From e00878be3f414a99234b3c9613e2fcab7497cd4b Mon Sep 17 00:00:00 2001 From: Paul Gortmaker Date: Sun, 3 Jul 2016 16:31:53 -0400 Subject: cxl: make base more explicitly non-modular The Kconfig/Makefile currently controlling compilation of this code is: drivers/misc/cxl/Kconfig:config CXL_BASE drivers/misc/cxl/Kconfig: bool drivers/misc/cxl/Makefile:obj-$(CONFIG_CXL_BASE) += base.o ...meaning that it currently is not being built as a module by anyone. Lets convert the one module_init into device_initcall so that when reading the driver it more clear that it is builtin-only. Since module_init translates to device_initcall in the non-modular case, the init ordering remains unchanged with this commit. We don't replace module.h with init.h since the file is doing other modular stuff (module_get/put) even though it is built-in. Cc: Ian Munsie Cc: Michael Neuling Cc: linuxppc-dev@lists.ozlabs.org Signed-off-by: Paul Gortmaker Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index 9b90ec6c07cd..e6f49aca0035 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -114,5 +114,4 @@ static int __init cxl_base_init(void) pr_devel("Found %d cxl device(s)\n", count); return 0; } - -module_init(cxl_base_init); +device_initcall(cxl_base_init); -- cgit From fc9f75ef2fdf46fc859b991dbf473a583edfb0e2 Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Tue, 12 Jul 2016 11:30:11 +0000 Subject: cxl: Use for_each_compatible_node() macro Use for_each_compatible_node() macro instead of open coding it. Generated by Coccinelle. Signed-off-by: Wei Yongjun Reviewed-by: Andrew Donnellan Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index e6f49aca0035..2330980bdde2 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -95,7 +95,7 @@ EXPORT_SYMBOL_GPL(cxl_update_properties); static int __init cxl_base_init(void) { - struct device_node *np = NULL; + struct device_node *np; struct platform_device *dev; int count = 0; @@ -105,8 +105,7 @@ static int __init cxl_base_init(void) if (cpu_has_feature(CPU_FTR_HVMODE)) return 0; - while ((np = of_find_compatible_node(np, NULL, - "ibm,coherent-platform-facility"))) { + for_each_compatible_node(np, NULL, "ibm,coherent-platform-facility") { dev = of_platform_device_create(np, NULL, NULL); if (dev) count++; -- cgit From 62ccf2d2efefa01d0eb92cd6ecbb45ea0499fb1c Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:03 +1000 Subject: cxl: Move cxl_afu_get / cxl_afu_put to base The Mellanox CX4 uses a model where the AFU is one physical function of the device, and is used by other peer physical functions of the same device. This will require those other devices to grab a reference on the AFU when they are initialised to make sure that it does not go away during their lifetime. Move the AFU refcount functions to base.c so they can be called from the PHB code. Signed-off-by: Ian Munsie Reviewed-by: Andrew Donnellan Reviewed-by: Frederic Barrat Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index 2330980bdde2..7557835cdfcd 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -54,6 +54,19 @@ static inline void cxl_calls_put(struct cxl_calls *calls) { } #endif /* CONFIG_CXL_MODULE */ +/* AFU refcount management */ +struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) +{ + return (get_device(&afu->dev) == NULL) ? NULL : afu; +} +EXPORT_SYMBOL_GPL(cxl_afu_get); + +void cxl_afu_put(struct cxl_afu *afu) +{ + put_device(&afu->dev); +} +EXPORT_SYMBOL_GPL(cxl_afu_put); + void cxl_slbia(struct mm_struct *mm) { struct cxl_calls *calls; -- cgit From a19bd79e31769626d288cc016e21a31b6f47bf6f Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:04 +1000 Subject: cxl: Allow a default context to be associated with an external pci_dev The cxl kernel API has a concept of a default context associated with each PCI device under the virtual PHB. The Mellanox CX4 will also use the cxl kernel API, but it does not use a virtual PHB - rather, the AFU appears as a physical function as a peer to the networking functions. In order to allow the kernel API to work with those networking functions, we will need to associate a default context with them as well. To this end, refactor the corresponding code to do this in vphb.c and export it so that it can be called from the PHB code. Signed-off-by: Ian Munsie Reviewed-by: Frederic Barrat Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index 7557835cdfcd..e1e80cb99ad9 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -106,6 +106,41 @@ int cxl_update_properties(struct device_node *dn, } EXPORT_SYMBOL_GPL(cxl_update_properties); +/* + * API calls into the driver that may be called from the PHB code and must be + * built in. + */ +bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) +{ + bool ret; + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return false; + + ret = calls->cxl_pci_associate_default_context(dev, afu); + + cxl_calls_put(calls); + + return ret; +} +EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context); + +void cxl_pci_disable_device(struct pci_dev *dev) +{ + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return; + + calls->cxl_pci_disable_device(dev); + + cxl_calls_put(calls); +} +EXPORT_SYMBOL_GPL(cxl_pci_disable_device); + static int __init cxl_base_init(void) { struct device_node *np; -- cgit From cbce0917e2e47d4bf5aa3b5fd6b1247f33e1a126 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:09 +1000 Subject: cxl: Add preliminary workaround for CX4 interrupt limitation The Mellanox CX4 has a hardware limitation where only 4 bits of the AFU interrupt number can be passed to the XSL when sending an interrupt, limiting it to only 15 interrupts per context (AFU interrupt number 0 is invalid). In order to overcome this, we will allocate additional contexts linked to the default context as extra address space for the extra interrupts - this will be implemented in the next patch. This patch adds the preliminary support to allow this, by way of adding a linked list in the context structure that we use to keep track of the contexts dedicated to interrupts, and an API to simultaneously iterate over the related context structures, AFU interrupt numbers and hardware interrupt numbers. The point of using a single API to iterate these is to hide some of the details of the iteration from external code, and to reduce the number of APIs that need to be exported via base.c to allow built in code to call. Signed-off-by: Ian Munsie Reviewed-by: Frederic Barrat Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index e1e80cb99ad9..fe90f895bb10 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -141,6 +141,23 @@ void cxl_pci_disable_device(struct pci_dev *dev) } EXPORT_SYMBOL_GPL(cxl_pci_disable_device); +int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq) +{ + int ret; + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return -EBUSY; + + ret = calls->cxl_next_msi_hwirq(pdev, ctx, afu_irq); + + cxl_calls_put(calls); + + return ret; +} +EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq); + static int __init cxl_base_init(void) { struct device_node *np; -- cgit From a2f67d5ee8d950caaa7a6144cf0bfb256500b73e Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:10 +1000 Subject: cxl: Add support for interrupts on the Mellanox CX4 The Mellanox CX4 in cxl mode uses a hybrid interrupt model, where interrupts are routed from the networking hardware to the XSL using the MSIX table, and from there will be transformed back into an MSIX interrupt using the cxl style interrupts (i.e. using IVTE entries and ranges to map a PE and AFU interrupt number to an MSIX address). We want to hide the implementation details of cxl interrupts as much as possible. To this end, we use a special version of the MSI setup & teardown routines in the PHB while in cxl mode to allocate the cxl interrupts and configure the IVTE entries in the process element. This function does not configure the MSIX table - the CX4 card uses a custom format in that table and it would not be appropriate to fill that out in generic code. The rest of the functionality is similar to the "Full MSI-X mode" described in the CAIA, and this could be easily extended to support other adapters that use that mode in the future. The interrupts will be associated with the default context. If the maximum number of interrupts per context has been limited (e.g. by the mlx5 driver), it will automatically allocate additional kernel contexts to associate extra interrupts as required. These contexts will be started using the same WED that was used to start the default context. Signed-off-by: Ian Munsie Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/base.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'drivers/misc/cxl/base.c') diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index fe90f895bb10..cd54ce6f6230 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -158,6 +158,37 @@ int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_ } EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq); +int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) +{ + int ret; + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return false; + + ret = calls->cxl_cx4_setup_msi_irqs(pdev, nvec, type); + + cxl_calls_put(calls); + + return ret; +} +EXPORT_SYMBOL_GPL(cxl_cx4_setup_msi_irqs); + +void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) +{ + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return; + + calls->cxl_cx4_teardown_msi_irqs(pdev); + + cxl_calls_put(calls); +} +EXPORT_SYMBOL_GPL(cxl_cx4_teardown_msi_irqs); + static int __init cxl_base_init(void) { struct device_node *np; -- cgit