From 386ae59bd7db4eafc74bdfdcf60182c2171aea2a Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:16:23 +0000 Subject: iommu/amd: Generalize log overflow handling Each IOMMU has three log buffers (Event, GA and PPR log). Once a buffer becomes full, IOMMU generates an interrupt with the corresponding overflow status bit, and stop processing the log. To handle an overflow, the IOMMU driver needs to disable the log, clear the overflow status bit, and re-enable the log. This procedure is same among all types of log buffer except it uses different overflow status bit and enabling bit. Hence, to consolidate the log buffer restarting logic, introduce a helper function amd_iommu_restart_log(), which caller can specify parameters specific for each type of log buffer. Also rename MMIO_STATUS_EVT_OVERFLOW_INT_MASK as MMIO_STATUS_EVT_OVERFLOW_MASK. Reviewed-by: Jerry Snitselaar Reviewed-by: Suravee Suthikulpanit Reviewed-by: Joao Martins Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20230628051624.5792-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 51 ++++++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 19 deletions(-) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index ea0f1ab94178..7fab6ecb6295 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -752,38 +752,51 @@ static int __init alloc_command_buffer(struct amd_iommu *iommu) return iommu->cmd_buf ? 0 : -ENOMEM; } +/* + * Interrupt handler has processed all pending events and adjusted head + * and tail pointer. Reset overflow mask and restart logging again. + */ +static void amd_iommu_restart_log(struct amd_iommu *iommu, const char *evt_type, + u8 cntrl_intr, u8 cntrl_log, + u32 status_run_mask, u32 status_overflow_mask) +{ + u32 status; + + status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); + if (status & status_run_mask) + return; + + pr_info_ratelimited("IOMMU %s log restarting\n", evt_type); + + iommu_feature_disable(iommu, cntrl_log); + iommu_feature_disable(iommu, cntrl_intr); + + writel(status_overflow_mask, iommu->mmio_base + MMIO_STATUS_OFFSET); + + iommu_feature_enable(iommu, cntrl_intr); + iommu_feature_enable(iommu, cntrl_log); +} + /* * This function restarts event logging in case the IOMMU experienced * an event log buffer overflow. */ void amd_iommu_restart_event_logging(struct amd_iommu *iommu) { - iommu_feature_disable(iommu, CONTROL_EVT_LOG_EN); - iommu_feature_enable(iommu, CONTROL_EVT_LOG_EN); + amd_iommu_restart_log(iommu, "Event", CONTROL_EVT_INT_EN, + CONTROL_EVT_LOG_EN, MMIO_STATUS_EVT_RUN_MASK, + MMIO_STATUS_EVT_OVERFLOW_MASK); } /* * This function restarts event logging in case the IOMMU experienced - * an GA log overflow. + * GA log overflow. */ void amd_iommu_restart_ga_log(struct amd_iommu *iommu) { - u32 status; - - status = readl(iommu->mmio_base + MMIO_STATUS_OFFSET); - if (status & MMIO_STATUS_GALOG_RUN_MASK) - return; - - pr_info_ratelimited("IOMMU GA Log restarting\n"); - - iommu_feature_disable(iommu, CONTROL_GALOG_EN); - iommu_feature_disable(iommu, CONTROL_GAINT_EN); - - writel(MMIO_STATUS_GALOG_OVERFLOW_MASK, - iommu->mmio_base + MMIO_STATUS_OFFSET); - - iommu_feature_enable(iommu, CONTROL_GAINT_EN); - iommu_feature_enable(iommu, CONTROL_GALOG_EN); + amd_iommu_restart_log(iommu, "GA", CONTROL_GAINT_EN, + CONTROL_GALOG_EN, MMIO_STATUS_GALOG_RUN_MASK, + MMIO_STATUS_GALOG_OVERFLOW_MASK); } /* -- cgit From 274c2218b8b29ea2e0404f97d52040deb16d42dd Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:16:24 +0000 Subject: iommu/amd: Handle PPR log overflow Some ATS-capable peripherals can issue requests to the processor to service peripheral page requests using PCIe PRI (the Page Request Interface). IOMMU supports PRI using PPR log buffer. IOMMU writes PRI request to PPR log buffer and sends PPR interrupt to host. When there is no space in the PPR log buffer (PPR log overflow) it will set PprOverflow bit in 'MMIO Offset 2020h IOMMU Status Register'. When this happens PPR log needs to be restarted as specified in IOMMU spec [1] section 2.6.2. When handling the event it just resumes the PPR log without resizing (similar to the way event and GA log overflow is handled). Failing to handle PPR overflow means device may not work properly as IOMMU stops processing new PPR events from device. [1] https://www.amd.com/system/files/TechDocs/48882_3.07_PUB.pdf Reviewed-by: Jerry Snitselaar Reviewed-by: Suravee Suthikulpanit Reviewed-by: Joao Martins Signed-off-by: Vasant Hegde Link: https://lore.kernel.org/r/20230628051624.5792-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 7fab6ecb6295..e78d7c4f41bd 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -799,6 +799,17 @@ void amd_iommu_restart_ga_log(struct amd_iommu *iommu) MMIO_STATUS_GALOG_OVERFLOW_MASK); } +/* + * This function restarts ppr logging in case the IOMMU experienced + * PPR log overflow. + */ +void amd_iommu_restart_ppr_log(struct amd_iommu *iommu) +{ + amd_iommu_restart_log(iommu, "PPR", CONTROL_PPRINT_EN, + CONTROL_PPRLOG_EN, MMIO_STATUS_PPR_RUN_MASK, + MMIO_STATUS_PPR_OVERFLOW_MASK); +} + /* * This function resets the command buffer if the IOMMU stopped fetching * commands from it. -- cgit From e5ebd90d1b5cbfa0df9b911cd3f24c1bb84c05dd Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:32:22 +0000 Subject: iommu/amd: Enable separate interrupt for PPR and GA log AMD IOMMU has three log buffers (i.e. Event, PPR, and GA). These logs can be configured to generate different interrupts when an entry is inserted into a log buffer. However, current implementation share single interrupt to handle all three logs. With increasing usages of the GA (for IOMMU AVIC) and PPR logs (for IOMMUv2 APIs and SVA), interrupt sharing could potentially become performance bottleneck. Hence, separate IOMMU interrupt into use three separate vectors and irq threads with corresponding name, which will be displayed in the /proc/interrupts as "AMD-Vi-[Evt/PPR/GA]", where "x" is an IOMMU id. Note that this patch changes interrupt handling only in IOMMU x2apic mode (MMIO 0x18[IntCapXTEn]=1). In legacy mode it will continue to use single MSI interrupt. Signed-off-by: Vasant Hegde Reviewed-by: Alexey Kardashevskiy Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20230628053222.5962-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 50 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 12 deletions(-) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index e78d7c4f41bd..30f09e786f7b 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2335,6 +2335,7 @@ static int intcapxt_irqdomain_alloc(struct irq_domain *domain, unsigned int virq struct irq_data *irqd = irq_domain_get_irq_data(domain, i); irqd->chip = &intcapxt_controller; + irqd->hwirq = info->hwirq; irqd->chip_data = info->data; __irq_set_handler(i, handle_edge_irq, 0, "edge"); } @@ -2361,22 +2362,14 @@ static void intcapxt_unmask_irq(struct irq_data *irqd) xt.destid_0_23 = cfg->dest_apicid & GENMASK(23, 0); xt.destid_24_31 = cfg->dest_apicid >> 24; - /** - * Current IOMMU implementation uses the same IRQ for all - * 3 IOMMU interrupts. - */ - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); - writeq(xt.capxt, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); + writeq(xt.capxt, iommu->mmio_base + irqd->hwirq); } static void intcapxt_mask_irq(struct irq_data *irqd) { struct amd_iommu *iommu = irqd->chip_data; - writeq(0, iommu->mmio_base + MMIO_INTCAPXT_EVT_OFFSET); - writeq(0, iommu->mmio_base + MMIO_INTCAPXT_PPR_OFFSET); - writeq(0, iommu->mmio_base + MMIO_INTCAPXT_GALOG_OFFSET); + writeq(0, iommu->mmio_base + irqd->hwirq); } @@ -2439,7 +2432,8 @@ static struct irq_domain *iommu_get_irqdomain(void) return iommu_irqdomain; } -static int iommu_setup_intcapxt(struct amd_iommu *iommu) +static int __iommu_setup_intcapxt(struct amd_iommu *iommu, const char *devname, + int hwirq, irq_handler_t thread_fn) { struct irq_domain *domain; struct irq_alloc_info info; @@ -2453,6 +2447,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) init_irq_alloc_info(&info, NULL); info.type = X86_IRQ_ALLOC_TYPE_AMDVI; info.data = iommu; + info.hwirq = hwirq; irq = irq_domain_alloc_irqs(domain, 1, node, &info); if (irq < 0) { @@ -2461,7 +2456,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) } ret = request_threaded_irq(irq, amd_iommu_int_handler, - amd_iommu_int_thread, 0, "AMD-Vi", iommu); + thread_fn, 0, devname, iommu); if (ret) { irq_domain_free_irqs(irq, 1); irq_domain_remove(domain); @@ -2471,6 +2466,37 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) return 0; } +static int iommu_setup_intcapxt(struct amd_iommu *iommu) +{ + int ret; + + snprintf(iommu->evt_irq_name, sizeof(iommu->evt_irq_name), + "AMD-Vi%d-Evt", iommu->index); + ret = __iommu_setup_intcapxt(iommu, iommu->evt_irq_name, + MMIO_INTCAPXT_EVT_OFFSET, + amd_iommu_int_thread_evtlog); + if (ret) + return ret; + + snprintf(iommu->ppr_irq_name, sizeof(iommu->ppr_irq_name), + "AMD-Vi%d-PPR", iommu->index); + ret = __iommu_setup_intcapxt(iommu, iommu->ppr_irq_name, + MMIO_INTCAPXT_PPR_OFFSET, + amd_iommu_int_thread_pprlog); + if (ret) + return ret; + +#ifdef CONFIG_IRQ_REMAP + snprintf(iommu->ga_irq_name, sizeof(iommu->ga_irq_name), + "AMD-Vi%d-GA", iommu->index); + ret = __iommu_setup_intcapxt(iommu, iommu->ga_irq_name, + MMIO_INTCAPXT_GALOG_OFFSET, + amd_iommu_int_thread_galog); +#endif + + return ret; +} + static int iommu_init_irq(struct amd_iommu *iommu) { int ret; -- cgit From 7827a2689e79bcd3096b4b49b8a8beb8574fd4aa Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:45:52 +0000 Subject: iommu/amd: Disable PPR log/interrupt in iommu_disable() Similar to other logs, disable PPR log/interrupt in iommu_disable() path. Reviewed-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20230628054554.6131-2-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 30f09e786f7b..94c91b6ee603 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -483,6 +483,10 @@ static void iommu_disable(struct amd_iommu *iommu) iommu_feature_disable(iommu, CONTROL_GALOG_EN); iommu_feature_disable(iommu, CONTROL_GAINT_EN); + /* Disable IOMMU PPR logging */ + iommu_feature_disable(iommu, CONTROL_PPRLOG_EN); + iommu_feature_disable(iommu, CONTROL_PPRINT_EN); + /* Disable IOMMU hardware itself */ iommu_feature_disable(iommu, CONTROL_IOMMU_EN); -- cgit From f52c895a2de8697108c6385e9695061585690dc8 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:45:53 +0000 Subject: iommu/amd: Consolidate PPR log enablement Move PPR log interrupt bit setting to iommu_enable_ppr_log(). Also rearrange iommu_enable_ppr_log() such that PPREn bit is enabled before enabling PPRLog and PPRInt bits. So that when PPRLog bit is set it will clear the PPRLogOverflow bit and sets the PPRLogRun bit in the IOMMU Status Register [MMIO Offset 2020h]. Reviewed-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20230628054554.6131-3-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 94c91b6ee603..f26c077eb9ea 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -934,6 +934,8 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) if (iommu->ppr_log == NULL) return; + iommu_feature_enable(iommu, CONTROL_PPR_EN); + entry = iommu_virt_to_phys(iommu->ppr_log) | PPR_LOG_SIZE_512; memcpy_toio(iommu->mmio_base + MMIO_PPR_LOG_OFFSET, @@ -944,7 +946,7 @@ static void iommu_enable_ppr_log(struct amd_iommu *iommu) writel(0x00, iommu->mmio_base + MMIO_PPR_TAIL_OFFSET); iommu_feature_enable(iommu, CONTROL_PPRLOG_EN); - iommu_feature_enable(iommu, CONTROL_PPR_EN); + iommu_feature_enable(iommu, CONTROL_PPRINT_EN); } static void __init free_ppr_log(struct amd_iommu *iommu) @@ -2526,8 +2528,6 @@ enable_faults: iommu_feature_enable(iommu, CONTROL_EVT_INT_EN); - if (iommu->ppr_log != NULL) - iommu_feature_enable(iommu, CONTROL_PPRINT_EN); return 0; } -- cgit From a48130e92f1c86638295a53d1735dfed7f55a2c4 Mon Sep 17 00:00:00 2001 From: Vasant Hegde Date: Wed, 28 Jun 2023 05:45:54 +0000 Subject: iommu/amd: Enable PPR/GA interrupt after interrupt handler setup Current code enables PPR and GA interrupts before setting up the interrupt handler (in state_next()). Make sure interrupt handler is in place before enabling these interrupt. amd_iommu_enable_interrupts() gets called in normal boot, kdump as well as in suspend/resume path. Hence moving interrupt enablement to this function works fine. Reviewed-by: Suravee Suthikulpanit Signed-off-by: Vasant Hegde Reviewed-by: Jerry Snitselaar Link: https://lore.kernel.org/r/20230628054554.6131-4-vasant.hegde@amd.com Signed-off-by: Joerg Roedel --- drivers/iommu/amd/init.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'drivers/iommu/amd/init.c') diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index f26c077eb9ea..45efb7e5d725 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2943,8 +2943,6 @@ static void enable_iommus_vapic(void) static void enable_iommus(void) { early_enable_iommus(); - enable_iommus_vapic(); - enable_iommus_v2(); } static void disable_iommus(void) @@ -3208,6 +3206,13 @@ static int amd_iommu_enable_interrupts(void) goto out; } + /* + * Interrupt handler is ready to process interrupts. Enable + * PPR and GA log interrupt for all IOMMUs. + */ + enable_iommus_vapic(); + enable_iommus_v2(); + out: return ret; } @@ -3287,8 +3292,6 @@ static int __init state_next(void) register_syscore_ops(&amd_iommu_syscore_ops); ret = amd_iommu_init_pci(); init_state = ret ? IOMMU_INIT_ERROR : IOMMU_PCI_INIT; - enable_iommus_vapic(); - enable_iommus_v2(); break; case IOMMU_PCI_INIT: ret = amd_iommu_enable_interrupts(); -- cgit