From 3f1f3234bc2db1c16b9818b9a15a5d58ad45251c Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:44 +0000 Subject: irqchip/gic-v3: Switch to PMR masking before calling IRQ handler Mask the IRQ priority through PMR and re-enable IRQs at CPU level, allowing only higher priority interrupts to be received during interrupt handling. Signed-off-by: Julien Thierry Acked-by: Catalin Marinas Acked-by: Marc Zyngier Cc: Will Deacon Cc: Thomas Gleixner Cc: Jason Cooper Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 0868a9d81c3c..8148a9250be2 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -356,6 +356,11 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs irqnr = gic_read_iar(); + if (gic_prio_masking_enabled()) { + gic_pmr_mask_irqs(); + gic_arch_enable_irqs(); + } + if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) { int err; -- cgit From b5cf607370d0ee257e4bfa80740952fa6110c2c7 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:54 +0000 Subject: irqchip/gic-v3: Factor group0 detection into functions The code to detect whether Linux has access to group0 interrupts can prove useful in other parts of the driver. Provide a separate function to do this. Signed-off-by: Julien Thierry Acked-by: Marc Zyngier Cc: Thomas Gleixner Cc: Jason Cooper Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 55 +++++++++++++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 19 deletions(-) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 8148a9250be2..da547e013c1a 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -400,6 +400,39 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs } } +static u32 gic_get_pribits(void) +{ + u32 pribits; + + pribits = gic_read_ctlr(); + pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; + pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; + pribits++; + + return pribits; +} + +static bool gic_has_group0(void) +{ + u32 val; + + /* + * Let's find out if Group0 is under control of EL3 or not by + * setting the highest possible, non-zero priority in PMR. + * + * If SCR_EL3.FIQ is set, the priority gets shifted down in + * order for the CPU interface to set bit 7, and keep the + * actual priority in the non-secure range. In the process, it + * looses the least significant bit and the actual priority + * becomes 0x80. Reading it back returns 0, indicating that + * we're don't have access to Group0. + */ + gic_write_pmr(BIT(8 - gic_get_pribits())); + val = gic_read_pmr(); + + return val != 0; +} + static void __init gic_dist_init(void) { unsigned int i; @@ -541,7 +574,7 @@ static void gic_cpu_sys_reg_init(void) u64 mpidr = cpu_logical_map(cpu); u64 need_rss = MPIDR_RS(mpidr); bool group0; - u32 val, pribits; + u32 pribits; /* * Need to check that the SRE bit has actually been set. If @@ -553,25 +586,9 @@ static void gic_cpu_sys_reg_init(void) if (!gic_enable_sre()) pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); - pribits = gic_read_ctlr(); - pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; - pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; - pribits++; + pribits = gic_get_pribits(); - /* - * Let's find out if Group0 is under control of EL3 or not by - * setting the highest possible, non-zero priority in PMR. - * - * If SCR_EL3.FIQ is set, the priority gets shifted down in - * order for the CPU interface to set bit 7, and keep the - * actual priority in the non-secure range. In the process, it - * looses the least significant bit and the actual priority - * becomes 0x80. Reading it back returns 0, indicating that - * we're don't have access to Group0. - */ - write_gicreg(BIT(8 - pribits), ICC_PMR_EL1); - val = read_gicreg(ICC_PMR_EL1); - group0 = val != 0; + group0 = gic_has_group0(); /* Set priority mask register */ write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); -- cgit From e79321883842ca7b77d8a58fe8303e8da35c085e Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:55 +0000 Subject: arm64: Switch to PMR masking when starting CPUs Once the boot CPU has been prepared or a new secondary CPU has been brought up, use ICC_PMR_EL1 to mask interrupts on that CPU and clear PSR.I bit. Since ICC_PMR_EL1 is initialized at CPU bringup, avoid overwriting it in the GICv3 driver. Signed-off-by: Julien Thierry Suggested-by: Daniel Thompson Acked-by: Marc Zyngier Cc: Will Deacon Cc: James Morse Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index da547e013c1a..5a703ae09ce5 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -415,6 +415,9 @@ static u32 gic_get_pribits(void) static bool gic_has_group0(void) { u32 val; + u32 old_pmr; + + old_pmr = gic_read_pmr(); /* * Let's find out if Group0 is under control of EL3 or not by @@ -430,6 +433,8 @@ static bool gic_has_group0(void) gic_write_pmr(BIT(8 - gic_get_pribits())); val = gic_read_pmr(); + gic_write_pmr(old_pmr); + return val != 0; } @@ -591,7 +596,8 @@ static void gic_cpu_sys_reg_init(void) group0 = gic_has_group0(); /* Set priority mask register */ - write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); + if (!gic_prio_masking_enabled()) + write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); /* * Some firmwares hand over to the kernel with the BPR changed from -- cgit From d98d0a990ca1446d3c0ca8f0b9ac127a66e40cdf Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:57 +0000 Subject: irqchip/gic-v3: Detect if GIC can support pseudo-NMIs The values non secure EL1 needs to use for PMR and RPR registers depends on the value of SCR_EL3.FIQ. The values non secure EL1 sees from the distributor and redistributor depend on whether security is enabled for the GIC or not. To avoid having to deal with two sets of values for PMR masking/unmasking, only enable pseudo-NMIs when GIC has non-secure view of priorities. Also, add firmware requirements related to SCR_EL3. Signed-off-by: Julien Thierry Acked-by: Marc Zyngier Cc: Will Deacon Cc: Jonathan Corbet Cc: Thomas Gleixner Cc: Jason Cooper Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 66 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 59 insertions(+), 7 deletions(-) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 5a703ae09ce5..eb9d9486f90b 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -66,6 +66,31 @@ struct gic_chip_data { static struct gic_chip_data gic_data __read_mostly; static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); +/* + * The behaviours of RPR and PMR registers differ depending on the value of + * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the + * distributor and redistributors depends on whether security is enabled in the + * GIC. + * + * When security is enabled, non-secure priority values from the (re)distributor + * are presented to the GIC CPUIF as follow: + * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; + * + * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure + * EL1 are subject to a similar operation thus matching the priorities presented + * from the (re)distributor when security is enabled. + * + * see GICv3/GICv4 Architecture Specification (IHI0069D): + * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt + * priorities. + * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 + * interrupt. + * + * For now, we only support pseudo-NMIs if we have non-secure view of + * priorities. + */ +static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); + static struct gic_kvm_info gic_v3_kvm_info; static DEFINE_PER_CPU(bool, has_rss); @@ -232,6 +257,12 @@ static void gic_unmask_irq(struct irq_data *d) gic_poke_irq(d, GICD_ISENABLER); } +static inline bool gic_supports_nmi(void) +{ + return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && + static_branch_likely(&supports_pseudo_nmis); +} + static int gic_irq_set_irqchip_state(struct irq_data *d, enum irqchip_irq_state which, bool val) { @@ -573,6 +604,12 @@ static void gic_update_vlpi_properties(void) !gic_data.rdists.has_direct_lpi ? "no " : ""); } +/* Check whether it's single security state view */ +static inline bool gic_dist_security_disabled(void) +{ + return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; +} + static void gic_cpu_sys_reg_init(void) { int i, cpu = smp_processor_id(); @@ -596,8 +633,17 @@ static void gic_cpu_sys_reg_init(void) group0 = gic_has_group0(); /* Set priority mask register */ - if (!gic_prio_masking_enabled()) + if (!gic_prio_masking_enabled()) { write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); + } else { + /* + * Mismatch configuration with boot CPU, the system is likely + * to die as interrupt masking will not work properly on all + * CPUs + */ + WARN_ON(gic_supports_nmi() && group0 && + !gic_dist_security_disabled()); + } /* * Some firmwares hand over to the kernel with the BPR changed from @@ -852,12 +898,6 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, #endif #ifdef CONFIG_CPU_PM -/* Check whether it's single security state view */ -static bool gic_dist_security_disabled(void) -{ - return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; -} - static int gic_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) { @@ -1110,6 +1150,11 @@ static bool gic_enable_quirk_msm8996(void *data) return true; } +static void gic_enable_nmi_support(void) +{ + static_branch_enable(&supports_pseudo_nmis); +} + static int __init gic_init_bases(void __iomem *dist_base, struct redist_region *rdist_regs, u32 nr_redist_regions, @@ -1179,6 +1224,13 @@ static int __init gic_init_bases(void __iomem *dist_base, its_cpu_init(); } + if (gic_prio_masking_enabled()) { + if (!gic_has_group0() || gic_dist_security_disabled()) + gic_enable_nmi_support(); + else + pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n"); + } + return 0; out_free: -- cgit From f32c926651dcd1683f4d896ee52609000a62a3dc Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:58 +0000 Subject: irqchip/gic-v3: Handle pseudo-NMIs Provide a higher priority to be used for pseudo-NMIs. When such an interrupt is received, keep interrupts fully disabled at CPU level to prevent receiving other pseudo-NMIs while handling the current one. Signed-off-by: Julien Thierry Acked-by: Marc Zyngier Cc: Thomas Gleixner Cc: Jason Cooper Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 42 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index eb9d9486f90b..c2767fbc2e66 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -41,6 +41,8 @@ #include "irq-gic-common.h" +#define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) + #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) struct redist_region { @@ -381,12 +383,45 @@ static u64 gic_mpidr_to_affinity(unsigned long mpidr) return aff; } +static void gic_deactivate_unhandled(u32 irqnr) +{ + if (static_branch_likely(&supports_deactivate_key)) { + if (irqnr < 8192) + gic_write_dir(irqnr); + } else { + gic_write_eoir(irqnr); + } +} + +static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs) +{ + int err; + + if (static_branch_likely(&supports_deactivate_key)) + gic_write_eoir(irqnr); + /* + * Leave the PSR.I bit set to prevent other NMIs to be + * received while handling this one. + * PSR.I will be restored when we ERET to the + * interrupted context. + */ + err = handle_domain_nmi(gic_data.domain, irqnr, regs); + if (err) + gic_deactivate_unhandled(irqnr); +} + static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) { u32 irqnr; irqnr = gic_read_iar(); + if (gic_supports_nmi() && + unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { + gic_handle_nmi(irqnr, regs); + return; + } + if (gic_prio_masking_enabled()) { gic_pmr_mask_irqs(); gic_arch_enable_irqs(); @@ -403,12 +438,7 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs err = handle_domain_irq(gic_data.domain, irqnr, regs); if (err) { WARN_ONCE(true, "Unexpected interrupt received!\n"); - if (static_branch_likely(&supports_deactivate_key)) { - if (irqnr < 8192) - gic_write_dir(irqnr); - } else { - gic_write_eoir(irqnr); - } + gic_deactivate_unhandled(irqnr); } return; } -- cgit From 101b35f7def1775bf589d86676983bc359843916 Mon Sep 17 00:00:00 2001 From: Julien Thierry Date: Thu, 31 Jan 2019 14:58:59 +0000 Subject: irqchip/gic-v3: Allow interrupts to be set as pseudo-NMI Implement NMI callbacks for GICv3 irqchip. Install NMI safe handlers when setting up interrupt line as NMI. Only SPIs and PPIs are allowed to be set up as NMI. Signed-off-by: Julien Thierry Reviewed-by: Marc Zyngier Cc: Thomas Gleixner Cc: Jason Cooper Cc: Marc Zyngier Signed-off-by: Catalin Marinas --- drivers/irqchip/irq-gic-v3.c | 91 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) (limited to 'drivers/irqchip') diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index c2767fbc2e66..15e55d327505 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -93,6 +94,9 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); */ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); +/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ +static refcount_t ppi_nmi_refs[16]; + static struct gic_kvm_info gic_v3_kvm_info; static DEFINE_PER_CPU(bool, has_rss); @@ -320,6 +324,79 @@ static int gic_irq_get_irqchip_state(struct irq_data *d, return 0; } +static void gic_irq_set_prio(struct irq_data *d, u8 prio) +{ + void __iomem *base = gic_dist_base(d); + + writeb_relaxed(prio, base + GICD_IPRIORITYR + gic_irq(d)); +} + +static int gic_irq_nmi_setup(struct irq_data *d) +{ + struct irq_desc *desc = irq_to_desc(d->irq); + + if (!gic_supports_nmi()) + return -EINVAL; + + if (gic_peek_irq(d, GICD_ISENABLER)) { + pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); + return -EINVAL; + } + + /* + * A secondary irq_chip should be in charge of LPI request, + * it should not be possible to get there + */ + if (WARN_ON(gic_irq(d) >= 8192)) + return -EINVAL; + + /* desc lock should already be held */ + if (gic_irq(d) < 32) { + /* Setting up PPI as NMI, only switch handler for first NMI */ + if (!refcount_inc_not_zero(&ppi_nmi_refs[gic_irq(d) - 16])) { + refcount_set(&ppi_nmi_refs[gic_irq(d) - 16], 1); + desc->handle_irq = handle_percpu_devid_fasteoi_nmi; + } + } else { + desc->handle_irq = handle_fasteoi_nmi; + } + + gic_irq_set_prio(d, GICD_INT_NMI_PRI); + + return 0; +} + +static void gic_irq_nmi_teardown(struct irq_data *d) +{ + struct irq_desc *desc = irq_to_desc(d->irq); + + if (WARN_ON(!gic_supports_nmi())) + return; + + if (gic_peek_irq(d, GICD_ISENABLER)) { + pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); + return; + } + + /* + * A secondary irq_chip should be in charge of LPI request, + * it should not be possible to get there + */ + if (WARN_ON(gic_irq(d) >= 8192)) + return; + + /* desc lock should already be held */ + if (gic_irq(d) < 32) { + /* Tearing down NMI, only switch handler for last NMI */ + if (refcount_dec_and_test(&ppi_nmi_refs[gic_irq(d) - 16])) + desc->handle_irq = handle_percpu_devid_irq; + } else { + desc->handle_irq = handle_fasteoi_irq; + } + + gic_irq_set_prio(d, GICD_INT_DEF_PRI); +} + static void gic_eoi_irq(struct irq_data *d) { gic_write_eoir(gic_irq(d)); @@ -964,6 +1041,8 @@ static struct irq_chip gic_chip = { .irq_set_affinity = gic_set_affinity, .irq_get_irqchip_state = gic_irq_get_irqchip_state, .irq_set_irqchip_state = gic_irq_set_irqchip_state, + .irq_nmi_setup = gic_irq_nmi_setup, + .irq_nmi_teardown = gic_irq_nmi_teardown, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, @@ -979,6 +1058,8 @@ static struct irq_chip gic_eoimode1_chip = { .irq_get_irqchip_state = gic_irq_get_irqchip_state, .irq_set_irqchip_state = gic_irq_set_irqchip_state, .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, + .irq_nmi_setup = gic_irq_nmi_setup, + .irq_nmi_teardown = gic_irq_nmi_teardown, .flags = IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, @@ -1182,7 +1263,17 @@ static bool gic_enable_quirk_msm8996(void *data) static void gic_enable_nmi_support(void) { + int i; + + for (i = 0; i < 16; i++) + refcount_set(&ppi_nmi_refs[i], 0); + static_branch_enable(&supports_pseudo_nmis); + + if (static_branch_likely(&supports_deactivate_key)) + gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; + else + gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; } static int __init gic_init_bases(void __iomem *dist_base, -- cgit