diff options
Diffstat (limited to 'drivers/irqchip/irq-gic.c')
| -rw-r--r-- | drivers/irqchip/irq-gic.c | 299 |
1 files changed, 157 insertions, 142 deletions
diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index d329ec3d64d8..ec70c84e9f91 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -19,6 +19,7 @@ */ #include <linux/init.h> #include <linux/kernel.h> +#include <linux/kstrtox.h> #include <linux/err.h> #include <linux/module.h> #include <linux/list.h> @@ -34,6 +35,7 @@ #include <linux/irqdomain.h> #include <linux/interrupt.h> #include <linux/percpu.h> +#include <linux/seq_file.h> #include <linux/slab.h> #include <linux/irqchip.h> #include <linux/irqchip/chained_irq.h> @@ -52,7 +54,7 @@ static void gic_check_cpu_features(void) { - WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_SYSREG_GIC_CPUIF), + WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF), TAINT_CPU_OUT_OF_SPEC, "GICv3 system registers enabled, broken firmware!\n"); } @@ -62,11 +64,10 @@ static void gic_check_cpu_features(void) union gic_base { void __iomem *common_base; - void __percpu * __iomem *percpu_base; + void __iomem * __percpu *percpu_base; }; struct gic_chip_data { - struct irq_chip chip; union gic_base dist_base; union gic_base cpu_base; void __iomem *raw_dist_base; @@ -107,6 +108,8 @@ static DEFINE_RAW_SPINLOCK(cpu_map_lock); #endif +static DEFINE_STATIC_KEY_FALSE(needs_rmw_access); + /* * The GIC mapping of CPU interfaces does not necessarily match * the logical CPU numbering. Let's use a mapping as returned @@ -159,11 +162,6 @@ static inline void __iomem *gic_cpu_base(struct irq_data *d) return gic_data_cpu_base(gic_data); } -static inline unsigned int gic_irq(struct irq_data *d) -{ - return d->hwirq; -} - static inline bool cascading_gic_irq(struct irq_data *d) { void *data = irq_data_get_irq_handler_data(d); @@ -180,14 +178,16 @@ static inline bool cascading_gic_irq(struct irq_data *d) */ static void gic_poke_irq(struct irq_data *d, u32 offset) { - u32 mask = 1 << (gic_irq(d) % 32); - writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4); + u32 mask = 1 << (irqd_to_hwirq(d) % 32); + + writel_relaxed(mask, gic_dist_base(d) + offset + (irqd_to_hwirq(d) / 32) * 4); } static int gic_peek_irq(struct irq_data *d, u32 offset) { - u32 mask = 1 << (gic_irq(d) % 32); - return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask); + u32 mask = 1 << (irqd_to_hwirq(d) % 32); + + return !!(readl_relaxed(gic_dist_base(d) + offset + (irqd_to_hwirq(d) / 32) * 4) & mask); } static void gic_mask_irq(struct irq_data *d) @@ -217,7 +217,7 @@ static void gic_unmask_irq(struct irq_data *d) static void gic_eoi_irq(struct irq_data *d) { - u32 hwirq = gic_irq(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); if (hwirq < 16) hwirq = this_cpu_read(sgi_intid); @@ -227,7 +227,7 @@ static void gic_eoi_irq(struct irq_data *d) static void gic_eoimode1_eoi_irq(struct irq_data *d) { - u32 hwirq = gic_irq(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); /* Do not deactivate an IRQ forwarded to a vcpu. */ if (irqd_is_forwarded_to_vcpu(d)) @@ -290,8 +290,8 @@ static int gic_irq_get_irqchip_state(struct irq_data *d, static int gic_set_type(struct irq_data *d, unsigned int type) { + irq_hw_number_t gicirq = irqd_to_hwirq(d); void __iomem *base = gic_dist_base(d); - unsigned int gicirq = gic_irq(d); int ret; /* Interrupt configuration for SGIs can't be changed */ @@ -303,10 +303,10 @@ static int gic_set_type(struct irq_data *d, unsigned int type) type != IRQ_TYPE_EDGE_RISING) return -EINVAL; - ret = gic_configure_irq(gicirq, type, base + GIC_DIST_CONFIG, NULL); + ret = gic_configure_irq(gicirq, type, base + GIC_DIST_CONFIG); if (ret && gicirq < 32) { /* Misconfigured PPIs are usually not fatal */ - pr_warn("GIC: PPI%d is secure or misconfigured\n", gicirq - 16); + pr_warn("GIC: PPI%ld is secure or misconfigured\n", gicirq - 16); ret = 0; } @@ -316,7 +316,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) { /* Only interrupts on the primary GIC can be forwarded to a vcpu. */ - if (cascading_gic_irq(d) || gic_irq(d) < 16) + if (cascading_gic_irq(d) || irqd_to_hwirq(d) < 16) return -EINVAL; if (vcpu) @@ -367,7 +367,7 @@ static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) this_cpu_write(sgi_intid, irqstat); } - handle_domain_irq(gic->domain, irqnr, regs); + generic_handle_domain_irq(gic->domain, irqnr); } while (1); } @@ -395,18 +395,15 @@ static void gic_handle_cascade_irq(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static const struct irq_chip gic_chip = { - .irq_mask = gic_mask_irq, - .irq_unmask = gic_unmask_irq, - .irq_eoi = gic_eoi_irq, - .irq_set_type = gic_set_type, - .irq_retrigger = gic_retrigger, - .irq_get_irqchip_state = gic_irq_get_irqchip_state, - .irq_set_irqchip_state = gic_irq_set_irqchip_state, - .flags = IRQCHIP_SET_TYPE_MASKED | - IRQCHIP_SKIP_SET_WAKE | - IRQCHIP_MASK_ON_SUSPEND, -}; +static void gic_irq_print_chip(struct irq_data *d, struct seq_file *p) +{ + struct gic_chip_data *gic = irq_data_get_irq_chip_data(d); + + if (gic->domain->pm_dev) + seq_puts(p, gic->domain->pm_dev->of_node->name); + else + seq_printf(p, "GIC-%d", (int)(gic - &gic_data[0])); +} void __init gic_cascade_irq(unsigned int gic_nr, unsigned int irq) { @@ -482,7 +479,7 @@ static void gic_dist_init(struct gic_chip_data *gic) for (i = 32; i < gic_irqs; i += 4) writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4); - gic_dist_config(base, gic_irqs, NULL); + gic_dist_config(base, gic_irqs, GICD_INT_DEF_PRI); writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL); } @@ -519,7 +516,7 @@ static int gic_cpu_init(struct gic_chip_data *gic) gic_cpu_map[i] &= ~cpu_mask; } - gic_cpu_config(dist_base, 32, NULL); + gic_cpu_config(dist_base, 32, GICD_INT_DEF_PRI); writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK); gic_cpu_if_up(gic); @@ -611,7 +608,7 @@ void gic_dist_restore(struct gic_chip_data *gic) dist_base + GIC_DIST_CONFIG + i * 4); for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) - writel_relaxed(GICD_INT_DEF_PRI_X4, + writel_relaxed(REPEAT_BYTE_U32(GICD_INT_DEF_PRI), dist_base + GIC_DIST_PRI + i * 4); for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) @@ -700,7 +697,7 @@ void gic_cpu_restore(struct gic_chip_data *gic) writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4); for (i = 0; i < DIV_ROUND_UP(32, 4); i++) - writel_relaxed(GICD_INT_DEF_PRI_X4, + writel_relaxed(REPEAT_BYTE_U32(GICD_INT_DEF_PRI), dist_base + GIC_DIST_PRI + i * 4); writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK); @@ -774,12 +771,35 @@ static int gic_pm_init(struct gic_chip_data *gic) #endif #ifdef CONFIG_SMP +static void rmw_writeb(u8 bval, void __iomem *addr) +{ + static DEFINE_RAW_SPINLOCK(rmw_lock); + unsigned long offset = (unsigned long)addr & 3UL; + unsigned long shift = offset * 8; + unsigned long flags; + u32 val; + + raw_spin_lock_irqsave(&rmw_lock, flags); + + addr -= offset; + val = readl_relaxed(addr); + val &= ~GENMASK(shift + 7, shift); + val |= bval << shift; + writel_relaxed(val, addr); + + raw_spin_unlock_irqrestore(&rmw_lock, flags); +} + static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { - void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d); + void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + irqd_to_hwirq(d); + struct gic_chip_data *gic = irq_data_get_irq_chip_data(d); unsigned int cpu; + if (unlikely(gic != &gic_data[0])) + return -EINVAL; + if (!force) cpu = cpumask_any_and(mask_val, cpu_online_mask); else @@ -788,7 +808,10 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, if (cpu >= NR_GIC_CPU_IF || cpu >= nr_cpu_ids) return -EINVAL; - writeb_relaxed(gic_cpu_map[cpu], reg); + if (static_branch_unlikely(&needs_rmw_access)) + rmw_writeb(gic_cpu_map[cpu], reg); + else + writeb_relaxed(gic_cpu_map[cpu], reg); irq_data_update_effective_affinity(d, cpumask_of(cpu)); return IRQ_SET_MASK_OK_DONE; @@ -842,9 +865,7 @@ static __init void gic_smp_init(void) "irqchip/arm/gic:starting", gic_starting_cpu, NULL); - base_sgi = __irq_domain_alloc_irqs(gic_data[0].domain, -1, 8, - NUMA_NO_NODE, &sgi_fwspec, - false, NULL); + base_sgi = irq_domain_alloc_irqs(gic_data[0].domain, 8, NUMA_NO_NODE, &sgi_fwspec); if (WARN_ON(base_sgi <= 0)) return; @@ -856,6 +877,39 @@ static __init void gic_smp_init(void) #define gic_ipi_send_mask NULL #endif +static const struct irq_chip gic_chip = { + .irq_mask = gic_mask_irq, + .irq_unmask = gic_unmask_irq, + .irq_eoi = gic_eoi_irq, + .irq_set_type = gic_set_type, + .irq_retrigger = gic_retrigger, + .irq_set_affinity = gic_set_affinity, + .ipi_send_mask = gic_ipi_send_mask, + .irq_get_irqchip_state = gic_irq_get_irqchip_state, + .irq_set_irqchip_state = gic_irq_set_irqchip_state, + .irq_print_chip = gic_irq_print_chip, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static const struct irq_chip gic_chip_mode1 = { + .name = "GICv2", + .irq_mask = gic_eoimode1_mask_irq, + .irq_unmask = gic_unmask_irq, + .irq_eoi = gic_eoimode1_eoi_irq, + .irq_set_type = gic_set_type, + .irq_retrigger = gic_retrigger, + .irq_set_affinity = gic_set_affinity, + .ipi_send_mask = gic_ipi_send_mask, + .irq_get_irqchip_state = gic_irq_get_irqchip_state, + .irq_set_irqchip_state = gic_irq_set_irqchip_state, + .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + #ifdef CONFIG_BL_SWITCHER /* * gic_send_sgi - send a SGI directly to given CPU interface number @@ -1000,15 +1054,19 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, { struct gic_chip_data *gic = d->host_data; struct irq_data *irqd = irq_desc_get_irq_data(irq_to_desc(irq)); + const struct irq_chip *chip; + + chip = (static_branch_likely(&supports_deactivate_key) && + gic == &gic_data[0]) ? &gic_chip_mode1 : &gic_chip; switch (hw) { case 0 ... 31: irq_set_percpu_devid(irq); - irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, + irq_domain_set_info(d, irq, hw, chip, d->host_data, handle_percpu_devid_irq, NULL, NULL); break; default: - irq_domain_set_info(d, irq, hw, &gic->chip, d->host_data, + irq_domain_set_info(d, irq, hw, chip, d->host_data, handle_fasteoi_irq, NULL, NULL); irq_set_probe(irq); irqd_set_single_target(irqd); @@ -1020,10 +1078,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, return 0; } -static void gic_irq_domain_unmap(struct irq_domain *d, unsigned int irq) -{ -} - static int gic_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, unsigned long *hwirq, @@ -1053,7 +1107,8 @@ static int gic_irq_domain_translate(struct irq_domain *d, *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; /* Make it clear that broken DTs are... broken */ - WARN_ON(*type == IRQ_TYPE_NONE); + WARN(*type == IRQ_TYPE_NONE, + "HW irq %ld has invalid type\n", *hwirq); return 0; } @@ -1061,10 +1116,17 @@ static int gic_irq_domain_translate(struct irq_domain *d, if(fwspec->param_count != 2) return -EINVAL; + if (fwspec->param[0] < 16) { + pr_err(FW_BUG "Illegal GSI%d translation request\n", + fwspec->param[0]); + return -EINVAL; + } + *hwirq = fwspec->param[0]; *type = fwspec->param[1]; - WARN_ON(*type == IRQ_TYPE_NONE); + WARN(*type == IRQ_TYPE_NONE, + "HW irq %ld has invalid type\n", *hwirq); return 0; } @@ -1098,31 +1160,6 @@ static const struct irq_domain_ops gic_irq_domain_hierarchy_ops = { .free = irq_domain_free_irqs_top, }; -static const struct irq_domain_ops gic_irq_domain_ops = { - .map = gic_irq_domain_map, - .unmap = gic_irq_domain_unmap, -}; - -static void gic_init_chip(struct gic_chip_data *gic, struct device *dev, - const char *name, bool use_eoimode1) -{ - /* Initialize irq_chip */ - gic->chip = gic_chip; - gic->chip.name = name; - gic->chip.parent_device = dev; - - if (use_eoimode1) { - gic->chip.irq_mask = gic_eoimode1_mask_irq; - gic->chip.irq_eoi = gic_eoimode1_eoi_irq; - gic->chip.irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity; - } - - if (gic == &gic_data[0]) { - gic->chip.irq_set_affinity = gic_set_affinity; - gic->chip.ipi_send_mask = gic_ipi_send_mask; - } -} - static int gic_init_bases(struct gic_chip_data *gic, struct fwnode_handle *handle) { @@ -1170,30 +1207,9 @@ static int gic_init_bases(struct gic_chip_data *gic, gic_irqs = 1020; gic->gic_irqs = gic_irqs; - if (handle) { /* DT/ACPI */ - gic->domain = irq_domain_create_linear(handle, gic_irqs, - &gic_irq_domain_hierarchy_ops, - gic); - } else { /* Legacy support */ - /* - * For primary GICs, skip over SGIs. - * No secondary GIC support whatsoever. - */ - int irq_base; - - gic_irqs -= 16; /* calculate # of irqs to allocate */ - - irq_base = irq_alloc_descs(16, 16, gic_irqs, - numa_node_id()); - if (irq_base < 0) { - WARN(1, "Cannot allocate irq_descs @ IRQ16, assuming pre-allocated\n"); - irq_base = 16; - } - - gic->domain = irq_domain_add_legacy(NULL, gic_irqs, irq_base, - 16, &gic_irq_domain_ops, gic); - } - + gic->domain = irq_domain_create_linear(handle, gic_irqs, + &gic_irq_domain_hierarchy_ops, + gic); if (WARN_ON(!gic->domain)) { ret = -ENODEV; goto error; @@ -1222,7 +1238,6 @@ error: static int __init __gic_init_bases(struct gic_chip_data *gic, struct fwnode_handle *handle) { - char *name; int i, ret; if (WARN_ON(!gic || gic->domain)) @@ -1242,40 +1257,13 @@ static int __init __gic_init_bases(struct gic_chip_data *gic, pr_info("GIC: Using split EOI/Deactivate mode\n"); } - if (static_branch_likely(&supports_deactivate_key) && gic == &gic_data[0]) { - name = kasprintf(GFP_KERNEL, "GICv2"); - gic_init_chip(gic, NULL, name, true); - } else { - name = kasprintf(GFP_KERNEL, "GIC-%d", (int)(gic-&gic_data[0])); - gic_init_chip(gic, NULL, name, false); - } - ret = gic_init_bases(gic, handle); - if (ret) - kfree(name); - else if (gic == &gic_data[0]) + if (gic == &gic_data[0]) gic_smp_init(); return ret; } -void __init gic_init(void __iomem *dist_base, void __iomem *cpu_base) -{ - struct gic_chip_data *gic; - - /* - * Non-DT/ACPI systems won't run a hypervisor, so let's not - * bother with these... - */ - static_branch_disable(&supports_deactivate_key); - - gic = &gic_data[0]; - gic->raw_dist_base = dist_base; - gic->raw_cpu_base = cpu_base; - - __gic_init_bases(gic, NULL); -} - static void gic_teardown(struct gic_chip_data *gic) { if (WARN_ON(!gic)) @@ -1287,13 +1275,12 @@ static void gic_teardown(struct gic_chip_data *gic) iounmap(gic->raw_cpu_base); } -#ifdef CONFIG_OF static int gic_cnt __initdata; static bool gicv2_force_probe; static int __init gicv2_force_probe_cfg(char *buf) { - return strtobool(buf, &gicv2_force_probe); + return kstrtobool(buf, &gicv2_force_probe); } early_param("irqchip.gicv2_force_probe", gicv2_force_probe_cfg); @@ -1375,6 +1362,30 @@ static bool gic_check_eoimode(struct device_node *node, void __iomem **base) return true; } +static bool gic_enable_rmw_access(void *data) +{ + /* + * The EMEV2 class of machines has a broken interconnect, and + * locks up on accesses that are less than 32bit. So far, only + * the affinity setting requires it. + */ + if (of_machine_is_compatible("renesas,emev2")) { + static_branch_enable(&needs_rmw_access); + return true; + } + + return false; +} + +static const struct gic_quirk gic_quirks[] = { + { + .desc = "broken byte access", + .compatible = "arm,pl390", + .init = gic_enable_rmw_access, + }, + { }, +}; + static int gic_of_setup(struct gic_chip_data *gic, struct device_node *node) { if (!gic || !node) @@ -1391,6 +1402,8 @@ static int gic_of_setup(struct gic_chip_data *gic, struct device_node *node) if (of_property_read_u32(node, "cpu-offset", &gic->percpu_offset)) gic->percpu_offset = 0; + gic_enable_of_quirks(node, gic_quirks, gic); + return 0; error: @@ -1410,8 +1423,6 @@ int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq) if (!*gic) return -ENOMEM; - gic_init_chip(*gic, dev, dev->of_node->name, false); - ret = gic_of_setup(*gic, dev->of_node); if (ret) return ret; @@ -1422,6 +1433,7 @@ int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq) return ret; } + irq_domain_set_pm_device((*gic)->domain, dev); irq_set_chained_handler_and_data(irq, gic_handle_cascade_irq, *gic); return 0; @@ -1447,6 +1459,8 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) if (ret) return; + gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base; + if (static_branch_likely(&supports_deactivate_key)) vgic_set_kvm_info(&gic_v2_kvm_info); } @@ -1507,12 +1521,6 @@ IRQCHIP_DECLARE(cortex_a7_gic, "arm,cortex-a7-gic", gic_of_init); IRQCHIP_DECLARE(msm_8660_qgic, "qcom,msm-8660-qgic", gic_of_init); IRQCHIP_DECLARE(msm_qgic2, "qcom,msm-qgic2", gic_of_init); IRQCHIP_DECLARE(pl390, "arm,pl390", gic_of_init); -#else -int gic_of_init_child(struct device *dev, struct gic_chip_data **gic, int irq) -{ - return -ENOTSUPP; -} -#endif #ifdef CONFIG_ACPI static struct @@ -1614,15 +1622,22 @@ static void __init gic_acpi_setup_kvm_info(void) return; gic_v2_kvm_info.maint_irq = irq; + gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base; vgic_set_kvm_info(&gic_v2_kvm_info); } +static struct fwnode_handle *gsi_domain_handle; + +static struct fwnode_handle *gic_v2_get_gsi_domain_id(u32 gsi) +{ + return gsi_domain_handle; +} + static int __init gic_v2_acpi_init(union acpi_subtable_headers *header, const unsigned long end) { struct acpi_madt_generic_distributor *dist; - struct fwnode_handle *domain_handle; struct gic_chip_data *gic = &gic_data[0]; int count, ret; @@ -1660,22 +1675,22 @@ static int __init gic_v2_acpi_init(union acpi_subtable_headers *header, /* * Initialize GIC instance zero (no multi-GIC support). */ - domain_handle = irq_domain_alloc_fwnode(&dist->base_address); - if (!domain_handle) { + gsi_domain_handle = irq_domain_alloc_fwnode(&dist->base_address); + if (!gsi_domain_handle) { pr_err("Unable to allocate domain handle\n"); gic_teardown(gic); return -ENOMEM; } - ret = __gic_init_bases(gic, domain_handle); + ret = __gic_init_bases(gic, gsi_domain_handle); if (ret) { pr_err("Failed to initialise GIC\n"); - irq_domain_free_fwnode(domain_handle); + irq_domain_free_fwnode(gsi_domain_handle); gic_teardown(gic); return ret; } - acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle); + acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, gic_v2_get_gsi_domain_id); if (IS_ENABLED(CONFIG_ARM_GIC_V2M)) gicv2m_init(NULL, gic_data[0].domain); |