diff options
Diffstat (limited to 'drivers/irqchip')
151 files changed, 15217 insertions, 4468 deletions
diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 4b9036c6d45b..f334f49c9791 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -26,7 +26,9 @@ config ARM_GIC_V2M bool depends on PCI select ARM_GIC + select IRQ_MSI_LIB select PCI_MSI + select IRQ_MSI_IOMMU config GIC_NON_BANKED bool @@ -34,21 +36,20 @@ config GIC_NON_BANKED config ARM_GIC_V3 bool select IRQ_DOMAIN_HIERARCHY - select PARTITION_PERCPU select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select HAVE_ARM_SMCCC_DISCOVERY + select IRQ_MSI_IOMMU + +config ARM_GIC_ITS_PARENT + bool config ARM_GIC_V3_ITS bool select GENERIC_MSI_IRQ + select IRQ_MSI_LIB + select ARM_GIC_ITS_PARENT default ARM_GIC_V3 - -config ARM_GIC_V3_ITS_PCI - bool - depends on ARM_GIC_V3_ITS - depends on PCI - depends on PCI_MSI - default ARM_GIC_V3_ITS + select IRQ_MSI_IOMMU config ARM_GIC_V3_ITS_FSL_MC bool @@ -56,6 +57,14 @@ config ARM_GIC_V3_ITS_FSL_MC depends on FSL_MC_BUS default ARM_GIC_V3_ITS +config ARM_GIC_V5 + bool + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_EFFECTIVE_AFF_MASK + select GENERIC_MSI_IRQ + select IRQ_MSI_LIB + select ARM_GIC_ITS_PARENT + config ARM_NVIC bool select IRQ_DOMAIN_HIERARCHY @@ -74,16 +83,22 @@ config ARM_VIC_NR The maximum number of VICs available in the system, for power management. +config IRQ_MSI_LIB + bool + select GENERIC_MSI_IRQ + config ARMADA_370_XP_IRQ bool select GENERIC_IRQ_CHIP select PCI_MSI if PCI + select IRQ_MSI_LIB if PCI select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP config ALPINE_MSI bool depends on PCI select PCI_MSI + select IRQ_MSI_LIB select GENERIC_IRQ_CHIP config AL_FIC @@ -111,6 +126,22 @@ config I8259 bool select IRQ_DOMAIN +config BCM2712_MIP + tristate "Broadcom BCM2712 MSI-X Interrupt Peripheral support" + depends on ARCH_BRCMSTB || ARCH_BCM2835 || COMPILE_TEST + default m if ARCH_BRCMSTB || ARCH_BCM2835 + depends on ARM_GIC + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN_HIERARCHY + select GENERIC_MSI_IRQ + select IRQ_MSI_LIB + help + Enable support for the Broadcom BCM2712 MSI-X target peripheral + (MIP) needed by brcmstb PCIe to handle MSI-X interrupts on + Raspberry Pi 5. + + If unsure say n. + config BCM6345_L1_IRQ bool select GENERIC_IRQ_CHIP @@ -119,7 +150,7 @@ config BCM6345_L1_IRQ config BCM7038_L1_IRQ tristate "Broadcom STB 7038-style L1/L2 interrupt controller driver" - depends on ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -127,14 +158,14 @@ config BCM7038_L1_IRQ config BCM7120_L2_IRQ tristate "Broadcom STB 7120-style L2 interrupt controller driver" - depends on ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN config BRCMSTB_L2_IRQ tristate "Broadcom STB generic L2 interrupt controller driver" - depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -149,6 +180,11 @@ config DW_APB_ICTL select GENERIC_IRQ_CHIP select IRQ_DOMAIN_HIERARCHY +config ECONET_EN751221_INTC + bool + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN + config FARADAY_FTINTC010 bool select IRQ_DOMAIN @@ -169,6 +205,19 @@ config IXP4XX_IRQ select IRQ_DOMAIN select SPARSE_IRQ +config LAN966X_OIC + tristate "Microchip LAN966x OIC Support" + depends on MCHP_LAN966X_PCI || COMPILE_TEST + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN + help + Enable support for the LAN966x Outbound Interrupt Controller. + This controller is present on the Microchip LAN966x PCI device and + maps the internal interrupts sources to PCIe interrupt. + + To compile this driver as a module, choose M here: the module + will be called irq-lan966x-oic. + config MADERA_IRQ tristate @@ -248,6 +297,13 @@ config RENESAS_RZG2L_IRQC Enable support for the Renesas RZ/G2L (and alike SoC) Interrupt Controller for external devices. +config RENESAS_RZV2H_ICU + bool "Renesas RZ/V2H(P) ICU support" if COMPILE_TEST + select GENERIC_IRQ_CHIP + select IRQ_DOMAIN_HIERARCHY + help + Enable support for the Renesas RZ/V2H(P) Interrupt Control Unit (ICU) + config SL28CPLD_INTC bool "Kontron sl28cpld IRQ controller" depends on MFD_SL28CPLD=y || COMPILE_TEST @@ -328,6 +384,7 @@ config KEYSTONE_IRQ config MIPS_GIC bool + select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select GENERIC_IRQ_IPI if SMP select IRQ_DOMAIN_HIERARCHY select MIPS_CM @@ -366,6 +423,7 @@ config MSCC_OCELOT_IRQ select GENERIC_IRQ_CHIP config MVEBU_GICP + select IRQ_MSI_LIB bool config MVEBU_ICU @@ -373,6 +431,7 @@ config MVEBU_ICU config MVEBU_ODMI bool + select IRQ_MSI_LIB select GENERIC_MSI_IRQ config MVEBU_PIC @@ -387,10 +446,18 @@ config LS_EXTIRQ config LS_SCFG_MSI def_bool y if SOC_LS1021A || ARCH_LAYERSCAPE + select IRQ_MSI_IOMMU depends on PCI_MSI + select IRQ_MSI_LIB -config PARTITION_PERCPU - bool +config STM32MP_EXTI + tristate "STM32MP extended interrupts and event controller" + depends on (ARCH_STM32 && !ARM_SINGLE_ARMV7M) || COMPILE_TEST + default ARCH_STM32 && !ARM_SINGLE_ARMV7M + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_CHIP + help + Support STM32MP EXTI (extended interrupts and event) controller. config STM32_EXTI bool @@ -483,10 +550,12 @@ config IMX_MU_MSI tristate "i.MX MU used as MSI controller" depends on OF && HAS_IOMEM depends on ARCH_MXC || COMPILE_TEST + depends on ARM || ARM64 default m if ARCH_MXC select IRQ_DOMAIN select IRQ_DOMAIN_HIERARCHY select GENERIC_MSI_IRQ + select IRQ_MSI_LIB help Provide a driver for the i.MX Messaging Unit block used as a CPU-to-CPU MSI controller. This requires a specially crafted DT @@ -504,8 +573,9 @@ config LS1X_IRQ Support for the Loongson-1 platform Interrupt Controller. config TI_SCI_INTR_IRQCHIP - bool + tristate "TI SCI INTR Interrupt Controller" depends on TI_SCI_PROTOCOL + depends on ARCH_K3 || COMPILE_TEST select IRQ_DOMAIN_HIERARCHY help This enables the irqchip driver support for K3 Interrupt router @@ -514,8 +584,9 @@ config TI_SCI_INTR_IRQCHIP TI System Controller, say Y here. Otherwise, say N. config TI_SCI_INTA_IRQCHIP - bool + tristate "TI SCI INTA Interrupt Controller" depends on TI_SCI_PROTOCOL + depends on ARCH_K3 || (COMPILE_TEST && ARM64) select IRQ_DOMAIN_HIERARCHY select TI_SCI_INTA_MSI_DOMAIN help @@ -540,12 +611,68 @@ config RISCV_INTC depends on RISCV select IRQ_DOMAIN_HIERARCHY +config RISCV_APLIC + bool + depends on RISCV + select IRQ_DOMAIN_HIERARCHY + +config RISCV_APLIC_MSI + bool + depends on RISCV_APLIC + select GENERIC_MSI_IRQ + default RISCV_APLIC + +config RISCV_IMSIC + bool + depends on RISCV + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_MATRIX_ALLOCATOR + select GENERIC_MSI_IRQ + select IRQ_MSI_LIB + +config RISCV_RPMI_SYSMSI + bool + depends on RISCV && MAILBOX + select IRQ_DOMAIN_HIERARCHY + select GENERIC_MSI_IRQ + default RISCV + config SIFIVE_PLIC bool depends on RISCV select IRQ_DOMAIN_HIERARCHY select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP +config STARFIVE_JH8100_INTC + bool "StarFive JH8100 External Interrupt Controller" + depends on ARCH_STARFIVE || COMPILE_TEST + default ARCH_STARFIVE + select IRQ_DOMAIN_HIERARCHY + help + This enables support for the INTC chip found in StarFive JH8100 + SoC. + + If you don't know what to do here, say Y. + +config ACLINT_SSWI + bool "RISC-V ACLINT S-mode IPI Interrupt Controller" + depends on RISCV + depends on SMP + select IRQ_DOMAIN_HIERARCHY + select GENERIC_IRQ_IPI_MUX + help + This enables support for variants of the RISC-V ACLINT-SSWI device. + Supported variants are: + - T-HEAD, with compatible "thead,c900-aclint-sswi" + - MIPS P8700, with compatible "mips,p8700-aclint-sswi" + + If you don't know what to do here, say Y. + +# Backwards compatibility so oldconfig does not drop it. +config THEAD_C900_ACLINT_SSWI + bool + select ACLINT_SSWI + config EXYNOS_IRQ_COMBINER bool "Samsung Exynos IRQ combiner support" if COMPILE_TEST depends on (ARCH_EXYNOS && ARM) || COMPILE_TEST @@ -557,7 +684,7 @@ config IRQ_LOONGARCH_CPU bool select GENERIC_IRQ_CHIP select IRQ_DOMAIN - select GENERIC_IRQ_EFFECTIVE_AFF_MASK + select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select LOONGSON_HTVEC select LOONGSON_LIOINTC select LOONGSON_EIOINTC @@ -567,7 +694,7 @@ config IRQ_LOONGARCH_CPU help Support for the LoongArch CPU Interrupt Controller. For details of irq chip hierarchy on LoongArch platforms please read the document - Documentation/loongarch/irq-chip-model.rst. + Documentation/arch/loongarch/irq-chip-model.rst. config LOONGSON_LIOINTC bool "Loongson Local I/O Interrupt Controller" @@ -620,6 +747,7 @@ config LOONGSON_PCH_MSI depends on PCI default MACH_LOONGSON64 select IRQ_DOMAIN_HIERARCHY + select IRQ_MSI_LIB select PCI_MSI help Support for the Loongson PCH MSI Controller. @@ -670,6 +798,18 @@ config MCHP_EIC help Support for Microchip External Interrupt Controller. +config SOPHGO_SG2042_MSI + bool "Sophgo SG2042 MSI Controller" + depends on ARCH_SOPHGO || COMPILE_TEST + depends on PCI + select IRQ_DOMAIN_HIERARCHY + select IRQ_MSI_LIB + select PCI_MSI + help + Support for the Sophgo SG2042 MSI Controller. + This on-chip interrupt controller enables MSI sources to be + routed to the primary PLIC controller on SoC. + config SUNPLUS_SP7021_INTC bool "Sunplus SP7021 interrupt controller" if COMPILE_TEST default SOC_SP7021 diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index ffd945fe71aa..6a229443efe0 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_ARCH_BCM2835) += irq-bcm2836.o obj-$(CONFIG_ARCH_ACTIONS) += irq-owl-sirq.o obj-$(CONFIG_DAVINCI_CP_INTC) += irq-davinci-cp-intc.o obj-$(CONFIG_EXYNOS_IRQ_COMBINER) += exynos-combiner.o +obj-$(CONFIG_ECONET_EN751221_INTC) += irq-econet-en751221.o obj-$(CONFIG_FARADAY_FTINTC010) += irq-ftintc010.o obj-$(CONFIG_ARCH_HIP04) += irq-hip04.o obj-$(CONFIG_ARCH_LPC32XX) += irq-lpc32xx.o @@ -29,12 +30,14 @@ obj-$(CONFIG_ARCH_SPEAR3XX) += spear-shirq.o obj-$(CONFIG_ARM_GIC) += irq-gic.o irq-gic-common.o obj-$(CONFIG_ARM_GIC_PM) += irq-gic-pm.o obj-$(CONFIG_ARCH_REALVIEW) += irq-gic-realview.o +obj-$(CONFIG_IRQ_MSI_LIB) += irq-msi-lib.o obj-$(CONFIG_ARM_GIC_V2M) += irq-gic-v2m.o obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o -obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v3-its-platform-msi.o irq-gic-v4.o -obj-$(CONFIG_ARM_GIC_V3_ITS_PCI) += irq-gic-v3-its-pci-msi.o +obj-$(CONFIG_ARM_GIC_ITS_PARENT) += irq-gic-its-msi-parent.o +obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC) += irq-gic-v3-its-fsl-mc-msi.o -obj-$(CONFIG_PARTITION_PERCPU) += irq-partition-percpu.o +obj-$(CONFIG_ARM_GIC_V5) += irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \ + irq-gic-v5-iwb.o obj-$(CONFIG_HISILICON_IRQ_MBIGEN) += irq-mbigen.o obj-$(CONFIG_ARM_NVIC) += irq-nvic.o obj-$(CONFIG_ARM_VIC) += irq-vic.o @@ -51,6 +54,7 @@ obj-$(CONFIG_RENESAS_INTC_IRQPIN) += irq-renesas-intc-irqpin.o obj-$(CONFIG_RENESAS_IRQC) += irq-renesas-irqc.o obj-$(CONFIG_RENESAS_RZA1_IRQC) += irq-renesas-rza1.o obj-$(CONFIG_RENESAS_RZG2L_IRQC) += irq-renesas-rzg2l.o +obj-$(CONFIG_RENESAS_RZV2H_ICU) += irq-renesas-rzv2h.o obj-$(CONFIG_VERSATILE_FPGA_IRQ) += irq-versatile-fpga.o obj-$(CONFIG_ARCH_NSPIRE) += irq-zevio.o obj-$(CONFIG_ARCH_VT8500) += irq-vt8500.o @@ -62,6 +66,7 @@ obj-$(CONFIG_XTENSA_MX) += irq-xtensa-mx.o obj-$(CONFIG_XILINX_INTC) += irq-xilinx-intc.o obj-$(CONFIG_IRQ_CROSSBAR) += irq-crossbar.o obj-$(CONFIG_SOC_VF610) += irq-vf610-mscm-ir.o +obj-$(CONFIG_BCM2712_MIP) += irq-bcm2712-mip.o obj-$(CONFIG_BCM6345_L1_IRQ) += irq-bcm6345-l1.o obj-$(CONFIG_BCM7038_L1_IRQ) += irq-bcm7038-l1.o obj-$(CONFIG_BCM7120_L2_IRQ) += irq-bcm7120-l2.o @@ -84,6 +89,8 @@ obj-$(CONFIG_MVEBU_SEI) += irq-mvebu-sei.o obj-$(CONFIG_LS_EXTIRQ) += irq-ls-extirq.o obj-$(CONFIG_LS_SCFG_MSI) += irq-ls-scfg-msi.o obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-vic.o irq-aspeed-i2c-ic.o irq-aspeed-scu-ic.o +obj-$(CONFIG_ARCH_ASPEED) += irq-aspeed-intc.o +obj-$(CONFIG_STM32MP_EXTI) += irq-stm32mp-exti.o obj-$(CONFIG_STM32_EXTI) += irq-stm32-exti.o obj-$(CONFIG_QCOM_IRQ_COMBINER) += qcom-irq-combiner.o obj-$(CONFIG_IRQ_UNIPHIER_AIDET) += irq-uniphier-aidet.o @@ -95,16 +102,23 @@ obj-$(CONFIG_QCOM_MPM) += irq-qcom-mpm.o obj-$(CONFIG_CSKY_MPINTC) += irq-csky-mpintc.o obj-$(CONFIG_CSKY_APB_INTC) += irq-csky-apb-intc.o obj-$(CONFIG_RISCV_INTC) += irq-riscv-intc.o +obj-$(CONFIG_RISCV_APLIC) += irq-riscv-aplic-main.o irq-riscv-aplic-direct.o +obj-$(CONFIG_RISCV_APLIC_MSI) += irq-riscv-aplic-msi.o +obj-$(CONFIG_RISCV_IMSIC) += irq-riscv-imsic-state.o irq-riscv-imsic-early.o irq-riscv-imsic-platform.o +obj-$(CONFIG_RISCV_RPMI_SYSMSI) += irq-riscv-rpmi-sysmsi.o obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o +obj-$(CONFIG_STARFIVE_JH8100_INTC) += irq-starfive-jh8100-intc.o +obj-$(CONFIG_ACLINT_SSWI) += irq-aclint-sswi.o obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o obj-$(CONFIG_IMX_MU_MSI) += irq-imx-mu-msi.o obj-$(CONFIG_MADERA_IRQ) += irq-madera.o +obj-$(CONFIG_LAN966X_OIC) += irq-lan966x-oic.o obj-$(CONFIG_LS1X_IRQ) += irq-ls1x.o obj-$(CONFIG_TI_SCI_INTR_IRQCHIP) += irq-ti-sci-intr.o obj-$(CONFIG_TI_SCI_INTA_IRQCHIP) += irq-ti-sci-inta.o obj-$(CONFIG_TI_PRUSS_INTC) += irq-pruss-intc.o -obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o +obj-$(CONFIG_IRQ_LOONGARCH_CPU) += irq-loongarch-cpu.o irq-loongarch-avec.o obj-$(CONFIG_LOONGSON_LIOINTC) += irq-loongson-liointc.o obj-$(CONFIG_LOONGSON_EIOINTC) += irq-loongson-eiointc.o obj-$(CONFIG_LOONGSON_HTPIC) += irq-loongson-htpic.o @@ -119,4 +133,5 @@ obj-$(CONFIG_WPCM450_AIC) += irq-wpcm450-aic.o obj-$(CONFIG_IRQ_IDT3243X) += irq-idt3243x.o obj-$(CONFIG_APPLE_AIC) += irq-apple-aic.o obj-$(CONFIG_MCHP_EIC) += irq-mchp-eic.o +obj-$(CONFIG_SOPHGO_SG2042_MSI) += irq-sg2042-msi.o obj-$(CONFIG_SUNPLUS_SP7021_INTC) += irq-sp7021-intc.o diff --git a/drivers/irqchip/exynos-combiner.c b/drivers/irqchip/exynos-combiner.c index 552aa04ff063..495848442b35 100644 --- a/drivers/irqchip/exynos-combiner.c +++ b/drivers/irqchip/exynos-combiner.c @@ -180,7 +180,7 @@ static void __init combiner_init(void __iomem *combiner_base, if (!combiner_data) return; - combiner_irq_domain = irq_domain_add_linear(np, nr_irq, + combiner_irq_domain = irq_domain_create_linear(of_fwnode_handle(np), nr_irq, &combiner_irq_domain_ops, combiner_data); if (WARN_ON(!combiner_irq_domain)) { pr_warn("%s: irq domain init failed\n", __func__); @@ -200,12 +200,13 @@ static void __init combiner_init(void __iomem *combiner_base, /** * combiner_suspend - save interrupt combiner state before suspend + * @data: syscore context * * Save the interrupt enable set register for all combiner groups since * the state is lost when the system enters into a sleep state. * */ -static int combiner_suspend(void) +static int combiner_suspend(void *data) { int i; @@ -218,12 +219,13 @@ static int combiner_suspend(void) /** * combiner_resume - restore interrupt combiner state after resume + * @data: syscore context * * Restore the interrupt enable set register for all combiner groups since * the state is lost when the system enters into a sleep state on suspend. * */ -static void combiner_resume(void) +static void combiner_resume(void *data) { int i; @@ -240,11 +242,15 @@ static void combiner_resume(void) #define combiner_resume NULL #endif -static struct syscore_ops combiner_syscore_ops = { +static const struct syscore_ops combiner_syscore_ops = { .suspend = combiner_suspend, .resume = combiner_resume, }; +static struct syscore combiner_syscore = { + .ops = &combiner_syscore_ops, +}; + static int __init combiner_of_init(struct device_node *np, struct device_node *parent) { @@ -264,7 +270,7 @@ static int __init combiner_of_init(struct device_node *np, combiner_init(combiner_base, np); - register_syscore_ops(&combiner_syscore_ops); + register_syscore(&combiner_syscore); return 0; } diff --git a/drivers/irqchip/irq-aclint-sswi.c b/drivers/irqchip/irq-aclint-sswi.c new file mode 100644 index 000000000000..fee30f3bc5ac --- /dev/null +++ b/drivers/irqchip/irq-aclint-sswi.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2024 Inochi Amaoto <inochiama@gmail.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/of_address.h> +#include <linux/spinlock.h> +#include <linux/smp.h> +#include <linux/string_choices.h> +#include <asm/sbi.h> +#include <asm/vendorid_list.h> + +static int sswi_ipi_virq __ro_after_init; +static DEFINE_PER_CPU(void __iomem *, sswi_cpu_regs); + +static void aclint_sswi_ipi_send(unsigned int cpu) +{ + writel(0x1, per_cpu(sswi_cpu_regs, cpu)); +} + +static void aclint_sswi_ipi_clear(void) +{ + writel_relaxed(0x0, this_cpu_read(sswi_cpu_regs)); +} + +static void aclint_sswi_ipi_handle(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + csr_clear(CSR_IP, IE_SIE); + aclint_sswi_ipi_clear(); + + ipi_mux_process(); + + chained_irq_exit(chip, desc); +} + +static int aclint_sswi_starting_cpu(unsigned int cpu) +{ + enable_percpu_irq(sswi_ipi_virq, irq_get_trigger_type(sswi_ipi_virq)); + + return 0; +} + +static int aclint_sswi_dying_cpu(unsigned int cpu) +{ + aclint_sswi_ipi_clear(); + + disable_percpu_irq(sswi_ipi_virq); + + return 0; +} + +static int __init aclint_sswi_parse_irq(struct fwnode_handle *fwnode, void __iomem *reg) +{ + u32 contexts = of_irq_count(to_of_node(fwnode)); + + if (!(contexts)) { + pr_err("%pfwP: no ACLINT SSWI context available\n", fwnode); + return -EINVAL; + } + + for (u32 i = 0; i < contexts; i++) { + struct of_phandle_args parent; + unsigned long hartid; + u32 hart_index; + int rc, cpu; + + rc = of_irq_parse_one(to_of_node(fwnode), i, &parent); + if (rc) + return rc; + + rc = riscv_of_parent_hartid(parent.np, &hartid); + if (rc) + return rc; + + if (parent.args[0] != RV_IRQ_SOFT) + return -ENOTSUPP; + + cpu = riscv_hartid_to_cpuid(hartid); + + rc = riscv_get_hart_index(fwnode, i, &hart_index); + if (rc) { + pr_warn("%pfwP: hart index [%d] not found\n", fwnode, i); + return -EINVAL; + } + per_cpu(sswi_cpu_regs, cpu) = reg + hart_index * 4; + } + + pr_info("%pfwP: register %u CPU%s\n", fwnode, contexts, str_plural(contexts)); + + return 0; +} + +static int __init aclint_sswi_probe(struct fwnode_handle *fwnode) +{ + struct irq_domain *domain; + void __iomem *reg; + int virq, rc; + + if (!is_of_node(fwnode)) + return -EINVAL; + + reg = of_iomap(to_of_node(fwnode), 0); + if (!reg) + return -ENOMEM; + + /* Parse SSWI setting */ + rc = aclint_sswi_parse_irq(fwnode, reg); + if (rc < 0) + return rc; + + /* If mulitple SSWI devices are present, do not register irq again */ + if (sswi_ipi_virq) + return 0; + + /* Find riscv intc domain and create IPI irq mapping */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); + if (!domain) { + pr_err("%pfwP: Failed to find INTC domain\n", fwnode); + return -ENOENT; + } + + sswi_ipi_virq = irq_create_mapping(domain, RV_IRQ_SOFT); + if (!sswi_ipi_virq) { + pr_err("unable to create ACLINT SSWI IRQ mapping\n"); + return -ENOMEM; + } + + /* Register SSWI irq and handler */ + virq = ipi_mux_create(BITS_PER_BYTE, aclint_sswi_ipi_send); + if (virq <= 0) { + pr_err("unable to create muxed IPIs\n"); + irq_dispose_mapping(sswi_ipi_virq); + return virq < 0 ? virq : -ENOMEM; + } + + irq_set_chained_handler(sswi_ipi_virq, aclint_sswi_ipi_handle); + + cpuhp_setup_state(CPUHP_AP_IRQ_ACLINT_SSWI_STARTING, + "irqchip/aclint-sswi:starting", + aclint_sswi_starting_cpu, + aclint_sswi_dying_cpu); + + riscv_ipi_set_virq_range(virq, BITS_PER_BYTE); + + return 0; +} + +/* generic/MIPS variant */ +static int __init generic_aclint_sswi_probe(struct fwnode_handle *fwnode) +{ + int rc; + + rc = aclint_sswi_probe(fwnode); + if (rc) + return rc; + + /* Announce that SSWI is providing IPIs */ + pr_info("providing IPIs using ACLINT SSWI\n"); + + return 0; +} + +static int __init generic_aclint_sswi_early_probe(struct device_node *node, + struct device_node *parent) +{ + return generic_aclint_sswi_probe(&node->fwnode); +} +IRQCHIP_DECLARE(mips_p8700_sswi, "mips,p8700-aclint-sswi", generic_aclint_sswi_early_probe); +IRQCHIP_DECLARE(nuclei_ux900_sswi, "nuclei,ux900-aclint-sswi", generic_aclint_sswi_early_probe); + +/* THEAD variant */ +#define THEAD_C9XX_CSR_SXSTATUS 0x5c0 +#define THEAD_C9XX_SXSTATUS_CLINTEE BIT(17) + +static int __init thead_aclint_sswi_probe(struct fwnode_handle *fwnode) +{ + int rc; + + /* If it is T-HEAD CPU, check whether SSWI is enabled */ + if (riscv_cached_mvendorid(0) == THEAD_VENDOR_ID && + !(csr_read(THEAD_C9XX_CSR_SXSTATUS) & THEAD_C9XX_SXSTATUS_CLINTEE)) + return -ENOTSUPP; + + rc = aclint_sswi_probe(fwnode); + if (rc) + return rc; + + /* Announce that SSWI is providing IPIs */ + pr_info("providing IPIs using THEAD ACLINT SSWI\n"); + + return 0; +} + +static int __init thead_aclint_sswi_early_probe(struct device_node *node, + struct device_node *parent) +{ + return thead_aclint_sswi_probe(&node->fwnode); +} +IRQCHIP_DECLARE(thead_aclint_sswi, "thead,c900-aclint-sswi", thead_aclint_sswi_early_probe); diff --git a/drivers/irqchip/irq-al-fic.c b/drivers/irqchip/irq-al-fic.c index dfb761e86c9c..8f300843bbca 100644 --- a/drivers/irqchip/irq-al-fic.c +++ b/drivers/irqchip/irq-al-fic.c @@ -65,15 +65,13 @@ static int al_fic_irq_set_type(struct irq_data *data, unsigned int flow_type) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); struct al_fic *fic = gc->private; enum al_fic_state new_state; - int ret = 0; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); if (((flow_type & IRQ_TYPE_SENSE_MASK) != IRQ_TYPE_LEVEL_HIGH) && ((flow_type & IRQ_TYPE_SENSE_MASK) != IRQ_TYPE_EDGE_RISING)) { pr_debug("fic doesn't support flow type %d\n", flow_type); - ret = -EINVAL; - goto err; + return -EINVAL; } new_state = (flow_type & IRQ_TYPE_LEVEL_HIGH) ? @@ -91,16 +89,10 @@ static int al_fic_irq_set_type(struct irq_data *data, unsigned int flow_type) if (fic->state == AL_FIC_UNCONFIGURED) { al_fic_set_trigger(fic, gc, new_state); } else if (fic->state != new_state) { - pr_debug("fic %s state already configured to %d\n", - fic->name, fic->state); - ret = -EINVAL; - goto err; + pr_debug("fic %s state already configured to %d\n", fic->name, fic->state); + return -EINVAL; } - -err: - irq_gc_unlock(gc); - - return ret; + return 0; } static void al_fic_irq_handler(struct irq_desc *desc) @@ -139,7 +131,7 @@ static int al_fic_register(struct device_node *node, struct irq_chip_generic *gc; int ret; - fic->domain = irq_domain_add_linear(node, + fic->domain = irq_domain_create_linear(of_fwnode_handle(node), NR_FIC_IRQS, &irq_generic_chip_ops, fic); diff --git a/drivers/irqchip/irq-alpine-msi.c b/drivers/irqchip/irq-alpine-msi.c index 9c8b1349ee17..159d9ec7c0dd 100644 --- a/drivers/irqchip/irq-alpine-msi.c +++ b/drivers/irqchip/irq-alpine-msi.c @@ -14,6 +14,7 @@ #include <linux/irqchip.h> #include <linux/irqchip/arm-gic.h> +#include <linux/irqchip/irq-msi-lib.h> #include <linux/msi.h> #include <linux/of.h> #include <linux/of_address.h> @@ -29,84 +30,45 @@ #define ALPINE_MSIX_SPI_TARGET_CLUSTER0 BIT(16) struct alpine_msix_data { - spinlock_t msi_map_lock; - phys_addr_t addr; - u32 spi_first; /* The SGI number that MSIs start */ - u32 num_spis; /* The number of SGIs for MSIs */ - unsigned long *msi_map; -}; - -static void alpine_msix_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void alpine_msix_unmask_msi_irq(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip alpine_msix_irq_chip = { - .name = "MSIx", - .irq_mask = alpine_msix_mask_msi_irq, - .irq_unmask = alpine_msix_unmask_msi_irq, - .irq_eoi = irq_chip_eoi_parent, - .irq_set_affinity = irq_chip_set_affinity_parent, + spinlock_t msi_map_lock; + phys_addr_t addr; + u32 spi_first; /* The SGI number that MSIs start */ + u32 num_spis; /* The number of SGIs for MSIs */ + unsigned long *msi_map; }; static int alpine_msix_allocate_sgi(struct alpine_msix_data *priv, int num_req) { int first; - spin_lock(&priv->msi_map_lock); - - first = bitmap_find_next_zero_area(priv->msi_map, priv->num_spis, 0, - num_req, 0); - if (first >= priv->num_spis) { - spin_unlock(&priv->msi_map_lock); + guard(spinlock)(&priv->msi_map_lock); + first = bitmap_find_next_zero_area(priv->msi_map, priv->num_spis, 0, num_req, 0); + if (first >= priv->num_spis) return -ENOSPC; - } bitmap_set(priv->msi_map, first, num_req); - - spin_unlock(&priv->msi_map_lock); - return priv->spi_first + first; } -static void alpine_msix_free_sgi(struct alpine_msix_data *priv, unsigned sgi, - int num_req) +static void alpine_msix_free_sgi(struct alpine_msix_data *priv, unsigned int sgi, int num_req) { int first = sgi - priv->spi_first; - spin_lock(&priv->msi_map_lock); - + guard(spinlock)(&priv->msi_map_lock); bitmap_clear(priv->msi_map, first, num_req); - - spin_unlock(&priv->msi_map_lock); } -static void alpine_msix_compose_msi_msg(struct irq_data *data, - struct msi_msg *msg) +static void alpine_msix_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { struct alpine_msix_data *priv = irq_data_get_irq_chip_data(data); phys_addr_t msg_addr = priv->addr; msg_addr |= (data->hwirq << 3); - msg->address_hi = upper_32_bits(msg_addr); msg->address_lo = lower_32_bits(msg_addr); msg->data = 0; } -static struct msi_domain_info alpine_msix_domain_info = { - .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX, - .chip = &alpine_msix_irq_chip, -}; - static struct irq_chip middle_irq_chip = { .name = "alpine_msix_middle", .irq_mask = irq_chip_mask_parent, @@ -116,8 +78,7 @@ static struct irq_chip middle_irq_chip = { .irq_compose_msi_msg = alpine_msix_compose_msi_msg, }; -static int alpine_msix_gic_domain_alloc(struct irq_domain *domain, - unsigned int virq, int sgi) +static int alpine_msix_gic_domain_alloc(struct irq_domain *domain, unsigned int virq, int sgi) { struct irq_fwspec fwspec; struct irq_data *d; @@ -138,12 +99,10 @@ static int alpine_msix_gic_domain_alloc(struct irq_domain *domain, d = irq_domain_get_irq_data(domain->parent, virq); d->chip->irq_set_type(d, IRQ_TYPE_EDGE_RISING); - return 0; } -static int alpine_msix_middle_domain_alloc(struct irq_domain *domain, - unsigned int virq, +static int alpine_msix_middle_domain_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, void *args) { struct alpine_msix_data *priv = domain->host_data; @@ -161,17 +120,15 @@ static int alpine_msix_middle_domain_alloc(struct irq_domain *domain, irq_domain_set_hwirq_and_chip(domain, virq + i, sgi + i, &middle_irq_chip, priv); } - return 0; err_sgi: - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); alpine_msix_free_sgi(priv, sgi, nr_irqs); return err; } -static void alpine_msix_middle_domain_free(struct irq_domain *domain, - unsigned int virq, +static void alpine_msix_middle_domain_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); @@ -182,14 +139,35 @@ static void alpine_msix_middle_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops alpine_msix_middle_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = alpine_msix_middle_domain_alloc, .free = alpine_msix_middle_domain_free, }; -static int alpine_msix_init_domains(struct alpine_msix_data *priv, - struct device_node *node) +#define ALPINE_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define ALPINE_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX) + +static struct msi_parent_ops alpine_msi_parent_ops = { + .supported_flags = ALPINE_MSI_FLAGS_SUPPORTED, + .required_flags = ALPINE_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI, + .prefix = "ALPINE-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int alpine_msix_init_domains(struct alpine_msix_data *priv, struct device_node *node) { - struct irq_domain *middle_domain, *msi_domain, *gic_domain; + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &alpine_msix_middle_domain_ops, + .host_data = priv, + }; struct device_node *gic_node; gic_node = of_irq_find_parent(node); @@ -198,41 +176,26 @@ static int alpine_msix_init_domains(struct alpine_msix_data *priv, return -ENODEV; } - gic_domain = irq_find_host(gic_node); + info.parent = irq_find_host(gic_node); of_node_put(gic_node); - if (!gic_domain) { + if (!info.parent) { pr_err("Failed to find the GIC domain\n"); return -ENXIO; } - middle_domain = irq_domain_add_hierarchy(gic_domain, 0, 0, NULL, - &alpine_msix_middle_domain_ops, - priv); - if (!middle_domain) { - pr_err("Failed to create the MSIX middle domain\n"); - return -ENOMEM; - } - - msi_domain = pci_msi_create_irq_domain(of_node_to_fwnode(node), - &alpine_msix_domain_info, - middle_domain); - if (!msi_domain) { + if (!msi_create_parent_irq_domain(&info, &alpine_msi_parent_ops)) { pr_err("Failed to create MSI domain\n"); - irq_domain_remove(middle_domain); return -ENOMEM; } - return 0; } -static int alpine_msix_init(struct device_node *node, - struct device_node *parent) +static int alpine_msix_init(struct device_node *node, struct device_node *parent) { - struct alpine_msix_data *priv; + struct alpine_msix_data *priv __free(kfree) = kzalloc(sizeof(*priv), GFP_KERNEL); struct resource res; int ret; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; @@ -241,7 +204,7 @@ static int alpine_msix_init(struct device_node *node, ret = of_address_to_resource(node, 0, &res); if (ret) { pr_err("Failed to allocate resource\n"); - goto err_priv; + return ret; } /* @@ -256,35 +219,28 @@ static int alpine_msix_init(struct device_node *node, if (of_property_read_u32(node, "al,msi-base-spi", &priv->spi_first)) { pr_err("Unable to parse MSI base\n"); - ret = -EINVAL; - goto err_priv; + return -EINVAL; } if (of_property_read_u32(node, "al,msi-num-spis", &priv->num_spis)) { pr_err("Unable to parse MSI numbers\n"); - ret = -EINVAL; - goto err_priv; + return -EINVAL; } - priv->msi_map = bitmap_zalloc(priv->num_spis, GFP_KERNEL); - if (!priv->msi_map) { - ret = -ENOMEM; - goto err_priv; - } + unsigned long *msi_map __free(kfree) = bitmap_zalloc(priv->num_spis, GFP_KERNEL); - pr_debug("Registering %d msixs, starting at %d\n", - priv->num_spis, priv->spi_first); + if (!msi_map) + return -ENOMEM; + priv->msi_map = msi_map; + + pr_debug("Registering %d msixs, starting at %d\n", priv->num_spis, priv->spi_first); ret = alpine_msix_init_domains(priv, node); if (ret) - goto err_map; + return ret; + retain_and_null_ptr(priv); + retain_and_null_ptr(msi_map); return 0; - -err_map: - bitmap_free(priv->msi_map); -err_priv: - kfree(priv); - return ret; } IRQCHIP_DECLARE(alpine_msix, "al,alpine-msix", alpine_msix_init); diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 5c534d9fd2b0..3c70364e7cdd 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -234,7 +234,10 @@ enum fiq_hwirq { AIC_NR_FIQ }; +/* True if UNCORE/UNCORE2 and Sn_... IPI registers are present and used (A11+) */ static DEFINE_STATIC_KEY_TRUE(use_fast_ipi); +/* True if SYS_IMP_APL_IPI_RR_LOCAL_EL1 exists for local fast IPIs (M1+) */ +static DEFINE_STATIC_KEY_TRUE(use_local_fast_ipi); struct aic_info { int version; @@ -252,6 +255,7 @@ struct aic_info { /* Features */ bool fast_ipi; + bool local_fast_ipi; }; static const struct aic_info aic1_info __initconst = { @@ -270,17 +274,32 @@ static const struct aic_info aic1_fipi_info __initconst = { .fast_ipi = true, }; +static const struct aic_info aic1_local_fipi_info __initconst = { + .version = 1, + + .event = AIC_EVENT, + .target_cpu = AIC_TARGET_CPU, + + .fast_ipi = true, + .local_fast_ipi = true, +}; + static const struct aic_info aic2_info __initconst = { .version = 2, .irq_cfg = AIC2_IRQ_CFG, .fast_ipi = true, + .local_fast_ipi = true, }; static const struct of_device_id aic_info_match[] = { { .compatible = "apple,t8103-aic", + .data = &aic1_local_fipi_info, + }, + { + .compatible = "apple,t8015-aic", .data = &aic1_fipi_info, }, { @@ -390,15 +409,18 @@ static void __exception_irq_entry aic_handle_irq(struct pt_regs *regs) * in use, and be cleared when coming back from the handler. */ if (is_kernel_in_hyp_mode() && - (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && + (read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EL2_En) && read_sysreg_s(SYS_ICH_MISR_EL2) != 0) { + u64 val; + generic_handle_domain_irq(aic_irqc->hw_domain, AIC_FIQ_HWIRQ(AIC_VGIC_MI)); - if (unlikely((read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EN) && - read_sysreg_s(SYS_ICH_MISR_EL2))) { - pr_err_ratelimited("vGIC IRQ fired and not handled by KVM, disabling.\n"); - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + if (unlikely((read_sysreg_s(SYS_ICH_HCR_EL2) & ICH_HCR_EL2_En) && + (val = read_sysreg_s(SYS_ICH_MISR_EL2)))) { + pr_err_ratelimited("vGIC IRQ fired and not handled by KVM (MISR=%llx), disabling.\n", + val); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EL2_En, 0); } } } @@ -532,14 +554,9 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) * we check for everything here, even things we don't support yet. */ - if (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING) { - if (static_branch_likely(&use_fast_ipi)) { - aic_handle_ipi(regs); - } else { - pr_err_ratelimited("Fast IPI fired. Acking.\n"); - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); - } - } + if (static_branch_likely(&use_fast_ipi) && + (read_sysreg_s(SYS_IMP_APL_IPI_SR_EL1) & IPI_SR_PENDING)) + aic_handle_ipi(regs); if (TIMER_FIRING(read_sysreg(cntp_ctl_el0))) generic_handle_domain_irq(aic_irqc->hw_domain, @@ -563,19 +580,14 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) AIC_FIQ_HWIRQ(AIC_TMR_EL02_VIRT)); } - if (read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & PMCR0_IACT) { - int irq; - if (cpumask_test_cpu(smp_processor_id(), - &aic_irqc->fiq_aff[AIC_CPU_PMU_P]->aff)) - irq = AIC_CPU_PMU_P; - else - irq = AIC_CPU_PMU_E; + if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) generic_handle_domain_irq(aic_irqc->hw_domain, - AIC_FIQ_HWIRQ(irq)); - } + AIC_FIQ_HWIRQ(AIC_CPU_PMU_P)); - if (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ && - (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { + if (static_branch_likely(&use_fast_ipi) && + (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ) && + (read_sysreg_s(SYS_IMP_APL_UPMSR_EL1) & UPMSR_IACT)) { /* Same story with uncore PMCs */ pr_err_ratelimited("Uncore PMC FIQ fired. Masking.\n"); sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, @@ -616,18 +628,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, handle_fasteoi_irq, NULL, NULL); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); } else { - int fiq = FIELD_GET(AIC_EVENT_NUM, hw); - - switch (fiq) { - case AIC_CPU_PMU_P: - case AIC_CPU_PMU_E: - irq_set_percpu_devid_partition(irq, &ic->fiq_aff[fiq]->aff); - break; - default: - irq_set_percpu_devid(irq); - break; - } - + irq_set_percpu_devid(irq); irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, handle_percpu_devid_irq, NULL, NULL); } @@ -635,6 +636,33 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, return 0; } +static int aic_irq_get_fwspec_info(struct irq_fwspec *fwspec, struct irq_fwspec_info *info) +{ + const struct cpumask *mask; + u32 intid; + + info->flags = 0; + info->affinity = NULL; + + if (fwspec->param[0] != AIC_FIQ) + return 0; + + if (fwspec->param_count == 3) + intid = fwspec->param[1]; + else + intid = fwspec->param[2]; + + if (aic_irqc->fiq_aff[intid]) + mask = &aic_irqc->fiq_aff[intid]->aff; + else + mask = cpu_possible_mask; + + info->affinity = mask; + info->flags = IRQ_FWSPEC_INFO_AFFINITY_VALID; + + return 0; +} + static int aic_irq_domain_translate(struct irq_domain *id, struct irq_fwspec *fwspec, unsigned long *hwirq, @@ -689,6 +717,10 @@ static int aic_irq_domain_translate(struct irq_domain *id, break; } } + + /* Merge the two PMUs on a single interrupt */ + if (*hwirq == AIC_CPU_PMU_E) + *hwirq = AIC_CPU_PMU_P; break; default: return -EINVAL; @@ -734,9 +766,10 @@ static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, } static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, + .get_fwspec_info = aic_irq_get_fwspec_info, }; /* @@ -750,12 +783,12 @@ static void aic_ipi_send_fast(int cpu) u64 cluster = MPIDR_CLUSTER(mpidr); u64 idx = MPIDR_CPU(mpidr); - if (MPIDR_CLUSTER(my_mpidr) == cluster) - write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx), - SYS_IMP_APL_IPI_RR_LOCAL_EL1); - else + if (static_branch_likely(&use_local_fast_ipi) && MPIDR_CLUSTER(my_mpidr) == cluster) { + write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx), SYS_IMP_APL_IPI_RR_LOCAL_EL1); + } else { write_sysreg_s(FIELD_PREP(IPI_RR_CPU, idx) | FIELD_PREP(IPI_RR_CLUSTER, cluster), SYS_IMP_APL_IPI_RR_GLOBAL_EL1); + } isb(); } @@ -811,7 +844,8 @@ static int aic_init_cpu(unsigned int cpu) /* Mask all hard-wired per-CPU IRQ/FIQ sources */ /* Pending Fast IPI FIQs */ - write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); + if (static_branch_likely(&use_fast_ipi)) + write_sysreg_s(IPI_SR_PENDING, SYS_IMP_APL_IPI_SR_EL1); /* Timer FIQs */ sysreg_clear_set(cntp_ctl_el0, 0, ARCH_TIMER_CTRL_IT_MASK); @@ -824,7 +858,7 @@ static int aic_init_cpu(unsigned int cpu) VM_TMR_FIQ_ENABLE_V | VM_TMR_FIQ_ENABLE_P, 0); /* vGIC maintenance IRQ */ - sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EN, 0); + sysreg_clear_set_s(SYS_ICH_HCR_EL2, ICH_HCR_EL2_En, 0); } /* PMC FIQ */ @@ -832,8 +866,10 @@ static int aic_init_cpu(unsigned int cpu) FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_OFF)); /* Uncore PMC FIQ */ - sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, - FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + if (static_branch_likely(&use_fast_ipi)) { + sysreg_clear_set_s(SYS_IMP_APL_UPMCR0_EL1, UPMCR0_IMODE, + FIELD_PREP(UPMCR0_IMODE, UPMCR0_IMODE_OFF)); + } /* Commit all of the above */ isb(); @@ -987,14 +1023,15 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p off += sizeof(u32) * (irqc->max_irq >> 5); /* MASK_CLR */ off += sizeof(u32) * (irqc->max_irq >> 5); /* HW_STATE */ - if (irqc->info.fast_ipi) - static_branch_enable(&use_fast_ipi); - else + if (!irqc->info.fast_ipi) static_branch_disable(&use_fast_ipi); + if (!irqc->info.local_fast_ipi) + static_branch_disable(&use_local_fast_ipi); + irqc->info.die_stride = off - start_off; - irqc->hw_domain = irq_domain_create_tree(of_node_to_fwnode(node), + irqc->hw_domain = irq_domain_create_tree(of_fwnode_handle(node), &aic_irq_domain_ops, irqc); if (WARN_ON(!irqc->hw_domain)) goto err_unmap; @@ -1047,7 +1084,7 @@ static int __init aic_of_ic_init(struct device_node *node, struct device_node *p if (is_kernel_in_hyp_mode()) { struct irq_fwspec mi = { - .fwnode = of_node_to_fwnode(node), + .fwnode = of_fwnode_handle(node), .param_count = 3, .param = { [0] = AIC_FIQ, /* This is a lie */ diff --git a/drivers/irqchip/irq-armada-370-xp.c b/drivers/irqchip/irq-armada-370-xp.c index a55528469278..a4d03a2d1569 100644 --- a/drivers/irqchip/irq-armada-370-xp.c +++ b/drivers/irqchip/irq-armada-370-xp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Marvell Armada 370 and Armada XP SoC IRQ handling * @@ -7,12 +8,11 @@ * Gregory CLEMENT <gregory.clement@free-electrons.com> * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> * Ben Dooks <ben.dooks@codethink.co.uk> - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ +#include <linux/bitfield.h> +#include <linux/bits.h> +#include <linux/err.h> #include <linux/kernel.h> #include <linux/module.h> #include <linux/init.h> @@ -20,6 +20,7 @@ #include <linux/interrupt.h> #include <linux/irqchip.h> #include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> #include <linux/cpu.h> #include <linux/io.h> #include <linux/of_address.h> @@ -29,6 +30,7 @@ #include <linux/slab.h> #include <linux/syscore_ops.h> #include <linux/msi.h> +#include <linux/types.h> #include <asm/mach/arch.h> #include <asm/exception.h> #include <asm/smp_plat.h> @@ -64,19 +66,17 @@ * device * * The "global interrupt mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_ENABLE_OFFS and - * ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS registers, which are relative - * to "main_int_base". + * MPIC_INT_SET_ENABLE and MPIC_INT_CLEAR_ENABLE + * registers, which are relative to "mpic->base". * - * The "per-CPU mask/unmask" is modified using the - * ARMADA_370_XP_INT_SET_MASK_OFFS and - * ARMADA_370_XP_INT_CLEAR_MASK_OFFS registers, which are relative to - * "per_cpu_int_base". This base address points to a special address, + * The "per-CPU mask/unmask" is modified using the MPIC_INT_SET_MASK + * and MPIC_INT_CLEAR_MASK registers, which are relative to + * "mpic->per_cpu". This base address points to a special address, * which automatically accesses the registers of the current CPU. * * The per-CPU mask/unmask can also be adjusted using the global - * per-interrupt ARMADA_370_XP_INT_SOURCE_CTL register, which we use - * to configure interrupt affinity. + * per-interrupt MPIC_INT_SOURCE_CTL register, which we use to + * configure interrupt affinity. * * Due to this model, all interrupts need to be mask/unmasked at two * different levels: at the global level and at the per-CPU level. @@ -90,9 +90,8 @@ * the current CPU, running the ->map() code. This allows to have * the interrupt unmasked at this level in non-SMP * configurations. In SMP configurations, the ->set_affinity() - * callback is called, which using the - * ARMADA_370_XP_INT_SOURCE_CTL() readjusts the per-CPU mask/unmask - * for the interrupt. + * callback is called, which using the MPIC_INT_SOURCE_CTL() + * readjusts the per-CPU mask/unmask for the interrupt. * * The ->mask() and ->unmask() operations only mask/unmask the * interrupt at the "global" level. @@ -114,54 +113,95 @@ * at the per-CPU level. */ -/* Registers relative to main_int_base */ -#define ARMADA_370_XP_INT_CONTROL (0x00) -#define ARMADA_370_XP_SW_TRIG_INT_OFFS (0x04) -#define ARMADA_370_XP_INT_SET_ENABLE_OFFS (0x30) -#define ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS (0x34) -#define ARMADA_370_XP_INT_SOURCE_CTL(irq) (0x100 + irq*4) -#define ARMADA_370_XP_INT_SOURCE_CPU_MASK 0xF -#define ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << cpuid) - -/* Registers relative to per_cpu_int_base */ -#define ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS (0x08) -#define ARMADA_370_XP_IN_DRBEL_MSK_OFFS (0x0c) -#define ARMADA_375_PPI_CAUSE (0x10) -#define ARMADA_370_XP_CPU_INTACK_OFFS (0x44) -#define ARMADA_370_XP_INT_SET_MASK_OFFS (0x48) -#define ARMADA_370_XP_INT_CLEAR_MASK_OFFS (0x4C) -#define ARMADA_370_XP_INT_FABRIC_MASK_OFFS (0x54) -#define ARMADA_370_XP_INT_CAUSE_PERF(cpu) (1 << cpu) - -#define ARMADA_370_XP_MAX_PER_CPU_IRQS (28) - -#define IPI_DOORBELL_START (0) -#define IPI_DOORBELL_END (8) -#define IPI_DOORBELL_MASK 0xFF -#define PCI_MSI_DOORBELL_START (16) -#define PCI_MSI_DOORBELL_NR (16) -#define PCI_MSI_DOORBELL_END (32) -#define PCI_MSI_DOORBELL_MASK 0xFFFF0000 - -static void __iomem *per_cpu_int_base; -static void __iomem *main_int_base; -static struct irq_domain *armada_370_xp_mpic_domain; -static u32 doorbell_mask_reg; -static int parent_irq; +/* Registers relative to mpic->base */ +#define MPIC_INT_CONTROL 0x00 +#define MPIC_INT_CONTROL_NUMINT_MASK GENMASK(12, 2) +#define MPIC_SW_TRIG_INT 0x04 +#define MPIC_INT_SET_ENABLE 0x30 +#define MPIC_INT_CLEAR_ENABLE 0x34 +#define MPIC_INT_SOURCE_CTL(hwirq) (0x100 + (hwirq) * 4) +#define MPIC_INT_SOURCE_CPU_MASK GENMASK(3, 0) +#define MPIC_INT_IRQ_FIQ_MASK(cpuid) ((BIT(0) | BIT(8)) << (cpuid)) + +/* Registers relative to mpic->per_cpu */ +#define MPIC_IN_DRBEL_CAUSE 0x08 +#define MPIC_IN_DRBEL_MASK 0x0c +#define MPIC_PPI_CAUSE 0x10 +#define MPIC_CPU_INTACK 0x44 +#define MPIC_CPU_INTACK_IID_MASK GENMASK(9, 0) +#define MPIC_INT_SET_MASK 0x48 +#define MPIC_INT_CLEAR_MASK 0x4C +#define MPIC_INT_FABRIC_MASK 0x54 +#define MPIC_INT_CAUSE_PERF(cpu) BIT(cpu) + +#define MPIC_PER_CPU_IRQS_NR 29 + +/* IPI and MSI interrupt definitions for IPI platforms */ +#define IPI_DOORBELL_NR 8 +#define IPI_DOORBELL_MASK GENMASK(7, 0) +#define PCI_MSI_DOORBELL_START 16 +#define PCI_MSI_DOORBELL_NR 16 +#define PCI_MSI_DOORBELL_MASK GENMASK(31, 16) + +/* MSI interrupt definitions for non-IPI platforms */ +#define PCI_MSI_FULL_DOORBELL_START 0 +#define PCI_MSI_FULL_DOORBELL_NR 32 +#define PCI_MSI_FULL_DOORBELL_MASK GENMASK(31, 0) +#define PCI_MSI_FULL_DOORBELL_SRC0_MASK GENMASK(15, 0) +#define PCI_MSI_FULL_DOORBELL_SRC1_MASK GENMASK(31, 16) + +/** + * struct mpic - MPIC private data structure + * @base: MPIC registers base address + * @per_cpu: per-CPU registers base address + * @parent_irq: parent IRQ if MPIC is not top-level interrupt controller + * @domain: MPIC main interrupt domain + * @ipi_domain: IPI domain + * @msi_inner_domain: MSI inner domain + * @msi_used: bitmap of used MSI numbers + * @msi_lock: mutex serializing access to @msi_used + * @msi_doorbell_addr: physical address of MSI doorbell register + * @msi_doorbell_mask: mask of available doorbell bits for MSIs (either PCI_MSI_DOORBELL_MASK or + * PCI_MSI_FULL_DOORBELL_MASK) + * @msi_doorbell_start: first set bit in @msi_doorbell_mask + * @msi_doorbell_size: number of set bits in @msi_doorbell_mask + * @doorbell_mask: doorbell mask of MSIs and IPIs, stored on suspend, restored on resume + */ +struct mpic { + void __iomem *base; + void __iomem *per_cpu; + int parent_irq; + struct irq_domain *domain; +#ifdef CONFIG_SMP + struct irq_domain *ipi_domain; +#endif #ifdef CONFIG_PCI_MSI -static struct irq_domain *armada_370_xp_msi_domain; -static struct irq_domain *armada_370_xp_msi_inner_domain; -static DECLARE_BITMAP(msi_used, PCI_MSI_DOORBELL_NR); -static DEFINE_MUTEX(msi_used_lock); -static phys_addr_t msi_doorbell_addr; + struct irq_domain *msi_inner_domain; + DECLARE_BITMAP(msi_used, PCI_MSI_FULL_DOORBELL_NR); + struct mutex msi_lock; + phys_addr_t msi_doorbell_addr; + u32 msi_doorbell_mask; + unsigned int msi_doorbell_start, msi_doorbell_size; #endif + u32 doorbell_mask; +}; + +static struct mpic *mpic_data __ro_after_init; -static inline bool is_percpu_irq(irq_hw_number_t irq) +static inline bool mpic_is_ipi_available(struct mpic *mpic) { - if (irq <= ARMADA_370_XP_MAX_PER_CPU_IRQS) - return true; + /* + * We distinguish IPI availability in the IC by the IC not having a + * parent irq defined. If a parent irq is defined, there is a parent + * interrupt controller (e.g. GIC) that takes care of inter-processor + * interrupts. + */ + return mpic->parent_irq <= 0; +} - return false; +static inline bool mpic_is_percpu_irq(irq_hw_number_t hwirq) +{ + return hwirq < MPIC_PER_CPU_IRQS_NR; } /* @@ -169,55 +209,41 @@ static inline bool is_percpu_irq(irq_hw_number_t irq) * For shared global interrupts, mask/unmask global enable bit * For CPU interrupts, mask/unmask the calling CPU's bit */ -static void armada_370_xp_irq_mask(struct irq_data *d) +static void mpic_irq_mask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); - if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + - ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); + if (!mpic_is_percpu_irq(hwirq)) + writel(hwirq, mpic->base + MPIC_INT_CLEAR_ENABLE); else - writel(hwirq, per_cpu_int_base + - ARMADA_370_XP_INT_SET_MASK_OFFS); + writel(hwirq, mpic->per_cpu + MPIC_INT_SET_MASK); } -static void armada_370_xp_irq_unmask(struct irq_data *d) +static void mpic_irq_unmask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); - if (!is_percpu_irq(hwirq)) - writel(hwirq, main_int_base + - ARMADA_370_XP_INT_SET_ENABLE_OFFS); + if (!mpic_is_percpu_irq(hwirq)) + writel(hwirq, mpic->base + MPIC_INT_SET_ENABLE); else - writel(hwirq, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(hwirq, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } #ifdef CONFIG_PCI_MSI -static struct irq_chip armada_370_xp_msi_irq_chip = { - .name = "MPIC MSI", - .irq_mask = pci_msi_mask_irq, - .irq_unmask = pci_msi_unmask_irq, -}; - -static struct msi_domain_info armada_370_xp_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), - .chip = &armada_370_xp_msi_irq_chip, -}; - -static void armada_370_xp_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) +static void mpic_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { - unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(data)); + unsigned int cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); + struct mpic *mpic = irq_data_get_irq_chip_data(d); - msg->address_lo = lower_32_bits(msi_doorbell_addr); - msg->address_hi = upper_32_bits(msi_doorbell_addr); - msg->data = BIT(cpu + 8) | (data->hwirq + PCI_MSI_DOORBELL_START); + msg->address_lo = lower_32_bits(mpic->msi_doorbell_addr); + msg->address_hi = upper_32_bits(mpic->msi_doorbell_addr); + msg->data = BIT(cpu + 8) | (d->hwirq + mpic->msi_doorbell_start); } -static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, - const struct cpumask *mask, bool force) +static int mpic_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force) { unsigned int cpu; @@ -229,33 +255,34 @@ static int armada_370_xp_msi_set_affinity(struct irq_data *irq_data, if (cpu >= nr_cpu_ids) return -EINVAL; - irq_data_update_effective_affinity(irq_data, cpumask_of(cpu)); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); return IRQ_SET_MASK_OK; } -static struct irq_chip armada_370_xp_msi_bottom_irq_chip = { +static struct irq_chip mpic_msi_bottom_irq_chip = { .name = "MPIC MSI", - .irq_compose_msi_msg = armada_370_xp_compose_msi_msg, - .irq_set_affinity = armada_370_xp_msi_set_affinity, + .irq_compose_msi_msg = mpic_compose_msi_msg, + .irq_set_affinity = mpic_msi_set_affinity, }; -static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *args) +static int mpic_msi_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, + void *args) { - int hwirq, i; + struct mpic *mpic = domain->host_data; + int hwirq; - mutex_lock(&msi_used_lock); - hwirq = bitmap_find_free_region(msi_used, PCI_MSI_DOORBELL_NR, + mutex_lock(&mpic->msi_lock); + hwirq = bitmap_find_free_region(mpic->msi_used, mpic->msi_doorbell_size, order_base_2(nr_irqs)); - mutex_unlock(&msi_used_lock); + mutex_unlock(&mpic->msi_lock); if (hwirq < 0) return -ENOSPC; - for (i = 0; i < nr_irqs; i++) { + for (unsigned int i = 0; i < nr_irqs; i++) { irq_domain_set_info(domain, virq + i, hwirq + i, - &armada_370_xp_msi_bottom_irq_chip, + &mpic_msi_bottom_irq_chip, domain->host_data, handle_simple_irq, NULL, NULL); } @@ -263,71 +290,99 @@ static int armada_370_xp_msi_alloc(struct irq_domain *domain, unsigned int virq, return 0; } -static void armada_370_xp_msi_free(struct irq_domain *domain, - unsigned int virq, unsigned int nr_irqs) +static void mpic_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct mpic *mpic = domain->host_data; - mutex_lock(&msi_used_lock); - bitmap_release_region(msi_used, d->hwirq, order_base_2(nr_irqs)); - mutex_unlock(&msi_used_lock); + mutex_lock(&mpic->msi_lock); + bitmap_release_region(mpic->msi_used, d->hwirq, order_base_2(nr_irqs)); + mutex_unlock(&mpic->msi_lock); } -static const struct irq_domain_ops armada_370_xp_msi_domain_ops = { - .alloc = armada_370_xp_msi_alloc, - .free = armada_370_xp_msi_free, +static const struct irq_domain_ops mpic_msi_domain_ops = { + .select = msi_lib_irq_domain_select, + .alloc = mpic_msi_alloc, + .free = mpic_msi_free, }; -static void armada_370_xp_msi_reenable_percpu(void) +static void mpic_msi_reenable_percpu(struct mpic *mpic) { u32 reg; /* Enable MSI doorbell mask and combined cpu local interrupt */ - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS) - | PCI_MSI_DOORBELL_MASK; - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); + reg |= mpic->msi_doorbell_mask; + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); + /* Unmask local doorbell interrupt */ - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } -static int armada_370_xp_msi_init(struct device_node *node, - phys_addr_t main_int_phys_base) +#define MPIC_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) +#define MPIC_MSI_FLAGS_SUPPORTED (MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX | \ + MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops mpic_msi_parent_ops = { + .required_flags = MPIC_MSI_FLAGS_REQUIRED, + .supported_flags = MPIC_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI, + .prefix = "MPIC-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int __init mpic_msi_init(struct mpic *mpic, struct device_node *node, + phys_addr_t main_int_phys_base) { - msi_doorbell_addr = main_int_phys_base + - ARMADA_370_XP_SW_TRIG_INT_OFFS; + mpic->msi_doorbell_addr = main_int_phys_base + MPIC_SW_TRIG_INT; - armada_370_xp_msi_inner_domain = - irq_domain_add_linear(NULL, PCI_MSI_DOORBELL_NR, - &armada_370_xp_msi_domain_ops, NULL); - if (!armada_370_xp_msi_inner_domain) - return -ENOMEM; + mutex_init(&mpic->msi_lock); - armada_370_xp_msi_domain = - pci_msi_create_irq_domain(of_node_to_fwnode(node), - &armada_370_xp_msi_domain_info, - armada_370_xp_msi_inner_domain); - if (!armada_370_xp_msi_domain) { - irq_domain_remove(armada_370_xp_msi_inner_domain); - return -ENOMEM; + if (mpic_is_ipi_available(mpic)) { + mpic->msi_doorbell_start = PCI_MSI_DOORBELL_START; + mpic->msi_doorbell_size = PCI_MSI_DOORBELL_NR; + mpic->msi_doorbell_mask = PCI_MSI_DOORBELL_MASK; + } else { + mpic->msi_doorbell_start = PCI_MSI_FULL_DOORBELL_START; + mpic->msi_doorbell_size = PCI_MSI_FULL_DOORBELL_NR; + mpic->msi_doorbell_mask = PCI_MSI_FULL_DOORBELL_MASK; } - armada_370_xp_msi_reenable_percpu(); + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &mpic_msi_domain_ops, + .host_data = mpic, + .size = mpic->msi_doorbell_size, + }; + + mpic->msi_inner_domain = msi_create_parent_irq_domain(&info, &mpic_msi_parent_ops); + if (!mpic->msi_inner_domain) + return -ENOMEM; + + mpic_msi_reenable_percpu(mpic); + + /* Unmask low 16 MSI irqs on non-IPI platforms */ + if (!mpic_is_ipi_available(mpic)) + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); return 0; } #else -static void armada_370_xp_msi_reenable_percpu(void) {} +static __maybe_unused void mpic_msi_reenable_percpu(struct mpic *mpic) {} -static inline int armada_370_xp_msi_init(struct device_node *node, - phys_addr_t main_int_phys_base) +static inline int mpic_msi_init(struct mpic *mpic, struct device_node *node, + phys_addr_t main_int_phys_base) { return 0; } #endif -static void armada_xp_mpic_perf_init(void) +static void mpic_perf_init(struct mpic *mpic) { - unsigned long cpuid; + u32 cpuid; /* * This Performance Counter Overflow interrupt is specific for @@ -339,38 +394,39 @@ static void armada_xp_mpic_perf_init(void) cpuid = cpu_logical_map(smp_processor_id()); /* Enable Performance Counter Overflow interrupts */ - writel(ARMADA_370_XP_INT_CAUSE_PERF(cpuid), - per_cpu_int_base + ARMADA_370_XP_INT_FABRIC_MASK_OFFS); + writel(MPIC_INT_CAUSE_PERF(cpuid), mpic->per_cpu + MPIC_INT_FABRIC_MASK); } #ifdef CONFIG_SMP -static struct irq_domain *ipi_domain; - -static void armada_370_xp_ipi_mask(struct irq_data *d) +static void mpic_ipi_mask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); reg &= ~BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); } -static void armada_370_xp_ipi_unmask(struct irq_data *d) +static void mpic_ipi_unmask(struct irq_data *d) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); u32 reg; - reg = readl(per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + + reg = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); reg |= BIT(d->hwirq); - writel(reg, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(reg, mpic->per_cpu + MPIC_IN_DRBEL_MASK); } -static void armada_370_xp_ipi_send_mask(struct irq_data *d, - const struct cpumask *mask) +static void mpic_ipi_send_mask(struct irq_data *d, const struct cpumask *mask) { - unsigned long map = 0; - int cpu; + struct mpic *mpic = irq_data_get_irq_chip_data(d); + unsigned int cpu; + u32 map = 0; /* Convert our logical CPU mask into a physical one. */ for_each_cpu(cpu, mask) - map |= 1 << cpu_logical_map(cpu); + map |= BIT(cpu_logical_map(cpu)); /* * Ensure that stores to Normal memory are visible to the @@ -379,433 +435,469 @@ static void armada_370_xp_ipi_send_mask(struct irq_data *d, dsb(); /* submit softirq */ - writel((map << 8) | d->hwirq, main_int_base + - ARMADA_370_XP_SW_TRIG_INT_OFFS); + writel((map << 8) | d->hwirq, mpic->base + MPIC_SW_TRIG_INT); } -static void armada_370_xp_ipi_ack(struct irq_data *d) +static void mpic_ipi_ack(struct irq_data *d) { - writel(~BIT(d->hwirq), per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + struct mpic *mpic = irq_data_get_irq_chip_data(d); + + writel(~BIT(d->hwirq), mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); } -static struct irq_chip ipi_irqchip = { +static struct irq_chip mpic_ipi_irqchip = { .name = "IPI", - .irq_ack = armada_370_xp_ipi_ack, - .irq_mask = armada_370_xp_ipi_mask, - .irq_unmask = armada_370_xp_ipi_unmask, - .ipi_send_mask = armada_370_xp_ipi_send_mask, + .irq_ack = mpic_ipi_ack, + .irq_mask = mpic_ipi_mask, + .irq_unmask = mpic_ipi_unmask, + .ipi_send_mask = mpic_ipi_send_mask, }; -static int armada_370_xp_ipi_alloc(struct irq_domain *d, - unsigned int virq, - unsigned int nr_irqs, void *args) +static int mpic_ipi_alloc(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs, void *args) { - int i; - - for (i = 0; i < nr_irqs; i++) { + for (unsigned int i = 0; i < nr_irqs; i++) { irq_set_percpu_devid(virq + i); - irq_domain_set_info(d, virq + i, i, &ipi_irqchip, - d->host_data, - handle_percpu_devid_irq, - NULL, NULL); + irq_domain_set_info(d, virq + i, i, &mpic_ipi_irqchip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); } return 0; } -static void armada_370_xp_ipi_free(struct irq_domain *d, - unsigned int virq, - unsigned int nr_irqs) +static void mpic_ipi_free(struct irq_domain *d, unsigned int virq, + unsigned int nr_irqs) { /* Not freeing IPIs */ } -static const struct irq_domain_ops ipi_domain_ops = { - .alloc = armada_370_xp_ipi_alloc, - .free = armada_370_xp_ipi_free, +static const struct irq_domain_ops mpic_ipi_domain_ops = { + .alloc = mpic_ipi_alloc, + .free = mpic_ipi_free, }; -static void ipi_resume(void) +static void mpic_ipi_resume(struct mpic *mpic) { - int i; - - for (i = 0; i < IPI_DOORBELL_END; i++) { - int irq; + for (irq_hw_number_t i = 0; i < IPI_DOORBELL_NR; i++) { + unsigned int virq = irq_find_mapping(mpic->ipi_domain, i); + struct irq_data *d; - irq = irq_find_mapping(ipi_domain, i); - if (irq <= 0) + if (!virq || !irq_percpu_is_enabled(virq)) continue; - if (irq_percpu_is_enabled(irq)) { - struct irq_data *d; - d = irq_domain_get_irq_data(ipi_domain, irq); - armada_370_xp_ipi_unmask(d); - } + + d = irq_domain_get_irq_data(mpic->ipi_domain, virq); + mpic_ipi_unmask(d); } } -static __init void armada_xp_ipi_init(struct device_node *node) +static int __init mpic_ipi_init(struct mpic *mpic, struct device_node *node) { int base_ipi; - ipi_domain = irq_domain_create_linear(of_node_to_fwnode(node), - IPI_DOORBELL_END, - &ipi_domain_ops, NULL); - if (WARN_ON(!ipi_domain)) - return; + mpic->ipi_domain = irq_domain_create_linear(of_fwnode_handle(node), IPI_DOORBELL_NR, + &mpic_ipi_domain_ops, mpic); + if (WARN_ON(!mpic->ipi_domain)) + return -ENOMEM; - irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); - base_ipi = irq_domain_alloc_irqs(ipi_domain, IPI_DOORBELL_END, NUMA_NO_NODE, NULL); + irq_domain_update_bus_token(mpic->ipi_domain, DOMAIN_BUS_IPI); + base_ipi = irq_domain_alloc_irqs(mpic->ipi_domain, IPI_DOORBELL_NR, NUMA_NO_NODE, NULL); if (WARN_ON(!base_ipi)) - return; + return -ENOMEM; - set_smp_ipi_range(base_ipi, IPI_DOORBELL_END); -} + set_smp_ipi_range(base_ipi, IPI_DOORBELL_NR); -static DEFINE_RAW_SPINLOCK(irq_controller_lock); + return 0; +} -static int armada_xp_set_affinity(struct irq_data *d, - const struct cpumask *mask_val, bool force) +static int mpic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { + struct mpic *mpic = irq_data_get_irq_chip_data(d); irq_hw_number_t hwirq = irqd_to_hwirq(d); - unsigned long reg, mask; - int cpu; + unsigned int cpu; /* Select a single core from the affinity mask which is online */ cpu = cpumask_any_and(mask_val, cpu_online_mask); - mask = 1UL << cpu_logical_map(cpu); - raw_spin_lock(&irq_controller_lock); - reg = readl(main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); - reg = (reg & (~ARMADA_370_XP_INT_SOURCE_CPU_MASK)) | mask; - writel(reg, main_int_base + ARMADA_370_XP_INT_SOURCE_CTL(hwirq)); - raw_spin_unlock(&irq_controller_lock); + atomic_io_modify(mpic->base + MPIC_INT_SOURCE_CTL(hwirq), + MPIC_INT_SOURCE_CPU_MASK, BIT(cpu_logical_map(cpu))); irq_data_update_effective_affinity(d, cpumask_of(cpu)); return IRQ_SET_MASK_OK; } -static void armada_xp_mpic_smp_cpu_init(void) +static void mpic_smp_cpu_init(struct mpic *mpic) { - u32 control; - int nr_irqs, i; + for (irq_hw_number_t i = 0; i < mpic->domain->hwirq_max; i++) + writel(i, mpic->per_cpu + MPIC_INT_SET_MASK); - control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); - nr_irqs = (control >> 2) & 0x3ff; - - for (i = 0; i < nr_irqs; i++) - writel(i, per_cpu_int_base + ARMADA_370_XP_INT_SET_MASK_OFFS); + if (!mpic_is_ipi_available(mpic)) + return; /* Disable all IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + writel(0, mpic->per_cpu + MPIC_IN_DRBEL_MASK); /* Clear pending IPIs */ - writel(0, per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); + writel(0, mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); /* Unmask IPI interrupt */ - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); } -static void armada_xp_mpic_reenable_percpu(void) +static void mpic_reenable_percpu(struct mpic *mpic) { - unsigned int irq; - /* Re-enable per-CPU interrupts that were enabled before suspend */ - for (irq = 0; irq < ARMADA_370_XP_MAX_PER_CPU_IRQS; irq++) { - struct irq_data *data; - int virq; + for (irq_hw_number_t i = 0; i < MPIC_PER_CPU_IRQS_NR; i++) { + unsigned int virq = irq_find_mapping(mpic->domain, i); + struct irq_data *d; - virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); - if (virq == 0) + if (!virq || !irq_percpu_is_enabled(virq)) continue; - data = irq_get_irq_data(virq); - - if (!irq_percpu_is_enabled(virq)) - continue; - - armada_370_xp_irq_unmask(data); + d = irq_get_irq_data(virq); + mpic_irq_unmask(d); } - ipi_resume(); + if (mpic_is_ipi_available(mpic)) + mpic_ipi_resume(mpic); - armada_370_xp_msi_reenable_percpu(); + mpic_msi_reenable_percpu(mpic); } -static int armada_xp_mpic_starting_cpu(unsigned int cpu) +static int mpic_starting_cpu(unsigned int cpu) { - armada_xp_mpic_perf_init(); - armada_xp_mpic_smp_cpu_init(); - armada_xp_mpic_reenable_percpu(); + struct mpic *mpic = irq_get_default_domain()->host_data; + + mpic_perf_init(mpic); + mpic_smp_cpu_init(mpic); + mpic_reenable_percpu(mpic); + return 0; } static int mpic_cascaded_starting_cpu(unsigned int cpu) { - armada_xp_mpic_perf_init(); - armada_xp_mpic_reenable_percpu(); - enable_percpu_irq(parent_irq, IRQ_TYPE_NONE); + struct mpic *mpic = mpic_data; + + mpic_perf_init(mpic); + mpic_reenable_percpu(mpic); + enable_percpu_irq(mpic->parent_irq, IRQ_TYPE_NONE); + return 0; } #else -static void armada_xp_mpic_smp_cpu_init(void) {} -static void ipi_resume(void) {} +static void mpic_smp_cpu_init(struct mpic *mpic) {} +static void mpic_ipi_resume(struct mpic *mpic) {} #endif -static struct irq_chip armada_370_xp_irq_chip = { +static struct irq_chip mpic_irq_chip = { .name = "MPIC", - .irq_mask = armada_370_xp_irq_mask, - .irq_mask_ack = armada_370_xp_irq_mask, - .irq_unmask = armada_370_xp_irq_unmask, + .irq_mask = mpic_irq_mask, + .irq_mask_ack = mpic_irq_mask, + .irq_unmask = mpic_irq_unmask, #ifdef CONFIG_SMP - .irq_set_affinity = armada_xp_set_affinity, + .irq_set_affinity = mpic_set_affinity, #endif .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, }; -static int armada_370_xp_mpic_irq_map(struct irq_domain *h, - unsigned int virq, irq_hw_number_t hw) +static int mpic_irq_map(struct irq_domain *domain, unsigned int virq, irq_hw_number_t hwirq) { - armada_370_xp_irq_mask(irq_get_irq_data(virq)); - if (!is_percpu_irq(hw)) - writel(hw, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + struct mpic *mpic = domain->host_data; + + /* IRQs 0 and 1 cannot be mapped, they are handled internally */ + if (hwirq <= 1) + return -EINVAL; + + irq_set_chip_data(virq, mpic); + + mpic_irq_mask(irq_get_irq_data(virq)); + if (!mpic_is_percpu_irq(hwirq)) + writel(hwirq, mpic->per_cpu + MPIC_INT_CLEAR_MASK); else - writel(hw, main_int_base + ARMADA_370_XP_INT_SET_ENABLE_OFFS); + writel(hwirq, mpic->base + MPIC_INT_SET_ENABLE); irq_set_status_flags(virq, IRQ_LEVEL); - if (is_percpu_irq(hw)) { + if (mpic_is_percpu_irq(hwirq)) { irq_set_percpu_devid(virq); - irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, - handle_percpu_devid_irq); + irq_set_chip_and_handler(virq, &mpic_irq_chip, handle_percpu_devid_irq); } else { - irq_set_chip_and_handler(virq, &armada_370_xp_irq_chip, - handle_level_irq); + irq_set_chip_and_handler(virq, &mpic_irq_chip, handle_level_irq); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq))); } irq_set_probe(virq); - return 0; } -static const struct irq_domain_ops armada_370_xp_mpic_irq_ops = { - .map = armada_370_xp_mpic_irq_map, - .xlate = irq_domain_xlate_onecell, +static const struct irq_domain_ops mpic_irq_ops = { + .map = mpic_irq_map, + .xlate = irq_domain_xlate_onecell, }; #ifdef CONFIG_PCI_MSI -static void armada_370_xp_handle_msi_irq(struct pt_regs *regs, bool is_chained) +static void mpic_handle_msi_irq(struct mpic *mpic) { - u32 msimask, msinr; + unsigned long cause; + unsigned int i; - msimask = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) - & PCI_MSI_DOORBELL_MASK; + cause = readl_relaxed(mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); + cause &= mpic->msi_doorbell_mask; + writel(~cause, mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); - writel(~msimask, per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS); - - for (msinr = PCI_MSI_DOORBELL_START; - msinr < PCI_MSI_DOORBELL_END; msinr++) { - unsigned int irq; + for_each_set_bit(i, &cause, BITS_PER_LONG) + generic_handle_domain_irq(mpic->msi_inner_domain, i - mpic->msi_doorbell_start); +} +#else +static void mpic_handle_msi_irq(struct mpic *mpic) {} +#endif - if (!(msimask & BIT(msinr))) - continue; +#ifdef CONFIG_SMP +static void mpic_handle_ipi_irq(struct mpic *mpic) +{ + unsigned long cause; + irq_hw_number_t i; - irq = msinr - PCI_MSI_DOORBELL_START; + cause = readl_relaxed(mpic->per_cpu + MPIC_IN_DRBEL_CAUSE); + cause &= IPI_DOORBELL_MASK; - generic_handle_domain_irq(armada_370_xp_msi_inner_domain, irq); - } + for_each_set_bit(i, &cause, IPI_DOORBELL_NR) + generic_handle_domain_irq(mpic->ipi_domain, i); } #else -static void armada_370_xp_handle_msi_irq(struct pt_regs *r, bool b) {} +static inline void mpic_handle_ipi_irq(struct mpic *mpic) {} #endif -static void armada_370_xp_mpic_handle_cascade_irq(struct irq_desc *desc) +static void mpic_handle_cascade_irq(struct irq_desc *desc) { + struct mpic *mpic = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned long irqmap, irqn, irqsrc, cpuid; + unsigned long cause; + u32 irqsrc, cpuid; + irq_hw_number_t i; chained_irq_enter(chip, desc); - irqmap = readl_relaxed(per_cpu_int_base + ARMADA_375_PPI_CAUSE); + cause = readl_relaxed(mpic->per_cpu + MPIC_PPI_CAUSE); cpuid = cpu_logical_map(smp_processor_id()); - for_each_set_bit(irqn, &irqmap, BITS_PER_LONG) { - irqsrc = readl_relaxed(main_int_base + - ARMADA_370_XP_INT_SOURCE_CTL(irqn)); + for_each_set_bit(i, &cause, MPIC_PER_CPU_IRQS_NR) { + irqsrc = readl_relaxed(mpic->base + MPIC_INT_SOURCE_CTL(i)); /* Check if the interrupt is not masked on current CPU. * Test IRQ (0-1) and FIQ (8-9) mask bits. */ - if (!(irqsrc & ARMADA_370_XP_INT_IRQ_FIQ_MASK(cpuid))) + if (!(irqsrc & MPIC_INT_IRQ_FIQ_MASK(cpuid))) continue; - if (irqn == 1) { - armada_370_xp_handle_msi_irq(NULL, true); + if (i == 0 || i == 1) { + mpic_handle_msi_irq(mpic); continue; } - generic_handle_domain_irq(armada_370_xp_mpic_domain, irqn); + generic_handle_domain_irq(mpic->domain, i); } chained_irq_exit(chip, desc); } -static void __exception_irq_entry -armada_370_xp_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry mpic_handle_irq(struct pt_regs *regs) { - u32 irqstat, irqnr; + struct mpic *mpic = irq_get_default_domain()->host_data; + irq_hw_number_t i; + u32 irqstat; do { - irqstat = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_CPU_INTACK_OFFS); - irqnr = irqstat & 0x3FF; + irqstat = readl_relaxed(mpic->per_cpu + MPIC_CPU_INTACK); + i = FIELD_GET(MPIC_CPU_INTACK_IID_MASK, irqstat); - if (irqnr > 1022) + if (i > 1022) break; - if (irqnr > 1) { - generic_handle_domain_irq(armada_370_xp_mpic_domain, - irqnr); - continue; - } + if (i > 1) + generic_handle_domain_irq(mpic->domain, i); /* MSI handling */ - if (irqnr == 1) - armada_370_xp_handle_msi_irq(regs, false); + if (i == 1) + mpic_handle_msi_irq(mpic); -#ifdef CONFIG_SMP /* IPI Handling */ - if (irqnr == 0) { - unsigned long ipimask; - int ipi; - - ipimask = readl_relaxed(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_CAUSE_OFFS) - & IPI_DOORBELL_MASK; - - for_each_set_bit(ipi, &ipimask, IPI_DOORBELL_END) - generic_handle_domain_irq(ipi_domain, ipi); - } -#endif - + if (i == 0) + mpic_handle_ipi_irq(mpic); } while (1); } -static int armada_370_xp_mpic_suspend(void) +static int mpic_suspend(void *data) { - doorbell_mask_reg = readl(per_cpu_int_base + - ARMADA_370_XP_IN_DRBEL_MSK_OFFS); + struct mpic *mpic = mpic_data; + + mpic->doorbell_mask = readl(mpic->per_cpu + MPIC_IN_DRBEL_MASK); + return 0; } -static void armada_370_xp_mpic_resume(void) +static void mpic_resume(void *data) { - int nirqs; - irq_hw_number_t irq; + struct mpic *mpic = mpic_data; + bool src0, src1; /* Re-enable interrupts */ - nirqs = (readl(main_int_base + ARMADA_370_XP_INT_CONTROL) >> 2) & 0x3ff; - for (irq = 0; irq < nirqs; irq++) { - struct irq_data *data; - int virq; + for (irq_hw_number_t i = 0; i < mpic->domain->hwirq_max; i++) { + unsigned int virq = irq_find_mapping(mpic->domain, i); + struct irq_data *d; - virq = irq_linear_revmap(armada_370_xp_mpic_domain, irq); - if (virq == 0) + if (!virq) continue; - data = irq_get_irq_data(virq); + d = irq_get_irq_data(virq); - if (!is_percpu_irq(irq)) { + if (!mpic_is_percpu_irq(i)) { /* Non per-CPU interrupts */ - writel(irq, per_cpu_int_base + - ARMADA_370_XP_INT_CLEAR_MASK_OFFS); - if (!irqd_irq_disabled(data)) - armada_370_xp_irq_unmask(data); + writel(i, mpic->per_cpu + MPIC_INT_CLEAR_MASK); + if (!irqd_irq_disabled(d)) + mpic_irq_unmask(d); } else { /* Per-CPU interrupts */ - writel(irq, main_int_base + - ARMADA_370_XP_INT_SET_ENABLE_OFFS); + writel(i, mpic->base + MPIC_INT_SET_ENABLE); /* - * Re-enable on the current CPU, - * armada_xp_mpic_reenable_percpu() will take - * care of secondary CPUs when they come up. + * Re-enable on the current CPU, mpic_reenable_percpu() + * will take care of secondary CPUs when they come up. */ if (irq_percpu_is_enabled(virq)) - armada_370_xp_irq_unmask(data); + mpic_irq_unmask(d); } } /* Reconfigure doorbells for IPIs and MSIs */ - writel(doorbell_mask_reg, - per_cpu_int_base + ARMADA_370_XP_IN_DRBEL_MSK_OFFS); - if (doorbell_mask_reg & IPI_DOORBELL_MASK) - writel(0, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); - if (doorbell_mask_reg & PCI_MSI_DOORBELL_MASK) - writel(1, per_cpu_int_base + ARMADA_370_XP_INT_CLEAR_MASK_OFFS); + writel(mpic->doorbell_mask, mpic->per_cpu + MPIC_IN_DRBEL_MASK); + + if (mpic_is_ipi_available(mpic)) { + src0 = mpic->doorbell_mask & IPI_DOORBELL_MASK; + src1 = mpic->doorbell_mask & PCI_MSI_DOORBELL_MASK; + } else { + src0 = mpic->doorbell_mask & PCI_MSI_FULL_DOORBELL_SRC0_MASK; + src1 = mpic->doorbell_mask & PCI_MSI_FULL_DOORBELL_SRC1_MASK; + } + + if (src0) + writel(0, mpic->per_cpu + MPIC_INT_CLEAR_MASK); + if (src1) + writel(1, mpic->per_cpu + MPIC_INT_CLEAR_MASK); - ipi_resume(); + if (mpic_is_ipi_available(mpic)) + mpic_ipi_resume(mpic); } -static struct syscore_ops armada_370_xp_mpic_syscore_ops = { - .suspend = armada_370_xp_mpic_suspend, - .resume = armada_370_xp_mpic_resume, +static const struct syscore_ops mpic_syscore_ops = { + .suspend = mpic_suspend, + .resume = mpic_resume, }; -static int __init armada_370_xp_mpic_of_init(struct device_node *node, - struct device_node *parent) +static struct syscore mpic_syscore = { + .ops = &mpic_syscore_ops, +}; + +static int __init mpic_map_region(struct device_node *np, int index, + void __iomem **base, phys_addr_t *phys_base) +{ + struct resource res; + int err; + + err = of_address_to_resource(np, index, &res); + if (WARN_ON(err)) + goto fail; + + if (WARN_ON(!request_mem_region(res.start, resource_size(&res), np->full_name))) { + err = -EBUSY; + goto fail; + } + + *base = ioremap(res.start, resource_size(&res)); + if (WARN_ON(!*base)) { + err = -ENOMEM; + goto fail; + } + + if (phys_base) + *phys_base = res.start; + + return 0; + +fail: + pr_err("%pOF: Unable to map resource %d: %pE\n", np, index, ERR_PTR(err)); + return err; +} + +static int __init mpic_of_init(struct device_node *node, struct device_node *parent) { - struct resource main_int_res, per_cpu_int_res; - int nr_irqs, i; - u32 control; + phys_addr_t phys_base; + unsigned int nr_irqs; + struct mpic *mpic; + int err; - BUG_ON(of_address_to_resource(node, 0, &main_int_res)); - BUG_ON(of_address_to_resource(node, 1, &per_cpu_int_res)); + mpic = kzalloc(sizeof(*mpic), GFP_KERNEL); + if (WARN_ON(!mpic)) + return -ENOMEM; - BUG_ON(!request_mem_region(main_int_res.start, - resource_size(&main_int_res), - node->full_name)); - BUG_ON(!request_mem_region(per_cpu_int_res.start, - resource_size(&per_cpu_int_res), - node->full_name)); + mpic_data = mpic; - main_int_base = ioremap(main_int_res.start, - resource_size(&main_int_res)); - BUG_ON(!main_int_base); + err = mpic_map_region(node, 0, &mpic->base, &phys_base); + if (err) + return err; - per_cpu_int_base = ioremap(per_cpu_int_res.start, - resource_size(&per_cpu_int_res)); - BUG_ON(!per_cpu_int_base); + err = mpic_map_region(node, 1, &mpic->per_cpu, NULL); + if (err) + return err; - control = readl(main_int_base + ARMADA_370_XP_INT_CONTROL); - nr_irqs = (control >> 2) & 0x3ff; + nr_irqs = FIELD_GET(MPIC_INT_CONTROL_NUMINT_MASK, readl(mpic->base + MPIC_INT_CONTROL)); - for (i = 0; i < nr_irqs; i++) - writel(i, main_int_base + ARMADA_370_XP_INT_CLEAR_ENABLE_OFFS); + for (irq_hw_number_t i = 0; i < nr_irqs; i++) + writel(i, mpic->base + MPIC_INT_CLEAR_ENABLE); - armada_370_xp_mpic_domain = - irq_domain_add_linear(node, nr_irqs, - &armada_370_xp_mpic_irq_ops, NULL); - BUG_ON(!armada_370_xp_mpic_domain); - irq_domain_update_bus_token(armada_370_xp_mpic_domain, DOMAIN_BUS_WIRED); + /* + * Initialize mpic->parent_irq before calling any other functions, since + * it is used to distinguish between IPI and non-IPI platforms. + */ + mpic->parent_irq = irq_of_parse_and_map(node, 0); + + /* + * On non-IPI platforms the driver currently supports only the per-CPU + * interrupts (the first 29 interrupts). See mpic_handle_cascade_irq(). + */ + if (!mpic_is_ipi_available(mpic)) + nr_irqs = MPIC_PER_CPU_IRQS_NR; + + mpic->domain = irq_domain_create_linear(of_fwnode_handle(node), nr_irqs, &mpic_irq_ops, mpic); + if (!mpic->domain) { + pr_err("%pOF: Unable to add IRQ domain\n", node); + return -ENOMEM; + } + + irq_domain_update_bus_token(mpic->domain, DOMAIN_BUS_WIRED); /* Setup for the boot CPU */ - armada_xp_mpic_perf_init(); - armada_xp_mpic_smp_cpu_init(); + mpic_perf_init(mpic); + mpic_smp_cpu_init(mpic); - armada_370_xp_msi_init(node, main_int_res.start); + err = mpic_msi_init(mpic, node, phys_base); + if (err) { + pr_err("%pOF: Unable to initialize MSI domain\n", node); + return err; + } - parent_irq = irq_of_parse_and_map(node, 0); - if (parent_irq <= 0) { - irq_set_default_host(armada_370_xp_mpic_domain); - set_handle_irq(armada_370_xp_handle_irq); + if (mpic_is_ipi_available(mpic)) { + irq_set_default_domain(mpic->domain); + set_handle_irq(mpic_handle_irq); #ifdef CONFIG_SMP - armada_xp_ipi_init(node); + err = mpic_ipi_init(mpic, node); + if (err) { + pr_err("%pOF: Unable to initialize IPI domain\n", node); + return err; + } + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_ARMADA_XP_STARTING, "irqchip/armada/ipi:starting", - armada_xp_mpic_starting_cpu, NULL); + mpic_starting_cpu, NULL); #endif } else { #ifdef CONFIG_SMP @@ -813,13 +905,13 @@ static int __init armada_370_xp_mpic_of_init(struct device_node *node, "irqchip/armada/cascade:starting", mpic_cascaded_starting_cpu, NULL); #endif - irq_set_chained_handler(parent_irq, - armada_370_xp_mpic_handle_cascade_irq); + irq_set_chained_handler_and_data(mpic->parent_irq, + mpic_handle_cascade_irq, mpic); } - register_syscore_ops(&armada_370_xp_mpic_syscore_ops); + register_syscore(&mpic_syscore); return 0; } -IRQCHIP_DECLARE(armada_370_xp_mpic, "marvell,mpic", armada_370_xp_mpic_of_init); +IRQCHIP_DECLARE(marvell_mpic, "marvell,mpic", mpic_of_init); diff --git a/drivers/irqchip/irq-aspeed-i2c-ic.c b/drivers/irqchip/irq-aspeed-i2c-ic.c index 9c9fc3e2967e..87c1feb999ff 100644 --- a/drivers/irqchip/irq-aspeed-i2c-ic.c +++ b/drivers/irqchip/irq-aspeed-i2c-ic.c @@ -82,7 +82,7 @@ static int __init aspeed_i2c_ic_of_init(struct device_node *node, goto err_iounmap; } - i2c_ic->irq_domain = irq_domain_add_linear(node, ASPEED_I2C_IC_NUM_BUS, + i2c_ic->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), ASPEED_I2C_IC_NUM_BUS, &aspeed_i2c_ic_irq_domain_ops, NULL); if (!i2c_ic->irq_domain) { diff --git a/drivers/irqchip/irq-aspeed-intc.c b/drivers/irqchip/irq-aspeed-intc.c new file mode 100644 index 000000000000..8330221799a0 --- /dev/null +++ b/drivers/irqchip/irq-aspeed-intc.c @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Aspeed Interrupt Controller. + * + * Copyright (C) 2023 ASPEED Technology Inc. + */ + +#include <linux/bitops.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/io.h> +#include <linux/spinlock.h> + +#define INTC_INT_ENABLE_REG 0x00 +#define INTC_INT_STATUS_REG 0x04 +#define INTC_IRQS_PER_WORD 32 + +struct aspeed_intc_ic { + void __iomem *base; + raw_spinlock_t gic_lock; + raw_spinlock_t intc_lock; + struct irq_domain *irq_domain; +}; + +static void aspeed_intc_ic_irq_handler(struct irq_desc *desc) +{ + struct aspeed_intc_ic *intc_ic = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + + scoped_guard(raw_spinlock, &intc_ic->gic_lock) { + unsigned long bit, status; + + status = readl(intc_ic->base + INTC_INT_STATUS_REG); + for_each_set_bit(bit, &status, INTC_IRQS_PER_WORD) { + generic_handle_domain_irq(intc_ic->irq_domain, bit); + writel(BIT(bit), intc_ic->base + INTC_INT_STATUS_REG); + } + } + + chained_irq_exit(chip, desc); +} + +static void aspeed_intc_irq_mask(struct irq_data *data) +{ + struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data); + unsigned int mask = readl(intc_ic->base + INTC_INT_ENABLE_REG) & ~BIT(data->hwirq); + + guard(raw_spinlock)(&intc_ic->intc_lock); + writel(mask, intc_ic->base + INTC_INT_ENABLE_REG); +} + +static void aspeed_intc_irq_unmask(struct irq_data *data) +{ + struct aspeed_intc_ic *intc_ic = irq_data_get_irq_chip_data(data); + unsigned int unmask = readl(intc_ic->base + INTC_INT_ENABLE_REG) | BIT(data->hwirq); + + guard(raw_spinlock)(&intc_ic->intc_lock); + writel(unmask, intc_ic->base + INTC_INT_ENABLE_REG); +} + +static struct irq_chip aspeed_intc_chip = { + .name = "ASPEED INTC", + .irq_mask = aspeed_intc_irq_mask, + .irq_unmask = aspeed_intc_irq_unmask, +}; + +static int aspeed_intc_ic_map_irq_domain(struct irq_domain *domain, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_chip_and_handler(irq, &aspeed_intc_chip, handle_level_irq); + irq_set_chip_data(irq, domain->host_data); + + return 0; +} + +static const struct irq_domain_ops aspeed_intc_ic_irq_domain_ops = { + .map = aspeed_intc_ic_map_irq_domain, +}; + +static int __init aspeed_intc_ic_of_init(struct device_node *node, + struct device_node *parent) +{ + struct aspeed_intc_ic *intc_ic; + int irq, i, ret = 0; + + intc_ic = kzalloc(sizeof(*intc_ic), GFP_KERNEL); + if (!intc_ic) + return -ENOMEM; + + intc_ic->base = of_iomap(node, 0); + if (!intc_ic->base) { + pr_err("Failed to iomap intc_ic base\n"); + ret = -ENOMEM; + goto err_free_ic; + } + writel(0xffffffff, intc_ic->base + INTC_INT_STATUS_REG); + writel(0x0, intc_ic->base + INTC_INT_ENABLE_REG); + + intc_ic->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), INTC_IRQS_PER_WORD, + &aspeed_intc_ic_irq_domain_ops, intc_ic); + if (!intc_ic->irq_domain) { + ret = -ENOMEM; + goto err_iounmap; + } + + raw_spin_lock_init(&intc_ic->gic_lock); + raw_spin_lock_init(&intc_ic->intc_lock); + + /* Check all the irq numbers valid. If not, unmaps all the base and frees the data. */ + for (i = 0; i < of_irq_count(node); i++) { + irq = irq_of_parse_and_map(node, i); + if (!irq) { + pr_err("Failed to get irq number\n"); + ret = -EINVAL; + goto err_iounmap; + } + } + + for (i = 0; i < of_irq_count(node); i++) { + irq = irq_of_parse_and_map(node, i); + irq_set_chained_handler_and_data(irq, aspeed_intc_ic_irq_handler, intc_ic); + } + + return 0; + +err_iounmap: + iounmap(intc_ic->base); +err_free_ic: + kfree(intc_ic); + return ret; +} + +IRQCHIP_DECLARE(ast2700_intc_ic, "aspeed,ast2700-intc-ic", aspeed_intc_ic_of_init); diff --git a/drivers/irqchip/irq-aspeed-scu-ic.c b/drivers/irqchip/irq-aspeed-scu-ic.c index 94a7223e95df..bee59c8c4c93 100644 --- a/drivers/irqchip/irq-aspeed-scu-ic.c +++ b/drivers/irqchip/irq-aspeed-scu-ic.c @@ -1,61 +1,78 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* - * Aspeed AST24XX, AST25XX, and AST26XX SCU Interrupt Controller + * Aspeed AST24XX, AST25XX, AST26XX, and AST27XX SCU Interrupt Controller * Copyright 2019 IBM Corporation * * Eddie James <eajames@linux.ibm.com> */ #include <linux/bitops.h> +#include <linux/io.h> #include <linux/irq.h> #include <linux/irqchip.h> #include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> -#include <linux/mfd/syscon.h> +#include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/regmap.h> -#define ASPEED_SCU_IC_REG 0x018 -#define ASPEED_SCU_IC_SHIFT 0 -#define ASPEED_SCU_IC_ENABLE GENMASK(15, ASPEED_SCU_IC_SHIFT) -#define ASPEED_SCU_IC_NUM_IRQS 7 #define ASPEED_SCU_IC_STATUS GENMASK(28, 16) #define ASPEED_SCU_IC_STATUS_SHIFT 16 +#define AST2700_SCU_IC_STATUS GENMASK(15, 0) + +struct aspeed_scu_ic_variant { + const char *compatible; + unsigned long irq_enable; + unsigned long irq_shift; + unsigned int num_irqs; + unsigned long ier; + unsigned long isr; +}; -#define ASPEED_AST2600_SCU_IC0_REG 0x560 -#define ASPEED_AST2600_SCU_IC0_SHIFT 0 -#define ASPEED_AST2600_SCU_IC0_ENABLE \ - GENMASK(5, ASPEED_AST2600_SCU_IC0_SHIFT) -#define ASPEED_AST2600_SCU_IC0_NUM_IRQS 6 +#define SCU_VARIANT(_compat, _shift, _enable, _num, _ier, _isr) { \ + .compatible = _compat, \ + .irq_shift = _shift, \ + .irq_enable = _enable, \ + .num_irqs = _num, \ + .ier = _ier, \ + .isr = _isr, \ +} -#define ASPEED_AST2600_SCU_IC1_REG 0x570 -#define ASPEED_AST2600_SCU_IC1_SHIFT 4 -#define ASPEED_AST2600_SCU_IC1_ENABLE \ - GENMASK(5, ASPEED_AST2600_SCU_IC1_SHIFT) -#define ASPEED_AST2600_SCU_IC1_NUM_IRQS 2 +static const struct aspeed_scu_ic_variant scu_ic_variants[] __initconst = { + SCU_VARIANT("aspeed,ast2400-scu-ic", 0, GENMASK(15, 0), 7, 0x00, 0x00), + SCU_VARIANT("aspeed,ast2500-scu-ic", 0, GENMASK(15, 0), 7, 0x00, 0x00), + SCU_VARIANT("aspeed,ast2600-scu-ic0", 0, GENMASK(5, 0), 6, 0x00, 0x00), + SCU_VARIANT("aspeed,ast2600-scu-ic1", 4, GENMASK(5, 4), 2, 0x00, 0x00), + SCU_VARIANT("aspeed,ast2700-scu-ic0", 0, GENMASK(3, 0), 4, 0x00, 0x04), + SCU_VARIANT("aspeed,ast2700-scu-ic1", 0, GENMASK(3, 0), 4, 0x00, 0x04), + SCU_VARIANT("aspeed,ast2700-scu-ic2", 0, GENMASK(3, 0), 4, 0x04, 0x00), + SCU_VARIANT("aspeed,ast2700-scu-ic3", 0, GENMASK(1, 0), 2, 0x04, 0x00), +}; struct aspeed_scu_ic { - unsigned long irq_enable; - unsigned long irq_shift; - unsigned int num_irqs; - unsigned int reg; - struct regmap *scu; - struct irq_domain *irq_domain; + unsigned long irq_enable; + unsigned long irq_shift; + unsigned int num_irqs; + void __iomem *base; + struct irq_domain *irq_domain; + unsigned long ier; + unsigned long isr; }; -static void aspeed_scu_ic_irq_handler(struct irq_desc *desc) +static inline bool scu_has_split_isr(struct aspeed_scu_ic *scu) +{ + return scu->ier != scu->isr; +} + +static void aspeed_scu_ic_irq_handler_combined(struct irq_desc *desc) { - unsigned int sts; - unsigned long bit; - unsigned long enabled; - unsigned long max; - unsigned long status; struct aspeed_scu_ic *scu_ic = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); - unsigned int mask = scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT; + unsigned long bit, enabled, max, status; + unsigned int sts, mask; chained_irq_enter(chip, desc); + mask = scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT; /* * The SCU IC has just one register to control its operation and read * status. The interrupt enable bits occupy the lower 16 bits of the @@ -66,7 +83,7 @@ static void aspeed_scu_ic_irq_handler(struct irq_desc *desc) * shifting the status down to get the mapping and then back up to * clear the bit. */ - regmap_read(scu_ic->scu, scu_ic->reg, &sts); + sts = readl(scu_ic->base); enabled = sts & scu_ic->irq_enable; status = (sts >> ASPEED_SCU_IC_STATUS_SHIFT) & enabled; @@ -74,43 +91,83 @@ static void aspeed_scu_ic_irq_handler(struct irq_desc *desc) max = scu_ic->num_irqs + bit; for_each_set_bit_from(bit, &status, max) { - generic_handle_domain_irq(scu_ic->irq_domain, - bit - scu_ic->irq_shift); + generic_handle_domain_irq(scu_ic->irq_domain, bit - scu_ic->irq_shift); + writel((readl(scu_ic->base) & ~mask) | BIT(bit + ASPEED_SCU_IC_STATUS_SHIFT), + scu_ic->base); + } + + chained_irq_exit(chip, desc); +} + +static void aspeed_scu_ic_irq_handler_split(struct irq_desc *desc) +{ + struct aspeed_scu_ic *scu_ic = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long bit, enabled, max, status; + unsigned int sts, mask; - regmap_write_bits(scu_ic->scu, scu_ic->reg, mask, - BIT(bit + ASPEED_SCU_IC_STATUS_SHIFT)); + chained_irq_enter(chip, desc); + + mask = scu_ic->irq_enable; + sts = readl(scu_ic->base + scu_ic->isr); + enabled = sts & scu_ic->irq_enable; + sts = readl(scu_ic->base + scu_ic->isr); + status = sts & enabled; + + bit = scu_ic->irq_shift; + max = scu_ic->num_irqs + bit; + + for_each_set_bit_from(bit, &status, max) { + generic_handle_domain_irq(scu_ic->irq_domain, bit - scu_ic->irq_shift); + /* Clear interrupt */ + writel(BIT(bit), scu_ic->base + scu_ic->isr); } chained_irq_exit(chip, desc); } -static void aspeed_scu_ic_irq_mask(struct irq_data *data) +static void aspeed_scu_ic_irq_mask_combined(struct irq_data *data) { struct aspeed_scu_ic *scu_ic = irq_data_get_irq_chip_data(data); - unsigned int mask = BIT(data->hwirq + scu_ic->irq_shift) | - (scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT); + unsigned int bit = BIT(data->hwirq + scu_ic->irq_shift); + unsigned int mask = bit | (scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT); /* * Status bits are cleared by writing 1. In order to prevent the mask * operation from clearing the status bits, they should be under the * mask and written with 0. */ - regmap_update_bits(scu_ic->scu, scu_ic->reg, mask, 0); + writel(readl(scu_ic->base) & ~mask, scu_ic->base); } -static void aspeed_scu_ic_irq_unmask(struct irq_data *data) +static void aspeed_scu_ic_irq_unmask_combined(struct irq_data *data) { struct aspeed_scu_ic *scu_ic = irq_data_get_irq_chip_data(data); unsigned int bit = BIT(data->hwirq + scu_ic->irq_shift); - unsigned int mask = bit | - (scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT); + unsigned int mask = bit | (scu_ic->irq_enable << ASPEED_SCU_IC_STATUS_SHIFT); /* * Status bits are cleared by writing 1. In order to prevent the unmask * operation from clearing the status bits, they should be under the * mask and written with 0. */ - regmap_update_bits(scu_ic->scu, scu_ic->reg, mask, bit); + writel((readl(scu_ic->base) & ~mask) | bit, scu_ic->base); +} + +static void aspeed_scu_ic_irq_mask_split(struct irq_data *data) +{ + struct aspeed_scu_ic *scu_ic = irq_data_get_irq_chip_data(data); + unsigned int mask = BIT(data->hwirq + scu_ic->irq_shift); + + writel(readl(scu_ic->base) & ~mask, scu_ic->base + scu_ic->ier); +} + +static void aspeed_scu_ic_irq_unmask_split(struct irq_data *data) +{ + struct aspeed_scu_ic *scu_ic = irq_data_get_irq_chip_data(data); + unsigned int bit = BIT(data->hwirq + scu_ic->irq_shift); + + writel(readl(scu_ic->base) | bit, scu_ic->base + scu_ic->ier); } static int aspeed_scu_ic_irq_set_affinity(struct irq_data *data, @@ -120,17 +177,29 @@ static int aspeed_scu_ic_irq_set_affinity(struct irq_data *data, return -EINVAL; } -static struct irq_chip aspeed_scu_ic_chip = { +static struct irq_chip aspeed_scu_ic_chip_combined = { .name = "aspeed-scu-ic", - .irq_mask = aspeed_scu_ic_irq_mask, - .irq_unmask = aspeed_scu_ic_irq_unmask, - .irq_set_affinity = aspeed_scu_ic_irq_set_affinity, + .irq_mask = aspeed_scu_ic_irq_mask_combined, + .irq_unmask = aspeed_scu_ic_irq_unmask_combined, + .irq_set_affinity = aspeed_scu_ic_irq_set_affinity, +}; + +static struct irq_chip aspeed_scu_ic_chip_split = { + .name = "ast2700-scu-ic", + .irq_mask = aspeed_scu_ic_irq_mask_split, + .irq_unmask = aspeed_scu_ic_irq_unmask_split, + .irq_set_affinity = aspeed_scu_ic_irq_set_affinity, }; static int aspeed_scu_ic_map(struct irq_domain *domain, unsigned int irq, irq_hw_number_t hwirq) { - irq_set_chip_and_handler(irq, &aspeed_scu_ic_chip, handle_level_irq); + struct aspeed_scu_ic *scu_ic = domain->host_data; + + if (scu_has_split_isr(scu_ic)) + irq_set_chip_and_handler(irq, &aspeed_scu_ic_chip_split, handle_level_irq); + else + irq_set_chip_and_handler(irq, &aspeed_scu_ic_chip_combined, handle_level_irq); irq_set_chip_data(irq, domain->host_data); return 0; @@ -143,21 +212,21 @@ static const struct irq_domain_ops aspeed_scu_ic_domain_ops = { static int aspeed_scu_ic_of_init_common(struct aspeed_scu_ic *scu_ic, struct device_node *node) { - int irq; - int rc = 0; + int irq, rc = 0; - if (!node->parent) { - rc = -ENODEV; + scu_ic->base = of_iomap(node, 0); + if (!scu_ic->base) { + rc = -ENOMEM; goto err; } - scu_ic->scu = syscon_node_to_regmap(node->parent); - if (IS_ERR(scu_ic->scu)) { - rc = PTR_ERR(scu_ic->scu); - goto err; + if (scu_has_split_isr(scu_ic)) { + writel(AST2700_SCU_IC_STATUS, scu_ic->base + scu_ic->isr); + writel(0, scu_ic->base + scu_ic->ier); + } else { + writel(ASPEED_SCU_IC_STATUS, scu_ic->base); + writel(0, scu_ic->base); } - regmap_write_bits(scu_ic->scu, scu_ic->reg, ASPEED_SCU_IC_STATUS, ASPEED_SCU_IC_STATUS); - regmap_write_bits(scu_ic->scu, scu_ic->reg, ASPEED_SCU_IC_ENABLE, 0); irq = irq_of_parse_and_map(node, 0); if (!irq) { @@ -165,76 +234,61 @@ static int aspeed_scu_ic_of_init_common(struct aspeed_scu_ic *scu_ic, goto err; } - scu_ic->irq_domain = irq_domain_add_linear(node, scu_ic->num_irqs, - &aspeed_scu_ic_domain_ops, - scu_ic); + scu_ic->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), scu_ic->num_irqs, + &aspeed_scu_ic_domain_ops, scu_ic); if (!scu_ic->irq_domain) { rc = -ENOMEM; goto err; } - irq_set_chained_handler_and_data(irq, aspeed_scu_ic_irq_handler, + irq_set_chained_handler_and_data(irq, scu_has_split_isr(scu_ic) ? + aspeed_scu_ic_irq_handler_split : + aspeed_scu_ic_irq_handler_combined, scu_ic); return 0; err: kfree(scu_ic); - return rc; } -static int __init aspeed_scu_ic_of_init(struct device_node *node, - struct device_node *parent) +static const struct aspeed_scu_ic_variant *aspeed_scu_ic_find_variant(struct device_node *np) { - struct aspeed_scu_ic *scu_ic = kzalloc(sizeof(*scu_ic), GFP_KERNEL); - - if (!scu_ic) - return -ENOMEM; - - scu_ic->irq_enable = ASPEED_SCU_IC_ENABLE; - scu_ic->irq_shift = ASPEED_SCU_IC_SHIFT; - scu_ic->num_irqs = ASPEED_SCU_IC_NUM_IRQS; - scu_ic->reg = ASPEED_SCU_IC_REG; - - return aspeed_scu_ic_of_init_common(scu_ic, node); + for (int i = 0; i < ARRAY_SIZE(scu_ic_variants); i++) { + if (of_device_is_compatible(np, scu_ic_variants[i].compatible)) + return &scu_ic_variants[i]; + } + return NULL; } -static int __init aspeed_ast2600_scu_ic0_of_init(struct device_node *node, - struct device_node *parent) +static int __init aspeed_scu_ic_of_init(struct device_node *node, struct device_node *parent) { - struct aspeed_scu_ic *scu_ic = kzalloc(sizeof(*scu_ic), GFP_KERNEL); + const struct aspeed_scu_ic_variant *variant; + struct aspeed_scu_ic *scu_ic; - if (!scu_ic) - return -ENOMEM; - - scu_ic->irq_enable = ASPEED_AST2600_SCU_IC0_ENABLE; - scu_ic->irq_shift = ASPEED_AST2600_SCU_IC0_SHIFT; - scu_ic->num_irqs = ASPEED_AST2600_SCU_IC0_NUM_IRQS; - scu_ic->reg = ASPEED_AST2600_SCU_IC0_REG; - - return aspeed_scu_ic_of_init_common(scu_ic, node); -} - -static int __init aspeed_ast2600_scu_ic1_of_init(struct device_node *node, - struct device_node *parent) -{ - struct aspeed_scu_ic *scu_ic = kzalloc(sizeof(*scu_ic), GFP_KERNEL); + variant = aspeed_scu_ic_find_variant(node); + if (!variant) + return -ENODEV; + scu_ic = kzalloc(sizeof(*scu_ic), GFP_KERNEL); if (!scu_ic) return -ENOMEM; - scu_ic->irq_enable = ASPEED_AST2600_SCU_IC1_ENABLE; - scu_ic->irq_shift = ASPEED_AST2600_SCU_IC1_SHIFT; - scu_ic->num_irqs = ASPEED_AST2600_SCU_IC1_NUM_IRQS; - scu_ic->reg = ASPEED_AST2600_SCU_IC1_REG; + scu_ic->irq_enable = variant->irq_enable; + scu_ic->irq_shift = variant->irq_shift; + scu_ic->num_irqs = variant->num_irqs; + scu_ic->ier = variant->ier; + scu_ic->isr = variant->isr; return aspeed_scu_ic_of_init_common(scu_ic, node); } IRQCHIP_DECLARE(ast2400_scu_ic, "aspeed,ast2400-scu-ic", aspeed_scu_ic_of_init); IRQCHIP_DECLARE(ast2500_scu_ic, "aspeed,ast2500-scu-ic", aspeed_scu_ic_of_init); -IRQCHIP_DECLARE(ast2600_scu_ic0, "aspeed,ast2600-scu-ic0", - aspeed_ast2600_scu_ic0_of_init); -IRQCHIP_DECLARE(ast2600_scu_ic1, "aspeed,ast2600-scu-ic1", - aspeed_ast2600_scu_ic1_of_init); +IRQCHIP_DECLARE(ast2600_scu_ic0, "aspeed,ast2600-scu-ic0", aspeed_scu_ic_of_init); +IRQCHIP_DECLARE(ast2600_scu_ic1, "aspeed,ast2600-scu-ic1", aspeed_scu_ic_of_init); +IRQCHIP_DECLARE(ast2700_scu_ic0, "aspeed,ast2700-scu-ic0", aspeed_scu_ic_of_init); +IRQCHIP_DECLARE(ast2700_scu_ic1, "aspeed,ast2700-scu-ic1", aspeed_scu_ic_of_init); +IRQCHIP_DECLARE(ast2700_scu_ic2, "aspeed,ast2700-scu-ic2", aspeed_scu_ic_of_init); +IRQCHIP_DECLARE(ast2700_scu_ic3, "aspeed,ast2700-scu-ic3", aspeed_scu_ic_of_init); diff --git a/drivers/irqchip/irq-aspeed-vic.c b/drivers/irqchip/irq-aspeed-vic.c index 62ccf2c0c414..9b665b5bb531 100644 --- a/drivers/irqchip/irq-aspeed-vic.c +++ b/drivers/irqchip/irq-aspeed-vic.c @@ -211,8 +211,8 @@ static int __init avic_of_init(struct device_node *node, set_handle_irq(avic_handle_irq); /* Register our domain */ - vic->dom = irq_domain_add_simple(node, NUM_IRQS, 0, - &avic_dom_ops, vic); + vic->dom = irq_domain_create_simple(of_fwnode_handle(node), NUM_IRQS, 0, + &avic_dom_ops, vic); return 0; } diff --git a/drivers/irqchip/irq-ath79-misc.c b/drivers/irqchip/irq-ath79-misc.c index 92f001a5ff8d..258b8e9a2d57 100644 --- a/drivers/irqchip/irq-ath79-misc.c +++ b/drivers/irqchip/irq-ath79-misc.c @@ -15,6 +15,8 @@ #include <linux/of_address.h> #include <linux/of_irq.h> +#include <asm/time.h> + #define AR71XX_RESET_REG_MISC_INT_STATUS 0 #define AR71XX_RESET_REG_MISC_INT_ENABLE 4 @@ -147,7 +149,7 @@ static int __init ath79_misc_intc_of_init( return -ENOMEM; } - domain = irq_domain_add_linear(node, ATH79_MISC_IRQ_COUNT, + domain = irq_domain_create_linear(of_fwnode_handle(node), ATH79_MISC_IRQ_COUNT, &misc_irq_domain_ops, base); if (!domain) { pr_err("Failed to add MISC irqdomain\n"); @@ -177,21 +179,3 @@ static int __init ar7240_misc_intc_of_init( IRQCHIP_DECLARE(ar7240_misc_intc, "qca,ar7240-misc-intc", ar7240_misc_intc_of_init); - -void __init ath79_misc_irq_init(void __iomem *regs, int irq, - int irq_base, bool is_ar71xx) -{ - struct irq_domain *domain; - - if (is_ar71xx) - ath79_misc_irq_chip.irq_mask_ack = ar71xx_misc_irq_mask; - else - ath79_misc_irq_chip.irq_ack = ar724x_misc_irq_ack; - - domain = irq_domain_add_legacy(NULL, ATH79_MISC_IRQ_COUNT, - irq_base, 0, &misc_irq_domain_ops, regs); - if (!domain) - panic("Failed to create MISC irqdomain"); - - ath79_misc_intc_domain_init(domain, irq); -} diff --git a/drivers/irqchip/irq-atmel-aic-common.c b/drivers/irqchip/irq-atmel-aic-common.c index 072bd227b6c6..e68853815c7a 100644 --- a/drivers/irqchip/irq-atmel-aic-common.c +++ b/drivers/irqchip/irq-atmel-aic-common.c @@ -111,8 +111,6 @@ static void __init aic_common_ext_irq_of_init(struct irq_domain *domain) struct device_node *node = irq_domain_get_of_node(domain); struct irq_chip_generic *gc; struct aic_chip_data *aic; - struct property *prop; - const __be32 *p; u32 hwirq; gc = irq_get_domain_generic_chip(domain, 0); @@ -120,7 +118,7 @@ static void __init aic_common_ext_irq_of_init(struct irq_domain *domain) aic = gc->private; aic->ext_irqs |= 1; - of_property_for_each_u32(node, "atmel,external-irqs", prop, p, hwirq) { + of_property_for_each_u32(node, "atmel,external-irqs", hwirq) { gc = irq_get_domain_generic_chip(domain, hwirq); if (!gc) { pr_warn("AIC: external irq %d >= %d skip it\n", @@ -189,20 +187,11 @@ void __init aic_common_rtt_irq_fixup(void) static void __init aic_common_irq_fixup(const struct of_device_id *matches) { - struct device_node *root = of_find_node_by_path("/"); - const struct of_device_id *match; + void (*fixup)(void); - if (!root) - return; - - match = of_match_node(matches, root); - - if (match) { - void (*fixup)(void) = match->data; + fixup = of_machine_get_match_data(matches); + if (fixup) fixup(); - } - - of_node_put(root); } struct irq_domain *__init aic_common_of_init(struct device_node *node, @@ -230,7 +219,7 @@ struct irq_domain *__init aic_common_of_init(struct device_node *node, goto err_iounmap; } - domain = irq_domain_add_linear(node, nchips * 32, ops, aic); + domain = irq_domain_create_linear(of_fwnode_handle(node), nchips * 32, ops, aic); if (!domain) { ret = -ENOMEM; goto err_free_aic; diff --git a/drivers/irqchip/irq-atmel-aic.c b/drivers/irqchip/irq-atmel-aic.c index 4631f6847953..1dcc52760eca 100644 --- a/drivers/irqchip/irq-atmel-aic.c +++ b/drivers/irqchip/irq-atmel-aic.c @@ -57,8 +57,7 @@ static struct irq_domain *aic_domain; -static asmlinkage void __exception_irq_entry -aic_handle(struct pt_regs *regs) +static void __exception_irq_entry aic_handle(struct pt_regs *regs) { struct irq_domain_chip_generic *dgc = aic_domain->gc; struct irq_chip_generic *gc = dgc->gc[0]; @@ -79,9 +78,8 @@ static int aic_retrigger(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); /* Enable interrupt on AIC5 */ - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, d->mask, AT91_AIC_ISCR); - irq_gc_unlock(gc); return 1; } @@ -107,30 +105,27 @@ static void aic_suspend(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, gc->mask_cache, AT91_AIC_IDCR); irq_reg_writel(gc, gc->wake_active, AT91_AIC_IECR); - irq_gc_unlock(gc); } static void aic_resume(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, gc->wake_active, AT91_AIC_IDCR); irq_reg_writel(gc, gc->mask_cache, AT91_AIC_IECR); - irq_gc_unlock(gc); } static void aic_pm_shutdown(struct irq_data *d) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, 0xffffffff, AT91_AIC_IDCR); irq_reg_writel(gc, 0xffffffff, AT91_AIC_ICCR); - irq_gc_unlock(gc); } #else #define aic_suspend NULL @@ -176,10 +171,8 @@ static int aic_irq_domain_xlate(struct irq_domain *d, { struct irq_domain_chip_generic *dgc = d->gc; struct irq_chip_generic *gc; - unsigned long flags; unsigned smr; - int idx; - int ret; + int idx, ret; if (!dgc) return -EINVAL; @@ -195,11 +188,10 @@ static int aic_irq_domain_xlate(struct irq_domain *d, gc = dgc->gc[idx]; - irq_gc_lock_irqsave(gc, flags); + guard(raw_spinlock_irqsave)(&gc->lock); smr = irq_reg_readl(gc, AT91_AIC_SMR(*out_hwirq)); aic_common_set_priority(intspec[2], &smr); irq_reg_writel(gc, smr, AT91_AIC_SMR(*out_hwirq)); - irq_gc_unlock_irqrestore(gc, flags); return ret; } diff --git a/drivers/irqchip/irq-atmel-aic5.c b/drivers/irqchip/irq-atmel-aic5.c index 145535bd7560..1f14b401f71d 100644 --- a/drivers/irqchip/irq-atmel-aic5.c +++ b/drivers/irqchip/irq-atmel-aic5.c @@ -67,8 +67,7 @@ static struct irq_domain *aic5_domain; -static asmlinkage void __exception_irq_entry -aic5_handle(struct pt_regs *regs) +static void __exception_irq_entry aic5_handle(struct pt_regs *regs) { struct irq_chip_generic *bgc = irq_get_domain_generic_chip(aic5_domain, 0); u32 irqnr; @@ -93,11 +92,10 @@ static void aic5_mask(struct irq_data *d) * Disable interrupt on AIC5. We always take the lock of the * first irq chip as all chips share the same registers. */ - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); irq_reg_writel(gc, 1, AT91_AIC5_IDCR); gc->mask_cache &= ~d->mask; - irq_gc_unlock(bgc); } static void aic5_unmask(struct irq_data *d) @@ -110,11 +108,10 @@ static void aic5_unmask(struct irq_data *d) * Enable interrupt on AIC5. We always take the lock of the * first irq chip as all chips share the same registers. */ - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); irq_reg_writel(gc, d->hwirq, AT91_AIC5_SSR); irq_reg_writel(gc, 1, AT91_AIC5_IECR); gc->mask_cache |= d->mask; - irq_gc_unlock(bgc); } static int aic5_retrigger(struct irq_data *d) @@ -123,11 +120,9 @@ static int aic5_retrigger(struct irq_data *d) struct irq_chip_generic *bgc = irq_get_domain_generic_chip(domain, 0); /* Enable interrupt on AIC5 */ - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); irq_reg_writel(bgc, d->hwirq, AT91_AIC5_SSR); irq_reg_writel(bgc, 1, AT91_AIC5_ISCR); - irq_gc_unlock(bgc); - return 1; } @@ -138,14 +133,12 @@ static int aic5_set_type(struct irq_data *d, unsigned type) unsigned int smr; int ret; - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); irq_reg_writel(bgc, d->hwirq, AT91_AIC5_SSR); smr = irq_reg_readl(bgc, AT91_AIC5_SMR); ret = aic_common_set_type(d, type, &smr); if (!ret) irq_reg_writel(bgc, smr, AT91_AIC5_SMR); - irq_gc_unlock(bgc); - return ret; } @@ -167,7 +160,7 @@ static void aic5_suspend(struct irq_data *d) smr_cache[i] = irq_reg_readl(bgc, AT91_AIC5_SMR); } - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); for (i = 0; i < dgc->irqs_per_chip; i++) { mask = 1 << i; if ((mask & gc->mask_cache) == (mask & gc->wake_active)) @@ -179,7 +172,6 @@ static void aic5_suspend(struct irq_data *d) else irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); } - irq_gc_unlock(bgc); } static void aic5_resume(struct irq_data *d) @@ -191,7 +183,7 @@ static void aic5_resume(struct irq_data *d) int i; u32 mask; - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); if (smr_cache) { irq_reg_writel(bgc, 0xffffffff, AT91_AIC5_SPU); @@ -215,7 +207,6 @@ static void aic5_resume(struct irq_data *d) else irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); } - irq_gc_unlock(bgc); } static void aic5_pm_shutdown(struct irq_data *d) @@ -226,13 +217,12 @@ static void aic5_pm_shutdown(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); int i; - irq_gc_lock(bgc); + guard(raw_spinlock)(&bgc->lock); for (i = 0; i < dgc->irqs_per_chip; i++) { irq_reg_writel(bgc, i + gc->irq_base, AT91_AIC5_SSR); irq_reg_writel(bgc, 1, AT91_AIC5_IDCR); irq_reg_writel(bgc, 1, AT91_AIC5_ICCR); } - irq_gc_unlock(bgc); } #else #define aic5_suspend NULL @@ -278,7 +268,6 @@ static int aic5_irq_domain_xlate(struct irq_domain *d, unsigned int *out_type) { struct irq_chip_generic *bgc = irq_get_domain_generic_chip(d, 0); - unsigned long flags; unsigned smr; int ret; @@ -290,13 +279,11 @@ static int aic5_irq_domain_xlate(struct irq_domain *d, if (ret) return ret; - irq_gc_lock_irqsave(bgc, flags); + guard(raw_spinlock_irqsave)(&bgc->lock); irq_reg_writel(bgc, *out_hwirq, AT91_AIC5_SSR); smr = irq_reg_readl(bgc, AT91_AIC5_SMR); aic_common_set_priority(intspec[2], &smr); irq_reg_writel(bgc, smr, AT91_AIC5_SMR); - irq_gc_unlock_irqrestore(bgc, flags); - return ret; } @@ -320,6 +307,7 @@ static const struct of_device_id aic5_irq_fixups[] __initconst = { { .compatible = "atmel,sama5d3", .data = sama5d3_aic_irq_fixup }, { .compatible = "atmel,sama5d4", .data = sama5d3_aic_irq_fixup }, { .compatible = "microchip,sam9x60", .data = sam9x60_aic_irq_fixup }, + { .compatible = "microchip,sam9x7", .data = sam9x60_aic_irq_fixup }, { /* sentinel */ }, }; @@ -406,3 +394,11 @@ static int __init sam9x60_aic5_of_init(struct device_node *node, return aic5_of_init(node, parent, NR_SAM9X60_IRQS); } IRQCHIP_DECLARE(sam9x60_aic5, "microchip,sam9x60-aic", sam9x60_aic5_of_init); + +#define NR_SAM9X7_IRQS 70 + +static int __init sam9x7_aic5_of_init(struct device_node *node, struct device_node *parent) +{ + return aic5_of_init(node, parent, NR_SAM9X7_IRQS); +} +IRQCHIP_DECLARE(sam9x7_aic5, "microchip,sam9x7-aic", sam9x7_aic5_of_init); diff --git a/drivers/irqchip/irq-bcm2712-mip.c b/drivers/irqchip/irq-bcm2712-mip.c new file mode 100644 index 000000000000..4761974ad650 --- /dev/null +++ b/drivers/irqchip/irq-bcm2712-mip.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024 Raspberry Pi Ltd., All Rights Reserved. + * Copyright (c) 2024 SUSE + */ + +#include <linux/bitmap.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/msi.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> + +#include <linux/irqchip/irq-msi-lib.h> + +#define MIP_INT_RAISE 0x00 +#define MIP_INT_CLEAR 0x10 +#define MIP_INT_CFGL_HOST 0x20 +#define MIP_INT_CFGH_HOST 0x30 +#define MIP_INT_MASKL_HOST 0x40 +#define MIP_INT_MASKH_HOST 0x50 +#define MIP_INT_MASKL_VPU 0x60 +#define MIP_INT_MASKH_VPU 0x70 +#define MIP_INT_STATUSL_HOST 0x80 +#define MIP_INT_STATUSH_HOST 0x90 +#define MIP_INT_STATUSL_VPU 0xa0 +#define MIP_INT_STATUSH_VPU 0xb0 + +/** + * struct mip_priv - MSI-X interrupt controller data + * @lock: Used to protect bitmap alloc/free + * @base: Base address of MMIO area + * @msg_addr: PCIe MSI-X address + * @msi_base: MSI base + * @num_msis: Count of MSIs + * @msi_offset: MSI offset + * @bitmap: A bitmap for hwirqs + * @parent: Parent domain (GIC) + * @dev: A device pointer + */ +struct mip_priv { + spinlock_t lock; + void __iomem *base; + u64 msg_addr; + u32 msi_base; + u32 num_msis; + u32 msi_offset; + unsigned long *bitmap; + struct irq_domain *parent; + struct device *dev; +}; + +static void mip_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct mip_priv *mip = irq_data_get_irq_chip_data(d); + + msg->address_hi = upper_32_bits(mip->msg_addr); + msg->address_lo = lower_32_bits(mip->msg_addr); + msg->data = d->hwirq; +} + +static struct irq_chip mip_middle_irq_chip = { + .name = "MIP", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_compose_msi_msg = mip_compose_msi_msg, +}; + +static int mip_alloc_hwirq(struct mip_priv *mip, unsigned int nr_irqs) +{ + guard(spinlock)(&mip->lock); + return bitmap_find_free_region(mip->bitmap, mip->num_msis, ilog2(nr_irqs)); +} + +static void mip_free_hwirq(struct mip_priv *mip, unsigned int hwirq, + unsigned int nr_irqs) +{ + guard(spinlock)(&mip->lock); + bitmap_release_region(mip->bitmap, hwirq, ilog2(nr_irqs)); +} + +static int mip_middle_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct mip_priv *mip = domain->host_data; + struct irq_fwspec fwspec = {0}; + unsigned int hwirq, i; + struct irq_data *irqd; + int irq, ret; + + irq = mip_alloc_hwirq(mip, nr_irqs); + if (irq < 0) + return irq; + + hwirq = irq + mip->msi_offset; + + fwspec.fwnode = domain->parent->fwnode; + fwspec.param_count = 3; + fwspec.param[0] = 0; + fwspec.param[1] = hwirq + mip->msi_base; + fwspec.param[2] = IRQ_TYPE_EDGE_RISING; + + ret = irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &fwspec); + if (ret) + goto err_free_hwirq; + + for (i = 0; i < nr_irqs; i++) { + irqd = irq_domain_get_irq_data(domain->parent, virq + i); + irqd->chip->irq_set_type(irqd, IRQ_TYPE_EDGE_RISING); + + ret = irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + &mip_middle_irq_chip, mip); + if (ret) + goto err_free; + + irqd = irq_get_irq_data(virq + i); + irqd_set_single_target(irqd); + irqd_set_affinity_on_activate(irqd); + } + + return 0; + +err_free: + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +err_free_hwirq: + mip_free_hwirq(mip, irq, nr_irqs); + return ret; +} + +static void mip_middle_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *irqd = irq_domain_get_irq_data(domain, virq); + struct mip_priv *mip; + unsigned int hwirq; + + if (!irqd) + return; + + mip = irq_data_get_irq_chip_data(irqd); + hwirq = irqd_to_hwirq(irqd); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); + mip_free_hwirq(mip, hwirq - mip->msi_offset, nr_irqs); +} + +static const struct irq_domain_ops mip_middle_domain_ops = { + .select = msi_lib_irq_domain_select, + .alloc = mip_middle_domain_alloc, + .free = mip_middle_domain_free, +}; + +#define MIP_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define MIP_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops mip_msi_parent_ops = { + .supported_flags = MIP_MSI_FLAGS_SUPPORTED, + .required_flags = MIP_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI | MSI_CHIP_FLAG_SET_ACK, + .bus_select_token = DOMAIN_BUS_GENERIC_MSI, + .bus_select_mask = MATCH_PCI_MSI, + .prefix = "MIP-MSI-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int mip_init_domains(struct mip_priv *mip, struct device_node *np) +{ + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(np), + .ops = &mip_middle_domain_ops, + .host_data = mip, + .size = mip->num_msis, + .parent = mip->parent, + .dev = mip->dev, + }; + + if (!msi_create_parent_irq_domain(&info, &mip_msi_parent_ops)) + return -ENOMEM; + + /* + * All MSI-X unmasked for the host, masked for the VPU, and edge-triggered. + */ + writel(0, mip->base + MIP_INT_MASKL_HOST); + writel(0, mip->base + MIP_INT_MASKH_HOST); + writel(~0, mip->base + MIP_INT_MASKL_VPU); + writel(~0, mip->base + MIP_INT_MASKH_VPU); + writel(~0, mip->base + MIP_INT_CFGL_HOST); + writel(~0, mip->base + MIP_INT_CFGH_HOST); + + return 0; +} + +static int mip_parse_dt(struct mip_priv *mip, struct device_node *np) +{ + struct of_phandle_args args; + u64 size; + int ret; + + ret = of_property_read_u32(np, "brcm,msi-offset", &mip->msi_offset); + if (ret) + mip->msi_offset = 0; + + ret = of_parse_phandle_with_args(np, "msi-ranges", "#interrupt-cells", + 0, &args); + if (ret) + return ret; + + ret = of_property_read_u32_index(np, "msi-ranges", args.args_count + 1, + &mip->num_msis); + if (ret) + goto err_put; + + ret = of_property_read_reg(np, 1, &mip->msg_addr, &size); + if (ret) + goto err_put; + + mip->msi_base = args.args[1]; + + mip->parent = irq_find_host(args.np); + if (!mip->parent) + ret = -EINVAL; + +err_put: + of_node_put(args.np); + return ret; +} + +static int mip_msi_probe(struct platform_device *pdev, struct device_node *parent) +{ + struct device_node *node = pdev->dev.of_node; + struct mip_priv *mip; + int ret; + + mip = kzalloc(sizeof(*mip), GFP_KERNEL); + if (!mip) + return -ENOMEM; + + spin_lock_init(&mip->lock); + mip->dev = &pdev->dev; + + ret = mip_parse_dt(mip, node); + if (ret) + goto err_priv; + + mip->base = of_iomap(node, 0); + if (!mip->base) { + ret = -ENXIO; + goto err_priv; + } + + mip->bitmap = bitmap_zalloc(mip->num_msis, GFP_KERNEL); + if (!mip->bitmap) { + ret = -ENOMEM; + goto err_base; + } + + ret = mip_init_domains(mip, node); + if (ret) + goto err_map; + + dev_dbg(&pdev->dev, "MIP: MSI-X count: %u, base: %u, offset: %u, msg_addr: %llx\n", + mip->num_msis, mip->msi_base, mip->msi_offset, mip->msg_addr); + + return 0; + +err_map: + bitmap_free(mip->bitmap); +err_base: + iounmap(mip->base); +err_priv: + kfree(mip); + return ret; +} + +IRQCHIP_PLATFORM_DRIVER_BEGIN(mip_msi) +IRQCHIP_MATCH("brcm,bcm2712-mip", mip_msi_probe) +IRQCHIP_PLATFORM_DRIVER_END(mip_msi) +MODULE_DESCRIPTION("Broadcom BCM2712 MSI-X interrupt controller"); +MODULE_AUTHOR("Phil Elwell <phil@raspberrypi.com>"); +MODULE_AUTHOR("Stanimir Varbanov <svarbanov@suse.de>"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-bcm2835.c b/drivers/irqchip/irq-bcm2835.c index e94e2882286c..1e384c870350 100644 --- a/drivers/irqchip/irq-bcm2835.c +++ b/drivers/irqchip/irq-bcm2835.c @@ -102,7 +102,9 @@ static void armctrl_unmask_irq(struct irq_data *d) static struct irq_chip armctrl_chip = { .name = "ARMCTRL-level", .irq_mask = armctrl_mask_irq, - .irq_unmask = armctrl_unmask_irq + .irq_unmask = armctrl_unmask_irq, + .flags = IRQCHIP_MASK_ON_SUSPEND | + IRQCHIP_SKIP_SET_WAKE, }; static int armctrl_xlate(struct irq_domain *d, struct device_node *ctrlr, @@ -142,7 +144,7 @@ static int __init armctrl_of_init(struct device_node *node, if (!base) panic("%pOF: unable to map IC registers\n", node); - intc.domain = irq_domain_add_linear(node, MAKE_HWIRQ(NR_BANKS, 0), + intc.domain = irq_domain_create_linear(of_fwnode_handle(node), MAKE_HWIRQ(NR_BANKS, 0), &armctrl_ops, NULL); if (!intc.domain) panic("%pOF: unable to create IRQ domain\n", node); diff --git a/drivers/irqchip/irq-bcm2836.c b/drivers/irqchip/irq-bcm2836.c index e5f1059b989f..fafd1f71348e 100644 --- a/drivers/irqchip/irq-bcm2836.c +++ b/drivers/irqchip/irq-bcm2836.c @@ -58,6 +58,7 @@ static struct irq_chip bcm2836_arm_irqchip_timer = { .name = "bcm2836-timer", .irq_mask = bcm2836_arm_irqchip_mask_timer_irq, .irq_unmask = bcm2836_arm_irqchip_unmask_timer_irq, + .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE, }; static void bcm2836_arm_irqchip_mask_pmu_irq(struct irq_data *d) @@ -74,6 +75,7 @@ static struct irq_chip bcm2836_arm_irqchip_pmu = { .name = "bcm2836-pmu", .irq_mask = bcm2836_arm_irqchip_mask_pmu_irq, .irq_unmask = bcm2836_arm_irqchip_unmask_pmu_irq, + .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE, }; static void bcm2836_arm_irqchip_mask_gpu_irq(struct irq_data *d) @@ -88,6 +90,7 @@ static struct irq_chip bcm2836_arm_irqchip_gpu = { .name = "bcm2836-gpu", .irq_mask = bcm2836_arm_irqchip_mask_gpu_irq, .irq_unmask = bcm2836_arm_irqchip_unmask_gpu_irq, + .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE, }; static void bcm2836_arm_irqchip_dummy_op(struct irq_data *d) @@ -322,7 +325,7 @@ static int __init bcm2836_arm_irqchip_l1_intc_of_init(struct device_node *node, bcm2835_init_local_timer_frequency(); - intc.domain = irq_domain_add_linear(node, LAST_IRQ + 1, + intc.domain = irq_domain_create_linear(of_fwnode_handle(node), LAST_IRQ + 1, &bcm2836_arm_irqchip_intc_ops, NULL); if (!intc.domain) diff --git a/drivers/irqchip/irq-bcm6345-l1.c b/drivers/irqchip/irq-bcm6345-l1.c index 9745a119d0e6..ca4e141c5bc2 100644 --- a/drivers/irqchip/irq-bcm6345-l1.c +++ b/drivers/irqchip/irq-bcm6345-l1.c @@ -192,14 +192,10 @@ static int bcm6345_l1_set_affinity(struct irq_data *d, u32 mask = BIT(d->hwirq % IRQS_PER_WORD); unsigned int old_cpu = cpu_for_irq(intc, d); unsigned int new_cpu; - struct cpumask valid; unsigned long flags; bool enabled; - if (!cpumask_and(&valid, &intc->cpumask, dest)) - return -EINVAL; - - new_cpu = cpumask_any_and(&valid, cpu_online_mask); + new_cpu = cpumask_first_and_and(&intc->cpumask, dest, cpu_online_mask); if (new_cpu >= nr_cpu_ids) return -EINVAL; @@ -242,7 +238,7 @@ static int __init bcm6345_l1_init_one(struct device_node *dn, else if (intc->n_words != n_words) return -EINVAL; - cpu = intc->cpus[idx] = kzalloc(sizeof(*cpu) + n_words * sizeof(u32), + cpu = intc->cpus[idx] = kzalloc(struct_size(cpu, enable_cache, n_words), GFP_KERNEL); if (!cpu) return -ENOMEM; @@ -320,7 +316,7 @@ static int __init bcm6345_l1_of_init(struct device_node *dn, raw_spin_lock_init(&intc->lock); - intc->domain = irq_domain_add_linear(dn, IRQS_PER_WORD * intc->n_words, + intc->domain = irq_domain_create_linear(of_fwnode_handle(dn), IRQS_PER_WORD * intc->n_words, &bcm6345_l1_domain_ops, intc); if (!intc->domain) { diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index 24ca1d656adc..45c4824be92f 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -82,12 +82,6 @@ static inline unsigned int reg_status(struct bcm7038_l1_chip *intc, return (0 * intc->n_words + word) * sizeof(u32); } -static inline unsigned int reg_mask_status(struct bcm7038_l1_chip *intc, - unsigned int word) -{ - return (1 * intc->n_words + word) * sizeof(u32); -} - static inline unsigned int reg_mask_set(struct bcm7038_l1_chip *intc, unsigned int word) { @@ -219,9 +213,8 @@ static int bcm7038_l1_set_affinity(struct irq_data *d, } #endif -static int __init bcm7038_l1_init_one(struct device_node *dn, - unsigned int idx, - struct bcm7038_l1_chip *intc) +static int bcm7038_l1_init_one(struct device_node *dn, unsigned int idx, + struct bcm7038_l1_chip *intc) { struct resource res; resource_size_t sz; @@ -249,7 +242,7 @@ static int __init bcm7038_l1_init_one(struct device_node *dn, return -EINVAL; } - cpu = intc->cpus[idx] = kzalloc(sizeof(*cpu) + n_words * sizeof(u32), + cpu = intc->cpus[idx] = kzalloc(struct_size(cpu, mask_cache, n_words), GFP_KERNEL); if (!cpu) return -ENOMEM; @@ -292,7 +285,7 @@ static int __init bcm7038_l1_init_one(struct device_node *dn, static LIST_HEAD(bcm7038_l1_intcs_list); static DEFINE_RAW_SPINLOCK(bcm7038_l1_intcs_lock); -static int bcm7038_l1_suspend(void) +static int bcm7038_l1_suspend(void *data) { struct bcm7038_l1_chip *intc; int boot_cpu, word; @@ -318,7 +311,7 @@ static int bcm7038_l1_suspend(void) return 0; } -static void bcm7038_l1_resume(void) +static void bcm7038_l1_resume(void *data) { struct bcm7038_l1_chip *intc; int boot_cpu, word; @@ -339,11 +332,15 @@ static void bcm7038_l1_resume(void) } } -static struct syscore_ops bcm7038_l1_syscore_ops = { +static const struct syscore_ops bcm7038_l1_syscore_ops = { .suspend = bcm7038_l1_suspend, .resume = bcm7038_l1_resume, }; +static struct syscore bcm7038_l1_syscore = { + .ops = &bcm7038_l1_syscore_ops, +}; + static int bcm7038_l1_set_wake(struct irq_data *d, unsigned int on) { struct bcm7038_l1_chip *intc = irq_data_get_irq_chip_data(d); @@ -395,9 +392,9 @@ static const struct irq_domain_ops bcm7038_l1_domain_ops = { .map = bcm7038_l1_map, }; -static int __init bcm7038_l1_of_init(struct device_node *dn, - struct device_node *parent) +static int bcm7038_l1_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *dn = pdev->dev.of_node; struct bcm7038_l1_chip *intc; int idx, ret; @@ -416,7 +413,7 @@ static int __init bcm7038_l1_of_init(struct device_node *dn, } } - intc->domain = irq_domain_add_linear(dn, IRQS_PER_WORD * intc->n_words, + intc->domain = irq_domain_create_linear(of_fwnode_handle(dn), IRQS_PER_WORD * intc->n_words, &bcm7038_l1_domain_ops, intc); if (!intc->domain) { @@ -431,7 +428,7 @@ static int __init bcm7038_l1_of_init(struct device_node *dn, raw_spin_unlock(&bcm7038_l1_intcs_lock); if (list_is_singular(&bcm7038_l1_intcs_list)) - register_syscore_ops(&bcm7038_l1_syscore_ops); + register_syscore(&bcm7038_l1_syscore); #endif pr_info("registered BCM7038 L1 intc (%pOF, IRQs: %d)\n", @@ -455,7 +452,7 @@ out_free: } IRQCHIP_PLATFORM_DRIVER_BEGIN(bcm7038_l1) -IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_of_init) +IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_probe) IRQCHIP_PLATFORM_DRIVER_END(bcm7038_l1) MODULE_DESCRIPTION("Broadcom STB 7038-style L1/L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index 1e9dab6e0d86..518c9d4366a5 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -63,16 +63,15 @@ static void bcm7120_l2_intc_irq_handle(struct irq_desc *desc) for (idx = 0; idx < b->n_words; idx++) { int base = idx * IRQS_PER_WORD; - struct irq_chip_generic *gc = - irq_get_domain_generic_chip(b->domain, base); + struct irq_chip_generic *gc; unsigned long pending; int hwirq; - irq_gc_lock(gc); - pending = irq_reg_readl(gc, b->stat_offset[idx]) & - gc->mask_cache & - data->irq_map_mask[idx]; - irq_gc_unlock(gc); + gc = irq_get_domain_generic_chip(b->domain, base); + scoped_guard (raw_spinlock, &gc->lock) { + pending = irq_reg_readl(gc, b->stat_offset[idx]) & gc->mask_cache & + data->irq_map_mask[idx]; + } for_each_set_bit(hwirq, &pending, IRQS_PER_WORD) generic_handle_domain_irq(b->domain, base + hwirq); @@ -86,11 +85,9 @@ static void bcm7120_l2_intc_suspend(struct irq_chip_generic *gc) struct bcm7120_l2_intc_data *b = gc->private; struct irq_chip_type *ct = gc->chip_types; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); if (b->can_wake) - irq_reg_writel(gc, gc->mask_cache | gc->wake_active, - ct->regs.mask); - irq_gc_unlock(gc); + irq_reg_writel(gc, gc->mask_cache | gc->wake_active, ct->regs.mask); } static void bcm7120_l2_intc_resume(struct irq_chip_generic *gc) @@ -98,9 +95,8 @@ static void bcm7120_l2_intc_resume(struct irq_chip_generic *gc) struct irq_chip_type *ct = gc->chip_types; /* Restore the saved mask */ - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, gc->mask_cache, ct->regs.mask); - irq_gc_unlock(gc); } static int bcm7120_l2_intc_init_one(struct device_node *dn, @@ -147,8 +143,7 @@ static int bcm7120_l2_intc_init_one(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_7120(struct device_node *dn, struct bcm7120_l2_intc_data *data) { int ret; @@ -181,8 +176,7 @@ static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_3380(struct device_node *dn, struct bcm7120_l2_intc_data *data) { unsigned int gc_idx; @@ -212,15 +206,14 @@ static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_probe(struct device_node *dn, - struct device_node *parent, +static int bcm7120_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, int (*iomap_regs_fn)(struct device_node *, - struct bcm7120_l2_intc_data *), + struct bcm7120_l2_intc_data *), const char *intc_name) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + struct device_node *dn = pdev->dev.of_node; struct bcm7120_l2_intc_data *data; - struct platform_device *pdev; struct irq_chip_generic *gc; struct irq_chip_type *ct; int ret = 0; @@ -231,14 +224,7 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, if (!data) return -ENOMEM; - pdev = of_find_device_by_node(dn); - if (!pdev) { - ret = -ENODEV; - goto out_free_data; - } - data->num_parent_irqs = platform_irq_count(pdev); - put_device(&pdev->dev); if (data->num_parent_irqs <= 0) { pr_err("invalid number of parent interrupts\n"); ret = -ENOMEM; @@ -264,7 +250,7 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, goto out_free_l1_data; } - data->domain = irq_domain_add_linear(dn, IRQS_PER_WORD * data->n_words, + data->domain = irq_domain_create_linear(of_fwnode_handle(dn), IRQS_PER_WORD * data->n_words, &irq_generic_chip_ops, NULL); if (!data->domain) { ret = -ENOMEM; @@ -338,22 +324,19 @@ out_unmap: if (data->map_base[idx]) iounmap(data->map_base[idx]); } -out_free_data: kfree(data); return ret; } -static int __init bcm7120_l2_intc_probe_7120(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_7120(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_7120, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_7120, "BCM7120 L2"); } -static int __init bcm7120_l2_intc_probe_3380(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_3380(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_3380, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_3380, "BCM3380 L2"); } diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 5559c943f03f..bb7078d6524f 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -2,7 +2,7 @@ /* * Generic Broadcom Set Top Box Level 2 Interrupt controller driver * - * Copyright (C) 2014-2017 Broadcom + * Copyright (C) 2014-2024 Broadcom */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -61,32 +61,6 @@ struct brcmstb_l2_intc_data { u32 saved_mask; /* for suspend/resume */ }; -/** - * brcmstb_l2_mask_and_ack - Mask and ack pending interrupt - * @d: irq_data - * - * Chip has separate enable/disable registers instead of a single mask - * register and pending interrupt is acknowledged by setting a bit. - * - * Note: This function is generic and could easily be added to the - * generic irqchip implementation if there ever becomes a will to do so. - * Perhaps with a name like irq_gc_mask_disable_and_ack_set(). - * - * e.g.: https://patchwork.kernel.org/patch/9831047/ - */ -static void brcmstb_l2_mask_and_ack(struct irq_data *d) -{ - struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); - struct irq_chip_type *ct = irq_data_get_chip_type(d); - u32 mask = d->mask; - - irq_gc_lock(gc); - irq_reg_writel(gc, mask, ct->regs.disable); - *ct->mask_cache &= ~mask; - irq_reg_writel(gc, mask, ct->regs.ack); - irq_gc_unlock(gc); -} - static void brcmstb_l2_intc_irq_handle(struct irq_desc *desc) { struct brcmstb_l2_intc_data *b = irq_desc_get_handler_data(desc); @@ -112,26 +86,38 @@ static void brcmstb_l2_intc_irq_handle(struct irq_desc *desc) generic_handle_domain_irq(b->domain, irq); } while (status); out: + /* Don't ack parent before all device writes are done */ + wmb(); + chained_irq_exit(chip, desc); } -static void brcmstb_l2_intc_suspend(struct irq_data *d) +static void __brcmstb_l2_intc_suspend(struct irq_data *d, bool save) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); struct brcmstb_l2_intc_data *b = gc->private; - unsigned long flags; - irq_gc_lock_irqsave(gc, flags); + guard(raw_spinlock_irqsave)(&gc->lock); /* Save the current mask */ - b->saved_mask = irq_reg_readl(gc, ct->regs.mask); + if (save) + b->saved_mask = irq_reg_readl(gc, ct->regs.mask); if (b->can_wake) { /* Program the wakeup mask */ irq_reg_writel(gc, ~gc->wake_active, ct->regs.disable); irq_reg_writel(gc, gc->wake_active, ct->regs.enable); } - irq_gc_unlock_irqrestore(gc, flags); +} + +static void brcmstb_l2_intc_shutdown(struct irq_data *d) +{ + __brcmstb_l2_intc_suspend(d, false); +} + +static void brcmstb_l2_intc_suspend(struct irq_data *d) +{ + __brcmstb_l2_intc_suspend(d, true); } static void brcmstb_l2_intc_resume(struct irq_data *d) @@ -139,9 +125,8 @@ static void brcmstb_l2_intc_resume(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); struct brcmstb_l2_intc_data *b = gc->private; - unsigned long flags; - irq_gc_lock_irqsave(gc, flags); + guard(raw_spinlock_irqsave)(&gc->lock); if (ct->chip.irq_ack) { /* Clear unmasked non-wakeup interrupts */ irq_reg_writel(gc, ~b->saved_mask & ~gc->wake_active, @@ -151,16 +136,14 @@ static void brcmstb_l2_intc_resume(struct irq_data *d) /* Restore the saved mask */ irq_reg_writel(gc, b->saved_mask, ct->regs.disable); irq_reg_writel(gc, ~b->saved_mask, ct->regs.enable); - irq_gc_unlock_irqrestore(gc, flags); } -static int __init brcmstb_l2_intc_of_init(struct device_node *np, - struct device_node *parent, - const struct brcmstb_intc_init_params - *init_params) +static int brcmstb_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, + const struct brcmstb_intc_init_params *init_params) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; unsigned int set = 0; + struct device_node *np = pdev->dev.of_node; struct brcmstb_l2_intc_data *data; struct irq_chip_type *ct; int ret; @@ -194,7 +177,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, goto out_unmap; } - data->domain = irq_domain_add_linear(np, 32, + data->domain = irq_domain_create_linear(of_fwnode_handle(np), 32, &irq_generic_chip_ops, NULL); if (!data->domain) { ret = -ENOMEM; @@ -234,7 +217,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, if (init_params->cpu_clear >= 0) { ct->regs.ack = init_params->cpu_clear; ct->chip.irq_ack = irq_gc_ack_set_bit; - ct->chip.irq_mask_ack = brcmstb_l2_mask_and_ack; + ct->chip.irq_mask_ack = irq_gc_mask_disable_and_ack_set; } else { /* No Ack - but still slightly more efficient to define this */ ct->chip.irq_mask_ack = irq_gc_mask_disable_reg; @@ -249,7 +232,7 @@ static int __init brcmstb_l2_intc_of_init(struct device_node *np, ct->chip.irq_suspend = brcmstb_l2_intc_suspend; ct->chip.irq_resume = brcmstb_l2_intc_resume; - ct->chip.irq_pm_shutdown = brcmstb_l2_intc_suspend; + ct->chip.irq_pm_shutdown = brcmstb_l2_intc_shutdown; if (data->can_wake) { /* This IRQ chip can wake the system, set all child interrupts @@ -273,23 +256,21 @@ out_free: return ret; } -static int __init brcmstb_l2_edge_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_edge_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_edge_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_edge_intc_init); } -static int __init brcmstb_l2_lvl_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_lvl_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_lvl_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_lvl_intc_init); } IRQCHIP_PLATFORM_DRIVER_BEGIN(brcmstb_l2) -IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_of_init) +IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_probe) IRQCHIP_PLATFORM_DRIVER_END(brcmstb_l2) MODULE_DESCRIPTION("Broadcom STB generic L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-clps711x.c b/drivers/irqchip/irq-clps711x.c index e731e0784f7e..c4b73ba2323b 100644 --- a/drivers/irqchip/irq-clps711x.c +++ b/drivers/irqchip/irq-clps711x.c @@ -69,7 +69,7 @@ static struct { struct irq_domain_ops ops; } *clps711x_intc; -static asmlinkage void __exception_irq_entry clps711x_irqh(struct pt_regs *regs) +static void __exception_irq_entry clps711x_irqh(struct pt_regs *regs) { u32 irqstat; @@ -184,14 +184,14 @@ static int __init _clps711x_intc_init(struct device_node *np, clps711x_intc->ops.map = clps711x_intc_irq_map; clps711x_intc->ops.xlate = irq_domain_xlate_onecell; clps711x_intc->domain = - irq_domain_add_legacy(np, ARRAY_SIZE(clps711x_irqs), - 0, 0, &clps711x_intc->ops, NULL); + irq_domain_create_legacy(of_fwnode_handle(np), ARRAY_SIZE(clps711x_irqs), 0, 0, + &clps711x_intc->ops, NULL); if (!clps711x_intc->domain) { err = -ENOMEM; goto out_irqfree; } - irq_set_default_host(clps711x_intc->domain); + irq_set_default_domain(clps711x_intc->domain); set_handle_irq(clps711x_irqh); #ifdef CONFIG_FIQ diff --git a/drivers/irqchip/irq-crossbar.c b/drivers/irqchip/irq-crossbar.c index a05a7501e107..66bb39e24a52 100644 --- a/drivers/irqchip/irq-crossbar.c +++ b/drivers/irqchip/irq-crossbar.c @@ -351,10 +351,8 @@ static int __init irqcrossbar_init(struct device_node *node, if (err) return err; - domain = irq_domain_add_hierarchy(parent_domain, 0, - cb->max_crossbar_sources, - node, &crossbar_domain_ops, - NULL); + domain = irq_domain_create_hierarchy(parent_domain, 0, cb->max_crossbar_sources, + of_fwnode_handle(node), &crossbar_domain_ops, NULL); if (!domain) { pr_err("%pOF: failed to allocated domain\n", node); return -ENOMEM; diff --git a/drivers/irqchip/irq-csky-apb-intc.c b/drivers/irqchip/irq-csky-apb-intc.c index 6710691e4c25..5b7150705d29 100644 --- a/drivers/irqchip/irq-csky-apb-intc.c +++ b/drivers/irqchip/irq-csky-apb-intc.c @@ -50,11 +50,10 @@ static void irq_ck_mask_set_bit(struct irq_data *d) unsigned long ifr = ct->regs.mask - 8; u32 mask = d->mask; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); *ct->mask_cache |= mask; irq_reg_writel(gc, *ct->mask_cache, ct->regs.mask); irq_reg_writel(gc, irq_reg_readl(gc, ifr) & ~mask, ifr); - irq_gc_unlock(gc); } static void __init ck_set_gc(struct device_node *node, void __iomem *reg_base, @@ -114,7 +113,7 @@ ck_intc_init_comm(struct device_node *node, struct device_node *parent) return -EINVAL; } - root_domain = irq_domain_add_linear(node, nr_irq, + root_domain = irq_domain_create_linear(of_fwnode_handle(node), nr_irq, &irq_generic_chip_ops, NULL); if (!root_domain) { pr_err("C-SKY Intc irq_domain_add failed.\n"); diff --git a/drivers/irqchip/irq-csky-mpintc.c b/drivers/irqchip/irq-csky-mpintc.c index 4aebd67d4f8f..1d1f5091f26f 100644 --- a/drivers/irqchip/irq-csky-mpintc.c +++ b/drivers/irqchip/irq-csky-mpintc.c @@ -255,7 +255,7 @@ csky_mpintc_init(struct device_node *node, struct device_node *parent) writel_relaxed(BIT(0), INTCG_base + INTCG_ICTLR); } - root_domain = irq_domain_add_linear(node, nr_irq, &csky_irqdomain_ops, + root_domain = irq_domain_create_linear(of_fwnode_handle(node), nr_irq, &csky_irqdomain_ops, NULL); if (!root_domain) return -ENXIO; diff --git a/drivers/irqchip/irq-davinci-cp-intc.c b/drivers/irqchip/irq-davinci-cp-intc.c index 7482c8ed34b2..00cdcc90f614 100644 --- a/drivers/irqchip/irq-davinci-cp-intc.c +++ b/drivers/irqchip/irq-davinci-cp-intc.c @@ -11,7 +11,6 @@ #include <linux/init.h> #include <linux/irq.h> #include <linux/irqchip.h> -#include <linux/irqchip/irq-davinci-cp-intc.h> #include <linux/irqdomain.h> #include <linux/io.h> #include <linux/of.h> @@ -116,8 +115,7 @@ static struct irq_chip davinci_cp_intc_irq_chip = { .flags = IRQCHIP_SKIP_SET_WAKE, }; -static asmlinkage void __exception_irq_entry -davinci_cp_intc_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry davinci_cp_intc_handle_irq(struct pt_regs *regs) { int gpir, irqnr, none; @@ -155,24 +153,20 @@ static const struct irq_domain_ops davinci_cp_intc_irq_domain_ops = { .xlate = irq_domain_xlate_onetwocell, }; -static int __init -davinci_cp_intc_do_init(const struct davinci_cp_intc_config *config, - struct device_node *node) +static int __init davinci_cp_intc_do_init(struct resource *res, unsigned int num_irqs, + struct device_node *node) { - unsigned int num_regs = BITS_TO_LONGS(config->num_irqs); + unsigned int num_regs = BITS_TO_LONGS(num_irqs); int offset, irq_base; void __iomem *req; - req = request_mem_region(config->reg.start, - resource_size(&config->reg), - "davinci-cp-intc"); + req = request_mem_region(res->start, resource_size(res), "davinci-cp-intc"); if (!req) { pr_err("%s: register range busy\n", __func__); return -EBUSY; } - davinci_cp_intc_base = ioremap(config->reg.start, - resource_size(&config->reg)); + davinci_cp_intc_base = ioremap(res->start, resource_size(res)); if (!davinci_cp_intc_base) { pr_err("%s: unable to ioremap register range\n", __func__); return -EINVAL; @@ -185,8 +179,7 @@ davinci_cp_intc_do_init(const struct davinci_cp_intc_config *config, /* Disable system interrupts */ for (offset = 0; offset < num_regs; offset++) - davinci_cp_intc_write(~0, - DAVINCI_CP_INTC_SYS_ENABLE_CLR(offset)); + davinci_cp_intc_write(~0, DAVINCI_CP_INTC_SYS_ENABLE_CLR(offset)); /* Set to normal mode, no nesting, no priority hold */ davinci_cp_intc_write(0, DAVINCI_CP_INTC_CTRL); @@ -194,28 +187,27 @@ davinci_cp_intc_do_init(const struct davinci_cp_intc_config *config, /* Clear system interrupt status */ for (offset = 0; offset < num_regs; offset++) - davinci_cp_intc_write(~0, - DAVINCI_CP_INTC_SYS_STAT_CLR(offset)); + davinci_cp_intc_write(~0, DAVINCI_CP_INTC_SYS_STAT_CLR(offset)); /* Enable nIRQ (what about nFIQ?) */ davinci_cp_intc_write(1, DAVINCI_CP_INTC_HOST_ENABLE_IDX_SET); + /* 4 channels per register */ + num_regs = (num_irqs + 3) >> 2; /* Default all priorities to channel 7. */ - num_regs = (config->num_irqs + 3) >> 2; /* 4 channels per register */ for (offset = 0; offset < num_regs; offset++) - davinci_cp_intc_write(0x07070707, - DAVINCI_CP_INTC_CHAN_MAP(offset)); + davinci_cp_intc_write(0x07070707, DAVINCI_CP_INTC_CHAN_MAP(offset)); - irq_base = irq_alloc_descs(-1, 0, config->num_irqs, 0); + irq_base = irq_alloc_descs(-1, 0, num_irqs, 0); if (irq_base < 0) { - pr_err("%s: unable to allocate interrupt descriptors: %d\n", - __func__, irq_base); + pr_err("%s: unable to allocate interrupt descriptors: %d\n", __func__, irq_base); return irq_base; } - davinci_cp_intc_irq_domain = irq_domain_add_legacy( - node, config->num_irqs, irq_base, 0, - &davinci_cp_intc_irq_domain_ops, NULL); + davinci_cp_intc_irq_domain = irq_domain_create_legacy(of_fwnode_handle(node), num_irqs, + irq_base, 0, + &davinci_cp_intc_irq_domain_ops, + NULL); if (!davinci_cp_intc_irq_domain) { pr_err("%s: unable to create an interrupt domain\n", __func__); @@ -230,31 +222,25 @@ davinci_cp_intc_do_init(const struct davinci_cp_intc_config *config, return 0; } -int __init davinci_cp_intc_init(const struct davinci_cp_intc_config *config) -{ - return davinci_cp_intc_do_init(config, NULL); -} - static int __init davinci_cp_intc_of_init(struct device_node *node, struct device_node *parent) { - struct davinci_cp_intc_config config = { }; + unsigned int num_irqs; + struct resource res; int ret; - ret = of_address_to_resource(node, 0, &config.reg); + ret = of_address_to_resource(node, 0, &res); if (ret) { - pr_err("%s: unable to get the register range from device-tree\n", - __func__); + pr_err("%s: unable to get the register range from device-tree\n", __func__); return ret; } - ret = of_property_read_u32(node, "ti,intc-size", &config.num_irqs); + ret = of_property_read_u32(node, "ti,intc-size", &num_irqs); if (ret) { - pr_err("%s: unable to read the 'ti,intc-size' property\n", - __func__); + pr_err("%s: unable to read the 'ti,intc-size' property\n", __func__); return ret; } - return davinci_cp_intc_do_init(&config, node); + return davinci_cp_intc_do_init(&res, num_irqs, node); } IRQCHIP_DECLARE(cp_intc, "ti,cp-intc", davinci_cp_intc_of_init); diff --git a/drivers/irqchip/irq-digicolor.c b/drivers/irqchip/irq-digicolor.c index 3b0d78aac13b..eb5a8de82751 100644 --- a/drivers/irqchip/irq-digicolor.c +++ b/drivers/irqchip/irq-digicolor.c @@ -95,7 +95,7 @@ static int __init digicolor_of_init(struct device_node *node, regmap_write(ucregs, UC_IRQ_CONTROL, 1); digicolor_irq_domain = - irq_domain_add_linear(node, 64, &irq_generic_chip_ops, NULL); + irq_domain_create_linear(of_fwnode_handle(node), 64, &irq_generic_chip_ops, NULL); if (!digicolor_irq_domain) { pr_err("%pOF: unable to create IRQ domain\n", node); return -ENOMEM; diff --git a/drivers/irqchip/irq-dw-apb-ictl.c b/drivers/irqchip/irq-dw-apb-ictl.c index d5c1c750c8d2..4240a0dbf627 100644 --- a/drivers/irqchip/irq-dw-apb-ictl.c +++ b/drivers/irqchip/irq-dw-apb-ictl.c @@ -101,10 +101,9 @@ static void dw_apb_ictl_resume(struct irq_data *d) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct irq_chip_type *ct = irq_data_get_chip_type(d); - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); writel_relaxed(~0, gc->reg_base + ct->regs.enable); writel_relaxed(*ct->mask_cache, gc->reg_base + ct->regs.mask); - irq_gc_unlock(gc); } #else #define dw_apb_ictl_resume NULL @@ -173,7 +172,7 @@ static int __init dw_apb_ictl_init(struct device_node *np, else nrirqs = fls(readl_relaxed(iobase + APB_INT_ENABLE_L)); - domain = irq_domain_add_linear(np, nrirqs, domain_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(np), nrirqs, domain_ops, NULL); if (!domain) { pr_err("%pOF: unable to add irq domain\n", np); ret = -ENOMEM; diff --git a/drivers/irqchip/irq-econet-en751221.c b/drivers/irqchip/irq-econet-en751221.c new file mode 100644 index 000000000000..d83d5eb12795 --- /dev/null +++ b/drivers/irqchip/irq-econet-en751221.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * EN751221 Interrupt Controller Driver. + * + * The EcoNet EN751221 Interrupt Controller is a simple interrupt controller + * designed for the MIPS 34Kc MT SMP processor with 2 VPEs. Each interrupt can + * be routed to either VPE but not both, so to support per-CPU interrupts, a + * secondary IRQ number is allocated to control masking/unmasking on VPE#1. In + * this driver, these are called "shadow interrupts". The assignment of shadow + * interrupts is defined by the SoC integrator when wiring the interrupt lines, + * so they are configurable in the device tree. + * + * If an interrupt (say 30) needs per-CPU capability, the SoC integrator + * allocates another IRQ number (say 29) to be its shadow. The device tree + * reflects this by adding the pair <30 29> to the "econet,shadow-interrupts" + * property. + * + * When VPE#1 requests IRQ 30, the driver manipulates the mask bit for IRQ 29, + * telling the hardware to mask VPE#1's view of IRQ 30. + * + * Copyright (C) 2025 Caleb James DeLisle <cjd@cjdns.fr> + */ + +#include <linux/cleanup.h> +#include <linux/io.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/irqdomain.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> + +#define IRQ_COUNT 40 + +#define NOT_PERCPU 0xff +#define IS_SHADOW 0xfe + +#define REG_MASK0 0x04 +#define REG_MASK1 0x50 +#define REG_PENDING0 0x08 +#define REG_PENDING1 0x54 + +/** + * @membase: Base address of the interrupt controller registers + * @interrupt_shadows: Array of all interrupts, for each value, + * - NOT_PERCPU: This interrupt is not per-cpu, so it has no shadow + * - IS_SHADOW: This interrupt is a shadow of another per-cpu interrupt + * - else: This is a per-cpu interrupt whose shadow is the value + */ +static struct { + void __iomem *membase; + u8 interrupt_shadows[IRQ_COUNT]; +} econet_intc __ro_after_init; + +static DEFINE_RAW_SPINLOCK(irq_lock); + +/* IRQs must be disabled */ +static void econet_wreg(u32 reg, u32 val, u32 mask) +{ + u32 v; + + guard(raw_spinlock)(&irq_lock); + + v = ioread32(econet_intc.membase + reg); + v &= ~mask; + v |= val & mask; + iowrite32(v, econet_intc.membase + reg); +} + +/* IRQs must be disabled */ +static void econet_chmask(u32 hwirq, bool unmask) +{ + u32 reg, mask; + u8 shadow; + + /* + * If the IRQ is a shadow, it should never be manipulated directly. + * It should only be masked/unmasked as a result of the "real" per-cpu + * irq being manipulated by a thread running on VPE#1. + * If it is per-cpu (has a shadow), and we're on VPE#1, the shadow is what we mask. + * This is single processor only, so smp_processor_id() never exceeds 1. + */ + shadow = econet_intc.interrupt_shadows[hwirq]; + if (WARN_ON_ONCE(shadow == IS_SHADOW)) + return; + else if (shadow != NOT_PERCPU && smp_processor_id() == 1) + hwirq = shadow; + + if (hwirq >= 32) { + reg = REG_MASK1; + mask = BIT(hwirq - 32); + } else { + reg = REG_MASK0; + mask = BIT(hwirq); + } + + econet_wreg(reg, unmask ? mask : 0, mask); +} + +/* IRQs must be disabled */ +static void econet_intc_mask(struct irq_data *d) +{ + econet_chmask(d->hwirq, false); +} + +/* IRQs must be disabled */ +static void econet_intc_unmask(struct irq_data *d) +{ + econet_chmask(d->hwirq, true); +} + +static void econet_mask_all(void) +{ + /* IRQs are generally disabled during init, but guarding here makes it non-obligatory. */ + guard(irqsave)(); + econet_wreg(REG_MASK0, 0, ~0); + econet_wreg(REG_MASK1, 0, ~0); +} + +static void econet_intc_handle_pending(struct irq_domain *d, u32 pending, u32 offset) +{ + int hwirq; + + while (pending) { + hwirq = fls(pending) - 1; + generic_handle_domain_irq(d, hwirq + offset); + pending &= ~BIT(hwirq); + } +} + +static void econet_intc_from_parent(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct irq_domain *domain; + u32 pending0, pending1; + + chained_irq_enter(chip, desc); + + pending0 = ioread32(econet_intc.membase + REG_PENDING0); + pending1 = ioread32(econet_intc.membase + REG_PENDING1); + + if (unlikely(!(pending0 | pending1))) { + spurious_interrupt(); + } else { + domain = irq_desc_get_handler_data(desc); + econet_intc_handle_pending(domain, pending0, 0); + econet_intc_handle_pending(domain, pending1, 32); + } + + chained_irq_exit(chip, desc); +} + +static const struct irq_chip econet_irq_chip; + +static int econet_intc_map(struct irq_domain *d, u32 irq, irq_hw_number_t hwirq) +{ + int ret; + + if (hwirq >= IRQ_COUNT) { + pr_err("%s: hwirq %lu out of range\n", __func__, hwirq); + return -EINVAL; + } else if (econet_intc.interrupt_shadows[hwirq] == IS_SHADOW) { + pr_err("%s: can't map hwirq %lu, it is a shadow interrupt\n", __func__, hwirq); + return -EINVAL; + } + + if (econet_intc.interrupt_shadows[hwirq] == NOT_PERCPU) { + irq_set_chip_and_handler(irq, &econet_irq_chip, handle_level_irq); + } else { + irq_set_chip_and_handler(irq, &econet_irq_chip, handle_percpu_devid_irq); + ret = irq_set_percpu_devid(irq); + if (ret) + pr_warn("%s: Failed irq_set_percpu_devid for %u: %d\n", d->name, irq, ret); + } + + irq_set_chip_data(irq, NULL); + return 0; +} + +static const struct irq_chip econet_irq_chip = { + .name = "en751221-intc", + .irq_unmask = econet_intc_unmask, + .irq_mask = econet_intc_mask, + .irq_mask_ack = econet_intc_mask, +}; + +static const struct irq_domain_ops econet_domain_ops = { + .xlate = irq_domain_xlate_onecell, + .map = econet_intc_map +}; + +static int __init get_shadow_interrupts(struct device_node *node) +{ + const char *field = "econet,shadow-interrupts"; + int num_shadows; + + num_shadows = of_property_count_u32_elems(node, field); + + memset(econet_intc.interrupt_shadows, NOT_PERCPU, + sizeof(econet_intc.interrupt_shadows)); + + if (num_shadows <= 0) { + return 0; + } else if (num_shadows % 2) { + pr_err("%pOF: %s count is odd, ignoring\n", node, field); + return 0; + } + + u32 *shadows __free(kfree) = kmalloc_array(num_shadows, sizeof(u32), GFP_KERNEL); + if (!shadows) + return -ENOMEM; + + if (of_property_read_u32_array(node, field, shadows, num_shadows)) { + pr_err("%pOF: Failed to read %s\n", node, field); + return -EINVAL; + } + + for (int i = 0; i < num_shadows; i += 2) { + u32 shadow = shadows[i + 1]; + u32 target = shadows[i]; + + if (shadow > IRQ_COUNT) { + pr_err("%pOF: %s[%d] shadow(%d) out of range\n", + node, field, i + 1, shadow); + continue; + } + + if (target >= IRQ_COUNT) { + pr_err("%pOF: %s[%d] target(%d) out of range\n", node, field, i, target); + continue; + } + + if (econet_intc.interrupt_shadows[target] != NOT_PERCPU) { + pr_err("%pOF: %s[%d] target(%d) already has a shadow\n", + node, field, i, target); + continue; + } + + if (econet_intc.interrupt_shadows[shadow] != NOT_PERCPU) { + pr_err("%pOF: %s[%d] shadow(%d) already has a target\n", + node, field, i + 1, shadow); + continue; + } + + econet_intc.interrupt_shadows[target] = shadow; + econet_intc.interrupt_shadows[shadow] = IS_SHADOW; + } + + return 0; +} + +static int __init econet_intc_of_init(struct device_node *node, struct device_node *parent) +{ + struct irq_domain *domain; + struct resource res; + int ret, irq; + + ret = get_shadow_interrupts(node); + if (ret) + return ret; + + irq = irq_of_parse_and_map(node, 0); + if (!irq) { + pr_err("%pOF: DT: Failed to get IRQ from 'interrupts'\n", node); + return -EINVAL; + } + + if (of_address_to_resource(node, 0, &res)) { + pr_err("%pOF: DT: Failed to get 'reg'\n", node); + ret = -EINVAL; + goto err_dispose_mapping; + } + + if (!request_mem_region(res.start, resource_size(&res), res.name)) { + pr_err("%pOF: Failed to request memory\n", node); + ret = -EBUSY; + goto err_dispose_mapping; + } + + econet_intc.membase = ioremap(res.start, resource_size(&res)); + if (!econet_intc.membase) { + pr_err("%pOF: Failed to remap membase\n", node); + ret = -ENOMEM; + goto err_release; + } + + econet_mask_all(); + + domain = irq_domain_create_linear(of_fwnode_handle(node), IRQ_COUNT, + &econet_domain_ops, NULL); + if (!domain) { + pr_err("%pOF: Failed to add irqdomain\n", node); + ret = -ENOMEM; + goto err_unmap; + } + + irq_set_chained_handler_and_data(irq, econet_intc_from_parent, domain); + + return 0; + +err_unmap: + iounmap(econet_intc.membase); +err_release: + release_mem_region(res.start, resource_size(&res)); +err_dispose_mapping: + irq_dispose_mapping(irq); + return ret; +} + +IRQCHIP_DECLARE(econet_en751221_intc, "econet,en751221-intc", econet_intc_of_init); diff --git a/drivers/irqchip/irq-ftintc010.c b/drivers/irqchip/irq-ftintc010.c index 359efc1d1be7..a59a66d79da6 100644 --- a/drivers/irqchip/irq-ftintc010.c +++ b/drivers/irqchip/irq-ftintc010.c @@ -125,7 +125,7 @@ static struct irq_chip ft010_irq_chip = { /* Local static for the IRQ entry call */ static struct ft010_irq_data firq; -static asmlinkage void __exception_irq_entry ft010_irqchip_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry ft010_irqchip_handle_irq(struct pt_regs *regs) { struct ft010_irq_data *f = &firq; int irq; @@ -180,8 +180,9 @@ static int __init ft010_of_init_irq(struct device_node *node, writel(0, FT010_IRQ_MASK(f->base)); writel(0, FT010_FIQ_MASK(f->base)); - f->domain = irq_domain_add_simple(node, FT010_NUM_IRQS, 0, - &ft010_irqdomain_ops, f); + f->domain = irq_domain_create_simple(of_fwnode_handle(node), + FT010_NUM_IRQS, 0, + &ft010_irqdomain_ops, f); set_handle_irq(ft010_irqchip_handle_irq); return 0; diff --git a/drivers/irqchip/irq-gic-common.c b/drivers/irqchip/irq-gic-common.c index afd6a1841715..c776f9142610 100644 --- a/drivers/irqchip/irq-gic-common.c +++ b/drivers/irqchip/irq-gic-common.c @@ -7,6 +7,7 @@ #include <linux/io.h> #include <linux/irq.h> #include <linux/irqchip/arm-gic.h> +#include <linux/kernel.h> #include "irq-gic-common.h" @@ -45,7 +46,7 @@ void gic_enable_quirks(u32 iidr, const struct gic_quirk *quirks, } int gic_configure_irq(unsigned int irq, unsigned int type, - void __iomem *base, void (*sync_access)(void)) + void __iomem *base) { u32 confmask = 0x2 << ((irq % 16) * 2); u32 confoff = (irq / 16) * 4; @@ -84,14 +85,10 @@ int gic_configure_irq(unsigned int irq, unsigned int type, raw_spin_unlock_irqrestore(&irq_controller_lock, flags); - if (sync_access) - sync_access(); - return ret; } -void gic_dist_config(void __iomem *base, int gic_irqs, - void (*sync_access)(void)) +void gic_dist_config(void __iomem *base, int gic_irqs, u8 priority) { unsigned int i; @@ -106,7 +103,8 @@ void gic_dist_config(void __iomem *base, int gic_irqs, * Set priority on all global interrupts. */ for (i = 32; i < gic_irqs; i += 4) - writel_relaxed(GICD_INT_DEF_PRI_X4, base + GIC_DIST_PRI + i); + writel_relaxed(REPEAT_BYTE_U32(priority), + base + GIC_DIST_PRI + i); /* * Deactivate and disable all SPIs. Leave the PPI and SGIs @@ -118,12 +116,9 @@ void gic_dist_config(void __iomem *base, int gic_irqs, writel_relaxed(GICD_INT_EN_CLR_X32, base + GIC_DIST_ENABLE_CLEAR + i / 8); } - - if (sync_access) - sync_access(); } -void gic_cpu_config(void __iomem *base, int nr, void (*sync_access)(void)) +void gic_cpu_config(void __iomem *base, int nr, u8 priority) { int i; @@ -142,9 +137,6 @@ void gic_cpu_config(void __iomem *base, int nr, void (*sync_access)(void)) * Set priority on PPI and SGI interrupts */ for (i = 0; i < nr; i += 4) - writel_relaxed(GICD_INT_DEF_PRI_X4, + writel_relaxed(REPEAT_BYTE_U32(priority), base + GIC_DIST_PRI + i * 4 / 4); - - if (sync_access) - sync_access(); } diff --git a/drivers/irqchip/irq-gic-common.h b/drivers/irqchip/irq-gic-common.h index 3db4592cda1c..710cab61d919 100644 --- a/drivers/irqchip/irq-gic-common.h +++ b/drivers/irqchip/irq-gic-common.h @@ -8,6 +8,7 @@ #include <linux/of.h> #include <linux/irqdomain.h> +#include <linux/msi.h> #include <linux/irqchip/arm-gic-common.h> struct gic_quirk { @@ -20,13 +21,16 @@ struct gic_quirk { }; int gic_configure_irq(unsigned int irq, unsigned int type, - void __iomem *base, void (*sync_access)(void)); -void gic_dist_config(void __iomem *base, int gic_irqs, - void (*sync_access)(void)); -void gic_cpu_config(void __iomem *base, int nr, void (*sync_access)(void)); + void __iomem *base); +void gic_dist_config(void __iomem *base, int gic_irqs, u8 priority); +void gic_cpu_config(void __iomem *base, int nr, u8 priority); void gic_enable_quirks(u32 iidr, const struct gic_quirk *quirks, void *data); void gic_enable_of_quirks(const struct device_node *np, const struct gic_quirk *quirks, void *data); +#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) +#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) +#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1 << 2) + #endif /* _IRQ_GIC_COMMON_H */ diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c new file mode 100644 index 000000000000..12f45228c867 --- /dev/null +++ b/drivers/irqchip/irq-gic-its-msi-parent.c @@ -0,0 +1,331 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2013-2015 ARM Limited, All Rights Reserved. +// Author: Marc Zyngier <marc.zyngier@arm.com> +// Copyright (C) 2022 Linutronix GmbH +// Copyright (C) 2022 Intel + +#include <linux/acpi_iort.h> +#include <linux/of_address.h> +#include <linux/pci.h> + +#include "irq-gic-its-msi-parent.h" +#include <linux/irqchip/irq-msi-lib.h> + +#define ITS_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define ITS_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static int its_translate_frame_address(struct device_node *msi_node, phys_addr_t *pa) +{ + struct resource res; + int ret; + + ret = of_property_match_string(msi_node, "reg-names", "ns-translate"); + if (ret < 0) + return ret; + + ret = of_address_to_resource(msi_node, ret, &res); + if (ret) + return ret; + + *pa = res.start; + return 0; +} + +#ifdef CONFIG_PCI_MSI +static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data) +{ + int msi, msix, *count = data; + + msi = max(pci_msi_vec_count(pdev), 0); + msix = max(pci_msix_vec_count(pdev), 0); + *count += max(msi, msix); + + return 0; +} + +static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data) +{ + struct pci_dev **alias_dev = data; + + *alias_dev = pdev; + + return 0; +} + +static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct pci_dev *pdev, *alias_dev; + struct msi_domain_info *msi_info; + int alias_count = 0, minnvec = 1; + + if (!dev_is_pci(dev)) + return -EINVAL; + + pdev = to_pci_dev(dev); + /* + * If pdev is downstream of any aliasing bridges, take an upper + * bound of how many other vectors could map to the same DevID. + * Also tell the ITS that the signalling will come from a proxy + * device, and that special allocation rules apply. + */ + pci_for_each_dma_alias(pdev, its_get_pci_alias, &alias_dev); + if (alias_dev != pdev) { + if (alias_dev->subordinate) + pci_walk_bus(alias_dev->subordinate, + its_pci_msi_vec_count, &alias_count); + info->flags |= MSI_ALLOC_FLAGS_PROXY_DEVICE; + } + + /* ITS specific DeviceID, as the core ITS ignores dev. */ + info->scratchpad[0].ul = pci_msi_domain_get_msi_rid(domain->parent, pdev); + + /* + * Always allocate a power of 2, and special case device 0 for + * broken systems where the DevID is not wired (and all devices + * appear as DevID 0). For that reason, we generously allocate a + * minimum of 32 MSIs for DevID 0. If you want more because all + * your devices are aliasing to DevID 0, consider fixing your HW. + */ + nvec = max(nvec, alias_count); + if (!info->scratchpad[0].ul) + minnvec = 32; + nvec = max_t(int, minnvec, roundup_pow_of_two(nvec)); + + msi_info = msi_get_domain_info(domain->parent); + return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info); +} + +static int its_v5_pci_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct device_node *msi_node = NULL; + struct msi_domain_info *msi_info; + struct pci_dev *pdev; + phys_addr_t pa; + u32 rid; + int ret; + + if (!dev_is_pci(dev)) + return -EINVAL; + + pdev = to_pci_dev(dev); + + rid = pci_msi_map_rid_ctlr_node(pdev, &msi_node); + if (!msi_node) + return -ENODEV; + + ret = its_translate_frame_address(msi_node, &pa); + if (ret) + return -ENODEV; + + of_node_put(msi_node); + + /* ITS specific DeviceID */ + info->scratchpad[0].ul = rid; + /* ITS translate frame physical address */ + info->scratchpad[1].ul = pa; + + /* Always allocate power of two vectors */ + nvec = roundup_pow_of_two(nvec); + + msi_info = msi_get_domain_info(domain->parent); + return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info); +} +#else /* CONFIG_PCI_MSI */ +#define its_pci_msi_prepare NULL +#define its_v5_pci_msi_prepare NULL +#endif /* !CONFIG_PCI_MSI */ + +static int of_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev, u32 *dev_id, + phys_addr_t *pa) +{ + struct of_phandle_iterator it; + int ret; + + /* Suck the DeviceID out of the msi-parent property */ + of_for_each_phandle(&it, ret, dev->of_node, "msi-parent", "#msi-cells", -1) { + /* GICv5 ITS domain matches the MSI controller node parent */ + struct device_node *np __free(device_node) = pa ? of_get_parent(it.node) + : of_node_get(it.node); + + if (np == irq_domain_get_of_node(domain)) { + u32 args; + + if (WARN_ON(of_phandle_iterator_args(&it, &args, 1) != 1)) + ret = -EINVAL; + + if (!ret && pa) + ret = its_translate_frame_address(it.node, pa); + + if (!ret) + *dev_id = args; + + of_node_put(it.node); + return ret; + } + } + + struct device_node *msi_ctrl __free(device_node) = NULL; + + return of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &msi_ctrl, dev_id); +} + +int __weak iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id) +{ + return -1; +} + +static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct msi_domain_info *msi_info; + u32 dev_id; + int ret; + + if (dev->of_node) + ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, NULL); + else + ret = iort_pmsi_get_dev_id(dev, &dev_id); + if (ret) + return ret; + + /* ITS specific DeviceID, as the core ITS ignores dev. */ + info->scratchpad[0].ul = dev_id; + + /* Allocate at least 32 MSIs, and always as a power of 2 */ + nvec = max_t(int, 32, roundup_pow_of_two(nvec)); + + msi_info = msi_get_domain_info(domain->parent); + return msi_info->ops->msi_prepare(domain->parent, + dev, nvec, info); +} + +static int its_v5_pmsi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + struct msi_domain_info *msi_info; + phys_addr_t pa; + u32 dev_id; + int ret; + + if (!dev->of_node) + return -ENODEV; + + ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, &pa); + if (ret) + return ret; + + /* ITS specific DeviceID */ + info->scratchpad[0].ul = dev_id; + /* ITS translate frame physical address */ + info->scratchpad[1].ul = pa; + + /* Allocate always as a power of 2 */ + nvec = roundup_pow_of_two(nvec); + + msi_info = msi_get_domain_info(domain->parent); + return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info); +} + +static void its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info) +{ + struct msi_domain_info *msi_info; + + msi_info = msi_get_domain_info(domain->parent); + msi_info->ops->msi_teardown(domain->parent, info); +} + +static bool its_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + switch(info->bus_token) { + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + /* + * FIXME: This probably should be done after a (not yet + * existing) post domain creation callback once to make + * support for dynamic post-enable MSI-X allocations + * work without having to reevaluate the domain size + * over and over. It is known already at allocation + * time via info->hwsize. + * + * That should work perfectly fine for MSI/MSI-X but needs + * some thoughts for purely software managed MSI domains + * where the index space is only limited artificially via + * %MSI_MAX_INDEX. + */ + info->ops->msi_prepare = its_pci_msi_prepare; + info->ops->msi_teardown = its_msi_teardown; + break; + case DOMAIN_BUS_DEVICE_MSI: + case DOMAIN_BUS_WIRED_TO_MSI: + /* + * FIXME: See the above PCI prepare comment. The domain + * size is also known at domain creation time. + */ + info->ops->msi_prepare = its_pmsi_prepare; + info->ops->msi_teardown = its_msi_teardown; + break; + default: + /* Confused. How did the lib return true? */ + WARN_ON_ONCE(1); + return false; + } + + return true; +} + +static bool its_v5_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + switch (info->bus_token) { + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + info->ops->msi_prepare = its_v5_pci_msi_prepare; + info->ops->msi_teardown = its_msi_teardown; + break; + case DOMAIN_BUS_DEVICE_MSI: + case DOMAIN_BUS_WIRED_TO_MSI: + info->ops->msi_prepare = its_v5_pmsi_prepare; + info->ops->msi_teardown = its_msi_teardown; + break; + default: + /* Confused. How did the lib return true? */ + WARN_ON_ONCE(1); + return false; + } + + return true; +} + +const struct msi_parent_ops gic_v3_its_msi_parent_ops = { + .supported_flags = ITS_MSI_FLAGS_SUPPORTED, + .required_flags = ITS_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .prefix = "ITS-", + .init_dev_msi_info = its_init_dev_msi_info, +}; + +const struct msi_parent_ops gic_v5_its_msi_parent_ops = { + .supported_flags = ITS_MSI_FLAGS_SUPPORTED, + .required_flags = ITS_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .prefix = "ITS-v5-", + .init_dev_msi_info = its_v5_init_dev_msi_info, +}; diff --git a/drivers/irqchip/irq-gic-its-msi-parent.h b/drivers/irqchip/irq-gic-its-msi-parent.h new file mode 100644 index 000000000000..df016f347337 --- /dev/null +++ b/drivers/irqchip/irq-gic-its-msi-parent.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2024 ARM Limited, All Rights Reserved. + */ + +#ifndef _IRQ_GIC_ITS_MSI_PARENT_H +#define _IRQ_GIC_ITS_MSI_PARENT_H + +extern const struct msi_parent_ops gic_v3_its_msi_parent_ops; +extern const struct msi_parent_ops gic_v5_its_msi_parent_ops; + +#endif /* _IRQ_GIC_ITS_MSI_PARENT_H */ diff --git a/drivers/irqchip/irq-gic-v2m.c b/drivers/irqchip/irq-gic-v2m.c index f2ff4387870d..8a3410c2b7b5 100644 --- a/drivers/irqchip/irq-gic-v2m.c +++ b/drivers/irqchip/irq-gic-v2m.c @@ -26,6 +26,8 @@ #include <linux/irqchip/arm-gic.h> #include <linux/irqchip/arm-gic-common.h> +#include <linux/irqchip/irq-msi-lib.h> + /* * MSI_TYPER: * [31:26] Reserved @@ -72,31 +74,6 @@ struct v2m_data { u32 flags; /* v2m flags for specific implementation */ }; -static void gicv2m_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void gicv2m_unmask_msi_irq(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip gicv2m_msi_irq_chip = { - .name = "MSI", - .irq_mask = gicv2m_mask_msi_irq, - .irq_unmask = gicv2m_unmask_msi_irq, - .irq_eoi = irq_chip_eoi_parent, -}; - -static struct msi_domain_info gicv2m_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI), - .chip = &gicv2m_msi_irq_chip, -}; - static phys_addr_t gicv2m_get_msi_addr(struct v2m_data *v2m, int hwirq) { if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) @@ -110,9 +87,6 @@ static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) struct v2m_data *v2m = irq_data_get_irq_chip_data(data); phys_addr_t addr = gicv2m_get_msi_addr(v2m, data->hwirq); - msg->address_hi = upper_32_bits(addr); - msg->address_lo = lower_32_bits(addr); - if (v2m->flags & GICV2M_GRAVITON_ADDRESS_ONLY) msg->data = 0; else @@ -120,7 +94,7 @@ static void gicv2m_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) if (v2m->flags & GICV2M_NEEDS_SPI_OFFSET) msg->data -= v2m->spi_offset; - iommu_dma_compose_msi_msg(irq_data_get_msi_desc(data), msg); + msi_msg_set_addr(irq_data_get_msi_desc(data), msg, addr); } static struct irq_chip gicv2m_irq_chip = { @@ -179,14 +153,19 @@ static int gicv2m_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, { msi_alloc_info_t *info = args; struct v2m_data *v2m = NULL, *tmp; - int hwirq, offset, i, err = 0; + int hwirq, i, err = 0; + unsigned long offset; + unsigned long align_mask = nr_irqs - 1; spin_lock(&v2m_lock); list_for_each_entry(tmp, &v2m_nodes, entry) { - offset = bitmap_find_free_region(tmp->bm, tmp->nr_spis, - get_count_order(nr_irqs)); - if (offset >= 0) { + unsigned long align_off = tmp->spi_start - (tmp->spi_start & ~align_mask); + + offset = bitmap_find_next_zero_area_off(tmp->bm, tmp->nr_spis, 0, + nr_irqs, align_mask, align_off); + if (offset < tmp->nr_spis) { v2m = tmp; + bitmap_set(v2m->bm, offset, nr_irqs); break; } } @@ -230,6 +209,7 @@ static void gicv2m_irq_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops gicv2m_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = gicv2m_irq_domain_alloc, .free = gicv2m_irq_domain_free, }; @@ -250,19 +230,6 @@ static bool is_msi_spi_valid(u32 base, u32 num) return true; } -static struct irq_chip gicv2m_pmsi_irq_chip = { - .name = "pMSI", -}; - -static struct msi_domain_ops gicv2m_pmsi_ops = { -}; - -static struct msi_domain_info gicv2m_pmsi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), - .ops = &gicv2m_pmsi_ops, - .chip = &gicv2m_pmsi_irq_chip, -}; - static void __init gicv2m_teardown(void) { struct v2m_data *v2m, *tmp; @@ -278,39 +245,44 @@ static void __init gicv2m_teardown(void) } } + +#define GICV2M_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define GICV2M_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static struct msi_parent_ops gicv2m_msi_parent_ops = { + .supported_flags = GICV2M_MSI_FLAGS_SUPPORTED, + .required_flags = GICV2M_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .prefix = "GICv2m-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + static __init int gicv2m_allocate_domains(struct irq_domain *parent) { - struct irq_domain *inner_domain, *pci_domain, *plat_domain; + struct irq_domain_info info = { + .ops = &gicv2m_domain_ops, + .parent = parent, + }; struct v2m_data *v2m; v2m = list_first_entry_or_null(&v2m_nodes, struct v2m_data, entry); if (!v2m) return 0; - inner_domain = irq_domain_create_hierarchy(parent, 0, 0, v2m->fwnode, - &gicv2m_domain_ops, v2m); - if (!inner_domain) { - pr_err("Failed to create GICv2m domain\n"); - return -ENOMEM; - } + info.host_data = v2m; + info.fwnode = v2m->fwnode; - irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); - pci_domain = pci_msi_create_irq_domain(v2m->fwnode, - &gicv2m_msi_domain_info, - inner_domain); - plat_domain = platform_msi_create_irq_domain(v2m->fwnode, - &gicv2m_pmsi_domain_info, - inner_domain); - if (!pci_domain || !plat_domain) { - pr_err("Failed to create MSI domains\n"); - if (plat_domain) - irq_domain_remove(plat_domain); - if (pci_domain) - irq_domain_remove(pci_domain); - irq_domain_remove(inner_domain); + if (!msi_create_parent_irq_domain(&info, &gicv2m_msi_parent_ops)) { + pr_err("Failed to create GICv2m domain\n"); return -ENOMEM; } - return 0; } @@ -438,12 +410,12 @@ static int __init gicv2m_of_init(struct fwnode_handle *parent_handle, ret = gicv2m_init_one(&child->fwnode, spi_start, nr_spis, &res, 0); - if (ret) { - of_node_put(child); + if (ret) break; - } } + if (ret && child) + of_node_put(child); if (!ret) ret = gicv2m_allocate_domains(parent); if (ret) @@ -454,7 +426,7 @@ static int __init gicv2m_of_init(struct fwnode_handle *parent_handle, #ifdef CONFIG_ACPI static int acpi_num_msi; -static __init struct fwnode_handle *gicv2m_get_fwnode(struct device *dev) +static struct fwnode_handle *gicv2m_get_fwnode(struct device *dev) { struct v2m_data *data; @@ -511,7 +483,7 @@ acpi_parse_madt_msi(union acpi_subtable_headers *header, pr_info("applying Amazon Graviton quirk\n"); res.end = res.start + SZ_8K - 1; flags |= GICV2M_GRAVITON_ADDRESS_ONLY; - gicv2m_msi_domain_info.flags &= ~MSI_FLAG_MULTI_PCI_MSI; + gicv2m_msi_parent_ops.supported_flags &= ~MSI_FLAG_MULTI_PCI_MSI; } if (m->flags & ACPI_MADT_OVERRIDE_SPI_VALUES) { diff --git a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c index 8e87fc35f8aa..b5785472765a 100644 --- a/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c +++ b/drivers/irqchip/irq-gic-v3-its-fsl-mc-msi.c @@ -30,7 +30,7 @@ static u32 fsl_mc_msi_domain_get_msi_id(struct irq_domain *domain, u32 out_id; of_node = irq_domain_get_of_node(domain); - out_id = of_node ? of_msi_map_id(&mc_dev->dev, of_node, mc_dev->icid) : + out_id = of_node ? of_msi_xlate(&mc_dev->dev, &of_node, mc_dev->icid) : iort_msi_map_id(&mc_dev->dev, mc_dev->icid); return out_id; @@ -152,7 +152,7 @@ static void __init its_fsl_mc_of_msi_init(void) if (!of_property_read_bool(np, "msi-controller")) continue; - its_fsl_mc_msi_init_one(of_node_to_fwnode(np), + its_fsl_mc_msi_init_one(of_fwnode_handle(np), np->full_name); } } diff --git a/drivers/irqchip/irq-gic-v3-its-pci-msi.c b/drivers/irqchip/irq-gic-v3-its-pci-msi.c deleted file mode 100644 index 93f77a8196da..000000000000 --- a/drivers/irqchip/irq-gic-v3-its-pci-msi.c +++ /dev/null @@ -1,202 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013-2015 ARM Limited, All Rights Reserved. - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/acpi_iort.h> -#include <linux/pci.h> -#include <linux/msi.h> -#include <linux/of.h> -#include <linux/of_irq.h> -#include <linux/of_pci.h> - -static void its_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void its_unmask_msi_irq(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip its_msi_irq_chip = { - .name = "ITS-MSI", - .irq_unmask = its_unmask_msi_irq, - .irq_mask = its_mask_msi_irq, - .irq_eoi = irq_chip_eoi_parent, -}; - -static int its_pci_msi_vec_count(struct pci_dev *pdev, void *data) -{ - int msi, msix, *count = data; - - msi = max(pci_msi_vec_count(pdev), 0); - msix = max(pci_msix_vec_count(pdev), 0); - *count += max(msi, msix); - - return 0; -} - -static int its_get_pci_alias(struct pci_dev *pdev, u16 alias, void *data) -{ - struct pci_dev **alias_dev = data; - - *alias_dev = pdev; - - return 0; -} - -static int its_pci_msi_prepare(struct irq_domain *domain, struct device *dev, - int nvec, msi_alloc_info_t *info) -{ - struct pci_dev *pdev, *alias_dev; - struct msi_domain_info *msi_info; - int alias_count = 0, minnvec = 1; - - if (!dev_is_pci(dev)) - return -EINVAL; - - msi_info = msi_get_domain_info(domain->parent); - - pdev = to_pci_dev(dev); - /* - * If pdev is downstream of any aliasing bridges, take an upper - * bound of how many other vectors could map to the same DevID. - * Also tell the ITS that the signalling will come from a proxy - * device, and that special allocation rules apply. - */ - pci_for_each_dma_alias(pdev, its_get_pci_alias, &alias_dev); - if (alias_dev != pdev) { - if (alias_dev->subordinate) - pci_walk_bus(alias_dev->subordinate, - its_pci_msi_vec_count, &alias_count); - info->flags |= MSI_ALLOC_FLAGS_PROXY_DEVICE; - } - - /* ITS specific DeviceID, as the core ITS ignores dev. */ - info->scratchpad[0].ul = pci_msi_domain_get_msi_rid(domain, pdev); - - /* - * Always allocate a power of 2, and special case device 0 for - * broken systems where the DevID is not wired (and all devices - * appear as DevID 0). For that reason, we generously allocate a - * minimum of 32 MSIs for DevID 0. If you want more because all - * your devices are aliasing to DevID 0, consider fixing your HW. - */ - nvec = max(nvec, alias_count); - if (!info->scratchpad[0].ul) - minnvec = 32; - nvec = max_t(int, minnvec, roundup_pow_of_two(nvec)); - return msi_info->ops->msi_prepare(domain->parent, dev, nvec, info); -} - -static struct msi_domain_ops its_pci_msi_ops = { - .msi_prepare = its_pci_msi_prepare, -}; - -static struct msi_domain_info its_pci_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX), - .ops = &its_pci_msi_ops, - .chip = &its_msi_irq_chip, -}; - -static struct of_device_id its_device_id[] = { - { .compatible = "arm,gic-v3-its", }, - {}, -}; - -static int __init its_pci_msi_init_one(struct fwnode_handle *handle, - const char *name) -{ - struct irq_domain *parent; - - parent = irq_find_matching_fwnode(handle, DOMAIN_BUS_NEXUS); - if (!parent || !msi_get_domain_info(parent)) { - pr_err("%s: Unable to locate ITS domain\n", name); - return -ENXIO; - } - - if (!pci_msi_create_irq_domain(handle, &its_pci_msi_domain_info, - parent)) { - pr_err("%s: Unable to create PCI domain\n", name); - return -ENOMEM; - } - - return 0; -} - -static int __init its_pci_of_msi_init(void) -{ - struct device_node *np; - - for (np = of_find_matching_node(NULL, its_device_id); np; - np = of_find_matching_node(np, its_device_id)) { - if (!of_device_is_available(np)) - continue; - if (!of_property_read_bool(np, "msi-controller")) - continue; - - if (its_pci_msi_init_one(of_node_to_fwnode(np), np->full_name)) - continue; - - pr_info("PCI/MSI: %pOF domain created\n", np); - } - - return 0; -} - -#ifdef CONFIG_ACPI - -static int __init -its_pci_msi_parse_madt(union acpi_subtable_headers *header, - const unsigned long end) -{ - struct acpi_madt_generic_translator *its_entry; - struct fwnode_handle *dom_handle; - const char *node_name; - int err = -ENXIO; - - its_entry = (struct acpi_madt_generic_translator *)header; - node_name = kasprintf(GFP_KERNEL, "ITS@0x%lx", - (long)its_entry->base_address); - dom_handle = iort_find_domain_token(its_entry->translation_id); - if (!dom_handle) { - pr_err("%s: Unable to locate ITS domain handle\n", node_name); - goto out; - } - - err = its_pci_msi_init_one(dom_handle, node_name); - if (!err) - pr_info("PCI/MSI: %s domain created\n", node_name); - -out: - kfree(node_name); - return err; -} - -static int __init its_pci_acpi_msi_init(void) -{ - acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, - its_pci_msi_parse_madt, 0); - return 0; -} -#else -static int __init its_pci_acpi_msi_init(void) -{ - return 0; -} -#endif - -static int __init its_pci_msi_init(void) -{ - its_pci_of_msi_init(); - its_pci_acpi_msi_init(); - - return 0; -} -early_initcall(its_pci_msi_init); diff --git a/drivers/irqchip/irq-gic-v3-its-platform-msi.c b/drivers/irqchip/irq-gic-v3-its-platform-msi.c deleted file mode 100644 index daa6d5053bc3..000000000000 --- a/drivers/irqchip/irq-gic-v3-its-platform-msi.c +++ /dev/null @@ -1,163 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2013-2015 ARM Limited, All Rights Reserved. - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/acpi_iort.h> -#include <linux/device.h> -#include <linux/msi.h> -#include <linux/of.h> -#include <linux/of_irq.h> - -static struct irq_chip its_pmsi_irq_chip = { - .name = "ITS-pMSI", -}; - -static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev, - u32 *dev_id) -{ - int ret, index = 0; - - /* Suck the DeviceID out of the msi-parent property */ - do { - struct of_phandle_args args; - - ret = of_parse_phandle_with_args(dev->of_node, - "msi-parent", "#msi-cells", - index, &args); - if (args.np == irq_domain_get_of_node(domain)) { - if (WARN_ON(args.args_count != 1)) - return -EINVAL; - *dev_id = args.args[0]; - break; - } - index++; - } while (!ret); - - return ret; -} - -int __weak iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id) -{ - return -1; -} - -static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev, - int nvec, msi_alloc_info_t *info) -{ - struct msi_domain_info *msi_info; - u32 dev_id; - int ret; - - msi_info = msi_get_domain_info(domain->parent); - - if (dev->of_node) - ret = of_pmsi_get_dev_id(domain, dev, &dev_id); - else - ret = iort_pmsi_get_dev_id(dev, &dev_id); - if (ret) - return ret; - - /* ITS specific DeviceID, as the core ITS ignores dev. */ - info->scratchpad[0].ul = dev_id; - - /* Allocate at least 32 MSIs, and always as a power of 2 */ - nvec = max_t(int, 32, roundup_pow_of_two(nvec)); - return msi_info->ops->msi_prepare(domain->parent, - dev, nvec, info); -} - -static struct msi_domain_ops its_pmsi_ops = { - .msi_prepare = its_pmsi_prepare, -}; - -static struct msi_domain_info its_pmsi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), - .ops = &its_pmsi_ops, - .chip = &its_pmsi_irq_chip, -}; - -static const struct of_device_id its_device_id[] = { - { .compatible = "arm,gic-v3-its", }, - {}, -}; - -static int __init its_pmsi_init_one(struct fwnode_handle *fwnode, - const char *name) -{ - struct irq_domain *parent; - - parent = irq_find_matching_fwnode(fwnode, DOMAIN_BUS_NEXUS); - if (!parent || !msi_get_domain_info(parent)) { - pr_err("%s: unable to locate ITS domain\n", name); - return -ENXIO; - } - - if (!platform_msi_create_irq_domain(fwnode, &its_pmsi_domain_info, - parent)) { - pr_err("%s: unable to create platform domain\n", name); - return -ENXIO; - } - - pr_info("Platform MSI: %s domain created\n", name); - return 0; -} - -#ifdef CONFIG_ACPI -static int __init -its_pmsi_parse_madt(union acpi_subtable_headers *header, - const unsigned long end) -{ - struct acpi_madt_generic_translator *its_entry; - struct fwnode_handle *domain_handle; - const char *node_name; - int err = -ENXIO; - - its_entry = (struct acpi_madt_generic_translator *)header; - node_name = kasprintf(GFP_KERNEL, "ITS@0x%lx", - (long)its_entry->base_address); - domain_handle = iort_find_domain_token(its_entry->translation_id); - if (!domain_handle) { - pr_err("%s: Unable to locate ITS domain handle\n", node_name); - goto out; - } - - err = its_pmsi_init_one(domain_handle, node_name); - -out: - kfree(node_name); - return err; -} - -static void __init its_pmsi_acpi_init(void) -{ - acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR, - its_pmsi_parse_madt, 0); -} -#else -static inline void its_pmsi_acpi_init(void) { } -#endif - -static void __init its_pmsi_of_init(void) -{ - struct device_node *np; - - for (np = of_find_matching_node(NULL, its_device_id); np; - np = of_find_matching_node(np, its_device_id)) { - if (!of_device_is_available(np)) - continue; - if (!of_property_read_bool(np, "msi-controller")) - continue; - - its_pmsi_init_one(of_node_to_fwnode(np), np->full_name); - } -} - -static int __init its_pmsi_init(void) -{ - its_pmsi_of_init(); - its_pmsi_acpi_init(); - return 0; -} -early_initcall(its_pmsi_init); diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index e0c2b10d154d..ada585bfa451 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -12,12 +12,14 @@ #include <linux/crash_dump.h> #include <linux/delay.h> #include <linux/efi.h> +#include <linux/genalloc.h> #include <linux/interrupt.h> #include <linux/iommu.h> #include <linux/iopoll.h> #include <linux/irqdomain.h> #include <linux/list.h> #include <linux/log2.h> +#include <linux/mem_encrypt.h> #include <linux/memblock.h> #include <linux/mm.h> #include <linux/msi.h> @@ -27,6 +29,7 @@ #include <linux/of_pci.h> #include <linux/of_platform.h> #include <linux/percpu.h> +#include <linux/set_memory.h> #include <linux/slab.h> #include <linux/syscore_ops.h> @@ -38,15 +41,14 @@ #include <asm/exception.h> #include "irq-gic-common.h" +#include "irq-gic-its-msi-parent.h" +#include <linux/irqchip/irq-msi-lib.h> #define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0) #define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1) #define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2) #define ITS_FLAGS_FORCE_NON_SHAREABLE (1ULL << 3) - -#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0) -#define RDIST_FLAGS_RD_TABLES_PREALLOCATED (1 << 1) -#define RDIST_FLAGS_FORCE_NON_SHAREABLE (1 << 2) +#define ITS_FLAGS_WORKAROUND_HISILICON_162100801 (1ULL << 4) #define RD_LOCAL_LPI_ENABLED BIT(0) #define RD_LOCAL_PENDTABLE_PREALLOCATED BIT(1) @@ -63,7 +65,8 @@ static u32 lpi_id_bits; #define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K) #define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K) -#define LPI_PROP_DEFAULT_PRIO GICD_INT_DEF_PRI +static u8 __ro_after_init lpi_prop_prio; +static struct its_node *find_4_1_its(void); /* * Collection structure - just an ID, and a redistributor address to @@ -123,6 +126,8 @@ struct its_node { int vlpi_redist_offset; }; +static DEFINE_PER_CPU(struct its_node *, local_4_1_its); + #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) @@ -167,6 +172,7 @@ struct its_device { struct its_node *its; struct event_lpi_map event_map; void *itt; + u32 itt_sz; u32 nr_ites; u32 device_id; bool shared; @@ -202,6 +208,89 @@ static DEFINE_IDA(its_vpeid_ida); #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) #define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K) +static gfp_t gfp_flags_quirk; + +static struct page *its_alloc_pages_node(int node, gfp_t gfp, + unsigned int order) +{ + struct page *page; + int ret = 0; + + page = alloc_pages_node(node, gfp | gfp_flags_quirk, order); + + if (!page) + return NULL; + + ret = set_memory_decrypted((unsigned long)page_address(page), + 1 << order); + /* + * If set_memory_decrypted() fails then we don't know what state the + * page is in, so we can't free it. Instead we leak it. + * set_memory_decrypted() will already have WARNed. + */ + if (ret) + return NULL; + + return page; +} + +static struct page *its_alloc_pages(gfp_t gfp, unsigned int order) +{ + return its_alloc_pages_node(NUMA_NO_NODE, gfp, order); +} + +static void its_free_pages(void *addr, unsigned int order) +{ + /* + * If the memory cannot be encrypted again then we must leak the pages. + * set_memory_encrypted() will already have WARNed. + */ + if (set_memory_encrypted((unsigned long)addr, 1 << order)) + return; + free_pages((unsigned long)addr, order); +} + +static struct gen_pool *itt_pool; + +static void *itt_alloc_pool(int node, int size) +{ + unsigned long addr; + struct page *page; + + if (size >= PAGE_SIZE) { + page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, get_order(size)); + + return page ? page_address(page) : NULL; + } + + do { + addr = gen_pool_alloc(itt_pool, size); + if (addr) + break; + + page = its_alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + if (!page) + break; + + gen_pool_add(itt_pool, (unsigned long)page_address(page), PAGE_SIZE, node); + } while (!addr); + + return (void *)addr; +} + +static void itt_free_pool(void *addr, int size) +{ + if (!addr) + return; + + if (size >= PAGE_SIZE) { + its_free_pages(addr, get_order(size)); + return; + } + + gen_pool_free(itt_pool, (unsigned long)addr, size); +} + /* * Skip ITSs that have no vLPIs mapped, unless we're on GICv4.1, as we * always have vSGIs mapped. @@ -211,6 +300,11 @@ static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); } +static bool rdists_support_shareable(void) +{ + return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE); +} + static u16 get_its_list(struct its_vm *vm) { struct its_node *its; @@ -619,7 +713,6 @@ static struct its_collection *its_build_mapd_cmd(struct its_node *its, u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); - itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN); its_encode_cmd(cmd, GITS_CMD_MAPD); its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); @@ -785,6 +878,7 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, struct its_cmd_block *cmd, struct its_cmd_desc *desc) { + struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe); unsigned long vpt_addr, vconf_addr; u64 target; bool alloc; @@ -794,9 +888,14 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, its_encode_valid(cmd, desc->its_vmapp_cmd.valid); if (!desc->its_vmapp_cmd.valid) { + alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); if (is_v4_1(its)) { - alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); its_encode_alloc(cmd, alloc); + /* + * Unmapping a VPE is self-synchronizing on GICv4.1, + * no need to issue a VSYNC. + */ + vpe = NULL; } goto out; @@ -809,13 +908,13 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, its_encode_vpt_addr(cmd, vpt_addr); its_encode_vpt_size(cmd, LPI_NRBITS - 1); + alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); + if (!is_v4_1(its)) goto out; vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); - alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); - its_encode_alloc(cmd, alloc); /* @@ -831,7 +930,7 @@ static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, out: its_fixup_cmd(cmd); - return valid_vpe(its, desc->its_vmapp_cmd.vpe); + return vpe; } static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, @@ -1310,7 +1409,6 @@ static void its_send_vmovp(struct its_vpe *vpe) { struct its_cmd_desc desc = {}; struct its_node *its; - unsigned long flags; int col_id = vpe->col_idx; desc.its_vmovp_cmd.vpe = vpe; @@ -1330,8 +1428,7 @@ static void its_send_vmovp(struct its_vpe *vpe) * * Wall <-- Head. */ - raw_spin_lock_irqsave(&vmovp_lock, flags); - + guard(raw_spinlock)(&vmovp_lock); desc.its_vmovp_cmd.seq_num = vmovp_seq_num++; desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm); @@ -1346,8 +1443,6 @@ static void its_send_vmovp(struct its_vpe *vpe) desc.its_vmovp_cmd.col = &its->collections[col_id]; its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); } - - raw_spin_unlock_irqrestore(&vmovp_lock, flags); } static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) @@ -1719,17 +1814,10 @@ static u64 its_irq_get_msi_base(struct its_device *its_dev) static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); - struct its_node *its; - u64 addr; - its = its_dev->its; - addr = its->get_msi_base(its_dev); - - msg->address_lo = lower_32_bits(addr); - msg->address_hi = upper_32_bits(addr); - msg->data = its_get_event_id(d); - - iommu_dma_compose_msi_msg(irq_data_get_msi_desc(d), msg); + msg->data = its_get_event_id(d); + msi_msg_set_addr(irq_data_get_msi_desc(d), msg, + its_dev->its->get_msi_base(its_dev)); } static int its_irq_set_irqchip_state(struct irq_data *d, @@ -1784,12 +1872,10 @@ static bool gic_requires_eager_mapping(void) static void its_map_vm(struct its_node *its, struct its_vm *vm) { - unsigned long flags; - if (gic_requires_eager_mapping()) return; - raw_spin_lock_irqsave(&vmovp_lock, flags); + guard(raw_spinlock_irqsave)(&vm->vmapp_lock); /* * If the VM wasn't mapped yet, iterate over the vpes and get @@ -1802,65 +1888,53 @@ static void its_map_vm(struct its_node *its, struct its_vm *vm) for (i = 0; i < vm->nr_vpes; i++) { struct its_vpe *vpe = vm->vpes[i]; - struct irq_data *d = irq_get_irq_data(vpe->irq); - /* Map the VPE to the first possible CPU */ - vpe->col_idx = cpumask_first(cpu_online_mask); - its_send_vmapp(its, vpe, true); + scoped_guard(raw_spinlock, &vpe->vpe_lock) + its_send_vmapp(its, vpe, true); + its_send_vinvall(its, vpe); - irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); } } - - raw_spin_unlock_irqrestore(&vmovp_lock, flags); } static void its_unmap_vm(struct its_node *its, struct its_vm *vm) { - unsigned long flags; - /* Not using the ITS list? Everything is always mapped. */ if (gic_requires_eager_mapping()) return; - raw_spin_lock_irqsave(&vmovp_lock, flags); + guard(raw_spinlock_irqsave)(&vm->vmapp_lock); if (!--vm->vlpi_count[its->list_nr]) { int i; - for (i = 0; i < vm->nr_vpes; i++) + for (i = 0; i < vm->nr_vpes; i++) { + guard(raw_spinlock)(&vm->vpes[i]->vpe_lock); its_send_vmapp(its, vm->vpes[i], false); + } } - - raw_spin_unlock_irqrestore(&vmovp_lock, flags); } static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); - int ret = 0; if (!info->map) return -EINVAL; - raw_spin_lock(&its_dev->event_map.vlpi_lock); - if (!its_dev->event_map.vm) { struct its_vlpi_map *maps; maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), GFP_ATOMIC); - if (!maps) { - ret = -ENOMEM; - goto out; - } + if (!maps) + return -ENOMEM; its_dev->event_map.vm = info->map->vm; its_dev->event_map.vlpi_maps = maps; } else if (its_dev->event_map.vm != info->map->vm) { - ret = -EINVAL; - goto out; + return -EINVAL; } /* Get our private copy of the mapping information */ @@ -1892,46 +1966,32 @@ static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info) its_dev->event_map.nr_vlpis++; } -out: - raw_spin_unlock(&its_dev->event_map.vlpi_lock); - return ret; + return 0; } static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); struct its_vlpi_map *map; - int ret = 0; - - raw_spin_lock(&its_dev->event_map.vlpi_lock); map = get_vlpi_map(d); - if (!its_dev->event_map.vm || !map) { - ret = -EINVAL; - goto out; - } + if (!its_dev->event_map.vm || !map) + return -EINVAL; /* Copy our mapping information to the incoming request */ *info->map = *map; -out: - raw_spin_unlock(&its_dev->event_map.vlpi_lock); - return ret; + return 0; } static int its_vlpi_unmap(struct irq_data *d) { struct its_device *its_dev = irq_data_get_irq_chip_data(d); u32 event = its_get_event_id(d); - int ret = 0; - raw_spin_lock(&its_dev->event_map.vlpi_lock); - - if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) { - ret = -EINVAL; - goto out; - } + if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) + return -EINVAL; /* Drop the virtual mapping */ its_send_discard(its_dev, event); @@ -1939,7 +1999,7 @@ static int its_vlpi_unmap(struct irq_data *d) /* and restore the physical one */ irqd_clr_forwarded_to_vcpu(d); its_send_mapti(its_dev, d->hwirq, event); - lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO | + lpi_update_config(d, 0xff, (lpi_prop_prio | LPI_PROP_ENABLED | LPI_PROP_GROUP1)); @@ -1955,9 +2015,7 @@ static int its_vlpi_unmap(struct irq_data *d) kfree(its_dev->event_map.vlpi_maps); } -out: - raw_spin_unlock(&its_dev->event_map.vlpi_lock); - return ret; + return 0; } static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info) @@ -1985,6 +2043,8 @@ static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) if (!is_v4(its_dev->its)) return -EINVAL; + guard(raw_spinlock)(&its_dev->event_map.vlpi_lock); + /* Unmap request? */ if (!info) return its_vlpi_unmap(d); @@ -2194,8 +2254,8 @@ static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids) static void gic_reset_prop_table(void *va) { - /* Priority 0xa0, Group-1, disabled */ - memset(va, LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1, LPI_PROPBASE_SZ); + /* Regular IRQ priority, Group-1, disabled */ + memset(va, lpi_prop_prio | LPI_PROP_GROUP1, LPI_PROPBASE_SZ); /* Make sure the GIC will observe the written configuration */ gic_flush_dcache_to_poc(va, LPI_PROPBASE_SZ); @@ -2205,7 +2265,8 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags) { struct page *prop_page; - prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ)); + prop_page = its_alloc_pages(gfp_flags, + get_order(LPI_PROPBASE_SZ)); if (!prop_page) return NULL; @@ -2216,8 +2277,7 @@ static struct page *its_allocate_prop_table(gfp_t gfp_flags) static void its_free_prop_table(struct page *prop_page) { - free_pages((unsigned long)page_address(prop_page), - get_order(LPI_PROPBASE_SZ)); + its_free_pages(page_address(prop_page), get_order(LPI_PROPBASE_SZ)); } static bool gic_check_reserved_range(phys_addr_t addr, unsigned long size) @@ -2339,7 +2399,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, order = get_order(GITS_BASER_PAGES_MAX * psz); } - page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); + page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); if (!page) return -ENOMEM; @@ -2352,7 +2412,7 @@ static int its_setup_baser(struct its_node *its, struct its_baser *baser, /* 52bit PA is supported only when PageSize=64K */ if (psz != SZ_64K) { pr_err("ITS: no 52bit PA support when psz=%d\n", psz); - free_pages((unsigned long)base, order); + its_free_pages(base, order); return -ENXIO; } @@ -2383,12 +2443,12 @@ retry_baser: break; } + if (!shr) + gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); + its_write_baser(its, baser, val); tmp = baser->val; - if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) - tmp &= ~GITS_BASER_SHAREABILITY_MASK; - if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) { /* * Shareability didn't stick. Just use @@ -2398,10 +2458,9 @@ retry_baser: * non-cacheable as well. */ shr = tmp & GITS_BASER_SHAREABILITY_MASK; - if (!shr) { + if (!shr) cache = GITS_BASER_nC; - gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order)); - } + goto retry_baser; } @@ -2409,7 +2468,7 @@ retry_baser: pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", &its->phys_base, its_base_type_string[type], val, tmp); - free_pages((unsigned long)base, order); + its_free_pages(base, order); return -ENXIO; } @@ -2470,8 +2529,8 @@ static bool its_parse_indirect_baser(struct its_node *its, * feature is not supported by hardware. */ new_order = max_t(u32, get_order(esz << ids), new_order); - if (new_order > MAX_ORDER) { - new_order = MAX_ORDER; + if (new_order > MAX_PAGE_ORDER) { + new_order = MAX_PAGE_ORDER; ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz); pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", &its->phys_base, its_base_type_string[type], @@ -2548,8 +2607,7 @@ static void its_free_tables(struct its_node *its) for (i = 0; i < GITS_BASER_NR_REGS; i++) { if (its->tables[i].base) { - free_pages((unsigned long)its->tables[i].base, - its->tables[i].order); + its_free_pages(its->tables[i].base, its->tables[i].order); its->tables[i].base = NULL; } } @@ -2613,6 +2671,11 @@ static int its_alloc_tables(struct its_node *its) /* erratum 24313: ignore memory access type */ cache = GITS_BASER_nCnB; + if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) { + cache = GITS_BASER_nC; + shr = 0; + } + for (i = 0; i < GITS_BASER_NR_REGS; i++) { struct its_baser *baser = its->tables + i; u64 val = its_read_baser(its, baser); @@ -2710,12 +2773,15 @@ static u64 inherit_vpe_l1_table_from_its(void) break; } val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, addr >> 12); - val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, - FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); - val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, - FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); + if (rdists_support_shareable()) { + val |= FIELD_PREP(GICR_VPROPBASER_SHAREABILITY_MASK, + FIELD_GET(GITS_BASER_SHAREABILITY_MASK, baser)); + val |= FIELD_PREP(GICR_VPROPBASER_INNER_CACHEABILITY_MASK, + FIELD_GET(GITS_BASER_INNER_CACHEABILITY_MASK, baser)); + } val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); + *this_cpu_ptr(&local_4_1_its) = its; return val; } @@ -2753,6 +2819,7 @@ static u64 inherit_vpe_l1_table_from_rd(cpumask_t **mask) gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; + *this_cpu_ptr(&local_4_1_its) = *per_cpu_ptr(&local_4_1_its, cpu); return val; } @@ -2808,7 +2875,7 @@ static bool allocate_vpe_l2_table(int cpu, u32 id) /* Allocate memory for 2nd level table */ if (!table[idx]) { - page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); + page = its_alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(psz)); if (!page) return false; @@ -2927,7 +2994,7 @@ static int allocate_vpe_l1_table(void) pr_debug("np = %d, npg = %lld, psz = %d, epp = %d, esz = %d\n", np, npg, psz, epp, esz); - page = alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); + page = its_alloc_pages(GFP_ATOMIC | __GFP_ZERO, get_order(np * PAGE_SIZE)); if (!page) return -ENOMEM; @@ -2936,8 +3003,10 @@ static int allocate_vpe_l1_table(void) WARN_ON(!IS_ALIGNED(pa, psz)); val |= FIELD_PREP(GICR_VPROPBASER_4_1_ADDR, pa >> 12); - val |= GICR_VPROPBASER_RaWb; - val |= GICR_VPROPBASER_InnerShareable; + if (rdists_support_shareable()) { + val |= GICR_VPROPBASER_RaWb; + val |= GICR_VPROPBASER_InnerShareable; + } val |= GICR_VPROPBASER_4_1_Z; val |= GICR_VPROPBASER_4_1_VALID; @@ -2971,8 +3040,7 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags) { struct page *pend_page; - pend_page = alloc_pages(gfp_flags | __GFP_ZERO, - get_order(LPI_PENDBASE_SZ)); + pend_page = its_alloc_pages(gfp_flags | __GFP_ZERO, get_order(LPI_PENDBASE_SZ)); if (!pend_page) return NULL; @@ -2984,7 +3052,7 @@ static struct page *its_allocate_pending_table(gfp_t gfp_flags) static void its_free_pending_table(struct page *pt) { - free_pages((unsigned long)page_address(pt), get_order(LPI_PENDBASE_SZ)); + its_free_pages(page_address(pt), get_order(LPI_PENDBASE_SZ)); } /* @@ -3126,7 +3194,7 @@ static void its_cpu_init_lpis(void) gicr_write_propbaser(val, rbase + GICR_PROPBASER); tmp = gicr_read_propbaser(rbase + GICR_PROPBASER); - if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE) + if (!rdists_support_shareable()) tmp &= ~GICR_PROPBASER_SHAREABILITY_MASK; if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) { @@ -3153,7 +3221,7 @@ static void its_cpu_init_lpis(void) gicr_write_pendbaser(val, rbase + GICR_PENDBASER); tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER); - if (gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE) + if (!rdists_support_shareable()) tmp &= ~GICR_PENDBASER_SHAREABILITY_MASK; if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) { @@ -3172,6 +3240,7 @@ static void its_cpu_init_lpis(void) val |= GICR_CTLR_ENABLE_LPIS; writel_relaxed(val, rbase + GICR_CTLR); +out: if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { void __iomem *vlpi_base = gic_data_rdist_vlpi_base(); @@ -3207,7 +3276,6 @@ static void its_cpu_init_lpis(void) /* Make sure the GIC has seen the above */ dsb(sy); -out: gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; pr_info("GICv3: CPU%d: using %s LPI pending table @%pa\n", smp_processor_id(), @@ -3319,8 +3387,8 @@ static bool its_alloc_table_entry(struct its_node *its, /* Allocate memory for 2nd level table */ if (!table[idx]) { - page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, - get_order(baser->psz)); + page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, + get_order(baser->psz)); if (!page) return false; @@ -3415,15 +3483,18 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, if (WARN_ON(!is_power_of_2(nvecs))) nvecs = roundup_pow_of_two(nvecs); - dev = kzalloc(sizeof(*dev), GFP_KERNEL); /* * Even if the device wants a single LPI, the ITT must be * sized as a power of two (and you need at least one bit...). */ nr_ites = max(2, nvecs); sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); - sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1; - itt = kzalloc_node(sz, GFP_KERNEL, its->numa_node); + sz = max(sz, ITS_ITT_ALIGN); + + itt = itt_alloc_pool(its->numa_node, sz); + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (alloc_lpis) { lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis); if (lpi_map) @@ -3435,9 +3506,9 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, lpi_base = 0; } - if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { + if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) { kfree(dev); - kfree(itt); + itt_free_pool(itt, sz); bitmap_free(lpi_map); kfree(col_map); return NULL; @@ -3447,6 +3518,7 @@ static struct its_device *its_create_device(struct its_node *its, u32 dev_id, dev->its = its; dev->itt = itt; + dev->itt_sz = sz; dev->nr_ites = nr_ites; dev->event_map.lpi_map = lpi_map; dev->event_map.col_map = col_map; @@ -3474,7 +3546,7 @@ static void its_free_device(struct its_device *its_dev) list_del(&its_dev->entry); raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); kfree(its_dev->event_map.col_map); - kfree(its_dev->itt); + itt_free_pool(its_dev->itt, its_dev->itt_sz); kfree(its_dev); } @@ -3553,8 +3625,33 @@ out: return err; } +static void its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info) +{ + struct its_device *its_dev = info->scratchpad[0].ptr; + + guard(mutex)(&its_dev->its->dev_alloc_lock); + + /* If the device is shared, keep everything around */ + if (its_dev->shared) + return; + + /* LPIs should have been already unmapped at this stage */ + if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map.lpi_map, + its_dev->event_map.nr_lpis))) + return; + + its_lpi_free(its_dev->event_map.lpi_map, + its_dev->event_map.lpi_base, + its_dev->event_map.nr_lpis); + + /* Unmap device/itt, and get rid of the tracking */ + its_send_mapd(its_dev, 0); + its_free_device(its_dev); +} + static struct msi_domain_ops its_msi_domain_ops = { .msi_prepare = its_msi_prepare, + .msi_teardown = its_msi_teardown, }; static int its_irq_gic_domain_alloc(struct irq_domain *domain, @@ -3655,7 +3752,6 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, { struct irq_data *d = irq_domain_get_irq_data(domain, virq); struct its_device *its_dev = irq_data_get_irq_chip_data(d); - struct its_node *its = its_dev->its; int i; bitmap_release_region(its_dev->event_map.lpi_map, @@ -3669,30 +3765,11 @@ static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq, irq_domain_reset_irq_data(data); } - mutex_lock(&its->dev_alloc_lock); - - /* - * If all interrupts have been freed, start mopping the - * floor. This is conditioned on the device not being shared. - */ - if (!its_dev->shared && - bitmap_empty(its_dev->event_map.lpi_map, - its_dev->event_map.nr_lpis)) { - its_lpi_free(its_dev->event_map.lpi_map, - its_dev->event_map.lpi_base, - its_dev->event_map.nr_lpis); - - /* Unmap device/itt */ - its_send_mapd(its_dev, 0); - its_free_device(its_dev); - } - - mutex_unlock(&its->dev_alloc_lock); - irq_domain_free_irqs_parent(domain, virq, nr_irqs); } static const struct irq_domain_ops its_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = its_irq_domain_alloc, .free = its_irq_domain_free, .activate = its_irq_domain_activate, @@ -3812,15 +3889,48 @@ static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to) raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags); } +static void its_vpe_4_1_invall_locked(int cpu, struct its_vpe *vpe) +{ + void __iomem *rdbase; + u64 val; + + val = GICR_INVALLR_V; + val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); + + guard(raw_spinlock)(&gic_data_rdist_cpu(cpu)->rd_lock); + rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; + gic_write_lpir(val, rdbase + GICR_INVALLR); + wait_for_syncr(rdbase); +} + static int its_vpe_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { struct its_vpe *vpe = irq_data_get_irq_chip_data(d); - int from, cpu = cpumask_first(mask_val); + unsigned int from, cpu = nr_cpu_ids; + struct cpumask *table_mask; + struct its_node *its; unsigned long flags; /* + * Check if we're racing against a VPE being destroyed, for + * which we don't want to allow a VMOVP. + */ + if (!atomic_read(&vpe->vmapp_count)) { + if (gic_requires_eager_mapping()) + return -EINVAL; + + /* + * If we lazily map the VPEs, this isn't an error and + * we can exit cleanly. + */ + cpu = cpumask_first(mask_val); + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + return IRQ_SET_MASK_OK_DONE; + } + + /* * Changing affinity is mega expensive, so let's be as lazy as * we can and only do it if we really have to. Also, if mapped * into the proxy device, we need to move the doorbell @@ -3832,28 +3942,51 @@ static int its_vpe_set_affinity(struct irq_data *d, * protect us, and that we must ensure nobody samples vpe->col_idx * during the update, hence the lock below which must also be * taken on any vLPI handling path that evaluates vpe->col_idx. + * + * Finally, we must protect ourselves against concurrent updates of + * the mapping state on this VM should the ITS list be in use (see + * the shortcut in its_send_vmovp() otherewise). */ - from = vpe_to_cpuid_lock(vpe, &flags); - if (from == cpu) - goto out; + if (its_list_map) + raw_spin_lock(&vpe->its_vm->vmapp_lock); - vpe->col_idx = cpu; + from = vpe_to_cpuid_lock(vpe, &flags); + table_mask = gic_data_rdist_cpu(from)->vpe_table_mask; /* - * GICv4.1 allows us to skip VMOVP if moving to a cpu whose RD - * is sharing its VPE table with the current one. + * If we are offered another CPU in the same GICv4.1 ITS + * affinity, pick this one. Otherwise, any CPU will do. */ - if (gic_data_rdist_cpu(cpu)->vpe_table_mask && - cpumask_test_cpu(from, gic_data_rdist_cpu(cpu)->vpe_table_mask)) + if (table_mask) + cpu = cpumask_any_and(mask_val, table_mask); + if (cpu < nr_cpu_ids) { + if (cpumask_test_cpu(from, mask_val) && + cpumask_test_cpu(from, table_mask)) + cpu = from; + } else { + cpu = cpumask_first(mask_val); + } + + if (from == cpu) goto out; + vpe->col_idx = cpu; + its_send_vmovp(vpe); + + its = find_4_1_its(); + if (its && its->flags & ITS_FLAGS_WORKAROUND_HISILICON_162100801) + its_vpe_4_1_invall_locked(cpu, vpe); + its_vpe_db_proxy_move(vpe, from, cpu); out: irq_data_update_effective_affinity(d, cpumask_of(cpu)); vpe_to_cpuid_unlock(vpe, flags); + if (its_list_map) + raw_spin_unlock(&vpe->its_vm->vmapp_lock); + return IRQ_SET_MASK_OK_DONE; } @@ -3880,14 +4013,18 @@ static void its_vpe_schedule(struct its_vpe *vpe) val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & GENMASK_ULL(51, 12); val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; - val |= GICR_VPROPBASER_RaWb; - val |= GICR_VPROPBASER_InnerShareable; + if (rdists_support_shareable()) { + val |= GICR_VPROPBASER_RaWb; + val |= GICR_VPROPBASER_InnerShareable; + } gicr_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER); val = virt_to_phys(page_address(vpe->vpt_page)) & GENMASK_ULL(51, 16); - val |= GICR_VPENDBASER_RaWaWb; - val |= GICR_VPENDBASER_InnerShareable; + if (rdists_support_shareable()) { + val |= GICR_VPENDBASER_RaWaWb; + val |= GICR_VPENDBASER_InnerShareable; + } /* * There is no good way of finding out if the pending table is * empty as we can race against the doorbell interrupt very @@ -3918,6 +4055,8 @@ static void its_vpe_invall(struct its_vpe *vpe) { struct its_node *its; + guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock); + list_for_each_entry(its, &its_nodes, entry) { if (!is_v4(its)) continue; @@ -4050,7 +4189,7 @@ static struct irq_chip its_vpe_irq_chip = { static struct its_node *find_4_1_its(void) { - static struct its_node *its = NULL; + struct its_node *its = *this_cpu_ptr(&local_4_1_its); if (!its) { list_for_each_entry(its, &its_nodes, entry) { @@ -4146,22 +4285,12 @@ static void its_vpe_4_1_deschedule(struct its_vpe *vpe, static void its_vpe_4_1_invall(struct its_vpe *vpe) { - void __iomem *rdbase; unsigned long flags; - u64 val; int cpu; - val = GICR_INVALLR_V; - val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); - /* Target the redistributor this vPE is currently known on */ cpu = vpe_to_cpuid_lock(vpe, &flags); - raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); - rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; - gic_write_lpir(val, rdbase + GICR_INVALLR); - - wait_for_syncr(rdbase); - raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); + its_vpe_4_1_invall_locked(cpu, vpe); vpe_to_cpuid_unlock(vpe, flags); } @@ -4419,12 +4548,12 @@ static const struct irq_domain_ops its_sgi_domain_ops = { static int its_vpe_id_alloc(void) { - return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL); + return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL); } static void its_vpe_id_free(u16 id) { - ida_simple_remove(&its_vpeid_ida, id); + ida_free(&its_vpeid_ida, id); } static int its_vpe_init(struct its_vpe *vpe) @@ -4453,9 +4582,8 @@ static int its_vpe_init(struct its_vpe *vpe) raw_spin_lock_init(&vpe->vpe_lock); vpe->vpe_id = vpe_id; vpe->vpt_page = vpt_page; - if (gic_rdists->has_rvpeid) - atomic_set(&vpe->vmapp_count, 0); - else + atomic_set(&vpe->vmapp_count, 0); + if (!gic_rdists->has_rvpeid) vpe->vpe_proxy_event = -1; return 0; @@ -4504,8 +4632,6 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq struct page *vprop_page; int base, nr_ids, i, err = 0; - BUG_ON(!vm); - bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids); if (!bitmap) return -ENOMEM; @@ -4525,6 +4651,7 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq vm->db_lpi_base = base; vm->nr_db_lpis = nr_ids; vm->vprop_page = vprop_page; + raw_spin_lock_init(&vm->vmapp_lock); if (gic_rdists->has_rvpeid) irqchip = &its_vpe_4_1_irq_chip; @@ -4544,13 +4671,8 @@ static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq irqd_set_resend_when_in_progress(irq_get_irq_data(virq + i)); } - if (err) { - if (i > 0) - its_vpe_irq_domain_free(domain, virq, i); - - its_lpi_free(bitmap, base, nr_ids); - its_free_prop_table(vprop_page); - } + if (err) + its_vpe_irq_domain_free(domain, virq, i); return err; } @@ -4561,6 +4683,10 @@ static int its_vpe_irq_domain_activate(struct irq_domain *domain, struct its_vpe *vpe = irq_data_get_irq_chip_data(d); struct its_node *its; + /* Map the VPE to the first possible CPU */ + vpe->col_idx = cpumask_first(cpu_online_mask); + irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); + /* * If we use the list map, we issue VMAPP on demand... Unless * we're on a GICv4.1 and we eagerly map the VPE on all ITSs @@ -4569,9 +4695,6 @@ static int its_vpe_irq_domain_activate(struct irq_domain *domain, if (!gic_requires_eager_mapping()) return 0; - /* Map the VPE to the first possible CPU */ - vpe->col_idx = cpumask_first(cpu_online_mask); - list_for_each_entry(its, &its_nodes, entry) { if (!is_v4(its)) continue; @@ -4580,8 +4703,6 @@ static int its_vpe_irq_domain_activate(struct irq_domain *domain, its_send_vinvall(its, vpe); } - irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); - return 0; } @@ -4754,6 +4875,33 @@ static bool __maybe_unused its_enable_rk3588001(void *data) return true; } +static bool its_set_non_coherent(void *data) +{ + struct its_node *its = data; + + its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; + return true; +} + +static bool __maybe_unused its_enable_quirk_hip09_162100801(void *data) +{ + struct its_node *its = data; + + its->flags |= ITS_FLAGS_WORKAROUND_HISILICON_162100801; + return true; +} + +static bool __maybe_unused its_enable_rk3568002(void *data) +{ + if (!of_machine_is_compatible("rockchip,rk3566") && + !of_machine_is_compatible("rockchip,rk3568")) + return false; + + gfp_flags_quirk |= GFP_DMA32; + + return true; +} + static const struct gic_quirk its_quirks[] = { #ifdef CONFIG_CAVIUM_ERRATUM_22375 { @@ -4800,6 +4948,14 @@ static const struct gic_quirk its_quirks[] = { .init = its_enable_quirk_hip07_161600802, }, #endif +#ifdef CONFIG_HISILICON_ERRATUM_162100801 + { + .desc = "ITS: Hip09 erratum 162100801", + .iidr = 0x00051736, + .mask = 0xffffffff, + .init = its_enable_quirk_hip09_162100801, + }, +#endif #ifdef CONFIG_ROCKCHIP_ERRATUM_3588001 { .desc = "ITS: Rockchip erratum RK3588001", @@ -4809,6 +4965,19 @@ static const struct gic_quirk its_quirks[] = { }, #endif { + .desc = "ITS: non-coherent attribute", + .property = "dma-noncoherent", + .init = its_set_non_coherent, + }, +#ifdef CONFIG_ROCKCHIP_ERRATUM_3568002 + { + .desc = "ITS: Rockchip erratum RK3568002", + .iidr = 0x0201743b, + .mask = 0xffffffff, + .init = its_enable_rk3568002, + }, +#endif + { } }; @@ -4817,9 +4986,13 @@ static void its_enable_quirks(struct its_node *its) u32 iidr = readl_relaxed(its->base + GITS_IIDR); gic_enable_quirks(iidr, its_quirks, its); + + if (is_of_node(its->fwnode_handle)) + gic_enable_of_quirks(to_of_node(its->fwnode_handle), + its_quirks, its); } -static int its_save_disable(void) +static int its_save_disable(void *data) { struct its_node *its; int err = 0; @@ -4855,7 +5028,7 @@ err: return err; } -static void its_restore_enable(void) +static void its_restore_enable(void *data) { struct its_node *its; int ret; @@ -4915,11 +5088,15 @@ static void its_restore_enable(void) raw_spin_unlock(&its_lock); } -static struct syscore_ops its_syscore_ops = { +static const struct syscore_ops its_syscore_ops = { .suspend = its_save_disable, .resume = its_restore_enable, }; +static struct syscore its_syscore = { + .ops = &its_syscore_ops, +}; + static void __init __iomem *its_map_one(struct resource *res, int *err) { void __iomem *its_base; @@ -4952,9 +5129,14 @@ out_unmap: return NULL; } -static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) +static int its_init_domain(struct its_node *its) { - struct irq_domain *inner_domain; + struct irq_domain_info dom_info = { + .fwnode = its->fwnode_handle, + .ops = &its_domain_ops, + .domain_flags = its->msi_domain_flags, + .parent = its_parent, + }; struct msi_domain_info *info; info = kzalloc(sizeof(*info), GFP_KERNEL); @@ -4963,18 +5145,12 @@ static int its_init_domain(struct fwnode_handle *handle, struct its_node *its) info->ops = &its_msi_domain_ops; info->data = its; + dom_info.host_data = info; - inner_domain = irq_domain_create_hierarchy(its_parent, - its->msi_domain_flags, 0, - handle, &its_domain_ops, - info); - if (!inner_domain) { + if (!msi_create_parent_irq_domain(&dom_info, &gic_v3_its_msi_parent_ops)) { kfree(info); return -ENOMEM; } - - irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS); - return 0; } @@ -5017,8 +5193,7 @@ static int its_init_vpe_domain(void) return 0; } -static int __init its_compute_its_list_map(struct resource *res, - void __iomem *its_base) +static int __init its_compute_its_list_map(struct its_node *its) { int its_number; u32 ctlr; @@ -5032,15 +5207,15 @@ static int __init its_compute_its_list_map(struct resource *res, its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX); if (its_number >= GICv4_ITS_LIST_MAX) { pr_err("ITS@%pa: No ITSList entry available!\n", - &res->start); + &its->phys_base); return -EINVAL; } - ctlr = readl_relaxed(its_base + GITS_CTLR); + ctlr = readl_relaxed(its->base + GITS_CTLR); ctlr &= ~GITS_CTLR_ITS_NUMBER; ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT; - writel_relaxed(ctlr, its_base + GITS_CTLR); - ctlr = readl_relaxed(its_base + GITS_CTLR); + writel_relaxed(ctlr, its->base + GITS_CTLR); + ctlr = readl_relaxed(its->base + GITS_CTLR); if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) { its_number = ctlr & GITS_CTLR_ITS_NUMBER; its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT; @@ -5048,88 +5223,61 @@ static int __init its_compute_its_list_map(struct resource *res, if (test_and_set_bit(its_number, &its_list_map)) { pr_err("ITS@%pa: Duplicate ITSList entry %d\n", - &res->start, its_number); + &its->phys_base, its_number); return -EINVAL; } return its_number; } -static int __init its_probe_one(struct resource *res, - struct fwnode_handle *handle, int numa_node) +static int __init its_probe_one(struct its_node *its) { - struct its_node *its; - void __iomem *its_base; - u64 baser, tmp, typer; + u64 baser, tmp; struct page *page; u32 ctlr; int err; - its_base = its_map_one(res, &err); - if (!its_base) - return err; - - pr_info("ITS %pR\n", res); - - its = kzalloc(sizeof(*its), GFP_KERNEL); - if (!its) { - err = -ENOMEM; - goto out_unmap; - } + its_enable_quirks(its); - raw_spin_lock_init(&its->lock); - mutex_init(&its->dev_alloc_lock); - INIT_LIST_HEAD(&its->entry); - INIT_LIST_HEAD(&its->its_device_list); - typer = gic_read_typer(its_base + GITS_TYPER); - its->typer = typer; - its->base = its_base; - its->phys_base = res->start; if (is_v4(its)) { - if (!(typer & GITS_TYPER_VMOVP)) { - err = its_compute_its_list_map(res, its_base); + if (!(its->typer & GITS_TYPER_VMOVP)) { + err = its_compute_its_list_map(its); if (err < 0) - goto out_free_its; + goto out; its->list_nr = err; pr_info("ITS@%pa: Using ITS number %d\n", - &res->start, err); + &its->phys_base, err); } else { - pr_info("ITS@%pa: Single VMOVP capable\n", &res->start); + pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base); } if (is_v4_1(its)) { - u32 svpet = FIELD_GET(GITS_TYPER_SVPET, typer); + u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); - its->sgir_base = ioremap(res->start + SZ_128K, SZ_64K); + its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K); if (!its->sgir_base) { err = -ENOMEM; - goto out_free_its; + goto out; } - its->mpidr = readl_relaxed(its_base + GITS_MPIDR); + its->mpidr = readl_relaxed(its->base + GITS_MPIDR); pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", - &res->start, its->mpidr, svpet); + &its->phys_base, its->mpidr, svpet); } } - its->numa_node = numa_node; - - page = alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, - get_order(ITS_CMD_QUEUE_SZ)); + page = its_alloc_pages_node(its->numa_node, + GFP_KERNEL | __GFP_ZERO, + get_order(ITS_CMD_QUEUE_SZ)); if (!page) { err = -ENOMEM; goto out_unmap_sgir; } its->cmd_base = (void *)page_address(page); its->cmd_write = its->cmd_base; - its->fwnode_handle = handle; - its->get_msi_base = its_irq_get_msi_base; - its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI; - - its_enable_quirks(its); err = its_alloc_tables(its); if (err) @@ -5174,7 +5322,7 @@ static int __init its_probe_one(struct resource *res, ctlr |= GITS_CTLR_ImDe; writel_relaxed(ctlr, its->base + GITS_CTLR); - err = its_init_domain(handle, its); + err = its_init_domain(its); if (err) goto out_free_tables; @@ -5187,15 +5335,12 @@ static int __init its_probe_one(struct resource *res, out_free_tables: its_free_tables(its); out_free_cmd: - free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); + its_free_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); out_unmap_sgir: if (its->sgir_base) iounmap(its->sgir_base); -out_free_its: - kfree(its); -out_unmap: - iounmap(its_base); - pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err); +out: + pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err); return err; } @@ -5356,10 +5501,55 @@ static const struct of_device_id its_device_id[] = { {}, }; +static struct its_node __init *its_node_init(struct resource *res, + struct fwnode_handle *handle, int numa_node) +{ + void __iomem *its_base; + struct its_node *its; + int err; + + its_base = its_map_one(res, &err); + if (!its_base) + return NULL; + + pr_info("ITS %pR\n", res); + + its = kzalloc(sizeof(*its), GFP_KERNEL); + if (!its) + goto out_unmap; + + raw_spin_lock_init(&its->lock); + mutex_init(&its->dev_alloc_lock); + INIT_LIST_HEAD(&its->entry); + INIT_LIST_HEAD(&its->its_device_list); + + its->typer = gic_read_typer(its_base + GITS_TYPER); + its->base = its_base; + its->phys_base = res->start; + its->get_msi_base = its_irq_get_msi_base; + its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI | IRQ_DOMAIN_FLAG_MSI_IMMUTABLE; + + its->numa_node = numa_node; + its->fwnode_handle = handle; + + return its; + +out_unmap: + iounmap(its_base); + return NULL; +} + +static void its_node_destroy(struct its_node *its) +{ + iounmap(its->base); + kfree(its); +} + static int __init its_of_probe(struct device_node *node) { struct device_node *np; struct resource res; + int err; /* * Make sure *all* the ITS are reset before we probe any, as @@ -5369,8 +5559,6 @@ static int __init its_of_probe(struct device_node *node) */ for (np = of_find_matching_node(node, its_device_id); np; np = of_find_matching_node(np, its_device_id)) { - int err; - if (!of_device_is_available(np) || !of_property_read_bool(np, "msi-controller") || of_address_to_resource(np, 0, &res)) @@ -5383,6 +5571,8 @@ static int __init its_of_probe(struct device_node *node) for (np = of_find_matching_node(node, its_device_id); np; np = of_find_matching_node(np, its_device_id)) { + struct its_node *its; + if (!of_device_is_available(np)) continue; if (!of_property_read_bool(np, "msi-controller")) { @@ -5396,7 +5586,16 @@ static int __init its_of_probe(struct device_node *node) continue; } - its_probe_one(&res, &np->fwnode, of_node_to_nid(np)); + + its = its_node_init(&res, &np->fwnode, of_node_to_nid(np)); + if (!its) + return -ENOMEM; + + err = its_probe_one(its); + if (err) { + its_node_destroy(its); + return err; + } } return 0; } @@ -5508,6 +5707,7 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, { struct acpi_madt_generic_translator *its_entry; struct fwnode_handle *dom_handle; + struct its_node *its; struct resource res; int err; @@ -5532,11 +5732,22 @@ static int __init gic_acpi_parse_madt_its(union acpi_subtable_headers *header, goto dom_err; } - err = its_probe_one(&res, dom_handle, - acpi_get_its_numa_node(its_entry->translation_id)); + its = its_node_init(&res, dom_handle, + acpi_get_its_numa_node(its_entry->translation_id)); + if (!its) { + err = -ENOMEM; + goto node_err; + } + + if (acpi_get_madt_revision() >= 7 && + (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT)) + its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; + + err = its_probe_one(its); if (!err) return 0; +node_err: iort_deregister_domain_token(its_entry->translation_id); dom_err: irq_domain_free_fwnode(dom_handle); @@ -5602,7 +5813,7 @@ int __init its_lpi_memreserve_init(void) } int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, - struct irq_domain *parent_domain) + struct irq_domain *parent_domain, u8 irq_prio) { struct device_node *of_node; struct its_node *its; @@ -5610,8 +5821,13 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, bool has_v4_1 = false; int err; + itt_pool = gen_pool_create(get_order(ITS_ITT_ALIGN), -1); + if (!itt_pool) + return -ENOMEM; + gic_rdists = rdists; + lpi_prop_prio = irq_prio; its_parent = parent_domain; of_node = to_of_node(handle); if (of_node) @@ -5652,7 +5868,7 @@ int __init its_init(struct fwnode_handle *handle, struct rdists *rdists, } } - register_syscore_ops(&its_syscore_ops); + register_syscore(&its_syscore); return 0; } diff --git a/drivers/irqchip/irq-gic-v3-mbi.c b/drivers/irqchip/irq-gic-v3-mbi.c index dbb8b1efda44..aa11bbe8026a 100644 --- a/drivers/irqchip/irq-gic-v3-mbi.c +++ b/drivers/irqchip/irq-gic-v3-mbi.c @@ -18,6 +18,8 @@ #include <linux/irqchip/arm-gic-v3.h> +#include <linux/irqchip/irq-msi-lib.h> + struct mbi_range { u32 spi_start; u32 nr_spis; @@ -138,124 +140,79 @@ static void mbi_irq_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops mbi_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = mbi_irq_domain_alloc, .free = mbi_irq_domain_free, }; static void mbi_compose_msi_msg(struct irq_data *data, struct msi_msg *msg) { - msg[0].address_hi = upper_32_bits(mbi_phys_base + GICD_SETSPI_NSR); - msg[0].address_lo = lower_32_bits(mbi_phys_base + GICD_SETSPI_NSR); msg[0].data = data->parent_data->hwirq; - - iommu_dma_compose_msi_msg(irq_data_get_msi_desc(data), msg); -} - -#ifdef CONFIG_PCI_MSI -/* PCI-specific irqchip */ -static void mbi_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void mbi_unmask_msi_irq(struct irq_data *d) -{ - pci_msi_unmask_irq(d); - irq_chip_unmask_parent(d); -} - -static struct irq_chip mbi_msi_irq_chip = { - .name = "MSI", - .irq_mask = mbi_mask_msi_irq, - .irq_unmask = mbi_unmask_msi_irq, - .irq_eoi = irq_chip_eoi_parent, - .irq_compose_msi_msg = mbi_compose_msi_msg, -}; - -static struct msi_domain_info mbi_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX | MSI_FLAG_MULTI_PCI_MSI), - .chip = &mbi_msi_irq_chip, -}; - -static int mbi_allocate_pci_domain(struct irq_domain *nexus_domain, - struct irq_domain **pci_domain) -{ - *pci_domain = pci_msi_create_irq_domain(nexus_domain->parent->fwnode, - &mbi_msi_domain_info, - nexus_domain); - if (!*pci_domain) - return -ENOMEM; - - return 0; -} -#else -static int mbi_allocate_pci_domain(struct irq_domain *nexus_domain, - struct irq_domain **pci_domain) -{ - *pci_domain = NULL; - return 0; + msi_msg_set_addr(irq_data_get_msi_desc(data), &msg[0], + mbi_phys_base + GICD_SETSPI_NSR); } -#endif static void mbi_compose_mbi_msg(struct irq_data *data, struct msi_msg *msg) { mbi_compose_msi_msg(data, msg); - msg[1].address_hi = upper_32_bits(mbi_phys_base + GICD_CLRSPI_NSR); - msg[1].address_lo = lower_32_bits(mbi_phys_base + GICD_CLRSPI_NSR); msg[1].data = data->parent_data->hwirq; - - iommu_dma_compose_msi_msg(irq_data_get_msi_desc(data), &msg[1]); + msi_msg_set_addr(irq_data_get_msi_desc(data), &msg[1], + mbi_phys_base + GICD_CLRSPI_NSR); } -/* Platform-MSI specific irqchip */ -static struct irq_chip mbi_pmsi_irq_chip = { - .name = "pMSI", - .irq_set_type = irq_chip_set_type_parent, - .irq_compose_msi_msg = mbi_compose_mbi_msg, - .flags = IRQCHIP_SUPPORTS_LEVEL_MSI, -}; - -static struct msi_domain_ops mbi_pmsi_ops = { -}; +static bool mbi_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + switch (info->bus_token) { + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + info->chip->irq_compose_msi_msg = mbi_compose_msi_msg; + return true; + + case DOMAIN_BUS_DEVICE_MSI: + info->chip->irq_compose_msi_msg = mbi_compose_mbi_msg; + info->chip->irq_set_type = irq_chip_set_type_parent; + info->chip->flags |= IRQCHIP_SUPPORTS_LEVEL_MSI; + info->flags |= MSI_FLAG_LEVEL_CAPABLE; + return true; + + default: + WARN_ON_ONCE(1); + return false; + } +} -static struct msi_domain_info mbi_pmsi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_LEVEL_CAPABLE), - .ops = &mbi_pmsi_ops, - .chip = &mbi_pmsi_irq_chip, +#define MBI_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define MBI_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static const struct msi_parent_ops gic_v3_mbi_msi_parent_ops = { + .supported_flags = MBI_MSI_FLAGS_SUPPORTED, + .required_flags = MBI_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .prefix = "MBI-", + .init_dev_msi_info = mbi_init_dev_msi_info, }; -static int mbi_allocate_domains(struct irq_domain *parent) +static int mbi_allocate_domain(struct irq_domain *parent) { - struct irq_domain *nexus_domain, *pci_domain, *plat_domain; - int err; - - nexus_domain = irq_domain_create_hierarchy(parent, 0, 0, parent->fwnode, - &mbi_domain_ops, NULL); - if (!nexus_domain) - return -ENOMEM; - - irq_domain_update_bus_token(nexus_domain, DOMAIN_BUS_NEXUS); - - err = mbi_allocate_pci_domain(nexus_domain, &pci_domain); + struct irq_domain_info info = { + .fwnode = parent->fwnode, + .ops = &mbi_domain_ops, + .parent = parent, + }; - plat_domain = platform_msi_create_irq_domain(parent->fwnode, - &mbi_pmsi_domain_info, - nexus_domain); - - if (err || !plat_domain) { - if (plat_domain) - irq_domain_remove(plat_domain); - if (pci_domain) - irq_domain_remove(pci_domain); - irq_domain_remove(nexus_domain); - return -ENOMEM; - } - - return 0; + return msi_create_parent_irq_domain(&info, &gic_v3_mbi_msi_parent_ops) ? 0 : -ENOMEM; } int __init mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent) @@ -317,7 +274,7 @@ int __init mbi_init(struct fwnode_handle *fwnode, struct irq_domain *parent) pr_info("Using MBI frame %pa\n", &mbi_phys_base); - ret = mbi_allocate_domains(parent); + ret = mbi_allocate_domain(parent); if (ret) goto err_free_mbi; diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index eedfa8e9f077..6607ab58f72e 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -12,6 +12,7 @@ #include <linux/delay.h> #include <linux/interrupt.h> #include <linux/irqdomain.h> +#include <linux/kernel.h> #include <linux/kstrtox.h> #include <linux/of.h> #include <linux/of_address.h> @@ -19,11 +20,12 @@ #include <linux/percpu.h> #include <linux/refcount.h> #include <linux/slab.h> +#include <linux/iopoll.h> #include <linux/irqchip.h> #include <linux/irqchip/arm-gic-common.h> #include <linux/irqchip/arm-gic-v3.h> -#include <linux/irqchip/irq-partition-percpu.h> +#include <linux/irqchip/arm-gic-v3-prio.h> #include <linux/bitfield.h> #include <linux/bits.h> #include <linux/arm-smccc.h> @@ -35,14 +37,15 @@ #include "irq-gic-common.h" -#define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) +static u8 dist_prio_irq __ro_after_init = GICV3_PRIO_IRQ; +static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI; #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) #define FLAGS_WORKAROUND_CAVIUM_ERRATUM_38539 (1ULL << 1) -#define FLAGS_WORKAROUND_MTK_GICR_SAVE (1ULL << 2) -#define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 3) +#define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 2) +#define FLAGS_WORKAROUND_INSECURE (1ULL << 3) -#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) +static struct cpumask broken_rdists __read_mostly __maybe_unused; struct redist_region { void __iomem *redist_base; @@ -62,7 +65,13 @@ struct gic_chip_data { u64 flags; bool has_rss; unsigned int ppi_nr; - struct partition_desc **ppi_descs; + struct partition_affinity *parts; + unsigned int nr_parts; +}; + +struct partition_affinity { + cpumask_t mask; + struct fwnode_handle *partition_id; }; #define T241_CHIPS_MAX 4 @@ -78,6 +87,15 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); #define GIC_LINE_NR min(GICD_TYPER_SPIS(gic_data.rdists.gicd_typer), 1020U) #define GIC_ESPI_NR GICD_TYPER_ESPIS(gic_data.rdists.gicd_typer) +static bool nmi_support_forbidden; + +/* + * There are 16 SGIs, though we only actually use 8 in Linux. The other 8 SGIs + * are potentially stolen by the secure side. Some code, especially code dealing + * with hwirq IDs, is simplified by accounting for all 16. + */ +#define SGI_NR 16 + /* * The behaviours of RPR and PMR registers differ depending on the value of * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the @@ -101,32 +119,117 @@ static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); */ static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); -DEFINE_STATIC_KEY_FALSE(gic_nonsecure_priorities); -EXPORT_SYMBOL(gic_nonsecure_priorities); +static u32 gic_get_pribits(void) +{ + u32 pribits; -/* - * When the Non-secure world has access to group 0 interrupts (as a - * consequence of SCR_EL3.FIQ == 0), reading the ICC_RPR_EL1 register will - * return the Distributor's view of the interrupt priority. - * - * When GIC security is enabled (GICD_CTLR.DS == 0), the interrupt priority - * written by software is moved to the Non-secure range by the Distributor. - * - * If both are true (which is when gic_nonsecure_priorities gets enabled), - * we need to shift down the priority programmed by software to match it - * against the value returned by ICC_RPR_EL1. - */ -#define GICD_INT_RPR_PRI(priority) \ - ({ \ - u32 __priority = (priority); \ - if (static_branch_unlikely(&gic_nonsecure_priorities)) \ - __priority = 0x80 | (__priority >> 1); \ - \ - __priority; \ - }) - -/* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ -static refcount_t *ppi_nmi_refs; + pribits = gic_read_ctlr(); + pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; + pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; + pribits++; + + return pribits; +} + +static bool gic_has_group0(void) +{ + u32 val; + u32 old_pmr; + + old_pmr = gic_read_pmr(); + + /* + * Let's find out if Group0 is under control of EL3 or not by + * setting the highest possible, non-zero priority in PMR. + * + * If SCR_EL3.FIQ is set, the priority gets shifted down in + * order for the CPU interface to set bit 7, and keep the + * actual priority in the non-secure range. In the process, it + * looses the least significant bit and the actual priority + * becomes 0x80. Reading it back returns 0, indicating that + * we're don't have access to Group0. + */ + gic_write_pmr(BIT(8 - gic_get_pribits())); + val = gic_read_pmr(); + + gic_write_pmr(old_pmr); + + return val != 0; +} + +static inline bool gic_dist_security_disabled(void) +{ + return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; +} + +static bool cpus_have_security_disabled __ro_after_init; +static bool cpus_have_group0 __ro_after_init; + +static void __init gic_prio_init(void) +{ + bool ds; + + cpus_have_group0 = gic_has_group0(); + + ds = gic_dist_security_disabled(); + if ((gic_data.flags & FLAGS_WORKAROUND_INSECURE) && !ds) { + if (cpus_have_group0) { + u32 val; + + val = readl_relaxed(gic_data.dist_base + GICD_CTLR); + val |= GICD_CTLR_DS; + writel_relaxed(val, gic_data.dist_base + GICD_CTLR); + + ds = gic_dist_security_disabled(); + if (ds) + pr_warn("Broken GIC integration, security disabled\n"); + } else { + pr_warn("Broken GIC integration, pNMI forbidden\n"); + nmi_support_forbidden = true; + } + } + + cpus_have_security_disabled = ds; + + /* + * How priority values are used by the GIC depends on two things: + * the security state of the GIC (controlled by the GICD_CTLR.DS bit) + * and if Group 0 interrupts can be delivered to Linux in the non-secure + * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the + * way priorities are presented in ICC_PMR_EL1 and in the distributor: + * + * GICD_CTLR.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Distributor + * ------------------------------------------------------- + * 1 | - | unchanged | unchanged + * ------------------------------------------------------- + * 0 | 1 | non-secure | non-secure + * ------------------------------------------------------- + * 0 | 0 | unchanged | non-secure + * + * In the non-secure view reads and writes are modified: + * + * - A value written is right-shifted by one and the MSB is set, + * forcing the priority into the non-secure range. + * + * - A value read is left-shifted by one. + * + * In the first two cases, where ICC_PMR_EL1 and the interrupt priority + * are both either modified or unchanged, we can use the same set of + * priorities. + * + * In the last case, where only the interrupt priorities are modified to + * be in the non-secure range, we program the non-secure values into + * the distributor to match the PMR values we want. + */ + if (cpus_have_group0 && !cpus_have_security_disabled) { + dist_prio_irq = __gicv3_prio_to_ns(dist_prio_irq); + dist_prio_nmi = __gicv3_prio_to_ns(dist_prio_nmi); + } + + pr_info("GICD_CTLR.DS=%d, SCR_EL3.FIQ=%d\n", + cpus_have_security_disabled, + !cpus_have_group0); +} static struct gic_kvm_info gic_v3_kvm_info __initdata; static DEFINE_PER_CPU(bool, has_rss); @@ -174,11 +277,6 @@ static enum gic_intid_range get_intid_range(struct irq_data *d) return __get_intid_range(d->hwirq); } -static inline unsigned int gic_irq(struct irq_data *d) -{ - return d->hwirq; -} - static inline bool gic_irq_in_rdist(struct irq_data *d) { switch (get_intid_range(d)) { @@ -245,17 +343,13 @@ static inline void __iomem *gic_dist_base(struct irq_data *d) static void gic_do_wait_for_rwp(void __iomem *base, u32 bit) { - u32 count = 1000000; /* 1s! */ + u32 val; + int ret; - while (readl_relaxed(base + GICD_CTLR) & bit) { - count--; - if (!count) { - pr_err_ratelimited("RWP timeout, gone fishing\n"); - return; - } - cpu_relax(); - udelay(1); - } + ret = readl_relaxed_poll_timeout_atomic(base + GICD_CTLR, val, !(val & bit), + 1, USEC_PER_SEC); + if (ret == -ETIMEDOUT) + pr_err_ratelimited("RWP timeout, gone fishing\n"); } /* Wait for completion of a distributor change */ @@ -270,22 +364,11 @@ static void gic_redist_wait_for_rwp(void) gic_do_wait_for_rwp(gic_data_rdist_rd_base(), GICR_CTLR_RWP); } -#ifdef CONFIG_ARM64 - -static u64 __maybe_unused gic_read_iar(void) -{ - if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) - return gic_read_iar_cavium_thunderx(); - else - return gic_read_iar_common(); -} -#endif - static void gic_enable_redist(bool enable) { void __iomem *rbase; - u32 count = 1000000; /* 1s! */ u32 val; + int ret; if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) return; @@ -306,16 +389,13 @@ static void gic_enable_redist(bool enable) return; /* No PM support in this redistributor */ } - while (--count) { - val = readl_relaxed(rbase + GICR_WAKER); - if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) - break; - cpu_relax(); - udelay(1); - } - if (!count) + ret = readl_relaxed_poll_timeout_atomic(rbase + GICR_WAKER, val, + enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep), + 1, USEC_PER_SEC); + if (ret == -ETIMEDOUT) { pr_err_ratelimited("redistributor failed to %s...\n", enable ? "wakeup" : "sleep"); + } } /* @@ -468,6 +548,13 @@ static int gic_irq_set_irqchip_state(struct irq_data *d, } gic_poke_irq(d, reg); + + /* + * Force read-back to guarantee that the active state has taken + * effect, and won't race with a guest-driven deactivation. + */ + if (reg == GICD_ISACTIVER) + gic_peek_irq(d, reg); return 0; } @@ -507,23 +594,6 @@ static void gic_irq_set_prio(struct irq_data *d, u8 prio) writeb_relaxed(prio, base + offset + index); } -static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) -{ - switch (__get_intid_range(hwirq)) { - case PPI_RANGE: - return hwirq - 16; - case EPPI_RANGE: - return hwirq - EPPI_BASE_INTID + 16; - default: - unreachable(); - } -} - -static u32 gic_get_ppi_index(struct irq_data *d) -{ - return __gic_get_ppi_index(d->hwirq); -} - static int gic_irq_nmi_setup(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); @@ -540,23 +610,14 @@ static int gic_irq_nmi_setup(struct irq_data *d) * A secondary irq_chip should be in charge of LPI request, * it should not be possible to get there */ - if (WARN_ON(gic_irq(d) >= 8192)) + if (WARN_ON(irqd_to_hwirq(d) >= 8192)) return -EINVAL; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); - - /* Setting up PPI as NMI, only switch handler for first NMI */ - if (!refcount_inc_not_zero(&ppi_nmi_refs[idx])) { - refcount_set(&ppi_nmi_refs[idx], 1); - desc->handle_irq = handle_percpu_devid_fasteoi_nmi; - } - } else { + if (!gic_irq_in_rdist(d)) desc->handle_irq = handle_fasteoi_nmi; - } - gic_irq_set_prio(d, GICD_INT_NMI_PRI); + gic_irq_set_prio(d, dist_prio_nmi); return 0; } @@ -577,21 +638,14 @@ static void gic_irq_nmi_teardown(struct irq_data *d) * A secondary irq_chip should be in charge of LPI request, * it should not be possible to get there */ - if (WARN_ON(gic_irq(d) >= 8192)) + if (WARN_ON(irqd_to_hwirq(d) >= 8192)) return; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_ppi_index(d); - - /* Tearing down NMI, only switch handler for last NMI */ - if (refcount_dec_and_test(&ppi_nmi_refs[idx])) - desc->handle_irq = handle_percpu_devid_irq; - } else { + if (!gic_irq_in_rdist(d)) desc->handle_irq = handle_fasteoi_irq; - } - gic_irq_set_prio(d, GICD_INT_DEF_PRI); + gic_irq_set_prio(d, dist_prio_irq); } static bool gic_arm64_erratum_2941627_needed(struct irq_data *d) @@ -615,7 +669,7 @@ static bool gic_arm64_erratum_2941627_needed(struct irq_data *d) static void gic_eoi_irq(struct irq_data *d) { - write_gicreg(gic_irq(d), ICC_EOIR1_EL1); + write_gicreg(irqd_to_hwirq(d), ICC_EOIR1_EL1); isb(); if (gic_arm64_erratum_2941627_needed(d)) { @@ -635,19 +689,19 @@ static void gic_eoimode1_eoi_irq(struct irq_data *d) * No need to deactivate an LPI, or an interrupt that * is is getting forwarded to a vcpu. */ - if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) + if (irqd_to_hwirq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) return; if (!gic_arm64_erratum_2941627_needed(d)) - gic_write_dir(gic_irq(d)); + gic_write_dir(irqd_to_hwirq(d)); else gic_poke_irq(d, GICD_ICACTIVER); } static int gic_set_type(struct irq_data *d, unsigned int type) { + irq_hw_number_t irq = irqd_to_hwirq(d); enum gic_intid_range range; - unsigned int irq = gic_irq(d); void __iomem *base; u32 offset, index; int ret; @@ -670,10 +724,10 @@ static int gic_set_type(struct irq_data *d, unsigned int type) offset = convert_offset_index(d, GICD_ICFGR, &index); - ret = gic_configure_irq(index, type, base + offset, NULL); + ret = gic_configure_irq(index, type, base + offset); if (ret && (range == PPI_RANGE || range == EPPI_RANGE)) { /* Misconfigured PPIs are usually not fatal */ - pr_warn("GIC: PPI INTID%d is secure or misconfigured\n", irq); + pr_warn("GIC: PPI INTID%ld is secure or misconfigured\n", irq); ret = 0; } @@ -738,7 +792,7 @@ static void gic_deactivate_unhandled(u32 irqnr) * register state is not stale, as these may have been indirectly written * *after* exception entry. * - * (2) Deactivate the interrupt when EOI mode 1 is in use. + * (2) Execute an interrupt priority drop when EOI mode 1 is in use. */ static inline void gic_complete_ack(u32 irqnr) { @@ -753,7 +807,7 @@ static bool gic_rpr_is_nmi_prio(void) if (!gic_supports_nmi()) return false; - return unlikely(gic_read_rpr() == GICD_INT_RPR_PRI(GICD_INT_NMI_PRI)); + return unlikely(gic_read_rpr() == GICV3_PRIO_NMI); } static bool gic_irqnr_is_special(u32 irqnr) @@ -858,7 +912,7 @@ static void __gic_handle_irq_from_irqsoff(struct pt_regs *regs) __gic_handle_nmi(irqnr, regs); } -static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) { if (unlikely(gic_supports_nmi() && !interrupts_enabled(regs))) __gic_handle_irq_from_irqsoff(regs); @@ -866,44 +920,6 @@ static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs __gic_handle_irq_from_irqson(regs); } -static u32 gic_get_pribits(void) -{ - u32 pribits; - - pribits = gic_read_ctlr(); - pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; - pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; - pribits++; - - return pribits; -} - -static bool gic_has_group0(void) -{ - u32 val; - u32 old_pmr; - - old_pmr = gic_read_pmr(); - - /* - * Let's find out if Group0 is under control of EL3 or not by - * setting the highest possible, non-zero priority in PMR. - * - * If SCR_EL3.FIQ is set, the priority gets shifted down in - * order for the CPU interface to set bit 7, and keep the - * actual priority in the non-secure range. In the process, it - * looses the least significant bit and the actual priority - * becomes 0x80. Reading it back returns 0, indicating that - * we're don't have access to Group0. - */ - gic_write_pmr(BIT(8 - gic_get_pribits())); - val = gic_read_pmr(); - - gic_write_pmr(old_pmr); - - return val != 0; -} - static void __init gic_dist_init(void) { unsigned int i; @@ -937,10 +953,11 @@ static void __init gic_dist_init(void) writel_relaxed(0, base + GICD_ICFGRnE + i / 4); for (i = 0; i < GIC_ESPI_NR; i += 4) - writel_relaxed(GICD_INT_DEF_PRI_X4, base + GICD_IPRIORITYRnE + i); + writel_relaxed(REPEAT_BYTE_U32(dist_prio_irq), + base + GICD_IPRIORITYRnE + i); /* Now do the common stuff */ - gic_dist_config(base, GIC_LINE_NR, NULL); + gic_dist_config(base, GIC_LINE_NR, dist_prio_irq); val = GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1; if (gic_data.rdists.gicd_typer2 & GICD_TYPER2_nASSGIcap) { @@ -1119,20 +1136,8 @@ static void gic_update_rdist_properties(void) gic_data.rdists.has_vpend_valid_dirty ? "Valid+Dirty " : ""); } -/* Check whether it's single security state view */ -static inline bool gic_dist_security_disabled(void) -{ - return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; -} - -static void gic_cpu_sys_reg_init(void) +static void gic_cpu_sys_reg_enable(void) { - int i, cpu = smp_processor_id(); - u64 mpidr = gic_cpu_to_affinity(cpu); - u64 need_rss = MPIDR_RS(mpidr); - bool group0; - u32 pribits; - /* * Need to check that the SRE bit has actually been set. If * not, it means that SRE is disabled at EL2. We're going to @@ -1143,6 +1148,16 @@ static void gic_cpu_sys_reg_init(void) if (!gic_enable_sre()) pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); +} + +static void gic_cpu_sys_reg_init(void) +{ + int i, cpu = smp_processor_id(); + u64 mpidr = gic_cpu_to_affinity(cpu); + u64 need_rss = MPIDR_RS(mpidr); + bool group0; + u32 pribits; + pribits = gic_get_pribits(); group0 = gic_has_group0(); @@ -1152,18 +1167,14 @@ static void gic_cpu_sys_reg_init(void) write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); } else if (gic_supports_nmi()) { /* - * Mismatch configuration with boot CPU, the system is likely - * to die as interrupt masking will not work properly on all - * CPUs + * Check that all CPUs use the same priority space. * - * The boot CPU calls this function before enabling NMI support, - * and as a result we'll never see this warning in the boot path - * for that CPU. + * If there's a mismatch with the boot CPU, the system is + * likely to die as interrupt masking will not work properly on + * all CPUs. */ - if (static_branch_unlikely(&gic_nonsecure_priorities)) - WARN_ON(!group0 || gic_dist_security_disabled()); - else - WARN_ON(group0 && !gic_dist_security_disabled()); + WARN_ON(group0 != cpus_have_group0); + WARN_ON(gic_dist_security_disabled() != cpus_have_security_disabled); } /* @@ -1279,10 +1290,11 @@ static void gic_cpu_init(void) rbase = gic_data_rdist_sgi_base(); /* Configure SGIs/PPIs as non-secure Group-1 */ - for (i = 0; i < gic_data.ppi_nr + 16; i += 32) + for (i = 0; i < gic_data.ppi_nr + SGI_NR; i += 32) writel_relaxed(~0, rbase + GICR_IGROUPR0 + i / 8); - gic_cpu_config(rbase, gic_data.ppi_nr + 16, gic_redist_wait_for_rwp); + gic_cpu_config(rbase, gic_data.ppi_nr + SGI_NR, dist_prio_irq); + gic_redist_wait_for_rwp(); /* initialise system registers */ gic_cpu_sys_reg_init(); @@ -1293,8 +1305,21 @@ static void gic_cpu_init(void) #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) +/* + * gic_starting_cpu() is called after the last point where cpuhp is allowed + * to fail. So pre check for problems earlier. + */ +static int gic_check_rdist(unsigned int cpu) +{ + if (cpumask_test_cpu(cpu, &broken_rdists)) + return -EINVAL; + + return 0; +} + static int gic_starting_cpu(unsigned int cpu) { + gic_cpu_sys_reg_enable(); gic_cpu_init(); if (gic_dist_supports_lpis()) @@ -1384,6 +1409,10 @@ static void __init gic_smp_init(void) }; int base_sgi; + cpuhp_setup_state_nocalls(CPUHP_BP_PREPARE_DYN, + "irqchip/arm/gicv3:checkrdist", + gic_check_rdist, NULL); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, "irqchip/arm/gicv3:starting", gic_starting_cpu, NULL); @@ -1453,9 +1482,10 @@ static int gic_retrigger(struct irq_data *data) static int gic_cpu_pm_notifier(struct notifier_block *self, unsigned long cmd, void *v) { - if (cmd == CPU_PM_EXIT) { + if (cmd == CPU_PM_EXIT || cmd == CPU_PM_ENTER_FAILED) { if (gic_dist_security_disabled()) gic_enable_redist(true); + gic_cpu_sys_reg_enable(); gic_cpu_sys_reg_init(); } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { gic_write_grpen1(0); @@ -1587,13 +1617,6 @@ static int gic_irq_domain_translate(struct irq_domain *d, case GIC_IRQ_TYPE_LPI: /* LPI */ *hwirq = fwspec->param[1]; break; - case GIC_IRQ_TYPE_PARTITION: - *hwirq = fwspec->param[1]; - if (fwspec->param[1] >= 16) - *hwirq += EPPI_BASE_INTID - 16; - else - *hwirq += 16; - break; default: return -EINVAL; } @@ -1602,10 +1625,8 @@ static int gic_irq_domain_translate(struct irq_domain *d, /* * Make it clear that broken DTs are... broken. - * Partitioned PPIs are an unfortunate exception. */ - WARN_ON(*type == IRQ_TYPE_NONE && - fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); + WARN_ON(*type == IRQ_TYPE_NONE); return 0; } @@ -1662,38 +1683,22 @@ static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, } } -static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, - irq_hw_number_t hwirq) -{ - enum gic_intid_range range; - - if (!gic_data.ppi_descs) - return false; - - if (!is_of_node(fwspec->fwnode)) - return false; - - if (fwspec->param_count < 4 || !fwspec->param[3]) - return false; - - range = __get_intid_range(hwirq); - if (range != PPI_RANGE && range != EPPI_RANGE) - return false; - - return true; -} - static int gic_irq_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, enum irq_domain_bus_token bus_token) { - unsigned int type, ret, ppi_idx; irq_hw_number_t hwirq; + unsigned int type; + int ret; /* Not for us */ - if (fwspec->fwnode != d->fwnode) + if (fwspec->fwnode != d->fwnode) return 0; + /* Handle pure domain searches */ + if (!fwspec->param_count) + return d->bus_token == bus_token; + /* If this is not DT, then we have a single domain */ if (!is_of_node(fwspec->fwnode)) return 1; @@ -1702,60 +1707,61 @@ static int gic_irq_domain_select(struct irq_domain *d, if (WARN_ON_ONCE(ret)) return 0; - if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) - return d == gic_data.domain; - - /* - * If this is a PPI and we have a 4th (non-null) parameter, - * then we need to match the partition domain. - */ - ppi_idx = __gic_get_ppi_index(hwirq); - return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); + return d == gic_data.domain; } -static const struct irq_domain_ops gic_irq_domain_ops = { - .translate = gic_irq_domain_translate, - .alloc = gic_irq_domain_alloc, - .free = gic_irq_domain_free, - .select = gic_irq_domain_select, -}; - -static int partition_domain_translate(struct irq_domain *d, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) +static int gic_irq_get_fwspec_info(struct irq_fwspec *fwspec, struct irq_fwspec_info *info) { - unsigned long ppi_intid; - struct device_node *np; - unsigned int ppi_idx; - int ret; + const struct cpumask *mask = NULL; - if (!gic_data.ppi_descs) - return -ENOMEM; - - np = of_find_node_by_phandle(fwspec->param[3]); - if (WARN_ON(!np)) - return -EINVAL; + info->flags = 0; + info->affinity = NULL; - ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); - if (WARN_ON_ONCE(ret)) + /* ACPI is not capable of describing PPI affinity -- yet */ + if (!is_of_node(fwspec->fwnode)) return 0; - ppi_idx = __gic_get_ppi_index(ppi_intid); - ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], - of_node_to_fwnode(np)); - if (ret < 0) - return ret; + /* If the specifier provides an affinity, use it */ + if (fwspec->param_count == 4 && fwspec->param[3]) { + struct fwnode_handle *fw; + + switch (fwspec->param[0]) { + case 1: /* PPI */ + case 3: /* EPPI */ + break; + default: + return 0; + } + + fw = of_fwnode_handle(of_find_node_by_phandle(fwspec->param[3])); + if (!fw) + return -ENOENT; + + for (int i = 0; i < gic_data.nr_parts; i++) { + if (gic_data.parts[i].partition_id == fw) { + mask = &gic_data.parts[i].mask; + break; + } + } - *hwirq = ret; - *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + if (!mask) + return -ENOENT; + } else { + mask = cpu_possible_mask; + } + + info->affinity = mask; + info->flags = IRQ_FWSPEC_INFO_AFFINITY_VALID; return 0; } -static const struct irq_domain_ops partition_domain_ops = { - .translate = partition_domain_translate, +static const struct irq_domain_ops gic_irq_domain_ops = { + .translate = gic_irq_domain_translate, + .alloc = gic_irq_domain_alloc, + .free = gic_irq_domain_free, .select = gic_irq_domain_select, + .get_fwspec_info = gic_irq_get_fwspec_info, }; static bool gic_enable_quirk_msm8996(void *data) @@ -1767,15 +1773,6 @@ static bool gic_enable_quirk_msm8996(void *data) return true; } -static bool gic_enable_quirk_mtk_gicr(void *data) -{ - struct gic_chip_data *d = data; - - d->flags |= FLAGS_WORKAROUND_MTK_GICR_SAVE; - - return true; -} - static bool gic_enable_quirk_cavium_38539(void *data) { struct gic_chip_data *d = data; @@ -1857,6 +1854,26 @@ static bool gic_enable_quirk_arm64_2941627(void *data) return true; } +static bool gic_enable_quirk_rk3399(void *data) +{ + struct gic_chip_data *d = data; + + if (of_machine_is_compatible("rockchip,rk3399")) { + d->flags |= FLAGS_WORKAROUND_INSECURE; + return true; + } + + return false; +} + +static bool rd_set_non_coherent(void *data) +{ + struct gic_chip_data *d = data; + + d->rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; + return true; +} + static const struct gic_quirk gic_quirks[] = { { .desc = "GICv3: Qualcomm MSM8996 broken firmware", @@ -1869,11 +1886,6 @@ static const struct gic_quirk gic_quirks[] = { .init = gic_enable_quirk_asr8601, }, { - .desc = "GICv3: Mediatek Chromebook GICR save problem", - .property = "mediatek,broken-save-restore-fw", - .init = gic_enable_quirk_mtk_gicr, - }, - { .desc = "GICv3: HIP06 erratum 161010803", .iidr = 0x0204043b, .mask = 0xffffffff, @@ -1924,61 +1936,28 @@ static const struct gic_quirk gic_quirks[] = { .init = gic_enable_quirk_arm64_2941627, }, { + .desc = "GICv3: non-coherent attribute", + .property = "dma-noncoherent", + .init = rd_set_non_coherent, + }, + { + .desc = "GICv3: Insecure RK3399 integration", + .iidr = 0x0000043b, + .mask = 0xff000fff, + .init = gic_enable_quirk_rk3399, + }, + { } }; static void gic_enable_nmi_support(void) { - int i; - - if (!gic_prio_masking_enabled()) - return; - - if (gic_data.flags & FLAGS_WORKAROUND_MTK_GICR_SAVE) { - pr_warn("Skipping NMI enable due to firmware issues\n"); - return; - } - - ppi_nmi_refs = kcalloc(gic_data.ppi_nr, sizeof(*ppi_nmi_refs), GFP_KERNEL); - if (!ppi_nmi_refs) + if (!gic_prio_masking_enabled() || nmi_support_forbidden) return; - for (i = 0; i < gic_data.ppi_nr; i++) - refcount_set(&ppi_nmi_refs[i], 0); - pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); - /* - * How priority values are used by the GIC depends on two things: - * the security state of the GIC (controlled by the GICD_CTRL.DS bit) - * and if Group 0 interrupts can be delivered to Linux in the non-secure - * world as FIQs (controlled by the SCR_EL3.FIQ bit). These affect the - * ICC_PMR_EL1 register and the priority that software assigns to - * interrupts: - * - * GICD_CTRL.DS | SCR_EL3.FIQ | ICC_PMR_EL1 | Group 1 priority - * ----------------------------------------------------------- - * 1 | - | unchanged | unchanged - * ----------------------------------------------------------- - * 0 | 1 | non-secure | non-secure - * ----------------------------------------------------------- - * 0 | 0 | unchanged | non-secure - * - * where non-secure means that the value is right-shifted by one and the - * MSB bit set, to make it fit in the non-secure priority range. - * - * In the first two cases, where ICC_PMR_EL1 and the interrupt priority - * are both either modified or unchanged, we can use the same set of - * priorities. - * - * In the last case, where only the interrupt priorities are modified to - * be in the non-secure range, we use a different PMR value to mask IRQs - * and the rest of the values that we use remain unchanged. - */ - if (gic_has_group0() && !gic_dist_security_disabled()) - static_branch_enable(&gic_nonsecure_priorities); - static_branch_enable(&supports_pseudo_nmis); if (static_branch_likely(&supports_deactivate_key)) @@ -2059,13 +2038,16 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gic_update_rdist_properties(); + gic_cpu_sys_reg_enable(); + gic_prio_init(); gic_dist_init(); gic_cpu_init(); + gic_enable_nmi_support(); gic_smp_init(); gic_cpu_pm_init(); if (gic_dist_supports_lpis()) { - its_init(handle, &gic_data.rdists, gic_data.domain); + its_init(handle, &gic_data.rdists, gic_data.domain, dist_prio_irq); its_cpu_init(); its_lpi_memreserve_init(); } else { @@ -2073,8 +2055,6 @@ static int __init gic_init_bases(phys_addr_t dist_phys_base, gicv2m_init(handle, gic_data.domain); } - gic_enable_nmi_support(); - return 0; out_free: @@ -2106,12 +2086,7 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node) if (!parts_node) return; - gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); - if (!gic_data.ppi_descs) - goto out_put_node; - nr_parts = of_get_child_count(parts_node); - if (!nr_parts) goto out_put_node; @@ -2125,7 +2100,7 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node) part = &parts[part_idx]; - part->partition_id = of_node_to_fwnode(child_part); + part->partition_id = of_fwnode_handle(child_part); pr_info("GIC: PPI partition %pOFn[%d] { ", child_part, part_idx); @@ -2164,39 +2139,17 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node) part_idx++; } - for (i = 0; i < gic_data.ppi_nr; i++) { - unsigned int irq; - struct partition_desc *desc; - struct irq_fwspec ppi_fwspec = { - .fwnode = gic_data.fwnode, - .param_count = 3, - .param = { - [0] = GIC_IRQ_TYPE_PARTITION, - [1] = i, - [2] = IRQ_TYPE_NONE, - }, - }; - - irq = irq_create_fwspec_mapping(&ppi_fwspec); - if (WARN_ON(!irq)) - continue; - desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, - irq, &partition_domain_ops); - if (WARN_ON(!desc)) - continue; - - gic_data.ppi_descs[i] = desc; - } + gic_data.parts = parts; + gic_data.nr_parts = nr_parts; out_put_node: of_node_put(parts_node); } -static void __init gic_of_setup_kvm_info(struct device_node *node) +static void __init gic_of_setup_kvm_info(struct device_node *node, u32 nr_redist_regions) { int ret; struct resource r; - u32 gicv_idx; gic_v3_kvm_info.type = GIC_V3; @@ -2204,12 +2157,8 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) if (!gic_v3_kvm_info.maint_irq) return; - if (of_property_read_u32(node, "#redistributor-regions", - &gicv_idx)) - gicv_idx = 1; - - gicv_idx += 3; /* Also skip GICD, GICC, GICH */ - ret = of_address_to_resource(node, gicv_idx, &r); + /* Also skip GICD, GICC, GICH */ + ret = of_address_to_resource(node, nr_redist_regions + 3, &r); if (!ret) gic_v3_kvm_info.vcpu = r; @@ -2299,7 +2248,7 @@ static int __init gic_of_init(struct device_node *node, struct device_node *pare gic_populate_ppi_partitions(node); if (static_branch_likely(&supports_deactivate_key)) - gic_of_setup_kvm_info(node); + gic_of_setup_kvm_info(node, nr_redist_regions); return 0; out_unmap_rdist: @@ -2351,6 +2300,11 @@ gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, pr_err("Couldn't map GICR region @%llx\n", redist->base_address); return -ENOMEM; } + + if (acpi_get_madt_revision() >= 7 && + (redist->flags & ACPI_MADT_GICR_NON_COHERENT)) + gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; + gic_request_region(redist->base_address, redist->length, "GICR"); gic_acpi_register_redist(redist->base_address, redist_base); @@ -2367,15 +2321,34 @@ gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; void __iomem *redist_base; - /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ - if (!(gicc->flags & ACPI_MADT_ENABLED)) + /* Neither enabled or online capable means it doesn't exist, skip it */ + if (!(gicc->flags & (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) + return 0; + + /* + * Capable but disabled CPUs can be brought online later. What about + * the redistributor? ACPI doesn't want to say! + * Virtual hotplug systems can use the MADT's "always-on" GICR entries. + * Otherwise, prevent such CPUs from being brought online. + */ + if (!(gicc->flags & ACPI_MADT_ENABLED)) { + int cpu = get_cpu_for_acpi_id(gicc->uid); + + pr_warn("CPU %u's redistributor is inaccessible: this CPU can't be brought online\n", cpu); + if (cpu >= 0) + cpumask_set_cpu(cpu, &broken_rdists); return 0; + } redist_base = ioremap(gicc->gicr_base_address, size); if (!redist_base) return -ENOMEM; gic_request_region(gicc->gicr_base_address, size, "GICR"); + if (acpi_get_madt_revision() >= 7 && + (gicc->flags & ACPI_MADT_GICC_NON_COHERENT)) + gic_data.rdists.flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; + gic_acpi_register_redist(gicc->gicr_base_address, redist_base); return 0; } @@ -2416,21 +2389,15 @@ static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, /* * If GICC is enabled and has valid gicr base address, then it means - * GICR base is presented via GICC + * GICR base is presented via GICC. The redistributor is only known to + * be accessible if the GICC is marked as enabled. If this bit is not + * set, we'd need to add the redistributor at runtime, which isn't + * supported. */ - if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) { + if (gicc->flags & ACPI_MADT_ENABLED && gicc->gicr_base_address) acpi_data.enabled_rdists++; - return 0; - } - - /* - * It's perfectly valid firmware can pass disabled GICC entry, driver - * should not treat as errors, skip the entry instead of probe fail. - */ - if (!(gicc->flags & ACPI_MADT_ENABLED)) - return 0; - return -ENODEV; + return 0; } static int __init gic_acpi_count_gicr_regions(void) @@ -2486,8 +2453,8 @@ static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *hea int maint_irq_mode; static int first_madt = true; - /* Skip unusable CPUs */ - if (!(gicc->flags & ACPI_MADT_ENABLED)) + if (!(gicc->flags & + (ACPI_MADT_ENABLED | ACPI_MADT_GICC_ONLINE_CAPABLE))) return 0; maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? diff --git a/drivers/irqchip/irq-gic-v4.c b/drivers/irqchip/irq-gic-v4.c index 94d56a03b175..8455b4a5fbb0 100644 --- a/drivers/irqchip/irq-gic-v4.c +++ b/drivers/irqchip/irq-gic-v4.c @@ -8,6 +8,7 @@ #include <linux/irq.h> #include <linux/irqdomain.h> #include <linux/msi.h> +#include <linux/pid.h> #include <linux/sched.h> #include <linux/irqchip/arm-gic-v4.h> @@ -96,7 +97,7 @@ bool gic_cpuif_has_vsgi(void) fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64PFR0_EL1_GIC_SHIFT); - return fld >= 0x3; + return fld >= ID_AA64PFR0_EL1_GIC_V4P1; } #else bool gic_cpuif_has_vsgi(void) @@ -341,10 +342,10 @@ int its_get_vlpi(int irq, struct its_vlpi_map *map) return irq_set_vcpu_affinity(irq, &info); } -int its_unmap_vlpi(int irq) +void its_unmap_vlpi(int irq) { irq_clear_status_flags(irq, IRQ_DISABLE_UNLAZY); - return irq_set_vcpu_affinity(irq, NULL); + WARN_ON_ONCE(irq_set_vcpu_affinity(irq, NULL)); } int its_prop_update_vlpi(int irq, u8 config, bool inv) diff --git a/drivers/irqchip/irq-gic-v5-irs.c b/drivers/irqchip/irq-gic-v5-irs.c new file mode 100644 index 000000000000..ce2732d649a3 --- /dev/null +++ b/drivers/irqchip/irq-gic-v5-irs.c @@ -0,0 +1,827 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. + */ + +#define pr_fmt(fmt) "GICv5 IRS: " fmt + +#include <linux/kmemleak.h> +#include <linux/log2.h> +#include <linux/of.h> +#include <linux/of_address.h> + +#include <linux/irqchip.h> +#include <linux/irqchip/arm-gic-v5.h> + +/* + * Hardcoded ID_BITS limit for systems supporting only a 1-level IST + * table. Systems supporting only a 1-level IST table aren't expected + * to require more than 2^12 LPIs. Tweak as required. + */ +#define LPI_ID_BITS_LINEAR 12 + +#define IRS_FLAGS_NON_COHERENT BIT(0) + +static DEFINE_PER_CPU_READ_MOSTLY(struct gicv5_irs_chip_data *, per_cpu_irs_data); +static LIST_HEAD(irs_nodes); + +static u32 irs_readl_relaxed(struct gicv5_irs_chip_data *irs_data, + const u32 reg_offset) +{ + return readl_relaxed(irs_data->irs_base + reg_offset); +} + +static void irs_writel_relaxed(struct gicv5_irs_chip_data *irs_data, + const u32 val, const u32 reg_offset) +{ + writel_relaxed(val, irs_data->irs_base + reg_offset); +} + +static u64 irs_readq_relaxed(struct gicv5_irs_chip_data *irs_data, + const u32 reg_offset) +{ + return readq_relaxed(irs_data->irs_base + reg_offset); +} + +static void irs_writeq_relaxed(struct gicv5_irs_chip_data *irs_data, + const u64 val, const u32 reg_offset) +{ + writeq_relaxed(val, irs_data->irs_base + reg_offset); +} + +/* + * The polling wait (in gicv5_wait_for_op_s_atomic()) on a GIC register + * provides the memory barriers (through MMIO accessors) + * required to synchronize CPU and GIC access to IST memory. + */ +static int gicv5_irs_ist_synchronise(struct gicv5_irs_chip_data *irs_data) +{ + return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_IST_STATUSR, + GICV5_IRS_IST_STATUSR_IDLE, NULL); +} + +static int __init gicv5_irs_init_ist_linear(struct gicv5_irs_chip_data *irs_data, + unsigned int lpi_id_bits, + unsigned int istsz) +{ + size_t l2istsz; + u32 n, cfgr; + void *ist; + u64 baser; + int ret; + + /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ + n = max(5, lpi_id_bits + 1 + istsz); + + l2istsz = BIT(n + 1); + /* + * Check memory requirements. For a linear IST we cap the + * number of ID bits to a value that should never exceed + * kmalloc interface memory allocation limits, so this + * check is really belt and braces. + */ + if (l2istsz > KMALLOC_MAX_SIZE) { + u8 lpi_id_cap = ilog2(KMALLOC_MAX_SIZE) - 2 + istsz; + + pr_warn("Limiting LPI ID bits from %u to %u\n", + lpi_id_bits, lpi_id_cap); + lpi_id_bits = lpi_id_cap; + l2istsz = KMALLOC_MAX_SIZE; + } + + ist = kzalloc(l2istsz, GFP_KERNEL); + if (!ist) + return -ENOMEM; + + if (irs_data->flags & IRS_FLAGS_NON_COHERENT) + dcache_clean_inval_poc((unsigned long)ist, + (unsigned long)ist + l2istsz); + else + dsb(ishst); + + cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, + GICV5_IRS_IST_CFGR_STRUCTURE_LINEAR) | + FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | + FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, + GICV5_IRS_IST_CFGR_L2SZ_4K) | + FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); + irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); + + gicv5_global_data.ist.l2 = false; + + baser = (virt_to_phys(ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | + FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); + irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); + + ret = gicv5_irs_ist_synchronise(irs_data); + if (ret) { + kfree(ist); + return ret; + } + kmemleak_ignore(ist); + + return 0; +} + +static int __init gicv5_irs_init_ist_two_level(struct gicv5_irs_chip_data *irs_data, + unsigned int lpi_id_bits, + unsigned int istsz, + unsigned int l2sz) +{ + __le64 *l1ist; + u32 cfgr, n; + size_t l1sz; + u64 baser; + int ret; + + /* Taken from GICv5 specifications 10.2.1.13 IRS_IST_BASER */ + n = max(5, lpi_id_bits - ((10 - istsz) + (2 * l2sz)) + 2); + + l1sz = BIT(n + 1); + + l1ist = kzalloc(l1sz, GFP_KERNEL); + if (!l1ist) + return -ENOMEM; + + if (irs_data->flags & IRS_FLAGS_NON_COHERENT) + dcache_clean_inval_poc((unsigned long)l1ist, + (unsigned long)l1ist + l1sz); + else + dsb(ishst); + + cfgr = FIELD_PREP(GICV5_IRS_IST_CFGR_STRUCTURE, + GICV5_IRS_IST_CFGR_STRUCTURE_TWO_LEVEL) | + FIELD_PREP(GICV5_IRS_IST_CFGR_ISTSZ, istsz) | + FIELD_PREP(GICV5_IRS_IST_CFGR_L2SZ, l2sz) | + FIELD_PREP(GICV5_IRS_IST_CFGR_LPI_ID_BITS, lpi_id_bits); + irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_IST_CFGR); + + /* + * The L2SZ determine bits required at L2 level. Number of bytes + * required by metadata is reported through istsz - the number of bits + * covered by L2 entries scales accordingly. + */ + gicv5_global_data.ist.l2_size = BIT(11 + (2 * l2sz) + 1); + gicv5_global_data.ist.l2_bits = (10 - istsz) + (2 * l2sz); + gicv5_global_data.ist.l1ist_addr = l1ist; + gicv5_global_data.ist.l2 = true; + + baser = (virt_to_phys(l1ist) & GICV5_IRS_IST_BASER_ADDR_MASK) | + FIELD_PREP(GICV5_IRS_IST_BASER_VALID, 0x1); + irs_writeq_relaxed(irs_data, baser, GICV5_IRS_IST_BASER); + + ret = gicv5_irs_ist_synchronise(irs_data); + if (ret) { + kfree(l1ist); + return ret; + } + + return 0; +} + +/* + * Alloc L2 IST entries on demand. + * + * Locking/serialization is guaranteed by irqdomain core code by + * taking the hierarchical domain struct irq_domain.root->mutex. + */ +int gicv5_irs_iste_alloc(const u32 lpi) +{ + struct gicv5_irs_chip_data *irs_data; + unsigned int index; + u32 l2istr, l2bits; + __le64 *l1ist; + size_t l2size; + void *l2ist; + int ret; + + if (!gicv5_global_data.ist.l2) + return 0; + + irs_data = per_cpu(per_cpu_irs_data, smp_processor_id()); + if (!irs_data) + return -ENOENT; + + l2size = gicv5_global_data.ist.l2_size; + l2bits = gicv5_global_data.ist.l2_bits; + l1ist = gicv5_global_data.ist.l1ist_addr; + index = lpi >> l2bits; + + if (FIELD_GET(GICV5_ISTL1E_VALID, le64_to_cpu(l1ist[index]))) + return 0; + + l2ist = kzalloc(l2size, GFP_KERNEL); + if (!l2ist) + return -ENOMEM; + + l1ist[index] = cpu_to_le64(virt_to_phys(l2ist) & GICV5_ISTL1E_L2_ADDR_MASK); + + if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { + dcache_clean_inval_poc((unsigned long)l2ist, + (unsigned long)l2ist + l2size); + dcache_clean_poc((unsigned long)(l1ist + index), + (unsigned long)(l1ist + index) + sizeof(*l1ist)); + } else { + dsb(ishst); + } + + l2istr = FIELD_PREP(GICV5_IRS_MAP_L2_ISTR_ID, lpi); + irs_writel_relaxed(irs_data, l2istr, GICV5_IRS_MAP_L2_ISTR); + + ret = gicv5_irs_ist_synchronise(irs_data); + if (ret) { + l1ist[index] = 0; + kfree(l2ist); + return ret; + } + kmemleak_ignore(l2ist); + + /* + * Make sure we invalidate the cache line pulled before the IRS + * had a chance to update the L1 entry and mark it valid. + */ + if (irs_data->flags & IRS_FLAGS_NON_COHERENT) { + /* + * gicv5_irs_ist_synchronise() includes memory + * barriers (MMIO accessors) required to guarantee that the + * following dcache invalidation is not executed before the + * IST mapping operation has completed. + */ + dcache_inval_poc((unsigned long)(l1ist + index), + (unsigned long)(l1ist + index) + sizeof(*l1ist)); + } + + return 0; +} + +/* + * Try to match the L2 IST size to the pagesize, and if this is not possible + * pick the smallest supported L2 size in order to minimise the requirement for + * physically contiguous blocks of memory as page-sized allocations are + * guaranteed to be physically contiguous, and are by definition the easiest to + * find. + * + * Fall back to the smallest supported size (in the event that the pagesize + * itself is not supported) again serves to make it easier to find physically + * contiguous blocks of memory. + */ +static unsigned int gicv5_irs_l2_sz(u32 idr2) +{ + switch (PAGE_SIZE) { + case SZ_64K: + if (GICV5_IRS_IST_L2SZ_SUPPORT_64KB(idr2)) + return GICV5_IRS_IST_CFGR_L2SZ_64K; + fallthrough; + case SZ_4K: + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + return GICV5_IRS_IST_CFGR_L2SZ_4K; + fallthrough; + case SZ_16K: + if (GICV5_IRS_IST_L2SZ_SUPPORT_16KB(idr2)) + return GICV5_IRS_IST_CFGR_L2SZ_16K; + break; + } + + if (GICV5_IRS_IST_L2SZ_SUPPORT_4KB(idr2)) + return GICV5_IRS_IST_CFGR_L2SZ_4K; + + return GICV5_IRS_IST_CFGR_L2SZ_64K; +} + +static int __init gicv5_irs_init_ist(struct gicv5_irs_chip_data *irs_data) +{ + u32 lpi_id_bits, idr2_id_bits, idr2_min_lpi_id_bits, l2_iste_sz, l2sz; + u32 l2_iste_sz_split, idr2; + bool two_levels, istmd; + u64 baser; + int ret; + + baser = irs_readq_relaxed(irs_data, GICV5_IRS_IST_BASER); + if (FIELD_GET(GICV5_IRS_IST_BASER_VALID, baser)) { + pr_err("IST is marked as valid already; cannot allocate\n"); + return -EPERM; + } + + idr2 = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); + two_levels = !!FIELD_GET(GICV5_IRS_IDR2_IST_LEVELS, idr2); + + idr2_id_bits = FIELD_GET(GICV5_IRS_IDR2_ID_BITS, idr2); + idr2_min_lpi_id_bits = FIELD_GET(GICV5_IRS_IDR2_MIN_LPI_ID_BITS, idr2); + + /* + * For two level tables we are always supporting the maximum allowed + * number of IDs. + * + * For 1-level tables, we should support a number of bits that + * is >= min_lpi_id_bits but cap it to LPI_ID_BITS_LINEAR lest + * the level 1-table gets too large and its memory allocation + * may fail. + */ + if (two_levels) { + lpi_id_bits = idr2_id_bits; + } else { + lpi_id_bits = max(LPI_ID_BITS_LINEAR, idr2_min_lpi_id_bits); + lpi_id_bits = min(lpi_id_bits, idr2_id_bits); + } + + /* + * Cap the ID bits according to the CPUIF supported ID bits + */ + lpi_id_bits = min(lpi_id_bits, gicv5_global_data.cpuif_id_bits); + + if (two_levels) + l2sz = gicv5_irs_l2_sz(idr2); + + istmd = !!FIELD_GET(GICV5_IRS_IDR2_ISTMD, idr2); + + l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_4; + + if (istmd) { + l2_iste_sz_split = FIELD_GET(GICV5_IRS_IDR2_ISTMD_SZ, idr2); + + if (lpi_id_bits < l2_iste_sz_split) + l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_8; + else + l2_iste_sz = GICV5_IRS_IST_CFGR_ISTSZ_16; + } + + /* + * Follow GICv5 specification recommendation to opt in for two + * level tables (ref: 10.2.1.14 IRS_IST_CFGR). + */ + if (two_levels && (lpi_id_bits > ((10 - l2_iste_sz) + (2 * l2sz)))) { + ret = gicv5_irs_init_ist_two_level(irs_data, lpi_id_bits, + l2_iste_sz, l2sz); + } else { + ret = gicv5_irs_init_ist_linear(irs_data, lpi_id_bits, + l2_iste_sz); + } + if (ret) + return ret; + + gicv5_init_lpis(BIT(lpi_id_bits)); + + return 0; +} + +struct iaffid_entry { + u16 iaffid; + bool valid; +}; + +static DEFINE_PER_CPU(struct iaffid_entry, cpu_iaffid); + +int gicv5_irs_cpu_to_iaffid(int cpuid, u16 *iaffid) +{ + if (!per_cpu(cpu_iaffid, cpuid).valid) { + pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); + return -ENODEV; + } + + *iaffid = per_cpu(cpu_iaffid, cpuid).iaffid; + + return 0; +} + +struct gicv5_irs_chip_data *gicv5_irs_lookup_by_spi_id(u32 spi_id) +{ + struct gicv5_irs_chip_data *irs_data; + u32 min, max; + + list_for_each_entry(irs_data, &irs_nodes, entry) { + if (!irs_data->spi_range) + continue; + + min = irs_data->spi_min; + max = irs_data->spi_min + irs_data->spi_range - 1; + if (spi_id >= min && spi_id <= max) + return irs_data; + } + + return NULL; +} + +static int gicv5_irs_wait_for_spi_op(struct gicv5_irs_chip_data *irs_data) +{ + u32 statusr; + int ret; + + ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_SPI_STATUSR, + GICV5_IRS_SPI_STATUSR_IDLE, &statusr); + if (ret) + return ret; + + return !!FIELD_GET(GICV5_IRS_SPI_STATUSR_V, statusr) ? 0 : -EIO; +} + +static int gicv5_irs_wait_for_irs_pe(struct gicv5_irs_chip_data *irs_data, + bool selr) +{ + bool valid = true; + u32 statusr; + int ret; + + ret = gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_PE_STATUSR, + GICV5_IRS_PE_STATUSR_IDLE, &statusr); + if (ret) + return ret; + + if (selr) + valid = !!FIELD_GET(GICV5_IRS_PE_STATUSR_V, statusr); + + return valid ? 0 : -EIO; +} + +static int gicv5_irs_wait_for_pe_selr(struct gicv5_irs_chip_data *irs_data) +{ + return gicv5_irs_wait_for_irs_pe(irs_data, true); +} + +static int gicv5_irs_wait_for_pe_cr0(struct gicv5_irs_chip_data *irs_data) +{ + return gicv5_irs_wait_for_irs_pe(irs_data, false); +} + +int gicv5_spi_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct gicv5_irs_chip_data *irs_data = d->chip_data; + u32 selr, cfgr; + bool level; + int ret; + + /* + * There is no distinction between HIGH/LOW for level IRQs + * and RISING/FALLING for edge IRQs in the architecture, + * hence consider them equivalent. + */ + switch (type) { + case IRQ_TYPE_EDGE_RISING: + case IRQ_TYPE_EDGE_FALLING: + level = false; + break; + case IRQ_TYPE_LEVEL_HIGH: + case IRQ_TYPE_LEVEL_LOW: + level = true; + break; + default: + return -EINVAL; + } + + guard(raw_spinlock)(&irs_data->spi_config_lock); + + selr = FIELD_PREP(GICV5_IRS_SPI_SELR_ID, d->hwirq); + irs_writel_relaxed(irs_data, selr, GICV5_IRS_SPI_SELR); + ret = gicv5_irs_wait_for_spi_op(irs_data); + if (ret) + return ret; + + cfgr = FIELD_PREP(GICV5_IRS_SPI_CFGR_TM, level); + irs_writel_relaxed(irs_data, cfgr, GICV5_IRS_SPI_CFGR); + + return gicv5_irs_wait_for_spi_op(irs_data); +} + +static int gicv5_irs_wait_for_idle(struct gicv5_irs_chip_data *irs_data) +{ + return gicv5_wait_for_op_atomic(irs_data->irs_base, GICV5_IRS_CR0, + GICV5_IRS_CR0_IDLE, NULL); +} + +void gicv5_irs_syncr(void) +{ + struct gicv5_irs_chip_data *irs_data; + u32 syncr; + + irs_data = list_first_entry_or_null(&irs_nodes, struct gicv5_irs_chip_data, entry); + if (WARN_ON_ONCE(!irs_data)) + return; + + syncr = FIELD_PREP(GICV5_IRS_SYNCR_SYNC, 1); + irs_writel_relaxed(irs_data, syncr, GICV5_IRS_SYNCR); + + gicv5_wait_for_op(irs_data->irs_base, GICV5_IRS_SYNC_STATUSR, + GICV5_IRS_SYNC_STATUSR_IDLE); +} + +int gicv5_irs_register_cpu(int cpuid) +{ + struct gicv5_irs_chip_data *irs_data; + u32 selr, cr0; + u16 iaffid; + int ret; + + ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); + if (ret) { + pr_err("IAFFID for CPU %d has not been initialised\n", cpuid); + return ret; + } + + irs_data = per_cpu(per_cpu_irs_data, cpuid); + if (!irs_data) { + pr_err("No IRS associated with CPU %u\n", cpuid); + return -ENXIO; + } + + selr = FIELD_PREP(GICV5_IRS_PE_SELR_IAFFID, iaffid); + irs_writel_relaxed(irs_data, selr, GICV5_IRS_PE_SELR); + + ret = gicv5_irs_wait_for_pe_selr(irs_data); + if (ret) { + pr_err("IAFFID 0x%x used in IRS_PE_SELR is invalid\n", iaffid); + return -ENXIO; + } + + cr0 = FIELD_PREP(GICV5_IRS_PE_CR0_DPS, 0x1); + irs_writel_relaxed(irs_data, cr0, GICV5_IRS_PE_CR0); + + ret = gicv5_irs_wait_for_pe_cr0(irs_data); + if (ret) + return ret; + + pr_debug("CPU %d enabled PE IAFFID 0x%x\n", cpuid, iaffid); + + return 0; +} + +static void __init gicv5_irs_init_bases(struct gicv5_irs_chip_data *irs_data, + void __iomem *irs_base, + struct fwnode_handle *handle) +{ + struct device_node *np = to_of_node(handle); + u32 cr0, cr1; + + irs_data->fwnode = handle; + irs_data->irs_base = irs_base; + + if (of_property_read_bool(np, "dma-noncoherent")) { + /* + * A non-coherent IRS implies that some cache levels cannot be + * used coherently by the cores and GIC. Our only option is to mark + * memory attributes for the GIC as non-cacheable; by default, + * non-cacheable memory attributes imply outer-shareable + * shareability, the value written into IRS_CR1_SH is ignored. + */ + cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_NO_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_NO_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_NO_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_NON_CACHE) | + FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_NON_CACHE); + irs_data->flags |= IRS_FLAGS_NON_COHERENT; + } else { + cr1 = FIELD_PREP(GICV5_IRS_CR1_VPED_WA, GICV5_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VPED_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMD_WA, GICV5_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMD_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VPET_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_VMT_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IST_WA, GICV5_WRITE_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IST_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_IRS_CR1_IC, GICV5_WB_CACHE) | + FIELD_PREP(GICV5_IRS_CR1_OC, GICV5_WB_CACHE) | + FIELD_PREP(GICV5_IRS_CR1_SH, GICV5_INNER_SHARE); + } + + irs_writel_relaxed(irs_data, cr1, GICV5_IRS_CR1); + + cr0 = FIELD_PREP(GICV5_IRS_CR0_IRSEN, 0x1); + irs_writel_relaxed(irs_data, cr0, GICV5_IRS_CR0); + gicv5_irs_wait_for_idle(irs_data); +} + +static int __init gicv5_irs_of_init_affinity(struct device_node *node, + struct gicv5_irs_chip_data *irs_data, + u8 iaffid_bits) +{ + /* + * Detect IAFFID<->CPU mappings from the device tree and + * record IRS<->CPU topology information. + */ + u16 iaffid_mask = GENMASK(iaffid_bits - 1, 0); + int ret, i, ncpus, niaffids; + + ncpus = of_count_phandle_with_args(node, "cpus", NULL); + if (ncpus < 0) + return -EINVAL; + + niaffids = of_property_count_elems_of_size(node, "arm,iaffids", + sizeof(u16)); + if (niaffids != ncpus) + return -EINVAL; + + u16 *iaffids __free(kfree) = kcalloc(niaffids, sizeof(*iaffids), GFP_KERNEL); + if (!iaffids) + return -ENOMEM; + + ret = of_property_read_u16_array(node, "arm,iaffids", iaffids, niaffids); + if (ret) + return ret; + + for (i = 0; i < ncpus; i++) { + struct device_node *cpu_node; + int cpu; + + cpu_node = of_parse_phandle(node, "cpus", i); + if (!cpu_node) { + pr_warn(FW_BUG "Erroneous CPU node phandle\n"); + continue; + } + + cpu = of_cpu_node_to_id(cpu_node); + of_node_put(cpu_node); + if (cpu < 0) + continue; + + if (iaffids[i] & ~iaffid_mask) { + pr_warn("CPU %d iaffid 0x%x exceeds IRS iaffid bits\n", + cpu, iaffids[i]); + continue; + } + + per_cpu(cpu_iaffid, cpu).iaffid = iaffids[i]; + per_cpu(cpu_iaffid, cpu).valid = true; + + /* We also know that the CPU is connected to this IRS */ + per_cpu(per_cpu_irs_data, cpu) = irs_data; + } + + return ret; +} + +static void irs_setup_pri_bits(u32 idr1) +{ + switch (FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)) { + case GICV5_IRS_IDR1_PRIORITY_BITS_1BITS: + gicv5_global_data.irs_pri_bits = 1; + break; + case GICV5_IRS_IDR1_PRIORITY_BITS_2BITS: + gicv5_global_data.irs_pri_bits = 2; + break; + case GICV5_IRS_IDR1_PRIORITY_BITS_3BITS: + gicv5_global_data.irs_pri_bits = 3; + break; + case GICV5_IRS_IDR1_PRIORITY_BITS_4BITS: + gicv5_global_data.irs_pri_bits = 4; + break; + case GICV5_IRS_IDR1_PRIORITY_BITS_5BITS: + gicv5_global_data.irs_pri_bits = 5; + break; + default: + pr_warn("Detected wrong IDR priority bits value 0x%lx\n", + FIELD_GET(GICV5_IRS_IDR1_PRIORITY_BITS, idr1)); + gicv5_global_data.irs_pri_bits = 1; + break; + } +} + +static int __init gicv5_irs_init(struct device_node *node) +{ + struct gicv5_irs_chip_data *irs_data; + void __iomem *irs_base; + u32 idr, spi_count; + u8 iaffid_bits; + int ret; + + irs_data = kzalloc(sizeof(*irs_data), GFP_KERNEL); + if (!irs_data) + return -ENOMEM; + + raw_spin_lock_init(&irs_data->spi_config_lock); + + ret = of_property_match_string(node, "reg-names", "ns-config"); + if (ret < 0) { + pr_err("%pOF: ns-config reg-name not present\n", node); + goto out_err; + } + + irs_base = of_io_request_and_map(node, ret, of_node_full_name(node)); + if (IS_ERR(irs_base)) { + pr_err("%pOF: unable to map GICv5 IRS registers\n", node); + ret = PTR_ERR(irs_base); + goto out_err; + } + + gicv5_irs_init_bases(irs_data, irs_base, &node->fwnode); + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); + iaffid_bits = FIELD_GET(GICV5_IRS_IDR1_IAFFID_BITS, idr) + 1; + + ret = gicv5_irs_of_init_affinity(node, irs_data, iaffid_bits); + if (ret) { + pr_err("Failed to parse CPU IAFFIDs from the device tree!\n"); + goto out_iomem; + } + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR2); + if (WARN(!FIELD_GET(GICV5_IRS_IDR2_LPI, idr), + "LPI support not available - no IPIs, can't proceed\n")) { + ret = -ENODEV; + goto out_iomem; + } + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR7); + irs_data->spi_min = FIELD_GET(GICV5_IRS_IDR7_SPI_BASE, idr); + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR6); + irs_data->spi_range = FIELD_GET(GICV5_IRS_IDR6_SPI_IRS_RANGE, idr); + + if (irs_data->spi_range) { + pr_info("%s detected SPI range [%u-%u]\n", + of_node_full_name(node), + irs_data->spi_min, + irs_data->spi_min + + irs_data->spi_range - 1); + } + + /* + * Do the global setting only on the first IRS. + * Global properties (iaffid_bits, global spi count) are guaranteed to + * be consistent across IRSes by the architecture. + */ + if (list_empty(&irs_nodes)) { + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR1); + irs_setup_pri_bits(idr); + + idr = irs_readl_relaxed(irs_data, GICV5_IRS_IDR5); + + spi_count = FIELD_GET(GICV5_IRS_IDR5_SPI_RANGE, idr); + gicv5_global_data.global_spi_count = spi_count; + + gicv5_init_lpi_domain(); + + pr_debug("Detected %u SPIs globally\n", spi_count); + } + + list_add_tail(&irs_data->entry, &irs_nodes); + + return 0; + +out_iomem: + iounmap(irs_base); +out_err: + kfree(irs_data); + return ret; +} + +void __init gicv5_irs_remove(void) +{ + struct gicv5_irs_chip_data *irs_data, *tmp_data; + + gicv5_free_lpi_domain(); + gicv5_deinit_lpis(); + + list_for_each_entry_safe(irs_data, tmp_data, &irs_nodes, entry) { + iounmap(irs_data->irs_base); + list_del(&irs_data->entry); + kfree(irs_data); + } +} + +int __init gicv5_irs_enable(void) +{ + struct gicv5_irs_chip_data *irs_data; + int ret; + + irs_data = list_first_entry_or_null(&irs_nodes, + struct gicv5_irs_chip_data, entry); + if (!irs_data) + return -ENODEV; + + ret = gicv5_irs_init_ist(irs_data); + if (ret) { + pr_err("Failed to init IST\n"); + return ret; + } + + return 0; +} + +void __init gicv5_irs_its_probe(void) +{ + struct gicv5_irs_chip_data *irs_data; + + list_for_each_entry(irs_data, &irs_nodes, entry) + gicv5_its_of_probe(to_of_node(irs_data->fwnode)); +} + +int __init gicv5_irs_of_probe(struct device_node *parent) +{ + struct device_node *np; + int ret; + + for_each_available_child_of_node(parent, np) { + if (!of_device_is_compatible(np, "arm,gic-v5-irs")) + continue; + + ret = gicv5_irs_init(np); + if (ret) + pr_err("Failed to init IRS %s\n", np->full_name); + } + + return list_empty(&irs_nodes) ? -ENODEV : 0; +} diff --git a/drivers/irqchip/irq-gic-v5-its.c b/drivers/irqchip/irq-gic-v5-its.c new file mode 100644 index 000000000000..554485f0be1f --- /dev/null +++ b/drivers/irqchip/irq-gic-v5-its.c @@ -0,0 +1,1233 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. + */ + +#define pr_fmt(fmt) "GICv5 ITS: " fmt + +#include <linux/bitmap.h> +#include <linux/iommu.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/slab.h> + +#include <linux/irqchip.h> +#include <linux/irqchip/arm-gic-v5.h> +#include <linux/irqchip/irq-msi-lib.h> + +#include "irq-gic-its-msi-parent.h" + +#define ITS_FLAGS_NON_COHERENT BIT(0) + +struct gicv5_its_chip_data { + struct xarray its_devices; + struct mutex dev_alloc_lock; + struct fwnode_handle *fwnode; + struct gicv5_its_devtab_cfg devtab_cfgr; + void __iomem *its_base; + u32 flags; + unsigned int msi_domain_flags; +}; + +struct gicv5_its_dev { + struct gicv5_its_chip_data *its_node; + struct gicv5_its_itt_cfg itt_cfg; + unsigned long *event_map; + u32 device_id; + u32 num_events; + phys_addr_t its_trans_phys_base; +}; + +static u32 its_readl_relaxed(struct gicv5_its_chip_data *its_node, const u32 reg_offset) +{ + return readl_relaxed(its_node->its_base + reg_offset); +} + +static void its_writel_relaxed(struct gicv5_its_chip_data *its_node, const u32 val, + const u32 reg_offset) +{ + writel_relaxed(val, its_node->its_base + reg_offset); +} + +static void its_writeq_relaxed(struct gicv5_its_chip_data *its_node, const u64 val, + const u32 reg_offset) +{ + writeq_relaxed(val, its_node->its_base + reg_offset); +} + +static void gicv5_its_dcache_clean(struct gicv5_its_chip_data *its, void *start, + size_t sz) +{ + void *end = start + sz; + + if (its->flags & ITS_FLAGS_NON_COHERENT) + dcache_clean_inval_poc((unsigned long)start, (unsigned long)end); + else + dsb(ishst); +} + +static void its_write_table_entry(struct gicv5_its_chip_data *its, __le64 *entry, + u64 val) +{ + WRITE_ONCE(*entry, cpu_to_le64(val)); + gicv5_its_dcache_clean(its, entry, sizeof(*entry)); +} + +#define devtab_cfgr_field(its, f) \ + FIELD_GET(GICV5_ITS_DT_CFGR_##f, (its)->devtab_cfgr.cfgr) + +static int gicv5_its_cache_sync(struct gicv5_its_chip_data *its) +{ + return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_STATUSR, + GICV5_ITS_STATUSR_IDLE, NULL); +} + +static void gicv5_its_syncr(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev) +{ + u64 syncr; + + syncr = FIELD_PREP(GICV5_ITS_SYNCR_SYNC, 1) | + FIELD_PREP(GICV5_ITS_SYNCR_DEVICEID, its_dev->device_id); + + its_writeq_relaxed(its, syncr, GICV5_ITS_SYNCR); + + gicv5_wait_for_op(its->its_base, GICV5_ITS_SYNC_STATUSR, GICV5_ITS_SYNC_STATUSR_IDLE); +} + +/* Number of bits required for each L2 {device/interrupt translation} table size */ +#define ITS_L2SZ_64K_L2_BITS 13 +#define ITS_L2SZ_16K_L2_BITS 11 +#define ITS_L2SZ_4K_L2_BITS 9 + +static unsigned int gicv5_its_l2sz_to_l2_bits(unsigned int sz) +{ + switch (sz) { + case GICV5_ITS_DT_ITT_CFGR_L2SZ_64k: + return ITS_L2SZ_64K_L2_BITS; + case GICV5_ITS_DT_ITT_CFGR_L2SZ_16k: + return ITS_L2SZ_16K_L2_BITS; + case GICV5_ITS_DT_ITT_CFGR_L2SZ_4k: + default: + return ITS_L2SZ_4K_L2_BITS; + } +} + +static int gicv5_its_itt_cache_inv(struct gicv5_its_chip_data *its, u32 device_id, + u16 event_id) +{ + u32 eventr, eidr; + u64 didr; + + didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, device_id); + eidr = FIELD_PREP(GICV5_ITS_EIDR_EVENTID, event_id); + eventr = FIELD_PREP(GICV5_ITS_INV_EVENTR_I, 0x1); + + its_writeq_relaxed(its, didr, GICV5_ITS_DIDR); + its_writel_relaxed(its, eidr, GICV5_ITS_EIDR); + its_writel_relaxed(its, eventr, GICV5_ITS_INV_EVENTR); + + return gicv5_its_cache_sync(its); +} + +static void gicv5_its_free_itt_linear(struct gicv5_its_dev *its_dev) +{ + kfree(its_dev->itt_cfg.linear.itt); +} + +static void gicv5_its_free_itt_two_level(struct gicv5_its_dev *its_dev) +{ + unsigned int i, num_ents = its_dev->itt_cfg.l2.num_l1_ents; + + for (i = 0; i < num_ents; i++) + kfree(its_dev->itt_cfg.l2.l2ptrs[i]); + + kfree(its_dev->itt_cfg.l2.l2ptrs); + kfree(its_dev->itt_cfg.l2.l1itt); +} + +static void gicv5_its_free_itt(struct gicv5_its_dev *its_dev) +{ + if (!its_dev->itt_cfg.l2itt) + gicv5_its_free_itt_linear(its_dev); + else + gicv5_its_free_itt_two_level(its_dev); +} + +static int gicv5_its_create_itt_linear(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev, + unsigned int event_id_bits) +{ + unsigned int num_ents = BIT(event_id_bits); + __le64 *itt; + + itt = kcalloc(num_ents, sizeof(*itt), GFP_KERNEL); + if (!itt) + return -ENOMEM; + + its_dev->itt_cfg.linear.itt = itt; + its_dev->itt_cfg.linear.num_ents = num_ents; + its_dev->itt_cfg.l2itt = false; + its_dev->itt_cfg.event_id_bits = event_id_bits; + + gicv5_its_dcache_clean(its, itt, num_ents * sizeof(*itt)); + + return 0; +} + +/* + * Allocate a two-level ITT. All ITT entries are allocated in one go, unlike + * with the device table. Span may be used to limit the second level table + * size, where possible. + */ +static int gicv5_its_create_itt_two_level(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev, + unsigned int event_id_bits, + unsigned int itt_l2sz, + unsigned int num_events) +{ + unsigned int l1_bits, l2_bits, span, events_per_l2_table; + unsigned int complete_tables, final_span, num_ents; + __le64 *itt_l1, *itt_l2, **l2ptrs; + int i, ret; + u64 val; + + ret = gicv5_its_l2sz_to_l2_bits(itt_l2sz); + if (ret >= event_id_bits) { + pr_debug("Incorrect l2sz (0x%x) for %u EventID bits. Cannot allocate ITT\n", + itt_l2sz, event_id_bits); + return -EINVAL; + } + + l2_bits = ret; + + l1_bits = event_id_bits - l2_bits; + + num_ents = BIT(l1_bits); + + itt_l1 = kcalloc(num_ents, sizeof(*itt_l1), GFP_KERNEL); + if (!itt_l1) + return -ENOMEM; + + l2ptrs = kcalloc(num_ents, sizeof(*l2ptrs), GFP_KERNEL); + if (!l2ptrs) { + kfree(itt_l1); + return -ENOMEM; + } + + its_dev->itt_cfg.l2.l2ptrs = l2ptrs; + + its_dev->itt_cfg.l2.l2sz = itt_l2sz; + its_dev->itt_cfg.l2.l1itt = itt_l1; + its_dev->itt_cfg.l2.num_l1_ents = num_ents; + its_dev->itt_cfg.l2itt = true; + its_dev->itt_cfg.event_id_bits = event_id_bits; + + /* + * Need to determine how many entries there are per L2 - this is based + * on the number of bits in the table. + */ + events_per_l2_table = BIT(l2_bits); + complete_tables = num_events / events_per_l2_table; + final_span = order_base_2(num_events % events_per_l2_table); + + for (i = 0; i < num_ents; i++) { + size_t l2sz; + + span = i == complete_tables ? final_span : l2_bits; + + itt_l2 = kcalloc(BIT(span), sizeof(*itt_l2), GFP_KERNEL); + if (!itt_l2) { + ret = -ENOMEM; + goto out_free; + } + + its_dev->itt_cfg.l2.l2ptrs[i] = itt_l2; + + l2sz = BIT(span) * sizeof(*itt_l2); + + gicv5_its_dcache_clean(its, itt_l2, l2sz); + + val = (virt_to_phys(itt_l2) & GICV5_ITTL1E_L2_ADDR_MASK) | + FIELD_PREP(GICV5_ITTL1E_SPAN, span) | + FIELD_PREP(GICV5_ITTL1E_VALID, 0x1); + + WRITE_ONCE(itt_l1[i], cpu_to_le64(val)); + } + + gicv5_its_dcache_clean(its, itt_l1, num_ents * sizeof(*itt_l1)); + + return 0; + +out_free: + for (i = i - 1; i >= 0; i--) + kfree(its_dev->itt_cfg.l2.l2ptrs[i]); + + kfree(its_dev->itt_cfg.l2.l2ptrs); + kfree(itt_l1); + return ret; +} + +/* + * Function to check whether the device table or ITT table support + * a two-level table and if so depending on the number of id_bits + * requested, determine whether a two-level table is required. + * + * Return the 2-level size value if a two level table is deemed + * necessary. + */ +static bool gicv5_its_l2sz_two_level(bool devtab, u32 its_idr1, u8 id_bits, u8 *sz) +{ + unsigned int l2_bits, l2_sz; + + if (devtab && !FIELD_GET(GICV5_ITS_IDR1_DT_LEVELS, its_idr1)) + return false; + + if (!devtab && !FIELD_GET(GICV5_ITS_IDR1_ITT_LEVELS, its_idr1)) + return false; + + /* + * Pick an L2 size that matches the pagesize; if a match + * is not found, go for the smallest supported l2 size granule. + * + * This ensures that we will always be able to allocate + * contiguous memory at L2. + */ + switch (PAGE_SIZE) { + case SZ_64K: + if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) { + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k; + break; + } + fallthrough; + case SZ_4K: + if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) { + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k; + break; + } + fallthrough; + case SZ_16K: + if (GICV5_ITS_IDR1_L2SZ_SUPPORT_16KB(its_idr1)) { + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_16k; + break; + } + if (GICV5_ITS_IDR1_L2SZ_SUPPORT_4KB(its_idr1)) { + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k; + break; + } + if (GICV5_ITS_IDR1_L2SZ_SUPPORT_64KB(its_idr1)) { + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_64k; + break; + } + + l2_sz = GICV5_ITS_DT_ITT_CFGR_L2SZ_4k; + break; + } + + l2_bits = gicv5_its_l2sz_to_l2_bits(l2_sz); + + if (l2_bits > id_bits) + return false; + + *sz = l2_sz; + + return true; +} + +static __le64 *gicv5_its_device_get_itte_ref(struct gicv5_its_dev *its_dev, + u16 event_id) +{ + unsigned int l1_idx, l2_idx, l2_bits; + __le64 *l2_itt; + + if (!its_dev->itt_cfg.l2itt) { + __le64 *itt = its_dev->itt_cfg.linear.itt; + + return &itt[event_id]; + } + + l2_bits = gicv5_its_l2sz_to_l2_bits(its_dev->itt_cfg.l2.l2sz); + l1_idx = event_id >> l2_bits; + l2_idx = event_id & GENMASK(l2_bits - 1, 0); + l2_itt = its_dev->itt_cfg.l2.l2ptrs[l1_idx]; + + return &l2_itt[l2_idx]; +} + +static int gicv5_its_device_cache_inv(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev) +{ + u32 devicer; + u64 didr; + + didr = FIELD_PREP(GICV5_ITS_DIDR_DEVICEID, its_dev->device_id); + devicer = FIELD_PREP(GICV5_ITS_INV_DEVICER_I, 0x1) | + FIELD_PREP(GICV5_ITS_INV_DEVICER_EVENTID_BITS, + its_dev->itt_cfg.event_id_bits) | + FIELD_PREP(GICV5_ITS_INV_DEVICER_L1, 0x0); + its_writeq_relaxed(its, didr, GICV5_ITS_DIDR); + its_writel_relaxed(its, devicer, GICV5_ITS_INV_DEVICER); + + return gicv5_its_cache_sync(its); +} + +/* + * Allocate a level 2 device table entry, update L1 parent to reference it. + * Only used for 2-level device tables, and it is called on demand. + */ +static int gicv5_its_alloc_l2_devtab(struct gicv5_its_chip_data *its, + unsigned int l1_index) +{ + __le64 *l2devtab, *l1devtab = its->devtab_cfgr.l2.l1devtab; + u8 span, l2sz, l2_bits; + u64 l1dte; + + if (FIELD_GET(GICV5_DTL1E_VALID, le64_to_cpu(l1devtab[l1_index]))) + return 0; + + span = FIELD_GET(GICV5_DTL1E_SPAN, le64_to_cpu(l1devtab[l1_index])); + l2sz = devtab_cfgr_field(its, L2SZ); + + l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz); + + /* + * Span allows us to create a smaller L2 device table. + * If it is too large, use the number of allowed L2 bits. + */ + if (span > l2_bits) + span = l2_bits; + + l2devtab = kcalloc(BIT(span), sizeof(*l2devtab), GFP_KERNEL); + if (!l2devtab) + return -ENOMEM; + + its->devtab_cfgr.l2.l2ptrs[l1_index] = l2devtab; + + l1dte = FIELD_PREP(GICV5_DTL1E_SPAN, span) | + (virt_to_phys(l2devtab) & GICV5_DTL1E_L2_ADDR_MASK) | + FIELD_PREP(GICV5_DTL1E_VALID, 0x1); + its_write_table_entry(its, &l1devtab[l1_index], l1dte); + + return 0; +} + +static __le64 *gicv5_its_devtab_get_dte_ref(struct gicv5_its_chip_data *its, + u32 device_id, bool alloc) +{ + u8 str = devtab_cfgr_field(its, STRUCTURE); + unsigned int l2sz, l2_bits, l1_idx, l2_idx; + __le64 *l2devtab; + int ret; + + if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) { + l2devtab = its->devtab_cfgr.linear.devtab; + return &l2devtab[device_id]; + } + + l2sz = devtab_cfgr_field(its, L2SZ); + l2_bits = gicv5_its_l2sz_to_l2_bits(l2sz); + l1_idx = device_id >> l2_bits; + l2_idx = device_id & GENMASK(l2_bits - 1, 0); + + if (alloc) { + /* + * Allocate a new L2 device table here before + * continuing. We make the assumption that the span in + * the L1 table has been set correctly, and blindly use + * that value. + */ + ret = gicv5_its_alloc_l2_devtab(its, l1_idx); + if (ret) + return NULL; + } + + l2devtab = its->devtab_cfgr.l2.l2ptrs[l1_idx]; + return &l2devtab[l2_idx]; +} + +/* + * Register a new device in the device table. Allocate an ITT and + * program the L2DTE entry according to the ITT structure that + * was chosen. + */ +static int gicv5_its_device_register(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev) +{ + u8 event_id_bits, device_id_bits, itt_struct, itt_l2sz; + phys_addr_t itt_phys_base; + bool two_level_itt; + u32 idr1, idr2; + __le64 *dte; + u64 val; + int ret; + + device_id_bits = devtab_cfgr_field(its, DEVICEID_BITS); + + if (its_dev->device_id >= BIT(device_id_bits)) { + pr_err("Supplied DeviceID (%u) outside of Device Table range (%u)!", + its_dev->device_id, (u32)GENMASK(device_id_bits - 1, 0)); + return -EINVAL; + } + + dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, true); + if (!dte) + return -ENOMEM; + + if (FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte))) + return -EBUSY; + + /* + * Determine how many bits we need, validate those against the max. + * Based on these, determine if we should go for a 1- or 2-level ITT. + */ + event_id_bits = order_base_2(its_dev->num_events); + + idr2 = its_readl_relaxed(its, GICV5_ITS_IDR2); + + if (event_id_bits > FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2)) { + pr_err("Required EventID bits (%u) larger than supported bits (%u)!", + event_id_bits, + (u8)FIELD_GET(GICV5_ITS_IDR2_EVENTID_BITS, idr2)); + return -EINVAL; + } + + idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1); + + /* + * L2 ITT size is programmed into the L2DTE regardless of + * whether a two-level or linear ITT is built, init it. + */ + itt_l2sz = 0; + + two_level_itt = gicv5_its_l2sz_two_level(false, idr1, event_id_bits, + &itt_l2sz); + if (two_level_itt) + ret = gicv5_its_create_itt_two_level(its, its_dev, event_id_bits, + itt_l2sz, + its_dev->num_events); + else + ret = gicv5_its_create_itt_linear(its, its_dev, event_id_bits); + if (ret) + return ret; + + itt_phys_base = two_level_itt ? virt_to_phys(its_dev->itt_cfg.l2.l1itt) : + virt_to_phys(its_dev->itt_cfg.linear.itt); + + itt_struct = two_level_itt ? GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL : + GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR; + + val = FIELD_PREP(GICV5_DTL2E_EVENT_ID_BITS, event_id_bits) | + FIELD_PREP(GICV5_DTL2E_ITT_STRUCTURE, itt_struct) | + (itt_phys_base & GICV5_DTL2E_ITT_ADDR_MASK) | + FIELD_PREP(GICV5_DTL2E_ITT_L2SZ, itt_l2sz) | + FIELD_PREP(GICV5_DTL2E_VALID, 0x1); + + its_write_table_entry(its, dte, val); + + ret = gicv5_its_device_cache_inv(its, its_dev); + if (ret) { + its_write_table_entry(its, dte, 0); + gicv5_its_free_itt(its_dev); + return ret; + } + + return 0; +} + +/* + * Unregister a device in the device table. Lookup the device by ID, free the + * corresponding ITT, mark the device as invalid in the device table. + */ +static int gicv5_its_device_unregister(struct gicv5_its_chip_data *its, + struct gicv5_its_dev *its_dev) +{ + __le64 *dte; + + dte = gicv5_its_devtab_get_dte_ref(its, its_dev->device_id, false); + + if (!FIELD_GET(GICV5_DTL2E_VALID, le64_to_cpu(*dte))) { + pr_debug("Device table entry for DeviceID 0x%x is not valid. Nothing to clean up!", + its_dev->device_id); + return -EINVAL; + } + + /* Zero everything - make it clear that this is an invalid entry */ + its_write_table_entry(its, dte, 0); + + gicv5_its_free_itt(its_dev); + + return gicv5_its_device_cache_inv(its, its_dev); +} + +/* + * Allocate a 1-level device table. All entries are allocated, but marked + * invalid. + */ +static int gicv5_its_alloc_devtab_linear(struct gicv5_its_chip_data *its, + u8 device_id_bits) +{ + __le64 *devtab; + size_t sz; + u64 baser; + u32 cfgr; + + /* + * We expect a GICv5 implementation requiring a large number of + * deviceID bits to support a 2-level device table. If that's not + * the case, cap the number of deviceIDs supported according to the + * kmalloc limits so that the system can chug along with a linear + * device table. + */ + sz = BIT_ULL(device_id_bits) * sizeof(*devtab); + if (sz > KMALLOC_MAX_SIZE) { + u8 device_id_cap = ilog2(KMALLOC_MAX_SIZE/sizeof(*devtab)); + + pr_warn("Limiting device ID bits from %u to %u\n", + device_id_bits, device_id_cap); + device_id_bits = device_id_cap; + } + + devtab = kcalloc(BIT(device_id_bits), sizeof(*devtab), GFP_KERNEL); + if (!devtab) + return -ENOMEM; + + gicv5_its_dcache_clean(its, devtab, sz); + + cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE, + GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) | + FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, 0) | + FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits); + its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR); + + baser = virt_to_phys(devtab) & GICV5_ITS_DT_BASER_ADDR_MASK; + its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER); + + its->devtab_cfgr.cfgr = cfgr; + its->devtab_cfgr.linear.devtab = devtab; + + return 0; +} + +/* + * Allocate a 2-level device table. L2 entries are not allocated, + * they are allocated on-demand. + */ +static int gicv5_its_alloc_devtab_two_level(struct gicv5_its_chip_data *its, + u8 device_id_bits, + u8 devtab_l2sz) +{ + unsigned int l1_bits, l2_bits, i; + __le64 *l1devtab, **l2ptrs; + size_t l1_sz; + u64 baser; + u32 cfgr; + + l2_bits = gicv5_its_l2sz_to_l2_bits(devtab_l2sz); + + l1_bits = device_id_bits - l2_bits; + l1_sz = BIT(l1_bits) * sizeof(*l1devtab); + /* + * With 2-level device table support it is highly unlikely + * that we are not able to allocate the required amount of + * device table memory to cover deviceID space; cap the + * deviceID space if we encounter such set-up. + * If this ever becomes a problem we could revisit the policy + * behind level 2 size selection to reduce level-1 deviceID bits. + */ + if (l1_sz > KMALLOC_MAX_SIZE) { + l1_bits = ilog2(KMALLOC_MAX_SIZE/sizeof(*l1devtab)); + + pr_warn("Limiting device ID bits from %u to %u\n", + device_id_bits, l1_bits + l2_bits); + device_id_bits = l1_bits + l2_bits; + l1_sz = KMALLOC_MAX_SIZE; + } + + l1devtab = kcalloc(BIT(l1_bits), sizeof(*l1devtab), GFP_KERNEL); + if (!l1devtab) + return -ENOMEM; + + l2ptrs = kcalloc(BIT(l1_bits), sizeof(*l2ptrs), GFP_KERNEL); + if (!l2ptrs) { + kfree(l1devtab); + return -ENOMEM; + } + + for (i = 0; i < BIT(l1_bits); i++) + l1devtab[i] = cpu_to_le64(FIELD_PREP(GICV5_DTL1E_SPAN, l2_bits)); + + gicv5_its_dcache_clean(its, l1devtab, l1_sz); + + cfgr = FIELD_PREP(GICV5_ITS_DT_CFGR_STRUCTURE, + GICV5_ITS_DT_ITT_CFGR_STRUCTURE_TWO_LEVEL) | + FIELD_PREP(GICV5_ITS_DT_CFGR_L2SZ, devtab_l2sz) | + FIELD_PREP(GICV5_ITS_DT_CFGR_DEVICEID_BITS, device_id_bits); + its_writel_relaxed(its, cfgr, GICV5_ITS_DT_CFGR); + + baser = virt_to_phys(l1devtab) & GICV5_ITS_DT_BASER_ADDR_MASK; + its_writeq_relaxed(its, baser, GICV5_ITS_DT_BASER); + + its->devtab_cfgr.cfgr = cfgr; + its->devtab_cfgr.l2.l1devtab = l1devtab; + its->devtab_cfgr.l2.l2ptrs = l2ptrs; + + return 0; +} + +/* + * Initialise the device table as either 1- or 2-level depending on what is + * supported by the hardware. + */ +static int gicv5_its_init_devtab(struct gicv5_its_chip_data *its) +{ + u8 device_id_bits, devtab_l2sz; + bool two_level_devtab; + u32 idr1; + + idr1 = its_readl_relaxed(its, GICV5_ITS_IDR1); + + device_id_bits = FIELD_GET(GICV5_ITS_IDR1_DEVICEID_BITS, idr1); + two_level_devtab = gicv5_its_l2sz_two_level(true, idr1, device_id_bits, + &devtab_l2sz); + if (two_level_devtab) + return gicv5_its_alloc_devtab_two_level(its, device_id_bits, + devtab_l2sz); + else + return gicv5_its_alloc_devtab_linear(its, device_id_bits); +} + +static void gicv5_its_deinit_devtab(struct gicv5_its_chip_data *its) +{ + u8 str = devtab_cfgr_field(its, STRUCTURE); + + if (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR) { + kfree(its->devtab_cfgr.linear.devtab); + } else { + kfree(its->devtab_cfgr.l2.l1devtab); + kfree(its->devtab_cfgr.l2.l2ptrs); + } +} + +static void gicv5_its_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d); + u64 addr = its_dev->its_trans_phys_base; + + msg->data = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq); + msi_msg_set_addr(irq_data_get_msi_desc(d), msg, addr); +} + +static const struct irq_chip gicv5_its_irq_chip = { + .name = "GICv5-ITS-MSI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_compose_msi_msg = gicv5_its_compose_msi_msg, +}; + +static struct gicv5_its_dev *gicv5_its_find_device(struct gicv5_its_chip_data *its, + u32 device_id) +{ + struct gicv5_its_dev *dev = xa_load(&its->its_devices, device_id); + + return dev ? dev : ERR_PTR(-ENODEV); +} + +static struct gicv5_its_dev *gicv5_its_alloc_device(struct gicv5_its_chip_data *its, int nvec, + u32 dev_id) +{ + struct gicv5_its_dev *its_dev; + void *entry; + int ret; + + its_dev = gicv5_its_find_device(its, dev_id); + if (!IS_ERR(its_dev)) { + pr_err("A device with this DeviceID (0x%x) has already been registered.\n", + dev_id); + + return ERR_PTR(-EBUSY); + } + + its_dev = kzalloc(sizeof(*its_dev), GFP_KERNEL); + if (!its_dev) + return ERR_PTR(-ENOMEM); + + its_dev->device_id = dev_id; + its_dev->num_events = nvec; + + ret = gicv5_its_device_register(its, its_dev); + if (ret) { + pr_err("Failed to register the device\n"); + goto out_dev_free; + } + + its_dev->its_node = its; + + its_dev->event_map = (unsigned long *)bitmap_zalloc(its_dev->num_events, GFP_KERNEL); + if (!its_dev->event_map) { + ret = -ENOMEM; + goto out_unregister; + } + + entry = xa_store(&its->its_devices, dev_id, its_dev, GFP_KERNEL); + if (xa_is_err(entry)) { + ret = xa_err(entry); + goto out_bitmap_free; + } + + return its_dev; + +out_bitmap_free: + bitmap_free(its_dev->event_map); +out_unregister: + gicv5_its_device_unregister(its, its_dev); +out_dev_free: + kfree(its_dev); + return ERR_PTR(ret); +} + +static int gicv5_its_msi_prepare(struct irq_domain *domain, struct device *dev, + int nvec, msi_alloc_info_t *info) +{ + u32 dev_id = info->scratchpad[0].ul; + struct msi_domain_info *msi_info; + struct gicv5_its_chip_data *its; + struct gicv5_its_dev *its_dev; + + msi_info = msi_get_domain_info(domain); + its = msi_info->data; + + guard(mutex)(&its->dev_alloc_lock); + + its_dev = gicv5_its_alloc_device(its, nvec, dev_id); + if (IS_ERR(its_dev)) + return PTR_ERR(its_dev); + + its_dev->its_trans_phys_base = info->scratchpad[1].ul; + info->scratchpad[0].ptr = its_dev; + + return 0; +} + +static void gicv5_its_msi_teardown(struct irq_domain *domain, msi_alloc_info_t *info) +{ + struct gicv5_its_dev *its_dev = info->scratchpad[0].ptr; + struct msi_domain_info *msi_info; + struct gicv5_its_chip_data *its; + + msi_info = msi_get_domain_info(domain); + its = msi_info->data; + + guard(mutex)(&its->dev_alloc_lock); + + if (WARN_ON_ONCE(!bitmap_empty(its_dev->event_map, its_dev->num_events))) + return; + + xa_erase(&its->its_devices, its_dev->device_id); + bitmap_free(its_dev->event_map); + gicv5_its_device_unregister(its, its_dev); + kfree(its_dev); +} + +static struct msi_domain_ops gicv5_its_msi_domain_ops = { + .msi_prepare = gicv5_its_msi_prepare, + .msi_teardown = gicv5_its_msi_teardown, +}; + +static int gicv5_its_map_event(struct gicv5_its_dev *its_dev, u16 event_id, u32 lpi) +{ + struct gicv5_its_chip_data *its = its_dev->its_node; + u64 itt_entry; + __le64 *itte; + + itte = gicv5_its_device_get_itte_ref(its_dev, event_id); + + if (FIELD_GET(GICV5_ITTL2E_VALID, *itte)) + return -EEXIST; + + itt_entry = FIELD_PREP(GICV5_ITTL2E_LPI_ID, lpi) | + FIELD_PREP(GICV5_ITTL2E_VALID, 0x1); + + its_write_table_entry(its, itte, itt_entry); + + gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id); + + return 0; +} + +static void gicv5_its_unmap_event(struct gicv5_its_dev *its_dev, u16 event_id) +{ + struct gicv5_its_chip_data *its = its_dev->its_node; + u64 itte_val; + __le64 *itte; + + itte = gicv5_its_device_get_itte_ref(its_dev, event_id); + + itte_val = le64_to_cpu(*itte); + itte_val &= ~GICV5_ITTL2E_VALID; + + its_write_table_entry(its, itte, itte_val); + + gicv5_its_itt_cache_inv(its, its_dev->device_id, event_id); +} + +static int gicv5_its_alloc_eventid(struct gicv5_its_dev *its_dev, msi_alloc_info_t *info, + unsigned int nr_irqs, u32 *eventid) +{ + int event_id_base; + + if (!(info->flags & MSI_ALLOC_FLAGS_FIXED_MSG_DATA)) { + event_id_base = bitmap_find_free_region(its_dev->event_map, + its_dev->num_events, + get_count_order(nr_irqs)); + if (event_id_base < 0) + return event_id_base; + } else { + /* + * We want to have a fixed EventID mapped for hardcoded + * message data allocations. + */ + if (WARN_ON_ONCE(nr_irqs != 1)) + return -EINVAL; + + event_id_base = info->hwirq; + + if (event_id_base >= its_dev->num_events) { + pr_err("EventID ouside of ITT range; cannot allocate an ITT entry!\n"); + + return -EINVAL; + } + + if (test_and_set_bit(event_id_base, its_dev->event_map)) { + pr_warn("Can't reserve event_id bitmap\n"); + return -EINVAL; + + } + } + + *eventid = event_id_base; + + return 0; +} + +static void gicv5_its_free_eventid(struct gicv5_its_dev *its_dev, u32 event_id_base, + unsigned int nr_irqs) +{ + bitmap_release_region(its_dev->event_map, event_id_base, + get_count_order(nr_irqs)); +} + +static int gicv5_its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + u32 device_id, event_id_base, lpi; + struct gicv5_its_dev *its_dev; + msi_alloc_info_t *info = arg; + irq_hw_number_t hwirq; + struct irq_data *irqd; + int ret, i; + + its_dev = info->scratchpad[0].ptr; + + ret = gicv5_its_alloc_eventid(its_dev, info, nr_irqs, &event_id_base); + if (ret) + return ret; + + ret = iommu_dma_prepare_msi(info->desc, its_dev->its_trans_phys_base); + if (ret) + goto out_eventid; + + device_id = its_dev->device_id; + + for (i = 0; i < nr_irqs; i++) { + ret = gicv5_alloc_lpi(); + if (ret < 0) { + pr_debug("Failed to find free LPI!\n"); + goto out_free_irqs; + } + lpi = ret; + + ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi); + if (ret) { + gicv5_free_lpi(lpi); + goto out_free_irqs; + } + + /* + * Store eventid and deviceid into the hwirq for later use. + * + * hwirq = event_id << 32 | device_id + */ + hwirq = FIELD_PREP(GICV5_ITS_HWIRQ_DEVICE_ID, device_id) | + FIELD_PREP(GICV5_ITS_HWIRQ_EVENT_ID, (u64)event_id_base + i); + irq_domain_set_info(domain, virq + i, hwirq, + &gicv5_its_irq_chip, its_dev, + handle_fasteoi_irq, NULL, NULL); + + irqd = irq_get_irq_data(virq + i); + irqd_set_single_target(irqd); + irqd_set_affinity_on_activate(irqd); + } + + return 0; + +out_free_irqs: + while (--i >= 0) { + irqd = irq_domain_get_irq_data(domain, virq + i); + gicv5_free_lpi(irqd->parent_data->hwirq); + irq_domain_reset_irq_data(irqd); + irq_domain_free_irqs_parent(domain, virq + i, 1); + } +out_eventid: + gicv5_its_free_eventid(its_dev, event_id_base, nr_irqs); + return ret; +} + +static void gicv5_its_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct gicv5_its_chip_data *its; + struct gicv5_its_dev *its_dev; + u16 event_id_base; + unsigned int i; + + its_dev = irq_data_get_irq_chip_data(d); + its = its_dev->its_node; + + event_id_base = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq); + + bitmap_release_region(its_dev->event_map, event_id_base, + get_count_order(nr_irqs)); + + /* Hierarchically free irq data */ + for (i = 0; i < nr_irqs; i++) { + d = irq_domain_get_irq_data(domain, virq + i); + + gicv5_free_lpi(d->parent_data->hwirq); + irq_domain_reset_irq_data(d); + irq_domain_free_irqs_parent(domain, virq + i, 1); + } + + gicv5_its_syncr(its, its_dev); + gicv5_irs_syncr(); +} + +static int gicv5_its_irq_domain_activate(struct irq_domain *domain, struct irq_data *d, + bool reserve) +{ + struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d); + u16 event_id; + u32 lpi; + + event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq); + lpi = d->parent_data->hwirq; + + return gicv5_its_map_event(its_dev, event_id, lpi); +} + +static void gicv5_its_irq_domain_deactivate(struct irq_domain *domain, + struct irq_data *d) +{ + struct gicv5_its_dev *its_dev = irq_data_get_irq_chip_data(d); + u16 event_id; + + event_id = FIELD_GET(GICV5_ITS_HWIRQ_EVENT_ID, d->hwirq); + + gicv5_its_unmap_event(its_dev, event_id); +} + +static const struct irq_domain_ops gicv5_its_irq_domain_ops = { + .alloc = gicv5_its_irq_domain_alloc, + .free = gicv5_its_irq_domain_free, + .activate = gicv5_its_irq_domain_activate, + .deactivate = gicv5_its_irq_domain_deactivate, + .select = msi_lib_irq_domain_select, +}; + +static int gicv5_its_write_cr0(struct gicv5_its_chip_data *its, bool enable) +{ + u32 cr0 = FIELD_PREP(GICV5_ITS_CR0_ITSEN, enable); + + its_writel_relaxed(its, cr0, GICV5_ITS_CR0); + return gicv5_wait_for_op_atomic(its->its_base, GICV5_ITS_CR0, + GICV5_ITS_CR0_IDLE, NULL); +} + +static int gicv5_its_enable(struct gicv5_its_chip_data *its) +{ + return gicv5_its_write_cr0(its, true); +} + +static int gicv5_its_disable(struct gicv5_its_chip_data *its) +{ + return gicv5_its_write_cr0(its, false); +} + +static void gicv5_its_print_info(struct gicv5_its_chip_data *its_node) +{ + bool devtab_linear; + u8 device_id_bits; + u8 str; + + device_id_bits = devtab_cfgr_field(its_node, DEVICEID_BITS); + + str = devtab_cfgr_field(its_node, STRUCTURE); + devtab_linear = (str == GICV5_ITS_DT_ITT_CFGR_STRUCTURE_LINEAR); + + pr_info("ITS %s enabled using %s device table device_id_bits %u\n", + fwnode_get_name(its_node->fwnode), + devtab_linear ? "linear" : "2-level", + device_id_bits); +} + +static int gicv5_its_init_domain(struct gicv5_its_chip_data *its, struct irq_domain *parent) +{ + struct irq_domain_info dom_info = { + .fwnode = its->fwnode, + .ops = &gicv5_its_irq_domain_ops, + .domain_flags = its->msi_domain_flags, + .parent = parent, + }; + struct msi_domain_info *info; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->ops = &gicv5_its_msi_domain_ops; + info->data = its; + dom_info.host_data = info; + + if (!msi_create_parent_irq_domain(&dom_info, &gic_v5_its_msi_parent_ops)) { + kfree(info); + return -ENOMEM; + } + + return 0; +} + +static int __init gicv5_its_init_bases(void __iomem *its_base, struct fwnode_handle *handle, + struct irq_domain *parent_domain) +{ + struct device_node *np = to_of_node(handle); + struct gicv5_its_chip_data *its_node; + u32 cr0, cr1; + bool enabled; + int ret; + + its_node = kzalloc(sizeof(*its_node), GFP_KERNEL); + if (!its_node) + return -ENOMEM; + + mutex_init(&its_node->dev_alloc_lock); + xa_init(&its_node->its_devices); + its_node->fwnode = handle; + its_node->its_base = its_base; + its_node->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI | + IRQ_DOMAIN_FLAG_FWNODE_PARENT; + + cr0 = its_readl_relaxed(its_node, GICV5_ITS_CR0); + enabled = FIELD_GET(GICV5_ITS_CR0_ITSEN, cr0); + if (WARN(enabled, "ITS %s enabled, disabling it before proceeding\n", np->full_name)) { + ret = gicv5_its_disable(its_node); + if (ret) + goto out_free_node; + } + + if (of_property_read_bool(np, "dma-noncoherent")) { + /* + * A non-coherent ITS implies that some cache levels cannot be + * used coherently by the cores and GIC. Our only option is to mark + * memory attributes for the GIC as non-cacheable; by default, + * non-cacheable memory attributes imply outer-shareable + * shareability, the value written into ITS_CR1_SH is ignored. + */ + cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_NO_READ_ALLOC) | + FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_NON_CACHE) | + FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_NON_CACHE); + its_node->flags |= ITS_FLAGS_NON_COHERENT; + } else { + cr1 = FIELD_PREP(GICV5_ITS_CR1_ITT_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_ITS_CR1_DT_RA, GICV5_READ_ALLOC) | + FIELD_PREP(GICV5_ITS_CR1_IC, GICV5_WB_CACHE) | + FIELD_PREP(GICV5_ITS_CR1_OC, GICV5_WB_CACHE) | + FIELD_PREP(GICV5_ITS_CR1_SH, GICV5_INNER_SHARE); + } + + its_writel_relaxed(its_node, cr1, GICV5_ITS_CR1); + + ret = gicv5_its_init_devtab(its_node); + if (ret) + goto out_free_node; + + ret = gicv5_its_enable(its_node); + if (ret) + goto out_free_devtab; + + ret = gicv5_its_init_domain(its_node, parent_domain); + if (ret) + goto out_disable_its; + + gicv5_its_print_info(its_node); + + return 0; + +out_disable_its: + gicv5_its_disable(its_node); +out_free_devtab: + gicv5_its_deinit_devtab(its_node); +out_free_node: + kfree(its_node); + return ret; +} + +static int __init gicv5_its_init(struct device_node *node) +{ + void __iomem *its_base; + int ret, idx; + + idx = of_property_match_string(node, "reg-names", "ns-config"); + if (idx < 0) { + pr_err("%pOF: ns-config reg-name not present\n", node); + return -ENODEV; + } + + its_base = of_io_request_and_map(node, idx, of_node_full_name(node)); + if (IS_ERR(its_base)) { + pr_err("%pOF: unable to map GICv5 ITS_CONFIG_FRAME\n", node); + return PTR_ERR(its_base); + } + + ret = gicv5_its_init_bases(its_base, of_fwnode_handle(node), + gicv5_global_data.lpi_domain); + if (ret) + goto out_unmap; + + return 0; + +out_unmap: + iounmap(its_base); + return ret; +} + +void __init gicv5_its_of_probe(struct device_node *parent) +{ + struct device_node *np; + + for_each_available_child_of_node(parent, np) { + if (!of_device_is_compatible(np, "arm,gic-v5-its")) + continue; + + if (gicv5_its_init(np)) + pr_err("Failed to init ITS %s\n", np->full_name); + } +} diff --git a/drivers/irqchip/irq-gic-v5-iwb.c b/drivers/irqchip/irq-gic-v5-iwb.c new file mode 100644 index 000000000000..ad9fdc14d1c6 --- /dev/null +++ b/drivers/irqchip/irq-gic-v5-iwb.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. + */ +#define pr_fmt(fmt) "GICv5 IWB: " fmt + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/msi.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_platform.h> + +#include <linux/irqchip.h> +#include <linux/irqchip/arm-gic-v5.h> + +struct gicv5_iwb_chip_data { + void __iomem *iwb_base; + u16 nr_regs; +}; + +static u32 iwb_readl_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 reg_offset) +{ + return readl_relaxed(iwb_node->iwb_base + reg_offset); +} + +static void iwb_writel_relaxed(struct gicv5_iwb_chip_data *iwb_node, const u32 val, + const u32 reg_offset) +{ + writel_relaxed(val, iwb_node->iwb_base + reg_offset); +} + +static int gicv5_iwb_wait_for_wenabler(struct gicv5_iwb_chip_data *iwb_node) +{ + return gicv5_wait_for_op_atomic(iwb_node->iwb_base, GICV5_IWB_WENABLE_STATUSR, + GICV5_IWB_WENABLE_STATUSR_IDLE, NULL); +} + +static int __gicv5_iwb_set_wire_enable(struct gicv5_iwb_chip_data *iwb_node, + u32 iwb_wire, bool enable) +{ + u32 n = iwb_wire / 32; + u8 i = iwb_wire % 32; + u32 val; + + if (n >= iwb_node->nr_regs) { + pr_err("IWB_WENABLER<n> is invalid for n=%u\n", n); + return -EINVAL; + } + + /* + * Enable IWB wire/pin at this point + * Note: This is not the same as enabling the interrupt + */ + val = iwb_readl_relaxed(iwb_node, GICV5_IWB_WENABLER + (4 * n)); + if (enable) + val |= BIT(i); + else + val &= ~BIT(i); + iwb_writel_relaxed(iwb_node, val, GICV5_IWB_WENABLER + (4 * n)); + + return gicv5_iwb_wait_for_wenabler(iwb_node); +} + +static int gicv5_iwb_enable_wire(struct gicv5_iwb_chip_data *iwb_node, + u32 iwb_wire) +{ + return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, true); +} + +static int gicv5_iwb_disable_wire(struct gicv5_iwb_chip_data *iwb_node, + u32 iwb_wire) +{ + return __gicv5_iwb_set_wire_enable(iwb_node, iwb_wire, false); +} + +static void gicv5_iwb_irq_disable(struct irq_data *d) +{ + struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d); + + gicv5_iwb_disable_wire(iwb_node, d->hwirq); + irq_chip_disable_parent(d); +} + +static void gicv5_iwb_irq_enable(struct irq_data *d) +{ + struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d); + + gicv5_iwb_enable_wire(iwb_node, d->hwirq); + irq_chip_enable_parent(d); +} + +static int gicv5_iwb_set_type(struct irq_data *d, unsigned int type) +{ + struct gicv5_iwb_chip_data *iwb_node = irq_data_get_irq_chip_data(d); + u32 iwb_wire, n, wtmr; + u8 i; + + iwb_wire = d->hwirq; + i = iwb_wire % 32; + n = iwb_wire / 32; + + if (n >= iwb_node->nr_regs) { + pr_err_once("reg %u out of range\n", n); + return -EINVAL; + } + + wtmr = iwb_readl_relaxed(iwb_node, GICV5_IWB_WTMR + (4 * n)); + + switch (type) { + case IRQ_TYPE_LEVEL_HIGH: + case IRQ_TYPE_LEVEL_LOW: + wtmr |= BIT(i); + break; + case IRQ_TYPE_EDGE_RISING: + case IRQ_TYPE_EDGE_FALLING: + wtmr &= ~BIT(i); + break; + default: + pr_debug("unexpected wire trigger mode"); + return -EINVAL; + } + + iwb_writel_relaxed(iwb_node, wtmr, GICV5_IWB_WTMR + (4 * n)); + + return 0; +} + +static void gicv5_iwb_domain_set_desc(msi_alloc_info_t *alloc_info, struct msi_desc *desc) +{ + alloc_info->desc = desc; + alloc_info->hwirq = (u32)desc->data.icookie.value; +} + +static int gicv5_iwb_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + irq_hw_number_t *hwirq, + unsigned int *type) +{ + if (!is_of_node(fwspec->fwnode)) + return -EINVAL; + + if (fwspec->param_count < 2) + return -EINVAL; + + /* + * param[0] is be the wire + * param[1] is the interrupt type + */ + *hwirq = fwspec->param[0]; + *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static void gicv5_iwb_write_msi_msg(struct irq_data *d, struct msi_msg *msg) {} + +static const struct msi_domain_template iwb_msi_template = { + .chip = { + .name = "GICv5-IWB", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_enable = gicv5_iwb_irq_enable, + .irq_disable = gicv5_iwb_irq_disable, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_type = gicv5_iwb_set_type, + .irq_write_msi_msg = gicv5_iwb_write_msi_msg, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, + }, + + .ops = { + .set_desc = gicv5_iwb_domain_set_desc, + .msi_translate = gicv5_iwb_irq_domain_translate, + }, + + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_USE_DEV_FWNODE, + }, + + .alloc_info = { + .flags = MSI_ALLOC_FLAGS_FIXED_MSG_DATA, + }, +}; + +static bool gicv5_iwb_create_device_domain(struct device *dev, unsigned int size, + struct gicv5_iwb_chip_data *iwb_node) +{ + if (WARN_ON_ONCE(!dev->msi.domain)) + return false; + + return msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, + &iwb_msi_template, size, + NULL, iwb_node); +} + +static struct gicv5_iwb_chip_data * +gicv5_iwb_init_bases(void __iomem *iwb_base, struct platform_device *pdev) +{ + u32 nr_wires, idr0, cr0; + unsigned int n; + int ret; + + struct gicv5_iwb_chip_data *iwb_node __free(kfree) = kzalloc(sizeof(*iwb_node), + GFP_KERNEL); + if (!iwb_node) + return ERR_PTR(-ENOMEM); + + iwb_node->iwb_base = iwb_base; + + idr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_IDR0); + nr_wires = (FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1) * 32; + + cr0 = iwb_readl_relaxed(iwb_node, GICV5_IWB_CR0); + if (!FIELD_GET(GICV5_IWB_CR0_IWBEN, cr0)) { + dev_err(&pdev->dev, "IWB must be enabled in firmware\n"); + return ERR_PTR(-EINVAL); + } + + iwb_node->nr_regs = FIELD_GET(GICV5_IWB_IDR0_IW_RANGE, idr0) + 1; + + for (n = 0; n < iwb_node->nr_regs; n++) + iwb_writel_relaxed(iwb_node, 0, GICV5_IWB_WENABLER + (sizeof(u32) * n)); + + ret = gicv5_iwb_wait_for_wenabler(iwb_node); + if (ret) + return ERR_PTR(ret); + + if (!gicv5_iwb_create_device_domain(&pdev->dev, nr_wires, iwb_node)) + return ERR_PTR(-ENOMEM); + + return_ptr(iwb_node); +} + +static int gicv5_iwb_device_probe(struct platform_device *pdev) +{ + struct gicv5_iwb_chip_data *iwb_node; + void __iomem *iwb_base; + struct resource *res; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!res) + return -EINVAL; + + iwb_base = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!iwb_base) { + dev_err(&pdev->dev, "failed to ioremap %pR\n", res); + return -ENOMEM; + } + + iwb_node = gicv5_iwb_init_bases(iwb_base, pdev); + if (IS_ERR(iwb_node)) + return PTR_ERR(iwb_node); + + return 0; +} + +static const struct of_device_id gicv5_iwb_of_match[] = { + { .compatible = "arm,gic-v5-iwb" }, + { /* END */ } +}; +MODULE_DEVICE_TABLE(of, gicv5_iwb_of_match); + +static struct platform_driver gicv5_iwb_platform_driver = { + .driver = { + .name = "GICv5 IWB", + .of_match_table = gicv5_iwb_of_match, + .suppress_bind_attrs = true, + }, + .probe = gicv5_iwb_device_probe, +}; + +module_platform_driver(gicv5_iwb_platform_driver); diff --git a/drivers/irqchip/irq-gic-v5.c b/drivers/irqchip/irq-gic-v5.c new file mode 100644 index 000000000000..41ef286c4d78 --- /dev/null +++ b/drivers/irqchip/irq-gic-v5.c @@ -0,0 +1,1130 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2024-2025 ARM Limited, All Rights Reserved. + */ + +#define pr_fmt(fmt) "GICv5: " fmt + +#include <linux/cpuhotplug.h> +#include <linux/idr.h> +#include <linux/irqdomain.h> +#include <linux/slab.h> +#include <linux/wordpart.h> + +#include <linux/irqchip.h> +#include <linux/irqchip/arm-gic-v5.h> +#include <linux/irqchip/arm-vgic-info.h> + +#include <asm/cpufeature.h> +#include <asm/exception.h> + +static u8 pri_bits __ro_after_init = 5; + +#define GICV5_IRQ_PRI_MASK 0x1f +#define GICV5_IRQ_PRI_MI (GICV5_IRQ_PRI_MASK & GENMASK(4, 5 - pri_bits)) + +#define PPI_NR 128 + +static bool gicv5_cpuif_has_gcie(void) +{ + return this_cpu_has_cap(ARM64_HAS_GICV5_CPUIF); +} + +struct gicv5_chip_data gicv5_global_data __read_mostly; + +static DEFINE_IDA(lpi_ida); +static u32 num_lpis __ro_after_init; + +void __init gicv5_init_lpis(u32 lpis) +{ + num_lpis = lpis; +} + +void __init gicv5_deinit_lpis(void) +{ + num_lpis = 0; +} + +static int alloc_lpi(void) +{ + if (!num_lpis) + return -ENOSPC; + + return ida_alloc_max(&lpi_ida, num_lpis - 1, GFP_KERNEL); +} + +static void release_lpi(u32 lpi) +{ + ida_free(&lpi_ida, lpi); +} + +int gicv5_alloc_lpi(void) +{ + return alloc_lpi(); +} + +void gicv5_free_lpi(u32 lpi) +{ + release_lpi(lpi); +} + +static void gicv5_ppi_priority_init(void) +{ + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR0_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR1_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR2_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR3_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR4_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR5_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR6_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR7_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR8_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR9_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR10_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR11_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR12_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR13_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR14_EL1); + write_sysreg_s(REPEAT_BYTE(GICV5_IRQ_PRI_MI), SYS_ICC_PPI_PRIORITYR15_EL1); + + /* + * Context syncronization required to make sure system register writes + * effects are synchronised. + */ + isb(); +} + +static void gicv5_hwirq_init(irq_hw_number_t hwirq, u8 priority, u8 hwirq_type) +{ + u64 cdpri, cdaff; + u16 iaffid; + int ret; + + if (hwirq_type == GICV5_HWIRQ_TYPE_LPI || hwirq_type == GICV5_HWIRQ_TYPE_SPI) { + cdpri = FIELD_PREP(GICV5_GIC_CDPRI_PRIORITY_MASK, priority) | + FIELD_PREP(GICV5_GIC_CDPRI_TYPE_MASK, hwirq_type) | + FIELD_PREP(GICV5_GIC_CDPRI_ID_MASK, hwirq); + gic_insn(cdpri, CDPRI); + + ret = gicv5_irs_cpu_to_iaffid(smp_processor_id(), &iaffid); + + if (WARN_ON_ONCE(ret)) + return; + + cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | + FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | + FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, hwirq); + gic_insn(cdaff, CDAFF); + } +} + +static void gicv5_ppi_irq_mask(struct irq_data *d) +{ + u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); + + if (d->hwirq < 64) + sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, hwirq_id_bit, 0); + else + sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, hwirq_id_bit, 0); + + /* + * We must ensure that the disable takes effect immediately to + * guarantee that the lazy-disabled IRQ mechanism works. + * A context synchronization event is required to guarantee it. + * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1. + */ + isb(); +} + +static void gicv5_iri_irq_mask(struct irq_data *d, u8 hwirq_type) +{ + u64 cddis; + + cddis = FIELD_PREP(GICV5_GIC_CDDIS_ID_MASK, d->hwirq) | + FIELD_PREP(GICV5_GIC_CDDIS_TYPE_MASK, hwirq_type); + + gic_insn(cddis, CDDIS); + /* + * We must make sure that GIC CDDIS write effects are propagated + * immediately to make sure the disable takes effect to guarantee + * that the lazy-disabled IRQ mechanism works. + * Rule R_XCLJC states that the effects of a GIC system instruction + * complete in finite time. + * The GSB ensures completion of the GIC instruction and prevents + * loads, stores and GIC instructions from executing part of their + * functionality before the GSB SYS. + */ + gsb_sys(); +} + +static void gicv5_spi_irq_mask(struct irq_data *d) +{ + gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_SPI); +} + +static void gicv5_lpi_irq_mask(struct irq_data *d) +{ + gicv5_iri_irq_mask(d, GICV5_HWIRQ_TYPE_LPI); +} + +static void gicv5_ppi_irq_unmask(struct irq_data *d) +{ + u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); + + if (d->hwirq < 64) + sysreg_clear_set_s(SYS_ICC_PPI_ENABLER0_EL1, 0, hwirq_id_bit); + else + sysreg_clear_set_s(SYS_ICC_PPI_ENABLER1_EL1, 0, hwirq_id_bit); + /* + * We must ensure that the enable takes effect in finite time - a + * context synchronization event is required to guarantee it, we + * can not take for granted that would happen (eg a core going straight + * into idle after enabling a PPI). + * Reference: I_ZLTKB/R_YRGMH GICv5 specification - section 2.9.1. + */ + isb(); +} + +static void gicv5_iri_irq_unmask(struct irq_data *d, u8 hwirq_type) +{ + u64 cden; + + cden = FIELD_PREP(GICV5_GIC_CDEN_ID_MASK, d->hwirq) | + FIELD_PREP(GICV5_GIC_CDEN_TYPE_MASK, hwirq_type); + /* + * Rule R_XCLJC states that the effects of a GIC system instruction + * complete in finite time and that's the only requirement when + * unmasking an SPI/LPI IRQ. + */ + gic_insn(cden, CDEN); +} + +static void gicv5_spi_irq_unmask(struct irq_data *d) +{ + gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_SPI); +} + +static void gicv5_lpi_irq_unmask(struct irq_data *d) +{ + gicv5_iri_irq_unmask(d, GICV5_HWIRQ_TYPE_LPI); +} + +static void gicv5_hwirq_eoi(u32 hwirq_id, u8 hwirq_type) +{ + u64 cddi; + + cddi = FIELD_PREP(GICV5_GIC_CDDI_ID_MASK, hwirq_id) | + FIELD_PREP(GICV5_GIC_CDDI_TYPE_MASK, hwirq_type); + + gic_insn(cddi, CDDI); + + gic_insn(0, CDEOI); +} + +static void gicv5_ppi_irq_eoi(struct irq_data *d) +{ + /* Skip deactivate for forwarded PPI interrupts */ + if (irqd_is_forwarded_to_vcpu(d)) { + gic_insn(0, CDEOI); + return; + } + + gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_PPI); +} + +static void gicv5_spi_irq_eoi(struct irq_data *d) +{ + gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_SPI); +} + +static void gicv5_lpi_irq_eoi(struct irq_data *d) +{ + gicv5_hwirq_eoi(d->hwirq, GICV5_HWIRQ_TYPE_LPI); +} + +static int gicv5_iri_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, + bool force, u8 hwirq_type) +{ + int ret, cpuid; + u16 iaffid; + u64 cdaff; + + if (force) + cpuid = cpumask_first(mask_val); + else + cpuid = cpumask_any_and(mask_val, cpu_online_mask); + + ret = gicv5_irs_cpu_to_iaffid(cpuid, &iaffid); + if (ret) + return ret; + + cdaff = FIELD_PREP(GICV5_GIC_CDAFF_IAFFID_MASK, iaffid) | + FIELD_PREP(GICV5_GIC_CDAFF_TYPE_MASK, hwirq_type) | + FIELD_PREP(GICV5_GIC_CDAFF_ID_MASK, d->hwirq); + gic_insn(cdaff, CDAFF); + + irq_data_update_effective_affinity(d, cpumask_of(cpuid)); + + return IRQ_SET_MASK_OK_DONE; +} + +static int gicv5_spi_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, + bool force) +{ + return gicv5_iri_irq_set_affinity(d, mask_val, force, + GICV5_HWIRQ_TYPE_SPI); +} + +static int gicv5_lpi_irq_set_affinity(struct irq_data *d, + const struct cpumask *mask_val, + bool force) +{ + return gicv5_iri_irq_set_affinity(d, mask_val, force, + GICV5_HWIRQ_TYPE_LPI); +} + +enum ppi_reg { + PPI_PENDING, + PPI_ACTIVE, + PPI_HM +}; + +static __always_inline u64 read_ppi_sysreg_s(unsigned int irq, + const enum ppi_reg which) +{ + switch (which) { + case PPI_PENDING: + return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SPENDR0_EL1) : + read_sysreg_s(SYS_ICC_PPI_SPENDR1_EL1); + case PPI_ACTIVE: + return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_SACTIVER0_EL1) : + read_sysreg_s(SYS_ICC_PPI_SACTIVER1_EL1); + case PPI_HM: + return irq < 64 ? read_sysreg_s(SYS_ICC_PPI_HMR0_EL1) : + read_sysreg_s(SYS_ICC_PPI_HMR1_EL1); + default: + BUILD_BUG_ON(1); + } +} + +static __always_inline void write_ppi_sysreg_s(unsigned int irq, bool set, + const enum ppi_reg which) +{ + u64 bit = BIT_ULL(irq % 64); + + switch (which) { + case PPI_PENDING: + if (set) { + if (irq < 64) + write_sysreg_s(bit, SYS_ICC_PPI_SPENDR0_EL1); + else + write_sysreg_s(bit, SYS_ICC_PPI_SPENDR1_EL1); + } else { + if (irq < 64) + write_sysreg_s(bit, SYS_ICC_PPI_CPENDR0_EL1); + else + write_sysreg_s(bit, SYS_ICC_PPI_CPENDR1_EL1); + } + return; + case PPI_ACTIVE: + if (set) { + if (irq < 64) + write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER0_EL1); + else + write_sysreg_s(bit, SYS_ICC_PPI_SACTIVER1_EL1); + } else { + if (irq < 64) + write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER0_EL1); + else + write_sysreg_s(bit, SYS_ICC_PPI_CACTIVER1_EL1); + } + return; + default: + BUILD_BUG_ON(1); + } +} + +static int gicv5_ppi_irq_get_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool *state) +{ + u64 hwirq_id_bit = BIT_ULL(d->hwirq % 64); + + switch (which) { + case IRQCHIP_STATE_PENDING: + *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_PENDING) & hwirq_id_bit); + return 0; + case IRQCHIP_STATE_ACTIVE: + *state = !!(read_ppi_sysreg_s(d->hwirq, PPI_ACTIVE) & hwirq_id_bit); + return 0; + default: + pr_debug("Unexpected PPI irqchip state\n"); + return -EINVAL; + } +} + +static int gicv5_iri_irq_get_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool *state, u8 hwirq_type) +{ + u64 icsr, cdrcfg; + + cdrcfg = d->hwirq | FIELD_PREP(GICV5_GIC_CDRCFG_TYPE_MASK, hwirq_type); + + gic_insn(cdrcfg, CDRCFG); + isb(); + icsr = read_sysreg_s(SYS_ICC_ICSR_EL1); + + if (FIELD_GET(ICC_ICSR_EL1_F, icsr)) { + pr_err("ICSR_EL1 is invalid\n"); + return -EINVAL; + } + + switch (which) { + case IRQCHIP_STATE_PENDING: + *state = !!(FIELD_GET(ICC_ICSR_EL1_Pending, icsr)); + return 0; + + case IRQCHIP_STATE_ACTIVE: + *state = !!(FIELD_GET(ICC_ICSR_EL1_Active, icsr)); + return 0; + + default: + pr_debug("Unexpected irqchip_irq_state\n"); + return -EINVAL; + } +} + +static int gicv5_spi_irq_get_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool *state) +{ + return gicv5_iri_irq_get_irqchip_state(d, which, state, + GICV5_HWIRQ_TYPE_SPI); +} + +static int gicv5_lpi_irq_get_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool *state) +{ + return gicv5_iri_irq_get_irqchip_state(d, which, state, + GICV5_HWIRQ_TYPE_LPI); +} + +static int gicv5_ppi_irq_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool state) +{ + switch (which) { + case IRQCHIP_STATE_PENDING: + write_ppi_sysreg_s(d->hwirq, state, PPI_PENDING); + return 0; + case IRQCHIP_STATE_ACTIVE: + write_ppi_sysreg_s(d->hwirq, state, PPI_ACTIVE); + return 0; + default: + pr_debug("Unexpected PPI irqchip state\n"); + return -EINVAL; + } +} + +static void gicv5_iri_irq_write_pending_state(struct irq_data *d, bool state, + u8 hwirq_type) +{ + u64 cdpend; + + cdpend = FIELD_PREP(GICV5_GIC_CDPEND_TYPE_MASK, hwirq_type) | + FIELD_PREP(GICV5_GIC_CDPEND_ID_MASK, d->hwirq) | + FIELD_PREP(GICV5_GIC_CDPEND_PENDING_MASK, state); + + gic_insn(cdpend, CDPEND); +} + +static void gicv5_spi_irq_write_pending_state(struct irq_data *d, bool state) +{ + gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_SPI); +} + +static void gicv5_lpi_irq_write_pending_state(struct irq_data *d, bool state) +{ + gicv5_iri_irq_write_pending_state(d, state, GICV5_HWIRQ_TYPE_LPI); +} + +static int gicv5_spi_irq_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool state) +{ + switch (which) { + case IRQCHIP_STATE_PENDING: + gicv5_spi_irq_write_pending_state(d, state); + break; + default: + pr_debug("Unexpected irqchip_irq_state\n"); + return -EINVAL; + } + + return 0; +} + +static int gicv5_lpi_irq_set_irqchip_state(struct irq_data *d, + enum irqchip_irq_state which, + bool state) +{ + switch (which) { + case IRQCHIP_STATE_PENDING: + gicv5_lpi_irq_write_pending_state(d, state); + break; + + default: + pr_debug("Unexpected irqchip_irq_state\n"); + return -EINVAL; + } + + return 0; +} + +static int gicv5_spi_irq_retrigger(struct irq_data *data) +{ + return !gicv5_spi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, + true); +} + +static int gicv5_lpi_irq_retrigger(struct irq_data *data) +{ + return !gicv5_lpi_irq_set_irqchip_state(data, IRQCHIP_STATE_PENDING, + true); +} + +static void gicv5_ipi_send_single(struct irq_data *d, unsigned int cpu) +{ + /* Mark the LPI pending */ + irq_chip_retrigger_hierarchy(d); +} + +static bool gicv5_ppi_irq_is_level(irq_hw_number_t hwirq) +{ + u64 bit = BIT_ULL(hwirq % 64); + + return !!(read_ppi_sysreg_s(hwirq, PPI_HM) & bit); +} + +static int gicv5_ppi_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) +{ + if (vcpu) + irqd_set_forwarded_to_vcpu(d); + else + irqd_clr_forwarded_to_vcpu(d); + + return 0; +} + +static const struct irq_chip gicv5_ppi_irq_chip = { + .name = "GICv5-PPI", + .irq_mask = gicv5_ppi_irq_mask, + .irq_unmask = gicv5_ppi_irq_unmask, + .irq_eoi = gicv5_ppi_irq_eoi, + .irq_get_irqchip_state = gicv5_ppi_irq_get_irqchip_state, + .irq_set_irqchip_state = gicv5_ppi_irq_set_irqchip_state, + .irq_set_vcpu_affinity = gicv5_ppi_irq_set_vcpu_affinity, + .flags = IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static const struct irq_chip gicv5_spi_irq_chip = { + .name = "GICv5-SPI", + .irq_mask = gicv5_spi_irq_mask, + .irq_unmask = gicv5_spi_irq_unmask, + .irq_eoi = gicv5_spi_irq_eoi, + .irq_set_type = gicv5_spi_irq_set_type, + .irq_set_affinity = gicv5_spi_irq_set_affinity, + .irq_retrigger = gicv5_spi_irq_retrigger, + .irq_get_irqchip_state = gicv5_spi_irq_get_irqchip_state, + .irq_set_irqchip_state = gicv5_spi_irq_set_irqchip_state, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static const struct irq_chip gicv5_lpi_irq_chip = { + .name = "GICv5-LPI", + .irq_mask = gicv5_lpi_irq_mask, + .irq_unmask = gicv5_lpi_irq_unmask, + .irq_eoi = gicv5_lpi_irq_eoi, + .irq_set_affinity = gicv5_lpi_irq_set_affinity, + .irq_retrigger = gicv5_lpi_irq_retrigger, + .irq_get_irqchip_state = gicv5_lpi_irq_get_irqchip_state, + .irq_set_irqchip_state = gicv5_lpi_irq_set_irqchip_state, + .flags = IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static const struct irq_chip gicv5_ipi_irq_chip = { + .name = "GICv5-IPI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_affinity = irq_chip_set_affinity_parent, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .ipi_send_single = gicv5_ipi_send_single, + .flags = IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static __always_inline int gicv5_irq_domain_translate(struct irq_domain *d, + struct irq_fwspec *fwspec, + irq_hw_number_t *hwirq, + unsigned int *type, + const u8 hwirq_type) +{ + if (!is_of_node(fwspec->fwnode)) + return -EINVAL; + + if (fwspec->param_count < 3) + return -EINVAL; + + if (fwspec->param[0] != hwirq_type) + return -EINVAL; + + *hwirq = fwspec->param[1]; + + switch (hwirq_type) { + case GICV5_HWIRQ_TYPE_PPI: + /* + * Handling mode is hardcoded for PPIs, set the type using + * HW reported value. + */ + *type = gicv5_ppi_irq_is_level(*hwirq) ? IRQ_TYPE_LEVEL_LOW : + IRQ_TYPE_EDGE_RISING; + break; + case GICV5_HWIRQ_TYPE_SPI: + *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + break; + default: + BUILD_BUG_ON(1); + } + + return 0; +} + +static int gicv5_irq_ppi_domain_translate(struct irq_domain *d, + struct irq_fwspec *fwspec, + irq_hw_number_t *hwirq, + unsigned int *type) +{ + return gicv5_irq_domain_translate(d, fwspec, hwirq, type, + GICV5_HWIRQ_TYPE_PPI); +} + +static int gicv5_irq_ppi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + int ret; + + if (WARN_ON_ONCE(nr_irqs != 1)) + return -EINVAL; + + ret = gicv5_irq_ppi_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + if (type & IRQ_TYPE_LEVEL_MASK) + irq_set_status_flags(virq, IRQ_LEVEL); + + irq_set_percpu_devid(virq); + irq_domain_set_info(domain, virq, hwirq, &gicv5_ppi_irq_chip, NULL, + handle_percpu_devid_irq, NULL, NULL); + + return 0; +} + +static void gicv5_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d; + + if (WARN_ON_ONCE(nr_irqs != 1)) + return; + + d = irq_domain_get_irq_data(domain, virq); + + irq_set_handler(virq, NULL); + irq_domain_reset_irq_data(d); +} + +static int gicv5_irq_ppi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, + enum irq_domain_bus_token bus_token) +{ + if (fwspec->fwnode != d->fwnode) + return 0; + + if (fwspec->param[0] != GICV5_HWIRQ_TYPE_PPI) + return 0; + + return (d == gicv5_global_data.ppi_domain); +} + +static const struct irq_domain_ops gicv5_irq_ppi_domain_ops = { + .translate = gicv5_irq_ppi_domain_translate, + .alloc = gicv5_irq_ppi_domain_alloc, + .free = gicv5_irq_domain_free, + .select = gicv5_irq_ppi_domain_select +}; + +static int gicv5_irq_spi_domain_translate(struct irq_domain *d, + struct irq_fwspec *fwspec, + irq_hw_number_t *hwirq, + unsigned int *type) +{ + return gicv5_irq_domain_translate(d, fwspec, hwirq, type, + GICV5_HWIRQ_TYPE_SPI); +} + +static int gicv5_irq_spi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct gicv5_irs_chip_data *chip_data; + unsigned int type = IRQ_TYPE_NONE; + struct irq_fwspec *fwspec = arg; + struct irq_data *irqd; + irq_hw_number_t hwirq; + int ret; + + if (WARN_ON_ONCE(nr_irqs != 1)) + return -EINVAL; + + ret = gicv5_irq_spi_domain_translate(domain, fwspec, &hwirq, &type); + if (ret) + return ret; + + irqd = irq_desc_get_irq_data(irq_to_desc(virq)); + chip_data = gicv5_irs_lookup_by_spi_id(hwirq); + + irq_domain_set_info(domain, virq, hwirq, &gicv5_spi_irq_chip, chip_data, + handle_fasteoi_irq, NULL, NULL); + irq_set_probe(virq); + irqd_set_single_target(irqd); + + gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_SPI); + + return 0; +} + +static int gicv5_irq_spi_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, + enum irq_domain_bus_token bus_token) +{ + if (fwspec->fwnode != d->fwnode) + return 0; + + if (fwspec->param[0] != GICV5_HWIRQ_TYPE_SPI) + return 0; + + return (d == gicv5_global_data.spi_domain); +} + +static const struct irq_domain_ops gicv5_irq_spi_domain_ops = { + .translate = gicv5_irq_spi_domain_translate, + .alloc = gicv5_irq_spi_domain_alloc, + .free = gicv5_irq_domain_free, + .select = gicv5_irq_spi_domain_select +}; + +static void gicv5_lpi_config_reset(struct irq_data *d) +{ + u64 cdhm; + + /* + * Reset LPIs handling mode to edge by default and clear pending + * state to make sure we start the LPI with a clean state from + * previous incarnations. + */ + cdhm = FIELD_PREP(GICV5_GIC_CDHM_HM_MASK, 0) | + FIELD_PREP(GICV5_GIC_CDHM_TYPE_MASK, GICV5_HWIRQ_TYPE_LPI) | + FIELD_PREP(GICV5_GIC_CDHM_ID_MASK, d->hwirq); + gic_insn(cdhm, CDHM); + + gicv5_lpi_irq_write_pending_state(d, false); +} + +static int gicv5_irq_lpi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + irq_hw_number_t hwirq; + struct irq_data *irqd; + u32 *lpi = arg; + int ret; + + if (WARN_ON_ONCE(nr_irqs != 1)) + return -EINVAL; + + hwirq = *lpi; + + irqd = irq_domain_get_irq_data(domain, virq); + + irq_domain_set_info(domain, virq, hwirq, &gicv5_lpi_irq_chip, NULL, + handle_fasteoi_irq, NULL, NULL); + irqd_set_single_target(irqd); + + ret = gicv5_irs_iste_alloc(hwirq); + if (ret < 0) + return ret; + + gicv5_hwirq_init(hwirq, GICV5_IRQ_PRI_MI, GICV5_HWIRQ_TYPE_LPI); + gicv5_lpi_config_reset(irqd); + + return 0; +} + +static const struct irq_domain_ops gicv5_irq_lpi_domain_ops = { + .alloc = gicv5_irq_lpi_domain_alloc, + .free = gicv5_irq_domain_free, +}; + +void __init gicv5_init_lpi_domain(void) +{ + struct irq_domain *d; + + d = irq_domain_create_tree(NULL, &gicv5_irq_lpi_domain_ops, NULL); + gicv5_global_data.lpi_domain = d; +} + +void __init gicv5_free_lpi_domain(void) +{ + irq_domain_remove(gicv5_global_data.lpi_domain); + gicv5_global_data.lpi_domain = NULL; +} + +static int gicv5_irq_ipi_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct irq_data *irqd; + int ret, i; + u32 lpi; + + for (i = 0; i < nr_irqs; i++) { + ret = gicv5_alloc_lpi(); + if (ret < 0) + return ret; + + lpi = ret; + + ret = irq_domain_alloc_irqs_parent(domain, virq + i, 1, &lpi); + if (ret) { + gicv5_free_lpi(lpi); + return ret; + } + + irqd = irq_domain_get_irq_data(domain, virq + i); + + irq_domain_set_hwirq_and_chip(domain, virq + i, i, + &gicv5_ipi_irq_chip, NULL); + + irqd_set_single_target(irqd); + + irq_set_handler(virq + i, handle_percpu_irq); + } + + return 0; +} + +static void gicv5_irq_ipi_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d; + unsigned int i; + + for (i = 0; i < nr_irqs; i++) { + d = irq_domain_get_irq_data(domain, virq + i); + + if (!d) + return; + + gicv5_free_lpi(d->parent_data->hwirq); + + irq_set_handler(virq + i, NULL); + irq_domain_reset_irq_data(d); + irq_domain_free_irqs_parent(domain, virq + i, 1); + } +} + +static const struct irq_domain_ops gicv5_irq_ipi_domain_ops = { + .alloc = gicv5_irq_ipi_domain_alloc, + .free = gicv5_irq_ipi_domain_free, +}; + +static void handle_irq_per_domain(u32 hwirq) +{ + u8 hwirq_type = FIELD_GET(GICV5_HWIRQ_TYPE, hwirq); + u32 hwirq_id = FIELD_GET(GICV5_HWIRQ_ID, hwirq); + struct irq_domain *domain; + + switch (hwirq_type) { + case GICV5_HWIRQ_TYPE_PPI: + domain = gicv5_global_data.ppi_domain; + break; + case GICV5_HWIRQ_TYPE_SPI: + domain = gicv5_global_data.spi_domain; + break; + case GICV5_HWIRQ_TYPE_LPI: + domain = gicv5_global_data.lpi_domain; + break; + default: + pr_err_once("Unknown IRQ type, bail out\n"); + return; + } + + if (generic_handle_domain_irq(domain, hwirq_id)) { + pr_err_once("Could not handle, hwirq = 0x%x", hwirq_id); + gicv5_hwirq_eoi(hwirq_id, hwirq_type); + } +} + +static void __exception_irq_entry gicv5_handle_irq(struct pt_regs *regs) +{ + bool valid; + u32 hwirq; + u64 ia; + + ia = gicr_insn(CDIA); + valid = GICV5_GICR_CDIA_VALID(ia); + + if (!valid) + return; + + /* + * Ensure that the CDIA instruction effects (ie IRQ activation) are + * completed before handling the interrupt. + */ + gsb_ack(); + + /* + * Ensure instruction ordering between an acknowledgment and subsequent + * instructions in the IRQ handler using an ISB. + */ + isb(); + + hwirq = FIELD_GET(GICV5_HWIRQ_INTID, ia); + + handle_irq_per_domain(hwirq); +} + +static void gicv5_cpu_disable_interrupts(void) +{ + u64 cr0; + + cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 0); + write_sysreg_s(cr0, SYS_ICC_CR0_EL1); +} + +static void gicv5_cpu_enable_interrupts(void) +{ + u64 cr0, pcr; + + write_sysreg_s(0, SYS_ICC_PPI_ENABLER0_EL1); + write_sysreg_s(0, SYS_ICC_PPI_ENABLER1_EL1); + + gicv5_ppi_priority_init(); + + pcr = FIELD_PREP(ICC_PCR_EL1_PRIORITY, GICV5_IRQ_PRI_MI); + write_sysreg_s(pcr, SYS_ICC_PCR_EL1); + + cr0 = FIELD_PREP(ICC_CR0_EL1_EN, 1); + write_sysreg_s(cr0, SYS_ICC_CR0_EL1); +} + +static int base_ipi_virq; + +static int gicv5_starting_cpu(unsigned int cpu) +{ + if (WARN(!gicv5_cpuif_has_gcie(), + "GICv5 system components present but CPU does not have FEAT_GCIE")) + return -ENODEV; + + gicv5_cpu_enable_interrupts(); + + return gicv5_irs_register_cpu(cpu); +} + +static void __init gicv5_smp_init(void) +{ + unsigned int num_ipis = GICV5_IPIS_PER_CPU * nr_cpu_ids; + + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, + "irqchip/arm/gicv5:starting", + gicv5_starting_cpu, NULL); + + base_ipi_virq = irq_domain_alloc_irqs(gicv5_global_data.ipi_domain, + num_ipis, NUMA_NO_NODE, NULL); + if (WARN(base_ipi_virq <= 0, "IPI IRQ allocation was not successful")) + return; + + set_smp_ipi_range_percpu(base_ipi_virq, GICV5_IPIS_PER_CPU, nr_cpu_ids); +} + +static void __init gicv5_free_domains(void) +{ + if (gicv5_global_data.ppi_domain) + irq_domain_remove(gicv5_global_data.ppi_domain); + if (gicv5_global_data.spi_domain) + irq_domain_remove(gicv5_global_data.spi_domain); + if (gicv5_global_data.ipi_domain) + irq_domain_remove(gicv5_global_data.ipi_domain); + + gicv5_global_data.ppi_domain = NULL; + gicv5_global_data.spi_domain = NULL; + gicv5_global_data.ipi_domain = NULL; +} + +static int __init gicv5_init_domains(struct fwnode_handle *handle) +{ + u32 spi_count = gicv5_global_data.global_spi_count; + struct irq_domain *d; + + d = irq_domain_create_linear(handle, PPI_NR, &gicv5_irq_ppi_domain_ops, NULL); + if (!d) + return -ENOMEM; + + irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED); + gicv5_global_data.ppi_domain = d; + + if (spi_count) { + d = irq_domain_create_linear(handle, spi_count, + &gicv5_irq_spi_domain_ops, NULL); + + if (!d) { + gicv5_free_domains(); + return -ENOMEM; + } + + gicv5_global_data.spi_domain = d; + irq_domain_update_bus_token(d, DOMAIN_BUS_WIRED); + } + + if (!WARN(!gicv5_global_data.lpi_domain, + "LPI domain uninitialized, can't set up IPIs")) { + d = irq_domain_create_hierarchy(gicv5_global_data.lpi_domain, + 0, GICV5_IPIS_PER_CPU * nr_cpu_ids, + NULL, &gicv5_irq_ipi_domain_ops, + NULL); + + if (!d) { + gicv5_free_domains(); + return -ENOMEM; + } + gicv5_global_data.ipi_domain = d; + } + gicv5_global_data.fwnode = handle; + + return 0; +} + +static void gicv5_set_cpuif_pribits(void) +{ + u64 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1); + + switch (FIELD_GET(ICC_IDR0_EL1_PRI_BITS, icc_idr0)) { + case ICC_IDR0_EL1_PRI_BITS_4BITS: + gicv5_global_data.cpuif_pri_bits = 4; + break; + case ICC_IDR0_EL1_PRI_BITS_5BITS: + gicv5_global_data.cpuif_pri_bits = 5; + break; + default: + pr_err("Unexpected ICC_IDR0_EL1_PRI_BITS value, default to 4"); + gicv5_global_data.cpuif_pri_bits = 4; + break; + } +} + +static void gicv5_set_cpuif_idbits(void) +{ + u32 icc_idr0 = read_sysreg_s(SYS_ICC_IDR0_EL1); + + switch (FIELD_GET(ICC_IDR0_EL1_ID_BITS, icc_idr0)) { + case ICC_IDR0_EL1_ID_BITS_16BITS: + gicv5_global_data.cpuif_id_bits = 16; + break; + case ICC_IDR0_EL1_ID_BITS_24BITS: + gicv5_global_data.cpuif_id_bits = 24; + break; + default: + pr_err("Unexpected ICC_IDR0_EL1_ID_BITS value, default to 16"); + gicv5_global_data.cpuif_id_bits = 16; + break; + } +} + +#ifdef CONFIG_KVM +static struct gic_kvm_info gic_v5_kvm_info __initdata; + +static void __init gic_of_setup_kvm_info(struct device_node *node) +{ + gic_v5_kvm_info.type = GIC_V5; + + /* GIC Virtual CPU interface maintenance interrupt */ + gic_v5_kvm_info.no_maint_irq_mask = false; + gic_v5_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); + if (!gic_v5_kvm_info.maint_irq) { + pr_warn("cannot find GICv5 virtual CPU interface maintenance interrupt\n"); + return; + } + + vgic_set_kvm_info(&gic_v5_kvm_info); +} +#else +static inline void __init gic_of_setup_kvm_info(struct device_node *node) +{ +} +#endif // CONFIG_KVM + +static int __init gicv5_of_init(struct device_node *node, struct device_node *parent) +{ + int ret = gicv5_irs_of_probe(node); + if (ret) + return ret; + + ret = gicv5_init_domains(of_fwnode_handle(node)); + if (ret) + goto out_irs; + + gicv5_set_cpuif_pribits(); + gicv5_set_cpuif_idbits(); + + pri_bits = min_not_zero(gicv5_global_data.cpuif_pri_bits, + gicv5_global_data.irs_pri_bits); + + ret = gicv5_starting_cpu(smp_processor_id()); + if (ret) + goto out_dom; + + ret = set_handle_irq(gicv5_handle_irq); + if (ret) + goto out_int; + + ret = gicv5_irs_enable(); + if (ret) + goto out_int; + + gicv5_smp_init(); + + gicv5_irs_its_probe(); + + gic_of_setup_kvm_info(node); + + return 0; + +out_int: + gicv5_cpu_disable_interrupts(); +out_dom: + gicv5_free_domains(); +out_irs: + gicv5_irs_remove(); + + return ret; +} +IRQCHIP_DECLARE(gic_v5, "arm,gic-v5", gicv5_of_init); diff --git a/drivers/irqchip/irq-gic.c b/drivers/irqchip/irq-gic.c index 412196a7dad5..ec70c84e9f91 100644 --- a/drivers/irqchip/irq-gic.c +++ b/drivers/irqchip/irq-gic.c @@ -54,7 +54,7 @@ static void gic_check_cpu_features(void) { - WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_GIC_CPUIF_SYSREGS), + WARN_TAINT_ONCE(this_cpu_has_cap(ARM64_HAS_GICV3_CPUIF), TAINT_CPU_OUT_OF_SPEC, "GICv3 system registers enabled, broken firmware!\n"); } @@ -64,7 +64,7 @@ static void gic_check_cpu_features(void) union gic_base { void __iomem *common_base; - void __percpu * __iomem *percpu_base; + void __iomem * __percpu *percpu_base; }; struct gic_chip_data { @@ -162,11 +162,6 @@ static inline void __iomem *gic_cpu_base(struct irq_data *d) return gic_data_cpu_base(gic_data); } -static inline unsigned int gic_irq(struct irq_data *d) -{ - return d->hwirq; -} - static inline bool cascading_gic_irq(struct irq_data *d) { void *data = irq_data_get_irq_handler_data(d); @@ -183,14 +178,16 @@ static inline bool cascading_gic_irq(struct irq_data *d) */ static void gic_poke_irq(struct irq_data *d, u32 offset) { - u32 mask = 1 << (gic_irq(d) % 32); - writel_relaxed(mask, gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4); + u32 mask = 1 << (irqd_to_hwirq(d) % 32); + + writel_relaxed(mask, gic_dist_base(d) + offset + (irqd_to_hwirq(d) / 32) * 4); } static int gic_peek_irq(struct irq_data *d, u32 offset) { - u32 mask = 1 << (gic_irq(d) % 32); - return !!(readl_relaxed(gic_dist_base(d) + offset + (gic_irq(d) / 32) * 4) & mask); + u32 mask = 1 << (irqd_to_hwirq(d) % 32); + + return !!(readl_relaxed(gic_dist_base(d) + offset + (irqd_to_hwirq(d) / 32) * 4) & mask); } static void gic_mask_irq(struct irq_data *d) @@ -220,7 +217,7 @@ static void gic_unmask_irq(struct irq_data *d) static void gic_eoi_irq(struct irq_data *d) { - u32 hwirq = gic_irq(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); if (hwirq < 16) hwirq = this_cpu_read(sgi_intid); @@ -230,7 +227,7 @@ static void gic_eoi_irq(struct irq_data *d) static void gic_eoimode1_eoi_irq(struct irq_data *d) { - u32 hwirq = gic_irq(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); /* Do not deactivate an IRQ forwarded to a vcpu. */ if (irqd_is_forwarded_to_vcpu(d)) @@ -293,8 +290,8 @@ static int gic_irq_get_irqchip_state(struct irq_data *d, static int gic_set_type(struct irq_data *d, unsigned int type) { + irq_hw_number_t gicirq = irqd_to_hwirq(d); void __iomem *base = gic_dist_base(d); - unsigned int gicirq = gic_irq(d); int ret; /* Interrupt configuration for SGIs can't be changed */ @@ -306,10 +303,10 @@ static int gic_set_type(struct irq_data *d, unsigned int type) type != IRQ_TYPE_EDGE_RISING) return -EINVAL; - ret = gic_configure_irq(gicirq, type, base + GIC_DIST_CONFIG, NULL); + ret = gic_configure_irq(gicirq, type, base + GIC_DIST_CONFIG); if (ret && gicirq < 32) { /* Misconfigured PPIs are usually not fatal */ - pr_warn("GIC: PPI%d is secure or misconfigured\n", gicirq - 16); + pr_warn("GIC: PPI%ld is secure or misconfigured\n", gicirq - 16); ret = 0; } @@ -319,7 +316,7 @@ static int gic_set_type(struct irq_data *d, unsigned int type) static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) { /* Only interrupts on the primary GIC can be forwarded to a vcpu. */ - if (cascading_gic_irq(d) || gic_irq(d) < 16) + if (cascading_gic_irq(d) || irqd_to_hwirq(d) < 16) return -EINVAL; if (vcpu) @@ -403,7 +400,7 @@ static void gic_irq_print_chip(struct irq_data *d, struct seq_file *p) struct gic_chip_data *gic = irq_data_get_irq_chip_data(d); if (gic->domain->pm_dev) - seq_printf(p, gic->domain->pm_dev->of_node->name); + seq_puts(p, gic->domain->pm_dev->of_node->name); else seq_printf(p, "GIC-%d", (int)(gic - &gic_data[0])); } @@ -482,7 +479,7 @@ static void gic_dist_init(struct gic_chip_data *gic) for (i = 32; i < gic_irqs; i += 4) writel_relaxed(cpumask, base + GIC_DIST_TARGET + i * 4 / 4); - gic_dist_config(base, gic_irqs, NULL); + gic_dist_config(base, gic_irqs, GICD_INT_DEF_PRI); writel_relaxed(GICD_ENABLE, base + GIC_DIST_CTRL); } @@ -519,7 +516,7 @@ static int gic_cpu_init(struct gic_chip_data *gic) gic_cpu_map[i] &= ~cpu_mask; } - gic_cpu_config(dist_base, 32, NULL); + gic_cpu_config(dist_base, 32, GICD_INT_DEF_PRI); writel_relaxed(GICC_INT_PRI_THRESHOLD, base + GIC_CPU_PRIMASK); gic_cpu_if_up(gic); @@ -611,7 +608,7 @@ void gic_dist_restore(struct gic_chip_data *gic) dist_base + GIC_DIST_CONFIG + i * 4); for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) - writel_relaxed(GICD_INT_DEF_PRI_X4, + writel_relaxed(REPEAT_BYTE_U32(GICD_INT_DEF_PRI), dist_base + GIC_DIST_PRI + i * 4); for (i = 0; i < DIV_ROUND_UP(gic_irqs, 4); i++) @@ -700,7 +697,7 @@ void gic_cpu_restore(struct gic_chip_data *gic) writel_relaxed(ptr[i], dist_base + GIC_DIST_CONFIG + i * 4); for (i = 0; i < DIV_ROUND_UP(32, 4); i++) - writel_relaxed(GICD_INT_DEF_PRI_X4, + writel_relaxed(REPEAT_BYTE_U32(GICD_INT_DEF_PRI), dist_base + GIC_DIST_PRI + i * 4); writel_relaxed(GICC_INT_PRI_THRESHOLD, cpu_base + GIC_CPU_PRIMASK); @@ -796,7 +793,7 @@ static void rmw_writeb(u8 bval, void __iomem *addr) static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { - void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + gic_irq(d); + void __iomem *reg = gic_dist_base(d) + GIC_DIST_TARGET + irqd_to_hwirq(d); struct gic_chip_data *gic = irq_data_get_irq_chip_data(d); unsigned int cpu; @@ -1462,6 +1459,8 @@ static void __init gic_of_setup_kvm_info(struct device_node *node) if (ret) return; + gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base; + if (static_branch_likely(&supports_deactivate_key)) vgic_set_kvm_info(&gic_v2_kvm_info); } @@ -1623,6 +1622,7 @@ static void __init gic_acpi_setup_kvm_info(void) return; gic_v2_kvm_info.maint_irq = irq; + gic_v2_kvm_info.gicc_base = gic_data[0].cpu_base.common_base; vgic_set_kvm_info(&gic_v2_kvm_info); } diff --git a/drivers/irqchip/irq-goldfish-pic.c b/drivers/irqchip/irq-goldfish-pic.c index 513f6edbbe95..a8b23b507ecd 100644 --- a/drivers/irqchip/irq-goldfish-pic.c +++ b/drivers/irqchip/irq-goldfish-pic.c @@ -101,10 +101,9 @@ static int __init goldfish_pic_of_init(struct device_node *of_node, irq_setup_generic_chip(gc, IRQ_MSK(GFPIC_NR_IRQS), 0, IRQ_NOPROBE | IRQ_LEVEL, 0); - gfpic->irq_domain = irq_domain_add_legacy(of_node, GFPIC_NR_IRQS, - GFPIC_IRQ_BASE, 0, - &goldfish_irq_domain_ops, - NULL); + gfpic->irq_domain = irq_domain_create_legacy(of_fwnode_handle(of_node), GFPIC_NR_IRQS, + GFPIC_IRQ_BASE, 0, &goldfish_irq_domain_ops, + NULL); if (!gfpic->irq_domain) { pr_err("Failed to add irqdomain!\n"); ret = -ENOMEM; diff --git a/drivers/irqchip/irq-hip04.c b/drivers/irqchip/irq-hip04.c index 46161f6ff289..b7958c5a1221 100644 --- a/drivers/irqchip/irq-hip04.c +++ b/drivers/irqchip/irq-hip04.c @@ -130,7 +130,7 @@ static int hip04_irq_set_type(struct irq_data *d, unsigned int type) raw_spin_lock(&irq_controller_lock); - ret = gic_configure_irq(irq, type, base + GIC_DIST_CONFIG, NULL); + ret = gic_configure_irq(irq, type, base + GIC_DIST_CONFIG); if (ret && irq < 32) { /* Misconfigured PPIs are usually not fatal */ pr_warn("GIC: PPI%d is secure or misconfigured\n", irq - 16); @@ -260,7 +260,7 @@ static void __init hip04_irq_dist_init(struct hip04_irq_data *intc) for (i = 32; i < nr_irqs; i += 2) writel_relaxed(cpumask, base + GIC_DIST_TARGET + ((i * 2) & ~3)); - gic_dist_config(base, nr_irqs, NULL); + gic_dist_config(base, nr_irqs, GICD_INT_DEF_PRI); writel_relaxed(1, base + GIC_DIST_CTRL); } @@ -287,7 +287,7 @@ static void hip04_irq_cpu_init(struct hip04_irq_data *intc) if (i != cpu) hip04_cpu_map[i] &= ~cpu_mask; - gic_cpu_config(dist_base, 32, NULL); + gic_cpu_config(dist_base, 32, GICD_INT_DEF_PRI); writel_relaxed(0xf0, base + GIC_CPU_PRIMASK); writel_relaxed(1, base + GIC_CPU_CTRL); @@ -386,10 +386,8 @@ hip04_of_init(struct device_node *node, struct device_node *parent) return -EINVAL; } - hip04_data.domain = irq_domain_add_legacy(node, nr_irqs, irq_base, - 0, - &hip04_irq_domain_ops, - &hip04_data); + hip04_data.domain = irq_domain_create_legacy(of_fwnode_handle(node), nr_irqs, irq_base, 0, + &hip04_irq_domain_ops, &hip04_data); if (WARN_ON(!hip04_data.domain)) return -EINVAL; diff --git a/drivers/irqchip/irq-i8259.c b/drivers/irqchip/irq-i8259.c index 115bdcffab24..cca77f9948a3 100644 --- a/drivers/irqchip/irq-i8259.c +++ b/drivers/irqchip/irq-i8259.c @@ -202,13 +202,13 @@ spurious_8259A_irq: } } -static void i8259A_resume(void) +static void i8259A_resume(void *data) { if (i8259A_auto_eoi >= 0) init_8259A(i8259A_auto_eoi); } -static void i8259A_shutdown(void) +static void i8259A_shutdown(void *data) { /* Put the i8259A into a quiescent state that * the kernel initialization code can get it @@ -220,11 +220,15 @@ static void i8259A_shutdown(void) } } -static struct syscore_ops i8259_syscore_ops = { +static const struct syscore_ops i8259_syscore_ops = { .resume = i8259A_resume, .shutdown = i8259A_shutdown, }; +static struct syscore i8259_syscore = { + .ops = &i8259_syscore_ops, +}; + static void init_8259A(int auto_eoi) { unsigned long flags; @@ -313,14 +317,14 @@ struct irq_domain * __init __init_i8259_irqs(struct device_node *node) init_8259A(0); - domain = irq_domain_add_legacy(node, 16, I8259A_IRQ_BASE, 0, - &i8259A_ops, NULL); + domain = irq_domain_create_legacy(of_fwnode_handle(node), 16, I8259A_IRQ_BASE, 0, + &i8259A_ops, NULL); if (!domain) panic("Failed to add i8259 IRQ domain"); if (request_irq(irq, no_action, IRQF_NO_THREAD, "cascade", NULL)) pr_err("Failed to register cascade interrupt\n"); - register_syscore_ops(&i8259_syscore_ops); + register_syscore(&i8259_syscore); return domain; } diff --git a/drivers/irqchip/irq-idt3243x.c b/drivers/irqchip/irq-idt3243x.c index 0732a0e9af62..f8324fb1fe8f 100644 --- a/drivers/irqchip/irq-idt3243x.c +++ b/drivers/irqchip/irq-idt3243x.c @@ -72,7 +72,7 @@ static int idt_pic_init(struct device_node *of_node, struct device_node *parent) goto out_unmap_irq; } - domain = irq_domain_add_linear(of_node, IDT_PIC_NR_IRQS, + domain = irq_domain_create_linear(of_fwnode_handle(of_node), IDT_PIC_NR_IRQS, &irq_generic_chip_ops, NULL); if (!domain) { pr_err("Failed to add irqdomain!\n"); diff --git a/drivers/irqchip/irq-imgpdc.c b/drivers/irqchip/irq-imgpdc.c index 5831be454673..e9ef2f5a7207 100644 --- a/drivers/irqchip/irq-imgpdc.c +++ b/drivers/irqchip/irq-imgpdc.c @@ -372,8 +372,8 @@ static int pdc_intc_probe(struct platform_device *pdev) priv->syswake_irq = irq; /* Set up an IRQ domain */ - priv->domain = irq_domain_add_linear(node, 16, &irq_generic_chip_ops, - priv); + priv->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), 16, &irq_generic_chip_ops, + priv); if (unlikely(!priv->domain)) { dev_err(&pdev->dev, "cannot add IRQ domain\n"); return -ENOMEM; @@ -461,12 +461,11 @@ err_generic: return ret; } -static int pdc_intc_remove(struct platform_device *pdev) +static void pdc_intc_remove(struct platform_device *pdev) { struct pdc_intc_priv *priv = platform_get_drvdata(pdev); irq_domain_remove(priv->domain); - return 0; } static const struct of_device_id pdc_intc_match[] = { @@ -479,8 +478,8 @@ static struct platform_driver pdc_intc_driver = { .name = "pdc-intc", .of_match_table = pdc_intc_match, }, - .probe = pdc_intc_probe, - .remove = pdc_intc_remove, + .probe = pdc_intc_probe, + .remove = pdc_intc_remove, }; static int __init pdc_intc_init(void) diff --git a/drivers/irqchip/irq-imx-gpcv2.c b/drivers/irqchip/irq-imx-gpcv2.c index 8a0e82067924..04f7ba0657be 100644 --- a/drivers/irqchip/irq-imx-gpcv2.c +++ b/drivers/irqchip/irq-imx-gpcv2.c @@ -33,7 +33,7 @@ static void __iomem *gpcv2_idx_to_reg(struct gpcv2_irqchip_data *cd, int i) return cd->gpc_base + cd->cpu2wakeup + i * 4; } -static int gpcv2_wakeup_source_save(void) +static int gpcv2_wakeup_source_save(void *data) { struct gpcv2_irqchip_data *cd; void __iomem *reg; @@ -52,7 +52,7 @@ static int gpcv2_wakeup_source_save(void) return 0; } -static void gpcv2_wakeup_source_restore(void) +static void gpcv2_wakeup_source_restore(void *data) { struct gpcv2_irqchip_data *cd; int i; @@ -65,9 +65,13 @@ static void gpcv2_wakeup_source_restore(void) writel_relaxed(cd->saved_irq_mask[i], gpcv2_idx_to_reg(cd, i)); } -static struct syscore_ops imx_gpcv2_syscore_ops = { - .suspend = gpcv2_wakeup_source_save, - .resume = gpcv2_wakeup_source_restore, +static const struct syscore_ops gpcv2_syscore_ops = { + .suspend = gpcv2_wakeup_source_save, + .resume = gpcv2_wakeup_source_restore, +}; + +static struct syscore gpcv2_syscore = { + .ops = &gpcv2_syscore_ops, }; static int imx_gpcv2_irq_set_wake(struct irq_data *d, unsigned int on) @@ -240,14 +244,14 @@ static int __init imx_gpcv2_irqchip_init(struct device_node *node, return -ENOMEM; } - domain = irq_domain_add_hierarchy(parent_domain, 0, GPC_MAX_IRQS, - node, &gpcv2_irqchip_data_domain_ops, cd); + domain = irq_domain_create_hierarchy(parent_domain, 0, GPC_MAX_IRQS, + of_fwnode_handle(node), &gpcv2_irqchip_data_domain_ops, cd); if (!domain) { iounmap(cd->gpc_base); kfree(cd); return -ENOMEM; } - irq_set_default_host(domain); + irq_set_default_domain(domain); /* Initially mask all interrupts */ for (i = 0; i < IMR_NUM; i++) { @@ -276,7 +280,7 @@ static int __init imx_gpcv2_irqchip_init(struct device_node *node, writel_relaxed(~0x1, cd->gpc_base + cd->cpu2wakeup); imx_gpcv2_instance = cd; - register_syscore_ops(&imx_gpcv2_syscore_ops); + register_syscore(&gpcv2_syscore); /* * Clear the OF_POPULATED flag set in of_irq_init so that diff --git a/drivers/irqchip/irq-imx-intmux.c b/drivers/irqchip/irq-imx-intmux.c index 6d9a08238c9d..5f9b204d350b 100644 --- a/drivers/irqchip/irq-imx-intmux.c +++ b/drivers/irqchip/irq-imx-intmux.c @@ -73,7 +73,7 @@ struct intmux_data { void __iomem *regs; struct clk *ipg_clk; int channum; - struct intmux_irqchip_data irqchip_data[]; + struct intmux_irqchip_data irqchip_data[] __counted_by(channum); }; static void imx_intmux_irq_mask(struct irq_data *d) @@ -166,6 +166,10 @@ static int imx_intmux_irq_select(struct irq_domain *d, struct irq_fwspec *fwspec if (fwspec->fwnode != d->fwnode) return false; + /* Handle pure domain searches */ + if (!fwspec->param_count) + return d->bus_token == bus_token; + return irqchip_data->chanidx == fwspec->param[1]; } @@ -250,7 +254,7 @@ static int imx_intmux_probe(struct platform_device *pdev) goto out; } - domain = irq_domain_add_linear(np, 32, &imx_intmux_domain_ops, + domain = irq_domain_create_linear(of_fwnode_handle(np), 32, &imx_intmux_domain_ops, &data->irqchip_data[i]); if (!domain) { ret = -ENOMEM; @@ -282,7 +286,7 @@ out: return ret; } -static int imx_intmux_remove(struct platform_device *pdev) +static void imx_intmux_remove(struct platform_device *pdev) { struct intmux_data *data = platform_get_drvdata(pdev); int i; @@ -298,8 +302,6 @@ static int imx_intmux_remove(struct platform_device *pdev) } pm_runtime_disable(&pdev->dev); - - return 0; } #ifdef CONFIG_PM @@ -354,11 +356,11 @@ static const struct of_device_id imx_intmux_id[] = { static struct platform_driver imx_intmux_driver = { .driver = { - .name = "imx-intmux", - .of_match_table = imx_intmux_id, - .pm = &imx_intmux_pm_ops, + .name = "imx-intmux", + .of_match_table = imx_intmux_id, + .pm = &imx_intmux_pm_ops, }, - .probe = imx_intmux_probe, - .remove = imx_intmux_remove, + .probe = imx_intmux_probe, + .remove = imx_intmux_remove, }; builtin_platform_driver(imx_intmux_driver); diff --git a/drivers/irqchip/irq-imx-irqsteer.c b/drivers/irqchip/irq-imx-irqsteer.c index bd9543314539..4682ce5bf8d3 100644 --- a/drivers/irqchip/irq-imx-irqsteer.c +++ b/drivers/irqchip/irq-imx-irqsteer.c @@ -24,7 +24,7 @@ #define CHAN_MINTDIS(t) (CTRL_STRIDE_OFF(t, 3) + 0x4) #define CHAN_MASTRSTAT(t) (CTRL_STRIDE_OFF(t, 3) + 0x8) -#define CHAN_MAX_OUTPUT_INT 0x8 +#define CHAN_MAX_OUTPUT_INT 0xF struct irqsteer_data { void __iomem *regs; @@ -36,6 +36,7 @@ struct irqsteer_data { int channel; struct irq_domain *domain; u32 *saved_reg; + struct device *dev; }; static int imx_irqsteer_get_reg_index(struct irqsteer_data *data, @@ -72,10 +73,26 @@ static void imx_irqsteer_irq_mask(struct irq_data *d) raw_spin_unlock_irqrestore(&data->lock, flags); } +static void imx_irqsteer_irq_bus_lock(struct irq_data *d) +{ + struct irqsteer_data *data = d->chip_data; + + pm_runtime_get_sync(data->dev); +} + +static void imx_irqsteer_irq_bus_sync_unlock(struct irq_data *d) +{ + struct irqsteer_data *data = d->chip_data; + + pm_runtime_put_autosuspend(data->dev); +} + static const struct irq_chip imx_irqsteer_irq_chip = { - .name = "irqsteer", - .irq_mask = imx_irqsteer_irq_mask, - .irq_unmask = imx_irqsteer_irq_unmask, + .name = "irqsteer", + .irq_mask = imx_irqsteer_irq_mask, + .irq_unmask = imx_irqsteer_irq_unmask, + .irq_bus_lock = imx_irqsteer_irq_bus_lock, + .irq_bus_sync_unlock = imx_irqsteer_irq_bus_sync_unlock, }; static int imx_irqsteer_irq_map(struct irq_domain *h, unsigned int irq, @@ -150,6 +167,7 @@ static int imx_irqsteer_probe(struct platform_device *pdev) if (!data) return -ENOMEM; + data->dev = &pdev->dev; data->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(data->regs)) { dev_err(&pdev->dev, "failed to initialize reg\n"); @@ -194,8 +212,8 @@ static int imx_irqsteer_probe(struct platform_device *pdev) /* steer all IRQs into configured channel */ writel_relaxed(BIT(data->channel), data->regs + CHANCTRL); - data->domain = irq_domain_add_linear(np, data->reg_num * 32, - &imx_irqsteer_domain_ops, data); + data->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), data->reg_num * 32, + &imx_irqsteer_domain_ops, data); if (!data->domain) { dev_err(&pdev->dev, "failed to create IRQ domain\n"); ret = -ENOMEM; @@ -210,10 +228,8 @@ static int imx_irqsteer_probe(struct platform_device *pdev) for (i = 0; i < data->irq_count; i++) { data->irq[i] = irq_of_parse_and_map(np, i); - if (!data->irq[i]) { - ret = -EINVAL; - goto out; - } + if (!data->irq[i]) + break; irq_set_chained_handler_and_data(data->irq[i], imx_irqsteer_irq_handler, @@ -231,20 +247,22 @@ out: return ret; } -static int imx_irqsteer_remove(struct platform_device *pdev) +static void imx_irqsteer_remove(struct platform_device *pdev) { struct irqsteer_data *irqsteer_data = platform_get_drvdata(pdev); int i; - for (i = 0; i < irqsteer_data->irq_count; i++) + for (i = 0; i < irqsteer_data->irq_count; i++) { + if (!irqsteer_data->irq[i]) + break; + irq_set_chained_handler_and_data(irqsteer_data->irq[i], NULL, NULL); + } irq_domain_remove(irqsteer_data->domain); clk_disable_unprepare(irqsteer_data->ipg_clk); - - return 0; } #ifdef CONFIG_PM @@ -307,11 +325,11 @@ static const struct of_device_id imx_irqsteer_dt_ids[] = { static struct platform_driver imx_irqsteer_driver = { .driver = { - .name = "imx-irqsteer", - .of_match_table = imx_irqsteer_dt_ids, - .pm = &imx_irqsteer_pm_ops, + .name = "imx-irqsteer", + .of_match_table = imx_irqsteer_dt_ids, + .pm = &imx_irqsteer_pm_ops, }, - .probe = imx_irqsteer_probe, - .remove = imx_irqsteer_remove, + .probe = imx_irqsteer_probe, + .remove = imx_irqsteer_remove, }; builtin_platform_driver(imx_irqsteer_driver); diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c index 90d41c1407ac..c598f2f52fc6 100644 --- a/drivers/irqchip/irq-imx-mu-msi.c +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -24,6 +24,8 @@ #include <linux/pm_domain.h> #include <linux/spinlock.h> +#include <linux/irqchip/irq-msi-lib.h> + #define IMX_MU_CHANS 4 enum imx_mu_xcr { @@ -114,20 +116,6 @@ static void imx_mu_msi_parent_ack_irq(struct irq_data *data) imx_mu_read(msi_data, msi_data->cfg->xRR + data->hwirq * 4); } -static struct irq_chip imx_mu_msi_irq_chip = { - .name = "MU-MSI", - .irq_ack = irq_chip_ack_parent, -}; - -static struct msi_domain_ops imx_mu_msi_irq_ops = { -}; - -static struct msi_domain_info imx_mu_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), - .ops = &imx_mu_msi_irq_ops, - .chip = &imx_mu_msi_irq_chip, -}; - static void imx_mu_msi_parent_compose_msg(struct irq_data *data, struct msi_msg *msg) { @@ -195,6 +183,7 @@ static void imx_mu_msi_domain_irq_free(struct irq_domain *domain, } static const struct irq_domain_ops imx_mu_msi_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = imx_mu_msi_domain_irq_alloc, .free = imx_mu_msi_domain_irq_free, }; @@ -216,35 +205,39 @@ static void imx_mu_msi_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } +#define IMX_MU_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PARENT_PM_DEV) + +#define IMX_MU_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops imx_mu_msi_parent_ops = { + .supported_flags = IMX_MU_MSI_FLAGS_SUPPORTED, + .required_flags = IMX_MU_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI | MSI_CHIP_FLAG_SET_ACK, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PLATFORM_MSI, + .prefix = "MU-MSI-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + static int imx_mu_msi_domains_init(struct imx_mu_msi *msi_data, struct device *dev) { - struct fwnode_handle *fwnodes = dev_fwnode(dev); + struct irq_domain_info info = { + .ops = &imx_mu_msi_domain_ops, + .fwnode = dev_fwnode(dev), + .size = IMX_MU_CHANS, + .host_data = msi_data, + }; struct irq_domain *parent; /* Initialize MSI domain parent */ - parent = irq_domain_create_linear(fwnodes, - IMX_MU_CHANS, - &imx_mu_msi_domain_ops, - msi_data); + parent = msi_create_parent_irq_domain(&info, &imx_mu_msi_parent_ops); if (!parent) { dev_err(dev, "failed to create IRQ domain\n"); return -ENOMEM; } - - irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); - - msi_data->msi_domain = platform_msi_create_irq_domain(fwnodes, - &imx_mu_msi_domain_info, - parent); - - if (!msi_data->msi_domain) { - dev_err(dev, "failed to create MSI domain\n"); - irq_domain_remove(parent); - return -ENOMEM; - } - - irq_domain_set_pm_device(msi_data->msi_domain, dev); - + parent->dev = parent->pm_dev = dev; return 0; } @@ -303,11 +296,9 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { }, }; -static int __init imx_mu_of_init(struct device_node *dn, - struct device_node *parent, - const struct imx_mu_dcfg *cfg) +static int imx_mu_probe(struct platform_device *pdev, struct device_node *parent, + const struct imx_mu_dcfg *cfg) { - struct platform_device *pdev = of_find_device_by_node(dn); struct device_link *pd_link_a; struct device_link *pd_link_b; struct imx_mu_msi *msi_data; @@ -423,31 +414,27 @@ static const struct dev_pm_ops imx_mu_pm_ops = { imx_mu_runtime_resume, NULL) }; -static int __init imx_mu_imx7ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx7ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx7ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx7ulp); } -static int __init imx_mu_imx6sx_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx6sx_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx6sx); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx6sx); } -static int __init imx_mu_imx8ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx8ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx8ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx8ulp); } IRQCHIP_PLATFORM_DRIVER_BEGIN(imx_mu_msi) -IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_of_init) -IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_of_init) -IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_of_init) +IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_probe) +IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_probe) +IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_probe) IRQCHIP_PLATFORM_DRIVER_END(imx_mu_msi, .pm = &imx_mu_pm_ops) - MODULE_AUTHOR("Frank Li <Frank.Li@nxp.com>"); MODULE_DESCRIPTION("Freescale MU MSI controller driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-ingenic-tcu.c b/drivers/irqchip/irq-ingenic-tcu.c index 3363f83bd7e9..794ecba717c9 100644 --- a/drivers/irqchip/irq-ingenic-tcu.c +++ b/drivers/irqchip/irq-ingenic-tcu.c @@ -52,11 +52,10 @@ static void ingenic_tcu_gc_unmask_enable_reg(struct irq_data *d) struct regmap *map = gc->private; u32 mask = d->mask; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); regmap_write(map, ct->regs.ack, mask); regmap_write(map, ct->regs.enable, mask); *ct->mask_cache |= mask; - irq_gc_unlock(gc); } static void ingenic_tcu_gc_mask_disable_reg(struct irq_data *d) @@ -66,10 +65,9 @@ static void ingenic_tcu_gc_mask_disable_reg(struct irq_data *d) struct regmap *map = gc->private; u32 mask = d->mask; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); regmap_write(map, ct->regs.disable, mask); *ct->mask_cache &= ~mask; - irq_gc_unlock(gc); } static void ingenic_tcu_gc_mask_disable_reg_and_ack(struct irq_data *d) @@ -79,10 +77,9 @@ static void ingenic_tcu_gc_mask_disable_reg_and_ack(struct irq_data *d) struct regmap *map = gc->private; u32 mask = d->mask; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); regmap_write(map, ct->regs.ack, mask); regmap_write(map, ct->regs.disable, mask); - irq_gc_unlock(gc); } static int __init ingenic_tcu_irq_init(struct device_node *np, @@ -114,8 +111,8 @@ static int __init ingenic_tcu_irq_init(struct device_node *np, tcu->nb_parent_irqs = irqs; - tcu->domain = irq_domain_add_linear(np, 32, &irq_generic_chip_ops, - NULL); + tcu->domain = irq_domain_create_linear(of_fwnode_handle(np), 32, &irq_generic_chip_ops, + NULL); if (!tcu->domain) { ret = -ENOMEM; goto err_free_tcu; diff --git a/drivers/irqchip/irq-ingenic.c b/drivers/irqchip/irq-ingenic.c index cee839ca627e..52393724f213 100644 --- a/drivers/irqchip/irq-ingenic.c +++ b/drivers/irqchip/irq-ingenic.c @@ -90,8 +90,8 @@ static int __init ingenic_intc_of_init(struct device_node *node, goto out_unmap_irq; } - domain = irq_domain_add_linear(node, num_chips * 32, - &irq_generic_chip_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(node), num_chips * 32, + &irq_generic_chip_ops, NULL); if (!domain) { err = -ENOMEM; goto out_unmap_base; diff --git a/drivers/irqchip/irq-ixp4xx.c b/drivers/irqchip/irq-ixp4xx.c index 5fba907b9052..a9a5a52b818a 100644 --- a/drivers/irqchip/irq-ixp4xx.c +++ b/drivers/irqchip/irq-ixp4xx.c @@ -105,8 +105,7 @@ static void ixp4xx_irq_unmask(struct irq_data *d) } } -static asmlinkage void __exception_irq_entry -ixp4xx_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry ixp4xx_handle_irq(struct pt_regs *regs) { struct ixp4xx_irq *ixi = &ixirq; unsigned long status; @@ -262,7 +261,7 @@ static int __init ixp4xx_of_init_irq(struct device_node *np, pr_crit("IXP4XX: could not ioremap interrupt controller\n"); return -ENODEV; } - fwnode = of_node_to_fwnode(np); + fwnode = of_fwnode_handle(np); /* These chip variants have 64 interrupts */ is_356 = of_device_is_compatible(np, "intel,ixp43x-interrupt") || diff --git a/drivers/irqchip/irq-jcore-aic.c b/drivers/irqchip/irq-jcore-aic.c index b9dcc8e78c75..94c05cf974be 100644 --- a/drivers/irqchip/irq-jcore-aic.c +++ b/drivers/irqchip/irq-jcore-aic.c @@ -38,7 +38,7 @@ static struct irq_chip jcore_aic; static void handle_jcore_irq(struct irq_desc *desc) { if (irqd_is_per_cpu(irq_desc_get_irq_data(desc))) - handle_percpu_irq(desc); + handle_percpu_devid_irq(desc); else handle_simple_irq(desc); } @@ -107,9 +107,8 @@ static int __init aic_irq_of_init(struct device_node *node, if (ret < 0) return ret; - domain = irq_domain_add_legacy(node, dom_sz - min_irq, min_irq, min_irq, - &jcore_aic_irqdomain_ops, - &jcore_aic); + domain = irq_domain_create_legacy(of_fwnode_handle(node), dom_sz - min_irq, min_irq, + min_irq, &jcore_aic_irqdomain_ops, &jcore_aic); if (!domain) return -ENOMEM; diff --git a/drivers/irqchip/irq-keystone.c b/drivers/irqchip/irq-keystone.c index a36396db4b08..922fff09354f 100644 --- a/drivers/irqchip/irq-keystone.c +++ b/drivers/irqchip/irq-keystone.c @@ -141,18 +141,11 @@ static int keystone_irq_probe(struct platform_device *pdev) if (!kirq) return -ENOMEM; - kirq->devctrl_regs = - syscon_regmap_lookup_by_phandle(np, "ti,syscon-dev"); + kirq->devctrl_regs = syscon_regmap_lookup_by_phandle_args(np, "ti,syscon-dev", + 1, &kirq->devctrl_offset); if (IS_ERR(kirq->devctrl_regs)) return PTR_ERR(kirq->devctrl_regs); - ret = of_property_read_u32_index(np, "ti,syscon-dev", 1, - &kirq->devctrl_offset); - if (ret) { - dev_err(dev, "couldn't read the devctrl_offset offset!\n"); - return ret; - } - kirq->irq = platform_get_irq(pdev, 0); if (kirq->irq < 0) return kirq->irq; @@ -164,8 +157,8 @@ static int keystone_irq_probe(struct platform_device *pdev) kirq->chip.irq_mask = keystone_irq_setmask; kirq->chip.irq_unmask = keystone_irq_unmask; - kirq->irqd = irq_domain_add_linear(np, KEYSTONE_N_IRQ, - &keystone_irq_ops, kirq); + kirq->irqd = irq_domain_create_linear(dev_fwnode(dev), KEYSTONE_N_IRQ, &keystone_irq_ops, + kirq); if (!kirq->irqd) { dev_err(dev, "IRQ domain registration failed\n"); return -ENODEV; @@ -190,7 +183,7 @@ static int keystone_irq_probe(struct platform_device *pdev) return 0; } -static int keystone_irq_remove(struct platform_device *pdev) +static void keystone_irq_remove(struct platform_device *pdev) { struct keystone_irq_device *kirq = platform_get_drvdata(pdev); int hwirq; @@ -201,7 +194,6 @@ static int keystone_irq_remove(struct platform_device *pdev) irq_dispose_mapping(irq_find_mapping(kirq->irqd, hwirq)); irq_domain_remove(kirq->irqd); - return 0; } static const struct of_device_id keystone_irq_dt_ids[] = { diff --git a/drivers/irqchip/irq-lan966x-oic.c b/drivers/irqchip/irq-lan966x-oic.c new file mode 100644 index 000000000000..11d3a0ffa261 --- /dev/null +++ b/drivers/irqchip/irq-lan966x-oic.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the Microchip LAN966x outbound interrupt controller + * + * Copyright (c) 2024 Technology Inc. and its subsidiaries. + * + * Authors: + * Horatiu Vultur <horatiu.vultur@microchip.com> + * Clément Léger <clement.leger@bootlin.com> + * Herve Codina <herve.codina@bootlin.com> + */ + +#include <linux/interrupt.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip.h> +#include <linux/irq.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +struct lan966x_oic_chip_regs { + int reg_off_ena_set; + int reg_off_ena_clr; + int reg_off_sticky; + int reg_off_ident; + int reg_off_map; +}; + +struct lan966x_oic_data { + void __iomem *regs; + int irq; +}; + +#define LAN966X_OIC_NR_IRQ 86 + +/* Interrupt sticky status */ +#define LAN966X_OIC_INTR_STICKY 0x30 +#define LAN966X_OIC_INTR_STICKY1 0x34 +#define LAN966X_OIC_INTR_STICKY2 0x38 + +/* Interrupt enable */ +#define LAN966X_OIC_INTR_ENA 0x48 +#define LAN966X_OIC_INTR_ENA1 0x4c +#define LAN966X_OIC_INTR_ENA2 0x50 + +/* Atomic clear of interrupt enable */ +#define LAN966X_OIC_INTR_ENA_CLR 0x54 +#define LAN966X_OIC_INTR_ENA_CLR1 0x58 +#define LAN966X_OIC_INTR_ENA_CLR2 0x5c + +/* Atomic set of interrupt */ +#define LAN966X_OIC_INTR_ENA_SET 0x60 +#define LAN966X_OIC_INTR_ENA_SET1 0x64 +#define LAN966X_OIC_INTR_ENA_SET2 0x68 + +/* Mapping of source to destination interrupts (_n = 0..8) */ +#define LAN966X_OIC_DST_INTR_MAP(_n) (0x78 + (_n) * 4) +#define LAN966X_OIC_DST_INTR_MAP1(_n) (0x9c + (_n) * 4) +#define LAN966X_OIC_DST_INTR_MAP2(_n) (0xc0 + (_n) * 4) + +/* Currently active interrupt sources per destination (_n = 0..8) */ +#define LAN966X_OIC_DST_INTR_IDENT(_n) (0xe4 + (_n) * 4) +#define LAN966X_OIC_DST_INTR_IDENT1(_n) (0x108 + (_n) * 4) +#define LAN966X_OIC_DST_INTR_IDENT2(_n) (0x12c + (_n) * 4) + +static unsigned int lan966x_oic_irq_startup(struct irq_data *data) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); + struct irq_chip_type *ct = irq_data_get_chip_type(data); + struct lan966x_oic_chip_regs *chip_regs = gc->private; + u32 map; + + scoped_guard (raw_spinlock, &gc->lock) { + /* Map the source interrupt to the destination */ + map = irq_reg_readl(gc, chip_regs->reg_off_map); + map |= data->mask; + irq_reg_writel(gc, map, chip_regs->reg_off_map); + } + + ct->chip.irq_ack(data); + ct->chip.irq_unmask(data); + + return 0; +} + +static void lan966x_oic_irq_shutdown(struct irq_data *data) +{ + struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); + struct irq_chip_type *ct = irq_data_get_chip_type(data); + struct lan966x_oic_chip_regs *chip_regs = gc->private; + u32 map; + + ct->chip.irq_mask(data); + + guard(raw_spinlock)(&gc->lock); + + /* Unmap the interrupt */ + map = irq_reg_readl(gc, chip_regs->reg_off_map); + map &= ~data->mask; + irq_reg_writel(gc, map, chip_regs->reg_off_map); +} + +static int lan966x_oic_irq_set_type(struct irq_data *data, + unsigned int flow_type) +{ + if (flow_type != IRQ_TYPE_LEVEL_HIGH) { + pr_err("lan966x oic doesn't support flow type %d\n", flow_type); + return -EINVAL; + } + + return 0; +} + +static void lan966x_oic_irq_handler_domain(struct irq_domain *d, u32 first_irq) +{ + struct irq_chip_generic *gc = irq_get_domain_generic_chip(d, first_irq); + struct lan966x_oic_chip_regs *chip_regs = gc->private; + unsigned long ident; + unsigned int hwirq; + + ident = irq_reg_readl(gc, chip_regs->reg_off_ident); + if (!ident) + return; + + for_each_set_bit(hwirq, &ident, 32) + generic_handle_domain_irq(d, hwirq + first_irq); +} + +static void lan966x_oic_irq_handler(struct irq_desc *desc) +{ + struct irq_domain *d = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + + chained_irq_enter(chip, desc); + lan966x_oic_irq_handler_domain(d, 0); + lan966x_oic_irq_handler_domain(d, 32); + lan966x_oic_irq_handler_domain(d, 64); + chained_irq_exit(chip, desc); +} + +static struct lan966x_oic_chip_regs lan966x_oic_chip_regs[3] = { + { + .reg_off_ena_set = LAN966X_OIC_INTR_ENA_SET, + .reg_off_ena_clr = LAN966X_OIC_INTR_ENA_CLR, + .reg_off_sticky = LAN966X_OIC_INTR_STICKY, + .reg_off_ident = LAN966X_OIC_DST_INTR_IDENT(0), + .reg_off_map = LAN966X_OIC_DST_INTR_MAP(0), + }, { + .reg_off_ena_set = LAN966X_OIC_INTR_ENA_SET1, + .reg_off_ena_clr = LAN966X_OIC_INTR_ENA_CLR1, + .reg_off_sticky = LAN966X_OIC_INTR_STICKY1, + .reg_off_ident = LAN966X_OIC_DST_INTR_IDENT1(0), + .reg_off_map = LAN966X_OIC_DST_INTR_MAP1(0), + }, { + .reg_off_ena_set = LAN966X_OIC_INTR_ENA_SET2, + .reg_off_ena_clr = LAN966X_OIC_INTR_ENA_CLR2, + .reg_off_sticky = LAN966X_OIC_INTR_STICKY2, + .reg_off_ident = LAN966X_OIC_DST_INTR_IDENT2(0), + .reg_off_map = LAN966X_OIC_DST_INTR_MAP2(0), + } +}; + +static int lan966x_oic_chip_init(struct irq_chip_generic *gc) +{ + struct lan966x_oic_data *lan966x_oic = gc->domain->host_data; + struct lan966x_oic_chip_regs *chip_regs; + + chip_regs = &lan966x_oic_chip_regs[gc->irq_base / 32]; + + gc->reg_base = lan966x_oic->regs; + gc->chip_types[0].regs.enable = chip_regs->reg_off_ena_set; + gc->chip_types[0].regs.disable = chip_regs->reg_off_ena_clr; + gc->chip_types[0].regs.ack = chip_regs->reg_off_sticky; + gc->chip_types[0].chip.irq_startup = lan966x_oic_irq_startup; + gc->chip_types[0].chip.irq_shutdown = lan966x_oic_irq_shutdown; + gc->chip_types[0].chip.irq_set_type = lan966x_oic_irq_set_type; + gc->chip_types[0].chip.irq_mask = irq_gc_mask_disable_reg; + gc->chip_types[0].chip.irq_unmask = irq_gc_unmask_enable_reg; + gc->chip_types[0].chip.irq_ack = irq_gc_ack_set_bit; + gc->private = chip_regs; + + /* Disable all interrupts handled by this chip */ + irq_reg_writel(gc, ~0U, chip_regs->reg_off_ena_clr); + + return 0; +} + +static void lan966x_oic_chip_exit(struct irq_chip_generic *gc) +{ + /* Disable and ack all interrupts handled by this chip */ + irq_reg_writel(gc, ~0U, gc->chip_types[0].regs.disable); + irq_reg_writel(gc, ~0U, gc->chip_types[0].regs.ack); +} + +static int lan966x_oic_domain_init(struct irq_domain *d) +{ + struct lan966x_oic_data *lan966x_oic = d->host_data; + + irq_set_chained_handler_and_data(lan966x_oic->irq, lan966x_oic_irq_handler, d); + + return 0; +} + +static void lan966x_oic_domain_exit(struct irq_domain *d) +{ + struct lan966x_oic_data *lan966x_oic = d->host_data; + + irq_set_chained_handler_and_data(lan966x_oic->irq, NULL, NULL); +} + +static int lan966x_oic_probe(struct platform_device *pdev) +{ + struct irq_domain_chip_generic_info dgc_info = { + .name = "lan966x-oic", + .handler = handle_level_irq, + .irqs_per_chip = 32, + .num_ct = 1, + .init = lan966x_oic_chip_init, + .exit = lan966x_oic_chip_exit, + }; + struct irq_domain_info d_info = { + .fwnode = of_fwnode_handle(pdev->dev.of_node), + .domain_flags = IRQ_DOMAIN_FLAG_DESTROY_GC, + .size = LAN966X_OIC_NR_IRQ, + .hwirq_max = LAN966X_OIC_NR_IRQ, + .ops = &irq_generic_chip_ops, + .dgc_info = &dgc_info, + .init = lan966x_oic_domain_init, + .exit = lan966x_oic_domain_exit, + }; + struct lan966x_oic_data *lan966x_oic; + struct device *dev = &pdev->dev; + struct irq_domain *domain; + + lan966x_oic = devm_kmalloc(dev, sizeof(*lan966x_oic), GFP_KERNEL); + if (!lan966x_oic) + return -ENOMEM; + + lan966x_oic->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(lan966x_oic->regs)) + return dev_err_probe(dev, PTR_ERR(lan966x_oic->regs), + "failed to map resource\n"); + + lan966x_oic->irq = platform_get_irq(pdev, 0); + if (lan966x_oic->irq < 0) + return dev_err_probe(dev, lan966x_oic->irq, "failed to get the IRQ\n"); + + d_info.host_data = lan966x_oic; + domain = devm_irq_domain_instantiate(dev, &d_info); + if (IS_ERR(domain)) + return dev_err_probe(dev, PTR_ERR(domain), + "failed to instantiate the IRQ domain\n"); + return 0; +} + +static const struct of_device_id lan966x_oic_of_match[] = { + { .compatible = "microchip,lan966x-oic" }, + {} /* sentinel */ +}; +MODULE_DEVICE_TABLE(of, lan966x_oic_of_match); + +static struct platform_driver lan966x_oic_driver = { + .probe = lan966x_oic_probe, + .driver = { + .name = "lan966x-oic", + .of_match_table = lan966x_oic_of_match, + }, +}; +module_platform_driver(lan966x_oic_driver); + +MODULE_AUTHOR("Herve Codina <herve.codina@bootlin.com>"); +MODULE_DESCRIPTION("Microchip LAN966x OIC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-loongarch-avec.c b/drivers/irqchip/irq-loongarch-avec.c new file mode 100644 index 000000000000..bf52dc8345f5 --- /dev/null +++ b/drivers/irqchip/irq-loongarch-avec.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2020-2024 Loongson Technologies, Inc. + */ + +#include <linux/cpuhotplug.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/msi.h> +#include <linux/radix-tree.h> +#include <linux/spinlock.h> + +#include <asm/loongarch.h> +#include <asm/setup.h> + +#include <linux/irqchip/irq-msi-lib.h> +#include "irq-loongson.h" + +#define VECTORS_PER_REG 64 +#define IRR_VECTOR_MASK 0xffUL +#define IRR_INVALID_MASK 0x80000000UL +#define AVEC_MSG_OFFSET 0x100000 + +#ifdef CONFIG_SMP +struct pending_list { + struct list_head head; +}; + +static struct cpumask intersect_mask; +static DEFINE_PER_CPU(struct pending_list, pending_list); +#endif + +static DEFINE_PER_CPU(struct irq_desc * [NR_VECTORS], irq_map); + +struct avecintc_chip { + raw_spinlock_t lock; + struct fwnode_handle *fwnode; + struct irq_domain *domain; + struct irq_matrix *vector_matrix; + phys_addr_t msi_base_addr; +}; + +static struct avecintc_chip loongarch_avec; + +struct avecintc_data { + struct list_head entry; + unsigned int cpu; + unsigned int vec; + unsigned int prev_cpu; + unsigned int prev_vec; + unsigned int moving; +}; + +static inline void avecintc_enable(void) +{ + u64 value; + + value = iocsr_read64(LOONGARCH_IOCSR_MISC_FUNC); + value |= IOCSR_MISC_FUNC_AVEC_EN; + iocsr_write64(value, LOONGARCH_IOCSR_MISC_FUNC); +} + +static inline void avecintc_ack_irq(struct irq_data *d) +{ +} + +static inline void avecintc_mask_irq(struct irq_data *d) +{ +} + +static inline void avecintc_unmask_irq(struct irq_data *d) +{ +} + +#ifdef CONFIG_SMP +static inline void pending_list_init(int cpu) +{ + struct pending_list *plist = per_cpu_ptr(&pending_list, cpu); + + INIT_LIST_HEAD(&plist->head); +} + +static void avecintc_sync(struct avecintc_data *adata) +{ + struct pending_list *plist; + + if (cpu_online(adata->prev_cpu)) { + plist = per_cpu_ptr(&pending_list, adata->prev_cpu); + list_add_tail(&adata->entry, &plist->head); + adata->moving = 1; + mp_ops.send_ipi_single(adata->prev_cpu, ACTION_CLEAR_VECTOR); + } +} + +static int avecintc_set_affinity(struct irq_data *data, const struct cpumask *dest, bool force) +{ + int cpu, ret, vector; + struct avecintc_data *adata; + + scoped_guard(raw_spinlock, &loongarch_avec.lock) { + adata = irq_data_get_irq_chip_data(data); + + if (adata->moving) + return -EBUSY; + + if (cpu_online(adata->cpu) && cpumask_test_cpu(adata->cpu, dest)) + return 0; + + cpumask_and(&intersect_mask, dest, cpu_online_mask); + + ret = irq_matrix_alloc(loongarch_avec.vector_matrix, &intersect_mask, false, &cpu); + if (ret < 0) + return ret; + + vector = ret; + adata->cpu = cpu; + adata->vec = vector; + per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(data); + avecintc_sync(adata); + } + + irq_data_update_effective_affinity(data, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK; +} + +static int avecintc_cpu_online(unsigned int cpu) +{ + if (!loongarch_avec.vector_matrix) + return 0; + + guard(raw_spinlock)(&loongarch_avec.lock); + + avecintc_enable(); + + irq_matrix_online(loongarch_avec.vector_matrix); + + pending_list_init(cpu); + + return 0; +} + +static int avecintc_cpu_offline(unsigned int cpu) +{ + struct pending_list *plist = per_cpu_ptr(&pending_list, cpu); + + if (!loongarch_avec.vector_matrix) + return 0; + + guard(raw_spinlock)(&loongarch_avec.lock); + + if (!list_empty(&plist->head)) + pr_warn("CPU#%d vector is busy\n", cpu); + + irq_matrix_offline(loongarch_avec.vector_matrix); + + return 0; +} + +void complete_irq_moving(void) +{ + struct pending_list *plist = this_cpu_ptr(&pending_list); + struct avecintc_data *adata, *tdata; + int cpu, vector, bias; + uint64_t isr; + + guard(raw_spinlock)(&loongarch_avec.lock); + + list_for_each_entry_safe(adata, tdata, &plist->head, entry) { + cpu = adata->prev_cpu; + vector = adata->prev_vec; + bias = vector / VECTORS_PER_REG; + switch (bias) { + case 0: + isr = csr_read64(LOONGARCH_CSR_ISR0); + break; + case 1: + isr = csr_read64(LOONGARCH_CSR_ISR1); + break; + case 2: + isr = csr_read64(LOONGARCH_CSR_ISR2); + break; + case 3: + isr = csr_read64(LOONGARCH_CSR_ISR3); + break; + } + + if (isr & (1UL << (vector % VECTORS_PER_REG))) { + mp_ops.send_ipi_single(cpu, ACTION_CLEAR_VECTOR); + continue; + } + list_del(&adata->entry); + irq_matrix_free(loongarch_avec.vector_matrix, cpu, vector, false); + this_cpu_write(irq_map[vector], NULL); + adata->moving = 0; + adata->prev_cpu = adata->cpu; + adata->prev_vec = adata->vec; + } +} +#endif + +static void avecintc_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct avecintc_data *adata = irq_data_get_irq_chip_data(d); + + msg->address_hi = 0x0; + msg->address_lo = (loongarch_avec.msi_base_addr | (adata->vec & 0xff) << 4) + | ((cpu_logical_map(adata->cpu & 0xffff)) << 12); + msg->data = 0x0; +} + +static struct irq_chip avec_irq_controller = { + .name = "AVECINTC", + .irq_ack = avecintc_ack_irq, + .irq_mask = avecintc_mask_irq, + .irq_unmask = avecintc_unmask_irq, +#ifdef CONFIG_SMP + .irq_set_affinity = avecintc_set_affinity, +#endif + .irq_compose_msi_msg = avecintc_compose_msi_msg, +}; + +static void avecintc_irq_dispatch(struct irq_desc *desc) +{ + struct irq_chip *chip = irq_desc_get_chip(desc); + struct irq_desc *d; + + chained_irq_enter(chip, desc); + + while (true) { + unsigned long vector = csr_read64(LOONGARCH_CSR_IRR); + if (vector & IRR_INVALID_MASK) + break; + + vector &= IRR_VECTOR_MASK; + + d = this_cpu_read(irq_map[vector]); + if (d) { + generic_handle_irq_desc(d); + } else { + spurious_interrupt(); + pr_warn("Unexpected IRQ occurs on CPU#%d [vector %ld]\n", smp_processor_id(), vector); + } + } + + chained_irq_exit(chip, desc); +} + +static int avecintc_alloc_vector(struct irq_data *irqd, struct avecintc_data *adata) +{ + int cpu, ret; + + guard(raw_spinlock_irqsave)(&loongarch_avec.lock); + + ret = irq_matrix_alloc(loongarch_avec.vector_matrix, cpu_online_mask, false, &cpu); + if (ret < 0) + return ret; + + adata->prev_cpu = adata->cpu = cpu; + adata->prev_vec = adata->vec = ret; + per_cpu_ptr(irq_map, adata->cpu)[adata->vec] = irq_data_to_desc(irqd); + + return 0; +} + +static int avecintc_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + for (unsigned int i = 0; i < nr_irqs; i++) { + struct irq_data *irqd = irq_domain_get_irq_data(domain, virq + i); + struct avecintc_data *adata = kzalloc(sizeof(*adata), GFP_KERNEL); + int ret; + + if (!adata) + return -ENOMEM; + + ret = avecintc_alloc_vector(irqd, adata); + if (ret < 0) { + kfree(adata); + return ret; + } + + irq_domain_set_info(domain, virq + i, virq + i, &avec_irq_controller, + adata, handle_edge_irq, NULL, NULL); + irqd_set_single_target(irqd); + irqd_set_affinity_on_activate(irqd); + } + + return 0; +} + +static void avecintc_free_vector(struct irq_data *irqd, struct avecintc_data *adata) +{ + guard(raw_spinlock_irqsave)(&loongarch_avec.lock); + + per_cpu(irq_map, adata->cpu)[adata->vec] = NULL; + irq_matrix_free(loongarch_avec.vector_matrix, adata->cpu, adata->vec, false); + +#ifdef CONFIG_SMP + if (!adata->moving) + return; + + per_cpu(irq_map, adata->prev_cpu)[adata->prev_vec] = NULL; + irq_matrix_free(loongarch_avec.vector_matrix, adata->prev_cpu, adata->prev_vec, false); + list_del_init(&adata->entry); +#endif +} + +static void avecintc_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + for (unsigned int i = 0; i < nr_irqs; i++) { + struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); + + if (d) { + struct avecintc_data *adata = irq_data_get_irq_chip_data(d); + + avecintc_free_vector(d, adata); + irq_domain_reset_irq_data(d); + kfree(adata); + } + } +} + +static const struct irq_domain_ops avecintc_domain_ops = { + .alloc = avecintc_domain_alloc, + .free = avecintc_domain_free, + .select = msi_lib_irq_domain_select, +}; + +static int __init irq_matrix_init(void) +{ + loongarch_avec.vector_matrix = irq_alloc_matrix(NR_VECTORS, 0, NR_VECTORS); + if (!loongarch_avec.vector_matrix) + return -ENOMEM; + + for (int i = 0; i < NR_LEGACY_VECTORS; i++) + irq_matrix_assign_system(loongarch_avec.vector_matrix, i, false); + + irq_matrix_online(loongarch_avec.vector_matrix); + + return 0; +} + +static int __init avecintc_init(struct irq_domain *parent) +{ + int ret, parent_irq; + + raw_spin_lock_init(&loongarch_avec.lock); + + loongarch_avec.fwnode = irq_domain_alloc_named_fwnode("AVECINTC"); + if (!loongarch_avec.fwnode) { + pr_err("Unable to allocate domain handle\n"); + ret = -ENOMEM; + goto out; + } + + loongarch_avec.domain = irq_domain_create_tree(loongarch_avec.fwnode, + &avecintc_domain_ops, NULL); + if (!loongarch_avec.domain) { + pr_err("Unable to create IRQ domain\n"); + ret = -ENOMEM; + goto out_free_handle; + } + + parent_irq = irq_create_mapping(parent, INT_AVEC); + if (!parent_irq) { + pr_err("Failed to mapping hwirq\n"); + ret = -EINVAL; + goto out_remove_domain; + } + + ret = irq_matrix_init(); + if (ret < 0) { + pr_err("Failed to init irq matrix\n"); + goto out_remove_domain; + } + irq_set_chained_handler_and_data(parent_irq, avecintc_irq_dispatch, NULL); + +#ifdef CONFIG_SMP + pending_list_init(0); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_AVECINTC_STARTING, + "irqchip/loongarch/avecintc:starting", + avecintc_cpu_online, avecintc_cpu_offline); +#endif + avecintc_enable(); + + return ret; + +out_remove_domain: + irq_domain_remove(loongarch_avec.domain); +out_free_handle: + irq_domain_free_fwnode(loongarch_avec.fwnode); +out: + return ret; +} + +static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_msi_pic *pchmsi_entry = (struct acpi_madt_msi_pic *)header; + + loongarch_avec.msi_base_addr = pchmsi_entry->msg_address - AVEC_MSG_OFFSET; + + return pch_msi_acpi_init_avec(loongarch_avec.domain); +} + +static inline int __init acpi_cascade_irqdomain_init(void) +{ + return acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1); +} + +int __init avecintc_acpi_init(struct irq_domain *parent) +{ + int ret = avecintc_init(parent); + if (ret < 0) { + pr_err("Failed to init IRQ domain\n"); + return ret; + } + + ret = acpi_cascade_irqdomain_init(); + if (ret < 0) { + pr_err("Failed to init cascade IRQ domain\n"); + return ret; + } + + return ret; +} diff --git a/drivers/irqchip/irq-loongarch-cpu.c b/drivers/irqchip/irq-loongarch-cpu.c index 9d8f2c406043..950bc087e388 100644 --- a/drivers/irqchip/irq-loongarch-cpu.c +++ b/drivers/irqchip/irq-loongarch-cpu.c @@ -13,16 +13,20 @@ #include <asm/loongarch.h> #include <asm/setup.h> +#include "irq-loongson.h" + static struct irq_domain *irq_domain; struct fwnode_handle *cpuintc_handle; static u32 lpic_gsi_to_irq(u32 gsi) { + int irq = 0; + /* Only pch irqdomain transferring is required for LoongArch. */ if (gsi >= GSI_MIN_PCH_IRQ && gsi <= GSI_MAX_PCH_IRQ) - return acpi_register_gsi(NULL, gsi, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); + irq = acpi_register_gsi(NULL, gsi, ACPI_LEVEL_SENSITIVE, ACPI_ACTIVE_HIGH); - return 0; + return (irq > 0) ? irq : 0; } static struct fwnode_handle *lpic_get_gsi_domain_id(u32 gsi) @@ -96,7 +100,7 @@ static const struct irq_domain_ops loongarch_cpu_intc_irq_domain_ops = { static int __init cpuintc_of_init(struct device_node *of_node, struct device_node *parent) { - cpuintc_handle = of_node_to_fwnode(of_node); + cpuintc_handle = of_fwnode_handle(of_node); irq_domain = irq_domain_create_linear(cpuintc_handle, EXCCODE_INT_NUM, &loongarch_cpu_intc_irq_domain_ops, NULL); @@ -138,7 +142,10 @@ static int __init acpi_cascade_irqdomain_init(void) if (r < 0) return r; - return 0; + if (cpu_has_avecint) + r = avecintc_acpi_init(irq_domain); + + return r; } static int __init cpuintc_acpi_init(union acpi_subtable_headers *header, diff --git a/drivers/irqchip/irq-loongson-eiointc.c b/drivers/irqchip/irq-loongson-eiointc.c index 1623cd779175..ad2105685b48 100644 --- a/drivers/irqchip/irq-loongson-eiointc.c +++ b/drivers/irqchip/irq-loongson-eiointc.c @@ -14,7 +14,11 @@ #include <linux/irqdomain.h> #include <linux/irqchip/chained_irq.h> #include <linux/kernel.h> +#include <linux/kvm_para.h> #include <linux/syscore_ops.h> +#include <asm/numa.h> + +#include "irq-loongson.h" #define EIOINTC_REG_NODEMAP 0x14a0 #define EIOINTC_REG_IPMAP 0x14c0 @@ -23,16 +27,47 @@ #define EIOINTC_REG_ISR 0x1800 #define EIOINTC_REG_ROUTE 0x1c00 +#define EXTIOI_VIRT_FEATURES 0x40000000 +#define EXTIOI_HAS_VIRT_EXTENSION BIT(0) +#define EXTIOI_HAS_ENABLE_OPTION BIT(1) +#define EXTIOI_HAS_INT_ENCODE BIT(2) +#define EXTIOI_HAS_CPU_ENCODE BIT(3) +#define EXTIOI_VIRT_CONFIG 0x40000004 +#define EXTIOI_ENABLE BIT(1) +#define EXTIOI_ENABLE_INT_ENCODE BIT(2) +#define EXTIOI_ENABLE_CPU_ENCODE BIT(3) + #define VEC_REG_COUNT 4 #define VEC_COUNT_PER_REG 64 #define VEC_COUNT (VEC_REG_COUNT * VEC_COUNT_PER_REG) #define VEC_REG_IDX(irq_id) ((irq_id) / VEC_COUNT_PER_REG) #define VEC_REG_BIT(irq_id) ((irq_id) % VEC_COUNT_PER_REG) #define EIOINTC_ALL_ENABLE 0xffffffff +#define EIOINTC_ALL_ENABLE_VEC_MASK(vector) (EIOINTC_ALL_ENABLE & ~BIT(vector & 0x1f)) +#define EIOINTC_REG_ENABLE_VEC(vector) (EIOINTC_REG_ENABLE + ((vector >> 5) << 2)) +#define EIOINTC_USE_CPU_ENCODE BIT(0) +#define EIOINTC_ROUTE_MULT_IP BIT(1) #define MAX_EIO_NODES (NR_CPUS / CORES_PER_EIO_NODE) +/* + * Routing registers are 32bit, and there is 8-bit route setting for every + * interrupt vector. So one Route register contains four vectors routing + * information. + */ +#define EIOINTC_REG_ROUTE_VEC(vector) (EIOINTC_REG_ROUTE + (vector & ~0x03)) +#define EIOINTC_REG_ROUTE_VEC_SHIFT(vector) ((vector & 0x03) << 3) +#define EIOINTC_REG_ROUTE_VEC_MASK(vector) (0xff << EIOINTC_REG_ROUTE_VEC_SHIFT(vector)) + static int nr_pics; +struct eiointc_priv; + +struct eiointc_ip_route { + struct eiointc_priv *priv; + /* Offset Routed destination IP */ + int start; + int end; +}; struct eiointc_priv { u32 node; @@ -41,6 +76,9 @@ struct eiointc_priv { cpumask_t cpuspan_map; struct fwnode_handle *domain_handle; struct irq_domain *eiointc_domain; + int flags; + irq_hw_number_t parent_hwirq; + struct eiointc_ip_route route_info[VEC_REG_COUNT]; }; static struct eiointc_priv *eiointc_priv[MAX_IO_PICS]; @@ -56,9 +94,13 @@ static void eiointc_enable(void) static int cpu_to_eio_node(int cpu) { - return cpu_logical_map(cpu) / CORES_PER_EIO_NODE; + if (!kvm_para_has_feature(KVM_FEATURE_VIRT_EXTIOI)) + return cpu_logical_map(cpu) / CORES_PER_EIO_NODE; + else + return cpu_logical_map(cpu) / CORES_PER_VEIO_NODE; } +#ifdef CONFIG_SMP static void eiointc_set_irq_route(int pos, unsigned int cpu, unsigned int mnode, nodemask_t *node_map) { int i, node, cpu_node, route_node; @@ -85,6 +127,17 @@ static void eiointc_set_irq_route(int pos, unsigned int cpu, unsigned int mnode, } } +static void veiointc_set_irq_route(unsigned int vector, unsigned int cpu) +{ + unsigned long reg = EIOINTC_REG_ROUTE_VEC(vector); + unsigned int data; + + data = iocsr_read32(reg); + data &= ~EIOINTC_REG_ROUTE_VEC_MASK(vector); + data |= cpu_logical_map(cpu) << EIOINTC_REG_ROUTE_VEC_SHIFT(vector); + iocsr_write32(data, reg); +} + static DEFINE_RAW_SPINLOCK(affinity_lock); static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *affinity, bool force) @@ -92,33 +145,35 @@ static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *af unsigned int cpu; unsigned long flags; uint32_t vector, regaddr; - struct cpumask intersect_affinity; struct eiointc_priv *priv = d->domain->host_data; raw_spin_lock_irqsave(&affinity_lock, flags); - cpumask_and(&intersect_affinity, affinity, cpu_online_mask); - cpumask_and(&intersect_affinity, &intersect_affinity, &priv->cpuspan_map); - - if (cpumask_empty(&intersect_affinity)) { + cpu = cpumask_first_and_and(&priv->cpuspan_map, affinity, cpu_online_mask); + if (cpu >= nr_cpu_ids) { raw_spin_unlock_irqrestore(&affinity_lock, flags); return -EINVAL; } - cpu = cpumask_first(&intersect_affinity); vector = d->hwirq; - regaddr = EIOINTC_REG_ENABLE + ((vector >> 5) << 2); - - /* Mask target vector */ - csr_any_send(regaddr, EIOINTC_ALL_ENABLE & (~BIT(vector & 0x1F)), - 0x0, priv->node * CORES_PER_EIO_NODE); - - /* Set route for target vector */ - eiointc_set_irq_route(vector, cpu, priv->node, &priv->node_map); - - /* Unmask target vector */ - csr_any_send(regaddr, EIOINTC_ALL_ENABLE, - 0x0, priv->node * CORES_PER_EIO_NODE); + regaddr = EIOINTC_REG_ENABLE_VEC(vector); + + if (priv->flags & EIOINTC_USE_CPU_ENCODE) { + iocsr_write32(EIOINTC_ALL_ENABLE_VEC_MASK(vector), regaddr); + veiointc_set_irq_route(vector, cpu); + iocsr_write32(EIOINTC_ALL_ENABLE, regaddr); + } else { + /* Mask target vector */ + csr_any_send(regaddr, EIOINTC_ALL_ENABLE_VEC_MASK(vector), + 0x0, priv->node * CORES_PER_EIO_NODE); + + /* Set route for target vector */ + eiointc_set_irq_route(vector, cpu, priv->node, &priv->node_map); + + /* Unmask target vector */ + csr_any_send(regaddr, EIOINTC_ALL_ENABLE, + 0x0, priv->node * CORES_PER_EIO_NODE); + } irq_data_update_effective_affinity(d, cpumask_of(cpu)); @@ -126,6 +181,7 @@ static int eiointc_set_irq_affinity(struct irq_data *d, const struct cpumask *af return IRQ_SET_MASK_OK; } +#endif static int eiointc_index(int node) { @@ -141,17 +197,31 @@ static int eiointc_index(int node) static int eiointc_router_init(unsigned int cpu) { - int i, bit; - uint32_t data; - uint32_t node = cpu_to_eio_node(cpu); - int index = eiointc_index(node); + int i, bit, cores, index, node; + unsigned int data; + int hwirq, mask; + + node = cpu_to_eio_node(cpu); + index = eiointc_index(node); if (index < 0) { pr_err("Error: invalid nodemap!\n"); - return -1; + return -EINVAL; } - if ((cpu_logical_map(cpu) % CORES_PER_EIO_NODE) == 0) { + /* Enable cpu interrupt pin from eiointc */ + hwirq = eiointc_priv[index]->parent_hwirq; + mask = BIT(hwirq); + if (eiointc_priv[index]->flags & EIOINTC_ROUTE_MULT_IP) + mask |= BIT(hwirq + 1) | BIT(hwirq + 2) | BIT(hwirq + 3); + set_csr_ecfg(mask); + + if (!(eiointc_priv[index]->flags & EIOINTC_USE_CPU_ENCODE)) + cores = CORES_PER_EIO_NODE; + else + cores = CORES_PER_VEIO_NODE; + + if ((cpu_logical_map(cpu) % cores) == 0) { eiointc_enable(); for (i = 0; i < eiointc_priv[0]->vec_count / 32; i++) { @@ -160,14 +230,39 @@ static int eiointc_router_init(unsigned int cpu) } for (i = 0; i < eiointc_priv[0]->vec_count / 32 / 4; i++) { - bit = BIT(1 + index); /* Route to IP[1 + index] */ - data = bit | (bit << 8) | (bit << 16) | (bit << 24); + /* + * Route to interrupt pin, relative offset used here + * Offset 0 means routing to IP0 and so on + * + * If flags is set with EIOINTC_ROUTE_MULT_IP, + * every 64 vector routes to different consecutive + * IPs, otherwise all vector routes to the same IP + */ + if (eiointc_priv[index]->flags & EIOINTC_ROUTE_MULT_IP) { + /* The first 64 vectors route to hwirq */ + bit = BIT(hwirq++ - INT_HWI0); + data = bit | (bit << 8); + + /* The second 64 vectors route to hwirq + 1 */ + bit = BIT(hwirq++ - INT_HWI0); + data |= (bit << 16) | (bit << 24); + + /* + * Route to hwirq + 2/hwirq + 3 separately + * in next loop + */ + } else { + bit = BIT(hwirq - INT_HWI0); + data = bit | (bit << 8) | (bit << 16) | (bit << 24); + } iocsr_write32(data, EIOINTC_REG_IPMAP + i * 4); } for (i = 0; i < eiointc_priv[0]->vec_count / 4; i++) { /* Route to Node-0 Core-0 */ - if (index == 0) + if (eiointc_priv[index]->flags & EIOINTC_USE_CPU_ENCODE) + bit = cpu_logical_map(0); + else if (index == 0) bit = BIT(cpu_logical_map(0)); else bit = (eiointc_priv[index]->node << 4) | 1; @@ -188,22 +283,35 @@ static int eiointc_router_init(unsigned int cpu) static void eiointc_irq_dispatch(struct irq_desc *desc) { - int i; - u64 pending; - bool handled = false; + struct eiointc_ip_route *info = irq_desc_get_handler_data(desc); struct irq_chip *chip = irq_desc_get_chip(desc); - struct eiointc_priv *priv = irq_desc_get_handler_data(desc); + bool handled = false; + u64 pending; + int i; chained_irq_enter(chip, desc); - for (i = 0; i < eiointc_priv[0]->vec_count / VEC_COUNT_PER_REG; i++) { + /* + * If EIOINTC_ROUTE_MULT_IP is set, every 64 interrupt vectors in + * eiointc interrupt controller routes to different cpu interrupt pins + * + * Every cpu interrupt pin has its own irq handler, it is ok to + * read ISR for these 64 interrupt vectors rather than all vectors + */ + for (i = info->start; i < info->end; i++) { pending = iocsr_read64(EIOINTC_REG_ISR + (i << 3)); + + /* Skip handling if pending bitmap is zero */ + if (!pending) + continue; + + /* Clear the IRQs */ iocsr_write64(pending, EIOINTC_REG_ISR + (i << 3)); while (pending) { int bit = __ffs(pending); int irq = bit + VEC_COUNT_PER_REG * i; - generic_handle_domain_irq(priv->eiointc_domain, irq); + generic_handle_domain_irq(info->priv->eiointc_domain, irq); pending &= ~BIT(bit); handled = true; } @@ -232,7 +340,9 @@ static struct irq_chip eiointc_irq_chip = { .irq_ack = eiointc_ack_irq, .irq_mask = eiointc_mask_irq, .irq_unmask = eiointc_unmask_irq, +#ifdef CONFIG_SMP .irq_set_affinity = eiointc_set_irq_affinity, +#endif }; static int eiointc_domain_alloc(struct irq_domain *domain, unsigned int virq, @@ -241,7 +351,7 @@ static int eiointc_domain_alloc(struct irq_domain *domain, unsigned int virq, int ret; unsigned int i, type; unsigned long hwirq = 0; - struct eiointc *priv = domain->host_data; + struct eiointc_priv *priv = domain->host_data; ret = irq_domain_translate_onecell(domain, arg, &hwirq, &type); if (ret) @@ -297,37 +407,25 @@ static struct irq_domain *acpi_get_vec_parent(int node, struct acpi_vector_group return NULL; } -static int eiointc_suspend(void) +static int eiointc_suspend(void *data) { return 0; } -static void eiointc_resume(void) +static void eiointc_resume(void *data) { - int i, j; - struct irq_desc *desc; - struct irq_data *irq_data; - eiointc_router_init(0); - - for (i = 0; i < nr_pics; i++) { - for (j = 0; j < eiointc_priv[0]->vec_count; j++) { - desc = irq_resolve_mapping(eiointc_priv[i]->eiointc_domain, j); - if (desc && desc->handle_irq && desc->handle_irq != handle_bad_irq) { - raw_spin_lock(&desc->lock); - irq_data = irq_domain_get_irq_data(eiointc_priv[i]->eiointc_domain, irq_desc_get_irq(desc)); - eiointc_set_irq_affinity(irq_data, irq_data->common->affinity, 0); - raw_spin_unlock(&desc->lock); - } - } - } } -static struct syscore_ops eiointc_syscore_ops = { +static const struct syscore_ops eiointc_syscore_ops = { .suspend = eiointc_suspend, .resume = eiointc_resume, }; +static struct syscore eiointc_syscore = { + .ops = &eiointc_syscore_ops, +}; + static int __init pch_pic_parse_madt(union acpi_subtable_headers *header, const unsigned long end) { @@ -349,7 +447,7 @@ static int __init pch_msi_parse_madt(union acpi_subtable_headers *header, int node; if (cpu_has_flatmode) - node = cpu_to_node(eiointc_priv[nr_pics - 1]->node * CORES_PER_EIO_NODE); + node = early_cpu_to_node(eiointc_priv[nr_pics - 1]->node * CORES_PER_EIO_NODE); else node = eiointc_priv[nr_pics - 1]->node; @@ -369,6 +467,9 @@ static int __init acpi_cascade_irqdomain_init(void) if (r < 0) return r; + if (cpu_has_avecint) + return 0; + r = acpi_table_parse_madt(ACPI_MADT_TYPE_MSI_PIC, pch_msi_parse_madt, 1); if (r < 0) return r; @@ -379,7 +480,7 @@ static int __init acpi_cascade_irqdomain_init(void) static int __init eiointc_init(struct eiointc_priv *priv, int parent_irq, u64 node_map) { - int i; + int i, val; node_map = node_map ? node_map : -1ULL; for_each_possible_cpu(i) { @@ -399,14 +500,53 @@ static int __init eiointc_init(struct eiointc_priv *priv, int parent_irq, return -ENOMEM; } + if (kvm_para_has_feature(KVM_FEATURE_VIRT_EXTIOI)) { + val = iocsr_read32(EXTIOI_VIRT_FEATURES); + /* + * With EXTIOI_ENABLE_CPU_ENCODE set + * interrupts can route to 256 vCPUs. + */ + if (val & EXTIOI_HAS_CPU_ENCODE) { + val = iocsr_read32(EXTIOI_VIRT_CONFIG); + val |= EXTIOI_ENABLE_CPU_ENCODE; + iocsr_write32(val, EXTIOI_VIRT_CONFIG); + priv->flags = EIOINTC_USE_CPU_ENCODE; + } + } + eiointc_priv[nr_pics++] = priv; + /* + * Only the first eiointc device on VM supports routing to + * different CPU interrupt pins. The later eiointc devices use + * generic method if there are multiple eiointc devices in future + */ + if (cpu_has_hypervisor && (nr_pics == 1)) { + priv->flags |= EIOINTC_ROUTE_MULT_IP; + priv->parent_hwirq = INT_HWI0; + } + + if (priv->flags & EIOINTC_ROUTE_MULT_IP) { + for (i = 0; i < priv->vec_count / VEC_COUNT_PER_REG; i++) { + priv->route_info[i].start = priv->parent_hwirq - INT_HWI0 + i; + priv->route_info[i].end = priv->route_info[i].start + 1; + priv->route_info[i].priv = priv; + parent_irq = get_percpu_irq(priv->parent_hwirq + i); + irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, + &priv->route_info[i]); + } + } else { + priv->route_info[0].start = 0; + priv->route_info[0].end = priv->vec_count / VEC_COUNT_PER_REG; + priv->route_info[0].priv = priv; + irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, + &priv->route_info[0]); + } eiointc_router_init(0); - irq_set_chained_handler_and_data(parent_irq, eiointc_irq_dispatch, priv); if (nr_pics == 1) { - register_syscore_ops(&eiointc_syscore_ops); - cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_LOONGARCH_STARTING, - "irqchip/loongarch/intc:starting", + register_syscore(&eiointc_syscore); + cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_EIOINTC_STARTING, + "irqchip/loongarch/eiointc:starting", eiointc_router_init, NULL); } @@ -433,7 +573,7 @@ int __init eiointc_acpi_init(struct irq_domain *parent, priv->vec_count = VEC_COUNT; priv->node = acpi_eiointc->node; - + priv->parent_hwirq = acpi_eiointc->cascade; parent_irq = irq_create_mapping(parent, acpi_eiointc->cascade); ret = eiointc_init(priv, parent_irq, acpi_eiointc->node_map); @@ -441,7 +581,7 @@ int __init eiointc_acpi_init(struct irq_domain *parent, goto out_free_handle; if (cpu_has_flatmode) - node = cpu_to_node(acpi_eiointc->node * CORES_PER_EIO_NODE); + node = early_cpu_to_node(acpi_eiointc->node * CORES_PER_EIO_NODE); else node = acpi_eiointc->node; acpi_set_vec_parent(node, priv->eiointc_domain, pch_group); @@ -465,8 +605,9 @@ out_free_priv: static int __init eiointc_of_init(struct device_node *of_node, struct device_node *parent) { - int parent_irq, ret; struct eiointc_priv *priv; + struct irq_data *irq_data; + int parent_irq, ret; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -482,6 +623,12 @@ static int __init eiointc_of_init(struct device_node *of_node, if (ret < 0) goto out_free_priv; + irq_data = irq_get_irq_data(parent_irq); + if (!irq_data) { + ret = -ENODEV; + goto out_free_priv; + } + /* * In particular, the number of devices supported by the LS2K0500 * extended I/O interrupt vector is 128. @@ -490,9 +637,9 @@ static int __init eiointc_of_init(struct device_node *of_node, priv->vec_count = 128; else priv->vec_count = VEC_COUNT; - + priv->parent_hwirq = irqd_to_hwirq(irq_data); priv->node = 0; - priv->domain_handle = of_node_to_fwnode(of_node); + priv->domain_handle = of_fwnode_handle(of_node); ret = eiointc_init(priv, parent_irq, 0); if (ret < 0) diff --git a/drivers/irqchip/irq-loongson-htpic.c b/drivers/irqchip/irq-loongson-htpic.c index f4abdf156de7..1c691c4be989 100644 --- a/drivers/irqchip/irq-loongson-htpic.c +++ b/drivers/irqchip/irq-loongson-htpic.c @@ -71,15 +71,19 @@ static void htpic_reg_init(void) writel(0xffff, htpic->base + HTINT_EN_OFF); } -static void htpic_resume(void) +static void htpic_resume(void *data) { htpic_reg_init(); } -struct syscore_ops htpic_syscore_ops = { +static const struct syscore_ops htpic_syscore_ops = { .resume = htpic_resume, }; +static struct syscore htpic_syscore = { + .ops = &htpic_syscore_ops, +}; + static int __init htpic_of_init(struct device_node *node, struct device_node *parent) { unsigned int parent_irq[4]; @@ -130,7 +134,7 @@ static int __init htpic_of_init(struct device_node *node, struct device_node *pa htpic_irq_dispatch, htpic); } - register_syscore_ops(&htpic_syscore_ops); + register_syscore(&htpic_syscore); return 0; diff --git a/drivers/irqchip/irq-loongson-htvec.c b/drivers/irqchip/irq-loongson-htvec.c index 0bff728b25e3..d2be8e954e92 100644 --- a/drivers/irqchip/irq-loongson-htvec.c +++ b/drivers/irqchip/irq-loongson-htvec.c @@ -17,6 +17,8 @@ #include <linux/of_irq.h> #include <linux/syscore_ops.h> +#include "irq-loongson.h" + /* Registers */ #define HTVEC_EN_OFF 0x20 #define HTVEC_MAX_PARENT_IRQ 8 @@ -157,7 +159,7 @@ static void htvec_reset(struct htvec *priv) } } -static int htvec_suspend(void) +static int htvec_suspend(void *data) { int i; @@ -167,7 +169,7 @@ static int htvec_suspend(void) return 0; } -static void htvec_resume(void) +static void htvec_resume(void *data) { int i; @@ -175,11 +177,15 @@ static void htvec_resume(void) writel(htvec_priv->saved_vec_en[i], htvec_priv->base + HTVEC_EN_OFF + 4 * i); } -static struct syscore_ops htvec_syscore_ops = { +static const struct syscore_ops htvec_syscore_ops = { .suspend = htvec_suspend, .resume = htvec_resume, }; +static struct syscore htvec_syscore = { + .ops = &htvec_syscore_ops, +}; + static int htvec_init(phys_addr_t addr, unsigned long size, int num_parents, int parent_irq[], struct fwnode_handle *domain_handle) { @@ -212,7 +218,7 @@ static int htvec_init(phys_addr_t addr, unsigned long size, htvec_priv = priv; - register_syscore_ops(&htvec_syscore_ops); + register_syscore(&htvec_syscore); return 0; @@ -246,7 +252,7 @@ static int htvec_of_init(struct device_node *node, } err = htvec_init(res.start, resource_size(&res), - num_parents, parent_irq, of_node_to_fwnode(node)); + num_parents, parent_irq, of_fwnode_handle(node)); if (err < 0) return err; diff --git a/drivers/irqchip/irq-loongson-liointc.c b/drivers/irqchip/irq-loongson-liointc.c index e4b33aed1c97..0033c2188abc 100644 --- a/drivers/irqchip/irq-loongson-liointc.c +++ b/drivers/irqchip/irq-loongson-liointc.c @@ -22,13 +22,15 @@ #include <asm/loongson.h> #endif +#include "irq-loongson.h" + #define LIOINTC_CHIP_IRQ 32 #define LIOINTC_NUM_PARENT 4 #define LIOINTC_NUM_CORES 4 #define LIOINTC_INTC_CHIP_START 0x20 -#define LIOINTC_REG_INTC_STATUS (LIOINTC_INTC_CHIP_START + 0x20) +#define LIOINTC_REG_INTC_STATUS(core) (LIOINTC_INTC_CHIP_START + 0x20 + (core) * 8) #define LIOINTC_REG_INTC_EN_STATUS (LIOINTC_INTC_CHIP_START + 0x04) #define LIOINTC_REG_INTC_ENABLE (LIOINTC_INTC_CHIP_START + 0x08) #define LIOINTC_REG_INTC_DISABLE (LIOINTC_INTC_CHIP_START + 0x0c) @@ -114,9 +116,8 @@ static int liointc_set_type(struct irq_data *data, unsigned int type) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); u32 mask = data->mask; - unsigned long flags; - irq_gc_lock_irqsave(gc, flags); + guard(raw_spinlock)(&gc->lock); switch (type) { case IRQ_TYPE_LEVEL_HIGH: liointc_set_bit(gc, LIOINTC_REG_INTC_EDGE, mask, false); @@ -135,10 +136,8 @@ static int liointc_set_type(struct irq_data *data, unsigned int type) liointc_set_bit(gc, LIOINTC_REG_INTC_POL, mask, true); break; default: - irq_gc_unlock_irqrestore(gc, flags); return -EINVAL; } - irq_gc_unlock_irqrestore(gc, flags); irqd_set_trigger_type(data, type); return 0; @@ -155,10 +154,9 @@ static void liointc_suspend(struct irq_chip_generic *gc) static void liointc_resume(struct irq_chip_generic *gc) { struct liointc_priv *priv = gc->private; - unsigned long flags; int i; - irq_gc_lock_irqsave(gc, flags); + guard(raw_spinlock_irqsave)(&gc->lock); /* Disable all at first */ writel(0xffffffff, gc->reg_base + LIOINTC_REG_INTC_DISABLE); /* Restore map cache */ @@ -168,7 +166,6 @@ static void liointc_resume(struct irq_chip_generic *gc) writel(priv->int_edge, gc->reg_base + LIOINTC_REG_INTC_EDGE); /* Restore mask cache */ writel(gc->mask_cache, gc->reg_base + LIOINTC_REG_INTC_ENABLE); - irq_gc_unlock_irqrestore(gc, flags); } static int parent_irq[LIOINTC_NUM_PARENT]; @@ -217,7 +214,7 @@ static int liointc_init(phys_addr_t addr, unsigned long size, int revision, goto out_free_priv; for (i = 0; i < LIOINTC_NUM_CORES; i++) - priv->core_isr[i] = base + LIOINTC_REG_INTC_STATUS; + priv->core_isr[i] = base + LIOINTC_REG_INTC_STATUS(i); for (i = 0; i < LIOINTC_NUM_PARENT; i++) priv->handler[i].parent_int_map = parent_int_map[i]; @@ -361,7 +358,7 @@ static int __init liointc_of_init(struct device_node *node, } err = liointc_init(res.start, resource_size(&res), - revision, of_node_to_fwnode(node), node); + revision, of_fwnode_handle(node), node); if (err < 0) return err; diff --git a/drivers/irqchip/irq-loongson-pch-lpc.c b/drivers/irqchip/irq-loongson-pch-lpc.c index 9b35492fb6be..3a125f3e4287 100644 --- a/drivers/irqchip/irq-loongson-pch-lpc.c +++ b/drivers/irqchip/irq-loongson-pch-lpc.c @@ -15,6 +15,8 @@ #include <linux/kernel.h> #include <linux/syscore_ops.h> +#include "irq-loongson.h" + /* Registers */ #define LPC_INT_CTL 0x00 #define LPC_INT_ENA 0x04 @@ -149,7 +151,7 @@ static int pch_lpc_disabled(struct pch_lpc *priv) (readl(priv->base + LPC_INT_STS) == 0xffffffff); } -static int pch_lpc_suspend(void) +static int pch_lpc_suspend(void *data) { pch_lpc_priv->saved_reg_ctl = readl(pch_lpc_priv->base + LPC_INT_CTL); pch_lpc_priv->saved_reg_ena = readl(pch_lpc_priv->base + LPC_INT_ENA); @@ -157,18 +159,22 @@ static int pch_lpc_suspend(void) return 0; } -static void pch_lpc_resume(void) +static void pch_lpc_resume(void *data) { writel(pch_lpc_priv->saved_reg_ctl, pch_lpc_priv->base + LPC_INT_CTL); writel(pch_lpc_priv->saved_reg_ena, pch_lpc_priv->base + LPC_INT_ENA); writel(pch_lpc_priv->saved_reg_pol, pch_lpc_priv->base + LPC_INT_POL); } -static struct syscore_ops pch_lpc_syscore_ops = { +static const struct syscore_ops pch_lpc_syscore_ops = { .suspend = pch_lpc_suspend, .resume = pch_lpc_resume, }; +static struct syscore pch_lpc_syscore = { + .ops = &pch_lpc_syscore_ops, +}; + int __init pch_lpc_acpi_init(struct irq_domain *parent, struct acpi_madt_lpc_pic *acpi_pchlpc) { @@ -198,8 +204,13 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent, goto iounmap_base; } - priv->lpc_domain = irq_domain_create_linear(irq_handle, LPC_COUNT, - &pch_lpc_domain_ops, priv); + /* + * The LPC interrupt controller is a legacy i8259-compatible device, + * which requires a static 1:1 mapping for IRQs 0-15. + * Use irq_domain_create_legacy to establish this static mapping early. + */ + priv->lpc_domain = irq_domain_create_legacy(irq_handle, LPC_COUNT, 0, 0, + &pch_lpc_domain_ops, priv); if (!priv->lpc_domain) { pr_err("Failed to create IRQ domain\n"); goto free_irq_handle; @@ -215,7 +226,7 @@ int __init pch_lpc_acpi_init(struct irq_domain *parent, pch_lpc_priv = priv; pch_lpc_handle = irq_handle; - register_syscore_ops(&pch_lpc_syscore_ops); + register_syscore(&pch_lpc_syscore); return 0; diff --git a/drivers/irqchip/irq-loongson-pch-msi.c b/drivers/irqchip/irq-loongson-pch-msi.c index 6e1e1f011bb2..4aedc9b90ff7 100644 --- a/drivers/irqchip/irq-loongson-pch-msi.c +++ b/drivers/irqchip/irq-loongson-pch-msi.c @@ -15,6 +15,9 @@ #include <linux/pci.h> #include <linux/slab.h> +#include <linux/irqchip/irq-msi-lib.h> +#include "irq-loongson.h" + static int nr_pics; struct pch_msi_data { @@ -27,26 +30,6 @@ struct pch_msi_data { static struct fwnode_handle *pch_msi_handle[MAX_IO_PICS]; -static void pch_msi_mask_msi_irq(struct irq_data *d) -{ - pci_msi_mask_irq(d); - irq_chip_mask_parent(d); -} - -static void pch_msi_unmask_msi_irq(struct irq_data *d) -{ - irq_chip_unmask_parent(d); - pci_msi_unmask_irq(d); -} - -static struct irq_chip pch_msi_irq_chip = { - .name = "PCH PCI MSI", - .irq_mask = pch_msi_mask_msi_irq, - .irq_unmask = pch_msi_unmask_msi_irq, - .irq_ack = irq_chip_ack_parent, - .irq_set_affinity = irq_chip_set_affinity_parent, -}; - static int pch_msi_allocate_hwirq(struct pch_msi_data *priv, int num_req) { int first; @@ -85,12 +68,6 @@ static void pch_msi_compose_msi_msg(struct irq_data *data, msg->data = data->hwirq; } -static struct msi_domain_info pch_msi_domain_info = { - .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_MULTI_PCI_MSI | MSI_FLAG_PCI_MSIX, - .chip = &pch_msi_irq_chip, -}; - static struct irq_chip middle_irq_chip = { .name = "PCH MSI", .irq_mask = irq_chip_mask_parent, @@ -136,7 +113,7 @@ static int pch_msi_middle_domain_alloc(struct irq_domain *domain, err_hwirq: pch_msi_free_hwirq(priv, hwirq, nr_irqs); - irq_domain_free_irqs_parent(domain, virq, i - 1); + irq_domain_free_irqs_parent(domain, virq, i); return err; } @@ -155,34 +132,42 @@ static void pch_msi_middle_domain_free(struct irq_domain *domain, static const struct irq_domain_ops pch_msi_middle_domain_ops = { .alloc = pch_msi_middle_domain_alloc, .free = pch_msi_middle_domain_free, + .select = msi_lib_irq_domain_select, +}; + +#define PCH_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT) + +#define PCH_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_PCI_MSIX | \ + MSI_FLAG_MULTI_PCI_MSI) + +static struct msi_parent_ops pch_msi_parent_ops = { + .required_flags = PCH_MSI_FLAGS_REQUIRED, + .supported_flags = PCH_MSI_FLAGS_SUPPORTED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI | MSI_CHIP_FLAG_SET_ACK, + .bus_select_mask = MATCH_PCI_MSI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .prefix = "PCH-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, }; -static int pch_msi_init_domains(struct pch_msi_data *priv, - struct irq_domain *parent, +static int pch_msi_init_domains(struct pch_msi_data *priv, struct irq_domain *parent, struct fwnode_handle *domain_handle) { - struct irq_domain *middle_domain, *msi_domain; - - middle_domain = irq_domain_create_hierarchy(parent, 0, priv->num_irqs, - domain_handle, - &pch_msi_middle_domain_ops, - priv); - if (!middle_domain) { + struct irq_domain_info info = { + .ops = &pch_msi_middle_domain_ops, + .size = priv->num_irqs, + .parent = parent, + .host_data = priv, + .fwnode = domain_handle, + }; + + if (!msi_create_parent_irq_domain(&info, &pch_msi_parent_ops)) { pr_err("Failed to create the MSI middle domain\n"); return -ENOMEM; } - - irq_domain_update_bus_token(middle_domain, DOMAIN_BUS_NEXUS); - - msi_domain = pci_msi_create_irq_domain(domain_handle, - &pch_msi_domain_info, - middle_domain); - if (!msi_domain) { - pr_err("Failed to create PCI MSI domain\n"); - irq_domain_remove(middle_domain); - return -ENOMEM; - } - return 0; } @@ -253,7 +238,7 @@ static int pch_msi_of_init(struct device_node *node, struct device_node *parent) return -EINVAL; } - err = pch_msi_init(res.start, irq_base, irq_count, parent_domain, of_node_to_fwnode(node)); + err = pch_msi_init(res.start, irq_base, irq_count, parent_domain, of_fwnode_handle(node)); if (err < 0) return err; @@ -266,17 +251,17 @@ IRQCHIP_DECLARE(pch_msi, "loongson,pch-msi-1.0", pch_msi_of_init); #ifdef CONFIG_ACPI struct fwnode_handle *get_pch_msi_handle(int pci_segment) { - int i; + if (cpu_has_avecint) + return pch_msi_handle[0]; - for (i = 0; i < MAX_IO_PICS; i++) { + for (int i = 0; i < MAX_IO_PICS; i++) { if (msi_group[i].pci_segment == pci_segment) return pch_msi_handle[i]; } - return NULL; + return pch_msi_handle[0]; } -int __init pch_msi_acpi_init(struct irq_domain *parent, - struct acpi_madt_msi_pic *acpi_pchmsi) +int __init pch_msi_acpi_init(struct irq_domain *parent, struct acpi_madt_msi_pic *acpi_pchmsi) { int ret; struct fwnode_handle *domain_handle; @@ -289,4 +274,18 @@ int __init pch_msi_acpi_init(struct irq_domain *parent, return ret; } + +int __init pch_msi_acpi_init_avec(struct irq_domain *parent) +{ + if (pch_msi_handle[0]) + return 0; + + pch_msi_handle[0] = parent->fwnode; + irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS); + + parent->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; + parent->msi_parent_ops = &pch_msi_parent_ops; + + return 0; +} #endif diff --git a/drivers/irqchip/irq-loongson-pch-pic.c b/drivers/irqchip/irq-loongson-pch-pic.c index 63db8e2172e0..c6b369a974a7 100644 --- a/drivers/irqchip/irq-loongson-pch-pic.c +++ b/drivers/irqchip/irq-loongson-pch-pic.c @@ -17,6 +17,8 @@ #include <linux/of_irq.h> #include <linux/syscore_ops.h> +#include "irq-loongson.h" + /* Registers */ #define PCH_PIC_MASK 0x20 #define PCH_PIC_HTMSI_EN 0x40 @@ -33,6 +35,7 @@ #define PIC_COUNT (PIC_COUNT_PER_REG * PIC_REG_COUNT) #define PIC_REG_IDX(irq_id) ((irq_id) / PIC_COUNT_PER_REG) #define PIC_REG_BIT(irq_id) ((irq_id) % PIC_COUNT_PER_REG) +#define PIC_UNDEF_VECTOR 255 static int nr_pics; @@ -46,12 +49,19 @@ struct pch_pic { u32 saved_vec_en[PIC_REG_COUNT]; u32 saved_vec_pol[PIC_REG_COUNT]; u32 saved_vec_edge[PIC_REG_COUNT]; + u8 table[PIC_COUNT]; + int inuse; }; static struct pch_pic *pch_pic_priv[MAX_IO_PICS]; struct fwnode_handle *pch_pic_handle[MAX_IO_PICS]; +static inline u8 hwirq_to_bit(struct pch_pic *priv, int hirq) +{ + return priv->table[hirq]; +} + static void pch_pic_bitset(struct pch_pic *priv, int offset, int bit) { u32 reg; @@ -80,45 +90,47 @@ static void pch_pic_mask_irq(struct irq_data *d) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); - pch_pic_bitset(priv, PCH_PIC_MASK, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_MASK, hwirq_to_bit(priv, d->hwirq)); irq_chip_mask_parent(d); } static void pch_pic_unmask_irq(struct irq_data *d) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); - writel(BIT(PIC_REG_BIT(d->hwirq)), - priv->base + PCH_PIC_CLR + PIC_REG_IDX(d->hwirq) * 4); + writel(BIT(PIC_REG_BIT(bit)), + priv->base + PCH_PIC_CLR + PIC_REG_IDX(bit) * 4); irq_chip_unmask_parent(d); - pch_pic_bitclr(priv, PCH_PIC_MASK, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_MASK, bit); } static int pch_pic_set_type(struct irq_data *d, unsigned int type) { struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); int ret = 0; switch (type) { case IRQ_TYPE_EDGE_RISING: - pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_EDGE, bit); + pch_pic_bitclr(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_edge_irq); break; case IRQ_TYPE_EDGE_FALLING: - pch_pic_bitset(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitset(priv, PCH_PIC_EDGE, bit); + pch_pic_bitset(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_edge_irq); break; case IRQ_TYPE_LEVEL_HIGH: - pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitclr(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_EDGE, bit); + pch_pic_bitclr(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_level_irq); break; case IRQ_TYPE_LEVEL_LOW: - pch_pic_bitclr(priv, PCH_PIC_EDGE, d->hwirq); - pch_pic_bitset(priv, PCH_PIC_POL, d->hwirq); + pch_pic_bitclr(priv, PCH_PIC_EDGE, bit); + pch_pic_bitset(priv, PCH_PIC_POL, bit); irq_set_handler_locked(d, handle_level_irq); break; default: @@ -133,11 +145,12 @@ static void pch_pic_ack_irq(struct irq_data *d) { unsigned int reg; struct pch_pic *priv = irq_data_get_irq_chip_data(d); + int bit = hwirq_to_bit(priv, d->hwirq); - reg = readl(priv->base + PCH_PIC_EDGE + PIC_REG_IDX(d->hwirq) * 4); - if (reg & BIT(PIC_REG_BIT(d->hwirq))) { - writel(BIT(PIC_REG_BIT(d->hwirq)), - priv->base + PCH_PIC_CLR + PIC_REG_IDX(d->hwirq) * 4); + reg = readl(priv->base + PCH_PIC_EDGE + PIC_REG_IDX(bit) * 4); + if (reg & BIT(PIC_REG_BIT(bit))) { + writel(BIT(PIC_REG_BIT(bit)), + priv->base + PCH_PIC_CLR + PIC_REG_IDX(bit) * 4); } irq_chip_ack_parent(d); } @@ -159,6 +172,8 @@ static int pch_pic_domain_translate(struct irq_domain *d, { struct pch_pic *priv = d->host_data; struct device_node *of_node = to_of_node(fwspec->fwnode); + unsigned long flags; + int i; if (of_node) { if (fwspec->param_count < 2) @@ -171,12 +186,33 @@ static int pch_pic_domain_translate(struct irq_domain *d, return -EINVAL; *hwirq = fwspec->param[0] - priv->gsi_base; + if (fwspec->param_count > 1) *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK; else *type = IRQ_TYPE_NONE; } + raw_spin_lock_irqsave(&priv->pic_lock, flags); + /* Check pic-table to confirm if the hwirq has been assigned */ + for (i = 0; i < priv->inuse; i++) { + if (priv->table[i] == *hwirq) { + *hwirq = i; + break; + } + } + if (i == priv->inuse) { + /* Assign a new hwirq in pic-table */ + if (priv->inuse >= PIC_COUNT) { + pr_err("pch-pic domain has no free vectors\n"); + raw_spin_unlock_irqrestore(&priv->pic_lock, flags); + return -EINVAL; + } + priv->table[priv->inuse] = *hwirq; + *hwirq = priv->inuse++; + } + raw_spin_unlock_irqrestore(&priv->pic_lock, flags); + return 0; } @@ -194,6 +230,9 @@ static int pch_pic_alloc(struct irq_domain *domain, unsigned int virq, if (err) return err; + /* Write vector ID */ + writeb(priv->ht_vec_base + hwirq, priv->base + PCH_INT_HTVEC(hwirq_to_bit(priv, hwirq))); + parent_fwspec.fwnode = domain->parent->fwnode; parent_fwspec.param_count = 1; parent_fwspec.param[0] = hwirq + priv->ht_vec_base; @@ -222,7 +261,7 @@ static void pch_pic_reset(struct pch_pic *priv) for (i = 0; i < PIC_COUNT; i++) { /* Write vector ID */ - writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(i)); + writeb(priv->ht_vec_base + i, priv->base + PCH_INT_HTVEC(hwirq_to_bit(priv, i))); /* Hardcode route to HT0 Lo */ writeb(1, priv->base + PCH_INT_ROUTE(i)); } @@ -239,7 +278,7 @@ static void pch_pic_reset(struct pch_pic *priv) } } -static int pch_pic_suspend(void) +static int pch_pic_suspend(void *data) { int i, j; @@ -257,7 +296,7 @@ static int pch_pic_suspend(void) return 0; } -static void pch_pic_resume(void) +static void pch_pic_resume(void *data) { int i, j; @@ -274,16 +313,21 @@ static void pch_pic_resume(void) } } -static struct syscore_ops pch_pic_syscore_ops = { +static const struct syscore_ops pch_pic_syscore_ops = { .suspend = pch_pic_suspend, .resume = pch_pic_resume, }; +static struct syscore pch_pic_syscore = { + .ops = &pch_pic_syscore_ops, +}; + static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, struct irq_domain *parent_domain, struct fwnode_handle *domain_handle, u32 gsi_base) { struct pch_pic *priv; + int i; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -294,6 +338,10 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, if (!priv->base) goto free_priv; + priv->inuse = 0; + for (i = 0; i < PIC_COUNT; i++) + priv->table[i] = PIC_UNDEF_VECTOR; + priv->ht_vec_base = vec_base; priv->vec_count = ((readq(priv->base) >> 48) & 0xff) + 1; priv->gsi_base = gsi_base; @@ -312,7 +360,7 @@ static int pch_pic_init(phys_addr_t addr, unsigned long size, int vec_base, pch_pic_priv[nr_pics++] = priv; if (nr_pics == 1) - register_syscore_ops(&pch_pic_syscore_ops); + register_syscore(&pch_pic_syscore); return 0; @@ -348,7 +396,7 @@ static int pch_pic_of_init(struct device_node *node, } err = pch_pic_init(res.start, resource_size(&res), vec_base, - parent_domain, of_node_to_fwnode(node), 0); + parent_domain, of_fwnode_handle(node), 0); if (err < 0) return err; diff --git a/drivers/irqchip/irq-loongson.h b/drivers/irqchip/irq-loongson.h new file mode 100644 index 000000000000..11fa138d1f44 --- /dev/null +++ b/drivers/irqchip/irq-loongson.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (C) 2024 Loongson Technology Corporation Limited + */ + +#ifndef _DRIVERS_IRQCHIP_IRQ_LOONGSON_H +#define _DRIVERS_IRQCHIP_IRQ_LOONGSON_H + +int find_pch_pic(u32 gsi); + +int liointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lio_pic *acpi_liointc); +int eiointc_acpi_init(struct irq_domain *parent, + struct acpi_madt_eio_pic *acpi_eiointc); +int avecintc_acpi_init(struct irq_domain *parent); + +int htvec_acpi_init(struct irq_domain *parent, + struct acpi_madt_ht_pic *acpi_htvec); +int pch_lpc_acpi_init(struct irq_domain *parent, + struct acpi_madt_lpc_pic *acpi_pchlpc); +int pch_pic_acpi_init(struct irq_domain *parent, + struct acpi_madt_bio_pic *acpi_pchpic); +int pch_msi_acpi_init(struct irq_domain *parent, + struct acpi_madt_msi_pic *acpi_pchmsi); +int pch_msi_acpi_init_avec(struct irq_domain *parent); + +#endif /* _DRIVERS_IRQCHIP_IRQ_LOONGSON_H */ diff --git a/drivers/irqchip/irq-lpc32xx.c b/drivers/irqchip/irq-lpc32xx.c index 4d70a857133f..14cca44baa14 100644 --- a/drivers/irqchip/irq-lpc32xx.c +++ b/drivers/irqchip/irq-lpc32xx.c @@ -210,8 +210,8 @@ static int __init lpc32xx_of_ic_init(struct device_node *node, return -EINVAL; } - irqc->domain = irq_domain_add_linear(node, NR_LPC32XX_IC_IRQS, - &lpc32xx_irq_domain_ops, irqc); + irqc->domain = irq_domain_create_linear(of_fwnode_handle(node), NR_LPC32XX_IC_IRQS, + &lpc32xx_irq_domain_ops, irqc); if (!irqc->domain) { pr_err("unable to add irq domain\n"); iounmap(irqc->base); diff --git a/drivers/irqchip/irq-ls-extirq.c b/drivers/irqchip/irq-ls-extirq.c index 139f26b0a6ef..50a7b38381b9 100644 --- a/drivers/irqchip/irq-ls-extirq.c +++ b/drivers/irqchip/irq-ls-extirq.c @@ -208,8 +208,8 @@ ls_extirq_of_init(struct device_node *node, struct device_node *parent) of_device_is_compatible(node, "fsl,ls1043a-extirq"); raw_spin_lock_init(&priv->lock); - domain = irq_domain_add_hierarchy(parent_domain, 0, priv->nirq, node, - &extirq_domain_ops, priv); + domain = irq_domain_create_hierarchy(parent_domain, 0, priv->nirq, of_fwnode_handle(node), + &extirq_domain_ops, priv); if (!domain) { ret = -ENOMEM; goto err_add_hierarchy; diff --git a/drivers/irqchip/irq-ls-scfg-msi.c b/drivers/irqchip/irq-ls-scfg-msi.c index f31a262fe438..4910f364e568 100644 --- a/drivers/irqchip/irq-ls-scfg-msi.c +++ b/drivers/irqchip/irq-ls-scfg-msi.c @@ -14,10 +14,12 @@ #include <linux/iommu.h> #include <linux/irq.h> #include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/irq-msi-lib.h> #include <linux/irqdomain.h> #include <linux/of_irq.h> #include <linux/of_pci.h> -#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/property.h> #include <linux/spinlock.h> #define MSI_IRQS_PER_MSIR 32 @@ -46,7 +48,6 @@ struct ls_scfg_msi { spinlock_t lock; struct platform_device *pdev; struct irq_domain *parent; - struct irq_domain *msi_domain; void __iomem *regs; phys_addr_t msiir_addr; struct ls_scfg_msi_cfg *cfg; @@ -56,17 +57,18 @@ struct ls_scfg_msi { unsigned long *used; }; -static struct irq_chip ls_scfg_msi_irq_chip = { - .name = "MSI", - .irq_mask = pci_msi_mask_irq, - .irq_unmask = pci_msi_unmask_irq, -}; - -static struct msi_domain_info ls_scfg_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | - MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_PCI_MSIX), - .chip = &ls_scfg_msi_irq_chip, +#define MPIC_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) +#define MPIC_MSI_FLAGS_SUPPORTED (MSI_FLAG_PCI_MSIX | \ + MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops ls_scfg_msi_parent_ops = { + .required_flags = MPIC_MSI_FLAGS_REQUIRED, + .supported_flags = MPIC_MSI_FLAGS_SUPPORTED, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI, + .prefix = "MSI-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, }; static int msi_affinity_flag = 1; @@ -86,8 +88,6 @@ static void ls_scfg_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) { struct ls_scfg_msi *msi_data = irq_data_get_irq_chip_data(data); - msg->address_hi = upper_32_bits(msi_data->msiir_addr); - msg->address_lo = lower_32_bits(msi_data->msiir_addr); msg->data = data->hwirq; if (msi_affinity_flag) { @@ -97,7 +97,8 @@ static void ls_scfg_msi_compose_msg(struct irq_data *data, struct msi_msg *msg) msg->data |= cpumask_first(mask); } - iommu_dma_compose_msi_msg(irq_data_get_msi_desc(data), msg); + msi_msg_set_addr(irq_data_get_msi_desc(data), msg, + msi_data->msiir_addr); } static int ls_scfg_msi_set_affinity(struct irq_data *irq_data, @@ -185,6 +186,7 @@ static void ls_scfg_msi_domain_irq_free(struct irq_domain *domain, } static const struct irq_domain_ops ls_scfg_msi_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = ls_scfg_msi_domain_irq_alloc, .free = ls_scfg_msi_domain_irq_free, }; @@ -214,23 +216,16 @@ static void ls_scfg_msi_irq_handler(struct irq_desc *desc) static int ls_scfg_msi_domains_init(struct ls_scfg_msi *msi_data) { - /* Initialize MSI domain parent */ - msi_data->parent = irq_domain_add_linear(NULL, - msi_data->irqs_num, - &ls_scfg_msi_domain_ops, - msi_data); + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(msi_data->pdev->dev.of_node), + .ops = &ls_scfg_msi_domain_ops, + .host_data = msi_data, + .size = msi_data->irqs_num, + }; + + msi_data->parent = msi_create_parent_irq_domain(&info, &ls_scfg_msi_parent_ops); if (!msi_data->parent) { - dev_err(&msi_data->pdev->dev, "failed to create IRQ domain\n"); - return -ENOMEM; - } - - msi_data->msi_domain = pci_msi_create_irq_domain( - of_node_to_fwnode(msi_data->pdev->dev.of_node), - &ls_scfg_msi_domain_info, - msi_data->parent); - if (!msi_data->msi_domain) { dev_err(&msi_data->pdev->dev, "failed to create MSI domain\n"); - irq_domain_remove(msi_data->parent); return -ENOMEM; } @@ -334,20 +329,17 @@ MODULE_DEVICE_TABLE(of, ls_scfg_msi_id); static int ls_scfg_msi_probe(struct platform_device *pdev) { - const struct of_device_id *match; struct ls_scfg_msi *msi_data; struct resource *res; int i, ret; - match = of_match_device(ls_scfg_msi_id, &pdev->dev); - if (!match) - return -ENODEV; - msi_data = devm_kzalloc(&pdev->dev, sizeof(*msi_data), GFP_KERNEL); if (!msi_data) return -ENOMEM; - msi_data->cfg = (struct ls_scfg_msi_cfg *) match->data; + msi_data->cfg = (struct ls_scfg_msi_cfg *)device_get_match_data(&pdev->dev); + if (!msi_data->cfg) + return -ENODEV; msi_data->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(msi_data->regs)) { @@ -400,7 +392,7 @@ static int ls_scfg_msi_probe(struct platform_device *pdev) return 0; } -static int ls_scfg_msi_remove(struct platform_device *pdev) +static void ls_scfg_msi_remove(struct platform_device *pdev) { struct ls_scfg_msi *msi_data = platform_get_drvdata(pdev); int i; @@ -408,21 +400,18 @@ static int ls_scfg_msi_remove(struct platform_device *pdev) for (i = 0; i < msi_data->msir_num; i++) ls_scfg_msi_teardown_hwirq(&msi_data->msir[i]); - irq_domain_remove(msi_data->msi_domain); irq_domain_remove(msi_data->parent); platform_set_drvdata(pdev, NULL); - - return 0; } static struct platform_driver ls_scfg_msi_driver = { .driver = { - .name = "ls-scfg-msi", - .of_match_table = ls_scfg_msi_id, + .name = "ls-scfg-msi", + .of_match_table = ls_scfg_msi_id, }, - .probe = ls_scfg_msi_probe, - .remove = ls_scfg_msi_remove, + .probe = ls_scfg_msi_probe, + .remove = ls_scfg_msi_remove, }; module_platform_driver(ls_scfg_msi_driver); diff --git a/drivers/irqchip/irq-ls1x.c b/drivers/irqchip/irq-ls1x.c index 77a3f7dfaaf0..589d32007fca 100644 --- a/drivers/irqchip/irq-ls1x.c +++ b/drivers/irqchip/irq-ls1x.c @@ -126,8 +126,8 @@ static int __init ls1x_intc_of_init(struct device_node *node, } /* Set up an IRQ domain */ - priv->domain = irq_domain_add_linear(node, 32, &irq_generic_chip_ops, - NULL); + priv->domain = irq_domain_create_linear(of_fwnode_handle(node), 32, &irq_generic_chip_ops, + NULL); if (!priv->domain) { pr_err("ls1x-irq: cannot add IRQ domain\n"); err = -ENOMEM; diff --git a/drivers/irqchip/irq-madera.c b/drivers/irqchip/irq-madera.c index 3eb1f8cdf674..b32982c11515 100644 --- a/drivers/irqchip/irq-madera.c +++ b/drivers/irqchip/irq-madera.c @@ -222,7 +222,7 @@ static int madera_irq_probe(struct platform_device *pdev) return 0; } -static int madera_irq_remove(struct platform_device *pdev) +static void madera_irq_remove(struct platform_device *pdev) { struct madera *madera = dev_get_drvdata(pdev->dev.parent); @@ -232,13 +232,11 @@ static int madera_irq_remove(struct platform_device *pdev) */ madera->irq_dev = NULL; regmap_del_irq_chip(madera->irq, madera->irq_data); - - return 0; } static struct platform_driver madera_irq_driver = { - .probe = &madera_irq_probe, - .remove = &madera_irq_remove, + .probe = madera_irq_probe, + .remove = madera_irq_remove, .driver = { .name = "madera-irq", .pm = &madera_irq_pm_ops, diff --git a/drivers/irqchip/irq-mbigen.c b/drivers/irqchip/irq-mbigen.c index 5101a3fb11df..6f69f4e5dbac 100644 --- a/drivers/irqchip/irq-mbigen.c +++ b/drivers/irqchip/irq-mbigen.c @@ -64,6 +64,20 @@ struct mbigen_device { void __iomem *base; }; +static inline unsigned int get_mbigen_node_offset(unsigned int nid) +{ + unsigned int offset = nid * MBIGEN_NODE_OFFSET; + + /* + * To avoid touched clear register in unexpected way, we need to directly + * skip clear register when access to more than 10 mbigen nodes. + */ + if (nid >= (REG_MBIGEN_CLEAR_OFFSET / MBIGEN_NODE_OFFSET)) + offset += MBIGEN_NODE_OFFSET; + + return offset; +} + static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) { unsigned int nid, pin; @@ -72,8 +86,7 @@ static inline unsigned int get_mbigen_vec_reg(irq_hw_number_t hwirq) nid = hwirq / IRQS_PER_MBIGEN_NODE + 1; pin = hwirq % IRQS_PER_MBIGEN_NODE; - return pin * 4 + nid * MBIGEN_NODE_OFFSET - + REG_MBIGEN_VEC_OFFSET; + return pin * 4 + get_mbigen_node_offset(nid) + REG_MBIGEN_VEC_OFFSET; } static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, @@ -88,8 +101,7 @@ static inline void get_mbigen_type_reg(irq_hw_number_t hwirq, *mask = 1 << (irq_ofst % 32); ofst = irq_ofst / 32 * 4; - *addr = ofst + nid * MBIGEN_NODE_OFFSET - + REG_MBIGEN_TYPE_OFFSET; + *addr = ofst + get_mbigen_node_offset(nid) + REG_MBIGEN_TYPE_OFFSET; } static inline void get_mbigen_clear_reg(irq_hw_number_t hwirq, @@ -135,24 +147,14 @@ static int mbigen_set_type(struct irq_data *data, unsigned int type) return 0; } -static struct irq_chip mbigen_irq_chip = { - .name = "mbigen-v2", - .irq_mask = irq_chip_mask_parent, - .irq_unmask = irq_chip_unmask_parent, - .irq_eoi = mbigen_eoi_irq, - .irq_set_type = mbigen_set_type, - .irq_set_affinity = irq_chip_set_affinity_parent, -}; - -static void mbigen_write_msg(struct msi_desc *desc, struct msi_msg *msg) +static void mbigen_write_msi_msg(struct irq_data *d, struct msi_msg *msg) { - struct irq_data *d = irq_get_irq_data(desc->irq); void __iomem *base = d->chip_data; u32 val; if (!msg->address_lo && !msg->address_hi) return; - + base += get_mbigen_vec_reg(d->hwirq); val = readl_relaxed(base); @@ -165,10 +167,8 @@ static void mbigen_write_msg(struct msi_desc *desc, struct msi_msg *msg) writel_relaxed(val, base); } -static int mbigen_domain_translate(struct irq_domain *d, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) +static int mbigen_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) { if (is_of_node(fwspec->fwnode) || is_acpi_device_node(fwspec->fwnode)) { if (fwspec->param_count != 2) @@ -192,92 +192,69 @@ static int mbigen_domain_translate(struct irq_domain *d, return -EINVAL; } -static int mbigen_irq_domain_alloc(struct irq_domain *domain, - unsigned int virq, - unsigned int nr_irqs, - void *args) +static void mbigen_domain_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) { - struct irq_fwspec *fwspec = args; - irq_hw_number_t hwirq; - unsigned int type; - struct mbigen_device *mgn_chip; - int i, err; - - err = mbigen_domain_translate(domain, fwspec, &hwirq, &type); - if (err) - return err; - - err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); - if (err) - return err; + arg->desc = desc; + arg->hwirq = (u32)desc->data.icookie.value; +} - mgn_chip = platform_msi_get_host_data(domain); +static const struct msi_domain_template mbigen_msi_template = { + .chip = { + .name = "mbigen-v2", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = mbigen_eoi_irq, + .irq_set_type = mbigen_set_type, + .irq_write_msi_msg = mbigen_write_msi_msg, + }, - for (i = 0; i < nr_irqs; i++) - irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, - &mbigen_irq_chip, mgn_chip->base); + .ops = { + .set_desc = mbigen_domain_set_desc, + .msi_translate = mbigen_domain_translate, + }, - return 0; -} + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_USE_DEV_FWNODE, + }, +}; -static void mbigen_irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) +static bool mbigen_create_device_domain(struct device *dev, unsigned int size, + struct mbigen_device *mgn_chip) { - platform_msi_device_domain_free(domain, virq, nr_irqs); -} + if (WARN_ON_ONCE(!dev->msi.domain)) + return false; -static const struct irq_domain_ops mbigen_domain_ops = { - .translate = mbigen_domain_translate, - .alloc = mbigen_irq_domain_alloc, - .free = mbigen_irq_domain_free, -}; + return msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, + &mbigen_msi_template, size, + NULL, mgn_chip->base); +} static int mbigen_of_create_domain(struct platform_device *pdev, struct mbigen_device *mgn_chip) { - struct device *parent; struct platform_device *child; - struct irq_domain *domain; - struct device_node *np; u32 num_pins; - int ret = 0; - - parent = bus_get_dev_root(&platform_bus_type); - if (!parent) - return -ENODEV; - for_each_child_of_node(pdev->dev.of_node, np) { + for_each_child_of_node_scoped(pdev->dev.of_node, np) { if (!of_property_read_bool(np, "interrupt-controller")) continue; - child = of_platform_device_create(np, NULL, parent); - if (!child) { - ret = -ENOMEM; - break; - } + child = of_platform_device_create(np, NULL, NULL); + if (!child) + return -ENOMEM; if (of_property_read_u32(child->dev.of_node, "num-pins", &num_pins) < 0) { dev_err(&pdev->dev, "No num-pins property\n"); - ret = -EINVAL; - break; + return -EINVAL; } - domain = platform_msi_create_device_domain(&child->dev, num_pins, - mbigen_write_msg, - &mbigen_domain_ops, - mgn_chip); - if (!domain) { - ret = -ENOMEM; - break; - } + if (!mbigen_create_device_domain(&child->dev, num_pins, mgn_chip)) + return -ENOMEM; } - put_device(parent); - if (ret) - of_node_put(np); - - return ret; + return 0; } #ifdef CONFIG_ACPI @@ -290,7 +267,6 @@ MODULE_DEVICE_TABLE(acpi, mbigen_acpi_match); static int mbigen_acpi_create_domain(struct platform_device *pdev, struct mbigen_device *mgn_chip) { - struct irq_domain *domain; u32 num_pins = 0; int ret; @@ -321,11 +297,7 @@ static int mbigen_acpi_create_domain(struct platform_device *pdev, if (ret || num_pins == 0) return -EINVAL; - domain = platform_msi_create_device_domain(&pdev->dev, num_pins, - mbigen_write_msg, - &mbigen_domain_ops, - mgn_chip); - if (!domain) + if (!mbigen_create_device_domain(&pdev->dev, num_pins, mgn_chip)) return -ENOMEM; return 0; diff --git a/drivers/irqchip/irq-mchp-eic.c b/drivers/irqchip/irq-mchp-eic.c index 5dcd94c000a2..2474fa467a05 100644 --- a/drivers/irqchip/irq-mchp-eic.c +++ b/drivers/irqchip/irq-mchp-eic.c @@ -109,7 +109,7 @@ static int mchp_eic_irq_set_wake(struct irq_data *d, unsigned int on) return 0; } -static int mchp_eic_irq_suspend(void) +static int mchp_eic_irq_suspend(void *data) { unsigned int hwirq; @@ -123,7 +123,7 @@ static int mchp_eic_irq_suspend(void) return 0; } -static void mchp_eic_irq_resume(void) +static void mchp_eic_irq_resume(void *data) { unsigned int hwirq; @@ -135,11 +135,15 @@ static void mchp_eic_irq_resume(void) MCHP_EIC_SCFG(hwirq)); } -static struct syscore_ops mchp_eic_syscore_ops = { +static const struct syscore_ops mchp_eic_syscore_ops = { .suspend = mchp_eic_irq_suspend, .resume = mchp_eic_irq_resume, }; +static struct syscore mchp_eic_syscore = { + .ops = &mchp_eic_syscore_ops, +}; + static struct irq_chip mchp_eic_chip = { .name = "eic", .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SET_TYPE_MASKED, @@ -199,8 +203,9 @@ static const struct irq_domain_ops mchp_eic_domain_ops = { .free = irq_domain_free_irqs_common, }; -static int mchp_eic_init(struct device_node *node, struct device_node *parent) +static int mchp_eic_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *parent_domain = NULL; int ret, i; @@ -248,15 +253,16 @@ static int mchp_eic_init(struct device_node *node, struct device_node *parent) eic->irqs[i] = irq.args[1]; } - eic->domain = irq_domain_add_hierarchy(parent_domain, 0, MCHP_EIC_NIRQ, - node, &mchp_eic_domain_ops, eic); + eic->domain = irq_domain_create_hierarchy(parent_domain, 0, MCHP_EIC_NIRQ, + of_fwnode_handle(node), &mchp_eic_domain_ops, + eic); if (!eic->domain) { pr_err("%pOF: Failed to add domain\n", node); ret = -ENODEV; goto clk_unprepare; } - register_syscore_ops(&mchp_eic_syscore_ops); + register_syscore(&mchp_eic_syscore); pr_info("%pOF: EIC registered, nr_irqs %u\n", node, MCHP_EIC_NIRQ); @@ -272,7 +278,7 @@ free: } IRQCHIP_PLATFORM_DRIVER_BEGIN(mchp_eic) -IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_init) +IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_probe) IRQCHIP_PLATFORM_DRIVER_END(mchp_eic) MODULE_DESCRIPTION("Microchip External Interrupt Controller"); diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index f88df39f4129..3fcbb044ae60 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -26,8 +26,6 @@ /* use for A1 like chips */ #define REG_PIN_A1_SEL 0x04 -/* Used for s4 chips */ -#define REG_EDGE_POL_S4 0x1c /* * Note: The S905X3 datasheet reports that BOTH_EDGE is controlled by @@ -72,6 +70,7 @@ struct meson_gpio_irq_params { bool support_edge_both; unsigned int edge_both_offset; unsigned int edge_single_offset; + unsigned int edge_pol_reg; unsigned int pol_low_offset; unsigned int pin_sel_mask; struct irq_ctl_ops ops; @@ -105,6 +104,18 @@ struct meson_gpio_irq_params { .pin_sel_mask = 0x7f, \ .nr_channels = 8, \ +#define INIT_MESON_A4_AO_COMMON_DATA(irqs) \ + INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init, \ + meson_a1_gpio_irq_sel_pin, \ + meson_s4_gpio_irq_set_type) \ + .support_edge_both = true, \ + .edge_both_offset = 0, \ + .edge_single_offset = 12, \ + .edge_pol_reg = 0x8, \ + .pol_low_offset = 0, \ + .pin_sel_mask = 0xff, \ + .nr_channels = 2, \ + #define INIT_MESON_S4_COMMON_DATA(irqs) \ INIT_MESON_COMMON(irqs, meson_a1_gpio_irq_init, \ meson_a1_gpio_irq_sel_pin, \ @@ -112,6 +123,7 @@ struct meson_gpio_irq_params { .support_edge_both = true, \ .edge_both_offset = 0, \ .edge_single_offset = 12, \ + .edge_pol_reg = 0x1c, \ .pol_low_offset = 0, \ .pin_sel_mask = 0xff, \ .nr_channels = 12, \ @@ -146,14 +158,38 @@ static const struct meson_gpio_irq_params a1_params = { INIT_MESON_A1_COMMON_DATA(62) }; +static const struct meson_gpio_irq_params a4_params = { + INIT_MESON_S4_COMMON_DATA(81) +}; + +static const struct meson_gpio_irq_params a4_ao_params = { + INIT_MESON_A4_AO_COMMON_DATA(8) +}; + +static const struct meson_gpio_irq_params a5_params = { + INIT_MESON_S4_COMMON_DATA(99) +}; + static const struct meson_gpio_irq_params s4_params = { INIT_MESON_S4_COMMON_DATA(82) }; +static const struct meson_gpio_irq_params s6_params = { + INIT_MESON_S4_COMMON_DATA(100) +}; + +static const struct meson_gpio_irq_params s7_params = { + INIT_MESON_S4_COMMON_DATA(84) +}; + static const struct meson_gpio_irq_params c3_params = { INIT_MESON_S4_COMMON_DATA(55) }; +static const struct meson_gpio_irq_params t7_params = { + INIT_MESON_S4_COMMON_DATA(157) +}; + static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson8-gpio-intc", .data = &meson8_params }, { .compatible = "amlogic,meson8b-gpio-intc", .data = &meson8b_params }, @@ -164,7 +200,14 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,meson-sm1-gpio-intc", .data = &sm1_params }, { .compatible = "amlogic,meson-a1-gpio-intc", .data = &a1_params }, { .compatible = "amlogic,meson-s4-gpio-intc", .data = &s4_params }, + { .compatible = "amlogic,a4-gpio-ao-intc", .data = &a4_ao_params }, + { .compatible = "amlogic,a4-gpio-intc", .data = &a4_params }, + { .compatible = "amlogic,a5-gpio-intc", .data = &a5_params }, + { .compatible = "amlogic,s6-gpio-intc", .data = &s6_params }, + { .compatible = "amlogic,s7-gpio-intc", .data = &s7_params }, + { .compatible = "amlogic,s7d-gpio-intc", .data = &s7_params }, { .compatible = "amlogic,c3-gpio-intc", .data = &c3_params }, + { .compatible = "amlogic,t7-gpio-intc", .data = &t7_params }, { } }; @@ -173,7 +216,7 @@ struct meson_gpio_irq_controller { void __iomem *base; u32 channel_irqs[MAX_NUM_CHANNEL]; DECLARE_BITMAP(channel_map, MAX_NUM_CHANNEL); - spinlock_t lock; + raw_spinlock_t lock; }; static void meson_gpio_irq_update_bits(struct meson_gpio_irq_controller *ctl, @@ -182,14 +225,14 @@ static void meson_gpio_irq_update_bits(struct meson_gpio_irq_controller *ctl, unsigned long flags; u32 tmp; - spin_lock_irqsave(&ctl->lock, flags); + raw_spin_lock_irqsave(&ctl->lock, flags); tmp = readl_relaxed(ctl->base + reg); tmp &= ~mask; tmp |= val; writel_relaxed(tmp, ctl->base + reg); - spin_unlock_irqrestore(&ctl->lock, flags); + raw_spin_unlock_irqrestore(&ctl->lock, flags); } static void meson_gpio_irq_init_dummy(struct meson_gpio_irq_controller *ctl) @@ -239,12 +282,12 @@ meson_gpio_irq_request_channel(struct meson_gpio_irq_controller *ctl, unsigned long flags; unsigned int idx; - spin_lock_irqsave(&ctl->lock, flags); + raw_spin_lock_irqsave(&ctl->lock, flags); /* Find a free channel */ idx = find_first_zero_bit(ctl->channel_map, ctl->params->nr_channels); if (idx >= ctl->params->nr_channels) { - spin_unlock_irqrestore(&ctl->lock, flags); + raw_spin_unlock_irqrestore(&ctl->lock, flags); pr_err("No channel available\n"); return -ENOSPC; } @@ -252,7 +295,7 @@ meson_gpio_irq_request_channel(struct meson_gpio_irq_controller *ctl, /* Mark the channel as used */ set_bit(idx, ctl->channel_map); - spin_unlock_irqrestore(&ctl->lock, flags); + raw_spin_unlock_irqrestore(&ctl->lock, flags); /* * Setup the mux of the channel to route the signal of the pad @@ -294,11 +337,10 @@ meson_gpio_irq_release_channel(struct meson_gpio_irq_controller *ctl, static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl, unsigned int type, u32 *channel_hwirq) { - u32 val = 0; + const struct meson_gpio_irq_params *params = ctl->params; unsigned int idx; - const struct meson_gpio_irq_params *params; + u32 val = 0; - params = ctl->params; idx = meson_gpio_irq_get_channel_idx(ctl, channel_hwirq); /* @@ -351,19 +393,19 @@ static int meson8_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl, static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl, unsigned int type, u32 *channel_hwirq) { - u32 val = 0; + const struct meson_gpio_irq_params *params = ctl->params; unsigned int idx; + u32 val = 0; idx = meson_gpio_irq_get_channel_idx(ctl, channel_hwirq); type &= IRQ_TYPE_SENSE_MASK; - meson_gpio_irq_update_bits(ctl, REG_EDGE_POL_S4, BIT(idx), 0); + meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, BIT(idx), 0); if (type == IRQ_TYPE_EDGE_BOTH) { - val |= BIT(ctl->params->edge_both_offset + idx); - meson_gpio_irq_update_bits(ctl, REG_EDGE_POL_S4, - BIT(ctl->params->edge_both_offset + idx), val); + val = BIT(ctl->params->edge_both_offset + idx); + meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, val, val); return 0; } @@ -373,7 +415,7 @@ static int meson_s4_gpio_irq_set_type(struct meson_gpio_irq_controller *ctl, if (type & (IRQ_TYPE_EDGE_RISING | IRQ_TYPE_EDGE_FALLING)) val |= BIT(ctl->params->edge_single_offset + idx); - meson_gpio_irq_update_bits(ctl, REG_EDGE_POL, + meson_gpio_irq_update_bits(ctl, params->edge_pol_reg, BIT(idx) | BIT(12 + idx), val); return 0; }; @@ -541,8 +583,9 @@ static int meson_gpio_irq_parse_dt(struct device_node *node, struct meson_gpio_i return 0; } -static int meson_gpio_irq_of_init(struct device_node *node, struct device_node *parent) +static int meson_gpio_irq_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *domain, *parent_domain; struct meson_gpio_irq_controller *ctl; int ret; @@ -562,7 +605,7 @@ static int meson_gpio_irq_of_init(struct device_node *node, struct device_node * if (!ctl) return -ENOMEM; - spin_lock_init(&ctl->lock); + raw_spin_lock_init(&ctl->lock); ctl->base = of_iomap(node, 0); if (!ctl->base) { @@ -576,7 +619,7 @@ static int meson_gpio_irq_of_init(struct device_node *node, struct device_node * domain = irq_domain_create_hierarchy(parent_domain, 0, ctl->params->nr_hwirq, - of_node_to_fwnode(node), + of_fwnode_handle(node), &meson_gpio_irq_domain_ops, ctl); if (!domain) { @@ -599,9 +642,9 @@ free_ctl: } IRQCHIP_PLATFORM_DRIVER_BEGIN(meson_gpio_intc) -IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_of_init) +IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_probe) IRQCHIP_PLATFORM_DRIVER_END(meson_gpio_intc) MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); +MODULE_DESCRIPTION("Meson GPIO Interrupt Multiplexer driver"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:meson-gpio-intc"); diff --git a/drivers/irqchip/irq-mips-cpu.c b/drivers/irqchip/irq-mips-cpu.c index 0c7ae71a0af0..ac784ef3ed4b 100644 --- a/drivers/irqchip/irq-mips-cpu.c +++ b/drivers/irqchip/irq-mips-cpu.c @@ -238,11 +238,9 @@ static void mips_cpu_register_ipi_domain(struct device_node *of_node) struct cpu_ipi_domain_state *ipi_domain_state; ipi_domain_state = kzalloc(sizeof(*ipi_domain_state), GFP_KERNEL); - ipi_domain = irq_domain_add_hierarchy(irq_domain, - IRQ_DOMAIN_FLAG_IPI_SINGLE, - 2, of_node, - &mips_cpu_ipi_chip_ops, - ipi_domain_state); + ipi_domain = irq_domain_create_hierarchy(irq_domain, IRQ_DOMAIN_FLAG_IPI_SINGLE, 2, + of_fwnode_handle(of_node), + &mips_cpu_ipi_chip_ops, ipi_domain_state); if (!ipi_domain) panic("Failed to add MIPS CPU IPI domain"); irq_domain_update_bus_token(ipi_domain, DOMAIN_BUS_IPI); @@ -260,9 +258,8 @@ static void __init __mips_cpu_irq_init(struct device_node *of_node) clear_c0_status(ST0_IM); clear_c0_cause(CAUSEF_IP); - irq_domain = irq_domain_add_legacy(of_node, 8, MIPS_CPU_IRQ_BASE, 0, - &mips_cpu_intc_irq_domain_ops, - NULL); + irq_domain = irq_domain_create_legacy(of_fwnode_handle(of_node), 8, MIPS_CPU_IRQ_BASE, 0, + &mips_cpu_intc_irq_domain_ops, NULL); if (!irq_domain) panic("Failed to add irqdomain for MIPS CPU"); diff --git a/drivers/irqchip/irq-mips-gic.c b/drivers/irqchip/irq-mips-gic.c index 76253e864f23..19a57c5e2b2e 100644 --- a/drivers/irqchip/irq-mips-gic.c +++ b/drivers/irqchip/irq-mips-gic.c @@ -66,6 +66,87 @@ static struct gic_all_vpes_chip_data { bool mask; } gic_all_vpes_chip_data[GIC_NUM_LOCAL_INTRS]; +static int __gic_with_next_online_cpu(int prev) +{ + unsigned int cpu; + + /* Discover the next online CPU */ + cpu = cpumask_next(prev, cpu_online_mask); + + /* If there isn't one, we're done */ + if (cpu >= nr_cpu_ids) + return cpu; + + /* + * Move the access lock to the next CPU's GIC local register block. + * + * Set GIC_VL_OTHER. Since the caller holds gic_lock nothing can + * clobber the written value. + */ + write_gic_vl_other(mips_cm_vp_id(cpu)); + + return cpu; +} + +static inline void gic_unlock_cluster(void) +{ + if (mips_cps_multicluster_cpus()) + mips_cm_unlock_other(); +} + +/** + * for_each_online_cpu_gic() - Iterate over online CPUs, access local registers + * @cpu: An integer variable to hold the current CPU number + * @gic_lock: A pointer to raw spin lock used as a guard + * + * Iterate over online CPUs & configure the other/redirect register region to + * access each CPUs GIC local register block, which can be accessed from the + * loop body using read_gic_vo_*() or write_gic_vo_*() accessor functions or + * their derivatives. + */ +#define for_each_online_cpu_gic(cpu, gic_lock) \ + guard(raw_spinlock_irqsave)(gic_lock); \ + for ((cpu) = __gic_with_next_online_cpu(-1); \ + (cpu) < nr_cpu_ids; \ + gic_unlock_cluster(), \ + (cpu) = __gic_with_next_online_cpu(cpu)) + +/** + * gic_irq_lock_cluster() - Lock redirect block access to IRQ's cluster + * @d: struct irq_data corresponding to the interrupt we're interested in + * + * Locks redirect register block access to the global register block of the GIC + * within the remote cluster that the IRQ corresponding to @d is affine to, + * returning true when this redirect block setup & locking has been performed. + * + * If @d is affine to the local cluster then no locking is performed and this + * function will return false, indicating to the caller that it should access + * the local clusters registers without the overhead of indirection through the + * redirect block. + * + * In summary, if this function returns true then the caller should access GIC + * registers using redirect register block accessors & then call + * mips_cm_unlock_other() when done. If this function returns false then the + * caller should trivially access GIC registers in the local cluster. + * + * Returns true if locking performed, else false. + */ +static bool gic_irq_lock_cluster(struct irq_data *d) +{ + unsigned int cpu, cl; + + cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); + BUG_ON(cpu >= NR_CPUS); + + cl = cpu_cluster(&cpu_data[cpu]); + if (cl == cpu_cluster(¤t_cpu_data)) + return false; + if (mips_cps_numcores(cl) == 0) + return false; + mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + return true; +} + static void gic_clear_pcpu_masks(unsigned int intr) { unsigned int i; @@ -112,7 +193,12 @@ static void gic_send_ipi(struct irq_data *d, unsigned int cpu) { irq_hw_number_t hwirq = GIC_HWIRQ_TO_SHARED(irqd_to_hwirq(d)); - write_gic_wedge(GIC_WEDGE_RW | hwirq); + if (gic_irq_lock_cluster(d)) { + write_gic_redir_wedge(GIC_WEDGE_RW | hwirq); + mips_cm_unlock_other(); + } else { + write_gic_wedge(GIC_WEDGE_RW | hwirq); + } } int gic_get_c0_compare_int(void) @@ -180,7 +266,13 @@ static void gic_mask_irq(struct irq_data *d) { unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); - write_gic_rmask(intr); + if (gic_irq_lock_cluster(d)) { + write_gic_redir_rmask(intr); + mips_cm_unlock_other(); + } else { + write_gic_rmask(intr); + } + gic_clear_pcpu_masks(intr); } @@ -189,7 +281,12 @@ static void gic_unmask_irq(struct irq_data *d) unsigned int intr = GIC_HWIRQ_TO_SHARED(d->hwirq); unsigned int cpu; - write_gic_smask(intr); + if (gic_irq_lock_cluster(d)) { + write_gic_redir_smask(intr); + mips_cm_unlock_other(); + } else { + write_gic_smask(intr); + } gic_clear_pcpu_masks(intr); cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); @@ -200,7 +297,12 @@ static void gic_ack_irq(struct irq_data *d) { unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); - write_gic_wedge(irq); + if (gic_irq_lock_cluster(d)) { + write_gic_redir_wedge(irq); + mips_cm_unlock_other(); + } else { + write_gic_wedge(irq); + } } static int gic_set_type(struct irq_data *d, unsigned int type) @@ -240,9 +342,16 @@ static int gic_set_type(struct irq_data *d, unsigned int type) break; } - change_gic_pol(irq, pol); - change_gic_trig(irq, trig); - change_gic_dual(irq, dual); + if (gic_irq_lock_cluster(d)) { + change_gic_redir_pol(irq, pol); + change_gic_redir_trig(irq, trig); + change_gic_redir_dual(irq, dual); + mips_cm_unlock_other(); + } else { + change_gic_pol(irq, pol); + change_gic_trig(irq, trig); + change_gic_dual(irq, dual); + } if (trig == GIC_TRIG_EDGE) irq_set_chip_handler_name_locked(d, &gic_edge_irq_controller, @@ -260,25 +369,76 @@ static int gic_set_affinity(struct irq_data *d, const struct cpumask *cpumask, bool force) { unsigned int irq = GIC_HWIRQ_TO_SHARED(d->hwirq); + unsigned int cpu, cl, old_cpu, old_cl; unsigned long flags; - unsigned int cpu; - cpu = cpumask_first_and(cpumask, cpu_online_mask); + /* + * The GIC specifies that we can only route an interrupt to one VP(E), + * ie. CPU in Linux parlance, at a time. Therefore we always route to + * the first forced or online CPU in the mask. + */ + if (force) + cpu = cpumask_first(cpumask); + else + cpu = cpumask_first_and(cpumask, cpu_online_mask); + if (cpu >= NR_CPUS) return -EINVAL; - /* Assumption : cpumask refers to a single CPU */ - raw_spin_lock_irqsave(&gic_lock, flags); + old_cpu = cpumask_first(irq_data_get_effective_affinity_mask(d)); + old_cl = cpu_cluster(&cpu_data[old_cpu]); + cl = cpu_cluster(&cpu_data[cpu]); - /* Re-route this IRQ */ - write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu))); + raw_spin_lock_irqsave(&gic_lock, flags); - /* Update the pcpu_masks */ - gic_clear_pcpu_masks(irq); - if (read_gic_mask(irq)) - set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); + /* + * If we're moving affinity between clusters, stop routing the + * interrupt to any VP(E) in the old cluster. + */ + if (cl != old_cl) { + if (gic_irq_lock_cluster(d)) { + write_gic_redir_map_vp(irq, 0); + mips_cm_unlock_other(); + } else { + write_gic_map_vp(irq, 0); + } + } + /* + * Update effective affinity - after this gic_irq_lock_cluster() will + * begin operating on the new cluster. + */ irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + /* + * If we're moving affinity between clusters, configure the interrupt + * trigger type in the new cluster. + */ + if (cl != old_cl) + gic_set_type(d, irqd_get_trigger_type(d)); + + /* Route the interrupt to its new VP(E) */ + if (gic_irq_lock_cluster(d)) { + write_gic_redir_map_pin(irq, + GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); + write_gic_redir_map_vp(irq, BIT(mips_cm_vp_id(cpu))); + + /* Update the pcpu_masks */ + gic_clear_pcpu_masks(irq); + if (read_gic_redir_mask(irq)) + set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); + + mips_cm_unlock_other(); + } else { + write_gic_map_pin(irq, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); + write_gic_map_vp(irq, BIT(mips_cm_vp_id(cpu))); + + /* Update the pcpu_masks */ + gic_clear_pcpu_masks(irq); + if (read_gic_mask(irq)) + set_bit(irq, per_cpu_ptr(pcpu_masks, cpu)); + } + raw_spin_unlock_irqrestore(&gic_lock, flags); return IRQ_SET_MASK_OK; @@ -350,37 +510,33 @@ static struct irq_chip gic_local_irq_controller = { static void gic_mask_local_irq_all_vpes(struct irq_data *d) { struct gic_all_vpes_chip_data *cd; - unsigned long flags; int intr, cpu; + if (!mips_cps_multicluster_cpus()) + return; + intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); cd = irq_data_get_irq_chip_data(d); cd->mask = false; - raw_spin_lock_irqsave(&gic_lock, flags); - for_each_online_cpu(cpu) { - write_gic_vl_other(mips_cm_vp_id(cpu)); + for_each_online_cpu_gic(cpu, &gic_lock) write_gic_vo_rmask(BIT(intr)); - } - raw_spin_unlock_irqrestore(&gic_lock, flags); } static void gic_unmask_local_irq_all_vpes(struct irq_data *d) { struct gic_all_vpes_chip_data *cd; - unsigned long flags; int intr, cpu; + if (!mips_cps_multicluster_cpus()) + return; + intr = GIC_HWIRQ_TO_LOCAL(d->hwirq); cd = irq_data_get_irq_chip_data(d); cd->mask = true; - raw_spin_lock_irqsave(&gic_lock, flags); - for_each_online_cpu(cpu) { - write_gic_vl_other(mips_cm_vp_id(cpu)); + for_each_online_cpu_gic(cpu, &gic_lock) write_gic_vo_smask(BIT(intr)); - } - raw_spin_unlock_irqrestore(&gic_lock, flags); } static void gic_all_vpes_irq_cpu_online(void) @@ -436,11 +592,21 @@ static int gic_shared_irq_domain_map(struct irq_domain *d, unsigned int virq, unsigned long flags; data = irq_get_irq_data(virq); + irq_data_update_effective_affinity(data, cpumask_of(cpu)); raw_spin_lock_irqsave(&gic_lock, flags); - write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); - write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu))); - irq_data_update_effective_affinity(data, cpumask_of(cpu)); + + /* Route the interrupt to its VP(E) */ + if (gic_irq_lock_cluster(data)) { + write_gic_redir_map_pin(intr, + GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); + write_gic_redir_map_vp(intr, BIT(mips_cm_vp_id(cpu))); + mips_cm_unlock_other(); + } else { + write_gic_map_pin(intr, GIC_MAP_PIN_MAP_TO_PIN | gic_cpu_pin); + write_gic_map_vp(intr, BIT(mips_cm_vp_id(cpu))); + } + raw_spin_unlock_irqrestore(&gic_lock, flags); return 0; @@ -469,7 +635,6 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq, irq_hw_number_t hwirq) { struct gic_all_vpes_chip_data *cd; - unsigned long flags; unsigned int intr; int err, cpu; u32 map; @@ -533,12 +698,10 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int virq, if (!gic_local_irq_is_routable(intr)) return -EPERM; - raw_spin_lock_irqsave(&gic_lock, flags); - for_each_online_cpu(cpu) { - write_gic_vl_other(mips_cm_vp_id(cpu)); - write_gic_vo_map(mips_gic_vx_map_reg(intr), map); + if (mips_cps_multicluster_cpus()) { + for_each_online_cpu_gic(cpu, &gic_lock) + write_gic_vo_map(mips_gic_vx_map_reg(intr), map); } - raw_spin_unlock_irqrestore(&gic_lock, flags); return 0; } @@ -621,6 +784,9 @@ static int gic_ipi_domain_alloc(struct irq_domain *d, unsigned int virq, if (ret) goto error; + /* Set affinity to cpu. */ + irq_data_update_effective_affinity(irq_get_irq_data(virq + i), + cpumask_of(cpu)); ret = irq_set_irq_type(virq + i, IRQ_TYPE_EDGE_RISING); if (ret) goto error; @@ -679,10 +845,10 @@ static int gic_register_ipi_domain(struct device_node *node) struct irq_domain *gic_ipi_domain; unsigned int v[2], num_ipis; - gic_ipi_domain = irq_domain_add_hierarchy(gic_irq_domain, - IRQ_DOMAIN_FLAG_IPI_PER_CPU, - GIC_NUM_LOCAL_INTRS + gic_shared_intrs, - node, &gic_ipi_domain_ops, NULL); + gic_ipi_domain = irq_domain_create_hierarchy(gic_irq_domain, IRQ_DOMAIN_FLAG_IPI_PER_CPU, + GIC_NUM_LOCAL_INTRS + gic_shared_intrs, + of_fwnode_handle(node), &gic_ipi_domain_ops, + NULL); if (!gic_ipi_domain) { pr_err("Failed to add IPI domain"); return -ENXIO; @@ -734,7 +900,7 @@ static int gic_cpu_startup(unsigned int cpu) static int __init gic_of_init(struct device_node *node, struct device_node *parent) { - unsigned int cpu_vec, i, gicconfig; + unsigned int cpu_vec, i, gicconfig, cl, nclusters; unsigned long reserved; phys_addr_t gic_base; struct resource res; @@ -801,9 +967,10 @@ static int __init gic_of_init(struct device_node *node, gic_irq_dispatch); } - gic_irq_domain = irq_domain_add_simple(node, GIC_NUM_LOCAL_INTRS + - gic_shared_intrs, 0, - &gic_irq_domain_ops, NULL); + gic_irq_domain = irq_domain_create_simple(of_fwnode_handle(node), + GIC_NUM_LOCAL_INTRS + + gic_shared_intrs, 0, + &gic_irq_domain_ops, NULL); if (!gic_irq_domain) { pr_err("Failed to add IRQ domain"); return -ENXIO; @@ -815,11 +982,32 @@ static int __init gic_of_init(struct device_node *node, board_bind_eic_interrupt = &gic_bind_eic_interrupt; - /* Setup defaults */ - for (i = 0; i < gic_shared_intrs; i++) { - change_gic_pol(i, GIC_POL_ACTIVE_HIGH); - change_gic_trig(i, GIC_TRIG_LEVEL); - write_gic_rmask(i); + /* + * Initialise each cluster's GIC shared registers to sane default + * values. + * Otherwise, the IPI set up will be erased if we move code + * to gic_cpu_startup for each cpu. + */ + nclusters = mips_cps_numclusters(); + for (cl = 0; cl < nclusters; cl++) { + if (cl == cpu_cluster(¤t_cpu_data)) { + for (i = 0; i < gic_shared_intrs; i++) { + change_gic_pol(i, GIC_POL_ACTIVE_HIGH); + change_gic_trig(i, GIC_TRIG_LEVEL); + write_gic_rmask(i); + } + } else if (mips_cps_numcores(cl) != 0) { + mips_cm_lock_other(cl, 0, 0, CM_GCR_Cx_OTHER_BLOCK_GLOBAL); + for (i = 0; i < gic_shared_intrs; i++) { + change_gic_redir_pol(i, GIC_POL_ACTIVE_HIGH); + change_gic_redir_trig(i, GIC_TRIG_LEVEL); + write_gic_redir_rmask(i); + } + mips_cm_unlock_other(); + + } else { + pr_warn("No CPU cores on the cluster %d skip it\n", cl); + } } return cpuhp_setup_state(CPUHP_AP_IRQ_MIPS_GIC_STARTING, diff --git a/drivers/irqchip/irq-mmp.c b/drivers/irqchip/irq-mmp.c index 25cf4f80e767..09e640430208 100644 --- a/drivers/irqchip/irq-mmp.c +++ b/drivers/irqchip/irq-mmp.c @@ -261,9 +261,9 @@ static int __init mmp_init_bases(struct device_node *node) } icu_data[0].virq_base = 0; - icu_data[0].domain = irq_domain_add_linear(node, nr_irqs, - &mmp_irq_domain_ops, - &icu_data[0]); + icu_data[0].domain = irq_domain_create_linear(of_fwnode_handle(node), nr_irqs, + &mmp_irq_domain_ops, + &icu_data[0]); for (irq = 0; irq < nr_irqs; irq++) { ret = irq_create_mapping(icu_data[0].domain, irq); if (!ret) { @@ -391,9 +391,9 @@ static int __init mmp2_mux_of_init(struct device_node *node, return -EINVAL; icu_data[i].virq_base = 0; - icu_data[i].domain = irq_domain_add_linear(node, nr_irqs, - &mmp_irq_domain_ops, - &icu_data[i]); + icu_data[i].domain = irq_domain_create_linear(of_fwnode_handle(node), nr_irqs, + &mmp_irq_domain_ops, + &icu_data[i]); for (irq = 0; irq < nr_irqs; irq++) { ret = irq_create_mapping(icu_data[i].domain, irq); if (!ret) { diff --git a/drivers/irqchip/irq-mscc-ocelot.c b/drivers/irqchip/irq-mscc-ocelot.c index 4d0c3532dbe7..8cbc191f750b 100644 --- a/drivers/irqchip/irq-mscc-ocelot.c +++ b/drivers/irqchip/irq-mscc-ocelot.c @@ -37,7 +37,7 @@ static struct chip_props ocelot_props = { .reg_off_ena_clr = 0x1c, .reg_off_ena_set = 0x20, .reg_off_ident = 0x38, - .reg_off_trigger = 0x5c, + .reg_off_trigger = 0x4, .n_irq = 24, }; @@ -70,7 +70,7 @@ static struct chip_props jaguar2_props = { .reg_off_ena_clr = 0x1c, .reg_off_ena_set = 0x20, .reg_off_ident = 0x38, - .reg_off_trigger = 0x5c, + .reg_off_trigger = 0x4, .n_irq = 29, }; @@ -83,7 +83,13 @@ static void ocelot_irq_unmask(struct irq_data *data) unsigned int mask = data->mask; u32 val; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); + /* + * Clear sticky bits for edge mode interrupts. + * Serval has only one trigger register replication, but the adjacent + * register is always read as zero, so there's no need to handle this + * case separately. + */ val = irq_reg_readl(gc, ICPU_CFG_INTR_INTR_TRIGGER(p, 0)) | irq_reg_readl(gc, ICPU_CFG_INTR_INTR_TRIGGER(p, 1)); if (!(val & mask)) @@ -91,7 +97,6 @@ static void ocelot_irq_unmask(struct irq_data *data) *ct->mask_cache &= ~mask; irq_reg_writel(gc, mask, p->reg_off_ena_set); - irq_gc_unlock(gc); } static void ocelot_irq_handler(struct irq_desc *desc) @@ -126,8 +131,8 @@ static int __init vcoreiii_irq_init(struct device_node *node, if (!parent_irq) return -EINVAL; - domain = irq_domain_add_linear(node, p->n_irq, - &irq_generic_chip_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(node), p->n_irq, + &irq_generic_chip_ops, NULL); if (!domain) { pr_err("%pOFn: unable to add irq domain\n", node); return -ENOMEM; diff --git a/drivers/irqchip/irq-msi-lib.c b/drivers/irqchip/irq-msi-lib.c new file mode 100644 index 000000000000..d5eefc3d7215 --- /dev/null +++ b/drivers/irqchip/irq-msi-lib.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2022 Linutronix GmbH +// Copyright (C) 2022 Intel + +#include <linux/export.h> + +#include <linux/irqchip/irq-msi-lib.h> + +/** + * msi_lib_init_dev_msi_info - Domain info setup for MSI domains + * @dev: The device for which the domain is created for + * @domain: The domain providing this callback + * @real_parent: The real parent domain of the domain to be initialized + * which might be a domain built on top of @domain or + * @domain itself + * @info: The domain info for the domain to be initialize + * + * This function is to be used for all types of MSI domains above the root + * parent domain and any intermediates. The topmost parent domain specific + * functionality is determined via @real_parent. + * + * All intermediate domains between the root and the device domain must + * have either msi_parent_ops.init_dev_msi_info = msi_parent_init_dev_msi_info + * or invoke it down the line. + */ +bool msi_lib_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, + struct msi_domain_info *info) +{ + const struct msi_parent_ops *pops = real_parent->msi_parent_ops; + struct irq_chip *chip = info->chip; + u32 required_flags; + + /* Parent ops available? */ + if (WARN_ON_ONCE(!pops)) + return false; + + /* + * MSI parent domain specific settings. For now there is only the + * root parent domain, e.g. NEXUS, acting as a MSI parent, but it is + * possible to stack MSI parents. See x86 vector -> irq remapping + */ + if (domain->bus_token == pops->bus_select_token) { + if (WARN_ON_ONCE(domain != real_parent)) + return false; + } else { + WARN_ON_ONCE(1); + return false; + } + + required_flags = pops->required_flags; + + /* Is the target domain bus token supported? */ + switch(info->bus_token) { + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_PCI_MSI))) + return false; + + break; + case DOMAIN_BUS_DEVICE_MSI: + /* + * Per device MSI should never have any MSI feature bits + * set. It's sole purpose is to create a dumb interrupt + * chip which has a device specific irq_write_msi_msg() + * callback. + */ + if (WARN_ON_ONCE(info->flags)) + return false; + + /* Core managed MSI descriptors */ + info->flags = MSI_FLAG_ALLOC_SIMPLE_MSI_DESCS | MSI_FLAG_FREE_MSI_DESCS; + fallthrough; + case DOMAIN_BUS_WIRED_TO_MSI: + /* Remove PCI specific flags */ + required_flags &= ~MSI_FLAG_PCI_MSI_MASK_PARENT; + break; + default: + /* + * This should never be reached. See + * msi_lib_irq_domain_select() + */ + WARN_ON_ONCE(1); + return false; + } + + /* + * Mask out the domain specific MSI feature flags which are not + * supported by the real parent. + */ + info->flags &= pops->supported_flags; + /* Enforce the required flags */ + info->flags |= required_flags; + + /* Chip updates for all child bus types */ + if (!chip->irq_eoi && (pops->chip_flags & MSI_CHIP_FLAG_SET_EOI)) + chip->irq_eoi = irq_chip_eoi_parent; + if (!chip->irq_ack && (pops->chip_flags & MSI_CHIP_FLAG_SET_ACK)) + chip->irq_ack = irq_chip_ack_parent; + + /* + * The device MSI domain can never have a set affinity callback. It + * always has to rely on the parent domain to handle affinity + * settings. The device MSI domain just has to write the resulting + * MSI message into the hardware which is the whole purpose of the + * device MSI domain aside of mask/unmask which is provided e.g. by + * PCI/MSI device domains. + * + * The exception to the rule is when the underlying domain + * tells you that affinity is not a thing -- for example when + * everything is muxed behind a single interrupt. + */ + if (!chip->irq_set_affinity && !(info->flags & MSI_FLAG_NO_AFFINITY)) + chip->irq_set_affinity = msi_domain_set_affinity; + + /* + * If the parent domain insists on being in charge of masking, obey + * blindly. The interrupt is un-masked at the PCI level on startup + * and masked on shutdown to prevent rogue interrupts after the + * driver freed the interrupt. Not masking it at the PCI level + * speeds up operation for disable/enable_irq() as it avoids + * getting all the way out to the PCI device. + */ + if (info->flags & MSI_FLAG_PCI_MSI_MASK_PARENT) { + chip->irq_mask = irq_chip_mask_parent; + chip->irq_unmask = irq_chip_unmask_parent; + } + + return true; +} +EXPORT_SYMBOL_GPL(msi_lib_init_dev_msi_info); + +/** + * msi_lib_irq_domain_select - Shared select function for NEXUS domains + * @d: Pointer to the irq domain on which select is invoked + * @fwspec: Firmware spec describing what is searched + * @bus_token: The bus token for which a matching irq domain is looked up + * + * Returns: %0 if @d is not what is being looked for + * + * %1 if @d is either the domain which is directly searched for or + * if @d is providing the parent MSI domain for the functionality + * requested with @bus_token. + */ +int msi_lib_irq_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, + enum irq_domain_bus_token bus_token) +{ + const struct msi_parent_ops *ops = d->msi_parent_ops; + u32 busmask = BIT(bus_token); + + if (!ops) + return 0; + + struct fwnode_handle *fwh __free(fwnode_handle) = + d->flags & IRQ_DOMAIN_FLAG_FWNODE_PARENT ? fwnode_get_parent(fwspec->fwnode) + : fwnode_handle_get(fwspec->fwnode); + if (fwh != d->fwnode || fwspec->param_count != 0) + return 0; + + /* Handle pure domain searches */ + if (bus_token == ops->bus_select_token) + return 1; + + return !!(ops->bus_select_mask & busmask); +} +EXPORT_SYMBOL_GPL(msi_lib_irq_domain_select); diff --git a/drivers/irqchip/irq-mst-intc.c b/drivers/irqchip/irq-mst-intc.c index f6133ae28155..7f760f555a76 100644 --- a/drivers/irqchip/irq-mst-intc.c +++ b/drivers/irqchip/irq-mst-intc.c @@ -143,7 +143,7 @@ static void mst_intc_polarity_restore(struct mst_intc_chip_data *cd) writew_relaxed(cd->saved_polarity_conf[i], addr + i * 4); } -static void mst_irq_resume(void) +static void mst_irq_resume(void *data) { struct mst_intc_chip_data *cd; @@ -151,7 +151,7 @@ static void mst_irq_resume(void) mst_intc_polarity_restore(cd); } -static int mst_irq_suspend(void) +static int mst_irq_suspend(void *data) { struct mst_intc_chip_data *cd; @@ -160,14 +160,18 @@ static int mst_irq_suspend(void) return 0; } -static struct syscore_ops mst_irq_syscore_ops = { +static const struct syscore_ops mst_irq_syscore_ops = { .suspend = mst_irq_suspend, .resume = mst_irq_resume, }; +static struct syscore mst_irq_syscore = { + .ops = &mst_irq_syscore_ops, +}; + static int __init mst_irq_pm_init(void) { - register_syscore_ops(&mst_irq_syscore_ops); + register_syscore(&mst_irq_syscore); return 0; } late_initcall(mst_irq_pm_init); @@ -273,8 +277,8 @@ static int __init mst_intc_of_init(struct device_node *dn, raw_spin_lock_init(&cd->lock); cd->irq_start = irq_start; cd->nr_irqs = irq_end - irq_start + 1; - domain = irq_domain_add_hierarchy(domain_parent, 0, cd->nr_irqs, dn, - &mst_intc_domain_ops, cd); + domain = irq_domain_create_hierarchy(domain_parent, 0, cd->nr_irqs, of_fwnode_handle(dn), + &mst_intc_domain_ops, cd); if (!domain) { iounmap(cd->base); kfree(cd); diff --git a/drivers/irqchip/irq-mtk-cirq.c b/drivers/irqchip/irq-mtk-cirq.c index 76bc0283e3b9..9571f622774e 100644 --- a/drivers/irqchip/irq-mtk-cirq.c +++ b/drivers/irqchip/irq-mtk-cirq.c @@ -199,7 +199,7 @@ static const struct irq_domain_ops cirq_domain_ops = { }; #ifdef CONFIG_PM_SLEEP -static int mtk_cirq_suspend(void) +static int mtk_cirq_suspend(void *data) { void __iomem *reg; u32 value, mask; @@ -257,7 +257,7 @@ static int mtk_cirq_suspend(void) return 0; } -static void mtk_cirq_resume(void) +static void mtk_cirq_resume(void *data) { void __iomem *reg = mtk_cirq_reg(cirq_data, CIRQ_CONTROL); u32 value; @@ -272,14 +272,18 @@ static void mtk_cirq_resume(void) writel_relaxed(value, reg); } -static struct syscore_ops mtk_cirq_syscore_ops = { +static const struct syscore_ops mtk_cirq_syscore_ops = { .suspend = mtk_cirq_suspend, .resume = mtk_cirq_resume, }; +static struct syscore mtk_cirq_syscore = { + .ops = &mtk_cirq_syscore_ops, +}; + static void mtk_cirq_syscore_init(void) { - register_syscore_ops(&mtk_cirq_syscore_ops); + register_syscore(&mtk_cirq_syscore); } #else static inline void mtk_cirq_syscore_init(void) {} @@ -336,9 +340,8 @@ static int __init mtk_cirq_of_init(struct device_node *node, cirq_data->offsets = match->data; irq_num = cirq_data->ext_irq_end - cirq_data->ext_irq_start + 1; - domain = irq_domain_add_hierarchy(domain_parent, 0, - irq_num, node, - &cirq_domain_ops, cirq_data); + domain = irq_domain_create_hierarchy(domain_parent, 0, irq_num, of_fwnode_handle(node), + &cirq_domain_ops, cirq_data); if (!domain) { ret = -ENOMEM; goto out_unmap; diff --git a/drivers/irqchip/irq-mtk-sysirq.c b/drivers/irqchip/irq-mtk-sysirq.c index 586e52d5442b..6895e7096b27 100644 --- a/drivers/irqchip/irq-mtk-sysirq.c +++ b/drivers/irqchip/irq-mtk-sysirq.c @@ -207,8 +207,8 @@ static int __init mtk_sysirq_of_init(struct device_node *node, chip_data->which_word[i] = word; } - domain = irq_domain_add_hierarchy(domain_parent, 0, intpol_num, node, - &sysirq_domain_ops, chip_data); + domain = irq_domain_create_hierarchy(domain_parent, 0, intpol_num, of_fwnode_handle(node), + &sysirq_domain_ops, chip_data); if (!domain) { ret = -ENOMEM; goto out_free_which_word; diff --git a/drivers/irqchip/irq-mvebu-gicp.c b/drivers/irqchip/irq-mvebu-gicp.c index c43a345061d5..667bde3c651f 100644 --- a/drivers/irqchip/irq-mvebu-gicp.c +++ b/drivers/irqchip/irq-mvebu-gicp.c @@ -17,6 +17,8 @@ #include <linux/of_platform.h> #include <linux/platform_device.h> +#include <linux/irqchip/irq-msi-lib.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> #define GICP_SETSPI_NSR_OFFSET 0x0 @@ -145,32 +147,37 @@ static void gicp_irq_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops gicp_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = gicp_irq_domain_alloc, .free = gicp_irq_domain_free, }; -static struct irq_chip gicp_msi_irq_chip = { - .name = "GICP", - .irq_set_type = irq_chip_set_type_parent, - .flags = IRQCHIP_SUPPORTS_LEVEL_MSI, -}; +#define GICP_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) -static struct msi_domain_ops gicp_msi_ops = { -}; +#define GICP_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_LEVEL_CAPABLE) -static struct msi_domain_info gicp_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS | - MSI_FLAG_LEVEL_CAPABLE), - .ops = &gicp_msi_ops, - .chip = &gicp_msi_irq_chip, +static const struct msi_parent_ops gicp_msi_parent_ops = { + .supported_flags = GICP_MSI_FLAGS_SUPPORTED, + .required_flags = GICP_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_GENERIC_MSI, + .bus_select_mask = MATCH_PLATFORM_MSI, + .prefix = "GICP-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, }; static int mvebu_gicp_probe(struct platform_device *pdev) { - struct mvebu_gicp *gicp; - struct irq_domain *inner_domain, *plat_domain, *parent_domain; struct device_node *node = pdev->dev.of_node; struct device_node *irq_parent_dn; + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &gicp_domain_ops, + }; + struct mvebu_gicp *gicp; + void __iomem *base; int ret, i; gicp = devm_kzalloc(&pdev->dev, sizeof(*gicp), GFP_KERNEL); @@ -214,38 +221,32 @@ static int mvebu_gicp_probe(struct platform_device *pdev) if (!gicp->spi_bitmap) return -ENOMEM; + info.size = gicp->spi_cnt; + info.host_data = gicp; + irq_parent_dn = of_irq_find_parent(node); if (!irq_parent_dn) { dev_err(&pdev->dev, "failed to find parent IRQ node\n"); return -ENODEV; } - parent_domain = irq_find_host(irq_parent_dn); + info.parent = irq_find_host(irq_parent_dn); of_node_put(irq_parent_dn); - if (!parent_domain) { + if (!info.parent) { dev_err(&pdev->dev, "failed to find parent IRQ domain\n"); return -ENODEV; } - inner_domain = irq_domain_create_hierarchy(parent_domain, 0, - gicp->spi_cnt, - of_node_to_fwnode(node), - &gicp_domain_ops, gicp); - if (!inner_domain) - return -ENOMEM; - - - plat_domain = platform_msi_create_irq_domain(of_node_to_fwnode(node), - &gicp_msi_domain_info, - inner_domain); - if (!plat_domain) { - irq_domain_remove(inner_domain); - return -ENOMEM; + base = ioremap(gicp->res->start, resource_size(gicp->res)); + if (!base) { + dev_err(&pdev->dev, "ioremap() failed. Unable to clear pending interrupts.\n"); + } else { + for (i = 0; i < 64; i++) + writel(i, base + GICP_CLRSPI_NSR_OFFSET); + iounmap(base); } - platform_set_drvdata(pdev, gicp); - - return 0; + return msi_create_parent_irq_domain(&info, &gicp_msi_parent_ops) ? 0 : -ENOMEM; } static const struct of_device_id mvebu_gicp_of_match[] = { diff --git a/drivers/irqchip/irq-mvebu-icu.c b/drivers/irqchip/irq-mvebu-icu.c index 3c77acc7ec6a..db5dbc6e88b0 100644 --- a/drivers/irqchip/irq-mvebu-icu.c +++ b/drivers/irqchip/irq-mvebu-icu.c @@ -20,6 +20,8 @@ #include <linux/of_platform.h> #include <linux/platform_device.h> +#include <linux/irqchip/irq-msi-lib.h> + #include <dt-bindings/interrupt-controller/mvebu-icu.h> /* ICU registers */ @@ -60,99 +62,14 @@ struct mvebu_icu_msi_data { const struct mvebu_icu_subset_data *subset_data; }; -struct mvebu_icu_irq_data { - struct mvebu_icu *icu; - unsigned int icu_group; - unsigned int type; -}; - static DEFINE_STATIC_KEY_FALSE(legacy_bindings); -static void mvebu_icu_init(struct mvebu_icu *icu, - struct mvebu_icu_msi_data *msi_data, - struct msi_msg *msg) -{ - const struct mvebu_icu_subset_data *subset = msi_data->subset_data; - - if (atomic_cmpxchg(&msi_data->initialized, false, true)) - return; - - /* Set 'SET' ICU SPI message address in AP */ - writel_relaxed(msg[0].address_hi, icu->base + subset->offset_set_ah); - writel_relaxed(msg[0].address_lo, icu->base + subset->offset_set_al); - - if (subset->icu_group != ICU_GRP_NSR) - return; - - /* Set 'CLEAR' ICU SPI message address in AP (level-MSI only) */ - writel_relaxed(msg[1].address_hi, icu->base + subset->offset_clr_ah); - writel_relaxed(msg[1].address_lo, icu->base + subset->offset_clr_al); -} - -static void mvebu_icu_write_msg(struct msi_desc *desc, struct msi_msg *msg) -{ - struct irq_data *d = irq_get_irq_data(desc->irq); - struct mvebu_icu_msi_data *msi_data = platform_msi_get_host_data(d->domain); - struct mvebu_icu_irq_data *icu_irqd = d->chip_data; - struct mvebu_icu *icu = icu_irqd->icu; - unsigned int icu_int; - - if (msg->address_lo || msg->address_hi) { - /* One off initialization per domain */ - mvebu_icu_init(icu, msi_data, msg); - /* Configure the ICU with irq number & type */ - icu_int = msg->data | ICU_INT_ENABLE; - if (icu_irqd->type & IRQ_TYPE_EDGE_RISING) - icu_int |= ICU_IS_EDGE; - icu_int |= icu_irqd->icu_group << ICU_GROUP_SHIFT; - } else { - /* De-configure the ICU */ - icu_int = 0; - } - - writel_relaxed(icu_int, icu->base + ICU_INT_CFG(d->hwirq)); - - /* - * The SATA unit has 2 ports, and a dedicated ICU entry per - * port. The ahci sata driver supports only one irq interrupt - * per SATA unit. To solve this conflict, we configure the 2 - * SATA wired interrupts in the south bridge into 1 GIC - * interrupt in the north bridge. Even if only a single port - * is enabled, if sata node is enabled, both interrupts are - * configured (regardless of which port is actually in use). - */ - if (d->hwirq == ICU_SATA0_ICU_ID || d->hwirq == ICU_SATA1_ICU_ID) { - writel_relaxed(icu_int, - icu->base + ICU_INT_CFG(ICU_SATA0_ICU_ID)); - writel_relaxed(icu_int, - icu->base + ICU_INT_CFG(ICU_SATA1_ICU_ID)); - } -} - -static struct irq_chip mvebu_icu_nsr_chip = { - .name = "ICU-NSR", - .irq_mask = irq_chip_mask_parent, - .irq_unmask = irq_chip_unmask_parent, - .irq_eoi = irq_chip_eoi_parent, - .irq_set_type = irq_chip_set_type_parent, - .irq_set_affinity = irq_chip_set_affinity_parent, -}; - -static struct irq_chip mvebu_icu_sei_chip = { - .name = "ICU-SEI", - .irq_ack = irq_chip_ack_parent, - .irq_mask = irq_chip_mask_parent, - .irq_unmask = irq_chip_unmask_parent, - .irq_set_type = irq_chip_set_type_parent, - .irq_set_affinity = irq_chip_set_affinity_parent, -}; - -static int -mvebu_icu_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, +static int mvebu_icu_translate(struct irq_domain *d, struct irq_fwspec *fwspec, unsigned long *hwirq, unsigned int *type) { unsigned int param_count = static_branch_unlikely(&legacy_bindings) ? 3 : 2; - struct mvebu_icu_msi_data *msi_data = platform_msi_get_host_data(d); + struct msi_domain_info *info = d->host_data; + struct mvebu_icu_msi_data *msi_data = info->chip_data; struct mvebu_icu *icu = msi_data->icu; /* Check the count of the parameters in dt */ @@ -192,81 +109,126 @@ mvebu_icu_irq_domain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, return 0; } -static int -mvebu_icu_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *args) +static void mvebu_icu_init(struct mvebu_icu *icu, + struct mvebu_icu_msi_data *msi_data, + struct msi_msg *msg) { - int err; - unsigned long hwirq; - struct irq_fwspec *fwspec = args; - struct mvebu_icu_msi_data *msi_data = platform_msi_get_host_data(domain); - struct mvebu_icu *icu = msi_data->icu; - struct mvebu_icu_irq_data *icu_irqd; - struct irq_chip *chip = &mvebu_icu_nsr_chip; + const struct mvebu_icu_subset_data *subset = msi_data->subset_data; - icu_irqd = kmalloc(sizeof(*icu_irqd), GFP_KERNEL); - if (!icu_irqd) - return -ENOMEM; + if (atomic_cmpxchg(&msi_data->initialized, false, true)) + return; - err = mvebu_icu_irq_domain_translate(domain, fwspec, &hwirq, - &icu_irqd->type); - if (err) { - dev_err(icu->dev, "failed to translate ICU parameters\n"); - goto free_irqd; - } + /* Set 'SET' ICU SPI message address in AP */ + writel_relaxed(msg[0].address_hi, icu->base + subset->offset_set_ah); + writel_relaxed(msg[0].address_lo, icu->base + subset->offset_set_al); - if (static_branch_unlikely(&legacy_bindings)) - icu_irqd->icu_group = fwspec->param[0]; - else - icu_irqd->icu_group = msi_data->subset_data->icu_group; - icu_irqd->icu = icu; + if (subset->icu_group != ICU_GRP_NSR) + return; - err = platform_msi_device_domain_alloc(domain, virq, nr_irqs); - if (err) { - dev_err(icu->dev, "failed to allocate ICU interrupt in parent domain\n"); - goto free_irqd; - } + /* Set 'CLEAR' ICU SPI message address in AP (level-MSI only) */ + writel_relaxed(msg[1].address_hi, icu->base + subset->offset_clr_ah); + writel_relaxed(msg[1].address_lo, icu->base + subset->offset_clr_al); +} - /* Make sure there is no interrupt left pending by the firmware */ - err = irq_set_irqchip_state(virq, IRQCHIP_STATE_PENDING, false); - if (err) - goto free_msi; +static int mvebu_icu_msi_init(struct irq_domain *domain, struct msi_domain_info *info, + unsigned int virq, irq_hw_number_t hwirq, msi_alloc_info_t *arg) +{ + irq_domain_set_hwirq_and_chip(domain, virq, hwirq, info->chip, info->chip_data); + return irq_set_irqchip_state(virq, IRQCHIP_STATE_PENDING, false); +} - if (icu_irqd->icu_group == ICU_GRP_SEI) - chip = &mvebu_icu_sei_chip; +static void mvebu_icu_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = (u32)desc->data.icookie.value; +} - err = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, - chip, icu_irqd); - if (err) { - dev_err(icu->dev, "failed to set the data to IRQ domain\n"); - goto free_msi; +static void mvebu_icu_write_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct mvebu_icu_msi_data *msi_data = d->chip_data; + unsigned int icu_group = msi_data->subset_data->icu_group; + struct msi_desc *desc = irq_data_get_msi_desc(d); + struct mvebu_icu *icu = msi_data->icu; + unsigned int type; + u32 icu_int; + + if (msg->address_lo || msg->address_hi) { + /* One off initialization per domain */ + mvebu_icu_init(icu, msi_data, msg); + /* Configure the ICU with irq number & type */ + icu_int = msg->data | ICU_INT_ENABLE; + type = (unsigned int)(desc->data.icookie.value >> 32); + if (type & IRQ_TYPE_EDGE_RISING) + icu_int |= ICU_IS_EDGE; + icu_int |= icu_group << ICU_GROUP_SHIFT; + } else { + /* De-configure the ICU */ + icu_int = 0; } - return 0; + writel_relaxed(icu_int, icu->base + ICU_INT_CFG(d->hwirq)); -free_msi: - platform_msi_device_domain_free(domain, virq, nr_irqs); -free_irqd: - kfree(icu_irqd); - return err; + /* + * The SATA unit has 2 ports, and a dedicated ICU entry per + * port. The ahci sata driver supports only one irq interrupt + * per SATA unit. To solve this conflict, we configure the 2 + * SATA wired interrupts in the south bridge into 1 GIC + * interrupt in the north bridge. Even if only a single port + * is enabled, if sata node is enabled, both interrupts are + * configured (regardless of which port is actually in use). + */ + if (d->hwirq == ICU_SATA0_ICU_ID || d->hwirq == ICU_SATA1_ICU_ID) { + writel_relaxed(icu_int, icu->base + ICU_INT_CFG(ICU_SATA0_ICU_ID)); + writel_relaxed(icu_int, icu->base + ICU_INT_CFG(ICU_SATA1_ICU_ID)); + } } -static void -mvebu_icu_irq_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - struct irq_data *d = irq_get_irq_data(virq); - struct mvebu_icu_irq_data *icu_irqd = d->chip_data; +static const struct msi_domain_template mvebu_icu_nsr_msi_template = { + .chip = { + .name = "ICU-NSR", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_eoi = irq_chip_eoi_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_write_msi_msg = mvebu_icu_write_msi_msg, + .flags = IRQCHIP_SUPPORTS_LEVEL_MSI, + }, - kfree(icu_irqd); + .ops = { + .msi_translate = mvebu_icu_translate, + .msi_init = mvebu_icu_msi_init, + .set_desc = mvebu_icu_set_desc, + }, - platform_msi_device_domain_free(domain, virq, nr_irqs); -} + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_LEVEL_CAPABLE | + MSI_FLAG_USE_DEV_FWNODE, + }, +}; + +static const struct msi_domain_template mvebu_icu_sei_msi_template = { + .chip = { + .name = "ICU-SEI", + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_ack = irq_chip_ack_parent, + .irq_set_type = irq_chip_set_type_parent, + .irq_write_msi_msg = mvebu_icu_write_msi_msg, + .flags = IRQCHIP_SUPPORTS_LEVEL_MSI, + }, -static const struct irq_domain_ops mvebu_icu_domain_ops = { - .translate = mvebu_icu_irq_domain_translate, - .alloc = mvebu_icu_irq_domain_alloc, - .free = mvebu_icu_irq_domain_free, + .ops = { + .msi_translate = mvebu_icu_translate, + .msi_init = mvebu_icu_msi_init, + .set_desc = mvebu_icu_set_desc, + }, + + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_LEVEL_CAPABLE | + MSI_FLAG_USE_DEV_FWNODE, + }, }; static const struct mvebu_icu_subset_data mvebu_icu_nsr_subset_data = { @@ -297,10 +259,10 @@ static const struct of_device_id mvebu_icu_subset_of_match[] = { static int mvebu_icu_subset_probe(struct platform_device *pdev) { + const struct msi_domain_template *tmpl; struct mvebu_icu_msi_data *msi_data; - struct device_node *msi_parent_dn; struct device *dev = &pdev->dev; - struct irq_domain *irq_domain; + bool sei; msi_data = devm_kzalloc(dev, sizeof(*msi_data), GFP_KERNEL); if (!msi_data) @@ -314,20 +276,18 @@ static int mvebu_icu_subset_probe(struct platform_device *pdev) msi_data->subset_data = of_device_get_match_data(dev); } - dev->msi.domain = of_msi_get_domain(dev, dev->of_node, - DOMAIN_BUS_PLATFORM_MSI); + dev->msi.domain = of_msi_get_domain(dev, dev->of_node, DOMAIN_BUS_PLATFORM_MSI); if (!dev->msi.domain) return -EPROBE_DEFER; - msi_parent_dn = irq_domain_get_of_node(dev->msi.domain); - if (!msi_parent_dn) + if (!irq_domain_get_of_node(dev->msi.domain)) return -ENODEV; - irq_domain = platform_msi_create_device_tree_domain(dev, ICU_MAX_IRQS, - mvebu_icu_write_msg, - &mvebu_icu_domain_ops, - msi_data); - if (!irq_domain) { + sei = msi_data->subset_data->icu_group == ICU_GRP_SEI; + tmpl = sei ? &mvebu_icu_sei_msi_template : &mvebu_icu_nsr_msi_template; + + if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, tmpl, + ICU_MAX_IRQS, NULL, msi_data)) { dev_err(dev, "Failed to create ICU MSI domain\n"); return -ENOMEM; } diff --git a/drivers/irqchip/irq-mvebu-odmi.c b/drivers/irqchip/irq-mvebu-odmi.c index 108091533e10..e5b2bde3d933 100644 --- a/drivers/irqchip/irq-mvebu-odmi.c +++ b/drivers/irqchip/irq-mvebu-odmi.c @@ -17,6 +17,9 @@ #include <linux/msi.h> #include <linux/of_address.h> #include <linux/slab.h> + +#include <linux/irqchip/irq-msi-lib.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> #define GICP_ODMIN_SET 0x40 @@ -141,27 +144,35 @@ static void odmi_irq_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops odmi_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = odmi_irq_domain_alloc, .free = odmi_irq_domain_free, }; -static struct irq_chip odmi_msi_irq_chip = { - .name = "ODMI", -}; +#define ODMI_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) -static struct msi_domain_ops odmi_msi_ops = { -}; +#define ODMI_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK) -static struct msi_domain_info odmi_msi_domain_info = { - .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS), - .ops = &odmi_msi_ops, - .chip = &odmi_msi_irq_chip, +static const struct msi_parent_ops odmi_msi_parent_ops = { + .supported_flags = ODMI_MSI_FLAGS_SUPPORTED, + .required_flags = ODMI_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI, + .bus_select_token = DOMAIN_BUS_GENERIC_MSI, + .bus_select_mask = MATCH_PLATFORM_MSI, + .prefix = "ODMI-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, }; static int __init mvebu_odmi_init(struct device_node *node, struct device_node *parent) { - struct irq_domain *parent_domain, *inner_domain, *plat_domain; + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &odmi_domain_ops, + .size = odmis_count * NODMIS_PER_FRAME, + .parent = irq_find_host(parent), + }; int ret, i; if (of_property_read_u32(node, "marvell,odmi-frames", &odmis_count)) @@ -197,29 +208,11 @@ static int __init mvebu_odmi_init(struct device_node *node, } } - parent_domain = irq_find_host(parent); + if (msi_create_parent_irq_domain(&info, &odmi_msi_parent_ops)) + return 0; - inner_domain = irq_domain_create_hierarchy(parent_domain, 0, - odmis_count * NODMIS_PER_FRAME, - of_node_to_fwnode(node), - &odmi_domain_ops, NULL); - if (!inner_domain) { - ret = -ENOMEM; - goto err_unmap; - } - - plat_domain = platform_msi_create_irq_domain(of_node_to_fwnode(node), - &odmi_msi_domain_info, - inner_domain); - if (!plat_domain) { - ret = -ENOMEM; - goto err_remove_inner; - } - - return 0; + ret = -ENOMEM; -err_remove_inner: - irq_domain_remove(inner_domain); err_unmap: for (i = 0; i < odmis_count; i++) { struct odmi_data *odmi = &odmis[i]; diff --git a/drivers/irqchip/irq-mvebu-pic.c b/drivers/irqchip/irq-mvebu-pic.c index ef3d3646ccc2..10b85128183a 100644 --- a/drivers/irqchip/irq-mvebu-pic.c +++ b/drivers/irqchip/irq-mvebu-pic.c @@ -71,7 +71,7 @@ static void mvebu_pic_print_chip(struct irq_data *d, struct seq_file *p) { struct mvebu_pic *pic = irq_data_get_irq_chip_data(d); - seq_printf(p, dev_name(&pic->pdev->dev)); + seq_puts(p, dev_name(&pic->pdev->dev)); } static const struct irq_chip mvebu_pic_chip = { @@ -150,8 +150,8 @@ static int mvebu_pic_probe(struct platform_device *pdev) return -EINVAL; } - pic->domain = irq_domain_add_linear(node, PIC_MAX_IRQS, - &mvebu_pic_domain_ops, pic); + pic->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), PIC_MAX_IRQS, + &mvebu_pic_domain_ops, pic); if (!pic->domain) { dev_err(&pdev->dev, "Failed to allocate irq domain\n"); return -ENOMEM; @@ -167,14 +167,12 @@ static int mvebu_pic_probe(struct platform_device *pdev) return 0; } -static int mvebu_pic_remove(struct platform_device *pdev) +static void mvebu_pic_remove(struct platform_device *pdev) { struct mvebu_pic *pic = platform_get_drvdata(pdev); on_each_cpu(mvebu_pic_disable_percpu_irq, pic, 1); irq_domain_remove(pic->domain); - - return 0; } static const struct of_device_id mvebu_pic_of_match[] = { @@ -184,17 +182,16 @@ static const struct of_device_id mvebu_pic_of_match[] = { MODULE_DEVICE_TABLE(of, mvebu_pic_of_match); static struct platform_driver mvebu_pic_driver = { - .probe = mvebu_pic_probe, - .remove = mvebu_pic_remove, + .probe = mvebu_pic_probe, + .remove = mvebu_pic_remove, .driver = { - .name = "mvebu-pic", - .of_match_table = mvebu_pic_of_match, + .name = "mvebu-pic", + .of_match_table = mvebu_pic_of_match, }, }; module_platform_driver(mvebu_pic_driver); MODULE_AUTHOR("Yehuda Yitschak <yehuday@marvell.com>"); MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); +MODULE_DESCRIPTION("Marvell Armada 7K/8K PIC driver"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:mvebu_pic"); - diff --git a/drivers/irqchip/irq-mvebu-sei.c b/drivers/irqchip/irq-mvebu-sei.c index a48dbe91b036..5822ea864765 100644 --- a/drivers/irqchip/irq-mvebu-sei.c +++ b/drivers/irqchip/irq-mvebu-sei.c @@ -14,6 +14,8 @@ #include <linux/of_irq.h> #include <linux/of_platform.h> +#include <linux/irqchip/irq-msi-lib.h> + /* Cause register */ #define GICP_SECR(idx) (0x0 + ((idx) * 0x4)) /* Mask register */ @@ -303,25 +305,11 @@ static void mvebu_sei_cp_domain_free(struct irq_domain *domain, } static const struct irq_domain_ops mvebu_sei_cp_domain_ops = { + .select = msi_lib_irq_domain_select, .alloc = mvebu_sei_cp_domain_alloc, .free = mvebu_sei_cp_domain_free, }; -static struct irq_chip mvebu_sei_msi_irq_chip = { - .name = "SEI pMSI", - .irq_ack = irq_chip_ack_parent, - .irq_set_type = irq_chip_set_type_parent, -}; - -static struct msi_domain_ops mvebu_sei_msi_ops = { -}; - -static struct msi_domain_info mvebu_sei_msi_domain_info = { - .flags = MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS, - .ops = &mvebu_sei_msi_ops, - .chip = &mvebu_sei_msi_irq_chip, -}; - static void mvebu_sei_handle_cascade_irq(struct irq_desc *desc) { struct mvebu_sei *sei = irq_desc_get_handler_data(desc); @@ -360,10 +348,28 @@ static void mvebu_sei_reset(struct mvebu_sei *sei) } } +#define SEI_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS) + +#define SEI_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK) + +static const struct msi_parent_ops sei_msi_parent_ops = { + .supported_flags = SEI_MSI_FLAGS_SUPPORTED, + .required_flags = SEI_MSI_FLAGS_REQUIRED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI | MSI_CHIP_FLAG_SET_ACK, + .bus_select_mask = MATCH_PLATFORM_MSI, + .bus_select_token = DOMAIN_BUS_GENERIC_MSI, + .prefix = "SEI-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + static int mvebu_sei_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; - struct irq_domain *plat_domain; + struct irq_domain_info info = { + .fwnode = of_fwnode_handle(node), + .ops = &mvebu_sei_cp_domain_ops, + }; struct mvebu_sei *sei; u32 parent_irq; int ret; @@ -400,7 +406,7 @@ static int mvebu_sei_probe(struct platform_device *pdev) } /* Create the root SEI domain */ - sei->sei_domain = irq_domain_create_linear(of_node_to_fwnode(node), + sei->sei_domain = irq_domain_create_linear(of_fwnode_handle(node), (sei->caps->ap_range.size + sei->caps->cp_range.size), &mvebu_sei_domain_ops, @@ -416,7 +422,7 @@ static int mvebu_sei_probe(struct platform_device *pdev) /* Create the 'wired' domain */ sei->ap_domain = irq_domain_create_hierarchy(sei->sei_domain, 0, sei->caps->ap_range.size, - of_node_to_fwnode(node), + of_fwnode_handle(node), &mvebu_sei_ap_domain_ops, sei); if (!sei->ap_domain) { @@ -428,45 +434,28 @@ static int mvebu_sei_probe(struct platform_device *pdev) irq_domain_update_bus_token(sei->ap_domain, DOMAIN_BUS_WIRED); /* Create the 'MSI' domain */ - sei->cp_domain = irq_domain_create_hierarchy(sei->sei_domain, 0, - sei->caps->cp_range.size, - of_node_to_fwnode(node), - &mvebu_sei_cp_domain_ops, - sei); + info.size = sei->caps->cp_range.size; + info.host_data = sei; + info.parent = sei->sei_domain; + + sei->cp_domain = msi_create_parent_irq_domain(&info, &sei_msi_parent_ops); if (!sei->cp_domain) { pr_err("Failed to create CPs IRQ domain\n"); ret = -ENOMEM; goto remove_ap_domain; } - irq_domain_update_bus_token(sei->cp_domain, DOMAIN_BUS_GENERIC_MSI); - - plat_domain = platform_msi_create_irq_domain(of_node_to_fwnode(node), - &mvebu_sei_msi_domain_info, - sei->cp_domain); - if (!plat_domain) { - pr_err("Failed to create CPs MSI domain\n"); - ret = -ENOMEM; - goto remove_cp_domain; - } - mvebu_sei_reset(sei); - irq_set_chained_handler_and_data(parent_irq, - mvebu_sei_handle_cascade_irq, - sei); - + irq_set_chained_handler_and_data(parent_irq, mvebu_sei_handle_cascade_irq, sei); return 0; -remove_cp_domain: - irq_domain_remove(sei->cp_domain); remove_ap_domain: irq_domain_remove(sei->ap_domain); remove_sei_domain: irq_domain_remove(sei->sei_domain); dispose_irq: irq_dispose_mapping(parent_irq); - return ret; } diff --git a/drivers/irqchip/irq-mxs.c b/drivers/irqchip/irq-mxs.c index be9680645545..0bb423dd5280 100644 --- a/drivers/irqchip/irq-mxs.c +++ b/drivers/irqchip/irq-mxs.c @@ -130,7 +130,7 @@ static struct irq_chip asm9260_icoll_chip = { IRQCHIP_SKIP_SET_WAKE, }; -asmlinkage void __exception_irq_entry icoll_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry icoll_handle_irq(struct pt_regs *regs) { u32 irqnr; @@ -162,8 +162,8 @@ static const struct irq_domain_ops icoll_irq_domain_ops = { static void __init icoll_add_domain(struct device_node *np, int num) { - icoll_domain = irq_domain_add_linear(np, num, - &icoll_irq_domain_ops, NULL); + icoll_domain = irq_domain_create_linear(of_fwnode_handle(np), num, + &icoll_irq_domain_ops, NULL); if (!icoll_domain) panic("%pOF: unable to create irq domain", np); diff --git a/drivers/irqchip/irq-nvic.c b/drivers/irqchip/irq-nvic.c index ba6332b00a0a..2191a2b79578 100644 --- a/drivers/irqchip/irq-nvic.c +++ b/drivers/irqchip/irq-nvic.c @@ -73,8 +73,9 @@ static int __init nvic_of_init(struct device_node *node, struct device_node *parent) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; - unsigned int irqs, i, ret, numbanks; + unsigned int irqs, i, numbanks; void __iomem *nvic_base; + int ret; numbanks = (readl_relaxed(V7M_SCS_ICTR) & V7M_SCS_ICTR_INTLINESNUM_MASK) + 1; @@ -90,7 +91,7 @@ static int __init nvic_of_init(struct device_node *node, irqs = NVIC_MAX_IRQ; nvic_irq_domain = - irq_domain_add_linear(node, irqs, &nvic_irq_domain_ops, NULL); + irq_domain_create_linear(of_fwnode_handle(node), irqs, &nvic_irq_domain_ops, NULL); if (!nvic_irq_domain) { pr_warn("Failed to allocate irq domain\n"); diff --git a/drivers/irqchip/irq-omap-intc.c b/drivers/irqchip/irq-omap-intc.c index dc82162ba763..16f00db570e7 100644 --- a/drivers/irqchip/irq-omap-intc.c +++ b/drivers/irqchip/irq-omap-intc.c @@ -248,7 +248,7 @@ static int __init omap_init_irq_of(struct device_node *node) if (WARN_ON(!omap_irq_base)) return -ENOMEM; - domain = irq_domain_add_linear(node, omap_nr_irqs, + domain = irq_domain_create_linear(of_fwnode_handle(node), omap_nr_irqs, &irq_generic_chip_ops, NULL); omap_irq_soft_reset(); @@ -274,7 +274,7 @@ static int __init omap_init_irq_legacy(u32 base, struct device_node *node) irq_base = 0; } - domain = irq_domain_add_legacy(node, omap_nr_irqs, irq_base, 0, + domain = irq_domain_create_legacy(of_fwnode_handle(node), omap_nr_irqs, irq_base, 0, &irq_domain_simple_ops, NULL); omap_irq_soft_reset(); @@ -325,8 +325,7 @@ static int __init omap_init_irq(u32 base, struct device_node *node) return ret; } -static asmlinkage void __exception_irq_entry -omap_intc_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry omap_intc_handle_irq(struct pt_regs *regs) { extern unsigned long irq_err_count; u32 irqnr; diff --git a/drivers/irqchip/irq-or1k-pic.c b/drivers/irqchip/irq-or1k-pic.c index f289ccd95291..48126067c54b 100644 --- a/drivers/irqchip/irq-or1k-pic.c +++ b/drivers/irqchip/irq-or1k-pic.c @@ -144,8 +144,8 @@ static int __init or1k_pic_init(struct device_node *node, /* Disable all interrupts until explicitly requested */ mtspr(SPR_PICMR, (0UL)); - root_domain = irq_domain_add_linear(node, 32, &or1k_irq_domain_ops, - pic); + root_domain = irq_domain_create_linear(of_fwnode_handle(node), 32, &or1k_irq_domain_ops, + pic); set_handle_irq(or1k_pic_handle_irq); diff --git a/drivers/irqchip/irq-orion.c b/drivers/irqchip/irq-orion.c index 4e4e874e09a8..dddbc05917c0 100644 --- a/drivers/irqchip/irq-orion.c +++ b/drivers/irqchip/irq-orion.c @@ -59,7 +59,7 @@ static int __init orion_irq_init(struct device_node *np, /* count number of irq chips by valid reg addresses */ num_chips = of_address_count(np); - orion_irq_domain = irq_domain_add_linear(np, + orion_irq_domain = irq_domain_create_linear(of_fwnode_handle(np), num_chips * ORION_IRQS_PER_CHIP, &irq_generic_chip_ops, NULL); if (!orion_irq_domain) @@ -146,8 +146,8 @@ static int __init orion_bridge_irq_init(struct device_node *np, /* get optional number of interrupts provided */ of_property_read_u32(np, "marvell,#interrupts", &nrirqs); - domain = irq_domain_add_linear(np, nrirqs, - &irq_generic_chip_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(np), nrirqs, + &irq_generic_chip_ops, NULL); if (!domain) { pr_err("%pOFn: unable to add irq domain\n", np); return -ENOMEM; diff --git a/drivers/irqchip/irq-owl-sirq.c b/drivers/irqchip/irq-owl-sirq.c index 6e4127465094..3d93d21f6732 100644 --- a/drivers/irqchip/irq-owl-sirq.c +++ b/drivers/irqchip/irq-owl-sirq.c @@ -323,8 +323,8 @@ static int __init owl_sirq_init(const struct owl_sirq_params *params, owl_sirq_clear_set_extctl(chip_data, 0, INTC_EXTCTL_CLK_SEL, i); } - domain = irq_domain_add_hierarchy(parent_domain, 0, NUM_SIRQ, node, - &owl_sirq_domain_ops, chip_data); + domain = irq_domain_create_hierarchy(parent_domain, 0, NUM_SIRQ, of_fwnode_handle(node), + &owl_sirq_domain_ops, chip_data); if (!domain) { pr_err("%pOF: failed to add domain\n", node); ret = -ENOMEM; diff --git a/drivers/irqchip/irq-partition-percpu.c b/drivers/irqchip/irq-partition-percpu.c deleted file mode 100644 index 8e76d2913e6b..000000000000 --- a/drivers/irqchip/irq-partition-percpu.c +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2016 ARM Limited, All Rights Reserved. - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/bitops.h> -#include <linux/interrupt.h> -#include <linux/irqchip.h> -#include <linux/irqchip/chained_irq.h> -#include <linux/irqchip/irq-partition-percpu.h> -#include <linux/irqdomain.h> -#include <linux/seq_file.h> -#include <linux/slab.h> - -struct partition_desc { - int nr_parts; - struct partition_affinity *parts; - struct irq_domain *domain; - struct irq_desc *chained_desc; - unsigned long *bitmap; - struct irq_domain_ops ops; -}; - -static bool partition_check_cpu(struct partition_desc *part, - unsigned int cpu, unsigned int hwirq) -{ - return cpumask_test_cpu(cpu, &part->parts[hwirq].mask); -} - -static void partition_irq_mask(struct irq_data *d) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_mask) - chip->irq_mask(data); -} - -static void partition_irq_unmask(struct irq_data *d) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_unmask) - chip->irq_unmask(data); -} - -static int partition_irq_set_irqchip_state(struct irq_data *d, - enum irqchip_irq_state which, - bool val) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_set_irqchip_state) - return chip->irq_set_irqchip_state(data, which, val); - - return -EINVAL; -} - -static int partition_irq_get_irqchip_state(struct irq_data *d, - enum irqchip_irq_state which, - bool *val) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_get_irqchip_state) - return chip->irq_get_irqchip_state(data, which, val); - - return -EINVAL; -} - -static int partition_irq_set_type(struct irq_data *d, unsigned int type) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (chip->irq_set_type) - return chip->irq_set_type(data, type); - - return -EINVAL; -} - -static void partition_irq_print_chip(struct irq_data *d, struct seq_file *p) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - seq_printf(p, " %5s-%lu", chip->name, data->hwirq); -} - -static struct irq_chip partition_irq_chip = { - .irq_mask = partition_irq_mask, - .irq_unmask = partition_irq_unmask, - .irq_set_type = partition_irq_set_type, - .irq_get_irqchip_state = partition_irq_get_irqchip_state, - .irq_set_irqchip_state = partition_irq_set_irqchip_state, - .irq_print_chip = partition_irq_print_chip, -}; - -static void partition_handle_irq(struct irq_desc *desc) -{ - struct partition_desc *part = irq_desc_get_handler_data(desc); - struct irq_chip *chip = irq_desc_get_chip(desc); - int cpu = smp_processor_id(); - int hwirq; - - chained_irq_enter(chip, desc); - - for_each_set_bit(hwirq, part->bitmap, part->nr_parts) { - if (partition_check_cpu(part, cpu, hwirq)) - break; - } - - if (unlikely(hwirq == part->nr_parts)) - handle_bad_irq(desc); - else - generic_handle_domain_irq(part->domain, hwirq); - - chained_irq_exit(chip, desc); -} - -static int partition_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) -{ - int ret; - irq_hw_number_t hwirq; - unsigned int type; - struct irq_fwspec *fwspec = arg; - struct partition_desc *part; - - BUG_ON(nr_irqs != 1); - ret = domain->ops->translate(domain, fwspec, &hwirq, &type); - if (ret) - return ret; - - part = domain->host_data; - - set_bit(hwirq, part->bitmap); - irq_set_chained_handler_and_data(irq_desc_get_irq(part->chained_desc), - partition_handle_irq, part); - irq_set_percpu_devid_partition(virq, &part->parts[hwirq].mask); - irq_domain_set_info(domain, virq, hwirq, &partition_irq_chip, part, - handle_percpu_devid_irq, NULL, NULL); - irq_set_status_flags(virq, IRQ_NOAUTOEN); - - return 0; -} - -static void partition_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - struct irq_data *d; - - BUG_ON(nr_irqs != 1); - - d = irq_domain_get_irq_data(domain, virq); - irq_set_handler(virq, NULL); - irq_domain_reset_irq_data(d); -} - -int partition_translate_id(struct partition_desc *desc, void *partition_id) -{ - struct partition_affinity *part = NULL; - int i; - - for (i = 0; i < desc->nr_parts; i++) { - if (desc->parts[i].partition_id == partition_id) { - part = &desc->parts[i]; - break; - } - } - - if (WARN_ON(!part)) { - pr_err("Failed to find partition\n"); - return -EINVAL; - } - - return i; -} - -struct partition_desc *partition_create_desc(struct fwnode_handle *fwnode, - struct partition_affinity *parts, - int nr_parts, - int chained_irq, - const struct irq_domain_ops *ops) -{ - struct partition_desc *desc; - struct irq_domain *d; - - BUG_ON(!ops->select || !ops->translate); - - desc = kzalloc(sizeof(*desc), GFP_KERNEL); - if (!desc) - return NULL; - - desc->ops = *ops; - desc->ops.free = partition_domain_free; - desc->ops.alloc = partition_domain_alloc; - - d = irq_domain_create_linear(fwnode, nr_parts, &desc->ops, desc); - if (!d) - goto out; - desc->domain = d; - - desc->bitmap = bitmap_zalloc(nr_parts, GFP_KERNEL); - if (WARN_ON(!desc->bitmap)) - goto out; - - desc->chained_desc = irq_to_desc(chained_irq); - desc->nr_parts = nr_parts; - desc->parts = parts; - - return desc; -out: - if (d) - irq_domain_remove(d); - kfree(desc); - - return NULL; -} - -struct irq_domain *partition_get_domain(struct partition_desc *dsc) -{ - if (dsc) - return dsc->domain; - - return NULL; -} diff --git a/drivers/irqchip/irq-pic32-evic.c b/drivers/irqchip/irq-pic32-evic.c index 1d9bb28d13e5..5dfda8e8df10 100644 --- a/drivers/irqchip/irq-pic32-evic.c +++ b/drivers/irqchip/irq-pic32-evic.c @@ -161,9 +161,9 @@ static int pic32_irq_domain_map(struct irq_domain *d, unsigned int virq, return ret; } -int pic32_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, - const u32 *intspec, unsigned int intsize, - irq_hw_number_t *out_hwirq, unsigned int *out_type) +static int pic32_irq_domain_xlate(struct irq_domain *d, struct device_node *ctrlr, + const u32 *intspec, unsigned int intsize, + irq_hw_number_t *out_hwirq, unsigned int *out_type) { struct evic_chip_data *priv = d->host_data; @@ -190,13 +190,11 @@ static void __init pic32_ext_irq_of_init(struct irq_domain *domain) { struct device_node *node = irq_domain_get_of_node(domain); struct evic_chip_data *priv = domain->host_data; - struct property *prop; - const __le32 *p; u32 hwirq; int i = 0; const char *pname = "microchip,external-irqs"; - of_property_for_each_u32(node, pname, prop, p, hwirq) { + of_property_for_each_u32(node, pname, hwirq) { if (i >= ARRAY_SIZE(priv->ext_irqs)) { pr_warn("More than %d external irq, skip rest\n", ARRAY_SIZE(priv->ext_irqs)); @@ -229,9 +227,9 @@ static int __init pic32_of_init(struct device_node *node, goto err_iounmap; } - evic_irq_domain = irq_domain_add_linear(node, nchips * 32, - &pic32_irq_domain_ops, - priv); + evic_irq_domain = irq_domain_create_linear(of_fwnode_handle(node), nchips * 32, + &pic32_irq_domain_ops, + priv); if (!evic_irq_domain) { ret = -ENOMEM; goto err_free_priv; @@ -293,7 +291,7 @@ static int __init pic32_of_init(struct device_node *node, gc->private = &priv[i]; } - irq_set_default_host(evic_irq_domain); + irq_set_default_domain(evic_irq_domain); /* * External interrupts have software configurable edge polarity. These diff --git a/drivers/irqchip/irq-pruss-intc.c b/drivers/irqchip/irq-pruss-intc.c index 0f64ecb9b1f4..81078d56f38d 100644 --- a/drivers/irqchip/irq-pruss-intc.c +++ b/drivers/irqchip/irq-pruss-intc.c @@ -555,8 +555,8 @@ static int pruss_intc_probe(struct platform_device *pdev) mutex_init(&intc->lock); - intc->domain = irq_domain_add_linear(dev->of_node, max_system_events, - &pruss_intc_irq_domain_ops, intc); + intc->domain = irq_domain_create_linear(dev_fwnode(dev), max_system_events, + &pruss_intc_irq_domain_ops, intc); if (!intc->domain) return -ENOMEM; @@ -581,8 +581,7 @@ static int pruss_intc_probe(struct platform_device *pdev) host_data->intc = intc; host_data->host_irq = i; - irq_set_handler_data(irq, host_data); - irq_set_chained_handler(irq, pruss_intc_irq_handler); + irq_set_chained_handler_and_data(irq, pruss_intc_irq_handler, host_data); } return 0; @@ -599,7 +598,7 @@ fail_irq: return ret; } -static int pruss_intc_remove(struct platform_device *pdev) +static void pruss_intc_remove(struct platform_device *pdev) { struct pruss_intc *intc = platform_get_drvdata(pdev); u8 max_system_events = intc->soc_config->num_system_events; @@ -616,8 +615,6 @@ static int pruss_intc_remove(struct platform_device *pdev) irq_dispose_mapping(irq_find_mapping(intc->domain, hwirq)); irq_domain_remove(intc->domain); - - return 0; } static const struct pruss_intc_match_data pruss_intc_data = { @@ -645,12 +642,12 @@ MODULE_DEVICE_TABLE(of, pruss_intc_of_match); static struct platform_driver pruss_intc_driver = { .driver = { - .name = "pruss-intc", - .of_match_table = pruss_intc_of_match, - .suppress_bind_attrs = true, + .name = "pruss-intc", + .of_match_table = pruss_intc_of_match, + .suppress_bind_attrs = true, }, - .probe = pruss_intc_probe, - .remove = pruss_intc_remove, + .probe = pruss_intc_probe, + .remove = pruss_intc_remove, }; module_platform_driver(pruss_intc_driver); diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index 7124565234a5..83f31ea657b7 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -14,6 +14,7 @@ #include <linux/mailbox_client.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm_domain.h> @@ -226,6 +227,9 @@ static int qcom_mpm_alloc(struct irq_domain *domain, unsigned int virq, if (ret) return ret; + if (pin == GPIO_NO_WAKE_IRQ) + return irq_domain_disconnect_hierarchy(domain, virq); + ret = irq_domain_set_hwirq_and_chip(domain, virq, pin, &qcom_mpm_chip, priv); if (ret) @@ -316,14 +320,16 @@ static bool gic_hwirq_is_mapped(struct mpm_gic_map *maps, int cnt, u32 hwirq) return false; } -static int qcom_mpm_init(struct device_node *np, struct device_node *parent) +static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *parent) { - struct platform_device *pdev = of_find_device_by_node(np); + struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct irq_domain *parent_domain; struct generic_pm_domain *genpd; + struct device_node *msgram_np; struct qcom_mpm_priv *priv; unsigned int pin_cnt; + struct resource res; int i, irq; int ret; @@ -374,9 +380,26 @@ static int qcom_mpm_init(struct device_node *np, struct device_node *parent) raw_spin_lock_init(&priv->lock); - priv->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(priv->base)) - return PTR_ERR(priv->base); + /* If we have a handle to an RPM message ram partition, use it. */ + msgram_np = of_parse_phandle(np, "qcom,rpm-msg-ram", 0); + if (msgram_np) { + ret = of_address_to_resource(msgram_np, 0, &res); + if (ret) { + of_node_put(msgram_np); + return ret; + } + + /* Don't use devm_ioremap_resource, as we're accessing a shared region. */ + priv->base = devm_ioremap(dev, res.start, resource_size(&res)); + of_node_put(msgram_np); + if (!priv->base) + return -ENOMEM; + } else { + /* Otherwise, fall back to simple MMIO. */ + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + } for (i = 0; i < priv->reg_stride; i++) { qcom_mpm_write(priv, MPM_REG_ENABLE, i, 0); @@ -427,7 +450,7 @@ static int qcom_mpm_init(struct device_node *np, struct device_node *parent) priv->domain = irq_domain_create_hierarchy(parent_domain, IRQ_DOMAIN_FLAG_QCOM_MPM_WAKEUP, pin_cnt, - of_node_to_fwnode(np), &qcom_mpm_ops, priv); + of_fwnode_handle(np), &qcom_mpm_ops, priv); if (!priv->domain) { dev_err(dev, "failed to create MPM domain\n"); ret = -ENOMEM; @@ -455,7 +478,7 @@ remove_genpd: } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_mpm) -IRQCHIP_MATCH("qcom,mpm", qcom_mpm_init) +IRQCHIP_MATCH("qcom,mpm", qcom_mpm_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_mpm) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. MSM Power Manager"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-realtek-rtl.c b/drivers/irqchip/irq-realtek-rtl.c index 2a349082af81..942c1f8c363d 100644 --- a/drivers/irqchip/irq-realtek-rtl.c +++ b/drivers/irqchip/irq-realtek-rtl.c @@ -162,7 +162,7 @@ static int __init realtek_rtl_of_init(struct device_node *node, struct device_no else if (!parent_irq) return -ENODEV; - domain = irq_domain_add_linear(node, RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(node), RTL_ICTL_NUM_INPUTS, &irq_domain_ops, NULL); if (!domain) return -ENOMEM; diff --git a/drivers/irqchip/irq-renesas-intc-irqpin.c b/drivers/irqchip/irq-renesas-intc-irqpin.c index fa19585f3dee..7951292d2d9b 100644 --- a/drivers/irqchip/irq-renesas-intc-irqpin.c +++ b/drivers/irqchip/irq-renesas-intc-irqpin.c @@ -513,8 +513,8 @@ static int intc_irqpin_probe(struct platform_device *pdev) irq_chip->irq_set_wake = intc_irqpin_irq_set_wake; irq_chip->flags = IRQCHIP_MASK_ON_SUSPEND; - p->irq_domain = irq_domain_add_simple(dev->of_node, nirqs, 0, - &intc_irqpin_irq_domain_ops, p); + p->irq_domain = irq_domain_create_simple(dev_fwnode(dev), nirqs, 0, + &intc_irqpin_irq_domain_ops, p); if (!p->irq_domain) { ret = -ENXIO; dev_err(dev, "cannot initialize irq domain\n"); @@ -561,17 +561,16 @@ err0: return ret; } -static int intc_irqpin_remove(struct platform_device *pdev) +static void intc_irqpin_remove(struct platform_device *pdev) { struct intc_irqpin_priv *p = platform_get_drvdata(pdev); irq_domain_remove(p->irq_domain); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } -static int __maybe_unused intc_irqpin_suspend(struct device *dev) +static int intc_irqpin_suspend(struct device *dev) { struct intc_irqpin_priv *p = dev_get_drvdata(dev); @@ -581,15 +580,15 @@ static int __maybe_unused intc_irqpin_suspend(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(intc_irqpin_pm_ops, intc_irqpin_suspend, NULL); +static DEFINE_SIMPLE_DEV_PM_OPS(intc_irqpin_pm_ops, intc_irqpin_suspend, NULL); static struct platform_driver intc_irqpin_device_driver = { .probe = intc_irqpin_probe, .remove = intc_irqpin_remove, .driver = { - .name = "renesas_intc_irqpin", - .of_match_table = intc_irqpin_dt_ids, - .pm = &intc_irqpin_pm_ops, + .name = "renesas_intc_irqpin", + .of_match_table = intc_irqpin_dt_ids, + .pm = pm_sleep_ptr(&intc_irqpin_pm_ops), } }; diff --git a/drivers/irqchip/irq-renesas-irqc.c b/drivers/irqchip/irq-renesas-irqc.c index 49b446b396f9..a20a6471b0e4 100644 --- a/drivers/irqchip/irq-renesas-irqc.c +++ b/drivers/irqchip/irq-renesas-irqc.c @@ -168,8 +168,8 @@ static int irqc_probe(struct platform_device *pdev) p->cpu_int_base = p->iomem + IRQC_INT_CPU_BASE(0); /* SYS-SPI */ - p->irq_domain = irq_domain_add_linear(dev->of_node, p->number_of_irqs, - &irq_generic_chip_ops, p); + p->irq_domain = irq_domain_create_linear(dev_fwnode(dev), p->number_of_irqs, + &irq_generic_chip_ops, p); if (!p->irq_domain) { ret = -ENXIO; dev_err(dev, "cannot initialize irq domain\n"); @@ -218,17 +218,16 @@ err_runtime_pm_disable: return ret; } -static int irqc_remove(struct platform_device *pdev) +static void irqc_remove(struct platform_device *pdev) { struct irqc_priv *p = platform_get_drvdata(pdev); irq_domain_remove(p->irq_domain); pm_runtime_put(&pdev->dev); pm_runtime_disable(&pdev->dev); - return 0; } -static int __maybe_unused irqc_suspend(struct device *dev) +static int irqc_suspend(struct device *dev) { struct irqc_priv *p = dev_get_drvdata(dev); @@ -238,7 +237,7 @@ static int __maybe_unused irqc_suspend(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(irqc_pm_ops, irqc_suspend, NULL); +static DEFINE_SIMPLE_DEV_PM_OPS(irqc_pm_ops, irqc_suspend, NULL); static const struct of_device_id irqc_dt_ids[] = { { .compatible = "renesas,irqc", }, @@ -250,9 +249,9 @@ static struct platform_driver irqc_device_driver = { .probe = irqc_probe, .remove = irqc_remove, .driver = { - .name = "renesas_irqc", + .name = "renesas_irqc", .of_match_table = irqc_dt_ids, - .pm = &irqc_pm_ops, + .pm = pm_sleep_ptr(&irqc_pm_ops), } }; diff --git a/drivers/irqchip/irq-renesas-rza1.c b/drivers/irqchip/irq-renesas-rza1.c index e4c99c2e0373..6047a524ac77 100644 --- a/drivers/irqchip/irq-renesas-rza1.c +++ b/drivers/irqchip/irq-renesas-rza1.c @@ -142,11 +142,12 @@ static const struct irq_domain_ops rza1_irqc_domain_ops = { static int rza1_irqc_parse_map(struct rza1_irqc_priv *priv, struct device_node *gic_node) { - unsigned int imaplen, i, j, ret; struct device *dev = priv->dev; + unsigned int imaplen, i, j; struct device_node *ipar; const __be32 *imap; u32 intsize; + int ret; imap = of_get_property(dev->of_node, "interrupt-map", &imaplen); if (!imap) @@ -231,9 +232,8 @@ static int rza1_irqc_probe(struct platform_device *pdev) priv->chip.irq_set_type = rza1_irqc_set_type; priv->chip.flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SKIP_SET_WAKE; - priv->irq_domain = irq_domain_add_hierarchy(parent, 0, IRQC_NUM_IRQ, - np, &rza1_irqc_domain_ops, - priv); + priv->irq_domain = irq_domain_create_hierarchy(parent, 0, IRQC_NUM_IRQ, dev_fwnode(dev), + &rza1_irqc_domain_ops, priv); if (!priv->irq_domain) { dev_err(dev, "cannot initialize irq domain\n"); ret = -ENOMEM; @@ -244,12 +244,11 @@ out_put_node: return ret; } -static int rza1_irqc_remove(struct platform_device *pdev) +static void rza1_irqc_remove(struct platform_device *pdev) { struct rza1_irqc_priv *priv = platform_get_drvdata(pdev); irq_domain_remove(priv->irq_domain); - return 0; } static const struct of_device_id rza1_irqc_dt_ids[] = { @@ -262,7 +261,7 @@ static struct platform_driver rza1_irqc_device_driver = { .probe = rza1_irqc_probe, .remove = rza1_irqc_remove, .driver = { - .name = "renesas_rza1_irqc", + .name = "renesas_rza1_irqc", .of_match_table = rza1_irqc_dt_ids, } }; diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index 4bbfa2b0a4df..e73d426cea6d 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -18,6 +18,7 @@ #include <linux/pm_runtime.h> #include <linux/reset.h> #include <linux/spinlock.h> +#include <linux/syscore_ops.h> #define IRQC_IRQ_START 1 #define IRQC_IRQ_COUNT 8 @@ -28,8 +29,7 @@ #define ISCR 0x10 #define IITSR 0x14 #define TSCR 0x20 -#define TITSR0 0x24 -#define TITSR1 0x28 +#define TITSR(n) (0x24 + (n) * 4) #define TITSR0_MAX_INT 16 #define TITSEL_WIDTH 0x2 #define TSSR(n) (0x30 + ((n) * 4)) @@ -37,6 +37,8 @@ #define TSSEL_SHIFT(n) (8 * (n)) #define TSSEL_MASK GENMASK(7, 0) #define IRQ_MASK 0x3 +#define IMSK 0x10010 +#define TMSK 0x10020 #define TSSR_OFFSET(n) ((n) % 4) #define TSSR_INDEX(n) ((n) / 4) @@ -53,42 +55,77 @@ #define IITSR_IITSEL_EDGE_BOTH 3 #define IITSR_IITSEL_MASK(n) IITSR_IITSEL((n), 3) -#define TINT_EXTRACT_HWIRQ(x) FIELD_GET(GENMASK(15, 0), (x)) -#define TINT_EXTRACT_GPIOINT(x) FIELD_GET(GENMASK(31, 16), (x)) +#define TINT_EXTRACT_HWIRQ(x) FIELD_GET(GENMASK(15, 0), (x)) +#define TINT_EXTRACT_GPIOINT(x) FIELD_GET(GENMASK(31, 16), (x)) -struct rzg2l_irqc_priv { - void __iomem *base; - struct irq_fwspec fwspec[IRQC_NUM_IRQ]; - raw_spinlock_t lock; +/** + * struct rzg2l_irqc_reg_cache - registers cache (necessary for suspend/resume) + * @iitsr: IITSR register + * @titsr: TITSR registers + */ +struct rzg2l_irqc_reg_cache { + u32 iitsr; + u32 titsr[2]; }; +/** + * struct rzg2l_irqc_priv - IRQ controller private data structure + * @base: Controller's base address + * @irqchip: Pointer to struct irq_chip + * @fwspec: IRQ firmware specific data + * @lock: Lock to serialize access to hardware registers + * @cache: Registers cache for suspend/resume + */ +static struct rzg2l_irqc_priv { + void __iomem *base; + const struct irq_chip *irqchip; + struct irq_fwspec fwspec[IRQC_NUM_IRQ]; + raw_spinlock_t lock; + struct rzg2l_irqc_reg_cache cache; +} *rzg2l_irqc_data; + static struct rzg2l_irqc_priv *irq_data_to_priv(struct irq_data *data) { return data->domain->host_data; } -static void rzg2l_irq_eoi(struct irq_data *d) +static void rzg2l_clear_irq_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq) { - unsigned int hw_irq = irqd_to_hwirq(d) - IRQC_IRQ_START; - struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); + unsigned int hw_irq = hwirq - IRQC_IRQ_START; u32 bit = BIT(hw_irq); - u32 reg; + u32 iitsr, iscr; + + iscr = readl_relaxed(priv->base + ISCR); + iitsr = readl_relaxed(priv->base + IITSR); - reg = readl_relaxed(priv->base + ISCR); - if (reg & bit) - writel_relaxed(reg & ~bit, priv->base + ISCR); + /* + * ISCR can only be cleared if the type is falling-edge, rising-edge or + * falling/rising-edge. + */ + if ((iscr & bit) && (iitsr & IITSR_IITSEL_MASK(hw_irq))) { + writel_relaxed(iscr & ~bit, priv->base + ISCR); + /* + * Enforce that the posted write is flushed to prevent that the + * just handled interrupt is raised again. + */ + readl_relaxed(priv->base + ISCR); + } } -static void rzg2l_tint_eoi(struct irq_data *d) +static void rzg2l_clear_tint_int(struct rzg2l_irqc_priv *priv, unsigned int hwirq) { - unsigned int hw_irq = irqd_to_hwirq(d) - IRQC_TINT_START; - struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); - u32 bit = BIT(hw_irq); + u32 bit = BIT(hwirq - IRQC_TINT_START); u32 reg; reg = readl_relaxed(priv->base + TSCR); - if (reg & bit) + if (reg & bit) { writel_relaxed(reg & ~bit, priv->base + TSCR); + /* + * Enforce that the posted write is flushed to prevent that the + * just handled interrupt is raised again. + */ + readl_relaxed(priv->base + TSCR); + } } static void rzg2l_irqc_eoi(struct irq_data *d) @@ -98,40 +135,124 @@ static void rzg2l_irqc_eoi(struct irq_data *d) raw_spin_lock(&priv->lock); if (hw_irq >= IRQC_IRQ_START && hw_irq <= IRQC_IRQ_COUNT) - rzg2l_irq_eoi(d); + rzg2l_clear_irq_int(priv, hw_irq); else if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ) - rzg2l_tint_eoi(d); + rzg2l_clear_tint_int(priv, hw_irq); raw_spin_unlock(&priv->lock); irq_chip_eoi_parent(d); } -static void rzg2l_irqc_irq_disable(struct irq_data *d) +static void rzfive_irqc_mask_irq_interrupt(struct rzg2l_irqc_priv *priv, + unsigned int hwirq) { - unsigned int hw_irq = irqd_to_hwirq(d); + u32 bit = BIT(hwirq - IRQC_IRQ_START); - if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ) { - struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); - u32 offset = hw_irq - IRQC_TINT_START; + writel_relaxed(readl_relaxed(priv->base + IMSK) | bit, priv->base + IMSK); +} + +static void rzfive_irqc_unmask_irq_interrupt(struct rzg2l_irqc_priv *priv, + unsigned int hwirq) +{ + u32 bit = BIT(hwirq - IRQC_IRQ_START); + + writel_relaxed(readl_relaxed(priv->base + IMSK) & ~bit, priv->base + IMSK); +} + +static void rzfive_irqc_mask_tint_interrupt(struct rzg2l_irqc_priv *priv, + unsigned int hwirq) +{ + u32 bit = BIT(hwirq - IRQC_TINT_START); + + writel_relaxed(readl_relaxed(priv->base + TMSK) | bit, priv->base + TMSK); +} + +static void rzfive_irqc_unmask_tint_interrupt(struct rzg2l_irqc_priv *priv, + unsigned int hwirq) +{ + u32 bit = BIT(hwirq - IRQC_TINT_START); + + writel_relaxed(readl_relaxed(priv->base + TMSK) & ~bit, priv->base + TMSK); +} + +static void rzfive_irqc_mask(struct irq_data *d) +{ + struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); + unsigned int hwirq = irqd_to_hwirq(d); + + raw_spin_lock(&priv->lock); + if (hwirq >= IRQC_IRQ_START && hwirq <= IRQC_IRQ_COUNT) + rzfive_irqc_mask_irq_interrupt(priv, hwirq); + else if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ) + rzfive_irqc_mask_tint_interrupt(priv, hwirq); + raw_spin_unlock(&priv->lock); + irq_chip_mask_parent(d); +} + +static void rzfive_irqc_unmask(struct irq_data *d) +{ + struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); + unsigned int hwirq = irqd_to_hwirq(d); + + raw_spin_lock(&priv->lock); + if (hwirq >= IRQC_IRQ_START && hwirq <= IRQC_IRQ_COUNT) + rzfive_irqc_unmask_irq_interrupt(priv, hwirq); + else if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ) + rzfive_irqc_unmask_tint_interrupt(priv, hwirq); + raw_spin_unlock(&priv->lock); + irq_chip_unmask_parent(d); +} + +static void rzfive_tint_irq_endisable(struct irq_data *d, bool enable) +{ + struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); + unsigned int hwirq = irqd_to_hwirq(d); + + if (hwirq >= IRQC_TINT_START && hwirq < IRQC_NUM_IRQ) { + u32 offset = hwirq - IRQC_TINT_START; u32 tssr_offset = TSSR_OFFSET(offset); u8 tssr_index = TSSR_INDEX(offset); u32 reg; raw_spin_lock(&priv->lock); + if (enable) + rzfive_irqc_unmask_tint_interrupt(priv, hwirq); + else + rzfive_irqc_mask_tint_interrupt(priv, hwirq); reg = readl_relaxed(priv->base + TSSR(tssr_index)); - reg &= ~(TSSEL_MASK << tssr_offset); + if (enable) + reg |= TIEN << TSSEL_SHIFT(tssr_offset); + else + reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset)); writel_relaxed(reg, priv->base + TSSR(tssr_index)); raw_spin_unlock(&priv->lock); + } else { + raw_spin_lock(&priv->lock); + if (enable) + rzfive_irqc_unmask_irq_interrupt(priv, hwirq); + else + rzfive_irqc_mask_irq_interrupt(priv, hwirq); + raw_spin_unlock(&priv->lock); } +} + +static void rzfive_irqc_irq_disable(struct irq_data *d) +{ irq_chip_disable_parent(d); + rzfive_tint_irq_endisable(d, false); } -static void rzg2l_irqc_irq_enable(struct irq_data *d) +static void rzfive_irqc_irq_enable(struct irq_data *d) +{ + rzfive_tint_irq_endisable(d, true); + irq_chip_enable_parent(d); +} + +static void rzg2l_tint_irq_endisable(struct irq_data *d, bool enable) { unsigned int hw_irq = irqd_to_hwirq(d); if (hw_irq >= IRQC_TINT_START && hw_irq < IRQC_NUM_IRQ) { struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); - unsigned long tint = (uintptr_t)d->chip_data; u32 offset = hw_irq - IRQC_TINT_START; u32 tssr_offset = TSSR_OFFSET(offset); u8 tssr_index = TSSR_INDEX(offset); @@ -139,17 +260,33 @@ static void rzg2l_irqc_irq_enable(struct irq_data *d) raw_spin_lock(&priv->lock); reg = readl_relaxed(priv->base + TSSR(tssr_index)); - reg |= (TIEN | tint) << TSSEL_SHIFT(tssr_offset); + if (enable) + reg |= TIEN << TSSEL_SHIFT(tssr_offset); + else + reg &= ~(TIEN << TSSEL_SHIFT(tssr_offset)); writel_relaxed(reg, priv->base + TSSR(tssr_index)); raw_spin_unlock(&priv->lock); } +} + +static void rzg2l_irqc_irq_disable(struct irq_data *d) +{ + irq_chip_disable_parent(d); + rzg2l_tint_irq_endisable(d, false); +} + +static void rzg2l_irqc_irq_enable(struct irq_data *d) +{ + rzg2l_tint_irq_endisable(d, true); irq_chip_enable_parent(d); } static int rzg2l_irq_set_type(struct irq_data *d, unsigned int type) { - unsigned int hw_irq = irqd_to_hwirq(d) - IRQC_IRQ_START; struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); + unsigned int hwirq = irqd_to_hwirq(d); + u32 iitseln = hwirq - IRQC_IRQ_START; + bool clear_irq_int = false; u16 sense, tmp; switch (type & IRQ_TYPE_SENSE_MASK) { @@ -159,14 +296,17 @@ static int rzg2l_irq_set_type(struct irq_data *d, unsigned int type) case IRQ_TYPE_EDGE_FALLING: sense = IITSR_IITSEL_EDGE_FALLING; + clear_irq_int = true; break; case IRQ_TYPE_EDGE_RISING: sense = IITSR_IITSEL_EDGE_RISING; + clear_irq_int = true; break; case IRQ_TYPE_EDGE_BOTH: sense = IITSR_IITSEL_EDGE_BOTH; + clear_irq_int = true; break; default: @@ -175,22 +315,40 @@ static int rzg2l_irq_set_type(struct irq_data *d, unsigned int type) raw_spin_lock(&priv->lock); tmp = readl_relaxed(priv->base + IITSR); - tmp &= ~IITSR_IITSEL_MASK(hw_irq); - tmp |= IITSR_IITSEL(hw_irq, sense); + tmp &= ~IITSR_IITSEL_MASK(iitseln); + tmp |= IITSR_IITSEL(iitseln, sense); + if (clear_irq_int) + rzg2l_clear_irq_int(priv, hwirq); writel_relaxed(tmp, priv->base + IITSR); raw_spin_unlock(&priv->lock); return 0; } +static u32 rzg2l_disable_tint_and_set_tint_source(struct irq_data *d, struct rzg2l_irqc_priv *priv, + u32 reg, u32 tssr_offset, u8 tssr_index) +{ + u32 tint = (u32)(uintptr_t)irq_data_get_irq_chip_data(d); + u32 tien = reg & (TIEN << TSSEL_SHIFT(tssr_offset)); + + /* Clear the relevant byte in reg */ + reg &= ~(TSSEL_MASK << TSSEL_SHIFT(tssr_offset)); + /* Set TINT and leave TIEN clear */ + reg |= tint << TSSEL_SHIFT(tssr_offset); + writel_relaxed(reg, priv->base + TSSR(tssr_index)); + + return reg | tien; +} + static int rzg2l_tint_set_edge(struct irq_data *d, unsigned int type) { struct rzg2l_irqc_priv *priv = irq_data_to_priv(d); unsigned int hwirq = irqd_to_hwirq(d); u32 titseln = hwirq - IRQC_TINT_START; - u32 offset; - u8 sense; - u32 reg; + u32 tssr_offset = TSSR_OFFSET(titseln); + u8 tssr_index = TSSR_INDEX(titseln); + u8 index, sense; + u32 reg, tssr; switch (type & IRQ_TYPE_SENSE_MASK) { case IRQ_TYPE_EDGE_RISING: @@ -205,17 +363,21 @@ static int rzg2l_tint_set_edge(struct irq_data *d, unsigned int type) return -EINVAL; } - offset = TITSR0; + index = 0; if (titseln >= TITSR0_MAX_INT) { titseln -= TITSR0_MAX_INT; - offset = TITSR1; + index = 1; } raw_spin_lock(&priv->lock); - reg = readl_relaxed(priv->base + offset); + tssr = readl_relaxed(priv->base + TSSR(tssr_index)); + tssr = rzg2l_disable_tint_and_set_tint_source(d, priv, tssr, tssr_offset, tssr_index); + reg = readl_relaxed(priv->base + TITSR(index)); reg &= ~(IRQ_MASK << (titseln * TITSEL_WIDTH)); reg |= sense << (titseln * TITSEL_WIDTH); - writel_relaxed(reg, priv->base + offset); + writel_relaxed(reg, priv->base + TITSR(index)); + rzg2l_clear_tint_int(priv, hwirq); + writel_relaxed(tssr, priv->base + TSSR(tssr_index)); raw_spin_unlock(&priv->lock); return 0; @@ -236,7 +398,43 @@ static int rzg2l_irqc_set_type(struct irq_data *d, unsigned int type) return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH); } -static const struct irq_chip irqc_chip = { +static int rzg2l_irqc_irq_suspend(void *data) +{ + struct rzg2l_irqc_reg_cache *cache = &rzg2l_irqc_data->cache; + void __iomem *base = rzg2l_irqc_data->base; + + cache->iitsr = readl_relaxed(base + IITSR); + for (u8 i = 0; i < 2; i++) + cache->titsr[i] = readl_relaxed(base + TITSR(i)); + + return 0; +} + +static void rzg2l_irqc_irq_resume(void *data) +{ + struct rzg2l_irqc_reg_cache *cache = &rzg2l_irqc_data->cache; + void __iomem *base = rzg2l_irqc_data->base; + + /* + * Restore only interrupt type. TSSRx will be restored at the + * request of pin controller to avoid spurious interrupts due + * to invalid PIN states. + */ + for (u8 i = 0; i < 2; i++) + writel_relaxed(cache->titsr[i], base + TITSR(i)); + writel_relaxed(cache->iitsr, base + IITSR); +} + +static const struct syscore_ops rzg2l_irqc_syscore_ops = { + .suspend = rzg2l_irqc_irq_suspend, + .resume = rzg2l_irqc_irq_resume, +}; + +static struct syscore rzg2l_irqc_syscore = { + .ops = &rzg2l_irqc_syscore_ops, +}; + +static const struct irq_chip rzg2l_irqc_chip = { .name = "rzg2l-irqc", .irq_eoi = rzg2l_irqc_eoi, .irq_mask = irq_chip_mask_parent, @@ -247,6 +445,24 @@ static const struct irq_chip irqc_chip = { .irq_set_irqchip_state = irq_chip_set_parent_state, .irq_retrigger = irq_chip_retrigger_hierarchy, .irq_set_type = rzg2l_irqc_set_type, + .irq_set_affinity = irq_chip_set_affinity_parent, + .flags = IRQCHIP_MASK_ON_SUSPEND | + IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE, +}; + +static const struct irq_chip rzfive_irqc_chip = { + .name = "rzfive-irqc", + .irq_eoi = rzg2l_irqc_eoi, + .irq_mask = rzfive_irqc_mask, + .irq_unmask = rzfive_irqc_unmask, + .irq_disable = rzfive_irqc_irq_disable, + .irq_enable = rzfive_irqc_irq_enable, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_retrigger = irq_chip_retrigger_hierarchy, + .irq_set_type = rzg2l_irqc_set_type, + .irq_set_affinity = irq_chip_set_affinity_parent, .flags = IRQCHIP_MASK_ON_SUSPEND | IRQCHIP_SET_TYPE_MASKED | IRQCHIP_SKIP_SET_WAKE, @@ -283,7 +499,7 @@ static int rzg2l_irqc_alloc(struct irq_domain *domain, unsigned int virq, if (hwirq > (IRQC_NUM_IRQ - 1)) return -EINVAL; - ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &irqc_chip, + ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, priv->irqchip, (void *)(uintptr_t)tint); if (ret) return ret; @@ -315,78 +531,74 @@ static int rzg2l_irqc_parse_interrupts(struct rzg2l_irqc_priv *priv, return 0; } -static int rzg2l_irqc_init(struct device_node *node, struct device_node *parent) +static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_node *parent, + const struct irq_chip *irq_chip) { struct irq_domain *irq_domain, *parent_domain; - struct platform_device *pdev; + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; struct reset_control *resetn; - struct rzg2l_irqc_priv *priv; int ret; - pdev = of_find_device_by_node(node); - if (!pdev) - return -ENODEV; - parent_domain = irq_find_host(parent); - if (!parent_domain) { - dev_err(&pdev->dev, "cannot find parent domain\n"); - return -ENODEV; - } + if (!parent_domain) + return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n"); - priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); - if (!priv) + rzg2l_irqc_data = devm_kzalloc(dev, sizeof(*rzg2l_irqc_data), GFP_KERNEL); + if (!rzg2l_irqc_data) return -ENOMEM; - priv->base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL); - if (IS_ERR(priv->base)) - return PTR_ERR(priv->base); + rzg2l_irqc_data->irqchip = irq_chip; - ret = rzg2l_irqc_parse_interrupts(priv, node); - if (ret) { - dev_err(&pdev->dev, "cannot parse interrupts: %d\n", ret); - return ret; - } + rzg2l_irqc_data->base = devm_of_iomap(dev, dev->of_node, 0, NULL); + if (IS_ERR(rzg2l_irqc_data->base)) + return PTR_ERR(rzg2l_irqc_data->base); - resetn = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(resetn)) - return PTR_ERR(resetn); + ret = rzg2l_irqc_parse_interrupts(rzg2l_irqc_data, node); + if (ret) + return dev_err_probe(dev, ret, "cannot parse interrupts: %d\n", ret); - ret = reset_control_deassert(resetn); - if (ret) { - dev_err(&pdev->dev, "failed to deassert resetn pin, %d\n", ret); - return ret; + resetn = devm_reset_control_get_exclusive_deasserted(dev, NULL); + if (IS_ERR(resetn)) { + return dev_err_probe(dev, PTR_ERR(resetn), + "failed to acquire deasserted reset: %d\n", ret); } - pm_runtime_enable(&pdev->dev); - ret = pm_runtime_resume_and_get(&pdev->dev); - if (ret < 0) { - dev_err(&pdev->dev, "pm_runtime_resume_and_get failed: %d\n", ret); - goto pm_disable; - } + ret = devm_pm_runtime_enable(dev); + if (ret) + return dev_err_probe(dev, ret, "devm_pm_runtime_enable failed: %d\n", ret); + + ret = pm_runtime_resume_and_get(dev); + if (ret) + return dev_err_probe(dev, ret, "pm_runtime_resume_and_get failed: %d\n", ret); - raw_spin_lock_init(&priv->lock); + raw_spin_lock_init(&rzg2l_irqc_data->lock); - irq_domain = irq_domain_add_hierarchy(parent_domain, 0, IRQC_NUM_IRQ, - node, &rzg2l_irqc_domain_ops, - priv); + irq_domain = irq_domain_create_hierarchy(parent_domain, 0, IRQC_NUM_IRQ, dev_fwnode(dev), + &rzg2l_irqc_domain_ops, rzg2l_irqc_data); if (!irq_domain) { - dev_err(&pdev->dev, "failed to add irq domain\n"); - ret = -ENOMEM; - goto pm_put; + pm_runtime_put(dev); + return -ENOMEM; } + register_syscore(&rzg2l_irqc_syscore); + return 0; +} -pm_put: - pm_runtime_put(&pdev->dev); -pm_disable: - pm_runtime_disable(&pdev->dev); - reset_control_assert(resetn); - return ret; +static int rzg2l_irqc_probe(struct platform_device *pdev, struct device_node *parent) +{ + return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_chip); +} + +static int rzfive_irqc_probe(struct platform_device *pdev, struct device_node *parent) +{ + return rzg2l_irqc_common_probe(pdev, parent, &rzfive_irqc_chip); } IRQCHIP_PLATFORM_DRIVER_BEGIN(rzg2l_irqc) -IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_init) +IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_probe) +IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_probe) IRQCHIP_PLATFORM_DRIVER_END(rzg2l_irqc) MODULE_AUTHOR("Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>"); MODULE_DESCRIPTION("Renesas RZ/G2L IRQC Driver"); diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c new file mode 100644 index 000000000000..899a423b5da8 --- /dev/null +++ b/drivers/irqchip/irq-renesas-rzv2h.c @@ -0,0 +1,622 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas RZ/V2H(P) ICU Driver + * + * Based on irq-renesas-rzg2l.c + * + * Copyright (C) 2024 Renesas Electronics Corporation. + * + * Author: Fabrizio Castro <fabrizio.castro.jz@renesas.com> + */ + +#include <linux/bitfield.h> +#include <linux/cleanup.h> +#include <linux/err.h> +#include <linux/io.h> +#include <linux/irqchip.h> +#include <linux/irqchip/irq-renesas-rzv2h.h> +#include <linux/irqdomain.h> +#include <linux/of_platform.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/spinlock.h> + +/* DT "interrupts" indexes */ +#define ICU_IRQ_START 1 +#define ICU_IRQ_COUNT 16 +#define ICU_TINT_START (ICU_IRQ_START + ICU_IRQ_COUNT) +#define ICU_TINT_COUNT 32 +#define ICU_NUM_IRQ (ICU_TINT_START + ICU_TINT_COUNT) + +/* Registers */ +#define ICU_NSCNT 0x00 +#define ICU_NSCLR 0x04 +#define ICU_NITSR 0x08 +#define ICU_ISCTR 0x10 +#define ICU_ISCLR 0x14 +#define ICU_IITSR 0x18 +#define ICU_TSCTR 0x20 +#define ICU_TSCLR 0x24 +#define ICU_TITSR(k) (0x28 + (k) * 4) +#define ICU_TSSR(k) (0x30 + (k) * 4) +#define ICU_DMkSELy(k, y) (0x420 + (k) * 0x20 + (y) * 4) +#define ICU_DMACKSELk(k) (0x500 + (k) * 4) + +/* NMI */ +#define ICU_NMI_EDGE_FALLING 0 +#define ICU_NMI_EDGE_RISING 1 + +#define ICU_NSCLR_NCLR BIT(0) + +/* IRQ */ +#define ICU_IRQ_LEVEL_LOW 0 +#define ICU_IRQ_EDGE_FALLING 1 +#define ICU_IRQ_EDGE_RISING 2 +#define ICU_IRQ_EDGE_BOTH 3 + +#define ICU_IITSR_IITSEL_PREP(iitsel, n) ((iitsel) << ((n) * 2)) +#define ICU_IITSR_IITSEL_GET(iitsr, n) (((iitsr) >> ((n) * 2)) & 0x03) +#define ICU_IITSR_IITSEL_MASK(n) ICU_IITSR_IITSEL_PREP(0x03, n) + +/* TINT */ +#define ICU_TINT_EDGE_RISING 0 +#define ICU_TINT_EDGE_FALLING 1 +#define ICU_TINT_LEVEL_HIGH 2 +#define ICU_TINT_LEVEL_LOW 3 + +#define ICU_TSSR_TSSEL_PREP(tssel, n, field_width) ((tssel) << ((n) * (field_width))) +#define ICU_TSSR_TSSEL_MASK(n, field_width) \ +({\ + typeof(field_width) (_field_width) = (field_width); \ + ICU_TSSR_TSSEL_PREP((GENMASK(((_field_width) - 2), 0)), (n), _field_width); \ +}) + +#define ICU_TSSR_TIEN(n, field_width) \ +({\ + typeof(field_width) (_field_width) = (field_width); \ + BIT((_field_width) - 1) << ((n) * (_field_width)); \ +}) + +#define ICU_TITSR_K(tint_nr) ((tint_nr) / 16) +#define ICU_TITSR_TITSEL_N(tint_nr) ((tint_nr) % 16) +#define ICU_TITSR_TITSEL_PREP(titsel, n) ICU_IITSR_IITSEL_PREP(titsel, n) +#define ICU_TITSR_TITSEL_MASK(n) ICU_IITSR_IITSEL_MASK(n) +#define ICU_TITSR_TITSEL_GET(titsr, n) ICU_IITSR_IITSEL_GET(titsr, n) + +#define ICU_TINT_EXTRACT_HWIRQ(x) FIELD_GET(GENMASK(15, 0), (x)) +#define ICU_TINT_EXTRACT_GPIOINT(x) FIELD_GET(GENMASK(31, 16), (x)) +#define ICU_RZG3E_TINT_OFFSET 0x800 +#define ICU_RZG3E_TSSEL_MAX_VAL 0x8c +#define ICU_RZV2H_TSSEL_MAX_VAL 0x55 + +/** + * struct rzv2h_hw_info - Interrupt Control Unit controller hardware info structure. + * @tssel_lut: TINT lookup table + * @t_offs: TINT offset + * @max_tssel: TSSEL max value + * @field_width: TSSR field width + */ +struct rzv2h_hw_info { + const u8 *tssel_lut; + u16 t_offs; + u8 max_tssel; + u8 field_width; +}; + +/* DMAC */ +#define ICU_DMAC_DkRQ_SEL_MASK GENMASK(9, 0) + +#define ICU_DMAC_DMAREQ_SHIFT(up) ((up) * 16) +#define ICU_DMAC_DMAREQ_MASK(up) (ICU_DMAC_DkRQ_SEL_MASK \ + << ICU_DMAC_DMAREQ_SHIFT(up)) +#define ICU_DMAC_PREP_DMAREQ(sel, up) (FIELD_PREP(ICU_DMAC_DkRQ_SEL_MASK, (sel)) \ + << ICU_DMAC_DMAREQ_SHIFT(up)) + +/** + * struct rzv2h_icu_priv - Interrupt Control Unit controller private data structure. + * @base: Controller's base address + * @fwspec: IRQ firmware specific data + * @lock: Lock to serialize access to hardware registers + * @info: Pointer to struct rzv2h_hw_info + */ +struct rzv2h_icu_priv { + void __iomem *base; + struct irq_fwspec fwspec[ICU_NUM_IRQ]; + raw_spinlock_t lock; + const struct rzv2h_hw_info *info; +}; + +void rzv2h_icu_register_dma_req(struct platform_device *icu_dev, u8 dmac_index, u8 dmac_channel, + u16 req_no) +{ + struct rzv2h_icu_priv *priv = platform_get_drvdata(icu_dev); + u32 icu_dmksely, dmareq, dmareq_mask; + u8 y, upper; + + y = dmac_channel / 2; + upper = dmac_channel % 2; + + dmareq = ICU_DMAC_PREP_DMAREQ(req_no, upper); + dmareq_mask = ICU_DMAC_DMAREQ_MASK(upper); + + guard(raw_spinlock_irqsave)(&priv->lock); + + icu_dmksely = readl(priv->base + ICU_DMkSELy(dmac_index, y)); + icu_dmksely = (icu_dmksely & ~dmareq_mask) | dmareq; + writel(icu_dmksely, priv->base + ICU_DMkSELy(dmac_index, y)); +} +EXPORT_SYMBOL_GPL(rzv2h_icu_register_dma_req); + +static inline struct rzv2h_icu_priv *irq_data_to_priv(struct irq_data *data) +{ + return data->domain->host_data; +} + +static void rzv2h_icu_eoi(struct irq_data *d) +{ + struct rzv2h_icu_priv *priv = irq_data_to_priv(d); + unsigned int hw_irq = irqd_to_hwirq(d); + unsigned int tintirq_nr; + u32 bit; + + scoped_guard(raw_spinlock, &priv->lock) { + if (hw_irq >= ICU_TINT_START) { + tintirq_nr = hw_irq - ICU_TINT_START; + bit = BIT(tintirq_nr); + if (!irqd_is_level_type(d)) + writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR); + } else if (hw_irq >= ICU_IRQ_START) { + tintirq_nr = hw_irq - ICU_IRQ_START; + bit = BIT(tintirq_nr); + if (!irqd_is_level_type(d)) + writel_relaxed(bit, priv->base + ICU_ISCLR); + } else { + writel_relaxed(ICU_NSCLR_NCLR, priv->base + ICU_NSCLR); + } + } + + irq_chip_eoi_parent(d); +} + +static void rzv2h_tint_irq_endisable(struct irq_data *d, bool enable) +{ + struct rzv2h_icu_priv *priv = irq_data_to_priv(d); + unsigned int hw_irq = irqd_to_hwirq(d); + u32 tint_nr, tssel_n, k, tssr; + u8 nr_tint; + + if (hw_irq < ICU_TINT_START) + return; + + tint_nr = hw_irq - ICU_TINT_START; + nr_tint = 32 / priv->info->field_width; + k = tint_nr / nr_tint; + tssel_n = tint_nr % nr_tint; + + guard(raw_spinlock)(&priv->lock); + tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(k)); + if (enable) + tssr |= ICU_TSSR_TIEN(tssel_n, priv->info->field_width); + else + tssr &= ~ICU_TSSR_TIEN(tssel_n, priv->info->field_width); + writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(k)); + + /* + * A glitch in the edge detection circuit can cause a spurious + * interrupt. Clear the status flag after setting the ICU_TSSRk + * registers, which is recommended by the hardware manual as a + * countermeasure. + */ + writel_relaxed(BIT(tint_nr), priv->base + priv->info->t_offs + ICU_TSCLR); +} + +static void rzv2h_icu_irq_disable(struct irq_data *d) +{ + irq_chip_disable_parent(d); + rzv2h_tint_irq_endisable(d, false); +} + +static void rzv2h_icu_irq_enable(struct irq_data *d) +{ + rzv2h_tint_irq_endisable(d, true); + irq_chip_enable_parent(d); +} + +static int rzv2h_nmi_set_type(struct irq_data *d, unsigned int type) +{ + struct rzv2h_icu_priv *priv = irq_data_to_priv(d); + u32 sense; + + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_EDGE_FALLING: + sense = ICU_NMI_EDGE_FALLING; + break; + + case IRQ_TYPE_EDGE_RISING: + sense = ICU_NMI_EDGE_RISING; + break; + + default: + return -EINVAL; + } + + writel_relaxed(sense, priv->base + ICU_NITSR); + + return 0; +} + +static void rzv2h_clear_irq_int(struct rzv2h_icu_priv *priv, unsigned int hwirq) +{ + unsigned int irq_nr = hwirq - ICU_IRQ_START; + u32 isctr, iitsr, iitsel; + u32 bit = BIT(irq_nr); + + isctr = readl_relaxed(priv->base + ICU_ISCTR); + iitsr = readl_relaxed(priv->base + ICU_IITSR); + iitsel = ICU_IITSR_IITSEL_GET(iitsr, irq_nr); + + /* + * When level sensing is used, the interrupt flag gets automatically cleared when the + * interrupt signal is de-asserted by the source of the interrupt request, therefore clear + * the interrupt only for edge triggered interrupts. + */ + if ((isctr & bit) && (iitsel != ICU_IRQ_LEVEL_LOW)) + writel_relaxed(bit, priv->base + ICU_ISCLR); +} + +static int rzv2h_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct rzv2h_icu_priv *priv = irq_data_to_priv(d); + unsigned int hwirq = irqd_to_hwirq(d); + u32 irq_nr = hwirq - ICU_IRQ_START; + u32 iitsr, sense; + + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_LEVEL_LOW: + sense = ICU_IRQ_LEVEL_LOW; + break; + + case IRQ_TYPE_EDGE_FALLING: + sense = ICU_IRQ_EDGE_FALLING; + break; + + case IRQ_TYPE_EDGE_RISING: + sense = ICU_IRQ_EDGE_RISING; + break; + + case IRQ_TYPE_EDGE_BOTH: + sense = ICU_IRQ_EDGE_BOTH; + break; + + default: + return -EINVAL; + } + + guard(raw_spinlock)(&priv->lock); + iitsr = readl_relaxed(priv->base + ICU_IITSR); + iitsr &= ~ICU_IITSR_IITSEL_MASK(irq_nr); + iitsr |= ICU_IITSR_IITSEL_PREP(sense, irq_nr); + rzv2h_clear_irq_int(priv, hwirq); + writel_relaxed(iitsr, priv->base + ICU_IITSR); + + return 0; +} + +static void rzv2h_clear_tint_int(struct rzv2h_icu_priv *priv, unsigned int hwirq) +{ + unsigned int tint_nr = hwirq - ICU_TINT_START; + int titsel_n = ICU_TITSR_TITSEL_N(tint_nr); + u32 tsctr, titsr, titsel; + u32 bit = BIT(tint_nr); + int k = tint_nr / 16; + + tsctr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSCTR); + titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(k)); + titsel = ICU_TITSR_TITSEL_GET(titsr, titsel_n); + + /* + * Writing 1 to the corresponding flag from register ICU_TSCTR only has effect if + * TSTATn = 1b and if it's a rising edge or a falling edge interrupt. + */ + if ((tsctr & bit) && ((titsel == ICU_TINT_EDGE_RISING) || + (titsel == ICU_TINT_EDGE_FALLING))) + writel_relaxed(bit, priv->base + priv->info->t_offs + ICU_TSCLR); +} + +static int rzv2h_tint_set_type(struct irq_data *d, unsigned int type) +{ + u32 titsr, titsr_k, titsel_n, tien; + struct rzv2h_icu_priv *priv; + u32 tssr, tssr_k, tssel_n; + unsigned int hwirq; + u32 tint, sense; + int tint_nr; + u8 nr_tint; + + switch (type & IRQ_TYPE_SENSE_MASK) { + case IRQ_TYPE_LEVEL_LOW: + sense = ICU_TINT_LEVEL_LOW; + break; + + case IRQ_TYPE_LEVEL_HIGH: + sense = ICU_TINT_LEVEL_HIGH; + break; + + case IRQ_TYPE_EDGE_RISING: + sense = ICU_TINT_EDGE_RISING; + break; + + case IRQ_TYPE_EDGE_FALLING: + sense = ICU_TINT_EDGE_FALLING; + break; + + default: + return -EINVAL; + } + + priv = irq_data_to_priv(d); + tint = (u32)(uintptr_t)irq_data_get_irq_chip_data(d); + if (tint > priv->info->max_tssel) + return -EINVAL; + + if (priv->info->tssel_lut) + tint = priv->info->tssel_lut[tint]; + + hwirq = irqd_to_hwirq(d); + tint_nr = hwirq - ICU_TINT_START; + + nr_tint = 32 / priv->info->field_width; + tssr_k = tint_nr / nr_tint; + tssel_n = tint_nr % nr_tint; + tien = ICU_TSSR_TIEN(tssel_n, priv->info->field_width); + + titsr_k = ICU_TITSR_K(tint_nr); + titsel_n = ICU_TITSR_TITSEL_N(tint_nr); + + guard(raw_spinlock)(&priv->lock); + + tssr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TSSR(tssr_k)); + tssr &= ~(ICU_TSSR_TSSEL_MASK(tssel_n, priv->info->field_width) | tien); + tssr |= ICU_TSSR_TSSEL_PREP(tint, tssel_n, priv->info->field_width); + + writel_relaxed(tssr, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k)); + + titsr = readl_relaxed(priv->base + priv->info->t_offs + ICU_TITSR(titsr_k)); + titsr &= ~ICU_TITSR_TITSEL_MASK(titsel_n); + titsr |= ICU_TITSR_TITSEL_PREP(sense, titsel_n); + + writel_relaxed(titsr, priv->base + priv->info->t_offs + ICU_TITSR(titsr_k)); + + rzv2h_clear_tint_int(priv, hwirq); + + writel_relaxed(tssr | tien, priv->base + priv->info->t_offs + ICU_TSSR(tssr_k)); + + return 0; +} + +static int rzv2h_icu_set_type(struct irq_data *d, unsigned int type) +{ + unsigned int hw_irq = irqd_to_hwirq(d); + int ret; + + if (hw_irq >= ICU_TINT_START) + ret = rzv2h_tint_set_type(d, type); + else if (hw_irq >= ICU_IRQ_START) + ret = rzv2h_irq_set_type(d, type); + else + ret = rzv2h_nmi_set_type(d, type); + + if (ret) + return ret; + + return irq_chip_set_type_parent(d, IRQ_TYPE_LEVEL_HIGH); +} + +static const struct irq_chip rzv2h_icu_chip = { + .name = "rzv2h-icu", + .irq_eoi = rzv2h_icu_eoi, + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, + .irq_disable = rzv2h_icu_irq_disable, + .irq_enable = rzv2h_icu_irq_enable, + .irq_get_irqchip_state = irq_chip_get_parent_state, + .irq_set_irqchip_state = irq_chip_set_parent_state, + .irq_retrigger = irq_chip_retrigger_hierarchy, + .irq_set_type = rzv2h_icu_set_type, + .irq_set_affinity = irq_chip_set_affinity_parent, + .flags = IRQCHIP_MASK_ON_SUSPEND | + IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE, +}; + +static int rzv2h_icu_alloc(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs, + void *arg) +{ + struct rzv2h_icu_priv *priv = domain->host_data; + unsigned long tint = 0; + irq_hw_number_t hwirq; + unsigned int type; + int ret; + + ret = irq_domain_translate_twocell(domain, arg, &hwirq, &type); + if (ret) + return ret; + + /* + * For TINT interrupts the hwirq and TINT are encoded in + * fwspec->param[0]. + * hwirq is embedded in bits 0-15. + * TINT is embedded in bits 16-31. + */ + if (hwirq >= ICU_TINT_START) { + tint = ICU_TINT_EXTRACT_GPIOINT(hwirq); + hwirq = ICU_TINT_EXTRACT_HWIRQ(hwirq); + + if (hwirq < ICU_TINT_START) + return -EINVAL; + } + + if (hwirq > (ICU_NUM_IRQ - 1)) + return -EINVAL; + + ret = irq_domain_set_hwirq_and_chip(domain, virq, hwirq, &rzv2h_icu_chip, + (void *)(uintptr_t)tint); + if (ret) + return ret; + + return irq_domain_alloc_irqs_parent(domain, virq, nr_irqs, &priv->fwspec[hwirq]); +} + +static const struct irq_domain_ops rzv2h_icu_domain_ops = { + .alloc = rzv2h_icu_alloc, + .free = irq_domain_free_irqs_common, + .translate = irq_domain_translate_twocell, +}; + +static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device_node *np) +{ + struct of_phandle_args map; + unsigned int i; + int ret; + + for (i = 0; i < ICU_NUM_IRQ; i++) { + ret = of_irq_parse_one(np, i, &map); + if (ret) + return ret; + + of_phandle_args_to_fwspec(np, map.args, map.args_count, &priv->fwspec[i]); + } + + return 0; +} + +static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_node *parent, + const struct rzv2h_hw_info *hw_info) +{ + struct irq_domain *irq_domain, *parent_domain; + struct device_node *node = pdev->dev.of_node; + struct rzv2h_icu_priv *rzv2h_icu_data; + struct reset_control *resetn; + int ret; + + parent_domain = irq_find_host(parent); + if (!parent_domain) { + dev_err(&pdev->dev, "cannot find parent domain\n"); + return -ENODEV; + } + + rzv2h_icu_data = devm_kzalloc(&pdev->dev, sizeof(*rzv2h_icu_data), GFP_KERNEL); + if (!rzv2h_icu_data) + return -ENOMEM; + + platform_set_drvdata(pdev, rzv2h_icu_data); + + rzv2h_icu_data->base = devm_of_iomap(&pdev->dev, pdev->dev.of_node, 0, NULL); + if (IS_ERR(rzv2h_icu_data->base)) + return PTR_ERR(rzv2h_icu_data->base); + + ret = rzv2h_icu_parse_interrupts(rzv2h_icu_data, node); + if (ret) { + dev_err(&pdev->dev, "cannot parse interrupts: %d\n", ret); + return ret; + } + + resetn = devm_reset_control_get_exclusive_deasserted(&pdev->dev, NULL); + if (IS_ERR(resetn)) { + ret = PTR_ERR(resetn); + dev_err(&pdev->dev, "failed to acquire deasserted reset: %d\n", ret); + return ret; + } + + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "devm_pm_runtime_enable failed, %d\n", ret); + return ret; + } + + ret = pm_runtime_resume_and_get(&pdev->dev); + if (ret < 0) { + dev_err(&pdev->dev, "pm_runtime_resume_and_get failed: %d\n", ret); + return ret; + } + + raw_spin_lock_init(&rzv2h_icu_data->lock); + + irq_domain = irq_domain_create_hierarchy(parent_domain, 0, ICU_NUM_IRQ, + dev_fwnode(&pdev->dev), &rzv2h_icu_domain_ops, + rzv2h_icu_data); + if (!irq_domain) { + dev_err(&pdev->dev, "failed to add irq domain\n"); + ret = -ENOMEM; + goto pm_put; + } + + rzv2h_icu_data->info = hw_info; + + /* + * coccicheck complains about a missing put_device call before returning, but it's a false + * positive. We still need &pdev->dev after successfully returning from this function. + */ + return 0; + +pm_put: + pm_runtime_put(&pdev->dev); + + return ret; +} + +/* Mapping based on port index on Table 4.2-6 and TSSEL bits on Table 4.6-4 */ +static const u8 rzg3e_tssel_lut[] = { + 81, 82, 83, 84, 85, 86, 87, 88, /* P00-P07 */ + 89, 90, 91, 92, 93, 94, 95, 96, /* P10-P17 */ + 111, 112, /* P20-P21 */ + 97, 98, 99, 100, 101, 102, 103, 104, /* P30-P37 */ + 105, 106, 107, 108, 109, 110, /* P40-P45 */ + 113, 114, 115, 116, 117, 118, 119, /* P50-P56 */ + 120, 121, 122, 123, 124, 125, 126, /* P60-P66 */ + 127, 128, 129, 130, 131, 132, 133, 134, /* P70-P77 */ + 135, 136, 137, 138, 139, 140, /* P80-P85 */ + 43, 44, 45, 46, 47, 48, 49, 50, /* PA0-PA7 */ + 51, 52, 53, 54, 55, 56, 57, 58, /* PB0-PB7 */ + 59, 60, 61, /* PC0-PC2 */ + 62, 63, 64, 65, 66, 67, 68, 69, /* PD0-PD7 */ + 70, 71, 72, 73, 74, 75, 76, 77, /* PE0-PE7 */ + 78, 79, 80, /* PF0-PF2 */ + 25, 26, 27, 28, 29, 30, 31, 32, /* PG0-PG7 */ + 33, 34, 35, 36, 37, 38, /* PH0-PH5 */ + 4, 5, 6, 7, 8, /* PJ0-PJ4 */ + 39, 40, 41, 42, /* PK0-PK3 */ + 9, 10, 11, 12, 21, 22, 23, 24, /* PL0-PL7 */ + 13, 14, 15, 16, 17, 18, 19, 20, /* PM0-PM7 */ + 0, 1, 2, 3 /* PS0-PS3 */ +}; + +static const struct rzv2h_hw_info rzg3e_hw_params = { + .tssel_lut = rzg3e_tssel_lut, + .t_offs = ICU_RZG3E_TINT_OFFSET, + .max_tssel = ICU_RZG3E_TSSEL_MAX_VAL, + .field_width = 16, +}; + +static const struct rzv2h_hw_info rzv2h_hw_params = { + .t_offs = 0, + .max_tssel = ICU_RZV2H_TSSEL_MAX_VAL, + .field_width = 8, +}; + +static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *parent) +{ + return rzv2h_icu_probe_common(pdev, parent, &rzg3e_hw_params); +} + +static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *parent) +{ + return rzv2h_icu_probe_common(pdev, parent, &rzv2h_hw_params); +} + +IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu) +IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe) +IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe) +IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu) +MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>"); +MODULE_DESCRIPTION("Renesas RZ/V2H(P) ICU Driver"); diff --git a/drivers/irqchip/irq-riscv-aplic-direct.c b/drivers/irqchip/irq-riscv-aplic-direct.c new file mode 100644 index 000000000000..c2a75bf3d20c --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-direct.c @@ -0,0 +1,333 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/printk.h> +#include <linux/smp.h> + +#include "irq-riscv-aplic-main.h" + +#define APLIC_DISABLE_IDELIVERY 0 +#define APLIC_ENABLE_IDELIVERY 1 +#define APLIC_DISABLE_ITHRESHOLD 1 +#define APLIC_ENABLE_ITHRESHOLD 0 + +struct aplic_direct { + struct aplic_priv priv; + struct irq_domain *irqdomain; + struct cpumask lmask; +}; + +struct aplic_idc { + u32 hart_index; + void __iomem *regs; + struct aplic_direct *direct; +}; + +static unsigned int aplic_direct_parent_irq; +static DEFINE_PER_CPU(struct aplic_idc, aplic_idcs); + +static void aplic_direct_irq_eoi(struct irq_data *d) +{ + /* + * The fasteoi_handler requires irq_eoi() callback hence + * provide a dummy handler. + */ +} + +#ifdef CONFIG_SMP +static int aplic_direct_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv); + struct aplic_idc *idc; + unsigned int cpu, val; + void __iomem *target; + + if (force) + cpu = cpumask_first_and(&direct->lmask, mask_val); + else + cpu = cpumask_first_and_and(&direct->lmask, mask_val, cpu_online_mask); + + if (cpu >= nr_cpu_ids) + return -EINVAL; + + idc = per_cpu_ptr(&aplic_idcs, cpu); + target = priv->regs + APLIC_TARGET_BASE + (d->hwirq - 1) * sizeof(u32); + val = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index); + val |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY); + writel(val, target); + + irq_data_update_effective_affinity(d, cpumask_of(cpu)); + + return IRQ_SET_MASK_OK_DONE; +} +#endif + +static struct irq_chip aplic_direct_chip = { + .name = "APLIC-DIRECT", + .irq_mask = aplic_irq_mask, + .irq_unmask = aplic_irq_unmask, + .irq_set_type = aplic_irq_set_type, + .irq_eoi = aplic_direct_irq_eoi, +#ifdef CONFIG_SMP + .irq_set_affinity = aplic_direct_set_affinity, +#endif + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, +}; + +static int aplic_direct_irqdomain_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) +{ + struct aplic_priv *priv = d->host_data; + + return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type); +} + +static int aplic_direct_irqdomain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *arg) +{ + struct aplic_priv *priv = domain->host_data; + struct aplic_direct *direct = container_of(priv, struct aplic_direct, priv); + struct irq_fwspec *fwspec = arg; + irq_hw_number_t hwirq; + unsigned int type; + int i, ret; + + ret = aplic_irqdomain_translate(fwspec, priv->gsi_base, &hwirq, &type); + if (ret) + return ret; + + for (i = 0; i < nr_irqs; i++) { + irq_domain_set_info(domain, virq + i, hwirq + i, &aplic_direct_chip, + priv, handle_fasteoi_irq, NULL, NULL); + irq_set_affinity(virq + i, &direct->lmask); + } + + return 0; +} + +static const struct irq_domain_ops aplic_direct_irqdomain_ops = { + .translate = aplic_direct_irqdomain_translate, + .alloc = aplic_direct_irqdomain_alloc, + .free = irq_domain_free_irqs_top, +}; + +/* + * To handle an APLIC direct interrupts, we just read the CLAIMI register + * which will return highest priority pending interrupt and clear the + * pending bit of the interrupt. This process is repeated until CLAIMI + * register return zero value. + */ +static void aplic_direct_handle_irq(struct irq_desc *desc) +{ + struct aplic_idc *idc = this_cpu_ptr(&aplic_idcs); + struct irq_domain *irqdomain = idc->direct->irqdomain; + struct irq_chip *chip = irq_desc_get_chip(desc); + irq_hw_number_t hw_irq; + int irq; + + chained_irq_enter(chip, desc); + + while ((hw_irq = readl(idc->regs + APLIC_IDC_CLAIMI))) { + hw_irq = hw_irq >> APLIC_IDC_TOPI_ID_SHIFT; + irq = irq_find_mapping(irqdomain, hw_irq); + + if (unlikely(irq <= 0)) { + dev_warn_ratelimited(idc->direct->priv.dev, + "hw_irq %lu mapping not found\n", hw_irq); + } else { + generic_handle_irq(irq); + } + } + + chained_irq_exit(chip, desc); +} + +static void aplic_idc_set_delivery(struct aplic_idc *idc, bool en) +{ + u32 de = (en) ? APLIC_ENABLE_IDELIVERY : APLIC_DISABLE_IDELIVERY; + u32 th = (en) ? APLIC_ENABLE_ITHRESHOLD : APLIC_DISABLE_ITHRESHOLD; + + /* Priority must be less than threshold for interrupt triggering */ + writel(th, idc->regs + APLIC_IDC_ITHRESHOLD); + + /* Delivery must be set to 1 for interrupt triggering */ + writel(de, idc->regs + APLIC_IDC_IDELIVERY); +} + +static int aplic_direct_dying_cpu(unsigned int cpu) +{ + if (aplic_direct_parent_irq) + disable_percpu_irq(aplic_direct_parent_irq); + + return 0; +} + +static int aplic_direct_starting_cpu(unsigned int cpu) +{ + if (aplic_direct_parent_irq) { + enable_percpu_irq(aplic_direct_parent_irq, + irq_get_trigger_type(aplic_direct_parent_irq)); + } + + return 0; +} + +static int aplic_direct_parse_parent_hwirq(struct device *dev, u32 index, + u32 *parent_hwirq, unsigned long *parent_hartid, + struct aplic_priv *priv) +{ + struct of_phandle_args parent; + unsigned long hartid; + int rc; + + if (!is_of_node(dev->fwnode)) { + hartid = acpi_rintc_ext_parent_to_hartid(priv->acpi_aplic_id, index); + if (hartid == INVALID_HARTID) + return -ENODEV; + + *parent_hartid = hartid; + *parent_hwirq = RV_IRQ_EXT; + return 0; + } + + rc = of_irq_parse_one(to_of_node(dev->fwnode), index, &parent); + if (rc) + return rc; + + rc = riscv_of_parent_hartid(parent.np, parent_hartid); + if (rc) + return rc; + + *parent_hwirq = parent.args[0]; + return 0; +} + +int aplic_direct_setup(struct device *dev, void __iomem *regs) +{ + int i, j, rc, cpu, current_cpu, setup_count = 0; + struct aplic_direct *direct; + struct irq_domain *domain; + struct aplic_priv *priv; + struct aplic_idc *idc; + unsigned long hartid; + u32 v, hwirq; + + direct = devm_kzalloc(dev, sizeof(*direct), GFP_KERNEL); + if (!direct) + return -ENOMEM; + priv = &direct->priv; + + rc = aplic_setup_priv(priv, dev, regs); + if (rc) { + dev_err(dev, "failed to create APLIC context\n"); + return rc; + } + + /* Setup per-CPU IDC and target CPU mask */ + current_cpu = get_cpu(); + for (i = 0; i < priv->nr_idcs; i++) { + rc = aplic_direct_parse_parent_hwirq(dev, i, &hwirq, &hartid, priv); + if (rc) { + dev_warn(dev, "parent irq for IDC%d not found\n", i); + continue; + } + + /* + * Skip interrupts other than external interrupts for + * current privilege level. + */ + if (hwirq != RV_IRQ_EXT) + continue; + + cpu = riscv_hartid_to_cpuid(hartid); + if (cpu < 0) { + dev_warn(dev, "invalid cpuid for IDC%d\n", i); + continue; + } + + cpumask_set_cpu(cpu, &direct->lmask); + + idc = per_cpu_ptr(&aplic_idcs, cpu); + rc = riscv_get_hart_index(dev->fwnode, i, &idc->hart_index); + if (rc) { + dev_warn(dev, "hart index not found for IDC%d\n", i); + continue; + } + idc->regs = priv->regs + APLIC_IDC_BASE + idc->hart_index * APLIC_IDC_SIZE; + idc->direct = direct; + + aplic_idc_set_delivery(idc, true); + + /* + * Boot cpu might not have APLIC hart_index = 0 so check + * and update target registers of all interrupts. + */ + if (cpu == current_cpu && idc->hart_index) { + v = FIELD_PREP(APLIC_TARGET_HART_IDX, idc->hart_index); + v |= FIELD_PREP(APLIC_TARGET_IPRIO, APLIC_DEFAULT_PRIORITY); + for (j = 1; j <= priv->nr_irqs; j++) + writel(v, priv->regs + APLIC_TARGET_BASE + (j - 1) * sizeof(u32)); + } + + setup_count++; + } + put_cpu(); + + /* Find parent domain and register chained handler */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), + DOMAIN_BUS_ANY); + if (!aplic_direct_parent_irq && domain) { + aplic_direct_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); + if (aplic_direct_parent_irq) { + irq_set_chained_handler(aplic_direct_parent_irq, + aplic_direct_handle_irq); + + /* + * Setup CPUHP notifier to enable parent + * interrupt on all CPUs + */ + cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, + "irqchip/riscv/aplic:starting", + aplic_direct_starting_cpu, + aplic_direct_dying_cpu); + } + } + + /* Fail if we were not able to setup IDC for any CPU */ + if (!setup_count) + return -ENODEV; + + /* Setup global config and interrupt delivery */ + aplic_init_hw_global(priv, false); + + /* Create irq domain instance for the APLIC */ + direct->irqdomain = irq_domain_create_linear(dev->fwnode, priv->nr_irqs + 1, + &aplic_direct_irqdomain_ops, priv); + if (!direct->irqdomain) { + dev_err(dev, "failed to create direct irq domain\n"); + return -ENOMEM; + } + + /* Advertise the interrupt controller */ + dev_info(dev, "%d interrupts directly connected to %d CPUs\n", + priv->nr_irqs, priv->nr_idcs); + + return 0; +} diff --git a/drivers/irqchip/irq-riscv-aplic-main.c b/drivers/irqchip/irq-riscv-aplic-main.c new file mode 100644 index 000000000000..93e7c51f944a --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-main.c @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/irqchip/riscv-imsic.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/printk.h> + +#include "irq-riscv-aplic-main.h" + +void aplic_irq_unmask(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + writel(d->hwirq, priv->regs + APLIC_SETIENUM); +} + +void aplic_irq_mask(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + writel(d->hwirq, priv->regs + APLIC_CLRIENUM); +} + +int aplic_irq_set_type(struct irq_data *d, unsigned int type) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + void __iomem *sourcecfg; + u32 val = 0; + + switch (type) { + case IRQ_TYPE_NONE: + val = APLIC_SOURCECFG_SM_INACTIVE; + break; + case IRQ_TYPE_LEVEL_LOW: + val = APLIC_SOURCECFG_SM_LEVEL_LOW; + break; + case IRQ_TYPE_LEVEL_HIGH: + val = APLIC_SOURCECFG_SM_LEVEL_HIGH; + break; + case IRQ_TYPE_EDGE_FALLING: + val = APLIC_SOURCECFG_SM_EDGE_FALL; + break; + case IRQ_TYPE_EDGE_RISING: + val = APLIC_SOURCECFG_SM_EDGE_RISE; + break; + default: + return -EINVAL; + } + + sourcecfg = priv->regs + APLIC_SOURCECFG_BASE; + sourcecfg += (d->hwirq - 1) * sizeof(u32); + writel(val, sourcecfg); + + return 0; +} + +int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, + unsigned long *hwirq, unsigned int *type) +{ + if (WARN_ON(fwspec->param_count < 2)) + return -EINVAL; + if (WARN_ON(!fwspec->param[0])) + return -EINVAL; + + /* For DT, gsi_base is always zero. */ + *hwirq = fwspec->param[0] - gsi_base; + *type = fwspec->param[1] & IRQ_TYPE_SENSE_MASK; + + WARN_ON(*type == IRQ_TYPE_NONE); + + return 0; +} + +void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode) +{ + u32 val; +#ifdef CONFIG_RISCV_M_MODE + u32 valh; + + if (msi_mode) { + val = lower_32_bits(priv->msicfg.base_ppn); + valh = FIELD_PREP(APLIC_xMSICFGADDRH_BAPPN, upper_32_bits(priv->msicfg.base_ppn)); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_LHXW, priv->msicfg.lhxw); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXW, priv->msicfg.hhxw); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_LHXS, priv->msicfg.lhxs); + valh |= FIELD_PREP(APLIC_xMSICFGADDRH_HHXS, priv->msicfg.hhxs); + writel(val, priv->regs + APLIC_xMSICFGADDR); + writel(valh, priv->regs + APLIC_xMSICFGADDRH); + } +#endif + + /* Setup APLIC domaincfg register */ + val = readl(priv->regs + APLIC_DOMAINCFG); + val |= APLIC_DOMAINCFG_IE; + if (msi_mode) + val |= APLIC_DOMAINCFG_DM; + writel(val, priv->regs + APLIC_DOMAINCFG); + if (readl(priv->regs + APLIC_DOMAINCFG) != val) + dev_warn(priv->dev, "unable to write 0x%x in domaincfg\n", val); +} + +static void aplic_init_hw_irqs(struct aplic_priv *priv) +{ + int i; + + /* Disable all interrupts */ + for (i = 0; i <= priv->nr_irqs; i += 32) + writel(-1U, priv->regs + APLIC_CLRIE_BASE + (i / 32) * sizeof(u32)); + + /* Set interrupt type and default priority for all interrupts */ + for (i = 1; i <= priv->nr_irqs; i++) { + writel(0, priv->regs + APLIC_SOURCECFG_BASE + (i - 1) * sizeof(u32)); + writel(APLIC_DEFAULT_PRIORITY, + priv->regs + APLIC_TARGET_BASE + (i - 1) * sizeof(u32)); + } + + /* Clear APLIC domaincfg */ + writel(0, priv->regs + APLIC_DOMAINCFG); +} + +#ifdef CONFIG_ACPI +static const struct acpi_device_id aplic_acpi_match[] = { + { "RSCV0002", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, aplic_acpi_match); + +#endif + +int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs) +{ + struct device_node *np = to_of_node(dev->fwnode); + struct of_phandle_args parent; + int rc; + + /* Save device pointer and register base */ + priv->dev = dev; + priv->regs = regs; + + if (np) { + /* Find out number of interrupt sources */ + rc = of_property_read_u32(np, "riscv,num-sources", &priv->nr_irqs); + if (rc) { + dev_err(dev, "failed to get number of interrupt sources\n"); + return rc; + } + + /* + * Find out number of IDCs based on parent interrupts + * + * If "msi-parent" property is present then we ignore the + * APLIC IDCs which forces the APLIC driver to use MSI mode. + */ + if (!of_property_present(np, "msi-parent")) { + while (!of_irq_parse_one(np, priv->nr_idcs, &parent)) + priv->nr_idcs++; + } + } else { + rc = riscv_acpi_get_gsi_info(dev->fwnode, &priv->gsi_base, &priv->acpi_aplic_id, + &priv->nr_irqs, &priv->nr_idcs); + if (rc) { + dev_err(dev, "failed to find GSI mapping\n"); + return rc; + } + } + + /* Setup initial state APLIC interrupts */ + aplic_init_hw_irqs(priv); + + return 0; +} + +static int aplic_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + bool msi_mode = false; + void __iomem *regs; + int rc; + + /* Map the MMIO registers */ + regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(regs)) { + dev_err(dev, "failed map MMIO registers\n"); + return PTR_ERR(regs); + } + + /* + * If msi-parent property is present then setup APLIC MSI + * mode otherwise setup APLIC direct mode. + */ + if (is_of_node(dev->fwnode)) + msi_mode = of_property_present(to_of_node(dev->fwnode), "msi-parent"); + else + msi_mode = imsic_acpi_get_fwnode(NULL) ? 1 : 0; + + if (msi_mode) + rc = aplic_msi_setup(dev, regs); + else + rc = aplic_direct_setup(dev, regs); + if (rc) + dev_err_probe(dev, rc, "failed to setup APLIC in %s mode\n", + msi_mode ? "MSI" : "direct"); + +#ifdef CONFIG_ACPI + if (!acpi_disabled) + acpi_dev_clear_dependencies(ACPI_COMPANION(dev)); +#endif + + return rc; +} + +static const struct of_device_id aplic_match[] = { + { .compatible = "riscv,aplic" }, + {} +}; + +static struct platform_driver aplic_driver = { + .driver = { + .name = "riscv-aplic", + .of_match_table = aplic_match, + .acpi_match_table = ACPI_PTR(aplic_acpi_match), + }, + .probe = aplic_probe, +}; +builtin_platform_driver(aplic_driver); diff --git a/drivers/irqchip/irq-riscv-aplic-main.h b/drivers/irqchip/irq-riscv-aplic-main.h new file mode 100644 index 000000000000..b0ad8cde69b1 --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-main.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _IRQ_RISCV_APLIC_MAIN_H +#define _IRQ_RISCV_APLIC_MAIN_H + +#include <linux/device.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/fwnode.h> + +#define APLIC_DEFAULT_PRIORITY 1 + +struct aplic_msicfg { + phys_addr_t base_ppn; + u32 hhxs; + u32 hhxw; + u32 lhxs; + u32 lhxw; +}; + +struct aplic_priv { + struct device *dev; + u32 gsi_base; + u32 nr_irqs; + u32 nr_idcs; + u32 acpi_aplic_id; + void __iomem *regs; + struct aplic_msicfg msicfg; +}; + +void aplic_irq_unmask(struct irq_data *d); +void aplic_irq_mask(struct irq_data *d); +int aplic_irq_set_type(struct irq_data *d, unsigned int type); +int aplic_irqdomain_translate(struct irq_fwspec *fwspec, u32 gsi_base, + unsigned long *hwirq, unsigned int *type); +void aplic_init_hw_global(struct aplic_priv *priv, bool msi_mode); +int aplic_setup_priv(struct aplic_priv *priv, struct device *dev, void __iomem *regs); +int aplic_direct_setup(struct device *dev, void __iomem *regs); +#ifdef CONFIG_RISCV_APLIC_MSI +int aplic_msi_setup(struct device *dev, void __iomem *regs); +#else +static inline int aplic_msi_setup(struct device *dev, void __iomem *regs) +{ + return -ENODEV; +} +#endif + +#endif diff --git a/drivers/irqchip/irq-riscv-aplic-msi.c b/drivers/irqchip/irq-riscv-aplic-msi.c new file mode 100644 index 000000000000..fb8d1838609f --- /dev/null +++ b/drivers/irqchip/irq-riscv-aplic-msi.c @@ -0,0 +1,285 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/irqchip.h> +#include <linux/irqchip/riscv-aplic.h> +#include <linux/irqchip/riscv-imsic.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/printk.h> +#include <linux/smp.h> + +#include "irq-riscv-aplic-main.h" + +static void aplic_msi_irq_mask(struct irq_data *d) +{ + aplic_irq_mask(d); + irq_chip_mask_parent(d); +} + +static void aplic_msi_irq_unmask(struct irq_data *d) +{ + irq_chip_unmask_parent(d); + aplic_irq_unmask(d); +} + +static void aplic_msi_irq_retrigger_level(struct irq_data *d) +{ + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + + switch (irqd_get_trigger_type(d)) { + case IRQ_TYPE_LEVEL_LOW: + case IRQ_TYPE_LEVEL_HIGH: + /* + * The section "4.9.2 Special consideration for level-sensitive interrupt + * sources" of the RISC-V AIA specification says: + * + * A second option is for the interrupt service routine to write the + * APLIC’s source identity number for the interrupt to the domain’s + * setipnum register just before exiting. This will cause the interrupt’s + * pending bit to be set to one again if the source is still asserting + * an interrupt, but not if the source is not asserting an interrupt. + */ + writel(d->hwirq, priv->regs + APLIC_SETIPNUM_LE); + break; + } +} + +static void aplic_msi_irq_eoi(struct irq_data *d) +{ + /* + * EOI handling is required only for level-triggered interrupts + * when APLIC is in MSI mode. + */ + aplic_msi_irq_retrigger_level(d); +} + +static int aplic_msi_irq_set_type(struct irq_data *d, unsigned int type) +{ + int rc = aplic_irq_set_type(d, type); + + if (rc) + return rc; + /* + * Updating sourcecfg register for level-triggered interrupts + * requires interrupt retriggering when APLIC is in MSI mode. + */ + aplic_msi_irq_retrigger_level(d); + return 0; +} + +static void aplic_msi_write_msg(struct irq_data *d, struct msi_msg *msg) +{ + unsigned int group_index, hart_index, guest_index, val; + struct aplic_priv *priv = irq_data_get_irq_chip_data(d); + struct aplic_msicfg *mc = &priv->msicfg; + phys_addr_t tppn, tbppn, msg_addr; + void __iomem *target; + + /* For zeroed MSI, simply write zero into the target register */ + if (!msg->address_hi && !msg->address_lo && !msg->data) { + target = priv->regs + APLIC_TARGET_BASE; + target += (d->hwirq - 1) * sizeof(u32); + writel(0, target); + return; + } + + /* Sanity check on message data */ + WARN_ON(msg->data > APLIC_TARGET_EIID_MASK); + + /* Compute target MSI address */ + msg_addr = (((u64)msg->address_hi) << 32) | msg->address_lo; + tppn = msg_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + + /* Compute target HART Base PPN */ + tbppn = tppn; + tbppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs); + tbppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs); + WARN_ON(tbppn != mc->base_ppn); + + /* Compute target group and hart indexes */ + group_index = (tppn >> APLIC_xMSICFGADDR_PPN_HHX_SHIFT(mc->hhxs)) & + APLIC_xMSICFGADDR_PPN_HHX_MASK(mc->hhxw); + hart_index = (tppn >> APLIC_xMSICFGADDR_PPN_LHX_SHIFT(mc->lhxs)) & + APLIC_xMSICFGADDR_PPN_LHX_MASK(mc->lhxw); + hart_index |= (group_index << mc->lhxw); + WARN_ON(hart_index > APLIC_TARGET_HART_IDX_MASK); + + /* Compute target guest index */ + guest_index = tppn & APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + WARN_ON(guest_index > APLIC_TARGET_GUEST_IDX_MASK); + + /* Update IRQ TARGET register */ + target = priv->regs + APLIC_TARGET_BASE; + target += (d->hwirq - 1) * sizeof(u32); + val = FIELD_PREP(APLIC_TARGET_HART_IDX, hart_index); + val |= FIELD_PREP(APLIC_TARGET_GUEST_IDX, guest_index); + val |= FIELD_PREP(APLIC_TARGET_EIID, msg->data); + writel(val, target); +} + +static void aplic_msi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = (u32)desc->data.icookie.value; +} + +static int aplic_msi_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) +{ + struct msi_domain_info *info = d->host_data; + struct aplic_priv *priv = info->data; + + return aplic_irqdomain_translate(fwspec, priv->gsi_base, hwirq, type); +} + +static const struct msi_domain_template aplic_msi_template = { + .chip = { + .name = "APLIC-MSI", + .irq_mask = aplic_msi_irq_mask, + .irq_unmask = aplic_msi_irq_unmask, + .irq_set_type = aplic_msi_irq_set_type, + .irq_eoi = aplic_msi_irq_eoi, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif + .irq_write_msi_msg = aplic_msi_write_msg, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, + }, + + .ops = { + .set_desc = aplic_msi_set_desc, + .msi_translate = aplic_msi_translate, + }, + + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_USE_DEV_FWNODE, + .handler = handle_fasteoi_irq, + .handler_name = "fasteoi", + }, +}; + +int aplic_msi_setup(struct device *dev, void __iomem *regs) +{ + const struct imsic_global_config *imsic_global; + struct irq_domain *msi_domain; + struct aplic_priv *priv; + struct aplic_msicfg *mc; + phys_addr_t pa; + int rc; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + rc = aplic_setup_priv(priv, dev, regs); + if (rc) { + dev_err(dev, "failed to create APLIC context\n"); + return rc; + } + mc = &priv->msicfg; + + /* + * The APLIC outgoing MSI config registers assume target MSI + * controller to be RISC-V AIA IMSIC controller. + */ + imsic_global = imsic_get_global_config(); + if (!imsic_global) { + dev_err(dev, "IMSIC global config not found\n"); + return -ENODEV; + } + + /* Find number of guest index bits (LHXS) */ + mc->lhxs = imsic_global->guest_index_bits; + if (APLIC_xMSICFGADDRH_LHXS_MASK < mc->lhxs) { + dev_err(dev, "IMSIC guest index bits big for APLIC LHXS\n"); + return -EINVAL; + } + + /* Find number of HART index bits (LHXW) */ + mc->lhxw = imsic_global->hart_index_bits; + if (APLIC_xMSICFGADDRH_LHXW_MASK < mc->lhxw) { + dev_err(dev, "IMSIC hart index bits big for APLIC LHXW\n"); + return -EINVAL; + } + + /* Find number of group index bits (HHXW) */ + mc->hhxw = imsic_global->group_index_bits; + if (APLIC_xMSICFGADDRH_HHXW_MASK < mc->hhxw) { + dev_err(dev, "IMSIC group index bits big for APLIC HHXW\n"); + return -EINVAL; + } + + /* Find first bit position of group index (HHXS) */ + mc->hhxs = imsic_global->group_index_shift; + if (mc->hhxs < (2 * APLIC_xMSICFGADDR_PPN_SHIFT)) { + dev_err(dev, "IMSIC group index shift should be >= %d\n", + (2 * APLIC_xMSICFGADDR_PPN_SHIFT)); + return -EINVAL; + } + mc->hhxs -= (2 * APLIC_xMSICFGADDR_PPN_SHIFT); + if (APLIC_xMSICFGADDRH_HHXS_MASK < mc->hhxs) { + dev_err(dev, "IMSIC group index shift big for APLIC HHXS\n"); + return -EINVAL; + } + + /* Compute PPN base */ + mc->base_ppn = imsic_global->base_addr >> APLIC_xMSICFGADDR_PPN_SHIFT; + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HART(mc->lhxs); + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_LHX(mc->lhxw, mc->lhxs); + mc->base_ppn &= ~APLIC_xMSICFGADDR_PPN_HHX(mc->hhxw, mc->hhxs); + + /* Setup global config and interrupt delivery */ + aplic_init_hw_global(priv, true); + + /* Set the APLIC device MSI domain if not available */ + if (!dev_get_msi_domain(dev)) { + /* + * The device MSI domain for OF devices is only set at the + * time of populating/creating OF device. If the device MSI + * domain is discovered later after the OF device is created + * then we need to set it explicitly before using any platform + * MSI functions. + * + * In case of APLIC device, the parent MSI domain is always + * IMSIC and the IMSIC MSI domains are created later through + * the platform driver probing so we set it explicitly here. + */ + if (is_of_node(dev->fwnode)) { + of_msi_configure(dev, to_of_node(dev->fwnode)); + } else { + msi_domain = irq_find_matching_fwnode(imsic_acpi_get_fwnode(dev), + DOMAIN_BUS_PLATFORM_MSI); + if (msi_domain) + dev_set_msi_domain(dev, msi_domain); + } + + if (!dev_get_msi_domain(dev)) + return -EPROBE_DEFER; + } + + if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, &aplic_msi_template, + priv->nr_irqs + 1, priv, priv)) { + dev_err(dev, "failed to create MSI irq domain\n"); + return -ENOMEM; + } + + /* Advertise the interrupt controller */ + pa = priv->msicfg.base_ppn << APLIC_xMSICFGADDR_PPN_SHIFT; + dev_info(dev, "%d interrupts forwarded to MSI base %pa\n", priv->nr_irqs, &pa); + + return 0; +} diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c new file mode 100644 index 000000000000..6bac67cc0b6d --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/acpi.h> +#include <linux/cpu.h> +#include <linux/export.h> +#include <linux/interrupt.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqchip/riscv-imsic.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/spinlock.h> +#include <linux/smp.h> + +#include "irq-riscv-imsic-state.h" + +static int imsic_parent_irq; +bool imsic_noipi __ro_after_init; + +static int __init imsic_noipi_cfg(char *buf) +{ + imsic_noipi = true; + return 0; +} +early_param("irqchip.riscv_imsic_noipi", imsic_noipi_cfg); + +#ifdef CONFIG_SMP +static void imsic_ipi_send(unsigned int cpu) +{ + struct imsic_local_config *local = per_cpu_ptr(imsic->global.local, cpu); + + writel(IMSIC_IPI_ID, local->msi_va); +} + +static void imsic_ipi_starting_cpu(void) +{ + if (imsic_noipi) + return; + + /* Enable IPIs for current CPU. */ + __imsic_id_set_enable(IMSIC_IPI_ID); +} + +static void imsic_ipi_dying_cpu(void) +{ + if (imsic_noipi) + return; + + /* Disable IPIs for current CPU. */ + __imsic_id_clear_enable(IMSIC_IPI_ID); +} + +static int __init imsic_ipi_domain_init(void) +{ + int virq; + + if (imsic_noipi) + return 0; + + /* Create IMSIC IPI multiplexing */ + virq = ipi_mux_create(IMSIC_NR_IPI, imsic_ipi_send); + if (virq <= 0) + return virq < 0 ? virq : -ENOMEM; + + /* Set vIRQ range */ + riscv_ipi_set_virq_range(virq, IMSIC_NR_IPI); + + /* Announce that IMSIC is providing IPIs */ + pr_info("%pfwP: providing IPIs using interrupt %d\n", imsic->fwnode, IMSIC_IPI_ID); + + return 0; +} +#else +static void imsic_ipi_starting_cpu(void) { } +static void imsic_ipi_dying_cpu(void) { } +static int __init imsic_ipi_domain_init(void) { return 0; } +#endif + +/* + * To handle an interrupt, we read the TOPEI CSR and write zero in one + * instruction. If TOPEI CSR is non-zero then we translate TOPEI.ID to + * Linux interrupt number and let Linux IRQ subsystem handle it. + */ +static void imsic_handle_irq(struct irq_desc *desc) +{ + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long local_id; + + /* + * Process pending local synchronization instead of waiting + * for per-CPU local timer to expire. + */ + imsic_local_sync_all(false); + + chained_irq_enter(chip, desc); + + while ((local_id = csr_swap(CSR_TOPEI, 0))) { + local_id >>= TOPEI_ID_SHIFT; + + if (!imsic_noipi && local_id == IMSIC_IPI_ID) { + if (IS_ENABLED(CONFIG_SMP)) + ipi_mux_process(); + continue; + } + + if (unlikely(local_id > imsic->global.nr_ids)) { + pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id); + continue; + } + + generic_handle_irq(lpriv->vectors[local_id].irq); + } + + chained_irq_exit(chip, desc); +} + +static int imsic_starting_cpu(unsigned int cpu) +{ + /* Mark per-CPU IMSIC state as online */ + imsic_state_online(); + + /* Enable per-CPU parent interrupt */ + enable_percpu_irq(imsic_parent_irq, irq_get_trigger_type(imsic_parent_irq)); + + /* Setup IPIs */ + imsic_ipi_starting_cpu(); + + /* + * Interrupts identities might have been enabled/disabled while + * this CPU was not running so sync-up local enable/disable state. + */ + imsic_local_sync_all(true); + + /* Enable local interrupt delivery */ + imsic_local_delivery(true); + + return 0; +} + +static int imsic_dying_cpu(unsigned int cpu) +{ + /* Cleanup IPIs */ + imsic_ipi_dying_cpu(); + + /* Mark per-CPU IMSIC state as offline */ + imsic_state_offline(); + + return 0; +} + +static int __init imsic_early_probe(struct fwnode_handle *fwnode) +{ + struct irq_domain *domain; + int rc; + + /* Find parent domain and register chained handler */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); + if (!domain) { + pr_err("%pfwP: Failed to find INTC domain\n", fwnode); + return -ENOENT; + } + imsic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); + if (!imsic_parent_irq) { + pr_err("%pfwP: Failed to create INTC mapping\n", fwnode); + return -ENOENT; + } + + /* Initialize IPI domain */ + rc = imsic_ipi_domain_init(); + if (rc) { + pr_err("%pfwP: Failed to initialize IPI domain\n", fwnode); + return rc; + } + + /* Setup chained handler to the parent domain interrupt */ + irq_set_chained_handler(imsic_parent_irq, imsic_handle_irq); + + /* + * Setup cpuhp state (must be done after setting imsic_parent_irq) + * + * Don't disable per-CPU IMSIC file when CPU goes offline + * because this affects IPI and the masking/unmasking of + * virtual IPIs is done via generic IPI-Mux + */ + cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_IMSIC_STARTING, "irqchip/riscv/imsic:starting", + imsic_starting_cpu, imsic_dying_cpu); + + return 0; +} + +static int __init imsic_early_dt_init(struct device_node *node, struct device_node *parent) +{ + struct fwnode_handle *fwnode = &node->fwnode; + int rc; + + /* Setup IMSIC state */ + rc = imsic_setup_state(fwnode, NULL); + if (rc) { + pr_err("%pfwP: failed to setup state (error %d)\n", fwnode, rc); + return rc; + } + + /* Do early setup of IPIs */ + rc = imsic_early_probe(fwnode); + if (rc) + return rc; + + /* Ensure that OF platform device gets probed */ + of_node_clear_flag(node, OF_POPULATED); + return 0; +} + +IRQCHIP_DECLARE(riscv_imsic, "riscv,imsics", imsic_early_dt_init); + +#ifdef CONFIG_ACPI + +static struct fwnode_handle *imsic_acpi_fwnode; + +struct fwnode_handle *imsic_acpi_get_fwnode(struct device *dev) +{ + return imsic_acpi_fwnode; +} +EXPORT_SYMBOL_GPL(imsic_acpi_get_fwnode); + +static int __init imsic_early_acpi_init(union acpi_subtable_headers *header, + const unsigned long end) +{ + struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)header; + int rc; + + imsic_acpi_fwnode = irq_domain_alloc_named_fwnode("imsic"); + if (!imsic_acpi_fwnode) { + pr_err("unable to allocate IMSIC FW node\n"); + return -ENOMEM; + } + + /* Setup IMSIC state */ + rc = imsic_setup_state(imsic_acpi_fwnode, imsic); + if (rc) { + pr_err("%pfwP: failed to setup state (error %d)\n", imsic_acpi_fwnode, rc); + return rc; + } + + /* Do early setup of IMSIC state and IPIs */ + rc = imsic_early_probe(imsic_acpi_fwnode); + if (rc) { + irq_domain_free_fwnode(imsic_acpi_fwnode); + imsic_acpi_fwnode = NULL; + return rc; + } + + rc = imsic_platform_acpi_probe(imsic_acpi_fwnode); + +#ifdef CONFIG_PCI + if (!rc) + pci_msi_register_fwnode_provider(&imsic_acpi_get_fwnode); +#endif + + if (rc) + pr_err("%pfwP: failed to register IMSIC for MSI functionality (error %d)\n", + imsic_acpi_fwnode, rc); + + /* + * Even if imsic_platform_acpi_probe() fails, the IPI part of IMSIC can + * continue to work. So, no need to return failure. This is similar to + * DT where IPI works but MSI probe fails for some reason. + */ + return 0; +} + +IRQCHIP_ACPI_DECLARE(riscv_imsic, ACPI_MADT_TYPE_IMSIC, NULL, + 1, imsic_early_acpi_init); +#endif diff --git a/drivers/irqchip/irq-riscv-imsic-platform.c b/drivers/irqchip/irq-riscv-imsic-platform.c new file mode 100644 index 000000000000..7228a33f6c37 --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-platform.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/acpi.h> +#include <linux/bitmap.h> +#include <linux/cpu.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/pci.h> +#include <linux/platform_device.h> +#include <linux/spinlock.h> +#include <linux/smp.h> + +#include <linux/irqchip/irq-msi-lib.h> +#include "irq-riscv-imsic-state.h" + +static bool imsic_cpu_page_phys(unsigned int cpu, unsigned int guest_index, + phys_addr_t *out_msi_pa) +{ + struct imsic_global_config *global; + struct imsic_local_config *local; + + global = &imsic->global; + local = per_cpu_ptr(global->local, cpu); + + if (BIT(global->guest_index_bits) <= guest_index) + return false; + + if (out_msi_pa) + *out_msi_pa = local->msi_pa + (guest_index * IMSIC_MMIO_PAGE_SZ); + + return true; +} + +static void imsic_irq_mask(struct irq_data *d) +{ + imsic_vector_mask(irq_data_get_irq_chip_data(d)); +} + +static void imsic_irq_unmask(struct irq_data *d) +{ + imsic_vector_unmask(irq_data_get_irq_chip_data(d)); +} + +static int imsic_irq_retrigger(struct irq_data *d) +{ + struct imsic_vector *vec = irq_data_get_irq_chip_data(d); + struct imsic_local_config *local; + + if (WARN_ON(!vec)) + return -ENOENT; + + local = per_cpu_ptr(imsic->global.local, vec->cpu); + writel_relaxed(vec->local_id, local->msi_va); + return 0; +} + +static void imsic_irq_ack(struct irq_data *d) +{ + irq_move_irq(d); +} + +static void imsic_irq_compose_vector_msg(struct imsic_vector *vec, struct msi_msg *msg) +{ + phys_addr_t msi_addr; + + if (WARN_ON(!vec)) + return; + + if (WARN_ON(!imsic_cpu_page_phys(vec->cpu, 0, &msi_addr))) + return; + + msg->address_hi = upper_32_bits(msi_addr); + msg->address_lo = lower_32_bits(msi_addr); + msg->data = vec->local_id; +} + +static void imsic_irq_compose_msg(struct irq_data *d, struct msi_msg *msg) +{ + imsic_irq_compose_vector_msg(irq_data_get_irq_chip_data(d), msg); +} + +#ifdef CONFIG_SMP +static void imsic_msi_update_msg(struct irq_data *d, struct imsic_vector *vec) +{ + struct msi_msg msg = { }; + + imsic_irq_compose_vector_msg(vec, &msg); + irq_data_get_irq_chip(d)->irq_write_msi_msg(d, &msg); +} + +static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask_val, + bool force) +{ + struct imsic_vector *old_vec, *new_vec; + struct imsic_vector tmp_vec; + + /* + * Requirements for the downstream irqdomains (or devices): + * + * 1) Downstream irqdomains (or devices) with atomic MSI update can + * happily do imsic_irq_set_affinity() in the process-context on + * any CPU so the irqchip of such irqdomains must not set the + * IRQCHIP_MOVE_DEFERRED flag. + * + * 2) Downstream irqdomains (or devices) with non-atomic MSI update + * must use imsic_irq_set_affinity() in nterrupt-context upon + * the next device interrupt so the irqchip of such irqdomains + * must set the IRQCHIP_MOVE_DEFERRED flag. + */ + + old_vec = irq_data_get_irq_chip_data(d); + if (WARN_ON(!old_vec)) + return -ENOENT; + + /* If old vector cpu belongs to the target cpumask then do nothing */ + if (cpumask_test_cpu(old_vec->cpu, mask_val)) + return IRQ_SET_MASK_OK_DONE; + + /* If move is already in-flight then return failure */ + if (imsic_vector_get_move(old_vec)) + return -EBUSY; + + /* Get a new vector on the desired set of CPUs */ + new_vec = imsic_vector_alloc(old_vec->irq, mask_val); + if (!new_vec) + return -ENOSPC; + + /* + * Device having non-atomic MSI update might see an intermediate + * state when changing target IMSIC vector from one CPU to another. + * + * To avoid losing interrupt to such intermediate state, do the + * following (just like x86 APIC): + * + * 1) First write a temporary IMSIC vector to the device which + * has MSI address same as the old IMSIC vector but MSI data + * matches the new IMSIC vector. + * + * 2) Next write the new IMSIC vector to the device. + * + * Based on the above, __imsic_local_sync() must check pending + * status of both old MSI data and new MSI data on the old CPU. + */ + if (!irq_can_move_in_process_context(d) && + new_vec->local_id != old_vec->local_id) { + /* Setup temporary vector */ + tmp_vec.cpu = old_vec->cpu; + tmp_vec.local_id = new_vec->local_id; + + /* Point device to the temporary vector */ + imsic_msi_update_msg(d, &tmp_vec); + } + + /* Point device to the new vector */ + imsic_msi_update_msg(d, new_vec); + + /* Update irq descriptors with the new vector */ + d->chip_data = new_vec; + + /* Update effective affinity */ + irq_data_update_effective_affinity(d, cpumask_of(new_vec->cpu)); + + /* Move state of the old vector to the new vector */ + imsic_vector_move(old_vec, new_vec); + + return IRQ_SET_MASK_OK_DONE; +} + +static void imsic_irq_force_complete_move(struct irq_data *d) +{ + struct imsic_vector *mvec, *vec = irq_data_get_irq_chip_data(d); + unsigned int cpu = smp_processor_id(); + + if (WARN_ON(!vec)) + return; + + /* Do nothing if there is no in-flight move */ + mvec = imsic_vector_get_move(vec); + if (!mvec) + return; + + /* Do nothing if the old IMSIC vector does not belong to current CPU */ + if (mvec->cpu != cpu) + return; + + /* + * The best we can do is force cleanup the old IMSIC vector. + * + * The challenges over here are same as x86 vector domain so + * refer to the comments in irq_force_complete_move() function + * implemented at arch/x86/kernel/apic/vector.c. + */ + + /* Force cleanup in-flight move */ + pr_info("IRQ fixup: irq %d move in progress, old vector cpu %d local_id %d\n", + d->irq, mvec->cpu, mvec->local_id); + imsic_vector_force_move_cleanup(vec); +} +#endif + +static struct irq_chip imsic_irq_base_chip = { + .name = "IMSIC", + .irq_mask = imsic_irq_mask, + .irq_unmask = imsic_irq_unmask, +#ifdef CONFIG_SMP + .irq_set_affinity = imsic_irq_set_affinity, + .irq_force_complete_move = imsic_irq_force_complete_move, +#endif + .irq_retrigger = imsic_irq_retrigger, + .irq_ack = imsic_irq_ack, + .irq_compose_msi_msg = imsic_irq_compose_msg, + .flags = IRQCHIP_SKIP_SET_WAKE | IRQCHIP_MASK_ON_SUSPEND, +}; + +static int imsic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct imsic_vector *vec; + + /* Multi-MSI is not supported yet. */ + if (nr_irqs > 1) + return -EOPNOTSUPP; + + vec = imsic_vector_alloc(virq, cpu_online_mask); + if (!vec) + return -ENOSPC; + + irq_domain_set_info(domain, virq, virq, &imsic_irq_base_chip, vec, + handle_edge_irq, NULL, NULL); + irq_set_noprobe(virq); + irq_set_affinity(virq, cpu_online_mask); + irq_data_update_effective_affinity(irq_get_irq_data(virq), cpumask_of(vec->cpu)); + + return 0; +} + +static void imsic_irq_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + + imsic_vector_free(irq_data_get_irq_chip_data(d)); + irq_domain_free_irqs_parent(domain, virq, nr_irqs); +} + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +static void imsic_irq_debug_show(struct seq_file *m, struct irq_domain *d, + struct irq_data *irqd, int ind) +{ + if (!irqd) { + imsic_vector_debug_show_summary(m, ind); + return; + } + + imsic_vector_debug_show(m, irq_data_get_irq_chip_data(irqd), ind); +} +#endif + +static const struct irq_domain_ops imsic_base_domain_ops = { + .alloc = imsic_irq_domain_alloc, + .free = imsic_irq_domain_free, + .select = msi_lib_irq_domain_select, +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS + .debug_show = imsic_irq_debug_show, +#endif +}; + +static bool imsic_init_dev_msi_info(struct device *dev, struct irq_domain *domain, + struct irq_domain *real_parent, struct msi_domain_info *info) +{ + if (!msi_lib_init_dev_msi_info(dev, domain, real_parent, info)) + return false; + + switch (info->bus_token) { + case DOMAIN_BUS_PCI_DEVICE_MSI: + case DOMAIN_BUS_PCI_DEVICE_MSIX: + info->chip->flags |= IRQCHIP_MOVE_DEFERRED; + break; + default: + break; + } + + return true; +} + +static const struct msi_parent_ops imsic_msi_parent_ops = { + .supported_flags = MSI_GENERIC_FLAGS_MASK | + MSI_FLAG_PCI_MSIX, + .required_flags = MSI_FLAG_USE_DEF_DOM_OPS | + MSI_FLAG_USE_DEF_CHIP_OPS | + MSI_FLAG_PCI_MSI_MASK_PARENT, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .bus_select_token = DOMAIN_BUS_NEXUS, + .bus_select_mask = MATCH_PCI_MSI | MATCH_PLATFORM_MSI, + .init_dev_msi_info = imsic_init_dev_msi_info, +}; + +int imsic_irqdomain_init(void) +{ + struct irq_domain_info info = { + .ops = &imsic_base_domain_ops, + .host_data = imsic, + }; + struct imsic_global_config *global; + + if (!imsic || !imsic->fwnode) { + pr_err("early driver not probed\n"); + return -ENODEV; + } + + if (imsic->base_domain) { + pr_err("%pfwP: irq domain already created\n", imsic->fwnode); + return -ENODEV; + } + + /* Create Base IRQ domain */ + info.fwnode = imsic->fwnode, + imsic->base_domain = msi_create_parent_irq_domain(&info, &imsic_msi_parent_ops); + if (!imsic->base_domain) { + pr_err("%pfwP: failed to create IMSIC base domain\n", imsic->fwnode); + return -ENOMEM; + } + + global = &imsic->global; + pr_info("%pfwP: hart-index-bits: %d, guest-index-bits: %d\n", + imsic->fwnode, global->hart_index_bits, global->guest_index_bits); + pr_info("%pfwP: group-index-bits: %d, group-index-shift: %d\n", + imsic->fwnode, global->group_index_bits, global->group_index_shift); + pr_info("%pfwP: per-CPU IDs %d at base address %pa\n", + imsic->fwnode, global->nr_ids, &global->base_addr); + pr_info("%pfwP: total %d interrupts available\n", + imsic->fwnode, num_possible_cpus() * (global->nr_ids - 1)); + + return 0; +} + +static int imsic_platform_probe_common(struct fwnode_handle *fwnode) +{ + if (imsic && imsic->fwnode != fwnode) { + pr_err("%pfwP: fwnode mismatch\n", fwnode); + return -ENODEV; + } + + return imsic_irqdomain_init(); +} + +static int imsic_platform_dt_probe(struct platform_device *pdev) +{ + return imsic_platform_probe_common(pdev->dev.fwnode); +} + +#ifdef CONFIG_ACPI + +/* + * On ACPI based systems, PCI enumeration happens early during boot in + * acpi_scan_init(). PCI enumeration expects MSI domain setup before + * it calls pci_set_msi_domain(). Hence, unlike in DT where + * imsic-platform drive probe happens late during boot, ACPI based + * systems need to setup the MSI domain early. + */ +int imsic_platform_acpi_probe(struct fwnode_handle *fwnode) +{ + return imsic_platform_probe_common(fwnode); +} + +#endif + +static const struct of_device_id imsic_platform_match[] = { + { .compatible = "riscv,imsics" }, + {} +}; + +static struct platform_driver imsic_platform_driver = { + .driver = { + .name = "riscv-imsic", + .of_match_table = imsic_platform_match, + }, + .probe = imsic_platform_dt_probe, +}; +builtin_platform_driver(imsic_platform_driver); diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c new file mode 100644 index 000000000000..385368052d5c --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-state.c @@ -0,0 +1,953 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#define pr_fmt(fmt) "riscv-imsic: " fmt +#include <linux/acpi.h> +#include <linux/cpu.h> +#include <linux/bitmap.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/seq_file.h> +#include <linux/spinlock.h> +#include <linux/smp.h> +#include <asm/hwcap.h> + +#include "irq-riscv-imsic-state.h" + +#define IMSIC_DISABLE_EIDELIVERY 0 +#define IMSIC_ENABLE_EIDELIVERY 1 +#define IMSIC_DISABLE_EITHRESHOLD 1 +#define IMSIC_ENABLE_EITHRESHOLD 0 + +static inline void imsic_csr_write(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_write(CSR_IREG, val); +} + +static inline unsigned long imsic_csr_read(unsigned long reg) +{ + csr_write(CSR_ISELECT, reg); + return csr_read(CSR_IREG); +} + +static inline unsigned long imsic_csr_read_clear(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + return csr_read_clear(CSR_IREG, val); +} + +static inline void imsic_csr_set(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_set(CSR_IREG, val); +} + +static inline void imsic_csr_clear(unsigned long reg, unsigned long val) +{ + csr_write(CSR_ISELECT, reg); + csr_clear(CSR_IREG, val); +} + +struct imsic_priv *imsic; + +const struct imsic_global_config *imsic_get_global_config(void) +{ + return imsic ? &imsic->global : NULL; +} +EXPORT_SYMBOL_GPL(imsic_get_global_config); + +static bool __imsic_eix_read_clear(unsigned long id, bool pend) +{ + unsigned long isel, imask; + + isel = id / BITS_PER_LONG; + isel *= BITS_PER_LONG / IMSIC_EIPx_BITS; + isel += pend ? IMSIC_EIP0 : IMSIC_EIE0; + imask = BIT(id & (__riscv_xlen - 1)); + + return !!(imsic_csr_read_clear(isel, imask) & imask); +} + +static inline bool __imsic_id_read_clear_enabled(unsigned long id) +{ + return __imsic_eix_read_clear(id, false); +} + +static inline bool __imsic_id_read_clear_pending(unsigned long id) +{ + return __imsic_eix_read_clear(id, true); +} + +void __imsic_eix_update(unsigned long base_id, unsigned long num_id, bool pend, bool val) +{ + unsigned long id = base_id, last_id = base_id + num_id; + unsigned long i, isel, ireg; + + while (id < last_id) { + isel = id / BITS_PER_LONG; + isel *= BITS_PER_LONG / IMSIC_EIPx_BITS; + isel += pend ? IMSIC_EIP0 : IMSIC_EIE0; + + /* + * Prepare the ID mask to be programmed in the + * IMSIC EIEx and EIPx registers. These registers + * are XLEN-wide and we must not touch IDs which + * are < base_id and >= (base_id + num_id). + */ + ireg = 0; + for (i = id & (__riscv_xlen - 1); id < last_id && i < __riscv_xlen; i++) { + ireg |= BIT(i); + id++; + } + + /* + * The IMSIC EIEx and EIPx registers are indirectly + * accessed via using ISELECT and IREG CSRs so we + * need to access these CSRs without getting preempted. + * + * All existing users of this function call this + * function with local IRQs disabled so we don't + * need to do anything special here. + */ + if (val) + imsic_csr_set(isel, ireg); + else + imsic_csr_clear(isel, ireg); + } +} + +static bool __imsic_local_sync(struct imsic_local_priv *lpriv) +{ + struct imsic_local_config *tlocal, *mlocal; + struct imsic_vector *vec, *tvec, *mvec; + bool ret = true; + int i; + + lockdep_assert_held(&lpriv->lock); + + for_each_set_bit(i, lpriv->dirty_bitmap, imsic->global.nr_ids + 1) { + if (!i || (!imsic_noipi && i == IMSIC_IPI_ID)) + goto skip; + vec = &lpriv->vectors[i]; + + if (READ_ONCE(vec->enable)) + __imsic_id_set_enable(i); + else + __imsic_id_clear_enable(i); + + /* + * Clear the previous vector pointer of the new vector only + * after the movement is complete on the old CPU. + */ + mvec = READ_ONCE(vec->move_prev); + if (mvec) { + /* + * If the old vector has not been updated then + * try again in the next sync-up call. + */ + if (READ_ONCE(mvec->move_next)) { + ret = false; + continue; + } + + WRITE_ONCE(vec->move_prev, NULL); + } + + /* + * If a vector was being moved to a new vector on some other + * CPU then we can get a MSI during the movement so check the + * ID pending bit and re-trigger the new ID on other CPU using + * MMIO write. + */ + mvec = READ_ONCE(vec->move_next); + if (mvec) { + /* + * Devices having non-atomic MSI update might see + * an intermediate state so check both old ID and + * new ID for pending interrupts. + * + * For details, see imsic_irq_set_affinity(). + */ + tvec = vec->local_id == mvec->local_id ? + NULL : &lpriv->vectors[mvec->local_id]; + + if (tvec && !irq_can_move_in_process_context(irq_get_irq_data(vec->irq)) && + __imsic_id_read_clear_pending(tvec->local_id)) { + /* Retrigger temporary vector if it was already in-use */ + if (READ_ONCE(tvec->enable)) { + tlocal = per_cpu_ptr(imsic->global.local, tvec->cpu); + writel_relaxed(tvec->local_id, tlocal->msi_va); + } + + mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); + writel_relaxed(mvec->local_id, mlocal->msi_va); + } + + if (__imsic_id_read_clear_pending(vec->local_id)) { + mlocal = per_cpu_ptr(imsic->global.local, mvec->cpu); + writel_relaxed(mvec->local_id, mlocal->msi_va); + } + + WRITE_ONCE(vec->move_next, NULL); + imsic_vector_free(vec); + } + +skip: + bitmap_clear(lpriv->dirty_bitmap, i, 1); + } + + return ret; +} + +#ifdef CONFIG_SMP +static void __imsic_local_timer_start(struct imsic_local_priv *lpriv, unsigned int cpu) +{ + lockdep_assert_held(&lpriv->lock); + + if (!timer_pending(&lpriv->timer)) { + lpriv->timer.expires = jiffies + 1; + add_timer_on(&lpriv->timer, cpu); + } +} +#else +static inline void __imsic_local_timer_start(struct imsic_local_priv *lpriv, unsigned int cpu) +{ +} +#endif + +void imsic_local_sync_all(bool force_all) +{ + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + unsigned long flags; + + raw_spin_lock_irqsave(&lpriv->lock, flags); + + if (force_all) + bitmap_fill(lpriv->dirty_bitmap, imsic->global.nr_ids + 1); + if (!__imsic_local_sync(lpriv)) + __imsic_local_timer_start(lpriv, smp_processor_id()); + + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +} + +void imsic_local_delivery(bool enable) +{ + if (enable) { + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_ENABLE_EITHRESHOLD); + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_ENABLE_EIDELIVERY); + return; + } + + imsic_csr_write(IMSIC_EIDELIVERY, IMSIC_DISABLE_EIDELIVERY); + imsic_csr_write(IMSIC_EITHRESHOLD, IMSIC_DISABLE_EITHRESHOLD); +} + +#ifdef CONFIG_SMP +static void imsic_local_timer_callback(struct timer_list *timer) +{ + imsic_local_sync_all(false); +} + +static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu) +{ + lockdep_assert_held(&lpriv->lock); + + /* + * The spinlock acquire/release semantics ensure that changes + * to vector enable, vector move and dirty bitmap are visible + * to the target CPU. + */ + + /* + * We schedule a timer on the target CPU if the target CPU is not + * same as the current CPU. An offline CPU will unconditionally + * synchronize IDs through imsic_starting_cpu() when the + * CPU is brought up. + */ + if (cpu_online(cpu)) { + if (cpu == smp_processor_id()) { + if (__imsic_local_sync(lpriv)) + return; + } + + __imsic_local_timer_start(lpriv, cpu); + } +} +#else +static void __imsic_remote_sync(struct imsic_local_priv *lpriv, unsigned int cpu) +{ + lockdep_assert_held(&lpriv->lock); + __imsic_local_sync(lpriv); +} +#endif + +void imsic_vector_mask(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + /* + * This function is called through Linux irq subsystem with + * irqs disabled so no need to save/restore irq flags. + */ + + raw_spin_lock(&lpriv->lock); + + WRITE_ONCE(vec->enable, false); + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock(&lpriv->lock); +} + +void imsic_vector_unmask(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + /* + * This function is called through Linux irq subsystem with + * irqs disabled so no need to save/restore irq flags. + */ + + raw_spin_lock(&lpriv->lock); + + WRITE_ONCE(vec->enable, true); + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock(&lpriv->lock); +} + +void imsic_vector_force_move_cleanup(struct imsic_vector *vec) +{ + struct imsic_local_priv *lpriv; + struct imsic_vector *mvec; + unsigned long flags; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + raw_spin_lock_irqsave(&lpriv->lock, flags); + + mvec = READ_ONCE(vec->move_prev); + WRITE_ONCE(vec->move_prev, NULL); + if (mvec) + imsic_vector_free(mvec); + + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +} + +static bool imsic_vector_move_update(struct imsic_local_priv *lpriv, + struct imsic_vector *vec, bool is_old_vec, + bool new_enable, struct imsic_vector *move_vec) +{ + unsigned long flags; + bool enabled; + + raw_spin_lock_irqsave(&lpriv->lock, flags); + + /* Update enable and move details */ + enabled = READ_ONCE(vec->enable); + WRITE_ONCE(vec->enable, new_enable); + if (is_old_vec) + WRITE_ONCE(vec->move_next, move_vec); + else + WRITE_ONCE(vec->move_prev, move_vec); + + /* Mark the vector as dirty and synchronize */ + bitmap_set(lpriv->dirty_bitmap, vec->local_id, 1); + __imsic_remote_sync(lpriv, vec->cpu); + + raw_spin_unlock_irqrestore(&lpriv->lock, flags); + + return enabled; +} + +void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_vec) +{ + struct imsic_local_priv *old_lpriv, *new_lpriv; + bool enabled; + + if (WARN_ON_ONCE(old_vec->cpu == new_vec->cpu)) + return; + + old_lpriv = per_cpu_ptr(imsic->lpriv, old_vec->cpu); + if (WARN_ON_ONCE(&old_lpriv->vectors[old_vec->local_id] != old_vec)) + return; + + new_lpriv = per_cpu_ptr(imsic->lpriv, new_vec->cpu); + if (WARN_ON_ONCE(&new_lpriv->vectors[new_vec->local_id] != new_vec)) + return; + + /* + * Move and re-trigger the new vector based on the pending + * state of the old vector because we might get a device + * interrupt on the old vector while device was being moved + * to the new vector. + */ + enabled = imsic_vector_move_update(old_lpriv, old_vec, true, false, new_vec); + imsic_vector_move_update(new_lpriv, new_vec, false, enabled, old_vec); +} + +#ifdef CONFIG_GENERIC_IRQ_DEBUGFS +void imsic_vector_debug_show(struct seq_file *m, struct imsic_vector *vec, int ind) +{ + struct imsic_local_priv *lpriv; + struct imsic_vector *mvec; + bool is_enabled; + + lpriv = per_cpu_ptr(imsic->lpriv, vec->cpu); + if (WARN_ON_ONCE(&lpriv->vectors[vec->local_id] != vec)) + return; + + is_enabled = imsic_vector_isenabled(vec); + mvec = imsic_vector_get_move(vec); + + seq_printf(m, "%*starget_cpu : %5u\n", ind, "", vec->cpu); + seq_printf(m, "%*starget_local_id : %5u\n", ind, "", vec->local_id); + seq_printf(m, "%*sis_reserved : %5u\n", ind, "", + (!imsic_noipi && vec->local_id <= IMSIC_IPI_ID) ? 1 : 0); + seq_printf(m, "%*sis_enabled : %5u\n", ind, "", is_enabled ? 1 : 0); + seq_printf(m, "%*sis_move_pending : %5u\n", ind, "", mvec ? 1 : 0); + if (mvec) { + seq_printf(m, "%*smove_cpu : %5u\n", ind, "", mvec->cpu); + seq_printf(m, "%*smove_local_id : %5u\n", ind, "", mvec->local_id); + } +} + +void imsic_vector_debug_show_summary(struct seq_file *m, int ind) +{ + irq_matrix_debug_show(m, imsic->matrix, ind); +} +#endif + +struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask) +{ + struct imsic_vector *vec = NULL; + struct imsic_local_priv *lpriv; + unsigned long flags; + unsigned int cpu; + int local_id; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + local_id = irq_matrix_alloc(imsic->matrix, mask, false, &cpu); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); + if (local_id < 0) + return NULL; + + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + vec = &lpriv->vectors[local_id]; + vec->irq = irq; + vec->enable = false; + vec->move_next = NULL; + vec->move_prev = NULL; + + return vec; +} + +void imsic_vector_free(struct imsic_vector *vec) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + vec->irq = 0; + irq_matrix_free(imsic->matrix, vec->cpu, vec->local_id, false); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); +} + +static void __init imsic_local_cleanup(void) +{ + struct imsic_local_priv *lpriv; + int cpu; + + for_each_possible_cpu(cpu) { + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + + bitmap_free(lpriv->dirty_bitmap); + } + + free_percpu(imsic->lpriv); +} + +static int __init imsic_local_init(void) +{ + struct imsic_global_config *global = &imsic->global; + struct imsic_local_priv *lpriv; + struct imsic_vector *vec; + int cpu, i; + + /* Allocate per-CPU private state */ + imsic->lpriv = __alloc_percpu(struct_size(imsic->lpriv, vectors, global->nr_ids + 1), + __alignof__(*imsic->lpriv)); + if (!imsic->lpriv) + return -ENOMEM; + + /* Setup per-CPU private state */ + for_each_possible_cpu(cpu) { + lpriv = per_cpu_ptr(imsic->lpriv, cpu); + + raw_spin_lock_init(&lpriv->lock); + + /* Allocate dirty bitmap */ + lpriv->dirty_bitmap = bitmap_zalloc(global->nr_ids + 1, GFP_KERNEL); + if (!lpriv->dirty_bitmap) + goto fail_local_cleanup; + +#ifdef CONFIG_SMP + /* Setup lazy timer for synchronization */ + timer_setup(&lpriv->timer, imsic_local_timer_callback, TIMER_PINNED); +#endif + + /* Setup vector array */ + for (i = 0; i <= global->nr_ids; i++) { + vec = &lpriv->vectors[i]; + vec->cpu = cpu; + vec->local_id = i; + vec->irq = 0; + } + } + + return 0; + +fail_local_cleanup: + imsic_local_cleanup(); + return -ENOMEM; +} + +void imsic_state_online(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + irq_matrix_online(imsic->matrix); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); +} + +void imsic_state_offline(void) +{ + unsigned long flags; + + raw_spin_lock_irqsave(&imsic->matrix_lock, flags); + irq_matrix_offline(imsic->matrix); + raw_spin_unlock_irqrestore(&imsic->matrix_lock, flags); + +#ifdef CONFIG_SMP + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); + + raw_spin_lock_irqsave(&lpriv->lock, flags); + WARN_ON_ONCE(timer_delete_sync_try(&lpriv->timer) < 0); + raw_spin_unlock_irqrestore(&lpriv->lock, flags); +#endif +} + +static int __init imsic_matrix_init(void) +{ + struct imsic_global_config *global = &imsic->global; + + raw_spin_lock_init(&imsic->matrix_lock); + imsic->matrix = irq_alloc_matrix(global->nr_ids + 1, + 0, global->nr_ids + 1); + if (!imsic->matrix) + return -ENOMEM; + + /* Reserve ID#0 because it is special and never implemented */ + irq_matrix_assign_system(imsic->matrix, 0, false); + + /* Reserve IPI ID because it is special and used internally */ + if (!imsic_noipi) + irq_matrix_assign_system(imsic->matrix, IMSIC_IPI_ID, false); + + return 0; +} + +static int __init imsic_populate_global_dt(struct fwnode_handle *fwnode, + struct imsic_global_config *global, + u32 *nr_parent_irqs) +{ + int rc; + + /* Find number of guest index bits in MSI address */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,guest-index-bits", + &global->guest_index_bits); + if (rc) + global->guest_index_bits = 0; + + /* Find number of HART index bits */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,hart-index-bits", + &global->hart_index_bits); + if (rc) { + /* Assume default value */ + global->hart_index_bits = __fls(*nr_parent_irqs); + if (BIT(global->hart_index_bits) < *nr_parent_irqs) + global->hart_index_bits++; + } + + /* Find number of group index bits */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-bits", + &global->group_index_bits); + if (rc) + global->group_index_bits = 0; + + /* + * Find first bit position of group index. + * If not specified assumed the default APLIC-IMSIC configuration. + */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,group-index-shift", + &global->group_index_shift); + if (rc) + global->group_index_shift = IMSIC_MMIO_PAGE_SHIFT * 2; + + /* Find number of interrupt identities */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-ids", + &global->nr_ids); + if (rc) { + pr_err("%pfwP: number of interrupt identities not found\n", fwnode); + return rc; + } + + /* Find number of guest interrupt identities */ + rc = of_property_read_u32(to_of_node(fwnode), "riscv,num-guest-ids", + &global->nr_guest_ids); + if (rc) + global->nr_guest_ids = global->nr_ids; + + return 0; +} + +static int __init imsic_populate_global_acpi(struct fwnode_handle *fwnode, + struct imsic_global_config *global, + u32 *nr_parent_irqs, void *opaque) +{ + struct acpi_madt_imsic *imsic = (struct acpi_madt_imsic *)opaque; + + global->guest_index_bits = imsic->guest_index_bits; + global->hart_index_bits = imsic->hart_index_bits; + global->group_index_bits = imsic->group_index_bits; + global->group_index_shift = imsic->group_index_shift; + global->nr_ids = imsic->num_ids; + global->nr_guest_ids = imsic->num_guest_ids; + return 0; +} + +static int __init imsic_get_parent_hartid(struct fwnode_handle *fwnode, + u32 index, unsigned long *hartid) +{ + struct of_phandle_args parent; + int rc; + + if (!is_of_node(fwnode)) { + if (hartid) + *hartid = acpi_rintc_index_to_hartid(index); + + if (!hartid || (*hartid == INVALID_HARTID)) + return -EINVAL; + + return 0; + } + + rc = of_irq_parse_one(to_of_node(fwnode), index, &parent); + if (rc) + return rc; + + /* + * Skip interrupts other than external interrupts for + * current privilege level. + */ + if (parent.args[0] != RV_IRQ_EXT) + return -EINVAL; + + return riscv_of_parent_hartid(parent.np, hartid); +} + +static int __init imsic_get_mmio_resource(struct fwnode_handle *fwnode, + u32 index, struct resource *res) +{ + if (!is_of_node(fwnode)) + return acpi_rintc_get_imsic_mmio_info(index, res); + + return of_address_to_resource(to_of_node(fwnode), index, res); +} + +static int __init imsic_parse_fwnode(struct fwnode_handle *fwnode, + struct imsic_global_config *global, + u32 *nr_parent_irqs, + u32 *nr_mmios, + void *opaque) +{ + unsigned long hartid; + struct resource res; + int rc; + u32 i; + + *nr_parent_irqs = 0; + *nr_mmios = 0; + + /* Find number of parent interrupts */ + while (!imsic_get_parent_hartid(fwnode, *nr_parent_irqs, &hartid)) + (*nr_parent_irqs)++; + if (!*nr_parent_irqs) { + pr_err("%pfwP: no parent irqs available\n", fwnode); + return -EINVAL; + } + + if (is_of_node(fwnode)) + rc = imsic_populate_global_dt(fwnode, global, nr_parent_irqs); + else + rc = imsic_populate_global_acpi(fwnode, global, nr_parent_irqs, opaque); + + if (rc) + return rc; + + /* Sanity check guest index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT; + if (i < global->guest_index_bits) { + pr_err("%pfwP: guest index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check HART index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - global->guest_index_bits; + if (i < global->hart_index_bits) { + pr_err("%pfwP: HART index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check group index bits */ + i = BITS_PER_LONG - IMSIC_MMIO_PAGE_SHIFT - + global->guest_index_bits - global->hart_index_bits; + if (i < global->group_index_bits) { + pr_err("%pfwP: group index bits too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check group index shift */ + i = global->group_index_bits + global->group_index_shift - 1; + if (i >= BITS_PER_LONG) { + pr_err("%pfwP: group index shift too big\n", fwnode); + return -EINVAL; + } + + /* Sanity check number of interrupt identities */ + if (global->nr_ids < IMSIC_MIN_ID || + global->nr_ids >= IMSIC_MAX_ID || + (global->nr_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { + pr_err("%pfwP: invalid number of interrupt identities\n", fwnode); + return -EINVAL; + } + + /* Sanity check number of guest interrupt identities */ + if (global->nr_guest_ids < IMSIC_MIN_ID || + global->nr_guest_ids >= IMSIC_MAX_ID || + (global->nr_guest_ids & IMSIC_MIN_ID) != IMSIC_MIN_ID) { + pr_err("%pfwP: invalid number of guest interrupt identities\n", fwnode); + return -EINVAL; + } + + /* Compute base address */ + rc = imsic_get_mmio_resource(fwnode, 0, &res); + if (rc) { + pr_err("%pfwP: first MMIO resource not found\n", fwnode); + return -EINVAL; + } + global->base_addr = res.start; + global->base_addr &= ~(BIT(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT) - 1); + global->base_addr &= ~((BIT(global->group_index_bits) - 1) << + global->group_index_shift); + + /* Find number of MMIO register sets */ + while (!imsic_get_mmio_resource(fwnode, *nr_mmios, &res)) + (*nr_mmios)++; + + return 0; +} + +int __init imsic_setup_state(struct fwnode_handle *fwnode, void *opaque) +{ + u32 i, j, index, nr_parent_irqs, nr_mmios, nr_handlers = 0; + struct imsic_global_config *global; + struct imsic_local_config *local; + void __iomem **mmios_va = NULL; + struct resource *mmios = NULL; + unsigned long reloff, hartid; + phys_addr_t base_addr; + int rc, cpu; + + /* + * Only one IMSIC instance allowed in a platform for clean + * implementation of SMP IRQ affinity and per-CPU IPIs. + * + * This means on a multi-socket (or multi-die) platform we + * will have multiple MMIO regions for one IMSIC instance. + */ + if (imsic) { + pr_err("%pfwP: already initialized hence ignoring\n", fwnode); + return -EALREADY; + } + + if (!riscv_isa_extension_available(NULL, SxAIA)) { + pr_err("%pfwP: AIA support not available\n", fwnode); + return -ENODEV; + } + + imsic = kzalloc(sizeof(*imsic), GFP_KERNEL); + if (!imsic) + return -ENOMEM; + imsic->fwnode = fwnode; + global = &imsic->global; + + global->local = alloc_percpu(typeof(*global->local)); + if (!global->local) { + rc = -ENOMEM; + goto out_free_priv; + } + + /* Parse IMSIC fwnode */ + rc = imsic_parse_fwnode(fwnode, global, &nr_parent_irqs, &nr_mmios, opaque); + if (rc) + goto out_free_local; + + /* Allocate MMIO resource array */ + mmios = kcalloc(nr_mmios, sizeof(*mmios), GFP_KERNEL); + if (!mmios) { + rc = -ENOMEM; + goto out_free_local; + } + + /* Allocate MMIO virtual address array */ + mmios_va = kcalloc(nr_mmios, sizeof(*mmios_va), GFP_KERNEL); + if (!mmios_va) { + rc = -ENOMEM; + goto out_iounmap; + } + + /* Parse and map MMIO register sets */ + for (i = 0; i < nr_mmios; i++) { + rc = imsic_get_mmio_resource(fwnode, i, &mmios[i]); + if (rc) { + pr_err("%pfwP: unable to parse MMIO regset %d\n", fwnode, i); + goto out_iounmap; + } + + base_addr = mmios[i].start; + base_addr &= ~(BIT(global->guest_index_bits + + global->hart_index_bits + + IMSIC_MMIO_PAGE_SHIFT) - 1); + base_addr &= ~((BIT(global->group_index_bits) - 1) << + global->group_index_shift); + if (base_addr != global->base_addr) { + rc = -EINVAL; + pr_err("%pfwP: address mismatch for regset %d\n", fwnode, i); + goto out_iounmap; + } + + mmios_va[i] = ioremap(mmios[i].start, resource_size(&mmios[i])); + if (!mmios_va[i]) { + rc = -EIO; + pr_err("%pfwP: unable to map MMIO regset %d\n", fwnode, i); + goto out_iounmap; + } + } + + /* Initialize local (or per-CPU )state */ + rc = imsic_local_init(); + if (rc) { + pr_err("%pfwP: failed to initialize local state\n", + fwnode); + goto out_iounmap; + } + + /* Configure handlers for target CPUs */ + for (i = 0; i < nr_parent_irqs; i++) { + rc = imsic_get_parent_hartid(fwnode, i, &hartid); + if (rc) { + pr_warn("%pfwP: hart ID for parent irq%d not found\n", fwnode, i); + continue; + } + + cpu = riscv_hartid_to_cpuid(hartid); + if (cpu < 0) { + pr_warn("%pfwP: invalid cpuid for parent irq%d\n", fwnode, i); + continue; + } + + /* Find MMIO location of MSI page */ + index = nr_mmios; + reloff = i * BIT(global->guest_index_bits) * + IMSIC_MMIO_PAGE_SZ; + for (j = 0; nr_mmios; j++) { + if (reloff < resource_size(&mmios[j])) { + index = j; + break; + } + + /* + * MMIO region size may not be aligned to + * BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ + * if holes are present. + */ + reloff -= ALIGN(resource_size(&mmios[j]), + BIT(global->guest_index_bits) * IMSIC_MMIO_PAGE_SZ); + } + if (index >= nr_mmios) { + pr_warn("%pfwP: MMIO not found for parent irq%d\n", fwnode, i); + continue; + } + + local = per_cpu_ptr(global->local, cpu); + local->msi_pa = mmios[index].start + reloff; + local->msi_va = mmios_va[index] + reloff; + + nr_handlers++; + } + + /* If no CPU handlers found then can't take interrupts */ + if (!nr_handlers) { + pr_err("%pfwP: No CPU handlers found\n", fwnode); + rc = -ENODEV; + goto out_local_cleanup; + } + + /* Initialize matrix allocator */ + rc = imsic_matrix_init(); + if (rc) { + pr_err("%pfwP: failed to create matrix allocator\n", fwnode); + goto out_local_cleanup; + } + + /* We don't need MMIO arrays anymore so let's free-up */ + kfree(mmios_va); + kfree(mmios); + + return 0; + +out_local_cleanup: + imsic_local_cleanup(); +out_iounmap: + for (i = 0; i < nr_mmios; i++) { + if (mmios_va[i]) + iounmap(mmios_va[i]); + } + kfree(mmios_va); + kfree(mmios); +out_free_local: + free_percpu(imsic->global.local); +out_free_priv: + kfree(imsic); + imsic = NULL; + return rc; +} diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h new file mode 100644 index 000000000000..6332501dcbd8 --- /dev/null +++ b/drivers/irqchip/irq-riscv-imsic-state.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2021 Western Digital Corporation or its affiliates. + * Copyright (C) 2022 Ventana Micro Systems Inc. + */ + +#ifndef _IRQ_RISCV_IMSIC_STATE_H +#define _IRQ_RISCV_IMSIC_STATE_H + +#include <linux/irqchip/riscv-imsic.h> +#include <linux/irqdomain.h> +#include <linux/fwnode.h> +#include <linux/timer.h> + +#define IMSIC_IPI_ID 1 +#define IMSIC_NR_IPI 8 + +struct imsic_vector { + /* Fixed details of the vector */ + unsigned int cpu; + unsigned int local_id; + /* Details saved by driver in the vector */ + unsigned int irq; + /* Details accessed using local lock held */ + bool enable; + struct imsic_vector *move_next; + struct imsic_vector *move_prev; +}; + +struct imsic_local_priv { + /* Local lock to protect vector enable/move variables and dirty bitmap */ + raw_spinlock_t lock; + + /* Local dirty bitmap for synchronization */ + unsigned long *dirty_bitmap; + +#ifdef CONFIG_SMP + /* Local timer for synchronization */ + struct timer_list timer; +#endif + + /* Local vector table */ + struct imsic_vector vectors[]; +}; + +struct imsic_priv { + /* Device details */ + struct fwnode_handle *fwnode; + + /* Global configuration common for all HARTs */ + struct imsic_global_config global; + + /* Per-CPU state */ + struct imsic_local_priv __percpu *lpriv; + + /* State of IRQ matrix allocator */ + raw_spinlock_t matrix_lock; + struct irq_matrix *matrix; + + /* IRQ domains (created by platform driver) */ + struct irq_domain *base_domain; +}; + +extern bool imsic_noipi; +extern struct imsic_priv *imsic; + +void __imsic_eix_update(unsigned long base_id, unsigned long num_id, bool pend, bool val); + +static inline void __imsic_id_set_enable(unsigned long id) +{ + __imsic_eix_update(id, 1, false, true); +} + +static inline void __imsic_id_clear_enable(unsigned long id) +{ + __imsic_eix_update(id, 1, false, false); +} + +void imsic_local_sync_all(bool force_all); +void imsic_local_delivery(bool enable); + +void imsic_vector_mask(struct imsic_vector *vec); +void imsic_vector_unmask(struct imsic_vector *vec); + +static inline bool imsic_vector_isenabled(struct imsic_vector *vec) +{ + return READ_ONCE(vec->enable); +} + +static inline struct imsic_vector *imsic_vector_get_move(struct imsic_vector *vec) +{ + return READ_ONCE(vec->move_prev); +} + +void imsic_vector_force_move_cleanup(struct imsic_vector *vec); +void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_vec); + +struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask); +void imsic_vector_free(struct imsic_vector *vector); + +void imsic_vector_debug_show(struct seq_file *m, struct imsic_vector *vec, int ind); +void imsic_vector_debug_show_summary(struct seq_file *m, int ind); + +void imsic_state_online(void); +void imsic_state_offline(void); +int imsic_setup_state(struct fwnode_handle *fwnode, void *opaque); +int imsic_irqdomain_init(void); + +#endif diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c index 4adeee1bc391..70290b35b317 100644 --- a/drivers/irqchip/irq-riscv-intc.c +++ b/drivers/irqchip/irq-riscv-intc.c @@ -17,17 +17,29 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/smp.h> +#include <linux/soc/andes/irq.h> + +#include <asm/hwcap.h> static struct irq_domain *intc_domain; +static unsigned int riscv_intc_nr_irqs __ro_after_init = BITS_PER_LONG; +static unsigned int riscv_intc_custom_base __ro_after_init = BITS_PER_LONG; +static unsigned int riscv_intc_custom_nr_irqs __ro_after_init; -static asmlinkage void riscv_intc_irq(struct pt_regs *regs) +static void riscv_intc_irq(struct pt_regs *regs) { unsigned long cause = regs->cause & ~CAUSE_IRQ_FLAG; - if (unlikely(cause >= BITS_PER_LONG)) - panic("unexpected interrupt cause"); + if (generic_handle_domain_irq(intc_domain, cause)) + pr_warn_ratelimited("Failed to handle interrupt (cause: %ld)\n", cause); +} - generic_handle_domain_irq(intc_domain, cause); +static void riscv_intc_aia_irq(struct pt_regs *regs) +{ + unsigned long topi; + + while ((topi = csr_read(CSR_TOPI))) + generic_handle_domain_irq(intc_domain, topi >> TOPI_IID_SHIFT); } /* @@ -39,12 +51,43 @@ static asmlinkage void riscv_intc_irq(struct pt_regs *regs) static void riscv_intc_irq_mask(struct irq_data *d) { - csr_clear(CSR_IE, BIT(d->hwirq)); + if (IS_ENABLED(CONFIG_32BIT) && d->hwirq >= BITS_PER_LONG) + csr_clear(CSR_IEH, BIT(d->hwirq - BITS_PER_LONG)); + else + csr_clear(CSR_IE, BIT(d->hwirq)); } static void riscv_intc_irq_unmask(struct irq_data *d) { - csr_set(CSR_IE, BIT(d->hwirq)); + if (IS_ENABLED(CONFIG_32BIT) && d->hwirq >= BITS_PER_LONG) + csr_set(CSR_IEH, BIT(d->hwirq - BITS_PER_LONG)); + else + csr_set(CSR_IE, BIT(d->hwirq)); +} + +static void andes_intc_irq_mask(struct irq_data *d) +{ + /* + * Andes specific S-mode local interrupt causes (hwirq) + * are defined as (256 + n) and controlled by n-th bit + * of SLIE. + */ + unsigned int mask = BIT(d->hwirq % BITS_PER_LONG); + + if (d->hwirq < ANDES_SLI_CAUSE_BASE) + csr_clear(CSR_IE, mask); + else + csr_clear(ANDES_CSR_SLIE, mask); +} + +static void andes_intc_irq_unmask(struct irq_data *d) +{ + unsigned int mask = BIT(d->hwirq % BITS_PER_LONG); + + if (d->hwirq < ANDES_SLI_CAUSE_BASE) + csr_set(CSR_IE, mask); + else + csr_set(ANDES_CSR_SLIE, mask); } static void riscv_intc_irq_eoi(struct irq_data *d) @@ -70,12 +113,21 @@ static struct irq_chip riscv_intc_chip = { .irq_eoi = riscv_intc_irq_eoi, }; +static struct irq_chip andes_intc_chip = { + .name = "RISC-V INTC", + .irq_mask = andes_intc_irq_mask, + .irq_unmask = andes_intc_irq_unmask, + .irq_eoi = riscv_intc_irq_eoi, +}; + static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) { + struct irq_chip *chip = d->host_data; + irq_set_percpu_devid(irq); - irq_domain_set_info(d, irq, hwirq, &riscv_intc_chip, d->host_data, - handle_percpu_devid_irq, NULL, NULL); + irq_domain_set_info(d, irq, hwirq, chip, NULL, handle_percpu_devid_irq, + NULL, NULL); return 0; } @@ -93,6 +145,15 @@ static int riscv_intc_domain_alloc(struct irq_domain *domain, if (ret) return ret; + /* + * Only allow hwirq for which we have corresponding standard or + * custom interrupt enable register. + */ + if (hwirq >= riscv_intc_nr_irqs && + (hwirq < riscv_intc_custom_base || + hwirq >= riscv_intc_custom_base + riscv_intc_custom_nr_irqs)) + return -EINVAL; + for (i = 0; i < nr_irqs; i++) { ret = riscv_intc_domain_map(domain, virq + i, hwirq + i); if (ret) @@ -105,7 +166,8 @@ static int riscv_intc_domain_alloc(struct irq_domain *domain, static const struct irq_domain_ops riscv_intc_domain_ops = { .map = riscv_intc_domain_map, .xlate = irq_domain_xlate_onecell, - .alloc = riscv_intc_domain_alloc + .alloc = riscv_intc_domain_alloc, + .free = irq_domain_free_irqs_top, }; static struct fwnode_handle *riscv_intc_hwnode(void) @@ -113,18 +175,22 @@ static struct fwnode_handle *riscv_intc_hwnode(void) return intc_domain->fwnode; } -static int __init riscv_intc_init_common(struct fwnode_handle *fn) +static int __init riscv_intc_init_common(struct fwnode_handle *fn, struct irq_chip *chip) { int rc; - intc_domain = irq_domain_create_linear(fn, BITS_PER_LONG, - &riscv_intc_domain_ops, NULL); + intc_domain = irq_domain_create_tree(fn, &riscv_intc_domain_ops, chip); if (!intc_domain) { pr_err("unable to add IRQ domain\n"); return -ENXIO; } - rc = set_handle_irq(&riscv_intc_irq); + if (riscv_isa_extension_available(NULL, SxAIA)) { + riscv_intc_nr_irqs = 64; + rc = set_handle_irq(&riscv_intc_aia_irq); + } else { + rc = set_handle_irq(&riscv_intc_irq); + } if (rc) { pr_err("failed to set irq handler\n"); return rc; @@ -132,7 +198,11 @@ static int __init riscv_intc_init_common(struct fwnode_handle *fn) riscv_set_intc_hwnode_fn(riscv_intc_hwnode); - pr_info("%d local interrupts mapped\n", BITS_PER_LONG); + pr_info("%d local interrupts mapped%s\n", + riscv_intc_nr_irqs, + riscv_isa_extension_available(NULL, SxAIA) ? " using AIA" : ""); + if (riscv_intc_custom_nr_irqs) + pr_info("%d custom local interrupts mapped\n", riscv_intc_custom_nr_irqs); return 0; } @@ -140,8 +210,9 @@ static int __init riscv_intc_init_common(struct fwnode_handle *fn) static int __init riscv_intc_init(struct device_node *node, struct device_node *parent) { - int rc; + struct irq_chip *chip = &riscv_intc_chip; unsigned long hartid; + int rc; rc = riscv_of_parent_hartid(node, &hartid); if (rc < 0) { @@ -155,23 +226,144 @@ static int __init riscv_intc_init(struct device_node *node, * for each INTC DT node. We only need to do INTC initialization * for the INTC DT node belonging to boot CPU (or boot HART). */ - if (riscv_hartid_to_cpuid(hartid) != smp_processor_id()) + if (riscv_hartid_to_cpuid(hartid) != smp_processor_id()) { + /* + * The INTC nodes of each CPU are suppliers for downstream + * interrupt controllers (such as PLIC, IMSIC and APLIC + * direct-mode) so we should mark an INTC node as initialized + * if we are not creating IRQ domain for it. + */ + fwnode_dev_initialized(of_fwnode_handle(node), true); return 0; + } + + if (of_device_is_compatible(node, "andestech,cpu-intc")) { + riscv_intc_custom_base = ANDES_SLI_CAUSE_BASE; + riscv_intc_custom_nr_irqs = ANDES_RV_IRQ_LAST; + chip = &andes_intc_chip; + } - return riscv_intc_init_common(of_node_to_fwnode(node)); + return riscv_intc_init_common(of_fwnode_handle(node), chip); } IRQCHIP_DECLARE(riscv, "riscv,cpu-intc", riscv_intc_init); +IRQCHIP_DECLARE(andes, "andestech,cpu-intc", riscv_intc_init); #ifdef CONFIG_ACPI +struct rintc_data { + union { + u32 ext_intc_id; + struct { + u32 context_id : 16, + reserved : 8, + aplic_plic_id : 8; + }; + }; + unsigned long hart_id; + u64 imsic_addr; + u32 imsic_size; +}; + +static u32 nr_rintc; +static struct rintc_data **rintc_acpi_data; + +#define for_each_matching_plic(_plic_id) \ + unsigned int _plic; \ + \ + for (_plic = 0; _plic < nr_rintc; _plic++) \ + if (rintc_acpi_data[_plic]->aplic_plic_id != _plic_id) \ + continue; \ + else + +unsigned int acpi_rintc_get_plic_nr_contexts(unsigned int plic_id) +{ + unsigned int nctx = 0; + + for_each_matching_plic(plic_id) + nctx++; + + return nctx; +} + +static struct rintc_data *get_plic_context(unsigned int plic_id, unsigned int ctxt_idx) +{ + unsigned int ctxt = 0; + + for_each_matching_plic(plic_id) { + if (ctxt == ctxt_idx) + return rintc_acpi_data[_plic]; + + ctxt++; + } + + return NULL; +} + +unsigned long acpi_rintc_ext_parent_to_hartid(unsigned int plic_id, unsigned int ctxt_idx) +{ + struct rintc_data *data = get_plic_context(plic_id, ctxt_idx); + + return data ? data->hart_id : INVALID_HARTID; +} + +unsigned int acpi_rintc_get_plic_context(unsigned int plic_id, unsigned int ctxt_idx) +{ + struct rintc_data *data = get_plic_context(plic_id, ctxt_idx); + + return data ? data->context_id : INVALID_CONTEXT; +} + +unsigned long acpi_rintc_index_to_hartid(u32 index) +{ + return index >= nr_rintc ? INVALID_HARTID : rintc_acpi_data[index]->hart_id; +} + +int acpi_rintc_get_imsic_mmio_info(u32 index, struct resource *res) +{ + if (index >= nr_rintc) + return -1; + + res->start = rintc_acpi_data[index]->imsic_addr; + res->end = res->start + rintc_acpi_data[index]->imsic_size - 1; + res->flags = IORESOURCE_MEM; + return 0; +} + +static int __init riscv_intc_acpi_match(union acpi_subtable_headers *header, + const unsigned long end) +{ + return 0; +} + static int __init riscv_intc_acpi_init(union acpi_subtable_headers *header, const unsigned long end) { - struct fwnode_handle *fn; struct acpi_madt_rintc *rintc; + struct fwnode_handle *fn; + int count; + int rc; + + if (!rintc_acpi_data) { + count = acpi_table_parse_madt(ACPI_MADT_TYPE_RINTC, riscv_intc_acpi_match, 0); + if (count <= 0) + return -EINVAL; + + rintc_acpi_data = kcalloc(count, sizeof(*rintc_acpi_data), GFP_KERNEL); + if (!rintc_acpi_data) + return -ENOMEM; + } rintc = (struct acpi_madt_rintc *)header; + rintc_acpi_data[nr_rintc] = kzalloc(sizeof(*rintc_acpi_data[0]), GFP_KERNEL); + if (!rintc_acpi_data[nr_rintc]) + return -ENOMEM; + + rintc_acpi_data[nr_rintc]->ext_intc_id = rintc->ext_intc_id; + rintc_acpi_data[nr_rintc]->hart_id = rintc->hart_id; + rintc_acpi_data[nr_rintc]->imsic_addr = rintc->imsic_addr; + rintc_acpi_data[nr_rintc]->imsic_size = rintc->imsic_size; + nr_rintc++; /* * The ACPI MADT will have one INTC for each CPU (or HART) @@ -188,7 +380,13 @@ static int __init riscv_intc_acpi_init(union acpi_subtable_headers *header, return -ENOMEM; } - return riscv_intc_init_common(fn); + rc = riscv_intc_init_common(fn, &riscv_intc_chip); + if (rc) + irq_domain_free_fwnode(fn); + else + acpi_set_irq_model(ACPI_IRQ_MODEL_RINTC, riscv_acpi_get_gsi_domain_id); + + return rc; } IRQCHIP_ACPI_DECLARE(riscv_intc, ACPI_MADT_TYPE_RINTC, NULL, diff --git a/drivers/irqchip/irq-riscv-rpmi-sysmsi.c b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c new file mode 100644 index 000000000000..5c74c561ce31 --- /dev/null +++ b/drivers/irqchip/irq-riscv-rpmi-sysmsi.c @@ -0,0 +1,328 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2025 Ventana Micro Systems Inc. */ + +#include <linux/acpi.h> +#include <linux/bits.h> +#include <linux/bug.h> +#include <linux/device.h> +#include <linux/device/devres.h> +#include <linux/dev_printk.h> +#include <linux/errno.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/irqchip/riscv-imsic.h> +#include <linux/mailbox_client.h> +#include <linux/mailbox/riscv-rpmi-message.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/types.h> + +struct rpmi_sysmsi_get_attrs_rx { + __le32 status; + __le32 sys_num_msi; + __le32 flag0; + __le32 flag1; +}; + +#define RPMI_SYSMSI_MSI_ATTRIBUTES_FLAG0_PREF_PRIV BIT(0) + +struct rpmi_sysmsi_set_msi_state_tx { + __le32 sys_msi_index; + __le32 sys_msi_state; +}; + +struct rpmi_sysmsi_set_msi_state_rx { + __le32 status; +}; + +#define RPMI_SYSMSI_MSI_STATE_ENABLE BIT(0) +#define RPMI_SYSMSI_MSI_STATE_PENDING BIT(1) + +struct rpmi_sysmsi_set_msi_target_tx { + __le32 sys_msi_index; + __le32 sys_msi_address_low; + __le32 sys_msi_address_high; + __le32 sys_msi_data; +}; + +struct rpmi_sysmsi_set_msi_target_rx { + __le32 status; +}; + +struct rpmi_sysmsi_priv { + struct device *dev; + struct mbox_client client; + struct mbox_chan *chan; + u32 nr_irqs; + u32 gsi_base; +}; + +static int rpmi_sysmsi_get_num_msi(struct rpmi_sysmsi_priv *priv) +{ + struct rpmi_sysmsi_get_attrs_rx rx; + struct rpmi_mbox_message msg; + int ret; + + rpmi_mbox_init_send_with_response(&msg, RPMI_SYSMSI_SRV_GET_ATTRIBUTES, + NULL, 0, &rx, sizeof(rx)); + ret = rpmi_mbox_send_message(priv->chan, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return le32_to_cpu(rx.sys_num_msi); +} + +static int rpmi_sysmsi_set_msi_state(struct rpmi_sysmsi_priv *priv, + u32 sys_msi_index, u32 sys_msi_state) +{ + struct rpmi_sysmsi_set_msi_state_tx tx; + struct rpmi_sysmsi_set_msi_state_rx rx; + struct rpmi_mbox_message msg; + int ret; + + tx.sys_msi_index = cpu_to_le32(sys_msi_index); + tx.sys_msi_state = cpu_to_le32(sys_msi_state); + rpmi_mbox_init_send_with_response(&msg, RPMI_SYSMSI_SRV_SET_MSI_STATE, + &tx, sizeof(tx), &rx, sizeof(rx)); + ret = rpmi_mbox_send_message(priv->chan, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + +static int rpmi_sysmsi_set_msi_target(struct rpmi_sysmsi_priv *priv, + u32 sys_msi_index, struct msi_msg *m) +{ + struct rpmi_sysmsi_set_msi_target_tx tx; + struct rpmi_sysmsi_set_msi_target_rx rx; + struct rpmi_mbox_message msg; + int ret; + + tx.sys_msi_index = cpu_to_le32(sys_msi_index); + tx.sys_msi_address_low = cpu_to_le32(m->address_lo); + tx.sys_msi_address_high = cpu_to_le32(m->address_hi); + tx.sys_msi_data = cpu_to_le32(m->data); + rpmi_mbox_init_send_with_response(&msg, RPMI_SYSMSI_SRV_SET_MSI_TARGET, + &tx, sizeof(tx), &rx, sizeof(rx)); + ret = rpmi_mbox_send_message(priv->chan, &msg); + if (ret) + return ret; + if (rx.status) + return rpmi_to_linux_error(le32_to_cpu(rx.status)); + + return 0; +} + +static void rpmi_sysmsi_irq_mask(struct irq_data *d) +{ + struct rpmi_sysmsi_priv *priv = irq_data_get_irq_chip_data(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + int ret; + + ret = rpmi_sysmsi_set_msi_state(priv, hwirq, 0); + if (ret) + dev_warn(priv->dev, "Failed to mask hwirq %lu (error %d)\n", hwirq, ret); + irq_chip_mask_parent(d); +} + +static void rpmi_sysmsi_irq_unmask(struct irq_data *d) +{ + struct rpmi_sysmsi_priv *priv = irq_data_get_irq_chip_data(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + int ret; + + irq_chip_unmask_parent(d); + ret = rpmi_sysmsi_set_msi_state(priv, hwirq, RPMI_SYSMSI_MSI_STATE_ENABLE); + if (ret) + dev_warn(priv->dev, "Failed to unmask hwirq %lu (error %d)\n", hwirq, ret); +} + +static void rpmi_sysmsi_write_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct rpmi_sysmsi_priv *priv = irq_data_get_irq_chip_data(d); + irq_hw_number_t hwirq = irqd_to_hwirq(d); + int ret; + + /* For zeroed MSI, do nothing as of now */ + if (!msg->address_hi && !msg->address_lo && !msg->data) + return; + + ret = rpmi_sysmsi_set_msi_target(priv, hwirq, msg); + if (ret) + dev_warn(priv->dev, "Failed to set target for hwirq %lu (error %d)\n", hwirq, ret); +} + +static void rpmi_sysmsi_set_desc(msi_alloc_info_t *arg, struct msi_desc *desc) +{ + arg->desc = desc; + arg->hwirq = desc->data.icookie.value; +} + +static int rpmi_sysmsi_translate(struct irq_domain *d, struct irq_fwspec *fwspec, + unsigned long *hwirq, unsigned int *type) +{ + struct msi_domain_info *info = d->host_data; + struct rpmi_sysmsi_priv *priv = info->data; + + if (WARN_ON(fwspec->param_count < 1)) + return -EINVAL; + + /* For DT, gsi_base is always zero. */ + *hwirq = fwspec->param[0] - priv->gsi_base; + *type = IRQ_TYPE_NONE; + return 0; +} + +static const struct msi_domain_template rpmi_sysmsi_template = { + .chip = { + .name = "RPMI-SYSMSI", + .irq_mask = rpmi_sysmsi_irq_mask, + .irq_unmask = rpmi_sysmsi_irq_unmask, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif + .irq_write_msi_msg = rpmi_sysmsi_write_msg, + .flags = IRQCHIP_SET_TYPE_MASKED | + IRQCHIP_SKIP_SET_WAKE | + IRQCHIP_MASK_ON_SUSPEND, + }, + + .ops = { + .set_desc = rpmi_sysmsi_set_desc, + .msi_translate = rpmi_sysmsi_translate, + }, + + .info = { + .bus_token = DOMAIN_BUS_WIRED_TO_MSI, + .flags = MSI_FLAG_USE_DEV_FWNODE, + .handler = handle_simple_irq, + .handler_name = "simple", + }, +}; + +static int rpmi_sysmsi_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct rpmi_sysmsi_priv *priv; + struct fwnode_handle *fwnode; + u32 id; + int rc; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + priv->dev = dev; + + /* Setup mailbox client */ + priv->client.dev = priv->dev; + priv->client.rx_callback = NULL; + priv->client.tx_block = false; + priv->client.knows_txdone = true; + priv->client.tx_tout = 0; + + /* Request mailbox channel */ + priv->chan = mbox_request_channel(&priv->client, 0); + if (IS_ERR(priv->chan)) + return PTR_ERR(priv->chan); + + /* Get number of system MSIs */ + rc = rpmi_sysmsi_get_num_msi(priv); + if (rc < 1) { + mbox_free_channel(priv->chan); + if (rc) + return dev_err_probe(dev, rc, "Failed to get number of system MSIs\n"); + else + return dev_err_probe(dev, -ENODEV, "No system MSIs found\n"); + } + priv->nr_irqs = rc; + + fwnode = dev_fwnode(dev); + if (is_acpi_node(fwnode)) { + u32 nr_irqs; + + rc = riscv_acpi_get_gsi_info(fwnode, &priv->gsi_base, &id, + &nr_irqs, NULL); + if (rc) { + dev_err(dev, "failed to find GSI mapping\n"); + return rc; + } + + /* Update with actual GSI range */ + if (nr_irqs != priv->nr_irqs) + riscv_acpi_update_gsi_range(priv->gsi_base, priv->nr_irqs); + } + + /* + * The device MSI domain for platform devices on RISC-V architecture + * is only available after the MSI controller driver is probed so, + * explicitly configure here. + */ + if (!dev_get_msi_domain(dev)) { + /* + * The device MSI domain for OF devices is only set at the + * time of populating/creating OF device. If the device MSI + * domain is discovered later after the OF device is created + * then we need to set it explicitly before using any platform + * MSI functions. + */ + if (is_of_node(fwnode)) { + of_msi_configure(dev, dev_of_node(dev)); + } else if (is_acpi_device_node(fwnode)) { + struct irq_domain *msi_domain; + + msi_domain = irq_find_matching_fwnode(imsic_acpi_get_fwnode(dev), + DOMAIN_BUS_PLATFORM_MSI); + dev_set_msi_domain(dev, msi_domain); + } + + if (!dev_get_msi_domain(dev)) { + mbox_free_channel(priv->chan); + return -EPROBE_DEFER; + } + } + + if (!msi_create_device_irq_domain(dev, MSI_DEFAULT_DOMAIN, + &rpmi_sysmsi_template, + priv->nr_irqs, priv, priv)) { + mbox_free_channel(priv->chan); + return dev_err_probe(dev, -ENOMEM, "failed to create MSI irq domain\n"); + } + +#ifdef CONFIG_ACPI + struct acpi_device *adev = ACPI_COMPANION(dev); + + if (adev) + acpi_dev_clear_dependencies(adev); +#endif + + dev_info(dev, "%u system MSIs registered\n", priv->nr_irqs); + return 0; +} + +static const struct of_device_id rpmi_sysmsi_match[] = { + { .compatible = "riscv,rpmi-system-msi" }, + {} +}; + +static const struct acpi_device_id acpi_rpmi_sysmsi_match[] = { + { "RSCV0006" }, + {} +}; +MODULE_DEVICE_TABLE(acpi, acpi_rpmi_sysmsi_match); + +static struct platform_driver rpmi_sysmsi_driver = { + .driver = { + .name = "rpmi-sysmsi", + .of_match_table = rpmi_sysmsi_match, + .acpi_match_table = acpi_rpmi_sysmsi_match, + }, + .probe = rpmi_sysmsi_probe, +}; +builtin_platform_driver(rpmi_sysmsi_driver); diff --git a/drivers/irqchip/irq-sa11x0.c b/drivers/irqchip/irq-sa11x0.c index 31c202a1ae62..e5f24c5f3f41 100644 --- a/drivers/irqchip/irq-sa11x0.c +++ b/drivers/irqchip/irq-sa11x0.c @@ -85,7 +85,7 @@ static struct sa1100irq_state { unsigned int iccr; } sa1100irq_state; -static int sa1100irq_suspend(void) +static int sa1100irq_suspend(void *data) { struct sa1100irq_state *st = &sa1100irq_state; @@ -102,7 +102,7 @@ static int sa1100irq_suspend(void) return 0; } -static void sa1100irq_resume(void) +static void sa1100irq_resume(void *data) { struct sa1100irq_state *st = &sa1100irq_state; @@ -114,21 +114,24 @@ static void sa1100irq_resume(void) } } -static struct syscore_ops sa1100irq_syscore_ops = { +static const struct syscore_ops sa1100irq_syscore_ops = { .suspend = sa1100irq_suspend, .resume = sa1100irq_resume, }; +static struct syscore sa1100irq_syscore = { + .ops = &sa1100irq_syscore_ops, +}; + static int __init sa1100irq_init_devicefs(void) { - register_syscore_ops(&sa1100irq_syscore_ops); + register_syscore(&sa1100irq_syscore); return 0; } device_initcall(sa1100irq_init_devicefs); -static asmlinkage void __exception_irq_entry -sa1100_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry sa1100_handle_irq(struct pt_regs *regs) { uint32_t icip, icmr, mask; @@ -163,7 +166,7 @@ void __init sa11x0_init_irq_nodt(int irq_start, resource_size_t io_start) */ writel_relaxed(1, iobase + ICCR); - sa1100_normal_irqdomain = irq_domain_add_simple(NULL, + sa1100_normal_irqdomain = irq_domain_create_simple(NULL, 32, irq_start, &sa1100_normal_irqdomain_ops, NULL); diff --git a/drivers/irqchip/irq-sg2042-msi.c b/drivers/irqchip/irq-sg2042-msi.c new file mode 100644 index 000000000000..f7cf0dc72eab --- /dev/null +++ b/drivers/irqchip/irq-sg2042-msi.c @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * SG2042 MSI Controller + * + * Copyright (C) 2024 Sophgo Technology Inc. + * Copyright (C) 2024 Chen Wang <unicorn_wang@outlook.com> + */ + +#include <linux/cleanup.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/msi.h> +#include <linux/platform_device.h> +#include <linux/property.h> +#include <linux/slab.h> + +#include <linux/irqchip/irq-msi-lib.h> + +struct sg204x_msi_chip_info { + const struct irq_chip *irqchip; + const struct msi_parent_ops *parent_ops; +}; + +/** + * struct sg204x_msi_chipdata - chip data for the SG204x MSI IRQ controller + * @reg_clr: clear reg, see TRM, 10.1.33, GP_INTR0_CLR + * @doorbell_addr: see TRM, 10.1.32, GP_INTR0_SET + * @irq_first: First vectors number that MSIs starts + * @num_irqs: Number of vectors for MSIs + * @irq_type: IRQ type for MSIs + * @msi_map: mapping for allocated MSI vectors. + * @msi_map_lock: Lock for msi_map + * @chip_info: chip specific infomations + */ +struct sg204x_msi_chipdata { + void __iomem *reg_clr; + + phys_addr_t doorbell_addr; + + u32 irq_first; + u32 num_irqs; + unsigned int irq_type; + + unsigned long *msi_map; + struct mutex msi_map_lock; + + const struct sg204x_msi_chip_info *chip_info; +}; + +static int sg204x_msi_allocate_hwirq(struct sg204x_msi_chipdata *data, int num_req) +{ + int first; + + guard(mutex)(&data->msi_map_lock); + first = bitmap_find_free_region(data->msi_map, data->num_irqs, + get_count_order(num_req)); + return first >= 0 ? first : -ENOSPC; +} + +static void sg204x_msi_free_hwirq(struct sg204x_msi_chipdata *data, int hwirq, int num_req) +{ + guard(mutex)(&data->msi_map_lock); + bitmap_release_region(data->msi_map, hwirq, get_count_order(num_req)); +} + +static void sg2042_msi_irq_ack(struct irq_data *d) +{ + struct sg204x_msi_chipdata *data = irq_data_get_irq_chip_data(d); + int bit_off = d->hwirq; + + writel(1 << bit_off, data->reg_clr); + + irq_chip_ack_parent(d); +} + +static void sg2042_msi_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct sg204x_msi_chipdata *data = irq_data_get_irq_chip_data(d); + + msg->address_hi = upper_32_bits(data->doorbell_addr); + msg->address_lo = lower_32_bits(data->doorbell_addr); + msg->data = 1 << d->hwirq; +} + +static const struct irq_chip sg2042_msi_middle_irq_chip = { + .name = "SG2042 MSI", + .irq_startup = irq_chip_startup_parent, + .irq_shutdown = irq_chip_shutdown_parent, + .irq_ack = sg2042_msi_irq_ack, + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif + .irq_compose_msi_msg = sg2042_msi_irq_compose_msi_msg, +}; + +static void sg2044_msi_irq_ack(struct irq_data *d) +{ + struct sg204x_msi_chipdata *data = irq_data_get_irq_chip_data(d); + + writel(0, (u32 __iomem *)data->reg_clr + d->hwirq); + irq_chip_ack_parent(d); +} + +static void sg2044_msi_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg) +{ + struct sg204x_msi_chipdata *data = irq_data_get_irq_chip_data(d); + phys_addr_t doorbell = data->doorbell_addr + 4 * (d->hwirq / 32); + + msg->address_lo = lower_32_bits(doorbell); + msg->address_hi = upper_32_bits(doorbell); + msg->data = d->hwirq % 32; +} + +static struct irq_chip sg2044_msi_middle_irq_chip = { + .name = "SG2044 MSI", + .irq_startup = irq_chip_startup_parent, + .irq_shutdown = irq_chip_shutdown_parent, + .irq_ack = sg2044_msi_irq_ack, + .irq_mask = irq_chip_mask_parent, + .irq_unmask = irq_chip_unmask_parent, +#ifdef CONFIG_SMP + .irq_set_affinity = irq_chip_set_affinity_parent, +#endif + .irq_compose_msi_msg = sg2044_msi_irq_compose_msi_msg, +}; + +static int sg204x_msi_parent_domain_alloc(struct irq_domain *domain, unsigned int virq, int hwirq) +{ + struct sg204x_msi_chipdata *data = domain->host_data; + struct irq_fwspec fwspec; + struct irq_data *d; + int ret; + + fwspec.fwnode = domain->parent->fwnode; + fwspec.param_count = 2; + fwspec.param[0] = data->irq_first + hwirq; + fwspec.param[1] = data->irq_type; + + ret = irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec); + if (ret) + return ret; + + d = irq_domain_get_irq_data(domain->parent, virq); + return d->chip->irq_set_type(d, data->irq_type); +} + +static int sg204x_msi_middle_domain_alloc(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs, void *args) +{ + struct sg204x_msi_chipdata *data = domain->host_data; + int hwirq, err, i; + + hwirq = sg204x_msi_allocate_hwirq(data, nr_irqs); + if (hwirq < 0) + return hwirq; + + for (i = 0; i < nr_irqs; i++) { + err = sg204x_msi_parent_domain_alloc(domain, virq + i, hwirq + i); + if (err) + goto err_hwirq; + + irq_domain_set_hwirq_and_chip(domain, virq + i, hwirq + i, + data->chip_info->irqchip, data); + } + return 0; + +err_hwirq: + sg204x_msi_free_hwirq(data, hwirq, nr_irqs); + irq_domain_free_irqs_parent(domain, virq, i); + return err; +} + +static void sg204x_msi_middle_domain_free(struct irq_domain *domain, unsigned int virq, + unsigned int nr_irqs) +{ + struct irq_data *d = irq_domain_get_irq_data(domain, virq); + struct sg204x_msi_chipdata *data = irq_data_get_irq_chip_data(d); + + irq_domain_free_irqs_parent(domain, virq, nr_irqs); + sg204x_msi_free_hwirq(data, d->hwirq, nr_irqs); +} + +static const struct irq_domain_ops sg204x_msi_middle_domain_ops = { + .alloc = sg204x_msi_middle_domain_alloc, + .free = sg204x_msi_middle_domain_free, + .select = msi_lib_irq_domain_select, +}; + +#define SG2042_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT | \ + MSI_FLAG_PCI_MSI_STARTUP_PARENT) + +#define SG2042_MSI_FLAGS_SUPPORTED MSI_GENERIC_FLAGS_MASK + +static const struct msi_parent_ops sg2042_msi_parent_ops = { + .required_flags = SG2042_MSI_FLAGS_REQUIRED, + .supported_flags = SG2042_MSI_FLAGS_SUPPORTED, + .chip_flags = MSI_CHIP_FLAG_SET_ACK, + .bus_select_mask = MATCH_PCI_MSI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .prefix = "SG2042-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +#define SG2044_MSI_FLAGS_REQUIRED (MSI_FLAG_USE_DEF_DOM_OPS | \ + MSI_FLAG_USE_DEF_CHIP_OPS | \ + MSI_FLAG_PCI_MSI_MASK_PARENT | \ + MSI_FLAG_PCI_MSI_STARTUP_PARENT) + +#define SG2044_MSI_FLAGS_SUPPORTED (MSI_GENERIC_FLAGS_MASK | \ + MSI_FLAG_MULTI_PCI_MSI | \ + MSI_FLAG_PCI_MSIX) + +static const struct msi_parent_ops sg2044_msi_parent_ops = { + .required_flags = SG2044_MSI_FLAGS_REQUIRED, + .supported_flags = SG2044_MSI_FLAGS_SUPPORTED, + .chip_flags = MSI_CHIP_FLAG_SET_EOI | MSI_CHIP_FLAG_SET_ACK, + .bus_select_mask = MATCH_PCI_MSI, + .bus_select_token = DOMAIN_BUS_NEXUS, + .prefix = "SG2044-", + .init_dev_msi_info = msi_lib_init_dev_msi_info, +}; + +static int sg204x_msi_init_domains(struct sg204x_msi_chipdata *data, + struct irq_domain *plic_domain, struct device *dev) +{ + struct irq_domain_info info = { + .ops = &sg204x_msi_middle_domain_ops, + .parent = plic_domain, + .size = data->num_irqs, + .fwnode = dev_fwnode(dev), + .host_data = data, + }; + + if (!msi_create_parent_irq_domain(&info, data->chip_info->parent_ops)) { + pr_err("Failed to create the MSI middle domain\n"); + return -ENOMEM; + } + return 0; +} + +static int sg2042_msi_probe(struct platform_device *pdev) +{ + struct fwnode_reference_args args = { }; + struct sg204x_msi_chipdata *data; + struct device *dev = &pdev->dev; + struct irq_domain *plic_domain; + struct resource *res; + int ret; + + data = devm_kzalloc(dev, sizeof(struct sg204x_msi_chipdata), GFP_KERNEL); + if (!data) + return -ENOMEM; + + data->chip_info = device_get_match_data(&pdev->dev); + if (!data->chip_info) { + dev_err(&pdev->dev, "Failed to get irqchip\n"); + return -EINVAL; + } + + data->reg_clr = devm_platform_ioremap_resource_byname(pdev, "clr"); + if (IS_ERR(data->reg_clr)) { + dev_err(dev, "Failed to map clear register\n"); + return PTR_ERR(data->reg_clr); + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "doorbell"); + if (!res) { + dev_err(dev, "Failed get resource from set\n"); + return -EINVAL; + } + data->doorbell_addr = res->start; + + ret = fwnode_property_get_reference_args(dev_fwnode(dev), "msi-ranges", + "#interrupt-cells", 0, 0, &args); + if (ret) { + dev_err(dev, "Unable to parse MSI vec base\n"); + return ret; + } + fwnode_handle_put(args.fwnode); + + ret = fwnode_property_get_reference_args(dev_fwnode(dev), "msi-ranges", NULL, + args.nargs + 1, 0, &args); + if (ret) { + dev_err(dev, "Unable to parse MSI vec number\n"); + return ret; + } + + plic_domain = irq_find_matching_fwnode(args.fwnode, DOMAIN_BUS_ANY); + fwnode_handle_put(args.fwnode); + if (!plic_domain) { + pr_err("Failed to find the PLIC domain\n"); + return -ENXIO; + } + + data->irq_first = (u32)args.args[0]; + data->irq_type = (unsigned int)args.args[1]; + data->num_irqs = (u32)args.args[args.nargs - 1]; + + mutex_init(&data->msi_map_lock); + + data->msi_map = devm_bitmap_zalloc(&pdev->dev, data->num_irqs, GFP_KERNEL); + if (!data->msi_map) { + dev_err(&pdev->dev, "Unable to allocate msi mapping\n"); + return -ENOMEM; + } + + return sg204x_msi_init_domains(data, plic_domain, dev); +} + +static const struct sg204x_msi_chip_info sg2042_chip_info = { + .irqchip = &sg2042_msi_middle_irq_chip, + .parent_ops = &sg2042_msi_parent_ops, +}; + +static const struct sg204x_msi_chip_info sg2044_chip_info = { + .irqchip = &sg2044_msi_middle_irq_chip, + .parent_ops = &sg2044_msi_parent_ops, +}; + +static const struct of_device_id sg2042_msi_of_match[] = { + { .compatible = "sophgo,sg2042-msi", .data = &sg2042_chip_info }, + { .compatible = "sophgo,sg2044-msi", .data = &sg2044_chip_info }, + { } +}; + +static struct platform_driver sg2042_msi_driver = { + .driver = { + .name = "sg2042-msi", + .of_match_table = sg2042_msi_of_match, + }, + .probe = sg2042_msi_probe, +}; +builtin_platform_driver(sg2042_msi_driver); diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index e1484905b7bd..210a57959637 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -3,7 +3,8 @@ * Copyright (C) 2017 SiFive * Copyright (C) 2018 Christoph Hellwig */ -#define pr_fmt(fmt) "plic: " fmt +#define pr_fmt(fmt) "riscv-plic: " fmt +#include <linux/acpi.h> #include <linux/cpu.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -48,6 +49,8 @@ #define CONTEXT_ENABLE_BASE 0x2000 #define CONTEXT_ENABLE_SIZE 0x80 +#define PENDING_BASE 0x1000 + /* * Each hart context has a set of control registers associated with it. Right * now there's only two: a source priority threshold over which the hart will @@ -62,14 +65,18 @@ #define PLIC_ENABLE_THRESHOLD 0 #define PLIC_QUIRK_EDGE_INTERRUPT 0 +#define PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM 1 struct plic_priv { + struct fwnode_handle *fwnode; struct cpumask lmask; struct irq_domain *irqdomain; void __iomem *regs; unsigned long plic_quirks; unsigned int nr_irqs; unsigned long *prio_save; + u32 gsi_base; + int acpi_plic_id; }; struct plic_handler { @@ -85,27 +92,36 @@ struct plic_handler { struct plic_priv *priv; }; static int plic_parent_irq __ro_after_init; -static bool plic_cpuhp_setup_done __ro_after_init; +static bool plic_global_setup_done __ro_after_init; static DEFINE_PER_CPU(struct plic_handler, plic_handlers); static int plic_irq_set_type(struct irq_data *d, unsigned int type); -static void __plic_toggle(void __iomem *enable_base, int hwirq, int enable) +static void __plic_toggle(struct plic_handler *handler, int hwirq, int enable) { - u32 __iomem *reg = enable_base + (hwirq / 32) * sizeof(u32); + u32 __iomem *base = handler->enable_base; u32 hwirq_mask = 1 << (hwirq % 32); + int group = hwirq / 32; + u32 value; + + value = readl(base + group); if (enable) - writel(readl(reg) | hwirq_mask, reg); + value |= hwirq_mask; else - writel(readl(reg) & ~hwirq_mask, reg); + value &= ~hwirq_mask; + + handler->enable_save[group] = value; + writel(value, base + group); } static void plic_toggle(struct plic_handler *handler, int hwirq, int enable) { - raw_spin_lock(&handler->enable_lock); - __plic_toggle(handler->enable_base, hwirq, enable); - raw_spin_unlock(&handler->enable_lock); + unsigned long flags; + + raw_spin_lock_irqsave(&handler->enable_lock, flags); + __plic_toggle(handler, hwirq, enable); + raw_spin_unlock_irqrestore(&handler->enable_lock, flags); } static inline void plic_irq_toggle(const struct cpumask *mask, @@ -120,16 +136,6 @@ static inline void plic_irq_toggle(const struct cpumask *mask, } } -static void plic_irq_enable(struct irq_data *d) -{ - plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1); -} - -static void plic_irq_disable(struct irq_data *d) -{ - plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0); -} - static void plic_irq_unmask(struct irq_data *d) { struct plic_priv *priv = irq_data_get_irq_chip_data(d); @@ -144,11 +150,28 @@ static void plic_irq_mask(struct irq_data *d) writel(0, priv->regs + PRIORITY_BASE + d->hwirq * PRIORITY_PER_ID); } +static void plic_irq_enable(struct irq_data *d) +{ + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1); + plic_irq_unmask(d); +} + +static void plic_irq_disable(struct irq_data *d) +{ + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0); +} + static void plic_irq_eoi(struct irq_data *d) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); - writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); + if (unlikely(irqd_irq_disabled(d))) { + plic_toggle(handler, d->hwirq, 1); + writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); + plic_toggle(handler, d->hwirq, 0); + } else { + writel(d->hwirq, handler->hart_base + CONTEXT_CLAIM); + } } #ifdef CONFIG_SMP @@ -156,25 +179,24 @@ static int plic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, bool force) { unsigned int cpu; - struct cpumask amask; struct plic_priv *priv = irq_data_get_irq_chip_data(d); - cpumask_and(&amask, &priv->lmask, mask_val); - if (force) - cpu = cpumask_first(&amask); + cpu = cpumask_first_and(&priv->lmask, mask_val); else - cpu = cpumask_any_and(&amask, cpu_online_mask); + cpu = cpumask_first_and_and(&priv->lmask, mask_val, cpu_online_mask); if (cpu >= nr_cpu_ids) return -EINVAL; - plic_irq_disable(d); + /* Invalidate the original routing entry */ + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 0); irq_data_update_effective_affinity(d, cpumask_of(cpu)); + /* Setting the new routing entry if irq is enabled */ if (!irqd_irq_disabled(d)) - plic_irq_enable(d); + plic_irq_toggle(irq_data_get_effective_affinity_mask(d), d, 1); return IRQ_SET_MASK_OK_DONE; } @@ -233,71 +255,61 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type) return IRQ_SET_MASK_OK; } -static int plic_irq_suspend(void) +static int plic_irq_suspend(void *data) { - unsigned int i, cpu; - u32 __iomem *reg; struct plic_priv *priv; priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - for (i = 0; i < priv->nr_irqs; i++) - if (readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)) - __set_bit(i, priv->prio_save); - else - __clear_bit(i, priv->prio_save); - - for_each_cpu(cpu, cpu_present_mask) { - struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu); - - if (!handler->present) - continue; - - raw_spin_lock(&handler->enable_lock); - for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) { - reg = handler->enable_base + i * sizeof(u32); - handler->enable_save[i] = readl(reg); - } - raw_spin_unlock(&handler->enable_lock); + /* irq ID 0 is reserved */ + for (unsigned int i = 1; i < priv->nr_irqs; i++) { + __assign_bit(i, priv->prio_save, + readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)); } return 0; } -static void plic_irq_resume(void) +static void plic_irq_resume(void *data) { unsigned int i, index, cpu; + unsigned long flags; u32 __iomem *reg; struct plic_priv *priv; priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; - for (i = 0; i < priv->nr_irqs; i++) { + /* irq ID 0 is reserved */ + for (i = 1; i < priv->nr_irqs; i++) { index = BIT_WORD(i); writel((priv->prio_save[index] & BIT_MASK(i)) ? 1 : 0, priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID); } - for_each_cpu(cpu, cpu_present_mask) { + for_each_present_cpu(cpu) { struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu); if (!handler->present) continue; - raw_spin_lock(&handler->enable_lock); + raw_spin_lock_irqsave(&handler->enable_lock, flags); for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) { reg = handler->enable_base + i * sizeof(u32); writel(handler->enable_save[i], reg); } - raw_spin_unlock(&handler->enable_lock); + raw_spin_unlock_irqrestore(&handler->enable_lock, flags); } } -static struct syscore_ops plic_irq_syscore_ops = { +static const struct syscore_ops plic_irq_syscore_ops = { .suspend = plic_irq_suspend, .resume = plic_irq_resume, }; +static struct syscore plic_irq_syscore = { + .ops = &plic_irq_syscore_ops, +}; + static int plic_irqdomain_map(struct irq_domain *d, unsigned int irq, irq_hw_number_t hwirq) { @@ -317,6 +329,10 @@ static int plic_irq_domain_translate(struct irq_domain *d, { struct plic_priv *priv = d->host_data; + /* For DT, gsi_base is always zero. */ + if (fwspec->param[0] >= priv->gsi_base) + fwspec->param[0] = fwspec->param[0] - priv->gsi_base; + if (test_bit(PLIC_QUIRK_EDGE_INTERRUPT, &priv->plic_quirks)) return irq_domain_translate_twocell(d, fwspec, hwirq, type); @@ -370,9 +386,102 @@ static void plic_handle_irq(struct irq_desc *desc) while ((hwirq = readl(claim))) { int err = generic_handle_domain_irq(handler->priv->irqdomain, hwirq); - if (unlikely(err)) - pr_warn_ratelimited("can't find mapping for hwirq %lu\n", - hwirq); + if (unlikely(err)) { + pr_warn_ratelimited("%pfwP: can't find mapping for hwirq %lu\n", + handler->priv->fwnode, hwirq); + } + } + + chained_irq_exit(chip, desc); +} + +static u32 cp100_isolate_pending_irq(int nr_irq_groups, struct plic_handler *handler) +{ + u32 __iomem *pending = handler->priv->regs + PENDING_BASE; + u32 __iomem *enable = handler->enable_base; + u32 pending_irqs = 0; + int i, j; + + /* Look for first pending interrupt */ + for (i = 0; i < nr_irq_groups; i++) { + /* Any pending interrupts would be annihilated, so skip checking them */ + if (!handler->enable_save[i]) + continue; + + pending_irqs = handler->enable_save[i] & readl_relaxed(pending + i); + if (pending_irqs) + break; + } + + if (!pending_irqs) + return 0; + + /* Isolate lowest set bit */ + pending_irqs &= -pending_irqs; + + /* Disable all interrupts but the first pending one */ + for (j = 0; j < nr_irq_groups; j++) { + u32 new_mask = j == i ? pending_irqs : 0; + + if (new_mask != handler->enable_save[j]) + writel_relaxed(new_mask, enable + j); + } + return pending_irqs; +} + +static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iomem *claim) +{ + int nr_irq_groups = DIV_ROUND_UP(handler->priv->nr_irqs, 32); + u32 __iomem *enable = handler->enable_base; + irq_hw_number_t hwirq = 0; + u32 iso_mask; + int i; + + guard(raw_spinlock)(&handler->enable_lock); + + /* Existing enable state is already cached in enable_save */ + iso_mask = cp100_isolate_pending_irq(nr_irq_groups, handler); + if (!iso_mask) + return 0; + + /* + * Interrupts delievered to hardware still become pending, but only + * interrupts that are both pending and enabled can be claimed. + * Clearing the enable bit for all interrupts but the first pending + * one avoids a hardware bug that occurs during read from the claim + * register with more than one eligible interrupt. + */ + hwirq = readl(claim); + + /* Restore previous state */ + for (i = 0; i < nr_irq_groups; i++) { + u32 written = i == hwirq / 32 ? iso_mask : 0; + u32 stored = handler->enable_save[i]; + + if (stored != written) + writel_relaxed(stored, enable + i); + } + return hwirq; +} + +static void plic_handle_irq_cp100(struct irq_desc *desc) +{ + struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + struct irq_chip *chip = irq_desc_get_chip(desc); + void __iomem *claim = handler->hart_base + CONTEXT_CLAIM; + irq_hw_number_t hwirq; + + WARN_ON_ONCE(!handler->present); + + chained_irq_enter(chip, desc); + + while ((hwirq = cp100_get_hwirq(handler, claim))) { + int err = generic_handle_domain_irq(handler->priv->irqdomain, hwirq); + + if (unlikely(err)) { + pr_warn_ratelimited("%pfwP: can't find mapping for hwirq %lu\n", + handler->priv->fwnode, hwirq); + } } chained_irq_exit(chip, desc); @@ -400,103 +509,194 @@ static int plic_starting_cpu(unsigned int cpu) enable_percpu_irq(plic_parent_irq, irq_get_trigger_type(plic_parent_irq)); else - pr_warn("cpu%d: parent irq not available\n", cpu); + pr_warn("%pfwP: cpu%d: parent irq not available\n", + handler->priv->fwnode, cpu); plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD); return 0; } -static int __init __plic_init(struct device_node *node, - struct device_node *parent, - unsigned long plic_quirks) +static const struct of_device_id plic_match[] = { + { .compatible = "sifive,plic-1.0.0" }, + { .compatible = "riscv,plic0" }, + { .compatible = "andestech,nceplic100", + .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) }, + { .compatible = "thead,c900-plic", + .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) }, + { .compatible = "ultrarisc,cp100-plic", + .data = (const void *)BIT(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM) }, + {} +}; + +#ifdef CONFIG_ACPI + +static const struct acpi_device_id plic_acpi_match[] = { + { "RSCV0001", 0 }, + {} +}; +MODULE_DEVICE_TABLE(acpi, plic_acpi_match); + +#endif +static int plic_parse_nr_irqs_and_contexts(struct fwnode_handle *fwnode, + u32 *nr_irqs, u32 *nr_contexts, + u32 *gsi_base, u32 *id) { - int error = 0, nr_contexts, nr_handlers = 0, i; - u32 nr_irqs; - struct plic_priv *priv; - struct plic_handler *handler; - unsigned int cpu; + int rc; - priv = kzalloc(sizeof(*priv), GFP_KERNEL); - if (!priv) - return -ENOMEM; + if (!is_of_node(fwnode)) { + rc = riscv_acpi_get_gsi_info(fwnode, gsi_base, id, nr_irqs, NULL); + if (rc) { + pr_err("%pfwP: failed to find GSI mapping\n", fwnode); + return rc; + } - priv->plic_quirks = plic_quirks; + *nr_contexts = acpi_rintc_get_plic_nr_contexts(*id); + if (WARN_ON(!*nr_contexts)) { + pr_err("%pfwP: no PLIC context available\n", fwnode); + return -EINVAL; + } - priv->regs = of_iomap(node, 0); - if (WARN_ON(!priv->regs)) { - error = -EIO; - goto out_free_priv; + return 0; } - error = -EINVAL; - of_property_read_u32(node, "riscv,ndev", &nr_irqs); - if (WARN_ON(!nr_irqs)) - goto out_iounmap; + rc = of_property_read_u32(to_of_node(fwnode), "riscv,ndev", nr_irqs); + if (rc) { + pr_err("%pfwP: riscv,ndev property not available\n", fwnode); + return rc; + } - priv->nr_irqs = nr_irqs; + *nr_contexts = of_irq_count(to_of_node(fwnode)); + if (WARN_ON(!(*nr_contexts))) { + pr_err("%pfwP: no PLIC context available\n", fwnode); + return -EINVAL; + } - priv->prio_save = bitmap_alloc(nr_irqs, GFP_KERNEL); - if (!priv->prio_save) - goto out_free_priority_reg; + *gsi_base = 0; + *id = 0; - nr_contexts = of_irq_count(node); - if (WARN_ON(!nr_contexts)) - goto out_free_priority_reg; + return 0; +} - error = -ENOMEM; - priv->irqdomain = irq_domain_add_linear(node, nr_irqs + 1, - &plic_irqdomain_ops, priv); - if (WARN_ON(!priv->irqdomain)) - goto out_free_priority_reg; +static int plic_parse_context_parent(struct fwnode_handle *fwnode, u32 context, + u32 *parent_hwirq, int *parent_cpu, u32 id) +{ + struct of_phandle_args parent; + unsigned long hartid; + int rc; + + if (!is_of_node(fwnode)) { + hartid = acpi_rintc_ext_parent_to_hartid(id, context); + if (hartid == INVALID_HARTID) + return -EINVAL; + + *parent_cpu = riscv_hartid_to_cpuid(hartid); + *parent_hwirq = RV_IRQ_EXT; + return 0; + } - for (i = 0; i < nr_contexts; i++) { - struct of_phandle_args parent; - irq_hw_number_t hwirq; - int cpu; - unsigned long hartid; + rc = of_irq_parse_one(to_of_node(fwnode), context, &parent); + if (rc) + return rc; + + rc = riscv_of_parent_hartid(parent.np, &hartid); + if (rc) + return rc; + + *parent_hwirq = parent.args[0]; + *parent_cpu = riscv_hartid_to_cpuid(hartid); + return 0; +} + +static int plic_probe(struct fwnode_handle *fwnode) +{ + int error = 0, nr_contexts, nr_handlers = 0, cpu, i; + unsigned long plic_quirks = 0; + struct plic_handler *handler; + u32 nr_irqs, parent_hwirq; + struct plic_priv *priv; + irq_hw_number_t hwirq; + void __iomem *regs; + int id, context_id; + u32 gsi_base; + + if (is_of_node(fwnode)) { + const struct of_device_id *id; + + id = of_match_node(plic_match, to_of_node(fwnode)); + if (id) + plic_quirks = (unsigned long)id->data; + + regs = of_iomap(to_of_node(fwnode), 0); + if (!regs) + return -ENOMEM; + } else { + regs = devm_platform_ioremap_resource(to_platform_device(fwnode->dev), 0); + if (IS_ERR(regs)) + return PTR_ERR(regs); + } + + error = plic_parse_nr_irqs_and_contexts(fwnode, &nr_irqs, &nr_contexts, &gsi_base, &id); + if (error) + goto fail_free_regs; + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) { + error = -ENOMEM; + goto fail_free_regs; + } - if (of_irq_parse_one(node, i, &parent)) { - pr_err("failed to parse parent for context %d.\n", i); + priv->fwnode = fwnode; + priv->plic_quirks = plic_quirks; + priv->nr_irqs = nr_irqs; + priv->regs = regs; + priv->gsi_base = gsi_base; + priv->acpi_plic_id = id; + + priv->prio_save = bitmap_zalloc(nr_irqs, GFP_KERNEL); + if (!priv->prio_save) { + error = -ENOMEM; + goto fail_free_priv; + } + + for (i = 0; i < nr_contexts; i++) { + error = plic_parse_context_parent(fwnode, i, &parent_hwirq, &cpu, + priv->acpi_plic_id); + if (error) { + pr_warn("%pfwP: hwirq for context%d not found\n", fwnode, i); continue; } + if (is_of_node(fwnode)) { + context_id = i; + } else { + context_id = acpi_rintc_get_plic_context(priv->acpi_plic_id, i); + if (context_id == INVALID_CONTEXT) { + pr_warn("%pfwP: invalid context id for context%d\n", fwnode, i); + continue; + } + } + /* * Skip contexts other than external interrupts for our * privilege level. */ - if (parent.args[0] != RV_IRQ_EXT) { + if (parent_hwirq != RV_IRQ_EXT) { /* Disable S-mode enable bits if running in M-mode. */ if (IS_ENABLED(CONFIG_RISCV_M_MODE)) { - void __iomem *enable_base = priv->regs + - CONTEXT_ENABLE_BASE + - i * CONTEXT_ENABLE_SIZE; + u32 __iomem *enable_base = priv->regs + CONTEXT_ENABLE_BASE + + i * CONTEXT_ENABLE_SIZE; - for (hwirq = 1; hwirq <= nr_irqs; hwirq++) - __plic_toggle(enable_base, hwirq, 0); + for (int j = 0; j <= nr_irqs / 32; j++) + writel(0, enable_base + j); } continue; } - error = riscv_of_parent_hartid(parent.np, &hartid); - if (error < 0) { - pr_warn("failed to parse hart ID for context %d.\n", i); - continue; - } - - cpu = riscv_hartid_to_cpuid(hartid); if (cpu < 0) { - pr_warn("Invalid cpuid for context %d\n", i); + pr_warn("%pfwP: Invalid cpuid for context %d\n", fwnode, i); continue; } - /* Find parent domain and register chained handler */ - if (!plic_parent_irq && irq_find_host(parent.np)) { - plic_parent_irq = irq_of_parse_and_map(node, i); - if (plic_parent_irq) - irq_set_chained_handler(plic_parent_irq, - plic_handle_irq); - } - /* * When running in M-mode we need to ignore the S-mode handler. * Here we assume it always comes later, but that might be a @@ -504,7 +704,7 @@ static int __init __plic_init(struct device_node *node, */ handler = per_cpu_ptr(&plic_handlers, cpu); if (handler->present) { - pr_warn("handler already present for context %d.\n", i); + pr_warn("%pfwP: handler already present for context %d.\n", fwnode, i); plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD); goto done; } @@ -512,16 +712,18 @@ static int __init __plic_init(struct device_node *node, cpumask_set_cpu(cpu, &priv->lmask); handler->present = true; handler->hart_base = priv->regs + CONTEXT_BASE + - i * CONTEXT_SIZE; + context_id * CONTEXT_SIZE; raw_spin_lock_init(&handler->enable_lock); handler->enable_base = priv->regs + CONTEXT_ENABLE_BASE + - i * CONTEXT_ENABLE_SIZE; + context_id * CONTEXT_ENABLE_SIZE; handler->priv = priv; - handler->enable_save = kcalloc(DIV_ROUND_UP(nr_irqs, 32), - sizeof(*handler->enable_save), GFP_KERNEL); - if (!handler->enable_save) - goto out_free_enable_reg; + handler->enable_save = kcalloc(DIV_ROUND_UP(nr_irqs, 32), + sizeof(*handler->enable_save), GFP_KERNEL); + if (!handler->enable_save) { + error = -ENOMEM; + goto fail_cleanup_contexts; + } done: for (hwirq = 1; hwirq <= nr_irqs; hwirq++) { plic_toggle(handler, hwirq, 0); @@ -531,51 +733,103 @@ done: nr_handlers++; } + priv->irqdomain = irq_domain_create_linear(fwnode, nr_irqs + 1, + &plic_irqdomain_ops, priv); + if (WARN_ON(!priv->irqdomain)) { + error = -ENOMEM; + goto fail_cleanup_contexts; + } + /* - * We can have multiple PLIC instances so setup cpuhp state only - * when context handler for current/boot CPU is present. + * We can have multiple PLIC instances so setup global state + * and register syscore operations only once after context + * handlers of all online CPUs are initialized. */ - handler = this_cpu_ptr(&plic_handlers); - if (handler->present && !plic_cpuhp_setup_done) { - cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, - "irqchip/sifive/plic:starting", - plic_starting_cpu, plic_dying_cpu); - plic_cpuhp_setup_done = true; + if (!plic_global_setup_done) { + struct irq_domain *domain; + bool global_setup = true; + + for_each_online_cpu(cpu) { + handler = per_cpu_ptr(&plic_handlers, cpu); + if (!handler->present) { + global_setup = false; + break; + } + } + + if (global_setup) { + void (*handler_fn)(struct irq_desc *) = plic_handle_irq; + + if (test_bit(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM, &handler->priv->plic_quirks)) + handler_fn = plic_handle_irq_cp100; + + /* Find parent domain and register chained handler */ + domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); + if (domain) + plic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); + if (plic_parent_irq) + irq_set_chained_handler(plic_parent_irq, handler_fn); + + cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, + "irqchip/sifive/plic:starting", + plic_starting_cpu, plic_dying_cpu); + register_syscore(&plic_irq_syscore); + plic_global_setup_done = true; + } } - register_syscore_ops(&plic_irq_syscore_ops); - pr_info("%pOFP: mapped %d interrupts with %d handlers for" - " %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts); +#ifdef CONFIG_ACPI + if (!acpi_disabled) + acpi_dev_clear_dependencies(ACPI_COMPANION(fwnode->dev)); +#endif + + pr_info("%pfwP: mapped %d interrupts with %d handlers for %d contexts.\n", + fwnode, nr_irqs, nr_handlers, nr_contexts); return 0; -out_free_enable_reg: - for_each_cpu(cpu, cpu_present_mask) { +fail_cleanup_contexts: + for (i = 0; i < nr_contexts; i++) { + if (plic_parse_context_parent(fwnode, i, &parent_hwirq, &cpu, priv->acpi_plic_id)) + continue; + if (parent_hwirq != RV_IRQ_EXT || cpu < 0) + continue; + handler = per_cpu_ptr(&plic_handlers, cpu); + handler->present = false; + handler->hart_base = NULL; + handler->enable_base = NULL; kfree(handler->enable_save); + handler->enable_save = NULL; + handler->priv = NULL; } -out_free_priority_reg: - kfree(priv->prio_save); -out_iounmap: - iounmap(priv->regs); -out_free_priv: + bitmap_free(priv->prio_save); +fail_free_priv: kfree(priv); +fail_free_regs: + iounmap(regs); return error; } -static int __init plic_init(struct device_node *node, - struct device_node *parent) +static int plic_platform_probe(struct platform_device *pdev) { - return __plic_init(node, parent, 0); + return plic_probe(pdev->dev.fwnode); } -IRQCHIP_DECLARE(sifive_plic, "sifive,plic-1.0.0", plic_init); -IRQCHIP_DECLARE(riscv_plic0, "riscv,plic0", plic_init); /* for legacy systems */ +static struct platform_driver plic_driver = { + .driver = { + .name = "riscv-plic", + .of_match_table = plic_match, + .suppress_bind_attrs = true, + .acpi_match_table = ACPI_PTR(plic_acpi_match), + }, + .probe = plic_platform_probe, +}; +builtin_platform_driver(plic_driver); -static int __init plic_edge_init(struct device_node *node, - struct device_node *parent) +static int __init plic_early_probe(struct device_node *node, + struct device_node *parent) { - return __plic_init(node, parent, BIT(PLIC_QUIRK_EDGE_INTERRUPT)); + return plic_probe(&node->fwnode); } -IRQCHIP_DECLARE(andestech_nceplic100, "andestech,nceplic100", plic_edge_init); -IRQCHIP_DECLARE(thead_c900_plic, "thead,c900-plic", plic_edge_init); +IRQCHIP_DECLARE(riscv, "allwinner,sun20i-d1-plic", plic_early_probe); diff --git a/drivers/irqchip/irq-sni-exiu.c b/drivers/irqchip/irq-sni-exiu.c index c7db617e1a2f..0cad68aa8388 100644 --- a/drivers/irqchip/irq-sni-exiu.c +++ b/drivers/irqchip/irq-sni-exiu.c @@ -249,12 +249,12 @@ static int __init exiu_dt_init(struct device_node *node, return -ENXIO; } - data = exiu_init(of_node_to_fwnode(node), &res); + data = exiu_init(of_fwnode_handle(node), &res); if (IS_ERR(data)) return PTR_ERR(data); - domain = irq_domain_add_hierarchy(parent_domain, 0, NUM_IRQS, node, - &exiu_domain_ops, data); + domain = irq_domain_create_hierarchy(parent_domain, 0, NUM_IRQS, of_fwnode_handle(node), + &exiu_domain_ops, data); if (!domain) { pr_err("%pOF: failed to allocate domain\n", node); goto out_unmap; diff --git a/drivers/irqchip/irq-sp7021-intc.c b/drivers/irqchip/irq-sp7021-intc.c index bed78d1def3d..2a6eda9ab62e 100644 --- a/drivers/irqchip/irq-sp7021-intc.c +++ b/drivers/irqchip/irq-sp7021-intc.c @@ -256,8 +256,8 @@ static int __init sp_intc_init_dt(struct device_node *node, struct device_node * writel_relaxed(~0, REG_INTR_CLEAR + i * 4); } - sp_intc.domain = irq_domain_add_linear(node, SP_INTC_NR_IRQS, - &sp_intc_dm_ops, &sp_intc); + sp_intc.domain = irq_domain_create_linear(of_fwnode_handle(node), SP_INTC_NR_IRQS, + &sp_intc_dm_ops, &sp_intc); if (!sp_intc.domain) { ret = -ENOMEM; goto out_unmap1; diff --git a/drivers/irqchip/irq-starfive-jh8100-intc.c b/drivers/irqchip/irq-starfive-jh8100-intc.c new file mode 100644 index 000000000000..705361b4ebe0 --- /dev/null +++ b/drivers/irqchip/irq-starfive-jh8100-intc.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * StarFive JH8100 External Interrupt Controller driver + * + * Copyright (C) 2023 StarFive Technology Co., Ltd. + * + * Author: Changhuang Liang <changhuang.liang@starfivetech.com> + */ + +#define pr_fmt(fmt) "irq-starfive-jh8100: " fmt + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> +#include <linux/irqdomain.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/reset.h> +#include <linux/spinlock.h> + +#define STARFIVE_INTC_SRC0_CLEAR 0x10 +#define STARFIVE_INTC_SRC0_MASK 0x14 +#define STARFIVE_INTC_SRC0_INT 0x1c + +#define STARFIVE_INTC_SRC_IRQ_NUM 32 + +struct starfive_irq_chip { + void __iomem *base; + struct irq_domain *domain; + raw_spinlock_t lock; +}; + +static void starfive_intc_bit_set(struct starfive_irq_chip *irqc, + u32 reg, u32 bit_mask) +{ + u32 value; + + value = ioread32(irqc->base + reg); + value |= bit_mask; + iowrite32(value, irqc->base + reg); +} + +static void starfive_intc_bit_clear(struct starfive_irq_chip *irqc, + u32 reg, u32 bit_mask) +{ + u32 value; + + value = ioread32(irqc->base + reg); + value &= ~bit_mask; + iowrite32(value, irqc->base + reg); +} + +static void starfive_intc_unmask(struct irq_data *d) +{ + struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d); + + raw_spin_lock(&irqc->lock); + starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq)); + raw_spin_unlock(&irqc->lock); +} + +static void starfive_intc_mask(struct irq_data *d) +{ + struct starfive_irq_chip *irqc = irq_data_get_irq_chip_data(d); + + raw_spin_lock(&irqc->lock); + starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_MASK, BIT(d->hwirq)); + raw_spin_unlock(&irqc->lock); +} + +static struct irq_chip intc_dev = { + .name = "StarFive JH8100 INTC", + .irq_unmask = starfive_intc_unmask, + .irq_mask = starfive_intc_mask, +}; + +static int starfive_intc_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_domain_set_info(d, irq, hwirq, &intc_dev, d->host_data, + handle_level_irq, NULL, NULL); + + return 0; +} + +static const struct irq_domain_ops starfive_intc_domain_ops = { + .xlate = irq_domain_xlate_onecell, + .map = starfive_intc_map, +}; + +static void starfive_intc_irq_handler(struct irq_desc *desc) +{ + struct starfive_irq_chip *irqc = irq_data_get_irq_handler_data(&desc->irq_data); + struct irq_chip *chip = irq_desc_get_chip(desc); + unsigned long value; + int hwirq; + + chained_irq_enter(chip, desc); + + value = ioread32(irqc->base + STARFIVE_INTC_SRC0_INT); + while (value) { + hwirq = ffs(value) - 1; + + generic_handle_domain_irq(irqc->domain, hwirq); + + starfive_intc_bit_set(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq)); + starfive_intc_bit_clear(irqc, STARFIVE_INTC_SRC0_CLEAR, BIT(hwirq)); + + __clear_bit(hwirq, &value); + } + + chained_irq_exit(chip, desc); +} + +static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent) +{ + struct device_node *intc = pdev->dev.of_node; + struct starfive_irq_chip *irqc; + struct reset_control *rst; + struct clk *clk; + int parent_irq; + int ret; + + irqc = kzalloc(sizeof(*irqc), GFP_KERNEL); + if (!irqc) + return -ENOMEM; + + irqc->base = of_iomap(intc, 0); + if (!irqc->base) { + pr_err("Unable to map registers\n"); + ret = -ENXIO; + goto err_free; + } + + rst = of_reset_control_get_exclusive(intc, NULL); + if (IS_ERR(rst)) { + pr_err("Unable to get reset control %pe\n", rst); + ret = PTR_ERR(rst); + goto err_unmap; + } + + clk = of_clk_get(intc, 0); + if (IS_ERR(clk)) { + pr_err("Unable to get clock %pe\n", clk); + ret = PTR_ERR(clk); + goto err_reset_put; + } + + ret = reset_control_deassert(rst); + if (ret) + goto err_clk_put; + + ret = clk_prepare_enable(clk); + if (ret) + goto err_reset_assert; + + raw_spin_lock_init(&irqc->lock); + + irqc->domain = irq_domain_create_linear(of_fwnode_handle(intc), STARFIVE_INTC_SRC_IRQ_NUM, + &starfive_intc_domain_ops, irqc); + if (!irqc->domain) { + pr_err("Unable to create IRQ domain\n"); + ret = -EINVAL; + goto err_clk_disable; + } + + parent_irq = of_irq_get(intc, 0); + if (parent_irq < 0) { + pr_err("Failed to get main IRQ: %d\n", parent_irq); + ret = parent_irq; + goto err_remove_domain; + } + + irq_set_chained_handler_and_data(parent_irq, starfive_intc_irq_handler, + irqc); + + pr_info("Interrupt controller register, nr_irqs %d\n", + STARFIVE_INTC_SRC_IRQ_NUM); + + return 0; + +err_remove_domain: + irq_domain_remove(irqc->domain); +err_clk_disable: + clk_disable_unprepare(clk); +err_reset_assert: + reset_control_assert(rst); +err_clk_put: + clk_put(clk); +err_reset_put: + reset_control_put(rst); +err_unmap: + iounmap(irqc->base); +err_free: + kfree(irqc); + return ret; +} + +IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc) +IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_probe) +IRQCHIP_PLATFORM_DRIVER_END(starfive_intc) + +MODULE_DESCRIPTION("StarFive JH8100 External Interrupt Controller"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Changhuang Liang <changhuang.liang@starfivetech.com>"); diff --git a/drivers/irqchip/irq-stm32-exti.c b/drivers/irqchip/irq-stm32-exti.c index d8ba5fba7450..978811f2abe8 100644 --- a/drivers/irqchip/irq-stm32-exti.c +++ b/drivers/irqchip/irq-stm32-exti.c @@ -1,31 +1,21 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (C) Maxime Coquelin 2015 - * Copyright (C) STMicroelectronics 2017 + * Copyright (C) STMicroelectronics 2017-2024 * Author: Maxime Coquelin <mcoquelin.stm32@gmail.com> */ #include <linux/bitops.h> -#include <linux/delay.h> -#include <linux/hwspinlock.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> #include <linux/irqchip.h> #include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> -#include <linux/mod_devicetable.h> -#include <linux/module.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <linux/platform_device.h> -#include <linux/syscore_ops.h> -#include <dt-bindings/interrupt-controller/arm-gic.h> - -#define IRQS_PER_BANK 32 - -#define HWSPNLCK_TIMEOUT 1000 /* usec */ +#define IRQS_PER_BANK 32 struct stm32_exti_bank { u32 imr_ofst; @@ -34,12 +24,8 @@ struct stm32_exti_bank { u32 ftsr_ofst; u32 swier_ofst; u32 rpr_ofst; - u32 fpr_ofst; - u32 trg_ofst; }; -#define UNDEF_REG ~0 - struct stm32_exti_drv_data { const struct stm32_exti_bank **exti_banks; const u8 *desc_irqs; @@ -49,22 +35,20 @@ struct stm32_exti_drv_data { struct stm32_exti_chip_data { struct stm32_exti_host_data *host_data; const struct stm32_exti_bank *reg_bank; - struct raw_spinlock rlock; u32 wake_active; u32 mask_cache; u32 rtsr_cache; u32 ftsr_cache; + u32 event_reserved; }; struct stm32_exti_host_data { void __iomem *base; + struct device *dev; struct stm32_exti_chip_data *chips_data; const struct stm32_exti_drv_data *drv_data; - struct hwspinlock *hwlock; }; -static struct stm32_exti_host_data *stm32_host_data; - static const struct stm32_exti_bank stm32f4xx_exti_b1 = { .imr_ofst = 0x00, .emr_ofst = 0x04, @@ -72,8 +56,6 @@ static const struct stm32_exti_bank stm32f4xx_exti_b1 = { .ftsr_ofst = 0x0C, .swier_ofst = 0x10, .rpr_ofst = 0x14, - .fpr_ofst = UNDEF_REG, - .trg_ofst = UNDEF_REG, }; static const struct stm32_exti_bank *stm32f4xx_exti_banks[] = { @@ -92,8 +74,6 @@ static const struct stm32_exti_bank stm32h7xx_exti_b1 = { .ftsr_ofst = 0x04, .swier_ofst = 0x08, .rpr_ofst = 0x88, - .fpr_ofst = UNDEF_REG, - .trg_ofst = UNDEF_REG, }; static const struct stm32_exti_bank stm32h7xx_exti_b2 = { @@ -103,8 +83,6 @@ static const struct stm32_exti_bank stm32h7xx_exti_b2 = { .ftsr_ofst = 0x24, .swier_ofst = 0x28, .rpr_ofst = 0x98, - .fpr_ofst = UNDEF_REG, - .trg_ofst = UNDEF_REG, }; static const struct stm32_exti_bank stm32h7xx_exti_b3 = { @@ -114,8 +92,6 @@ static const struct stm32_exti_bank stm32h7xx_exti_b3 = { .ftsr_ofst = 0x44, .swier_ofst = 0x48, .rpr_ofst = 0xA8, - .fpr_ofst = UNDEF_REG, - .trg_ofst = UNDEF_REG, }; static const struct stm32_exti_bank *stm32h7xx_exti_banks[] = { @@ -129,180 +105,12 @@ static const struct stm32_exti_drv_data stm32h7xx_drv_data = { .bank_nr = ARRAY_SIZE(stm32h7xx_exti_banks), }; -static const struct stm32_exti_bank stm32mp1_exti_b1 = { - .imr_ofst = 0x80, - .emr_ofst = UNDEF_REG, - .rtsr_ofst = 0x00, - .ftsr_ofst = 0x04, - .swier_ofst = 0x08, - .rpr_ofst = 0x0C, - .fpr_ofst = 0x10, - .trg_ofst = 0x3EC, -}; - -static const struct stm32_exti_bank stm32mp1_exti_b2 = { - .imr_ofst = 0x90, - .emr_ofst = UNDEF_REG, - .rtsr_ofst = 0x20, - .ftsr_ofst = 0x24, - .swier_ofst = 0x28, - .rpr_ofst = 0x2C, - .fpr_ofst = 0x30, - .trg_ofst = 0x3E8, -}; - -static const struct stm32_exti_bank stm32mp1_exti_b3 = { - .imr_ofst = 0xA0, - .emr_ofst = UNDEF_REG, - .rtsr_ofst = 0x40, - .ftsr_ofst = 0x44, - .swier_ofst = 0x48, - .rpr_ofst = 0x4C, - .fpr_ofst = 0x50, - .trg_ofst = 0x3E4, -}; - -static const struct stm32_exti_bank *stm32mp1_exti_banks[] = { - &stm32mp1_exti_b1, - &stm32mp1_exti_b2, - &stm32mp1_exti_b3, -}; - -static struct irq_chip stm32_exti_h_chip; -static struct irq_chip stm32_exti_h_chip_direct; - -#define EXTI_INVALID_IRQ U8_MAX -#define STM32MP1_DESC_IRQ_SIZE (ARRAY_SIZE(stm32mp1_exti_banks) * IRQS_PER_BANK) - -/* - * Use some intentionally tricky logic here to initialize the whole array to - * EXTI_INVALID_IRQ, but then override certain fields, requiring us to indicate - * that we "know" that there are overrides in this structure, and we'll need to - * disable that warning from W=1 builds. - */ -__diag_push(); -__diag_ignore_all("-Woverride-init", - "logic to initialize all and then override some is OK"); - -static const u8 stm32mp1_desc_irq[] = { - /* default value */ - [0 ... (STM32MP1_DESC_IRQ_SIZE - 1)] = EXTI_INVALID_IRQ, - - [0] = 6, - [1] = 7, - [2] = 8, - [3] = 9, - [4] = 10, - [5] = 23, - [6] = 64, - [7] = 65, - [8] = 66, - [9] = 67, - [10] = 40, - [11] = 42, - [12] = 76, - [13] = 77, - [14] = 121, - [15] = 127, - [16] = 1, - [19] = 3, - [21] = 31, - [22] = 33, - [23] = 72, - [24] = 95, - [25] = 107, - [26] = 37, - [27] = 38, - [28] = 39, - [29] = 71, - [30] = 52, - [31] = 53, - [32] = 82, - [33] = 83, - [46] = 151, - [47] = 93, - [48] = 138, - [50] = 139, - [52] = 140, - [53] = 141, - [54] = 135, - [61] = 100, - [65] = 144, - [68] = 143, - [70] = 62, - [73] = 129, -}; - -static const u8 stm32mp13_desc_irq[] = { - /* default value */ - [0 ... (STM32MP1_DESC_IRQ_SIZE - 1)] = EXTI_INVALID_IRQ, - - [0] = 6, - [1] = 7, - [2] = 8, - [3] = 9, - [4] = 10, - [5] = 24, - [6] = 65, - [7] = 66, - [8] = 67, - [9] = 68, - [10] = 41, - [11] = 43, - [12] = 77, - [13] = 78, - [14] = 106, - [15] = 109, - [16] = 1, - [19] = 3, - [21] = 32, - [22] = 34, - [23] = 73, - [24] = 93, - [25] = 114, - [26] = 38, - [27] = 39, - [28] = 40, - [29] = 72, - [30] = 53, - [31] = 54, - [32] = 83, - [33] = 84, - [44] = 96, - [47] = 92, - [48] = 116, - [50] = 117, - [52] = 118, - [53] = 119, - [68] = 63, - [70] = 98, -}; - -__diag_pop(); - -static const struct stm32_exti_drv_data stm32mp1_drv_data = { - .exti_banks = stm32mp1_exti_banks, - .bank_nr = ARRAY_SIZE(stm32mp1_exti_banks), - .desc_irqs = stm32mp1_desc_irq, -}; - -static const struct stm32_exti_drv_data stm32mp13_drv_data = { - .exti_banks = stm32mp1_exti_banks, - .bank_nr = ARRAY_SIZE(stm32mp1_exti_banks), - .desc_irqs = stm32mp13_desc_irq, -}; - static unsigned long stm32_exti_pending(struct irq_chip_generic *gc) { struct stm32_exti_chip_data *chip_data = gc->private; const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - unsigned long pending; - - pending = irq_reg_readl(gc, stm32_bank->rpr_ofst); - if (stm32_bank->fpr_ofst != UNDEF_REG) - pending |= irq_reg_readl(gc, stm32_bank->fpr_ofst); - return pending; + return irq_reg_readl(gc, stm32_bank->rpr_ofst); } static void stm32_irq_handler(struct irq_desc *desc) @@ -322,7 +130,7 @@ static void stm32_irq_handler(struct irq_desc *desc) while ((pending = stm32_exti_pending(gc))) { for_each_set_bit(n, &pending, IRQS_PER_BANK) generic_handle_domain_irq(domain, irq_base + n); - } + } } chained_irq_exit(chip, desc); @@ -358,37 +166,21 @@ static int stm32_irq_set_type(struct irq_data *d, unsigned int type) struct irq_chip_generic *gc = irq_data_get_irq_chip_data(d); struct stm32_exti_chip_data *chip_data = gc->private; const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - struct hwspinlock *hwlock = chip_data->host_data->hwlock; u32 rtsr, ftsr; int err; - irq_gc_lock(gc); - - if (hwlock) { - err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT); - if (err) { - pr_err("%s can't get hwspinlock (%d)\n", __func__, err); - goto unlock; - } - } + guard(raw_spinlock)(&gc->lock); rtsr = irq_reg_readl(gc, stm32_bank->rtsr_ofst); ftsr = irq_reg_readl(gc, stm32_bank->ftsr_ofst); err = stm32_exti_set_type(d, type, &rtsr, &ftsr); if (err) - goto unspinlock; + return err; irq_reg_writel(gc, rtsr, stm32_bank->rtsr_ofst); irq_reg_writel(gc, ftsr, stm32_bank->ftsr_ofst); - -unspinlock: - if (hwlock) - hwspin_unlock_in_atomic(hwlock); -unlock: - irq_gc_unlock(gc); - - return err; + return 0; } static void stm32_chip_suspend(struct stm32_exti_chip_data *chip_data, @@ -421,18 +213,16 @@ static void stm32_irq_suspend(struct irq_chip_generic *gc) { struct stm32_exti_chip_data *chip_data = gc->private; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); stm32_chip_suspend(chip_data, gc->wake_active); - irq_gc_unlock(gc); } static void stm32_irq_resume(struct irq_chip_generic *gc) { struct stm32_exti_chip_data *chip_data = gc->private; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); stm32_chip_resume(chip_data, gc->mask_cache); - irq_gc_unlock(gc); } static int stm32_exti_alloc(struct irq_domain *d, unsigned int virq, @@ -460,6 +250,7 @@ static const struct irq_domain_ops irq_exti_domain_ops = { .map = irq_map_generic_chip, .alloc = stm32_exti_alloc, .free = stm32_exti_free, + .xlate = irq_domain_xlate_twocell, }; static void stm32_irq_ack(struct irq_data *d) @@ -468,283 +259,8 @@ static void stm32_irq_ack(struct irq_data *d) struct stm32_exti_chip_data *chip_data = gc->private; const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - irq_gc_lock(gc); - + guard(raw_spinlock)(&gc->lock); irq_reg_writel(gc, d->mask, stm32_bank->rpr_ofst); - if (stm32_bank->fpr_ofst != UNDEF_REG) - irq_reg_writel(gc, d->mask, stm32_bank->fpr_ofst); - - irq_gc_unlock(gc); -} - -/* directly set the target bit without reading first. */ -static inline void stm32_exti_write_bit(struct irq_data *d, u32 reg) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - void __iomem *base = chip_data->host_data->base; - u32 val = BIT(d->hwirq % IRQS_PER_BANK); - - writel_relaxed(val, base + reg); -} - -static inline u32 stm32_exti_set_bit(struct irq_data *d, u32 reg) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - void __iomem *base = chip_data->host_data->base; - u32 val; - - val = readl_relaxed(base + reg); - val |= BIT(d->hwirq % IRQS_PER_BANK); - writel_relaxed(val, base + reg); - - return val; -} - -static inline u32 stm32_exti_clr_bit(struct irq_data *d, u32 reg) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - void __iomem *base = chip_data->host_data->base; - u32 val; - - val = readl_relaxed(base + reg); - val &= ~BIT(d->hwirq % IRQS_PER_BANK); - writel_relaxed(val, base + reg); - - return val; -} - -static void stm32_exti_h_eoi(struct irq_data *d) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - - raw_spin_lock(&chip_data->rlock); - - stm32_exti_write_bit(d, stm32_bank->rpr_ofst); - if (stm32_bank->fpr_ofst != UNDEF_REG) - stm32_exti_write_bit(d, stm32_bank->fpr_ofst); - - raw_spin_unlock(&chip_data->rlock); - - if (d->parent_data->chip) - irq_chip_eoi_parent(d); -} - -static void stm32_exti_h_mask(struct irq_data *d) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - - raw_spin_lock(&chip_data->rlock); - chip_data->mask_cache = stm32_exti_clr_bit(d, stm32_bank->imr_ofst); - raw_spin_unlock(&chip_data->rlock); - - if (d->parent_data->chip) - irq_chip_mask_parent(d); -} - -static void stm32_exti_h_unmask(struct irq_data *d) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - - raw_spin_lock(&chip_data->rlock); - chip_data->mask_cache = stm32_exti_set_bit(d, stm32_bank->imr_ofst); - raw_spin_unlock(&chip_data->rlock); - - if (d->parent_data->chip) - irq_chip_unmask_parent(d); -} - -static int stm32_exti_h_set_type(struct irq_data *d, unsigned int type) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - struct hwspinlock *hwlock = chip_data->host_data->hwlock; - void __iomem *base = chip_data->host_data->base; - u32 rtsr, ftsr; - int err; - - raw_spin_lock(&chip_data->rlock); - - if (hwlock) { - err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT); - if (err) { - pr_err("%s can't get hwspinlock (%d)\n", __func__, err); - goto unlock; - } - } - - rtsr = readl_relaxed(base + stm32_bank->rtsr_ofst); - ftsr = readl_relaxed(base + stm32_bank->ftsr_ofst); - - err = stm32_exti_set_type(d, type, &rtsr, &ftsr); - if (err) - goto unspinlock; - - writel_relaxed(rtsr, base + stm32_bank->rtsr_ofst); - writel_relaxed(ftsr, base + stm32_bank->ftsr_ofst); - -unspinlock: - if (hwlock) - hwspin_unlock_in_atomic(hwlock); -unlock: - raw_spin_unlock(&chip_data->rlock); - - return err; -} - -static int stm32_exti_h_set_wake(struct irq_data *d, unsigned int on) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - u32 mask = BIT(d->hwirq % IRQS_PER_BANK); - - raw_spin_lock(&chip_data->rlock); - - if (on) - chip_data->wake_active |= mask; - else - chip_data->wake_active &= ~mask; - - raw_spin_unlock(&chip_data->rlock); - - return 0; -} - -static int stm32_exti_h_set_affinity(struct irq_data *d, - const struct cpumask *dest, bool force) -{ - if (d->parent_data->chip) - return irq_chip_set_affinity_parent(d, dest, force); - - return IRQ_SET_MASK_OK_DONE; -} - -static int __maybe_unused stm32_exti_h_suspend(void) -{ - struct stm32_exti_chip_data *chip_data; - int i; - - for (i = 0; i < stm32_host_data->drv_data->bank_nr; i++) { - chip_data = &stm32_host_data->chips_data[i]; - raw_spin_lock(&chip_data->rlock); - stm32_chip_suspend(chip_data, chip_data->wake_active); - raw_spin_unlock(&chip_data->rlock); - } - - return 0; -} - -static void __maybe_unused stm32_exti_h_resume(void) -{ - struct stm32_exti_chip_data *chip_data; - int i; - - for (i = 0; i < stm32_host_data->drv_data->bank_nr; i++) { - chip_data = &stm32_host_data->chips_data[i]; - raw_spin_lock(&chip_data->rlock); - stm32_chip_resume(chip_data, chip_data->mask_cache); - raw_spin_unlock(&chip_data->rlock); - } -} - -static struct syscore_ops stm32_exti_h_syscore_ops = { -#ifdef CONFIG_PM_SLEEP - .suspend = stm32_exti_h_suspend, - .resume = stm32_exti_h_resume, -#endif -}; - -static void stm32_exti_h_syscore_init(struct stm32_exti_host_data *host_data) -{ - stm32_host_data = host_data; - register_syscore_ops(&stm32_exti_h_syscore_ops); -} - -static void stm32_exti_h_syscore_deinit(void) -{ - unregister_syscore_ops(&stm32_exti_h_syscore_ops); -} - -static int stm32_exti_h_retrigger(struct irq_data *d) -{ - struct stm32_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); - const struct stm32_exti_bank *stm32_bank = chip_data->reg_bank; - void __iomem *base = chip_data->host_data->base; - u32 mask = BIT(d->hwirq % IRQS_PER_BANK); - - writel_relaxed(mask, base + stm32_bank->swier_ofst); - - return 0; -} - -static struct irq_chip stm32_exti_h_chip = { - .name = "stm32-exti-h", - .irq_eoi = stm32_exti_h_eoi, - .irq_mask = stm32_exti_h_mask, - .irq_unmask = stm32_exti_h_unmask, - .irq_retrigger = stm32_exti_h_retrigger, - .irq_set_type = stm32_exti_h_set_type, - .irq_set_wake = stm32_exti_h_set_wake, - .flags = IRQCHIP_MASK_ON_SUSPEND, - .irq_set_affinity = IS_ENABLED(CONFIG_SMP) ? stm32_exti_h_set_affinity : NULL, -}; - -static struct irq_chip stm32_exti_h_chip_direct = { - .name = "stm32-exti-h-direct", - .irq_eoi = irq_chip_eoi_parent, - .irq_ack = irq_chip_ack_parent, - .irq_mask = stm32_exti_h_mask, - .irq_unmask = stm32_exti_h_unmask, - .irq_retrigger = irq_chip_retrigger_hierarchy, - .irq_set_type = irq_chip_set_type_parent, - .irq_set_wake = stm32_exti_h_set_wake, - .flags = IRQCHIP_MASK_ON_SUSPEND, - .irq_set_affinity = IS_ENABLED(CONFIG_SMP) ? irq_chip_set_affinity_parent : NULL, -}; - -static int stm32_exti_h_domain_alloc(struct irq_domain *dm, - unsigned int virq, - unsigned int nr_irqs, void *data) -{ - struct stm32_exti_host_data *host_data = dm->host_data; - struct stm32_exti_chip_data *chip_data; - u8 desc_irq; - struct irq_fwspec *fwspec = data; - struct irq_fwspec p_fwspec; - irq_hw_number_t hwirq; - int bank; - u32 event_trg; - struct irq_chip *chip; - - hwirq = fwspec->param[0]; - if (hwirq >= host_data->drv_data->bank_nr * IRQS_PER_BANK) - return -EINVAL; - - bank = hwirq / IRQS_PER_BANK; - chip_data = &host_data->chips_data[bank]; - - event_trg = readl_relaxed(host_data->base + chip_data->reg_bank->trg_ofst); - chip = (event_trg & BIT(hwirq % IRQS_PER_BANK)) ? - &stm32_exti_h_chip : &stm32_exti_h_chip_direct; - - irq_domain_set_hwirq_and_chip(dm, virq, hwirq, chip, chip_data); - - if (!host_data->drv_data->desc_irqs) - return -EINVAL; - - desc_irq = host_data->drv_data->desc_irqs[hwirq]; - if (desc_irq != EXTI_INVALID_IRQ) { - p_fwspec.fwnode = dm->parent->fwnode; - p_fwspec.param_count = 3; - p_fwspec.param[0] = GIC_SPI; - p_fwspec.param[1] = desc_irq; - p_fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH; - - return irq_domain_alloc_irqs_parent(dm, virq, 1, &p_fwspec); - } - - return 0; } static struct @@ -770,8 +286,6 @@ stm32_exti_host_data *stm32_exti_host_init(const struct stm32_exti_drv_data *dd, goto free_chips_data; } - stm32_host_data = host_data; - return host_data; free_chips_data: @@ -796,15 +310,12 @@ stm32_exti_chip_data *stm32_exti_chip_init(struct stm32_exti_host_data *h_data, chip_data->host_data = h_data; chip_data->reg_bank = stm32_bank; - raw_spin_lock_init(&chip_data->rlock); - /* * This IP has no reset, so after hot reboot we should * clear registers to avoid residue */ writel_relaxed(0, base + stm32_bank->imr_ofst); - if (stm32_bank->emr_ofst != UNDEF_REG) - writel_relaxed(0, base + stm32_bank->emr_ofst); + writel_relaxed(0, base + stm32_bank->emr_ofst); pr_info("%pOF: bank%d\n", node, bank_idx); @@ -824,8 +335,8 @@ static int __init stm32_exti_init(const struct stm32_exti_drv_data *drv_data, if (!host_data) return -ENOMEM; - domain = irq_domain_add_linear(node, drv_data->bank_nr * IRQS_PER_BANK, - &irq_exti_domain_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(node), drv_data->bank_nr * IRQS_PER_BANK, + &irq_exti_domain_ops, NULL); if (!domain) { pr_err("%pOFn: Could not register interrupt domain.\n", node); @@ -884,133 +395,6 @@ out_unmap: return ret; } -static const struct irq_domain_ops stm32_exti_h_domain_ops = { - .alloc = stm32_exti_h_domain_alloc, - .free = irq_domain_free_irqs_common, - .xlate = irq_domain_xlate_twocell, -}; - -static void stm32_exti_remove_irq(void *data) -{ - struct irq_domain *domain = data; - - irq_domain_remove(domain); -} - -static int stm32_exti_remove(struct platform_device *pdev) -{ - stm32_exti_h_syscore_deinit(); - return 0; -} - -static int stm32_exti_probe(struct platform_device *pdev) -{ - int ret, i; - struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; - struct irq_domain *parent_domain, *domain; - struct stm32_exti_host_data *host_data; - const struct stm32_exti_drv_data *drv_data; - - host_data = devm_kzalloc(dev, sizeof(*host_data), GFP_KERNEL); - if (!host_data) - return -ENOMEM; - - /* check for optional hwspinlock which may be not available yet */ - ret = of_hwspin_lock_get_id(np, 0); - if (ret == -EPROBE_DEFER) - /* hwspinlock framework not yet ready */ - return ret; - - if (ret >= 0) { - host_data->hwlock = devm_hwspin_lock_request_specific(dev, ret); - if (!host_data->hwlock) { - dev_err(dev, "Failed to request hwspinlock\n"); - return -EINVAL; - } - } else if (ret != -ENOENT) { - /* note: ENOENT is a valid case (means 'no hwspinlock') */ - dev_err(dev, "Failed to get hwspinlock\n"); - return ret; - } - - /* initialize host_data */ - drv_data = of_device_get_match_data(dev); - if (!drv_data) { - dev_err(dev, "no of match data\n"); - return -ENODEV; - } - host_data->drv_data = drv_data; - - host_data->chips_data = devm_kcalloc(dev, drv_data->bank_nr, - sizeof(*host_data->chips_data), - GFP_KERNEL); - if (!host_data->chips_data) - return -ENOMEM; - - host_data->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(host_data->base)) - return PTR_ERR(host_data->base); - - for (i = 0; i < drv_data->bank_nr; i++) - stm32_exti_chip_init(host_data, i, np); - - parent_domain = irq_find_host(of_irq_find_parent(np)); - if (!parent_domain) { - dev_err(dev, "GIC interrupt-parent not found\n"); - return -EINVAL; - } - - domain = irq_domain_add_hierarchy(parent_domain, 0, - drv_data->bank_nr * IRQS_PER_BANK, - np, &stm32_exti_h_domain_ops, - host_data); - - if (!domain) { - dev_err(dev, "Could not register exti domain\n"); - return -ENOMEM; - } - - ret = devm_add_action_or_reset(dev, stm32_exti_remove_irq, domain); - if (ret) - return ret; - - stm32_exti_h_syscore_init(host_data); - - return 0; -} - -/* platform driver only for MP1 */ -static const struct of_device_id stm32_exti_ids[] = { - { .compatible = "st,stm32mp1-exti", .data = &stm32mp1_drv_data}, - { .compatible = "st,stm32mp13-exti", .data = &stm32mp13_drv_data}, - {}, -}; -MODULE_DEVICE_TABLE(of, stm32_exti_ids); - -static struct platform_driver stm32_exti_driver = { - .probe = stm32_exti_probe, - .remove = stm32_exti_remove, - .driver = { - .name = "stm32_exti", - .of_match_table = stm32_exti_ids, - }, -}; - -static int __init stm32_exti_arch_init(void) -{ - return platform_driver_register(&stm32_exti_driver); -} - -static void __exit stm32_exti_arch_exit(void) -{ - return platform_driver_unregister(&stm32_exti_driver); -} - -arch_initcall(stm32_exti_arch_init); -module_exit(stm32_exti_arch_exit); - -/* no platform driver for F4 and H7 */ static int __init stm32f4_exti_of_init(struct device_node *np, struct device_node *parent) { diff --git a/drivers/irqchip/irq-stm32mp-exti.c b/drivers/irqchip/irq-stm32mp-exti.c new file mode 100644 index 000000000000..a24f4f1a4f8f --- /dev/null +++ b/drivers/irqchip/irq-stm32mp-exti.c @@ -0,0 +1,725 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) Maxime Coquelin 2015 + * Copyright (C) STMicroelectronics 2017-2024 + * Author: Maxime Coquelin <mcoquelin.stm32@gmail.com> + */ + +#include <linux/bitops.h> +#include <linux/hwspinlock.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/pm.h> + +#include <dt-bindings/interrupt-controller/arm-gic.h> + +#define IRQS_PER_BANK 32 + +#define HWSPNLCK_TIMEOUT 1000 /* usec */ + +#define EXTI_EnCIDCFGR(n) (0x180 + (n) * 4) +#define EXTI_HWCFGR1 0x3f0 + +/* Register: EXTI_EnCIDCFGR(n) */ +#define EXTI_CIDCFGR_CFEN_MASK BIT(0) +#define EXTI_CIDCFGR_CID_MASK GENMASK(6, 4) +#define EXTI_CIDCFGR_CID_SHIFT 4 + +/* Register: EXTI_HWCFGR1 */ +#define EXTI_HWCFGR1_CIDWIDTH_MASK GENMASK(27, 24) + +#define EXTI_CID1 1 + +struct stm32mp_exti_bank { + u32 imr_ofst; + u32 rtsr_ofst; + u32 ftsr_ofst; + u32 swier_ofst; + u32 rpr_ofst; + u32 fpr_ofst; + u32 trg_ofst; + u32 seccfgr_ofst; +}; + +struct stm32mp_exti_drv_data { + const struct stm32mp_exti_bank **exti_banks; + const u8 *desc_irqs; + u32 bank_nr; +}; + +struct stm32mp_exti_chip_data { + struct stm32mp_exti_host_data *host_data; + const struct stm32mp_exti_bank *reg_bank; + struct raw_spinlock rlock; + u32 wake_active; + u32 mask_cache; + u32 rtsr_cache; + u32 ftsr_cache; + u32 event_reserved; +}; + +struct stm32mp_exti_host_data { + void __iomem *base; + struct device *dev; + struct stm32mp_exti_chip_data *chips_data; + const struct stm32mp_exti_drv_data *drv_data; + struct hwspinlock *hwlock; + /* skip internal desc_irqs array and get it from DT */ + bool dt_has_irqs_desc; +}; + +static const struct stm32mp_exti_bank stm32mp_exti_b1 = { + .imr_ofst = 0x80, + .rtsr_ofst = 0x00, + .ftsr_ofst = 0x04, + .swier_ofst = 0x08, + .rpr_ofst = 0x0C, + .fpr_ofst = 0x10, + .trg_ofst = 0x3EC, + .seccfgr_ofst = 0x14, +}; + +static const struct stm32mp_exti_bank stm32mp_exti_b2 = { + .imr_ofst = 0x90, + .rtsr_ofst = 0x20, + .ftsr_ofst = 0x24, + .swier_ofst = 0x28, + .rpr_ofst = 0x2C, + .fpr_ofst = 0x30, + .trg_ofst = 0x3E8, + .seccfgr_ofst = 0x34, +}; + +static const struct stm32mp_exti_bank stm32mp_exti_b3 = { + .imr_ofst = 0xA0, + .rtsr_ofst = 0x40, + .ftsr_ofst = 0x44, + .swier_ofst = 0x48, + .rpr_ofst = 0x4C, + .fpr_ofst = 0x50, + .trg_ofst = 0x3E4, + .seccfgr_ofst = 0x54, +}; + +static const struct stm32mp_exti_bank *stm32mp_exti_banks[] = { + &stm32mp_exti_b1, + &stm32mp_exti_b2, + &stm32mp_exti_b3, +}; + +static struct irq_chip stm32mp_exti_chip; +static struct irq_chip stm32mp_exti_chip_direct; + +#define EXTI_INVALID_IRQ U8_MAX +#define STM32MP_DESC_IRQ_SIZE (ARRAY_SIZE(stm32mp_exti_banks) * IRQS_PER_BANK) + +/* + * Use some intentionally tricky logic here to initialize the whole array to + * EXTI_INVALID_IRQ, but then override certain fields, requiring us to indicate + * that we "know" that there are overrides in this structure, and we'll need to + * disable that warning from W=1 builds. + */ +__diag_push(); +__diag_ignore_all("-Woverride-init", + "logic to initialize all and then override some is OK"); + +static const u8 stm32mp1_desc_irq[] = { + /* default value */ + [0 ... (STM32MP_DESC_IRQ_SIZE - 1)] = EXTI_INVALID_IRQ, + + [0] = 6, + [1] = 7, + [2] = 8, + [3] = 9, + [4] = 10, + [5] = 23, + [6] = 64, + [7] = 65, + [8] = 66, + [9] = 67, + [10] = 40, + [11] = 42, + [12] = 76, + [13] = 77, + [14] = 121, + [15] = 127, + [16] = 1, + [19] = 3, + [21] = 31, + [22] = 33, + [23] = 72, + [24] = 95, + [25] = 107, + [26] = 37, + [27] = 38, + [28] = 39, + [29] = 71, + [30] = 52, + [31] = 53, + [32] = 82, + [33] = 83, + [46] = 151, + [47] = 93, + [48] = 138, + [50] = 139, + [52] = 140, + [53] = 141, + [54] = 135, + [61] = 100, + [65] = 144, + [68] = 143, + [70] = 62, + [73] = 129, +}; + +static const u8 stm32mp13_desc_irq[] = { + /* default value */ + [0 ... (STM32MP_DESC_IRQ_SIZE - 1)] = EXTI_INVALID_IRQ, + + [0] = 6, + [1] = 7, + [2] = 8, + [3] = 9, + [4] = 10, + [5] = 24, + [6] = 65, + [7] = 66, + [8] = 67, + [9] = 68, + [10] = 41, + [11] = 43, + [12] = 77, + [13] = 78, + [14] = 106, + [15] = 109, + [16] = 1, + [19] = 3, + [21] = 32, + [22] = 34, + [23] = 73, + [24] = 93, + [25] = 114, + [26] = 38, + [27] = 39, + [28] = 40, + [29] = 72, + [30] = 53, + [31] = 54, + [32] = 83, + [33] = 84, + [44] = 96, + [47] = 92, + [48] = 116, + [50] = 117, + [52] = 118, + [53] = 119, + [68] = 63, + [70] = 98, +}; + +__diag_pop(); + +static const struct stm32mp_exti_drv_data stm32mp1_drv_data = { + .exti_banks = stm32mp_exti_banks, + .bank_nr = ARRAY_SIZE(stm32mp_exti_banks), + .desc_irqs = stm32mp1_desc_irq, +}; + +static const struct stm32mp_exti_drv_data stm32mp13_drv_data = { + .exti_banks = stm32mp_exti_banks, + .bank_nr = ARRAY_SIZE(stm32mp_exti_banks), + .desc_irqs = stm32mp13_desc_irq, +}; + +static int stm32mp_exti_convert_type(struct irq_data *d, unsigned int type, u32 *rtsr, u32 *ftsr) +{ + u32 mask = BIT(d->hwirq % IRQS_PER_BANK); + + switch (type) { + case IRQ_TYPE_EDGE_RISING: + *rtsr |= mask; + *ftsr &= ~mask; + break; + case IRQ_TYPE_EDGE_FALLING: + *rtsr &= ~mask; + *ftsr |= mask; + break; + case IRQ_TYPE_EDGE_BOTH: + *rtsr |= mask; + *ftsr |= mask; + break; + default: + return -EINVAL; + } + + return 0; +} + +static void stm32mp_chip_suspend(struct stm32mp_exti_chip_data *chip_data, u32 wake_active) +{ + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + void __iomem *base = chip_data->host_data->base; + + /* save rtsr, ftsr registers */ + chip_data->rtsr_cache = readl_relaxed(base + bank->rtsr_ofst); + chip_data->ftsr_cache = readl_relaxed(base + bank->ftsr_ofst); + + writel_relaxed(wake_active, base + bank->imr_ofst); +} + +static void stm32mp_chip_resume(struct stm32mp_exti_chip_data *chip_data, u32 mask_cache) +{ + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + void __iomem *base = chip_data->host_data->base; + + /* restore rtsr, ftsr, registers */ + writel_relaxed(chip_data->rtsr_cache, base + bank->rtsr_ofst); + writel_relaxed(chip_data->ftsr_cache, base + bank->ftsr_ofst); + + writel_relaxed(mask_cache, base + bank->imr_ofst); +} + +/* directly set the target bit without reading first. */ +static inline void stm32mp_exti_write_bit(struct irq_data *d, u32 reg) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + void __iomem *base = chip_data->host_data->base; + u32 val = BIT(d->hwirq % IRQS_PER_BANK); + + writel_relaxed(val, base + reg); +} + +static inline u32 stm32mp_exti_set_bit(struct irq_data *d, u32 reg) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + void __iomem *base = chip_data->host_data->base; + u32 val; + + val = readl_relaxed(base + reg); + val |= BIT(d->hwirq % IRQS_PER_BANK); + writel_relaxed(val, base + reg); + + return val; +} + +static inline u32 stm32mp_exti_clr_bit(struct irq_data *d, u32 reg) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + void __iomem *base = chip_data->host_data->base; + u32 val; + + val = readl_relaxed(base + reg); + val &= ~BIT(d->hwirq % IRQS_PER_BANK); + writel_relaxed(val, base + reg); + + return val; +} + +static void stm32mp_exti_eoi(struct irq_data *d) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + + raw_spin_lock(&chip_data->rlock); + + stm32mp_exti_write_bit(d, bank->rpr_ofst); + stm32mp_exti_write_bit(d, bank->fpr_ofst); + + raw_spin_unlock(&chip_data->rlock); + + if (d->parent_data->chip) + irq_chip_eoi_parent(d); +} + +static void stm32mp_exti_mask(struct irq_data *d) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + + raw_spin_lock(&chip_data->rlock); + chip_data->mask_cache = stm32mp_exti_clr_bit(d, bank->imr_ofst); + raw_spin_unlock(&chip_data->rlock); + + if (d->parent_data->chip) + irq_chip_mask_parent(d); +} + +static void stm32mp_exti_unmask(struct irq_data *d) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + + raw_spin_lock(&chip_data->rlock); + chip_data->mask_cache = stm32mp_exti_set_bit(d, bank->imr_ofst); + raw_spin_unlock(&chip_data->rlock); + + if (d->parent_data->chip) + irq_chip_unmask_parent(d); +} + +static int stm32mp_exti_set_type(struct irq_data *d, unsigned int type) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + struct hwspinlock *hwlock = chip_data->host_data->hwlock; + void __iomem *base = chip_data->host_data->base; + u32 rtsr, ftsr; + int err; + + raw_spin_lock(&chip_data->rlock); + + if (hwlock) { + err = hwspin_lock_timeout_in_atomic(hwlock, HWSPNLCK_TIMEOUT); + if (err) { + pr_err("%s can't get hwspinlock (%d)\n", __func__, err); + goto unlock; + } + } + + rtsr = readl_relaxed(base + bank->rtsr_ofst); + ftsr = readl_relaxed(base + bank->ftsr_ofst); + + err = stm32mp_exti_convert_type(d, type, &rtsr, &ftsr); + if (!err) { + writel_relaxed(rtsr, base + bank->rtsr_ofst); + writel_relaxed(ftsr, base + bank->ftsr_ofst); + } + + if (hwlock) + hwspin_unlock_in_atomic(hwlock); +unlock: + raw_spin_unlock(&chip_data->rlock); + return err; +} + +static int stm32mp_exti_set_wake(struct irq_data *d, unsigned int on) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + u32 mask = BIT(d->hwirq % IRQS_PER_BANK); + + raw_spin_lock(&chip_data->rlock); + + if (on) + chip_data->wake_active |= mask; + else + chip_data->wake_active &= ~mask; + + raw_spin_unlock(&chip_data->rlock); + + return 0; +} + +static int stm32mp_exti_set_affinity(struct irq_data *d, const struct cpumask *dest, bool force) +{ + if (d->parent_data->chip) + return irq_chip_set_affinity_parent(d, dest, force); + + return IRQ_SET_MASK_OK_DONE; +} + +static int stm32mp_exti_suspend(struct device *dev) +{ + struct stm32mp_exti_host_data *host_data = dev_get_drvdata(dev); + struct stm32mp_exti_chip_data *chip_data; + int i; + + for (i = 0; i < host_data->drv_data->bank_nr; i++) { + chip_data = &host_data->chips_data[i]; + stm32mp_chip_suspend(chip_data, chip_data->wake_active); + } + + return 0; +} + +static int stm32mp_exti_resume(struct device *dev) +{ + struct stm32mp_exti_host_data *host_data = dev_get_drvdata(dev); + struct stm32mp_exti_chip_data *chip_data; + int i; + + for (i = 0; i < host_data->drv_data->bank_nr; i++) { + chip_data = &host_data->chips_data[i]; + stm32mp_chip_resume(chip_data, chip_data->mask_cache); + } + + return 0; +} + +static int stm32mp_exti_retrigger(struct irq_data *d) +{ + struct stm32mp_exti_chip_data *chip_data = irq_data_get_irq_chip_data(d); + const struct stm32mp_exti_bank *bank = chip_data->reg_bank; + void __iomem *base = chip_data->host_data->base; + u32 mask = BIT(d->hwirq % IRQS_PER_BANK); + + writel_relaxed(mask, base + bank->swier_ofst); + + return 0; +} + +static struct irq_chip stm32mp_exti_chip = { + .name = "stm32mp-exti", + .irq_eoi = stm32mp_exti_eoi, + .irq_mask = stm32mp_exti_mask, + .irq_unmask = stm32mp_exti_unmask, + .irq_retrigger = stm32mp_exti_retrigger, + .irq_set_type = stm32mp_exti_set_type, + .irq_set_wake = stm32mp_exti_set_wake, + .flags = IRQCHIP_MASK_ON_SUSPEND, + .irq_set_affinity = IS_ENABLED(CONFIG_SMP) ? stm32mp_exti_set_affinity : NULL, +}; + +static struct irq_chip stm32mp_exti_chip_direct = { + .name = "stm32mp-exti-direct", + .irq_eoi = irq_chip_eoi_parent, + .irq_ack = irq_chip_ack_parent, + .irq_mask = stm32mp_exti_mask, + .irq_unmask = stm32mp_exti_unmask, + .irq_retrigger = irq_chip_retrigger_hierarchy, + .irq_set_type = irq_chip_set_type_parent, + .irq_set_wake = stm32mp_exti_set_wake, + .flags = IRQCHIP_MASK_ON_SUSPEND, + .irq_set_affinity = IS_ENABLED(CONFIG_SMP) ? irq_chip_set_affinity_parent : NULL, +}; + +static int stm32mp_exti_domain_alloc(struct irq_domain *dm, + unsigned int virq, + unsigned int nr_irqs, void *data) +{ + struct stm32mp_exti_host_data *host_data = dm->host_data; + struct stm32mp_exti_chip_data *chip_data; + struct irq_fwspec *fwspec = data; + struct irq_fwspec p_fwspec; + irq_hw_number_t hwirq; + struct irq_chip *chip; + u32 event_trg; + u8 desc_irq; + int bank; + + hwirq = fwspec->param[0]; + if (hwirq >= host_data->drv_data->bank_nr * IRQS_PER_BANK) + return -EINVAL; + + bank = hwirq / IRQS_PER_BANK; + chip_data = &host_data->chips_data[bank]; + + /* Check if event is reserved (Secure) */ + if (chip_data->event_reserved & BIT(hwirq % IRQS_PER_BANK)) { + dev_err(host_data->dev, "event %lu is reserved, secure\n", hwirq); + return -EPERM; + } + + event_trg = readl_relaxed(host_data->base + chip_data->reg_bank->trg_ofst); + chip = (event_trg & BIT(hwirq % IRQS_PER_BANK)) ? + &stm32mp_exti_chip : &stm32mp_exti_chip_direct; + + irq_domain_set_hwirq_and_chip(dm, virq, hwirq, chip, chip_data); + + if (host_data->dt_has_irqs_desc) { + struct of_phandle_args out_irq; + int ret; + + ret = of_irq_parse_one(host_data->dev->of_node, hwirq, &out_irq); + if (ret) + return ret; + /* we only support one parent, so far */ + if (of_fwnode_handle(out_irq.np) != dm->parent->fwnode) + return -EINVAL; + + of_phandle_args_to_fwspec(out_irq.np, out_irq.args, + out_irq.args_count, &p_fwspec); + + return irq_domain_alloc_irqs_parent(dm, virq, 1, &p_fwspec); + } + + if (!host_data->drv_data->desc_irqs) + return -EINVAL; + + desc_irq = host_data->drv_data->desc_irqs[hwirq]; + if (desc_irq != EXTI_INVALID_IRQ) { + p_fwspec.fwnode = dm->parent->fwnode; + p_fwspec.param_count = 3; + p_fwspec.param[0] = GIC_SPI; + p_fwspec.param[1] = desc_irq; + p_fwspec.param[2] = IRQ_TYPE_LEVEL_HIGH; + + return irq_domain_alloc_irqs_parent(dm, virq, 1, &p_fwspec); + } + + return 0; +} + +static struct stm32mp_exti_chip_data *stm32mp_exti_chip_init(struct stm32mp_exti_host_data *h_data, + u32 bank_idx, struct device_node *node) +{ + struct stm32mp_exti_chip_data *chip_data; + const struct stm32mp_exti_bank *bank; + void __iomem *base = h_data->base; + + bank = h_data->drv_data->exti_banks[bank_idx]; + chip_data = &h_data->chips_data[bank_idx]; + chip_data->host_data = h_data; + chip_data->reg_bank = bank; + + raw_spin_lock_init(&chip_data->rlock); + + /* + * This IP has no reset, so after hot reboot we should + * clear registers to avoid residue + */ + writel_relaxed(0, base + bank->imr_ofst); + + /* reserve Secure events */ + chip_data->event_reserved = readl_relaxed(base + bank->seccfgr_ofst); + + pr_info("%pOF: bank%d\n", node, bank_idx); + + return chip_data; +} + +static const struct irq_domain_ops stm32mp_exti_domain_ops = { + .alloc = stm32mp_exti_domain_alloc, + .free = irq_domain_free_irqs_common, + .xlate = irq_domain_xlate_twocell, +}; + +static void stm32mp_exti_check_rif(struct stm32mp_exti_host_data *host_data) +{ + unsigned int bank, i, event; + u32 cid, cidcfgr, hwcfgr1; + + /* quit on CID not supported */ + hwcfgr1 = readl_relaxed(host_data->base + EXTI_HWCFGR1); + if ((hwcfgr1 & EXTI_HWCFGR1_CIDWIDTH_MASK) == 0) + return; + + for (bank = 0; bank < host_data->drv_data->bank_nr; bank++) { + for (i = 0; i < IRQS_PER_BANK; i++) { + event = bank * IRQS_PER_BANK + i; + cidcfgr = readl_relaxed(host_data->base + EXTI_EnCIDCFGR(event)); + cid = (cidcfgr & EXTI_CIDCFGR_CID_MASK) >> EXTI_CIDCFGR_CID_SHIFT; + if ((cidcfgr & EXTI_CIDCFGR_CFEN_MASK) && cid != EXTI_CID1) + host_data->chips_data[bank].event_reserved |= BIT(i); + } + } +} + +static void stm32mp_exti_remove_irq(void *data) +{ + struct irq_domain *domain = data; + + irq_domain_remove(domain); +} + +static int stm32mp_exti_probe(struct platform_device *pdev) +{ + const struct stm32mp_exti_drv_data *drv_data; + struct irq_domain *parent_domain, *domain; + struct stm32mp_exti_host_data *host_data; + struct device *dev = &pdev->dev; + struct device_node *np = dev->of_node; + int ret, i; + + host_data = devm_kzalloc(dev, sizeof(*host_data), GFP_KERNEL); + if (!host_data) + return -ENOMEM; + + dev_set_drvdata(dev, host_data); + host_data->dev = dev; + + /* check for optional hwspinlock which may be not available yet */ + ret = of_hwspin_lock_get_id(np, 0); + if (ret == -EPROBE_DEFER) + /* hwspinlock framework not yet ready */ + return ret; + + if (ret >= 0) { + host_data->hwlock = devm_hwspin_lock_request_specific(dev, ret); + if (!host_data->hwlock) { + dev_err(dev, "Failed to request hwspinlock\n"); + return -EINVAL; + } + } else if (ret != -ENOENT) { + /* note: ENOENT is a valid case (means 'no hwspinlock') */ + dev_err(dev, "Failed to get hwspinlock\n"); + return ret; + } + + /* initialize host_data */ + drv_data = of_device_get_match_data(dev); + if (!drv_data) { + dev_err(dev, "no of match data\n"); + return -ENODEV; + } + host_data->drv_data = drv_data; + + host_data->chips_data = devm_kcalloc(dev, drv_data->bank_nr, + sizeof(*host_data->chips_data), + GFP_KERNEL); + if (!host_data->chips_data) + return -ENOMEM; + + host_data->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(host_data->base)) + return PTR_ERR(host_data->base); + + for (i = 0; i < drv_data->bank_nr; i++) + stm32mp_exti_chip_init(host_data, i, np); + + stm32mp_exti_check_rif(host_data); + + parent_domain = irq_find_host(of_irq_find_parent(np)); + if (!parent_domain) { + dev_err(dev, "GIC interrupt-parent not found\n"); + return -EINVAL; + } + + domain = irq_domain_create_hierarchy(parent_domain, 0, drv_data->bank_nr * IRQS_PER_BANK, + dev_fwnode(dev), &stm32mp_exti_domain_ops, host_data); + if (!domain) { + dev_err(dev, "Could not register exti domain\n"); + return -ENOMEM; + } + + ret = devm_add_action_or_reset(dev, stm32mp_exti_remove_irq, domain); + if (ret) + return ret; + + host_data->dt_has_irqs_desc = of_property_present(np, "interrupts-extended"); + + return 0; +} + +static const struct of_device_id stm32mp_exti_ids[] = { + { .compatible = "st,stm32mp1-exti", .data = &stm32mp1_drv_data}, + { .compatible = "st,stm32mp13-exti", .data = &stm32mp13_drv_data}, + {}, +}; +MODULE_DEVICE_TABLE(of, stm32mp_exti_ids); + +static const struct dev_pm_ops stm32mp_exti_dev_pm_ops = { + NOIRQ_SYSTEM_SLEEP_PM_OPS(stm32mp_exti_suspend, stm32mp_exti_resume) +}; + +static struct platform_driver stm32mp_exti_driver = { + .probe = stm32mp_exti_probe, + .driver = { + .name = "stm32mp_exti", + .of_match_table = stm32mp_exti_ids, + .pm = &stm32mp_exti_dev_pm_ops, + }, +}; + +module_platform_driver(stm32mp_exti_driver); + +MODULE_AUTHOR("Maxime Coquelin <mcoquelin.stm32@gmail.com>"); +MODULE_DESCRIPTION("STM32MP EXTI driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-sun4i.c b/drivers/irqchip/irq-sun4i.c index dd506ebfdacb..9c2c9caeca2a 100644 --- a/drivers/irqchip/irq-sun4i.c +++ b/drivers/irqchip/irq-sun4i.c @@ -133,7 +133,7 @@ static int __init sun4i_of_init(struct device_node *node, /* Configure the external interrupt source type */ writel(0x00, irq_ic_data->irq_base + SUN4I_IRQ_NMI_CTRL_REG); - irq_ic_data->irq_domain = irq_domain_add_linear(node, 3 * 32, + irq_ic_data->irq_domain = irq_domain_create_linear(of_fwnode_handle(node), 3 * 32, &sun4i_irq_ops, NULL); if (!irq_ic_data->irq_domain) panic("%pOF: unable to create IRQ domain\n", node); diff --git a/drivers/irqchip/irq-sun6i-r.c b/drivers/irqchip/irq-sun6i-r.c index a01e44049415..23251831c06e 100644 --- a/drivers/irqchip/irq-sun6i-r.c +++ b/drivers/irqchip/irq-sun6i-r.c @@ -268,9 +268,9 @@ static const struct irq_domain_ops sun6i_r_intc_domain_ops = { .free = irq_domain_free_irqs_common, }; -static int sun6i_r_intc_suspend(void) +static int sun6i_r_intc_suspend(void *data) { - u32 buf[BITS_TO_U32(max(SUN6I_NR_TOP_LEVEL_IRQS, SUN6I_NR_MUX_BITS))]; + u32 buf[BITS_TO_U32(MAX(SUN6I_NR_TOP_LEVEL_IRQS, SUN6I_NR_MUX_BITS))]; int i; /* Wake IRQs are enabled during system sleep and shutdown. */ @@ -284,7 +284,7 @@ static int sun6i_r_intc_suspend(void) return 0; } -static void sun6i_r_intc_resume(void) +static void sun6i_r_intc_resume(void *data) { int i; @@ -294,17 +294,21 @@ static void sun6i_r_intc_resume(void) writel_relaxed(0, base + SUN6I_IRQ_ENABLE(i)); } -static void sun6i_r_intc_shutdown(void) +static void sun6i_r_intc_shutdown(void *data) { - sun6i_r_intc_suspend(); + sun6i_r_intc_suspend(data); } -static struct syscore_ops sun6i_r_intc_syscore_ops = { +static const struct syscore_ops sun6i_r_intc_syscore_ops = { .suspend = sun6i_r_intc_suspend, .resume = sun6i_r_intc_resume, .shutdown = sun6i_r_intc_shutdown, }; +static struct syscore sun6i_r_intc_syscore = { + .ops = &sun6i_r_intc_syscore_ops, +}; + static int __init sun6i_r_intc_init(struct device_node *node, struct device_node *parent, const struct sun6i_r_intc_variant *v) @@ -338,18 +342,18 @@ static int __init sun6i_r_intc_init(struct device_node *node, return PTR_ERR(base); } - domain = irq_domain_add_hierarchy(parent_domain, 0, 0, node, - &sun6i_r_intc_domain_ops, NULL); + domain = irq_domain_create_hierarchy(parent_domain, 0, 0, of_fwnode_handle(node), + &sun6i_r_intc_domain_ops, NULL); if (!domain) { pr_err("%pOF: Failed to allocate domain\n", node); iounmap(base); return -ENOMEM; } - register_syscore_ops(&sun6i_r_intc_syscore_ops); + register_syscore(&sun6i_r_intc_syscore); sun6i_r_intc_ack_nmi(); - sun6i_r_intc_resume(); + sun6i_r_intc_resume(NULL); return 0; } diff --git a/drivers/irqchip/irq-sunxi-nmi.c b/drivers/irqchip/irq-sunxi-nmi.c index e760b1278143..fe32dfdc2dd0 100644 --- a/drivers/irqchip/irq-sunxi-nmi.c +++ b/drivers/irqchip/irq-sunxi-nmi.c @@ -48,32 +48,41 @@ enum { SUNXI_SRC_TYPE_EDGE_RISING, }; -struct sunxi_sc_nmi_reg_offs { - u32 ctrl; - u32 pend; - u32 enable; +struct sunxi_sc_nmi_data { + struct { + u32 ctrl; + u32 pend; + u32 enable; + } reg_offs; + u32 enable_val; }; -static const struct sunxi_sc_nmi_reg_offs sun6i_reg_offs __initconst = { - .ctrl = SUN6I_NMI_CTRL, - .pend = SUN6I_NMI_PENDING, - .enable = SUN6I_NMI_ENABLE, +static const struct sunxi_sc_nmi_data sun6i_data __initconst = { + .reg_offs.ctrl = SUN6I_NMI_CTRL, + .reg_offs.pend = SUN6I_NMI_PENDING, + .reg_offs.enable = SUN6I_NMI_ENABLE, }; -static const struct sunxi_sc_nmi_reg_offs sun7i_reg_offs __initconst = { - .ctrl = SUN7I_NMI_CTRL, - .pend = SUN7I_NMI_PENDING, - .enable = SUN7I_NMI_ENABLE, +static const struct sunxi_sc_nmi_data sun7i_data __initconst = { + .reg_offs.ctrl = SUN7I_NMI_CTRL, + .reg_offs.pend = SUN7I_NMI_PENDING, + .reg_offs.enable = SUN7I_NMI_ENABLE, }; -static const struct sunxi_sc_nmi_reg_offs sun9i_reg_offs __initconst = { - .ctrl = SUN9I_NMI_CTRL, - .pend = SUN9I_NMI_PENDING, - .enable = SUN9I_NMI_ENABLE, +static const struct sunxi_sc_nmi_data sun9i_data __initconst = { + .reg_offs.ctrl = SUN9I_NMI_CTRL, + .reg_offs.pend = SUN9I_NMI_PENDING, + .reg_offs.enable = SUN9I_NMI_ENABLE, }; -static inline void sunxi_sc_nmi_write(struct irq_chip_generic *gc, u32 off, - u32 val) +static const struct sunxi_sc_nmi_data sun55i_a523_data __initconst = { + .reg_offs.ctrl = SUN9I_NMI_CTRL, + .reg_offs.pend = SUN9I_NMI_PENDING, + .reg_offs.enable = SUN9I_NMI_ENABLE, + .enable_val = BIT(31), +}; + +static inline void sunxi_sc_nmi_write(struct irq_chip_generic *gc, u32 off, u32 val) { irq_reg_writel(gc, val, off); } @@ -102,7 +111,7 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) unsigned int src_type; unsigned int i; - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); switch (flow_type & IRQF_TRIGGER_MASK) { case IRQ_TYPE_EDGE_FALLING: @@ -119,9 +128,7 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) src_type = SUNXI_SRC_TYPE_LEVEL_LOW; break; default: - irq_gc_unlock(gc); - pr_err("Cannot assign multiple trigger modes to IRQ %d.\n", - data->irq); + pr_err("Cannot assign multiple trigger modes to IRQ %d.\n", data->irq); return -EBADR; } @@ -136,23 +143,18 @@ static int sunxi_sc_nmi_set_type(struct irq_data *data, unsigned int flow_type) src_type_reg &= ~SUNXI_NMI_SRC_TYPE_MASK; src_type_reg |= src_type; sunxi_sc_nmi_write(gc, ctrl_off, src_type_reg); - - irq_gc_unlock(gc); - return IRQ_SET_MASK_OK; } static int __init sunxi_sc_nmi_irq_init(struct device_node *node, - const struct sunxi_sc_nmi_reg_offs *reg_offs) + const struct sunxi_sc_nmi_data *data) { - struct irq_domain *domain; + unsigned int irq, clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; struct irq_chip_generic *gc; - unsigned int irq; - unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + struct irq_domain *domain; int ret; - - domain = irq_domain_add_linear(node, 1, &irq_generic_chip_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(node), 1, &irq_generic_chip_ops, NULL); if (!domain) { pr_err("Could not register interrupt domain.\n"); return -ENOMEM; @@ -186,27 +188,28 @@ static int __init sunxi_sc_nmi_irq_init(struct device_node *node, gc->chip_types[0].chip.irq_unmask = irq_gc_mask_set_bit; gc->chip_types[0].chip.irq_eoi = irq_gc_ack_set_bit; gc->chip_types[0].chip.irq_set_type = sunxi_sc_nmi_set_type; - gc->chip_types[0].chip.flags = IRQCHIP_EOI_THREADED | IRQCHIP_EOI_IF_HANDLED; - gc->chip_types[0].regs.ack = reg_offs->pend; - gc->chip_types[0].regs.mask = reg_offs->enable; - gc->chip_types[0].regs.type = reg_offs->ctrl; + gc->chip_types[0].chip.flags = IRQCHIP_EOI_THREADED | + IRQCHIP_EOI_IF_HANDLED | + IRQCHIP_SKIP_SET_WAKE; + gc->chip_types[0].regs.ack = data->reg_offs.pend; + gc->chip_types[0].regs.mask = data->reg_offs.enable; + gc->chip_types[0].regs.type = data->reg_offs.ctrl; gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH; - gc->chip_types[1].chip.name = gc->chip_types[0].chip.name; gc->chip_types[1].chip.irq_ack = irq_gc_ack_set_bit; gc->chip_types[1].chip.irq_mask = irq_gc_mask_clr_bit; gc->chip_types[1].chip.irq_unmask = irq_gc_mask_set_bit; gc->chip_types[1].chip.irq_set_type = sunxi_sc_nmi_set_type; - gc->chip_types[1].regs.ack = reg_offs->pend; - gc->chip_types[1].regs.mask = reg_offs->enable; - gc->chip_types[1].regs.type = reg_offs->ctrl; + gc->chip_types[1].regs.ack = data->reg_offs.pend; + gc->chip_types[1].regs.mask = data->reg_offs.enable; + gc->chip_types[1].regs.type = data->reg_offs.ctrl; gc->chip_types[1].handler = handle_edge_irq; /* Disable any active interrupts */ - sunxi_sc_nmi_write(gc, reg_offs->enable, 0); + sunxi_sc_nmi_write(gc, data->reg_offs.enable, data->enable_val); /* Clear any pending NMI interrupts */ - sunxi_sc_nmi_write(gc, reg_offs->pend, SUNXI_NMI_IRQ_BIT); + sunxi_sc_nmi_write(gc, data->reg_offs.pend, SUNXI_NMI_IRQ_BIT); irq_set_chained_handler_and_data(irq, sunxi_sc_nmi_handle_irq, domain); @@ -221,20 +224,27 @@ fail_irqd_remove: static int __init sun6i_sc_nmi_irq_init(struct device_node *node, struct device_node *parent) { - return sunxi_sc_nmi_irq_init(node, &sun6i_reg_offs); + return sunxi_sc_nmi_irq_init(node, &sun6i_data); } IRQCHIP_DECLARE(sun6i_sc_nmi, "allwinner,sun6i-a31-sc-nmi", sun6i_sc_nmi_irq_init); static int __init sun7i_sc_nmi_irq_init(struct device_node *node, struct device_node *parent) { - return sunxi_sc_nmi_irq_init(node, &sun7i_reg_offs); + return sunxi_sc_nmi_irq_init(node, &sun7i_data); } IRQCHIP_DECLARE(sun7i_sc_nmi, "allwinner,sun7i-a20-sc-nmi", sun7i_sc_nmi_irq_init); static int __init sun9i_nmi_irq_init(struct device_node *node, struct device_node *parent) { - return sunxi_sc_nmi_irq_init(node, &sun9i_reg_offs); + return sunxi_sc_nmi_irq_init(node, &sun9i_data); } IRQCHIP_DECLARE(sun9i_nmi, "allwinner,sun9i-a80-nmi", sun9i_nmi_irq_init); + +static int __init sun55i_nmi_irq_init(struct device_node *node, + struct device_node *parent) +{ + return sunxi_sc_nmi_irq_init(node, &sun55i_a523_data); +} +IRQCHIP_DECLARE(sun55i_nmi, "allwinner,sun55i-a523-nmi", sun55i_nmi_irq_init); diff --git a/drivers/irqchip/irq-tb10x.c b/drivers/irqchip/irq-tb10x.c index 680586354d12..94cbc5111d7e 100644 --- a/drivers/irqchip/irq-tb10x.c +++ b/drivers/irqchip/irq-tb10x.c @@ -41,11 +41,9 @@ static inline u32 ab_irqctl_readreg(struct irq_chip_generic *gc, u32 reg) static int tb10x_irq_set_type(struct irq_data *data, unsigned int flow_type) { struct irq_chip_generic *gc = irq_data_get_irq_chip_data(data); - uint32_t im, mod, pol; + uint32_t mod, pol, im = data->mask; - im = data->mask; - - irq_gc_lock(gc); + guard(raw_spinlock)(&gc->lock); mod = ab_irqctl_readreg(gc, AB_IRQCTL_SRC_MODE) | im; pol = ab_irqctl_readreg(gc, AB_IRQCTL_SRC_POLARITY) | im; @@ -67,9 +65,7 @@ static int tb10x_irq_set_type(struct irq_data *data, unsigned int flow_type) case IRQ_TYPE_EDGE_RISING: break; default: - irq_gc_unlock(gc); - pr_err("%s: Cannot assign multiple trigger modes to IRQ %d.\n", - __func__, data->irq); + pr_err("%s: Cannot assign multiple trigger modes to IRQ %d.\n", __func__, data->irq); return -EBADR; } @@ -79,9 +75,6 @@ static int tb10x_irq_set_type(struct irq_data *data, unsigned int flow_type) ab_irqctl_writereg(gc, AB_IRQCTL_SRC_MODE, mod); ab_irqctl_writereg(gc, AB_IRQCTL_SRC_POLARITY, pol); ab_irqctl_writereg(gc, AB_IRQCTL_INT_STATUS, im); - - irq_gc_unlock(gc); - return IRQ_SET_MASK_OK; } @@ -121,13 +114,13 @@ static int __init of_tb10x_init_irq(struct device_node *ictl, goto ioremap_fail; } - domain = irq_domain_add_linear(ictl, AB_IRQCTL_MAXIRQ, - &irq_generic_chip_ops, NULL); + domain = irq_domain_create_linear(of_fwnode_handle(ictl), AB_IRQCTL_MAXIRQ, + &irq_generic_chip_ops, NULL); if (!domain) { ret = -ENOMEM; pr_err("%pOFn: Could not register interrupt domain.\n", ictl); - goto irq_domain_add_fail; + goto irq_domain_create_fail; } ret = irq_alloc_domain_generic_chips(domain, AB_IRQCTL_MAXIRQ, @@ -150,7 +143,6 @@ static int __init of_tb10x_init_irq(struct device_node *ictl, gc->chip_types[0].regs.mask = AB_IRQCTL_INT_ENABLE; gc->chip_types[1].type = IRQ_TYPE_EDGE_BOTH; - gc->chip_types[1].chip.name = gc->chip_types[0].chip.name; gc->chip_types[1].chip.irq_ack = irq_gc_ack_set_bit; gc->chip_types[1].chip.irq_mask = irq_gc_mask_clr_bit; gc->chip_types[1].chip.irq_unmask = irq_gc_mask_set_bit; @@ -175,7 +167,7 @@ static int __init of_tb10x_init_irq(struct device_node *ictl, gc_alloc_fail: irq_domain_remove(domain); -irq_domain_add_fail: +irq_domain_create_fail: iounmap(reg_base); ioremap_fail: release_mem_region(mem.start, resource_size(&mem)); diff --git a/drivers/irqchip/irq-tegra.c b/drivers/irqchip/irq-tegra.c index ad3e2c1b3c87..b6382cf6359a 100644 --- a/drivers/irqchip/irq-tegra.c +++ b/drivers/irqchip/irq-tegra.c @@ -132,7 +132,7 @@ static int tegra_set_wake(struct irq_data *d, unsigned int enable) return 0; } -static int tegra_ictlr_suspend(void) +static int tegra_ictlr_suspend(void *data) { unsigned long flags; unsigned int i; @@ -161,7 +161,7 @@ static int tegra_ictlr_suspend(void) return 0; } -static void tegra_ictlr_resume(void) +static void tegra_ictlr_resume(void *data) { unsigned long flags; unsigned int i; @@ -184,14 +184,18 @@ static void tegra_ictlr_resume(void) local_irq_restore(flags); } -static struct syscore_ops tegra_ictlr_syscore_ops = { +static const struct syscore_ops tegra_ictlr_syscore_ops = { .suspend = tegra_ictlr_suspend, .resume = tegra_ictlr_resume, }; +static struct syscore tegra_ictlr_syscore = { + .ops = &tegra_ictlr_syscore_ops, +}; + static void tegra_ictlr_syscore_init(void) { - register_syscore_ops(&tegra_ictlr_syscore_ops); + register_syscore(&tegra_ictlr_syscore); } #else #define tegra_set_wake NULL @@ -330,9 +334,8 @@ static int __init tegra_ictlr_init(struct device_node *node, node, num_ictlrs, soc->num_ictlrs); - domain = irq_domain_add_hierarchy(parent_domain, 0, num_ictlrs * 32, - node, &tegra_ictlr_domain_ops, - lic); + domain = irq_domain_create_hierarchy(parent_domain, 0, num_ictlrs * 32, + of_fwnode_handle(node), &tegra_ictlr_domain_ops, lic); if (!domain) { pr_err("%pOF: failed to allocated domain\n", node); err = -ENOMEM; diff --git a/drivers/irqchip/irq-ti-sci-inta.c b/drivers/irqchip/irq-ti-sci-inta.c index b83f5cbab123..01963d36cfaf 100644 --- a/drivers/irqchip/irq-ti-sci-inta.c +++ b/drivers/irqchip/irq-ti-sci-inta.c @@ -233,7 +233,7 @@ static struct ti_sci_inta_vint_desc *ti_sci_inta_alloc_parent_irq(struct irq_dom INIT_LIST_HEAD(&vint_desc->list); parent_node = of_irq_find_parent(dev_of_node(&inta->pdev->dev)); - parent_fwspec.fwnode = of_node_to_fwnode(parent_node); + parent_fwspec.fwnode = of_fwnode_handle(parent_node); if (of_device_is_compatible(parent_node, "arm,gic-v3")) { /* Parent is GIC */ @@ -701,15 +701,14 @@ static int ti_sci_inta_irq_domain_probe(struct platform_device *pdev) if (ret) return ret; - domain = irq_domain_add_linear(dev_of_node(dev), - ti_sci_get_num_resources(inta->vint), - &ti_sci_inta_irq_domain_ops, inta); + domain = irq_domain_create_linear(dev_fwnode(dev), ti_sci_get_num_resources(inta->vint), + &ti_sci_inta_irq_domain_ops, inta); if (!domain) { dev_err(dev, "Failed to allocate IRQ domain\n"); return -ENOMEM; } - msi_domain = ti_sci_inta_msi_create_irq_domain(of_node_to_fwnode(node), + msi_domain = ti_sci_inta_msi_create_irq_domain(of_fwnode_handle(node), &ti_sci_inta_msi_domain_info, domain); if (!msi_domain) { @@ -743,3 +742,4 @@ module_platform_driver(ti_sci_inta_irq_domain_driver); MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ti.com>"); MODULE_DESCRIPTION("K3 Interrupt Aggregator driver over TI SCI protocol"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-ti-sci-intr.c b/drivers/irqchip/irq-ti-sci-intr.c index c027cd9e4a69..354613e74ad0 100644 --- a/drivers/irqchip/irq-ti-sci-intr.c +++ b/drivers/irqchip/irq-ti-sci-intr.c @@ -149,7 +149,7 @@ static int ti_sci_intr_alloc_parent_irq(struct irq_domain *domain, goto err_irqs; parent_node = of_irq_find_parent(dev_of_node(intr->dev)); - fwspec.fwnode = of_node_to_fwnode(parent_node); + fwspec.fwnode = of_fwnode_handle(parent_node); if (of_device_is_compatible(parent_node, "arm,gic-v3")) { /* Parent is GIC */ @@ -274,8 +274,8 @@ static int ti_sci_intr_irq_domain_probe(struct platform_device *pdev) return PTR_ERR(intr->out_irqs); } - domain = irq_domain_add_hierarchy(parent_domain, 0, 0, dev_of_node(dev), - &ti_sci_intr_irq_domain_ops, intr); + domain = irq_domain_create_hierarchy(parent_domain, 0, 0, dev_fwnode(dev), + &ti_sci_intr_irq_domain_ops, intr); if (!domain) { dev_err(dev, "Failed to allocate IRQ domain\n"); return -ENOMEM; @@ -303,3 +303,4 @@ module_platform_driver(ti_sci_intr_irq_domain_driver); MODULE_AUTHOR("Lokesh Vutla <lokeshvutla@ticom>"); MODULE_DESCRIPTION("K3 Interrupt Router driver over TI SCI protocol"); +MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-ts4800.c b/drivers/irqchip/irq-ts4800.c index b2d61d4f6fe6..2e4013c6834d 100644 --- a/drivers/irqchip/irq-ts4800.c +++ b/drivers/irqchip/irq-ts4800.c @@ -52,7 +52,7 @@ static void ts4800_irq_print_chip(struct irq_data *d, struct seq_file *p) { struct ts4800_irq_data *data = irq_data_get_irq_chip_data(d); - seq_printf(p, "%s", dev_name(&data->pdev->dev)); + seq_puts(p, dev_name(&data->pdev->dev)); } static const struct irq_chip ts4800_chip = { @@ -125,7 +125,7 @@ static int ts4800_ic_probe(struct platform_device *pdev) return -EINVAL; } - data->domain = irq_domain_add_linear(node, 8, &ts4800_ic_ops, data); + data->domain = irq_domain_create_linear(dev_fwnode(&pdev->dev), 8, &ts4800_ic_ops, data); if (!data->domain) { dev_err(&pdev->dev, "cannot add IRQ domain\n"); return -ENOMEM; @@ -139,13 +139,11 @@ static int ts4800_ic_probe(struct platform_device *pdev) return 0; } -static int ts4800_ic_remove(struct platform_device *pdev) +static void ts4800_ic_remove(struct platform_device *pdev) { struct ts4800_irq_data *data = platform_get_drvdata(pdev); irq_domain_remove(data->domain); - - return 0; } static const struct of_device_id ts4800_ic_of_match[] = { @@ -155,15 +153,15 @@ static const struct of_device_id ts4800_ic_of_match[] = { MODULE_DEVICE_TABLE(of, ts4800_ic_of_match); static struct platform_driver ts4800_ic_driver = { - .probe = ts4800_ic_probe, - .remove = ts4800_ic_remove, + .probe = ts4800_ic_probe, + .remove = ts4800_ic_remove, .driver = { - .name = "ts4800-irqc", - .of_match_table = ts4800_ic_of_match, + .name = "ts4800-irqc", + .of_match_table = ts4800_ic_of_match, }, }; module_platform_driver(ts4800_ic_driver); MODULE_AUTHOR("Damien Riegel <damien.riegel@savoirfairelinux.com>"); +MODULE_DESCRIPTION("Multiplexed-IRQs driver for TS-4800's FPGA"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:ts4800_irqc"); diff --git a/drivers/irqchip/irq-uniphier-aidet.c b/drivers/irqchip/irq-uniphier-aidet.c index 601f9343d5b3..6005c2d28dd9 100644 --- a/drivers/irqchip/irq-uniphier-aidet.c +++ b/drivers/irqchip/irq-uniphier-aidet.c @@ -188,7 +188,7 @@ static int uniphier_aidet_probe(struct platform_device *pdev) priv->domain = irq_domain_create_hierarchy( parent_domain, 0, UNIPHIER_AIDET_NR_IRQS, - of_node_to_fwnode(dev->of_node), + of_fwnode_handle(dev->of_node), &uniphier_aidet_domain_ops, priv); if (!priv->domain) return -ENOMEM; diff --git a/drivers/irqchip/irq-versatile-fpga.c b/drivers/irqchip/irq-versatile-fpga.c index 5018a06060e6..034ce6afe170 100644 --- a/drivers/irqchip/irq-versatile-fpga.c +++ b/drivers/irqchip/irq-versatile-fpga.c @@ -69,7 +69,7 @@ static void fpga_irq_print_chip(struct irq_data *d, struct seq_file *p) { struct fpga_irq_data *f = irq_data_get_irq_chip_data(d); - seq_printf(p, irq_domain_get_of_node(f->domain)->name); + seq_puts(p, irq_domain_get_of_node(f->domain)->name); } static const struct irq_chip fpga_chip = { @@ -128,7 +128,7 @@ static int handle_one_fpga(struct fpga_irq_data *f, struct pt_regs *regs) * Keep iterating over all registered FPGA IRQ controllers until there are * no pending interrupts. */ -static asmlinkage void __exception_irq_entry fpga_handle_irq(struct pt_regs *regs) +static void __exception_irq_entry fpga_handle_irq(struct pt_regs *regs) { int i, handled; @@ -176,8 +176,8 @@ static void __init fpga_irq_init(void __iomem *base, int parent_irq, f); } - f->domain = irq_domain_add_linear(node, fls(valid), - &fpga_irqdomain_ops, f); + f->domain = irq_domain_create_linear(of_fwnode_handle(node), fls(valid), + &fpga_irqdomain_ops, f); /* This will allocate all valid descriptors in the linear case */ for (i = 0; i < fls(valid); i++) diff --git a/drivers/irqchip/irq-vf610-mscm-ir.c b/drivers/irqchip/irq-vf610-mscm-ir.c index 2b9a8ba58e26..5d9c7503aa7f 100644 --- a/drivers/irqchip/irq-vf610-mscm-ir.c +++ b/drivers/irqchip/irq-vf610-mscm-ir.c @@ -209,9 +209,9 @@ static int __init vf610_mscm_ir_of_init(struct device_node *node, regmap_read(mscm_cp_regmap, MSCM_CPxNUM, &cpuid); mscm_ir_data->cpu_mask = 0x1 << cpuid; - domain = irq_domain_add_hierarchy(domain_parent, 0, - MSCM_IRSPRC_NUM, node, - &mscm_irq_domain_ops, mscm_ir_data); + domain = irq_domain_create_hierarchy(domain_parent, 0, MSCM_IRSPRC_NUM, + of_fwnode_handle(node), &mscm_irq_domain_ops, + mscm_ir_data); if (!domain) { ret = -ENOMEM; goto out_unmap; diff --git a/drivers/irqchip/irq-vic.c b/drivers/irqchip/irq-vic.c index 9e3d5561e04e..e38104c5064e 100644 --- a/drivers/irqchip/irq-vic.c +++ b/drivers/irqchip/irq-vic.c @@ -47,9 +47,8 @@ /** * struct vic_device - VIC PM device - * @parent_irq: The parent IRQ number of the VIC if cascaded, or 0. - * @irq: The IRQ number for the base of the VIC. * @base: The register base for the VIC. + * @irq: The IRQ number for the base of the VIC. * @valid_sources: A bitmask of valid interrupts * @resume_sources: A bitmask of interrupts for resume. * @resume_irqs: The IRQs enabled for resume. @@ -121,7 +120,7 @@ static void resume_one_vic(struct vic_device *vic) writel(~vic->soft_int, base + VIC_INT_SOFT_CLEAR); } -static void vic_resume(void) +static void vic_resume(void *data) { int id; @@ -147,7 +146,7 @@ static void suspend_one_vic(struct vic_device *vic) writel(~vic->resume_irqs, base + VIC_INT_ENABLE_CLEAR); } -static int vic_suspend(void) +static int vic_suspend(void *data) { int id; @@ -157,11 +156,15 @@ static int vic_suspend(void) return 0; } -static struct syscore_ops vic_syscore_ops = { +static const struct syscore_ops vic_syscore_ops = { .suspend = vic_suspend, .resume = vic_resume, }; +static struct syscore vic_syscore = { + .ops = &vic_syscore_ops, +}; + /** * vic_pm_init - initcall to register VIC pm * @@ -172,7 +175,7 @@ static struct syscore_ops vic_syscore_ops = { static int __init vic_pm_init(void) { if (vic_id > 0) - register_syscore_ops(&vic_syscore_ops); + register_syscore(&vic_syscore); return 0; } @@ -290,8 +293,9 @@ static void __init vic_register(void __iomem *base, unsigned int parent_irq, vic_handle_irq_cascaded, v); } - v->domain = irq_domain_add_simple(node, fls(valid_sources), irq, - &vic_irqdomain_ops, v); + v->domain = irq_domain_create_simple(of_fwnode_handle(node), + fls(valid_sources), irq, + &vic_irqdomain_ops, v); /* create an IRQ mapping for each valid IRQ */ for (i = 0; i < fls(valid_sources); i++) if (valid_sources & (1 << i)) diff --git a/drivers/irqchip/irq-vt8500.c b/drivers/irqchip/irq-vt8500.c index e17dd3a8c2d5..3b742590aec8 100644 --- a/drivers/irqchip/irq-vt8500.c +++ b/drivers/irqchip/irq-vt8500.c @@ -15,6 +15,7 @@ #include <linux/io.h> #include <linux/irq.h> #include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> #include <linux/interrupt.h> #include <linux/bitops.h> @@ -63,29 +64,28 @@ struct vt8500_irq_data { struct irq_domain *domain; /* Domain for this controller */ }; -/* Global variable for accessing io-mem addresses */ -static struct vt8500_irq_data intc[VT8500_INTC_MAX]; -static u32 active_cnt = 0; +/* Primary interrupt controller data */ +static struct vt8500_irq_data *primary_intc; -static void vt8500_irq_mask(struct irq_data *d) +static void vt8500_irq_ack(struct irq_data *d) { struct vt8500_irq_data *priv = d->domain->host_data; void __iomem *base = priv->base; void __iomem *stat_reg = base + VT8500_ICIS + (d->hwirq < 32 ? 0 : 4); - u8 edge, dctr; - u32 status; + u32 status = (1 << (d->hwirq & 0x1f)); - edge = readb(base + VT8500_ICDC + d->hwirq) & VT8500_EDGE; - if (edge) { - status = readl(stat_reg); + writel(status, stat_reg); +} - status |= (1 << (d->hwirq & 0x1f)); - writel(status, stat_reg); - } else { - dctr = readb(base + VT8500_ICDC + d->hwirq); - dctr &= ~VT8500_INT_ENABLE; - writeb(dctr, base + VT8500_ICDC + d->hwirq); - } +static void vt8500_irq_mask(struct irq_data *d) +{ + struct vt8500_irq_data *priv = d->domain->host_data; + void __iomem *base = priv->base; + u8 dctr; + + dctr = readb(base + VT8500_ICDC + d->hwirq); + dctr &= ~VT8500_INT_ENABLE; + writeb(dctr, base + VT8500_ICDC + d->hwirq); } static void vt8500_irq_unmask(struct irq_data *d) @@ -130,11 +130,11 @@ static int vt8500_irq_set_type(struct irq_data *d, unsigned int flow_type) } static struct irq_chip vt8500_irq_chip = { - .name = "vt8500", - .irq_ack = vt8500_irq_mask, - .irq_mask = vt8500_irq_mask, - .irq_unmask = vt8500_irq_unmask, - .irq_set_type = vt8500_irq_set_type, + .name = "vt8500", + .irq_ack = vt8500_irq_ack, + .irq_mask = vt8500_irq_mask, + .irq_unmask = vt8500_irq_unmask, + .irq_set_type = vt8500_irq_set_type, }; static void __init vt8500_init_irq_hw(void __iomem *base) @@ -163,82 +163,89 @@ static const struct irq_domain_ops vt8500_irq_domain_ops = { .xlate = irq_domain_xlate_onecell, }; +static inline void vt8500_handle_irq_common(struct vt8500_irq_data *intc) +{ + unsigned long irqnr = readl_relaxed(intc->base) & 0x3F; + unsigned long stat; + + /* + * Highest Priority register default = 63, so check that this + * is a real interrupt by checking the status register + */ + if (irqnr == 63) { + stat = readl_relaxed(intc->base + VT8500_ICIS + 4); + if (!(stat & BIT(31))) + return; + } + + generic_handle_domain_irq(intc->domain, irqnr); +} + static void __exception_irq_entry vt8500_handle_irq(struct pt_regs *regs) { - u32 stat, i; - int irqnr; - void __iomem *base; - - /* Loop through each active controller */ - for (i=0; i<active_cnt; i++) { - base = intc[i].base; - irqnr = readl_relaxed(base) & 0x3F; - /* - Highest Priority register default = 63, so check that this - is a real interrupt by checking the status register - */ - if (irqnr == 63) { - stat = readl_relaxed(base + VT8500_ICIS + 4); - if (!(stat & BIT(31))) - continue; - } + vt8500_handle_irq_common(primary_intc); +} - generic_handle_domain_irq(intc[i].domain, irqnr); - } +static void vt8500_handle_irq_chained(struct irq_desc *desc) +{ + struct irq_domain *d = irq_desc_get_handler_data(desc); + struct irq_chip *chip = irq_desc_get_chip(desc); + struct vt8500_irq_data *intc = d->host_data; + + chained_irq_enter(chip, desc); + vt8500_handle_irq_common(intc); + chained_irq_exit(chip, desc); } static int __init vt8500_irq_init(struct device_node *node, struct device_node *parent) { - int irq, i; - struct device_node *np = node; + struct vt8500_irq_data *intc; + int irq, i, ret = 0; - if (active_cnt == VT8500_INTC_MAX) { - pr_err("%s: Interrupt controllers > VT8500_INTC_MAX\n", - __func__); - goto out; - } + intc = kzalloc(sizeof(*intc), GFP_KERNEL); + if (!intc) + return -ENOMEM; - intc[active_cnt].base = of_iomap(np, 0); - intc[active_cnt].domain = irq_domain_add_linear(node, 64, - &vt8500_irq_domain_ops, &intc[active_cnt]); - - if (!intc[active_cnt].base) { + intc->base = of_iomap(node, 0); + if (!intc->base) { pr_err("%s: Unable to map IO memory\n", __func__); - goto out; + ret = -ENOMEM; + goto err_free; } - if (!intc[active_cnt].domain) { + intc->domain = irq_domain_create_linear(of_fwnode_handle(node), 64, + &vt8500_irq_domain_ops, intc); + if (!intc->domain) { pr_err("%s: Unable to add irq domain!\n", __func__); - goto out; + ret = -ENOMEM; + goto err_unmap; } - set_handle_irq(vt8500_handle_irq); - - vt8500_init_irq_hw(intc[active_cnt].base); + vt8500_init_irq_hw(intc->base); pr_info("vt8500-irq: Added interrupt controller\n"); - active_cnt++; - - /* check if this is a slaved controller */ - if (of_irq_count(np) != 0) { - /* check that we have the correct number of interrupts */ - if (of_irq_count(np) != 8) { - pr_err("%s: Incorrect IRQ map for slaved controller\n", - __func__); - return -EINVAL; - } - - for (i = 0; i < 8; i++) { - irq = irq_of_parse_and_map(np, i); - enable_irq(irq); + /* check if this is a chained controller */ + if (of_irq_count(node) != 0) { + for (i = 0; i < of_irq_count(node); i++) { + irq = irq_of_parse_and_map(node, i); + irq_set_chained_handler_and_data(irq, vt8500_handle_irq_chained, + intc); } pr_info("vt8500-irq: Enabled slave->parent interrupts\n"); + } else { + primary_intc = intc; + set_handle_irq(vt8500_handle_irq); } -out: return 0; + +err_unmap: + iounmap(intc->base); +err_free: + kfree(intc); + return ret; } IRQCHIP_DECLARE(vt8500_irq, "via,vt8500-intc", vt8500_irq_init); diff --git a/drivers/irqchip/irq-wpcm450-aic.c b/drivers/irqchip/irq-wpcm450-aic.c index 91df62a64cd9..a8ed4894d29e 100644 --- a/drivers/irqchip/irq-wpcm450-aic.c +++ b/drivers/irqchip/irq-wpcm450-aic.c @@ -154,7 +154,7 @@ static int __init wpcm450_aic_of_init(struct device_node *node, set_handle_irq(wpcm450_aic_handle_irq); - aic->domain = irq_domain_add_linear(node, AIC_NUM_IRQS, &wpcm450_aic_ops, aic); + aic->domain = irq_domain_create_linear(of_fwnode_handle(node), AIC_NUM_IRQS, &wpcm450_aic_ops, aic); return 0; } diff --git a/drivers/irqchip/irq-xilinx-intc.c b/drivers/irqchip/irq-xilinx-intc.c index 238d3d344949..92dcb9fdcb25 100644 --- a/drivers/irqchip/irq-xilinx-intc.c +++ b/drivers/irqchip/irq-xilinx-intc.c @@ -189,7 +189,7 @@ static int __init xilinx_intc_of_init(struct device_node *intc, irqc->intr_mask = 0; } - if (irqc->intr_mask >> irqc->nr_irq) + if ((u64)irqc->intr_mask >> irqc->nr_irq) pr_warn("irq-xilinx: mismatch in kind-of-intr param\n"); pr_info("irq-xilinx: %pOF: num_irq=%d, edge=0x%x\n", @@ -212,8 +212,8 @@ static int __init xilinx_intc_of_init(struct device_node *intc, xintc_write(irqc, MER, MER_HIE | MER_ME); } - irqc->root_domain = irq_domain_add_linear(intc, irqc->nr_irq, - &xintc_irq_domain_ops, irqc); + irqc->root_domain = irq_domain_create_linear(of_fwnode_handle(intc), irqc->nr_irq, + &xintc_irq_domain_ops, irqc); if (!irqc->root_domain) { pr_err("irq-xilinx: Unable to create IRQ domain\n"); ret = -EINVAL; @@ -233,7 +233,7 @@ static int __init xilinx_intc_of_init(struct device_node *intc, } } else { primary_intc = irqc; - irq_set_default_host(primary_intc->root_domain); + irq_set_default_domain(primary_intc->root_domain); set_handle_irq(xil_intc_handle_irq); } diff --git a/drivers/irqchip/irq-xtensa-mx.c b/drivers/irqchip/irq-xtensa-mx.c index 8c581c985aa7..9fdacbd89a63 100644 --- a/drivers/irqchip/irq-xtensa-mx.c +++ b/drivers/irqchip/irq-xtensa-mx.c @@ -12,6 +12,7 @@ #include <linux/irqdomain.h> #include <linux/irq.h> #include <linux/irqchip.h> +#include <linux/irqchip/xtensa-mx.h> #include <linux/of.h> #include <asm/mxregs.h> @@ -155,7 +156,7 @@ static void __init xtensa_mx_init_common(struct irq_domain *root_domain) { unsigned int i; - irq_set_default_host(root_domain); + irq_set_default_domain(root_domain); secondary_init_irq(); /* Initialize default IRQ routing to CPU 0 */ @@ -166,8 +167,7 @@ static void __init xtensa_mx_init_common(struct irq_domain *root_domain) int __init xtensa_mx_init_legacy(struct device_node *interrupt_parent) { struct irq_domain *root_domain = - irq_domain_add_legacy(NULL, NR_IRQS - 1, 1, 0, - &xtensa_mx_irq_domain_ops, + irq_domain_create_legacy(NULL, NR_IRQS - 1, 1, 0, &xtensa_mx_irq_domain_ops, &xtensa_mx_irq_chip); xtensa_mx_init_common(root_domain); return 0; @@ -177,7 +177,7 @@ static int __init xtensa_mx_init(struct device_node *np, struct device_node *interrupt_parent) { struct irq_domain *root_domain = - irq_domain_add_linear(np, NR_IRQS, &xtensa_mx_irq_domain_ops, + irq_domain_create_linear(of_fwnode_handle(np), NR_IRQS, &xtensa_mx_irq_domain_ops, &xtensa_mx_irq_chip); xtensa_mx_init_common(root_domain); return 0; diff --git a/drivers/irqchip/irq-xtensa-pic.c b/drivers/irqchip/irq-xtensa-pic.c index 0c18d1f1e264..44e7be051a2e 100644 --- a/drivers/irqchip/irq-xtensa-pic.c +++ b/drivers/irqchip/irq-xtensa-pic.c @@ -12,6 +12,7 @@ * Kevin Chea */ +#include <linux/bits.h> #include <linux/interrupt.h> #include <linux/irqdomain.h> #include <linux/irq.h> @@ -19,8 +20,6 @@ #include <linux/irqchip/xtensa-pic.h> #include <linux/of.h> -unsigned int cached_irq_mask; - /* * Device Tree IRQ specifier translation function which works with one or * two cell bindings. First cell value maps directly to the hwirq number. @@ -44,34 +43,30 @@ static const struct irq_domain_ops xtensa_irq_domain_ops = { static void xtensa_irq_mask(struct irq_data *d) { - cached_irq_mask &= ~(1 << d->hwirq); - xtensa_set_sr(cached_irq_mask, intenable); -} + u32 irq_mask; -static void xtensa_irq_unmask(struct irq_data *d) -{ - cached_irq_mask |= 1 << d->hwirq; - xtensa_set_sr(cached_irq_mask, intenable); + irq_mask = xtensa_get_sr(intenable); + irq_mask &= ~BIT(d->hwirq); + xtensa_set_sr(irq_mask, intenable); } -static void xtensa_irq_enable(struct irq_data *d) +static void xtensa_irq_unmask(struct irq_data *d) { - xtensa_irq_unmask(d); -} + u32 irq_mask; -static void xtensa_irq_disable(struct irq_data *d) -{ - xtensa_irq_mask(d); + irq_mask = xtensa_get_sr(intenable); + irq_mask |= BIT(d->hwirq); + xtensa_set_sr(irq_mask, intenable); } static void xtensa_irq_ack(struct irq_data *d) { - xtensa_set_sr(1 << d->hwirq, intclear); + xtensa_set_sr(BIT(d->hwirq), intclear); } static int xtensa_irq_retrigger(struct irq_data *d) { - unsigned int mask = 1u << d->hwirq; + unsigned int mask = BIT(d->hwirq); if (WARN_ON(mask & ~XCHAL_INTTYPE_MASK_SOFTWARE)) return 0; @@ -81,8 +76,6 @@ static int xtensa_irq_retrigger(struct irq_data *d) static struct irq_chip xtensa_irq_chip = { .name = "xtensa", - .irq_enable = xtensa_irq_enable, - .irq_disable = xtensa_irq_disable, .irq_mask = xtensa_irq_mask, .irq_unmask = xtensa_irq_unmask, .irq_ack = xtensa_irq_ack, @@ -92,9 +85,9 @@ static struct irq_chip xtensa_irq_chip = { int __init xtensa_pic_init_legacy(struct device_node *interrupt_parent) { struct irq_domain *root_domain = - irq_domain_add_legacy(NULL, NR_IRQS - 1, 1, 0, + irq_domain_create_legacy(NULL, NR_IRQS - 1, 1, 0, &xtensa_irq_domain_ops, &xtensa_irq_chip); - irq_set_default_host(root_domain); + irq_set_default_domain(root_domain); return 0; } @@ -102,9 +95,9 @@ static int __init xtensa_pic_init(struct device_node *np, struct device_node *interrupt_parent) { struct irq_domain *root_domain = - irq_domain_add_linear(np, NR_IRQS, &xtensa_irq_domain_ops, + irq_domain_create_linear(of_fwnode_handle(np), NR_IRQS, &xtensa_irq_domain_ops, &xtensa_irq_chip); - irq_set_default_host(root_domain); + irq_set_default_domain(root_domain); return 0; } IRQCHIP_DECLARE(xtensa_irq_chip, "cdns,xtensa-pic", xtensa_pic_init); diff --git a/drivers/irqchip/irq-zevio.c b/drivers/irqchip/irq-zevio.c index 7a72620fc478..22d46c246594 100644 --- a/drivers/irqchip/irq-zevio.c +++ b/drivers/irqchip/irq-zevio.c @@ -92,8 +92,8 @@ static int __init zevio_of_init(struct device_node *node, zevio_init_irq_base(zevio_irq_io + IO_IRQ_BASE); zevio_init_irq_base(zevio_irq_io + IO_FIQ_BASE); - zevio_irq_domain = irq_domain_add_linear(node, MAX_INTRS, - &irq_generic_chip_ops, NULL); + zevio_irq_domain = irq_domain_create_linear(of_fwnode_handle(node), MAX_INTRS, + &irq_generic_chip_ops, NULL); BUG_ON(!zevio_irq_domain); ret = irq_alloc_domain_generic_chips(zevio_irq_domain, MAX_INTRS, 1, diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 1eeb0d0156ce..689c8e448901 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -35,13 +35,11 @@ void __init irqchip_init(void) int platform_irqchip_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; - struct device_node *par_np = of_irq_find_parent(np); - of_irq_init_cb_t irq_init_cb = of_device_get_match_data(&pdev->dev); + struct device_node *par_np __free(device_node) = of_irq_find_parent(np); + platform_irq_probe_t irq_probe = of_device_get_match_data(&pdev->dev); - if (!irq_init_cb) { - of_node_put(par_np); + if (!irq_probe) return -EINVAL; - } if (par_np == np) par_np = NULL; @@ -54,11 +52,9 @@ int platform_irqchip_probe(struct platform_device *pdev) * interrupt controller. The actual initialization callback of this * interrupt controller can check for specific domains as necessary. */ - if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) { - of_node_put(par_np); + if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) return -EPROBE_DEFER; - } - return irq_init_cb(np, par_np); + return irq_probe(pdev, par_np); } EXPORT_SYMBOL_GPL(platform_irqchip_probe); diff --git a/drivers/irqchip/qcom-irq-combiner.c b/drivers/irqchip/qcom-irq-combiner.c index 18e696dc7f4d..09819007d08e 100644 --- a/drivers/irqchip/qcom-irq-combiner.c +++ b/drivers/irqchip/qcom-irq-combiner.c @@ -222,7 +222,7 @@ static int get_registers(struct platform_device *pdev, struct combiner *comb) return 0; } -static int __init combiner_probe(struct platform_device *pdev) +static int combiner_probe(struct platform_device *pdev) { struct combiner *combiner; int nregs; @@ -266,11 +266,11 @@ static const struct acpi_device_id qcom_irq_combiner_ids[] = { { } }; -static struct platform_driver qcom_irq_combiner_probe = { +static struct platform_driver qcom_irq_combiner_driver = { .driver = { .name = "qcom-irq-combiner", .acpi_match_table = ACPI_PTR(qcom_irq_combiner_ids), }, .probe = combiner_probe, }; -builtin_platform_driver(qcom_irq_combiner_probe); +builtin_platform_driver(qcom_irq_combiner_driver); diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index a32c0d28d038..518f7f0f3dab 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -21,10 +21,23 @@ #include <linux/types.h> #define PDC_MAX_GPIO_IRQS 256 +#define PDC_DRV_OFFSET 0x10000 +/* Valid only on HW version < 3.2 */ #define IRQ_ENABLE_BANK 0x10 +#define IRQ_ENABLE_BANK_MAX (IRQ_ENABLE_BANK + BITS_TO_BYTES(PDC_MAX_GPIO_IRQS)) #define IRQ_i_CFG 0x110 +/* Valid only on HW version >= 3.2 */ +#define IRQ_i_CFG_IRQ_ENABLE 3 + +#define IRQ_i_CFG_TYPE_MASK GENMASK(2, 0) + +#define PDC_VERSION_REG 0x1000 + +/* Notable PDC versions */ +#define PDC_VERSION_3_2 0x30200 + struct pdc_pin_region { u32 pin_base; u32 parent_base; @@ -35,12 +48,20 @@ struct pdc_pin_region { static DEFINE_RAW_SPINLOCK(pdc_lock); static void __iomem *pdc_base; +static void __iomem *pdc_prev_base; static struct pdc_pin_region *pdc_region; static int pdc_region_cnt; +static unsigned int pdc_version; +static bool pdc_x1e_quirk; + +static void pdc_base_reg_write(void __iomem *base, int reg, u32 i, u32 val) +{ + writel_relaxed(val, base + reg + i * sizeof(u32)); +} static void pdc_reg_write(int reg, u32 i, u32 val) { - writel_relaxed(val, pdc_base + reg + i * sizeof(u32)); + pdc_base_reg_write(pdc_base, reg, i, val); } static u32 pdc_reg_read(int reg, u32 i) @@ -48,20 +69,64 @@ static u32 pdc_reg_read(int reg, u32 i) return readl_relaxed(pdc_base + reg + i * sizeof(u32)); } -static void pdc_enable_intr(struct irq_data *d, bool on) +static void pdc_x1e_irq_enable_write(u32 bank, u32 enable) +{ + void __iomem *base; + + /* Remap the write access to work around a hardware bug on X1E */ + switch (bank) { + case 0 ... 1: + /* Use previous DRV (client) region and shift to bank 3-4 */ + base = pdc_prev_base; + bank += 3; + break; + case 2 ... 4: + /* Use our own region and shift to bank 0-2 */ + base = pdc_base; + bank -= 2; + break; + case 5: + /* No fixup required for bank 5 */ + base = pdc_base; + break; + default: + WARN_ON(1); + return; + } + + pdc_base_reg_write(base, IRQ_ENABLE_BANK, bank, enable); +} + +static void __pdc_enable_intr(int pin_out, bool on) { - int pin_out = d->hwirq; unsigned long enable; - unsigned long flags; - u32 index, mask; - index = pin_out / 32; - mask = pin_out % 32; + if (pdc_version < PDC_VERSION_3_2) { + u32 index, mask; + + index = pin_out / 32; + mask = pin_out % 32; + + enable = pdc_reg_read(IRQ_ENABLE_BANK, index); + __assign_bit(mask, &enable, on); + + if (pdc_x1e_quirk) + pdc_x1e_irq_enable_write(index, enable); + else + pdc_reg_write(IRQ_ENABLE_BANK, index, enable); + } else { + enable = pdc_reg_read(IRQ_i_CFG, pin_out); + __assign_bit(IRQ_i_CFG_IRQ_ENABLE, &enable, on); + pdc_reg_write(IRQ_i_CFG, pin_out, enable); + } +} + +static void pdc_enable_intr(struct irq_data *d, bool on) +{ + unsigned long flags; raw_spin_lock_irqsave(&pdc_lock, flags); - enable = pdc_reg_read(IRQ_ENABLE_BANK, index); - __assign_bit(mask, &enable, on); - pdc_reg_write(IRQ_ENABLE_BANK, index, enable); + __pdc_enable_intr(d->hwirq, on); raw_spin_unlock_irqrestore(&pdc_lock, flags); } @@ -142,6 +207,7 @@ static int qcom_pdc_gic_set_type(struct irq_data *d, unsigned int type) } old_pdc_type = pdc_reg_read(IRQ_i_CFG, d->hwirq); + pdc_type |= (old_pdc_type & ~IRQ_i_CFG_TYPE_MASK); pdc_reg_write(IRQ_i_CFG, d->hwirq, pdc_type); ret = irq_chip_set_type_parent(d, type); @@ -246,7 +312,6 @@ static const struct irq_domain_ops qcom_pdc_ops = { static int pdc_setup_pin_mapping(struct device_node *np) { int ret, n, i; - u32 irq_index, reg_index, val; n = of_property_count_elems_of_size(np, "qcom,pdc-ranges", sizeof(u32)); if (n <= 0 || n % 3) @@ -276,29 +341,58 @@ static int pdc_setup_pin_mapping(struct device_node *np) if (ret) return ret; - for (i = 0; i < pdc_region[n].cnt; i++) { - reg_index = (i + pdc_region[n].pin_base) >> 5; - irq_index = (i + pdc_region[n].pin_base) & 0x1f; - val = pdc_reg_read(IRQ_ENABLE_BANK, reg_index); - val &= ~BIT(irq_index); - pdc_reg_write(IRQ_ENABLE_BANK, reg_index, val); - } + for (i = 0; i < pdc_region[n].cnt; i++) + __pdc_enable_intr(i + pdc_region[n].pin_base, 0); } return 0; } -static int qcom_pdc_init(struct device_node *node, struct device_node *parent) +#define QCOM_PDC_SIZE 0x30000 + +static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *parent) { struct irq_domain *parent_domain, *pdc_domain; + struct device_node *node = pdev->dev.of_node; + resource_size_t res_size; + struct resource res; int ret; - pdc_base = of_iomap(node, 0); + /* compat with old sm8150 DT which had very small region for PDC */ + if (of_address_to_resource(node, 0, &res)) + return -EINVAL; + + res_size = max_t(resource_size_t, resource_size(&res), QCOM_PDC_SIZE); + if (res_size > resource_size(&res)) + pr_warn("%pOF: invalid reg size, please fix DT\n", node); + + /* + * PDC has multiple DRV regions, each one provides the same set of + * registers for a particular client in the system. Due to a hardware + * bug on X1E, some writes to the IRQ_ENABLE_BANK register must be + * issued inside the previous region. This region belongs to + * a different client and is not described in the device tree. Map the + * region with the expected offset to preserve support for old DTs. + */ + if (of_device_is_compatible(node, "qcom,x1e80100-pdc")) { + pdc_prev_base = ioremap(res.start - PDC_DRV_OFFSET, IRQ_ENABLE_BANK_MAX); + if (!pdc_prev_base) { + pr_err("%pOF: unable to map previous PDC DRV region\n", node); + return -ENXIO; + } + + pdc_x1e_quirk = true; + } + + pdc_base = ioremap(res.start, res_size); if (!pdc_base) { pr_err("%pOF: unable to map PDC registers\n", node); - return -ENXIO; + ret = -ENXIO; + goto fail; } + pdc_version = pdc_reg_read(PDC_VERSION_REG, 0); + parent_domain = irq_find_host(parent); if (!parent_domain) { pr_err("%pOF: unable to find PDC's parent domain\n", node); @@ -330,11 +424,12 @@ static int qcom_pdc_init(struct device_node *node, struct device_node *parent) fail: kfree(pdc_region); iounmap(pdc_base); + iounmap(pdc_prev_base); return ret; } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_pdc) -IRQCHIP_MATCH("qcom,pdc", qcom_pdc_init) +IRQCHIP_MATCH("qcom,pdc", qcom_pdc_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_pdc) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Power Domain Controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/spear-shirq.c b/drivers/irqchip/spear-shirq.c index 7c17a6f643ef..576e55569d77 100644 --- a/drivers/irqchip/spear-shirq.c +++ b/drivers/irqchip/spear-shirq.c @@ -239,7 +239,7 @@ static int __init shirq_init(struct spear_shirq **shirq_blocks, int block_nr, goto err_unmap; } - shirq_domain = irq_domain_add_legacy(np, nr_irqs, virq_base, 0, + shirq_domain = irq_domain_create_legacy(of_fwnode_handle(np), nr_irqs, virq_base, 0, &irq_domain_simple_ops, NULL); if (WARN_ON(!shirq_domain)) { pr_warn("%s: irq domain init failed\n", __func__); |