diff options
Diffstat (limited to 'drivers/iommu')
30 files changed, 559 insertions, 334 deletions
diff --git a/drivers/iommu/amd/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index c160a332ce33..e98f20a9bdd8 100644 --- a/drivers/iommu/amd/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h @@ -34,6 +34,7 @@ extern int amd_iommu_reenable(int); extern int amd_iommu_enable_faulting(void); extern int amd_iommu_guest_ir; extern enum io_pgtable_fmt amd_iommu_pgtable; +extern int amd_iommu_gpt_level; /* IOMMUv2 specific functions */ struct iommu_domain; @@ -122,6 +123,14 @@ static inline int get_pci_sbdf_id(struct pci_dev *pdev) return PCI_SEG_DEVID_TO_SBDF(seg, devid); } +static inline void *alloc_pgtable_page(int nid, gfp_t gfp) +{ + struct page *page; + + page = alloc_pages_node(nid, gfp | __GFP_ZERO, 0); + return page ? page_address(page) : NULL; +} + extern bool translation_pre_enabled(struct amd_iommu *iommu); extern bool amd_iommu_is_attach_deferred(struct device *dev); extern int __init add_special_device(u8 type, u8 id, u32 *devid, diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 3d684190b4d5..2ddbda3a4374 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -93,6 +93,8 @@ #define FEATURE_GA (1ULL<<7) #define FEATURE_HE (1ULL<<8) #define FEATURE_PC (1ULL<<9) +#define FEATURE_GATS_SHIFT (12) +#define FEATURE_GATS_MASK (3ULL) #define FEATURE_GAM_VAPIC (1ULL<<21) #define FEATURE_GIOSUP (1ULL<<48) #define FEATURE_EPHSUP (1ULL<<50) @@ -305,6 +307,9 @@ #define PAGE_MODE_6_LEVEL 0x06 #define PAGE_MODE_7_LEVEL 0x07 +#define GUEST_PGTABLE_4_LEVEL 0x00 +#define GUEST_PGTABLE_5_LEVEL 0x01 + #define PM_LEVEL_SHIFT(x) (12 + ((x) * 9)) #define PM_LEVEL_SIZE(x) (((x) < 6) ? \ ((1ULL << PM_LEVEL_SHIFT((x))) - 1): \ @@ -398,6 +403,8 @@ #define DTE_GCR3_SHIFT_B 16 #define DTE_GCR3_SHIFT_C 43 +#define DTE_GPT_LEVEL_SHIFT 54 + #define GCR3_VALID 0x01ULL #define IOMMU_PAGE_MASK (((1ULL << 52) - 1) & ~0xfffULL) @@ -549,6 +556,7 @@ struct protection_domain { spinlock_t lock; /* mostly used to lock the page table*/ u16 id; /* the domain id written to the device table */ int glx; /* Number of levels for GCR3 table */ + int nid; /* Node ID */ u64 *gcr3_tbl; /* Guest CR3 table */ unsigned long flags; /* flags to find out type of domain */ unsigned dev_cnt; /* devices assigned to this domain */ @@ -1001,8 +1009,8 @@ struct amd_ir_data { */ struct irq_cfg *cfg; int ga_vector; - int ga_root_ptr; - int ga_tag; + u64 ga_root_ptr; + u32 ga_tag; }; struct amd_irte_ops { diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index a98202018140..329a406cc37d 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -153,6 +153,8 @@ bool amd_iommu_dump; bool amd_iommu_irq_remap __read_mostly; enum io_pgtable_fmt amd_iommu_pgtable = AMD_IOMMU_V1; +/* Guest page table level */ +int amd_iommu_gpt_level = PAGE_MODE_4_LEVEL; int amd_iommu_guest_ir = AMD_IOMMU_GUEST_IR_VAPIC; static int amd_iommu_xt_mode = IRQ_REMAP_XAPIC_MODE; @@ -306,6 +308,11 @@ static bool check_feature_on_all_iommus(u64 mask) return !!(amd_iommu_efr & mask); } +static inline int check_feature_gpt_level(void) +{ + return ((amd_iommu_efr >> FEATURE_GATS_SHIFT) & FEATURE_GATS_MASK); +} + /* * For IVHD type 0x11/0x40, EFR is also available via IVHD. * Default to IVHD EFR since it is available sooner @@ -2155,8 +2162,10 @@ static void print_iommu_info(void) if (amd_iommu_xt_mode == IRQ_REMAP_X2APIC_MODE) pr_info("X2APIC enabled\n"); } - if (amd_iommu_pgtable == AMD_IOMMU_V2) - pr_info("V2 page table enabled\n"); + if (amd_iommu_pgtable == AMD_IOMMU_V2) { + pr_info("V2 page table enabled (Paging mode : %d level)\n", + amd_iommu_gpt_level); + } } static int __init amd_iommu_init_pci(void) @@ -2383,6 +2392,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) struct irq_domain *domain; struct irq_alloc_info info; int irq, ret; + int node = dev_to_node(&iommu->dev->dev); domain = iommu_get_irqdomain(); if (!domain) @@ -2392,7 +2402,7 @@ static int iommu_setup_intcapxt(struct amd_iommu *iommu) info.type = X86_IRQ_ALLOC_TYPE_AMDVI; info.data = iommu; - irq = irq_domain_alloc_irqs(domain, 1, NUMA_NO_NODE, &info); + irq = irq_domain_alloc_irqs(domain, 1, node, &info); if (irq < 0) { irq_domain_remove(domain); return irq; @@ -3025,6 +3035,11 @@ static int __init early_amd_iommu_init(void) if (ret) goto out; + /* 5 level guest page table */ + if (cpu_feature_enabled(X86_FEATURE_LA57) && + check_feature_gpt_level() == GUEST_PGTABLE_5_LEVEL) + amd_iommu_gpt_level = PAGE_MODE_5_LEVEL; + /* Disable any previously enabled IOMMUs */ if (!is_kdump_kernel() || amd_iommu_disabled) disable_iommus(); @@ -3556,6 +3571,11 @@ __setup("ivrs_acpihid", parse_ivrs_acpihid); bool amd_iommu_v2_supported(void) { + /* CPU page table size should match IOMMU guest page table size */ + if (cpu_feature_enabled(X86_FEATURE_LA57) && + amd_iommu_gpt_level != PAGE_MODE_5_LEVEL) + return false; + /* * Since DTE[Mode]=0 is prohibited on SNP-enabled system * (i.e. EFR[SNPSup]=1), IOMMUv2 page table cannot be used without diff --git a/drivers/iommu/amd/io_pgtable.c b/drivers/iommu/amd/io_pgtable.c index ace0e9b8b913..1b67116882be 100644 --- a/drivers/iommu/amd/io_pgtable.c +++ b/drivers/iommu/amd/io_pgtable.c @@ -156,7 +156,7 @@ static bool increase_address_space(struct protection_domain *domain, bool ret = true; u64 *pte; - pte = (void *)get_zeroed_page(gfp); + pte = alloc_pgtable_page(domain->nid, gfp); if (!pte) return false; @@ -250,7 +250,7 @@ static u64 *alloc_pte(struct protection_domain *domain, if (!IOMMU_PTE_PRESENT(__pte) || pte_level == PAGE_MODE_NONE) { - page = (u64 *)get_zeroed_page(gfp); + page = alloc_pgtable_page(domain->nid, gfp); if (!page) return NULL; diff --git a/drivers/iommu/amd/io_pgtable_v2.c b/drivers/iommu/amd/io_pgtable_v2.c index 8638ddf6fb3b..27c3015947e6 100644 --- a/drivers/iommu/amd/io_pgtable_v2.c +++ b/drivers/iommu/amd/io_pgtable_v2.c @@ -37,8 +37,7 @@ static inline int get_pgtable_level(void) { - /* 5 level page table is not supported */ - return PAGE_MODE_4_LEVEL; + return amd_iommu_gpt_level; } static inline bool is_large_pte(u64 pte) @@ -46,11 +45,6 @@ static inline bool is_large_pte(u64 pte) return (pte & IOMMU_PAGE_PSE); } -static inline void *alloc_pgtable_page(void) -{ - return (void *)get_zeroed_page(GFP_KERNEL); -} - static inline u64 set_pgtable_attr(u64 *page) { u64 prot; @@ -138,8 +132,8 @@ static void free_pgtable(u64 *pt, int level) } /* Allocate page table */ -static u64 *v2_alloc_pte(u64 *pgd, unsigned long iova, - unsigned long pg_size, bool *updated) +static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, + unsigned long pg_size, gfp_t gfp, bool *updated) { u64 *pte, *page; int level, end_level; @@ -162,7 +156,7 @@ static u64 *v2_alloc_pte(u64 *pgd, unsigned long iova, } if (!IOMMU_PTE_PRESENT(__pte)) { - page = alloc_pgtable_page(); + page = alloc_pgtable_page(nid, gfp); if (!page) return NULL; @@ -262,7 +256,8 @@ static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, while (mapped_size < size) { map_size = get_alloc_page_size(pgsize); - pte = v2_alloc_pte(pdom->iop.pgd, iova, map_size, &updated); + pte = v2_alloc_pte(pdom->nid, pdom->iop.pgd, + iova, map_size, gfp, &updated); if (!pte) { ret = -EINVAL; goto out; @@ -383,8 +378,9 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo struct amd_io_pgtable *pgtable = io_pgtable_cfg_to_data(cfg); struct protection_domain *pdom = (struct protection_domain *)cookie; int ret; + int ias = IOMMU_IN_ADDR_BIT_SIZE; - pgtable->pgd = alloc_pgtable_page(); + pgtable->pgd = alloc_pgtable_page(pdom->nid, GFP_ATOMIC); if (!pgtable->pgd) return NULL; @@ -392,12 +388,15 @@ static struct io_pgtable *v2_alloc_pgtable(struct io_pgtable_cfg *cfg, void *coo if (ret) goto err_free_pgd; + if (get_pgtable_level() == PAGE_MODE_5_LEVEL) + ias = 57; + pgtable->iop.ops.map_pages = iommu_v2_map_pages; pgtable->iop.ops.unmap_pages = iommu_v2_unmap_pages; pgtable->iop.ops.iova_to_phys = iommu_v2_iova_to_phys; cfg->pgsize_bitmap = AMD_IOMMU_PGSIZES_V2, - cfg->ias = IOMMU_IN_ADDR_BIT_SIZE, + cfg->ias = ias, cfg->oas = IOMMU_OUT_ADDR_BIT_SIZE, cfg->tlb = &v2_flush_ops; diff --git a/drivers/iommu/amd/iommu.c b/drivers/iommu/amd/iommu.c index 5a505ba5467e..4a314647d1f7 100644 --- a/drivers/iommu/amd/iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -1611,6 +1611,11 @@ static void set_dte_entry(struct amd_iommu *iommu, u16 devid, tmp = DTE_GCR3_VAL_C(gcr3) << DTE_GCR3_SHIFT_C; flags |= tmp; + if (amd_iommu_gpt_level == PAGE_MODE_5_LEVEL) { + dev_table[devid].data[2] |= + ((u64)GUEST_PGTABLE_5_LEVEL << DTE_GPT_LEVEL_SHIFT); + } + if (domain->flags & PD_GIOV_MASK) pte_root |= DTE_FLAG_GIOV; } @@ -1662,14 +1667,14 @@ static void do_attach(struct iommu_dev_data *dev_data, dev_data->domain = domain; list_add(&dev_data->list, &domain->dev_list); + /* Update NUMA Node ID */ + if (domain->nid == NUMA_NO_NODE) + domain->nid = dev_to_node(dev_data->dev); + /* Do reference counting */ domain->dev_iommu[iommu->index] += 1; domain->dev_cnt += 1; - /* Override supported page sizes */ - if (domain->flags & PD_GIOV_MASK) - domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; - /* Update device table */ set_dte_entry(iommu, dev_data->devid, domain, ats, dev_data->iommu_v2); @@ -2048,6 +2053,8 @@ static int protection_domain_init_v2(struct protection_domain *domain) domain->flags |= PD_GIOV_MASK; + domain->domain.pgsize_bitmap = AMD_IOMMU_PGSIZES_V2; + if (domain_enable_v2(domain, 1)) { domain_id_free(domain->id); return -ENOMEM; @@ -2097,6 +2104,8 @@ static struct protection_domain *protection_domain_alloc(unsigned int type) if (type == IOMMU_DOMAIN_IDENTITY) return domain; + domain->nid = NUMA_NO_NODE; + pgtbl_ops = alloc_io_pgtable_ops(pgtable, &domain->iop.pgtbl_cfg, domain); if (!pgtbl_ops) { domain_id_free(domain->id); diff --git a/drivers/iommu/apple-dart.c b/drivers/iommu/apple-dart.c index 06169d36eab8..8af64b57f048 100644 --- a/drivers/iommu/apple-dart.c +++ b/drivers/iommu/apple-dart.c @@ -1150,7 +1150,7 @@ err_clk_disable: return ret; } -static int apple_dart_remove(struct platform_device *pdev) +static void apple_dart_remove(struct platform_device *pdev) { struct apple_dart *dart = platform_get_drvdata(pdev); @@ -1161,8 +1161,6 @@ static int apple_dart_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&dart->iommu); clk_bulk_disable_unprepare(dart->num_clks, dart->clks); - - return 0; } static const struct apple_dart_hw apple_dart_hw_t8103 = { @@ -1296,7 +1294,7 @@ static struct platform_driver apple_dart_driver = { .pm = pm_sleep_ptr(&apple_dart_pm_ops), }, .probe = apple_dart_probe, - .remove = apple_dart_remove, + .remove_new = apple_dart_remove, }; module_platform_driver(apple_dart_driver); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index f2425b0f0cd6..3fd83fb75722 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -152,6 +152,18 @@ static void queue_inc_cons(struct arm_smmu_ll_queue *q) q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons); } +static void queue_sync_cons_ovf(struct arm_smmu_queue *q) +{ + struct arm_smmu_ll_queue *llq = &q->llq; + + if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons))) + return; + + llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | + Q_IDX(llq, llq->cons); + queue_sync_cons_out(q); +} + static int queue_sync_prod_in(struct arm_smmu_queue *q) { u32 prod; @@ -1577,8 +1589,7 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev) } while (!queue_empty(llq)); /* Sync our overflow flag, as we believe we're up to speed */ - llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | - Q_IDX(llq, llq->cons); + queue_sync_cons_ovf(q); return IRQ_HANDLED; } @@ -1636,9 +1647,7 @@ static irqreturn_t arm_smmu_priq_thread(int irq, void *dev) } while (!queue_empty(llq)); /* Sync our overflow flag, as we believe we're up to speed */ - llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) | - Q_IDX(llq, llq->cons); - queue_sync_cons_out(q); + queue_sync_cons_ovf(q); return IRQ_HANDLED; } @@ -2447,6 +2456,13 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) master->domain = smmu_domain; + /* + * The SMMU does not support enabling ATS with bypass. When the STE is + * in bypass (STE.Config[2:0] == 0b100), ATS Translation Requests and + * Translated transactions are denied as though ATS is disabled for the + * stream (STE.EATS == 0b00), causing F_BAD_ATS_TREQ and + * F_TRANSL_FORBIDDEN events (IHI0070Ea 5.2 Stream Table Entry). + */ if (smmu_domain->stage != ARM_SMMU_DOMAIN_BYPASS) master->ats_enabled = arm_smmu_ats_supported(master); @@ -3844,7 +3860,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev) return 0; } -static int arm_smmu_device_remove(struct platform_device *pdev) +static void arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); @@ -3852,8 +3868,6 @@ static int arm_smmu_device_remove(struct platform_device *pdev) iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_disable(smmu); iopf_queue_free(smmu->evtq.iopf); - - return 0; } static void arm_smmu_device_shutdown(struct platform_device *pdev) @@ -3882,7 +3896,7 @@ static struct platform_driver arm_smmu_driver = { .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, - .remove = arm_smmu_device_remove, + .remove_new = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; module_driver(arm_smmu_driver, platform_driver_register, diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index d1b296b95c86..ae09c627bc84 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -268,13 +268,27 @@ static int qcom_smmu_init_context(struct arm_smmu_domain *smmu_domain, static int qcom_smmu_cfg_probe(struct arm_smmu_device *smmu) { - unsigned int last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); struct qcom_smmu *qsmmu = to_qcom_smmu(smmu); + unsigned int last_s2cr; u32 reg; u32 smr; int i; /* + * Some platforms support more than the Arm SMMU architected maximum of + * 128 stream matching groups. For unknown reasons, the additional + * groups don't exhibit the same behavior as the architected registers, + * so limit the groups to 128 until the behavior is fixed for the other + * groups. + */ + if (smmu->num_mapping_groups > 128) { + dev_notice(smmu->dev, "\tLimiting the stream matching groups to 128\n"); + smmu->num_mapping_groups = 128; + } + + last_s2cr = ARM_SMMU_GR0_S2CR(smmu->num_mapping_groups - 1); + + /* * With some firmware versions writes to S2CR of type FAULT are * ignored, and writing BYPASS will end up written as FAULT in the * register. Perform a write to S2CR to detect if this is the case and diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu.c b/drivers/iommu/arm/arm-smmu/arm-smmu.c index d0843caf8760..6e0813b26fb6 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu.c @@ -2195,9 +2195,6 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - if (!smmu) - return; - if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) dev_notice(&pdev->dev, "disabling translation\n"); @@ -2214,19 +2211,14 @@ static void arm_smmu_device_shutdown(struct platform_device *pdev) clk_bulk_unprepare(smmu->num_clks, smmu->clks); } -static int arm_smmu_device_remove(struct platform_device *pdev) +static void arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); - if (!smmu) - return -ENODEV; - iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_shutdown(pdev); - - return 0; } static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) @@ -2302,7 +2294,7 @@ static struct platform_driver arm_smmu_driver = { .suppress_bind_attrs = true, }, .probe = arm_smmu_device_probe, - .remove = arm_smmu_device_remove, + .remove_new = arm_smmu_device_remove, .shutdown = arm_smmu_device_shutdown, }; module_platform_driver(arm_smmu_driver); diff --git a/drivers/iommu/arm/arm-smmu/qcom_iommu.c b/drivers/iommu/arm/arm-smmu/qcom_iommu.c index c8b70f476cd8..a503ed758ec3 100644 --- a/drivers/iommu/arm/arm-smmu/qcom_iommu.c +++ b/drivers/iommu/arm/arm-smmu/qcom_iommu.c @@ -682,7 +682,7 @@ static int qcom_iommu_ctx_probe(struct platform_device *pdev) return 0; } -static int qcom_iommu_ctx_remove(struct platform_device *pdev) +static void qcom_iommu_ctx_remove(struct platform_device *pdev) { struct qcom_iommu_dev *qcom_iommu = dev_get_drvdata(pdev->dev.parent); struct qcom_iommu_ctx *ctx = platform_get_drvdata(pdev); @@ -690,8 +690,6 @@ static int qcom_iommu_ctx_remove(struct platform_device *pdev) platform_set_drvdata(pdev, NULL); qcom_iommu->ctxs[ctx->asid - 1] = NULL; - - return 0; } static const struct of_device_id ctx_of_match[] = { @@ -706,7 +704,7 @@ static struct platform_driver qcom_iommu_ctx_driver = { .of_match_table = ctx_of_match, }, .probe = qcom_iommu_ctx_probe, - .remove = qcom_iommu_ctx_remove, + .remove_new = qcom_iommu_ctx_remove, }; static bool qcom_iommu_has_secure_context(struct qcom_iommu_dev *qcom_iommu) @@ -824,7 +822,7 @@ err_pm_disable: return ret; } -static int qcom_iommu_device_remove(struct platform_device *pdev) +static void qcom_iommu_device_remove(struct platform_device *pdev) { struct qcom_iommu_dev *qcom_iommu = platform_get_drvdata(pdev); @@ -832,8 +830,6 @@ static int qcom_iommu_device_remove(struct platform_device *pdev) platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&qcom_iommu->iommu); iommu_device_unregister(&qcom_iommu->iommu); - - return 0; } static int __maybe_unused qcom_iommu_resume(struct device *dev) @@ -870,7 +866,7 @@ static struct platform_driver qcom_iommu_driver = { .pm = &qcom_iommu_pm_ops, }, .probe = qcom_iommu_device_probe, - .remove = qcom_iommu_device_remove, + .remove_new = qcom_iommu_device_remove, }; static int __init qcom_iommu_init(void) diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 483aaaeb6dae..c275fe71c4db 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -747,22 +747,16 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) return ret; } - data->clk = devm_clk_get(dev, "sysmmu"); - if (PTR_ERR(data->clk) == -ENOENT) - data->clk = NULL; - else if (IS_ERR(data->clk)) + data->clk = devm_clk_get_optional(dev, "sysmmu"); + if (IS_ERR(data->clk)) return PTR_ERR(data->clk); - data->aclk = devm_clk_get(dev, "aclk"); - if (PTR_ERR(data->aclk) == -ENOENT) - data->aclk = NULL; - else if (IS_ERR(data->aclk)) + data->aclk = devm_clk_get_optional(dev, "aclk"); + if (IS_ERR(data->aclk)) return PTR_ERR(data->aclk); - data->pclk = devm_clk_get(dev, "pclk"); - if (PTR_ERR(data->pclk) == -ENOENT) - data->pclk = NULL; - else if (IS_ERR(data->pclk)) + data->pclk = devm_clk_get_optional(dev, "pclk"); + if (IS_ERR(data->pclk)) return PTR_ERR(data->pclk); if (!data->clk && (!data->aclk || !data->pclk)) { @@ -770,10 +764,8 @@ static int exynos_sysmmu_probe(struct platform_device *pdev) return -ENOSYS; } - data->clk_master = devm_clk_get(dev, "master"); - if (PTR_ERR(data->clk_master) == -ENOENT) - data->clk_master = NULL; - else if (IS_ERR(data->clk_master)) + data->clk_master = devm_clk_get_optional(dev, "master"); + if (IS_ERR(data->clk_master)) return PTR_ERR(data->clk_master); data->sysmmu = dev; @@ -1415,23 +1407,26 @@ static struct iommu_device *exynos_iommu_probe_device(struct device *dev) return &data->iommu; } -static void exynos_iommu_release_device(struct device *dev) +static void exynos_iommu_set_platform_dma(struct device *dev) { struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); - struct sysmmu_drvdata *data; if (owner->domain) { struct iommu_group *group = iommu_group_get(dev); if (group) { -#ifndef CONFIG_ARM - WARN_ON(owner->domain != - iommu_group_default_domain(group)); -#endif exynos_iommu_detach_device(owner->domain, dev); iommu_group_put(group); } } +} + +static void exynos_iommu_release_device(struct device *dev) +{ + struct exynos_iommu_owner *owner = dev_iommu_priv_get(dev); + struct sysmmu_drvdata *data; + + exynos_iommu_set_platform_dma(dev); list_for_each_entry(data, &owner->controllers, owner_node) device_link_del(data->link); @@ -1479,7 +1474,7 @@ static const struct iommu_ops exynos_iommu_ops = { .domain_alloc = exynos_iommu_domain_alloc, .device_group = generic_device_group, #ifdef CONFIG_ARM - .set_platform_dma_ops = exynos_iommu_release_device, + .set_platform_dma_ops = exynos_iommu_set_platform_dma, #endif .probe_device = exynos_iommu_probe_device, .release_device = exynos_iommu_release_device, diff --git a/drivers/iommu/fsl_pamu.c b/drivers/iommu/fsl_pamu.c index 05d820fb1d0b..f37d3b044131 100644 --- a/drivers/iommu/fsl_pamu.c +++ b/drivers/iommu/fsl_pamu.c @@ -178,7 +178,7 @@ int pamu_update_paace_stash(int liodn, u32 value) } /** - * pamu_config_paace() - Sets up PPAACE entry for specified liodn + * pamu_config_ppaace() - Sets up PPAACE entry for specified liodn * * @liodn: Logical IO device number * @omi: Operation mapping index -- if ~omi == 0 then omi not defined @@ -232,7 +232,8 @@ int pamu_config_ppaace(int liodn, u32 omi, u32 stashid, int prot) /** * get_ome_index() - Returns the index in the operation mapping table * for device. - * @*omi_index: pointer for storing the index value + * @omi_index: pointer for storing the index value + * @dev: target device * */ void get_ome_index(u32 *omi_index, struct device *dev) @@ -328,7 +329,7 @@ found_cpu_node: #define QMAN_PORTAL_PAACE 2 #define BMAN_PAACE 3 -/** +/* * Setup operation mapping and stash destinations for QMAN and QMAN portal. * Memory accesses to QMAN and BMAN private memory need not be coherent, so * clear the PAACE entry coherency attribute for them. @@ -357,7 +358,7 @@ static void setup_qbman_paace(struct paace *ppaace, int paace_type) } } -/** +/* * Setup the operation mapping table for various devices. This is a static * table where each table index corresponds to a particular device. PAMU uses * this table to translate device transaction to appropriate corenet diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 10bb20ff50fd..a3414afe11b0 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -127,8 +127,6 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event) struct pci_dev *tmp; struct dmar_pci_notify_info *info; - BUG_ON(dev->is_virtfn); - /* * Ignore devices that have a domain number higher than what can * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000 @@ -264,7 +262,8 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, get_device(dev)); return 1; } - BUG_ON(i >= devices_cnt); + if (WARN_ON(i >= devices_cnt)) + return -EINVAL; } return 0; @@ -1069,7 +1068,8 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) } err = -EINVAL; - if (cap_sagaw(iommu->cap) == 0) { + if (!cap_sagaw(iommu->cap) && + (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) { pr_info("%s: No supported address widths. Not attempting DMA translation.\n", iommu->name); drhd->ignored = 1; @@ -1687,7 +1687,7 @@ static void __dmar_enable_qi(struct intel_iommu *iommu) * is present. */ if (ecap_smts(iommu->ecap)) - val |= (1 << 11) | 1; + val |= BIT_ULL(11) | BIT_ULL(0); raw_spin_lock_irqsave(&iommu->register_lock, flags); diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 350c33605fd3..b871a6afd803 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -915,8 +915,6 @@ static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain, int level = agaw_to_level(domain->agaw); int offset; - BUG_ON(!domain->pgd); - if (!domain_pfn_supported(domain, pfn)) /* Address beyond IOMMU's addressing capabilities. */ return NULL; @@ -1005,9 +1003,9 @@ static void dma_pte_clear_range(struct dmar_domain *domain, unsigned int large_page; struct dma_pte *first_pte, *pte; - BUG_ON(!domain_pfn_supported(domain, start_pfn)); - BUG_ON(!domain_pfn_supported(domain, last_pfn)); - BUG_ON(start_pfn > last_pfn); + if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) || + WARN_ON(start_pfn > last_pfn)) + return; /* we don't need lock here; nobody else touches the iova range */ do { @@ -1166,9 +1164,9 @@ next: static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn, unsigned long last_pfn, struct list_head *freelist) { - BUG_ON(!domain_pfn_supported(domain, start_pfn)); - BUG_ON(!domain_pfn_supported(domain, last_pfn)); - BUG_ON(start_pfn > last_pfn); + if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) || + WARN_ON(start_pfn > last_pfn)) + return; /* we don't need lock here; nobody else touches the iova range */ dma_pte_clear_level(domain, agaw_to_level(domain->agaw), @@ -1272,7 +1270,9 @@ static void __iommu_flush_context(struct intel_iommu *iommu, | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask); break; default: - BUG(); + pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n", + iommu->name, type); + return; } val |= DMA_CCMD_ICC; @@ -1308,7 +1308,9 @@ static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, val_iva = size_order | addr; break; default: - BUG(); + pr_warn("%s: Unexpected iotlb invalidation type 0x%llx\n", + iommu->name, type); + return; } /* Note: set drain read/write */ #if 0 @@ -1406,20 +1408,6 @@ static void iommu_enable_pci_caps(struct device_domain_info *info) return; pdev = to_pci_dev(info->dev); - /* For IOMMU that supports device IOTLB throttling (DIT), we assign - * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge - * queue depth at PF level. If DIT is not set, PFSID will be treated as - * reserved, which should be set to 0. - */ - if (!ecap_dit(info->iommu->ecap)) - info->pfsid = 0; - else { - struct pci_dev *pf_pdev; - - /* pdev will be returned if device is not a vf */ - pf_pdev = pci_physfn(pdev); - info->pfsid = pci_dev_id(pf_pdev); - } /* The PCIe spec, in its wisdom, declares that the behaviour of the device if you enable PASID support after ATS support is @@ -1429,16 +1417,10 @@ static void iommu_enable_pci_caps(struct device_domain_info *info) if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1)) info->pasid_enabled = 1; - if (info->pri_supported && - (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) && - !pci_reset_pri(pdev) && !pci_enable_pri(pdev, PRQ_DEPTH)) - info->pri_enabled = 1; - if (info->ats_supported && pci_ats_page_aligned(pdev) && !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) { info->ats_enabled = 1; domain_update_iotlb(info->domain); - info->ats_qdep = pci_ats_queue_depth(pdev); } } @@ -1457,11 +1439,6 @@ static void iommu_disable_pci_caps(struct device_domain_info *info) domain_update_iotlb(info->domain); } - if (info->pri_enabled) { - pci_disable_pri(pdev); - info->pri_enabled = 0; - } - if (info->pasid_enabled) { pci_disable_pasid(pdev); info->pasid_enabled = 0; @@ -1508,7 +1485,8 @@ static void iommu_flush_iotlb_psi(struct intel_iommu *iommu, uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT; u16 did = domain_id_iommu(domain, iommu); - BUG_ON(pages == 0); + if (WARN_ON(!pages)) + return; if (ih) ih = 1 << 6; @@ -1892,7 +1870,7 @@ context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid) */ static inline void context_set_sm_dte(struct context_entry *context) { - context->lo |= (1 << 2); + context->lo |= BIT_ULL(2); } /* @@ -1901,7 +1879,7 @@ static inline void context_set_sm_dte(struct context_entry *context) */ static inline void context_set_sm_pre(struct context_entry *context) { - context->lo |= (1 << 4); + context->lo |= BIT_ULL(4); } /* Convert value to context PASID directory size field coding. */ @@ -1927,8 +1905,6 @@ static int domain_context_mapping_one(struct dmar_domain *domain, pr_debug("Set context mapping for %02x:%02x.%d\n", bus, PCI_SLOT(devfn), PCI_FUNC(devfn)); - BUG_ON(!domain->pgd); - spin_lock(&iommu->lock); ret = -ENOMEM; context = iommu_context_addr(iommu, bus, devfn, 1); @@ -2180,7 +2156,8 @@ __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn, phys_addr_t pteval; u64 attr; - BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)); + if (unlikely(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1))) + return -EINVAL; if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0) return -EINVAL; @@ -2338,8 +2315,6 @@ static int domain_setup_first_level(struct intel_iommu *iommu, if (level != 4 && level != 5) return -EINVAL; - if (pasid != PASID_RID2PASID) - flags |= PASID_FLAG_SUPERVISOR_MODE; if (level == 5) flags |= PASID_FLAG_FL5LP; @@ -4256,8 +4231,9 @@ static size_t intel_iommu_unmap(struct iommu_domain *domain, /* Cope with horrid API which requires us to unmap more than the size argument if it happens to be a large-page mapping. */ - BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level, - GFP_ATOMIC)); + if (unlikely(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, + &level, GFP_ATOMIC))) + return 0; if (size < VTD_PAGE_SIZE << level_to_offset_bits(level)) size = VTD_PAGE_SIZE << level_to_offset_bits(level); @@ -4437,6 +4413,17 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev) dmar_ats_supported(pdev, iommu)) { info->ats_supported = 1; info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev); + + /* + * For IOMMU that supports device IOTLB throttling + * (DIT), we assign PFSID to the invalidation desc + * of a VF such that IOMMU HW can gauge queue depth + * at PF level. If DIT is not set, PFSID will be + * treated as reserved, which should be set to 0. + */ + if (ecap_dit(iommu->ecap)) + info->pfsid = pci_dev_id(pci_physfn(pdev)); + info->ats_qdep = pci_ats_queue_depth(pdev); } if (sm_supported(iommu)) { if (pasid_supported(iommu)) { @@ -4554,7 +4541,6 @@ static int intel_iommu_enable_sva(struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); struct intel_iommu *iommu; - int ret; if (!info || dmar_disabled) return -EINVAL; @@ -4566,45 +4552,102 @@ static int intel_iommu_enable_sva(struct device *dev) if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE)) return -ENODEV; - if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled) + if (!info->pasid_enabled || !info->ats_enabled) return -EINVAL; - ret = iopf_queue_add_device(iommu->iopf_queue, dev); - if (ret) - return ret; + /* + * Devices having device-specific I/O fault handling should not + * support PCI/PRI. The IOMMU side has no means to check the + * capability of device-specific IOPF. Therefore, IOMMU can only + * default that if the device driver enables SVA on a non-PRI + * device, it will handle IOPF in its own way. + */ + if (!info->pri_supported) + return 0; - ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); - if (ret) - iopf_queue_remove_device(iommu->iopf_queue, dev); + /* Devices supporting PRI should have it enabled. */ + if (!info->pri_enabled) + return -EINVAL; - return ret; + return 0; } -static int intel_iommu_disable_sva(struct device *dev) +static int intel_iommu_enable_iopf(struct device *dev) { + struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL; struct device_domain_info *info = dev_iommu_priv_get(dev); - struct intel_iommu *iommu = info->iommu; + struct intel_iommu *iommu; int ret; - ret = iommu_unregister_device_fault_handler(dev); + if (!pdev || !info || !info->ats_enabled || !info->pri_supported) + return -ENODEV; + + if (info->pri_enabled) + return -EBUSY; + + iommu = info->iommu; + if (!iommu) + return -EINVAL; + + /* PASID is required in PRG Response Message. */ + if (info->pasid_enabled && !pci_prg_resp_pasid_required(pdev)) + return -EINVAL; + + ret = pci_reset_pri(pdev); + if (ret) + return ret; + + ret = iopf_queue_add_device(iommu->iopf_queue, dev); if (ret) return ret; - ret = iopf_queue_remove_device(iommu->iopf_queue, dev); + ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); + if (ret) + goto iopf_remove_device; + + ret = pci_enable_pri(pdev, PRQ_DEPTH); if (ret) - iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev); + goto iopf_unregister_handler; + info->pri_enabled = 1; + + return 0; + +iopf_unregister_handler: + iommu_unregister_device_fault_handler(dev); +iopf_remove_device: + iopf_queue_remove_device(iommu->iopf_queue, dev); return ret; } -static int intel_iommu_enable_iopf(struct device *dev) +static int intel_iommu_disable_iopf(struct device *dev) { struct device_domain_info *info = dev_iommu_priv_get(dev); + struct intel_iommu *iommu = info->iommu; - if (info && info->pri_supported) - return 0; + if (!info->pri_enabled) + return -EINVAL; - return -ENODEV; + /* + * PCIe spec states that by clearing PRI enable bit, the Page + * Request Interface will not issue new page requests, but has + * outstanding page requests that have been transmitted or are + * queued for transmission. This is supposed to be called after + * the device driver has stopped DMA, all PASIDs have been + * unbound and the outstanding PRQs have been drained. + */ + pci_disable_pri(to_pci_dev(dev)); + info->pri_enabled = 0; + + /* + * With PRI disabled and outstanding PRQs drained, unregistering + * fault handler and removing device from iopf queue should never + * fail. + */ + WARN_ON(iommu_unregister_device_fault_handler(dev)); + WARN_ON(iopf_queue_remove_device(iommu->iopf_queue, dev)); + + return 0; } static int @@ -4627,10 +4670,10 @@ intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat) { switch (feat) { case IOMMU_DEV_FEAT_IOPF: - return 0; + return intel_iommu_disable_iopf(dev); case IOMMU_DEV_FEAT_SVA: - return intel_iommu_disable_sva(dev); + return 0; default: return -ENODEV; diff --git a/drivers/iommu/intel/iommu.h b/drivers/iommu/intel/iommu.h index 65b15be72878..1c5e1d88862b 100644 --- a/drivers/iommu/intel/iommu.h +++ b/drivers/iommu/intel/iommu.h @@ -639,6 +639,8 @@ struct iommu_pmu { DECLARE_BITMAP(used_mask, IOMMU_PMU_IDX_MAX); struct perf_event *event_list[IOMMU_PMU_IDX_MAX]; unsigned char irq_name[16]; + struct hlist_node cpuhp_node; + int cpu; }; #define IOMMU_IRQ_ID_OFFSET_PRQ (DMAR_UNITS_SUPPORTED) @@ -793,18 +795,18 @@ static inline bool context_present(struct context_entry *context) return (context->lo & 1); } -extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev); +struct dmar_drhd_unit *dmar_find_matched_drhd_unit(struct pci_dev *dev); -extern int dmar_enable_qi(struct intel_iommu *iommu); -extern void dmar_disable_qi(struct intel_iommu *iommu); -extern int dmar_reenable_qi(struct intel_iommu *iommu); -extern void qi_global_iec(struct intel_iommu *iommu); +int dmar_enable_qi(struct intel_iommu *iommu); +void dmar_disable_qi(struct intel_iommu *iommu); +int dmar_reenable_qi(struct intel_iommu *iommu); +void qi_global_iec(struct intel_iommu *iommu); -extern void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, - u8 fm, u64 type); -extern void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, - unsigned int size_order, u64 type); -extern void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, +void qi_flush_context(struct intel_iommu *iommu, u16 did, + u16 sid, u8 fm, u64 type); +void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr, + unsigned int size_order, u64 type); +void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid, u16 qdep, u64 addr, unsigned mask); void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr, @@ -827,7 +829,7 @@ int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc, */ #define QI_OPT_WAIT_DRAIN BIT(0) -extern int dmar_ir_support(void); +int dmar_ir_support(void); void *alloc_pgtable_page(int node, gfp_t gfp); void free_pgtable_page(void *vaddr); @@ -835,9 +837,9 @@ void iommu_flush_write_buffer(struct intel_iommu *iommu); struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn); #ifdef CONFIG_INTEL_IOMMU_SVM -extern void intel_svm_check(struct intel_iommu *iommu); -extern int intel_svm_enable_prq(struct intel_iommu *iommu); -extern int intel_svm_finish_prq(struct intel_iommu *iommu); +void intel_svm_check(struct intel_iommu *iommu); +int intel_svm_enable_prq(struct intel_iommu *iommu); +int intel_svm_finish_prq(struct intel_iommu *iommu); int intel_svm_page_response(struct device *dev, struct iommu_fault_event *evt, struct iommu_page_response *msg); struct iommu_domain *intel_svm_domain_alloc(void); @@ -884,8 +886,8 @@ extern const struct iommu_ops intel_iommu_ops; #ifdef CONFIG_INTEL_IOMMU extern int intel_iommu_sm; -extern int iommu_calculate_agaw(struct intel_iommu *iommu); -extern int iommu_calculate_max_sagaw(struct intel_iommu *iommu); +int iommu_calculate_agaw(struct intel_iommu *iommu); +int iommu_calculate_max_sagaw(struct intel_iommu *iommu); int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob); static inline bool ecmd_has_pmu_essential(struct intel_iommu *iommu) diff --git a/drivers/iommu/intel/irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index 6d01fa078c36..a1b987335b31 100644 --- a/drivers/iommu/intel/irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -311,14 +311,12 @@ static int set_ioapic_sid(struct irte *irte, int apic) if (!irte) return -1; - down_read(&dmar_global_lock); for (i = 0; i < MAX_IO_APICS; i++) { if (ir_ioapic[i].iommu && ir_ioapic[i].id == apic) { sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; break; } } - up_read(&dmar_global_lock); if (sid == 0) { pr_warn("Failed to set source-id of IOAPIC (%d)\n", apic); @@ -338,14 +336,12 @@ static int set_hpet_sid(struct irte *irte, u8 id) if (!irte) return -1; - down_read(&dmar_global_lock); for (i = 0; i < MAX_HPET_TBS; i++) { if (ir_hpet[i].iommu && ir_hpet[i].id == id) { sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; break; } } - up_read(&dmar_global_lock); if (sid == 0) { pr_warn("Failed to set source-id of HPET block (%d)\n", id); @@ -552,7 +548,7 @@ static int intel_setup_irq_remapping(struct intel_iommu *iommu) goto out_free_table; } - bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_ATOMIC); + bitmap = bitmap_zalloc(INTR_REMAP_TABLE_ENTRIES, GFP_KERNEL); if (bitmap == NULL) { pr_err("IR%d: failed to allocate bitmap\n", iommu->seq_id); goto out_free_pages; @@ -1339,9 +1335,7 @@ static int intel_irq_remapping_alloc(struct irq_domain *domain, if (!data) goto out_free_parent; - down_read(&dmar_global_lock); index = alloc_irte(iommu, &data->irq_2_iommu, nr_irqs); - up_read(&dmar_global_lock); if (index < 0) { pr_warn("Failed to allocate IRTE\n"); kfree(data); diff --git a/drivers/iommu/intel/pasid.c b/drivers/iommu/intel/pasid.c index 633e0a4a01e7..c5d479770e12 100644 --- a/drivers/iommu/intel/pasid.c +++ b/drivers/iommu/intel/pasid.c @@ -336,15 +336,6 @@ static inline void pasid_set_fault_enable(struct pasid_entry *pe) } /* - * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a - * scalable mode PASID entry. - */ -static inline void pasid_set_sre(struct pasid_entry *pe) -{ - pasid_set_bits(&pe->val[2], 1 << 0, 1); -} - -/* * Setup the WPE(Write Protect Enable) field (Bit 132) of a * scalable mode PASID entry. */ @@ -521,23 +512,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, return -EINVAL; } - if (flags & PASID_FLAG_SUPERVISOR_MODE) { -#ifdef CONFIG_X86 - unsigned long cr0 = read_cr0(); - - /* CR0.WP is normally set but just to be sure */ - if (unlikely(!(cr0 & X86_CR0_WP))) { - pr_err("No CPU write protect!\n"); - return -EINVAL; - } -#endif - if (!ecap_srs(iommu->ecap)) { - pr_err("No supervisor request support on %s\n", - iommu->name); - return -EINVAL; - } - } - if ((flags & PASID_FLAG_FL5LP) && !cap_fl5lp_support(iommu->cap)) { pr_err("No 5-level paging support for first-level on %s\n", iommu->name); @@ -560,10 +534,6 @@ int intel_pasid_setup_first_level(struct intel_iommu *iommu, /* Setup the first level page table pointer: */ pasid_set_flptr(pte, (u64)__pa(pgd)); - if (flags & PASID_FLAG_SUPERVISOR_MODE) { - pasid_set_sre(pte); - pasid_set_wpe(pte); - } if (flags & PASID_FLAG_FL5LP) pasid_set_flpm(pte, 1); @@ -658,12 +628,6 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu, pasid_set_fault_enable(pte); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); - /* - * Since it is a second level only translation setup, we should - * set SRE bit as well (addresses are expected to be GPAs). - */ - if (pasid != PASID_RID2PASID && ecap_srs(iommu->ecap)) - pasid_set_sre(pte); pasid_set_present(pte); spin_unlock(&iommu->lock); @@ -700,13 +664,6 @@ int intel_pasid_setup_pass_through(struct intel_iommu *iommu, pasid_set_translation_type(pte, PASID_ENTRY_PGTT_PT); pasid_set_fault_enable(pte); pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap)); - - /* - * We should set SRE bit as well since the addresses are expected - * to be GPAs. - */ - if (ecap_srs(iommu->ecap)) - pasid_set_sre(pte); pasid_set_present(pte); spin_unlock(&iommu->lock); diff --git a/drivers/iommu/intel/pasid.h b/drivers/iommu/intel/pasid.h index 20c54e50f533..d6b7d21244b1 100644 --- a/drivers/iommu/intel/pasid.h +++ b/drivers/iommu/intel/pasid.h @@ -41,13 +41,6 @@ #define FLPT_DEFAULT_DID 1 #define NUM_RESERVED_DID 2 -/* - * The SUPERVISOR_MODE flag indicates a first level translation which - * can be used for access to kernel addresses. It is valid only for - * access to the kernel's static 1:1 mapping of physical memory — not - * to vmalloc or even module mappings. - */ -#define PASID_FLAG_SUPERVISOR_MODE BIT(0) #define PASID_FLAG_NESTED BIT(1) #define PASID_FLAG_PAGE_SNOOP BIT(2) diff --git a/drivers/iommu/intel/perfmon.c b/drivers/iommu/intel/perfmon.c index e17d9743a0d8..cf43e798eca4 100644 --- a/drivers/iommu/intel/perfmon.c +++ b/drivers/iommu/intel/perfmon.c @@ -773,19 +773,34 @@ static void iommu_pmu_unset_interrupt(struct intel_iommu *iommu) iommu->perf_irq = 0; } -static int iommu_pmu_cpu_online(unsigned int cpu) +static int iommu_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) { + struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); + if (cpumask_empty(&iommu_pmu_cpu_mask)) cpumask_set_cpu(cpu, &iommu_pmu_cpu_mask); + if (cpumask_test_cpu(cpu, &iommu_pmu_cpu_mask)) + iommu_pmu->cpu = cpu; + return 0; } -static int iommu_pmu_cpu_offline(unsigned int cpu) +static int iommu_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) { - struct dmar_drhd_unit *drhd; - struct intel_iommu *iommu; - int target; + struct iommu_pmu *iommu_pmu = hlist_entry_safe(node, typeof(*iommu_pmu), cpuhp_node); + int target = cpumask_first(&iommu_pmu_cpu_mask); + + /* + * The iommu_pmu_cpu_mask has been updated when offline the CPU + * for the first iommu_pmu. Migrate the other iommu_pmu to the + * new target. + */ + if (target < nr_cpu_ids && target != iommu_pmu->cpu) { + perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); + iommu_pmu->cpu = target; + return 0; + } if (!cpumask_test_and_clear_cpu(cpu, &iommu_pmu_cpu_mask)) return 0; @@ -795,45 +810,50 @@ static int iommu_pmu_cpu_offline(unsigned int cpu) if (target < nr_cpu_ids) cpumask_set_cpu(target, &iommu_pmu_cpu_mask); else - target = -1; + return 0; - rcu_read_lock(); - - for_each_iommu(iommu, drhd) { - if (!iommu->pmu) - continue; - perf_pmu_migrate_context(&iommu->pmu->pmu, cpu, target); - } - rcu_read_unlock(); + perf_pmu_migrate_context(&iommu_pmu->pmu, cpu, target); + iommu_pmu->cpu = target; return 0; } static int nr_iommu_pmu; +static enum cpuhp_state iommu_cpuhp_slot; static int iommu_pmu_cpuhp_setup(struct iommu_pmu *iommu_pmu) { int ret; - if (nr_iommu_pmu++) - return 0; + if (!nr_iommu_pmu) { + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "driver/iommu/intel/perfmon:online", + iommu_pmu_cpu_online, + iommu_pmu_cpu_offline); + if (ret < 0) + return ret; + iommu_cpuhp_slot = ret; + } - ret = cpuhp_setup_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE, - "driver/iommu/intel/perfmon:online", - iommu_pmu_cpu_online, - iommu_pmu_cpu_offline); - if (ret) - nr_iommu_pmu = 0; + ret = cpuhp_state_add_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); + if (ret) { + if (!nr_iommu_pmu) + cpuhp_remove_multi_state(iommu_cpuhp_slot); + return ret; + } + nr_iommu_pmu++; - return ret; + return 0; } static void iommu_pmu_cpuhp_free(struct iommu_pmu *iommu_pmu) { + cpuhp_state_remove_instance(iommu_cpuhp_slot, &iommu_pmu->cpuhp_node); + if (--nr_iommu_pmu) return; - cpuhp_remove_state(CPUHP_AP_PERF_X86_IOMMU_PERF_ONLINE); + cpuhp_remove_multi_state(iommu_cpuhp_slot); } void iommu_pmu_register(struct intel_iommu *iommu) diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index f35058f1d68e..153a3dab568c 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1985,8 +1985,13 @@ static struct iommu_domain *__iommu_domain_alloc(struct bus_type *bus, return NULL; domain->type = type; - /* Assume all sizes by default; the driver may override this later */ - domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; + /* + * If not already set, assume all sizes by default; the driver + * may override this later + */ + if (!domain->pgsize_bitmap) + domain->pgsize_bitmap = bus->iommu_ops->pgsize_bitmap; + if (!domain->ops) domain->ops = bus->iommu_ops->default_domain_ops; diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index f8d92c9bb65b..3c47846cc5ef 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -294,9 +294,9 @@ static void batch_clear_carry(struct pfn_batch *batch, unsigned int keep_pfns) batch->npfns[batch->end - 1] < keep_pfns); batch->total_pfns = keep_pfns; - batch->npfns[0] = keep_pfns; batch->pfns[0] = batch->pfns[batch->end - 1] + (batch->npfns[batch->end - 1] - keep_pfns); + batch->npfns[0] = keep_pfns; batch->end = 0; } @@ -1142,6 +1142,7 @@ struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length, bool writable) { struct iopt_pages *pages; + unsigned long end; /* * The iommu API uses size_t as the length, and protect the DIV_ROUND_UP @@ -1150,6 +1151,9 @@ struct iopt_pages *iopt_alloc_pages(void __user *uptr, unsigned long length, if (length > SIZE_MAX - PAGE_SIZE || length == 0) return ERR_PTR(-EINVAL); + if (check_add_overflow((unsigned long)uptr, length, &end)) + return ERR_PTR(-EOVERFLOW); + pages = kzalloc(sizeof(*pages), GFP_KERNEL_ACCOUNT); if (!pages) return ERR_PTR(-ENOMEM); @@ -1203,13 +1207,21 @@ iopt_area_unpin_domain(struct pfn_batch *batch, struct iopt_area *area, unsigned long start = max(start_index, *unmapped_end_index); + if (IS_ENABLED(CONFIG_IOMMUFD_TEST) && + batch->total_pfns) + WARN_ON(*unmapped_end_index - + batch->total_pfns != + start_index); batch_from_domain(batch, domain, area, start, last_index); - batch_last_index = start + batch->total_pfns - 1; + batch_last_index = start_index + batch->total_pfns - 1; } else { batch_last_index = last_index; } + if (IS_ENABLED(CONFIG_IOMMUFD_TEST)) + WARN_ON(batch_last_index > real_last_index); + /* * unmaps must always 'cut' at a place where the pfns are not * contiguous to pair with the maps that always install diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c index 7fcbef8802de..9f64c5c9f5b9 100644 --- a/drivers/iommu/ipmmu-vmsa.c +++ b/drivers/iommu/ipmmu-vmsa.c @@ -696,7 +696,6 @@ static const struct soc_device_attribute soc_needs_opt_in[] = { static const struct soc_device_attribute soc_denylist[] = { { .soc_id = "r8a774a1", }, - { .soc_id = "r8a7795", .revision = "ES1.*" }, { .soc_id = "r8a7795", .revision = "ES2.*" }, { .soc_id = "r8a7796", }, { /* sentinel */ } @@ -1083,7 +1082,7 @@ static int ipmmu_probe(struct platform_device *pdev) return 0; } -static int ipmmu_remove(struct platform_device *pdev) +static void ipmmu_remove(struct platform_device *pdev) { struct ipmmu_vmsa_device *mmu = platform_get_drvdata(pdev); @@ -1093,8 +1092,6 @@ static int ipmmu_remove(struct platform_device *pdev) arm_iommu_release_mapping(mmu->mapping); ipmmu_device_reset(mmu); - - return 0; } #ifdef CONFIG_PM_SLEEP @@ -1141,6 +1138,6 @@ static struct platform_driver ipmmu_driver = { .pm = DEV_PM_OPS, }, .probe = ipmmu_probe, - .remove = ipmmu_remove, + .remove_new = ipmmu_remove, }; builtin_platform_driver(ipmmu_driver); diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c index 454f6331c889..79d89bad5132 100644 --- a/drivers/iommu/msm_iommu.c +++ b/drivers/iommu/msm_iommu.c @@ -811,13 +811,12 @@ static const struct of_device_id msm_iommu_dt_match[] = { {} }; -static int msm_iommu_remove(struct platform_device *pdev) +static void msm_iommu_remove(struct platform_device *pdev) { struct msm_iommu_dev *iommu = platform_get_drvdata(pdev); clk_unprepare(iommu->clk); clk_unprepare(iommu->pclk); - return 0; } static struct platform_driver msm_iommu_driver = { @@ -826,6 +825,6 @@ static struct platform_driver msm_iommu_driver = { .of_match_table = msm_iommu_dt_match, }, .probe = msm_iommu_probe, - .remove = msm_iommu_remove, + .remove_new = msm_iommu_remove, }; builtin_platform_driver(msm_iommu_driver); diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c index d5a4955910ff..aecc7d154f28 100644 --- a/drivers/iommu/mtk_iommu.c +++ b/drivers/iommu/mtk_iommu.c @@ -8,7 +8,6 @@ #include <linux/clk.h> #include <linux/component.h> #include <linux/device.h> -#include <linux/dma-direct.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -197,12 +196,42 @@ struct mtk_iommu_plat_data { char *pericfg_comp_str; struct list_head *hw_list; - unsigned int iova_region_nr; - const struct mtk_iommu_iova_region *iova_region; - u8 banks_num; - bool banks_enable[MTK_IOMMU_BANK_MAX]; - unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX]; + /* + * The IOMMU HW may support 16GB iova. In order to balance the IOVA ranges, + * different masters will be put in different iova ranges, for example vcodec + * is in 4G-8G and cam is in 8G-12G. Meanwhile, some masters may have the + * special IOVA range requirement, like CCU can only support the address + * 0x40000000-0x44000000. + * Here list the iova ranges this SoC supports and which larbs/ports are in + * which region. + * + * 16GB iova all use one pgtable, but each a region is a iommu group. + */ + struct { + unsigned int iova_region_nr; + const struct mtk_iommu_iova_region *iova_region; + /* + * Indicate the correspondance between larbs, ports and regions. + * + * The index is the same as iova_region and larb port numbers are + * described as bit positions. + * For example, storing BIT(0) at index 2,1 means "larb 1, port0 is in region 2". + * [2] = { [1] = BIT(0) } + */ + const u32 (*iova_region_larb_msk)[MTK_LARB_NR_MAX]; + }; + + /* + * The IOMMU HW may have 5 banks. Each bank has a independent pgtable. + * Here list how many banks this SoC supports/enables and which ports are in which bank. + */ + struct { + u8 banks_num; + bool banks_enable[MTK_IOMMU_BANK_MAX]; + unsigned int banks_portmsk[MTK_IOMMU_BANK_MAX]; + }; + unsigned char larbid_remap[MTK_LARB_COM_MAX][MTK_LARB_SUBCOM_MAX]; }; @@ -303,16 +332,23 @@ static LIST_HEAD(m4ulist); /* List all the M4U HWs */ #define for_each_m4u(data, head) list_for_each_entry(data, head, list) +#define MTK_IOMMU_IOVA_SZ_4G (SZ_4G - SZ_8M) /* 8M as gap */ + static const struct mtk_iommu_iova_region single_domain[] = { - {.iova_base = 0, .size = SZ_4G}, + {.iova_base = 0, .size = MTK_IOMMU_IOVA_SZ_4G}, }; -static const struct mtk_iommu_iova_region mt8192_multi_dom[] = { - { .iova_base = 0x0, .size = SZ_4G}, /* 0 ~ 4G */ +#define MT8192_MULTI_REGION_NR_MAX 6 + +#define MT8192_MULTI_REGION_NR (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) ? \ + MT8192_MULTI_REGION_NR_MAX : 1) + +static const struct mtk_iommu_iova_region mt8192_multi_dom[MT8192_MULTI_REGION_NR] = { + { .iova_base = 0x0, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 0 ~ 4G, */ #if IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT) - { .iova_base = SZ_4G, .size = SZ_4G}, /* 4G ~ 8G */ - { .iova_base = SZ_4G * 2, .size = SZ_4G}, /* 8G ~ 12G */ - { .iova_base = SZ_4G * 3, .size = SZ_4G}, /* 12G ~ 16G */ + { .iova_base = SZ_4G, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 4G ~ 8G */ + { .iova_base = SZ_4G * 2, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 8G ~ 12G */ + { .iova_base = SZ_4G * 3, .size = MTK_IOMMU_IOVA_SZ_4G}, /* 12G ~ 16G */ { .iova_base = 0x240000000ULL, .size = 0x4000000}, /* CCU0 */ { .iova_base = 0x244000000ULL, .size = 0x4000000}, /* CCU1 */ @@ -508,30 +544,29 @@ static unsigned int mtk_iommu_get_bank_id(struct device *dev, static int mtk_iommu_get_iova_region_id(struct device *dev, const struct mtk_iommu_plat_data *plat_data) { - const struct mtk_iommu_iova_region *rgn = plat_data->iova_region; - const struct bus_dma_region *dma_rgn = dev->dma_range_map; - int i, candidate = -1; - dma_addr_t dma_end; + struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); + unsigned int portidmsk = 0, larbid; + const u32 *rgn_larb_msk; + int i; - if (!dma_rgn || plat_data->iova_region_nr == 1) + if (plat_data->iova_region_nr == 1) return 0; - dma_end = dma_rgn->dma_start + dma_rgn->size - 1; - for (i = 0; i < plat_data->iova_region_nr; i++, rgn++) { - /* Best fit. */ - if (dma_rgn->dma_start == rgn->iova_base && - dma_end == rgn->iova_base + rgn->size - 1) + larbid = MTK_M4U_TO_LARB(fwspec->ids[0]); + for (i = 0; i < fwspec->num_ids; i++) + portidmsk |= BIT(MTK_M4U_TO_PORT(fwspec->ids[i])); + + for (i = 0; i < plat_data->iova_region_nr; i++) { + rgn_larb_msk = plat_data->iova_region_larb_msk[i]; + if (!rgn_larb_msk) + continue; + + if ((rgn_larb_msk[larbid] & portidmsk) == portidmsk) return i; - /* ok if it is inside this region. */ - if (dma_rgn->dma_start >= rgn->iova_base && - dma_end < rgn->iova_base + rgn->size) - candidate = i; } - if (candidate >= 0) - return candidate; - dev_err(dev, "Can NOT find the iommu domain id(%pad 0x%llx).\n", - &dma_rgn->dma_start, dma_rgn->size); + dev_err(dev, "Can NOT find the region for larb(%d-%x).\n", + larbid, portidmsk); return -EINVAL; } @@ -703,6 +738,14 @@ static int mtk_iommu_attach_device(struct iommu_domain *domain, } mutex_unlock(&data->mutex); + if (region_id > 0) { + ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(34)); + if (ret) { + dev_err(m4udev, "Failed to set dma_mask for %s(%d).\n", dev_name(dev), ret); + return ret; + } + } + return mtk_iommu_config(data, dev, true, region_id); err_unlock: @@ -1258,6 +1301,14 @@ static int mtk_iommu_probe(struct platform_device *pdev) return PTR_ERR(data->bclk); } + if (MTK_IOMMU_HAS_FLAG(data->plat_data, PGTABLE_PA_35_EN)) { + ret = dma_set_mask(dev, DMA_BIT_MASK(35)); + if (ret) { + dev_err(dev, "Failed to set dma_mask 35.\n"); + return ret; + } + } + pm_runtime_enable(dev); if (MTK_IOMMU_IS_TYPE(data->plat_data, MTK_IOMMU_TYPE_MM)) { @@ -1316,7 +1367,7 @@ out_runtime_disable: return ret; } -static int mtk_iommu_remove(struct platform_device *pdev) +static void mtk_iommu_remove(struct platform_device *pdev) { struct mtk_iommu_data *data = platform_get_drvdata(pdev); struct mtk_iommu_bank_data *bank; @@ -1338,7 +1389,6 @@ static int mtk_iommu_remove(struct platform_device *pdev) continue; devm_free_irq(&pdev->dev, bank->irq, bank); } - return 0; } static int __maybe_unused mtk_iommu_runtime_suspend(struct device *dev) @@ -1492,6 +1542,18 @@ static const struct mtk_iommu_plat_data mt8183_data = { .larbid_remap = {{0}, {4}, {5}, {6}, {7}, {2}, {3}, {1}}, }; +static const unsigned int mt8186_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0, ~0}, /* Region0: all ports for larb0/1/2 */ + [1] = {0, 0, 0, 0, ~0, 0, 0, ~0}, /* Region1: larb4/7 */ + [2] = {0, 0, 0, 0, 0, 0, 0, 0, /* Region2: larb8/9/11/13/16/17/19/20 */ + ~0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), 0, 0, + /* larb13: the other ports except port9/10 */ + ~0, ~0, 0, ~0, ~0}, + [3] = {0}, + [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ + [5] = {[14] = ~0}, /* larb14 */ +}; + static const struct mtk_iommu_plat_data mt8186_data_mm = { .m4u_plat = M4U_MT8186, .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | @@ -1504,6 +1566,18 @@ static const struct mtk_iommu_plat_data mt8186_data_mm = { .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8186_larb_region_msk, +}; + +static const unsigned int mt8192_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0}, /* Region0: larb0/1 */ + [1] = {0, 0, 0, 0, ~0, ~0, 0, ~0}, /* Region1: larb4/5/7 */ + [2] = {0, 0, ~0, 0, 0, 0, 0, 0, /* Region2: larb2/9/11/13/14/16/17/18/19/20 */ + 0, ~0, 0, ~0, 0, ~(u32)(BIT(9) | BIT(10)), ~(u32)(BIT(4) | BIT(5)), 0, + ~0, ~0, ~0, ~0, ~0}, + [3] = {0}, + [4] = {[13] = BIT(9) | BIT(10)}, /* larb13 port9/10 */ + [5] = {[14] = BIT(4) | BIT(5)}, /* larb14 port4/5 */ }; static const struct mtk_iommu_plat_data mt8192_data = { @@ -1515,6 +1589,7 @@ static const struct mtk_iommu_plat_data mt8192_data = { .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8192_larb_region_msk, .larbid_remap = {{0}, {1}, {4, 5}, {7}, {2}, {9, 11, 19, 20}, {0, 14, 16}, {0, 13, 18, 17}}, }; @@ -1534,6 +1609,21 @@ static const struct mtk_iommu_plat_data mt8195_data_infra = { .iova_region_nr = ARRAY_SIZE(single_domain), }; +static const unsigned int mt8195_larb_region_msk[MT8192_MULTI_REGION_NR_MAX][MTK_LARB_NR_MAX] = { + [0] = {~0, ~0, ~0, ~0}, /* Region0: all ports for larb0/1/2/3 */ + [1] = {0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, ~0, ~0, ~0, ~0, ~0, /* Region1: larb19/20/21/22/23/24 */ + ~0}, + [2] = {0, 0, 0, 0, ~0, ~0, ~0, ~0, /* Region2: the other larbs. */ + ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, + ~0, ~0, 0, 0, 0, 0, 0, 0, + 0, ~0, ~0, ~0, ~0}, + [3] = {0}, + [4] = {[18] = BIT(0) | BIT(1)}, /* Only larb18 port0/1 */ + [5] = {[18] = BIT(2) | BIT(3)}, /* Only larb18 port2/3 */ +}; + static const struct mtk_iommu_plat_data mt8195_data_vdo = { .m4u_plat = M4U_MT8195, .flags = HAS_BCLK | HAS_SUB_COMM_2BITS | OUT_ORDER_WR_EN | @@ -1544,6 +1634,7 @@ static const struct mtk_iommu_plat_data mt8195_data_vdo = { .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8195_larb_region_msk, .larbid_remap = {{2, 0}, {21}, {24}, {7}, {19}, {9, 10, 11}, {13, 17, 15/* 17b */, 25}, {5}}, }; @@ -1558,6 +1649,7 @@ static const struct mtk_iommu_plat_data mt8195_data_vpp = { .banks_enable = {true}, .iova_region = mt8192_multi_dom, .iova_region_nr = ARRAY_SIZE(mt8192_multi_dom), + .iova_region_larb_msk = mt8195_larb_region_msk, .larbid_remap = {{1}, {3}, {22, MTK_INVALID_LARBID, MTK_INVALID_LARBID, MTK_INVALID_LARBID, 23}, {8}, {20}, {12}, @@ -1595,7 +1687,7 @@ static const struct of_device_id mtk_iommu_of_ids[] = { static struct platform_driver mtk_iommu_driver = { .probe = mtk_iommu_probe, - .remove = mtk_iommu_remove, + .remove_new = mtk_iommu_remove, .driver = { .name = "mtk-iommu", .of_match_table = mtk_iommu_of_ids, diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c index 43e4c8f89e23..8a0a5e5d049f 100644 --- a/drivers/iommu/mtk_iommu_v1.c +++ b/drivers/iommu/mtk_iommu_v1.c @@ -703,7 +703,7 @@ out_clk_unprepare: return ret; } -static int mtk_iommu_v1_remove(struct platform_device *pdev) +static void mtk_iommu_v1_remove(struct platform_device *pdev) { struct mtk_iommu_v1_data *data = platform_get_drvdata(pdev); @@ -713,7 +713,6 @@ static int mtk_iommu_v1_remove(struct platform_device *pdev) clk_disable_unprepare(data->bclk); devm_free_irq(&pdev->dev, data->irq, data); component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops); - return 0; } static int __maybe_unused mtk_iommu_v1_suspend(struct device *dev) @@ -752,7 +751,7 @@ static const struct dev_pm_ops mtk_iommu_v1_pm_ops = { static struct platform_driver mtk_iommu_v1_driver = { .probe = mtk_iommu_v1_probe, - .remove = mtk_iommu_v1_remove, + .remove_new = mtk_iommu_v1_remove, .driver = { .name = "mtk-iommu-v1", .of_match_table = mtk_iommu_v1_of_ids, diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c index 3ab078112a7c..537e402f9bba 100644 --- a/drivers/iommu/omap-iommu.c +++ b/drivers/iommu/omap-iommu.c @@ -1191,7 +1191,7 @@ static int omap_iommu_probe(struct platform_device *pdev) return err; if (obj->nr_tlb_entries != 32 && obj->nr_tlb_entries != 8) return -EINVAL; - if (of_find_property(of, "ti,iommu-bus-err-back", NULL)) + if (of_property_read_bool(of, "ti,iommu-bus-err-back")) obj->has_bus_err_back = MMU_GP_REG_BUS_ERR_BACK_EN; obj->dev = &pdev->dev; @@ -1257,7 +1257,7 @@ out_group: return err; } -static int omap_iommu_remove(struct platform_device *pdev) +static void omap_iommu_remove(struct platform_device *pdev) { struct omap_iommu *obj = platform_get_drvdata(pdev); @@ -1274,7 +1274,6 @@ static int omap_iommu_remove(struct platform_device *pdev) pm_runtime_disable(obj->dev); dev_info(&pdev->dev, "%s removed\n", obj->name); - return 0; } static const struct dev_pm_ops omap_iommu_pm_ops = { @@ -1295,7 +1294,7 @@ static const struct of_device_id omap_iommu_of_match[] = { static struct platform_driver omap_iommu_driver = { .probe = omap_iommu_probe, - .remove = omap_iommu_remove, + .remove_new = omap_iommu_remove, .driver = { .name = "omap-iommu", .pm = &omap_iommu_pm_ops, diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c index f30db22ea5d7..ea5a3088bb7e 100644 --- a/drivers/iommu/rockchip-iommu.c +++ b/drivers/iommu/rockchip-iommu.c @@ -124,6 +124,7 @@ struct rk_iommudata { static struct device *dma_dev; static const struct rk_iommu_ops *rk_ops; +static struct iommu_domain rk_identity_domain; static inline void rk_table_flush(struct rk_iommu_domain *dom, dma_addr_t dma, unsigned int count) @@ -646,7 +647,7 @@ static irqreturn_t rk_iommu_irq(int irq, void *dev_id) * Ignore the return code, though, since we always zap cache * and clear the page fault anyway. */ - if (iommu->domain) + if (iommu->domain != &rk_identity_domain) report_iommu_fault(iommu->domain, iommu->dev, iova, flags); else @@ -980,26 +981,27 @@ out_disable_clocks: return ret; } -static void rk_iommu_detach_device(struct iommu_domain *domain, - struct device *dev) +static int rk_iommu_identity_attach(struct iommu_domain *identity_domain, + struct device *dev) { struct rk_iommu *iommu; - struct rk_iommu_domain *rk_domain = to_rk_domain(domain); + struct rk_iommu_domain *rk_domain; unsigned long flags; int ret; /* Allow 'virtual devices' (eg drm) to detach from domain */ iommu = rk_iommu_from_dev(dev); if (!iommu) - return; + return -ENODEV; + + rk_domain = to_rk_domain(iommu->domain); dev_dbg(dev, "Detaching from iommu domain\n"); - /* iommu already detached */ - if (iommu->domain != domain) - return; + if (iommu->domain == identity_domain) + return 0; - iommu->domain = NULL; + iommu->domain = identity_domain; spin_lock_irqsave(&rk_domain->iommus_lock, flags); list_del_init(&iommu->node); @@ -1011,8 +1013,31 @@ static void rk_iommu_detach_device(struct iommu_domain *domain, rk_iommu_disable(iommu); pm_runtime_put(iommu->dev); } + + return 0; } +static void rk_iommu_identity_free(struct iommu_domain *domain) +{ +} + +static struct iommu_domain_ops rk_identity_ops = { + .attach_dev = rk_iommu_identity_attach, + .free = rk_iommu_identity_free, +}; + +static struct iommu_domain rk_identity_domain = { + .type = IOMMU_DOMAIN_IDENTITY, + .ops = &rk_identity_ops, +}; + +#ifdef CONFIG_ARM +static void rk_iommu_set_platform_dma(struct device *dev) +{ + WARN_ON(rk_iommu_identity_attach(&rk_identity_domain, dev)); +} +#endif + static int rk_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { @@ -1035,8 +1060,9 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, if (iommu->domain == domain) return 0; - if (iommu->domain) - rk_iommu_detach_device(iommu->domain, dev); + ret = rk_iommu_identity_attach(&rk_identity_domain, dev); + if (ret) + return ret; iommu->domain = domain; @@ -1050,7 +1076,7 @@ static int rk_iommu_attach_device(struct iommu_domain *domain, ret = rk_iommu_enable(iommu); if (ret) - rk_iommu_detach_device(iommu->domain, dev); + WARN_ON(rk_iommu_identity_attach(&rk_identity_domain, dev)); pm_runtime_put(iommu->dev); @@ -1061,6 +1087,9 @@ static struct iommu_domain *rk_iommu_domain_alloc(unsigned type) { struct rk_iommu_domain *rk_domain; + if (type == IOMMU_DOMAIN_IDENTITY) + return &rk_identity_domain; + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; @@ -1176,6 +1205,7 @@ static int rk_iommu_of_xlate(struct device *dev, iommu_dev = of_find_device_by_node(args->np); data->iommu = platform_get_drvdata(iommu_dev); + data->iommu->domain = &rk_identity_domain; dev_iommu_priv_set(dev, data); platform_device_put(iommu_dev); @@ -1188,6 +1218,9 @@ static const struct iommu_ops rk_iommu_ops = { .probe_device = rk_iommu_probe_device, .release_device = rk_iommu_release_device, .device_group = rk_iommu_device_group, +#ifdef CONFIG_ARM + .set_platform_dma_ops = rk_iommu_set_platform_dma, +#endif .pgsize_bitmap = RK_IOMMU_PGSIZE_BITMAP, .of_xlate = rk_iommu_of_xlate, .default_domain_ops = &(const struct iommu_domain_ops) { @@ -1343,7 +1376,7 @@ static int __maybe_unused rk_iommu_suspend(struct device *dev) { struct rk_iommu *iommu = dev_get_drvdata(dev); - if (!iommu->domain) + if (iommu->domain == &rk_identity_domain) return 0; rk_iommu_disable(iommu); @@ -1354,7 +1387,7 @@ static int __maybe_unused rk_iommu_resume(struct device *dev) { struct rk_iommu *iommu = dev_get_drvdata(dev); - if (!iommu->domain) + if (iommu->domain == &rk_identity_domain) return 0; return rk_iommu_enable(iommu); diff --git a/drivers/iommu/sprd-iommu.c b/drivers/iommu/sprd-iommu.c index ae94d74b73f4..39e34fdeccda 100644 --- a/drivers/iommu/sprd-iommu.c +++ b/drivers/iommu/sprd-iommu.c @@ -62,6 +62,7 @@ enum sprd_iommu_version { * @eb: gate clock which controls IOMMU access */ struct sprd_iommu_device { + struct sprd_iommu_domain *dom; enum sprd_iommu_version ver; u32 *prot_page_va; dma_addr_t prot_page_pa; @@ -151,13 +152,6 @@ static struct iommu_domain *sprd_iommu_domain_alloc(unsigned int domain_type) return &dom->domain; } -static void sprd_iommu_domain_free(struct iommu_domain *domain) -{ - struct sprd_iommu_domain *dom = to_sprd_domain(domain); - - kfree(dom); -} - static void sprd_iommu_first_vpn(struct sprd_iommu_domain *dom) { struct sprd_iommu_device *sdev = dom->sdev; @@ -230,6 +224,28 @@ static void sprd_iommu_hw_en(struct sprd_iommu_device *sdev, bool en) sprd_iommu_update_bits(sdev, reg_cfg, mask, 0, val); } +static void sprd_iommu_cleanup(struct sprd_iommu_domain *dom) +{ + size_t pgt_size; + + /* Nothing need to do if the domain hasn't been attached */ + if (!dom->sdev) + return; + + pgt_size = sprd_iommu_pgt_size(&dom->domain); + dma_free_coherent(dom->sdev->dev, pgt_size, dom->pgt_va, dom->pgt_pa); + dom->sdev = NULL; + sprd_iommu_hw_en(dom->sdev, false); +} + +static void sprd_iommu_domain_free(struct iommu_domain *domain) +{ + struct sprd_iommu_domain *dom = to_sprd_domain(domain); + + sprd_iommu_cleanup(dom); + kfree(dom); +} + static int sprd_iommu_attach_device(struct iommu_domain *domain, struct device *dev) { @@ -237,15 +253,27 @@ static int sprd_iommu_attach_device(struct iommu_domain *domain, struct sprd_iommu_domain *dom = to_sprd_domain(domain); size_t pgt_size = sprd_iommu_pgt_size(domain); - if (dom->sdev) - return -EINVAL; + /* The device is attached to this domain */ + if (sdev->dom == dom) + return 0; - dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL); - if (!dom->pgt_va) - return -ENOMEM; + /* The first time that domain is attaching to a device */ + if (!dom->pgt_va) { + dom->pgt_va = dma_alloc_coherent(sdev->dev, pgt_size, &dom->pgt_pa, GFP_KERNEL); + if (!dom->pgt_va) + return -ENOMEM; + + dom->sdev = sdev; + } - dom->sdev = sdev; + sdev->dom = dom; + /* + * One sprd IOMMU serves one client device only, disabled it before + * configure mapping table to avoid access conflict in case other + * mapping table is stored in. + */ + sprd_iommu_hw_en(sdev, false); sprd_iommu_first_ppn(dom); sprd_iommu_first_vpn(dom); sprd_iommu_vpn_range(dom); @@ -507,7 +535,7 @@ free_page: return ret; } -static int sprd_iommu_remove(struct platform_device *pdev) +static void sprd_iommu_remove(struct platform_device *pdev) { struct sprd_iommu_device *sdev = platform_get_drvdata(pdev); @@ -519,8 +547,6 @@ static int sprd_iommu_remove(struct platform_device *pdev) platform_set_drvdata(pdev, NULL); iommu_device_sysfs_remove(&sdev->iommu); iommu_device_unregister(&sdev->iommu); - - return 0; } static struct platform_driver sprd_iommu_driver = { @@ -530,7 +556,7 @@ static struct platform_driver sprd_iommu_driver = { .suppress_bind_attrs = true, }, .probe = sprd_iommu_probe, - .remove = sprd_iommu_remove, + .remove_new = sprd_iommu_remove, }; module_platform_driver(sprd_iommu_driver); diff --git a/drivers/iommu/sun50i-iommu.c b/drivers/iommu/sun50i-iommu.c index 2d993d0cea7d..74c5cb93e900 100644 --- a/drivers/iommu/sun50i-iommu.c +++ b/drivers/iommu/sun50i-iommu.c @@ -1076,4 +1076,3 @@ builtin_platform_driver_probe(sun50i_iommu_driver, sun50i_iommu_probe); MODULE_DESCRIPTION("Allwinner H6 IOMMU driver"); MODULE_AUTHOR("Maxime Ripard <maxime@cerno.tech>"); MODULE_AUTHOR("zhuxianbin <zhuxianbin@allwinnertech.com>"); -MODULE_LICENSE("Dual BSD/GPL"); |