From 1da027362a7db422243601e895e6f8288389f435 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Fri, 2 Apr 2021 12:50:14 +0100 Subject: irqdomain: Reimplement irq_linear_revmap() with irq_find_mapping() irq_linear_revmap() is supposed to be a fast path for domain lookups, but it only exposes low-level details of the irqdomain implementation, details which are better kept private. The *overhead* between the two is only a function call and a couple of tests, so it is likely that noone can show any meaningful difference compared to the cost of taking an interrupt. Reimplement irq_linear_revmap() with irq_find_mapping() in order to preserve source code compatibility, and rename the internal field for a measure. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 6284443b87ec..8bd012253989 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -486,7 +486,7 @@ static void irq_domain_clear_mapping(struct irq_domain *domain, irq_hw_number_t hwirq) { if (hwirq < domain->revmap_size) { - domain->linear_revmap[hwirq] = 0; + domain->revmap[hwirq] = 0; } else { mutex_lock(&domain->revmap_tree_mutex); radix_tree_delete(&domain->revmap_tree, hwirq); @@ -499,7 +499,7 @@ static void irq_domain_set_mapping(struct irq_domain *domain, struct irq_data *irq_data) { if (hwirq < domain->revmap_size) { - domain->linear_revmap[hwirq] = irq_data->irq; + domain->revmap[hwirq] = irq_data->irq; } else { mutex_lock(&domain->revmap_tree_mutex); radix_tree_insert(&domain->revmap_tree, hwirq, irq_data); @@ -885,7 +885,7 @@ unsigned int irq_find_mapping(struct irq_domain *domain, /* Check if the hwirq is in the linear revmap. */ if (hwirq < domain->revmap_size) - return domain->linear_revmap[hwirq]; + return domain->revmap[hwirq]; rcu_read_lock(); data = radix_tree_lookup(&domain->revmap_tree, hwirq); -- cgit From e37af8011a9631996e6cd32dd81a152708eee7d4 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Sun, 4 Apr 2021 13:06:39 +0100 Subject: powerpc: Move the use of irq_domain_add_nomap() behind a config option Only a handful of old PPC systems are still using the old 'nomap' variant of the irqdomain library. Move the associated definitions behind a configuration option, which will allow us to make some more radical changes. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 8bd012253989..e0143e640683 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -604,6 +604,7 @@ void irq_domain_associate_many(struct irq_domain *domain, unsigned int irq_base, } EXPORT_SYMBOL_GPL(irq_domain_associate_many); +#ifdef CONFIG_IRQ_DOMAIN_NOMAP /** * irq_create_direct_mapping() - Allocate an irq for direct mapping * @domain: domain to allocate the irq for or NULL for default domain @@ -644,6 +645,7 @@ unsigned int irq_create_direct_mapping(struct irq_domain *domain) return virq; } EXPORT_SYMBOL_GPL(irq_create_direct_mapping); +#endif /** * irq_create_mapping_affinity() - Map a hardware interrupt into linux irq space -- cgit From 4f86a06e2d6ece5316e4c42fbf946ee22acb30f3 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 10 Sep 2018 18:33:46 +0100 Subject: irqdomain: Make normal and nomap irqdomains exclusive Direct mappings are completely exclusive of normal mappings, meaning that we can refactor the code slightly so that we can get rid of the revmap_direct_max_irq field and use the revmap_size field instead, reducing the size of the irqdomain structure. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 45 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 10 deletions(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index e0143e640683..fa94c86e47d4 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -146,6 +146,10 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, static atomic_t unknown_domains; + if (WARN_ON((size && direct_max) || + (!IS_ENABLED(CONFIG_IRQ_DOMAIN_NOMAP) && direct_max))) + return NULL; + domain = kzalloc_node(sizeof(*domain) + (sizeof(unsigned int) * size), GFP_KERNEL, of_node_to_nid(to_of_node(fwnode))); if (!domain) @@ -213,8 +217,14 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, domain->ops = ops; domain->host_data = host_data; domain->hwirq_max = hwirq_max; + + if (direct_max) { + size = direct_max; + domain->flags |= IRQ_DOMAIN_FLAG_NO_MAP; + } + domain->revmap_size = size; - domain->revmap_direct_max_irq = direct_max; + irq_domain_check_hierarchy(domain); mutex_lock(&irq_domain_mutex); @@ -482,9 +492,18 @@ struct irq_domain *irq_get_default_host(void) return irq_default_domain; } +static bool irq_domain_is_nomap(struct irq_domain *domain) +{ + return IS_ENABLED(CONFIG_IRQ_DOMAIN_NOMAP) && + (domain->flags & IRQ_DOMAIN_FLAG_NO_MAP); +} + static void irq_domain_clear_mapping(struct irq_domain *domain, irq_hw_number_t hwirq) { + if (irq_domain_is_nomap(domain)) + return; + if (hwirq < domain->revmap_size) { domain->revmap[hwirq] = 0; } else { @@ -498,6 +517,9 @@ static void irq_domain_set_mapping(struct irq_domain *domain, irq_hw_number_t hwirq, struct irq_data *irq_data) { + if (irq_domain_is_nomap(domain)) + return; + if (hwirq < domain->revmap_size) { domain->revmap[hwirq] = irq_data->irq; } else { @@ -629,9 +651,9 @@ unsigned int irq_create_direct_mapping(struct irq_domain *domain) pr_debug("create_direct virq allocation failed\n"); return 0; } - if (virq >= domain->revmap_direct_max_irq) { + if (virq >= domain->revmap_size) { pr_err("ERROR: no free irqs available below %i maximum\n", - domain->revmap_direct_max_irq); + domain->revmap_size); irq_free_desc(virq); return 0; } @@ -879,10 +901,14 @@ unsigned int irq_find_mapping(struct irq_domain *domain, if (domain == NULL) return 0; - if (hwirq < domain->revmap_direct_max_irq) { - data = irq_domain_get_irq_data(domain, hwirq); - if (data && data->hwirq == hwirq) - return hwirq; + if (irq_domain_is_nomap(domain)) { + if (hwirq < domain->revmap_size) { + data = irq_domain_get_irq_data(domain, hwirq); + if (data && data->hwirq == hwirq) + return hwirq; + } + + return 0; } /* Check if the hwirq is in the linear revmap. */ @@ -1470,7 +1496,7 @@ static void irq_domain_fix_revmap(struct irq_data *d) { void __rcu **slot; - if (d->hwirq < d->domain->revmap_size) + if (irq_domain_is_nomap(d->domain) || d->hwirq < d->domain->revmap_size) return; /* Not using radix tree. */ /* Fix up the revmap. */ @@ -1830,8 +1856,7 @@ static void irq_domain_debug_show_one(struct seq_file *m, struct irq_domain *d, int ind) { seq_printf(m, "%*sname: %s\n", ind, "", d->name); - seq_printf(m, "%*ssize: %u\n", ind + 1, "", - d->revmap_size + d->revmap_direct_max_irq); + seq_printf(m, "%*ssize: %u\n", ind + 1, "", d->revmap_size); seq_printf(m, "%*smapped: %u\n", ind + 1, "", d->mapcount); seq_printf(m, "%*sflags: 0x%08x\n", ind +1 , "", d->flags); if (d->ops && d->ops->debug_show) -- cgit From 426fa316148bccabf48f9c91a13c387ee911eadc Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 5 Apr 2021 11:30:51 +0100 Subject: irqdomain: Use struct_size() helper when allocating irqdomain Instead of open-coding the size computation of struct irqdomain, use the struct_size() helper instead. This is going to be handy as we change the type of the revmap array. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index fa94c86e47d4..cdcb1989cd20 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -150,7 +150,7 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, (!IS_ENABLED(CONFIG_IRQ_DOMAIN_NOMAP) && direct_max))) return NULL; - domain = kzalloc_node(sizeof(*domain) + (sizeof(unsigned int) * size), + domain = kzalloc_node(struct_size(domain, revmap, size), GFP_KERNEL, of_node_to_nid(to_of_node(fwnode))); if (!domain) return NULL; -- cgit From 48b15a7921d60680babe59f64e127816585a585c Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 5 Apr 2021 11:46:53 +0100 Subject: irqdomain: Cache irq_data instead of a virq number in the revmap Caching a virq number in the revmap is pretty inefficient, as it means we will need to convert it back to either an irq_data or irq_desc to do anything with it. It is also a bit odd, as the radix tree does cache irq_data pointers. Change the revmap type to be an irq_data pointer instead of an unsigned int, and preserve the current API for now. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index cdcb1989cd20..7a4e38804487 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -505,7 +505,7 @@ static void irq_domain_clear_mapping(struct irq_domain *domain, return; if (hwirq < domain->revmap_size) { - domain->revmap[hwirq] = 0; + domain->revmap[hwirq] = NULL; } else { mutex_lock(&domain->revmap_tree_mutex); radix_tree_delete(&domain->revmap_tree, hwirq); @@ -521,7 +521,7 @@ static void irq_domain_set_mapping(struct irq_domain *domain, return; if (hwirq < domain->revmap_size) { - domain->revmap[hwirq] = irq_data->irq; + domain->revmap[hwirq] = irq_data; } else { mutex_lock(&domain->revmap_tree_mutex); radix_tree_insert(&domain->revmap_tree, hwirq, irq_data); @@ -913,7 +913,7 @@ unsigned int irq_find_mapping(struct irq_domain *domain, /* Check if the hwirq is in the linear revmap. */ if (hwirq < domain->revmap_size) - return domain->revmap[hwirq]; + return domain->revmap[hwirq]->irq; rcu_read_lock(); data = radix_tree_lookup(&domain->revmap_tree, hwirq); @@ -1496,8 +1496,14 @@ static void irq_domain_fix_revmap(struct irq_data *d) { void __rcu **slot; - if (irq_domain_is_nomap(d->domain) || d->hwirq < d->domain->revmap_size) - return; /* Not using radix tree. */ + if (irq_domain_is_nomap(d->domain)) + return; + + if (d->hwirq < d->domain->revmap_size) { + /* Not using radix tree */ + d->domain->revmap[d->hwirq] = d; + return; + } /* Fix up the revmap. */ mutex_lock(&d->domain->revmap_tree_mutex); -- cgit From d4a45c68dc81f9117ceaff9f058d5fae674181b9 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Mon, 5 Apr 2021 12:57:27 +0100 Subject: irqdomain: Protect the linear revmap with RCU It is pretty odd that the radix tree uses RCU while the linear portion doesn't, leading to potential surprises for the users, depending on how the irqdomain has been created. Fix this by moving the update of the linear revmap under the mutex, and the lookup under the RCU read-side lock. The mutex name is updated to reflect that it doesn't only cover the radix-tree anymore. Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 49 +++++++++++++++++++++++-------------------------- 1 file changed, 23 insertions(+), 26 deletions(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 7a4e38804487..8fbadeefc814 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -213,7 +213,7 @@ struct irq_domain *__irq_domain_add(struct fwnode_handle *fwnode, int size, /* Fill structure */ INIT_RADIX_TREE(&domain->revmap_tree, GFP_KERNEL); - mutex_init(&domain->revmap_tree_mutex); + mutex_init(&domain->revmap_mutex); domain->ops = ops; domain->host_data = host_data; domain->hwirq_max = hwirq_max; @@ -504,13 +504,12 @@ static void irq_domain_clear_mapping(struct irq_domain *domain, if (irq_domain_is_nomap(domain)) return; - if (hwirq < domain->revmap_size) { - domain->revmap[hwirq] = NULL; - } else { - mutex_lock(&domain->revmap_tree_mutex); + mutex_lock(&domain->revmap_mutex); + if (hwirq < domain->revmap_size) + rcu_assign_pointer(domain->revmap[hwirq], NULL); + else radix_tree_delete(&domain->revmap_tree, hwirq); - mutex_unlock(&domain->revmap_tree_mutex); - } + mutex_unlock(&domain->revmap_mutex); } static void irq_domain_set_mapping(struct irq_domain *domain, @@ -520,13 +519,12 @@ static void irq_domain_set_mapping(struct irq_domain *domain, if (irq_domain_is_nomap(domain)) return; - if (hwirq < domain->revmap_size) { - domain->revmap[hwirq] = irq_data; - } else { - mutex_lock(&domain->revmap_tree_mutex); + mutex_lock(&domain->revmap_mutex); + if (hwirq < domain->revmap_size) + rcu_assign_pointer(domain->revmap[hwirq], irq_data); + else radix_tree_insert(&domain->revmap_tree, hwirq, irq_data); - mutex_unlock(&domain->revmap_tree_mutex); - } + mutex_unlock(&domain->revmap_mutex); } static void irq_domain_disassociate(struct irq_domain *domain, unsigned int irq) @@ -911,12 +909,12 @@ unsigned int irq_find_mapping(struct irq_domain *domain, return 0; } + rcu_read_lock(); /* Check if the hwirq is in the linear revmap. */ if (hwirq < domain->revmap_size) - return domain->revmap[hwirq]->irq; - - rcu_read_lock(); - data = radix_tree_lookup(&domain->revmap_tree, hwirq); + data = rcu_dereference(domain->revmap[hwirq]); + else + data = radix_tree_lookup(&domain->revmap_tree, hwirq); rcu_read_unlock(); return data ? data->irq : 0; } @@ -1499,18 +1497,17 @@ static void irq_domain_fix_revmap(struct irq_data *d) if (irq_domain_is_nomap(d->domain)) return; + /* Fix up the revmap. */ + mutex_lock(&d->domain->revmap_mutex); if (d->hwirq < d->domain->revmap_size) { /* Not using radix tree */ - d->domain->revmap[d->hwirq] = d; - return; + rcu_assign_pointer(d->domain->revmap[d->hwirq], d); + } else { + slot = radix_tree_lookup_slot(&d->domain->revmap_tree, d->hwirq); + if (slot) + radix_tree_replace_slot(&d->domain->revmap_tree, slot, d); } - - /* Fix up the revmap. */ - mutex_lock(&d->domain->revmap_tree_mutex); - slot = radix_tree_lookup_slot(&d->domain->revmap_tree, d->hwirq); - if (slot) - radix_tree_replace_slot(&d->domain->revmap_tree, slot, d); - mutex_unlock(&d->domain->revmap_tree_mutex); + mutex_unlock(&d->domain->revmap_mutex); } /** -- cgit From d22558dd0a6c888b1829f9d3a0a627e330e27585 Mon Sep 17 00:00:00 2001 From: Marc Zyngier Date: Tue, 4 May 2021 14:00:13 +0100 Subject: irqdomain: Introduce irq_resolve_mapping() Rework irq_find_mapping() to return an both an irq_desc pointer, optionally the virtual irq number, and rename the result to __irq_resolve_mapping(). a new helper called irq_resolve_mapping() is provided for code that doesn't need the virtual irq number. irq_find_mapping() is also rewritten in terms of __irq_resolve_mapping(). Signed-off-by: Marc Zyngier --- kernel/irq/irqdomain.c | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) (limited to 'kernel/irq/irqdomain.c') diff --git a/kernel/irq/irqdomain.c b/kernel/irq/irqdomain.c index 8fbadeefc814..51c483ce2447 100644 --- a/kernel/irq/irqdomain.c +++ b/kernel/irq/irqdomain.c @@ -884,29 +884,34 @@ void irq_dispose_mapping(unsigned int virq) EXPORT_SYMBOL_GPL(irq_dispose_mapping); /** - * irq_find_mapping() - Find a linux irq from a hw irq number. + * __irq_resolve_mapping() - Find a linux irq from a hw irq number. * @domain: domain owning this hardware interrupt * @hwirq: hardware irq number in that domain space + * @irq: optional pointer to return the Linux irq if required + * + * Returns the interrupt descriptor. */ -unsigned int irq_find_mapping(struct irq_domain *domain, - irq_hw_number_t hwirq) +struct irq_desc *__irq_resolve_mapping(struct irq_domain *domain, + irq_hw_number_t hwirq, + unsigned int *irq) { + struct irq_desc *desc = NULL; struct irq_data *data; /* Look for default domain if necessary */ if (domain == NULL) domain = irq_default_domain; if (domain == NULL) - return 0; + return desc; if (irq_domain_is_nomap(domain)) { if (hwirq < domain->revmap_size) { data = irq_domain_get_irq_data(domain, hwirq); if (data && data->hwirq == hwirq) - return hwirq; + desc = irq_data_to_desc(data); } - return 0; + return desc; } rcu_read_lock(); @@ -915,10 +920,17 @@ unsigned int irq_find_mapping(struct irq_domain *domain, data = rcu_dereference(domain->revmap[hwirq]); else data = radix_tree_lookup(&domain->revmap_tree, hwirq); + + if (likely(data)) { + desc = irq_data_to_desc(data); + if (irq) + *irq = data->irq; + } + rcu_read_unlock(); - return data ? data->irq : 0; + return desc; } -EXPORT_SYMBOL_GPL(irq_find_mapping); +EXPORT_SYMBOL_GPL(__irq_resolve_mapping); /** * irq_domain_xlate_onecell() - Generic xlate for direct one cell bindings -- cgit