diff options
Diffstat (limited to 'drivers/infiniband/hw/hfi1/affinity.c')
| -rw-r--r-- | drivers/infiniband/hw/hfi1/affinity.c | 181 |
1 files changed, 53 insertions, 128 deletions
diff --git a/drivers/infiniband/hw/hfi1/affinity.c b/drivers/infiniband/hw/hfi1/affinity.c index 2a91b8d95e12..ee7fedc67b86 100644 --- a/drivers/infiniband/hw/hfi1/affinity.c +++ b/drivers/infiniband/hw/hfi1/affinity.c @@ -1,52 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* * Copyright(c) 2015 - 2020 Intel Corporation. - * - * This file is provided under a dual BSD/GPLv2 license. When using or - * redistributing this file, you may do so under either license. - * - * GPL LICENSE SUMMARY - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of version 2 of the GNU General Public License as - * published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * General Public License for more details. - * - * BSD LICENSE - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * - * - Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * - Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in - * the documentation and/or other materials provided with the - * distribution. - * - Neither the name of Intel Corporation nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - * */ + #include <linux/topology.h> #include <linux/cpumask.h> -#include <linux/module.h> #include <linux/interrupt.h> #include <linux/numa.h> @@ -134,9 +92,7 @@ static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu) /* Initialize non-HT cpu cores mask */ void init_real_cpu_mask(void) { - int possible, curr_cpu, i, ht; - - cpumask_clear(&node_affinity.real_cpu_mask); + int possible, curr_cpu, ht; /* Start with cpu online mask as the real cpu mask */ cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask); @@ -152,17 +108,10 @@ void init_real_cpu_mask(void) * "real" cores. Assumes that HT cores are not enumerated in * succession (except in the single core case). */ - curr_cpu = cpumask_first(&node_affinity.real_cpu_mask); - for (i = 0; i < possible / ht; i++) - curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); - /* - * Step 2. Remove the remaining HT siblings. Use cpumask_next() to - * skip any gaps. - */ - for (; i < possible; i++) { - cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask); - curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask); - } + curr_cpu = cpumask_nth(possible / ht, &node_affinity.real_cpu_mask) + 1; + + /* Step 2. Remove the remaining HT siblings. */ + cpumask_clear_cpus(&node_affinity.real_cpu_mask, curr_cpu, nr_cpu_ids - curr_cpu); } int node_affinity_init(void) @@ -219,6 +168,8 @@ out: for (node = 0; node < node_affinity.num_possible_nodes; node++) hfi1_per_node_cntr[node] = 1; + pci_dev_put(dev); + return 0; } @@ -270,11 +221,9 @@ static void node_affinity_add_tail(struct hfi1_affinity_node *entry) /* It must be called with node_affinity.lock held */ static struct hfi1_affinity_node *node_affinity_lookup(int node) { - struct list_head *pos; struct hfi1_affinity_node *entry; - list_for_each(pos, &node_affinity.list) { - entry = list_entry(pos, struct hfi1_affinity_node, list); + list_for_each_entry(entry, &node_affinity.list, list) { if (entry->node == node) return entry; } @@ -388,9 +337,10 @@ static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd, &entry->def_intr.used); /* If there are non-interrupt CPUs available, use them first */ - if (!cpumask_empty(non_intr_cpus)) - cpu = cpumask_first(non_intr_cpus); - else /* Otherwise, use interrupt CPUs */ + cpu = cpumask_first(non_intr_cpus); + + /* Otherwise, use interrupt CPUs */ + if (cpu >= nr_cpu_ids) cpu = cpumask_first(available_cpus); if (cpu >= nr_cpu_ids) { /* empty */ @@ -632,22 +582,11 @@ static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd, */ int hfi1_dev_affinity_init(struct hfi1_devdata *dd) { - int node = pcibus_to_node(dd->pcidev->bus); struct hfi1_affinity_node *entry; const struct cpumask *local_mask; int curr_cpu, possible, i, ret; bool new_entry = false; - /* - * If the BIOS does not have the NUMA node information set, select - * NUMA 0 so we get consistent performance. - */ - if (node < 0) { - dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n"); - node = 0; - } - dd->node = node; - local_mask = cpumask_of_node(dd->node); if (cpumask_first(local_mask) >= nr_cpu_ids) local_mask = topology_core_cpumask(0); @@ -660,7 +599,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd) * create an entry in the global affinity structure and initialize it. */ if (!entry) { - entry = node_affinity_allocate(node); + entry = node_affinity_allocate(dd->node); if (!entry) { dd_dev_err(dd, "Unable to allocate global affinity node\n"); @@ -719,7 +658,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd) * engines, use the same CPU cores as general/control * context. */ - if (cpumask_weight(&entry->def_intr.mask) == 0) + if (cpumask_empty(&entry->def_intr.mask)) cpumask_copy(&entry->def_intr.mask, &entry->general_intr_mask); } @@ -739,7 +678,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd) * vectors, use the same CPU core as the general/control * context. */ - if (cpumask_weight(&entry->comp_vect_mask) == 0) + if (cpumask_empty(&entry->comp_vect_mask)) cpumask_copy(&entry->comp_vect_mask, &entry->general_intr_mask); } @@ -751,6 +690,7 @@ int hfi1_dev_affinity_init(struct hfi1_devdata *dd) if (new_entry) node_affinity_add_tail(entry); + dd->affinity_entry = entry; mutex_unlock(&node_affinity.lock); return 0; @@ -766,10 +706,9 @@ void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd) { struct hfi1_affinity_node *entry; - if (dd->node < 0) - return; - mutex_lock(&node_affinity.lock); + if (!dd->affinity_entry) + goto unlock; entry = node_affinity_lookup(dd->node); if (!entry) goto unlock; @@ -780,8 +719,8 @@ void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd) */ _dev_comp_vect_cpu_mask_clean_up(dd, entry); unlock: + dd->affinity_entry = NULL; mutex_unlock(&node_affinity.lock); - dd->node = NUMA_NO_NODE; } /* @@ -973,7 +912,6 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix) { struct cpu_mask_set *set = NULL; - struct hfi1_ctxtdata *rcd; struct hfi1_affinity_node *entry; mutex_lock(&node_affinity.lock); @@ -987,14 +925,15 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, case IRQ_GENERAL: /* Don't do accounting for general contexts */ break; - case IRQ_RCVCTXT: - rcd = (struct hfi1_ctxtdata *)msix->arg; + case IRQ_RCVCTXT: { + struct hfi1_ctxtdata *rcd = msix->arg; + /* Don't do accounting for control contexts */ if (rcd->ctxt != HFI1_CTRL_CTXT) set = &entry->rcv_intr; break; + } case IRQ_NETDEVCTXT: - rcd = (struct hfi1_ctxtdata *)msix->arg; set = &entry->def_intr; break; default: @@ -1016,32 +955,23 @@ void hfi1_put_irq_affinity(struct hfi1_devdata *dd, static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask, struct hfi1_affinity_node_list *affinity) { - int possible, curr_cpu, i; - uint num_cores_per_socket = node_affinity.num_online_cpus / - affinity->num_core_siblings / - node_affinity.num_online_nodes; + int curr_cpu; + uint num_cores; cpumask_copy(hw_thread_mask, &affinity->proc.mask); - if (affinity->num_core_siblings > 0) { - /* Removing other siblings not needed for now */ - possible = cpumask_weight(hw_thread_mask); - curr_cpu = cpumask_first(hw_thread_mask); - for (i = 0; - i < num_cores_per_socket * node_affinity.num_online_nodes; - i++) - curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); - - for (; i < possible; i++) { - cpumask_clear_cpu(curr_cpu, hw_thread_mask); - curr_cpu = cpumask_next(curr_cpu, hw_thread_mask); - } - /* Identifying correct HW threads within physical cores */ - cpumask_shift_left(hw_thread_mask, hw_thread_mask, - num_cores_per_socket * - node_affinity.num_online_nodes * - hw_thread_no); - } + if (affinity->num_core_siblings == 0) + return; + + num_cores = rounddown(node_affinity.num_online_cpus / affinity->num_core_siblings, + node_affinity.num_online_nodes); + + /* Removing other siblings not needed for now */ + curr_cpu = cpumask_nth(num_cores * node_affinity.num_online_nodes, hw_thread_mask) + 1; + cpumask_clear_cpus(hw_thread_mask, curr_cpu, nr_cpu_ids - curr_cpu); + + /* Identifying correct HW threads within physical cores */ + cpumask_shift_left(hw_thread_mask, hw_thread_mask, num_cores * hw_thread_no); } int hfi1_get_proc_affinity(int node) @@ -1140,22 +1070,19 @@ int hfi1_get_proc_affinity(int node) * If HT cores are enabled, identify which HW threads within the * physical cores should be used. */ - if (affinity->num_core_siblings > 0) { - for (i = 0; i < affinity->num_core_siblings; i++) { - find_hw_thread_mask(i, hw_thread_mask, affinity); + for (i = 0; i < affinity->num_core_siblings; i++) { + find_hw_thread_mask(i, hw_thread_mask, affinity); - /* - * If there's at least one available core for this HW - * thread number, stop looking for a core. - * - * diff will always be not empty at least once in this - * loop as the used mask gets reset when - * (set->mask == set->used) before this loop. - */ - cpumask_andnot(diff, hw_thread_mask, &set->used); - if (!cpumask_empty(diff)) - break; - } + /* + * If there's at least one available core for this HW + * thread number, stop looking for a core. + * + * diff will always be not empty at least once in this + * loop as the used mask gets reset when + * (set->mask == set->used) before this loop. + */ + if (cpumask_andnot(diff, hw_thread_mask, &set->used)) + break; } hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl", cpumask_pr_args(hw_thread_mask)); @@ -1186,8 +1113,7 @@ int hfi1_get_proc_affinity(int node) * used for process assignments using the same method as * the preferred NUMA node. */ - cpumask_andnot(diff, available_mask, intrs_mask); - if (!cpumask_empty(diff)) + if (cpumask_andnot(diff, available_mask, intrs_mask)) cpumask_copy(available_mask, diff); /* If we don't have CPUs on the preferred node, use other NUMA nodes */ @@ -1203,8 +1129,7 @@ int hfi1_get_proc_affinity(int node) * At first, we don't want to place processes on the same * CPUs as interrupt handlers. */ - cpumask_andnot(diff, available_mask, intrs_mask); - if (!cpumask_empty(diff)) + if (cpumask_andnot(diff, available_mask, intrs_mask)) cpumask_copy(available_mask, diff); } hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl", |