// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2019 Mellanox Technologies. */ #include #include #include #include #include "mlx5_core.h" #include "mlx5_irq.h" #include "lib/sf.h" #ifdef CONFIG_RFS_ACCEL #include #endif #define MLX5_MAX_IRQ_NAME (32) /* max irq_index is 2047, so four chars */ #define MLX5_MAX_IRQ_IDX_CHARS (4) #define MLX5_SFS_PER_CTRL_IRQ 64 #define MLX5_IRQ_CTRL_SF_MAX 8 /* min num of vectors for SFs to be enabled */ #define MLX5_IRQ_VEC_COMP_BASE_SF 2 #define MLX5_EQ_SHARE_IRQ_MAX_COMP (8) #define MLX5_EQ_SHARE_IRQ_MAX_CTRL (UINT_MAX) #define MLX5_EQ_SHARE_IRQ_MIN_COMP (1) #define MLX5_EQ_SHARE_IRQ_MIN_CTRL (4) #define MLX5_EQ_REFS_PER_IRQ (2) struct mlx5_irq { struct atomic_notifier_head nh; cpumask_var_t mask; char name[MLX5_MAX_IRQ_NAME]; struct mlx5_irq_pool *pool; int refcount; u32 index; int irqn; }; struct mlx5_irq_pool { char name[MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS]; struct xa_limit xa_num_irqs; struct mutex lock; /* sync IRQs creations */ struct xarray irqs; u32 max_threshold; u32 min_threshold; struct mlx5_core_dev *dev; }; struct mlx5_irq_table { struct mlx5_irq_pool *pf_pool; struct mlx5_irq_pool *sf_ctrl_pool; struct mlx5_irq_pool *sf_comp_pool; }; /** * mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors * to be ssigned to each VF. * @dev: PF to work on * @num_vfs: Number of enabled VFs */ int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs) { int num_vf_msix, min_msix, max_msix; num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix); if (!num_vf_msix) return 0; min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size); max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size); /* Limit maximum number of MSI-X vectors so the default configuration * has some available in the pool. This will allow the user to increase * the number of vectors in a VF without having to first size-down other * VFs. */ return max(min(num_vf_msix / num_vfs, max_msix / 2), min_msix); } /** * mlx5_set_msix_vec_count - Set dynamically allocated MSI-X on the VF * @dev: PF to work on * @function_id: Internal PCI VF function IDd * @msix_vec_count: Number of MSI-X vectors to set */ int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id, int msix_vec_count) { int query_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in); void *hca_cap = NULL, *query_cap = NULL, *cap; int num_vf_msix, min_msix, max_msix; int ret; num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix); if (!num_vf_msix) return 0; if (!MLX5_CAP_GEN(dev, vport_group_manager) || !mlx5_core_is_pf(dev)) return -EOPNOTSUPP; min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size); max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size); if (msix_vec_count < min_msix) return -EINVAL; if (msix_vec_count > max_msix) return -EOVERFLOW; query_cap = kzalloc(query_sz, GFP_KERNEL); hca_cap = kzalloc(set_sz, GFP_KERNEL); if (!hca_cap || !query_cap) { ret = -ENOMEM; goto out; } ret = mlx5_vport_get_other_func_cap(dev, function_id, query_cap); if (ret) goto out; cap = MLX5_ADDR_OF(set_hca_cap_in, hca_cap, capability); memcpy(cap, MLX5_ADDR_OF(query_hca_cap_out, query_cap, capability), MLX5_UN_SZ_BYTES(hca_cap_union)); MLX5_SET(cmd_hca_cap, cap, dynamic_msix_table_size, msix_vec_count); MLX5_SET(set_hca_cap_in, hca_cap, opcode, MLX5_CMD_OP_SET_HCA_CAP); MLX5_SET(set_hca_cap_in, hca_cap, other_function, 1); MLX5_SET(set_hca_cap_in, hca_cap, function_id, function_id); MLX5_SET(set_hca_cap_in, hca_cap, op_mod, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1); ret = mlx5_cmd_exec_in(dev, set_hca_cap, hca_cap); out: kfree(hca_cap); kfree(query_cap); return ret; } static void irq_release(struct mlx5_irq *irq) { struct mlx5_irq_pool *pool = irq->pool; xa_erase(&pool->irqs, irq->index); /* free_irq requires that affinity and rmap will be cleared * before calling it. This is why there is asymmetry with set_rmap * which should be called after alloc_irq but before request_irq. */ irq_set_affinity_hint(irq->irqn, NULL); free_cpumask_var(irq->mask); free_irq(irq->irqn, &irq->nh); kfree(irq); } static void irq_put(struct mlx5_irq *irq) { struct mlx5_irq_pool *pool = irq->pool; mutex_lock(&pool->lock); irq->refcount--; if (!irq->refcount) irq_release(irq); mutex_unlock(&pool->lock); } static int irq_get_locked(struct mlx5_irq *irq) { lockdep_assert_held(&irq->pool->lock); if (WARN_ON_ONCE(!irq->refcount)) return 0; irq->refcount++; return 1; } static int irq_get(struct mlx5_irq *irq) { int err; mutex_lock(&irq->pool->lock); err = irq_get_locked(irq); mutex_unlock(&irq->pool->lock); return err; } static irqreturn_t irq_int_handler(int irq, void *nh) { atomic_notifier_call_chain(nh, 0, NULL); return IRQ_HANDLED; } static void irq_sf_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx) { snprintf(name, MLX5_MAX_IRQ_NAME, "%s%d", pool->name, vecidx); } static void irq_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx) { if (!pool->xa_num_irqs.max) { /* in case we only have a single irq for the device */ snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_combined%d", vecidx); return; } if (vecidx == pool->xa_num_irqs.max) { snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async%d", vecidx); return; } snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", vecidx); } static bool irq_pool_is_sf_pool(struct mlx5_irq_pool *pool) { return !strncmp("mlx5_sf", pool->name, strlen("mlx5_sf")); } static struct mlx5_irq *irq_request(struct mlx5_irq_pool *pool, int i) { struct mlx5_core_dev *dev = pool->dev; char name[MLX5_MAX_IRQ_NAME]; struct mlx5_irq *irq; int err; irq = kzalloc(sizeof(*irq), GFP_KERNEL); if (!irq) return ERR_PTR(-ENOMEM); irq->irqn = pci_irq_vector(dev->pdev, i); if (!irq_pool_is_sf_pool(pool)) irq_set_name(pool, name, i); else irq_sf_set_name(pool, name, i); ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh); snprintf(irq->name, MLX5_MAX_IRQ_NAME, "%s@pci:%s", name, pci_name(dev->pdev)); err = request_irq(irq->irqn, irq_int_handler, 0, irq->name, &irq->nh); if (err) { mlx5_core_err(dev, "Failed to request irq. err = %d\n", err); goto err_req_irq; } if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) { mlx5_core_warn(dev, "zalloc_cpumask_var failed\n"); err = -ENOMEM; goto err_cpumask; } irq->pool = pool; irq->refcount = 1; irq->index = i; err = xa_err(xa_store(&pool->irqs, irq->index, irq, GFP_KERNEL)); if (err) { mlx5_core_err(dev, "Failed to alloc xa entry for irq(%u). err = %d\n", irq->index, err); goto err_xa; } return irq; err_xa: free_cpumask_var(irq->mask); err_cpumask: free_irq(irq->irqn, &irq->nh); err_req_irq: kfree(irq); return ERR_PTR(err); } int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb) { int ret; ret = irq_get(irq); if (!ret) /* Something very bad happens here, we are enabling EQ * on non-existing IRQ. */ return -ENOENT; ret = atomic_notifier_chain_register(&irq->nh, nb); if (ret) irq_put(irq); return ret; } int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb) { int err = 0; err = atomic_notifier_chain_unregister(&irq->nh, nb); irq_put(irq); return err; } struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq) { return irq->mask; } int mlx5_irq_get_index(struct mlx5_irq *irq) { return irq->index; } /* irq_pool API */ /* creating an irq from irq_pool */ static struct mlx5_irq *irq_pool_create_irq(struct mlx5_irq_pool *pool, struct cpumask *affinity) { struct mlx5_irq *irq; u32 irq_index; int err; err = xa_alloc(&pool->irqs, &irq_index, NULL, pool->xa_num_irqs, GFP_KERNEL); if (err) return ERR_PTR(err); irq = irq_request(pool, irq_index); if (IS_ERR(irq)) return irq; cpumask_copy(irq->mask, affinity); irq_set_affinity_hint(irq->irqn, irq->mask); return irq; } /* looking for the irq with the smallest refcount and the same affinity */ static struct mlx5_irq *irq_pool_find_least_loaded(struct mlx5_irq_pool *pool, struct cpumask *affinity) { int start = pool->xa_num_irqs.min; int end = pool->xa_num_irqs.max; struct mlx5_irq *irq = NULL; struct mlx5_irq *iter; unsigned long index; lockdep_assert_held(&pool->lock); xa_for_each_range(&pool->irqs, index, iter, start, end) { if (!cpumask_equal(iter->mask, affinity)) continue; if (iter->refcount < pool->min_threshold) return iter; if (!irq || iter->refcount < irq->refcount) irq = iter; } return irq; } /* requesting an irq from a given pool according to given affinity */ static struct mlx5_irq *irq_pool_request_affinity(struct mlx5_irq_pool *pool, struct cpumask *affinity) { struct mlx5_irq *least_loaded_irq, *new_irq; mutex_lock(&pool->lock); least_loaded_irq = irq_pool_find_least_loaded(pool, affinity); if (least_loaded_irq && least_loaded_irq->refcount < pool->min_threshold) goto out; new_irq = irq_pool_create_irq(pool, affinity); if (IS_ERR(new_irq)) { if (!least_loaded_irq) { mlx5_core_err(pool->dev, "Didn't find a matching IRQ. err = %ld\n", PTR_ERR(new_irq)); mutex_unlock(&pool->lock); return new_irq; } /* We failed to create a new IRQ for the requested affinity, * sharing existing IRQ. */ goto out; } least_loaded_irq = new_irq; goto unlock; out: irq_get_locked(least_loaded_irq); if (least_loaded_irq->refcount > pool->max_threshold) mlx5_core_dbg(pool->dev, "IRQ %u overloaded, pool_name: %s, %u EQs on this irq\n", least_loaded_irq->irqn, pool->name, least_loaded_irq->refcount / MLX5_EQ_REFS_PER_IRQ); unlock: mutex_unlock(&pool->lock); return least_loaded_irq; } /* requesting an irq from a given pool according to given index */ static struct mlx5_irq * irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx, struct cpumask *affinity) { struct mlx5_irq *irq; mutex_lock(&pool->lock); irq = xa_load(&pool->irqs, vecidx); if (irq) { irq_get_locked(irq); goto unlock; } irq = irq_request(pool, vecidx); if (IS_ERR(irq) || !affinity) goto unlock; cpumask_copy(irq->mask, affinity); if (!irq_pool_is_sf_pool(pool) && !pool->xa_num_irqs.max && cpumask_empty(irq->mask)) cpumask_set_cpu(cpumask_first(cpu_online_mask), irq->mask); irq_set_affinity_hint(irq->irqn, irq->mask); unlock: mutex_unlock(&pool->lock); return irq; } static struct mlx5_irq_pool *find_sf_irq_pool(struct mlx5_irq_table *irq_table, int i, struct cpumask *affinity) { if (cpumask_empty(affinity) && i == MLX5_IRQ_EQ_CTRL) return irq_table->sf_ctrl_pool; return irq_table->sf_comp_pool; } /** * mlx5_irq_release - release an IRQ back to the system. * @irq: irq to be released. */ void mlx5_irq_release(struct mlx5_irq *irq) { synchronize_irq(irq->irqn); irq_put(irq); } /** * mlx5_irq_request - request an IRQ for mlx5 device. * @dev: mlx5 device that requesting the IRQ. * @vecidx: vector index of the IRQ. This argument is ignore if affinity is * provided. * @affinity: cpumask requested for this IRQ. * * This function returns a pointer to IRQ, or ERR_PTR in case of error. */ struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx, struct cpumask *affinity) { struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev); struct mlx5_irq_pool *pool; struct mlx5_irq *irq; if (mlx5_core_is_sf(dev)) { pool = find_sf_irq_pool(irq_table, vecidx, affinity); if (!pool) /* we don't have IRQs for SFs, using the PF IRQs */ goto pf_irq; if (cpumask_empty(affinity) && !strcmp(pool->name, "mlx5_sf_comp")) /* In case an SF user request IRQ with vecidx */ irq = irq_pool_request_vector(pool, vecidx, NULL); else irq = irq_pool_request_affinity(pool, affinity); goto out; } pf_irq: pool = irq_table->pf_pool; vecidx = (vecidx == MLX5_IRQ_EQ_CTRL) ? pool->xa_num_irqs.max : vecidx; irq = irq_pool_request_vector(pool, vecidx, affinity); out: if (IS_ERR(irq)) return irq; mlx5_core_dbg(dev, "irq %u mapped to cpu %*pbl, %u EQs on this irq\n", irq->irqn, cpumask_pr_args(affinity), irq->refcount / MLX5_EQ_REFS_PER_IRQ); return irq; } static struct mlx5_irq_pool * irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name, u32 min_threshold, u32 max_threshold) { struct mlx5_irq_pool *pool = kvzalloc(sizeof(*pool), GFP_KERNEL); if (!pool) return ERR_PTR(-ENOMEM); pool->dev = dev; mutex_init(&pool->lock); xa_init_flags(&pool->irqs, XA_FLAGS_ALLOC); pool->xa_num_irqs.min = start; pool->xa_num_irqs.max = start + size - 1; if (name) snprintf(pool->name, MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS, name); pool->min_threshold = min_threshold * MLX5_EQ_REFS_PER_IRQ; pool->max_threshold = max_threshold * MLX5_EQ_REFS_PER_IRQ; mlx5_core_dbg(dev, "pool->name = %s, pool->size = %d, pool->start = %d", name, size, start); return pool; } static void irq_pool_free(struct mlx5_irq_pool *pool) { struct mlx5_irq *irq; unsigned long index; /* There are cases in which we are destrying the irq_table before * freeing all the IRQs, fast teardown for example. Hence, free the irqs * which might not have been freed. */ xa_for_each(&pool->irqs, index, irq) irq_release(irq); xa_destroy(&pool->irqs); mutex_destroy(&pool->lock); kvfree(pool); } static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pf_vec) { struct mlx5_irq_table *table = dev->priv.irq_table; int num_sf_ctrl_by_msix; int num_sf_ctrl_by_sfs; int num_sf_ctrl; int err; /* init pf_pool */ table->pf_pool = irq_pool_alloc(dev, 0, pf_vec, NULL, MLX5_EQ_SHARE_IRQ_MIN_COMP, MLX5_EQ_SHARE_IRQ_MAX_COMP); if (IS_ERR(table->pf_pool)) return PTR_ERR(table->pf_pool); if (!mlx5_sf_max_functions(dev)) return 0; if (sf_vec < MLX5_IRQ_VEC_COMP_BASE_SF) { mlx5_core_dbg(dev, "Not enught IRQs for SFs. SF may run at lower performance\n"); return 0; } /* init sf_ctrl_pool */ num_sf_ctrl_by_msix = DIV_ROUND_UP(sf_vec, MLX5_COMP_EQS_PER_SF); num_sf_ctrl_by_sfs = DIV_ROUND_UP(mlx5_sf_max_functions(dev), MLX5_SFS_PER_CTRL_IRQ); num_sf_ctrl = min_t(int, num_sf_ctrl_by_msix, num_sf_ctrl_by_sfs); num_sf_ctrl = min_t(int, MLX5_IRQ_CTRL_SF_MAX, num_sf_ctrl); table->sf_ctrl_pool = irq_pool_alloc(dev, pf_vec, num_sf_ctrl, "mlx5_sf_ctrl", MLX5_EQ_SHARE_IRQ_MIN_CTRL, MLX5_EQ_SHARE_IRQ_MAX_CTRL); if (IS_ERR(table->sf_ctrl_pool)) { err = PTR_ERR(table->sf_ctrl_pool); goto err_pf; } /* init sf_comp_pool */ table->sf_comp_pool = irq_pool_alloc(dev, pf_vec + num_sf_ctrl, sf_vec - num_sf_ctrl, "mlx5_sf_comp", MLX5_EQ_SHARE_IRQ_MIN_COMP, MLX5_EQ_SHARE_IRQ_MAX_COMP); if (IS_ERR(table->sf_comp_pool)) { err = PTR_ERR(table->sf_comp_pool); goto err_sf_ctrl; } return 0; err_sf_ctrl: irq_pool_free(table->sf_ctrl_pool); err_pf: irq_pool_free(table->pf_pool); return err; } static void irq_pools_destroy(struct mlx5_irq_table *table) { if (table->sf_ctrl_pool) { irq_pool_free(table->sf_comp_pool); irq_pool_free(table->sf_ctrl_pool); } irq_pool_free(table->pf_pool); } /* irq_table API */ int mlx5_irq_table_init(struct mlx5_core_dev *dev) { struct mlx5_irq_table *irq_table; if (mlx5_core_is_sf(dev)) return 0; irq_table = kvzalloc(sizeof(*irq_table), GFP_KERNEL); if (!irq_table) return -ENOMEM; dev->priv.irq_table = irq_table; return 0; } void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev) { if (mlx5_core_is_sf(dev)) return; kvfree(dev->priv.irq_table); } int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table) { if (!table->pf_pool->xa_num_irqs.max) return 1; return table->pf_pool->xa_num_irqs.max - table->pf_pool->xa_num_irqs.min; } int mlx5_irq_table_create(struct mlx5_core_dev *dev) { int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ? MLX5_CAP_GEN(dev, max_num_eqs) : 1 << MLX5_CAP_GEN(dev, log_max_eq); int total_vec; int pf_vec; int err; if (mlx5_core_is_sf(dev)) return 0; pf_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 1; pf_vec = min_t(int, pf_vec, num_eqs); total_vec = pf_vec; if (mlx5_sf_max_functions(dev)) total_vec += MLX5_IRQ_CTRL_SF_MAX + MLX5_COMP_EQS_PER_SF * mlx5_sf_max_functions(dev); total_vec = pci_alloc_irq_vectors(dev->pdev, 1, total_vec, PCI_IRQ_MSIX); if (total_vec < 0) return total_vec; pf_vec = min(pf_vec, total_vec); err = irq_pools_init(dev, total_vec - pf_vec, pf_vec); if (err) pci_free_irq_vectors(dev->pdev); return err; } void mlx5_irq_table_destroy(struct mlx5_core_dev *dev) { struct mlx5_irq_table *table = dev->priv.irq_table; if (mlx5_core_is_sf(dev)) return; /* There are cases where IRQs still will be in used when we reaching * to here. Hence, making sure all the irqs are released. */ irq_pools_destroy(table); pci_free_irq_vectors(dev->pdev); } int mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table *table) { if (table->sf_comp_pool) return min_t(int, num_online_cpus(), table->sf_comp_pool->xa_num_irqs.max - table->sf_comp_pool->xa_num_irqs.min + 1); else return mlx5_irq_table_get_num_comp(table); } struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev) { #ifdef CONFIG_MLX5_SF if (mlx5_core_is_sf(dev)) return dev->priv.parent_mdev->priv.irq_table; #endif return dev->priv.irq_table; }