From 2c925db0a7d69b404d6bfe4c037935c2d367913d Mon Sep 17 00:00:00 2001 From: Ofer Levi Date: Tue, 9 Feb 2021 17:48:11 +0200 Subject: net/mlx5e: Support enhanced CQE compression CQE compression feature improves performance by reducing PCI bandwidth bottleneck on CQEs write. Enhanced CQE compression introduced in ConnectX-6 and it aims to reduce CPU utilization of SW side packets decompression by eliminating the need to rewrite ownership bit, which is likely to cost a cache-miss, is replaced by validity byte handled solely by HW. Another advantage of the enhanced feature is that session packets are available to SW as soon as a single CQE slot is filled, instead of waiting for session to close, this improves packet latency from NIC to host. Performance: Following are tested scenarios and reults comparing basic and enahnced CQE compression. setup: IXIA 100GbE connected directly to port 0 and port 1 of ConnectX-6 Dx 100GbE dual port. Case #1 RX only, single flow goes to single queue: IRQ rate reduced by ~ 30%, CPU utilization improved by 2%. Case #2 IP forwarding from port 1 to port 0 single flow goes to single queue: Avg latency improved from 60us to 21us, frame loss improved from 0.5% to 0.0%. Case #3 IP forwarding from port 1 to port 0 Max Throughput IXIA sends 100%, 8192 UDP flows, goes to 24 queues: Enhanced is equal or slightly better than basic. Testing the basic compression feature with this patch shows there is no perfrormance degradation of the basic compression feature. Signed-off-by: Ofer Levi Reviewed-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 150 ++++++++++++++++++++---- 1 file changed, 129 insertions(+), 21 deletions(-) (limited to 'drivers/net/ethernet/mellanox/mlx5/core/en_rx.c') diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index a61a43fc8d5c..b1ea0b995d9c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -89,6 +89,25 @@ static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq, memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64)); } +static void mlx5e_read_enhanced_title_slot(struct mlx5e_rq *rq, + struct mlx5_cqe64 *cqe) +{ + struct mlx5e_cq_decomp *cqd = &rq->cqd; + struct mlx5_cqe64 *title = &cqd->title; + + memcpy(title, cqe, sizeof(struct mlx5_cqe64)); + + if (likely(test_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state))) + return; + + if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) + cqd->wqe_counter = mpwrq_get_cqe_stride_index(title) + + mpwrq_get_cqe_consumed_strides(title); + else + cqd->wqe_counter = + mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, be16_to_cpu(title->wqe_counter) + 1); +} + static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq, struct mlx5_cqwq *wq, u32 cqcc) @@ -175,6 +194,38 @@ static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq, cqd->title.rss_hash_result = 0; } +static u32 mlx5e_decompress_enhanced_cqe(struct mlx5e_rq *rq, + struct mlx5_cqwq *wq, + struct mlx5_cqe64 *cqe, + int budget_rem) +{ + struct mlx5e_cq_decomp *cqd = &rq->cqd; + u32 cqcc, left; + u32 i; + + left = get_cqe_enhanced_num_mini_cqes(cqe); + /* Here we avoid breaking the cqe compression session in the middle + * in case budget is not sufficient to handle all of it. In this case + * we return work_done == budget_rem to give 'busy' napi indication. + */ + if (unlikely(left > budget_rem)) + return budget_rem; + + cqcc = wq->cc; + cqd->mini_arr_idx = 0; + memcpy(cqd->mini_arr, cqe, sizeof(struct mlx5_cqe64)); + for (i = 0; i < left; i++, cqd->mini_arr_idx++, cqcc++) { + mlx5e_decompress_cqe_no_hash(rq, wq, cqcc); + INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, + mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, + rq, &cqd->title); + } + wq->cc = cqcc; + rq->stats->cqe_compress_pkts += left; + + return left; +} + static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq, struct mlx5_cqwq *wq, int update_owner_only, @@ -220,7 +271,7 @@ static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq, rq, &cqd->title); cqd->mini_arr_idx++; - return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1; + return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem); } static inline bool mlx5e_rx_cache_put(struct mlx5e_rq *rq, struct page *page) @@ -2211,45 +2262,102 @@ mpwrq_cqe_out: mlx5_wq_ll_pop(wq, cqe->wqe_id, &wqe->next.next_wqe_index); } -int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) +static int mlx5e_rx_cq_process_enhanced_cqe_comp(struct mlx5e_rq *rq, + struct mlx5_cqwq *cqwq, + int budget_rem) { - struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); - struct mlx5_cqwq *cqwq = &cq->wq; - struct mlx5_cqe64 *cqe; + struct mlx5_cqe64 *cqe, *title_cqe = NULL; + struct mlx5e_cq_decomp *cqd = &rq->cqd; int work_done = 0; - if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) - return 0; + cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq); + if (!cqe) + return work_done; - if (rq->cqd.left) { - work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget); - if (work_done >= budget) - goto out; + if (cqd->last_cqe_title && + (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED)) { + rq->stats->cqe_compress_blks++; + cqd->last_cqe_title = false; } - cqe = mlx5_cqwq_get_cqe(cqwq); - if (!cqe) { - if (unlikely(work_done)) - goto out; - return 0; + do { + if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { + if (title_cqe) { + mlx5e_read_enhanced_title_slot(rq, title_cqe); + title_cqe = NULL; + rq->stats->cqe_compress_blks++; + } + work_done += + mlx5e_decompress_enhanced_cqe(rq, cqwq, cqe, + budget_rem - work_done); + continue; + } + title_cqe = cqe; + mlx5_cqwq_pop(cqwq); + + INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, + mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, + rq, cqe); + work_done++; + } while (work_done < budget_rem && + (cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq))); + + /* last cqe might be title on next poll bulk */ + if (title_cqe) { + mlx5e_read_enhanced_title_slot(rq, title_cqe); + cqd->last_cqe_title = true; } - do { + return work_done; +} + +static int mlx5e_rx_cq_process_basic_cqe_comp(struct mlx5e_rq *rq, + struct mlx5_cqwq *cqwq, + int budget_rem) +{ + struct mlx5_cqe64 *cqe; + int work_done = 0; + + if (rq->cqd.left) + work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget_rem); + + while (work_done < budget_rem && (cqe = mlx5_cqwq_get_cqe(cqwq))) { if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) { work_done += mlx5e_decompress_cqes_start(rq, cqwq, - budget - work_done); + budget_rem - work_done); continue; } mlx5_cqwq_pop(cqwq); - INDIRECT_CALL_3(rq->handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq, mlx5e_handle_rx_cqe, mlx5e_handle_rx_cqe_mpwrq_shampo, rq, cqe); - } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq))); + work_done++; + } + + return work_done; +} + +int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget) +{ + struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq); + struct mlx5_cqwq *cqwq = &cq->wq; + int work_done; + + if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state))) + return 0; + + if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) + work_done = mlx5e_rx_cq_process_enhanced_cqe_comp(rq, cqwq, + budget); + else + work_done = mlx5e_rx_cq_process_basic_cqe_comp(rq, cqwq, + budget); + + if (work_done == 0) + return 0; -out: if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) && rq->hw_gro_data->skb) mlx5e_shampo_flush_skb(rq, NULL, false); -- cgit