diff options
Diffstat (limited to 'drivers/net/ethernet/amazon')
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_admin_defs.h | 72 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_com.c | 541 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_com.h | 103 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_eth_com.c | 84 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_eth_com.h | 41 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_ethtool.c | 194 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_netdev.c | 324 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_netdev.h | 3 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_regs_defs.h | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/amazon/ena/ena_xdp.c | 5 |
10 files changed, 818 insertions, 551 deletions
diff --git a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h index 6de0d590be34..9d9fa6559354 100644 --- a/drivers/net/ethernet/amazon/ena/ena_admin_defs.h +++ b/drivers/net/ethernet/amazon/ena/ena_admin_defs.h @@ -7,6 +7,21 @@ #define ENA_ADMIN_RSS_KEY_PARTS 10 +#define ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK 0x3F +#define ENA_ADMIN_CUSTOMER_METRICS_MIN_SUPPORT_MASK 0x1F + + /* customer metrics - in correlation with + * ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK + */ +enum ena_admin_customer_metrics_id { + ENA_ADMIN_BW_IN_ALLOWANCE_EXCEEDED = 0, + ENA_ADMIN_BW_OUT_ALLOWANCE_EXCEEDED = 1, + ENA_ADMIN_PPS_ALLOWANCE_EXCEEDED = 2, + ENA_ADMIN_CONNTRACK_ALLOWANCE_EXCEEDED = 3, + ENA_ADMIN_LINKLOCAL_ALLOWANCE_EXCEEDED = 4, + ENA_ADMIN_CONNTRACK_ALLOWANCE_AVAILABLE = 5, +}; + enum ena_admin_aq_opcode { ENA_ADMIN_CREATE_SQ = 1, ENA_ADMIN_DESTROY_SQ = 2, @@ -51,6 +66,9 @@ enum ena_admin_aq_feature_id { /* device capabilities */ enum ena_admin_aq_caps_id { ENA_ADMIN_ENI_STATS = 0, + /* ENA SRD customer metrics */ + ENA_ADMIN_ENA_SRD_INFO = 1, + ENA_ADMIN_CUSTOMER_METRICS = 2, }; enum ena_admin_placement_policy_type { @@ -99,6 +117,9 @@ enum ena_admin_get_stats_type { ENA_ADMIN_GET_STATS_TYPE_EXTENDED = 1, /* extra HW stats for specific network interface */ ENA_ADMIN_GET_STATS_TYPE_ENI = 2, + /* extra HW stats for ENA SRD */ + ENA_ADMIN_GET_STATS_TYPE_ENA_SRD = 3, + ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS = 4, }; enum ena_admin_get_stats_scope { @@ -106,6 +127,16 @@ enum ena_admin_get_stats_scope { ENA_ADMIN_ETH_TRAFFIC = 1, }; +/* ENA SRD configuration for ENI */ +enum ena_admin_ena_srd_flags { + /* Feature enabled */ + ENA_ADMIN_ENA_SRD_ENABLED = BIT(0), + /* UDP support enabled */ + ENA_ADMIN_ENA_SRD_UDP_ENABLED = BIT(1), + /* Bypass Rx UDP ordering */ + ENA_ADMIN_ENA_SRD_UDP_ORDERING_BYPASS_ENABLED = BIT(2), +}; + struct ena_admin_aq_common_desc { /* 11:0 : command_id * 15:12 : reserved12 @@ -363,6 +394,9 @@ struct ena_admin_aq_get_stats_cmd { * stats of other device */ u16 device_id; + + /* a bitmap representing the requested metric values */ + u64 requested_metrics; }; /* Basic Statistics Command. */ @@ -419,6 +453,40 @@ struct ena_admin_eni_stats { u64 linklocal_allowance_exceeded; }; +struct ena_admin_ena_srd_stats { + /* Number of packets transmitted over ENA SRD */ + u64 ena_srd_tx_pkts; + + /* Number of packets transmitted or could have been + * transmitted over ENA SRD + */ + u64 ena_srd_eligible_tx_pkts; + + /* Number of packets received over ENA SRD */ + u64 ena_srd_rx_pkts; + + /* Percentage of the ENA SRD resources that is in use */ + u64 ena_srd_resource_utilization; +}; + +/* ENA SRD Statistics Command */ +struct ena_admin_ena_srd_info { + /* ENA SRD configuration bitmap. See ena_admin_ena_srd_flags for + * details + */ + u64 flags; + + struct ena_admin_ena_srd_stats ena_srd_stats; +}; + +/* Customer Metrics Command. */ +struct ena_admin_customer_metrics { + /* A bitmap representing the reported customer metrics according to + * the order they are reported + */ + u64 reported_metrics; +}; + struct ena_admin_acq_get_stats_resp { struct ena_admin_acq_common_desc acq_common_desc; @@ -428,6 +496,10 @@ struct ena_admin_acq_get_stats_resp { struct ena_admin_basic_stats basic_stats; struct ena_admin_eni_stats eni_stats; + + struct ena_admin_ena_srd_info ena_srd_info; + + struct ena_admin_customer_metrics customer_metrics; } u; }; diff --git a/drivers/net/ethernet/amazon/ena/ena_com.c b/drivers/net/ethernet/amazon/ena/ena_com.c index 633b321d7fdd..66445617fbfb 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_com.c @@ -90,8 +90,7 @@ static int ena_com_admin_init_sq(struct ena_com_admin_queue *admin_queue) struct ena_com_admin_sq *sq = &admin_queue->sq; u16 size = ADMIN_SQ_SIZE(admin_queue->q_depth); - sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, - &sq->dma_addr, GFP_KERNEL); + sq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &sq->dma_addr, GFP_KERNEL); if (!sq->entries) { netdev_err(ena_dev->net_device, "Memory allocation failed\n"); @@ -113,8 +112,7 @@ static int ena_com_admin_init_cq(struct ena_com_admin_queue *admin_queue) struct ena_com_admin_cq *cq = &admin_queue->cq; u16 size = ADMIN_CQ_SIZE(admin_queue->q_depth); - cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, - &cq->dma_addr, GFP_KERNEL); + cq->entries = dma_alloc_coherent(admin_queue->q_dmadev, size, &cq->dma_addr, GFP_KERNEL); if (!cq->entries) { netdev_err(ena_dev->net_device, "Memory allocation failed\n"); @@ -136,8 +134,7 @@ static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev, ena_dev->aenq.q_depth = ENA_ASYNC_QUEUE_DEPTH; size = ADMIN_AENQ_SIZE(ENA_ASYNC_QUEUE_DEPTH); - aenq->entries = dma_alloc_coherent(ena_dev->dmadev, size, - &aenq->dma_addr, GFP_KERNEL); + aenq->entries = dma_alloc_coherent(ena_dev->dmadev, size, &aenq->dma_addr, GFP_KERNEL); if (!aenq->entries) { netdev_err(ena_dev->net_device, "Memory allocation failed\n"); @@ -155,14 +152,13 @@ static int ena_com_admin_init_aenq(struct ena_com_dev *ena_dev, aenq_caps = 0; aenq_caps |= ena_dev->aenq.q_depth & ENA_REGS_AENQ_CAPS_AENQ_DEPTH_MASK; - aenq_caps |= (sizeof(struct ena_admin_aenq_entry) - << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) & - ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK; + aenq_caps |= + (sizeof(struct ena_admin_aenq_entry) << ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_SHIFT) & + ENA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE_MASK; writel(aenq_caps, ena_dev->reg_bar + ENA_REGS_AENQ_CAPS_OFF); if (unlikely(!aenq_handlers)) { - netdev_err(ena_dev->net_device, - "AENQ handlers pointer is NULL\n"); + netdev_err(ena_dev->net_device, "AENQ handlers pointer is NULL\n"); return -EINVAL; } @@ -189,14 +185,12 @@ static struct ena_comp_ctx *get_comp_ctxt(struct ena_com_admin_queue *admin_queu } if (unlikely(!admin_queue->comp_ctx)) { - netdev_err(admin_queue->ena_dev->net_device, - "Completion context is NULL\n"); + netdev_err(admin_queue->ena_dev->net_device, "Completion context is NULL\n"); return NULL; } if (unlikely(admin_queue->comp_ctx[command_id].occupied && capture)) { - netdev_err(admin_queue->ena_dev->net_device, - "Completion context is occupied\n"); + netdev_err(admin_queue->ena_dev->net_device, "Completion context is occupied\n"); return NULL; } @@ -226,8 +220,7 @@ static struct ena_comp_ctx *__ena_com_submit_admin_cmd(struct ena_com_admin_queu /* In case of queue FULL */ cnt = (u16)atomic_read(&admin_queue->outstanding_cmds); if (cnt >= admin_queue->q_depth) { - netdev_dbg(admin_queue->ena_dev->net_device, - "Admin queue is full.\n"); + netdev_dbg(admin_queue->ena_dev->net_device, "Admin queue is full.\n"); admin_queue->stats.out_of_space++; return ERR_PTR(-ENOSPC); } @@ -274,8 +267,7 @@ static int ena_com_init_comp_ctxt(struct ena_com_admin_queue *admin_queue) struct ena_comp_ctx *comp_ctx; u16 i; - admin_queue->comp_ctx = - devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL); + admin_queue->comp_ctx = devm_kzalloc(admin_queue->q_dmadev, size, GFP_KERNEL); if (unlikely(!admin_queue->comp_ctx)) { netdev_err(ena_dev->net_device, "Memory allocation failed\n"); return -ENOMEM; @@ -320,7 +312,6 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, struct ena_com_io_sq *io_sq) { size_t size; - int dev_node = 0; memset(&io_sq->desc_addr, 0x0, sizeof(io_sq->desc_addr)); @@ -333,23 +324,17 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, size = io_sq->desc_entry_size * io_sq->q_depth; if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_HOST) { - dev_node = dev_to_node(ena_dev->dmadev); - set_dev_node(ena_dev->dmadev, ctx->numa_node); io_sq->desc_addr.virt_addr = - dma_alloc_coherent(ena_dev->dmadev, size, - &io_sq->desc_addr.phys_addr, + dma_alloc_coherent(ena_dev->dmadev, size, &io_sq->desc_addr.phys_addr, GFP_KERNEL); - set_dev_node(ena_dev->dmadev, dev_node); if (!io_sq->desc_addr.virt_addr) { io_sq->desc_addr.virt_addr = dma_alloc_coherent(ena_dev->dmadev, size, - &io_sq->desc_addr.phys_addr, - GFP_KERNEL); + &io_sq->desc_addr.phys_addr, GFP_KERNEL); } if (!io_sq->desc_addr.virt_addr) { - netdev_err(ena_dev->net_device, - "Memory allocation failed\n"); + netdev_err(ena_dev->net_device, "Memory allocation failed\n"); return -ENOMEM; } } @@ -362,21 +347,16 @@ static int ena_com_init_io_sq(struct ena_com_dev *ena_dev, ENA_COM_BOUNCE_BUFFER_CNTRL_CNT; io_sq->bounce_buf_ctrl.next_to_use = 0; - size = io_sq->bounce_buf_ctrl.buffer_size * + size = (size_t)io_sq->bounce_buf_ctrl.buffer_size * io_sq->bounce_buf_ctrl.buffers_num; - dev_node = dev_to_node(ena_dev->dmadev); - set_dev_node(ena_dev->dmadev, ctx->numa_node); - io_sq->bounce_buf_ctrl.base_buffer = - devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL); - set_dev_node(ena_dev->dmadev, dev_node); + io_sq->bounce_buf_ctrl.base_buffer = devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL); if (!io_sq->bounce_buf_ctrl.base_buffer) io_sq->bounce_buf_ctrl.base_buffer = devm_kzalloc(ena_dev->dmadev, size, GFP_KERNEL); if (!io_sq->bounce_buf_ctrl.base_buffer) { - netdev_err(ena_dev->net_device, - "Bounce buffer memory allocation failed\n"); + netdev_err(ena_dev->net_device, "Bounce buffer memory allocation failed\n"); return -ENOMEM; } @@ -410,7 +390,6 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev, struct ena_com_io_cq *io_cq) { size_t size; - int prev_node = 0; memset(&io_cq->cdesc_addr, 0x0, sizeof(io_cq->cdesc_addr)); @@ -422,16 +401,11 @@ static int ena_com_init_io_cq(struct ena_com_dev *ena_dev, size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; - prev_node = dev_to_node(ena_dev->dmadev); - set_dev_node(ena_dev->dmadev, ctx->numa_node); io_cq->cdesc_addr.virt_addr = - dma_alloc_coherent(ena_dev->dmadev, size, - &io_cq->cdesc_addr.phys_addr, GFP_KERNEL); - set_dev_node(ena_dev->dmadev, prev_node); + dma_alloc_coherent(ena_dev->dmadev, size, &io_cq->cdesc_addr.phys_addr, GFP_KERNEL); if (!io_cq->cdesc_addr.virt_addr) { io_cq->cdesc_addr.virt_addr = - dma_alloc_coherent(ena_dev->dmadev, size, - &io_cq->cdesc_addr.phys_addr, + dma_alloc_coherent(ena_dev->dmadev, size, &io_cq->cdesc_addr.phys_addr, GFP_KERNEL); } @@ -514,8 +488,8 @@ static int ena_com_comp_status_to_errno(struct ena_com_admin_queue *admin_queue, u8 comp_status) { if (unlikely(comp_status != 0)) - netdev_err(admin_queue->ena_dev->net_device, - "Admin command failed[%u]\n", comp_status); + netdev_err(admin_queue->ena_dev->net_device, "Admin command failed[%u]\n", + comp_status); switch (comp_status) { case ENA_ADMIN_SUCCESS: @@ -580,8 +554,7 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c } if (unlikely(comp_ctx->status == ENA_CMD_ABORTED)) { - netdev_err(admin_queue->ena_dev->net_device, - "Command was aborted\n"); + netdev_err(admin_queue->ena_dev->net_device, "Command was aborted\n"); spin_lock_irqsave(&admin_queue->q_lock, flags); admin_queue->stats.aborted_cmd++; spin_unlock_irqrestore(&admin_queue->q_lock, flags); @@ -589,8 +562,7 @@ static int ena_com_wait_and_process_admin_cq_polling(struct ena_comp_ctx *comp_c goto err; } - WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n", - comp_ctx->status); + WARN(comp_ctx->status != ENA_CMD_COMPLETED, "Invalid comp status %d\n", comp_ctx->status); ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); err: @@ -634,8 +606,7 @@ static int ena_com_set_llq(struct ena_com_dev *ena_dev) sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to set LLQ configurations: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to set LLQ configurations: %d\n", ret); return ret; } @@ -658,8 +629,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, llq_default_cfg->llq_header_location; } else { netdev_err(ena_dev->net_device, - "Invalid header location control, supported: 0x%x\n", - supported_feat); + "Invalid header location control, supported: 0x%x\n", supported_feat); return -EINVAL; } @@ -681,8 +651,8 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, netdev_err(ena_dev->net_device, "Default llq stride ctrl is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", - llq_default_cfg->llq_stride_ctrl, - supported_feat, llq_info->desc_stride_ctrl); + llq_default_cfg->llq_stride_ctrl, supported_feat, + llq_info->desc_stride_ctrl); } } else { llq_info->desc_stride_ctrl = 0; @@ -704,8 +674,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, llq_info->desc_list_entry_size = 256; } else { netdev_err(ena_dev->net_device, - "Invalid entry_size_ctrl, supported: 0x%x\n", - supported_feat); + "Invalid entry_size_ctrl, supported: 0x%x\n", supported_feat); return -EINVAL; } @@ -750,8 +719,8 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, netdev_err(ena_dev->net_device, "Default llq num descs before header is not supported, performing fallback, default: 0x%x, supported: 0x%x, used: 0x%x\n", - llq_default_cfg->llq_num_decs_before_header, - supported_feat, llq_info->descs_num_before_header); + llq_default_cfg->llq_num_decs_before_header, supported_feat, + llq_info->descs_num_before_header); } /* Check for accelerated queue supported */ llq_accel_mode_get = llq_features->accel_mode.u.get; @@ -767,8 +736,7 @@ static int ena_com_config_llq_info(struct ena_com_dev *ena_dev, rc = ena_com_set_llq(ena_dev); if (rc) - netdev_err(ena_dev->net_device, - "Cannot set LLQ configuration: %d\n", rc); + netdev_err(ena_dev->net_device, "Cannot set LLQ configuration: %d\n", rc); return rc; } @@ -780,8 +748,7 @@ static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *com int ret; wait_for_completion_timeout(&comp_ctx->wait_event, - usecs_to_jiffies( - admin_queue->completion_timeout)); + usecs_to_jiffies(admin_queue->completion_timeout)); /* In case the command wasn't completed find out the root cause. * There might be 2 kinds of errors @@ -796,26 +763,16 @@ static int ena_com_wait_and_process_admin_cq_interrupts(struct ena_comp_ctx *com if (comp_ctx->status == ENA_CMD_COMPLETED) { netdev_err(admin_queue->ena_dev->net_device, - "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d), autopolling mode is %s\n", - comp_ctx->cmd_opcode, - admin_queue->auto_polling ? "ON" : "OFF"); - /* Check if fallback to polling is enabled */ - if (admin_queue->auto_polling) - admin_queue->polling = true; + "The ena device sent a completion but the driver didn't receive a MSI-X interrupt (cmd %d)\n", + comp_ctx->cmd_opcode); } else { netdev_err(admin_queue->ena_dev->net_device, "The ena device didn't send a completion for the admin cmd %d status %d\n", comp_ctx->cmd_opcode, comp_ctx->status); } - /* Check if shifted to polling mode. - * This will happen if there is a completion without an interrupt - * and autopolling mode is enabled. Continuing normal execution in such case - */ - if (!admin_queue->polling) { - admin_queue->running_state = false; - ret = -ETIME; - goto err; - } + admin_queue->running_state = false; + ret = -ETIME; + goto err; } ret = ena_com_comp_status_to_errno(admin_queue, comp_ctx->comp_status); @@ -867,15 +824,13 @@ static u32 ena_com_reg_bar_read32(struct ena_com_dev *ena_dev, u16 offset) if (unlikely(i == timeout)) { netdev_err(ena_dev->net_device, "Reading reg failed for timeout. expected: req id[%u] offset[%u] actual: req id[%u] offset[%u]\n", - mmio_read->seq_num, offset, read_resp->req_id, - read_resp->reg_off); + mmio_read->seq_num, offset, read_resp->req_id, read_resp->reg_off); ret = ENA_MMIO_READ_TIMEOUT; goto err; } if (read_resp->reg_off != offset) { - netdev_err(ena_dev->net_device, - "Read failure: wrong offset provided\n"); + netdev_err(ena_dev->net_device, "Read failure: wrong offset provided\n"); ret = ENA_MMIO_READ_TIMEOUT; } else { ret = read_resp->reg_val; @@ -934,8 +889,7 @@ static int ena_com_destroy_io_sq(struct ena_com_dev *ena_dev, sizeof(destroy_resp)); if (unlikely(ret && (ret != -ENODEV))) - netdev_err(ena_dev->net_device, - "Failed to destroy io sq error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to destroy io sq error: %d\n", ret); return ret; } @@ -949,8 +903,7 @@ static void ena_com_io_queue_free(struct ena_com_dev *ena_dev, if (io_cq->cdesc_addr.virt_addr) { size = io_cq->cdesc_entry_size_in_bytes * io_cq->q_depth; - dma_free_coherent(ena_dev->dmadev, size, - io_cq->cdesc_addr.virt_addr, + dma_free_coherent(ena_dev->dmadev, size, io_cq->cdesc_addr.virt_addr, io_cq->cdesc_addr.phys_addr); io_cq->cdesc_addr.virt_addr = NULL; @@ -959,8 +912,7 @@ static void ena_com_io_queue_free(struct ena_com_dev *ena_dev, if (io_sq->desc_addr.virt_addr) { size = io_sq->desc_entry_size * io_sq->q_depth; - dma_free_coherent(ena_dev->dmadev, size, - io_sq->desc_addr.virt_addr, + dma_free_coherent(ena_dev->dmadev, size, io_sq->desc_addr.virt_addr, io_sq->desc_addr.phys_addr); io_sq->desc_addr.virt_addr = NULL; @@ -985,8 +937,7 @@ static int wait_for_reset_state(struct ena_com_dev *ena_dev, u32 timeout, val = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); if (unlikely(val == ENA_MMIO_READ_TIMEOUT)) { - netdev_err(ena_dev->net_device, - "Reg read timeout occurred\n"); + netdev_err(ena_dev->net_device, "Reg read timeout occurred\n"); return -ETIME; } @@ -1026,8 +977,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev, int ret; if (!ena_com_check_supported_feature_id(ena_dev, feature_id)) { - netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", - feature_id); + netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", feature_id); return -EOPNOTSUPP; } @@ -1064,8 +1014,7 @@ static int ena_com_get_feature_ex(struct ena_com_dev *ena_dev, if (unlikely(ret)) netdev_err(ena_dev->net_device, - "Failed to submit get_feature command %d error: %d\n", - feature_id, ret); + "Failed to submit get_feature command %d error: %d\n", feature_id, ret); return ret; } @@ -1104,13 +1053,11 @@ static int ena_com_hash_key_allocate(struct ena_com_dev *ena_dev) { struct ena_rss *rss = &ena_dev->rss; - if (!ena_com_check_supported_feature_id(ena_dev, - ENA_ADMIN_RSS_HASH_FUNCTION)) + if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_FUNCTION)) return -EOPNOTSUPP; - rss->hash_key = - dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key), - &rss->hash_key_dma_addr, GFP_KERNEL); + rss->hash_key = dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_key), + &rss->hash_key_dma_addr, GFP_KERNEL); if (unlikely(!rss->hash_key)) return -ENOMEM; @@ -1123,8 +1070,8 @@ static void ena_com_hash_key_destroy(struct ena_com_dev *ena_dev) struct ena_rss *rss = &ena_dev->rss; if (rss->hash_key) - dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key), - rss->hash_key, rss->hash_key_dma_addr); + dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_key), rss->hash_key, + rss->hash_key_dma_addr); rss->hash_key = NULL; } @@ -1132,9 +1079,8 @@ static int ena_com_hash_ctrl_init(struct ena_com_dev *ena_dev) { struct ena_rss *rss = &ena_dev->rss; - rss->hash_ctrl = - dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl), - &rss->hash_ctrl_dma_addr, GFP_KERNEL); + rss->hash_ctrl = dma_alloc_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl), + &rss->hash_ctrl_dma_addr, GFP_KERNEL); if (unlikely(!rss->hash_ctrl)) return -ENOMEM; @@ -1147,8 +1093,8 @@ static void ena_com_hash_ctrl_destroy(struct ena_com_dev *ena_dev) struct ena_rss *rss = &ena_dev->rss; if (rss->hash_ctrl) - dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl), - rss->hash_ctrl, rss->hash_ctrl_dma_addr); + dma_free_coherent(ena_dev->dmadev, sizeof(*rss->hash_ctrl), rss->hash_ctrl, + rss->hash_ctrl_dma_addr); rss->hash_ctrl = NULL; } @@ -1177,15 +1123,13 @@ static int ena_com_indirect_table_allocate(struct ena_com_dev *ena_dev, tbl_size = (1ULL << log_size) * sizeof(struct ena_admin_rss_ind_table_entry); - rss->rss_ind_tbl = - dma_alloc_coherent(ena_dev->dmadev, tbl_size, - &rss->rss_ind_tbl_dma_addr, GFP_KERNEL); + rss->rss_ind_tbl = dma_alloc_coherent(ena_dev->dmadev, tbl_size, &rss->rss_ind_tbl_dma_addr, + GFP_KERNEL); if (unlikely(!rss->rss_ind_tbl)) goto mem_err1; tbl_size = (1ULL << log_size) * sizeof(u16); - rss->host_rss_ind_tbl = - devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL); + rss->host_rss_ind_tbl = devm_kzalloc(ena_dev->dmadev, tbl_size, GFP_KERNEL); if (unlikely(!rss->host_rss_ind_tbl)) goto mem_err2; @@ -1197,8 +1141,7 @@ mem_err2: tbl_size = (1ULL << log_size) * sizeof(struct ena_admin_rss_ind_table_entry); - dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl, - rss->rss_ind_tbl_dma_addr); + dma_free_coherent(ena_dev->dmadev, tbl_size, rss->rss_ind_tbl, rss->rss_ind_tbl_dma_addr); rss->rss_ind_tbl = NULL; mem_err1: rss->tbl_log_size = 0; @@ -1261,8 +1204,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, &create_cmd.sq_ba, io_sq->desc_addr.phys_addr); if (unlikely(ret)) { - netdev_err(ena_dev->net_device, - "Memory address set failed\n"); + netdev_err(ena_dev->net_device, "Memory address set failed\n"); return ret; } } @@ -1273,8 +1215,7 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, (struct ena_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (unlikely(ret)) { - netdev_err(ena_dev->net_device, - "Failed to create IO SQ. error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to create IO SQ. error: %d\n", ret); return ret; } @@ -1284,16 +1225,12 @@ static int ena_com_create_io_sq(struct ena_com_dev *ena_dev, (uintptr_t)cmd_completion.sq_doorbell_offset); if (io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) { - io_sq->header_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar - + cmd_completion.llq_headers_offset); - io_sq->desc_addr.pbuf_dev_addr = (u8 __iomem *)((uintptr_t)ena_dev->mem_bar + cmd_completion.llq_descriptors_offset); } - netdev_dbg(ena_dev->net_device, "Created sq[%u], depth[%u]\n", - io_sq->idx, io_sq->q_depth); + netdev_dbg(ena_dev->net_device, "Created sq[%u], depth[%u]\n", io_sq->idx, io_sq->q_depth); return ret; } @@ -1420,8 +1357,7 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev, (struct ena_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (unlikely(ret)) { - netdev_err(ena_dev->net_device, - "Failed to create IO CQ. error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to create IO CQ. error: %d\n", ret); return ret; } @@ -1430,18 +1366,12 @@ int ena_com_create_io_cq(struct ena_com_dev *ena_dev, io_cq->unmask_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + cmd_completion.cq_interrupt_unmask_register_offset); - if (cmd_completion.cq_head_db_register_offset) - io_cq->cq_head_db_reg = - (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + - cmd_completion.cq_head_db_register_offset); - if (cmd_completion.numa_node_register_offset) io_cq->numa_node_cfg_reg = (u32 __iomem *)((uintptr_t)ena_dev->reg_bar + cmd_completion.numa_node_register_offset); - netdev_dbg(ena_dev->net_device, "Created cq[%u], depth[%u]\n", - io_cq->idx, io_cq->q_depth); + netdev_dbg(ena_dev->net_device, "Created cq[%u], depth[%u]\n", io_cq->idx, io_cq->q_depth); return ret; } @@ -1451,8 +1381,7 @@ int ena_com_get_io_handlers(struct ena_com_dev *ena_dev, u16 qid, struct ena_com_io_cq **io_cq) { if (qid >= ENA_TOTAL_NUM_QUEUES) { - netdev_err(ena_dev->net_device, - "Invalid queue number %d but the max is %d\n", qid, + netdev_err(ena_dev->net_device, "Invalid queue number %d but the max is %d\n", qid, ENA_TOTAL_NUM_QUEUES); return -EINVAL; } @@ -1492,8 +1421,7 @@ void ena_com_wait_for_abort_completion(struct ena_com_dev *ena_dev) spin_lock_irqsave(&admin_queue->q_lock, flags); while (atomic_read(&admin_queue->outstanding_cmds) != 0) { spin_unlock_irqrestore(&admin_queue->q_lock, flags); - ena_delay_exponential_backoff_us(exp++, - ena_dev->ena_min_poll_delay_us); + ena_delay_exponential_backoff_us(exp++, ena_dev->ena_min_poll_delay_us); spin_lock_irqsave(&admin_queue->q_lock, flags); } spin_unlock_irqrestore(&admin_queue->q_lock, flags); @@ -1519,8 +1447,7 @@ int ena_com_destroy_io_cq(struct ena_com_dev *ena_dev, sizeof(destroy_resp)); if (unlikely(ret && (ret != -ENODEV))) - netdev_err(ena_dev->net_device, - "Failed to destroy IO CQ. error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to destroy IO CQ. error: %d\n", ret); return ret; } @@ -1588,8 +1515,7 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag) sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to config AENQ ret: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to config AENQ ret: %d\n", ret); return ret; } @@ -1610,8 +1536,7 @@ int ena_com_get_dma_width(struct ena_com_dev *ena_dev) netdev_dbg(ena_dev->net_device, "ENA dma width: %d\n", width); if ((width < 32) || width > ENA_MAX_PHYS_ADDR_SIZE_BITS) { - netdev_err(ena_dev->net_device, "DMA width illegal value: %d\n", - width); + netdev_err(ena_dev->net_device, "DMA width illegal value: %d\n", width); return -EINVAL; } @@ -1633,19 +1558,16 @@ int ena_com_validate_version(struct ena_com_dev *ena_dev) ctrl_ver = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CONTROLLER_VERSION_OFF); - if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) || - (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) { + if (unlikely((ver == ENA_MMIO_READ_TIMEOUT) || (ctrl_ver == ENA_MMIO_READ_TIMEOUT))) { netdev_err(ena_dev->net_device, "Reg read timeout occurred\n"); return -ETIME; } dev_info(ena_dev->dmadev, "ENA device version: %d.%d\n", - (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >> - ENA_REGS_VERSION_MAJOR_VERSION_SHIFT, + (ver & ENA_REGS_VERSION_MAJOR_VERSION_MASK) >> ENA_REGS_VERSION_MAJOR_VERSION_SHIFT, ver & ENA_REGS_VERSION_MINOR_VERSION_MASK); - dev_info(ena_dev->dmadev, - "ENA controller version: %d.%d.%d implementation version %d\n", + dev_info(ena_dev->dmadev, "ENA controller version: %d.%d.%d implementation version %d\n", (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_MASK) >> ENA_REGS_CONTROLLER_VERSION_MAJOR_VERSION_SHIFT, (ctrl_ver & ENA_REGS_CONTROLLER_VERSION_MINOR_VERSION_MASK) >> @@ -1694,20 +1616,17 @@ void ena_com_admin_destroy(struct ena_com_dev *ena_dev) size = ADMIN_SQ_SIZE(admin_queue->q_depth); if (sq->entries) - dma_free_coherent(ena_dev->dmadev, size, sq->entries, - sq->dma_addr); + dma_free_coherent(ena_dev->dmadev, size, sq->entries, sq->dma_addr); sq->entries = NULL; size = ADMIN_CQ_SIZE(admin_queue->q_depth); if (cq->entries) - dma_free_coherent(ena_dev->dmadev, size, cq->entries, - cq->dma_addr); + dma_free_coherent(ena_dev->dmadev, size, cq->entries, cq->dma_addr); cq->entries = NULL; size = ADMIN_AENQ_SIZE(aenq->q_depth); if (ena_dev->aenq.entries) - dma_free_coherent(ena_dev->dmadev, size, aenq->entries, - aenq->dma_addr); + dma_free_coherent(ena_dev->dmadev, size, aenq->entries, aenq->dma_addr); aenq->entries = NULL; } @@ -1722,21 +1641,13 @@ void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling) ena_dev->admin_queue.polling = polling; } -void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, - bool polling) -{ - ena_dev->admin_queue.auto_polling = polling; -} - int ena_com_mmio_reg_read_request_init(struct ena_com_dev *ena_dev) { struct ena_com_mmio_read *mmio_read = &ena_dev->mmio_read; spin_lock_init(&mmio_read->lock); - mmio_read->read_resp = - dma_alloc_coherent(ena_dev->dmadev, - sizeof(*mmio_read->read_resp), - &mmio_read->read_resp_dma_addr, GFP_KERNEL); + mmio_read->read_resp = dma_alloc_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp), + &mmio_read->read_resp_dma_addr, GFP_KERNEL); if (unlikely(!mmio_read->read_resp)) goto err; @@ -1767,8 +1678,8 @@ void ena_com_mmio_reg_read_request_destroy(struct ena_com_dev *ena_dev) writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_LO_OFF); writel(0x0, ena_dev->reg_bar + ENA_REGS_MMIO_RESP_HI_OFF); - dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp), - mmio_read->read_resp, mmio_read->read_resp_dma_addr); + dma_free_coherent(ena_dev->dmadev, sizeof(*mmio_read->read_resp), mmio_read->read_resp, + mmio_read->read_resp_dma_addr); mmio_read->read_resp = NULL; } @@ -1800,8 +1711,7 @@ int ena_com_admin_init(struct ena_com_dev *ena_dev, } if (!(dev_sts & ENA_REGS_DEV_STS_READY_MASK)) { - netdev_err(ena_dev->net_device, - "Device isn't ready, abort com init\n"); + netdev_err(ena_dev->net_device, "Device isn't ready, abort com init\n"); return -ENODEV; } @@ -1878,8 +1788,7 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev, int ret; if (ctx->qid >= ENA_TOTAL_NUM_QUEUES) { - netdev_err(ena_dev->net_device, - "Qid (%d) is bigger than max num of queues (%d)\n", + netdev_err(ena_dev->net_device, "Qid (%d) is bigger than max num of queues (%d)\n", ctx->qid, ENA_TOTAL_NUM_QUEUES); return -EINVAL; } @@ -1905,8 +1814,7 @@ int ena_com_create_io_queue(struct ena_com_dev *ena_dev, if (ctx->direction == ENA_COM_IO_QUEUE_DIRECTION_TX) /* header length is limited to 8 bits */ - io_sq->tx_max_header_size = - min_t(u32, ena_dev->tx_max_header_size, SZ_256); + io_sq->tx_max_header_size = min_t(u32, ena_dev->tx_max_header_size, SZ_256); ret = ena_com_init_io_sq(ena_dev, ctx, io_sq); if (ret) @@ -1938,8 +1846,7 @@ void ena_com_destroy_io_queue(struct ena_com_dev *ena_dev, u16 qid) struct ena_com_io_cq *io_cq; if (qid >= ENA_TOTAL_NUM_QUEUES) { - netdev_err(ena_dev->net_device, - "Qid (%d) is bigger than max num of queues (%d)\n", + netdev_err(ena_dev->net_device, "Qid (%d) is bigger than max num of queues (%d)\n", qid, ENA_TOTAL_NUM_QUEUES); return; } @@ -1959,6 +1866,56 @@ int ena_com_get_link_params(struct ena_com_dev *ena_dev, return ena_com_get_feature(ena_dev, resp, ENA_ADMIN_LINK_CONFIG, 0); } +static int ena_get_dev_stats(struct ena_com_dev *ena_dev, + struct ena_com_stats_ctx *ctx, + enum ena_admin_get_stats_type type) +{ + struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; + struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; + struct ena_com_admin_queue *admin_queue; + int ret; + + admin_queue = &ena_dev->admin_queue; + + get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; + get_cmd->aq_common_descriptor.flags = 0; + get_cmd->type = type; + + ret = ena_com_execute_admin_command(admin_queue, + (struct ena_admin_aq_entry *)get_cmd, + sizeof(*get_cmd), + (struct ena_admin_acq_entry *)get_resp, + sizeof(*get_resp)); + + if (unlikely(ret)) + netdev_err(ena_dev->net_device, "Failed to get stats. error: %d\n", ret); + + return ret; +} + +static void ena_com_set_supported_customer_metrics(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics; + struct ena_com_stats_ctx ctx; + int ret; + + customer_metrics = &ena_dev->customer_metrics; + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) { + customer_metrics->supported_metrics = ENA_ADMIN_CUSTOMER_METRICS_MIN_SUPPORT_MASK; + return; + } + + memset(&ctx, 0x0, sizeof(ctx)); + ctx.get_cmd.requested_metrics = ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK; + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS); + if (likely(ret == 0)) + customer_metrics->supported_metrics = + ctx.get_resp.u.customer_metrics.reported_metrics; + else + netdev_err(ena_dev->net_device, + "Failed to query customer metrics support. error: %d\n", ret); +} + int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, struct ena_com_dev_get_features_ctx *get_feat_ctx) { @@ -1983,8 +1940,7 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, if (rc) return rc; - if (get_resp.u.max_queue_ext.version != - ENA_FEATURE_MAX_QUEUE_EXT_VER) + if (get_resp.u.max_queue_ext.version != ENA_FEATURE_MAX_QUEUE_EXT_VER) return -EINVAL; memcpy(&get_feat_ctx->max_queue_ext, &get_resp.u.max_queue_ext, @@ -2025,23 +1981,22 @@ int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_HW_HINTS, 0); if (!rc) - memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, - sizeof(get_resp.u.hw_hints)); + memcpy(&get_feat_ctx->hw_hints, &get_resp.u.hw_hints, sizeof(get_resp.u.hw_hints)); else if (rc == -EOPNOTSUPP) - memset(&get_feat_ctx->hw_hints, 0x0, - sizeof(get_feat_ctx->hw_hints)); + memset(&get_feat_ctx->hw_hints, 0x0, sizeof(get_feat_ctx->hw_hints)); else return rc; rc = ena_com_get_feature(ena_dev, &get_resp, ENA_ADMIN_LLQ, 0); if (!rc) - memcpy(&get_feat_ctx->llq, &get_resp.u.llq, - sizeof(get_resp.u.llq)); + memcpy(&get_feat_ctx->llq, &get_resp.u.llq, sizeof(get_resp.u.llq)); else if (rc == -EOPNOTSUPP) memset(&get_feat_ctx->llq, 0x0, sizeof(get_feat_ctx->llq)); else return rc; + ena_com_set_supported_customer_metrics(ena_dev); + return 0; } @@ -2084,8 +2039,7 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data) aenq_common = &aenq_e->aenq_common_desc; /* Go over all the events */ - while ((READ_ONCE(aenq_common->flags) & - ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { + while ((READ_ONCE(aenq_common->flags) & ENA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) { /* Make sure the phase bit (ownership) is as expected before * reading the rest of the descriptor. */ @@ -2094,8 +2048,7 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data) timestamp = (u64)aenq_common->timestamp_low | ((u64)aenq_common->timestamp_high << 32); - netdev_dbg(ena_dev->net_device, - "AENQ! Group[%x] Syndrome[%x] timestamp: [%llus]\n", + netdev_dbg(ena_dev->net_device, "AENQ! Group[%x] Syndrome[%x] timestamp: [%llus]\n", aenq_common->group, aenq_common->syndrome, timestamp); /* Handle specific event*/ @@ -2124,8 +2077,7 @@ void ena_com_aenq_intr_handler(struct ena_com_dev *ena_dev, void *data) /* write the aenq doorbell after all AENQ descriptors were read */ mb(); - writel_relaxed((u32)aenq->head, - ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); + writel_relaxed((u32)aenq->head, ena_dev->reg_bar + ENA_REGS_AENQ_HEAD_DB_OFF); } int ena_com_dev_reset(struct ena_com_dev *ena_dev, @@ -2137,15 +2089,13 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev, stat = ena_com_reg_bar_read32(ena_dev, ENA_REGS_DEV_STS_OFF); cap = ena_com_reg_bar_read32(ena_dev, ENA_REGS_CAPS_OFF); - if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) || - (cap == ENA_MMIO_READ_TIMEOUT))) { + if (unlikely((stat == ENA_MMIO_READ_TIMEOUT) || (cap == ENA_MMIO_READ_TIMEOUT))) { netdev_err(ena_dev->net_device, "Reg read32 timeout occurred\n"); return -ETIME; } if ((stat & ENA_REGS_DEV_STS_READY_MASK) == 0) { - netdev_err(ena_dev->net_device, - "Device isn't ready, can't reset device\n"); + netdev_err(ena_dev->net_device, "Device isn't ready, can't reset device\n"); return -EINVAL; } @@ -2168,8 +2118,7 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev, rc = wait_for_reset_state(ena_dev, timeout, ENA_REGS_DEV_STS_RESET_IN_PROGRESS_MASK); if (rc != 0) { - netdev_err(ena_dev->net_device, - "Reset indication didn't turn on\n"); + netdev_err(ena_dev->net_device, "Reset indication didn't turn on\n"); return rc; } @@ -2177,8 +2126,7 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev, writel(0, ena_dev->reg_bar + ENA_REGS_DEV_CTL_OFF); rc = wait_for_reset_state(ena_dev, timeout, 0); if (rc != 0) { - netdev_err(ena_dev->net_device, - "Reset indication didn't turn off\n"); + netdev_err(ena_dev->net_device, "Reset indication didn't turn off\n"); return rc; } @@ -2193,67 +2141,88 @@ int ena_com_dev_reset(struct ena_com_dev *ena_dev, return 0; } -static int ena_get_dev_stats(struct ena_com_dev *ena_dev, - struct ena_com_stats_ctx *ctx, - enum ena_admin_get_stats_type type) +int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, + struct ena_admin_eni_stats *stats) { - struct ena_admin_aq_get_stats_cmd *get_cmd = &ctx->get_cmd; - struct ena_admin_acq_get_stats_resp *get_resp = &ctx->get_resp; - struct ena_com_admin_queue *admin_queue; + struct ena_com_stats_ctx ctx; int ret; - admin_queue = &ena_dev->admin_queue; - - get_cmd->aq_common_descriptor.opcode = ENA_ADMIN_GET_STATS; - get_cmd->aq_common_descriptor.flags = 0; - get_cmd->type = type; - - ret = ena_com_execute_admin_command(admin_queue, - (struct ena_admin_aq_entry *)get_cmd, - sizeof(*get_cmd), - (struct ena_admin_acq_entry *)get_resp, - sizeof(*get_resp)); + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENI_STATS)) { + netdev_err(ena_dev->net_device, "Capability %d isn't supported\n", + ENA_ADMIN_ENI_STATS); + return -EOPNOTSUPP; + } - if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to get stats. error: %d\n", ret); + memset(&ctx, 0x0, sizeof(ctx)); + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI); + if (likely(ret == 0)) + memcpy(stats, &ctx.get_resp.u.eni_stats, + sizeof(ctx.get_resp.u.eni_stats)); return ret; } -int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, - struct ena_admin_eni_stats *stats) +int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, + struct ena_admin_ena_srd_info *info) { struct ena_com_stats_ctx ctx; int ret; - if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENI_STATS)) { - netdev_err(ena_dev->net_device, - "Capability %d isn't supported\n", - ENA_ADMIN_ENI_STATS); + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_ENA_SRD_INFO)) { + netdev_err(ena_dev->net_device, "Capability %d isn't supported\n", + ENA_ADMIN_ENA_SRD_INFO); return -EOPNOTSUPP; } memset(&ctx, 0x0, sizeof(ctx)); - ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENI); + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_ENA_SRD); if (likely(ret == 0)) - memcpy(stats, &ctx.get_resp.u.eni_stats, - sizeof(ctx.get_resp.u.eni_stats)); + memcpy(info, &ctx.get_resp.u.ena_srd_info, + sizeof(ctx.get_resp.u.ena_srd_info)); return ret; } -int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, - struct ena_admin_basic_stats *stats) +int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len) { + struct ena_admin_aq_get_stats_cmd *get_cmd; struct ena_com_stats_ctx ctx; int ret; + if (unlikely(len > ena_dev->customer_metrics.buffer_len)) { + netdev_err(ena_dev->net_device, + "Invalid buffer size %u. The given buffer is too big.\n", len); + return -EINVAL; + } + + if (!ena_com_get_cap(ena_dev, ENA_ADMIN_CUSTOMER_METRICS)) { + netdev_err(ena_dev->net_device, "Capability %d not supported.\n", + ENA_ADMIN_CUSTOMER_METRICS); + return -EOPNOTSUPP; + } + + if (!ena_dev->customer_metrics.supported_metrics) { + netdev_err(ena_dev->net_device, "No supported customer metrics.\n"); + return -EOPNOTSUPP; + } + + get_cmd = &ctx.get_cmd; memset(&ctx, 0x0, sizeof(ctx)); - ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_BASIC); + ret = ena_com_mem_addr_set(ena_dev, + &get_cmd->u.control_buffer.address, + ena_dev->customer_metrics.buffer_dma_addr); + if (unlikely(ret)) { + netdev_err(ena_dev->net_device, "Memory address set failed.\n"); + return ret; + } + + get_cmd->u.control_buffer.length = ena_dev->customer_metrics.buffer_len; + get_cmd->requested_metrics = ena_dev->customer_metrics.supported_metrics; + ret = ena_get_dev_stats(ena_dev, &ctx, ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS); if (likely(ret == 0)) - memcpy(stats, &ctx.get_resp.u.basic_stats, - sizeof(ctx.get_resp.u.basic_stats)); + memcpy(buffer, ena_dev->customer_metrics.buffer_virt_addr, len); + else + netdev_err(ena_dev->net_device, "Failed to get customer metrics. error: %d\n", ret); return ret; } @@ -2266,8 +2235,7 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu) int ret; if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_MTU)) { - netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", - ENA_ADMIN_MTU); + netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", ENA_ADMIN_MTU); return -EOPNOTSUPP; } @@ -2286,31 +2254,11 @@ int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu) sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to set mtu %d. error: %d\n", mtu, ret); + netdev_err(ena_dev->net_device, "Failed to set mtu %d. error: %d\n", mtu, ret); return ret; } -int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, - struct ena_admin_feature_offload_desc *offload) -{ - int ret; - struct ena_admin_get_feat_resp resp; - - ret = ena_com_get_feature(ena_dev, &resp, - ENA_ADMIN_STATELESS_OFFLOAD_CONFIG, 0); - if (unlikely(ret)) { - netdev_err(ena_dev->net_device, - "Failed to get offload capabilities %d\n", ret); - return ret; - } - - memcpy(offload, &resp.u.offload, sizeof(resp.u.offload)); - - return 0; -} - int ena_com_set_hash_function(struct ena_com_dev *ena_dev) { struct ena_com_admin_queue *admin_queue = &ena_dev->admin_queue; @@ -2320,8 +2268,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev) struct ena_admin_get_feat_resp get_resp; int ret; - if (!ena_com_check_supported_feature_id(ena_dev, - ENA_ADMIN_RSS_HASH_FUNCTION)) { + if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_FUNCTION)) { netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", ENA_ADMIN_RSS_HASH_FUNCTION); return -EOPNOTSUPP; @@ -2334,8 +2281,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev) return ret; if (!(get_resp.u.flow_hash_func.supported_func & BIT(rss->hash_func))) { - netdev_err(ena_dev->net_device, - "Func hash %d isn't supported by device, abort\n", + netdev_err(ena_dev->net_device, "Func hash %d isn't supported by device, abort\n", rss->hash_func); return -EOPNOTSUPP; } @@ -2365,8 +2311,7 @@ int ena_com_set_hash_function(struct ena_com_dev *ena_dev) (struct ena_admin_acq_entry *)&resp, sizeof(resp)); if (unlikely(ret)) { - netdev_err(ena_dev->net_device, - "Failed to set hash function %d. error: %d\n", + netdev_err(ena_dev->net_device, "Failed to set hash function %d. error: %d\n", rss->hash_func, ret); return -EINVAL; } @@ -2398,16 +2343,15 @@ int ena_com_fill_hash_function(struct ena_com_dev *ena_dev, return rc; if (!(BIT(func) & get_resp.u.flow_hash_func.supported_func)) { - netdev_err(ena_dev->net_device, - "Flow hash function %d isn't supported\n", func); + netdev_err(ena_dev->net_device, "Flow hash function %d isn't supported\n", func); return -EOPNOTSUPP; } if ((func == ENA_ADMIN_TOEPLITZ) && key) { if (key_len != sizeof(hash_key->key)) { netdev_err(ena_dev->net_device, - "key len (%u) doesn't equal the supported size (%zu)\n", - key_len, sizeof(hash_key->key)); + "key len (%u) doesn't equal the supported size (%zu)\n", key_len, + sizeof(hash_key->key)); return -EINVAL; } memcpy(hash_key->key, key, key_len); @@ -2495,8 +2439,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) struct ena_admin_set_feat_resp resp; int ret; - if (!ena_com_check_supported_feature_id(ena_dev, - ENA_ADMIN_RSS_HASH_INPUT)) { + if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_HASH_INPUT)) { netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", ENA_ADMIN_RSS_HASH_INPUT); return -EOPNOTSUPP; @@ -2527,8 +2470,7 @@ int ena_com_set_hash_ctrl(struct ena_com_dev *ena_dev) (struct ena_admin_acq_entry *)&resp, sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to set hash input. error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to set hash input. error: %d\n", ret); return ret; } @@ -2605,8 +2547,7 @@ int ena_com_fill_hash_ctrl(struct ena_com_dev *ena_dev, int rc; if (proto >= ENA_ADMIN_RSS_PROTO_NUM) { - netdev_err(ena_dev->net_device, "Invalid proto num (%u)\n", - proto); + netdev_err(ena_dev->net_device, "Invalid proto num (%u)\n", proto); return -EINVAL; } @@ -2658,8 +2599,7 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev) struct ena_admin_set_feat_resp resp; int ret; - if (!ena_com_check_supported_feature_id( - ena_dev, ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) { + if (!ena_com_check_supported_feature_id(ena_dev, ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG)) { netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", ENA_ADMIN_RSS_INDIRECTION_TABLE_CONFIG); return -EOPNOTSUPP; @@ -2699,8 +2639,7 @@ int ena_com_indirect_table_set(struct ena_com_dev *ena_dev) sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to set indirect table. error: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to set indirect table. error: %d\n", ret); return ret; } @@ -2779,9 +2718,8 @@ int ena_com_allocate_host_info(struct ena_com_dev *ena_dev) { struct ena_host_attribute *host_attr = &ena_dev->host_attr; - host_attr->host_info = - dma_alloc_coherent(ena_dev->dmadev, SZ_4K, - &host_attr->host_info_dma_addr, GFP_KERNEL); + host_attr->host_info = dma_alloc_coherent(ena_dev->dmadev, SZ_4K, + &host_attr->host_info_dma_addr, GFP_KERNEL); if (unlikely(!host_attr->host_info)) return -ENOMEM; @@ -2810,6 +2748,24 @@ int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, return 0; } +int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; + + customer_metrics->buffer_len = ENA_CUSTOMER_METRICS_BUFFER_SIZE; + customer_metrics->buffer_virt_addr = NULL; + + customer_metrics->buffer_virt_addr = + dma_alloc_coherent(ena_dev->dmadev, customer_metrics->buffer_len, + &customer_metrics->buffer_dma_addr, GFP_KERNEL); + if (!customer_metrics->buffer_virt_addr) { + customer_metrics->buffer_len = 0; + return -ENOMEM; + } + + return 0; +} + void ena_com_delete_host_info(struct ena_com_dev *ena_dev) { struct ena_host_attribute *host_attr = &ena_dev->host_attr; @@ -2827,12 +2783,24 @@ void ena_com_delete_debug_area(struct ena_com_dev *ena_dev) if (host_attr->debug_area_virt_addr) { dma_free_coherent(ena_dev->dmadev, host_attr->debug_area_size, - host_attr->debug_area_virt_addr, - host_attr->debug_area_dma_addr); + host_attr->debug_area_virt_addr, host_attr->debug_area_dma_addr); host_attr->debug_area_virt_addr = NULL; } } +void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev) +{ + struct ena_customer_metrics *customer_metrics = &ena_dev->customer_metrics; + + if (customer_metrics->buffer_virt_addr) { + dma_free_coherent(ena_dev->dmadev, customer_metrics->buffer_len, + customer_metrics->buffer_virt_addr, + customer_metrics->buffer_dma_addr); + customer_metrics->buffer_virt_addr = NULL; + customer_metrics->buffer_len = 0; + } +} + int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) { struct ena_host_attribute *host_attr = &ena_dev->host_attr; @@ -2877,8 +2845,7 @@ int ena_com_set_host_attributes(struct ena_com_dev *ena_dev) sizeof(resp)); if (unlikely(ret)) - netdev_err(ena_dev->net_device, - "Failed to set host attributes: %d\n", ret); + netdev_err(ena_dev->net_device, "Failed to set host attributes: %d\n", ret); return ret; } @@ -2896,8 +2863,7 @@ static int ena_com_update_nonadaptive_moderation_interval(struct ena_com_dev *en u32 *intr_moder_interval) { if (!intr_delay_resolution) { - netdev_err(ena_dev->net_device, - "Illegal interrupt delay granularity value\n"); + netdev_err(ena_dev->net_device, "Illegal interrupt delay granularity value\n"); return -EFAULT; } @@ -2935,14 +2901,12 @@ int ena_com_init_interrupt_moderation(struct ena_com_dev *ena_dev) if (rc) { if (rc == -EOPNOTSUPP) { - netdev_dbg(ena_dev->net_device, - "Feature %d isn't supported\n", + netdev_dbg(ena_dev->net_device, "Feature %d isn't supported\n", ENA_ADMIN_INTERRUPT_MODERATION); rc = 0; } else { netdev_err(ena_dev->net_device, - "Failed to get interrupt moderation admin cmd. rc: %d\n", - rc); + "Failed to get interrupt moderation admin cmd. rc: %d\n", rc); } /* no moderation supported, disable adaptive support */ @@ -2990,8 +2954,7 @@ int ena_com_config_dev_mode(struct ena_com_dev *ena_dev, (llq_info->descs_num_before_header * sizeof(struct ena_eth_io_tx_desc)); if (unlikely(ena_dev->tx_max_header_size == 0)) { - netdev_err(ena_dev->net_device, - "The size of the LLQ entry is smaller than needed\n"); + netdev_err(ena_dev->net_device, "The size of the LLQ entry is smaller than needed\n"); return -EINVAL; } diff --git a/drivers/net/ethernet/amazon/ena/ena_com.h b/drivers/net/ethernet/amazon/ena/ena_com.h index 3c5081d9d25d..9414e93d107b 100644 --- a/drivers/net/ethernet/amazon/ena/ena_com.h +++ b/drivers/net/ethernet/amazon/ena/ena_com.h @@ -42,12 +42,14 @@ #define ADMIN_CQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_acq_entry)) #define ADMIN_AENQ_SIZE(depth) ((depth) * sizeof(struct ena_admin_aenq_entry)) +#define ENA_CUSTOMER_METRICS_BUFFER_SIZE 512 + /*****************************************************************************/ /*****************************************************************************/ /* ENA adaptive interrupt moderation settings */ #define ENA_INTR_INITIAL_TX_INTERVAL_USECS 64 -#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 0 +#define ENA_INTR_INITIAL_RX_INTERVAL_USECS 20 #define ENA_DEFAULT_INTR_DELAY_RESOLUTION 1 #define ENA_HASH_KEY_SIZE 40 @@ -109,16 +111,13 @@ struct ena_com_io_cq { /* Interrupt unmask register */ u32 __iomem *unmask_reg; - /* The completion queue head doorbell register */ - u32 __iomem *cq_head_db_reg; - /* numa configuration register (for TPH) */ u32 __iomem *numa_node_cfg_reg; /* The value to write to the above register to unmask * the interrupt of this queue */ - u32 msix_vector; + u32 msix_vector ____cacheline_aligned; enum queue_direction direction; @@ -134,7 +133,6 @@ struct ena_com_io_cq { /* Device queue index */ u16 idx; u16 head; - u16 last_head_update; u8 phase; u8 cdesc_entry_size_in_bytes; @@ -158,7 +156,6 @@ struct ena_com_io_sq { struct ena_com_io_desc_addr desc_addr; u32 __iomem *db_addr; - u8 __iomem *header_addr; enum queue_direction direction; enum ena_admin_placement_policy_type mem_queue_type; @@ -227,9 +224,6 @@ struct ena_com_admin_queue { /* Indicate if the admin queue should poll for completion */ bool polling; - /* Define if fallback to polling mode should occur */ - bool auto_polling; - u16 curr_cmd_id; /* Indicate that the ena was initialized and can @@ -283,6 +277,16 @@ struct ena_rss { }; +struct ena_customer_metrics { + /* in correlation with ENA_ADMIN_CUSTOMER_METRICS_SUPPORT_MASK + * and ena_admin_customer_metrics_id + */ + u64 supported_metrics; + dma_addr_t buffer_dma_addr; + void *buffer_virt_addr; + u32 buffer_len; +}; + struct ena_host_attribute { /* Debug area */ u8 *debug_area_virt_addr; @@ -310,6 +314,8 @@ struct ena_com_dev { u16 stats_func; /* Selected function for extended statistic dump */ u16 stats_queue; /* Selected queue for extended statistic dump */ + u32 ena_min_poll_delay_us; + struct ena_com_mmio_read mmio_read; struct ena_rss rss; @@ -331,7 +337,7 @@ struct ena_com_dev { struct ena_com_llq_info llq_info; - u32 ena_min_poll_delay_us; + struct ena_customer_metrics customer_metrics; }; struct ena_com_dev_get_features_ctx { @@ -484,17 +490,6 @@ bool ena_com_get_admin_running_state(struct ena_com_dev *ena_dev); */ void ena_com_set_admin_polling_mode(struct ena_com_dev *ena_dev, bool polling); -/* ena_com_set_admin_auto_polling_mode - Enable autoswitch to polling mode - * @ena_dev: ENA communication layer struct - * @polling: Enable/Disable polling mode - * - * Set the autopolling mode. - * If autopolling is on: - * In case of missing interrupt when data is available switch to polling. - */ -void ena_com_set_admin_auto_polling_mode(struct ena_com_dev *ena_dev, - bool polling); - /* ena_com_admin_q_comp_intr_handler - admin queue interrupt handler * @ena_dev: ENA communication layer struct * @@ -582,40 +577,40 @@ int ena_com_set_aenq_config(struct ena_com_dev *ena_dev, u32 groups_flag); int ena_com_get_dev_attr_feat(struct ena_com_dev *ena_dev, struct ena_com_dev_get_features_ctx *get_feat_ctx); -/* ena_com_get_dev_basic_stats - Get device basic statistics +/* ena_com_get_eni_stats - Get extended network interface statistics * @ena_dev: ENA communication layer struct * @stats: stats return value * * @return: 0 on Success and negative value otherwise. */ -int ena_com_get_dev_basic_stats(struct ena_com_dev *ena_dev, - struct ena_admin_basic_stats *stats); +int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, + struct ena_admin_eni_stats *stats); -/* ena_com_get_eni_stats - Get extended network interface statistics +/* ena_com_get_ena_srd_info - Get ENA SRD network interface statistics * @ena_dev: ENA communication layer struct - * @stats: stats return value + * @info: ena srd stats and flags * * @return: 0 on Success and negative value otherwise. */ -int ena_com_get_eni_stats(struct ena_com_dev *ena_dev, - struct ena_admin_eni_stats *stats); +int ena_com_get_ena_srd_info(struct ena_com_dev *ena_dev, + struct ena_admin_ena_srd_info *info); -/* ena_com_set_dev_mtu - Configure the device mtu. +/* ena_com_get_customer_metrics - Get customer metrics for network interface * @ena_dev: ENA communication layer struct - * @mtu: mtu value + * @buffer: buffer for returned customer metrics + * @len: size of the buffer * * @return: 0 on Success and negative value otherwise. */ -int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu); +int ena_com_get_customer_metrics(struct ena_com_dev *ena_dev, char *buffer, u32 len); -/* ena_com_get_offload_settings - Retrieve the device offloads capabilities +/* ena_com_set_dev_mtu - Configure the device mtu. * @ena_dev: ENA communication layer struct - * @offlad: offload return value + * @mtu: mtu value * * @return: 0 on Success and negative value otherwise. */ -int ena_com_get_offload_settings(struct ena_com_dev *ena_dev, - struct ena_admin_feature_offload_desc *offload); +int ena_com_set_dev_mtu(struct ena_com_dev *ena_dev, u32 mtu); /* ena_com_rss_init - Init RSS * @ena_dev: ENA communication layer struct @@ -810,6 +805,13 @@ int ena_com_allocate_host_info(struct ena_com_dev *ena_dev); int ena_com_allocate_debug_area(struct ena_com_dev *ena_dev, u32 debug_area_size); +/* ena_com_allocate_customer_metrics_buffer - Allocate customer metrics resources. + * @ena_dev: ENA communication layer struct + * + * @return: 0 on Success and negative value otherwise. + */ +int ena_com_allocate_customer_metrics_buffer(struct ena_com_dev *ena_dev); + /* ena_com_delete_debug_area - Free the debug area resources. * @ena_dev: ENA communication layer struct * @@ -824,6 +826,13 @@ void ena_com_delete_debug_area(struct ena_com_dev *ena_dev); */ void ena_com_delete_host_info(struct ena_com_dev *ena_dev); +/* ena_com_delete_customer_metrics_buffer - Free the customer metrics resources. + * @ena_dev: ENA communication layer struct + * + * Free the allocated customer metrics area. + */ +void ena_com_delete_customer_metrics_buffer(struct ena_com_dev *ena_dev); + /* ena_com_set_host_attributes - Update the device with the host * attributes (debug area and host info) base address. * @ena_dev: ENA communication layer struct @@ -980,6 +989,28 @@ static inline bool ena_com_get_cap(struct ena_com_dev *ena_dev, return !!(ena_dev->capabilities & BIT(cap_id)); } +/* ena_com_get_customer_metric_support - query whether device supports a given customer metric. + * @ena_dev: ENA communication layer struct + * @metric_id: enum value representing the customer metric + * + * @return - true if customer metric is supported or false otherwise + */ +static inline bool ena_com_get_customer_metric_support(struct ena_com_dev *ena_dev, + enum ena_admin_customer_metrics_id metric_id) +{ + return !!(ena_dev->customer_metrics.supported_metrics & BIT(metric_id)); +} + +/* ena_com_get_customer_metric_count - return the number of supported customer metrics. + * @ena_dev: ENA communication layer struct + * + * @return - the number of supported customer metrics + */ +static inline int ena_com_get_customer_metric_count(struct ena_com_dev *ena_dev) +{ + return hweight64(ena_dev->customer_metrics.supported_metrics); +} + /* ena_com_update_intr_reg - Prepare interrupt register * @intr_reg: interrupt register to update. * @rx_delay_interval: Rx interval in usecs diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.c b/drivers/net/ethernet/amazon/ena/ena_eth_com.c index f9f886289b97..4c6e07aa4bbb 100644 --- a/drivers/net/ethernet/amazon/ena/ena_eth_com.c +++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.c @@ -18,8 +18,7 @@ static struct ena_eth_io_rx_cdesc_base *ena_com_get_next_rx_cdesc( cdesc = (struct ena_eth_io_rx_cdesc_base *)(io_cq->cdesc_addr.virt_addr + (head_masked * io_cq->cdesc_entry_size_in_bytes)); - desc_phase = (READ_ONCE(cdesc->status) & - ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> + desc_phase = (READ_ONCE(cdesc->status) & ENA_ETH_IO_RX_CDESC_BASE_PHASE_MASK) >> ENA_ETH_IO_RX_CDESC_BASE_PHASE_SHIFT; if (desc_phase != expected_phase) @@ -65,8 +64,8 @@ static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq, io_sq->entries_in_tx_burst_left--; netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Decreasing entries_in_tx_burst_left of queue %d to %d\n", - io_sq->qid, io_sq->entries_in_tx_burst_left); + "Decreasing entries_in_tx_burst_left of queue %d to %d\n", io_sq->qid, + io_sq->entries_in_tx_burst_left); } /* Make sure everything was written into the bounce buffer before @@ -75,8 +74,8 @@ static int ena_com_write_bounce_buffer_to_dev(struct ena_com_io_sq *io_sq, wmb(); /* The line is completed. Copy it to dev */ - __iowrite64_copy(io_sq->desc_addr.pbuf_dev_addr + dst_offset, - bounce_buffer, (llq_info->desc_list_entry_size) / 8); + __iowrite64_copy(io_sq->desc_addr.pbuf_dev_addr + dst_offset, bounce_buffer, + (llq_info->desc_list_entry_size) / 8); io_sq->tail++; @@ -102,16 +101,14 @@ static int ena_com_write_header_to_bounce(struct ena_com_io_sq *io_sq, header_offset = llq_info->descs_num_before_header * io_sq->desc_entry_size; - if (unlikely((header_offset + header_len) > - llq_info->desc_list_entry_size)) { + if (unlikely((header_offset + header_len) > llq_info->desc_list_entry_size)) { netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, "Trying to write header larger than llq entry can accommodate\n"); return -EFAULT; } if (unlikely(!bounce_buffer)) { - netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Bounce buffer is NULL\n"); + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, "Bounce buffer is NULL\n"); return -EFAULT; } @@ -129,8 +126,7 @@ static void *get_sq_desc_llq(struct ena_com_io_sq *io_sq) bounce_buffer = pkt_ctrl->curr_bounce_buf; if (unlikely(!bounce_buffer)) { - netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Bounce buffer is NULL\n"); + netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, "Bounce buffer is NULL\n"); return NULL; } @@ -233,31 +229,43 @@ static struct ena_eth_io_rx_cdesc_base * idx * io_cq->cdesc_entry_size_in_bytes); } -static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, - u16 *first_cdesc_idx) +static int ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, + u16 *first_cdesc_idx, + u16 *num_descs) { + u16 count = io_cq->cur_rx_pkt_cdesc_count, head_masked; struct ena_eth_io_rx_cdesc_base *cdesc; - u16 count = 0, head_masked; u32 last = 0; do { + u32 status; + cdesc = ena_com_get_next_rx_cdesc(io_cq); if (!cdesc) break; + status = READ_ONCE(cdesc->status); ena_com_cq_inc_head(io_cq); + if (unlikely((status & ENA_ETH_IO_RX_CDESC_BASE_FIRST_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_FIRST_SHIFT && count != 0)) { + struct ena_com_dev *dev = ena_com_io_cq_to_ena_dev(io_cq); + + netdev_err(dev->net_device, + "First bit is on in descriptor #%d on q_id: %d, req_id: %u\n", + count, io_cq->qid, cdesc->req_id); + return -EFAULT; + } count++; - last = (READ_ONCE(cdesc->status) & - ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> - ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; + last = (status & ENA_ETH_IO_RX_CDESC_BASE_LAST_MASK) >> + ENA_ETH_IO_RX_CDESC_BASE_LAST_SHIFT; } while (!last); if (last) { *first_cdesc_idx = io_cq->cur_rx_pkt_cdesc_start_idx; - count += io_cq->cur_rx_pkt_cdesc_count; head_masked = io_cq->head & (io_cq->q_depth - 1); + *num_descs = count; io_cq->cur_rx_pkt_cdesc_count = 0; io_cq->cur_rx_pkt_cdesc_start_idx = head_masked; @@ -265,11 +273,11 @@ static u16 ena_com_cdesc_rx_pkt_get(struct ena_com_io_cq *io_cq, "ENA q_id: %d packets were completed. first desc idx %u descs# %d\n", io_cq->qid, *first_cdesc_idx, count); } else { - io_cq->cur_rx_pkt_cdesc_count += count; - count = 0; + io_cq->cur_rx_pkt_cdesc_count = count; + *num_descs = 0; } - return count; + return 0; } static int ena_com_create_meta(struct ena_com_io_sq *io_sq, @@ -369,9 +377,8 @@ static void ena_com_rx_set_flags(struct ena_com_io_cq *io_cq, netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device, "l3_proto %d l4_proto %d l3_csum_err %d l4_csum_err %d hash %d frag %d cdesc_status %x\n", - ena_rx_ctx->l3_proto, ena_rx_ctx->l4_proto, - ena_rx_ctx->l3_csum_err, ena_rx_ctx->l4_csum_err, - ena_rx_ctx->hash, ena_rx_ctx->frag, cdesc->status); + ena_rx_ctx->l3_proto, ena_rx_ctx->l4_proto, ena_rx_ctx->l3_csum_err, + ena_rx_ctx->l4_csum_err, ena_rx_ctx->hash, ena_rx_ctx->frag, cdesc->status); } /*****************************************************************************/ @@ -403,13 +410,12 @@ int ena_com_prepare_tx(struct ena_com_io_sq *io_sq, if (unlikely(header_len > io_sq->tx_max_header_size)) { netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Header size is too large %d max header: %d\n", - header_len, io_sq->tx_max_header_size); + "Header size is too large %d max header: %d\n", header_len, + io_sq->tx_max_header_size); return -EINVAL; } - if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && - !buffer_to_push)) { + if (unlikely(io_sq->mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV && !buffer_to_push)) { netdev_err(ena_com_io_sq_to_ena_dev(io_sq)->net_device, "Push header wasn't provided in LLQ mode\n"); return -EINVAL; @@ -546,23 +552,25 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, u16 cdesc_idx = 0; u16 nb_hw_desc; u16 i = 0; + int rc; WARN(io_cq->direction != ENA_COM_IO_QUEUE_DIRECTION_RX, "wrong Q type"); - nb_hw_desc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx); + rc = ena_com_cdesc_rx_pkt_get(io_cq, &cdesc_idx, &nb_hw_desc); + if (unlikely(rc != 0)) + return -EFAULT; + if (nb_hw_desc == 0) { ena_rx_ctx->descs = nb_hw_desc; return 0; } netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device, - "Fetch rx packet: queue %d completed desc: %d\n", io_cq->qid, - nb_hw_desc); + "Fetch rx packet: queue %d completed desc: %d\n", io_cq->qid, nb_hw_desc); if (unlikely(nb_hw_desc > ena_rx_ctx->max_bufs)) { netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device, - "Too many RX cdescs (%d) > MAX(%d)\n", nb_hw_desc, - ena_rx_ctx->max_bufs); + "Too many RX cdescs (%d) > MAX(%d)\n", nb_hw_desc, ena_rx_ctx->max_bufs); return -ENOSPC; } @@ -586,8 +594,8 @@ int ena_com_rx_pkt(struct ena_com_io_cq *io_cq, io_sq->next_to_comp += nb_hw_desc; netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device, - "[%s][QID#%d] Updating SQ head to: %d\n", __func__, - io_sq->qid, io_sq->next_to_comp); + "[%s][QID#%d] Updating SQ head to: %d\n", __func__, io_sq->qid, + io_sq->next_to_comp); /* Get rx flags from the last pkt */ ena_com_rx_set_flags(io_cq, ena_rx_ctx, cdesc); @@ -624,8 +632,8 @@ int ena_com_add_single_rx_desc(struct ena_com_io_sq *io_sq, desc->req_id = req_id; netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "[%s] Adding single RX desc, Queue: %u, req_id: %u\n", - __func__, io_sq->qid, req_id); + "[%s] Adding single RX desc, Queue: %u, req_id: %u\n", __func__, io_sq->qid, + req_id); desc->buff_addr_lo = (u32)ena_buf->paddr; desc->buff_addr_hi = diff --git a/drivers/net/ethernet/amazon/ena/ena_eth_com.h b/drivers/net/ethernet/amazon/ena/ena_eth_com.h index 372b259279ec..449bc4960ccc 100644 --- a/drivers/net/ethernet/amazon/ena/ena_eth_com.h +++ b/drivers/net/ethernet/amazon/ena/ena_eth_com.h @@ -8,8 +8,6 @@ #include "ena_com.h" -/* head update threshold in units of (queue size / ENA_COMP_HEAD_THRESH) */ -#define ENA_COMP_HEAD_THRESH 4 /* we allow 2 DMA descriptors per LLQ entry */ #define ENA_LLQ_ENTRY_DESC_CHUNK_SIZE (2 * sizeof(struct ena_eth_io_tx_desc)) #define ENA_LLQ_HEADER (128UL - ENA_LLQ_ENTRY_DESC_CHUNK_SIZE) @@ -49,7 +47,7 @@ struct ena_com_rx_ctx { bool frag; u32 hash; u16 descs; - int max_bufs; + u16 max_bufs; u8 pkt_offset; }; @@ -145,8 +143,8 @@ static inline bool ena_com_is_doorbell_needed(struct ena_com_io_sq *io_sq, } netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Queue: %d num_descs: %d num_entries_needed: %d\n", - io_sq->qid, num_descs, num_entries_needed); + "Queue: %d num_descs: %d num_entries_needed: %d\n", io_sq->qid, num_descs, + num_entries_needed); return num_entries_needed > io_sq->entries_in_tx_burst_left; } @@ -157,43 +155,20 @@ static inline int ena_com_write_sq_doorbell(struct ena_com_io_sq *io_sq) u16 tail = io_sq->tail; netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Write submission queue doorbell for queue: %d tail: %d\n", - io_sq->qid, tail); + "Write submission queue doorbell for queue: %d tail: %d\n", io_sq->qid, tail); writel(tail, io_sq->db_addr); if (is_llq_max_tx_burst_exists(io_sq)) { netdev_dbg(ena_com_io_sq_to_ena_dev(io_sq)->net_device, - "Reset available entries in tx burst for queue %d to %d\n", - io_sq->qid, max_entries_in_tx_burst); + "Reset available entries in tx burst for queue %d to %d\n", io_sq->qid, + max_entries_in_tx_burst); io_sq->entries_in_tx_burst_left = max_entries_in_tx_burst; } return 0; } -static inline int ena_com_update_dev_comp_head(struct ena_com_io_cq *io_cq) -{ - u16 unreported_comp, head; - bool need_update; - - if (unlikely(io_cq->cq_head_db_reg)) { - head = io_cq->head; - unreported_comp = head - io_cq->last_head_update; - need_update = unreported_comp > (io_cq->q_depth / ENA_COMP_HEAD_THRESH); - - if (unlikely(need_update)) { - netdev_dbg(ena_com_io_cq_to_ena_dev(io_cq)->net_device, - "Write completion queue doorbell for queue %d: head: %d\n", - io_cq->qid, head); - writel(head, io_cq->cq_head_db_reg); - io_cq->last_head_update = head; - } - } - - return 0; -} - static inline void ena_com_update_numa_node(struct ena_com_io_cq *io_cq, u8 numa_node) { @@ -248,8 +223,8 @@ static inline int ena_com_tx_comp_req_id_get(struct ena_com_io_cq *io_cq, *req_id = READ_ONCE(cdesc->req_id); if (unlikely(*req_id >= io_cq->q_depth)) { - netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device, - "Invalid req id %d\n", cdesc->req_id); + netdev_err(ena_com_io_cq_to_ena_dev(io_cq)->net_device, "Invalid req id %d\n", + cdesc->req_id); return -EINVAL; } diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 0cb6cc1cef56..a3c934c3de71 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -14,6 +14,10 @@ struct ena_stats { int stat_offset; }; +struct ena_hw_metrics { + char name[ETH_GSTRING_LEN]; +}; + #define ENA_STAT_ENA_COM_ENTRY(stat) { \ .name = #stat, \ .stat_offset = offsetof(struct ena_com_stats_admin, stat) / sizeof(u64) \ @@ -41,6 +45,18 @@ struct ena_stats { #define ENA_STAT_ENI_ENTRY(stat) \ ENA_STAT_HW_ENTRY(stat, eni_stats) +#define ENA_STAT_ENA_SRD_ENTRY(stat) \ + ENA_STAT_HW_ENTRY(stat, ena_srd_stats) + +#define ENA_STAT_ENA_SRD_MODE_ENTRY(stat) { \ + .name = #stat, \ + .stat_offset = offsetof(struct ena_admin_ena_srd_info, flags) / sizeof(u64) \ +} + +#define ENA_METRIC_ENI_ENTRY(stat) { \ + .name = #stat \ +} + static const struct ena_stats ena_stats_global_strings[] = { ENA_STAT_GLOBAL_ENTRY(tx_timeout), ENA_STAT_GLOBAL_ENTRY(suspend), @@ -49,8 +65,12 @@ static const struct ena_stats ena_stats_global_strings[] = { ENA_STAT_GLOBAL_ENTRY(interface_up), ENA_STAT_GLOBAL_ENTRY(interface_down), ENA_STAT_GLOBAL_ENTRY(admin_q_pause), + ENA_STAT_GLOBAL_ENTRY(reset_fail), }; +/* A partial list of hw stats. Used when admin command + * with type ENA_ADMIN_GET_STATS_TYPE_CUSTOMER_METRICS is not supported + */ static const struct ena_stats ena_stats_eni_strings[] = { ENA_STAT_ENI_ENTRY(bw_in_allowance_exceeded), ENA_STAT_ENI_ENTRY(bw_out_allowance_exceeded), @@ -59,6 +79,23 @@ static const struct ena_stats ena_stats_eni_strings[] = { ENA_STAT_ENI_ENTRY(linklocal_allowance_exceeded), }; +static const struct ena_hw_metrics ena_hw_stats_strings[] = { + ENA_METRIC_ENI_ENTRY(bw_in_allowance_exceeded), + ENA_METRIC_ENI_ENTRY(bw_out_allowance_exceeded), + ENA_METRIC_ENI_ENTRY(pps_allowance_exceeded), + ENA_METRIC_ENI_ENTRY(conntrack_allowance_exceeded), + ENA_METRIC_ENI_ENTRY(linklocal_allowance_exceeded), + ENA_METRIC_ENI_ENTRY(conntrack_allowance_available), +}; + +static const struct ena_stats ena_srd_info_strings[] = { + ENA_STAT_ENA_SRD_MODE_ENTRY(ena_srd_mode), + ENA_STAT_ENA_SRD_ENTRY(ena_srd_tx_pkts), + ENA_STAT_ENA_SRD_ENTRY(ena_srd_eligible_tx_pkts), + ENA_STAT_ENA_SRD_ENTRY(ena_srd_rx_pkts), + ENA_STAT_ENA_SRD_ENTRY(ena_srd_resource_utilization) +}; + static const struct ena_stats ena_stats_tx_strings[] = { ENA_STAT_TX_ENTRY(cnt), ENA_STAT_TX_ENTRY(bytes), @@ -111,7 +148,9 @@ static const struct ena_stats ena_stats_ena_com_strings[] = { #define ENA_STATS_ARRAY_TX ARRAY_SIZE(ena_stats_tx_strings) #define ENA_STATS_ARRAY_RX ARRAY_SIZE(ena_stats_rx_strings) #define ENA_STATS_ARRAY_ENA_COM ARRAY_SIZE(ena_stats_ena_com_strings) -#define ENA_STATS_ARRAY_ENI(adapter) ARRAY_SIZE(ena_stats_eni_strings) +#define ENA_STATS_ARRAY_ENI ARRAY_SIZE(ena_stats_eni_strings) +#define ENA_STATS_ARRAY_ENA_SRD ARRAY_SIZE(ena_srd_info_strings) +#define ENA_METRICS_ARRAY_ENI ARRAY_SIZE(ena_hw_stats_strings) static void ena_safe_update_stat(u64 *src, u64 *dst, struct u64_stats_sync *syncp) @@ -124,6 +163,57 @@ static void ena_safe_update_stat(u64 *src, u64 *dst, } while (u64_stats_fetch_retry(syncp, start)); } +static void ena_metrics_stats(struct ena_adapter *adapter, u64 **data) +{ + struct ena_com_dev *dev = adapter->ena_dev; + const struct ena_stats *ena_stats; + u64 *ptr; + int i; + + if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { + u32 supported_metrics_count; + int len; + + supported_metrics_count = ena_com_get_customer_metric_count(dev); + len = supported_metrics_count * sizeof(u64); + + /* Fill the data buffer, and advance its pointer */ + ena_com_get_customer_metrics(dev, (char *)(*data), len); + (*data) += supported_metrics_count; + + } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { + ena_com_get_eni_stats(dev, &adapter->eni_stats); + /* Updating regardless of rc - once we told ethtool how many stats we have + * it will print that much stats. We can't leave holes in the stats + */ + for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { + ena_stats = &ena_stats_eni_strings[i]; + + ptr = (u64 *)&adapter->eni_stats + + ena_stats->stat_offset; + + ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); + } + } + + if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { + ena_com_get_ena_srd_info(dev, &adapter->ena_srd_info); + /* Get ENA SRD mode */ + ptr = (u64 *)&adapter->ena_srd_info; + ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); + for (i = 1; i < ENA_STATS_ARRAY_ENA_SRD; i++) { + ena_stats = &ena_srd_info_strings[i]; + /* Wrapped within an outer struct - need to accommodate an + * additional offset of the ENA SRD mode that was already processed + */ + ptr = (u64 *)&adapter->ena_srd_info + + ena_stats->stat_offset + 1; + + ena_safe_update_stat(ptr, (*data)++, &adapter->syncp); + } + } +} + static void ena_queue_stats(struct ena_adapter *adapter, u64 **data) { const struct ena_stats *ena_stats; @@ -178,7 +268,7 @@ static void ena_dev_admin_queue_stats(struct ena_adapter *adapter, u64 **data) static void ena_get_stats(struct ena_adapter *adapter, u64 *data, - bool eni_stats_needed) + bool hw_stats_needed) { const struct ena_stats *ena_stats; u64 *ptr; @@ -192,17 +282,8 @@ static void ena_get_stats(struct ena_adapter *adapter, ena_safe_update_stat(ptr, data++, &adapter->syncp); } - if (eni_stats_needed) { - ena_update_hw_stats(adapter); - for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) { - ena_stats = &ena_stats_eni_strings[i]; - - ptr = (u64 *)&adapter->eni_stats + - ena_stats->stat_offset; - - ena_safe_update_stat(ptr, data++, &adapter->syncp); - } - } + if (hw_stats_needed) + ena_metrics_stats(adapter, &data); ena_queue_stats(adapter, &data); ena_dev_admin_queue_stats(adapter, &data); @@ -213,9 +294,8 @@ static void ena_get_ethtool_stats(struct net_device *netdev, u64 *data) { struct ena_adapter *adapter = netdev_priv(netdev); - struct ena_com_dev *dev = adapter->ena_dev; - ena_get_stats(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)); + ena_get_stats(adapter, data, true); } static int ena_get_sw_stats_count(struct ena_adapter *adapter) @@ -227,9 +307,17 @@ static int ena_get_sw_stats_count(struct ena_adapter *adapter) static int ena_get_hw_stats_count(struct ena_adapter *adapter) { - bool supported = ena_com_get_cap(adapter->ena_dev, ENA_ADMIN_ENI_STATS); + struct ena_com_dev *dev = adapter->ena_dev; + int count; + + count = ENA_STATS_ARRAY_ENA_SRD * ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO); + + if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) + count += ena_com_get_customer_metric_count(dev); + else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) + count += ENA_STATS_ARRAY_ENI; - return ENA_STATS_ARRAY_ENI(adapter) * supported; + return count; } int ena_get_sset_count(struct net_device *netdev, int sset) @@ -245,6 +333,35 @@ int ena_get_sset_count(struct net_device *netdev, int sset) return -EOPNOTSUPP; } +static void ena_metrics_stats_strings(struct ena_adapter *adapter, u8 **data) +{ + struct ena_com_dev *dev = adapter->ena_dev; + const struct ena_hw_metrics *ena_metrics; + const struct ena_stats *ena_stats; + int i; + + if (ena_com_get_cap(dev, ENA_ADMIN_CUSTOMER_METRICS)) { + for (i = 0; i < ENA_METRICS_ARRAY_ENI; i++) { + if (ena_com_get_customer_metric_support(dev, i)) { + ena_metrics = &ena_hw_stats_strings[i]; + ethtool_puts(data, ena_metrics->name); + } + } + } else if (ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)) { + for (i = 0; i < ENA_STATS_ARRAY_ENI; i++) { + ena_stats = &ena_stats_eni_strings[i]; + ethtool_puts(data, ena_stats->name); + } + } + + if (ena_com_get_cap(dev, ENA_ADMIN_ENA_SRD_INFO)) { + for (i = 0; i < ENA_STATS_ARRAY_ENA_SRD; i++) { + ena_stats = &ena_srd_info_strings[i]; + ethtool_puts(data, ena_stats->name); + } + } +} + static void ena_queue_strings(struct ena_adapter *adapter, u8 **data) { const struct ena_stats *ena_stats; @@ -290,7 +407,7 @@ static void ena_com_dev_strings(u8 **data) static void ena_get_strings(struct ena_adapter *adapter, u8 *data, - bool eni_stats_needed) + bool hw_stats_needed) { const struct ena_stats *ena_stats; int i; @@ -300,12 +417,8 @@ static void ena_get_strings(struct ena_adapter *adapter, ethtool_puts(&data, ena_stats->name); } - if (eni_stats_needed) { - for (i = 0; i < ENA_STATS_ARRAY_ENI(adapter); i++) { - ena_stats = &ena_stats_eni_strings[i]; - ethtool_puts(&data, ena_stats->name); - } - } + if (hw_stats_needed) + ena_metrics_stats_strings(adapter, &data); ena_queue_strings(adapter, &data); ena_com_dev_strings(&data); @@ -316,11 +429,10 @@ static void ena_get_ethtool_strings(struct net_device *netdev, u8 *data) { struct ena_adapter *adapter = netdev_priv(netdev); - struct ena_com_dev *dev = adapter->ena_dev; switch (sset) { case ETH_SS_STATS: - ena_get_strings(adapter, data, ena_com_get_cap(dev, ENA_ADMIN_ENI_STATS)); + ena_get_strings(adapter, data, true); break; } } @@ -459,10 +571,18 @@ static void ena_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) { struct ena_adapter *adapter = netdev_priv(dev); - - strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); - strscpy(info->bus_info, pci_name(adapter->pdev), - sizeof(info->bus_info)); + ssize_t ret = 0; + + ret = strscpy(info->driver, DRV_MODULE_NAME, sizeof(info->driver)); + if (ret < 0) + netif_dbg(adapter, drv, dev, + "module name will be truncated, status = %zd\n", ret); + + ret = strscpy(info->bus_info, pci_name(adapter->pdev), + sizeof(info->bus_info)); + if (ret < 0) + netif_dbg(adapter, drv, dev, + "bus info will be truncated, status = %zd\n", ret); } static void ena_get_ringparam(struct net_device *netdev, @@ -1009,22 +1129,18 @@ static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf) return; } - strings_buf = devm_kcalloc(&adapter->pdev->dev, - ETH_GSTRING_LEN, strings_num, - GFP_ATOMIC); + strings_buf = kcalloc(strings_num, ETH_GSTRING_LEN, GFP_ATOMIC); if (!strings_buf) { netif_err(adapter, drv, netdev, "Failed to allocate strings_buf\n"); return; } - data_buf = devm_kcalloc(&adapter->pdev->dev, - strings_num, sizeof(u64), - GFP_ATOMIC); + data_buf = kcalloc(strings_num, sizeof(u64), GFP_ATOMIC); if (!data_buf) { netif_err(adapter, drv, netdev, "Failed to allocate data buf\n"); - devm_kfree(&adapter->pdev->dev, strings_buf); + kfree(strings_buf); return; } @@ -1046,8 +1162,8 @@ static void ena_dump_stats_ex(struct ena_adapter *adapter, u8 *buf) strings_buf + i * ETH_GSTRING_LEN, data_buf[i]); - devm_kfree(&adapter->pdev->dev, strings_buf); - devm_kfree(&adapter->pdev->dev, data_buf); + kfree(strings_buf); + kfree(data_buf); } void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf) diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.c b/drivers/net/ethernet/amazon/ena/ena_netdev.c index 1c0a7828d397..c1295dfad0d0 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.c +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.c @@ -32,7 +32,7 @@ MODULE_LICENSE("GPL"); #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus()) #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \ - NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR) + NETIF_MSG_IFDOWN | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR) static struct ena_aenq_handlers aenq_handlers; @@ -42,24 +42,49 @@ MODULE_DEVICE_TABLE(pci, ena_pci_tbl); static int ena_rss_init_default(struct ena_adapter *adapter); static void check_for_admin_com_state(struct ena_adapter *adapter); -static void ena_destroy_device(struct ena_adapter *adapter, bool graceful); +static int ena_destroy_device(struct ena_adapter *adapter, bool graceful); static int ena_restore_device(struct ena_adapter *adapter); static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue) { + enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_OS_NETDEV_WD; struct ena_adapter *adapter = netdev_priv(dev); + unsigned int time_since_last_napi, threshold; + struct ena_ring *tx_ring; + int napi_scheduled; + + if (txqueue >= adapter->num_io_queues) { + netdev_err(dev, "TX timeout on invalid queue %u\n", txqueue); + goto schedule_reset; + } + + threshold = jiffies_to_usecs(dev->watchdog_timeo); + tx_ring = &adapter->tx_ring[txqueue]; + time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies); + napi_scheduled = !!(tx_ring->napi->state & NAPIF_STATE_SCHED); + + netdev_err(dev, + "TX q %d is paused for too long (threshold %u). Time since last napi %u usec. napi scheduled: %d\n", + txqueue, + threshold, + time_since_last_napi, + napi_scheduled); + + if (threshold < time_since_last_napi && napi_scheduled) { + netdev_err(dev, + "napi handler hasn't been called for a long time but is scheduled\n"); + reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION; + } +schedule_reset: /* Change the state of the device to trigger reset * Check that we are not in the middle or a trigger already */ - if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) return; - ena_reset_device(adapter, ENA_REGS_RESET_OS_NETDEV_WD); + ena_reset_device(adapter, reset_reason); ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp); - - netif_err(adapter, tx_err, dev, "Transmit time out\n"); } static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu) @@ -79,7 +104,7 @@ static int ena_change_mtu(struct net_device *dev, int new_mtu) if (!ret) { netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu); update_rx_ring_mtu(adapter, new_mtu); - dev->mtu = new_mtu; + WRITE_ONCE(dev->mtu, new_mtu); } else { netif_err(adapter, drv, dev, "Failed to set MTU to %d\n", new_mtu); @@ -116,11 +141,9 @@ int ena_xmit_common(struct ena_adapter *adapter, if (unlikely(rc)) { netif_err(adapter, tx_queued, adapter->netdev, "Failed to prepare tx bufs\n"); - ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, - &ring->syncp); + ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1, &ring->syncp); if (rc != -ENOMEM) - ena_reset_device(adapter, - ENA_REGS_RESET_DRIVER_INVALID_STATE); + ena_reset_device(adapter, ENA_REGS_RESET_DRIVER_INVALID_STATE); return rc; } @@ -485,8 +508,7 @@ static struct page *ena_alloc_map_page(struct ena_ring *rx_ring, */ page = dev_alloc_page(); if (!page) { - ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, - &rx_ring->syncp); + ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1, &rx_ring->syncp); return ERR_PTR(-ENOSPC); } @@ -523,7 +545,7 @@ static int ena_alloc_rx_buffer(struct ena_ring *rx_ring, /* We handle DMA here */ page = ena_alloc_map_page(rx_ring, &dma); - if (unlikely(IS_ERR(page))) + if (IS_ERR(page)) return PTR_ERR(page); netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev, @@ -545,8 +567,8 @@ static void ena_unmap_rx_buff_attrs(struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info, unsigned long attrs) { - dma_unmap_page_attrs(rx_ring->dev, rx_info->dma_addr, ENA_PAGE_SIZE, - DMA_BIDIRECTIONAL, attrs); + dma_unmap_page_attrs(rx_ring->dev, rx_info->dma_addr, ENA_PAGE_SIZE, DMA_BIDIRECTIONAL, + attrs); } static void ena_free_rx_page(struct ena_ring *rx_ring, @@ -696,8 +718,11 @@ void ena_unmap_tx_buff(struct ena_ring *tx_ring, static void ena_free_tx_bufs(struct ena_ring *tx_ring) { bool print_once = true; + bool is_xdp_ring; u32 i; + is_xdp_ring = ENA_IS_XDP_INDEX(tx_ring->adapter, tx_ring->qid); + for (i = 0; i < tx_ring->ring_size; i++) { struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i]; @@ -717,10 +742,15 @@ static void ena_free_tx_bufs(struct ena_ring *tx_ring) ena_unmap_tx_buff(tx_ring, tx_info); - dev_kfree_skb_any(tx_info->skb); + if (is_xdp_ring) + xdp_return_frame(tx_info->xdpf); + else + dev_kfree_skb_any(tx_info->skb); } - netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev, - tx_ring->qid)); + + if (!is_xdp_ring) + netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev, + tx_ring->qid)); } static void ena_free_all_tx_bufs(struct ena_adapter *adapter) @@ -819,8 +849,7 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) &req_id); if (rc) { if (unlikely(rc == -EINVAL)) - handle_invalid_req_id(tx_ring, req_id, NULL, - false); + handle_invalid_req_id(tx_ring, req_id, NULL, false); break; } @@ -856,7 +885,6 @@ static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget) tx_ring->next_to_clean = next_to_clean; ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done); - ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); netdev_tx_completed_queue(txq, tx_pkts, tx_bytes); @@ -1046,8 +1074,7 @@ static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring, DMA_FROM_DEVICE); if (!reuse_rx_buf_page) - ena_unmap_rx_buff_attrs(rx_ring, rx_info, - DMA_ATTR_SKIP_CPU_SYNC); + ena_unmap_rx_buff_attrs(rx_ring, rx_info, DMA_ATTR_SKIP_CPU_SYNC); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page, page_offset + buf_offset, len, buf_len); @@ -1303,10 +1330,8 @@ static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi, ENA_RX_REFILL_THRESH_PACKET); /* Optimization, try to batch new rx buffers */ - if (refill_required > refill_threshold) { - ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq); + if (refill_required > refill_threshold) ena_refill_rx_bufs(rx_ring, refill_required); - } if (xdp_flags & ENA_XDP_REDIRECT) xdp_do_flush(); @@ -1320,9 +1345,10 @@ error: adapter = netdev_priv(rx_ring->netdev); if (rc == -ENOSPC) { - ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, - &rx_ring->syncp); + ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1, &rx_ring->syncp); ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS); + } else if (rc == -EFAULT) { + ena_reset_device(adapter, ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED); } else { ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp); @@ -1357,7 +1383,7 @@ static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi) rx_ring->rx_stats.bytes, &dim_sample); - net_dim(&ena_napi->dim, dim_sample); + net_dim(&ena_napi->dim, &dim_sample); rx_ring->per_napi_packets = 0; } @@ -1651,9 +1677,9 @@ static int ena_request_mgmnt_irq(struct ena_adapter *adapter) static int ena_request_io_irq(struct ena_adapter *adapter) { u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; + int rc = 0, i, k, irq_idx; unsigned long flags = 0; struct ena_irq *irq; - int rc = 0, i, k; if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) { netif_err(adapter, ifup, adapter->netdev, @@ -1679,6 +1705,16 @@ static int ena_request_io_irq(struct ena_adapter *adapter) irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask); } + /* Now that IO IRQs have been successfully allocated map them to the + * corresponding IO NAPI instance. Note that the mgmnt IRQ does not + * have a NAPI, so care must be taken to correctly map IRQs to NAPIs. + */ + for (i = 0; i < io_queue_count; i++) { + irq_idx = ENA_IO_IRQ_IDX(i); + irq = &adapter->irq_tbl[irq_idx]; + netif_napi_set_irq(&adapter->ena_napi[i].napi, irq->vector); + } + return rc; err: @@ -1785,20 +1821,40 @@ static void ena_napi_disable_in_range(struct ena_adapter *adapter, int first_index, int count) { + struct napi_struct *napi; int i; - for (i = first_index; i < first_index + count; i++) - napi_disable(&adapter->ena_napi[i].napi); + for (i = first_index; i < first_index + count; i++) { + napi = &adapter->ena_napi[i].napi; + if (!ENA_IS_XDP_INDEX(adapter, i)) { + /* This API is supported for non-XDP queues only */ + netif_queue_set_napi(adapter->netdev, i, + NETDEV_QUEUE_TYPE_TX, NULL); + netif_queue_set_napi(adapter->netdev, i, + NETDEV_QUEUE_TYPE_RX, NULL); + } + napi_disable(napi); + } } static void ena_napi_enable_in_range(struct ena_adapter *adapter, int first_index, int count) { + struct napi_struct *napi; int i; - for (i = first_index; i < first_index + count; i++) - napi_enable(&adapter->ena_napi[i].napi); + for (i = first_index; i < first_index + count; i++) { + napi = &adapter->ena_napi[i].napi; + napi_enable(napi); + if (!ENA_IS_XDP_INDEX(adapter, i)) { + /* This API is supported for non-XDP queues only */ + netif_queue_set_napi(adapter->netdev, i, + NETDEV_QUEUE_TYPE_RX, napi); + netif_queue_set_napi(adapter->netdev, i, + NETDEV_QUEUE_TYPE_TX, napi); + } + } } /* Configure the Rx forwarding */ @@ -1811,8 +1867,7 @@ static int ena_rss_configure(struct ena_adapter *adapter) if (!ena_dev->rss.tbl_log_size) { rc = ena_rss_init_default(adapter); if (rc && (rc != -EOPNOTSUPP)) { - netif_err(adapter, ifup, adapter->netdev, - "Failed to init RSS rc: %d\n", rc); + netif_err(adapter, ifup, adapter->netdev, "Failed to init RSS rc: %d\n", rc); return rc; } } @@ -2134,6 +2189,12 @@ int ena_up(struct ena_adapter *adapter) */ ena_init_napi_in_range(adapter, 0, io_queue_count); + /* Enabling DIM needs to happen before enabling IRQs since DIM + * is run from napi routine + */ + if (ena_com_interrupt_moderation_supported(adapter->ena_dev)) + ena_com_enable_adaptive_moderation(adapter->ena_dev); + rc = ena_request_io_irq(adapter); if (rc) goto err_req_irq; @@ -2184,7 +2245,7 @@ void ena_down(struct ena_adapter *adapter) { int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues; - netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__); + netif_dbg(adapter, ifdown, adapter->netdev, "%s\n", __func__); clear_bit(ENA_FLAG_DEV_UP, &adapter->flags); @@ -2197,8 +2258,6 @@ void ena_down(struct ena_adapter *adapter) /* After this point the napi handler won't enable the tx queue */ ena_napi_disable_in_range(adapter, 0, io_queue_count); - /* After destroy the queue there won't be any new interrupts */ - if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) { int rc; @@ -2588,8 +2647,6 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) if (unlikely(rc)) goto error_drop_packet; - skb_tx_timestamp(skb); - next_to_use = tx_ring->next_to_use; req_id = tx_ring->free_ids[next_to_use]; tx_info = &tx_ring->tx_buffer_info[req_id]; @@ -2653,6 +2710,8 @@ static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev) } } + skb_tx_timestamp(skb); + if (netif_xmit_stopped(txq) || !netdev_xmit_more()) /* trigger the dma engine. ena_ring_tx_doorbell() * calls a memory barrier inside it. @@ -2670,26 +2729,11 @@ error_drop_packet: return NETDEV_TX_OK; } -static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) -{ - u16 qid; - /* we suspect that this is good for in--kernel network services that - * want to loop incoming skb rx to tx in normal user generated traffic, - * most probably we will not get to this - */ - if (skb_rx_queue_recorded(skb)) - qid = skb_get_rx_queue(skb); - else - qid = netdev_pick_tx(dev, skb, NULL); - - return qid; -} - static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev) { struct device *dev = &pdev->dev; struct ena_admin_host_info *host_info; + ssize_t ret; int rc; /* Allocate only the host info */ @@ -2704,11 +2748,19 @@ static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pd host_info->bdf = pci_dev_id(pdev); host_info->os_type = ENA_ADMIN_OS_LINUX; host_info->kernel_ver = LINUX_VERSION_CODE; - strscpy(host_info->kernel_ver_str, utsname()->version, - sizeof(host_info->kernel_ver_str) - 1); + ret = strscpy(host_info->kernel_ver_str, utsname()->version, + sizeof(host_info->kernel_ver_str)); + if (ret < 0) + dev_dbg(dev, + "kernel version string will be truncated, status = %zd\n", ret); + host_info->os_dist = 0; - strscpy(host_info->os_dist_str, utsname()->release, - sizeof(host_info->os_dist_str)); + ret = strscpy(host_info->os_dist_str, utsname()->release, + sizeof(host_info->os_dist_str)); + if (ret < 0) + dev_dbg(dev, + "OS distribution string will be truncated, status = %zd\n", ret); + host_info->driver_version = (DRV_MODULE_GEN_MAJOR) | (DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) | @@ -2764,8 +2816,7 @@ static void ena_config_debug_area(struct ena_adapter *adapter) rc = ena_com_set_host_attributes(adapter->ena_dev); if (rc) { if (rc == -EOPNOTSUPP) - netif_warn(adapter, drv, adapter->netdev, - "Cannot set host attributes\n"); + netif_warn(adapter, drv, adapter->netdev, "Cannot set host attributes\n"); else netif_err(adapter, drv, adapter->netdev, "Cannot set host attributes\n"); @@ -2777,19 +2828,6 @@ err: ena_com_delete_debug_area(adapter->ena_dev); } -int ena_update_hw_stats(struct ena_adapter *adapter) -{ - int rc; - - rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats); - if (rc) { - netdev_err(adapter->netdev, "Failed to get ENI stats\n"); - return rc; - } - - return 0; -} - static void ena_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { @@ -2863,18 +2901,16 @@ static const struct net_device_ops ena_netdev_ops = { .ndo_open = ena_open, .ndo_stop = ena_close, .ndo_start_xmit = ena_start_xmit, - .ndo_select_queue = ena_select_queue, .ndo_get_stats64 = ena_get_stats64, .ndo_tx_timeout = ena_tx_timeout, .ndo_change_mtu = ena_change_mtu, - .ndo_set_mac_address = NULL, .ndo_validate_addr = eth_validate_addr, .ndo_bpf = ena_xdp, .ndo_xdp_xmit = ena_xdp_xmit, }; -static void ena_calc_io_queue_size(struct ena_adapter *adapter, - struct ena_com_dev_get_features_ctx *get_feat_ctx) +static int ena_calc_io_queue_size(struct ena_adapter *adapter, + struct ena_com_dev_get_features_ctx *get_feat_ctx) { struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq; struct ena_com_dev *ena_dev = adapter->ena_dev; @@ -2933,6 +2969,18 @@ static void ena_calc_io_queue_size(struct ena_adapter *adapter, max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size); max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size); + if (max_tx_queue_size < ENA_MIN_RING_SIZE) { + netdev_err(adapter->netdev, "Device max TX queue size: %d < minimum: %d\n", + max_tx_queue_size, ENA_MIN_RING_SIZE); + return -EINVAL; + } + + if (max_rx_queue_size < ENA_MIN_RING_SIZE) { + netdev_err(adapter->netdev, "Device max RX queue size: %d < minimum: %d\n", + max_rx_queue_size, ENA_MIN_RING_SIZE); + return -EINVAL; + } + /* When forcing large headers, we multiply the entry size by 2, and therefore divide * the queue size by 2, leaving the amount of memory used by the queues unchanged. */ @@ -2963,6 +3011,8 @@ static void ena_calc_io_queue_size(struct ena_adapter *adapter, adapter->max_rx_ring_size = max_rx_queue_size; adapter->requested_tx_ring_size = tx_queue_size; adapter->requested_rx_ring_size = rx_queue_size; + + return 0; } static int ena_device_validate_params(struct ena_adapter *adapter, @@ -3070,6 +3120,7 @@ static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev, bool *wd_state) { struct ena_com_dev *ena_dev = adapter->ena_dev; + struct net_device *netdev = adapter->netdev; struct ena_llq_configurations llq_config; struct device *dev = &pdev->dev; bool readless_supported; @@ -3159,15 +3210,19 @@ static int ena_device_init(struct ena_adapter *adapter, struct pci_dev *pdev, rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq, &llq_config); if (rc) { - dev_err(dev, "ENA device init failed\n"); + netdev_err(netdev, "Cannot set queues placement policy rc= %d\n", rc); goto err_admin_init; } - ena_calc_io_queue_size(adapter, get_feat_ctx); + rc = ena_calc_io_queue_size(adapter, get_feat_ctx); + if (unlikely(rc)) + goto err_admin_init; return 0; err_admin_init: + ena_com_abort_admin_commands(ena_dev); + ena_com_wait_for_abort_completion(ena_dev); ena_com_delete_host_info(ena_dev); ena_com_admin_destroy(ena_dev); err_mmio_read_less: @@ -3208,14 +3263,15 @@ err_disable_msix: return rc; } -static void ena_destroy_device(struct ena_adapter *adapter, bool graceful) +static int ena_destroy_device(struct ena_adapter *adapter, bool graceful) { struct net_device *netdev = adapter->netdev; struct ena_com_dev *ena_dev = adapter->ena_dev; bool dev_up; + int rc = 0; if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)) - return; + return 0; netif_carrier_off(netdev); @@ -3226,14 +3282,14 @@ static void ena_destroy_device(struct ena_adapter *adapter, bool graceful) if (!graceful) ena_com_set_admin_running_state(ena_dev, false); - if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags)) + if (dev_up) ena_down(adapter); /* Stop the device from sending AENQ events (in case reset flag is set * and device is up, ena_down() already reset the device. */ if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up)) - ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); + rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason); ena_free_mgmnt_irq(adapter); @@ -3252,6 +3308,8 @@ static void ena_destroy_device(struct ena_adapter *adapter, bool graceful) clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags); clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags); + + return rc; } static int ena_restore_device(struct ena_adapter *adapter) @@ -3328,14 +3386,17 @@ err: static void ena_fw_reset_device(struct work_struct *work) { + int rc = 0; + struct ena_adapter *adapter = container_of(work, struct ena_adapter, reset_task); rtnl_lock(); if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) { - ena_destroy_device(adapter, false); - ena_restore_device(adapter); + rc |= ena_destroy_device(adapter, false); + rc |= ena_restore_device(adapter); + adapter->dev_stats.reset_fail += !!rc; dev_err(&adapter->pdev->dev, "Device reset completed successfully\n"); } @@ -3372,14 +3433,18 @@ static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, struct ena_ring *tx_ring) { struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi); + enum ena_regs_reset_reason_types reset_reason = ENA_REGS_RESET_MISS_TX_CMPL; unsigned int time_since_last_napi; unsigned int missing_tx_comp_to; bool is_tx_comp_time_expired; struct ena_tx_buffer *tx_buf; unsigned long last_jiffies; + int napi_scheduled; u32 missed_tx = 0; int i, rc = 0; + missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to); + for (i = 0; i < tx_ring->ring_size; i++) { tx_buf = &tx_ring->tx_buffer_info[i]; last_jiffies = tx_buf->last_jiffies; @@ -3406,25 +3471,45 @@ static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter, adapter->missing_tx_completion_to); if (unlikely(is_tx_comp_time_expired)) { - if (!tx_buf->print_once) { - time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies); - missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to); - netif_notice(adapter, tx_err, adapter->netdev, - "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n", - tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to); + time_since_last_napi = + jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies); + napi_scheduled = !!(ena_napi->napi.state & NAPIF_STATE_SCHED); + + if (missing_tx_comp_to < time_since_last_napi && napi_scheduled) { + /* We suspect napi isn't called because the + * bottom half is not run. Require a bigger + * timeout for these cases + */ + if (!time_is_before_jiffies(last_jiffies + + 2 * adapter->missing_tx_completion_to)) + continue; + + reset_reason = ENA_REGS_RESET_SUSPECTED_POLL_STARVATION; } - tx_buf->print_once = 1; missed_tx++; + + if (tx_buf->print_once) + continue; + + netif_notice(adapter, tx_err, adapter->netdev, + "TX hasn't completed, qid %d, index %d. %u usecs from last napi execution, napi scheduled: %d\n", + tx_ring->qid, i, time_since_last_napi, napi_scheduled); + + tx_buf->print_once = 1; } } if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) { netif_err(adapter, tx_err, adapter->netdev, - "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n", + "Lost TX completions are above the threshold (%d > %d). Completion transmission timeout: %u.\n", missed_tx, - adapter->missing_tx_completion_threshold); - ena_reset_device(adapter, ENA_REGS_RESET_MISS_TX_CMPL); + adapter->missing_tx_completion_threshold, + missing_tx_comp_to); + netif_err(adapter, tx_err, adapter->netdev, + "Resetting the device\n"); + + ena_reset_device(adapter, reset_reason); rc = -EIO; } @@ -3438,10 +3523,11 @@ static void check_for_missing_completions(struct ena_adapter *adapter) { struct ena_ring *tx_ring; struct ena_ring *rx_ring; - int i, budget, rc; + int qid, budget, rc; int io_queue_count; io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues; + /* Make sure the driver doesn't turn the device in other process */ smp_rmb(); @@ -3454,27 +3540,29 @@ static void check_for_missing_completions(struct ena_adapter *adapter) if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT) return; - budget = ENA_MONITORED_TX_QUEUES; + budget = min_t(u32, io_queue_count, ENA_MONITORED_TX_QUEUES); - for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) { - tx_ring = &adapter->tx_ring[i]; - rx_ring = &adapter->rx_ring[i]; + qid = adapter->last_monitored_tx_qid; + + while (budget) { + qid = (qid + 1) % io_queue_count; + + tx_ring = &adapter->tx_ring[qid]; + rx_ring = &adapter->rx_ring[qid]; rc = check_missing_comp_in_tx_queue(adapter, tx_ring); if (unlikely(rc)) return; - rc = !ENA_IS_XDP_INDEX(adapter, i) ? + rc = !ENA_IS_XDP_INDEX(adapter, qid) ? check_for_rx_interrupt_queue(adapter, rx_ring) : 0; if (unlikely(rc)) return; budget--; - if (!budget) - break; } - adapter->last_monitored_tx_qid = i % io_queue_count; + adapter->last_monitored_tx_qid = qid; } /* trigger napi schedule after 2 consecutive detections */ @@ -3762,8 +3850,8 @@ static int ena_rss_init_default(struct ena_adapter *adapter) } } - rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, - ENA_HASH_KEY_SIZE, 0xFFFFFFFF); + rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL, ENA_HASH_KEY_SIZE, + 0xFFFFFFFF); if (unlikely(rc && (rc != -EOPNOTSUPP))) { dev_err(dev, "Cannot fill hash function\n"); goto err_fill_indir; @@ -3873,10 +3961,16 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) pci_set_drvdata(pdev, adapter); + rc = ena_com_allocate_customer_metrics_buffer(ena_dev); + if (rc) { + netdev_err(netdev, "ena_com_allocate_customer_metrics_buffer failed\n"); + goto err_netdev_destroy; + } + rc = ena_map_llq_mem_bar(pdev, ena_dev, bars); if (rc) { dev_err(&pdev->dev, "ENA LLQ bar mapping failed\n"); - goto err_netdev_destroy; + goto err_metrics_destroy; } rc = ena_device_init(adapter, pdev, &get_feat_ctx, &wd_state); @@ -3884,7 +3978,7 @@ static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent) dev_err(&pdev->dev, "ENA device init failed\n"); if (rc == -ETIME) rc = -EPROBE_DEFER; - goto err_netdev_destroy; + goto err_metrics_destroy; } /* Initial TX and RX interrupt delay. Assumes 1 usec granularity. @@ -4005,6 +4099,8 @@ err_worker_destroy: err_device_destroy: ena_com_delete_host_info(ena_dev); ena_com_admin_destroy(ena_dev); +err_metrics_destroy: + ena_com_delete_customer_metrics_buffer(ena_dev); err_netdev_destroy: free_netdev(netdev); err_free_region: @@ -4040,8 +4136,8 @@ static void __ena_shutoff(struct pci_dev *pdev, bool shutdown) free_irq_cpu_rmap(netdev->rx_cpu_rmap); netdev->rx_cpu_rmap = NULL; } -#endif /* CONFIG_RFS_ACCEL */ +#endif /* CONFIG_RFS_ACCEL */ /* Make sure timer and reset routine won't be called after * freeing device resources. */ @@ -4068,6 +4164,8 @@ static void __ena_shutoff(struct pci_dev *pdev, bool shutdown) ena_com_delete_host_info(ena_dev); + ena_com_delete_customer_metrics_buffer(ena_dev); + ena_release_bars(ena_dev, pdev); pci_disable_device(pdev); diff --git a/drivers/net/ethernet/amazon/ena/ena_netdev.h b/drivers/net/ethernet/amazon/ena/ena_netdev.h index 6d2cc20210cc..6e12ae3b12e5 100644 --- a/drivers/net/ethernet/amazon/ena/ena_netdev.h +++ b/drivers/net/ethernet/amazon/ena/ena_netdev.h @@ -290,6 +290,7 @@ struct ena_stats_dev { u64 admin_q_pause; u64 rx_drops; u64 tx_drops; + u64 reset_fail; }; enum ena_flags_t { @@ -372,6 +373,7 @@ struct ena_adapter { struct u64_stats_sync syncp; struct ena_stats_dev dev_stats; struct ena_admin_eni_stats eni_stats; + struct ena_admin_ena_srd_info ena_srd_info; /* last queue index that was checked for uncompleted tx packets */ u32 last_monitored_tx_qid; @@ -389,7 +391,6 @@ void ena_dump_stats_to_dmesg(struct ena_adapter *adapter); void ena_dump_stats_to_buf(struct ena_adapter *adapter, u8 *buf); -int ena_update_hw_stats(struct ena_adapter *adapter); int ena_update_queue_params(struct ena_adapter *adapter, u32 new_tx_size, diff --git a/drivers/net/ethernet/amazon/ena/ena_regs_defs.h b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h index 1e007a41a525..a2efebafd686 100644 --- a/drivers/net/ethernet/amazon/ena/ena_regs_defs.h +++ b/drivers/net/ethernet/amazon/ena/ena_regs_defs.h @@ -21,6 +21,8 @@ enum ena_regs_reset_reason_types { ENA_REGS_RESET_USER_TRIGGER = 12, ENA_REGS_RESET_GENERIC = 13, ENA_REGS_RESET_MISS_INTERRUPT = 14, + ENA_REGS_RESET_SUSPECTED_POLL_STARVATION = 15, + ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED = 16, }; /* ena_registers offsets */ diff --git a/drivers/net/ethernet/amazon/ena/ena_xdp.c b/drivers/net/ethernet/amazon/ena/ena_xdp.c index fc1c4ef73ba3..5b175e7e92a1 100644 --- a/drivers/net/ethernet/amazon/ena/ena_xdp.c +++ b/drivers/net/ethernet/amazon/ena/ena_xdp.c @@ -89,7 +89,7 @@ int ena_xdp_xmit_frame(struct ena_ring *tx_ring, rc = ena_xdp_tx_map_frame(tx_ring, tx_info, xdpf, &ena_tx_ctx); if (unlikely(rc)) - return rc; + goto err; ena_tx_ctx.req_id = req_id; @@ -112,7 +112,9 @@ int ena_xdp_xmit_frame(struct ena_ring *tx_ring, error_unmap_dma: ena_unmap_tx_buff(tx_ring, tx_info); +err: tx_info->xdpf = NULL; + return rc; } @@ -412,7 +414,6 @@ static int ena_clean_xdp_irq(struct ena_ring *tx_ring, u32 budget) tx_ring->next_to_clean = next_to_clean; ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done); - ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq); netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev, "tx_poll: q %d done. total pkts: %d\n", |