diff options
Diffstat (limited to 'drivers/net/wireless/realtek/rtw89/pci.c')
| -rw-r--r-- | drivers/net/wireless/realtek/rtw89/pci.c | 1527 |
1 files changed, 1227 insertions, 300 deletions
diff --git a/drivers/net/wireless/realtek/rtw89/pci.c b/drivers/net/wireless/realtek/rtw89/pci.c index 1c4500ba777c..a66fcdb0293b 100644 --- a/drivers/net/wireless/realtek/rtw89/pci.c +++ b/drivers/net/wireless/realtek/rtw89/pci.c @@ -19,28 +19,50 @@ MODULE_PARM_DESC(disable_clkreq, "Set Y to disable PCI clkreq support"); MODULE_PARM_DESC(disable_aspm_l1, "Set Y to disable PCI ASPM L1 support"); MODULE_PARM_DESC(disable_aspm_l1ss, "Set Y to disable PCI L1SS support"); -static int rtw89_pci_rst_bdram_pcie(struct rtw89_dev *rtwdev) +static int rtw89_pci_get_phy_offset_by_link_speed(struct rtw89_dev *rtwdev, + u32 *phy_offset) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + u32 val; + int ret; + + ret = pci_read_config_dword(pdev, RTW89_PCIE_L1_STS_V1, &val); + if (ret) + return ret; + + val = u32_get_bits(val, RTW89_BCFG_LINK_SPEED_MASK); + if (val == RTW89_PCIE_GEN1_SPEED) { + *phy_offset = R_RAC_DIRECT_OFFSET_G1; + } else if (val == RTW89_PCIE_GEN2_SPEED) { + *phy_offset = R_RAC_DIRECT_OFFSET_G2; + } else { + rtw89_warn(rtwdev, "Unknown PCI link speed %d\n", val); + return -EFAULT; + } + + return 0; +} + +static int rtw89_pci_rst_bdram_ax(struct rtw89_dev *rtwdev) { u32 val; int ret; - rtw89_write32(rtwdev, R_AX_PCIE_INIT_CFG1, - rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) | B_AX_RST_BDRAM); + rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_RST_BDRAM); ret = read_poll_timeout_atomic(rtw89_read32, val, !(val & B_AX_RST_BDRAM), 1, RTW89_PCI_POLL_BDRAM_RST_CNT, false, rtwdev, R_AX_PCIE_INIT_CFG1); - if (ret) - return -EBUSY; - - return 0; + return ret; } static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev, struct rtw89_pci_dma_ring *bd_ring, u32 cur_idx, bool tx) { + const struct rtw89_pci_info *info = rtwdev->pci_info; u32 cnt, cur_rp, wp, rp, len; rp = bd_ring->rp; @@ -48,10 +70,14 @@ static u32 rtw89_pci_dma_recalc(struct rtw89_dev *rtwdev, len = bd_ring->len; cur_rp = FIELD_GET(TXBD_HW_IDX_MASK, cur_idx); - if (tx) + if (tx) { cnt = cur_rp >= rp ? cur_rp - rp : len - (rp - cur_rp); - else + } else { + if (info->rx_ring_eq_is_full) + wp += 1; + cnt = cur_rp >= wp ? cur_rp - wp : len - (wp - cur_rp); + } bd_ring->rp = cur_rp; @@ -108,7 +134,7 @@ static void rtw89_pci_release_fwcmd(struct rtw89_dev *rtwdev, static void rtw89_pci_reclaim_tx_fwcmd(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) { - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12]; u32 cnt; cnt = rtw89_pci_txbd_recalc(rtwdev, tx_ring); @@ -154,22 +180,73 @@ static void rtw89_pci_sync_skb_for_device(struct rtw89_dev *rtwdev, DMA_FROM_DEVICE); } -static int rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev, - struct sk_buff *skb) +static void rtw89_pci_rxbd_info_update(struct rtw89_dev *rtwdev, + struct sk_buff *skb) { - struct rtw89_pci_rxbd_info *rxbd_info; struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); + struct rtw89_pci_rxbd_info *rxbd_info; + __le32 info; rxbd_info = (struct rtw89_pci_rxbd_info *)skb->data; - rx_info->fs = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_FS); - rx_info->ls = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_LS); - rx_info->len = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_WRITE_SIZE); - rx_info->tag = le32_get_bits(rxbd_info->dword, RTW89_PCI_RXBD_TAG); + info = rxbd_info->dword; + + rx_info->fs = le32_get_bits(info, RTW89_PCI_RXBD_FS); + rx_info->ls = le32_get_bits(info, RTW89_PCI_RXBD_LS); + rx_info->len = le32_get_bits(info, RTW89_PCI_RXBD_WRITE_SIZE); + rx_info->tag = le32_get_bits(info, RTW89_PCI_RXBD_TAG); +} + +static int rtw89_pci_validate_rx_tag(struct rtw89_dev *rtwdev, + struct rtw89_pci_rx_ring *rx_ring, + struct sk_buff *skb) +{ + struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); + const struct rtw89_pci_info *info = rtwdev->pci_info; + u32 target_rx_tag; + + if (!info->check_rx_tag) + return 0; + + /* valid range is 1 ~ 0x1FFF */ + if (rx_ring->target_rx_tag == 0) + target_rx_tag = 1; + else + target_rx_tag = rx_ring->target_rx_tag; + + if (rx_info->tag != target_rx_tag) { + rtw89_debug(rtwdev, RTW89_DBG_UNEXP, "mismatch RX tag 0x%x 0x%x\n", + rx_info->tag, target_rx_tag); + return -EAGAIN; + } return 0; } -static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable) +static +int rtw89_pci_sync_skb_for_device_and_validate_rx_info(struct rtw89_dev *rtwdev, + struct rtw89_pci_rx_ring *rx_ring, + struct sk_buff *skb) +{ + struct rtw89_pci_rx_info *rx_info = RTW89_PCI_RX_SKB_CB(skb); + int rx_tag_retry = 1000; + int ret; + + do { + rtw89_pci_sync_skb_for_cpu(rtwdev, skb); + rtw89_pci_rxbd_info_update(rtwdev, skb); + + ret = rtw89_pci_validate_rx_tag(rtwdev, rx_ring, skb); + if (ret != -EAGAIN) + break; + } while (rx_tag_retry--); + + /* update target rx_tag for next RX */ + rx_ring->target_rx_tag = rx_info->tag + 1; + + return ret; +} + +static void rtw89_pci_ctrl_txdma_ch_ax(struct rtw89_dev *rtwdev, bool enable) { const struct rtw89_pci_info *info = rtwdev->pci_info; const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; @@ -186,7 +263,7 @@ static void rtw89_pci_ctrl_txdma_ch_pcie(struct rtw89_dev *rtwdev, bool enable) } } -static void rtw89_pci_ctrl_txdma_fw_ch_pcie(struct rtw89_dev *rtwdev, bool enable) +static void rtw89_pci_ctrl_txdma_fw_ch_ax(struct rtw89_dev *rtwdev, bool enable) { const struct rtw89_pci_info *info = rtwdev->pci_info; const struct rtw89_reg_def *dma_stop1 = &info->dma_stop1; @@ -226,24 +303,41 @@ rtw89_skb_put_rx_data(struct rtw89_dev *rtwdev, bool fs, bool ls, return true; } +static u32 rtw89_pci_get_rx_skb_idx(struct rtw89_dev *rtwdev, + struct rtw89_pci_dma_ring *bd_ring) +{ + const struct rtw89_pci_info *info = rtwdev->pci_info; + u32 wp = bd_ring->wp; + + if (!info->rx_ring_eq_is_full) + return wp; + + if (++wp >= bd_ring->len) + wp = 0; + + return wp; +} + static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, struct rtw89_pci_rx_ring *rx_ring) { - struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; - struct rtw89_pci_rx_info *rx_info; struct rtw89_rx_desc_info *desc_info = &rx_ring->diliver_desc; + struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct sk_buff *new = rx_ring->diliver_skb; + struct rtw89_pci_rx_info *rx_info; struct sk_buff *skb; u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info); + u32 skb_idx; u32 offset; u32 cnt = 1; bool fs, ls; int ret; - skb = rx_ring->buf[bd_ring->wp]; - rtw89_pci_sync_skb_for_cpu(rtwdev, skb); + skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring); + skb = rx_ring->buf[skb_idx]; - ret = rtw89_pci_rxbd_info_update(rtwdev, skb); + ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb); if (ret) { rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n", bd_ring->wp, ret); @@ -251,9 +345,14 @@ static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, } rx_info = RTW89_PCI_RX_SKB_CB(skb); - fs = rx_info->fs; + fs = info->no_rxbd_fs ? !new : rx_info->fs; ls = rx_info->ls; + if (unlikely(!fs || !ls)) + rtw89_debug(rtwdev, RTW89_DBG_UNEXP, + "unexpected fs/ls=%d/%d tag=%u len=%u new->len=%u\n", + fs, ls, rx_info->tag, rx_info->len, new ? new->len : 0); + if (fs) { if (new) { rtw89_debug(rtwdev, RTW89_DBG_UNEXP, @@ -265,7 +364,7 @@ static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, goto err_sync_device; } - rtw89_core_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size); + rtw89_chip_query_rxdesc(rtwdev, desc_info, skb->data, rxinfo_size); new = rtw89_alloc_skb_for_rx(rtwdev, desc_info->pkt_size); if (!new) @@ -274,9 +373,7 @@ static u32 rtw89_pci_rxbd_deliver_skbs(struct rtw89_dev *rtwdev, rx_ring->diliver_skb = new; /* first segment has RX desc */ - offset = desc_info->offset; - offset += desc_info->long_rxdesc ? sizeof(struct rtw89_rxdesc_long) : - sizeof(struct rtw89_rxdesc_short); + offset = desc_info->offset + desc_info->rxd_len; } else { offset = sizeof(struct rtw89_pci_rxbd_info); if (!new) { @@ -343,7 +440,7 @@ static int rtw89_pci_poll_rxq_dma(struct rtw89_dev *rtwdev, int countdown = rtwdev->napi_budget_countdown; u32 cnt; - rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RXQ]; + rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RXQ]; cnt = rtw89_pci_rxbd_recalc(rtwdev, rx_ring); if (!cnt) @@ -364,8 +461,12 @@ static void rtw89_pci_tx_status(struct rtw89_dev *rtwdev, struct rtw89_pci_tx_ring *tx_ring, struct sk_buff *skb, u8 tx_status) { + struct rtw89_tx_skb_data *skb_data = RTW89_TX_SKB_CB(skb); struct ieee80211_tx_info *info; + if (rtw89_core_tx_wait_complete(rtwdev, skb_data, tx_status)) + return; + info = IEEE80211_SKB_CB(skb); ieee80211_tx_info_clear_status(info); @@ -468,31 +569,52 @@ static void rtw89_pci_release_txwd_skb(struct rtw89_dev *rtwdev, rtw89_pci_enqueue_txwd(tx_ring, txwd); } -static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, - struct rtw89_pci_rpp_fmt *rpp) +void rtw89_pci_parse_rpp(struct rtw89_dev *rtwdev, void *_rpp, + struct rtw89_pci_rpp_info *rpp_info) +{ + const struct rtw89_pci_rpp_fmt *rpp = _rpp; + + rpp_info->seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ); + rpp_info->qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL); + rpp_info->tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS); + rpp_info->txch = rtw89_chip_get_ch_dma(rtwdev, rpp_info->qsel); +} +EXPORT_SYMBOL(rtw89_pci_parse_rpp); + +void rtw89_pci_parse_rpp_v1(struct rtw89_dev *rtwdev, void *_rpp, + struct rtw89_pci_rpp_info *rpp_info) +{ + const struct rtw89_pci_rpp_fmt_v1 *rpp = _rpp; + + rpp_info->seq = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_PCIE_SEQ_V1_MASK); + rpp_info->qsel = le32_get_bits(rpp->w1, RTW89_PCI_RPP_W1_QSEL_V1_MASK); + rpp_info->tx_status = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_TX_STATUS_V1_MASK); + rpp_info->txch = le32_get_bits(rpp->w0, RTW89_PCI_RPP_W0_DMA_CH_MASK); +} +EXPORT_SYMBOL(rtw89_pci_parse_rpp_v1); + +static void rtw89_pci_release_rpp(struct rtw89_dev *rtwdev, void *rpp) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring; + const struct rtw89_pci_info *info = rtwdev->pci_info; + struct rtw89_pci_rpp_info rpp_info = {}; struct rtw89_pci_tx_wd_ring *wd_ring; + struct rtw89_pci_tx_ring *tx_ring; struct rtw89_pci_tx_wd *txwd; - u16 seq; - u8 qsel, tx_status, txch; - seq = le32_get_bits(rpp->dword, RTW89_PCI_RPP_SEQ); - qsel = le32_get_bits(rpp->dword, RTW89_PCI_RPP_QSEL); - tx_status = le32_get_bits(rpp->dword, RTW89_PCI_RPP_TX_STATUS); - txch = rtw89_core_get_ch_dma(rtwdev, qsel); + info->parse_rpp(rtwdev, rpp, &rpp_info); - if (txch == RTW89_TXCH_CH12) { + if (rpp_info.txch == RTW89_TXCH_CH12) { rtw89_warn(rtwdev, "should no fwcmd release report\n"); return; } - tx_ring = &rtwpci->tx_rings[txch]; + tx_ring = &rtwpci->tx.rings[rpp_info.txch]; wd_ring = &tx_ring->wd_ring; - txwd = &wd_ring->pages[seq]; + txwd = &wd_ring->pages[rpp_info.seq]; - rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, seq, tx_status); + rtw89_pci_release_txwd_skb(rtwdev, tx_ring, txwd, rpp_info.seq, + rpp_info.tx_status); } static void rtw89_pci_release_pending_txwd_skb(struct rtw89_dev *rtwdev, @@ -517,20 +639,22 @@ static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev, u32 max_cnt) { struct rtw89_pci_dma_ring *bd_ring = &rx_ring->bd_ring; - struct rtw89_pci_rx_info *rx_info; - struct rtw89_pci_rpp_fmt *rpp; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_rx_desc_info desc_info = {}; + struct rtw89_pci_rx_info *rx_info; struct sk_buff *skb; - u32 cnt = 0; - u32 rpp_size = sizeof(struct rtw89_pci_rpp_fmt); + void *rpp; u32 rxinfo_size = sizeof(struct rtw89_pci_rxbd_info); + u32 rpp_size = info->rpp_fmt_size; + u32 cnt = 0; + u32 skb_idx; u32 offset; int ret; - skb = rx_ring->buf[bd_ring->wp]; - rtw89_pci_sync_skb_for_cpu(rtwdev, skb); + skb_idx = rtw89_pci_get_rx_skb_idx(rtwdev, bd_ring); + skb = rx_ring->buf[skb_idx]; - ret = rtw89_pci_rxbd_info_update(rtwdev, skb); + ret = rtw89_pci_sync_skb_for_device_and_validate_rx_info(rtwdev, rx_ring, skb); if (ret) { rtw89_err(rtwdev, "failed to update %d RXBD info: %d\n", bd_ring->wp, ret); @@ -543,14 +667,12 @@ static u32 rtw89_pci_release_tx_skbs(struct rtw89_dev *rtwdev, return cnt; } - rtw89_core_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size); + rtw89_chip_query_rxdesc(rtwdev, &desc_info, skb->data, rxinfo_size); /* first segment has RX desc */ - offset = desc_info.offset; - offset += desc_info.long_rxdesc ? sizeof(struct rtw89_rxdesc_long) : - sizeof(struct rtw89_rxdesc_short); + offset = desc_info.offset + desc_info.rxd_len; for (; offset + rpp_size <= rx_info->len; offset += rpp_size) { - rpp = (struct rtw89_pci_rpp_fmt *)(skb->data + offset); + rpp = skb->data + offset; rtw89_pci_release_rpp(rtwdev, rpp); } @@ -595,7 +717,7 @@ static int rtw89_pci_poll_rpq_dma(struct rtw89_dev *rtwdev, u32 cnt; int work_done; - rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ]; + rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ]; spin_lock_bh(&rtwpci->trx_lock); @@ -625,7 +747,7 @@ static void rtw89_pci_isr_rxd_unavail(struct rtw89_dev *rtwdev, int i; for (i = 0; i < RTW89_RXCH_NUM; i++) { - rx_ring = &rtwpci->rx_rings[i]; + rx_ring = &rtwpci->rx.rings[i]; bd_ring = &rx_ring->bd_ring; reg_idx = rtw89_read32(rtwdev, bd_ring->addr.idx); @@ -677,11 +799,49 @@ void rtw89_pci_recognize_intrs_v1(struct rtw89_dev *rtwdev, } EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v1); -static void rtw89_pci_clear_isr0(struct rtw89_dev *rtwdev, u32 isr00) +void rtw89_pci_recognize_intrs_v2(struct rtw89_dev *rtwdev, + struct rtw89_pci *rtwpci, + struct rtw89_pci_isrs *isrs) { - /* write 1 clear */ - rtw89_write32(rtwdev, R_AX_PCIE_HISR00, isr00); + isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs; + isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ? + rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0; + isrs->isrs[0] = isrs->ind_isrs & B_BE_HCI_AXIDMA_INT ? + rtw89_read32(rtwdev, R_BE_HAXI_HISR00) & rtwpci->intrs[0] : 0; + isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1]; + + if (isrs->halt_c2h_isrs) + rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs); + if (isrs->isrs[0]) + rtw89_write32(rtwdev, R_BE_HAXI_HISR00, isrs->isrs[0]); + if (isrs->isrs[1]) + rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]); + rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs); } +EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v2); + +void rtw89_pci_recognize_intrs_v3(struct rtw89_dev *rtwdev, + struct rtw89_pci *rtwpci, + struct rtw89_pci_isrs *isrs) +{ + isrs->ind_isrs = rtw89_read32(rtwdev, R_BE_PCIE_HISR) & rtwpci->ind_intrs; + isrs->halt_c2h_isrs = isrs->ind_isrs & B_BE_HS0ISR_IND_INT ? + rtw89_read32(rtwdev, R_BE_HISR0) & rtwpci->halt_c2h_intrs : 0; + isrs->isrs[1] = rtw89_read32(rtwdev, R_BE_PCIE_DMA_ISR) & rtwpci->intrs[1]; + + /* isrs[0] is not used, so borrow to store RDU status to share common + * flow in rtw89_pci_interrupt_threadfn(). + */ + isrs->isrs[0] = isrs->isrs[1] & (B_BE_PCIE_RDU_CH1_INT | + B_BE_PCIE_RDU_CH0_INT); + + if (isrs->halt_c2h_isrs) + rtw89_write32(rtwdev, R_BE_HISR0, isrs->halt_c2h_isrs); + if (isrs->isrs[1]) + rtw89_write32(rtwdev, R_BE_PCIE_DMA_ISR, isrs->isrs[1]); + rtw89_write32(rtwdev, R_BE_PCIE_HISR, isrs->ind_isrs); +} +EXPORT_SYMBOL(rtw89_pci_recognize_intrs_v3); void rtw89_pci_enable_intr(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) { @@ -714,6 +874,37 @@ void rtw89_pci_disable_intr_v1(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpc } EXPORT_SYMBOL(rtw89_pci_disable_intr_v1); +void rtw89_pci_enable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) +{ + rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs); + rtw89_write32(rtwdev, R_BE_HAXI_HIMR00, rtwpci->intrs[0]); + rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]); + rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs); +} +EXPORT_SYMBOL(rtw89_pci_enable_intr_v2); + +void rtw89_pci_disable_intr_v2(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) +{ + rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0); + rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0); +} +EXPORT_SYMBOL(rtw89_pci_disable_intr_v2); + +void rtw89_pci_enable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) +{ + rtw89_write32(rtwdev, R_BE_HIMR0, rtwpci->halt_c2h_intrs); + rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, rtwpci->intrs[1]); + rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, rtwpci->ind_intrs); +} +EXPORT_SYMBOL(rtw89_pci_enable_intr_v3); + +void rtw89_pci_disable_intr_v3(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci) +{ + rtw89_write32(rtwdev, R_BE_PCIE_HIMR0, 0); + rtw89_write32(rtwdev, R_BE_PCIE_DMA_IMR_0_V1, 0); +} +EXPORT_SYMBOL(rtw89_pci_disable_intr_v3); + static void rtw89_pci_ops_recovery_start(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; @@ -754,6 +945,8 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev) { struct rtw89_dev *rtwdev = dev; struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_isr_def *isr_def = info->isr_def; struct rtw89_pci_isrs isrs; unsigned long flags; @@ -761,13 +954,13 @@ static irqreturn_t rtw89_pci_interrupt_threadfn(int irq, void *dev) rtw89_chip_recognize_intrs(rtwdev, rtwpci, &isrs); spin_unlock_irqrestore(&rtwpci->irq_lock, flags); - if (unlikely(isrs.isrs[0] & B_AX_RDU_INT)) + if (unlikely(isrs.isrs[0] & isr_def->isr_rdu)) rtw89_pci_isr_rxd_unavail(rtwdev, rtwpci); - if (unlikely(isrs.halt_c2h_isrs & B_AX_HALT_C2H_INT_EN)) + if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_halt_c2h)) rtw89_ser_notify(rtwdev, rtw89_mac_get_err_status(rtwdev)); - if (unlikely(isrs.halt_c2h_isrs & B_AX_WDT_TIMEOUT_INT_EN)) + if (unlikely(isrs.halt_c2h_isrs & isr_def->isr_wdt_timeout)) rtw89_ser_notify(rtwdev, MAC_AX_ERR_L2_ERR_WDT_TIMEOUT_INT); if (unlikely(rtwpci->under_recovery)) @@ -818,6 +1011,33 @@ exit: return irqret; } +#define DEF_TXCHADDRS_TYPE3(gen, ch_idx, txch, v...) \ + [RTW89_TXCH_##ch_idx] = { \ + .num = R_##gen##_##txch##_TXBD_CFG, \ + .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ + .bdram = 0, \ + .desa_l = 0, \ + .desa_h = 0, \ + } + +#define DEF_TXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, txch, grp, v...) \ + [RTW89_TXCH_##ch_idx] = { \ + .num = R_##gen##_##txch##_TXBD_CFG, \ + .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ + .bdram = 0, \ + .desa_l = R_##gen##_##grp##_TXBD_DESA_L, \ + .desa_h = R_##gen##_##grp##_TXBD_DESA_H, \ + } + +#define DEF_TXCHADDRS_TYPE2(gen, ch_idx, txch, v...) \ + [RTW89_TXCH_##ch_idx] = { \ + .num = R_##gen##_##txch##_TXBD_NUM ##v, \ + .idx = R_##gen##_##txch##_TXBD_IDX ##v, \ + .bdram = 0, \ + .desa_l = R_##gen##_##txch##_TXBD_DESA_L ##v, \ + .desa_h = R_##gen##_##txch##_TXBD_DESA_H ##v, \ + } + #define DEF_TXCHADDRS_TYPE1(info, txch, v...) \ [RTW89_TXCH_##txch] = { \ .num = R_AX_##txch##_TXBD_NUM ##v, \ @@ -836,12 +1056,28 @@ exit: .desa_h = R_AX_##txch##_TXBD_DESA_H ##v, \ } -#define DEF_RXCHADDRS(info, rxch, v...) \ - [RTW89_RXCH_##rxch] = { \ - .num = R_AX_##rxch##_RXBD_NUM ##v, \ - .idx = R_AX_##rxch##_RXBD_IDX ##v, \ - .desa_l = R_AX_##rxch##_RXBD_DESA_L ##v, \ - .desa_h = R_AX_##rxch##_RXBD_DESA_H ##v, \ +#define DEF_RXCHADDRS_TYPE3(gen, ch_idx, rxch, v...) \ + [RTW89_RXCH_##ch_idx] = { \ + .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \ + .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \ + .desa_l = 0, \ + .desa_h = 0, \ + } + +#define DEF_RXCHADDRS_TYPE3_GRP_BASE(gen, ch_idx, rxch, grp, v...) \ + [RTW89_RXCH_##ch_idx] = { \ + .num = R_##gen##_RX_##rxch##_RXBD_CONFIG, \ + .idx = R_##gen##_##ch_idx##0_RXBD_IDX ##v, \ + .desa_l = R_##gen##_##grp##_RXBD_DESA_L, \ + .desa_h = R_##gen##_##grp##_RXBD_DESA_H, \ + } + +#define DEF_RXCHADDRS(gen, ch_idx, rxch, v...) \ + [RTW89_RXCH_##ch_idx] = { \ + .num = R_##gen##_##rxch##_RXBD_NUM ##v, \ + .idx = R_##gen##_##rxch##_RXBD_IDX ##v, \ + .desa_l = R_##gen##_##rxch##_RXBD_DESA_L ##v, \ + .desa_h = R_##gen##_##rxch##_RXBD_DESA_H ##v, \ } const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = { @@ -861,8 +1097,8 @@ const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set = { DEF_TXCHADDRS(info, CH12), }, .rx = { - DEF_RXCHADDRS(info, RXQ), - DEF_RXCHADDRS(info, RPQ), + DEF_RXCHADDRS(AX, RXQ, RXQ), + DEF_RXCHADDRS(AX, RPQ, RPQ), }, }; EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set); @@ -884,14 +1120,65 @@ const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_v1 = { DEF_TXCHADDRS(info, CH12, _V1), }, .rx = { - DEF_RXCHADDRS(info, RXQ, _V1), - DEF_RXCHADDRS(info, RPQ, _V1), + DEF_RXCHADDRS(AX, RXQ, RXQ, _V1), + DEF_RXCHADDRS(AX, RPQ, RPQ, _V1), }, }; EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_v1); +const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be = { + .tx = { + DEF_TXCHADDRS_TYPE2(BE, ACH0, CH0, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH1, CH1, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH2, CH2, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH3, CH3, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH4, CH4, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH5, CH5, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH6, CH6, _V1), + DEF_TXCHADDRS_TYPE2(BE, ACH7, CH7, _V1), + DEF_TXCHADDRS_TYPE2(BE, CH8, CH8, _V1), + DEF_TXCHADDRS_TYPE2(BE, CH9, CH9, _V1), + DEF_TXCHADDRS_TYPE2(BE, CH10, CH10, _V1), + DEF_TXCHADDRS_TYPE2(BE, CH11, CH11, _V1), + DEF_TXCHADDRS_TYPE2(BE, CH12, CH12, _V1), + }, + .rx = { + DEF_RXCHADDRS(BE, RXQ, RXQ0, _V1), + DEF_RXCHADDRS(BE, RPQ, RPQ0, _V1), + }, +}; +EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be); + +const struct rtw89_pci_ch_dma_addr_set rtw89_pci_ch_dma_addr_set_be_v1 = { + .tx = { + DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, ACH0, CH0, ACQ, _V1), + /* no CH1 */ + DEF_TXCHADDRS_TYPE3(BE, ACH2, CH2, _V1), + /* no CH3 */ + DEF_TXCHADDRS_TYPE3(BE, ACH4, CH4, _V1), + /* no CH5 */ + DEF_TXCHADDRS_TYPE3(BE, ACH6, CH6, _V1), + /* no CH7 */ + DEF_TXCHADDRS_TYPE3_GRP_BASE(BE, CH8, CH8, NACQ, _V1), + /* no CH9 */ + DEF_TXCHADDRS_TYPE3(BE, CH10, CH10, _V1), + /* no CH11 */ + DEF_TXCHADDRS_TYPE3(BE, CH12, CH12, _V1), + }, + .rx = { + DEF_RXCHADDRS_TYPE3_GRP_BASE(BE, RXQ, CH0, HOST0, _V1), + DEF_RXCHADDRS_TYPE3(BE, RPQ, CH1, _V1), + }, +}; +EXPORT_SYMBOL(rtw89_pci_ch_dma_addr_set_be_v1); + +#undef DEF_TXCHADDRS_TYPE3 +#undef DEF_TXCHADDRS_TYPE3_GRP_BASE +#undef DEF_TXCHADDRS_TYPE2 #undef DEF_TXCHADDRS_TYPE1 #undef DEF_TXCHADDRS +#undef DEF_RXCHADDRS_TYPE3 +#undef DEF_RXCHADDRS_TYPE3_GRP_BASE #undef DEF_RXCHADDRS static int rtw89_pci_get_txch_addrs(struct rtw89_dev *rtwdev, @@ -937,7 +1224,7 @@ static u32 __rtw89_pci_check_and_reclaim_tx_fwcmd_resource(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[RTW89_TXCH_CH12]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[RTW89_TXCH_CH12]; u32 cnt; spin_lock_bh(&rtwpci->trx_lock); @@ -953,13 +1240,14 @@ u32 __rtw89_pci_check_and_reclaim_tx_resource_noio(struct rtw89_dev *rtwdev, u8 txch) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; u32 cnt; spin_lock_bh(&rtwpci->trx_lock); cnt = rtw89_pci_get_avail_txbd_num(tx_ring); - cnt = min(cnt, wd_ring->curr_num); + if (txch != RTW89_TXCH_CH12) + cnt = min(cnt, wd_ring->curr_num); spin_unlock_bh(&rtwpci->trx_lock); return cnt; @@ -969,7 +1257,7 @@ static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, u8 txch) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; struct rtw89_pci_tx_wd_ring *wd_ring = &tx_ring->wd_ring; const struct rtw89_chip_info *chip = rtwdev->chip; u32 bd_cnt, wd_cnt, min_cnt = 0; @@ -977,7 +1265,7 @@ static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, enum rtw89_debug_mask debug_mask; u32 cnt; - rx_ring = &rtwpci->rx_rings[RTW89_RXCH_RPQ]; + rx_ring = &rtwpci->rx.rings[RTW89_RXCH_RPQ]; spin_lock_bh(&rtwpci->trx_lock); bd_cnt = rtw89_pci_get_avail_txbd_num(tx_ring); @@ -1000,10 +1288,10 @@ static u32 __rtw89_pci_check_and_reclaim_tx_resource(struct rtw89_dev *rtwdev, min_cnt = min(bd_cnt, wd_cnt); if (min_cnt == 0) { /* This message can be frequently shown in low power mode or - * high traffic with 8852B, and we have recognized it as normal + * high traffic with small FIFO chips, and we have recognized it as normal * behavior, so print with mask RTW89_DBG_TXRX in these situations. */ - if (rtwpci->low_power || chip->chip_id == RTL8852B) + if (rtwpci->low_power || chip->small_fifo_size) debug_mask = RTW89_DBG_TXRX; else debug_mask = RTW89_DBG_UNEXP; @@ -1062,7 +1350,7 @@ static void rtw89_pci_tx_bd_ring_update(struct rtw89_dev *rtwdev, struct rtw89_p static void rtw89_pci_ops_tx_kick_off(struct rtw89_dev *rtwdev, u8 txch) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; if (rtwdev->hci.paused) { set_bit(txch, rtwpci->kick_map); @@ -1082,7 +1370,7 @@ static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev) if (!test_and_clear_bit(txch, rtwpci->kick_map)) continue; - tx_ring = &rtwpci->tx_rings[txch]; + tx_ring = &rtwpci->tx.rings[txch]; __rtw89_pci_tx_kick_off(rtwdev, tx_ring); } } @@ -1090,7 +1378,7 @@ static void rtw89_pci_tx_kick_off_pending(struct rtw89_dev *rtwdev) static void __pci_flush_txch(struct rtw89_dev *rtwdev, u8 txch, bool drop) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; - struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx_rings[txch]; + struct rtw89_pci_tx_ring *tx_ring = &rtwpci->tx.rings[txch]; struct rtw89_pci_dma_ring *bd_ring = &tx_ring->bd_ring; u32 cur_idx, cur_rp; u8 i; @@ -1142,10 +1430,12 @@ u32 rtw89_pci_fill_txaddr_info(struct rtw89_dev *rtwdev, dma_addr_t dma, u8 *add_info_nr) { struct rtw89_pci_tx_addr_info_32 *txaddr_info = txaddr_info_addr; + __le16 option; txaddr_info->length = cpu_to_le16(total_len); - txaddr_info->option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | - RTW89_PCI_ADDR_NUM(1)); + option = cpu_to_le16(RTW89_PCI_ADDR_MSDU_LS | RTW89_PCI_ADDR_NUM(1)); + option |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_ADDR_HIGH_MASK); + txaddr_info->option = option; txaddr_info->dma = cpu_to_le32(dma); *add_info_nr = 1; @@ -1172,6 +1462,8 @@ u32 rtw89_pci_fill_txaddr_info_v1(struct rtw89_dev *rtwdev, length_option = FIELD_PREP(B_PCIADDR_LEN_V1_MASK, len) | FIELD_PREP(B_PCIADDR_HIGH_SEL_V1_MASK, 0) | FIELD_PREP(B_PCIADDR_LS_V1_MASK, remain == 0); + length_option |= u16_encode_bits(upper_32_bits(dma), + B_PCIADDR_HIGH_SEL_V1_MASK); txaddr_info->length_opt = cpu_to_le16(length_option); txaddr_info->dma_low_lsb = cpu_to_le16(FIELD_GET(GENMASK(15, 0), dma)); txaddr_info->dma_low_msb = cpu_to_le16(FIELD_GET(GENMASK(31, 16), dma)); @@ -1197,7 +1489,6 @@ static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev, struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; const struct rtw89_chip_info *chip = rtwdev->chip; struct rtw89_tx_desc_info *desc_info = &tx_req->desc_info; - struct rtw89_txwd_info *txwd_info; struct rtw89_pci_tx_wp_info *txwp_info; void *txaddr_info_addr; struct pci_dev *pdev = rtwpci->pdev; @@ -1221,7 +1512,7 @@ static int rtw89_pci_txwd_submit(struct rtw89_dev *rtwdev, txwp_len = sizeof(*txwp_info); txwd_len = chip->txwd_body_size; - txwd_len += en_wd_info ? sizeof(*txwd_info) : 0; + txwd_len += en_wd_info ? chip->txwd_info_size : 0; txwp_info = txwd->vaddr + txwd_len; txwp_info->seq0 = cpu_to_le16(txwd->seq | RTW89_PCI_TXWP_VALID); @@ -1261,6 +1552,7 @@ static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev, struct sk_buff *skb = tx_req->skb; struct rtw89_pci_tx_data *tx_data = RTW89_PCI_TX_SKB_CB(skb); dma_addr_t dma; + __le16 opt; txdesc = skb_push(skb, txdesc_size); memset(txdesc, 0, txdesc_size); @@ -1273,7 +1565,9 @@ static int rtw89_pci_fwcmd_submit(struct rtw89_dev *rtwdev, } tx_data->dma = dma; - txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS); + opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS); + opt |= le16_encode_bits(upper_32_bits(dma), RTW89_PCI_TXBD_OPT_DMA_HI); + txbd->opt = opt; txbd->length = cpu_to_le16(skb->len); txbd->dma = cpu_to_le32(tx_data->dma); skb_queue_tail(&rtwpci->h2c_queue, skb); @@ -1289,6 +1583,7 @@ static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev, struct rtw89_core_tx_request *tx_req) { struct rtw89_pci_tx_wd *txwd; + __le16 opt; int ret; /* FWCMD queue doesn't have wd pages. Instead, it submits the CMD @@ -1313,7 +1608,9 @@ static int rtw89_pci_txbd_submit(struct rtw89_dev *rtwdev, list_add_tail(&txwd->list, &tx_ring->busy_pages); - txbd->option = cpu_to_le16(RTW89_PCI_TXBD_OPTION_LS); + opt = cpu_to_le16(RTW89_PCI_TXBD_OPT_LS); + opt |= le16_encode_bits(upper_32_bits(txwd->paddr), RTW89_PCI_TXBD_OPT_DMA_HI); + txbd->opt = opt; txbd->length = cpu_to_le16(txwd->len); txbd->dma = cpu_to_le32(txwd->paddr); @@ -1345,7 +1642,7 @@ static int rtw89_pci_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx_req return -EINVAL; } - tx_ring = &rtwpci->tx_rings[txch]; + tx_ring = &rtwpci->tx.rings[txch]; spin_lock_bh(&rtwpci->trx_lock); n_avail_txbd = rtw89_pci_get_avail_txbd_num(tx_ring); @@ -1384,7 +1681,7 @@ static int rtw89_pci_ops_tx_write(struct rtw89_dev *rtwdev, struct rtw89_core_tx return 0; } -static const struct rtw89_pci_bd_ram bd_ram_table[RTW89_TXCH_NUM] = { +const struct rtw89_pci_bd_ram rtw89_bd_ram_table_dual[RTW89_TXCH_NUM] = { [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2}, [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2}, [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2}, @@ -1399,18 +1696,88 @@ static const struct rtw89_pci_bd_ram bd_ram_table[RTW89_TXCH_NUM] = { [RTW89_TXCH_CH11] = {.start_idx = 55, .max_num = 5, .min_num = 1}, [RTW89_TXCH_CH12] = {.start_idx = 60, .max_num = 4, .min_num = 1}, }; +EXPORT_SYMBOL(rtw89_bd_ram_table_dual); + +const struct rtw89_pci_bd_ram rtw89_bd_ram_table_single[RTW89_TXCH_NUM] = { + [RTW89_TXCH_ACH0] = {.start_idx = 0, .max_num = 5, .min_num = 2}, + [RTW89_TXCH_ACH1] = {.start_idx = 5, .max_num = 5, .min_num = 2}, + [RTW89_TXCH_ACH2] = {.start_idx = 10, .max_num = 5, .min_num = 2}, + [RTW89_TXCH_ACH3] = {.start_idx = 15, .max_num = 5, .min_num = 2}, + [RTW89_TXCH_CH8] = {.start_idx = 20, .max_num = 4, .min_num = 1}, + [RTW89_TXCH_CH9] = {.start_idx = 24, .max_num = 4, .min_num = 1}, + [RTW89_TXCH_CH12] = {.start_idx = 28, .max_num = 4, .min_num = 1}, +}; +EXPORT_SYMBOL(rtw89_bd_ram_table_single); + +static void rtw89_pci_init_wp_16sel(struct rtw89_dev *rtwdev) +{ + const struct rtw89_pci_info *info = rtwdev->pci_info; + u32 addr = info->wp_sel_addr; + u32 val; + int i; + + if (!info->wp_sel_addr) + return; + + for (i = 0; i < 16; i += 4) { + val = u32_encode_bits(i + 0, MASKBYTE0) | + u32_encode_bits(i + 1, MASKBYTE1) | + u32_encode_bits(i + 2, MASKBYTE2) | + u32_encode_bits(i + 3, MASKBYTE3); + rtw89_write32(rtwdev, addr + i, val); + } +} + +static u16 rtw89_pci_enc_bd_cfg(struct rtw89_dev *rtwdev, u16 bd_num, + u32 dma_offset) +{ + u16 dma_offset_sel; + u16 num_sel; + + /* B_BE_TX_NUM_SEL_MASK, B_BE_RX_NUM_SEL_MASK: + * 0 -> 0 + * 1 -> 64 = 2^6 + * 2 -> 128 = 2^7 + * ... + * 7 -> 4096 = 2^12 + */ + num_sel = ilog2(bd_num) - 5; + + if (hweight16(bd_num) != 1) + rtw89_warn(rtwdev, "bd_num %u is not power of 2\n", bd_num); + + /* B_BE_TX_START_OFFSET_MASK, B_BE_RX_START_OFFSET_MASK: + * 0 -> 0 = 0 * 2^9 + * 1 -> 512 = 1 * 2^9 + * 2 -> 1024 = 2 * 2^9 + * 3 -> 1536 = 3 * 2^9 + * ... + * 255 -> 130560 = 255 * 2^9 + */ + dma_offset_sel = dma_offset >> 9; + + if (dma_offset % 512) + rtw89_warn(rtwdev, "offset %u is not multiple of 512\n", dma_offset); + + return u16_encode_bits(num_sel, B_BE_TX_NUM_SEL_MASK) | + u16_encode_bits(dma_offset_sel, B_BE_TX_START_OFFSET_MASK); +} static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_bd_ram *bd_ram_table = *info->bd_ram_table; struct rtw89_pci_tx_ring *tx_ring; struct rtw89_pci_rx_ring *rx_ring; struct rtw89_pci_dma_ring *bd_ring; const struct rtw89_pci_bd_ram *bd_ram; - u32 addr_num; - u32 addr_bdram; + dma_addr_t group_dma_base = 0; + u16 num_or_offset; u32 addr_desa_l; + u32 addr_bdram; + u32 addr_num; + u32 addr_idx; u32 val32; int i; @@ -1418,37 +1785,77 @@ static void rtw89_pci_reset_trx_rings(struct rtw89_dev *rtwdev) if (info->tx_dma_ch_mask & BIT(i)) continue; - tx_ring = &rtwpci->tx_rings[i]; + tx_ring = &rtwpci->tx.rings[i]; bd_ring = &tx_ring->bd_ring; - bd_ram = &bd_ram_table[i]; + bd_ram = bd_ram_table ? &bd_ram_table[i] : NULL; addr_num = bd_ring->addr.num; addr_bdram = bd_ring->addr.bdram; addr_desa_l = bd_ring->addr.desa_l; bd_ring->wp = 0; bd_ring->rp = 0; - val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) | - FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) | - FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num); + if (info->group_bd_addr) { + if (addr_desa_l) + group_dma_base = bd_ring->dma; + + num_or_offset = + rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len, + bd_ring->dma - group_dma_base); + } else { + num_or_offset = bd_ring->len; + } + rtw89_write16(rtwdev, addr_num, num_or_offset); + + if (addr_bdram && bd_ram) { + val32 = FIELD_PREP(BDRAM_SIDX_MASK, bd_ram->start_idx) | + FIELD_PREP(BDRAM_MAX_MASK, bd_ram->max_num) | + FIELD_PREP(BDRAM_MIN_MASK, bd_ram->min_num); - rtw89_write16(rtwdev, addr_num, bd_ring->len); - rtw89_write32(rtwdev, addr_bdram, val32); - rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); + rtw89_write32(rtwdev, addr_bdram, val32); + } + if (addr_desa_l) { + rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); + rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); + } } for (i = 0; i < RTW89_RXCH_NUM; i++) { - rx_ring = &rtwpci->rx_rings[i]; + rx_ring = &rtwpci->rx.rings[i]; bd_ring = &rx_ring->bd_ring; addr_num = bd_ring->addr.num; + addr_idx = bd_ring->addr.idx; addr_desa_l = bd_ring->addr.desa_l; - bd_ring->wp = 0; + if (info->rx_ring_eq_is_full) + bd_ring->wp = bd_ring->len - 1; + else + bd_ring->wp = 0; bd_ring->rp = 0; rx_ring->diliver_skb = NULL; rx_ring->diliver_desc.ready = false; + rx_ring->target_rx_tag = 0; + + if (info->group_bd_addr) { + if (addr_desa_l) + group_dma_base = bd_ring->dma; + + num_or_offset = + rtw89_pci_enc_bd_cfg(rtwdev, bd_ring->len, + bd_ring->dma - group_dma_base); + } else { + num_or_offset = bd_ring->len; + } + rtw89_write16(rtwdev, addr_num, num_or_offset); - rtw89_write16(rtwdev, addr_num, bd_ring->len); - rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); + if (addr_desa_l) { + rtw89_write32(rtwdev, addr_desa_l, bd_ring->dma); + rtw89_write32(rtwdev, addr_desa_l + 4, upper_32_bits(bd_ring->dma)); + } + + if (info->rx_ring_eq_is_full) + rtw89_write16(rtwdev, addr_idx, bd_ring->wp); } + + rtw89_pci_init_wp_16sel(rtwdev); } static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev, @@ -1458,7 +1865,7 @@ static void rtw89_pci_release_tx_ring(struct rtw89_dev *rtwdev, rtw89_pci_release_pending_txwd_skb(rtwdev, tx_ring); } -static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev) +void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; const struct rtw89_pci_info *info = rtwdev->pci_info; @@ -1475,7 +1882,7 @@ static void rtw89_pci_ops_reset(struct rtw89_dev *rtwdev) skb_queue_len(&rtwpci->h2c_queue), true); continue; } - rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx_rings[txch]); + rtw89_pci_release_tx_ring(rtwdev, &rtwpci->tx.rings[txch]); } spin_unlock_bh(&rtwpci->trx_lock); } @@ -1551,14 +1958,14 @@ void rtw89_pci_switch_bd_idx_addr(struct rtw89_dev *rtwdev, bool low_power) return; for (i = 0; i < RTW89_TXCH_NUM; i++) { - tx_ring = &rtwpci->tx_rings[i]; + tx_ring = &rtwpci->tx.rings[i]; tx_ring->bd_ring.addr.idx = low_power ? bd_idx_addr->tx_bd_addrs[i] : dma_addr_set->tx[i].idx; } for (i = 0; i < RTW89_RXCH_NUM; i++) { - rx_ring = &rtwpci->rx_rings[i]; + rx_ring = &rtwpci->rx.rings[i]; rx_ring->bd_ring.addr.idx = low_power ? bd_idx_addr->rx_bd_addrs[i] : dma_addr_set->rx[i].idx; @@ -1657,6 +2064,20 @@ static void rtw89_pci_ops_write32(struct rtw89_dev *rtwdev, u32 addr, u32 data) writel(data, rtwpci->mmap + addr); } +static u32 rtw89_pci_ops_read32_pci_cfg(struct rtw89_dev *rtwdev, u32 addr) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + u32 value; + int ret; + + ret = pci_read_config_dword(pdev, addr, &value); + if (ret) + return RTW89_R32_EA; + + return value; +} + static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable) { const struct rtw89_pci_info *info = rtwdev->pci_info; @@ -1671,24 +2092,16 @@ static void rtw89_pci_ctrl_dma_trx(struct rtw89_dev *rtwdev, bool enable) static void rtw89_pci_ctrl_dma_io(struct rtw89_dev *rtwdev, bool enable) { - enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - u32 reg, mask; - - if (chip_id == RTL8852C) { - reg = R_AX_HAXI_INIT_CFG1; - mask = B_AX_STOP_AXI_MST; - } else { - reg = R_AX_PCIE_DMA_STOP1; - mask = B_AX_STOP_PCIEIO; - } + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_reg_def *reg = &info->dma_io_stop; if (enable) - rtw89_write32_clr(rtwdev, reg, mask); + rtw89_write32_clr(rtwdev, reg->addr, reg->mask); else - rtw89_write32_set(rtwdev, reg, mask); + rtw89_write32_set(rtwdev, reg->addr, reg->mask); } -static void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable) +void rtw89_pci_ctrl_dma_all(struct rtw89_dev *rtwdev, bool enable) { rtw89_pci_ctrl_dma_io(rtwdev, enable); rtw89_pci_ctrl_dma_trx(rtwdev, enable); @@ -1807,22 +2220,87 @@ static int rtw89_write16_mdio_clr(struct rtw89_dev *rtwdev, u8 addr, u16 mask, u return 0; } +static int rtw89_dbi_write8(struct rtw89_dev *rtwdev, u16 addr, u8 data) +{ + u16 addr_2lsb = addr & B_AX_DBI_2LSB; + u16 write_addr; + u8 flag; + int ret; + + write_addr = addr & B_AX_DBI_ADDR_MSK; + write_addr |= u16_encode_bits(BIT(addr_2lsb), B_AX_DBI_WREN_MSK); + rtw89_write8(rtwdev, R_AX_DBI_WDATA + addr_2lsb, data); + rtw89_write16(rtwdev, R_AX_DBI_FLAG, write_addr); + rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_WFLAG >> 16); + + ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, + 10 * RTW89_PCI_WR_RETRY_CNT, false, + rtwdev, R_AX_DBI_FLAG + 2); + if (ret) + rtw89_err(rtwdev, "failed to write DBI register, addr=0x%X\n", + addr); + + return ret; +} + +static int rtw89_dbi_read8(struct rtw89_dev *rtwdev, u16 addr, u8 *value) +{ + u16 read_addr = addr & B_AX_DBI_ADDR_MSK; + u8 flag; + int ret; + + rtw89_write16(rtwdev, R_AX_DBI_FLAG, read_addr); + rtw89_write8(rtwdev, R_AX_DBI_FLAG + 2, B_AX_DBI_RFLAG >> 16); + + ret = read_poll_timeout_atomic(rtw89_read8, flag, !flag, 10, + 10 * RTW89_PCI_WR_RETRY_CNT, false, + rtwdev, R_AX_DBI_FLAG + 2); + if (ret) { + rtw89_err(rtwdev, "failed to read DBI register, addr=0x%X\n", + addr); + return ret; + } + + read_addr = R_AX_DBI_RDATA + (addr & 3); + *value = rtw89_read8(rtwdev, read_addr); + + return 0; +} + static int rtw89_pci_write_config_byte(struct rtw89_dev *rtwdev, u16 addr, u8 data) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; struct pci_dev *pdev = rtwpci->pdev; + int ret; + + ret = pci_write_config_byte(pdev, addr, data); + if (!ret) + return 0; - return pci_write_config_byte(pdev, addr, data); + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) + ret = rtw89_dbi_write8(rtwdev, addr, data); + + return ret; } static int rtw89_pci_read_config_byte(struct rtw89_dev *rtwdev, u16 addr, u8 *value) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; struct pci_dev *pdev = rtwpci->pdev; + int ret; - return pci_read_config_byte(pdev, addr, value); + ret = pci_read_config_byte(pdev, addr, value); + if (!ret) + return 0; + + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) + ret = rtw89_dbi_read8(rtwdev, addr, value); + + return ret; } static int rtw89_pci_config_byte_set(struct rtw89_dev *rtwdev, u16 addr, @@ -1901,7 +2379,7 @@ static int rtw89_pci_autok_x(struct rtw89_dev *rtwdev) { int ret; - if (rtwdev->chip->chip_id != RTL8852B) + if (!rtw89_is_rtl885xb(rtwdev)) return 0; ret = rtw89_write16_mdio_mask(rtwdev, RAC_REG_FLD_0, BAC_AUTOK_N_MASK, @@ -1917,7 +2395,7 @@ static int rtw89_pci_auto_refclk_cal(struct rtw89_dev *rtwdev, bool autook_en) bool l1_flag = false; int ret = 0; - if (rtwdev->chip->chip_id != RTL8852B) + if (!rtw89_is_rtl885xb(rtwdev)) return 0; ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_PHY_RATE, &val8); @@ -2084,6 +2562,92 @@ static int rtw89_pci_deglitch_setting(struct rtw89_dev *rtwdev) return 0; } +static void rtw89_pci_disable_eq_ax(struct rtw89_dev *rtwdev) +{ + u16 g1_oobs, g2_oobs; + u32 backup_aspm; + u32 phy_offset; + u16 offset_cal; + u16 oobs_val; + int ret; + u8 gen; + + if (rtwdev->chip->chip_id != RTL8852C) + return; + + g1_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + + RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL); + g2_oobs = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + + RAC_ANA09 * RAC_MULT, BAC_OOBS_SEL); + if (g1_oobs && g2_oobs) + return; + + backup_aspm = rtw89_read32(rtwdev, R_AX_PCIE_MIX_CFG_V1); + rtw89_write32_clr(rtwdev, R_AX_PCIE_MIX_CFG_V1, B_AX_ASPM_CTRL_MASK); + + ret = rtw89_pci_get_phy_offset_by_link_speed(rtwdev, &phy_offset); + if (ret) + goto out; + + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, BAC_RX_TEST_EN); + rtw89_write16(rtwdev, phy_offset + RAC_ANA10 * RAC_MULT, ADDR_SEL_PINOUT_DIS_VAL); + rtw89_write16_set(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, B_PCIE_BIT_RD_SEL); + + oobs_val = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT, + OOBS_LEVEL_MASK); + + rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA03 * RAC_MULT, + OOBS_SEN_MASK, oobs_val); + rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G1 + RAC_ANA09 * RAC_MULT, + BAC_OOBS_SEL); + + rtw89_write16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA03 * RAC_MULT, + OOBS_SEN_MASK, oobs_val); + rtw89_write16_set(rtwdev, R_RAC_DIRECT_OFFSET_G2 + RAC_ANA09 * RAC_MULT, + BAC_OOBS_SEL); + + /* offset K */ + for (gen = 1; gen <= 2; gen++) { + phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 : + R_RAC_DIRECT_OFFSET_G2; + + rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA19 * RAC_MULT, + B_PCIE_BIT_RD_SEL); + } + + offset_cal = rtw89_read16_mask(rtwdev, R_RAC_DIRECT_OFFSET_G1 + + RAC_ANA1F * RAC_MULT, OFFSET_CAL_MASK); + + for (gen = 1; gen <= 2; gen++) { + phy_offset = gen == 1 ? R_RAC_DIRECT_OFFSET_G1 : + R_RAC_DIRECT_OFFSET_G2; + + rtw89_write16_mask(rtwdev, phy_offset + RAC_ANA0B * RAC_MULT, + MANUAL_LVL_MASK, offset_cal); + rtw89_write16_clr(rtwdev, phy_offset + RAC_ANA0D * RAC_MULT, + OFFSET_CAL_MODE); + } + +out: + rtw89_write32(rtwdev, R_AX_PCIE_MIX_CFG_V1, backup_aspm); +} + +static void rtw89_pci_ber(struct rtw89_dev *rtwdev) +{ + u32 phy_offset; + + if (!test_bit(RTW89_QUIRK_PCI_BER, rtwdev->quirks)) + return; + + phy_offset = R_RAC_DIRECT_OFFSET_G1; + rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G1_VAL); + rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL); + + phy_offset = R_RAC_DIRECT_OFFSET_G2; + rtw89_write16(rtwdev, phy_offset + RAC_ANA1E * RAC_MULT, RAC_ANA1E_G2_VAL); + rtw89_write16(rtwdev, phy_offset + RAC_ANA2E * RAC_MULT, RAC_ANA2E_VAL); +} + static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev) { if (rtwdev->chip->chip_id != RTL8852A) @@ -2094,7 +2658,9 @@ static void rtw89_pci_rxdma_prefth(struct rtw89_dev *rtwdev) static void rtw89_pci_l1off_pwroff(struct rtw89_dev *rtwdev) { - if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B) + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + + if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev)) return; rtw89_write32_clr(rtwdev, R_AX_PCIE_PS_CTRL, B_AX_L1OFF_PWR_OFF_EN); @@ -2122,7 +2688,9 @@ static u32 rtw89_pci_l2_rxen_lat(struct rtw89_dev *rtwdev) static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev) { - if (rtwdev->chip->chip_id != RTL8852A && rtwdev->chip->chip_id != RTL8852B) + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + + if (chip_id != RTL8852A && !rtw89_is_rtl885xb(rtwdev)) return; rtw89_write32_clr(rtwdev, R_AX_SYS_PW_CTRL, B_AX_PSUS_OFF_CAPC_EN); @@ -2130,8 +2698,9 @@ static void rtw89_pci_aphy_pwrcut(struct rtw89_dev *rtwdev) static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev) { - if (rtwdev->chip->chip_id == RTL8852A || - rtwdev->chip->chip_id == RTL8852B) { + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, @@ -2144,14 +2713,14 @@ static void rtw89_pci_hci_ldo(struct rtw89_dev *rtwdev) static int rtw89_pci_dphy_delay(struct rtw89_dev *rtwdev) { - if (rtwdev->chip->chip_id != RTL8852B) + if (!rtw89_is_rtl885xb(rtwdev)) return 0; return rtw89_write16_mdio_mask(rtwdev, RAC_REG_REV2, BAC_CMU_EN_DLY_MASK, PCIE_DPHY_DLY_25US, PCIE_PHY_GEN1); } -static void rtw89_pci_power_wake(struct rtw89_dev *rtwdev, bool pwr_up) +static void rtw89_pci_power_wake_ax(struct rtw89_dev *rtwdev, bool pwr_up) { if (pwr_up) rtw89_write32_set(rtwdev, R_AX_HCI_OPT_CTRL, BIT_WAKE_CTRL); @@ -2267,6 +2836,10 @@ static void rtw89_pci_set_dbg(struct rtw89_dev *rtwdev) rtw89_write32_set(rtwdev, R_AX_PCIE_DBG_CTRL, B_AX_ASFF_FULL_NO_STK | B_AX_EN_STUCK_DBG); + rtw89_write32_mask(rtwdev, R_AX_PCIE_EXP_CTRL, + B_AX_EN_STUCK_DBG | B_AX_ASFF_FULL_NO_STK, + B_AX_EN_STUCK_DBG); + if (rtwdev->chip->chip_id == RTL8852A) rtw89_write32_set(rtwdev, R_AX_PCIE_EXP_CTRL, B_AX_EN_CHKDSC_NO_RX_STUCK); @@ -2281,7 +2854,7 @@ static void rtw89_pci_set_keep_reg(struct rtw89_dev *rtwdev) B_AX_PCIE_TXRST_KEEP_REG | B_AX_PCIE_RXRST_KEEP_REG); } -static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev) +static void rtw89_pci_clr_idx_all_ax(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; @@ -2303,12 +2876,13 @@ static void rtw89_pci_clr_idx_all(struct rtw89_dev *rtwdev) B_AX_CLR_RXQ_IDX | B_AX_CLR_RPQ_IDX); } -static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev) +static int rtw89_pci_poll_txdma_ch_idle_ax(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; - u32 ret, check, dma_busy; u32 dma_busy1 = info->dma_busy1.addr; u32 dma_busy2 = info->dma_busy2_reg; + u32 check, dma_busy; + int ret; check = info->dma_busy1.mask; @@ -2330,11 +2904,12 @@ static int rtw89_poll_txdma_ch_idle_pcie(struct rtw89_dev *rtwdev) return 0; } -static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev) +static int rtw89_pci_poll_rxdma_ch_idle_ax(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; - u32 ret, check, dma_busy; u32 dma_busy3 = info->dma_busy3_reg; + u32 check, dma_busy; + int ret; check = B_AX_RXQ_BUSY | B_AX_RPQ_BUSY; @@ -2348,15 +2923,15 @@ static int rtw89_poll_rxdma_ch_idle_pcie(struct rtw89_dev *rtwdev) static int rtw89_pci_poll_dma_all_idle(struct rtw89_dev *rtwdev) { - u32 ret; + int ret; - ret = rtw89_poll_txdma_ch_idle_pcie(rtwdev); + ret = rtw89_pci_poll_txdma_ch_idle_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "txdma ch busy\n"); return ret; } - ret = rtw89_poll_rxdma_ch_idle_pcie(rtwdev); + ret = rtw89_pci_poll_rxdma_ch_idle_ax(rtwdev); if (ret) { rtw89_err(rtwdev, "rxdma ch busy\n"); return ret; @@ -2406,7 +2981,7 @@ static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev) B_AX_PCIE_RX_APPLEN_MASK, 0); } - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_TXDMA_MASK, tx_burst); rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG1, B_AX_PCIE_MAX_RXDMA_MASK, rx_burst); } else if (chip_id == RTL8852C) { @@ -2414,7 +2989,7 @@ static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev) rtw89_write32_mask(rtwdev, R_AX_HAXI_INIT_CFG1, B_AX_HAXI_MAX_RXDMA_MASK, rx_burst); } - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { if (tag_mode == MAC_AX_TAG_SGL) { val32 = rtw89_read32(rtwdev, R_AX_PCIE_INIT_CFG1) & ~B_AX_LATENCY_CONTROL; @@ -2429,7 +3004,7 @@ static int rtw89_pci_mode_op(struct rtw89_dev *rtwdev) rtw89_write32_mask(rtwdev, info->exp_ctrl_reg, info->max_tag_num_mask, info->multi_tag_num); - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_IDLE, wd_dma_idle_intvl); rtw89_write32_mask(rtwdev, R_AX_PCIE_INIT_CFG2, B_AX_WD_ITVL_ACT, @@ -2458,6 +3033,8 @@ static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; + rtw89_pci_power_wake(rtwdev, false); + if (rtwdev->chip->chip_id == RTL8852A) { /* ltr sw trigger */ rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_IDLE); @@ -2469,11 +3046,12 @@ static int rtw89_pci_ops_deinit(struct rtw89_dev *rtwdev) return 0; } -static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) +static int rtw89_pci_ops_mac_pre_init_ax(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; int ret; + rtw89_pci_ber(rtwdev); rtw89_pci_rxdma_prefth(rtwdev); rtw89_pci_l1off_pwroff(rtwdev); rtw89_pci_deglitch_setting(rtwdev); @@ -2499,7 +3077,7 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) return ret; } - rtw89_pci_power_wake(rtwdev, true); + rtw89_pci_power_wake_ax(rtwdev, true); rtw89_pci_autoload_hang(rtwdev); rtw89_pci_l12_vmain(rtwdev); rtw89_pci_gen2_force_ib(rtwdev); @@ -2528,15 +3106,15 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) /* fill TRX BD indexes */ rtw89_pci_ops_reset(rtwdev); - ret = rtw89_pci_rst_bdram_pcie(rtwdev); + ret = rtw89_pci_rst_bdram_ax(rtwdev); if (ret) { rtw89_warn(rtwdev, "reset bdram busy\n"); return ret; } /* disable all channels except to FW CMD channel to download firmware */ - rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, false); - rtw89_pci_ctrl_txdma_fw_ch_pcie(rtwdev, true); + rtw89_pci_ctrl_txdma_ch_ax(rtwdev, false); + rtw89_pci_ctrl_txdma_fw_ch_ax(rtwdev, true); /* start DMA activities */ rtw89_pci_ctrl_dma_all(rtwdev, true); @@ -2544,6 +3122,13 @@ static int rtw89_pci_ops_mac_pre_init(struct rtw89_dev *rtwdev) return 0; } +static int rtw89_pci_ops_mac_pre_deinit_ax(struct rtw89_dev *rtwdev) +{ + rtw89_pci_power_wake_ax(rtwdev, false); + + return 0; +} + int rtw89_pci_ltr_set(struct rtw89_dev *rtwdev, bool en) { u32 val; @@ -2626,7 +3211,7 @@ int rtw89_pci_ltr_set_v1(struct rtw89_dev *rtwdev, bool en) } EXPORT_SYMBOL(rtw89_pci_ltr_set_v1); -static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev) +static int rtw89_pci_ops_mac_post_init_ax(struct rtw89_dev *rtwdev) { const struct rtw89_pci_info *info = rtwdev->pci_info; enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; @@ -2641,7 +3226,7 @@ static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev) /* ltr sw trigger */ rtw89_write32_set(rtwdev, R_AX_LTR_CTRL_0, B_AX_APP_LTR_ACT); } - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { /* ADDR info 8-byte mode */ rtw89_write32_set(rtwdev, R_AX_TX_ADDRESS_INFO_MODE_SETTING, B_AX_HOST_ADDR_INFO_8B_SEL); @@ -2649,7 +3234,7 @@ static int rtw89_pci_ops_mac_post_init(struct rtw89_dev *rtwdev) } /* enable DMA for all queues */ - rtw89_pci_ctrl_txdma_ch_pcie(rtwdev, true); + rtw89_pci_ctrl_txdma_ch_ax(rtwdev, true); /* Release PCI IO */ rtw89_write32_clr(rtwdev, info->dma_stop1.addr, @@ -2681,10 +3266,69 @@ static int rtw89_pci_claim_device(struct rtw89_dev *rtwdev, static void rtw89_pci_declaim_device(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { - pci_clear_master(pdev); pci_disable_device(pdev); } +static bool rtw89_pci_chip_is_manual_dac(struct rtw89_dev *rtwdev) +{ + const struct rtw89_chip_info *chip = rtwdev->chip; + + switch (chip->chip_id) { + case RTL8852A: + case RTL8852B: + case RTL8851B: + case RTL8852BT: + return true; + default: + return false; + } +} + +static bool rtw89_pci_is_dac_compatible_bridge(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *bridge = pci_upstream_bridge(rtwpci->pdev); + + if (!rtw89_pci_chip_is_manual_dac(rtwdev)) + return true; + + if (!bridge) + return false; + + switch (bridge->vendor) { + case PCI_VENDOR_ID_INTEL: + return true; + case PCI_VENDOR_ID_ASMEDIA: + if (bridge->device == 0x2806) + return true; + break; + } + + return false; +} + +static int rtw89_pci_cfg_dac(struct rtw89_dev *rtwdev, bool force) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + int ret; + u8 val; + + if (!rtwpci->enable_dac && !force) + return 0; + + if (!rtw89_pci_chip_is_manual_dac(rtwdev)) + return 0; + + /* Configure DAC only via PCI config API, not DBI interfaces */ + ret = pci_read_config_byte(pdev, RTW89_PCIE_L1_CTRL, &val); + if (ret) + return ret; + + val |= RTW89_PCIE_BIT_EN_64BITS; + return pci_write_config_byte(pdev, RTW89_PCIE_L1_CTRL, val); +} + static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { @@ -2699,17 +3343,25 @@ static int rtw89_pci_setup_mapping(struct rtw89_dev *rtwdev, goto err; } - ret = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); - if (ret) { - rtw89_err(rtwdev, "failed to set dma mask to 32-bit\n"); - goto err_release_regions; - } + if (!rtw89_pci_is_dac_compatible_bridge(rtwdev)) + goto try_dac_done; - ret = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); - if (ret) { - rtw89_err(rtwdev, "failed to set consistent dma mask to 32-bit\n"); - goto err_release_regions; + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(36)); + if (!ret) { + ret = rtw89_pci_cfg_dac(rtwdev, true); + if (!ret) { + rtwpci->enable_dac = true; + goto try_dac_done; + } + + ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (ret) { + rtw89_err(rtwdev, + "failed to set dma and consistent mask to 32/36-bit\n"); + goto err_release_regions; + } } +try_dac_done: resource_len = pci_resource_len(pdev, bar_id); rtwpci->mmap = pci_iomap(pdev, bar_id, resource_len); @@ -2757,15 +3409,6 @@ static void rtw89_pci_free_tx_ring(struct rtw89_dev *rtwdev, struct pci_dev *pdev, struct rtw89_pci_tx_ring *tx_ring) { - int ring_sz; - u8 *head; - dma_addr_t dma; - - head = tx_ring->bd_ring.head; - dma = tx_ring->bd_ring.dma; - ring_sz = tx_ring->bd_ring.desc_size * tx_ring->bd_ring.len; - dma_free_coherent(&pdev->dev, ring_sz, head, dma); - tx_ring->bd_ring.head = NULL; } @@ -2773,6 +3416,7 @@ static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool; const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_pci_tx_ring *tx_ring; int i; @@ -2780,10 +3424,12 @@ static void rtw89_pci_free_tx_rings(struct rtw89_dev *rtwdev, for (i = 0; i < RTW89_TXCH_NUM; i++) { if (info->tx_dma_ch_mask & BIT(i)) continue; - tx_ring = &rtwpci->tx_rings[i]; + tx_ring = &rtwpci->tx.rings[i]; rtw89_pci_free_tx_wd_ring(rtwdev, pdev, tx_ring); rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring); } + + dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); } static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev, @@ -2794,8 +3440,6 @@ static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev, struct sk_buff *skb; dma_addr_t dma; u32 buf_sz; - u8 *head; - int ring_sz = rx_ring->bd_ring.desc_size * rx_ring->bd_ring.len; int i; buf_sz = rx_ring->buf_sz; @@ -2811,10 +3455,6 @@ static void rtw89_pci_free_rx_ring(struct rtw89_dev *rtwdev, rx_ring->buf[i] = NULL; } - head = rx_ring->bd_ring.head; - dma = rx_ring->bd_ring.dma; - dma_free_coherent(&pdev->dev, ring_sz, head, dma); - rx_ring->bd_ring.head = NULL; } @@ -2822,13 +3462,16 @@ static void rtw89_pci_free_rx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool; struct rtw89_pci_rx_ring *rx_ring; int i; for (i = 0; i < RTW89_RXCH_NUM; i++) { - rx_ring = &rtwpci->rx_rings[i]; + rx_ring = &rtwpci->rx.rings[i]; rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring); } + + dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); } static void rtw89_pci_free_trx_rings(struct rtw89_dev *rtwdev, @@ -2859,6 +3502,7 @@ static int rtw89_pci_init_rx_bd(struct rtw89_dev *rtwdev, struct pci_dev *pdev, memset(rx_bd, 0, sizeof(*rx_bd)); rx_bd->buf_size = cpu_to_le16(buf_sz); rx_bd->dma = cpu_to_le32(dma); + rx_bd->opt = le16_encode_bits(upper_32_bits(dma), RTW89_PCI_RXBD_OPT_DMA_HI); rx_info->dma = dma; return 0; @@ -2919,12 +3563,10 @@ static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev, struct pci_dev *pdev, struct rtw89_pci_tx_ring *tx_ring, u32 desc_size, u32 len, - enum rtw89_tx_channel txch) + enum rtw89_tx_channel txch, + void *head, dma_addr_t dma) { const struct rtw89_pci_ch_dma_addr *txch_addr; - int ring_sz = desc_size * len; - u8 *head; - dma_addr_t dma; int ret; ret = rtw89_pci_alloc_tx_wd_ring(rtwdev, pdev, tx_ring, txch); @@ -2939,12 +3581,6 @@ static int rtw89_pci_alloc_tx_ring(struct rtw89_dev *rtwdev, goto err_free_wd_ring; } - head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); - if (!head) { - ret = -ENOMEM; - goto err_free_wd_ring; - } - INIT_LIST_HEAD(&tx_ring->busy_pages); tx_ring->bd_ring.head = head; tx_ring->bd_ring.dma = dma; @@ -2967,25 +3603,48 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct rtw89_pci_dma_pool *bd_pool = &rtwpci->tx.bd_pool; const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_pci_tx_ring *tx_ring; + u32 i, tx_allocated; + dma_addr_t dma; u32 desc_size; + u32 ring_sz; + u32 pool_sz; + u32 ch_num; + void *head; u32 len; - u32 i, tx_allocated; int ret; + BUILD_BUG_ON(RTW89_PCI_TXBD_NUM_MAX % 16); + + desc_size = sizeof(struct rtw89_pci_tx_bd_32); + len = RTW89_PCI_TXBD_NUM_MAX; + ch_num = RTW89_TXCH_NUM - hweight32(info->tx_dma_ch_mask); + ring_sz = desc_size * len; + pool_sz = ring_sz * ch_num; + + head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL); + if (!head) + return -ENOMEM; + + bd_pool->head = head; + bd_pool->dma = dma; + bd_pool->size = pool_sz; + for (i = 0; i < RTW89_TXCH_NUM; i++) { if (info->tx_dma_ch_mask & BIT(i)) continue; - tx_ring = &rtwpci->tx_rings[i]; - desc_size = sizeof(struct rtw89_pci_tx_bd_32); - len = RTW89_PCI_TXBD_NUM_MAX; + tx_ring = &rtwpci->tx.rings[i]; ret = rtw89_pci_alloc_tx_ring(rtwdev, pdev, tx_ring, - desc_size, len, i); + desc_size, len, i, head, dma); if (ret) { rtw89_err(rtwdev, "failed to alloc tx ring %d\n", i); goto err_free; } + + head += ring_sz; + dma += ring_sz; } return 0; @@ -2993,23 +3652,24 @@ static int rtw89_pci_alloc_tx_rings(struct rtw89_dev *rtwdev, err_free: tx_allocated = i; for (i = 0; i < tx_allocated; i++) { - tx_ring = &rtwpci->tx_rings[i]; + tx_ring = &rtwpci->tx.rings[i]; rtw89_pci_free_tx_ring(rtwdev, pdev, tx_ring); } + dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); + return ret; } static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev, struct pci_dev *pdev, struct rtw89_pci_rx_ring *rx_ring, - u32 desc_size, u32 len, u32 rxch) + u32 desc_size, u32 len, u32 rxch, + void *head, dma_addr_t dma) { + const struct rtw89_pci_info *info = rtwdev->pci_info; const struct rtw89_pci_ch_dma_addr *rxch_addr; struct sk_buff *skb; - u8 *head; - dma_addr_t dma; - int ring_sz = desc_size * len; int buf_sz = RTW89_PCI_RX_BUF_SIZE; int i, allocated; int ret; @@ -3020,22 +3680,20 @@ static int rtw89_pci_alloc_rx_ring(struct rtw89_dev *rtwdev, return ret; } - head = dma_alloc_coherent(&pdev->dev, ring_sz, &dma, GFP_KERNEL); - if (!head) { - ret = -ENOMEM; - goto err; - } - rx_ring->bd_ring.head = head; rx_ring->bd_ring.dma = dma; rx_ring->bd_ring.len = len; rx_ring->bd_ring.desc_size = desc_size; rx_ring->bd_ring.addr = *rxch_addr; - rx_ring->bd_ring.wp = 0; + if (info->rx_ring_eq_is_full) + rx_ring->bd_ring.wp = len - 1; + else + rx_ring->bd_ring.wp = 0; rx_ring->bd_ring.rp = 0; rx_ring->buf_sz = buf_sz; rx_ring->diliver_skb = NULL; rx_ring->diliver_desc.ready = false; + rx_ring->target_rx_tag = 0; for (i = 0; i < len; i++) { skb = dev_alloc_skb(buf_sz); @@ -3070,12 +3728,8 @@ err_free: rx_ring->buf[i] = NULL; } - head = rx_ring->bd_ring.head; - dma = rx_ring->bd_ring.dma; - dma_free_coherent(&pdev->dev, ring_sz, head, dma); - rx_ring->bd_ring.head = NULL; -err: + return ret; } @@ -3083,22 +3737,43 @@ static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct rtw89_pci_dma_pool *bd_pool = &rtwpci->rx.bd_pool; struct rtw89_pci_rx_ring *rx_ring; + int i, rx_allocated; + dma_addr_t dma; u32 desc_size; + u32 ring_sz; + u32 pool_sz; + void *head; u32 len; - int i, rx_allocated; int ret; + desc_size = sizeof(struct rtw89_pci_rx_bd_32); + len = RTW89_PCI_RXBD_NUM_MAX; + ring_sz = desc_size * len; + pool_sz = ring_sz * RTW89_RXCH_NUM; + + head = dma_alloc_coherent(&pdev->dev, pool_sz, &dma, GFP_KERNEL); + if (!head) + return -ENOMEM; + + bd_pool->head = head; + bd_pool->dma = dma; + bd_pool->size = pool_sz; + for (i = 0; i < RTW89_RXCH_NUM; i++) { - rx_ring = &rtwpci->rx_rings[i]; - desc_size = sizeof(struct rtw89_pci_rx_bd_32); - len = RTW89_PCI_RXBD_NUM_MAX; + rx_ring = &rtwpci->rx.rings[i]; + ret = rtw89_pci_alloc_rx_ring(rtwdev, pdev, rx_ring, - desc_size, len, i); + desc_size, len, i, + head, dma); if (ret) { rtw89_err(rtwdev, "failed to alloc rx ring %d\n", i); goto err_free; } + + head += ring_sz; + dma += ring_sz; } return 0; @@ -3106,10 +3781,12 @@ static int rtw89_pci_alloc_rx_rings(struct rtw89_dev *rtwdev, err_free: rx_allocated = i; for (i = 0; i < rx_allocated; i++) { - rx_ring = &rtwpci->rx_rings[i]; + rx_ring = &rtwpci->rx.rings[i]; rtw89_pci_free_rx_ring(rtwdev, pdev, rx_ring); } + dma_free_coherent(&pdev->dev, bd_pool->size, bd_pool->head, bd_pool->dma); + return ret; } @@ -3190,11 +3867,16 @@ static void rtw89_pci_clear_resource(struct rtw89_dev *rtwdev, void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev) { struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_chip_info *chip = rtwdev->chip; + u32 hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN; + + if (chip->chip_id == RTL8851B) + hs0isr_ind_int_en = B_AX_HS0ISR_IND_INT_EN_WKARND; rtwpci->halt_c2h_intrs = B_AX_HALT_C2H_INT_EN | 0; if (rtwpci->under_recovery) { - rtwpci->intrs[0] = B_AX_HS0ISR_IND_INT_EN; + rtwpci->intrs[0] = hs0isr_ind_int_en; rtwpci->intrs[1] = 0; } else { rtwpci->intrs[0] = B_AX_TXDMA_STUCK_INT_EN | @@ -3204,7 +3886,7 @@ void rtw89_pci_config_intr_mask(struct rtw89_dev *rtwdev) B_AX_RXDMA_STUCK_INT_EN | B_AX_RDU_INT_EN | B_AX_RPQBD_FULL_INT_EN | - B_AX_HS0ISR_IND_INT_EN; + hs0isr_ind_int_en; rtwpci->intrs[1] = B_AX_HC10ISR_IND_INT_EN; } @@ -3263,13 +3945,95 @@ void rtw89_pci_config_intr_mask_v1(struct rtw89_dev *rtwdev) } EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v1); +static void rtw89_pci_recovery_intr_mask_v2(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; + rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; + rtwpci->intrs[0] = 0; + rtwpci->intrs[1] = 0; +} + +static void rtw89_pci_default_intr_mask_v2(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + rtwpci->ind_intrs = B_BE_HCI_AXIDMA_INT_EN0 | + B_BE_HS0_IND_INT_EN0; + rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; + rtwpci->intrs[0] = B_BE_RDU_CH1_INT_IMR_V1 | + B_BE_RDU_CH0_INT_IMR_V1; + rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | + B_BE_PCIE_RX_RPQ0_IMR0_V1; +} + +static void rtw89_pci_low_power_intr_mask_v2(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0 | + B_BE_HS1_IND_INT_EN0; + rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; + rtwpci->intrs[0] = 0; + rtwpci->intrs[1] = B_BE_PCIE_RX_RX0P2_IMR0_V1 | + B_BE_PCIE_RX_RPQ0_IMR0_V1; +} + +void rtw89_pci_config_intr_mask_v2(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + if (rtwpci->under_recovery) + rtw89_pci_recovery_intr_mask_v2(rtwdev); + else if (rtwpci->low_power) + rtw89_pci_low_power_intr_mask_v2(rtwdev); + else + rtw89_pci_default_intr_mask_v2(rtwdev); +} +EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v2); + +static void rtw89_pci_recovery_intr_mask_v3(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; + rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; + rtwpci->intrs[0] = 0; + rtwpci->intrs[1] = 0; +} + +static void rtw89_pci_default_intr_mask_v3(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + rtwpci->ind_intrs = B_BE_HS0_IND_INT_EN0; + rtwpci->halt_c2h_intrs = B_BE_HALT_C2H_INT_EN | B_BE_WDT_TIMEOUT_INT_EN; + rtwpci->intrs[0] = 0; + rtwpci->intrs[1] = B_BE_PCIE_RDU_CH1_IMR | + B_BE_PCIE_RDU_CH0_IMR | + B_BE_PCIE_RX_RX0P2_IMR0_V1 | + B_BE_PCIE_RX_RPQ0_IMR0_V1; +} + +void rtw89_pci_config_intr_mask_v3(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + + if (rtwpci->under_recovery) + rtw89_pci_recovery_intr_mask_v3(rtwdev); + else + rtw89_pci_default_intr_mask_v3(rtwdev); +} +EXPORT_SYMBOL(rtw89_pci_config_intr_mask_v3); + static int rtw89_pci_request_irq(struct rtw89_dev *rtwdev, struct pci_dev *pdev) { unsigned long flags = 0; int ret; - flags |= PCI_IRQ_LEGACY | PCI_IRQ_MSI; + flags |= PCI_IRQ_INTX | PCI_IRQ_MSI; ret = pci_alloc_irq_vectors(pdev, 1, 1, flags); if (ret < 0) { rtw89_err(rtwdev, "failed to alloc irq vectors, ret %d\n", ret); @@ -3302,19 +4066,16 @@ static void rtw89_pci_free_irq(struct rtw89_dev *rtwdev, pci_free_irq_vectors(pdev); } -static u16 gray_code_to_bin(u16 gray_code, u32 bit_num) +static u16 gray_code_to_bin(u16 gray_code) { - u16 bin = 0, gray_bit; - u32 bit_idx; + u16 binary = gray_code; - for (bit_idx = 0; bit_idx < bit_num; bit_idx++) { - gray_bit = (gray_code >> bit_idx) & 0x1; - if (bit_num - bit_idx > 1) - gray_bit ^= (gray_code >> (bit_idx + 1)) & 0x1; - bin |= (gray_bit << bit_idx); + while (gray_code) { + gray_code >>= 1; + binary ^= gray_code; } - return bin; + return binary; } static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev) @@ -3350,7 +4111,7 @@ static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev) val16 = rtw89_read16_mask(rtwdev, phy_offset + RAC_ANA1F * RAC_MULT, FILTER_OUT_EQ_MASK); - val16 = gray_code_to_bin(val16, hweight16(val16)); + val16 = gray_code_to_bin(val16); filter_out_val = rtw89_read16(rtwdev, phy_offset + RAC_ANA24 * RAC_MULT); filter_out_val &= ~REG_FILTER_OUT_MASK; @@ -3374,18 +4135,26 @@ static int rtw89_pci_filter_out(struct rtw89_dev *rtwdev) static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable) { - enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - int ret; + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_gen_def *gen_def = info->gen_def; if (rtw89_pci_disable_clkreq) return; + gen_def->clkreq_set(rtwdev, enable); +} + +static void rtw89_pci_clkreq_set_ax(struct rtw89_dev *rtwdev, bool enable) +{ + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + int ret; + ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_CLK_CTRL, PCIE_CLKDLY_HW_30US); if (ret) rtw89_err(rtwdev, "failed to set CLKREQ Delay\n"); - if (chip_id == RTL8852A) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { if (enable) ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, @@ -3411,26 +4180,33 @@ static void rtw89_pci_clkreq_set(struct rtw89_dev *rtwdev, bool enable) static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable) { - enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; - u8 value = 0; - int ret; + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_gen_def *gen_def = info->gen_def; if (rtw89_pci_disable_aspm_l1) return; + gen_def->aspm_set(rtwdev, enable); +} + +static void rtw89_pci_aspm_set_ax(struct rtw89_dev *rtwdev, bool enable) +{ + enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; + u8 value = 0; + int ret; + ret = rtw89_pci_read_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, &value); if (ret) - rtw89_err(rtwdev, "failed to read ASPM Delay\n"); + rtw89_warn(rtwdev, "failed to read ASPM Delay\n"); - value &= ~(RTW89_L1DLY_MASK | RTW89_L0DLY_MASK); - value |= FIELD_PREP(RTW89_L1DLY_MASK, PCIE_L1DLY_16US) | - FIELD_PREP(RTW89_L0DLY_MASK, PCIE_L0SDLY_4US); + u8p_replace_bits(&value, PCIE_L1DLY_16US, RTW89_L1DLY_MASK); + u8p_replace_bits(&value, PCIE_L0SDLY_4US, RTW89_L0DLY_MASK); ret = rtw89_pci_write_config_byte(rtwdev, RTW89_PCIE_ASPM_CTRL, value); if (ret) - rtw89_err(rtwdev, "failed to read ASPM Delay\n"); + rtw89_warn(rtwdev, "failed to read ASPM Delay\n"); - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { if (enable) ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_L1_CTRL, @@ -3454,19 +4230,27 @@ static void rtw89_pci_aspm_set(struct rtw89_dev *rtwdev, bool enable) static void rtw89_pci_recalc_int_mit(struct rtw89_dev *rtwdev) { + enum rtw89_chip_gen chip_gen = rtwdev->chip->chip_gen; + const struct rtw89_pci_info *info = rtwdev->pci_info; struct rtw89_traffic_stats *stats = &rtwdev->stats; enum rtw89_tfc_lv tx_tfc_lv = stats->tx_tfc_lv; enum rtw89_tfc_lv rx_tfc_lv = stats->rx_tfc_lv; u32 val = 0; - if (!rtwdev->scanning && - (tx_tfc_lv >= RTW89_TFC_HIGH || rx_tfc_lv >= RTW89_TFC_HIGH)) + if (rtwdev->scanning || + (tx_tfc_lv < RTW89_TFC_HIGH && rx_tfc_lv < RTW89_TFC_HIGH)) + goto out; + + if (chip_gen == RTW89_CHIP_BE) + val = B_BE_PCIE_MIT_RX0P2_EN | B_BE_PCIE_MIT_RX0P1_EN; + else val = B_AX_RXMIT_RXP2_SEL | B_AX_RXMIT_RXP1_SEL | FIELD_PREP(B_AX_RXCOUNTER_MATCH_MASK, RTW89_PCI_RXBD_NUM_MAX / 2) | FIELD_PREP(B_AX_RXTIMER_UNIT_MASK, AX_RXTIMER_UNIT_64US) | FIELD_PREP(B_AX_RXTIMER_MATCH_MASK, 2048 / 64); - rtw89_write32(rtwdev, R_AX_INT_MIT_RX, val); +out: + rtw89_write32(rtwdev, info->mit_addr, val); } static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev) @@ -3507,10 +4291,21 @@ static void rtw89_pci_link_cfg(struct rtw89_dev *rtwdev) static void rtw89_pci_l1ss_set(struct rtw89_dev *rtwdev, bool enable) { + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_gen_def *gen_def = info->gen_def; + + if (rtw89_pci_disable_l1ss) + return; + + gen_def->l1ss_set(rtwdev, enable); +} + +static void rtw89_pci_l1ss_set_ax(struct rtw89_dev *rtwdev, bool enable) +{ enum rtw89_core_chip_id chip_id = rtwdev->chip->chip_id; int ret; - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { if (enable) ret = rtw89_pci_config_byte_set(rtwdev, RTW89_PCIE_TIMER_CTRL, @@ -3556,7 +4351,16 @@ static void rtw89_pci_l1ss_cfg(struct rtw89_dev *rtwdev) rtw89_pci_l1ss_set(rtwdev, true); } -static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev) +static void rtw89_pci_cpl_timeout_cfg(struct rtw89_dev *rtwdev) +{ + struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + struct pci_dev *pdev = rtwpci->pdev; + + pcie_capability_set_word(pdev, PCI_EXP_DEVCTL2, + PCI_EXP_DEVCTL2_COMP_TMOUT_DIS); +} + +static int rtw89_pci_poll_io_idle_ax(struct rtw89_dev *rtwdev) { int ret = 0; u32 sts; @@ -3573,7 +4377,7 @@ static int rtw89_pci_poll_io_idle(struct rtw89_dev *rtwdev) return ret; } -static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) +static int rtw89_pci_lv1rst_stop_dma_ax(struct rtw89_dev *rtwdev) { u32 val; int ret; @@ -3582,7 +4386,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) return 0; rtw89_pci_ctrl_dma_all(rtwdev, false); - ret = rtw89_pci_poll_io_idle(rtwdev); + ret = rtw89_pci_poll_io_idle_ax(rtwdev); if (ret) { val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); rtw89_debug(rtwdev, RTW89_DBG_HCI, @@ -3593,7 +4397,7 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) if (val & B_AX_RX_STUCK) rtw89_mac_ctrl_hci_dma_rx(rtwdev, false); rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); - ret = rtw89_pci_poll_io_idle(rtwdev); + ret = rtw89_pci_poll_io_idle_ax(rtwdev); val = rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG); rtw89_debug(rtwdev, RTW89_DBG_HCI, "[PCIe] poll_io_idle fail, after 0x%08x: 0x%08x\n", @@ -3603,25 +4407,9 @@ static int rtw89_pci_lv1rst_stop_dma(struct rtw89_dev *rtwdev) return ret; } - - -static int rtw89_pci_rst_bdram(struct rtw89_dev *rtwdev) +static int rtw89_pci_lv1rst_start_dma_ax(struct rtw89_dev *rtwdev) { - int ret = 0; - u32 val32, sts; - - val32 = B_AX_RST_BDRAM; - rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, val32); - - ret = read_poll_timeout_atomic(rtw89_read32, sts, - (sts & B_AX_RST_BDRAM) == 0x0, 1, 100, - true, rtwdev, R_AX_PCIE_INIT_CFG1); - return ret; -} - -static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev) -{ - u32 ret; + int ret; if (rtwdev->chip->chip_id == RTL8852C) return 0; @@ -3630,29 +4418,31 @@ static int rtw89_pci_lv1rst_start_dma(struct rtw89_dev *rtwdev) rtw89_mac_ctrl_hci_dma_trx(rtwdev, true); rtw89_pci_clr_idx_all(rtwdev); - ret = rtw89_pci_rst_bdram(rtwdev); + ret = rtw89_pci_rst_bdram_ax(rtwdev); if (ret) return ret; rtw89_pci_ctrl_dma_all(rtwdev, true); - return ret; + return 0; } static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev, enum rtw89_lv1_rcvy_step step) { + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_gen_def *gen_def = info->gen_def; int ret; switch (step) { case RTW89_LV1_RCVY_STEP_1: - ret = rtw89_pci_lv1rst_stop_dma(rtwdev); + ret = gen_def->lv1rst_stop_dma(rtwdev); if (ret) rtw89_err(rtwdev, "lv1 rcvy pci stop dma fail\n"); break; case RTW89_LV1_RCVY_STEP_2: - ret = rtw89_pci_lv1rst_start_dma(rtwdev); + ret = gen_def->lv1rst_start_dma(rtwdev); if (ret) rtw89_err(rtwdev, "lv1 rcvy pci start dma fail\n"); break; @@ -3666,29 +4456,41 @@ static int rtw89_pci_ops_mac_lv1_recovery(struct rtw89_dev *rtwdev, static void rtw89_pci_ops_dump_err_status(struct rtw89_dev *rtwdev) { - rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n", - rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX)); - rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", - rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG)); - rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", - rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG)); + if (rtwdev->chip->chip_gen == RTW89_CHIP_BE) + return; + + if (rtwdev->chip->chip_id == RTL8852C) { + rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG_V1)); + rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG_V1)); + } else { + rtw89_info(rtwdev, "R_AX_RPQ_RXBD_IDX =0x%08x\n", + rtw89_read32(rtwdev, R_AX_RPQ_RXBD_IDX)); + rtw89_info(rtwdev, "R_AX_DBG_ERR_FLAG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_DBG_ERR_FLAG)); + rtw89_info(rtwdev, "R_AX_LBC_WATCHDOG=0x%08x\n", + rtw89_read32(rtwdev, R_AX_LBC_WATCHDOG)); + } } static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget) { struct rtw89_dev *rtwdev = container_of(napi, struct rtw89_dev, napi); struct rtw89_pci *rtwpci = (struct rtw89_pci *)rtwdev->priv; + const struct rtw89_pci_info *info = rtwdev->pci_info; + const struct rtw89_pci_isr_def *isr_def = info->isr_def; unsigned long flags; int work_done; rtwdev->napi_budget_countdown = budget; - rtw89_pci_clear_isr0(rtwdev, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT); + rtw89_write32(rtwdev, isr_def->isr_clear_rpq.addr, isr_def->isr_clear_rpq.data); work_done = rtw89_pci_poll_rpq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); if (work_done == budget) return budget; - rtw89_pci_clear_isr0(rtwdev, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | B_AX_RDU_INT); + rtw89_write32(rtwdev, isr_def->isr_clear_rxq.addr, isr_def->isr_clear_rxq.data); work_done += rtw89_pci_poll_rxq_dma(rtwdev, rtwpci, rtwdev->napi_budget_countdown); if (work_done < budget && napi_complete_done(napi, work_done)) { spin_lock_irqsave(&rtwpci->irq_lock, flags); @@ -3700,6 +4502,36 @@ static int rtw89_pci_napi_poll(struct napi_struct *napi, int budget) return work_done; } +static +void rtw89_check_pci_ssid_quirks(struct rtw89_dev *rtwdev, + struct pci_dev *pdev, + const struct rtw89_pci_ssid_quirk *ssid_quirks) +{ + int i; + + if (!ssid_quirks) + return; + + for (i = 0; i < 200; i++, ssid_quirks++) { + if (ssid_quirks->vendor == 0 && ssid_quirks->device == 0) + break; + + if (ssid_quirks->vendor != pdev->vendor || + ssid_quirks->device != pdev->device || + ssid_quirks->subsystem_vendor != pdev->subsystem_vendor || + ssid_quirks->subsystem_device != pdev->subsystem_device) + continue; + + bitmap_or(rtwdev->quirks, rtwdev->quirks, &ssid_quirks->bitmap, + NUM_OF_RTW89_QUIRKS); + rtwdev->custid = ssid_quirks->custid; + break; + } + + rtw89_debug(rtwdev, RTW89_DBG_HCI, "quirks=%*ph custid=%d\n", + (int)sizeof(rtwdev->quirks), rtwdev->quirks, rtwdev->custid); +} + static int __maybe_unused rtw89_pci_suspend(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); @@ -3709,7 +4541,7 @@ static int __maybe_unused rtw89_pci_suspend(struct device *dev) rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { rtw89_write32_clr(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); rtw89_write32_set(rtwdev, R_AX_PCIE_INIT_CFG1, @@ -3734,6 +4566,18 @@ static void rtw89_pci_l2_hci_ldo(struct rtw89_dev *rtwdev) RTW89_PCIE_BIT_CFG_RST_MSTATE); } +void rtw89_pci_basic_cfg(struct rtw89_dev *rtwdev, bool resume) +{ + if (resume) + rtw89_pci_cfg_dac(rtwdev, false); + + rtw89_pci_disable_eq(rtwdev); + rtw89_pci_filter_out(rtwdev); + rtw89_pci_cpl_timeout_cfg(rtwdev); + rtw89_pci_link_cfg(rtwdev); + rtw89_pci_l1ss_cfg(rtwdev); +} + static int __maybe_unused rtw89_pci_resume(struct device *dev) { struct ieee80211_hw *hw = dev_get_drvdata(dev); @@ -3743,7 +4587,7 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) rtw89_write32_set(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_R_DIS_PRST); rtw89_write32_clr(rtwdev, R_AX_RSV_CTRL, B_AX_WLOCK_1C_BIT6); - if (chip_id == RTL8852A || chip_id == RTL8852B) { + if (chip_id == RTL8852A || rtw89_is_rtl885xb(rtwdev)) { rtw89_write32_set(rtwdev, R_AX_SYS_SDIO_CTRL, B_AX_PCIE_DIS_L2_CTRL_LDO_HCI); rtw89_write32_clr(rtwdev, R_AX_PCIE_INIT_CFG1, @@ -3755,9 +4599,8 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) B_AX_SEL_REQ_ENTR_L1); } rtw89_pci_l2_hci_ldo(rtwdev); - rtw89_pci_filter_out(rtwdev); - rtw89_pci_link_cfg(rtwdev); - rtw89_pci_l1ss_cfg(rtwdev); + + rtw89_pci_basic_cfg(rtwdev, true); return 0; } @@ -3765,6 +4608,77 @@ static int __maybe_unused rtw89_pci_resume(struct device *dev) SIMPLE_DEV_PM_OPS(rtw89_pm_ops, rtw89_pci_suspend, rtw89_pci_resume); EXPORT_SYMBOL(rtw89_pm_ops); +static pci_ers_result_t rtw89_pci_io_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + + netif_device_detach(netdev); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t rtw89_pci_io_slot_reset(struct pci_dev *pdev) +{ + struct ieee80211_hw *hw = pci_get_drvdata(pdev); + struct rtw89_dev *rtwdev = hw->priv; + + rtw89_ser_notify(rtwdev, MAC_AX_ERR_ASSERTION); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void rtw89_pci_io_resume(struct pci_dev *pdev) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + + /* ack any pending wake events, disable PME */ + pci_enable_wake(pdev, PCI_D0, 0); + + netif_device_attach(netdev); +} + +const struct pci_error_handlers rtw89_pci_err_handler = { + .error_detected = rtw89_pci_io_error_detected, + .slot_reset = rtw89_pci_io_slot_reset, + .resume = rtw89_pci_io_resume, +}; +EXPORT_SYMBOL(rtw89_pci_err_handler); + +const struct rtw89_pci_isr_def rtw89_pci_isr_ax = { + .isr_rdu = B_AX_RDU_INT, + .isr_halt_c2h = B_AX_HALT_C2H_INT_EN, + .isr_wdt_timeout = B_AX_WDT_TIMEOUT_INT_EN, + .isr_clear_rpq = {R_AX_PCIE_HISR00, B_AX_RPQDMA_INT | B_AX_RPQBD_FULL_INT}, + .isr_clear_rxq = {R_AX_PCIE_HISR00, B_AX_RXP1DMA_INT | B_AX_RXDMA_INT | + B_AX_RDU_INT}, +}; +EXPORT_SYMBOL(rtw89_pci_isr_ax); + +const struct rtw89_pci_gen_def rtw89_pci_gen_ax = { + .mac_pre_init = rtw89_pci_ops_mac_pre_init_ax, + .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit_ax, + .mac_post_init = rtw89_pci_ops_mac_post_init_ax, + + .clr_idx_all = rtw89_pci_clr_idx_all_ax, + .rst_bdram = rtw89_pci_rst_bdram_ax, + + .lv1rst_stop_dma = rtw89_pci_lv1rst_stop_dma_ax, + .lv1rst_start_dma = rtw89_pci_lv1rst_start_dma_ax, + + .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_ax, + .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_ax, + .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle_ax, + + .aspm_set = rtw89_pci_aspm_set_ax, + .clkreq_set = rtw89_pci_clkreq_set_ax, + .l1ss_set = rtw89_pci_l1ss_set_ax, + + .disable_eq = rtw89_pci_disable_eq_ax, + .power_wake = rtw89_pci_power_wake_ax, +}; +EXPORT_SYMBOL(rtw89_pci_gen_ax); + static const struct rtw89_hci_ops rtw89_pci_ops = { .tx_write = rtw89_pci_ops_tx_write, .tx_kick_off = rtw89_pci_ops_tx_kick_off, @@ -3783,7 +4697,10 @@ static const struct rtw89_hci_ops rtw89_pci_ops = { .write16 = rtw89_pci_ops_write16, .write32 = rtw89_pci_ops_write32, + .read32_pci_cfg = rtw89_pci_ops_read32_pci_cfg, + .mac_pre_init = rtw89_pci_ops_mac_pre_init, + .mac_pre_deinit = rtw89_pci_ops_mac_pre_deinit, .mac_post_init = rtw89_pci_ops_mac_post_init, .deinit = rtw89_pci_ops_deinit, @@ -3795,15 +4712,16 @@ static const struct rtw89_hci_ops rtw89_pci_ops = { .recovery_start = rtw89_pci_ops_recovery_start, .recovery_complete = rtw89_pci_ops_recovery_complete, - .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch_pcie, - .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch_pcie, + .ctrl_txdma_ch = rtw89_pci_ctrl_txdma_ch, + .ctrl_txdma_fw_ch = rtw89_pci_ctrl_txdma_fw_ch, .ctrl_trxhci = rtw89_pci_ctrl_dma_trx, - .poll_txdma_ch = rtw89_poll_txdma_ch_idle_pcie, + .poll_txdma_ch_idle = rtw89_pci_poll_txdma_ch_idle, + .clr_idx_all = rtw89_pci_clr_idx_all, .clear = rtw89_pci_clear_resource, .disable_intr = rtw89_pci_disable_intr_lock, .enable_intr = rtw89_pci_enable_intr_lock, - .rst_bdram = rtw89_pci_rst_bdram_pcie, + .rst_bdram = rtw89_pci_reset_bdram, }; int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) @@ -3817,7 +4735,7 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rtwdev = rtw89_alloc_ieee80211_hw(&pdev->dev, sizeof(struct rtw89_pci), - info->chip); + info->chip, info->variant); if (!rtwdev) { dev_err(&pdev->dev, "failed to allocate hw\n"); return -ENOMEM; @@ -3828,9 +4746,13 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) rtwdev->pci_info = info->bus.pci; rtwdev->hci.ops = &rtw89_pci_ops; rtwdev->hci.type = RTW89_HCI_TYPE_PCIE; + rtwdev->hci.dle_type = RTW89_HCI_DLE_TYPE_PCIE; rtwdev->hci.rpwm_addr = pci_info->rpwm_addr; rtwdev->hci.cpwm_addr = pci_info->cpwm_addr; + rtw89_check_quirks(rtwdev, info->quirks); + rtw89_check_pci_ssid_quirks(rtwdev, pdev, pci_info->ssid_quirks); + SET_IEEE80211_DEV(rtwdev->hw, &pdev->dev); ret = rtw89_core_init(rtwdev); @@ -3857,29 +4779,34 @@ int rtw89_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) goto err_clear_resource; } - rtw89_pci_filter_out(rtwdev); - rtw89_pci_link_cfg(rtwdev); - rtw89_pci_l1ss_cfg(rtwdev); + rtw89_pci_basic_cfg(rtwdev, false); - ret = rtw89_core_register(rtwdev); + ret = rtw89_core_napi_init(rtwdev); if (ret) { - rtw89_err(rtwdev, "failed to register core\n"); + rtw89_err(rtwdev, "failed to init napi\n"); goto err_clear_resource; } - rtw89_core_napi_init(rtwdev); - ret = rtw89_pci_request_irq(rtwdev, pdev); if (ret) { rtw89_err(rtwdev, "failed to request pci irq\n"); - goto err_unregister; + goto err_deinit_napi; } + ret = rtw89_core_register(rtwdev); + if (ret) { + rtw89_err(rtwdev, "failed to register core\n"); + goto err_free_irq; + } + + set_bit(RTW89_FLAG_PROBE_DONE, rtwdev->flags); + return 0; -err_unregister: +err_free_irq: + rtw89_pci_free_irq(rtwdev, pdev); +err_deinit_napi: rtw89_core_napi_deinit(rtwdev); - rtw89_core_unregister(rtwdev); err_clear_resource: rtw89_pci_clear_resource(rtwdev, pdev); err_declaim_pci: @@ -3911,5 +4838,5 @@ void rtw89_pci_remove(struct pci_dev *pdev) EXPORT_SYMBOL(rtw89_pci_remove); MODULE_AUTHOR("Realtek Corporation"); -MODULE_DESCRIPTION("Realtek 802.11ax wireless PCI driver"); +MODULE_DESCRIPTION("Realtek PCI 802.11ax wireless driver"); MODULE_LICENSE("Dual BSD/GPL"); |