diff options
Diffstat (limited to 'drivers/ntb')
| -rw-r--r-- | drivers/ntb/hw/idt/ntb_hw_idt.c | 2 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/Makefile | 1 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen1.c (renamed from drivers/ntb/hw/intel/ntb_hw_intel.c) | 713 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen1.h | 182 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen3.c | 597 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_gen3.h | 110 | ||||
| -rw-r--r-- | drivers/ntb/hw/intel/ntb_hw_intel.h | 203 | ||||
| -rw-r--r-- | drivers/ntb/ntb_transport.c | 6 |
8 files changed, 1001 insertions, 813 deletions
diff --git a/drivers/ntb/hw/idt/ntb_hw_idt.c b/drivers/ntb/hw/idt/ntb_hw_idt.c index 8d98872d0983..dbe72f116017 100644 --- a/drivers/ntb/hw/idt/ntb_hw_idt.c +++ b/drivers/ntb/hw/idt/ntb_hw_idt.c @@ -1401,7 +1401,7 @@ static int idt_ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx, * 5. Doorbell operations * * Doorbell functionality of IDT PCIe-switches is pretty unusual. First of - * all there is global doorbell register which state can by changed by any + * all there is global doorbell register which state can be changed by any * NT-function of the IDT device in accordance with global permissions. These * permissions configs are not supported by NTB API, so it must be done by * either BIOS or EEPROM settings. In the same way the state of the global diff --git a/drivers/ntb/hw/intel/Makefile b/drivers/ntb/hw/intel/Makefile index 1b434568d2ad..4ff22af967c6 100644 --- a/drivers/ntb/hw/intel/Makefile +++ b/drivers/ntb/hw/intel/Makefile @@ -1 +1,2 @@ obj-$(CONFIG_NTB_INTEL) += ntb_hw_intel.o +ntb_hw_intel-y := ntb_hw_gen1.o ntb_hw_gen3.o diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_gen1.c index 3be323132896..6aa573227279 100644 --- a/drivers/ntb/hw/intel/ntb_hw_intel.c +++ b/drivers/ntb/hw/intel/ntb_hw_gen1.c @@ -45,9 +45,6 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. * * Intel PCIe NTB Linux driver - * - * Contact Information: - * Jon Mason <jon.mason@intel.com> */ #include <linux/debugfs.h> @@ -61,6 +58,8 @@ #include <linux/ntb.h> #include "ntb_hw_intel.h" +#include "ntb_hw_gen1.h" +#include "ntb_hw_gen3.h" #define NTB_NAME "ntb_hw_intel" #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver" @@ -80,14 +79,7 @@ static const struct intel_ntb_alt_reg xeon_sec_reg; static const struct intel_ntb_alt_reg xeon_b2b_reg; static const struct intel_ntb_xlat_reg xeon_pri_xlat; static const struct intel_ntb_xlat_reg xeon_sec_xlat; -static struct intel_b2b_addr xeon_b2b_usd_addr; -static struct intel_b2b_addr xeon_b2b_dsd_addr; -static const struct intel_ntb_reg skx_reg; -static const struct intel_ntb_alt_reg skx_pri_reg; -static const struct intel_ntb_alt_reg skx_b2b_reg; -static const struct intel_ntb_xlat_reg skx_sec_xlat; static const struct ntb_dev_ops intel_ntb_ops; -static const struct ntb_dev_ops intel_ntb3_ops; static const struct file_operations intel_ntb_debugfs_info; static struct dentry *debugfs_dir; @@ -146,68 +138,8 @@ module_param_named(xeon_b2b_dsd_bar5_addr32, MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32, "XEON B2B DSD split-BAR 5 32-bit address"); -static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd); -static int xeon_init_isr(struct intel_ntb_dev *ndev); - -#ifndef ioread64 -#ifdef readq -#define ioread64 readq -#else -#define ioread64 _ioread64 -static inline u64 _ioread64(void __iomem *mmio) -{ - u64 low, high; - - low = ioread32(mmio); - high = ioread32(mmio + sizeof(u32)); - return low | (high << 32); -} -#endif -#endif - -#ifndef iowrite64 -#ifdef writeq -#define iowrite64 writeq -#else -#define iowrite64 _iowrite64 -static inline void _iowrite64(u64 val, void __iomem *mmio) -{ - iowrite32(val, mmio); - iowrite32(val >> 32, mmio + sizeof(u32)); -} -#endif -#endif - -static inline int pdev_is_xeon(struct pci_dev *pdev) -{ - switch (pdev->device) { - case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: - case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: - case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: - case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: - case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: - case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: - case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: - case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: - case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: - case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: - case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: - return 1; - } - return 0; -} - -static inline int pdev_is_skx_xeon(struct pci_dev *pdev) -{ - if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX) - return 1; - return 0; -} +static int xeon_init_isr(struct intel_ntb_dev *ndev); static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev) { @@ -241,7 +173,7 @@ static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev, return !!flag; } -static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) +int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx) { if (idx < 0 || idx >= ndev->mw_count) return -EINVAL; @@ -268,7 +200,7 @@ static inline int ndev_db_addr(struct intel_ntb_dev *ndev, return 0; } -static inline u64 ndev_db_read(struct intel_ntb_dev *ndev, +u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio) { if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) @@ -277,7 +209,7 @@ static inline u64 ndev_db_read(struct intel_ntb_dev *ndev, return ndev->reg->db_ioread(mmio); } -static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, +int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, void __iomem *mmio) { if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB)) @@ -429,7 +361,7 @@ static irqreturn_t ndev_irq_isr(int irq, void *dev) return ndev_interrupt(ndev, irq - ndev->ntb.pdev->irq); } -static int ndev_init_isr(struct intel_ntb_dev *ndev, +int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max, int msix_shift, int total_shift) { @@ -557,169 +489,6 @@ static void ndev_deinit_isr(struct intel_ntb_dev *ndev) } } -static ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf, - size_t count, loff_t *offp) -{ - struct intel_ntb_dev *ndev; - void __iomem *mmio; - char *buf; - size_t buf_size; - ssize_t ret, off; - union { u64 v64; u32 v32; u16 v16; } u; - - ndev = filp->private_data; - mmio = ndev->self_mmio; - - buf_size = min(count, 0x800ul); - - buf = kmalloc(buf_size, GFP_KERNEL); - if (!buf) - return -ENOMEM; - - off = 0; - - off += scnprintf(buf + off, buf_size - off, - "NTB Device Information:\n"); - - off += scnprintf(buf + off, buf_size - off, - "Connection Topology -\t%s\n", - ntb_topo_string(ndev->ntb.topo)); - - off += scnprintf(buf + off, buf_size - off, - "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); - off += scnprintf(buf + off, buf_size - off, - "LNK STA -\t\t%#06x\n", ndev->lnk_sta); - - if (!ndev->reg->link_is_up(ndev)) - off += scnprintf(buf + off, buf_size - off, - "Link Status -\t\tDown\n"); - else { - off += scnprintf(buf + off, buf_size - off, - "Link Status -\t\tUp\n"); - off += scnprintf(buf + off, buf_size - off, - "Link Speed -\t\tPCI-E Gen %u\n", - NTB_LNK_STA_SPEED(ndev->lnk_sta)); - off += scnprintf(buf + off, buf_size - off, - "Link Width -\t\tx%u\n", - NTB_LNK_STA_WIDTH(ndev->lnk_sta)); - } - - off += scnprintf(buf + off, buf_size - off, - "Memory Window Count -\t%u\n", ndev->mw_count); - off += scnprintf(buf + off, buf_size - off, - "Scratchpad Count -\t%u\n", ndev->spad_count); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Count -\t%u\n", ndev->db_count); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); - - off += scnprintf(buf + off, buf_size - off, - "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); - - u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Mask -\t\t%#llx\n", u.v64); - - u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); - off += scnprintf(buf + off, buf_size - off, - "Doorbell Bell -\t\t%#llx\n", u.v64); - - off += scnprintf(buf + off, buf_size - off, - "\nNTB Incoming XLAT:\n"); - - u.v64 = ioread64(mmio + SKX_IMBAR1XBASE_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "IMBAR1XBASE -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_IMBAR2XBASE_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "IMBAR2XBASE -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64); - - if (ntb_topo_is_b2b(ndev->ntb.topo)) { - off += scnprintf(buf + off, buf_size - off, - "\nNTB Outgoing B2B XLAT:\n"); - - u.v64 = ioread64(mmio + SKX_EMBAR1XBASE_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR1XBASE -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_EMBAR2XBASE_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR2XBASE -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_EMBAR1XLMT_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR1XLMT -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_EMBAR2XLMT_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR2XLMT -\t\t%#018llx\n", u.v64); - - off += scnprintf(buf + off, buf_size - off, - "\nNTB Secondary BAR:\n"); - - u.v64 = ioread64(mmio + SKX_EMBAR0_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR0 -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_EMBAR1_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR1 -\t\t%#018llx\n", u.v64); - - u.v64 = ioread64(mmio + SKX_EMBAR2_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "EMBAR2 -\t\t%#018llx\n", u.v64); - } - - off += scnprintf(buf + off, buf_size - off, - "\nNTB Statistics:\n"); - - u.v16 = ioread16(mmio + SKX_USMEMMISS_OFFSET); - off += scnprintf(buf + off, buf_size - off, - "Upstream Memory Miss -\t%u\n", u.v16); - - off += scnprintf(buf + off, buf_size - off, - "\nNTB Hardware Errors:\n"); - - if (!pci_read_config_word(ndev->ntb.pdev, - SKX_DEVSTS_OFFSET, &u.v16)) - off += scnprintf(buf + off, buf_size - off, - "DEVSTS -\t\t%#06x\n", u.v16); - - if (!pci_read_config_word(ndev->ntb.pdev, - SKX_LINK_STATUS_OFFSET, &u.v16)) - off += scnprintf(buf + off, buf_size - off, - "LNKSTS -\t\t%#06x\n", u.v16); - - if (!pci_read_config_dword(ndev->ntb.pdev, - SKX_UNCERRSTS_OFFSET, &u.v32)) - off += scnprintf(buf + off, buf_size - off, - "UNCERRSTS -\t\t%#06x\n", u.v32); - - if (!pci_read_config_dword(ndev->ntb.pdev, - SKX_CORERRSTS_OFFSET, &u.v32)) - off += scnprintf(buf + off, buf_size - off, - "CORERRSTS -\t\t%#06x\n", u.v32); - - ret = simple_read_from_buffer(ubuf, count, offp, buf, off); - kfree(buf); - return ret; -} - static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf, size_t count, loff_t *offp) { @@ -879,7 +648,7 @@ static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf, "LMT45 -\t\t\t%#018llx\n", u.v64); } - if (pdev_is_xeon(pdev)) { + if (pdev_is_gen1(pdev)) { if (ntb_topo_is_b2b(ndev->ntb.topo)) { off += scnprintf(buf + off, buf_size - off, "\nNTB Outgoing B2B XLAT:\n"); @@ -991,9 +760,9 @@ static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf, { struct intel_ntb_dev *ndev = filp->private_data; - if (pdev_is_xeon(ndev->ntb.pdev)) + if (pdev_is_gen1(ndev->ntb.pdev)) return ndev_ntb_debugfs_read(filp, ubuf, count, offp); - else if (pdev_is_skx_xeon(ndev->ntb.pdev)) + else if (pdev_is_gen3(ndev->ntb.pdev)) return ndev_ntb3_debugfs_read(filp, ubuf, count, offp); return -ENXIO; @@ -1023,7 +792,7 @@ static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev) debugfs_remove_recursive(ndev->debugfs_dir); } -static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) +int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) { if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; @@ -1031,10 +800,10 @@ static int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx) return ntb_ndev(ntb)->mw_count; } -static int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, - resource_size_t *addr_align, - resource_size_t *size_align, - resource_size_t *size_max) +int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, + resource_size_t *addr_align, + resource_size_t *size_align, + resource_size_t *size_max) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); resource_size_t bar_size, mw_size; @@ -1170,9 +939,8 @@ static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, return 0; } -static u64 intel_ntb_link_is_up(struct ntb_dev *ntb, - enum ntb_speed *speed, - enum ntb_width *width) +u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, + enum ntb_width *width) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1224,7 +992,7 @@ static int intel_ntb_link_enable(struct ntb_dev *ntb, return 0; } -static int intel_ntb_link_disable(struct ntb_dev *ntb) +int intel_ntb_link_disable(struct ntb_dev *ntb) { struct intel_ntb_dev *ndev; u32 ntb_cntl; @@ -1248,14 +1016,14 @@ static int intel_ntb_link_disable(struct ntb_dev *ntb) return 0; } -static int intel_ntb_peer_mw_count(struct ntb_dev *ntb) +int intel_ntb_peer_mw_count(struct ntb_dev *ntb) { /* Numbers of inbound and outbound memory windows match */ return ntb_ndev(ntb)->mw_count; } -static int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, - phys_addr_t *base, resource_size_t *size) +int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, + phys_addr_t *base, resource_size_t *size) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); int bar; @@ -1283,12 +1051,12 @@ static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb) return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB); } -static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) +u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb) { return ntb_ndev(ntb)->db_valid_mask; } -static int intel_ntb_db_vector_count(struct ntb_dev *ntb) +int intel_ntb_db_vector_count(struct ntb_dev *ntb) { struct intel_ntb_dev *ndev; @@ -1297,7 +1065,7 @@ static int intel_ntb_db_vector_count(struct ntb_dev *ntb) return ndev->db_vec_count; } -static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) +u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1325,7 +1093,7 @@ static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) ndev->self_reg->db_bell); } -static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) +int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1334,7 +1102,7 @@ static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) ndev->self_reg->db_mask); } -static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) +int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1343,9 +1111,8 @@ static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) ndev->self_reg->db_mask); } -static int intel_ntb_peer_db_addr(struct ntb_dev *ntb, - phys_addr_t *db_addr, - resource_size_t *db_size) +int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, + resource_size_t *db_size) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1362,12 +1129,12 @@ static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) ndev->peer_reg->db_bell); } -static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) +int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb) { return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD); } -static int intel_ntb_spad_count(struct ntb_dev *ntb) +int intel_ntb_spad_count(struct ntb_dev *ntb) { struct intel_ntb_dev *ndev; @@ -1376,7 +1143,7 @@ static int intel_ntb_spad_count(struct ntb_dev *ntb) return ndev->spad_count; } -static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) +u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1385,8 +1152,7 @@ static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx) ndev->self_reg->spad); } -static int intel_ntb_spad_write(struct ntb_dev *ntb, - int idx, u32 val) +int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1395,8 +1161,8 @@ static int intel_ntb_spad_write(struct ntb_dev *ntb, ndev->self_reg->spad); } -static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, - phys_addr_t *spad_addr) +int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, + phys_addr_t *spad_addr) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1404,7 +1170,7 @@ static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, ndev->peer_reg->spad); } -static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) +u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1413,8 +1179,8 @@ static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) ndev->peer_reg->spad); } -static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, - int sidx, u32 val) +int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, + u32 val) { struct intel_ntb_dev *ndev = ntb_ndev(ntb); @@ -1423,336 +1189,6 @@ static int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, ndev->peer_reg->spad); } -/* Skylake Xeon NTB */ - -static int skx_poll_link(struct intel_ntb_dev *ndev) -{ - u16 reg_val; - int rc; - - ndev->reg->db_iowrite(ndev->db_link_mask, - ndev->self_mmio + - ndev->self_reg->db_clear); - - rc = pci_read_config_word(ndev->ntb.pdev, - SKX_LINK_STATUS_OFFSET, ®_val); - if (rc) - return 0; - - if (reg_val == ndev->lnk_sta) - return 0; - - ndev->lnk_sta = reg_val; - - return 1; -} - -static u64 skx_db_ioread(void __iomem *mmio) -{ - return ioread64(mmio); -} - -static void skx_db_iowrite(u64 bits, void __iomem *mmio) -{ - iowrite64(bits, mmio); -} - -static int skx_init_isr(struct intel_ntb_dev *ndev) -{ - int i; - - /* - * The MSIX vectors and the interrupt status bits are not lined up - * on Skylake. By default the link status bit is bit 32, however it - * is by default MSIX vector0. We need to fixup to line them up. - * The vectors at reset is 1-32,0. We need to reprogram to 0-32. - */ - - for (i = 0; i < SKX_DB_MSIX_VECTOR_COUNT; i++) - iowrite8(i, ndev->self_mmio + SKX_INTVEC_OFFSET + i); - - /* move link status down one as workaround */ - if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) { - iowrite8(SKX_DB_MSIX_VECTOR_COUNT - 2, - ndev->self_mmio + SKX_INTVEC_OFFSET + - (SKX_DB_MSIX_VECTOR_COUNT - 1)); - } - - return ndev_init_isr(ndev, SKX_DB_MSIX_VECTOR_COUNT, - SKX_DB_MSIX_VECTOR_COUNT, - SKX_DB_MSIX_VECTOR_SHIFT, - SKX_DB_TOTAL_SHIFT); -} - -static int skx_setup_b2b_mw(struct intel_ntb_dev *ndev, - const struct intel_b2b_addr *addr, - const struct intel_b2b_addr *peer_addr) -{ - struct pci_dev *pdev; - void __iomem *mmio; - phys_addr_t bar_addr; - - pdev = ndev->ntb.pdev; - mmio = ndev->self_mmio; - - /* setup incoming bar limits == base addrs (zero length windows) */ - bar_addr = addr->bar2_addr64; - iowrite64(bar_addr, mmio + SKX_IMBAR1XLMT_OFFSET); - bar_addr = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET); - dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr); - - bar_addr = addr->bar4_addr64; - iowrite64(bar_addr, mmio + SKX_IMBAR2XLMT_OFFSET); - bar_addr = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET); - dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr); - - /* zero incoming translation addrs */ - iowrite64(0, mmio + SKX_IMBAR1XBASE_OFFSET); - iowrite64(0, mmio + SKX_IMBAR2XBASE_OFFSET); - - ndev->peer_mmio = ndev->self_mmio; - - return 0; -} - -static int skx_init_ntb(struct intel_ntb_dev *ndev) -{ - int rc; - - - ndev->mw_count = XEON_MW_COUNT; - ndev->spad_count = SKX_SPAD_COUNT; - ndev->db_count = SKX_DB_COUNT; - ndev->db_link_mask = SKX_DB_LINK_BIT; - - /* DB fixup for using 31 right now */ - if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) - ndev->db_link_mask |= BIT_ULL(31); - - switch (ndev->ntb.topo) { - case NTB_TOPO_B2B_USD: - case NTB_TOPO_B2B_DSD: - ndev->self_reg = &skx_pri_reg; - ndev->peer_reg = &skx_b2b_reg; - ndev->xlat_reg = &skx_sec_xlat; - - if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { - rc = skx_setup_b2b_mw(ndev, - &xeon_b2b_dsd_addr, - &xeon_b2b_usd_addr); - } else { - rc = skx_setup_b2b_mw(ndev, - &xeon_b2b_usd_addr, - &xeon_b2b_dsd_addr); - } - - if (rc) - return rc; - - /* Enable Bus Master and Memory Space on the secondary side */ - iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, - ndev->self_mmio + SKX_SPCICMD_OFFSET); - - break; - - default: - return -EINVAL; - } - - ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; - - ndev->reg->db_iowrite(ndev->db_valid_mask, - ndev->self_mmio + - ndev->self_reg->db_mask); - - return 0; -} - -static int skx_init_dev(struct intel_ntb_dev *ndev) -{ - struct pci_dev *pdev; - u8 ppd; - int rc; - - pdev = ndev->ntb.pdev; - - ndev->reg = &skx_reg; - - rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); - if (rc) - return -EIO; - - ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); - dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd, - ntb_topo_string(ndev->ntb.topo)); - if (ndev->ntb.topo == NTB_TOPO_NONE) - return -EINVAL; - - if (pdev_is_skx_xeon(pdev)) - ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD; - - rc = skx_init_ntb(ndev); - if (rc) - return rc; - - return skx_init_isr(ndev); -} - -static int intel_ntb3_link_enable(struct ntb_dev *ntb, - enum ntb_speed max_speed, - enum ntb_width max_width) -{ - struct intel_ntb_dev *ndev; - u32 ntb_ctl; - - ndev = container_of(ntb, struct intel_ntb_dev, ntb); - - dev_dbg(&ntb->pdev->dev, - "Enabling link with max_speed %d max_width %d\n", - max_speed, max_width); - - if (max_speed != NTB_SPEED_AUTO) - dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed); - if (max_width != NTB_WIDTH_AUTO) - dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width); - - ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); - ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); - ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; - ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; - iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); - - return 0; -} -static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, - dma_addr_t addr, resource_size_t size) -{ - struct intel_ntb_dev *ndev = ntb_ndev(ntb); - unsigned long xlat_reg, limit_reg; - resource_size_t bar_size, mw_size; - void __iomem *mmio; - u64 base, limit, reg_val; - int bar; - - if (pidx != NTB_DEF_PEER_IDX) - return -EINVAL; - - if (idx >= ndev->b2b_idx && !ndev->b2b_off) - idx += 1; - - bar = ndev_mw_to_bar(ndev, idx); - if (bar < 0) - return bar; - - bar_size = pci_resource_len(ndev->ntb.pdev, bar); - - if (idx == ndev->b2b_idx) - mw_size = bar_size - ndev->b2b_off; - else - mw_size = bar_size; - - /* hardware requires that addr is aligned to bar size */ - if (addr & (bar_size - 1)) - return -EINVAL; - - /* make sure the range fits in the usable mw size */ - if (size > mw_size) - return -EINVAL; - - mmio = ndev->self_mmio; - xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10); - limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10); - base = pci_resource_start(ndev->ntb.pdev, bar); - - /* Set the limit if supported, if size is not mw_size */ - if (limit_reg && size != mw_size) - limit = base + size; - else - limit = base + mw_size; - - /* set and verify setting the translation address */ - iowrite64(addr, mmio + xlat_reg); - reg_val = ioread64(mmio + xlat_reg); - if (reg_val != addr) { - iowrite64(0, mmio + xlat_reg); - return -EIO; - } - - dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val); - - /* set and verify setting the limit */ - iowrite64(limit, mmio + limit_reg); - reg_val = ioread64(mmio + limit_reg); - if (reg_val != limit) { - iowrite64(base, mmio + limit_reg); - iowrite64(0, mmio + xlat_reg); - return -EIO; - } - - dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val); - - /* setup the EP */ - limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000; - base = ioread64(mmio + SKX_EMBAR1_OFFSET + (8 * idx)); - base &= ~0xf; - - if (limit_reg && size != mw_size) - limit = base + size; - else - limit = base + mw_size; - - /* set and verify setting the limit */ - iowrite64(limit, mmio + limit_reg); - reg_val = ioread64(mmio + limit_reg); - if (reg_val != limit) { - iowrite64(base, mmio + limit_reg); - iowrite64(0, mmio + xlat_reg); - return -EIO; - } - - dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val); - - return 0; -} - -static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits) -{ - struct intel_ntb_dev *ndev = ntb_ndev(ntb); - int bit; - - if (db_bits & ~ndev->db_valid_mask) - return -EINVAL; - - while (db_bits) { - bit = __ffs(db_bits); - iowrite32(1, ndev->peer_mmio + - ndev->peer_reg->db_bell + (bit * 4)); - db_bits &= db_bits - 1; - } - - return 0; -} - -static u64 intel_ntb3_db_read(struct ntb_dev *ntb) -{ - struct intel_ntb_dev *ndev = ntb_ndev(ntb); - - return ndev_db_read(ndev, - ndev->self_mmio + - ndev->self_reg->db_clear); -} - -static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits) -{ - struct intel_ntb_dev *ndev = ntb_ndev(ntb); - - return ndev_db_write(ndev, db_bits, - ndev->self_mmio + - ndev->self_reg->db_clear); -} - -/* XEON */ - static u64 xeon_db_ioread(void __iomem *mmio) { return (u64)ioread16(mmio); @@ -1785,7 +1221,7 @@ static int xeon_poll_link(struct intel_ntb_dev *ndev) return 1; } -static int xeon_link_is_up(struct intel_ntb_dev *ndev) +int xeon_link_is_up(struct intel_ntb_dev *ndev) { if (ndev->ntb.topo == NTB_TOPO_SEC) return 1; @@ -1793,7 +1229,7 @@ static int xeon_link_is_up(struct intel_ntb_dev *ndev) return NTB_LNK_STA_ACTIVE(ndev->lnk_sta); } -static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) +enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd) { switch (ppd & XEON_PPD_TOPO_MASK) { case XEON_PPD_TOPO_B2B_USD: @@ -2410,7 +1846,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev, node = dev_to_node(&pdev->dev); - if (pdev_is_xeon(pdev)) { + if (pdev_is_gen1(pdev)) { ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); if (!ndev) { rc = -ENOMEM; @@ -2427,7 +1863,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev, if (rc) goto err_init_dev; - } else if (pdev_is_skx_xeon(pdev)) { + } else if (pdev_is_gen3(pdev)) { ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node); if (!ndev) { rc = -ENOMEM; @@ -2441,7 +1877,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev, if (rc) goto err_init_pci; - rc = skx_init_dev(ndev); + rc = gen3_init_dev(ndev); if (rc) goto err_init_dev; @@ -2466,7 +1902,7 @@ static int intel_ntb_pci_probe(struct pci_dev *pdev, err_register: ndev_deinit_debugfs(ndev); - if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev)) + if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev)) xeon_deinit_dev(ndev); err_init_dev: intel_ntb_deinit_pci(ndev); @@ -2482,7 +1918,7 @@ static void intel_ntb_pci_remove(struct pci_dev *pdev) ntb_unregister_device(&ndev->ntb); ndev_deinit_debugfs(ndev); - if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev)) + if (pdev_is_gen1(pdev) || pdev_is_gen3(pdev)) xeon_deinit_dev(ndev); intel_ntb_deinit_pci(ndev); kfree(ndev); @@ -2537,50 +1973,20 @@ static const struct intel_ntb_xlat_reg xeon_sec_xlat = { .bar2_xlat = XEON_SBAR23XLAT_OFFSET, }; -static struct intel_b2b_addr xeon_b2b_usd_addr = { +struct intel_b2b_addr xeon_b2b_usd_addr = { .bar2_addr64 = XEON_B2B_BAR2_ADDR64, .bar4_addr64 = XEON_B2B_BAR4_ADDR64, .bar4_addr32 = XEON_B2B_BAR4_ADDR32, .bar5_addr32 = XEON_B2B_BAR5_ADDR32, }; -static struct intel_b2b_addr xeon_b2b_dsd_addr = { +struct intel_b2b_addr xeon_b2b_dsd_addr = { .bar2_addr64 = XEON_B2B_BAR2_ADDR64, .bar4_addr64 = XEON_B2B_BAR4_ADDR64, .bar4_addr32 = XEON_B2B_BAR4_ADDR32, .bar5_addr32 = XEON_B2B_BAR5_ADDR32, }; -static const struct intel_ntb_reg skx_reg = { - .poll_link = skx_poll_link, - .link_is_up = xeon_link_is_up, - .db_ioread = skx_db_ioread, - .db_iowrite = skx_db_iowrite, - .db_size = sizeof(u32), - .ntb_ctl = SKX_NTBCNTL_OFFSET, - .mw_bar = {2, 4}, -}; - -static const struct intel_ntb_alt_reg skx_pri_reg = { - .db_bell = SKX_EM_DOORBELL_OFFSET, - .db_clear = SKX_IM_INT_STATUS_OFFSET, - .db_mask = SKX_IM_INT_DISABLE_OFFSET, - .spad = SKX_IM_SPAD_OFFSET, -}; - -static const struct intel_ntb_alt_reg skx_b2b_reg = { - .db_bell = SKX_IM_DOORBELL_OFFSET, - .db_clear = SKX_EM_INT_STATUS_OFFSET, - .db_mask = SKX_EM_INT_DISABLE_OFFSET, - .spad = SKX_B2B_SPAD_OFFSET, -}; - -static const struct intel_ntb_xlat_reg skx_sec_xlat = { -/* .bar0_base = SKX_EMBAR0_OFFSET, */ - .bar2_limit = SKX_IMBAR1XLMT_OFFSET, - .bar2_xlat = SKX_IMBAR1XBASE_OFFSET, -}; - /* operations for primary side of local ntb */ static const struct ntb_dev_ops intel_ntb_ops = { .mw_count = intel_ntb_mw_count, @@ -2610,33 +2016,6 @@ static const struct ntb_dev_ops intel_ntb_ops = { .peer_spad_write = intel_ntb_peer_spad_write, }; -static const struct ntb_dev_ops intel_ntb3_ops = { - .mw_count = intel_ntb_mw_count, - .mw_get_align = intel_ntb_mw_get_align, - .mw_set_trans = intel_ntb3_mw_set_trans, - .peer_mw_count = intel_ntb_peer_mw_count, - .peer_mw_get_addr = intel_ntb_peer_mw_get_addr, - .link_is_up = intel_ntb_link_is_up, - .link_enable = intel_ntb3_link_enable, - .link_disable = intel_ntb_link_disable, - .db_valid_mask = intel_ntb_db_valid_mask, - .db_vector_count = intel_ntb_db_vector_count, - .db_vector_mask = intel_ntb_db_vector_mask, - .db_read = intel_ntb3_db_read, - .db_clear = intel_ntb3_db_clear, - .db_set_mask = intel_ntb_db_set_mask, - .db_clear_mask = intel_ntb_db_clear_mask, - .peer_db_addr = intel_ntb_peer_db_addr, - .peer_db_set = intel_ntb3_peer_db_set, - .spad_is_unsafe = intel_ntb_spad_is_unsafe, - .spad_count = intel_ntb_spad_count, - .spad_read = intel_ntb_spad_read, - .spad_write = intel_ntb_spad_write, - .peer_spad_addr = intel_ntb_peer_spad_addr, - .peer_spad_read = intel_ntb_peer_spad_read, - .peer_spad_write = intel_ntb_peer_spad_write, -}; - static const struct file_operations intel_ntb_debugfs_info = { .owner = THIS_MODULE, .open = simple_open, diff --git a/drivers/ntb/hw/intel/ntb_hw_gen1.h b/drivers/ntb/hw/intel/ntb_hw_gen1.h new file mode 100644 index 000000000000..ad8ec1444436 --- /dev/null +++ b/drivers/ntb/hw/intel/ntb_hw_gen1.h @@ -0,0 +1,182 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2012-2017 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright(c) 2012-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _NTB_INTEL_GEN1_H_ +#define _NTB_INTEL_GEN1_H_ + +#include "ntb_hw_intel.h" + +/* Intel Gen1 Xeon hardware */ +#define XEON_PBAR23LMT_OFFSET 0x0000 +#define XEON_PBAR45LMT_OFFSET 0x0008 +#define XEON_PBAR4LMT_OFFSET 0x0008 +#define XEON_PBAR5LMT_OFFSET 0x000c +#define XEON_PBAR23XLAT_OFFSET 0x0010 +#define XEON_PBAR45XLAT_OFFSET 0x0018 +#define XEON_PBAR4XLAT_OFFSET 0x0018 +#define XEON_PBAR5XLAT_OFFSET 0x001c +#define XEON_SBAR23LMT_OFFSET 0x0020 +#define XEON_SBAR45LMT_OFFSET 0x0028 +#define XEON_SBAR4LMT_OFFSET 0x0028 +#define XEON_SBAR5LMT_OFFSET 0x002c +#define XEON_SBAR23XLAT_OFFSET 0x0030 +#define XEON_SBAR45XLAT_OFFSET 0x0038 +#define XEON_SBAR4XLAT_OFFSET 0x0038 +#define XEON_SBAR5XLAT_OFFSET 0x003c +#define XEON_SBAR0BASE_OFFSET 0x0040 +#define XEON_SBAR23BASE_OFFSET 0x0048 +#define XEON_SBAR45BASE_OFFSET 0x0050 +#define XEON_SBAR4BASE_OFFSET 0x0050 +#define XEON_SBAR5BASE_OFFSET 0x0054 +#define XEON_SBDF_OFFSET 0x005c +#define XEON_NTBCNTL_OFFSET 0x0058 +#define XEON_PDOORBELL_OFFSET 0x0060 +#define XEON_PDBMSK_OFFSET 0x0062 +#define XEON_SDOORBELL_OFFSET 0x0064 +#define XEON_SDBMSK_OFFSET 0x0066 +#define XEON_USMEMMISS_OFFSET 0x0070 +#define XEON_SPAD_OFFSET 0x0080 +#define XEON_PBAR23SZ_OFFSET 0x00d0 +#define XEON_PBAR45SZ_OFFSET 0x00d1 +#define XEON_PBAR4SZ_OFFSET 0x00d1 +#define XEON_SBAR23SZ_OFFSET 0x00d2 +#define XEON_SBAR45SZ_OFFSET 0x00d3 +#define XEON_SBAR4SZ_OFFSET 0x00d3 +#define XEON_PPD_OFFSET 0x00d4 +#define XEON_PBAR5SZ_OFFSET 0x00d5 +#define XEON_SBAR5SZ_OFFSET 0x00d6 +#define XEON_WCCNTRL_OFFSET 0x00e0 +#define XEON_UNCERRSTS_OFFSET 0x014c +#define XEON_CORERRSTS_OFFSET 0x0158 +#define XEON_LINK_STATUS_OFFSET 0x01a2 +#define XEON_SPCICMD_OFFSET 0x0504 +#define XEON_DEVCTRL_OFFSET 0x0598 +#define XEON_DEVSTS_OFFSET 0x059a +#define XEON_SLINK_STATUS_OFFSET 0x05a2 +#define XEON_B2B_SPAD_OFFSET 0x0100 +#define XEON_B2B_DOORBELL_OFFSET 0x0140 +#define XEON_B2B_XLAT_OFFSETL 0x0144 +#define XEON_B2B_XLAT_OFFSETU 0x0148 +#define XEON_PPD_CONN_MASK 0x03 +#define XEON_PPD_CONN_TRANSPARENT 0x00 +#define XEON_PPD_CONN_B2B 0x01 +#define XEON_PPD_CONN_RP 0x02 +#define XEON_PPD_DEV_MASK 0x10 +#define XEON_PPD_DEV_USD 0x00 +#define XEON_PPD_DEV_DSD 0x10 +#define XEON_PPD_SPLIT_BAR_MASK 0x40 + +#define XEON_PPD_TOPO_MASK (XEON_PPD_CONN_MASK | XEON_PPD_DEV_MASK) +#define XEON_PPD_TOPO_PRI_USD (XEON_PPD_CONN_RP | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_PRI_DSD (XEON_PPD_CONN_RP | XEON_PPD_DEV_DSD) +#define XEON_PPD_TOPO_SEC_USD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_SEC_DSD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_DSD) +#define XEON_PPD_TOPO_B2B_USD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_USD) +#define XEON_PPD_TOPO_B2B_DSD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_DSD) + +#define XEON_MW_COUNT 2 +#define HSX_SPLIT_BAR_MW_COUNT 3 +#define XEON_DB_COUNT 15 +#define XEON_DB_LINK 15 +#define XEON_DB_LINK_BIT BIT_ULL(XEON_DB_LINK) +#define XEON_DB_MSIX_VECTOR_COUNT 4 +#define XEON_DB_MSIX_VECTOR_SHIFT 5 +#define XEON_DB_TOTAL_SHIFT 16 +#define XEON_SPAD_COUNT 16 + +/* Use the following addresses for translation between b2b ntb devices in case + * the hardware default values are not reliable. */ +#define XEON_B2B_BAR0_ADDR 0x1000000000000000ull +#define XEON_B2B_BAR2_ADDR64 0x2000000000000000ull +#define XEON_B2B_BAR4_ADDR64 0x4000000000000000ull +#define XEON_B2B_BAR4_ADDR32 0x20000000u +#define XEON_B2B_BAR5_ADDR32 0x40000000u + +/* The peer ntb secondary config space is 32KB fixed size */ +#define XEON_B2B_MIN_SIZE 0x8000 + +/* flags to indicate hardware errata */ +#define NTB_HWERR_SDOORBELL_LOCKUP BIT_ULL(0) +#define NTB_HWERR_SB01BASE_LOCKUP BIT_ULL(1) +#define NTB_HWERR_B2BDOORBELL_BIT14 BIT_ULL(2) +#define NTB_HWERR_MSIX_VECTOR32_BAD BIT_ULL(3) + +extern struct intel_b2b_addr xeon_b2b_usd_addr; +extern struct intel_b2b_addr xeon_b2b_dsd_addr; + +int ndev_init_isr(struct intel_ntb_dev *ndev, int msix_min, int msix_max, + int msix_shift, int total_shift); +enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd); +u64 ndev_db_read(struct intel_ntb_dev *ndev, void __iomem *mmio); +int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits, + void __iomem *mmio); +int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx); +int intel_ntb_mw_count(struct ntb_dev *ntb, int pidx); +int intel_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int idx, + resource_size_t *addr_align, resource_size_t *size_align, + resource_size_t *size_max); +int intel_ntb_peer_mw_count(struct ntb_dev *ntb); +int intel_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, + phys_addr_t *base, resource_size_t *size); +u64 intel_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, + enum ntb_width *width); +int intel_ntb_link_disable(struct ntb_dev *ntb); +u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb); +int intel_ntb_db_vector_count(struct ntb_dev *ntb); +u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector); +int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits); +int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits); +int intel_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, + resource_size_t *db_size); +int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb); +int intel_ntb_spad_count(struct ntb_dev *ntb); +u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx); +int intel_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val); +u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx); +int intel_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, + u32 val); +int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, + phys_addr_t *spad_addr); +int xeon_link_is_up(struct intel_ntb_dev *ndev); + +#endif diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.c b/drivers/ntb/hw/intel/ntb_hw_gen3.c new file mode 100644 index 000000000000..b3fa24778f94 --- /dev/null +++ b/drivers/ntb/hw/intel/ntb_hw_gen3.c @@ -0,0 +1,597 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright(c) 2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * Intel PCIe GEN3 NTB Linux driver + * + */ + +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/random.h> +#include <linux/slab.h> +#include <linux/ntb.h> + +#include "ntb_hw_intel.h" +#include "ntb_hw_gen1.h" +#include "ntb_hw_gen3.h" + +static int gen3_poll_link(struct intel_ntb_dev *ndev); + +static const struct intel_ntb_reg gen3_reg = { + .poll_link = gen3_poll_link, + .link_is_up = xeon_link_is_up, + .db_ioread = gen3_db_ioread, + .db_iowrite = gen3_db_iowrite, + .db_size = sizeof(u32), + .ntb_ctl = GEN3_NTBCNTL_OFFSET, + .mw_bar = {2, 4}, +}; + +static const struct intel_ntb_alt_reg gen3_pri_reg = { + .db_bell = GEN3_EM_DOORBELL_OFFSET, + .db_clear = GEN3_IM_INT_STATUS_OFFSET, + .db_mask = GEN3_IM_INT_DISABLE_OFFSET, + .spad = GEN3_IM_SPAD_OFFSET, +}; + +static const struct intel_ntb_alt_reg gen3_b2b_reg = { + .db_bell = GEN3_IM_DOORBELL_OFFSET, + .db_clear = GEN3_EM_INT_STATUS_OFFSET, + .db_mask = GEN3_EM_INT_DISABLE_OFFSET, + .spad = GEN3_B2B_SPAD_OFFSET, +}; + +static const struct intel_ntb_xlat_reg gen3_sec_xlat = { +/* .bar0_base = GEN3_EMBAR0_OFFSET, */ + .bar2_limit = GEN3_IMBAR1XLMT_OFFSET, + .bar2_xlat = GEN3_IMBAR1XBASE_OFFSET, +}; + +static int gen3_poll_link(struct intel_ntb_dev *ndev) +{ + u16 reg_val; + int rc; + + ndev->reg->db_iowrite(ndev->db_link_mask, + ndev->self_mmio + + ndev->self_reg->db_clear); + + rc = pci_read_config_word(ndev->ntb.pdev, + GEN3_LINK_STATUS_OFFSET, ®_val); + if (rc) + return 0; + + if (reg_val == ndev->lnk_sta) + return 0; + + ndev->lnk_sta = reg_val; + + return 1; +} + +static int gen3_init_isr(struct intel_ntb_dev *ndev) +{ + int i; + + /* + * The MSIX vectors and the interrupt status bits are not lined up + * on Skylake. By default the link status bit is bit 32, however it + * is by default MSIX vector0. We need to fixup to line them up. + * The vectors at reset is 1-32,0. We need to reprogram to 0-32. + */ + + for (i = 0; i < GEN3_DB_MSIX_VECTOR_COUNT; i++) + iowrite8(i, ndev->self_mmio + GEN3_INTVEC_OFFSET + i); + + /* move link status down one as workaround */ + if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) { + iowrite8(GEN3_DB_MSIX_VECTOR_COUNT - 2, + ndev->self_mmio + GEN3_INTVEC_OFFSET + + (GEN3_DB_MSIX_VECTOR_COUNT - 1)); + } + + return ndev_init_isr(ndev, GEN3_DB_MSIX_VECTOR_COUNT, + GEN3_DB_MSIX_VECTOR_COUNT, + GEN3_DB_MSIX_VECTOR_SHIFT, + GEN3_DB_TOTAL_SHIFT); +} + +static int gen3_setup_b2b_mw(struct intel_ntb_dev *ndev, + const struct intel_b2b_addr *addr, + const struct intel_b2b_addr *peer_addr) +{ + struct pci_dev *pdev; + void __iomem *mmio; + phys_addr_t bar_addr; + + pdev = ndev->ntb.pdev; + mmio = ndev->self_mmio; + + /* setup incoming bar limits == base addrs (zero length windows) */ + bar_addr = addr->bar2_addr64; + iowrite64(bar_addr, mmio + GEN3_IMBAR1XLMT_OFFSET); + bar_addr = ioread64(mmio + GEN3_IMBAR1XLMT_OFFSET); + dev_dbg(&pdev->dev, "IMBAR1XLMT %#018llx\n", bar_addr); + + bar_addr = addr->bar4_addr64; + iowrite64(bar_addr, mmio + GEN3_IMBAR2XLMT_OFFSET); + bar_addr = ioread64(mmio + GEN3_IMBAR2XLMT_OFFSET); + dev_dbg(&pdev->dev, "IMBAR2XLMT %#018llx\n", bar_addr); + + /* zero incoming translation addrs */ + iowrite64(0, mmio + GEN3_IMBAR1XBASE_OFFSET); + iowrite64(0, mmio + GEN3_IMBAR2XBASE_OFFSET); + + ndev->peer_mmio = ndev->self_mmio; + + return 0; +} + +static int gen3_init_ntb(struct intel_ntb_dev *ndev) +{ + int rc; + + + ndev->mw_count = XEON_MW_COUNT; + ndev->spad_count = GEN3_SPAD_COUNT; + ndev->db_count = GEN3_DB_COUNT; + ndev->db_link_mask = GEN3_DB_LINK_BIT; + + /* DB fixup for using 31 right now */ + if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) + ndev->db_link_mask |= BIT_ULL(31); + + switch (ndev->ntb.topo) { + case NTB_TOPO_B2B_USD: + case NTB_TOPO_B2B_DSD: + ndev->self_reg = &gen3_pri_reg; + ndev->peer_reg = &gen3_b2b_reg; + ndev->xlat_reg = &gen3_sec_xlat; + + if (ndev->ntb.topo == NTB_TOPO_B2B_USD) { + rc = gen3_setup_b2b_mw(ndev, + &xeon_b2b_dsd_addr, + &xeon_b2b_usd_addr); + } else { + rc = gen3_setup_b2b_mw(ndev, + &xeon_b2b_usd_addr, + &xeon_b2b_dsd_addr); + } + + if (rc) + return rc; + + /* Enable Bus Master and Memory Space on the secondary side */ + iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER, + ndev->self_mmio + GEN3_SPCICMD_OFFSET); + + break; + + default: + return -EINVAL; + } + + ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1; + + ndev->reg->db_iowrite(ndev->db_valid_mask, + ndev->self_mmio + + ndev->self_reg->db_mask); + + return 0; +} + +int gen3_init_dev(struct intel_ntb_dev *ndev) +{ + struct pci_dev *pdev; + u8 ppd; + int rc; + + pdev = ndev->ntb.pdev; + + ndev->reg = &gen3_reg; + + rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd); + if (rc) + return -EIO; + + ndev->ntb.topo = xeon_ppd_topo(ndev, ppd); + dev_dbg(&pdev->dev, "ppd %#x topo %s\n", ppd, + ntb_topo_string(ndev->ntb.topo)); + if (ndev->ntb.topo == NTB_TOPO_NONE) + return -EINVAL; + + ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD; + + rc = gen3_init_ntb(ndev); + if (rc) + return rc; + + return gen3_init_isr(ndev); +} + +ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp) +{ + struct intel_ntb_dev *ndev; + void __iomem *mmio; + char *buf; + size_t buf_size; + ssize_t ret, off; + union { u64 v64; u32 v32; u16 v16; } u; + + ndev = filp->private_data; + mmio = ndev->self_mmio; + + buf_size = min(count, 0x800ul); + + buf = kmalloc(buf_size, GFP_KERNEL); + if (!buf) + return -ENOMEM; + + off = 0; + + off += scnprintf(buf + off, buf_size - off, + "NTB Device Information:\n"); + + off += scnprintf(buf + off, buf_size - off, + "Connection Topology -\t%s\n", + ntb_topo_string(ndev->ntb.topo)); + + off += scnprintf(buf + off, buf_size - off, + "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl); + off += scnprintf(buf + off, buf_size - off, + "LNK STA -\t\t%#06x\n", ndev->lnk_sta); + + if (!ndev->reg->link_is_up(ndev)) + off += scnprintf(buf + off, buf_size - off, + "Link Status -\t\tDown\n"); + else { + off += scnprintf(buf + off, buf_size - off, + "Link Status -\t\tUp\n"); + off += scnprintf(buf + off, buf_size - off, + "Link Speed -\t\tPCI-E Gen %u\n", + NTB_LNK_STA_SPEED(ndev->lnk_sta)); + off += scnprintf(buf + off, buf_size - off, + "Link Width -\t\tx%u\n", + NTB_LNK_STA_WIDTH(ndev->lnk_sta)); + } + + off += scnprintf(buf + off, buf_size - off, + "Memory Window Count -\t%u\n", ndev->mw_count); + off += scnprintf(buf + off, buf_size - off, + "Scratchpad Count -\t%u\n", ndev->spad_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Count -\t%u\n", ndev->db_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Vector Count -\t%u\n", ndev->db_vec_count); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift); + + off += scnprintf(buf + off, buf_size - off, + "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask); + + u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Mask -\t\t%#llx\n", u.v64); + + u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell); + off += scnprintf(buf + off, buf_size - off, + "Doorbell Bell -\t\t%#llx\n", u.v64); + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Incoming XLAT:\n"); + + u.v64 = ioread64(mmio + GEN3_IMBAR1XBASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "IMBAR1XBASE -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_IMBAR2XBASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "IMBAR2XBASE -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_IMBAR1XLMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_IMBAR2XLMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64); + + if (ntb_topo_is_b2b(ndev->ntb.topo)) { + off += scnprintf(buf + off, buf_size - off, + "\nNTB Outgoing B2B XLAT:\n"); + + u.v64 = ioread64(mmio + GEN3_EMBAR1XBASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR1XBASE -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_EMBAR2XBASE_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR2XBASE -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_EMBAR1XLMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR1XLMT -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_EMBAR2XLMT_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR2XLMT -\t\t%#018llx\n", u.v64); + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Secondary BAR:\n"); + + u.v64 = ioread64(mmio + GEN3_EMBAR0_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR0 -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_EMBAR1_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR1 -\t\t%#018llx\n", u.v64); + + u.v64 = ioread64(mmio + GEN3_EMBAR2_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "EMBAR2 -\t\t%#018llx\n", u.v64); + } + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Statistics:\n"); + + u.v16 = ioread16(mmio + GEN3_USMEMMISS_OFFSET); + off += scnprintf(buf + off, buf_size - off, + "Upstream Memory Miss -\t%u\n", u.v16); + + off += scnprintf(buf + off, buf_size - off, + "\nNTB Hardware Errors:\n"); + + if (!pci_read_config_word(ndev->ntb.pdev, + GEN3_DEVSTS_OFFSET, &u.v16)) + off += scnprintf(buf + off, buf_size - off, + "DEVSTS -\t\t%#06x\n", u.v16); + + if (!pci_read_config_word(ndev->ntb.pdev, + GEN3_LINK_STATUS_OFFSET, &u.v16)) + off += scnprintf(buf + off, buf_size - off, + "LNKSTS -\t\t%#06x\n", u.v16); + + if (!pci_read_config_dword(ndev->ntb.pdev, + GEN3_UNCERRSTS_OFFSET, &u.v32)) + off += scnprintf(buf + off, buf_size - off, + "UNCERRSTS -\t\t%#06x\n", u.v32); + + if (!pci_read_config_dword(ndev->ntb.pdev, + GEN3_CORERRSTS_OFFSET, &u.v32)) + off += scnprintf(buf + off, buf_size - off, + "CORERRSTS -\t\t%#06x\n", u.v32); + + ret = simple_read_from_buffer(ubuf, count, offp, buf, off); + kfree(buf); + return ret; +} + +static int intel_ntb3_link_enable(struct ntb_dev *ntb, + enum ntb_speed max_speed, + enum ntb_width max_width) +{ + struct intel_ntb_dev *ndev; + u32 ntb_ctl; + + ndev = container_of(ntb, struct intel_ntb_dev, ntb); + + dev_dbg(&ntb->pdev->dev, + "Enabling link with max_speed %d max_width %d\n", + max_speed, max_width); + + if (max_speed != NTB_SPEED_AUTO) + dev_dbg(&ntb->pdev->dev, "ignoring max_speed %d\n", max_speed); + if (max_width != NTB_WIDTH_AUTO) + dev_dbg(&ntb->pdev->dev, "ignoring max_width %d\n", max_width); + + ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl); + ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK); + ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP; + ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP; + iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl); + + return 0; +} +static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int pidx, int idx, + dma_addr_t addr, resource_size_t size) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + unsigned long xlat_reg, limit_reg; + resource_size_t bar_size, mw_size; + void __iomem *mmio; + u64 base, limit, reg_val; + int bar; + + if (pidx != NTB_DEF_PEER_IDX) + return -EINVAL; + + if (idx >= ndev->b2b_idx && !ndev->b2b_off) + idx += 1; + + bar = ndev_mw_to_bar(ndev, idx); + if (bar < 0) + return bar; + + bar_size = pci_resource_len(ndev->ntb.pdev, bar); + + if (idx == ndev->b2b_idx) + mw_size = bar_size - ndev->b2b_off; + else + mw_size = bar_size; + + /* hardware requires that addr is aligned to bar size */ + if (addr & (bar_size - 1)) + return -EINVAL; + + /* make sure the range fits in the usable mw size */ + if (size > mw_size) + return -EINVAL; + + mmio = ndev->self_mmio; + xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10); + limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10); + base = pci_resource_start(ndev->ntb.pdev, bar); + + /* Set the limit if supported, if size is not mw_size */ + if (limit_reg && size != mw_size) + limit = base + size; + else + limit = base + mw_size; + + /* set and verify setting the translation address */ + iowrite64(addr, mmio + xlat_reg); + reg_val = ioread64(mmio + xlat_reg); + if (reg_val != addr) { + iowrite64(0, mmio + xlat_reg); + return -EIO; + } + + dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val); + + /* set and verify setting the limit */ + iowrite64(limit, mmio + limit_reg); + reg_val = ioread64(mmio + limit_reg); + if (reg_val != limit) { + iowrite64(base, mmio + limit_reg); + iowrite64(0, mmio + xlat_reg); + return -EIO; + } + + dev_dbg(&ntb->pdev->dev, "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val); + + /* setup the EP */ + limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000; + base = ioread64(mmio + GEN3_EMBAR1_OFFSET + (8 * idx)); + base &= ~0xf; + + if (limit_reg && size != mw_size) + limit = base + size; + else + limit = base + mw_size; + + /* set and verify setting the limit */ + iowrite64(limit, mmio + limit_reg); + reg_val = ioread64(mmio + limit_reg); + if (reg_val != limit) { + iowrite64(base, mmio + limit_reg); + iowrite64(0, mmio + xlat_reg); + return -EIO; + } + + dev_dbg(&ntb->pdev->dev, "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val); + + return 0; +} + +static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + int bit; + + if (db_bits & ~ndev->db_valid_mask) + return -EINVAL; + + while (db_bits) { + bit = __ffs(db_bits); + iowrite32(1, ndev->peer_mmio + + ndev->peer_reg->db_bell + (bit * 4)); + db_bits &= db_bits - 1; + } + + return 0; +} + +static u64 intel_ntb3_db_read(struct ntb_dev *ntb) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_read(ndev, + ndev->self_mmio + + ndev->self_reg->db_clear); +} + +static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits) +{ + struct intel_ntb_dev *ndev = ntb_ndev(ntb); + + return ndev_db_write(ndev, db_bits, + ndev->self_mmio + + ndev->self_reg->db_clear); +} + +const struct ntb_dev_ops intel_ntb3_ops = { + .mw_count = intel_ntb_mw_count, + .mw_get_align = intel_ntb_mw_get_align, + .mw_set_trans = intel_ntb3_mw_set_trans, + .peer_mw_count = intel_ntb_peer_mw_count, + .peer_mw_get_addr = intel_ntb_peer_mw_get_addr, + .link_is_up = intel_ntb_link_is_up, + .link_enable = intel_ntb3_link_enable, + .link_disable = intel_ntb_link_disable, + .db_valid_mask = intel_ntb_db_valid_mask, + .db_vector_count = intel_ntb_db_vector_count, + .db_vector_mask = intel_ntb_db_vector_mask, + .db_read = intel_ntb3_db_read, + .db_clear = intel_ntb3_db_clear, + .db_set_mask = intel_ntb_db_set_mask, + .db_clear_mask = intel_ntb_db_clear_mask, + .peer_db_addr = intel_ntb_peer_db_addr, + .peer_db_set = intel_ntb3_peer_db_set, + .spad_is_unsafe = intel_ntb_spad_is_unsafe, + .spad_count = intel_ntb_spad_count, + .spad_read = intel_ntb_spad_read, + .spad_write = intel_ntb_spad_write, + .peer_spad_addr = intel_ntb_peer_spad_addr, + .peer_spad_read = intel_ntb_peer_spad_read, + .peer_spad_write = intel_ntb_peer_spad_write, +}; + diff --git a/drivers/ntb/hw/intel/ntb_hw_gen3.h b/drivers/ntb/hw/intel/ntb_hw_gen3.h new file mode 100644 index 000000000000..75fb86ca27bb --- /dev/null +++ b/drivers/ntb/hw/intel/ntb_hw_gen3.h @@ -0,0 +1,110 @@ +/* + * This file is provided under a dual BSD/GPLv2 license. When using or + * redistributing this file, you may do so under either license. + * + * GPL LICENSE SUMMARY + * + * Copyright(c) 2012-2017 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * BSD LICENSE + * + * Copyright(c) 2012-2017 Intel Corporation. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copy + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Intel Corporation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef _NTB_INTEL_GEN3_H_ +#define _NTB_INTEL_GEN3_H_ + +#include "ntb_hw_intel.h" + +/* Intel Skylake Xeon hardware */ +#define GEN3_IMBAR1SZ_OFFSET 0x00d0 +#define GEN3_IMBAR2SZ_OFFSET 0x00d1 +#define GEN3_EMBAR1SZ_OFFSET 0x00d2 +#define GEN3_EMBAR2SZ_OFFSET 0x00d3 +#define GEN3_DEVCTRL_OFFSET 0x0098 +#define GEN3_DEVSTS_OFFSET 0x009a +#define GEN3_UNCERRSTS_OFFSET 0x014c +#define GEN3_CORERRSTS_OFFSET 0x0158 +#define GEN3_LINK_STATUS_OFFSET 0x01a2 + +#define GEN3_NTBCNTL_OFFSET 0x0000 +#define GEN3_IMBAR1XBASE_OFFSET 0x0010 /* SBAR2XLAT */ +#define GEN3_IMBAR1XLMT_OFFSET 0x0018 /* SBAR2LMT */ +#define GEN3_IMBAR2XBASE_OFFSET 0x0020 /* SBAR4XLAT */ +#define GEN3_IMBAR2XLMT_OFFSET 0x0028 /* SBAR4LMT */ +#define GEN3_IM_INT_STATUS_OFFSET 0x0040 +#define GEN3_IM_INT_DISABLE_OFFSET 0x0048 +#define GEN3_IM_SPAD_OFFSET 0x0080 /* SPAD */ +#define GEN3_USMEMMISS_OFFSET 0x0070 +#define GEN3_INTVEC_OFFSET 0x00d0 +#define GEN3_IM_DOORBELL_OFFSET 0x0100 /* SDOORBELL0 */ +#define GEN3_B2B_SPAD_OFFSET 0x0180 /* B2B SPAD */ +#define GEN3_EMBAR0XBASE_OFFSET 0x4008 /* B2B_XLAT */ +#define GEN3_EMBAR1XBASE_OFFSET 0x4010 /* PBAR2XLAT */ +#define GEN3_EMBAR1XLMT_OFFSET 0x4018 /* PBAR2LMT */ +#define GEN3_EMBAR2XBASE_OFFSET 0x4020 /* PBAR4XLAT */ +#define GEN3_EMBAR2XLMT_OFFSET 0x4028 /* PBAR4LMT */ +#define GEN3_EM_INT_STATUS_OFFSET 0x4040 +#define GEN3_EM_INT_DISABLE_OFFSET 0x4048 +#define GEN3_EM_SPAD_OFFSET 0x4080 /* remote SPAD */ +#define GEN3_EM_DOORBELL_OFFSET 0x4100 /* PDOORBELL0 */ +#define GEN3_SPCICMD_OFFSET 0x4504 /* SPCICMD */ +#define GEN3_EMBAR0_OFFSET 0x4510 /* SBAR0BASE */ +#define GEN3_EMBAR1_OFFSET 0x4518 /* SBAR23BASE */ +#define GEN3_EMBAR2_OFFSET 0x4520 /* SBAR45BASE */ + +#define GEN3_DB_COUNT 32 +#define GEN3_DB_LINK 32 +#define GEN3_DB_LINK_BIT BIT_ULL(GEN3_DB_LINK) +#define GEN3_DB_MSIX_VECTOR_COUNT 33 +#define GEN3_DB_MSIX_VECTOR_SHIFT 1 +#define GEN3_DB_TOTAL_SHIFT 33 +#define GEN3_SPAD_COUNT 16 + +static inline u64 gen3_db_ioread(void __iomem *mmio) +{ + return ioread64(mmio); +} + +static inline void gen3_db_iowrite(u64 bits, void __iomem *mmio) +{ + iowrite64(bits, mmio); +} + +ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf, + size_t count, loff_t *offp); +int gen3_init_dev(struct intel_ntb_dev *ndev); + +extern const struct ntb_dev_ops intel_ntb3_ops; + +#endif diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.h b/drivers/ntb/hw/intel/ntb_hw_intel.h index 4415aa7ea775..c49ff8970ce3 100644 --- a/drivers/ntb/hw/intel/ntb_hw_intel.h +++ b/drivers/ntb/hw/intel/ntb_hw_intel.h @@ -54,6 +54,7 @@ #include <linux/ntb.h> #include <linux/pci.h> +/* PCI device IDs */ #define PCI_DEVICE_ID_INTEL_NTB_B2B_JSF 0x3725 #define PCI_DEVICE_ID_INTEL_NTB_PS_JSF 0x3726 #define PCI_DEVICE_ID_INTEL_NTB_SS_JSF 0x3727 @@ -71,132 +72,7 @@ #define PCI_DEVICE_ID_INTEL_NTB_SS_BDX 0x6F0F #define PCI_DEVICE_ID_INTEL_NTB_B2B_SKX 0x201C -/* Intel Xeon hardware */ - -#define XEON_PBAR23LMT_OFFSET 0x0000 -#define XEON_PBAR45LMT_OFFSET 0x0008 -#define XEON_PBAR4LMT_OFFSET 0x0008 -#define XEON_PBAR5LMT_OFFSET 0x000c -#define XEON_PBAR23XLAT_OFFSET 0x0010 -#define XEON_PBAR45XLAT_OFFSET 0x0018 -#define XEON_PBAR4XLAT_OFFSET 0x0018 -#define XEON_PBAR5XLAT_OFFSET 0x001c -#define XEON_SBAR23LMT_OFFSET 0x0020 -#define XEON_SBAR45LMT_OFFSET 0x0028 -#define XEON_SBAR4LMT_OFFSET 0x0028 -#define XEON_SBAR5LMT_OFFSET 0x002c -#define XEON_SBAR23XLAT_OFFSET 0x0030 -#define XEON_SBAR45XLAT_OFFSET 0x0038 -#define XEON_SBAR4XLAT_OFFSET 0x0038 -#define XEON_SBAR5XLAT_OFFSET 0x003c -#define XEON_SBAR0BASE_OFFSET 0x0040 -#define XEON_SBAR23BASE_OFFSET 0x0048 -#define XEON_SBAR45BASE_OFFSET 0x0050 -#define XEON_SBAR4BASE_OFFSET 0x0050 -#define XEON_SBAR5BASE_OFFSET 0x0054 -#define XEON_SBDF_OFFSET 0x005c -#define XEON_NTBCNTL_OFFSET 0x0058 -#define XEON_PDOORBELL_OFFSET 0x0060 -#define XEON_PDBMSK_OFFSET 0x0062 -#define XEON_SDOORBELL_OFFSET 0x0064 -#define XEON_SDBMSK_OFFSET 0x0066 -#define XEON_USMEMMISS_OFFSET 0x0070 -#define XEON_SPAD_OFFSET 0x0080 -#define XEON_PBAR23SZ_OFFSET 0x00d0 -#define XEON_PBAR45SZ_OFFSET 0x00d1 -#define XEON_PBAR4SZ_OFFSET 0x00d1 -#define XEON_SBAR23SZ_OFFSET 0x00d2 -#define XEON_SBAR45SZ_OFFSET 0x00d3 -#define XEON_SBAR4SZ_OFFSET 0x00d3 -#define XEON_PPD_OFFSET 0x00d4 -#define XEON_PBAR5SZ_OFFSET 0x00d5 -#define XEON_SBAR5SZ_OFFSET 0x00d6 -#define XEON_WCCNTRL_OFFSET 0x00e0 -#define XEON_UNCERRSTS_OFFSET 0x014c -#define XEON_CORERRSTS_OFFSET 0x0158 -#define XEON_LINK_STATUS_OFFSET 0x01a2 -#define XEON_SPCICMD_OFFSET 0x0504 -#define XEON_DEVCTRL_OFFSET 0x0598 -#define XEON_DEVSTS_OFFSET 0x059a -#define XEON_SLINK_STATUS_OFFSET 0x05a2 -#define XEON_B2B_SPAD_OFFSET 0x0100 -#define XEON_B2B_DOORBELL_OFFSET 0x0140 -#define XEON_B2B_XLAT_OFFSETL 0x0144 -#define XEON_B2B_XLAT_OFFSETU 0x0148 -#define XEON_PPD_CONN_MASK 0x03 -#define XEON_PPD_CONN_TRANSPARENT 0x00 -#define XEON_PPD_CONN_B2B 0x01 -#define XEON_PPD_CONN_RP 0x02 -#define XEON_PPD_DEV_MASK 0x10 -#define XEON_PPD_DEV_USD 0x00 -#define XEON_PPD_DEV_DSD 0x10 -#define XEON_PPD_SPLIT_BAR_MASK 0x40 - -#define XEON_PPD_TOPO_MASK (XEON_PPD_CONN_MASK | XEON_PPD_DEV_MASK) -#define XEON_PPD_TOPO_PRI_USD (XEON_PPD_CONN_RP | XEON_PPD_DEV_USD) -#define XEON_PPD_TOPO_PRI_DSD (XEON_PPD_CONN_RP | XEON_PPD_DEV_DSD) -#define XEON_PPD_TOPO_SEC_USD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_USD) -#define XEON_PPD_TOPO_SEC_DSD (XEON_PPD_CONN_TRANSPARENT | XEON_PPD_DEV_DSD) -#define XEON_PPD_TOPO_B2B_USD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_USD) -#define XEON_PPD_TOPO_B2B_DSD (XEON_PPD_CONN_B2B | XEON_PPD_DEV_DSD) - -#define XEON_MW_COUNT 2 -#define HSX_SPLIT_BAR_MW_COUNT 3 -#define XEON_DB_COUNT 15 -#define XEON_DB_LINK 15 -#define XEON_DB_LINK_BIT BIT_ULL(XEON_DB_LINK) -#define XEON_DB_MSIX_VECTOR_COUNT 4 -#define XEON_DB_MSIX_VECTOR_SHIFT 5 -#define XEON_DB_TOTAL_SHIFT 16 -#define XEON_SPAD_COUNT 16 - -/* Intel Skylake Xeon hardware */ -#define SKX_IMBAR1SZ_OFFSET 0x00d0 -#define SKX_IMBAR2SZ_OFFSET 0x00d1 -#define SKX_EMBAR1SZ_OFFSET 0x00d2 -#define SKX_EMBAR2SZ_OFFSET 0x00d3 -#define SKX_DEVCTRL_OFFSET 0x0098 -#define SKX_DEVSTS_OFFSET 0x009a -#define SKX_UNCERRSTS_OFFSET 0x014c -#define SKX_CORERRSTS_OFFSET 0x0158 -#define SKX_LINK_STATUS_OFFSET 0x01a2 - -#define SKX_NTBCNTL_OFFSET 0x0000 -#define SKX_IMBAR1XBASE_OFFSET 0x0010 /* SBAR2XLAT */ -#define SKX_IMBAR1XLMT_OFFSET 0x0018 /* SBAR2LMT */ -#define SKX_IMBAR2XBASE_OFFSET 0x0020 /* SBAR4XLAT */ -#define SKX_IMBAR2XLMT_OFFSET 0x0028 /* SBAR4LMT */ -#define SKX_IM_INT_STATUS_OFFSET 0x0040 -#define SKX_IM_INT_DISABLE_OFFSET 0x0048 -#define SKX_IM_SPAD_OFFSET 0x0080 /* SPAD */ -#define SKX_USMEMMISS_OFFSET 0x0070 -#define SKX_INTVEC_OFFSET 0x00d0 -#define SKX_IM_DOORBELL_OFFSET 0x0100 /* SDOORBELL0 */ -#define SKX_B2B_SPAD_OFFSET 0x0180 /* B2B SPAD */ -#define SKX_EMBAR0XBASE_OFFSET 0x4008 /* B2B_XLAT */ -#define SKX_EMBAR1XBASE_OFFSET 0x4010 /* PBAR2XLAT */ -#define SKX_EMBAR1XLMT_OFFSET 0x4018 /* PBAR2LMT */ -#define SKX_EMBAR2XBASE_OFFSET 0x4020 /* PBAR4XLAT */ -#define SKX_EMBAR2XLMT_OFFSET 0x4028 /* PBAR4LMT */ -#define SKX_EM_INT_STATUS_OFFSET 0x4040 -#define SKX_EM_INT_DISABLE_OFFSET 0x4048 -#define SKX_EM_SPAD_OFFSET 0x4080 /* remote SPAD */ -#define SKX_EM_DOORBELL_OFFSET 0x4100 /* PDOORBELL0 */ -#define SKX_SPCICMD_OFFSET 0x4504 /* SPCICMD */ -#define SKX_EMBAR0_OFFSET 0x4510 /* SBAR0BASE */ -#define SKX_EMBAR1_OFFSET 0x4518 /* SBAR23BASE */ -#define SKX_EMBAR2_OFFSET 0x4520 /* SBAR45BASE */ - -#define SKX_DB_COUNT 32 -#define SKX_DB_LINK 32 -#define SKX_DB_LINK_BIT BIT_ULL(SKX_DB_LINK) -#define SKX_DB_MSIX_VECTOR_COUNT 33 -#define SKX_DB_MSIX_VECTOR_SHIFT 1 -#define SKX_DB_TOTAL_SHIFT 33 -#define SKX_SPAD_COUNT 16 - /* Ntb control and link status */ - #define NTB_CTL_CFG_LOCK BIT(0) #define NTB_CTL_DISABLE BIT(1) #define NTB_CTL_S2P_BAR2_SNOOP BIT(2) @@ -213,23 +89,6 @@ #define NTB_LNK_STA_SPEED(x) ((x) & NTB_LNK_STA_SPEED_MASK) #define NTB_LNK_STA_WIDTH(x) (((x) & NTB_LNK_STA_WIDTH_MASK) >> 4) -/* Use the following addresses for translation between b2b ntb devices in case - * the hardware default values are not reliable. */ -#define XEON_B2B_BAR0_ADDR 0x1000000000000000ull -#define XEON_B2B_BAR2_ADDR64 0x2000000000000000ull -#define XEON_B2B_BAR4_ADDR64 0x4000000000000000ull -#define XEON_B2B_BAR4_ADDR32 0x20000000u -#define XEON_B2B_BAR5_ADDR32 0x40000000u - -/* The peer ntb secondary config space is 32KB fixed size */ -#define XEON_B2B_MIN_SIZE 0x8000 - -/* flags to indicate hardware errata */ -#define NTB_HWERR_SDOORBELL_LOCKUP BIT_ULL(0) -#define NTB_HWERR_SB01BASE_LOCKUP BIT_ULL(1) -#define NTB_HWERR_B2BDOORBELL_BIT14 BIT_ULL(2) -#define NTB_HWERR_MSIX_VECTOR32_BAD BIT_ULL(3) - /* flags to indicate unsafe api */ #define NTB_UNSAFE_DB BIT_ULL(0) #define NTB_UNSAFE_SPAD BIT_ULL(1) @@ -328,4 +187,64 @@ struct intel_ntb_dev { #define hb_ndev(__work) container_of(__work, struct intel_ntb_dev, \ hb_timer.work) +static inline int pdev_is_gen1(struct pci_dev *pdev) +{ + switch (pdev->device) { + case PCI_DEVICE_ID_INTEL_NTB_SS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_SS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_SS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_SS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_SS_BDX: + case PCI_DEVICE_ID_INTEL_NTB_PS_JSF: + case PCI_DEVICE_ID_INTEL_NTB_PS_SNB: + case PCI_DEVICE_ID_INTEL_NTB_PS_IVT: + case PCI_DEVICE_ID_INTEL_NTB_PS_HSX: + case PCI_DEVICE_ID_INTEL_NTB_PS_BDX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF: + case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB: + case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT: + case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX: + case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX: + return 1; + } + return 0; +} + +static inline int pdev_is_gen3(struct pci_dev *pdev) +{ + if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX) + return 1; + + return 0; +} + +#ifndef ioread64 +#ifdef readq +#define ioread64 readq +#else +#define ioread64 _ioread64 +static inline u64 _ioread64(void __iomem *mmio) +{ + u64 low, high; + + low = ioread32(mmio); + high = ioread32(mmio + sizeof(u32)); + return low | (high << 32); +} +#endif +#endif + +#ifndef iowrite64 +#ifdef writeq +#define iowrite64 writeq +#else +#define iowrite64 _iowrite64 +static inline void _iowrite64(u64 val, void __iomem *mmio) +{ + iowrite32(val, mmio); + iowrite32(val >> 32, mmio + sizeof(u32)); +} +#endif +#endif + #endif diff --git a/drivers/ntb/ntb_transport.c b/drivers/ntb/ntb_transport.c index 504bdcc57ae8..939895966476 100644 --- a/drivers/ntb/ntb_transport.c +++ b/drivers/ntb/ntb_transport.c @@ -637,7 +637,7 @@ static int ntb_transport_setup_qp_mw(struct ntb_transport_ctx *nt, */ node = dev_to_node(&ndev->dev); for (i = qp->rx_alloc_entry; i < qp->rx_max_entry; i++) { - entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); if (!entry) return -ENOMEM; @@ -1828,7 +1828,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev, qp->rx_dma_chan ? "DMA" : "CPU"); for (i = 0; i < NTB_QP_DEF_NUM_ENTRIES; i++) { - entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); if (!entry) goto err1; @@ -1839,7 +1839,7 @@ ntb_transport_create_queue(void *data, struct device *client_dev, qp->rx_alloc_entry = NTB_QP_DEF_NUM_ENTRIES; for (i = 0; i < qp->tx_max_entry; i++) { - entry = kzalloc_node(sizeof(*entry), GFP_ATOMIC, node); + entry = kzalloc_node(sizeof(*entry), GFP_KERNEL, node); if (!entry) goto err2; |