summaryrefslogtreecommitdiff
path: root/drivers/ntb/hw/intel/ntb_hw_intel.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/ntb/hw/intel/ntb_hw_intel.c')
-rw-r--r--drivers/ntb/hw/intel/ntb_hw_intel.c3073
1 files changed, 1647 insertions, 1426 deletions
diff --git a/drivers/ntb/hw/intel/ntb_hw_intel.c b/drivers/ntb/hw/intel/ntb_hw_intel.c
index 044534a995f1..686091756ba4 100644
--- a/drivers/ntb/hw/intel/ntb_hw_intel.c
+++ b/drivers/ntb/hw/intel/ntb_hw_intel.c
@@ -5,6 +5,7 @@
* GPL LICENSE SUMMARY
*
* Copyright(c) 2012 Intel Corporation. All rights reserved.
+ * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
@@ -13,6 +14,7 @@
* BSD LICENSE
*
* Copyright(c) 2012 Intel Corporation. All rights reserved.
+ * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
@@ -45,6 +47,7 @@
* Contact Information:
* Jon Mason <jon.mason@intel.com>
*/
+
#include <linux/debugfs.h>
#include <linux/delay.h>
#include <linux/init.h>
@@ -53,99 +56,97 @@
#include <linux/pci.h>
#include <linux/random.h>
#include <linux/slab.h>
+#include <linux/ntb.h>
+
#include "ntb_hw_intel.h"
-#define NTB_NAME "Intel(R) PCI-E Non-Transparent Bridge Driver"
-#define NTB_VER "1.0"
+#define NTB_NAME "ntb_hw_intel"
+#define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
+#define NTB_VER "2.0"
-MODULE_DESCRIPTION(NTB_NAME);
+MODULE_DESCRIPTION(NTB_DESC);
MODULE_VERSION(NTB_VER);
MODULE_LICENSE("Dual BSD/GPL");
MODULE_AUTHOR("Intel Corporation");
-enum {
- NTB_CONN_TRANSPARENT = 0,
- NTB_CONN_B2B,
- NTB_CONN_RP,
-};
-
-enum {
- NTB_DEV_USD = 0,
- NTB_DEV_DSD,
-};
-
-enum {
- SNB_HW = 0,
- BWD_HW,
-};
-
+#define bar0_off(base, bar) ((base) + ((bar) << 2))
+#define bar2_off(base, bar) bar0_off(base, (bar) - 2)
+
+static int b2b_mw_idx = -1;
+module_param(b2b_mw_idx, int, 0644);
+MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
+ "value of zero or positive starts from first mw idx, and a "
+ "negative value starts from last mw idx. Both sides MUST "
+ "set the same value here!");
+
+static unsigned int b2b_mw_share;
+module_param(b2b_mw_share, uint, 0644);
+MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
+ "ntb so that the peer ntb only occupies the first half of "
+ "the mw, so the second half can still be used as a mw. Both "
+ "sides MUST set the same value here!");
+
+static const struct intel_ntb_reg bwd_reg;
+static const struct intel_ntb_alt_reg bwd_pri_reg;
+static const struct intel_ntb_alt_reg bwd_sec_reg;
+static const struct intel_ntb_alt_reg bwd_b2b_reg;
+static const struct intel_ntb_xlat_reg bwd_pri_xlat;
+static const struct intel_ntb_xlat_reg bwd_sec_xlat;
+static const struct intel_ntb_reg snb_reg;
+static const struct intel_ntb_alt_reg snb_pri_reg;
+static const struct intel_ntb_alt_reg snb_sec_reg;
+static const struct intel_ntb_alt_reg snb_b2b_reg;
+static const struct intel_ntb_xlat_reg snb_pri_xlat;
+static const struct intel_ntb_xlat_reg snb_sec_xlat;
+static const struct intel_b2b_addr snb_b2b_usd_addr;
+static const struct intel_b2b_addr snb_b2b_dsd_addr;
+
+static const struct ntb_dev_ops intel_ntb_ops;
+
+static const struct file_operations intel_ntb_debugfs_info;
static struct dentry *debugfs_dir;
-#define BWD_LINK_RECOVERY_TIME 500
-
-/* Translate memory window 0,1,2 to BAR 2,4,5 */
-#define MW_TO_BAR(mw) (mw == 0 ? 2 : (mw == 1 ? 4 : 5))
-
-static const struct pci_device_id ntb_pci_tbl[] = {
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
- {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
- {0}
-};
-MODULE_DEVICE_TABLE(pci, ntb_pci_tbl);
-
-static int is_ntb_xeon(struct ntb_device *ndev)
+#ifndef ioread64
+#ifdef readq
+#define ioread64 readq
+#else
+#define ioread64 _ioread64
+static inline u64 _ioread64(void __iomem *mmio)
{
- switch (ndev->pdev->device) {
- case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
- case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
- case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
- case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
- case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
- case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
- case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
- case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
- case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
- case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
- case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
- case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
- return 1;
- default:
- return 0;
- }
+ u64 low, high;
- return 0;
+ low = ioread32(mmio);
+ high = ioread32(mmio + sizeof(u32));
+ return low | (high << 32);
+}
+#endif
+#endif
+
+#ifndef iowrite64
+#ifdef writeq
+#define iowrite64 writeq
+#else
+#define iowrite64 _iowrite64
+static inline void _iowrite64(u64 val, void __iomem *mmio)
+{
+ iowrite32(val, mmio);
+ iowrite32(val >> 32, mmio + sizeof(u32));
}
+#endif
+#endif
-static int is_ntb_atom(struct ntb_device *ndev)
+static inline int pdev_is_bwd(struct pci_dev *pdev)
{
- switch (ndev->pdev->device) {
+ switch (pdev->device) {
case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
return 1;
- default:
- return 0;
}
-
return 0;
}
-static void ntb_set_errata_flags(struct ntb_device *ndev)
+static inline int pdev_is_snb(struct pci_dev *pdev)
{
- switch (ndev->pdev->device) {
- /*
- * this workaround applies to all platform up to IvyBridge
- * Haswell has splitbar support and use a different workaround
- */
+ switch (pdev->device) {
case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
@@ -158,1737 +159,1957 @@ static void ntb_set_errata_flags(struct ntb_device *ndev)
case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
- ndev->wa_flags |= WA_SNB_ERR;
- break;
+ return 1;
}
+ return 0;
}
-/**
- * ntb_register_event_callback() - register event callback
- * @ndev: pointer to ntb_device instance
- * @func: callback function to register
- *
- * This function registers a callback for any HW driver events such as link
- * up/down, power management notices and etc.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_register_event_callback(struct ntb_device *ndev,
- void (*func)(void *handle,
- enum ntb_hw_event event))
+static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
{
- if (ndev->event_cb)
- return -EINVAL;
-
- ndev->event_cb = func;
-
- return 0;
+ ndev->unsafe_flags = 0;
+ ndev->unsafe_flags_ignore = 0;
+
+ /* Only B2B has a workaround to avoid SDOORBELL */
+ if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
+ if (!ntb_topo_is_b2b(ndev->ntb.topo))
+ ndev->unsafe_flags |= NTB_UNSAFE_DB;
+
+ /* No low level workaround to avoid SB01BASE */
+ if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
+ ndev->unsafe_flags |= NTB_UNSAFE_DB;
+ ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
+ }
}
-/**
- * ntb_unregister_event_callback() - unregisters the event callback
- * @ndev: pointer to ntb_device instance
- *
- * This function unregisters the existing callback from transport
- */
-void ntb_unregister_event_callback(struct ntb_device *ndev)
+static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
+ unsigned long flag)
{
- ndev->event_cb = NULL;
+ return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
}
-static void ntb_irq_work(unsigned long data)
+static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
+ unsigned long flag)
{
- struct ntb_db_cb *db_cb = (struct ntb_db_cb *)data;
- int rc;
+ flag &= ndev->unsafe_flags;
+ ndev->unsafe_flags_ignore |= flag;
- rc = db_cb->callback(db_cb->data, db_cb->db_num);
- if (rc)
- tasklet_schedule(&db_cb->irq_work);
- else {
- struct ntb_device *ndev = db_cb->ndev;
- unsigned long mask;
-
- mask = readw(ndev->reg_ofs.ldb_mask);
- clear_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
- writew(mask, ndev->reg_ofs.ldb_mask);
- }
+ return !!flag;
}
-/**
- * ntb_register_db_callback() - register a callback for doorbell interrupt
- * @ndev: pointer to ntb_device instance
- * @idx: doorbell index to register callback, zero based
- * @data: pointer to be returned to caller with every callback
- * @func: callback function to register
- *
- * This function registers a callback function for the doorbell interrupt
- * on the primary side. The function will unmask the doorbell as well to
- * allow interrupt.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_register_db_callback(struct ntb_device *ndev, unsigned int idx,
- void *data, int (*func)(void *data, int db_num))
+static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
{
- unsigned long mask;
-
- if (idx >= ndev->max_cbs || ndev->db_cb[idx].callback) {
- dev_warn(&ndev->pdev->dev, "Invalid Index.\n");
+ if (idx < 0 || idx > ndev->mw_count)
return -EINVAL;
- }
+ return ndev->reg->mw_bar[idx];
+}
- ndev->db_cb[idx].callback = func;
- ndev->db_cb[idx].data = data;
- ndev->db_cb[idx].ndev = ndev;
+static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
+ phys_addr_t *db_addr, resource_size_t *db_size,
+ phys_addr_t reg_addr, unsigned long reg)
+{
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
- tasklet_init(&ndev->db_cb[idx].irq_work, ntb_irq_work,
- (unsigned long) &ndev->db_cb[idx]);
+ if (db_addr) {
+ *db_addr = reg_addr + reg;
+ dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
+ }
- /* unmask interrupt */
- mask = readw(ndev->reg_ofs.ldb_mask);
- clear_bit(idx * ndev->bits_per_vector, &mask);
- writew(mask, ndev->reg_ofs.ldb_mask);
+ if (db_size) {
+ *db_size = ndev->reg->db_size;
+ dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
+ }
return 0;
}
-/**
- * ntb_unregister_db_callback() - unregister a callback for doorbell interrupt
- * @ndev: pointer to ntb_device instance
- * @idx: doorbell index to register callback, zero based
- *
- * This function unregisters a callback function for the doorbell interrupt
- * on the primary side. The function will also mask the said doorbell.
- */
-void ntb_unregister_db_callback(struct ntb_device *ndev, unsigned int idx)
+static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
+ void __iomem *mmio)
{
- unsigned long mask;
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
- if (idx >= ndev->max_cbs || !ndev->db_cb[idx].callback)
- return;
+ return ndev->reg->db_ioread(mmio);
+}
- mask = readw(ndev->reg_ofs.ldb_mask);
- set_bit(idx * ndev->bits_per_vector, &mask);
- writew(mask, ndev->reg_ofs.ldb_mask);
+static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
+ void __iomem *mmio)
+{
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
- tasklet_disable(&ndev->db_cb[idx].irq_work);
+ if (db_bits & ~ndev->db_valid_mask)
+ return -EINVAL;
- ndev->db_cb[idx].callback = NULL;
-}
+ ndev->reg->db_iowrite(db_bits, mmio);
-/**
- * ntb_find_transport() - find the transport pointer
- * @transport: pointer to pci device
- *
- * Given the pci device pointer, return the transport pointer passed in when
- * the transport attached when it was inited.
- *
- * RETURNS: pointer to transport.
- */
-void *ntb_find_transport(struct pci_dev *pdev)
-{
- struct ntb_device *ndev = pci_get_drvdata(pdev);
- return ndev->ntb_transport;
+ return 0;
}
-/**
- * ntb_register_transport() - Register NTB transport with NTB HW driver
- * @transport: transport identifier
- *
- * This function allows a transport to reserve the hardware driver for
- * NTB usage.
- *
- * RETURNS: pointer to ntb_device, NULL on error.
- */
-struct ntb_device *ntb_register_transport(struct pci_dev *pdev, void *transport)
+static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
+ void __iomem *mmio)
{
- struct ntb_device *ndev = pci_get_drvdata(pdev);
+ unsigned long irqflags;
- if (ndev->ntb_transport)
- return NULL;
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
- ndev->ntb_transport = transport;
- return ndev;
+ if (db_bits & ~ndev->db_valid_mask)
+ return -EINVAL;
+
+ spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
+ {
+ ndev->db_mask |= db_bits;
+ ndev->reg->db_iowrite(ndev->db_mask, mmio);
+ }
+ spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
+
+ return 0;
}
-/**
- * ntb_unregister_transport() - Unregister the transport with the NTB HW driver
- * @ndev - ntb_device of the transport to be freed
- *
- * This function unregisters the transport from the HW driver and performs any
- * necessary cleanups.
- */
-void ntb_unregister_transport(struct ntb_device *ndev)
+static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
+ void __iomem *mmio)
{
- int i;
+ unsigned long irqflags;
- if (!ndev->ntb_transport)
- return;
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_DB));
+
+ if (db_bits & ~ndev->db_valid_mask)
+ return -EINVAL;
- for (i = 0; i < ndev->max_cbs; i++)
- ntb_unregister_db_callback(ndev, i);
+ spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
+ {
+ ndev->db_mask &= ~db_bits;
+ ndev->reg->db_iowrite(ndev->db_mask, mmio);
+ }
+ spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
- ntb_unregister_event_callback(ndev);
- ndev->ntb_transport = NULL;
+ return 0;
}
-/**
- * ntb_write_local_spad() - write to the secondary scratchpad register
- * @ndev: pointer to ntb_device instance
- * @idx: index to the scratchpad register, 0 based
- * @val: the data value to put into the register
- *
- * This function allows writing of a 32bit value to the indexed scratchpad
- * register. This writes over the data mirrored to the local scratchpad register
- * by the remote system.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_write_local_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
+static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
{
- if (idx >= ndev->limits.max_spads)
- return -EINVAL;
+ u64 shift, mask;
- dev_dbg(&ndev->pdev->dev, "Writing %x to local scratch pad index %d\n",
- val, idx);
- writel(val, ndev->reg_ofs.spad_read + idx * 4);
+ shift = ndev->db_vec_shift;
+ mask = BIT_ULL(shift) - 1;
- return 0;
+ return mask << (shift * db_vector);
}
-/**
- * ntb_read_local_spad() - read from the primary scratchpad register
- * @ndev: pointer to ntb_device instance
- * @idx: index to scratchpad register, 0 based
- * @val: pointer to 32bit integer for storing the register value
- *
- * This function allows reading of the 32bit scratchpad register on
- * the primary (internal) side. This allows the local system to read data
- * written and mirrored to the scratchpad register by the remote system.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_read_local_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
+static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
+ phys_addr_t *spad_addr, phys_addr_t reg_addr,
+ unsigned long reg)
{
- if (idx >= ndev->limits.max_spads)
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
+
+ if (idx < 0 || idx >= ndev->spad_count)
return -EINVAL;
- *val = readl(ndev->reg_ofs.spad_write + idx * 4);
- dev_dbg(&ndev->pdev->dev,
- "Reading %x from local scratch pad index %d\n", *val, idx);
+ if (spad_addr) {
+ *spad_addr = reg_addr + reg + (idx << 2);
+ dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
+ }
return 0;
}
-/**
- * ntb_write_remote_spad() - write to the secondary scratchpad register
- * @ndev: pointer to ntb_device instance
- * @idx: index to the scratchpad register, 0 based
- * @val: the data value to put into the register
- *
- * This function allows writing of a 32bit value to the indexed scratchpad
- * register. The register resides on the secondary (external) side. This allows
- * the local system to write data to be mirrored to the remote systems
- * scratchpad register.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_write_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 val)
+static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
+ void __iomem *mmio)
{
- if (idx >= ndev->limits.max_spads)
- return -EINVAL;
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
- dev_dbg(&ndev->pdev->dev, "Writing %x to remote scratch pad index %d\n",
- val, idx);
- writel(val, ndev->reg_ofs.spad_write + idx * 4);
+ if (idx < 0 || idx >= ndev->spad_count)
+ return 0;
- return 0;
+ return ioread32(mmio + (idx << 2));
}
-/**
- * ntb_read_remote_spad() - read from the primary scratchpad register
- * @ndev: pointer to ntb_device instance
- * @idx: index to scratchpad register, 0 based
- * @val: pointer to 32bit integer for storing the register value
- *
- * This function allows reading of the 32bit scratchpad register on
- * the primary (internal) side. This alloows the local system to read the data
- * it wrote to be mirrored on the remote system.
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-int ntb_read_remote_spad(struct ntb_device *ndev, unsigned int idx, u32 *val)
+static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
+ void __iomem *mmio)
{
- if (idx >= ndev->limits.max_spads)
+ WARN_ON_ONCE(ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD));
+
+ if (idx < 0 || idx >= ndev->spad_count)
return -EINVAL;
- *val = readl(ndev->reg_ofs.spad_read + idx * 4);
- dev_dbg(&ndev->pdev->dev,
- "Reading %x from remote scratch pad index %d\n", *val, idx);
+ iowrite32(val, mmio + (idx << 2));
return 0;
}
-/**
- * ntb_get_mw_base() - get addr for the NTB memory window
- * @ndev: pointer to ntb_device instance
- * @mw: memory window number
- *
- * This function provides the base address of the memory window specified.
- *
- * RETURNS: address, or NULL on error.
- */
-resource_size_t ntb_get_mw_base(struct ntb_device *ndev, unsigned int mw)
+static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
{
- if (mw >= ntb_max_mw(ndev))
- return 0;
+ u64 vec_mask;
+
+ vec_mask = ndev_vec_mask(ndev, vec);
+
+ dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
+
+ ndev->last_ts = jiffies;
- return pci_resource_start(ndev->pdev, MW_TO_BAR(mw));
+ if (vec_mask & ndev->db_link_mask) {
+ if (ndev->reg->poll_link(ndev))
+ ntb_link_event(&ndev->ntb);
+ }
+
+ if (vec_mask & ndev->db_valid_mask)
+ ntb_db_event(&ndev->ntb, vec);
+
+ return IRQ_HANDLED;
}
-/**
- * ntb_get_mw_vbase() - get virtual addr for the NTB memory window
- * @ndev: pointer to ntb_device instance
- * @mw: memory window number
- *
- * This function provides the base virtual address of the memory window
- * specified.
- *
- * RETURNS: pointer to virtual address, or NULL on error.
- */
-void __iomem *ntb_get_mw_vbase(struct ntb_device *ndev, unsigned int mw)
+static irqreturn_t ndev_vec_isr(int irq, void *dev)
{
- if (mw >= ntb_max_mw(ndev))
- return NULL;
+ struct intel_ntb_vec *nvec = dev;
- return ndev->mw[mw].vbase;
+ return ndev_interrupt(nvec->ndev, nvec->num);
}
-/**
- * ntb_get_mw_size() - return size of NTB memory window
- * @ndev: pointer to ntb_device instance
- * @mw: memory window number
- *
- * This function provides the physical size of the memory window specified
- *
- * RETURNS: the size of the memory window or zero on error
- */
-u64 ntb_get_mw_size(struct ntb_device *ndev, unsigned int mw)
+static irqreturn_t ndev_irq_isr(int irq, void *dev)
{
- if (mw >= ntb_max_mw(ndev))
- return 0;
+ struct intel_ntb_dev *ndev = dev;
- return ndev->mw[mw].bar_sz;
+ return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
}
-/**
- * ntb_set_mw_addr - set the memory window address
- * @ndev: pointer to ntb_device instance
- * @mw: memory window number
- * @addr: base address for data
- *
- * This function sets the base physical address of the memory window. This
- * memory address is where data from the remote system will be transfered into
- * or out of depending on how the transport is configured.
- */
-void ntb_set_mw_addr(struct ntb_device *ndev, unsigned int mw, u64 addr)
+static int ndev_init_isr(struct intel_ntb_dev *ndev,
+ int msix_min, int msix_max,
+ int msix_shift, int total_shift)
{
- if (mw >= ntb_max_mw(ndev))
- return;
+ struct pci_dev *pdev;
+ int rc, i, msix_count;
- dev_dbg(&ndev->pdev->dev, "Writing addr %Lx to BAR %d\n", addr,
- MW_TO_BAR(mw));
+ pdev = ndev_pdev(ndev);
- ndev->mw[mw].phys_addr = addr;
+ /* Mask all doorbell interrupts */
+ ndev->db_mask = ndev->db_valid_mask;
+ ndev->reg->db_iowrite(ndev->db_mask,
+ ndev->self_mmio +
+ ndev->self_reg->db_mask);
- switch (MW_TO_BAR(mw)) {
- case NTB_BAR_23:
- writeq(addr, ndev->reg_ofs.bar2_xlat);
- break;
- case NTB_BAR_4:
- if (ndev->split_bar)
- writel(addr, ndev->reg_ofs.bar4_xlat);
- else
- writeq(addr, ndev->reg_ofs.bar4_xlat);
- break;
- case NTB_BAR_5:
- writel(addr, ndev->reg_ofs.bar5_xlat);
- break;
+ /* Try to set up msix irq */
+
+ ndev->vec = kcalloc(msix_max, sizeof(*ndev->vec), GFP_KERNEL);
+ if (!ndev->vec)
+ goto err_msix_vec_alloc;
+
+ ndev->msix = kcalloc(msix_max, sizeof(*ndev->msix), GFP_KERNEL);
+ if (!ndev->msix)
+ goto err_msix_alloc;
+
+ for (i = 0; i < msix_max; ++i)
+ ndev->msix[i].entry = i;
+
+ msix_count = pci_enable_msix_range(pdev, ndev->msix,
+ msix_min, msix_max);
+ if (msix_count < 0)
+ goto err_msix_enable;
+
+ for (i = 0; i < msix_count; ++i) {
+ ndev->vec[i].ndev = ndev;
+ ndev->vec[i].num = i;
+ rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
+ "ndev_vec_isr", &ndev->vec[i]);
+ if (rc)
+ goto err_msix_request;
}
-}
-/**
- * ntb_ring_doorbell() - Set the doorbell on the secondary/external side
- * @ndev: pointer to ntb_device instance
- * @db: doorbell to ring
- *
- * This function allows triggering of a doorbell on the secondary/external
- * side that will initiate an interrupt on the remote host
- *
- * RETURNS: An appropriate -ERRNO error value on error, or zero for success.
- */
-void ntb_ring_doorbell(struct ntb_device *ndev, unsigned int db)
-{
- dev_dbg(&ndev->pdev->dev, "%s: ringing doorbell %d\n", __func__, db);
+ dev_dbg(ndev_dev(ndev), "Using msix interrupts\n");
+ ndev->db_vec_count = msix_count;
+ ndev->db_vec_shift = msix_shift;
+ return 0;
- if (ndev->hw_type == BWD_HW)
- writeq((u64) 1 << db, ndev->reg_ofs.rdb);
- else
- writew(((1 << ndev->bits_per_vector) - 1) <<
- (db * ndev->bits_per_vector), ndev->reg_ofs.rdb);
-}
+err_msix_request:
+ while (i-- > 0)
+ free_irq(ndev->msix[i].vector, ndev);
+ pci_disable_msix(pdev);
+err_msix_enable:
+ kfree(ndev->msix);
+err_msix_alloc:
+ kfree(ndev->vec);
+err_msix_vec_alloc:
+ ndev->msix = NULL;
+ ndev->vec = NULL;
-static void bwd_recover_link(struct ntb_device *ndev)
-{
- u32 status;
+ /* Try to set up msi irq */
- /* Driver resets the NTB ModPhy lanes - magic! */
- writeb(0xe0, ndev->reg_base + BWD_MODPHY_PCSREG6);
- writeb(0x40, ndev->reg_base + BWD_MODPHY_PCSREG4);
- writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG4);
- writeb(0x60, ndev->reg_base + BWD_MODPHY_PCSREG6);
+ rc = pci_enable_msi(pdev);
+ if (rc)
+ goto err_msi_enable;
- /* Driver waits 100ms to allow the NTB ModPhy to settle */
- msleep(100);
+ rc = request_irq(pdev->irq, ndev_irq_isr, 0,
+ "ndev_irq_isr", ndev);
+ if (rc)
+ goto err_msi_request;
- /* Clear AER Errors, write to clear */
- status = readl(ndev->reg_base + BWD_ERRCORSTS_OFFSET);
- dev_dbg(&ndev->pdev->dev, "ERRCORSTS = %x\n", status);
- status &= PCI_ERR_COR_REP_ROLL;
- writel(status, ndev->reg_base + BWD_ERRCORSTS_OFFSET);
+ dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
+ ndev->db_vec_count = 1;
+ ndev->db_vec_shift = total_shift;
+ return 0;
- /* Clear unexpected electrical idle event in LTSSM, write to clear */
- status = readl(ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
- dev_dbg(&ndev->pdev->dev, "LTSSMERRSTS0 = %x\n", status);
- status |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
- writel(status, ndev->reg_base + BWD_LTSSMERRSTS0_OFFSET);
+err_msi_request:
+ pci_disable_msi(pdev);
+err_msi_enable:
- /* Clear DeSkew Buffer error, write to clear */
- status = readl(ndev->reg_base + BWD_DESKEWSTS_OFFSET);
- dev_dbg(&ndev->pdev->dev, "DESKEWSTS = %x\n", status);
- status |= BWD_DESKEWSTS_DBERR;
- writel(status, ndev->reg_base + BWD_DESKEWSTS_OFFSET);
+ /* Try to set up intx irq */
- status = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
- dev_dbg(&ndev->pdev->dev, "IBSTERRRCRVSTS0 = %x\n", status);
- status &= BWD_IBIST_ERR_OFLOW;
- writel(status, ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
+ pci_intx(pdev, 1);
- /* Releases the NTB state machine to allow the link to retrain */
- status = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
- dev_dbg(&ndev->pdev->dev, "LTSSMSTATEJMP = %x\n", status);
- status &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
- writel(status, ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
+ rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
+ "ndev_irq_isr", ndev);
+ if (rc)
+ goto err_intx_request;
+
+ dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
+ ndev->db_vec_count = 1;
+ ndev->db_vec_shift = total_shift;
+ return 0;
+
+err_intx_request:
+ return rc;
}
-static void ntb_link_event(struct ntb_device *ndev, int link_state)
+static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
{
- unsigned int event;
+ struct pci_dev *pdev;
+ int i;
- if (ndev->link_status == link_state)
- return;
+ pdev = ndev_pdev(ndev);
- if (link_state == NTB_LINK_UP) {
- u16 status;
-
- dev_info(&ndev->pdev->dev, "Link Up\n");
- ndev->link_status = NTB_LINK_UP;
- event = NTB_EVENT_HW_LINK_UP;
-
- if (is_ntb_atom(ndev) ||
- ndev->conn_type == NTB_CONN_TRANSPARENT)
- status = readw(ndev->reg_ofs.lnk_stat);
- else {
- int rc = pci_read_config_word(ndev->pdev,
- SNB_LINK_STATUS_OFFSET,
- &status);
- if (rc)
- return;
- }
+ /* Mask all doorbell interrupts */
+ ndev->db_mask = ndev->db_valid_mask;
+ ndev->reg->db_iowrite(ndev->db_mask,
+ ndev->self_mmio +
+ ndev->self_reg->db_mask);
- ndev->link_width = (status & NTB_LINK_WIDTH_MASK) >> 4;
- ndev->link_speed = (status & NTB_LINK_SPEED_MASK);
- dev_info(&ndev->pdev->dev, "Link Width %d, Link Speed %d\n",
- ndev->link_width, ndev->link_speed);
+ if (ndev->msix) {
+ i = ndev->db_vec_count;
+ while (i--)
+ free_irq(ndev->msix[i].vector, &ndev->vec[i]);
+ pci_disable_msix(pdev);
+ kfree(ndev->msix);
+ kfree(ndev->vec);
} else {
- dev_info(&ndev->pdev->dev, "Link Down\n");
- ndev->link_status = NTB_LINK_DOWN;
- event = NTB_EVENT_HW_LINK_DOWN;
- /* Don't modify link width/speed, we need it in link recovery */
+ free_irq(pdev->irq, ndev);
+ if (pci_dev_msi_enabled(pdev))
+ pci_disable_msi(pdev);
}
-
- /* notify the upper layer if we have an event change */
- if (ndev->event_cb)
- ndev->event_cb(ndev->ntb_transport, event);
}
-static int ntb_link_status(struct ntb_device *ndev)
+static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
+ size_t count, loff_t *offp)
{
- int link_state;
+ struct intel_ntb_dev *ndev;
+ void __iomem *mmio;
+ char *buf;
+ size_t buf_size;
+ ssize_t ret, off;
+ union { u64 v64; u32 v32; u16 v16; } u;
- if (is_ntb_atom(ndev)) {
- u32 ntb_cntl;
+ ndev = filp->private_data;
+ mmio = ndev->self_mmio;
- ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
- if (ntb_cntl & BWD_CNTL_LINK_DOWN)
- link_state = NTB_LINK_DOWN;
- else
- link_state = NTB_LINK_UP;
- } else {
- u16 status;
- int rc;
+ buf_size = min(count, 0x800ul);
- rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
- &status);
- if (rc)
- return rc;
+ buf = kmalloc(buf_size, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
- if (status & NTB_LINK_STATUS_ACTIVE)
- link_state = NTB_LINK_UP;
- else
- link_state = NTB_LINK_DOWN;
+ off = 0;
+
+ off += scnprintf(buf + off, buf_size - off,
+ "NTB Device Information:\n");
+
+ off += scnprintf(buf + off, buf_size - off,
+ "Connection Topology -\t%s\n",
+ ntb_topo_string(ndev->ntb.topo));
+
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B MW Idx -\t\t%d\n", ndev->b2b_idx);
+ off += scnprintf(buf + off, buf_size - off,
+ "BAR4 Split -\t\t%s\n",
+ ndev->bar4_split ? "yes" : "no");
+
+ off += scnprintf(buf + off, buf_size - off,
+ "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
+ off += scnprintf(buf + off, buf_size - off,
+ "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
+
+ if (!ndev->reg->link_is_up(ndev)) {
+ off += scnprintf(buf + off, buf_size - off,
+ "Link Status -\t\tDown\n");
+ } else {
+ off += scnprintf(buf + off, buf_size - off,
+ "Link Status -\t\tUp\n");
+ off += scnprintf(buf + off, buf_size - off,
+ "Link Speed -\t\tPCI-E Gen %u\n",
+ NTB_LNK_STA_SPEED(ndev->lnk_sta));
+ off += scnprintf(buf + off, buf_size - off,
+ "Link Width -\t\tx%u\n",
+ NTB_LNK_STA_WIDTH(ndev->lnk_sta));
}
- ntb_link_event(ndev, link_state);
+ off += scnprintf(buf + off, buf_size - off,
+ "Memory Window Count -\t%u\n", ndev->mw_count);
+ off += scnprintf(buf + off, buf_size - off,
+ "Scratchpad Count -\t%u\n", ndev->spad_count);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Count -\t%u\n", ndev->db_count);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
+
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
+
+ u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Mask -\t\t%#llx\n", u.v64);
+
+ u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
+ off += scnprintf(buf + off, buf_size - off,
+ "Doorbell Bell -\t\t%#llx\n", u.v64);
+
+ off += scnprintf(buf + off, buf_size - off,
+ "\nNTB Incoming XLAT:\n");
+
+ u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
+ off += scnprintf(buf + off, buf_size - off,
+ "XLAT23 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
+ off += scnprintf(buf + off, buf_size - off,
+ "XLAT45 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
+ off += scnprintf(buf + off, buf_size - off,
+ "LMT23 -\t\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
+ off += scnprintf(buf + off, buf_size - off,
+ "LMT45 -\t\t\t%#018llx\n", u.v64);
+
+ if (pdev_is_snb(ndev->ntb.pdev)) {
+ if (ntb_topo_is_b2b(ndev->ntb.topo)) {
+ off += scnprintf(buf + off, buf_size - off,
+ "\nNTB Outgoing B2B XLAT:\n");
+
+ u.v64 = ioread64(mmio + SNB_PBAR23XLAT_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B XLAT23 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + SNB_PBAR45XLAT_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B XLAT45 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + SNB_PBAR23LMT_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B LMT23 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + SNB_PBAR45LMT_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "B2B LMT45 -\t\t%#018llx\n", u.v64);
+
+ off += scnprintf(buf + off, buf_size - off,
+ "\nNTB Secondary BAR:\n");
+
+ u.v64 = ioread64(mmio + SNB_SBAR0BASE_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "SBAR01 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + SNB_SBAR23BASE_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "SBAR23 -\t\t%#018llx\n", u.v64);
+
+ u.v64 = ioread64(mmio + SNB_SBAR45BASE_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "SBAR45 -\t\t%#018llx\n", u.v64);
+ }
+
+ off += scnprintf(buf + off, buf_size - off,
+ "\nSNB NTB Statistics:\n");
- return 0;
+ u.v16 = ioread16(mmio + SNB_USMEMMISS_OFFSET);
+ off += scnprintf(buf + off, buf_size - off,
+ "Upstream Memory Miss -\t%u\n", u.v16);
+
+ off += scnprintf(buf + off, buf_size - off,
+ "\nSNB NTB Hardware Errors:\n");
+
+ if (!pci_read_config_word(ndev->ntb.pdev,
+ SNB_DEVSTS_OFFSET, &u.v16))
+ off += scnprintf(buf + off, buf_size - off,
+ "DEVSTS -\t\t%#06x\n", u.v16);
+
+ if (!pci_read_config_word(ndev->ntb.pdev,
+ SNB_LINK_STATUS_OFFSET, &u.v16))
+ off += scnprintf(buf + off, buf_size - off,
+ "LNKSTS -\t\t%#06x\n", u.v16);
+
+ if (!pci_read_config_dword(ndev->ntb.pdev,
+ SNB_UNCERRSTS_OFFSET, &u.v32))
+ off += scnprintf(buf + off, buf_size - off,
+ "UNCERRSTS -\t\t%#06x\n", u.v32);
+
+ if (!pci_read_config_dword(ndev->ntb.pdev,
+ SNB_CORERRSTS_OFFSET, &u.v32))
+ off += scnprintf(buf + off, buf_size - off,
+ "CORERRSTS -\t\t%#06x\n", u.v32);
+ }
+
+ ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
+ kfree(buf);
+ return ret;
}
-static void bwd_link_recovery(struct work_struct *work)
+static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
{
- struct ntb_device *ndev = container_of(work, struct ntb_device,
- lr_timer.work);
- u32 status32;
-
- bwd_recover_link(ndev);
- /* There is a potential race between the 2 NTB devices recovering at the
- * same time. If the times are the same, the link will not recover and
- * the driver will be stuck in this loop forever. Add a random interval
- * to the recovery time to prevent this race.
- */
- msleep(BWD_LINK_RECOVERY_TIME + prandom_u32() % BWD_LINK_RECOVERY_TIME);
-
- status32 = readl(ndev->reg_base + BWD_LTSSMSTATEJMP_OFFSET);
- if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT)
- goto retry;
-
- status32 = readl(ndev->reg_base + BWD_IBSTERRRCRVSTS0_OFFSET);
- if (status32 & BWD_IBIST_ERR_OFLOW)
- goto retry;
-
- status32 = readl(ndev->reg_ofs.lnk_cntl);
- if (!(status32 & BWD_CNTL_LINK_DOWN)) {
- unsigned char speed, width;
- u16 status16;
-
- status16 = readw(ndev->reg_ofs.lnk_stat);
- width = (status16 & NTB_LINK_WIDTH_MASK) >> 4;
- speed = (status16 & NTB_LINK_SPEED_MASK);
- if (ndev->link_width != width || ndev->link_speed != speed)
- goto retry;
+ if (!debugfs_dir) {
+ ndev->debugfs_dir = NULL;
+ ndev->debugfs_info = NULL;
+ } else {
+ ndev->debugfs_dir =
+ debugfs_create_dir(ndev_name(ndev), debugfs_dir);
+ if (!ndev->debugfs_dir)
+ ndev->debugfs_info = NULL;
+ else
+ ndev->debugfs_info =
+ debugfs_create_file("info", S_IRUSR,
+ ndev->debugfs_dir, ndev,
+ &intel_ntb_debugfs_info);
}
+}
- schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
- return;
+static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
+{
+ debugfs_remove_recursive(ndev->debugfs_dir);
+}
-retry:
- schedule_delayed_work(&ndev->lr_timer, NTB_HB_TIMEOUT);
+static int intel_ntb_mw_count(struct ntb_dev *ntb)
+{
+ return ntb_ndev(ntb)->mw_count;
}
-/* BWD doesn't have link status interrupt, poll on that platform */
-static void bwd_link_poll(struct work_struct *work)
+static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
+ phys_addr_t *base,
+ resource_size_t *size,
+ resource_size_t *align,
+ resource_size_t *align_size)
{
- struct ntb_device *ndev = container_of(work, struct ntb_device,
- hb_timer.work);
- unsigned long ts = jiffies;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+ int bar;
- /* If we haven't gotten an interrupt in a while, check the BWD link
- * status bit
- */
- if (ts > ndev->last_ts + NTB_HB_TIMEOUT) {
- int rc = ntb_link_status(ndev);
- if (rc)
- dev_err(&ndev->pdev->dev,
- "Error determining link status\n");
-
- /* Check to see if a link error is the cause of the link down */
- if (ndev->link_status == NTB_LINK_DOWN) {
- u32 status32 = readl(ndev->reg_base +
- BWD_LTSSMSTATEJMP_OFFSET);
- if (status32 & BWD_LTSSMSTATEJMP_FORCEDETECT) {
- schedule_delayed_work(&ndev->lr_timer, 0);
- return;
- }
- }
- }
+ if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+ idx += 1;
+
+ bar = ndev_mw_to_bar(ndev, idx);
+ if (bar < 0)
+ return bar;
- schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
+ if (base)
+ *base = pci_resource_start(ndev->ntb.pdev, bar) +
+ (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+ if (size)
+ *size = pci_resource_len(ndev->ntb.pdev, bar) -
+ (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
+
+ if (align)
+ *align = pci_resource_len(ndev->ntb.pdev, bar);
+
+ if (align_size)
+ *align_size = 1;
+
+ return 0;
}
-static int ntb_xeon_setup(struct ntb_device *ndev)
+static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
+ dma_addr_t addr, resource_size_t size)
{
- switch (ndev->conn_type) {
- case NTB_CONN_B2B:
- ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
- ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
- ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
- ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
- ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
- if (ndev->split_bar)
- ndev->reg_ofs.bar5_xlat =
- ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
- ndev->limits.max_spads = SNB_MAX_B2B_SPADS;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
+ unsigned long base_reg, xlat_reg, limit_reg;
+ resource_size_t bar_size, mw_size;
+ void __iomem *mmio;
+ u64 base, limit, reg_val;
+ int bar;
- /* There is a Xeon hardware errata related to writes to
- * SDOORBELL or B2BDOORBELL in conjunction with inbound access
- * to NTB MMIO Space, which may hang the system. To workaround
- * this use the second memory window to access the interrupt and
- * scratch pad registers on the remote system.
- */
- if (ndev->wa_flags & WA_SNB_ERR) {
- if (!ndev->mw[ndev->limits.max_mw - 1].bar_sz)
- return -EINVAL;
-
- ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
- ndev->reg_ofs.spad_write =
- ndev->mw[ndev->limits.max_mw - 1].vbase +
- SNB_SPAD_OFFSET;
- ndev->reg_ofs.rdb =
- ndev->mw[ndev->limits.max_mw - 1].vbase +
- SNB_PDOORBELL_OFFSET;
-
- /* Set the Limit register to 4k, the minimum size, to
- * prevent an illegal access
- */
- writeq(ndev->mw[1].bar_sz + 0x1000, ndev->reg_base +
- SNB_PBAR4LMT_OFFSET);
- /* HW errata on the Limit registers. They can only be
- * written when the base register is 4GB aligned and
- * < 32bit. This should already be the case based on
- * the driver defaults, but write the Limit registers
- * first just in case.
- */
-
- ndev->limits.max_mw = SNB_ERRATA_MAX_MW;
- } else {
- /* HW Errata on bit 14 of b2bdoorbell register. Writes
- * will not be mirrored to the remote system. Shrink
- * the number of bits by one, since bit 14 is the last
- * bit.
- */
- ndev->limits.max_db_bits = SNB_MAX_DB_BITS - 1;
- ndev->reg_ofs.spad_write = ndev->reg_base +
- SNB_B2B_SPAD_OFFSET;
- ndev->reg_ofs.rdb = ndev->reg_base +
- SNB_B2B_DOORBELL_OFFSET;
-
- /* Disable the Limit register, just incase it is set to
- * something silly. A 64bit write should handle it
- * regardless of whether it has a split BAR or not.
- */
- writeq(0, ndev->reg_base + SNB_PBAR4LMT_OFFSET);
- /* HW errata on the Limit registers. They can only be
- * written when the base register is 4GB aligned and
- * < 32bit. This should already be the case based on
- * the driver defaults, but write the Limit registers
- * first just in case.
- */
- if (ndev->split_bar)
- ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
- else
- ndev->limits.max_mw = SNB_MAX_MW;
- }
+ if (idx >= ndev->b2b_idx && !ndev->b2b_off)
+ idx += 1;
- /* The Xeon errata workaround requires setting SBAR Base
- * addresses to known values, so that the PBAR XLAT can be
- * pointed at SBAR0 of the remote system.
- */
- if (ndev->dev_type == NTB_DEV_USD) {
- writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
- SNB_PBAR2XLAT_OFFSET);
- if (ndev->wa_flags & WA_SNB_ERR)
- writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
- else {
- if (ndev->split_bar) {
- writel(SNB_MBAR4_DSD_ADDR,
- ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
- writel(SNB_MBAR5_DSD_ADDR,
- ndev->reg_base +
- SNB_PBAR5XLAT_OFFSET);
- } else
- writeq(SNB_MBAR4_DSD_ADDR,
- ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
-
- /* B2B_XLAT_OFFSET is a 64bit register, but can
- * only take 32bit writes
- */
- writel(SNB_MBAR01_DSD_ADDR & 0xffffffff,
- ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
- writel(SNB_MBAR01_DSD_ADDR >> 32,
- ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
- }
-
- writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
- SNB_SBAR0BASE_OFFSET);
- writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
- SNB_SBAR2BASE_OFFSET);
- if (ndev->split_bar) {
- writel(SNB_MBAR4_USD_ADDR, ndev->reg_base +
- SNB_SBAR4BASE_OFFSET);
- writel(SNB_MBAR5_USD_ADDR, ndev->reg_base +
- SNB_SBAR5BASE_OFFSET);
- } else
- writeq(SNB_MBAR4_USD_ADDR, ndev->reg_base +
- SNB_SBAR4BASE_OFFSET);
- } else {
- writeq(SNB_MBAR23_USD_ADDR, ndev->reg_base +
- SNB_PBAR2XLAT_OFFSET);
- if (ndev->wa_flags & WA_SNB_ERR)
- writeq(SNB_MBAR01_USD_ADDR, ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
- else {
- if (ndev->split_bar) {
- writel(SNB_MBAR4_USD_ADDR,
- ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
- writel(SNB_MBAR5_USD_ADDR,
- ndev->reg_base +
- SNB_PBAR5XLAT_OFFSET);
- } else
- writeq(SNB_MBAR4_USD_ADDR,
- ndev->reg_base +
- SNB_PBAR4XLAT_OFFSET);
-
- /*
- * B2B_XLAT_OFFSET is a 64bit register, but can
- * only take 32bit writes
- */
- writel(SNB_MBAR01_USD_ADDR & 0xffffffff,
- ndev->reg_base + SNB_B2B_XLAT_OFFSETL);
- writel(SNB_MBAR01_USD_ADDR >> 32,
- ndev->reg_base + SNB_B2B_XLAT_OFFSETU);
- }
- writeq(SNB_MBAR01_DSD_ADDR, ndev->reg_base +
- SNB_SBAR0BASE_OFFSET);
- writeq(SNB_MBAR23_DSD_ADDR, ndev->reg_base +
- SNB_SBAR2BASE_OFFSET);
- if (ndev->split_bar) {
- writel(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
- SNB_SBAR4BASE_OFFSET);
- writel(SNB_MBAR5_DSD_ADDR, ndev->reg_base +
- SNB_SBAR5BASE_OFFSET);
- } else
- writeq(SNB_MBAR4_DSD_ADDR, ndev->reg_base +
- SNB_SBAR4BASE_OFFSET);
+ bar = ndev_mw_to_bar(ndev, idx);
+ if (bar < 0)
+ return bar;
- }
- break;
- case NTB_CONN_RP:
- if (ndev->wa_flags & WA_SNB_ERR) {
- dev_err(&ndev->pdev->dev,
- "NTB-RP disabled due to hardware errata.\n");
- return -EINVAL;
+ bar_size = pci_resource_len(ndev->ntb.pdev, bar);
+
+ if (idx == ndev->b2b_idx)
+ mw_size = bar_size - ndev->b2b_off;
+ else
+ mw_size = bar_size;
+
+ /* hardware requires that addr is aligned to bar size */
+ if (addr & (bar_size - 1))
+ return -EINVAL;
+
+ /* make sure the range fits in the usable mw size */
+ if (size > mw_size)
+ return -EINVAL;
+
+ mmio = ndev->self_mmio;
+ base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
+ xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
+ limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
+
+ if (bar < 4 || !ndev->bar4_split) {
+ base = ioread64(mmio + base_reg);
+
+ /* Set the limit if supported, if size is not mw_size */
+ if (limit_reg && size != mw_size)
+ limit = base + size;
+ else
+ limit = 0;
+
+ /* set and verify setting the translation address */
+ iowrite64(addr, mmio + xlat_reg);
+ reg_val = ioread64(mmio + xlat_reg);
+ if (reg_val != addr) {
+ iowrite64(0, mmio + xlat_reg);
+ return -EIO;
}
- /* Scratch pads need to have exclusive access from the primary
- * or secondary side. Halve the num spads so that each side can
- * have an equal amount.
- */
- ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
- ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
- /* Note: The SDOORBELL is the cause of the errata. You REALLY
- * don't want to touch it.
- */
- ndev->reg_ofs.rdb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
- ndev->reg_ofs.ldb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
- ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_PDBMSK_OFFSET;
- /* Offset the start of the spads to correspond to whether it is
- * primary or secondary
- */
- ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET +
- ndev->limits.max_spads * 4;
- ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET;
- ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_SBAR2XLAT_OFFSET;
- ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_SBAR4XLAT_OFFSET;
- if (ndev->split_bar) {
- ndev->reg_ofs.bar5_xlat =
- ndev->reg_base + SNB_SBAR5XLAT_OFFSET;
- ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
- } else
- ndev->limits.max_mw = SNB_MAX_MW;
- break;
- case NTB_CONN_TRANSPARENT:
- if (ndev->wa_flags & WA_SNB_ERR) {
- dev_err(&ndev->pdev->dev,
- "NTB-TRANSPARENT disabled due to hardware errata.\n");
- return -EINVAL;
+ /* set and verify setting the limit */
+ iowrite64(limit, mmio + limit_reg);
+ reg_val = ioread64(mmio + limit_reg);
+ if (reg_val != limit) {
+ iowrite64(base, mmio + limit_reg);
+ iowrite64(0, mmio + xlat_reg);
+ return -EIO;
}
+ } else {
+ /* split bar addr range must all be 32 bit */
+ if (addr & (~0ull << 32))
+ return -EINVAL;
+ if ((addr + size) & (~0ull << 32))
+ return -EINVAL;
- /* Scratch pads need to have exclusive access from the primary
- * or secondary side. Halve the num spads so that each side can
- * have an equal amount.
- */
- ndev->limits.max_spads = SNB_MAX_COMPAT_SPADS / 2;
- ndev->limits.max_db_bits = SNB_MAX_DB_BITS;
- ndev->reg_ofs.rdb = ndev->reg_base + SNB_PDOORBELL_OFFSET;
- ndev->reg_ofs.ldb = ndev->reg_base + SNB_SDOORBELL_OFFSET;
- ndev->reg_ofs.ldb_mask = ndev->reg_base + SNB_SDBMSK_OFFSET;
- ndev->reg_ofs.spad_write = ndev->reg_base + SNB_SPAD_OFFSET;
- /* Offset the start of the spads to correspond to whether it is
- * primary or secondary
- */
- ndev->reg_ofs.spad_read = ndev->reg_base + SNB_SPAD_OFFSET +
- ndev->limits.max_spads * 4;
- ndev->reg_ofs.bar2_xlat = ndev->reg_base + SNB_PBAR2XLAT_OFFSET;
- ndev->reg_ofs.bar4_xlat = ndev->reg_base + SNB_PBAR4XLAT_OFFSET;
-
- if (ndev->split_bar) {
- ndev->reg_ofs.bar5_xlat =
- ndev->reg_base + SNB_PBAR5XLAT_OFFSET;
- ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
- } else
- ndev->limits.max_mw = SNB_MAX_MW;
- break;
- default:
- /*
- * we should never hit this. the detect function should've
- * take cared of everything.
- */
- return -EINVAL;
- }
+ base = ioread32(mmio + base_reg);
- ndev->reg_ofs.lnk_cntl = ndev->reg_base + SNB_NTBCNTL_OFFSET;
- ndev->reg_ofs.lnk_stat = ndev->reg_base + SNB_SLINK_STATUS_OFFSET;
- ndev->reg_ofs.spci_cmd = ndev->reg_base + SNB_PCICMD_OFFSET;
+ /* Set the limit if supported, if size is not mw_size */
+ if (limit_reg && size != mw_size)
+ limit = base + size;
+ else
+ limit = 0;
+
+ /* set and verify setting the translation address */
+ iowrite32(addr, mmio + xlat_reg);
+ reg_val = ioread32(mmio + xlat_reg);
+ if (reg_val != addr) {
+ iowrite32(0, mmio + xlat_reg);
+ return -EIO;
+ }
- ndev->limits.msix_cnt = SNB_MSIX_CNT;
- ndev->bits_per_vector = SNB_DB_BITS_PER_VEC;
+ /* set and verify setting the limit */
+ iowrite32(limit, mmio + limit_reg);
+ reg_val = ioread32(mmio + limit_reg);
+ if (reg_val != limit) {
+ iowrite32(base, mmio + limit_reg);
+ iowrite32(0, mmio + xlat_reg);
+ return -EIO;
+ }
+ }
return 0;
}
-static int ntb_bwd_setup(struct ntb_device *ndev)
+static int intel_ntb_link_is_up(struct ntb_dev *ntb,
+ enum ntb_speed *speed,
+ enum ntb_width *width)
{
- int rc;
- u32 val;
-
- ndev->hw_type = BWD_HW;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &val);
- if (rc)
- return rc;
-
- switch ((val & BWD_PPD_CONN_TYPE) >> 8) {
- case NTB_CONN_B2B:
- ndev->conn_type = NTB_CONN_B2B;
- break;
- case NTB_CONN_RP:
- default:
- dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
- return -EINVAL;
+ if (ndev->reg->link_is_up(ndev)) {
+ if (speed)
+ *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
+ if (width)
+ *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
+ return 1;
+ } else {
+ /* TODO MAYBE: is it possible to observe the link speed and
+ * width while link is training? */
+ if (speed)
+ *speed = NTB_SPEED_NONE;
+ if (width)
+ *width = NTB_WIDTH_NONE;
+ return 0;
}
+}
- if (val & BWD_PPD_DEV_TYPE)
- ndev->dev_type = NTB_DEV_DSD;
- else
- ndev->dev_type = NTB_DEV_USD;
+static int intel_ntb_link_enable(struct ntb_dev *ntb,
+ enum ntb_speed max_speed,
+ enum ntb_width max_width)
+{
+ struct intel_ntb_dev *ndev;
+ u32 ntb_ctl;
- /* Initiate PCI-E link training */
- rc = pci_write_config_dword(ndev->pdev, NTB_PPD_OFFSET,
- val | BWD_PPD_INIT_LINK);
- if (rc)
- return rc;
+ ndev = container_of(ntb, struct intel_ntb_dev, ntb);
- ndev->reg_ofs.ldb = ndev->reg_base + BWD_PDOORBELL_OFFSET;
- ndev->reg_ofs.ldb_mask = ndev->reg_base + BWD_PDBMSK_OFFSET;
- ndev->reg_ofs.rdb = ndev->reg_base + BWD_B2B_DOORBELL_OFFSET;
- ndev->reg_ofs.bar2_xlat = ndev->reg_base + BWD_SBAR2XLAT_OFFSET;
- ndev->reg_ofs.bar4_xlat = ndev->reg_base + BWD_SBAR4XLAT_OFFSET;
- ndev->reg_ofs.lnk_cntl = ndev->reg_base + BWD_NTBCNTL_OFFSET;
- ndev->reg_ofs.lnk_stat = ndev->reg_base + BWD_LINK_STATUS_OFFSET;
- ndev->reg_ofs.spad_read = ndev->reg_base + BWD_SPAD_OFFSET;
- ndev->reg_ofs.spad_write = ndev->reg_base + BWD_B2B_SPAD_OFFSET;
- ndev->reg_ofs.spci_cmd = ndev->reg_base + BWD_PCICMD_OFFSET;
- ndev->limits.max_mw = BWD_MAX_MW;
- ndev->limits.max_spads = BWD_MAX_SPADS;
- ndev->limits.max_db_bits = BWD_MAX_DB_BITS;
- ndev->limits.msix_cnt = BWD_MSIX_CNT;
- ndev->bits_per_vector = BWD_DB_BITS_PER_VEC;
-
- /* Since bwd doesn't have a link interrupt, setup a poll timer */
- INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_poll);
- INIT_DELAYED_WORK(&ndev->lr_timer, bwd_link_recovery);
- schedule_delayed_work(&ndev->hb_timer, NTB_HB_TIMEOUT);
+ if (ndev->ntb.topo == NTB_TOPO_SEC)
+ return -EINVAL;
+
+ dev_dbg(ndev_dev(ndev),
+ "Enabling link with max_speed %d max_width %d\n",
+ max_speed, max_width);
+ if (max_speed != NTB_SPEED_AUTO)
+ dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
+ if (max_width != NTB_WIDTH_AUTO)
+ dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
+
+ ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+ ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
+ ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
+ ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
+ if (ndev->bar4_split)
+ ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
+ iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
return 0;
}
-static int ntb_device_setup(struct ntb_device *ndev)
+static int intel_ntb_link_disable(struct ntb_dev *ntb)
{
- int rc;
+ struct intel_ntb_dev *ndev;
+ u32 ntb_cntl;
- if (is_ntb_xeon(ndev))
- rc = ntb_xeon_setup(ndev);
- else if (is_ntb_atom(ndev))
- rc = ntb_bwd_setup(ndev);
- else
- rc = -ENODEV;
+ ndev = container_of(ntb, struct intel_ntb_dev, ntb);
- if (rc)
- return rc;
+ if (ndev->ntb.topo == NTB_TOPO_SEC)
+ return -EINVAL;
- if (ndev->conn_type == NTB_CONN_B2B)
- /* Enable Bus Master and Memory Space on the secondary side */
- writew(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
- ndev->reg_ofs.spci_cmd);
+ dev_dbg(ndev_dev(ndev), "Disabling link\n");
+
+ /* Bring NTB link down */
+ ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
+ ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
+ ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
+ if (ndev->bar4_split)
+ ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
+ ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
+ iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
return 0;
}
-static void ntb_device_free(struct ntb_device *ndev)
+static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
{
- if (is_ntb_atom(ndev)) {
- cancel_delayed_work_sync(&ndev->hb_timer);
- cancel_delayed_work_sync(&ndev->lr_timer);
- }
+ return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
}
-static irqreturn_t bwd_callback_msix_irq(int irq, void *data)
+static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
{
- struct ntb_db_cb *db_cb = data;
- struct ntb_device *ndev = db_cb->ndev;
- unsigned long mask;
+ return ntb_ndev(ntb)->db_valid_mask;
+}
- dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
- db_cb->db_num);
+static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
+{
+ struct intel_ntb_dev *ndev;
- mask = readw(ndev->reg_ofs.ldb_mask);
- set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
- writew(mask, ndev->reg_ofs.ldb_mask);
+ ndev = container_of(ntb, struct intel_ntb_dev, ntb);
- tasklet_schedule(&db_cb->irq_work);
+ return ndev->db_vec_count;
+}
- /* No need to check for the specific HB irq, any interrupt means
- * we're connected.
- */
- ndev->last_ts = jiffies;
+static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- writeq((u64) 1 << db_cb->db_num, ndev->reg_ofs.ldb);
+ if (db_vector < 0 || db_vector > ndev->db_vec_count)
+ return 0;
- return IRQ_HANDLED;
+ return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
}
-static irqreturn_t xeon_callback_msix_irq(int irq, void *data)
+static u64 intel_ntb_db_read(struct ntb_dev *ntb)
{
- struct ntb_db_cb *db_cb = data;
- struct ntb_device *ndev = db_cb->ndev;
- unsigned long mask;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for DB %d\n", irq,
- db_cb->db_num);
+ return ndev_db_read(ndev,
+ ndev->self_mmio +
+ ndev->self_reg->db_bell);
+}
- mask = readw(ndev->reg_ofs.ldb_mask);
- set_bit(db_cb->db_num * ndev->bits_per_vector, &mask);
- writew(mask, ndev->reg_ofs.ldb_mask);
+static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- tasklet_schedule(&db_cb->irq_work);
+ return ndev_db_write(ndev, db_bits,
+ ndev->self_mmio +
+ ndev->self_reg->db_bell);
+}
- /* On Sandybridge, there are 16 bits in the interrupt register
- * but only 4 vectors. So, 5 bits are assigned to the first 3
- * vectors, with the 4th having a single bit for link
- * interrupts.
- */
- writew(((1 << ndev->bits_per_vector) - 1) <<
- (db_cb->db_num * ndev->bits_per_vector), ndev->reg_ofs.ldb);
+static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- return IRQ_HANDLED;
+ return ndev_db_set_mask(ndev, db_bits,
+ ndev->self_mmio +
+ ndev->self_reg->db_mask);
}
-/* Since we do not have a HW doorbell in BWD, this is only used in JF/JT */
-static irqreturn_t xeon_event_msix_irq(int irq, void *dev)
+static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
{
- struct ntb_device *ndev = dev;
- int rc;
-
- dev_dbg(&ndev->pdev->dev, "MSI-X irq %d received for Events\n", irq);
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- rc = ntb_link_status(ndev);
- if (rc)
- dev_err(&ndev->pdev->dev, "Error determining link status\n");
+ return ndev_db_clear_mask(ndev, db_bits,
+ ndev->self_mmio +
+ ndev->self_reg->db_mask);
+}
- /* bit 15 is always the link bit */
- writew(1 << SNB_LINK_DB, ndev->reg_ofs.ldb);
+static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
+ phys_addr_t *db_addr,
+ resource_size_t *db_size)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- return IRQ_HANDLED;
+ return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
+ ndev->peer_reg->db_bell);
}
-static irqreturn_t ntb_interrupt(int irq, void *dev)
+static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
{
- struct ntb_device *ndev = dev;
- unsigned int i = 0;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- if (is_ntb_atom(ndev)) {
- u64 ldb = readq(ndev->reg_ofs.ldb);
+ return ndev_db_write(ndev, db_bits,
+ ndev->peer_mmio +
+ ndev->peer_reg->db_bell);
+}
- dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %Lx\n", irq, ldb);
+static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
+{
+ return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
+}
- while (ldb) {
- i = __ffs(ldb);
- ldb &= ldb - 1;
- bwd_callback_msix_irq(irq, &ndev->db_cb[i]);
- }
- } else {
- u16 ldb = readw(ndev->reg_ofs.ldb);
+static int intel_ntb_spad_count(struct ntb_dev *ntb)
+{
+ struct intel_ntb_dev *ndev;
- dev_dbg(&ndev->pdev->dev, "irq %d - ldb = %x\n", irq, ldb);
+ ndev = container_of(ntb, struct intel_ntb_dev, ntb);
- if (ldb & SNB_DB_HW_LINK) {
- xeon_event_msix_irq(irq, dev);
- ldb &= ~SNB_DB_HW_LINK;
- }
+ return ndev->spad_count;
+}
- while (ldb) {
- i = __ffs(ldb);
- ldb &= ldb - 1;
- xeon_callback_msix_irq(irq, &ndev->db_cb[i]);
- }
- }
+static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- return IRQ_HANDLED;
+ return ndev_spad_read(ndev, idx,
+ ndev->self_mmio +
+ ndev->self_reg->spad);
}
-static int ntb_setup_snb_msix(struct ntb_device *ndev, int msix_entries)
+static int intel_ntb_spad_write(struct ntb_dev *ntb,
+ int idx, u32 val)
{
- struct pci_dev *pdev = ndev->pdev;
- struct msix_entry *msix;
- int rc, i;
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- if (msix_entries < ndev->limits.msix_cnt)
- return -ENOSPC;
+ return ndev_spad_write(ndev, idx, val,
+ ndev->self_mmio +
+ ndev->self_reg->spad);
+}
- rc = pci_enable_msix_exact(pdev, ndev->msix_entries, msix_entries);
- if (rc < 0)
- return rc;
+static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
+ phys_addr_t *spad_addr)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- for (i = 0; i < msix_entries; i++) {
- msix = &ndev->msix_entries[i];
- WARN_ON(!msix->vector);
+ return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
+ ndev->peer_reg->spad);
+}
- if (i == msix_entries - 1) {
- rc = request_irq(msix->vector,
- xeon_event_msix_irq, 0,
- "ntb-event-msix", ndev);
- if (rc)
- goto err;
- } else {
- rc = request_irq(msix->vector,
- xeon_callback_msix_irq, 0,
- "ntb-callback-msix",
- &ndev->db_cb[i]);
- if (rc)
- goto err;
- }
- }
+static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
- ndev->num_msix = msix_entries;
- ndev->max_cbs = msix_entries - 1;
+ return ndev_spad_read(ndev, idx,
+ ndev->peer_mmio +
+ ndev->peer_reg->spad);
+}
- return 0;
+static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
+ int idx, u32 val)
+{
+ struct intel_ntb_dev *ndev = ntb_ndev(ntb);
-err:
- while (--i >= 0) {
- /* Code never reaches here for entry nr 'ndev->num_msix - 1' */
- msix = &ndev->msix_entries[i];
- free_irq(msix->vector, &ndev->db_cb[i]);
- }
+ return ndev_spad_write(ndev, idx, val,
+ ndev->peer_mmio +
+ ndev->peer_reg->spad);
+}
- pci_disable_msix(pdev);
- ndev->num_msix = 0;
+/* BWD */
- return rc;
+static u64 bwd_db_ioread(void __iomem *mmio)
+{
+ return ioread64(mmio);
}
-static int ntb_setup_bwd_msix(struct ntb_device *ndev, int msix_entries)
+static void bwd_db_iowrite(u64 bits, void __iomem *mmio)
{
- struct pci_dev *pdev = ndev->pdev;
- struct msix_entry *msix;
- int rc, i;
+ iowrite64(bits, mmio);
+}
- msix_entries = pci_enable_msix_range(pdev, ndev->msix_entries,
- 1, msix_entries);
- if (msix_entries < 0)
- return msix_entries;
+static int bwd_poll_link(struct intel_ntb_dev *ndev)
+{
+ u32 ntb_ctl;
- for (i = 0; i < msix_entries; i++) {
- msix = &ndev->msix_entries[i];
- WARN_ON(!msix->vector);
+ ntb_ctl = ioread32(ndev->self_mmio + BWD_NTBCNTL_OFFSET);
- rc = request_irq(msix->vector, bwd_callback_msix_irq, 0,
- "ntb-callback-msix", &ndev->db_cb[i]);
- if (rc)
- goto err;
- }
+ if (ntb_ctl == ndev->ntb_ctl)
+ return 0;
- ndev->num_msix = msix_entries;
- ndev->max_cbs = msix_entries;
+ ndev->ntb_ctl = ntb_ctl;
- return 0;
+ ndev->lnk_sta = ioread32(ndev->self_mmio + BWD_LINK_STATUS_OFFSET);
-err:
- while (--i >= 0)
- free_irq(msix->vector, &ndev->db_cb[i]);
+ return 1;
+}
- pci_disable_msix(pdev);
- ndev->num_msix = 0;
+static int bwd_link_is_up(struct intel_ntb_dev *ndev)
+{
+ return BWD_NTB_CTL_ACTIVE(ndev->ntb_ctl);
+}
- return rc;
+static int bwd_link_is_err(struct intel_ntb_dev *ndev)
+{
+ if (ioread32(ndev->self_mmio + BWD_LTSSMSTATEJMP_OFFSET)
+ & BWD_LTSSMSTATEJMP_FORCEDETECT)
+ return 1;
+
+ if (ioread32(ndev->self_mmio + BWD_IBSTERRRCRVSTS0_OFFSET)
+ & BWD_IBIST_ERR_OFLOW)
+ return 1;
+
+ return 0;
}
-static int ntb_setup_msix(struct ntb_device *ndev)
+static inline enum ntb_topo bwd_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
{
- struct pci_dev *pdev = ndev->pdev;
- int msix_entries;
- int rc, i;
+ switch (ppd & BWD_PPD_TOPO_MASK) {
+ case BWD_PPD_TOPO_B2B_USD:
+ dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
+ return NTB_TOPO_B2B_USD;
+
+ case BWD_PPD_TOPO_B2B_DSD:
+ dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
+ return NTB_TOPO_B2B_DSD;
+
+ case BWD_PPD_TOPO_PRI_USD:
+ case BWD_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
+ case BWD_PPD_TOPO_SEC_USD:
+ case BWD_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
+ dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
+ return NTB_TOPO_NONE;
+ }
- msix_entries = pci_msix_vec_count(pdev);
- if (msix_entries < 0) {
- rc = msix_entries;
- goto err;
- } else if (msix_entries > ndev->limits.msix_cnt) {
- rc = -EINVAL;
- goto err;
+ dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
+ return NTB_TOPO_NONE;
+}
+
+static void bwd_link_hb(struct work_struct *work)
+{
+ struct intel_ntb_dev *ndev = hb_ndev(work);
+ unsigned long poll_ts;
+ void __iomem *mmio;
+ u32 status32;
+
+ poll_ts = ndev->last_ts + BWD_LINK_HB_TIMEOUT;
+
+ /* Delay polling the link status if an interrupt was received,
+ * unless the cached link status says the link is down.
+ */
+ if (time_after(poll_ts, jiffies) && bwd_link_is_up(ndev)) {
+ schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
+ return;
}
- ndev->msix_entries = kmalloc(sizeof(struct msix_entry) * msix_entries,
- GFP_KERNEL);
- if (!ndev->msix_entries) {
- rc = -ENOMEM;
- goto err;
+ if (bwd_poll_link(ndev))
+ ntb_link_event(&ndev->ntb);
+
+ if (bwd_link_is_up(ndev) || !bwd_link_is_err(ndev)) {
+ schedule_delayed_work(&ndev->hb_timer, BWD_LINK_HB_TIMEOUT);
+ return;
}
- for (i = 0; i < msix_entries; i++)
- ndev->msix_entries[i].entry = i;
+ /* Link is down with error: recover the link! */
- if (is_ntb_atom(ndev))
- rc = ntb_setup_bwd_msix(ndev, msix_entries);
- else
- rc = ntb_setup_snb_msix(ndev, msix_entries);
- if (rc)
- goto err1;
+ mmio = ndev->self_mmio;
- return 0;
+ /* Driver resets the NTB ModPhy lanes - magic! */
+ iowrite8(0xe0, mmio + BWD_MODPHY_PCSREG6);
+ iowrite8(0x40, mmio + BWD_MODPHY_PCSREG4);
+ iowrite8(0x60, mmio + BWD_MODPHY_PCSREG4);
+ iowrite8(0x60, mmio + BWD_MODPHY_PCSREG6);
-err1:
- kfree(ndev->msix_entries);
-err:
- dev_err(&pdev->dev, "Error allocating MSI-X interrupt\n");
- return rc;
+ /* Driver waits 100ms to allow the NTB ModPhy to settle */
+ msleep(100);
+
+ /* Clear AER Errors, write to clear */
+ status32 = ioread32(mmio + BWD_ERRCORSTS_OFFSET);
+ dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
+ status32 &= PCI_ERR_COR_REP_ROLL;
+ iowrite32(status32, mmio + BWD_ERRCORSTS_OFFSET);
+
+ /* Clear unexpected electrical idle event in LTSSM, write to clear */
+ status32 = ioread32(mmio + BWD_LTSSMERRSTS0_OFFSET);
+ dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
+ status32 |= BWD_LTSSMERRSTS0_UNEXPECTEDEI;
+ iowrite32(status32, mmio + BWD_LTSSMERRSTS0_OFFSET);
+
+ /* Clear DeSkew Buffer error, write to clear */
+ status32 = ioread32(mmio + BWD_DESKEWSTS_OFFSET);
+ dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
+ status32 |= BWD_DESKEWSTS_DBERR;
+ iowrite32(status32, mmio + BWD_DESKEWSTS_OFFSET);
+
+ status32 = ioread32(mmio + BWD_IBSTERRRCRVSTS0_OFFSET);
+ dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
+ status32 &= BWD_IBIST_ERR_OFLOW;
+ iowrite32(status32, mmio + BWD_IBSTERRRCRVSTS0_OFFSET);
+
+ /* Releases the NTB state machine to allow the link to retrain */
+ status32 = ioread32(mmio + BWD_LTSSMSTATEJMP_OFFSET);
+ dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
+ status32 &= ~BWD_LTSSMSTATEJMP_FORCEDETECT;
+ iowrite32(status32, mmio + BWD_LTSSMSTATEJMP_OFFSET);
+
+ /* There is a potential race between the 2 NTB devices recovering at the
+ * same time. If the times are the same, the link will not recover and
+ * the driver will be stuck in this loop forever. Add a random interval
+ * to the recovery time to prevent this race.
+ */
+ schedule_delayed_work(&ndev->hb_timer, BWD_LINK_RECOVERY_TIME
+ + prandom_u32() % BWD_LINK_RECOVERY_TIME);
}
-static int ntb_setup_msi(struct ntb_device *ndev)
+static int bwd_init_isr(struct intel_ntb_dev *ndev)
{
- struct pci_dev *pdev = ndev->pdev;
int rc;
- rc = pci_enable_msi(pdev);
+ rc = ndev_init_isr(ndev, 1, BWD_DB_MSIX_VECTOR_COUNT,
+ BWD_DB_MSIX_VECTOR_SHIFT, BWD_DB_TOTAL_SHIFT);
if (rc)
return rc;
- rc = request_irq(pdev->irq, ntb_interrupt, 0, "ntb-msi", ndev);
- if (rc) {
- pci_disable_msi(pdev);
- dev_err(&pdev->dev, "Error allocating MSI interrupt\n");
- return rc;
- }
+ /* BWD doesn't have link status interrupt, poll on that platform */
+ ndev->last_ts = jiffies;
+ INIT_DELAYED_WORK(&ndev->hb_timer, bwd_link_hb);
+ schedule_delayed_work(&ndev->hb_timer, BWD_LINK_HB_TIMEOUT);
return 0;
}
-static int ntb_setup_intx(struct ntb_device *ndev)
+static void bwd_deinit_isr(struct intel_ntb_dev *ndev)
{
- struct pci_dev *pdev = ndev->pdev;
- int rc;
+ cancel_delayed_work_sync(&ndev->hb_timer);
+ ndev_deinit_isr(ndev);
+}
- /* Verify intx is enabled */
- pci_intx(pdev, 1);
+static int bwd_init_ntb(struct intel_ntb_dev *ndev)
+{
+ ndev->mw_count = BWD_MW_COUNT;
+ ndev->spad_count = BWD_SPAD_COUNT;
+ ndev->db_count = BWD_DB_COUNT;
- rc = request_irq(pdev->irq, ntb_interrupt, IRQF_SHARED, "ntb-intx",
- ndev);
- if (rc)
- return rc;
+ switch (ndev->ntb.topo) {
+ case NTB_TOPO_B2B_USD:
+ case NTB_TOPO_B2B_DSD:
+ ndev->self_reg = &bwd_pri_reg;
+ ndev->peer_reg = &bwd_b2b_reg;
+ ndev->xlat_reg = &bwd_sec_xlat;
+
+ /* Enable Bus Master and Memory Space on the secondary side */
+ iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
+ ndev->self_mmio + BWD_SPCICMD_OFFSET);
+
+ break;
+
+ default:
+ return -EINVAL;
+ }
+
+ ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
return 0;
}
-static int ntb_setup_interrupts(struct ntb_device *ndev)
+static int bwd_init_dev(struct intel_ntb_dev *ndev)
{
+ u32 ppd;
int rc;
- /* On BWD, disable all interrupts. On SNB, disable all but Link
- * Interrupt. The rest will be unmasked as callbacks are registered.
- */
- if (is_ntb_atom(ndev))
- writeq(~0, ndev->reg_ofs.ldb_mask);
- else {
- u16 var = 1 << SNB_LINK_DB;
- writew(~var, ndev->reg_ofs.ldb_mask);
- }
-
- rc = ntb_setup_msix(ndev);
- if (!rc)
- goto done;
+ rc = pci_read_config_dword(ndev->ntb.pdev, BWD_PPD_OFFSET, &ppd);
+ if (rc)
+ return -EIO;
- ndev->bits_per_vector = 1;
- ndev->max_cbs = ndev->limits.max_db_bits;
+ ndev->ntb.topo = bwd_ppd_topo(ndev, ppd);
+ if (ndev->ntb.topo == NTB_TOPO_NONE)
+ return -EINVAL;
- rc = ntb_setup_msi(ndev);
- if (!rc)
- goto done;
+ rc = bwd_init_ntb(ndev);
+ if (rc)
+ return rc;
- rc = ntb_setup_intx(ndev);
- if (rc) {
- dev_err(&ndev->pdev->dev, "no usable interrupts\n");
+ rc = bwd_init_isr(ndev);
+ if (rc)
return rc;
+
+ if (ndev->ntb.topo != NTB_TOPO_SEC) {
+ /* Initiate PCI-E link training */
+ rc = pci_write_config_dword(ndev->ntb.pdev, BWD_PPD_OFFSET,
+ ppd | BWD_PPD_INIT_LINK);
+ if (rc)
+ return rc;
}
-done:
return 0;
}
-static void ntb_free_interrupts(struct ntb_device *ndev)
+static void bwd_deinit_dev(struct intel_ntb_dev *ndev)
{
- struct pci_dev *pdev = ndev->pdev;
+ bwd_deinit_isr(ndev);
+}
- /* mask interrupts */
- if (is_ntb_atom(ndev))
- writeq(~0, ndev->reg_ofs.ldb_mask);
- else
- writew(~0, ndev->reg_ofs.ldb_mask);
+/* SNB */
- if (ndev->num_msix) {
- struct msix_entry *msix;
- u32 i;
+static u64 snb_db_ioread(void __iomem *mmio)
+{
+ return (u64)ioread16(mmio);
+}
- for (i = 0; i < ndev->num_msix; i++) {
- msix = &ndev->msix_entries[i];
- if (is_ntb_xeon(ndev) && i == ndev->num_msix - 1)
- free_irq(msix->vector, ndev);
- else
- free_irq(msix->vector, &ndev->db_cb[i]);
- }
- pci_disable_msix(pdev);
- kfree(ndev->msix_entries);
- } else {
- free_irq(pdev->irq, ndev);
+static void snb_db_iowrite(u64 bits, void __iomem *mmio)
+{
+ iowrite16((u16)bits, mmio);
+}
- if (pci_dev_msi_enabled(pdev))
- pci_disable_msi(pdev);
- }
+static int snb_poll_link(struct intel_ntb_dev *ndev)
+{
+ u16 reg_val;
+ int rc;
+
+ ndev->reg->db_iowrite(ndev->db_link_mask,
+ ndev->self_mmio +
+ ndev->self_reg->db_bell);
+
+ rc = pci_read_config_word(ndev->ntb.pdev,
+ SNB_LINK_STATUS_OFFSET, &reg_val);
+ if (rc)
+ return 0;
+
+ if (reg_val == ndev->lnk_sta)
+ return 0;
+
+ ndev->lnk_sta = reg_val;
+
+ return 1;
}
-static int ntb_create_callbacks(struct ntb_device *ndev)
+static int snb_link_is_up(struct intel_ntb_dev *ndev)
{
- int i;
+ return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
+}
- /* Chicken-egg issue. We won't know how many callbacks are necessary
- * until we see how many MSI-X vectors we get, but these pointers need
- * to be passed into the MSI-X register function. So, we allocate the
- * max, knowing that they might not all be used, to work around this.
- */
- ndev->db_cb = kcalloc(ndev->limits.max_db_bits,
- sizeof(struct ntb_db_cb),
- GFP_KERNEL);
- if (!ndev->db_cb)
- return -ENOMEM;
+static inline enum ntb_topo snb_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
+{
+ switch (ppd & SNB_PPD_TOPO_MASK) {
+ case SNB_PPD_TOPO_B2B_USD:
+ return NTB_TOPO_B2B_USD;
+
+ case SNB_PPD_TOPO_B2B_DSD:
+ return NTB_TOPO_B2B_DSD;
+
+ case SNB_PPD_TOPO_PRI_USD:
+ case SNB_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
+ return NTB_TOPO_PRI;
- for (i = 0; i < ndev->limits.max_db_bits; i++) {
- ndev->db_cb[i].db_num = i;
- ndev->db_cb[i].ndev = ndev;
+ case SNB_PPD_TOPO_SEC_USD:
+ case SNB_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
+ return NTB_TOPO_SEC;
}
- return 0;
+ return NTB_TOPO_NONE;
}
-static void ntb_free_callbacks(struct ntb_device *ndev)
+static inline int snb_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
{
- int i;
+ if (ppd & SNB_PPD_SPLIT_BAR_MASK) {
+ dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
+ return 1;
+ }
+ return 0;
+}
- for (i = 0; i < ndev->limits.max_db_bits; i++)
- ntb_unregister_db_callback(ndev, i);
+static int snb_init_isr(struct intel_ntb_dev *ndev)
+{
+ return ndev_init_isr(ndev, SNB_DB_MSIX_VECTOR_COUNT,
+ SNB_DB_MSIX_VECTOR_COUNT,
+ SNB_DB_MSIX_VECTOR_SHIFT,
+ SNB_DB_TOTAL_SHIFT);
+}
- kfree(ndev->db_cb);
+static void snb_deinit_isr(struct intel_ntb_dev *ndev)
+{
+ ndev_deinit_isr(ndev);
}
-static ssize_t ntb_debugfs_read(struct file *filp, char __user *ubuf,
- size_t count, loff_t *offp)
+static int snb_setup_b2b_mw(struct intel_ntb_dev *ndev,
+ const struct intel_b2b_addr *addr,
+ const struct intel_b2b_addr *peer_addr)
{
- struct ntb_device *ndev;
- char *buf;
- ssize_t ret, offset, out_count;
+ struct pci_dev *pdev;
+ void __iomem *mmio;
+ resource_size_t bar_size;
+ phys_addr_t bar_addr;
+ int b2b_bar;
+ u8 bar_sz;
+
+ pdev = ndev_pdev(ndev);
+ mmio = ndev->self_mmio;
+
+ if (ndev->b2b_idx >= ndev->mw_count) {
+ dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
+ b2b_bar = 0;
+ ndev->b2b_off = 0;
+ } else {
+ b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
+ if (b2b_bar < 0)
+ return -EIO;
- out_count = 500;
+ dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
- buf = kmalloc(out_count, GFP_KERNEL);
- if (!buf)
- return -ENOMEM;
+ bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
- ndev = filp->private_data;
- offset = 0;
- offset += snprintf(buf + offset, out_count - offset,
- "NTB Device Information:\n");
- offset += snprintf(buf + offset, out_count - offset,
- "Connection Type - \t\t%s\n",
- ndev->conn_type == NTB_CONN_TRANSPARENT ?
- "Transparent" : (ndev->conn_type == NTB_CONN_B2B) ?
- "Back to back" : "Root Port");
- offset += snprintf(buf + offset, out_count - offset,
- "Device Type - \t\t\t%s\n",
- ndev->dev_type == NTB_DEV_USD ?
- "DSD/USP" : "USD/DSP");
- offset += snprintf(buf + offset, out_count - offset,
- "Max Number of Callbacks - \t%u\n",
- ntb_max_cbs(ndev));
- offset += snprintf(buf + offset, out_count - offset,
- "Link Status - \t\t\t%s\n",
- ntb_hw_link_status(ndev) ? "Up" : "Down");
- if (ntb_hw_link_status(ndev)) {
- offset += snprintf(buf + offset, out_count - offset,
- "Link Speed - \t\t\tPCI-E Gen %u\n",
- ndev->link_speed);
- offset += snprintf(buf + offset, out_count - offset,
- "Link Width - \t\t\tx%u\n",
- ndev->link_width);
- }
+ dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
- if (is_ntb_xeon(ndev)) {
- u32 status32;
- u16 status16;
- int rc;
-
- offset += snprintf(buf + offset, out_count - offset,
- "\nNTB Device Statistics:\n");
- offset += snprintf(buf + offset, out_count - offset,
- "Upstream Memory Miss - \t%u\n",
- readw(ndev->reg_base +
- SNB_USMEMMISS_OFFSET));
-
- offset += snprintf(buf + offset, out_count - offset,
- "\nNTB Hardware Errors:\n");
-
- rc = pci_read_config_word(ndev->pdev, SNB_DEVSTS_OFFSET,
- &status16);
- if (!rc)
- offset += snprintf(buf + offset, out_count - offset,
- "DEVSTS - \t%#06x\n", status16);
-
- rc = pci_read_config_word(ndev->pdev, SNB_LINK_STATUS_OFFSET,
- &status16);
- if (!rc)
- offset += snprintf(buf + offset, out_count - offset,
- "LNKSTS - \t%#06x\n", status16);
-
- rc = pci_read_config_dword(ndev->pdev, SNB_UNCERRSTS_OFFSET,
- &status32);
- if (!rc)
- offset += snprintf(buf + offset, out_count - offset,
- "UNCERRSTS - \t%#010x\n", status32);
-
- rc = pci_read_config_dword(ndev->pdev, SNB_CORERRSTS_OFFSET,
- &status32);
- if (!rc)
- offset += snprintf(buf + offset, out_count - offset,
- "CORERRSTS - \t%#010x\n", status32);
+ if (b2b_mw_share && SNB_B2B_MIN_SIZE <= bar_size >> 1) {
+ dev_dbg(ndev_dev(ndev),
+ "b2b using first half of bar\n");
+ ndev->b2b_off = bar_size >> 1;
+ } else if (SNB_B2B_MIN_SIZE <= bar_size) {
+ dev_dbg(ndev_dev(ndev),
+ "b2b using whole bar\n");
+ ndev->b2b_off = 0;
+ --ndev->mw_count;
+ } else {
+ dev_dbg(ndev_dev(ndev),
+ "b2b bar size is too small\n");
+ return -EIO;
+ }
}
- if (offset > out_count)
- offset = out_count;
+ /* Reset the secondary bar sizes to match the primary bar sizes,
+ * except disable or halve the size of the b2b secondary bar.
+ *
+ * Note: code for each specific bar size register, because the register
+ * offsets are not in a consistent order (bar5sz comes after ppd, odd).
+ */
+ pci_read_config_byte(pdev, SNB_PBAR23SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
+ if (b2b_bar == 2) {
+ if (ndev->b2b_off)
+ bar_sz -= 1;
+ else
+ bar_sz = 0;
+ }
+ pci_write_config_byte(pdev, SNB_SBAR23SZ_OFFSET, bar_sz);
+ pci_read_config_byte(pdev, SNB_SBAR23SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
+
+ if (!ndev->bar4_split) {
+ pci_read_config_byte(pdev, SNB_PBAR45SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
+ if (b2b_bar == 4) {
+ if (ndev->b2b_off)
+ bar_sz -= 1;
+ else
+ bar_sz = 0;
+ }
+ pci_write_config_byte(pdev, SNB_SBAR45SZ_OFFSET, bar_sz);
+ pci_read_config_byte(pdev, SNB_SBAR45SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
+ } else {
+ pci_read_config_byte(pdev, SNB_PBAR4SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
+ if (b2b_bar == 4) {
+ if (ndev->b2b_off)
+ bar_sz -= 1;
+ else
+ bar_sz = 0;
+ }
+ pci_write_config_byte(pdev, SNB_SBAR4SZ_OFFSET, bar_sz);
+ pci_read_config_byte(pdev, SNB_SBAR4SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
+
+ pci_read_config_byte(pdev, SNB_PBAR5SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
+ if (b2b_bar == 5) {
+ if (ndev->b2b_off)
+ bar_sz -= 1;
+ else
+ bar_sz = 0;
+ }
+ pci_write_config_byte(pdev, SNB_SBAR5SZ_OFFSET, bar_sz);
+ pci_read_config_byte(pdev, SNB_SBAR5SZ_OFFSET, &bar_sz);
+ dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
+ }
- ret = simple_read_from_buffer(ubuf, count, offp, buf, offset);
- kfree(buf);
- return ret;
-}
+ /* SBAR01 hit by first part of the b2b bar */
+ if (b2b_bar == 0)
+ bar_addr = addr->bar0_addr;
+ else if (b2b_bar == 2)
+ bar_addr = addr->bar2_addr64;
+ else if (b2b_bar == 4 && !ndev->bar4_split)
+ bar_addr = addr->bar4_addr64;
+ else if (b2b_bar == 4)
+ bar_addr = addr->bar4_addr32;
+ else if (b2b_bar == 5)
+ bar_addr = addr->bar5_addr32;
+ else
+ return -EIO;
-static const struct file_operations ntb_debugfs_info = {
- .owner = THIS_MODULE,
- .open = simple_open,
- .read = ntb_debugfs_read,
-};
+ dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
+ iowrite64(bar_addr, mmio + SNB_SBAR0BASE_OFFSET);
-static void ntb_setup_debugfs(struct ntb_device *ndev)
-{
- if (!debugfs_initialized())
- return;
+ /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
+ * The b2b bar is either disabled above, or configured half-size, and
+ * it starts at the PBAR xlat + offset.
+ */
- if (!debugfs_dir)
- debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+ bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
+ iowrite64(bar_addr, mmio + SNB_SBAR23BASE_OFFSET);
+ bar_addr = ioread64(mmio + SNB_SBAR23BASE_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
+
+ if (!ndev->bar4_split) {
+ bar_addr = addr->bar4_addr64 +
+ (b2b_bar == 4 ? ndev->b2b_off : 0);
+ iowrite64(bar_addr, mmio + SNB_SBAR45BASE_OFFSET);
+ bar_addr = ioread64(mmio + SNB_SBAR45BASE_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
+ } else {
+ bar_addr = addr->bar4_addr32 +
+ (b2b_bar == 4 ? ndev->b2b_off : 0);
+ iowrite32(bar_addr, mmio + SNB_SBAR4BASE_OFFSET);
+ bar_addr = ioread32(mmio + SNB_SBAR4BASE_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
+
+ bar_addr = addr->bar5_addr32 +
+ (b2b_bar == 5 ? ndev->b2b_off : 0);
+ iowrite32(bar_addr, mmio + SNB_SBAR5BASE_OFFSET);
+ bar_addr = ioread32(mmio + SNB_SBAR5BASE_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
+ }
- ndev->debugfs_dir = debugfs_create_dir(pci_name(ndev->pdev),
- debugfs_dir);
- if (ndev->debugfs_dir)
- ndev->debugfs_info = debugfs_create_file("info", S_IRUSR,
- ndev->debugfs_dir,
- ndev,
- &ntb_debugfs_info);
-}
+ /* setup incoming bar limits == base addrs (zero length windows) */
-static void ntb_free_debugfs(struct ntb_device *ndev)
-{
- debugfs_remove_recursive(ndev->debugfs_dir);
+ bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
+ iowrite64(bar_addr, mmio + SNB_SBAR23LMT_OFFSET);
+ bar_addr = ioread64(mmio + SNB_SBAR23LMT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
- if (debugfs_dir && simple_empty(debugfs_dir)) {
- debugfs_remove_recursive(debugfs_dir);
- debugfs_dir = NULL;
+ if (!ndev->bar4_split) {
+ bar_addr = addr->bar4_addr64 +
+ (b2b_bar == 4 ? ndev->b2b_off : 0);
+ iowrite64(bar_addr, mmio + SNB_SBAR45LMT_OFFSET);
+ bar_addr = ioread64(mmio + SNB_SBAR45LMT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
+ } else {
+ bar_addr = addr->bar4_addr32 +
+ (b2b_bar == 4 ? ndev->b2b_off : 0);
+ iowrite32(bar_addr, mmio + SNB_SBAR4LMT_OFFSET);
+ bar_addr = ioread32(mmio + SNB_SBAR4LMT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
+
+ bar_addr = addr->bar5_addr32 +
+ (b2b_bar == 5 ? ndev->b2b_off : 0);
+ iowrite32(bar_addr, mmio + SNB_SBAR5LMT_OFFSET);
+ bar_addr = ioread32(mmio + SNB_SBAR5LMT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
}
-}
-static void ntb_hw_link_up(struct ntb_device *ndev)
-{
- if (ndev->conn_type == NTB_CONN_TRANSPARENT)
- ntb_link_event(ndev, NTB_LINK_UP);
- else {
- u32 ntb_cntl;
+ /* zero incoming translation addrs */
+ iowrite64(0, mmio + SNB_SBAR23XLAT_OFFSET);
- /* Let's bring the NTB link up */
- ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
- ntb_cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK);
- ntb_cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP;
- ntb_cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP;
- if (ndev->split_bar)
- ntb_cntl |= NTB_CNTL_P2S_BAR5_SNOOP |
- NTB_CNTL_S2P_BAR5_SNOOP;
+ if (!ndev->bar4_split) {
+ iowrite64(0, mmio + SNB_SBAR45XLAT_OFFSET);
+ } else {
+ iowrite32(0, mmio + SNB_SBAR4XLAT_OFFSET);
+ iowrite32(0, mmio + SNB_SBAR5XLAT_OFFSET);
+ }
- writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
+ /* zero outgoing translation limits (whole bar size windows) */
+ iowrite64(0, mmio + SNB_PBAR23LMT_OFFSET);
+ if (!ndev->bar4_split) {
+ iowrite64(0, mmio + SNB_PBAR45LMT_OFFSET);
+ } else {
+ iowrite32(0, mmio + SNB_PBAR4LMT_OFFSET);
+ iowrite32(0, mmio + SNB_PBAR5LMT_OFFSET);
}
-}
-static void ntb_hw_link_down(struct ntb_device *ndev)
-{
- u32 ntb_cntl;
+ /* set outgoing translation offsets */
+ bar_addr = peer_addr->bar2_addr64;
+ iowrite64(bar_addr, mmio + SNB_PBAR23XLAT_OFFSET);
+ bar_addr = ioread64(mmio + SNB_PBAR23XLAT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
+
+ if (!ndev->bar4_split) {
+ bar_addr = peer_addr->bar4_addr64;
+ iowrite64(bar_addr, mmio + SNB_PBAR45XLAT_OFFSET);
+ bar_addr = ioread64(mmio + SNB_PBAR45XLAT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
+ } else {
+ bar_addr = peer_addr->bar4_addr32;
+ iowrite32(bar_addr, mmio + SNB_PBAR4XLAT_OFFSET);
+ bar_addr = ioread32(mmio + SNB_PBAR4XLAT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
+
+ bar_addr = peer_addr->bar5_addr32;
+ iowrite32(bar_addr, mmio + SNB_PBAR5XLAT_OFFSET);
+ bar_addr = ioread32(mmio + SNB_PBAR5XLAT_OFFSET);
+ dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
+ }
- if (ndev->conn_type == NTB_CONN_TRANSPARENT) {
- ntb_link_event(ndev, NTB_LINK_DOWN);
- return;
+ /* set the translation offset for b2b registers */
+ if (b2b_bar == 0)
+ bar_addr = peer_addr->bar0_addr;
+ else if (b2b_bar == 2)
+ bar_addr = peer_addr->bar2_addr64;
+ else if (b2b_bar == 4 && !ndev->bar4_split)
+ bar_addr = peer_addr->bar4_addr64;
+ else if (b2b_bar == 4)
+ bar_addr = peer_addr->bar4_addr32;
+ else if (b2b_bar == 5)
+ bar_addr = peer_addr->bar5_addr32;
+ else
+ return -EIO;
+
+ /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
+ dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
+ iowrite32(bar_addr, mmio + SNB_B2B_XLAT_OFFSETL);
+ iowrite32(bar_addr >> 32, mmio + SNB_B2B_XLAT_OFFSETU);
+
+ if (b2b_bar) {
+ /* map peer ntb mmio config space registers */
+ ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
+ SNB_B2B_MIN_SIZE);
+ if (!ndev->peer_mmio)
+ return -EIO;
}
- /* Bring NTB link down */
- ntb_cntl = readl(ndev->reg_ofs.lnk_cntl);
- ntb_cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP);
- ntb_cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP);
- if (ndev->split_bar)
- ntb_cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP |
- NTB_CNTL_S2P_BAR5_SNOOP);
- ntb_cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK;
- writel(ntb_cntl, ndev->reg_ofs.lnk_cntl);
+ return 0;
}
-static void ntb_max_mw_detect(struct ntb_device *ndev)
+static int snb_init_ntb(struct intel_ntb_dev *ndev)
{
- if (ndev->split_bar)
- ndev->limits.max_mw = HSX_SPLITBAR_MAX_MW;
+ int rc;
+
+ if (ndev->bar4_split)
+ ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
else
- ndev->limits.max_mw = SNB_MAX_MW;
-}
+ ndev->mw_count = SNB_MW_COUNT;
-static int ntb_xeon_detect(struct ntb_device *ndev)
-{
- int rc, bars_mask;
- u32 bars;
- u8 ppd;
+ ndev->spad_count = SNB_SPAD_COUNT;
+ ndev->db_count = SNB_DB_COUNT;
+ ndev->db_link_mask = SNB_DB_LINK_BIT;
- ndev->hw_type = SNB_HW;
+ switch (ndev->ntb.topo) {
+ case NTB_TOPO_PRI:
+ if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+ dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
+ return -EINVAL;
+ }
+ /* use half the spads for the peer */
+ ndev->spad_count >>= 1;
+ ndev->self_reg = &snb_pri_reg;
+ ndev->peer_reg = &snb_sec_reg;
+ ndev->xlat_reg = &snb_sec_xlat;
+ break;
- rc = pci_read_config_byte(ndev->pdev, NTB_PPD_OFFSET, &ppd);
- if (rc)
- return -EIO;
+ case NTB_TOPO_SEC:
+ if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+ dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
+ return -EINVAL;
+ }
+ /* use half the spads for the peer */
+ ndev->spad_count >>= 1;
+ ndev->self_reg = &snb_sec_reg;
+ ndev->peer_reg = &snb_pri_reg;
+ ndev->xlat_reg = &snb_pri_xlat;
+ break;
- if (ppd & SNB_PPD_DEV_TYPE)
- ndev->dev_type = NTB_DEV_USD;
- else
- ndev->dev_type = NTB_DEV_DSD;
+ case NTB_TOPO_B2B_USD:
+ case NTB_TOPO_B2B_DSD:
+ ndev->self_reg = &snb_pri_reg;
+ ndev->peer_reg = &snb_b2b_reg;
+ ndev->xlat_reg = &snb_sec_xlat;
- ndev->split_bar = (ppd & SNB_PPD_SPLIT_BAR) ? 1 : 0;
+ if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
+ ndev->peer_reg = &snb_pri_reg;
- switch (ppd & SNB_PPD_CONN_TYPE) {
- case NTB_CONN_B2B:
- dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
- ndev->conn_type = NTB_CONN_B2B;
- break;
- case NTB_CONN_RP:
- dev_info(&ndev->pdev->dev, "Conn Type = RP\n");
- ndev->conn_type = NTB_CONN_RP;
- break;
- case NTB_CONN_TRANSPARENT:
- dev_info(&ndev->pdev->dev, "Conn Type = TRANSPARENT\n");
- ndev->conn_type = NTB_CONN_TRANSPARENT;
- /*
- * This mode is default to USD/DSP. HW does not report
- * properly in transparent mode as it has no knowledge of
- * NTB. We will just force correct here.
- */
- ndev->dev_type = NTB_DEV_USD;
+ if (b2b_mw_idx < 0)
+ ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
+ else
+ ndev->b2b_idx = b2b_mw_idx;
- /*
- * This is a way for transparent BAR to figure out if we
- * are doing split BAR or not. There is no way for the hw
- * on the transparent side to know and set the PPD.
- */
- bars_mask = pci_select_bars(ndev->pdev, IORESOURCE_MEM);
- bars = hweight32(bars_mask);
- if (bars == (HSX_SPLITBAR_MAX_MW + 1))
- ndev->split_bar = 1;
+ dev_dbg(ndev_dev(ndev),
+ "setting up b2b mw idx %d means %d\n",
+ b2b_mw_idx, ndev->b2b_idx);
+
+ } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
+ dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
+ ndev->db_count -= 1;
+ }
+
+ if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
+ rc = snb_setup_b2b_mw(ndev,
+ &snb_b2b_dsd_addr,
+ &snb_b2b_usd_addr);
+ } else {
+ rc = snb_setup_b2b_mw(ndev,
+ &snb_b2b_usd_addr,
+ &snb_b2b_dsd_addr);
+ }
+ if (rc)
+ return rc;
+
+ /* Enable Bus Master and Memory Space on the secondary side */
+ iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
+ ndev->self_mmio + SNB_SPCICMD_OFFSET);
break;
+
default:
- dev_err(&ndev->pdev->dev, "Unknown PPD %x\n", ppd);
- return -ENODEV;
+ return -EINVAL;
}
- ntb_max_mw_detect(ndev);
+ ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
+
+ ndev->reg->db_iowrite(ndev->db_valid_mask,
+ ndev->self_mmio +
+ ndev->self_reg->db_mask);
return 0;
}
-static int ntb_atom_detect(struct ntb_device *ndev)
+static int snb_init_dev(struct intel_ntb_dev *ndev)
{
- int rc;
- u32 ppd;
+ struct pci_dev *pdev;
+ u8 ppd;
+ int rc, mem;
+
+ /* There is a Xeon hardware errata related to writes to SDOORBELL or
+ * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
+ * which may hang the system. To workaround this use the second memory
+ * window to access the interrupt and scratch pad registers on the
+ * remote system.
+ */
+ ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
+
+ /* There is a hardware errata related to accessing any register in
+ * SB01BASE in the presence of bidirectional traffic crossing the NTB.
+ */
+ ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
- ndev->hw_type = BWD_HW;
- ndev->limits.max_mw = BWD_MAX_MW;
+ /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
+ * mirrored to the remote system. Shrink the number of bits by one,
+ * since bit 14 is the last bit.
+ */
+ ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
+
+ ndev->reg = &snb_reg;
+
+ pdev = ndev_pdev(ndev);
- rc = pci_read_config_dword(ndev->pdev, NTB_PPD_OFFSET, &ppd);
+ rc = pci_read_config_byte(pdev, SNB_PPD_OFFSET, &ppd);
if (rc)
- return rc;
+ return -EIO;
- switch ((ppd & BWD_PPD_CONN_TYPE) >> 8) {
- case NTB_CONN_B2B:
- dev_info(&ndev->pdev->dev, "Conn Type = B2B\n");
- ndev->conn_type = NTB_CONN_B2B;
- break;
- case NTB_CONN_RP:
- default:
- dev_err(&ndev->pdev->dev, "Unsupported NTB configuration\n");
+ ndev->ntb.topo = snb_ppd_topo(ndev, ppd);
+ dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
+ ntb_topo_string(ndev->ntb.topo));
+ if (ndev->ntb.topo == NTB_TOPO_NONE)
return -EINVAL;
+
+ if (ndev->ntb.topo != NTB_TOPO_SEC) {
+ ndev->bar4_split = snb_ppd_bar4_split(ndev, ppd);
+ dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
+ ppd, ndev->bar4_split);
+ } else {
+ /* This is a way for transparent BAR to figure out if we are
+ * doing split BAR or not. There is no way for the hw on the
+ * transparent side to know and set the PPD.
+ */
+ mem = pci_select_bars(pdev, IORESOURCE_MEM);
+ ndev->bar4_split = hweight32(mem) ==
+ HSX_SPLIT_BAR_MW_COUNT + 1;
+ dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
+ mem, ndev->bar4_split);
}
- if (ppd & BWD_PPD_DEV_TYPE)
- ndev->dev_type = NTB_DEV_DSD;
- else
- ndev->dev_type = NTB_DEV_USD;
+ rc = snb_init_ntb(ndev);
+ if (rc)
+ return rc;
- return 0;
+ return snb_init_isr(ndev);
+}
+
+static void snb_deinit_dev(struct intel_ntb_dev *ndev)
+{
+ snb_deinit_isr(ndev);
}
-static int ntb_device_detect(struct ntb_device *ndev)
+static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
{
int rc;
- if (is_ntb_xeon(ndev))
- rc = ntb_xeon_detect(ndev);
- else if (is_ntb_atom(ndev))
- rc = ntb_atom_detect(ndev);
- else
- rc = -ENODEV;
+ pci_set_drvdata(pdev, ndev);
+
+ rc = pci_enable_device(pdev);
+ if (rc)
+ goto err_pci_enable;
+
+ rc = pci_request_regions(pdev, NTB_NAME);
+ if (rc)
+ goto err_pci_regions;
+
+ pci_set_master(pdev);
+
+ rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (rc) {
+ rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
+ if (rc)
+ goto err_dma_mask;
+ dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
+ }
+
+ rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
+ if (rc) {
+ rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+ if (rc)
+ goto err_dma_mask;
+ dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
+ }
- dev_info(&ndev->pdev->dev, "Device Type = %s\n",
- ndev->dev_type == NTB_DEV_USD ? "USD/DSP" : "DSD/USP");
+ ndev->self_mmio = pci_iomap(pdev, 0, 0);
+ if (!ndev->self_mmio) {
+ rc = -EIO;
+ goto err_mmio;
+ }
+ ndev->peer_mmio = ndev->self_mmio;
return 0;
+
+err_mmio:
+err_dma_mask:
+ pci_clear_master(pdev);
+ pci_release_regions(pdev);
+err_pci_regions:
+ pci_disable_device(pdev);
+err_pci_enable:
+ pci_set_drvdata(pdev, NULL);
+ return rc;
}
-static int ntb_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
{
- struct ntb_device *ndev;
- int rc, i;
+ struct pci_dev *pdev = ndev_pdev(ndev);
- ndev = kzalloc(sizeof(struct ntb_device), GFP_KERNEL);
- if (!ndev)
- return -ENOMEM;
+ if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
+ pci_iounmap(pdev, ndev->peer_mmio);
+ pci_iounmap(pdev, ndev->self_mmio);
- ndev->pdev = pdev;
+ pci_clear_master(pdev);
+ pci_release_regions(pdev);
+ pci_disable_device(pdev);
+ pci_set_drvdata(pdev, NULL);
+}
- ntb_set_errata_flags(ndev);
+static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
+ struct pci_dev *pdev)
+{
+ ndev->ntb.pdev = pdev;
+ ndev->ntb.topo = NTB_TOPO_NONE;
+ ndev->ntb.ops = &intel_ntb_ops;
- ndev->link_status = NTB_LINK_DOWN;
- pci_set_drvdata(pdev, ndev);
- ntb_setup_debugfs(ndev);
+ ndev->b2b_off = 0;
+ ndev->b2b_idx = INT_MAX;
- rc = pci_enable_device(pdev);
- if (rc)
- goto err;
+ ndev->bar4_split = 0;
- pci_set_master(ndev->pdev);
+ ndev->mw_count = 0;
+ ndev->spad_count = 0;
+ ndev->db_count = 0;
+ ndev->db_vec_count = 0;
+ ndev->db_vec_shift = 0;
- rc = ntb_device_detect(ndev);
- if (rc)
- goto err;
+ ndev->ntb_ctl = 0;
+ ndev->lnk_sta = 0;
- ndev->mw = kcalloc(ndev->limits.max_mw, sizeof(struct ntb_mw),
- GFP_KERNEL);
- if (!ndev->mw) {
- rc = -ENOMEM;
- goto err1;
- }
+ ndev->db_valid_mask = 0;
+ ndev->db_link_mask = 0;
+ ndev->db_mask = 0;
- if (ndev->split_bar)
- rc = pci_request_selected_regions(pdev, NTB_SPLITBAR_MASK,
- KBUILD_MODNAME);
- else
- rc = pci_request_selected_regions(pdev, NTB_BAR_MASK,
- KBUILD_MODNAME);
+ spin_lock_init(&ndev->db_mask_lock);
+}
- if (rc)
- goto err2;
+static int intel_ntb_pci_probe(struct pci_dev *pdev,
+ const struct pci_device_id *id)
+{
+ struct intel_ntb_dev *ndev;
+ int rc;
- ndev->reg_base = pci_ioremap_bar(pdev, NTB_BAR_MMIO);
- if (!ndev->reg_base) {
- dev_warn(&pdev->dev, "Cannot remap BAR 0\n");
- rc = -EIO;
- goto err3;
- }
+ if (pdev_is_bwd(pdev)) {
+ ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
+ if (!ndev) {
+ rc = -ENOMEM;
+ goto err_ndev;
+ }
- for (i = 0; i < ndev->limits.max_mw; i++) {
- ndev->mw[i].bar_sz = pci_resource_len(pdev, MW_TO_BAR(i));
+ ndev_init_struct(ndev, pdev);
- /*
- * with the errata we need to steal last of the memory
- * windows for workarounds and they point to MMIO registers.
- */
- if ((ndev->wa_flags & WA_SNB_ERR) &&
- (i == (ndev->limits.max_mw - 1))) {
- ndev->mw[i].vbase =
- ioremap_nocache(pci_resource_start(pdev,
- MW_TO_BAR(i)),
- ndev->mw[i].bar_sz);
- } else {
- ndev->mw[i].vbase =
- ioremap_wc(pci_resource_start(pdev,
- MW_TO_BAR(i)),
- ndev->mw[i].bar_sz);
- }
+ rc = intel_ntb_init_pci(ndev, pdev);
+ if (rc)
+ goto err_init_pci;
+
+ rc = bwd_init_dev(ndev);
+ if (rc)
+ goto err_init_dev;
- dev_info(&pdev->dev, "MW %d size %llu\n", i,
- (unsigned long long) ndev->mw[i].bar_sz);
- if (!ndev->mw[i].vbase) {
- dev_warn(&pdev->dev, "Cannot remap BAR %d\n",
- MW_TO_BAR(i));
- rc = -EIO;
- goto err4;
+ } else if (pdev_is_snb(pdev)) {
+ ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
+ if (!ndev) {
+ rc = -ENOMEM;
+ goto err_ndev;
}
- }
- rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
- if (rc) {
- rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
- if (rc)
- goto err4;
+ ndev_init_struct(ndev, pdev);
- dev_warn(&pdev->dev, "Cannot DMA highmem\n");
- }
+ rc = intel_ntb_init_pci(ndev, pdev);
+ if (rc)
+ goto err_init_pci;
- rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
- if (rc) {
- rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
+ rc = snb_init_dev(ndev);
if (rc)
- goto err4;
+ goto err_init_dev;
- dev_warn(&pdev->dev, "Cannot DMA consistent highmem\n");
+ } else {
+ rc = -EINVAL;
+ goto err_ndev;
}
- rc = ntb_device_setup(ndev);
- if (rc)
- goto err4;
+ ndev_reset_unsafe_flags(ndev);
- rc = ntb_create_callbacks(ndev);
- if (rc)
- goto err5;
-
- rc = ntb_setup_interrupts(ndev);
- if (rc)
- goto err6;
+ ndev->reg->poll_link(ndev);
- /* The scratchpad registers keep the values between rmmod/insmod,
- * blast them now
- */
- for (i = 0; i < ndev->limits.max_spads; i++) {
- ntb_write_local_spad(ndev, i, 0);
- ntb_write_remote_spad(ndev, i, 0);
- }
+ ndev_init_debugfs(ndev);
- rc = ntb_transport_init(pdev);
+ rc = ntb_register_device(&ndev->ntb);
if (rc)
- goto err7;
-
- ntb_hw_link_up(ndev);
+ goto err_register;
return 0;
-err7:
- ntb_free_interrupts(ndev);
-err6:
- ntb_free_callbacks(ndev);
-err5:
- ntb_device_free(ndev);
-err4:
- for (i--; i >= 0; i--)
- iounmap(ndev->mw[i].vbase);
- iounmap(ndev->reg_base);
-err3:
- if (ndev->split_bar)
- pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
- else
- pci_release_selected_regions(pdev, NTB_BAR_MASK);
-err2:
- kfree(ndev->mw);
-err1:
- pci_disable_device(pdev);
-err:
- ntb_free_debugfs(ndev);
+err_register:
+ ndev_deinit_debugfs(ndev);
+ if (pdev_is_bwd(pdev))
+ bwd_deinit_dev(ndev);
+ else if (pdev_is_snb(pdev))
+ snb_deinit_dev(ndev);
+err_init_dev:
+ intel_ntb_deinit_pci(ndev);
+err_init_pci:
kfree(ndev);
-
- dev_err(&pdev->dev, "Error loading %s module\n", KBUILD_MODNAME);
+err_ndev:
return rc;
}
-static void ntb_pci_remove(struct pci_dev *pdev)
+static void intel_ntb_pci_remove(struct pci_dev *pdev)
{
- struct ntb_device *ndev = pci_get_drvdata(pdev);
- int i;
+ struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
+
+ ntb_unregister_device(&ndev->ntb);
+ ndev_deinit_debugfs(ndev);
+ if (pdev_is_bwd(pdev))
+ bwd_deinit_dev(ndev);
+ else if (pdev_is_snb(pdev))
+ snb_deinit_dev(ndev);
+ intel_ntb_deinit_pci(ndev);
+ kfree(ndev);
+}
- ntb_hw_link_down(ndev);
+static const struct intel_ntb_reg bwd_reg = {
+ .poll_link = bwd_poll_link,
+ .link_is_up = bwd_link_is_up,
+ .db_ioread = bwd_db_ioread,
+ .db_iowrite = bwd_db_iowrite,
+ .db_size = sizeof(u64),
+ .ntb_ctl = BWD_NTBCNTL_OFFSET,
+ .mw_bar = {2, 4},
+};
- ntb_transport_free(ndev->ntb_transport);
+static const struct intel_ntb_alt_reg bwd_pri_reg = {
+ .db_bell = BWD_PDOORBELL_OFFSET,
+ .db_mask = BWD_PDBMSK_OFFSET,
+ .spad = BWD_SPAD_OFFSET,
+};
- ntb_free_interrupts(ndev);
- ntb_free_callbacks(ndev);
- ntb_device_free(ndev);
+static const struct intel_ntb_alt_reg bwd_b2b_reg = {
+ .db_bell = BWD_B2B_DOORBELL_OFFSET,
+ .spad = BWD_B2B_SPAD_OFFSET,
+};
- /* need to reset max_mw limits so we can unmap properly */
- if (ndev->hw_type == SNB_HW)
- ntb_max_mw_detect(ndev);
+static const struct intel_ntb_xlat_reg bwd_sec_xlat = {
+ /* FIXME : .bar0_base = BWD_SBAR0BASE_OFFSET, */
+ /* FIXME : .bar2_limit = BWD_SBAR2LMT_OFFSET, */
+ .bar2_xlat = BWD_SBAR2XLAT_OFFSET,
+};
- for (i = 0; i < ndev->limits.max_mw; i++)
- iounmap(ndev->mw[i].vbase);
+static const struct intel_ntb_reg snb_reg = {
+ .poll_link = snb_poll_link,
+ .link_is_up = snb_link_is_up,
+ .db_ioread = snb_db_ioread,
+ .db_iowrite = snb_db_iowrite,
+ .db_size = sizeof(u32),
+ .ntb_ctl = SNB_NTBCNTL_OFFSET,
+ .mw_bar = {2, 4, 5},
+};
- kfree(ndev->mw);
- iounmap(ndev->reg_base);
- if (ndev->split_bar)
- pci_release_selected_regions(pdev, NTB_SPLITBAR_MASK);
- else
- pci_release_selected_regions(pdev, NTB_BAR_MASK);
- pci_disable_device(pdev);
- ntb_free_debugfs(ndev);
- kfree(ndev);
-}
+static const struct intel_ntb_alt_reg snb_pri_reg = {
+ .db_bell = SNB_PDOORBELL_OFFSET,
+ .db_mask = SNB_PDBMSK_OFFSET,
+ .spad = SNB_SPAD_OFFSET,
+};
-static struct pci_driver ntb_pci_driver = {
+static const struct intel_ntb_alt_reg snb_sec_reg = {
+ .db_bell = SNB_SDOORBELL_OFFSET,
+ .db_mask = SNB_SDBMSK_OFFSET,
+ /* second half of the scratchpads */
+ .spad = SNB_SPAD_OFFSET + (SNB_SPAD_COUNT << 1),
+};
+
+static const struct intel_ntb_alt_reg snb_b2b_reg = {
+ .db_bell = SNB_B2B_DOORBELL_OFFSET,
+ .spad = SNB_B2B_SPAD_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg snb_pri_xlat = {
+ /* Note: no primary .bar0_base visible to the secondary side.
+ *
+ * The secondary side cannot get the base address stored in primary
+ * bars. The base address is necessary to set the limit register to
+ * any value other than zero, or unlimited.
+ *
+ * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
+ * window by setting the limit equal to base, nor can it limit the size
+ * of the memory window by setting the limit to base + size.
+ */
+ .bar2_limit = SNB_PBAR23LMT_OFFSET,
+ .bar2_xlat = SNB_PBAR23XLAT_OFFSET,
+};
+
+static const struct intel_ntb_xlat_reg snb_sec_xlat = {
+ .bar0_base = SNB_SBAR0BASE_OFFSET,
+ .bar2_limit = SNB_SBAR23LMT_OFFSET,
+ .bar2_xlat = SNB_SBAR23XLAT_OFFSET,
+};
+
+static const struct intel_b2b_addr snb_b2b_usd_addr = {
+ .bar2_addr64 = SNB_B2B_BAR2_USD_ADDR64,
+ .bar4_addr64 = SNB_B2B_BAR4_USD_ADDR64,
+ .bar4_addr32 = SNB_B2B_BAR4_USD_ADDR32,
+ .bar5_addr32 = SNB_B2B_BAR5_USD_ADDR32,
+};
+
+static const struct intel_b2b_addr snb_b2b_dsd_addr = {
+ .bar2_addr64 = SNB_B2B_BAR2_DSD_ADDR64,
+ .bar4_addr64 = SNB_B2B_BAR4_DSD_ADDR64,
+ .bar4_addr32 = SNB_B2B_BAR4_DSD_ADDR32,
+ .bar5_addr32 = SNB_B2B_BAR5_DSD_ADDR32,
+};
+
+/* operations for primary side of local ntb */
+static const struct ntb_dev_ops intel_ntb_ops = {
+ .mw_count = intel_ntb_mw_count,
+ .mw_get_range = intel_ntb_mw_get_range,
+ .mw_set_trans = intel_ntb_mw_set_trans,
+ .link_is_up = intel_ntb_link_is_up,
+ .link_enable = intel_ntb_link_enable,
+ .link_disable = intel_ntb_link_disable,
+ .db_is_unsafe = intel_ntb_db_is_unsafe,
+ .db_valid_mask = intel_ntb_db_valid_mask,
+ .db_vector_count = intel_ntb_db_vector_count,
+ .db_vector_mask = intel_ntb_db_vector_mask,
+ .db_read = intel_ntb_db_read,
+ .db_clear = intel_ntb_db_clear,
+ .db_set_mask = intel_ntb_db_set_mask,
+ .db_clear_mask = intel_ntb_db_clear_mask,
+ .peer_db_addr = intel_ntb_peer_db_addr,
+ .peer_db_set = intel_ntb_peer_db_set,
+ .spad_is_unsafe = intel_ntb_spad_is_unsafe,
+ .spad_count = intel_ntb_spad_count,
+ .spad_read = intel_ntb_spad_read,
+ .spad_write = intel_ntb_spad_write,
+ .peer_spad_addr = intel_ntb_peer_spad_addr,
+ .peer_spad_read = intel_ntb_peer_spad_read,
+ .peer_spad_write = intel_ntb_peer_spad_write,
+};
+
+static const struct file_operations intel_ntb_debugfs_info = {
+ .owner = THIS_MODULE,
+ .open = simple_open,
+ .read = ndev_debugfs_read,
+};
+
+static const struct pci_device_id intel_ntb_pci_tbl[] = {
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
+ {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
+ {0}
+};
+MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
+
+static struct pci_driver intel_ntb_pci_driver = {
.name = KBUILD_MODNAME,
- .id_table = ntb_pci_tbl,
- .probe = ntb_pci_probe,
- .remove = ntb_pci_remove,
+ .id_table = intel_ntb_pci_tbl,
+ .probe = intel_ntb_pci_probe,
+ .remove = intel_ntb_pci_remove,
};
-module_pci_driver(ntb_pci_driver);
+static int __init intel_ntb_pci_driver_init(void)
+{
+ if (debugfs_initialized())
+ debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
+
+ return pci_register_driver(&intel_ntb_pci_driver);
+}
+module_init(intel_ntb_pci_driver_init);
+
+static void __exit intel_ntb_pci_driver_exit(void)
+{
+ pci_unregister_driver(&intel_ntb_pci_driver);
+
+ debugfs_remove_recursive(debugfs_dir);
+}
+module_exit(intel_ntb_pci_driver_exit);
+