/* * Microsemi Switchtec(tm) PCIe Management Driver * Copyright (c) 2017, Microsemi Corporation * * This program is free software; you can redistribute it and/or modify it * under the terms and conditions of the GNU General Public License, * version 2, as published by the Free Software Foundation. * * This program is distributed in the hope it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for * more details. * */ #include #include #include #include #include #include #include MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); MODULE_VERSION("0.1"); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Microsemi Corporation"); static ulong max_mw_size = SZ_2M; module_param(max_mw_size, ulong, 0644); MODULE_PARM_DESC(max_mw_size, "Max memory window size reported to the upper layer"); static bool use_lut_mws; module_param(use_lut_mws, bool, 0644); MODULE_PARM_DESC(use_lut_mws, "Enable the use of the LUT based memory windows"); #ifndef ioread64 #ifdef readq #define ioread64 readq #else #define ioread64 _ioread64 static inline u64 _ioread64(void __iomem *mmio) { u64 low, high; low = ioread32(mmio); high = ioread32(mmio + sizeof(u32)); return low | (high << 32); } #endif #endif #ifndef iowrite64 #ifdef writeq #define iowrite64 writeq #else #define iowrite64 _iowrite64 static inline void _iowrite64(u64 val, void __iomem *mmio) { iowrite32(val, mmio); iowrite32(val >> 32, mmio + sizeof(u32)); } #endif #endif #define SWITCHTEC_NTB_MAGIC 0x45CC0001 #define MAX_MWS 128 struct shared_mw { u32 magic; u32 link_sta; u32 partition_id; u64 mw_sizes[MAX_MWS]; u32 spad[128]; }; #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) #define LUT_SIZE SZ_64K struct switchtec_ntb { struct ntb_dev ntb; struct switchtec_dev *stdev; int self_partition; int peer_partition; int doorbell_irq; int message_irq; struct ntb_info_regs __iomem *mmio_ntb; struct ntb_ctrl_regs __iomem *mmio_ctrl; struct ntb_dbmsg_regs __iomem *mmio_dbmsg; struct ntb_ctrl_regs __iomem *mmio_self_ctrl; struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; void __iomem *mmio_xlink_win; struct shared_mw *self_shared; struct shared_mw __iomem *peer_shared; dma_addr_t self_shared_dma; u64 db_mask; u64 db_valid_mask; int db_shift; int db_peer_shift; /* synchronize rmw access of db_mask and hw reg */ spinlock_t db_mask_lock; int nr_direct_mw; int nr_lut_mw; int nr_rsvd_luts; int direct_mw_to_bar[MAX_DIRECT_MW]; int peer_nr_direct_mw; int peer_nr_lut_mw; int peer_direct_mw_to_bar[MAX_DIRECT_MW]; bool link_is_up; enum ntb_speed link_speed; enum ntb_width link_width; struct work_struct link_reinit_work; }; static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) { return container_of(ntb, struct switchtec_ntb, ntb); } static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, struct ntb_ctrl_regs __iomem *ctl, u32 op, int wait_status) { static const char * const op_text[] = { [NTB_CTRL_PART_OP_LOCK] = "lock", [NTB_CTRL_PART_OP_CFG] = "configure", [NTB_CTRL_PART_OP_RESET] = "reset", }; int i; u32 ps; int status; switch (op) { case NTB_CTRL_PART_OP_LOCK: status = NTB_CTRL_PART_STATUS_LOCKING; break; case NTB_CTRL_PART_OP_CFG: status = NTB_CTRL_PART_STATUS_CONFIGURING; break; case NTB_CTRL_PART_OP_RESET: status = NTB_CTRL_PART_STATUS_RESETTING; break; default: return -EINVAL; } iowrite32(op, &ctl->partition_op); for (i = 0; i < 1000; i++) { if (msleep_interruptible(50) != 0) { iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); return -EINTR; } ps = ioread32(&ctl->partition_status) & 0xFFFF; if (ps != status) break; } if (ps == wait_status) return 0; if (ps == status) { dev_err(&sndev->stdev->dev, "Timed out while performing %s (%d). (%08x)\n", op_text[op], op, ioread32(&ctl->partition_status)); return -ETIMEDOUT; } return -EIO; } static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, u32 val) { if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) return -EINVAL; iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); return 0; } static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) { struct switchtec_ntb *sndev = ntb_sndev(ntb); int nr_direct_mw = sndev->peer_nr_direct_mw; int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; if (!use_lut_mws) nr_lut_mw = 0; return nr_direct_mw + nr_lut_mw; } static int lut_index(struct switchtec_ntb *sndev, int mw_idx) { return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; } static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) { return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; } static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx, resource_size_t *addr_align, resource_size_t *size_align, resource_size_t *size_max) { struct switchtec_ntb *sndev = ntb_sndev(ntb); int lut; resource_size_t size; if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; lut = widx >= sndev->peer_nr_direct_mw; size = ioread64(&sndev->peer_shared->mw_sizes[widx]); if (size == 0) return -EINVAL; if (addr_align) *addr_align = lut ? size : SZ_4K; if (size_align) *size_align = lut ? size : SZ_4K; if (size_max) *size_max = size; return 0; } static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) { struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; int bar = sndev->peer_direct_mw_to_bar[idx]; u32 ctl_val; ctl_val = ioread32(&ctl->bar_entry[bar].ctl); ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); iowrite32(0, &ctl->bar_entry[bar].win_size); iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); } static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) { struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); } static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, dma_addr_t addr, resource_size_t size) { int xlate_pos = ilog2(size); int bar = sndev->peer_direct_mw_to_bar[idx]; struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; u32 ctl_val; ctl_val = ioread32(&ctl->bar_entry[bar].ctl); ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); iowrite64(sndev->self_partition | addr, &ctl->bar_entry[bar].xlate_addr); } static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, dma_addr_t addr, resource_size_t size) { struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), &ctl->lut_entry[peer_lut_index(sndev, idx)]); } static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, dma_addr_t addr, resource_size_t size) { struct switchtec_ntb *sndev = ntb_sndev(ntb); struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; int xlate_pos = ilog2(size); int nr_direct_mw = sndev->peer_nr_direct_mw; int rc; if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", widx, pidx, &addr, &size); if (widx >= switchtec_ntb_mw_count(ntb, pidx)) return -EINVAL; if (xlate_pos < 12) return -EINVAL; if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { /* * In certain circumstances we can get a buffer that is * not aligned to its size. (Most of the time * dma_alloc_coherent ensures this). This can happen when * using large buffers allocated by the CMA * (see CMA_CONFIG_ALIGNMENT) */ dev_err(&sndev->stdev->dev, "ERROR: Memory window address is not aligned to it's size!\n"); return -EINVAL; } rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, NTB_CTRL_PART_STATUS_LOCKED); if (rc) return rc; if (addr == 0 || size == 0) { if (widx < nr_direct_mw) switchtec_ntb_mw_clr_direct(sndev, widx); else switchtec_ntb_mw_clr_lut(sndev, widx); } else { if (widx < nr_direct_mw) switchtec_ntb_mw_set_direct(sndev, widx, addr, size); else switchtec_ntb_mw_set_lut(sndev, widx, addr, size); } rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, NTB_CTRL_PART_STATUS_NORMAL); if (rc == -EIO) { dev_err(&sndev->stdev->dev, "Hardware reported an error configuring mw %d: %08x\n", widx, ioread32(&ctl->bar_error)); if (widx < nr_direct_mw) switchtec_ntb_mw_clr_direct(sndev, widx); else switchtec_ntb_mw_clr_lut(sndev, widx); switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, NTB_CTRL_PART_STATUS_NORMAL); } return rc; } static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) { struct switchtec_ntb *sndev = ntb_sndev(ntb); int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); } static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, int idx, phys_addr_t *base, resource_size_t *size) { int bar = sndev->direct_mw_to_bar[idx]; size_t offset = 0; if (bar < 0) return -EINVAL; if (idx == 0) { /* * This is the direct BAR shared with the LUTs * which means the actual window will be offset * by the size of all the LUT entries. */ offset = LUT_SIZE * sndev->nr_lut_mw; } if (base) *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; if (size) { *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; if (offset && *size > offset) *size = offset; if (*size > max_mw_size) *size = max_mw_size; } return 0; } static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, int idx, phys_addr_t *base, resource_size_t *size) { int bar = sndev->direct_mw_to_bar[0]; int offset; offset = LUT_SIZE * lut_index(sndev, idx); if (base) *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; if (size) *size = LUT_SIZE; return 0; } static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, phys_addr_t *base, resource_size_t *size) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (idx < sndev->nr_direct_mw) return switchtec_ntb_direct_get_addr(sndev, idx, base, size); else if (idx < switchtec_ntb_peer_mw_count(ntb)) return switchtec_ntb_lut_get_addr(sndev, idx, base, size); else return -EINVAL; } static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, int partition, enum ntb_speed *speed, enum ntb_width *width) { struct switchtec_dev *stdev = sndev->stdev; u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); if (speed) *speed = (linksta >> 16) & 0xF; if (width) *width = (linksta >> 20) & 0x3F; } static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) { enum ntb_speed self_speed, peer_speed; enum ntb_width self_width, peer_width; if (!sndev->link_is_up) { sndev->link_speed = NTB_SPEED_NONE; sndev->link_width = NTB_WIDTH_NONE; return; } switchtec_ntb_part_link_speed(sndev, sndev->self_partition, &self_speed, &self_width); switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, &peer_speed, &peer_width); sndev->link_speed = min(self_speed, peer_speed); sndev->link_width = min(self_width, peer_width); } static int crosslink_is_enabled(struct switchtec_ntb *sndev) { struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); } static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) { int i; u32 msg_map = 0; if (!crosslink_is_enabled(sndev)) return; for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { int m = i | sndev->self_partition << 2; msg_map |= m << i * 8; } iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, &sndev->mmio_peer_dbmsg->odb_mask); } enum switchtec_msg { LINK_MESSAGE = 0, MSG_LINK_UP = 1, MSG_LINK_DOWN = 2, MSG_CHECK_LINK = 3, MSG_LINK_FORCE_DOWN = 4, }; static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); static void link_reinit_work(struct work_struct *work) { struct switchtec_ntb *sndev; sndev = container_of(work, struct switchtec_ntb, link_reinit_work); switchtec_ntb_reinit_peer(sndev); } static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, enum switchtec_msg msg) { int link_sta; int old = sndev->link_is_up; if (msg == MSG_LINK_FORCE_DOWN) { schedule_work(&sndev->link_reinit_work); if (sndev->link_is_up) { sndev->link_is_up = 0; ntb_link_event(&sndev->ntb); dev_info(&sndev->stdev->dev, "ntb link forced down\n"); } return; } link_sta = sndev->self_shared->link_sta; if (link_sta) { u64 peer = ioread64(&sndev->peer_shared->magic); if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) link_sta = peer >> 32; else link_sta = 0; } sndev->link_is_up = link_sta; switchtec_ntb_set_link_speed(sndev); if (link_sta != old) { switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); ntb_link_event(&sndev->ntb); dev_info(&sndev->stdev->dev, "ntb link %s\n", link_sta ? "up" : "down"); if (link_sta) crosslink_init_dbmsgs(sndev); } } static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) { struct switchtec_ntb *sndev = stdev->sndev; switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); } static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, enum ntb_speed *speed, enum ntb_width *width) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (speed) *speed = sndev->link_speed; if (width) *width = sndev->link_width; return sndev->link_is_up; } static int switchtec_ntb_link_enable(struct ntb_dev *ntb, enum ntb_speed max_speed, enum ntb_width max_width) { struct switchtec_ntb *sndev = ntb_sndev(ntb); dev_dbg(&sndev->stdev->dev, "enabling link\n"); sndev->self_shared->link_sta = 1; switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); return 0; } static int switchtec_ntb_link_disable(struct ntb_dev *ntb) { struct switchtec_ntb *sndev = ntb_sndev(ntb); dev_dbg(&sndev->stdev->dev, "disabling link\n"); sndev->self_shared->link_sta = 0; switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); return 0; } static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) { struct switchtec_ntb *sndev = ntb_sndev(ntb); return sndev->db_valid_mask; } static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) { return 1; } static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (db_vector < 0 || db_vector > 1) return 0; return sndev->db_valid_mask; } static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) { u64 ret; struct switchtec_ntb *sndev = ntb_sndev(ntb); ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; return ret & sndev->db_valid_mask; } static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) { struct switchtec_ntb *sndev = ntb_sndev(ntb); iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); return 0; } static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) { unsigned long irqflags; struct switchtec_ntb *sndev = ntb_sndev(ntb); if (db_bits & ~sndev->db_valid_mask) return -EINVAL; spin_lock_irqsave(&sndev->db_mask_lock, irqflags); sndev->db_mask |= db_bits << sndev->db_shift; iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); return 0; } static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) { unsigned long irqflags; struct switchtec_ntb *sndev = ntb_sndev(ntb); if (db_bits & ~sndev->db_valid_mask) return -EINVAL; spin_lock_irqsave(&sndev->db_mask_lock, irqflags); sndev->db_mask &= ~(db_bits << sndev->db_shift); iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); return 0; } static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) { struct switchtec_ntb *sndev = ntb_sndev(ntb); return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; } static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, phys_addr_t *db_addr, resource_size_t *db_size) { struct switchtec_ntb *sndev = ntb_sndev(ntb); unsigned long offset; offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - (unsigned long)sndev->stdev->mmio; offset += sndev->db_shift / 8; if (db_addr) *db_addr = pci_resource_start(ntb->pdev, 0) + offset; if (db_size) *db_size = sizeof(u32); return 0; } static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) { struct switchtec_ntb *sndev = ntb_sndev(ntb); iowrite64(db_bits << sndev->db_peer_shift, &sndev->mmio_peer_dbmsg->odb); return 0; } static int switchtec_ntb_spad_count(struct ntb_dev *ntb) { struct switchtec_ntb *sndev = ntb_sndev(ntb); return ARRAY_SIZE(sndev->self_shared->spad); } static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) return 0; if (!sndev->self_shared) return 0; return sndev->self_shared->spad[idx]; } static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) return -EINVAL; if (!sndev->self_shared) return -EIO; sndev->self_shared->spad[idx] = val; return 0; } static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, int sidx) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) return 0; if (!sndev->peer_shared) return 0; return ioread32(&sndev->peer_shared->spad[sidx]); } static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, int sidx, u32 val) { struct switchtec_ntb *sndev = ntb_sndev(ntb); if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) return -EINVAL; if (!sndev->peer_shared) return -EIO; iowrite32(val, &sndev->peer_shared->spad[sidx]); return 0; } static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, int sidx, phys_addr_t *spad_addr) { struct switchtec_ntb *sndev = ntb_sndev(ntb); unsigned long offset; if (pidx != NTB_DEF_PEER_IDX) return -EINVAL; offset = (unsigned long)&sndev->peer_shared->spad[sidx] - (unsigned long)sndev->stdev->mmio; if (spad_addr) *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; return 0; } static const struct ntb_dev_ops switchtec_ntb_ops = { .mw_count = switchtec_ntb_mw_count, .mw_get_align = switchtec_ntb_mw_get_align, .mw_set_trans = switchtec_ntb_mw_set_trans, .peer_mw_count = switchtec_ntb_peer_mw_count, .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, .link_is_up = switchtec_ntb_link_is_up, .link_enable = switchtec_ntb_link_enable, .link_disable = switchtec_ntb_link_disable, .db_valid_mask = switchtec_ntb_db_valid_mask, .db_vector_count = switchtec_ntb_db_vector_count, .db_vector_mask = switchtec_ntb_db_vector_mask, .db_read = switchtec_ntb_db_read, .db_clear = switchtec_ntb_db_clear, .db_set_mask = switchtec_ntb_db_set_mask, .db_clear_mask = switchtec_ntb_db_clear_mask, .db_read_mask = switchtec_ntb_db_read_mask, .peer_db_addr = switchtec_ntb_peer_db_addr, .peer_db_set = switchtec_ntb_peer_db_set, .spad_count = switchtec_ntb_spad_count, .spad_read = switchtec_ntb_spad_read, .spad_write = switchtec_ntb_spad_write, .peer_spad_read = switchtec_ntb_peer_spad_read, .peer_spad_write = switchtec_ntb_peer_spad_write, .peer_spad_addr = switchtec_ntb_peer_spad_addr, }; static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) { u64 tpart_vec; int self; u64 part_map; int bit; sndev->ntb.pdev = sndev->stdev->pdev; sndev->ntb.topo = NTB_TOPO_SWITCH; sndev->ntb.ops = &switchtec_ntb_ops; INIT_WORK(&sndev->link_reinit_work, link_reinit_work); sndev->self_partition = sndev->stdev->partition; sndev->mmio_ntb = sndev->stdev->mmio_ntb; self = sndev->self_partition; tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); tpart_vec <<= 32; tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); part_map = ioread64(&sndev->mmio_ntb->ep_map); part_map &= ~(1 << sndev->self_partition); if (!ffs(tpart_vec)) { if (sndev->stdev->partition_count != 2) { dev_err(&sndev->stdev->dev, "ntb target partition not defined\n"); return -ENODEV; } bit = ffs(part_map); if (!bit) { dev_err(&sndev->stdev->dev, "peer partition is not NT partition\n"); return -ENODEV; } sndev->peer_partition = bit - 1; } else { if (ffs(tpart_vec) != fls(tpart_vec)) { dev_err(&sndev->stdev->dev, "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); return -ENODEV; } sndev->peer_partition = ffs(tpart_vec) - 1; if (!(part_map & (1 << sndev->peer_partition))) { dev_err(&sndev->stdev->dev, "ntb target partition is not NT partition\n"); return -ENODEV; } } dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", sndev->self_partition, sndev->stdev->partition_count); sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + SWITCHTEC_NTB_REG_CTRL_OFFSET; sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + SWITCHTEC_NTB_REG_DBMSG_OFFSET; sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; return 0; } static int config_rsvd_lut_win(struct switchtec_ntb *sndev, struct ntb_ctrl_regs __iomem *ctl, int lut_idx, int partition, u64 addr) { int peer_bar = sndev->peer_direct_mw_to_bar[0]; u32 ctl_val; int rc; rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, NTB_CTRL_PART_STATUS_LOCKED); if (rc) return rc; ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); ctl_val &= 0xFF; ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; ctl_val |= ilog2(LUT_SIZE) << 8; ctl_val |= (sndev->nr_lut_mw - 1) << 14; iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), &ctl->lut_entry[lut_idx]); rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, NTB_CTRL_PART_STATUS_NORMAL); if (rc) { u32 bar_error, lut_error; bar_error = ioread32(&ctl->bar_error); lut_error = ioread32(&ctl->lut_error); dev_err(&sndev->stdev->dev, "Error setting up reserved lut window: %08x / %08x\n", bar_error, lut_error); return rc; } return 0; } static int config_req_id_table(struct switchtec_ntb *sndev, struct ntb_ctrl_regs __iomem *mmio_ctrl, int *req_ids, int count) { int i, rc = 0; u32 error; u32 proxy_id; if (ioread32(&mmio_ctrl->req_id_table_size) < count) { dev_err(&sndev->stdev->dev, "Not enough requester IDs available.\n"); return -EFAULT; } rc = switchtec_ntb_part_op(sndev, mmio_ctrl, NTB_CTRL_PART_OP_LOCK, NTB_CTRL_PART_STATUS_LOCKED); if (rc) return rc; iowrite32(NTB_PART_CTRL_ID_PROT_DIS, &mmio_ctrl->partition_ctrl); for (i = 0; i < count; i++) { iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, &mmio_ctrl->req_id_table[i]); proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); dev_dbg(&sndev->stdev->dev, "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, (proxy_id >> 1) & 0x7); } rc = switchtec_ntb_part_op(sndev, mmio_ctrl, NTB_CTRL_PART_OP_CFG, NTB_CTRL_PART_STATUS_NORMAL); if (rc == -EIO) { error = ioread32(&mmio_ctrl->req_id_error); dev_err(&sndev->stdev->dev, "Error setting up the requester ID table: %08x\n", error); } return 0; } static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, u64 *mw_addrs, int mw_count) { int rc, i; struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; u64 addr; size_t size, offset; int bar; int xlate_pos; u32 ctl_val; rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, NTB_CTRL_PART_STATUS_LOCKED); if (rc) return rc; for (i = 0; i < sndev->nr_lut_mw; i++) { if (i == ntb_lut_idx) continue; addr = mw_addrs[0] + LUT_SIZE * i; iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | addr), &ctl->lut_entry[i]); } sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); for (i = 0; i < sndev->nr_direct_mw; i++) { bar = sndev->direct_mw_to_bar[i]; offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; addr = mw_addrs[i] + offset; size = pci_resource_len(sndev->ntb.pdev, bar) - offset; xlate_pos = ilog2(size); if (offset && size > offset) size = offset; ctl_val = ioread32(&ctl->bar_entry[bar].ctl); ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); iowrite64(sndev->peer_partition | addr, &ctl->bar_entry[bar].xlate_addr); } rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, NTB_CTRL_PART_STATUS_NORMAL); if (rc) { u32 bar_error, lut_error; bar_error = ioread32(&ctl->bar_error); lut_error = ioread32(&ctl->lut_error); dev_err(&sndev->stdev->dev, "Error setting up cross link windows: %08x / %08x\n", bar_error, lut_error); return rc; } return 0; } static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, struct ntb_ctrl_regs __iomem *mmio_ctrl) { int req_ids[16]; int i; u32 proxy_id; for (i = 0; i < ARRAY_SIZE(req_ids); i++) { proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) break; req_ids[i] = ((proxy_id >> 1) & 0xFF); } return config_req_id_table(sndev, mmio_ctrl, req_ids, i); } /* * In crosslink configuration there is a virtual partition in the * middle of the two switches. The BARs in this partition have to be * enumerated and assigned addresses. */ static int crosslink_enum_partition(struct switchtec_ntb *sndev, u64 *bar_addrs) { struct part_cfg_regs __iomem *part_cfg = &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; u32 pff = ioread32(&part_cfg->vep_pff_inst_id); struct pff_csr_regs __iomem *mmio_pff = &sndev->stdev->mmio_pff_csr[pff]; const u64 bar_space = 0x1000000000LL; u64 bar_addr; int bar_cnt = 0; int i; iowrite16(0x6, &mmio_pff->pcicmd); for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); bar_addr = ioread64(&mmio_pff->pci_bar64[i]); bar_addr &= ~0xf; dev_dbg(&sndev->stdev->dev, "Crosslink BAR%d addr: %llx\n", i, bar_addr); if (bar_addr != bar_space * i) continue; bar_addrs[bar_cnt++] = bar_addr; } return bar_cnt; } static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) { int rc; int bar = sndev->direct_mw_to_bar[0]; const int ntb_lut_idx = 1; u64 bar_addrs[6]; u64 addr; int offset; int bar_cnt; if (!crosslink_is_enabled(sndev)) return 0; dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); sndev->ntb.topo = NTB_TOPO_CROSSLINK; bar_cnt = crosslink_enum_partition(sndev, bar_addrs); if (bar_cnt < sndev->nr_direct_mw + 1) { dev_err(&sndev->stdev->dev, "Error enumerating crosslink partition\n"); return -EINVAL; } addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + SWITCHTEC_NTB_REG_DBMSG_OFFSET + sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); offset = addr & (LUT_SIZE - 1); addr -= offset; rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, sndev->peer_partition, addr); if (rc) return rc; rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], bar_cnt - 1); if (rc) return rc; rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); if (rc) return rc; sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, LUT_SIZE, LUT_SIZE); if (!sndev->mmio_xlink_win) { rc = -ENOMEM; return rc; } sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; sndev->nr_rsvd_luts++; crosslink_init_dbmsgs(sndev); return 0; } static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) { if (sndev->mmio_xlink_win) pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); } static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) { int i; int cnt = 0; for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { u32 r = ioread32(&ctrl->bar_entry[i].ctl); if (r & NTB_CTRL_BAR_VALID) map[cnt++] = i; } return cnt; } static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) { sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, sndev->mmio_self_ctrl); sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", sndev->nr_direct_mw, sndev->nr_lut_mw); sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, sndev->mmio_peer_ctrl); sndev->peer_nr_lut_mw = ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); } /* * There are 64 doorbells in the switch hardware but this is * shared among all partitions. So we must split them in half * (32 for each partition). However, the message interrupts are * also shared with the top 4 doorbells so we just limit this to * 28 doorbells per partition. * * In crosslink mode, each side has it's own dbmsg register so * they can each use all 60 of the available doorbells. */ static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) { sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { sndev->db_shift = 0; sndev->db_peer_shift = 0; sndev->db_valid_mask = sndev->db_mask; } else if (sndev->self_partition < sndev->peer_partition) { sndev->db_shift = 0; sndev->db_peer_shift = 32; sndev->db_valid_mask = 0x0FFFFFFF; } else { sndev->db_shift = 32; sndev->db_peer_shift = 0; sndev->db_valid_mask = 0x0FFFFFFF; } iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, &sndev->mmio_peer_dbmsg->odb_mask); dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); } static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) { int i; u32 msg_map = 0; for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { int m = i | sndev->peer_partition << 2; msg_map |= m << i * 8; } iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, &sndev->mmio_self_dbmsg->imsg[i]); } static int switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) { int req_ids[2]; /* * Root Complex Requester ID (which is 0:00.0) */ req_ids[0] = 0; /* * Host Bridge Requester ID (as read from the mmap address) */ req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, ARRAY_SIZE(req_ids)); } static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) { int i; memset(sndev->self_shared, 0, LUT_SIZE); sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; sndev->self_shared->partition_id = sndev->stdev->partition; for (i = 0; i < sndev->nr_direct_mw; i++) { int bar = sndev->direct_mw_to_bar[i]; resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); if (i == 0) sz = min_t(resource_size_t, sz, LUT_SIZE * sndev->nr_lut_mw); sndev->self_shared->mw_sizes[i] = sz; } for (i = 0; i < sndev->nr_lut_mw; i++) { int idx = sndev->nr_direct_mw + i; sndev->self_shared->mw_sizes[idx] = LUT_SIZE; } } static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) { int self_bar = sndev->direct_mw_to_bar[0]; int rc; sndev->nr_rsvd_luts++; sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, &sndev->self_shared_dma, GFP_KERNEL); if (!sndev->self_shared) { dev_err(&sndev->stdev->dev, "unable to allocate memory for shared mw\n"); return -ENOMEM; } switchtec_ntb_init_shared(sndev); rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, sndev->self_partition, sndev->self_shared_dma); if (rc) goto unalloc_and_exit; sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); if (!sndev->peer_shared) { rc = -ENOMEM; goto unalloc_and_exit; } dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); return 0; unalloc_and_exit: dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, sndev->self_shared, sndev->self_shared_dma); return rc; } static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) { if (sndev->peer_shared) pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); if (sndev->self_shared) dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, sndev->self_shared, sndev->self_shared_dma); sndev->nr_rsvd_luts--; } static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) { struct switchtec_ntb *sndev = dev; dev_dbg(&sndev->stdev->dev, "doorbell\n"); ntb_db_event(&sndev->ntb, 0); return IRQ_HANDLED; } static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) { int i; struct switchtec_ntb *sndev = dev; for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); if (msg & NTB_DBMSG_IMSG_STATUS) { dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", i, (u32)msg); iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); if (i == LINK_MESSAGE) switchtec_ntb_check_link(sndev, msg); } } return IRQ_HANDLED; } static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) { int i; int rc; int doorbell_irq = 0; int message_irq = 0; int event_irq; int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); while (doorbell_irq == event_irq) doorbell_irq++; while (message_irq == doorbell_irq || message_irq == event_irq) message_irq++; dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", event_irq, doorbell_irq, message_irq); for (i = 0; i < idb_vecs - 4; i++) iowrite8(doorbell_irq, &sndev->mmio_self_dbmsg->idb_vec_map[i]); for (; i < idb_vecs; i++) iowrite8(message_irq, &sndev->mmio_self_dbmsg->idb_vec_map[i]); sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); rc = request_irq(sndev->doorbell_irq, switchtec_ntb_doorbell_isr, 0, "switchtec_ntb_doorbell", sndev); if (rc) return rc; rc = request_irq(sndev->message_irq, switchtec_ntb_message_isr, 0, "switchtec_ntb_message", sndev); if (rc) { free_irq(sndev->doorbell_irq, sndev); return rc; } return 0; } static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) { free_irq(sndev->doorbell_irq, sndev); free_irq(sndev->message_irq, sndev); } static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) { dev_info(&sndev->stdev->dev, "peer reinitialized\n"); switchtec_ntb_deinit_shared_mw(sndev); switchtec_ntb_init_mw(sndev); return switchtec_ntb_init_shared_mw(sndev); } static int switchtec_ntb_add(struct device *dev, struct class_interface *class_intf) { struct switchtec_dev *stdev = to_stdev(dev); struct switchtec_ntb *sndev; int rc; stdev->sndev = NULL; if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) return -ENODEV; sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); if (!sndev) return -ENOMEM; sndev->stdev = stdev; rc = switchtec_ntb_init_sndev(sndev); if (rc) goto free_and_exit; switchtec_ntb_init_mw(sndev); rc = switchtec_ntb_init_req_id_table(sndev); if (rc) goto free_and_exit; rc = switchtec_ntb_init_crosslink(sndev); if (rc) goto free_and_exit; switchtec_ntb_init_db(sndev); switchtec_ntb_init_msgs(sndev); rc = switchtec_ntb_init_shared_mw(sndev); if (rc) goto deinit_crosslink; rc = switchtec_ntb_init_db_msg_irq(sndev); if (rc) goto deinit_shared_and_exit; /* * If this host crashed, the other host may think the link is * still up. Tell them to force it down (it will go back up * once we register the ntb device). */ switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); rc = ntb_register_device(&sndev->ntb); if (rc) goto deinit_and_exit; stdev->sndev = sndev; stdev->link_notifier = switchtec_ntb_link_notification; dev_info(dev, "NTB device registered\n"); return 0; deinit_and_exit: switchtec_ntb_deinit_db_msg_irq(sndev); deinit_shared_and_exit: switchtec_ntb_deinit_shared_mw(sndev); deinit_crosslink: switchtec_ntb_deinit_crosslink(sndev); free_and_exit: kfree(sndev); dev_err(dev, "failed to register ntb device: %d\n", rc); return rc; } static void switchtec_ntb_remove(struct device *dev, struct class_interface *class_intf) { struct switchtec_dev *stdev = to_stdev(dev); struct switchtec_ntb *sndev = stdev->sndev; if (!sndev) return; stdev->link_notifier = NULL; stdev->sndev = NULL; ntb_unregister_device(&sndev->ntb); switchtec_ntb_deinit_db_msg_irq(sndev); switchtec_ntb_deinit_shared_mw(sndev); switchtec_ntb_deinit_crosslink(sndev); kfree(sndev); dev_info(dev, "ntb device unregistered\n"); } static struct class_interface switchtec_interface = { .add_dev = switchtec_ntb_add, .remove_dev = switchtec_ntb_remove, }; static int __init switchtec_ntb_init(void) { switchtec_interface.class = switchtec_class; return class_interface_register(&switchtec_interface); } module_init(switchtec_ntb_init); static void __exit switchtec_ntb_exit(void) { class_interface_unregister(&switchtec_interface); } module_exit(switchtec_ntb_exit);