// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) #include #include #include #include #include #include #include #include #include #include #include #include #include "fun_queue.h" #include "fun_dev.h" #define FUN_ADMIN_CMD_TO_MS 3000 enum { AQA_ASQS_SHIFT = 0, AQA_ACQS_SHIFT = 16, AQA_MIN_QUEUE_SIZE = 2, AQA_MAX_QUEUE_SIZE = 4096 }; /* context for admin commands */ struct fun_cmd_ctx { fun_admin_callback_t cb; /* callback to invoke on completion */ void *cb_data; /* user data provided to callback */ int cpu; /* CPU where the cmd's tag was allocated */ }; /* Context for synchronous admin commands. */ struct fun_sync_cmd_ctx { struct completion compl; u8 *rsp_buf; /* caller provided response buffer */ unsigned int rsp_len; /* response buffer size */ u8 rsp_status; /* command response status */ }; /* Wait for the CSTS.RDY bit to match @enabled. */ static int fun_wait_ready(struct fun_dev *fdev, bool enabled) { unsigned int cap_to = NVME_CAP_TIMEOUT(fdev->cap_reg); u32 bit = enabled ? NVME_CSTS_RDY : 0; unsigned long deadline; deadline = ((cap_to + 1) * HZ / 2) + jiffies; /* CAP.TO is in 500ms */ for (;;) { u32 csts = readl(fdev->bar + NVME_REG_CSTS); if (csts == ~0) { dev_err(fdev->dev, "CSTS register read %#x\n", csts); return -EIO; } if ((csts & NVME_CSTS_RDY) == bit) return 0; if (time_is_before_jiffies(deadline)) break; msleep(100); } dev_err(fdev->dev, "Timed out waiting for device to indicate RDY %u; aborting %s\n", enabled, enabled ? "initialization" : "reset"); return -ETIMEDOUT; } /* Check CSTS and return an error if it is unreadable or has unexpected * RDY value. */ static int fun_check_csts_rdy(struct fun_dev *fdev, unsigned int expected_rdy) { u32 csts = readl(fdev->bar + NVME_REG_CSTS); u32 actual_rdy = csts & NVME_CSTS_RDY; if (csts == ~0) { dev_err(fdev->dev, "CSTS register read %#x\n", csts); return -EIO; } if (actual_rdy != expected_rdy) { dev_err(fdev->dev, "Unexpected CSTS RDY %u\n", actual_rdy); return -EINVAL; } return 0; } /* Check that CSTS RDY has the expected value. Then write a new value to the CC * register and wait for CSTS RDY to match the new CC ENABLE state. */ static int fun_update_cc_enable(struct fun_dev *fdev, unsigned int initial_rdy) { int rc = fun_check_csts_rdy(fdev, initial_rdy); if (rc) return rc; writel(fdev->cc_reg, fdev->bar + NVME_REG_CC); return fun_wait_ready(fdev, !!(fdev->cc_reg & NVME_CC_ENABLE)); } static int fun_disable_ctrl(struct fun_dev *fdev) { fdev->cc_reg &= ~(NVME_CC_SHN_MASK | NVME_CC_ENABLE); return fun_update_cc_enable(fdev, 1); } static int fun_enable_ctrl(struct fun_dev *fdev, u32 admin_cqesz_log2, u32 admin_sqesz_log2) { fdev->cc_reg = (admin_cqesz_log2 << NVME_CC_IOCQES_SHIFT) | (admin_sqesz_log2 << NVME_CC_IOSQES_SHIFT) | ((PAGE_SHIFT - 12) << NVME_CC_MPS_SHIFT) | NVME_CC_ENABLE; return fun_update_cc_enable(fdev, 0); } static int fun_map_bars(struct fun_dev *fdev, const char *name) { struct pci_dev *pdev = to_pci_dev(fdev->dev); int err; err = pci_request_mem_regions(pdev, name); if (err) { dev_err(&pdev->dev, "Couldn't get PCI memory resources, err %d\n", err); return err; } fdev->bar = pci_ioremap_bar(pdev, 0); if (!fdev->bar) { dev_err(&pdev->dev, "Couldn't map BAR 0\n"); pci_release_mem_regions(pdev); return -ENOMEM; } return 0; } static void fun_unmap_bars(struct fun_dev *fdev) { struct pci_dev *pdev = to_pci_dev(fdev->dev); if (fdev->bar) { iounmap(fdev->bar); fdev->bar = NULL; pci_release_mem_regions(pdev); } } static int fun_set_dma_masks(struct device *dev) { int err; err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (err) dev_err(dev, "DMA mask configuration failed, err %d\n", err); return err; } static irqreturn_t fun_admin_irq(int irq, void *data) { struct fun_queue *funq = data; return fun_process_cq(funq, 0) ? IRQ_HANDLED : IRQ_NONE; } static void fun_complete_admin_cmd(struct fun_queue *funq, void *data, void *entry, const struct fun_cqe_info *info) { const struct fun_admin_rsp_common *rsp_common = entry; struct fun_dev *fdev = funq->fdev; struct fun_cmd_ctx *cmd_ctx; int cpu; u16 cid; if (info->sqhd == cpu_to_be16(0xffff)) { dev_dbg(fdev->dev, "adminq event"); if (fdev->adminq_cb) fdev->adminq_cb(fdev, entry); return; } cid = be16_to_cpu(rsp_common->cid); dev_dbg(fdev->dev, "admin CQE cid %u, op %u, ret %u\n", cid, rsp_common->op, rsp_common->ret); cmd_ctx = &fdev->cmd_ctx[cid]; if (cmd_ctx->cpu < 0) { dev_err(fdev->dev, "admin CQE with CID=%u, op=%u does not match a pending command\n", cid, rsp_common->op); return; } if (cmd_ctx->cb) cmd_ctx->cb(fdev, entry, xchg(&cmd_ctx->cb_data, NULL)); cpu = cmd_ctx->cpu; cmd_ctx->cpu = -1; sbitmap_queue_clear(&fdev->admin_sbq, cid, cpu); } static int fun_init_cmd_ctx(struct fun_dev *fdev, unsigned int ntags) { unsigned int i; fdev->cmd_ctx = kvcalloc(ntags, sizeof(*fdev->cmd_ctx), GFP_KERNEL); if (!fdev->cmd_ctx) return -ENOMEM; for (i = 0; i < ntags; i++) fdev->cmd_ctx[i].cpu = -1; return 0; } /* Allocate and enable an admin queue and assign it the first IRQ vector. */ static int fun_enable_admin_queue(struct fun_dev *fdev, const struct fun_dev_params *areq) { struct fun_queue_alloc_req qreq = { .cqe_size_log2 = areq->cqe_size_log2, .sqe_size_log2 = areq->sqe_size_log2, .cq_depth = areq->cq_depth, .sq_depth = areq->sq_depth, .rq_depth = areq->rq_depth, }; unsigned int ntags = areq->sq_depth - 1; struct fun_queue *funq; int rc; if (fdev->admin_q) return -EEXIST; if (areq->sq_depth < AQA_MIN_QUEUE_SIZE || areq->sq_depth > AQA_MAX_QUEUE_SIZE || areq->cq_depth < AQA_MIN_QUEUE_SIZE || areq->cq_depth > AQA_MAX_QUEUE_SIZE) return -EINVAL; fdev->admin_q = fun_alloc_queue(fdev, 0, &qreq); if (!fdev->admin_q) return -ENOMEM; rc = fun_init_cmd_ctx(fdev, ntags); if (rc) goto free_q; rc = sbitmap_queue_init_node(&fdev->admin_sbq, ntags, -1, false, GFP_KERNEL, dev_to_node(fdev->dev)); if (rc) goto free_cmd_ctx; funq = fdev->admin_q; funq->cq_vector = 0; rc = fun_request_irq(funq, dev_name(fdev->dev), fun_admin_irq, funq); if (rc) goto free_sbq; fun_set_cq_callback(funq, fun_complete_admin_cmd, NULL); fdev->adminq_cb = areq->event_cb; writel((funq->sq_depth - 1) << AQA_ASQS_SHIFT | (funq->cq_depth - 1) << AQA_ACQS_SHIFT, fdev->bar + NVME_REG_AQA); writeq(funq->sq_dma_addr, fdev->bar + NVME_REG_ASQ); writeq(funq->cq_dma_addr, fdev->bar + NVME_REG_ACQ); rc = fun_enable_ctrl(fdev, areq->cqe_size_log2, areq->sqe_size_log2); if (rc) goto free_irq; if (areq->rq_depth) { rc = fun_create_rq(funq); if (rc) goto disable_ctrl; funq_rq_post(funq); } return 0; disable_ctrl: fun_disable_ctrl(fdev); free_irq: fun_free_irq(funq); free_sbq: sbitmap_queue_free(&fdev->admin_sbq); free_cmd_ctx: kvfree(fdev->cmd_ctx); fdev->cmd_ctx = NULL; free_q: fun_free_queue(fdev->admin_q); fdev->admin_q = NULL; return rc; } static void fun_disable_admin_queue(struct fun_dev *fdev) { struct fun_queue *admq = fdev->admin_q; if (!admq) return; fun_disable_ctrl(fdev); fun_free_irq(admq); __fun_process_cq(admq, 0); sbitmap_queue_free(&fdev->admin_sbq); kvfree(fdev->cmd_ctx); fdev->cmd_ctx = NULL; fun_free_queue(admq); fdev->admin_q = NULL; } /* Return %true if the admin queue has stopped servicing commands as can be * detected through registers. This isn't exhaustive and may provide false * negatives. */ static bool fun_adminq_stopped(struct fun_dev *fdev) { u32 csts = readl(fdev->bar + NVME_REG_CSTS); return (csts & (NVME_CSTS_CFS | NVME_CSTS_RDY)) != NVME_CSTS_RDY; } static int fun_wait_for_tag(struct fun_dev *fdev, int *cpup) { struct sbitmap_queue *sbq = &fdev->admin_sbq; struct sbq_wait_state *ws = &sbq->ws[0]; DEFINE_SBQ_WAIT(wait); int tag; for (;;) { sbitmap_prepare_to_wait(sbq, ws, &wait, TASK_UNINTERRUPTIBLE); if (fdev->suppress_cmds) { tag = -ESHUTDOWN; break; } tag = sbitmap_queue_get(sbq, cpup); if (tag >= 0) break; schedule(); } sbitmap_finish_wait(sbq, ws, &wait); return tag; } /* Submit an asynchronous admin command. Caller is responsible for implementing * any waiting or timeout. Upon command completion the callback @cb is called. */ int fun_submit_admin_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd, fun_admin_callback_t cb, void *cb_data, bool wait_ok) { struct fun_queue *funq = fdev->admin_q; unsigned int cmdsize = cmd->len8 * 8; struct fun_cmd_ctx *cmd_ctx; int tag, cpu, rc = 0; if (WARN_ON(cmdsize > (1 << funq->sqe_size_log2))) return -EMSGSIZE; tag = sbitmap_queue_get(&fdev->admin_sbq, &cpu); if (tag < 0) { if (!wait_ok) return -EAGAIN; tag = fun_wait_for_tag(fdev, &cpu); if (tag < 0) return tag; } cmd->cid = cpu_to_be16(tag); cmd_ctx = &fdev->cmd_ctx[tag]; cmd_ctx->cb = cb; cmd_ctx->cb_data = cb_data; spin_lock(&funq->sq_lock); if (unlikely(fdev->suppress_cmds)) { rc = -ESHUTDOWN; sbitmap_queue_clear(&fdev->admin_sbq, tag, cpu); } else { cmd_ctx->cpu = cpu; memcpy(fun_sqe_at(funq, funq->sq_tail), cmd, cmdsize); dev_dbg(fdev->dev, "admin cmd @ %u: %8ph\n", funq->sq_tail, cmd); if (++funq->sq_tail == funq->sq_depth) funq->sq_tail = 0; writel(funq->sq_tail, funq->sq_db); } spin_unlock(&funq->sq_lock); return rc; } /* Abandon a pending admin command by clearing the issuer's callback data. * Failure indicates that the command either has already completed or its * completion is racing with this call. */ static bool fun_abandon_admin_cmd(struct fun_dev *fd, const struct fun_admin_req_common *cmd, void *cb_data) { u16 cid = be16_to_cpu(cmd->cid); struct fun_cmd_ctx *cmd_ctx = &fd->cmd_ctx[cid]; return cmpxchg(&cmd_ctx->cb_data, cb_data, NULL) == cb_data; } /* Stop submission of new admin commands and wake up any processes waiting for * tags. Already submitted commands are left to complete or time out. */ static void fun_admin_stop(struct fun_dev *fdev) { spin_lock(&fdev->admin_q->sq_lock); fdev->suppress_cmds = true; spin_unlock(&fdev->admin_q->sq_lock); sbitmap_queue_wake_all(&fdev->admin_sbq); } /* The callback for synchronous execution of admin commands. It copies the * command response to the caller's buffer and signals completion. */ static void fun_admin_cmd_sync_cb(struct fun_dev *fd, void *rsp, void *cb_data) { const struct fun_admin_rsp_common *rsp_common = rsp; struct fun_sync_cmd_ctx *ctx = cb_data; if (!ctx) return; /* command issuer timed out and left */ if (ctx->rsp_buf) { unsigned int rsp_len = rsp_common->len8 * 8; if (unlikely(rsp_len > ctx->rsp_len)) { dev_err(fd->dev, "response for op %u is %uB > response buffer %uB\n", rsp_common->op, rsp_len, ctx->rsp_len); rsp_len = ctx->rsp_len; } memcpy(ctx->rsp_buf, rsp, rsp_len); } ctx->rsp_status = rsp_common->ret; complete(&ctx->compl); } /* Submit a synchronous admin command. */ int fun_submit_admin_sync_cmd(struct fun_dev *fdev, struct fun_admin_req_common *cmd, void *rsp, size_t rspsize, unsigned int timeout) { struct fun_sync_cmd_ctx ctx = { .compl = COMPLETION_INITIALIZER_ONSTACK(ctx.compl), .rsp_buf = rsp, .rsp_len = rspsize, }; unsigned int cmdlen = cmd->len8 * 8; unsigned long jiffies_left; int ret; ret = fun_submit_admin_cmd(fdev, cmd, fun_admin_cmd_sync_cb, &ctx, true); if (ret) return ret; if (!timeout) timeout = FUN_ADMIN_CMD_TO_MS; jiffies_left = wait_for_completion_timeout(&ctx.compl, msecs_to_jiffies(timeout)); if (!jiffies_left) { /* The command timed out. Attempt to cancel it so we can return. * But if the command is in the process of completing we'll * wait for it. */ if (fun_abandon_admin_cmd(fdev, cmd, &ctx)) { dev_err(fdev->dev, "admin command timed out: %*ph\n", cmdlen, cmd); fun_admin_stop(fdev); /* see if the timeout was due to a queue failure */ if (fun_adminq_stopped(fdev)) dev_err(fdev->dev, "device does not accept admin commands\n"); return -ETIMEDOUT; } wait_for_completion(&ctx.compl); } if (ctx.rsp_status) { dev_err(fdev->dev, "admin command failed, err %d: %*ph\n", ctx.rsp_status, cmdlen, cmd); } return -ctx.rsp_status; } EXPORT_SYMBOL_GPL(fun_submit_admin_sync_cmd); /* Return the number of device resources of the requested type. */ int fun_get_res_count(struct fun_dev *fdev, enum fun_admin_op res) { union { struct fun_admin_res_count_req req; struct fun_admin_res_count_rsp rsp; } cmd; int rc; cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(cmd.req)); cmd.req.count = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_RES_COUNT, 0, 0); rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common, &cmd.rsp, sizeof(cmd), 0); return rc ? rc : be32_to_cpu(cmd.rsp.count.data); } EXPORT_SYMBOL_GPL(fun_get_res_count); /* Request that the instance of resource @res with the given id be deleted. */ int fun_res_destroy(struct fun_dev *fdev, enum fun_admin_op res, unsigned int flags, u32 id) { struct fun_admin_generic_destroy_req req = { .common = FUN_ADMIN_REQ_COMMON_INIT2(res, sizeof(req)), .destroy = FUN_ADMIN_SIMPLE_SUBOP_INIT(FUN_ADMIN_SUBOP_DESTROY, flags, id) }; return fun_submit_admin_sync_cmd(fdev, &req.common, NULL, 0, 0); } EXPORT_SYMBOL_GPL(fun_res_destroy); /* Bind two entities of the given types and IDs. */ int fun_bind(struct fun_dev *fdev, enum fun_admin_bind_type type0, unsigned int id0, enum fun_admin_bind_type type1, unsigned int id1) { struct { struct fun_admin_bind_req req; struct fun_admin_bind_entry entry[2]; } cmd = { .req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_BIND, sizeof(cmd)), .entry[0] = FUN_ADMIN_BIND_ENTRY_INIT(type0, id0), .entry[1] = FUN_ADMIN_BIND_ENTRY_INIT(type1, id1), }; return fun_submit_admin_sync_cmd(fdev, &cmd.req.common, NULL, 0, 0); } EXPORT_SYMBOL_GPL(fun_bind); static int fun_get_dev_limits(struct fun_dev *fdev) { struct pci_dev *pdev = to_pci_dev(fdev->dev); unsigned int cq_count, sq_count, num_dbs; int rc; rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPCQ); if (rc < 0) return rc; cq_count = rc; rc = fun_get_res_count(fdev, FUN_ADMIN_OP_EPSQ); if (rc < 0) return rc; sq_count = rc; /* The admin queue consumes 1 CQ and at least 1 SQ. To be usable the * device must provide additional queues. */ if (cq_count < 2 || sq_count < 2 + !!fdev->admin_q->rq_depth) return -EINVAL; /* Calculate the max QID based on SQ/CQ/doorbell counts. * SQ/CQ doorbells alternate. */ num_dbs = (pci_resource_len(pdev, 0) - NVME_REG_DBS) / (fdev->db_stride * 4); fdev->max_qid = min3(cq_count, sq_count, num_dbs / 2) - 1; fdev->kern_end_qid = fdev->max_qid + 1; return 0; } /* Allocate all MSI-X vectors available on a function and at least @min_vecs. */ static int fun_alloc_irqs(struct pci_dev *pdev, unsigned int min_vecs) { int vecs, num_msix = pci_msix_vec_count(pdev); if (num_msix < 0) return num_msix; if (min_vecs > num_msix) return -ERANGE; vecs = pci_alloc_irq_vectors(pdev, min_vecs, num_msix, PCI_IRQ_MSIX); if (vecs > 0) { dev_info(&pdev->dev, "Allocated %d IRQ vectors of %d requested\n", vecs, num_msix); } else { dev_err(&pdev->dev, "Unable to allocate at least %u IRQ vectors\n", min_vecs); } return vecs; } /* Allocate and initialize the IRQ manager state. */ static int fun_alloc_irq_mgr(struct fun_dev *fdev) { fdev->irq_map = bitmap_zalloc(fdev->num_irqs, GFP_KERNEL); if (!fdev->irq_map) return -ENOMEM; spin_lock_init(&fdev->irqmgr_lock); /* mark IRQ 0 allocated, it is used by the admin queue */ __set_bit(0, fdev->irq_map); fdev->irqs_avail = fdev->num_irqs - 1; return 0; } /* Reserve @nirqs of the currently available IRQs and return their indices. */ int fun_reserve_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices) { unsigned int b, n = 0; int err = -ENOSPC; if (!nirqs) return 0; spin_lock(&fdev->irqmgr_lock); if (nirqs > fdev->irqs_avail) goto unlock; for_each_clear_bit(b, fdev->irq_map, fdev->num_irqs) { __set_bit(b, fdev->irq_map); irq_indices[n++] = b; if (n >= nirqs) break; } WARN_ON(n < nirqs); fdev->irqs_avail -= n; err = n; unlock: spin_unlock(&fdev->irqmgr_lock); return err; } EXPORT_SYMBOL(fun_reserve_irqs); /* Release @nirqs previously allocated IRQS with the supplied indices. */ void fun_release_irqs(struct fun_dev *fdev, unsigned int nirqs, u16 *irq_indices) { unsigned int i; spin_lock(&fdev->irqmgr_lock); for (i = 0; i < nirqs; i++) __clear_bit(irq_indices[i], fdev->irq_map); fdev->irqs_avail += nirqs; spin_unlock(&fdev->irqmgr_lock); } EXPORT_SYMBOL(fun_release_irqs); static void fun_serv_handler(struct work_struct *work) { struct fun_dev *fd = container_of(work, struct fun_dev, service_task); if (test_bit(FUN_SERV_DISABLED, &fd->service_flags)) return; if (fd->serv_cb) fd->serv_cb(fd); } void fun_serv_stop(struct fun_dev *fd) { set_bit(FUN_SERV_DISABLED, &fd->service_flags); cancel_work_sync(&fd->service_task); } EXPORT_SYMBOL_GPL(fun_serv_stop); void fun_serv_restart(struct fun_dev *fd) { clear_bit(FUN_SERV_DISABLED, &fd->service_flags); if (fd->service_flags) schedule_work(&fd->service_task); } EXPORT_SYMBOL_GPL(fun_serv_restart); void fun_serv_sched(struct fun_dev *fd) { if (!test_bit(FUN_SERV_DISABLED, &fd->service_flags)) schedule_work(&fd->service_task); } EXPORT_SYMBOL_GPL(fun_serv_sched); /* Check and try to get the device into a proper state for initialization, * i.e., CSTS.RDY = CC.EN = 0. */ static int sanitize_dev(struct fun_dev *fdev) { int rc; fdev->cap_reg = readq(fdev->bar + NVME_REG_CAP); fdev->cc_reg = readl(fdev->bar + NVME_REG_CC); /* First get RDY to agree with the current EN. Give RDY the opportunity * to complete a potential recent EN change. */ rc = fun_wait_ready(fdev, fdev->cc_reg & NVME_CC_ENABLE); if (rc) return rc; /* Next, reset the device if EN is currently 1. */ if (fdev->cc_reg & NVME_CC_ENABLE) rc = fun_disable_ctrl(fdev); return rc; } /* Undo the device initialization of fun_dev_enable(). */ void fun_dev_disable(struct fun_dev *fdev) { struct pci_dev *pdev = to_pci_dev(fdev->dev); pci_set_drvdata(pdev, NULL); if (fdev->fw_handle != FUN_HCI_ID_INVALID) { fun_res_destroy(fdev, FUN_ADMIN_OP_SWUPGRADE, 0, fdev->fw_handle); fdev->fw_handle = FUN_HCI_ID_INVALID; } fun_disable_admin_queue(fdev); bitmap_free(fdev->irq_map); pci_free_irq_vectors(pdev); pci_clear_master(pdev); pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); fun_unmap_bars(fdev); } EXPORT_SYMBOL(fun_dev_disable); /* Perform basic initialization of a device, including * - PCI config space setup and BAR0 mapping * - interrupt management initialization * - 1 admin queue setup * - determination of some device limits, such as number of queues. */ int fun_dev_enable(struct fun_dev *fdev, struct pci_dev *pdev, const struct fun_dev_params *areq, const char *name) { int rc; fdev->dev = &pdev->dev; rc = fun_map_bars(fdev, name); if (rc) return rc; rc = fun_set_dma_masks(fdev->dev); if (rc) goto unmap; rc = pci_enable_device_mem(pdev); if (rc) { dev_err(&pdev->dev, "Couldn't enable device, err %d\n", rc); goto unmap; } pci_enable_pcie_error_reporting(pdev); rc = sanitize_dev(fdev); if (rc) goto disable_dev; fdev->fw_handle = FUN_HCI_ID_INVALID; fdev->q_depth = NVME_CAP_MQES(fdev->cap_reg) + 1; fdev->db_stride = 1 << NVME_CAP_STRIDE(fdev->cap_reg); fdev->dbs = fdev->bar + NVME_REG_DBS; INIT_WORK(&fdev->service_task, fun_serv_handler); fdev->service_flags = FUN_SERV_DISABLED; fdev->serv_cb = areq->serv_cb; rc = fun_alloc_irqs(pdev, areq->min_msix + 1); /* +1 for admin CQ */ if (rc < 0) goto disable_dev; fdev->num_irqs = rc; rc = fun_alloc_irq_mgr(fdev); if (rc) goto free_irqs; pci_set_master(pdev); rc = fun_enable_admin_queue(fdev, areq); if (rc) goto free_irq_mgr; rc = fun_get_dev_limits(fdev); if (rc < 0) goto disable_admin; pci_save_state(pdev); pci_set_drvdata(pdev, fdev); pcie_print_link_status(pdev); dev_dbg(fdev->dev, "q_depth %u, db_stride %u, max qid %d kern_end_qid %d\n", fdev->q_depth, fdev->db_stride, fdev->max_qid, fdev->kern_end_qid); return 0; disable_admin: fun_disable_admin_queue(fdev); free_irq_mgr: pci_clear_master(pdev); bitmap_free(fdev->irq_map); free_irqs: pci_free_irq_vectors(pdev); disable_dev: pci_disable_pcie_error_reporting(pdev); pci_disable_device(pdev); unmap: fun_unmap_bars(fdev); return rc; } EXPORT_SYMBOL(fun_dev_enable); MODULE_AUTHOR("Dimitris Michailidis "); MODULE_DESCRIPTION("Core services driver for Fungible devices"); MODULE_LICENSE("Dual BSD/GPL");