diff options
Diffstat (limited to 'drivers/scsi/cxlflash/ocxl_hw.c')
| -rw-r--r-- | drivers/scsi/cxlflash/ocxl_hw.c | 1399 |
1 files changed, 0 insertions, 1399 deletions
diff --git a/drivers/scsi/cxlflash/ocxl_hw.c b/drivers/scsi/cxlflash/ocxl_hw.c deleted file mode 100644 index 6542818e595a..000000000000 --- a/drivers/scsi/cxlflash/ocxl_hw.c +++ /dev/null @@ -1,1399 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * CXL Flash Device Driver - * - * Written by: Matthew R. Ochs <mrochs@linux.vnet.ibm.com>, IBM Corporation - * Uma Krishnan <ukrishn@linux.vnet.ibm.com>, IBM Corporation - * - * Copyright (C) 2018 IBM Corporation - */ - -#include <linux/file.h> -#include <linux/idr.h> -#include <linux/module.h> -#include <linux/mount.h> -#include <linux/pseudo_fs.h> -#include <linux/poll.h> -#include <linux/sched/signal.h> -#include <linux/interrupt.h> -#include <linux/irqdomain.h> -#include <asm/xive.h> -#include <misc/ocxl.h> - -#include <uapi/misc/cxl.h> - -#include "backend.h" -#include "ocxl_hw.h" - -/* - * Pseudo-filesystem to allocate inodes. - */ - -#define OCXLFLASH_FS_MAGIC 0x1697698f - -static int ocxlflash_fs_cnt; -static struct vfsmount *ocxlflash_vfs_mount; - -static int ocxlflash_fs_init_fs_context(struct fs_context *fc) -{ - return init_pseudo(fc, OCXLFLASH_FS_MAGIC) ? 0 : -ENOMEM; -} - -static struct file_system_type ocxlflash_fs_type = { - .name = "ocxlflash", - .owner = THIS_MODULE, - .init_fs_context = ocxlflash_fs_init_fs_context, - .kill_sb = kill_anon_super, -}; - -/* - * ocxlflash_release_mapping() - release the memory mapping - * @ctx: Context whose mapping is to be released. - */ -static void ocxlflash_release_mapping(struct ocxlflash_context *ctx) -{ - if (ctx->mapping) - simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt); - ctx->mapping = NULL; -} - -/* - * ocxlflash_getfile() - allocate pseudo filesystem, inode, and the file - * @dev: Generic device of the host. - * @name: Name of the pseudo filesystem. - * @fops: File operations. - * @priv: Private data. - * @flags: Flags for the file. - * - * Return: pointer to the file on success, ERR_PTR on failure - */ -static struct file *ocxlflash_getfile(struct device *dev, const char *name, - const struct file_operations *fops, - void *priv, int flags) -{ - struct file *file; - struct inode *inode; - int rc; - - if (fops->owner && !try_module_get(fops->owner)) { - dev_err(dev, "%s: Owner does not exist\n", __func__); - rc = -ENOENT; - goto err1; - } - - rc = simple_pin_fs(&ocxlflash_fs_type, &ocxlflash_vfs_mount, - &ocxlflash_fs_cnt); - if (unlikely(rc < 0)) { - dev_err(dev, "%s: Cannot mount ocxlflash pseudofs rc=%d\n", - __func__, rc); - goto err2; - } - - inode = alloc_anon_inode(ocxlflash_vfs_mount->mnt_sb); - if (IS_ERR(inode)) { - rc = PTR_ERR(inode); - dev_err(dev, "%s: alloc_anon_inode failed rc=%d\n", - __func__, rc); - goto err3; - } - - file = alloc_file_pseudo(inode, ocxlflash_vfs_mount, name, - flags & (O_ACCMODE | O_NONBLOCK), fops); - if (IS_ERR(file)) { - rc = PTR_ERR(file); - dev_err(dev, "%s: alloc_file failed rc=%d\n", - __func__, rc); - goto err4; - } - - file->private_data = priv; -out: - return file; -err4: - iput(inode); -err3: - simple_release_fs(&ocxlflash_vfs_mount, &ocxlflash_fs_cnt); -err2: - module_put(fops->owner); -err1: - file = ERR_PTR(rc); - goto out; -} - -/** - * ocxlflash_psa_map() - map the process specific MMIO space - * @ctx_cookie: Adapter context for which the mapping needs to be done. - * - * Return: MMIO pointer of the mapped region - */ -static void __iomem *ocxlflash_psa_map(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct device *dev = ctx->hw_afu->dev; - - mutex_lock(&ctx->state_mutex); - if (ctx->state != STARTED) { - dev_err(dev, "%s: Context not started, state=%d\n", __func__, - ctx->state); - mutex_unlock(&ctx->state_mutex); - return NULL; - } - mutex_unlock(&ctx->state_mutex); - - return ioremap(ctx->psn_phys, ctx->psn_size); -} - -/** - * ocxlflash_psa_unmap() - unmap the process specific MMIO space - * @addr: MMIO pointer to unmap. - */ -static void ocxlflash_psa_unmap(void __iomem *addr) -{ - iounmap(addr); -} - -/** - * ocxlflash_process_element() - get process element of the adapter context - * @ctx_cookie: Adapter context associated with the process element. - * - * Return: process element of the adapter context - */ -static int ocxlflash_process_element(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - - return ctx->pe; -} - -/** - * afu_map_irq() - map the interrupt of the adapter context - * @flags: Flags. - * @ctx: Adapter context. - * @num: Per-context AFU interrupt number. - * @handler: Interrupt handler to register. - * @cookie: Interrupt handler private data. - * @name: Name of the interrupt. - * - * Return: 0 on success, -errno on failure - */ -static int afu_map_irq(u64 flags, struct ocxlflash_context *ctx, int num, - irq_handler_t handler, void *cookie, char *name) -{ - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct device *dev = afu->dev; - struct ocxlflash_irqs *irq; - struct xive_irq_data *xd; - u32 virq; - int rc = 0; - - if (num < 0 || num >= ctx->num_irqs) { - dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num); - rc = -ENOENT; - goto out; - } - - irq = &ctx->irqs[num]; - virq = irq_create_mapping(NULL, irq->hwirq); - if (unlikely(!virq)) { - dev_err(dev, "%s: irq_create_mapping failed\n", __func__); - rc = -ENOMEM; - goto out; - } - - rc = request_irq(virq, handler, 0, name, cookie); - if (unlikely(rc)) { - dev_err(dev, "%s: request_irq failed rc=%d\n", __func__, rc); - goto err1; - } - - xd = irq_get_handler_data(virq); - if (unlikely(!xd)) { - dev_err(dev, "%s: Can't get interrupt data\n", __func__); - rc = -ENXIO; - goto err2; - } - - irq->virq = virq; - irq->vtrig = xd->trig_mmio; -out: - return rc; -err2: - free_irq(virq, cookie); -err1: - irq_dispose_mapping(virq); - goto out; -} - -/** - * ocxlflash_map_afu_irq() - map the interrupt of the adapter context - * @ctx_cookie: Adapter context. - * @num: Per-context AFU interrupt number. - * @handler: Interrupt handler to register. - * @cookie: Interrupt handler private data. - * @name: Name of the interrupt. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_map_afu_irq(void *ctx_cookie, int num, - irq_handler_t handler, void *cookie, - char *name) -{ - return afu_map_irq(0, ctx_cookie, num, handler, cookie, name); -} - -/** - * afu_unmap_irq() - unmap the interrupt - * @flags: Flags. - * @ctx: Adapter context. - * @num: Per-context AFU interrupt number. - * @cookie: Interrupt handler private data. - */ -static void afu_unmap_irq(u64 flags, struct ocxlflash_context *ctx, int num, - void *cookie) -{ - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct device *dev = afu->dev; - struct ocxlflash_irqs *irq; - - if (num < 0 || num >= ctx->num_irqs) { - dev_err(dev, "%s: Interrupt %d not allocated\n", __func__, num); - return; - } - - irq = &ctx->irqs[num]; - - if (irq_find_mapping(NULL, irq->hwirq)) { - free_irq(irq->virq, cookie); - irq_dispose_mapping(irq->virq); - } - - memset(irq, 0, sizeof(*irq)); -} - -/** - * ocxlflash_unmap_afu_irq() - unmap the interrupt - * @ctx_cookie: Adapter context. - * @num: Per-context AFU interrupt number. - * @cookie: Interrupt handler private data. - */ -static void ocxlflash_unmap_afu_irq(void *ctx_cookie, int num, void *cookie) -{ - return afu_unmap_irq(0, ctx_cookie, num, cookie); -} - -/** - * ocxlflash_get_irq_objhndl() - get the object handle for an interrupt - * @ctx_cookie: Context associated with the interrupt. - * @irq: Interrupt number. - * - * Return: effective address of the mapped region - */ -static u64 ocxlflash_get_irq_objhndl(void *ctx_cookie, int irq) -{ - struct ocxlflash_context *ctx = ctx_cookie; - - if (irq < 0 || irq >= ctx->num_irqs) - return 0; - - return (__force u64)ctx->irqs[irq].vtrig; -} - -/** - * ocxlflash_xsl_fault() - callback when translation error is triggered - * @data: Private data provided at callback registration, the context. - * @addr: Address that triggered the error. - * @dsisr: Value of dsisr register. - */ -static void ocxlflash_xsl_fault(void *data, u64 addr, u64 dsisr) -{ - struct ocxlflash_context *ctx = data; - - spin_lock(&ctx->slock); - ctx->fault_addr = addr; - ctx->fault_dsisr = dsisr; - ctx->pending_fault = true; - spin_unlock(&ctx->slock); - - wake_up_all(&ctx->wq); -} - -/** - * start_context() - local routine to start a context - * @ctx: Adapter context to be started. - * - * Assign the context specific MMIO space, add and enable the PE. - * - * Return: 0 on success, -errno on failure - */ -static int start_context(struct ocxlflash_context *ctx) -{ - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct ocxl_afu_config *acfg = &afu->acfg; - void *link_token = afu->link_token; - struct pci_dev *pdev = afu->pdev; - struct device *dev = afu->dev; - bool master = ctx->master; - struct mm_struct *mm; - int rc = 0; - u32 pid; - - mutex_lock(&ctx->state_mutex); - if (ctx->state != OPENED) { - dev_err(dev, "%s: Context state invalid, state=%d\n", - __func__, ctx->state); - rc = -EINVAL; - goto out; - } - - if (master) { - ctx->psn_size = acfg->global_mmio_size; - ctx->psn_phys = afu->gmmio_phys; - } else { - ctx->psn_size = acfg->pp_mmio_stride; - ctx->psn_phys = afu->ppmmio_phys + (ctx->pe * ctx->psn_size); - } - - /* pid and mm not set for master contexts */ - if (master) { - pid = 0; - mm = NULL; - } else { - pid = current->mm->context.id; - mm = current->mm; - } - - rc = ocxl_link_add_pe(link_token, ctx->pe, pid, 0, 0, - pci_dev_id(pdev), mm, ocxlflash_xsl_fault, - ctx); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_link_add_pe failed rc=%d\n", - __func__, rc); - goto out; - } - - ctx->state = STARTED; -out: - mutex_unlock(&ctx->state_mutex); - return rc; -} - -/** - * ocxlflash_start_context() - start a kernel context - * @ctx_cookie: Adapter context to be started. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_start_context(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - - return start_context(ctx); -} - -/** - * ocxlflash_stop_context() - stop a context - * @ctx_cookie: Adapter context to be stopped. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_stop_context(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct ocxl_afu_config *acfg = &afu->acfg; - struct pci_dev *pdev = afu->pdev; - struct device *dev = afu->dev; - enum ocxlflash_ctx_state state; - int rc = 0; - - mutex_lock(&ctx->state_mutex); - state = ctx->state; - ctx->state = CLOSED; - mutex_unlock(&ctx->state_mutex); - if (state != STARTED) - goto out; - - rc = ocxl_config_terminate_pasid(pdev, acfg->dvsec_afu_control_pos, - ctx->pe); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_config_terminate_pasid failed rc=%d\n", - __func__, rc); - /* If EBUSY, PE could be referenced in future by the AFU */ - if (rc == -EBUSY) - goto out; - } - - rc = ocxl_link_remove_pe(afu->link_token, ctx->pe); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_link_remove_pe failed rc=%d\n", - __func__, rc); - goto out; - } -out: - return rc; -} - -/** - * ocxlflash_afu_reset() - reset the AFU - * @ctx_cookie: Adapter context. - */ -static int ocxlflash_afu_reset(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct device *dev = ctx->hw_afu->dev; - - /* Pending implementation from OCXL transport services */ - dev_err_once(dev, "%s: afu_reset() fop not supported\n", __func__); - - /* Silently return success until it is implemented */ - return 0; -} - -/** - * ocxlflash_set_master() - sets the context as master - * @ctx_cookie: Adapter context to set as master. - */ -static void ocxlflash_set_master(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - - ctx->master = true; -} - -/** - * ocxlflash_get_context() - obtains the context associated with the host - * @pdev: PCI device associated with the host. - * @afu_cookie: Hardware AFU associated with the host. - * - * Return: returns the pointer to host adapter context - */ -static void *ocxlflash_get_context(struct pci_dev *pdev, void *afu_cookie) -{ - struct ocxl_hw_afu *afu = afu_cookie; - - return afu->ocxl_ctx; -} - -/** - * ocxlflash_dev_context_init() - allocate and initialize an adapter context - * @pdev: PCI device associated with the host. - * @afu_cookie: Hardware AFU associated with the host. - * - * Return: returns the adapter context on success, ERR_PTR on failure - */ -static void *ocxlflash_dev_context_init(struct pci_dev *pdev, void *afu_cookie) -{ - struct ocxl_hw_afu *afu = afu_cookie; - struct device *dev = afu->dev; - struct ocxlflash_context *ctx; - int rc; - - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); - if (unlikely(!ctx)) { - dev_err(dev, "%s: Context allocation failed\n", __func__); - rc = -ENOMEM; - goto err1; - } - - idr_preload(GFP_KERNEL); - rc = idr_alloc(&afu->idr, ctx, 0, afu->max_pasid, GFP_NOWAIT); - idr_preload_end(); - if (unlikely(rc < 0)) { - dev_err(dev, "%s: idr_alloc failed rc=%d\n", __func__, rc); - goto err2; - } - - spin_lock_init(&ctx->slock); - init_waitqueue_head(&ctx->wq); - mutex_init(&ctx->state_mutex); - - ctx->state = OPENED; - ctx->pe = rc; - ctx->master = false; - ctx->mapping = NULL; - ctx->hw_afu = afu; - ctx->irq_bitmap = 0; - ctx->pending_irq = false; - ctx->pending_fault = false; -out: - return ctx; -err2: - kfree(ctx); -err1: - ctx = ERR_PTR(rc); - goto out; -} - -/** - * ocxlflash_release_context() - releases an adapter context - * @ctx_cookie: Adapter context to be released. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_release_context(void *ctx_cookie) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct device *dev; - int rc = 0; - - if (!ctx) - goto out; - - dev = ctx->hw_afu->dev; - mutex_lock(&ctx->state_mutex); - if (ctx->state >= STARTED) { - dev_err(dev, "%s: Context in use, state=%d\n", __func__, - ctx->state); - mutex_unlock(&ctx->state_mutex); - rc = -EBUSY; - goto out; - } - mutex_unlock(&ctx->state_mutex); - - idr_remove(&ctx->hw_afu->idr, ctx->pe); - ocxlflash_release_mapping(ctx); - kfree(ctx); -out: - return rc; -} - -/** - * ocxlflash_perst_reloads_same_image() - sets the image reload policy - * @afu_cookie: Hardware AFU associated with the host. - * @image: Whether to load the same image on PERST. - */ -static void ocxlflash_perst_reloads_same_image(void *afu_cookie, bool image) -{ - struct ocxl_hw_afu *afu = afu_cookie; - - afu->perst_same_image = image; -} - -/** - * ocxlflash_read_adapter_vpd() - reads the adapter VPD - * @pdev: PCI device associated with the host. - * @buf: Buffer to get the VPD data. - * @count: Size of buffer (maximum bytes that can be read). - * - * Return: size of VPD on success, -errno on failure - */ -static ssize_t ocxlflash_read_adapter_vpd(struct pci_dev *pdev, void *buf, - size_t count) -{ - return pci_read_vpd(pdev, 0, count, buf); -} - -/** - * free_afu_irqs() - internal service to free interrupts - * @ctx: Adapter context. - */ -static void free_afu_irqs(struct ocxlflash_context *ctx) -{ - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct device *dev = afu->dev; - int i; - - if (!ctx->irqs) { - dev_err(dev, "%s: Interrupts not allocated\n", __func__); - return; - } - - for (i = ctx->num_irqs; i >= 0; i--) - ocxl_link_free_irq(afu->link_token, ctx->irqs[i].hwirq); - - kfree(ctx->irqs); - ctx->irqs = NULL; -} - -/** - * alloc_afu_irqs() - internal service to allocate interrupts - * @ctx: Context associated with the request. - * @num: Number of interrupts requested. - * - * Return: 0 on success, -errno on failure - */ -static int alloc_afu_irqs(struct ocxlflash_context *ctx, int num) -{ - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct device *dev = afu->dev; - struct ocxlflash_irqs *irqs; - int rc = 0; - int hwirq; - int i; - - if (ctx->irqs) { - dev_err(dev, "%s: Interrupts already allocated\n", __func__); - rc = -EEXIST; - goto out; - } - - if (num > OCXL_MAX_IRQS) { - dev_err(dev, "%s: Too many interrupts num=%d\n", __func__, num); - rc = -EINVAL; - goto out; - } - - irqs = kcalloc(num, sizeof(*irqs), GFP_KERNEL); - if (unlikely(!irqs)) { - dev_err(dev, "%s: Context irqs allocation failed\n", __func__); - rc = -ENOMEM; - goto out; - } - - for (i = 0; i < num; i++) { - rc = ocxl_link_irq_alloc(afu->link_token, &hwirq); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_link_irq_alloc failed rc=%d\n", - __func__, rc); - goto err; - } - - irqs[i].hwirq = hwirq; - } - - ctx->irqs = irqs; - ctx->num_irqs = num; -out: - return rc; -err: - for (i = i-1; i >= 0; i--) - ocxl_link_free_irq(afu->link_token, irqs[i].hwirq); - kfree(irqs); - goto out; -} - -/** - * ocxlflash_allocate_afu_irqs() - allocates the requested number of interrupts - * @ctx_cookie: Context associated with the request. - * @num: Number of interrupts requested. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_allocate_afu_irqs(void *ctx_cookie, int num) -{ - return alloc_afu_irqs(ctx_cookie, num); -} - -/** - * ocxlflash_free_afu_irqs() - frees the interrupts of an adapter context - * @ctx_cookie: Adapter context. - */ -static void ocxlflash_free_afu_irqs(void *ctx_cookie) -{ - free_afu_irqs(ctx_cookie); -} - -/** - * ocxlflash_unconfig_afu() - unconfigure the AFU - * @afu: AFU associated with the host. - */ -static void ocxlflash_unconfig_afu(struct ocxl_hw_afu *afu) -{ - if (afu->gmmio_virt) { - iounmap(afu->gmmio_virt); - afu->gmmio_virt = NULL; - } -} - -/** - * ocxlflash_destroy_afu() - destroy the AFU structure - * @afu_cookie: AFU to be freed. - */ -static void ocxlflash_destroy_afu(void *afu_cookie) -{ - struct ocxl_hw_afu *afu = afu_cookie; - int pos; - - if (!afu) - return; - - ocxlflash_release_context(afu->ocxl_ctx); - idr_destroy(&afu->idr); - - /* Disable the AFU */ - pos = afu->acfg.dvsec_afu_control_pos; - ocxl_config_set_afu_state(afu->pdev, pos, 0); - - ocxlflash_unconfig_afu(afu); - kfree(afu); -} - -/** - * ocxlflash_config_fn() - configure the host function - * @pdev: PCI device associated with the host. - * @afu: AFU associated with the host. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_config_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu) -{ - struct ocxl_fn_config *fcfg = &afu->fcfg; - struct device *dev = &pdev->dev; - u16 base, enabled, supported; - int rc = 0; - - /* Read DVSEC config of the function */ - rc = ocxl_config_read_function(pdev, fcfg); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_config_read_function failed rc=%d\n", - __func__, rc); - goto out; - } - - /* Check if function has AFUs defined, only 1 per function supported */ - if (fcfg->max_afu_index >= 0) { - afu->is_present = true; - if (fcfg->max_afu_index != 0) - dev_warn(dev, "%s: Unexpected AFU index value %d\n", - __func__, fcfg->max_afu_index); - } - - rc = ocxl_config_get_actag_info(pdev, &base, &enabled, &supported); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_config_get_actag_info failed rc=%d\n", - __func__, rc); - goto out; - } - - afu->fn_actag_base = base; - afu->fn_actag_enabled = enabled; - - ocxl_config_set_actag(pdev, fcfg->dvsec_function_pos, base, enabled); - dev_dbg(dev, "%s: Function acTag range base=%u enabled=%u\n", - __func__, base, enabled); - - rc = ocxl_link_setup(pdev, 0, &afu->link_token); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_link_setup failed rc=%d\n", - __func__, rc); - goto out; - } - - rc = ocxl_config_set_TL(pdev, fcfg->dvsec_tl_pos); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_config_set_TL failed rc=%d\n", - __func__, rc); - goto err; - } -out: - return rc; -err: - ocxl_link_release(pdev, afu->link_token); - goto out; -} - -/** - * ocxlflash_unconfig_fn() - unconfigure the host function - * @pdev: PCI device associated with the host. - * @afu: AFU associated with the host. - */ -static void ocxlflash_unconfig_fn(struct pci_dev *pdev, struct ocxl_hw_afu *afu) -{ - ocxl_link_release(pdev, afu->link_token); -} - -/** - * ocxlflash_map_mmio() - map the AFU MMIO space - * @afu: AFU associated with the host. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_map_mmio(struct ocxl_hw_afu *afu) -{ - struct ocxl_afu_config *acfg = &afu->acfg; - struct pci_dev *pdev = afu->pdev; - struct device *dev = afu->dev; - phys_addr_t gmmio, ppmmio; - int rc = 0; - - rc = pci_request_region(pdev, acfg->global_mmio_bar, "ocxlflash"); - if (unlikely(rc)) { - dev_err(dev, "%s: pci_request_region for global failed rc=%d\n", - __func__, rc); - goto out; - } - gmmio = pci_resource_start(pdev, acfg->global_mmio_bar); - gmmio += acfg->global_mmio_offset; - - rc = pci_request_region(pdev, acfg->pp_mmio_bar, "ocxlflash"); - if (unlikely(rc)) { - dev_err(dev, "%s: pci_request_region for pp bar failed rc=%d\n", - __func__, rc); - goto err1; - } - ppmmio = pci_resource_start(pdev, acfg->pp_mmio_bar); - ppmmio += acfg->pp_mmio_offset; - - afu->gmmio_virt = ioremap(gmmio, acfg->global_mmio_size); - if (unlikely(!afu->gmmio_virt)) { - dev_err(dev, "%s: MMIO mapping failed\n", __func__); - rc = -ENOMEM; - goto err2; - } - - afu->gmmio_phys = gmmio; - afu->ppmmio_phys = ppmmio; -out: - return rc; -err2: - pci_release_region(pdev, acfg->pp_mmio_bar); -err1: - pci_release_region(pdev, acfg->global_mmio_bar); - goto out; -} - -/** - * ocxlflash_config_afu() - configure the host AFU - * @pdev: PCI device associated with the host. - * @afu: AFU associated with the host. - * - * Must be called _after_ host function configuration. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_config_afu(struct pci_dev *pdev, struct ocxl_hw_afu *afu) -{ - struct ocxl_afu_config *acfg = &afu->acfg; - struct ocxl_fn_config *fcfg = &afu->fcfg; - struct device *dev = &pdev->dev; - int count; - int base; - int pos; - int rc = 0; - - /* This HW AFU function does not have any AFUs defined */ - if (!afu->is_present) - goto out; - - /* Read AFU config at index 0 */ - rc = ocxl_config_read_afu(pdev, fcfg, acfg, 0); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxl_config_read_afu failed rc=%d\n", - __func__, rc); - goto out; - } - - /* Only one AFU per function is supported, so actag_base is same */ - base = afu->fn_actag_base; - count = min_t(int, acfg->actag_supported, afu->fn_actag_enabled); - pos = acfg->dvsec_afu_control_pos; - - ocxl_config_set_afu_actag(pdev, pos, base, count); - dev_dbg(dev, "%s: acTag base=%d enabled=%d\n", __func__, base, count); - afu->afu_actag_base = base; - afu->afu_actag_enabled = count; - afu->max_pasid = 1 << acfg->pasid_supported_log; - - ocxl_config_set_afu_pasid(pdev, pos, 0, acfg->pasid_supported_log); - - rc = ocxlflash_map_mmio(afu); - if (unlikely(rc)) { - dev_err(dev, "%s: ocxlflash_map_mmio failed rc=%d\n", - __func__, rc); - goto out; - } - - /* Enable the AFU */ - ocxl_config_set_afu_state(pdev, acfg->dvsec_afu_control_pos, 1); -out: - return rc; -} - -/** - * ocxlflash_create_afu() - create the AFU for OCXL - * @pdev: PCI device associated with the host. - * - * Return: AFU on success, NULL on failure - */ -static void *ocxlflash_create_afu(struct pci_dev *pdev) -{ - struct device *dev = &pdev->dev; - struct ocxlflash_context *ctx; - struct ocxl_hw_afu *afu; - int rc; - - afu = kzalloc(sizeof(*afu), GFP_KERNEL); - if (unlikely(!afu)) { - dev_err(dev, "%s: HW AFU allocation failed\n", __func__); - goto out; - } - - afu->pdev = pdev; - afu->dev = dev; - idr_init(&afu->idr); - - rc = ocxlflash_config_fn(pdev, afu); - if (unlikely(rc)) { - dev_err(dev, "%s: Function configuration failed rc=%d\n", - __func__, rc); - goto err1; - } - - rc = ocxlflash_config_afu(pdev, afu); - if (unlikely(rc)) { - dev_err(dev, "%s: AFU configuration failed rc=%d\n", - __func__, rc); - goto err2; - } - - ctx = ocxlflash_dev_context_init(pdev, afu); - if (IS_ERR(ctx)) { - rc = PTR_ERR(ctx); - dev_err(dev, "%s: ocxlflash_dev_context_init failed rc=%d\n", - __func__, rc); - goto err3; - } - - afu->ocxl_ctx = ctx; -out: - return afu; -err3: - ocxlflash_unconfig_afu(afu); -err2: - ocxlflash_unconfig_fn(pdev, afu); -err1: - idr_destroy(&afu->idr); - kfree(afu); - afu = NULL; - goto out; -} - -/** - * ctx_event_pending() - check for any event pending on the context - * @ctx: Context to be checked. - * - * Return: true if there is an event pending, false if none pending - */ -static inline bool ctx_event_pending(struct ocxlflash_context *ctx) -{ - if (ctx->pending_irq || ctx->pending_fault) - return true; - - return false; -} - -/** - * afu_poll() - poll the AFU for events on the context - * @file: File associated with the adapter context. - * @poll: Poll structure from the user. - * - * Return: poll mask - */ -static unsigned int afu_poll(struct file *file, struct poll_table_struct *poll) -{ - struct ocxlflash_context *ctx = file->private_data; - struct device *dev = ctx->hw_afu->dev; - ulong lock_flags; - int mask = 0; - - poll_wait(file, &ctx->wq, poll); - - spin_lock_irqsave(&ctx->slock, lock_flags); - if (ctx_event_pending(ctx)) - mask |= POLLIN | POLLRDNORM; - else if (ctx->state == CLOSED) - mask |= POLLERR; - spin_unlock_irqrestore(&ctx->slock, lock_flags); - - dev_dbg(dev, "%s: Poll wait completed for pe %i mask %i\n", - __func__, ctx->pe, mask); - - return mask; -} - -/** - * afu_read() - perform a read on the context for any event - * @file: File associated with the adapter context. - * @buf: Buffer to receive the data. - * @count: Size of buffer (maximum bytes that can be read). - * @off: Offset. - * - * Return: size of the data read on success, -errno on failure - */ -static ssize_t afu_read(struct file *file, char __user *buf, size_t count, - loff_t *off) -{ - struct ocxlflash_context *ctx = file->private_data; - struct device *dev = ctx->hw_afu->dev; - struct cxl_event event; - ulong lock_flags; - ssize_t esize; - ssize_t rc; - int bit; - DEFINE_WAIT(event_wait); - - if (*off != 0) { - dev_err(dev, "%s: Non-zero offset not supported, off=%lld\n", - __func__, *off); - rc = -EINVAL; - goto out; - } - - spin_lock_irqsave(&ctx->slock, lock_flags); - - for (;;) { - prepare_to_wait(&ctx->wq, &event_wait, TASK_INTERRUPTIBLE); - - if (ctx_event_pending(ctx) || (ctx->state == CLOSED)) - break; - - if (file->f_flags & O_NONBLOCK) { - dev_err(dev, "%s: File cannot be blocked on I/O\n", - __func__); - rc = -EAGAIN; - goto err; - } - - if (signal_pending(current)) { - dev_err(dev, "%s: Signal pending on the process\n", - __func__); - rc = -ERESTARTSYS; - goto err; - } - - spin_unlock_irqrestore(&ctx->slock, lock_flags); - schedule(); - spin_lock_irqsave(&ctx->slock, lock_flags); - } - - finish_wait(&ctx->wq, &event_wait); - - memset(&event, 0, sizeof(event)); - event.header.process_element = ctx->pe; - event.header.size = sizeof(struct cxl_event_header); - if (ctx->pending_irq) { - esize = sizeof(struct cxl_event_afu_interrupt); - event.header.size += esize; - event.header.type = CXL_EVENT_AFU_INTERRUPT; - - bit = find_first_bit(&ctx->irq_bitmap, ctx->num_irqs); - clear_bit(bit, &ctx->irq_bitmap); - event.irq.irq = bit + 1; - if (bitmap_empty(&ctx->irq_bitmap, ctx->num_irqs)) - ctx->pending_irq = false; - } else if (ctx->pending_fault) { - event.header.size += sizeof(struct cxl_event_data_storage); - event.header.type = CXL_EVENT_DATA_STORAGE; - event.fault.addr = ctx->fault_addr; - event.fault.dsisr = ctx->fault_dsisr; - ctx->pending_fault = false; - } - - spin_unlock_irqrestore(&ctx->slock, lock_flags); - - if (copy_to_user(buf, &event, event.header.size)) { - dev_err(dev, "%s: copy_to_user failed\n", __func__); - rc = -EFAULT; - goto out; - } - - rc = event.header.size; -out: - return rc; -err: - finish_wait(&ctx->wq, &event_wait); - spin_unlock_irqrestore(&ctx->slock, lock_flags); - goto out; -} - -/** - * afu_release() - release and free the context - * @inode: File inode pointer. - * @file: File associated with the context. - * - * Return: 0 on success, -errno on failure - */ -static int afu_release(struct inode *inode, struct file *file) -{ - struct ocxlflash_context *ctx = file->private_data; - int i; - - /* Unmap and free the interrupts associated with the context */ - for (i = ctx->num_irqs; i >= 0; i--) - afu_unmap_irq(0, ctx, i, ctx); - free_afu_irqs(ctx); - - return ocxlflash_release_context(ctx); -} - -/** - * ocxlflash_mmap_fault() - mmap fault handler - * @vmf: VM fault associated with current fault. - * - * Return: 0 on success, -errno on failure - */ -static vm_fault_t ocxlflash_mmap_fault(struct vm_fault *vmf) -{ - struct vm_area_struct *vma = vmf->vma; - struct ocxlflash_context *ctx = vma->vm_file->private_data; - struct device *dev = ctx->hw_afu->dev; - u64 mmio_area, offset; - - offset = vmf->pgoff << PAGE_SHIFT; - if (offset >= ctx->psn_size) - return VM_FAULT_SIGBUS; - - mutex_lock(&ctx->state_mutex); - if (ctx->state != STARTED) { - dev_err(dev, "%s: Context not started, state=%d\n", - __func__, ctx->state); - mutex_unlock(&ctx->state_mutex); - return VM_FAULT_SIGBUS; - } - mutex_unlock(&ctx->state_mutex); - - mmio_area = ctx->psn_phys; - mmio_area += offset; - - return vmf_insert_pfn(vma, vmf->address, mmio_area >> PAGE_SHIFT); -} - -static const struct vm_operations_struct ocxlflash_vmops = { - .fault = ocxlflash_mmap_fault, -}; - -/** - * afu_mmap() - map the fault handler operations - * @file: File associated with the context. - * @vma: VM area associated with mapping. - * - * Return: 0 on success, -errno on failure - */ -static int afu_mmap(struct file *file, struct vm_area_struct *vma) -{ - struct ocxlflash_context *ctx = file->private_data; - - if ((vma_pages(vma) + vma->vm_pgoff) > - (ctx->psn_size >> PAGE_SHIFT)) - return -EINVAL; - - vm_flags_set(vma, VM_IO | VM_PFNMAP); - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - vma->vm_ops = &ocxlflash_vmops; - return 0; -} - -static const struct file_operations ocxl_afu_fops = { - .owner = THIS_MODULE, - .poll = afu_poll, - .read = afu_read, - .release = afu_release, - .mmap = afu_mmap, -}; - -#define PATCH_FOPS(NAME) \ - do { if (!fops->NAME) fops->NAME = ocxl_afu_fops.NAME; } while (0) - -/** - * ocxlflash_get_fd() - get file descriptor for an adapter context - * @ctx_cookie: Adapter context. - * @fops: File operations to be associated. - * @fd: File descriptor to be returned back. - * - * Return: pointer to the file on success, ERR_PTR on failure - */ -static struct file *ocxlflash_get_fd(void *ctx_cookie, - struct file_operations *fops, int *fd) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct device *dev = ctx->hw_afu->dev; - struct file *file; - int flags, fdtmp; - int rc = 0; - char *name = NULL; - - /* Only allow one fd per context */ - if (ctx->mapping) { - dev_err(dev, "%s: Context is already mapped to an fd\n", - __func__); - rc = -EEXIST; - goto err1; - } - - flags = O_RDWR | O_CLOEXEC; - - /* This code is similar to anon_inode_getfd() */ - rc = get_unused_fd_flags(flags); - if (unlikely(rc < 0)) { - dev_err(dev, "%s: get_unused_fd_flags failed rc=%d\n", - __func__, rc); - goto err1; - } - fdtmp = rc; - - /* Patch the file ops that are not defined */ - if (fops) { - PATCH_FOPS(poll); - PATCH_FOPS(read); - PATCH_FOPS(release); - PATCH_FOPS(mmap); - } else /* Use default ops */ - fops = (struct file_operations *)&ocxl_afu_fops; - - name = kasprintf(GFP_KERNEL, "ocxlflash:%d", ctx->pe); - file = ocxlflash_getfile(dev, name, fops, ctx, flags); - kfree(name); - if (IS_ERR(file)) { - rc = PTR_ERR(file); - dev_err(dev, "%s: ocxlflash_getfile failed rc=%d\n", - __func__, rc); - goto err2; - } - - ctx->mapping = file->f_mapping; - *fd = fdtmp; -out: - return file; -err2: - put_unused_fd(fdtmp); -err1: - file = ERR_PTR(rc); - goto out; -} - -/** - * ocxlflash_fops_get_context() - get the context associated with the file - * @file: File associated with the adapter context. - * - * Return: pointer to the context - */ -static void *ocxlflash_fops_get_context(struct file *file) -{ - return file->private_data; -} - -/** - * ocxlflash_afu_irq() - interrupt handler for user contexts - * @irq: Interrupt number. - * @data: Private data provided at interrupt registration, the context. - * - * Return: Always return IRQ_HANDLED. - */ -static irqreturn_t ocxlflash_afu_irq(int irq, void *data) -{ - struct ocxlflash_context *ctx = data; - struct device *dev = ctx->hw_afu->dev; - int i; - - dev_dbg(dev, "%s: Interrupt raised for pe %i virq %i\n", - __func__, ctx->pe, irq); - - for (i = 0; i < ctx->num_irqs; i++) { - if (ctx->irqs[i].virq == irq) - break; - } - if (unlikely(i >= ctx->num_irqs)) { - dev_err(dev, "%s: Received AFU IRQ out of range\n", __func__); - goto out; - } - - spin_lock(&ctx->slock); - set_bit(i - 1, &ctx->irq_bitmap); - ctx->pending_irq = true; - spin_unlock(&ctx->slock); - - wake_up_all(&ctx->wq); -out: - return IRQ_HANDLED; -} - -/** - * ocxlflash_start_work() - start a user context - * @ctx_cookie: Context to be started. - * @num_irqs: Number of interrupts requested. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_start_work(void *ctx_cookie, u64 num_irqs) -{ - struct ocxlflash_context *ctx = ctx_cookie; - struct ocxl_hw_afu *afu = ctx->hw_afu; - struct device *dev = afu->dev; - char *name; - int rc = 0; - int i; - - rc = alloc_afu_irqs(ctx, num_irqs); - if (unlikely(rc < 0)) { - dev_err(dev, "%s: alloc_afu_irqs failed rc=%d\n", __func__, rc); - goto out; - } - - for (i = 0; i < num_irqs; i++) { - name = kasprintf(GFP_KERNEL, "ocxlflash-%s-pe%i-%i", - dev_name(dev), ctx->pe, i); - rc = afu_map_irq(0, ctx, i, ocxlflash_afu_irq, ctx, name); - kfree(name); - if (unlikely(rc < 0)) { - dev_err(dev, "%s: afu_map_irq failed rc=%d\n", - __func__, rc); - goto err; - } - } - - rc = start_context(ctx); - if (unlikely(rc)) { - dev_err(dev, "%s: start_context failed rc=%d\n", __func__, rc); - goto err; - } -out: - return rc; -err: - for (i = i-1; i >= 0; i--) - afu_unmap_irq(0, ctx, i, ctx); - free_afu_irqs(ctx); - goto out; -}; - -/** - * ocxlflash_fd_mmap() - mmap handler for adapter file descriptor - * @file: File installed with adapter file descriptor. - * @vma: VM area associated with mapping. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_fd_mmap(struct file *file, struct vm_area_struct *vma) -{ - return afu_mmap(file, vma); -} - -/** - * ocxlflash_fd_release() - release the context associated with the file - * @inode: File inode pointer. - * @file: File associated with the adapter context. - * - * Return: 0 on success, -errno on failure - */ -static int ocxlflash_fd_release(struct inode *inode, struct file *file) -{ - return afu_release(inode, file); -} - -/* Backend ops to ocxlflash services */ -const struct cxlflash_backend_ops cxlflash_ocxl_ops = { - .module = THIS_MODULE, - .psa_map = ocxlflash_psa_map, - .psa_unmap = ocxlflash_psa_unmap, - .process_element = ocxlflash_process_element, - .map_afu_irq = ocxlflash_map_afu_irq, - .unmap_afu_irq = ocxlflash_unmap_afu_irq, - .get_irq_objhndl = ocxlflash_get_irq_objhndl, - .start_context = ocxlflash_start_context, - .stop_context = ocxlflash_stop_context, - .afu_reset = ocxlflash_afu_reset, - .set_master = ocxlflash_set_master, - .get_context = ocxlflash_get_context, - .dev_context_init = ocxlflash_dev_context_init, - .release_context = ocxlflash_release_context, - .perst_reloads_same_image = ocxlflash_perst_reloads_same_image, - .read_adapter_vpd = ocxlflash_read_adapter_vpd, - .allocate_afu_irqs = ocxlflash_allocate_afu_irqs, - .free_afu_irqs = ocxlflash_free_afu_irqs, - .create_afu = ocxlflash_create_afu, - .destroy_afu = ocxlflash_destroy_afu, - .get_fd = ocxlflash_get_fd, - .fops_get_context = ocxlflash_fops_get_context, - .start_work = ocxlflash_start_work, - .fd_mmap = ocxlflash_fd_mmap, - .fd_release = ocxlflash_fd_release, -}; |