diff options
Diffstat (limited to 'drivers/misc/vmw_vmci/vmci_guest.c')
| -rw-r--r-- | drivers/misc/vmw_vmci/vmci_guest.c | 556 |
1 files changed, 390 insertions, 166 deletions
diff --git a/drivers/misc/vmw_vmci/vmci_guest.c b/drivers/misc/vmw_vmci/vmci_guest.c index 60c01999f489..476af89e751b 100644 --- a/drivers/misc/vmw_vmci/vmci_guest.c +++ b/drivers/misc/vmw_vmci/vmci_guest.c @@ -1,16 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * VMware VMCI Driver * * Copyright (C) 2012 VMware, Inc. All rights reserved. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation version 2 and no later version. - * - * This program is distributed in the hope that it will be useful, but - * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * for more details. */ #include <linux/vmw_vmci_defs.h> @@ -21,6 +13,7 @@ #include <linux/kernel.h> #include <linux/mm.h> #include <linux/module.h> +#include <linux/processor.h> #include <linux/sched.h> #include <linux/slab.h> #include <linux/init.h> @@ -35,11 +28,16 @@ #include "vmci_driver.h" #include "vmci_event.h" -#define PCI_VENDOR_ID_VMWARE 0x15AD #define PCI_DEVICE_ID_VMWARE_VMCI 0x0740 #define VMCI_UTIL_NUM_RESOURCES 1 +/* + * Datagram buffers for DMA send/receive must accommodate at least + * a maximum sized datagram and the header. + */ +#define VMCI_DMA_DG_BUFFER_SIZE (VMCI_MAX_DG_SIZE + PAGE_SIZE) + static bool vmci_disable_msi; module_param_named(disable_msi, vmci_disable_msi, bool, 0); MODULE_PARM_DESC(disable_msi, "Disable MSI use in driver - (default=0)"); @@ -54,20 +52,29 @@ static u32 vm_context_id = VMCI_INVALID_ID; struct vmci_guest_device { struct device *dev; /* PCI device we are attached to */ void __iomem *iobase; + void __iomem *mmio_base; - unsigned int irq; - unsigned int intr_type; bool exclusive_vectors; - struct msix_entry msix_entries[VMCI_MAX_INTRS]; - struct tasklet_struct datagram_tasklet; - struct tasklet_struct bm_tasklet; + struct wait_queue_head inout_wq; void *data_buffer; + dma_addr_t data_buffer_base; + void *tx_buffer; + dma_addr_t tx_buffer_base; void *notification_bitmap; + dma_addr_t notification_base; }; +static bool use_ppn64; + +bool vmci_use_ppn64(void) +{ + return use_ppn64; +} + /* vmci_dev singleton device and supporting data*/ +struct pci_dev *vmci_pdev; static struct vmci_guest_device *vmci_dev_g; static DEFINE_SPINLOCK(vmci_dev_spinlock); @@ -92,6 +99,92 @@ u32 vmci_get_vm_context_id(void) return vm_context_id; } +static unsigned int vmci_read_reg(struct vmci_guest_device *dev, u32 reg) +{ + if (dev->mmio_base != NULL) + return readl(dev->mmio_base + reg); + return ioread32(dev->iobase + reg); +} + +static void vmci_write_reg(struct vmci_guest_device *dev, u32 val, u32 reg) +{ + if (dev->mmio_base != NULL) + writel(val, dev->mmio_base + reg); + else + iowrite32(val, dev->iobase + reg); +} + +static void vmci_read_data(struct vmci_guest_device *vmci_dev, + void *dest, size_t size) +{ + if (vmci_dev->mmio_base == NULL) + ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, + dest, size); + else { + /* + * For DMA datagrams, the data_buffer will contain the header on the + * first page, followed by the incoming datagram(s) on the following + * pages. The header uses an S/G element immediately following the + * header on the first page to point to the data area. + */ + struct vmci_data_in_out_header *buffer_header = vmci_dev->data_buffer; + struct vmci_sg_elem *sg_array = (struct vmci_sg_elem *)(buffer_header + 1); + size_t buffer_offset = dest - vmci_dev->data_buffer; + + buffer_header->opcode = 1; + buffer_header->size = 1; + buffer_header->busy = 0; + sg_array[0].addr = vmci_dev->data_buffer_base + buffer_offset; + sg_array[0].size = size; + + vmci_write_reg(vmci_dev, lower_32_bits(vmci_dev->data_buffer_base), + VMCI_DATA_IN_LOW_ADDR); + + wait_event(vmci_dev->inout_wq, buffer_header->busy == 1); + } +} + +static int vmci_write_data(struct vmci_guest_device *dev, + struct vmci_datagram *dg) +{ + int result; + + if (dev->mmio_base != NULL) { + struct vmci_data_in_out_header *buffer_header = dev->tx_buffer; + u8 *dg_out_buffer = (u8 *)(buffer_header + 1); + + if (VMCI_DG_SIZE(dg) > VMCI_MAX_DG_SIZE) + return VMCI_ERROR_INVALID_ARGS; + + /* + * Initialize send buffer with outgoing datagram + * and set up header for inline data. Device will + * not access buffer asynchronously - only after + * the write to VMCI_DATA_OUT_LOW_ADDR. + */ + memcpy(dg_out_buffer, dg, VMCI_DG_SIZE(dg)); + buffer_header->opcode = 0; + buffer_header->size = VMCI_DG_SIZE(dg); + buffer_header->busy = 1; + + vmci_write_reg(dev, lower_32_bits(dev->tx_buffer_base), + VMCI_DATA_OUT_LOW_ADDR); + + /* Caller holds a spinlock, so cannot block. */ + spin_until_cond(buffer_header->busy == 0); + + result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); + if (result == VMCI_SUCCESS) + result = (int)buffer_header->result; + } else { + iowrite8_rep(dev->iobase + VMCI_DATA_OUT_ADDR, + dg, VMCI_DG_SIZE(dg)); + result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); + } + + return result; +} + /* * VM to hypervisor call mechanism. We use the standard VMware naming * convention since shared code is calling this function as well. @@ -117,9 +210,8 @@ int vmci_send_datagram(struct vmci_datagram *dg) spin_lock_irqsave(&vmci_dev_spinlock, flags); if (vmci_dev_g) { - iowrite8_rep(vmci_dev_g->iobase + VMCI_DATA_OUT_ADDR, - dg, VMCI_DG_SIZE(dg)); - result = ioread32(vmci_dev_g->iobase + VMCI_RESULT_LOW_ADDR); + vmci_write_data(vmci_dev_g, dg); + result = vmci_read_reg(vmci_dev_g, VMCI_RESULT_LOW_ADDR); } else { result = VMCI_ERROR_UNAVAILABLE; } @@ -159,11 +251,11 @@ static void vmci_guest_cid_update(u32 sub_id, /* * Verify that the host supports the hypercalls we need. If it does not, - * try to find fallback hypercalls and use those instead. Returns - * true if required hypercalls (or fallback hypercalls) are - * supported by the host, false otherwise. + * try to find fallback hypercalls and use those instead. Returns 0 if + * required hypercalls (or fallback hypercalls) are supported by the host, + * an error code otherwise. */ -static bool vmci_check_host_caps(struct pci_dev *pdev) +static int vmci_check_host_caps(struct pci_dev *pdev) { bool result; struct vmci_resource_query_msg *msg; @@ -171,10 +263,10 @@ static bool vmci_check_host_caps(struct pci_dev *pdev) VMCI_UTIL_NUM_RESOURCES * sizeof(u32); struct vmci_datagram *check_msg; - check_msg = kmalloc(msg_size, GFP_KERNEL); + check_msg = kzalloc(msg_size, GFP_KERNEL); if (!check_msg) { dev_err(&pdev->dev, "%s: Insufficient memory\n", __func__); - return false; + return -ENOMEM; } check_msg->dst = vmci_make_handle(VMCI_HYPERVISOR_CONTEXT_ID, @@ -194,43 +286,62 @@ static bool vmci_check_host_caps(struct pci_dev *pdev) __func__, result ? "PASSED" : "FAILED"); /* We need the vector. There are no fallbacks. */ - return result; + return result ? 0 : -ENXIO; } /* - * Reads datagrams from the data in port and dispatches them. We - * always start reading datagrams into only the first page of the - * datagram buffer. If the datagrams don't fit into one page, we - * use the maximum datagram buffer size for the remainder of the - * invocation. This is a simple heuristic for not penalizing - * small datagrams. + * Reads datagrams from the device and dispatches them. For IO port + * based access to the device, we always start reading datagrams into + * only the first page of the datagram buffer. If the datagrams don't + * fit into one page, we use the maximum datagram buffer size for the + * remainder of the invocation. This is a simple heuristic for not + * penalizing small datagrams. For DMA-based datagrams, we always + * use the maximum datagram buffer size, since there is no performance + * penalty for doing so. * * This function assumes that it has exclusive access to the data - * in port for the duration of the call. + * in register(s) for the duration of the call. */ -static void vmci_dispatch_dgs(unsigned long data) +static void vmci_dispatch_dgs(struct vmci_guest_device *vmci_dev) { - struct vmci_guest_device *vmci_dev = (struct vmci_guest_device *)data; u8 *dg_in_buffer = vmci_dev->data_buffer; struct vmci_datagram *dg; size_t dg_in_buffer_size = VMCI_MAX_DG_SIZE; - size_t current_dg_in_buffer_size = PAGE_SIZE; + size_t current_dg_in_buffer_size; size_t remaining_bytes; + bool is_io_port = vmci_dev->mmio_base == NULL; BUILD_BUG_ON(VMCI_MAX_DG_SIZE < PAGE_SIZE); - ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, - vmci_dev->data_buffer, current_dg_in_buffer_size); + if (!is_io_port) { + /* For mmio, the first page is used for the header. */ + dg_in_buffer += PAGE_SIZE; + + /* + * For DMA-based datagram operations, there is no performance + * penalty for reading the maximum buffer size. + */ + current_dg_in_buffer_size = VMCI_MAX_DG_SIZE; + } else { + current_dg_in_buffer_size = PAGE_SIZE; + } + vmci_read_data(vmci_dev, dg_in_buffer, current_dg_in_buffer_size); dg = (struct vmci_datagram *)dg_in_buffer; remaining_bytes = current_dg_in_buffer_size; + /* + * Read through the buffer until an invalid datagram header is + * encountered. The exit condition for datagrams read through + * VMCI_DATA_IN_ADDR is a bit more complicated, since a datagram + * can start on any page boundary in the buffer. + */ while (dg->dst.resource != VMCI_INVALID_ID || - remaining_bytes > PAGE_SIZE) { + (is_io_port && remaining_bytes > PAGE_SIZE)) { unsigned dg_in_size; /* - * When the input buffer spans multiple pages, a datagram can - * start on any page boundary in the buffer. + * If using VMCI_DATA_IN_ADDR, skip to the next page + * as a datagram can start on any page boundary. */ if (dg->dst.resource == VMCI_INVALID_ID) { dg = (struct vmci_datagram *)roundup( @@ -280,11 +391,10 @@ static void vmci_dispatch_dgs(unsigned long data) current_dg_in_buffer_size = dg_in_buffer_size; - ioread8_rep(vmci_dev->iobase + - VMCI_DATA_IN_ADDR, - vmci_dev->data_buffer + + vmci_read_data(vmci_dev, + dg_in_buffer + remaining_bytes, - current_dg_in_buffer_size - + current_dg_in_buffer_size - remaining_bytes); } @@ -322,10 +432,8 @@ static void vmci_dispatch_dgs(unsigned long data) current_dg_in_buffer_size = dg_in_buffer_size; for (;;) { - ioread8_rep(vmci_dev->iobase + - VMCI_DATA_IN_ADDR, - vmci_dev->data_buffer, - current_dg_in_buffer_size); + vmci_read_data(vmci_dev, dg_in_buffer, + current_dg_in_buffer_size); if (bytes_to_skip <= current_dg_in_buffer_size) break; @@ -342,8 +450,7 @@ static void vmci_dispatch_dgs(unsigned long data) if (remaining_bytes < VMCI_DG_HEADERSIZE) { /* Get the next batch of datagrams. */ - ioread8_rep(vmci_dev->iobase + VMCI_DATA_IN_ADDR, - vmci_dev->data_buffer, + vmci_read_data(vmci_dev, dg_in_buffer, current_dg_in_buffer_size); dg = (struct vmci_datagram *)dg_in_buffer; remaining_bytes = current_dg_in_buffer_size; @@ -355,10 +462,8 @@ static void vmci_dispatch_dgs(unsigned long data) * Scans the notification bitmap for raised flags, clears them * and handles the notifications. */ -static void vmci_process_bitmap(unsigned long data) +static void vmci_process_bitmap(struct vmci_guest_device *dev) { - struct vmci_guest_device *dev = (struct vmci_guest_device *)data; - if (!dev->notification_bitmap) { dev_dbg(dev->dev, "No bitmap present in %s\n", __func__); return; @@ -368,29 +473,6 @@ static void vmci_process_bitmap(unsigned long data) } /* - * Enable MSI-X. Try exclusive vectors first, then shared vectors. - */ -static int vmci_enable_msix(struct pci_dev *pdev, - struct vmci_guest_device *vmci_dev) -{ - int i; - int result; - - for (i = 0; i < VMCI_MAX_INTRS; ++i) { - vmci_dev->msix_entries[i].entry = i; - vmci_dev->msix_entries[i].vector = i; - } - - result = pci_enable_msix(pdev, vmci_dev->msix_entries, VMCI_MAX_INTRS); - if (result == 0) - vmci_dev->exclusive_vectors = true; - else if (result > 0) - result = pci_enable_msix(pdev, vmci_dev->msix_entries, 1); - - return result; -} - -/* * Interrupt handler for legacy or MSI interrupt, or for first MSI-X * interrupt (vector VMCI_INTR_DATAGRAM). */ @@ -399,31 +481,37 @@ static irqreturn_t vmci_interrupt(int irq, void *_dev) struct vmci_guest_device *dev = _dev; /* - * If we are using MSI-X with exclusive vectors then we simply schedule - * the datagram tasklet, since we know the interrupt was meant for us. + * If we are using MSI-X with exclusive vectors then we simply call + * vmci_dispatch_dgs(), since we know the interrupt was meant for us. * Otherwise we must read the ICR to determine what to do. */ - if (dev->intr_type == VMCI_INTR_TYPE_MSIX && dev->exclusive_vectors) { - tasklet_schedule(&dev->datagram_tasklet); + if (dev->exclusive_vectors) { + vmci_dispatch_dgs(dev); } else { unsigned int icr; /* Acknowledge interrupt and determine what needs doing. */ - icr = ioread32(dev->iobase + VMCI_ICR_ADDR); + icr = vmci_read_reg(dev, VMCI_ICR_ADDR); if (icr == 0 || icr == ~0) return IRQ_NONE; if (icr & VMCI_ICR_DATAGRAM) { - tasklet_schedule(&dev->datagram_tasklet); + vmci_dispatch_dgs(dev); icr &= ~VMCI_ICR_DATAGRAM; } if (icr & VMCI_ICR_NOTIFICATION) { - tasklet_schedule(&dev->bm_tasklet); + vmci_process_bitmap(dev); icr &= ~VMCI_ICR_NOTIFICATION; } + + if (icr & VMCI_ICR_DMA_DATAGRAM) { + wake_up_all(&dev->inout_wq); + icr &= ~VMCI_ICR_DMA_DATAGRAM; + } + if (icr != 0) dev_warn(dev->dev, "Ignoring unknown interrupt cause (%d)\n", @@ -443,11 +531,43 @@ static irqreturn_t vmci_interrupt_bm(int irq, void *_dev) struct vmci_guest_device *dev = _dev; /* For MSI-X we can just assume it was meant for us. */ - tasklet_schedule(&dev->bm_tasklet); + vmci_process_bitmap(dev); + + return IRQ_HANDLED; +} + +/* + * Interrupt handler for MSI-X interrupt vector VMCI_INTR_DMA_DATAGRAM, + * which is for the completion of a DMA datagram send or receive operation. + * Will only get called if we are using MSI-X with exclusive vectors. + */ +static irqreturn_t vmci_interrupt_dma_datagram(int irq, void *_dev) +{ + struct vmci_guest_device *dev = _dev; + + wake_up_all(&dev->inout_wq); return IRQ_HANDLED; } +static void vmci_free_dg_buffers(struct vmci_guest_device *vmci_dev) +{ + if (vmci_dev->mmio_base != NULL) { + if (vmci_dev->tx_buffer != NULL) + dma_free_coherent(vmci_dev->dev, + VMCI_DMA_DG_BUFFER_SIZE, + vmci_dev->tx_buffer, + vmci_dev->tx_buffer_base); + if (vmci_dev->data_buffer != NULL) + dma_free_coherent(vmci_dev->dev, + VMCI_DMA_DG_BUFFER_SIZE, + vmci_dev->data_buffer, + vmci_dev->data_buffer_base); + } else { + vfree(vmci_dev->data_buffer); + } +} + /* * Most of the initialization at module load time is done here. */ @@ -455,8 +575,11 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, const struct pci_device_id *id) { struct vmci_guest_device *vmci_dev; - void __iomem *iobase; + void __iomem *iobase = NULL; + void __iomem *mmio_base = NULL; + unsigned int num_irq_vectors; unsigned int capabilities; + unsigned int caps_in_use; unsigned long cmd; int vmci_err; int error; @@ -470,39 +593,71 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, return error; } - error = pcim_iomap_regions(pdev, 1 << 0, KBUILD_MODNAME); - if (error) { - dev_err(&pdev->dev, "Failed to reserve/map IO regions\n"); - return error; - } + /* + * The VMCI device with mmio access to registers requests 256KB + * for BAR1. If present, driver will use new VMCI device + * functionality for register access and datagram send/recv. + */ - iobase = pcim_iomap_table(pdev)[0]; + if (pci_resource_len(pdev, 1) == VMCI_WITH_MMIO_ACCESS_BAR_SIZE) { + dev_info(&pdev->dev, "MMIO register access is available\n"); + mmio_base = pci_iomap_range(pdev, 1, VMCI_MMIO_ACCESS_OFFSET, + VMCI_MMIO_ACCESS_SIZE); + /* If the map fails, we fall back to IOIO access. */ + if (!mmio_base) + dev_warn(&pdev->dev, "Failed to map MMIO register access\n"); + } - dev_info(&pdev->dev, "Found VMCI PCI device at %#lx, irq %u\n", - (unsigned long)iobase, pdev->irq); + if (!mmio_base) { + if (IS_ENABLED(CONFIG_ARM64)) { + dev_err(&pdev->dev, "MMIO base is invalid\n"); + return -ENXIO; + } + error = pcim_iomap_regions(pdev, BIT(0), KBUILD_MODNAME); + if (error) { + dev_err(&pdev->dev, "Failed to reserve/map IO regions\n"); + return error; + } + iobase = pcim_iomap_table(pdev)[0]; + } vmci_dev = devm_kzalloc(&pdev->dev, sizeof(*vmci_dev), GFP_KERNEL); if (!vmci_dev) { dev_err(&pdev->dev, "Can't allocate memory for VMCI device\n"); - return -ENOMEM; + error = -ENOMEM; + goto err_unmap_mmio_base; } vmci_dev->dev = &pdev->dev; - vmci_dev->intr_type = VMCI_INTR_TYPE_INTX; vmci_dev->exclusive_vectors = false; vmci_dev->iobase = iobase; + vmci_dev->mmio_base = mmio_base; + + init_waitqueue_head(&vmci_dev->inout_wq); - tasklet_init(&vmci_dev->datagram_tasklet, - vmci_dispatch_dgs, (unsigned long)vmci_dev); - tasklet_init(&vmci_dev->bm_tasklet, - vmci_process_bitmap, (unsigned long)vmci_dev); + if (mmio_base != NULL) { + vmci_dev->tx_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE, + &vmci_dev->tx_buffer_base, + GFP_KERNEL); + if (!vmci_dev->tx_buffer) { + dev_err(&pdev->dev, + "Can't allocate memory for datagram tx buffer\n"); + error = -ENOMEM; + goto err_unmap_mmio_base; + } - vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE); + vmci_dev->data_buffer = dma_alloc_coherent(&pdev->dev, VMCI_DMA_DG_BUFFER_SIZE, + &vmci_dev->data_buffer_base, + GFP_KERNEL); + } else { + vmci_dev->data_buffer = vmalloc(VMCI_MAX_DG_SIZE); + } if (!vmci_dev->data_buffer) { dev_err(&pdev->dev, "Can't allocate memory for datagram buffer\n"); - return -ENOMEM; + error = -ENOMEM; + goto err_free_data_buffers; } pci_set_master(pdev); /* To enable queue_pair functionality. */ @@ -516,11 +671,28 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, * * Right now, we need datagrams. There are no fallbacks. */ - capabilities = ioread32(vmci_dev->iobase + VMCI_CAPS_ADDR); + capabilities = vmci_read_reg(vmci_dev, VMCI_CAPS_ADDR); if (!(capabilities & VMCI_CAPS_DATAGRAM)) { dev_err(&pdev->dev, "Device does not support datagrams\n"); error = -ENXIO; - goto err_free_data_buffer; + goto err_free_data_buffers; + } + caps_in_use = VMCI_CAPS_DATAGRAM; + + /* + * Use 64-bit PPNs if the device supports. + * + * There is no check for the return value of dma_set_mask_and_coherent + * since this driver can handle the default mask values if + * dma_set_mask_and_coherent fails. + */ + if (capabilities & VMCI_CAPS_PPN64) { + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + use_ppn64 = true; + caps_in_use |= VMCI_CAPS_PPN64; + } else { + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44)); + use_ppn64 = false; } /* @@ -528,45 +700,69 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, * well. */ if (capabilities & VMCI_CAPS_NOTIFICATIONS) { - vmci_dev->notification_bitmap = vmalloc(PAGE_SIZE); - if (!vmci_dev->notification_bitmap) { + vmci_dev->notification_bitmap = dma_alloc_coherent( + &pdev->dev, PAGE_SIZE, &vmci_dev->notification_base, + GFP_KERNEL); + if (!vmci_dev->notification_bitmap) dev_warn(&pdev->dev, "Unable to allocate notification bitmap\n"); + else + caps_in_use |= VMCI_CAPS_NOTIFICATIONS; + } + + if (mmio_base != NULL) { + if (capabilities & VMCI_CAPS_DMA_DATAGRAM) { + caps_in_use |= VMCI_CAPS_DMA_DATAGRAM; } else { - memset(vmci_dev->notification_bitmap, 0, PAGE_SIZE); - capabilities |= VMCI_CAPS_NOTIFICATIONS; + dev_err(&pdev->dev, + "Missing capability: VMCI_CAPS_DMA_DATAGRAM\n"); + error = -ENXIO; + goto err_free_notification_bitmap; } } - dev_info(&pdev->dev, "Using capabilities 0x%x\n", capabilities); + dev_info(&pdev->dev, "Using capabilities 0x%x\n", caps_in_use); /* Let the host know which capabilities we intend to use. */ - iowrite32(capabilities, vmci_dev->iobase + VMCI_CAPS_ADDR); + vmci_write_reg(vmci_dev, caps_in_use, VMCI_CAPS_ADDR); + + if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) { + /* Let the device know the size for pages passed down. */ + vmci_write_reg(vmci_dev, PAGE_SHIFT, VMCI_GUEST_PAGE_SHIFT); + + /* Configure the high order parts of the data in/out buffers. */ + vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->data_buffer_base), + VMCI_DATA_IN_HIGH_ADDR); + vmci_write_reg(vmci_dev, upper_32_bits(vmci_dev->tx_buffer_base), + VMCI_DATA_OUT_HIGH_ADDR); + } /* Set up global device so that we can start sending datagrams */ spin_lock_irq(&vmci_dev_spinlock); vmci_dev_g = vmci_dev; + vmci_pdev = pdev; spin_unlock_irq(&vmci_dev_spinlock); /* * Register notification bitmap with device if that capability is * used. */ - if (capabilities & VMCI_CAPS_NOTIFICATIONS) { - struct page *page = - vmalloc_to_page(vmci_dev->notification_bitmap); - unsigned long bitmap_ppn = page_to_pfn(page); + if (caps_in_use & VMCI_CAPS_NOTIFICATIONS) { + unsigned long bitmap_ppn = + vmci_dev->notification_base >> PAGE_SHIFT; if (!vmci_dbell_register_notification_bitmap(bitmap_ppn)) { dev_warn(&pdev->dev, - "VMCI device unable to register notification bitmap with PPN 0x%x\n", - (u32) bitmap_ppn); + "VMCI device unable to register notification bitmap with PPN 0x%lx\n", + bitmap_ppn); + error = -ENXIO; goto err_remove_vmci_dev_g; } } /* Check host capabilities. */ - if (!vmci_check_host_caps(pdev)) - goto err_remove_bitmap; + error = vmci_check_host_caps(pdev); + if (error) + goto err_remove_vmci_dev_g; /* Enable device. */ @@ -586,26 +782,30 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, * Enable interrupts. Try MSI-X first, then MSI, and then fallback on * legacy interrupts. */ - if (!vmci_disable_msix && !vmci_enable_msix(pdev, vmci_dev)) { - vmci_dev->intr_type = VMCI_INTR_TYPE_MSIX; - vmci_dev->irq = vmci_dev->msix_entries[0].vector; - } else if (!vmci_disable_msi && !pci_enable_msi(pdev)) { - vmci_dev->intr_type = VMCI_INTR_TYPE_MSI; - vmci_dev->irq = pdev->irq; + if (vmci_dev->mmio_base != NULL) + num_irq_vectors = VMCI_MAX_INTRS; + else + num_irq_vectors = VMCI_MAX_INTRS_NOTIFICATION; + error = pci_alloc_irq_vectors(pdev, num_irq_vectors, num_irq_vectors, + PCI_IRQ_MSIX); + if (error < 0) { + error = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES); + if (error < 0) + goto err_unsubscribe_event; } else { - vmci_dev->intr_type = VMCI_INTR_TYPE_INTX; - vmci_dev->irq = pdev->irq; + vmci_dev->exclusive_vectors = true; } /* * Request IRQ for legacy or MSI interrupts, or for first * MSI-X vector. */ - error = request_irq(vmci_dev->irq, vmci_interrupt, IRQF_SHARED, - KBUILD_MODNAME, vmci_dev); + error = request_threaded_irq(pci_irq_vector(pdev, 0), NULL, + vmci_interrupt, IRQF_SHARED, + KBUILD_MODNAME, vmci_dev); if (error) { dev_err(&pdev->dev, "Irq %u in use: %d\n", - vmci_dev->irq, error); + pci_irq_vector(pdev, 0), error); goto err_disable_msi; } @@ -616,15 +816,28 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, * between the vectors. */ if (vmci_dev->exclusive_vectors) { - error = request_irq(vmci_dev->msix_entries[1].vector, - vmci_interrupt_bm, 0, KBUILD_MODNAME, - vmci_dev); + error = request_threaded_irq(pci_irq_vector(pdev, 1), NULL, + vmci_interrupt_bm, 0, + KBUILD_MODNAME, vmci_dev); if (error) { dev_err(&pdev->dev, "Failed to allocate irq %u: %d\n", - vmci_dev->msix_entries[1].vector, error); + pci_irq_vector(pdev, 1), error); goto err_free_irq; } + if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) { + error = request_threaded_irq(pci_irq_vector(pdev, 2), + NULL, + vmci_interrupt_dma_datagram, + 0, KBUILD_MODNAME, + vmci_dev); + if (error) { + dev_err(&pdev->dev, + "Failed to allocate irq %u: %d\n", + pci_irq_vector(pdev, 2), error); + goto err_free_bm_irq; + } + } } dev_dbg(&pdev->dev, "Registered device\n"); @@ -633,48 +846,57 @@ static int vmci_guest_probe_device(struct pci_dev *pdev, /* Enable specific interrupt bits. */ cmd = VMCI_IMR_DATAGRAM; - if (capabilities & VMCI_CAPS_NOTIFICATIONS) + if (caps_in_use & VMCI_CAPS_NOTIFICATIONS) cmd |= VMCI_IMR_NOTIFICATION; - iowrite32(cmd, vmci_dev->iobase + VMCI_IMR_ADDR); + if (caps_in_use & VMCI_CAPS_DMA_DATAGRAM) + cmd |= VMCI_IMR_DMA_DATAGRAM; + vmci_write_reg(vmci_dev, cmd, VMCI_IMR_ADDR); /* Enable interrupts. */ - iowrite32(VMCI_CONTROL_INT_ENABLE, - vmci_dev->iobase + VMCI_CONTROL_ADDR); + vmci_write_reg(vmci_dev, VMCI_CONTROL_INT_ENABLE, VMCI_CONTROL_ADDR); pci_set_drvdata(pdev, vmci_dev); + + vmci_call_vsock_callback(false); return 0; +err_free_bm_irq: + if (vmci_dev->exclusive_vectors) + free_irq(pci_irq_vector(pdev, 1), vmci_dev); + err_free_irq: - free_irq(vmci_dev->irq, &vmci_dev); - tasklet_kill(&vmci_dev->datagram_tasklet); - tasklet_kill(&vmci_dev->bm_tasklet); + free_irq(pci_irq_vector(pdev, 0), vmci_dev); err_disable_msi: - if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX) - pci_disable_msix(pdev); - else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI) - pci_disable_msi(pdev); + pci_free_irq_vectors(pdev); +err_unsubscribe_event: vmci_err = vmci_event_unsubscribe(ctx_update_sub_id); if (vmci_err < VMCI_SUCCESS) dev_warn(&pdev->dev, "Failed to unsubscribe from event (type=%d) with subscriber (ID=0x%x): %d\n", VMCI_EVENT_CTX_ID_UPDATE, ctx_update_sub_id, vmci_err); -err_remove_bitmap: - if (vmci_dev->notification_bitmap) { - iowrite32(VMCI_CONTROL_RESET, - vmci_dev->iobase + VMCI_CONTROL_ADDR); - vfree(vmci_dev->notification_bitmap); - } - err_remove_vmci_dev_g: spin_lock_irq(&vmci_dev_spinlock); + vmci_pdev = NULL; vmci_dev_g = NULL; spin_unlock_irq(&vmci_dev_spinlock); -err_free_data_buffer: - vfree(vmci_dev->data_buffer); +err_free_notification_bitmap: + if (vmci_dev->notification_bitmap) { + vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR); + dma_free_coherent(&pdev->dev, PAGE_SIZE, + vmci_dev->notification_bitmap, + vmci_dev->notification_base); + } + +err_free_data_buffers: + vmci_free_dg_buffers(vmci_dev); + +err_unmap_mmio_base: + if (mmio_base != NULL) + pci_iounmap(pdev, mmio_base); /* The rest are managed resources and will be freed by PCI core */ return error; @@ -699,27 +921,24 @@ static void vmci_guest_remove_device(struct pci_dev *pdev) spin_lock_irq(&vmci_dev_spinlock); vmci_dev_g = NULL; + vmci_pdev = NULL; spin_unlock_irq(&vmci_dev_spinlock); dev_dbg(&pdev->dev, "Resetting vmci device\n"); - iowrite32(VMCI_CONTROL_RESET, vmci_dev->iobase + VMCI_CONTROL_ADDR); + vmci_write_reg(vmci_dev, VMCI_CONTROL_RESET, VMCI_CONTROL_ADDR); /* * Free IRQ and then disable MSI/MSI-X as appropriate. For * MSI-X, we might have multiple vectors, each with their own * IRQ, which we must free too. */ - free_irq(vmci_dev->irq, vmci_dev); - if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSIX) { - if (vmci_dev->exclusive_vectors) - free_irq(vmci_dev->msix_entries[1].vector, vmci_dev); - pci_disable_msix(pdev); - } else if (vmci_dev->intr_type == VMCI_INTR_TYPE_MSI) { - pci_disable_msi(pdev); + if (vmci_dev->exclusive_vectors) { + free_irq(pci_irq_vector(pdev, 1), vmci_dev); + if (vmci_dev->mmio_base != NULL) + free_irq(pci_irq_vector(pdev, 2), vmci_dev); } - - tasklet_kill(&vmci_dev->datagram_tasklet); - tasklet_kill(&vmci_dev->bm_tasklet); + free_irq(pci_irq_vector(pdev, 0), vmci_dev); + pci_free_irq_vectors(pdev); if (vmci_dev->notification_bitmap) { /* @@ -727,15 +946,20 @@ static void vmci_guest_remove_device(struct pci_dev *pdev) * device, so we can safely free it here. */ - vfree(vmci_dev->notification_bitmap); + dma_free_coherent(&pdev->dev, PAGE_SIZE, + vmci_dev->notification_bitmap, + vmci_dev->notification_base); } - vfree(vmci_dev->data_buffer); + vmci_free_dg_buffers(vmci_dev); + + if (vmci_dev->mmio_base != NULL) + pci_iounmap(pdev, vmci_dev->mmio_base); /* The rest are managed resources and will be freed by PCI core */ } -static DEFINE_PCI_DEVICE_TABLE(vmci_ids) = { +static const struct pci_device_id vmci_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_VMWARE, PCI_DEVICE_ID_VMWARE_VMCI), }, { 0 }, }; |