diff options
Diffstat (limited to 'drivers/vfio/pci/vfio_pci_rdwr.c')
| -rw-r--r-- | drivers/vfio/pci/vfio_pci_rdwr.c | 279 |
1 files changed, 198 insertions, 81 deletions
diff --git a/drivers/vfio/pci/vfio_pci_rdwr.c b/drivers/vfio/pci/vfio_pci_rdwr.c index a6029d0a5524..6192788c8ba3 100644 --- a/drivers/vfio/pci/vfio_pci_rdwr.c +++ b/drivers/vfio/pci/vfio_pci_rdwr.c @@ -1,13 +1,10 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * VFIO PCI I/O Port & MMIO access * * Copyright (C) 2012 Red Hat, Inc. All rights reserved. * Author: Alex Williamson <alex.williamson@redhat.com> * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. - * * Derived from original vfio: * Copyright 2010 Cisco Systems, Inc. All rights reserved. * Author: Tom Lyon, pugs@cisco.com @@ -19,8 +16,9 @@ #include <linux/io.h> #include <linux/vfio.h> #include <linux/vgaarb.h> +#include <linux/io-64-nonatomic-lo-hi.h> -#include "vfio_pci_private.h" +#include "vfio_pci_priv.h" #ifdef __LITTLE_ENDIAN #define vfio_ioread64 ioread64 @@ -40,17 +38,107 @@ #define vfio_ioread8 ioread8 #define vfio_iowrite8 iowrite8 +#define VFIO_IOWRITE(size) \ +int vfio_pci_core_iowrite##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + vfio_iowrite##size(val, io); \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} \ +EXPORT_SYMBOL_GPL(vfio_pci_core_iowrite##size); + +VFIO_IOWRITE(8) +VFIO_IOWRITE(16) +VFIO_IOWRITE(32) +VFIO_IOWRITE(64) + +#define VFIO_IOREAD(size) \ +int vfio_pci_core_ioread##size(struct vfio_pci_core_device *vdev, \ + bool test_mem, u##size *val, void __iomem *io) \ +{ \ + if (test_mem) { \ + down_read(&vdev->memory_lock); \ + if (!__vfio_pci_memory_enabled(vdev)) { \ + up_read(&vdev->memory_lock); \ + return -EIO; \ + } \ + } \ + \ + *val = vfio_ioread##size(io); \ + \ + if (test_mem) \ + up_read(&vdev->memory_lock); \ + \ + return 0; \ +} \ +EXPORT_SYMBOL_GPL(vfio_pci_core_ioread##size); + +VFIO_IOREAD(8) +VFIO_IOREAD(16) +VFIO_IOREAD(32) +VFIO_IOREAD(64) + +#define VFIO_IORDWR(size) \ +static int vfio_pci_iordwr##size(struct vfio_pci_core_device *vdev,\ + bool iswrite, bool test_mem, \ + void __iomem *io, char __user *buf, \ + loff_t off, size_t *filled) \ +{ \ + u##size val; \ + int ret; \ + \ + if (iswrite) { \ + if (copy_from_user(&val, buf, sizeof(val))) \ + return -EFAULT; \ + \ + ret = vfio_pci_core_iowrite##size(vdev, test_mem, \ + val, io + off); \ + if (ret) \ + return ret; \ + } else { \ + ret = vfio_pci_core_ioread##size(vdev, test_mem, \ + &val, io + off); \ + if (ret) \ + return ret; \ + \ + if (copy_to_user(buf, &val, sizeof(val))) \ + return -EFAULT; \ + } \ + \ + *filled = sizeof(val); \ + return 0; \ +} \ + +VFIO_IORDWR(8) +VFIO_IORDWR(16) +VFIO_IORDWR(32) +VFIO_IORDWR(64) + /* * Read or write from an __iomem region (MMIO or I/O port) with an excluded * range which is inaccessible. The excluded range drops writes and fills * reads with -1. This is intended for handling MSI-X vector tables and * leftover space for ROM BARs. */ -static ssize_t do_io_rw(void __iomem *io, char __user *buf, - loff_t off, size_t count, size_t x_start, - size_t x_end, bool iswrite) +ssize_t vfio_pci_core_do_io_rw(struct vfio_pci_core_device *vdev, bool test_mem, + void __iomem *io, char __user *buf, + loff_t off, size_t count, size_t x_start, + size_t x_end, bool iswrite) { ssize_t done = 0; + int ret; while (count) { size_t fillable, filled; @@ -62,54 +150,31 @@ static ssize_t do_io_rw(void __iomem *io, char __user *buf, else fillable = 0; - if (fillable >= 4 && !(off % 4)) { - u32 val; - - if (iswrite) { - if (copy_from_user(&val, buf, 4)) - return -EFAULT; - - vfio_iowrite32(val, io + off); - } else { - val = vfio_ioread32(io + off); + if (fillable >= 8 && !(off % 8)) { + ret = vfio_pci_iordwr64(vdev, iswrite, test_mem, + io, buf, off, &filled); + if (ret) + return ret; - if (copy_to_user(buf, &val, 4)) - return -EFAULT; - } + } else + if (fillable >= 4 && !(off % 4)) { + ret = vfio_pci_iordwr32(vdev, iswrite, test_mem, + io, buf, off, &filled); + if (ret) + return ret; - filled = 4; } else if (fillable >= 2 && !(off % 2)) { - u16 val; - - if (iswrite) { - if (copy_from_user(&val, buf, 2)) - return -EFAULT; + ret = vfio_pci_iordwr16(vdev, iswrite, test_mem, + io, buf, off, &filled); + if (ret) + return ret; - vfio_iowrite16(val, io + off); - } else { - val = vfio_ioread16(io + off); - - if (copy_to_user(buf, &val, 2)) - return -EFAULT; - } - - filled = 2; } else if (fillable) { - u8 val; - - if (iswrite) { - if (copy_from_user(&val, buf, 1)) - return -EFAULT; - - vfio_iowrite8(val, io + off); - } else { - val = vfio_ioread8(io + off); - - if (copy_to_user(buf, &val, 1)) - return -EFAULT; - } + ret = vfio_pci_iordwr8(vdev, iswrite, test_mem, + io, buf, off, &filled); + if (ret) + return ret; - filled = 1; } else { /* Fill reads with -1, drop writes */ filled = min(count, (size_t)(x_end - off)); @@ -131,8 +196,9 @@ static ssize_t do_io_rw(void __iomem *io, char __user *buf, return done; } +EXPORT_SYMBOL_GPL(vfio_pci_core_do_io_rw); -static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) +int vfio_pci_core_setup_barmap(struct vfio_pci_core_device *vdev, int bar) { struct pci_dev *pdev = vdev->pdev; int ret; @@ -155,8 +221,9 @@ static int vfio_pci_setup_barmap(struct vfio_pci_device *vdev, int bar) return 0; } +EXPORT_SYMBOL_GPL(vfio_pci_core_setup_barmap); -ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, +ssize_t vfio_pci_bar_rw(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { struct pci_dev *pdev = vdev->pdev; @@ -165,13 +232,13 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, size_t x_start = 0, x_end = 0; resource_size_t end; void __iomem *io; + struct resource *res = &vdev->pdev->resource[bar]; ssize_t done; if (pci_resource_start(pdev, bar)) end = pci_resource_len(pdev, bar); - else if (bar == PCI_ROM_RESOURCE && - pdev->resource[bar].flags & IORESOURCE_ROM_SHADOW) - end = 0x20000; + else if (bar == PCI_ROM_RESOURCE && pdev->rom && pdev->romlen) + end = roundup_pow_of_two(pdev->romlen); else return -EINVAL; @@ -186,14 +253,21 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, * excluded range at the end of the actual ROM. This makes * filling large ROM BARs much faster. */ - io = pci_map_rom(pdev, &x_start); + if (pci_resource_start(pdev, bar)) { + io = pci_map_rom(pdev, &x_start); + } else { + io = ioremap(pdev->rom, pdev->romlen); + x_start = pdev->romlen; + } if (!io) return -ENOMEM; x_end = end; } else { - int ret = vfio_pci_setup_barmap(vdev, bar); - if (ret) - return ret; + int ret = vfio_pci_core_setup_barmap(vdev, bar); + if (ret) { + done = ret; + goto out; + } io = vdev->barmap[bar]; } @@ -203,18 +277,25 @@ ssize_t vfio_pci_bar_rw(struct vfio_pci_device *vdev, char __user *buf, x_end = vdev->msix_offset + vdev->msix_size; } - done = do_io_rw(io, buf, pos, count, x_start, x_end, iswrite); + done = vfio_pci_core_do_io_rw(vdev, res->flags & IORESOURCE_MEM, io, buf, pos, + count, x_start, x_end, iswrite); if (done >= 0) *ppos += done; - if (bar == PCI_ROM_RESOURCE) - pci_unmap_rom(pdev, io); + if (bar == PCI_ROM_RESOURCE) { + if (pci_resource_start(pdev, bar)) + pci_unmap_rom(pdev, io); + else + iounmap(io); + } +out: return done; } -ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, +#ifdef CONFIG_VFIO_PCI_VGA +ssize_t vfio_pci_vga_rw(struct vfio_pci_core_device *vdev, char __user *buf, size_t count, loff_t *ppos, bool iswrite) { int ret; @@ -233,7 +314,7 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, switch ((u32)pos) { case 0xa0000 ... 0xbffff: count = min(count, (size_t)(0xc0000 - pos)); - iomem = ioremap_nocache(0xa0000, 0xbffff - 0xa0000 + 1); + iomem = ioremap(0xa0000, 0xbffff - 0xa0000 + 1); off = pos - 0xa0000; rsrc = VGA_RSRC_LEGACY_MEM; is_ioport = false; @@ -265,7 +346,13 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, return ret; } - done = do_io_rw(iomem, buf, off, count, 0, 0, iswrite); + /* + * VGA MMIO is a legacy, non-BAR resource that hopefully allows + * probing, so we don't currently worry about access in relation + * to the memory enable bit in the command register. + */ + done = vfio_pci_core_do_io_rw(vdev, false, iomem, buf, off, count, + 0, 0, iswrite); vga_put(vdev->pdev, rsrc); @@ -276,33 +363,62 @@ ssize_t vfio_pci_vga_rw(struct vfio_pci_device *vdev, char __user *buf, return done; } +#endif -static int vfio_pci_ioeventfd_handler(void *opaque, void *unused) +static void vfio_pci_ioeventfd_do_write(struct vfio_pci_ioeventfd *ioeventfd, + bool test_mem) { - struct vfio_pci_ioeventfd *ioeventfd = opaque; - switch (ioeventfd->count) { case 1: - vfio_iowrite8(ioeventfd->data, ioeventfd->addr); + vfio_pci_core_iowrite8(ioeventfd->vdev, test_mem, + ioeventfd->data, ioeventfd->addr); break; case 2: - vfio_iowrite16(ioeventfd->data, ioeventfd->addr); + vfio_pci_core_iowrite16(ioeventfd->vdev, test_mem, + ioeventfd->data, ioeventfd->addr); break; case 4: - vfio_iowrite32(ioeventfd->data, ioeventfd->addr); + vfio_pci_core_iowrite32(ioeventfd->vdev, test_mem, + ioeventfd->data, ioeventfd->addr); break; -#ifdef iowrite64 case 8: - vfio_iowrite64(ioeventfd->data, ioeventfd->addr); + vfio_pci_core_iowrite64(ioeventfd->vdev, test_mem, + ioeventfd->data, ioeventfd->addr); break; -#endif } +} + +static int vfio_pci_ioeventfd_handler(void *opaque, void *unused) +{ + struct vfio_pci_ioeventfd *ioeventfd = opaque; + struct vfio_pci_core_device *vdev = ioeventfd->vdev; + + if (ioeventfd->test_mem) { + if (!down_read_trylock(&vdev->memory_lock)) + return 1; /* Lock contended, use thread */ + if (!__vfio_pci_memory_enabled(vdev)) { + up_read(&vdev->memory_lock); + return 0; + } + } + + vfio_pci_ioeventfd_do_write(ioeventfd, false); + + if (ioeventfd->test_mem) + up_read(&vdev->memory_lock); return 0; } -long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset, - uint64_t data, int count, int fd) +static void vfio_pci_ioeventfd_thread(void *opaque, void *unused) +{ + struct vfio_pci_ioeventfd *ioeventfd = opaque; + + vfio_pci_ioeventfd_do_write(ioeventfd, ioeventfd->test_mem); +} + +int vfio_pci_ioeventfd(struct vfio_pci_core_device *vdev, loff_t offset, + uint64_t data, int count, int fd) { struct pci_dev *pdev = vdev->pdev; loff_t pos = offset & VFIO_PCI_OFFSET_MASK; @@ -322,12 +438,10 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset, pos >= vdev->msix_offset + vdev->msix_size)) return -EINVAL; -#ifndef iowrite64 if (count == 8) return -EINVAL; -#endif - ret = vfio_pci_setup_barmap(vdev, bar); + ret = vfio_pci_core_setup_barmap(vdev, bar); if (ret) return ret; @@ -359,20 +473,23 @@ long vfio_pci_ioeventfd(struct vfio_pci_device *vdev, loff_t offset, goto out_unlock; } - ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL); + ioeventfd = kzalloc(sizeof(*ioeventfd), GFP_KERNEL_ACCOUNT); if (!ioeventfd) { ret = -ENOMEM; goto out_unlock; } + ioeventfd->vdev = vdev; ioeventfd->addr = vdev->barmap[bar] + pos; ioeventfd->data = data; ioeventfd->pos = pos; ioeventfd->bar = bar; ioeventfd->count = count; + ioeventfd->test_mem = vdev->pdev->resource[bar].flags & IORESOURCE_MEM; ret = vfio_virqfd_enable(ioeventfd, vfio_pci_ioeventfd_handler, - NULL, NULL, &ioeventfd->virqfd, fd); + vfio_pci_ioeventfd_thread, NULL, + &ioeventfd->virqfd, fd); if (ret) { kfree(ioeventfd); goto out_unlock; |