From c2a052a4a949df53f50a5024843432d2234cb824 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Fri, 24 Jun 2022 10:55:41 +0800 Subject: remoteproc: rename len of rpoc_vring to num Rename the member len in the structure rpoc_vring to num. And remove 'in bytes' from the comment of it. This is misleading. Because this actually refers to the size of the virtio vring to be created. The unit is not bytes. Signed-off-by: Xuan Zhuo Message-Id: <20220624025621.128843-2-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/remoteproc/remoteproc_core.c | 4 ++-- drivers/remoteproc/remoteproc_virtio.c | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'drivers') diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 02a04ab34a23..2d2f3bab5888 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -334,7 +334,7 @@ int rproc_alloc_vring(struct rproc_vdev *rvdev, int i) size_t size; /* actual size of vring (in bytes) */ - size = PAGE_ALIGN(vring_size(rvring->len, rvring->align)); + size = PAGE_ALIGN(vring_size(rvring->num, rvring->align)); rsc = (void *)rproc->table_ptr + rvdev->rsc_offset; @@ -401,7 +401,7 @@ rproc_parse_vring(struct rproc_vdev *rvdev, struct fw_rsc_vdev *rsc, int i) return -EINVAL; } - rvring->len = vring->num; + rvring->num = vring->num; rvring->align = vring->align; rvring->rvdev = rvdev; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 70ab496d0431..d43d74733f0a 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -87,7 +87,7 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, struct fw_rsc_vdev *rsc; struct virtqueue *vq; void *addr; - int len, size; + int num, size; /* we're temporarily limited to two virtqueues per rvdev */ if (id >= ARRAY_SIZE(rvdev->vring)) @@ -104,20 +104,20 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, rvring = &rvdev->vring[id]; addr = mem->va; - len = rvring->len; + num = rvring->num; /* zero vring */ - size = vring_size(len, rvring->align); + size = vring_size(num, rvring->align); memset(addr, 0, size); dev_dbg(dev, "vring%d: va %pK qsz %d notifyid %d\n", - id, addr, len, rvring->notifyid); + id, addr, num, rvring->notifyid); /* * Create the new vq, and tell virtio we're not interested in * the 'weak' smp barriers, since we're talking with a real device. */ - vq = vring_new_virtqueue(id, len, rvring->align, vdev, false, ctx, + vq = vring_new_virtqueue(id, num, rvring->align, vdev, false, ctx, addr, rproc_virtio_notify, callback, name); if (!vq) { dev_err(dev, "vring_new_virtqueue %s failed\n", name); -- cgit From 96ef18a24b87bef7c3d2dad72cf4c3013a9f7f35 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Fri, 24 Jun 2022 10:55:45 +0800 Subject: virtio_ring: remove the arg vq of vring_alloc_desc_extra() The parameter vq of vring_alloc_desc_extra() is useless. This patch removes this parameter. Subsequent patches will call this function to avoid passing useless arguments. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220624025621.128843-6-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 643ca779fcc6..a5ec724c01d8 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1637,8 +1637,7 @@ static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq) return NULL; } -static struct vring_desc_extra *vring_alloc_desc_extra(struct vring_virtqueue *vq, - unsigned int num) +static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num) { struct vring_desc_extra *desc_extra; unsigned int i; @@ -1759,7 +1758,7 @@ static struct virtqueue *vring_create_virtqueue_packed( /* Put everything in free lists. */ vq->free_head = 0; - vq->packed.desc_extra = vring_alloc_desc_extra(vq, num); + vq->packed.desc_extra = vring_alloc_desc_extra(num); if (!vq->packed.desc_extra) goto err_desc_extra; @@ -2248,7 +2247,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, if (!vq->split.desc_state) goto err_state; - vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num); + vq->split.desc_extra = vring_alloc_desc_extra(vring.num); if (!vq->split.desc_extra) goto err_extra; -- cgit From 309bba39c945ac8ab8083ac05cd6cfe5822968e0 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Fri, 24 Jun 2022 09:56:56 +0200 Subject: vringh: iterate on iotlb_translate to handle large translations iotlb_translate() can return -ENOBUFS if the bio_vec is not big enough to contain all the ranges for translation. This can happen for example if the VMM maps a large bounce buffer, without using hugepages, that requires more than 16 ranges to translate the addresses. To handle this case, let's extend iotlb_translate() to also return the number of bytes successfully translated. In copy_from_iotlb()/copy_to_iotlb() loops by calling iotlb_translate() several times until we complete the translation. Signed-off-by: Stefano Garzarella Message-Id: <20220624075656.13997-1-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vringh.c | 78 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 22 deletions(-) (limited to 'drivers') diff --git a/drivers/vhost/vringh.c b/drivers/vhost/vringh.c index eab55accf381..11f59dd06a74 100644 --- a/drivers/vhost/vringh.c +++ b/drivers/vhost/vringh.c @@ -1095,7 +1095,8 @@ EXPORT_SYMBOL(vringh_need_notify_kern); #if IS_REACHABLE(CONFIG_VHOST_IOTLB) static int iotlb_translate(const struct vringh *vrh, - u64 addr, u64 len, struct bio_vec iov[], + u64 addr, u64 len, u64 *translated, + struct bio_vec iov[], int iov_size, u32 perm) { struct vhost_iotlb_map *map; @@ -1136,43 +1137,76 @@ static int iotlb_translate(const struct vringh *vrh, spin_unlock(vrh->iotlb_lock); + if (translated) + *translated = min(len, s); + return ret; } static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, void *src, size_t len) { - struct iov_iter iter; - struct bio_vec iov[16]; - int ret; + u64 total_translated = 0; - ret = iotlb_translate(vrh, (u64)(uintptr_t)src, - len, iov, 16, VHOST_MAP_RO); - if (ret < 0) - return ret; + while (total_translated < len) { + struct bio_vec iov[16]; + struct iov_iter iter; + u64 translated; + int ret; - iov_iter_bvec(&iter, READ, iov, ret, len); + ret = iotlb_translate(vrh, (u64)(uintptr_t)src, + len - total_translated, &translated, + iov, ARRAY_SIZE(iov), VHOST_MAP_RO); + if (ret == -ENOBUFS) + ret = ARRAY_SIZE(iov); + else if (ret < 0) + return ret; - ret = copy_from_iter(dst, len, &iter); + iov_iter_bvec(&iter, READ, iov, ret, translated); - return ret; + ret = copy_from_iter(dst, translated, &iter); + if (ret < 0) + return ret; + + src += translated; + dst += translated; + total_translated += translated; + } + + return total_translated; } static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, void *src, size_t len) { - struct iov_iter iter; - struct bio_vec iov[16]; - int ret; + u64 total_translated = 0; - ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, - len, iov, 16, VHOST_MAP_WO); - if (ret < 0) - return ret; + while (total_translated < len) { + struct bio_vec iov[16]; + struct iov_iter iter; + u64 translated; + int ret; + + ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, + len - total_translated, &translated, + iov, ARRAY_SIZE(iov), VHOST_MAP_WO); + if (ret == -ENOBUFS) + ret = ARRAY_SIZE(iov); + else if (ret < 0) + return ret; - iov_iter_bvec(&iter, WRITE, iov, ret, len); + iov_iter_bvec(&iter, WRITE, iov, ret, translated); + + ret = copy_to_iter(src, translated, &iter); + if (ret < 0) + return ret; + + src += translated; + dst += translated; + total_translated += translated; + } - return copy_to_iter(src, len, &iter); + return total_translated; } static inline int getu16_iotlb(const struct vringh *vrh, @@ -1183,7 +1217,7 @@ static inline int getu16_iotlb(const struct vringh *vrh, int ret; /* Atomic read is needed for getu16 */ - ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, &iov, 1, VHOST_MAP_RO); if (ret < 0) return ret; @@ -1204,7 +1238,7 @@ static inline int putu16_iotlb(const struct vringh *vrh, int ret; /* Atomic write is needed for putu16 */ - ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), + ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, &iov, 1, VHOST_MAP_WO); if (ret < 0) return ret; -- cgit From 020e1aede7ee31d8b0e24a46ee364c7bb0939f02 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Jun 2022 16:34:29 +0800 Subject: virtio_pmem: initialize provider_data through nd_region_desc We used to initialize the provider_data manually after nvdimm_pemm_region_create(). This seems to be racy if the flush is issued before the initialization of provider_data[1]. Fixing this by initializing the provider_data through nd_region_desc to make sure the provider_data is ready after the pmem is created. [1]: [ 80.152281] nd_pmem namespace0.0: unable to guarantee persistence of writes [ 92.393956] BUG: kernel NULL pointer dereference, address: 0000000000000318 [ 92.394551] #PF: supervisor read access in kernel mode [ 92.394955] #PF: error_code(0x0000) - not-present page [ 92.395365] PGD 0 P4D 0 [ 92.395566] Oops: 0000 [#1] PREEMPT SMP PTI [ 92.395867] CPU: 2 PID: 506 Comm: mkfs.ext4 Not tainted 5.19.0-rc1+ #453 [ 92.396365] Hardware name: QEMU Standard PC (Q35 + ICH9, 2009), BIOS rel-1.16.0-0-gd239552ce722-prebuilt.qemu.org 04/01/2014 [ 92.397178] RIP: 0010:virtio_pmem_flush+0x2f/0x1f0 [ 92.397521] Code: 55 41 54 55 53 48 81 ec a0 00 00 00 65 48 8b 04 25 28 00 00 00 48 89 84 24 98 00 00 00 31 c0 48 8b 87 78 03 00 00 48 89 04 24 <48> 8b 98 18 03 00 00 e8 85 bf 6b 00 ba 58 00 00 00 be c0 0c 00 00 [ 92.398982] RSP: 0018:ffff9a7380aefc88 EFLAGS: 00010246 [ 92.399349] RAX: 0000000000000000 RBX: ffff8e77c3f86f00 RCX: 0000000000000000 [ 92.399833] RDX: ffffffffad4ea720 RSI: ffff8e77c41e39c0 RDI: ffff8e77c41c5c00 [ 92.400388] RBP: ffff8e77c41e39c0 R08: ffff8e77c19f0600 R09: 0000000000000000 [ 92.400874] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8e77c0814e28 [ 92.401364] R13: 0000000000000000 R14: 0000000000000000 R15: ffff8e77c41e39c0 [ 92.401849] FS: 00007f3cd75b2780(0000) GS:ffff8e7937d00000(0000) knlGS:0000000000000000 [ 92.402423] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 92.402821] CR2: 0000000000000318 CR3: 0000000103c80002 CR4: 0000000000370ee0 [ 92.403307] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 [ 92.403793] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 [ 92.404278] Call Trace: [ 92.404481] [ 92.404654] ? mempool_alloc+0x5d/0x160 [ 92.404939] ? terminate_walk+0x5f/0xf0 [ 92.405226] ? bio_alloc_bioset+0xbb/0x3f0 [ 92.405525] async_pmem_flush+0x17/0x80 [ 92.405806] nvdimm_flush+0x11/0x30 [ 92.406067] pmem_submit_bio+0x1e9/0x200 [ 92.406354] __submit_bio+0x80/0x120 [ 92.406621] submit_bio_noacct_nocheck+0xdc/0x2a0 [ 92.406958] submit_bio_wait+0x4e/0x80 [ 92.407234] blkdev_issue_flush+0x31/0x50 [ 92.407526] ? punt_bios_to_rescuer+0x230/0x230 [ 92.407852] blkdev_fsync+0x1e/0x30 [ 92.408112] do_fsync+0x33/0x70 [ 92.408354] __x64_sys_fsync+0xb/0x10 [ 92.408625] do_syscall_64+0x43/0x90 [ 92.408895] entry_SYSCALL_64_after_hwframe+0x46/0xb0 [ 92.409257] RIP: 0033:0x7f3cd76c6c44 Fixes 6e84200c0a29 ("virtio-pmem: Add virtio pmem driver") Acked-by: Pankaj Gupta Reviewed-by: Dan Williams Signed-off-by: Jason Wang Message-Id: <20220628083430.61856-1-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/nvdimm/virtio_pmem.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 995b6cdc67ed..48f8327d0431 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -81,6 +81,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) ndr_desc.res = &res; ndr_desc.numa_node = nid; ndr_desc.flush = async_pmem_flush; + ndr_desc.provider_data = vdev; set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); set_bit(ND_REGION_ASYNC, &ndr_desc.flags); nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc); @@ -89,7 +90,6 @@ static int virtio_pmem_probe(struct virtio_device *vdev) err = -ENXIO; goto out_nd; } - nd_region->provider_data = dev_to_virtio(nd_region->dev.parent->parent); return 0; out_nd: nvdimm_bus_unregister(vpmem->nvdimm_bus); -- cgit From 5d66322b2b23507e9907a68e9c504181ffccd62f Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Tue, 28 Jun 2022 16:34:30 +0800 Subject: virtio_pmem: set device ready in probe() The NVDIMM region could be available before the virtio_device_ready() that is called by virtio_dev_probe(). This means the driver tries to use device before DRIVER_OK which violates the spec, fixing this by set device ready before the nvdimm_pmem_region_create(). Note that this means the virtio_pmem_host_ack() could be triggered before the creation of the nd region, this is safe since the pmem_lock has been initialized and whether or not any available buffer is added before is validated by virtio_pmem_host_ack(). Fixes 6e84200c0a29 ("virtio-pmem: Add virtio pmem driver") Acked-by: Pankaj Gupta Signed-off-by: Jason Wang Message-Id: <20220628083430.61856-2-jasowang@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/nvdimm/virtio_pmem.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'drivers') diff --git a/drivers/nvdimm/virtio_pmem.c b/drivers/nvdimm/virtio_pmem.c index 48f8327d0431..20da455d2ef6 100644 --- a/drivers/nvdimm/virtio_pmem.c +++ b/drivers/nvdimm/virtio_pmem.c @@ -84,6 +84,12 @@ static int virtio_pmem_probe(struct virtio_device *vdev) ndr_desc.provider_data = vdev; set_bit(ND_REGION_PAGEMAP, &ndr_desc.flags); set_bit(ND_REGION_ASYNC, &ndr_desc.flags); + /* + * The NVDIMM region could be available before the + * virtio_device_ready() that is called by + * virtio_dev_probe(), so we set device ready here. + */ + virtio_device_ready(vdev); nd_region = nvdimm_pmem_region_create(vpmem->nvdimm_bus, &ndr_desc); if (!nd_region) { dev_err(&vdev->dev, "failed to create nvdimm region\n"); @@ -92,6 +98,7 @@ static int virtio_pmem_probe(struct virtio_device *vdev) } return 0; out_nd: + virtio_reset_device(vdev); nvdimm_bus_unregister(vpmem->nvdimm_bus); out_vq: vdev->config->del_vqs(vdev); -- cgit From ebe797f25f68f28581f46a9cb9c1997ac15c39a0 Mon Sep 17 00:00:00 2001 From: "Michael S. Tsirkin" Date: Thu, 30 Jun 2022 15:10:57 -0400 Subject: virtio: VIRTIO_HARDEN_NOTIFICATION is broken This option doesn't really work and breaks too many drivers. Not yet sure what's the right thing to do, for now let's make sure randconfig isn't broken by this. Fixes: c346dae4f3fb ("virtio: disable notification hardening by default") Cc: "Jason Wang" Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/virtio/Kconfig | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index e1556d2a355a..afb9051e0125 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -31,11 +31,12 @@ if VIRTIO_MENU config VIRTIO_HARDEN_NOTIFICATION bool "Harden virtio notification" + depends on BROKEN help Enable this to harden the device notifications and suppress those that happen at a time where notifications are illegal. - Experimental: Note that several drivers still have bugs that + Experimental: Note that several drivers still have issues that may cause crashes or hangs when correct handling of notifications is enforced; depending on the subset of drivers and devices you use, this may or may not work. -- cgit From 366958a7fd7b541cba478e2866ef08f34e20471b Mon Sep 17 00:00:00 2001 From: Robin Murphy Date: Wed, 8 Jun 2022 12:48:26 +0100 Subject: vdpa: Use device_iommu_capable() Use the new interface to check the capability for our device specifically. Signed-off-by: Robin Murphy Message-Id: <548e316fa282ce513fabb991a4c4d92258062eb5.1654688822.git.robin.murphy@arm.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vhost/vdpa.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 23dcbfdfa13b..250ad4160ccb 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -1076,7 +1076,7 @@ static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v) if (!bus) return -EFAULT; - if (!iommu_capable(bus, IOMMU_CAP_CACHE_COHERENCY)) + if (!device_iommu_capable(dma_dev, IOMMU_CAP_CACHE_COHERENCY)) return -ENOTSUPP; v->domain = iommu_domain_alloc(bus); -- cgit From 02213273f72a43bad3a8e9c18595a2c74ee487e8 Mon Sep 17 00:00:00 2001 From: Minghao Xue Date: Fri, 10 Jun 2022 16:58:27 +0800 Subject: virtio_mmio: add support to set IRQ of a virtio device as wakeup source According to virtio_mmio wakeup flag in device trees, set its IRQ as wakeup source in virtqueue initialization. Signed-off-by: Minghao Xue Message-Id: <1654851507-13891-3-git-send-email-quic_mingxue@quicinc.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 083ff1eb743d..945cb8fb60b6 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -487,6 +487,9 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, if (err) return err; + if (of_property_read_bool(vm_dev->pdev->dev.of_node, "wakeup-source")) + enable_irq_wake(irq); + for (i = 0; i < nvqs; ++i) { if (!names[i]) { vqs[i] = NULL; -- cgit From 0b6fd46ec5f5720b76cbb01300ed9f7b7c6365c4 Mon Sep 17 00:00:00 2001 From: David Hildenbrand Date: Fri, 10 Jun 2022 11:47:37 +0200 Subject: drivers/virtio: Clarify CONFIG_VIRTIO_MEM for unsupported architectures Let's make it clearer that simply unlocking CONFIG_VIRTIO_MEM on an architecture is most probably not sufficient to have it working as expected. Cc: "Michael S. Tsirkin" Cc: Jason Wang Cc: Gavin Shan Signed-off-by: David Hildenbrand Message-Id: <20220610094737.65254-1-david@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/Kconfig | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/Kconfig b/drivers/virtio/Kconfig index afb9051e0125..02bbccd602c0 100644 --- a/drivers/virtio/Kconfig +++ b/drivers/virtio/Kconfig @@ -123,9 +123,11 @@ config VIRTIO_MEM This driver provides access to virtio-mem paravirtualized memory devices, allowing to hotplug and hotunplug memory. - This driver was only tested under x86-64 and arm64, but should - theoretically work on all architectures that support memory hotplug - and hotremove. + This driver currently only supports x86-64 and arm64. Although it + should compile on other architectures that implement memory + hot(un)plug, architecture-specific and/or common + code changes may be required for virtio-mem, kdump and kexec to work as + expected. If unsure, say M. -- cgit From da802961832f9852886304290135457519815497 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:21 +0800 Subject: virtio: record the maximum queue num supported by the device. virtio-net can display the maximum (supported by hardware) ring size in ethtool -g eth0. When the subsequent patch implements vring reset, it can judge whether the ring size passed by the driver is legal based on this. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-2-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/platform/mellanox/mlxbf-tmfifo.c | 2 ++ drivers/remoteproc/remoteproc_virtio.c | 2 ++ drivers/s390/virtio/virtio_ccw.c | 3 +++ drivers/virtio/virtio_mmio.c | 2 ++ drivers/virtio/virtio_pci_legacy.c | 2 ++ drivers/virtio/virtio_pci_modern.c | 2 ++ drivers/virtio/virtio_vdpa.c | 2 ++ 7 files changed, 15 insertions(+) (limited to 'drivers') diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 38800e86ed8a..1ae3c56b66b0 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -959,6 +959,8 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, goto error; } + vq->num_max = vring->num; + vqs[i] = vq; vring->vq = vq; vq->priv = vring; diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index d43d74733f0a..0f7706e23eb9 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -125,6 +125,8 @@ static struct virtqueue *rp_find_vq(struct virtio_device *vdev, return ERR_PTR(-ENOMEM); } + vq->num_max = num; + rvring->vq = vq; vq->priv = rvring; diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 161d3b141f0d..6b86d0280d6b 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -530,6 +530,9 @@ static struct virtqueue *virtio_ccw_setup_vq(struct virtio_device *vdev, err = -ENOMEM; goto out_err; } + + vq->num_max = info->num; + /* it may have been reduced */ info->num = virtqueue_get_vring_size(vq); diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 945cb8fb60b6..3ff746e3f24a 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -403,6 +403,8 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_new_virtqueue; } + vq->num_max = num; + /* Activate the queue */ writel(virtqueue_get_vring_size(vq), vm_dev->base + VIRTIO_MMIO_QUEUE_NUM); if (vm_dev->version == 1) { diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index a5e5721145c7..2257f1b3d8ae 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -135,6 +135,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + q_pfn = virtqueue_get_desc_addr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT; if (q_pfn >> 32) { dev_err(&vp_dev->pci_dev->dev, diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 623906b4996c..e7e0b8c850f6 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -218,6 +218,8 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!vq) return ERR_PTR(-ENOMEM); + vq->num_max = num; + /* activate the queue */ vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index c40f7deb6b5a..9670cc79371d 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -183,6 +183,8 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, goto error_new_virtqueue; } + vq->num_max = max_num; + /* Setup virtqueue callback */ cb.callback = callback ? virtio_vdpa_virtqueue_cb : NULL; cb.private = info; -- cgit From a62eecb3a9c086fa7b47c5cc74ab8ca4e655a682 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:23 +0800 Subject: virtio_ring: update the document of the virtqueue_detach_unused_buf for queue reset Added documentation for virtqueue_detach_unused_buf, allowing it to be called on queue reset. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-4-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a5ec724c01d8..17024389b62c 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2130,8 +2130,8 @@ EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); * @_vq: the struct virtqueue we're talking about. * * Returns NULL or the "data" token handed to virtqueue_add_*(). - * This is not valid on an active queue; it is useful only for device - * shutdown. + * This is not valid on an active queue; it is useful for device + * shutdown or the reset queue. */ void *virtqueue_detach_unused_buf(struct virtqueue *_vq) { -- cgit From 3ea19e3265d8d140bdd2905e7c40216368cac589 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:24 +0800 Subject: virtio_ring: extract the logic of freeing vring Introduce vring_free() to free the vring of vq. Subsequent patches will use vring_free() alone. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-5-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 17024389b62c..a3d76fd87983 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2316,14 +2316,10 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, } EXPORT_SYMBOL_GPL(vring_new_virtqueue); -void vring_del_virtqueue(struct virtqueue *_vq) +static void vring_free(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); - spin_lock(&vq->vq.vdev->vqs_list_lock); - list_del(&_vq->list); - spin_unlock(&vq->vq.vdev->vqs_list_lock); - if (vq->we_own_ring) { if (vq->packed_ring) { vring_free_queue(vq->vq.vdev, @@ -2354,6 +2350,18 @@ void vring_del_virtqueue(struct virtqueue *_vq) kfree(vq->split.desc_state); kfree(vq->split.desc_extra); } +} + +void vring_del_virtqueue(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + spin_lock(&vq->vq.vdev->vqs_list_lock); + list_del(&_vq->list); + spin_unlock(&vq->vq.vdev->vqs_list_lock); + + vring_free(_vq); + kfree(vq); } EXPORT_SYMBOL_GPL(vring_del_virtqueue); -- cgit From d76136e434973691d8928cec1b3f93d38e30fe6e Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:25 +0800 Subject: virtio_ring: split vring_virtqueue Separate the two inline structures(split and packed) from the structure vring_virtqueue. In this way, we can use these two structures later to pass parameters and retain temporary variables. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-6-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 116 ++++++++++++++++++++++--------------------- 1 file changed, 60 insertions(+), 56 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a3d76fd87983..1bc5794e9739 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -85,6 +85,64 @@ struct vring_desc_extra { u16 next; /* The next desc state in a list. */ }; +struct vring_virtqueue_split { + /* Actual memory layout for this queue. */ + struct vring vring; + + /* Last written value to avail->flags */ + u16 avail_flags_shadow; + + /* + * Last written value to avail->idx in + * guest byte order. + */ + u16 avail_idx_shadow; + + /* Per-descriptor state. */ + struct vring_desc_state_split *desc_state; + struct vring_desc_extra *desc_extra; + + /* DMA address and size information */ + dma_addr_t queue_dma_addr; + size_t queue_size_in_bytes; +}; + +struct vring_virtqueue_packed { + /* Actual memory layout for this queue. */ + struct { + unsigned int num; + struct vring_packed_desc *desc; + struct vring_packed_desc_event *driver; + struct vring_packed_desc_event *device; + } vring; + + /* Driver ring wrap counter. */ + bool avail_wrap_counter; + + /* Avail used flags. */ + u16 avail_used_flags; + + /* Index of the next avail descriptor. */ + u16 next_avail_idx; + + /* + * Last written value to driver->flags in + * guest byte order. + */ + u16 event_flags_shadow; + + /* Per-descriptor state. */ + struct vring_desc_state_packed *desc_state; + struct vring_desc_extra *desc_extra; + + /* DMA address and size information */ + dma_addr_t ring_dma_addr; + dma_addr_t driver_event_dma_addr; + dma_addr_t device_event_dma_addr; + size_t ring_size_in_bytes; + size_t event_size_in_bytes; +}; + struct vring_virtqueue { struct virtqueue vq; @@ -124,64 +182,10 @@ struct vring_virtqueue { union { /* Available for split ring */ - struct { - /* Actual memory layout for this queue. */ - struct vring vring; - - /* Last written value to avail->flags */ - u16 avail_flags_shadow; - - /* - * Last written value to avail->idx in - * guest byte order. - */ - u16 avail_idx_shadow; - - /* Per-descriptor state. */ - struct vring_desc_state_split *desc_state; - struct vring_desc_extra *desc_extra; - - /* DMA address and size information */ - dma_addr_t queue_dma_addr; - size_t queue_size_in_bytes; - } split; + struct vring_virtqueue_split split; /* Available for packed ring */ - struct { - /* Actual memory layout for this queue. */ - struct { - unsigned int num; - struct vring_packed_desc *desc; - struct vring_packed_desc_event *driver; - struct vring_packed_desc_event *device; - } vring; - - /* Driver ring wrap counter. */ - bool avail_wrap_counter; - - /* Avail used flags. */ - u16 avail_used_flags; - - /* Index of the next avail descriptor. */ - u16 next_avail_idx; - - /* - * Last written value to driver->flags in - * guest byte order. - */ - u16 event_flags_shadow; - - /* Per-descriptor state. */ - struct vring_desc_state_packed *desc_state; - struct vring_desc_extra *desc_extra; - - /* DMA address and size information */ - dma_addr_t ring_dma_addr; - dma_addr_t driver_event_dma_addr; - dma_addr_t device_event_dma_addr; - size_t ring_size_in_bytes; - size_t event_size_in_bytes; - } packed; + struct vring_virtqueue_packed packed; }; /* How to notify other side. FIXME: commonalize hcalls! */ -- cgit From 3a897128d3193459a87551e54a7f0a71edb65153 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:26 +0800 Subject: virtio_ring: introduce virtqueue_init() Separate the logic of virtqueue initialization. These variables should be reset during reset. This logic can be called independently when implementing resize/reset later. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-7-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1bc5794e9739..a63ef2d99955 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -368,6 +368,24 @@ static int vring_mapping_error(const struct vring_virtqueue *vq, return dma_mapping_error(vring_dma_dev(vq), addr); } +static void virtqueue_init(struct vring_virtqueue *vq, u32 num) +{ + vq->vq.num_free = num; + + if (vq->packed_ring) + vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); + else + vq->last_used_idx = 0; + + vq->event_triggered = false; + vq->num_added = 0; + +#ifdef DEBUG + vq->in_use = false; + vq->last_add_time_valid = false; +#endif +} + /* * Split ring specific functions - *_split(). @@ -1706,7 +1724,6 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; - vq->vq.num_free = num; vq->vq.index = index; vq->we_own_ring = true; vq->notify = notify; @@ -1716,15 +1733,8 @@ static struct virtqueue *vring_create_virtqueue_packed( #else vq->broken = false; #endif - vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); - vq->event_triggered = false; - vq->num_added = 0; vq->packed_ring = true; vq->use_dma_api = vring_use_dma_api(vdev); -#ifdef DEBUG - vq->in_use = false; - vq->last_add_time_valid = false; -#endif vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -1773,6 +1783,8 @@ static struct virtqueue *vring_create_virtqueue_packed( cpu_to_le16(vq->packed.event_flags_shadow); } + virtqueue_init(vq, num); + spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); @@ -2205,7 +2217,6 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->vq.callback = callback; vq->vq.vdev = vdev; vq->vq.name = name; - vq->vq.num_free = vring.num; vq->vq.index = index; vq->we_own_ring = false; vq->notify = notify; @@ -2215,14 +2226,7 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, #else vq->broken = false; #endif - vq->last_used_idx = 0; - vq->event_triggered = false; - vq->num_added = 0; vq->use_dma_api = vring_use_dma_api(vdev); -#ifdef DEBUG - vq->in_use = false; - vq->last_add_time_valid = false; -#endif vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && !context; @@ -2260,6 +2264,8 @@ struct virtqueue *__vring_new_virtqueue(unsigned int index, memset(vq->split.desc_state, 0, vring.num * sizeof(struct vring_desc_state_split)); + virtqueue_init(vq, vring.num); + spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); -- cgit From 07d9629d49584b6f79faa6158cd7aef7e6919703 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:27 +0800 Subject: virtio_ring: split: stop __vring_new_virtqueue as export symbol There is currently only one place to reference __vring_new_virtqueue() directly from the outside of virtio core. And here vring_new_virtqueue() can be used instead. Subsequent patches will modify __vring_new_virtqueue, so stop it as an export symbol for now. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-8-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a63ef2d99955..8ce6cc73d814 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -204,6 +204,14 @@ struct vring_virtqueue { #endif }; +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name); /* * Helpers. @@ -2195,14 +2203,14 @@ irqreturn_t vring_interrupt(int irq, void *_vq) EXPORT_SYMBOL_GPL(vring_interrupt); /* Only available for split ring */ -struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, - struct virtio_device *vdev, - bool weak_barriers, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static struct virtqueue *__vring_new_virtqueue(unsigned int index, + struct vring vring, + struct virtio_device *vdev, + bool weak_barriers, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) { struct vring_virtqueue *vq; @@ -2277,7 +2285,6 @@ err_state: kfree(vq); return NULL; } -EXPORT_SYMBOL_GPL(__vring_new_virtqueue); struct virtqueue *vring_create_virtqueue( unsigned int index, -- cgit From cd4c812acb8390a3cfa0a178b777979f9cdd3eeb Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:28 +0800 Subject: virtio_ring: split: __vring_new_virtqueue() accept struct vring_virtqueue_split __vring_new_virtqueue() instead accepts struct vring_virtqueue_split. The purpose of this is to pass more information into __vring_new_virtqueue() to make the code simpler and the structure cleaner. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-9-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8ce6cc73d814..1d51ed46b295 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -205,7 +205,7 @@ struct vring_virtqueue { }; static struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, + struct vring_virtqueue_split *vring_split, struct virtio_device *vdev, bool weak_barriers, bool context, @@ -949,6 +949,7 @@ static struct virtqueue *vring_create_virtqueue_split( void (*callback)(struct virtqueue *), const char *name) { + struct vring_virtqueue_split vring_split = {}; struct virtqueue *vq; void *queue = NULL; dma_addr_t dma_addr; @@ -984,10 +985,10 @@ static struct virtqueue *vring_create_virtqueue_split( return NULL; queue_size_in_bytes = vring_size(num, vring_align); - vring_init(&vring, num, queue, vring_align); + vring_init(&vring_split.vring, num, queue, vring_align); - vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, - notify, callback, name); + vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, + context, notify, callback, name); if (!vq) { vring_free_queue(vdev, queue_size_in_bytes, queue, dma_addr); @@ -2204,7 +2205,7 @@ EXPORT_SYMBOL_GPL(vring_interrupt); /* Only available for split ring */ static struct virtqueue *__vring_new_virtqueue(unsigned int index, - struct vring vring, + struct vring_virtqueue_split *vring_split, struct virtio_device *vdev, bool weak_barriers, bool context, @@ -2246,7 +2247,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.queue_dma_addr = 0; vq->split.queue_size_in_bytes = 0; - vq->split.vring = vring; + vq->split.vring = vring_split->vring; vq->split.avail_flags_shadow = 0; vq->split.avail_idx_shadow = 0; @@ -2258,21 +2259,21 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.avail_flags_shadow); } - vq->split.desc_state = kmalloc_array(vring.num, + vq->split.desc_state = kmalloc_array(vring_split->vring.num, sizeof(struct vring_desc_state_split), GFP_KERNEL); if (!vq->split.desc_state) goto err_state; - vq->split.desc_extra = vring_alloc_desc_extra(vring.num); + vq->split.desc_extra = vring_alloc_desc_extra(vring_split->vring.num); if (!vq->split.desc_extra) goto err_extra; /* Put everything in free lists. */ vq->free_head = 0; - memset(vq->split.desc_state, 0, vring.num * + memset(vq->split.desc_state, 0, vring_split->vring.num * sizeof(struct vring_desc_state_split)); - virtqueue_init(vq, vring.num); + virtqueue_init(vq, vring_split->vring.num); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); @@ -2322,14 +2323,14 @@ struct virtqueue *vring_new_virtqueue(unsigned int index, void (*callback)(struct virtqueue *vq), const char *name) { - struct vring vring; + struct vring_virtqueue_split vring_split = {}; if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) return NULL; - vring_init(&vring, num, pages, vring_align); - return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context, - notify, callback, name); + vring_init(&vring_split.vring, num, pages, vring_align); + return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, + context, notify, callback, name); } EXPORT_SYMBOL_GPL(vring_new_virtqueue); -- cgit From 89f05d94a33abec2d38f644065ca5cff5f50c4d7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:29 +0800 Subject: virtio_ring: split: introduce vring_free_split() Free the structure struct vring_vritqueue_split. Subsequent patches require it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-10-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1d51ed46b295..71cc7762ec51 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -937,6 +937,17 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static void vring_free_split(struct vring_virtqueue_split *vring_split, + struct virtio_device *vdev) +{ + vring_free_queue(vdev, vring_split->queue_size_in_bytes, + vring_split->vring.desc, + vring_split->queue_dma_addr); + + kfree(vring_split->desc_state); + kfree(vring_split->desc_extra); +} + static struct virtqueue *vring_create_virtqueue_split( unsigned int index, unsigned int num, -- cgit From c2d87fe68d13f45f42861e25dcda21b132dbf554 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:30 +0800 Subject: virtio_ring: split: extract the logic of alloc queue Separate the logic of split to create vring queue. This feature is required for subsequent virtuqueue reset vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-11-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 65 +++++++++++++++++++++++++++----------------- 1 file changed, 40 insertions(+), 25 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 71cc7762ec51..5597a9b9e518 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -948,29 +948,19 @@ static void vring_free_split(struct vring_virtqueue_split *vring_split, kfree(vring_split->desc_extra); } -static struct virtqueue *vring_create_virtqueue_split( - unsigned int index, - unsigned int num, - unsigned int vring_align, - struct virtio_device *vdev, - bool weak_barriers, - bool may_reduce_num, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, + struct virtio_device *vdev, + u32 num, + unsigned int vring_align, + bool may_reduce_num) { - struct vring_virtqueue_split vring_split = {}; - struct virtqueue *vq; void *queue = NULL; dma_addr_t dma_addr; - size_t queue_size_in_bytes; - struct vring vring; /* We assume num is a power of 2. */ if (num & (num - 1)) { dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); - return NULL; + return -EINVAL; } /* TODO: allocate each queue chunk individually */ @@ -981,11 +971,11 @@ static struct virtqueue *vring_create_virtqueue_split( if (queue) break; if (!may_reduce_num) - return NULL; + return -ENOMEM; } if (!num) - return NULL; + return -ENOMEM; if (!queue) { /* Try to get a single page. You are my only hope! */ @@ -993,21 +983,46 @@ static struct virtqueue *vring_create_virtqueue_split( &dma_addr, GFP_KERNEL|__GFP_ZERO); } if (!queue) - return NULL; + return -ENOMEM; + + vring_init(&vring_split->vring, num, queue, vring_align); - queue_size_in_bytes = vring_size(num, vring_align); - vring_init(&vring_split.vring, num, queue, vring_align); + vring_split->queue_dma_addr = dma_addr; + vring_split->queue_size_in_bytes = vring_size(num, vring_align); + + return 0; +} + +static struct virtqueue *vring_create_virtqueue_split( + unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) +{ + struct vring_virtqueue_split vring_split = {}; + struct virtqueue *vq; + int err; + + err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align, + may_reduce_num); + if (err) + return NULL; vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, context, notify, callback, name); if (!vq) { - vring_free_queue(vdev, queue_size_in_bytes, queue, - dma_addr); + vring_free_split(&vring_split, vdev); return NULL; } - to_vvq(vq)->split.queue_dma_addr = dma_addr; - to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes; + to_vvq(vq)->split.queue_dma_addr = vring_split.queue_dma_addr; + to_vvq(vq)->split.queue_size_in_bytes = vring_split.queue_size_in_bytes; to_vvq(vq)->we_own_ring = true; return vq; -- cgit From a2b36c8d7ddbaff801bcaa4592522cd3b7b08112 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:31 +0800 Subject: virtio_ring: split: extract the logic of alloc state and extra Separate the logic of creating desc_state, desc_extra, and subsequent patches will call it independently. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-12-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 52 ++++++++++++++++++++++++++++++-------------- 1 file changed, 36 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 5597a9b9e518..03fc656b1f40 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -212,6 +212,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, bool (*notify)(struct virtqueue *), void (*callback)(struct virtqueue *), const char *name); +static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); /* * Helpers. @@ -937,6 +938,32 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) +{ + struct vring_desc_state_split *state; + struct vring_desc_extra *extra; + u32 num = vring_split->vring.num; + + state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL); + if (!state) + goto err_state; + + extra = vring_alloc_desc_extra(num); + if (!extra) + goto err_extra; + + memset(state, 0, num * sizeof(struct vring_desc_state_split)); + + vring_split->desc_state = state; + vring_split->desc_extra = extra; + return 0; + +err_extra: + kfree(state); +err_state: + return -ENOMEM; +} + static void vring_free_split(struct vring_virtqueue_split *vring_split, struct virtio_device *vdev) { @@ -2240,6 +2267,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, const char *name) { struct vring_virtqueue *vq; + int err; if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) return NULL; @@ -2285,19 +2313,17 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.avail_flags_shadow); } - vq->split.desc_state = kmalloc_array(vring_split->vring.num, - sizeof(struct vring_desc_state_split), GFP_KERNEL); - if (!vq->split.desc_state) - goto err_state; - - vq->split.desc_extra = vring_alloc_desc_extra(vring_split->vring.num); - if (!vq->split.desc_extra) - goto err_extra; + err = vring_alloc_state_extra_split(vring_split); + if (err) { + kfree(vq); + return NULL; + } /* Put everything in free lists. */ vq->free_head = 0; - memset(vq->split.desc_state, 0, vring_split->vring.num * - sizeof(struct vring_desc_state_split)); + + vq->split.desc_state = vring_split->desc_state; + vq->split.desc_extra = vring_split->desc_extra; virtqueue_init(vq, vring_split->vring.num); @@ -2305,12 +2331,6 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, list_add_tail(&vq->vq.list, &vdev->vqs); spin_unlock(&vdev->vqs_list_lock); return &vq->vq; - -err_extra: - kfree(vq->split.desc_state); -err_state: - kfree(vq); - return NULL; } struct virtqueue *vring_create_virtqueue( -- cgit From 198fa7be96e52bc89c9e8a7e1c3b9e059ff203a0 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:32 +0800 Subject: virtio_ring: split: extract the logic of vring init Separate the logic of initializing vring, and subsequent patches will call it separately. This function completes the variable initialization of split vring. It together with the logic of atatch constitutes the initialization of vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-13-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 03fc656b1f40..e456cc16ea2a 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -938,6 +938,25 @@ static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) return NULL; } +static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split, + struct vring_virtqueue *vq) +{ + struct virtio_device *vdev; + + vdev = vq->vq.vdev; + + vring_split->avail_flags_shadow = 0; + vring_split->avail_idx_shadow = 0; + + /* No callback? Tell other side not to bother us. */ + if (!vq->vq.callback) { + vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; + if (!vq->event) + vring_split->vring.avail->flags = cpu_to_virtio16(vdev, + vring_split->avail_flags_shadow); + } +} + static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) { struct vring_desc_state_split *state; @@ -2302,16 +2321,6 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.queue_size_in_bytes = 0; vq->split.vring = vring_split->vring; - vq->split.avail_flags_shadow = 0; - vq->split.avail_idx_shadow = 0; - - /* No callback? Tell other side not to bother us. */ - if (!callback) { - vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; - if (!vq->event) - vq->split.vring.avail->flags = cpu_to_virtio16(vdev, - vq->split.avail_flags_shadow); - } err = vring_alloc_state_extra_split(vring_split); if (err) { @@ -2325,6 +2334,8 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->split.desc_state = vring_split->desc_state; vq->split.desc_extra = vring_split->desc_extra; + virtqueue_vring_init_split(vring_split, vq); + virtqueue_init(vq, vring_split->vring.num); spin_lock(&vdev->vqs_list_lock); -- cgit From e1d6a423ea1867a3a84f12a99981f036acb8f354 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:33 +0800 Subject: virtio_ring: split: extract the logic of attach vring Separate the logic of attach vring, subsequent patches will call it separately. virtqueue_vring_init_split() completes the initialization of other variables of vring split. We can directly use vq->split = *vring_split to complete attach. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-14-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index e456cc16ea2a..d0df887c3ada 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -957,6 +957,15 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split } } +static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, + struct vring_virtqueue_split *vring_split) +{ + vq->split = *vring_split; + + /* Put everything in free lists. */ + vq->free_head = 0; +} + static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) { struct vring_desc_state_split *state; @@ -1067,8 +1076,6 @@ static struct virtqueue *vring_create_virtqueue_split( return NULL; } - to_vvq(vq)->split.queue_dma_addr = vring_split.queue_dma_addr; - to_vvq(vq)->split.queue_size_in_bytes = vring_split.queue_size_in_bytes; to_vvq(vq)->we_own_ring = true; return vq; @@ -2317,26 +2324,16 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->split.queue_dma_addr = 0; - vq->split.queue_size_in_bytes = 0; - - vq->split.vring = vring_split->vring; - err = vring_alloc_state_extra_split(vring_split); if (err) { kfree(vq); return NULL; } - /* Put everything in free lists. */ - vq->free_head = 0; - - vq->split.desc_state = vring_split->desc_state; - vq->split.desc_extra = vring_split->desc_extra; - virtqueue_vring_init_split(vring_split, vq); virtqueue_init(vq, vring_split->vring.num); + virtqueue_vring_attach_split(vq, vring_split); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); -- cgit From e5175b419a1394e77ff418dd9bfaf15555dfe594 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:34 +0800 Subject: virtio_ring: split: introduce virtqueue_reinit_split() Introduce a function to initialize vq without allocating new ring, desc_state, desc_extra. Subsequent patches will call this function after reset vq to reinitialize vq. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-15-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index d0df887c3ada..948f8da7b780 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -957,6 +957,29 @@ static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split } } +static void virtqueue_reinit_split(struct vring_virtqueue *vq) +{ + int num; + + num = vq->split.vring.num; + + vq->split.vring.avail->flags = 0; + vq->split.vring.avail->idx = 0; + + /* reset avail event */ + vq->split.vring.avail->ring[num] = 0; + + vq->split.vring.used->flags = 0; + vq->split.vring.used->idx = 0; + + /* reset used event */ + *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0; + + virtqueue_init(vq, num); + + virtqueue_vring_init_split(&vq->split, vq); +} + static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, struct vring_virtqueue_split *vring_split) { -- cgit From af36b16f6c1e51975a3815eb21c21c47f3114393 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:35 +0800 Subject: virtio_ring: split: reserve vring_align, may_reduce_num In vring_alloc_queue_split() save vring_align, may_reduce_num to structure vring_virtqueue_split. Used to create a new vring when implementing resize. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-16-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 948f8da7b780..8cbb19cdb1ce 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -105,6 +105,13 @@ struct vring_virtqueue_split { /* DMA address and size information */ dma_addr_t queue_dma_addr; size_t queue_size_in_bytes; + + /* + * The parameters for creating vrings are reserved for creating new + * vring. + */ + u32 vring_align; + bool may_reduce_num; }; struct vring_virtqueue_packed { @@ -1068,6 +1075,9 @@ static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, vring_split->queue_dma_addr = dma_addr; vring_split->queue_size_in_bytes = vring_size(num, vring_align); + vring_split->vring_align = vring_align; + vring_split->may_reduce_num = may_reduce_num; + return 0; } -- cgit From 6fea20e5611fc5c8264f04a878a98c06b6209eed Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:36 +0800 Subject: virtio_ring: split: introduce virtqueue_resize_split() virtio ring split supports resize. Only after the new vring is successfully allocated based on the new num, we will release the old vring. In any case, an error is returned, indicating that the vring still points to the old vring. In the case of an error, re-initialize(virtqueue_reinit_split()) the virtqueue to ensure that the vring can be used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-17-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8cbb19cdb1ce..1852912d711e 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -220,6 +220,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, void (*callback)(struct virtqueue *), const char *name); static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); +static void vring_free(struct virtqueue *_vq); /* * Helpers. @@ -1114,6 +1115,39 @@ static struct virtqueue *vring_create_virtqueue_split( return vq; } +static int virtqueue_resize_split(struct virtqueue *_vq, u32 num) +{ + struct vring_virtqueue_split vring_split = {}; + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = _vq->vdev; + int err; + + err = vring_alloc_queue_split(&vring_split, vdev, num, + vq->split.vring_align, + vq->split.may_reduce_num); + if (err) + goto err; + + err = vring_alloc_state_extra_split(&vring_split); + if (err) + goto err_state_extra; + + vring_free(&vq->vq); + + virtqueue_vring_init_split(&vring_split, vq); + + virtqueue_init(vq, vring_split.vring.num); + virtqueue_vring_attach_split(vq, &vring_split); + + return 0; + +err_state_extra: + vring_free_split(&vring_split, vdev); +err: + virtqueue_reinit_split(vq); + return -ENOMEM; +} + /* * Packed ring specific functions - *_packed(). -- cgit From 6356f8bb01ddf4a6d9f2da16f7c0e3c093c7f548 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:37 +0800 Subject: virtio_ring: packed: introduce vring_free_packed Free the structure struct vring_vritqueue_packed. Subsequent patches require it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-18-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 1852912d711e..2993d817871a 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1832,6 +1832,28 @@ static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num) return desc_extra; } +static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, + struct virtio_device *vdev) +{ + if (vring_packed->vring.desc) + vring_free_queue(vdev, vring_packed->ring_size_in_bytes, + vring_packed->vring.desc, + vring_packed->ring_dma_addr); + + if (vring_packed->vring.driver) + vring_free_queue(vdev, vring_packed->event_size_in_bytes, + vring_packed->vring.driver, + vring_packed->driver_event_dma_addr); + + if (vring_packed->vring.device) + vring_free_queue(vdev, vring_packed->event_size_in_bytes, + vring_packed->vring.device, + vring_packed->device_event_dma_addr); + + kfree(vring_packed->desc_state); + kfree(vring_packed->desc_extra); +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, -- cgit From 6b60b9c008e56e43cdda6ae618f7f58679a8901a Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:38 +0800 Subject: virtio_ring: packed: extract the logic of alloc queue Separate the logic of packed to create vring queue. This feature is required for subsequent virtuqueue reset vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-19-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 80 ++++++++++++++++++++++++++++---------------- 1 file changed, 51 insertions(+), 29 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 2993d817871a..8209923ea7d7 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1854,19 +1854,10 @@ static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, kfree(vring_packed->desc_extra); } -static struct virtqueue *vring_create_virtqueue_packed( - unsigned int index, - unsigned int num, - unsigned int vring_align, - struct virtio_device *vdev, - bool weak_barriers, - bool may_reduce_num, - bool context, - bool (*notify)(struct virtqueue *), - void (*callback)(struct virtqueue *), - const char *name) +static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, + struct virtio_device *vdev, + u32 num) { - struct vring_virtqueue *vq; struct vring_packed_desc *ring; struct vring_packed_desc_event *driver, *device; dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr; @@ -1878,7 +1869,11 @@ static struct virtqueue *vring_create_virtqueue_packed( &ring_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!ring) - goto err_ring; + goto err; + + vring_packed->vring.desc = ring; + vring_packed->ring_dma_addr = ring_dma_addr; + vring_packed->ring_size_in_bytes = ring_size_in_bytes; event_size_in_bytes = sizeof(struct vring_packed_desc_event); @@ -1886,13 +1881,47 @@ static struct virtqueue *vring_create_virtqueue_packed( &driver_event_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!driver) - goto err_driver; + goto err; + + vring_packed->vring.driver = driver; + vring_packed->event_size_in_bytes = event_size_in_bytes; + vring_packed->driver_event_dma_addr = driver_event_dma_addr; device = vring_alloc_queue(vdev, event_size_in_bytes, &device_event_dma_addr, GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO); if (!device) - goto err_device; + goto err; + + vring_packed->vring.device = device; + vring_packed->device_event_dma_addr = device_event_dma_addr; + + vring_packed->vring.num = num; + + return 0; + +err: + vring_free_packed(vring_packed, vdev); + return -ENOMEM; +} + +static struct virtqueue *vring_create_virtqueue_packed( + unsigned int index, + unsigned int num, + unsigned int vring_align, + struct virtio_device *vdev, + bool weak_barriers, + bool may_reduce_num, + bool context, + bool (*notify)(struct virtqueue *), + void (*callback)(struct virtqueue *), + const char *name) +{ + struct vring_virtqueue_packed vring_packed = {}; + struct vring_virtqueue *vq; + + if (vring_alloc_queue_packed(&vring_packed, vdev, num)) + goto err_ring; vq = kmalloc(sizeof(*vq), GFP_KERNEL); if (!vq) @@ -1920,17 +1949,14 @@ static struct virtqueue *vring_create_virtqueue_packed( if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->packed.ring_dma_addr = ring_dma_addr; - vq->packed.driver_event_dma_addr = driver_event_dma_addr; - vq->packed.device_event_dma_addr = device_event_dma_addr; + vq->packed.ring_dma_addr = vring_packed.ring_dma_addr; + vq->packed.driver_event_dma_addr = vring_packed.driver_event_dma_addr; + vq->packed.device_event_dma_addr = vring_packed.device_event_dma_addr; - vq->packed.ring_size_in_bytes = ring_size_in_bytes; - vq->packed.event_size_in_bytes = event_size_in_bytes; + vq->packed.ring_size_in_bytes = vring_packed.ring_size_in_bytes; + vq->packed.event_size_in_bytes = vring_packed.event_size_in_bytes; - vq->packed.vring.num = num; - vq->packed.vring.desc = ring; - vq->packed.vring.driver = driver; - vq->packed.vring.device = device; + vq->packed.vring = vring_packed.vring; vq->packed.next_avail_idx = 0; vq->packed.avail_wrap_counter = 1; @@ -1972,11 +1998,7 @@ err_desc_extra: err_desc_state: kfree(vq); err_vq: - vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr); -err_device: - vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr); -err_driver: - vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr); + vring_free_packed(&vring_packed, vdev); err_ring: return NULL; } -- cgit From ef3167cfd5f28bc86d68e82fc52d07ed03dff2f7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:39 +0800 Subject: virtio_ring: packed: extract the logic of alloc state and extra Separate the logic for alloc desc_state and desc_extra, which will be called separately by subsequent patches. Use struct vring_packed to pass desc_state, desc_extra. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-20-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 48 +++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 8209923ea7d7..533af061d009 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1905,6 +1905,33 @@ err: return -ENOMEM; } +static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed) +{ + struct vring_desc_state_packed *state; + struct vring_desc_extra *extra; + u32 num = vring_packed->vring.num; + + state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL); + if (!state) + goto err_desc_state; + + memset(state, 0, num * sizeof(struct vring_desc_state_packed)); + + extra = vring_alloc_desc_extra(num); + if (!extra) + goto err_desc_extra; + + vring_packed->desc_state = state; + vring_packed->desc_extra = extra; + + return 0; + +err_desc_extra: + kfree(state); +err_desc_state: + return -ENOMEM; +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1919,6 +1946,7 @@ static struct virtqueue *vring_create_virtqueue_packed( { struct vring_virtqueue_packed vring_packed = {}; struct vring_virtqueue *vq; + int err; if (vring_alloc_queue_packed(&vring_packed, vdev, num)) goto err_ring; @@ -1963,21 +1991,15 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.event_flags_shadow = 0; vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; - vq->packed.desc_state = kmalloc_array(num, - sizeof(struct vring_desc_state_packed), - GFP_KERNEL); - if (!vq->packed.desc_state) - goto err_desc_state; - - memset(vq->packed.desc_state, 0, - num * sizeof(struct vring_desc_state_packed)); + err = vring_alloc_state_extra_packed(&vring_packed); + if (err) + goto err_state_extra; /* Put everything in free lists. */ vq->free_head = 0; - vq->packed.desc_extra = vring_alloc_desc_extra(num); - if (!vq->packed.desc_extra) - goto err_desc_extra; + vq->packed.desc_state = vring_packed.desc_state; + vq->packed.desc_extra = vring_packed.desc_extra; /* No callback? Tell other side not to bother us. */ if (!callback) { @@ -1993,9 +2015,7 @@ static struct virtqueue *vring_create_virtqueue_packed( spin_unlock(&vdev->vqs_list_lock); return &vq->vq; -err_desc_extra: - kfree(vq->packed.desc_state); -err_desc_state: +err_state_extra: kfree(vq); err_vq: vring_free_packed(&vring_packed, vdev); -- cgit From 1a107c87ebcf04ede1b86e5de192df34b6e50bd7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:40 +0800 Subject: virtio_ring: packed: extract the logic of vring init Separate the logic of initializing vring, and subsequent patches will call it separately. This function completes the variable initialization of packed vring. It together with the logic of atatch constitutes the initialization of vring. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-21-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 533af061d009..a6f90e6e621b 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1932,6 +1932,22 @@ err_desc_state: return -ENOMEM; } +static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed, + bool callback) +{ + vring_packed->next_avail_idx = 0; + vring_packed->avail_wrap_counter = 1; + vring_packed->event_flags_shadow = 0; + vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; + + /* No callback? Tell other side not to bother us. */ + if (!callback) { + vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; + vring_packed->vring.driver->flags = + cpu_to_le16(vring_packed->event_flags_shadow); + } +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1986,11 +2002,6 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.vring = vring_packed.vring; - vq->packed.next_avail_idx = 0; - vq->packed.avail_wrap_counter = 1; - vq->packed.event_flags_shadow = 0; - vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; - err = vring_alloc_state_extra_packed(&vring_packed); if (err) goto err_state_extra; @@ -2001,12 +2012,7 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->packed.desc_state = vring_packed.desc_state; vq->packed.desc_extra = vring_packed.desc_extra; - /* No callback? Tell other side not to bother us. */ - if (!callback) { - vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; - vq->packed.vring.driver->flags = - cpu_to_le16(vq->packed.event_flags_shadow); - } + virtqueue_vring_init_packed(&vring_packed, !!callback); virtqueue_init(vq, num); -- cgit From 51d649f14aae0986c62cf798b262f99b49836e88 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:41 +0800 Subject: virtio_ring: packed: extract the logic of attach vring Separate the logic of attach vring, the subsequent patch will call it separately. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-22-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index a6f90e6e621b..93d2a950f536 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1948,6 +1948,15 @@ static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_pac } } +static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, + struct vring_virtqueue_packed *vring_packed) +{ + vq->packed = *vring_packed; + + /* Put everything in free lists. */ + vq->free_head = 0; +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, @@ -1993,28 +2002,14 @@ static struct virtqueue *vring_create_virtqueue_packed( if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) vq->weak_barriers = false; - vq->packed.ring_dma_addr = vring_packed.ring_dma_addr; - vq->packed.driver_event_dma_addr = vring_packed.driver_event_dma_addr; - vq->packed.device_event_dma_addr = vring_packed.device_event_dma_addr; - - vq->packed.ring_size_in_bytes = vring_packed.ring_size_in_bytes; - vq->packed.event_size_in_bytes = vring_packed.event_size_in_bytes; - - vq->packed.vring = vring_packed.vring; - err = vring_alloc_state_extra_packed(&vring_packed); if (err) goto err_state_extra; - /* Put everything in free lists. */ - vq->free_head = 0; - - vq->packed.desc_state = vring_packed.desc_state; - vq->packed.desc_extra = vring_packed.desc_extra; - virtqueue_vring_init_packed(&vring_packed, !!callback); virtqueue_init(vq, num); + virtqueue_vring_attach_packed(vq, &vring_packed); spin_lock(&vdev->vqs_list_lock); list_add_tail(&vq->vq.list, &vdev->vqs); -- cgit From 56775e141b18790f70e05537f3a1417565e766ac Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:42 +0800 Subject: virtio_ring: packed: introduce virtqueue_reinit_packed() Introduce a function to initialize vq without allocating new ring, desc_state, desc_extra. Subsequent patches will call this function after reset vq to reinitialize vq. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-23-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 93d2a950f536..db06fb0ddfd6 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1957,6 +1957,18 @@ static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, vq->free_head = 0; } +static void virtqueue_reinit_packed(struct vring_virtqueue *vq) +{ + memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes); + memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes); + + /* we need to reset the desc.flags. For more, see is_used_desc_packed() */ + memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); + + virtqueue_init(vq, vq->packed.vring.num); + virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); +} + static struct virtqueue *vring_create_virtqueue_packed( unsigned int index, unsigned int num, -- cgit From 947f9fcf674f8f3f09e01dfb5d8e564650875878 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:43 +0800 Subject: virtio_ring: packed: introduce virtqueue_resize_packed() virtio ring packed supports resize. Only after the new vring is successfully allocated based on the new num, we will release the old vring. In any case, an error is returned, indicating that the vring still points to the old vring. In the case of an error, re-initialize(by virtqueue_reinit_packed()) the virtqueue to ensure that the vring can be used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-24-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index db06fb0ddfd6..bea5a3448217 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2036,6 +2036,36 @@ err_ring: return NULL; } +static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num) +{ + struct vring_virtqueue_packed vring_packed = {}; + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = _vq->vdev; + int err; + + if (vring_alloc_queue_packed(&vring_packed, vdev, num)) + goto err_ring; + + err = vring_alloc_state_extra_packed(&vring_packed); + if (err) + goto err_state_extra; + + vring_free(&vq->vq); + + virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback); + + virtqueue_init(vq, vring_packed.vring.num); + virtqueue_vring_attach_packed(vq, &vring_packed); + + return 0; + +err_state_extra: + vring_free_packed(&vring_packed, vdev); +err_ring: + virtqueue_reinit_packed(vq); + return -ENOMEM; +} + /* * Generic functions and exported symbols. -- cgit From c790e8e1817f1a17c05e64f1c4f16f231b8529d5 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:44 +0800 Subject: virtio_ring: introduce virtqueue_resize() Introduce virtqueue_resize() to implement the resize of vring. Based on these, the driver can dynamically adjust the size of the vring. For example: ethtool -G. virtqueue_resize() implements resize based on the vq reset function. In case of failure to allocate a new vring, it will give up resize and use the original vring. During this process, if the re-enable reset vq fails, the vq can no longer be used. Although the probability of this situation is not high. The parameter recycle is used to recycle the buffer that is no longer used. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-25-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 69 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index bea5a3448217..6447a09e2e38 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2539,6 +2539,75 @@ struct virtqueue *vring_create_virtqueue( } EXPORT_SYMBOL_GPL(vring_create_virtqueue); +/** + * virtqueue_resize - resize the vring of vq + * @_vq: the struct virtqueue we're talking about. + * @num: new ring num + * @recycle: callback for recycle the useless buffer + * + * When it is really necessary to create a new vring, it will set the current vq + * into the reset state. Then call the passed callback to recycle the buffer + * that is no longer used. Only after the new vring is successfully created, the + * old vring will be released. + * + * Caller must ensure we don't call this with other virtqueue operations + * at the same time (except where noted). + * + * Returns zero or a negative error. + * 0: success. + * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size. + * vq can still work normally + * -EBUSY: Failed to sync with device, vq may not work properly + * -ENOENT: Transport or device not supported + * -E2BIG/-EINVAL: num error + * -EPERM: Operation not permitted + * + */ +int virtqueue_resize(struct virtqueue *_vq, u32 num, + void (*recycle)(struct virtqueue *vq, void *buf)) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + struct virtio_device *vdev = vq->vq.vdev; + void *buf; + int err; + + if (!vq->we_own_ring) + return -EPERM; + + if (num > vq->vq.num_max) + return -E2BIG; + + if (!num) + return -EINVAL; + + if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) + return 0; + + if (!vdev->config->disable_vq_and_reset) + return -ENOENT; + + if (!vdev->config->enable_vq_after_reset) + return -ENOENT; + + err = vdev->config->disable_vq_and_reset(_vq); + if (err) + return err; + + while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL) + recycle(_vq, buf); + + if (vq->packed_ring) + err = virtqueue_resize_packed(_vq, num); + else + err = virtqueue_resize_split(_vq, num); + + if (vdev->config->enable_vq_after_reset(_vq)) + return -EBUSY; + + return err; +} +EXPORT_SYMBOL_GPL(virtqueue_resize); + /* Only available for split ring */ struct virtqueue *vring_new_virtqueue(unsigned int index, unsigned int num, -- cgit From 3251063155032729b8793ac3957136ae25c0bafa Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:46 +0800 Subject: virtio: allow to unbreak/break virtqueue individually This patch allows the new introduced __virtqueue_break()/__virtqueue_unbreak() to break/unbreak the virtqueue. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-27-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 6447a09e2e38..accb3ae6cc95 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -2724,6 +2724,30 @@ unsigned int virtqueue_get_vring_size(struct virtqueue *_vq) } EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_break(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, true); +} +EXPORT_SYMBOL_GPL(__virtqueue_break); + +/* + * This function should only be called by the core, not directly by the driver. + */ +void __virtqueue_unbreak(struct virtqueue *_vq) +{ + struct vring_virtqueue *vq = to_vvq(_vq); + + /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ + WRITE_ONCE(vq->broken, false); +} +EXPORT_SYMBOL_GPL(__virtqueue_unbreak); + bool virtqueue_is_broken(struct virtqueue *_vq) { struct vring_virtqueue *vq = to_vvq(_vq); -- cgit From 4913e85441b40386c4bb093f188b955d8165f1b7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:48 +0800 Subject: virtio_ring: struct virtqueue introduce reset Introduce a new member reset to the structure virtqueue to determine whether the current vq is in the reset state. Subsequent patches will use it. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-29-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_ring.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index accb3ae6cc95..d66c8e6d0ef3 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -1996,6 +1996,7 @@ static struct virtqueue *vring_create_virtqueue_packed( vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = true; vq->notify = notify; vq->weak_barriers = weak_barriers; @@ -2481,6 +2482,7 @@ static struct virtqueue *__vring_new_virtqueue(unsigned int index, vq->vq.vdev = vdev; vq->vq.name = name; vq->vq.index = index; + vq->vq.reset = false; vq->we_own_ring = false; vq->notify = notify; vq->weak_barriers = weak_barriers; -- cgit From 0b50cece0b7857732d2055f2c77f8730c10f9196 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:50 +0800 Subject: virtio_pci: introduce helper to get/set queue reset Introduce new helpers to implement queue reset and get queue reset status. https://github.com/oasis-tcs/virtio-spec/issues/124 https://github.com/oasis-tcs/virtio-spec/issues/139 Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-31-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern_dev.c | 39 ++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'drivers') diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index fa2a9445bb18..869cb46bef96 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -3,6 +3,7 @@ #include #include #include +#include /* * vp_modern_map_capability - map a part of virtio pci capability @@ -474,6 +475,44 @@ void vp_modern_set_status(struct virtio_pci_modern_device *mdev, } EXPORT_SYMBOL_GPL(vp_modern_set_status); +/* + * vp_modern_get_queue_reset - get the queue reset status + * @mdev: the modern virtio-pci device + * @index: queue index + */ +int vp_modern_get_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + return vp_ioread16(&cfg->queue_reset); +} +EXPORT_SYMBOL_GPL(vp_modern_get_queue_reset); + +/* + * vp_modern_set_queue_reset - reset the queue + * @mdev: the modern virtio-pci device + * @index: queue index + */ +void vp_modern_set_queue_reset(struct virtio_pci_modern_device *mdev, u16 index) +{ + struct virtio_pci_modern_common_cfg __iomem *cfg; + + cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common; + + vp_iowrite16(index, &cfg->cfg.queue_select); + vp_iowrite16(1, &cfg->queue_reset); + + while (vp_ioread16(&cfg->queue_reset)) + msleep(1); + + while (vp_ioread16(&cfg->cfg.queue_enable)) + msleep(1); +} +EXPORT_SYMBOL_GPL(vp_modern_set_queue_reset); + /* * vp_modern_queue_vector - set the MSIX vector for a specific virtqueue * @mdev: the modern virtio-pci device -- cgit From 56bdc06139404b3b06ed75ec99b93445d7e0b8c3 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:51 +0800 Subject: virtio_pci: extract the logic of active vq for modern pci Introduce vp_active_vq() to configure vring to backend after vq attach vring. And configure vq vector if necessary. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-32-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_modern.c | 46 +++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 18 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index e7e0b8c850f6..9041d9a41b7d 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -176,6 +176,29 @@ static void vp_reset(struct virtio_device *vdev) vp_synchronize_vectors(vdev); } +static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + unsigned long index; + + index = vq->index; + + /* activate the queue */ + vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); + vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), + virtqueue_get_avail_addr(vq), + virtqueue_get_used_addr(vq)); + + if (msix_vec != VIRTIO_MSI_NO_VECTOR) { + msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); + if (msix_vec == VIRTIO_MSI_NO_VECTOR) + return -EBUSY; + } + + return 0; +} + static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) { return vp_modern_config_vector(&vp_dev->mdev, vector); @@ -220,32 +243,19 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, vq->num_max = num; - /* activate the queue */ - vp_modern_set_queue_size(mdev, index, virtqueue_get_vring_size(vq)); - vp_modern_queue_address(mdev, index, virtqueue_get_desc_addr(vq), - virtqueue_get_avail_addr(vq), - virtqueue_get_used_addr(vq)); + err = vp_active_vq(vq, msix_vec); + if (err) + goto err; vq->priv = (void __force *)vp_modern_map_vq_notify(mdev, index, NULL); if (!vq->priv) { err = -ENOMEM; - goto err_map_notify; - } - - if (msix_vec != VIRTIO_MSI_NO_VECTOR) { - msix_vec = vp_modern_queue_vector(mdev, index, msix_vec); - if (msix_vec == VIRTIO_MSI_NO_VECTOR) { - err = -EBUSY; - goto err_assign_vector; - } + goto err; } return vq; -err_assign_vector: - if (!mdev->notify_base) - pci_iounmap(mdev->pci_dev, (void __iomem __force *)vq->priv); -err_map_notify: +err: vring_del_virtqueue(vq); return ERR_PTR(err); } -- cgit From 04ca0b0b16f11faf74fa92468dab51b8372586cd Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:52 +0800 Subject: virtio_pci: support VIRTIO_F_RING_RESET This patch implements virtio pci support for QUEUE RESET. Performing reset on a queue is divided into these steps: 1. notify the device to reset the queue 2. recycle the buffer submitted 3. reset the vring (may re-alloc) 4. mmap vring to device, and enable the queue This patch implements virtio_reset_vq(), virtio_enable_resetq() in the pci scenario. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-33-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_common.c | 12 ++++-- drivers/virtio/virtio_pci_modern.c | 88 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ca51fcc9daab..ad258a9d3b9f 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -214,9 +214,15 @@ static void vp_del_vq(struct virtqueue *vq) struct virtio_pci_vq_info *info = vp_dev->vqs[vq->index]; unsigned long flags; - spin_lock_irqsave(&vp_dev->lock, flags); - list_del(&info->node); - spin_unlock_irqrestore(&vp_dev->lock, flags); + /* + * If it fails during re-enable reset vq. This way we won't rejoin + * info->node to the queue. Prevent unexpected irqs. + */ + if (!vq->reset) { + spin_lock_irqsave(&vp_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vp_dev->lock, flags); + } vp_dev->del_vq(info); kfree(info); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index 9041d9a41b7d..c3b9f2761849 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -34,6 +34,9 @@ static void vp_transport_features(struct virtio_device *vdev, u64 features) if ((features & BIT_ULL(VIRTIO_F_SR_IOV)) && pci_find_ext_capability(pci_dev, PCI_EXT_CAP_ID_SRIOV)) __virtio_set_bit(vdev, VIRTIO_F_SR_IOV); + + if (features & BIT_ULL(VIRTIO_F_RING_RESET)) + __virtio_set_bit(vdev, VIRTIO_F_RING_RESET); } /* virtio config->finalize_features() implementation */ @@ -199,6 +202,87 @@ static int vp_active_vq(struct virtqueue *vq, u16 msix_vec) return 0; } +static int vp_modern_disable_vq_and_reset(struct virtqueue *vq) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + struct virtio_pci_vq_info *info; + unsigned long flags; + + if (!virtio_has_feature(vq->vdev, VIRTIO_F_RING_RESET)) + return -ENOENT; + + vp_modern_set_queue_reset(mdev, vq->index); + + info = vp_dev->vqs[vq->index]; + + /* delete vq from irq handler */ + spin_lock_irqsave(&vp_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vp_dev->lock, flags); + + INIT_LIST_HEAD(&info->node); + +#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION + __virtqueue_break(vq); +#endif + + /* For the case where vq has an exclusive irq, call synchronize_irq() to + * wait for completion. + * + * note: We can't use disable_irq() since it conflicts with the affinity + * managed IRQ that is used by some drivers. + */ + if (vp_dev->per_vq_vectors && info->msix_vector != VIRTIO_MSI_NO_VECTOR) + synchronize_irq(pci_irq_vector(vp_dev->pci_dev, info->msix_vector)); + + vq->reset = true; + + return 0; +} + +static int vp_modern_enable_vq_after_reset(struct virtqueue *vq) +{ + struct virtio_pci_device *vp_dev = to_vp_device(vq->vdev); + struct virtio_pci_modern_device *mdev = &vp_dev->mdev; + struct virtio_pci_vq_info *info; + unsigned long flags, index; + int err; + + if (!vq->reset) + return -EBUSY; + + index = vq->index; + info = vp_dev->vqs[index]; + + if (vp_modern_get_queue_reset(mdev, index)) + return -EBUSY; + + if (vp_modern_get_queue_enable(mdev, index)) + return -EBUSY; + + err = vp_active_vq(vq, info->msix_vector); + if (err) + return err; + + if (vq->callback) { + spin_lock_irqsave(&vp_dev->lock, flags); + list_add(&info->node, &vp_dev->virtqueues); + spin_unlock_irqrestore(&vp_dev->lock, flags); + } else { + INIT_LIST_HEAD(&info->node); + } + +#ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION + __virtqueue_unbreak(vq); +#endif + + vp_modern_set_queue_enable(&vp_dev->mdev, index, true); + vq->reset = false; + + return 0; +} + static u16 vp_config_vector(struct virtio_pci_device *vp_dev, u16 vector) { return vp_modern_config_vector(&vp_dev->mdev, vector); @@ -413,6 +497,8 @@ static const struct virtio_config_ops virtio_pci_config_nodev_ops = { .set_vq_affinity = vp_set_vq_affinity, .get_vq_affinity = vp_get_vq_affinity, .get_shm_region = vp_get_shm_region, + .disable_vq_and_reset = vp_modern_disable_vq_and_reset, + .enable_vq_after_reset = vp_modern_enable_vq_after_reset, }; static const struct virtio_config_ops virtio_pci_config_ops = { @@ -431,6 +517,8 @@ static const struct virtio_config_ops virtio_pci_config_ops = { .set_vq_affinity = vp_set_vq_affinity, .get_vq_affinity = vp_get_vq_affinity, .get_shm_region = vp_get_shm_region, + .disable_vq_and_reset = vp_modern_disable_vq_and_reset, + .enable_vq_after_reset = vp_modern_enable_vq_after_reset, }; /* the PCI probing function */ -- cgit From a10fba0377145fccefea4dc4dd5915b7ed87e546 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:53 +0800 Subject: virtio: find_vqs() add arg sizes find_vqs() adds a new parameter sizes to specify the size of each vq vring. NULL as sizes means that all queues in find_vqs() use the maximum size. A value in the array is 0, which means that the corresponding queue uses the maximum size. In the split scenario, the meaning of size is the largest size, because it may be limited by memory, the virtio core will try a smaller size. And the size is power of 2. Signed-off-by: Xuan Zhuo Acked-by: Hans de Goede Reviewed-by: Mathieu Poirier Acked-by: Jason Wang Message-Id: <20220801063902.129329-34-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/platform/mellanox/mlxbf-tmfifo.c | 1 + drivers/remoteproc/remoteproc_virtio.c | 1 + drivers/s390/virtio/virtio_ccw.c | 1 + drivers/virtio/virtio_mmio.c | 1 + drivers/virtio/virtio_pci_common.c | 2 +- drivers/virtio/virtio_pci_common.h | 2 +- drivers/virtio/virtio_pci_modern.c | 7 +++++-- drivers/virtio/virtio_vdpa.c | 1 + 8 files changed, 12 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/platform/mellanox/mlxbf-tmfifo.c b/drivers/platform/mellanox/mlxbf-tmfifo.c index 1ae3c56b66b0..8be13d416f48 100644 --- a/drivers/platform/mellanox/mlxbf-tmfifo.c +++ b/drivers/platform/mellanox/mlxbf-tmfifo.c @@ -928,6 +928,7 @@ static int mlxbf_tmfifo_virtio_find_vqs(struct virtio_device *vdev, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/remoteproc/remoteproc_virtio.c b/drivers/remoteproc/remoteproc_virtio.c index 0f7706e23eb9..81c4f5776109 100644 --- a/drivers/remoteproc/remoteproc_virtio.c +++ b/drivers/remoteproc/remoteproc_virtio.c @@ -158,6 +158,7 @@ static int rproc_virtio_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool * ctx, struct irq_affinity *desc) { diff --git a/drivers/s390/virtio/virtio_ccw.c b/drivers/s390/virtio/virtio_ccw.c index 6b86d0280d6b..72500cd2dbf5 100644 --- a/drivers/s390/virtio/virtio_ccw.c +++ b/drivers/s390/virtio/virtio_ccw.c @@ -635,6 +635,7 @@ static int virtio_ccw_find_vqs(struct virtio_device *vdev, unsigned nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index 3ff746e3f24a..dfcecfd7aba1 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -474,6 +474,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index ad258a9d3b9f..7ad734584823 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -396,7 +396,7 @@ out_del_vqs: /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc) { int err; diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index 23112d84218f..a5ff838b85a5 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -110,7 +110,7 @@ void vp_del_vqs(struct virtio_device *vdev); /* the config->find_vqs() implementation */ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], u32 sizes[], const bool *ctx, struct irq_affinity *desc); const char *vp_bus_name(struct virtio_device *vdev); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index c3b9f2761849..be51ec849252 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -347,12 +347,15 @@ err: static int vp_modern_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx, + const char * const names[], + u32 sizes[], + const bool *ctx, struct irq_affinity *desc) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); struct virtqueue *vq; - int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, ctx, desc); + int rc = vp_find_vqs(vdev, nvqs, vqs, callbacks, names, sizes, ctx, + desc); if (rc) return rc; diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 9670cc79371d..832d2c5b1b19 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -269,6 +269,7 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], const char * const names[], + u32 sizes[], const bool *ctx, struct irq_affinity *desc) { -- cgit From cdb44806fca2d0ad29ca644cbf1505433902ee0c Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:54 +0800 Subject: virtio_pci: support the arg sizes of find_vqs() Virtio PCI supports new parameter sizes of find_vqs(). Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-35-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_pci_common.c | 18 ++++++++++-------- drivers/virtio/virtio_pci_common.h | 1 + drivers/virtio/virtio_pci_legacy.c | 6 +++++- drivers/virtio/virtio_pci_modern.c | 10 +++++++--- 4 files changed, 23 insertions(+), 12 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 7ad734584823..00ad476a815d 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -174,6 +174,7 @@ error: static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -186,7 +187,7 @@ static struct virtqueue *vp_setup_vq(struct virtio_device *vdev, unsigned int in if (!info) return ERR_PTR(-ENOMEM); - vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, ctx, + vq = vp_dev->setup_vq(vp_dev, info, index, callback, name, size, ctx, msix_vec); if (IS_ERR(vq)) goto out_info; @@ -283,7 +284,7 @@ void vp_del_vqs(struct virtio_device *vdev) static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], bool per_vq_vectors, + const char * const names[], u32 sizes[], bool per_vq_vectors, const bool *ctx, struct irq_affinity *desc) { @@ -326,8 +327,8 @@ static int vp_find_vqs_msix(struct virtio_device *vdev, unsigned int nvqs, else msix_vec = VP_MSIX_VQ_VECTOR; vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false, - msix_vec); + sizes ? sizes[i] : 0, + ctx ? ctx[i] : false, msix_vec); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto error_find; @@ -357,7 +358,7 @@ error_find: static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, struct virtqueue *vqs[], vq_callback_t *callbacks[], - const char * const names[], const bool *ctx) + const char * const names[], u32 sizes[], const bool *ctx) { struct virtio_pci_device *vp_dev = to_vp_device(vdev); int i, err, queue_idx = 0; @@ -379,6 +380,7 @@ static int vp_find_vqs_intx(struct virtio_device *vdev, unsigned int nvqs, continue; } vqs[i] = vp_setup_vq(vdev, queue_idx++, callbacks[i], names[i], + sizes ? sizes[i] : 0, ctx ? ctx[i] : false, VIRTIO_MSI_NO_VECTOR); if (IS_ERR(vqs[i])) { @@ -402,15 +404,15 @@ int vp_find_vqs(struct virtio_device *vdev, unsigned int nvqs, int err; /* Try MSI-X with one vector per queue. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, true, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, true, ctx, desc); if (!err) return 0; /* Fallback: MSI-X with one vector for config, one shared for queues. */ - err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, false, ctx, desc); + err = vp_find_vqs_msix(vdev, nvqs, vqs, callbacks, names, sizes, false, ctx, desc); if (!err) return 0; /* Finally fall back to regular interrupts. */ - return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, ctx); + return vp_find_vqs_intx(vdev, nvqs, vqs, callbacks, names, sizes, ctx); } const char *vp_bus_name(struct virtio_device *vdev) diff --git a/drivers/virtio/virtio_pci_common.h b/drivers/virtio/virtio_pci_common.h index a5ff838b85a5..c0448378b698 100644 --- a/drivers/virtio/virtio_pci_common.h +++ b/drivers/virtio/virtio_pci_common.h @@ -80,6 +80,7 @@ struct virtio_pci_device { unsigned int idx, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec); void (*del_vq)(struct virtio_pci_vq_info *info); diff --git a/drivers/virtio/virtio_pci_legacy.c b/drivers/virtio/virtio_pci_legacy.c index 2257f1b3d8ae..d75e5c4e637f 100644 --- a/drivers/virtio/virtio_pci_legacy.c +++ b/drivers/virtio/virtio_pci_legacy.c @@ -112,6 +112,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -125,10 +126,13 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!num || vp_legacy_get_queue_enable(&vp_dev->ldev, index)) return ERR_PTR(-ENOENT); + if (!size || size > num) + size = num; + info->msix_vector = msix_vec; /* create the vring */ - vq = vring_create_virtqueue(index, num, + vq = vring_create_virtqueue(index, size, VIRTIO_PCI_VRING_ALIGN, &vp_dev->vdev, true, false, ctx, vp_notify, callback, name); diff --git a/drivers/virtio/virtio_pci_modern.c b/drivers/virtio/virtio_pci_modern.c index be51ec849252..f7965c5dd36b 100644 --- a/drivers/virtio/virtio_pci_modern.c +++ b/drivers/virtio/virtio_pci_modern.c @@ -293,6 +293,7 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, unsigned int index, void (*callback)(struct virtqueue *vq), const char *name, + u32 size, bool ctx, u16 msix_vec) { @@ -310,15 +311,18 @@ static struct virtqueue *setup_vq(struct virtio_pci_device *vp_dev, if (!num || vp_modern_get_queue_enable(mdev, index)) return ERR_PTR(-ENOENT); - if (num & (num - 1)) { - dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", num); + if (!size || size > num) + size = num; + + if (size & (size - 1)) { + dev_warn(&vp_dev->pci_dev->dev, "bad queue size %u", size); return ERR_PTR(-EINVAL); } info->msix_vector = msix_vec; /* create the vring */ - vq = vring_create_virtqueue(index, num, + vq = vring_create_virtqueue(index, size, SMP_CACHE_BYTES, &vp_dev->vdev, true, true, ctx, vp_notify, callback, name); -- cgit From fbed86abba6e0472d98079790e58060e4332608a Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:55 +0800 Subject: virtio_mmio: support the arg sizes of find_vqs() Virtio MMIO support the new parameter sizes of find_vqs(). Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-36-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_mmio.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index dfcecfd7aba1..c492a57531c6 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -360,7 +360,7 @@ static void vm_synchronize_cbs(struct virtio_device *vdev) static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), - const char *name, bool ctx) + const char *name, u32 size, bool ctx) { struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); struct virtio_mmio_vq_info *info; @@ -395,8 +395,11 @@ static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int in goto error_new_virtqueue; } + if (!size || size > num) + size = num; + /* Create the vring */ - vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, + vq = vring_create_virtqueue(index, size, VIRTIO_MMIO_VRING_ALIGN, vdev, true, true, ctx, vm_notify, callback, name); if (!vq) { err = -ENOMEM; @@ -500,6 +503,7 @@ static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, } vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i], + sizes ? sizes[i] : 0, ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { vm_del_vqs(vdev); -- cgit From 762faee5a2678559d3dc09d95f8f2c54cd0466a7 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:57 +0800 Subject: virtio_net: set the default max ring size by find_vqs() Use virtio_find_vqs_ctx_size() to specify the maximum ring size of tx, rx at the same time. | rx/tx ring size ------------------------------------------- speed == UNKNOWN or < 10G| 1024 speed < 40G | 4096 speed >= 40G | 8192 Call virtnet_update_settings() once before calling init_vqs() to update speed. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-38-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index ec8e1b3108c3..00948b0fb589 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3225,6 +3225,29 @@ static unsigned int mergeable_min_buf_len(struct virtnet_info *vi, struct virtqu (unsigned int)GOOD_PACKET_LEN); } +static void virtnet_config_sizes(struct virtnet_info *vi, u32 *sizes) +{ + u32 i, rx_size, tx_size; + + if (vi->speed == SPEED_UNKNOWN || vi->speed < SPEED_10000) { + rx_size = 1024; + tx_size = 1024; + + } else if (vi->speed < SPEED_40000) { + rx_size = 1024 * 4; + tx_size = 1024 * 4; + + } else { + rx_size = 1024 * 8; + tx_size = 1024 * 8; + } + + for (i = 0; i < vi->max_queue_pairs; i++) { + sizes[rxq2vq(i)] = rx_size; + sizes[txq2vq(i)] = tx_size; + } +} + static int virtnet_find_vqs(struct virtnet_info *vi) { vq_callback_t **callbacks; @@ -3232,6 +3255,7 @@ static int virtnet_find_vqs(struct virtnet_info *vi) int ret = -ENOMEM; int i, total_vqs; const char **names; + u32 *sizes; bool *ctx; /* We expect 1 RX virtqueue followed by 1 TX virtqueue, followed by @@ -3259,10 +3283,15 @@ static int virtnet_find_vqs(struct virtnet_info *vi) ctx = NULL; } + sizes = kmalloc_array(total_vqs, sizeof(*sizes), GFP_KERNEL); + if (!sizes) + goto err_sizes; + /* Parameters for control virtqueue, if any */ if (vi->has_cvq) { callbacks[total_vqs - 1] = NULL; names[total_vqs - 1] = "control"; + sizes[total_vqs - 1] = 64; } /* Allocate/initialize parameters for send/receive virtqueues */ @@ -3277,8 +3306,10 @@ static int virtnet_find_vqs(struct virtnet_info *vi) ctx[rxq2vq(i)] = true; } - ret = virtio_find_vqs_ctx(vi->vdev, total_vqs, vqs, callbacks, - names, ctx, NULL); + virtnet_config_sizes(vi, sizes); + + ret = virtio_find_vqs_ctx_size(vi->vdev, total_vqs, vqs, callbacks, + names, sizes, ctx, NULL); if (ret) goto err_find; @@ -3298,6 +3329,8 @@ static int virtnet_find_vqs(struct virtnet_info *vi) err_find: + kfree(sizes); +err_sizes: kfree(ctx); err_ctx: kfree(names); @@ -3648,6 +3681,9 @@ static int virtnet_probe(struct virtio_device *vdev) vi->curr_queue_pairs = num_online_cpus(); vi->max_queue_pairs = max_queue_pairs; + virtnet_init_settings(dev); + virtnet_update_settings(vi); + /* Allocate/initialize the rx/tx queues, and invoke find_vqs */ err = init_vqs(vi); if (err) @@ -3660,8 +3696,6 @@ static int virtnet_probe(struct virtio_device *vdev) netif_set_real_num_tx_queues(dev, vi->curr_queue_pairs); netif_set_real_num_rx_queues(dev, vi->curr_queue_pairs); - virtnet_init_settings(dev); - if (virtio_has_feature(vdev, VIRTIO_NET_F_STANDBY)) { vi->failover = net_failover_create(vi->dev); if (IS_ERR(vi->failover)) { -- cgit From 8597b5ddcbba840ab852e1083f1a8dd1c6d41d1b Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:58 +0800 Subject: virtio_net: get ringparam by virtqueue_get_vring_max_size() Use virtqueue_get_vring_max_size() in virtnet_get_ringparam() to set tx,rx_max_pending. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-39-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 00948b0fb589..204bfe49d6b4 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2282,10 +2282,10 @@ static void virtnet_get_ringparam(struct net_device *dev, { struct virtnet_info *vi = netdev_priv(dev); - ring->rx_max_pending = virtqueue_get_vring_size(vi->rq[0].vq); - ring->tx_max_pending = virtqueue_get_vring_size(vi->sq[0].vq); - ring->rx_pending = ring->rx_max_pending; - ring->tx_pending = ring->tx_max_pending; + ring->rx_max_pending = vi->rq[0].vq->num_max; + ring->tx_max_pending = vi->sq[0].vq->num_max; + ring->rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); + ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); } static bool virtnet_commit_rss_command(struct virtnet_info *vi) -- cgit From 6e345f8c7cd029ad3aaece15ad4425ac26e4eb63 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:38:59 +0800 Subject: virtio_net: split free_unused_bufs() This patch separates two functions for freeing sq buf and rq buf from free_unused_bufs(). When supporting the enable/disable tx/rq queue in the future, it is necessary to support separate recovery of a sq buf or a rq buf. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-40-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 204bfe49d6b4..8cad913926e5 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -3168,6 +3168,27 @@ static void free_receive_page_frags(struct virtnet_info *vi) put_page(vi->rq[i].alloc_frag.page); } +static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf) +{ + if (!is_xdp_frame(buf)) + dev_kfree_skb(buf); + else + xdp_return_frame(ptr_to_xdp(buf)); +} + +static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf) +{ + struct virtnet_info *vi = vq->vdev->priv; + int i = vq2rxq(vq); + + if (vi->mergeable_rx_bufs) + put_page(virt_to_head_page(buf)); + else if (vi->big_packets) + give_pages(&vi->rq[i], buf); + else + put_page(virt_to_head_page(buf)); +} + static void free_unused_bufs(struct virtnet_info *vi) { void *buf; @@ -3175,26 +3196,14 @@ static void free_unused_bufs(struct virtnet_info *vi) for (i = 0; i < vi->max_queue_pairs; i++) { struct virtqueue *vq = vi->sq[i].vq; - while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { - if (!is_xdp_frame(buf)) - dev_kfree_skb(buf); - else - xdp_return_frame(ptr_to_xdp(buf)); - } + while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) + virtnet_sq_free_unused_buf(vq, buf); } for (i = 0; i < vi->max_queue_pairs; i++) { struct virtqueue *vq = vi->rq[i].vq; - - while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) { - if (vi->mergeable_rx_bufs) { - put_page(virt_to_head_page(buf)); - } else if (vi->big_packets) { - give_pages(&vi->rq[i], buf); - } else { - put_page(virt_to_head_page(buf)); - } - } + while ((buf = virtqueue_detach_unused_buf(vq)) != NULL) + virtnet_rq_free_unused_buf(vq, buf); } } -- cgit From 6a4763e268030362340bc3286560d44be15a7421 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:39:00 +0800 Subject: virtio_net: support rx queue resize This patch implements the resize function of the rx queues. Based on this function, it is possible to modify the ring num of the queue. Includes fixup: virtio_net: fix for stuck when change rx ring size with dev down When dev is set to DOWN state, napi has been disabled, if we modify the ring size at this time, we should not call napi_disable() again, which will cause stuck. And all operations are under the protection of rtnl_lock, so there is no need to consider concurrency issues. Message-Id: <20220801063902.129329-41-xuanzhuo@linux.alibaba.com> Acked-by: Jason Wang Message-Id: <20220811080258.79398-2-xuanzhuo@linux.alibaba.com> Reported-by: Kangjie Xu Signed-off-by: Xuan Zhuo Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 8cad913926e5..4cf278dd9004 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -284,6 +284,8 @@ struct padded_vnet_hdr { char padding[12]; }; +static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf); + static bool is_xdp_frame(void *ptr) { return (unsigned long)ptr & VIRTIO_XDP_FLAG; @@ -1872,6 +1874,29 @@ static netdev_tx_t start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; } +static int virtnet_rx_resize(struct virtnet_info *vi, + struct receive_queue *rq, u32 ring_num) +{ + bool running = netif_running(vi->dev); + int err, qindex; + + qindex = rq - vi->rq; + + if (running) + napi_disable(&rq->napi); + + err = virtqueue_resize(rq->vq, ring_num, virtnet_rq_free_unused_buf); + if (err) + netdev_err(vi->dev, "resize rx fail: rx queue index: %d err: %d\n", qindex, err); + + if (!try_fill_recv(vi, rq, GFP_KERNEL)) + schedule_delayed_work(&vi->refill, 0); + + if (running) + virtnet_napi_enable(rq->vq, &rq->napi); + return err; +} + /* * Send command via the control virtqueue and check status. Commands * supported by the hypervisor, as indicated by feature bits, should -- cgit From ebcce492636506443e4361db6587e6acd1a624f9 Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:39:01 +0800 Subject: virtio_net: support tx queue resize This patch implements the resize function of the tx queues. Based on this function, it is possible to modify the ring num of the queue. Inludes fixup: virtio_net: fix for stuck when change tx ring size with dev down When dev is set to DOWN state, napi has been disabled, if we modify the ring size at this time, we should not call napi_disable() again, which will cause stuck. And all operations are under the protection of rtnl_lock, so there is no need to consider concurrency issues. Message-Id: <20220801063902.129329-42-xuanzhuo@linux.alibaba.com> Acked-by: Jason Wang Message-Id: <20220811080258.79398-3-xuanzhuo@linux.alibaba.com> Reported-by: Kangjie Xu Signed-off-by: Xuan Zhuo Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 4cf278dd9004..6aa0871b438c 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -135,6 +135,9 @@ struct send_queue { struct virtnet_sq_stats stats; struct napi_struct napi; + + /* Record whether sq is in reset state. */ + bool reset; }; /* Internal representation of a receive virtqueue */ @@ -285,6 +288,7 @@ struct padded_vnet_hdr { }; static void virtnet_rq_free_unused_buf(struct virtqueue *vq, void *buf); +static void virtnet_sq_free_unused_buf(struct virtqueue *vq, void *buf); static bool is_xdp_frame(void *ptr) { @@ -1627,6 +1631,11 @@ static void virtnet_poll_cleantx(struct receive_queue *rq) return; if (__netif_tx_trylock(txq)) { + if (sq->reset) { + __netif_tx_unlock(txq); + return; + } + do { virtqueue_disable_cb(sq->vq); free_old_xmit_skbs(sq, true); @@ -1897,6 +1906,47 @@ static int virtnet_rx_resize(struct virtnet_info *vi, return err; } +static int virtnet_tx_resize(struct virtnet_info *vi, + struct send_queue *sq, u32 ring_num) +{ + bool running = netif_running(vi->dev); + struct netdev_queue *txq; + int err, qindex; + + qindex = sq - vi->sq; + + if (running) + virtnet_napi_tx_disable(&sq->napi); + + txq = netdev_get_tx_queue(vi->dev, qindex); + + /* 1. wait all ximt complete + * 2. fix the race of netif_stop_subqueue() vs netif_start_subqueue() + */ + __netif_tx_lock_bh(txq); + + /* Prevent rx poll from accessing sq. */ + sq->reset = true; + + /* Prevent the upper layer from trying to send packets. */ + netif_stop_subqueue(vi->dev, qindex); + + __netif_tx_unlock_bh(txq); + + err = virtqueue_resize(sq->vq, ring_num, virtnet_sq_free_unused_buf); + if (err) + netdev_err(vi->dev, "resize tx fail: tx queue index: %d err: %d\n", qindex, err); + + __netif_tx_lock_bh(txq); + sq->reset = false; + netif_tx_wake_queue(txq); + __netif_tx_unlock_bh(txq); + + if (running) + virtnet_napi_tx_enable(vi, sq->vq, &sq->napi); + return err; +} + /* * Send command via the control virtqueue and check status. Commands * supported by the hypervisor, as indicated by feature bits, should -- cgit From a335b33f4f354c5b77a79d505708f6d2f486ceab Mon Sep 17 00:00:00 2001 From: Xuan Zhuo Date: Mon, 1 Aug 2022 14:39:02 +0800 Subject: virtio_net: support set_ringparam Support set_ringparam based on virtio queue reset. Users can use ethtool -G eth0 to modify the ring size of virtio-net. Signed-off-by: Xuan Zhuo Acked-by: Jason Wang Message-Id: <20220801063902.129329-43-xuanzhuo@linux.alibaba.com> Signed-off-by: Michael S. Tsirkin --- drivers/net/virtio_net.c | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 6aa0871b438c..16783ed782c5 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2363,6 +2363,53 @@ static void virtnet_get_ringparam(struct net_device *dev, ring->tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); } +static int virtnet_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct virtnet_info *vi = netdev_priv(dev); + u32 rx_pending, tx_pending; + struct receive_queue *rq; + struct send_queue *sq; + int i, err; + + if (ring->rx_mini_pending || ring->rx_jumbo_pending) + return -EINVAL; + + rx_pending = virtqueue_get_vring_size(vi->rq[0].vq); + tx_pending = virtqueue_get_vring_size(vi->sq[0].vq); + + if (ring->rx_pending == rx_pending && + ring->tx_pending == tx_pending) + return 0; + + if (ring->rx_pending > vi->rq[0].vq->num_max) + return -EINVAL; + + if (ring->tx_pending > vi->sq[0].vq->num_max) + return -EINVAL; + + for (i = 0; i < vi->max_queue_pairs; i++) { + rq = vi->rq + i; + sq = vi->sq + i; + + if (ring->tx_pending != tx_pending) { + err = virtnet_tx_resize(vi, sq, ring->tx_pending); + if (err) + return err; + } + + if (ring->rx_pending != rx_pending) { + err = virtnet_rx_resize(vi, rq, ring->rx_pending); + if (err) + return err; + } + } + + return 0; +} + static bool virtnet_commit_rss_command(struct virtnet_info *vi) { struct net_device *dev = vi->dev; @@ -2850,6 +2897,7 @@ static const struct ethtool_ops virtnet_ethtool_ops = { .get_drvinfo = virtnet_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = virtnet_get_ringparam, + .set_ringparam = virtnet_set_ringparam, .get_strings = virtnet_get_strings, .get_sset_count = virtnet_get_sset_count, .get_ethtool_stats = virtnet_get_ethtool_stats, -- cgit From 42a84c09eff355d895723866b1a5ff48f093112a Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 30 Jun 2022 17:32:19 +0200 Subject: vdpa_sim_blk: use dev_dbg() to print errors Use dev_dbg() instead of dev_err()/dev_warn() to avoid flooding the host with prints, when the guest driver is misbehaving. In this way, prints can be dynamically enabled when the vDPA block simulator is used to validate a driver. Suggested-by: Jason Wang Acked-by: Jason Wang Signed-off-by: Stefano Garzarella Message-Id: <20220630153221.83371-2-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index 42d401d43911..a83a5c76f620 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -76,13 +76,13 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, return false; if (vq->out_iov.used < 1 || vq->in_iov.used < 1) { - dev_err(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n", + dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n", vq->out_iov.used, vq->in_iov.used); return false; } if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) { - dev_err(&vdpasim->vdpa.dev, "request in header too short\n"); + dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n"); return false; } @@ -96,7 +96,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &hdr, sizeof(hdr)); if (bytes != sizeof(hdr)) { - dev_err(&vdpasim->vdpa.dev, "request out header too short\n"); + dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n"); return false; } @@ -110,7 +110,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, switch (type) { case VIRTIO_BLK_T_IN: if (!vdpasim_blk_check_range(sector, to_push)) { - dev_err(&vdpasim->vdpa.dev, + dev_dbg(&vdpasim->vdpa.dev, "reading over the capacity - offset: 0x%llx len: 0x%zx\n", offset, to_push); status = VIRTIO_BLK_S_IOERR; @@ -121,7 +121,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, vdpasim->buffer + offset, to_push); if (bytes < 0) { - dev_err(&vdpasim->vdpa.dev, + dev_dbg(&vdpasim->vdpa.dev, "vringh_iov_push_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", bytes, offset, to_push); status = VIRTIO_BLK_S_IOERR; @@ -133,7 +133,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, case VIRTIO_BLK_T_OUT: if (!vdpasim_blk_check_range(sector, to_pull)) { - dev_err(&vdpasim->vdpa.dev, + dev_dbg(&vdpasim->vdpa.dev, "writing over the capacity - offset: 0x%llx len: 0x%zx\n", offset, to_pull); status = VIRTIO_BLK_S_IOERR; @@ -144,7 +144,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, vdpasim->buffer + offset, to_pull); if (bytes < 0) { - dev_err(&vdpasim->vdpa.dev, + dev_dbg(&vdpasim->vdpa.dev, "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", bytes, offset, to_pull); status = VIRTIO_BLK_S_IOERR; @@ -157,7 +157,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, vdpasim_blk_id, VIRTIO_BLK_ID_BYTES); if (bytes < 0) { - dev_err(&vdpasim->vdpa.dev, + dev_dbg(&vdpasim->vdpa.dev, "vringh_iov_push_iotlb() error: %zd\n", bytes); status = VIRTIO_BLK_S_IOERR; break; @@ -167,8 +167,8 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, break; default: - dev_warn(&vdpasim->vdpa.dev, - "Unsupported request type %d\n", type); + dev_dbg(&vdpasim->vdpa.dev, + "Unsupported request type %d\n", type); status = VIRTIO_BLK_S_IOERR; break; } -- cgit From 9c4df0900f376f28e8e5cca640a10a0085f16b66 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 30 Jun 2022 17:32:20 +0200 Subject: vdpa_sim_blk: limit the number of request handled per batch MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limit the number of requests (4 per queue as for vdpa_sim_net) handled in a batch to prevent the worker from using the CPU for too long. Suggested-by: Eugenio Pérez Signed-off-by: Stefano Garzarella Message-Id: <20220630153221.83371-3-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index a83a5c76f620..b2d75efec63a 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -197,6 +197,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, static void vdpasim_blk_work(struct work_struct *work) { struct vdpasim *vdpasim = container_of(work, struct vdpasim, work); + bool reschedule = false; int i; spin_lock(&vdpasim->lock); @@ -206,6 +207,7 @@ static void vdpasim_blk_work(struct work_struct *work) for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) { struct vdpasim_virtqueue *vq = &vdpasim->vqs[i]; + int reqs = 0; if (!vq->ready) continue; @@ -218,10 +220,18 @@ static void vdpasim_blk_work(struct work_struct *work) if (vringh_need_notify_iotlb(&vq->vring) > 0) vringh_notify(&vq->vring); local_bh_enable(); + + if (++reqs > 4) { + reschedule = true; + break; + } } } out: spin_unlock(&vdpasim->lock); + + if (reschedule) + schedule_work(&vdpasim->work); } static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config) -- cgit From 8472019eec2ccff60144d8585367b2856a59416e Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 30 Jun 2022 17:32:21 +0200 Subject: vdpa_sim_blk: call vringh_complete_iotlb() also in the error path Call vringh_complete_iotlb() even when we encounter a serious error that prevents us from writing the status in the "in" header (e.g. the header length is incorrect, etc.). The guest is misbehaving, so maybe the ring is in a bad state, but let's avoid making things worse. Acked-by: Jason Wang Signed-off-by: Stefano Garzarella Message-Id: <20220630153221.83371-4-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index b2d75efec63a..43422b3854d2 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -63,6 +63,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, { size_t pushed = 0, to_pull, to_push; struct virtio_blk_outhdr hdr; + bool handled = false; ssize_t bytes; loff_t offset; u64 sector; @@ -78,12 +79,12 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, if (vq->out_iov.used < 1 || vq->in_iov.used < 1) { dev_dbg(&vdpasim->vdpa.dev, "missing headers - out_iov: %u in_iov %u\n", vq->out_iov.used, vq->in_iov.used); - return false; + goto err; } if (vq->in_iov.iov[vq->in_iov.used - 1].iov_len < 1) { dev_dbg(&vdpasim->vdpa.dev, "request in header too short\n"); - return false; + goto err; } /* The last byte is the status and we checked if the last iov has @@ -97,7 +98,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, sizeof(hdr)); if (bytes != sizeof(hdr)) { dev_dbg(&vdpasim->vdpa.dev, "request out header too short\n"); - return false; + goto err; } to_pull -= bytes; @@ -182,16 +183,19 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, /* Last byte is the status */ bytes = vringh_iov_push_iotlb(&vq->vring, &vq->in_iov, &status, 1); if (bytes != 1) - return false; + goto err; pushed += bytes; /* Make sure data is wrote before advancing index */ smp_wmb(); + handled = true; + +err: vringh_complete_iotlb(&vq->vring, vq->head, pushed); - return true; + return handled; } static void vdpasim_blk_work(struct work_struct *work) -- cgit From 19cd4a5471b8eaa4bd161b0fdb4567f2fc88d809 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Tue, 21 Jun 2022 17:13:23 +0200 Subject: vdpa_sim_blk: set number of address spaces and virtqueue groups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit bda324fd037a ("vdpasim: control virtqueue support") added two new fields (nas, ngroups) to vdpasim_dev_attr, but we forgot to initialize them for vdpa_sim_blk. When creating a new vdpa_sim_blk device this causes the kernel to panic in this way:    $ vdpa dev add mgmtdev vdpasim_blk name blk0    BUG: kernel NULL pointer dereference, address: 0000000000000030    ...    RIP: 0010:vhost_iotlb_add_range_ctx+0x41/0x220 [vhost_iotlb]    ...    Call Trace:         vhost_iotlb_add_range+0x11/0x800 [vhost_iotlb]     vdpasim_map_range+0x91/0xd0 [vdpa_sim]     vdpasim_alloc_coherent+0x56/0x90 [vdpa_sim]     ... This happens because vdpasim->iommu[0] is not initialized when dev_attr.nas is 0. Let's fix this issue by initializing both (nas, ngroups) to 1 for vdpa_sim_blk. Fixes: bda324fd037a ("vdpasim: control virtqueue support") Cc: gautam.dawar@xilinx.com Signed-off-by: Stefano Garzarella Message-Id: <20220621151323.190431-1-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin Acked-by: Eugenio Pérez --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index 43422b3854d2..dde965a826bb 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -34,7 +34,11 @@ #define VDPASIM_BLK_CAPACITY 0x40000 #define VDPASIM_BLK_SIZE_MAX 0x1000 #define VDPASIM_BLK_SEG_MAX 32 + +/* 1 virtqueue, 1 address space, 1 virtqueue group */ #define VDPASIM_BLK_VQ_NUM 1 +#define VDPASIM_BLK_AS_NUM 1 +#define VDPASIM_BLK_GROUP_NUM 1 static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim"; @@ -274,6 +278,8 @@ static int vdpasim_blk_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, dev_attr.id = VIRTIO_ID_BLOCK; dev_attr.supported_features = VDPASIM_BLK_FEATURES; dev_attr.nvqs = VDPASIM_BLK_VQ_NUM; + dev_attr.ngroups = VDPASIM_BLK_GROUP_NUM; + dev_attr.nas = VDPASIM_BLK_AS_NUM; dev_attr.config_size = sizeof(struct virtio_blk_config); dev_attr.get_config = vdpasim_blk_get_config; dev_attr.work_fn = vdpasim_blk_work; -- cgit From 67f8f10c0bd78c4a0f0e983e050ab90da015323b Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Tue, 21 Jun 2022 17:12:08 +0200 Subject: vdpa_sim: use max_iotlb_entries as a limit in vhost_iotlb_init MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Commit bda324fd037a ("vdpasim: control virtqueue support") changed the allocation of iotlbs calling vhost_iotlb_init() for each address space, instead of vhost_iotlb_alloc(). With this change we forgot to use the limit we had introduced with the `max_iotlb_entries` module parameter. Fixes: bda324fd037a ("vdpasim: control virtqueue support") Cc: gautam.dawar@xilinx.com Signed-off-by: Stefano Garzarella Message-Id: <20220621151208.189959-1-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin Acked-by: Eugenio Pérez --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 0f2865899647..3e81532c01cb 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -33,7 +33,7 @@ MODULE_PARM_DESC(batch_mapping, "Batched mapping 1 -Enable; 0 - Disable"); static int max_iotlb_entries = 2048; module_param(max_iotlb_entries, int, 0444); MODULE_PARM_DESC(max_iotlb_entries, - "Maximum number of iotlb entries. 0 means unlimited. (default: 2048)"); + "Maximum number of iotlb entries for each address space. 0 means unlimited. (default: 2048)"); #define VDPASIM_QUEUE_ALIGN PAGE_SIZE #define VDPASIM_QUEUE_MAX 256 @@ -291,7 +291,7 @@ struct vdpasim *vdpasim_create(struct vdpasim_dev_attr *dev_attr) goto err_iommu; for (i = 0; i < vdpasim->dev_attr.nas; i++) - vhost_iotlb_init(&vdpasim->iommu[i], 0, 0); + vhost_iotlb_init(&vdpasim->iommu[i], max_iotlb_entries, 0); vdpasim->buffer = kvmalloc(dev_attr->buffer_size, GFP_KERNEL); if (!vdpasim->buffer) -- cgit From 71aa95a69c765bd0ec1c6d9d6263e6fefa1f5072 Mon Sep 17 00:00:00 2001 From: Xu Qiang Date: Mon, 4 Jul 2022 02:14:05 +0000 Subject: vdpa/mlx5: Use eth_broadcast_addr() to assign broadcast address Using eth_broadcast_addr() to assign broadcast address instead of memset(). Reported-by: Hulk Robot Signed-off-by: Xu Qiang Message-Id: <20220704021405.64545-1-xuqiang36@huawei.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/mlx5/net/mlx5_vnet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index e85c1d71f4ed..99bbbf38c8b1 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1440,7 +1440,7 @@ static int mlx5_vdpa_add_mac_vlan_rules(struct mlx5_vdpa_net *ndev, u8 *mac, headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers); dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, outer_headers.dmac_47_16); dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, outer_headers.dmac_47_16); - memset(dmac_c, 0xff, ETH_ALEN); + eth_broadcast_addr(dmac_c); ether_addr_copy(dmac_v, mac); MLX5_SET(fte_match_set_lyr_2_4, headers_c, cvlan_tag, 1); if (tagged) { -- cgit From 6e33fa094f1f5a96140cc2439b17f4f3ef9e4a9d Mon Sep 17 00:00:00 2001 From: Zhang Jiaming Date: Mon, 4 Jul 2022 10:41:04 +0800 Subject: vdpa: ifcvf: Fix spelling mistake in comments There is a typo(does't) in comments. It maybe 'doesn't' instead of 'does't'. Signed-off-by: Zhang Jiaming Message-Id: <20220704024104.15535-1-jiaming@nfschina.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/ifcvf/ifcvf_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index 0a5670729412..e2e201885713 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -685,7 +685,7 @@ static struct vdpa_notification_area ifcvf_get_vq_notification(struct vdpa_devic } /* - * IFCVF currently does't have on-chip IOMMU, so not + * IFCVF currently doesn't have on-chip IOMMU, so not * implemented set_map()/dma_map()/dma_unmap() */ static const struct vdpa_config_ops ifc_vdpa_ops = { -- cgit From 994cea53183c211f218d5dbe14ab634f5762f068 Mon Sep 17 00:00:00 2001 From: Colin Ian King Date: Mon, 4 Jul 2022 20:04:56 +0100 Subject: vDPA/ifcvf: remove duplicated assignment to pointer cfg The assignment to pointer cfg is duplicated, the second assignment is redundant and can be removed. Signed-off-by: Colin Ian King Message-Id: <20220704190456.593464-1-colin.i.king@gmail.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/ifcvf/ifcvf_base.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 48c4dadb0c7c..47b94091733c 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -29,7 +29,6 @@ u16 ifcvf_set_config_vector(struct ifcvf_hw *hw, int vector) { struct virtio_pci_common_cfg __iomem *cfg = hw->common_cfg; - cfg = hw->common_cfg; vp_iowrite16(vector, &cfg->msix_config); return vp_ioread16(&cfg->msix_config); -- cgit From 95bf9798779af166b1d6d184d794834214a1a261 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Wed, 6 Jul 2022 23:17:51 -0400 Subject: virtio: Check dev_set_name() return value It's possible that dev_set_name() returns -ENOMEM, catch and handle this. Signed-off-by: Bo Liu Message-Id: <20220707031751.4802-1-liubo03@inspur.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/virtio/virtio.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index 7deeed30d1f3..ddd4466da83b 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -428,7 +428,9 @@ int register_virtio_device(struct virtio_device *dev) goto out; dev->index = err; - dev_set_name(&dev->dev, "virtio%u", dev->index); + err = dev_set_name(&dev->dev, "virtio%u", dev->index); + if (err) + goto out_ida_remove; err = virtio_device_of_init(dev); if (err) -- cgit From 699b045a8e43bd1063db4795be685bfd659649dc Mon Sep 17 00:00:00 2001 From: Alvaro Karsz Date: Mon, 18 Jul 2022 12:11:02 +0300 Subject: net: virtio_net: notifications coalescing support New VirtIO network feature: VIRTIO_NET_F_NOTF_COAL. Control a Virtio network device notifications coalescing parameters using the control virtqueue. A device that supports this fetature can receive VIRTIO_NET_CTRL_NOTF_COAL control commands. - VIRTIO_NET_CTRL_NOTF_COAL_TX_SET: Ask the network device to change the following parameters: - tx_usecs: Maximum number of usecs to delay a TX notification. - tx_max_packets: Maximum number of packets to send before a TX notification. - VIRTIO_NET_CTRL_NOTF_COAL_RX_SET: Ask the network device to change the following parameters: - rx_usecs: Maximum number of usecs to delay a RX notification. - rx_max_packets: Maximum number of packets to receive before a RX notification. VirtIO spec. patch: https://lists.oasis-open.org/archives/virtio-comment/202206/msg00100.html Signed-off-by: Alvaro Karsz Message-Id: <20220718091102.498774-1-alvaro.karsz@solid-run.com> Signed-off-by: Michael S. Tsirkin Reviewed-by: Jakub Kicinski Acked-by: Jason Wang --- drivers/net/virtio_net.c | 111 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 96 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 16783ed782c5..d9c434b00e9b 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -270,6 +270,12 @@ struct virtnet_info { u8 duplex; u32 speed; + /* Interrupt coalescing settings */ + u32 tx_usecs; + u32 rx_usecs; + u32 tx_max_packets; + u32 rx_max_packets; + unsigned long guest_offloads; unsigned long guest_offloads_capable; @@ -2737,27 +2743,89 @@ static int virtnet_get_link_ksettings(struct net_device *dev, return 0; } +static int virtnet_send_notf_coal_cmds(struct virtnet_info *vi, + struct ethtool_coalesce *ec) +{ + struct scatterlist sgs_tx, sgs_rx; + struct virtio_net_ctrl_coal_tx coal_tx; + struct virtio_net_ctrl_coal_rx coal_rx; + + coal_tx.tx_usecs = cpu_to_le32(ec->tx_coalesce_usecs); + coal_tx.tx_max_packets = cpu_to_le32(ec->tx_max_coalesced_frames); + sg_init_one(&sgs_tx, &coal_tx, sizeof(coal_tx)); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, + VIRTIO_NET_CTRL_NOTF_COAL_TX_SET, + &sgs_tx)) + return -EINVAL; + + /* Save parameters */ + vi->tx_usecs = ec->tx_coalesce_usecs; + vi->tx_max_packets = ec->tx_max_coalesced_frames; + + coal_rx.rx_usecs = cpu_to_le32(ec->rx_coalesce_usecs); + coal_rx.rx_max_packets = cpu_to_le32(ec->rx_max_coalesced_frames); + sg_init_one(&sgs_rx, &coal_rx, sizeof(coal_rx)); + + if (!virtnet_send_command(vi, VIRTIO_NET_CTRL_NOTF_COAL, + VIRTIO_NET_CTRL_NOTF_COAL_RX_SET, + &sgs_rx)) + return -EINVAL; + + /* Save parameters */ + vi->rx_usecs = ec->rx_coalesce_usecs; + vi->rx_max_packets = ec->rx_max_coalesced_frames; + + return 0; +} + +static int virtnet_coal_params_supported(struct ethtool_coalesce *ec) +{ + /* usecs coalescing is supported only if VIRTIO_NET_F_NOTF_COAL + * feature is negotiated. + */ + if (ec->rx_coalesce_usecs || ec->tx_coalesce_usecs) + return -EOPNOTSUPP; + + if (ec->tx_max_coalesced_frames > 1 || + ec->rx_max_coalesced_frames != 1) + return -EINVAL; + + return 0; +} + static int virtnet_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); - int i, napi_weight; - - if (ec->tx_max_coalesced_frames > 1 || - ec->rx_max_coalesced_frames != 1) - return -EINVAL; + int ret, i, napi_weight; + bool update_napi = false; + /* Can't change NAPI weight if the link is up */ napi_weight = ec->tx_max_coalesced_frames ? NAPI_POLL_WEIGHT : 0; if (napi_weight ^ vi->sq[0].napi.weight) { if (dev->flags & IFF_UP) return -EBUSY; + else + update_napi = true; + } + + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) + ret = virtnet_send_notf_coal_cmds(vi, ec); + else + ret = virtnet_coal_params_supported(ec); + + if (ret) + return ret; + + if (update_napi) { for (i = 0; i < vi->max_queue_pairs; i++) vi->sq[i].napi.weight = napi_weight; } - return 0; + return ret; } static int virtnet_get_coalesce(struct net_device *dev, @@ -2765,16 +2833,19 @@ static int virtnet_get_coalesce(struct net_device *dev, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { - struct ethtool_coalesce ec_default = { - .cmd = ETHTOOL_GCOALESCE, - .rx_max_coalesced_frames = 1, - }; struct virtnet_info *vi = netdev_priv(dev); - memcpy(ec, &ec_default, sizeof(ec_default)); + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { + ec->rx_coalesce_usecs = vi->rx_usecs; + ec->tx_coalesce_usecs = vi->tx_usecs; + ec->tx_max_coalesced_frames = vi->tx_max_packets; + ec->rx_max_coalesced_frames = vi->rx_max_packets; + } else { + ec->rx_max_coalesced_frames = 1; - if (vi->sq[0].napi.weight) - ec->tx_max_coalesced_frames = 1; + if (vi->sq[0].napi.weight) + ec->tx_max_coalesced_frames = 1; + } return 0; } @@ -2893,7 +2964,8 @@ static int virtnet_set_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info) } static const struct ethtool_ops virtnet_ethtool_ops = { - .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES, + .supported_coalesce_params = ETHTOOL_COALESCE_MAX_FRAMES | + ETHTOOL_COALESCE_USECS, .get_drvinfo = virtnet_get_drvinfo, .get_link = ethtool_op_get_link, .get_ringparam = virtnet_get_ringparam, @@ -3606,6 +3678,8 @@ static bool virtnet_validate_features(struct virtio_device *vdev) VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_RSS, "VIRTIO_NET_F_CTRL_VQ") || VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_HASH_REPORT, + "VIRTIO_NET_F_CTRL_VQ") || + VIRTNET_FAIL_ON(vdev, VIRTIO_NET_F_NOTF_COAL, "VIRTIO_NET_F_CTRL_VQ"))) { return false; } @@ -3742,6 +3816,13 @@ static int virtnet_probe(struct virtio_device *vdev) if (virtio_has_feature(vdev, VIRTIO_NET_F_MRG_RXBUF)) vi->mergeable_rx_bufs = true; + if (virtio_has_feature(vi->vdev, VIRTIO_NET_F_NOTF_COAL)) { + vi->rx_usecs = 0; + vi->tx_usecs = 0; + vi->tx_max_packets = 0; + vi->rx_max_packets = 0; + } + if (virtio_has_feature(vdev, VIRTIO_NET_F_HASH_REPORT)) vi->has_rss_hash_report = true; @@ -3977,7 +4058,7 @@ static struct virtio_device_id id_table[] = { VIRTIO_NET_F_CTRL_MAC_ADDR, \ VIRTIO_NET_F_MTU, VIRTIO_NET_F_CTRL_GUEST_OFFLOADS, \ VIRTIO_NET_F_SPEED_DUPLEX, VIRTIO_NET_F_STANDBY, \ - VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT + VIRTIO_NET_F_RSS, VIRTIO_NET_F_HASH_REPORT, VIRTIO_NET_F_NOTF_COAL static unsigned int features[] = { VIRTNET_FEATURES, -- cgit From c32ee693125aa19a021dad32abbd49f5c6f70031 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:19 +0800 Subject: vduse: Remove unnecessary spin lock protection Now we use domain->iotlb_lock to protect two different variables: domain->bounce_maps->bounce_page and domain->iotlb. But for domain->bounce_maps->bounce_page, we actually don't need any synchronization between vduse_domain_get_bounce_page() and vduse_domain_free_bounce_pages() since vduse_domain_get_bounce_page() will only be called in page fault handler and vduse_domain_free_bounce_pages() will be called during file release. So let's remove the unnecessary spin lock protection in vduse_domain_get_bounce_page(). Then the usage of domain->iotlb_lock could be more clear: the lock will be only used to protect the domain->iotlb. Signed-off-by: Xie Yongji Acked-by: Jason Wang Message-Id: <20220803045523.23851-2-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/iova_domain.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 6daa3978d290..bca1f0b8850c 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -211,17 +211,14 @@ static struct page * vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova) { struct vduse_bounce_map *map; - struct page *page = NULL; + struct page *page; - spin_lock(&domain->iotlb_lock); map = &domain->bounce_maps[iova >> PAGE_SHIFT]; if (!map->bounce_page) - goto out; + return NULL; page = map->bounce_page; get_page(page); -out: - spin_unlock(&domain->iotlb_lock); return page; } -- cgit From 82eb46f94ae5faa87ace248d950292c87384a097 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:20 +0800 Subject: vduse: Use memcpy_{to,from}_page() in do_bounce() kmap_atomic() is being deprecated in favor of kmap_local_page(). The use of kmap_atomic() in do_bounce() is all thread local therefore kmap_local_page() is a sufficient replacement. Convert to kmap_local_page() but, instead of open coding it, use the helpers memcpy_to_page() and memcpy_from_page(). Signed-off-by: Xie Yongji Acked-by: Jason Wang Reviewed-by: Muchun Song Message-Id: <20220803045523.23851-3-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/iova_domain.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index bca1f0b8850c..50d7c08d5450 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -138,18 +138,17 @@ static void do_bounce(phys_addr_t orig, void *addr, size_t size, { unsigned long pfn = PFN_DOWN(orig); unsigned int offset = offset_in_page(orig); - char *buffer; + struct page *page; unsigned int sz = 0; while (size) { sz = min_t(size_t, PAGE_SIZE - offset, size); - buffer = kmap_atomic(pfn_to_page(pfn)); + page = pfn_to_page(pfn); if (dir == DMA_TO_DEVICE) - memcpy(addr, buffer + offset, sz); + memcpy_from_page(addr, page, offset, sz); else - memcpy(buffer + offset, addr, sz); - kunmap_atomic(buffer); + memcpy_to_page(page, offset, addr, sz); size -= sz; pfn++; -- cgit From 6c77ed22880d3e8ae91b1d2cce26dbd6c69f38b7 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:21 +0800 Subject: vduse: Support using userspace pages as bounce buffer Introduce two APIs: vduse_domain_add_user_bounce_pages() and vduse_domain_remove_user_bounce_pages() to support adding and removing userspace pages for bounce buffers. During adding and removing, the DMA data would be copied from the kernel bounce pages to the userspace bounce pages and back. Signed-off-by: Xie Yongji Acked-by: Jason Wang Message-Id: <20220803045523.23851-4-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/iova_domain.c | 96 +++++++++++++++++++++++++++++++++--- drivers/vdpa/vdpa_user/iova_domain.h | 8 +++ 2 files changed, 96 insertions(+), 8 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_user/iova_domain.c b/drivers/vdpa/vdpa_user/iova_domain.c index 50d7c08d5450..e682bc7ee6c9 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.c +++ b/drivers/vdpa/vdpa_user/iova_domain.c @@ -178,8 +178,9 @@ static void vduse_domain_bounce(struct vduse_iova_domain *domain, map->orig_phys == INVALID_PHYS_ADDR)) return; - addr = page_address(map->bounce_page) + offset; - do_bounce(map->orig_phys + offset, addr, sz, dir); + addr = kmap_local_page(map->bounce_page); + do_bounce(map->orig_phys + offset, addr + offset, sz, dir); + kunmap_local(addr); size -= sz; iova += sz; } @@ -210,20 +211,23 @@ static struct page * vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova) { struct vduse_bounce_map *map; - struct page *page; + struct page *page = NULL; + read_lock(&domain->bounce_lock); map = &domain->bounce_maps[iova >> PAGE_SHIFT]; - if (!map->bounce_page) - return NULL; + if (domain->user_bounce_pages || !map->bounce_page) + goto out; page = map->bounce_page; get_page(page); +out: + read_unlock(&domain->bounce_lock); return page; } static void -vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain) +vduse_domain_free_kernel_bounce_pages(struct vduse_iova_domain *domain) { struct vduse_bounce_map *map; unsigned long pfn, bounce_pfns; @@ -243,6 +247,73 @@ vduse_domain_free_bounce_pages(struct vduse_iova_domain *domain) } } +int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain, + struct page **pages, int count) +{ + struct vduse_bounce_map *map; + int i, ret; + + /* Now we don't support partial mapping */ + if (count != (domain->bounce_size >> PAGE_SHIFT)) + return -EINVAL; + + write_lock(&domain->bounce_lock); + ret = -EEXIST; + if (domain->user_bounce_pages) + goto out; + + for (i = 0; i < count; i++) { + map = &domain->bounce_maps[i]; + if (map->bounce_page) { + /* Copy kernel page to user page if it's in use */ + if (map->orig_phys != INVALID_PHYS_ADDR) + memcpy_to_page(pages[i], 0, + page_address(map->bounce_page), + PAGE_SIZE); + __free_page(map->bounce_page); + } + map->bounce_page = pages[i]; + get_page(pages[i]); + } + domain->user_bounce_pages = true; + ret = 0; +out: + write_unlock(&domain->bounce_lock); + + return ret; +} + +void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain) +{ + struct vduse_bounce_map *map; + unsigned long i, count; + + write_lock(&domain->bounce_lock); + if (!domain->user_bounce_pages) + goto out; + + count = domain->bounce_size >> PAGE_SHIFT; + for (i = 0; i < count; i++) { + struct page *page = NULL; + + map = &domain->bounce_maps[i]; + if (WARN_ON(!map->bounce_page)) + continue; + + /* Copy user page to kernel page if it's in use */ + if (map->orig_phys != INVALID_PHYS_ADDR) { + page = alloc_page(GFP_ATOMIC | __GFP_NOFAIL); + memcpy_from_page(page_address(page), + map->bounce_page, 0, PAGE_SIZE); + } + put_page(map->bounce_page); + map->bounce_page = page; + } + domain->user_bounce_pages = false; +out: + write_unlock(&domain->bounce_lock); +} + void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain) { if (!domain->bounce_map) @@ -318,13 +389,18 @@ dma_addr_t vduse_domain_map_page(struct vduse_iova_domain *domain, if (vduse_domain_init_bounce_map(domain)) goto err; + read_lock(&domain->bounce_lock); if (vduse_domain_map_bounce_page(domain, (u64)iova, (u64)size, pa)) - goto err; + goto err_unlock; if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL) vduse_domain_bounce(domain, iova, size, DMA_TO_DEVICE); + read_unlock(&domain->bounce_lock); + return iova; +err_unlock: + read_unlock(&domain->bounce_lock); err: vduse_domain_free_iova(iovad, iova, size); return DMA_MAPPING_ERROR; @@ -336,10 +412,12 @@ void vduse_domain_unmap_page(struct vduse_iova_domain *domain, { struct iova_domain *iovad = &domain->stream_iovad; + read_lock(&domain->bounce_lock); if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL) vduse_domain_bounce(domain, dma_addr, size, DMA_FROM_DEVICE); vduse_domain_unmap_bounce_page(domain, (u64)dma_addr, (u64)size); + read_unlock(&domain->bounce_lock); vduse_domain_free_iova(iovad, dma_addr, size); } @@ -447,7 +525,8 @@ static int vduse_domain_release(struct inode *inode, struct file *file) spin_lock(&domain->iotlb_lock); vduse_iotlb_del_range(domain, 0, ULLONG_MAX); - vduse_domain_free_bounce_pages(domain); + vduse_domain_remove_user_bounce_pages(domain); + vduse_domain_free_kernel_bounce_pages(domain); spin_unlock(&domain->iotlb_lock); put_iova_domain(&domain->stream_iovad); put_iova_domain(&domain->consistent_iovad); @@ -507,6 +586,7 @@ vduse_domain_create(unsigned long iova_limit, size_t bounce_size) goto err_file; domain->file = file; + rwlock_init(&domain->bounce_lock); spin_lock_init(&domain->iotlb_lock); init_iova_domain(&domain->stream_iovad, PAGE_SIZE, IOVA_START_PFN); diff --git a/drivers/vdpa/vdpa_user/iova_domain.h b/drivers/vdpa/vdpa_user/iova_domain.h index 2722d9b8e21a..4e0e50e7ac15 100644 --- a/drivers/vdpa/vdpa_user/iova_domain.h +++ b/drivers/vdpa/vdpa_user/iova_domain.h @@ -14,6 +14,7 @@ #include #include #include +#include #define IOVA_START_PFN 1 @@ -34,6 +35,8 @@ struct vduse_iova_domain { struct vhost_iotlb *iotlb; spinlock_t iotlb_lock; struct file *file; + bool user_bounce_pages; + rwlock_t bounce_lock; }; int vduse_domain_set_map(struct vduse_iova_domain *domain, @@ -61,6 +64,11 @@ void vduse_domain_free_coherent(struct vduse_iova_domain *domain, size_t size, void vduse_domain_reset_bounce_map(struct vduse_iova_domain *domain); +int vduse_domain_add_user_bounce_pages(struct vduse_iova_domain *domain, + struct page **pages, int count); + +void vduse_domain_remove_user_bounce_pages(struct vduse_iova_domain *domain); + void vduse_domain_destroy(struct vduse_iova_domain *domain); struct vduse_iova_domain *vduse_domain_create(unsigned long iova_limit, -- cgit From 79a463be9e0051997508d52cf411ed5e91d657f6 Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:22 +0800 Subject: vduse: Support registering userspace memory for IOVA regions Introduce two ioctls: VDUSE_IOTLB_REG_UMEM and VDUSE_IOTLB_DEREG_UMEM to support registering and de-registering userspace memory for IOVA regions. Now it only supports registering userspace memory for bounce buffer region in virtio-vdpa case. Signed-off-by: Xie Yongji Acked-by: Jason Wang Message-Id: <20220803045523.23851-5-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_user/vduse_dev.c | 141 +++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index 3bc27de58f46..eedff0a3885a 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -21,6 +21,8 @@ #include #include #include +#include +#include #include #include #include @@ -64,6 +66,13 @@ struct vduse_vdpa { struct vduse_dev *dev; }; +struct vduse_umem { + unsigned long iova; + unsigned long npages; + struct page **pages; + struct mm_struct *mm; +}; + struct vduse_dev { struct vduse_vdpa *vdev; struct device *dev; @@ -95,6 +104,8 @@ struct vduse_dev { u8 status; u32 vq_num; u32 vq_align; + struct vduse_umem *umem; + struct mutex mem_lock; }; struct vduse_dev_msg { @@ -917,6 +928,102 @@ unlock: return ret; } +static int vduse_dev_dereg_umem(struct vduse_dev *dev, + u64 iova, u64 size) +{ + int ret; + + mutex_lock(&dev->mem_lock); + ret = -ENOENT; + if (!dev->umem) + goto unlock; + + ret = -EINVAL; + if (dev->umem->iova != iova || size != dev->domain->bounce_size) + goto unlock; + + vduse_domain_remove_user_bounce_pages(dev->domain); + unpin_user_pages_dirty_lock(dev->umem->pages, + dev->umem->npages, true); + atomic64_sub(dev->umem->npages, &dev->umem->mm->pinned_vm); + mmdrop(dev->umem->mm); + vfree(dev->umem->pages); + kfree(dev->umem); + dev->umem = NULL; + ret = 0; +unlock: + mutex_unlock(&dev->mem_lock); + return ret; +} + +static int vduse_dev_reg_umem(struct vduse_dev *dev, + u64 iova, u64 uaddr, u64 size) +{ + struct page **page_list = NULL; + struct vduse_umem *umem = NULL; + long pinned = 0; + unsigned long npages, lock_limit; + int ret; + + if (!dev->domain->bounce_map || + size != dev->domain->bounce_size || + iova != 0 || uaddr & ~PAGE_MASK) + return -EINVAL; + + mutex_lock(&dev->mem_lock); + ret = -EEXIST; + if (dev->umem) + goto unlock; + + ret = -ENOMEM; + npages = size >> PAGE_SHIFT; + page_list = __vmalloc(array_size(npages, sizeof(struct page *)), + GFP_KERNEL_ACCOUNT); + umem = kzalloc(sizeof(*umem), GFP_KERNEL); + if (!page_list || !umem) + goto unlock; + + mmap_read_lock(current->mm); + + lock_limit = PFN_DOWN(rlimit(RLIMIT_MEMLOCK)); + if (npages + atomic64_read(¤t->mm->pinned_vm) > lock_limit) + goto out; + + pinned = pin_user_pages(uaddr, npages, FOLL_LONGTERM | FOLL_WRITE, + page_list, NULL); + if (pinned != npages) { + ret = pinned < 0 ? pinned : -ENOMEM; + goto out; + } + + ret = vduse_domain_add_user_bounce_pages(dev->domain, + page_list, pinned); + if (ret) + goto out; + + atomic64_add(npages, ¤t->mm->pinned_vm); + + umem->pages = page_list; + umem->npages = pinned; + umem->iova = iova; + umem->mm = current->mm; + mmgrab(current->mm); + + dev->umem = umem; +out: + if (ret && pinned > 0) + unpin_user_pages(page_list, pinned); + + mmap_read_unlock(current->mm); +unlock: + if (ret) { + vfree(page_list); + kfree(umem); + } + mutex_unlock(&dev->mem_lock); + return ret; +} + static long vduse_dev_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -1089,6 +1196,38 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject); break; } + case VDUSE_IOTLB_REG_UMEM: { + struct vduse_iova_umem umem; + + ret = -EFAULT; + if (copy_from_user(&umem, argp, sizeof(umem))) + break; + + ret = -EINVAL; + if (!is_mem_zero((const char *)umem.reserved, + sizeof(umem.reserved))) + break; + + ret = vduse_dev_reg_umem(dev, umem.iova, + umem.uaddr, umem.size); + break; + } + case VDUSE_IOTLB_DEREG_UMEM: { + struct vduse_iova_umem umem; + + ret = -EFAULT; + if (copy_from_user(&umem, argp, sizeof(umem))) + break; + + ret = -EINVAL; + if (!is_mem_zero((const char *)umem.reserved, + sizeof(umem.reserved))) + break; + + ret = vduse_dev_dereg_umem(dev, umem.iova, + umem.size); + break; + } default: ret = -ENOIOCTLCMD; break; @@ -1101,6 +1240,7 @@ static int vduse_dev_release(struct inode *inode, struct file *file) { struct vduse_dev *dev = file->private_data; + vduse_dev_dereg_umem(dev, 0, dev->domain->bounce_size); spin_lock(&dev->msg_lock); /* Make sure the inflight messages can processed after reconncection */ list_splice_init(&dev->recv_list, &dev->send_list); @@ -1163,6 +1303,7 @@ static struct vduse_dev *vduse_dev_create(void) return NULL; mutex_init(&dev->lock); + mutex_init(&dev->mem_lock); spin_lock_init(&dev->msg_lock); INIT_LIST_HEAD(&dev->send_list); INIT_LIST_HEAD(&dev->recv_list); -- cgit From ad146355bfad627bd0717ece73997c6c93b1b82e Mon Sep 17 00:00:00 2001 From: Xie Yongji Date: Wed, 3 Aug 2022 12:55:23 +0800 Subject: vduse: Support querying information of IOVA regions This introduces a new ioctl: VDUSE_IOTLB_GET_INFO to support querying some information of IOVA regions. Now it can be used to query whether the IOVA region supports userspace memory registration. Signed-off-by: Xie Yongji Message-Id: <20220803045523.23851-6-xieyongji@bytedance.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vdpa/vdpa_user/vduse_dev.c | 39 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index eedff0a3885a..41c0b29739f1 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -1228,6 +1228,45 @@ static long vduse_dev_ioctl(struct file *file, unsigned int cmd, umem.size); break; } + case VDUSE_IOTLB_GET_INFO: { + struct vduse_iova_info info; + struct vhost_iotlb_map *map; + struct vduse_iova_domain *domain = dev->domain; + + ret = -EFAULT; + if (copy_from_user(&info, argp, sizeof(info))) + break; + + ret = -EINVAL; + if (info.start > info.last) + break; + + if (!is_mem_zero((const char *)info.reserved, + sizeof(info.reserved))) + break; + + spin_lock(&domain->iotlb_lock); + map = vhost_iotlb_itree_first(domain->iotlb, + info.start, info.last); + if (map) { + info.start = map->start; + info.last = map->last; + info.capability = 0; + if (domain->bounce_map && map->start == 0 && + map->last == domain->bounce_size - 1) + info.capability |= VDUSE_IOVA_CAP_UMEM; + } + spin_unlock(&domain->iotlb_lock); + if (!map) + break; + + ret = -EFAULT; + if (copy_to_user(argp, &info, sizeof(info))) + break; + + ret = 0; + break; + } default: ret = -ENOIOCTLCMD; break; -- cgit From cae15c2ed8e6e058bd5e32de292ab7982640161e Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Thu, 14 Jul 2022 14:39:26 +0300 Subject: vdpa/mlx5: Implement susupend virtqueue callback Implement the suspend callback allowing to suspend the virtqueues so they stop processing descriptors. This is required to allow to query a consistent state of the virtqueue while live migration is taking place. Signed-off-by: Eli Cohen Message-Id: <20220714113927.85729-2-elic@nvidia.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/mlx5/net/mlx5_vnet.c | 83 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 80 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 99bbbf38c8b1..a476e06476f6 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -164,6 +164,7 @@ struct mlx5_vdpa_net { bool setup; u32 cur_num_vqs; u32 rqt_size; + bool nb_registered; struct notifier_block nb; struct vdpa_callback config_cb; struct mlx5_vdpa_wq_ent cvq_ent; @@ -895,6 +896,7 @@ static int create_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque if (err) goto err_cmd; + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT; kfree(in); mvq->virtq_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); @@ -922,6 +924,7 @@ static void destroy_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtq mlx5_vdpa_warn(&ndev->mvdev, "destroy virtqueue 0x%x\n", mvq->virtq_id); return; } + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; umems_destroy(ndev, mvq); } @@ -1121,6 +1124,20 @@ err_cmd: return err; } +static bool is_valid_state_change(int oldstate, int newstate) +{ + switch (oldstate) { + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_INIT: + return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY: + return newstate == MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND; + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_SUSPEND: + case MLX5_VIRTIO_NET_Q_OBJECT_STATE_ERR: + default: + return false; + } +} + static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtqueue *mvq, int state) { int inlen = MLX5_ST_SZ_BYTES(modify_virtio_net_q_in); @@ -1130,6 +1147,12 @@ static int modify_virtqueue(struct mlx5_vdpa_net *ndev, struct mlx5_vdpa_virtque void *in; int err; + if (mvq->fw_state == MLX5_VIRTIO_NET_Q_OBJECT_NONE) + return 0; + + if (!is_valid_state_change(mvq->fw_state, state)) + return -EINVAL; + in = kzalloc(inlen, GFP_KERNEL); if (!in) return -ENOMEM; @@ -1992,6 +2015,7 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct mlx5_vdpa_virtqueue *mvq; + int err; if (!mvdev->actual_features) return; @@ -2005,8 +2029,16 @@ static void mlx5_vdpa_set_vq_ready(struct vdpa_device *vdev, u16 idx, bool ready } mvq = &ndev->vqs[idx]; - if (!ready) + if (!ready) { suspend_vq(ndev, mvq); + } else { + err = modify_virtqueue(ndev, mvq, MLX5_VIRTIO_NET_Q_OBJECT_STATE_RDY); + if (err) { + mlx5_vdpa_warn(mvdev, "modify VQ %d to ready failed (%d)\n", idx, err); + ready = false; + } + } + mvq->ready = ready; } @@ -2733,6 +2765,37 @@ out_err: return err; } +static void mlx5_vdpa_cvq_suspend(struct mlx5_vdpa_dev *mvdev) +{ + struct mlx5_control_vq *cvq; + + if (!(mvdev->actual_features & BIT_ULL(VIRTIO_NET_F_CTRL_VQ))) + return; + + cvq = &mvdev->cvq; + cvq->ready = false; +} + +static int mlx5_vdpa_suspend(struct vdpa_device *vdev) +{ + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + struct mlx5_vdpa_virtqueue *mvq; + int i; + + down_write(&ndev->reslock); + mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + ndev->nb_registered = false; + flush_workqueue(ndev->mvdev.wq); + for (i = 0; i < ndev->cur_num_vqs; i++) { + mvq = &ndev->vqs[i]; + suspend_vq(ndev, mvq); + } + mlx5_vdpa_cvq_suspend(mvdev); + up_write(&ndev->reslock); + return 0; +} + static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_vq_address = mlx5_vdpa_set_vq_address, .set_vq_num = mlx5_vdpa_set_vq_num, @@ -2763,6 +2826,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { .get_generation = mlx5_vdpa_get_generation, .set_map = mlx5_vdpa_set_map, .free = mlx5_vdpa_free, + .suspend = mlx5_vdpa_suspend, }; static int query_mtu(struct mlx5_core_dev *mdev, u16 *mtu) @@ -2828,6 +2892,7 @@ static void init_mvqs(struct mlx5_vdpa_net *ndev) mvq->index = i; mvq->ndev = ndev; mvq->fwqp.fw = true; + mvq->fw_state = MLX5_VIRTIO_NET_Q_OBJECT_NONE; } for (; i < ndev->mvdev.max_vqs; i++) { mvq = &ndev->vqs[i]; @@ -2902,13 +2967,21 @@ static int event_handler(struct notifier_block *nb, unsigned long event, void *p switch (eqe->sub_type) { case MLX5_PORT_CHANGE_SUBTYPE_DOWN: case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: + down_read(&ndev->reslock); + if (!ndev->nb_registered) { + up_read(&ndev->reslock); + return NOTIFY_DONE; + } wqent = kzalloc(sizeof(*wqent), GFP_ATOMIC); - if (!wqent) + if (!wqent) { + up_read(&ndev->reslock); return NOTIFY_DONE; + } wqent->mvdev = &ndev->mvdev; INIT_WORK(&wqent->work, update_carrier); queue_work(ndev->mvdev.wq, &wqent->work); + up_read(&ndev->reslock); ret = NOTIFY_OK; break; default: @@ -3062,6 +3135,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, ndev->nb.notifier_call = event_handler; mlx5_notifier_register(mdev, &ndev->nb); + ndev->nb_registered = true; mvdev->vdev.mdev = &mgtdev->mgtdev; err = _vdpa_register_device(&mvdev->vdev, max_vqs + 1); if (err) @@ -3093,7 +3167,10 @@ static void mlx5_vdpa_dev_del(struct vdpa_mgmt_dev *v_mdev, struct vdpa_device * struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); struct workqueue_struct *wq; - mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + if (ndev->nb_registered) { + mlx5_notifier_unregister(mvdev->mdev, &ndev->nb); + ndev->nb_registered = false; + } wq = mvdev->wq; mvdev->wq = NULL; destroy_workqueue(wq); -- cgit From 8fcd20c307042b0bb4fd3aee7fc23c94080306ad Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Thu, 14 Jul 2022 14:39:27 +0300 Subject: vdpa/mlx5: Support different address spaces for control and data Partition virtqueues to two different address spaces: one for control virtqueue which is implemented in software, and one for data virtqueues. Based-on: <20220526124338.36247-1-eperezma@redhat.com> Signed-off-by: Eli Cohen Message-Id: <20220714113927.85729-3-elic@nvidia.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/mlx5/core/mlx5_vdpa.h | 11 +++++ drivers/vdpa/mlx5/net/mlx5_vnet.c | 88 +++++++++++++++++++++++++++++++++----- 2 files changed, 88 insertions(+), 11 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/mlx5/core/mlx5_vdpa.h b/drivers/vdpa/mlx5/core/mlx5_vdpa.h index 44104093163b..6af9fdbb86b7 100644 --- a/drivers/vdpa/mlx5/core/mlx5_vdpa.h +++ b/drivers/vdpa/mlx5/core/mlx5_vdpa.h @@ -70,6 +70,16 @@ struct mlx5_vdpa_wq_ent { struct mlx5_vdpa_dev *mvdev; }; +enum { + MLX5_VDPA_DATAVQ_GROUP, + MLX5_VDPA_CVQ_GROUP, + MLX5_VDPA_NUMVQ_GROUPS +}; + +enum { + MLX5_VDPA_NUM_AS = MLX5_VDPA_NUMVQ_GROUPS +}; + struct mlx5_vdpa_dev { struct vdpa_device vdev; struct mlx5_core_dev *mdev; @@ -85,6 +95,7 @@ struct mlx5_vdpa_dev { struct mlx5_vdpa_mr mr; struct mlx5_control_vq cvq; struct workqueue_struct *wq; + unsigned int group2asid[MLX5_VDPA_NUMVQ_GROUPS]; }; int mlx5_vdpa_alloc_pd(struct mlx5_vdpa_dev *dev, u32 *pdn, u16 uid); diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index a476e06476f6..289cf91b1ff5 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -2127,9 +2127,14 @@ static u32 mlx5_vdpa_get_vq_align(struct vdpa_device *vdev) return PAGE_SIZE; } -static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdpa, u16 idx) +static u32 mlx5_vdpa_get_vq_group(struct vdpa_device *vdev, u16 idx) { - return 0; + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + + if (is_ctrl_vq_idx(mvdev, idx)) + return MLX5_VDPA_CVQ_GROUP; + + return MLX5_VDPA_DATAVQ_GROUP; } enum { MLX5_VIRTIO_NET_F_GUEST_CSUM = 1 << 9, @@ -2543,6 +2548,15 @@ err_clear: up_write(&ndev->reslock); } +static void init_group_to_asid_map(struct mlx5_vdpa_dev *mvdev) +{ + int i; + + /* default mapping all groups are mapped to asid 0 */ + for (i = 0; i < MLX5_VDPA_NUMVQ_GROUPS; i++) + mvdev->group2asid[i] = 0; +} + static int mlx5_vdpa_reset(struct vdpa_device *vdev) { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); @@ -2561,7 +2575,9 @@ static int mlx5_vdpa_reset(struct vdpa_device *vdev) ndev->mvdev.cvq.completed_desc = 0; memset(ndev->event_cbs, 0, sizeof(*ndev->event_cbs) * (mvdev->max_vqs + 1)); ndev->mvdev.actual_features = 0; + init_group_to_asid_map(mvdev); ++mvdev->generation; + if (MLX5_CAP_GEN(mvdev->mdev, umem_uid_0)) { if (mlx5_vdpa_create_mr(mvdev, NULL)) mlx5_vdpa_warn(mvdev, "create MR failed\n"); @@ -2599,26 +2615,63 @@ static u32 mlx5_vdpa_get_generation(struct vdpa_device *vdev) return mvdev->generation; } -static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, - struct vhost_iotlb *iotlb) +static int set_map_control(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) +{ + u64 start = 0ULL, last = 0ULL - 1; + struct vhost_iotlb_map *map; + int err = 0; + + spin_lock(&mvdev->cvq.iommu_lock); + vhost_iotlb_reset(mvdev->cvq.iotlb); + + for (map = vhost_iotlb_itree_first(iotlb, start, last); map; + map = vhost_iotlb_itree_next(map, start, last)) { + err = vhost_iotlb_add_range(mvdev->cvq.iotlb, map->start, + map->last, map->addr, map->perm); + if (err) + goto out; + } + +out: + spin_unlock(&mvdev->cvq.iommu_lock); + return err; +} + +static int set_map_data(struct mlx5_vdpa_dev *mvdev, struct vhost_iotlb *iotlb) { - struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); - struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); bool change_map; int err; - down_write(&ndev->reslock); - err = mlx5_vdpa_handle_set_map(mvdev, iotlb, &change_map); if (err) { mlx5_vdpa_warn(mvdev, "set map failed(%d)\n", err); - goto err; + return err; } if (change_map) err = mlx5_vdpa_change_map(mvdev, iotlb); -err: + return err; +} + +static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, + struct vhost_iotlb *iotlb) +{ + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); + int err; + + down_write(&ndev->reslock); + if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] == asid) { + err = set_map_data(mvdev, iotlb); + if (err) + goto out; + } + + if (mvdev->group2asid[MLX5_VDPA_CVQ_GROUP] == asid) + err = set_map_control(mvdev, iotlb); + +out: up_write(&ndev->reslock); return err; } @@ -2796,6 +2849,18 @@ static int mlx5_vdpa_suspend(struct vdpa_device *vdev) return 0; } +static int mlx5_set_group_asid(struct vdpa_device *vdev, u32 group, + unsigned int asid) +{ + struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); + + if (group >= MLX5_VDPA_NUMVQ_GROUPS) + return -EINVAL; + + mvdev->group2asid[group] = asid; + return 0; +} + static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_vq_address = mlx5_vdpa_set_vq_address, .set_vq_num = mlx5_vdpa_set_vq_num, @@ -2825,6 +2890,7 @@ static const struct vdpa_config_ops mlx5_vdpa_ops = { .set_config = mlx5_vdpa_set_config, .get_generation = mlx5_vdpa_get_generation, .set_map = mlx5_vdpa_set_map, + .set_group_asid = mlx5_set_group_asid, .free = mlx5_vdpa_free, .suspend = mlx5_vdpa_suspend, }; @@ -3055,7 +3121,7 @@ static int mlx5_vdpa_dev_add(struct vdpa_mgmt_dev *v_mdev, const char *name, } ndev = vdpa_alloc_device(struct mlx5_vdpa_net, mvdev.vdev, mdev->device, &mlx5_vdpa_ops, - 1, 1, name, false); + MLX5_VDPA_NUMVQ_GROUPS, MLX5_VDPA_NUM_AS, name, false); if (IS_ERR(ndev)) return PTR_ERR(ndev); -- cgit From 5a4b0420b28fed7e8df81ac7e3b53834142053dd Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Thu, 7 Jul 2022 22:05:24 -0500 Subject: vhost-scsi: Fix max number of virtqueues Qemu takes it's num_queues limit then adds the fixed queues (control and event) to the total it will request from the kernel. So when a user requests 128 (or qemu does it's num_queues calculation based on vCPUS and other system limits), we hit errors due to userspace trying to setup 130 queues when vhost-scsi has a hard coded limit of 128. This has vhost-scsi adjust it's max so we can do a total of 130 virtqueues (128 IO and 2 fixed). For the case where the user has 128 vCPUs the guest OS can then nicely map each IO virtqueue to a vCPU and not have the odd case where 2 vCPUs share a virtqueue. Signed-off-by: Mike Christie Message-Id: <20220708030525.5065-2-michael.christie@oracle.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vhost/scsi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index ffd9e6c2ffc1..8d6b4eef554d 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -159,7 +159,7 @@ enum { }; #define VHOST_SCSI_MAX_TARGET 256 -#define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_VQ 128 + VHOST_SCSI_VQ_IO #define VHOST_SCSI_MAX_EVENT 128 struct vhost_scsi_virtqueue { -- cgit From f49c2226af8444563897e25bf293ea29e377995a Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Thu, 7 Jul 2022 22:05:25 -0500 Subject: vhost scsi: Allow user to control num virtqueues We are currently hard coded to always create 128 IO virtqueues, so this adds a modparam to control it. For large systems where we are ok with using memory for virtqueues it allows us to add up to 1024. This limit was just selected becuase that's qemu's limit. Signed-off-by: Mike Christie Message-Id: <20220708030525.5065-3-michael.christie@oracle.com> Signed-off-by: Michael S. Tsirkin Acked-by: Jason Wang --- drivers/vhost/scsi.c | 85 +++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 61 insertions(+), 24 deletions(-) (limited to 'drivers') diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 8d6b4eef554d..d9861ab2c300 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -159,9 +159,13 @@ enum { }; #define VHOST_SCSI_MAX_TARGET 256 -#define VHOST_SCSI_MAX_VQ 128 + VHOST_SCSI_VQ_IO +#define VHOST_SCSI_MAX_IO_VQ 1024 #define VHOST_SCSI_MAX_EVENT 128 +static unsigned vhost_scsi_max_io_vqs = 128; +module_param_named(max_io_vqs, vhost_scsi_max_io_vqs, uint, 0644); +MODULE_PARM_DESC(max_io_vqs, "Set the max number of IO virtqueues a vhost scsi device can support. The default is 128. The max is 1024."); + struct vhost_scsi_virtqueue { struct vhost_virtqueue vq; /* @@ -186,7 +190,9 @@ struct vhost_scsi { char vs_vhost_wwpn[TRANSPORT_IQN_LEN]; struct vhost_dev dev; - struct vhost_scsi_virtqueue vqs[VHOST_SCSI_MAX_VQ]; + struct vhost_scsi_virtqueue *vqs; + unsigned long *compl_bitmap; + struct vhost_scsi_inflight **old_inflight; struct vhost_work vs_completion_work; /* cmd completion work item */ struct llist_head vs_completion_list; /* cmd completion queue */ @@ -245,7 +251,7 @@ static void vhost_scsi_init_inflight(struct vhost_scsi *vs, struct vhost_virtqueue *vq; int idx, i; - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); @@ -533,7 +539,6 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) { struct vhost_scsi *vs = container_of(work, struct vhost_scsi, vs_completion_work); - DECLARE_BITMAP(signal, VHOST_SCSI_MAX_VQ); struct virtio_scsi_cmd_resp v_rsp; struct vhost_scsi_cmd *cmd, *t; struct llist_node *llnode; @@ -541,7 +546,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) struct iov_iter iov_iter; int ret, vq; - bitmap_zero(signal, VHOST_SCSI_MAX_VQ); + bitmap_zero(vs->compl_bitmap, vs->dev.nvqs); llnode = llist_del_all(&vs->vs_completion_list); llist_for_each_entry_safe(cmd, t, llnode, tvc_completion_list) { se_cmd = &cmd->tvc_se_cmd; @@ -566,7 +571,7 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0); q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq); vq = q - vs->vqs; - __set_bit(vq, signal); + __set_bit(vq, vs->compl_bitmap); } else pr_err("Faulted on virtio_scsi_cmd_resp\n"); @@ -574,8 +579,8 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work *work) } vq = -1; - while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1)) - < VHOST_SCSI_MAX_VQ) + while ((vq = find_next_bit(vs->compl_bitmap, vs->dev.nvqs, vq + 1)) + < vs->dev.nvqs) vhost_signal(&vs->dev, &vs->vqs[vq].vq); } @@ -1421,26 +1426,25 @@ static void vhost_scsi_handle_kick(struct vhost_work *work) /* Callers must hold dev mutex */ static void vhost_scsi_flush(struct vhost_scsi *vs) { - struct vhost_scsi_inflight *old_inflight[VHOST_SCSI_MAX_VQ]; int i; /* Init new inflight and remember the old inflight */ - vhost_scsi_init_inflight(vs, old_inflight); + vhost_scsi_init_inflight(vs, vs->old_inflight); /* * The inflight->kref was initialized to 1. We decrement it here to * indicate the start of the flush operation so that it will reach 0 * when all the reqs are finished. */ - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) - kref_put(&old_inflight[i]->kref, vhost_scsi_done_inflight); + for (i = 0; i < vs->dev.nvqs; i++) + kref_put(&vs->old_inflight[i]->kref, vhost_scsi_done_inflight); /* Flush both the vhost poll and vhost work */ vhost_dev_flush(&vs->dev); /* Wait for all reqs issued before the flush to be finished */ - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) - wait_for_completion(&old_inflight[i]->comp); + for (i = 0; i < vs->dev.nvqs; i++) + wait_for_completion(&vs->old_inflight[i]->comp); } static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq) @@ -1603,7 +1607,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn, sizeof(vs->vs_vhost_wwpn)); - for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = VHOST_SCSI_VQ_IO; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; if (!vhost_vq_is_setup(vq)) continue; @@ -1613,7 +1617,7 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs, goto destroy_vq_cmds; } - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); vhost_vq_set_backend(vq, vs_tpg); @@ -1715,7 +1719,7 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, target_undepend_item(&se_tpg->tpg_group.cg_item); } if (match) { - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); vhost_vq_set_backend(vq, NULL); @@ -1724,7 +1728,7 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs, /* Make sure cmds are not running before tearing them down. */ vhost_scsi_flush(vs); - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; vhost_scsi_destroy_vq_cmds(vq); } @@ -1764,7 +1768,7 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs, u64 features) return -EFAULT; } - for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = 0; i < vs->dev.nvqs; i++) { vq = &vs->vqs[i].vq; mutex_lock(&vq->mutex); vq->acked_features = features; @@ -1778,16 +1782,40 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) { struct vhost_scsi *vs; struct vhost_virtqueue **vqs; - int r = -ENOMEM, i; + int r = -ENOMEM, i, nvqs = vhost_scsi_max_io_vqs; vs = kvzalloc(sizeof(*vs), GFP_KERNEL); if (!vs) goto err_vs; - vqs = kmalloc_array(VHOST_SCSI_MAX_VQ, sizeof(*vqs), GFP_KERNEL); - if (!vqs) + if (nvqs > VHOST_SCSI_MAX_IO_VQ) { + pr_err("Invalid max_io_vqs of %d. Using %d.\n", nvqs, + VHOST_SCSI_MAX_IO_VQ); + nvqs = VHOST_SCSI_MAX_IO_VQ; + } else if (nvqs == 0) { + pr_err("Invalid max_io_vqs of %d. Using 1.\n", nvqs); + nvqs = 1; + } + nvqs += VHOST_SCSI_VQ_IO; + + vs->compl_bitmap = bitmap_alloc(nvqs, GFP_KERNEL); + if (!vs->compl_bitmap) + goto err_compl_bitmap; + + vs->old_inflight = kmalloc_array(nvqs, sizeof(*vs->old_inflight), + GFP_KERNEL | __GFP_ZERO); + if (!vs->old_inflight) + goto err_inflight; + + vs->vqs = kmalloc_array(nvqs, sizeof(*vs->vqs), + GFP_KERNEL | __GFP_ZERO); + if (!vs->vqs) goto err_vqs; + vqs = kmalloc_array(nvqs, sizeof(*vqs), GFP_KERNEL); + if (!vqs) + goto err_local_vqs; + vhost_work_init(&vs->vs_completion_work, vhost_scsi_complete_cmd_work); vhost_work_init(&vs->vs_event_work, vhost_scsi_evt_work); @@ -1798,11 +1826,11 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq; vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick; vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick; - for (i = VHOST_SCSI_VQ_IO; i < VHOST_SCSI_MAX_VQ; i++) { + for (i = VHOST_SCSI_VQ_IO; i < nvqs; i++) { vqs[i] = &vs->vqs[i].vq; vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick; } - vhost_dev_init(&vs->dev, vqs, VHOST_SCSI_MAX_VQ, UIO_MAXIOV, + vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV, VHOST_SCSI_WEIGHT, 0, true, NULL); vhost_scsi_init_inflight(vs, NULL); @@ -1810,7 +1838,13 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) f->private_data = vs; return 0; +err_local_vqs: + kfree(vs->vqs); err_vqs: + kfree(vs->old_inflight); +err_inflight: + bitmap_free(vs->compl_bitmap); +err_compl_bitmap: kvfree(vs); err_vs: return r; @@ -1828,6 +1862,9 @@ static int vhost_scsi_release(struct inode *inode, struct file *f) vhost_dev_stop(&vs->dev); vhost_dev_cleanup(&vs->dev); kfree(vs->dev.vqs); + kfree(vs->vqs); + kfree(vs->old_inflight); + bitmap_free(vs->compl_bitmap); kvfree(vs); return 0; } -- cgit From 0d6e5e8d1686ce5d055c5ecaa76e5f51609ce9da Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 22 Jul 2022 19:53:04 +0800 Subject: vDPA/ifcvf: get_config_size should return a value no greater than dev implementation Drivers must not access a BAR outside the capability length, and for a virtio device, ifcvf driver should not report any non-standard capability contents to the upper layers. Function ifcvf_get_config_size() is introduced here to return a safe value of the device config capability size. Signed-off-by: Zhu Lingshan Message-Id: <20220722115309.82746-2-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_base.c | 13 +++++++++++-- drivers/vdpa/ifcvf/ifcvf_base.h | 2 ++ 2 files changed, 13 insertions(+), 2 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/ifcvf/ifcvf_base.c b/drivers/vdpa/ifcvf/ifcvf_base.c index 47b94091733c..75a703b803a2 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.c +++ b/drivers/vdpa/ifcvf/ifcvf_base.c @@ -127,6 +127,7 @@ int ifcvf_init_hw(struct ifcvf_hw *hw, struct pci_dev *pdev) break; case VIRTIO_PCI_CAP_DEVICE_CFG: hw->dev_cfg = get_cap_addr(hw, &cap); + hw->cap_dev_config_size = le32_to_cpu(cap.length); IFCVF_DBG(pdev, "hw->dev_cfg = %p\n", hw->dev_cfg); break; } @@ -232,15 +233,23 @@ int ifcvf_verify_min_features(struct ifcvf_hw *hw, u64 features) u32 ifcvf_get_config_size(struct ifcvf_hw *hw) { struct ifcvf_adapter *adapter; + u32 net_config_size = sizeof(struct virtio_net_config); + u32 blk_config_size = sizeof(struct virtio_blk_config); + u32 cap_size = hw->cap_dev_config_size; u32 config_size; adapter = vf_to_adapter(hw); + /* If the onboard device config space size is greater than + * the size of struct virtio_net/blk_config, only the spec + * implementing contents size is returned, this is very + * unlikely, defensive programming. + */ switch (hw->dev_type) { case VIRTIO_ID_NET: - config_size = sizeof(struct virtio_net_config); + config_size = min(cap_size, net_config_size); break; case VIRTIO_ID_BLOCK: - config_size = sizeof(struct virtio_blk_config); + config_size = min(cap_size, blk_config_size); break; default: config_size = 0; diff --git a/drivers/vdpa/ifcvf/ifcvf_base.h b/drivers/vdpa/ifcvf/ifcvf_base.h index 115b61f4924b..f5563f665cc6 100644 --- a/drivers/vdpa/ifcvf/ifcvf_base.h +++ b/drivers/vdpa/ifcvf/ifcvf_base.h @@ -87,6 +87,8 @@ struct ifcvf_hw { int config_irq; int vqs_reused_irq; u16 nr_vring; + /* VIRTIO_PCI_CAP_DEVICE_CFG size */ + u32 cap_dev_config_size; }; struct ifcvf_adapter { -- cgit From 378b2e956820ff5c082d05f42828badcfbabb614 Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 22 Jul 2022 19:53:05 +0800 Subject: vDPA/ifcvf: support userspace to query features and MQ of a management device Adapting to current netlink interfaces, this commit allows userspace to query feature bits and MQ capability of a management device. Currently both the vDPA device and the management device are the VF itself, thus this ifcvf should initialize the virtio capabilities in probe() before setting up the struct vdpa_mgmt_dev. Signed-off-by: Zhu Lingshan Message-Id: <20220722115309.82746-3-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/ifcvf/ifcvf_main.c | 142 +++++++++++++++++++++------------------- 1 file changed, 76 insertions(+), 66 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/ifcvf/ifcvf_main.c b/drivers/vdpa/ifcvf/ifcvf_main.c index e2e201885713..f9c0044c6442 100644 --- a/drivers/vdpa/ifcvf/ifcvf_main.c +++ b/drivers/vdpa/ifcvf/ifcvf_main.c @@ -752,59 +752,36 @@ static int ifcvf_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, { struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev; struct ifcvf_adapter *adapter; + struct vdpa_device *vdpa_dev; struct pci_dev *pdev; struct ifcvf_hw *vf; - struct device *dev; - int ret, i; + int ret; ifcvf_mgmt_dev = container_of(mdev, struct ifcvf_vdpa_mgmt_dev, mdev); - if (ifcvf_mgmt_dev->adapter) + if (!ifcvf_mgmt_dev->adapter) return -EOPNOTSUPP; - pdev = ifcvf_mgmt_dev->pdev; - dev = &pdev->dev; - adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, - dev, &ifc_vdpa_ops, 1, 1, name, false); - if (IS_ERR(adapter)) { - IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); - return PTR_ERR(adapter); - } - - ifcvf_mgmt_dev->adapter = adapter; - + adapter = ifcvf_mgmt_dev->adapter; vf = &adapter->vf; - vf->dev_type = get_dev_type(pdev); - vf->base = pcim_iomap_table(pdev); + pdev = adapter->pdev; + vdpa_dev = &adapter->vdpa; - adapter->pdev = pdev; - adapter->vdpa.dma_dev = &pdev->dev; - - ret = ifcvf_init_hw(vf, pdev); - if (ret) { - IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); - goto err; - } - - for (i = 0; i < vf->nr_vring; i++) - vf->vring[i].irq = -EINVAL; - - vf->hw_features = ifcvf_get_hw_features(vf); - vf->config_size = ifcvf_get_config_size(vf); + if (name) + ret = dev_set_name(&vdpa_dev->dev, "%s", name); + else + ret = dev_set_name(&vdpa_dev->dev, "vdpa%u", vdpa_dev->index); - adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev; ret = _vdpa_register_device(&adapter->vdpa, vf->nr_vring); if (ret) { + put_device(&adapter->vdpa.dev); IFCVF_ERR(pdev, "Failed to register to vDPA bus"); - goto err; + return ret; } return 0; - -err: - put_device(&adapter->vdpa.dev); - return ret; } + static void ifcvf_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev) { struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev; @@ -823,61 +800,94 @@ static int ifcvf_probe(struct pci_dev *pdev, const struct pci_device_id *id) { struct ifcvf_vdpa_mgmt_dev *ifcvf_mgmt_dev; struct device *dev = &pdev->dev; + struct ifcvf_adapter *adapter; + struct ifcvf_hw *vf; u32 dev_type; - int ret; - - ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL); - if (!ifcvf_mgmt_dev) { - IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n"); - return -ENOMEM; - } - - dev_type = get_dev_type(pdev); - switch (dev_type) { - case VIRTIO_ID_NET: - ifcvf_mgmt_dev->mdev.id_table = id_table_net; - break; - case VIRTIO_ID_BLOCK: - ifcvf_mgmt_dev->mdev.id_table = id_table_blk; - break; - default: - IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type); - ret = -EOPNOTSUPP; - goto err; - } - - ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops; - ifcvf_mgmt_dev->mdev.device = dev; - ifcvf_mgmt_dev->pdev = pdev; + int ret, i; ret = pcim_enable_device(pdev); if (ret) { IFCVF_ERR(pdev, "Failed to enable device\n"); - goto err; + return ret; } - ret = pcim_iomap_regions(pdev, BIT(0) | BIT(2) | BIT(4), IFCVF_DRIVER_NAME); if (ret) { IFCVF_ERR(pdev, "Failed to request MMIO region\n"); - goto err; + return ret; } ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); if (ret) { IFCVF_ERR(pdev, "No usable DMA configuration\n"); - goto err; + return ret; } ret = devm_add_action_or_reset(dev, ifcvf_free_irq_vectors, pdev); if (ret) { IFCVF_ERR(pdev, "Failed for adding devres for freeing irq vectors\n"); - goto err; + return ret; } pci_set_master(pdev); + adapter = vdpa_alloc_device(struct ifcvf_adapter, vdpa, + dev, &ifc_vdpa_ops, 1, 1, NULL, false); + if (IS_ERR(adapter)) { + IFCVF_ERR(pdev, "Failed to allocate vDPA structure"); + return PTR_ERR(adapter); + } + + vf = &adapter->vf; + vf->dev_type = get_dev_type(pdev); + vf->base = pcim_iomap_table(pdev); + + adapter->pdev = pdev; + adapter->vdpa.dma_dev = &pdev->dev; + + ret = ifcvf_init_hw(vf, pdev); + if (ret) { + IFCVF_ERR(pdev, "Failed to init IFCVF hw\n"); + return ret; + } + + for (i = 0; i < vf->nr_vring; i++) + vf->vring[i].irq = -EINVAL; + + vf->hw_features = ifcvf_get_hw_features(vf); + vf->config_size = ifcvf_get_config_size(vf); + + ifcvf_mgmt_dev = kzalloc(sizeof(struct ifcvf_vdpa_mgmt_dev), GFP_KERNEL); + if (!ifcvf_mgmt_dev) { + IFCVF_ERR(pdev, "Failed to alloc memory for the vDPA management device\n"); + return -ENOMEM; + } + + ifcvf_mgmt_dev->mdev.ops = &ifcvf_vdpa_mgmt_dev_ops; + ifcvf_mgmt_dev->mdev.device = dev; + ifcvf_mgmt_dev->adapter = adapter; + + dev_type = get_dev_type(pdev); + switch (dev_type) { + case VIRTIO_ID_NET: + ifcvf_mgmt_dev->mdev.id_table = id_table_net; + break; + case VIRTIO_ID_BLOCK: + ifcvf_mgmt_dev->mdev.id_table = id_table_blk; + break; + default: + IFCVF_ERR(pdev, "VIRTIO ID %u not supported\n", dev_type); + ret = -EOPNOTSUPP; + goto err; + } + + ifcvf_mgmt_dev->mdev.max_supported_vqs = vf->nr_vring; + ifcvf_mgmt_dev->mdev.supported_features = vf->hw_features; + + adapter->vdpa.mdev = &ifcvf_mgmt_dev->mdev; + + ret = vdpa_mgmtdev_register(&ifcvf_mgmt_dev->mdev); if (ret) { IFCVF_ERR(pdev, -- cgit From a34bed37fc9d3da319bb75dfbf02a7d3e95e12de Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 22 Jul 2022 19:53:07 +0800 Subject: vDPA: !FEATURES_OK should not block querying device config space Users may want to query the config space of a vDPA device, to choose a appropriate one for a certain guest. This means the users need to read the config space before FEATURES_OK, and the existence of config space contents does not depend on FEATURES_OK. The spec says: The device MUST allow reading of any device-specific configuration field before FEATURES_OK is set by the driver. This includes fields which are conditional on feature bits, as long as those feature bits are offered by the device. Signed-off-by: Zhu Lingshan Message-Id: <20220722115309.82746-5-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index ebf2f363fbe7..0e99b20e942e 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -846,17 +846,9 @@ vdpa_dev_config_fill(struct vdpa_device *vdev, struct sk_buff *msg, u32 portid, { u32 device_id; void *hdr; - u8 status; int err; down_read(&vdev->cf_lock); - status = vdev->config->get_status(vdev); - if (!(status & VIRTIO_CONFIG_S_FEATURES_OK)) { - NL_SET_ERR_MSG_MOD(extack, "Features negotiation not completed"); - err = -EAGAIN; - goto out; - } - hdr = genlmsg_put(msg, portid, seq, &vdpa_nl_family, flags, VDPA_CMD_DEV_CONFIG_GET); if (!hdr) { -- cgit From 79e0034cb3485e64622ec0aabf8a6f4f8143f47b Mon Sep 17 00:00:00 2001 From: Zhu Lingshan Date: Fri, 22 Jul 2022 19:53:09 +0800 Subject: vDPA: fix 'cast to restricted le16' warnings in vdpa.c This commit fixes spars warnings: cast to restricted __le16 in function vdpa_dev_net_config_fill() and vdpa_fill_stats_rec() Signed-off-by: Zhu Lingshan Reviewed-by: Parav Pandit Message-Id: <20220722115309.82746-7-lingshan.zhu@intel.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa.c b/drivers/vdpa/vdpa.c index 0e99b20e942e..c06c02704461 100644 --- a/drivers/vdpa/vdpa.c +++ b/drivers/vdpa/vdpa.c @@ -824,11 +824,11 @@ static int vdpa_dev_net_config_fill(struct vdpa_device *vdev, struct sk_buff *ms config.mac)) return -EMSGSIZE; - val_u16 = le16_to_cpu(config.status); + val_u16 = __virtio16_to_cpu(true, config.status); if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_STATUS, val_u16)) return -EMSGSIZE; - val_u16 = le16_to_cpu(config.mtu); + val_u16 = __virtio16_to_cpu(true, config.mtu); if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MTU, val_u16)) return -EMSGSIZE; @@ -905,7 +905,7 @@ static int vdpa_fill_stats_rec(struct vdpa_device *vdev, struct sk_buff *msg, } vdpa_get_config_unlocked(vdev, 0, &config, sizeof(config)); - max_vqp = le16_to_cpu(config.max_virtqueue_pairs); + max_vqp = __virtio16_to_cpu(true, config.max_virtqueue_pairs); if (nla_put_u16(msg, VDPA_ATTR_DEV_NET_CFG_MAX_VQP, max_vqp)) return -EMSGSIZE; -- cgit From ebe6a354fa7e0a7d5b581da31ad031b19d8693f9 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Fri, 5 Aug 2022 05:12:54 -0400 Subject: vhost-vdpa: Call ida_simple_remove() when failed In function vhost_vdpa_probe(), when code execution fails, we should call ida_simple_remove() to free ida. Signed-off-by: Bo Liu Message-Id: <20220805091254.20026-1-liubo03@inspur.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 250ad4160ccb..2c997d77d266 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -1363,6 +1363,7 @@ static int vhost_vdpa_probe(struct vdpa_device *vdpa) err: put_device(&v->dev); + ida_simple_remove(&vhost_vdpa_ida, v->minor); return r; } -- cgit From 99e8927d8a4da8eb8a8a5904dc13a3156be8e7c0 Mon Sep 17 00:00:00 2001 From: Bo Liu Date: Wed, 10 Aug 2022 04:51:51 -0400 Subject: virtio_vdpa: support the arg sizes of find_vqs() Virtio vdpa support the new parameter sizes of find_vqs(). Signed-off-by: Bo Liu Message-Id: <20220810085151.7251-1-liubo03@inspur.com> Signed-off-by: Michael S. Tsirkin --- drivers/virtio/virtio_vdpa.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers') diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index 832d2c5b1b19..9bc4d110b800 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -131,7 +131,7 @@ static irqreturn_t virtio_vdpa_virtqueue_cb(void *private) static struct virtqueue * virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, void (*callback)(struct virtqueue *vq), - const char *name, bool ctx) + const char *name, u32 size, bool ctx) { struct virtio_vdpa_device *vd_dev = to_virtio_vdpa_device(vdev); struct vdpa_device *vdpa = vd_get_vdpa(vdev); @@ -168,14 +168,17 @@ virtio_vdpa_setup_vq(struct virtio_device *vdev, unsigned int index, goto error_new_virtqueue; } + if (!size || size > max_num) + size = max_num; + if (ops->get_vq_num_min) min_num = ops->get_vq_num_min(vdpa); - may_reduce_num = (max_num == min_num) ? false : true; + may_reduce_num = (size == min_num) ? false : true; /* Create the vring */ align = ops->get_vq_align(vdpa); - vq = vring_create_virtqueue(index, max_num, align, vdev, + vq = vring_create_virtqueue(index, size, align, vdev, true, may_reduce_num, ctx, virtio_vdpa_notify, callback, name); if (!vq) { @@ -285,9 +288,9 @@ static int virtio_vdpa_find_vqs(struct virtio_device *vdev, unsigned int nvqs, continue; } - vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, - callbacks[i], names[i], ctx ? - ctx[i] : false); + vqs[i] = virtio_vdpa_setup_vq(vdev, queue_idx++, callbacks[i], + names[i], sizes ? sizes[i] : 0, + ctx ? ctx[i] : false); if (IS_ERR(vqs[i])) { err = PTR_ERR(vqs[i]); goto err_setup_vq; -- cgit From 8d12ec10292877751ee4463b11a63bd850bc09b5 Mon Sep 17 00:00:00 2001 From: Shigeru Yoshida Date: Thu, 11 Aug 2022 01:09:48 +0900 Subject: virtio-blk: Avoid use-after-free on suspend/resume hctx->user_data is set to vq in virtblk_init_hctx(). However, vq is freed on suspend and reallocated on resume. So, hctx->user_data is invalid after resume, and it will cause use-after-free accessing which will result in the kernel crash something like below: [ 22.428391] Call Trace: [ 22.428899] [ 22.429339] virtqueue_add_split+0x3eb/0x620 [ 22.430035] ? __blk_mq_alloc_requests+0x17f/0x2d0 [ 22.430789] ? kvm_clock_get_cycles+0x14/0x30 [ 22.431496] virtqueue_add_sgs+0xad/0xd0 [ 22.432108] virtblk_add_req+0xe8/0x150 [ 22.432692] virtio_queue_rqs+0xeb/0x210 [ 22.433330] blk_mq_flush_plug_list+0x1b8/0x280 [ 22.434059] __blk_flush_plug+0xe1/0x140 [ 22.434853] blk_finish_plug+0x20/0x40 [ 22.435512] read_pages+0x20a/0x2e0 [ 22.436063] ? folio_add_lru+0x62/0xa0 [ 22.436652] page_cache_ra_unbounded+0x112/0x160 [ 22.437365] filemap_get_pages+0xe1/0x5b0 [ 22.437964] ? context_to_sid+0x70/0x100 [ 22.438580] ? sidtab_context_to_sid+0x32/0x400 [ 22.439979] filemap_read+0xcd/0x3d0 [ 22.440917] xfs_file_buffered_read+0x4a/0xc0 [ 22.441984] xfs_file_read_iter+0x65/0xd0 [ 22.442970] __kernel_read+0x160/0x2e0 [ 22.443921] bprm_execve+0x21b/0x640 [ 22.444809] do_execveat_common.isra.0+0x1a8/0x220 [ 22.446008] __x64_sys_execve+0x2d/0x40 [ 22.446920] do_syscall_64+0x37/0x90 [ 22.447773] entry_SYSCALL_64_after_hwframe+0x63/0xcd This patch fixes this issue by getting vq from vblk, and removes virtblk_init_hctx(). Fixes: 4e0400525691 ("virtio-blk: support polling I/O") Cc: "Suwan Kim" Signed-off-by: Shigeru Yoshida Message-Id: <20220810160948.959781-1-syoshida@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/block/virtio_blk.c | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) (limited to 'drivers') diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index 6fc7850c2b0a..d756423e0059 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -101,6 +101,14 @@ static inline blk_status_t virtblk_result(struct virtblk_req *vbr) } } +static inline struct virtio_blk_vq *get_virtio_blk_vq(struct blk_mq_hw_ctx *hctx) +{ + struct virtio_blk *vblk = hctx->queue->queuedata; + struct virtio_blk_vq *vq = &vblk->vqs[hctx->queue_num]; + + return vq; +} + static int virtblk_add_req(struct virtqueue *vq, struct virtblk_req *vbr) { struct scatterlist hdr, status, *sgs[3]; @@ -416,7 +424,7 @@ static void virtio_queue_rqs(struct request **rqlist) struct request *requeue_list = NULL; rq_list_for_each_safe(rqlist, req, next) { - struct virtio_blk_vq *vq = req->mq_hctx->driver_data; + struct virtio_blk_vq *vq = get_virtio_blk_vq(req->mq_hctx); bool kick; if (!virtblk_prep_rq_batch(req)) { @@ -837,7 +845,7 @@ static void virtblk_complete_batch(struct io_comp_batch *iob) static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) { struct virtio_blk *vblk = hctx->queue->queuedata; - struct virtio_blk_vq *vq = hctx->driver_data; + struct virtio_blk_vq *vq = get_virtio_blk_vq(hctx); struct virtblk_req *vbr; unsigned long flags; unsigned int len; @@ -862,22 +870,10 @@ static int virtblk_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob) return found; } -static int virtblk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data, - unsigned int hctx_idx) -{ - struct virtio_blk *vblk = data; - struct virtio_blk_vq *vq = &vblk->vqs[hctx_idx]; - - WARN_ON(vblk->tag_set.tags[hctx_idx] != hctx->tags); - hctx->driver_data = vq; - return 0; -} - static const struct blk_mq_ops virtio_mq_ops = { .queue_rq = virtio_queue_rq, .queue_rqs = virtio_queue_rqs, .commit_rqs = virtio_commit_rqs, - .init_hctx = virtblk_init_hctx, .complete = virtblk_request_done, .map_queues = virtblk_map_queues, .poll = virtblk_poll, -- cgit From 0723f1df5c3ec8a1112d150dab98e149361ef488 Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:10 +0200 Subject: vhost-vdpa: introduce SUSPEND backend feature bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Userland knows if it can suspend the device or not by checking this feature bit. It's only offered if the vdpa driver backend implements the suspend() operation callback, and to offer it or userland to ack it if the backend does not offer that callback is an error. Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-3-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 2c997d77d266..092752fea8e1 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -347,6 +347,14 @@ static long vhost_vdpa_set_config(struct vhost_vdpa *v, return 0; } +static bool vhost_vdpa_can_suspend(const struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + return ops->suspend; +} + static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep) { struct vdpa_device *vdpa = v->vdpa; @@ -577,7 +585,11 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, if (cmd == VHOST_SET_BACKEND_FEATURES) { if (copy_from_user(&features, featurep, sizeof(features))) return -EFAULT; - if (features & ~VHOST_VDPA_BACKEND_FEATURES) + if (features & ~(VHOST_VDPA_BACKEND_FEATURES | + BIT_ULL(VHOST_BACKEND_F_SUSPEND))) + return -EOPNOTSUPP; + if ((features & BIT_ULL(VHOST_BACKEND_F_SUSPEND)) && + !vhost_vdpa_can_suspend(v)) return -EOPNOTSUPP; vhost_set_backend_features(&v->vdev, features); return 0; @@ -628,6 +640,8 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, break; case VHOST_GET_BACKEND_FEATURES: features = VHOST_VDPA_BACKEND_FEATURES; + if (vhost_vdpa_can_suspend(v)) + features |= BIT_ULL(VHOST_BACKEND_F_SUSPEND); if (copy_to_user(featurep, &features, sizeof(features))) r = -EFAULT; break; -- cgit From f345a0143b4dd1cfc850009c6979a3801b86a06f Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:11 +0200 Subject: vhost-vdpa: uAPI to suspend the device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ioctl adds support for suspending the device from userspace. This is a must before getting virtqueue indexes (base) for live migration, since the device could modify them after userland gets them. There are individual ways to perform that action for some devices (VHOST_NET_SET_BACKEND, VHOST_VSOCK_SET_RUNNING, ...) but there was no way to perform it for any vhost device (and, in particular, vhost-vdpa). After a successful return of the ioctl call the device must not process more virtqueue descriptors. The device can answer to read or writes of config fields as if it were not suspended. In particular, writing to "queue_enable" with a value of 1 will not make the device start processing buffers of the virtqueue. Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-4-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vhost/vdpa.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'drivers') diff --git a/drivers/vhost/vdpa.c b/drivers/vhost/vdpa.c index 092752fea8e1..166044642fd5 100644 --- a/drivers/vhost/vdpa.c +++ b/drivers/vhost/vdpa.c @@ -478,6 +478,22 @@ static long vhost_vdpa_get_vqs_count(struct vhost_vdpa *v, u32 __user *argp) return 0; } +/* After a successful return of ioctl the device must not process more + * virtqueue descriptors. The device can answer to read or writes of config + * fields as if it were not suspended. In particular, writing to "queue_enable" + * with a value of 1 will not make the device start processing buffers. + */ +static long vhost_vdpa_suspend(struct vhost_vdpa *v) +{ + struct vdpa_device *vdpa = v->vdpa; + const struct vdpa_config_ops *ops = vdpa->config; + + if (!ops->suspend) + return -EOPNOTSUPP; + + return ops->suspend(vdpa); +} + static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd, void __user *argp) { @@ -654,6 +670,9 @@ static long vhost_vdpa_unlocked_ioctl(struct file *filep, case VHOST_VDPA_GET_VQS_COUNT: r = vhost_vdpa_get_vqs_count(v, argp); break; + case VHOST_VDPA_SUSPEND: + r = vhost_vdpa_suspend(v); + break; default: r = vhost_dev_ioctl(&v->vdev, cmd, argp); if (r == -ENOIOCTLCMD) -- cgit From 0c89e2a3a9d0815b6eb769ed22157bd9996cbb58 Mon Sep 17 00:00:00 2001 From: Eugenio Pérez Date: Wed, 10 Aug 2022 19:15:12 +0200 Subject: vdpa_sim: Implement suspend vdpa op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement suspend operation for vdpa_sim devices, so vhost-vdpa will offer that backend feature and userspace can effectively suspend the device. This is a must before get virtqueue indexes (base) for live migration, since the device could modify them after userland gets them. There are individual ways to perform that action for some devices (VHOST_NET_SET_BACKEND, VHOST_VSOCK_SET_RUNNING, ...) but there was no way to perform it for any vhost device (and, in particular, vhost-vdpa). Reviewed-by: Stefano Garzarella Signed-off-by: Eugenio Pérez Message-Id: <20220810171512.2343333-5-eperezma@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim.c | 14 ++++++++++++++ drivers/vdpa/vdpa_sim/vdpa_sim.h | 1 + drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 3 +++ drivers/vdpa/vdpa_sim/vdpa_sim_net.c | 3 +++ 4 files changed, 21 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.c b/drivers/vdpa/vdpa_sim/vdpa_sim.c index 3e81532c01cb..225b7f5d8be3 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.c @@ -107,6 +107,7 @@ static void vdpasim_do_reset(struct vdpasim *vdpasim) for (i = 0; i < vdpasim->dev_attr.nas; i++) vhost_iotlb_reset(&vdpasim->iommu[i]); + vdpasim->running = true; spin_unlock(&vdpasim->iommu_lock); vdpasim->features = 0; @@ -505,6 +506,17 @@ static int vdpasim_reset(struct vdpa_device *vdpa) return 0; } +static int vdpasim_suspend(struct vdpa_device *vdpa) +{ + struct vdpasim *vdpasim = vdpa_to_sim(vdpa); + + spin_lock(&vdpasim->lock); + vdpasim->running = false; + spin_unlock(&vdpasim->lock); + + return 0; +} + static size_t vdpasim_get_config_size(struct vdpa_device *vdpa) { struct vdpasim *vdpasim = vdpa_to_sim(vdpa); @@ -694,6 +706,7 @@ static const struct vdpa_config_ops vdpasim_config_ops = { .get_status = vdpasim_get_status, .set_status = vdpasim_set_status, .reset = vdpasim_reset, + .suspend = vdpasim_suspend, .get_config_size = vdpasim_get_config_size, .get_config = vdpasim_get_config, .set_config = vdpasim_set_config, @@ -726,6 +739,7 @@ static const struct vdpa_config_ops vdpasim_batch_config_ops = { .get_status = vdpasim_get_status, .set_status = vdpasim_set_status, .reset = vdpasim_reset, + .suspend = vdpasim_suspend, .get_config_size = vdpasim_get_config_size, .get_config = vdpasim_get_config, .set_config = vdpasim_set_config, diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim.h b/drivers/vdpa/vdpa_sim/vdpa_sim.h index 622782e92239..061986f30911 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim.h +++ b/drivers/vdpa/vdpa_sim/vdpa_sim.h @@ -66,6 +66,7 @@ struct vdpasim { u32 generation; u64 features; u32 groups; + bool running; /* spinlock to synchronize iommu table */ spinlock_t iommu_lock; }; diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index dde965a826bb..ba253f8bce32 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -213,6 +213,9 @@ static void vdpasim_blk_work(struct work_struct *work) if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) goto out; + if (!vdpasim->running) + goto out; + for (i = 0; i < VDPASIM_BLK_VQ_NUM; i++) { struct vdpasim_virtqueue *vq = &vdpasim->vqs[i]; int reqs = 0; diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c index 5125976a4df8..886449e88502 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_net.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_net.c @@ -154,6 +154,9 @@ static void vdpasim_net_work(struct work_struct *work) spin_lock(&vdpasim->lock); + if (!vdpasim->running) + goto out; + if (!(vdpasim->status & VIRTIO_CONFIG_S_DRIVER_OK)) goto out; -- cgit From b91cf6e95b4f987d0d26def0c9cca3168d7752cb Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 11 Aug 2022 10:36:29 +0200 Subject: vdpa_sim_blk: check if sector is 0 for commands other than read or write VIRTIO spec states: "The sector number indicates the offset (multiplied by 512) where the read or write is to occur. This field is unused and set to 0 for commands other than read or write." Signed-off-by: Stefano Garzarella Message-Id: <20220811083632.77525-2-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index ba253f8bce32..69c9f4f36f9a 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -112,6 +112,15 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, offset = sector << SECTOR_SHIFT; status = VIRTIO_BLK_S_OK; + if (type != VIRTIO_BLK_T_IN && type != VIRTIO_BLK_T_OUT && + sector != 0) { + dev_dbg(&vdpasim->vdpa.dev, + "sector must be 0 for %u request - sector: 0x%llx\n", + type, sector); + status = VIRTIO_BLK_S_IOERR; + goto err_status; + } + switch (type) { case VIRTIO_BLK_T_IN: if (!vdpasim_blk_check_range(sector, to_push)) { @@ -178,6 +187,7 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, break; } +err_status: /* If some operations fail, we need to skip the remaining bytes * to put the status in the last byte */ -- cgit From ac926e1b468e03818e31568f6e520390b945b038 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 11 Aug 2022 10:36:30 +0200 Subject: vdpa_sim_blk: make vdpasim_blk_check_range usable by other requests Next patches will add handling of other requests, where will be useful to reuse vdpasim_blk_check_range(). So let's make it more generic by adding the `max_sectors` parameter, since different requests allow different numbers of maximum sectors. Let's also print the messages directly in vdpasim_blk_check_range() to avoid duplicate prints. Signed-off-by: Stefano Garzarella Message-Id: <20220811083632.77525-3-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 38 ++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 15 deletions(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index 69c9f4f36f9a..24dd9cae6450 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -42,18 +42,28 @@ static char vdpasim_blk_id[VIRTIO_BLK_ID_BYTES] = "vdpa_blk_sim"; -static bool vdpasim_blk_check_range(u64 start_sector, size_t range_size) +static bool vdpasim_blk_check_range(struct vdpasim *vdpasim, u64 start_sector, + u64 num_sectors, u64 max_sectors) { - u64 range_sectors = range_size >> SECTOR_SHIFT; - - if (range_size > VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX) - return false; + if (start_sector > VDPASIM_BLK_CAPACITY) { + dev_dbg(&vdpasim->vdpa.dev, + "starting sector exceeds the capacity - start: 0x%llx capacity: 0x%x\n", + start_sector, VDPASIM_BLK_CAPACITY); + } - if (start_sector > VDPASIM_BLK_CAPACITY) + if (num_sectors > max_sectors) { + dev_dbg(&vdpasim->vdpa.dev, + "number of sectors exceeds the max allowed in a request - num: 0x%llx max: 0x%llx\n", + num_sectors, max_sectors); return false; + } - if (range_sectors > VDPASIM_BLK_CAPACITY - start_sector) + if (num_sectors > VDPASIM_BLK_CAPACITY - start_sector) { + dev_dbg(&vdpasim->vdpa.dev, + "request exceeds the capacity - start: 0x%llx num: 0x%llx capacity: 0x%x\n", + start_sector, num_sectors, VDPASIM_BLK_CAPACITY); return false; + } return true; } @@ -123,10 +133,9 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, switch (type) { case VIRTIO_BLK_T_IN: - if (!vdpasim_blk_check_range(sector, to_push)) { - dev_dbg(&vdpasim->vdpa.dev, - "reading over the capacity - offset: 0x%llx len: 0x%zx\n", - offset, to_push); + if (!vdpasim_blk_check_range(vdpasim, sector, + to_push >> SECTOR_SHIFT, + VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) { status = VIRTIO_BLK_S_IOERR; break; } @@ -146,10 +155,9 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, break; case VIRTIO_BLK_T_OUT: - if (!vdpasim_blk_check_range(sector, to_pull)) { - dev_dbg(&vdpasim->vdpa.dev, - "writing over the capacity - offset: 0x%llx len: 0x%zx\n", - offset, to_pull); + if (!vdpasim_blk_check_range(vdpasim, sector, + to_pull >> SECTOR_SHIFT, + VDPASIM_BLK_SIZE_MAX * VDPASIM_BLK_SEG_MAX)) { status = VIRTIO_BLK_S_IOERR; break; } -- cgit From 518083d2f5b2e7c91549ac1723923e8ae3d679ca Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 11 Aug 2022 10:36:31 +0200 Subject: vdpa_sim_blk: add support for VIRTIO_BLK_T_FLUSH The simulator behaves like a ramdisk, so we don't have to do anything when a VIRTIO_BLK_T_FLUSH request is received, but it could be useful to test driver behavior. Let's expose the VIRTIO_BLK_F_FLUSH feature to inform the driver that we support the flush command. Signed-off-by: Stefano Garzarella Message-Id: <20220811083632.77525-4-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index 24dd9cae6450..1f18f336cf0c 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -25,6 +25,7 @@ #define DRV_LICENSE "GPL v2" #define VDPASIM_BLK_FEATURES (VDPASIM_FEATURES | \ + (1ULL << VIRTIO_BLK_F_FLUSH) | \ (1ULL << VIRTIO_BLK_F_SIZE_MAX) | \ (1ULL << VIRTIO_BLK_F_SEG_MAX) | \ (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \ @@ -188,6 +189,10 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, pushed += bytes; break; + case VIRTIO_BLK_T_FLUSH: + /* nothing to do */ + break; + default: dev_dbg(&vdpasim->vdpa.dev, "Unsupported request type %d\n", type); -- cgit From 4a44a5eda245453ed7bd14239c596262cb53b665 Mon Sep 17 00:00:00 2001 From: Stefano Garzarella Date: Thu, 11 Aug 2022 10:36:32 +0200 Subject: vdpa_sim_blk: add support for discard and write-zeroes Expose VIRTIO_BLK_F_DISCARD and VIRTIO_BLK_F_WRITE_ZEROES features to the drivers and handle VIRTIO_BLK_T_DISCARD and VIRTIO_BLK_T_WRITE_ZEROES requests checking ranges and flags. The simulator behaves like a ramdisk, so for VIRTIO_BLK_F_DISCARD does nothing, while for VIRTIO_BLK_T_WRITE_ZEROES sets to 0 the specified region. Signed-off-by: Stefano Garzarella Message-Id: <20220811083632.77525-5-sgarzare@redhat.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/vdpa_sim/vdpa_sim_blk.c | 74 +++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c index 1f18f336cf0c..c8bfea3b7db2 100644 --- a/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c +++ b/drivers/vdpa/vdpa_sim/vdpa_sim_blk.c @@ -30,11 +30,14 @@ (1ULL << VIRTIO_BLK_F_SEG_MAX) | \ (1ULL << VIRTIO_BLK_F_BLK_SIZE) | \ (1ULL << VIRTIO_BLK_F_TOPOLOGY) | \ - (1ULL << VIRTIO_BLK_F_MQ)) + (1ULL << VIRTIO_BLK_F_MQ) | \ + (1ULL << VIRTIO_BLK_F_DISCARD) | \ + (1ULL << VIRTIO_BLK_F_WRITE_ZEROES)) #define VDPASIM_BLK_CAPACITY 0x40000 #define VDPASIM_BLK_SIZE_MAX 0x1000 #define VDPASIM_BLK_SEG_MAX 32 +#define VDPASIM_BLK_DWZ_MAX_SECTORS UINT_MAX /* 1 virtqueue, 1 address space, 1 virtqueue group */ #define VDPASIM_BLK_VQ_NUM 1 @@ -193,6 +196,64 @@ static bool vdpasim_blk_handle_req(struct vdpasim *vdpasim, /* nothing to do */ break; + case VIRTIO_BLK_T_DISCARD: + case VIRTIO_BLK_T_WRITE_ZEROES: { + struct virtio_blk_discard_write_zeroes range; + u32 num_sectors, flags; + + if (to_pull != sizeof(range)) { + dev_dbg(&vdpasim->vdpa.dev, + "discard/write_zeroes header len: 0x%zx [expected: 0x%zx]\n", + to_pull, sizeof(range)); + status = VIRTIO_BLK_S_IOERR; + break; + } + + bytes = vringh_iov_pull_iotlb(&vq->vring, &vq->out_iov, &range, + to_pull); + if (bytes < 0) { + dev_dbg(&vdpasim->vdpa.dev, + "vringh_iov_pull_iotlb() error: %zd offset: 0x%llx len: 0x%zx\n", + bytes, offset, to_pull); + status = VIRTIO_BLK_S_IOERR; + break; + } + + sector = le64_to_cpu(range.sector); + offset = sector << SECTOR_SHIFT; + num_sectors = le32_to_cpu(range.num_sectors); + flags = le32_to_cpu(range.flags); + + if (type == VIRTIO_BLK_T_DISCARD && flags != 0) { + dev_dbg(&vdpasim->vdpa.dev, + "discard unexpected flags set - flags: 0x%x\n", + flags); + status = VIRTIO_BLK_S_UNSUPP; + break; + } + + if (type == VIRTIO_BLK_T_WRITE_ZEROES && + flags & ~VIRTIO_BLK_WRITE_ZEROES_FLAG_UNMAP) { + dev_dbg(&vdpasim->vdpa.dev, + "write_zeroes unexpected flags set - flags: 0x%x\n", + flags); + status = VIRTIO_BLK_S_UNSUPP; + break; + } + + if (!vdpasim_blk_check_range(vdpasim, sector, num_sectors, + VDPASIM_BLK_DWZ_MAX_SECTORS)) { + status = VIRTIO_BLK_S_IOERR; + break; + } + + if (type == VIRTIO_BLK_T_WRITE_ZEROES) { + memset(vdpasim->buffer + offset, 0, + num_sectors << SECTOR_SHIFT); + } + + break; + } default: dev_dbg(&vdpasim->vdpa.dev, "Unsupported request type %d\n", type); @@ -281,6 +342,17 @@ static void vdpasim_blk_get_config(struct vdpasim *vdpasim, void *config) blk_config->min_io_size = cpu_to_vdpasim16(vdpasim, 1); blk_config->opt_io_size = cpu_to_vdpasim32(vdpasim, 1); blk_config->blk_size = cpu_to_vdpasim32(vdpasim, SECTOR_SIZE); + /* VIRTIO_BLK_F_DISCARD */ + blk_config->discard_sector_alignment = + cpu_to_vdpasim32(vdpasim, SECTOR_SIZE); + blk_config->max_discard_sectors = + cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS); + blk_config->max_discard_seg = cpu_to_vdpasim32(vdpasim, 1); + /* VIRTIO_BLK_F_WRITE_ZEROES */ + blk_config->max_write_zeroes_sectors = + cpu_to_vdpasim32(vdpasim, VDPASIM_BLK_DWZ_MAX_SECTORS); + blk_config->max_write_zeroes_seg = cpu_to_vdpasim32(vdpasim, 1); + } static void vdpasim_blk_mgmtdev_release(struct device *dev) -- cgit From 93e530d2a1c4c0fcce45e01ae6c5c6287a08d3e3 Mon Sep 17 00:00:00 2001 From: Eli Cohen Date: Thu, 11 Aug 2022 16:40:10 +0300 Subject: vdpa/mlx5: Fix possible uninitialized return value Initialize err local variable to return -EAGAIN if the asid cannot be found thus avoiding returning uninitialized value. Fixes: 8fcd20c30704 ("vdpa/mlx5: Support different address spaces for control and data") Reported-by: Dan Carpenter Signed-off-by: Eli Cohen Message-Id: <20220811134010.952291-1-elic@nvidia.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/mlx5/net/mlx5_vnet.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers') diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index 289cf91b1ff5..ed100a35e596 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -2659,7 +2659,7 @@ static int mlx5_vdpa_set_map(struct vdpa_device *vdev, unsigned int asid, { struct mlx5_vdpa_dev *mvdev = to_mvdev(vdev); struct mlx5_vdpa_net *ndev = to_mlx5_vdpa_ndev(mvdev); - int err; + int err = -EINVAL; down_write(&ndev->reslock); if (mvdev->group2asid[MLX5_VDPA_DATAVQ_GROUP] == asid) { -- cgit