From f090a00df9ecdab5d066b099c1797e0070e27a36 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Tue, 15 Aug 2017 13:14:04 +0800 Subject: drm/i915/gvt: Add emulation for BAR2 (aperture) with normal file RW approach For vfio-pci, if the region support MMAP then it should support both mmap and normal file access. The user-space is free to choose which is being used. For qemu, we just need add 'x-no-mmap=on' for vfio-pci option. Currently GVTg only support MMAP for BAR2. So GVTg will not work when user turn on x-no-mmap option. This patch added file style access for BAR2, aka the GPU aperture. We map the entire aperture partition of active vGPU to kernel space when guest driver try to enable PCI Memory Space. Then we redirect the file RW operation from kvmgt to this mapped area. Link: https://bugzilla.redhat.com/show_bug.cgi?id=1458032 Signed-off-by: Changbin Du Signed-off-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/cfg_space.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'drivers/gpu/drm/i915/gvt/cfg_space.c') diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index 40af17ec6312..be0dd8534993 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -110,13 +110,25 @@ int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, static int map_aperture(struct intel_vgpu *vgpu, bool map) { - u64 first_gfn, first_mfn; + phys_addr_t aperture_pa = vgpu_aperture_pa_base(vgpu); + unsigned long aperture_sz = vgpu_aperture_sz(vgpu); + u64 first_gfn; u64 val; int ret; if (map == vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked) return 0; + if (map) { + vgpu->gm.aperture_va = memremap(aperture_pa, aperture_sz, + MEMREMAP_WC); + if (!vgpu->gm.aperture_va) + return -ENOMEM; + } else { + memunmap(vgpu->gm.aperture_va); + vgpu->gm.aperture_va = NULL; + } + val = vgpu_cfg_space(vgpu)[PCI_BASE_ADDRESS_2]; if (val & PCI_BASE_ADDRESS_MEM_TYPE_64) val = *(u64 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2); @@ -124,14 +136,16 @@ static int map_aperture(struct intel_vgpu *vgpu, bool map) val = *(u32 *)(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_2); first_gfn = (val + vgpu_aperture_offset(vgpu)) >> PAGE_SHIFT; - first_mfn = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT; ret = intel_gvt_hypervisor_map_gfn_to_mfn(vgpu, first_gfn, - first_mfn, - vgpu_aperture_sz(vgpu) >> - PAGE_SHIFT, map); - if (ret) + aperture_pa >> PAGE_SHIFT, + aperture_sz >> PAGE_SHIFT, + map); + if (ret) { + memunmap(vgpu->gm.aperture_va); + vgpu->gm.aperture_va = NULL; return ret; + } vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].tracked = map; return 0; -- cgit From f1751362d6357a90bc6e53176cec715ff2dbed74 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Fri, 18 Aug 2017 17:49:58 +0800 Subject: drm/i915/gvt: Fix incorrect PCI BARs reporting Looking at our virtual PCI device, we can see surprising Region 4 and Region 5. 00:10.0 VGA compatible controller: Intel Corporation Sky Lake Integrated Graphics (rev 06) (prog-if 00 [VGA controller]) .... Region 0: Memory at 140000000 (64-bit, non-prefetchable) [size=16M] Region 2: Memory at 180000000 (64-bit, prefetchable) [size=1G] Region 4: Memory at (32-bit, non-prefetchable) Region 5: Memory at (32-bit, non-prefetchable) Expansion ROM at febd6000 [disabled] [size=2K] The fact is that we only implemented BAR0 and BAR2. Surprising Region 4 and Region 5 are shown because we report their size as 0xffffffff. They should report size 0 instead. BTW, the physical GPU has a PIO BAR. GVTg hasn't implemented PIO access, so we ignored this BAR for vGPU device. v2: fix BAR size value calculation. Link: https://bugzilla.redhat.com/show_bug.cgi?id=1458032 Signed-off-by: Changbin Du Cc: stable@vger.kernel.org Signed-off-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/cfg_space.c | 113 +++++++++++++++-------------------- 1 file changed, 48 insertions(+), 65 deletions(-) (limited to 'drivers/gpu/drm/i915/gvt/cfg_space.c') diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index be0dd8534993..ce71b5878d65 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -211,78 +211,65 @@ static int emulate_pci_command_write(struct intel_vgpu *vgpu, static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset, void *p_data, unsigned int bytes) { - unsigned int bar_index = - (rounddown(offset, 8) % PCI_BASE_ADDRESS_0) / 8; u32 new = *(u32 *)(p_data); bool lo = IS_ALIGNED(offset, 8); u64 size; int ret = 0; bool mmio_enabled = vgpu_cfg_space(vgpu)[PCI_COMMAND] & PCI_COMMAND_MEMORY; + struct intel_vgpu_pci_bar *bars = vgpu->cfg_space.bar; - if (WARN_ON(bar_index >= INTEL_GVT_PCI_BAR_MAX)) - return -EINVAL; - + /* + * Power-up software can determine how much address + * space the device requires by writing a value of + * all 1's to the register and then reading the value + * back. The device will return 0's in all don't-care + * address bits. + */ if (new == 0xffffffff) { - /* - * Power-up software can determine how much address - * space the device requires by writing a value of - * all 1's to the register and then reading the value - * back. The device will return 0's in all don't-care - * address bits. - */ - size = vgpu->cfg_space.bar[bar_index].size; - if (lo) { - new = rounddown(new, size); - } else { - u32 val = vgpu_cfg_space(vgpu)[rounddown(offset, 8)]; - /* for 32bit mode bar it returns all-0 in upper 32 - * bit, for 64bit mode bar it will calculate the - * size with lower 32bit and return the corresponding - * value + switch (offset) { + case PCI_BASE_ADDRESS_0: + case PCI_BASE_ADDRESS_1: + size = ~(bars[INTEL_GVT_PCI_BAR_GTTMMIO].size -1); + intel_vgpu_write_pci_bar(vgpu, offset, + size >> (lo ? 0 : 32), lo); + /* + * Untrap the BAR, since guest hasn't configured a + * valid GPA */ - if (val & PCI_BASE_ADDRESS_MEM_TYPE_64) - new &= (~(size-1)) >> 32; - else - new = 0; - } - /* - * Unmapp & untrap the BAR, since guest hasn't configured a - * valid GPA - */ - switch (bar_index) { - case INTEL_GVT_PCI_BAR_GTTMMIO: ret = trap_gttmmio(vgpu, false); break; - case INTEL_GVT_PCI_BAR_APERTURE: + case PCI_BASE_ADDRESS_2: + case PCI_BASE_ADDRESS_3: + size = ~(bars[INTEL_GVT_PCI_BAR_APERTURE].size -1); + intel_vgpu_write_pci_bar(vgpu, offset, + size >> (lo ? 0 : 32), lo); ret = map_aperture(vgpu, false); break; + default: + /* Unimplemented BARs */ + intel_vgpu_write_pci_bar(vgpu, offset, 0x0, false); } - intel_vgpu_write_pci_bar(vgpu, offset, new, lo); } else { - /* - * Unmapp & untrap the old BAR first, since guest has - * re-configured the BAR - */ - switch (bar_index) { - case INTEL_GVT_PCI_BAR_GTTMMIO: - ret = trap_gttmmio(vgpu, false); + switch (offset) { + case PCI_BASE_ADDRESS_0: + case PCI_BASE_ADDRESS_1: + /* + * Untrap the old BAR first, since guest has + * re-configured the BAR + */ + trap_gttmmio(vgpu, false); + intel_vgpu_write_pci_bar(vgpu, offset, new, lo); + ret = trap_gttmmio(vgpu, mmio_enabled); break; - case INTEL_GVT_PCI_BAR_APERTURE: - ret = map_aperture(vgpu, false); + case PCI_BASE_ADDRESS_2: + case PCI_BASE_ADDRESS_3: + map_aperture(vgpu, false); + intel_vgpu_write_pci_bar(vgpu, offset, new, lo); + ret = map_aperture(vgpu, mmio_enabled); break; - } - intel_vgpu_write_pci_bar(vgpu, offset, new, lo); - /* Track the new BAR */ - if (mmio_enabled) { - switch (bar_index) { - case INTEL_GVT_PCI_BAR_GTTMMIO: - ret = trap_gttmmio(vgpu, true); - break; - case INTEL_GVT_PCI_BAR_APERTURE: - ret = map_aperture(vgpu, true); - break; - } + default: + intel_vgpu_write_pci_bar(vgpu, offset, new, lo); } } return ret; @@ -313,10 +300,7 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, } switch (rounddown(offset, 4)) { - case PCI_BASE_ADDRESS_0: - case PCI_BASE_ADDRESS_1: - case PCI_BASE_ADDRESS_2: - case PCI_BASE_ADDRESS_3: + case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_5: if (WARN_ON(!IS_ALIGNED(offset, 4))) return -EINVAL; return emulate_pci_bar_write(vgpu, offset, p_data, bytes); @@ -358,7 +342,6 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, struct intel_gvt *gvt = vgpu->gvt; const struct intel_gvt_device_info *info = &gvt->device_info; u16 *gmch_ctl; - int i; memcpy(vgpu_cfg_space(vgpu), gvt->firmware.cfg_space, info->cfg_space_size); @@ -385,13 +368,13 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu, */ memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_1, 0, 4); memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_3, 0, 4); + memset(vgpu_cfg_space(vgpu) + PCI_BASE_ADDRESS_4, 0, 8); memset(vgpu_cfg_space(vgpu) + INTEL_GVT_PCI_OPREGION, 0, 4); - for (i = 0; i < INTEL_GVT_MAX_BAR_NUM; i++) { - vgpu->cfg_space.bar[i].size = pci_resource_len( - gvt->dev_priv->drm.pdev, i * 2); - vgpu->cfg_space.bar[i].tracked = false; - } + vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_GTTMMIO].size = + pci_resource_len(gvt->dev_priv->drm.pdev, 0); + vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size = + pci_resource_len(gvt->dev_priv->drm.pdev, 2); } /** -- cgit From 02d578e5edd980eac3fbed15db4d9e5665f22089 Mon Sep 17 00:00:00 2001 From: Changbin Du Date: Wed, 23 Aug 2017 14:08:10 +0800 Subject: drm/i915/gvt: Add support for PCIe extended configuration space IGD is PCIe device and has extended configuration space. Checking the binary dump, we can see we have Caps located out of PCI compatible Configuration Space range. 0x000: 86 80 12 19 17 04 10 00 06 00 00 03 00 00 00 00 0x010: 04 00 00 10 08 00 00 00 0c 00 00 00 08 00 00 00 0x020: 00 00 00 00 00 00 00 00 00 00 00 00 28 10 b9 06 0x030: 00 f8 ff ff 40 00 00 00 00 00 00 00 0b 01 00 00 0x040: 09 70 0c 01 71 26 01 62 c8 00 04 84 00 00 00 00 0x050: c1 00 00 00 39 00 00 00 00 00 00 00 01 00 00 a2 0x060: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00 0x070: 10 ac 92 00 00 80 00 10 00 00 00 00 00 00 00 00 0x080: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x090: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x0a0: 00 00 00 00 00 00 00 00 00 00 00 00 05 d0 01 00 0x0b0: 18 00 e0 fe 00 00 00 00 00 00 00 00 00 00 00 00 0x0c0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x0d0: 01 00 22 00 00 80 00 00 00 00 00 00 00 00 00 00 0x0e0: 00 00 00 00 00 00 00 00 00 80 00 00 00 00 00 00 0x0f0: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 0x100: 1b 00 01 20 02 14 00 00 00 00 00 00 00 00 00 00 ... Currently, we only emulate the PCI compatible Configuration Space. This is okay if we attach vGPU to PCI bus. But when we attach to a PCI Express bus (when Qemu emulates a Intel Q35 chipset which has PCIe slot), it will not work. Extended Configuration Space is required for a PCIe device. This patch extended the virtual configuration space from 256 bytes to 4KB bytes. So we are to be a *real* PCIe device. And for the Extended CapList we keep same to physical GPU. Cc: Laszlo Ersek Tested-by: Laszlo Ersek Signed-off-by: Changbin Du Signed-off-by: Zhenyu Wang --- drivers/gpu/drm/i915/gvt/cfg_space.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/gpu/drm/i915/gvt/cfg_space.c') diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c index ce71b5878d65..ab19545d59a1 100644 --- a/drivers/gpu/drm/i915/gvt/cfg_space.c +++ b/drivers/gpu/drm/i915/gvt/cfg_space.c @@ -101,7 +101,7 @@ int intel_vgpu_emulate_cfg_read(struct intel_vgpu *vgpu, unsigned int offset, if (WARN_ON(bytes > 4)) return -EINVAL; - if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ)) + if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size)) return -EINVAL; memcpy(p_data, vgpu_cfg_space(vgpu) + offset, bytes); @@ -289,7 +289,7 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset, if (WARN_ON(bytes > 4)) return -EINVAL; - if (WARN_ON(offset + bytes > INTEL_GVT_MAX_CFG_SPACE_SZ)) + if (WARN_ON(offset + bytes > vgpu->gvt->device_info.cfg_space_size)) return -EINVAL; /* First check if it's PCI_COMMAND */ -- cgit