summaryrefslogtreecommitdiff
path: root/drivers/gpu/drm/i915/gvt/gtt.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/gpu/drm/i915/gvt/gtt.c')
-rw-r--r--drivers/gpu/drm/i915/gvt/gtt.c523
1 files changed, 274 insertions, 249 deletions
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index c7103dd2d8d5..076d9139edc6 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -33,11 +33,16 @@
*
*/
+#include <drm/drm_print.h>
+
#include "i915_drv.h"
#include "gvt.h"
#include "i915_pvinfo.h"
#include "trace.h"
+#include "gt/intel_gt_regs.h"
+#include <linux/vmalloc.h>
+
#if defined(VERBOSE_DEBUG)
#define gvt_vdbg_mm(fmt, args...) gvt_dbg_mm(fmt, ##args)
#else
@@ -53,75 +58,19 @@ static int preallocated_oos_pages = 8192;
*/
bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size)
{
- if ((!vgpu_gmadr_is_valid(vgpu, addr)) || (size
- && !vgpu_gmadr_is_valid(vgpu, addr + size - 1))) {
- gvt_vgpu_err("invalid range gmadr 0x%llx size 0x%x\n",
- addr, size);
- return false;
- }
- return true;
-}
-
-/* translate a guest gmadr to host gmadr */
-int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr)
-{
- if (WARN(!vgpu_gmadr_is_valid(vgpu, g_addr),
- "invalid guest gmadr %llx\n", g_addr))
- return -EACCES;
-
- if (vgpu_gmadr_is_aperture(vgpu, g_addr))
- *h_addr = vgpu_aperture_gmadr_base(vgpu)
- + (g_addr - vgpu_aperture_offset(vgpu));
- else
- *h_addr = vgpu_hidden_gmadr_base(vgpu)
- + (g_addr - vgpu_hidden_offset(vgpu));
- return 0;
-}
-
-/* translate a host gmadr to guest gmadr */
-int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr)
-{
- if (WARN(!gvt_gmadr_is_valid(vgpu->gvt, h_addr),
- "invalid host gmadr %llx\n", h_addr))
- return -EACCES;
-
- if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))
- *g_addr = vgpu_aperture_gmadr_base(vgpu)
- + (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));
- else
- *g_addr = vgpu_hidden_gmadr_base(vgpu)
- + (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));
- return 0;
-}
-
-int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
- unsigned long *h_index)
-{
- u64 h_addr;
- int ret;
-
- ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT,
- &h_addr);
- if (ret)
- return ret;
-
- *h_index = h_addr >> I915_GTT_PAGE_SHIFT;
- return 0;
-}
-
-int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
- unsigned long *g_index)
-{
- u64 g_addr;
- int ret;
+ if (size == 0)
+ return vgpu_gmadr_is_valid(vgpu, addr);
- ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT,
- &g_addr);
- if (ret)
- return ret;
+ if (vgpu_gmadr_is_aperture(vgpu, addr) &&
+ vgpu_gmadr_is_aperture(vgpu, addr + size - 1))
+ return true;
+ else if (vgpu_gmadr_is_hidden(vgpu, addr) &&
+ vgpu_gmadr_is_hidden(vgpu, addr + size - 1))
+ return true;
- *g_index = g_addr >> I915_GTT_PAGE_SHIFT;
- return 0;
+ gvt_dbg_mm("Invalid ggtt range at 0x%llx, size: 0x%x\n",
+ addr, size);
+ return false;
}
#define gtt_type_is_entry(type) \
@@ -175,7 +124,7 @@ struct gtt_type_table_entry {
.pse_entry_type = pse_type, \
}
-static struct gtt_type_table_entry gtt_type_table[] = {
+static const struct gtt_type_table_entry gtt_type_table[] = {
GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
GTT_TYPE_INVALID,
@@ -254,11 +203,6 @@ static inline int get_next_pt_type(int type)
return gtt_type_table[type].next_pt_type;
}
-static inline int get_pt_type(int type)
-{
- return gtt_type_table[type].pt_type;
-}
-
static inline int get_entry_type(int type)
{
return gtt_type_table[type].entry_type;
@@ -269,24 +213,25 @@ static inline int get_pse_type(int type)
return gtt_type_table[type].pse_entry_type;
}
-static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
+static u64 read_pte64(struct i915_ggtt *ggtt, unsigned long index)
{
- void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+ void __iomem *addr = (gen8_pte_t __iomem *)ggtt->gsm + index;
return readq(addr);
}
-static void ggtt_invalidate(struct drm_i915_private *dev_priv)
+static void ggtt_invalidate(struct intel_gt *gt)
{
- mmio_hw_access_pre(dev_priv);
- I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
- mmio_hw_access_post(dev_priv);
+ intel_wakeref_t wakeref;
+
+ wakeref = mmio_hw_access_pre(gt);
+ intel_uncore_write(gt->uncore, GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+ mmio_hw_access_post(gt, wakeref);
}
-static void write_pte64(struct drm_i915_private *dev_priv,
- unsigned long index, u64 pte)
+static void write_pte64(struct i915_ggtt *ggtt, unsigned long index, u64 pte)
{
- void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
+ void __iomem *addr = (gen8_pte_t __iomem *)ggtt->gsm + index;
writeq(pte, addr);
}
@@ -303,13 +248,13 @@ static inline int gtt_get_entry64(void *pt,
return -EINVAL;
if (hypervisor_access) {
- ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +
+ ret = intel_gvt_read_gpa(vgpu, gpa +
(index << info->gtt_entry_size_shift),
&e->val64, 8);
if (WARN_ON(ret))
return ret;
} else if (!pt) {
- e->val64 = read_pte64(vgpu->gvt->dev_priv, index);
+ e->val64 = read_pte64(vgpu->gvt->gt->ggtt, index);
} else {
e->val64 = *((u64 *)pt + index);
}
@@ -328,13 +273,13 @@ static inline int gtt_set_entry64(void *pt,
return -EINVAL;
if (hypervisor_access) {
- ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +
+ ret = intel_gvt_write_gpa(vgpu, gpa +
(index << info->gtt_entry_size_shift),
&e->val64, 8);
if (WARN_ON(ret))
return ret;
} else if (!pt) {
- write_pte64(vgpu->gvt->dev_priv, index, e->val64);
+ write_pte64(vgpu->gvt->gt->ggtt, index, e->val64);
} else {
*((u64 *)pt + index) = e->val64;
}
@@ -437,17 +382,17 @@ static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
|| e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
return (e->val64 != 0);
else
- return (e->val64 & _PAGE_PRESENT);
+ return (e->val64 & GEN8_PAGE_PRESENT);
}
static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
{
- e->val64 &= ~_PAGE_PRESENT;
+ e->val64 &= ~GEN8_PAGE_PRESENT;
}
static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e)
{
- e->val64 |= _PAGE_PRESENT;
+ e->val64 |= GEN8_PAGE_PRESENT;
}
static bool gen8_gtt_test_64k_splited(struct intel_gvt_gtt_entry *e)
@@ -490,7 +435,7 @@ DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3));
DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff));
DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff));
-static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
+static const struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
.get_entry = gtt_get_entry64,
.set_entry = gtt_set_entry64,
.clear_present = gtt_entry_clear_present,
@@ -507,7 +452,7 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
.set_pfn = gen8_gtt_set_pfn,
};
-static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
+static const struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
.gma_to_ggtt_pte_index = gma_to_ggtt_pte_index,
.gma_to_pte_index = gen8_gma_to_pte_index,
.gma_to_pde_index = gen8_gma_to_pde_index,
@@ -517,7 +462,7 @@ static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
};
/* Update entry type per pse and ips bit. */
-static void update_entry_type_for_real(struct intel_gvt_gtt_pte_ops *pte_ops,
+static void update_entry_type_for_real(const struct intel_gvt_gtt_pte_ops *pte_ops,
struct intel_gvt_gtt_entry *entry, bool ips)
{
switch (entry->type) {
@@ -544,7 +489,7 @@ static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index,
bool guest)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT);
@@ -571,19 +516,13 @@ static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index,
bool guest)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps :
mm->ppgtt_mm.shadow_pdps,
entry, index, false, 0, mm->vgpu);
}
-static inline void ppgtt_set_guest_root_entry(struct intel_vgpu_mm *mm,
- struct intel_gvt_gtt_entry *entry, unsigned long index)
-{
- _ppgtt_set_root_entry(mm, entry, index, true);
-}
-
static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index)
{
@@ -593,7 +532,7 @@ static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm,
static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
@@ -605,7 +544,7 @@ static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm,
static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
@@ -616,7 +555,7 @@ static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm,
static void ggtt_get_host_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
@@ -626,10 +565,19 @@ static void ggtt_get_host_entry(struct intel_vgpu_mm *mm,
static void ggtt_set_host_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *entry, unsigned long index)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops;
+ unsigned long offset = index;
GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT);
+ if (vgpu_gmadr_is_aperture(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) {
+ offset -= (vgpu_aperture_gmadr_base(mm->vgpu) >> PAGE_SHIFT);
+ mm->ggtt_mm.host_ggtt_aperture[offset] = entry->val64;
+ } else if (vgpu_gmadr_is_hidden(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) {
+ offset -= (vgpu_hidden_gmadr_base(mm->vgpu) >> PAGE_SHIFT);
+ mm->ggtt_mm.host_ggtt_hidden[offset] = entry->val64;
+ }
+
pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu);
}
@@ -643,7 +591,7 @@ static inline int ppgtt_spt_get_entry(
bool guest)
{
struct intel_gvt *gvt = spt->vgpu->gvt;
- struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
int ret;
e->type = get_entry_type(type);
@@ -672,7 +620,7 @@ static inline int ppgtt_spt_set_entry(
bool guest)
{
struct intel_gvt *gvt = spt->vgpu->gvt;
- struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
return -EINVAL;
@@ -728,12 +676,12 @@ static int detach_oos_page(struct intel_vgpu *vgpu,
static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt)
{
- struct device *kdev = &spt->vgpu->gvt->dev_priv->drm.pdev->dev;
+ struct device *kdev = spt->vgpu->gvt->gt->i915->drm.dev;
trace_spt_free(spt->vgpu->id, spt, spt->guest_page.type);
dma_unmap_page(kdev, spt->shadow_page.mfn << I915_GTT_PAGE_SHIFT, 4096,
- PCI_DMA_BIDIRECTIONAL);
+ DMA_BIDIRECTIONAL);
radix_tree_delete(&spt->vgpu->gtt.spt_tree, spt->shadow_page.mfn);
@@ -750,14 +698,20 @@ static void ppgtt_free_spt(struct intel_vgpu_ppgtt_spt *spt)
static void ppgtt_free_all_spt(struct intel_vgpu *vgpu)
{
- struct intel_vgpu_ppgtt_spt *spt;
+ struct intel_vgpu_ppgtt_spt *spt, *spn;
struct radix_tree_iter iter;
- void **slot;
+ LIST_HEAD(all_spt);
+ void __rcu **slot;
+ rcu_read_lock();
radix_tree_for_each_slot(slot, &vgpu->gtt.spt_tree, &iter, 0) {
spt = radix_tree_deref_slot(slot);
- ppgtt_free_spt(spt);
+ list_move(&spt->post_shadow_list, &all_spt);
}
+ rcu_read_unlock();
+
+ list_for_each_entry_safe(spt, spn, &all_spt, post_shadow_list)
+ ppgtt_free_spt(spt);
}
static int ppgtt_handle_guest_write_page_table_bytes(
@@ -805,9 +759,9 @@ static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt);
/* Allocate shadow page table without guest page. */
static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt(
- struct intel_vgpu *vgpu, intel_gvt_gtt_type_t type)
+ struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type)
{
- struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
+ struct device *kdev = vgpu->gvt->gt->i915->drm.dev;
struct intel_vgpu_ppgtt_spt *spt = NULL;
dma_addr_t daddr;
int ret;
@@ -831,7 +785,7 @@ retry:
*/
spt->shadow_page.type = type;
daddr = dma_map_page(kdev, spt->shadow_page.page,
- 0, 4096, PCI_DMA_BIDIRECTIONAL);
+ 0, 4096, DMA_BIDIRECTIONAL);
if (dma_mapping_error(kdev, daddr)) {
gvt_vgpu_err("fail to map dma addr\n");
ret = -EINVAL;
@@ -847,7 +801,7 @@ retry:
return spt;
err_unmap_dma:
- dma_unmap_page(kdev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
+ dma_unmap_page(kdev, daddr, PAGE_SIZE, DMA_BIDIRECTIONAL);
err_free_spt:
free_spt(spt);
return ERR_PTR(ret);
@@ -855,7 +809,7 @@ err_free_spt:
/* Allocate shadow page table associated with specific gfn. */
static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_spt_gfn(
- struct intel_vgpu *vgpu, intel_gvt_gtt_type_t type,
+ struct intel_vgpu *vgpu, enum intel_gvt_gtt_type type,
unsigned long gfn, bool guest_pde_ips)
{
struct intel_vgpu_ppgtt_spt *spt;
@@ -928,15 +882,27 @@ static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt);
static int ppgtt_invalidate_spt_by_shadow_entry(struct intel_vgpu *vgpu,
struct intel_gvt_gtt_entry *e)
{
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *s;
- intel_gvt_gtt_type_t cur_pt_type;
+ enum intel_gvt_gtt_type cur_pt_type;
GEM_BUG_ON(!gtt_type_is_pt(get_next_pt_type(e->type)));
if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY
&& e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
- cur_pt_type = get_next_pt_type(e->type) + 1;
+ cur_pt_type = get_next_pt_type(e->type);
+
+ if (!gtt_type_is_pt(cur_pt_type) ||
+ !gtt_type_is_pt(cur_pt_type + 1)) {
+ drm_WARN(&i915->drm, 1,
+ "Invalid page table type, cur_pt_type is: %d\n",
+ cur_pt_type);
+ return -EINVAL;
+ }
+
+ cur_pt_type += 1;
+
if (ops->get_pfn(e) ==
vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)
return 0;
@@ -954,7 +920,7 @@ static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt,
struct intel_gvt_gtt_entry *entry)
{
struct intel_vgpu *vgpu = spt->vgpu;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
unsigned long pfn;
int type;
@@ -965,7 +931,7 @@ static inline void ppgtt_invalidate_pte(struct intel_vgpu_ppgtt_spt *spt,
if (!pfn || pfn == vgpu->gtt.scratch_pt[type].page_mfn)
return;
- intel_gvt_hypervisor_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);
+ intel_gvt_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);
}
static int ppgtt_invalidate_spt(struct intel_vgpu_ppgtt_spt *spt)
@@ -1023,14 +989,14 @@ fail:
static bool vgpu_ips_enabled(struct intel_vgpu *vgpu)
{
- struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+ struct drm_i915_private *dev_priv = vgpu->gvt->gt->i915;
- if (INTEL_GEN(dev_priv) == 9 || INTEL_GEN(dev_priv) == 10) {
+ if (GRAPHICS_VER(dev_priv) == 9) {
u32 ips = vgpu_vreg_t(vgpu, GEN8_GAMW_ECO_DEV_RW_IA) &
GAMW_ECO_ENABLE_64K_IPS_FIELD;
return ips == GAMW_ECO_ENABLE_64K_IPS_FIELD;
- } else if (INTEL_GEN(dev_priv) >= 11) {
+ } else if (GRAPHICS_VER(dev_priv) >= 11) {
/* 64K paging only controlled by IPS bit in PTE now. */
return true;
} else
@@ -1042,7 +1008,7 @@ static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt);
static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)
{
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *spt = NULL;
bool ips = false;
int ret;
@@ -1070,6 +1036,11 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
} else {
int type = get_next_pt_type(we->type);
+ if (!gtt_type_is_pt(type)) {
+ ret = -EINVAL;
+ goto err;
+ }
+
spt = ppgtt_alloc_spt_gfn(vgpu, type, ops->get_pfn(we), ips);
if (IS_ERR(spt)) {
ret = PTR_ERR(spt);
@@ -1091,6 +1062,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_spt_by_guest_entry(
err_free_spt:
ppgtt_free_spt(spt);
+ spt = NULL;
err:
gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
spt, we->val64, we->type);
@@ -1100,7 +1072,7 @@ err:
static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge)
{
- struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;
se->type = ge->type;
se->val64 = ge->val64;
@@ -1112,35 +1084,11 @@ static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
ops->set_pfn(se, s->shadow_page.mfn);
}
-/**
- * Check if can do 2M page
- * @vgpu: target vgpu
- * @entry: target pfn's gtt entry
- *
- * Return 1 if 2MB huge gtt shadowing is possilbe, 0 if miscondition,
- * negtive if found err.
- */
-static int is_2MB_gtt_possible(struct intel_vgpu *vgpu,
- struct intel_gvt_gtt_entry *entry)
-{
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
- unsigned long pfn;
-
- if (!HAS_PAGE_SIZES(vgpu->gvt->dev_priv, I915_GTT_PAGE_SIZE_2M))
- return 0;
-
- pfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, ops->get_pfn(entry));
- if (pfn == INTEL_GVT_INVALID_ADDR)
- return -EINVAL;
-
- return PageTransHuge(pfn_to_page(pfn));
-}
-
static int split_2MB_gtt_entry(struct intel_vgpu *vgpu,
struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
struct intel_gvt_gtt_entry *se)
{
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *sub_spt;
struct intel_gvt_gtt_entry sub_se;
unsigned long start_gfn;
@@ -1157,12 +1105,10 @@ static int split_2MB_gtt_entry(struct intel_vgpu *vgpu,
return PTR_ERR(sub_spt);
for_each_shadow_entry(sub_spt, &sub_se, sub_index) {
- ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,
- start_gfn + sub_index, PAGE_SIZE, &dma_addr);
- if (ret) {
- ppgtt_invalidate_spt(spt);
- return ret;
- }
+ ret = intel_gvt_dma_map_guest_page(vgpu, start_gfn + sub_index,
+ PAGE_SIZE, &dma_addr);
+ if (ret)
+ goto err;
sub_se.val64 = se->val64;
/* Copy the PAT field from PDE. */
@@ -1181,13 +1127,24 @@ static int split_2MB_gtt_entry(struct intel_vgpu *vgpu,
ops->set_pfn(se, sub_spt->shadow_page.mfn);
ppgtt_set_shadow_entry(spt, se, index);
return 0;
+err:
+ /* Cancel the existing address mappings of DMA addr. */
+ for_each_present_shadow_entry(sub_spt, &sub_se, sub_index) {
+ gvt_vdbg_mm("invalidate 4K entry\n");
+ ppgtt_invalidate_pte(sub_spt, &sub_se);
+ }
+ /* Release the new allocated spt. */
+ trace_spt_change(sub_spt->vgpu->id, "release", sub_spt,
+ sub_spt->guest_page.gfn, sub_spt->shadow_page.type);
+ ppgtt_free_spt(sub_spt);
+ return ret;
}
static int split_64KB_gtt_entry(struct intel_vgpu *vgpu,
struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
struct intel_gvt_gtt_entry *se)
{
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_gvt_gtt_entry entry = *se;
unsigned long start_gfn;
dma_addr_t dma_addr;
@@ -1203,8 +1160,8 @@ static int split_64KB_gtt_entry(struct intel_vgpu *vgpu,
ops->set_64k_splited(&entry);
for (i = 0; i < GTT_64K_PTE_STRIDE; i++) {
- ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu,
- start_gfn + i, PAGE_SIZE, &dma_addr);
+ ret = intel_gvt_dma_map_guest_page(vgpu, start_gfn + i,
+ PAGE_SIZE, &dma_addr);
if (ret)
return ret;
@@ -1218,9 +1175,9 @@ static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
struct intel_vgpu_ppgtt_spt *spt, unsigned long index,
struct intel_gvt_gtt_entry *ge)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
struct intel_gvt_gtt_entry se = *ge;
- unsigned long gfn, page_size = PAGE_SIZE;
+ unsigned long gfn;
dma_addr_t dma_addr;
int ret;
@@ -1232,37 +1189,34 @@ static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
switch (ge->type) {
case GTT_TYPE_PPGTT_PTE_4K_ENTRY:
gvt_vdbg_mm("shadow 4K gtt entry\n");
+ ret = intel_gvt_dma_map_guest_page(vgpu, gfn, PAGE_SIZE, &dma_addr);
+ if (ret)
+ return -ENXIO;
break;
case GTT_TYPE_PPGTT_PTE_64K_ENTRY:
gvt_vdbg_mm("shadow 64K gtt entry\n");
/*
* The layout of 64K page is special, the page size is
- * controlled by uper PDE. To be simple, we always split
+ * controlled by upper PDE. To be simple, we always split
* 64K page to smaller 4K pages in shadow PT.
*/
return split_64KB_gtt_entry(vgpu, spt, index, &se);
case GTT_TYPE_PPGTT_PTE_2M_ENTRY:
gvt_vdbg_mm("shadow 2M gtt entry\n");
- ret = is_2MB_gtt_possible(vgpu, ge);
- if (ret == 0)
+ if (!HAS_PAGE_SIZES(vgpu->gvt->gt->i915, I915_GTT_PAGE_SIZE_2M) ||
+ intel_gvt_dma_map_guest_page(vgpu, gfn,
+ I915_GTT_PAGE_SIZE_2M, &dma_addr))
return split_2MB_gtt_entry(vgpu, spt, index, &se);
- else if (ret < 0)
- return ret;
- page_size = I915_GTT_PAGE_SIZE_2M;
break;
case GTT_TYPE_PPGTT_PTE_1G_ENTRY:
gvt_vgpu_err("GVT doesn't support 1GB entry\n");
return -EINVAL;
default:
GEM_BUG_ON(1);
- };
-
- /* direct shadow */
- ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn, page_size,
- &dma_addr);
- if (ret)
- return -ENXIO;
+ return -EINVAL;
+ }
+ /* Successfully shadowed a 4K or 2M page (without splitting). */
pte_ops->set_pfn(&se, dma_addr >> PAGE_SHIFT);
ppgtt_set_shadow_entry(spt, &se, index);
return 0;
@@ -1271,11 +1225,9 @@ static int ppgtt_populate_shadow_entry(struct intel_vgpu *vgpu,
static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt)
{
struct intel_vgpu *vgpu = spt->vgpu;
- struct intel_gvt *gvt = vgpu->gvt;
- struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *s;
struct intel_gvt_gtt_entry se, ge;
- unsigned long gfn, i;
+ unsigned long i;
int ret;
trace_spt_change(spt->vgpu->id, "born", spt,
@@ -1292,13 +1244,6 @@ static int ppgtt_populate_spt(struct intel_vgpu_ppgtt_spt *spt)
ppgtt_generate_shadow_entry(&se, s, &ge);
ppgtt_set_shadow_entry(spt, &se, i);
} else {
- gfn = ops->get_pfn(&ge);
- if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {
- ops->set_pfn(&se, gvt->gtt.scratch_mfn);
- ppgtt_set_shadow_entry(spt, &se, i);
- continue;
- }
-
ret = ppgtt_populate_shadow_entry(vgpu, spt, i, &ge);
if (ret)
goto fail;
@@ -1315,7 +1260,7 @@ static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_ppgtt_spt *spt,
struct intel_gvt_gtt_entry *se, unsigned long index)
{
struct intel_vgpu *vgpu = spt->vgpu;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
int ret;
trace_spt_guest_change(spt->vgpu->id, "remove", spt,
@@ -1396,7 +1341,7 @@ static int sync_oos_page(struct intel_vgpu *vgpu,
{
const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
struct intel_gvt *gvt = vgpu->gvt;
- struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *spt = oos_page->spt;
struct intel_gvt_gtt_entry old, new;
int index;
@@ -1459,7 +1404,7 @@ static int attach_oos_page(struct intel_vgpu_oos_page *oos_page,
struct intel_gvt *gvt = spt->vgpu->gvt;
int ret;
- ret = intel_gvt_hypervisor_read_gpa(spt->vgpu,
+ ret = intel_gvt_read_gpa(spt->vgpu,
spt->guest_page.gfn << I915_GTT_PAGE_SHIFT,
oos_page->mem, I915_GTT_PAGE_SIZE);
if (ret)
@@ -1567,7 +1512,7 @@ static int ppgtt_handle_guest_write_page_table(
{
struct intel_vgpu *vgpu = spt->vgpu;
int type = spt->shadow_page.type;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_gvt_gtt_entry old_se;
int new_present;
int i, ret;
@@ -1684,7 +1629,7 @@ static int ppgtt_handle_guest_write_page_table_bytes(
u64 pa, void *p_data, int bytes)
{
struct intel_vgpu *vgpu = spt->vgpu;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
struct intel_gvt_gtt_entry we, se;
unsigned long index;
@@ -1749,7 +1694,7 @@ static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm)
struct intel_vgpu *vgpu = mm->vgpu;
struct intel_gvt *gvt = vgpu->gvt;
struct intel_gvt_gtt *gtt = &gvt->gtt;
- struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
struct intel_gvt_gtt_entry se;
int index;
@@ -1779,7 +1724,7 @@ static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm)
struct intel_vgpu *vgpu = mm->vgpu;
struct intel_gvt *gvt = vgpu->gvt;
struct intel_gvt_gtt *gtt = &gvt->gtt;
- struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
struct intel_vgpu_ppgtt_spt *spt;
struct intel_gvt_gtt_entry ge, se;
int index, ret;
@@ -1787,6 +1732,9 @@ static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm)
if (mm->ppgtt_mm.shadowed)
return 0;
+ if (!test_bit(INTEL_VGPU_STATUS_ATTACHED, vgpu->status))
+ return -EINVAL;
+
mm->ppgtt_mm.shadowed = true;
for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.guest_pdps); index++) {
@@ -1849,7 +1797,7 @@ static void vgpu_free_mm(struct intel_vgpu_mm *mm)
* Zero on success, negative error code in pointer if failed.
*/
struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,
- intel_gvt_gtt_type_t root_entry_type, u64 pdps[])
+ enum intel_gvt_gtt_type root_entry_type, u64 pdps[])
{
struct intel_gvt *gvt = vgpu->gvt;
struct intel_vgpu_mm *mm;
@@ -1867,6 +1815,7 @@ struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,
INIT_LIST_HEAD(&mm->ppgtt_mm.list);
INIT_LIST_HEAD(&mm->ppgtt_mm.lru_list);
+ INIT_LIST_HEAD(&mm->ppgtt_mm.link);
if (root_entry_type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
mm->ppgtt_mm.guest_pdps[0] = pdps[0];
@@ -1882,7 +1831,11 @@ struct intel_vgpu_mm *intel_vgpu_create_ppgtt_mm(struct intel_vgpu *vgpu,
}
list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head);
+
+ mutex_lock(&gvt->gtt.ppgtt_mm_lock);
list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head);
+ mutex_unlock(&gvt->gtt.ppgtt_mm_lock);
+
return mm;
}
@@ -1906,6 +1859,21 @@ static struct intel_vgpu_mm *intel_vgpu_create_ggtt_mm(struct intel_vgpu *vgpu)
return ERR_PTR(-ENOMEM);
}
+ mm->ggtt_mm.host_ggtt_aperture = vzalloc((vgpu_aperture_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64));
+ if (!mm->ggtt_mm.host_ggtt_aperture) {
+ vfree(mm->ggtt_mm.virtual_ggtt);
+ vgpu_free_mm(mm);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ mm->ggtt_mm.host_ggtt_hidden = vzalloc((vgpu_hidden_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64));
+ if (!mm->ggtt_mm.host_ggtt_hidden) {
+ vfree(mm->ggtt_mm.host_ggtt_aperture);
+ vfree(mm->ggtt_mm.virtual_ggtt);
+ vgpu_free_mm(mm);
+ return ERR_PTR(-ENOMEM);
+ }
+
return mm;
}
@@ -1925,10 +1893,16 @@ void _intel_vgpu_mm_release(struct kref *mm_ref)
if (mm->type == INTEL_GVT_MM_PPGTT) {
list_del(&mm->ppgtt_mm.list);
+
+ mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
list_del(&mm->ppgtt_mm.lru_list);
+ mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
+
invalidate_ppgtt_mm(mm);
} else {
vfree(mm->ggtt_mm.virtual_ggtt);
+ vfree(mm->ggtt_mm.host_ggtt_aperture);
+ vfree(mm->ggtt_mm.host_ggtt_hidden);
}
vgpu_free_mm(mm);
@@ -1942,7 +1916,7 @@ void _intel_vgpu_mm_release(struct kref *mm_ref)
*/
void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
{
- atomic_dec(&mm->pincount);
+ atomic_dec_if_positive(&mm->pincount);
}
/**
@@ -1967,9 +1941,10 @@ int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
if (ret)
return ret;
+ mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
list_move_tail(&mm->ppgtt_mm.lru_list,
&mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head);
-
+ mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock);
}
return 0;
@@ -1980,6 +1955,8 @@ static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt)
struct intel_vgpu_mm *mm;
struct list_head *pos, *n;
+ mutex_lock(&gvt->gtt.ppgtt_mm_lock);
+
list_for_each_safe(pos, n, &gvt->gtt.ppgtt_mm_lru_list_head) {
mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list);
@@ -1987,9 +1964,11 @@ static int reclaim_one_ppgtt_mm(struct intel_gvt *gvt)
continue;
list_del_init(&mm->ppgtt_mm.lru_list);
+ mutex_unlock(&gvt->gtt.ppgtt_mm_lock);
invalidate_ppgtt_mm(mm);
return 1;
}
+ mutex_unlock(&gvt->gtt.ppgtt_mm_lock);
return 0;
}
@@ -2000,7 +1979,7 @@ static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
struct intel_gvt_gtt_entry *e, unsigned long index, bool guest)
{
struct intel_vgpu *vgpu = mm->vgpu;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
struct intel_vgpu_ppgtt_spt *s;
s = intel_vgpu_find_spt_by_mfn(vgpu, ops->get_pfn(e));
@@ -2029,8 +2008,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
{
struct intel_vgpu *vgpu = mm->vgpu;
struct intel_gvt *gvt = vgpu->gvt;
- struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;
- struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;
unsigned long gpa = INTEL_GVT_INVALID_ADDR;
unsigned long gma_index[4];
struct intel_gvt_gtt_entry e;
@@ -2105,11 +2084,20 @@ static int emulate_ggtt_mmio_read(struct intel_vgpu *vgpu,
struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
unsigned long index = off >> info->gtt_entry_size_shift;
+ unsigned long gma;
struct intel_gvt_gtt_entry e;
if (bytes != 4 && bytes != 8)
return -EINVAL;
+ gma = index << I915_GTT_PAGE_SHIFT;
+ if (!intel_gvt_ggtt_validate_range(vgpu,
+ gma, 1 << I915_GTT_PAGE_SHIFT)) {
+ gvt_dbg_mm("read invalid ggtt at 0x%lx\n", gma);
+ memset(p_data, 0, bytes);
+ return 0;
+ }
+
ggtt_get_guest_entry(ggtt_mm, &e, index);
memcpy(p_data, (void *)&e.val64 + (off & (info->gtt_entry_size - 1)),
bytes);
@@ -2117,7 +2105,7 @@ static int emulate_ggtt_mmio_read(struct intel_vgpu *vgpu,
}
/**
- * intel_vgpu_emulate_gtt_mmio_read - emulate GTT MMIO register read
+ * intel_vgpu_emulate_ggtt_mmio_read - emulate GTT MMIO register read
* @vgpu: a vGPU
* @off: register offset
* @p_data: data will be returned to guest
@@ -2145,13 +2133,12 @@ int intel_vgpu_emulate_ggtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
static void ggtt_invalidate_pte(struct intel_vgpu *vgpu,
struct intel_gvt_gtt_entry *entry)
{
- struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
unsigned long pfn;
pfn = pte_ops->get_pfn(entry);
if (pfn != vgpu->gvt->gtt.scratch_mfn)
- intel_gvt_hypervisor_dma_unmap_guest_page(vgpu,
- pfn << PAGE_SHIFT);
+ intel_gvt_dma_unmap_guest_page(vgpu, pfn << PAGE_SHIFT);
}
static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
@@ -2160,10 +2147,11 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
struct intel_gvt *gvt = vgpu->gvt;
const struct intel_gvt_device_info *info = &gvt->device_info;
struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
- struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
unsigned long gma, gfn;
- struct intel_gvt_gtt_entry e, m;
+ struct intel_gvt_gtt_entry e = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE};
+ struct intel_gvt_gtt_entry m = {.val64 = 0, .type = GTT_TYPE_GGTT_PTE};
dma_addr_t dma_addr;
int ret;
struct intel_gvt_partial_pte *partial_pte, *pos, *n;
@@ -2230,23 +2218,16 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
if (!partial_update && (ops->test_present(&e))) {
gfn = ops->get_pfn(&e);
- m = e;
-
- /* one PTE update may be issued in multiple writes and the
- * first write may not construct a valid gfn
- */
- if (!intel_gvt_hypervisor_is_valid_gfn(vgpu, gfn)) {
- ops->set_pfn(&m, gvt->gtt.scratch_mfn);
- goto out;
- }
+ m.val64 = e.val64;
+ m.type = e.type;
- ret = intel_gvt_hypervisor_dma_map_guest_page(vgpu, gfn,
- PAGE_SIZE, &dma_addr);
+ ret = intel_gvt_dma_map_guest_page(vgpu, gfn, PAGE_SIZE,
+ &dma_addr);
if (ret) {
gvt_vgpu_err("fail to populate guest ggtt entry\n");
/* guest driver may read/write the entry when partial
* update the entry in this situation p2m will fail
- * settting the shadow entry to point to a scratch page
+ * setting the shadow entry to point to a scratch page
*/
ops->set_pfn(&m, gvt->gtt.scratch_mfn);
} else
@@ -2256,14 +2237,13 @@ static int emulate_ggtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
ops->clear_present(&m);
}
-out:
ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
ggtt_get_host_entry(ggtt_mm, &e, g_gtt_index);
ggtt_invalidate_pte(vgpu, &e);
ggtt_set_host_entry(ggtt_mm, &m, g_gtt_index);
- ggtt_invalidate(gvt->dev_priv);
+ ggtt_invalidate(gvt->gt);
return 0;
}
@@ -2284,28 +2264,45 @@ int intel_vgpu_emulate_ggtt_mmio_write(struct intel_vgpu *vgpu,
{
const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
int ret;
+ struct intel_vgpu_submission *s = &vgpu->submission;
+ struct intel_engine_cs *engine;
+ int i;
if (bytes != 4 && bytes != 8)
return -EINVAL;
off -= info->gtt_start_offset;
ret = emulate_ggtt_mmio_write(vgpu, off, p_data, bytes);
+
+ /* if ggtt of last submitted context is written,
+ * that context is probably got unpinned.
+ * Set last shadowed ctx to invalid.
+ */
+ for_each_engine(engine, vgpu->gvt->gt, i) {
+ if (!s->last_ctx[i].valid)
+ continue;
+
+ if (s->last_ctx[i].lrca == (off >> info->gtt_entry_size_shift))
+ s->last_ctx[i].valid = false;
+ }
return ret;
}
static int alloc_scratch_pages(struct intel_vgpu *vgpu,
- intel_gvt_gtt_type_t type)
+ enum intel_gvt_gtt_type type)
{
+ struct drm_i915_private *i915 = vgpu->gvt->gt->i915;
struct intel_vgpu_gtt *gtt = &vgpu->gtt;
- struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
int page_entry_num = I915_GTT_PAGE_SIZE >>
vgpu->gvt->device_info.gtt_entry_size_shift;
void *scratch_pt;
int i;
- struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
+ struct device *dev = vgpu->gvt->gt->i915->drm.dev;
dma_addr_t daddr;
- if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX))
+ if (drm_WARN_ON(&i915->drm,
+ type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX))
return -EINVAL;
scratch_pt = (void *)get_zeroed_page(GFP_KERNEL);
@@ -2314,8 +2311,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
return -ENOMEM;
}
- daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0,
- 4096, PCI_DMA_BIDIRECTIONAL);
+ daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0, 4096, DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, daddr)) {
gvt_vgpu_err("fail to dmamap scratch_pt\n");
__free_page(virt_to_page(scratch_pt));
@@ -2345,7 +2341,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
/* The entry parameters like present/writeable/cache type
* set to the same as i915's scratch page tree.
*/
- se.val64 |= _PAGE_PRESENT | _PAGE_RW;
+ se.val64 |= GEN8_PAGE_PRESENT | GEN8_PAGE_RW;
if (type == GTT_TYPE_PPGTT_PDE_PT)
se.val64 |= PPAT_CACHED;
@@ -2359,14 +2355,14 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
static int release_scratch_page_tree(struct intel_vgpu *vgpu)
{
int i;
- struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
+ struct device *dev = vgpu->gvt->gt->i915->drm.dev;
dma_addr_t daddr;
for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
if (vgpu->gtt.scratch_pt[i].page != NULL) {
daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<
I915_GTT_PAGE_SHIFT);
- dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
+ dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL);
__free_page(vgpu->gtt.scratch_pt[i].page);
vgpu->gtt.scratch_pt[i].page = NULL;
vgpu->gtt.scratch_pt[i].page_mfn = 0;
@@ -2426,7 +2422,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
return create_scratch_page_tree(vgpu);
}
-static void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu)
+void intel_vgpu_destroy_all_ppgtt_mm(struct intel_vgpu *vgpu)
{
struct list_head *pos, *n;
struct intel_vgpu_mm *mm;
@@ -2489,6 +2485,7 @@ static void clean_spt_oos(struct intel_gvt *gvt)
list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {
oos_page = container_of(pos, struct intel_vgpu_oos_page, list);
list_del(&oos_page->list);
+ free_page((unsigned long)oos_page->mem);
kfree(oos_page);
}
}
@@ -2509,6 +2506,12 @@ static int setup_spt_oos(struct intel_gvt *gvt)
ret = -ENOMEM;
goto fail;
}
+ oos_page->mem = (void *)__get_free_pages(GFP_KERNEL, 0);
+ if (!oos_page->mem) {
+ ret = -ENOMEM;
+ kfree(oos_page);
+ goto fail;
+ }
INIT_LIST_HEAD(&oos_page->list);
INIT_LIST_HEAD(&oos_page->vm_list);
@@ -2572,7 +2575,7 @@ struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
* Zero on success, negative error code if failed.
*/
struct intel_vgpu_mm *intel_vgpu_get_ppgtt_mm(struct intel_vgpu *vgpu,
- intel_gvt_gtt_type_t root_entry_type, u64 pdps[])
+ enum intel_gvt_gtt_type root_entry_type, u64 pdps[])
{
struct intel_vgpu_mm *mm;
@@ -2624,7 +2627,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
{
int ret;
void *page;
- struct device *dev = &gvt->dev_priv->drm.pdev->dev;
+ struct device *dev = gvt->gt->i915->drm.dev;
dma_addr_t daddr;
gvt_dbg_core("init gtt\n");
@@ -2639,7 +2642,7 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
}
daddr = dma_map_page(dev, virt_to_page(page), 0,
- 4096, PCI_DMA_BIDIRECTIONAL);
+ 4096, DMA_BIDIRECTIONAL);
if (dma_mapping_error(dev, daddr)) {
gvt_err("fail to dmamap scratch ggtt page\n");
__free_page(virt_to_page(page));
@@ -2653,12 +2656,13 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
ret = setup_spt_oos(gvt);
if (ret) {
gvt_err("fail to initialize SPT oos\n");
- dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
+ dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL);
__free_page(gvt->gtt.scratch_page);
return ret;
}
}
INIT_LIST_HEAD(&gvt->gtt.ppgtt_mm_lru_list_head);
+ mutex_init(&gvt->gtt.ppgtt_mm_lock);
return 0;
}
@@ -2666,17 +2670,17 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
* intel_gvt_clean_gtt - clean up mm components of a GVT device
* @gvt: GVT device
*
- * This function is called at the driver unloading stage, to clean up the
+ * This function is called at the driver unloading stage, to clean up
* the mm components of a GVT device.
*
*/
void intel_gvt_clean_gtt(struct intel_gvt *gvt)
{
- struct device *dev = &gvt->dev_priv->drm.pdev->dev;
+ struct device *dev = gvt->gt->i915->drm.dev;
dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<
I915_GTT_PAGE_SHIFT);
- dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
+ dma_unmap_page(dev, daddr, 4096, DMA_BIDIRECTIONAL);
__free_page(gvt->gtt.scratch_page);
@@ -2699,7 +2703,9 @@ void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu)
list_for_each_safe(pos, n, &vgpu->gtt.ppgtt_mm_list_head) {
mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list);
if (mm->type == INTEL_GVT_MM_PPGTT) {
+ mutex_lock(&vgpu->gvt->gtt.ppgtt_mm_lock);
list_del_init(&mm->ppgtt_mm.lru_list);
+ mutex_unlock(&vgpu->gvt->gtt.ppgtt_mm_lock);
if (mm->ppgtt_mm.shadowed)
invalidate_ppgtt_mm(mm);
}
@@ -2718,8 +2724,7 @@ void intel_vgpu_invalidate_ppgtt(struct intel_vgpu *vgpu)
void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old)
{
struct intel_gvt *gvt = vgpu->gvt;
- struct drm_i915_private *dev_priv = gvt->dev_priv;
- struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
+ const struct intel_gvt_gtt_pte_ops *pte_ops = vgpu->gvt->gtt.pte_ops;
struct intel_gvt_gtt_entry entry = {.type = GTT_TYPE_GGTT_PTE};
struct intel_gvt_gtt_entry old_entry;
u32 index;
@@ -2748,23 +2753,43 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu, bool invalidate_old)
ggtt_set_host_entry(vgpu->gtt.ggtt_mm, &entry, index++);
}
- ggtt_invalidate(dev_priv);
+ ggtt_invalidate(gvt->gt);
}
/**
- * intel_vgpu_reset_gtt - reset the all GTT related status
- * @vgpu: a vGPU
+ * intel_gvt_restore_ggtt - restore all vGPU's ggtt entries
+ * @gvt: intel gvt device
*
- * This function is called from vfio core to reset reset all
- * GTT related status, including GGTT, PPGTT, scratch page.
+ * This function is called at driver resume stage to restore
+ * GGTT entries of every vGPU.
*
*/
-void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
+void intel_gvt_restore_ggtt(struct intel_gvt *gvt)
{
- /* Shadow pages are only created when there is no page
- * table tracking data, so remove page tracking data after
- * removing the shadow pages.
- */
- intel_vgpu_destroy_all_ppgtt_mm(vgpu);
- intel_vgpu_reset_ggtt(vgpu, true);
+ struct intel_vgpu *vgpu;
+ struct intel_vgpu_mm *mm;
+ int id;
+ gen8_pte_t pte;
+ u32 idx, num_low, num_hi, offset;
+
+ /* Restore dirty host ggtt for all vGPUs */
+ idr_for_each_entry(&(gvt)->vgpu_idr, vgpu, id) {
+ mm = vgpu->gtt.ggtt_mm;
+
+ num_low = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT;
+ offset = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;
+ for (idx = 0; idx < num_low; idx++) {
+ pte = mm->ggtt_mm.host_ggtt_aperture[idx];
+ if (pte & GEN8_PAGE_PRESENT)
+ write_pte64(vgpu->gvt->gt->ggtt, offset + idx, pte);
+ }
+
+ num_hi = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT;
+ offset = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT;
+ for (idx = 0; idx < num_hi; idx++) {
+ pte = mm->ggtt_mm.host_ggtt_hidden[idx];
+ if (pte & GEN8_PAGE_PRESENT)
+ write_pte64(vgpu->gvt->gt->ggtt, offset + idx, pte);
+ }
+ }
}
generated by cgit (git 2.34.1) at 2025-12-25 08:31:15 +0000