diff options
Diffstat (limited to 'drivers/gpu/drm/i915')
| -rw-r--r-- | drivers/gpu/drm/i915/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/display/intel_dp_link_training.c | 71 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gem/i915_gem_pages.c | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gt/gen7_renderclear.c | 5 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/gvt.c | 124 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/gvt.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/hypercall.h | 2 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/kvmgt.c | 122 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/gvt/mpt.h | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_gem.c | 11 | ||||
| -rw-r--r-- | drivers/gpu/drm/i915/i915_mm.c | 73 |
11 files changed, 212 insertions, 206 deletions
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig index 69f57ca9c68d..93f4d059fc89 100644 --- a/drivers/gpu/drm/i915/Kconfig +++ b/drivers/gpu/drm/i915/Kconfig @@ -102,7 +102,6 @@ config DRM_I915_GVT bool "Enable Intel GVT-g graphics virtualization host support" depends on DRM_I915 depends on 64BIT - depends on VFIO_MDEV=y || VFIO_MDEV=DRM_I915 default n help Choose this option if you want to enable Intel GVT-g graphics diff --git a/drivers/gpu/drm/i915/display/intel_dp_link_training.c b/drivers/gpu/drm/i915/display/intel_dp_link_training.c index 02a003fd48fb..50cae0198a3d 100644 --- a/drivers/gpu/drm/i915/display/intel_dp_link_training.c +++ b/drivers/gpu/drm/i915/display/intel_dp_link_training.c @@ -128,49 +128,13 @@ intel_dp_set_lttpr_transparent_mode(struct intel_dp *intel_dp, bool enable) return drm_dp_dpcd_write(&intel_dp->aux, DP_PHY_REPEATER_MODE, &val, 1) == 1; } -/** - * intel_dp_init_lttpr_and_dprx_caps - detect LTTPR and DPRX caps, init the LTTPR link training mode - * @intel_dp: Intel DP struct - * - * Read the LTTPR common and DPRX capabilities and switch to non-transparent - * link training mode if any is detected and read the PHY capabilities for all - * detected LTTPRs. In case of an LTTPR detection error or if the number of - * LTTPRs is more than is supported (8), fall back to the no-LTTPR, - * transparent mode link training mode. - * - * Returns: - * >0 if LTTPRs were detected and the non-transparent LT mode was set. The - * DPRX capabilities are read out. - * 0 if no LTTPRs or more than 8 LTTPRs were detected or in case of a - * detection failure and the transparent LT mode was set. The DPRX - * capabilities are read out. - * <0 Reading out the DPRX capabilities failed. - */ -int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) +static int intel_dp_init_lttpr(struct intel_dp *intel_dp) { int lttpr_count; - bool ret; int i; - ret = intel_dp_read_lttpr_common_caps(intel_dp); - - /* The DPTX shall read the DPRX caps after LTTPR detection. */ - if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd)) { - intel_dp_reset_lttpr_common_caps(intel_dp); - return -EIO; - } - - if (!ret) - return 0; - - /* - * The 0xF0000-0xF02FF range is only valid if the DPCD revision is - * at least 1.4. - */ - if (intel_dp->dpcd[DP_DPCD_REV] < 0x14) { - intel_dp_reset_lttpr_common_caps(intel_dp); + if (!intel_dp_read_lttpr_common_caps(intel_dp)) return 0; - } lttpr_count = drm_dp_lttpr_count(intel_dp->lttpr_common_caps); /* @@ -211,6 +175,37 @@ int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) return lttpr_count; } + +/** + * intel_dp_init_lttpr_and_dprx_caps - detect LTTPR and DPRX caps, init the LTTPR link training mode + * @intel_dp: Intel DP struct + * + * Read the LTTPR common and DPRX capabilities and switch to non-transparent + * link training mode if any is detected and read the PHY capabilities for all + * detected LTTPRs. In case of an LTTPR detection error or if the number of + * LTTPRs is more than is supported (8), fall back to the no-LTTPR, + * transparent mode link training mode. + * + * Returns: + * >0 if LTTPRs were detected and the non-transparent LT mode was set. The + * DPRX capabilities are read out. + * 0 if no LTTPRs or more than 8 LTTPRs were detected or in case of a + * detection failure and the transparent LT mode was set. The DPRX + * capabilities are read out. + * <0 Reading out the DPRX capabilities failed. + */ +int intel_dp_init_lttpr_and_dprx_caps(struct intel_dp *intel_dp) +{ + int lttpr_count = intel_dp_init_lttpr(intel_dp); + + /* The DPTX shall read the DPRX caps after LTTPR detection. */ + if (drm_dp_read_dpcd_caps(&intel_dp->aux, intel_dp->dpcd)) { + intel_dp_reset_lttpr_common_caps(intel_dp); + return -EIO; + } + + return lttpr_count; +} EXPORT_SYMBOL(intel_dp_init_lttpr_and_dprx_caps); static u8 dp_voltage_max(u8 preemph) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pages.c b/drivers/gpu/drm/i915/gem/i915_gem_pages.c index aed8a37ccdc9..7361971c177d 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pages.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pages.c @@ -63,6 +63,8 @@ void __i915_gem_object_set_pages(struct drm_i915_gem_object *obj, i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) { GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj)); i915_gem_object_set_tiling_quirk(obj); + GEM_BUG_ON(!list_empty(&obj->mm.link)); + atomic_inc(&obj->mm.shrink_pin); shrinkable = false; } diff --git a/drivers/gpu/drm/i915/gt/gen7_renderclear.c b/drivers/gpu/drm/i915/gt/gen7_renderclear.c index de575fdb033f..21f08e53889c 100644 --- a/drivers/gpu/drm/i915/gt/gen7_renderclear.c +++ b/drivers/gpu/drm/i915/gt/gen7_renderclear.c @@ -397,7 +397,10 @@ static void emit_batch(struct i915_vma * const vma, gen7_emit_pipeline_invalidate(&cmds); batch_add(&cmds, MI_LOAD_REGISTER_IMM(2)); batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7)); - batch_add(&cmds, 0xffff0000); + batch_add(&cmds, 0xffff0000 | + ((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ? + HIZ_RAW_STALL_OPT_DISABLE : + 0)); batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1)); batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); gen7_emit_pipeline_invalidate(&cmds); diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index e7c2babcee8b..cbac409f6c8a 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -46,118 +46,6 @@ static const char * const supported_hypervisors[] = { [INTEL_GVT_HYPERVISOR_KVM] = "KVM", }; -static struct intel_vgpu_type * -intel_gvt_find_vgpu_type(struct intel_gvt *gvt, unsigned int type_group_id) -{ - if (WARN_ON(type_group_id >= gvt->num_types)) - return NULL; - return &gvt->types[type_group_id]; -} - -static ssize_t available_instances_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, - char *buf) -{ - struct intel_vgpu_type *type; - unsigned int num = 0; - void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype)); - if (!type) - num = 0; - else - num = type->avail_instance; - - return sprintf(buf, "%u\n", num); -} - -static ssize_t device_api_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); -} - -static ssize_t description_show(struct mdev_type *mtype, - struct mdev_type_attribute *attr, char *buf) -{ - struct intel_vgpu_type *type; - void *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; - - type = intel_gvt_find_vgpu_type(gvt, mtype_get_type_group_id(mtype)); - if (!type) - return 0; - - return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" - "fence: %d\nresolution: %s\n" - "weight: %d\n", - BYTES_TO_MB(type->low_gm_size), - BYTES_TO_MB(type->high_gm_size), - type->fence, vgpu_edid_str(type->resolution), - type->weight); -} - -static MDEV_TYPE_ATTR_RO(available_instances); -static MDEV_TYPE_ATTR_RO(device_api); -static MDEV_TYPE_ATTR_RO(description); - -static struct attribute *gvt_type_attrs[] = { - &mdev_type_attr_available_instances.attr, - &mdev_type_attr_device_api.attr, - &mdev_type_attr_description.attr, - NULL, -}; - -static struct attribute_group *gvt_vgpu_type_groups[] = { - [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, -}; - -static bool intel_get_gvt_attrs(struct attribute_group ***intel_vgpu_type_groups) -{ - *intel_vgpu_type_groups = gvt_vgpu_type_groups; - return true; -} - -static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i, j; - struct intel_vgpu_type *type; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - type = &gvt->types[i]; - - group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); - if (WARN_ON(!group)) - goto unwind; - - group->name = type->name; - group->attrs = gvt_type_attrs; - gvt_vgpu_type_groups[i] = group; - } - - return 0; - -unwind: - for (j = 0; j < i; j++) { - group = gvt_vgpu_type_groups[j]; - kfree(group); - } - - return -ENOMEM; -} - -static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) -{ - int i; - struct attribute_group *group; - - for (i = 0; i < gvt->num_types; i++) { - group = gvt_vgpu_type_groups[i]; - gvt_vgpu_type_groups[i] = NULL; - kfree(group); - } -} - static const struct intel_gvt_ops intel_gvt_ops = { .emulate_cfg_read = intel_vgpu_emulate_cfg_read, .emulate_cfg_write = intel_vgpu_emulate_cfg_write, @@ -169,8 +57,6 @@ static const struct intel_gvt_ops intel_gvt_ops = { .vgpu_reset = intel_gvt_reset_vgpu, .vgpu_activate = intel_gvt_activate_vgpu, .vgpu_deactivate = intel_gvt_deactivate_vgpu, - .gvt_find_vgpu_type = intel_gvt_find_vgpu_type, - .get_gvt_attrs = intel_get_gvt_attrs, .vgpu_query_plane = intel_vgpu_query_plane, .vgpu_get_dmabuf = intel_vgpu_get_dmabuf, .write_protect_handler = intel_vgpu_page_track_handler, @@ -274,7 +160,6 @@ void intel_gvt_clean_device(struct drm_i915_private *i915) return; intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); - intel_gvt_cleanup_vgpu_type_groups(gvt); intel_gvt_clean_vgpu_types(gvt); intel_gvt_debugfs_clean(gvt); @@ -363,12 +248,6 @@ int intel_gvt_init_device(struct drm_i915_private *i915) if (ret) goto out_clean_thread; - ret = intel_gvt_init_vgpu_type_groups(gvt); - if (ret) { - gvt_err("failed to init vgpu type groups: %d\n", ret); - goto out_clean_types; - } - vgpu = intel_gvt_create_idle_vgpu(gvt); if (IS_ERR(vgpu)) { ret = PTR_ERR(vgpu); @@ -454,7 +333,8 @@ EXPORT_SYMBOL_GPL(intel_gvt_register_hypervisor); void intel_gvt_unregister_hypervisor(void) { - intel_gvt_hypervisor_host_exit(intel_gvt_host.dev); + void *gvt = (void *)kdev_to_i915(intel_gvt_host.dev)->gvt; + intel_gvt_hypervisor_host_exit(intel_gvt_host.dev, gvt); module_put(THIS_MODULE); } EXPORT_SYMBOL_GPL(intel_gvt_unregister_hypervisor); diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 88ab360fcb31..0c0615602343 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -574,9 +574,6 @@ struct intel_gvt_ops { void (*vgpu_reset)(struct intel_vgpu *); void (*vgpu_activate)(struct intel_vgpu *); void (*vgpu_deactivate)(struct intel_vgpu *); - struct intel_vgpu_type *(*gvt_find_vgpu_type)( - struct intel_gvt *gvt, unsigned int type_group_id); - bool (*get_gvt_attrs)(struct attribute_group ***intel_vgpu_type_groups); int (*vgpu_query_plane)(struct intel_vgpu *vgpu, void *); int (*vgpu_get_dmabuf)(struct intel_vgpu *vgpu, unsigned int); int (*write_protect_handler)(struct intel_vgpu *, u64, void *, diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h index b79da5124f83..f33e3cbd0439 100644 --- a/drivers/gpu/drm/i915/gvt/hypercall.h +++ b/drivers/gpu/drm/i915/gvt/hypercall.h @@ -49,7 +49,7 @@ enum hypervisor_type { struct intel_gvt_mpt { enum hypervisor_type type; int (*host_init)(struct device *dev, void *gvt, const void *ops); - void (*host_exit)(struct device *dev); + void (*host_exit)(struct device *dev, void *gvt); int (*attach_vgpu)(void *vgpu, unsigned long *handle); void (*detach_vgpu)(void *vgpu); int (*inject_msi)(unsigned long handle, u32 addr, u16 data); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 65ff43cfc0f7..48b4d4cf805d 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -144,6 +144,104 @@ static inline bool handle_valid(unsigned long handle) return !!(handle & ~0xff); } +static ssize_t available_instances_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, + char *buf) +{ + struct intel_vgpu_type *type; + unsigned int num = 0; + struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; + + type = &gvt->types[mtype_get_type_group_id(mtype)]; + if (!type) + num = 0; + else + num = type->avail_instance; + + return sprintf(buf, "%u\n", num); +} + +static ssize_t device_api_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING); +} + +static ssize_t description_show(struct mdev_type *mtype, + struct mdev_type_attribute *attr, char *buf) +{ + struct intel_vgpu_type *type; + struct intel_gvt *gvt = kdev_to_i915(mtype_get_parent_dev(mtype))->gvt; + + type = &gvt->types[mtype_get_type_group_id(mtype)]; + if (!type) + return 0; + + return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n" + "fence: %d\nresolution: %s\n" + "weight: %d\n", + BYTES_TO_MB(type->low_gm_size), + BYTES_TO_MB(type->high_gm_size), + type->fence, vgpu_edid_str(type->resolution), + type->weight); +} + +static MDEV_TYPE_ATTR_RO(available_instances); +static MDEV_TYPE_ATTR_RO(device_api); +static MDEV_TYPE_ATTR_RO(description); + +static struct attribute *gvt_type_attrs[] = { + &mdev_type_attr_available_instances.attr, + &mdev_type_attr_device_api.attr, + &mdev_type_attr_description.attr, + NULL, +}; + +static struct attribute_group *gvt_vgpu_type_groups[] = { + [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL, +}; + +static int intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i, j; + struct intel_vgpu_type *type; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + type = &gvt->types[i]; + + group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL); + if (!group) + goto unwind; + + group->name = type->name; + group->attrs = gvt_type_attrs; + gvt_vgpu_type_groups[i] = group; + } + + return 0; + +unwind: + for (j = 0; j < i; j++) { + group = gvt_vgpu_type_groups[j]; + kfree(group); + } + + return -ENOMEM; +} + +static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt) +{ + int i; + struct attribute_group *group; + + for (i = 0; i < gvt->num_types; i++) { + group = gvt_vgpu_type_groups[i]; + gvt_vgpu_type_groups[i] = NULL; + kfree(group); + } +} + static int kvmgt_guest_init(struct mdev_device *mdev); static void intel_vgpu_release_work(struct work_struct *work); static bool kvmgt_guest_exit(struct kvmgt_guest_info *info); @@ -694,14 +792,13 @@ static int intel_vgpu_create(struct mdev_device *mdev) struct intel_vgpu *vgpu = NULL; struct intel_vgpu_type *type; struct device *pdev; - void *gvt; + struct intel_gvt *gvt; int ret; pdev = mdev_parent_dev(mdev); gvt = kdev_to_i915(pdev)->gvt; - type = intel_gvt_ops->gvt_find_vgpu_type(gvt, - mdev_get_type_group_id(mdev)); + type = &gvt->types[mdev_get_type_group_id(mdev)]; if (!type) { ret = -EINVAL; goto out; @@ -1667,19 +1764,26 @@ static struct mdev_parent_ops intel_vgpu_ops = { static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops) { - struct attribute_group **kvm_vgpu_type_groups; + int ret; + + ret = intel_gvt_init_vgpu_type_groups((struct intel_gvt *)gvt); + if (ret) + return ret; intel_gvt_ops = ops; - if (!intel_gvt_ops->get_gvt_attrs(&kvm_vgpu_type_groups)) - return -EFAULT; - intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups; + intel_vgpu_ops.supported_type_groups = gvt_vgpu_type_groups; - return mdev_register_device(dev, &intel_vgpu_ops); + ret = mdev_register_device(dev, &intel_vgpu_ops); + if (ret) + intel_gvt_cleanup_vgpu_type_groups((struct intel_gvt *)gvt); + + return ret; } -static void kvmgt_host_exit(struct device *dev) +static void kvmgt_host_exit(struct device *dev, void *gvt) { mdev_unregister_device(dev); + intel_gvt_cleanup_vgpu_type_groups((struct intel_gvt *)gvt); } static int kvmgt_page_track_add(unsigned long handle, u64 gfn) diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h index 550a456e936f..e6c5a792a49a 100644 --- a/drivers/gpu/drm/i915/gvt/mpt.h +++ b/drivers/gpu/drm/i915/gvt/mpt.h @@ -63,13 +63,13 @@ static inline int intel_gvt_hypervisor_host_init(struct device *dev, /** * intel_gvt_hypervisor_host_exit - exit GVT-g host side */ -static inline void intel_gvt_hypervisor_host_exit(struct device *dev) +static inline void intel_gvt_hypervisor_host_exit(struct device *dev, void *gvt) { /* optional to provide */ if (!intel_gvt_host.mpt->host_exit) return; - intel_gvt_host.mpt->host_exit(dev); + intel_gvt_host.mpt->host_exit(dev, gvt); } /** diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index b23f58e94cfb..b3cedd20f365 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -999,12 +999,11 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, obj->mm.madv = args->madv; if (i915_gem_object_has_pages(obj)) { - struct list_head *list; + unsigned long flags; - if (i915_gem_object_is_shrinkable(obj)) { - unsigned long flags; - - spin_lock_irqsave(&i915->mm.obj_lock, flags); + spin_lock_irqsave(&i915->mm.obj_lock, flags); + if (!list_empty(&obj->mm.link)) { + struct list_head *list; if (obj->mm.madv != I915_MADV_WILLNEED) list = &i915->mm.purge_list; @@ -1012,8 +1011,8 @@ i915_gem_madvise_ioctl(struct drm_device *dev, void *data, list = &i915->mm.shrink_list; list_move_tail(&obj->mm.link, list); - spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } + spin_unlock_irqrestore(&i915->mm.obj_lock, flags); } /* if the object is no longer attached, discard its backing storage */ diff --git a/drivers/gpu/drm/i915/i915_mm.c b/drivers/gpu/drm/i915/i915_mm.c index 4c8cd08c672d..9a777b0ff59b 100644 --- a/drivers/gpu/drm/i915/i915_mm.c +++ b/drivers/gpu/drm/i915/i915_mm.c @@ -28,10 +28,46 @@ #include "i915_drv.h" -#define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP) +struct remap_pfn { + struct mm_struct *mm; + unsigned long pfn; + pgprot_t prot; + + struct sgt_iter sgt; + resource_size_t iobase; +}; #define use_dma(io) ((io) != -1) +static inline unsigned long sgt_pfn(const struct remap_pfn *r) +{ + if (use_dma(r->iobase)) + return (r->sgt.dma + r->sgt.curr + r->iobase) >> PAGE_SHIFT; + else + return r->sgt.pfn + (r->sgt.curr >> PAGE_SHIFT); +} + +static int remap_sg(pte_t *pte, unsigned long addr, void *data) +{ + struct remap_pfn *r = data; + + if (GEM_WARN_ON(!r->sgt.sgp)) + return -EINVAL; + + /* Special PTE are not associated with any struct page */ + set_pte_at(r->mm, addr, pte, + pte_mkspecial(pfn_pte(sgt_pfn(r), r->prot))); + r->pfn++; /* track insertions in case we need to unwind later */ + + r->sgt.curr += PAGE_SIZE; + if (r->sgt.curr >= r->sgt.max) + r->sgt = __sgt_iter(__sg_next(r->sgt.sgp), use_dma(r->iobase)); + + return 0; +} + +#define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP) + /** * remap_io_sg - remap an IO mapping to userspace * @vma: user vma to map to @@ -46,7 +82,12 @@ int remap_io_sg(struct vm_area_struct *vma, unsigned long addr, unsigned long size, struct scatterlist *sgl, resource_size_t iobase) { - unsigned long pfn, len, remapped = 0; + struct remap_pfn r = { + .mm = vma->vm_mm, + .prot = vma->vm_page_prot, + .sgt = __sgt_iter(sgl, use_dma(iobase)), + .iobase = iobase, + }; int err; /* We rely on prevalidation of the io-mapping to skip track_pfn(). */ @@ -55,25 +96,11 @@ int remap_io_sg(struct vm_area_struct *vma, if (!use_dma(iobase)) flush_cache_range(vma, addr, size); - do { - if (use_dma(iobase)) { - if (!sg_dma_len(sgl)) - break; - pfn = (sg_dma_address(sgl) + iobase) >> PAGE_SHIFT; - len = sg_dma_len(sgl); - } else { - pfn = page_to_pfn(sg_page(sgl)); - len = sgl->length; - } - - err = remap_pfn_range(vma, addr + remapped, pfn, len, - vma->vm_page_prot); - if (err) - break; - remapped += len; - } while ((sgl = __sg_next(sgl))); - - if (err) - zap_vma_ptes(vma, addr, remapped); - return err; + err = apply_to_page_range(r.mm, addr, size, remap_sg, &r); + if (unlikely(err)) { + zap_vma_ptes(vma, addr, r.pfn << PAGE_SHIFT); + return err; + } + + return 0; } |