diff options
Diffstat (limited to 'drivers/gpu/drm/nouveau/nouveau_uvmm.c')
| -rw-r--r-- | drivers/gpu/drm/nouveau/nouveau_uvmm.c | 1917 |
1 files changed, 1917 insertions, 0 deletions
diff --git a/drivers/gpu/drm/nouveau/nouveau_uvmm.c b/drivers/gpu/drm/nouveau/nouveau_uvmm.c new file mode 100644 index 000000000000..aae780e4a4aa --- /dev/null +++ b/drivers/gpu/drm/nouveau/nouveau_uvmm.c @@ -0,0 +1,1917 @@ +// SPDX-License-Identifier: MIT + +/* + * Locking: + * + * The uvmm mutex protects any operations on the GPU VA space provided by the + * DRM GPU VA manager. + * + * The GEMs dma_resv lock protects the GEMs GPUVA list, hence link/unlink of a + * mapping to it's backing GEM must be performed under this lock. + * + * Actual map/unmap operations within the fence signalling critical path are + * protected by installing DMA fences to the corresponding GEMs DMA + * reservations, such that concurrent BO moves, which itself walk the GEMs GPUVA + * list in order to map/unmap it's entries, can't occur concurrently. + * + * Accessing the DRM_GPUVA_INVALIDATED flag doesn't need any separate + * protection, since there are no accesses other than from BO move callbacks + * and from the fence signalling critical path, which are already protected by + * the corresponding GEMs DMA reservation fence. + */ + +#include "nouveau_drv.h" +#include "nouveau_gem.h" +#include "nouveau_mem.h" +#include "nouveau_uvmm.h" + +#include <nvif/vmm.h> +#include <nvif/mem.h> + +#include <nvif/class.h> +#include <nvif/if000c.h> +#include <nvif/if900d.h> + +#define NOUVEAU_VA_SPACE_BITS 47 /* FIXME */ +#define NOUVEAU_VA_SPACE_START 0x0 +#define NOUVEAU_VA_SPACE_END (1ULL << NOUVEAU_VA_SPACE_BITS) + +#define list_last_op(_ops) list_last_entry(_ops, struct bind_job_op, entry) +#define list_prev_op(_op) list_prev_entry(_op, entry) +#define list_for_each_op(_op, _ops) list_for_each_entry(_op, _ops, entry) +#define list_for_each_op_from_reverse(_op, _ops) \ + list_for_each_entry_from_reverse(_op, _ops, entry) +#define list_for_each_op_safe(_op, _n, _ops) list_for_each_entry_safe(_op, _n, _ops, entry) + +enum vm_bind_op { + OP_MAP = DRM_NOUVEAU_VM_BIND_OP_MAP, + OP_UNMAP = DRM_NOUVEAU_VM_BIND_OP_UNMAP, + OP_MAP_SPARSE, + OP_UNMAP_SPARSE, +}; + +struct nouveau_uvma_prealloc { + struct nouveau_uvma *map; + struct nouveau_uvma *prev; + struct nouveau_uvma *next; +}; + +struct bind_job_op { + struct list_head entry; + + enum vm_bind_op op; + u32 flags; + + struct { + u64 addr; + u64 range; + } va; + + struct { + u32 handle; + u64 offset; + struct drm_gem_object *obj; + } gem; + + struct nouveau_uvma_region *reg; + struct nouveau_uvma_prealloc new; + struct drm_gpuva_ops *ops; +}; + +struct uvmm_map_args { + struct nouveau_uvma_region *region; + u64 addr; + u64 range; + u8 kind; +}; + +static int +nouveau_uvmm_vmm_sparse_ref(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + + return nvif_vmm_raw_sparse(vmm, addr, range, true); +} + +static int +nouveau_uvmm_vmm_sparse_unref(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + + return nvif_vmm_raw_sparse(vmm, addr, range, false); +} + +static int +nouveau_uvmm_vmm_get(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + + return nvif_vmm_raw_get(vmm, addr, range, PAGE_SHIFT); +} + +static int +nouveau_uvmm_vmm_put(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + + return nvif_vmm_raw_put(vmm, addr, range, PAGE_SHIFT); +} + +static int +nouveau_uvmm_vmm_unmap(struct nouveau_uvmm *uvmm, + u64 addr, u64 range, bool sparse) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + + return nvif_vmm_raw_unmap(vmm, addr, range, PAGE_SHIFT, sparse); +} + +static int +nouveau_uvmm_vmm_map(struct nouveau_uvmm *uvmm, + u64 addr, u64 range, + u64 bo_offset, u8 kind, + struct nouveau_mem *mem) +{ + struct nvif_vmm *vmm = &uvmm->vmm.vmm; + union { + struct gf100_vmm_map_v0 gf100; + } args; + u32 argc = 0; + + switch (vmm->object.oclass) { + case NVIF_CLASS_VMM_GF100: + case NVIF_CLASS_VMM_GM200: + case NVIF_CLASS_VMM_GP100: + args.gf100.version = 0; + if (mem->mem.type & NVIF_MEM_VRAM) + args.gf100.vol = 0; + else + args.gf100.vol = 1; + args.gf100.ro = 0; + args.gf100.priv = 0; + args.gf100.kind = kind; + argc = sizeof(args.gf100); + break; + default: + WARN_ON(1); + return -ENOSYS; + } + + return nvif_vmm_raw_map(vmm, addr, range, PAGE_SHIFT, + &args, argc, + &mem->mem, bo_offset); +} + +static int +nouveau_uvma_region_sparse_unref(struct nouveau_uvma_region *reg) +{ + u64 addr = reg->va.addr; + u64 range = reg->va.range; + + return nouveau_uvmm_vmm_sparse_unref(reg->uvmm, addr, range); +} + +static int +nouveau_uvma_vmm_put(struct nouveau_uvma *uvma) +{ + u64 addr = uvma->va.va.addr; + u64 range = uvma->va.va.range; + + return nouveau_uvmm_vmm_put(to_uvmm(uvma), addr, range); +} + +static int +nouveau_uvma_map(struct nouveau_uvma *uvma, + struct nouveau_mem *mem) +{ + u64 addr = uvma->va.va.addr; + u64 offset = uvma->va.gem.offset; + u64 range = uvma->va.va.range; + + return nouveau_uvmm_vmm_map(to_uvmm(uvma), addr, range, + offset, uvma->kind, mem); +} + +static int +nouveau_uvma_unmap(struct nouveau_uvma *uvma) +{ + u64 addr = uvma->va.va.addr; + u64 range = uvma->va.va.range; + bool sparse = !!uvma->region; + + if (drm_gpuva_invalidated(&uvma->va)) + return 0; + + return nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); +} + +static int +nouveau_uvma_alloc(struct nouveau_uvma **puvma) +{ + *puvma = kzalloc(sizeof(**puvma), GFP_KERNEL); + if (!*puvma) + return -ENOMEM; + + return 0; +} + +static void +nouveau_uvma_free(struct nouveau_uvma *uvma) +{ + kfree(uvma); +} + +static void +nouveau_uvma_gem_get(struct nouveau_uvma *uvma) +{ + drm_gem_object_get(uvma->va.gem.obj); +} + +static void +nouveau_uvma_gem_put(struct nouveau_uvma *uvma) +{ + drm_gem_object_put(uvma->va.gem.obj); +} + +static int +nouveau_uvma_region_alloc(struct nouveau_uvma_region **preg) +{ + *preg = kzalloc(sizeof(**preg), GFP_KERNEL); + if (!*preg) + return -ENOMEM; + + kref_init(&(*preg)->kref); + + return 0; +} + +static void +nouveau_uvma_region_free(struct kref *kref) +{ + struct nouveau_uvma_region *reg = + container_of(kref, struct nouveau_uvma_region, kref); + + kfree(reg); +} + +static void +nouveau_uvma_region_get(struct nouveau_uvma_region *reg) +{ + kref_get(®->kref); +} + +static void +nouveau_uvma_region_put(struct nouveau_uvma_region *reg) +{ + kref_put(®->kref, nouveau_uvma_region_free); +} + +static int +__nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_region *reg) +{ + u64 addr = reg->va.addr; + u64 range = reg->va.range; + u64 last = addr + range - 1; + MA_STATE(mas, &uvmm->region_mt, addr, addr); + + if (unlikely(mas_walk(&mas))) + return -EEXIST; + + if (unlikely(mas.last < last)) + return -EEXIST; + + mas.index = addr; + mas.last = last; + + mas_store_gfp(&mas, reg, GFP_KERNEL); + + reg->uvmm = uvmm; + + return 0; +} + +static int +nouveau_uvma_region_insert(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_region *reg, + u64 addr, u64 range) +{ + int ret; + + reg->uvmm = uvmm; + reg->va.addr = addr; + reg->va.range = range; + + ret = __nouveau_uvma_region_insert(uvmm, reg); + if (ret) + return ret; + + return 0; +} + +static void +nouveau_uvma_region_remove(struct nouveau_uvma_region *reg) +{ + struct nouveau_uvmm *uvmm = reg->uvmm; + MA_STATE(mas, &uvmm->region_mt, reg->va.addr, 0); + + mas_erase(&mas); +} + +static int +nouveau_uvma_region_create(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nouveau_uvma_region *reg; + int ret; + + if (!drm_gpuva_interval_empty(&uvmm->umgr, addr, range)) + return -ENOSPC; + + ret = nouveau_uvma_region_alloc(®); + if (ret) + return ret; + + ret = nouveau_uvma_region_insert(uvmm, reg, addr, range); + if (ret) + goto err_free_region; + + ret = nouveau_uvmm_vmm_sparse_ref(uvmm, addr, range); + if (ret) + goto err_region_remove; + + return 0; + +err_region_remove: + nouveau_uvma_region_remove(reg); +err_free_region: + nouveau_uvma_region_put(reg); + return ret; +} + +static struct nouveau_uvma_region * +nouveau_uvma_region_find_first(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + MA_STATE(mas, &uvmm->region_mt, addr, 0); + + return mas_find(&mas, addr + range - 1); +} + +static struct nouveau_uvma_region * +nouveau_uvma_region_find(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nouveau_uvma_region *reg; + + reg = nouveau_uvma_region_find_first(uvmm, addr, range); + if (!reg) + return NULL; + + if (reg->va.addr != addr || + reg->va.range != range) + return NULL; + + return reg; +} + +static bool +nouveau_uvma_region_empty(struct nouveau_uvma_region *reg) +{ + struct nouveau_uvmm *uvmm = reg->uvmm; + + return drm_gpuva_interval_empty(&uvmm->umgr, + reg->va.addr, + reg->va.range); +} + +static int +__nouveau_uvma_region_destroy(struct nouveau_uvma_region *reg) +{ + struct nouveau_uvmm *uvmm = reg->uvmm; + u64 addr = reg->va.addr; + u64 range = reg->va.range; + + if (!nouveau_uvma_region_empty(reg)) + return -EBUSY; + + nouveau_uvma_region_remove(reg); + nouveau_uvmm_vmm_sparse_unref(uvmm, addr, range); + nouveau_uvma_region_put(reg); + + return 0; +} + +static int +nouveau_uvma_region_destroy(struct nouveau_uvmm *uvmm, + u64 addr, u64 range) +{ + struct nouveau_uvma_region *reg; + + reg = nouveau_uvma_region_find(uvmm, addr, range); + if (!reg) + return -ENOENT; + + return __nouveau_uvma_region_destroy(reg); +} + +static void +nouveau_uvma_region_dirty(struct nouveau_uvma_region *reg) +{ + + init_completion(®->complete); + reg->dirty = true; +} + +static void +nouveau_uvma_region_complete(struct nouveau_uvma_region *reg) +{ + complete_all(®->complete); +} + +static void +op_map_prepare_unwind(struct nouveau_uvma *uvma) +{ + nouveau_uvma_gem_put(uvma); + drm_gpuva_remove(&uvma->va); + nouveau_uvma_free(uvma); +} + +static void +op_unmap_prepare_unwind(struct drm_gpuva *va) +{ + drm_gpuva_insert(va->mgr, va); +} + +static void +nouveau_uvmm_sm_prepare_unwind(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops, + struct drm_gpuva_op *last, + struct uvmm_map_args *args) +{ + struct drm_gpuva_op *op = last; + u64 vmm_get_start = args ? args->addr : 0; + u64 vmm_get_end = args ? args->addr + args->range : 0; + + /* Unwind GPUVA space. */ + drm_gpuva_for_each_op_from_reverse(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: + op_map_prepare_unwind(new->map); + break; + case DRM_GPUVA_OP_REMAP: { + struct drm_gpuva_op_remap *r = &op->remap; + + if (r->next) + op_map_prepare_unwind(new->next); + + if (r->prev) + op_map_prepare_unwind(new->prev); + + op_unmap_prepare_unwind(r->unmap->va); + break; + } + case DRM_GPUVA_OP_UNMAP: + op_unmap_prepare_unwind(op->unmap.va); + break; + default: + break; + } + } + + /* Unmap operation don't allocate page tables, hence skip the following + * page table unwind. + */ + if (!args) + return; + + drm_gpuva_for_each_op(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: { + u64 vmm_get_range = vmm_get_end - vmm_get_start; + + if (vmm_get_range) + nouveau_uvmm_vmm_put(uvmm, vmm_get_start, + vmm_get_range); + break; + } + case DRM_GPUVA_OP_REMAP: { + struct drm_gpuva_op_remap *r = &op->remap; + struct drm_gpuva *va = r->unmap->va; + u64 ustart = va->va.addr; + u64 urange = va->va.range; + u64 uend = ustart + urange; + + if (r->prev) + vmm_get_start = uend; + + if (r->next) + vmm_get_end = ustart; + + if (r->prev && r->next) + vmm_get_start = vmm_get_end = 0; + + break; + } + case DRM_GPUVA_OP_UNMAP: { + struct drm_gpuva_op_unmap *u = &op->unmap; + struct drm_gpuva *va = u->va; + u64 ustart = va->va.addr; + u64 urange = va->va.range; + u64 uend = ustart + urange; + + /* Nothing to do for mappings we merge with. */ + if (uend == vmm_get_start || + ustart == vmm_get_end) + break; + + if (ustart > vmm_get_start) { + u64 vmm_get_range = ustart - vmm_get_start; + + nouveau_uvmm_vmm_put(uvmm, vmm_get_start, + vmm_get_range); + } + vmm_get_start = uend; + break; + } + default: + break; + } + + if (op == last) + break; + } +} + +static void +nouveau_uvmm_sm_map_prepare_unwind(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops, + u64 addr, u64 range) +{ + struct drm_gpuva_op *last = drm_gpuva_last_op(ops); + struct uvmm_map_args args = { + .addr = addr, + .range = range, + }; + + nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, &args); +} + +static void +nouveau_uvmm_sm_unmap_prepare_unwind(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + struct drm_gpuva_op *last = drm_gpuva_last_op(ops); + + nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, last, NULL); +} + +static int +op_map_prepare(struct nouveau_uvmm *uvmm, + struct nouveau_uvma **puvma, + struct drm_gpuva_op_map *op, + struct uvmm_map_args *args) +{ + struct nouveau_uvma *uvma; + int ret; + + ret = nouveau_uvma_alloc(&uvma); + if (ret) + return ret; + + uvma->region = args->region; + uvma->kind = args->kind; + + drm_gpuva_map(&uvmm->umgr, &uvma->va, op); + + /* Keep a reference until this uvma is destroyed. */ + nouveau_uvma_gem_get(uvma); + + *puvma = uvma; + return 0; +} + +static void +op_unmap_prepare(struct drm_gpuva_op_unmap *u) +{ + drm_gpuva_unmap(u); +} + +static int +nouveau_uvmm_sm_prepare(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops, + struct uvmm_map_args *args) +{ + struct drm_gpuva_op *op; + u64 vmm_get_start = args ? args->addr : 0; + u64 vmm_get_end = args ? args->addr + args->range : 0; + int ret; + + drm_gpuva_for_each_op(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: { + u64 vmm_get_range = vmm_get_end - vmm_get_start; + + ret = op_map_prepare(uvmm, &new->map, &op->map, args); + if (ret) + goto unwind; + + if (args && vmm_get_range) { + ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start, + vmm_get_range); + if (ret) { + op_map_prepare_unwind(new->map); + goto unwind; + } + } + break; + } + case DRM_GPUVA_OP_REMAP: { + struct drm_gpuva_op_remap *r = &op->remap; + struct drm_gpuva *va = r->unmap->va; + struct uvmm_map_args remap_args = { + .kind = uvma_from_va(va)->kind, + .region = uvma_from_va(va)->region, + }; + u64 ustart = va->va.addr; + u64 urange = va->va.range; + u64 uend = ustart + urange; + + op_unmap_prepare(r->unmap); + + if (r->prev) { + ret = op_map_prepare(uvmm, &new->prev, r->prev, + &remap_args); + if (ret) + goto unwind; + + if (args) + vmm_get_start = uend; + } + + if (r->next) { + ret = op_map_prepare(uvmm, &new->next, r->next, + &remap_args); + if (ret) { + if (r->prev) + op_map_prepare_unwind(new->prev); + goto unwind; + } + + if (args) + vmm_get_end = ustart; + } + + if (args && (r->prev && r->next)) + vmm_get_start = vmm_get_end = 0; + + break; + } + case DRM_GPUVA_OP_UNMAP: { + struct drm_gpuva_op_unmap *u = &op->unmap; + struct drm_gpuva *va = u->va; + u64 ustart = va->va.addr; + u64 urange = va->va.range; + u64 uend = ustart + urange; + + op_unmap_prepare(u); + + if (!args) + break; + + /* Nothing to do for mappings we merge with. */ + if (uend == vmm_get_start || + ustart == vmm_get_end) + break; + + if (ustart > vmm_get_start) { + u64 vmm_get_range = ustart - vmm_get_start; + + ret = nouveau_uvmm_vmm_get(uvmm, vmm_get_start, + vmm_get_range); + if (ret) { + op_unmap_prepare_unwind(va); + goto unwind; + } + } + vmm_get_start = uend; + + break; + } + default: + ret = -EINVAL; + goto unwind; + } + } + + return 0; + +unwind: + if (op != drm_gpuva_first_op(ops)) + nouveau_uvmm_sm_prepare_unwind(uvmm, new, ops, + drm_gpuva_prev_op(op), + args); + return ret; +} + +static int +nouveau_uvmm_sm_map_prepare(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct nouveau_uvma_region *region, + struct drm_gpuva_ops *ops, + u64 addr, u64 range, u8 kind) +{ + struct uvmm_map_args args = { + .region = region, + .addr = addr, + .range = range, + .kind = kind, + }; + + return nouveau_uvmm_sm_prepare(uvmm, new, ops, &args); +} + +static int +nouveau_uvmm_sm_unmap_prepare(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + return nouveau_uvmm_sm_prepare(uvmm, new, ops, NULL); +} + +static struct drm_gem_object * +op_gem_obj(struct drm_gpuva_op *op) +{ + switch (op->op) { + case DRM_GPUVA_OP_MAP: + return op->map.gem.obj; + case DRM_GPUVA_OP_REMAP: + /* Actually, we're looking for the GEMs backing remap.prev and + * remap.next, but since this is a remap they're identical to + * the GEM backing the unmapped GPUVA. + */ + return op->remap.unmap->va->gem.obj; + case DRM_GPUVA_OP_UNMAP: + return op->unmap.va->gem.obj; + default: + WARN(1, "Unknown operation.\n"); + return NULL; + } +} + +static void +op_map(struct nouveau_uvma *uvma) +{ + struct nouveau_bo *nvbo = nouveau_gem_object(uvma->va.gem.obj); + + nouveau_uvma_map(uvma, nouveau_mem(nvbo->bo.resource)); +} + +static void +op_unmap(struct drm_gpuva_op_unmap *u) +{ + struct drm_gpuva *va = u->va; + struct nouveau_uvma *uvma = uvma_from_va(va); + + /* nouveau_uvma_unmap() does not unmap if backing BO is evicted. */ + if (!u->keep) + nouveau_uvma_unmap(uvma); +} + +static void +op_unmap_range(struct drm_gpuva_op_unmap *u, + u64 addr, u64 range) +{ + struct nouveau_uvma *uvma = uvma_from_va(u->va); + bool sparse = !!uvma->region; + + if (!drm_gpuva_invalidated(u->va)) + nouveau_uvmm_vmm_unmap(to_uvmm(uvma), addr, range, sparse); +} + +static void +op_remap(struct drm_gpuva_op_remap *r, + struct nouveau_uvma_prealloc *new) +{ + struct drm_gpuva_op_unmap *u = r->unmap; + struct nouveau_uvma *uvma = uvma_from_va(u->va); + u64 addr = uvma->va.va.addr; + u64 range = uvma->va.va.range; + + if (r->prev) + addr = r->prev->va.addr + r->prev->va.range; + + if (r->next) + range = r->next->va.addr - addr; + + op_unmap_range(u, addr, range); +} + +static int +nouveau_uvmm_sm(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + struct drm_gpuva_op *op; + + drm_gpuva_for_each_op(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: + op_map(new->map); + break; + case DRM_GPUVA_OP_REMAP: + op_remap(&op->remap, new); + break; + case DRM_GPUVA_OP_UNMAP: + op_unmap(&op->unmap); + break; + default: + break; + } + } + + return 0; +} + +static int +nouveau_uvmm_sm_map(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + return nouveau_uvmm_sm(uvmm, new, ops); +} + +static int +nouveau_uvmm_sm_unmap(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + return nouveau_uvmm_sm(uvmm, new, ops); +} + +static void +nouveau_uvmm_sm_cleanup(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops, bool unmap) +{ + struct drm_gpuva_op *op; + + drm_gpuva_for_each_op(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: + break; + case DRM_GPUVA_OP_REMAP: { + struct drm_gpuva_op_remap *r = &op->remap; + struct drm_gpuva_op_map *p = r->prev; + struct drm_gpuva_op_map *n = r->next; + struct drm_gpuva *va = r->unmap->va; + struct nouveau_uvma *uvma = uvma_from_va(va); + + if (unmap) { + u64 addr = va->va.addr; + u64 end = addr + va->va.range; + + if (p) + addr = p->va.addr + p->va.range; + + if (n) + end = n->va.addr; + + nouveau_uvmm_vmm_put(uvmm, addr, end - addr); + } + + nouveau_uvma_gem_put(uvma); + nouveau_uvma_free(uvma); + break; + } + case DRM_GPUVA_OP_UNMAP: { + struct drm_gpuva_op_unmap *u = &op->unmap; + struct drm_gpuva *va = u->va; + struct nouveau_uvma *uvma = uvma_from_va(va); + + if (unmap) + nouveau_uvma_vmm_put(uvma); + + nouveau_uvma_gem_put(uvma); + nouveau_uvma_free(uvma); + break; + } + default: + break; + } + } +} + +static void +nouveau_uvmm_sm_map_cleanup(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + nouveau_uvmm_sm_cleanup(uvmm, new, ops, false); +} + +static void +nouveau_uvmm_sm_unmap_cleanup(struct nouveau_uvmm *uvmm, + struct nouveau_uvma_prealloc *new, + struct drm_gpuva_ops *ops) +{ + nouveau_uvmm_sm_cleanup(uvmm, new, ops, true); +} + +static int +nouveau_uvmm_validate_range(struct nouveau_uvmm *uvmm, u64 addr, u64 range) +{ + u64 end = addr + range; + u64 kernel_managed_end = uvmm->kernel_managed_addr + + uvmm->kernel_managed_size; + + if (addr & ~PAGE_MASK) + return -EINVAL; + + if (range & ~PAGE_MASK) + return -EINVAL; + + if (end <= addr) + return -EINVAL; + + if (addr < NOUVEAU_VA_SPACE_START || + end > NOUVEAU_VA_SPACE_END) + return -EINVAL; + + if (addr < kernel_managed_end && + end > uvmm->kernel_managed_addr) + return -EINVAL; + + return 0; +} + +static int +nouveau_uvmm_bind_job_alloc(struct nouveau_uvmm_bind_job **pjob) +{ + *pjob = kzalloc(sizeof(**pjob), GFP_KERNEL); + if (!*pjob) + return -ENOMEM; + + kref_init(&(*pjob)->kref); + + return 0; +} + +static void +nouveau_uvmm_bind_job_free(struct kref *kref) +{ + struct nouveau_uvmm_bind_job *job = + container_of(kref, struct nouveau_uvmm_bind_job, kref); + + nouveau_job_free(&job->base); + kfree(job); +} + +static void +nouveau_uvmm_bind_job_get(struct nouveau_uvmm_bind_job *job) +{ + kref_get(&job->kref); +} + +static void +nouveau_uvmm_bind_job_put(struct nouveau_uvmm_bind_job *job) +{ + kref_put(&job->kref, nouveau_uvmm_bind_job_free); +} + +static int +bind_validate_op(struct nouveau_job *job, + struct bind_job_op *op) +{ + struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); + struct drm_gem_object *obj = op->gem.obj; + + if (op->op == OP_MAP) { + if (op->gem.offset & ~PAGE_MASK) + return -EINVAL; + + if (obj->size <= op->gem.offset) + return -EINVAL; + + if (op->va.range > (obj->size - op->gem.offset)) + return -EINVAL; + } + + return nouveau_uvmm_validate_range(uvmm, op->va.addr, op->va.range); +} + +static void +bind_validate_map_sparse(struct nouveau_job *job, u64 addr, u64 range) +{ + struct nouveau_uvmm_bind_job *bind_job; + struct nouveau_sched_entity *entity = job->entity; + struct bind_job_op *op; + u64 end = addr + range; + +again: + spin_lock(&entity->job.list.lock); + list_for_each_entry(bind_job, &entity->job.list.head, entry) { + list_for_each_op(op, &bind_job->ops) { + if (op->op == OP_UNMAP) { + u64 op_addr = op->va.addr; + u64 op_end = op_addr + op->va.range; + + if (!(end <= op_addr || addr >= op_end)) { + nouveau_uvmm_bind_job_get(bind_job); + spin_unlock(&entity->job.list.lock); + wait_for_completion(&bind_job->complete); + nouveau_uvmm_bind_job_put(bind_job); + goto again; + } + } + } + } + spin_unlock(&entity->job.list.lock); +} + +static int +bind_validate_map_common(struct nouveau_job *job, u64 addr, u64 range, + bool sparse) +{ + struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); + struct nouveau_uvma_region *reg; + u64 reg_addr, reg_end; + u64 end = addr + range; + +again: + nouveau_uvmm_lock(uvmm); + reg = nouveau_uvma_region_find_first(uvmm, addr, range); + if (!reg) { + nouveau_uvmm_unlock(uvmm); + return 0; + } + + /* Generally, job submits are serialized, hence only + * dirty regions can be modified concurrently. + */ + if (reg->dirty) { + nouveau_uvma_region_get(reg); + nouveau_uvmm_unlock(uvmm); + wait_for_completion(®->complete); + nouveau_uvma_region_put(reg); + goto again; + } + nouveau_uvmm_unlock(uvmm); + + if (sparse) + return -ENOSPC; + + reg_addr = reg->va.addr; + reg_end = reg_addr + reg->va.range; + + /* Make sure the mapping is either outside of a + * region or fully enclosed by a region. + */ + if (reg_addr > addr || reg_end < end) + return -ENOSPC; + + return 0; +} + +static int +bind_validate_region(struct nouveau_job *job) +{ + struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); + struct bind_job_op *op; + int ret; + + list_for_each_op(op, &bind_job->ops) { + u64 op_addr = op->va.addr; + u64 op_range = op->va.range; + bool sparse = false; + + switch (op->op) { + case OP_MAP_SPARSE: + sparse = true; + bind_validate_map_sparse(job, op_addr, op_range); + fallthrough; + case OP_MAP: + ret = bind_validate_map_common(job, op_addr, op_range, + sparse); + if (ret) + return ret; + break; + default: + break; + } + } + + return 0; +} + +static void +bind_link_gpuvas(struct drm_gpuva_ops *ops, struct nouveau_uvma_prealloc *new) +{ + struct drm_gpuva_op *op; + + drm_gpuva_for_each_op(op, ops) { + switch (op->op) { + case DRM_GPUVA_OP_MAP: + drm_gpuva_link(&new->map->va); + break; + case DRM_GPUVA_OP_REMAP: + if (op->remap.prev) + drm_gpuva_link(&new->prev->va); + if (op->remap.next) + drm_gpuva_link(&new->next->va); + drm_gpuva_unlink(op->remap.unmap->va); + break; + case DRM_GPUVA_OP_UNMAP: + drm_gpuva_unlink(op->unmap.va); + break; + default: + break; + } + } +} + +static int +nouveau_uvmm_bind_job_submit(struct nouveau_job *job) +{ + struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); + struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); + struct nouveau_sched_entity *entity = job->entity; + struct drm_exec *exec = &job->exec; + struct bind_job_op *op; + int ret; + + list_for_each_op(op, &bind_job->ops) { + if (op->op == OP_MAP) { + op->gem.obj = drm_gem_object_lookup(job->file_priv, + op->gem.handle); + if (!op->gem.obj) + return -ENOENT; + } + + ret = bind_validate_op(job, op); + if (ret) + return ret; + } + + /* If a sparse region or mapping overlaps a dirty region, we need to + * wait for the region to complete the unbind process. This is due to + * how page table management is currently implemented. A future + * implementation might change this. + */ + ret = bind_validate_region(job); + if (ret) + return ret; + + /* Once we start modifying the GPU VA space we need to keep holding the + * uvmm lock until we can't fail anymore. This is due to the set of GPU + * VA space changes must appear atomically and we need to be able to + * unwind all GPU VA space changes on failure. + */ + nouveau_uvmm_lock(uvmm); + list_for_each_op(op, &bind_job->ops) { + switch (op->op) { + case OP_MAP_SPARSE: + ret = nouveau_uvma_region_create(uvmm, + op->va.addr, + op->va.range); + if (ret) + goto unwind_continue; + + break; + case OP_UNMAP_SPARSE: + op->reg = nouveau_uvma_region_find(uvmm, op->va.addr, + op->va.range); + if (!op->reg || op->reg->dirty) { + ret = -ENOENT; + goto unwind_continue; + } + + op->ops = drm_gpuva_sm_unmap_ops_create(&uvmm->umgr, + op->va.addr, + op->va.range); + if (IS_ERR(op->ops)) { + ret = PTR_ERR(op->ops); + goto unwind_continue; + } + + ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new, + op->ops); + if (ret) { + drm_gpuva_ops_free(&uvmm->umgr, op->ops); + op->ops = NULL; + op->reg = NULL; + goto unwind_continue; + } + + nouveau_uvma_region_dirty(op->reg); + + break; + case OP_MAP: { + struct nouveau_uvma_region *reg; + + reg = nouveau_uvma_region_find_first(uvmm, + op->va.addr, + op->va.range); + if (reg) { + u64 reg_addr = reg->va.addr; + u64 reg_end = reg_addr + reg->va.range; + u64 op_addr = op->va.addr; + u64 op_end = op_addr + op->va.range; + + if (unlikely(reg->dirty)) { + ret = -EINVAL; + goto unwind_continue; + } + + /* Make sure the mapping is either outside of a + * region or fully enclosed by a region. + */ + if (reg_addr > op_addr || reg_end < op_end) { + ret = -ENOSPC; + goto unwind_continue; + } + } + + op->ops = drm_gpuva_sm_map_ops_create(&uvmm->umgr, + op->va.addr, + op->va.range, + op->gem.obj, + op->gem.offset); + if (IS_ERR(op->ops)) { + ret = PTR_ERR(op->ops); + goto unwind_continue; + } + + ret = nouveau_uvmm_sm_map_prepare(uvmm, &op->new, + reg, op->ops, + op->va.addr, + op->va.range, + op->flags & 0xff); + if (ret) { + drm_gpuva_ops_free(&uvmm->umgr, op->ops); + op->ops = NULL; + goto unwind_continue; + } + + break; + } + case OP_UNMAP: + op->ops = drm_gpuva_sm_unmap_ops_create(&uvmm->umgr, + op->va.addr, + op->va.range); + if (IS_ERR(op->ops)) { + ret = PTR_ERR(op->ops); + goto unwind_continue; + } + + ret = nouveau_uvmm_sm_unmap_prepare(uvmm, &op->new, + op->ops); + if (ret) { + drm_gpuva_ops_free(&uvmm->umgr, op->ops); + op->ops = NULL; + goto unwind_continue; + } + + break; + default: + ret = -EINVAL; + goto unwind_continue; + } + } + + drm_exec_init(exec, DRM_EXEC_INTERRUPTIBLE_WAIT | + DRM_EXEC_IGNORE_DUPLICATES); + drm_exec_until_all_locked(exec) { + list_for_each_op(op, &bind_job->ops) { + struct drm_gpuva_op *va_op; + + if (IS_ERR_OR_NULL(op->ops)) + continue; + + drm_gpuva_for_each_op(va_op, op->ops) { + struct drm_gem_object *obj = op_gem_obj(va_op); + + if (unlikely(!obj)) + continue; + + ret = drm_exec_prepare_obj(exec, obj, 1); + drm_exec_retry_on_contention(exec); + if (ret) { + op = list_last_op(&bind_job->ops); + goto unwind; + } + } + } + } + + list_for_each_op(op, &bind_job->ops) { + struct drm_gpuva_op *va_op; + + if (IS_ERR_OR_NULL(op->ops)) + continue; + + drm_gpuva_for_each_op(va_op, op->ops) { + struct drm_gem_object *obj = op_gem_obj(va_op); + + if (unlikely(!obj)) + continue; + + /* Don't validate GEMs backing mappings we're about to + * unmap, it's not worth the effort. + */ + if (unlikely(va_op->op == DRM_GPUVA_OP_UNMAP)) + continue; + + ret = nouveau_bo_validate(nouveau_gem_object(obj), + true, false); + if (ret) { + op = list_last_op(&bind_job->ops); + goto unwind; + } + } + } + + /* Link and unlink GPUVAs while holding the dma_resv lock. + * + * As long as we validate() all GEMs and add fences to all GEMs DMA + * reservations backing map and remap operations we can be sure there + * won't be any concurrent (in)validations during job execution, hence + * we're safe to check drm_gpuva_invalidated() within the fence + * signalling critical path without holding a separate lock. + * + * GPUVAs about to be unmapped are safe as well, since they're unlinked + * already. + * + * GEMs from map and remap operations must be validated before linking + * their corresponding mappings to prevent the actual PT update to + * happen right away in validate() rather than asynchronously as + * intended. + * + * Note that after linking and unlinking the GPUVAs in this loop this + * function cannot fail anymore, hence there is no need for an unwind + * path. + */ + list_for_each_op(op, &bind_job->ops) { + switch (op->op) { + case OP_UNMAP_SPARSE: + case OP_MAP: + case OP_UNMAP: + bind_link_gpuvas(op->ops, &op->new); + break; + default: + break; + } + } + nouveau_uvmm_unlock(uvmm); + + spin_lock(&entity->job.list.lock); + list_add(&bind_job->entry, &entity->job.list.head); + spin_unlock(&entity->job.list.lock); + + return 0; + +unwind_continue: + op = list_prev_op(op); +unwind: + list_for_each_op_from_reverse(op, &bind_job->ops) { + switch (op->op) { + case OP_MAP_SPARSE: + nouveau_uvma_region_destroy(uvmm, op->va.addr, + op->va.range); + break; + case OP_UNMAP_SPARSE: + __nouveau_uvma_region_insert(uvmm, op->reg); + nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, + op->ops); + break; + case OP_MAP: + nouveau_uvmm_sm_map_prepare_unwind(uvmm, &op->new, + op->ops, + op->va.addr, + op->va.range); + break; + case OP_UNMAP: + nouveau_uvmm_sm_unmap_prepare_unwind(uvmm, &op->new, + op->ops); + break; + } + + drm_gpuva_ops_free(&uvmm->umgr, op->ops); + op->ops = NULL; + op->reg = NULL; + } + + nouveau_uvmm_unlock(uvmm); + drm_exec_fini(exec); + return ret; +} + +static void +nouveau_uvmm_bind_job_armed_submit(struct nouveau_job *job) +{ + struct drm_exec *exec = &job->exec; + struct drm_gem_object *obj; + unsigned long index; + + drm_exec_for_each_locked_object(exec, index, obj) + dma_resv_add_fence(obj->resv, job->done_fence, job->resv_usage); + + drm_exec_fini(exec); +} + +static struct dma_fence * +nouveau_uvmm_bind_job_run(struct nouveau_job *job) +{ + struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); + struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); + struct bind_job_op *op; + int ret = 0; + + list_for_each_op(op, &bind_job->ops) { + switch (op->op) { + case OP_MAP_SPARSE: + /* noop */ + break; + case OP_MAP: + ret = nouveau_uvmm_sm_map(uvmm, &op->new, op->ops); + if (ret) + goto out; + break; + case OP_UNMAP_SPARSE: + fallthrough; + case OP_UNMAP: + ret = nouveau_uvmm_sm_unmap(uvmm, &op->new, op->ops); + if (ret) + goto out; + break; + } + } + +out: + if (ret) + NV_PRINTK(err, job->cli, "bind job failed: %d\n", ret); + return ERR_PTR(ret); +} + +static void +nouveau_uvmm_bind_job_free_work_fn(struct work_struct *work) +{ + struct nouveau_uvmm_bind_job *bind_job = + container_of(work, struct nouveau_uvmm_bind_job, work); + struct nouveau_job *job = &bind_job->base; + struct nouveau_uvmm *uvmm = nouveau_cli_uvmm(job->cli); + struct nouveau_sched_entity *entity = job->entity; + struct bind_job_op *op, *next; + + list_for_each_op(op, &bind_job->ops) { + struct drm_gem_object *obj = op->gem.obj; + + /* When nouveau_uvmm_bind_job_submit() fails op->ops and op->reg + * will be NULL, hence skip the cleanup. + */ + switch (op->op) { + case OP_MAP_SPARSE: + /* noop */ + break; + case OP_UNMAP_SPARSE: + if (!IS_ERR_OR_NULL(op->ops)) + nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new, + op->ops); + + if (op->reg) { + nouveau_uvma_region_sparse_unref(op->reg); + nouveau_uvmm_lock(uvmm); + nouveau_uvma_region_remove(op->reg); + nouveau_uvmm_unlock(uvmm); + nouveau_uvma_region_complete(op->reg); + nouveau_uvma_region_put(op->reg); + } + + break; + case OP_MAP: + if (!IS_ERR_OR_NULL(op->ops)) + nouveau_uvmm_sm_map_cleanup(uvmm, &op->new, + op->ops); + break; + case OP_UNMAP: + if (!IS_ERR_OR_NULL(op->ops)) + nouveau_uvmm_sm_unmap_cleanup(uvmm, &op->new, + op->ops); + break; + } + + if (!IS_ERR_OR_NULL(op->ops)) + drm_gpuva_ops_free(&uvmm->umgr, op->ops); + + if (obj) + drm_gem_object_put(obj); + } + + spin_lock(&entity->job.list.lock); + list_del(&bind_job->entry); + spin_unlock(&entity->job.list.lock); + + complete_all(&bind_job->complete); + wake_up(&entity->job.wq); + + /* Remove and free ops after removing the bind job from the job list to + * avoid races against bind_validate_map_sparse(). + */ + list_for_each_op_safe(op, next, &bind_job->ops) { + list_del(&op->entry); + kfree(op); + } + + nouveau_uvmm_bind_job_put(bind_job); +} + +static void +nouveau_uvmm_bind_job_free_qwork(struct nouveau_job *job) +{ + struct nouveau_uvmm_bind_job *bind_job = to_uvmm_bind_job(job); + struct nouveau_sched_entity *entity = job->entity; + + nouveau_sched_entity_qwork(entity, &bind_job->work); +} + +static struct nouveau_job_ops nouveau_bind_job_ops = { + .submit = nouveau_uvmm_bind_job_submit, + .armed_submit = nouveau_uvmm_bind_job_armed_submit, + .run = nouveau_uvmm_bind_job_run, + .free = nouveau_uvmm_bind_job_free_qwork, +}; + +static int +bind_job_op_from_uop(struct bind_job_op **pop, + struct drm_nouveau_vm_bind_op *uop) +{ + struct bind_job_op *op; + + op = *pop = kzalloc(sizeof(*op), GFP_KERNEL); + if (!op) + return -ENOMEM; + + switch (uop->op) { + case OP_MAP: + op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? + OP_MAP_SPARSE : OP_MAP; + break; + case OP_UNMAP: + op->op = uop->flags & DRM_NOUVEAU_VM_BIND_SPARSE ? + OP_UNMAP_SPARSE : OP_UNMAP; + break; + default: + op->op = uop->op; + break; + } + + op->flags = uop->flags; + op->va.addr = uop->addr; + op->va.range = uop->range; + op->gem.handle = uop->handle; + op->gem.offset = uop->bo_offset; + + return 0; +} + +static void +bind_job_ops_free(struct list_head *ops) +{ + struct bind_job_op *op, *next; + + list_for_each_op_safe(op, next, ops) { + list_del(&op->entry); + kfree(op); + } +} + +static int +nouveau_uvmm_bind_job_init(struct nouveau_uvmm_bind_job **pjob, + struct nouveau_uvmm_bind_job_args *__args) +{ + struct nouveau_uvmm_bind_job *job; + struct nouveau_job_args args = {}; + struct bind_job_op *op; + int i, ret; + + ret = nouveau_uvmm_bind_job_alloc(&job); + if (ret) + return ret; + + INIT_LIST_HEAD(&job->ops); + INIT_LIST_HEAD(&job->entry); + + for (i = 0; i < __args->op.count; i++) { + ret = bind_job_op_from_uop(&op, &__args->op.s[i]); + if (ret) + goto err_free; + + list_add_tail(&op->entry, &job->ops); + } + + init_completion(&job->complete); + INIT_WORK(&job->work, nouveau_uvmm_bind_job_free_work_fn); + + args.sched_entity = __args->sched_entity; + args.file_priv = __args->file_priv; + + args.in_sync.count = __args->in_sync.count; + args.in_sync.s = __args->in_sync.s; + + args.out_sync.count = __args->out_sync.count; + args.out_sync.s = __args->out_sync.s; + + args.sync = !(__args->flags & DRM_NOUVEAU_VM_BIND_RUN_ASYNC); + args.ops = &nouveau_bind_job_ops; + args.resv_usage = DMA_RESV_USAGE_BOOKKEEP; + + ret = nouveau_job_init(&job->base, &args); + if (ret) + goto err_free; + + *pjob = job; + return 0; + +err_free: + bind_job_ops_free(&job->ops); + kfree(job); + *pjob = NULL; + + return ret; +} + +int +nouveau_uvmm_ioctl_vm_init(struct drm_device *dev, + void *data, + struct drm_file *file_priv) +{ + struct nouveau_cli *cli = nouveau_cli(file_priv); + struct drm_nouveau_vm_init *init = data; + + return nouveau_uvmm_init(&cli->uvmm, cli, init->kernel_managed_addr, + init->kernel_managed_size); +} + +static int +nouveau_uvmm_vm_bind(struct nouveau_uvmm_bind_job_args *args) +{ + struct nouveau_uvmm_bind_job *job; + int ret; + + ret = nouveau_uvmm_bind_job_init(&job, args); + if (ret) + return ret; + + ret = nouveau_job_submit(&job->base); + if (ret) + goto err_job_fini; + + return 0; + +err_job_fini: + nouveau_job_fini(&job->base); + return ret; +} + +static int +nouveau_uvmm_vm_bind_ucopy(struct nouveau_uvmm_bind_job_args *args, + struct drm_nouveau_vm_bind *req) +{ + struct drm_nouveau_sync **s; + u32 inc = req->wait_count; + u64 ins = req->wait_ptr; + u32 outc = req->sig_count; + u64 outs = req->sig_ptr; + u32 opc = req->op_count; + u64 ops = req->op_ptr; + int ret; + + args->flags = req->flags; + + if (opc) { + args->op.count = opc; + args->op.s = u_memcpya(ops, opc, + sizeof(*args->op.s)); + if (IS_ERR(args->op.s)) + return PTR_ERR(args->op.s); + } + + if (inc) { + s = &args->in_sync.s; + + args->in_sync.count = inc; + *s = u_memcpya(ins, inc, sizeof(**s)); + if (IS_ERR(*s)) { + ret = PTR_ERR(*s); + goto err_free_ops; + } + } + + if (outc) { + s = &args->out_sync.s; + + args->out_sync.count = outc; + *s = u_memcpya(outs, outc, sizeof(**s)); + if (IS_ERR(*s)) { + ret = PTR_ERR(*s); + goto err_free_ins; + } + } + + return 0; + +err_free_ops: + u_free(args->op.s); +err_free_ins: + u_free(args->in_sync.s); + return ret; +} + +static void +nouveau_uvmm_vm_bind_ufree(struct nouveau_uvmm_bind_job_args *args) +{ + u_free(args->op.s); + u_free(args->in_sync.s); + u_free(args->out_sync.s); +} + +int +nouveau_uvmm_ioctl_vm_bind(struct drm_device *dev, + void *data, + struct drm_file *file_priv) +{ + struct nouveau_cli *cli = nouveau_cli(file_priv); + struct nouveau_uvmm_bind_job_args args = {}; + struct drm_nouveau_vm_bind *req = data; + int ret = 0; + + if (unlikely(!nouveau_cli_uvmm_locked(cli))) + return -ENOSYS; + + ret = nouveau_uvmm_vm_bind_ucopy(&args, req); + if (ret) + return ret; + + args.sched_entity = &cli->sched_entity; + args.file_priv = file_priv; + + ret = nouveau_uvmm_vm_bind(&args); + if (ret) + goto out_free_args; + +out_free_args: + nouveau_uvmm_vm_bind_ufree(&args); + return ret; +} + +void +nouveau_uvmm_bo_map_all(struct nouveau_bo *nvbo, struct nouveau_mem *mem) +{ + struct drm_gem_object *obj = &nvbo->bo.base; + struct drm_gpuva *va; + + dma_resv_assert_held(obj->resv); + + drm_gem_for_each_gpuva(va, obj) { + struct nouveau_uvma *uvma = uvma_from_va(va); + + nouveau_uvma_map(uvma, mem); + drm_gpuva_invalidate(va, false); + } +} + +void +nouveau_uvmm_bo_unmap_all(struct nouveau_bo *nvbo) +{ + struct drm_gem_object *obj = &nvbo->bo.base; + struct drm_gpuva *va; + + dma_resv_assert_held(obj->resv); + + drm_gem_for_each_gpuva(va, obj) { + struct nouveau_uvma *uvma = uvma_from_va(va); + + nouveau_uvma_unmap(uvma); + drm_gpuva_invalidate(va, true); + } +} + +int +nouveau_uvmm_init(struct nouveau_uvmm *uvmm, struct nouveau_cli *cli, + u64 kernel_managed_addr, u64 kernel_managed_size) +{ + int ret; + u64 kernel_managed_end = kernel_managed_addr + kernel_managed_size; + + mutex_init(&uvmm->mutex); + dma_resv_init(&uvmm->resv); + mt_init_flags(&uvmm->region_mt, MT_FLAGS_LOCK_EXTERN); + mt_set_external_lock(&uvmm->region_mt, &uvmm->mutex); + + mutex_lock(&cli->mutex); + + if (unlikely(cli->uvmm.disabled)) { + ret = -ENOSYS; + goto out_unlock; + } + + if (kernel_managed_end <= kernel_managed_addr) { + ret = -EINVAL; + goto out_unlock; + } + + if (kernel_managed_end > NOUVEAU_VA_SPACE_END) { + ret = -EINVAL; + goto out_unlock; + } + + uvmm->kernel_managed_addr = kernel_managed_addr; + uvmm->kernel_managed_size = kernel_managed_size; + + drm_gpuva_manager_init(&uvmm->umgr, cli->name, + NOUVEAU_VA_SPACE_START, + NOUVEAU_VA_SPACE_END, + kernel_managed_addr, kernel_managed_size, + NULL); + + ret = nvif_vmm_ctor(&cli->mmu, "uvmm", + cli->vmm.vmm.object.oclass, RAW, + kernel_managed_addr, kernel_managed_size, + NULL, 0, &cli->uvmm.vmm.vmm); + if (ret) + goto out_free_gpuva_mgr; + + cli->uvmm.vmm.cli = cli; + mutex_unlock(&cli->mutex); + + return 0; + +out_free_gpuva_mgr: + drm_gpuva_manager_destroy(&uvmm->umgr); +out_unlock: + mutex_unlock(&cli->mutex); + return ret; +} + +void +nouveau_uvmm_fini(struct nouveau_uvmm *uvmm) +{ + MA_STATE(mas, &uvmm->region_mt, 0, 0); + struct nouveau_uvma_region *reg; + struct nouveau_cli *cli = uvmm->vmm.cli; + struct nouveau_sched_entity *entity = &cli->sched_entity; + struct drm_gpuva *va, *next; + + if (!cli) + return; + + rmb(); /* for list_empty to work without lock */ + wait_event(entity->job.wq, list_empty(&entity->job.list.head)); + + nouveau_uvmm_lock(uvmm); + drm_gpuva_for_each_va_safe(va, next, &uvmm->umgr) { + struct nouveau_uvma *uvma = uvma_from_va(va); + struct drm_gem_object *obj = va->gem.obj; + + if (unlikely(va == &uvmm->umgr.kernel_alloc_node)) + continue; + + drm_gpuva_remove(va); + + dma_resv_lock(obj->resv, NULL); + drm_gpuva_unlink(va); + dma_resv_unlock(obj->resv); + + nouveau_uvma_unmap(uvma); + nouveau_uvma_vmm_put(uvma); + + nouveau_uvma_gem_put(uvma); + nouveau_uvma_free(uvma); + } + + mas_for_each(&mas, reg, ULONG_MAX) { + mas_erase(&mas); + nouveau_uvma_region_sparse_unref(reg); + nouveau_uvma_region_put(reg); + } + + WARN(!mtree_empty(&uvmm->region_mt), + "nouveau_uvma_region tree not empty, potentially leaking memory."); + __mt_destroy(&uvmm->region_mt); + nouveau_uvmm_unlock(uvmm); + + mutex_lock(&cli->mutex); + nouveau_vmm_fini(&uvmm->vmm); + drm_gpuva_manager_destroy(&uvmm->umgr); + mutex_unlock(&cli->mutex); + + dma_resv_fini(&uvmm->resv); +} |