diff options
Diffstat (limited to 'drivers/gpu/drm/etnaviv/etnaviv_gem.c')
| -rw-r--r-- | drivers/gpu/drm/etnaviv/etnaviv_gem.c | 273 |
1 files changed, 132 insertions, 141 deletions
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c index 1fa74226db91..5d8f3b03d4ae 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c @@ -3,10 +3,12 @@ * Copyright (C) 2015-2018 Etnaviv Project */ -#include <linux/spinlock.h> +#include <drm/drm_prime.h> +#include <drm/drm_print.h> +#include <linux/dma-mapping.h> #include <linux/shmem_fs.h> -#include <linux/sched/mm.h> -#include <linux/sched/task.h> +#include <linux/spinlock.h> +#include <linux/vmalloc.h> #include "etnaviv_drv.h" #include "etnaviv_gem.h" @@ -26,7 +28,7 @@ static void etnaviv_gem_scatter_map(struct etnaviv_gem_object *etnaviv_obj) * because display controller, GPU, etc. are not coherent. */ if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) - dma_map_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); + dma_map_sgtable(dev->dev, sgt, DMA_BIDIRECTIONAL, 0); } static void etnaviv_gem_scatterlist_unmap(struct etnaviv_gem_object *etnaviv_obj) @@ -50,7 +52,7 @@ static void etnaviv_gem_scatterlist_unmap(struct etnaviv_gem_object *etnaviv_obj * discard those writes. */ if (etnaviv_obj->flags & ETNA_BO_CACHE_MASK) - dma_unmap_sg(dev->dev, sgt->sgl, sgt->nents, DMA_BIDIRECTIONAL); + dma_unmap_sgtable(dev->dev, sgt, DMA_BIDIRECTIONAL, 0); } /* called with etnaviv_obj->lock held */ @@ -99,10 +101,10 @@ struct page **etnaviv_gem_get_pages(struct etnaviv_gem_object *etnaviv_obj) if (!etnaviv_obj->sgt) { struct drm_device *dev = etnaviv_obj->base.dev; - int npages = etnaviv_obj->base.size >> PAGE_SHIFT; + unsigned int npages = etnaviv_obj->base.size >> PAGE_SHIFT; struct sg_table *sgt; - sgt = drm_prime_pages_to_sg(etnaviv_obj->pages, npages); + sgt = drm_prime_pages_to_sg(dev, etnaviv_obj->pages, npages); if (IS_ERR(sgt)) { dev_err(dev->dev, "failed to allocate sgt: %ld\n", PTR_ERR(sgt)); @@ -128,8 +130,7 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, { pgprot_t vm_page_prot; - vma->vm_flags &= ~VM_PFNMAP; - vma->vm_flags |= VM_MIXEDMAP; + vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); vm_page_prot = vm_get_page_prot(vma->vm_flags); @@ -143,10 +144,8 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, * address_space (so unmap_mapping_range does what we want, * in particular in the case of mmap'd dmabufs) */ - fput(vma->vm_file); - get_file(etnaviv_obj->base.filp); vma->vm_pgoff = 0; - vma->vm_file = etnaviv_obj->base.filp; + vma_set_file(vma, etnaviv_obj->base.filp); vma->vm_page_prot = vm_page_prot; } @@ -154,27 +153,20 @@ static int etnaviv_gem_mmap_obj(struct etnaviv_gem_object *etnaviv_obj, return 0; } -int etnaviv_gem_mmap(struct file *filp, struct vm_area_struct *vma) +static int etnaviv_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) { - struct etnaviv_gem_object *obj; - int ret; - - ret = drm_gem_mmap(filp, vma); - if (ret) { - DBG("mmap failed: %d", ret); - return ret; - } + struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - obj = to_etnaviv_bo(vma->vm_private_data); - return obj->ops->mmap(obj, vma); + return etnaviv_obj->ops->mmap(etnaviv_obj, vma); } -vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) +static vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - struct page **pages, *page; + struct page **pages; + unsigned long pfn; pgoff_t pgoff; int err; @@ -198,12 +190,12 @@ vm_fault_t etnaviv_gem_fault(struct vm_fault *vmf) /* We don't use vmf->pgoff since that has the fake offset: */ pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; - page = pages[pgoff]; + pfn = page_to_pfn(pages[pgoff]); VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address, - page_to_pfn(page), page_to_pfn(page) << PAGE_SHIFT); + pfn, pfn << PAGE_SHIFT); - return vmf_insert_page(vma, vmf->address, page); + return vmf_insert_pfn(vma, vmf->address, pfn); } int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset) @@ -222,30 +214,18 @@ int etnaviv_gem_mmap_offset(struct drm_gem_object *obj, u64 *offset) static struct etnaviv_vram_mapping * etnaviv_gem_get_vram_mapping(struct etnaviv_gem_object *obj, - struct etnaviv_iommu *mmu) + struct etnaviv_iommu_context *context) { struct etnaviv_vram_mapping *mapping; list_for_each_entry(mapping, &obj->vram_list, obj_node) { - if (mapping->mmu == mmu) + if (mapping->context == context) return mapping; } return NULL; } -void etnaviv_gem_mapping_reference(struct etnaviv_vram_mapping *mapping) -{ - struct etnaviv_gem_object *etnaviv_obj = mapping->object; - - drm_gem_object_get(&etnaviv_obj->base); - - mutex_lock(&etnaviv_obj->lock); - WARN_ON(mapping->use == 0); - mapping->use += 1; - mutex_unlock(&etnaviv_obj->lock); -} - void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping) { struct etnaviv_gem_object *etnaviv_obj = mapping->object; @@ -255,11 +235,12 @@ void etnaviv_gem_mapping_unreference(struct etnaviv_vram_mapping *mapping) mapping->use -= 1; mutex_unlock(&etnaviv_obj->lock); - drm_gem_object_put_unlocked(&etnaviv_obj->base); + drm_gem_object_put(&etnaviv_obj->base); } struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( - struct drm_gem_object *obj, struct etnaviv_gpu *gpu) + struct drm_gem_object *obj, struct etnaviv_iommu_context *mmu_context, + u64 va) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); struct etnaviv_vram_mapping *mapping; @@ -267,7 +248,7 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( int ret = 0; mutex_lock(&etnaviv_obj->lock); - mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, gpu->mmu); + mapping = etnaviv_gem_get_vram_mapping(etnaviv_obj, mmu_context); if (mapping) { /* * Holding the object lock prevents the use count changing @@ -276,12 +257,17 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( * the MMU owns this mapping to close this race. */ if (mapping->use == 0) { - mutex_lock(&gpu->mmu->lock); - if (mapping->mmu == gpu->mmu) - mapping->use += 1; + mutex_lock(&mmu_context->lock); + if (mapping->context == mmu_context) + if (va && mapping->iova != va) { + etnaviv_iommu_reap_mapping(mapping); + mapping = NULL; + } else { + mapping->use += 1; + } else mapping = NULL; - mutex_unlock(&gpu->mmu->lock); + mutex_unlock(&mmu_context->lock); if (mapping) goto out; } else { @@ -314,11 +300,11 @@ struct etnaviv_vram_mapping *etnaviv_gem_mapping_get( list_del(&mapping->obj_node); } - mapping->mmu = gpu->mmu; mapping->use = 1; - ret = etnaviv_iommu_map_gem(gpu->mmu, etnaviv_obj, gpu->memory_base, - mapping); + ret = etnaviv_iommu_map_gem(mmu_context, etnaviv_obj, + mmu_context->global->memory_base, + mapping, va); if (ret < 0) kfree(mapping); else @@ -357,6 +343,7 @@ void *etnaviv_gem_vmap(struct drm_gem_object *obj) static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj) { struct page **pages; + pgprot_t prot; lockdep_assert_held(&obj->lock); @@ -364,22 +351,35 @@ static void *etnaviv_gem_vmap_impl(struct etnaviv_gem_object *obj) if (IS_ERR(pages)) return NULL; - return vmap(pages, obj->base.size >> PAGE_SHIFT, - VM_MAP, pgprot_writecombine(PAGE_KERNEL)); + switch (obj->flags & ETNA_BO_CACHE_MASK) { + case ETNA_BO_CACHED: + prot = PAGE_KERNEL; + break; + case ETNA_BO_UNCACHED: + prot = pgprot_noncached(PAGE_KERNEL); + break; + case ETNA_BO_WC: + default: + prot = pgprot_writecombine(PAGE_KERNEL); + } + + return vmap(pages, obj->base.size >> PAGE_SHIFT, VM_MAP, prot); } static inline enum dma_data_direction etnaviv_op_to_dma_dir(u32 op) { - if (op & ETNA_PREP_READ) + op &= ETNA_PREP_READ | ETNA_PREP_WRITE; + + if (op == ETNA_PREP_READ) return DMA_FROM_DEVICE; - else if (op & ETNA_PREP_WRITE) + else if (op == ETNA_PREP_WRITE) return DMA_TO_DEVICE; else return DMA_BIDIRECTIONAL; } int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op, - struct timespec *timeout) + struct drm_etnaviv_timespec *timeout) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); struct drm_device *dev = obj->dev; @@ -397,22 +397,21 @@ int etnaviv_gem_cpu_prep(struct drm_gem_object *obj, u32 op, } if (op & ETNA_PREP_NOSYNC) { - if (!reservation_object_test_signaled_rcu(etnaviv_obj->resv, - write)) + if (!dma_resv_test_signaled(obj->resv, + dma_resv_usage_rw(write))) return -EBUSY; } else { unsigned long remain = etnaviv_timeout_to_jiffies(timeout); - ret = reservation_object_wait_timeout_rcu(etnaviv_obj->resv, - write, true, remain); + ret = dma_resv_wait_timeout(obj->resv, dma_resv_usage_rw(write), + true, remain); if (ret <= 0) return ret == 0 ? -ETIMEDOUT : ret; } if (etnaviv_obj->flags & ETNA_BO_CACHED) { - dma_sync_sg_for_cpu(dev->dev, etnaviv_obj->sgt->sgl, - etnaviv_obj->sgt->nents, - etnaviv_op_to_dma_dir(op)); + dma_sync_sgtable_for_cpu(dev->dev, etnaviv_obj->sgt, + etnaviv_op_to_dma_dir(op)); etnaviv_obj->last_cpu_prep_op = op; } @@ -427,8 +426,7 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj) if (etnaviv_obj->flags & ETNA_BO_CACHED) { /* fini without a prep is almost certainly a userspace error */ WARN_ON(etnaviv_obj->last_cpu_prep_op == 0); - dma_sync_sg_for_device(dev->dev, etnaviv_obj->sgt->sgl, - etnaviv_obj->sgt->nents, + dma_sync_sgtable_for_device(dev->dev, etnaviv_obj->sgt, etnaviv_op_to_dma_dir(etnaviv_obj->last_cpu_prep_op)); etnaviv_obj->last_cpu_prep_op = 0; } @@ -437,7 +435,7 @@ int etnaviv_gem_cpu_fini(struct drm_gem_object *obj) } int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, - struct timespec *timeout) + struct drm_etnaviv_timespec *timeout) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); @@ -445,45 +443,24 @@ int etnaviv_gem_wait_bo(struct etnaviv_gpu *gpu, struct drm_gem_object *obj, } #ifdef CONFIG_DEBUG_FS -static void etnaviv_gem_describe_fence(struct dma_fence *fence, - const char *type, struct seq_file *m) -{ - if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) - seq_printf(m, "\t%9s: %s %s seq %u\n", - type, - fence->ops->get_driver_name(fence), - fence->ops->get_timeline_name(fence), - fence->seqno); -} - static void etnaviv_gem_describe(struct drm_gem_object *obj, struct seq_file *m) { struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); - struct reservation_object *robj = etnaviv_obj->resv; - struct reservation_object_list *fobj; - struct dma_fence *fence; + struct dma_resv *robj = obj->resv; unsigned long off = drm_vma_node_start(&obj->vma_node); + int r; seq_printf(m, "%08x: %c %2d (%2d) %08lx %p %zd\n", etnaviv_obj->flags, is_active(etnaviv_obj) ? 'A' : 'I', obj->name, kref_read(&obj->refcount), off, etnaviv_obj->vaddr, obj->size); - rcu_read_lock(); - fobj = rcu_dereference(robj->fence); - if (fobj) { - unsigned int i, shared_count = fobj->shared_count; - - for (i = 0; i < shared_count; i++) { - fence = rcu_dereference(fobj->shared[i]); - etnaviv_gem_describe_fence(fence, "Shared", m); - } - } + r = dma_resv_lock(robj, NULL); + if (r) + return; - fence = rcu_dereference(robj->fence_excl); - if (fence) - etnaviv_gem_describe_fence(fence, "Exclusive", m); - rcu_read_unlock(); + dma_resv_describe(robj, m); + dma_resv_unlock(robj); } void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv, @@ -536,23 +513,21 @@ void etnaviv_gem_free_object(struct drm_gem_object *obj) list_for_each_entry_safe(mapping, tmp, &etnaviv_obj->vram_list, obj_node) { - struct etnaviv_iommu *mmu = mapping->mmu; + struct etnaviv_iommu_context *context = mapping->context; WARN_ON(mapping->use); - if (mmu) - etnaviv_iommu_unmap_gem(mmu, mapping); + if (context) + etnaviv_iommu_unmap_gem(context, mapping); list_del(&mapping->obj_node); kfree(mapping); } - drm_gem_free_mmap_offset(obj); etnaviv_obj->ops->release(etnaviv_obj); - if (etnaviv_obj->resv == &etnaviv_obj->_resv) - reservation_object_fini(&etnaviv_obj->_resv); drm_gem_object_release(obj); + mutex_destroy(&etnaviv_obj->lock); kfree(etnaviv_obj); } @@ -566,9 +541,36 @@ void etnaviv_gem_obj_add(struct drm_device *dev, struct drm_gem_object *obj) mutex_unlock(&priv->gem_lock); } +static enum drm_gem_object_status etnaviv_gem_status(struct drm_gem_object *obj) +{ + struct etnaviv_gem_object *etnaviv_obj = to_etnaviv_bo(obj); + enum drm_gem_object_status status = 0; + + if (etnaviv_obj->pages) + status |= DRM_GEM_OBJECT_RESIDENT; + + return status; +} + +static const struct vm_operations_struct vm_ops = { + .fault = etnaviv_gem_fault, + .open = drm_gem_vm_open, + .close = drm_gem_vm_close, +}; + +static const struct drm_gem_object_funcs etnaviv_gem_object_funcs = { + .free = etnaviv_gem_free_object, + .pin = etnaviv_gem_prime_pin, + .unpin = etnaviv_gem_prime_unpin, + .get_sg_table = etnaviv_gem_prime_get_sg_table, + .vmap = etnaviv_gem_prime_vmap, + .mmap = etnaviv_gem_mmap, + .status = etnaviv_gem_status, + .vm_ops = &vm_ops, +}; + static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, - struct reservation_object *robj, const struct etnaviv_gem_ops *ops, - struct drm_gem_object **obj) + const struct etnaviv_gem_ops *ops, struct drm_gem_object **obj) { struct etnaviv_gem_object *etnaviv_obj; unsigned sz = sizeof(*etnaviv_obj); @@ -594,19 +596,15 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, if (!etnaviv_obj) return -ENOMEM; + etnaviv_obj->size = ALIGN(size, SZ_4K); etnaviv_obj->flags = flags; etnaviv_obj->ops = ops; - if (robj) { - etnaviv_obj->resv = robj; - } else { - etnaviv_obj->resv = &etnaviv_obj->_resv; - reservation_object_init(&etnaviv_obj->_resv); - } mutex_init(&etnaviv_obj->lock); INIT_LIST_HEAD(&etnaviv_obj->vram_list); *obj = &etnaviv_obj->base; + (*obj)->funcs = &etnaviv_gem_object_funcs; return 0; } @@ -615,56 +613,46 @@ static int etnaviv_gem_new_impl(struct drm_device *dev, u32 size, u32 flags, int etnaviv_gem_new_handle(struct drm_device *dev, struct drm_file *file, u32 size, u32 flags, u32 *handle) { + struct etnaviv_drm_private *priv = dev->dev_private; struct drm_gem_object *obj = NULL; int ret; - size = PAGE_ALIGN(size); - - ret = etnaviv_gem_new_impl(dev, size, flags, NULL, - &etnaviv_gem_shmem_ops, &obj); + ret = etnaviv_gem_new_impl(dev, size, flags, &etnaviv_gem_shmem_ops, &obj); if (ret) goto fail; lockdep_set_class(&to_etnaviv_bo(obj)->lock, &etnaviv_shm_lock_class); - ret = drm_gem_object_init(dev, obj, size); - if (ret == 0) { - struct address_space *mapping; - - /* - * Our buffers are kept pinned, so allocating them - * from the MOVABLE zone is a really bad idea, and - * conflicts with CMA. See comments above new_inode() - * why this is required _and_ expected if you're - * going to pin these pages. - */ - mapping = obj->filp->f_mapping; - mapping_set_gfp_mask(mapping, GFP_HIGHUSER | - __GFP_RETRY_MAYFAIL | __GFP_NOWARN); - } - + ret = drm_gem_object_init(dev, obj, PAGE_ALIGN(size)); if (ret) goto fail; + /* + * Our buffers are kept pinned, so allocating them from the MOVABLE + * zone is a really bad idea, and conflicts with CMA. See comments + * above new_inode() why this is required _and_ expected if you're + * going to pin these pages. + */ + mapping_set_gfp_mask(obj->filp->f_mapping, priv->shm_gfp_mask); + etnaviv_gem_obj_add(dev, obj); ret = drm_gem_handle_create(file, obj, handle); /* drop reference from allocate - handle holds it now */ fail: - drm_gem_object_put_unlocked(obj); + drm_gem_object_put(obj); return ret; } int etnaviv_gem_new_private(struct drm_device *dev, size_t size, u32 flags, - struct reservation_object *robj, const struct etnaviv_gem_ops *ops, - struct etnaviv_gem_object **res) + const struct etnaviv_gem_ops *ops, struct etnaviv_gem_object **res) { struct drm_gem_object *obj; int ret; - ret = etnaviv_gem_new_impl(dev, size, flags, robj, ops, &obj); + ret = etnaviv_gem_new_impl(dev, size, flags, ops, &obj); if (ret) return ret; @@ -680,8 +668,9 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) struct page **pvec = NULL; struct etnaviv_gem_userptr *userptr = &etnaviv_obj->userptr; int ret, pinned = 0, npages = etnaviv_obj->base.size >> PAGE_SHIFT; + unsigned int gup_flags = FOLL_LONGTERM; - might_lock_read(¤t->mm->mmap_sem); + might_lock_read(¤t->mm->mmap_lock); if (userptr->mm != current->mm) return -EPERM; @@ -690,15 +679,17 @@ static int etnaviv_gem_userptr_get_pages(struct etnaviv_gem_object *etnaviv_obj) if (!pvec) return -ENOMEM; + if (!userptr->ro) + gup_flags |= FOLL_WRITE; + do { unsigned num_pages = npages - pinned; uint64_t ptr = userptr->ptr + pinned * PAGE_SIZE; struct page **pages = pvec + pinned; - ret = get_user_pages_fast(ptr, num_pages, - !userptr->ro ? FOLL_WRITE : 0, pages); + ret = pin_user_pages_fast(ptr, num_pages, gup_flags, pages); if (ret < 0) { - release_pages(pvec, pinned); + unpin_user_pages(pvec, pinned); kvfree(pvec); return ret; } @@ -720,9 +711,9 @@ static void etnaviv_gem_userptr_release(struct etnaviv_gem_object *etnaviv_obj) kfree(etnaviv_obj->sgt); } if (etnaviv_obj->pages) { - int npages = etnaviv_obj->base.size >> PAGE_SHIFT; + unsigned int npages = etnaviv_obj->base.size >> PAGE_SHIFT; - release_pages(etnaviv_obj->pages, npages); + unpin_user_pages(etnaviv_obj->pages, npages); kvfree(etnaviv_obj->pages); } } @@ -746,7 +737,7 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, struct etnaviv_gem_object *etnaviv_obj; int ret; - ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, NULL, + ret = etnaviv_gem_new_private(dev, size, ETNA_BO_CACHED, &etnaviv_gem_userptr_ops, &etnaviv_obj); if (ret) return ret; @@ -762,6 +753,6 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle); /* drop reference from allocate - handle holds it now */ - drm_gem_object_put_unlocked(&etnaviv_obj->base); + drm_gem_object_put(&etnaviv_obj->base); return ret; } |
