diff options
| -rw-r--r-- | drivers/infiniband/core/umem_odp.c | 35 | ||||
| -rw-r--r-- | drivers/infiniband/hw/mlx5/odp.c | 2 | ||||
| -rw-r--r-- | include/rdma/ib_umem_odp.h | 2 |
3 files changed, 27 insertions, 12 deletions
diff --git a/drivers/infiniband/core/umem_odp.c b/drivers/infiniband/core/umem_odp.c index ad3b2ec688e8..323f6cf00682 100644 --- a/drivers/infiniband/core/umem_odp.c +++ b/drivers/infiniband/core/umem_odp.c @@ -347,9 +347,10 @@ static int ib_umem_odp_map_dma_single_page( * the return value. * @access_mask: bit mask of the requested access permissions for the given * range. + * @fault: is faulting required for the given range */ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, - u64 bcnt, u64 access_mask) + u64 bcnt, u64 access_mask, bool fault) __acquires(&umem_odp->umem_mutex) { struct task_struct *owning_process = NULL; @@ -385,10 +386,12 @@ int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 user_virt, range.end = ALIGN(user_virt + bcnt, 1UL << page_shift); pfn_start_idx = (range.start - ib_umem_start(umem_odp)) >> PAGE_SHIFT; num_pfns = (range.end - range.start) >> PAGE_SHIFT; - range.default_flags = HMM_PFN_REQ_FAULT; + if (fault) { + range.default_flags = HMM_PFN_REQ_FAULT; - if (access_mask & ODP_WRITE_ALLOWED_BIT) - range.default_flags |= HMM_PFN_REQ_WRITE; + if (access_mask & ODP_WRITE_ALLOWED_BIT) + range.default_flags |= HMM_PFN_REQ_WRITE; + } range.hmm_pfns = &(umem_odp->pfn_list[pfn_start_idx]); timeout = jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT); @@ -417,12 +420,24 @@ retry: for (pfn_index = 0; pfn_index < num_pfns; pfn_index += 1 << (page_shift - PAGE_SHIFT), dma_index++) { - /* - * Since we asked for hmm_range_fault() to populate pages, - * it shouldn't return an error entry on success. - */ - WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); - WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); + + if (fault) { + /* + * Since we asked for hmm_range_fault() to populate + * pages it shouldn't return an error entry on success. + */ + WARN_ON(range.hmm_pfns[pfn_index] & HMM_PFN_ERROR); + WARN_ON(!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)); + } else { + if (!(range.hmm_pfns[pfn_index] & HMM_PFN_VALID)) { + WARN_ON(umem_odp->dma_list[dma_index]); + continue; + } + access_mask = ODP_READ_ALLOWED_BIT; + if (range.hmm_pfns[pfn_index] & HMM_PFN_WRITE) + access_mask |= ODP_WRITE_ALLOWED_BIT; + } + hmm_order = hmm_pfn_to_map_order(range.hmm_pfns[pfn_index]); /* If a hugepage was detected and ODP wasn't set for, the umem * page_shift will be used, the opposite case is an error. diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index cca9e85b2ea4..4743ef78afe0 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -681,7 +681,7 @@ static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, if (odp->umem.writable && !downgrade) access_mask |= ODP_WRITE_ALLOWED_BIT; - np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask); + np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, true); if (np < 0) return np; diff --git a/include/rdma/ib_umem_odp.h b/include/rdma/ib_umem_odp.h index a53b62ac8a9d..0844c1d05ac6 100644 --- a/include/rdma/ib_umem_odp.h +++ b/include/rdma/ib_umem_odp.h @@ -94,7 +94,7 @@ ib_umem_odp_alloc_child(struct ib_umem_odp *root_umem, unsigned long addr, void ib_umem_odp_release(struct ib_umem_odp *umem_odp); int ib_umem_odp_map_dma_and_lock(struct ib_umem_odp *umem_odp, u64 start_offset, - u64 bcnt, u64 access_mask); + u64 bcnt, u64 access_mask, bool fault); void ib_umem_odp_unmap_dma_pages(struct ib_umem_odp *umem_odp, u64 start_offset, u64 bound); |