summaryrefslogtreecommitdiff
path: root/lib/scatterlist.c
diff options
context:
space:
mode:
Diffstat (limited to 'lib/scatterlist.c')
-rw-r--r--lib/scatterlist.c110
1 files changed, 78 insertions, 32 deletions
diff --git a/lib/scatterlist.c b/lib/scatterlist.c
index 68b45c82c37a..7582dfab7fe3 100644
--- a/lib/scatterlist.c
+++ b/lib/scatterlist.c
@@ -11,29 +11,7 @@
#include <linux/kmemleak.h>
#include <linux/bvec.h>
#include <linux/uio.h>
-
-/**
- * sg_next - return the next scatterlist entry in a list
- * @sg: The current sg entry
- *
- * Description:
- * Usually the next entry will be @sg@ + 1, but if this sg element is part
- * of a chained scatterlist, it could jump to the start of a new
- * scatterlist array.
- *
- **/
-struct scatterlist *sg_next(struct scatterlist *sg)
-{
- if (sg_is_last(sg))
- return NULL;
-
- sg++;
- if (unlikely(sg_is_chain(sg)))
- sg = sg_chain_ptr(sg);
-
- return sg;
-}
-EXPORT_SYMBOL(sg_next);
+#include <linux/folio_queue.h>
/**
* sg_nents - return total count of entries in scatterlist
@@ -473,14 +451,14 @@ int sg_alloc_append_table_from_pages(struct sg_append_table *sgt_append,
return -EOPNOTSUPP;
if (sgt_append->prv) {
- unsigned long next_pfn = (page_to_phys(sg_page(sgt_append->prv)) +
- sgt_append->prv->offset + sgt_append->prv->length) / PAGE_SIZE;
+ unsigned long next_pfn;
if (WARN_ON(offset))
return -EINVAL;
/* Merge contiguous pages into the last SG */
prv_len = sgt_append->prv->length;
+ next_pfn = (sg_phys(sgt_append->prv) + prv_len) / PAGE_SIZE;
if (page_to_pfn(pages[0]) == next_pfn) {
last_pg = pfn_to_page(next_pfn - 1);
while (n_pages && pages_are_mergeable(pages[0], last_pg)) {
@@ -878,7 +856,7 @@ EXPORT_SYMBOL(sg_miter_skip);
* @miter->addr and @miter->length point to the current mapping.
*
* Context:
- * May sleep if !SG_MITER_ATOMIC.
+ * May sleep if !SG_MITER_ATOMIC && !SG_MITER_LOCAL.
*
* Returns:
* true if @miter contains the next mapping. false if end of sg
@@ -900,6 +878,8 @@ bool sg_miter_next(struct sg_mapping_iter *miter)
if (miter->__flags & SG_MITER_ATOMIC)
miter->addr = kmap_atomic(miter->page) + miter->__offset;
+ else if (miter->__flags & SG_MITER_LOCAL)
+ miter->addr = kmap_local_page(miter->page) + miter->__offset;
else
miter->addr = kmap(miter->page) + miter->__offset;
@@ -935,7 +915,9 @@ void sg_miter_stop(struct sg_mapping_iter *miter)
if (miter->__flags & SG_MITER_ATOMIC) {
WARN_ON_ONCE(!pagefault_disabled());
kunmap_atomic(miter->addr);
- } else
+ } else if (miter->__flags & SG_MITER_LOCAL)
+ kunmap_local(miter->addr);
+ else
kunmap(miter->page);
miter->page = NULL;
@@ -964,7 +946,7 @@ size_t sg_copy_buffer(struct scatterlist *sgl, unsigned int nents, void *buf,
{
unsigned int offset = 0;
struct sg_mapping_iter miter;
- unsigned int sg_flags = SG_MITER_ATOMIC;
+ unsigned int sg_flags = SG_MITER_LOCAL;
if (to_buffer)
sg_flags |= SG_MITER_FROM_SG;
@@ -1079,7 +1061,7 @@ size_t sg_zero_buffer(struct scatterlist *sgl, unsigned int nents,
{
unsigned int offset = 0;
struct sg_mapping_iter miter;
- unsigned int sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
+ unsigned int sg_flags = SG_MITER_LOCAL | SG_MITER_TO_SG;
sg_miter_start(&miter, sgl, nents, sg_flags);
@@ -1124,7 +1106,7 @@ static ssize_t extract_user_to_sg(struct iov_iter *iter,
do {
res = iov_iter_extract_pages(iter, &pages, maxsize, sg_max,
extraction_flags, &off);
- if (res < 0)
+ if (res <= 0)
goto failed;
len = res;
@@ -1262,6 +1244,67 @@ static ssize_t extract_kvec_to_sg(struct iov_iter *iter,
}
/*
+ * Extract up to sg_max folios from an FOLIOQ-type iterator and add them to
+ * the scatterlist. The pages are not pinned.
+ */
+static ssize_t extract_folioq_to_sg(struct iov_iter *iter,
+ ssize_t maxsize,
+ struct sg_table *sgtable,
+ unsigned int sg_max,
+ iov_iter_extraction_t extraction_flags)
+{
+ const struct folio_queue *folioq = iter->folioq;
+ struct scatterlist *sg = sgtable->sgl + sgtable->nents;
+ unsigned int slot = iter->folioq_slot;
+ ssize_t ret = 0;
+ size_t offset = iter->iov_offset;
+
+ BUG_ON(!folioq);
+
+ if (slot >= folioq_nr_slots(folioq)) {
+ folioq = folioq->next;
+ if (WARN_ON_ONCE(!folioq))
+ return 0;
+ slot = 0;
+ }
+
+ do {
+ struct folio *folio = folioq_folio(folioq, slot);
+ size_t fsize = folioq_folio_size(folioq, slot);
+
+ if (offset < fsize) {
+ size_t part = umin(maxsize - ret, fsize - offset);
+
+ sg_set_page(sg, folio_page(folio, 0), part, offset);
+ sgtable->nents++;
+ sg++;
+ sg_max--;
+ offset += part;
+ ret += part;
+ }
+
+ if (offset >= fsize) {
+ offset = 0;
+ slot++;
+ if (slot >= folioq_nr_slots(folioq)) {
+ if (!folioq->next) {
+ WARN_ON_ONCE(ret < iter->count);
+ break;
+ }
+ folioq = folioq->next;
+ slot = 0;
+ }
+ }
+ } while (sg_max > 0 && ret < maxsize);
+
+ iter->folioq = folioq;
+ iter->folioq_slot = slot;
+ iter->iov_offset = offset;
+ iter->count -= ret;
+ return ret;
+}
+
+/*
* Extract up to sg_max folios from an XARRAY-type iterator and add them to
* the scatterlist. The pages are not pinned.
*/
@@ -1323,8 +1366,8 @@ static ssize_t extract_xarray_to_sg(struct iov_iter *iter,
* addition of @sg_max elements.
*
* The pages referred to by UBUF- and IOVEC-type iterators are extracted and
- * pinned; BVEC-, KVEC- and XARRAY-type are extracted but aren't pinned; PIPE-
- * and DISCARD-type are not supported.
+ * pinned; BVEC-, KVEC-, FOLIOQ- and XARRAY-type are extracted but aren't
+ * pinned; DISCARD-type is not supported.
*
* No end mark is placed on the scatterlist; that's left to the caller.
*
@@ -1356,6 +1399,9 @@ ssize_t extract_iter_to_sg(struct iov_iter *iter, size_t maxsize,
case ITER_KVEC:
return extract_kvec_to_sg(iter, maxsize, sgtable, sg_max,
extraction_flags);
+ case ITER_FOLIOQ:
+ return extract_folioq_to_sg(iter, maxsize, sgtable, sg_max,
+ extraction_flags);
case ITER_XARRAY:
return extract_xarray_to_sg(iter, maxsize, sgtable, sg_max,
extraction_flags);