summaryrefslogtreecommitdiff
path: root/block/fops.c
diff options
context:
space:
mode:
authorJens Axboe <axboe@kernel.dk>2021-10-28 08:57:09 -0600
committerJens Axboe <axboe@kernel.dk>2021-12-03 14:51:29 -0700
commitceaa762527f41a431b552bc000de4b626d2d8cb7 (patch)
tree74e9ff872d0cbfe0373c25edb60840d77b06c245 /block/fops.c
parent4bdcd1dd4d2f973b1a89fb20ba720d879e9e506b (diff)
block: move direct_IO into our own read_iter handler
Don't call into generic_file_read_iter() if we know it's O_DIRECT, just set it up ourselves and call our own handler. This avoids an indirect call for O_DIRECT. Fall back to filemap_read() if we fail. Signed-off-by: Jens Axboe <axboe@kernel.dk>
Diffstat (limited to 'block/fops.c')
-rw-r--r--block/fops.c37
1 files changed, 32 insertions, 5 deletions
diff --git a/block/fops.c b/block/fops.c
index ad732a36f9b3..4d0e220f379e 100644
--- a/block/fops.c
+++ b/block/fops.c
@@ -566,21 +566,48 @@ static ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
{
struct block_device *bdev = iocb->ki_filp->private_data;
loff_t size = bdev_nr_bytes(bdev);
+ size_t count = iov_iter_count(to);
loff_t pos = iocb->ki_pos;
size_t shorted = 0;
- ssize_t ret;
+ ssize_t ret = 0;
- if (unlikely(pos + iov_iter_count(to) > size)) {
+ if (unlikely(pos + count > size)) {
if (pos >= size)
return 0;
size -= pos;
- if (iov_iter_count(to) > size) {
- shorted = iov_iter_count(to) - size;
+ if (count > size) {
+ shorted = count - size;
iov_iter_truncate(to, size);
}
}
- ret = generic_file_read_iter(iocb, to);
+ if (iocb->ki_flags & IOCB_DIRECT) {
+ struct address_space *mapping = iocb->ki_filp->f_mapping;
+
+ if (iocb->ki_flags & IOCB_NOWAIT) {
+ if (filemap_range_needs_writeback(mapping, iocb->ki_pos,
+ iocb->ki_pos + count - 1))
+ return -EAGAIN;
+ } else {
+ ret = filemap_write_and_wait_range(mapping,
+ iocb->ki_pos,
+ iocb->ki_pos + count - 1);
+ if (ret < 0)
+ return ret;
+ }
+
+ file_accessed(iocb->ki_filp);
+
+ ret = blkdev_direct_IO(iocb, to);
+ if (ret >= 0) {
+ iocb->ki_pos += ret;
+ count -= ret;
+ }
+ if (ret < 0 || !count)
+ return ret;
+ }
+
+ ret = filemap_read(iocb, to, ret);
if (unlikely(shorted))
iov_iter_reexpand(to, iov_iter_count(to) + shorted);