summaryrefslogtreecommitdiff
path: root/fs/verity
diff options
context:
space:
mode:
Diffstat (limited to 'fs/verity')
-rw-r--r--fs/verity/enable.c18
-rw-r--r--fs/verity/fsverity_private.h11
-rw-r--r--fs/verity/hash_algs.c3
-rw-r--r--fs/verity/open.c23
-rw-r--r--fs/verity/verify.c177
5 files changed, 168 insertions, 64 deletions
diff --git a/fs/verity/enable.c b/fs/verity/enable.c
index 503268cf4296..95ec42b84797 100644
--- a/fs/verity/enable.c
+++ b/fs/verity/enable.c
@@ -19,8 +19,7 @@ struct block_buffer {
};
/* Hash a block, writing the result to the next level's pending block buffer. */
-static int hash_one_block(struct inode *inode,
- const struct merkle_tree_params *params,
+static int hash_one_block(const struct merkle_tree_params *params,
struct block_buffer *cur)
{
struct block_buffer *next = cur + 1;
@@ -36,8 +35,7 @@ static int hash_one_block(struct inode *inode,
/* Zero-pad the block if it's shorter than the block size. */
memset(&cur->data[cur->filled], 0, params->block_size - cur->filled);
- fsverity_hash_block(params, inode, cur->data,
- &next->data[next->filled]);
+ fsverity_hash_block(params, cur->data, &next->data[next->filled]);
next->filled += params->digest_size;
cur->filled = 0;
return 0;
@@ -123,7 +121,7 @@ static int build_merkle_tree(struct file *filp,
fsverity_err(inode, "Short read of file data");
goto out;
}
- err = hash_one_block(inode, params, &buffers[-1]);
+ err = hash_one_block(params, &buffers[-1]);
if (err)
goto out;
for (level = 0; level < num_levels; level++) {
@@ -134,7 +132,7 @@ static int build_merkle_tree(struct file *filp,
}
/* Next block at @level is full */
- err = hash_one_block(inode, params, &buffers[level]);
+ err = hash_one_block(params, &buffers[level]);
if (err)
goto out;
err = write_merkle_tree_block(inode,
@@ -154,7 +152,7 @@ static int build_merkle_tree(struct file *filp,
/* Finish all nonempty pending tree blocks. */
for (level = 0; level < num_levels; level++) {
if (buffers[level].filled != 0) {
- err = hash_one_block(inode, params, &buffers[level]);
+ err = hash_one_block(params, &buffers[level]);
if (err)
goto out;
err = write_merkle_tree_block(inode,
@@ -284,9 +282,9 @@ static int enable_verity(struct file *filp,
/* Successfully enabled verity */
/*
- * Readers can start using ->i_verity_info immediately, so it
- * can't be rolled back once set. So don't set it until just
- * after the filesystem has successfully enabled verity.
+ * Readers can start using the inode's verity info immediately,
+ * so it can't be rolled back once set. So don't set it until
+ * just after the filesystem has successfully enabled verity.
*/
fsverity_set_info(inode, vi);
}
diff --git a/fs/verity/fsverity_private.h b/fs/verity/fsverity_private.h
index 5fe854a5b9ad..dd20b138d452 100644
--- a/fs/verity/fsverity_private.h
+++ b/fs/verity/fsverity_private.h
@@ -63,10 +63,11 @@ struct merkle_tree_params {
* fsverity_info - cached verity metadata for an inode
*
* When a verity file is first opened, an instance of this struct is allocated
- * and stored in ->i_verity_info; it remains until the inode is evicted. It
- * caches information about the Merkle tree that's needed to efficiently verify
- * data read from the file. It also caches the file digest. The Merkle tree
- * pages themselves are not cached here, but the filesystem may cache them.
+ * and a pointer to it is stored in the file's in-memory inode. It remains
+ * until the inode is evicted. It caches information about the Merkle tree
+ * that's needed to efficiently verify data read from the file. It also caches
+ * the file digest. The Merkle tree pages themselves are not cached here, but
+ * the filesystem may cache them.
*/
struct fsverity_info {
struct merkle_tree_params tree_params;
@@ -89,7 +90,7 @@ union fsverity_hash_ctx *
fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
const u8 *salt, size_t salt_size);
void fsverity_hash_block(const struct merkle_tree_params *params,
- const struct inode *inode, const void *data, u8 *out);
+ const void *data, u8 *out);
void fsverity_hash_buffer(const struct fsverity_hash_alg *alg,
const void *data, size_t size, u8 *out);
void __init fsverity_check_hash_algs(void);
diff --git a/fs/verity/hash_algs.c b/fs/verity/hash_algs.c
index 9bb3c6344907..de53e14c8aa7 100644
--- a/fs/verity/hash_algs.c
+++ b/fs/verity/hash_algs.c
@@ -94,7 +94,6 @@ fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
/**
* fsverity_hash_block() - hash a single data or hash block
* @params: the Merkle tree's parameters
- * @inode: inode for which the hashing is being done
* @data: virtual address of a buffer containing the block to hash
* @out: output digest, size 'params->digest_size' bytes
*
@@ -102,7 +101,7 @@ fsverity_prepare_hash_state(const struct fsverity_hash_alg *alg,
* in the Merkle tree parameters.
*/
void fsverity_hash_block(const struct merkle_tree_params *params,
- const struct inode *inode, const void *data, u8 *out)
+ const void *data, u8 *out)
{
union fsverity_hash_ctx ctx;
diff --git a/fs/verity/open.c b/fs/verity/open.c
index c561e130cd0c..77b1c977af02 100644
--- a/fs/verity/open.c
+++ b/fs/verity/open.c
@@ -244,17 +244,17 @@ fail:
void fsverity_set_info(struct inode *inode, struct fsverity_info *vi)
{
/*
- * Multiple tasks may race to set ->i_verity_info, so use
- * cmpxchg_release(). This pairs with the smp_load_acquire() in
- * fsverity_get_info(). I.e., here we publish ->i_verity_info with a
- * RELEASE barrier so that other tasks can ACQUIRE it.
+ * Multiple tasks may race to set the inode's verity info pointer, so
+ * use cmpxchg_release(). This pairs with the smp_load_acquire() in
+ * fsverity_get_info(). I.e., publish the pointer with a RELEASE
+ * barrier so that other tasks can ACQUIRE it.
*/
- if (cmpxchg_release(&inode->i_verity_info, NULL, vi) != NULL) {
- /* Lost the race, so free the fsverity_info we allocated. */
+ if (cmpxchg_release(fsverity_info_addr(inode), NULL, vi) != NULL) {
+ /* Lost the race, so free the verity info we allocated. */
fsverity_free_info(vi);
/*
- * Afterwards, the caller may access ->i_verity_info directly,
- * so make sure to ACQUIRE the winning fsverity_info.
+ * Afterwards, the caller may access the inode's verity info
+ * directly, so make sure to ACQUIRE the winning verity info.
*/
(void)fsverity_get_info(inode);
}
@@ -350,7 +350,6 @@ int fsverity_get_descriptor(struct inode *inode,
return 0;
}
-/* Ensure the inode has an ->i_verity_info */
static int ensure_verity_info(struct inode *inode)
{
struct fsverity_info *vi = fsverity_get_info(inode);
@@ -395,8 +394,10 @@ EXPORT_SYMBOL_GPL(__fsverity_prepare_setattr);
void __fsverity_cleanup_inode(struct inode *inode)
{
- fsverity_free_info(inode->i_verity_info);
- inode->i_verity_info = NULL;
+ struct fsverity_info **vi_addr = fsverity_info_addr(inode);
+
+ fsverity_free_info(*vi_addr);
+ *vi_addr = NULL;
}
EXPORT_SYMBOL_GPL(__fsverity_cleanup_inode);
diff --git a/fs/verity/verify.c b/fs/verity/verify.c
index a1f00c3fd3b2..86067c8b40cf 100644
--- a/fs/verity/verify.c
+++ b/fs/verity/verify.c
@@ -10,6 +10,31 @@
#include <linux/bio.h>
#include <linux/export.h>
+#define FS_VERITY_MAX_PENDING_BLOCKS 2
+
+struct fsverity_pending_block {
+ const void *data;
+ u64 pos;
+ u8 real_hash[FS_VERITY_MAX_DIGEST_SIZE];
+};
+
+struct fsverity_verification_context {
+ struct inode *inode;
+ struct fsverity_info *vi;
+ unsigned long max_ra_pages;
+
+ /*
+ * This is the queue of data blocks that are pending verification. When
+ * the crypto layer supports interleaved hashing, we allow multiple
+ * blocks to be queued up in order to utilize it. This can improve
+ * performance significantly vs. sequential hashing of each block.
+ */
+ int num_pending;
+ int max_pending;
+ struct fsverity_pending_block
+ pending_blocks[FS_VERITY_MAX_PENDING_BLOCKS];
+};
+
static struct workqueue_struct *fsverity_read_workqueue;
/*
@@ -79,7 +104,7 @@ static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage,
}
/*
- * Verify a single data block against the file's Merkle tree.
+ * Verify the hash of a single data block against the file's Merkle tree.
*
* In principle, we need to verify the entire path to the root node. However,
* for efficiency the filesystem may cache the hash blocks. Therefore we need
@@ -88,10 +113,11 @@ static bool is_hash_block_verified(struct fsverity_info *vi, struct page *hpage,
*
* Return: %true if the data block is valid, else %false.
*/
-static bool
-verify_data_block(struct inode *inode, struct fsverity_info *vi,
- const void *data, u64 data_pos, unsigned long max_ra_pages)
+static bool verify_data_block(struct inode *inode, struct fsverity_info *vi,
+ const struct fsverity_pending_block *dblock,
+ unsigned long max_ra_pages)
{
+ const u64 data_pos = dblock->pos;
const struct merkle_tree_params *params = &vi->tree_params;
const unsigned int hsize = params->digest_size;
int level;
@@ -115,8 +141,12 @@ verify_data_block(struct inode *inode, struct fsverity_info *vi,
*/
u64 hidx = data_pos >> params->log_blocksize;
- /* Up to 1 + FS_VERITY_MAX_LEVELS pages may be mapped at once */
- BUILD_BUG_ON(1 + FS_VERITY_MAX_LEVELS > KM_MAX_IDX);
+ /*
+ * Up to FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS pages may
+ * be mapped at once.
+ */
+ static_assert(FS_VERITY_MAX_PENDING_BLOCKS + FS_VERITY_MAX_LEVELS <=
+ KM_MAX_IDX);
if (unlikely(data_pos >= inode->i_size)) {
/*
@@ -127,7 +157,7 @@ verify_data_block(struct inode *inode, struct fsverity_info *vi,
* any part past EOF should be all zeroes. Therefore, we need
* to verify that any data blocks fully past EOF are all zeroes.
*/
- if (memchr_inv(data, 0, params->block_size)) {
+ if (memchr_inv(dblock->data, 0, params->block_size)) {
fsverity_err(inode,
"FILE CORRUPTED! Data past EOF is not zeroed");
return false;
@@ -202,7 +232,7 @@ descend:
unsigned long hblock_idx = hblocks[level - 1].index;
unsigned int hoffset = hblocks[level - 1].hoffset;
- fsverity_hash_block(params, inode, haddr, real_hash);
+ fsverity_hash_block(params, haddr, real_hash);
if (memcmp(want_hash, real_hash, hsize) != 0)
goto corrupted;
/*
@@ -220,18 +250,18 @@ descend:
put_page(hpage);
}
- /* Finally, verify the data block. */
- fsverity_hash_block(params, inode, data, real_hash);
- if (memcmp(want_hash, real_hash, hsize) != 0)
+ /* Finally, verify the hash of the data block. */
+ if (memcmp(want_hash, dblock->real_hash, hsize) != 0)
goto corrupted;
return true;
corrupted:
- fsverity_err(inode,
- "FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN",
- data_pos, level - 1,
- params->hash_alg->name, hsize, want_hash,
- params->hash_alg->name, hsize, real_hash);
+ fsverity_err(
+ inode,
+ "FILE CORRUPTED! pos=%llu, level=%d, want_hash=%s:%*phN, real_hash=%s:%*phN",
+ data_pos, level - 1, params->hash_alg->name, hsize, want_hash,
+ params->hash_alg->name, hsize,
+ level == 0 ? dblock->real_hash : real_hash);
error:
for (; level > 0; level--) {
kunmap_local(hblocks[level - 1].addr);
@@ -240,13 +270,73 @@ error:
return false;
}
+static void
+fsverity_init_verification_context(struct fsverity_verification_context *ctx,
+ struct inode *inode,
+ unsigned long max_ra_pages)
+{
+ struct fsverity_info *vi = *fsverity_info_addr(inode);
+
+ ctx->inode = inode;
+ ctx->vi = vi;
+ ctx->max_ra_pages = max_ra_pages;
+ ctx->num_pending = 0;
+ if (vi->tree_params.hash_alg->algo_id == HASH_ALGO_SHA256 &&
+ sha256_finup_2x_is_optimized())
+ ctx->max_pending = 2;
+ else
+ ctx->max_pending = 1;
+}
+
+static void
+fsverity_clear_pending_blocks(struct fsverity_verification_context *ctx)
+{
+ int i;
+
+ for (i = ctx->num_pending - 1; i >= 0; i--) {
+ kunmap_local(ctx->pending_blocks[i].data);
+ ctx->pending_blocks[i].data = NULL;
+ }
+ ctx->num_pending = 0;
+}
+
static bool
-verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
- unsigned long max_ra_pages)
+fsverity_verify_pending_blocks(struct fsverity_verification_context *ctx)
{
- struct inode *inode = data_folio->mapping->host;
- struct fsverity_info *vi = inode->i_verity_info;
- const unsigned int block_size = vi->tree_params.block_size;
+ struct fsverity_info *vi = ctx->vi;
+ const struct merkle_tree_params *params = &vi->tree_params;
+ int i;
+
+ if (ctx->num_pending == 2) {
+ /* num_pending == 2 implies that the algorithm is SHA-256 */
+ sha256_finup_2x(params->hashstate ? &params->hashstate->sha256 :
+ NULL,
+ ctx->pending_blocks[0].data,
+ ctx->pending_blocks[1].data, params->block_size,
+ ctx->pending_blocks[0].real_hash,
+ ctx->pending_blocks[1].real_hash);
+ } else {
+ for (i = 0; i < ctx->num_pending; i++)
+ fsverity_hash_block(params, ctx->pending_blocks[i].data,
+ ctx->pending_blocks[i].real_hash);
+ }
+
+ for (i = 0; i < ctx->num_pending; i++) {
+ if (!verify_data_block(ctx->inode, vi, &ctx->pending_blocks[i],
+ ctx->max_ra_pages))
+ return false;
+ }
+ fsverity_clear_pending_blocks(ctx);
+ return true;
+}
+
+static bool fsverity_add_data_blocks(struct fsverity_verification_context *ctx,
+ struct folio *data_folio, size_t len,
+ size_t offset)
+{
+ struct fsverity_info *vi = ctx->vi;
+ const struct merkle_tree_params *params = &vi->tree_params;
+ const unsigned int block_size = params->block_size;
u64 pos = (u64)data_folio->index << PAGE_SHIFT;
if (WARN_ON_ONCE(len <= 0 || !IS_ALIGNED(len | offset, block_size)))
@@ -255,14 +345,11 @@ verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
folio_test_uptodate(data_folio)))
return false;
do {
- void *data;
- bool valid;
-
- data = kmap_local_folio(data_folio, offset);
- valid = verify_data_block(inode, vi, data, pos + offset,
- max_ra_pages);
- kunmap_local(data);
- if (!valid)
+ ctx->pending_blocks[ctx->num_pending].data =
+ kmap_local_folio(data_folio, offset);
+ ctx->pending_blocks[ctx->num_pending].pos = pos + offset;
+ if (++ctx->num_pending == ctx->max_pending &&
+ !fsverity_verify_pending_blocks(ctx))
return false;
offset += block_size;
len -= block_size;
@@ -284,7 +371,15 @@ verify_data_blocks(struct folio *data_folio, size_t len, size_t offset,
*/
bool fsverity_verify_blocks(struct folio *folio, size_t len, size_t offset)
{
- return verify_data_blocks(folio, len, offset, 0);
+ struct fsverity_verification_context ctx;
+
+ fsverity_init_verification_context(&ctx, folio->mapping->host, 0);
+
+ if (fsverity_add_data_blocks(&ctx, folio, len, offset) &&
+ fsverity_verify_pending_blocks(&ctx))
+ return true;
+ fsverity_clear_pending_blocks(&ctx);
+ return false;
}
EXPORT_SYMBOL_GPL(fsverity_verify_blocks);
@@ -305,6 +400,8 @@ EXPORT_SYMBOL_GPL(fsverity_verify_blocks);
*/
void fsverity_verify_bio(struct bio *bio)
{
+ struct inode *inode = bio_first_folio_all(bio)->mapping->host;
+ struct fsverity_verification_context ctx;
struct folio_iter fi;
unsigned long max_ra_pages = 0;
@@ -321,13 +418,21 @@ void fsverity_verify_bio(struct bio *bio)
max_ra_pages = bio->bi_iter.bi_size >> (PAGE_SHIFT + 2);
}
+ fsverity_init_verification_context(&ctx, inode, max_ra_pages);
+
bio_for_each_folio_all(fi, bio) {
- if (!verify_data_blocks(fi.folio, fi.length, fi.offset,
- max_ra_pages)) {
- bio->bi_status = BLK_STS_IOERR;
- break;
- }
+ if (!fsverity_add_data_blocks(&ctx, fi.folio, fi.length,
+ fi.offset))
+ goto ioerr;
}
+
+ if (!fsverity_verify_pending_blocks(&ctx))
+ goto ioerr;
+ return;
+
+ioerr:
+ fsverity_clear_pending_blocks(&ctx);
+ bio->bi_status = BLK_STS_IOERR;
}
EXPORT_SYMBOL_GPL(fsverity_verify_bio);
#endif /* CONFIG_BLOCK */
@@ -355,7 +460,7 @@ void __init fsverity_init_workqueue(void)
* latency on ARM64.
*/
fsverity_read_workqueue = alloc_workqueue("fsverity_read_queue",
- WQ_HIGHPRI,
+ WQ_HIGHPRI | WQ_PERCPU,
num_online_cpus());
if (!fsverity_read_workqueue)
panic("failed to allocate fsverity_read_queue");
generated by cgit (git 2.34.1) at 2025-10-04 20:26:40 +0000