diff options
Diffstat (limited to 'drivers/md/dm-verity-fec.c')
| -rw-r--r-- | drivers/md/dm-verity-fec.c | 180 |
1 files changed, 83 insertions, 97 deletions
diff --git a/drivers/md/dm-verity-fec.c b/drivers/md/dm-verity-fec.c index 0ce04e5b4afb..72047b47a7a0 100644 --- a/drivers/md/dm-verity-fec.c +++ b/drivers/md/dm-verity-fec.c @@ -1,12 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (C) 2015 Google, Inc. * * Author: Sami Tolvanen <samitolvanen@google.com> - * - * This program is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the Free - * Software Foundation; either version 2 of the License, or (at your option) - * any later version. */ #include "dm-verity-fec.h" @@ -28,7 +24,8 @@ bool verity_fec_is_enabled(struct dm_verity *v) */ static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io) { - return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io); + return (struct dm_verity_fec_io *) + ((char *)io + io->v->ti->per_io_data_size - sizeof(struct dm_verity_fec_io)); } /* @@ -43,41 +40,29 @@ static inline u64 fec_interleave(struct dm_verity *v, u64 offset) } /* - * Decode an RS block using Reed-Solomon. - */ -static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio, - u8 *data, u8 *fec, int neras) -{ - int i; - uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; - - for (i = 0; i < v->fec->roots; i++) - par[i] = fec[i]; - - return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras, - fio->erasures, 0, NULL); -} - -/* * Read error-correcting codes for the requested RS block. Returns a pointer * to the data block. Caller is responsible for releasing buf. */ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, - unsigned *offset, struct dm_buffer **buf) + unsigned int *offset, unsigned int par_buf_offset, + struct dm_buffer **buf, unsigned short ioprio) { - u64 position, block; + u64 position, block, rem; u8 *res; + /* We have already part of parity bytes read, skip to the next block */ + if (par_buf_offset) + index++; + position = (index + rsb) * v->fec->roots; - block = position >> v->data_dev_block_bits; - *offset = (unsigned)(position - (block << v->data_dev_block_bits)); + block = div64_u64_rem(position, v->fec->io_size, &rem); + *offset = par_buf_offset ? 0 : (unsigned int)rem; - res = dm_bufio_read(v->fec->bufio, v->fec->start + block, buf); - if (unlikely(IS_ERR(res))) { + res = dm_bufio_read_with_ioprio(v->fec->bufio, block, buf, ioprio); + if (IS_ERR(res)) { DMERR("%s: FEC %llu: parity read failed (block %llu): %ld", v->data_dev->name, (unsigned long long)rsb, - (unsigned long long)(v->fec->start + block), - PTR_ERR(res)); + (unsigned long long)block, PTR_ERR(res)); *buf = NULL; } @@ -107,7 +92,7 @@ static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index, */ static inline u8 *fec_buffer_rs_block(struct dm_verity *v, struct dm_verity_fec_io *fio, - unsigned i, unsigned j) + unsigned int i, unsigned int j) { return &fio->bufs[i][j * v->fec->rsn]; } @@ -116,7 +101,7 @@ static inline u8 *fec_buffer_rs_block(struct dm_verity *v, * Return an index to the current RS block when called inside * fec_for_each_buffer_rs_block. */ -static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) +static inline unsigned int fec_buffer_rs_index(unsigned int i, unsigned int j) { return (i << DM_VERITY_FEC_BUF_RS_BITS) + j; } @@ -125,16 +110,19 @@ static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j) * Decode all RS blocks from buffers and copy corrected bytes into fio->output * starting from block_offset. */ -static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, - u64 rsb, int byte_index, unsigned block_offset, - int neras) +static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_io *io, + struct dm_verity_fec_io *fio, u64 rsb, int byte_index, + unsigned int block_offset, int neras) { int r, corrected = 0, res; struct dm_buffer *buf; - unsigned n, i, offset; + unsigned int n, i, j, offset, par_buf_offset = 0; + uint16_t par_buf[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN]; u8 *par, *block; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); + par = fec_read_parity(v, rsb, block_offset, &offset, + par_buf_offset, &buf, bio->bi_ioprio); if (IS_ERR(par)) return PTR_ERR(par); @@ -144,7 +132,11 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, */ fec_for_each_buffer_rs_block(fio, n, i) { block = fec_buffer_rs_block(v, fio, n, i); - res = fec_decode_rs8(v, fio, block, &par[offset], neras); + for (j = 0; j < v->fec->roots - par_buf_offset; j++) + par_buf[par_buf_offset + j] = par[offset + j]; + /* Decode an RS block using Reed-Solomon */ + res = decode_rs8(fio->rs, block, par_buf, v->fec->rsn, + NULL, neras, fio->erasures, 0, NULL); if (res < 0) { r = res; goto error; @@ -157,13 +149,23 @@ static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio, if (block_offset >= 1 << v->data_dev_block_bits) goto done; - /* read the next block when we run out of parity bytes */ - offset += v->fec->roots; - if (offset >= 1 << v->data_dev_block_bits) { + /* Read the next block when we run out of parity bytes */ + offset += (v->fec->roots - par_buf_offset); + /* Check if parity bytes are split between blocks */ + if (offset < v->fec->io_size && (offset + v->fec->roots) > v->fec->io_size) { + par_buf_offset = v->fec->io_size - offset; + for (j = 0; j < par_buf_offset; j++) + par_buf[j] = par[offset + j]; + offset += par_buf_offset; + } else + par_buf_offset = 0; + + if (offset >= v->fec->io_size) { dm_bufio_release(buf); - par = fec_read_parity(v, rsb, block_offset, &offset, &buf); - if (unlikely(IS_ERR(par))) + par = fec_read_parity(v, rsb, block_offset, &offset, + par_buf_offset, &buf, bio->bi_ioprio); + if (IS_ERR(par)) return PTR_ERR(par); } } @@ -188,8 +190,7 @@ error: static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, u8 *want_digest, u8 *data) { - if (unlikely(verity_hash(v, verity_io_hash_req(v, io), - data, 1 << v->data_dev_block_bits, + if (unlikely(verity_hash(v, io, data, 1 << v->data_dev_block_bits, verity_io_real_digest(v, io)))) return 0; @@ -202,7 +203,7 @@ static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io, * fits into buffers. Check for erasure locations if @neras is non-NULL. */ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, - u64 rsb, u64 target, unsigned block_offset, + u64 rsb, u64 target, unsigned int block_offset, int *neras) { bool is_zero; @@ -213,7 +214,8 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, u64 block, ileaved; u8 *bbuf, *rs_block; u8 want_digest[HASH_MAX_DIGESTSIZE]; - unsigned n, k; + unsigned int n, k; + struct bio *bio = dm_bio_from_per_bio_data(io, v->ti->per_io_data_size); if (neras) *neras = 0; @@ -252,8 +254,8 @@ static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io, bufio = v->bufio; } - bbuf = dm_bufio_read(bufio, block, &buf); - if (unlikely(IS_ERR(bbuf))) { + bbuf = dm_bufio_read_with_ioprio(bufio, block, &buf, bio->bi_ioprio); + if (IS_ERR(bbuf)) { DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld", v->data_dev->name, (unsigned long long)rsb, @@ -309,7 +311,7 @@ done: */ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) { - unsigned n; + unsigned int n; if (!fio->rs) fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO); @@ -318,11 +320,7 @@ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) if (fio->bufs[n]) continue; - fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT); - if (unlikely(!fio->bufs[n])) { - DMERR("failed to allocate FEC buffer"); - return -ENOMEM; - } + fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOIO); } /* try to allocate the maximum number of buffers */ @@ -349,7 +347,7 @@ static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) */ static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio) { - unsigned n; + unsigned int n; fec_for_each_buffer(fio, n) memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS); @@ -367,7 +365,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, bool use_erasures) { int r, neras = 0; - unsigned pos; + unsigned int pos; r = fec_alloc_bufs(v, fio); if (unlikely(r < 0)) @@ -381,7 +379,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, if (unlikely(r < 0)) return r; - r = fec_decode_bufs(v, fio, rsb, r, pos, neras); + r = fec_decode_bufs(v, io, fio, rsb, r, pos, neras); if (r < 0) return r; @@ -389,8 +387,7 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, } /* Always re-validate the corrected block against the expected hash */ - r = verity_hash(v, verity_io_hash_req(v, io), fio->output, - 1 << v->data_dev_block_bits, + r = verity_hash(v, io, fio->output, 1 << v->data_dev_block_bits, verity_io_real_digest(v, io)); if (unlikely(r < 0)) return r; @@ -405,24 +402,9 @@ static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io, return 0; } -static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data, - size_t len) -{ - struct dm_verity_fec_io *fio = fec_io(io); - - memcpy(data, &fio->output[fio->output_pos], len); - fio->output_pos += len; - - return 0; -} - -/* - * Correct errors in a block. Copies corrected block to dest if non-NULL, - * otherwise to a bio_vec starting from iter. - */ +/* Correct errors in a block. Copies corrected block to dest. */ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, - enum verity_block_type type, sector_t block, u8 *dest, - struct bvec_iter *iter) + enum verity_block_type type, sector_t block, u8 *dest) { int r; struct dm_verity_fec_io *fio = fec_io(io); @@ -439,7 +421,7 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, fio->level++; if (type == DM_VERITY_BLOCK_TYPE_METADATA) - block += v->data_blocks; + block = block - v->hash_start + v->data_blocks; /* * For RS(M, N), the continuous FEC data is divided into blocks of N @@ -472,12 +454,7 @@ int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io, goto done; } - if (dest) - memcpy(dest, fio->output, 1 << v->data_dev_block_bits); - else if (iter) { - fio->output_pos = 0; - r = verity_for_bv_block(v, io, iter, fec_bv_copy); - } + memcpy(dest, fio->output, 1 << v->data_dev_block_bits); done: fio->level--; @@ -489,7 +466,7 @@ done: */ void verity_fec_finish_io(struct dm_verity_io *io) { - unsigned n; + unsigned int n; struct dm_verity_fec *f = io->v->fec; struct dm_verity_fec_io *fio = fec_io(io); @@ -527,8 +504,8 @@ void verity_fec_init_io(struct dm_verity_io *io) /* * Append feature arguments and values to the status table. */ -unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz, - char *result, unsigned maxlen) +unsigned int verity_fec_status_table(struct dm_verity *v, unsigned int sz, + char *result, unsigned int maxlen) { if (!verity_fec_is_enabled(v)) return sz; @@ -555,6 +532,7 @@ void verity_fec_dtr(struct dm_verity *v) mempool_exit(&f->rs_pool); mempool_exit(&f->prealloc_pool); mempool_exit(&f->extra_pool); + mempool_exit(&f->output_pool); kmem_cache_destroy(f->cache); if (f->data_bufio) @@ -571,14 +549,14 @@ out: static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data) { - struct dm_verity *v = (struct dm_verity *)pool_data; + struct dm_verity *v = pool_data; return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask); } static void fec_rs_free(void *element, void *pool_data) { - struct rs_control *rs = (struct rs_control *)element; + struct rs_control *rs = element; if (rs) free_rs(rs); @@ -593,7 +571,7 @@ bool verity_is_fec_opt_arg(const char *arg_name) } int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, - unsigned *argc, const char *arg_name) + unsigned int *argc, const char *arg_name) { int r; struct dm_target *ti = v->ti; @@ -611,7 +589,11 @@ int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v, (*argc)--; if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) { - r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev); + if (v->fec->dev) { + ti->error = "FEC device already specified"; + return -EINVAL; + } + r = dm_get_device(ti, arg_value, BLK_OPEN_READ, &v->fec->dev); if (r) { ti->error = "FEC device lookup failed"; return r; @@ -677,7 +659,7 @@ int verity_fec_ctr(struct dm_verity *v) { struct dm_verity_fec *f = v->fec; struct dm_target *ti = v->ti; - u64 hash_blocks; + u64 hash_blocks, fec_blocks; int ret; if (!verity_fec_is_enabled(v)) { @@ -746,23 +728,27 @@ int verity_fec_ctr(struct dm_verity *v) return -E2BIG; } + f->io_size = 1 << v->data_dev_block_bits; + f->bufio = dm_bufio_client_create(f->dev->bdev, - 1 << v->data_dev_block_bits, - 1, 0, NULL, NULL); + f->io_size, + 1, 0, NULL, NULL, 0); if (IS_ERR(f->bufio)) { ti->error = "Cannot initialize FEC bufio client"; return PTR_ERR(f->bufio); } - if (dm_bufio_get_device_size(f->bufio) < - ((f->start + f->rounds * f->roots) >> v->data_dev_block_bits)) { + dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT)); + + fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT); + if (dm_bufio_get_device_size(f->bufio) < fec_blocks) { ti->error = "FEC device is too small"; return -E2BIG; } f->data_bufio = dm_bufio_client_create(v->data_dev->bdev, 1 << v->data_dev_block_bits, - 1, 0, NULL, NULL); + 1, 0, NULL, NULL, 0); if (IS_ERR(f->data_bufio)) { ti->error = "Cannot initialize FEC data bufio client"; return PTR_ERR(f->data_bufio); |