diff options
Diffstat (limited to 'drivers/md/dm-integrity.c')
| -rw-r--r-- | drivers/md/dm-integrity.c | 3029 |
1 files changed, 2420 insertions, 609 deletions
diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 457200ca6287..170bf67a2edd 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Copyright (C) 2016-2017 Red Hat, Inc. All rights reserved. * Copyright (C) 2016-2017 Milan Broz @@ -6,6 +7,8 @@ * This file is released under the GPL. */ +#include "dm-bio-record.h" + #include <linux/compiler.h> #include <linux/module.h> #include <linux/device-mapper.h> @@ -15,24 +18,34 @@ #include <linux/rbtree.h> #include <linux/delay.h> #include <linux/random.h> +#include <linux/reboot.h> #include <crypto/hash.h> #include <crypto/skcipher.h> +#include <crypto/utils.h> #include <linux/async_tx.h> #include <linux/dm-bufio.h> +#include "dm-audit.h" + #define DM_MSG_PREFIX "integrity" #define DEFAULT_INTERLEAVE_SECTORS 32768 #define DEFAULT_JOURNAL_SIZE_FACTOR 7 +#define DEFAULT_SECTORS_PER_BITMAP_BIT 32768 #define DEFAULT_BUFFER_SECTORS 128 #define DEFAULT_JOURNAL_WATERMARK 50 #define DEFAULT_SYNC_MSEC 10000 -#define DEFAULT_MAX_JOURNAL_SECTORS 131072 +#define DEFAULT_MAX_JOURNAL_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 131072 : 8192) #define MIN_LOG2_INTERLEAVE_SECTORS 3 #define MAX_LOG2_INTERLEAVE_SECTORS 31 #define METADATA_WORKQUEUE_MAX_ACTIVE 16 -#define RECALC_SECTORS 8192 +#define RECALC_SECTORS (IS_ENABLED(CONFIG_64BIT) ? 32768 : 2048) #define RECALC_WRITE_SUPER 16 +#define BITMAP_BLOCK_SIZE 4096 /* don't change it */ +#define BITMAP_FLUSH_INTERVAL (10 * HZ) +#define DISCARD_FILLER 0xf6 +#define SALT_SIZE 16 +#define RECHECK_POOL_SIZE 256 /* * Warning - DEBUG_PRINT prints security-sensitive data to the log, @@ -48,6 +61,10 @@ #define SB_MAGIC "integrt" #define SB_VERSION_1 1 #define SB_VERSION_2 2 +#define SB_VERSION_3 3 +#define SB_VERSION_4 4 +#define SB_VERSION_5 5 +#define SB_VERSION_6 6 #define SB_SECTORS 8 #define MAX_SECTORS_PER_BLOCK 8 @@ -55,32 +72,39 @@ struct superblock { __u8 magic[8]; __u8 version; __u8 log2_interleave_sectors; - __u16 integrity_tag_size; - __u32 journal_sections; - __u64 provided_data_sectors; /* userspace uses this value */ - __u32 flags; + __le16 integrity_tag_size; + __le32 journal_sections; + __le64 provided_data_sectors; /* userspace uses this value */ + __le32 flags; __u8 log2_sectors_per_block; - __u8 pad[3]; - __u64 recalc_sector; + __u8 log2_blocks_per_bitmap_bit; + __u8 pad[2]; + __le64 recalc_sector; + __u8 pad2[8]; + __u8 salt[SALT_SIZE]; }; #define SB_FLAG_HAVE_JOURNAL_MAC 0x1 #define SB_FLAG_RECALCULATING 0x2 +#define SB_FLAG_DIRTY_BITMAP 0x4 +#define SB_FLAG_FIXED_PADDING 0x8 +#define SB_FLAG_FIXED_HMAC 0x10 +#define SB_FLAG_INLINE 0x20 #define JOURNAL_ENTRY_ROUNDUP 8 -typedef __u64 commit_id_t; +typedef __le64 commit_id_t; #define JOURNAL_MAC_PER_SECTOR 8 struct journal_entry { union { struct { - __u32 sector_lo; - __u32 sector_hi; + __le32 sector_lo; + __le32 sector_hi; } s; - __u64 sector; + __le64 sector; } u; - commit_id_t last_bytes[0]; + commit_id_t last_bytes[]; /* __u8 tag[0]; */ }; @@ -88,30 +112,28 @@ struct journal_entry { #if BITS_PER_LONG == 64 #define journal_entry_set_sector(je, x) do { smp_wmb(); WRITE_ONCE((je)->u.sector, cpu_to_le64(x)); } while (0) -#define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) -#elif defined(CONFIG_LBDAF) -#define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0) -#define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) #else -#define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32(0)); } while (0) -#define journal_entry_get_sector(je) le32_to_cpu((je)->u.s.sector_lo) +#define journal_entry_set_sector(je, x) do { (je)->u.s.sector_lo = cpu_to_le32(x); smp_wmb(); WRITE_ONCE((je)->u.s.sector_hi, cpu_to_le32((x) >> 32)); } while (0) #endif +#define journal_entry_get_sector(je) le64_to_cpu((je)->u.sector) #define journal_entry_is_unused(je) ((je)->u.s.sector_hi == cpu_to_le32(-1)) -#define journal_entry_set_unused(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-1)); } while (0) +#define journal_entry_set_unused(je) ((je)->u.s.sector_hi = cpu_to_le32(-1)) #define journal_entry_is_inprogress(je) ((je)->u.s.sector_hi == cpu_to_le32(-2)) -#define journal_entry_set_inprogress(je) do { ((je)->u.s.sector_hi = cpu_to_le32(-2)); } while (0) +#define journal_entry_set_inprogress(je) ((je)->u.s.sector_hi = cpu_to_le32(-2)) #define JOURNAL_BLOCK_SECTORS 8 #define JOURNAL_SECTOR_DATA ((1 << SECTOR_SHIFT) - sizeof(commit_id_t)) #define JOURNAL_MAC_SIZE (JOURNAL_MAC_PER_SECTOR * JOURNAL_BLOCK_SECTORS) struct journal_sector { - __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR]; - __u8 mac[JOURNAL_MAC_PER_SECTOR]; + struct_group(sectors, + __u8 entries[JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR]; + __u8 mac[JOURNAL_MAC_PER_SECTOR]; + ); commit_id_t commit_id; }; -#define MAX_TAG_SIZE (JOURNAL_SECTOR_DATA - JOURNAL_MAC_PER_SECTOR - offsetof(struct journal_entry, last_bytes[MAX_SECTORS_PER_BLOCK])) +#define MAX_TAG_SIZE 255 #define METADATA_PADDING_SECTORS 8 @@ -140,24 +162,34 @@ struct alg_spec { char *alg_string; char *key_string; __u8 *key; - unsigned key_size; + unsigned int key_size; }; struct dm_integrity_c { struct dm_dev *dev; struct dm_dev *meta_dev; - unsigned tag_size; + unsigned int tag_size; __s8 log2_tag_size; + unsigned int tuple_size; sector_t start; mempool_t journal_io_mempool; struct dm_io_client *io; struct dm_bufio_client *bufio; struct workqueue_struct *metadata_wq; struct superblock *sb; - unsigned journal_pages; + unsigned int journal_pages; + unsigned int n_bitmap_blocks; + struct page_list *journal; struct page_list *journal_io; struct page_list *journal_xor; + struct page_list *recalc_bitmap; + struct page_list *may_write_bitmap; + struct bitmap_block_status *bbs; + unsigned int bitmap_flush_interval; + int synchronous_mode; + struct bio_list synchronous_bios; + struct delayed_work bitmap_flush_work; struct crypto_skcipher *journal_crypt; struct scatterlist **journal_scatterlist; @@ -175,43 +207,49 @@ struct dm_integrity_c { unsigned char journal_entries_per_sector; unsigned char journal_section_entries; unsigned short journal_section_sectors; - unsigned journal_sections; - unsigned journal_entries; + unsigned int journal_sections; + unsigned int journal_entries; sector_t data_device_sectors; sector_t meta_device_sectors; - unsigned initial_sectors; - unsigned metadata_run; + unsigned int initial_sectors; + unsigned int metadata_run; __s8 log2_metadata_run; __u8 log2_buffer_sectors; __u8 sectors_per_block; + __u8 log2_blocks_per_bitmap_bit; unsigned char mode; - int suspending; + bool internal_hash; int failed; - struct crypto_shash *internal_hash; + struct crypto_shash *internal_shash; + struct crypto_ahash *internal_ahash; + unsigned int internal_hash_digestsize; + + struct dm_target *ti; /* these variables are locked with endio_wait.lock */ struct rb_root in_progress; struct list_head wait_list; wait_queue_head_t endio_wait; struct workqueue_struct *wait_wq; + struct workqueue_struct *offload_wq; unsigned char commit_seq; commit_id_t commit_ids[N_COMMIT_IDS]; - unsigned committed_section; - unsigned n_committed_sections; + unsigned int committed_section; + unsigned int n_committed_sections; - unsigned uncommitted_section; - unsigned n_uncommitted_sections; + unsigned int uncommitted_section; + unsigned int n_uncommitted_sections; - unsigned free_section; + unsigned int free_section; unsigned char free_section_entry; - unsigned free_sectors; + unsigned int free_sectors; - unsigned free_sectors_threshold; + unsigned int free_sectors_threshold; struct workqueue_struct *commit_wq; struct work_struct commit_work; @@ -221,32 +259,46 @@ struct dm_integrity_c { struct workqueue_struct *recalc_wq; struct work_struct recalc_work; - u8 *recalc_buffer; - u8 *recalc_tags; struct bio_list flush_bio_list; unsigned long autocommit_jiffies; struct timer_list autocommit_timer; - unsigned autocommit_msec; + unsigned int autocommit_msec; wait_queue_head_t copy_to_journal_wait; struct completion crypto_backoff; + bool wrote_to_journal; bool journal_uptodate; bool just_formatted; + bool recalculate_flag; + bool reset_recalculate_flag; + bool discard; + bool fix_padding; + bool fix_hmac; + bool legacy_recalculate; + + mempool_t ahash_req_pool; + struct ahash_request *journal_ahash_req; struct alg_spec internal_hash_alg; struct alg_spec journal_crypt_alg; struct alg_spec journal_mac_alg; atomic64_t number_of_mismatches; + + mempool_t recheck_pool; + struct bio_set recheck_bios; + struct bio_set recalc_bios; + + struct notifier_block reboot_notifier; }; struct dm_integrity_range { sector_t logical_sector; - unsigned n_sectors; + sector_t n_sectors; bool waiting; union { struct rb_node node; @@ -261,24 +313,27 @@ struct dm_integrity_io { struct work_struct work; struct dm_integrity_c *ic; - bool write; + enum req_op op; bool fua; struct dm_integrity_range range; sector_t metadata_block; - unsigned metadata_offset; + unsigned int metadata_offset; atomic_t in_flight; blk_status_t bi_status; struct completion *completion; - struct gendisk *orig_bi_disk; - u8 orig_bi_partno; - bio_end_io_t *orig_bi_end_io; - struct bio_integrity_payload *orig_bi_integrity; - struct bvec_iter orig_bi_iter; + struct dm_bio_details bio_details; + + char *integrity_payload; + unsigned payload_len; + bool integrity_payload_from_mempool; + bool integrity_range_locked; + + struct ahash_request *ahash_req; }; struct journal_completion { @@ -292,43 +347,32 @@ struct journal_io { struct journal_completion *comp; }; +struct bitmap_block_status { + struct work_struct work; + struct dm_integrity_c *ic; + unsigned int idx; + unsigned long *bitmap; + struct bio_list bio_queue; + spinlock_t bio_queue_lock; + +}; + static struct kmem_cache *journal_io_cache; #define JOURNAL_IO_MEMPOOL 32 +#define AHASH_MEMPOOL 32 #ifdef DEBUG_PRINT -#define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__) -static void __DEBUG_bytes(__u8 *bytes, size_t len, const char *msg, ...) -{ - va_list args; - va_start(args, msg); - vprintk(msg, args); - va_end(args); - if (len) - pr_cont(":"); - while (len) { - pr_cont(" %02x", *bytes); - bytes++; - len--; - } - pr_cont("\n"); -} -#define DEBUG_bytes(bytes, len, msg, ...) __DEBUG_bytes(bytes, len, KERN_DEBUG msg, ##__VA_ARGS__) +#define DEBUG_print(x, ...) printk(KERN_DEBUG x, ##__VA_ARGS__) +#define DEBUG_bytes(bytes, len, msg, ...) printk(KERN_DEBUG msg "%s%*ph\n", ##__VA_ARGS__, \ + len ? ": " : "", len, bytes) #else #define DEBUG_print(x, ...) do { } while (0) #define DEBUG_bytes(bytes, len, msg, ...) do { } while (0) #endif -/* - * DM Integrity profile, protection is performed layer above (dm-crypt) - */ -static const struct blk_integrity_profile dm_integrity_profile = { - .name = "DM-DIF-EXT-TAG", - .generate_fn = NULL, - .verify_fn = NULL, -}; - static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map); +static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map); static void integrity_bio_wait(struct work_struct *w); static void dm_integrity_dtr(struct dm_target *ti); @@ -345,8 +389,19 @@ static int dm_integrity_failed(struct dm_integrity_c *ic) return READ_ONCE(ic->failed); } -static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i, - unsigned j, unsigned char seq) +static bool dm_integrity_disable_recalculate(struct dm_integrity_c *ic) +{ + if (ic->legacy_recalculate) + return false; + if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) ? + ic->internal_hash_alg.key || ic->journal_mac_alg.key : + ic->internal_hash_alg.key && !ic->journal_mac_alg.key) + return true; + return false; +} + +static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned int i, + unsigned int j, unsigned char seq) { /* * Xor the number with section and sector, so that if a piece of @@ -361,7 +416,7 @@ static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, if (!ic->meta_dev) { __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; *area = data_sector >> log2_interleave_sectors; - *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1); + *offset = (unsigned int)data_sector & ((1U << log2_interleave_sectors) - 1); } else { *area = 0; *offset = data_sector; @@ -370,15 +425,15 @@ static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, #define sector_to_block(ic, n) \ do { \ - BUG_ON((n) & (unsigned)((ic)->sectors_per_block - 1)); \ + BUG_ON((n) & (unsigned int)((ic)->sectors_per_block - 1)); \ (n) >>= (ic)->sb->log2_sectors_per_block; \ } while (0) static __u64 get_metadata_sector_and_offset(struct dm_integrity_c *ic, sector_t area, - sector_t offset, unsigned *metadata_offset) + sector_t offset, unsigned int *metadata_offset) { __u64 ms; - unsigned mo; + unsigned int mo; ms = area << ic->sb->log2_interleave_sectors; if (likely(ic->log2_metadata_run >= 0)) @@ -419,7 +474,7 @@ static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector return result; } -static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr) +static void wraparound_section(struct dm_integrity_c *ic, unsigned int *sec_ptr) { if (unlikely(*sec_ptr >= ic->journal_sections)) *sec_ptr -= ic->journal_sections; @@ -427,19 +482,68 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr) static void sb_set_version(struct dm_integrity_c *ic) { - if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) + if (ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) + ic->sb->version = SB_VERSION_6; + else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) + ic->sb->version = SB_VERSION_5; + else if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) + ic->sb->version = SB_VERSION_4; + else if (ic->mode == 'B' || ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) + ic->sb->version = SB_VERSION_3; + else if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ic->sb->version = SB_VERSION_2; else ic->sb->version = SB_VERSION_1; } -static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) +static int sb_mac(struct dm_integrity_c *ic, bool wr) +{ + SHASH_DESC_ON_STACK(desc, ic->journal_mac); + int r; + unsigned int mac_size = crypto_shash_digestsize(ic->journal_mac); + __u8 *sb = (__u8 *)ic->sb; + __u8 *mac = sb + (1 << SECTOR_SHIFT) - mac_size; + + if (sizeof(struct superblock) + mac_size > 1 << SECTOR_SHIFT || + mac_size > HASH_MAX_DIGESTSIZE) { + dm_integrity_io_error(ic, "digest is too long", -EINVAL); + return -EINVAL; + } + + desc->tfm = ic->journal_mac; + + if (likely(wr)) { + r = crypto_shash_digest(desc, sb, mac - sb, mac); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_digest", r); + return r; + } + } else { + __u8 actual_mac[HASH_MAX_DIGESTSIZE]; + + r = crypto_shash_digest(desc, sb, mac - sb, actual_mac); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_digest", r); + return r; + } + if (crypto_memneq(mac, actual_mac, mac_size)) { + dm_integrity_io_error(ic, "superblock mac", -EILSEQ); + dm_audit_log_target(DM_MSG_PREFIX, "mac-superblock", ic->ti, 0); + return -EILSEQ; + } + } + + return 0; +} + +static int sync_rw_sb(struct dm_integrity_c *ic, blk_opf_t opf) { struct dm_io_request io_req; struct dm_io_region io_loc; + const enum req_op op = opf & REQ_OP_MASK; + int r; - io_req.bi_op = op; - io_req.bi_op_flags = op_flags; + io_req.bi_opf = opf; io_req.mem.type = DM_IO_KMEM; io_req.mem.ptr.addr = ic->sb; io_req.notify.fn = NULL; @@ -448,28 +552,182 @@ static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) io_loc.sector = ic->start; io_loc.count = SB_SECTORS; - return dm_io(&io_req, 1, &io_loc, NULL); + if (op == REQ_OP_WRITE) { + sb_set_version(ic); + if (ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { + r = sb_mac(ic, true); + if (unlikely(r)) + return r; + } + } + + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + if (unlikely(r)) + return r; + + if (op == REQ_OP_READ) { + if (ic->mode != 'R' && ic->journal_mac && ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { + r = sb_mac(ic, false); + if (unlikely(r)) + return r; + } + } + + return 0; +} + +#define BITMAP_OP_TEST_ALL_SET 0 +#define BITMAP_OP_TEST_ALL_CLEAR 1 +#define BITMAP_OP_SET 2 +#define BITMAP_OP_CLEAR 3 + +static bool block_bitmap_op(struct dm_integrity_c *ic, struct page_list *bitmap, + sector_t sector, sector_t n_sectors, int mode) +{ + unsigned long bit, end_bit, this_end_bit, page, end_page; + unsigned long *data; + + if (unlikely(((sector | n_sectors) & ((1 << ic->sb->log2_sectors_per_block) - 1)) != 0)) { + DMCRIT("invalid bitmap access (%llx,%llx,%d,%d,%d)", + sector, + n_sectors, + ic->sb->log2_sectors_per_block, + ic->log2_blocks_per_bitmap_bit, + mode); + BUG(); + } + + if (unlikely(!n_sectors)) + return true; + + bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + end_bit = (sector + n_sectors - 1) >> + (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + + page = bit / (PAGE_SIZE * 8); + bit %= PAGE_SIZE * 8; + + end_page = end_bit / (PAGE_SIZE * 8); + end_bit %= PAGE_SIZE * 8; + +repeat: + if (page < end_page) + this_end_bit = PAGE_SIZE * 8 - 1; + else + this_end_bit = end_bit; + + data = lowmem_page_address(bitmap[page].page); + + if (mode == BITMAP_OP_TEST_ALL_SET) { + while (bit <= this_end_bit) { + if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { + do { + if (data[bit / BITS_PER_LONG] != -1) + return false; + bit += BITS_PER_LONG; + } while (this_end_bit >= bit + BITS_PER_LONG - 1); + continue; + } + if (!test_bit(bit, data)) + return false; + bit++; + } + } else if (mode == BITMAP_OP_TEST_ALL_CLEAR) { + while (bit <= this_end_bit) { + if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { + do { + if (data[bit / BITS_PER_LONG] != 0) + return false; + bit += BITS_PER_LONG; + } while (this_end_bit >= bit + BITS_PER_LONG - 1); + continue; + } + if (test_bit(bit, data)) + return false; + bit++; + } + } else if (mode == BITMAP_OP_SET) { + while (bit <= this_end_bit) { + if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { + do { + data[bit / BITS_PER_LONG] = -1; + bit += BITS_PER_LONG; + } while (this_end_bit >= bit + BITS_PER_LONG - 1); + continue; + } + __set_bit(bit, data); + bit++; + } + } else if (mode == BITMAP_OP_CLEAR) { + if (!bit && this_end_bit == PAGE_SIZE * 8 - 1) + clear_page(data); + else { + while (bit <= this_end_bit) { + if (!(bit % BITS_PER_LONG) && this_end_bit >= bit + BITS_PER_LONG - 1) { + do { + data[bit / BITS_PER_LONG] = 0; + bit += BITS_PER_LONG; + } while (this_end_bit >= bit + BITS_PER_LONG - 1); + continue; + } + __clear_bit(bit, data); + bit++; + } + } + } else { + BUG(); + } + + if (unlikely(page < end_page)) { + bit = 0; + page++; + goto repeat; + } + + return true; } -static void access_journal_check(struct dm_integrity_c *ic, unsigned section, unsigned offset, +static void block_bitmap_copy(struct dm_integrity_c *ic, struct page_list *dst, struct page_list *src) +{ + unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); + unsigned int i; + + for (i = 0; i < n_bitmap_pages; i++) { + unsigned long *dst_data = lowmem_page_address(dst[i].page); + unsigned long *src_data = lowmem_page_address(src[i].page); + + copy_page(dst_data, src_data); + } +} + +static struct bitmap_block_status *sector_to_bitmap_block(struct dm_integrity_c *ic, sector_t sector) +{ + unsigned int bit = sector >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + unsigned int bitmap_block = bit / (BITMAP_BLOCK_SIZE * 8); + + BUG_ON(bitmap_block >= ic->n_bitmap_blocks); + return &ic->bbs[bitmap_block]; +} + +static void access_journal_check(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, bool e, const char *function) { #if defined(CONFIG_DM_DEBUG) || defined(INTERNAL_VERIFY) - unsigned limit = e ? ic->journal_section_entries : ic->journal_section_sectors; + unsigned int limit = e ? ic->journal_section_entries : ic->journal_section_sectors; if (unlikely(section >= ic->journal_sections) || unlikely(offset >= limit)) { - printk(KERN_CRIT "%s: invalid access at (%u,%u), limit (%u,%u)\n", - function, section, offset, ic->journal_sections, limit); + DMCRIT("%s: invalid access at (%u,%u), limit (%u,%u)", + function, section, offset, ic->journal_sections, limit); BUG(); } #endif } -static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsigned offset, - unsigned *pl_index, unsigned *pl_offset) +static void page_list_location(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, + unsigned int *pl_index, unsigned int *pl_offset) { - unsigned sector; + unsigned int sector; access_journal_check(ic, section, offset, false, "page_list_location"); @@ -480,9 +738,9 @@ static void page_list_location(struct dm_integrity_c *ic, unsigned section, unsi } static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct page_list *pl, - unsigned section, unsigned offset, unsigned *n_sectors) + unsigned int section, unsigned int offset, unsigned int *n_sectors) { - unsigned pl_index, pl_offset; + unsigned int pl_index, pl_offset; char *va; page_list_location(ic, section, offset, &pl_index, &pl_offset); @@ -495,14 +753,14 @@ static struct journal_sector *access_page_list(struct dm_integrity_c *ic, struct return (struct journal_sector *)(va + pl_offset); } -static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset) +static struct journal_sector *access_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset) { return access_page_list(ic, ic->journal, section, offset, NULL); } -static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned section, unsigned n) +static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, unsigned int section, unsigned int n) { - unsigned rel_sector, offset; + unsigned int rel_sector, offset; struct journal_sector *js; access_journal_check(ic, section, n, true, "access_journal_entry"); @@ -514,7 +772,7 @@ static struct journal_entry *access_journal_entry(struct dm_integrity_c *ic, uns return (struct journal_entry *)((char *)js + offset * ic->journal_entry_size); } -static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned section, unsigned n) +static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, unsigned int section, unsigned int n) { n <<= ic->sb->log2_sectors_per_block; @@ -525,25 +783,42 @@ static struct journal_sector *access_journal_data(struct dm_integrity_c *ic, uns return access_journal(ic, section, n); } -static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result[JOURNAL_MAC_SIZE]) +static void section_mac(struct dm_integrity_c *ic, unsigned int section, __u8 result[JOURNAL_MAC_SIZE]) { SHASH_DESC_ON_STACK(desc, ic->journal_mac); int r; - unsigned j, size; + unsigned int j, size; desc->tfm = ic->journal_mac; - desc->flags = 0; r = crypto_shash_init(desc); - if (unlikely(r)) { + if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_init", r); goto err; } + if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { + __le64 section_le; + + r = crypto_shash_update(desc, (__u8 *)&ic->sb->salt, SALT_SIZE); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto err; + } + + section_le = cpu_to_le64(section); + r = crypto_shash_update(desc, (__u8 *)§ion_le, sizeof(section_le)); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto err; + } + } + for (j = 0; j < ic->journal_section_entries; j++) { struct journal_entry *je = access_journal_entry(ic, section, j); - r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof je->u.sector); - if (unlikely(r)) { + + r = crypto_shash_update(desc, (__u8 *)&je->u.sector, sizeof(je->u.sector)); + if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_update", r); goto err; } @@ -553,7 +828,7 @@ static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result if (likely(size <= JOURNAL_MAC_SIZE)) { r = crypto_shash_final(desc, result); - if (unlikely(r)) { + if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_final", r); goto err; } @@ -566,7 +841,7 @@ static void section_mac(struct dm_integrity_c *ic, unsigned section, __u8 result goto err; } r = crypto_shash_final(desc, digest); - if (unlikely(r)) { + if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_final", r); goto err; } @@ -578,10 +853,10 @@ err: memset(result, 0, JOURNAL_MAC_SIZE); } -static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr) +static void rw_section_mac(struct dm_integrity_c *ic, unsigned int section, bool wr) { __u8 result[JOURNAL_MAC_SIZE]; - unsigned j; + unsigned int j; if (!ic->journal_mac) return; @@ -594,8 +869,10 @@ static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr) if (likely(wr)) memcpy(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR); else { - if (memcmp(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) + if (crypto_memneq(&js->mac, result + (j * JOURNAL_MAC_PER_SECTOR), JOURNAL_MAC_PER_SECTOR)) { dm_integrity_io_error(ic, "journal mac", -EILSEQ); + dm_audit_log_target(DM_MSG_PREFIX, "mac-journal", ic->ti, 0); + } } } } @@ -603,17 +880,18 @@ static void rw_section_mac(struct dm_integrity_c *ic, unsigned section, bool wr) static void complete_journal_op(void *context) { struct journal_completion *comp = context; + BUG_ON(!atomic_read(&comp->in_flight)); if (likely(atomic_dec_and_test(&comp->in_flight))) complete(&comp->comp); } -static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, - unsigned n_sections, struct journal_completion *comp) +static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, + unsigned int n_sections, struct journal_completion *comp) { struct async_submit_ctl submit; size_t n_bytes = (size_t)(n_sections * ic->journal_section_sectors) << SECTOR_SHIFT; - unsigned pl_index, pl_offset, section_index; + unsigned int pl_index, pl_offset, section_index; struct page_list *source_pl, *target_pl; if (likely(encrypt)) { @@ -638,7 +916,8 @@ static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned sectio struct page *dst_page; while (unlikely(pl_index == section_index)) { - unsigned dummy; + unsigned int dummy; + if (likely(encrypt)) rw_section_mac(ic, section, true); section++; @@ -665,9 +944,10 @@ static void xor_journal(struct dm_integrity_c *ic, bool encrypt, unsigned sectio async_tx_issue_pending_all(); } -static void complete_journal_encrypt(struct crypto_async_request *req, int err) +static void complete_journal_encrypt(void *data, int err) { - struct journal_completion *comp = req->data; + struct journal_completion *comp = data; + if (unlikely(err)) { if (likely(err == -EINPROGRESS)) { complete(&comp->ic->crypto_backoff); @@ -681,6 +961,7 @@ static void complete_journal_encrypt(struct crypto_async_request *req, int err) static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_completion *comp) { int r; + skcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, complete_journal_encrypt, comp); if (likely(encrypt)) @@ -700,8 +981,8 @@ static bool do_crypt(bool encrypt, struct skcipher_request *req, struct journal_ return false; } -static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, - unsigned n_sections, struct journal_completion *comp) +static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, + unsigned int n_sections, struct journal_completion *comp) { struct scatterlist **source_sg; struct scatterlist **target_sg; @@ -718,7 +999,7 @@ static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned sect do { struct skcipher_request *req; - unsigned ivsize; + unsigned int ivsize; char *iv; if (likely(encrypt)) @@ -744,8 +1025,8 @@ static void crypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned sect complete_journal_op(comp); } -static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned section, - unsigned n_sections, struct journal_completion *comp) +static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned int section, + unsigned int n_sections, struct journal_completion *comp) { if (ic->journal_xor) return xor_journal(ic, encrypt, section, n_sections, comp); @@ -756,17 +1037,19 @@ static void encrypt_journal(struct dm_integrity_c *ic, bool encrypt, unsigned se static void complete_journal_io(unsigned long error, void *context) { struct journal_completion *comp = context; + if (unlikely(error != 0)) dm_integrity_io_error(comp->ic, "writing journal", -EIO); complete_journal_op(comp); } -static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned section, - unsigned n_sections, struct journal_completion *comp) +static void rw_journal_sectors(struct dm_integrity_c *ic, blk_opf_t opf, + unsigned int sector, unsigned int n_sectors, + struct journal_completion *comp) { struct dm_io_request io_req; struct dm_io_region io_loc; - unsigned sector, n_sectors, pl_index, pl_offset; + unsigned int pl_index, pl_offset; int r; if (unlikely(dm_integrity_failed(ic))) { @@ -775,14 +1058,10 @@ static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned return; } - sector = section * ic->journal_section_sectors; - n_sectors = n_sections * ic->journal_section_sectors; - pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); - io_req.bi_op = op; - io_req.bi_op_flags = op_flags; + io_req.bi_opf = opf; io_req.mem.type = DM_IO_PAGE_LIST; if (ic->journal_io) io_req.mem.ptr.pl = &ic->journal_io[pl_index]; @@ -800,9 +1079,10 @@ static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { - dm_integrity_io_error(ic, op == REQ_OP_READ ? "reading journal" : "writing journal", r); + dm_integrity_io_error(ic, (opf & REQ_OP_MASK) == REQ_OP_READ ? + "reading journal" : "writing journal", r); if (comp) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); complete_journal_io(-1UL, comp); @@ -810,12 +1090,24 @@ static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned } } -static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsigned commit_sections) +static void rw_journal(struct dm_integrity_c *ic, blk_opf_t opf, + unsigned int section, unsigned int n_sections, + struct journal_completion *comp) +{ + unsigned int sector, n_sectors; + + sector = section * ic->journal_section_sectors; + n_sectors = n_sections * ic->journal_section_sectors; + + rw_journal_sectors(ic, opf, sector, n_sectors, comp); +} + +static void write_journal(struct dm_integrity_c *ic, unsigned int commit_start, unsigned int commit_sections) { struct journal_completion io_comp; struct journal_completion crypt_comp_1; struct journal_completion crypt_comp_2; - unsigned i; + unsigned int i; io_comp.ic = ic; init_completion(&io_comp.comp); @@ -832,10 +1124,11 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi for (i = 0; i < commit_sections; i++) rw_section_mac(ic, commit_start + i, true); } - rw_journal(ic, REQ_OP_WRITE, REQ_FUA | REQ_SYNC, commit_start, + rw_journal(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, commit_start, commit_sections, &io_comp); } else { - unsigned to_end; + unsigned int to_end; + io_comp.in_flight = (atomic_t)ATOMIC_INIT(2); to_end = ic->journal_sections - commit_start; if (ic->journal_io) { @@ -844,7 +1137,8 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); encrypt_journal(ic, true, commit_start, to_end, &crypt_comp_1); if (try_wait_for_completion(&crypt_comp_1.comp)) { - rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); + rw_journal(ic, REQ_OP_WRITE | REQ_FUA, + commit_start, to_end, &io_comp); reinit_completion(&crypt_comp_1.comp); crypt_comp_1.in_flight = (atomic_t)ATOMIC_INIT(0); encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_1); @@ -855,31 +1149,31 @@ static void write_journal(struct dm_integrity_c *ic, unsigned commit_start, unsi crypt_comp_2.in_flight = (atomic_t)ATOMIC_INIT(0); encrypt_journal(ic, true, 0, commit_sections - to_end, &crypt_comp_2); wait_for_completion_io(&crypt_comp_1.comp); - rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); + rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); wait_for_completion_io(&crypt_comp_2.comp); } } else { for (i = 0; i < to_end; i++) rw_section_mac(ic, commit_start + i, true); - rw_journal(ic, REQ_OP_WRITE, REQ_FUA, commit_start, to_end, &io_comp); + rw_journal(ic, REQ_OP_WRITE | REQ_FUA, commit_start, to_end, &io_comp); for (i = 0; i < commit_sections - to_end; i++) rw_section_mac(ic, i, true); } - rw_journal(ic, REQ_OP_WRITE, REQ_FUA, 0, commit_sections - to_end, &io_comp); + rw_journal(ic, REQ_OP_WRITE | REQ_FUA, 0, commit_sections - to_end, &io_comp); } wait_for_completion_io(&io_comp.comp); } -static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsigned offset, - unsigned n_sectors, sector_t target, io_notify_fn fn, void *data) +static void copy_from_journal(struct dm_integrity_c *ic, unsigned int section, unsigned int offset, + unsigned int n_sectors, sector_t target, io_notify_fn fn, void *data) { struct dm_io_request io_req; struct dm_io_region io_loc; int r; - unsigned sector, pl_index, pl_offset; + unsigned int sector, pl_index, pl_offset; - BUG_ON((target | n_sectors | offset) & (unsigned)(ic->sectors_per_block - 1)); + BUG_ON((target | n_sectors | offset) & (unsigned int)(ic->sectors_per_block - 1)); if (unlikely(dm_integrity_failed(ic))) { fn(-1UL, data); @@ -891,8 +1185,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); - io_req.bi_op = REQ_OP_WRITE; - io_req.bi_op_flags = 0; + io_req.bi_opf = REQ_OP_WRITE; io_req.mem.type = DM_IO_PAGE_LIST; io_req.mem.ptr.pl = &ic->journal[pl_index]; io_req.mem.offset = pl_offset; @@ -903,7 +1196,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig io_loc.sector = target; io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { WARN_ONCE(1, "asynchronous dm_io failed: %d", r); fn(-1UL, data); @@ -913,7 +1206,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2) { return range1->logical_sector < range2->logical_sector + range2->n_sectors && - range2->logical_sector + range2->n_sectors > range2->logical_sector; + range1->logical_sector + range1->n_sectors > range2->logical_sector; } static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting) @@ -921,10 +1214,11 @@ static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range * struct rb_node **n = &ic->in_progress.rb_node; struct rb_node *parent; - BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1)); + BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned int)(ic->sectors_per_block - 1)); if (likely(check_waiting)) { struct dm_integrity_range *range; + list_for_each_entry(range, &ic->wait_list, wait_entry) { if (unlikely(ranges_overlap(range, new_range))) return false; @@ -937,13 +1231,12 @@ static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range * struct dm_integrity_range *range = container_of(*n, struct dm_integrity_range, node); parent = *n; - if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) { + if (new_range->logical_sector + new_range->n_sectors <= range->logical_sector) n = &range->node.rb_left; - } else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) { + else if (new_range->logical_sector >= range->logical_sector + range->n_sectors) n = &range->node.rb_right; - } else { + else return false; - } } rb_link_node(&new_range->node, parent, n); @@ -959,8 +1252,7 @@ static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity struct dm_integrity_range *last_range = list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); struct task_struct *last_range_task; - if (!ranges_overlap(range, last_range)) - break; + last_range_task = last_range->task; list_del(&last_range->wait_entry); if (!add_new_range(ic, last_range, false)) { @@ -995,6 +1287,12 @@ static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrit } while (unlikely(new_range->waiting)); } +static void add_new_range_and_wait(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) +{ + if (unlikely(!add_new_range(ic, new_range, true))) + wait_and_add_new_range(ic, new_range); +} + static void init_journal_node(struct journal_node *node) { RB_CLEAR_NODE(&node->node); @@ -1014,6 +1312,7 @@ static void add_journal_node(struct dm_integrity_c *ic, struct journal_node *nod while (*link) { struct journal_node *j; + parent = *link; j = container_of(parent, struct journal_node, node); if (sector < j->sector) @@ -1035,28 +1334,29 @@ static void remove_journal_node(struct dm_integrity_c *ic, struct journal_node * #define NOT_FOUND (-1U) -static unsigned find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) +static unsigned int find_journal_node(struct dm_integrity_c *ic, sector_t sector, sector_t *next_sector) { struct rb_node *n = ic->journal_tree_root.rb_node; - unsigned found = NOT_FOUND; + unsigned int found = NOT_FOUND; + *next_sector = (sector_t)-1; while (n) { struct journal_node *j = container_of(n, struct journal_node, node); - if (sector == j->sector) { + + if (sector == j->sector) found = j - ic->journal_tree; - } + if (sector < j->sector) { *next_sector = j->sector; n = j->node.rb_left; - } else { + } else n = j->node.rb_right; - } } return found; } -static bool test_journal_node(struct dm_integrity_c *ic, unsigned pos, sector_t sector) +static bool test_journal_node(struct dm_integrity_c *ic, unsigned int pos, sector_t sector) { struct journal_node *node, *next_node; struct rb_node *next; @@ -1081,7 +1381,7 @@ static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_ { struct rb_node *next; struct journal_node *next_node; - unsigned next_section; + unsigned int next_section; BUG_ON(RB_EMPTY_NODE(&node->node)); @@ -1094,7 +1394,7 @@ static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_ if (next_node->sector != node->sector) return false; - next_section = (unsigned)(next_node - ic->journal_tree) / ic->journal_section_entries; + next_section = (unsigned int)(next_node - ic->journal_tree) / ic->journal_section_entries; if (next_section >= ic->committed_section && next_section < ic->committed_section + ic->n_committed_sections) return true; @@ -1109,12 +1409,16 @@ static bool find_newer_committed_node(struct dm_integrity_c *ic, struct journal_ #define TAG_CMP 2 static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, sector_t *metadata_block, - unsigned *metadata_offset, unsigned total_size, int op) + unsigned int *metadata_offset, unsigned int total_size, int op) { + unsigned int hash_offset = 0; + unsigned char mismatch_hash = 0; + unsigned char mismatch_filler = !ic->discard; + do { unsigned char *data, *dp; struct dm_buffer *b; - unsigned to_copy; + unsigned int to_copy; int r; r = dm_integrity_failed(ic); @@ -1122,7 +1426,7 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se return r; data = dm_bufio_read(ic->bufio, *metadata_block, &b); - if (unlikely(IS_ERR(data))) + if (IS_ERR(data)) return PTR_ERR(data); to_copy = min((1U << SECTOR_SHIFT << ic->log2_buffer_sectors) - *metadata_offset, total_size); @@ -1130,20 +1434,40 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se if (op == TAG_READ) { memcpy(tag, dp, to_copy); } else if (op == TAG_WRITE) { - memcpy(dp, tag, to_copy); - dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy); - } else { + if (crypto_memneq(dp, tag, to_copy)) { + memcpy(dp, tag, to_copy); + dm_bufio_mark_partial_buffer_dirty(b, *metadata_offset, *metadata_offset + to_copy); + } + } else { /* e.g.: op == TAG_CMP */ - if (unlikely(memcmp(dp, tag, to_copy))) { - unsigned i; - for (i = 0; i < to_copy; i++) { - if (dp[i] != tag[i]) - break; - total_size--; + if (likely(is_power_of_2(ic->tag_size))) { + if (unlikely(crypto_memneq(dp, tag, to_copy))) + goto thorough_test; + } else { + unsigned int i, ts; +thorough_test: + ts = total_size; + + for (i = 0; i < to_copy; i++, ts--) { + /* + * Warning: the control flow must not be + * dependent on match/mismatch of + * individual bytes. + */ + mismatch_hash |= dp[i] ^ tag[i]; + mismatch_filler |= dp[i] ^ DISCARD_FILLER; + hash_offset++; + if (unlikely(hash_offset == ic->tag_size)) { + if (unlikely(mismatch_hash) && unlikely(mismatch_filler)) { + dm_bufio_release(b); + return ts; + } + hash_offset = 0; + mismatch_hash = 0; + mismatch_filler = !ic->discard; + } } - dm_bufio_release(b); - return total_size; } } dm_bufio_release(b); @@ -1154,23 +1478,67 @@ static int dm_integrity_rw_tag(struct dm_integrity_c *ic, unsigned char *tag, se (*metadata_block)++; *metadata_offset = 0; } + + if (unlikely(!is_power_of_2(ic->tag_size))) + hash_offset = (hash_offset + to_copy) % ic->tag_size; + total_size -= to_copy; } while (unlikely(total_size)); return 0; } -static void dm_integrity_flush_buffers(struct dm_integrity_c *ic) +struct flush_request { + struct dm_io_request io_req; + struct dm_io_region io_reg; + struct dm_integrity_c *ic; + struct completion comp; +}; + +static void flush_notify(unsigned long error, void *fr_) +{ + struct flush_request *fr = fr_; + + if (unlikely(error != 0)) + dm_integrity_io_error(fr->ic, "flushing disk cache", -EIO); + complete(&fr->comp); +} + +static void dm_integrity_flush_buffers(struct dm_integrity_c *ic, bool flush_data) { int r; + struct flush_request fr; + + if (!ic->meta_dev) + flush_data = false; + if (flush_data) { + fr.io_req.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC; + fr.io_req.mem.type = DM_IO_KMEM; + fr.io_req.mem.ptr.addr = NULL; + fr.io_req.notify.fn = flush_notify; + fr.io_req.notify.context = &fr; + fr.io_req.client = dm_bufio_get_dm_io_client(ic->bufio); + fr.io_reg.bdev = ic->dev->bdev; + fr.io_reg.sector = 0; + fr.io_reg.count = 0; + fr.ic = ic; + init_completion(&fr.comp); + r = dm_io(&fr.io_req, 1, &fr.io_reg, NULL, IOPRIO_DEFAULT); + BUG_ON(r); + } + r = dm_bufio_write_dirty_buffers(ic->bufio); if (unlikely(r)) dm_integrity_io_error(ic, "writing tags", r); + + if (flush_data) + wait_for_completion(&fr.comp); } static void sleep_on_endio_wait(struct dm_integrity_c *ic) { DECLARE_WAITQUEUE(wait, current); + __add_wait_queue(&ic->endio_wait, &wait); __set_current_state(TASK_UNINTERRUPTIBLE); spin_unlock_irq(&ic->endio_wait.lock); @@ -1181,7 +1549,8 @@ static void sleep_on_endio_wait(struct dm_integrity_c *ic) static void autocommit_fn(struct timer_list *t) { - struct dm_integrity_c *ic = from_timer(ic, t, autocommit_timer); + struct dm_integrity_c *ic = timer_container_of(ic, t, + autocommit_timer); if (likely(!dm_integrity_failed(ic))) queue_work(ic->commit_wq, &ic->commit_work); @@ -1208,9 +1577,20 @@ static void submit_flush_bio(struct dm_integrity_c *ic, struct dm_integrity_io * static void do_endio(struct dm_integrity_c *ic, struct bio *bio) { - int r = dm_integrity_failed(ic); + int r; + + r = dm_integrity_failed(ic); if (unlikely(r) && !bio->bi_status) bio->bi_status = errno_to_blk_status(r); + if (unlikely(ic->synchronous_mode) && bio_op(bio) == REQ_OP_WRITE) { + unsigned long flags; + + spin_lock_irqsave(&ic->endio_wait.lock, flags); + bio_list_add(&ic->synchronous_bios, bio); + queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); + spin_unlock_irqrestore(&ic->endio_wait.lock, flags); + return; + } bio_endio(bio); } @@ -1232,18 +1612,17 @@ static void dec_in_flight(struct dm_integrity_io *dio) remove_range(ic, &dio->range); - if (unlikely(dio->write)) + if (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD)) schedule_autocommit(ic); bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); - if (unlikely(dio->bi_status) && !bio->bi_status) bio->bi_status = dio->bi_status; if (likely(!bio->bi_status) && unlikely(bio_sectors(bio) != dio->range.n_sectors)) { dio->range.logical_sector += dio->range.n_sectors; bio_advance(bio, dio->range.n_sectors << SECTOR_SHIFT); INIT_WORK(&dio->work, integrity_bio_wait); - queue_work(ic->wait_wq, &dio->work); + queue_work(ic->offload_wq, &dio->work); return; } do_endio_flush(ic, dio); @@ -1254,14 +1633,9 @@ static void integrity_end_io(struct bio *bio) { struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); - bio->bi_iter = dio->orig_bi_iter; - bio->bi_disk = dio->orig_bi_disk; - bio->bi_partno = dio->orig_bi_partno; - if (dio->orig_bi_integrity) { - bio->bi_integrity = dio->orig_bi_integrity; + dm_bio_restore(&dio->bio_details, bio); + if (bio->bi_integrity) bio->bi_opf |= REQ_INTEGRITY; - } - bio->bi_end_io = dio->orig_bi_end_io; if (dio->completion) complete(dio->completion); @@ -1269,16 +1643,15 @@ static void integrity_end_io(struct bio *bio) dec_in_flight(dio); } -static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector, - const char *data, char *result) +static void integrity_sector_checksum_shash(struct dm_integrity_c *ic, sector_t sector, + const char *data, unsigned offset, char *result) { - __u64 sector_le = cpu_to_le64(sector); - SHASH_DESC_ON_STACK(req, ic->internal_hash); + __le64 sector_le = cpu_to_le64(sector); + SHASH_DESC_ON_STACK(req, ic->internal_shash); int r; - unsigned digest_size; + unsigned int digest_size; - req->tfm = ic->internal_hash; - req->flags = 0; + req->tfm = ic->internal_shash; r = crypto_shash_init(req); if (unlikely(r < 0)) { @@ -1286,13 +1659,21 @@ static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector goto failed; } - r = crypto_shash_update(req, (const __u8 *)§or_le, sizeof sector_le); + if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { + r = crypto_shash_update(req, (__u8 *)&ic->sb->salt, SALT_SIZE); + if (unlikely(r < 0)) { + dm_integrity_io_error(ic, "crypto_shash_update", r); + goto failed; + } + } + + r = crypto_shash_update(req, (const __u8 *)§or_le, sizeof(sector_le)); if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_update", r); goto failed; } - r = crypto_shash_update(req, data, ic->sectors_per_block << SECTOR_SHIFT); + r = crypto_shash_update(req, data + offset, ic->sectors_per_block << SECTOR_SHIFT); if (unlikely(r < 0)) { dm_integrity_io_error(ic, "crypto_shash_update", r); goto failed; @@ -1304,7 +1685,70 @@ static void integrity_sector_checksum(struct dm_integrity_c *ic, sector_t sector goto failed; } - digest_size = crypto_shash_digestsize(ic->internal_hash); + digest_size = ic->internal_hash_digestsize; + if (unlikely(digest_size < ic->tag_size)) + memset(result + digest_size, 0, ic->tag_size - digest_size); + + return; + +failed: + /* this shouldn't happen anyway, the hash functions have no reason to fail */ + get_random_bytes(result, ic->tag_size); +} + +static void integrity_sector_checksum_ahash(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + sector_t sector, struct page *page, unsigned offset, char *result) +{ + __le64 sector_le = cpu_to_le64(sector); + struct ahash_request *req; + DECLARE_CRYPTO_WAIT(wait); + struct scatterlist sg[3], *s = sg; + int r; + unsigned int digest_size; + unsigned int nbytes = 0; + + might_sleep(); + + req = *ahash_req; + if (unlikely(!req)) { + req = mempool_alloc(&ic->ahash_req_pool, GFP_NOIO); + *ahash_req = req; + } + + ahash_request_set_tfm(req, ic->internal_ahash); + ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, crypto_req_done, &wait); + + if (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) { + sg_init_table(sg, 3); + sg_set_buf(s, (const __u8 *)&ic->sb->salt, SALT_SIZE); + nbytes += SALT_SIZE; + s++; + } else { + sg_init_table(sg, 2); + } + + if (likely(!is_vmalloc_addr(§or_le))) { + sg_set_buf(s, §or_le, sizeof(sector_le)); + } else { + struct page *sec_page = vmalloc_to_page(§or_le); + unsigned int sec_off = offset_in_page(§or_le); + sg_set_page(s, sec_page, sizeof(sector_le), sec_off); + } + nbytes += sizeof(sector_le); + s++; + + sg_set_page(s, page, ic->sectors_per_block << SECTOR_SHIFT, offset); + nbytes += ic->sectors_per_block << SECTOR_SHIFT; + + ahash_request_set_crypt(req, sg, result, nbytes); + + r = crypto_wait_req(crypto_ahash_digest(req), &wait); + if (unlikely(r)) { + dm_integrity_io_error(ic, "crypto_ahash_digest", r); + goto failed; + } + + digest_size = ic->internal_hash_digestsize; if (unlikely(digest_size < ic->tag_size)) memset(result + digest_size, 0, ic->tag_size - digest_size); @@ -1315,6 +1759,121 @@ failed: get_random_bytes(result, ic->tag_size); } +static void integrity_sector_checksum(struct dm_integrity_c *ic, struct ahash_request **ahash_req, + sector_t sector, const char *data, unsigned offset, char *result) +{ + if (likely(ic->internal_shash != NULL)) + integrity_sector_checksum_shash(ic, sector, data, offset, result); + else + integrity_sector_checksum_ahash(ic, ahash_req, sector, (struct page *)data, offset, result); +} + +static void *integrity_kmap(struct dm_integrity_c *ic, struct page *p) +{ + if (likely(ic->internal_shash != NULL)) + return kmap_local_page(p); + else + return p; +} + +static void integrity_kunmap(struct dm_integrity_c *ic, const void *ptr) +{ + if (likely(ic->internal_shash != NULL)) + kunmap_local(ptr); +} + +static void *integrity_identity(struct dm_integrity_c *ic, void *data) +{ +#ifdef CONFIG_DEBUG_SG + BUG_ON(offset_in_page(data)); + BUG_ON(!virt_addr_valid(data)); +#endif + if (likely(ic->internal_shash != NULL)) + return data; + else + return virt_to_page(data); +} + +static noinline void integrity_recheck(struct dm_integrity_io *dio, char *checksum) +{ + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + struct dm_integrity_c *ic = dio->ic; + struct bvec_iter iter; + struct bio_vec bv; + sector_t sector, logical_sector, area, offset; + struct page *page; + + get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); + dio->metadata_block = get_metadata_sector_and_offset(ic, area, offset, + &dio->metadata_offset); + sector = get_data_sector(ic, area, offset); + logical_sector = dio->range.logical_sector; + + page = mempool_alloc(&ic->recheck_pool, GFP_NOIO); + + __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { + unsigned pos = 0; + + do { + sector_t alignment; + char *mem; + char *buffer = page_to_virt(page); + unsigned int buffer_offset; + int r; + struct dm_io_request io_req; + struct dm_io_region io_loc; + io_req.bi_opf = REQ_OP_READ; + io_req.mem.type = DM_IO_KMEM; + io_req.mem.ptr.addr = buffer; + io_req.notify.fn = NULL; + io_req.client = ic->io; + io_loc.bdev = ic->dev->bdev; + io_loc.sector = sector; + io_loc.count = ic->sectors_per_block; + + /* Align the bio to logical block size */ + alignment = dio->range.logical_sector | bio_sectors(bio) | (PAGE_SIZE >> SECTOR_SHIFT); + alignment &= -alignment; + io_loc.sector = round_down(io_loc.sector, alignment); + io_loc.count += sector - io_loc.sector; + buffer_offset = (sector - io_loc.sector) << SECTOR_SHIFT; + io_loc.count = round_up(io_loc.count, alignment); + + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); + if (unlikely(r)) { + dio->bi_status = errno_to_blk_status(r); + goto free_ret; + } + + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, integrity_identity(ic, buffer), buffer_offset, checksum); + r = dm_integrity_rw_tag(ic, checksum, &dio->metadata_block, + &dio->metadata_offset, ic->tag_size, TAG_CMP); + if (r) { + if (r > 0) { + DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", + bio->bi_bdev, logical_sector); + atomic64_inc(&ic->number_of_mismatches); + dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum", + bio, logical_sector, 0); + r = -EILSEQ; + } + dio->bi_status = errno_to_blk_status(r); + goto free_ret; + } + + mem = bvec_kmap_local(&bv); + memcpy(mem + pos, buffer + buffer_offset, ic->sectors_per_block << SECTOR_SHIFT); + kunmap_local(mem); + + pos += ic->sectors_per_block << SECTOR_SHIFT; + sector += ic->sectors_per_block; + logical_sector += ic->sectors_per_block; + } while (pos < bv.bv_len); + } +free_ret: + mempool_free(page, &ic->recheck_pool); +} + static void integrity_metadata(struct work_struct *w) { struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); @@ -1325,19 +1884,22 @@ static void integrity_metadata(struct work_struct *w) if (ic->internal_hash) { struct bvec_iter iter; struct bio_vec bv; - unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); + unsigned int digest_size = ic->internal_hash_digestsize; struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); char *checksums; - unsigned extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; - char checksums_onstack[HASH_MAX_DIGESTSIZE]; - unsigned sectors_to_process = dio->range.n_sectors; - sector_t sector = dio->range.logical_sector; + unsigned int extra_space = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; + char checksums_onstack[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; + sector_t sector; + unsigned int sectors_to_process; if (unlikely(ic->mode == 'R')) goto skip_io; - checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space, - GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); + if (likely(dio->op != REQ_OP_DISCARD)) + checksums = kmalloc((PAGE_SIZE >> SECTOR_SHIFT >> ic->sb->log2_sectors_per_block) * ic->tag_size + extra_space, + GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); + else + checksums = kmalloc(PAGE_SIZE, GFP_NOIO | __GFP_NORETRY | __GFP_NOWARN); if (!checksums) { checksums = checksums_onstack; if (WARN_ON(extra_space && @@ -1347,43 +1909,72 @@ static void integrity_metadata(struct work_struct *w) } } - __bio_for_each_segment(bv, bio, iter, dio->orig_bi_iter) { - unsigned pos; + if (unlikely(dio->op == REQ_OP_DISCARD)) { + unsigned int bi_size = dio->bio_details.bi_iter.bi_size; + unsigned int max_size = likely(checksums != checksums_onstack) ? PAGE_SIZE : HASH_MAX_DIGESTSIZE; + unsigned int max_blocks = max_size / ic->tag_size; + + memset(checksums, DISCARD_FILLER, max_size); + + while (bi_size) { + unsigned int this_step_blocks = bi_size >> (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); + + this_step_blocks = min(this_step_blocks, max_blocks); + r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, + this_step_blocks * ic->tag_size, TAG_WRITE); + if (unlikely(r)) { + if (likely(checksums != checksums_onstack)) + kfree(checksums); + goto error; + } + + bi_size -= this_step_blocks << (SECTOR_SHIFT + ic->sb->log2_sectors_per_block); + } + + if (likely(checksums != checksums_onstack)) + kfree(checksums); + goto skip_io; + } + + sector = dio->range.logical_sector; + sectors_to_process = dio->range.n_sectors; + + __bio_for_each_segment(bv, bio, iter, dio->bio_details.bi_iter) { + struct bio_vec bv_copy = bv; + unsigned int pos; char *mem, *checksums_ptr; again: - mem = (char *)kmap_atomic(bv.bv_page) + bv.bv_offset; + mem = integrity_kmap(ic, bv_copy.bv_page); pos = 0; checksums_ptr = checksums; do { - integrity_sector_checksum(ic, sector, mem + pos, checksums_ptr); + integrity_sector_checksum(ic, &dio->ahash_req, sector, mem, bv_copy.bv_offset + pos, checksums_ptr); checksums_ptr += ic->tag_size; sectors_to_process -= ic->sectors_per_block; pos += ic->sectors_per_block << SECTOR_SHIFT; sector += ic->sectors_per_block; - } while (pos < bv.bv_len && sectors_to_process && checksums != checksums_onstack); - kunmap_atomic(mem); + } while (pos < bv_copy.bv_len && sectors_to_process && checksums != checksums_onstack); + integrity_kunmap(ic, mem); r = dm_integrity_rw_tag(ic, checksums, &dio->metadata_block, &dio->metadata_offset, - checksums_ptr - checksums, !dio->write ? TAG_CMP : TAG_WRITE); + checksums_ptr - checksums, dio->op == REQ_OP_READ ? TAG_CMP : TAG_WRITE); if (unlikely(r)) { - if (r > 0) { - DMERR("Checksum failed at sector 0x%llx", - (unsigned long long)(sector - ((r + ic->tag_size - 1) / ic->tag_size))); - r = -EILSEQ; - atomic64_inc(&ic->number_of_mismatches); - } if (likely(checksums != checksums_onstack)) kfree(checksums); + if (r > 0) { + integrity_recheck(dio, checksums_onstack); + goto skip_io; + } goto error; } if (!sectors_to_process) break; - if (unlikely(pos < bv.bv_len)) { - bv.bv_offset += pos; - bv.bv_len -= pos; + if (unlikely(pos < bv_copy.bv_len)) { + bv_copy.bv_offset += pos; + bv_copy.bv_len -= pos; goto again; } } @@ -1391,24 +1982,25 @@ again: if (likely(checksums != checksums_onstack)) kfree(checksums); } else { - struct bio_integrity_payload *bip = dio->orig_bi_integrity; + struct bio_integrity_payload *bip = dio->bio_details.bi_integrity; if (bip) { struct bio_vec biv; struct bvec_iter iter; - unsigned data_to_process = dio->range.n_sectors; + unsigned int data_to_process = dio->range.n_sectors; + sector_to_block(ic, data_to_process); data_to_process *= ic->tag_size; bip_for_each_vec(biv, bip, iter) { unsigned char *tag; - unsigned this_len; + unsigned int this_len; BUG_ON(PageHighMem(biv.bv_page)); - tag = lowmem_page_address(biv.bv_page) + biv.bv_offset; + tag = bvec_virt(&biv); this_len = min(biv.bv_len, data_to_process); r = dm_integrity_rw_tag(ic, tag, &dio->metadata_block, &dio->metadata_offset, - this_len, !dio->write ? TAG_READ : TAG_WRITE); + this_len, dio->op == REQ_OP_READ ? TAG_READ : TAG_WRITE); if (unlikely(r)) goto error; data_to_process -= this_len; @@ -1425,6 +2017,35 @@ error: dec_in_flight(dio); } +static inline bool dm_integrity_check_limits(struct dm_integrity_c *ic, sector_t logical_sector, struct bio *bio) +{ + if (unlikely(logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { + DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx", + logical_sector, bio_sectors(bio), + ic->provided_data_sectors); + return false; + } + if (unlikely((logical_sector | bio_sectors(bio)) & (unsigned int)(ic->sectors_per_block - 1))) { + DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x", + ic->sectors_per_block, + logical_sector, bio_sectors(bio)); + return false; + } + if (ic->sectors_per_block > 1 && likely(bio_op(bio) != REQ_OP_DISCARD)) { + struct bvec_iter iter; + struct bio_vec bv; + + bio_for_each_segment(bv, bio, iter) { + if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { + DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary", + bv.bv_offset, bv.bv_len, ic->sectors_per_block); + return false; + } + } + } + return true; +} + static int dm_integrity_map(struct dm_target *ti, struct bio *bio) { struct dm_integrity_c *ic = ti->private; @@ -1435,6 +2056,31 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) dio->ic = ic; dio->bi_status = 0; + dio->op = bio_op(bio); + dio->ahash_req = NULL; + + if (ic->mode == 'I') { + bio->bi_iter.bi_sector = dm_target_offset(ic->ti, bio->bi_iter.bi_sector); + dio->integrity_payload = NULL; + dio->integrity_payload_from_mempool = false; + dio->integrity_range_locked = false; + return dm_integrity_map_inline(dio, true); + } + + if (unlikely(dio->op == REQ_OP_DISCARD)) { + if (ti->max_io_len) { + sector_t sec = dm_target_offset(ti, bio->bi_iter.bi_sector); + unsigned int log2_max_io_len = __fls(ti->max_io_len); + sector_t start_boundary = sec >> log2_max_io_len; + sector_t end_boundary = (sec + bio_sectors(bio) - 1) >> log2_max_io_len; + + if (start_boundary < end_boundary) { + sector_t len = ti->max_io_len - (sec & (ti->max_io_len - 1)); + + dm_accept_partial_bio(bio, len); + } + } + } if (unlikely(bio->bi_opf & REQ_PREFLUSH)) { submit_flush_bio(ic, dio); @@ -1442,8 +2088,7 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) } dio->range.logical_sector = dm_target_offset(ti, bio->bi_iter.bi_sector); - dio->write = bio_op(bio) == REQ_OP_WRITE; - dio->fua = dio->write && bio->bi_opf & REQ_FUA; + dio->fua = dio->op == REQ_OP_WRITE && bio->bi_opf & REQ_FUA; if (unlikely(dio->fua)) { /* * Don't pass down the FUA flag because we have to flush @@ -1451,41 +2096,21 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) */ bio->bi_opf &= ~REQ_FUA; } - if (unlikely(dio->range.logical_sector + bio_sectors(bio) > ic->provided_data_sectors)) { - DMERR("Too big sector number: 0x%llx + 0x%x > 0x%llx", - (unsigned long long)dio->range.logical_sector, bio_sectors(bio), - (unsigned long long)ic->provided_data_sectors); - return DM_MAPIO_KILL; - } - if (unlikely((dio->range.logical_sector | bio_sectors(bio)) & (unsigned)(ic->sectors_per_block - 1))) { - DMERR("Bio not aligned on %u sectors: 0x%llx, 0x%x", - ic->sectors_per_block, - (unsigned long long)dio->range.logical_sector, bio_sectors(bio)); + if (unlikely(!dm_integrity_check_limits(ic, dio->range.logical_sector, bio))) return DM_MAPIO_KILL; - } - - if (ic->sectors_per_block > 1) { - struct bvec_iter iter; - struct bio_vec bv; - bio_for_each_segment(bv, bio, iter) { - if (unlikely(bv.bv_len & ((ic->sectors_per_block << SECTOR_SHIFT) - 1))) { - DMERR("Bio vector (%u,%u) is not aligned on %u-sector boundary", - bv.bv_offset, bv.bv_len, ic->sectors_per_block); - return DM_MAPIO_KILL; - } - } - } bip = bio_integrity(bio); if (!ic->internal_hash) { if (bip) { - unsigned wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; + unsigned int wanted_tag_size = bio_sectors(bio) >> ic->sb->log2_sectors_per_block; + if (ic->log2_tag_size >= 0) wanted_tag_size <<= ic->log2_tag_size; else wanted_tag_size *= ic->tag_size; if (unlikely(wanted_tag_size != bip->bip_iter.bi_size)) { - DMERR("Invalid integrity data size %u, expected %u", bip->bip_iter.bi_size, wanted_tag_size); + DMERR("Invalid integrity data size %u, expected %u", + bip->bip_iter.bi_size, wanted_tag_size); return DM_MAPIO_KILL; } } @@ -1496,7 +2121,7 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) } } - if (unlikely(ic->mode == 'R') && unlikely(dio->write)) + if (unlikely(ic->mode == 'R') && unlikely(dio->op != REQ_OP_READ)) return DM_MAPIO_KILL; get_area_and_offset(ic, dio->range.logical_sector, &area, &offset); @@ -1508,11 +2133,11 @@ static int dm_integrity_map(struct dm_target *ti, struct bio *bio) } static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, - unsigned journal_section, unsigned journal_entry) + unsigned int journal_section, unsigned int journal_entry) { struct dm_integrity_c *ic = dio->ic; sector_t logical_sector; - unsigned n_sectors; + unsigned int n_sectors; logical_sector = dio->range.logical_sector; n_sectors = dio->range.n_sectors; @@ -1525,21 +2150,21 @@ static bool __journal_read_write(struct dm_integrity_io *dio, struct bio *bio, n_sectors -= bv.bv_len >> SECTOR_SHIFT; bio_advance_iter(bio, &bio->bi_iter, bv.bv_len); retry_kmap: - mem = kmap_atomic(bv.bv_page); - if (likely(dio->write)) + mem = kmap_local_page(bv.bv_page); + if (likely(dio->op == REQ_OP_WRITE)) flush_dcache_page(bv.bv_page); do { struct journal_entry *je = access_journal_entry(ic, journal_section, journal_entry); - if (unlikely(!dio->write)) { + if (unlikely(dio->op == REQ_OP_READ)) { struct journal_sector *js; char *mem_ptr; - unsigned s; + unsigned int s; if (unlikely(journal_entry_is_inprogress(je))) { flush_dcache_page(bv.bv_page); - kunmap_atomic(mem); + kunmap_local(mem); __io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); goto retry_kmap; @@ -1555,46 +2180,36 @@ retry_kmap: js++; mem_ptr += 1 << SECTOR_SHIFT; } while (++s < ic->sectors_per_block); -#ifdef INTERNAL_VERIFY - if (ic->internal_hash) { - char checksums_onstack[max(HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; - - integrity_sector_checksum(ic, logical_sector, mem + bv.bv_offset, checksums_onstack); - if (unlikely(memcmp(checksums_onstack, journal_entry_tag(ic, je), ic->tag_size))) { - DMERR("Checksum failed when reading from journal, at sector 0x%llx", - (unsigned long long)logical_sector); - } - } -#endif } if (!ic->internal_hash) { struct bio_integrity_payload *bip = bio_integrity(bio); - unsigned tag_todo = ic->tag_size; + unsigned int tag_todo = ic->tag_size; char *tag_ptr = journal_entry_tag(ic, je); - if (bip) do { - struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); - unsigned tag_now = min(biv.bv_len, tag_todo); - char *tag_addr; - BUG_ON(PageHighMem(biv.bv_page)); - tag_addr = lowmem_page_address(biv.bv_page) + biv.bv_offset; - if (likely(dio->write)) - memcpy(tag_ptr, tag_addr, tag_now); - else - memcpy(tag_addr, tag_ptr, tag_now); - bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now); - tag_ptr += tag_now; - tag_todo -= tag_now; - } while (unlikely(tag_todo)); else { - if (likely(dio->write)) - memset(tag_ptr, 0, tag_todo); - } + if (bip) { + do { + struct bio_vec biv = bvec_iter_bvec(bip->bip_vec, bip->bip_iter); + unsigned int tag_now = min(biv.bv_len, tag_todo); + char *tag_addr; + + BUG_ON(PageHighMem(biv.bv_page)); + tag_addr = bvec_virt(&biv); + if (likely(dio->op == REQ_OP_WRITE)) + memcpy(tag_ptr, tag_addr, tag_now); + else + memcpy(tag_addr, tag_ptr, tag_now); + bvec_iter_advance(bip->bip_vec, &bip->bip_iter, tag_now); + tag_ptr += tag_now; + tag_todo -= tag_now; + } while (unlikely(tag_todo)); + } else if (likely(dio->op == REQ_OP_WRITE)) + memset(tag_ptr, 0, tag_todo); } - if (likely(dio->write)) { + if (likely(dio->op == REQ_OP_WRITE)) { struct journal_sector *js; - unsigned s; + unsigned int s; js = access_journal_data(ic, journal_section, journal_entry); memcpy(js, mem + bv.bv_offset, ic->sectors_per_block << SECTOR_SHIFT); @@ -1605,13 +2220,17 @@ retry_kmap: } while (++s < ic->sectors_per_block); if (ic->internal_hash) { - unsigned digest_size = crypto_shash_digestsize(ic->internal_hash); + unsigned int digest_size = ic->internal_hash_digestsize; + void *js_page = integrity_identity(ic, (char *)js - offset_in_page(js)); + unsigned js_offset = offset_in_page(js); + if (unlikely(digest_size > ic->tag_size)) { char checksums_onstack[HASH_MAX_DIGESTSIZE]; - integrity_sector_checksum(ic, logical_sector, (char *)js, checksums_onstack); + + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, checksums_onstack); memcpy(journal_entry_tag(ic, je), checksums_onstack, ic->tag_size); } else - integrity_sector_checksum(ic, logical_sector, (char *)js, journal_entry_tag(ic, je)); + integrity_sector_checksum(ic, &dio->ahash_req, logical_sector, js_page, js_offset, journal_entry_tag(ic, je)); } journal_entry_set_sector(je, logical_sector); @@ -1628,23 +2247,21 @@ retry_kmap: bv.bv_offset += ic->sectors_per_block << SECTOR_SHIFT; } while (bv.bv_len -= ic->sectors_per_block << SECTOR_SHIFT); - if (unlikely(!dio->write)) + if (unlikely(dio->op == REQ_OP_READ)) flush_dcache_page(bv.bv_page); - kunmap_atomic(mem); + kunmap_local(mem); } while (n_sectors); - if (likely(dio->write)) { + if (likely(dio->op == REQ_OP_WRITE)) { smp_mb(); if (unlikely(waitqueue_active(&ic->copy_to_journal_wait))) wake_up(&ic->copy_to_journal_wait); - if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) { + if (READ_ONCE(ic->free_sectors) <= ic->free_sectors_threshold) queue_work(ic->commit_wq, &ic->commit_work); - } else { + else schedule_autocommit(ic); - } - } else { + } else remove_range(ic, &dio->range); - } if (unlikely(bio->bi_iter.bi_size)) { sector_t area, offset; @@ -1662,14 +2279,19 @@ static void dm_integrity_map_continue(struct dm_integrity_io *dio, bool from_map { struct dm_integrity_c *ic = dio->ic; struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); - unsigned journal_section, journal_entry; - unsigned journal_read_pos; + unsigned int journal_section, journal_entry; + unsigned int journal_read_pos; + sector_t recalc_sector; struct completion read_comp; - bool need_sync_io = ic->internal_hash && !dio->write; + bool discard_retried = false; + bool need_sync_io = ic->internal_hash && dio->op == REQ_OP_READ; + + if (unlikely(dio->op == REQ_OP_DISCARD) && ic->mode != 'D') + need_sync_io = true; if (need_sync_io && from_map) { INIT_WORK(&dio->work, integrity_bio_wait); - queue_work(ic->metadata_wq, &dio->work); + queue_work(ic->offload_wq, &dio->work); return; } @@ -1683,13 +2305,13 @@ retry: } dio->range.n_sectors = bio_sectors(bio); journal_read_pos = NOT_FOUND; - if (likely(ic->mode == 'J')) { - if (dio->write) { - unsigned next_entry, i, pos; - unsigned ws, we, range_sectors; + if (ic->mode == 'J' && likely(dio->op != REQ_OP_DISCARD)) { + if (dio->op == REQ_OP_WRITE) { + unsigned int next_entry, i, pos; + unsigned int ws, we, range_sectors; dio->range.n_sectors = min(dio->range.n_sectors, - ic->free_sectors << ic->sb->log2_sectors_per_block); + (sector_t)ic->free_sectors << ic->sb->log2_sectors_per_block); if (unlikely(!dio->range.n_sectors)) { if (from_map) goto offload_to_thread; @@ -1734,13 +2356,15 @@ retry: goto journal_read_write; } else { sector_t next_sector; + journal_read_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); if (likely(journal_read_pos == NOT_FOUND)) { if (unlikely(dio->range.n_sectors > next_sector - dio->range.logical_sector)) dio->range.n_sectors = next_sector - dio->range.logical_sector; } else { - unsigned i; - unsigned jp = journal_read_pos + 1; + unsigned int i; + unsigned int jp = journal_read_pos + 1; + for (i = ic->sectors_per_block; i < dio->range.n_sectors; i += ic->sectors_per_block, jp++) { if (!test_journal_node(ic, jp, dio->range.logical_sector + i)) break; @@ -1762,8 +2386,43 @@ offload_to_thread: queue_work(ic->wait_wq, &dio->work); return; } + if (journal_read_pos != NOT_FOUND) + dio->range.n_sectors = ic->sectors_per_block; wait_and_add_new_range(ic, &dio->range); + /* + * wait_and_add_new_range drops the spinlock, so the journal + * may have been changed arbitrarily. We need to recheck. + * To simplify the code, we restrict I/O size to just one block. + */ + if (journal_read_pos != NOT_FOUND) { + sector_t next_sector; + unsigned int new_pos; + + new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); + if (unlikely(new_pos != journal_read_pos)) { + remove_range_unlocked(ic, &dio->range); + goto retry; + } + } } + if (ic->mode == 'J' && likely(dio->op == REQ_OP_DISCARD) && !discard_retried) { + sector_t next_sector; + unsigned int new_pos; + + new_pos = find_journal_node(ic, dio->range.logical_sector, &next_sector); + if (unlikely(new_pos != NOT_FOUND) || + unlikely(next_sector < dio->range.logical_sector - dio->range.n_sectors)) { + remove_range_unlocked(ic, &dio->range); + spin_unlock_irq(&ic->endio_wait.lock); + queue_work(ic->commit_wq, &ic->commit_work); + flush_workqueue(ic->commit_wq); + queue_work(ic->writer_wq, &ic->writer_work); + flush_workqueue(ic->writer_wq); + discard_retried = true; + goto lock_retry; + } + } + recalc_sector = le64_to_cpu(ic->sb->recalc_sector); spin_unlock_irq(&ic->endio_wait.lock); if (unlikely(journal_read_pos != NOT_FOUND)) { @@ -1772,6 +2431,20 @@ offload_to_thread: goto journal_read_write; } + if (ic->mode == 'B' && (dio->op == REQ_OP_WRITE || unlikely(dio->op == REQ_OP_DISCARD))) { + if (!block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, + dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { + struct bitmap_block_status *bbs; + + bbs = sector_to_bitmap_block(ic, dio->range.logical_sector); + spin_lock(&bbs->bio_queue_lock); + bio_list_add(&bbs->bio_queue, bio); + spin_unlock(&bbs->bio_queue_lock); + queue_work(ic->writer_wq, &bbs->work); + return; + } + } + dio->in_flight = (atomic_t)ATOMIC_INIT(2); if (need_sync_io) { @@ -1780,34 +2453,43 @@ offload_to_thread: } else dio->completion = NULL; - dio->orig_bi_iter = bio->bi_iter; - - dio->orig_bi_disk = bio->bi_disk; - dio->orig_bi_partno = bio->bi_partno; + dm_bio_record(&dio->bio_details, bio); bio_set_dev(bio, ic->dev->bdev); - - dio->orig_bi_integrity = bio_integrity(bio); bio->bi_integrity = NULL; bio->bi_opf &= ~REQ_INTEGRITY; - - dio->orig_bi_end_io = bio->bi_end_io; bio->bi_end_io = integrity_end_io; - bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; - generic_make_request(bio); + + if (unlikely(dio->op == REQ_OP_DISCARD) && likely(ic->mode != 'D')) { + integrity_metadata(&dio->work); + dm_integrity_flush_buffers(ic, false); + + dio->in_flight = (atomic_t)ATOMIC_INIT(1); + dio->completion = NULL; + + submit_bio_noacct(bio); + + return; + } + + submit_bio_noacct(bio); if (need_sync_io) { wait_for_completion_io(&read_comp); - if (unlikely(ic->recalc_wq != NULL) && - ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && - dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + dio->range.logical_sector + dio->range.n_sectors > recalc_sector) goto skip_check; + if (ic->mode == 'B') { + if (!block_bitmap_op(ic, ic->recalc_bitmap, dio->range.logical_sector, + dio->range.n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) + goto skip_check; + } + if (likely(!bio->bi_status)) integrity_metadata(&dio->work); else skip_check: dec_in_flight(dio); - } else { INIT_WORK(&dio->work, integrity_metadata); queue_work(ic->metadata_wq, &dio->work); @@ -1822,12 +2504,299 @@ journal_read_write: do_endio_flush(ic, dio); } +static int dm_integrity_map_inline(struct dm_integrity_io *dio, bool from_map) +{ + struct dm_integrity_c *ic = dio->ic; + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + struct bio_integrity_payload *bip; + unsigned ret; + sector_t recalc_sector; + + if (unlikely(bio_integrity(bio))) { + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + return DM_MAPIO_SUBMITTED; + } + + bio_set_dev(bio, ic->dev->bdev); + if (unlikely((bio->bi_opf & REQ_PREFLUSH) != 0)) + return DM_MAPIO_REMAPPED; + +retry: + if (!dio->integrity_payload) { + unsigned digest_size, extra_size; + dio->payload_len = ic->tuple_size * (bio_sectors(bio) >> ic->sb->log2_sectors_per_block); + digest_size = ic->internal_hash_digestsize; + extra_size = unlikely(digest_size > ic->tag_size) ? digest_size - ic->tag_size : 0; + dio->payload_len += extra_size; + dio->integrity_payload = kmalloc(dio->payload_len, GFP_NOIO | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN); + if (unlikely(!dio->integrity_payload)) { + const unsigned x_size = PAGE_SIZE << 1; + if (dio->payload_len > x_size) { + unsigned sectors = ((x_size - extra_size) / ic->tuple_size) << ic->sb->log2_sectors_per_block; + if (WARN_ON(!sectors || sectors >= bio_sectors(bio))) { + bio->bi_status = BLK_STS_NOTSUPP; + bio_endio(bio); + return DM_MAPIO_SUBMITTED; + } + dm_accept_partial_bio(bio, sectors); + goto retry; + } + } + } + + dio->range.logical_sector = bio->bi_iter.bi_sector; + dio->range.n_sectors = bio_sectors(bio); + + if (!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) + goto skip_spinlock; +#ifdef CONFIG_64BIT + /* + * On 64-bit CPUs we can optimize the lock away (so that it won't cause + * cache line bouncing) and use acquire/release barriers instead. + * + * Paired with smp_store_release in integrity_recalc_inline. + */ + recalc_sector = le64_to_cpu(smp_load_acquire(&ic->sb->recalc_sector)); + if (likely(dio->range.logical_sector + dio->range.n_sectors <= recalc_sector)) + goto skip_spinlock; +#endif + spin_lock_irq(&ic->endio_wait.lock); + recalc_sector = le64_to_cpu(ic->sb->recalc_sector); + if (dio->range.logical_sector + dio->range.n_sectors <= recalc_sector) + goto skip_unlock; + if (unlikely(!add_new_range(ic, &dio->range, true))) { + if (from_map) { + spin_unlock_irq(&ic->endio_wait.lock); + INIT_WORK(&dio->work, integrity_bio_wait); + queue_work(ic->wait_wq, &dio->work); + return DM_MAPIO_SUBMITTED; + } + wait_and_add_new_range(ic, &dio->range); + } + dio->integrity_range_locked = true; +skip_unlock: + spin_unlock_irq(&ic->endio_wait.lock); +skip_spinlock: + + if (unlikely(!dio->integrity_payload)) { + dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO)); + dio->integrity_payload_from_mempool = true; + } + + dio->bio_details.bi_iter = bio->bi_iter; + + if (unlikely(!dm_integrity_check_limits(ic, bio->bi_iter.bi_sector, bio))) { + return DM_MAPIO_KILL; + } + + bio->bi_iter.bi_sector += ic->start + SB_SECTORS; + + bip = bio_integrity_alloc(bio, GFP_NOIO, 1); + if (IS_ERR(bip)) { + bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); + bio_endio(bio); + return DM_MAPIO_SUBMITTED; + } + + if (dio->op == REQ_OP_WRITE) { + unsigned pos = 0; + while (dio->bio_details.bi_iter.bi_size) { + struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); + const char *mem = integrity_kmap(ic, bv.bv_page); + if (ic->tag_size < ic->tuple_size) + memset(dio->integrity_payload + pos + ic->tag_size, 0, ic->tuple_size - ic->tuple_size); + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, dio->integrity_payload + pos); + integrity_kunmap(ic, mem); + pos += ic->tuple_size; + bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); + } + } + + ret = bio_integrity_add_page(bio, virt_to_page(dio->integrity_payload), + dio->payload_len, offset_in_page(dio->integrity_payload)); + if (unlikely(ret != dio->payload_len)) { + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return DM_MAPIO_SUBMITTED; + } + + return DM_MAPIO_REMAPPED; +} + +static inline void dm_integrity_free_payload(struct dm_integrity_io *dio) +{ + struct dm_integrity_c *ic = dio->ic; + if (unlikely(dio->integrity_payload_from_mempool)) + mempool_free(virt_to_page(dio->integrity_payload), &ic->recheck_pool); + else + kfree(dio->integrity_payload); + dio->integrity_payload = NULL; + dio->integrity_payload_from_mempool = false; +} + +static void dm_integrity_inline_recheck(struct work_struct *w) +{ + struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + struct dm_integrity_c *ic = dio->ic; + struct bio *outgoing_bio; + void *outgoing_data; + + dio->integrity_payload = page_to_virt((struct page *)mempool_alloc(&ic->recheck_pool, GFP_NOIO)); + dio->integrity_payload_from_mempool = true; + + outgoing_data = dio->integrity_payload + PAGE_SIZE; + + while (dio->bio_details.bi_iter.bi_size) { + char digest[HASH_MAX_DIGESTSIZE]; + int r; + struct bio_integrity_payload *bip; + struct bio_vec bv; + char *mem; + + outgoing_bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recheck_bios); + bio_add_virt_nofail(outgoing_bio, outgoing_data, + ic->sectors_per_block << SECTOR_SHIFT); + + bip = bio_integrity_alloc(outgoing_bio, GFP_NOIO, 1); + if (IS_ERR(bip)) { + bio_put(outgoing_bio); + bio->bi_status = errno_to_blk_status(PTR_ERR(bip)); + bio_endio(bio); + return; + } + + r = bio_integrity_add_page(outgoing_bio, virt_to_page(dio->integrity_payload), ic->tuple_size, 0); + if (unlikely(r != ic->tuple_size)) { + bio_put(outgoing_bio); + bio->bi_status = BLK_STS_RESOURCE; + bio_endio(bio); + return; + } + + outgoing_bio->bi_iter.bi_sector = dio->bio_details.bi_iter.bi_sector + ic->start + SB_SECTORS; + + r = submit_bio_wait(outgoing_bio); + if (unlikely(r != 0)) { + bio_put(outgoing_bio); + bio->bi_status = errno_to_blk_status(r); + bio_endio(bio); + return; + } + bio_put(outgoing_bio); + + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, integrity_identity(ic, outgoing_data), 0, digest); + if (unlikely(crypto_memneq(digest, dio->integrity_payload, min(ic->internal_hash_digestsize, ic->tag_size)))) { + DMERR_LIMIT("%pg: Checksum failed at sector 0x%llx", + ic->dev->bdev, dio->bio_details.bi_iter.bi_sector); + atomic64_inc(&ic->number_of_mismatches); + dm_audit_log_bio(DM_MSG_PREFIX, "integrity-checksum", + bio, dio->bio_details.bi_iter.bi_sector, 0); + + bio->bi_status = BLK_STS_PROTECTION; + bio_endio(bio); + return; + } + + bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); + mem = bvec_kmap_local(&bv); + memcpy(mem, outgoing_data, ic->sectors_per_block << SECTOR_SHIFT); + kunmap_local(mem); + + bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); + } + + bio_endio(bio); +} + +static inline bool dm_integrity_check(struct dm_integrity_c *ic, struct dm_integrity_io *dio) +{ + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + unsigned pos = 0; + + while (dio->bio_details.bi_iter.bi_size) { + char digest[HASH_MAX_DIGESTSIZE]; + struct bio_vec bv = bio_iter_iovec(bio, dio->bio_details.bi_iter); + char *mem = integrity_kmap(ic, bv.bv_page); + integrity_sector_checksum(ic, &dio->ahash_req, dio->bio_details.bi_iter.bi_sector, mem, bv.bv_offset, digest); + if (unlikely(crypto_memneq(digest, dio->integrity_payload + pos, + min(ic->internal_hash_digestsize, ic->tag_size)))) { + integrity_kunmap(ic, mem); + dm_integrity_free_payload(dio); + INIT_WORK(&dio->work, dm_integrity_inline_recheck); + queue_work(ic->offload_wq, &dio->work); + return false; + } + integrity_kunmap(ic, mem); + pos += ic->tuple_size; + bio_advance_iter_single(bio, &dio->bio_details.bi_iter, ic->sectors_per_block << SECTOR_SHIFT); + } + + return true; +} + +static void dm_integrity_inline_async_check(struct work_struct *w) +{ + struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); + struct dm_integrity_c *ic = dio->ic; + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + + if (likely(dm_integrity_check(ic, dio))) + bio_endio(bio); +} + +static int dm_integrity_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status) +{ + struct dm_integrity_c *ic = ti->private; + struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); + if (ic->mode == 'I') { + if (dio->op == REQ_OP_READ && likely(*status == BLK_STS_OK) && likely(dio->bio_details.bi_iter.bi_size != 0)) { + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + unlikely(dio->integrity_range_locked)) + goto skip_check; + if (likely(ic->internal_shash != NULL)) { + if (unlikely(!dm_integrity_check(ic, dio))) + return DM_ENDIO_INCOMPLETE; + } else { + INIT_WORK(&dio->work, dm_integrity_inline_async_check); + queue_work(ic->offload_wq, &dio->work); + return DM_ENDIO_INCOMPLETE; + } + } +skip_check: + dm_integrity_free_payload(dio); + if (unlikely(dio->integrity_range_locked)) + remove_range(ic, &dio->range); + } + if (unlikely(dio->ahash_req)) + mempool_free(dio->ahash_req, &ic->ahash_req_pool); + return DM_ENDIO_DONE; +} static void integrity_bio_wait(struct work_struct *w) { struct dm_integrity_io *dio = container_of(w, struct dm_integrity_io, work); + struct dm_integrity_c *ic = dio->ic; - dm_integrity_map_continue(dio, false); + if (ic->mode == 'I') { + struct bio *bio = dm_bio_from_per_bio_data(dio, sizeof(struct dm_integrity_io)); + int r = dm_integrity_map_inline(dio, false); + switch (r) { + case DM_MAPIO_KILL: + bio->bi_status = BLK_STS_IOERR; + fallthrough; + case DM_MAPIO_REMAPPED: + submit_bio_noacct(bio); + fallthrough; + case DM_MAPIO_SUBMITTED: + return; + default: + BUG(); + } + } else { + dm_integrity_map_continue(dio, false); + } } static void pad_uncommitted(struct dm_integrity_c *ic) @@ -1839,24 +2808,35 @@ static void pad_uncommitted(struct dm_integrity_c *ic) wraparound_section(ic, &ic->free_section); ic->n_uncommitted_sections++; } - WARN_ON(ic->journal_sections * ic->journal_section_entries != - (ic->n_uncommitted_sections + ic->n_committed_sections) * ic->journal_section_entries + ic->free_sectors); + if (WARN_ON(ic->journal_sections * ic->journal_section_entries != + (ic->n_uncommitted_sections + ic->n_committed_sections) * + ic->journal_section_entries + ic->free_sectors)) { + DMCRIT("journal_sections %u, journal_section_entries %u, " + "n_uncommitted_sections %u, n_committed_sections %u, " + "journal_section_entries %u, free_sectors %u", + ic->journal_sections, ic->journal_section_entries, + ic->n_uncommitted_sections, ic->n_committed_sections, + ic->journal_section_entries, ic->free_sectors); + } } static void integrity_commit(struct work_struct *w) { struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, commit_work); - unsigned commit_start, commit_sections; - unsigned i, j, n; + unsigned int commit_start, commit_sections; + unsigned int i, j, n; struct bio *flushes; - del_timer(&ic->autocommit_timer); + timer_delete(&ic->autocommit_timer); + + if (ic->mode == 'I') + return; spin_lock_irq(&ic->endio_wait.lock); flushes = bio_list_get(&ic->flush_bio_list); if (unlikely(ic->mode != 'J')) { spin_unlock_irq(&ic->endio_wait.lock); - dm_integrity_flush_buffers(ic); + dm_integrity_flush_buffers(ic, true); goto release_flush_bios; } @@ -1868,15 +2848,19 @@ static void integrity_commit(struct work_struct *w) if (!commit_sections) goto release_flush_bios; + ic->wrote_to_journal = true; + i = commit_start; for (n = 0; n < commit_sections; n++) { for (j = 0; j < ic->journal_section_entries; j++) { struct journal_entry *je; + je = access_journal_entry(ic, i, j); io_wait_event(ic->copy_to_journal_wait, !journal_entry_is_inprogress(je)); } for (j = 0; j < ic->journal_section_sectors; j++) { struct journal_sector *js; + js = access_journal(ic, i, j); js->commit_id = dm_integrity_commit_id(ic, i, j, ic->commit_seq); } @@ -1902,6 +2886,7 @@ static void integrity_commit(struct work_struct *w) release_flush_bios: while (flushes) { struct bio *next = flushes->bi_next; + flushes->bi_next = NULL; do_endio(ic, flushes); flushes = next; @@ -1913,6 +2898,7 @@ static void complete_copy_from_journal(unsigned long error, void *context) struct journal_io *io = context; struct journal_completion *comp = io->comp; struct dm_integrity_c *ic = comp->ic; + remove_range(ic, &io->range); mempool_free(io, &ic->journal_io_mempool); if (unlikely(error != 0)) @@ -1923,17 +2909,18 @@ static void complete_copy_from_journal(unsigned long error, void *context) static void restore_last_bytes(struct dm_integrity_c *ic, struct journal_sector *js, struct journal_entry *je) { - unsigned s = 0; + unsigned int s = 0; + do { js->commit_id = je->last_bytes[s]; js++; } while (++s < ic->sectors_per_block); } -static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, - unsigned write_sections, bool from_replay) +static void do_journal_write(struct dm_integrity_c *ic, unsigned int write_start, + unsigned int write_sections, bool from_replay) { - unsigned i, j, n; + unsigned int i, j, n; struct journal_completion comp; struct blk_plug plug; @@ -1952,9 +2939,9 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, for (j = 0; j < ic->journal_section_entries; j++) { struct journal_entry *je = access_journal_entry(ic, i, j); sector_t sec, area, offset; - unsigned k, l, next_loop; + unsigned int k, l, next_loop; sector_t metadata_block; - unsigned metadata_offset; + unsigned int metadata_offset; struct journal_io *io; if (journal_entry_is_unused(je)) @@ -1962,20 +2949,27 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, BUG_ON(unlikely(journal_entry_is_inprogress(je)) && !from_replay); sec = journal_entry_get_sector(je); if (unlikely(from_replay)) { - if (unlikely(sec & (unsigned)(ic->sectors_per_block - 1))) { + if (unlikely(sec & (unsigned int)(ic->sectors_per_block - 1))) { dm_integrity_io_error(ic, "invalid sector in journal", -EIO); sec &= ~(sector_t)(ic->sectors_per_block - 1); } + if (unlikely(sec >= ic->provided_data_sectors)) { + journal_entry_set_unused(je); + continue; + } } get_area_and_offset(ic, sec, &area, &offset); restore_last_bytes(ic, access_journal_data(ic, i, j), je); for (k = j + 1; k < ic->journal_section_entries; k++) { struct journal_entry *je2 = access_journal_entry(ic, i, k); sector_t sec2, area2, offset2; + if (journal_entry_is_unused(je2)) break; BUG_ON(unlikely(journal_entry_is_inprogress(je2)) && !from_replay); sec2 = journal_entry_get_sector(je2); + if (unlikely(sec2 >= ic->provided_data_sectors)) + break; get_area_and_offset(ic, sec2, &area2, &offset2); if (area2 != area || offset2 != offset + ((k - j) << ic->sb->log2_sectors_per_block)) break; @@ -1989,8 +2983,7 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; spin_lock_irq(&ic->endio_wait.lock); - if (unlikely(!add_new_range(ic, &io->range, true))) - wait_and_add_new_range(ic, &io->range); + add_new_range_and_wait(ic, &io->range); if (likely(!from_replay)) { struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; @@ -2018,9 +3011,8 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, mempool_free(io, &ic->journal_io_mempool); goto skip_io; } - for (l = j; l < k; l++) { + for (l = j; l < k; l++) remove_journal_node(ic, §ion_node[l]); - } } spin_unlock_irq(&ic->endio_wait.lock); @@ -2034,20 +3026,24 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, unlikely(from_replay) && #endif ic->internal_hash) { - char test_tag[max_t(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; - - integrity_sector_checksum(ic, sec + ((l - j) << ic->sb->log2_sectors_per_block), - (char *)access_journal_data(ic, i, l), test_tag); - if (unlikely(memcmp(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) + char test_tag[MAX_T(size_t, HASH_MAX_DIGESTSIZE, MAX_TAG_SIZE)]; + struct journal_sector *js = access_journal_data(ic, i, l); + void *js_page = integrity_identity(ic, (char *)js - offset_in_page(js)); + unsigned js_offset = offset_in_page(js); + + integrity_sector_checksum(ic, &ic->journal_ahash_req, sec + ((l - j) << ic->sb->log2_sectors_per_block), + js_page, js_offset, test_tag); + if (unlikely(crypto_memneq(test_tag, journal_entry_tag(ic, je2), ic->tag_size))) { dm_integrity_io_error(ic, "tag mismatch when replaying journal", -EILSEQ); + dm_audit_log_target(DM_MSG_PREFIX, "integrity-replay-journal", ic->ti, 0); + } } journal_entry_set_unused(je2); r = dm_integrity_rw_tag(ic, journal_entry_tag(ic, je2), &metadata_block, &metadata_offset, ic->tag_size, TAG_WRITE); - if (unlikely(r)) { + if (unlikely(r)) dm_integrity_io_error(ic, "reading tags", r); - } } atomic_inc(&comp.in_flight); @@ -2067,19 +3063,14 @@ skip_io: complete_journal_op(&comp); wait_for_completion_io(&comp.comp); - dm_integrity_flush_buffers(ic); + dm_integrity_flush_buffers(ic, true); } static void integrity_writer(struct work_struct *w) { struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, writer_work); - unsigned write_start, write_sections; - - unsigned prev_free_sectors; - - /* the following test is not needed, but it tests the replay code */ - if (READ_ONCE(ic->suspending) && !ic->meta_dev) - return; + unsigned int write_start, write_sections; + unsigned int prev_free_sectors; spin_lock_irq(&ic->endio_wait.lock); write_start = ic->committed_section; @@ -2109,12 +3100,11 @@ static void recalc_write_super(struct dm_integrity_c *ic) { int r; - dm_integrity_flush_buffers(ic); + dm_integrity_flush_buffers(ic, false); if (dm_integrity_failed(ic)) return; - sb_set_version(ic); - r = sync_rw_sb(ic, REQ_OP_WRITE, 0); + r = sync_rw_sb(ic, REQ_OP_WRITE); if (unlikely(r)) dm_integrity_io_error(ic, "writing superblock", r); } @@ -2122,76 +3112,150 @@ static void recalc_write_super(struct dm_integrity_c *ic) static void integrity_recalc(struct work_struct *w) { struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); + size_t recalc_tags_size; + u8 *recalc_buffer = NULL; + u8 *recalc_tags = NULL; + struct ahash_request *ahash_req = NULL; struct dm_integrity_range range; struct dm_io_request io_req; struct dm_io_region io_loc; sector_t area, offset; sector_t metadata_block; - unsigned metadata_offset; + unsigned int metadata_offset; + sector_t logical_sector, n_sectors; __u8 *t; - unsigned i; + unsigned int i; int r; - unsigned super_counter = 0; + unsigned int super_counter = 0; + unsigned recalc_sectors = RECALC_SECTORS; + +retry: + recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN); + if (!recalc_buffer) { +oom: + recalc_sectors >>= 1; + if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block) + goto retry; + DMCRIT("out of memory for recalculate buffer - recalculation disabled"); + goto free_ret; + } + recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; + if (ic->internal_hash_digestsize > ic->tag_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tag_size; + recalc_tags = kvmalloc(recalc_tags_size, GFP_NOIO); + if (!recalc_tags) { + kfree(recalc_buffer); + recalc_buffer = NULL; + goto oom; + } + + DEBUG_print("start recalculation... (position %llx)\n", le64_to_cpu(ic->sb->recalc_sector)); spin_lock_irq(&ic->endio_wait.lock); next_chunk: - if (unlikely(READ_ONCE(ic->suspending))) + if (unlikely(dm_post_suspending(ic->ti))) goto unlock_ret; range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); - if (unlikely(range.logical_sector >= ic->provided_data_sectors)) + if (unlikely(range.logical_sector >= ic->provided_data_sectors)) { + if (ic->mode == 'B') { + block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); + DEBUG_print("queue_delayed_work: bitmap_flush_work\n"); + queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); + } goto unlock_ret; + } get_area_and_offset(ic, range.logical_sector, &area, &offset); - range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector); + range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector); if (!ic->meta_dev) - range.n_sectors = min(range.n_sectors, (1U << ic->sb->log2_interleave_sectors) - (unsigned)offset); - - if (unlikely(!add_new_range(ic, &range, true))) - wait_and_add_new_range(ic, &range); + range.n_sectors = min(range.n_sectors, ((sector_t)1U << ic->sb->log2_interleave_sectors) - (unsigned int)offset); + add_new_range_and_wait(ic, &range); spin_unlock_irq(&ic->endio_wait.lock); + logical_sector = range.logical_sector; + n_sectors = range.n_sectors; + + if (ic->mode == 'B') { + if (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, n_sectors, BITMAP_OP_TEST_ALL_CLEAR)) + goto advance_and_next; + + while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector, + ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { + logical_sector += ic->sectors_per_block; + n_sectors -= ic->sectors_per_block; + cond_resched(); + } + while (block_bitmap_op(ic, ic->recalc_bitmap, logical_sector + n_sectors - ic->sectors_per_block, + ic->sectors_per_block, BITMAP_OP_TEST_ALL_CLEAR)) { + n_sectors -= ic->sectors_per_block; + cond_resched(); + } + get_area_and_offset(ic, logical_sector, &area, &offset); + } + + DEBUG_print("recalculating: %llx, %llx\n", logical_sector, n_sectors); if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { recalc_write_super(ic); + if (ic->mode == 'B') + queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); + super_counter = 0; } if (unlikely(dm_integrity_failed(ic))) goto err; - io_req.bi_op = REQ_OP_READ; - io_req.bi_op_flags = 0; - io_req.mem.type = DM_IO_VMA; - io_req.mem.ptr.addr = ic->recalc_buffer; + io_req.bi_opf = REQ_OP_READ; + io_req.mem.type = DM_IO_KMEM; + io_req.mem.ptr.addr = recalc_buffer; io_req.notify.fn = NULL; io_req.client = ic->io; io_loc.bdev = ic->dev->bdev; io_loc.sector = get_data_sector(ic, area, offset); - io_loc.count = range.n_sectors; + io_loc.count = n_sectors; - r = dm_io(&io_req, 1, &io_loc, NULL); + r = dm_io(&io_req, 1, &io_loc, NULL, IOPRIO_DEFAULT); if (unlikely(r)) { dm_integrity_io_error(ic, "reading data", r); goto err; } - t = ic->recalc_tags; - for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { - integrity_sector_checksum(ic, range.logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t); + t = recalc_tags; + for (i = 0; i < n_sectors; i += ic->sectors_per_block) { + void *ptr = recalc_buffer + (i << SECTOR_SHIFT); + void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr)); + unsigned ptr_offset = offset_in_page(ptr); + integrity_sector_checksum(ic, &ahash_req, logical_sector + i, ptr_page, ptr_offset, t); t += ic->tag_size; } metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); - r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE); + r = dm_integrity_rw_tag(ic, recalc_tags, &metadata_block, &metadata_offset, t - recalc_tags, TAG_WRITE); if (unlikely(r)) { dm_integrity_io_error(ic, "writing tags", r); goto err; } + if (ic->mode == 'B') { + sector_t start, end; + + start = (range.logical_sector >> + (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << + (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + end = ((range.logical_sector + range.n_sectors) >> + (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit)) << + (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + block_bitmap_op(ic, ic->recalc_bitmap, start, end - start, BITMAP_OP_CLEAR); + } + +advance_and_next: + cond_resched(); + spin_lock_irq(&ic->endio_wait.lock); remove_range_unlocked(ic, &range); ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); @@ -2199,18 +3263,252 @@ next_chunk: err: remove_range(ic, &range); - return; + goto free_ret; unlock_ret: spin_unlock_irq(&ic->endio_wait.lock); recalc_write_super(ic); + +free_ret: + kfree(recalc_buffer); + kvfree(recalc_tags); + mempool_free(ahash_req, &ic->ahash_req_pool); } -static void init_journal(struct dm_integrity_c *ic, unsigned start_section, - unsigned n_sections, unsigned char commit_seq) +static void integrity_recalc_inline(struct work_struct *w) { - unsigned i, j, n; + struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); + size_t recalc_tags_size; + u8 *recalc_buffer = NULL; + u8 *recalc_tags = NULL; + struct ahash_request *ahash_req = NULL; + struct dm_integrity_range range; + struct bio *bio; + struct bio_integrity_payload *bip; + __u8 *t; + unsigned int i; + int r; + unsigned ret; + unsigned int super_counter = 0; + unsigned recalc_sectors = RECALC_SECTORS; + +retry: + recalc_buffer = kmalloc(recalc_sectors << SECTOR_SHIFT, GFP_NOIO | __GFP_NOWARN); + if (!recalc_buffer) { +oom: + recalc_sectors >>= 1; + if (recalc_sectors >= 1U << ic->sb->log2_sectors_per_block) + goto retry; + DMCRIT("out of memory for recalculate buffer - recalculation disabled"); + goto free_ret; + } + + recalc_tags_size = (recalc_sectors >> ic->sb->log2_sectors_per_block) * ic->tuple_size; + if (ic->internal_hash_digestsize > ic->tuple_size) + recalc_tags_size += ic->internal_hash_digestsize - ic->tuple_size; + recalc_tags = kmalloc(recalc_tags_size, GFP_NOIO | __GFP_NOWARN); + if (!recalc_tags) { + kfree(recalc_buffer); + recalc_buffer = NULL; + goto oom; + } + + spin_lock_irq(&ic->endio_wait.lock); + +next_chunk: + if (unlikely(dm_post_suspending(ic->ti))) + goto unlock_ret; + + range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); + if (unlikely(range.logical_sector >= ic->provided_data_sectors)) + goto unlock_ret; + range.n_sectors = min((sector_t)recalc_sectors, ic->provided_data_sectors - range.logical_sector); + + add_new_range_and_wait(ic, &range); + spin_unlock_irq(&ic->endio_wait.lock); + + if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { + recalc_write_super(ic); + super_counter = 0; + } + + if (unlikely(dm_integrity_failed(ic))) + goto err; + + DEBUG_print("recalculating: %llx - %llx\n", range.logical_sector, range.n_sectors); + + bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_READ, GFP_NOIO, &ic->recalc_bios); + bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; + bio_add_virt_nofail(bio, recalc_buffer, + range.n_sectors << SECTOR_SHIFT); + r = submit_bio_wait(bio); + bio_put(bio); + if (unlikely(r)) { + dm_integrity_io_error(ic, "reading data", r); + goto err; + } + + t = recalc_tags; + for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { + void *ptr = recalc_buffer + (i << SECTOR_SHIFT); + void *ptr_page = integrity_identity(ic, (char *)ptr - offset_in_page(ptr)); + unsigned ptr_offset = offset_in_page(ptr); + memset(t, 0, ic->tuple_size); + integrity_sector_checksum(ic, &ahash_req, range.logical_sector + i, ptr_page, ptr_offset, t); + t += ic->tuple_size; + } + + bio = bio_alloc_bioset(ic->dev->bdev, 1, REQ_OP_WRITE, GFP_NOIO, &ic->recalc_bios); + bio->bi_iter.bi_sector = ic->start + SB_SECTORS + range.logical_sector; + bio_add_virt_nofail(bio, recalc_buffer, + range.n_sectors << SECTOR_SHIFT); + + bip = bio_integrity_alloc(bio, GFP_NOIO, 1); + if (unlikely(IS_ERR(bip))) { + bio_put(bio); + DMCRIT("out of memory for bio integrity payload - recalculation disabled"); + goto err; + } + ret = bio_integrity_add_page(bio, virt_to_page(recalc_tags), t - recalc_tags, offset_in_page(recalc_tags)); + if (unlikely(ret != t - recalc_tags)) { + bio_put(bio); + dm_integrity_io_error(ic, "attaching integrity tags", -ENOMEM); + goto err; + } + + r = submit_bio_wait(bio); + bio_put(bio); + if (unlikely(r)) { + dm_integrity_io_error(ic, "writing data", r); + goto err; + } + + cond_resched(); + spin_lock_irq(&ic->endio_wait.lock); + remove_range_unlocked(ic, &range); +#ifdef CONFIG_64BIT + /* Paired with smp_load_acquire in dm_integrity_map_inline. */ + smp_store_release(&ic->sb->recalc_sector, cpu_to_le64(range.logical_sector + range.n_sectors)); +#else + ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); +#endif + goto next_chunk; + +err: + remove_range(ic, &range); + goto free_ret; + +unlock_ret: + spin_unlock_irq(&ic->endio_wait.lock); + + recalc_write_super(ic); + +free_ret: + kfree(recalc_buffer); + kfree(recalc_tags); + mempool_free(ahash_req, &ic->ahash_req_pool); +} + +static void bitmap_block_work(struct work_struct *w) +{ + struct bitmap_block_status *bbs = container_of(w, struct bitmap_block_status, work); + struct dm_integrity_c *ic = bbs->ic; + struct bio *bio; + struct bio_list bio_queue; + struct bio_list waiting; + + bio_list_init(&waiting); + + spin_lock(&bbs->bio_queue_lock); + bio_queue = bbs->bio_queue; + bio_list_init(&bbs->bio_queue); + spin_unlock(&bbs->bio_queue_lock); + + while ((bio = bio_list_pop(&bio_queue))) { + struct dm_integrity_io *dio; + + dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); + + if (block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, + dio->range.n_sectors, BITMAP_OP_TEST_ALL_SET)) { + remove_range(ic, &dio->range); + INIT_WORK(&dio->work, integrity_bio_wait); + queue_work(ic->offload_wq, &dio->work); + } else { + block_bitmap_op(ic, ic->journal, dio->range.logical_sector, + dio->range.n_sectors, BITMAP_OP_SET); + bio_list_add(&waiting, bio); + } + } + + if (bio_list_empty(&waiting)) + return; + + rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, + bbs->idx * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), + BITMAP_BLOCK_SIZE >> SECTOR_SHIFT, NULL); + + while ((bio = bio_list_pop(&waiting))) { + struct dm_integrity_io *dio = dm_per_bio_data(bio, sizeof(struct dm_integrity_io)); + + block_bitmap_op(ic, ic->may_write_bitmap, dio->range.logical_sector, + dio->range.n_sectors, BITMAP_OP_SET); + + remove_range(ic, &dio->range); + INIT_WORK(&dio->work, integrity_bio_wait); + queue_work(ic->offload_wq, &dio->work); + } + + queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, ic->bitmap_flush_interval); +} + +static void bitmap_flush_work(struct work_struct *work) +{ + struct dm_integrity_c *ic = container_of(work, struct dm_integrity_c, bitmap_flush_work.work); + struct dm_integrity_range range; + unsigned long limit; + struct bio *bio; + + dm_integrity_flush_buffers(ic, false); + + range.logical_sector = 0; + range.n_sectors = ic->provided_data_sectors; + + spin_lock_irq(&ic->endio_wait.lock); + add_new_range_and_wait(ic, &range); + spin_unlock_irq(&ic->endio_wait.lock); + + dm_integrity_flush_buffers(ic, true); + + limit = ic->provided_data_sectors; + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { + limit = le64_to_cpu(ic->sb->recalc_sector) + >> (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit) + << (ic->sb->log2_sectors_per_block + ic->log2_blocks_per_bitmap_bit); + } + /*DEBUG_print("zeroing journal\n");*/ + block_bitmap_op(ic, ic->journal, 0, limit, BITMAP_OP_CLEAR); + block_bitmap_op(ic, ic->may_write_bitmap, 0, limit, BITMAP_OP_CLEAR); + + rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + + spin_lock_irq(&ic->endio_wait.lock); + remove_range_unlocked(ic, &range); + while (unlikely((bio = bio_list_pop(&ic->synchronous_bios)) != NULL)) { + bio_endio(bio); + spin_unlock_irq(&ic->endio_wait.lock); + spin_lock_irq(&ic->endio_wait.lock); + } + spin_unlock_irq(&ic->endio_wait.lock); +} + + +static void init_journal(struct dm_integrity_c *ic, unsigned int start_section, + unsigned int n_sections, unsigned char commit_seq) +{ + unsigned int i, j, n; if (!n_sections) return; @@ -2220,11 +3518,14 @@ static void init_journal(struct dm_integrity_c *ic, unsigned start_section, wraparound_section(ic, &i); for (j = 0; j < ic->journal_section_sectors; j++) { struct journal_sector *js = access_journal(ic, i, j); - memset(&js->entries, 0, JOURNAL_SECTOR_DATA); + + BUILD_BUG_ON(sizeof(js->sectors) != JOURNAL_SECTOR_DATA); + memset(&js->sectors, 0, sizeof(js->sectors)); js->commit_id = dm_integrity_commit_id(ic, i, j, commit_seq); } for (j = 0; j < ic->journal_section_entries; j++) { struct journal_entry *je = access_journal_entry(ic, i, j); + journal_entry_set_unused(je); } } @@ -2232,9 +3533,10 @@ static void init_journal(struct dm_integrity_c *ic, unsigned start_section, write_journal(ic, start_section, n_sections); } -static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, commit_id_t id) +static int find_commit_seq(struct dm_integrity_c *ic, unsigned int i, unsigned int j, commit_id_t id) { unsigned char k; + for (k = 0; k < N_COMMIT_IDS; k++) { if (dm_integrity_commit_id(ic, i, j, k) == id) return k; @@ -2245,11 +3547,11 @@ static int find_commit_seq(struct dm_integrity_c *ic, unsigned i, unsigned j, co static void replay_journal(struct dm_integrity_c *ic) { - unsigned i, j; + unsigned int i, j; bool used_commit_ids[N_COMMIT_IDS]; - unsigned max_commit_id_sections[N_COMMIT_IDS]; - unsigned write_start, write_sections; - unsigned continue_section; + unsigned int max_commit_id_sections[N_COMMIT_IDS]; + unsigned int write_start, write_sections; + unsigned int continue_section; bool journal_empty; unsigned char unused, last_used, want_commit_seq; @@ -2264,11 +3566,12 @@ static void replay_journal(struct dm_integrity_c *ic) if (!ic->just_formatted) { DEBUG_print("reading journal\n"); - rw_journal(ic, REQ_OP_READ, 0, 0, ic->journal_sections, NULL); + rw_journal(ic, REQ_OP_READ, 0, ic->journal_sections, NULL); if (ic->journal_io) DEBUG_bytes(lowmem_page_address(ic->journal_io[0].page), 64, "read journal"); if (ic->journal_io) { struct journal_completion crypt_comp; + crypt_comp.ic = ic; init_completion(&crypt_comp.comp); crypt_comp.in_flight = (atomic_t)ATOMIC_INIT(0); @@ -2282,12 +3585,13 @@ static void replay_journal(struct dm_integrity_c *ic) goto clear_journal; journal_empty = true; - memset(used_commit_ids, 0, sizeof used_commit_ids); - memset(max_commit_id_sections, 0, sizeof max_commit_id_sections); + memset(used_commit_ids, 0, sizeof(used_commit_ids)); + memset(max_commit_id_sections, 0, sizeof(max_commit_id_sections)); for (i = 0; i < ic->journal_sections; i++) { for (j = 0; j < ic->journal_section_sectors; j++) { int k; struct journal_sector *js = access_journal(ic, i, j); + k = find_commit_seq(ic, i, j, js->commit_id); if (k < 0) goto clear_journal; @@ -2297,6 +3601,7 @@ static void replay_journal(struct dm_integrity_c *ic) if (journal_empty) { for (j = 0; j < ic->journal_section_entries; j++) { struct journal_entry *je = access_journal_entry(ic, i, j); + if (!journal_entry_is_unused(je)) { journal_empty = false; break; @@ -2367,8 +3672,9 @@ brk: ic->commit_seq = want_commit_seq; DEBUG_print("continuing from section %u, commit seq %d\n", write_start, ic->commit_seq); } else { - unsigned s; + unsigned int s; unsigned char erase_seq; + clear_journal: DEBUG_print("clearing journal\n"); @@ -2403,28 +3709,74 @@ clear_journal: init_journal_node(&ic->journal_tree[i]); } +static void dm_integrity_enter_synchronous_mode(struct dm_integrity_c *ic) +{ + DEBUG_print("%s\n", __func__); + + if (ic->mode == 'B') { + ic->bitmap_flush_interval = msecs_to_jiffies(10) + 1; + ic->synchronous_mode = 1; + + cancel_delayed_work_sync(&ic->bitmap_flush_work); + queue_delayed_work(ic->commit_wq, &ic->bitmap_flush_work, 0); + flush_workqueue(ic->commit_wq); + } +} + +static int dm_integrity_reboot(struct notifier_block *n, unsigned long code, void *x) +{ + struct dm_integrity_c *ic = container_of(n, struct dm_integrity_c, reboot_notifier); + + DEBUG_print("%s\n", __func__); + + dm_integrity_enter_synchronous_mode(ic); + + return NOTIFY_DONE; +} + static void dm_integrity_postsuspend(struct dm_target *ti) { - struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; + struct dm_integrity_c *ic = ti->private; + int r; - del_timer_sync(&ic->autocommit_timer); + WARN_ON(unregister_reboot_notifier(&ic->reboot_notifier)); - WRITE_ONCE(ic->suspending, 1); + timer_delete_sync(&ic->autocommit_timer); if (ic->recalc_wq) drain_workqueue(ic->recalc_wq); + if (ic->mode == 'B') + cancel_delayed_work_sync(&ic->bitmap_flush_work); + queue_work(ic->commit_wq, &ic->commit_work); drain_workqueue(ic->commit_wq); if (ic->mode == 'J') { - if (ic->meta_dev) - queue_work(ic->writer_wq, &ic->writer_work); + queue_work(ic->writer_wq, &ic->writer_work); drain_workqueue(ic->writer_wq); - dm_integrity_flush_buffers(ic); + dm_integrity_flush_buffers(ic, true); + if (ic->wrote_to_journal) { + init_journal(ic, ic->free_section, + ic->journal_sections - ic->free_section, ic->commit_seq); + if (ic->free_section) { + init_journal(ic, 0, ic->free_section, + next_commit_seq(ic->commit_seq)); + } + } } - WRITE_ONCE(ic->suspending, 0); + if (ic->mode == 'B') { + dm_integrity_flush_buffers(ic, true); +#if 1 + /* set to 0 to test bitmap replay code */ + init_journal(ic, 0, ic->journal_sections, 0); + ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); +#endif + } BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); @@ -2433,12 +3785,107 @@ static void dm_integrity_postsuspend(struct dm_target *ti) static void dm_integrity_resume(struct dm_target *ti) { - struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; + struct dm_integrity_c *ic = ti->private; + __u64 old_provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); + int r; + + DEBUG_print("resume\n"); + + ic->wrote_to_journal = false; + + if (ic->provided_data_sectors != old_provided_data_sectors) { + if (ic->provided_data_sectors > old_provided_data_sectors && + ic->mode == 'B' && + ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit) { + rw_journal_sectors(ic, REQ_OP_READ, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + block_bitmap_op(ic, ic->journal, old_provided_data_sectors, + ic->provided_data_sectors - old_provided_data_sectors, BITMAP_OP_SET); + rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + } - replay_journal(ic); + ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + } - if (ic->recalc_wq && ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { + if (ic->sb->flags & cpu_to_le32(SB_FLAG_DIRTY_BITMAP)) { + DEBUG_print("resume dirty_bitmap\n"); + rw_journal_sectors(ic, REQ_OP_READ, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + if (ic->mode == 'B') { + if (ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && + !ic->reset_recalculate_flag) { + block_bitmap_copy(ic, ic->recalc_bitmap, ic->journal); + block_bitmap_copy(ic, ic->may_write_bitmap, ic->journal); + if (!block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, + BITMAP_OP_TEST_ALL_CLEAR)) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } + } else { + DEBUG_print("non-matching blocks_per_bitmap_bit: %u, %u\n", + ic->sb->log2_blocks_per_bitmap_bit, ic->log2_blocks_per_bitmap_bit); + ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; + block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); + block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_SET); + block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_SET); + rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } + } else { + if (!(ic->sb->log2_blocks_per_bitmap_bit == ic->log2_blocks_per_bitmap_bit && + block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_TEST_ALL_CLEAR)) || + ic->reset_recalculate_flag) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } + init_journal(ic, 0, ic->journal_sections, 0); + replay_journal(ic); + ic->sb->flags &= ~cpu_to_le32(SB_FLAG_DIRTY_BITMAP); + } + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + } else { + replay_journal(ic); + if (ic->reset_recalculate_flag) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } + if (ic->mode == 'B') { + ic->sb->flags |= cpu_to_le32(SB_FLAG_DIRTY_BITMAP); + ic->sb->log2_blocks_per_bitmap_bit = ic->log2_blocks_per_bitmap_bit; + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); + + block_bitmap_op(ic, ic->journal, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); + block_bitmap_op(ic, ic->recalc_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); + block_bitmap_op(ic, ic->may_write_bitmap, 0, ic->provided_data_sectors, BITMAP_OP_CLEAR); + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors) { + block_bitmap_op(ic, ic->journal, le64_to_cpu(ic->sb->recalc_sector), + ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); + block_bitmap_op(ic, ic->recalc_bitmap, le64_to_cpu(ic->sb->recalc_sector), + ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); + block_bitmap_op(ic, ic->may_write_bitmap, le64_to_cpu(ic->sb->recalc_sector), + ic->provided_data_sectors - le64_to_cpu(ic->sb->recalc_sector), BITMAP_OP_SET); + } + rw_journal_sectors(ic, REQ_OP_WRITE | REQ_FUA | REQ_SYNC, 0, + ic->n_bitmap_blocks * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT), NULL); + } + } + + DEBUG_print("testing recalc: %x\n", ic->sb->flags); + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); + + DEBUG_print("recalc pos: %llx / %llx\n", recalc_pos, ic->provided_data_sectors); if (recalc_pos < ic->provided_data_sectors) { queue_work(ic->recalc_wq, &ic->recalc_work); } else if (recalc_pos > ic->provided_data_sectors) { @@ -2446,38 +3893,56 @@ static void dm_integrity_resume(struct dm_target *ti) recalc_write_super(ic); } } + + ic->reboot_notifier.notifier_call = dm_integrity_reboot; + ic->reboot_notifier.next = NULL; + ic->reboot_notifier.priority = INT_MAX - 1; /* be notified after md and before hardware drivers */ + WARN_ON(register_reboot_notifier(&ic->reboot_notifier)); + +#if 0 + /* set to 1 to stress test synchronous mode */ + dm_integrity_enter_synchronous_mode(ic); +#endif } static void dm_integrity_status(struct dm_target *ti, status_type_t type, - unsigned status_flags, char *result, unsigned maxlen) + unsigned int status_flags, char *result, unsigned int maxlen) { - struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; - unsigned arg_count; + struct dm_integrity_c *ic = ti->private; + unsigned int arg_count; size_t sz = 0; switch (type) { case STATUSTYPE_INFO: DMEMIT("%llu %llu", (unsigned long long)atomic64_read(&ic->number_of_mismatches), - (unsigned long long)ic->provided_data_sectors); + ic->provided_data_sectors); if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) - DMEMIT(" %llu", (unsigned long long)le64_to_cpu(ic->sb->recalc_sector)); + DMEMIT(" %llu", le64_to_cpu(ic->sb->recalc_sector)); else DMEMIT(" -"); break; case STATUSTYPE_TABLE: { - __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; - watermark_percentage += ic->journal_entries / 2; - do_div(watermark_percentage, ic->journal_entries); - arg_count = 5; + arg_count = 1; /* buffer_sectors */ arg_count += !!ic->meta_dev; arg_count += ic->sectors_per_block != 1; arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); + arg_count += ic->reset_recalculate_flag; + arg_count += ic->discard; + arg_count += ic->mode != 'I'; /* interleave_sectors */ + arg_count += ic->mode == 'J'; /* journal_sectors */ + arg_count += ic->mode == 'J'; /* journal_watermark */ + arg_count += ic->mode == 'J'; /* commit_time */ + arg_count += ic->mode == 'B'; /* sectors_per_bit */ + arg_count += ic->mode == 'B'; /* bitmap_flush_interval */ arg_count += !!ic->internal_hash_alg.alg_string; arg_count += !!ic->journal_crypt_alg.alg_string; arg_count += !!ic->journal_mac_alg.alg_string; - DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start, + arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0; + arg_count += (ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0; + arg_count += ic->legacy_recalculate; + DMEMIT("%s %llu %u %c %u", ic->dev->name, ic->start, ic->tag_size, ic->mode, arg_count); if (ic->meta_dev) DMEMIT(" meta_device:%s", ic->meta_dev->name); @@ -2485,11 +3950,32 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) DMEMIT(" recalculate"); - DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); - DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); + if (ic->reset_recalculate_flag) + DMEMIT(" reset_recalculate"); + if (ic->discard) + DMEMIT(" allow_discards"); + if (ic->mode != 'I') + DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); - DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage); - DMEMIT(" commit_time:%u", ic->autocommit_msec); + if (ic->mode == 'J') { + __u64 watermark_percentage = (__u64)(ic->journal_entries - ic->free_sectors_threshold) * 100; + + watermark_percentage += ic->journal_entries / 2; + do_div(watermark_percentage, ic->journal_entries); + DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); + DMEMIT(" journal_watermark:%u", (unsigned int)watermark_percentage); + DMEMIT(" commit_time:%u", ic->autocommit_msec); + } + if (ic->mode == 'B') { + DMEMIT(" sectors_per_bit:%llu", (sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit); + DMEMIT(" bitmap_flush_interval:%u", jiffies_to_msecs(ic->bitmap_flush_interval)); + } + if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) + DMEMIT(" fix_padding"); + if ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) + DMEMIT(" fix_hmac"); + if (ic->legacy_recalculate) + DMEMIT(" legacy_recalculate"); #define EMIT_ALG(a, n) \ do { \ @@ -2504,6 +3990,30 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, EMIT_ALG(journal_mac_alg, "journal_mac"); break; } + case STATUSTYPE_IMA: + DMEMIT_TARGET_NAME_VERSION(ti->type); + DMEMIT(",dev_name=%s,start=%llu,tag_size=%u,mode=%c", + ic->dev->name, ic->start, ic->tag_size, ic->mode); + + if (ic->meta_dev) + DMEMIT(",meta_device=%s", ic->meta_dev->name); + if (ic->sectors_per_block != 1) + DMEMIT(",block_size=%u", ic->sectors_per_block << SECTOR_SHIFT); + + DMEMIT(",recalculate=%c", (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) ? + 'y' : 'n'); + DMEMIT(",allow_discards=%c", ic->discard ? 'y' : 'n'); + DMEMIT(",fix_padding=%c", + ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING)) != 0) ? 'y' : 'n'); + DMEMIT(",fix_hmac=%c", + ((ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_HMAC)) != 0) ? 'y' : 'n'); + DMEMIT(",legacy_recalculate=%c", ic->legacy_recalculate ? 'y' : 'n'); + + DMEMIT(",journal_sectors=%u", ic->initial_sectors - SB_SECTORS); + DMEMIT(",interleave_sectors=%u", 1U << ic->sb->log2_interleave_sectors); + DMEMIT(",buffer_sectors=%u", 1U << ic->log2_buffer_sectors); + DMEMIT(";"); + break; } } @@ -2525,13 +4035,27 @@ static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *lim if (ic->sectors_per_block > 1) { limits->logical_block_size = ic->sectors_per_block << SECTOR_SHIFT; limits->physical_block_size = ic->sectors_per_block << SECTOR_SHIFT; - blk_limits_io_min(limits, ic->sectors_per_block << SECTOR_SHIFT); + limits->io_min = ic->sectors_per_block << SECTOR_SHIFT; + limits->dma_alignment = limits->logical_block_size - 1; + limits->discard_granularity = ic->sectors_per_block << SECTOR_SHIFT; + } + + if (!ic->internal_hash) { + struct blk_integrity *bi = &limits->integrity; + + memset(bi, 0, sizeof(*bi)); + bi->metadata_size = ic->tag_size; + bi->tag_size = bi->metadata_size; + bi->interval_exp = + ic->sb->log2_sectors_per_block + SECTOR_SHIFT; } + + limits->max_integrity_segments = USHRT_MAX; } static void calculate_journal_section_size(struct dm_integrity_c *ic) { - unsigned sector_space = JOURNAL_SECTOR_DATA; + unsigned int sector_space = JOURNAL_SECTOR_DATA; ic->journal_sections = le32_to_cpu(ic->sb->journal_sections); ic->journal_entry_size = roundup(offsetof(struct journal_entry, last_bytes[ic->sectors_per_block]) + ic->tag_size, @@ -2555,11 +4079,20 @@ static int calculate_device_limits(struct dm_integrity_c *ic) return -EINVAL; ic->initial_sectors = initial_sectors; - if (!ic->meta_dev) { + if (ic->mode == 'I') { + if (ic->initial_sectors + ic->provided_data_sectors > ic->meta_device_sectors) + return -EINVAL; + } else if (!ic->meta_dev) { sector_t last_sector, last_area, last_offset; - ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), - (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT; + /* we have to maintain excessive padding for compatibility with existing volumes */ + __u64 metadata_run_padding = + ic->sb->flags & cpu_to_le32(SB_FLAG_FIXED_PADDING) ? + (__u64)(METADATA_PADDING_SECTORS << SECTOR_SHIFT) : + (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS); + + ic->metadata_run = round_up((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), + metadata_run_padding) >> SECTOR_SHIFT; if (!(ic->metadata_run & (ic->metadata_run - 1))) ic->log2_metadata_run = __ffs(ic->metadata_run); else @@ -2570,7 +4103,8 @@ static int calculate_device_limits(struct dm_integrity_c *ic) if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) return -EINVAL; } else { - __u64 meta_size = ic->provided_data_sectors * ic->tag_size; + __u64 meta_size = (ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) * ic->tag_size; + meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) >> (ic->log2_buffer_sectors + SECTOR_SHIFT); meta_size <<= ic->log2_buffer_sectors; @@ -2584,13 +4118,35 @@ static int calculate_device_limits(struct dm_integrity_c *ic) return 0; } -static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sectors, unsigned interleave_sectors) +static void get_provided_data_sectors(struct dm_integrity_c *ic) +{ + if (!ic->meta_dev) { + int test_bit; + + ic->provided_data_sectors = 0; + for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { + __u64 prev_data_sectors = ic->provided_data_sectors; + + ic->provided_data_sectors |= (sector_t)1 << test_bit; + if (calculate_device_limits(ic)) + ic->provided_data_sectors = prev_data_sectors; + } + } else { + ic->provided_data_sectors = ic->data_device_sectors; + ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); + } +} + +static int initialize_superblock(struct dm_integrity_c *ic, + unsigned int journal_sectors, unsigned int interleave_sectors) { - unsigned journal_sections; + unsigned int journal_sections; int test_bit; memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); memcpy(ic->sb->magic, SB_MAGIC, 8); + if (ic->mode == 'I') + ic->sb->flags |= cpu_to_le32(SB_FLAG_INLINE); ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); if (ic->journal_mac_alg.alg_string) @@ -2600,35 +4156,40 @@ static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sec journal_sections = journal_sectors / ic->journal_section_sectors; if (!journal_sections) journal_sections = 1; + if (ic->mode == 'I') + journal_sections = 0; + + if (ic->fix_hmac && (ic->internal_hash_alg.alg_string || ic->journal_mac_alg.alg_string)) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_HMAC); + get_random_bytes(ic->sb->salt, SALT_SIZE); + } if (!ic->meta_dev) { + if (ic->fix_padding) + ic->sb->flags |= cpu_to_le32(SB_FLAG_FIXED_PADDING); ic->sb->journal_sections = cpu_to_le32(journal_sections); if (!interleave_sectors) interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; ic->sb->log2_interleave_sectors = __fls(interleave_sectors); - ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); - ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); - - ic->provided_data_sectors = 0; - for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { - __u64 prev_data_sectors = ic->provided_data_sectors; + ic->sb->log2_interleave_sectors = max_t(__u8, MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); + ic->sb->log2_interleave_sectors = min_t(__u8, MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); - ic->provided_data_sectors |= (sector_t)1 << test_bit; - if (calculate_device_limits(ic)) - ic->provided_data_sectors = prev_data_sectors; - } + get_provided_data_sectors(ic); if (!ic->provided_data_sectors) return -EINVAL; } else { ic->sb->log2_interleave_sectors = 0; - ic->provided_data_sectors = ic->data_device_sectors; - ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); + + get_provided_data_sectors(ic); + if (!ic->provided_data_sectors) + return -EINVAL; try_smaller_buffer: ic->sb->journal_sections = cpu_to_le32(0); for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) { __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); __u32 test_journal_sections = prev_journal_sections | (1U << test_bit); + if (test_journal_sections > journal_sections) continue; ic->sb->journal_sections = cpu_to_le32(test_journal_sections); @@ -2652,68 +4213,55 @@ try_smaller_buffer: return 0; } -static void dm_integrity_set(struct dm_target *ti, struct dm_integrity_c *ic) +static void dm_integrity_free_page_list(struct page_list *pl) { - struct gendisk *disk = dm_disk(dm_table_get_md(ti->table)); - struct blk_integrity bi; - - memset(&bi, 0, sizeof(bi)); - bi.profile = &dm_integrity_profile; - bi.tuple_size = ic->tag_size; - bi.tag_size = bi.tuple_size; - bi.interval_exp = ic->sb->log2_sectors_per_block + SECTOR_SHIFT; - - blk_integrity_register(disk, &bi); - blk_queue_max_integrity_segments(disk->queue, UINT_MAX); -} - -static void dm_integrity_free_page_list(struct dm_integrity_c *ic, struct page_list *pl) -{ - unsigned i; + unsigned int i; if (!pl) return; - for (i = 0; i < ic->journal_pages; i++) - if (pl[i].page) - __free_page(pl[i].page); + for (i = 0; pl[i].page; i++) + __free_page(pl[i].page); kvfree(pl); } -static struct page_list *dm_integrity_alloc_page_list(struct dm_integrity_c *ic) +static struct page_list *dm_integrity_alloc_page_list(unsigned int n_pages) { - size_t page_list_desc_size = ic->journal_pages * sizeof(struct page_list); struct page_list *pl; - unsigned i; + unsigned int i; - pl = kvmalloc(page_list_desc_size, GFP_KERNEL | __GFP_ZERO); + pl = kvmalloc_array(n_pages + 1, sizeof(struct page_list), GFP_KERNEL | __GFP_ZERO); if (!pl) return NULL; - for (i = 0; i < ic->journal_pages; i++) { + for (i = 0; i < n_pages; i++) { pl[i].page = alloc_page(GFP_KERNEL); if (!pl[i].page) { - dm_integrity_free_page_list(ic, pl); + dm_integrity_free_page_list(pl); return NULL; } if (i) pl[i - 1].next = &pl[i]; } + pl[i].page = NULL; + pl[i].next = NULL; return pl; } static void dm_integrity_free_journal_scatterlist(struct dm_integrity_c *ic, struct scatterlist **sl) { - unsigned i; + unsigned int i; + for (i = 0; i < ic->journal_sections; i++) kvfree(sl[i]); kvfree(sl); } -static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, struct page_list *pl) +static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_integrity_c *ic, + struct page_list *pl) { struct scatterlist **sl; - unsigned i; + unsigned int i; sl = kvmalloc_array(ic->journal_sections, sizeof(struct scatterlist *), @@ -2723,13 +4271,14 @@ static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_int for (i = 0; i < ic->journal_sections; i++) { struct scatterlist *s; - unsigned start_index, start_offset; - unsigned end_index, end_offset; - unsigned n_pages; - unsigned idx; + unsigned int start_index, start_offset; + unsigned int end_index, end_offset; + unsigned int n_pages; + unsigned int idx; page_list_location(ic, i, 0, &start_index, &start_offset); - page_list_location(ic, i, ic->journal_section_sectors - 1, &end_index, &end_offset); + page_list_location(ic, i, ic->journal_section_sectors - 1, + &end_index, &end_offset); n_pages = (end_index - start_index + 1); @@ -2743,7 +4292,8 @@ static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_int sg_init_table(s, n_pages); for (idx = start_index; idx <= end_index; idx++) { char *va = lowmem_page_address(pl[idx].page); - unsigned start = 0, end = PAGE_SIZE; + unsigned int start = 0, end = PAGE_SIZE; + if (idx == start_index) start = start_offset; if (idx == end_index) @@ -2759,9 +4309,9 @@ static struct scatterlist **dm_integrity_alloc_journal_scatterlist(struct dm_int static void free_alg(struct alg_spec *a) { - kzfree(a->alg_string); - kzfree(a->key); - memset(a, 0, sizeof *a); + kfree_sensitive(a->alg_string); + kfree_sensitive(a->key); + memset(a, 0, sizeof(*a)); } static int get_alg_and_key(const char *arg, struct alg_spec *a, char **error, char *error_inval) @@ -2798,30 +4348,53 @@ nomem: return -ENOMEM; } -static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, - char *error_alg, char *error_key) +static int get_mac(struct crypto_shash **shash, struct crypto_ahash **ahash, + struct alg_spec *a, char **error, char *error_alg, char *error_key) { int r; if (a->alg_string) { - *hash = crypto_alloc_shash(a->alg_string, 0, 0); - if (IS_ERR(*hash)) { - *error = error_alg; - r = PTR_ERR(*hash); - *hash = NULL; - return r; - } - - if (a->key) { - r = crypto_shash_setkey(*hash, a->key, a->key_size); - if (r) { + if (shash) { + *shash = crypto_alloc_shash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); + if (IS_ERR(*shash)) { + *shash = NULL; + goto try_ahash; + } + if (a->key) { + r = crypto_shash_setkey(*shash, a->key, a->key_size); + if (r) { + *error = error_key; + return r; + } + } else if (crypto_shash_get_flags(*shash) & CRYPTO_TFM_NEED_KEY) { *error = error_key; + return -ENOKEY; + } + return 0; + } +try_ahash: + if (ahash) { + *ahash = crypto_alloc_ahash(a->alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); + if (IS_ERR(*ahash)) { + *error = error_alg; + r = PTR_ERR(*ahash); + *ahash = NULL; return r; } - } else if (crypto_shash_get_flags(*hash) & CRYPTO_TFM_NEED_KEY) { - *error = error_key; - return -ENOKEY; + if (a->key) { + r = crypto_ahash_setkey(*ahash, a->key, a->key_size); + if (r) { + *error = error_key; + return r; + } + } else if (crypto_ahash_get_flags(*ahash) & CRYPTO_TFM_NEED_KEY) { + *error = error_key; + return -ENOKEY; + } + return 0; } + *error = error_alg; + return -ENOENT; } return 0; @@ -2830,7 +4403,7 @@ static int get_mac(struct crypto_shash **hash, struct alg_spec *a, char **error, static int create_journal(struct dm_integrity_c *ic, char **error) { int r = 0; - unsigned i; + unsigned int i; __u64 journal_pages, journal_desc_size, journal_tree_size; unsigned char *crypt_data = NULL, *crypt_iv = NULL; struct skcipher_request *req = NULL; @@ -2850,18 +4423,18 @@ static int create_journal(struct dm_integrity_c *ic, char **error) } ic->journal_pages = journal_pages; - ic->journal = dm_integrity_alloc_page_list(ic); + ic->journal = dm_integrity_alloc_page_list(ic->journal_pages); if (!ic->journal) { *error = "Could not allocate memory for journal"; r = -ENOMEM; goto bad; } if (ic->journal_crypt_alg.alg_string) { - unsigned ivsize, blocksize; + unsigned int ivsize, blocksize; struct journal_completion comp; comp.ic = ic; - ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, 0); + ic->journal_crypt = crypto_alloc_skcipher(ic->journal_crypt_alg.alg_string, 0, CRYPTO_ALG_ALLOCATES_MEMORY); if (IS_ERR(ic->journal_crypt)) { *error = "Invalid journal cipher"; r = PTR_ERR(ic->journal_crypt); @@ -2882,7 +4455,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error) DEBUG_print("cipher %s, block size %u iv size %u\n", ic->journal_crypt_alg.alg_string, blocksize, ivsize); - ic->journal_io = dm_integrity_alloc_page_list(ic); + ic->journal_io = dm_integrity_alloc_page_list(ic->journal_pages); if (!ic->journal_io) { *error = "Could not allocate memory for journal io"; r = -ENOMEM; @@ -2899,14 +4472,14 @@ static int create_journal(struct dm_integrity_c *ic, char **error) goto bad; } - crypt_iv = kmalloc(ivsize, GFP_KERNEL); + crypt_iv = kzalloc(ivsize, GFP_KERNEL); if (!crypt_iv) { *error = "Could not allocate iv"; r = -ENOMEM; goto bad; } - ic->journal_xor = dm_integrity_alloc_page_list(ic); + ic->journal_xor = dm_integrity_alloc_page_list(ic->journal_pages); if (!ic->journal_xor) { *error = "Could not allocate memory for journal xor"; r = -ENOMEM; @@ -2924,13 +4497,14 @@ static int create_journal(struct dm_integrity_c *ic, char **error) sg_init_table(sg, ic->journal_pages + 1); for (i = 0; i < ic->journal_pages; i++) { char *va = lowmem_page_address(ic->journal_xor[i].page); + clear_page(va); sg_set_buf(&sg[i], va, PAGE_SIZE); } - sg_set_buf(&sg[i], &ic->commit_ids, sizeof ic->commit_ids); - memset(crypt_iv, 0x00, ivsize); + sg_set_buf(&sg[i], &ic->commit_ids, sizeof(ic->commit_ids)); - skcipher_request_set_crypt(req, sg, sg, PAGE_SIZE * ic->journal_pages + sizeof ic->commit_ids, crypt_iv); + skcipher_request_set_crypt(req, sg, sg, + PAGE_SIZE * ic->journal_pages + sizeof(ic->commit_ids), crypt_iv); init_completion(&comp.comp); comp.in_flight = (atomic_t)ATOMIC_INIT(1); if (do_crypt(true, req, &comp)) @@ -2946,7 +4520,7 @@ static int create_journal(struct dm_integrity_c *ic, char **error) crypto_free_skcipher(ic->journal_crypt); ic->journal_crypt = NULL; } else { - unsigned crypt_len = roundup(ivsize, blocksize); + unsigned int crypt_len = roundup(ivsize, blocksize); req = skcipher_request_alloc(ic->journal_crypt, GFP_KERNEL); if (!req) { @@ -2992,11 +4566,11 @@ static int create_journal(struct dm_integrity_c *ic, char **error) for (i = 0; i < ic->journal_sections; i++) { struct scatterlist sg; struct skcipher_request *section_req; - __u32 section_le = cpu_to_le32(i); + __le32 section_le = cpu_to_le32(i); memset(crypt_iv, 0x00, ivsize); memset(crypt_data, 0x00, crypt_len); - memcpy(crypt_data, §ion_le, min((size_t)crypt_len, sizeof(section_le))); + memcpy(crypt_data, §ion_le, min_t(size_t, crypt_len, sizeof(section_le))); sg_init_one(&sg, crypt_data, crypt_len); skcipher_request_set_crypt(req, &sg, &sg, crypt_len, crypt_iv); @@ -3034,7 +4608,8 @@ static int create_journal(struct dm_integrity_c *ic, char **error) } for (i = 0; i < N_COMMIT_IDS; i++) { - unsigned j; + unsigned int j; + retest_commit_id: for (j = 0; j < i; j++) { if (ic->commit_ids[j] == ic->commit_ids[i]) { @@ -3071,7 +4646,7 @@ bad: * device * offset from the start of the device * tag size - * D - direct writes, J - journal writes, R - recovery mode + * D - direct writes, J - journal writes, B - bitmap mode, R - recovery mode * number of optional arguments * optional arguments: * journal_sectors @@ -3079,26 +4654,33 @@ bad: * buffer_sectors * journal_watermark * commit_time + * meta_device + * block_size + * sectors_per_bit + * bitmap_flush_interval * internal_hash * journal_crypt * journal_mac - * block_size + * recalculate */ -static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) +static int dm_integrity_ctr(struct dm_target *ti, unsigned int argc, char **argv) { struct dm_integrity_c *ic; char dummy; int r; - unsigned extra_args; + unsigned int extra_args; struct dm_arg_set as; static const struct dm_arg _args[] = { - {0, 9, "Invalid number of feature args"}, + {0, 18, "Invalid number of feature args"}, }; - unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; - bool recalculate; + unsigned int journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; bool should_write_sb; __u64 threshold; unsigned long long start; + __s8 log2_sectors_per_bitmap_bit = -1; + __s8 log2_blocks_per_bitmap_bit; + __u64 bits_in_journal; + __u64 n_bitmap_bits; #define DIRECT_ARGUMENTS 4 @@ -3114,6 +4696,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } ti->private = ic; ti->per_io_data_size = sizeof(struct dm_integrity_io); + ic->ti = ti; ic->in_progress = RB_ROOT; INIT_LIST_HEAD(&ic->wait_list); @@ -3122,6 +4705,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) init_waitqueue_head(&ic->copy_to_journal_wait); init_completion(&ic->crypto_backoff); atomic64_set(&ic->number_of_mismatches, 0); + ic->bitmap_flush_interval = BITMAP_FLUSH_INTERVAL; r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ic->dev); if (r) { @@ -3144,10 +4728,12 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } } - if (!strcmp(argv[3], "J") || !strcmp(argv[3], "D") || !strcmp(argv[3], "R")) + if (!strcmp(argv[3], "J") || !strcmp(argv[3], "B") || + !strcmp(argv[3], "D") || !strcmp(argv[3], "R") || + !strcmp(argv[3], "I")) { ic->mode = argv[3][0]; - else { - ti->error = "Invalid mode (expecting J, D, R)"; + } else { + ti->error = "Invalid mode (expecting J, B, D, R, I)"; r = -EINVAL; goto bad; } @@ -3157,7 +4743,6 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) buffer_sectors = DEFAULT_BUFFER_SECTORS; journal_watermark = DEFAULT_JOURNAL_WATERMARK; sync_msec = DEFAULT_SYNC_MSEC; - recalculate = false; ic->sectors_per_block = 1; as.argc = argc - DIRECT_ARGUMENTS; @@ -3168,7 +4753,9 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) while (extra_args--) { const char *opt_string; - unsigned val; + unsigned int val; + unsigned long long llval; + opt_string = dm_shift_arg(&as); if (!opt_string) { r = -EINVAL; @@ -3185,12 +4772,13 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) journal_watermark = val; else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1) sync_msec = val; - else if (!memcmp(opt_string, "meta_device:", strlen("meta_device:"))) { + else if (!strncmp(opt_string, "meta_device:", strlen("meta_device:"))) { if (ic->meta_dev) { dm_put_device(ti, ic->meta_dev); ic->meta_dev = NULL; } - r = dm_get_device(ti, strchr(opt_string, ':') + 1, dm_table_get_mode(ti->table), &ic->meta_dev); + r = dm_get_device(ti, strchr(opt_string, ':') + 1, + dm_table_get_mode(ti->table), &ic->meta_dev); if (r) { ti->error = "Device lookup failed"; goto bad; @@ -3198,29 +4786,49 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) { if (val < 1 << SECTOR_SHIFT || val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT || - (val & (val -1))) { + (val & (val - 1))) { r = -EINVAL; ti->error = "Invalid block_size argument"; goto bad; } ic->sectors_per_block = val >> SECTOR_SHIFT; - } else if (!memcmp(opt_string, "internal_hash:", strlen("internal_hash:"))) { + } else if (sscanf(opt_string, "sectors_per_bit:%llu%c", &llval, &dummy) == 1) { + log2_sectors_per_bitmap_bit = !llval ? 0 : __ilog2_u64(llval); + } else if (sscanf(opt_string, "bitmap_flush_interval:%u%c", &val, &dummy) == 1) { + if ((uint64_t)val >= (uint64_t)UINT_MAX * 1000 / HZ) { + r = -EINVAL; + ti->error = "Invalid bitmap_flush_interval argument"; + goto bad; + } + ic->bitmap_flush_interval = msecs_to_jiffies(val); + } else if (!strncmp(opt_string, "internal_hash:", strlen("internal_hash:"))) { r = get_alg_and_key(opt_string, &ic->internal_hash_alg, &ti->error, "Invalid internal_hash argument"); if (r) goto bad; - } else if (!memcmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) { + } else if (!strncmp(opt_string, "journal_crypt:", strlen("journal_crypt:"))) { r = get_alg_and_key(opt_string, &ic->journal_crypt_alg, &ti->error, "Invalid journal_crypt argument"); if (r) goto bad; - } else if (!memcmp(opt_string, "journal_mac:", strlen("journal_mac:"))) { - r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, + } else if (!strncmp(opt_string, "journal_mac:", strlen("journal_mac:"))) { + r = get_alg_and_key(opt_string, &ic->journal_mac_alg, &ti->error, "Invalid journal_mac argument"); if (r) goto bad; } else if (!strcmp(opt_string, "recalculate")) { - recalculate = true; + ic->recalculate_flag = true; + } else if (!strcmp(opt_string, "reset_recalculate")) { + ic->recalculate_flag = true; + ic->reset_recalculate_flag = true; + } else if (!strcmp(opt_string, "allow_discards")) { + ic->discard = true; + } else if (!strcmp(opt_string, "fix_padding")) { + ic->fix_padding = true; + } else if (!strcmp(opt_string, "fix_hmac")) { + ic->fix_hmac = true; + } else if (!strcmp(opt_string, "legacy_recalculate")) { + ic->legacy_recalculate = true; } else { r = -EINVAL; ti->error = "Invalid argument"; @@ -3228,27 +4836,41 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } } - ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; + ic->data_device_sectors = bdev_nr_sectors(ic->dev->bdev); if (!ic->meta_dev) ic->meta_device_sectors = ic->data_device_sectors; else - ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT; + ic->meta_device_sectors = bdev_nr_sectors(ic->meta_dev->bdev); if (!journal_sectors) { journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, - ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); + ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); } if (!buffer_sectors) buffer_sectors = 1; ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); - r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, + r = get_mac(&ic->internal_shash, &ic->internal_ahash, &ic->internal_hash_alg, &ti->error, "Invalid internal hash", "Error setting internal hash key"); if (r) goto bad; + if (ic->internal_shash) { + ic->internal_hash = true; + ic->internal_hash_digestsize = crypto_shash_digestsize(ic->internal_shash); + } + if (ic->internal_ahash) { + ic->internal_hash = true; + ic->internal_hash_digestsize = crypto_ahash_digestsize(ic->internal_ahash); + r = mempool_init_kmalloc_pool(&ic->ahash_req_pool, AHASH_MEMPOOL, + sizeof(struct ahash_request) + crypto_ahash_reqsize(ic->internal_ahash)); + if (r) { + ti->error = "Cannot allocate mempool"; + goto bad; + } + } - r = get_mac(&ic->journal_mac, &ic->journal_mac_alg, &ti->error, + r = get_mac(&ic->journal_mac, NULL, &ic->journal_mac_alg, &ti->error, "Invalid journal mac", "Error setting journal mac key"); if (r) goto bad; @@ -3259,7 +4881,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) r = -EINVAL; goto bad; } - ic->tag_size = crypto_shash_digestsize(ic->internal_hash); + ic->tag_size = ic->internal_hash_digestsize; } if (ic->tag_size > MAX_TAG_SIZE) { ti->error = "Too big tag size"; @@ -3271,6 +4893,65 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) else ic->log2_tag_size = -1; + if (ic->mode == 'I') { + struct blk_integrity *bi; + if (ic->meta_dev) { + r = -EINVAL; + ti->error = "Metadata device not supported in inline mode"; + goto bad; + } + if (!ic->internal_hash_alg.alg_string) { + r = -EINVAL; + ti->error = "Internal hash not set in inline mode"; + goto bad; + } + if (ic->journal_crypt_alg.alg_string || ic->journal_mac_alg.alg_string) { + r = -EINVAL; + ti->error = "Journal crypt not supported in inline mode"; + goto bad; + } + if (ic->discard) { + r = -EINVAL; + ti->error = "Discards not supported in inline mode"; + goto bad; + } + bi = blk_get_integrity(ic->dev->bdev->bd_disk); + if (!bi || bi->csum_type != BLK_INTEGRITY_CSUM_NONE) { + r = -EINVAL; + ti->error = "Integrity profile not supported"; + goto bad; + } + /*printk("tag_size: %u, metadata_size: %u\n", bi->tag_size, bi->metadata_size);*/ + if (bi->metadata_size < ic->tag_size) { + r = -EINVAL; + ti->error = "The integrity profile is smaller than tag size"; + goto bad; + } + if ((unsigned long)bi->metadata_size > PAGE_SIZE / 2) { + r = -EINVAL; + ti->error = "Too big tuple size"; + goto bad; + } + ic->tuple_size = bi->metadata_size; + if (1 << bi->interval_exp != ic->sectors_per_block << SECTOR_SHIFT) { + r = -EINVAL; + ti->error = "Integrity profile sector size mismatch"; + goto bad; + } + } + + if (ic->mode == 'B' && !ic->internal_hash) { + r = -EINVAL; + ti->error = "Bitmap mode can be only used with internal hash"; + goto bad; + } + + if (ic->discard && !ic->internal_hash) { + r = -EINVAL; + ti->error = "Discard can be only used with internal hash"; + goto bad; + } + ic->autocommit_jiffies = msecs_to_jiffies(sync_msec); ic->autocommit_msec = sync_msec; timer_setup(&ic->autocommit_timer, autocommit_fn, 0); @@ -3289,6 +4970,25 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } + r = mempool_init_page_pool(&ic->recheck_pool, 1, ic->mode == 'I' ? 1 : 0); + if (r) { + ti->error = "Cannot allocate mempool"; + goto bad; + } + + if (ic->mode == 'I') { + r = bioset_init(&ic->recheck_bios, RECHECK_POOL_SIZE, 0, BIOSET_NEED_BVECS); + if (r) { + ti->error = "Cannot allocate bio set"; + goto bad; + } + r = bioset_init(&ic->recalc_bios, 1, 0, BIOSET_NEED_BVECS); + if (r) { + ti->error = "Cannot allocate bio set"; + goto bad; + } + } + ic->metadata_wq = alloc_workqueue("dm-integrity-metadata", WQ_MEM_RECLAIM, METADATA_WORKQUEUE_MAX_ACTIVE); if (!ic->metadata_wq) { @@ -3298,16 +4998,24 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } /* - * If this workqueue were percpu, it would cause bio reordering + * If this workqueue weren't ordered, it would cause bio reordering * and reduced performance. */ - ic->wait_wq = alloc_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM | WQ_UNBOUND, 1); + ic->wait_wq = alloc_ordered_workqueue("dm-integrity-wait", WQ_MEM_RECLAIM); if (!ic->wait_wq) { ti->error = "Cannot allocate workqueue"; r = -ENOMEM; goto bad; } + ic->offload_wq = alloc_workqueue("dm-integrity-offload", WQ_MEM_RECLAIM, + METADATA_WORKQUEUE_MAX_ACTIVE); + if (!ic->offload_wq) { + ti->error = "Cannot allocate workqueue"; + r = -ENOMEM; + goto bad; + } + ic->commit_wq = alloc_workqueue("dm-integrity-commit", WQ_MEM_RECLAIM, 1); if (!ic->commit_wq) { ti->error = "Cannot allocate workqueue"; @@ -3316,7 +5024,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } INIT_WORK(&ic->commit_work, integrity_commit); - if (ic->mode == 'J') { + if (ic->mode == 'J' || ic->mode == 'B') { ic->writer_wq = alloc_workqueue("dm-integrity-writer", WQ_MEM_RECLAIM, 1); if (!ic->writer_wq) { ti->error = "Cannot allocate workqueue"; @@ -3333,7 +5041,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } - r = sync_rw_sb(ic, REQ_OP_READ, 0); + r = sync_rw_sb(ic, REQ_OP_READ); if (r) { ti->error = "Error reading superblock"; goto bad; @@ -3357,11 +5065,16 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) should_write_sb = true; } - if (!ic->sb->version || ic->sb->version > SB_VERSION_2) { + if (!ic->sb->version || ic->sb->version > SB_VERSION_6) { r = -EINVAL; ti->error = "Unknown version"; goto bad; } + if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_INLINE)) != (ic->mode == 'I')) { + r = -EINVAL; + ti->error = "Inline flag mismatch"; + goto bad; + } if (le16_to_cpu(ic->sb->integrity_tag_size) != ic->tag_size) { r = -EINVAL; ti->error = "Tag size doesn't match the information in superblock"; @@ -3372,10 +5085,18 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ti->error = "Block size doesn't match the information in superblock"; goto bad; } - if (!le32_to_cpu(ic->sb->journal_sections)) { - r = -EINVAL; - ti->error = "Corrupted superblock, journal_sections is 0"; - goto bad; + if (ic->mode != 'I') { + if (!le32_to_cpu(ic->sb->journal_sections)) { + r = -EINVAL; + ti->error = "Corrupted superblock, journal_sections is 0"; + goto bad; + } + } else { + if (le32_to_cpu(ic->sb->journal_sections)) { + r = -EINVAL; + ti->error = "Corrupted superblock, journal_sections is not 0"; + goto bad; + } } /* make sure that ti->max_io_len doesn't overflow */ if (!ic->meta_dev) { @@ -3392,16 +5113,16 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } } - ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); - if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) { - /* test for overflow */ + if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { r = -EINVAL; - ti->error = "The superblock has 64-bit device size, but the kernel was compiled with 32-bit sectors"; + ti->error = "Journal mac mismatch"; goto bad; } - if (!!(ic->sb->flags & cpu_to_le32(SB_FLAG_HAVE_JOURNAL_MAC)) != !!ic->journal_mac_alg.alg_string) { + + get_provided_data_sectors(ic); + if (!ic->provided_data_sectors) { r = -EINVAL; - ti->error = "Journal mac mismatch"; + ti->error = "The device is too small"; goto bad; } @@ -3417,6 +5138,28 @@ try_smaller_buffer: ti->error = "The device is too small"; goto bad; } + + if (log2_sectors_per_bitmap_bit < 0) + log2_sectors_per_bitmap_bit = __fls(DEFAULT_SECTORS_PER_BITMAP_BIT); + if (log2_sectors_per_bitmap_bit < ic->sb->log2_sectors_per_block) + log2_sectors_per_bitmap_bit = ic->sb->log2_sectors_per_block; + + bits_in_journal = ((__u64)ic->journal_section_sectors * ic->journal_sections) << (SECTOR_SHIFT + 3); + if (bits_in_journal > UINT_MAX) + bits_in_journal = UINT_MAX; + if (bits_in_journal) + while (bits_in_journal < (ic->provided_data_sectors + ((sector_t)1 << log2_sectors_per_bitmap_bit) - 1) >> log2_sectors_per_bitmap_bit) + log2_sectors_per_bitmap_bit++; + + log2_blocks_per_bitmap_bit = log2_sectors_per_bitmap_bit - ic->sb->log2_sectors_per_block; + ic->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; + if (should_write_sb) + ic->sb->log2_blocks_per_bitmap_bit = log2_blocks_per_bitmap_bit; + + n_bitmap_bits = ((ic->provided_data_sectors >> ic->sb->log2_sectors_per_block) + + (((sector_t)1 << log2_blocks_per_bitmap_bit) - 1)) >> log2_blocks_per_bitmap_bit; + ic->n_bitmap_blocks = DIV_ROUND_UP(n_bitmap_bits, BITMAP_BLOCK_SIZE * 8); + if (!ic->meta_dev) ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); @@ -3426,7 +5169,6 @@ try_smaller_buffer: goto bad; } - threshold = (__u64)ic->journal_entries * (100 - journal_watermark); threshold += 50; do_div(threshold, 100); @@ -3438,52 +5180,48 @@ try_smaller_buffer: DEBUG_print(" journal_entries_per_sector %u\n", ic->journal_entries_per_sector); DEBUG_print(" journal_section_entries %u\n", ic->journal_section_entries); DEBUG_print(" journal_section_sectors %u\n", ic->journal_section_sectors); - DEBUG_print(" journal_sections %u\n", (unsigned)le32_to_cpu(ic->sb->journal_sections)); + DEBUG_print(" journal_sections %u\n", (unsigned int)le32_to_cpu(ic->sb->journal_sections)); DEBUG_print(" journal_entries %u\n", ic->journal_entries); DEBUG_print(" log2_interleave_sectors %d\n", ic->sb->log2_interleave_sectors); - DEBUG_print(" device_sectors 0x%llx\n", (unsigned long long)ic->device_sectors); + DEBUG_print(" data_device_sectors 0x%llx\n", bdev_nr_sectors(ic->dev->bdev)); DEBUG_print(" initial_sectors 0x%x\n", ic->initial_sectors); DEBUG_print(" metadata_run 0x%x\n", ic->metadata_run); DEBUG_print(" log2_metadata_run %d\n", ic->log2_metadata_run); - DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", (unsigned long long)ic->provided_data_sectors, - (unsigned long long)ic->provided_data_sectors); + DEBUG_print(" provided_data_sectors 0x%llx (%llu)\n", ic->provided_data_sectors, ic->provided_data_sectors); DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); + DEBUG_print(" bits_in_journal %llu\n", bits_in_journal); - if (recalculate && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { + if (ic->recalculate_flag && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); ic->sb->recalc_sector = cpu_to_le64(0); } - if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { - if (!ic->internal_hash) { - r = -EINVAL; - ti->error = "Recalculate is only valid with internal hash"; - goto bad; - } + if (ic->internal_hash) { ic->recalc_wq = alloc_workqueue("dm-integrity-recalc", WQ_MEM_RECLAIM, 1); - if (!ic->recalc_wq ) { + if (!ic->recalc_wq) { ti->error = "Cannot allocate workqueue"; r = -ENOMEM; goto bad; } - INIT_WORK(&ic->recalc_work, integrity_recalc); - ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT); - if (!ic->recalc_buffer) { - ti->error = "Cannot allocate buffer for recalculating"; - r = -ENOMEM; - goto bad; - } - ic->recalc_tags = kvmalloc_array(RECALC_SECTORS >> ic->sb->log2_sectors_per_block, - ic->tag_size, GFP_KERNEL); - if (!ic->recalc_tags) { - ti->error = "Cannot allocate tags for recalculating"; - r = -ENOMEM; + INIT_WORK(&ic->recalc_work, ic->mode == 'I' ? integrity_recalc_inline : integrity_recalc); + } else { + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { + ti->error = "Recalculate can only be specified with internal_hash"; + r = -EINVAL; goto bad; } } + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + le64_to_cpu(ic->sb->recalc_sector) < ic->provided_data_sectors && + dm_integrity_disable_recalculate(ic)) { + ti->error = "Recalculating with HMAC is disabled for security reasons - if you really need it, use the argument \"legacy_recalculate\""; + r = -EOPNOTSUPP; + goto bad; + } + ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, - 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL); + 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL, 0); if (IS_ERR(ic->bufio)) { r = PTR_ERR(ic->bufio); ti->error = "Cannot initialize dm-bufio"; @@ -3492,22 +5230,62 @@ try_smaller_buffer: } dm_bufio_set_sector_offset(ic->bufio, ic->start + ic->initial_sectors); - if (ic->mode != 'R') { + if (ic->mode != 'R' && ic->mode != 'I') { r = create_journal(ic, &ti->error); if (r) goto bad; + } - if (should_write_sb) { - int r; + if (ic->mode == 'B') { + unsigned int i; + unsigned int n_bitmap_pages = DIV_ROUND_UP(ic->n_bitmap_blocks, PAGE_SIZE / BITMAP_BLOCK_SIZE); + + ic->recalc_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); + if (!ic->recalc_bitmap) { + ti->error = "Could not allocate memory for bitmap"; + r = -ENOMEM; + goto bad; + } + ic->may_write_bitmap = dm_integrity_alloc_page_list(n_bitmap_pages); + if (!ic->may_write_bitmap) { + ti->error = "Could not allocate memory for bitmap"; + r = -ENOMEM; + goto bad; + } + ic->bbs = kvmalloc_array(ic->n_bitmap_blocks, sizeof(struct bitmap_block_status), GFP_KERNEL); + if (!ic->bbs) { + ti->error = "Could not allocate memory for bitmap"; + r = -ENOMEM; + goto bad; + } + INIT_DELAYED_WORK(&ic->bitmap_flush_work, bitmap_flush_work); + for (i = 0; i < ic->n_bitmap_blocks; i++) { + struct bitmap_block_status *bbs = &ic->bbs[i]; + unsigned int sector, pl_index, pl_offset; + + INIT_WORK(&bbs->work, bitmap_block_work); + bbs->ic = ic; + bbs->idx = i; + bio_list_init(&bbs->bio_queue); + spin_lock_init(&bbs->bio_queue_lock); + + sector = i * (BITMAP_BLOCK_SIZE >> SECTOR_SHIFT); + pl_index = sector >> (PAGE_SHIFT - SECTOR_SHIFT); + pl_offset = (sector << SECTOR_SHIFT) & (PAGE_SIZE - 1); + + bbs->bitmap = lowmem_page_address(ic->journal[pl_index].page) + pl_offset; + } + } + if (should_write_sb) { init_journal(ic, 0, ic->journal_sections, 0); r = dm_integrity_failed(ic); if (unlikely(r)) { ti->error = "Error initializing journal"; goto bad; } - r = sync_rw_sb(ic, REQ_OP_WRITE, REQ_FUA); + r = sync_rw_sb(ic, REQ_OP_WRITE | REQ_FUA); if (r) { ti->error = "Error initializing superblock"; goto bad; @@ -3515,20 +5293,38 @@ try_smaller_buffer: ic->just_formatted = true; } - if (!ic->meta_dev) { + if (!ic->meta_dev && ic->mode != 'I') { r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); if (r) goto bad; } - - if (!ic->internal_hash) - dm_integrity_set(ti, ic); + if (ic->mode == 'B') { + unsigned int max_io_len; + + max_io_len = ((sector_t)ic->sectors_per_block << ic->log2_blocks_per_bitmap_bit) * (BITMAP_BLOCK_SIZE * 8); + if (!max_io_len) + max_io_len = 1U << 31; + DEBUG_print("max_io_len: old %u, new %u\n", ti->max_io_len, max_io_len); + if (!ti->max_io_len || ti->max_io_len > max_io_len) { + r = dm_set_target_max_io_len(ti, max_io_len); + if (r) + goto bad; + } + } ti->num_flush_bios = 1; ti->flush_supported = true; + if (ic->discard) + ti->num_discard_bios = 1; + if (ic->mode == 'I') + ti->mempool_needs_integrity = true; + + dm_audit_log_ctr(DM_MSG_PREFIX, ti, 1); return 0; + bad: + dm_audit_log_ctr(DM_MSG_PREFIX, ti, 0); dm_integrity_dtr(ti); return r; } @@ -3540,22 +5336,28 @@ static void dm_integrity_dtr(struct dm_target *ti) BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); BUG_ON(!list_empty(&ic->wait_list)); + if (ic->mode == 'B' && ic->bitmap_flush_work.work.func) + cancel_delayed_work_sync(&ic->bitmap_flush_work); if (ic->metadata_wq) destroy_workqueue(ic->metadata_wq); if (ic->wait_wq) destroy_workqueue(ic->wait_wq); + if (ic->offload_wq) + destroy_workqueue(ic->offload_wq); if (ic->commit_wq) destroy_workqueue(ic->commit_wq); if (ic->writer_wq) destroy_workqueue(ic->writer_wq); if (ic->recalc_wq) destroy_workqueue(ic->recalc_wq); - if (ic->recalc_buffer) - vfree(ic->recalc_buffer); - if (ic->recalc_tags) - kvfree(ic->recalc_tags); + kvfree(ic->bbs); if (ic->bufio) dm_bufio_client_destroy(ic->bufio); + mempool_free(ic->journal_ahash_req, &ic->ahash_req_pool); + mempool_exit(&ic->ahash_req_pool); + bioset_exit(&ic->recalc_bios); + bioset_exit(&ic->recheck_bios); + mempool_exit(&ic->recheck_pool); mempool_exit(&ic->journal_io_mempool); if (ic->io) dm_io_client_destroy(ic->io); @@ -3563,20 +5365,24 @@ static void dm_integrity_dtr(struct dm_target *ti) dm_put_device(ti, ic->dev); if (ic->meta_dev) dm_put_device(ti, ic->meta_dev); - dm_integrity_free_page_list(ic, ic->journal); - dm_integrity_free_page_list(ic, ic->journal_io); - dm_integrity_free_page_list(ic, ic->journal_xor); + dm_integrity_free_page_list(ic->journal); + dm_integrity_free_page_list(ic->journal_io); + dm_integrity_free_page_list(ic->journal_xor); + dm_integrity_free_page_list(ic->recalc_bitmap); + dm_integrity_free_page_list(ic->may_write_bitmap); if (ic->journal_scatterlist) dm_integrity_free_journal_scatterlist(ic, ic->journal_scatterlist); if (ic->journal_io_scatterlist) dm_integrity_free_journal_scatterlist(ic, ic->journal_io_scatterlist); if (ic->sk_requests) { - unsigned i; + unsigned int i; for (i = 0; i < ic->journal_sections; i++) { - struct skcipher_request *req = ic->sk_requests[i]; + struct skcipher_request *req; + + req = ic->sk_requests[i]; if (req) { - kzfree(req->iv); + kfree_sensitive(req->iv); skcipher_request_free(req); } } @@ -3586,8 +5392,10 @@ static void dm_integrity_dtr(struct dm_target *ti) if (ic->sb) free_pages_exact(ic->sb, SB_SECTORS << SECTOR_SHIFT); - if (ic->internal_hash) - crypto_free_shash(ic->internal_hash); + if (ic->internal_shash) + crypto_free_shash(ic->internal_shash); + if (ic->internal_ahash) + crypto_free_ahash(ic->internal_ahash); free_alg(&ic->internal_hash_alg); if (ic->journal_crypt) @@ -3599,16 +5407,18 @@ static void dm_integrity_dtr(struct dm_target *ti) free_alg(&ic->journal_mac_alg); kfree(ic); + dm_audit_log_dtr(DM_MSG_PREFIX, ti, 1); } static struct target_type integrity_target = { .name = "integrity", - .version = {1, 2, 0}, + .version = {1, 14, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, .dtr = dm_integrity_dtr, .map = dm_integrity_map, + .end_io = dm_integrity_end_io, .postsuspend = dm_integrity_postsuspend, .resume = dm_integrity_resume, .status = dm_integrity_status, @@ -3616,7 +5426,7 @@ static struct target_type integrity_target = { .io_hints = dm_integrity_io_hints, }; -int __init dm_integrity_init(void) +static int __init dm_integrity_init(void) { int r; @@ -3628,14 +5438,15 @@ int __init dm_integrity_init(void) } r = dm_register_target(&integrity_target); + if (r < 0) { + kmem_cache_destroy(journal_io_cache); + return r; + } - if (r < 0) - DMERR("register failed %d", r); - - return r; + return 0; } -void dm_integrity_exit(void) +static void __exit dm_integrity_exit(void) { dm_unregister_target(&integrity_target); kmem_cache_destroy(journal_io_cache); |