diff options
Diffstat (limited to 'drivers/md/dm-stats.c')
| -rw-r--r-- | drivers/md/dm-stats.c | 168 |
1 files changed, 112 insertions, 56 deletions
diff --git a/drivers/md/dm-stats.c b/drivers/md/dm-stats.c index 35d368c418d0..1e5d988f44da 100644 --- a/drivers/md/dm-stats.c +++ b/drivers/md/dm-stats.c @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +// SPDX-License-Identifier: GPL-2.0-only #include <linux/errno.h> #include <linux/numa.h> #include <linux/slab.h> @@ -42,12 +42,12 @@ struct dm_stat_shared { struct dm_stat { struct list_head list_entry; int id; - unsigned stat_flags; + unsigned int stat_flags; size_t n_entries; sector_t start; sector_t end; sector_t step; - unsigned n_histogram_entries; + unsigned int n_histogram_entries; unsigned long long *histogram_boundaries; const char *program_id; const char *aux_data; @@ -56,16 +56,19 @@ struct dm_stat { size_t percpu_alloc_size; size_t histogram_alloc_size; struct dm_stat_percpu *stat_percpu[NR_CPUS]; - struct dm_stat_shared stat_shared[]; + struct dm_stat_shared stat_shared[] __counted_by(n_entries); }; #define STAT_PRECISE_TIMESTAMPS 1 struct dm_stats_last_position { sector_t last_sector; - unsigned last_rw; + unsigned int last_rw; }; +#define DM_STAT_MAX_ENTRIES 8388608 +#define DM_STAT_MAX_HISTOGRAM_ENTRIES 134217728 + /* * A typo on the command line could possibly make the kernel run out of memory * and crash. To prevent the crash we account all used memory. We fail if we @@ -188,19 +191,25 @@ static int dm_stat_in_flight(struct dm_stat_shared *shared) atomic_read(&shared->in_flight[WRITE]); } -void dm_stats_init(struct dm_stats *stats) +int dm_stats_init(struct dm_stats *stats) { int cpu; struct dm_stats_last_position *last; mutex_init(&stats->mutex); INIT_LIST_HEAD(&stats->list); + stats->precise_timestamps = false; stats->last = alloc_percpu(struct dm_stats_last_position); + if (!stats->last) + return -ENOMEM; + for_each_possible_cpu(cpu) { last = per_cpu_ptr(stats->last, cpu); last->last_sector = (sector_t)ULLONG_MAX; last->last_rw = UINT_MAX; } + + return 0; } void dm_stats_cleanup(struct dm_stats *stats) @@ -224,6 +233,7 @@ void dm_stats_cleanup(struct dm_stats *stats) atomic_read(&shared->in_flight[READ]), atomic_read(&shared->in_flight[WRITE])); } + cond_resched(); } dm_stat_free(&s->rcu_head); } @@ -231,9 +241,25 @@ void dm_stats_cleanup(struct dm_stats *stats) mutex_destroy(&stats->mutex); } +static void dm_stats_recalc_precise_timestamps(struct dm_stats *stats) +{ + struct list_head *l; + struct dm_stat *tmp_s; + bool precise_timestamps = false; + + list_for_each(l, &stats->list) { + tmp_s = container_of(l, struct dm_stat, list_entry); + if (tmp_s->stat_flags & STAT_PRECISE_TIMESTAMPS) { + precise_timestamps = true; + break; + } + } + stats->precise_timestamps = precise_timestamps; +} + static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, - sector_t step, unsigned stat_flags, - unsigned n_histogram_entries, + sector_t step, unsigned int stat_flags, + unsigned int n_histogram_entries, unsigned long long *histogram_boundaries, const char *program_id, const char *aux_data, void (*suspend_callback)(struct mapped_device *), @@ -262,6 +288,9 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, if (n_entries != (size_t)n_entries || !(size_t)(n_entries + 1)) return -EOVERFLOW; + if (n_entries > DM_STAT_MAX_ENTRIES) + return -EOVERFLOW; + shared_alloc_size = struct_size(s, stat_shared, n_entries); if ((shared_alloc_size - sizeof(struct dm_stat)) / sizeof(struct dm_stat_shared) != n_entries) return -EOVERFLOW; @@ -274,6 +303,9 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, if (histogram_alloc_size / (n_histogram_entries + 1) != (size_t)n_entries * sizeof(unsigned long long)) return -EOVERFLOW; + if ((n_histogram_entries + 1) * (size_t)n_entries > DM_STAT_MAX_HISTOGRAM_ENTRIES) + return -EOVERFLOW; + if (!check_shared_memory(shared_alloc_size + histogram_alloc_size + num_possible_cpus() * (percpu_alloc_size + histogram_alloc_size))) return -ENOMEM; @@ -313,10 +345,12 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { atomic_set(&s->stat_shared[ni].in_flight[READ], 0); atomic_set(&s->stat_shared[ni].in_flight[WRITE], 0); + cond_resched(); } if (s->n_histogram_entries) { unsigned long long *hi; + hi = dm_kvzalloc(s->histogram_alloc_size, NUMA_NO_NODE); if (!hi) { r = -ENOMEM; @@ -325,6 +359,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { s->stat_shared[ni].tmp.histogram = hi; hi += s->n_histogram_entries + 1; + cond_resched(); } } @@ -337,6 +372,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, s->stat_percpu[cpu] = p; if (s->n_histogram_entries) { unsigned long long *hi; + hi = dm_kvzalloc(s->histogram_alloc_size, cpu_to_node(cpu)); if (!hi) { r = -ENOMEM; @@ -345,6 +381,7 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, for (ni = 0; ni < n_entries; ni++) { p[ni].histogram = hi; hi += s->n_histogram_entries + 1; + cond_resched(); } } } @@ -376,6 +413,12 @@ static int dm_stats_create(struct dm_stats *stats, sector_t start, sector_t end, } ret_id = s->id; list_add_tail_rcu(&s->list_entry, l); + + dm_stats_recalc_precise_timestamps(stats); + + if (!static_key_enabled(&stats_enabled.key)) + static_branch_enable(&stats_enabled); + mutex_unlock(&stats->mutex); resume_callback(md); @@ -418,6 +461,9 @@ static int dm_stats_delete(struct dm_stats *stats, int id) } list_del_rcu(&s->list_entry); + + dm_stats_recalc_precise_timestamps(stats); + mutex_unlock(&stats->mutex); /* @@ -440,11 +486,11 @@ do_sync_free: } static int dm_stats_list(struct dm_stats *stats, const char *program, - char *result, unsigned maxlen) + char *result, unsigned int maxlen) { struct dm_stat *s; sector_t len; - unsigned sz = 0; + unsigned int sz = 0; /* * Output format: @@ -464,7 +510,8 @@ static int dm_stats_list(struct dm_stats *stats, const char *program, if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) DMEMIT(" precise_timestamps"); if (s->n_histogram_entries) { - unsigned i; + unsigned int i; + DMEMIT(" histogram:"); for (i = 0; i < s->n_histogram_entries; i++) { if (i) @@ -474,6 +521,7 @@ static int dm_stats_list(struct dm_stats *stats, const char *program, } DMEMIT("\n"); } + cond_resched(); } mutex_unlock(&stats->mutex); @@ -487,7 +535,7 @@ static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, * This is racy, but so is part_round_stats_single. */ unsigned long long now, difference; - unsigned in_flight_read, in_flight_write; + unsigned int in_flight_read, in_flight_write; if (likely(!(s->stat_flags & STAT_PRECISE_TIMESTAMPS))) now = jiffies; @@ -498,8 +546,8 @@ static void dm_stat_round(struct dm_stat *s, struct dm_stat_shared *shared, if (!difference) return; - in_flight_read = (unsigned)atomic_read(&shared->in_flight[READ]); - in_flight_write = (unsigned)atomic_read(&shared->in_flight[WRITE]); + in_flight_read = (unsigned int)atomic_read(&shared->in_flight[READ]); + in_flight_write = (unsigned int)atomic_read(&shared->in_flight[WRITE]); if (in_flight_read) p->io_ticks[READ] += difference; if (in_flight_write) @@ -536,6 +584,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, */ #if BITS_PER_LONG == 32 unsigned long flags; + local_irq_save(flags); #else preempt_disable(); @@ -547,6 +596,7 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, atomic_inc(&shared->in_flight[idx]); } else { unsigned long long duration; + dm_stat_round(s, shared, p); atomic_dec(&shared->in_flight[idx]); p->sectors[idx] += len; @@ -560,15 +610,15 @@ static void dm_stat_for_entry(struct dm_stat *s, size_t entry, duration = stats_aux->duration_ns; } if (s->n_histogram_entries) { - unsigned lo = 0, hi = s->n_histogram_entries + 1; + unsigned int lo = 0, hi = s->n_histogram_entries + 1; + while (lo + 1 < hi) { - unsigned mid = (lo + hi) / 2; - if (s->histogram_boundaries[mid - 1] > duration) { + unsigned int mid = (lo + hi) / 2; + + if (s->histogram_boundaries[mid - 1] > duration) hi = mid; - } else { + else lo = mid; - } - } p->histogram[lo]++; } @@ -620,14 +670,15 @@ static void __dm_stat_bio(struct dm_stat *s, int bi_rw, } void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, - sector_t bi_sector, unsigned bi_sectors, bool end, - unsigned long duration_jiffies, + sector_t bi_sector, unsigned int bi_sectors, bool end, + unsigned long start_time, struct dm_stats_aux *stats_aux) { struct dm_stat *s; sector_t end_sector; struct dm_stats_last_position *last; bool got_precise_time; + unsigned long duration_jiffies = 0; if (unlikely(!bi_sectors)) return; @@ -647,16 +698,16 @@ void dm_stats_account_io(struct dm_stats *stats, unsigned long bi_rw, )); WRITE_ONCE(last->last_sector, end_sector); WRITE_ONCE(last->last_rw, bi_rw); - } + } else + duration_jiffies = jiffies - start_time; rcu_read_lock(); got_precise_time = false; list_for_each_entry_rcu(s, &stats->list, list_entry) { if (s->stat_flags & STAT_PRECISE_TIMESTAMPS && !got_precise_time) { - if (!end) - stats_aux->duration_ns = ktime_to_ns(ktime_get()); - else + /* start (!end) duration_ns is set by DM core's alloc_io() */ + if (end) stats_aux->duration_ns = ktime_to_ns(ktime_get()) - stats_aux->duration_ns; got_precise_time = true; } @@ -708,7 +759,8 @@ static void __dm_stat_init_temporary_percpu_totals(struct dm_stat_shared *shared shared->tmp.io_ticks_total += READ_ONCE(p->io_ticks_total); shared->tmp.time_in_queue += READ_ONCE(p->time_in_queue); if (s->n_histogram_entries) { - unsigned i; + unsigned int i; + for (i = 0; i < s->n_histogram_entries + 1; i++) shared->tmp.histogram[i] += READ_ONCE(p->histogram[i]); } @@ -742,7 +794,8 @@ static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, p->time_in_queue -= shared->tmp.time_in_queue; local_irq_enable(); if (s->n_histogram_entries) { - unsigned i; + unsigned int i; + for (i = 0; i < s->n_histogram_entries + 1; i++) { local_irq_disable(); p = &s->stat_percpu[smp_processor_id()][x]; @@ -750,6 +803,7 @@ static void __dm_stat_clear(struct dm_stat *s, size_t idx_start, size_t idx_end, local_irq_enable(); } } + cond_resched(); } } @@ -778,7 +832,7 @@ static int dm_stats_clear(struct dm_stats *stats, int id) static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long long j) { unsigned long long result; - unsigned mult; + unsigned int mult; if (s->stat_flags & STAT_PRECISE_TIMESTAMPS) return j; @@ -798,9 +852,9 @@ static unsigned long long dm_jiffies_to_msec64(struct dm_stat *s, unsigned long static int dm_stats_print(struct dm_stats *stats, int id, size_t idx_start, size_t idx_len, - bool clear, char *result, unsigned maxlen) + bool clear, char *result, unsigned int maxlen) { - unsigned sz = 0; + unsigned int sz = 0; struct dm_stat *s; size_t x; sector_t start, end, step; @@ -856,15 +910,17 @@ static int dm_stats_print(struct dm_stats *stats, int id, dm_jiffies_to_msec64(s, shared->tmp.io_ticks[READ]), dm_jiffies_to_msec64(s, shared->tmp.io_ticks[WRITE])); if (s->n_histogram_entries) { - unsigned i; - for (i = 0; i < s->n_histogram_entries + 1; i++) { + unsigned int i; + + for (i = 0; i < s->n_histogram_entries + 1; i++) DMEMIT("%s%llu", !i ? " " : ":", shared->tmp.histogram[i]); - } } DMEMIT("\n"); if (unlikely(sz + 1 >= maxlen)) goto buffer_overflow; + + cond_resched(); } if (clear) @@ -903,11 +959,11 @@ static int dm_stats_set_aux(struct dm_stats *stats, int id, const char *aux_data return 0; } -static int parse_histogram(const char *h, unsigned *n_histogram_entries, +static int parse_histogram(const char *h, unsigned int *n_histogram_entries, unsigned long long **histogram_boundaries) { const char *q; - unsigned n; + unsigned int n; unsigned long long last; *n_histogram_entries = 1; @@ -927,6 +983,7 @@ static int parse_histogram(const char *h, unsigned *n_histogram_entries, unsigned long long hi; int s; char ch; + s = sscanf(h, "%llu%c", &hi, &ch); if (!s || (s == 2 && ch != ',')) return -EINVAL; @@ -942,23 +999,21 @@ static int parse_histogram(const char *h, unsigned *n_histogram_entries, } static int message_stats_create(struct mapped_device *md, - unsigned argc, char **argv, - char *result, unsigned maxlen) + unsigned int argc, char **argv, + char *result, unsigned int maxlen) { int r; int id; char dummy; unsigned long long start, end, len, step; - unsigned divisor; + unsigned int divisor; const char *program_id, *aux_data; - unsigned stat_flags = 0; - - unsigned n_histogram_entries = 0; + unsigned int stat_flags = 0; + unsigned int n_histogram_entries = 0; unsigned long long *histogram_boundaries = NULL; - struct dm_arg_set as, as_backup; const char *a; - unsigned feature_args; + unsigned int feature_args; /* * Input format: @@ -1011,7 +1066,8 @@ static int message_stats_create(struct mapped_device *md, else if (!strncasecmp(a, "histogram:", 10)) { if (n_histogram_entries) goto ret_einval; - if ((r = parse_histogram(a + 10, &n_histogram_entries, &histogram_boundaries))) + r = parse_histogram(a + 10, &n_histogram_entries, &histogram_boundaries); + if (r) goto ret; } else goto ret_einval; @@ -1067,7 +1123,7 @@ ret: } static int message_stats_delete(struct mapped_device *md, - unsigned argc, char **argv) + unsigned int argc, char **argv) { int id; char dummy; @@ -1082,7 +1138,7 @@ static int message_stats_delete(struct mapped_device *md, } static int message_stats_clear(struct mapped_device *md, - unsigned argc, char **argv) + unsigned int argc, char **argv) { int id; char dummy; @@ -1097,8 +1153,8 @@ static int message_stats_clear(struct mapped_device *md, } static int message_stats_list(struct mapped_device *md, - unsigned argc, char **argv, - char *result, unsigned maxlen) + unsigned int argc, char **argv, + char *result, unsigned int maxlen) { int r; const char *program = NULL; @@ -1120,8 +1176,8 @@ static int message_stats_list(struct mapped_device *md, } static int message_stats_print(struct mapped_device *md, - unsigned argc, char **argv, bool clear, - char *result, unsigned maxlen) + unsigned int argc, char **argv, bool clear, + char *result, unsigned int maxlen) { int id; char dummy; @@ -1147,7 +1203,7 @@ static int message_stats_print(struct mapped_device *md, } static int message_stats_set_aux(struct mapped_device *md, - unsigned argc, char **argv) + unsigned int argc, char **argv) { int id; char dummy; @@ -1161,8 +1217,8 @@ static int message_stats_set_aux(struct mapped_device *md, return dm_stats_set_aux(dm_get_stats(md), id, argv[2]); } -int dm_stats_message(struct mapped_device *md, unsigned argc, char **argv, - char *result, unsigned maxlen) +int dm_stats_message(struct mapped_device *md, unsigned int argc, char **argv, + char *result, unsigned int maxlen) { int r; @@ -1185,7 +1241,7 @@ int dm_stats_message(struct mapped_device *md, unsigned argc, char **argv, return 2; /* this wasn't a stats message */ if (r == -EINVAL) - DMWARN("Invalid parameters for message %s", argv[0]); + DMCRIT("Invalid parameters for message %s", argv[0]); return r; } @@ -1205,5 +1261,5 @@ void dm_statistics_exit(void) DMCRIT("shared_memory_amount leaked: %lu", shared_memory_amount); } -module_param_named(stats_current_allocated_bytes, shared_memory_amount, ulong, S_IRUGO); +module_param_named(stats_current_allocated_bytes, shared_memory_amount, ulong, 0444); MODULE_PARM_DESC(stats_current_allocated_bytes, "Memory currently used by statistics"); |