summaryrefslogtreecommitdiff
path: root/kernel/kcsan/report.c
diff options
context:
space:
mode:
Diffstat (limited to 'kernel/kcsan/report.c')
-rw-r--r--kernel/kcsan/report.c455
1 files changed, 287 insertions, 168 deletions
diff --git a/kernel/kcsan/report.c b/kernel/kcsan/report.c
index 11c791b886f3..ac5f8345bae9 100644
--- a/kernel/kcsan/report.c
+++ b/kernel/kcsan/report.c
@@ -1,5 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
+#include <linux/debug_locks.h>
+#include <linux/delay.h>
#include <linux/jiffies.h>
#include <linux/kernel.h>
#include <linux/lockdep.h>
@@ -17,21 +19,49 @@
*/
#define NUM_STACK_ENTRIES 64
-/*
- * Other thread info: communicated from other racing thread to thread that set
- * up the watchpoint, which then prints the complete report atomically. Only
- * need one struct, as all threads should to be serialized regardless to print
- * the reports, with reporting being in the slow-path.
- */
-static struct {
+/* Common access info. */
+struct access_info {
const volatile void *ptr;
size_t size;
int access_type;
int task_pid;
int cpu_id;
+};
+
+/*
+ * Other thread info: communicated from other racing thread to thread that set
+ * up the watchpoint, which then prints the complete report atomically.
+ */
+struct other_info {
+ struct access_info ai;
unsigned long stack_entries[NUM_STACK_ENTRIES];
int num_stack_entries;
-} other_info = { .ptr = NULL };
+
+ /*
+ * Optionally pass @current. Typically we do not need to pass @current
+ * via @other_info since just @task_pid is sufficient. Passing @current
+ * has additional overhead.
+ *
+ * To safely pass @current, we must either use get_task_struct/
+ * put_task_struct, or stall the thread that populated @other_info.
+ *
+ * We cannot rely on get_task_struct/put_task_struct in case
+ * release_report() races with a task being released, and would have to
+ * free it in release_report(). This may result in deadlock if we want
+ * to use KCSAN on the allocators.
+ *
+ * Since we also want to reliably print held locks for
+ * CONFIG_KCSAN_VERBOSE, the current implementation stalls the thread
+ * that populated @other_info until it has been consumed.
+ */
+ struct task_struct *task;
+};
+
+/*
+ * To never block any producers of struct other_info, we need as many elements
+ * as we have watchpoints (upper bound on concurrent races to report).
+ */
+static struct other_info other_infos[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1];
/*
* Information about reported races; used to rate limit reporting.
@@ -68,10 +98,11 @@ struct report_time {
static struct report_time report_times[REPORT_TIMES_SIZE];
/*
- * This spinlock protects reporting and other_info, since other_info is usually
- * required when reporting.
+ * Spinlock serializing report generation, and access to @other_infos. Although
+ * it could make sense to have a finer-grained locking story for @other_infos,
+ * report generation needs to be serialized either way, so not much is gained.
*/
-static DEFINE_SPINLOCK(report_lock);
+static DEFINE_RAW_SPINLOCK(report_lock);
/*
* Checks if the race identified by thread frames frame1 and frame2 has
@@ -161,11 +192,11 @@ skip_report(enum kcsan_value_change value_change, unsigned long top_frame)
* maintainers.
*/
char buf[64];
+ int len = scnprintf(buf, sizeof(buf), "%ps", (void *)top_frame);
- snprintf(buf, sizeof(buf), "%ps", (void *)top_frame);
- if (!strnstr(buf, "rcu_", sizeof(buf)) &&
- !strnstr(buf, "_rcu", sizeof(buf)) &&
- !strnstr(buf, "_srcu", sizeof(buf)))
+ if (!strnstr(buf, "rcu_", len) &&
+ !strnstr(buf, "_rcu", len) &&
+ !strnstr(buf, "_srcu", len))
return true;
}
@@ -174,6 +205,20 @@ skip_report(enum kcsan_value_change value_change, unsigned long top_frame)
static const char *get_access_type(int type)
{
+ if (type & KCSAN_ACCESS_ASSERT) {
+ if (type & KCSAN_ACCESS_SCOPED) {
+ if (type & KCSAN_ACCESS_WRITE)
+ return "assert no accesses (scoped)";
+ else
+ return "assert no writes (scoped)";
+ } else {
+ if (type & KCSAN_ACCESS_WRITE)
+ return "assert no accesses";
+ else
+ return "assert no writes";
+ }
+ }
+
switch (type) {
case 0:
return "read";
@@ -183,17 +228,14 @@ static const char *get_access_type(int type)
return "write";
case KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
return "write (marked)";
-
- /*
- * ASSERT variants:
- */
- case KCSAN_ACCESS_ASSERT:
- case KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_ATOMIC:
- return "assert no writes";
- case KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE:
- case KCSAN_ACCESS_ASSERT | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
- return "assert no accesses";
-
+ case KCSAN_ACCESS_SCOPED:
+ return "read (scoped)";
+ case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_ATOMIC:
+ return "read (marked, scoped)";
+ case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE:
+ return "write (scoped)";
+ case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
+ return "write (marked, scoped)";
default:
BUG();
}
@@ -217,19 +259,35 @@ static const char *get_thread_desc(int task_id)
}
/* Helper to skip KCSAN-related functions in stack-trace. */
-static int get_stack_skipnr(unsigned long stack_entries[], int num_entries)
+static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries)
{
char buf[64];
- int skip = 0;
-
- for (; skip < num_entries; ++skip) {
- snprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]);
- if (!strnstr(buf, "csan_", sizeof(buf)) &&
- !strnstr(buf, "tsan_", sizeof(buf)) &&
- !strnstr(buf, "_once_size", sizeof(buf))) {
- break;
+ char *cur;
+ int len, skip;
+
+ for (skip = 0; skip < num_entries; ++skip) {
+ len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]);
+
+ /* Never show tsan_* or {read,write}_once_size. */
+ if (strnstr(buf, "tsan_", len) ||
+ strnstr(buf, "_once_size", len))
+ continue;
+
+ cur = strnstr(buf, "kcsan_", len);
+ if (cur) {
+ cur += sizeof("kcsan_") - 1;
+ if (strncmp(cur, "test", sizeof("test") - 1))
+ continue; /* KCSAN runtime function. */
+ /* KCSAN related test. */
}
+
+ /*
+ * No match for runtime functions -- @skip entries to skip to
+ * get to first frame of interest.
+ */
+ break;
}
+
return skip;
}
@@ -245,12 +303,23 @@ static int sym_strcmp(void *addr1, void *addr2)
return strncmp(buf1, buf2, sizeof(buf1));
}
+static void print_verbose_info(struct task_struct *task)
+{
+ if (!task)
+ return;
+
+ pr_err("\n");
+ debug_show_held_locks(task);
+ print_irqtrace_events(task);
+}
+
/*
* Returns true if a report was generated, false otherwise.
*/
-static bool print_report(const volatile void *ptr, size_t size, int access_type,
- enum kcsan_value_change value_change, int cpu_id,
- enum kcsan_report_type type)
+static bool print_report(enum kcsan_value_change value_change,
+ enum kcsan_report_type type,
+ const struct access_info *ai,
+ const struct other_info *other_info)
{
unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 };
int num_stack_entries = stack_trace_save(stack_entries, NUM_STACK_ENTRIES, 1);
@@ -266,9 +335,9 @@ static bool print_report(const volatile void *ptr, size_t size, int access_type,
return false;
if (type == KCSAN_REPORT_RACE_SIGNAL) {
- other_skipnr = get_stack_skipnr(other_info.stack_entries,
- other_info.num_stack_entries);
- other_frame = other_info.stack_entries[other_skipnr];
+ other_skipnr = get_stack_skipnr(other_info->stack_entries,
+ other_info->num_stack_entries);
+ other_frame = other_info->stack_entries[other_skipnr];
/* @value_change is only known for the other thread */
if (skip_report(value_change, other_frame))
@@ -290,13 +359,13 @@ static bool print_report(const volatile void *ptr, size_t size, int access_type,
*/
cmp = sym_strcmp((void *)other_frame, (void *)this_frame);
pr_err("BUG: KCSAN: %s in %ps / %ps\n",
- get_bug_type(access_type | other_info.access_type),
+ get_bug_type(ai->access_type | other_info->ai.access_type),
(void *)(cmp < 0 ? other_frame : this_frame),
(void *)(cmp < 0 ? this_frame : other_frame));
} break;
case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
- pr_err("BUG: KCSAN: %s in %pS\n", get_bug_type(access_type),
+ pr_err("BUG: KCSAN: %s in %pS\n", get_bug_type(ai->access_type),
(void *)this_frame);
break;
@@ -310,27 +379,28 @@ static bool print_report(const volatile void *ptr, size_t size, int access_type,
switch (type) {
case KCSAN_REPORT_RACE_SIGNAL:
pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
- get_access_type(other_info.access_type), other_info.ptr,
- other_info.size, get_thread_desc(other_info.task_pid),
- other_info.cpu_id);
+ get_access_type(other_info->ai.access_type), other_info->ai.ptr,
+ other_info->ai.size, get_thread_desc(other_info->ai.task_pid),
+ other_info->ai.cpu_id);
/* Print the other thread's stack trace. */
- stack_trace_print(other_info.stack_entries + other_skipnr,
- other_info.num_stack_entries - other_skipnr,
+ stack_trace_print(other_info->stack_entries + other_skipnr,
+ other_info->num_stack_entries - other_skipnr,
0);
+ if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
+ print_verbose_info(other_info->task);
+
pr_err("\n");
pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
- get_access_type(access_type), ptr, size,
- get_thread_desc(in_task() ? task_pid_nr(current) : -1),
- cpu_id);
+ get_access_type(ai->access_type), ai->ptr, ai->size,
+ get_thread_desc(ai->task_pid), ai->cpu_id);
break;
case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN:
pr_err("race at unknown origin, with %s to 0x%px of %zu bytes by %s on cpu %i:\n",
- get_access_type(access_type), ptr, size,
- get_thread_desc(in_task() ? task_pid_nr(current) : -1),
- cpu_id);
+ get_access_type(ai->access_type), ai->ptr, ai->size,
+ get_thread_desc(ai->task_pid), ai->cpu_id);
break;
default:
@@ -340,6 +410,9 @@ static bool print_report(const volatile void *ptr, size_t size, int access_type,
stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr,
0);
+ if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
+ print_verbose_info(current);
+
/* Print report footer. */
pr_err("\n");
pr_err("Reported by Kernel Concurrency Sanitizer on:\n");
@@ -349,142 +422,188 @@ static bool print_report(const volatile void *ptr, size_t size, int access_type,
return true;
}
-static void release_report(unsigned long *flags, enum kcsan_report_type type)
+static void release_report(unsigned long *flags, struct other_info *other_info)
{
- if (type == KCSAN_REPORT_RACE_SIGNAL)
- other_info.ptr = NULL; /* mark for reuse */
+ if (other_info)
+ /*
+ * Use size to denote valid/invalid, since KCSAN entirely
+ * ignores 0-sized accesses.
+ */
+ other_info->ai.size = 0;
- spin_unlock_irqrestore(&report_lock, *flags);
+ raw_spin_unlock_irqrestore(&report_lock, *flags);
}
/*
- * Depending on the report type either sets other_info and returns false, or
- * acquires the matching other_info and returns true. If other_info is not
- * required for the report type, simply acquires report_lock and returns true.
+ * Sets @other_info->task and awaits consumption of @other_info.
+ *
+ * Precondition: report_lock is held.
+ * Postcondition: report_lock is held.
*/
-static bool prepare_report(unsigned long *flags, const volatile void *ptr,
- size_t size, int access_type, int cpu_id,
- enum kcsan_report_type type)
+static void set_other_info_task_blocking(unsigned long *flags,
+ const struct access_info *ai,
+ struct other_info *other_info)
{
- if (type != KCSAN_REPORT_CONSUMED_WATCHPOINT &&
- type != KCSAN_REPORT_RACE_SIGNAL) {
- /* other_info not required; just acquire report_lock */
- spin_lock_irqsave(&report_lock, *flags);
- return true;
- }
+ /*
+ * We may be instrumenting a code-path where current->state is already
+ * something other than TASK_RUNNING.
+ */
+ const bool is_running = current->state == TASK_RUNNING;
+ /*
+ * To avoid deadlock in case we are in an interrupt here and this is a
+ * race with a task on the same CPU (KCSAN_INTERRUPT_WATCHER), provide a
+ * timeout to ensure this works in all contexts.
+ *
+ * Await approximately the worst case delay of the reporting thread (if
+ * we are not interrupted).
+ */
+ int timeout = max(kcsan_udelay_task, kcsan_udelay_interrupt);
+
+ other_info->task = current;
+ do {
+ if (is_running) {
+ /*
+ * Let lockdep know the real task is sleeping, to print
+ * the held locks (recall we turned lockdep off, so
+ * locking/unlocking @report_lock won't be recorded).
+ */
+ set_current_state(TASK_UNINTERRUPTIBLE);
+ }
+ raw_spin_unlock_irqrestore(&report_lock, *flags);
+ /*
+ * We cannot call schedule() since we also cannot reliably
+ * determine if sleeping here is permitted -- see in_atomic().
+ */
-retry:
- spin_lock_irqsave(&report_lock, *flags);
+ udelay(1);
+ raw_spin_lock_irqsave(&report_lock, *flags);
+ if (timeout-- < 0) {
+ /*
+ * Abort. Reset @other_info->task to NULL, since it
+ * appears the other thread is still going to consume
+ * it. It will result in no verbose info printed for
+ * this task.
+ */
+ other_info->task = NULL;
+ break;
+ }
+ /*
+ * If invalid, or @ptr nor @current matches, then @other_info
+ * has been consumed and we may continue. If not, retry.
+ */
+ } while (other_info->ai.size && other_info->ai.ptr == ai->ptr &&
+ other_info->task == current);
+ if (is_running)
+ set_current_state(TASK_RUNNING);
+}
- switch (type) {
- case KCSAN_REPORT_CONSUMED_WATCHPOINT:
- if (other_info.ptr != NULL)
- break; /* still in use, retry */
+/* Populate @other_info; requires that the provided @other_info not in use. */
+static void prepare_report_producer(unsigned long *flags,
+ const struct access_info *ai,
+ struct other_info *other_info)
+{
+ raw_spin_lock_irqsave(&report_lock, *flags);
- other_info.ptr = ptr;
- other_info.size = size;
- other_info.access_type = access_type;
- other_info.task_pid = in_task() ? task_pid_nr(current) : -1;
- other_info.cpu_id = cpu_id;
- other_info.num_stack_entries = stack_trace_save(other_info.stack_entries, NUM_STACK_ENTRIES, 1);
+ /*
+ * The same @other_infos entry cannot be used concurrently, because
+ * there is a one-to-one mapping to watchpoint slots (@watchpoints in
+ * core.c), and a watchpoint is only released for reuse after reporting
+ * is done by the consumer of @other_info. Therefore, it is impossible
+ * for another concurrent prepare_report_producer() to set the same
+ * @other_info, and are guaranteed exclusivity for the @other_infos
+ * entry pointed to by @other_info.
+ *
+ * To check this property holds, size should never be non-zero here,
+ * because every consumer of struct other_info resets size to 0 in
+ * release_report().
+ */
+ WARN_ON(other_info->ai.size);
- spin_unlock_irqrestore(&report_lock, *flags);
+ other_info->ai = *ai;
+ other_info->num_stack_entries = stack_trace_save(other_info->stack_entries, NUM_STACK_ENTRIES, 2);
- /*
- * The other thread will print the summary; other_info may now
- * be consumed.
- */
- return false;
+ if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
+ set_other_info_task_blocking(flags, ai, other_info);
- case KCSAN_REPORT_RACE_SIGNAL:
- if (other_info.ptr == NULL)
- break; /* no data available yet, retry */
+ raw_spin_unlock_irqrestore(&report_lock, *flags);
+}
- /*
- * First check if this is the other_info we are expecting, i.e.
- * matches based on how watchpoint was encoded.
- */
- if (!matching_access((unsigned long)other_info.ptr &
- WATCHPOINT_ADDR_MASK,
- other_info.size,
- (unsigned long)ptr & WATCHPOINT_ADDR_MASK,
- size))
- break; /* mismatching watchpoint, retry */
-
- if (!matching_access((unsigned long)other_info.ptr,
- other_info.size, (unsigned long)ptr,
- size)) {
- /*
- * If the actual accesses to not match, this was a false
- * positive due to watchpoint encoding.
- */
- kcsan_counter_inc(
- KCSAN_COUNTER_ENCODING_FALSE_POSITIVES);
+/* Awaits producer to fill @other_info and then returns. */
+static bool prepare_report_consumer(unsigned long *flags,
+ const struct access_info *ai,
+ struct other_info *other_info)
+{
- /* discard this other_info */
- release_report(flags, KCSAN_REPORT_RACE_SIGNAL);
- return false;
- }
+ raw_spin_lock_irqsave(&report_lock, *flags);
+ while (!other_info->ai.size) { /* Await valid @other_info. */
+ raw_spin_unlock_irqrestore(&report_lock, *flags);
+ cpu_relax();
+ raw_spin_lock_irqsave(&report_lock, *flags);
+ }
- access_type |= other_info.access_type;
- if ((access_type & KCSAN_ACCESS_WRITE) == 0) {
- /*
- * While the address matches, this is not the other_info
- * from the thread that consumed our watchpoint, since
- * neither this nor the access in other_info is a write.
- * It is invalid to continue with the report, since we
- * only have information about reads.
- *
- * This can happen due to concurrent races on the same
- * address, with at least 4 threads. To avoid locking up
- * other_info and all other threads, we have to consume
- * it regardless.
- *
- * A concrete case to illustrate why we might lock up if
- * we do not consume other_info:
- *
- * We have 4 threads, all accessing the same address
- * (or matching address ranges). Assume the following
- * watcher and watchpoint consumer pairs:
- * write1-read1, read2-write2. The first to populate
- * other_info is write2, however, write1 consumes it,
- * resulting in a report of write1-write2. This report
- * is valid, however, now read1 populates other_info;
- * read2-read1 is an invalid conflict, yet, no other
- * conflicting access is left. Therefore, we must
- * consume read1's other_info.
- *
- * Since this case is assumed to be rare, it is
- * reasonable to omit this report: one of the other
- * reports includes information about the same shared
- * data, and at this point the likelihood that we
- * re-report the same race again is high.
- */
- release_report(flags, KCSAN_REPORT_RACE_SIGNAL);
- return false;
- }
+ /* Should always have a matching access based on watchpoint encoding. */
+ if (WARN_ON(!matching_access((unsigned long)other_info->ai.ptr & WATCHPOINT_ADDR_MASK, other_info->ai.size,
+ (unsigned long)ai->ptr & WATCHPOINT_ADDR_MASK, ai->size)))
+ goto discard;
+ if (!matching_access((unsigned long)other_info->ai.ptr, other_info->ai.size,
+ (unsigned long)ai->ptr, ai->size)) {
/*
- * Matching & usable access in other_info: keep other_info_lock
- * locked, as this thread consumes it to print the full report;
- * unlocked in release_report.
+ * If the actual accesses to not match, this was a false
+ * positive due to watchpoint encoding.
*/
- return true;
-
- default:
- BUG();
+ kcsan_counter_inc(KCSAN_COUNTER_ENCODING_FALSE_POSITIVES);
+ goto discard;
}
- spin_unlock_irqrestore(&report_lock, *flags);
+ return true;
- goto retry;
+discard:
+ release_report(flags, other_info);
+ return false;
+}
+
+/*
+ * Depending on the report type either sets @other_info and returns false, or
+ * awaits @other_info and returns true. If @other_info is not required for the
+ * report type, simply acquires @report_lock and returns true.
+ */
+static noinline bool prepare_report(unsigned long *flags,
+ enum kcsan_report_type type,
+ const struct access_info *ai,
+ struct other_info *other_info)
+{
+ switch (type) {
+ case KCSAN_REPORT_CONSUMED_WATCHPOINT:
+ prepare_report_producer(flags, ai, other_info);
+ return false;
+ case KCSAN_REPORT_RACE_SIGNAL:
+ return prepare_report_consumer(flags, ai, other_info);
+ default:
+ /* @other_info not required; just acquire @report_lock. */
+ raw_spin_lock_irqsave(&report_lock, *flags);
+ return true;
+ }
}
void kcsan_report(const volatile void *ptr, size_t size, int access_type,
- enum kcsan_value_change value_change, int cpu_id,
- enum kcsan_report_type type)
+ enum kcsan_value_change value_change,
+ enum kcsan_report_type type, int watchpoint_idx)
{
unsigned long flags = 0;
+ const struct access_info ai = {
+ .ptr = ptr,
+ .size = size,
+ .access_type = access_type,
+ .task_pid = in_task() ? task_pid_nr(current) : -1,
+ .cpu_id = raw_smp_processor_id()
+ };
+ struct other_info *other_info = type == KCSAN_REPORT_RACE_UNKNOWN_ORIGIN
+ ? NULL : &other_infos[watchpoint_idx];
+
+ kcsan_disable_current();
+ if (WARN_ON(watchpoint_idx < 0 || watchpoint_idx >= ARRAY_SIZE(other_infos)))
+ goto out;
/*
* With TRACE_IRQFLAGS, lockdep's IRQ trace state becomes corrupted if
@@ -494,22 +613,22 @@ void kcsan_report(const volatile void *ptr, size_t size, int access_type,
*/
lockdep_off();
- kcsan_disable_current();
- if (prepare_report(&flags, ptr, size, access_type, cpu_id, type)) {
+ if (prepare_report(&flags, type, &ai, other_info)) {
/*
* Never report if value_change is FALSE, only if we it is
* either TRUE or MAYBE. In case of MAYBE, further filtering may
* be done once we know the full stack trace in print_report().
*/
bool reported = value_change != KCSAN_VALUE_CHANGE_FALSE &&
- print_report(ptr, size, access_type, value_change, cpu_id, type);
+ print_report(value_change, type, &ai, other_info);
if (reported && panic_on_warn)
panic("panic_on_warn set ...\n");
- release_report(&flags, type);
+ release_report(&flags, other_info);
}
- kcsan_enable_current();
lockdep_on();
+out:
+ kcsan_enable_current();
}