diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2022-09-15 13:11:31 +0200 |
|---|---|---|
| committer | Peter Zijlstra <peterz@infradead.org> | 2022-10-17 16:41:16 +0200 |
| commit | 7825451fa4dc04660f1f53d236e4302161d0ebd1 (patch) | |
| tree | 60fb5490d9e73fc9b505e941af7a2259a879318d /arch | |
| parent | f5c1bb2afe93396d41c5cbdcb909b08a75b8dde4 (diff) | |
static_call: Add call depth tracking support
When indirect calls are switched to direct calls then it has to be ensured
that the call target is not the function, but the call thunk when call
depth tracking is enabled. But static calls are available before call
thunks have been set up.
Ensure a second run through the static call patching code after call thunks
have been created. When call thunks are not enabled this has no side
effects.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20220915111148.306100465@infradead.org
Diffstat (limited to 'arch')
| -rw-r--r-- | arch/x86/include/asm/alternative.h | 5 | ||||
| -rw-r--r-- | arch/x86/kernel/callthunks.c | 18 | ||||
| -rw-r--r-- | arch/x86/kernel/static_call.c | 1 |
3 files changed, 24 insertions, 0 deletions
diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index 4c416b21bac8..07ac25793a3f 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -91,11 +91,16 @@ struct callthunk_sites { extern void callthunks_patch_builtin_calls(void); extern void callthunks_patch_module_calls(struct callthunk_sites *sites, struct module *mod); +extern void *callthunks_translate_call_dest(void *dest); #else static __always_inline void callthunks_patch_builtin_calls(void) {} static __always_inline void callthunks_patch_module_calls(struct callthunk_sites *sites, struct module *mod) {} +static __always_inline void *callthunks_translate_call_dest(void *dest) +{ + return dest; +} #endif #ifdef CONFIG_SMP diff --git a/arch/x86/kernel/callthunks.c b/arch/x86/kernel/callthunks.c index dfe7ffff88b9..071003605a86 100644 --- a/arch/x86/kernel/callthunks.c +++ b/arch/x86/kernel/callthunks.c @@ -6,6 +6,7 @@ #include <linux/kallsyms.h> #include <linux/memory.h> #include <linux/moduleloader.h> +#include <linux/static_call.h> #include <asm/alternative.h> #include <asm/asm-offsets.h> @@ -271,10 +272,27 @@ void __init callthunks_patch_builtin_calls(void) pr_info("Setting up call depth tracking\n"); mutex_lock(&text_mutex); callthunks_setup(&cs, &builtin_coretext); + static_call_force_reinit(); thunks_initialized = true; mutex_unlock(&text_mutex); } +void *callthunks_translate_call_dest(void *dest) +{ + void *target; + + lockdep_assert_held(&text_mutex); + + if (!thunks_initialized || skip_addr(dest)) + return dest; + + if (!is_coretext(NULL, dest)) + return dest; + + target = patch_dest(dest, false); + return target ? : dest; +} + #ifdef CONFIG_MODULES void noinline callthunks_patch_module_calls(struct callthunk_sites *cs, struct module *mod) diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 5d3844a98373..2ebc338980bc 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -34,6 +34,7 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, switch (type) { case CALL: + func = callthunks_translate_call_dest(func); code = text_gen_insn(CALL_INSN_OPCODE, insn, func); if (func == &__static_call_return0) { emulate = code; |