diff options
| author | Peter Zijlstra <peterz@infradead.org> | 2020-08-18 15:57:49 +0200 |
|---|---|---|
| committer | Ingo Molnar <mingo@kernel.org> | 2020-09-01 09:58:06 +0200 |
| commit | 5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997 (patch) | |
| tree | 533fcad5bdd98fd31ccd3956dec2e3478f15ca91 /arch/x86/kernel/static_call.c | |
| parent | 452cddbff74b6a15b9354505671011700fe03710 (diff) | |
static_call: Handle tail-calls
GCC can turn our static_call(name)(args...) into a tail call, in which
case we get a JMP.d32 into the trampoline (which then does a further
tail-call).
Teach objtool to recognise and mark these in .static_call_sites and
adjust the code patching to deal with this.
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Link: https://lore.kernel.org/r/20200818135805.101186767@infradead.org
Diffstat (limited to 'arch/x86/kernel/static_call.c')
| -rw-r--r-- | arch/x86/kernel/static_call.c | 21 |
1 files changed, 18 insertions, 3 deletions
diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index ead6726fb06d..60a325c731df 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -41,15 +41,30 @@ static void __static_call_transform(void *insn, enum insn_type type, void *func) text_poke_bp(insn, code, size, NULL); } -void arch_static_call_transform(void *site, void *tramp, void *func) +static inline enum insn_type __sc_insn(bool null, bool tail) +{ + /* + * Encode the following table without branches: + * + * tail null insn + * -----+-------+------ + * 0 | 0 | CALL + * 0 | 1 | NOP + * 1 | 0 | JMP + * 1 | 1 | RET + */ + return 2*tail + null; +} + +void arch_static_call_transform(void *site, void *tramp, void *func, bool tail) { mutex_lock(&text_mutex); if (tramp) - __static_call_transform(tramp, func ? JMP : RET, func); + __static_call_transform(tramp, __sc_insn(!func, true), func); if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) - __static_call_transform(site, func ? CALL : NOP, func); + __static_call_transform(site, __sc_insn(!func, tail), func); mutex_unlock(&text_mutex); } |