summaryrefslogtreecommitdiff
path: root/kernel/static_call.c
diff options
context:
space:
mode:
authorPeter Zijlstra <peterz@infradead.org>2020-08-18 15:57:49 +0200
committerIngo Molnar <mingo@kernel.org>2020-09-01 09:58:06 +0200
commit5b06fd3bb9cdce4f3e731c48eb5b74c4acc47997 (patch)
tree533fcad5bdd98fd31ccd3956dec2e3478f15ca91 /kernel/static_call.c
parent452cddbff74b6a15b9354505671011700fe03710 (diff)
static_call: Handle tail-calls
GCC can turn our static_call(name)(args...) into a tail call, in which case we get a JMP.d32 into the trampoline (which then does a further tail-call). Teach objtool to recognise and mark these in .static_call_sites and adjust the code patching to deal with this. Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org> Signed-off-by: Ingo Molnar <mingo@kernel.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Link: https://lore.kernel.org/r/20200818135805.101186767@infradead.org
Diffstat (limited to 'kernel/static_call.c')
-rw-r--r--kernel/static_call.c21
1 files changed, 13 insertions, 8 deletions
diff --git a/kernel/static_call.c b/kernel/static_call.c
index 97142cb6bfa6..d98e0e4272c1 100644
--- a/kernel/static_call.c
+++ b/kernel/static_call.c
@@ -15,8 +15,6 @@ extern struct static_call_site __start_static_call_sites[],
static bool static_call_initialized;
-#define STATIC_CALL_INIT 1UL
-
/* mutex to protect key modules/sites */
static DEFINE_MUTEX(static_call_mutex);
@@ -39,18 +37,23 @@ static inline void *static_call_addr(struct static_call_site *site)
static inline struct static_call_key *static_call_key(const struct static_call_site *site)
{
return (struct static_call_key *)
- (((long)site->key + (long)&site->key) & ~STATIC_CALL_INIT);
+ (((long)site->key + (long)&site->key) & ~STATIC_CALL_SITE_FLAGS);
}
/* These assume the key is word-aligned. */
static inline bool static_call_is_init(struct static_call_site *site)
{
- return ((long)site->key + (long)&site->key) & STATIC_CALL_INIT;
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_INIT;
+}
+
+static inline bool static_call_is_tail(struct static_call_site *site)
+{
+ return ((long)site->key + (long)&site->key) & STATIC_CALL_SITE_TAIL;
}
static inline void static_call_set_init(struct static_call_site *site)
{
- site->key = ((long)static_call_key(site) | STATIC_CALL_INIT) -
+ site->key = ((long)static_call_key(site) | STATIC_CALL_SITE_INIT) -
(long)&site->key;
}
@@ -104,7 +107,7 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
key->func = func;
- arch_static_call_transform(NULL, tramp, func);
+ arch_static_call_transform(NULL, tramp, func, false);
/*
* If uninitialized, we'll not update the callsites, but they still
@@ -154,7 +157,8 @@ void __static_call_update(struct static_call_key *key, void *tramp, void *func)
continue;
}
- arch_static_call_transform(site_addr, NULL, func);
+ arch_static_call_transform(site_addr, NULL, func,
+ static_call_is_tail(site));
}
}
@@ -198,7 +202,8 @@ static int __static_call_init(struct module *mod,
key->mods = site_mod;
}
- arch_static_call_transform(site_addr, NULL, key->func);
+ arch_static_call_transform(site_addr, NULL, key->func,
+ static_call_is_tail(site));
}
return 0;