From f105dfec0a951cd0d5bfbfe9dc067ea69f71ad5c Mon Sep 17 00:00:00 2001 From: Peter Zijlstra Date: Thu, 13 May 2021 01:29:15 +0200 Subject: tick/nohz: Evaluate the CPU expression after the static key When tick_nohz_full_cpu() is called with smp_processor_id(), the latter is unconditionally evaluated whether the static key is on or off. It is not necessary in the off-case though, so make sure the cpu expression is executed at the last moment. Illustrate with the following test function: int tick_nohz_test(void) { return tick_nohz_full_cpu(smp_processor_id()); } The resulting code before was: mov %gs:0x7eea92d1(%rip),%eax # smp_processor_id() fetch nopl 0x0(%rax,%rax,1) xor %eax,%eax retq cmpb $0x0,0x29d393a(%rip) # je tick_nohz_test+0x29 # jump to below eax clear mov %eax,%eax bt %rax,0x29d3936(%rip) # setb %al movzbl %al,%eax retq xor %eax,%eax retq Now it becomes: nopl 0x0(%rax,%rax,1) xor %eax,%eax retq cmpb $0x0,0x29d3871(%rip) # je tick_nohz_test+0x29 # jump to below eax clear mov %gs:0x7eea91f0(%rip),%eax # smp_processor_id() fetch, after static key mov %eax,%eax bt %rax,0x29d3866(%rip) # setb %al movzbl %al,%eax retq xor %eax,%eax retq Signed-off-by: Peter Zijlstra Signed-off-by: Frederic Weisbecker Signed-off-by: Ingo Molnar Link: https://lore.kernel.org/r/20210512232924.150322-2-frederic@kernel.org --- include/linux/tick.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/include/linux/tick.h b/include/linux/tick.h index 7340613c7eff..2258984a0e8a 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -185,13 +185,17 @@ static inline bool tick_nohz_full_enabled(void) return tick_nohz_full_running; } -static inline bool tick_nohz_full_cpu(int cpu) -{ - if (!tick_nohz_full_enabled()) - return false; - - return cpumask_test_cpu(cpu, tick_nohz_full_mask); -} +/* + * Check if a CPU is part of the nohz_full subset. Arrange for evaluating + * the cpu expression (typically smp_processor_id()) _after_ the static + * key. + */ +#define tick_nohz_full_cpu(_cpu) ({ \ + bool __ret = false; \ + if (tick_nohz_full_enabled()) \ + __ret = cpumask_test_cpu((_cpu), tick_nohz_full_mask); \ + __ret; \ +}) static inline void tick_nohz_full_add_cpus_to(struct cpumask *mask) { -- cgit From 1e4ca26d367ae71743e25068e5cd8750ef3f5f7d Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 13 May 2021 01:29:21 +0200 Subject: tick/nohz: Change signal tick dependency to wake up CPUs of member tasks Rather than waking up all nohz_full CPUs on the system, only wake up the target CPUs of member threads of the signal. Reduces interruptions to nohz_full CPUs. Signed-off-by: Marcelo Tosatti Signed-off-by: Frederic Weisbecker Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20210512232924.150322-8-frederic@kernel.org --- include/linux/tick.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/include/linux/tick.h b/include/linux/tick.h index 2258984a0e8a..0bb80a7f05b9 100644 --- a/include/linux/tick.h +++ b/include/linux/tick.h @@ -211,7 +211,7 @@ extern void tick_nohz_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit); extern void tick_nohz_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit); -extern void tick_nohz_dep_set_signal(struct signal_struct *signal, +extern void tick_nohz_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit); extern void tick_nohz_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit); @@ -256,11 +256,11 @@ static inline void tick_dep_clear_task(struct task_struct *tsk, if (tick_nohz_full_enabled()) tick_nohz_dep_clear_task(tsk, bit); } -static inline void tick_dep_set_signal(struct signal_struct *signal, +static inline void tick_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit) { if (tick_nohz_full_enabled()) - tick_nohz_dep_set_signal(signal, bit); + tick_nohz_dep_set_signal(tsk, bit); } static inline void tick_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit) @@ -288,7 +288,7 @@ static inline void tick_dep_set_task(struct task_struct *tsk, enum tick_dep_bits bit) { } static inline void tick_dep_clear_task(struct task_struct *tsk, enum tick_dep_bits bit) { } -static inline void tick_dep_set_signal(struct signal_struct *signal, +static inline void tick_dep_set_signal(struct task_struct *tsk, enum tick_dep_bits bit) { } static inline void tick_dep_clear_signal(struct signal_struct *signal, enum tick_dep_bits bit) { } -- cgit From a1dfb6311c7739e21e160bc4c5575a1b21b48c87 Mon Sep 17 00:00:00 2001 From: Marcelo Tosatti Date: Thu, 13 May 2021 01:29:22 +0200 Subject: tick/nohz: Kick only _queued_ task whose tick dependency is updated When the tick dependency of a task is updated, we want it to aknowledge the new state and restart the tick if needed. If the task is not running, we don't need to kick it because it will observe the new dependency upon scheduling in. But if the task is running, we may need to send an IPI to it so that it gets notified. Unfortunately we don't have the means to check if a task is running in a race free way. Checking p->on_cpu in a synchronized way against p->tick_dep_mask would imply adding a full barrier between prepare_task_switch() and tick_nohz_task_switch(), which we want to avoid in this fast-path. Therefore we blindly fire an IPI to the task's CPU. Meanwhile we can check if the task is queued on the CPU rq because p->on_rq is always set to TASK_ON_RQ_QUEUED _before_ schedule() and its full barrier that precedes tick_nohz_task_switch(). And if the task is queued on a nohz_full CPU, it also has fair chances to be running as the isolation constraints prescribe running single tasks on full dynticks CPUs. So use this as a trick to check if we can spare an IPI toward a non-running task. NOTE: For the ordering to be correct, it is assumed that we never deactivate a task while it is running, the only exception being the task deactivating itself while scheduling out. Suggested-by: Peter Zijlstra Signed-off-by: Marcelo Tosatti Signed-off-by: Frederic Weisbecker Signed-off-by: Ingo Molnar Acked-by: Peter Zijlstra Link: https://lore.kernel.org/r/20210512232924.150322-9-frederic@kernel.org --- include/linux/sched.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'include') diff --git a/include/linux/sched.h b/include/linux/sched.h index d2c881384517..3341ae2e8231 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2011,6 +2011,8 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) #endif /* CONFIG_SMP */ +extern bool sched_task_on_rq(struct task_struct *p); + /* * In order to reduce various lock holder preemption latencies provide an * interface to see if a vCPU is currently running or not. -- cgit