diff options
| -rw-r--r-- | kernel/rcu/tree.c | 15 | ||||
| -rw-r--r-- | kernel/rcu/tree_plugin.h | 13 |
2 files changed, 24 insertions, 4 deletions
diff --git a/kernel/rcu/tree.c b/kernel/rcu/tree.c index fb6b80aa34f6..a6ddfae6978d 100644 --- a/kernel/rcu/tree.c +++ b/kernel/rcu/tree.c @@ -1335,6 +1335,19 @@ static bool rcu_advance_cbs(struct rcu_node *rnp, struct rcu_data *rdp) } /* + * Move and classify callbacks, but only if doing so won't require + * that the RCU grace-period kthread be awakened. + */ +static void __maybe_unused rcu_advance_cbs_nowake(struct rcu_node *rnp, + struct rcu_data *rdp) +{ + raw_lockdep_assert_held_rcu_node(rnp); + if (!rcu_seq_state(rcu_seq_current(&rnp->gp_seq))) + return; + WARN_ON_ONCE(rcu_advance_cbs(rnp, rdp)); +} + +/* * Update CPU-local rcu_data state to record the beginnings and ends of * grace periods. The caller must hold the ->lock of the leaf rcu_node * structure corresponding to the current CPU, and must have irqs disabled. @@ -2118,6 +2131,8 @@ static void rcu_do_batch(struct rcu_data *rdp) rcu_segcblist_n_lazy_cbs(&rdp->cblist), rcu_segcblist_n_cbs(&rdp->cblist), bl); rcu_segcblist_extract_done_cbs(&rdp->cblist, &rcl); + if (offloaded) + rdp->qlen_last_fqs_check = rcu_segcblist_n_cbs(&rdp->cblist); rcu_nocb_unlock_irqrestore(rdp, flags); /* Invoke callbacks. */ diff --git a/kernel/rcu/tree_plugin.h b/kernel/rcu/tree_plugin.h index 5cbc0848647c..c10afe778430 100644 --- a/kernel/rcu/tree_plugin.h +++ b/kernel/rcu/tree_plugin.h @@ -1643,10 +1643,15 @@ static void __call_rcu_nocb_wake(struct rcu_data *rdp, bool was_alldone, } else if (len > rdp->qlen_last_fqs_check + qhimark) { /* ... or if many callbacks queued. */ rdp->qlen_last_fqs_check = len; - if (!irqs_disabled_flags(flags)) { - wake_nocb_gp(rdp, true, flags); - trace_rcu_nocb_wake(rcu_state.name, rdp->cpu, - TPS("WakeOvf")); + if (!rdp->nocb_cb_sleep && + rcu_segcblist_ready_cbs(&rdp->cblist)) { + // Already going full tilt, so don't try to rewake. + rcu_nocb_unlock_irqrestore(rdp, flags); + } else if (rcu_segcblist_pend_cbs(&rdp->cblist) && + raw_spin_trylock_rcu_node(rdp->mynode)) { + rcu_advance_cbs_nowake(rdp->mynode, rdp); + raw_spin_unlock_rcu_node(rdp->mynode); + rcu_nocb_unlock_irqrestore(rdp, flags); } else { wake_nocb_gp_defer(rdp, RCU_NOCB_WAKE_FORCE, TPS("WakeOvfIsDeferred")); |