diff options
Diffstat (limited to 'include/trace/events/rcu.h')
| -rw-r--r-- | include/trace/events/rcu.h | 498 |
1 files changed, 255 insertions, 243 deletions
diff --git a/include/trace/events/rcu.h b/include/trace/events/rcu.h index 91dc089d65b7..5fbdabe3faea 100644 --- a/include/trace/events/rcu.h +++ b/include/trace/events/rcu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #undef TRACE_SYSTEM #define TRACE_SYSTEM rcu @@ -6,6 +7,12 @@ #include <linux/tracepoint.h> +#ifdef CONFIG_RCU_TRACE +#define TRACE_EVENT_RCU TRACE_EVENT +#else +#define TRACE_EVENT_RCU TRACE_EVENT_NOP +#endif + /* * Tracepoint for start/end markers used for utilization calculations. * By convention, the string is of the following forms: @@ -34,16 +41,14 @@ TRACE_EVENT(rcu_utilization, TP_printk("%s", __entry->s) ); -#ifdef CONFIG_RCU_TRACE - -#if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) +#if defined(CONFIG_TREE_RCU) /* * Tracepoint for grace-period events. Takes a string identifying the * RCU flavor, the grace-period number, and a string identifying the * grace-period-related event as follows: * - * "AccReadyCB": CPU acclerates new callbacks to RCU_NEXT_READY_TAIL. + * "AccReadyCB": CPU accelerates new callbacks to RCU_NEXT_READY_TAIL. * "AccWaitCB": CPU accelerates new callbacks to RCU_WAIT_TAIL. * "newreq": Request a new grace period. * "start": Start a grace period. @@ -51,6 +56,7 @@ TRACE_EVENT(rcu_utilization, * "cpuqs": CPU passes through a quiescent state. * "cpuonl": CPU comes online. * "cpuofl": CPU goes offline. + * "cpuofl-bgp": CPU goes offline while blocking a grace period. * "reqwait": GP kthread sleeps waiting for grace-period request. * "reqwaitsig": GP kthread awakened by signal from reqwait state. * "fqswait": GP kthread waiting until time to force quiescent states. @@ -60,57 +66,56 @@ TRACE_EVENT(rcu_utilization, * "end": End a grace period. * "cpuend": CPU first notices a grace-period end. */ -TRACE_EVENT(rcu_grace_period, +TRACE_EVENT_RCU(rcu_grace_period, - TP_PROTO(const char *rcuname, unsigned long gpnum, const char *gpevent), + TP_PROTO(const char *rcuname, unsigned long gp_seq, const char *gpevent), - TP_ARGS(rcuname, gpnum, gpevent), + TP_ARGS(rcuname, gp_seq, gpevent), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(const char *, gpevent) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->gpevent = gpevent; ), - TP_printk("%s %lu %s", - __entry->rcuname, __entry->gpnum, __entry->gpevent) + TP_printk("%s %ld %s", + __entry->rcuname, __entry->gp_seq, __entry->gpevent) ); /* - * Tracepoint for future grace-period events, including those for no-callbacks - * CPUs. The caller should pull the data from the rcu_node structure, - * other than rcuname, which comes from the rcu_state structure, and event, - * which is one of the following: + * Tracepoint for future grace-period events. The caller should pull + * the data from the rcu_node structure, other than rcuname, which comes + * from the rcu_state structure, and event, which is one of the following: * - * "Startleaf": Request a nocb grace period based on leaf-node data. - * "Startedleaf": Leaf-node start proved sufficient. - * "Startedleafroot": Leaf-node start proved sufficient after checking root. + * "Cleanup": Clean up rcu_node structure after previous GP. + * "CleanupMore": Clean up, and another GP is needed. + * "EndWait": Complete wait. + * "NoGPkthread": The RCU grace-period kthread has not yet started. + * "Prestarted": Someone beat us to the request + * "Startedleaf": Leaf node marked for future GP. + * "Startedleafroot": All nodes from leaf to root marked for future GP. * "Startedroot": Requested a nocb grace period based on root-node data. + * "Startleaf": Request a grace period based on leaf-node data. * "StartWait": Start waiting for the requested grace period. - * "ResumeWait": Resume waiting after signal. - * "EndWait": Complete wait. - * "Cleanup": Clean up rcu_node structure after previous GP. - * "CleanupMore": Clean up, and another no-CB GP is needed. */ -TRACE_EVENT(rcu_future_grace_period, +TRACE_EVENT_RCU(rcu_future_grace_period, - TP_PROTO(const char *rcuname, unsigned long gpnum, unsigned long completed, - unsigned long c, u8 level, int grplo, int grphi, + TP_PROTO(const char *rcuname, unsigned long gp_seq, + unsigned long gp_seq_req, u8 level, int grplo, int grphi, const char *gpevent), - TP_ARGS(rcuname, gpnum, completed, c, level, grplo, grphi, gpevent), + TP_ARGS(rcuname, gp_seq, gp_seq_req, level, grplo, grphi, gpevent), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) - __field(unsigned long, completed) - __field(unsigned long, c) + __field(long, gp_seq) + __field(long, gp_seq_req) __field(u8, level) __field(int, grplo) __field(int, grphi) @@ -119,19 +124,17 @@ TRACE_EVENT(rcu_future_grace_period, TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; - __entry->completed = completed; - __entry->c = c; + __entry->gp_seq = (long)gp_seq; + __entry->gp_seq_req = (long)gp_seq_req; __entry->level = level; __entry->grplo = grplo; __entry->grphi = grphi; __entry->gpevent = gpevent; ), - TP_printk("%s %lu %lu %lu %u %d %d %s", - __entry->rcuname, __entry->gpnum, __entry->completed, - __entry->c, __entry->level, __entry->grplo, __entry->grphi, - __entry->gpevent) + TP_printk("%s %ld %ld %u %d %d %s", + __entry->rcuname, (long)__entry->gp_seq, (long)__entry->gp_seq_req, __entry->level, + __entry->grplo, __entry->grphi, __entry->gpevent) ); /* @@ -141,16 +144,16 @@ TRACE_EVENT(rcu_future_grace_period, * rcu_node structure, and the mask of CPUs that will be waited for. * All but the type of RCU are extracted from the rcu_node structure. */ -TRACE_EVENT(rcu_grace_period_init, +TRACE_EVENT_RCU(rcu_grace_period_init, - TP_PROTO(const char *rcuname, unsigned long gpnum, u8 level, + TP_PROTO(const char *rcuname, unsigned long gp_seq, u8 level, int grplo, int grphi, unsigned long qsmask), - TP_ARGS(rcuname, gpnum, level, grplo, grphi, qsmask), + TP_ARGS(rcuname, gp_seq, level, grplo, grphi, qsmask), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(u8, level) __field(int, grplo) __field(int, grphi) @@ -159,15 +162,15 @@ TRACE_EVENT(rcu_grace_period_init, TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->level = level; __entry->grplo = grplo; __entry->grphi = grphi; __entry->qsmask = qsmask; ), - TP_printk("%s %lu %u %d %d %lx", - __entry->rcuname, __entry->gpnum, __entry->level, + TP_printk("%s %ld %u %d %d %lx", + __entry->rcuname, __entry->gp_seq, __entry->level, __entry->grplo, __entry->grphi, __entry->qsmask) ); @@ -178,11 +181,15 @@ TRACE_EVENT(rcu_grace_period_init, * * "snap": Captured snapshot of expedited grace period sequence number. * "start": Started a real expedited grace period. + * "reset": Started resetting the tree + * "select": Started selecting the CPUs to wait on. + * "selectofl": Selected CPU partially offline. + * "startwait": Started waiting on selected CPUs. * "end": Ended a real expedited grace period. * "endwake": Woke piggybackers up. * "done": Someone else did the expedited grace period for us. */ -TRACE_EVENT(rcu_exp_grace_period, +TRACE_EVENT_RCU(rcu_exp_grace_period, TP_PROTO(const char *rcuname, unsigned long gpseq, const char *gpevent), @@ -190,17 +197,17 @@ TRACE_EVENT(rcu_exp_grace_period, TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpseq) + __field(long, gpseq) __field(const char *, gpevent) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpseq = gpseq; + __entry->gpseq = (long)gpseq; __entry->gpevent = gpevent; ), - TP_printk("%s %lu %s", + TP_printk("%s %#lx %s", __entry->rcuname, __entry->gpseq, __entry->gpevent) ); @@ -214,7 +221,7 @@ TRACE_EVENT(rcu_exp_grace_period, * "nxtlvl": Advance to next level of rcu_node funnel * "wait": Wait for someone else to do expedited GP */ -TRACE_EVENT(rcu_exp_funnel_lock, +TRACE_EVENT_RCU(rcu_exp_funnel_lock, TP_PROTO(const char *rcuname, u8 level, int grplo, int grphi, const char *gpevent), @@ -242,6 +249,7 @@ TRACE_EVENT(rcu_exp_funnel_lock, __entry->grphi, __entry->gpevent) ); +#ifdef CONFIG_RCU_NOCB_CPU /* * Tracepoint for RCU no-CBs CPU callback handoffs. This event is intended * to assist debugging of these handoffs. @@ -250,21 +258,30 @@ TRACE_EVENT(rcu_exp_funnel_lock, * the number of the offloaded CPU are extracted. The third and final * argument is a string as follows: * - * "WakeEmpty": Wake rcuo kthread, first CB to empty list. - * "WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list. - * "WakeOvf": Wake rcuo kthread, CB list is huge. - * "WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge. - * "WakeNot": Don't wake rcuo kthread. - * "WakeNotPoll": Don't wake rcuo kthread because it is polling. - * "DeferredWake": Carried out the "IsDeferred" wakeup. - * "Poll": Start of new polling cycle for rcu_nocb_poll. - * "Sleep": Sleep waiting for CBs for !rcu_nocb_poll. - * "WokeEmpty": rcuo kthread woke to find empty list. - * "WokeNonEmpty": rcuo kthread woke to find non-empty list. - * "WaitQueue": Enqueue partially done, timed wait for it to complete. - * "WokeQueue": Partial enqueue now complete. + * "AlreadyAwake": The to-be-awakened rcuo kthread is already awake. + * "Bypass": rcuo GP kthread sees non-empty ->nocb_bypass. + * "CBSleep": rcuo CB kthread sleeping waiting for CBs. + * "Check": rcuo GP kthread checking specified CPU for work. + * "DeferredWake": Timer expired or polled check, time to wake. + * "DoWake": The to-be-awakened rcuo kthread needs to be awakened. + * "EndSleep": Done waiting for GP for !rcu_nocb_poll. + * "FirstBQ": New CB to empty ->nocb_bypass (->cblist maybe non-empty). + * "FirstBQnoWake": FirstBQ plus rcuo kthread need not be awakened. + * "FirstBQwake": FirstBQ plus rcuo kthread must be awakened. + * "FirstQ": New CB to empty ->cblist (->nocb_bypass maybe non-empty). + * "NeedWaitGP": rcuo GP kthread must wait on a grace period. + * "Poll": Start of new polling cycle for rcu_nocb_poll. + * "Sleep": Sleep waiting for GP for !rcu_nocb_poll. + * "Timer": Deferred-wake timer expired. + * "WakeEmptyIsDeferred": Wake rcuo kthread later, first CB to empty list. + * "WakeEmpty": Wake rcuo kthread, first CB to empty list. + * "WakeNot": Don't wake rcuo kthread. + * "WakeNotPoll": Don't wake rcuo kthread because it is polling. + * "WakeOvfIsDeferred": Wake rcuo kthread later, CB list is huge. + * "WakeBypassIsDeferred": Wake rcuo kthread later, bypass list is contended. + * "WokeEmpty": rcuo CB kthread woke to find empty list. */ -TRACE_EVENT(rcu_nocb_wake, +TRACE_EVENT_RCU(rcu_nocb_wake, TP_PROTO(const char *rcuname, int cpu, const char *reason), @@ -284,6 +301,7 @@ TRACE_EVENT(rcu_nocb_wake, TP_printk("%s %d %s", __entry->rcuname, __entry->cpu, __entry->reason) ); +#endif /* * Tracepoint for tasks blocking within preemptible-RCU read-side @@ -291,26 +309,26 @@ TRACE_EVENT(rcu_nocb_wake, * include SRCU), the grace-period number that the task is blocking * (the current or the next), and the task's PID. */ -TRACE_EVENT(rcu_preempt_task, +TRACE_EVENT_RCU(rcu_preempt_task, - TP_PROTO(const char *rcuname, int pid, unsigned long gpnum), + TP_PROTO(const char *rcuname, int pid, unsigned long gp_seq), - TP_ARGS(rcuname, pid, gpnum), + TP_ARGS(rcuname, pid, gp_seq), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(int, pid) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->pid = pid; ), - TP_printk("%s %lu %d", - __entry->rcuname, __entry->gpnum, __entry->pid) + TP_printk("%s %ld %d", + __entry->rcuname, __entry->gp_seq, __entry->pid) ); /* @@ -318,25 +336,25 @@ TRACE_EVENT(rcu_preempt_task, * read-side critical section exiting that critical section. Track the * type of RCU (which one day might include SRCU) and the task's PID. */ -TRACE_EVENT(rcu_unlock_preempted_task, +TRACE_EVENT_RCU(rcu_unlock_preempted_task, - TP_PROTO(const char *rcuname, unsigned long gpnum, int pid), + TP_PROTO(const char *rcuname, unsigned long gp_seq, int pid), - TP_ARGS(rcuname, gpnum, pid), + TP_ARGS(rcuname, gp_seq, pid), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(int, pid) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->pid = pid; ), - TP_printk("%s %lu %d", __entry->rcuname, __entry->gpnum, __entry->pid) + TP_printk("%s %ld %d", __entry->rcuname, __entry->gp_seq, __entry->pid) ); /* @@ -347,17 +365,17 @@ TRACE_EVENT(rcu_unlock_preempted_task, * whether there are any blocked tasks blocking the current grace period. * All but the type of RCU are extracted from the rcu_node structure. */ -TRACE_EVENT(rcu_quiescent_state_report, +TRACE_EVENT_RCU(rcu_quiescent_state_report, - TP_PROTO(const char *rcuname, unsigned long gpnum, + TP_PROTO(const char *rcuname, unsigned long gp_seq, unsigned long mask, unsigned long qsmask, u8 level, int grplo, int grphi, int gp_tasks), - TP_ARGS(rcuname, gpnum, mask, qsmask, level, grplo, grphi, gp_tasks), + TP_ARGS(rcuname, gp_seq, mask, qsmask, level, grplo, grphi, gp_tasks), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(unsigned long, mask) __field(unsigned long, qsmask) __field(u8, level) @@ -368,7 +386,7 @@ TRACE_EVENT(rcu_quiescent_state_report, TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->mask = mask; __entry->qsmask = qsmask; __entry->level = level; @@ -377,119 +395,111 @@ TRACE_EVENT(rcu_quiescent_state_report, __entry->gp_tasks = gp_tasks; ), - TP_printk("%s %lu %lx>%lx %u %d %d %u", - __entry->rcuname, __entry->gpnum, + TP_printk("%s %ld %lx>%lx %u %d %d %u", + __entry->rcuname, __entry->gp_seq, __entry->mask, __entry->qsmask, __entry->level, __entry->grplo, __entry->grphi, __entry->gp_tasks) ); /* * Tracepoint for quiescent states detected by force_quiescent_state(). - * These trace events include the type of RCU, the grace-period number that - * was blocked by the CPU, the CPU itself, and the type of quiescent state, - * which can be "dti" for dyntick-idle mode, "ofl" for CPU offline, "kick" - * when kicking a CPU that has been in dyntick-idle mode for too long, or - * "rqc" if the CPU got a quiescent state via its rcu_qs_ctr. + * These trace events include the type of RCU, the grace-period number + * that was blocked by the CPU, the CPU itself, and the type of quiescent + * state, which can be "dti" for dyntick-idle mode or "kick" when kicking + * a CPU that has been in dyntick-idle mode for too long. */ -TRACE_EVENT(rcu_fqs, +TRACE_EVENT_RCU(rcu_fqs, - TP_PROTO(const char *rcuname, unsigned long gpnum, int cpu, const char *qsevent), + TP_PROTO(const char *rcuname, unsigned long gp_seq, int cpu, const char *qsevent), - TP_ARGS(rcuname, gpnum, cpu, qsevent), + TP_ARGS(rcuname, gp_seq, cpu, qsevent), TP_STRUCT__entry( __field(const char *, rcuname) - __field(unsigned long, gpnum) + __field(long, gp_seq) __field(int, cpu) __field(const char *, qsevent) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->gpnum = gpnum; + __entry->gp_seq = (long)gp_seq; __entry->cpu = cpu; __entry->qsevent = qsevent; ), - TP_printk("%s %lu %d %s", - __entry->rcuname, __entry->gpnum, + TP_printk("%s %ld %d %s", + __entry->rcuname, __entry->gp_seq, __entry->cpu, __entry->qsevent) ); -#endif /* #if defined(CONFIG_TREE_RCU) || defined(CONFIG_PREEMPT_RCU) */ - /* - * Tracepoint for dyntick-idle entry/exit events. These take a string - * as argument: "Start" for entering dyntick-idle mode, "End" for - * leaving it, "--=" for events moving towards idle, and "++=" for events - * moving away from idle. "Error on entry: not idle task" and "Error on - * exit: not idle task" indicate that a non-idle task is erroneously - * toying with the idle loop. + * Tracepoint for RCU stall events. Takes a string identifying the RCU flavor + * and a string identifying which function detected the RCU stall as follows: * - * These events also take a pair of numbers, which indicate the nesting - * depth before and after the event of interest. Note that task-related - * events use the upper bits of each number, while interrupt-related - * events use the lower bits. + * "StallDetected": Scheduler-tick detects other CPU's stalls. + * "SelfDetected": Scheduler-tick detects a current CPU's stall. + * "ExpeditedStall": Expedited grace period detects stalls. */ -TRACE_EVENT(rcu_dyntick, +TRACE_EVENT(rcu_stall_warning, - TP_PROTO(const char *polarity, long long oldnesting, long long newnesting), + TP_PROTO(const char *rcuname, const char *msg), - TP_ARGS(polarity, oldnesting, newnesting), + TP_ARGS(rcuname, msg), TP_STRUCT__entry( - __field(const char *, polarity) - __field(long long, oldnesting) - __field(long long, newnesting) + __field(const char *, rcuname) + __field(const char *, msg) ), TP_fast_assign( - __entry->polarity = polarity; - __entry->oldnesting = oldnesting; - __entry->newnesting = newnesting; + __entry->rcuname = rcuname; + __entry->msg = msg; ), - TP_printk("%s %llx %llx", __entry->polarity, - __entry->oldnesting, __entry->newnesting) + TP_printk("%s %s", + __entry->rcuname, __entry->msg) ); +#endif /* #if defined(CONFIG_TREE_RCU) */ + /* - * Tracepoint for RCU preparation for idle, the goal being to get RCU - * processing done so that the current CPU can shut off its scheduling - * clock and enter dyntick-idle mode. One way to accomplish this is - * to drain all RCU callbacks from this CPU, and the other is to have - * done everything RCU requires for the current grace period. In this - * latter case, the CPU will be awakened at the end of the current grace - * period in order to process the remainder of its callbacks. - * - * These tracepoints take a string as argument: + * Tracepoint for dyntick-idle entry/exit events. These take 2 strings + * as argument: + * polarity: "Start", "End", "StillWatching" for entering, exiting or still not + * being in EQS mode. + * context: "USER" or "IDLE" or "IRQ". + * NMIs nested in IRQs are inferred with nesting > 1 in IRQ context. * - * "No callbacks": Nothing to do, no callbacks on this CPU. - * "In holdoff": Nothing to do, holding off after unsuccessful attempt. - * "Begin holdoff": Attempt failed, don't retry until next jiffy. - * "Dyntick with callbacks": Entering dyntick-idle despite callbacks. - * "Dyntick with lazy callbacks": Entering dyntick-idle w/lazy callbacks. - * "More callbacks": Still more callbacks, try again to clear them out. - * "Callbacks drained": All callbacks processed, off to dyntick idle! - * "Timer": Timer fired to cause CPU to continue processing callbacks. - * "Demigrate": Timer fired on wrong CPU, woke up correct CPU. - * "Cleanup after idle": Idle exited, timer canceled. + * These events also take a pair of numbers, which indicate the nesting + * depth before and after the event of interest, and a third number that is + * the RCU_WATCHING counter. Note that task-related and interrupt-related + * events use two separate counters, and that the "++=" and "--=" events + * for irq/NMI will change the counter by two, otherwise by one. */ -TRACE_EVENT(rcu_prep_idle, +TRACE_EVENT_RCU(rcu_watching, - TP_PROTO(const char *reason), + TP_PROTO(const char *polarity, long oldnesting, long newnesting, int counter), - TP_ARGS(reason), + TP_ARGS(polarity, oldnesting, newnesting, counter), TP_STRUCT__entry( - __field(const char *, reason) + __field(const char *, polarity) + __field(long, oldnesting) + __field(long, newnesting) + __field(int, counter) ), TP_fast_assign( - __entry->reason = reason; + __entry->polarity = polarity; + __entry->oldnesting = oldnesting; + __entry->newnesting = newnesting; + __entry->counter = counter; ), - TP_printk("%s", __entry->reason) + TP_printk("%s %lx %lx %#3x", __entry->polarity, + __entry->oldnesting, __entry->newnesting, + __entry->counter & 0xfff) ); /* @@ -499,18 +509,16 @@ TRACE_EVENT(rcu_prep_idle, * number of lazy callbacks queued, and the fourth element is the * total number of callbacks queued. */ -TRACE_EVENT(rcu_callback, +TRACE_EVENT_RCU(rcu_callback, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen_lazy, - long qlen), + TP_PROTO(const char *rcuname, struct rcu_head *rhp, long qlen), - TP_ARGS(rcuname, rhp, qlen_lazy, qlen), + TP_ARGS(rcuname, rhp, qlen), TP_STRUCT__entry( __field(const char *, rcuname) __field(void *, rhp) __field(void *, func) - __field(long, qlen_lazy) __field(long, qlen) ), @@ -518,49 +526,38 @@ TRACE_EVENT(rcu_callback, __entry->rcuname = rcuname; __entry->rhp = rhp; __entry->func = rhp->func; - __entry->qlen_lazy = qlen_lazy; __entry->qlen = qlen; ), - TP_printk("%s rhp=%p func=%pf %ld/%ld", + TP_printk("%s rhp=%p func=%ps %ld", __entry->rcuname, __entry->rhp, __entry->func, - __entry->qlen_lazy, __entry->qlen) + __entry->qlen) ); -/* - * Tracepoint for the registration of a single RCU callback of the special - * kfree() form. The first argument is the RCU type, the second argument - * is a pointer to the RCU callback, the third argument is the offset - * of the callback within the enclosing RCU-protected data structure, - * the fourth argument is the number of lazy callbacks queued, and the - * fifth argument is the total number of callbacks queued. - */ -TRACE_EVENT(rcu_kfree_callback, +TRACE_EVENT_RCU(rcu_segcb_stats, - TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset, - long qlen_lazy, long qlen), + TP_PROTO(struct rcu_segcblist *rs, const char *ctx), - TP_ARGS(rcuname, rhp, offset, qlen_lazy, qlen), + TP_ARGS(rs, ctx), - TP_STRUCT__entry( - __field(const char *, rcuname) - __field(void *, rhp) - __field(unsigned long, offset) - __field(long, qlen_lazy) - __field(long, qlen) - ), + TP_STRUCT__entry( + __field(const char *, ctx) + __array(unsigned long, gp_seq, RCU_CBLIST_NSEGS) + __array(long, seglen, RCU_CBLIST_NSEGS) + ), - TP_fast_assign( - __entry->rcuname = rcuname; - __entry->rhp = rhp; - __entry->offset = offset; - __entry->qlen_lazy = qlen_lazy; - __entry->qlen = qlen; - ), + TP_fast_assign( + __entry->ctx = ctx; + memcpy(__entry->seglen, rs->seglen, RCU_CBLIST_NSEGS * sizeof(long)); + memcpy(__entry->gp_seq, rs->gp_seq, RCU_CBLIST_NSEGS * sizeof(unsigned long)); + + ), + + TP_printk("%s seglen: (DONE=%ld, WAIT=%ld, NEXT_READY=%ld, NEXT=%ld) " + "gp_seq: (DONE=%lu, WAIT=%lu, NEXT_READY=%lu, NEXT=%lu)", __entry->ctx, + __entry->seglen[0], __entry->seglen[1], __entry->seglen[2], __entry->seglen[3], + __entry->gp_seq[0], __entry->gp_seq[1], __entry->gp_seq[2], __entry->gp_seq[3]) - TP_printk("%s rhp=%p func=%ld %ld/%ld", - __entry->rcuname, __entry->rhp, __entry->offset, - __entry->qlen_lazy, __entry->qlen) ); /* @@ -570,29 +567,26 @@ TRACE_EVENT(rcu_kfree_callback, * the total number of callbacks queued, and the fourth argument is * the current RCU-callback batch limit. */ -TRACE_EVENT(rcu_batch_start, +TRACE_EVENT_RCU(rcu_batch_start, - TP_PROTO(const char *rcuname, long qlen_lazy, long qlen, long blimit), + TP_PROTO(const char *rcuname, long qlen, long blimit), - TP_ARGS(rcuname, qlen_lazy, qlen, blimit), + TP_ARGS(rcuname, qlen, blimit), TP_STRUCT__entry( __field(const char *, rcuname) - __field(long, qlen_lazy) __field(long, qlen) __field(long, blimit) ), TP_fast_assign( __entry->rcuname = rcuname; - __entry->qlen_lazy = qlen_lazy; __entry->qlen = qlen; __entry->blimit = blimit; ), - TP_printk("%s CBs=%ld/%ld bl=%ld", - __entry->rcuname, __entry->qlen_lazy, __entry->qlen, - __entry->blimit) + TP_printk("%s CBs=%ld bl=%ld", + __entry->rcuname, __entry->qlen, __entry->blimit) ); /* @@ -600,7 +594,7 @@ TRACE_EVENT(rcu_batch_start, * The first argument is the type of RCU, and the second argument is * a pointer to the RCU callback itself. */ -TRACE_EVENT(rcu_invoke_callback, +TRACE_EVENT_RCU(rcu_invoke_callback, TP_PROTO(const char *rcuname, struct rcu_head *rhp), @@ -618,18 +612,18 @@ TRACE_EVENT(rcu_invoke_callback, __entry->func = rhp->func; ), - TP_printk("%s rhp=%p func=%pf", + TP_printk("%s rhp=%p func=%ps", __entry->rcuname, __entry->rhp, __entry->func) ); /* * Tracepoint for the invocation of a single RCU callback of the special - * kfree() form. The first argument is the RCU flavor, the second + * kvfree() form. The first argument is the RCU flavor, the second * argument is a pointer to the RCU callback, and the third argument * is the offset of the callback within the enclosing RCU-protected * data structure. */ -TRACE_EVENT(rcu_invoke_kfree_callback, +TRACE_EVENT_RCU(rcu_invoke_kvfree_callback, TP_PROTO(const char *rcuname, struct rcu_head *rhp, unsigned long offset), @@ -652,6 +646,61 @@ TRACE_EVENT(rcu_invoke_kfree_callback, ); /* + * Tracepoint for the invocation of a single RCU callback of the special + * kfree_bulk() form. The first argument is the RCU flavor, the second + * argument is a number of elements in array to free, the third is an + * address of the array holding nr_records entries. + */ +TRACE_EVENT_RCU(rcu_invoke_kfree_bulk_callback, + + TP_PROTO(const char *rcuname, unsigned long nr_records, void **p), + + TP_ARGS(rcuname, nr_records, p), + + TP_STRUCT__entry( + __field(const char *, rcuname) + __field(unsigned long, nr_records) + __field(void **, p) + ), + + TP_fast_assign( + __entry->rcuname = rcuname; + __entry->nr_records = nr_records; + __entry->p = p; + ), + + TP_printk("%s bulk=0x%p nr_records=%lu", + __entry->rcuname, __entry->p, __entry->nr_records) +); + +/* + * Tracepoint for a normal synchronize_rcu() states. The first argument + * is the RCU flavor, the second argument is a pointer to rcu_head the + * last one is an event. + */ +TRACE_EVENT_RCU(rcu_sr_normal, + + TP_PROTO(const char *rcuname, struct rcu_head *rhp, const char *srevent), + + TP_ARGS(rcuname, rhp, srevent), + + TP_STRUCT__entry( + __field(const char *, rcuname) + __field(void *, rhp) + __field(const char *, srevent) + ), + + TP_fast_assign( + __entry->rcuname = rcuname; + __entry->rhp = rhp; + __entry->srevent = srevent; + ), + + TP_printk("%s rhp=0x%p event=%s", + __entry->rcuname, __entry->rhp, __entry->srevent) +); + +/* * Tracepoint for exiting rcu_do_batch after RCU callbacks have been * invoked. The first argument is the name of the RCU flavor, * the second argument is number of callbacks actually invoked, @@ -662,7 +711,7 @@ TRACE_EVENT(rcu_invoke_kfree_callback, * and the sixth argument (risk) is the return value from * rcu_is_callbacks_kthread(). */ -TRACE_EVENT(rcu_batch_end, +TRACE_EVENT_RCU(rcu_batch_end, TP_PROTO(const char *rcuname, int callbacks_invoked, char cb, char nr, char iit, char risk), @@ -703,7 +752,8 @@ TRACE_EVENT(rcu_batch_end, * at the beginning and end of the read, respectively. Note that the * callback address can be NULL. */ -TRACE_EVENT(rcu_torture_read, +#define RCUTORTURENAME_LEN 8 +TRACE_EVENT_RCU(rcu_torture_read, TP_PROTO(const char *rcutorturename, struct rcu_head *rhp, unsigned long secs, unsigned long c_old, unsigned long c), @@ -711,7 +761,7 @@ TRACE_EVENT(rcu_torture_read, TP_ARGS(rcutorturename, rhp, secs, c_old, c), TP_STRUCT__entry( - __field(const char *, rcutorturename) + __array(char, rcutorturename, RCUTORTURENAME_LEN) __field(struct rcu_head *, rhp) __field(unsigned long, secs) __field(unsigned long, c_old) @@ -719,7 +769,7 @@ TRACE_EVENT(rcu_torture_read, ), TP_fast_assign( - __entry->rcutorturename = rcutorturename; + strscpy(__entry->rcutorturename, rcutorturename, RCUTORTURENAME_LEN); __entry->rhp = rhp; __entry->secs = secs; __entry->c_old = c_old; @@ -732,24 +782,22 @@ TRACE_EVENT(rcu_torture_read, ); /* - * Tracepoint for _rcu_barrier() execution. The string "s" describes - * the _rcu_barrier phase: - * "Begin": _rcu_barrier() started. - * "EarlyExit": _rcu_barrier() piggybacked, thus early exit. - * "Inc1": _rcu_barrier() piggyback check counter incremented. - * "OfflineNoCB": _rcu_barrier() found callback on never-online CPU - * "OnlineNoCB": _rcu_barrier() found online no-CBs CPU. - * "OnlineQ": _rcu_barrier() found online CPU with callbacks. - * "OnlineNQ": _rcu_barrier() found online CPU, no callbacks. + * Tracepoint for rcu_barrier() execution. The string "s" describes + * the rcu_barrier phase: + * "Begin": rcu_barrier() started. + * "CB": An rcu_barrier_callback() invoked a callback, not the last. + * "EarlyExit": rcu_barrier() piggybacked, thus early exit. + * "Inc1": rcu_barrier() piggyback check counter incremented. + * "Inc2": rcu_barrier() piggyback check counter incremented. * "IRQ": An rcu_barrier_callback() callback posted on remote CPU. * "IRQNQ": An rcu_barrier_callback() callback found no callbacks. - * "CB": An rcu_barrier_callback() invoked a callback, not the last. * "LastCB": An rcu_barrier_callback() invoked the last callback. - * "Inc2": _rcu_barrier() piggyback check counter incremented. + * "NQ": rcu_barrier() found a CPU with no callbacks. + * "OnlineQ": rcu_barrier() found online CPU with callbacks. * The "cpu" argument is the CPU or -1 if meaningless, the "cnt" argument * is the count of remaining callbacks, and "done" is the piggybacking count. */ -TRACE_EVENT(rcu_barrier, +TRACE_EVENT_RCU(rcu_barrier, TP_PROTO(const char *rcuname, const char *s, int cpu, int cnt, unsigned long done), @@ -776,42 +824,6 @@ TRACE_EVENT(rcu_barrier, __entry->done) ); -#else /* #ifdef CONFIG_RCU_TRACE */ - -#define trace_rcu_grace_period(rcuname, gpnum, gpevent) do { } while (0) -#define trace_rcu_future_grace_period(rcuname, gpnum, completed, c, \ - level, grplo, grphi, event) \ - do { } while (0) -#define trace_rcu_grace_period_init(rcuname, gpnum, level, grplo, grphi, \ - qsmask) do { } while (0) -#define trace_rcu_exp_grace_period(rcuname, gqseq, gpevent) \ - do { } while (0) -#define trace_rcu_exp_funnel_lock(rcuname, level, grplo, grphi, gpevent) \ - do { } while (0) -#define trace_rcu_nocb_wake(rcuname, cpu, reason) do { } while (0) -#define trace_rcu_preempt_task(rcuname, pid, gpnum) do { } while (0) -#define trace_rcu_unlock_preempted_task(rcuname, gpnum, pid) do { } while (0) -#define trace_rcu_quiescent_state_report(rcuname, gpnum, mask, qsmask, level, \ - grplo, grphi, gp_tasks) do { } \ - while (0) -#define trace_rcu_fqs(rcuname, gpnum, cpu, qsevent) do { } while (0) -#define trace_rcu_dyntick(polarity, oldnesting, newnesting) do { } while (0) -#define trace_rcu_prep_idle(reason) do { } while (0) -#define trace_rcu_callback(rcuname, rhp, qlen_lazy, qlen) do { } while (0) -#define trace_rcu_kfree_callback(rcuname, rhp, offset, qlen_lazy, qlen) \ - do { } while (0) -#define trace_rcu_batch_start(rcuname, qlen_lazy, qlen, blimit) \ - do { } while (0) -#define trace_rcu_invoke_callback(rcuname, rhp) do { } while (0) -#define trace_rcu_invoke_kfree_callback(rcuname, rhp, offset) do { } while (0) -#define trace_rcu_batch_end(rcuname, callbacks_invoked, cb, nr, iit, risk) \ - do { } while (0) -#define trace_rcu_torture_read(rcutorturename, rhp, secs, c_old, c) \ - do { } while (0) -#define trace_rcu_barrier(name, s, cpu, cnt, done) do { } while (0) - -#endif /* #else #ifdef CONFIG_RCU_TRACE */ - #endif /* _TRACE_RCU_H */ /* This part must be outside protection */ |