From 401013e2ad9e1ba329a91a7b66779d8cb780bcd0 Mon Sep 17 00:00:00 2001 From: Tycho Andersen Date: Wed, 7 Jun 2023 08:15:21 -0600 Subject: documentation/rcu: Fix typo s/slat/splat/ Signed-off-by: Tycho Andersen Signed-off-by: Paul E. McKenney --- Documentation/RCU/lockdep-splat.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Documentation') diff --git a/Documentation/RCU/lockdep-splat.rst b/Documentation/RCU/lockdep-splat.rst index 2a5c79db57dc..bcbc4b3c88d7 100644 --- a/Documentation/RCU/lockdep-splat.rst +++ b/Documentation/RCU/lockdep-splat.rst @@ -10,7 +10,7 @@ misuses of the RCU API, most notably using one of the rcu_dereference() family to access an RCU-protected pointer without the proper protection. When such misuse is detected, an lockdep-RCU splat is emitted. -The usual cause of a lockdep-RCU slat is someone accessing an +The usual cause of a lockdep-RCU splat is someone accessing an RCU-protected data structure without either (1) being in the right kind of RCU read-side critical section or (2) holding the right update-side lock. This problem can therefore be serious: it might result in random memory -- cgit From bc25e7c3598e56992120505ea09f6cf0651da63b Mon Sep 17 00:00:00 2001 From: Alan Huang Date: Tue, 13 Jun 2023 00:57:01 +0000 Subject: docs/RCU: Add the missing rcu_read_unlock() We should exit the RCU read-side critical section before re-entering. Signed-off-by: Alan Huang Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 9a734bf54b76..0612a6387d8e 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -29,8 +29,10 @@ algorithms: rcu_read_lock() obj = lockless_lookup(key); if (obj) { - if (!try_get_ref(obj)) // might fail for free objects + if (!try_get_ref(obj)) { // might fail for free objects + rcu_read_unlock(); goto begin; + } /* * Because a writer could delete object, and a writer could * reuse these object before the RCU grace period, we -- cgit From 3f831e38cecdd72a139a130da11f95be38481034 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 13 Jun 2023 18:24:31 +0000 Subject: Docs/RCU/rculist_nulls: Fix trivial coding style Lookup example of non-hlist_nulls management is missing a semicolon, and having inconsistent indentation (one line is using single space indentation while others are using two spaces indentation). Fix the trivial issues. Signed-off-by: SeongJae Park Reviewed-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 0612a6387d8e..25b739885cfa 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -26,7 +26,7 @@ algorithms: :: begin: - rcu_read_lock() + rcu_read_lock(); obj = lockless_lookup(key); if (obj) { if (!try_get_ref(obj)) { // might fail for free objects @@ -70,8 +70,8 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: pos && ({ prefetch(pos->next); 1; }) && ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); pos = rcu_dereference(pos->next)) - if (obj->key == key) - return obj; + if (obj->key == key) + return obj; return NULL; Quoting Corey Minyard:: -- cgit From 674dd365736125b56297aaea8845a90e5ea990b9 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Tue, 13 Jun 2023 18:24:32 +0000 Subject: Docs/RCU/rculist_nulls: Assign 'obj' before use from the examples Lookup example code snippets in rculist_nulls.rst are using 'obj' without assignment. Fix the code to assign it properly. Signed-off-by: SeongJae Park Reviewed-by: Joel Fernandes (Google) Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 25b739885cfa..4d6f077552ed 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -56,7 +56,7 @@ but a version with an additional memory barrier (smp_rmb()) struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); pos = rcu_dereference(next)) if (obj->key == key) return obj; @@ -68,7 +68,7 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); pos && ({ prefetch(pos->next); 1; }) && - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); pos = rcu_dereference(pos->next)) if (obj->key == key) return obj; -- cgit From 5326caa7a147743340f68b5b7ed4c2a91b4abb56 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:24 +0000 Subject: Docs/RCU/rculist_nulls: Specify type of the object in examples The type of 'obj' in example code of rculist_nulls.rst is implicit. Provide the specific type of it before the example code. Suggested-by: Paul E. McKenney Link: https://lore.kernel.org/rcu/43943609-f80c-4b6a-9844-994eef800757@paulmck-laptop/ Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 4d6f077552ed..479cedfec446 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -18,7 +18,16 @@ to solve following problem. Without 'nulls', a typical RCU linked list managing objects which are allocated with SLAB_TYPESAFE_BY_RCU kmem_cache can use the following -algorithms: +algorithms. Following examples assume 'obj' is a pointer to such +objects, which is having below type. + +:: + + struct object { + struct hlist_node obj_node; + atomic_t refcnt; + unsigned int key; + }; 1) Lookup algorithm ------------------- @@ -144,6 +153,9 @@ the beginning. If the object was moved to the same chain, then the reader doesn't care: It might occasionally scan the list again without harm. +Note that using hlist_nulls means the type of 'obj_node' field of +'struct object' becomes 'struct hlist_nulls_node'. + 1) lookup algorithm ------------------- -- cgit From d186204a98541c7d693290bfc91853bbfdfcf0cb Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:25 +0000 Subject: Docs/RCU/rculist_nulls: Fix hlist_[nulls]_head field names of 'obj' The example code snippets on rculist_nulls.rst are assuming 'obj' to have the 'hlist_head' or 'hlist_nulls_head' field named 'obj_node', but a sentence and some code snippets are wrongly calling 'obj->obj_node.next' as 'obj->obj_next', or 'obj->obj_node' as 'member'. Fix it. Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 479cedfec446..1fb086066377 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -65,7 +65,7 @@ but a version with an additional memory barrier (smp_rmb()) struct hlist_node *node, *next; for (pos = rcu_dereference((head)->first); pos && ({ next = pos->next; smp_rmb(); prefetch(next); 1; }) && - ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(next)) if (obj->key == key) return obj; @@ -77,7 +77,7 @@ And note the traditional hlist_for_each_entry_rcu() misses this smp_rmb():: struct hlist_node *node; for (pos = rcu_dereference((head)->first); pos && ({ prefetch(pos->next); 1; }) && - ({ obj = hlist_entry(pos, typeof(*obj), member); 1; }); + ({ obj = hlist_entry(pos, typeof(*obj), obj_node); 1; }); pos = rcu_dereference(pos->next)) if (obj->key == key) return obj; @@ -97,7 +97,7 @@ Quoting Corey Minyard:: 2) Insertion algorithm ---------------------- -We need to make sure a reader cannot read the new 'obj->obj_next' value +We need to make sure a reader cannot read the new 'obj->obj_node.next' value and previous value of 'obj->key'. Otherwise, an item could be deleted from a chain, and inserted into another chain. If new chain was empty before the move, 'next' pointer is NULL, and lockless reader can not @@ -165,7 +165,7 @@ Note that using hlist_nulls means the type of 'obj_node' field of head = &table[slot]; begin: rcu_read_lock(); - hlist_nulls_for_each_entry_rcu(obj, node, head, member) { + hlist_nulls_for_each_entry_rcu(obj, node, head, obj_node) { if (obj->key == key) { if (!try_get_ref(obj)) { // might fail for free objects rcu_read_unlock(); -- cgit From 47d63d7a3918d8bc1ccd0b0527da8dbb66429100 Mon Sep 17 00:00:00 2001 From: SeongJae Park Date: Fri, 16 Jun 2023 23:36:26 +0000 Subject: Docs/RCU/rculist_nulls: Fix text about atomic_set_release() The document says we can avoid extra _release() in insert function when hlist_nulls is used, but that's not true[1]. Drop it. [1] https://lore.kernel.org/rcu/46440869-644a-4982-b790-b71b43976c66@paulmck-laptop/ Signed-off-by: SeongJae Park Signed-off-by: Paul E. McKenney --- Documentation/RCU/rculist_nulls.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'Documentation') diff --git a/Documentation/RCU/rculist_nulls.rst b/Documentation/RCU/rculist_nulls.rst index 1fb086066377..21e40fcc08de 100644 --- a/Documentation/RCU/rculist_nulls.rst +++ b/Documentation/RCU/rculist_nulls.rst @@ -140,8 +140,7 @@ very very fast (before the end of RCU grace period) Avoiding extra smp_rmb() ======================== -With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup() -and extra _release() in insert function. +With hlist_nulls we can avoid extra smp_rmb() in lockless_lookup(). For example, if we choose to store the slot number as the 'nulls' end-of-list marker for each slot of the hash table, we can detect @@ -196,6 +195,9 @@ Note that using hlist_nulls means the type of 'obj_node' field of 2) Insert algorithm ------------------- +Same to the above one, but uses hlist_nulls_add_head_rcu() instead of +hlist_add_head_rcu(). + :: /* -- cgit From 450d461aa629024c0f01022122838b5243fe0f2a Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Wed, 17 May 2023 06:40:23 -0700 Subject: rcu-tasks: Add kernel boot parameters for callback laziness This commit adds kernel boot parameters for callback laziness, allowing the RCU Tasks flavors to be individually adjusted. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..a0f427e1a562 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5293,6 +5293,29 @@ A change in value does not take effect until the beginning of the next grace period. + rcupdate.rcu_tasks_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks asynchronous + callback batching for call_rcu_tasks(). + A negative value will take the default. A value + of zero will disable batching. Batching is + always disabled for synchronize_rcu_tasks(). + + rcupdate.rcu_tasks_rude_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Rude asynchronous callback batching for + call_rcu_tasks_rude(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_rude(). + + rcupdate.rcu_tasks_trace_lazy_ms= [KNL] + Set timeout in milliseconds RCU Tasks + Trace asynchronous callback batching for + call_rcu_tasks_trace(). A negative value + will take the default. A value of zero will + disable batching. Batching is always disabled + for synchronize_rcu_tasks_trace(). + rcupdate.rcu_self_test= [KNL] Run the RCU early boot self tests -- cgit From db13710a0365ba66f8f232eb3bdfaa1a10d820f7 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Mon, 26 Jun 2023 13:45:13 -0700 Subject: rcu-tasks: Cancel callback laziness if too many callbacks The various RCU Tasks flavors now do lazy grace periods when there are only asynchronous grace period requests. By default, the system will let 250 milliseconds elapse after the first call_rcu_tasks*() callbacki is queued before starting a grace period. In contrast, synchronous grace period requests such as synchronize_rcu_tasks*() will start a grace period immediately. However, invoking one of the call_rcu_tasks*() functions in a too-tight loop can result in a callback flood, which in turn can exhaust memory if grace periods are delayed for too long. This commit therefore sets a limit so that the grace-period kthread will be awakened when any CPU's callback list expands to contain rcupdate.rcu_task_lazy_lim callbacks elements (defaulting to 32, set to -1 to disable), the grace-period kthread will be awakened, thus cancelling any ongoing laziness and getting out in front of the potential callback flood. Cc: Alexei Starovoitov Cc: Martin KaFai Lau Cc: Andrii Nakryiko Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a0f427e1a562..44705a3618b9 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -5264,6 +5264,13 @@ number avoids disturbing real-time workloads, but lengthens grace periods. + rcupdate.rcu_task_lazy_lim= [KNL] + Number of callbacks on a given CPU that will + cancel laziness on that CPU. Use -1 to disable + cancellation of laziness, but be advised that + doing so increases the danger of OOM due to + callback flooding. + rcupdate.rcu_task_stall_info= [KNL] Set initial timeout in jiffies for RCU task stall informational messages, which give some indication -- cgit From 2226f3dc05a988ce81308bfc3d125f5d0b0d592c Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 07:02:01 -0700 Subject: rcuscale: Permit blocking delays between writers Some workloads do isolated RCU work, and this can affect operation latencies. This commit therefore adds a writer_holdoff_jiffies module parameter that causes writers to block for the specified number of jiffies between each pair of consecutive write-side operations. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..5ba231b786f8 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4983,6 +4983,11 @@ in microseconds. The default of zero says no holdoff. + rcuscale.writer_holdoff_jiffies= [KNL] + Additional write-side holdoff between grace + periods, but in jiffies. The default of zero + says no holdoff. + rcutorture.fqs_duration= [KNL] Set duration of force_quiescent_state bursts in microseconds. -- cgit From 7221f493c5ff8bee5c39de1e43cb8a8997c2cda5 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Tue, 16 May 2023 08:22:31 -0700 Subject: rcuscale: Add minruntime module parameter By default, rcuscale collects only 100 points of data per writer, but arranging for all kthreads to be actively collecting (if not recording) data during the time that any kthread might be recording. This works well, but does not allow much time to bring external performance tools to bear. This commit therefore adds a minruntime module parameter that specifies a minimum data-collection interval in seconds. Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index 5ba231b786f8..a6888e3dfc20 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4953,6 +4953,12 @@ Number of loops doing rcuscale.kfree_alloc_num number of allocations and frees. + rcuscale.minruntime= [KNL] + Set the minimum test run time in seconds. This + does not affect the data-collection interval, + but instead allows better measurement of things + like CPU consumption. + rcuscale.nreaders= [KNL] Set number of RCU readers. The value -1 selects N, where N is the number of CPUs. A value -- cgit From 2d7b2b344c0c665d47a3a9b540e03b834852b679 Mon Sep 17 00:00:00 2001 From: "Paul E. McKenney" Date: Thu, 1 Jun 2023 09:38:50 -0700 Subject: rcuscale: Add kfree_by_call_rcu and kfree_mult to documentation This commit adds the kfree_by_call_rcu and kfree_mult rcuscale module parameters to kernel-parameters.txt. While in the area, it updates to rcuscale.scale_type. Signed-off-by: Paul E. McKenney Cc: Uladzislau Rezki (Sony) --- Documentation/admin-guide/kernel-parameters.txt | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a6888e3dfc20..3329dc22f53b 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -4928,6 +4928,15 @@ test until boot completes in order to avoid interference. + rcuscale.kfree_by_call_rcu= [KNL] + In kernels built with CONFIG_RCU_LAZY=y, test + call_rcu() instead of kfree_rcu(). + + rcuscale.kfree_mult= [KNL] + Instead of allocating an object of size kfree_obj, + allocate one of kfree_mult * sizeof(kfree_obj). + Defaults to 1. + rcuscale.kfree_rcu_test= [KNL] Set to measure performance of kfree_rcu() flooding. @@ -4973,7 +4982,7 @@ the same as for rcuscale.nreaders. N, where N is the number of CPUs - rcuscale.perf_type= [KNL] + rcuscale.scale_type= [KNL] Specify the RCU implementation to test. rcuscale.shutdown= [KNL] -- cgit From 5d248bb39fe1388943acb6510f8f48fa5570e0ec Mon Sep 17 00:00:00 2001 From: Dietmar Eggemann Date: Fri, 2 Jun 2023 22:02:10 +0000 Subject: torture: Add lock_torture writer_fifo module parameter This commit adds a module parameter that causes the locktorture writer to run at real-time priority. To use it: insmod /lib/modules/torture.ko random_shuffle=1 insmod /lib/modules/locktorture.ko torture_type=mutex_lock rt_boost=1 rt_boost_factor=50 nested_locks=3 writer_fifo=1 ^^^^^^^^^^^^^ A predecessor to this patch has been helpful to uncover issues with the proxy-execution series. [ paulmck: Remove locktorture-specific code from kernel/torture.c. ] Cc: "Paul E. McKenney" Cc: Josh Triplett Cc: Joel Fernandes Cc: Juri Lelli Cc: Valentin Schneider Cc: kernel-team@android.com Signed-off-by: Dietmar Eggemann [jstultz: Include header change to build, reword commit message] Signed-off-by: John Stultz Acked-by: Davidlohr Bueso Signed-off-by: Paul E. McKenney --- Documentation/admin-guide/kernel-parameters.txt | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Documentation') diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index a1457995fd41..7b94455e9ae2 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2918,6 +2918,10 @@ locktorture.torture_type= [KNL] Specify the locking implementation to test. + locktorture.writer_fifo= [KNL] + Run the write-side locktorture kthreads at + sched_set_fifo() real-time priority. + locktorture.verbose= [KNL] Enable additional printk() statements. -- cgit