summaryrefslogtreecommitdiff
path: root/lib
diff options
context:
space:
mode:
Diffstat (limited to 'lib')
-rw-r--r--lib/Kconfig3
-rw-r--r--lib/Kconfig.debug9
-rw-r--r--lib/Makefile2
-rw-r--r--lib/alloc_tag.c2
-rw-r--r--lib/cache_maint.c138
-rw-r--r--lib/raid6/recov_rvv.c7
-rw-r--r--lib/raid6/rvv.c299
-rw-r--r--lib/raid6/rvv.h17
-rw-r--r--lib/raid6/test/Makefile8
-rw-r--r--lib/test_hmm.c457
-rw-r--r--lib/test_hmm_uapi.h3
-rw-r--r--lib/test_vmalloc.c28
-rw-r--r--lib/tests/test_fprobe.c99
13 files changed, 826 insertions, 246 deletions
diff --git a/lib/Kconfig b/lib/Kconfig
index e629449dd2a3..2923924bea78 100644
--- a/lib/Kconfig
+++ b/lib/Kconfig
@@ -542,6 +542,9 @@ config MEMREGION
config ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION
bool
+config GENERIC_CPU_CACHE_MAINTENANCE
+ bool
+
config ARCH_HAS_MEMREMAP_COMPAT_ALIGN
bool
diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug
index 742b23ef0d8b..c2654075377e 100644
--- a/lib/Kconfig.debug
+++ b/lib/Kconfig.debug
@@ -688,7 +688,7 @@ choice
help
This selects the default access restrictions for debugfs.
It can be overridden with kernel command line option
- debugfs=[on,no-mount,off]. The restrictions apply for API access
+ debugfs=[on,off]. The restrictions apply for API access
and filesystem registration.
config DEBUG_FS_ALLOW_ALL
@@ -697,13 +697,6 @@ config DEBUG_FS_ALLOW_ALL
No restrictions apply. Both API and filesystem registration
is on. This is the normal default operation.
-config DEBUG_FS_DISALLOW_MOUNT
- bool "Do not register debugfs as filesystem"
- help
- The API is open but filesystem is not loaded. Clients can still do
- their work and read with debug tools that do not need
- debugfs filesystem.
-
config DEBUG_FS_ALLOW_NONE
bool "No access"
help
diff --git a/lib/Makefile b/lib/Makefile
index 1ab2c4be3b66..aaf677cf4527 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -127,6 +127,8 @@ obj-$(CONFIG_HAS_IOMEM) += iomap_copy.o devres.o
obj-$(CONFIG_CHECK_SIGNATURE) += check_signature.o
obj-$(CONFIG_DEBUG_LOCKING_API_SELFTESTS) += locking-selftest.o
+obj-$(CONFIG_GENERIC_CPU_CACHE_MAINTENANCE) += cache_maint.o
+
lib-y += logic_pio.o
lib-$(CONFIG_INDIRECT_IOMEM) += logic_iomem.o
diff --git a/lib/alloc_tag.c b/lib/alloc_tag.c
index f26456988445..27fee57a5c91 100644
--- a/lib/alloc_tag.c
+++ b/lib/alloc_tag.c
@@ -845,7 +845,7 @@ static int __init alloc_tag_init(void)
alloc_tag_cttype = codetag_register_type(&desc);
if (IS_ERR(alloc_tag_cttype)) {
- pr_err("Allocation tags registration failed, errno = %ld\n", PTR_ERR(alloc_tag_cttype));
+ pr_err("Allocation tags registration failed, errno = %pe\n", alloc_tag_cttype);
free_mod_tags_mem();
shutdown_mem_profiling(true);
return PTR_ERR(alloc_tag_cttype);
diff --git a/lib/cache_maint.c b/lib/cache_maint.c
new file mode 100644
index 000000000000..9256a9ffc34c
--- /dev/null
+++ b/lib/cache_maint.c
@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Generic support for Memory System Cache Maintenance operations.
+ *
+ * Coherency maintenance drivers register with this simple framework that will
+ * iterate over each registered instance to first kick off invalidation and
+ * then to wait until it is complete.
+ *
+ * If no implementations are registered yet cpu_cache_has_invalidate_memregion()
+ * will return false. If this runs concurrently with unregistration then a
+ * race exists but this is no worse than the case where the operations instance
+ * responsible for a given memory region has not yet registered.
+ */
+#include <linux/cache_coherency.h>
+#include <linux/cleanup.h>
+#include <linux/container_of.h>
+#include <linux/export.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/memregion.h>
+#include <linux/module.h>
+#include <linux/rwsem.h>
+#include <linux/slab.h>
+
+static LIST_HEAD(cache_ops_instance_list);
+static DECLARE_RWSEM(cache_ops_instance_list_lock);
+
+static void __cache_coherency_ops_instance_free(struct kref *kref)
+{
+ struct cache_coherency_ops_inst *cci =
+ container_of(kref, struct cache_coherency_ops_inst, kref);
+ kfree(cci);
+}
+
+void cache_coherency_ops_instance_put(struct cache_coherency_ops_inst *cci)
+{
+ kref_put(&cci->kref, __cache_coherency_ops_instance_free);
+}
+EXPORT_SYMBOL_GPL(cache_coherency_ops_instance_put);
+
+static int cache_inval_one(struct cache_coherency_ops_inst *cci, void *data)
+{
+ if (!cci->ops)
+ return -EINVAL;
+
+ return cci->ops->wbinv(cci, data);
+}
+
+static int cache_inval_done_one(struct cache_coherency_ops_inst *cci)
+{
+ if (!cci->ops)
+ return -EINVAL;
+
+ if (!cci->ops->done)
+ return 0;
+
+ return cci->ops->done(cci);
+}
+
+static int cache_invalidate_memregion(phys_addr_t addr, size_t size)
+{
+ int ret;
+ struct cache_coherency_ops_inst *cci;
+ struct cc_inval_params params = {
+ .addr = addr,
+ .size = size,
+ };
+
+ guard(rwsem_read)(&cache_ops_instance_list_lock);
+ list_for_each_entry(cci, &cache_ops_instance_list, node) {
+ ret = cache_inval_one(cci, &params);
+ if (ret)
+ return ret;
+ }
+ list_for_each_entry(cci, &cache_ops_instance_list, node) {
+ ret = cache_inval_done_one(cci);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
+struct cache_coherency_ops_inst *
+_cache_coherency_ops_instance_alloc(const struct cache_coherency_ops *ops,
+ size_t size)
+{
+ struct cache_coherency_ops_inst *cci;
+
+ if (!ops || !ops->wbinv)
+ return NULL;
+
+ cci = kzalloc(size, GFP_KERNEL);
+ if (!cci)
+ return NULL;
+
+ cci->ops = ops;
+ INIT_LIST_HEAD(&cci->node);
+ kref_init(&cci->kref);
+
+ return cci;
+}
+EXPORT_SYMBOL_NS_GPL(_cache_coherency_ops_instance_alloc, "CACHE_COHERENCY");
+
+int cache_coherency_ops_instance_register(struct cache_coherency_ops_inst *cci)
+{
+ guard(rwsem_write)(&cache_ops_instance_list_lock);
+ list_add(&cci->node, &cache_ops_instance_list);
+
+ return 0;
+}
+EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_register, "CACHE_COHERENCY");
+
+void cache_coherency_ops_instance_unregister(struct cache_coherency_ops_inst *cci)
+{
+ guard(rwsem_write)(&cache_ops_instance_list_lock);
+ list_del(&cci->node);
+}
+EXPORT_SYMBOL_NS_GPL(cache_coherency_ops_instance_unregister, "CACHE_COHERENCY");
+
+int cpu_cache_invalidate_memregion(phys_addr_t start, size_t len)
+{
+ return cache_invalidate_memregion(start, len);
+}
+EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, "DEVMEM");
+
+/*
+ * Used for optimization / debug purposes only as removal can race
+ *
+ * Machines that do not support invalidation, e.g. VMs, will not have any
+ * operations instance to register and so this will always return false.
+ */
+bool cpu_cache_has_invalidate_memregion(void)
+{
+ guard(rwsem_read)(&cache_ops_instance_list_lock);
+ return !list_empty(&cache_ops_instance_list);
+}
+EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, "DEVMEM");
diff --git a/lib/raid6/recov_rvv.c b/lib/raid6/recov_rvv.c
index 5f779719c3d3..40c393206b6a 100644
--- a/lib/raid6/recov_rvv.c
+++ b/lib/raid6/recov_rvv.c
@@ -4,13 +4,8 @@
* Author: Chunyan Zhang <zhangchunyan@iscas.ac.cn>
*/
-#include <asm/vector.h>
#include <linux/raid/pq.h>
-
-static int rvv_has_vector(void)
-{
- return has_vector();
-}
+#include "rvv.h"
static void __raid6_2data_recov_rvv(int bytes, u8 *p, u8 *q, u8 *dp,
u8 *dq, const u8 *pbmul,
diff --git a/lib/raid6/rvv.c b/lib/raid6/rvv.c
index 89da5fc247aa..75c9dafedb28 100644
--- a/lib/raid6/rvv.c
+++ b/lib/raid6/rvv.c
@@ -9,22 +9,17 @@
* Copyright 2002-2004 H. Peter Anvin
*/
-#include <asm/vector.h>
-#include <linux/raid/pq.h>
#include "rvv.h"
-#define NSIZE (riscv_v_vsize / 32) /* NSIZE = vlenb */
-
-static int rvv_has_vector(void)
-{
- return has_vector();
-}
+#ifdef __riscv_vector
+#error "This code must be built without compiler support for vector"
+#endif
static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **ptrs)
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = disks - 3; /* Highest data disk */
@@ -38,8 +33,10 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
: "=&r" (vl)
);
+ nsize = vl;
+
/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
- for (d = 0; d < bytes; d += NSIZE * 1) {
+ for (d = 0; d < bytes; d += nsize * 1) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -47,7 +44,7 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
"vmv.v.v v1, v0\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize])
);
for (z = z0 - 1 ; z >= 0 ; z--) {
@@ -71,7 +68,7 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
"vxor.vv v0, v0, v2\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -86,8 +83,8 @@ static void raid6_rvv1_gen_syndrome_real(int disks, unsigned long bytes, void **
"vse8.v v1, (%[wq0])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0])
);
}
}
@@ -97,7 +94,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = stop; /* P/Q right side optimization */
@@ -111,8 +108,10 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
: "=&r" (vl)
);
+ nsize = vl;
+
/* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10 */
- for (d = 0 ; d < bytes ; d += NSIZE * 1) {
+ for (d = 0 ; d < bytes ; d += nsize * 1) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -120,7 +119,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
"vmv.v.v v1, v0\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize])
);
/* P/Q data pages */
@@ -145,7 +144,7 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
"vxor.vv v0, v0, v2\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -185,8 +184,8 @@ static void raid6_rvv1_xor_syndrome_real(int disks, int start, int stop,
"vse8.v v3, (%[wq0])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0])
);
}
}
@@ -195,7 +194,7 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = disks - 3; /* Highest data disk */
@@ -209,11 +208,13 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
*/
- for (d = 0; d < bytes; d += NSIZE * 2) {
+ for (d = 0; d < bytes; d += nsize * 2) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -223,8 +224,8 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
"vmv.v.v v5, v4\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize])
);
for (z = z0 - 1; z >= 0; z--) {
@@ -256,8 +257,8 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
"vxor.vv v4, v4, v6\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -274,10 +275,10 @@ static void raid6_rvv2_gen_syndrome_real(int disks, unsigned long bytes, void **
"vse8.v v5, (%[wq1])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1])
);
}
}
@@ -287,7 +288,7 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = stop; /* P/Q right side optimization */
@@ -301,11 +302,13 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
*/
- for (d = 0; d < bytes; d += NSIZE * 2) {
+ for (d = 0; d < bytes; d += nsize * 2) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -315,8 +318,8 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
"vmv.v.v v5, v4\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize])
);
/* P/Q data pages */
@@ -349,8 +352,8 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
"vxor.vv v4, v4, v6\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -403,10 +406,10 @@ static void raid6_rvv2_xor_syndrome_real(int disks, int start, int stop,
"vse8.v v7, (%[wq1])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1])
);
}
}
@@ -415,7 +418,7 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = disks - 3; /* Highest data disk */
@@ -429,13 +432,15 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
* v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
* v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
*/
- for (d = 0; d < bytes; d += NSIZE * 4) {
+ for (d = 0; d < bytes; d += nsize * 4) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -449,10 +454,10 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
"vmv.v.v v13, v12\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
- [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
- [wp3]"r"(&dptr[z0][d + 3 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+ [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+ [wp3]"r"(&dptr[z0][d + 3 * nsize])
);
for (z = z0 - 1; z >= 0; z--) {
@@ -500,10 +505,10 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
"vxor.vv v12, v12, v14\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
- [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
- [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
+ [wd2]"r"(&dptr[z][d + 2 * nsize]),
+ [wd3]"r"(&dptr[z][d + 3 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -524,14 +529,14 @@ static void raid6_rvv4_gen_syndrome_real(int disks, unsigned long bytes, void **
"vse8.v v13, (%[wq3])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1]),
- [wp2]"r"(&p[d + NSIZE * 2]),
- [wq2]"r"(&q[d + NSIZE * 2]),
- [wp3]"r"(&p[d + NSIZE * 3]),
- [wq3]"r"(&q[d + NSIZE * 3])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1]),
+ [wp2]"r"(&p[d + nsize * 2]),
+ [wq2]"r"(&q[d + nsize * 2]),
+ [wp3]"r"(&p[d + nsize * 3]),
+ [wq3]"r"(&q[d + nsize * 3])
);
}
}
@@ -541,7 +546,7 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = stop; /* P/Q right side optimization */
@@ -555,13 +560,15 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
* v8:wp2, v9:wq2, v10:wd2/w22, v11:w12
* v12:wp3, v13:wq3, v14:wd3/w23, v15:w13
*/
- for (d = 0; d < bytes; d += NSIZE * 4) {
+ for (d = 0; d < bytes; d += nsize * 4) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -575,10 +582,10 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
"vmv.v.v v13, v12\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
- [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
- [wp3]"r"(&dptr[z0][d + 3 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+ [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+ [wp3]"r"(&dptr[z0][d + 3 * nsize])
);
/* P/Q data pages */
@@ -627,10 +634,10 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
"vxor.vv v12, v12, v14\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
- [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
- [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
+ [wd2]"r"(&dptr[z][d + 2 * nsize]),
+ [wd3]"r"(&dptr[z][d + 3 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -709,14 +716,14 @@ static void raid6_rvv4_xor_syndrome_real(int disks, int start, int stop,
"vse8.v v15, (%[wq3])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1]),
- [wp2]"r"(&p[d + NSIZE * 2]),
- [wq2]"r"(&q[d + NSIZE * 2]),
- [wp3]"r"(&p[d + NSIZE * 3]),
- [wq3]"r"(&q[d + NSIZE * 3])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1]),
+ [wp2]"r"(&p[d + nsize * 2]),
+ [wq2]"r"(&q[d + nsize * 2]),
+ [wp3]"r"(&p[d + nsize * 3]),
+ [wq3]"r"(&q[d + nsize * 3])
);
}
}
@@ -725,7 +732,7 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = disks - 3; /* Highest data disk */
@@ -739,6 +746,8 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
@@ -749,7 +758,7 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
* v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
* v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
*/
- for (d = 0; d < bytes; d += NSIZE * 8) {
+ for (d = 0; d < bytes; d += nsize * 8) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -771,14 +780,14 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
"vmv.v.v v29, v28\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
- [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
- [wp3]"r"(&dptr[z0][d + 3 * NSIZE]),
- [wp4]"r"(&dptr[z0][d + 4 * NSIZE]),
- [wp5]"r"(&dptr[z0][d + 5 * NSIZE]),
- [wp6]"r"(&dptr[z0][d + 6 * NSIZE]),
- [wp7]"r"(&dptr[z0][d + 7 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+ [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+ [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+ [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+ [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+ [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+ [wp7]"r"(&dptr[z0][d + 7 * nsize])
);
for (z = z0 - 1; z >= 0; z--) {
@@ -858,14 +867,14 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
"vxor.vv v28, v28, v30\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
- [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
- [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
- [wd4]"r"(&dptr[z][d + 4 * NSIZE]),
- [wd5]"r"(&dptr[z][d + 5 * NSIZE]),
- [wd6]"r"(&dptr[z][d + 6 * NSIZE]),
- [wd7]"r"(&dptr[z][d + 7 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
+ [wd2]"r"(&dptr[z][d + 2 * nsize]),
+ [wd3]"r"(&dptr[z][d + 3 * nsize]),
+ [wd4]"r"(&dptr[z][d + 4 * nsize]),
+ [wd5]"r"(&dptr[z][d + 5 * nsize]),
+ [wd6]"r"(&dptr[z][d + 6 * nsize]),
+ [wd7]"r"(&dptr[z][d + 7 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -894,22 +903,22 @@ static void raid6_rvv8_gen_syndrome_real(int disks, unsigned long bytes, void **
"vse8.v v29, (%[wq7])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1]),
- [wp2]"r"(&p[d + NSIZE * 2]),
- [wq2]"r"(&q[d + NSIZE * 2]),
- [wp3]"r"(&p[d + NSIZE * 3]),
- [wq3]"r"(&q[d + NSIZE * 3]),
- [wp4]"r"(&p[d + NSIZE * 4]),
- [wq4]"r"(&q[d + NSIZE * 4]),
- [wp5]"r"(&p[d + NSIZE * 5]),
- [wq5]"r"(&q[d + NSIZE * 5]),
- [wp6]"r"(&p[d + NSIZE * 6]),
- [wq6]"r"(&q[d + NSIZE * 6]),
- [wp7]"r"(&p[d + NSIZE * 7]),
- [wq7]"r"(&q[d + NSIZE * 7])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1]),
+ [wp2]"r"(&p[d + nsize * 2]),
+ [wq2]"r"(&q[d + nsize * 2]),
+ [wp3]"r"(&p[d + nsize * 3]),
+ [wq3]"r"(&q[d + nsize * 3]),
+ [wp4]"r"(&p[d + nsize * 4]),
+ [wq4]"r"(&q[d + nsize * 4]),
+ [wp5]"r"(&p[d + nsize * 5]),
+ [wq5]"r"(&q[d + nsize * 5]),
+ [wp6]"r"(&p[d + nsize * 6]),
+ [wq6]"r"(&q[d + nsize * 6]),
+ [wp7]"r"(&p[d + nsize * 7]),
+ [wq7]"r"(&q[d + nsize * 7])
);
}
}
@@ -919,7 +928,7 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
{
u8 **dptr = (u8 **)ptrs;
u8 *p, *q;
- unsigned long vl, d;
+ unsigned long vl, d, nsize;
int z, z0;
z0 = stop; /* P/Q right side optimization */
@@ -933,6 +942,8 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
: "=&r" (vl)
);
+ nsize = vl;
+
/*
* v0:wp0, v1:wq0, v2:wd0/w20, v3:w10
* v4:wp1, v5:wq1, v6:wd1/w21, v7:w11
@@ -943,7 +954,7 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
* v24:wp6, v25:wq6, v26:wd6/w26, v27:w16
* v28:wp7, v29:wq7, v30:wd7/w27, v31:w17
*/
- for (d = 0; d < bytes; d += NSIZE * 8) {
+ for (d = 0; d < bytes; d += nsize * 8) {
/* wq$$ = wp$$ = *(unative_t *)&dptr[z0][d+$$*NSIZE]; */
asm volatile (".option push\n"
".option arch,+v\n"
@@ -965,14 +976,14 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
"vmv.v.v v29, v28\n"
".option pop\n"
: :
- [wp0]"r"(&dptr[z0][d + 0 * NSIZE]),
- [wp1]"r"(&dptr[z0][d + 1 * NSIZE]),
- [wp2]"r"(&dptr[z0][d + 2 * NSIZE]),
- [wp3]"r"(&dptr[z0][d + 3 * NSIZE]),
- [wp4]"r"(&dptr[z0][d + 4 * NSIZE]),
- [wp5]"r"(&dptr[z0][d + 5 * NSIZE]),
- [wp6]"r"(&dptr[z0][d + 6 * NSIZE]),
- [wp7]"r"(&dptr[z0][d + 7 * NSIZE])
+ [wp0]"r"(&dptr[z0][d + 0 * nsize]),
+ [wp1]"r"(&dptr[z0][d + 1 * nsize]),
+ [wp2]"r"(&dptr[z0][d + 2 * nsize]),
+ [wp3]"r"(&dptr[z0][d + 3 * nsize]),
+ [wp4]"r"(&dptr[z0][d + 4 * nsize]),
+ [wp5]"r"(&dptr[z0][d + 5 * nsize]),
+ [wp6]"r"(&dptr[z0][d + 6 * nsize]),
+ [wp7]"r"(&dptr[z0][d + 7 * nsize])
);
/* P/Q data pages */
@@ -1053,14 +1064,14 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
"vxor.vv v28, v28, v30\n"
".option pop\n"
: :
- [wd0]"r"(&dptr[z][d + 0 * NSIZE]),
- [wd1]"r"(&dptr[z][d + 1 * NSIZE]),
- [wd2]"r"(&dptr[z][d + 2 * NSIZE]),
- [wd3]"r"(&dptr[z][d + 3 * NSIZE]),
- [wd4]"r"(&dptr[z][d + 4 * NSIZE]),
- [wd5]"r"(&dptr[z][d + 5 * NSIZE]),
- [wd6]"r"(&dptr[z][d + 6 * NSIZE]),
- [wd7]"r"(&dptr[z][d + 7 * NSIZE]),
+ [wd0]"r"(&dptr[z][d + 0 * nsize]),
+ [wd1]"r"(&dptr[z][d + 1 * nsize]),
+ [wd2]"r"(&dptr[z][d + 2 * nsize]),
+ [wd3]"r"(&dptr[z][d + 3 * nsize]),
+ [wd4]"r"(&dptr[z][d + 4 * nsize]),
+ [wd5]"r"(&dptr[z][d + 5 * nsize]),
+ [wd6]"r"(&dptr[z][d + 6 * nsize]),
+ [wd7]"r"(&dptr[z][d + 7 * nsize]),
[x1d]"r"(0x1d)
);
}
@@ -1191,22 +1202,22 @@ static void raid6_rvv8_xor_syndrome_real(int disks, int start, int stop,
"vse8.v v31, (%[wq7])\n"
".option pop\n"
: :
- [wp0]"r"(&p[d + NSIZE * 0]),
- [wq0]"r"(&q[d + NSIZE * 0]),
- [wp1]"r"(&p[d + NSIZE * 1]),
- [wq1]"r"(&q[d + NSIZE * 1]),
- [wp2]"r"(&p[d + NSIZE * 2]),
- [wq2]"r"(&q[d + NSIZE * 2]),
- [wp3]"r"(&p[d + NSIZE * 3]),
- [wq3]"r"(&q[d + NSIZE * 3]),
- [wp4]"r"(&p[d + NSIZE * 4]),
- [wq4]"r"(&q[d + NSIZE * 4]),
- [wp5]"r"(&p[d + NSIZE * 5]),
- [wq5]"r"(&q[d + NSIZE * 5]),
- [wp6]"r"(&p[d + NSIZE * 6]),
- [wq6]"r"(&q[d + NSIZE * 6]),
- [wp7]"r"(&p[d + NSIZE * 7]),
- [wq7]"r"(&q[d + NSIZE * 7])
+ [wp0]"r"(&p[d + nsize * 0]),
+ [wq0]"r"(&q[d + nsize * 0]),
+ [wp1]"r"(&p[d + nsize * 1]),
+ [wq1]"r"(&q[d + nsize * 1]),
+ [wp2]"r"(&p[d + nsize * 2]),
+ [wq2]"r"(&q[d + nsize * 2]),
+ [wp3]"r"(&p[d + nsize * 3]),
+ [wq3]"r"(&q[d + nsize * 3]),
+ [wp4]"r"(&p[d + nsize * 4]),
+ [wq4]"r"(&q[d + nsize * 4]),
+ [wp5]"r"(&p[d + nsize * 5]),
+ [wq5]"r"(&q[d + nsize * 5]),
+ [wp6]"r"(&p[d + nsize * 6]),
+ [wq6]"r"(&q[d + nsize * 6]),
+ [wp7]"r"(&p[d + nsize * 7]),
+ [wq7]"r"(&q[d + nsize * 7])
);
}
}
diff --git a/lib/raid6/rvv.h b/lib/raid6/rvv.h
index 94044a1b707b..6d0708a2c8a4 100644
--- a/lib/raid6/rvv.h
+++ b/lib/raid6/rvv.h
@@ -7,6 +7,23 @@
* Definitions for RISC-V RAID-6 code
*/
+#ifdef __KERNEL__
+#include <asm/vector.h>
+#else
+#define kernel_vector_begin()
+#define kernel_vector_end()
+#include <sys/auxv.h>
+#include <asm/hwcap.h>
+#define has_vector() (getauxval(AT_HWCAP) & COMPAT_HWCAP_ISA_V)
+#endif
+
+#include <linux/raid/pq.h>
+
+static int rvv_has_vector(void)
+{
+ return has_vector();
+}
+
#define RAID6_RVV_WRAPPER(_n) \
static void raid6_rvv ## _n ## _gen_syndrome(int disks, \
size_t bytes, void **ptrs) \
diff --git a/lib/raid6/test/Makefile b/lib/raid6/test/Makefile
index 8f2dd2210ba8..09bbe2b14cce 100644
--- a/lib/raid6/test/Makefile
+++ b/lib/raid6/test/Makefile
@@ -35,6 +35,11 @@ ifeq ($(ARCH),aarch64)
HAS_NEON = yes
endif
+ifeq ($(findstring riscv,$(ARCH)),riscv)
+ CFLAGS += -I../../../arch/riscv/include -DCONFIG_RISCV=1
+ HAS_RVV = yes
+endif
+
ifeq ($(findstring ppc,$(ARCH)),ppc)
CFLAGS += -I../../../arch/powerpc/include
HAS_ALTIVEC := $(shell printf '$(pound)include <altivec.h>\nvector int a;\n' |\
@@ -63,6 +68,9 @@ else ifeq ($(HAS_ALTIVEC),yes)
vpermxor1.o vpermxor2.o vpermxor4.o vpermxor8.o
else ifeq ($(ARCH),loongarch64)
OBJS += loongarch_simd.o recov_loongarch_simd.o
+else ifeq ($(HAS_RVV),yes)
+ OBJS += rvv.o recov_rvv.o
+ CFLAGS += -DCONFIG_RISCV_ISA_V=1
endif
.c.o:
diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index 83e3d8208a54..8af169d3873a 100644
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -92,6 +92,7 @@ struct dmirror {
struct xarray pt;
struct mmu_interval_notifier notifier;
struct mutex mutex;
+ __u64 flags;
};
/*
@@ -119,6 +120,7 @@ struct dmirror_device {
unsigned long calloc;
unsigned long cfree;
struct page *free_pages;
+ struct folio *free_folios;
spinlock_t lock; /* protects the above */
};
@@ -492,7 +494,7 @@ fini:
}
static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
- struct page **ppage)
+ struct page **ppage, bool is_large)
{
struct dmirror_chunk *devmem;
struct resource *res = NULL;
@@ -572,20 +574,45 @@ static int dmirror_allocate_chunk(struct dmirror_device *mdevice,
pfn_first, pfn_last);
spin_lock(&mdevice->lock);
- for (pfn = pfn_first; pfn < pfn_last; pfn++) {
+ for (pfn = pfn_first; pfn < pfn_last; ) {
struct page *page = pfn_to_page(pfn);
+ if (is_large && IS_ALIGNED(pfn, HPAGE_PMD_NR)
+ && (pfn + HPAGE_PMD_NR <= pfn_last)) {
+ page->zone_device_data = mdevice->free_folios;
+ mdevice->free_folios = page_folio(page);
+ pfn += HPAGE_PMD_NR;
+ continue;
+ }
+
page->zone_device_data = mdevice->free_pages;
mdevice->free_pages = page;
+ pfn++;
}
+
+ ret = 0;
if (ppage) {
- *ppage = mdevice->free_pages;
- mdevice->free_pages = (*ppage)->zone_device_data;
- mdevice->calloc++;
+ if (is_large) {
+ if (!mdevice->free_folios) {
+ ret = -ENOMEM;
+ goto err_unlock;
+ }
+ *ppage = folio_page(mdevice->free_folios, 0);
+ mdevice->free_folios = (*ppage)->zone_device_data;
+ mdevice->calloc += HPAGE_PMD_NR;
+ } else if (mdevice->free_pages) {
+ *ppage = mdevice->free_pages;
+ mdevice->free_pages = (*ppage)->zone_device_data;
+ mdevice->calloc++;
+ } else {
+ ret = -ENOMEM;
+ goto err_unlock;
+ }
}
+err_unlock:
spin_unlock(&mdevice->lock);
- return 0;
+ return ret;
err_release:
mutex_unlock(&mdevice->devmem_lock);
@@ -598,10 +625,13 @@ err_devmem:
return ret;
}
-static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
+static struct page *dmirror_devmem_alloc_page(struct dmirror *dmirror,
+ bool is_large)
{
struct page *dpage = NULL;
struct page *rpage = NULL;
+ unsigned int order = is_large ? HPAGE_PMD_ORDER : 0;
+ struct dmirror_device *mdevice = dmirror->mdevice;
/*
* For ZONE_DEVICE private type, this is a fake device so we allocate
@@ -610,49 +640,55 @@ static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
* data and ignore rpage.
*/
if (dmirror_is_private_zone(mdevice)) {
- rpage = alloc_page(GFP_HIGHUSER);
+ rpage = folio_page(folio_alloc(GFP_HIGHUSER, order), 0);
if (!rpage)
return NULL;
}
spin_lock(&mdevice->lock);
- if (mdevice->free_pages) {
+ if (is_large && mdevice->free_folios) {
+ dpage = folio_page(mdevice->free_folios, 0);
+ mdevice->free_folios = dpage->zone_device_data;
+ mdevice->calloc += 1 << order;
+ spin_unlock(&mdevice->lock);
+ } else if (!is_large && mdevice->free_pages) {
dpage = mdevice->free_pages;
mdevice->free_pages = dpage->zone_device_data;
mdevice->calloc++;
spin_unlock(&mdevice->lock);
} else {
spin_unlock(&mdevice->lock);
- if (dmirror_allocate_chunk(mdevice, &dpage))
+ if (dmirror_allocate_chunk(mdevice, &dpage, is_large))
goto error;
}
- zone_device_page_init(dpage);
+ zone_device_folio_init(page_folio(dpage), order);
dpage->zone_device_data = rpage;
return dpage;
error:
if (rpage)
- __free_page(rpage);
+ __free_pages(rpage, order);
return NULL;
}
static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
struct dmirror *dmirror)
{
- struct dmirror_device *mdevice = dmirror->mdevice;
const unsigned long *src = args->src;
unsigned long *dst = args->dst;
unsigned long addr;
- for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
- src++, dst++) {
+ for (addr = args->start; addr < args->end; ) {
struct page *spage;
struct page *dpage;
struct page *rpage;
+ bool is_large = *src & MIGRATE_PFN_COMPOUND;
+ int write = (*src & MIGRATE_PFN_WRITE) ? MIGRATE_PFN_WRITE : 0;
+ unsigned long nr = 1;
if (!(*src & MIGRATE_PFN_MIGRATE))
- continue;
+ goto next;
/*
* Note that spage might be NULL which is OK since it is an
@@ -662,17 +698,50 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
if (WARN(spage && is_zone_device_page(spage),
"page already in device spage pfn: 0x%lx\n",
page_to_pfn(spage)))
+ goto next;
+
+ if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
+ dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
+ dpage = NULL;
+ } else
+ dpage = dmirror_devmem_alloc_page(dmirror, is_large);
+
+ if (!dpage) {
+ struct folio *folio;
+ unsigned long i;
+ unsigned long spfn = *src >> MIGRATE_PFN_SHIFT;
+ struct page *src_page;
+
+ if (!is_large)
+ goto next;
+
+ if (!spage && is_large) {
+ nr = HPAGE_PMD_NR;
+ } else {
+ folio = page_folio(spage);
+ nr = folio_nr_pages(folio);
+ }
+
+ for (i = 0; i < nr && addr < args->end; i++) {
+ dpage = dmirror_devmem_alloc_page(dmirror, false);
+ rpage = BACKING_PAGE(dpage);
+ rpage->zone_device_data = dmirror;
+
+ *dst = migrate_pfn(page_to_pfn(dpage)) | write;
+ src_page = pfn_to_page(spfn + i);
+
+ if (spage)
+ copy_highpage(rpage, src_page);
+ else
+ clear_highpage(rpage);
+ src++;
+ dst++;
+ addr += PAGE_SIZE;
+ }
continue;
-
- dpage = dmirror_devmem_alloc_page(mdevice);
- if (!dpage)
- continue;
+ }
rpage = BACKING_PAGE(dpage);
- if (spage)
- copy_highpage(rpage, spage);
- else
- clear_highpage(rpage);
/*
* Normally, a device would use the page->zone_device_data to
@@ -684,10 +753,42 @@ static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
page_to_pfn(spage), page_to_pfn(dpage));
- *dst = migrate_pfn(page_to_pfn(dpage));
- if ((*src & MIGRATE_PFN_WRITE) ||
- (!spage && args->vma->vm_flags & VM_WRITE))
- *dst |= MIGRATE_PFN_WRITE;
+
+ *dst = migrate_pfn(page_to_pfn(dpage)) | write;
+
+ if (is_large) {
+ int i;
+ struct folio *folio = page_folio(dpage);
+ *dst |= MIGRATE_PFN_COMPOUND;
+
+ if (folio_test_large(folio)) {
+ for (i = 0; i < folio_nr_pages(folio); i++) {
+ struct page *dst_page =
+ pfn_to_page(page_to_pfn(rpage) + i);
+ struct page *src_page =
+ pfn_to_page(page_to_pfn(spage) + i);
+
+ if (spage)
+ copy_highpage(dst_page, src_page);
+ else
+ clear_highpage(dst_page);
+ src++;
+ dst++;
+ addr += PAGE_SIZE;
+ }
+ continue;
+ }
+ }
+
+ if (spage)
+ copy_highpage(rpage, spage);
+ else
+ clear_highpage(rpage);
+
+next:
+ src++;
+ dst++;
+ addr += PAGE_SIZE;
}
}
@@ -734,14 +835,17 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
const unsigned long *src = args->src;
const unsigned long *dst = args->dst;
unsigned long pfn;
+ const unsigned long start_pfn = start >> PAGE_SHIFT;
+ const unsigned long end_pfn = end >> PAGE_SHIFT;
/* Map the migrated pages into the device's page tables. */
mutex_lock(&dmirror->mutex);
- for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
- src++, dst++) {
+ for (pfn = start_pfn; pfn < end_pfn; pfn++, src++, dst++) {
struct page *dpage;
void *entry;
+ int nr, i;
+ struct page *rpage;
if (!(*src & MIGRATE_PFN_MIGRATE))
continue;
@@ -750,13 +854,25 @@ static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
if (!dpage)
continue;
- entry = BACKING_PAGE(dpage);
- if (*dst & MIGRATE_PFN_WRITE)
- entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
- entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
- if (xa_is_err(entry)) {
- mutex_unlock(&dmirror->mutex);
- return xa_err(entry);
+ if (*dst & MIGRATE_PFN_COMPOUND)
+ nr = folio_nr_pages(page_folio(dpage));
+ else
+ nr = 1;
+
+ WARN_ON_ONCE(end_pfn < start_pfn + nr);
+
+ rpage = BACKING_PAGE(dpage);
+ VM_WARN_ON(folio_nr_pages(page_folio(rpage)) != nr);
+
+ for (i = 0; i < nr; i++) {
+ entry = folio_page(page_folio(rpage), i);
+ if (*dst & MIGRATE_PFN_WRITE)
+ entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
+ entry = xa_store(&dmirror->pt, pfn + i, entry, GFP_ATOMIC);
+ if (xa_is_err(entry)) {
+ mutex_unlock(&dmirror->mutex);
+ return xa_err(entry);
+ }
}
}
@@ -829,31 +945,77 @@ static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
unsigned long start = args->start;
unsigned long end = args->end;
unsigned long addr;
+ unsigned int order = 0;
+ int i;
- for (addr = start; addr < end; addr += PAGE_SIZE,
- src++, dst++) {
+ for (addr = start; addr < end; ) {
struct page *dpage, *spage;
spage = migrate_pfn_to_page(*src);
- if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
- continue;
+ if (!spage || !(*src & MIGRATE_PFN_MIGRATE)) {
+ addr += PAGE_SIZE;
+ goto next;
+ }
if (WARN_ON(!is_device_private_page(spage) &&
- !is_device_coherent_page(spage)))
- continue;
- spage = BACKING_PAGE(spage);
- dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
- if (!dpage)
- continue;
- pr_debug("migrating from dev to sys pfn src: 0x%lx pfn dst: 0x%lx\n",
- page_to_pfn(spage), page_to_pfn(dpage));
+ !is_device_coherent_page(spage))) {
+ addr += PAGE_SIZE;
+ goto next;
+ }
- lock_page(dpage);
- xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
- copy_highpage(dpage, spage);
- *dst = migrate_pfn(page_to_pfn(dpage));
+ spage = BACKING_PAGE(spage);
+ order = folio_order(page_folio(spage));
+ if (order)
+ *dst = MIGRATE_PFN_COMPOUND;
if (*src & MIGRATE_PFN_WRITE)
*dst |= MIGRATE_PFN_WRITE;
+
+ if (dmirror->flags & HMM_DMIRROR_FLAG_FAIL_ALLOC) {
+ dmirror->flags &= ~HMM_DMIRROR_FLAG_FAIL_ALLOC;
+ *dst &= ~MIGRATE_PFN_COMPOUND;
+ dpage = NULL;
+ } else if (order) {
+ dpage = folio_page(vma_alloc_folio(GFP_HIGHUSER_MOVABLE,
+ order, args->vma, addr), 0);
+ } else {
+ dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
+ }
+
+ if (!dpage && !order)
+ return VM_FAULT_OOM;
+
+ pr_debug("migrating from sys to dev pfn src: 0x%lx pfn dst: 0x%lx\n",
+ page_to_pfn(spage), page_to_pfn(dpage));
+
+ if (dpage) {
+ lock_page(dpage);
+ *dst |= migrate_pfn(page_to_pfn(dpage));
+ }
+
+ for (i = 0; i < (1 << order); i++) {
+ struct page *src_page;
+ struct page *dst_page;
+
+ /* Try with smaller pages if large allocation fails */
+ if (!dpage && order) {
+ dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
+ lock_page(dpage);
+ dst[i] = migrate_pfn(page_to_pfn(dpage));
+ dst_page = pfn_to_page(page_to_pfn(dpage));
+ dpage = NULL; /* For the next iteration */
+ } else {
+ dst_page = pfn_to_page(page_to_pfn(dpage) + i);
+ }
+
+ src_page = pfn_to_page(page_to_pfn(spage) + i);
+
+ xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
+ addr += PAGE_SIZE;
+ copy_highpage(dst_page, src_page);
+ }
+next:
+ src += 1 << order;
+ dst += 1 << order;
}
return 0;
}
@@ -879,11 +1041,14 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
unsigned long size = cmd->npages << PAGE_SHIFT;
struct mm_struct *mm = dmirror->notifier.mm;
struct vm_area_struct *vma;
- unsigned long src_pfns[32] = { 0 };
- unsigned long dst_pfns[32] = { 0 };
struct migrate_vma args = { 0 };
unsigned long next;
int ret;
+ unsigned long *src_pfns;
+ unsigned long *dst_pfns;
+
+ src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
+ dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
start = cmd->addr;
end = start + size;
@@ -902,7 +1067,7 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
ret = -EINVAL;
goto out;
}
- next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
+ next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
if (next > vma->vm_end)
next = vma->vm_end;
@@ -912,7 +1077,7 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
args.start = addr;
args.end = next;
args.pgmap_owner = dmirror->mdevice;
- args.flags = dmirror_select_device(dmirror);
+ args.flags = dmirror_select_device(dmirror) | MIGRATE_VMA_SELECT_COMPOUND;
ret = migrate_vma_setup(&args);
if (ret)
@@ -928,6 +1093,8 @@ static int dmirror_migrate_to_system(struct dmirror *dmirror,
out:
mmap_read_unlock(mm);
mmput(mm);
+ kvfree(src_pfns);
+ kvfree(dst_pfns);
return ret;
}
@@ -939,12 +1106,12 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
unsigned long size = cmd->npages << PAGE_SHIFT;
struct mm_struct *mm = dmirror->notifier.mm;
struct vm_area_struct *vma;
- unsigned long src_pfns[32] = { 0 };
- unsigned long dst_pfns[32] = { 0 };
struct dmirror_bounce bounce;
struct migrate_vma args = { 0 };
unsigned long next;
int ret;
+ unsigned long *src_pfns = NULL;
+ unsigned long *dst_pfns = NULL;
start = cmd->addr;
end = start + size;
@@ -955,6 +1122,18 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
if (!mmget_not_zero(mm))
return -EINVAL;
+ ret = -ENOMEM;
+ src_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*src_pfns),
+ GFP_KERNEL | __GFP_NOFAIL);
+ if (!src_pfns)
+ goto free_mem;
+
+ dst_pfns = kvcalloc(PTRS_PER_PTE, sizeof(*dst_pfns),
+ GFP_KERNEL | __GFP_NOFAIL);
+ if (!dst_pfns)
+ goto free_mem;
+
+ ret = 0;
mmap_read_lock(mm);
for (addr = start; addr < end; addr = next) {
vma = vma_lookup(mm, addr);
@@ -962,7 +1141,7 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
ret = -EINVAL;
goto out;
}
- next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
+ next = min(end, addr + (PTRS_PER_PTE << PAGE_SHIFT));
if (next > vma->vm_end)
next = vma->vm_end;
@@ -972,7 +1151,8 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
args.start = addr;
args.end = next;
args.pgmap_owner = dmirror->mdevice;
- args.flags = MIGRATE_VMA_SELECT_SYSTEM;
+ args.flags = MIGRATE_VMA_SELECT_SYSTEM |
+ MIGRATE_VMA_SELECT_COMPOUND;
ret = migrate_vma_setup(&args);
if (ret)
goto out;
@@ -992,7 +1172,7 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
*/
ret = dmirror_bounce_init(&bounce, start, size);
if (ret)
- return ret;
+ goto free_mem;
mutex_lock(&dmirror->mutex);
ret = dmirror_do_read(dmirror, start, end, &bounce);
mutex_unlock(&dmirror->mutex);
@@ -1003,11 +1183,14 @@ static int dmirror_migrate_to_device(struct dmirror *dmirror,
}
cmd->cpages = bounce.cpages;
dmirror_bounce_fini(&bounce);
- return ret;
+ goto free_mem;
out:
mmap_read_unlock(mm);
mmput(mm);
+free_mem:
+ kfree(src_pfns);
+ kfree(dst_pfns);
return ret;
}
@@ -1200,6 +1383,7 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
unsigned long i;
unsigned long *src_pfns;
unsigned long *dst_pfns;
+ unsigned int order = 0;
src_pfns = kvcalloc(npages, sizeof(*src_pfns), GFP_KERNEL | __GFP_NOFAIL);
dst_pfns = kvcalloc(npages, sizeof(*dst_pfns), GFP_KERNEL | __GFP_NOFAIL);
@@ -1215,13 +1399,25 @@ static void dmirror_device_evict_chunk(struct dmirror_chunk *chunk)
if (WARN_ON(!is_device_private_page(spage) &&
!is_device_coherent_page(spage)))
continue;
+
+ order = folio_order(page_folio(spage));
spage = BACKING_PAGE(spage);
- dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
+ if (src_pfns[i] & MIGRATE_PFN_COMPOUND) {
+ dpage = folio_page(folio_alloc(GFP_HIGHUSER_MOVABLE,
+ order), 0);
+ } else {
+ dpage = alloc_page(GFP_HIGHUSER_MOVABLE | __GFP_NOFAIL);
+ order = 0;
+ }
+
+ /* TODO Support splitting here */
lock_page(dpage);
- copy_highpage(dpage, spage);
dst_pfns[i] = migrate_pfn(page_to_pfn(dpage));
if (src_pfns[i] & MIGRATE_PFN_WRITE)
dst_pfns[i] |= MIGRATE_PFN_WRITE;
+ if (order)
+ dst_pfns[i] |= MIGRATE_PFN_COMPOUND;
+ folio_copy(page_folio(dpage), page_folio(spage));
}
migrate_device_pages(src_pfns, dst_pfns, npages);
migrate_device_finalize(src_pfns, dst_pfns, npages);
@@ -1234,7 +1430,12 @@ static void dmirror_remove_free_pages(struct dmirror_chunk *devmem)
{
struct dmirror_device *mdevice = devmem->mdevice;
struct page *page;
+ struct folio *folio;
+
+ for (folio = mdevice->free_folios; folio; folio = folio_zone_device_data(folio))
+ if (dmirror_page_to_chunk(folio_page(folio, 0)) == devmem)
+ mdevice->free_folios = folio_zone_device_data(folio);
for (page = mdevice->free_pages; page; page = page->zone_device_data)
if (dmirror_page_to_chunk(page) == devmem)
mdevice->free_pages = page->zone_device_data;
@@ -1265,6 +1466,7 @@ static void dmirror_device_remove_chunks(struct dmirror_device *mdevice)
mdevice->devmem_count = 0;
mdevice->devmem_capacity = 0;
mdevice->free_pages = NULL;
+ mdevice->free_folios = NULL;
kfree(mdevice->devmem_chunks);
mdevice->devmem_chunks = NULL;
}
@@ -1329,6 +1531,10 @@ static long dmirror_fops_unlocked_ioctl(struct file *filp,
dmirror_device_remove_chunks(dmirror->mdevice);
ret = 0;
break;
+ case HMM_DMIRROR_FLAGS:
+ dmirror->flags = cmd.npages;
+ ret = 0;
+ break;
default:
return -EINVAL;
@@ -1374,22 +1580,35 @@ static const struct file_operations dmirror_fops = {
.owner = THIS_MODULE,
};
-static void dmirror_devmem_free(struct page *page)
+static void dmirror_devmem_free(struct folio *folio)
{
+ struct page *page = &folio->page;
struct page *rpage = BACKING_PAGE(page);
struct dmirror_device *mdevice;
+ struct folio *rfolio = page_folio(rpage);
+ unsigned int order = folio_order(rfolio);
- if (rpage != page)
- __free_page(rpage);
+ if (rpage != page) {
+ if (order)
+ __free_pages(rpage, order);
+ else
+ __free_page(rpage);
+ rpage = NULL;
+ }
mdevice = dmirror_page_to_device(page);
spin_lock(&mdevice->lock);
/* Return page to our allocator if not freeing the chunk */
if (!dmirror_page_to_chunk(page)->remove) {
- mdevice->cfree++;
- page->zone_device_data = mdevice->free_pages;
- mdevice->free_pages = page;
+ mdevice->cfree += 1 << order;
+ if (order) {
+ page->zone_device_data = mdevice->free_folios;
+ mdevice->free_folios = page_folio(page);
+ } else {
+ page->zone_device_data = mdevice->free_pages;
+ mdevice->free_pages = page;
+ }
}
spin_unlock(&mdevice->lock);
}
@@ -1397,36 +1616,61 @@ static void dmirror_devmem_free(struct page *page)
static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
{
struct migrate_vma args = { 0 };
- unsigned long src_pfns = 0;
- unsigned long dst_pfns = 0;
struct page *rpage;
struct dmirror *dmirror;
- vm_fault_t ret;
+ vm_fault_t ret = 0;
+ unsigned int order, nr;
/*
* Normally, a device would use the page->zone_device_data to point to
* the mirror but here we use it to hold the page for the simulated
* device memory and that page holds the pointer to the mirror.
*/
- rpage = vmf->page->zone_device_data;
+ rpage = folio_zone_device_data(page_folio(vmf->page));
dmirror = rpage->zone_device_data;
/* FIXME demonstrate how we can adjust migrate range */
+ order = folio_order(page_folio(vmf->page));
+ nr = 1 << order;
+
+ /*
+ * When folios are partially mapped, we can't rely on the folio
+ * order of vmf->page as the folio might not be fully split yet
+ */
+ if (vmf->pte) {
+ order = 0;
+ nr = 1;
+ }
+
+ /*
+ * Consider a per-cpu cache of src and dst pfns, but with
+ * large number of cpus that might not scale well.
+ */
+ args.start = ALIGN_DOWN(vmf->address, (PAGE_SIZE << order));
args.vma = vmf->vma;
- args.start = vmf->address;
- args.end = args.start + PAGE_SIZE;
- args.src = &src_pfns;
- args.dst = &dst_pfns;
+ args.end = args.start + (PAGE_SIZE << order);
+
+ nr = (args.end - args.start) >> PAGE_SHIFT;
+ args.src = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
+ args.dst = kcalloc(nr, sizeof(unsigned long), GFP_KERNEL);
args.pgmap_owner = dmirror->mdevice;
args.flags = dmirror_select_device(dmirror);
args.fault_page = vmf->page;
+ if (!args.src || !args.dst) {
+ ret = VM_FAULT_OOM;
+ goto err;
+ }
+
+ if (order)
+ args.flags |= MIGRATE_VMA_SELECT_COMPOUND;
+
if (migrate_vma_setup(&args))
return VM_FAULT_SIGBUS;
ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
if (ret)
- return ret;
+ goto err;
migrate_vma_pages(&args);
/*
* No device finalize step is needed since
@@ -1434,12 +1678,50 @@ static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
* invalidated the device page table.
*/
migrate_vma_finalize(&args);
- return 0;
+err:
+ kfree(args.src);
+ kfree(args.dst);
+ return ret;
+}
+
+static void dmirror_devmem_folio_split(struct folio *head, struct folio *tail)
+{
+ struct page *rpage = BACKING_PAGE(folio_page(head, 0));
+ struct page *rpage_tail;
+ struct folio *rfolio;
+ unsigned long offset = 0;
+
+ if (!rpage) {
+ tail->page.zone_device_data = NULL;
+ return;
+ }
+
+ rfolio = page_folio(rpage);
+
+ if (tail == NULL) {
+ folio_reset_order(rfolio);
+ rfolio->mapping = NULL;
+ folio_set_count(rfolio, 1);
+ return;
+ }
+
+ offset = folio_pfn(tail) - folio_pfn(head);
+
+ rpage_tail = folio_page(rfolio, offset);
+ tail->page.zone_device_data = rpage_tail;
+ rpage_tail->zone_device_data = rpage->zone_device_data;
+ clear_compound_head(rpage_tail);
+ rpage_tail->mapping = NULL;
+
+ folio_page(tail, 0)->mapping = folio_page(head, 0)->mapping;
+ tail->pgmap = head->pgmap;
+ folio_set_count(page_folio(rpage_tail), 1);
}
static const struct dev_pagemap_ops dmirror_devmem_ops = {
- .page_free = dmirror_devmem_free,
+ .folio_free = dmirror_devmem_free,
.migrate_to_ram = dmirror_devmem_fault,
+ .folio_split = dmirror_devmem_folio_split,
};
static int dmirror_device_init(struct dmirror_device *mdevice, int id)
@@ -1458,20 +1740,25 @@ static int dmirror_device_init(struct dmirror_device *mdevice, int id)
ret = dev_set_name(&mdevice->device, "hmm_dmirror%u", id);
if (ret)
- return ret;
+ goto put_device;
ret = cdev_device_add(&mdevice->cdevice, &mdevice->device);
if (ret)
- return ret;
+ goto put_device;
/* Build a list of free ZONE_DEVICE struct pages */
- return dmirror_allocate_chunk(mdevice, NULL);
+ return dmirror_allocate_chunk(mdevice, NULL, false);
+
+put_device:
+ put_device(&mdevice->device);
+ return ret;
}
static void dmirror_device_remove(struct dmirror_device *mdevice)
{
dmirror_device_remove_chunks(mdevice);
cdev_device_del(&mdevice->cdevice, &mdevice->device);
+ put_device(&mdevice->device);
}
static int __init hmm_dmirror_init(void)
diff --git a/lib/test_hmm_uapi.h b/lib/test_hmm_uapi.h
index 8c818a2cf4f6..f94c6d457338 100644
--- a/lib/test_hmm_uapi.h
+++ b/lib/test_hmm_uapi.h
@@ -37,6 +37,9 @@ struct hmm_dmirror_cmd {
#define HMM_DMIRROR_EXCLUSIVE _IOWR('H', 0x05, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_CHECK_EXCLUSIVE _IOWR('H', 0x06, struct hmm_dmirror_cmd)
#define HMM_DMIRROR_RELEASE _IOWR('H', 0x07, struct hmm_dmirror_cmd)
+#define HMM_DMIRROR_FLAGS _IOWR('H', 0x08, struct hmm_dmirror_cmd)
+
+#define HMM_DMIRROR_FLAG_FAIL_ALLOC (1ULL << 0)
/*
* Values returned in hmm_dmirror_cmd.ptr for HMM_DMIRROR_SNAPSHOT.
diff --git a/lib/test_vmalloc.c b/lib/test_vmalloc.c
index 2815658ccc37..6521c05c7816 100644
--- a/lib/test_vmalloc.c
+++ b/lib/test_vmalloc.c
@@ -54,6 +54,7 @@ __param(int, run_test_mask, 7,
"\t\tid: 256, name: kvfree_rcu_1_arg_vmalloc_test\n"
"\t\tid: 512, name: kvfree_rcu_2_arg_vmalloc_test\n"
"\t\tid: 1024, name: vm_map_ram_test\n"
+ "\t\tid: 2048, name: no_block_alloc_test\n"
/* Add a new test case description here. */
);
@@ -283,6 +284,30 @@ static int fix_size_alloc_test(void)
return 0;
}
+static int no_block_alloc_test(void)
+{
+ void *ptr;
+ int i;
+
+ for (i = 0; i < test_loop_count; i++) {
+ bool use_atomic = !!(get_random_u8() % 2);
+ gfp_t gfp = use_atomic ? GFP_ATOMIC : GFP_NOWAIT;
+ unsigned long size = (nr_pages > 0 ? nr_pages : 1) * PAGE_SIZE;
+
+ preempt_disable();
+ ptr = __vmalloc(size, gfp);
+ preempt_enable();
+
+ if (!ptr)
+ return -1;
+
+ *((__u8 *)ptr) = 0;
+ vfree(ptr);
+ }
+
+ return 0;
+}
+
static int
pcpu_alloc_test(void)
{
@@ -411,6 +436,7 @@ static struct test_case_desc test_case_array[] = {
{ "kvfree_rcu_1_arg_vmalloc_test", kvfree_rcu_1_arg_vmalloc_test, },
{ "kvfree_rcu_2_arg_vmalloc_test", kvfree_rcu_2_arg_vmalloc_test, },
{ "vm_map_ram_test", vm_map_ram_test, },
+ { "no_block_alloc_test", no_block_alloc_test, true },
/* Add a new test case here. */
};
@@ -474,7 +500,7 @@ static int test_func(void *private)
for (j = 0; j < test_repeat_count; j++) {
ret = test_case_array[index].test_func();
- if (!ret && !test_case_array[index].xfail)
+ if (!ret)
t->data[index].test_passed++;
else if (ret && test_case_array[index].xfail)
t->data[index].test_xfailed++;
diff --git a/lib/tests/test_fprobe.c b/lib/tests/test_fprobe.c
index cf92111b5c79..108c7aa33cb4 100644
--- a/lib/tests/test_fprobe.c
+++ b/lib/tests/test_fprobe.c
@@ -12,7 +12,8 @@
static struct kunit *current_test;
-static u32 rand1, entry_val, exit_val;
+static u32 rand1, entry_only_val, entry_val, exit_val;
+static u32 entry_only_count, entry_count, exit_count;
/* Use indirect calls to avoid inlining the target functions */
static u32 (*target)(u32 value);
@@ -190,6 +191,101 @@ static void test_fprobe_skip(struct kunit *test)
KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp));
}
+/* Handler for fprobe entry only case */
+static notrace int entry_only_handler(struct fprobe *fp, unsigned long ip,
+ unsigned long ret_ip,
+ struct ftrace_regs *fregs, void *data)
+{
+ KUNIT_EXPECT_FALSE(current_test, preemptible());
+ KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+
+ entry_only_count++;
+ entry_only_val = (rand1 / div_factor);
+
+ return 0;
+}
+
+static notrace int fprobe_entry_multi_handler(struct fprobe *fp, unsigned long ip,
+ unsigned long ret_ip,
+ struct ftrace_regs *fregs,
+ void *data)
+{
+ KUNIT_EXPECT_FALSE(current_test, preemptible());
+ KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+
+ entry_count++;
+ entry_val = (rand1 / div_factor);
+
+ return 0;
+}
+
+static notrace void fprobe_exit_multi_handler(struct fprobe *fp, unsigned long ip,
+ unsigned long ret_ip,
+ struct ftrace_regs *fregs,
+ void *data)
+{
+ unsigned long ret = ftrace_regs_get_return_value(fregs);
+
+ KUNIT_EXPECT_FALSE(current_test, preemptible());
+ KUNIT_EXPECT_EQ(current_test, ip, target_ip);
+ KUNIT_EXPECT_EQ(current_test, ret, (rand1 / div_factor));
+
+ exit_count++;
+ exit_val = ret;
+}
+
+static void check_fprobe_multi(struct kunit *test)
+{
+ entry_only_count = entry_count = exit_count = 0;
+ entry_only_val = entry_val = exit_val = 0;
+
+ target(rand1);
+
+ /* Verify all handlers were called */
+ KUNIT_EXPECT_EQ(test, 1, entry_only_count);
+ KUNIT_EXPECT_EQ(test, 1, entry_count);
+ KUNIT_EXPECT_EQ(test, 1, exit_count);
+
+ /* Verify values are correct */
+ KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_only_val);
+ KUNIT_EXPECT_EQ(test, (rand1 / div_factor), entry_val);
+ KUNIT_EXPECT_EQ(test, (rand1 / div_factor), exit_val);
+}
+
+/* Test multiple fprobes hooking the same target function */
+static void test_fprobe_multi(struct kunit *test)
+{
+ struct fprobe fp1 = {
+ .entry_handler = fprobe_entry_multi_handler,
+ .exit_handler = fprobe_exit_multi_handler,
+ };
+ struct fprobe fp2 = {
+ .entry_handler = entry_only_handler,
+ };
+
+ current_test = test;
+
+ /* Test Case 1: Register in order 1 -> 2 */
+ KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
+ KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
+
+ check_fprobe_multi(test);
+
+ /* Unregister all */
+ KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
+ KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
+
+ /* Test Case 2: Register in order 2 -> 1 */
+ KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp2, "fprobe_selftest_target", NULL));
+ KUNIT_EXPECT_EQ(test, 0, register_fprobe(&fp1, "fprobe_selftest_target", NULL));
+
+ check_fprobe_multi(test);
+
+ /* Unregister all */
+ KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp1));
+ KUNIT_EXPECT_EQ(test, 0, unregister_fprobe(&fp2));
+}
+
static unsigned long get_ftrace_location(void *func)
{
unsigned long size, addr = (unsigned long)func;
@@ -217,6 +313,7 @@ static struct kunit_case fprobe_testcases[] = {
KUNIT_CASE(test_fprobe_syms),
KUNIT_CASE(test_fprobe_data),
KUNIT_CASE(test_fprobe_skip),
+ KUNIT_CASE(test_fprobe_multi),
{}
};
generated by cgit (git 2.34.1) at 2025-12-06 17:18:30 +0000