From cbc59c9a4e5785796ccac9a975a94cb52c87feb1 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:10 +0100 Subject: arm64: idreg-override: Omit non-NULL checks for override pointer Now that override pointers are always set, we can drop the various non-NULL checks that we have in the code. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-57-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 3addc09f8746..536bc33859bc 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -216,9 +216,6 @@ static void __init match_options(const char *cmdline) for (i = 0; i < ARRAY_SIZE(regs); i++) { int f; - if (!regs[i]->override) - continue; - for (f = 0; strlen(regs[i]->fields[f].name); f++) { u64 shift = regs[i]->fields[f].shift; u64 width = regs[i]->fields[f].width ?: 4; @@ -319,10 +316,8 @@ asmlinkage void __init init_feature_override(u64 boot_status) int i; for (i = 0; i < ARRAY_SIZE(regs); i++) { - if (regs[i]->override) { - regs[i]->override->val = 0; - regs[i]->override->mask = 0; - } + regs[i]->override->val = 0; + regs[i]->override->mask = 0; } __boot_status = boot_status; @@ -330,9 +325,8 @@ asmlinkage void __init init_feature_override(u64 boot_status) parse_cmdline(); for (i = 0; i < ARRAY_SIZE(regs); i++) { - if (regs[i]->override) - dcache_clean_inval_poc((unsigned long)regs[i]->override, - (unsigned long)regs[i]->override + - sizeof(*regs[i]->override)); + dcache_clean_inval_poc((unsigned long)regs[i]->override, + (unsigned long)regs[i]->override + + sizeof(*regs[i]->override)); } } -- cgit From 01fd29092a35833ef87bd13c0a025e726550d646 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:11 +0100 Subject: arm64: idreg-override: Prepare for place relative reloc patching The ID reg override handling code uses a rather elaborate data structure that relies on statically initialized absolute address values in pointer fields. This means that this code cannot run until relocation fixups have been applied, and this is unfortunate, because it means we cannot discover overrides for KASLR or LVA/LPA without creating the kernel mapping and performing the relocations first. This can be solved by switching to place-relative relocations, which can be applied by the linker at build time. This means some additional arithmetic is required when dereferencing these pointers, as we can no longer dereference the pointer members directly. So let's implement this for idreg-override.c in a preliminary way, i.e., convert all the references in code to use a special accessor that produces the correct absolute value at runtime. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-58-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 89 ++++++++++++++++++++++++-------------- 1 file changed, 56 insertions(+), 33 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 536bc33859bc..ca1b8d2dbe99 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -21,14 +21,25 @@ static u64 __boot_status __initdata; +// temporary __prel64 related definitions +// to be removed when this code is moved under pi/ + +#define __prel64_initconst __initconst + +#define PREL64(type, name) union { type *name; } + +#define prel64_pointer(__d) (__d) + +typedef bool filter_t(u64 val); + struct ftr_set_desc { char name[FTR_DESC_NAME_LEN]; - struct arm64_ftr_override *override; + PREL64(struct arm64_ftr_override, override); struct { char name[FTR_DESC_FIELD_LEN]; u8 shift; u8 width; - bool (*filter)(u64 val); + PREL64(filter_t, filter); } fields[]; }; @@ -46,7 +57,7 @@ static bool __init mmfr1_vh_filter(u64 val) val == 0); } -static const struct ftr_set_desc mmfr1 __initconst = { +static const struct ftr_set_desc mmfr1 __prel64_initconst = { .name = "id_aa64mmfr1", .override = &id_aa64mmfr1_override, .fields = { @@ -70,7 +81,7 @@ static bool __init pfr0_sve_filter(u64 val) return true; } -static const struct ftr_set_desc pfr0 __initconst = { +static const struct ftr_set_desc pfr0 __prel64_initconst = { .name = "id_aa64pfr0", .override = &id_aa64pfr0_override, .fields = { @@ -94,7 +105,7 @@ static bool __init pfr1_sme_filter(u64 val) return true; } -static const struct ftr_set_desc pfr1 __initconst = { +static const struct ftr_set_desc pfr1 __prel64_initconst = { .name = "id_aa64pfr1", .override = &id_aa64pfr1_override, .fields = { @@ -105,7 +116,7 @@ static const struct ftr_set_desc pfr1 __initconst = { }, }; -static const struct ftr_set_desc isar1 __initconst = { +static const struct ftr_set_desc isar1 __prel64_initconst = { .name = "id_aa64isar1", .override = &id_aa64isar1_override, .fields = { @@ -117,7 +128,7 @@ static const struct ftr_set_desc isar1 __initconst = { }, }; -static const struct ftr_set_desc isar2 __initconst = { +static const struct ftr_set_desc isar2 __prel64_initconst = { .name = "id_aa64isar2", .override = &id_aa64isar2_override, .fields = { @@ -128,7 +139,7 @@ static const struct ftr_set_desc isar2 __initconst = { }, }; -static const struct ftr_set_desc smfr0 __initconst = { +static const struct ftr_set_desc smfr0 __prel64_initconst = { .name = "id_aa64smfr0", .override = &id_aa64smfr0_override, .fields = { @@ -149,7 +160,7 @@ static bool __init hvhe_filter(u64 val) ID_AA64MMFR1_EL1_VH_SHIFT)); } -static const struct ftr_set_desc sw_features __initconst = { +static const struct ftr_set_desc sw_features __prel64_initconst = { .name = "arm64_sw", .override = &arm64_sw_feature_override, .fields = { @@ -159,14 +170,15 @@ static const struct ftr_set_desc sw_features __initconst = { }, }; -static const struct ftr_set_desc * const regs[] __initconst = { - &mmfr1, - &pfr0, - &pfr1, - &isar1, - &isar2, - &smfr0, - &sw_features, +static const +PREL64(const struct ftr_set_desc, reg) regs[] __prel64_initconst = { + { &mmfr1 }, + { &pfr0 }, + { &pfr1 }, + { &isar1 }, + { &isar2 }, + { &smfr0 }, + { &sw_features }, }; static const struct { @@ -214,15 +226,20 @@ static void __init match_options(const char *cmdline) int i; for (i = 0; i < ARRAY_SIZE(regs); i++) { + const struct ftr_set_desc *reg = prel64_pointer(regs[i].reg); + struct arm64_ftr_override *override; int f; - for (f = 0; strlen(regs[i]->fields[f].name); f++) { - u64 shift = regs[i]->fields[f].shift; - u64 width = regs[i]->fields[f].width ?: 4; + override = prel64_pointer(reg->override); + + for (f = 0; strlen(reg->fields[f].name); f++) { + u64 shift = reg->fields[f].shift; + u64 width = reg->fields[f].width ?: 4; u64 mask = GENMASK_ULL(shift + width - 1, shift); + bool (*filter)(u64 val); u64 v; - if (find_field(cmdline, regs[i], f, &v)) + if (find_field(cmdline, reg, f, &v)) continue; /* @@ -230,16 +247,16 @@ static void __init match_options(const char *cmdline) * it by setting the value to the all-ones while * clearing the mask... Yes, this is fragile. */ - if (regs[i]->fields[f].filter && - !regs[i]->fields[f].filter(v)) { - regs[i]->override->val |= mask; - regs[i]->override->mask &= ~mask; + filter = prel64_pointer(reg->fields[f].filter); + if (filter && !filter(v)) { + override->val |= mask; + override->mask &= ~mask; continue; } - regs[i]->override->val &= ~mask; - regs[i]->override->val |= (v << shift) & mask; - regs[i]->override->mask |= mask; + override->val &= ~mask; + override->val |= (v << shift) & mask; + override->mask |= mask; return; } @@ -313,11 +330,16 @@ void init_feature_override(u64 boot_status); asmlinkage void __init init_feature_override(u64 boot_status) { + struct arm64_ftr_override *override; + const struct ftr_set_desc *reg; int i; for (i = 0; i < ARRAY_SIZE(regs); i++) { - regs[i]->override->val = 0; - regs[i]->override->mask = 0; + reg = prel64_pointer(regs[i].reg); + override = prel64_pointer(reg->override); + + override->val = 0; + override->mask = 0; } __boot_status = boot_status; @@ -325,8 +347,9 @@ asmlinkage void __init init_feature_override(u64 boot_status) parse_cmdline(); for (i = 0; i < ARRAY_SIZE(regs); i++) { - dcache_clean_inval_poc((unsigned long)regs[i]->override, - (unsigned long)regs[i]->override + - sizeof(*regs[i]->override)); + reg = prel64_pointer(regs[i].reg); + override = prel64_pointer(reg->override); + dcache_clean_inval_poc((unsigned long)override, + (unsigned long)(override + 1)); } } -- cgit From dc3f5aae06381b43bc9d0d416bd15ee1682940e9 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:12 +0100 Subject: arm64: idreg-override: Avoid parameq() and parameqn() The only way parameq() and parameqn() deviate from the ordinary string and memory routines is that they ignore the difference between dashes and underscores. Since we copy each command line argument into a buffer before passing it to parameq() and parameqn() numerous times, let's just convert all dashes to underscores just once, and update the alias array accordingly. This also helps reduce the dependency on kernel APIs that are no longer available once we move this code into the early mini C runtime. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-59-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index ca1b8d2dbe99..1eca93446345 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -185,8 +185,8 @@ static const struct { char alias[FTR_ALIAS_NAME_LEN]; char feature[FTR_ALIAS_OPTION_LEN]; } aliases[] __initconst = { - { "kvm-arm.mode=nvhe", "id_aa64mmfr1.vh=0" }, - { "kvm-arm.mode=protected", "id_aa64mmfr1.vh=0" }, + { "kvm_arm.mode=nvhe", "id_aa64mmfr1.vh=0" }, + { "kvm_arm.mode=protected", "id_aa64mmfr1.vh=0" }, { "arm64.nosve", "id_aa64pfr0.sve=0" }, { "arm64.nosme", "id_aa64pfr1.sme=0" }, { "arm64.nobti", "id_aa64pfr1.bt=0" }, @@ -215,7 +215,7 @@ static int __init find_field(const char *cmdline, len = snprintf(opt, ARRAY_SIZE(opt), "%s.%s=", reg->name, reg->fields[f].name); - if (!parameqn(cmdline, opt, len)) + if (memcmp(cmdline, opt, len)) return -1; return kstrtou64(cmdline + len, 0, v); @@ -272,23 +272,29 @@ static __init void __parse_cmdline(const char *cmdline, bool parse_aliases) cmdline = skip_spaces(cmdline); - for (len = 0; cmdline[len] && !isspace(cmdline[len]); len++); - if (!len) + /* terminate on "--" appearing on the command line by itself */ + if (cmdline[0] == '-' && cmdline[1] == '-' && isspace(cmdline[2])) return; - len = min(len, ARRAY_SIZE(buf) - 1); - memcpy(buf, cmdline, len); - buf[len] = '\0'; - - if (strcmp(buf, "--") == 0) + for (len = 0; cmdline[len] && !isspace(cmdline[len]); len++) { + if (len >= sizeof(buf) - 1) + break; + if (cmdline[len] == '-') + buf[len] = '_'; + else + buf[len] = cmdline[len]; + } + if (!len) return; + buf[len] = 0; + cmdline += len; match_options(buf); for (i = 0; parse_aliases && i < ARRAY_SIZE(aliases); i++) - if (parameq(buf, aliases[i].alias)) + if (!memcmp(buf, aliases[i].alias, len + 1)) __parse_cmdline(aliases[i].feature, false); } while (1); } -- cgit From bcf1eed3f8a0b83824b41da82d226cf36320be76 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:13 +0100 Subject: arm64: idreg-override: avoid strlen() to check for empty strings strlen() is a costly way to decide whether a string is empty, as in that case, the first character will be NUL so we can check for that directly. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-60-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 1eca93446345..8b22ca523186 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -232,7 +232,7 @@ static void __init match_options(const char *cmdline) override = prel64_pointer(reg->override); - for (f = 0; strlen(reg->fields[f].name); f++) { + for (f = 0; reg->fields[f].name[0] != '\0'; f++) { u64 shift = reg->fields[f].shift; u64 width = reg->fields[f].width ?: 4; u64 mask = GENMASK_ULL(shift + width - 1, shift); -- cgit From 060260a6be47ae163b0c71a6d0902d065b68f3d2 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:14 +0100 Subject: arm64: idreg-override: Avoid sprintf() for simple string concatenation Instead of using sprintf() with the "%s.%s=" format, where the first string argument is always the same in the inner loop of match_options(), use simple memcpy() for string concatenation, and move the first copy to the outer loop. This removes the dependency on sprintf(), which will be difficult to fulfil when we move this code into the early mini C runtime. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-61-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 8b22ca523186..cf1df5f6fbc1 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -206,14 +206,15 @@ static int __init parse_nokaslr(char *unused) } early_param("nokaslr", parse_nokaslr); -static int __init find_field(const char *cmdline, +static int __init find_field(const char *cmdline, char *opt, int len, const struct ftr_set_desc *reg, int f, u64 *v) { - char opt[FTR_DESC_NAME_LEN + FTR_DESC_FIELD_LEN + 2]; - int len; + int flen = strlen(reg->fields[f].name); - len = snprintf(opt, ARRAY_SIZE(opt), "%s.%s=", - reg->name, reg->fields[f].name); + // append '=' to obtain '.=' + memcpy(opt + len, reg->fields[f].name, flen); + len += flen; + opt[len++] = '='; if (memcmp(cmdline, opt, len)) return -1; @@ -223,15 +224,21 @@ static int __init find_field(const char *cmdline, static void __init match_options(const char *cmdline) { + char opt[FTR_DESC_NAME_LEN + FTR_DESC_FIELD_LEN + 2]; int i; for (i = 0; i < ARRAY_SIZE(regs); i++) { const struct ftr_set_desc *reg = prel64_pointer(regs[i].reg); struct arm64_ftr_override *override; + int len = strlen(reg->name); int f; override = prel64_pointer(reg->override); + // set opt[] to '.' + memcpy(opt, reg->name, len); + opt[len++] = '.'; + for (f = 0; reg->fields[f].name[0] != '\0'; f++) { u64 shift = reg->fields[f].shift; u64 width = reg->fields[f].width ?: 4; @@ -239,7 +246,7 @@ static void __init match_options(const char *cmdline) bool (*filter)(u64 val); u64 v; - if (find_field(cmdline, reg, f, &v)) + if (find_field(cmdline, opt, len, reg, f, &v)) continue; /* -- cgit From ea48626f8f0efc697555d17bb5853df92461e280 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:15 +0100 Subject: arm64: idreg-override: Avoid kstrtou64() to parse a single hex digit All ID register value overrides are =0 with the exception of the nokaslr pseudo feature which uses =1. In order to remove the dependency on kstrtou64(), which is part of the core kernel and no longer usable once we move idreg-override into the early mini C runtime, let's just parse a single hex digit (with optional leading 0x) and set the output value accordingly. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-62-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index cf1df5f6fbc1..9646f94094ed 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -206,6 +206,20 @@ static int __init parse_nokaslr(char *unused) } early_param("nokaslr", parse_nokaslr); +static int __init parse_hexdigit(const char *p, u64 *v) +{ + // skip "0x" if it comes next + if (p[0] == '0' && tolower(p[1]) == 'x') + p += 2; + + // check whether the RHS is a single hex digit + if (!isxdigit(p[0]) || (p[1] && !isspace(p[1]))) + return -EINVAL; + + *v = tolower(*p) - (isdigit(*p) ? '0' : 'a' - 10); + return 0; +} + static int __init find_field(const char *cmdline, char *opt, int len, const struct ftr_set_desc *reg, int f, u64 *v) { @@ -219,7 +233,7 @@ static int __init find_field(const char *cmdline, char *opt, int len, if (memcmp(cmdline, opt, len)) return -1; - return kstrtou64(cmdline + len, 0, v); + return parse_hexdigit(cmdline + len, v); } static void __init match_options(const char *cmdline) -- cgit From 50f176175e96ea5d7cbd8536c0dd774de796ef63 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Wed, 29 Nov 2023 12:16:16 +0100 Subject: arm64/kernel: Move 'nokaslr' parsing out of early idreg code Parsing and ignoring 'nokaslr' can be done from anywhere, except from the code that runs very early and is therefore built with limitations on the kind of relocations it is permitted to use. So move it to a source file that is part of the ordinary kernel build. Signed-off-by: Ard Biesheuvel Link: https://lore.kernel.org/r/20231129111555.3594833-63-ardb@google.com Signed-off-by: Will Deacon --- arch/arm64/kernel/idreg-override.c | 7 ------- arch/arm64/kernel/kaslr.c | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) (limited to 'arch/arm64/kernel') diff --git a/arch/arm64/kernel/idreg-override.c b/arch/arm64/kernel/idreg-override.c index 9646f94094ed..e30fd9e32ef3 100644 --- a/arch/arm64/kernel/idreg-override.c +++ b/arch/arm64/kernel/idreg-override.c @@ -199,13 +199,6 @@ static const struct { { "nokaslr", "arm64_sw.nokaslr=1" }, }; -static int __init parse_nokaslr(char *unused) -{ - /* nokaslr param handling is done by early cpufeature code */ - return 0; -} -early_param("nokaslr", parse_nokaslr); - static int __init parse_hexdigit(const char *p, u64 *v) { // skip "0x" if it comes next diff --git a/arch/arm64/kernel/kaslr.c b/arch/arm64/kernel/kaslr.c index 94a269cd1f07..12c7f3c8ba76 100644 --- a/arch/arm64/kernel/kaslr.c +++ b/arch/arm64/kernel/kaslr.c @@ -36,3 +36,10 @@ void __init kaslr_init(void) pr_info("KASLR enabled\n"); __kaslr_is_enabled = true; } + +static int __init parse_nokaslr(char *unused) +{ + /* nokaslr param handling is done by early cpufeature code */ + return 0; +} +early_param("nokaslr", parse_nokaslr); -- cgit