From 01ab991fc0ee5019aecc4ca461311fc8aa75ece5 Mon Sep 17 00:00:00 2001 From: Ard Biesheuvel Date: Mon, 7 Nov 2022 18:24:00 +0100 Subject: arm64: Enable data independent timing (DIT) in the kernel The ARM architecture revision v8.4 introduces a data independent timing control (DIT) which can be set at any exception level, and instructs the CPU to avoid optimizations that may result in a correlation between the execution time of certain instructions and the value of the data they operate on. The DIT bit is part of PSTATE, and is therefore context switched as usual, given that it becomes part of the saved program state (SPSR) when taking an exception. We have also defined a hwcap for DIT, and so user space can discover already whether or nor DIT is available. This means that, as far as user space is concerned, DIT is wired up and fully functional. In the kernel, however, we never bothered with DIT: we disable at it boot (i.e., INIT_PSTATE_EL1 has DIT cleared) and ignore the fact that we might run with DIT enabled if user space happened to set it. Currently, we have no idea whether or not running privileged code with DIT disabled on a CPU that implements support for it may result in a side channel that exposes privileged data to unprivileged user space processes, so let's be cautious and just enable DIT while running in the kernel if supported by all CPUs. Cc: Catalin Marinas Cc: Will Deacon Cc: Mark Rutland Cc: Marc Zyngier Cc: Eric Biggers Cc: Jason A. Donenfeld Cc: Kees Cook Cc: Suzuki K Poulose Cc: Adam Langley Link: https://lore.kernel.org/all/YwgCrqutxmX0W72r@gmail.com/ Signed-off-by: Ard Biesheuvel Acked-by: Mark Rutland Link: https://lore.kernel.org/r/20221107172400.1851434-1-ardb@kernel.org [will: Removed cpu_has_dit() as per Mark's suggestion on the list] Signed-off-by: Will Deacon --- arch/arm64/include/asm/sysreg.h | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'arch/arm64/include') diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h index 7d301700d1a9..1f3f52ce407f 100644 --- a/arch/arm64/include/asm/sysreg.h +++ b/arch/arm64/include/asm/sysreg.h @@ -90,20 +90,24 @@ */ #define pstate_field(op1, op2) ((op1) << Op1_shift | (op2) << Op2_shift) #define PSTATE_Imm_shift CRm_shift +#define SET_PSTATE(x, r) __emit_inst(0xd500401f | PSTATE_ ## r | ((!!x) << PSTATE_Imm_shift)) #define PSTATE_PAN pstate_field(0, 4) #define PSTATE_UAO pstate_field(0, 3) #define PSTATE_SSBS pstate_field(3, 1) +#define PSTATE_DIT pstate_field(3, 2) #define PSTATE_TCO pstate_field(3, 4) -#define SET_PSTATE_PAN(x) __emit_inst(0xd500401f | PSTATE_PAN | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_UAO(x) __emit_inst(0xd500401f | PSTATE_UAO | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_SSBS(x) __emit_inst(0xd500401f | PSTATE_SSBS | ((!!x) << PSTATE_Imm_shift)) -#define SET_PSTATE_TCO(x) __emit_inst(0xd500401f | PSTATE_TCO | ((!!x) << PSTATE_Imm_shift)) +#define SET_PSTATE_PAN(x) SET_PSTATE((x), PAN) +#define SET_PSTATE_UAO(x) SET_PSTATE((x), UAO) +#define SET_PSTATE_SSBS(x) SET_PSTATE((x), SSBS) +#define SET_PSTATE_DIT(x) SET_PSTATE((x), DIT) +#define SET_PSTATE_TCO(x) SET_PSTATE((x), TCO) #define set_pstate_pan(x) asm volatile(SET_PSTATE_PAN(x)) #define set_pstate_uao(x) asm volatile(SET_PSTATE_UAO(x)) #define set_pstate_ssbs(x) asm volatile(SET_PSTATE_SSBS(x)) +#define set_pstate_dit(x) asm volatile(SET_PSTATE_DIT(x)) #define __SYS_BARRIER_INSN(CRm, op2, Rt) \ __emit_inst(0xd5000000 | sys_insn(0, 3, 3, (CRm), (op2)) | ((Rt) & 0x1f)) -- cgit From 95aa6860d60881e8f08956e02199f7d65c9cbf7b Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:15 +0100 Subject: arm64/hwcap: Add support for FEAT_CSSC FEAT_CSSC adds a number of new instructions usable to optimise common short sequences of instructions, add a hwcap indicating that the feature is available and can be used by userspace. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-2-broonie@kernel.org Signed-off-by: Will Deacon --- arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + 2 files changed, 2 insertions(+) (limited to 'arch/arm64/include') diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 298b386d3ebe..a0e080df9a62 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -120,6 +120,7 @@ #define KERNEL_HWCAP_WFXT __khwcap2_feature(WFXT) #define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16) #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) +#define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 9b245da6f507..a43dddd94b4a 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -93,5 +93,6 @@ #define HWCAP2_WFXT (1UL << 31) #define HWCAP2_EBF16 (1UL << 32) #define HWCAP2_SVE_EBF16 (1UL << 33) +#define HWCAP2_CSSC (1UL << 34) #endif /* _UAPI__ASM_HWCAP_H */ -- cgit From 939e4649d4fd54b622d08cd57100828df7a82074 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:17 +0100 Subject: arm64/hwcap: Add support for FEAT_RPRFM FEAT_RPRFM adds a new range prefetch hint within the existing PRFM space for range prefetch hinting. Add a new hwcap to allow userspace to discover support for the new instruction. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-4-broonie@kernel.org Signed-off-by: Will Deacon --- arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + 2 files changed, 2 insertions(+) (limited to 'arch/arm64/include') diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index a0e080df9a62..605ec55cee70 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -121,6 +121,7 @@ #define KERNEL_HWCAP_EBF16 __khwcap2_feature(EBF16) #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) #define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) +#define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index a43dddd94b4a..063cc6ea560f 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -94,5 +94,6 @@ #define HWCAP2_EBF16 (1UL << 32) #define HWCAP2_SVE_EBF16 (1UL << 33) #define HWCAP2_CSSC (1UL << 34) +#define HWCAP2_RPRFM (1UL << 35) #endif /* _UAPI__ASM_HWCAP_H */ -- cgit From d12aada8dfb030f7df8ec0bb0ce4bb01a2ab3944 Mon Sep 17 00:00:00 2001 From: Mark Brown Date: Mon, 17 Oct 2022 16:25:19 +0100 Subject: arm64/hwcap: Add support for SVE 2.1 FEAT_SVE2p1 introduces a number of new SVE instructions. Since there is no new architectural state added kernel support is simply a new hwcap which lets userspace know that the feature is supported. Signed-off-by: Mark Brown Reviewed-by: Catalin Marinas Link: https://lore.kernel.org/r/20221017152520.1039165-6-broonie@kernel.org Signed-off-by: Will Deacon --- arch/arm64/include/asm/hwcap.h | 1 + arch/arm64/include/uapi/asm/hwcap.h | 1 + 2 files changed, 2 insertions(+) (limited to 'arch/arm64/include') diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 605ec55cee70..06dd12c514e6 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -122,6 +122,7 @@ #define KERNEL_HWCAP_SVE_EBF16 __khwcap2_feature(SVE_EBF16) #define KERNEL_HWCAP_CSSC __khwcap2_feature(CSSC) #define KERNEL_HWCAP_RPRFM __khwcap2_feature(RPRFM) +#define KERNEL_HWCAP_SVE2P1 __khwcap2_feature(SVE2P1) /* * This yields a mask that user programs can use to figure out what diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index 063cc6ea560f..b713d30544f1 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -95,5 +95,6 @@ #define HWCAP2_SVE_EBF16 (1UL << 33) #define HWCAP2_CSSC (1UL << 34) #define HWCAP2_RPRFM (1UL << 35) +#define HWCAP2_SVE2P1 (1UL << 36) #endif /* _UAPI__ASM_HWCAP_H */ -- cgit