diff options
Diffstat (limited to 'arch/riscv')
64 files changed, 1091 insertions, 346 deletions
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index 8ebafe337eac..be09c8836d56 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -41,7 +41,7 @@ config RISCV select ARCH_HAS_SYNC_CORE_BEFORE_USERMODE select ARCH_HAS_SYSCALL_WRAPPER select ARCH_HAS_TICK_BROADCAST if GENERIC_CLOCKEVENTS_BROADCAST - select ARCH_HAS_UBSAN_SANITIZE_ALL + select ARCH_HAS_UBSAN select ARCH_HAS_VDSO_DATA select ARCH_KEEP_MEMBLOCK if ACPI select ARCH_OPTIONAL_KERNEL_RWX if ARCH_HAS_STRICT_KERNEL_RWX @@ -145,6 +145,7 @@ config RISCV select HAVE_LD_DEAD_CODE_DATA_ELIMINATION if !LD_IS_LLD select HAVE_MOVE_PMD select HAVE_MOVE_PUD + select HAVE_PAGE_SIZE_4KB select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS @@ -184,8 +185,6 @@ config RISCV config CLANG_SUPPORTS_DYNAMIC_FTRACE def_bool CC_IS_CLANG - # https://github.com/llvm/llvm-project/commit/6ab8927931851bb42b2c93a00801dc499d7d9b1e - depends on CLANG_VERSION >= 130000 # https://github.com/ClangBuiltLinux/linux/issues/1817 depends on AS_IS_GNU || (AS_IS_LLVM && (LD_IS_LLD || LD_VERSION >= 23600)) @@ -322,7 +321,7 @@ config AS_HAS_INSN def_bool $(as-instr,.insn r 51$(comma) 0$(comma) 0$(comma) t0$(comma) t0$(comma) zero) config AS_HAS_OPTION_ARCH - # https://reviews.llvm.org/D123515 + # https://github.com/llvm/llvm-project/commit/9e8ed3403c191ab9c4903e8eeb8f732ff8a43cb4 def_bool y depends on $(as-instr, .option arch$(comma) +m) @@ -817,7 +816,7 @@ config ARCH_SUPPORTS_CRASH_DUMP def_bool y config ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION - def_bool CRASH_CORE + def_bool CRASH_RESERVE config COMPAT bool "Kernel support for 32-bit U-mode" @@ -1033,7 +1032,19 @@ config RISCV_ISA_FALLBACK config BUILTIN_DTB bool "Built-in device tree" depends on OF && NONPORTABLE - default y if XIP_KERNEL + help + Build a device tree into the Linux image. + This option should be selected if no bootloader is being used. + If unsure, say N. + + +config BUILTIN_DTB_SOURCE + string "Built-in device tree source" + depends on BUILTIN_DTB + help + DTS file path (without suffix, relative to arch/riscv/boot/dts) + for the DTS file that will be used to produce the DTB linked into the + kernel. endmenu # "Boot options" diff --git a/arch/riscv/Kconfig.socs b/arch/riscv/Kconfig.socs index e08e91c49abe..623de5f8a208 100644 --- a/arch/riscv/Kconfig.socs +++ b/arch/riscv/Kconfig.socs @@ -84,36 +84,4 @@ config SOC_CANAAN help This enables support for Canaan Kendryte K210 SoC platform hardware. -if ARCH_CANAAN - -config ARCH_CANAAN_K210_DTB_BUILTIN - def_bool SOC_CANAAN_K210_DTB_BUILTIN - -config SOC_CANAAN_K210_DTB_BUILTIN - bool "Builtin device tree for the Canaan Kendryte K210" - depends on ARCH_CANAAN - default y - select OF - select BUILTIN_DTB - help - Build a device tree for the Kendryte K210 into the Linux image. - This option should be selected if no bootloader is being used. - If unsure, say Y. - -config ARCH_CANAAN_K210_DTB_SOURCE - string - default SOC_CANAAN_K210_DTB_SOURCE - -config SOC_CANAAN_K210_DTB_SOURCE - string "Source file for the Canaan Kendryte K210 builtin DTB" - depends on ARCH_CANAAN - depends on ARCH_CANAAN_K210_DTB_BUILTIN - default "k210_generic" - help - Base name (without suffix, relative to arch/riscv/boot/dts/canaan) - for the DTS file that will be used to produce the DTB linked into the - kernel. - -endif # ARCH_CANAAN - endmenu # "SoC selection" diff --git a/arch/riscv/boot/dts/Makefile b/arch/riscv/boot/dts/Makefile index 72030fd727af..fdae05bbf556 100644 --- a/arch/riscv/boot/dts/Makefile +++ b/arch/riscv/boot/dts/Makefile @@ -8,4 +8,4 @@ subdir-y += sophgo subdir-y += starfive subdir-y += thead -obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix /, $(subdir-y)) +obj-$(CONFIG_BUILTIN_DTB) := $(addsuffix .dtb.o, $(CONFIG_BUILTIN_DTB_SOURCE)) diff --git a/arch/riscv/boot/dts/canaan/Makefile b/arch/riscv/boot/dts/canaan/Makefile index 520623264c87..987d1f0c41f0 100644 --- a/arch/riscv/boot/dts/canaan/Makefile +++ b/arch/riscv/boot/dts/canaan/Makefile @@ -5,5 +5,3 @@ dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_bit.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_dock.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maix_go.dtb dtb-$(CONFIG_ARCH_CANAAN) += sipeed_maixduino.dtb - -obj-$(CONFIG_ARCH_CANAAN_K210_DTB_BUILTIN) += $(addsuffix .dtb.o, $(CONFIG_ARCH_CANAAN_K210_DTB_SOURCE)) diff --git a/arch/riscv/boot/dts/microchip/Makefile b/arch/riscv/boot/dts/microchip/Makefile index 45adc4926e79..e177815bf1a2 100644 --- a/arch/riscv/boot/dts/microchip/Makefile +++ b/arch/riscv/boot/dts/microchip/Makefile @@ -4,4 +4,3 @@ dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-m100pfsevp.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-polarberry.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-sev-kit.dtb dtb-$(CONFIG_ARCH_MICROCHIP_POLARFIRE) += mpfs-tysom-m.dtb -obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/microchip/mpfs.dtsi b/arch/riscv/boot/dts/microchip/mpfs.dtsi index 59fd2d4ea523..9883ca3554c5 100644 --- a/arch/riscv/boot/dts/microchip/mpfs.dtsi +++ b/arch/riscv/boot/dts/microchip/mpfs.dtsi @@ -243,7 +243,7 @@ }; pdma: dma-controller@3000000 { - compatible = "sifive,fu540-c000-pdma", "sifive,pdma0"; + compatible = "microchip,mpfs-pdma", "sifive,pdma0"; reg = <0x0 0x3000000 0x0 0x8000>; interrupt-parent = <&plic>; interrupts = <5 6>, <7 8>, <9 10>, <11 12>; @@ -422,7 +422,7 @@ can0: can@2010c000 { compatible = "microchip,mpfs-can"; reg = <0x0 0x2010c000 0x0 0x1000>; - clocks = <&clkcfg CLK_CAN0>; + clocks = <&clkcfg CLK_CAN0>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <56>; status = "disabled"; @@ -431,7 +431,7 @@ can1: can@2010d000 { compatible = "microchip,mpfs-can"; reg = <0x0 0x2010d000 0x0 0x1000>; - clocks = <&clkcfg CLK_CAN1>; + clocks = <&clkcfg CLK_CAN1>, <&clkcfg CLK_MSSPLL3>; interrupt-parent = <&plic>; interrupts = <57>; status = "disabled"; diff --git a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi index d7a66043f13b..f35324b9173c 100644 --- a/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi +++ b/arch/riscv/boot/dts/renesas/r9a07g043f.dtsi @@ -46,6 +46,10 @@ }; }; +&pinctrl { + gpio-ranges = <&pinctrl 0 0 232>; +}; + &soc { dma-noncoherent; interrupt-parent = <&plic>; diff --git a/arch/riscv/boot/dts/sifive/Makefile b/arch/riscv/boot/dts/sifive/Makefile index 6a5fbd4ed96a..495bf760a909 100644 --- a/arch/riscv/boot/dts/sifive/Makefile +++ b/arch/riscv/boot/dts/sifive/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0 dtb-$(CONFIG_ARCH_SIFIVE) += hifive-unleashed-a00.dtb \ hifive-unmatched-a00.dtb -obj-$(CONFIG_BUILTIN_DTB) += $(addsuffix .o, $(dtb-y)) diff --git a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts index 07387f9c135c..72b87b08ab44 100644 --- a/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts +++ b/arch/riscv/boot/dts/sifive/hifive-unmatched-a00.dts @@ -123,6 +123,7 @@ interrupt-parent = <&gpio>; interrupts = <1 IRQ_TYPE_LEVEL_LOW>; interrupt-controller; + #interrupt-cells = <2>; onkey { compatible = "dlg,da9063-onkey"; diff --git a/arch/riscv/boot/dts/sophgo/sg2042.dtsi b/arch/riscv/boot/dts/sophgo/sg2042.dtsi index 93256540d078..81fda312f988 100644 --- a/arch/riscv/boot/dts/sophgo/sg2042.dtsi +++ b/arch/riscv/boot/dts/sophgo/sg2042.dtsi @@ -6,6 +6,8 @@ /dts-v1/; #include <dt-bindings/interrupt-controller/irq.h> +#include <dt-bindings/reset/sophgo,sg2042-reset.h> + #include "sg2042-cpus.dtsi" / { @@ -93,144 +95,160 @@ <&cpu63_intc 3>; }; - clint_mtimer0: timer@70ac000000 { + clint_mtimer0: timer@70ac004000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac000000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac004000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu0_intc 7>, <&cpu1_intc 7>, <&cpu2_intc 7>, <&cpu3_intc 7>; }; - clint_mtimer1: timer@70ac010000 { + clint_mtimer1: timer@70ac014000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac010000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac014000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu4_intc 7>, <&cpu5_intc 7>, <&cpu6_intc 7>, <&cpu7_intc 7>; }; - clint_mtimer2: timer@70ac020000 { + clint_mtimer2: timer@70ac024000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac020000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac024000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu8_intc 7>, <&cpu9_intc 7>, <&cpu10_intc 7>, <&cpu11_intc 7>; }; - clint_mtimer3: timer@70ac030000 { + clint_mtimer3: timer@70ac034000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac030000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac034000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu12_intc 7>, <&cpu13_intc 7>, <&cpu14_intc 7>, <&cpu15_intc 7>; }; - clint_mtimer4: timer@70ac040000 { + clint_mtimer4: timer@70ac044000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac040000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac044000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu16_intc 7>, <&cpu17_intc 7>, <&cpu18_intc 7>, <&cpu19_intc 7>; }; - clint_mtimer5: timer@70ac050000 { + clint_mtimer5: timer@70ac054000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac050000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac054000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu20_intc 7>, <&cpu21_intc 7>, <&cpu22_intc 7>, <&cpu23_intc 7>; }; - clint_mtimer6: timer@70ac060000 { + clint_mtimer6: timer@70ac064000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac060000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac064000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu24_intc 7>, <&cpu25_intc 7>, <&cpu26_intc 7>, <&cpu27_intc 7>; }; - clint_mtimer7: timer@70ac070000 { + clint_mtimer7: timer@70ac074000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac070000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac074000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu28_intc 7>, <&cpu29_intc 7>, <&cpu30_intc 7>, <&cpu31_intc 7>; }; - clint_mtimer8: timer@70ac080000 { + clint_mtimer8: timer@70ac084000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac080000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac084000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu32_intc 7>, <&cpu33_intc 7>, <&cpu34_intc 7>, <&cpu35_intc 7>; }; - clint_mtimer9: timer@70ac090000 { + clint_mtimer9: timer@70ac094000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac090000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac094000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu36_intc 7>, <&cpu37_intc 7>, <&cpu38_intc 7>, <&cpu39_intc 7>; }; - clint_mtimer10: timer@70ac0a0000 { + clint_mtimer10: timer@70ac0a4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0a0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0a4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu40_intc 7>, <&cpu41_intc 7>, <&cpu42_intc 7>, <&cpu43_intc 7>; }; - clint_mtimer11: timer@70ac0b0000 { + clint_mtimer11: timer@70ac0b4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0b0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0b4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu44_intc 7>, <&cpu45_intc 7>, <&cpu46_intc 7>, <&cpu47_intc 7>; }; - clint_mtimer12: timer@70ac0c0000 { + clint_mtimer12: timer@70ac0c4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0c0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0c4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu48_intc 7>, <&cpu49_intc 7>, <&cpu50_intc 7>, <&cpu51_intc 7>; }; - clint_mtimer13: timer@70ac0d0000 { + clint_mtimer13: timer@70ac0d4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0d0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0d4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu52_intc 7>, <&cpu53_intc 7>, <&cpu54_intc 7>, <&cpu55_intc 7>; }; - clint_mtimer14: timer@70ac0e0000 { + clint_mtimer14: timer@70ac0e4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0e0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0e4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu56_intc 7>, <&cpu57_intc 7>, <&cpu58_intc 7>, <&cpu59_intc 7>; }; - clint_mtimer15: timer@70ac0f0000 { + clint_mtimer15: timer@70ac0f4000 { compatible = "sophgo,sg2042-aclint-mtimer", "thead,c900-aclint-mtimer"; - reg = <0x00000070 0xac0f0000 0x00000000 0x00007ff8>; + reg = <0x00000070 0xac0f4000 0x00000000 0x0000c000>; + reg-names = "mtimecmp"; interrupts-extended = <&cpu60_intc 7>, <&cpu61_intc 7>, <&cpu62_intc 7>, @@ -311,6 +329,12 @@ riscv,ndev = <224>; }; + rstgen: reset-controller@7030013000 { + compatible = "sophgo,sg2042-reset"; + reg = <0x00000070 0x30013000 0x00000000 0x0000000c>; + #reset-cells = <1>; + }; + uart0: serial@7040000000 { compatible = "snps,dw-apb-uart"; reg = <0x00000070 0x40000000 0x00000000 0x00001000>; @@ -319,6 +343,7 @@ clock-frequency = <500000000>; reg-shift = <2>; reg-io-width = <4>; + resets = <&rstgen RST_UART0>; status = "disabled"; }; }; diff --git a/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts b/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts index 7cda3a89020a..168f5d9895a9 100644 --- a/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts +++ b/arch/riscv/boot/dts/starfive/jh7100-beaglev-starlight.dts @@ -11,3 +11,14 @@ model = "BeagleV Starlight Beta"; compatible = "beagle,beaglev-starlight-jh7100-r0", "starfive,jh7100"; }; + +&gmac { + phy-handle = <&phy>; +}; + +&mdio { + phy: ethernet-phy@7 { + reg = <7>; + reset-gpios = <&gpio 63 GPIO_ACTIVE_LOW>; + }; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi index 42fb61c36068..ae1a6aeb0aea 100644 --- a/arch/riscv/boot/dts/starfive/jh7100-common.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7100-common.dtsi @@ -72,7 +72,91 @@ }; }; +&gmac { + pinctrl-names = "default"; + pinctrl-0 = <&gmac_pins>; + phy-mode = "rgmii-id"; + status = "okay"; + + mdio: mdio { + #address-cells = <1>; + #size-cells = <0>; + compatible = "snps,dwmac-mdio"; + }; +}; + &gpio { + gmac_pins: gmac-0 { + gtxclk-pins { + pins = <PAD_FUNC_SHARE(115)>; + bias-pull-up; + drive-strength = <35>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + miitxclk-pins { + pins = <PAD_FUNC_SHARE(116)>; + bias-pull-up; + drive-strength = <14>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + tx-pins { + pins = <PAD_FUNC_SHARE(117)>, + <PAD_FUNC_SHARE(119)>, + <PAD_FUNC_SHARE(120)>, + <PAD_FUNC_SHARE(121)>, + <PAD_FUNC_SHARE(122)>, + <PAD_FUNC_SHARE(123)>, + <PAD_FUNC_SHARE(124)>, + <PAD_FUNC_SHARE(125)>, + <PAD_FUNC_SHARE(126)>; + bias-pull-up; + drive-strength = <35>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + rxclk-pins { + pins = <PAD_FUNC_SHARE(127)>; + bias-pull-up; + drive-strength = <14>; + input-enable; + input-schmitt-disable; + slew-rate = <6>; + }; + rxer-pins { + pins = <PAD_FUNC_SHARE(129)>; + bias-pull-up; + drive-strength = <14>; + input-enable; + input-schmitt-disable; + slew-rate = <0>; + }; + rx-pins { + pins = <PAD_FUNC_SHARE(128)>, + <PAD_FUNC_SHARE(130)>, + <PAD_FUNC_SHARE(131)>, + <PAD_FUNC_SHARE(132)>, + <PAD_FUNC_SHARE(133)>, + <PAD_FUNC_SHARE(134)>, + <PAD_FUNC_SHARE(135)>, + <PAD_FUNC_SHARE(136)>, + <PAD_FUNC_SHARE(137)>, + <PAD_FUNC_SHARE(138)>, + <PAD_FUNC_SHARE(139)>, + <PAD_FUNC_SHARE(140)>, + <PAD_FUNC_SHARE(141)>; + bias-pull-up; + drive-strength = <14>; + input-enable; + input-schmitt-enable; + slew-rate = <0>; + }; + }; + i2c0_pins: i2c0-0 { i2c-pins { pinmux = <GPIOMUX(62, GPO_LOW, @@ -115,6 +199,24 @@ }; }; + pwm_pins: pwm-0 { + pwm-pins { + pinmux = <GPIOMUX(7, + GPO_PWM_PAD_OUT_BIT0, + GPO_PWM_PAD_OE_N_BIT0, + GPI_NONE)>, + <GPIOMUX(5, + GPO_PWM_PAD_OUT_BIT1, + GPO_PWM_PAD_OE_N_BIT1, + GPI_NONE)>; + bias-disable; + drive-strength = <35>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + sdio0_pins: sdio0-0 { clk-pins { pinmux = <GPIOMUX(54, GPO_SDIO0_PAD_CCLK_OUT, @@ -257,6 +359,12 @@ clock-frequency = <27000000>; }; +&pwm { + pinctrl-names = "default"; + pinctrl-0 = <&pwm_pins>; + status = "okay"; +}; + &sdio0 { broken-cd; bus-width = <4>; diff --git a/arch/riscv/boot/dts/starfive/jh7100-starfive-visionfive-v1.dts b/arch/riscv/boot/dts/starfive/jh7100-starfive-visionfive-v1.dts index e82af72f1aaf..692c696e1ab4 100644 --- a/arch/riscv/boot/dts/starfive/jh7100-starfive-visionfive-v1.dts +++ b/arch/riscv/boot/dts/starfive/jh7100-starfive-visionfive-v1.dts @@ -6,7 +6,6 @@ /dts-v1/; #include "jh7100-common.dtsi" -#include <dt-bindings/gpio/gpio.h> / { model = "StarFive VisionFive V1"; @@ -18,3 +17,24 @@ priority = <224>; }; }; + +&gmac { + phy-handle = <&phy>; +}; + +/* + * The board uses a Motorcomm YT8521 PHY supporting RGMII-ID, but requires + * manual adjustment of the RX internal delay to work properly. The default + * RX delay provided by the driver (1.95ns) is too high, but applying a 50% + * reduction seems to mitigate the issue. + * + * It is worth noting the adjustment is not necessary on BeagleV Starlight SBC, + * which uses a Microchip PHY. Hence, most likely the Motorcomm PHY is the one + * responsible for the misbehaviour, not the GMAC. + */ +&mdio { + phy: ethernet-phy@0 { + reg = <0>; + rx-internal-delay-ps = <900>; + }; +}; diff --git a/arch/riscv/boot/dts/starfive/jh7100.dtsi b/arch/riscv/boot/dts/starfive/jh7100.dtsi index c216aaecac53..9a2e9583af88 100644 --- a/arch/riscv/boot/dts/starfive/jh7100.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7100.dtsi @@ -96,14 +96,14 @@ thermal-sensors = <&sfctemp>; trips { - cpu_alert0 { + cpu-alert0 { /* milliCelsius */ temperature = <75000>; hysteresis = <2000>; type = "passive"; }; - cpu_crit { + cpu-crit { /* milliCelsius */ temperature = <90000>; hysteresis = <2000>; @@ -113,30 +113,34 @@ }; }; - osc_sys: osc_sys { + osc_sys: osc-sys { compatible = "fixed-clock"; #clock-cells = <0>; + clock-output-names = "osc_sys"; /* This value must be overridden by the board */ clock-frequency = <0>; }; - osc_aud: osc_aud { + osc_aud: osc-aud { compatible = "fixed-clock"; #clock-cells = <0>; + clock-output-names = "osc_aud"; /* This value must be overridden by the board */ clock-frequency = <0>; }; - gmac_rmii_ref: gmac_rmii_ref { + gmac_rmii_ref: gmac-rmii-ref { compatible = "fixed-clock"; #clock-cells = <0>; + clock-output-names = "gmac_rmii_ref"; /* Should be overridden by the board when needed */ clock-frequency = <0>; }; - gmac_gr_mii_rxclk: gmac_gr_mii_rxclk { + gmac_gr_mii_rxclk: gmac-gr-mii-rxclk { compatible = "fixed-clock"; #clock-cells = <0>; + clock-output-names = "gmac_gr_mii_rxclk"; /* Should be overridden by the board when needed */ clock-frequency = <0>; }; @@ -204,6 +208,37 @@ status = "disabled"; }; + gmac: ethernet@10020000 { + compatible = "starfive,jh7100-dwmac", "snps,dwmac"; + reg = <0x0 0x10020000 0x0 0x10000>; + clocks = <&clkgen JH7100_CLK_GMAC_ROOT_DIV>, + <&clkgen JH7100_CLK_GMAC_AHB>, + <&clkgen JH7100_CLK_GMAC_PTP_REF>, + <&clkgen JH7100_CLK_GMAC_TX_INV>, + <&clkgen JH7100_CLK_GMAC_GTX>; + clock-names = "stmmaceth", "pclk", "ptp_ref", "tx", "gtx"; + resets = <&rstgen JH7100_RSTN_GMAC_AHB>; + reset-names = "ahb"; + interrupts = <6>, <7>; + interrupt-names = "macirq", "eth_wake_irq"; + max-frame-size = <9000>; + snps,multicast-filter-bins = <32>; + snps,perfect-filter-entries = <128>; + starfive,syscon = <&sysmain 0x70 0>; + rx-fifo-depth = <32768>; + tx-fifo-depth = <16384>; + snps,axi-config = <&stmmac_axi_setup>; + snps,fixed-burst; + snps,force_thresh_dma_mode; + status = "disabled"; + + stmmac_axi_setup: stmmac-axi-config { + snps,wr_osr_lmt = <16>; + snps,rd_osr_lmt = <16>; + snps,blen = <256 128 64 32 0 0 0>; + }; + }; + clkgen: clock-controller@11800000 { compatible = "starfive,jh7100-clkgen"; reg = <0x0 0x11800000 0x0 0x10000>; @@ -218,6 +253,11 @@ #reset-cells = <1>; }; + sysmain: syscon@11850000 { + compatible = "starfive,jh7100-sysmain", "syscon"; + reg = <0x0 0x11850000 0x0 0x10000>; + }; + i2c0: i2c@118b0000 { compatible = "snps,designware-i2c"; reg = <0x0 0x118b0000 0x0 0x10000>; @@ -320,6 +360,15 @@ <&rstgen JH7100_RSTN_WDT>; }; + pwm: pwm@12490000 { + compatible = "starfive,jh7100-pwm", "opencores,pwm-v1"; + reg = <0x0 0x12490000 0x0 0x10000>; + clocks = <&clkgen JH7100_CLK_PWM_APB>; + resets = <&rstgen JH7100_RSTN_PWM_APB>; + #pwm-cells = <3>; + status = "disabled"; + }; + sfctemp: temperature-sensor@124a0000 { compatible = "starfive,jh7100-temp"; reg = <0x0 0x124a0000 0x0 0x10000>; diff --git a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi index b89e9791efa7..45b58b6f3df8 100644 --- a/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110-starfive-visionfive-2.dtsi @@ -125,6 +125,55 @@ clock-frequency = <49152000>; }; +&camss { + assigned-clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, + <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>; + assigned-clock-rates = <49500000>, <198000000>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + }; + + port@1 { + reg = <1>; + + camss_from_csi2rx: endpoint { + remote-endpoint = <&csi2rx_to_camss>; + }; + }; + }; +}; + +&csi2rx { + assigned-clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>; + assigned-clock-rates = <297000000>; + status = "okay"; + + ports { + #address-cells = <1>; + #size-cells = <0>; + + port@0 { + reg = <0>; + + /* remote MIPI sensor endpoint */ + }; + + port@1 { + reg = <1>; + + csi2rx_to_camss: endpoint { + remote-endpoint = <&camss_from_csi2rx>; + }; + }; + }; +}; + &gmac0 { phy-handle = <&phy0>; phy-mode = "rgmii-id"; @@ -323,6 +372,12 @@ }; }; +&pwm { + pinctrl-names = "default"; + pinctrl-0 = <&pwm_pins>; + status = "okay"; +}; + &spi0 { pinctrl-names = "default"; pinctrl-0 = <&spi0_pins>; @@ -513,6 +568,22 @@ }; }; + pwm_pins: pwm-0 { + pwm-pins { + pinmux = <GPIOMUX(46, GPOUT_SYS_PWM_CHANNEL0, + GPOEN_SYS_PWM0_CHANNEL0, + GPI_NONE)>, + <GPIOMUX(59, GPOUT_SYS_PWM_CHANNEL1, + GPOEN_SYS_PWM0_CHANNEL1, + GPI_NONE)>; + bias-disable; + drive-strength = <12>; + input-disable; + input-schmitt-disable; + slew-rate = <0>; + }; + }; + spi0_pins: spi0-0 { mosi-pins { pinmux = <GPIOMUX(52, GPOUT_SYS_SPI0_TXD, diff --git a/arch/riscv/boot/dts/starfive/jh7110.dtsi b/arch/riscv/boot/dts/starfive/jh7110.dtsi index 45213cdf50dc..4a5708f7fcf7 100644 --- a/arch/riscv/boot/dts/starfive/jh7110.dtsi +++ b/arch/riscv/boot/dts/starfive/jh7110.dtsi @@ -237,14 +237,14 @@ }; trips { - cpu_alert0: cpu_alert0 { + cpu_alert0: cpu-alert0 { /* milliCelsius */ temperature = <85000>; hysteresis = <2000>; type = "passive"; }; - cpu_crit { + cpu-crit { /* milliCelsius */ temperature = <100000>; hysteresis = <2000>; @@ -829,6 +829,15 @@ status = "disabled"; }; + pwm: pwm@120d0000 { + compatible = "starfive,jh7110-pwm", "opencores,pwm-v1"; + reg = <0x0 0x120d0000 0x0 0x10000>; + clocks = <&syscrg JH7110_SYSCLK_PWM_APB>; + resets = <&syscrg JH7110_SYSRST_PWM_APB>; + #pwm-cells = <3>; + status = "disabled"; + }; + sfctemp: temperature-sensor@120e0000 { compatible = "starfive,jh7110-temp"; reg = <0x0 0x120e0000 0x0 0x10000>; @@ -1104,6 +1113,32 @@ #power-domain-cells = <1>; }; + csi2rx: csi@19800000 { + compatible = "starfive,jh7110-csi2rx", "cdns,csi2rx"; + reg = <0x0 0x19800000 0x0 0x10000>; + clocks = <&ispcrg JH7110_ISPCLK_VIN_SYS>, + <&ispcrg JH7110_ISPCLK_VIN_APB>, + <&ispcrg JH7110_ISPCLK_VIN_PIXEL_IF0>, + <&ispcrg JH7110_ISPCLK_VIN_PIXEL_IF1>, + <&ispcrg JH7110_ISPCLK_VIN_PIXEL_IF2>, + <&ispcrg JH7110_ISPCLK_VIN_PIXEL_IF3>; + clock-names = "sys_clk", "p_clk", + "pixel_if0_clk", "pixel_if1_clk", + "pixel_if2_clk", "pixel_if3_clk"; + resets = <&ispcrg JH7110_ISPRST_VIN_SYS>, + <&ispcrg JH7110_ISPRST_VIN_APB>, + <&ispcrg JH7110_ISPRST_VIN_PIXEL_IF0>, + <&ispcrg JH7110_ISPRST_VIN_PIXEL_IF1>, + <&ispcrg JH7110_ISPRST_VIN_PIXEL_IF2>, + <&ispcrg JH7110_ISPRST_VIN_PIXEL_IF3>; + reset-names = "sys", "reg_bank", + "pixel_if0", "pixel_if1", + "pixel_if2", "pixel_if3"; + phys = <&csi_phy>; + phy-names = "dphy"; + status = "disabled"; + }; + ispcrg: clock-controller@19810000 { compatible = "starfive,jh7110-ispcrg"; reg = <0x0 0x19810000 0x0 0x10000>; @@ -1121,6 +1156,47 @@ power-domains = <&pwrc JH7110_PD_ISP>; }; + csi_phy: phy@19820000 { + compatible = "starfive,jh7110-dphy-rx"; + reg = <0x0 0x19820000 0x0 0x10000>; + clocks = <&ispcrg JH7110_ISPCLK_M31DPHY_CFG_IN>, + <&ispcrg JH7110_ISPCLK_M31DPHY_REF_IN>, + <&ispcrg JH7110_ISPCLK_M31DPHY_TX_ESC_LAN0>; + clock-names = "cfg", "ref", "tx"; + resets = <&ispcrg JH7110_ISPRST_M31DPHY_HW>, + <&ispcrg JH7110_ISPRST_M31DPHY_B09_AON>; + power-domains = <&aon_syscon JH7110_AON_PD_DPHY_RX>; + #phy-cells = <0>; + }; + + camss: isp@19840000 { + compatible = "starfive,jh7110-camss"; + reg = <0x0 0x19840000 0x0 0x10000>, + <0x0 0x19870000 0x0 0x30000>; + reg-names = "syscon", "isp"; + clocks = <&ispcrg JH7110_ISPCLK_DOM4_APB_FUNC>, + <&ispcrg JH7110_ISPCLK_ISPV2_TOP_WRAPPER_C>, + <&ispcrg JH7110_ISPCLK_DVP_INV>, + <&ispcrg JH7110_ISPCLK_VIN_P_AXI_WR>, + <&ispcrg JH7110_ISPCLK_MIPI_RX0_PXL>, + <&syscrg JH7110_SYSCLK_ISP_TOP_CORE>, + <&syscrg JH7110_SYSCLK_ISP_TOP_AXI>; + clock-names = "apb_func", "wrapper_clk_c", "dvp_inv", + "axiwr", "mipi_rx0_pxl", "ispcore_2x", + "isp_axi"; + resets = <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_P>, + <&ispcrg JH7110_ISPRST_ISPV2_TOP_WRAPPER_C>, + <&ispcrg JH7110_ISPRST_VIN_P_AXI_RD>, + <&ispcrg JH7110_ISPRST_VIN_P_AXI_WR>, + <&syscrg JH7110_SYSRST_ISP_TOP>, + <&syscrg JH7110_SYSRST_ISP_TOP_AXI>; + reset-names = "wrapper_p", "wrapper_c", "axird", + "axiwr", "isp_top_n", "isp_top_axi"; + power-domains = <&pwrc JH7110_PD_ISP>; + interrupts = <92>, <87>, <90>, <88>; + status = "disabled"; + }; + voutcrg: clock-controller@295c0000 { compatible = "starfive,jh7110-voutcrg"; reg = <0x0 0x295c0000 0x0 0x10000>; diff --git a/arch/riscv/configs/nommu_k210_defconfig b/arch/riscv/configs/nommu_k210_defconfig index 146c46d0525b..7e75200543f4 100644 --- a/arch/riscv/configs/nommu_k210_defconfig +++ b/arch/riscv/configs/nommu_k210_defconfig @@ -33,6 +33,8 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0" CONFIG_CMDLINE_FORCE=y +CONFIG_BUILTIN_DTB=y +CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set diff --git a/arch/riscv/configs/nommu_k210_sdcard_defconfig b/arch/riscv/configs/nommu_k210_sdcard_defconfig index 95d8d1808f19..0ba353e9ca71 100644 --- a/arch/riscv/configs/nommu_k210_sdcard_defconfig +++ b/arch/riscv/configs/nommu_k210_sdcard_defconfig @@ -25,6 +25,8 @@ CONFIG_SMP=y CONFIG_NR_CPUS=2 CONFIG_CMDLINE="earlycon console=ttySIF0 root=/dev/mmcblk0p1 rootwait ro" CONFIG_CMDLINE_FORCE=y +CONFIG_BUILTIN_DTB=y +CONFIG_BUILTIN_DTB_SOURCE="canaan/k210_generic" # CONFIG_SECCOMP is not set # CONFIG_STACKPROTECTOR is not set # CONFIG_GCC_PLUGINS is not set diff --git a/arch/riscv/include/asm/arch_hweight.h b/arch/riscv/include/asm/arch_hweight.h index c20236a0725b..85b2c443823e 100644 --- a/arch/riscv/include/asm/arch_hweight.h +++ b/arch/riscv/include/asm/arch_hweight.h @@ -20,7 +20,7 @@ static __always_inline unsigned int __arch_hweight32(unsigned int w) { #ifdef CONFIG_RISCV_ISA_ZBB - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -51,7 +51,7 @@ static inline unsigned int __arch_hweight8(unsigned int w) static __always_inline unsigned long __arch_hweight64(__u64 w) { # ifdef CONFIG_RISCV_ISA_ZBB - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h index c4c2173dfe99..880606b0469a 100644 --- a/arch/riscv/include/asm/bitops.h +++ b/arch/riscv/include/asm/bitops.h @@ -47,7 +47,7 @@ static __always_inline unsigned long variable__ffs(unsigned long word) { - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -76,7 +76,7 @@ legacy: static __always_inline unsigned long variable__fls(unsigned long word) { - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -105,7 +105,7 @@ legacy: static __always_inline int variable_ffs(int x) { - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); @@ -137,7 +137,7 @@ legacy: static __always_inline int variable_fls(unsigned int x) { - asm_volatile_goto(ALTERNATIVE("j %l[legacy]", "nop", 0, + asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : : : legacy); diff --git a/arch/riscv/include/asm/cfi.h b/arch/riscv/include/asm/cfi.h index 8f7a62257044..fb9696d7a3f2 100644 --- a/arch/riscv/include/asm/cfi.h +++ b/arch/riscv/include/asm/cfi.h @@ -13,11 +13,28 @@ struct pt_regs; #ifdef CONFIG_CFI_CLANG enum bug_trap_type handle_cfi_failure(struct pt_regs *regs); +#define __bpfcall +static inline int cfi_get_offset(void) +{ + return 4; +} + +#define cfi_get_offset cfi_get_offset +extern u32 cfi_bpf_hash; +extern u32 cfi_bpf_subprog_hash; +extern u32 cfi_get_func_hash(void *func); #else static inline enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) { return BUG_TRAP_TYPE_NONE; } + +#define cfi_bpf_hash 0U +#define cfi_bpf_subprog_hash 0U +static inline u32 cfi_get_func_hash(void *func) +{ + return 0; +} #endif /* CONFIG_CFI_CLANG */ #endif /* _ASM_RISCV_CFI_H */ diff --git a/arch/riscv/include/asm/checksum.h b/arch/riscv/include/asm/checksum.h index a5b60b54b101..88e6f1499e88 100644 --- a/arch/riscv/include/asm/checksum.h +++ b/arch/riscv/include/asm/checksum.h @@ -53,7 +53,7 @@ static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { unsigned long fold_temp; - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : diff --git a/arch/riscv/include/asm/cpufeature.h b/arch/riscv/include/asm/cpufeature.h index 46061f5e9764..347805446151 100644 --- a/arch/riscv/include/asm/cpufeature.h +++ b/arch/riscv/include/asm/cpufeature.h @@ -89,7 +89,7 @@ riscv_has_extension_likely(const unsigned long ext) "ext must be < RISCV_ISA_EXT_MAX"); if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { - asm_volatile_goto( + asm goto( ALTERNATIVE("j %l[l_no]", "nop", 0, %[ext], 1) : : [ext] "i" (ext) @@ -112,7 +112,7 @@ riscv_has_extension_unlikely(const unsigned long ext) "ext must be < RISCV_ISA_EXT_MAX"); if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) { - asm_volatile_goto( + asm goto( ALTERNATIVE("nop", "j %l[l_yes]", 0, %[ext], 1) : : [ext] "i" (ext) diff --git a/arch/riscv/include/asm/crash_core.h b/arch/riscv/include/asm/crash_reserve.h index e1874b23feaf..013962e63587 100644 --- a/arch/riscv/include/asm/crash_core.h +++ b/arch/riscv/include/asm/crash_reserve.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0-only */ -#ifndef _RISCV_CRASH_CORE_H -#define _RISCV_CRASH_CORE_H +#ifndef _RISCV_CRASH_RESERVE_H +#define _RISCV_CRASH_RESERVE_H #define CRASH_ALIGN PMD_SIZE diff --git a/arch/riscv/include/asm/csr.h b/arch/riscv/include/asm/csr.h index 510014051f5d..2468c55933cd 100644 --- a/arch/riscv/include/asm/csr.h +++ b/arch/riscv/include/asm/csr.h @@ -424,6 +424,7 @@ # define CSR_STATUS CSR_MSTATUS # define CSR_IE CSR_MIE # define CSR_TVEC CSR_MTVEC +# define CSR_ENVCFG CSR_MENVCFG # define CSR_SCRATCH CSR_MSCRATCH # define CSR_EPC CSR_MEPC # define CSR_CAUSE CSR_MCAUSE @@ -448,6 +449,7 @@ # define CSR_STATUS CSR_SSTATUS # define CSR_IE CSR_SIE # define CSR_TVEC CSR_STVEC +# define CSR_ENVCFG CSR_SENVCFG # define CSR_SCRATCH CSR_SSCRATCH # define CSR_EPC CSR_SEPC # define CSR_CAUSE CSR_SCAUSE diff --git a/arch/riscv/include/asm/ftrace.h b/arch/riscv/include/asm/ftrace.h index 329172122952..1276d7d9ca8b 100644 --- a/arch/riscv/include/asm/ftrace.h +++ b/arch/riscv/include/asm/ftrace.h @@ -13,19 +13,14 @@ #endif #define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR -/* - * Clang prior to 13 had "mcount" instead of "_mcount": - * https://reviews.llvm.org/D98881 - */ -#if defined(CONFIG_CC_IS_GCC) || CONFIG_CLANG_VERSION >= 130000 -#define MCOUNT_NAME _mcount -#else -#define MCOUNT_NAME mcount -#endif - #define ARCH_SUPPORTS_FTRACE_OPS 1 #ifndef __ASSEMBLY__ -void MCOUNT_NAME(void); + +extern void *return_address(unsigned int level); + +#define ftrace_return_address(n) return_address(n) + +void _mcount(void); static inline unsigned long ftrace_call_adjust(unsigned long addr) { return addr; @@ -75,7 +70,7 @@ struct dyn_arch_ftrace { * both auipc and jalr at the same time. */ -#define MCOUNT_ADDR ((unsigned long)MCOUNT_NAME) +#define MCOUNT_ADDR ((unsigned long)_mcount) #define JALR_SIGN_MASK (0x00000800) #define JALR_OFFSET_MASK (0x00000fff) #define AUIPC_OFFSET_MASK (0xfffff000) diff --git a/arch/riscv/include/asm/hugetlb.h b/arch/riscv/include/asm/hugetlb.h index 4c5b0e929890..22deb7a2a6ec 100644 --- a/arch/riscv/include/asm/hugetlb.h +++ b/arch/riscv/include/asm/hugetlb.h @@ -11,6 +11,11 @@ static inline void arch_clear_hugepage_flags(struct page *page) } #define arch_clear_hugepage_flags arch_clear_hugepage_flags +#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION +bool arch_hugetlb_migration_supported(struct hstate *h); +#define arch_hugetlb_migration_supported arch_hugetlb_migration_supported +#endif + #ifdef CONFIG_RISCV_ISA_SVNAPOT #define __HAVE_ARCH_HUGE_PTE_CLEAR void huge_pte_clear(struct mm_struct *mm, unsigned long addr, diff --git a/arch/riscv/include/asm/hwcap.h b/arch/riscv/include/asm/hwcap.h index bae7eac76c18..e17d0078a651 100644 --- a/arch/riscv/include/asm/hwcap.h +++ b/arch/riscv/include/asm/hwcap.h @@ -82,6 +82,8 @@ #define RISCV_ISA_EXT_ZACAS 73 #define RISCV_ISA_EXT_XANDESPMU 74 +#define RISCV_ISA_EXT_XLINUXENVCFG 127 + #define RISCV_ISA_EXT_MAX 128 #define RISCV_ISA_EXT_INVALID U32_MAX diff --git a/arch/riscv/include/asm/jump_label.h b/arch/riscv/include/asm/jump_label.h index 14a5ea8d8ef0..4a35d787c019 100644 --- a/arch/riscv/include/asm/jump_label.h +++ b/arch/riscv/include/asm/jump_label.h @@ -17,7 +17,7 @@ static __always_inline bool arch_static_branch(struct static_key * const key, const bool branch) { - asm_volatile_goto( + asm goto( " .align 2 \n\t" " .option push \n\t" " .option norelax \n\t" @@ -39,7 +39,7 @@ label: static __always_inline bool arch_static_branch_jump(struct static_key * const key, const bool branch) { - asm_volatile_goto( + asm goto( " .align 2 \n\t" " .option push \n\t" " .option norelax \n\t" diff --git a/arch/riscv/include/asm/page.h b/arch/riscv/include/asm/page.h index 57e887bfa34c..2947423b5082 100644 --- a/arch/riscv/include/asm/page.h +++ b/arch/riscv/include/asm/page.h @@ -12,7 +12,7 @@ #include <linux/pfn.h> #include <linux/const.h> -#define PAGE_SHIFT (12) +#define PAGE_SHIFT CONFIG_PAGE_SHIFT #define PAGE_SIZE (_AC(1, UL) << PAGE_SHIFT) #define PAGE_MASK (~(PAGE_SIZE - 1)) diff --git a/arch/riscv/include/asm/pgtable-64.h b/arch/riscv/include/asm/pgtable-64.h index b42017d76924..221a5c1ee287 100644 --- a/arch/riscv/include/asm/pgtable-64.h +++ b/arch/riscv/include/asm/pgtable-64.h @@ -136,7 +136,7 @@ enum napot_cont_order { * 10010 - IO Strongly-ordered, Non-cacheable, Non-bufferable, Shareable, Non-trustable */ #define _PAGE_PMA_THEAD ((1UL << 62) | (1UL << 61) | (1UL << 60)) -#define _PAGE_NOCACHE_THEAD ((1UL < 61) | (1UL << 60)) +#define _PAGE_NOCACHE_THEAD ((1UL << 61) | (1UL << 60)) #define _PAGE_IO_THEAD ((1UL << 63) | (1UL << 60)) #define _PAGE_MTMASK_THEAD (_PAGE_PMA_THEAD | _PAGE_IO_THEAD | (1UL << 59)) @@ -190,7 +190,7 @@ static inline int pud_bad(pud_t pud) } #define pud_leaf pud_leaf -static inline int pud_leaf(pud_t pud) +static inline bool pud_leaf(pud_t pud) { return pud_present(pud) && (pud_val(pud) & _PAGE_LEAF); } diff --git a/arch/riscv/include/asm/pgtable.h b/arch/riscv/include/asm/pgtable.h index b2e6965748f2..97fcde30e247 100644 --- a/arch/riscv/include/asm/pgtable.h +++ b/arch/riscv/include/asm/pgtable.h @@ -84,7 +84,7 @@ * Define vmemmap for pfn_to_page & page_to_pfn calls. Needed if kernel * is configured with CONFIG_SPARSEMEM_VMEMMAP enabled. */ -#define vmemmap ((struct page *)VMEMMAP_START) +#define vmemmap ((struct page *)VMEMMAP_START - (phys_ram_base >> PAGE_SHIFT)) #define PCI_IO_SIZE SZ_16M #define PCI_IO_END VMEMMAP_START @@ -235,7 +235,7 @@ static inline int pmd_bad(pmd_t pmd) } #define pmd_leaf pmd_leaf -static inline int pmd_leaf(pmd_t pmd) +static inline bool pmd_leaf(pmd_t pmd) { return pmd_present(pmd) && (pmd_val(pmd) & _PAGE_LEAF); } @@ -523,6 +523,8 @@ static inline void __set_pte_at(struct mm_struct *mm, pte_t *ptep, pte_t pteval) set_pte(ptep, pteval); } +#define PFN_PTE_SHIFT _PAGE_PFN_SHIFT + static inline void set_ptes(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval, unsigned int nr) { diff --git a/arch/riscv/include/asm/ptdump.h b/arch/riscv/include/asm/ptdump.h deleted file mode 100644 index 3c9ea6dd5af7..000000000000 --- a/arch/riscv/include/asm/ptdump.h +++ /dev/null @@ -1,22 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Copyright (C) 2019 SiFive - */ - -#ifndef _ASM_RISCV_PTDUMP_H -#define _ASM_RISCV_PTDUMP_H - -void ptdump_check_wx(void); - -#ifdef CONFIG_DEBUG_WX -static inline void debug_checkwx(void) -{ - ptdump_check_wx(); -} -#else -static inline void debug_checkwx(void) -{ -} -#endif - -#endif /* _ASM_RISCV_PTDUMP_H */ diff --git a/arch/riscv/include/asm/stacktrace.h b/arch/riscv/include/asm/stacktrace.h index f7e8ef2418b9..b1495a7e06ce 100644 --- a/arch/riscv/include/asm/stacktrace.h +++ b/arch/riscv/include/asm/stacktrace.h @@ -21,4 +21,9 @@ static inline bool on_thread_stack(void) return !(((unsigned long)(current->stack) ^ current_stack_pointer) & ~(THREAD_SIZE - 1)); } + +#ifdef CONFIG_VMAP_STACK +DECLARE_PER_CPU(unsigned long [OVERFLOW_STACK_SIZE/sizeof(long)], overflow_stack); +#endif /* CONFIG_VMAP_STACK */ + #endif /* _ASM_RISCV_STACKTRACE_H */ diff --git a/arch/riscv/include/asm/suspend.h b/arch/riscv/include/asm/suspend.h index 076f8a9437cf..4718096fa5e3 100644 --- a/arch/riscv/include/asm/suspend.h +++ b/arch/riscv/include/asm/suspend.h @@ -14,6 +14,7 @@ struct suspend_context { struct pt_regs regs; /* Saved and restored by high-level functions */ unsigned long scratch; + unsigned long envcfg; unsigned long tvec; unsigned long ie; #ifdef CONFIG_MMU diff --git a/arch/riscv/include/asm/tlb.h b/arch/riscv/include/asm/tlb.h index a0b8b853503f..1f6c38420d8e 100644 --- a/arch/riscv/include/asm/tlb.h +++ b/arch/riscv/include/asm/tlb.h @@ -34,7 +34,7 @@ static inline void __tlb_remove_table(void *table) static inline void tlb_flush(struct mmu_gather *tlb) { #ifdef CONFIG_MMU - if (tlb->fullmm || tlb->need_flush_all) + if (tlb->fullmm || tlb->need_flush_all || tlb->freed_tables) flush_tlb_mm(tlb->mm); else flush_tlb_mm_range(tlb->mm, tlb->start, tlb->end, diff --git a/arch/riscv/include/asm/tlbflush.h b/arch/riscv/include/asm/tlbflush.h index 928f096dca21..4112cc8d1d69 100644 --- a/arch/riscv/include/asm/tlbflush.h +++ b/arch/riscv/include/asm/tlbflush.h @@ -75,6 +75,7 @@ static inline void flush_tlb_kernel_range(unsigned long start, #define flush_tlb_mm(mm) flush_tlb_all() #define flush_tlb_mm_range(mm, start, end, page_size) flush_tlb_all() +#define local_flush_tlb_kernel_range(start, end) flush_tlb_all() #endif /* !CONFIG_SMP || !CONFIG_MMU */ #endif /* _ASM_RISCV_TLBFLUSH_H */ diff --git a/arch/riscv/include/asm/word-at-a-time.h b/arch/riscv/include/asm/word-at-a-time.h index f3f031e34191..3802cda71ab7 100644 --- a/arch/riscv/include/asm/word-at-a-time.h +++ b/arch/riscv/include/asm/word-at-a-time.h @@ -10,7 +10,8 @@ #include <asm/asm-extable.h> -#include <linux/kernel.h> +#include <linux/bitops.h> +#include <linux/wordpart.h> struct word_at_a_time { const unsigned long one_bits, high_bits; diff --git a/arch/riscv/include/uapi/asm/kvm.h b/arch/riscv/include/uapi/asm/kvm.h index d6b7a5b95874..b1c503c2959c 100644 --- a/arch/riscv/include/uapi/asm/kvm.h +++ b/arch/riscv/include/uapi/asm/kvm.h @@ -16,7 +16,6 @@ #include <asm/ptrace.h> #define __KVM_HAVE_IRQ_LINE -#define __KVM_HAVE_READONLY_MEM #define KVM_COALESCED_MMIO_PAGE_OFFSET 1 @@ -139,6 +138,35 @@ enum KVM_RISCV_ISA_EXT_ID { KVM_RISCV_ISA_EXT_ZIHPM, KVM_RISCV_ISA_EXT_SMSTATEEN, KVM_RISCV_ISA_EXT_ZICOND, + KVM_RISCV_ISA_EXT_ZBC, + KVM_RISCV_ISA_EXT_ZBKB, + KVM_RISCV_ISA_EXT_ZBKC, + KVM_RISCV_ISA_EXT_ZBKX, + KVM_RISCV_ISA_EXT_ZKND, + KVM_RISCV_ISA_EXT_ZKNE, + KVM_RISCV_ISA_EXT_ZKNH, + KVM_RISCV_ISA_EXT_ZKR, + KVM_RISCV_ISA_EXT_ZKSED, + KVM_RISCV_ISA_EXT_ZKSH, + KVM_RISCV_ISA_EXT_ZKT, + KVM_RISCV_ISA_EXT_ZVBB, + KVM_RISCV_ISA_EXT_ZVBC, + KVM_RISCV_ISA_EXT_ZVKB, + KVM_RISCV_ISA_EXT_ZVKG, + KVM_RISCV_ISA_EXT_ZVKNED, + KVM_RISCV_ISA_EXT_ZVKNHA, + KVM_RISCV_ISA_EXT_ZVKNHB, + KVM_RISCV_ISA_EXT_ZVKSED, + KVM_RISCV_ISA_EXT_ZVKSH, + KVM_RISCV_ISA_EXT_ZVKT, + KVM_RISCV_ISA_EXT_ZFH, + KVM_RISCV_ISA_EXT_ZFHMIN, + KVM_RISCV_ISA_EXT_ZIHINTNTL, + KVM_RISCV_ISA_EXT_ZVFH, + KVM_RISCV_ISA_EXT_ZVFHMIN, + KVM_RISCV_ISA_EXT_ZFA, + KVM_RISCV_ISA_EXT_ZTSO, + KVM_RISCV_ISA_EXT_ZACAS, KVM_RISCV_ISA_EXT_MAX, }; diff --git a/arch/riscv/kernel/Makefile b/arch/riscv/kernel/Makefile index c8085126a6f9..81d94a8ee10f 100644 --- a/arch/riscv/kernel/Makefile +++ b/arch/riscv/kernel/Makefile @@ -7,6 +7,7 @@ ifdef CONFIG_FTRACE CFLAGS_REMOVE_ftrace.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_patch.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_sbi.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_return_address.o = $(CC_FLAGS_FTRACE) endif CFLAGS_syscall_table.o += $(call cc-option,-Wno-override-init,) CFLAGS_compat_syscall_table.o += $(call cc-option,-Wno-override-init,) @@ -45,6 +46,7 @@ obj-y += irq.o obj-y += process.o obj-y += ptrace.o obj-y += reset.o +obj-y += return_address.o obj-y += setup.o obj-y += signal.o obj-y += syscall_table.o @@ -94,7 +96,7 @@ obj-$(CONFIG_KGDB) += kgdb.o obj-$(CONFIG_KEXEC_CORE) += kexec_relocate.o crash_save_regs.o machine_kexec.o obj-$(CONFIG_KEXEC_FILE) += elf_kexec.o machine_kexec_file.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o -obj-$(CONFIG_CRASH_CORE) += crash_core.o +obj-$(CONFIG_VMCORE_INFO) += vmcore_info.o obj-$(CONFIG_JUMP_LABEL) += jump_label.o diff --git a/arch/riscv/kernel/cfi.c b/arch/riscv/kernel/cfi.c index 6ec9dbd7292e..64bdd3e1ab8c 100644 --- a/arch/riscv/kernel/cfi.c +++ b/arch/riscv/kernel/cfi.c @@ -75,3 +75,56 @@ enum bug_trap_type handle_cfi_failure(struct pt_regs *regs) return report_cfi_failure(regs, regs->epc, &target, type); } + +#ifdef CONFIG_CFI_CLANG +struct bpf_insn; + +/* Must match bpf_func_t / DEFINE_BPF_PROG_RUN() */ +extern unsigned int __bpf_prog_runX(const void *ctx, + const struct bpf_insn *insn); + +/* + * Force a reference to the external symbol so the compiler generates + * __kcfi_typid. + */ +__ADDRESSABLE(__bpf_prog_runX); + +/* u32 __ro_after_init cfi_bpf_hash = __kcfi_typeid___bpf_prog_runX; */ +asm ( +" .pushsection .data..ro_after_init,\"aw\",@progbits \n" +" .type cfi_bpf_hash,@object \n" +" .globl cfi_bpf_hash \n" +" .p2align 2, 0x0 \n" +"cfi_bpf_hash: \n" +" .word __kcfi_typeid___bpf_prog_runX \n" +" .size cfi_bpf_hash, 4 \n" +" .popsection \n" +); + +/* Must match bpf_callback_t */ +extern u64 __bpf_callback_fn(u64, u64, u64, u64, u64); + +__ADDRESSABLE(__bpf_callback_fn); + +/* u32 __ro_after_init cfi_bpf_subprog_hash = __kcfi_typeid___bpf_callback_fn; */ +asm ( +" .pushsection .data..ro_after_init,\"aw\",@progbits \n" +" .type cfi_bpf_subprog_hash,@object \n" +" .globl cfi_bpf_subprog_hash \n" +" .p2align 2, 0x0 \n" +"cfi_bpf_subprog_hash: \n" +" .word __kcfi_typeid___bpf_callback_fn \n" +" .size cfi_bpf_subprog_hash, 4 \n" +" .popsection \n" +); + +u32 cfi_get_func_hash(void *func) +{ + u32 hash; + + if (get_kernel_nofault(hash, func - cfi_get_offset())) + return 0; + + return hash; +} +#endif diff --git a/arch/riscv/kernel/cpufeature.c b/arch/riscv/kernel/cpufeature.c index afeae3ff43dc..3ed2359eae35 100644 --- a/arch/riscv/kernel/cpufeature.c +++ b/arch/riscv/kernel/cpufeature.c @@ -22,6 +22,7 @@ #include <asm/hwcap.h> #include <asm/patch.h> #include <asm/processor.h> +#include <asm/sbi.h> #include <asm/vector.h> #define NUM_ALPHA_EXTS ('z' - 'a' + 1) @@ -188,6 +189,16 @@ static const unsigned int riscv_zvbb_exts[] = { }; /* + * While the [ms]envcfg CSRs were not defined until version 1.12 of the RISC-V + * privileged ISA, the existence of the CSRs is implied by any extension which + * specifies [ms]envcfg bit(s). Hence, we define a custom ISA extension for the + * existence of the CSR, and treat it as a subset of those other extensions. + */ +static const unsigned int riscv_xlinuxenvcfg_exts[] = { + RISCV_ISA_EXT_XLINUXENVCFG +}; + +/* * The canonical order of ISA extension names in the ISA string is defined in * chapter 27 of the unprivileged specification. * @@ -236,8 +247,8 @@ const struct riscv_isa_ext_data riscv_isa_ext[] = { __RISCV_ISA_EXT_DATA(c, RISCV_ISA_EXT_c), __RISCV_ISA_EXT_DATA(v, RISCV_ISA_EXT_v), __RISCV_ISA_EXT_DATA(h, RISCV_ISA_EXT_h), - __RISCV_ISA_EXT_DATA(zicbom, RISCV_ISA_EXT_ZICBOM), - __RISCV_ISA_EXT_DATA(zicboz, RISCV_ISA_EXT_ZICBOZ), + __RISCV_ISA_EXT_SUPERSET(zicbom, RISCV_ISA_EXT_ZICBOM, riscv_xlinuxenvcfg_exts), + __RISCV_ISA_EXT_SUPERSET(zicboz, RISCV_ISA_EXT_ZICBOZ, riscv_xlinuxenvcfg_exts), __RISCV_ISA_EXT_DATA(zicntr, RISCV_ISA_EXT_ZICNTR), __RISCV_ISA_EXT_DATA(zicond, RISCV_ISA_EXT_ZICOND), __RISCV_ISA_EXT_DATA(zicsr, RISCV_ISA_EXT_ZICSR), @@ -526,6 +537,20 @@ static void __init riscv_fill_hwcap_from_isa_string(unsigned long *isa2hwcap) } /* + * "V" in ISA strings is ambiguous in practice: it should mean + * just the standard V-1.0 but vendors aren't well behaved. + * Many vendors with T-Head CPU cores which implement the 0.7.1 + * version of the vector specification put "v" into their DTs. + * CPU cores with the ratified spec will contain non-zero + * marchid. + */ + if (acpi_disabled && riscv_cached_mvendorid(cpu) == THEAD_VENDOR_ID && + riscv_cached_marchid(cpu) == 0x0) { + this_hwcap &= ~isa2hwcap[RISCV_ISA_EXT_v]; + clear_bit(RISCV_ISA_EXT_v, isainfo->isa); + } + + /* * All "okay" hart should have same isa. Set HWCAP based on * common capabilities of every "okay" hart, in case they don't * have. @@ -696,7 +721,7 @@ unsigned long riscv_get_elf_hwcap(void) void riscv_user_isa_enable(void) { if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_ZICBOZ)) - csr_set(CSR_SENVCFG, ENVCFG_CBZE); + csr_set(CSR_ENVCFG, ENVCFG_CBZE); } #ifdef CONFIG_RISCV_ALTERNATIVE diff --git a/arch/riscv/kernel/elf_kexec.c b/arch/riscv/kernel/elf_kexec.c index 5bd1ec3341fe..54260c16f991 100644 --- a/arch/riscv/kernel/elf_kexec.c +++ b/arch/riscv/kernel/elf_kexec.c @@ -117,6 +117,7 @@ static int elf_find_pbase(struct kimage *image, unsigned long kernel_len, return ret; } +#ifdef CONFIG_CRASH_DUMP static int get_nr_ram_ranges_callback(struct resource *res, void *arg) { unsigned int *nr_ranges = arg; @@ -189,6 +190,7 @@ static char *setup_kdump_cmdline(struct kimage *image, char *cmdline, cmdline_ptr[COMMAND_LINE_SIZE - 1] = '\0'; return cmdline_ptr; } +#endif static void *elf_kexec_load(struct kimage *image, char *kernel_buf, unsigned long kernel_len, char *initrd, @@ -196,12 +198,11 @@ static void *elf_kexec_load(struct kimage *image, char *kernel_buf, unsigned long cmdline_len) { int ret; + void *fdt; unsigned long old_kernel_pbase = ULONG_MAX; unsigned long new_kernel_pbase = 0UL; unsigned long initrd_pbase = 0UL; - unsigned long headers_sz; unsigned long kernel_start; - void *fdt, *headers; struct elfhdr ehdr; struct kexec_buf kbuf; struct kexec_elf_info elf_info; @@ -227,8 +228,11 @@ static void *elf_kexec_load(struct kimage *image, char *kernel_buf, kbuf.buf_min = new_kernel_pbase + kernel_len; kbuf.buf_max = ULONG_MAX; +#ifdef CONFIG_CRASH_DUMP /* Add elfcorehdr */ if (image->type == KEXEC_TYPE_CRASH) { + void *headers; + unsigned long headers_sz; ret = prepare_elf_headers(&headers, &headers_sz); if (ret) { pr_err("Preparing elf core header failed\n"); @@ -264,6 +268,7 @@ static void *elf_kexec_load(struct kimage *image, char *kernel_buf, } cmdline = modified_cmdline; } +#endif #ifdef CONFIG_ARCH_SUPPORTS_KEXEC_PURGATORY /* Add purgatory to the image */ diff --git a/arch/riscv/kernel/mcount.S b/arch/riscv/kernel/mcount.S index d7ec69ac6910..3a42f6287909 100644 --- a/arch/riscv/kernel/mcount.S +++ b/arch/riscv/kernel/mcount.S @@ -50,8 +50,8 @@ SYM_TYPED_FUNC_START(ftrace_stub) #ifdef CONFIG_DYNAMIC_FTRACE - .global MCOUNT_NAME - .set MCOUNT_NAME, ftrace_stub + .global _mcount + .set _mcount, ftrace_stub #endif ret SYM_FUNC_END(ftrace_stub) @@ -80,7 +80,7 @@ SYM_FUNC_END(return_to_handler) #endif #ifndef CONFIG_DYNAMIC_FTRACE -SYM_FUNC_START(MCOUNT_NAME) +SYM_FUNC_START(_mcount) la t4, ftrace_stub #ifdef CONFIG_FUNCTION_GRAPH_TRACER la t0, ftrace_graph_return @@ -126,6 +126,6 @@ SYM_FUNC_START(MCOUNT_NAME) jalr t5 RESTORE_ABI_STATE ret -SYM_FUNC_END(MCOUNT_NAME) +SYM_FUNC_END(_mcount) #endif -EXPORT_SYMBOL(MCOUNT_NAME) +EXPORT_SYMBOL(_mcount) diff --git a/arch/riscv/kernel/paravirt.c b/arch/riscv/kernel/paravirt.c index 8e114f5930ce..0d6225fd3194 100644 --- a/arch/riscv/kernel/paravirt.c +++ b/arch/riscv/kernel/paravirt.c @@ -41,7 +41,7 @@ static int __init parse_no_stealacc(char *arg) early_param("no-steal-acc", parse_no_stealacc); -DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64); +static DEFINE_PER_CPU(struct sbi_sta_struct, steal_time) __aligned(64); static bool __init has_pv_steal_clock(void) { @@ -91,8 +91,8 @@ static int pv_time_cpu_down_prepare(unsigned int cpu) static u64 pv_time_steal_clock(int cpu) { struct sbi_sta_struct *st = per_cpu_ptr(&steal_time, cpu); - u32 sequence; - u64 steal; + __le32 sequence; + __le64 steal; /* * Check the sequence field before and after reading the steal diff --git a/arch/riscv/kernel/return_address.c b/arch/riscv/kernel/return_address.c new file mode 100644 index 000000000000..c8115ec8fb30 --- /dev/null +++ b/arch/riscv/kernel/return_address.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * This code come from arch/arm64/kernel/return_address.c + * + * Copyright (C) 2023 SiFive. + */ + +#include <linux/export.h> +#include <linux/kprobes.h> +#include <linux/stacktrace.h> + +struct return_address_data { + unsigned int level; + void *addr; +}; + +static bool save_return_addr(void *d, unsigned long pc) +{ + struct return_address_data *data = d; + + if (!data->level) { + data->addr = (void *)pc; + return false; + } + + --data->level; + + return true; +} +NOKPROBE_SYMBOL(save_return_addr); + +noinline void *return_address(unsigned int level) +{ + struct return_address_data data; + + data.level = level + 3; + data.addr = NULL; + + arch_stack_walk(save_return_addr, &data, current, NULL); + + if (!data.level) + return data.addr; + else + return NULL; + +} +EXPORT_SYMBOL_GPL(return_address); +NOKPROBE_SYMBOL(return_address); diff --git a/arch/riscv/kernel/smpboot.c b/arch/riscv/kernel/smpboot.c index cfbe4b840d42..d41090fc3203 100644 --- a/arch/riscv/kernel/smpboot.c +++ b/arch/riscv/kernel/smpboot.c @@ -41,10 +41,6 @@ static DECLARE_COMPLETION(cpu_running); -void __init smp_prepare_boot_cpu(void) -{ -} - void __init smp_prepare_cpus(unsigned int max_cpus) { int cpuid; diff --git a/arch/riscv/kernel/suspend.c b/arch/riscv/kernel/suspend.c index b20f2cb5879f..8a327b485b90 100644 --- a/arch/riscv/kernel/suspend.c +++ b/arch/riscv/kernel/suspend.c @@ -15,6 +15,8 @@ void suspend_save_csrs(struct suspend_context *context) { context->scratch = csr_read(CSR_SCRATCH); + if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG)) + context->envcfg = csr_read(CSR_ENVCFG); context->tvec = csr_read(CSR_TVEC); context->ie = csr_read(CSR_IE); @@ -36,6 +38,8 @@ void suspend_save_csrs(struct suspend_context *context) void suspend_restore_csrs(struct suspend_context *context) { csr_write(CSR_SCRATCH, context->scratch); + if (riscv_cpu_has_extension_unlikely(smp_processor_id(), RISCV_ISA_EXT_XLINUXENVCFG)) + csr_write(CSR_ENVCFG, context->envcfg); csr_write(CSR_TVEC, context->tvec); csr_write(CSR_IE, context->ie); diff --git a/arch/riscv/kernel/tests/Kconfig.debug b/arch/riscv/kernel/tests/Kconfig.debug index 5dba64e8e977..78cea5d2c270 100644 --- a/arch/riscv/kernel/tests/Kconfig.debug +++ b/arch/riscv/kernel/tests/Kconfig.debug @@ -6,7 +6,7 @@ config AS_HAS_ULEB128 menuconfig RUNTIME_KERNEL_TESTING_MENU bool "arch/riscv/kernel runtime Testing" - def_bool y + default y help Enable riscv kernel runtime testing. diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index 2cf76218a5bd..98315b98256d 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -30,14 +30,8 @@ enum rv_vdso_map { #define VVAR_SIZE (VVAR_NR_PAGES << PAGE_SHIFT) -/* - * The vDSO data page. - */ -static union { - struct vdso_data data; - u8 page[PAGE_SIZE]; -} vdso_data_store __page_aligned_data; -struct vdso_data *vdso_data = &vdso_data_store.data; +static union vdso_data_store vdso_data_store __page_aligned_data; +struct vdso_data *vdso_data = vdso_data_store.data; struct __vdso_info { const char *name; diff --git a/arch/riscv/kernel/crash_core.c b/arch/riscv/kernel/vmcore_info.c index 8706736fd4e2..6d7a22522d63 100644 --- a/arch/riscv/kernel/crash_core.c +++ b/arch/riscv/kernel/vmcore_info.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <linux/crash_core.h> +#include <linux/vmcore_info.h> #include <linux/pagemap.h> void arch_crash_save_vmcoreinfo(void) @@ -8,7 +8,6 @@ void arch_crash_save_vmcoreinfo(void) VMCOREINFO_NUMBER(phys_ram_base); vmcoreinfo_append_str("NUMBER(PAGE_OFFSET)=0x%lx\n", PAGE_OFFSET); - vmcoreinfo_append_str("NUMBER(VMALLOC_START)=0x%lx\n", VMALLOC_START); vmcoreinfo_append_str("NUMBER(VMALLOC_END)=0x%lx\n", VMALLOC_END); #ifdef CONFIG_MMU VMCOREINFO_NUMBER(VA_BITS); diff --git a/arch/riscv/kvm/Kconfig b/arch/riscv/kvm/Kconfig index d490db943858..26d1727f0550 100644 --- a/arch/riscv/kvm/Kconfig +++ b/arch/riscv/kvm/Kconfig @@ -24,6 +24,7 @@ config KVM select HAVE_KVM_IRQ_ROUTING select HAVE_KVM_MSI select HAVE_KVM_VCPU_ASYNC_IOCTL + select HAVE_KVM_READONLY_MEM select KVM_COMMON select KVM_GENERIC_DIRTYLOG_READ_PROTECT select KVM_GENERIC_HARDWARE_ENABLING diff --git a/arch/riscv/kvm/vcpu_insn.c b/arch/riscv/kvm/vcpu_insn.c index 7a6abed41bc1..ee7215f4071f 100644 --- a/arch/riscv/kvm/vcpu_insn.c +++ b/arch/riscv/kvm/vcpu_insn.c @@ -7,6 +7,8 @@ #include <linux/bitops.h> #include <linux/kvm_host.h> +#include <asm/cpufeature.h> + #define INSN_OPCODE_MASK 0x007c #define INSN_OPCODE_SHIFT 2 #define INSN_OPCODE_SYSTEM 28 @@ -213,9 +215,20 @@ struct csr_func { unsigned long wr_mask); }; +static int seed_csr_rmw(struct kvm_vcpu *vcpu, unsigned int csr_num, + unsigned long *val, unsigned long new_val, + unsigned long wr_mask) +{ + if (!riscv_isa_extension_available(vcpu->arch.isa, ZKR)) + return KVM_INSN_ILLEGAL_TRAP; + + return KVM_INSN_EXIT_TO_USER_SPACE; +} + static const struct csr_func csr_funcs[] = { KVM_RISCV_VCPU_AIA_CSR_FUNCS KVM_RISCV_VCPU_HPMCOUNTER_CSR_FUNCS + { .base = CSR_SEED, .count = 1, .func = seed_csr_rmw }, }; /** diff --git a/arch/riscv/kvm/vcpu_onereg.c b/arch/riscv/kvm/vcpu_onereg.c index fc34557f5356..f4a6124d25c9 100644 --- a/arch/riscv/kvm/vcpu_onereg.c +++ b/arch/riscv/kvm/vcpu_onereg.c @@ -40,17 +40,46 @@ static const unsigned long kvm_isa_ext_arr[] = { KVM_ISA_EXT_ARR(SVINVAL), KVM_ISA_EXT_ARR(SVNAPOT), KVM_ISA_EXT_ARR(SVPBMT), + KVM_ISA_EXT_ARR(ZACAS), KVM_ISA_EXT_ARR(ZBA), KVM_ISA_EXT_ARR(ZBB), + KVM_ISA_EXT_ARR(ZBC), + KVM_ISA_EXT_ARR(ZBKB), + KVM_ISA_EXT_ARR(ZBKC), + KVM_ISA_EXT_ARR(ZBKX), KVM_ISA_EXT_ARR(ZBS), + KVM_ISA_EXT_ARR(ZFA), + KVM_ISA_EXT_ARR(ZFH), + KVM_ISA_EXT_ARR(ZFHMIN), KVM_ISA_EXT_ARR(ZICBOM), KVM_ISA_EXT_ARR(ZICBOZ), KVM_ISA_EXT_ARR(ZICNTR), KVM_ISA_EXT_ARR(ZICOND), KVM_ISA_EXT_ARR(ZICSR), KVM_ISA_EXT_ARR(ZIFENCEI), + KVM_ISA_EXT_ARR(ZIHINTNTL), KVM_ISA_EXT_ARR(ZIHINTPAUSE), KVM_ISA_EXT_ARR(ZIHPM), + KVM_ISA_EXT_ARR(ZKND), + KVM_ISA_EXT_ARR(ZKNE), + KVM_ISA_EXT_ARR(ZKNH), + KVM_ISA_EXT_ARR(ZKR), + KVM_ISA_EXT_ARR(ZKSED), + KVM_ISA_EXT_ARR(ZKSH), + KVM_ISA_EXT_ARR(ZKT), + KVM_ISA_EXT_ARR(ZTSO), + KVM_ISA_EXT_ARR(ZVBB), + KVM_ISA_EXT_ARR(ZVBC), + KVM_ISA_EXT_ARR(ZVFH), + KVM_ISA_EXT_ARR(ZVFHMIN), + KVM_ISA_EXT_ARR(ZVKB), + KVM_ISA_EXT_ARR(ZVKG), + KVM_ISA_EXT_ARR(ZVKNED), + KVM_ISA_EXT_ARR(ZVKNHA), + KVM_ISA_EXT_ARR(ZVKNHB), + KVM_ISA_EXT_ARR(ZVKSED), + KVM_ISA_EXT_ARR(ZVKSH), + KVM_ISA_EXT_ARR(ZVKT), }; static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) @@ -90,15 +119,44 @@ static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) case KVM_RISCV_ISA_EXT_SSTC: case KVM_RISCV_ISA_EXT_SVINVAL: case KVM_RISCV_ISA_EXT_SVNAPOT: + case KVM_RISCV_ISA_EXT_ZACAS: case KVM_RISCV_ISA_EXT_ZBA: case KVM_RISCV_ISA_EXT_ZBB: + case KVM_RISCV_ISA_EXT_ZBC: + case KVM_RISCV_ISA_EXT_ZBKB: + case KVM_RISCV_ISA_EXT_ZBKC: + case KVM_RISCV_ISA_EXT_ZBKX: case KVM_RISCV_ISA_EXT_ZBS: + case KVM_RISCV_ISA_EXT_ZFA: + case KVM_RISCV_ISA_EXT_ZFH: + case KVM_RISCV_ISA_EXT_ZFHMIN: case KVM_RISCV_ISA_EXT_ZICNTR: case KVM_RISCV_ISA_EXT_ZICOND: case KVM_RISCV_ISA_EXT_ZICSR: case KVM_RISCV_ISA_EXT_ZIFENCEI: + case KVM_RISCV_ISA_EXT_ZIHINTNTL: case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: case KVM_RISCV_ISA_EXT_ZIHPM: + case KVM_RISCV_ISA_EXT_ZKND: + case KVM_RISCV_ISA_EXT_ZKNE: + case KVM_RISCV_ISA_EXT_ZKNH: + case KVM_RISCV_ISA_EXT_ZKR: + case KVM_RISCV_ISA_EXT_ZKSED: + case KVM_RISCV_ISA_EXT_ZKSH: + case KVM_RISCV_ISA_EXT_ZKT: + case KVM_RISCV_ISA_EXT_ZTSO: + case KVM_RISCV_ISA_EXT_ZVBB: + case KVM_RISCV_ISA_EXT_ZVBC: + case KVM_RISCV_ISA_EXT_ZVFH: + case KVM_RISCV_ISA_EXT_ZVFHMIN: + case KVM_RISCV_ISA_EXT_ZVKB: + case KVM_RISCV_ISA_EXT_ZVKG: + case KVM_RISCV_ISA_EXT_ZVKNED: + case KVM_RISCV_ISA_EXT_ZVKNHA: + case KVM_RISCV_ISA_EXT_ZVKNHB: + case KVM_RISCV_ISA_EXT_ZVKSED: + case KVM_RISCV_ISA_EXT_ZVKSH: + case KVM_RISCV_ISA_EXT_ZVKT: return false; /* Extensions which can be disabled using Smstateen */ case KVM_RISCV_ISA_EXT_SSAIA: diff --git a/arch/riscv/kvm/vcpu_sbi_sta.c b/arch/riscv/kvm/vcpu_sbi_sta.c index 01f09fe8c3b0..d8cf9ca28c61 100644 --- a/arch/riscv/kvm/vcpu_sbi_sta.c +++ b/arch/riscv/kvm/vcpu_sbi_sta.c @@ -26,8 +26,12 @@ void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) { gpa_t shmem = vcpu->arch.sta.shmem; u64 last_steal = vcpu->arch.sta.last_steal; - u32 *sequence_ptr, sequence; - u64 *steal_ptr, steal; + __le32 __user *sequence_ptr; + __le64 __user *steal_ptr; + __le32 sequence_le; + __le64 steal_le; + u32 sequence; + u64 steal; unsigned long hva; gfn_t gfn; @@ -47,22 +51,22 @@ void kvm_riscv_vcpu_record_steal_time(struct kvm_vcpu *vcpu) return; } - sequence_ptr = (u32 *)(hva + offset_in_page(shmem) + + sequence_ptr = (__le32 __user *)(hva + offset_in_page(shmem) + offsetof(struct sbi_sta_struct, sequence)); - steal_ptr = (u64 *)(hva + offset_in_page(shmem) + + steal_ptr = (__le64 __user *)(hva + offset_in_page(shmem) + offsetof(struct sbi_sta_struct, steal)); - if (WARN_ON(get_user(sequence, sequence_ptr))) + if (WARN_ON(get_user(sequence_le, sequence_ptr))) return; - sequence = le32_to_cpu(sequence); + sequence = le32_to_cpu(sequence_le); sequence += 1; if (WARN_ON(put_user(cpu_to_le32(sequence), sequence_ptr))) return; - if (!WARN_ON(get_user(steal, steal_ptr))) { - steal = le64_to_cpu(steal); + if (!WARN_ON(get_user(steal_le, steal_ptr))) { + steal = le64_to_cpu(steal_le); vcpu->arch.sta.last_steal = READ_ONCE(current->sched_info.run_delay); steal += vcpu->arch.sta.last_steal - last_steal; WARN_ON(put_user(cpu_to_le64(steal), steal_ptr)); diff --git a/arch/riscv/lib/csum.c b/arch/riscv/lib/csum.c index 7178e0acfa22..7fb12c59e571 100644 --- a/arch/riscv/lib/csum.c +++ b/arch/riscv/lib/csum.c @@ -53,7 +53,7 @@ __sum16 csum_ipv6_magic(const struct in6_addr *saddr, * support, so nop when Zbb is available and jump when Zbb is * not available. */ - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : @@ -170,7 +170,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) * support, so nop when Zbb is available and jump when Zbb is * not available. */ - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : @@ -178,7 +178,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) : no_zbb); #ifdef CONFIG_32BIT - asm_volatile_goto(".option push \n\ + asm_goto_output(".option push \n\ .option arch,+zbb \n\ rori %[fold_temp], %[csum], 16 \n\ andi %[offset], %[offset], 1 \n\ @@ -193,7 +193,7 @@ do_csum_with_alignment(const unsigned char *buff, int len) return (unsigned short)csum; #else /* !CONFIG_32BIT */ - asm_volatile_goto(".option push \n\ + asm_goto_output(".option push \n\ .option arch,+zbb \n\ rori %[fold_temp], %[csum], 32 \n\ add %[csum], %[fold_temp], %[csum] \n\ @@ -257,7 +257,7 @@ do_csum_no_alignment(const unsigned char *buff, int len) * support, so nop when Zbb is available and jump when Zbb is * not available. */ - asm_volatile_goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, + asm goto(ALTERNATIVE("j %l[no_zbb]", "nop", 0, RISCV_ISA_EXT_ZBB, 1) : : diff --git a/arch/riscv/mm/hugetlbpage.c b/arch/riscv/mm/hugetlbpage.c index 431596c0e20e..5ef2a6891158 100644 --- a/arch/riscv/mm/hugetlbpage.c +++ b/arch/riscv/mm/hugetlbpage.c @@ -125,6 +125,26 @@ pte_t *huge_pte_offset(struct mm_struct *mm, return pte; } +unsigned long hugetlb_mask_last_page(struct hstate *h) +{ + unsigned long hp_size = huge_page_size(h); + + switch (hp_size) { +#ifndef __PAGETABLE_PMD_FOLDED + case PUD_SIZE: + return P4D_SIZE - PUD_SIZE; +#endif + case PMD_SIZE: + return PUD_SIZE - PMD_SIZE; + case napot_cont_size(NAPOT_CONT64KB_ORDER): + return PMD_SIZE - napot_cont_size(NAPOT_CONT64KB_ORDER); + default: + break; + } + + return 0UL; +} + static pte_t get_clear_contig(struct mm_struct *mm, unsigned long addr, pte_t *ptep, @@ -177,13 +197,36 @@ pte_t arch_make_huge_pte(pte_t entry, unsigned int shift, vm_flags_t flags) return entry; } +static void clear_flush(struct mm_struct *mm, + unsigned long addr, + pte_t *ptep, + unsigned long pgsize, + unsigned long ncontig) +{ + struct vm_area_struct vma = TLB_FLUSH_VMA(mm, 0); + unsigned long i, saddr = addr; + + for (i = 0; i < ncontig; i++, addr += pgsize, ptep++) + ptep_get_and_clear(mm, addr, ptep); + + flush_tlb_range(&vma, saddr, addr); +} + +/* + * When dealing with NAPOT mappings, the privileged specification indicates that + * "if an update needs to be made, the OS generally should first mark all of the + * PTEs invalid, then issue SFENCE.VMA instruction(s) covering all 4 KiB regions + * within the range, [...] then update the PTE(s), as described in Section + * 4.2.1.". That's the equivalent of the Break-Before-Make approach used by + * arm64. + */ void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte, unsigned long sz) { - unsigned long hugepage_shift; + unsigned long hugepage_shift, pgsize; int i, pte_num; if (sz >= PGDIR_SIZE) @@ -198,7 +241,22 @@ void set_huge_pte_at(struct mm_struct *mm, hugepage_shift = PAGE_SHIFT; pte_num = sz >> hugepage_shift; - for (i = 0; i < pte_num; i++, ptep++, addr += (1 << hugepage_shift)) + pgsize = 1 << hugepage_shift; + + if (!pte_present(pte)) { + for (i = 0; i < pte_num; i++, ptep++, addr += pgsize) + set_ptes(mm, addr, ptep, pte, 1); + return; + } + + if (!pte_napot(pte)) { + set_ptes(mm, addr, ptep, pte, 1); + return; + } + + clear_flush(mm, addr, ptep, pgsize, pte_num); + + for (i = 0; i < pte_num; i++, ptep++, addr += pgsize) set_pte_at(mm, addr, ptep, pte); } @@ -306,7 +364,7 @@ void huge_pte_clear(struct mm_struct *mm, pte_clear(mm, addr, ptep); } -static __init bool is_napot_size(unsigned long size) +static bool is_napot_size(unsigned long size) { unsigned long order; @@ -334,7 +392,7 @@ arch_initcall(napot_hugetlbpages_init); #else -static __init bool is_napot_size(unsigned long size) +static bool is_napot_size(unsigned long size) { return false; } @@ -351,7 +409,7 @@ int pmd_huge(pmd_t pmd) return pmd_leaf(pmd); } -bool __init arch_hugetlb_valid_size(unsigned long size) +static bool __hugetlb_valid_size(unsigned long size) { if (size == HPAGE_SIZE) return true; @@ -363,6 +421,18 @@ bool __init arch_hugetlb_valid_size(unsigned long size) return false; } +bool __init arch_hugetlb_valid_size(unsigned long size) +{ + return __hugetlb_valid_size(size); +} + +#ifdef CONFIG_ARCH_ENABLE_HUGEPAGE_MIGRATION +bool arch_hugetlb_migration_supported(struct hstate *h) +{ + return __hugetlb_valid_size(huge_page_size(h)); +} +#endif + #ifdef CONFIG_CONTIG_ALLOC static __init int gigantic_pages_init(void) { diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index c55915554836..fe8e159394d8 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -29,7 +29,6 @@ #include <asm/io.h> #include <asm/numa.h> #include <asm/pgtable.h> -#include <asm/ptdump.h> #include <asm/sections.h> #include <asm/soc.h> #include <asm/tlbflush.h> @@ -723,8 +722,6 @@ void mark_rodata_ro(void) if (IS_ENABLED(CONFIG_64BIT)) set_kernel_memory(lm_alias(__start_rodata), lm_alias(_data), set_memory_ro); - - debug_checkwx(); } #else static __init pgprot_t pgprot_from_va(uintptr_t va) @@ -1364,7 +1361,7 @@ static void __init arch_reserve_crashkernel(void) bool high = false; int ret; - if (!IS_ENABLED(CONFIG_KEXEC_CORE)) + if (!IS_ENABLED(CONFIG_CRASH_RESERVE)) return; ret = parse_crashkernel(cmdline, memblock_phys_mem_size(), @@ -1391,6 +1388,10 @@ void __init misc_mem_init(void) early_memtest(min_low_pfn << PAGE_SHIFT, max_low_pfn << PAGE_SHIFT); arch_numa_init(); sparse_init(); +#ifdef CONFIG_SPARSEMEM_VMEMMAP + /* The entire VMEMMAP region has been populated. Flush TLB for this region */ + local_flush_tlb_kernel_range(VMEMMAP_START, VMEMMAP_END); +#endif zone_sizes_init(); arch_reserve_crashkernel(); memblock_dump_all(); diff --git a/arch/riscv/mm/ptdump.c b/arch/riscv/mm/ptdump.c index 657c27bc07a7..1289cc6d3700 100644 --- a/arch/riscv/mm/ptdump.c +++ b/arch/riscv/mm/ptdump.c @@ -9,7 +9,6 @@ #include <linux/seq_file.h> #include <linux/ptdump.h> -#include <asm/ptdump.h> #include <linux/pgtable.h> #include <asm/kasan.h> @@ -336,7 +335,7 @@ static void ptdump_walk(struct seq_file *s, struct ptd_mm_info *pinfo) ptdump_walk_pgd(&st.ptdump, pinfo->mm, NULL); } -void ptdump_check_wx(void) +bool ptdump_check_wx(void) { struct pg_state st = { .seq = NULL, @@ -357,11 +356,16 @@ void ptdump_check_wx(void) ptdump_walk_pgd(&st.ptdump, &init_mm, NULL); - if (st.wx_pages) + if (st.wx_pages) { pr_warn("Checked W+X mappings: failed, %lu W+X pages found\n", st.wx_pages); - else + + return false; + } else { pr_info("Checked W+X mappings: passed, no W+X pages found\n"); + + return true; + } } static int ptdump_show(struct seq_file *m, void *v) diff --git a/arch/riscv/mm/tlbflush.c b/arch/riscv/mm/tlbflush.c index 8d12b26f5ac3..893566e004b7 100644 --- a/arch/riscv/mm/tlbflush.c +++ b/arch/riscv/mm/tlbflush.c @@ -66,9 +66,10 @@ static inline void local_flush_tlb_range_asid(unsigned long start, local_flush_tlb_range_threshold_asid(start, size, stride, asid); } +/* Flush a range of kernel pages without broadcasting */ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end) { - local_flush_tlb_range_asid(start, end, PAGE_SIZE, FLUSH_TLB_NO_ASID); + local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID); } static void __ipi_flush_tlb_all(void *info) @@ -233,4 +234,5 @@ void arch_tlbbatch_flush(struct arch_tlbflush_unmap_batch *batch) { __flush_tlb_range(&batch->cpumask, FLUSH_TLB_NO_ASID, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE); + cpumask_clear(&batch->cpumask); } diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h index a5ce1ab76ece..f4b6b3b9edda 100644 --- a/arch/riscv/net/bpf_jit.h +++ b/arch/riscv/net/bpf_jit.h @@ -18,6 +18,11 @@ static inline bool rvc_enabled(void) return IS_ENABLED(CONFIG_RISCV_ISA_C); } +static inline bool rvzbb_enabled(void) +{ + return IS_ENABLED(CONFIG_RISCV_ISA_ZBB) && riscv_has_extension_likely(RISCV_ISA_EXT_ZBB); +} + enum { RV_REG_ZERO = 0, /* The constant value 0 */ RV_REG_RA = 1, /* Return address */ @@ -730,6 +735,33 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2) return rv_css_insn(0x6, imm, rs2, 0x2); } +/* RVZBB instrutions. */ +static inline u32 rvzbb_sextb(u8 rd, u8 rs1) +{ + return rv_i_insn(0x604, rs1, 1, rd, 0x13); +} + +static inline u32 rvzbb_sexth(u8 rd, u8 rs1) +{ + return rv_i_insn(0x605, rs1, 1, rd, 0x13); +} + +static inline u32 rvzbb_zexth(u8 rd, u8 rs) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return rv_i_insn(0x80, rs, 4, rd, 0x3b); + + return rv_i_insn(0x80, rs, 4, rd, 0x33); +} + +static inline u32 rvzbb_rev8(u8 rd, u8 rs) +{ + if (IS_ENABLED(CONFIG_64BIT)) + return rv_i_insn(0x6b8, rs, 5, rd, 0x13); + + return rv_i_insn(0x698, rs, 5, rd, 0x13); +} + /* * RV64-only instructions. * @@ -1087,9 +1119,111 @@ static inline void emit_subw(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx) emit(rv_subw(rd, rs1, rs2), ctx); } +static inline void emit_sextb(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + if (rvzbb_enabled()) { + emit(rvzbb_sextb(rd, rs), ctx); + return; + } + + emit_slli(rd, rs, 56, ctx); + emit_srai(rd, rd, 56, ctx); +} + +static inline void emit_sexth(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + if (rvzbb_enabled()) { + emit(rvzbb_sexth(rd, rs), ctx); + return; + } + + emit_slli(rd, rs, 48, ctx); + emit_srai(rd, rd, 48, ctx); +} + +static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + emit_addiw(rd, rs, 0, ctx); +} + +static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + if (rvzbb_enabled()) { + emit(rvzbb_zexth(rd, rs), ctx); + return; + } + + emit_slli(rd, rs, 48, ctx); + emit_srli(rd, rd, 48, ctx); +} + +static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx) +{ + emit_slli(rd, rs, 32, ctx); + emit_srli(rd, rd, 32, ctx); +} + +static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx) +{ + if (rvzbb_enabled()) { + int bits = 64 - imm; + + emit(rvzbb_rev8(rd, rd), ctx); + if (bits) + emit_srli(rd, rd, bits, ctx); + return; + } + + emit_li(RV_REG_T2, 0, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + if (imm == 16) + goto out_be; + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + if (imm == 32) + goto out_be; + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); + + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); + emit_srli(rd, rd, 8, ctx); +out_be: + emit_andi(RV_REG_T1, rd, 0xff, ctx); + emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); + + emit_mv(rd, RV_REG_T2, ctx); +} + #endif /* __riscv_xlen == 64 */ -void bpf_jit_build_prologue(struct rv_jit_context *ctx); +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog); void bpf_jit_build_epilogue(struct rv_jit_context *ctx); int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c index 529a83b85c1c..f5ba73bb153d 100644 --- a/arch/riscv/net/bpf_jit_comp32.c +++ b/arch/riscv/net/bpf_jit_comp32.c @@ -1301,7 +1301,7 @@ notsupported: return 0; } -void bpf_jit_build_prologue(struct rv_jit_context *ctx) +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { const s8 *fp = bpf2rv32[BPF_REG_FP]; const s8 *r1 = bpf2rv32[BPF_REG_1]; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 58dc64dd94a8..aac190085472 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -11,6 +11,7 @@ #include <linux/memory.h> #include <linux/stop_machine.h> #include <asm/patch.h> +#include <asm/cfi.h> #include "bpf_jit.h" #define RV_FENTRY_NINSNS 2 @@ -141,6 +142,19 @@ static bool in_auipc_jalr_range(s64 val) val < ((1L << 31) - (1L << 11)); } +/* Modify rd pointer to alternate reg to avoid corrupting original reg */ +static void emit_sextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) +{ + emit_sextw(ra, *rd, ctx); + *rd = ra; +} + +static void emit_zextw_alt(u8 *rd, u8 ra, struct rv_jit_context *ctx) +{ + emit_zextw(ra, *rd, ctx); + *rd = ra; +} + /* Emit fixed-length instructions for address */ static int emit_addr(u8 rd, u64 addr, bool extra_pass, struct rv_jit_context *ctx) { @@ -326,12 +340,6 @@ static void emit_branch(u8 cond, u8 rd, u8 rs, int rvoff, emit(rv_jalr(RV_REG_ZERO, RV_REG_T1, lower), ctx); } -static void emit_zext_32(u8 reg, struct rv_jit_context *ctx) -{ - emit_slli(reg, reg, 32, ctx); - emit_srli(reg, reg, 32, ctx); -} - static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) { int tc_ninsn, off, start_insn = ctx->ninsns; @@ -346,7 +354,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx) */ tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : ctx->offset[0]; - emit_zext_32(RV_REG_A2, ctx); + emit_zextw(RV_REG_A2, RV_REG_A2, ctx); off = offsetof(struct bpf_array, map.max_entries); if (is_12b_check(off, insn)) @@ -405,38 +413,6 @@ static void init_regs(u8 *rd, u8 *rs, const struct bpf_insn *insn, *rs = bpf_to_rv_reg(insn->src_reg, ctx); } -static void emit_zext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) -{ - emit_mv(RV_REG_T2, *rd, ctx); - emit_zext_32(RV_REG_T2, ctx); - emit_mv(RV_REG_T1, *rs, ctx); - emit_zext_32(RV_REG_T1, ctx); - *rd = RV_REG_T2; - *rs = RV_REG_T1; -} - -static void emit_sext_32_rd_rs(u8 *rd, u8 *rs, struct rv_jit_context *ctx) -{ - emit_addiw(RV_REG_T2, *rd, 0, ctx); - emit_addiw(RV_REG_T1, *rs, 0, ctx); - *rd = RV_REG_T2; - *rs = RV_REG_T1; -} - -static void emit_zext_32_rd_t1(u8 *rd, struct rv_jit_context *ctx) -{ - emit_mv(RV_REG_T2, *rd, ctx); - emit_zext_32(RV_REG_T2, ctx); - emit_zext_32(RV_REG_T1, ctx); - *rd = RV_REG_T2; -} - -static void emit_sext_32_rd(u8 *rd, struct rv_jit_context *ctx) -{ - emit_addiw(RV_REG_T2, *rd, 0, ctx); - *rd = RV_REG_T2; -} - static int emit_jump_and_link(u8 rd, s64 rvoff, bool fixed_addr, struct rv_jit_context *ctx) { @@ -480,6 +456,12 @@ static int emit_call(u64 addr, bool fixed_addr, struct rv_jit_context *ctx) return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); } +static inline void emit_kcfi(u32 hash, struct rv_jit_context *ctx) +{ + if (IS_ENABLED(CONFIG_CFI_CLANG)) + emit(hash, ctx); +} + static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, struct rv_jit_context *ctx) { @@ -519,32 +501,32 @@ static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, emit(is64 ? rv_amoadd_d(rs, rs, rd, 0, 0) : rv_amoadd_w(rs, rs, rd, 0, 0), ctx); if (!is64) - emit_zext_32(rs, ctx); + emit_zextw(rs, rs, ctx); break; case BPF_AND | BPF_FETCH: emit(is64 ? rv_amoand_d(rs, rs, rd, 0, 0) : rv_amoand_w(rs, rs, rd, 0, 0), ctx); if (!is64) - emit_zext_32(rs, ctx); + emit_zextw(rs, rs, ctx); break; case BPF_OR | BPF_FETCH: emit(is64 ? rv_amoor_d(rs, rs, rd, 0, 0) : rv_amoor_w(rs, rs, rd, 0, 0), ctx); if (!is64) - emit_zext_32(rs, ctx); + emit_zextw(rs, rs, ctx); break; case BPF_XOR | BPF_FETCH: emit(is64 ? rv_amoxor_d(rs, rs, rd, 0, 0) : rv_amoxor_w(rs, rs, rd, 0, 0), ctx); if (!is64) - emit_zext_32(rs, ctx); + emit_zextw(rs, rs, ctx); break; /* src_reg = atomic_xchg(dst_reg + off16, src_reg); */ case BPF_XCHG: emit(is64 ? rv_amoswap_d(rs, rs, rd, 0, 0) : rv_amoswap_w(rs, rs, rd, 0, 0), ctx); if (!is64) - emit_zext_32(rs, ctx); + emit_zextw(rs, rs, ctx); break; /* r0 = atomic_cmpxchg(dst_reg + off16, r0, src_reg); */ case BPF_CMPXCHG: @@ -795,6 +777,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; + bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; void *orig_call = func_addr; bool save_ret; u32 insn; @@ -878,7 +861,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, stack_size = round_up(stack_size, 16); - if (func_addr) { + if (!is_struct_ops) { /* For the trampoline called from function entry, * the frame of traced function and the frame of * trampoline need to be considered. @@ -893,6 +876,8 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); emit_addi(RV_REG_FP, RV_REG_SP, stack_size, ctx); } else { + /* emit kcfi hash */ + emit_kcfi(cfi_get_func_hash(func_addr), ctx); /* For the trampoline called directly, just handle * the frame of trampoline. */ @@ -998,7 +983,7 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); - if (func_addr) { + if (!is_struct_ops) { /* trampoline called from function entry */ emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); @@ -1090,7 +1075,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU64 | BPF_MOV | BPF_X: if (imm == 1) { /* Special mov32 for zext */ - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; } switch (insn->off) { @@ -1098,16 +1083,17 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_mv(rd, rs, ctx); break; case 8: + emit_sextb(rd, rs, ctx); + break; case 16: - emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx); - emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx); + emit_sexth(rd, rs, ctx); break; case 32: - emit_addiw(rd, rs, 0, ctx); + emit_sextw(rd, rs, ctx); break; } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; /* dst = dst OP src */ @@ -1115,7 +1101,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU64 | BPF_ADD | BPF_X: emit_add(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_SUB | BPF_X: case BPF_ALU64 | BPF_SUB | BPF_X: @@ -1125,31 +1111,31 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, emit_subw(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_AND | BPF_X: case BPF_ALU64 | BPF_AND | BPF_X: emit_and(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_OR | BPF_X: case BPF_ALU64 | BPF_OR | BPF_X: emit_or(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_XOR | BPF_X: case BPF_ALU64 | BPF_XOR | BPF_X: emit_xor(rd, rd, rs, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_MUL | BPF_X: case BPF_ALU64 | BPF_MUL | BPF_X: emit(is64 ? rv_mul(rd, rd, rs) : rv_mulw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_DIV | BPF_X: case BPF_ALU64 | BPF_DIV | BPF_X: @@ -1158,7 +1144,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else emit(is64 ? rv_divu(rd, rd, rs) : rv_divuw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_MOD | BPF_X: case BPF_ALU64 | BPF_MOD | BPF_X: @@ -1167,25 +1153,25 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, else emit(is64 ? rv_remu(rd, rd, rs) : rv_remuw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_LSH | BPF_X: case BPF_ALU64 | BPF_LSH | BPF_X: emit(is64 ? rv_sll(rd, rd, rs) : rv_sllw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_X: case BPF_ALU64 | BPF_RSH | BPF_X: emit(is64 ? rv_srl(rd, rd, rs) : rv_srlw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_ARSH | BPF_X: case BPF_ALU64 | BPF_ARSH | BPF_X: emit(is64 ? rv_sra(rd, rd, rs) : rv_sraw(rd, rd, rs), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; /* dst = -dst */ @@ -1193,73 +1179,27 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx, case BPF_ALU64 | BPF_NEG: emit_sub(rd, RV_REG_ZERO, rd, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; /* dst = BSWAP##imm(dst) */ case BPF_ALU | BPF_END | BPF_FROM_LE: switch (imm) { case 16: - emit_slli(rd, rd, 48, ctx); - emit_srli(rd, rd, 48, ctx); + emit_zexth(rd, rd, ctx); break; case 32: if (!aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case 64: /* Do nothing */ break; } break; - case BPF_ALU | BPF_END | BPF_FROM_BE: case BPF_ALU64 | BPF_END | BPF_FROM_LE: - emit_li(RV_REG_T2, 0, ctx); - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - if (imm == 16) - goto out_be; - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - if (imm == 32) - goto out_be; - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); - - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx); - emit_srli(rd, rd, 8, ctx); -out_be: - emit_andi(RV_REG_T1, rd, 0xff, ctx); - emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx); - - emit_mv(rd, RV_REG_T2, ctx); + emit_bswap(rd, imm, ctx); break; /* dst = imm */ @@ -1267,7 +1207,7 @@ out_be: case BPF_ALU64 | BPF_MOV | BPF_K: emit_imm(rd, imm, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; /* dst = dst OP imm */ @@ -1280,7 +1220,7 @@ out_be: emit_add(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_SUB | BPF_K: case BPF_ALU64 | BPF_SUB | BPF_K: @@ -1291,7 +1231,7 @@ out_be: emit_sub(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_AND | BPF_K: case BPF_ALU64 | BPF_AND | BPF_K: @@ -1302,7 +1242,7 @@ out_be: emit_and(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_OR | BPF_K: case BPF_ALU64 | BPF_OR | BPF_K: @@ -1313,7 +1253,7 @@ out_be: emit_or(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_XOR | BPF_K: case BPF_ALU64 | BPF_XOR | BPF_K: @@ -1324,7 +1264,7 @@ out_be: emit_xor(rd, rd, RV_REG_T1, ctx); } if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_MUL | BPF_K: case BPF_ALU64 | BPF_MUL | BPF_K: @@ -1332,7 +1272,7 @@ out_be: emit(is64 ? rv_mul(rd, rd, RV_REG_T1) : rv_mulw(rd, rd, RV_REG_T1), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_DIV | BPF_K: case BPF_ALU64 | BPF_DIV | BPF_K: @@ -1344,7 +1284,7 @@ out_be: emit(is64 ? rv_divu(rd, rd, RV_REG_T1) : rv_divuw(rd, rd, RV_REG_T1), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_MOD | BPF_K: case BPF_ALU64 | BPF_MOD | BPF_K: @@ -1356,14 +1296,14 @@ out_be: emit(is64 ? rv_remu(rd, rd, RV_REG_T1) : rv_remuw(rd, rd, RV_REG_T1), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_LSH | BPF_K: case BPF_ALU64 | BPF_LSH | BPF_K: emit_slli(rd, rd, imm, ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_RSH | BPF_K: case BPF_ALU64 | BPF_RSH | BPF_K: @@ -1373,7 +1313,7 @@ out_be: emit(rv_srliw(rd, rd, imm), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; case BPF_ALU | BPF_ARSH | BPF_K: case BPF_ALU64 | BPF_ARSH | BPF_K: @@ -1383,7 +1323,7 @@ out_be: emit(rv_sraiw(rd, rd, imm), ctx); if (!is64 && !aux->verifier_zext) - emit_zext_32(rd, ctx); + emit_zextw(rd, rd, ctx); break; /* JUMP off */ @@ -1424,10 +1364,13 @@ out_be: rvoff = rv_offset(i, off, ctx); if (!is64) { s = ctx->ninsns; - if (is_signed_bpf_cond(BPF_OP(code))) - emit_sext_32_rd_rs(&rd, &rs, ctx); - else - emit_zext_32_rd_rs(&rd, &rs, ctx); + if (is_signed_bpf_cond(BPF_OP(code))) { + emit_sextw_alt(&rs, RV_REG_T1, ctx); + emit_sextw_alt(&rd, RV_REG_T2, ctx); + } else { + emit_zextw_alt(&rs, RV_REG_T1, ctx); + emit_zextw_alt(&rd, RV_REG_T2, ctx); + } e = ctx->ninsns; /* Adjust for extra insns */ @@ -1438,8 +1381,7 @@ out_be: /* Adjust for and */ rvoff -= 4; emit_and(RV_REG_T1, rd, rs, ctx); - emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, - ctx); + emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); } else { emit_branch(BPF_OP(code), rd, rs, rvoff, ctx); } @@ -1468,18 +1410,18 @@ out_be: case BPF_JMP32 | BPF_JSLE | BPF_K: rvoff = rv_offset(i, off, ctx); s = ctx->ninsns; - if (imm) { + if (imm) emit_imm(RV_REG_T1, imm, ctx); - rs = RV_REG_T1; - } else { - /* If imm is 0, simply use zero register. */ - rs = RV_REG_ZERO; - } + rs = imm ? RV_REG_T1 : RV_REG_ZERO; if (!is64) { - if (is_signed_bpf_cond(BPF_OP(code))) - emit_sext_32_rd(&rd, ctx); - else - emit_zext_32_rd_t1(&rd, ctx); + if (is_signed_bpf_cond(BPF_OP(code))) { + emit_sextw_alt(&rd, RV_REG_T2, ctx); + /* rs has been sign extended */ + } else { + emit_zextw_alt(&rd, RV_REG_T2, ctx); + if (imm) + emit_zextw(rs, rs, ctx); + } } e = ctx->ninsns; @@ -1503,7 +1445,7 @@ out_be: * as t1 is used only in comparison against zero. */ if (!is64 && imm < 0) - emit_addiw(RV_REG_T1, RV_REG_T1, 0, ctx); + emit_sextw(RV_REG_T1, RV_REG_T1, ctx); e = ctx->ninsns; rvoff -= ninsns_rvoff(e - s); emit_branch(BPF_JNE, RV_REG_T1, RV_REG_ZERO, rvoff, ctx); @@ -1778,7 +1720,7 @@ out_be: return 0; } -void bpf_jit_build_prologue(struct rv_jit_context *ctx) +void bpf_jit_build_prologue(struct rv_jit_context *ctx, bool is_subprog) { int i, stack_adjust = 0, store_offset, bpf_stack_adjust; @@ -1807,6 +1749,9 @@ void bpf_jit_build_prologue(struct rv_jit_context *ctx) store_offset = stack_adjust - 8; + /* emit kcfi type preamble immediately before the first insn */ + emit_kcfi(is_subprog ? cfi_bpf_subprog_hash : cfi_bpf_hash, ctx); + /* nops reserved for auipc+jalr pair */ for (i = 0; i < RV_FENTRY_NINSNS; i++) emit(rv_nop(), ctx); @@ -1873,3 +1818,8 @@ bool bpf_jit_supports_kfunc_call(void) { return true; } + +bool bpf_jit_supports_ptr_xchg(void) +{ + return true; +} diff --git a/arch/riscv/net/bpf_jit_core.c b/arch/riscv/net/bpf_jit_core.c index 7b70ccb7fec3..6b3acac30c06 100644 --- a/arch/riscv/net/bpf_jit_core.c +++ b/arch/riscv/net/bpf_jit_core.c @@ -10,6 +10,7 @@ #include <linux/filter.h> #include <linux/memory.h> #include <asm/patch.h> +#include <asm/cfi.h> #include "bpf_jit.h" /* Number of iterations to try until offsets converge. */ @@ -100,7 +101,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) pass++; ctx->ninsns = 0; - bpf_jit_build_prologue(ctx); + bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); ctx->prologue_len = ctx->ninsns; if (build_body(ctx, extra_pass, ctx->offset)) { @@ -160,7 +161,7 @@ skip_init_ctx: ctx->ninsns = 0; ctx->nexentries = 0; - bpf_jit_build_prologue(ctx); + bpf_jit_build_prologue(ctx, bpf_is_subprog(prog)); if (build_body(ctx, extra_pass, NULL)) { prog = orig_prog; goto out_free_hdr; @@ -170,9 +171,9 @@ skip_init_ctx: if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, prog_size, pass, ctx->insns); - prog->bpf_func = (void *)ctx->ro_insns; + prog->bpf_func = (void *)ctx->ro_insns + cfi_get_offset(); prog->jited = 1; - prog->jited_len = prog_size; + prog->jited_len = prog_size - cfi_get_offset(); if (!prog->is_func || extra_pass) { if (WARN_ON(bpf_jit_binary_pack_finalize(prog, jit_data->ro_header, |