diff options
285 files changed, 5839 insertions, 2680 deletions
diff --git a/Documentation/ABI/testing/sysfs-firmware-efi b/Documentation/ABI/testing/sysfs-firmware-efi index 5e4d0b27cdfe..927e362d4974 100644 --- a/Documentation/ABI/testing/sysfs-firmware-efi +++ b/Documentation/ABI/testing/sysfs-firmware-efi @@ -36,3 +36,10 @@ Description: Displays the content of the Runtime Configuration Interface Table version 2 on Dell EMC PowerEdge systems in binary format Users: It is used by Dell EMC OpenManage Server Administrator tool to populate BIOS setup page. + +What: /sys/firmware/efi/ovmf_debug_log +Date: July 2025 +Contact: Gerd Hoffmann <kraxel@redhat.com>, linux-efi@vger.kernel.org +Description: Displays the content of the OVMF debug log buffer. The file is + only present in case the firmware supports logging to a memory + buffer. diff --git a/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml b/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml index 75d5d97305e1..87d31963c1b7 100644 --- a/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/allwinner,sun6i-a31-msgbox.yaml @@ -68,13 +68,13 @@ examples: #include <dt-bindings/reset/sun8i-h3-ccu.h> msgbox: mailbox@1c17000 { - compatible = "allwinner,sun8i-h3-msgbox", - "allwinner,sun6i-a31-msgbox"; - reg = <0x01c17000 0x1000>; - clocks = <&ccu CLK_BUS_MSGBOX>; - resets = <&ccu RST_BUS_MSGBOX>; - interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>; - #mbox-cells = <1>; + compatible = "allwinner,sun8i-h3-msgbox", + "allwinner,sun6i-a31-msgbox"; + reg = <0x01c17000 0x1000>; + clocks = <&ccu CLK_BUS_MSGBOX>; + resets = <&ccu RST_BUS_MSGBOX>; + interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>; + #mbox-cells = <1>; }; ... diff --git a/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml b/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml index 385809ed1569..79963c9878ba 100644 --- a/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml +++ b/Documentation/devicetree/bindings/mailbox/amlogic,meson-gxbb-mhu.yaml @@ -27,7 +27,7 @@ properties: maxItems: 1 interrupts: - minItems: 3 + maxItems: 3 description: Contains the interrupt information corresponding to each of the 3 links of MHU. @@ -46,8 +46,8 @@ additionalProperties: false examples: - | mailbox@c883c404 { - compatible = "amlogic,meson-gxbb-mhu"; - reg = <0xc883c404 0x4c>; - interrupts = <208>, <209>, <210>; - #mbox-cells = <1>; + compatible = "amlogic,meson-gxbb-mhu"; + reg = <0xc883c404 0x4c>; + interrupts = <208>, <209>, <210>; + #mbox-cells = <1>; }; diff --git a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml index 4c0668e5f0bd..474c1a0f99f3 100644 --- a/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/apple,mailbox.yaml @@ -78,11 +78,11 @@ additionalProperties: false examples: - | - mailbox@77408000 { - compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; - reg = <0x77408000 0x4000>; - interrupts = <1 583 4>, <1 584 4>, <1 585 4>, <1 586 4>; - interrupt-names = "send-empty", "send-not-empty", - "recv-empty", "recv-not-empty"; - #mbox-cells = <0>; - }; + mailbox@77408000 { + compatible = "apple,t8103-asc-mailbox", "apple,asc-mailbox-v4"; + reg = <0x77408000 0x4000>; + interrupts = <1 583 4>, <1 584 4>, <1 585 4>, <1 586 4>; + interrupt-names = "send-empty", "send-not-empty", + "recv-empty", "recv-not-empty"; + #mbox-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/mailbox/aspeed,ast2700-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/aspeed,ast2700-mailbox.yaml new file mode 100644 index 000000000000..600e2d63fccd --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/aspeed,ast2700-mailbox.yaml @@ -0,0 +1,68 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/aspeed,ast2700-mailbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: ASPEED AST2700 mailbox controller + +maintainers: + - Jammy Huang <jammy_huang@aspeedtech.com> + +description: > + ASPEED AST2700 has multiple processors that need to communicate with each + other. The mailbox controller provides a way for these processors to send + messages to each other. It is a hardware-based inter-processor communication + mechanism that allows processors to send and receive messages through + dedicated channels. + + The mailbox's tx/rx are independent, meaning that one processor can send a + message while another processor is receiving a message simultaneously. + There are 4 channels available for both tx and rx operations. Each channel + has a FIFO buffer that can hold messages of a fixed size (32 bytes in this + case). + + The mailbox controller also supports interrupt generation, allowing + processors to notify each other when a message is available or when an event + occurs. + +properties: + compatible: + const: aspeed,ast2700-mailbox + + reg: + items: + - description: TX control register + - description: RX control register + + reg-names: + items: + - const: tx + - const: rx + + interrupts: + maxItems: 1 + + "#mbox-cells": + const: 1 + +required: + - compatible + - reg + - reg-names + - interrupts + - "#mbox-cells" + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + + mailbox@12c1c200 { + compatible = "aspeed,ast2700-mailbox"; + reg = <0x12c1c200 0x100>, <0x12c1c300 0x100>; + reg-names = "tx", "rx"; + interrupts = <GIC_SPI 103 IRQ_TYPE_LEVEL_HIGH>; + #mbox-cells = <1>; + }; diff --git a/Documentation/devicetree/bindings/mailbox/brcm,bcm74110-mbox.yaml b/Documentation/devicetree/bindings/mailbox/brcm,bcm74110-mbox.yaml new file mode 100644 index 000000000000..750cc96edb46 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/brcm,bcm74110-mbox.yaml @@ -0,0 +1,64 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/brcm,bcm74110-mbox.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Broadcom BCM74110 Mailbox + +maintainers: + - Justin Chen <justin.chen@broadcom.com> + - Florian Fainelli <florian.fainelli@broadcom.com> + +description: Broadcom mailbox hardware first introduced with 74110 + +properties: + compatible: + enum: + - brcm,bcm74110-mbox + + reg: + maxItems: 1 + + interrupts: + items: + - description: RX doorbell and watermark interrupts + - description: TX doorbell and watermark interrupts + + "#mbox-cells": + const: 2 + description: + The first cell is channel type and second cell is shared memory slot + + brcm,rx: + $ref: /schemas/types.yaml#/definitions/uint32 + description: RX Mailbox number + + brcm,tx: + $ref: /schemas/types.yaml#/definitions/uint32 + description: TX Mailbox number + +required: + - compatible + - reg + - interrupts + - "#mbox-cells" + - brcm,rx + - brcm,tx + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + mailbox@a552000 { + compatible = "brcm,bcm74110-mbox"; + reg = <0xa552000 0x1104>; + interrupts = <GIC_SPI 0x67 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 0x66 IRQ_TYPE_LEVEL_HIGH>; + #mbox-cells = <0x2>; + brcm,rx = <0x7>; + brcm,tx = <0x6>; + }; diff --git a/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml index d97050e40fbf..f833b845de0d 100644 --- a/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml +++ b/Documentation/devicetree/bindings/mailbox/nvidia,tegra186-hsp.yaml @@ -59,9 +59,6 @@ description: | <dt-bindings/mailbox/tegra186-hsp.h> properties: - $nodename: - pattern: "^hsp@[0-9a-f]+$" - compatible: oneOf: - enum: @@ -131,14 +128,10 @@ examples: #include <dt-bindings/interrupt-controller/arm-gic.h> #include <dt-bindings/mailbox/tegra186-hsp.h> - hsp_top0: hsp@3c00000 { + mailbox@3c00000 { compatible = "nvidia,tegra186-hsp"; reg = <0x03c00000 0xa0000>; interrupts = <GIC_SPI 176 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "doorbell"; #mbox-cells = <2>; }; - - client { - mboxes = <&hsp_top0 TEGRA_HSP_MBOX_TYPE_DB TEGRA_HSP_DB_MASTER_CCPLEX>; - }; diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml index ac726136f7e5..615ed103b7e6 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -251,7 +251,7 @@ examples: # Example apcs with msm8996 - | #include <dt-bindings/interrupt-controller/arm-gic.h> - apcs_glb: mailbox@9820000 { + mailbox@9820000 { compatible = "qcom,msm8996-apcs-hmss-global"; reg = <0x9820000 0x1000>; @@ -259,13 +259,6 @@ examples: #clock-cells = <0>; }; - rpm-glink { - compatible = "qcom,glink-rpm"; - interrupts = <GIC_SPI 168 IRQ_TYPE_EDGE_RISING>; - qcom,rpm-msg-ram = <&rpm_msg_ram>; - mboxes = <&apcs_glb 0>; - }; - # Example apcs with qcs404 - | #define GCC_APSS_AHB_CLK_SRC 1 diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml index f69c0ec5d19d..e5c423130db6 100644 --- a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml +++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml @@ -24,6 +24,7 @@ properties: compatible: items: - enum: + - qcom,milos-ipcc - qcom,qcs8300-ipcc - qcom,qdu1000-ipcc - qcom,sa8255p-ipcc diff --git a/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml index 1a2001e58880..8504ceb64806 100644 --- a/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml +++ b/Documentation/devicetree/bindings/mailbox/ti,omap-mailbox.yaml @@ -242,7 +242,7 @@ examples: - | /* OMAP4 */ #include <dt-bindings/interrupt-controller/arm-gic.h> - mailbox: mailbox@4a0f4000 { + mailbox@4a0f4000 { compatible = "ti,omap4-mailbox"; reg = <0x4a0f4000 0x200>; interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>; @@ -260,13 +260,9 @@ examples: }; }; - dsp { - mboxes = <&mailbox &mbox_dsp>; - }; - - | /* AM33xx */ - mailbox1: mailbox@480c8000 { + mailbox@480c8000 { compatible = "ti,omap4-mailbox"; reg = <0x480c8000 0x200>; interrupts = <77>; @@ -283,7 +279,7 @@ examples: - | /* AM65x */ - mailbox0_cluster0: mailbox@31f80000 { + mailbox@31f80000 { compatible = "ti,am654-mailbox"; reg = <0x31f80000 0x200>; #mbox-cells = <1>; diff --git a/Documentation/devicetree/bindings/mailbox/ti,secure-proxy.yaml b/Documentation/devicetree/bindings/mailbox/ti,secure-proxy.yaml index eea822861804..c321b69f0ccd 100644 --- a/Documentation/devicetree/bindings/mailbox/ti,secure-proxy.yaml +++ b/Documentation/devicetree/bindings/mailbox/ti,secure-proxy.yaml @@ -36,7 +36,7 @@ properties: - const: scfg reg: - minItems: 3 + maxItems: 3 interrupt-names: minItems: 1 @@ -68,12 +68,12 @@ examples: - | #include <dt-bindings/interrupt-controller/arm-gic.h> secure_proxy: mailbox@32c00000 { - compatible = "ti,am654-secure-proxy"; - #mbox-cells = <1>; - reg-names = "target_data", "rt", "scfg"; - reg = <0x32c00000 0x100000>, - <0x32400000 0x100000>, - <0x32800000 0x100000>; - interrupt-names = "rx_011"; - interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>; + compatible = "ti,am654-secure-proxy"; + #mbox-cells = <1>; + reg-names = "target_data", "rt", "scfg"; + reg = <0x32c00000 0x100000>, + <0x32400000 0x100000>, + <0x32800000 0x100000>; + interrupt-names = "rx_011"; + interrupts = <GIC_SPI 32 IRQ_TYPE_LEVEL_HIGH>; }; diff --git a/Documentation/devicetree/bindings/net/adi,adin.yaml b/Documentation/devicetree/bindings/net/adi,adin.yaml index 929cf8c0b0fd..c425a9f1886d 100644 --- a/Documentation/devicetree/bindings/net/adi,adin.yaml +++ b/Documentation/devicetree/bindings/net/adi,adin.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: Analog Devices ADIN1200/ADIN1300 PHY maintainers: - - Alexandru Tachici <alexandru.tachici@analog.com> + - Marcelo Schmitt <marcelo.schmitt@analog.com> description: | Bindings for Analog Devices Industrial Ethernet PHYs diff --git a/Documentation/devicetree/bindings/net/adi,adin1110.yaml b/Documentation/devicetree/bindings/net/adi,adin1110.yaml index 9de865295d7a..0a73e01d7f97 100644 --- a/Documentation/devicetree/bindings/net/adi,adin1110.yaml +++ b/Documentation/devicetree/bindings/net/adi,adin1110.yaml @@ -7,7 +7,7 @@ $schema: http://devicetree.org/meta-schemas/core.yaml# title: ADI ADIN1110 MAC-PHY maintainers: - - Alexandru Tachici <alexandru.tachici@analog.com> + - Marcelo Schmitt <marcelo.schmitt@analog.com> description: | The ADIN1110 is a low power single port 10BASE-T1L MAC- diff --git a/Documentation/netlink/specs/ethtool.yaml b/Documentation/netlink/specs/ethtool.yaml index 1063d5d32fea..1bc1bd7d33c2 100644 --- a/Documentation/netlink/specs/ethtool.yaml +++ b/Documentation/netlink/specs/ethtool.yaml @@ -2344,14 +2344,14 @@ operations: request: attributes: - header - reply: - attributes: - - header - offset - length - page - bank - i2c-address + reply: + attributes: + - header - data dump: *module-eeprom-get-op - diff --git a/MAINTAINERS b/MAINTAINERS index bd62ad58a47f..fe168477caa4 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -21487,6 +21487,14 @@ S: Maintained F: Documentation/devicetree/bindings/net/renesas,rzv2h-gbeth.yaml F: drivers/net/ethernet/stmicro/stmmac/dwmac-renesas-gbeth.c +RENESAS RZ/V2H(P) RSPI DRIVER +M: Fabrizio Castro <fabrizio.castro.jz@renesas.com> +L: linux-spi@vger.kernel.org +L: linux-renesas-soc@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/spi/renesas,rzv2h-rspi.yaml +F: drivers/spi/spi-rzv2h-rspi.c + RENESAS RZ/V2H(P) USB2PHY PORT RESET DRIVER M: Fabrizio Castro <fabrizio.castro.jz@renesas.com> M: Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> diff --git a/arch/arm/boot/dts/broadcom/bcm7445.dtsi b/arch/arm/boot/dts/broadcom/bcm7445.dtsi index 5ac2042515b8..c6307c7437e3 100644 --- a/arch/arm/boot/dts/broadcom/bcm7445.dtsi +++ b/arch/arm/boot/dts/broadcom/bcm7445.dtsi @@ -237,7 +237,8 @@ ranges = <0x0 0x0 0x80000>; memc-ddr@2000 { - compatible = "brcm,brcmstb-memc-ddr"; + compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x", + "brcm,brcmstb-memc-ddr"; reg = <0x2000 0x800>; }; @@ -259,7 +260,8 @@ ranges = <0x0 0x80000 0x80000>; memc-ddr@2000 { - compatible = "brcm,brcmstb-memc-ddr"; + compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x", + "brcm,brcmstb-memc-ddr"; reg = <0x2000 0x800>; }; @@ -281,7 +283,8 @@ ranges = <0x0 0x100000 0x80000>; memc-ddr@2000 { - compatible = "brcm,brcmstb-memc-ddr"; + compatible = "brcm,brcmstb-memc-ddr-rev-b.1.x", + "brcm,brcmstb-memc-ddr"; reg = <0x2000 0x800>; }; diff --git a/arch/arm64/boot/dts/nvidia/tegra264.dtsi b/arch/arm64/boot/dts/nvidia/tegra264.dtsi index 62c87a387b14..e02659efa233 100644 --- a/arch/arm64/boot/dts/nvidia/tegra264.dtsi +++ b/arch/arm64/boot/dts/nvidia/tegra264.dtsi @@ -11,7 +11,6 @@ interrupt-parent = <&gic>; #address-cells = <2>; #size-cells = <2>; - numa-node-id = <0>; reserved-memory { #address-cells = <2>; @@ -341,7 +340,6 @@ status = "okay"; enable-method = "psci"; - numa-node-id = <0>; i-cache-size = <65536>; i-cache-line-size = <64>; @@ -358,7 +356,6 @@ status = "okay"; enable-method = "psci"; - numa-node-id = <0>; i-cache-size = <65536>; i-cache-line-size = <64>; diff --git a/arch/loongarch/boot/dts/loongson-2k0500-ref.dts b/arch/loongarch/boot/dts/loongson-2k0500-ref.dts index a34734a6c3ce..018ed904352a 100644 --- a/arch/loongarch/boot/dts/loongson-2k0500-ref.dts +++ b/arch/loongarch/boot/dts/loongson-2k0500-ref.dts @@ -41,6 +41,15 @@ }; }; +&apbdma3 { + status = "okay"; +}; + +&mmc0 { + status = "okay"; + bus-width = <4>; +}; + &gmac0 { status = "okay"; diff --git a/arch/loongarch/boot/dts/loongson-2k0500.dtsi b/arch/loongarch/boot/dts/loongson-2k0500.dtsi index 760c60eebb89..588ebc3bded4 100644 --- a/arch/loongarch/boot/dts/loongson-2k0500.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k0500.dtsi @@ -104,7 +104,7 @@ status = "disabled"; }; - dma-controller@1fe10c20 { + apbdma2: dma-controller@1fe10c20 { compatible = "loongson,ls2k0500-apbdma", "loongson,ls2k1000-apbdma"; reg = <0 0x1fe10c20 0 0x8>; interrupt-parent = <&eiointc>; @@ -114,7 +114,7 @@ status = "disabled"; }; - dma-controller@1fe10c30 { + apbdma3: dma-controller@1fe10c30 { compatible = "loongson,ls2k0500-apbdma", "loongson,ls2k1000-apbdma"; reg = <0 0x1fe10c30 0 0x8>; interrupt-parent = <&eiointc>; @@ -437,6 +437,30 @@ status = "disabled"; }; + mmc0: mmc@1ff64000 { + compatible = "loongson,ls2k0500-mmc"; + reg = <0 0x1ff64000 0 0x2000>, + <0 0x1fe10100 0 0x4>; + interrupt-parent = <&eiointc>; + interrupts = <57>; + dmas = <&apbdma3 0>; + dma-names = "rx-tx"; + clocks = <&clk LOONGSON2_APB_CLK>; + status = "disabled"; + }; + + mmc@1ff66000 { + compatible = "loongson,ls2k0500-mmc"; + reg = <0 0x1ff66000 0 0x2000>, + <0 0x1fe10100 0 0x4>; + interrupt-parent = <&eiointc>; + interrupts = <58>; + dmas = <&apbdma2 0>; + dma-names = "rx-tx"; + clocks = <&clk LOONGSON2_APB_CLK>; + status = "disabled"; + }; + pmc: power-management@1ff6c000 { compatible = "loongson,ls2k0500-pmc", "syscon"; reg = <0x0 0x1ff6c000 0x0 0x58>; diff --git a/arch/loongarch/boot/dts/loongson-2k1000-ref.dts b/arch/loongarch/boot/dts/loongson-2k1000-ref.dts index 78ea995abf1c..d9a452ada5d7 100644 --- a/arch/loongarch/boot/dts/loongson-2k1000-ref.dts +++ b/arch/loongarch/boot/dts/loongson-2k1000-ref.dts @@ -48,6 +48,19 @@ }; }; +&apbdma1 { + status = "okay"; +}; + +&mmc { + status = "okay"; + + pinctrl-0 = <&sdio_pins_default>; + pinctrl-names = "default"; + bus-width = <4>; + cd-gpios = <&gpio0 22 GPIO_ACTIVE_LOW>; +}; + &gmac0 { status = "okay"; diff --git a/arch/loongarch/boot/dts/loongson-2k1000.dtsi b/arch/loongarch/boot/dts/loongson-2k1000.dtsi index 1da3beb00f0e..d8e01e2534dd 100644 --- a/arch/loongarch/boot/dts/loongson-2k1000.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k1000.dtsi @@ -187,14 +187,14 @@ <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, - <>, - <26 IRQ_TYPE_LEVEL_HIGH>, + <0 IRQ_TYPE_NONE>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, + <26 IRQ_TYPE_NONE>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, <26 IRQ_TYPE_LEVEL_HIGH>, @@ -209,13 +209,13 @@ <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, - <>, + <0 IRQ_TYPE_NONE>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, - <>, - <>, + <0 IRQ_TYPE_NONE>, + <0 IRQ_TYPE_NONE>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, <27 IRQ_TYPE_LEVEL_HIGH>, @@ -256,7 +256,7 @@ status = "disabled"; }; - dma-controller@1fe00c10 { + apbdma1: dma-controller@1fe00c10 { compatible = "loongson,ls2k1000-apbdma"; reg = <0x0 0x1fe00c10 0x0 0x8>; interrupt-parent = <&liointc1>; @@ -405,6 +405,18 @@ status = "disabled"; }; + mmc: mmc@1fe2c000 { + compatible = "loongson,ls2k1000-mmc"; + reg = <0 0x1fe2c000 0 0x68>, + <0 0x1fe00438 0 0x8>; + interrupt-parent = <&liointc0>; + interrupts = <31 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk LOONGSON2_APB_CLK>; + dmas = <&apbdma1 0>; + dma-names = "rx-tx"; + status = "disabled"; + }; + spi0: spi@1fff0220 { compatible = "loongson,ls2k1000-spi"; reg = <0x0 0x1fff0220 0x0 0x10>; diff --git a/arch/loongarch/boot/dts/loongson-2k2000-ref.dts b/arch/loongarch/boot/dts/loongson-2k2000-ref.dts index ea9e6985d0e9..3c6b12220386 100644 --- a/arch/loongarch/boot/dts/loongson-2k2000-ref.dts +++ b/arch/loongarch/boot/dts/loongson-2k2000-ref.dts @@ -39,6 +39,16 @@ }; }; +&emmc { + status = "okay"; + + bus-width = <8>; + cap-mmc-highspeed; + mmc-hs200-1_8v; + no-sd; + no-sdio; +}; + &sata { status = "okay"; }; diff --git a/arch/loongarch/boot/dts/loongson-2k2000.dtsi b/arch/loongarch/boot/dts/loongson-2k2000.dtsi index 9e0411f2754c..00cc485b753b 100644 --- a/arch/loongarch/boot/dts/loongson-2k2000.dtsi +++ b/arch/loongarch/boot/dts/loongson-2k2000.dtsi @@ -259,6 +259,24 @@ status = "disabled"; }; + emmc: mmc@79990000 { + compatible = "loongson,ls2k2000-mmc"; + reg = <0x0 0x79990000 0x0 0x1000>; + interrupt-parent = <&pic>; + interrupts = <51 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk LOONGSON2_EMMC_CLK>; + status = "disabled"; + }; + + mmc@79991000 { + compatible = "loongson,ls2k2000-mmc"; + reg = <0x0 0x79991000 0x0 0x1000>; + interrupt-parent = <&pic>; + interrupts = <50 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&clk LOONGSON2_EMMC_CLK>; + status = "disabled"; + }; + pcie@1a000000 { compatible = "loongson,ls2k-pci"; reg = <0x0 0x1a000000 0x0 0x02000000>, diff --git a/arch/loongarch/include/asm/inst.h b/arch/loongarch/include/asm/inst.h index 3089785ca97e..277d2140676b 100644 --- a/arch/loongarch/include/asm/inst.h +++ b/arch/loongarch/include/asm/inst.h @@ -497,6 +497,7 @@ void arch_simulate_insn(union loongarch_instruction insn, struct pt_regs *regs); int larch_insn_read(void *addr, u32 *insnp); int larch_insn_write(void *addr, u32 insn); int larch_insn_patch_text(void *addr, u32 insn); +int larch_insn_text_copy(void *dst, void *src, size_t len); u32 larch_insn_gen_nop(void); u32 larch_insn_gen_b(unsigned long pc, unsigned long dest); @@ -510,6 +511,8 @@ u32 larch_insn_gen_move(enum loongarch_gpr rd, enum loongarch_gpr rj); u32 larch_insn_gen_lu12iw(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu32id(enum loongarch_gpr rd, int imm); u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm); static inline bool signed_imm_check(long val, unsigned int bit) diff --git a/arch/loongarch/include/asm/loongarch.h b/arch/loongarch/include/asm/loongarch.h index a0994d226eff..09dfd7eb406e 100644 --- a/arch/loongarch/include/asm/loongarch.h +++ b/arch/loongarch/include/asm/loongarch.h @@ -451,6 +451,13 @@ #define LOONGARCH_CSR_KS6 0x36 #define LOONGARCH_CSR_KS7 0x37 #define LOONGARCH_CSR_KS8 0x38 +#define LOONGARCH_CSR_KS9 0x39 +#define LOONGARCH_CSR_KS10 0x3a +#define LOONGARCH_CSR_KS11 0x3b +#define LOONGARCH_CSR_KS12 0x3c +#define LOONGARCH_CSR_KS13 0x3d +#define LOONGARCH_CSR_KS14 0x3e +#define LOONGARCH_CSR_KS15 0x3f /* Exception allocated KS0, KS1 and KS2 statically */ #define EXCEPTION_KS0 LOONGARCH_CSR_KS0 diff --git a/arch/loongarch/kernel/env.c b/arch/loongarch/kernel/env.c index 27144de5c5fe..c0a5dc9aeae2 100644 --- a/arch/loongarch/kernel/env.c +++ b/arch/loongarch/kernel/env.c @@ -39,16 +39,19 @@ void __init init_environ(void) static int __init init_cpu_fullname(void) { - struct device_node *root; int cpu, ret; - char *model; + char *cpuname; + const char *model; + struct device_node *root; /* Parsing cpuname from DTS model property */ root = of_find_node_by_path("/"); - ret = of_property_read_string(root, "model", (const char **)&model); + ret = of_property_read_string(root, "model", &model); + if (ret == 0) { + cpuname = kstrdup(model, GFP_KERNEL); + loongson_sysconf.cpuname = strsep(&cpuname, " "); + } of_node_put(root); - if (ret == 0) - loongson_sysconf.cpuname = strsep(&model, " "); if (loongson_sysconf.cpuname && !strncmp(loongson_sysconf.cpuname, "Loongson", 8)) { for (cpu = 0; cpu < NR_CPUS; cpu++) diff --git a/arch/loongarch/kernel/inst.c b/arch/loongarch/kernel/inst.c index 14d7d700bcb9..72ecfed29d55 100644 --- a/arch/loongarch/kernel/inst.c +++ b/arch/loongarch/kernel/inst.c @@ -4,6 +4,8 @@ */ #include <linux/sizes.h> #include <linux/uaccess.h> +#include <linux/set_memory.h> +#include <linux/stop_machine.h> #include <asm/cacheflush.h> #include <asm/inst.h> @@ -218,6 +220,50 @@ int larch_insn_patch_text(void *addr, u32 insn) return ret; } +struct insn_copy { + void *dst; + void *src; + size_t len; + unsigned int cpu; +}; + +static int text_copy_cb(void *data) +{ + int ret = 0; + struct insn_copy *copy = data; + + if (smp_processor_id() == copy->cpu) { + ret = copy_to_kernel_nofault(copy->dst, copy->src, copy->len); + if (ret) + pr_err("%s: operation failed\n", __func__); + } + + flush_icache_range((unsigned long)copy->dst, (unsigned long)copy->dst + copy->len); + + return ret; +} + +int larch_insn_text_copy(void *dst, void *src, size_t len) +{ + int ret = 0; + size_t start, end; + struct insn_copy copy = { + .dst = dst, + .src = src, + .len = len, + .cpu = smp_processor_id(), + }; + + start = round_down((size_t)dst, PAGE_SIZE); + end = round_up((size_t)dst + len, PAGE_SIZE); + + set_memory_rw(start, (end - start) / PAGE_SIZE); + ret = stop_machine(text_copy_cb, ©, cpu_online_mask); + set_memory_rox(start, (end - start) / PAGE_SIZE); + + return ret; +} + u32 larch_insn_gen_nop(void) { return INSN_NOP; @@ -323,6 +369,34 @@ u32 larch_insn_gen_lu52id(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) return insn.word; } +u32 larch_insn_gen_beq(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + union loongarch_instruction insn; + + if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) { + pr_warn("The generated beq instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_beq(&insn, rj, rd, imm >> 2); + + return insn.word; +} + +u32 larch_insn_gen_bne(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) +{ + union loongarch_instruction insn; + + if ((imm & 3) || imm < -SZ_128K || imm >= SZ_128K) { + pr_warn("The generated bne instruction is out of range.\n"); + return INSN_BREAK; + } + + emit_bne(&insn, rj, rd, imm >> 2); + + return insn.word; +} + u32 larch_insn_gen_jirl(enum loongarch_gpr rd, enum loongarch_gpr rj, int imm) { union loongarch_instruction insn; diff --git a/arch/loongarch/kernel/relocate_kernel.S b/arch/loongarch/kernel/relocate_kernel.S index 84e6de2fd973..8b5140ac9ea1 100644 --- a/arch/loongarch/kernel/relocate_kernel.S +++ b/arch/loongarch/kernel/relocate_kernel.S @@ -109,4 +109,4 @@ SYM_CODE_END(kexec_smp_wait) relocate_new_kernel_end: .section ".data" -SYM_DATA(relocate_new_kernel_size, .long relocate_new_kernel_end - relocate_new_kernel) +SYM_DATA(relocate_new_kernel_size, .quad relocate_new_kernel_end - relocate_new_kernel) diff --git a/arch/loongarch/kernel/setup.c b/arch/loongarch/kernel/setup.c index 22b27cd447a1..075b79b2c1d3 100644 --- a/arch/loongarch/kernel/setup.c +++ b/arch/loongarch/kernel/setup.c @@ -191,6 +191,16 @@ static int __init early_parse_mem(char *p) return -EINVAL; } + start = 0; + size = memparse(p, &p); + if (*p == '@') /* Every mem=... should contain '@' */ + start = memparse(p + 1, &p); + else { /* Only one mem=... is allowed if no '@' */ + usermem = 1; + memblock_enforce_memory_limit(size); + return 0; + } + /* * If a user specifies memory size, we * blow away any automatically generated @@ -201,14 +211,6 @@ static int __init early_parse_mem(char *p) memblock_remove(memblock_start_of_DRAM(), memblock_end_of_DRAM() - memblock_start_of_DRAM()); } - start = 0; - size = memparse(p, &p); - if (*p == '@') - start = memparse(p + 1, &p); - else { - pr_err("Invalid format!\n"); - return -EINVAL; - } if (!IS_ENABLED(CONFIG_NUMA)) memblock_add(start, size); diff --git a/arch/loongarch/kernel/unwind_orc.c b/arch/loongarch/kernel/unwind_orc.c index 0005be49b056..0d5fa64a2225 100644 --- a/arch/loongarch/kernel/unwind_orc.c +++ b/arch/loongarch/kernel/unwind_orc.c @@ -508,7 +508,7 @@ bool unwind_next_frame(struct unwind_state *state) state->pc = bt_address(pc); if (!state->pc) { - pr_err("cannot find unwind pc at %pK\n", (void *)pc); + pr_err("cannot find unwind pc at %p\n", (void *)pc); goto err; } diff --git a/arch/loongarch/net/bpf_jit.c b/arch/loongarch/net/bpf_jit.c index fa1500d4aa3e..abfdb6bb5c38 100644 --- a/arch/loongarch/net/bpf_jit.c +++ b/arch/loongarch/net/bpf_jit.c @@ -4,13 +4,20 @@ * * Copyright (C) 2022 Loongson Technology Corporation Limited */ +#include <linux/memory.h> #include "bpf_jit.h" -#define REG_TCC LOONGARCH_GPR_A6 -#define TCC_SAVED LOONGARCH_GPR_S5 +#define LOONGARCH_MAX_REG_ARGS 8 + +#define LOONGARCH_LONG_JUMP_NINSNS 5 +#define LOONGARCH_LONG_JUMP_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) -#define SAVE_RA BIT(0) -#define SAVE_TCC BIT(1) +#define LOONGARCH_FENTRY_NINSNS 2 +#define LOONGARCH_FENTRY_NBYTES (LOONGARCH_FENTRY_NINSNS * 4) +#define LOONGARCH_BPF_FENTRY_NBYTES (LOONGARCH_LONG_JUMP_NINSNS * 4) + +#define REG_TCC LOONGARCH_GPR_A6 +#define BPF_TAIL_CALL_CNT_PTR_STACK_OFF(stack) (round_up(stack, 16) - 80) static const int regmap[] = { /* return value from in-kernel function, and exit value for eBPF program */ @@ -32,32 +39,57 @@ static const int regmap[] = { [BPF_REG_AX] = LOONGARCH_GPR_T0, }; -static void mark_call(struct jit_ctx *ctx) +static void prepare_bpf_tail_call_cnt(struct jit_ctx *ctx, int *store_offset) { - ctx->flags |= SAVE_RA; -} + const struct bpf_prog *prog = ctx->prog; + const bool is_main_prog = !bpf_is_subprog(prog); -static void mark_tail_call(struct jit_ctx *ctx) -{ - ctx->flags |= SAVE_TCC; -} + if (is_main_prog) { + /* + * LOONGARCH_GPR_T3 = MAX_TAIL_CALL_CNT + * if (REG_TCC > T3 ) + * std REG_TCC -> LOONGARCH_GPR_SP + store_offset + * else + * std REG_TCC -> LOONGARCH_GPR_SP + store_offset + * REG_TCC = LOONGARCH_GPR_SP + store_offset + * + * std REG_TCC -> LOONGARCH_GPR_SP + store_offset + * + * The purpose of this code is to first push the TCC into stack, + * and then push the address of TCC into stack. + * In cases where bpf2bpf and tailcall are used in combination, + * the value in REG_TCC may be a count or an address, + * these two cases need to be judged and handled separately. + */ + emit_insn(ctx, addid, LOONGARCH_GPR_T3, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + *store_offset -= sizeof(long); -static bool seen_call(struct jit_ctx *ctx) -{ - return (ctx->flags & SAVE_RA); -} + emit_cond_jmp(ctx, BPF_JGT, REG_TCC, LOONGARCH_GPR_T3, 4); -static bool seen_tail_call(struct jit_ctx *ctx) -{ - return (ctx->flags & SAVE_TCC); -} + /* + * If REG_TCC < MAX_TAIL_CALL_CNT, the value in REG_TCC is a count, + * push tcc into stack + */ + emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); -static u8 tail_call_reg(struct jit_ctx *ctx) -{ - if (seen_call(ctx)) - return TCC_SAVED; + /* Push the address of TCC into the REG_TCC */ + emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + + emit_uncond_jmp(ctx, 2); + + /* + * If REG_TCC > MAX_TAIL_CALL_CNT, the value in REG_TCC is an address, + * push tcc_ptr into stack + */ + emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + } else { + *store_offset -= sizeof(long); + emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); + } - return REG_TCC; + /* Push tcc_ptr into stack */ + *store_offset -= sizeof(long); + emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_SP, *store_offset); } /* @@ -80,6 +112,10 @@ static u8 tail_call_reg(struct jit_ctx *ctx) * | $s4 | * +-------------------------+ * | $s5 | + * +-------------------------+ + * | tcc | + * +-------------------------+ + * | tcc_ptr | * +-------------------------+ <--BPF_REG_FP * | prog->aux->stack_depth | * | (optional) | @@ -88,22 +124,32 @@ static u8 tail_call_reg(struct jit_ctx *ctx) */ static void build_prologue(struct jit_ctx *ctx) { - int stack_adjust = 0, store_offset, bpf_stack_adjust; + int i, stack_adjust = 0, store_offset, bpf_stack_adjust; + const struct bpf_prog *prog = ctx->prog; + const bool is_main_prog = !bpf_is_subprog(prog); bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); - /* To store ra, fp, s0, s1, s2, s3, s4 and s5. */ + /* To store ra, fp, s0, s1, s2, s3, s4, s5 */ stack_adjust += sizeof(long) * 8; + /* To store tcc and tcc_ptr */ + stack_adjust += sizeof(long) * 2; + stack_adjust = round_up(stack_adjust, 16); stack_adjust += bpf_stack_adjust; + /* Reserve space for the move_imm + jirl instruction */ + for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn(ctx, nop); + /* - * First instruction initializes the tail call count (TCC). - * On tail call we skip this instruction, and the TCC is - * passed in REG_TCC from the caller. + * First instruction initializes the tail call count (TCC) + * register to zero. On tail call we skip this instruction, + * and the TCC is passed in REG_TCC from the caller. */ - emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + if (is_main_prog) + emit_insn(ctx, addid, REG_TCC, LOONGARCH_GPR_ZERO, 0); emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_adjust); @@ -131,20 +177,13 @@ static void build_prologue(struct jit_ctx *ctx) store_offset -= sizeof(long); emit_insn(ctx, std, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, store_offset); + prepare_bpf_tail_call_cnt(ctx, &store_offset); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_adjust); if (bpf_stack_adjust) emit_insn(ctx, addid, regmap[BPF_REG_FP], LOONGARCH_GPR_SP, bpf_stack_adjust); - /* - * Program contains calls and tail calls, so REG_TCC need - * to be saved across calls. - */ - if (seen_tail_call(ctx) && seen_call(ctx)) - move_reg(ctx, TCC_SAVED, REG_TCC); - else - emit_insn(ctx, nop); - ctx->stack_size = stack_adjust; } @@ -177,6 +216,16 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) load_offset -= sizeof(long); emit_insn(ctx, ldd, LOONGARCH_GPR_S5, LOONGARCH_GPR_SP, load_offset); + /* + * When push into the stack, follow the order of tcc then tcc_ptr. + * When pop from the stack, first pop tcc_ptr then followed by tcc. + */ + load_offset -= 2 * sizeof(long); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); + + load_offset += sizeof(long); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, load_offset); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_adjust); if (!is_tail_call) { @@ -189,7 +238,7 @@ static void __build_epilogue(struct jit_ctx *ctx, bool is_tail_call) * Call the next bpf prog and skip the first instruction * of TCC initialization. */ - emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 1); + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T3, 6); } } @@ -208,12 +257,10 @@ bool bpf_jit_supports_far_kfunc_call(void) return true; } -/* initialized on the first pass of build_body() */ -static int out_offset = -1; -static int emit_bpf_tail_call(struct jit_ctx *ctx) +static int emit_bpf_tail_call(struct jit_ctx *ctx, int insn) { - int off; - u8 tcc = tail_call_reg(ctx); + int off, tc_ninsn = 0; + int tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); u8 a1 = LOONGARCH_GPR_A1; u8 a2 = LOONGARCH_GPR_A2; u8 t1 = LOONGARCH_GPR_T1; @@ -222,7 +269,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) const int idx0 = ctx->idx; #define cur_offset (ctx->idx - idx0) -#define jmp_offset (out_offset - (cur_offset)) +#define jmp_offset (tc_ninsn - (cur_offset)) /* * a0: &ctx @@ -232,6 +279,7 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) * if (index >= array->map.max_entries) * goto out; */ + tc_ninsn = insn ? ctx->offset[insn+1] - ctx->offset[insn] : ctx->offset[0]; off = offsetof(struct bpf_array, map.max_entries); emit_insn(ctx, ldwu, t1, a1, off); /* bgeu $a2, $t1, jmp_offset */ @@ -239,11 +287,15 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) goto toofar; /* - * if (--TCC < 0) - * goto out; + * if ((*tcc_ptr)++ >= MAX_TAIL_CALL_CNT) + * goto out; */ - emit_insn(ctx, addid, REG_TCC, tcc, -1); - if (emit_tailcall_jmp(ctx, BPF_JSLT, REG_TCC, LOONGARCH_GPR_ZERO, jmp_offset) < 0) + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); + emit_insn(ctx, ldd, t3, REG_TCC, 0); + emit_insn(ctx, addid, t3, t3, 1); + emit_insn(ctx, std, t3, REG_TCC, 0); + emit_insn(ctx, addid, t2, LOONGARCH_GPR_ZERO, MAX_TAIL_CALL_CNT); + if (emit_tailcall_jmp(ctx, BPF_JSGT, t3, t2, jmp_offset) < 0) goto toofar; /* @@ -263,15 +315,6 @@ static int emit_bpf_tail_call(struct jit_ctx *ctx) emit_insn(ctx, ldd, t3, t2, off); __build_epilogue(ctx, true); - /* out: */ - if (out_offset == -1) - out_offset = cur_offset; - if (cur_offset != out_offset) { - pr_err_once("tail_call out_offset = %d, expected %d!\n", - cur_offset, out_offset); - return -1; - } - return 0; toofar: @@ -463,7 +506,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext u64 func_addr; bool func_addr_fixed, sign_extend; int i = insn - ctx->prog->insnsi; - int ret, jmp_offset; + int ret, jmp_offset, tcc_ptr_off; const u8 code = insn->code; const u8 cond = BPF_OP(code); const u8 t1 = LOONGARCH_GPR_T1; @@ -899,12 +942,16 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* function call */ case BPF_JMP | BPF_CALL: - mark_call(ctx); ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, &func_addr, &func_addr_fixed); if (ret < 0) return ret; + if (insn->src_reg == BPF_PSEUDO_CALL) { + tcc_ptr_off = BPF_TAIL_CALL_CNT_PTR_STACK_OFF(ctx->stack_size); + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_SP, tcc_ptr_off); + } + move_addr(ctx, t1, func_addr); emit_insn(ctx, jirl, LOONGARCH_GPR_RA, t1, 0); @@ -915,8 +962,7 @@ static int build_insn(const struct bpf_insn *insn, struct jit_ctx *ctx, bool ext /* tail call */ case BPF_JMP | BPF_TAIL_CALL: - mark_tail_call(ctx); - if (emit_bpf_tail_call(ctx) < 0) + if (emit_bpf_tail_call(ctx, i) < 0) return -EINVAL; break; @@ -1180,12 +1226,528 @@ static int validate_code(struct jit_ctx *ctx) return -1; } + return 0; +} + +static int validate_ctx(struct jit_ctx *ctx) +{ + if (validate_code(ctx)) + return -1; + if (WARN_ON_ONCE(ctx->num_exentries != ctx->prog->aux->num_exentries)) return -1; return 0; } +static int emit_jump_and_link(struct jit_ctx *ctx, u8 rd, u64 target) +{ + if (!target) { + pr_err("bpf_jit: jump target address is error\n"); + return -EFAULT; + } + + move_imm(ctx, LOONGARCH_GPR_T1, target, false); + emit_insn(ctx, jirl, rd, LOONGARCH_GPR_T1, 0); + + return 0; +} + +static int emit_jump_or_nops(void *target, void *ip, u32 *insns, bool is_call) +{ + int i; + struct jit_ctx ctx; + + ctx.idx = 0; + ctx.image = (union loongarch_instruction *)insns; + + if (!target) { + for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn((&ctx), nop); + return 0; + } + + return emit_jump_and_link(&ctx, is_call ? LOONGARCH_GPR_T0 : LOONGARCH_GPR_ZERO, (u64)target); +} + +static int emit_call(struct jit_ctx *ctx, u64 addr) +{ + return emit_jump_and_link(ctx, LOONGARCH_GPR_RA, addr); +} + +void *bpf_arch_text_copy(void *dst, void *src, size_t len) +{ + int ret; + + mutex_lock(&text_mutex); + ret = larch_insn_text_copy(dst, src, len); + mutex_unlock(&text_mutex); + + return ret ? ERR_PTR(-EINVAL) : dst; +} + +int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type poke_type, + void *old_addr, void *new_addr) +{ + int ret; + bool is_call = (poke_type == BPF_MOD_CALL); + u32 old_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; + u32 new_insns[LOONGARCH_LONG_JUMP_NINSNS] = {[0 ... 4] = INSN_NOP}; + + if (!is_kernel_text((unsigned long)ip) && + !is_bpf_text_address((unsigned long)ip)) + return -ENOTSUPP; + + ret = emit_jump_or_nops(old_addr, ip, old_insns, is_call); + if (ret) + return ret; + + if (memcmp(ip, old_insns, LOONGARCH_LONG_JUMP_NBYTES)) + return -EFAULT; + + ret = emit_jump_or_nops(new_addr, ip, new_insns, is_call); + if (ret) + return ret; + + mutex_lock(&text_mutex); + if (memcmp(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES)) + ret = larch_insn_text_copy(ip, new_insns, LOONGARCH_LONG_JUMP_NBYTES); + mutex_unlock(&text_mutex); + + return ret; +} + +int bpf_arch_text_invalidate(void *dst, size_t len) +{ + int i; + int ret = 0; + u32 *inst; + + inst = kvmalloc(len, GFP_KERNEL); + if (!inst) + return -ENOMEM; + + for (i = 0; i < (len / sizeof(u32)); i++) + inst[i] = INSN_BREAK; + + mutex_lock(&text_mutex); + if (larch_insn_text_copy(dst, inst, len)) + ret = -EINVAL; + mutex_unlock(&text_mutex); + + kvfree(inst); + + return ret; +} + +static void store_args(struct jit_ctx *ctx, int nargs, int args_off) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit_insn(ctx, std, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + args_off -= 8; + } +} + +static void restore_args(struct jit_ctx *ctx, int nargs, int args_off) +{ + int i; + + for (i = 0; i < nargs; i++) { + emit_insn(ctx, ldd, LOONGARCH_GPR_A0 + i, LOONGARCH_GPR_FP, -args_off); + args_off -= 8; + } +} + +static int invoke_bpf_prog(struct jit_ctx *ctx, struct bpf_tramp_link *l, + int args_off, int retval_off, int run_ctx_off, bool save_ret) +{ + int ret; + u32 *branch; + struct bpf_prog *p = l->link.prog; + int cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie); + + if (l->cookie) { + move_imm(ctx, LOONGARCH_GPR_T1, l->cookie, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); + } else { + emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -run_ctx_off + cookie_off); + } + + /* arg1: prog */ + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); + /* arg2: &run_ctx */ + emit_insn(ctx, addid, LOONGARCH_GPR_A1, LOONGARCH_GPR_FP, -run_ctx_off); + ret = emit_call(ctx, (const u64)bpf_trampoline_enter(p)); + if (ret) + return ret; + + /* store prog start time */ + move_reg(ctx, LOONGARCH_GPR_S1, LOONGARCH_GPR_A0); + + /* + * if (__bpf_prog_enter(prog) == 0) + * goto skip_exec_of_prog; + */ + branch = (u32 *)ctx->image + ctx->idx; + /* nop reserved for conditional jump */ + emit_insn(ctx, nop); + + /* arg1: &args_off */ + emit_insn(ctx, addid, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -args_off); + if (!p->jited) + move_imm(ctx, LOONGARCH_GPR_A1, (const s64)p->insnsi, false); + ret = emit_call(ctx, (const u64)p->bpf_func); + if (ret) + return ret; + + if (save_ret) { + emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); + } + + /* update branch with beqz */ + if (ctx->image) { + int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branch; + *branch = larch_insn_gen_beq(LOONGARCH_GPR_A0, LOONGARCH_GPR_ZERO, offset); + } + + /* arg1: prog */ + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)p, false); + /* arg2: prog start time */ + move_reg(ctx, LOONGARCH_GPR_A1, LOONGARCH_GPR_S1); + /* arg3: &run_ctx */ + emit_insn(ctx, addid, LOONGARCH_GPR_A2, LOONGARCH_GPR_FP, -run_ctx_off); + ret = emit_call(ctx, (const u64)bpf_trampoline_exit(p)); + + return ret; +} + +static void invoke_bpf_mod_ret(struct jit_ctx *ctx, struct bpf_tramp_links *tl, + int args_off, int retval_off, int run_ctx_off, u32 **branches) +{ + int i; + + emit_insn(ctx, std, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_FP, -retval_off); + for (i = 0; i < tl->nr_links; i++) { + invoke_bpf_prog(ctx, tl->links[i], args_off, retval_off, run_ctx_off, true); + emit_insn(ctx, ldd, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -retval_off); + branches[i] = (u32 *)ctx->image + ctx->idx; + emit_insn(ctx, nop); + } +} + +void *arch_alloc_bpf_trampoline(unsigned int size) +{ + return bpf_prog_pack_alloc(size, jit_fill_hole); +} + +void arch_free_bpf_trampoline(void *image, unsigned int size) +{ + bpf_prog_pack_free(image, size); +} + +static int __arch_prepare_bpf_trampoline(struct jit_ctx *ctx, struct bpf_tramp_image *im, + const struct btf_func_model *m, struct bpf_tramp_links *tlinks, + void *func_addr, u32 flags) +{ + int i, ret, save_ret; + int stack_size = 0, nargs = 0; + int retval_off, args_off, nargs_off, ip_off, run_ctx_off, sreg_off, tcc_ptr_off; + bool is_struct_ops = flags & BPF_TRAMP_F_INDIRECT; + void *orig_call = func_addr; + struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY]; + struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT]; + struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN]; + u32 **branches = NULL; + + if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) + return -ENOTSUPP; + + /* + * FP + 8 [ RA to parent func ] return address to parent + * function + * FP + 0 [ FP of parent func ] frame pointer of parent + * function + * FP - 8 [ T0 to traced func ] return address of traced + * function + * FP - 16 [ FP of traced func ] frame pointer of traced + * function + * + * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or + * BPF_TRAMP_F_RET_FENTRY_RET + * [ argN ] + * [ ... ] + * FP - args_off [ arg1 ] + * + * FP - nargs_off [ regs count ] + * + * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG + * + * FP - run_ctx_off [ bpf_tramp_run_ctx ] + * + * FP - sreg_off [ callee saved reg ] + * + * FP - tcc_ptr_off [ tail_call_cnt_ptr ] + */ + + if (m->nr_args > LOONGARCH_MAX_REG_ARGS) + return -ENOTSUPP; + + if (flags & (BPF_TRAMP_F_ORIG_STACK | BPF_TRAMP_F_SHARE_IPMODIFY)) + return -ENOTSUPP; + + stack_size = 0; + + /* Room of trampoline frame to store return address and frame pointer */ + stack_size += 16; + + save_ret = flags & (BPF_TRAMP_F_CALL_ORIG | BPF_TRAMP_F_RET_FENTRY_RET); + if (save_ret) { + /* Save BPF R0 and A0 */ + stack_size += 16; + retval_off = stack_size; + } + + /* Room of trampoline frame to store args */ + nargs = m->nr_args; + stack_size += nargs * 8; + args_off = stack_size; + + /* Room of trampoline frame to store args number */ + stack_size += 8; + nargs_off = stack_size; + + /* Room of trampoline frame to store ip address */ + if (flags & BPF_TRAMP_F_IP_ARG) { + stack_size += 8; + ip_off = stack_size; + } + + /* Room of trampoline frame to store struct bpf_tramp_run_ctx */ + stack_size += round_up(sizeof(struct bpf_tramp_run_ctx), 8); + run_ctx_off = stack_size; + + stack_size += 8; + sreg_off = stack_size; + + /* Room of trampoline frame to store tail_call_cnt_ptr */ + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) { + stack_size += 8; + tcc_ptr_off = stack_size; + } + + stack_size = round_up(stack_size, 16); + + if (is_struct_ops) { + /* + * For the trampoline called directly, just handle + * the frame of trampoline. + */ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + } else { + /* + * For the trampoline called from function entry, + * the frame of traced function and the frame of + * trampoline need to be considered. + */ + /* RA and FP for parent function */ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -16); + emit_insn(ctx, std, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 16); + + /* RA and FP for traced function */ + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, -stack_size); + emit_insn(ctx, std, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, std, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size); + } + + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_insn(ctx, std, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); + + /* callee saved register S1 to pass start time */ + emit_insn(ctx, std, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); + + /* store ip address of the traced function */ + if (flags & BPF_TRAMP_F_IP_ARG) { + move_imm(ctx, LOONGARCH_GPR_T1, (const s64)func_addr, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -ip_off); + } + + /* store nargs number */ + move_imm(ctx, LOONGARCH_GPR_T1, nargs, false); + emit_insn(ctx, std, LOONGARCH_GPR_T1, LOONGARCH_GPR_FP, -nargs_off); + + store_args(ctx, nargs, args_off); + + /* To traced function */ + /* Ftrace jump skips 2 NOP instructions */ + if (is_kernel_text((unsigned long)orig_call)) + orig_call += LOONGARCH_FENTRY_NBYTES; + /* Direct jump skips 5 NOP instructions */ + else if (is_bpf_text_address((unsigned long)orig_call)) + orig_call += LOONGARCH_BPF_FENTRY_NBYTES; + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); + ret = emit_call(ctx, (const u64)__bpf_tramp_enter); + if (ret) + return ret; + } + + for (i = 0; i < fentry->nr_links; i++) { + ret = invoke_bpf_prog(ctx, fentry->links[i], args_off, retval_off, + run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET); + if (ret) + return ret; + } + if (fmod_ret->nr_links) { + branches = kcalloc(fmod_ret->nr_links, sizeof(u32 *), GFP_KERNEL); + if (!branches) + return -ENOMEM; + + invoke_bpf_mod_ret(ctx, fmod_ret, args_off, retval_off, run_ctx_off, branches); + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + restore_args(ctx, m->nr_args, args_off); + + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); + + ret = emit_call(ctx, (const u64)orig_call); + if (ret) + goto out; + emit_insn(ctx, std, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, std, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); + im->ip_after_call = ctx->ro_image + ctx->idx; + /* Reserve space for the move_imm + jirl instruction */ + for (i = 0; i < LOONGARCH_LONG_JUMP_NINSNS; i++) + emit_insn(ctx, nop); + } + + for (i = 0; ctx->image && i < fmod_ret->nr_links; i++) { + int offset = (void *)(&ctx->image[ctx->idx]) - (void *)branches[i]; + *branches[i] = larch_insn_gen_bne(LOONGARCH_GPR_T1, LOONGARCH_GPR_ZERO, offset); + } + + for (i = 0; i < fexit->nr_links; i++) { + ret = invoke_bpf_prog(ctx, fexit->links[i], args_off, retval_off, run_ctx_off, false); + if (ret) + goto out; + } + + if (flags & BPF_TRAMP_F_CALL_ORIG) { + im->ip_epilogue = ctx->ro_image + ctx->idx; + move_imm(ctx, LOONGARCH_GPR_A0, (const s64)im, false); + ret = emit_call(ctx, (const u64)__bpf_tramp_exit); + if (ret) + goto out; + } + + if (flags & BPF_TRAMP_F_RESTORE_REGS) + restore_args(ctx, m->nr_args, args_off); + + if (save_ret) { + emit_insn(ctx, ldd, LOONGARCH_GPR_A0, LOONGARCH_GPR_FP, -retval_off); + emit_insn(ctx, ldd, regmap[BPF_REG_0], LOONGARCH_GPR_FP, -(retval_off - 8)); + } + + emit_insn(ctx, ldd, LOONGARCH_GPR_S1, LOONGARCH_GPR_FP, -sreg_off); + + if (flags & BPF_TRAMP_F_TAIL_CALL_CTX) + emit_insn(ctx, ldd, REG_TCC, LOONGARCH_GPR_FP, -tcc_ptr_off); + + if (is_struct_ops) { + /* trampoline called directly */ + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); + + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); + } else { + /* trampoline called from function entry */ + emit_insn(ctx, ldd, LOONGARCH_GPR_T0, LOONGARCH_GPR_SP, stack_size - 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, stack_size - 16); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, stack_size); + + emit_insn(ctx, ldd, LOONGARCH_GPR_RA, LOONGARCH_GPR_SP, 8); + emit_insn(ctx, ldd, LOONGARCH_GPR_FP, LOONGARCH_GPR_SP, 0); + emit_insn(ctx, addid, LOONGARCH_GPR_SP, LOONGARCH_GPR_SP, 16); + + if (flags & BPF_TRAMP_F_SKIP_FRAME) + /* return to parent function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_RA, 0); + else + /* return to traced function */ + emit_insn(ctx, jirl, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_T0, 0); + } + + ret = ctx->idx; +out: + kfree(branches); + + return ret; +} + +int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *ro_image, + void *ro_image_end, const struct btf_func_model *m, + u32 flags, struct bpf_tramp_links *tlinks, void *func_addr) +{ + int ret, size; + void *image, *tmp; + struct jit_ctx ctx; + + size = ro_image_end - ro_image; + image = kvmalloc(size, GFP_KERNEL); + if (!image) + return -ENOMEM; + + ctx.image = (union loongarch_instruction *)image; + ctx.ro_image = (union loongarch_instruction *)ro_image; + ctx.idx = 0; + + jit_fill_hole(image, (unsigned int)(ro_image_end - ro_image)); + ret = __arch_prepare_bpf_trampoline(&ctx, im, m, tlinks, func_addr, flags); + if (ret > 0 && validate_code(&ctx) < 0) { + ret = -EINVAL; + goto out; + } + + tmp = bpf_arch_text_copy(ro_image, image, size); + if (IS_ERR(tmp)) { + ret = PTR_ERR(tmp); + goto out; + } + + bpf_flush_icache(ro_image, ro_image_end); +out: + kvfree(image); + return ret < 0 ? ret : size; +} + +int arch_bpf_trampoline_size(const struct btf_func_model *m, u32 flags, + struct bpf_tramp_links *tlinks, void *func_addr) +{ + int ret; + struct jit_ctx ctx; + struct bpf_tramp_image im; + + ctx.image = NULL; + ctx.idx = 0; + + ret = __arch_prepare_bpf_trampoline(&ctx, &im, m, tlinks, func_addr, flags); + + /* Page align */ + return ret < 0 ? ret : round_up(ret * LOONGARCH_INSN_SIZE, PAGE_SIZE); +} + struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { bool tmp_blinded = false, extra_pass = false; @@ -1288,7 +1850,7 @@ skip_init_ctx: build_epilogue(&ctx); /* 3. Extra pass to validate JITed code */ - if (validate_code(&ctx)) { + if (validate_ctx(&ctx)) { bpf_jit_binary_free(header); prog = orig_prog; goto out_offset; @@ -1342,7 +1904,6 @@ out: if (tmp_blinded) bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); - out_offset = -1; return prog; @@ -1354,6 +1915,16 @@ out_free: goto out_offset; } +bool bpf_jit_bypass_spec_v1(void) +{ + return true; +} + +bool bpf_jit_bypass_spec_v4(void) +{ + return true; +} + /* Indicate the JIT backend supports mixing bpf2bpf and tailcalls. */ bool bpf_jit_supports_subprog_tailcalls(void) { diff --git a/arch/loongarch/net/bpf_jit.h b/arch/loongarch/net/bpf_jit.h index f9c569f53949..5697158fd164 100644 --- a/arch/loongarch/net/bpf_jit.h +++ b/arch/loongarch/net/bpf_jit.h @@ -18,6 +18,7 @@ struct jit_ctx { u32 *offset; int num_exentries; union loongarch_instruction *image; + union loongarch_instruction *ro_image; u32 stack_size; }; @@ -308,3 +309,8 @@ static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch return -EINVAL; } + +static inline void bpf_flush_icache(void *start, void *end) +{ + flush_icache_range((unsigned long)start, (unsigned long)end); +} diff --git a/arch/loongarch/vdso/Makefile b/arch/loongarch/vdso/Makefile index ccd2c5e135c6..d8316f993482 100644 --- a/arch/loongarch/vdso/Makefile +++ b/arch/loongarch/vdso/Makefile @@ -36,7 +36,7 @@ endif # VDSO linker flags. ldflags-y := -Bsymbolic --no-undefined -soname=linux-vdso.so.1 \ - $(filter -E%,$(KBUILD_CFLAGS)) -nostdlib -shared --build-id -T + $(filter -E%,$(KBUILD_CFLAGS)) -shared --build-id -T # # Shared build commands. diff --git a/arch/s390/Kconfig b/arch/s390/Kconfig index f0c0469e553d..bf680c26a33c 100644 --- a/arch/s390/Kconfig +++ b/arch/s390/Kconfig @@ -74,6 +74,7 @@ config S390 select ARCH_ENABLE_MEMORY_HOTPLUG if SPARSEMEM select ARCH_ENABLE_MEMORY_HOTREMOVE select ARCH_ENABLE_SPLIT_PMD_PTLOCK if PGTABLE_LEVELS > 2 + select ARCH_ENABLE_THP_MIGRATION if TRANSPARENT_HUGEPAGE select ARCH_HAS_CPU_FINALIZE_INIT select ARCH_HAS_CURRENT_STACK_POINTER select ARCH_HAS_DEBUG_VIRTUAL @@ -102,6 +103,7 @@ config S390 select ARCH_HAS_UBSAN select ARCH_HAS_VDSO_TIME_DATA select ARCH_HAVE_NMI_SAFE_CMPXCHG + select ARCH_HAVE_TRACE_MMIO_ACCESS select ARCH_INLINE_READ_LOCK select ARCH_INLINE_READ_LOCK_BH select ARCH_INLINE_READ_LOCK_IRQ @@ -150,6 +152,7 @@ config S390 select ARCH_WANT_KERNEL_PMD_MKWRITE select ARCH_WANT_LD_ORPHAN_WARN select ARCH_WANT_OPTIMIZE_HUGETLB_VMEMMAP + select ARCH_WANTS_THP_SWAP select BUILDTIME_TABLE_SORT select CLONE_BACKWARDS2 select DCACHE_WORD_ACCESS if !KMSAN diff --git a/arch/s390/boot/startup.c b/arch/s390/boot/startup.c index 305e6c791071..93684a775716 100644 --- a/arch/s390/boot/startup.c +++ b/arch/s390/boot/startup.c @@ -384,7 +384,7 @@ static unsigned long setup_kernel_memory_layout(unsigned long kernel_size) kernel_start = round_down(kernel_end - kernel_size, THREAD_SIZE); boot_debug("Randomization range: 0x%016lx-0x%016lx\n", vmax - kaslr_len, vmax); boot_debug("kernel image: 0x%016lx-0x%016lx (kaslr)\n", kernel_start, - kernel_size + kernel_size); + kernel_start + kernel_size); } else if (vmax < __NO_KASLR_END_KERNEL || vsize > __NO_KASLR_END_KERNEL) { kernel_start = round_down(vmax - kernel_size, THREAD_SIZE); boot_debug("kernel image: 0x%016lx-0x%016lx (constrained)\n", kernel_start, diff --git a/arch/s390/include/asm/ap.h b/arch/s390/include/asm/ap.h index 395b02d6a133..352108727d7e 100644 --- a/arch/s390/include/asm/ap.h +++ b/arch/s390/include/asm/ap.h @@ -103,7 +103,7 @@ struct ap_tapq_hwinfo { unsigned int accel : 1; /* A */ unsigned int ep11 : 1; /* X */ unsigned int apxa : 1; /* APXA */ - unsigned int : 1; + unsigned int slcf : 1; /* Cmd filtering avail. */ unsigned int class : 8; unsigned int bs : 2; /* SE bind/assoc */ unsigned int : 14; diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 6d8bc27a366e..c1a7a92f0575 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -963,6 +963,12 @@ static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) return clear_pmd_bit(pmd, __pgprot(_SEGMENT_ENTRY_SOFT_DIRTY)); } +#ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION +#define pmd_swp_soft_dirty(pmd) pmd_soft_dirty(pmd) +#define pmd_swp_mksoft_dirty(pmd) pmd_mksoft_dirty(pmd) +#define pmd_swp_clear_soft_dirty(pmd) pmd_clear_soft_dirty(pmd) +#endif + /* * query functions pte_write/pte_dirty/pte_young only work if * pte_present() is true. Undefined behaviour if not.. @@ -1979,6 +1985,45 @@ static inline unsigned long __swp_offset_rste(swp_entry_t entry) #define __rste_to_swp_entry(rste) ((swp_entry_t) { rste }) +/* + * s390 has different layout for PTE and region / segment table entries (RSTE). + * This is also true for swap entries, and their swap type and offset encoding. + * For hugetlbfs PTE_MARKER support, s390 has internal __swp_type_rste() and + * __swp_offset_rste() helpers to correctly handle RSTE swap entries. + * + * But common swap code does not know about this difference, and only uses + * __swp_type(), __swp_offset() and __swp_entry() helpers for conversion between + * arch-dependent and arch-independent representation of swp_entry_t for all + * pagetable levels. On s390, those helpers only work for PTE swap entries. + * + * Therefore, implement __pmd_to_swp_entry() to build a fake PTE swap entry + * and return the arch-dependent representation of that. Correspondingly, + * implement __swp_entry_to_pmd() to convert that into a proper PMD swap + * entry again. With this, the arch-dependent swp_entry_t representation will + * always look like a PTE swap entry in common code. + * + * This is somewhat similar to fake PTEs in hugetlbfs code for s390, but only + * requires conversion of the swap type and offset, and not all the possible + * PTE bits. + */ +static inline swp_entry_t __pmd_to_swp_entry(pmd_t pmd) +{ + swp_entry_t arch_entry; + pte_t pte; + + arch_entry = __rste_to_swp_entry(pmd_val(pmd)); + pte = mk_swap_pte(__swp_type_rste(arch_entry), __swp_offset_rste(arch_entry)); + return __pte_to_swp_entry(pte); +} + +static inline pmd_t __swp_entry_to_pmd(swp_entry_t arch_entry) +{ + pmd_t pmd; + + pmd = __pmd(mk_swap_rste(__swp_type(arch_entry), __swp_offset(arch_entry))); + return pmd; +} + extern int vmem_add_mapping(unsigned long start, unsigned long size); extern void vmem_remove_mapping(unsigned long start, unsigned long size); extern int __vmem_map_4k_page(unsigned long addr, unsigned long phys, pgprot_t prot, bool alloc); diff --git a/arch/s390/kernel/debug.c b/arch/s390/kernel/debug.c index 2a41be2f7925..c62100dc62c8 100644 --- a/arch/s390/kernel/debug.c +++ b/arch/s390/kernel/debug.c @@ -1677,7 +1677,7 @@ EXPORT_SYMBOL(debug_dflt_header_fn); /* * prints debug data sprintf-formatted: - * debug_sprinf_event/exception calls must be used together with this view + * debug_sprintf_event/exception calls must be used together with this view */ #define DEBUG_SPRINTF_MAX_ARGS 10 diff --git a/arch/s390/kernel/setup.c b/arch/s390/kernel/setup.c index b99aeb0db2ee..7b529868789f 100644 --- a/arch/s390/kernel/setup.c +++ b/arch/s390/kernel/setup.c @@ -719,6 +719,11 @@ static void __init memblock_add_physmem_info(void) memblock_set_node(0, ULONG_MAX, &memblock.memory, 0); } +static void __init setup_high_memory(void) +{ + high_memory = __va(ident_map_size); +} + /* * Reserve memory used for lowcore. */ @@ -951,6 +956,7 @@ void __init setup_arch(char **cmdline_p) free_physmem_info(); setup_memory_end(); + setup_high_memory(); memblock_dump_all(); setup_memory(); diff --git a/arch/s390/mm/vmem.c b/arch/s390/mm/vmem.c index 448dd6ed1069..f48ef361bc83 100644 --- a/arch/s390/mm/vmem.c +++ b/arch/s390/mm/vmem.c @@ -64,13 +64,12 @@ void *vmem_crst_alloc(unsigned long val) pte_t __ref *vmem_pte_alloc(void) { - unsigned long size = PTRS_PER_PTE * sizeof(pte_t); pte_t *pte; if (slab_is_available()) - pte = (pte_t *) page_table_alloc(&init_mm); + pte = (pte_t *)page_table_alloc(&init_mm); else - pte = (pte_t *) memblock_alloc(size, size); + pte = (pte_t *)memblock_alloc(PAGE_SIZE, PAGE_SIZE); if (!pte) return NULL; memset64((u64 *)pte, _PAGE_INVALID, PTRS_PER_PTE); diff --git a/arch/xtensa/include/asm/bootparam.h b/arch/xtensa/include/asm/bootparam.h index 6333bd1eb9d2..a459ffbaf7ab 100644 --- a/arch/xtensa/include/asm/bootparam.h +++ b/arch/xtensa/include/asm/bootparam.h @@ -27,7 +27,7 @@ #define BP_TAG_FIRST 0x7B0B /* first tag with a version number */ #define BP_TAG_LAST 0x7E0B /* last tag */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* All records are aligned to 4 bytes */ diff --git a/arch/xtensa/include/asm/cmpxchg.h b/arch/xtensa/include/asm/cmpxchg.h index 95e33a913962..b6db4838b175 100644 --- a/arch/xtensa/include/asm/cmpxchg.h +++ b/arch/xtensa/include/asm/cmpxchg.h @@ -11,7 +11,7 @@ #ifndef _XTENSA_CMPXCHG_H #define _XTENSA_CMPXCHG_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/bits.h> #include <linux/stringify.h> @@ -220,6 +220,6 @@ __arch_xchg(unsigned long x, volatile void * ptr, int size) } } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _XTENSA_CMPXCHG_H */ diff --git a/arch/xtensa/include/asm/coprocessor.h b/arch/xtensa/include/asm/coprocessor.h index 3b1a0d5d2169..e0447bcc52c5 100644 --- a/arch/xtensa/include/asm/coprocessor.h +++ b/arch/xtensa/include/asm/coprocessor.h @@ -16,7 +16,7 @@ #include <asm/core.h> #include <asm/types.h> -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ # include <variant/tie-asm.h> .macro xchal_sa_start a b @@ -69,7 +69,7 @@ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ /* * XTENSA_HAVE_COPROCESSOR(x) returns 1 if coprocessor x is configured. @@ -87,7 +87,7 @@ #define XTENSA_HAVE_IO_PORTS \ XCHAL_CP_PORT_MASK -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* * Additional registers. @@ -151,5 +151,5 @@ void local_coprocessors_flush_release_all(void); #endif /* XTENSA_HAVE_COPROCESSORS */ -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _XTENSA_COPROCESSOR_H */ diff --git a/arch/xtensa/include/asm/current.h b/arch/xtensa/include/asm/current.h index df275d554788..7b483538f066 100644 --- a/arch/xtensa/include/asm/current.h +++ b/arch/xtensa/include/asm/current.h @@ -13,7 +13,7 @@ #include <asm/thread_info.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/thread_info.h> diff --git a/arch/xtensa/include/asm/ftrace.h b/arch/xtensa/include/asm/ftrace.h index 0ea4f84cd558..f676d209d110 100644 --- a/arch/xtensa/include/asm/ftrace.h +++ b/arch/xtensa/include/asm/ftrace.h @@ -12,20 +12,20 @@ #include <asm/processor.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned long return_address(unsigned level); #define ftrace_return_address(n) return_address(n) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #ifdef CONFIG_FUNCTION_TRACER #define MCOUNT_ADDR ((unsigned long)(_mcount)) #define MCOUNT_INSN_SIZE 3 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern void _mcount(void); #define mcount _mcount -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* CONFIG_FUNCTION_TRACER */ #endif /* _XTENSA_FTRACE_H */ diff --git a/arch/xtensa/include/asm/initialize_mmu.h b/arch/xtensa/include/asm/initialize_mmu.h index 574795a20d6f..101bcb87e15b 100644 --- a/arch/xtensa/include/asm/initialize_mmu.h +++ b/arch/xtensa/include/asm/initialize_mmu.h @@ -34,7 +34,7 @@ #define CA_WRITEBACK (0x4) #endif -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define XTENSA_HWVERSION_RC_2009_0 230000 @@ -240,6 +240,6 @@ .endm -#endif /*__ASSEMBLY__*/ +#endif /*__ASSEMBLER__*/ #endif /* _XTENSA_INITIALIZE_MMU_H */ diff --git a/arch/xtensa/include/asm/jump_label.h b/arch/xtensa/include/asm/jump_label.h index 46c8596259d2..38e3e2a9b0fb 100644 --- a/arch/xtensa/include/asm/jump_label.h +++ b/arch/xtensa/include/asm/jump_label.h @@ -4,7 +4,7 @@ #ifndef _ASM_XTENSA_JUMP_LABEL_H #define _ASM_XTENSA_JUMP_LABEL_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -61,5 +61,5 @@ struct jump_entry { jump_label_t key; }; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif diff --git a/arch/xtensa/include/asm/kasan.h b/arch/xtensa/include/asm/kasan.h index 8d2b4248466f..0da91b64fab9 100644 --- a/arch/xtensa/include/asm/kasan.h +++ b/arch/xtensa/include/asm/kasan.h @@ -2,7 +2,7 @@ #ifndef __ASM_KASAN_H #define __ASM_KASAN_H -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #ifdef CONFIG_KASAN diff --git a/arch/xtensa/include/asm/kmem_layout.h b/arch/xtensa/include/asm/kmem_layout.h index 6fc05cba61a2..6949724625a0 100644 --- a/arch/xtensa/include/asm/kmem_layout.h +++ b/arch/xtensa/include/asm/kmem_layout.h @@ -80,7 +80,7 @@ #if (!XCHAL_HAVE_PTP_MMU || XCHAL_HAVE_SPANNING_WAY) && defined(CONFIG_USE_OF) #define XCHAL_KIO_PADDR xtensa_get_kio_paddr() -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ extern unsigned long xtensa_kio_paddr; static inline unsigned long xtensa_get_kio_paddr(void) diff --git a/arch/xtensa/include/asm/page.h b/arch/xtensa/include/asm/page.h index 644413792bf3..20655174b111 100644 --- a/arch/xtensa/include/asm/page.h +++ b/arch/xtensa/include/asm/page.h @@ -80,7 +80,7 @@ #endif -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ #define __pgprot(x) (x) @@ -172,7 +172,7 @@ static inline unsigned long ___pa(unsigned long va) #define page_to_virt(page) __va(page_to_pfn(page) << PAGE_SHIFT) #define virt_addr_valid(kaddr) pfn_valid(__pa(kaddr) >> PAGE_SHIFT) -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #include <asm-generic/memory_model.h> #endif /* _XTENSA_PAGE_H */ diff --git a/arch/xtensa/include/asm/pgtable.h b/arch/xtensa/include/asm/pgtable.h index d62aa1c316fc..d6eb695f2b26 100644 --- a/arch/xtensa/include/asm/pgtable.h +++ b/arch/xtensa/include/asm/pgtable.h @@ -203,7 +203,7 @@ * What follows is the closest we can get by reasonable means.. * See linux/mm/mmap.c for protection_map[] array that uses these definitions. */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define pte_ERROR(e) \ printk("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) @@ -366,10 +366,10 @@ static inline pte_t pte_swp_clear_exclusive(pte_t pte) return pte; } -#endif /* !defined (__ASSEMBLY__) */ +#endif /* !defined (__ASSEMBLER__) */ -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ /* Assembly macro _PGD_INDEX is the same as C pgd_index(unsigned long), * _PGD_OFFSET as C pgd_offset(struct mm_struct*, unsigned long), @@ -408,7 +408,7 @@ void update_mmu_tlb_range(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, unsigned int nr); #define update_mmu_tlb_range update_mmu_tlb_range -#endif /* !defined (__ASSEMBLY__) */ +#endif /* !defined (__ASSEMBLER__) */ #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG #define __HAVE_ARCH_PTEP_GET_AND_CLEAR diff --git a/arch/xtensa/include/asm/processor.h b/arch/xtensa/include/asm/processor.h index 47b5df86ab5c..60a211356335 100644 --- a/arch/xtensa/include/asm/processor.h +++ b/arch/xtensa/include/asm/processor.h @@ -105,7 +105,7 @@ #error Unsupported xtensa ABI #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #if defined(__XTENSA_WINDOWED_ABI__) @@ -263,5 +263,5 @@ static inline unsigned long get_er(unsigned long addr) #endif /* XCHAL_HAVE_EXTERN_REGS */ -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _XTENSA_PROCESSOR_H */ diff --git a/arch/xtensa/include/asm/ptrace.h b/arch/xtensa/include/asm/ptrace.h index 4871e5a4d6fb..d0568ff6d349 100644 --- a/arch/xtensa/include/asm/ptrace.h +++ b/arch/xtensa/include/asm/ptrace.h @@ -41,7 +41,7 @@ #define NO_SYSCALL (-1) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm/coprocessor.h> #include <asm/core.h> @@ -106,11 +106,11 @@ static inline unsigned long regs_return_value(struct pt_regs *regs) int do_syscall_trace_enter(struct pt_regs *regs); void do_syscall_trace_leave(struct pt_regs *regs); -#else /* __ASSEMBLY__ */ +#else /* __ASSEMBLER__ */ # include <asm/asm-offsets.h> #define PT_REGS_OFFSET (KERNEL_STACK_SIZE - PT_USER_SIZE) -#endif /* !__ASSEMBLY__ */ +#endif /* !__ASSEMBLER__ */ #endif /* _XTENSA_PTRACE_H */ diff --git a/arch/xtensa/include/asm/signal.h b/arch/xtensa/include/asm/signal.h index de169b4eaeef..d301e68573cc 100644 --- a/arch/xtensa/include/asm/signal.h +++ b/arch/xtensa/include/asm/signal.h @@ -14,10 +14,10 @@ #include <uapi/asm/signal.h> -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define __ARCH_HAS_SA_RESTORER #include <asm/sigcontext.h> -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _XTENSA_SIGNAL_H */ diff --git a/arch/xtensa/include/asm/thread_info.h b/arch/xtensa/include/asm/thread_info.h index e0dffcc43b9e..5b74dfc35ef9 100644 --- a/arch/xtensa/include/asm/thread_info.h +++ b/arch/xtensa/include/asm/thread_info.h @@ -16,7 +16,7 @@ #define CURRENT_SHIFT KERNEL_STACK_SHIFT -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ # include <asm/processor.h> #endif @@ -28,7 +28,7 @@ * must also be changed */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #if XTENSA_HAVE_COPROCESSORS @@ -80,7 +80,7 @@ struct thread_info { * macros/functions for gaining access to the thread information structure */ -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #define INIT_THREAD_INFO(tsk) \ { \ @@ -99,7 +99,7 @@ static __always_inline struct thread_info *current_thread_info(void) return ti; } -#else /* !__ASSEMBLY__ */ +#else /* !__ASSEMBLER__ */ /* how to get the thread information struct from ASM */ #define GET_THREAD_INFO(reg,sp) \ diff --git a/arch/xtensa/include/asm/tlbflush.h b/arch/xtensa/include/asm/tlbflush.h index 573df8cea200..3edaebeef423 100644 --- a/arch/xtensa/include/asm/tlbflush.h +++ b/arch/xtensa/include/asm/tlbflush.h @@ -20,7 +20,7 @@ #define ITLB_HIT_BIT 3 #define DTLB_HIT_BIT 4 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ /* TLB flushing: * @@ -201,5 +201,5 @@ static inline unsigned long read_itlb_translation (int way) return tmp; } -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _XTENSA_TLBFLUSH_H */ diff --git a/arch/xtensa/include/uapi/asm/ptrace.h b/arch/xtensa/include/uapi/asm/ptrace.h index 9115e86ebc75..6e89ea301438 100644 --- a/arch/xtensa/include/uapi/asm/ptrace.h +++ b/arch/xtensa/include/uapi/asm/ptrace.h @@ -42,7 +42,7 @@ #define PTRACE_GETFDPIC_EXEC 0 #define PTRACE_GETFDPIC_INTERP 1 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ struct user_pt_regs { __u32 pc; diff --git a/arch/xtensa/include/uapi/asm/signal.h b/arch/xtensa/include/uapi/asm/signal.h index b8c824dd4b74..8060f1914400 100644 --- a/arch/xtensa/include/uapi/asm/signal.h +++ b/arch/xtensa/include/uapi/asm/signal.h @@ -19,7 +19,7 @@ #define _NSIG_BPW 32 #define _NSIG_WORDS (_NSIG / _NSIG_BPW) -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <linux/types.h> @@ -77,7 +77,7 @@ typedef struct { #define MINSIGSTKSZ 2048 #define SIGSTKSZ 8192 -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #include <asm-generic/signal-defs.h> @@ -106,5 +106,5 @@ typedef struct sigaltstack { __kernel_size_t ss_size; } stack_t; -#endif /* __ASSEMBLY__ */ +#endif /* __ASSEMBLER__ */ #endif /* _UAPI_XTENSA_SIGNAL_H */ diff --git a/arch/xtensa/include/uapi/asm/types.h b/arch/xtensa/include/uapi/asm/types.h index 12db8ac38750..2e9217a06ebf 100644 --- a/arch/xtensa/include/uapi/asm/types.h +++ b/arch/xtensa/include/uapi/asm/types.h @@ -14,7 +14,7 @@ #include <asm-generic/int-ll64.h> -#ifdef __ASSEMBLY__ +#ifdef __ASSEMBLER__ # define __XTENSA_UL(x) (x) # define __XTENSA_UL_CONST(x) x #else @@ -23,7 +23,7 @@ # define __XTENSA_UL_CONST(x) ___XTENSA_UL_CONST(x) #endif -#ifndef __ASSEMBLY__ +#ifndef __ASSEMBLER__ #endif diff --git a/block/bfq-iosched.c b/block/bfq-iosched.c index 0cb1e9873aab..3bf76902f07f 100644 --- a/block/bfq-iosched.c +++ b/block/bfq-iosched.c @@ -454,17 +454,10 @@ static struct bfq_io_cq *icq_to_bic(struct io_cq *icq) */ static struct bfq_io_cq *bfq_bic_lookup(struct request_queue *q) { - struct bfq_io_cq *icq; - unsigned long flags; - if (!current->io_context) return NULL; - spin_lock_irqsave(&q->queue_lock, flags); - icq = icq_to_bic(ioc_lookup_icq(q)); - spin_unlock_irqrestore(&q->queue_lock, flags); - - return icq; + return icq_to_bic(ioc_lookup_icq(q)); } /* @@ -701,17 +694,13 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) { struct bfq_data *bfqd = data->q->elevator->elevator_data; struct bfq_io_cq *bic = bfq_bic_lookup(data->q); - int depth; - unsigned limit = data->q->nr_requests; - unsigned int act_idx; + unsigned int limit, act_idx; /* Sync reads have full depth available */ - if (op_is_sync(opf) && !op_is_write(opf)) { - depth = 0; - } else { - depth = bfqd->word_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; - limit = (limit * depth) >> bfqd->full_depth_shift; - } + if (op_is_sync(opf) && !op_is_write(opf)) + limit = data->q->nr_requests; + else + limit = bfqd->async_depths[!!bfqd->wr_busy_queues][op_is_sync(opf)]; for (act_idx = 0; bic && act_idx < bfqd->num_actuators; act_idx++) { /* Fast path to check if bfqq is already allocated. */ @@ -725,14 +714,16 @@ static void bfq_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * available requests and thus starve other entities. */ if (bfqq_request_over_limit(bfqd, bic, opf, act_idx, limit)) { - depth = 1; + limit = 1; break; } } + bfq_log(bfqd, "[%s] wr_busy %d sync %d depth %u", - __func__, bfqd->wr_busy_queues, op_is_sync(opf), depth); - if (depth) - data->shallow_depth = depth; + __func__, bfqd->wr_busy_queues, op_is_sync(opf), limit); + + if (limit < data->q->nr_requests) + data->shallow_depth = limit; } static struct bfq_queue * @@ -2457,15 +2448,8 @@ static bool bfq_bio_merge(struct request_queue *q, struct bio *bio, unsigned int nr_segs) { struct bfq_data *bfqd = q->elevator->elevator_data; - struct request *free = NULL; - /* - * bfq_bic_lookup grabs the queue_lock: invoke it now and - * store its return value for later use, to avoid nesting - * queue_lock inside the bfqd->lock. We assume that the bic - * returned by bfq_bic_lookup does not go away before - * bfqd->lock is taken. - */ struct bfq_io_cq *bic = bfq_bic_lookup(q); + struct request *free = NULL; bool ret; spin_lock_irq(&bfqd->lock); @@ -7128,9 +7112,8 @@ void bfq_put_async_queues(struct bfq_data *bfqd, struct bfq_group *bfqg) */ static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt) { - unsigned int depth = 1U << bt->sb.shift; + unsigned int nr_requests = bfqd->queue->nr_requests; - bfqd->full_depth_shift = bt->sb.shift; /* * In-word depths if no bfq_queue is being weight-raised: * leaving 25% of tags only for sync reads. @@ -7142,13 +7125,13 @@ static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt) * limit 'something'. */ /* no more than 50% of tags for async I/O */ - bfqd->word_depths[0][0] = max(depth >> 1, 1U); + bfqd->async_depths[0][0] = max(nr_requests >> 1, 1U); /* * no more than 75% of tags for sync writes (25% extra tags * w.r.t. async I/O, to prevent async I/O from starving sync * writes) */ - bfqd->word_depths[0][1] = max((depth * 3) >> 2, 1U); + bfqd->async_depths[0][1] = max((nr_requests * 3) >> 2, 1U); /* * In-word depths in case some bfq_queue is being weight- @@ -7158,9 +7141,9 @@ static void bfq_update_depths(struct bfq_data *bfqd, struct sbitmap_queue *bt) * shortage. */ /* no more than ~18% of tags for async I/O */ - bfqd->word_depths[1][0] = max((depth * 3) >> 4, 1U); + bfqd->async_depths[1][0] = max((nr_requests * 3) >> 4, 1U); /* no more than ~37% of tags for sync writes (~20% extra tags) */ - bfqd->word_depths[1][1] = max((depth * 6) >> 4, 1U); + bfqd->async_depths[1][1] = max((nr_requests * 6) >> 4, 1U); } static void bfq_depth_updated(struct blk_mq_hw_ctx *hctx) @@ -7232,22 +7215,16 @@ static void bfq_init_root_group(struct bfq_group *root_group, root_group->sched_data.bfq_class_idle_last_service = jiffies; } -static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) +static int bfq_init_queue(struct request_queue *q, struct elevator_queue *eq) { struct bfq_data *bfqd; - struct elevator_queue *eq; unsigned int i; struct blk_independent_access_ranges *ia_ranges = q->disk->ia_ranges; - eq = elevator_alloc(q, e); - if (!eq) - return -ENOMEM; - bfqd = kzalloc_node(sizeof(*bfqd), GFP_KERNEL, q->node); - if (!bfqd) { - kobject_put(&eq->kobj); + if (!bfqd) return -ENOMEM; - } + eq->elevator_data = bfqd; spin_lock_irq(&q->queue_lock); @@ -7405,7 +7382,6 @@ static int bfq_init_queue(struct request_queue *q, struct elevator_type *e) out_free: kfree(bfqd); - kobject_put(&eq->kobj); return -ENOMEM; } diff --git a/block/bfq-iosched.h b/block/bfq-iosched.h index 687a3a7ba784..34a498e6b2a5 100644 --- a/block/bfq-iosched.h +++ b/block/bfq-iosched.h @@ -427,9 +427,6 @@ struct bfq_iocq_bfqq_data { */ bool saved_IO_bound; - u64 saved_io_start_time; - u64 saved_tot_idle_time; - /* * Same purpose as the previous fields for the values of the * field keeping the queue's belonging to a large burst @@ -450,6 +447,9 @@ struct bfq_iocq_bfqq_data { */ unsigned int saved_weight; + u64 saved_io_start_time; + u64 saved_tot_idle_time; + /* * Similar to previous fields: save wr information. */ @@ -457,13 +457,13 @@ struct bfq_iocq_bfqq_data { unsigned long saved_last_wr_start_finish; unsigned long saved_service_from_wr; unsigned long saved_wr_start_at_switch_to_srt; - unsigned int saved_wr_cur_max_time; struct bfq_ttime saved_ttime; + unsigned int saved_wr_cur_max_time; /* Save also injection state */ - u64 saved_last_serv_time_ns; unsigned int saved_inject_limit; unsigned long saved_decrease_time_jif; + u64 saved_last_serv_time_ns; /* candidate queue for a stable merge (due to close creation time) */ struct bfq_queue *stable_merge_bfqq; @@ -813,8 +813,7 @@ struct bfq_data { * Depth limits used in bfq_limit_depth (see comments on the * function) */ - unsigned int word_depths[2][2]; - unsigned int full_depth_shift; + unsigned int async_depths[2][2]; /* * Number of independent actuators. This is equal to 1 in diff --git a/block/blk-ioc.c b/block/blk-ioc.c index ce82770c72ab..9fda3906e5f5 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -308,24 +308,23 @@ int __copy_io(unsigned long clone_flags, struct task_struct *tsk) #ifdef CONFIG_BLK_ICQ /** - * ioc_lookup_icq - lookup io_cq from ioc + * ioc_lookup_icq - lookup io_cq from ioc in io issue path * @q: the associated request_queue * * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called - * with @q->queue_lock held. + * from io issue path, either return NULL if current issue io to @q for the + * first time, or return a valid icq. */ struct io_cq *ioc_lookup_icq(struct request_queue *q) { struct io_context *ioc = current->io_context; struct io_cq *icq; - lockdep_assert_held(&q->queue_lock); - /* * icq's are indexed from @ioc using radix tree and hint pointer, - * both of which are protected with RCU. All removals are done - * holding both q and ioc locks, and we're holding q lock - if we - * find a icq which points to us, it's guaranteed to be valid. + * both of which are protected with RCU, io issue path ensures that + * both request_queue and current task are valid, the found icq + * is guaranteed to be valid until the io is done. */ rcu_read_lock(); icq = rcu_dereference(ioc->icq_hint); @@ -419,10 +418,7 @@ struct io_cq *ioc_find_get_icq(struct request_queue *q) task_unlock(current); } else { get_io_context(ioc); - - spin_lock_irq(&q->queue_lock); icq = ioc_lookup_icq(q); - spin_unlock_irq(&q->queue_lock); } if (!icq) { diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index 55a0fd105147..e2ce4a28e6c9 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -374,64 +374,17 @@ bool blk_mq_sched_try_insert_merge(struct request_queue *q, struct request *rq, } EXPORT_SYMBOL_GPL(blk_mq_sched_try_insert_merge); -static int blk_mq_sched_alloc_map_and_rqs(struct request_queue *q, - struct blk_mq_hw_ctx *hctx, - unsigned int hctx_idx) -{ - if (blk_mq_is_shared_tags(q->tag_set->flags)) { - hctx->sched_tags = q->sched_shared_tags; - return 0; - } - - hctx->sched_tags = blk_mq_alloc_map_and_rqs(q->tag_set, hctx_idx, - q->nr_requests); - - if (!hctx->sched_tags) - return -ENOMEM; - return 0; -} - -static void blk_mq_exit_sched_shared_tags(struct request_queue *queue) -{ - blk_mq_free_rq_map(queue->sched_shared_tags); - queue->sched_shared_tags = NULL; -} - /* called in queue's release handler, tagset has gone away */ static void blk_mq_sched_tags_teardown(struct request_queue *q, unsigned int flags) { struct blk_mq_hw_ctx *hctx; unsigned long i; - queue_for_each_hw_ctx(q, hctx, i) { - if (hctx->sched_tags) { - if (!blk_mq_is_shared_tags(flags)) - blk_mq_free_rq_map(hctx->sched_tags); - hctx->sched_tags = NULL; - } - } + queue_for_each_hw_ctx(q, hctx, i) + hctx->sched_tags = NULL; if (blk_mq_is_shared_tags(flags)) - blk_mq_exit_sched_shared_tags(q); -} - -static int blk_mq_init_sched_shared_tags(struct request_queue *queue) -{ - struct blk_mq_tag_set *set = queue->tag_set; - - /* - * Set initial depth at max so that we don't need to reallocate for - * updating nr_requests. - */ - queue->sched_shared_tags = blk_mq_alloc_map_and_rqs(set, - BLK_MQ_NO_HCTX_IDX, - MAX_SCHED_RQ); - if (!queue->sched_shared_tags) - return -ENOMEM; - - blk_mq_tag_update_sched_shared_tags(queue); - - return 0; + q->sched_shared_tags = NULL; } void blk_mq_sched_reg_debugfs(struct request_queue *q) @@ -458,8 +411,140 @@ void blk_mq_sched_unreg_debugfs(struct request_queue *q) mutex_unlock(&q->debugfs_mutex); } +void blk_mq_free_sched_tags(struct elevator_tags *et, + struct blk_mq_tag_set *set) +{ + unsigned long i; + + /* Shared tags are stored at index 0 in @tags. */ + if (blk_mq_is_shared_tags(set->flags)) + blk_mq_free_map_and_rqs(set, et->tags[0], BLK_MQ_NO_HCTX_IDX); + else { + for (i = 0; i < et->nr_hw_queues; i++) + blk_mq_free_map_and_rqs(set, et->tags[i], i); + } + + kfree(et); +} + +void blk_mq_free_sched_tags_batch(struct xarray *et_table, + struct blk_mq_tag_set *set) +{ + struct request_queue *q; + struct elevator_tags *et; + + lockdep_assert_held_write(&set->update_nr_hwq_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + /* + * Accessing q->elevator without holding q->elevator_lock is + * safe because we're holding here set->update_nr_hwq_lock in + * the writer context. So, scheduler update/switch code (which + * acquires the same lock but in the reader context) can't run + * concurrently. + */ + if (q->elevator) { + et = xa_load(et_table, q->id); + if (unlikely(!et)) + WARN_ON_ONCE(1); + else + blk_mq_free_sched_tags(et, set); + } + } +} + +struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, + unsigned int nr_hw_queues) +{ + unsigned int nr_tags; + int i; + struct elevator_tags *et; + gfp_t gfp = GFP_NOIO | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; + + if (blk_mq_is_shared_tags(set->flags)) + nr_tags = 1; + else + nr_tags = nr_hw_queues; + + et = kmalloc(sizeof(struct elevator_tags) + + nr_tags * sizeof(struct blk_mq_tags *), gfp); + if (!et) + return NULL; + /* + * Default to double of smaller one between hw queue_depth and + * 128, since we don't split into sync/async like the old code + * did. Additionally, this is a per-hw queue depth. + */ + et->nr_requests = 2 * min_t(unsigned int, set->queue_depth, + BLKDEV_DEFAULT_RQ); + et->nr_hw_queues = nr_hw_queues; + + if (blk_mq_is_shared_tags(set->flags)) { + /* Shared tags are stored at index 0 in @tags. */ + et->tags[0] = blk_mq_alloc_map_and_rqs(set, BLK_MQ_NO_HCTX_IDX, + MAX_SCHED_RQ); + if (!et->tags[0]) + goto out; + } else { + for (i = 0; i < et->nr_hw_queues; i++) { + et->tags[i] = blk_mq_alloc_map_and_rqs(set, i, + et->nr_requests); + if (!et->tags[i]) + goto out_unwind; + } + } + + return et; +out_unwind: + while (--i >= 0) + blk_mq_free_map_and_rqs(set, et->tags[i], i); +out: + kfree(et); + return NULL; +} + +int blk_mq_alloc_sched_tags_batch(struct xarray *et_table, + struct blk_mq_tag_set *set, unsigned int nr_hw_queues) +{ + struct request_queue *q; + struct elevator_tags *et; + gfp_t gfp = GFP_NOIO | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY; + + lockdep_assert_held_write(&set->update_nr_hwq_lock); + + list_for_each_entry(q, &set->tag_list, tag_set_list) { + /* + * Accessing q->elevator without holding q->elevator_lock is + * safe because we're holding here set->update_nr_hwq_lock in + * the writer context. So, scheduler update/switch code (which + * acquires the same lock but in the reader context) can't run + * concurrently. + */ + if (q->elevator) { + et = blk_mq_alloc_sched_tags(set, nr_hw_queues); + if (!et) + goto out_unwind; + if (xa_insert(et_table, q->id, et, gfp)) + goto out_free_tags; + } + } + return 0; +out_free_tags: + blk_mq_free_sched_tags(et, set); +out_unwind: + list_for_each_entry_continue_reverse(q, &set->tag_list, tag_set_list) { + if (q->elevator) { + et = xa_load(et_table, q->id); + if (et) + blk_mq_free_sched_tags(et, set); + } + } + return -ENOMEM; +} + /* caller must have a reference to @e, will grab another one if successful */ -int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e) +int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, + struct elevator_tags *et) { unsigned int flags = q->tag_set->flags; struct blk_mq_hw_ctx *hctx; @@ -467,36 +552,33 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e) unsigned long i; int ret; - /* - * Default to double of smaller one between hw queue_depth and 128, - * since we don't split into sync/async like the old code did. - * Additionally, this is a per-hw queue depth. - */ - q->nr_requests = 2 * min_t(unsigned int, q->tag_set->queue_depth, - BLKDEV_DEFAULT_RQ); + eq = elevator_alloc(q, e, et); + if (!eq) + return -ENOMEM; + + q->nr_requests = et->nr_requests; if (blk_mq_is_shared_tags(flags)) { - ret = blk_mq_init_sched_shared_tags(q); - if (ret) - return ret; + /* Shared tags are stored at index 0 in @et->tags. */ + q->sched_shared_tags = et->tags[0]; + blk_mq_tag_update_sched_shared_tags(q); } queue_for_each_hw_ctx(q, hctx, i) { - ret = blk_mq_sched_alloc_map_and_rqs(q, hctx, i); - if (ret) - goto err_free_map_and_rqs; + if (blk_mq_is_shared_tags(flags)) + hctx->sched_tags = q->sched_shared_tags; + else + hctx->sched_tags = et->tags[i]; } - ret = e->ops.init_sched(q, e); + ret = e->ops.init_sched(q, eq); if (ret) - goto err_free_map_and_rqs; + goto out; queue_for_each_hw_ctx(q, hctx, i) { if (e->ops.init_hctx) { ret = e->ops.init_hctx(hctx, i); if (ret) { - eq = q->elevator; - blk_mq_sched_free_rqs(q); blk_mq_exit_sched(q, eq); kobject_put(&eq->kobj); return ret; @@ -505,10 +587,9 @@ int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e) } return 0; -err_free_map_and_rqs: - blk_mq_sched_free_rqs(q); +out: blk_mq_sched_tags_teardown(q, flags); - + kobject_put(&eq->kobj); q->elevator = NULL; return ret; } diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h index 1326526bb733..b554e1d55950 100644 --- a/block/blk-mq-sched.h +++ b/block/blk-mq-sched.h @@ -18,10 +18,20 @@ void __blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx); void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx); -int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e); +int blk_mq_init_sched(struct request_queue *q, struct elevator_type *e, + struct elevator_tags *et); void blk_mq_exit_sched(struct request_queue *q, struct elevator_queue *e); void blk_mq_sched_free_rqs(struct request_queue *q); +struct elevator_tags *blk_mq_alloc_sched_tags(struct blk_mq_tag_set *set, + unsigned int nr_hw_queues); +int blk_mq_alloc_sched_tags_batch(struct xarray *et_table, + struct blk_mq_tag_set *set, unsigned int nr_hw_queues); +void blk_mq_free_sched_tags(struct elevator_tags *et, + struct blk_mq_tag_set *set); +void blk_mq_free_sched_tags_batch(struct xarray *et_table, + struct blk_mq_tag_set *set); + static inline void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx) { if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state)) diff --git a/block/blk-mq.c b/block/blk-mq.c index 9692fa4c3ef2..b67d6c02eceb 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -4974,12 +4974,13 @@ int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr) * Switch back to the elevator type stored in the xarray. */ static void blk_mq_elv_switch_back(struct request_queue *q, - struct xarray *elv_tbl) + struct xarray *elv_tbl, struct xarray *et_tbl) { struct elevator_type *e = xa_load(elv_tbl, q->id); + struct elevator_tags *t = xa_load(et_tbl, q->id); /* The elv_update_nr_hw_queues unfreezes the queue. */ - elv_update_nr_hw_queues(q, e); + elv_update_nr_hw_queues(q, e, t); /* Drop the reference acquired in blk_mq_elv_switch_none. */ if (e) @@ -5031,7 +5032,7 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int prev_nr_hw_queues = set->nr_hw_queues; unsigned int memflags; int i; - struct xarray elv_tbl; + struct xarray elv_tbl, et_tbl; lockdep_assert_held(&set->tag_list_lock); @@ -5044,6 +5045,10 @@ static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, memflags = memalloc_noio_save(); + xa_init(&et_tbl); + if (blk_mq_alloc_sched_tags_batch(&et_tbl, set, nr_hw_queues) < 0) + goto out_memalloc_restore; + xa_init(&elv_tbl); list_for_each_entry(q, &set->tag_list, tag_set_list) { @@ -5087,7 +5092,7 @@ fallback: switch_back: /* The blk_mq_elv_switch_back unfreezes queue for us. */ list_for_each_entry(q, &set->tag_list, tag_set_list) - blk_mq_elv_switch_back(q, &elv_tbl); + blk_mq_elv_switch_back(q, &elv_tbl, &et_tbl); list_for_each_entry(q, &set->tag_list, tag_set_list) { blk_mq_sysfs_register_hctxs(q); @@ -5098,7 +5103,8 @@ switch_back: } xa_destroy(&elv_tbl); - + xa_destroy(&et_tbl); +out_memalloc_restore: memalloc_noio_restore(memflags); /* Free the excess tags when nr_hw_queues shrink. */ diff --git a/block/blk-settings.c b/block/blk-settings.c index 91449147bae9..07874e9b609f 100644 --- a/block/blk-settings.c +++ b/block/blk-settings.c @@ -62,16 +62,24 @@ EXPORT_SYMBOL(blk_set_stacking_limits); void blk_apply_bdi_limits(struct backing_dev_info *bdi, struct queue_limits *lim) { + u64 io_opt = lim->io_opt; + /* * For read-ahead of large files to be effective, we need to read ahead - * at least twice the optimal I/O size. + * at least twice the optimal I/O size. For rotational devices that do + * not report an optimal I/O size (e.g. ATA HDDs), use the maximum I/O + * size to avoid falling back to the (rather inefficient) small default + * read-ahead size. * * There is no hardware limitation for the read-ahead size and the user * might have increased the read-ahead size through sysfs, so don't ever * decrease it. */ + if (!io_opt && (lim->features & BLK_FEAT_ROTATIONAL)) + io_opt = (u64)lim->max_sectors << SECTOR_SHIFT; + bdi->ra_pages = max3(bdi->ra_pages, - lim->io_opt * 2 / PAGE_SIZE, + io_opt * 2 >> PAGE_SHIFT, VM_READAHEAD_PAGES); bdi->io_pages = lim->max_sectors >> PAGE_SECTORS_SHIFT; } @@ -312,8 +320,12 @@ int blk_validate_limits(struct queue_limits *lim) pr_warn("Invalid logical block size (%d)\n", lim->logical_block_size); return -EINVAL; } - if (lim->physical_block_size < lim->logical_block_size) + if (lim->physical_block_size < lim->logical_block_size) { lim->physical_block_size = lim->logical_block_size; + } else if (!is_power_of_2(lim->physical_block_size)) { + pr_warn("Invalid physical block size (%d)\n", lim->physical_block_size); + return -EINVAL; + } /* * The minimum I/O size defaults to the physical block size unless @@ -388,12 +400,19 @@ int blk_validate_limits(struct queue_limits *lim) lim->max_discard_sectors = min(lim->max_hw_discard_sectors, lim->max_user_discard_sectors); + /* + * When discard is not supported, discard_granularity should be reported + * as 0 to userspace. + */ + if (lim->max_discard_sectors) + lim->discard_granularity = + max(lim->discard_granularity, lim->physical_block_size); + else + lim->discard_granularity = 0; + if (!lim->max_discard_segments) lim->max_discard_segments = 1; - if (lim->discard_granularity < lim->physical_block_size) - lim->discard_granularity = lim->physical_block_size; - /* * By default there is no limit on the segment boundary alignment, * but if there is one it can't be smaller than the page size as @@ -849,7 +868,7 @@ int blk_stack_limits(struct queue_limits *t, struct queue_limits *b, } /* chunk_sectors a multiple of the physical block size? */ - if ((t->chunk_sectors << 9) & (t->physical_block_size - 1)) { + if (t->chunk_sectors % (t->physical_block_size >> SECTOR_SHIFT)) { t->chunk_sectors = 0; t->flags |= BLK_FLAG_MISALIGNED; ret = -1; diff --git a/block/blk.h b/block/blk.h index 76901a39997f..0a2eccf28ca4 100644 --- a/block/blk.h +++ b/block/blk.h @@ -12,6 +12,7 @@ #include "blk-crypto-internal.h" struct elevator_type; +struct elevator_tags; /* * Default upper limit for the software max_sectors limit used for regular I/Os. @@ -330,7 +331,8 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list, bool blk_insert_flush(struct request *rq); -void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e); +void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e, + struct elevator_tags *t); void elevator_set_default(struct request_queue *q); void elevator_set_none(struct request_queue *q); diff --git a/block/elevator.c b/block/elevator.c index 88f8f36bed98..fe96c6f4753c 100644 --- a/block/elevator.c +++ b/block/elevator.c @@ -54,6 +54,8 @@ struct elv_change_ctx { struct elevator_queue *old; /* for registering new elevator */ struct elevator_queue *new; + /* holds sched tags data */ + struct elevator_tags *et; }; static DEFINE_SPINLOCK(elv_list_lock); @@ -132,7 +134,7 @@ static struct elevator_type *elevator_find_get(const char *name) static const struct kobj_type elv_ktype; struct elevator_queue *elevator_alloc(struct request_queue *q, - struct elevator_type *e) + struct elevator_type *e, struct elevator_tags *et) { struct elevator_queue *eq; @@ -145,10 +147,10 @@ struct elevator_queue *elevator_alloc(struct request_queue *q, kobject_init(&eq->kobj, &elv_ktype); mutex_init(&eq->sysfs_lock); hash_init(eq->hash); + eq->et = et; return eq; } -EXPORT_SYMBOL(elevator_alloc); static void elevator_release(struct kobject *kobj) { @@ -166,7 +168,6 @@ static void elevator_exit(struct request_queue *q) lockdep_assert_held(&q->elevator_lock); ioc_clear_queue(q); - blk_mq_sched_free_rqs(q); mutex_lock(&e->sysfs_lock); blk_mq_exit_sched(q, e); @@ -592,7 +593,7 @@ static int elevator_switch(struct request_queue *q, struct elv_change_ctx *ctx) } if (new_e) { - ret = blk_mq_init_sched(q, new_e); + ret = blk_mq_init_sched(q, new_e, ctx->et); if (ret) goto out_unfreeze; ctx->new = q->elevator; @@ -627,8 +628,10 @@ static void elv_exit_and_release(struct request_queue *q) elevator_exit(q); mutex_unlock(&q->elevator_lock); blk_mq_unfreeze_queue(q, memflags); - if (e) + if (e) { + blk_mq_free_sched_tags(e->et, q->tag_set); kobject_put(&e->kobj); + } } static int elevator_change_done(struct request_queue *q, @@ -641,6 +644,7 @@ static int elevator_change_done(struct request_queue *q, &ctx->old->flags); elv_unregister_queue(q, ctx->old); + blk_mq_free_sched_tags(ctx->old->et, q->tag_set); kobject_put(&ctx->old->kobj); if (enable_wbt) wbt_enable_default(q->disk); @@ -659,9 +663,16 @@ static int elevator_change_done(struct request_queue *q, static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) { unsigned int memflags; + struct blk_mq_tag_set *set = q->tag_set; int ret = 0; - lockdep_assert_held(&q->tag_set->update_nr_hwq_lock); + lockdep_assert_held(&set->update_nr_hwq_lock); + + if (strncmp(ctx->name, "none", 4)) { + ctx->et = blk_mq_alloc_sched_tags(set, set->nr_hw_queues); + if (!ctx->et) + return -ENOMEM; + } memflags = blk_mq_freeze_queue(q); /* @@ -681,6 +692,11 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) blk_mq_unfreeze_queue(q, memflags); if (!ret) ret = elevator_change_done(q, ctx); + /* + * Free sched tags if it's allocated but we couldn't switch elevator. + */ + if (ctx->et && !ctx->new) + blk_mq_free_sched_tags(ctx->et, set); return ret; } @@ -689,8 +705,10 @@ static int elevator_change(struct request_queue *q, struct elv_change_ctx *ctx) * The I/O scheduler depends on the number of hardware queues, this forces a * reattachment when nr_hw_queues changes. */ -void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e) +void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e, + struct elevator_tags *t) { + struct blk_mq_tag_set *set = q->tag_set; struct elv_change_ctx ctx = {}; int ret = -ENODEV; @@ -698,6 +716,7 @@ void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e) if (e && !blk_queue_dying(q) && blk_queue_registered(q)) { ctx.name = e->elevator_name; + ctx.et = t; mutex_lock(&q->elevator_lock); /* force to reattach elevator after nr_hw_queue is updated */ @@ -707,6 +726,11 @@ void elv_update_nr_hw_queues(struct request_queue *q, struct elevator_type *e) blk_mq_unfreeze_queue_nomemrestore(q); if (!ret) WARN_ON_ONCE(elevator_change_done(q, &ctx)); + /* + * Free sched tags if it's allocated but we couldn't switch elevator. + */ + if (t && !ctx.new) + blk_mq_free_sched_tags(t, set); } /* diff --git a/block/elevator.h b/block/elevator.h index a07ce773a38f..adc5c157e17e 100644 --- a/block/elevator.h +++ b/block/elevator.h @@ -23,8 +23,17 @@ enum elv_merge { struct blk_mq_alloc_data; struct blk_mq_hw_ctx; +struct elevator_tags { + /* num. of hardware queues for which tags are allocated */ + unsigned int nr_hw_queues; + /* depth used while allocating tags */ + unsigned int nr_requests; + /* shared tag is stored at index 0 */ + struct blk_mq_tags *tags[]; +}; + struct elevator_mq_ops { - int (*init_sched)(struct request_queue *, struct elevator_type *); + int (*init_sched)(struct request_queue *, struct elevator_queue *); void (*exit_sched)(struct elevator_queue *); int (*init_hctx)(struct blk_mq_hw_ctx *, unsigned int); void (*exit_hctx)(struct blk_mq_hw_ctx *, unsigned int); @@ -113,6 +122,7 @@ struct request *elv_rqhash_find(struct request_queue *q, sector_t offset); struct elevator_queue { struct elevator_type *type; + struct elevator_tags *et; void *elevator_data; struct kobject kobj; struct mutex sysfs_lock; @@ -152,8 +162,8 @@ ssize_t elv_iosched_show(struct gendisk *disk, char *page); ssize_t elv_iosched_store(struct gendisk *disk, const char *page, size_t count); extern bool elv_bio_merge_ok(struct request *, struct bio *); -extern struct elevator_queue *elevator_alloc(struct request_queue *, - struct elevator_type *); +struct elevator_queue *elevator_alloc(struct request_queue *, + struct elevator_type *, struct elevator_tags *); /* * Helper functions. diff --git a/block/kyber-iosched.c b/block/kyber-iosched.c index 4dba8405bd01..70cbc7b2deb4 100644 --- a/block/kyber-iosched.c +++ b/block/kyber-iosched.c @@ -157,10 +157,7 @@ struct kyber_queue_data { */ struct sbitmap_queue domain_tokens[KYBER_NUM_DOMAINS]; - /* - * Async request percentage, converted to per-word depth for - * sbitmap_get_shallow(). - */ + /* Number of allowed async requests. */ unsigned int async_depth; struct kyber_cpu_latency __percpu *cpu_latency; @@ -402,20 +399,13 @@ err: return ERR_PTR(ret); } -static int kyber_init_sched(struct request_queue *q, struct elevator_type *e) +static int kyber_init_sched(struct request_queue *q, struct elevator_queue *eq) { struct kyber_queue_data *kqd; - struct elevator_queue *eq; - - eq = elevator_alloc(q, e); - if (!eq) - return -ENOMEM; kqd = kyber_queue_data_alloc(q); - if (IS_ERR(kqd)) { - kobject_put(&eq->kobj); + if (IS_ERR(kqd)) return PTR_ERR(kqd); - } blk_stat_enable_accounting(q); @@ -454,10 +444,8 @@ static void kyber_depth_updated(struct blk_mq_hw_ctx *hctx) { struct kyber_queue_data *kqd = hctx->queue->elevator->elevator_data; struct blk_mq_tags *tags = hctx->sched_tags; - unsigned int shift = tags->bitmap_tags.sb.shift; - - kqd->async_depth = (1U << shift) * KYBER_ASYNC_PERCENT / 100U; + kqd->async_depth = hctx->queue->nr_requests * KYBER_ASYNC_PERCENT / 100U; sbitmap_queue_min_shallow_depth(&tags->bitmap_tags, kqd->async_depth); } diff --git a/block/mq-deadline.c b/block/mq-deadline.c index 2edf1cac06d5..b9b7cdf1d3c9 100644 --- a/block/mq-deadline.c +++ b/block/mq-deadline.c @@ -488,20 +488,6 @@ unlock: } /* - * 'depth' is a number in the range 1..INT_MAX representing a number of - * requests. Scale it with a factor (1 << bt->sb.shift) / q->nr_requests since - * 1..(1 << bt->sb.shift) is the range expected by sbitmap_get_shallow(). - * Values larger than q->nr_requests have the same effect as q->nr_requests. - */ -static int dd_to_word_depth(struct blk_mq_hw_ctx *hctx, unsigned int qdepth) -{ - struct sbitmap_queue *bt = &hctx->sched_tags->bitmap_tags; - const unsigned int nrr = hctx->queue->nr_requests; - - return ((qdepth << bt->sb.shift) + nrr - 1) / nrr; -} - -/* * Called by __blk_mq_alloc_request(). The shallow_depth value set by this * function is used by __blk_mq_get_tag(). */ @@ -517,7 +503,7 @@ static void dd_limit_depth(blk_opf_t opf, struct blk_mq_alloc_data *data) * Throttle asynchronous requests and writes such that these requests * do not block the allocation of synchronous requests. */ - data->shallow_depth = dd_to_word_depth(data->hctx, dd->async_depth); + data->shallow_depth = dd->async_depth; } /* Called by blk_mq_update_nr_requests(). */ @@ -568,20 +554,14 @@ static void dd_exit_sched(struct elevator_queue *e) /* * initialize elevator private data (deadline_data). */ -static int dd_init_sched(struct request_queue *q, struct elevator_type *e) +static int dd_init_sched(struct request_queue *q, struct elevator_queue *eq) { struct deadline_data *dd; - struct elevator_queue *eq; enum dd_prio prio; - int ret = -ENOMEM; - - eq = elevator_alloc(q, e); - if (!eq) - return ret; dd = kzalloc_node(sizeof(*dd), GFP_KERNEL, q->node); if (!dd) - goto put_eq; + return -ENOMEM; eq->elevator_data = dd; @@ -608,10 +588,6 @@ static int dd_init_sched(struct request_queue *q, struct elevator_type *e) q->elevator = eq; return 0; - -put_eq: - kobject_put(&eq->kobj); - return ret; } /* diff --git a/drivers/base/regmap/regmap-irq.c b/drivers/base/regmap/regmap-irq.c index d1585f073776..6112d942499b 100644 --- a/drivers/base/regmap/regmap-irq.c +++ b/drivers/base/regmap/regmap-irq.c @@ -21,6 +21,7 @@ struct regmap_irq_chip_data { struct mutex lock; + struct lock_class_key lock_key; struct irq_chip irq_chip; struct regmap *map; @@ -801,7 +802,13 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, goto err_alloc; } - mutex_init(&d->lock); + /* + * If one regmap-irq is the parent of another then we'll try + * to lock the child with the parent locked, use an explicit + * lock_key so lockdep can figure out what's going on. + */ + lockdep_register_key(&d->lock_key); + mutex_init_with_key(&d->lock, &d->lock_key); for (i = 0; i < chip->num_irqs; i++) d->mask_buf_def[chip->irqs[i].reg_offset / map->reg_stride] @@ -816,7 +823,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, d->mask_buf[i], chip->irq_drv_data); if (ret) - goto err_alloc; + goto err_mutex; } if (chip->mask_base && !chip->handle_mask_sync) { @@ -827,7 +834,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (ret) { dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", reg, ret); - goto err_alloc; + goto err_mutex; } } @@ -838,7 +845,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (ret) { dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", reg, ret); - goto err_alloc; + goto err_mutex; } } @@ -855,7 +862,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (ret != 0) { dev_err(map->dev, "Failed to read IRQ status: %d\n", ret); - goto err_alloc; + goto err_mutex; } } @@ -879,7 +886,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (ret != 0) { dev_err(map->dev, "Failed to ack 0x%x: %d\n", reg, ret); - goto err_alloc; + goto err_mutex; } } } @@ -901,7 +908,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (ret != 0) { dev_err(map->dev, "Failed to set masks in 0x%x: %d\n", reg, ret); - goto err_alloc; + goto err_mutex; } } } @@ -910,7 +917,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, if (chip->status_is_level) { ret = read_irq_data(d); if (ret < 0) - goto err_alloc; + goto err_mutex; memcpy(d->prev_status_buf, d->status_buf, array_size(d->chip->num_regs, sizeof(d->prev_status_buf[0]))); @@ -918,7 +925,7 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, ret = regmap_irq_create_domain(fwnode, irq_base, chip, d); if (ret) - goto err_alloc; + goto err_mutex; ret = request_threaded_irq(irq, NULL, regmap_irq_thread, irq_flags | IRQF_ONESHOT, @@ -935,6 +942,9 @@ int regmap_add_irq_chip_fwnode(struct fwnode_handle *fwnode, err_domain: /* Should really dispose of the domain but... */ +err_mutex: + mutex_destroy(&d->lock); + lockdep_unregister_key(&d->lock_key); err_alloc: kfree(d->type_buf); kfree(d->type_buf_def); @@ -1027,6 +1037,8 @@ void regmap_del_irq_chip(int irq, struct regmap_irq_chip_data *d) kfree(d->config_buf[i]); kfree(d->config_buf); } + mutex_destroy(&d->lock); + lockdep_unregister_key(&d->lock_key); kfree(d); } EXPORT_SYMBOL_GPL(regmap_del_irq_chip); diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index 553b1a713ab9..a423228e201b 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -700,6 +700,8 @@ static void zloop_free_disk(struct gendisk *disk) struct zloop_device *zlo = disk->private_data; unsigned int i; + blk_mq_free_tag_set(&zlo->tag_set); + for (i = 0; i < zlo->nr_zones; i++) { struct zloop_zone *zone = &zlo->zones[i]; @@ -1080,7 +1082,6 @@ static int zloop_ctl_remove(struct zloop_options *opts) del_gendisk(zlo->disk); put_disk(zlo->disk); - blk_mq_free_tag_set(&zlo->tag_set); pr_info("Removed device %d\n", opts->id); diff --git a/drivers/dpll/zl3073x/Kconfig b/drivers/dpll/zl3073x/Kconfig index 7db262ab8458..5bbca1400581 100644 --- a/drivers/dpll/zl3073x/Kconfig +++ b/drivers/dpll/zl3073x/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config ZL3073X - tristate "Microchip Azurite DPLL/PTP/SyncE devices" + tristate "Microchip Azurite DPLL/PTP/SyncE devices" if COMPILE_TEST depends on NET select DPLL select NET_DEVLINK @@ -16,9 +16,9 @@ config ZL3073X config ZL3073X_I2C tristate "I2C bus implementation for Microchip Azurite devices" - depends on I2C && ZL3073X + depends on I2C && NET select REGMAP_I2C - default m + select ZL3073X help This is I2C bus implementation for Microchip Azurite DPLL/PTP/SyncE devices. @@ -28,9 +28,9 @@ config ZL3073X_I2C config ZL3073X_SPI tristate "SPI bus implementation for Microchip Azurite devices" - depends on SPI && ZL3073X + depends on NET && SPI select REGMAP_SPI - default m + select ZL3073X help This is SPI bus implementation for Microchip Azurite DPLL/PTP/SyncE devices. diff --git a/drivers/firmware/efi/Kconfig b/drivers/firmware/efi/Kconfig index 16baa038d412..d528c94c5859 100644 --- a/drivers/firmware/efi/Kconfig +++ b/drivers/firmware/efi/Kconfig @@ -263,6 +263,14 @@ config EFI_COCO_SECRET virt/coco/efi_secret module to access the secrets, which in turn allows userspace programs to access the injected secrets. +config OVMF_DEBUG_LOG + bool "Expose OVMF firmware debug log via sysfs" + depends on EFI + help + Recent OVMF versions (edk2-stable202508 + newer) can write + their debug log to a memory buffer. This driver exposes the + log content via sysfs (/sys/firmware/efi/ovmf_debug_log). + config UNACCEPTED_MEMORY bool depends on EFI_STUB diff --git a/drivers/firmware/efi/Makefile b/drivers/firmware/efi/Makefile index a2d0009560d0..8efbcf699e4f 100644 --- a/drivers/firmware/efi/Makefile +++ b/drivers/firmware/efi/Makefile @@ -29,6 +29,7 @@ obj-$(CONFIG_APPLE_PROPERTIES) += apple-properties.o obj-$(CONFIG_EFI_RCI2_TABLE) += rci2-table.o obj-$(CONFIG_EFI_EMBEDDED_FIRMWARE) += embedded-firmware.o obj-$(CONFIG_LOAD_UEFI_KEYS) += mokvar-table.o +obj-$(CONFIG_OVMF_DEBUG_LOG) += ovmf-debug-log.o obj-$(CONFIG_SYSFB) += sysfb_efi.o diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index e57bff702b5f..1ce428e2ac8a 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -45,6 +45,7 @@ struct efi __read_mostly efi = { .esrt = EFI_INVALID_TABLE_ADDR, .tpm_log = EFI_INVALID_TABLE_ADDR, .tpm_final_log = EFI_INVALID_TABLE_ADDR, + .ovmf_debug_log = EFI_INVALID_TABLE_ADDR, #ifdef CONFIG_LOAD_UEFI_KEYS .mokvar_table = EFI_INVALID_TABLE_ADDR, #endif @@ -473,6 +474,10 @@ static int __init efisubsys_init(void) platform_device_register_simple("efi_secret", 0, NULL, 0); #endif + if (IS_ENABLED(CONFIG_OVMF_DEBUG_LOG) && + efi.ovmf_debug_log != EFI_INVALID_TABLE_ADDR) + ovmf_log_probe(efi.ovmf_debug_log); + return 0; err_remove_group: @@ -617,6 +622,9 @@ static const efi_config_table_type_t common_tables[] __initconst = { {LINUX_EFI_MEMRESERVE_TABLE_GUID, &mem_reserve, "MEMRESERVE" }, {LINUX_EFI_INITRD_MEDIA_GUID, &initrd, "INITRD" }, {EFI_RT_PROPERTIES_TABLE_GUID, &rt_prop, "RTPROP" }, +#ifdef CONFIG_OVMF_DEBUG_LOG + {OVMF_MEMORY_LOG_TABLE_GUID, &efi.ovmf_debug_log, "OvmfDebugLog" }, +#endif #ifdef CONFIG_EFI_RCI2_TABLE {DELLEMC_EFI_RCI2_TABLE_GUID, &rci2_table_phys }, #endif diff --git a/drivers/firmware/efi/libstub/printk.c b/drivers/firmware/efi/libstub/printk.c index 3a67a2cea7bd..bc599212c05d 100644 --- a/drivers/firmware/efi/libstub/printk.c +++ b/drivers/firmware/efi/libstub/printk.c @@ -5,13 +5,13 @@ #include <linux/ctype.h> #include <linux/efi.h> #include <linux/kernel.h> -#include <linux/printk.h> /* For CONSOLE_LOGLEVEL_* */ +#include <linux/kern_levels.h> #include <asm/efi.h> #include <asm/setup.h> #include "efistub.h" -int efi_loglevel = CONSOLE_LOGLEVEL_DEFAULT; +int efi_loglevel = LOGLEVEL_NOTICE; /** * efi_char16_puts() - Write a UCS-2 encoded string to the console diff --git a/drivers/firmware/efi/ovmf-debug-log.c b/drivers/firmware/efi/ovmf-debug-log.c new file mode 100644 index 000000000000..5b2471ffaeed --- /dev/null +++ b/drivers/firmware/efi/ovmf-debug-log.c @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/efi.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/kobject.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/sysfs.h> + +#define OVMF_DEBUG_LOG_MAGIC1 0x3167646d666d766f // "ovmfmdg1" +#define OVMF_DEBUG_LOG_MAGIC2 0x3267646d666d766f // "ovmfmdg2" + +struct ovmf_debug_log_header { + u64 magic1; + u64 magic2; + u64 hdr_size; + u64 log_size; + u64 lock; // edk2 spinlock + u64 head_off; + u64 tail_off; + u64 truncated; + u8 fw_version[128]; +}; + +static struct ovmf_debug_log_header *hdr; +static u8 *logbuf; +static u64 logbufsize; + +static ssize_t ovmf_log_read(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, char *buf, + loff_t offset, size_t count) +{ + u64 start, end; + + start = hdr->head_off + offset; + if (hdr->head_off > hdr->tail_off && start >= hdr->log_size) + start -= hdr->log_size; + + end = start + count; + if (start > hdr->tail_off) { + if (end > hdr->log_size) + end = hdr->log_size; + } else { + if (end > hdr->tail_off) + end = hdr->tail_off; + } + + if (start > logbufsize || end > logbufsize) + return 0; + if (start >= end) + return 0; + + memcpy(buf, logbuf + start, end - start); + return end - start; +} + +static struct bin_attribute ovmf_log_bin_attr = { + .attr = { + .name = "ovmf_debug_log", + .mode = 0444, + }, + .read = ovmf_log_read, +}; + +int __init ovmf_log_probe(unsigned long ovmf_debug_log_table) +{ + int ret = -EINVAL; + u64 size; + + /* map + verify header */ + hdr = memremap(ovmf_debug_log_table, sizeof(*hdr), MEMREMAP_WB); + if (!hdr) { + pr_err("OVMF debug log: header map failed\n"); + return -EINVAL; + } + + if (hdr->magic1 != OVMF_DEBUG_LOG_MAGIC1 || + hdr->magic2 != OVMF_DEBUG_LOG_MAGIC2) { + printk(KERN_ERR "OVMF debug log: magic mismatch\n"); + goto err_unmap; + } + + size = hdr->hdr_size + hdr->log_size; + pr_info("OVMF debug log: firmware version: \"%s\"\n", hdr->fw_version); + pr_info("OVMF debug log: buffer size: %lluk\n", size / 1024); + + /* map complete log buffer */ + memunmap(hdr); + hdr = memremap(ovmf_debug_log_table, size, MEMREMAP_WB); + if (!hdr) { + pr_err("OVMF debug log: buffer map failed\n"); + return -EINVAL; + } + logbuf = (void *)hdr + hdr->hdr_size; + logbufsize = hdr->log_size; + + ovmf_log_bin_attr.size = size; + ret = sysfs_create_bin_file(efi_kobj, &ovmf_log_bin_attr); + if (ret != 0) { + pr_err("OVMF debug log: sysfs register failed\n"); + goto err_unmap; + } + + return 0; + +err_unmap: + memunmap(hdr); + return ret; +} diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 6f93473436be..01d234cf8156 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2570,9 +2570,6 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) adev->firmware.gpu_info_fw = NULL; - if (adev->mman.discovery_bin) - return 0; - switch (adev->asic_type) { default: return 0; @@ -2594,6 +2591,8 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) chip_name = "arcturus"; break; case CHIP_NAVI12: + if (adev->mman.discovery_bin) + return 0; chip_name = "navi12"; break; } @@ -3271,6 +3270,7 @@ static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev) * always assumed to be lost. */ switch (amdgpu_asic_reset_method(adev)) { + case AMD_RESET_METHOD_LEGACY: case AMD_RESET_METHOD_LINK: case AMD_RESET_METHOD_BACO: case AMD_RESET_METHOD_MODE1: diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c index 81b3443c8d7f..efe0058b48ca 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_discovery.c @@ -276,7 +276,7 @@ static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev, u32 msg; if (!amdgpu_sriov_vf(adev)) { - /* It can take up to a second for IFWI init to complete on some dGPUs, + /* It can take up to two second for IFWI init to complete on some dGPUs, * but generally it should be in the 60-100ms range. Normally this starts * as soon as the device gets power so by the time the OS loads this has long * completed. However, when a card is hotplugged via e.g., USB4, we need to @@ -284,7 +284,7 @@ static int amdgpu_discovery_read_binary_from_mem(struct amdgpu_device *adev, * continue. */ - for (i = 0; i < 1000; i++) { + for (i = 0; i < 2000; i++) { msg = RREG32(mmMP0_SMN_C2PMSG_33); if (msg & 0x80000000) break; @@ -2555,40 +2555,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) switch (adev->asic_type) { case CHIP_VEGA10: - case CHIP_VEGA12: - case CHIP_RAVEN: - case CHIP_VEGA20: - case CHIP_ARCTURUS: - case CHIP_ALDEBARAN: - /* this is not fatal. We have a fallback below - * if the new firmwares are not present. some of - * this will be overridden below to keep things - * consistent with the current behavior. + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. */ - r = amdgpu_discovery_reg_base_init(adev); - if (!r) { - amdgpu_discovery_harvest_ip(adev); - amdgpu_discovery_get_gfx_info(adev); - amdgpu_discovery_get_mall_info(adev); - amdgpu_discovery_get_vcn_info(adev); - } - break; - default: - r = amdgpu_discovery_reg_base_init(adev); - if (r) { - drm_err(&adev->ddev, "discovery failed: %d\n", r); - return r; - } - - amdgpu_discovery_harvest_ip(adev); - amdgpu_discovery_get_gfx_info(adev); - amdgpu_discovery_get_mall_info(adev); - amdgpu_discovery_get_vcn_info(adev); - break; - } - - switch (adev->asic_type) { - case CHIP_VEGA10: + amdgpu_discovery_init(adev); vega10_reg_base_init(adev); adev->sdma.num_instances = 2; adev->gmc.num_umc = 4; @@ -2611,6 +2582,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->ip_versions[DCI_HWIP][0] = IP_VERSION(12, 0, 0); break; case CHIP_VEGA12: + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. + */ + amdgpu_discovery_init(adev); vega10_reg_base_init(adev); adev->sdma.num_instances = 2; adev->gmc.num_umc = 4; @@ -2633,6 +2609,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->ip_versions[DCI_HWIP][0] = IP_VERSION(12, 0, 1); break; case CHIP_RAVEN: + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. + */ + amdgpu_discovery_init(adev); vega10_reg_base_init(adev); adev->sdma.num_instances = 1; adev->vcn.num_vcn_inst = 1; @@ -2674,6 +2655,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) } break; case CHIP_VEGA20: + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. + */ + amdgpu_discovery_init(adev); vega20_reg_base_init(adev); adev->sdma.num_instances = 2; adev->gmc.num_umc = 8; @@ -2697,6 +2683,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->ip_versions[DCI_HWIP][0] = IP_VERSION(12, 1, 0); break; case CHIP_ARCTURUS: + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. + */ + amdgpu_discovery_init(adev); arct_reg_base_init(adev); adev->sdma.num_instances = 8; adev->vcn.num_vcn_inst = 2; @@ -2725,6 +2716,11 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->ip_versions[UVD_HWIP][1] = IP_VERSION(2, 5, 0); break; case CHIP_ALDEBARAN: + /* This is not fatal. We only need the discovery + * binary for sysfs. We don't need it for a + * functional system. + */ + amdgpu_discovery_init(adev); aldebaran_reg_base_init(adev); adev->sdma.num_instances = 5; adev->vcn.num_vcn_inst = 2; @@ -2751,6 +2747,16 @@ int amdgpu_discovery_set_ip_blocks(struct amdgpu_device *adev) adev->ip_versions[XGMI_HWIP][0] = IP_VERSION(6, 1, 0); break; default: + r = amdgpu_discovery_reg_base_init(adev); + if (r) { + drm_err(&adev->ddev, "discovery failed: %d\n", r); + return r; + } + + amdgpu_discovery_harvest_ip(adev); + amdgpu_discovery_get_gfx_info(adev); + amdgpu_discovery_get_mall_info(adev); + amdgpu_discovery_get_vcn_info(adev); break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c index e6061d45f142..9b1c55115921 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c @@ -365,13 +365,6 @@ amdgpu_job_prepare_job(struct drm_sched_job *sched_job, dev_err(ring->adev->dev, "Error getting VM ID (%d)\n", r); goto error; } - /* - * The VM structure might be released after the VMID is - * assigned, we had multiple problems with people trying to use - * the VM pointer so better set it to NULL. - */ - if (!fence) - job->vm = NULL; return fence; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c index e56ba93a8df6..a974265837f0 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_nbio.c @@ -55,7 +55,8 @@ u64 amdgpu_nbio_get_pcie_replay_count(struct amdgpu_device *adev) bool amdgpu_nbio_is_replay_cnt_supported(struct amdgpu_device *adev) { - if (amdgpu_sriov_vf(adev) || !adev->asic_funcs->get_pcie_replay_count || + if (amdgpu_sriov_vf(adev) || !adev->asic_funcs || + !adev->asic_funcs->get_pcie_replay_count || (!adev->nbio.funcs || !adev->nbio.funcs->get_pcie_replay_count)) return false; diff --git a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c index 914cf4bfb033..811124ff88a8 100644 --- a/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c +++ b/drivers/gpu/drm/amd/amdgpu/aqua_vanjaram.c @@ -227,6 +227,7 @@ static int __aqua_vanjaram_get_px_mode_info(struct amdgpu_xcp_mgr *xcp_mgr, uint16_t *nps_modes) { struct amdgpu_device *adev = xcp_mgr->adev; + uint32_t gc_ver = amdgpu_ip_version(adev, GC_HWIP, 0); if (!num_xcp || !nps_modes || !(xcp_mgr->supp_xcp_modes & BIT(px_mode))) return -EINVAL; @@ -250,12 +251,14 @@ static int __aqua_vanjaram_get_px_mode_info(struct amdgpu_xcp_mgr *xcp_mgr, *num_xcp = 4; *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE); + if (gc_ver == IP_VERSION(9, 5, 0)) + *nps_modes |= BIT(AMDGPU_NPS2_PARTITION_MODE); break; case AMDGPU_CPX_PARTITION_MODE: *num_xcp = NUM_XCC(adev->gfx.xcc_mask); *nps_modes = BIT(AMDGPU_NPS1_PARTITION_MODE) | BIT(AMDGPU_NPS4_PARTITION_MODE); - if (amdgpu_sriov_vf(adev)) + if (gc_ver == IP_VERSION(9, 5, 0)) *nps_modes |= BIT(AMDGPU_NPS2_PARTITION_MODE); break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c index 134c4ec10887..910337dc28d1 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_0_1.c @@ -36,40 +36,47 @@ static const char *mmhub_client_ids_v3_0_1[][2] = { [0][0] = "VMC", + [1][0] = "ISPXT", + [2][0] = "ISPIXT", [4][0] = "DCEDMC", [5][0] = "DCEVGA", [6][0] = "MP0", [7][0] = "MP1", - [8][0] = "MPIO", - [16][0] = "HDP", - [17][0] = "LSDMA", - [18][0] = "JPEG", - [19][0] = "VCNU0", - [21][0] = "VSCH", - [22][0] = "VCNU1", - [23][0] = "VCN1", - [32+20][0] = "VCN0", - [2][1] = "DBGUNBIO", + [8][0] = "MPM", + [12][0] = "ISPTNR", + [14][0] = "ISPCRD0", + [15][0] = "ISPCRD1", + [16][0] = "ISPCRD2", + [22][0] = "HDP", + [23][0] = "LSDMA", + [24][0] = "JPEG", + [27][0] = "VSCH", + [28][0] = "VCNU", + [29][0] = "VCN", + [1][1] = "ISPXT", + [2][1] = "ISPIXT", [3][1] = "DCEDWB", [4][1] = "DCEDMC", [5][1] = "DCEVGA", [6][1] = "MP0", [7][1] = "MP1", - [8][1] = "MPIO", - [10][1] = "DBGU0", - [11][1] = "DBGU1", - [12][1] = "DBGU2", - [13][1] = "DBGU3", - [14][1] = "XDP", - [15][1] = "OSSSYS", - [16][1] = "HDP", - [17][1] = "LSDMA", - [18][1] = "JPEG", - [19][1] = "VCNU0", - [20][1] = "VCN0", - [21][1] = "VSCH", - [22][1] = "VCNU1", - [23][1] = "VCN1", + [8][1] = "MPM", + [10][1] = "ISPMWR0", + [11][1] = "ISPMWR1", + [12][1] = "ISPTNR", + [13][1] = "ISPSWR", + [14][1] = "ISPCWR0", + [15][1] = "ISPCWR1", + [16][1] = "ISPCWR2", + [17][1] = "ISPCWR3", + [18][1] = "XDP", + [21][1] = "OSSSYS", + [22][1] = "HDP", + [23][1] = "LSDMA", + [24][1] = "JPEG", + [27][1] = "VSCH", + [28][1] = "VCNU", + [29][1] = "VCN", }; static uint32_t mmhub_v3_0_1_get_invalidate_req(unsigned int vmid, diff --git a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c index bc3d6c2fc87a..f6fc9778bc30 100644 --- a/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c +++ b/drivers/gpu/drm/amd/amdgpu/mmhub_v3_3.c @@ -40,30 +40,129 @@ static const char *mmhub_client_ids_v3_3[][2] = { [0][0] = "VMC", + [1][0] = "ISPXT", + [2][0] = "ISPIXT", [4][0] = "DCEDMC", [6][0] = "MP0", [7][0] = "MP1", [8][0] = "MPM", + [9][0] = "ISPPDPRD", + [10][0] = "ISPCSTATRD", + [11][0] = "ISPBYRPRD", + [12][0] = "ISPRGBPRD", + [13][0] = "ISPMCFPRD", + [14][0] = "ISPMCFPRD1", + [15][0] = "ISPYUVPRD", + [16][0] = "ISPMCSCRD", + [17][0] = "ISPGDCRD", + [18][0] = "ISPLMERD", + [22][0] = "ISPXT1", + [23][0] = "ISPIXT1", [24][0] = "HDP", [25][0] = "LSDMA", [26][0] = "JPEG", [27][0] = "VPE", + [28][0] = "VSCH", [29][0] = "VCNU", [30][0] = "VCN", + [1][1] = "ISPXT", + [2][1] = "ISPIXT", [3][1] = "DCEDWB", [4][1] = "DCEDMC", + [5][1] = "ISPCSISWR", [6][1] = "MP0", [7][1] = "MP1", [8][1] = "MPM", + [9][1] = "ISPPDPWR", + [10][1] = "ISPCSTATWR", + [11][1] = "ISPBYRPWR", + [12][1] = "ISPRGBPWR", + [13][1] = "ISPMCFPWR", + [14][1] = "ISPMWR0", + [15][1] = "ISPYUVPWR", + [16][1] = "ISPMCSCWR", + [17][1] = "ISPGDCWR", + [18][1] = "ISPLMEWR", + [20][1] = "ISPMWR2", [21][1] = "OSSSYS", + [22][1] = "ISPXT1", + [23][1] = "ISPIXT1", [24][1] = "HDP", [25][1] = "LSDMA", [26][1] = "JPEG", [27][1] = "VPE", + [28][1] = "VSCH", [29][1] = "VCNU", [30][1] = "VCN", }; +static const char *mmhub_client_ids_v3_3_1[][2] = { + [0][0] = "VMC", + [4][0] = "DCEDMC", + [6][0] = "MP0", + [7][0] = "MP1", + [8][0] = "MPM", + [24][0] = "HDP", + [25][0] = "LSDMA", + [26][0] = "JPEG0", + [27][0] = "VPE0", + [28][0] = "VSCH", + [29][0] = "VCNU0", + [30][0] = "VCN0", + [32+1][0] = "ISPXT", + [32+2][0] = "ISPIXT", + [32+9][0] = "ISPPDPRD", + [32+10][0] = "ISPCSTATRD", + [32+11][0] = "ISPBYRPRD", + [32+12][0] = "ISPRGBPRD", + [32+13][0] = "ISPMCFPRD", + [32+14][0] = "ISPMCFPRD1", + [32+15][0] = "ISPYUVPRD", + [32+16][0] = "ISPMCSCRD", + [32+17][0] = "ISPGDCRD", + [32+18][0] = "ISPLMERD", + [32+22][0] = "ISPXT1", + [32+23][0] = "ISPIXT1", + [32+26][0] = "JPEG1", + [32+27][0] = "VPE1", + [32+29][0] = "VCNU1", + [32+30][0] = "VCN1", + [3][1] = "DCEDWB", + [4][1] = "DCEDMC", + [6][1] = "MP0", + [7][1] = "MP1", + [8][1] = "MPM", + [21][1] = "OSSSYS", + [24][1] = "HDP", + [25][1] = "LSDMA", + [26][1] = "JPEG0", + [27][1] = "VPE0", + [28][1] = "VSCH", + [29][1] = "VCNU0", + [30][1] = "VCN0", + [32+1][1] = "ISPXT", + [32+2][1] = "ISPIXT", + [32+5][1] = "ISPCSISWR", + [32+9][1] = "ISPPDPWR", + [32+10][1] = "ISPCSTATWR", + [32+11][1] = "ISPBYRPWR", + [32+12][1] = "ISPRGBPWR", + [32+13][1] = "ISPMCFPWR", + [32+14][1] = "ISPMWR0", + [32+15][1] = "ISPYUVPWR", + [32+16][1] = "ISPMCSCWR", + [32+17][1] = "ISPGDCWR", + [32+18][1] = "ISPLMEWR", + [32+19][1] = "ISPMWR1", + [32+20][1] = "ISPMWR2", + [32+22][1] = "ISPXT1", + [32+23][1] = "ISPIXT1", + [32+26][1] = "JPEG1", + [32+27][1] = "VPE1", + [32+29][1] = "VCNU1", + [32+30][1] = "VCN1", +}; + static uint32_t mmhub_v3_3_get_invalidate_req(unsigned int vmid, uint32_t flush_type) { @@ -102,12 +201,16 @@ mmhub_v3_3_print_l2_protection_fault_status(struct amdgpu_device *adev, switch (amdgpu_ip_version(adev, MMHUB_HWIP, 0)) { case IP_VERSION(3, 3, 0): - case IP_VERSION(3, 3, 1): case IP_VERSION(3, 3, 2): mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3) ? mmhub_client_ids_v3_3[cid][rw] : cid == 0x140 ? "UMSCH" : NULL; break; + case IP_VERSION(3, 3, 1): + mmhub_cid = cid < ARRAY_SIZE(mmhub_client_ids_v3_3_1) ? + mmhub_client_ids_v3_3_1[cid][rw] : + cid == 0x140 ? "UMSCH" : NULL; + break; default: mmhub_cid = NULL; break; diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c index b8b06d4c5882..326ecc8d37d2 100644 --- a/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/sdma_v7_0.c @@ -1353,7 +1353,7 @@ static int sdma_v7_0_sw_init(struct amdgpu_ip_block *ip_block) switch (amdgpu_ip_version(adev, SDMA0_HWIP, 0)) { case IP_VERSION(7, 0, 0): case IP_VERSION(7, 0, 1): - if ((adev->sdma.instance[0].fw_version >= 7836028) && !adev->sdma.disable_uq) + if ((adev->sdma.instance[0].fw_version >= 7966358) && !adev->sdma.disable_uq) adev->userq_funcs[AMDGPU_HW_IP_DMA] = &userq_mes_funcs; break; default: diff --git a/drivers/gpu/drm/amd/amdgpu/soc15.c b/drivers/gpu/drm/amd/amdgpu/soc15.c index c457be3a3c56..9e74c9822e62 100644 --- a/drivers/gpu/drm/amd/amdgpu/soc15.c +++ b/drivers/gpu/drm/amd/amdgpu/soc15.c @@ -1218,6 +1218,8 @@ static int soc15_common_early_init(struct amdgpu_ip_block *ip_block) AMD_PG_SUPPORT_JPEG; /*TODO: need a new external_rev_id for GC 9.4.4? */ adev->external_rev_id = adev->rev_id + 0x46; + if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 5, 0)) + adev->external_rev_id = adev->rev_id + 0x50; break; default: /* FIXME: not supported yet */ diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c index 2d91027e2a74..6c5c7c1bf5ed 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.c @@ -2725,7 +2725,7 @@ static void get_queue_checkpoint_info(struct device_queue_manager *dqm, dqm_lock(dqm); mqd_mgr = dqm->mqd_mgrs[mqd_type]; - *mqd_size = mqd_mgr->mqd_size; + *mqd_size = mqd_mgr->mqd_size * NUM_XCC(mqd_mgr->dev->xcc_mask); *ctl_stack_size = 0; if (q->properties.type == KFD_QUEUE_TYPE_COMPUTE && mqd_mgr->get_checkpoint_info) diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_module.c b/drivers/gpu/drm/amd/amdkfd/kfd_module.c index aee2212e52f6..33aa23450b3f 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_module.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_module.c @@ -78,8 +78,8 @@ err_ioctl: static void kfd_exit(void) { kfd_cleanup_processes(); - kfd_debugfs_fini(); kfd_process_destroy_wq(); + kfd_debugfs_fini(); kfd_procfs_shutdown(); kfd_topology_shutdown(); kfd_chardev_exit(); diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c index 97933d2a3803..f2dee320fada 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_mqd_manager_v9.c @@ -373,7 +373,7 @@ static void get_checkpoint_info(struct mqd_manager *mm, void *mqd, u32 *ctl_stac { struct v9_mqd *m = get_mqd(mqd); - *ctl_stack_size = m->cp_hqd_cntl_stack_size; + *ctl_stack_size = m->cp_hqd_cntl_stack_size * NUM_XCC(mm->dev->xcc_mask); } static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, void *ctl_stack_dst) @@ -388,6 +388,24 @@ static void checkpoint_mqd(struct mqd_manager *mm, void *mqd, void *mqd_dst, voi memcpy(ctl_stack_dst, ctl_stack, m->cp_hqd_cntl_stack_size); } +static void checkpoint_mqd_v9_4_3(struct mqd_manager *mm, + void *mqd, + void *mqd_dst, + void *ctl_stack_dst) +{ + struct v9_mqd *m; + int xcc; + uint64_t size = get_mqd(mqd)->cp_mqd_stride_size; + + for (xcc = 0; xcc < NUM_XCC(mm->dev->xcc_mask); xcc++) { + m = get_mqd(mqd + size * xcc); + + checkpoint_mqd(mm, m, + (uint8_t *)mqd_dst + sizeof(*m) * xcc, + (uint8_t *)ctl_stack_dst + m->cp_hqd_cntl_stack_size * xcc); + } +} + static void restore_mqd(struct mqd_manager *mm, void **mqd, struct kfd_mem_obj *mqd_mem_obj, uint64_t *gart_addr, struct queue_properties *qp, @@ -764,13 +782,35 @@ static void restore_mqd_v9_4_3(struct mqd_manager *mm, void **mqd, const void *mqd_src, const void *ctl_stack_src, u32 ctl_stack_size) { - restore_mqd(mm, mqd, mqd_mem_obj, gart_addr, qp, mqd_src, ctl_stack_src, ctl_stack_size); - if (amdgpu_sriov_multi_vf_mode(mm->dev->adev)) { - struct v9_mqd *m; + struct kfd_mem_obj xcc_mqd_mem_obj; + u32 mqd_ctl_stack_size; + struct v9_mqd *m; + u32 num_xcc; + int xcc; - m = (struct v9_mqd *) mqd_mem_obj->cpu_ptr; - m->cp_hqd_pq_doorbell_control |= 1 << - CP_HQD_PQ_DOORBELL_CONTROL__DOORBELL_MODE__SHIFT; + uint64_t offset = mm->mqd_stride(mm, qp); + + mm->dev->dqm->current_logical_xcc_start++; + + num_xcc = NUM_XCC(mm->dev->xcc_mask); + mqd_ctl_stack_size = ctl_stack_size / num_xcc; + + memset(&xcc_mqd_mem_obj, 0x0, sizeof(struct kfd_mem_obj)); + + /* Set the MQD pointer and gart address to XCC0 MQD */ + *mqd = mqd_mem_obj->cpu_ptr; + if (gart_addr) + *gart_addr = mqd_mem_obj->gpu_addr; + + for (xcc = 0; xcc < num_xcc; xcc++) { + get_xcc_mqd(mqd_mem_obj, &xcc_mqd_mem_obj, offset * xcc); + restore_mqd(mm, (void **)&m, + &xcc_mqd_mem_obj, + NULL, + qp, + (uint8_t *)mqd_src + xcc * sizeof(*m), + (uint8_t *)ctl_stack_src + xcc * mqd_ctl_stack_size, + mqd_ctl_stack_size); } } static int destroy_mqd_v9_4_3(struct mqd_manager *mm, void *mqd, @@ -906,7 +946,6 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->free_mqd = kfd_free_mqd_cp; mqd->is_occupied = kfd_is_occupied_cp; mqd->get_checkpoint_info = get_checkpoint_info; - mqd->checkpoint_mqd = checkpoint_mqd; mqd->mqd_size = sizeof(struct v9_mqd); mqd->mqd_stride = mqd_stride_v9; #if defined(CONFIG_DEBUG_FS) @@ -918,16 +957,18 @@ struct mqd_manager *mqd_manager_init_v9(enum KFD_MQD_TYPE type, mqd->init_mqd = init_mqd_v9_4_3; mqd->load_mqd = load_mqd_v9_4_3; mqd->update_mqd = update_mqd_v9_4_3; - mqd->restore_mqd = restore_mqd_v9_4_3; mqd->destroy_mqd = destroy_mqd_v9_4_3; mqd->get_wave_state = get_wave_state_v9_4_3; + mqd->checkpoint_mqd = checkpoint_mqd_v9_4_3; + mqd->restore_mqd = restore_mqd_v9_4_3; } else { mqd->init_mqd = init_mqd; mqd->load_mqd = load_mqd; mqd->update_mqd = update_mqd; - mqd->restore_mqd = restore_mqd; mqd->destroy_mqd = kfd_destroy_mqd_cp; mqd->get_wave_state = get_wave_state; + mqd->checkpoint_mqd = checkpoint_mqd; + mqd->restore_mqd = restore_mqd; } break; case KFD_MQD_TYPE_HIQ: diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c index c643e0ccec52..7fbb5c274ccc 100644 --- a/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c +++ b/drivers/gpu/drm/amd/amdkfd/kfd_process_queue_manager.c @@ -914,7 +914,10 @@ static int criu_checkpoint_queues_device(struct kfd_process_device *pdd, q_data = (struct kfd_criu_queue_priv_data *)q_private_data; - /* data stored in this order: priv_data, mqd, ctl_stack */ + /* + * data stored in this order: + * priv_data, mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]... + */ q_data->mqd_size = mqd_size; q_data->ctl_stack_size = ctl_stack_size; @@ -963,7 +966,7 @@ int kfd_criu_checkpoint_queues(struct kfd_process *p, } static void set_queue_properties_from_criu(struct queue_properties *qp, - struct kfd_criu_queue_priv_data *q_data) + struct kfd_criu_queue_priv_data *q_data, uint32_t num_xcc) { qp->is_interop = false; qp->queue_percent = q_data->q_percent; @@ -976,7 +979,11 @@ static void set_queue_properties_from_criu(struct queue_properties *qp, qp->eop_ring_buffer_size = q_data->eop_ring_buffer_size; qp->ctx_save_restore_area_address = q_data->ctx_save_restore_area_address; qp->ctx_save_restore_area_size = q_data->ctx_save_restore_area_size; - qp->ctl_stack_size = q_data->ctl_stack_size; + if (q_data->type == KFD_QUEUE_TYPE_COMPUTE) + qp->ctl_stack_size = q_data->ctl_stack_size / num_xcc; + else + qp->ctl_stack_size = q_data->ctl_stack_size; + qp->type = q_data->type; qp->format = q_data->format; } @@ -1036,12 +1043,15 @@ int kfd_criu_restore_queue(struct kfd_process *p, goto exit; } - /* data stored in this order: mqd, ctl_stack */ + /* + * data stored in this order: + * mqd[xcc0], mqd[xcc1],..., ctl_stack[xcc0], ctl_stack[xcc1]... + */ mqd = q_extra_data; ctl_stack = mqd + q_data->mqd_size; memset(&qp, 0, sizeof(qp)); - set_queue_properties_from_criu(&qp, q_data); + set_queue_properties_from_criu(&qp, q_data, NUM_XCC(pdd->dev->adev->gfx.xcc_mask)); print_queue_properties(&qp); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2a175fc0399c..cd0e2976e268 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -4756,16 +4756,16 @@ static int get_brightness_range(const struct amdgpu_dm_backlight_caps *caps, return 1; } -/* Rescale from [min..max] to [0..MAX_BACKLIGHT_LEVEL] */ +/* Rescale from [min..max] to [0..AMDGPU_MAX_BL_LEVEL] */ static inline u32 scale_input_to_fw(int min, int max, u64 input) { - return DIV_ROUND_CLOSEST_ULL(input * MAX_BACKLIGHT_LEVEL, max - min); + return DIV_ROUND_CLOSEST_ULL(input * AMDGPU_MAX_BL_LEVEL, max - min); } -/* Rescale from [0..MAX_BACKLIGHT_LEVEL] to [min..max] */ +/* Rescale from [0..AMDGPU_MAX_BL_LEVEL] to [min..max] */ static inline u32 scale_fw_to_input(int min, int max, u64 input) { - return min + DIV_ROUND_CLOSEST_ULL(input * (max - min), MAX_BACKLIGHT_LEVEL); + return min + DIV_ROUND_CLOSEST_ULL(input * (max - min), AMDGPU_MAX_BL_LEVEL); } static void convert_custom_brightness(const struct amdgpu_dm_backlight_caps *caps, diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c index 2551823382f8..010172f930ae 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crtc.c @@ -661,6 +661,15 @@ static int amdgpu_dm_crtc_helper_atomic_check(struct drm_crtc *crtc, return -EINVAL; } + if (!state->legacy_cursor_update && amdgpu_dm_crtc_vrr_active(dm_crtc_state)) { + struct drm_plane_state *primary_state; + + /* Pull in primary plane for correct VRR handling */ + primary_state = drm_atomic_get_plane_state(state, crtc->primary); + if (IS_ERR(primary_state)) + return PTR_ERR(primary_state); + } + /* In some use cases, like reset, no stream is attached */ if (!dm_crtc_state->stream) return 0; diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c index 33b9d36619ff..4071851f9e86 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/clk_mgr.c @@ -158,7 +158,6 @@ struct clk_mgr *dc_clk_mgr_create(struct dc_context *ctx, struct pp_smu_funcs *p return NULL; } dce60_clk_mgr_construct(ctx, clk_mgr); - dce_clk_mgr_construct(ctx, clk_mgr); return &clk_mgr->base; } #endif diff --git a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c index 26feefbb8990..f5ad0a177038 100644 --- a/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c +++ b/drivers/gpu/drm/amd/display/dc/clk_mgr/dce100/dce_clk_mgr.c @@ -245,6 +245,11 @@ int dce_set_clock( pxl_clk_params.target_pixel_clock_100hz = requested_clk_khz * 10; pxl_clk_params.pll_id = CLOCK_SOURCE_ID_DFS; + /* DCE 6.0, DCE 6.4: engine clock is the same as PLL0 */ + if (clk_mgr_base->ctx->dce_version == DCE_VERSION_6_0 || + clk_mgr_base->ctx->dce_version == DCE_VERSION_6_4) + pxl_clk_params.pll_id = CLOCK_SOURCE_ID_PLL0; + if (clk_mgr_dce->dfs_bypass_active) pxl_clk_params.flags.SET_DISPCLK_DFS_BYPASS = true; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc.c b/drivers/gpu/drm/amd/display/dc/core/dc.c index 28aca7017f0f..9ab0ee20ca6f 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc.c @@ -938,17 +938,18 @@ static void dc_destruct(struct dc *dc) if (dc->link_srv) link_destroy_link_service(&dc->link_srv); - if (dc->ctx->gpio_service) - dal_gpio_service_destroy(&dc->ctx->gpio_service); + if (dc->ctx) { + if (dc->ctx->gpio_service) + dal_gpio_service_destroy(&dc->ctx->gpio_service); - if (dc->ctx->created_bios) - dal_bios_parser_destroy(&dc->ctx->dc_bios); + if (dc->ctx->created_bios) + dal_bios_parser_destroy(&dc->ctx->dc_bios); + kfree(dc->ctx->logger); + dc_perf_trace_destroy(&dc->ctx->perf_trace); - kfree(dc->ctx->logger); - dc_perf_trace_destroy(&dc->ctx->perf_trace); - - kfree(dc->ctx); - dc->ctx = NULL; + kfree(dc->ctx); + dc->ctx = NULL; + } kfree(dc->bw_vbios); dc->bw_vbios = NULL; diff --git a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c index 58b59d52dc9d..53b60044653f 100644 --- a/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c +++ b/drivers/gpu/drm/amd/display/dc/resource/dce60/dce60_resource.c @@ -373,7 +373,7 @@ static const struct resource_caps res_cap = { .num_timing_generator = 6, .num_audio = 6, .num_stream_encoder = 6, - .num_pll = 2, + .num_pll = 3, .num_ddc = 6, }; @@ -389,7 +389,7 @@ static const struct resource_caps res_cap_64 = { .num_timing_generator = 2, .num_audio = 2, .num_stream_encoder = 2, - .num_pll = 2, + .num_pll = 3, .num_ddc = 2, }; @@ -973,21 +973,24 @@ static bool dce60_construct( if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) { pool->base.dp_clock_source = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + /* DCE 6.0 and 6.4: PLL0 can only be used with DP. Don't initialize it here. */ pool->base.clock_sources[0] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], false); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); pool->base.clock_sources[1] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); pool->base.clk_src_count = 2; } else { pool->base.dp_clock_source = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true); pool->base.clock_sources[0] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); - pool->base.clk_src_count = 1; + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); + pool->base.clk_src_count = 2; } if (pool->base.dp_clock_source == NULL) { @@ -1365,21 +1368,24 @@ static bool dce64_construct( if (bp->fw_info_valid && bp->fw_info.external_clock_source_frequency_for_dp != 0) { pool->base.dp_clock_source = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_EXTERNAL, NULL, true); + /* DCE 6.0 and 6.4: PLL0 can only be used with DP. Don't initialize it here. */ pool->base.clock_sources[0] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], false); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); pool->base.clock_sources[1] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); pool->base.clk_src_count = 2; } else { pool->base.dp_clock_source = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[0], true); + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL0, &clk_src_regs[0], true); pool->base.clock_sources[0] = - dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[1], false); - pool->base.clk_src_count = 1; + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL1, &clk_src_regs[1], false); + pool->base.clock_sources[1] = + dce60_clock_source_create(ctx, bp, CLOCK_SOURCE_ID_PLL2, &clk_src_regs[2], false); + pool->base.clk_src_count = 2; } if (pool->base.dp_clock_source == NULL) { diff --git a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c index 756afe78a6e5..b47cb4a5f488 100644 --- a/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c +++ b/drivers/gpu/drm/amd/pm/swsmu/amdgpu_smu.c @@ -77,6 +77,9 @@ static void smu_power_profile_mode_get(struct smu_context *smu, static void smu_power_profile_mode_put(struct smu_context *smu, enum PP_SMC_POWER_PROFILE profile_mode); static enum smu_clk_type smu_convert_to_smuclk(enum pp_clock_type type); +static int smu_od_edit_dpm_table(void *handle, + enum PP_OD_DPM_TABLE_COMMAND type, + long *input, uint32_t size); static int smu_sys_get_pp_feature_mask(void *handle, char *buf) @@ -2195,6 +2198,7 @@ static int smu_resume(struct amdgpu_ip_block *ip_block) int ret; struct amdgpu_device *adev = ip_block->adev; struct smu_context *smu = adev->powerplay.pp_handle; + struct smu_dpm_context *smu_dpm_ctx = &(smu->smu_dpm); if (amdgpu_sriov_multi_vf_mode(adev)) return 0; @@ -2226,6 +2230,18 @@ static int smu_resume(struct amdgpu_ip_block *ip_block) adev->pm.dpm_enabled = true; + if (smu->current_power_limit) { + ret = smu_set_power_limit(smu, smu->current_power_limit); + if (ret && ret != -EOPNOTSUPP) + return ret; + } + + if (smu_dpm_ctx->dpm_level == AMD_DPM_FORCED_LEVEL_MANUAL) { + ret = smu_od_edit_dpm_table(smu, PP_OD_COMMIT_DPM_TABLE, NULL, 0); + if (ret) + return ret; + } + dev_info(adev->dev, "SMU is resumed successfully!\n"); return 0; diff --git a/drivers/gpu/drm/i915/display/intel_cx0_phy.c b/drivers/gpu/drm/i915/display/intel_cx0_phy.c index ed8e640b96b0..801235a5bc0a 100644 --- a/drivers/gpu/drm/i915/display/intel_cx0_phy.c +++ b/drivers/gpu/drm/i915/display/intel_cx0_phy.c @@ -3239,14 +3239,22 @@ void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder, const struct intel_crtc_state *crtc_state) { struct intel_display *display = to_intel_display(encoder); - u8 owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder); - bool enable = intel_alpm_is_alpm_aux_less(enc_to_intel_dp(encoder), - crtc_state); + intel_wakeref_t wakeref; int i; + u8 owned_lane_mask; - if (DISPLAY_VER(display) < 20) + if (DISPLAY_VER(display) < 20 || + !intel_alpm_is_alpm_aux_less(enc_to_intel_dp(encoder), crtc_state)) return; + owned_lane_mask = intel_cx0_get_owned_lane_mask(encoder); + + wakeref = intel_cx0_phy_transaction_begin(encoder); + + if (intel_encoder_is_c10phy(encoder)) + intel_cx0_rmw(encoder, owned_lane_mask, PHY_C10_VDR_CONTROL(1), 0, + C10_VDR_CTRL_MSGBUS_ACCESS, MB_WRITE_COMMITTED); + for (i = 0; i < 4; i++) { int tx = i % 2 + 1; u8 lane_mask = i < 2 ? INTEL_CX0_LANE0 : INTEL_CX0_LANE1; @@ -3256,9 +3264,10 @@ void intel_lnl_mac_transmit_lfps(struct intel_encoder *encoder, intel_cx0_rmw(encoder, lane_mask, PHY_CMN1_CONTROL(tx, 0), CONTROL0_MAC_TRANSMIT_LFPS, - enable ? CONTROL0_MAC_TRANSMIT_LFPS : 0, - MB_WRITE_COMMITTED); + CONTROL0_MAC_TRANSMIT_LFPS, MB_WRITE_COMMITTED); } + + intel_cx0_phy_transaction_end(encoder, wakeref); } static u8 cx0_power_control_disable_val(struct intel_encoder *encoder) diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig index 2bb2bc052120..714d5702dfd7 100644 --- a/drivers/gpu/drm/xe/Kconfig +++ b/drivers/gpu/drm/xe/Kconfig @@ -5,6 +5,7 @@ config DRM_XE depends on KUNIT || !KUNIT depends on INTEL_VSEC || !INTEL_VSEC depends on X86_PLATFORM_DEVICES || !(X86 && ACPI) + depends on PAGE_SIZE_4KB || COMPILE_TEST || BROKEN select INTERVAL_TREE # we need shmfs for the swappable backing store, and in particular # the shmem_readpage() which depends upon tmpfs diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 5bd2f7d7b4ea..6ece4defa9df 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -802,10 +802,6 @@ int xe_device_probe(struct xe_device *xe) return err; } - err = xe_devcoredump_init(xe); - if (err) - return err; - /* * From here on, if a step fails, make sure a Driver-FLR is triggereed */ @@ -870,6 +866,10 @@ int xe_device_probe(struct xe_device *xe) XE_WA(xe->tiles->media_gt, 15015404425_disable)) XE_DEVICE_WA_DISABLE(xe, 15015404425); + err = xe_devcoredump_init(xe); + if (err) + return err; + xe_nvm_init(xe); err = xe_heci_gsc_init(xe); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c index 35489fa81825..bdbd15f3afe3 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.c @@ -16,6 +16,7 @@ #include "xe_gt_sriov_pf_migration.h" #include "xe_gt_sriov_pf_service.h" #include "xe_gt_sriov_printk.h" +#include "xe_guc_submit.h" #include "xe_mmio.h" #include "xe_pm.h" @@ -47,9 +48,16 @@ static int pf_alloc_metadata(struct xe_gt *gt) static void pf_init_workers(struct xe_gt *gt) { + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); INIT_WORK(>->sriov.pf.workers.restart, pf_worker_restart_func); } +static void pf_fini_workers(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + disable_work_sync(>->sriov.pf.workers.restart); +} + /** * xe_gt_sriov_pf_init_early - Prepare SR-IOV PF data structures on PF. * @gt: the &xe_gt to initialize @@ -79,6 +87,21 @@ int xe_gt_sriov_pf_init_early(struct xe_gt *gt) return 0; } +static void pf_fini_action(void *arg) +{ + struct xe_gt *gt = arg; + + pf_fini_workers(gt); +} + +static int pf_init_late(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + xe_gt_assert(gt, IS_SRIOV_PF(xe)); + return devm_add_action_or_reset(xe->drm.dev, pf_fini_action, gt); +} + /** * xe_gt_sriov_pf_init - Prepare SR-IOV PF data structures on PF. * @gt: the &xe_gt to initialize @@ -95,7 +118,15 @@ int xe_gt_sriov_pf_init(struct xe_gt *gt) if (err) return err; - return xe_gt_sriov_pf_migration_init(gt); + err = xe_gt_sriov_pf_migration_init(gt); + if (err) + return err; + + err = pf_init_late(gt); + if (err) + return err; + + return 0; } static bool pf_needs_enable_ggtt_guest_update(struct xe_device *xe) @@ -230,3 +261,27 @@ void xe_gt_sriov_pf_restart(struct xe_gt *gt) { pf_queue_restart(gt); } + +static void pf_flush_restart(struct xe_gt *gt) +{ + xe_gt_assert(gt, IS_SRIOV_PF(gt_to_xe(gt))); + flush_work(>->sriov.pf.workers.restart); +} + +/** + * xe_gt_sriov_pf_wait_ready() - Wait until per-GT PF SR-IOV support is ready. + * @gt: the &xe_gt + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt) +{ + /* don't wait if there is another ongoing reset */ + if (xe_guc_read_stopped(>->uc.guc)) + return -EBUSY; + + pf_flush_restart(gt); + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h index e2b2ff8132dc..e7fde3f9937a 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf.h +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf.h @@ -11,6 +11,7 @@ struct xe_gt; #ifdef CONFIG_PCI_IOV int xe_gt_sriov_pf_init_early(struct xe_gt *gt); int xe_gt_sriov_pf_init(struct xe_gt *gt); +int xe_gt_sriov_pf_wait_ready(struct xe_gt *gt); void xe_gt_sriov_pf_init_hw(struct xe_gt *gt); void xe_gt_sriov_pf_sanitize_hw(struct xe_gt *gt, unsigned int vfid); void xe_gt_sriov_pf_stop_prepare(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c index bf679b21f485..3ed245e04d0c 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_debugfs.c @@ -22,6 +22,7 @@ #include "xe_gt_sriov_pf_policy.h" #include "xe_gt_sriov_pf_service.h" #include "xe_pm.h" +#include "xe_sriov_pf.h" /* * /sys/kernel/debug/dri/0/ @@ -205,7 +206,8 @@ static int CONFIG##_set(void *data, u64 val) \ return -EOVERFLOW; \ \ xe_pm_runtime_get(xe); \ - err = xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ + err = xe_sriov_pf_wait_ready(xe) ?: \ + xe_gt_sriov_pf_config_set_##CONFIG(gt, vfid, val); \ xe_pm_runtime_put(xe); \ \ return err; \ diff --git a/drivers/gpu/drm/xe/xe_guc_capture.c b/drivers/gpu/drm/xe/xe_guc_capture.c index 859a3ba91be5..243dad3e2418 100644 --- a/drivers/gpu/drm/xe/xe_guc_capture.c +++ b/drivers/gpu/drm/xe/xe_guc_capture.c @@ -1817,6 +1817,12 @@ void xe_engine_snapshot_print(struct xe_hw_engine_snapshot *snapshot, struct drm str_yes_no(snapshot->kernel_reserved)); for (type = GUC_STATE_CAPTURE_TYPE_GLOBAL; type < GUC_STATE_CAPTURE_TYPE_MAX; type++) { + /* + * FIXME: During devcoredump print we should avoid accessing the + * driver pointers for gt or engine. Printing should be done only + * using the snapshot captured. Here we are accessing the gt + * pointer. It should be fixed. + */ list = xe_guc_capture_get_reg_desc_list(gt, GUC_CAPTURE_LIST_INDEX_PF, type, capture_class, false); snapshot_print_by_list_order(snapshot, p, type, list); diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index 107ffe87808c..d9391bd08194 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -27,6 +27,8 @@ #define DEFAULT_PROBE_DISPLAY true #define DEFAULT_VRAM_BAR_SIZE 0 #define DEFAULT_FORCE_PROBE CONFIG_DRM_XE_FORCE_PROBE +#define DEFAULT_MAX_VFS ~0 +#define DEFAULT_MAX_VFS_STR "unlimited" #define DEFAULT_WEDGED_MODE 1 #define DEFAULT_SVM_NOTIFIER_SIZE 512 @@ -34,6 +36,9 @@ struct xe_modparam xe_modparam = { .probe_display = DEFAULT_PROBE_DISPLAY, .guc_log_level = DEFAULT_GUC_LOG_LEVEL, .force_probe = DEFAULT_FORCE_PROBE, +#ifdef CONFIG_PCI_IOV + .max_vfs = DEFAULT_MAX_VFS, +#endif .wedged_mode = DEFAULT_WEDGED_MODE, .svm_notifier_size = DEFAULT_SVM_NOTIFIER_SIZE, /* the rest are 0 by default */ @@ -79,7 +84,8 @@ MODULE_PARM_DESC(force_probe, module_param_named(max_vfs, xe_modparam.max_vfs, uint, 0400); MODULE_PARM_DESC(max_vfs, "Limit number of Virtual Functions (VFs) that could be managed. " - "(0 = no VFs [default]; N = allow up to N VFs)"); + "(0=no VFs; N=allow up to N VFs " + "[default=" DEFAULT_MAX_VFS_STR "])"); #endif module_param_named_unsafe(wedged_mode, xe_modparam.wedged_mode, int, 0600); diff --git a/drivers/gpu/drm/xe/xe_pci_sriov.c b/drivers/gpu/drm/xe/xe_pci_sriov.c index 8813efdcafbb..447a7867eecb 100644 --- a/drivers/gpu/drm/xe/xe_pci_sriov.c +++ b/drivers/gpu/drm/xe/xe_pci_sriov.c @@ -12,6 +12,7 @@ #include "xe_pci_sriov.h" #include "xe_pm.h" #include "xe_sriov.h" +#include "xe_sriov_pf.h" #include "xe_sriov_pf_helpers.h" #include "xe_sriov_printk.h" @@ -138,6 +139,10 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) xe_assert(xe, num_vfs <= total_vfs); xe_sriov_dbg(xe, "enabling %u VF%s\n", num_vfs, str_plural(num_vfs)); + err = xe_sriov_pf_wait_ready(xe); + if (err) + goto out; + /* * We must hold additional reference to the runtime PM to keep PF in D0 * during VFs lifetime, as our VFs do not implement the PM capability. @@ -169,7 +174,7 @@ static int pf_enable_vfs(struct xe_device *xe, int num_vfs) failed: pf_unprovision_vfs(xe, num_vfs); xe_pm_runtime_put(xe); - +out: xe_sriov_notice(xe, "Failed to enable %u VF%s (%pe)\n", num_vfs, str_plural(num_vfs), ERR_PTR(err)); return err; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.c b/drivers/gpu/drm/xe/xe_sriov_pf.c index afbdd894bd6e..27ddf3cc80e9 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf.c @@ -9,6 +9,7 @@ #include "xe_assert.h" #include "xe_device.h" +#include "xe_gt_sriov_pf.h" #include "xe_module.h" #include "xe_sriov.h" #include "xe_sriov_pf.h" @@ -103,6 +104,32 @@ int xe_sriov_pf_init_early(struct xe_device *xe) } /** + * xe_sriov_pf_wait_ready() - Wait until PF is ready to operate. + * @xe: the &xe_device to test + * + * This function can only be called on PF. + * + * Return: 0 on success or a negative error code on failure. + */ +int xe_sriov_pf_wait_ready(struct xe_device *xe) +{ + struct xe_gt *gt; + unsigned int id; + int err; + + if (xe_device_wedged(xe)) + return -ECANCELED; + + for_each_gt(gt, xe, id) { + err = xe_gt_sriov_pf_wait_ready(gt); + if (err) + return err; + } + + return 0; +} + +/** * xe_sriov_pf_print_vfs_summary - Print SR-IOV PF information. * @xe: the &xe_device to print info from * @p: the &drm_printer diff --git a/drivers/gpu/drm/xe/xe_sriov_pf.h b/drivers/gpu/drm/xe/xe_sriov_pf.h index c392c3fcf085..e3b34f8f5e04 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf.h @@ -15,6 +15,7 @@ struct xe_device; #ifdef CONFIG_PCI_IOV bool xe_sriov_pf_readiness(struct xe_device *xe); int xe_sriov_pf_init_early(struct xe_device *xe); +int xe_sriov_pf_wait_ready(struct xe_device *xe); void xe_sriov_pf_debugfs_register(struct xe_device *xe, struct dentry *root); void xe_sriov_pf_print_vfs_summary(struct xe_device *xe, struct drm_printer *p); #else diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig index 4fef4797b110..02432d4a5ccd 100644 --- a/drivers/mailbox/Kconfig +++ b/drivers/mailbox/Kconfig @@ -36,6 +36,15 @@ config ARM_MHU_V3 that provides different means of transports: supported extensions will be discovered and possibly managed at probe-time. +config AST2700_MBOX + tristate "ASPEED AST2700 IPC driver" + depends on ARCH_ASPEED || COMPILE_TEST + help + Mailbox driver implementation for ASPEED AST27XX SoCs. This driver + can be used to send message between different processors in SoC. + The driver provides mailbox support for sending interrupts to the + clients. Say Y here if you want to build this driver. + config CV1800_MBOX tristate "cv1800 mailbox" depends on ARCH_SOPHGO || COMPILE_TEST @@ -350,4 +359,14 @@ config CIX_MBOX is unidirectional. Say Y here if you want to use the CIX Mailbox support. +config BCM74110_MAILBOX + tristate "Brcmstb BCM74110 Mailbox" + depends on ARCH_BRCMSTB || COMPILE_TEST + default ARCH_BRCMSTB + help + Broadcom STB mailbox driver present starting with brcmstb bcm74110 + SoCs. The mailbox is a communication channel between the host + processor and coprocessor that handles various power management task + and more. + endif diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index 786a46587ba1..98a68f838486 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -11,6 +11,8 @@ obj-$(CONFIG_ARM_MHU_V2) += arm_mhuv2.o obj-$(CONFIG_ARM_MHU_V3) += arm_mhuv3.o +obj-$(CONFIG_AST2700_MBOX) += ast2700-mailbox.o + obj-$(CONFIG_CV1800_MBOX) += cv1800-mailbox.o obj-$(CONFIG_EXYNOS_MBOX) += exynos-mailbox.o @@ -74,3 +76,5 @@ obj-$(CONFIG_QCOM_IPCC) += qcom-ipcc.o obj-$(CONFIG_THEAD_TH1520_MBOX) += mailbox-th1520.o obj-$(CONFIG_CIX_MBOX) += cix-mailbox.o + +obj-$(CONFIG_BCM74110_MAILBOX) += bcm74110-mailbox.o diff --git a/drivers/mailbox/ast2700-mailbox.c b/drivers/mailbox/ast2700-mailbox.c new file mode 100644 index 000000000000..83c6afe5411f --- /dev/null +++ b/drivers/mailbox/ast2700-mailbox.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright Aspeed Technology Inc. (C) 2025. All rights reserved + */ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/mailbox_controller.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/slab.h> + +/* Each bit in the register represents an IPC ID */ +#define IPCR_TX_TRIG 0x00 +#define IPCR_ENABLE 0x04 +#define IPCR_STATUS 0x08 +#define RX_IRQ(n) BIT(n) +#define RX_IRQ_MASK 0xf +#define IPCR_DATA 0x10 + +struct ast2700_mbox_data { + u8 num_chans; + u8 msg_size; +}; + +struct ast2700_mbox { + struct mbox_controller mbox; + u8 msg_size; + void __iomem *tx_regs; + void __iomem *rx_regs; + spinlock_t lock; +}; + +static inline int ch_num(struct mbox_chan *chan) +{ + return chan - chan->mbox->chans; +} + +static inline bool ast2700_mbox_tx_done(struct ast2700_mbox *mb, int idx) +{ + return !(readl(mb->tx_regs + IPCR_STATUS) & BIT(idx)); +} + +static irqreturn_t ast2700_mbox_irq(int irq, void *p) +{ + struct ast2700_mbox *mb = p; + void __iomem *data_reg; + int num_words = mb->msg_size / sizeof(u32); + u32 *word_data; + u32 status; + int n, i; + + /* Only examine channels that are currently enabled. */ + status = readl(mb->rx_regs + IPCR_ENABLE) & + readl(mb->rx_regs + IPCR_STATUS); + + if (!(status & RX_IRQ_MASK)) + return IRQ_NONE; + + for (n = 0; n < mb->mbox.num_chans; ++n) { + struct mbox_chan *chan = &mb->mbox.chans[n]; + + if (!(status & RX_IRQ(n))) + continue; + + data_reg = mb->rx_regs + IPCR_DATA + mb->msg_size * n; + word_data = chan->con_priv; + /* Read the message data */ + for (i = 0; i < num_words; i++) + word_data[i] = readl(data_reg + i * sizeof(u32)); + + mbox_chan_received_data(chan, chan->con_priv); + + /* The IRQ can be cleared only once the FIFO is empty. */ + writel(RX_IRQ(n), mb->rx_regs + IPCR_STATUS); + } + + return IRQ_HANDLED; +} + +static int ast2700_mbox_send_data(struct mbox_chan *chan, void *data) +{ + struct ast2700_mbox *mb = dev_get_drvdata(chan->mbox->dev); + int idx = ch_num(chan); + void __iomem *data_reg = mb->tx_regs + IPCR_DATA + mb->msg_size * idx; + u32 *word_data = data; + int num_words = mb->msg_size / sizeof(u32); + int i; + + if (!(readl(mb->tx_regs + IPCR_ENABLE) & BIT(idx))) { + dev_warn(mb->mbox.dev, "%s: Ch-%d not enabled yet\n", __func__, idx); + return -ENODEV; + } + + if (!(ast2700_mbox_tx_done(mb, idx))) { + dev_warn(mb->mbox.dev, "%s: Ch-%d last data has not finished\n", __func__, idx); + return -EBUSY; + } + + /* Write the message data */ + for (i = 0 ; i < num_words; i++) + writel(word_data[i], data_reg + i * sizeof(u32)); + + writel(BIT(idx), mb->tx_regs + IPCR_TX_TRIG); + dev_dbg(mb->mbox.dev, "%s: Ch-%d sent\n", __func__, idx); + + return 0; +} + +static int ast2700_mbox_startup(struct mbox_chan *chan) +{ + struct ast2700_mbox *mb = dev_get_drvdata(chan->mbox->dev); + int idx = ch_num(chan); + void __iomem *reg = mb->rx_regs + IPCR_ENABLE; + unsigned long flags; + + spin_lock_irqsave(&mb->lock, flags); + writel(readl(reg) | BIT(idx), reg); + spin_unlock_irqrestore(&mb->lock, flags); + + return 0; +} + +static void ast2700_mbox_shutdown(struct mbox_chan *chan) +{ + struct ast2700_mbox *mb = dev_get_drvdata(chan->mbox->dev); + int idx = ch_num(chan); + void __iomem *reg = mb->rx_regs + IPCR_ENABLE; + unsigned long flags; + + spin_lock_irqsave(&mb->lock, flags); + writel(readl(reg) & ~BIT(idx), reg); + spin_unlock_irqrestore(&mb->lock, flags); +} + +static bool ast2700_mbox_last_tx_done(struct mbox_chan *chan) +{ + struct ast2700_mbox *mb = dev_get_drvdata(chan->mbox->dev); + int idx = ch_num(chan); + + return ast2700_mbox_tx_done(mb, idx); +} + +static const struct mbox_chan_ops ast2700_mbox_chan_ops = { + .send_data = ast2700_mbox_send_data, + .startup = ast2700_mbox_startup, + .shutdown = ast2700_mbox_shutdown, + .last_tx_done = ast2700_mbox_last_tx_done, +}; + +static int ast2700_mbox_probe(struct platform_device *pdev) +{ + struct ast2700_mbox *mb; + const struct ast2700_mbox_data *dev_data; + struct device *dev = &pdev->dev; + int irq, ret; + + if (!pdev->dev.of_node) + return -ENODEV; + + dev_data = device_get_match_data(&pdev->dev); + + mb = devm_kzalloc(dev, sizeof(*mb), GFP_KERNEL); + if (!mb) + return -ENOMEM; + + mb->mbox.chans = devm_kcalloc(&pdev->dev, dev_data->num_chans, + sizeof(*mb->mbox.chans), GFP_KERNEL); + if (!mb->mbox.chans) + return -ENOMEM; + + /* con_priv of each channel is used to store the message received */ + for (int i = 0; i < dev_data->num_chans; i++) { + mb->mbox.chans[i].con_priv = devm_kcalloc(dev, dev_data->msg_size, + sizeof(u8), GFP_KERNEL); + if (!mb->mbox.chans[i].con_priv) + return -ENOMEM; + } + + platform_set_drvdata(pdev, mb); + + mb->tx_regs = devm_platform_ioremap_resource_byname(pdev, "tx"); + if (IS_ERR(mb->tx_regs)) + return PTR_ERR(mb->tx_regs); + + mb->rx_regs = devm_platform_ioremap_resource_byname(pdev, "rx"); + if (IS_ERR(mb->rx_regs)) + return PTR_ERR(mb->rx_regs); + + mb->msg_size = dev_data->msg_size; + mb->mbox.dev = dev; + mb->mbox.num_chans = dev_data->num_chans; + mb->mbox.ops = &ast2700_mbox_chan_ops; + mb->mbox.txdone_irq = false; + mb->mbox.txdone_poll = true; + mb->mbox.txpoll_period = 5; + spin_lock_init(&mb->lock); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(dev, irq, ast2700_mbox_irq, 0, dev_name(dev), mb); + if (ret) + return ret; + + return devm_mbox_controller_register(dev, &mb->mbox); +} + +static const struct ast2700_mbox_data ast2700_dev_data = { + .num_chans = 4, + .msg_size = 0x20, +}; + +static const struct of_device_id ast2700_mbox_of_match[] = { + { .compatible = "aspeed,ast2700-mailbox", .data = &ast2700_dev_data }, + {} +}; +MODULE_DEVICE_TABLE(of, ast2700_mbox_of_match); + +static struct platform_driver ast2700_mbox_driver = { + .driver = { + .name = "ast2700-mailbox", + .of_match_table = ast2700_mbox_of_match, + }, + .probe = ast2700_mbox_probe, +}; +module_platform_driver(ast2700_mbox_driver); + +MODULE_AUTHOR("Jammy Huang <jammy_huang@aspeedtech.com>"); +MODULE_DESCRIPTION("ASPEED AST2700 IPC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mailbox/bcm74110-mailbox.c b/drivers/mailbox/bcm74110-mailbox.c new file mode 100644 index 000000000000..2e7e86f3e6a4 --- /dev/null +++ b/drivers/mailbox/bcm74110-mailbox.c @@ -0,0 +1,656 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Broadcom BCM74110 Mailbox Driver + * + * Copyright (c) 2025 Broadcom + */ +#include <linux/list.h> +#include <linux/types.h> +#include <linux/workqueue.h> +#include <linux/io-64-nonatomic-hi-lo.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/of.h> +#include <linux/delay.h> +#include <linux/mailbox_controller.h> +#include <linux/bitfield.h> +#include <linux/slab.h> + +#define BCM_MBOX_BASE(sel) ((sel) * 0x40) +#define BCM_MBOX_IRQ_BASE(sel) (((sel) * 0x20) + 0x800) + +#define BCM_MBOX_CFGA 0x0 +#define BCM_MBOX_CFGB 0x4 +#define BCM_MBOX_CFGC 0x8 +#define BCM_MBOX_CFGD 0xc +#define BCM_MBOX_CTRL 0x10 +#define BCM_MBOX_CTRL_EN BIT(0) +#define BCM_MBOX_CTRL_CLR BIT(1) +#define BCM_MBOX_STATUS0 0x14 +#define BCM_MBOX_STATUS0_NOT_EMPTY BIT(28) +#define BCM_MBOX_STATUS0_FULL BIT(29) +#define BCM_MBOX_STATUS1 0x18 +#define BCM_MBOX_STATUS2 0x1c +#define BCM_MBOX_WDATA 0x20 +#define BCM_MBOX_RDATA 0x28 + +#define BCM_MBOX_IRQ_STATUS 0x0 +#define BCM_MBOX_IRQ_SET 0x4 +#define BCM_MBOX_IRQ_CLEAR 0x8 +#define BCM_MBOX_IRQ_MASK_STATUS 0xc +#define BCM_MBOX_IRQ_MASK_SET 0x10 +#define BCM_MBOX_IRQ_MASK_CLEAR 0x14 +#define BCM_MBOX_IRQ_TIMEOUT BIT(0) +#define BCM_MBOX_IRQ_NOT_EMPTY BIT(1) +#define BCM_MBOX_IRQ_FULL BIT(2) +#define BCM_MBOX_IRQ_LOW_WM BIT(3) +#define BCM_MBOX_IRQ_HIGH_WM BIT(4) + +#define BCM_LINK_CODE0 0xbe0 +#define BCM_LINK_CODE1 0xbe1 +#define BCM_LINK_CODE2 0xbe2 + +enum { + BCM_MSG_FUNC_LINK_START = 0, + BCM_MSG_FUNC_LINK_STOP, + BCM_MSG_FUNC_SHMEM_TX, + BCM_MSG_FUNC_SHMEM_RX, + BCM_MSG_FUNC_SHMEM_STOP, + BCM_MSG_FUNC_MAX, +}; + +enum { + BCM_MSG_SVC_INIT = 0, + BCM_MSG_SVC_PMC, + BCM_MSG_SVC_SCMI, + BCM_MSG_SVC_DPFE, + BCM_MSG_SVC_MAX, +}; + +struct bcm74110_mbox_msg { + struct list_head list_entry; +#define BCM_MSG_VERSION_MASK GENMASK(31, 29) +#define BCM_MSG_VERSION 0x1 +#define BCM_MSG_REQ_MASK BIT(28) +#define BCM_MSG_RPLY_MASK BIT(27) +#define BCM_MSG_SVC_MASK GENMASK(26, 24) +#define BCM_MSG_FUNC_MASK GENMASK(23, 16) +#define BCM_MSG_LENGTH_MASK GENMASK(15, 4) +#define BCM_MSG_SLOT_MASK GENMASK(3, 0) + +#define BCM_MSG_SET_FIELD(hdr, field, val) \ + do { \ + hdr &= ~BCM_MSG_##field##_MASK; \ + hdr |= FIELD_PREP(BCM_MSG_##field##_MASK, val); \ + } while (0) + +#define BCM_MSG_GET_FIELD(hdr, field) \ + FIELD_GET(BCM_MSG_##field##_MASK, hdr) + u32 msg; +}; + +struct bcm74110_mbox_chan { + struct bcm74110_mbox *mbox; + bool en; + int slot; + int type; +}; + +struct bcm74110_mbox { + struct platform_device *pdev; + void __iomem *base; + + int tx_chan; + int rx_chan; + int rx_irq; + struct list_head rx_svc_init_list; + spinlock_t rx_svc_list_lock; + + struct mbox_controller controller; + struct bcm74110_mbox_chan *mbox_chan; +}; + +#define BCM74110_OFFSET_IO_WRITEL_MACRO(name, offset_base) \ +static void bcm74110_##name##_writel(struct bcm74110_mbox *mbox,\ + u32 val, u32 off) \ +{ \ + writel_relaxed(val, mbox->base + offset_base + off); \ +} +BCM74110_OFFSET_IO_WRITEL_MACRO(tx, BCM_MBOX_BASE(mbox->tx_chan)); +BCM74110_OFFSET_IO_WRITEL_MACRO(irq, BCM_MBOX_IRQ_BASE(mbox->rx_chan)); + +#define BCM74110_OFFSET_IO_READL_MACRO(name, offset_base) \ +static u32 bcm74110_##name##_readl(struct bcm74110_mbox *mbox, \ + u32 off) \ +{ \ + return readl_relaxed(mbox->base + offset_base + off); \ +} +BCM74110_OFFSET_IO_READL_MACRO(tx, BCM_MBOX_BASE(mbox->tx_chan)); +BCM74110_OFFSET_IO_READL_MACRO(rx, BCM_MBOX_BASE(mbox->rx_chan)); +BCM74110_OFFSET_IO_READL_MACRO(irq, BCM_MBOX_IRQ_BASE(mbox->rx_chan)); + +static inline struct bcm74110_mbox *bcm74110_mbox_from_cntrl( + struct mbox_controller *cntrl) +{ + return container_of(cntrl, struct bcm74110_mbox, controller); +} + +static void bcm74110_rx_push_init_msg(struct bcm74110_mbox *mbox, u32 val) +{ + struct bcm74110_mbox_msg *msg; + + msg = kzalloc(sizeof(*msg), GFP_ATOMIC); + if (!msg) + return; + + INIT_LIST_HEAD(&msg->list_entry); + msg->msg = val; + + spin_lock(&mbox->rx_svc_list_lock); + list_add_tail(&msg->list_entry, &mbox->rx_svc_init_list); + spin_unlock(&mbox->rx_svc_list_lock); +} + +static void bcm74110_rx_process_msg(struct bcm74110_mbox *mbox) +{ + struct device *dev = &mbox->pdev->dev; + struct bcm74110_mbox_chan *chan_priv; + struct mbox_chan *chan; + u32 msg, status; + int type; + + do { + msg = bcm74110_rx_readl(mbox, BCM_MBOX_RDATA); + status = bcm74110_rx_readl(mbox, BCM_MBOX_STATUS0); + + dev_dbg(dev, "rx: [{req=%lu|rply=%lu|srv=%lu|fn=%lu|length=%lu|slot=%lu]\n", + BCM_MSG_GET_FIELD(msg, REQ), BCM_MSG_GET_FIELD(msg, RPLY), + BCM_MSG_GET_FIELD(msg, SVC), BCM_MSG_GET_FIELD(msg, FUNC), + BCM_MSG_GET_FIELD(msg, LENGTH), BCM_MSG_GET_FIELD(msg, SLOT)); + + type = BCM_MSG_GET_FIELD(msg, SVC); + switch (type) { + case BCM_MSG_SVC_INIT: + bcm74110_rx_push_init_msg(mbox, msg); + break; + case BCM_MSG_SVC_PMC: + case BCM_MSG_SVC_SCMI: + case BCM_MSG_SVC_DPFE: + chan = &mbox->controller.chans[type]; + chan_priv = chan->con_priv; + if (chan_priv->en) + mbox_chan_received_data(chan, NULL); + else + dev_warn(dev, "Channel not enabled\n"); + break; + default: + dev_warn(dev, "Unsupported msg received\n"); + } + } while (status & BCM_MBOX_STATUS0_NOT_EMPTY); +} + +static irqreturn_t bcm74110_mbox_isr(int irq, void *data) +{ + struct bcm74110_mbox *mbox = data; + u32 status; + + status = bcm74110_irq_readl(mbox, BCM_MBOX_IRQ_STATUS); + + bcm74110_irq_writel(mbox, 0xffffffff, BCM_MBOX_IRQ_CLEAR); + + if (status & BCM_MBOX_IRQ_NOT_EMPTY) + bcm74110_rx_process_msg(mbox); + else + dev_warn(&mbox->pdev->dev, "Spurious interrupt\n"); + + return IRQ_HANDLED; +} + +static void bcm74110_mbox_mask_and_clear(struct bcm74110_mbox *mbox) +{ + bcm74110_irq_writel(mbox, 0xffffffff, BCM_MBOX_IRQ_MASK_SET); + bcm74110_irq_writel(mbox, 0xffffffff, BCM_MBOX_IRQ_CLEAR); +} + +static int bcm74110_rx_pop_init_msg(struct bcm74110_mbox *mbox, u32 func_type, + u32 *val) +{ + struct bcm74110_mbox_msg *msg, *msg_tmp; + unsigned long flags; + bool found = false; + + spin_lock_irqsave(&mbox->rx_svc_list_lock, flags); + list_for_each_entry_safe(msg, msg_tmp, &mbox->rx_svc_init_list, + list_entry) { + if (BCM_MSG_GET_FIELD(msg->msg, FUNC) == func_type) { + list_del(&msg->list_entry); + found = true; + break; + } + } + spin_unlock_irqrestore(&mbox->rx_svc_list_lock, flags); + + if (!found) + return -EINVAL; + + *val = msg->msg; + kfree(msg); + + return 0; +} + +static void bcm74110_rx_flush_msg(struct bcm74110_mbox *mbox) +{ + struct bcm74110_mbox_msg *msg, *msg_tmp; + LIST_HEAD(list_temp); + unsigned long flags; + + spin_lock_irqsave(&mbox->rx_svc_list_lock, flags); + list_splice_init(&mbox->rx_svc_init_list, &list_temp); + spin_unlock_irqrestore(&mbox->rx_svc_list_lock, flags); + + list_for_each_entry_safe(msg, msg_tmp, &list_temp, list_entry) { + list_del(&msg->list_entry); + kfree(msg); + } +} + +#define BCM_DEQUEUE_TIMEOUT_MS 30 +static int bcm74110_rx_pop_init_msg_block(struct bcm74110_mbox *mbox, u32 func_type, + u32 *val) +{ + int ret, timeout = 0; + + do { + ret = bcm74110_rx_pop_init_msg(mbox, func_type, val); + + if (!ret) + return 0; + + /* TODO: Figure out what is a good sleep here. */ + usleep_range(1000, 2000); + timeout++; + } while (timeout < BCM_DEQUEUE_TIMEOUT_MS); + + dev_warn(&mbox->pdev->dev, "Timeout waiting for service init response\n"); + return -ETIMEDOUT; +} + +static int bcm74110_mbox_create_msg(int req, int rply, int svc, int func, + int length, int slot) +{ + u32 msg = 0; + + BCM_MSG_SET_FIELD(msg, REQ, req); + BCM_MSG_SET_FIELD(msg, RPLY, rply); + BCM_MSG_SET_FIELD(msg, SVC, svc); + BCM_MSG_SET_FIELD(msg, FUNC, func); + BCM_MSG_SET_FIELD(msg, LENGTH, length); + BCM_MSG_SET_FIELD(msg, SLOT, slot); + + return msg; +} + +static int bcm74110_mbox_tx_msg(struct bcm74110_mbox *mbox, u32 msg) +{ + int val; + + /* We can potentially poll with timeout here instead */ + val = bcm74110_tx_readl(mbox, BCM_MBOX_STATUS0); + if (val & BCM_MBOX_STATUS0_FULL) { + dev_err(&mbox->pdev->dev, "Mailbox full\n"); + return -EINVAL; + } + + dev_dbg(&mbox->pdev->dev, "tx: [{req=%lu|rply=%lu|srv=%lu|fn=%lu|length=%lu|slot=%lu]\n", + BCM_MSG_GET_FIELD(msg, REQ), BCM_MSG_GET_FIELD(msg, RPLY), + BCM_MSG_GET_FIELD(msg, SVC), BCM_MSG_GET_FIELD(msg, FUNC), + BCM_MSG_GET_FIELD(msg, LENGTH), BCM_MSG_GET_FIELD(msg, SLOT)); + + bcm74110_tx_writel(mbox, msg, BCM_MBOX_WDATA); + + return 0; +} + +#define BCM_MBOX_LINK_TRAINING_RETRIES 5 +static int bcm74110_mbox_link_training(struct bcm74110_mbox *mbox) +{ + int ret, retries = 0; + u32 msg = 0, orig_len = 0, len = BCM_LINK_CODE0; + + do { + switch (len) { + case 0: + retries++; + dev_warn(&mbox->pdev->dev, + "Link train failed, trying again... %d\n", + retries); + if (retries > BCM_MBOX_LINK_TRAINING_RETRIES) + return -EINVAL; + len = BCM_LINK_CODE0; + fallthrough; + case BCM_LINK_CODE0: + case BCM_LINK_CODE1: + case BCM_LINK_CODE2: + msg = bcm74110_mbox_create_msg(1, 0, BCM_MSG_SVC_INIT, + BCM_MSG_FUNC_LINK_START, + len, BCM_MSG_SVC_INIT); + break; + default: + break; + } + + bcm74110_mbox_tx_msg(mbox, msg); + + /* No response expected for LINK_CODE2 */ + if (len == BCM_LINK_CODE2) + return 0; + + orig_len = len; + + ret = bcm74110_rx_pop_init_msg_block(mbox, + BCM_MSG_GET_FIELD(msg, FUNC), + &msg); + if (ret) { + len = 0; + continue; + } + + if ((BCM_MSG_GET_FIELD(msg, SVC) != BCM_MSG_SVC_INIT) || + (BCM_MSG_GET_FIELD(msg, FUNC) != BCM_MSG_FUNC_LINK_START) || + (BCM_MSG_GET_FIELD(msg, SLOT) != 0) || + (BCM_MSG_GET_FIELD(msg, RPLY) != 1) || + (BCM_MSG_GET_FIELD(msg, REQ) != 0)) { + len = 0; + continue; + } + + len = BCM_MSG_GET_FIELD(msg, LENGTH); + + /* Make sure sequence is good */ + if (len != (orig_len + 1)) { + len = 0; + continue; + } + } while (1); + + return -EINVAL; +} + +static int bcm74110_mbox_tx_msg_and_wait_ack(struct bcm74110_mbox *mbox, u32 msg) +{ + int ret; + u32 recv_msg; + + ret = bcm74110_mbox_tx_msg(mbox, msg); + if (ret) + return ret; + + ret = bcm74110_rx_pop_init_msg_block(mbox, BCM_MSG_GET_FIELD(msg, FUNC), + &recv_msg); + if (ret) + return ret; + + /* + * Modify tx message to verify rx ack. + * Flip RPLY/REQ for synchronous messages + */ + if (BCM_MSG_GET_FIELD(msg, REQ) == 1) { + BCM_MSG_SET_FIELD(msg, RPLY, 1); + BCM_MSG_SET_FIELD(msg, REQ, 0); + } + + if (msg != recv_msg) { + dev_err(&mbox->pdev->dev, "Found ack, but ack is invalid\n"); + return -EINVAL; + } + + return 0; +} + +/* Each index points to 0x100 of HAB MEM. IDX size counts from 0 */ +#define BCM_MBOX_HAB_MEM_IDX_START 0x30 +#define BCM_MBOX_HAB_MEM_IDX_SIZE 0x0 +static int bcm74110_mbox_shmem_init(struct bcm74110_mbox *mbox) +{ + u32 msg = 0; + int ret; + + msg = bcm74110_mbox_create_msg(1, 0, BCM_MSG_SVC_INIT, + BCM_MSG_FUNC_SHMEM_STOP, + 0, BCM_MSG_SVC_INIT); + ret = bcm74110_mbox_tx_msg_and_wait_ack(mbox, msg); + if (ret) + return -EINVAL; + + msg = bcm74110_mbox_create_msg(1, 0, BCM_MSG_SVC_INIT, + BCM_MSG_FUNC_SHMEM_TX, + BCM_MBOX_HAB_MEM_IDX_START, + BCM_MBOX_HAB_MEM_IDX_SIZE); + ret = bcm74110_mbox_tx_msg_and_wait_ack(mbox, msg); + if (ret) + return -EINVAL; + + msg = bcm74110_mbox_create_msg(1, 0, BCM_MSG_SVC_INIT, + BCM_MSG_FUNC_SHMEM_RX, + BCM_MBOX_HAB_MEM_IDX_START, + BCM_MBOX_HAB_MEM_IDX_SIZE); + ret = bcm74110_mbox_tx_msg_and_wait_ack(mbox, msg); + if (ret) + return -EINVAL; + + return 0; +} + +static int bcm74110_mbox_init(struct bcm74110_mbox *mbox) +{ + int ret = 0; + + /* Disable queues tx/rx */ + bcm74110_tx_writel(mbox, 0x0, BCM_MBOX_CTRL); + + /* Clear status & restart tx/rx*/ + bcm74110_tx_writel(mbox, BCM_MBOX_CTRL_EN | BCM_MBOX_CTRL_CLR, + BCM_MBOX_CTRL); + + /* Unmask irq */ + bcm74110_irq_writel(mbox, BCM_MBOX_IRQ_NOT_EMPTY, BCM_MBOX_IRQ_MASK_CLEAR); + + ret = bcm74110_mbox_link_training(mbox); + if (ret) { + dev_err(&mbox->pdev->dev, "Training failed\n"); + return ret; + } + + return bcm74110_mbox_shmem_init(mbox); +} + +static int bcm74110_mbox_send_data(struct mbox_chan *chan, void *data) +{ + struct bcm74110_mbox_chan *chan_priv = chan->con_priv; + u32 msg; + + switch (chan_priv->type) { + case BCM_MSG_SVC_PMC: + case BCM_MSG_SVC_SCMI: + case BCM_MSG_SVC_DPFE: + msg = bcm74110_mbox_create_msg(1, 0, chan_priv->type, 0, + 128 + 28, chan_priv->slot); + break; + default: + return -EINVAL; + } + + return bcm74110_mbox_tx_msg(chan_priv->mbox, msg); +} + +static int bcm74110_mbox_chan_startup(struct mbox_chan *chan) +{ + struct bcm74110_mbox_chan *chan_priv = chan->con_priv; + + chan_priv->en = true; + + return 0; +} + +static void bcm74110_mbox_chan_shutdown(struct mbox_chan *chan) +{ + struct bcm74110_mbox_chan *chan_priv = chan->con_priv; + + chan_priv->en = false; +} + +static const struct mbox_chan_ops bcm74110_mbox_chan_ops = { + .send_data = bcm74110_mbox_send_data, + .startup = bcm74110_mbox_chan_startup, + .shutdown = bcm74110_mbox_chan_shutdown, +}; + +static void bcm74110_mbox_shutdown(struct platform_device *pdev) +{ + struct bcm74110_mbox *mbox = dev_get_drvdata(&pdev->dev); + u32 msg; + + msg = bcm74110_mbox_create_msg(1, 0, BCM_MSG_SVC_INIT, + BCM_MSG_FUNC_LINK_STOP, + 0, 0); + + bcm74110_mbox_tx_msg_and_wait_ack(mbox, msg); + + /* Even if we don't receive ACK, lets shut it down */ + + bcm74110_mbox_mask_and_clear(mbox); + + /* Disable queues tx/rx */ + bcm74110_tx_writel(mbox, 0x0, BCM_MBOX_CTRL); + + /* Flush queues */ + bcm74110_rx_flush_msg(mbox); +} + +static struct mbox_chan *bcm74110_mbox_of_xlate(struct mbox_controller *cntrl, + const struct of_phandle_args *p) +{ + struct bcm74110_mbox *mbox = bcm74110_mbox_from_cntrl(cntrl); + struct device *dev = &mbox->pdev->dev; + struct bcm74110_mbox_chan *chan_priv; + int slot, type; + + if (p->args_count != 2) { + dev_err(dev, "Invalid arguments\n"); + return ERR_PTR(-EINVAL); + } + + type = p->args[0]; + slot = p->args[1]; + + switch (type) { + case BCM_MSG_SVC_PMC: + case BCM_MSG_SVC_SCMI: + case BCM_MSG_SVC_DPFE: + if (slot > BCM_MBOX_HAB_MEM_IDX_SIZE) { + dev_err(dev, "Not enough shared memory\n"); + return ERR_PTR(-EINVAL); + } + chan_priv = cntrl->chans[type].con_priv; + chan_priv->slot = slot; + chan_priv->type = type; + break; + default: + dev_err(dev, "Invalid channel type: %d\n", type); + return ERR_PTR(-EINVAL); + } + + return &cntrl->chans[type]; +} + +static int bcm74110_mbox_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct bcm74110_mbox *mbox; + int i, ret; + + mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); + if (!mbox) + return -ENOMEM; + + mbox->pdev = pdev; + platform_set_drvdata(pdev, mbox); + + mbox->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(mbox->base)) + return dev_err_probe(dev, PTR_ERR(mbox->base), "Failed to iomap\n"); + + ret = of_property_read_u32(dev->of_node, "brcm,tx", &mbox->tx_chan); + if (ret) + return dev_err_probe(dev, ret, "Failed to find tx channel\n"); + + ret = of_property_read_u32(dev->of_node, "brcm,rx", &mbox->rx_chan); + if (ret) + return dev_err_probe(dev, ret, "Failed to find rx channel\n"); + + mbox->rx_irq = platform_get_irq(pdev, 0); + if (mbox->rx_irq < 0) + return mbox->rx_irq; + + INIT_LIST_HEAD(&mbox->rx_svc_init_list); + spin_lock_init(&mbox->rx_svc_list_lock); + bcm74110_mbox_mask_and_clear(mbox); + + ret = devm_request_irq(dev, mbox->rx_irq, bcm74110_mbox_isr, + IRQF_NO_SUSPEND, pdev->name, mbox); + if (ret) + return dev_err_probe(dev, ret, "Failed to request irq\n"); + + mbox->controller.ops = &bcm74110_mbox_chan_ops; + mbox->controller.dev = dev; + mbox->controller.num_chans = BCM_MSG_SVC_MAX; + mbox->controller.of_xlate = &bcm74110_mbox_of_xlate; + mbox->controller.chans = devm_kcalloc(dev, BCM_MSG_SVC_MAX, + sizeof(*mbox->controller.chans), + GFP_KERNEL); + if (!mbox->controller.chans) + return -ENOMEM; + + mbox->mbox_chan = devm_kcalloc(dev, BCM_MSG_SVC_MAX, + sizeof(*mbox->mbox_chan), + GFP_KERNEL); + if (!mbox->mbox_chan) + return -ENOMEM; + + for (i = 0; i < BCM_MSG_SVC_MAX; i++) { + mbox->mbox_chan[i].mbox = mbox; + mbox->controller.chans[i].con_priv = &mbox->mbox_chan[i]; + } + + ret = devm_mbox_controller_register(dev, &mbox->controller); + if (ret) + return ret; + + ret = bcm74110_mbox_init(mbox); + if (ret) + return ret; + + return 0; +} + +static const struct of_device_id bcm74110_mbox_of_match[] = { + { .compatible = "brcm,bcm74110-mbox", }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, bcm74110_mbox_of_match); + +static struct platform_driver bcm74110_mbox_driver = { + .driver = { + .name = "bcm74110-mbox", + .of_match_table = bcm74110_mbox_of_match, + }, + .probe = bcm74110_mbox_probe, + .shutdown = bcm74110_mbox_shutdown, +}; +module_platform_driver(bcm74110_mbox_driver); + +MODULE_AUTHOR("Justin Chen <justin.chen@broadcom.com>"); +MODULE_DESCRIPTION("BCM74110 mailbox driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index ab4e8d1954a1..532929916e99 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -390,7 +390,7 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) task = kzalloc(sizeof(*task), GFP_ATOMIC); if (!task) { - __pm_runtime_put_autosuspend(cmdq->mbox.dev); + pm_runtime_put_autosuspend(cmdq->mbox.dev); return -ENOMEM; } @@ -440,7 +440,7 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) list_move_tail(&task->list_entry, &thread->task_busy_list); pm_runtime_mark_last_busy(cmdq->mbox.dev); - __pm_runtime_put_autosuspend(cmdq->mbox.dev); + pm_runtime_put_autosuspend(cmdq->mbox.dev); return 0; } @@ -488,7 +488,7 @@ done: spin_unlock_irqrestore(&thread->chan->lock, flags); pm_runtime_mark_last_busy(cmdq->mbox.dev); - __pm_runtime_put_autosuspend(cmdq->mbox.dev); + pm_runtime_put_autosuspend(cmdq->mbox.dev); } static int cmdq_mbox_flush(struct mbox_chan *chan, unsigned long timeout) @@ -528,7 +528,7 @@ static int cmdq_mbox_flush(struct mbox_chan *chan, unsigned long timeout) out: spin_unlock_irqrestore(&thread->chan->lock, flags); pm_runtime_mark_last_busy(cmdq->mbox.dev); - __pm_runtime_put_autosuspend(cmdq->mbox.dev); + pm_runtime_put_autosuspend(cmdq->mbox.dev); return 0; @@ -543,7 +543,7 @@ wait: return -EFAULT; } pm_runtime_mark_last_busy(cmdq->mbox.dev); - __pm_runtime_put_autosuspend(cmdq->mbox.dev); + pm_runtime_put_autosuspend(cmdq->mbox.dev); return 0; } diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index f6714c233f5a..0a00719b2482 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -306,6 +306,22 @@ static void pcc_chan_acknowledge(struct pcc_chan_info *pchan) pcc_chan_reg_read_modify_write(&pchan->db); } +static void *write_response(struct pcc_chan_info *pchan) +{ + struct pcc_header pcc_header; + void *buffer; + int data_len; + + memcpy_fromio(&pcc_header, pchan->chan.shmem, + sizeof(pcc_header)); + data_len = pcc_header.length - sizeof(u32) + sizeof(struct pcc_header); + + buffer = pchan->chan.rx_alloc(pchan->chan.mchan->cl, data_len); + if (buffer != NULL) + memcpy_fromio(buffer, pchan->chan.shmem, data_len); + return buffer; +} + /** * pcc_mbox_irq - PCC mailbox interrupt handler * @irq: interrupt number @@ -317,6 +333,8 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) { struct pcc_chan_info *pchan; struct mbox_chan *chan = p; + struct pcc_header *pcc_header = chan->active_req; + void *handle = NULL; pchan = chan->con_priv; @@ -340,7 +358,17 @@ static irqreturn_t pcc_mbox_irq(int irq, void *p) * required to avoid any possible race in updatation of this flag. */ pchan->chan_in_use = false; - mbox_chan_received_data(chan, NULL); + + if (pchan->chan.rx_alloc) + handle = write_response(pchan); + + if (chan->active_req) { + pcc_header = chan->active_req; + if (pcc_header->flags & PCC_CMD_COMPLETION_NOTIFY) + mbox_chan_txdone(chan, 0); + } + + mbox_chan_received_data(chan, handle); pcc_chan_acknowledge(pchan); @@ -384,9 +412,24 @@ pcc_mbox_request_channel(struct mbox_client *cl, int subspace_id) pcc_mchan = &pchan->chan; pcc_mchan->shmem = acpi_os_ioremap(pcc_mchan->shmem_base_addr, pcc_mchan->shmem_size); - if (pcc_mchan->shmem) - return pcc_mchan; + if (!pcc_mchan->shmem) + goto err; + + pcc_mchan->manage_writes = false; + + /* This indicates that the channel is ready to accept messages. + * This needs to happen after the channel has registered + * its callback. There is no access point to do that in + * the mailbox API. That implies that the mailbox client must + * have set the allocate callback function prior to + * sending any messages. + */ + if (pchan->type == ACPI_PCCT_TYPE_EXT_PCC_SLAVE_SUBSPACE) + pcc_chan_reg_read_modify_write(&pchan->cmd_update); + + return pcc_mchan; +err: mbox_free_channel(chan); return ERR_PTR(-ENXIO); } @@ -417,8 +460,38 @@ void pcc_mbox_free_channel(struct pcc_mbox_chan *pchan) } EXPORT_SYMBOL_GPL(pcc_mbox_free_channel); +static int pcc_write_to_buffer(struct mbox_chan *chan, void *data) +{ + struct pcc_chan_info *pchan = chan->con_priv; + struct pcc_mbox_chan *pcc_mbox_chan = &pchan->chan; + struct pcc_header *pcc_header = data; + + if (!pchan->chan.manage_writes) + return 0; + + /* The PCC header length includes the command field + * but not the other values from the header. + */ + int len = pcc_header->length - sizeof(u32) + sizeof(struct pcc_header); + u64 val; + + pcc_chan_reg_read(&pchan->cmd_complete, &val); + if (!val) { + pr_info("%s pchan->cmd_complete not set", __func__); + return -1; + } + memcpy_toio(pcc_mbox_chan->shmem, data, len); + return 0; +} + + /** - * pcc_send_data - Called from Mailbox Controller code. Used + * pcc_send_data - Called from Mailbox Controller code. If + * pchan->chan.rx_alloc is set, then the command complete + * flag is checked and the data is written to the shared + * buffer io memory. + * + * If pchan->chan.rx_alloc is not set, then it is used * here only to ring the channel doorbell. The PCC client * specific read/write is done in the client driver in * order to maintain atomicity over PCC channel once @@ -434,17 +507,37 @@ static int pcc_send_data(struct mbox_chan *chan, void *data) int ret; struct pcc_chan_info *pchan = chan->con_priv; + ret = pcc_write_to_buffer(chan, data); + if (ret) + return ret; + ret = pcc_chan_reg_read_modify_write(&pchan->cmd_update); if (ret) return ret; ret = pcc_chan_reg_read_modify_write(&pchan->db); + if (!ret && pchan->plat_irq > 0) pchan->chan_in_use = true; return ret; } + +static bool pcc_last_tx_done(struct mbox_chan *chan) +{ + struct pcc_chan_info *pchan = chan->con_priv; + u64 val; + + pcc_chan_reg_read(&pchan->cmd_complete, &val); + if (!val) + return false; + else + return true; +} + + + /** * pcc_startup - Called from Mailbox Controller code. Used here * to request the interrupt. @@ -490,6 +583,7 @@ static const struct mbox_chan_ops pcc_chan_ops = { .send_data = pcc_send_data, .startup = pcc_startup, .shutdown = pcc_shutdown, + .last_tx_done = pcc_last_tx_done, }; /** diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c index ea44ffb5ce1a..d957d989c0ce 100644 --- a/drivers/mailbox/qcom-ipcc.c +++ b/drivers/mailbox/qcom-ipcc.c @@ -312,8 +312,7 @@ static int qcom_ipcc_probe(struct platform_device *pdev) if (!name) return -ENOMEM; - ipcc->irq_domain = irq_domain_create_tree(of_fwnode_handle(pdev->dev.of_node), - &qcom_ipcc_irq_ops, ipcc); + ipcc->irq_domain = irq_domain_create_tree(dev_fwnode(&pdev->dev), &qcom_ipcc_irq_ops, ipcc); if (!ipcc->irq_domain) return -ENOMEM; diff --git a/drivers/md/dm-raid.c b/drivers/md/dm-raid.c index 15c538ee9537..79ea85d18e24 100644 --- a/drivers/md/dm-raid.c +++ b/drivers/md/dm-raid.c @@ -438,7 +438,7 @@ static bool rs_is_reshapable(struct raid_set *rs) /* Return true, if raid set in @rs is recovering */ static bool rs_is_recovering(struct raid_set *rs) { - return rs->md.recovery_cp < rs->md.dev_sectors; + return rs->md.resync_offset < rs->md.dev_sectors; } /* Return true, if raid set in @rs is reshaping */ @@ -768,7 +768,7 @@ static struct raid_set *raid_set_alloc(struct dm_target *ti, struct raid_type *r rs->md.layout = raid_type->algorithm; rs->md.new_layout = rs->md.layout; rs->md.delta_disks = 0; - rs->md.recovery_cp = MaxSector; + rs->md.resync_offset = MaxSector; for (i = 0; i < raid_devs; i++) md_rdev_init(&rs->dev[i].rdev); @@ -912,7 +912,7 @@ static int parse_dev_params(struct raid_set *rs, struct dm_arg_set *as) rs->md.external = 0; rs->md.persistent = 1; rs->md.major_version = 2; - } else if (rebuild && !rs->md.recovery_cp) { + } else if (rebuild && !rs->md.resync_offset) { /* * Without metadata, we will not be able to tell if the array * is in-sync or not - we must assume it is not. Therefore, @@ -1695,20 +1695,20 @@ static void rs_setup_recovery(struct raid_set *rs, sector_t dev_sectors) { /* raid0 does not recover */ if (rs_is_raid0(rs)) - rs->md.recovery_cp = MaxSector; + rs->md.resync_offset = MaxSector; /* * A raid6 set has to be recovered either * completely or for the grown part to * ensure proper parity and Q-Syndrome */ else if (rs_is_raid6(rs)) - rs->md.recovery_cp = dev_sectors; + rs->md.resync_offset = dev_sectors; /* * Other raid set types may skip recovery * depending on the 'nosync' flag. */ else - rs->md.recovery_cp = test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags) + rs->md.resync_offset = test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags) ? MaxSector : dev_sectors; } @@ -2143,7 +2143,7 @@ static void super_sync(struct mddev *mddev, struct md_rdev *rdev) sb->events = cpu_to_le64(mddev->events); sb->disk_recovery_offset = cpu_to_le64(rdev->recovery_offset); - sb->array_resync_offset = cpu_to_le64(mddev->recovery_cp); + sb->array_resync_offset = cpu_to_le64(mddev->resync_offset); sb->level = cpu_to_le32(mddev->level); sb->layout = cpu_to_le32(mddev->layout); @@ -2334,18 +2334,18 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) } if (!test_bit(__CTR_FLAG_NOSYNC, &rs->ctr_flags)) - mddev->recovery_cp = le64_to_cpu(sb->array_resync_offset); + mddev->resync_offset = le64_to_cpu(sb->array_resync_offset); /* * During load, we set FirstUse if a new superblock was written. * There are two reasons we might not have a superblock: * 1) The raid set is brand new - in which case, all of the * devices must have their In_sync bit set. Also, - * recovery_cp must be 0, unless forced. + * resync_offset must be 0, unless forced. * 2) This is a new device being added to an old raid set * and the new device needs to be rebuilt - in which * case the In_sync bit will /not/ be set and - * recovery_cp must be MaxSector. + * resync_offset must be MaxSector. * 3) This is/are a new device(s) being added to an old * raid set during takeover to a higher raid level * to provide capacity for redundancy or during reshape @@ -2390,8 +2390,8 @@ static int super_init_validation(struct raid_set *rs, struct md_rdev *rdev) new_devs > 1 ? "s" : ""); return -EINVAL; } else if (!test_bit(__CTR_FLAG_REBUILD, &rs->ctr_flags) && rs_is_recovering(rs)) { - DMERR("'rebuild' specified while raid set is not in-sync (recovery_cp=%llu)", - (unsigned long long) mddev->recovery_cp); + DMERR("'rebuild' specified while raid set is not in-sync (resync_offset=%llu)", + (unsigned long long) mddev->resync_offset); return -EINVAL; } else if (rs_is_reshaping(rs)) { DMERR("'rebuild' specified while raid set is being reshaped (reshape_position=%llu)", @@ -2700,11 +2700,11 @@ static int rs_adjust_data_offsets(struct raid_set *rs) } out: /* - * Raise recovery_cp in case data_offset != 0 to + * Raise resync_offset in case data_offset != 0 to * avoid false recovery positives in the constructor. */ - if (rs->md.recovery_cp < rs->md.dev_sectors) - rs->md.recovery_cp += rs->dev[0].rdev.data_offset; + if (rs->md.resync_offset < rs->md.dev_sectors) + rs->md.resync_offset += rs->dev[0].rdev.data_offset; /* Adjust data offsets on all rdevs but on any raid4/5/6 journal device */ rdev_for_each(rdev, &rs->md) { @@ -2759,7 +2759,7 @@ static int rs_setup_takeover(struct raid_set *rs) } clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags); - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; while (d--) { rdev = &rs->dev[d].rdev; @@ -2767,7 +2767,7 @@ static int rs_setup_takeover(struct raid_set *rs) if (test_bit(d, (void *) rs->rebuild_disks)) { clear_bit(In_sync, &rdev->flags); clear_bit(Faulty, &rdev->flags); - mddev->recovery_cp = rdev->recovery_offset = 0; + mddev->resync_offset = rdev->recovery_offset = 0; /* Bitmap has to be created when we do an "up" takeover */ set_bit(MD_ARRAY_FIRST_USE, &mddev->flags); } @@ -3225,7 +3225,7 @@ size_check: if (r) goto bad; - rs_setup_recovery(rs, rs->md.recovery_cp < rs->md.dev_sectors ? rs->md.recovery_cp : rs->md.dev_sectors); + rs_setup_recovery(rs, rs->md.resync_offset < rs->md.dev_sectors ? rs->md.resync_offset : rs->md.dev_sectors); } else { /* This is no size change or it is shrinking, update size and record in superblocks */ r = rs_set_dev_and_array_sectors(rs, rs->ti->len, false); @@ -3449,7 +3449,7 @@ static sector_t rs_get_progress(struct raid_set *rs, unsigned long recovery, } else { if (state == st_idle && !test_bit(MD_RECOVERY_INTR, &recovery)) - r = mddev->recovery_cp; + r = mddev->resync_offset; else r = mddev->curr_resync_completed; @@ -4077,9 +4077,9 @@ static int raid_preresume(struct dm_target *ti) } /* Check for any resize/reshape on @rs and adjust/initiate */ - if (mddev->recovery_cp && mddev->recovery_cp < MaxSector) { + if (mddev->resync_offset && mddev->resync_offset < MaxSector) { set_bit(MD_RECOVERY_REQUESTED, &mddev->recovery); - mddev->resync_min = mddev->recovery_cp; + mddev->resync_min = mddev->resync_offset; if (test_bit(RT_FLAG_RS_GROW, &rs->runtime_flags)) mddev->resync_max_sectors = mddev->dev_sectors; } diff --git a/drivers/md/md-bitmap.c b/drivers/md/md-bitmap.c index 7f524a26cebc..334b71404930 100644 --- a/drivers/md/md-bitmap.c +++ b/drivers/md/md-bitmap.c @@ -1987,12 +1987,12 @@ static void bitmap_dirty_bits(struct mddev *mddev, unsigned long s, md_bitmap_set_memory_bits(bitmap, sec, 1); md_bitmap_file_set_bit(bitmap, sec); - if (sec < bitmap->mddev->recovery_cp) + if (sec < bitmap->mddev->resync_offset) /* We are asserting that the array is dirty, - * so move the recovery_cp address back so + * so move the resync_offset address back so * that it is obvious that it is dirty */ - bitmap->mddev->recovery_cp = sec; + bitmap->mddev->resync_offset = sec; } } @@ -2258,7 +2258,7 @@ static int bitmap_load(struct mddev *mddev) || bitmap->events_cleared == mddev->events) /* no need to keep dirty bits to optimise a * re-add of a missing device */ - start = mddev->recovery_cp; + start = mddev->resync_offset; mutex_lock(&mddev->bitmap_info.mutex); err = md_bitmap_init_from_disk(bitmap, start); diff --git a/drivers/md/md-cluster.c b/drivers/md/md-cluster.c index 94221d964d4f..5497eaee96e7 100644 --- a/drivers/md/md-cluster.c +++ b/drivers/md/md-cluster.c @@ -337,11 +337,11 @@ static void recover_bitmaps(struct md_thread *thread) md_wakeup_thread(mddev->sync_thread); if (hi > 0) { - if (lo < mddev->recovery_cp) - mddev->recovery_cp = lo; + if (lo < mddev->resync_offset) + mddev->resync_offset = lo; /* wake up thread to continue resync in case resync * is not finished */ - if (mddev->recovery_cp != MaxSector) { + if (mddev->resync_offset != MaxSector) { /* * clear the REMOTE flag since we will launch * resync thread in current node. @@ -863,9 +863,9 @@ static int gather_all_resync_info(struct mddev *mddev, int total_slots) lockres_free(bm_lockres); continue; } - if ((hi > 0) && (lo < mddev->recovery_cp)) { + if ((hi > 0) && (lo < mddev->resync_offset)) { set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); - mddev->recovery_cp = lo; + mddev->resync_offset = lo; md_check_recovery(mddev); } @@ -1027,7 +1027,7 @@ static int leave(struct mddev *mddev) * Also, we should send BITMAP_NEEDS_SYNC message in * case reshaping is interrupted. */ - if ((cinfo->slot_number > 0 && mddev->recovery_cp != MaxSector) || + if ((cinfo->slot_number > 0 && mddev->resync_offset != MaxSector) || (mddev->reshape_position != MaxSector && test_bit(MD_CLOSING, &mddev->flags))) resync_bitmap(mddev); @@ -1605,8 +1605,8 @@ static int gather_bitmaps(struct md_rdev *rdev) pr_warn("md-cluster: Could not gather bitmaps from slot %d", sn); goto out; } - if ((hi > 0) && (lo < mddev->recovery_cp)) - mddev->recovery_cp = lo; + if ((hi > 0) && (lo < mddev->resync_offset)) + mddev->resync_offset = lo; } out: return err; diff --git a/drivers/md/md.c b/drivers/md/md.c index 046fe85c76fe..ac85ec73a409 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -637,6 +637,12 @@ static void __mddev_put(struct mddev *mddev) return; /* + * If array is freed by stopping array, MD_DELETED is set by + * do_md_stop(), MD_DELETED is still set here in case mddev is freed + * directly by closing a mddev that is created by create_on_open. + */ + set_bit(MD_DELETED, &mddev->flags); + /* * Call queue_work inside the spinlock so that flush_workqueue() after * mddev_find will succeed in waiting for the work to be done. */ @@ -1409,13 +1415,13 @@ static int super_90_validate(struct mddev *mddev, struct md_rdev *freshest, stru mddev->layout = -1; if (sb->state & (1<<MD_SB_CLEAN)) - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; else { if (sb->events_hi == sb->cp_events_hi && sb->events_lo == sb->cp_events_lo) { - mddev->recovery_cp = sb->recovery_cp; + mddev->resync_offset = sb->resync_offset; } else - mddev->recovery_cp = 0; + mddev->resync_offset = 0; } memcpy(mddev->uuid+0, &sb->set_uuid0, 4); @@ -1541,13 +1547,13 @@ static void super_90_sync(struct mddev *mddev, struct md_rdev *rdev) mddev->minor_version = sb->minor_version; if (mddev->in_sync) { - sb->recovery_cp = mddev->recovery_cp; + sb->resync_offset = mddev->resync_offset; sb->cp_events_hi = (mddev->events>>32); sb->cp_events_lo = (u32)mddev->events; - if (mddev->recovery_cp == MaxSector) + if (mddev->resync_offset == MaxSector) sb->state = (1<< MD_SB_CLEAN); } else - sb->recovery_cp = 0; + sb->resync_offset = 0; sb->layout = mddev->layout; sb->chunk_size = mddev->chunk_sectors << 9; @@ -1895,7 +1901,7 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *freshest, struc mddev->bitmap_info.default_space = (4096-1024) >> 9; mddev->reshape_backwards = 0; - mddev->recovery_cp = le64_to_cpu(sb->resync_offset); + mddev->resync_offset = le64_to_cpu(sb->resync_offset); memcpy(mddev->uuid, sb->set_uuid, 16); mddev->max_disks = (4096-256)/2; @@ -2081,7 +2087,7 @@ static void super_1_sync(struct mddev *mddev, struct md_rdev *rdev) sb->utime = cpu_to_le64((__u64)mddev->utime); sb->events = cpu_to_le64(mddev->events); if (mddev->in_sync) - sb->resync_offset = cpu_to_le64(mddev->recovery_cp); + sb->resync_offset = cpu_to_le64(mddev->resync_offset); else if (test_bit(MD_JOURNAL_CLEAN, &mddev->flags)) sb->resync_offset = cpu_to_le64(MaxSector); else @@ -2761,7 +2767,7 @@ repeat: /* If this is just a dirty<->clean transition, and the array is clean * and 'events' is odd, we can roll back to the previous clean state */ if (nospares - && (mddev->in_sync && mddev->recovery_cp == MaxSector) + && (mddev->in_sync && mddev->resync_offset == MaxSector) && mddev->can_decrease_events && mddev->events != 1) { mddev->events--; @@ -4297,9 +4303,9 @@ __ATTR(chunk_size, S_IRUGO|S_IWUSR, chunk_size_show, chunk_size_store); static ssize_t resync_start_show(struct mddev *mddev, char *page) { - if (mddev->recovery_cp == MaxSector) + if (mddev->resync_offset == MaxSector) return sprintf(page, "none\n"); - return sprintf(page, "%llu\n", (unsigned long long)mddev->recovery_cp); + return sprintf(page, "%llu\n", (unsigned long long)mddev->resync_offset); } static ssize_t @@ -4325,7 +4331,7 @@ resync_start_store(struct mddev *mddev, const char *buf, size_t len) err = -EBUSY; if (!err) { - mddev->recovery_cp = n; + mddev->resync_offset = n; if (mddev->pers) set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); } @@ -6417,7 +6423,7 @@ static void md_clean(struct mddev *mddev) mddev->external_size = 0; mddev->dev_sectors = 0; mddev->raid_disks = 0; - mddev->recovery_cp = 0; + mddev->resync_offset = 0; mddev->resync_min = 0; mddev->resync_max = MaxSector; mddev->reshape_position = MaxSector; @@ -7362,9 +7368,9 @@ int md_set_array_info(struct mddev *mddev, struct mdu_array_info_s *info) * openned */ if (info->state & (1<<MD_SB_CLEAN)) - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; else - mddev->recovery_cp = 0; + mddev->resync_offset = 0; mddev->persistent = ! info->not_persistent; mddev->external = 0; @@ -8303,7 +8309,7 @@ static int status_resync(struct seq_file *seq, struct mddev *mddev) seq_printf(seq, "\tresync=REMOTE"); return 1; } - if (mddev->recovery_cp < MaxSector) { + if (mddev->resync_offset < MaxSector) { seq_printf(seq, "\tresync=PENDING"); return 1; } @@ -8946,7 +8952,7 @@ static sector_t md_sync_position(struct mddev *mddev, enum sync_action action) return mddev->resync_min; case ACTION_RESYNC: if (!mddev->bitmap) - return mddev->recovery_cp; + return mddev->resync_offset; return 0; case ACTION_RESHAPE: /* @@ -9184,8 +9190,8 @@ void md_do_sync(struct md_thread *thread) atomic_read(&mddev->recovery_active) == 0); mddev->curr_resync_completed = j; if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && - j > mddev->recovery_cp) - mddev->recovery_cp = j; + j > mddev->resync_offset) + mddev->resync_offset = j; update_time = jiffies; set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); sysfs_notify_dirent_safe(mddev->sysfs_completed); @@ -9305,19 +9311,19 @@ void md_do_sync(struct md_thread *thread) mddev->curr_resync > MD_RESYNC_ACTIVE) { if (test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) { if (test_bit(MD_RECOVERY_INTR, &mddev->recovery)) { - if (mddev->curr_resync >= mddev->recovery_cp) { + if (mddev->curr_resync >= mddev->resync_offset) { pr_debug("md: checkpointing %s of %s.\n", desc, mdname(mddev)); if (test_bit(MD_RECOVERY_ERROR, &mddev->recovery)) - mddev->recovery_cp = + mddev->resync_offset = mddev->curr_resync_completed; else - mddev->recovery_cp = + mddev->resync_offset = mddev->curr_resync; } } else - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; } else { if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) mddev->curr_resync = MaxSector; @@ -9421,6 +9427,12 @@ static bool rdev_is_spare(struct md_rdev *rdev) static bool rdev_addable(struct md_rdev *rdev) { + struct mddev *mddev; + + mddev = READ_ONCE(rdev->mddev); + if (!mddev) + return false; + /* rdev is already used, don't add it again. */ if (test_bit(Candidate, &rdev->flags) || rdev->raid_disk >= 0 || test_bit(Faulty, &rdev->flags)) @@ -9431,7 +9443,7 @@ static bool rdev_addable(struct md_rdev *rdev) return true; /* Allow to add if array is read-write. */ - if (md_is_rdwr(rdev->mddev)) + if (md_is_rdwr(mddev)) return true; /* @@ -9533,7 +9545,7 @@ static bool md_choose_sync_action(struct mddev *mddev, int *spares) } /* Check if resync is in progress. */ - if (mddev->recovery_cp < MaxSector) { + if (mddev->resync_offset < MaxSector) { remove_spares(mddev, NULL); set_bit(MD_RECOVERY_SYNC, &mddev->recovery); clear_bit(MD_RECOVERY_RECOVER, &mddev->recovery); @@ -9714,7 +9726,7 @@ void md_check_recovery(struct mddev *mddev) test_bit(MD_RECOVERY_DONE, &mddev->recovery) || (mddev->external == 0 && mddev->safemode == 1) || (mddev->safemode == 2 - && !mddev->in_sync && mddev->recovery_cp == MaxSector) + && !mddev->in_sync && mddev->resync_offset == MaxSector) )) return; @@ -9771,8 +9783,8 @@ void md_check_recovery(struct mddev *mddev) * remove disk. */ rdev_for_each_safe(rdev, tmp, mddev) { - if (test_and_clear_bit(ClusterRemove, &rdev->flags) && - rdev->raid_disk < 0) + if (rdev->raid_disk < 0 && + test_and_clear_bit(ClusterRemove, &rdev->flags)) md_kick_rdev_from_array(rdev); } } @@ -10078,8 +10090,11 @@ static void check_sb_changes(struct mddev *mddev, struct md_rdev *rdev) /* Check for change of roles in the active devices */ rdev_for_each_safe(rdev2, tmp, mddev) { - if (test_bit(Faulty, &rdev2->flags)) + if (test_bit(Faulty, &rdev2->flags)) { + if (test_bit(ClusterRemove, &rdev2->flags)) + set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); continue; + } /* Check if the roles changed */ role = le16_to_cpu(sb->dev_roles[rdev2->desc_nr]); diff --git a/drivers/md/md.h b/drivers/md/md.h index 67b365621507..51af29a03079 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -523,7 +523,7 @@ struct mddev { unsigned long normal_io_events; /* IO event timestamp */ atomic_t recovery_active; /* blocks scheduled, but not written */ wait_queue_head_t recovery_wait; - sector_t recovery_cp; + sector_t resync_offset; sector_t resync_min; /* user requested sync * starts here */ sector_t resync_max; /* resync should pause diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index cbe2a9054cb9..f1d8811a542a 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -674,7 +674,7 @@ static void *raid0_takeover_raid45(struct mddev *mddev) mddev->raid_disks--; mddev->delta_disks = -1; /* make sure it will be not marked as dirty */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); create_strip_zones(mddev, &priv_conf); @@ -717,7 +717,7 @@ static void *raid0_takeover_raid10(struct mddev *mddev) mddev->raid_disks += mddev->delta_disks; mddev->degraded = 0; /* make sure it will be not marked as dirty */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); create_strip_zones(mddev, &priv_conf); @@ -760,7 +760,7 @@ static void *raid0_takeover_raid1(struct mddev *mddev) mddev->delta_disks = 1 - mddev->raid_disks; mddev->raid_disks = 1; /* make sure it will be not marked as dirty */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; mddev_clear_unsupported_flags(mddev, UNSUPPORTED_MDDEV_FLAGS); create_strip_zones(mddev, &priv_conf); diff --git a/drivers/md/raid1-10.c b/drivers/md/raid1-10.c index b8b3a9069701..52881e6032da 100644 --- a/drivers/md/raid1-10.c +++ b/drivers/md/raid1-10.c @@ -283,7 +283,7 @@ static inline int raid1_check_read_range(struct md_rdev *rdev, static inline bool raid1_should_read_first(struct mddev *mddev, sector_t this_sector, int len) { - if ((mddev->recovery_cp < this_sector + len)) + if ((mddev->resync_offset < this_sector + len)) return true; if (mddev_is_clustered(mddev) && diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 64b8176907a9..408c26398321 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -127,10 +127,9 @@ static inline struct r1bio *get_resync_r1bio(struct bio *bio) return get_resync_pages(bio)->raid_bio; } -static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) +static void *r1bio_pool_alloc(gfp_t gfp_flags, struct r1conf *conf) { - struct pool_info *pi = data; - int size = offsetof(struct r1bio, bios[pi->raid_disks]); + int size = offsetof(struct r1bio, bios[conf->raid_disks * 2]); /* allocate a r1bio with room for raid_disks entries in the bios array */ return kzalloc(size, gfp_flags); @@ -145,18 +144,18 @@ static void * r1bio_pool_alloc(gfp_t gfp_flags, void *data) static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) { - struct pool_info *pi = data; + struct r1conf *conf = data; struct r1bio *r1_bio; struct bio *bio; int need_pages; int j; struct resync_pages *rps; - r1_bio = r1bio_pool_alloc(gfp_flags, pi); + r1_bio = r1bio_pool_alloc(gfp_flags, conf); if (!r1_bio) return NULL; - rps = kmalloc_array(pi->raid_disks, sizeof(struct resync_pages), + rps = kmalloc_array(conf->raid_disks * 2, sizeof(struct resync_pages), gfp_flags); if (!rps) goto out_free_r1bio; @@ -164,7 +163,7 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) /* * Allocate bios : 1 for reading, n-1 for writing */ - for (j = pi->raid_disks ; j-- ; ) { + for (j = conf->raid_disks * 2; j-- ; ) { bio = bio_kmalloc(RESYNC_PAGES, gfp_flags); if (!bio) goto out_free_bio; @@ -177,11 +176,11 @@ static void * r1buf_pool_alloc(gfp_t gfp_flags, void *data) * If this is a user-requested check/repair, allocate * RESYNC_PAGES for each bio. */ - if (test_bit(MD_RECOVERY_REQUESTED, &pi->mddev->recovery)) - need_pages = pi->raid_disks; + if (test_bit(MD_RECOVERY_REQUESTED, &conf->mddev->recovery)) + need_pages = conf->raid_disks * 2; else need_pages = 1; - for (j = 0; j < pi->raid_disks; j++) { + for (j = 0; j < conf->raid_disks * 2; j++) { struct resync_pages *rp = &rps[j]; bio = r1_bio->bios[j]; @@ -207,7 +206,7 @@ out_free_pages: resync_free_pages(&rps[j]); out_free_bio: - while (++j < pi->raid_disks) { + while (++j < conf->raid_disks * 2) { bio_uninit(r1_bio->bios[j]); kfree(r1_bio->bios[j]); } @@ -220,12 +219,12 @@ out_free_r1bio: static void r1buf_pool_free(void *__r1_bio, void *data) { - struct pool_info *pi = data; + struct r1conf *conf = data; int i; struct r1bio *r1bio = __r1_bio; struct resync_pages *rp = NULL; - for (i = pi->raid_disks; i--; ) { + for (i = conf->raid_disks * 2; i--; ) { rp = get_resync_pages(r1bio->bios[i]); resync_free_pages(rp); bio_uninit(r1bio->bios[i]); @@ -255,7 +254,7 @@ static void free_r1bio(struct r1bio *r1_bio) struct r1conf *conf = r1_bio->mddev->private; put_all_bios(conf, r1_bio); - mempool_free(r1_bio, &conf->r1bio_pool); + mempool_free(r1_bio, conf->r1bio_pool); } static void put_buf(struct r1bio *r1_bio) @@ -1305,9 +1304,8 @@ alloc_r1bio(struct mddev *mddev, struct bio *bio) struct r1conf *conf = mddev->private; struct r1bio *r1_bio; - r1_bio = mempool_alloc(&conf->r1bio_pool, GFP_NOIO); - /* Ensure no bio records IO_BLOCKED */ - memset(r1_bio->bios, 0, conf->raid_disks * sizeof(r1_bio->bios[0])); + r1_bio = mempool_alloc(conf->r1bio_pool, GFP_NOIO); + memset(r1_bio, 0, offsetof(struct r1bio, bios[conf->raid_disks * 2])); init_r1bio(r1_bio, mddev, bio); return r1_bio; } @@ -2747,7 +2745,7 @@ static int init_resync(struct r1conf *conf) BUG_ON(mempool_initialized(&conf->r1buf_pool)); return mempool_init(&conf->r1buf_pool, buffs, r1buf_pool_alloc, - r1buf_pool_free, conf->poolinfo); + r1buf_pool_free, conf); } static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf) @@ -2757,7 +2755,7 @@ static struct r1bio *raid1_alloc_init_r1buf(struct r1conf *conf) struct bio *bio; int i; - for (i = conf->poolinfo->raid_disks; i--; ) { + for (i = conf->raid_disks * 2; i--; ) { bio = r1bio->bios[i]; rps = bio->bi_private; bio_reset(bio, NULL, 0); @@ -2822,7 +2820,7 @@ static sector_t raid1_sync_request(struct mddev *mddev, sector_t sector_nr, } if (mddev->bitmap == NULL && - mddev->recovery_cp == MaxSector && + mddev->resync_offset == MaxSector && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && conf->fullsync == 0) { *skipped = 1; @@ -3085,6 +3083,7 @@ static struct r1conf *setup_conf(struct mddev *mddev) int i; struct raid1_info *disk; struct md_rdev *rdev; + size_t r1bio_size; int err = -ENOMEM; conf = kzalloc(sizeof(struct r1conf), GFP_KERNEL); @@ -3121,21 +3120,15 @@ static struct r1conf *setup_conf(struct mddev *mddev) if (!conf->tmppage) goto abort; - conf->poolinfo = kzalloc(sizeof(*conf->poolinfo), GFP_KERNEL); - if (!conf->poolinfo) - goto abort; - conf->poolinfo->raid_disks = mddev->raid_disks * 2; - err = mempool_init(&conf->r1bio_pool, NR_RAID_BIOS, r1bio_pool_alloc, - rbio_pool_free, conf->poolinfo); - if (err) + r1bio_size = offsetof(struct r1bio, bios[mddev->raid_disks * 2]); + conf->r1bio_pool = mempool_create_kmalloc_pool(NR_RAID_BIOS, r1bio_size); + if (!conf->r1bio_pool) goto abort; err = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0); if (err) goto abort; - conf->poolinfo->mddev = mddev; - err = -EINVAL; spin_lock_init(&conf->device_lock); conf->raid_disks = mddev->raid_disks; @@ -3198,10 +3191,9 @@ static struct r1conf *setup_conf(struct mddev *mddev) abort: if (conf) { - mempool_exit(&conf->r1bio_pool); + mempool_destroy(conf->r1bio_pool); kfree(conf->mirrors); safe_put_page(conf->tmppage); - kfree(conf->poolinfo); kfree(conf->nr_pending); kfree(conf->nr_waiting); kfree(conf->nr_queued); @@ -3282,9 +3274,9 @@ static int raid1_run(struct mddev *mddev) } if (conf->raid_disks - mddev->degraded == 1) - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; - if (mddev->recovery_cp != MaxSector) + if (mddev->resync_offset != MaxSector) pr_info("md/raid1:%s: not clean -- starting background reconstruction\n", mdname(mddev)); pr_info("md/raid1:%s: active with %d out of %d mirrors\n", @@ -3311,10 +3303,9 @@ static void raid1_free(struct mddev *mddev, void *priv) { struct r1conf *conf = priv; - mempool_exit(&conf->r1bio_pool); + mempool_destroy(conf->r1bio_pool); kfree(conf->mirrors); safe_put_page(conf->tmppage); - kfree(conf->poolinfo); kfree(conf->nr_pending); kfree(conf->nr_waiting); kfree(conf->nr_queued); @@ -3345,8 +3336,8 @@ static int raid1_resize(struct mddev *mddev, sector_t sectors) md_set_array_sectors(mddev, newsize); if (sectors > mddev->dev_sectors && - mddev->recovery_cp > mddev->dev_sectors) { - mddev->recovery_cp = mddev->dev_sectors; + mddev->resync_offset > mddev->dev_sectors) { + mddev->resync_offset = mddev->dev_sectors; set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); } mddev->dev_sectors = sectors; @@ -3367,17 +3358,13 @@ static int raid1_reshape(struct mddev *mddev) * At the same time, we "pack" the devices so that all the missing * devices have the higher raid_disk numbers. */ - mempool_t newpool, oldpool; - struct pool_info *newpoolinfo; + mempool_t *newpool, *oldpool; + size_t new_r1bio_size; struct raid1_info *newmirrors; struct r1conf *conf = mddev->private; int cnt, raid_disks; unsigned long flags; int d, d2; - int ret; - - memset(&newpool, 0, sizeof(newpool)); - memset(&oldpool, 0, sizeof(oldpool)); /* Cannot change chunk_size, layout, or level */ if (mddev->chunk_sectors != mddev->new_chunk_sectors || @@ -3403,24 +3390,16 @@ static int raid1_reshape(struct mddev *mddev) return -EBUSY; } - newpoolinfo = kmalloc(sizeof(*newpoolinfo), GFP_KERNEL); - if (!newpoolinfo) + new_r1bio_size = offsetof(struct r1bio, bios[raid_disks * 2]); + newpool = mempool_create_kmalloc_pool(NR_RAID_BIOS, new_r1bio_size); + if (!newpool) { return -ENOMEM; - newpoolinfo->mddev = mddev; - newpoolinfo->raid_disks = raid_disks * 2; - - ret = mempool_init(&newpool, NR_RAID_BIOS, r1bio_pool_alloc, - rbio_pool_free, newpoolinfo); - if (ret) { - kfree(newpoolinfo); - return ret; } newmirrors = kzalloc(array3_size(sizeof(struct raid1_info), raid_disks, 2), GFP_KERNEL); if (!newmirrors) { - kfree(newpoolinfo); - mempool_exit(&newpool); + mempool_destroy(newpool); return -ENOMEM; } @@ -3429,7 +3408,6 @@ static int raid1_reshape(struct mddev *mddev) /* ok, everything is stopped */ oldpool = conf->r1bio_pool; conf->r1bio_pool = newpool; - init_waitqueue_head(&conf->r1bio_pool.wait); for (d = d2 = 0; d < conf->raid_disks; d++) { struct md_rdev *rdev = conf->mirrors[d].rdev; @@ -3446,8 +3424,6 @@ static int raid1_reshape(struct mddev *mddev) } kfree(conf->mirrors); conf->mirrors = newmirrors; - kfree(conf->poolinfo); - conf->poolinfo = newpoolinfo; spin_lock_irqsave(&conf->device_lock, flags); mddev->degraded += (raid_disks - conf->raid_disks); @@ -3461,7 +3437,7 @@ static int raid1_reshape(struct mddev *mddev) set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); md_wakeup_thread(mddev->thread); - mempool_exit(&oldpool); + mempool_destroy(oldpool); return 0; } diff --git a/drivers/md/raid1.h b/drivers/md/raid1.h index 33f318fcc268..d236ef179cfb 100644 --- a/drivers/md/raid1.h +++ b/drivers/md/raid1.h @@ -49,22 +49,6 @@ struct raid1_info { sector_t seq_start; }; -/* - * memory pools need a pointer to the mddev, so they can force an unplug - * when memory is tight, and a count of the number of drives that the - * pool was allocated for, so they know how much to allocate and free. - * mddev->raid_disks cannot be used, as it can change while a pool is active - * These two datums are stored in a kmalloced struct. - * The 'raid_disks' here is twice the raid_disks in r1conf. - * This allows space for each 'real' device can have a replacement in the - * second half of the array. - */ - -struct pool_info { - struct mddev *mddev; - int raid_disks; -}; - struct r1conf { struct mddev *mddev; struct raid1_info *mirrors; /* twice 'raid_disks' to @@ -114,11 +98,7 @@ struct r1conf { */ int recovery_disabled; - /* poolinfo contains information about the content of the - * mempools - it changes when the array grows or shrinks - */ - struct pool_info *poolinfo; - mempool_t r1bio_pool; + mempool_t *r1bio_pool; mempool_t r1buf_pool; struct bio_set bio_split; diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 95dc354a86a0..b60c30bfb6c7 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -2117,7 +2117,7 @@ static int raid10_add_disk(struct mddev *mddev, struct md_rdev *rdev) int last = conf->geo.raid_disks - 1; struct raid10_info *p; - if (mddev->recovery_cp < MaxSector) + if (mddev->resync_offset < MaxSector) /* only hot-add to in-sync arrays, as recovery is * very different from resync */ @@ -3185,7 +3185,7 @@ static sector_t raid10_sync_request(struct mddev *mddev, sector_t sector_nr, * of a clean array, like RAID1 does. */ if (mddev->bitmap == NULL && - mddev->recovery_cp == MaxSector && + mddev->resync_offset == MaxSector && mddev->reshape_position == MaxSector && !test_bit(MD_RECOVERY_SYNC, &mddev->recovery) && !test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) && @@ -4145,7 +4145,7 @@ static int raid10_run(struct mddev *mddev) disk->recovery_disabled = mddev->recovery_disabled - 1; } - if (mddev->recovery_cp != MaxSector) + if (mddev->resync_offset != MaxSector) pr_notice("md/raid10:%s: not clean -- starting background reconstruction\n", mdname(mddev)); pr_info("md/raid10:%s: active with %d out of %d devices\n", @@ -4245,8 +4245,8 @@ static int raid10_resize(struct mddev *mddev, sector_t sectors) md_set_array_sectors(mddev, size); if (sectors > mddev->dev_sectors && - mddev->recovery_cp > oldsize) { - mddev->recovery_cp = oldsize; + mddev->resync_offset > oldsize) { + mddev->resync_offset = oldsize; set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); } calc_sectors(conf, sectors); @@ -4275,7 +4275,7 @@ static void *raid10_takeover_raid0(struct mddev *mddev, sector_t size, int devs) mddev->delta_disks = mddev->raid_disks; mddev->raid_disks *= 2; /* make sure it will be not marked as dirty */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; mddev->dev_sectors = size; conf = setup_conf(mddev); @@ -5087,8 +5087,8 @@ static void raid10_finish_reshape(struct mddev *mddev) return; if (mddev->delta_disks > 0) { - if (mddev->recovery_cp > mddev->resync_max_sectors) { - mddev->recovery_cp = mddev->resync_max_sectors; + if (mddev->resync_offset > mddev->resync_max_sectors) { + mddev->resync_offset = mddev->resync_max_sectors; set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); } mddev->resync_max_sectors = mddev->array_sectors; diff --git a/drivers/md/raid5-ppl.c b/drivers/md/raid5-ppl.c index c0fb335311aa..56b234683ee6 100644 --- a/drivers/md/raid5-ppl.c +++ b/drivers/md/raid5-ppl.c @@ -1163,7 +1163,7 @@ static int ppl_load_distributed(struct ppl_log *log) le64_to_cpu(pplhdr->generation)); /* attempt to recover from log if we are starting a dirty array */ - if (pplhdr && !mddev->pers && mddev->recovery_cp != MaxSector) + if (pplhdr && !mddev->pers && mddev->resync_offset != MaxSector) ret = ppl_recover(log, pplhdr, pplhdr_offset); /* write empty header if we are starting the array */ @@ -1422,14 +1422,14 @@ int ppl_init_log(struct r5conf *conf) if (ret) { goto err; - } else if (!mddev->pers && mddev->recovery_cp == 0 && + } else if (!mddev->pers && mddev->resync_offset == 0 && ppl_conf->recovered_entries > 0 && ppl_conf->mismatch_count == 0) { /* * If we are starting a dirty array and the recovery succeeds * without any issues, set the array as clean. */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; set_bit(MD_SB_CHANGE_CLEAN, &mddev->sb_flags); } else if (mddev->pers && ppl_conf->mismatch_count > 0) { /* no mismatch allowed when enabling PPL for a running array */ diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 7ec61ee7b218..023649fe2476 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -3740,7 +3740,7 @@ static int want_replace(struct stripe_head *sh, int disk_idx) && !test_bit(Faulty, &rdev->flags) && !test_bit(In_sync, &rdev->flags) && (rdev->recovery_offset <= sh->sector - || rdev->mddev->recovery_cp <= sh->sector)) + || rdev->mddev->resync_offset <= sh->sector)) rv = 1; return rv; } @@ -3832,7 +3832,7 @@ static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s, * is missing/faulty, then we need to read everything we can. */ if (!force_rcw && - sh->sector < sh->raid_conf->mddev->recovery_cp) + sh->sector < sh->raid_conf->mddev->resync_offset) /* reconstruct-write isn't being forced */ return 0; for (i = 0; i < s->failed && i < 2; i++) { @@ -4097,7 +4097,7 @@ static int handle_stripe_dirtying(struct r5conf *conf, int disks) { int rmw = 0, rcw = 0, i; - sector_t recovery_cp = conf->mddev->recovery_cp; + sector_t resync_offset = conf->mddev->resync_offset; /* Check whether resync is now happening or should start. * If yes, then the array is dirty (after unclean shutdown or @@ -4107,14 +4107,14 @@ static int handle_stripe_dirtying(struct r5conf *conf, * generate correct data from the parity. */ if (conf->rmw_level == PARITY_DISABLE_RMW || - (recovery_cp < MaxSector && sh->sector >= recovery_cp && + (resync_offset < MaxSector && sh->sector >= resync_offset && s->failed == 0)) { /* Calculate the real rcw later - for now make it * look like rcw is cheaper */ rcw = 1; rmw = 2; - pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n", - conf->rmw_level, (unsigned long long)recovery_cp, + pr_debug("force RCW rmw_level=%u, resync_offset=%llu sh->sector=%llu\n", + conf->rmw_level, (unsigned long long)resync_offset, (unsigned long long)sh->sector); } else for (i = disks; i--; ) { /* would I have to read this buffer for read_modify_write */ @@ -4770,14 +4770,14 @@ static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s) if (test_bit(STRIPE_SYNCING, &sh->state)) { /* If there is a failed device being replaced, * we must be recovering. - * else if we are after recovery_cp, we must be syncing + * else if we are after resync_offset, we must be syncing * else if MD_RECOVERY_REQUESTED is set, we also are syncing. * else we can only be replacing * sync and recovery both need to read all devices, and so * use the same flag. */ if (do_recovery || - sh->sector >= conf->mddev->recovery_cp || + sh->sector >= conf->mddev->resync_offset || test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery))) s->syncing = 1; else @@ -7780,7 +7780,7 @@ static int raid5_run(struct mddev *mddev) int first = 1; int ret = -EIO; - if (mddev->recovery_cp != MaxSector) + if (mddev->resync_offset != MaxSector) pr_notice("md/raid:%s: not clean -- starting background reconstruction\n", mdname(mddev)); @@ -7921,7 +7921,7 @@ static int raid5_run(struct mddev *mddev) mdname(mddev)); mddev->ro = 1; set_disk_ro(mddev->gendisk, 1); - } else if (mddev->recovery_cp == MaxSector) + } else if (mddev->resync_offset == MaxSector) set_bit(MD_JOURNAL_CLEAN, &mddev->flags); } @@ -7988,7 +7988,7 @@ static int raid5_run(struct mddev *mddev) mddev->resync_max_sectors = mddev->dev_sectors; if (mddev->degraded > dirty_parity_disks && - mddev->recovery_cp != MaxSector) { + mddev->resync_offset != MaxSector) { if (test_bit(MD_HAS_PPL, &mddev->flags)) pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n", mdname(mddev)); @@ -8328,8 +8328,8 @@ static int raid5_resize(struct mddev *mddev, sector_t sectors) md_set_array_sectors(mddev, newsize); if (sectors > mddev->dev_sectors && - mddev->recovery_cp > mddev->dev_sectors) { - mddev->recovery_cp = mddev->dev_sectors; + mddev->resync_offset > mddev->dev_sectors) { + mddev->resync_offset = mddev->dev_sectors; set_bit(MD_RECOVERY_NEEDED, &mddev->recovery); } mddev->dev_sectors = sectors; @@ -8423,7 +8423,7 @@ static int raid5_start_reshape(struct mddev *mddev) return -EINVAL; /* raid5 can't handle concurrent reshape and recovery */ - if (mddev->recovery_cp < MaxSector) + if (mddev->resync_offset < MaxSector) return -EBUSY; for (i = 0; i < conf->raid_disks; i++) if (conf->disks[i].replacement) @@ -8648,7 +8648,7 @@ static void *raid45_takeover_raid0(struct mddev *mddev, int level) mddev->raid_disks += 1; mddev->delta_disks = 1; /* make sure it will be not marked as dirty */ - mddev->recovery_cp = MaxSector; + mddev->resync_offset = MaxSector; return setup_conf(mddev); } diff --git a/drivers/net/ethernet/airoha/airoha_npu.c b/drivers/net/ethernet/airoha/airoha_npu.c index 9ab964c536e1..a802f95df99d 100644 --- a/drivers/net/ethernet/airoha/airoha_npu.c +++ b/drivers/net/ethernet/airoha/airoha_npu.c @@ -579,6 +579,8 @@ static struct platform_driver airoha_npu_driver = { }; module_platform_driver(airoha_npu_driver); +MODULE_FIRMWARE(NPU_EN7581_FIRMWARE_DATA); +MODULE_FIRMWARE(NPU_EN7581_FIRMWARE_RV32); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Lorenzo Bianconi <lorenzo@kernel.org>"); MODULE_DESCRIPTION("Airoha Network Processor Unit driver"); diff --git a/drivers/net/ethernet/airoha/airoha_ppe.c b/drivers/net/ethernet/airoha/airoha_ppe.c index c354d536bc66..47411d2cbd28 100644 --- a/drivers/net/ethernet/airoha/airoha_ppe.c +++ b/drivers/net/ethernet/airoha/airoha_ppe.c @@ -508,9 +508,11 @@ static void airoha_ppe_foe_flow_stats_update(struct airoha_ppe *ppe, FIELD_PREP(AIROHA_FOE_IB2_NBQ, nbq); } -struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, - u32 hash) +static struct airoha_foe_entry * +airoha_ppe_foe_get_entry_locked(struct airoha_ppe *ppe, u32 hash) { + lockdep_assert_held(&ppe_lock); + if (hash < PPE_SRAM_NUM_ENTRIES) { u32 *hwe = ppe->foe + hash * sizeof(struct airoha_foe_entry); struct airoha_eth *eth = ppe->eth; @@ -537,6 +539,18 @@ struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, return ppe->foe + hash * sizeof(struct airoha_foe_entry); } +struct airoha_foe_entry *airoha_ppe_foe_get_entry(struct airoha_ppe *ppe, + u32 hash) +{ + struct airoha_foe_entry *hwe; + + spin_lock_bh(&ppe_lock); + hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); + spin_unlock_bh(&ppe_lock); + + return hwe; +} + static bool airoha_ppe_foe_compare_entry(struct airoha_flow_table_entry *e, struct airoha_foe_entry *hwe) { @@ -651,7 +665,7 @@ airoha_ppe_foe_commit_subflow_entry(struct airoha_ppe *ppe, struct airoha_flow_table_entry *f; int type; - hwe_p = airoha_ppe_foe_get_entry(ppe, hash); + hwe_p = airoha_ppe_foe_get_entry_locked(ppe, hash); if (!hwe_p) return -EINVAL; @@ -703,7 +717,7 @@ static void airoha_ppe_foe_insert_entry(struct airoha_ppe *ppe, spin_lock_bh(&ppe_lock); - hwe = airoha_ppe_foe_get_entry(ppe, hash); + hwe = airoha_ppe_foe_get_entry_locked(ppe, hash); if (!hwe) goto unlock; @@ -818,7 +832,7 @@ airoha_ppe_foe_flow_l2_entry_update(struct airoha_ppe *ppe, u32 ib1, state; int idle; - hwe = airoha_ppe_foe_get_entry(ppe, iter->hash); + hwe = airoha_ppe_foe_get_entry_locked(ppe, iter->hash); if (!hwe) continue; @@ -855,7 +869,7 @@ static void airoha_ppe_foe_flow_entry_update(struct airoha_ppe *ppe, if (e->hash == 0xffff) goto unlock; - hwe_p = airoha_ppe_foe_get_entry(ppe, e->hash); + hwe_p = airoha_ppe_foe_get_entry_locked(ppe, e->hash); if (!hwe_p) goto unlock; diff --git a/drivers/net/ethernet/emulex/benet/be_cmds.c b/drivers/net/ethernet/emulex/benet/be_cmds.c index d730af4a50c7..bb5d2fa15736 100644 --- a/drivers/net/ethernet/emulex/benet/be_cmds.c +++ b/drivers/net/ethernet/emulex/benet/be_cmds.c @@ -3856,8 +3856,8 @@ int be_cmd_set_mac_list(struct be_adapter *adapter, u8 *mac_array, status = be_mcc_notify_wait(adapter); err: - dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); spin_unlock_bh(&adapter->mcc_lock); + dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); return status; } diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 5d0c0906878d..a863f7841210 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1750,16 +1750,17 @@ err_register_mdiobus: static void ftgmac100_phy_disconnect(struct net_device *netdev) { struct ftgmac100 *priv = netdev_priv(netdev); + struct phy_device *phydev = netdev->phydev; - if (!netdev->phydev) + if (!phydev) return; - phy_disconnect(netdev->phydev); + phy_disconnect(phydev); if (of_phy_is_fixed_link(priv->dev->of_node)) of_phy_deregister_fixed_link(priv->dev->of_node); if (priv->use_ncsi) - fixed_phy_unregister(netdev->phydev); + fixed_phy_unregister(phydev); } static void ftgmac100_destroy_mdio(struct net_device *netdev) diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c index 0c588e03b15e..d09e456f14c0 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_ethtool.c @@ -371,8 +371,10 @@ static int dpaa_get_ts_info(struct net_device *net_dev, of_node_put(ptp_node); } - if (ptp_dev) + if (ptp_dev) { ptp = platform_get_drvdata(ptp_dev); + put_device(&ptp_dev->dev); + } if (ptp) info->phc_index = ptp->phc_index; diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index f63a29e2e031..de0fb272c847 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -829,19 +829,29 @@ static int enetc_pf_register_with_ierb(struct pci_dev *pdev) { struct platform_device *ierb_pdev; struct device_node *ierb_node; + int ret; ierb_node = of_find_compatible_node(NULL, NULL, "fsl,ls1028a-enetc-ierb"); - if (!ierb_node || !of_device_is_available(ierb_node)) + if (!ierb_node) return -ENODEV; + if (!of_device_is_available(ierb_node)) { + of_node_put(ierb_node); + return -ENODEV; + } + ierb_pdev = of_find_device_by_node(ierb_node); of_node_put(ierb_node); if (!ierb_pdev) return -EPROBE_DEFER; - return enetc_ierb_register_pf(ierb_pdev, pdev); + ret = enetc_ierb_register_pf(ierb_pdev, pdev); + + put_device(&ierb_pdev->dev); + + return ret; } static const struct enetc_si_ops enetc_psi_ops = { diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index 28f53cf2a174..5fd1f7327680 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -1475,8 +1475,10 @@ static int gfar_get_ts_info(struct net_device *dev, if (ptp_node) { ptp_dev = of_find_device_by_node(ptp_node); of_node_put(ptp_node); - if (ptp_dev) + if (ptp_dev) { ptp = platform_get_drvdata(ptp_dev); + put_device(&ptp_dev->dev); + } } if (ptp) diff --git a/drivers/net/ethernet/mediatek/mtk_wed.c b/drivers/net/ethernet/mediatek/mtk_wed.c index 73c26fcfd85e..0a80d8f8cff7 100644 --- a/drivers/net/ethernet/mediatek/mtk_wed.c +++ b/drivers/net/ethernet/mediatek/mtk_wed.c @@ -2782,7 +2782,6 @@ void mtk_wed_add_hw(struct device_node *np, struct mtk_eth *eth, if (!pdev) goto err_of_node_put; - get_device(&pdev->dev); irq = platform_get_irq(pdev, 0); if (irq < 0) goto err_put_device; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 218b1a09534c..b8c609d91d11 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -1574,6 +1574,7 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe, unsigned int hdrlen = mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt); skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt - hdrlen, lro_num_seg); + skb_shinfo(skb)->gso_segs = lro_num_seg; /* Subtract one since we already counted this as one * "regular" packet in mlx5e_complete_rx_cqe() */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c index 7bd7812d9c06..e67e99487a27 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -33,7 +33,7 @@ int __fbnic_open(struct fbnic_net *fbn) dev_warn(fbd->dev, "Error %d sending host ownership message to the firmware\n", err); - goto free_resources; + goto err_reset_queues; } err = fbnic_time_start(fbn); @@ -57,6 +57,8 @@ time_stop: fbnic_time_stop(fbn); release_ownership: fbnic_fw_xmit_ownership_msg(fbn->fbd, false); +err_reset_queues: + fbnic_reset_netif_queues(fbn); free_resources: fbnic_free_resources(fbn); free_napi_vectors: @@ -420,15 +422,17 @@ static void fbnic_get_stats64(struct net_device *dev, tx_packets = stats->packets; tx_dropped = stats->dropped; - stats64->tx_bytes = tx_bytes; - stats64->tx_packets = tx_packets; - stats64->tx_dropped = tx_dropped; - /* Record drops from Tx HW Datapath */ + spin_lock(&fbd->hw_stats_lock); tx_dropped += fbd->hw_stats.tmi.drop.frames.value + fbd->hw_stats.tti.cm_drop.frames.value + fbd->hw_stats.tti.frame_drop.frames.value + fbd->hw_stats.tti.tbi_drop.frames.value; + spin_unlock(&fbd->hw_stats_lock); + + stats64->tx_bytes = tx_bytes; + stats64->tx_packets = tx_packets; + stats64->tx_dropped = tx_dropped; for (i = 0; i < fbn->num_tx_queues; i++) { struct fbnic_ring *txr = fbn->tx[i]; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c index ac11389a764c..f9543d03485f 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -661,8 +661,8 @@ static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, { struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; - page_pool_fragment_page(page, PAGECNT_BIAS_MAX); - rx_buf->pagecnt_bias = PAGECNT_BIAS_MAX; + page_pool_fragment_page(page, FBNIC_PAGECNT_BIAS_MAX); + rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; rx_buf->page = page; } diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h index 2e361d6f03ff..34693596e5eb 100644 --- a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -91,10 +91,8 @@ struct fbnic_queue_stats { struct u64_stats_sync syncp; }; -/* Pagecnt bias is long max to reserve the last bit to catch overflow - * cases where if we overcharge the bias it will flip over to be negative. - */ -#define PAGECNT_BIAS_MAX LONG_MAX +#define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE + struct fbnic_rx_buf { struct page *page; long pagecnt_bias; diff --git a/drivers/net/ethernet/sfc/tc_encap_actions.c b/drivers/net/ethernet/sfc/tc_encap_actions.c index 2258f854e5be..e872f926e438 100644 --- a/drivers/net/ethernet/sfc/tc_encap_actions.c +++ b/drivers/net/ethernet/sfc/tc_encap_actions.c @@ -442,7 +442,7 @@ static void efx_tc_update_encap(struct efx_nic *efx, rule = container_of(acts, struct efx_tc_flow_rule, acts); if (rule->fallback) fallback = rule->fallback; - else /* fallback: deliver to PF */ + else /* fallback of the fallback: deliver to PF */ fallback = &efx->tc->facts.pf; rc = efx_mae_update_rule(efx, fallback->fw_id, rule->fw_id); diff --git a/drivers/net/ethernet/ti/icssg/icss_iep.c b/drivers/net/ethernet/ti/icssg/icss_iep.c index 2a1c43316f46..50bfbc2779e4 100644 --- a/drivers/net/ethernet/ti/icssg/icss_iep.c +++ b/drivers/net/ethernet/ti/icssg/icss_iep.c @@ -685,11 +685,17 @@ struct icss_iep *icss_iep_get_idx(struct device_node *np, int idx) struct platform_device *pdev; struct device_node *iep_np; struct icss_iep *iep; + int ret; iep_np = of_parse_phandle(np, "ti,iep", idx); - if (!iep_np || !of_device_is_available(iep_np)) + if (!iep_np) return ERR_PTR(-ENODEV); + if (!of_device_is_available(iep_np)) { + of_node_put(iep_np); + return ERR_PTR(-ENODEV); + } + pdev = of_find_device_by_node(iep_np); of_node_put(iep_np); @@ -698,21 +704,28 @@ struct icss_iep *icss_iep_get_idx(struct device_node *np, int idx) return ERR_PTR(-EPROBE_DEFER); iep = platform_get_drvdata(pdev); - if (!iep) - return ERR_PTR(-EPROBE_DEFER); + if (!iep) { + ret = -EPROBE_DEFER; + goto err_put_pdev; + } device_lock(iep->dev); if (iep->client_np) { device_unlock(iep->dev); dev_err(iep->dev, "IEP is already acquired by %s", iep->client_np->name); - return ERR_PTR(-EBUSY); + ret = -EBUSY; + goto err_put_pdev; } iep->client_np = np; device_unlock(iep->dev); - get_device(iep->dev); return iep; + +err_put_pdev: + put_device(&pdev->dev); + + return ERR_PTR(ret); } EXPORT_SYMBOL_GPL(icss_iep_get_idx); diff --git a/drivers/net/ethernet/ti/icssg/icssg_common.c b/drivers/net/ethernet/ti/icssg/icssg_common.c index 12f25cec6255..57e5f1c88f50 100644 --- a/drivers/net/ethernet/ti/icssg/icssg_common.c +++ b/drivers/net/ethernet/ti/icssg/icssg_common.c @@ -706,9 +706,9 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) struct page_pool *pool; struct sk_buff *skb; struct xdp_buff xdp; + int headroom, ret; u32 *psdata; void *pa; - int ret; *xdp_state = 0; pool = rx_chn->pg_pool; @@ -757,22 +757,23 @@ static int emac_rx_packet(struct prueth_emac *emac, u32 flow_id, u32 *xdp_state) xdp_prepare_buff(&xdp, pa, PRUETH_HEADROOM, pkt_len, false); *xdp_state = emac_run_xdp(emac, &xdp, page, &pkt_len); - if (*xdp_state == ICSSG_XDP_PASS) - skb = xdp_build_skb_from_buff(&xdp); - else + if (*xdp_state != ICSSG_XDP_PASS) goto requeue; + headroom = xdp.data - xdp.data_hard_start; + pkt_len = xdp.data_end - xdp.data; } else { - /* prepare skb and send to n/w stack */ - skb = napi_build_skb(pa, PAGE_SIZE); + headroom = PRUETH_HEADROOM; } + /* prepare skb and send to n/w stack */ + skb = napi_build_skb(pa, PAGE_SIZE); if (!skb) { ndev->stats.rx_dropped++; page_pool_recycle_direct(pool, page); goto requeue; } - skb_reserve(skb, PRUETH_HEADROOM); + skb_reserve(skb, headroom); skb_put(skb, pkt_len); skb->dev = ndev; diff --git a/drivers/net/ipa/Kconfig b/drivers/net/ipa/Kconfig index 6782c2cbf542..01d219d3760c 100644 --- a/drivers/net/ipa/Kconfig +++ b/drivers/net/ipa/Kconfig @@ -5,7 +5,7 @@ config QCOM_IPA depends on INTERCONNECT depends on QCOM_RPROC_COMMON || (QCOM_RPROC_COMMON=n && COMPILE_TEST) depends on QCOM_AOSS_QMP || QCOM_AOSS_QMP=n - select QCOM_MDT_LOADER if ARCH_QCOM + select QCOM_MDT_LOADER select QCOM_SCM select QCOM_QMI_HELPERS help diff --git a/drivers/net/ipa/ipa_sysfs.c b/drivers/net/ipa/ipa_sysfs.c index a59bd215494c..a53e9e6f6cdf 100644 --- a/drivers/net/ipa/ipa_sysfs.c +++ b/drivers/net/ipa/ipa_sysfs.c @@ -37,8 +37,12 @@ static const char *ipa_version_string(struct ipa *ipa) return "4.11"; case IPA_VERSION_5_0: return "5.0"; + case IPA_VERSION_5_1: + return "5.1"; + case IPA_VERSION_5_5: + return "5.5"; default: - return "0.0"; /* Won't happen (checked at probe time) */ + return "0.0"; /* Should not happen */ } } diff --git a/drivers/net/mdio/mdio-bcm-unimac.c b/drivers/net/mdio/mdio-bcm-unimac.c index b6e30bdf5325..7baab230008a 100644 --- a/drivers/net/mdio/mdio-bcm-unimac.c +++ b/drivers/net/mdio/mdio-bcm-unimac.c @@ -209,10 +209,9 @@ static int unimac_mdio_clk_set(struct unimac_mdio_priv *priv) if (ret) return ret; - if (!priv->clk) + rate = clk_get_rate(priv->clk); + if (!rate) rate = 250000000; - else - rate = clk_get_rate(priv->clk); div = (rate / (2 * priv->clk_freq)) - 1; if (div & ~MDIO_CLK_DIV_MASK) { diff --git a/drivers/net/phy/mscc/mscc_ptp.c b/drivers/net/phy/mscc/mscc_ptp.c index 6b800081eed5..275706de5847 100644 --- a/drivers/net/phy/mscc/mscc_ptp.c +++ b/drivers/net/phy/mscc/mscc_ptp.c @@ -900,6 +900,7 @@ static int vsc85xx_eth1_conf(struct phy_device *phydev, enum ts_blk blk, get_unaligned_be32(ptp_multicast)); } else { val |= ANA_ETH1_FLOW_ADDR_MATCH2_ANY_MULTICAST; + val |= ANA_ETH1_FLOW_ADDR_MATCH2_ANY_UNICAST; vsc85xx_ts_write_csr(phydev, blk, MSCC_ANA_ETH1_FLOW_ADDR_MATCH2(0), val); vsc85xx_ts_write_csr(phydev, blk, diff --git a/drivers/net/phy/mscc/mscc_ptp.h b/drivers/net/phy/mscc/mscc_ptp.h index da3465360e90..ae9ad925bfa8 100644 --- a/drivers/net/phy/mscc/mscc_ptp.h +++ b/drivers/net/phy/mscc/mscc_ptp.h @@ -98,6 +98,7 @@ #define MSCC_ANA_ETH1_FLOW_ADDR_MATCH2(x) (MSCC_ANA_ETH1_FLOW_ENA(x) + 3) #define ANA_ETH1_FLOW_ADDR_MATCH2_MASK_MASK GENMASK(22, 20) #define ANA_ETH1_FLOW_ADDR_MATCH2_ANY_MULTICAST 0x400000 +#define ANA_ETH1_FLOW_ADDR_MATCH2_ANY_UNICAST 0x200000 #define ANA_ETH1_FLOW_ADDR_MATCH2_FULL_ADDR 0x100000 #define ANA_ETH1_FLOW_ADDR_MATCH2_SRC_DEST_MASK GENMASK(17, 16) #define ANA_ETH1_FLOW_ADDR_MATCH2_SRC_DEST 0x020000 diff --git a/drivers/net/phy/smsc.c b/drivers/net/phy/smsc.c index b6489da5cfcd..48487149c225 100644 --- a/drivers/net/phy/smsc.c +++ b/drivers/net/phy/smsc.c @@ -785,6 +785,7 @@ static struct phy_driver smsc_phy_driver[] = { /* PHY_BASIC_FEATURES */ + .flags = PHY_RST_AFTER_CLK_EN, .probe = smsc_phy_probe, /* basic functions */ diff --git a/drivers/net/ppp/pptp.c b/drivers/net/ppp/pptp.c index 5feaa70b5f47..90737cb71892 100644 --- a/drivers/net/ppp/pptp.c +++ b/drivers/net/ppp/pptp.c @@ -159,19 +159,17 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) int len; unsigned char *data; __u32 seq_recv; - - struct rtable *rt; struct net_device *tdev; struct iphdr *iph; int max_headroom; if (sk_pppox(po)->sk_state & PPPOX_DEAD) - goto tx_error; + goto tx_drop; rt = pptp_route_output(po, &fl4); if (IS_ERR(rt)) - goto tx_error; + goto tx_drop; tdev = rt->dst.dev; @@ -179,16 +177,20 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) if (skb_headroom(skb) < max_headroom || skb_cloned(skb) || skb_shared(skb)) { struct sk_buff *new_skb = skb_realloc_headroom(skb, max_headroom); - if (!new_skb) { - ip_rt_put(rt); + + if (!new_skb) goto tx_error; - } + if (skb->sk) skb_set_owner_w(new_skb, skb->sk); consume_skb(skb); skb = new_skb; } + /* Ensure we can safely access protocol field and LCP code */ + if (!pskb_may_pull(skb, 3)) + goto tx_error; + data = skb->data; islcp = ((data[0] << 8) + data[1]) == PPP_LCP && 1 <= data[2] && data[2] <= 7; @@ -262,6 +264,8 @@ static int pptp_xmit(struct ppp_channel *chan, struct sk_buff *skb) return 1; tx_error: + ip_rt_put(rt); +tx_drop: kfree_skb(skb); return 1; } diff --git a/drivers/nvme/host/auth.c b/drivers/nvme/host/auth.c index f6ddbe553289..201fc8809a62 100644 --- a/drivers/nvme/host/auth.c +++ b/drivers/nvme/host/auth.c @@ -742,7 +742,7 @@ static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl, "%s: qid %d failed to generate digest, error %d\n", __func__, chap->qid, ret); goto out_free_psk; - }; + } dev_dbg(ctrl->device, "%s: generated digest %s\n", __func__, digest); ret = nvme_auth_derive_tls_psk(chap->hash_id, psk, psk_len, @@ -752,7 +752,7 @@ static int nvme_auth_secure_concat(struct nvme_ctrl *ctrl, "%s: qid %d failed to derive TLS psk, error %d\n", __func__, chap->qid, ret); goto out_free_digest; - }; + } tls_key = nvme_tls_psk_refresh(ctrl->opts->keyring, ctrl->opts->host->nqn, diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 9d988f4cb87a..812c1565114f 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3158,6 +3158,11 @@ static inline bool nvme_discovery_ctrl(struct nvme_ctrl *ctrl) return ctrl->opts && ctrl->opts->discovery_nqn; } +static inline bool nvme_admin_ctrl(struct nvme_ctrl *ctrl) +{ + return ctrl->cntrltype == NVME_CTRL_ADMIN; +} + static bool nvme_validate_cntlid(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id) { @@ -3670,6 +3675,17 @@ int nvme_init_ctrl_finish(struct nvme_ctrl *ctrl, bool was_suspended) if (ret) return ret; + if (nvme_admin_ctrl(ctrl)) { + /* + * An admin controller has one admin queue, but no I/O queues. + * Override queue_count so it only creates an admin queue. + */ + dev_dbg(ctrl->device, + "Subsystem %s is an administrative controller", + ctrl->subsys->subnqn); + ctrl->queue_count = 1; + } + ret = nvme_configure_apst(ctrl); if (ret < 0) return ret; diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 08a5ea3e9383..3e12d4683ac7 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -1363,7 +1363,7 @@ nvme_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) * down, and the related FC-NVME Association ID and Connection IDs * become invalid. * - * The behavior of the fc-nvme initiator is such that it's + * The behavior of the fc-nvme initiator is such that its * understanding of the association and connections will implicitly * be torn down. The action is implicit as it may be due to a loss of * connectivity with the fc-nvme target, so you may never get a @@ -2777,7 +2777,7 @@ nvme_fc_queue_rq(struct blk_mq_hw_ctx *hctx, * as WRITE ZEROES will return a non-zero rq payload_bytes yet * there is no actual payload to be transferred. * To get it right, key data transmission on there being 1 or - * more physical segments in the sg list. If there is no + * more physical segments in the sg list. If there are no * physical segments, there is no payload. */ if (blk_rq_nr_phys_segments(rq)) { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 071efec25346..2c6d9506b172 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -935,7 +935,7 @@ static blk_status_t nvme_pci_setup_data_sgl(struct request *req, nvme_pci_sgl_set_seg(&iod->cmd.common.dptr.sgl, sgl_dma, mapped); if (unlikely(iter->status)) - nvme_free_sgls(req); + nvme_unmap_data(req); return iter->status; } diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 9233f088fac8..c0fe8cfb7229 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2179,7 +2179,7 @@ static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new) /* * Only start IO queues for which we have allocated the tagset - * and limitted it to the available queues. On reconnects, the + * and limited it to the available queues. On reconnects, the * queue number might have changed. */ nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count); diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 884286f90688..0dd7bd99afa3 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -1960,24 +1960,24 @@ static int __init nvmet_init(void) if (!nvmet_wq) goto out_free_buffered_work_queue; - error = nvmet_init_discovery(); + error = nvmet_init_debugfs(); if (error) goto out_free_nvmet_work_queue; - error = nvmet_init_debugfs(); + error = nvmet_init_discovery(); if (error) - goto out_exit_discovery; + goto out_exit_debugfs; error = nvmet_init_configfs(); if (error) - goto out_exit_debugfs; + goto out_exit_discovery; return 0; -out_exit_debugfs: - nvmet_exit_debugfs(); out_exit_discovery: nvmet_exit_discovery(); +out_exit_debugfs: + nvmet_exit_debugfs(); out_free_nvmet_work_queue: destroy_workqueue(nvmet_wq); out_free_buffered_work_queue: @@ -1992,8 +1992,8 @@ out_destroy_bvec_cache: static void __exit nvmet_exit(void) { nvmet_exit_configfs(); - nvmet_exit_debugfs(); nvmet_exit_discovery(); + nvmet_exit_debugfs(); ida_destroy(&cntlid_ida); destroy_workqueue(nvmet_wq); destroy_workqueue(buffered_io_wq); diff --git a/drivers/nvme/target/fc.c b/drivers/nvme/target/fc.c index 25598a46bf0d..a9b18c051f5b 100644 --- a/drivers/nvme/target/fc.c +++ b/drivers/nvme/target/fc.c @@ -459,7 +459,7 @@ nvmet_fc_disconnect_assoc_done(struct nvmefc_ls_req *lsreq, int status) * down, and the related FC-NVME Association ID and Connection IDs * become invalid. * - * The behavior of the fc-nvme target is such that it's + * The behavior of the fc-nvme target is such that its * understanding of the association and connections will implicitly * be torn down. The action is implicit as it may be due to a loss of * connectivity with the fc-nvme host, so the target may never get a @@ -2313,7 +2313,7 @@ nvmet_fc_transfer_fcp_data(struct nvmet_fc_tgtport *tgtport, ret = tgtport->ops->fcp_op(&tgtport->fc_target_port, fod->fcpreq); if (ret) { /* - * should be ok to set w/o lock as its in the thread of + * should be ok to set w/o lock as it's in the thread of * execution (not an async timer routine) and doesn't * contend with any clearing action */ @@ -2629,7 +2629,7 @@ transport_error: * and the api of the FC LLDD which may issue a hw command to send the * response, but the LLDD may not get the hw completion for that command * and upcall the nvmet_fc layer before a new command may be - * asynchronously received - its possible for a command to be received + * asynchronously received - it's possible for a command to be received * before the LLDD and nvmet_fc have recycled the job structure. It gives * the appearance of more commands received than fits in the sq. * To alleviate this scenario, a temporary queue is maintained in the diff --git a/drivers/nvme/target/passthru.c b/drivers/nvme/target/passthru.c index 3b4b0df8f879..0c361b1e3566 100644 --- a/drivers/nvme/target/passthru.c +++ b/drivers/nvme/target/passthru.c @@ -533,6 +533,8 @@ u16 nvmet_parse_passthru_admin_cmd(struct nvmet_req *req) case NVME_FEAT_HOST_ID: req->execute = nvmet_execute_get_features; return NVME_SC_SUCCESS; + case NVME_FEAT_FDP: + return nvmet_setup_passthru_command(req); default: return nvmet_passthru_get_set_features(req); } diff --git a/drivers/nvme/target/rdma.c b/drivers/nvme/target/rdma.c index 67f61c67c167..0485e25ab797 100644 --- a/drivers/nvme/target/rdma.c +++ b/drivers/nvme/target/rdma.c @@ -1731,7 +1731,7 @@ static void nvmet_rdma_queue_connect_fail(struct rdma_cm_id *cm_id, * We registered an ib_client to handle device removal for queues, * so we only need to handle the listening port cm_ids. In this case * we nullify the priv to prevent double cm_id destruction and destroying - * the cm_id implicitely by returning a non-zero rc to the callout. + * the cm_id implicitly by returning a non-zero rc to the callout. */ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, struct nvmet_rdma_queue *queue) @@ -1742,7 +1742,7 @@ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, /* * This is a queue cm_id. we have registered * an ib_client to handle queues removal - * so don't interfear and just return. + * so don't interfere and just return. */ return 0; } @@ -1760,7 +1760,7 @@ static int nvmet_rdma_device_removal(struct rdma_cm_id *cm_id, /* * We need to return 1 so that the core will destroy - * it's own ID. What a great API design.. + * its own ID. What a great API design.. */ return 1; } diff --git a/drivers/pci/controller/vmd.c b/drivers/pci/controller/vmd.c index 9bbb0ff4cc15..b679c7f28f51 100644 --- a/drivers/pci/controller/vmd.c +++ b/drivers/pci/controller/vmd.c @@ -280,10 +280,12 @@ static int vmd_msi_alloc(struct irq_domain *domain, unsigned int virq, static void vmd_msi_free(struct irq_domain *domain, unsigned int virq, unsigned int nr_irqs) { + struct irq_data *irq_data; struct vmd_irq *vmdirq; for (int i = 0; i < nr_irqs; ++i) { - vmdirq = irq_get_chip_data(virq + i); + irq_data = irq_domain_get_irq_data(domain, virq + i); + vmdirq = irq_data->chip_data; synchronize_srcu(&vmdirq->irq->srcu); diff --git a/drivers/regulator/core.c b/drivers/regulator/core.c index 8ed9b96518cf..554d83c4af0c 100644 --- a/drivers/regulator/core.c +++ b/drivers/regulator/core.c @@ -3884,7 +3884,7 @@ static int regulator_set_voltage_unlocked(struct regulator *regulator, new_delta = ret; /* check that voltage is converging quickly enough */ - if (new_delta - delta > rdev->constraints->max_uV_step) { + if (delta - new_delta < rdev->constraints->max_uV_step) { ret = -EWOULDBLOCK; goto out; } diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 88b625ba1978..4b7ffa840563 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -180,7 +180,7 @@ struct ap_card { atomic64_t total_request_count; /* # requests ever for this AP device.*/ }; -#define TAPQ_CARD_HWINFO_MASK 0xFEFF0000FFFF0F0FUL +#define TAPQ_CARD_HWINFO_MASK 0xFFFF0000FFFF0F0FUL #define ASSOC_IDX_INVALID 0x10000 #define to_ap_card(x) container_of((x), struct ap_card, ap_dev.device) diff --git a/drivers/spi/spi-cs42l43.c b/drivers/spi/spi-cs42l43.c index b28a840b3b04..14307dd800b7 100644 --- a/drivers/spi/spi-cs42l43.c +++ b/drivers/spi/spi-cs42l43.c @@ -295,7 +295,7 @@ static struct spi_board_info *cs42l43_create_bridge_amp(struct cs42l43_spi *priv struct spi_board_info *info; if (spkid >= 0) { - props = devm_kmalloc(priv->dev, sizeof(*props), GFP_KERNEL); + props = devm_kcalloc(priv->dev, 2, sizeof(*props), GFP_KERNEL); if (!props) return NULL; diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 23286e4d7b49..8570fdf2e14a 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -615,6 +615,7 @@ void vhost_dev_init(struct vhost_dev *dev, vq->log = NULL; vq->indirect = NULL; vq->heads = NULL; + vq->nheads = NULL; vq->dev = dev; mutex_init(&vq->mutex); vhost_vq_reset(dev, vq); diff --git a/drivers/video/console/vgacon.c b/drivers/video/console/vgacon.c index f9cdbf8c53e3..37bd18730fe0 100644 --- a/drivers/video/console/vgacon.c +++ b/drivers/video/console/vgacon.c @@ -1168,7 +1168,7 @@ static bool vgacon_scroll(struct vc_data *c, unsigned int t, unsigned int b, c->vc_screenbuf_size - delta); c->vc_origin = vga_vram_end - c->vc_screenbuf_size; vga_rolled_over = 0; - } else if (oldo - delta >= (unsigned long)c->vc_screenbuf) + } else c->vc_origin -= delta; c->vc_scr_end = c->vc_origin + c->vc_screenbuf_size; scr_memsetw((u16 *) (c->vc_origin), c->vc_video_erase_char, diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index eabbc4bd7cf6..55f5731e94c3 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -837,7 +837,8 @@ static void con2fb_init_display(struct vc_data *vc, struct fb_info *info, fg_vc->vc_rows); } - update_screen(vc_cons[fg_console].d); + if (fg_console != unit) + update_screen(vc_cons[fg_console].d); } /** @@ -1375,6 +1376,7 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, struct vc_data *svc; struct fbcon_ops *ops = info->fbcon_par; int rows, cols; + unsigned long ret = 0; p = &fb_display[unit]; @@ -1425,11 +1427,10 @@ static void fbcon_set_disp(struct fb_info *info, struct fb_var_screeninfo *var, rows = FBCON_SWAP(ops->rotate, info->var.yres, info->var.xres); cols /= vc->vc_font.width; rows /= vc->vc_font.height; - vc_resize(vc, cols, rows); + ret = vc_resize(vc, cols, rows); - if (con_is_visible(vc)) { + if (con_is_visible(vc) && !ret) update_screen(vc); - } } static __inline__ void ywrap_up(struct vc_data *vc, int count) diff --git a/fs/nfs/blocklayout/blocklayout.c b/fs/nfs/blocklayout/blocklayout.c index 47189476b553..5d6edafbed20 100644 --- a/fs/nfs/blocklayout/blocklayout.c +++ b/fs/nfs/blocklayout/blocklayout.c @@ -149,8 +149,8 @@ do_add_page_to_bio(struct bio *bio, int npg, enum req_op op, sector_t isect, /* limit length to what the device mapping allows */ end = disk_addr + *len; - if (end >= map->start + map->len) - *len = map->start + map->len - disk_addr; + if (end >= map->disk_offset + map->len) + *len = map->disk_offset + map->len - disk_addr; retry: if (!bio) { diff --git a/fs/nfs/blocklayout/dev.c b/fs/nfs/blocklayout/dev.c index cab8809f0e0f..44306ac22353 100644 --- a/fs/nfs/blocklayout/dev.c +++ b/fs/nfs/blocklayout/dev.c @@ -257,10 +257,11 @@ static bool bl_map_stripe(struct pnfs_block_dev *dev, u64 offset, struct pnfs_block_dev *child; u64 chunk; u32 chunk_idx; + u64 disk_chunk; u64 disk_offset; chunk = div_u64(offset, dev->chunk_size); - div_u64_rem(chunk, dev->nr_children, &chunk_idx); + disk_chunk = div_u64_rem(chunk, dev->nr_children, &chunk_idx); if (chunk_idx >= dev->nr_children) { dprintk("%s: invalid chunk idx %d (%lld/%lld)\n", @@ -273,7 +274,7 @@ static bool bl_map_stripe(struct pnfs_block_dev *dev, u64 offset, offset = chunk * dev->chunk_size; /* disk offset of the stripe */ - disk_offset = div_u64(offset, dev->nr_children); + disk_offset = disk_chunk * dev->chunk_size; child = &dev->children[chunk_idx]; child->map(child, disk_offset, map); diff --git a/fs/nfs/blocklayout/extent_tree.c b/fs/nfs/blocklayout/extent_tree.c index 8f7cff7a4293..315949a7e92d 100644 --- a/fs/nfs/blocklayout/extent_tree.c +++ b/fs/nfs/blocklayout/extent_tree.c @@ -6,6 +6,7 @@ #include <linux/vmalloc.h> #include "blocklayout.h" +#include "../nfs4trace.h" #define NFSDBG_FACILITY NFSDBG_PNFS_LD @@ -520,10 +521,71 @@ static __be32 *encode_scsi_range(struct pnfs_block_extent *be, __be32 *p) return xdr_encode_hyper(p, be->be_length << SECTOR_SHIFT); } -static int ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, +/** + * ext_tree_try_encode_commit - try to encode all extents into the buffer + * @bl: pointer to the layout + * @p: pointer to the output buffer + * @buffer_size: size of the output buffer + * @count: output pointer to the number of encoded extents + * @lastbyte: output pointer to the last written byte + * + * Return values: + * %0: Success, all required extents encoded, outputs are valid + * %-ENOSPC: Buffer too small, nothing encoded, outputs are invalid + */ +static int +ext_tree_try_encode_commit(struct pnfs_block_layout *bl, __be32 *p, size_t buffer_size, size_t *count, __u64 *lastbyte) { struct pnfs_block_extent *be; + + spin_lock(&bl->bl_ext_lock); + for (be = ext_tree_first(&bl->bl_ext_rw); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_WRITTEN) + continue; + + (*count)++; + if (ext_tree_layoutupdate_size(bl, *count) > buffer_size) { + spin_unlock(&bl->bl_ext_lock); + return -ENOSPC; + } + } + for (be = ext_tree_first(&bl->bl_ext_rw); be; be = ext_tree_next(be)) { + if (be->be_state != PNFS_BLOCK_INVALID_DATA || + be->be_tag != EXTENT_WRITTEN) + continue; + + if (bl->bl_scsi_layout) + p = encode_scsi_range(be, p); + else + p = encode_block_extent(be, p); + be->be_tag = EXTENT_COMMITTING; + } + *lastbyte = (bl->bl_lwb != 0) ? bl->bl_lwb - 1 : U64_MAX; + bl->bl_lwb = 0; + spin_unlock(&bl->bl_ext_lock); + + return 0; +} + +/** + * ext_tree_encode_commit - encode as much as possible extents into the buffer + * @bl: pointer to the layout + * @p: pointer to the output buffer + * @buffer_size: size of the output buffer + * @count: output pointer to the number of encoded extents + * @lastbyte: output pointer to the last written byte + * + * Return values: + * %0: Success, all required extents encoded, outputs are valid + * %-ENOSPC: Buffer too small, some extents are encoded, outputs are valid + */ +static int +ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, + size_t buffer_size, size_t *count, __u64 *lastbyte) +{ + struct pnfs_block_extent *be, *be_prev; int ret = 0; spin_lock(&bl->bl_ext_lock); @@ -534,9 +596,9 @@ static int ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, (*count)++; if (ext_tree_layoutupdate_size(bl, *count) > buffer_size) { - /* keep counting.. */ + (*count)--; ret = -ENOSPC; - continue; + break; } if (bl->bl_scsi_layout) @@ -544,14 +606,30 @@ static int ext_tree_encode_commit(struct pnfs_block_layout *bl, __be32 *p, else p = encode_block_extent(be, p); be->be_tag = EXTENT_COMMITTING; + be_prev = be; + } + if (!ret) { + *lastbyte = (bl->bl_lwb != 0) ? bl->bl_lwb - 1 : U64_MAX; + bl->bl_lwb = 0; + } else { + *lastbyte = be_prev->be_f_offset + be_prev->be_length; + *lastbyte <<= SECTOR_SHIFT; + *lastbyte -= 1; } - *lastbyte = bl->bl_lwb - 1; - bl->bl_lwb = 0; spin_unlock(&bl->bl_ext_lock); return ret; } +/** + * ext_tree_prepare_commit - encode extents that need to be committed + * @arg: layout commit data + * + * Return values: + * %0: Success, all required extents are encoded + * %-ENOSPC: Some extents are encoded, but not all, due to RPC size limit + * %-ENOMEM: Out of memory, extents not encoded + */ int ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) { @@ -560,20 +638,18 @@ ext_tree_prepare_commit(struct nfs4_layoutcommit_args *arg) __be32 *start_p; int ret; - dprintk("%s enter\n", __func__); - arg->layoutupdate_page = alloc_page(GFP_NOFS); if (!arg->layoutupdate_page) return -ENOMEM; start_p = page_address(arg->layoutupdate_page); arg->layoutupdate_pages = &arg->layoutupdate_page; -retry: - ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, &count, &arg->lastbytewritten); + ret = ext_tree_try_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); if (unlikely(ret)) { ext_tree_free_commitdata(arg, buffer_size); - buffer_size = ext_tree_layoutupdate_size(bl, count); + buffer_size = NFS_SERVER(arg->inode)->wsize; count = 0; arg->layoutupdate_pages = @@ -588,7 +664,8 @@ retry: return -ENOMEM; } - goto retry; + ret = ext_tree_encode_commit(bl, start_p + 1, buffer_size, + &count, &arg->lastbytewritten); } *start_p = cpu_to_be32(count); @@ -607,8 +684,9 @@ retry: } } - dprintk("%s found %zu ranges\n", __func__, count); - return 0; + trace_bl_ext_tree_prepare_commit(ret, count, + arg->lastbytewritten, !!ret); + return ret; } void diff --git a/fs/nfs/client.c b/fs/nfs/client.c index cf35ad3f818a..8fb4a950dd55 100644 --- a/fs/nfs/client.c +++ b/fs/nfs/client.c @@ -682,6 +682,44 @@ struct nfs_client *nfs_init_client(struct nfs_client *clp, } EXPORT_SYMBOL_GPL(nfs_init_client); +static void nfs4_server_set_init_caps(struct nfs_server *server) +{ +#if IS_ENABLED(CONFIG_NFS_V4) + /* Set the basic capabilities */ + server->caps = server->nfs_client->cl_mvops->init_caps; + if (server->flags & NFS_MOUNT_NORDIRPLUS) + server->caps &= ~NFS_CAP_READDIRPLUS; + if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA) + server->caps &= ~NFS_CAP_READ_PLUS; + + /* + * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower + * authentication. + */ + if (nfs4_disable_idmapping && + server->client->cl_auth->au_flavor == RPC_AUTH_UNIX) + server->caps |= NFS_CAP_UIDGID_NOMAP; +#endif +} + +void nfs_server_set_init_caps(struct nfs_server *server) +{ + switch (server->nfs_client->rpc_ops->version) { + case 2: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + break; + case 3: + server->caps = NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; + if (!(server->flags & NFS_MOUNT_NORDIRPLUS)) + server->caps |= NFS_CAP_READDIRPLUS; + break; + default: + nfs4_server_set_init_caps(server); + break; + } +} +EXPORT_SYMBOL_GPL(nfs_server_set_init_caps); + /* * Create a version 2 or 3 client */ @@ -726,7 +764,6 @@ static int nfs_init_server(struct nfs_server *server, /* Initialise the client representation from the mount data */ server->flags = ctx->flags; server->options = ctx->options; - server->caps |= NFS_CAP_HARDLINKS | NFS_CAP_SYMLINKS; switch (clp->rpc_ops->version) { case 2: @@ -762,6 +799,8 @@ static int nfs_init_server(struct nfs_server *server, if (error < 0) goto error; + nfs_server_set_init_caps(server); + /* Preserve the values of mount_server-related mount options */ if (ctx->mount_server.addrlen) { memcpy(&server->mountd_address, &ctx->mount_server.address, @@ -814,7 +853,6 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, server->wsize = max_rpc_payload; if (server->wsize > NFS_MAX_FILE_IO_SIZE) server->wsize = NFS_MAX_FILE_IO_SIZE; - server->wpages = (server->wsize + PAGE_SIZE - 1) >> PAGE_SHIFT; server->wtmult = nfs_block_bits(fsinfo->wtmult, NULL); @@ -831,7 +869,6 @@ static void nfs_server_set_fsinfo(struct nfs_server *server, server->maxfilesize = fsinfo->maxfilesize; - server->time_delta = fsinfo->time_delta; server->change_attr_type = fsinfo->change_attr_type; server->clone_blksize = fsinfo->clone_blksize; @@ -936,7 +973,6 @@ void nfs_server_copy_userdata(struct nfs_server *target, struct nfs_server *sour target->acregmax = source->acregmax; target->acdirmin = source->acdirmin; target->acdirmax = source->acdirmax; - target->caps = source->caps; target->options = source->options; target->auth_info = source->auth_info; target->port = source->port; @@ -1007,6 +1043,7 @@ struct nfs_server *nfs_alloc_server(void) INIT_LIST_HEAD(&server->ss_src_copies); atomic_set(&server->active, 0); + atomic_long_set(&server->nr_active_delegations, 0); server->io_stats = nfs_alloc_iostats(); if (!server->io_stats) { @@ -1170,6 +1207,8 @@ struct nfs_server *nfs_clone_server(struct nfs_server *source, if (error < 0) goto out_free_server; + nfs_server_set_init_caps(server); + /* probe the filesystem info for this server filesystem */ error = nfs_probe_server(server, fh); if (error < 0) diff --git a/fs/nfs/delegation.c b/fs/nfs/delegation.c index 10ef46e29b25..9d3a5f29f17f 100644 --- a/fs/nfs/delegation.c +++ b/fs/nfs/delegation.c @@ -27,8 +27,15 @@ #define NFS_DEFAULT_DELEGATION_WATERMARK (5000U) -static atomic_long_t nfs_active_delegations; static unsigned nfs_delegation_watermark = NFS_DEFAULT_DELEGATION_WATERMARK; +module_param_named(delegation_watermark, nfs_delegation_watermark, uint, 0644); + +static struct hlist_head *nfs_delegation_hash(struct nfs_server *server, + const struct nfs_fh *fhandle) +{ + return server->delegation_hash_table + + (nfs_fhandle_hash(fhandle) & server->delegation_hash_mask); +} static void __nfs_free_delegation(struct nfs_delegation *delegation) { @@ -37,11 +44,12 @@ static void __nfs_free_delegation(struct nfs_delegation *delegation) kfree_rcu(delegation, rcu); } -static void nfs_mark_delegation_revoked(struct nfs_delegation *delegation) +static void nfs_mark_delegation_revoked(struct nfs_server *server, + struct nfs_delegation *delegation) { if (!test_and_set_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) { delegation->stateid.type = NFS4_INVALID_STATEID_TYPE; - atomic_long_dec(&nfs_active_delegations); + atomic_long_dec(&server->nr_active_delegations); if (!test_bit(NFS_DELEGATION_RETURNING, &delegation->flags)) nfs_clear_verifier_delegated(delegation->inode); } @@ -59,9 +67,10 @@ static void nfs_put_delegation(struct nfs_delegation *delegation) __nfs_free_delegation(delegation); } -static void nfs_free_delegation(struct nfs_delegation *delegation) +static void nfs_free_delegation(struct nfs_server *server, + struct nfs_delegation *delegation) { - nfs_mark_delegation_revoked(delegation); + nfs_mark_delegation_revoked(server, delegation); nfs_put_delegation(delegation); } @@ -237,34 +246,34 @@ void nfs_inode_reclaim_delegation(struct inode *inode, const struct cred *cred, rcu_read_lock(); delegation = rcu_dereference(NFS_I(inode)->delegation); - if (delegation != NULL) { - spin_lock(&delegation->lock); - nfs4_stateid_copy(&delegation->stateid, stateid); - delegation->type = type; - delegation->pagemod_limit = pagemod_limit; - oldcred = delegation->cred; - delegation->cred = get_cred(cred); - switch (deleg_type) { - case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: - case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: - set_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); - break; - default: - clear_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); - } - clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); - if (test_and_clear_bit(NFS_DELEGATION_REVOKED, - &delegation->flags)) - atomic_long_inc(&nfs_active_delegations); - spin_unlock(&delegation->lock); - rcu_read_unlock(); - put_cred(oldcred); - trace_nfs4_reclaim_delegation(inode, type); - } else { + if (!delegation) { rcu_read_unlock(); nfs_inode_set_delegation(inode, cred, type, stateid, pagemod_limit, deleg_type); + return; } + + spin_lock(&delegation->lock); + nfs4_stateid_copy(&delegation->stateid, stateid); + delegation->type = type; + delegation->pagemod_limit = pagemod_limit; + oldcred = delegation->cred; + delegation->cred = get_cred(cred); + switch (deleg_type) { + case NFS4_OPEN_DELEGATE_READ_ATTRS_DELEG: + case NFS4_OPEN_DELEGATE_WRITE_ATTRS_DELEG: + set_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); + break; + default: + clear_bit(NFS_DELEGATION_DELEGTIME, &delegation->flags); + } + clear_bit(NFS_DELEGATION_NEED_RECLAIM, &delegation->flags); + if (test_and_clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags)) + atomic_long_inc(&NFS_SERVER(inode)->nr_active_delegations); + spin_unlock(&delegation->lock); + rcu_read_unlock(); + put_cred(oldcred); + trace_nfs4_reclaim_delegation(inode, type); } static int nfs_do_return_delegation(struct inode *inode, @@ -355,6 +364,8 @@ nfs_detach_delegation_locked(struct nfs_inode *nfsi, rcu_dereference_protected(nfsi->delegation, lockdep_is_held(&clp->cl_lock)); + trace_nfs4_detach_delegation(&nfsi->vfs_inode, delegation->type); + if (deleg_cur == NULL || delegation != deleg_cur) return NULL; @@ -363,6 +374,7 @@ nfs_detach_delegation_locked(struct nfs_inode *nfsi, spin_unlock(&delegation->lock); return NULL; } + hlist_del_init_rcu(&delegation->hash); list_del_rcu(&delegation->super_list); delegation->inode = NULL; rcu_assign_pointer(nfsi->delegation, NULL); @@ -410,7 +422,8 @@ nfs_update_delegation_cred(struct nfs_delegation *delegation, } static void -nfs_update_inplace_delegation(struct nfs_delegation *delegation, +nfs_update_inplace_delegation(struct nfs_server *server, + struct nfs_delegation *delegation, const struct nfs_delegation *update) { if (nfs4_stateid_is_newer(&update->stateid, &delegation->stateid)) { @@ -423,7 +436,7 @@ nfs_update_inplace_delegation(struct nfs_delegation *delegation, nfs_update_delegation_cred(delegation, update->cred); /* smp_mb__before_atomic() is implicit due to xchg() */ clear_bit(NFS_DELEGATION_REVOKED, &delegation->flags); - atomic_long_inc(&nfs_active_delegations); + atomic_long_inc(&server->nr_active_delegations); } } } @@ -478,7 +491,7 @@ int nfs_inode_set_delegation(struct inode *inode, const struct cred *cred, if (nfs4_stateid_match_other(&old_delegation->stateid, &delegation->stateid)) { spin_lock(&old_delegation->lock); - nfs_update_inplace_delegation(old_delegation, + nfs_update_inplace_delegation(server, old_delegation, delegation); spin_unlock(&old_delegation->lock); goto out; @@ -524,10 +537,12 @@ add_new: spin_unlock(&inode->i_lock); list_add_tail_rcu(&delegation->super_list, &server->delegations); + hlist_add_head_rcu(&delegation->hash, + nfs_delegation_hash(server, &NFS_I(inode)->fh)); rcu_assign_pointer(nfsi->delegation, delegation); delegation = NULL; - atomic_long_inc(&nfs_active_delegations); + atomic_long_inc(&server->nr_active_delegations); trace_nfs4_set_delegation(inode, type); @@ -541,7 +556,7 @@ out: __nfs_free_delegation(delegation); if (freeme != NULL) { nfs_do_return_delegation(inode, freeme, 0); - nfs_free_delegation(freeme); + nfs_free_delegation(server, freeme); } return status; } @@ -592,6 +607,8 @@ static bool nfs_delegation_need_return(struct nfs_delegation *delegation) { bool ret = false; + trace_nfs_delegation_need_return(delegation); + if (test_and_clear_bit(NFS_DELEGATION_RETURN, &delegation->flags)) ret = true; if (test_bit(NFS_DELEGATION_RETURNING, &delegation->flags) || @@ -751,7 +768,7 @@ void nfs_inode_evict_delegation(struct inode *inode) set_bit(NFS_DELEGATION_RETURNING, &delegation->flags); set_bit(NFS_DELEGATION_INODE_FREEING, &delegation->flags); nfs_do_return_delegation(inode, delegation, 1); - nfs_free_delegation(delegation); + nfs_free_delegation(NFS_SERVER(inode), delegation); } } @@ -837,7 +854,8 @@ void nfs4_inode_return_delegation_on_close(struct inode *inode) if (!delegation) goto out; if (test_bit(NFS_DELEGATION_RETURN_IF_CLOSED, &delegation->flags) || - atomic_long_read(&nfs_active_delegations) >= nfs_delegation_watermark) { + atomic_long_read(&NFS_SERVER(inode)->nr_active_delegations) >= + nfs_delegation_watermark) { spin_lock(&delegation->lock); if (delegation->inode && list_empty(&NFS_I(inode)->open_files) && @@ -1013,7 +1031,7 @@ static void nfs_revoke_delegation(struct inode *inode, } spin_unlock(&delegation->lock); } - nfs_mark_delegation_revoked(delegation); + nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation); ret = true; out: rcu_read_unlock(); @@ -1045,7 +1063,7 @@ void nfs_delegation_mark_returned(struct inode *inode, delegation->stateid.seqid = stateid->seqid; } - nfs_mark_delegation_revoked(delegation); + nfs_mark_delegation_revoked(NFS_SERVER(inode), delegation); clear_bit(NFS_DELEGATION_RETURNING, &delegation->flags); spin_unlock(&delegation->lock); if (nfs_detach_delegation(NFS_I(inode), delegation, NFS_SERVER(inode))) @@ -1158,11 +1176,12 @@ static struct inode * nfs_delegation_find_inode_server(struct nfs_server *server, const struct nfs_fh *fhandle) { + struct hlist_head *head = nfs_delegation_hash(server, fhandle); struct nfs_delegation *delegation; struct super_block *freeme = NULL; struct inode *res = NULL; - list_for_each_entry_rcu(delegation, &server->delegations, super_list) { + hlist_for_each_entry_rcu(delegation, head, hash) { spin_lock(&delegation->lock); if (delegation->inode != NULL && !test_bit(NFS_DELEGATION_REVOKED, &delegation->flags) && @@ -1265,7 +1284,7 @@ restart: if (delegation != NULL) { if (nfs_detach_delegation(NFS_I(inode), delegation, server) != NULL) - nfs_free_delegation(delegation); + nfs_free_delegation(server, delegation); /* Match nfs_start_delegation_return_locked */ nfs_put_delegation(delegation); } @@ -1570,4 +1589,17 @@ out: return ret; } -module_param_named(delegation_watermark, nfs_delegation_watermark, uint, 0644); +int nfs4_delegation_hash_alloc(struct nfs_server *server) +{ + int delegation_buckets, i; + + delegation_buckets = roundup_pow_of_two(nfs_delegation_watermark / 16); + server->delegation_hash_mask = delegation_buckets - 1; + server->delegation_hash_table = kmalloc_array(delegation_buckets, + sizeof(*server->delegation_hash_table), GFP_KERNEL); + if (!server->delegation_hash_table) + return -ENOMEM; + for (i = 0; i < delegation_buckets; i++) + INIT_HLIST_HEAD(&server->delegation_hash_table[i]); + return 0; +} diff --git a/fs/nfs/delegation.h b/fs/nfs/delegation.h index 8ff5ab9c5c25..08ec2e9c68a4 100644 --- a/fs/nfs/delegation.h +++ b/fs/nfs/delegation.h @@ -14,6 +14,7 @@ * NFSv4 delegation */ struct nfs_delegation { + struct hlist_node hash; struct list_head super_list; const struct cred *cred; struct inode *inode; @@ -123,4 +124,6 @@ static inline int nfs_have_delegated_mtime(struct inode *inode) NFS_DELEGATION_FLAG_TIME); } +int nfs4_delegation_hash_alloc(struct nfs_server *server); + #endif diff --git a/fs/nfs/dir.c b/fs/nfs/dir.c index d0e0b435a843..d81217923936 100644 --- a/fs/nfs/dir.c +++ b/fs/nfs/dir.c @@ -1828,9 +1828,7 @@ static void block_revalidate(struct dentry *dentry) static void unblock_revalidate(struct dentry *dentry) { - /* store_release ensures wait_var_event() sees the update */ - smp_store_release(&dentry->d_fsdata, NULL); - wake_up_var(&dentry->d_fsdata); + store_release_wake_up(&dentry->d_fsdata, NULL); } /* diff --git a/fs/nfs/export.c b/fs/nfs/export.c index e9c233b6fd20..a10dd5f9d078 100644 --- a/fs/nfs/export.c +++ b/fs/nfs/export.c @@ -66,14 +66,21 @@ nfs_fh_to_dentry(struct super_block *sb, struct fid *fid, { struct nfs_fattr *fattr = NULL; struct nfs_fh *server_fh = nfs_exp_embedfh(fid->raw); - size_t fh_size = offsetof(struct nfs_fh, data) + server_fh->size; + size_t fh_size = offsetof(struct nfs_fh, data); const struct nfs_rpc_ops *rpc_ops; struct dentry *dentry; struct inode *inode; - int len = EMBED_FH_OFF + XDR_QUADLEN(fh_size); + int len = EMBED_FH_OFF; u32 *p = fid->raw; int ret; + /* Initial check of bounds */ + if (fh_len < len + XDR_QUADLEN(fh_size) || + fh_len > XDR_QUADLEN(NFS_MAXFHSIZE)) + return NULL; + /* Calculate embedded filehandle size */ + fh_size += server_fh->size; + len += XDR_QUADLEN(fh_size); /* NULL translates to ESTALE */ if (fh_len < len || fh_type != len) return NULL; diff --git a/fs/nfs/flexfilelayout/flexfilelayout.c b/fs/nfs/flexfilelayout/flexfilelayout.c index 4bea008dbebd..8dc921d83538 100644 --- a/fs/nfs/flexfilelayout/flexfilelayout.c +++ b/fs/nfs/flexfilelayout/flexfilelayout.c @@ -762,14 +762,14 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, { struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg); struct nfs4_ff_layout_mirror *mirror; - struct nfs4_pnfs_ds *ds; + struct nfs4_pnfs_ds *ds = ERR_PTR(-EAGAIN); u32 idx; /* mirrors are initially sorted by efficiency */ for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) { mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false); - if (!ds) + if (IS_ERR(ds)) continue; if (check_device && @@ -777,10 +777,10 @@ ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg, continue; *best_idx = idx; - return ds; + break; } - return NULL; + return ds; } static struct nfs4_pnfs_ds * @@ -942,7 +942,7 @@ retry: for (i = 0; i < pgio->pg_mirror_count; i++) { mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i); ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, true); - if (!ds) { + if (IS_ERR(ds)) { if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg)) goto out_mds; pnfs_generic_pg_cleanup(pgio); @@ -1867,6 +1867,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) u32 idx = hdr->pgio_mirror_idx; int vers; struct nfs_fh *fh; + bool ds_fatal_error = false; dprintk("--> %s ino %lu pgbase %u req %zu@%llu\n", __func__, hdr->inode->i_ino, @@ -1874,8 +1875,10 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false); - if (!ds) + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; + } ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, hdr->inode); @@ -1923,7 +1926,7 @@ ff_layout_read_pagelist(struct nfs_pgio_header *hdr) return PNFS_ATTEMPTED; out_failed: - if (ff_layout_avoid_mds_available_ds(lseg)) + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; trace_pnfs_mds_fallback_read_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, @@ -1945,11 +1948,14 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) int vers; struct nfs_fh *fh; u32 idx = hdr->pgio_mirror_idx; + bool ds_fatal_error = false; mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true); - if (!ds) + if (IS_ERR(ds)) { + ds_fatal_error = nfs_error_is_fatal(PTR_ERR(ds)); goto out_failed; + } ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, hdr->inode); @@ -2000,7 +2006,7 @@ ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync) return PNFS_ATTEMPTED; out_failed: - if (ff_layout_avoid_mds_available_ds(lseg)) + if (ff_layout_avoid_mds_available_ds(lseg) && !ds_fatal_error) return PNFS_TRY_AGAIN; trace_pnfs_mds_fallback_write_pagelist(hdr->inode, hdr->args.offset, hdr->args.count, @@ -2043,7 +2049,7 @@ static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how) idx = calc_ds_index_from_commit(lseg, data->ds_commit_index); mirror = FF_LAYOUT_COMP(lseg, idx); ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true); - if (!ds) + if (IS_ERR(ds)) goto out_err; ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp, diff --git a/fs/nfs/flexfilelayout/flexfilelayoutdev.c b/fs/nfs/flexfilelayout/flexfilelayoutdev.c index 656d5c50bbce..30365ec782bb 100644 --- a/fs/nfs/flexfilelayout/flexfilelayoutdev.c +++ b/fs/nfs/flexfilelayout/flexfilelayoutdev.c @@ -370,11 +370,11 @@ nfs4_ff_layout_prepare_ds(struct pnfs_layout_segment *lseg, struct nfs4_ff_layout_mirror *mirror, bool fail_return) { - struct nfs4_pnfs_ds *ds = NULL; + struct nfs4_pnfs_ds *ds; struct inode *ino = lseg->pls_layout->plh_inode; struct nfs_server *s = NFS_SERVER(ino); unsigned int max_payload; - int status; + int status = -EAGAIN; if (!ff_layout_init_mirror_ds(lseg->pls_layout, mirror)) goto noconnect; @@ -418,7 +418,7 @@ noconnect: ff_layout_send_layouterror(lseg); if (fail_return || !ff_layout_has_available_ds(lseg)) pnfs_error_mark_layout_for_return(ino, lseg); - ds = NULL; + ds = ERR_PTR(status); out: return ds; } diff --git a/fs/nfs/fs_context.c b/fs/nfs/fs_context.c index 13f71ca8c974..9e94d18448ff 100644 --- a/fs/nfs/fs_context.c +++ b/fs/nfs/fs_context.c @@ -96,6 +96,8 @@ enum nfs_param { Opt_wsize, Opt_write, Opt_xprtsec, + Opt_cert_serial, + Opt_privkey_serial, }; enum { @@ -221,6 +223,8 @@ static const struct fs_parameter_spec nfs_fs_parameters[] = { fsparam_enum ("write", Opt_write, nfs_param_enums_write), fsparam_u32 ("wsize", Opt_wsize), fsparam_string("xprtsec", Opt_xprtsec), + fsparam_s32("cert_serial", Opt_cert_serial), + fsparam_s32("privkey_serial", Opt_privkey_serial), {} }; @@ -551,6 +555,32 @@ static int nfs_parse_version_string(struct fs_context *fc, return 0; } +#ifdef CONFIG_KEYS +static int nfs_tls_key_verify(key_serial_t key_id) +{ + struct key *key = key_lookup(key_id); + int error = 0; + + if (IS_ERR(key)) { + pr_err("key id %08x not found\n", key_id); + return PTR_ERR(key); + } + if (test_bit(KEY_FLAG_REVOKED, &key->flags) || + test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { + pr_err("key id %08x revoked\n", key_id); + error = -EKEYREVOKED; + } + + key_put(key); + return error; +} +#else +static inline int nfs_tls_key_verify(key_serial_t key_id) +{ + return -ENOENT; +} +#endif /* CONFIG_KEYS */ + /* * Parse a single mount parameter. */ @@ -807,6 +837,18 @@ static int nfs_fs_context_parse_param(struct fs_context *fc, if (ret < 0) return ret; break; + case Opt_cert_serial: + ret = nfs_tls_key_verify(result.int_32); + if (ret < 0) + return ret; + ctx->xprtsec.cert_serial = result.int_32; + break; + case Opt_privkey_serial: + ret = nfs_tls_key_verify(result.int_32); + if (ret < 0) + return ret; + ctx->xprtsec.privkey_serial = result.int_32; + break; case Opt_proto: if (!param->string) diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index a2fa6bc4d74e..338ef77ae423 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -197,6 +197,7 @@ void nfs_set_cache_invalid(struct inode *inode, unsigned long flags) if (!(flags & NFS_INO_REVAL_FORCED)) flags &= ~(NFS_INO_INVALID_MODE | NFS_INO_INVALID_OTHER | + NFS_INO_INVALID_BTIME | NFS_INO_INVALID_XATTR); flags &= ~(NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_SIZE); } @@ -522,6 +523,7 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) inode_set_atime(inode, 0, 0); inode_set_mtime(inode, 0, 0); inode_set_ctime(inode, 0, 0); + memset(&nfsi->btime, 0, sizeof(nfsi->btime)); inode_set_iversion_raw(inode, 0); inode->i_size = 0; clear_nlink(inode); @@ -545,6 +547,10 @@ nfs_fhget(struct super_block *sb, struct nfs_fh *fh, struct nfs_fattr *fattr) inode_set_ctime_to_ts(inode, fattr->ctime); else if (fattr_supported & NFS_ATTR_FATTR_CTIME) nfs_set_cache_invalid(inode, NFS_INO_INVALID_CTIME); + if (fattr->valid & NFS_ATTR_FATTR_BTIME) + nfsi->btime = fattr->btime; + else if (fattr_supported & NFS_ATTR_FATTR_BTIME) + nfs_set_cache_invalid(inode, NFS_INO_INVALID_BTIME); if (fattr->valid & NFS_ATTR_FATTR_CHANGE) inode_set_iversion_raw(inode, fattr->change_attr); else @@ -931,6 +937,7 @@ static void nfs_readdirplus_parent_cache_hit(struct dentry *dentry) static u32 nfs_get_valid_attrmask(struct inode *inode) { + u64 fattr_valid = NFS_SERVER(inode)->fattr_valid; unsigned long cache_validity = READ_ONCE(NFS_I(inode)->cache_validity); u32 reply_mask = STATX_INO | STATX_TYPE; @@ -950,6 +957,9 @@ static u32 nfs_get_valid_attrmask(struct inode *inode) reply_mask |= STATX_UID | STATX_GID; if (!(cache_validity & NFS_INO_INVALID_BLOCKS)) reply_mask |= STATX_BLOCKS; + if (!(cache_validity & NFS_INO_INVALID_BTIME) && + (fattr_valid & NFS_ATTR_FATTR_BTIME)) + reply_mask |= STATX_BTIME; if (!(cache_validity & NFS_INO_INVALID_CHANGE)) reply_mask |= STATX_CHANGE_COOKIE; return reply_mask; @@ -960,6 +970,7 @@ int nfs_getattr(struct mnt_idmap *idmap, const struct path *path, { struct inode *inode = d_inode(path->dentry); struct nfs_server *server = NFS_SERVER(inode); + u64 fattr_valid = server->fattr_valid; unsigned long cache_validity; int err = 0; bool force_sync = query_flags & AT_STATX_FORCE_SYNC; @@ -970,9 +981,12 @@ int nfs_getattr(struct mnt_idmap *idmap, const struct path *path, request_mask &= STATX_TYPE | STATX_MODE | STATX_NLINK | STATX_UID | STATX_GID | STATX_ATIME | STATX_MTIME | STATX_CTIME | - STATX_INO | STATX_SIZE | STATX_BLOCKS | + STATX_INO | STATX_SIZE | STATX_BLOCKS | STATX_BTIME | STATX_CHANGE_COOKIE; + if (!(fattr_valid & NFS_ATTR_FATTR_BTIME)) + request_mask &= ~STATX_BTIME; + if ((query_flags & AT_STATX_DONT_SYNC) && !force_sync) { if (readdirplus_enabled) nfs_readdirplus_parent_cache_hit(path->dentry); @@ -1004,7 +1018,7 @@ int nfs_getattr(struct mnt_idmap *idmap, const struct path *path, /* Is the user requesting attributes that might need revalidation? */ if (!(request_mask & (STATX_MODE|STATX_NLINK|STATX_ATIME|STATX_CTIME| STATX_MTIME|STATX_UID|STATX_GID| - STATX_SIZE|STATX_BLOCKS| + STATX_SIZE|STATX_BLOCKS|STATX_BTIME| STATX_CHANGE_COOKIE))) goto out_no_revalidate; @@ -1028,6 +1042,8 @@ int nfs_getattr(struct mnt_idmap *idmap, const struct path *path, do_update |= cache_validity & NFS_INO_INVALID_OTHER; if (request_mask & STATX_BLOCKS) do_update |= cache_validity & NFS_INO_INVALID_BLOCKS; + if (request_mask & STATX_BTIME) + do_update |= cache_validity & NFS_INO_INVALID_BTIME; if (do_update) { if (readdirplus_enabled) @@ -1049,6 +1065,7 @@ out_no_revalidate: stat->attributes |= STATX_ATTR_CHANGE_MONOTONIC; if (S_ISDIR(inode->i_mode)) stat->blksize = NFS_SERVER(inode)->dtsize; + stat->btime = NFS_I(inode)->btime; out: trace_nfs_getattr_exit(inode, err); return err; @@ -1943,7 +1960,7 @@ static int nfs_inode_finish_partial_attr_update(const struct nfs_fattr *fattr, NFS_INO_INVALID_ATIME | NFS_INO_INVALID_CTIME | NFS_INO_INVALID_MTIME | NFS_INO_INVALID_SIZE | NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_OTHER | - NFS_INO_INVALID_NLINK; + NFS_INO_INVALID_NLINK | NFS_INO_INVALID_BTIME; unsigned long cache_validity = NFS_I(inode)->cache_validity; enum nfs4_change_attr_type ctype = NFS_SERVER(inode)->change_attr_type; @@ -2209,7 +2226,7 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) bool attr_changed = false; bool have_delegation; - dfprintk(VFS, "NFS: %s(%s/%lu fh_crc=0x%08x ct=%d info=0x%x)\n", + dfprintk(VFS, "NFS: %s(%s/%lu fh_crc=0x%08x ct=%d info=0x%llx)\n", __func__, inode->i_sb->s_id, inode->i_ino, nfs_display_fhandle_hash(NFS_FH(inode)), atomic_read(&inode->i_count), fattr->valid); @@ -2304,7 +2321,8 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) | NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK | NFS_INO_INVALID_MODE - | NFS_INO_INVALID_OTHER; + | NFS_INO_INVALID_OTHER + | NFS_INO_INVALID_BTIME; if (S_ISDIR(inode->i_mode)) nfs_force_lookup_revalidate(inode); attr_changed = true; @@ -2338,6 +2356,12 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) nfsi->cache_validity |= save_cache_validity & NFS_INO_INVALID_CTIME; + if (fattr->valid & NFS_ATTR_FATTR_BTIME) + nfsi->btime = fattr->btime; + else if (fattr_supported & NFS_ATTR_FATTR_BTIME) + nfsi->cache_validity |= + save_cache_validity & NFS_INO_INVALID_BTIME; + /* Check if our cached file size is stale */ if (fattr->valid & NFS_ATTR_FATTR_SIZE) { new_isize = nfs_size_to_loff_t(fattr->size); @@ -2625,6 +2649,35 @@ static struct pernet_operations nfs_net_ops = { .size = sizeof(struct nfs_net), }; +#ifdef CONFIG_KEYS +static struct key *nfs_keyring; + +static int __init nfs_init_keyring(void) +{ + nfs_keyring = keyring_alloc(".nfs", + GLOBAL_ROOT_UID, GLOBAL_ROOT_GID, + current_cred(), + (KEY_POS_ALL & ~KEY_POS_SETATTR) | + (KEY_USR_ALL & ~KEY_USR_SETATTR), + KEY_ALLOC_NOT_IN_QUOTA, NULL, NULL); + return PTR_ERR_OR_ZERO(nfs_keyring); +} + +static void nfs_exit_keyring(void) +{ + key_put(nfs_keyring); +} +#else +static inline int nfs_init_keyring(void) +{ + return 0; +} + +static inline void nfs_exit_keyring(void) +{ +} +#endif /* CONFIG_KEYS */ + /* * Initialize NFS */ @@ -2632,6 +2685,10 @@ static int __init init_nfs_fs(void) { int err; + err = nfs_init_keyring(); + if (err) + return err; + err = nfs_sysfs_init(); if (err < 0) goto out10; @@ -2692,6 +2749,7 @@ out7: out9: nfs_sysfs_exit(); out10: + nfs_exit_keyring(); return err; } @@ -2707,6 +2765,7 @@ static void __exit exit_nfs_fs(void) nfs_fs_proc_exit(); nfsiod_stop(); nfs_sysfs_exit(); + nfs_exit_keyring(); } /* Not quite true; I just maintain it */ diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h index 26551ff09a52..74d712b58423 100644 --- a/fs/nfs/internal.h +++ b/fs/nfs/internal.h @@ -207,7 +207,6 @@ struct nfs_mount_request { }; extern int nfs_mount(struct nfs_mount_request *info, int timeo, int retrans); -extern void nfs_umount(const struct nfs_mount_request *info); /* client.c */ extern const struct rpc_program nfs_program; @@ -232,7 +231,7 @@ extern struct nfs_client * nfs4_find_client_sessionid(struct net *, const struct sockaddr *, struct nfs4_sessionid *, u32); extern struct nfs_server *nfs_create_server(struct fs_context *); -extern void nfs4_server_set_init_caps(struct nfs_server *); +extern void nfs_server_set_init_caps(struct nfs_server *); extern struct nfs_server *nfs4_create_server(struct fs_context *); extern struct nfs_server *nfs4_create_referral_server(struct fs_context *); extern int nfs4_update_server(struct nfs_server *server, const char *hostname, @@ -671,9 +670,12 @@ nfs_write_match_verf(const struct nfs_writeverf *verf, static inline gfp_t nfs_io_gfp_mask(void) { - if (current->flags & PF_WQ_WORKER) - return GFP_KERNEL | __GFP_NORETRY | __GFP_NOWARN; - return GFP_KERNEL; + gfp_t ret = current_gfp_context(GFP_KERNEL); + + /* For workers __GFP_NORETRY only with __GFP_IO or __GFP_FS */ + if ((current->flags & PF_WQ_WORKER) && ret == GFP_KERNEL) + ret |= __GFP_NORETRY | __GFP_NOWARN; + return ret; } /* diff --git a/fs/nfs/localio.c b/fs/nfs/localio.c index 510d0a16cfe9..bd5fca285899 100644 --- a/fs/nfs/localio.c +++ b/fs/nfs/localio.c @@ -500,14 +500,13 @@ nfs_copy_boot_verifier(struct nfs_write_verifier *verifier, struct inode *inode) { struct nfs_client *clp = NFS_SERVER(inode)->nfs_client; u32 *verf = (u32 *)verifier->data; - int seq = 0; + unsigned int seq; do { - read_seqbegin_or_lock(&clp->cl_boot_lock, &seq); + seq = read_seqbegin(&clp->cl_boot_lock); verf[0] = (u32)clp->cl_nfssvc_boot.tv_sec; verf[1] = (u32)clp->cl_nfssvc_boot.tv_nsec; - } while (need_seqretry(&clp->cl_boot_lock, seq)); - done_seqretry(&clp->cl_boot_lock, seq); + } while (read_seqretry(&clp->cl_boot_lock, seq)); } static void diff --git a/fs/nfs/mount_clnt.c b/fs/nfs/mount_clnt.c index 57c9dd700b58..db8dfb920394 100644 --- a/fs/nfs/mount_clnt.c +++ b/fs/nfs/mount_clnt.c @@ -223,74 +223,6 @@ out_mnt_err: goto out; } -/** - * nfs_umount - Notify a server that we have unmounted this export - * @info: pointer to umount request arguments - * - * MOUNTPROC_UMNT is advisory, so we set a short timeout, and always - * use UDP. - */ -void nfs_umount(const struct nfs_mount_request *info) -{ - static const struct rpc_timeout nfs_umnt_timeout = { - .to_initval = 1 * HZ, - .to_maxval = 3 * HZ, - .to_retries = 2, - }; - struct rpc_create_args args = { - .net = info->net, - .protocol = IPPROTO_UDP, - .address = (struct sockaddr *)info->sap, - .addrsize = info->salen, - .timeout = &nfs_umnt_timeout, - .servername = info->hostname, - .program = &mnt_program, - .version = info->version, - .authflavor = RPC_AUTH_UNIX, - .flags = RPC_CLNT_CREATE_NOPING, - .cred = current_cred(), - }; - struct rpc_message msg = { - .rpc_argp = info->dirpath, - }; - struct rpc_clnt *clnt; - int status; - - if (strlen(info->dirpath) > MNTPATHLEN) - return; - - if (info->noresvport) - args.flags |= RPC_CLNT_CREATE_NONPRIVPORT; - - clnt = rpc_create(&args); - if (IS_ERR(clnt)) - goto out_clnt_err; - - dprintk("NFS: sending UMNT request for %s:%s\n", - (info->hostname ? info->hostname : "server"), info->dirpath); - - if (info->version == NFS_MNT3_VERSION) - msg.rpc_proc = &clnt->cl_procinfo[MOUNTPROC3_UMNT]; - else - msg.rpc_proc = &clnt->cl_procinfo[MOUNTPROC_UMNT]; - - status = rpc_call_sync(clnt, &msg, 0); - rpc_shutdown_client(clnt); - - if (unlikely(status < 0)) - goto out_call_err; - - return; - -out_clnt_err: - dprintk("NFS: failed to create UMNT RPC client, status=%ld\n", - PTR_ERR(clnt)); - return; - -out_call_err: - dprintk("NFS: UMNT request failed, status=%d\n", status); -} - /* * XDR encode/decode functions for MOUNT */ diff --git a/fs/nfs/nfs4_fs.h b/fs/nfs/nfs4_fs.h index d3ca91f60fc1..c34c89af9c7d 100644 --- a/fs/nfs/nfs4_fs.h +++ b/fs/nfs/nfs4_fs.h @@ -63,7 +63,7 @@ struct nfs4_minor_version_ops { bool (*match_stateid)(const nfs4_stateid *, const nfs4_stateid *); int (*find_root_sec)(struct nfs_server *, struct nfs_fh *, - struct nfs_fsinfo *); + struct nfs_fattr *); void (*free_lock_state)(struct nfs_server *, struct nfs4_lock_state *); int (*test_and_free_expired)(struct nfs_server *, @@ -296,7 +296,8 @@ extern int nfs4_call_sync(struct rpc_clnt *, struct nfs_server *, extern void nfs4_init_sequence(struct nfs4_sequence_args *, struct nfs4_sequence_res *, int, int); extern int nfs4_proc_setclientid(struct nfs_client *, u32, unsigned short, const struct cred *, struct nfs4_setclientid_res *); extern int nfs4_proc_setclientid_confirm(struct nfs_client *, struct nfs4_setclientid_res *arg, const struct cred *); -extern int nfs4_proc_get_rootfh(struct nfs_server *, struct nfs_fh *, struct nfs_fsinfo *, bool); +extern int nfs4_proc_get_rootfh(struct nfs_server *, struct nfs_fh *, + struct nfs_fattr *, bool); extern int nfs4_proc_bind_conn_to_session(struct nfs_client *, const struct cred *cred); extern int nfs4_proc_exchange_id(struct nfs_client *clp, const struct cred *cred); extern int nfs4_destroy_clientid(struct nfs_client *clp); diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c index 162c85a83a14..6fddf43d729c 100644 --- a/fs/nfs/nfs4client.c +++ b/fs/nfs/nfs4client.c @@ -802,6 +802,7 @@ static void nfs4_destroy_server(struct nfs_server *server) unset_pnfs_layoutdriver(server); nfs4_purge_state_owners(server, &freeme); nfs4_free_state_owners(&freeme); + kfree(server->delegation_hash_table); } /* @@ -895,55 +896,40 @@ nfs4_find_client_sessionid(struct net *net, const struct sockaddr *addr, * Set up an NFS4 client */ static int nfs4_set_client(struct nfs_server *server, - const char *hostname, - const struct sockaddr_storage *addr, - const size_t addrlen, - const char *ip_addr, - int proto, const struct rpc_timeout *timeparms, - u32 minorversion, unsigned int nconnect, - unsigned int max_connect, - struct net *net, - struct xprtsec_parms *xprtsec) + struct nfs_client_initdata *cl_init) { - struct nfs_client_initdata cl_init = { - .hostname = hostname, - .addr = addr, - .addrlen = addrlen, - .ip_addr = ip_addr, - .nfs_mod = &nfs_v4, - .proto = proto, - .minorversion = minorversion, - .net = net, - .timeparms = timeparms, - .cred = server->cred, - .xprtsec = *xprtsec, - }; struct nfs_client *clp; - if (minorversion == 0) - __set_bit(NFS_CS_REUSEPORT, &cl_init.init_flags); - else - cl_init.max_connect = max_connect; - switch (proto) { + cl_init->nfs_mod = &nfs_v4; + cl_init->cred = server->cred; + + if (cl_init->minorversion == 0) { + __set_bit(NFS_CS_REUSEPORT, &cl_init->init_flags); + cl_init->max_connect = 0; + } + + switch (cl_init->proto) { case XPRT_TRANSPORT_RDMA: case XPRT_TRANSPORT_TCP: case XPRT_TRANSPORT_TCP_TLS: - cl_init.nconnect = nconnect; + break; + default: + cl_init->nconnect = 0; } if (server->flags & NFS_MOUNT_NORESVPORT) - __set_bit(NFS_CS_NORESVPORT, &cl_init.init_flags); + __set_bit(NFS_CS_NORESVPORT, &cl_init->init_flags); if (server->options & NFS_OPTION_MIGRATION) - __set_bit(NFS_CS_MIGRATION, &cl_init.init_flags); + __set_bit(NFS_CS_MIGRATION, &cl_init->init_flags); if (test_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status)) - __set_bit(NFS_CS_TSM_POSSIBLE, &cl_init.init_flags); - server->port = rpc_get_port((struct sockaddr *)addr); + __set_bit(NFS_CS_TSM_POSSIBLE, &cl_init->init_flags); + server->port = rpc_get_port((struct sockaddr *)cl_init->addr); if (server->flags & NFS_MOUNT_NETUNREACH_FATAL) - __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init.init_flags); + __set_bit(NFS_CS_NETUNREACH_FATAL, &cl_init->init_flags); /* Allocate or find a client reference we can use */ - clp = nfs_get_client(&cl_init); + clp = nfs_get_client(cl_init); if (IS_ERR(clp)) return PTR_ERR(clp); @@ -1088,29 +1074,15 @@ static void nfs4_session_limit_xasize(struct nfs_server *server) #endif } -void nfs4_server_set_init_caps(struct nfs_server *server) -{ - /* Set the basic capabilities */ - server->caps |= server->nfs_client->cl_mvops->init_caps; - if (server->flags & NFS_MOUNT_NORDIRPLUS) - server->caps &= ~NFS_CAP_READDIRPLUS; - if (server->nfs_client->cl_proto == XPRT_TRANSPORT_RDMA) - server->caps &= ~NFS_CAP_READ_PLUS; - - /* - * Don't use NFS uid/gid mapping if we're using AUTH_SYS or lower - * authentication. - */ - if (nfs4_disable_idmapping && - server->client->cl_auth->au_flavor == RPC_AUTH_UNIX) - server->caps |= NFS_CAP_UIDGID_NOMAP; -} - static int nfs4_server_common_setup(struct nfs_server *server, struct nfs_fh *mntfh, bool auth_probe) { int error; + error = nfs4_delegation_hash_alloc(server); + if (error) + return error; + /* data servers support only a subset of NFSv4.1 */ if (is_ds_only_client(server->nfs_client)) return -EPROTONOSUPPORT; @@ -1118,14 +1090,14 @@ static int nfs4_server_common_setup(struct nfs_server *server, /* We must ensure the session is initialised first */ error = nfs4_init_session(server->nfs_client); if (error < 0) - goto out; + return error; - nfs4_server_set_init_caps(server); + nfs_server_set_init_caps(server); /* Probe the root fh to retrieve its FSID and filehandle */ error = nfs4_get_rootfh(server, mntfh, auth_probe); if (error < 0) - goto out; + return error; dprintk("Server FSID: %llx:%llx\n", (unsigned long long) server->fsid.major, @@ -1134,7 +1106,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, error = nfs_probe_server(server, mntfh); if (error < 0) - goto out; + return error; nfs4_session_limit_rwsize(server); nfs4_session_limit_xasize(server); @@ -1145,8 +1117,7 @@ static int nfs4_server_common_setup(struct nfs_server *server, nfs_server_insert_lists(server); server->mount_time = jiffies; server->destroy = nfs4_destroy_server; -out: - return error; + return 0; } /* @@ -1156,6 +1127,19 @@ static int nfs4_init_server(struct nfs_server *server, struct fs_context *fc) { struct nfs_fs_context *ctx = nfs_fc2context(fc); struct rpc_timeout timeparms; + struct nfs_client_initdata cl_init = { + .hostname = ctx->nfs_server.hostname, + .addr = &ctx->nfs_server._address, + .addrlen = ctx->nfs_server.addrlen, + .ip_addr = ctx->client_address, + .proto = ctx->nfs_server.protocol, + .minorversion = ctx->minorversion, + .net = fc->net_ns, + .timeparms = &timeparms, + .xprtsec = ctx->xprtsec, + .nconnect = ctx->nfs_server.nconnect, + .max_connect = ctx->nfs_server.max_connect, + }; int error; nfs_init_timeout_values(&timeparms, ctx->nfs_server.protocol, @@ -1175,18 +1159,7 @@ static int nfs4_init_server(struct nfs_server *server, struct fs_context *fc) ctx->selected_flavor = RPC_AUTH_UNIX; /* Get a client record */ - error = nfs4_set_client(server, - ctx->nfs_server.hostname, - &ctx->nfs_server._address, - ctx->nfs_server.addrlen, - ctx->client_address, - ctx->nfs_server.protocol, - &timeparms, - ctx->minorversion, - ctx->nfs_server.nconnect, - ctx->nfs_server.max_connect, - fc->net_ns, - &ctx->xprtsec); + error = nfs4_set_client(server, &cl_init); if (error < 0) return error; @@ -1246,18 +1219,28 @@ error: struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) { struct nfs_fs_context *ctx = nfs_fc2context(fc); - struct nfs_client *parent_client; - struct nfs_server *server, *parent_server; - int proto, error; + struct nfs_server *parent_server = NFS_SB(ctx->clone_data.sb); + struct nfs_client *parent_client = parent_server->nfs_client; + struct nfs_client_initdata cl_init = { + .hostname = ctx->nfs_server.hostname, + .addr = &ctx->nfs_server._address, + .addrlen = ctx->nfs_server.addrlen, + .ip_addr = parent_client->cl_ipaddr, + .minorversion = parent_client->cl_mvops->minor_version, + .net = parent_client->cl_net, + .timeparms = parent_server->client->cl_timeout, + .xprtsec = parent_client->cl_xprtsec, + .nconnect = parent_client->cl_nconnect, + .max_connect = parent_client->cl_max_connect, + }; + struct nfs_server *server; bool auth_probe; + int error; server = nfs_alloc_server(); if (!server) return ERR_PTR(-ENOMEM); - parent_server = NFS_SB(ctx->clone_data.sb); - parent_client = parent_server->nfs_client; - server->cred = get_cred(parent_server->cred); /* Initialise the client representation from the parent server */ @@ -1266,38 +1249,17 @@ struct nfs_server *nfs4_create_referral_server(struct fs_context *fc) /* Get a client representation */ #if IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) rpc_set_port(&ctx->nfs_server.address, NFS_RDMA_PORT); - error = nfs4_set_client(server, - ctx->nfs_server.hostname, - &ctx->nfs_server._address, - ctx->nfs_server.addrlen, - parent_client->cl_ipaddr, - XPRT_TRANSPORT_RDMA, - parent_server->client->cl_timeout, - parent_client->cl_mvops->minor_version, - parent_client->cl_nconnect, - parent_client->cl_max_connect, - parent_client->cl_net, - &parent_client->cl_xprtsec); + cl_init.proto = XPRT_TRANSPORT_RDMA; + error = nfs4_set_client(server, &cl_init); if (!error) goto init_server; #endif /* IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) */ - proto = XPRT_TRANSPORT_TCP; + cl_init.proto = XPRT_TRANSPORT_TCP; if (parent_client->cl_xprtsec.policy != RPC_XPRTSEC_NONE) - proto = XPRT_TRANSPORT_TCP_TLS; + cl_init.proto = XPRT_TRANSPORT_TCP_TLS; rpc_set_port(&ctx->nfs_server.address, NFS_PORT); - error = nfs4_set_client(server, - ctx->nfs_server.hostname, - &ctx->nfs_server._address, - ctx->nfs_server.addrlen, - parent_client->cl_ipaddr, - proto, - parent_server->client->cl_timeout, - parent_client->cl_mvops->minor_version, - parent_client->cl_nconnect, - parent_client->cl_max_connect, - parent_client->cl_net, - &parent_client->cl_xprtsec); + error = nfs4_set_client(server, &cl_init); if (error < 0) goto error; @@ -1353,6 +1315,19 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname, char buf[INET6_ADDRSTRLEN + 1]; struct sockaddr_storage address; struct sockaddr *localaddr = (struct sockaddr *)&address; + struct nfs_client_initdata cl_init = { + .hostname = hostname, + .addr = sap, + .addrlen = salen, + .ip_addr = buf, + .proto = clp->cl_proto, + .minorversion = clp->cl_minorversion, + .net = net, + .timeparms = clnt->cl_timeout, + .xprtsec = clp->cl_xprtsec, + .nconnect = clp->cl_nconnect, + .max_connect = clp->cl_max_connect, + }; int error; error = rpc_switch_client_transport(clnt, &xargs, clnt->cl_timeout); @@ -1368,11 +1343,7 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname, nfs_server_remove_lists(server); set_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); - error = nfs4_set_client(server, hostname, sap, salen, buf, - clp->cl_proto, clnt->cl_timeout, - clp->cl_minorversion, - clp->cl_nconnect, clp->cl_max_connect, - net, &clp->cl_xprtsec); + error = nfs4_set_client(server, &cl_init); clear_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); if (error != 0) { nfs_server_insert_lists(server); diff --git a/fs/nfs/nfs4file.c b/fs/nfs/nfs4file.c index 5c749b6117bb..1d6b5f4230c9 100644 --- a/fs/nfs/nfs4file.c +++ b/fs/nfs/nfs4file.c @@ -253,7 +253,6 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, struct nfs_server *server = NFS_SERVER(dst_inode); struct inode *src_inode = file_inode(src_file); unsigned int bs = server->clone_blksize; - bool same_inode = false; int ret; /* NFS does not support deduplication. */ @@ -275,20 +274,8 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, goto out; } - if (src_inode == dst_inode) - same_inode = true; - /* XXX: do we lock at all? what if server needs CB_RECALL_LAYOUT? */ - if (same_inode) { - inode_lock(src_inode); - } else if (dst_inode < src_inode) { - inode_lock_nested(dst_inode, I_MUTEX_PARENT); - inode_lock_nested(src_inode, I_MUTEX_CHILD); - } else { - inode_lock_nested(src_inode, I_MUTEX_PARENT); - inode_lock_nested(dst_inode, I_MUTEX_CHILD); - } - + lock_two_nondirectories(src_inode, dst_inode); /* flush all pending writes on both src and dst so that server * has the latest data */ ret = nfs_sync_inode(src_inode); @@ -306,15 +293,7 @@ static loff_t nfs42_remap_file_range(struct file *src_file, loff_t src_off, truncate_inode_pages_range(&dst_inode->i_data, dst_off, dst_off + count - 1); out_unlock: - if (same_inode) { - inode_unlock(src_inode); - } else if (dst_inode < src_inode) { - inode_unlock(src_inode); - inode_unlock(dst_inode); - } else { - inode_unlock(dst_inode); - inode_unlock(src_inode); - } + unlock_two_nondirectories(src_inode, dst_inode); out: return ret < 0 ? ret : count; } diff --git a/fs/nfs/nfs4getroot.c b/fs/nfs/nfs4getroot.c index 1a69479a3a59..e67ea345de69 100644 --- a/fs/nfs/nfs4getroot.c +++ b/fs/nfs/nfs4getroot.c @@ -12,30 +12,28 @@ int nfs4_get_rootfh(struct nfs_server *server, struct nfs_fh *mntfh, bool auth_probe) { - struct nfs_fsinfo fsinfo; + struct nfs_fattr *fattr = nfs_alloc_fattr(); int ret = -ENOMEM; - fsinfo.fattr = nfs_alloc_fattr(); - if (fsinfo.fattr == NULL) + if (fattr == NULL) goto out; /* Start by getting the root filehandle from the server */ - ret = nfs4_proc_get_rootfh(server, mntfh, &fsinfo, auth_probe); + ret = nfs4_proc_get_rootfh(server, mntfh, fattr, auth_probe); if (ret < 0) { dprintk("nfs4_get_rootfh: getroot error = %d\n", -ret); goto out; } - if (!(fsinfo.fattr->valid & NFS_ATTR_FATTR_TYPE) - || !S_ISDIR(fsinfo.fattr->mode)) { + if (!(fattr->valid & NFS_ATTR_FATTR_TYPE) || !S_ISDIR(fattr->mode)) { printk(KERN_ERR "nfs4_get_rootfh:" " getroot encountered non-directory\n"); ret = -ENOTDIR; goto out; } - memcpy(&server->fsid, &fsinfo.fattr->fsid, sizeof(server->fsid)); + memcpy(&server->fsid, &fattr->fsid, sizeof(server->fsid)); out: - nfs_free_fattr(fsinfo.fattr); + nfs_free_fattr(fattr); return ret; } diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 341740fa293d..7d2b67e06cc3 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -222,6 +222,7 @@ const u32 nfs4_fattr_bitmap[3] = { | FATTR4_WORD1_RAWDEV | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_MODIFY | FATTR4_WORD1_MOUNTED_ON_FILEID, @@ -243,6 +244,7 @@ static const u32 nfs4_pnfs_open_bitmap[3] = { | FATTR4_WORD1_RAWDEV | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_MODIFY, FATTR4_WORD2_MDSTHRESHOLD @@ -323,6 +325,9 @@ static void nfs4_bitmap_copy_adjust(__u32 *dst, const __u32 *src, if (!(cache_validity & NFS_INO_INVALID_OTHER)) dst[1] &= ~(FATTR4_WORD1_OWNER | FATTR4_WORD1_OWNER_GROUP); + if (!(cache_validity & NFS_INO_INVALID_BTIME)) + dst[1] &= ~FATTR4_WORD1_TIME_CREATE; + if (nfs_have_delegated_mtime(inode)) { if (!(cache_validity & NFS_INO_INVALID_ATIME)) dst[1] &= ~(FATTR4_WORD1_TIME_ACCESS|FATTR4_WORD1_TIME_ACCESS_SET); @@ -1307,7 +1312,8 @@ nfs4_update_changeattr_locked(struct inode *inode, NFS_INO_INVALID_ACCESS | NFS_INO_INVALID_ACL | NFS_INO_INVALID_SIZE | NFS_INO_INVALID_OTHER | NFS_INO_INVALID_BLOCKS | NFS_INO_INVALID_NLINK | - NFS_INO_INVALID_MODE | NFS_INO_INVALID_XATTR; + NFS_INO_INVALID_MODE | NFS_INO_INVALID_BTIME | + NFS_INO_INVALID_XATTR; nfsi->attrtimeo = NFS_MINATTRTIMEO(inode); } nfsi->attrtimeo_timestamp = jiffies; @@ -4047,6 +4053,10 @@ static int _nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *f server->fattr_valid &= ~NFS_ATTR_FATTR_CTIME; if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_MODIFY)) server->fattr_valid &= ~NFS_ATTR_FATTR_MTIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_MODIFY)) + server->fattr_valid &= ~NFS_ATTR_FATTR_MTIME; + if (!(res.attr_bitmask[1] & FATTR4_WORD1_TIME_CREATE)) + server->fattr_valid &= ~NFS_ATTR_FATTR_BTIME; memcpy(server->attr_bitmask_nl, res.attr_bitmask, sizeof(server->attr_bitmask)); server->attr_bitmask_nl[2] &= ~FATTR4_WORD2_SECURITY_LABEL; @@ -4082,7 +4092,7 @@ int nfs4_server_capabilities(struct nfs_server *server, struct nfs_fh *fhandle) }; int err; - nfs4_server_set_init_caps(server); + nfs_server_set_init_caps(server); do { err = nfs4_handle_exception(server, _nfs4_server_capabilities(server, fhandle), @@ -4230,15 +4240,18 @@ out: } static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info) + struct nfs_fattr *fattr) { - u32 bitmask[3]; + u32 bitmask[3] = { + [0] = FATTR4_WORD0_TYPE | FATTR4_WORD0_CHANGE | + FATTR4_WORD0_SIZE | FATTR4_WORD0_FSID, + }; struct nfs4_lookup_root_arg args = { .bitmask = bitmask, }; struct nfs4_lookup_res res = { .server = server, - .fattr = info->fattr, + .fattr = fattr, .fh = fhandle, }; struct rpc_message msg = { @@ -4247,27 +4260,20 @@ static int _nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, .rpc_resp = &res, }; - bitmask[0] = nfs4_fattr_bitmap[0]; - bitmask[1] = nfs4_fattr_bitmap[1]; - /* - * Process the label in the upcoming getfattr - */ - bitmask[2] = nfs4_fattr_bitmap[2] & ~FATTR4_WORD2_SECURITY_LABEL; - - nfs_fattr_init(info->fattr); + nfs_fattr_init(fattr); return nfs4_call_sync(server->client, server, &msg, &args.seq_args, &res.seq_res, 0); } static int nfs4_lookup_root(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info) + struct nfs_fattr *fattr) { struct nfs4_exception exception = { .interruptible = true, }; int err; do { - err = _nfs4_lookup_root(server, fhandle, info); - trace_nfs4_lookup_root(server, fhandle, info->fattr, err); + err = _nfs4_lookup_root(server, fhandle, fattr); + trace_nfs4_lookup_root(server, fhandle, fattr, err); switch (err) { case 0: case -NFS4ERR_WRONGSEC: @@ -4280,8 +4286,9 @@ out: return err; } -static int nfs4_lookup_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info, rpc_authflavor_t flavor) +static int nfs4_lookup_root_sec(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs_fattr *fattr, + rpc_authflavor_t flavor) { struct rpc_auth_create_args auth_args = { .pseudoflavor = flavor, @@ -4291,7 +4298,7 @@ static int nfs4_lookup_root_sec(struct nfs_server *server, struct nfs_fh *fhandl auth = rpcauth_create(&auth_args, server->client); if (IS_ERR(auth)) return -EACCES; - return nfs4_lookup_root(server, fhandle, info); + return nfs4_lookup_root(server, fhandle, fattr); } /* @@ -4304,7 +4311,7 @@ static int nfs4_lookup_root_sec(struct nfs_server *server, struct nfs_fh *fhandl * negative errno value. */ static int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info) + struct nfs_fattr *fattr) { /* Per 3530bis 15.33.5 */ static const rpc_authflavor_t flav_array[] = { @@ -4320,8 +4327,9 @@ static int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, if (server->auth_info.flavor_len > 0) { /* try each flavor specified by user */ for (i = 0; i < server->auth_info.flavor_len; i++) { - status = nfs4_lookup_root_sec(server, fhandle, info, - server->auth_info.flavors[i]); + status = nfs4_lookup_root_sec( + server, fhandle, fattr, + server->auth_info.flavors[i]); if (status == -NFS4ERR_WRONGSEC || status == -EACCES) continue; break; @@ -4329,7 +4337,7 @@ static int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, } else { /* no flavors specified by user, try default list */ for (i = 0; i < ARRAY_SIZE(flav_array); i++) { - status = nfs4_lookup_root_sec(server, fhandle, info, + status = nfs4_lookup_root_sec(server, fhandle, fattr, flav_array[i]); if (status == -NFS4ERR_WRONGSEC || status == -EACCES) continue; @@ -4353,28 +4361,22 @@ static int nfs4_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, * nfs4_proc_get_rootfh - get file handle for server's pseudoroot * @server: initialized nfs_server handle * @fhandle: we fill in the pseudo-fs root file handle - * @info: we fill in an FSINFO struct + * @fattr: we fill in a bare bones struct fattr * @auth_probe: probe the auth flavours * * Returns zero on success, or a negative errno. */ int nfs4_proc_get_rootfh(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info, - bool auth_probe) + struct nfs_fattr *fattr, bool auth_probe) { int status = 0; if (!auth_probe) - status = nfs4_lookup_root(server, fhandle, info); + status = nfs4_lookup_root(server, fhandle, fattr); if (auth_probe || status == NFS4ERR_WRONGSEC) - status = server->nfs_client->cl_mvops->find_root_sec(server, - fhandle, info); - - if (status == 0) - status = nfs4_server_capabilities(server, fhandle); - if (status == 0) - status = nfs4_do_fsinfo(server, fhandle, info); + status = server->nfs_client->cl_mvops->find_root_sec( + server, fhandle, fattr); return nfs4_map_errors(status); } @@ -5781,6 +5783,8 @@ void nfs4_bitmask_set(__u32 bitmask[], const __u32 src[], bitmask[1] |= FATTR4_WORD1_TIME_MODIFY; if (cache_validity & NFS_INO_INVALID_BLOCKS) bitmask[1] |= FATTR4_WORD1_SPACE_USED; + if (cache_validity & NFS_INO_INVALID_BTIME) + bitmask[1] |= FATTR4_WORD1_TIME_CREATE; if (cache_validity & NFS_INO_INVALID_SIZE) bitmask[0] |= FATTR4_WORD0_SIZE; @@ -10339,10 +10343,10 @@ nfs4_proc_layoutcommit(struct nfs4_layoutcommit_data *data, bool sync) * Use the state managment nfs_client cl_rpcclient, which uses krb5i (if * possible) as per RFC3530bis and RFC5661 Security Considerations sections */ -static int -_nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info, - struct nfs4_secinfo_flavors *flavors, bool use_integrity) +static int _nfs41_proc_secinfo_no_name(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_secinfo_flavors *flavors, + bool use_integrity) { struct nfs41_secinfo_no_name_args args = { .style = SECINFO_STYLE_CURRENT_FH, @@ -10386,9 +10390,9 @@ _nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle, return status; } -static int -nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info, struct nfs4_secinfo_flavors *flavors) +static int nfs41_proc_secinfo_no_name(struct nfs_server *server, + struct nfs_fh *fhandle, + struct nfs4_secinfo_flavors *flavors) { struct nfs4_exception exception = { .interruptible = true, @@ -10400,7 +10404,7 @@ nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle, /* try to use integrity protection with machine cred */ if (_nfs4_is_integrity_protected(server->nfs_client)) - err = _nfs41_proc_secinfo_no_name(server, fhandle, info, + err = _nfs41_proc_secinfo_no_name(server, fhandle, flavors, true); /* @@ -10410,7 +10414,7 @@ nfs41_proc_secinfo_no_name(struct nfs_server *server, struct nfs_fh *fhandle, * the current filesystem's rpc_client and the user cred. */ if (err == -NFS4ERR_WRONGSEC) - err = _nfs41_proc_secinfo_no_name(server, fhandle, info, + err = _nfs41_proc_secinfo_no_name(server, fhandle, flavors, false); switch (err) { @@ -10426,9 +10430,8 @@ out: return err; } -static int -nfs41_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, - struct nfs_fsinfo *info) +static int nfs41_find_root_sec(struct nfs_server *server, + struct nfs_fh *fhandle, struct nfs_fattr *fattr) { int err; struct page *page; @@ -10444,14 +10447,14 @@ nfs41_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, } flavors = page_address(page); - err = nfs41_proc_secinfo_no_name(server, fhandle, info, flavors); + err = nfs41_proc_secinfo_no_name(server, fhandle, flavors); /* * Fall back on "guess and check" method if * the server doesn't support SECINFO_NO_NAME */ if (err == -NFS4ERR_WRONGSEC || err == -ENOTSUPP) { - err = nfs4_find_root_sec(server, fhandle, info); + err = nfs4_find_root_sec(server, fhandle, fattr); goto out_freepage; } if (err) @@ -10476,8 +10479,8 @@ nfs41_find_root_sec(struct nfs_server *server, struct nfs_fh *fhandle, flavor = RPC_AUTH_MAXFLAVOR; if (flavor != RPC_AUTH_MAXFLAVOR) { - err = nfs4_lookup_root_sec(server, fhandle, - info, flavor); + err = nfs4_lookup_root_sec(server, fhandle, fattr, + flavor); if (!err) break; } @@ -10680,6 +10683,8 @@ nfs41_free_lock_state(struct nfs_server *server, struct nfs4_lock_state *lsp) static bool nfs41_match_stateid(const nfs4_stateid *s1, const nfs4_stateid *s2) { + trace_nfs41_match_stateid(s1, s2); + if (s1->type != s2->type) return false; @@ -10697,6 +10702,8 @@ static bool nfs41_match_stateid(const nfs4_stateid *s1, static bool nfs4_match_stateid(const nfs4_stateid *s1, const nfs4_stateid *s2) { + trace_nfs4_match_stateid(s1, s2); + return nfs4_stateid_match(s1, s2); } @@ -10867,7 +10874,7 @@ const struct nfs4_minor_version_ops *nfs_v4_minor_ops[] = { static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) { - ssize_t error, error2, error3, error4; + ssize_t error, error2, error3, error4 = 0; size_t left = size; error = generic_listxattr(dentry, list, left); @@ -10895,9 +10902,11 @@ static ssize_t nfs4_listxattr(struct dentry *dentry, char *list, size_t size) left -= error3; } - error4 = security_inode_listsecurity(d_inode(dentry), list, left); - if (error4 < 0) - return error4; + if (!nfs_server_capable(d_inode(dentry), NFS_CAP_SECURITY_LABEL)) { + error4 = security_inode_listsecurity(d_inode(dentry), list, left); + if (error4 < 0) + return error4; + } error += error2 + error3 + error4; if (size && error > size) @@ -10951,6 +10960,26 @@ static const struct inode_operations nfs4_file_inode_operations = { .listxattr = nfs4_listxattr, }; +static struct nfs_server *nfs4_clone_server(struct nfs_server *source, + struct nfs_fh *fh, struct nfs_fattr *fattr, + rpc_authflavor_t flavor) +{ + struct nfs_server *server; + int error; + + server = nfs_clone_server(source, fh, fattr, flavor); + if (IS_ERR(server)) + return server; + + error = nfs4_delegation_hash_alloc(server); + if (error) { + nfs_free_server(server); + return ERR_PTR(error); + } + + return server; +} + const struct nfs_rpc_ops nfs_v4_clientops = { .version = 4, /* protocol version */ .dentry_ops = &nfs4_dentry_operations, @@ -11003,7 +11032,7 @@ const struct nfs_rpc_ops nfs_v4_clientops = { .init_client = nfs4_init_client, .free_client = nfs4_free_client, .create_server = nfs4_create_server, - .clone_server = nfs_clone_server, + .clone_server = nfs4_clone_server, .discover_trunking = nfs4_discover_trunking, .enable_swap = nfs4_enable_swap, .disable_swap = nfs4_disable_swap, diff --git a/fs/nfs/nfs4trace.c b/fs/nfs/nfs4trace.c index 389941ccc9c9..987c92d6364b 100644 --- a/fs/nfs/nfs4trace.c +++ b/fs/nfs/nfs4trace.c @@ -26,11 +26,13 @@ EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_done); EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_done); EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_read_pagelist); EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_mds_fallback_write_pagelist); +EXPORT_TRACEPOINT_SYMBOL_GPL(pnfs_ds_connect); EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_read_error); EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_write_error); EXPORT_TRACEPOINT_SYMBOL_GPL(ff_layout_commit_error); +EXPORT_TRACEPOINT_SYMBOL_GPL(bl_ext_tree_prepare_commit); EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_reg); EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_reg_err); EXPORT_TRACEPOINT_SYMBOL_GPL(bl_pr_key_unreg); diff --git a/fs/nfs/nfs4trace.h b/fs/nfs/nfs4trace.h index deab4c0e21a0..9776d220cec3 100644 --- a/fs/nfs/nfs4trace.h +++ b/fs/nfs/nfs4trace.h @@ -14,6 +14,8 @@ #include <trace/misc/fs.h> #include <trace/misc/nfs.h> +#include "delegation.h" + #define show_nfs_fattr_flags(valid) \ __print_flags((unsigned long)valid, "|", \ { NFS_ATTR_FATTR_TYPE, "TYPE" }, \ @@ -30,7 +32,8 @@ { NFS_ATTR_FATTR_CTIME, "CTIME" }, \ { NFS_ATTR_FATTR_CHANGE, "CHANGE" }, \ { NFS_ATTR_FATTR_OWNER_NAME, "OWNER_NAME" }, \ - { NFS_ATTR_FATTR_GROUP_NAME, "GROUP_NAME" }) + { NFS_ATTR_FATTR_GROUP_NAME, "GROUP_NAME" }, \ + { NFS_ATTR_FATTR_BTIME, "BTIME" }) DECLARE_EVENT_CLASS(nfs4_clientid_event, TP_PROTO( @@ -273,6 +276,32 @@ TRACE_EVENT(nfs4_cb_offload, show_nfs_stable_how(__entry->cb_how) ) ); + +TRACE_EVENT(pnfs_ds_connect, + TP_PROTO( + char *ds_remotestr, + int status + ), + + TP_ARGS(ds_remotestr, status), + + TP_STRUCT__entry( + __string(ds_ips, ds_remotestr) + __field(int, status) + ), + + TP_fast_assign( + __assign_str(ds_ips); + __entry->status = status; + ), + + TP_printk( + "ds_ips=%s, status=%d", + __get_str(ds_ips), + __entry->status + ) +); + #endif /* CONFIG_NFS_V4_1 */ TRACE_EVENT(nfs4_setup_sequence, @@ -956,6 +985,52 @@ DECLARE_EVENT_CLASS(nfs4_set_delegation_event, TP_ARGS(inode, fmode)) DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_set_delegation); DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_reclaim_delegation); +DEFINE_NFS4_SET_DELEGATION_EVENT(nfs4_detach_delegation); + +#define show_delegation_flags(flags) \ + __print_flags(flags, "|", \ + { BIT(NFS_DELEGATION_NEED_RECLAIM), "NEED_RECLAIM" }, \ + { BIT(NFS_DELEGATION_RETURN), "RETURN" }, \ + { BIT(NFS_DELEGATION_RETURN_IF_CLOSED), "RETURN_IF_CLOSED" }, \ + { BIT(NFS_DELEGATION_REFERENCED), "REFERENCED" }, \ + { BIT(NFS_DELEGATION_RETURNING), "RETURNING" }, \ + { BIT(NFS_DELEGATION_REVOKED), "REVOKED" }, \ + { BIT(NFS_DELEGATION_TEST_EXPIRED), "TEST_EXPIRED" }, \ + { BIT(NFS_DELEGATION_INODE_FREEING), "INODE_FREEING" }, \ + { BIT(NFS_DELEGATION_RETURN_DELAYED), "RETURN_DELAYED" }) + +DECLARE_EVENT_CLASS(nfs4_delegation_event, + TP_PROTO( + const struct nfs_delegation *delegation + ), + + TP_ARGS(delegation), + + TP_STRUCT__entry( + __field(u32, fhandle) + __field(unsigned int, fmode) + __field(unsigned long, flags) + ), + + TP_fast_assign( + __entry->fhandle = nfs_fhandle_hash(NFS_FH(delegation->inode)); + __entry->fmode = delegation->type; + __entry->flags = delegation->flags; + ), + + TP_printk( + "fhandle=0x%08x fmode=%s flags=%s", + __entry->fhandle, show_fs_fmode_flags(__entry->fmode), + show_delegation_flags(__entry->flags) + ) +); +#define DEFINE_NFS4_DELEGATION_EVENT(name) \ + DEFINE_EVENT(nfs4_delegation_event, name, \ + TP_PROTO( \ + const struct nfs_delegation *delegation \ + ), \ + TP_ARGS(delegation)) +DEFINE_NFS4_DELEGATION_EVENT(nfs_delegation_need_return); TRACE_EVENT(nfs4_delegreturn_exit, TP_PROTO( @@ -1449,6 +1524,63 @@ DECLARE_EVENT_CLASS(nfs4_inode_stateid_callback_event, DEFINE_NFS4_INODE_STATEID_CALLBACK_EVENT(nfs4_cb_recall); DEFINE_NFS4_INODE_STATEID_CALLBACK_EVENT(nfs4_cb_layoutrecall_file); +#define show_stateid_type(type) \ + __print_symbolic(type, \ + { NFS4_INVALID_STATEID_TYPE, "INVALID" }, \ + { NFS4_SPECIAL_STATEID_TYPE, "SPECIAL" }, \ + { NFS4_OPEN_STATEID_TYPE, "OPEN" }, \ + { NFS4_LOCK_STATEID_TYPE, "LOCK" }, \ + { NFS4_DELEGATION_STATEID_TYPE, "DELEGATION" }, \ + { NFS4_LAYOUT_STATEID_TYPE, "LAYOUT" }, \ + { NFS4_PNFS_DS_STATEID_TYPE, "PNFS_DS" }, \ + { NFS4_REVOKED_STATEID_TYPE, "REVOKED" }, \ + { NFS4_FREED_STATEID_TYPE, "FREED" }) + +DECLARE_EVENT_CLASS(nfs4_match_stateid_event, + TP_PROTO( + const nfs4_stateid *s1, + const nfs4_stateid *s2 + ), + + TP_ARGS(s1, s2), + + TP_STRUCT__entry( + __field(int, s1_seq) + __field(int, s2_seq) + __field(u32, s1_hash) + __field(u32, s2_hash) + __field(int, s1_type) + __field(int, s2_type) + ), + + TP_fast_assign( + __entry->s1_seq = s1->seqid; + __entry->s1_hash = nfs_stateid_hash(s1); + __entry->s1_type = s1->type; + __entry->s2_seq = s2->seqid; + __entry->s2_hash = nfs_stateid_hash(s2); + __entry->s2_type = s2->type; + ), + + TP_printk( + "s1=%s:%x:%u s2=%s:%x:%u", + show_stateid_type(__entry->s1_type), + __entry->s1_hash, __entry->s1_seq, + show_stateid_type(__entry->s2_type), + __entry->s2_hash, __entry->s2_seq + ) +); + +#define DEFINE_NFS4_MATCH_STATEID_EVENT(name) \ + DEFINE_EVENT(nfs4_match_stateid_event, name, \ + TP_PROTO( \ + const nfs4_stateid *s1, \ + const nfs4_stateid *s2 \ + ), \ + TP_ARGS(s1, s2)) +DEFINE_NFS4_MATCH_STATEID_EVENT(nfs41_match_stateid); +DEFINE_NFS4_MATCH_STATEID_EVENT(nfs4_match_stateid); + DECLARE_EVENT_CLASS(nfs4_idmap_event, TP_PROTO( const char *name, @@ -2163,6 +2295,40 @@ TRACE_EVENT(ff_layout_commit_error, ) ); +TRACE_EVENT(bl_ext_tree_prepare_commit, + TP_PROTO( + int ret, + size_t count, + u64 lwb, + bool not_all_ranges + ), + + TP_ARGS(ret, count, lwb, not_all_ranges), + + TP_STRUCT__entry( + __field(int, ret) + __field(size_t, count) + __field(u64, lwb) + __field(bool, not_all_ranges) + ), + + TP_fast_assign( + __entry->ret = ret; + __entry->count = count; + __entry->lwb = lwb; + __entry->not_all_ranges = not_all_ranges; + ), + + TP_printk( + "ret=%d, found %zu ranges, lwb=%llu%s", + __entry->ret, + __entry->count, + __entry->lwb, + __entry->not_all_ranges ? ", not all ranges encoded" : + "" + ) +); + DECLARE_EVENT_CLASS(pnfs_bl_pr_key_class, TP_PROTO( const struct block_device *bdev, diff --git a/fs/nfs/nfs4xdr.c b/fs/nfs/nfs4xdr.c index 318afde38057..49ff98571fa5 100644 --- a/fs/nfs/nfs4xdr.c +++ b/fs/nfs/nfs4xdr.c @@ -1623,6 +1623,7 @@ static void encode_readdir(struct xdr_stream *xdr, const struct nfs4_readdir_arg | FATTR4_WORD1_RAWDEV | FATTR4_WORD1_SPACE_USED | FATTR4_WORD1_TIME_ACCESS + | FATTR4_WORD1_TIME_CREATE | FATTR4_WORD1_TIME_METADATA | FATTR4_WORD1_TIME_MODIFY; attrs[2] |= FATTR4_WORD2_SECURITY_LABEL; @@ -4207,6 +4208,24 @@ static int decode_attr_time_access(struct xdr_stream *xdr, uint32_t *bitmap, str return status; } +static int decode_attr_time_create(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) +{ + int status = 0; + + time->tv_sec = 0; + time->tv_nsec = 0; + if (unlikely(bitmap[1] & (FATTR4_WORD1_TIME_CREATE - 1U))) + return -EIO; + if (likely(bitmap[1] & FATTR4_WORD1_TIME_CREATE)) { + status = decode_attr_time(xdr, time); + if (status == 0) + status = NFS_ATTR_FATTR_BTIME; + bitmap[1] &= ~FATTR4_WORD1_TIME_CREATE; + } + dprintk("%s: btime=%lld\n", __func__, time->tv_sec); + return status; +} + static int decode_attr_time_metadata(struct xdr_stream *xdr, uint32_t *bitmap, struct timespec64 *time) { int status = 0; @@ -4781,6 +4800,11 @@ static int decode_getfattr_attrs(struct xdr_stream *xdr, uint32_t *bitmap, goto xdr_error; fattr->valid |= status; + status = decode_attr_time_create(xdr, bitmap, &fattr->btime); + if (status < 0) + goto xdr_error; + fattr->valid |= status; + status = decode_attr_time_metadata(xdr, bitmap, &fattr->ctime); if (status < 0) goto xdr_error; diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index 7a058bd8c566..96b1323318c2 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -32,7 +32,8 @@ { NFS_INO_INVALID_BLOCKS, "INVALID_BLOCKS" }, \ { NFS_INO_INVALID_XATTR, "INVALID_XATTR" }, \ { NFS_INO_INVALID_NLINK, "INVALID_NLINK" }, \ - { NFS_INO_INVALID_MODE, "INVALID_MODE" }) + { NFS_INO_INVALID_MODE, "INVALID_MODE" }, \ + { NFS_INO_INVALID_BTIME, "INVALID_BTIME" }) #define nfs_show_nfsi_flags(v) \ __print_flags(v, "|", \ @@ -56,6 +57,7 @@ DECLARE_EVENT_CLASS(nfs_inode_event, __field(u32, fhandle) __field(u64, fileid) __field(u64, version) + __field(unsigned long, cache_validity) ), TP_fast_assign( @@ -64,14 +66,17 @@ DECLARE_EVENT_CLASS(nfs_inode_event, __entry->fileid = nfsi->fileid; __entry->fhandle = nfs_fhandle_hash(&nfsi->fh); __entry->version = inode_peek_iversion_raw(inode); + __entry->cache_validity = nfsi->cache_validity; ), TP_printk( - "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu ", + "fileid=%02x:%02x:%llu fhandle=0x%08x version=%llu cache_validity=0x%lx (%s)", MAJOR(__entry->dev), MINOR(__entry->dev), (unsigned long long)__entry->fileid, __entry->fhandle, - (unsigned long long)__entry->version + (unsigned long long)__entry->version, + __entry->cache_validity, + nfs_show_cache_validity(__entry->cache_validity) ) ); diff --git a/fs/nfs/pnfs.c b/fs/nfs/pnfs.c index 1a7ec68bde15..a3135b5af7ee 100644 --- a/fs/nfs/pnfs.c +++ b/fs/nfs/pnfs.c @@ -306,7 +306,6 @@ void pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) { struct inode *inode; - unsigned long i_state; if (!lo) return; @@ -317,12 +316,11 @@ pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo) if (!list_empty(&lo->plh_segs)) WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n"); pnfs_detach_layout_hdr(lo); - i_state = inode->i_state; + /* Notify pnfs_destroy_layout_final() that we're done */ + if (inode->i_state & (I_FREEING | I_CLEAR)) + wake_up_var_locked(lo, &inode->i_lock); spin_unlock(&inode->i_lock); pnfs_free_layout_hdr(lo); - /* Notify pnfs_destroy_layout_final() that we're done */ - if (i_state & (I_FREEING | I_CLEAR)) - wake_up_var(lo); } } @@ -809,23 +807,17 @@ void pnfs_destroy_layout(struct nfs_inode *nfsi) } EXPORT_SYMBOL_GPL(pnfs_destroy_layout); -static bool pnfs_layout_removed(struct nfs_inode *nfsi, - struct pnfs_layout_hdr *lo) -{ - bool ret; - - spin_lock(&nfsi->vfs_inode.i_lock); - ret = nfsi->layout != lo; - spin_unlock(&nfsi->vfs_inode.i_lock); - return ret; -} - void pnfs_destroy_layout_final(struct nfs_inode *nfsi) { struct pnfs_layout_hdr *lo = __pnfs_destroy_layout(nfsi); + struct inode *inode = &nfsi->vfs_inode; - if (lo) - wait_var_event(lo, pnfs_layout_removed(nfsi, lo)); + if (lo) { + spin_lock(&inode->i_lock); + wait_var_event_spinlock(lo, nfsi->layout != lo, + &inode->i_lock); + spin_unlock(&inode->i_lock); + } } static bool @@ -3340,6 +3332,7 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) struct nfs_inode *nfsi = NFS_I(inode); loff_t end_pos; int status; + bool mark_as_dirty = false; if (!pnfs_layoutcommit_outstanding(inode)) return 0; @@ -3391,19 +3384,23 @@ pnfs_layoutcommit_inode(struct inode *inode, bool sync) if (ld->prepare_layoutcommit) { status = ld->prepare_layoutcommit(&data->args); if (status) { - put_cred(data->cred); + if (status != -ENOSPC) + put_cred(data->cred); spin_lock(&inode->i_lock); set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags); if (end_pos > nfsi->layout->plh_lwb) nfsi->layout->plh_lwb = end_pos; - goto out_unlock; + if (status != -ENOSPC) + goto out_unlock; + spin_unlock(&inode->i_lock); + mark_as_dirty = true; } } status = nfs4_proc_layoutcommit(data, sync); out: - if (status) + if (status || mark_as_dirty) mark_inode_dirty_sync(inode); dprintk("<-- %s status %d\n", __func__, status); return status; diff --git a/fs/nfs/pnfs_nfs.c b/fs/nfs/pnfs_nfs.c index b4ccdf78d4dd..7b32afb29782 100644 --- a/fs/nfs/pnfs_nfs.c +++ b/fs/nfs/pnfs_nfs.c @@ -17,6 +17,7 @@ #include "internal.h" #include "pnfs.h" #include "netns.h" +#include "nfs4trace.h" #define NFSDBG_FACILITY NFSDBG_PNFS @@ -1007,8 +1008,10 @@ int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds, err = nfs4_wait_ds_connect(ds); if (err || ds->ds_clp) goto out; - if (nfs4_test_deviceid_unavailable(devid)) - return -ENODEV; + if (nfs4_test_deviceid_unavailable(devid)) { + err = -ENODEV; + goto out; + } } while (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) != 0); if (ds->ds_clp) @@ -1038,11 +1041,12 @@ out: if (!ds->ds_clp || !nfs_client_init_is_complete(ds->ds_clp)) { WARN_ON_ONCE(ds->ds_clp || !nfs4_test_deviceid_unavailable(devid)); - return -EINVAL; - } - err = nfs_client_init_status(ds->ds_clp); + err = -EINVAL; + } else + err = nfs_client_init_status(ds->ds_clp); } + trace_pnfs_ds_connect(ds->ds_remotestr, err); return err; } EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_connect); diff --git a/fs/nfs/write.c b/fs/nfs/write.c index cf1d720b8251..fa5c41d0989a 100644 --- a/fs/nfs/write.c +++ b/fs/nfs/write.c @@ -2113,8 +2113,12 @@ int nfs_migrate_folio(struct address_space *mapping, struct folio *dst, * that we can safely release the inode reference while holding * the folio lock. */ - if (folio_test_private(src)) - return -EBUSY; + if (folio_test_private(src)) { + if (mode == MIGRATE_SYNC) + nfs_wb_folio(src->mapping->host, src); + if (folio_test_private(src)) + return -EBUSY; + } if (folio_test_private_2(src)) { /* [DEPRECATED] */ if (mode == MIGRATE_ASYNC) diff --git a/fs/nfs_common/nfslocalio.c b/fs/nfs_common/nfslocalio.c index 05c7c16e37ab..dd715cdb6c04 100644 --- a/fs/nfs_common/nfslocalio.c +++ b/fs/nfs_common/nfslocalio.c @@ -177,7 +177,7 @@ static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) /* nfs_close_local_fh() is doing the * close and we must wait. until it unlinks */ - wait_var_event_spinlock(nfl, + wait_var_event_spinlock(nfs_uuid, list_first_entry_or_null( &nfs_uuid->files, struct nfs_file_localio, @@ -198,8 +198,7 @@ static bool nfs_uuid_put(nfs_uuid_t *nfs_uuid) /* Now we can allow racing nfs_close_local_fh() to * skip the locking. */ - RCU_INIT_POINTER(nfl->nfs_uuid, NULL); - wake_up_var_locked(&nfl->nfs_uuid, &nfs_uuid->lock); + store_release_wake_up(&nfl->nfs_uuid, RCU_INITIALIZER(NULL)); } /* Remove client from nn->local_clients */ @@ -243,15 +242,20 @@ void nfs_localio_invalidate_clients(struct list_head *nn_local_clients, } EXPORT_SYMBOL_GPL(nfs_localio_invalidate_clients); -static void nfs_uuid_add_file(nfs_uuid_t *nfs_uuid, struct nfs_file_localio *nfl) +static int nfs_uuid_add_file(nfs_uuid_t *nfs_uuid, struct nfs_file_localio *nfl) { + int ret = 0; + /* Add nfl to nfs_uuid->files if it isn't already */ spin_lock(&nfs_uuid->lock); - if (list_empty(&nfl->list)) { + if (rcu_access_pointer(nfs_uuid->net) == NULL) { + ret = -ENXIO; + } else if (list_empty(&nfl->list)) { rcu_assign_pointer(nfl->nfs_uuid, nfs_uuid); list_add_tail(&nfl->list, &nfs_uuid->files); } spin_unlock(&nfs_uuid->lock); + return ret; } /* @@ -285,11 +289,13 @@ struct nfsd_file *nfs_open_local_fh(nfs_uuid_t *uuid, } rcu_read_unlock(); /* We have an implied reference to net thanks to nfsd_net_try_get */ - localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, - cred, nfs_fh, pnf, fmode); + localio = nfs_to->nfsd_open_local_fh(net, uuid->dom, rpc_clnt, cred, + nfs_fh, pnf, fmode); + if (!IS_ERR(localio) && nfs_uuid_add_file(uuid, nfl) < 0) { + /* Delete the cached file when racing with nfs_uuid_put() */ + nfs_to_nfsd_file_put_local(pnf); + } nfs_to_nfsd_net_put(net); - if (!IS_ERR(localio)) - nfs_uuid_add_file(uuid, nfl); return localio; } @@ -314,7 +320,7 @@ void nfs_close_local_fh(struct nfs_file_localio *nfl) rcu_read_unlock(); return; } - if (list_empty(&nfs_uuid->files)) { + if (list_empty(&nfl->list)) { /* nfs_uuid_put() has started closing files, wait for it * to finished */ @@ -338,7 +344,7 @@ void nfs_close_local_fh(struct nfs_file_localio *nfl) */ spin_lock(&nfs_uuid->lock); list_del_init(&nfl->list); - wake_up_var_locked(&nfl->nfs_uuid, &nfs_uuid->lock); + wake_up_var_locked(nfs_uuid, &nfs_uuid->lock); spin_unlock(&nfs_uuid->lock); } EXPORT_SYMBOL_GPL(nfs_close_local_fh); diff --git a/fs/smb/client/Makefile b/fs/smb/client/Makefile index 22023e30915b..4c97b31a25c2 100644 --- a/fs/smb/client/Makefile +++ b/fs/smb/client/Makefile @@ -32,6 +32,6 @@ cifs-$(CONFIG_CIFS_SMB_DIRECT) += smbdirect.o cifs-$(CONFIG_CIFS_ROOT) += cifsroot.o -cifs-$(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) += smb1ops.o cifssmb.o +cifs-$(CONFIG_CIFS_ALLOW_INSECURE_LEGACY) += smb1ops.o cifssmb.o cifstransport.o cifs-$(CONFIG_CIFS_COMPRESSION) += compress.o compress/lz77.o diff --git a/fs/smb/client/cifs_debug.c b/fs/smb/client/cifs_debug.c index f1cea365b6f1..beb4f18f05ef 100644 --- a/fs/smb/client/cifs_debug.c +++ b/fs/smb/client/cifs_debug.c @@ -60,7 +60,7 @@ void cifs_dump_mids(struct TCP_Server_Info *server) return; cifs_dbg(VFS, "Dump pending requests:\n"); - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_entry(mid_entry, &server->pending_mid_q, qhead) { cifs_dbg(VFS, "State: %d Cmd: %d Pid: %d Cbdata: %p Mid %llu\n", mid_entry->mid_state, @@ -83,7 +83,7 @@ void cifs_dump_mids(struct TCP_Server_Info *server) mid_entry->resp_buf, 62); } } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); #endif /* CONFIG_CIFS_DEBUG2 */ } @@ -412,6 +412,7 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) spin_lock(&cifs_tcp_ses_lock); list_for_each_entry(server, &cifs_tcp_ses_list, tcp_ses_list) { #ifdef CONFIG_CIFS_SMB_DIRECT + struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; #endif @@ -436,7 +437,8 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) seq_printf(m, "\nSMBDirect transport not available"); goto skip_rdma; } - sp = &server->smbd_conn->socket.parameters; + sc = &server->smbd_conn->socket; + sp = &sc->parameters; seq_printf(m, "\nSMBDirect (in hex) protocol version: %x " "transport status: %x", @@ -465,15 +467,13 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) seq_printf(m, "\nRead Queue count_reassembly_queue: %x " "count_enqueue_reassembly_queue: %x " "count_dequeue_reassembly_queue: %x " - "fragment_reassembly_remaining: %x " "reassembly_data_length: %x " "reassembly_queue_length: %x", server->smbd_conn->count_reassembly_queue, server->smbd_conn->count_enqueue_reassembly_queue, server->smbd_conn->count_dequeue_reassembly_queue, - server->smbd_conn->fragment_reassembly_remaining, - server->smbd_conn->reassembly_data_length, - server->smbd_conn->reassembly_queue_length); + sc->recv_io.reassembly.data_length, + sc->recv_io.reassembly.queue_length); seq_printf(m, "\nCurrent Credits send_credits: %x " "receive_credits: %x receive_credit_target: %x", atomic_read(&server->smbd_conn->send_credits), @@ -481,10 +481,8 @@ static int cifs_debug_data_proc_show(struct seq_file *m, void *v) server->smbd_conn->receive_credit_target); seq_printf(m, "\nPending send_pending: %x ", atomic_read(&server->smbd_conn->send_pending)); - seq_printf(m, "\nReceive buffers count_receive_queue: %x " - "count_empty_packet_queue: %x", - server->smbd_conn->count_receive_queue, - server->smbd_conn->count_empty_packet_queue); + seq_printf(m, "\nReceive buffers count_receive_queue: %x ", + server->smbd_conn->count_receive_queue); seq_printf(m, "\nMR responder_resources: %x " "max_frmr_depth: %x mr_type: %x", server->smbd_conn->responder_resources, @@ -672,7 +670,7 @@ skip_rdma: seq_printf(m, "\n\tServer ConnectionId: 0x%llx", chan_server->conn_id); - spin_lock(&chan_server->mid_lock); + spin_lock(&chan_server->mid_queue_lock); list_for_each_entry(mid_entry, &chan_server->pending_mid_q, qhead) { seq_printf(m, "\n\t\tState: %d com: %d pid: %d cbdata: %p mid %llu", mid_entry->mid_state, @@ -681,7 +679,7 @@ skip_rdma: mid_entry->callback_data, mid_entry->mid); } - spin_unlock(&chan_server->mid_lock); + spin_unlock(&chan_server->mid_queue_lock); } spin_unlock(&ses->chan_lock); seq_puts(m, "\n--\n"); diff --git a/fs/smb/client/cifsfs.c b/fs/smb/client/cifsfs.c index 0fdadd668a81..3bd85ab2deb1 100644 --- a/fs/smb/client/cifsfs.c +++ b/fs/smb/client/cifsfs.c @@ -77,7 +77,7 @@ unsigned int global_secflags = CIFSSEC_DEF; unsigned int GlobalCurrentXid; /* protected by GlobalMid_Lock */ unsigned int GlobalTotalActiveXid; /* prot by GlobalMid_Lock */ unsigned int GlobalMaxActiveXid; /* prot by GlobalMid_Lock */ -spinlock_t GlobalMid_Lock; /* protects above & list operations on midQ entries */ +DEFINE_SPINLOCK(GlobalMid_Lock); /* protects above & list operations on midQ entries */ /* * Global counters, updated atomically @@ -97,7 +97,7 @@ atomic_t total_buf_alloc_count; atomic_t total_small_buf_alloc_count; #endif/* STATS2 */ struct list_head cifs_tcp_ses_list; -spinlock_t cifs_tcp_ses_lock; +DEFINE_SPINLOCK(cifs_tcp_ses_lock); static const struct super_operations cifs_super_ops; unsigned int CIFSMaxBufSize = CIFS_MAX_MSGSIZE; module_param(CIFSMaxBufSize, uint, 0444); @@ -723,7 +723,7 @@ cifs_show_options(struct seq_file *s, struct dentry *root) else seq_puts(s, ",nativesocket"); seq_show_option(s, "symlink", - cifs_symlink_type_str(get_cifs_symlink_type(cifs_sb))); + cifs_symlink_type_str(cifs_symlink_type(cifs_sb))); seq_printf(s, ",rsize=%u", cifs_sb->ctx->rsize); seq_printf(s, ",wsize=%u", cifs_sb->ctx->wsize); @@ -1863,8 +1863,6 @@ init_cifs(void) GlobalCurrentXid = 0; GlobalTotalActiveXid = 0; GlobalMaxActiveXid = 0; - spin_lock_init(&cifs_tcp_ses_lock); - spin_lock_init(&GlobalMid_Lock); cifs_lock_secret = get_random_u32(); diff --git a/fs/smb/client/cifsglob.h b/fs/smb/client/cifsglob.h index 19dd901fe8ab..e6830ab3a546 100644 --- a/fs/smb/client/cifsglob.h +++ b/fs/smb/client/cifsglob.h @@ -732,7 +732,8 @@ struct TCP_Server_Info { #endif wait_queue_head_t response_q; wait_queue_head_t request_q; /* if more than maxmpx to srvr must block*/ - spinlock_t mid_lock; /* protect mid queue and it's entries */ + spinlock_t mid_queue_lock; /* protect mid queue */ + spinlock_t mid_counter_lock; struct list_head pending_mid_q; bool noblocksnd; /* use blocking sendmsg */ bool noautotune; /* do not autotune send buf sizes */ @@ -770,7 +771,7 @@ struct TCP_Server_Info { /* SMB_COM_WRITE_RAW or SMB_COM_READ_RAW. */ unsigned int capabilities; /* selective disabling of caps by smb sess */ int timeAdj; /* Adjust for difference in server time zone in sec */ - __u64 CurrentMid; /* multiplex id - rotating counter, protected by GlobalMid_Lock */ + __u64 current_mid; /* multiplex id - rotating counter, protected by mid_counter_lock */ char cryptkey[CIFS_CRYPTO_KEY_SIZE]; /* used by ntlm, ntlmv2 etc */ /* 16th byte of RFC1001 workstation name is always null */ char workstation_RFC1001_name[RFC1001_NAME_LEN_WITH_NULL]; @@ -1729,9 +1730,10 @@ struct mid_q_entry { unsigned int resp_buf_size; int mid_state; /* wish this were enum but can not pass to wait_event */ int mid_rc; /* rc for MID_RC */ - unsigned int mid_flags; __le16 command; /* smb command code */ unsigned int optype; /* operation type */ + bool wait_cancelled:1; /* Cancelled while waiting for response */ + bool deleted_from_q:1; /* Whether Mid has been dequeued frem pending_mid_q */ bool large_buf:1; /* if valid response, is pointer to large buf */ bool multiRsp:1; /* multiple trans2 responses for one request */ bool multiEnd:1; /* both received */ @@ -1893,10 +1895,6 @@ static inline bool is_replayable_error(int error) #define MID_RESPONSE_READY 0x40 /* ready for other process handle the rsp */ #define MID_RC 0x80 /* mid_rc contains custom rc */ -/* Flags */ -#define MID_WAIT_CANCELLED 1 /* Cancelled while waiting for response */ -#define MID_DELETED 2 /* Mid has been dequeued/deleted */ - /* Types of response buffer returned from SendReceive2 */ #define CIFS_NO_BUFFER 0 /* Response buffer not returned */ #define CIFS_SMALL_BUFFER 1 @@ -2007,9 +2005,9 @@ require use of the stronger protocol */ * GlobalCurrentXid * GlobalTotalActiveXid * TCP_Server_Info->srv_lock (anything in struct not protected by another lock and can change) - * TCP_Server_Info->mid_lock TCP_Server_Info->pending_mid_q cifs_get_tcp_session - * ->CurrentMid - * (any changes in mid_q_entry fields) + * TCP_Server_Info->mid_queue_lock TCP_Server_Info->pending_mid_q cifs_get_tcp_session + * mid_q_entry->deleted_from_q + * TCP_Server_Info->mid_counter_lock TCP_Server_Info->current_mid cifs_get_tcp_session * TCP_Server_Info->req_lock TCP_Server_Info->in_flight cifs_get_tcp_session * ->credits * ->echo_credits @@ -2377,4 +2375,9 @@ static inline bool cifs_netbios_name(const char *name, size_t namelen) return ret; } +#define CIFS_REPARSE_SUPPORT(tcon) \ + ((tcon)->posix_extensions || \ + (le32_to_cpu((tcon)->fsAttrInfo.Attributes) & \ + FILE_SUPPORTS_REPARSE_POINTS)) + #endif /* _CIFS_GLOB_H */ diff --git a/fs/smb/client/cifsproto.h b/fs/smb/client/cifsproto.h index 40ec0634377f..c34c533b2efa 100644 --- a/fs/smb/client/cifsproto.h +++ b/fs/smb/client/cifsproto.h @@ -116,16 +116,31 @@ extern int SendReceive(const unsigned int /* xid */ , struct cifs_ses *, int * /* bytes returned */ , const int); extern int SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, char *in_buf, int flags); +int cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server); extern struct mid_q_entry *cifs_setup_request(struct cifs_ses *, struct TCP_Server_Info *, struct smb_rqst *); extern struct mid_q_entry *cifs_setup_async_request(struct TCP_Server_Info *, struct smb_rqst *); +int __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst); extern int cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, bool log_error); +int wait_for_free_request(struct TCP_Server_Info *server, const int flags, + unsigned int *instance); extern int cifs_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, size_t *num, struct cifs_credits *credits); + +static inline int +send_cancel(struct TCP_Server_Info *server, struct smb_rqst *rqst, + struct mid_q_entry *mid) +{ + return server->ops->send_cancel ? + server->ops->send_cancel(server, rqst, mid) : 0; +} + +int wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *midQ); extern int SendReceive2(const unsigned int /* xid */ , struct cifs_ses *, struct kvec *, int /* nvec to send */, int * /* type of buf returned */, const int flags, diff --git a/fs/smb/client/cifssmb.c b/fs/smb/client/cifssmb.c index 6c890db06593..d20766f664c4 100644 --- a/fs/smb/client/cifssmb.c +++ b/fs/smb/client/cifssmb.c @@ -2751,7 +2751,7 @@ int cifs_query_reparse_point(const unsigned int xid, if (cap_unix(tcon->ses)) return -EOPNOTSUPP; - if (!(le32_to_cpu(tcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS)) + if (!CIFS_REPARSE_SUPPORT(tcon)) return -EOPNOTSUPP; oparms = (struct cifs_open_parms) { @@ -2879,7 +2879,7 @@ struct inode *cifs_create_reparse_inode(struct cifs_open_info_data *data, * attempt to create reparse point. This will prevent creating unusable * empty object on the server. */ - if (!(le32_to_cpu(tcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS)) + if (!CIFS_REPARSE_SUPPORT(tcon)) return ERR_PTR(-EOPNOTSUPP); #ifndef CONFIG_CIFS_XATTR diff --git a/fs/smb/client/cifstransport.c b/fs/smb/client/cifstransport.c new file mode 100644 index 000000000000..352dafb888dd --- /dev/null +++ b/fs/smb/client/cifstransport.c @@ -0,0 +1,566 @@ +// SPDX-License-Identifier: LGPL-2.1 +/* + * + * Copyright (C) International Business Machines Corp., 2002,2008 + * Author(s): Steve French (sfrench@us.ibm.com) + * Jeremy Allison (jra@samba.org) 2006. + * + */ + +#include <linux/fs.h> +#include <linux/list.h> +#include <linux/gfp.h> +#include <linux/wait.h> +#include <linux/net.h> +#include <linux/delay.h> +#include <linux/freezer.h> +#include <linux/tcp.h> +#include <linux/bvec.h> +#include <linux/highmem.h> +#include <linux/uaccess.h> +#include <linux/processor.h> +#include <linux/mempool.h> +#include <linux/sched/signal.h> +#include <linux/task_io_accounting_ops.h> +#include "cifspdu.h" +#include "cifsglob.h" +#include "cifsproto.h" +#include "cifs_debug.h" +#include "smb2proto.h" +#include "smbdirect.h" +#include "compress.h" + +/* Max number of iovectors we can use off the stack when sending requests. */ +#define CIFS_MAX_IOV_SIZE 8 + +static struct mid_q_entry * +alloc_mid(const struct smb_hdr *smb_buffer, struct TCP_Server_Info *server) +{ + struct mid_q_entry *temp; + + if (server == NULL) { + cifs_dbg(VFS, "%s: null TCP session\n", __func__); + return NULL; + } + + temp = mempool_alloc(cifs_mid_poolp, GFP_NOFS); + memset(temp, 0, sizeof(struct mid_q_entry)); + kref_init(&temp->refcount); + temp->mid = get_mid(smb_buffer); + temp->pid = current->pid; + temp->command = cpu_to_le16(smb_buffer->Command); + cifs_dbg(FYI, "For smb_command %d\n", smb_buffer->Command); + /* easier to use jiffies */ + /* when mid allocated can be before when sent */ + temp->when_alloc = jiffies; + temp->server = server; + + /* + * The default is for the mid to be synchronous, so the + * default callback just wakes up the current task. + */ + get_task_struct(current); + temp->creator = current; + temp->callback = cifs_wake_up_task; + temp->callback_data = current; + + atomic_inc(&mid_count); + temp->mid_state = MID_REQUEST_ALLOCATED; + return temp; +} + +int +smb_send(struct TCP_Server_Info *server, struct smb_hdr *smb_buffer, + unsigned int smb_buf_length) +{ + struct kvec iov[2]; + struct smb_rqst rqst = { .rq_iov = iov, + .rq_nvec = 2 }; + + iov[0].iov_base = smb_buffer; + iov[0].iov_len = 4; + iov[1].iov_base = (char *)smb_buffer + 4; + iov[1].iov_len = smb_buf_length; + + return __smb_send_rqst(server, 1, &rqst); +} + +static int allocate_mid(struct cifs_ses *ses, struct smb_hdr *in_buf, + struct mid_q_entry **ppmidQ) +{ + spin_lock(&ses->ses_lock); + if (ses->ses_status == SES_NEW) { + if ((in_buf->Command != SMB_COM_SESSION_SETUP_ANDX) && + (in_buf->Command != SMB_COM_NEGOTIATE)) { + spin_unlock(&ses->ses_lock); + return -EAGAIN; + } + /* else ok - we are setting up session */ + } + + if (ses->ses_status == SES_EXITING) { + /* check if SMB session is bad because we are setting it up */ + if (in_buf->Command != SMB_COM_LOGOFF_ANDX) { + spin_unlock(&ses->ses_lock); + return -EAGAIN; + } + /* else ok - we are shutting down session */ + } + spin_unlock(&ses->ses_lock); + + *ppmidQ = alloc_mid(in_buf, ses->server); + if (*ppmidQ == NULL) + return -ENOMEM; + spin_lock(&ses->server->mid_queue_lock); + list_add_tail(&(*ppmidQ)->qhead, &ses->server->pending_mid_q); + spin_unlock(&ses->server->mid_queue_lock); + return 0; +} + +struct mid_q_entry * +cifs_setup_async_request(struct TCP_Server_Info *server, struct smb_rqst *rqst) +{ + int rc; + struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + if (rqst->rq_iov[0].iov_len != 4 || + rqst->rq_iov[0].iov_base + 4 != rqst->rq_iov[1].iov_base) + return ERR_PTR(-EIO); + + /* enable signing if server requires it */ + if (server->sign) + hdr->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; + + mid = alloc_mid(hdr, server); + if (mid == NULL) + return ERR_PTR(-ENOMEM); + + rc = cifs_sign_rqst(rqst, server, &mid->sequence_number); + if (rc) { + release_mid(mid); + return ERR_PTR(rc); + } + + return mid; +} + +/* + * + * Send an SMB Request. No response info (other than return code) + * needs to be parsed. + * + * flags indicate the type of request buffer and how long to wait + * and whether to log NT STATUS code (error) before mapping it to POSIX error + * + */ +int +SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, + char *in_buf, int flags) +{ + int rc; + struct kvec iov[1]; + struct kvec rsp_iov; + int resp_buf_type; + + iov[0].iov_base = in_buf; + iov[0].iov_len = get_rfc1002_length(in_buf) + 4; + flags |= CIFS_NO_RSP_BUF; + rc = SendReceive2(xid, ses, iov, 1, &resp_buf_type, flags, &rsp_iov); + cifs_dbg(NOISY, "SendRcvNoRsp flags %d rc %d\n", flags, rc); + + return rc; +} + +int +cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, + bool log_error) +{ + unsigned int len = get_rfc1002_length(mid->resp_buf) + 4; + + dump_smb(mid->resp_buf, min_t(u32, 92, len)); + + /* convert the length into a more usable form */ + if (server->sign) { + struct kvec iov[2]; + int rc = 0; + struct smb_rqst rqst = { .rq_iov = iov, + .rq_nvec = 2 }; + + iov[0].iov_base = mid->resp_buf; + iov[0].iov_len = 4; + iov[1].iov_base = (char *)mid->resp_buf + 4; + iov[1].iov_len = len - 4; + /* FIXME: add code to kill session */ + rc = cifs_verify_signature(&rqst, server, + mid->sequence_number); + if (rc) + cifs_server_dbg(VFS, "SMB signature verification returned error = %d\n", + rc); + } + + /* BB special case reconnect tid and uid here? */ + return map_and_check_smb_error(mid, log_error); +} + +struct mid_q_entry * +cifs_setup_request(struct cifs_ses *ses, struct TCP_Server_Info *ignored, + struct smb_rqst *rqst) +{ + int rc; + struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; + struct mid_q_entry *mid; + + if (rqst->rq_iov[0].iov_len != 4 || + rqst->rq_iov[0].iov_base + 4 != rqst->rq_iov[1].iov_base) + return ERR_PTR(-EIO); + + rc = allocate_mid(ses, hdr, &mid); + if (rc) + return ERR_PTR(rc); + rc = cifs_sign_rqst(rqst, ses->server, &mid->sequence_number); + if (rc) { + delete_mid(mid); + return ERR_PTR(rc); + } + return mid; +} + +int +SendReceive2(const unsigned int xid, struct cifs_ses *ses, + struct kvec *iov, int n_vec, int *resp_buf_type /* ret */, + const int flags, struct kvec *resp_iov) +{ + struct smb_rqst rqst; + struct kvec s_iov[CIFS_MAX_IOV_SIZE], *new_iov; + int rc; + + if (n_vec + 1 > CIFS_MAX_IOV_SIZE) { + new_iov = kmalloc_array(n_vec + 1, sizeof(struct kvec), + GFP_KERNEL); + if (!new_iov) { + /* otherwise cifs_send_recv below sets resp_buf_type */ + *resp_buf_type = CIFS_NO_BUFFER; + return -ENOMEM; + } + } else + new_iov = s_iov; + + /* 1st iov is a RFC1001 length followed by the rest of the packet */ + memcpy(new_iov + 1, iov, (sizeof(struct kvec) * n_vec)); + + new_iov[0].iov_base = new_iov[1].iov_base; + new_iov[0].iov_len = 4; + new_iov[1].iov_base += 4; + new_iov[1].iov_len -= 4; + + memset(&rqst, 0, sizeof(struct smb_rqst)); + rqst.rq_iov = new_iov; + rqst.rq_nvec = n_vec + 1; + + rc = cifs_send_recv(xid, ses, ses->server, + &rqst, resp_buf_type, flags, resp_iov); + if (n_vec + 1 > CIFS_MAX_IOV_SIZE) + kfree(new_iov); + return rc; +} + +int +SendReceive(const unsigned int xid, struct cifs_ses *ses, + struct smb_hdr *in_buf, struct smb_hdr *out_buf, + int *pbytes_returned, const int flags) +{ + int rc = 0; + struct mid_q_entry *midQ; + unsigned int len = be32_to_cpu(in_buf->smb_buf_length); + struct kvec iov = { .iov_base = in_buf, .iov_len = len }; + struct smb_rqst rqst = { .rq_iov = &iov, .rq_nvec = 1 }; + struct cifs_credits credits = { .value = 1, .instance = 0 }; + struct TCP_Server_Info *server; + + if (ses == NULL) { + cifs_dbg(VFS, "Null smb session\n"); + return -EIO; + } + server = ses->server; + if (server == NULL) { + cifs_dbg(VFS, "Null tcp session\n"); + return -EIO; + } + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ENOENT; + } + spin_unlock(&server->srv_lock); + + /* Ensure that we do not send more than 50 overlapping requests + to the same server. We may make this configurable later or + use ses->maxReq */ + + if (len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { + cifs_server_dbg(VFS, "Invalid length, greater than maximum frame, %d\n", + len); + return -EIO; + } + + rc = wait_for_free_request(server, flags, &credits.instance); + if (rc) + return rc; + + /* make sure that we sign in the same order that we send on this socket + and avoid races inside tcp sendmsg code that could cause corruption + of smb data */ + + cifs_server_lock(server); + + rc = allocate_mid(ses, in_buf, &midQ); + if (rc) { + cifs_server_unlock(server); + /* Update # of requests on wire to server */ + add_credits(server, &credits, 0); + return rc; + } + + rc = cifs_sign_smb(in_buf, server, &midQ->sequence_number); + if (rc) { + cifs_server_unlock(server); + goto out; + } + + midQ->mid_state = MID_REQUEST_SUBMITTED; + + rc = smb_send(server, in_buf, len); + cifs_save_when_sent(midQ); + + if (rc < 0) + server->sequence_number -= 2; + + cifs_server_unlock(server); + + if (rc < 0) + goto out; + + rc = wait_for_response(server, midQ); + if (rc != 0) { + send_cancel(server, &rqst, midQ); + spin_lock(&server->mid_queue_lock); + if (midQ->mid_state == MID_REQUEST_SUBMITTED || + midQ->mid_state == MID_RESPONSE_RECEIVED) { + /* no longer considered to be "in-flight" */ + midQ->callback = release_mid; + spin_unlock(&server->mid_queue_lock); + add_credits(server, &credits, 0); + return rc; + } + spin_unlock(&server->mid_queue_lock); + } + + rc = cifs_sync_mid_result(midQ, server); + if (rc != 0) { + add_credits(server, &credits, 0); + return rc; + } + + if (!midQ->resp_buf || !out_buf || + midQ->mid_state != MID_RESPONSE_READY) { + rc = -EIO; + cifs_server_dbg(VFS, "Bad MID state?\n"); + goto out; + } + + *pbytes_returned = get_rfc1002_length(midQ->resp_buf); + memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); + rc = cifs_check_receive(midQ, server, 0); +out: + delete_mid(midQ); + add_credits(server, &credits, 0); + + return rc; +} + +/* We send a LOCKINGX_CANCEL_LOCK to cause the Windows + blocking lock to return. */ + +static int +send_lock_cancel(const unsigned int xid, struct cifs_tcon *tcon, + struct smb_hdr *in_buf, + struct smb_hdr *out_buf) +{ + int bytes_returned; + struct cifs_ses *ses = tcon->ses; + LOCK_REQ *pSMB = (LOCK_REQ *)in_buf; + + /* We just modify the current in_buf to change + the type of lock from LOCKING_ANDX_SHARED_LOCK + or LOCKING_ANDX_EXCLUSIVE_LOCK to + LOCKING_ANDX_CANCEL_LOCK. */ + + pSMB->LockType = LOCKING_ANDX_CANCEL_LOCK|LOCKING_ANDX_LARGE_FILES; + pSMB->Timeout = 0; + pSMB->hdr.Mid = get_next_mid(ses->server); + + return SendReceive(xid, ses, in_buf, out_buf, + &bytes_returned, 0); +} + +int +SendReceiveBlockingLock(const unsigned int xid, struct cifs_tcon *tcon, + struct smb_hdr *in_buf, struct smb_hdr *out_buf, + int *pbytes_returned) +{ + int rc = 0; + int rstart = 0; + struct mid_q_entry *midQ; + struct cifs_ses *ses; + unsigned int len = be32_to_cpu(in_buf->smb_buf_length); + struct kvec iov = { .iov_base = in_buf, .iov_len = len }; + struct smb_rqst rqst = { .rq_iov = &iov, .rq_nvec = 1 }; + unsigned int instance; + struct TCP_Server_Info *server; + + if (tcon == NULL || tcon->ses == NULL) { + cifs_dbg(VFS, "Null smb session\n"); + return -EIO; + } + ses = tcon->ses; + server = ses->server; + + if (server == NULL) { + cifs_dbg(VFS, "Null tcp session\n"); + return -EIO; + } + + spin_lock(&server->srv_lock); + if (server->tcpStatus == CifsExiting) { + spin_unlock(&server->srv_lock); + return -ENOENT; + } + spin_unlock(&server->srv_lock); + + /* Ensure that we do not send more than 50 overlapping requests + to the same server. We may make this configurable later or + use ses->maxReq */ + + if (len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { + cifs_tcon_dbg(VFS, "Invalid length, greater than maximum frame, %d\n", + len); + return -EIO; + } + + rc = wait_for_free_request(server, CIFS_BLOCKING_OP, &instance); + if (rc) + return rc; + + /* make sure that we sign in the same order that we send on this socket + and avoid races inside tcp sendmsg code that could cause corruption + of smb data */ + + cifs_server_lock(server); + + rc = allocate_mid(ses, in_buf, &midQ); + if (rc) { + cifs_server_unlock(server); + return rc; + } + + rc = cifs_sign_smb(in_buf, server, &midQ->sequence_number); + if (rc) { + delete_mid(midQ); + cifs_server_unlock(server); + return rc; + } + + midQ->mid_state = MID_REQUEST_SUBMITTED; + rc = smb_send(server, in_buf, len); + cifs_save_when_sent(midQ); + + if (rc < 0) + server->sequence_number -= 2; + + cifs_server_unlock(server); + + if (rc < 0) { + delete_mid(midQ); + return rc; + } + + /* Wait for a reply - allow signals to interrupt. */ + rc = wait_event_interruptible(server->response_q, + (!(midQ->mid_state == MID_REQUEST_SUBMITTED || + midQ->mid_state == MID_RESPONSE_RECEIVED)) || + ((server->tcpStatus != CifsGood) && + (server->tcpStatus != CifsNew))); + + /* Were we interrupted by a signal ? */ + spin_lock(&server->srv_lock); + if ((rc == -ERESTARTSYS) && + (midQ->mid_state == MID_REQUEST_SUBMITTED || + midQ->mid_state == MID_RESPONSE_RECEIVED) && + ((server->tcpStatus == CifsGood) || + (server->tcpStatus == CifsNew))) { + spin_unlock(&server->srv_lock); + + if (in_buf->Command == SMB_COM_TRANSACTION2) { + /* POSIX lock. We send a NT_CANCEL SMB to cause the + blocking lock to return. */ + rc = send_cancel(server, &rqst, midQ); + if (rc) { + delete_mid(midQ); + return rc; + } + } else { + /* Windows lock. We send a LOCKINGX_CANCEL_LOCK + to cause the blocking lock to return. */ + + rc = send_lock_cancel(xid, tcon, in_buf, out_buf); + + /* If we get -ENOLCK back the lock may have + already been removed. Don't exit in this case. */ + if (rc && rc != -ENOLCK) { + delete_mid(midQ); + return rc; + } + } + + rc = wait_for_response(server, midQ); + if (rc) { + send_cancel(server, &rqst, midQ); + spin_lock(&server->mid_queue_lock); + if (midQ->mid_state == MID_REQUEST_SUBMITTED || + midQ->mid_state == MID_RESPONSE_RECEIVED) { + /* no longer considered to be "in-flight" */ + midQ->callback = release_mid; + spin_unlock(&server->mid_queue_lock); + return rc; + } + spin_unlock(&server->mid_queue_lock); + } + + /* We got the response - restart system call. */ + rstart = 1; + spin_lock(&server->srv_lock); + } + spin_unlock(&server->srv_lock); + + rc = cifs_sync_mid_result(midQ, server); + if (rc != 0) + return rc; + + /* rcvd frame is ok */ + if (out_buf == NULL || midQ->mid_state != MID_RESPONSE_READY) { + rc = -EIO; + cifs_tcon_dbg(VFS, "Bad MID state?\n"); + goto out; + } + + *pbytes_returned = get_rfc1002_length(midQ->resp_buf); + memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); + rc = cifs_check_receive(midQ, server, 0); +out: + delete_mid(midQ); + if (rstart && rc == -EACCES) + return -ERESTARTSYS; + return rc; +} diff --git a/fs/smb/client/connect.c b/fs/smb/client/connect.c index 5eec8957f2a9..587845a2452d 100644 --- a/fs/smb/client/connect.c +++ b/fs/smb/client/connect.c @@ -321,15 +321,15 @@ cifs_abort_connection(struct TCP_Server_Info *server) /* mark submitted MIDs for retry and issue callback */ INIT_LIST_HEAD(&retry_list); cifs_dbg(FYI, "%s: moving mids to private list\n", __func__); - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_entry_safe(mid, nmid, &server->pending_mid_q, qhead) { kref_get(&mid->refcount); if (mid->mid_state == MID_REQUEST_SUBMITTED) mid->mid_state = MID_RETRY_NEEDED; list_move(&mid->qhead, &retry_list); - mid->mid_flags |= MID_DELETED; + mid->deleted_from_q = true; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); cifs_server_unlock(server); cifs_dbg(FYI, "%s: issuing mid callbacks\n", __func__); @@ -358,7 +358,7 @@ static bool cifs_tcp_ses_needs_reconnect(struct TCP_Server_Info *server, int num } cifs_dbg(FYI, "Mark tcp session as need reconnect\n"); - trace_smb3_reconnect(server->CurrentMid, server->conn_id, + trace_smb3_reconnect(server->current_mid, server->conn_id, server->hostname); server->tcpStatus = CifsNeedReconnect; @@ -884,13 +884,13 @@ is_smb_response(struct TCP_Server_Info *server, unsigned char type) * server there should be exactly one pending mid * corresponding to SMB1/SMB2 Negotiate packet. */ - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_entry_safe(mid, nmid, &server->pending_mid_q, qhead) { kref_get(&mid->refcount); list_move(&mid->qhead, &dispose_list); - mid->mid_flags |= MID_DELETED; + mid->deleted_from_q = true; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); /* Now try to reconnect once with NetBIOS session. */ server->with_rfc1001 = true; @@ -957,7 +957,7 @@ dequeue_mid(struct mid_q_entry *mid, bool malformed) #ifdef CONFIG_CIFS_STATS2 mid->when_received = jiffies; #endif - spin_lock(&mid->server->mid_lock); + spin_lock(&mid->server->mid_queue_lock); if (!malformed) mid->mid_state = MID_RESPONSE_RECEIVED; else @@ -966,13 +966,13 @@ dequeue_mid(struct mid_q_entry *mid, bool malformed) * Trying to handle/dequeue a mid after the send_recv() * function has finished processing it is a bug. */ - if (mid->mid_flags & MID_DELETED) { - spin_unlock(&mid->server->mid_lock); + if (mid->deleted_from_q == true) { + spin_unlock(&mid->server->mid_queue_lock); pr_warn_once("trying to dequeue a deleted mid\n"); } else { list_del_init(&mid->qhead); - mid->mid_flags |= MID_DELETED; - spin_unlock(&mid->server->mid_lock); + mid->deleted_from_q = true; + spin_unlock(&mid->server->mid_queue_lock); } } @@ -1101,16 +1101,16 @@ clean_demultiplex_info(struct TCP_Server_Info *server) struct list_head *tmp, *tmp2; LIST_HEAD(dispose_list); - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_safe(tmp, tmp2, &server->pending_mid_q) { mid_entry = list_entry(tmp, struct mid_q_entry, qhead); cifs_dbg(FYI, "Clearing mid %llu\n", mid_entry->mid); kref_get(&mid_entry->refcount); mid_entry->mid_state = MID_SHUTDOWN; list_move(&mid_entry->qhead, &dispose_list); - mid_entry->mid_flags |= MID_DELETED; + mid_entry->deleted_from_q = true; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); /* now walk dispose list and issue callbacks */ list_for_each_safe(tmp, tmp2, &dispose_list) { @@ -1242,7 +1242,7 @@ smb2_add_credits_from_hdr(char *buffer, struct TCP_Server_Info *server) spin_unlock(&server->req_lock); wake_up(&server->request_q); - trace_smb3_hdr_credits(server->CurrentMid, + trace_smb3_hdr_credits(server->current_mid, server->conn_id, server->hostname, scredits, le16_to_cpu(shdr->CreditRequest), in_flight); cifs_server_dbg(FYI, "%s: added %u credits total=%d\n", @@ -1822,7 +1822,8 @@ cifs_get_tcp_session(struct smb3_fs_context *ctx, tcp_ses->compression.requested = ctx->compress; spin_lock_init(&tcp_ses->req_lock); spin_lock_init(&tcp_ses->srv_lock); - spin_lock_init(&tcp_ses->mid_lock); + spin_lock_init(&tcp_ses->mid_queue_lock); + spin_lock_init(&tcp_ses->mid_counter_lock); INIT_LIST_HEAD(&tcp_ses->tcp_ses_list); INIT_LIST_HEAD(&tcp_ses->smb_ses_list); INIT_DELAYED_WORK(&tcp_ses->echo, cifs_echo_request); diff --git a/fs/smb/client/fs_context.c b/fs/smb/client/fs_context.c index 3f34bb07997b..072383899e81 100644 --- a/fs/smb/client/fs_context.c +++ b/fs/smb/client/fs_context.c @@ -1652,6 +1652,7 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, pr_warn_once("conflicting posix mount options specified\n"); ctx->linux_ext = 1; ctx->no_linux_ext = 0; + ctx->nonativesocket = 1; /* POSIX mounts use NFS style reparse points */ } break; case Opt_nocase: @@ -1829,24 +1830,6 @@ static int smb3_fs_context_parse_param(struct fs_context *fc, return -EINVAL; } -enum cifs_symlink_type get_cifs_symlink_type(struct cifs_sb_info *cifs_sb) -{ - if (cifs_sb->ctx->symlink_type == CIFS_SYMLINK_TYPE_DEFAULT) { - if (cifs_sb->ctx->mfsymlinks) - return CIFS_SYMLINK_TYPE_MFSYMLINKS; - else if (cifs_sb->ctx->sfu_emul) - return CIFS_SYMLINK_TYPE_SFU; - else if (cifs_sb->ctx->linux_ext && !cifs_sb->ctx->no_linux_ext) - return CIFS_SYMLINK_TYPE_UNIX; - else if (cifs_sb->ctx->reparse_type != CIFS_REPARSE_TYPE_NONE) - return CIFS_SYMLINK_TYPE_NATIVE; - else - return CIFS_SYMLINK_TYPE_NONE; - } else { - return cifs_sb->ctx->symlink_type; - } -} - int smb3_init_fs_context(struct fs_context *fc) { struct smb3_fs_context *ctx; diff --git a/fs/smb/client/fs_context.h b/fs/smb/client/fs_context.h index 9e83302ce4b8..b0fec6b9a23b 100644 --- a/fs/smb/client/fs_context.h +++ b/fs/smb/client/fs_context.h @@ -341,7 +341,23 @@ struct smb3_fs_context { extern const struct fs_parameter_spec smb3_fs_parameters[]; -extern enum cifs_symlink_type get_cifs_symlink_type(struct cifs_sb_info *cifs_sb); +static inline enum cifs_symlink_type cifs_symlink_type(struct cifs_sb_info *cifs_sb) +{ + bool posix = cifs_sb_master_tcon(cifs_sb)->posix_extensions; + + if (cifs_sb->ctx->symlink_type != CIFS_SYMLINK_TYPE_DEFAULT) + return cifs_sb->ctx->symlink_type; + + if (cifs_sb->ctx->mfsymlinks) + return CIFS_SYMLINK_TYPE_MFSYMLINKS; + else if (cifs_sb->ctx->sfu_emul) + return CIFS_SYMLINK_TYPE_SFU; + else if (cifs_sb->ctx->linux_ext && !cifs_sb->ctx->no_linux_ext) + return posix ? CIFS_SYMLINK_TYPE_NATIVE : CIFS_SYMLINK_TYPE_UNIX; + else if (cifs_sb->ctx->reparse_type != CIFS_REPARSE_TYPE_NONE) + return CIFS_SYMLINK_TYPE_NATIVE; + return CIFS_SYMLINK_TYPE_NONE; +} extern int smb3_init_fs_context(struct fs_context *fc); extern void smb3_cleanup_fs_context_contents(struct smb3_fs_context *ctx); diff --git a/fs/smb/client/link.c b/fs/smb/client/link.c index 2ecd705e9e8c..fe80e711cd75 100644 --- a/fs/smb/client/link.c +++ b/fs/smb/client/link.c @@ -605,14 +605,7 @@ cifs_symlink(struct mnt_idmap *idmap, struct inode *inode, /* BB what if DFS and this volume is on different share? BB */ rc = -EOPNOTSUPP; - switch (get_cifs_symlink_type(cifs_sb)) { - case CIFS_SYMLINK_TYPE_DEFAULT: - /* should not happen, get_cifs_symlink_type() resolves the default */ - break; - - case CIFS_SYMLINK_TYPE_NONE: - break; - + switch (cifs_symlink_type(cifs_sb)) { case CIFS_SYMLINK_TYPE_UNIX: #ifdef CONFIG_CIFS_ALLOW_INSECURE_LEGACY if (pTcon->unix_ext) { @@ -642,12 +635,14 @@ cifs_symlink(struct mnt_idmap *idmap, struct inode *inode, case CIFS_SYMLINK_TYPE_NATIVE: case CIFS_SYMLINK_TYPE_NFS: case CIFS_SYMLINK_TYPE_WSL: - if (le32_to_cpu(pTcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS) { + if (CIFS_REPARSE_SUPPORT(pTcon)) { rc = create_reparse_symlink(xid, inode, direntry, pTcon, full_path, symname); goto symlink_exit; } break; + default: + break; } if (rc == 0) { diff --git a/fs/smb/client/reparse.c b/fs/smb/client/reparse.c index 33c1d970747c..7869cec58f52 100644 --- a/fs/smb/client/reparse.c +++ b/fs/smb/client/reparse.c @@ -38,7 +38,7 @@ int create_reparse_symlink(const unsigned int xid, struct inode *inode, struct dentry *dentry, struct cifs_tcon *tcon, const char *full_path, const char *symname) { - switch (get_cifs_symlink_type(CIFS_SB(inode->i_sb))) { + switch (cifs_symlink_type(CIFS_SB(inode->i_sb))) { case CIFS_SYMLINK_TYPE_NATIVE: return create_native_symlink(xid, inode, dentry, tcon, full_path, symname); case CIFS_SYMLINK_TYPE_NFS: diff --git a/fs/smb/client/smb1ops.c b/fs/smb/client/smb1ops.c index e364b6515af3..893a1ea8c000 100644 --- a/fs/smb/client/smb1ops.c +++ b/fs/smb/client/smb1ops.c @@ -95,17 +95,17 @@ cifs_find_mid(struct TCP_Server_Info *server, char *buffer) struct smb_hdr *buf = (struct smb_hdr *)buffer; struct mid_q_entry *mid; - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_entry(mid, &server->pending_mid_q, qhead) { if (compare_mid(mid->mid, buf) && mid->mid_state == MID_REQUEST_SUBMITTED && le16_to_cpu(mid->command) == buf->Command) { kref_get(&mid->refcount); - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return mid; } } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return NULL; } @@ -169,10 +169,9 @@ cifs_get_next_mid(struct TCP_Server_Info *server) __u16 last_mid, cur_mid; bool collision, reconnect = false; - spin_lock(&server->mid_lock); - + spin_lock(&server->mid_counter_lock); /* mid is 16 bit only for CIFS/SMB */ - cur_mid = (__u16)((server->CurrentMid) & 0xffff); + cur_mid = (__u16)((server->current_mid) & 0xffff); /* we do not want to loop forever */ last_mid = cur_mid; cur_mid++; @@ -198,6 +197,7 @@ cifs_get_next_mid(struct TCP_Server_Info *server) cur_mid++; num_mids = 0; + spin_lock(&server->mid_queue_lock); list_for_each_entry(mid_entry, &server->pending_mid_q, qhead) { ++num_mids; if (mid_entry->mid == cur_mid && @@ -207,6 +207,7 @@ cifs_get_next_mid(struct TCP_Server_Info *server) break; } } + spin_unlock(&server->mid_queue_lock); /* * if we have more than 32k mids in the list, then something @@ -223,12 +224,12 @@ cifs_get_next_mid(struct TCP_Server_Info *server) if (!collision) { mid = (__u64)cur_mid; - server->CurrentMid = mid; + server->current_mid = mid; break; } cur_mid++; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_counter_lock); if (reconnect) { cifs_signal_cifsd_for_reconnect(server, false); @@ -1272,7 +1273,7 @@ cifs_make_node(unsigned int xid, struct inode *inode, */ return cifs_sfu_make_node(xid, inode, dentry, tcon, full_path, mode, dev); - } else if (le32_to_cpu(tcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS) { + } else if (CIFS_REPARSE_SUPPORT(tcon)) { /* * mknod via reparse points requires server support for * storing reparse points, which is available since diff --git a/fs/smb/client/smb2inode.c b/fs/smb/client/smb2inode.c index 69d251726c02..2a0316c514e4 100644 --- a/fs/smb/client/smb2inode.c +++ b/fs/smb/client/smb2inode.c @@ -1346,9 +1346,8 @@ struct inode *smb2_create_reparse_inode(struct cifs_open_info_data *data, * attempt to create reparse point. This will prevent creating unusable * empty object on the server. */ - if (!(le32_to_cpu(tcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS)) - if (!tcon->posix_extensions) - return ERR_PTR(-EOPNOTSUPP); + if (!CIFS_REPARSE_SUPPORT(tcon)) + return ERR_PTR(-EOPNOTSUPP); oparms = CIFS_OPARMS(cifs_sb, tcon, full_path, SYNCHRONIZE | DELETE | diff --git a/fs/smb/client/smb2ops.c b/fs/smb/client/smb2ops.c index 1b4a31894f43..ad8947434b71 100644 --- a/fs/smb/client/smb2ops.c +++ b/fs/smb/client/smb2ops.c @@ -91,7 +91,7 @@ smb2_add_credits(struct TCP_Server_Info *server, if (*val > 65000) { *val = 65000; /* Don't get near 64K credits, avoid srv bugs */ pr_warn_once("server overflowed SMB3 credits\n"); - trace_smb3_overflow_credits(server->CurrentMid, + trace_smb3_overflow_credits(server->current_mid, server->conn_id, server->hostname, *val, add, server->in_flight); } @@ -136,7 +136,7 @@ smb2_add_credits(struct TCP_Server_Info *server, wake_up(&server->request_q); if (reconnect_detected) { - trace_smb3_reconnect_detected(server->CurrentMid, + trace_smb3_reconnect_detected(server->current_mid, server->conn_id, server->hostname, scredits, add, in_flight); cifs_dbg(FYI, "trying to put %d credits from the old server instance %d\n", @@ -144,7 +144,7 @@ smb2_add_credits(struct TCP_Server_Info *server, } if (reconnect_with_invalid_credits) { - trace_smb3_reconnect_with_invalid_credits(server->CurrentMid, + trace_smb3_reconnect_with_invalid_credits(server->current_mid, server->conn_id, server->hostname, scredits, add, in_flight); cifs_dbg(FYI, "Negotiate operation when server credits is non-zero. Optype: %d, server credits: %d, credits added: %d\n", optype, scredits, add); @@ -176,7 +176,7 @@ smb2_add_credits(struct TCP_Server_Info *server, break; } - trace_smb3_add_credits(server->CurrentMid, + trace_smb3_add_credits(server->current_mid, server->conn_id, server->hostname, scredits, add, in_flight); cifs_dbg(FYI, "%s: added %u credits total=%d\n", __func__, add, scredits); } @@ -203,7 +203,7 @@ smb2_set_credits(struct TCP_Server_Info *server, const int val) in_flight = server->in_flight; spin_unlock(&server->req_lock); - trace_smb3_set_credits(server->CurrentMid, + trace_smb3_set_credits(server->current_mid, server->conn_id, server->hostname, scredits, val, in_flight); cifs_dbg(FYI, "%s: set %u credits\n", __func__, val); @@ -288,7 +288,7 @@ smb2_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, in_flight = server->in_flight; spin_unlock(&server->req_lock); - trace_smb3_wait_credits(server->CurrentMid, + trace_smb3_wait_credits(server->current_mid, server->conn_id, server->hostname, scredits, -(credits->value), in_flight); cifs_dbg(FYI, "%s: removed %u credits total=%d\n", __func__, credits->value, scredits); @@ -316,7 +316,7 @@ smb2_adjust_credits(struct TCP_Server_Info *server, server->credits, server->in_flight, new_val - credits->value, cifs_trace_rw_credits_no_adjust_up); - trace_smb3_too_many_credits(server->CurrentMid, + trace_smb3_too_many_credits(server->current_mid, server->conn_id, server->hostname, 0, credits->value - new_val, 0); cifs_server_dbg(VFS, "R=%x[%x] request has less credits (%d) than required (%d)", subreq->rreq->debug_id, subreq->subreq.debug_index, @@ -338,7 +338,7 @@ smb2_adjust_credits(struct TCP_Server_Info *server, server->credits, server->in_flight, new_val - credits->value, cifs_trace_rw_credits_old_session); - trace_smb3_reconnect_detected(server->CurrentMid, + trace_smb3_reconnect_detected(server->current_mid, server->conn_id, server->hostname, scredits, credits->value - new_val, in_flight); cifs_server_dbg(VFS, "R=%x[%x] trying to return %d credits to old session\n", @@ -358,7 +358,7 @@ smb2_adjust_credits(struct TCP_Server_Info *server, spin_unlock(&server->req_lock); wake_up(&server->request_q); - trace_smb3_adj_credits(server->CurrentMid, + trace_smb3_adj_credits(server->current_mid, server->conn_id, server->hostname, scredits, credits->value - new_val, in_flight); cifs_dbg(FYI, "%s: adjust added %u credits total=%d\n", @@ -374,19 +374,19 @@ smb2_get_next_mid(struct TCP_Server_Info *server) { __u64 mid; /* for SMB2 we need the current value */ - spin_lock(&server->mid_lock); - mid = server->CurrentMid++; - spin_unlock(&server->mid_lock); + spin_lock(&server->mid_counter_lock); + mid = server->current_mid++; + spin_unlock(&server->mid_counter_lock); return mid; } static void smb2_revert_current_mid(struct TCP_Server_Info *server, const unsigned int val) { - spin_lock(&server->mid_lock); - if (server->CurrentMid >= val) - server->CurrentMid -= val; - spin_unlock(&server->mid_lock); + spin_lock(&server->mid_counter_lock); + if (server->current_mid >= val) + server->current_mid -= val; + spin_unlock(&server->mid_counter_lock); } static struct mid_q_entry * @@ -401,7 +401,7 @@ __smb2_find_mid(struct TCP_Server_Info *server, char *buf, bool dequeue) return NULL; } - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_for_each_entry(mid, &server->pending_mid_q, qhead) { if ((mid->mid == wire_mid) && (mid->mid_state == MID_REQUEST_SUBMITTED) && @@ -409,13 +409,13 @@ __smb2_find_mid(struct TCP_Server_Info *server, char *buf, bool dequeue) kref_get(&mid->refcount); if (dequeue) { list_del_init(&mid->qhead); - mid->mid_flags |= MID_DELETED; + mid->deleted_from_q = true; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return mid; } } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return NULL; } @@ -460,9 +460,9 @@ smb2_negotiate(const unsigned int xid, { int rc; - spin_lock(&server->mid_lock); - server->CurrentMid = 0; - spin_unlock(&server->mid_lock); + spin_lock(&server->mid_counter_lock); + server->current_mid = 0; + spin_unlock(&server->mid_counter_lock); rc = SMB2_negotiate(xid, ses, server); return rc; } @@ -2498,7 +2498,7 @@ smb2_is_status_pending(char *buf, struct TCP_Server_Info *server) spin_unlock(&server->req_lock); wake_up(&server->request_q); - trace_smb3_pend_credits(server->CurrentMid, + trace_smb3_pend_credits(server->current_mid, server->conn_id, server->hostname, scredits, le16_to_cpu(shdr->CreditRequest), in_flight); cifs_dbg(FYI, "%s: status pending add %u credits total=%d\n", @@ -4809,18 +4809,18 @@ static void smb2_decrypt_offload(struct work_struct *work) } else { spin_lock(&dw->server->srv_lock); if (dw->server->tcpStatus == CifsNeedReconnect) { - spin_lock(&dw->server->mid_lock); + spin_lock(&dw->server->mid_queue_lock); mid->mid_state = MID_RETRY_NEEDED; - spin_unlock(&dw->server->mid_lock); + spin_unlock(&dw->server->mid_queue_lock); spin_unlock(&dw->server->srv_lock); mid->callback(mid); } else { - spin_lock(&dw->server->mid_lock); + spin_lock(&dw->server->mid_queue_lock); mid->mid_state = MID_REQUEST_SUBMITTED; - mid->mid_flags &= ~(MID_DELETED); + mid->deleted_from_q = false; list_add_tail(&mid->qhead, &dw->server->pending_mid_q); - spin_unlock(&dw->server->mid_lock); + spin_unlock(&dw->server->mid_queue_lock); spin_unlock(&dw->server->srv_lock); } } @@ -5260,10 +5260,9 @@ static int smb2_make_node(unsigned int xid, struct inode *inode, if (cifs_sb->mnt_cifs_flags & CIFS_MOUNT_UNX_EMUL) { rc = cifs_sfu_make_node(xid, inode, dentry, tcon, full_path, mode, dev); - } else if ((le32_to_cpu(tcon->fsAttrInfo.Attributes) & FILE_SUPPORTS_REPARSE_POINTS) - || (tcon->posix_extensions)) { + } else if (CIFS_REPARSE_SUPPORT(tcon)) { rc = mknod_reparse(xid, inode, dentry, tcon, - full_path, mode, dev); + full_path, mode, dev); } return rc; } diff --git a/fs/smb/client/smb2transport.c b/fs/smb/client/smb2transport.c index 475b36c27f65..ff9ef7fcd010 100644 --- a/fs/smb/client/smb2transport.c +++ b/fs/smb/client/smb2transport.c @@ -840,9 +840,9 @@ smb2_get_mid_entry(struct cifs_ses *ses, struct TCP_Server_Info *server, *mid = smb2_mid_entry_alloc(shdr, server); if (*mid == NULL) return -ENOMEM; - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_add_tail(&(*mid)->qhead, &server->pending_mid_q); - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return 0; } diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c index 754e94a0e07f..c628e91c328b 100644 --- a/fs/smb/client/smbdirect.c +++ b/fs/smb/client/smbdirect.c @@ -13,27 +13,23 @@ #include "cifsproto.h" #include "smb2proto.h" -static struct smbd_response *get_empty_queue_buffer( - struct smbd_connection *info); -static struct smbd_response *get_receive_buffer( +static struct smbdirect_recv_io *get_receive_buffer( struct smbd_connection *info); static void put_receive_buffer( struct smbd_connection *info, - struct smbd_response *response); + struct smbdirect_recv_io *response); static int allocate_receive_buffers(struct smbd_connection *info, int num_buf); static void destroy_receive_buffers(struct smbd_connection *info); -static void put_empty_packet( - struct smbd_connection *info, struct smbd_response *response); static void enqueue_reassembly( struct smbd_connection *info, - struct smbd_response *response, int data_length); -static struct smbd_response *_get_first_reassembly( + struct smbdirect_recv_io *response, int data_length); +static struct smbdirect_recv_io *_get_first_reassembly( struct smbd_connection *info); static int smbd_post_recv( struct smbd_connection *info, - struct smbd_response *response); + struct smbdirect_recv_io *response); static int smbd_post_send_empty(struct smbd_connection *info); @@ -182,9 +178,10 @@ static int smbd_conn_upcall( { struct smbd_connection *info = id->context; struct smbdirect_socket *sc = &info->socket; + const char *event_name = rdma_event_msg(event->event); - log_rdma_event(INFO, "event=%d status=%d\n", - event->event, event->status); + log_rdma_event(INFO, "event=%s status=%d\n", + event_name, event->status); switch (event->event) { case RDMA_CM_EVENT_ADDR_RESOLVED: @@ -194,45 +191,50 @@ static int smbd_conn_upcall( break; case RDMA_CM_EVENT_ADDR_ERROR: + log_rdma_event(ERR, "connecting failed event=%s\n", event_name); info->ri_rc = -EHOSTUNREACH; complete(&info->ri_done); break; case RDMA_CM_EVENT_ROUTE_ERROR: + log_rdma_event(ERR, "connecting failed event=%s\n", event_name); info->ri_rc = -ENETUNREACH; complete(&info->ri_done); break; case RDMA_CM_EVENT_ESTABLISHED: - log_rdma_event(INFO, "connected event=%d\n", event->event); + log_rdma_event(INFO, "connected event=%s\n", event_name); sc->status = SMBDIRECT_SOCKET_CONNECTED; - wake_up_interruptible(&info->conn_wait); + wake_up_interruptible(&info->status_wait); break; case RDMA_CM_EVENT_CONNECT_ERROR: case RDMA_CM_EVENT_UNREACHABLE: case RDMA_CM_EVENT_REJECTED: - log_rdma_event(INFO, "connecting failed event=%d\n", event->event); + log_rdma_event(ERR, "connecting failed event=%s\n", event_name); sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - wake_up_interruptible(&info->conn_wait); + wake_up_interruptible(&info->status_wait); break; case RDMA_CM_EVENT_DEVICE_REMOVAL: case RDMA_CM_EVENT_DISCONNECTED: /* This happens when we fail the negotiation */ if (sc->status == SMBDIRECT_SOCKET_NEGOTIATE_FAILED) { + log_rdma_event(ERR, "event=%s during negotiation\n", event_name); sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - wake_up(&info->conn_wait); + wake_up(&info->status_wait); break; } sc->status = SMBDIRECT_SOCKET_DISCONNECTED; - wake_up_interruptible(&info->disconn_wait); - wake_up_interruptible(&info->wait_reassembly_queue); + wake_up_interruptible(&info->status_wait); + wake_up_interruptible(&sc->recv_io.reassembly.wait_queue); wake_up_interruptible_all(&info->wait_send_queue); break; default: + log_rdma_event(ERR, "unexpected event=%s status=%d\n", + event_name, event->status); break; } @@ -259,12 +261,12 @@ smbd_qp_async_error_upcall(struct ib_event *event, void *context) } } -static inline void *smbd_request_payload(struct smbd_request *request) +static inline void *smbdirect_send_io_payload(struct smbdirect_send_io *request) { return (void *)request->packet; } -static inline void *smbd_response_payload(struct smbd_response *response) +static inline void *smbdirect_recv_io_payload(struct smbdirect_recv_io *response) { return (void *)response->packet; } @@ -273,32 +275,35 @@ static inline void *smbd_response_payload(struct smbd_response *response) static void send_done(struct ib_cq *cq, struct ib_wc *wc) { int i; - struct smbd_request *request = - container_of(wc->wr_cqe, struct smbd_request, cqe); - struct smbd_connection *info = request->info; - struct smbdirect_socket *sc = &info->socket; + struct smbdirect_send_io *request = + container_of(wc->wr_cqe, struct smbdirect_send_io, cqe); + struct smbdirect_socket *sc = request->socket; + struct smbd_connection *info = + container_of(sc, struct smbd_connection, socket); - log_rdma_send(INFO, "smbd_request 0x%p completed wc->status=%d\n", + log_rdma_send(INFO, "smbdirect_send_io 0x%p completed wc->status=%d\n", request, wc->status); - if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { - log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", - wc->status, wc->opcode); - smbd_disconnect_rdma_connection(request->info); - } - for (i = 0; i < request->num_sge; i++) ib_dma_unmap_single(sc->ib.dev, request->sge[i].addr, request->sge[i].length, DMA_TO_DEVICE); - if (atomic_dec_and_test(&request->info->send_pending)) - wake_up(&request->info->wait_send_pending); + if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_SEND) { + log_rdma_send(ERR, "wc->status=%d wc->opcode=%d\n", + wc->status, wc->opcode); + mempool_free(request, sc->send_io.mem.pool); + smbd_disconnect_rdma_connection(info); + return; + } - wake_up(&request->info->wait_post_send); + if (atomic_dec_and_test(&info->send_pending)) + wake_up(&info->wait_send_pending); + + wake_up(&info->wait_post_send); - mempool_free(request, request->info->request_mempool); + mempool_free(request, sc->send_io.mem.pool); } static void dump_smbdirect_negotiate_resp(struct smbdirect_negotiate_resp *resp) @@ -317,12 +322,13 @@ static void dump_smbdirect_negotiate_resp(struct smbdirect_negotiate_resp *resp) * return value: true if negotiation is a success, false if failed */ static bool process_negotiation_response( - struct smbd_response *response, int packet_length) + struct smbdirect_recv_io *response, int packet_length) { - struct smbd_connection *info = response->info; - struct smbdirect_socket *sc = &info->socket; + struct smbdirect_socket *sc = response->socket; + struct smbd_connection *info = + container_of(sc, struct smbd_connection, socket); struct smbdirect_socket_parameters *sp = &sc->parameters; - struct smbdirect_negotiate_resp *packet = smbd_response_payload(response); + struct smbdirect_negotiate_resp *packet = smbdirect_recv_io_payload(response); if (packet_length < sizeof(struct smbdirect_negotiate_resp)) { log_rdma_event(ERR, @@ -385,15 +391,15 @@ static bool process_negotiation_response( info->max_frmr_depth * PAGE_SIZE); info->max_frmr_depth = sp->max_read_write_size / PAGE_SIZE; + sc->recv_io.expected = SMBDIRECT_EXPECT_DATA_TRANSFER; return true; } static void smbd_post_send_credits(struct work_struct *work) { int ret = 0; - int use_receive_queue = 1; int rc; - struct smbd_response *response; + struct smbdirect_recv_io *response; struct smbd_connection *info = container_of(work, struct smbd_connection, post_send_credits_work); @@ -407,20 +413,10 @@ static void smbd_post_send_credits(struct work_struct *work) if (info->receive_credit_target > atomic_read(&info->receive_credits)) { while (true) { - if (use_receive_queue) - response = get_receive_buffer(info); - else - response = get_empty_queue_buffer(info); - if (!response) { - /* now switch to empty packet queue */ - if (use_receive_queue) { - use_receive_queue = 0; - continue; - } else - break; - } + response = get_receive_buffer(info); + if (!response) + break; - response->type = SMBD_TRANSFER_DATA; response->first_segment = false; rc = smbd_post_recv(info, response); if (rc) { @@ -454,19 +450,20 @@ static void smbd_post_send_credits(struct work_struct *work) static void recv_done(struct ib_cq *cq, struct ib_wc *wc) { struct smbdirect_data_transfer *data_transfer; - struct smbd_response *response = - container_of(wc->wr_cqe, struct smbd_response, cqe); - struct smbd_connection *info = response->info; + struct smbdirect_recv_io *response = + container_of(wc->wr_cqe, struct smbdirect_recv_io, cqe); + struct smbdirect_socket *sc = response->socket; + struct smbd_connection *info = + container_of(sc, struct smbd_connection, socket); int data_length = 0; log_rdma_recv(INFO, "response=0x%p type=%d wc status=%d wc opcode %d byte_len=%d pkey_index=%u\n", - response, response->type, wc->status, wc->opcode, + response, sc->recv_io.expected, wc->status, wc->opcode, wc->byte_len, wc->pkey_index); if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { log_rdma_recv(INFO, "wc->status=%d opcode=%d\n", wc->status, wc->opcode); - smbd_disconnect_rdma_connection(info); goto error; } @@ -476,43 +473,31 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) response->sge.length, DMA_FROM_DEVICE); - switch (response->type) { + switch (sc->recv_io.expected) { /* SMBD negotiation response */ - case SMBD_NEGOTIATE_RESP: - dump_smbdirect_negotiate_resp(smbd_response_payload(response)); - info->full_packet_received = true; + case SMBDIRECT_EXPECT_NEGOTIATE_REP: + dump_smbdirect_negotiate_resp(smbdirect_recv_io_payload(response)); + sc->recv_io.reassembly.full_packet_received = true; info->negotiate_done = process_negotiation_response(response, wc->byte_len); + put_receive_buffer(info, response); complete(&info->negotiate_completion); - break; + return; /* SMBD data transfer packet */ - case SMBD_TRANSFER_DATA: - data_transfer = smbd_response_payload(response); + case SMBDIRECT_EXPECT_DATA_TRANSFER: + data_transfer = smbdirect_recv_io_payload(response); data_length = le32_to_cpu(data_transfer->data_length); - /* - * If this is a packet with data playload place the data in - * reassembly queue and wake up the reading thread - */ if (data_length) { - if (info->full_packet_received) + if (sc->recv_io.reassembly.full_packet_received) response->first_segment = true; if (le32_to_cpu(data_transfer->remaining_data_length)) - info->full_packet_received = false; + sc->recv_io.reassembly.full_packet_received = false; else - info->full_packet_received = true; - - enqueue_reassembly( - info, - response, - data_length); - } else - put_empty_packet(info, response); - - if (data_length) - wake_up_interruptible(&info->wait_reassembly_queue); + sc->recv_io.reassembly.full_packet_received = true; + } atomic_dec(&info->receive_credits); info->receive_credit_target = @@ -540,15 +525,31 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) info->keep_alive_requested = KEEP_ALIVE_PENDING; } + /* + * If this is a packet with data playload place the data in + * reassembly queue and wake up the reading thread + */ + if (data_length) { + enqueue_reassembly(info, response, data_length); + wake_up_interruptible(&sc->recv_io.reassembly.wait_queue); + } else + put_receive_buffer(info, response); + return; - default: - log_rdma_recv(ERR, - "unexpected response type=%d\n", response->type); + case SMBDIRECT_EXPECT_NEGOTIATE_REQ: + /* Only server... */ + break; } + /* + * This is an internal error! + */ + log_rdma_recv(ERR, "unexpected response type=%d\n", sc->recv_io.expected); + WARN_ON_ONCE(sc->recv_io.expected != SMBDIRECT_EXPECT_DATA_TRANSFER); error: put_receive_buffer(info, response); + smbd_disconnect_rdma_connection(info); } static struct rdma_cm_id *smbd_create_id( @@ -694,16 +695,16 @@ static int smbd_post_send_negotiate_req(struct smbd_connection *info) struct smbdirect_socket_parameters *sp = &sc->parameters; struct ib_send_wr send_wr; int rc = -ENOMEM; - struct smbd_request *request; + struct smbdirect_send_io *request; struct smbdirect_negotiate_req *packet; - request = mempool_alloc(info->request_mempool, GFP_KERNEL); + request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); if (!request) return rc; - request->info = info; + request->socket = sc; - packet = smbd_request_payload(request); + packet = smbdirect_send_io_payload(request); packet->min_version = cpu_to_le16(SMBDIRECT_V1); packet->max_version = cpu_to_le16(SMBDIRECT_V1); packet->reserved = 0; @@ -756,7 +757,7 @@ static int smbd_post_send_negotiate_req(struct smbd_connection *info) smbd_disconnect_rdma_connection(info); dma_mapping_failed: - mempool_free(request, info->request_mempool); + mempool_free(request, sc->send_io.mem.pool); return rc; } @@ -800,7 +801,7 @@ static int manage_keep_alive_before_sending(struct smbd_connection *info) /* Post the send request */ static int smbd_post_send(struct smbd_connection *info, - struct smbd_request *request) + struct smbdirect_send_io *request) { struct smbdirect_socket *sc = &info->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; @@ -849,7 +850,7 @@ static int smbd_post_send_iter(struct smbd_connection *info, int i, rc; int header_length; int data_length; - struct smbd_request *request; + struct smbdirect_send_io *request; struct smbdirect_data_transfer *packet; int new_credits = 0; @@ -888,20 +889,20 @@ wait_send_queue: goto wait_send_queue; } - request = mempool_alloc(info->request_mempool, GFP_KERNEL); + request = mempool_alloc(sc->send_io.mem.pool, GFP_KERNEL); if (!request) { rc = -ENOMEM; goto err_alloc; } - request->info = info; + request->socket = sc; memset(request->sge, 0, sizeof(request->sge)); /* Fill in the data payload to find out how much data we can add */ if (iter) { struct smb_extract_to_rdma extract = { .nr_sge = 1, - .max_sge = SMBDIRECT_MAX_SEND_SGE, + .max_sge = SMBDIRECT_SEND_IO_MAX_SGE, .sge = request->sge, .device = sc->ib.dev, .local_dma_lkey = sc->ib.pd->local_dma_lkey, @@ -923,7 +924,7 @@ wait_send_queue: } /* Fill in the packet header */ - packet = smbd_request_payload(request); + packet = smbdirect_send_io_payload(request); packet->credits_requested = cpu_to_le16(sp->send_credit_target); new_credits = manage_credits_prior_sending(info); @@ -982,7 +983,7 @@ err_dma: request->sge[i].addr, request->sge[i].length, DMA_TO_DEVICE); - mempool_free(request, info->request_mempool); + mempool_free(request, sc->send_io.mem.pool); /* roll back receive credits and credits to be offered */ spin_lock(&info->lock_new_credits_offered); @@ -1042,7 +1043,7 @@ static int smbd_post_send_full_iter(struct smbd_connection *info, * The interaction is controlled by send/receive credit system */ static int smbd_post_recv( - struct smbd_connection *info, struct smbd_response *response) + struct smbd_connection *info, struct smbdirect_recv_io *response) { struct smbdirect_socket *sc = &info->socket; struct smbdirect_socket_parameters *sp = &sc->parameters; @@ -1069,6 +1070,7 @@ static int smbd_post_recv( if (rc) { ib_dma_unmap_single(sc->ib.dev, response->sge.addr, response->sge.length, DMA_FROM_DEVICE); + response->sge.length = 0; smbd_disconnect_rdma_connection(info); log_rdma_recv(ERR, "ib_post_recv failed rc=%d\n", rc); } @@ -1079,10 +1081,11 @@ static int smbd_post_recv( /* Perform SMBD negotiate according to [MS-SMBD] 3.1.5.2 */ static int smbd_negotiate(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; int rc; - struct smbd_response *response = get_receive_buffer(info); + struct smbdirect_recv_io *response = get_receive_buffer(info); - response->type = SMBD_NEGOTIATE_RESP; + sc->recv_io.expected = SMBDIRECT_EXPECT_NEGOTIATE_REP; rc = smbd_post_recv(info, response); log_rdma_event(INFO, "smbd_post_recv rc=%d iov.addr=0x%llx iov.length=%u iov.lkey=0x%x\n", rc, response->sge.addr, @@ -1113,17 +1116,6 @@ static int smbd_negotiate(struct smbd_connection *info) return rc; } -static void put_empty_packet( - struct smbd_connection *info, struct smbd_response *response) -{ - spin_lock(&info->empty_packet_queue_lock); - list_add_tail(&response->list, &info->empty_packet_queue); - info->count_empty_packet_queue++; - spin_unlock(&info->empty_packet_queue_lock); - - queue_work(info->workqueue, &info->post_send_credits_work); -} - /* * Implement Connection.FragmentReassemblyBuffer defined in [MS-SMBD] 3.1.1.1 * This is a queue for reassembling upper layer payload and present to upper @@ -1136,12 +1128,14 @@ static void put_empty_packet( */ static void enqueue_reassembly( struct smbd_connection *info, - struct smbd_response *response, + struct smbdirect_recv_io *response, int data_length) { - spin_lock(&info->reassembly_queue_lock); - list_add_tail(&response->list, &info->reassembly_queue); - info->reassembly_queue_length++; + struct smbdirect_socket *sc = &info->socket; + + spin_lock(&sc->recv_io.reassembly.lock); + list_add_tail(&response->list, &sc->recv_io.reassembly.list); + sc->recv_io.reassembly.queue_length++; /* * Make sure reassembly_data_length is updated after list and * reassembly_queue_length are updated. On the dequeue side @@ -1149,8 +1143,8 @@ static void enqueue_reassembly( * if reassembly_queue_length and list is up to date */ virt_wmb(); - info->reassembly_data_length += data_length; - spin_unlock(&info->reassembly_queue_lock); + sc->recv_io.reassembly.data_length += data_length; + spin_unlock(&sc->recv_io.reassembly.lock); info->count_reassembly_queue++; info->count_enqueue_reassembly_queue++; } @@ -1160,34 +1154,16 @@ static void enqueue_reassembly( * Caller is responsible for locking * return value: the first entry if any, NULL if queue is empty */ -static struct smbd_response *_get_first_reassembly(struct smbd_connection *info) -{ - struct smbd_response *ret = NULL; - - if (!list_empty(&info->reassembly_queue)) { - ret = list_first_entry( - &info->reassembly_queue, - struct smbd_response, list); - } - return ret; -} - -static struct smbd_response *get_empty_queue_buffer( - struct smbd_connection *info) +static struct smbdirect_recv_io *_get_first_reassembly(struct smbd_connection *info) { - struct smbd_response *ret = NULL; - unsigned long flags; + struct smbdirect_socket *sc = &info->socket; + struct smbdirect_recv_io *ret = NULL; - spin_lock_irqsave(&info->empty_packet_queue_lock, flags); - if (!list_empty(&info->empty_packet_queue)) { + if (!list_empty(&sc->recv_io.reassembly.list)) { ret = list_first_entry( - &info->empty_packet_queue, - struct smbd_response, list); - list_del(&ret->list); - info->count_empty_packet_queue--; + &sc->recv_io.reassembly.list, + struct smbdirect_recv_io, list); } - spin_unlock_irqrestore(&info->empty_packet_queue_lock, flags); - return ret; } @@ -1197,21 +1173,22 @@ static struct smbd_response *get_empty_queue_buffer( * pre-allocated in advance. * return value: the receive buffer, NULL if none is available */ -static struct smbd_response *get_receive_buffer(struct smbd_connection *info) +static struct smbdirect_recv_io *get_receive_buffer(struct smbd_connection *info) { - struct smbd_response *ret = NULL; + struct smbdirect_socket *sc = &info->socket; + struct smbdirect_recv_io *ret = NULL; unsigned long flags; - spin_lock_irqsave(&info->receive_queue_lock, flags); - if (!list_empty(&info->receive_queue)) { + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + if (!list_empty(&sc->recv_io.free.list)) { ret = list_first_entry( - &info->receive_queue, - struct smbd_response, list); + &sc->recv_io.free.list, + struct smbdirect_recv_io, list); list_del(&ret->list); info->count_receive_queue--; info->count_get_receive_buffer++; } - spin_unlock_irqrestore(&info->receive_queue_lock, flags); + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); return ret; } @@ -1223,19 +1200,24 @@ static struct smbd_response *get_receive_buffer(struct smbd_connection *info) * receive buffer is returned. */ static void put_receive_buffer( - struct smbd_connection *info, struct smbd_response *response) + struct smbd_connection *info, struct smbdirect_recv_io *response) { struct smbdirect_socket *sc = &info->socket; unsigned long flags; - ib_dma_unmap_single(sc->ib.dev, response->sge.addr, - response->sge.length, DMA_FROM_DEVICE); + if (likely(response->sge.length != 0)) { + ib_dma_unmap_single(sc->ib.dev, + response->sge.addr, + response->sge.length, + DMA_FROM_DEVICE); + response->sge.length = 0; + } - spin_lock_irqsave(&info->receive_queue_lock, flags); - list_add_tail(&response->list, &info->receive_queue); + spin_lock_irqsave(&sc->recv_io.free.lock, flags); + list_add_tail(&response->list, &sc->recv_io.free.list); info->count_receive_queue++; info->count_put_receive_buffer++; - spin_unlock_irqrestore(&info->receive_queue_lock, flags); + spin_unlock_irqrestore(&sc->recv_io.free.lock, flags); queue_work(info->workqueue, &info->post_send_credits_work); } @@ -1243,58 +1225,54 @@ static void put_receive_buffer( /* Preallocate all receive buffer on transport establishment */ static int allocate_receive_buffers(struct smbd_connection *info, int num_buf) { + struct smbdirect_socket *sc = &info->socket; + struct smbdirect_recv_io *response; int i; - struct smbd_response *response; - INIT_LIST_HEAD(&info->reassembly_queue); - spin_lock_init(&info->reassembly_queue_lock); - info->reassembly_data_length = 0; - info->reassembly_queue_length = 0; + INIT_LIST_HEAD(&sc->recv_io.reassembly.list); + spin_lock_init(&sc->recv_io.reassembly.lock); + sc->recv_io.reassembly.data_length = 0; + sc->recv_io.reassembly.queue_length = 0; - INIT_LIST_HEAD(&info->receive_queue); - spin_lock_init(&info->receive_queue_lock); + INIT_LIST_HEAD(&sc->recv_io.free.list); + spin_lock_init(&sc->recv_io.free.lock); info->count_receive_queue = 0; - INIT_LIST_HEAD(&info->empty_packet_queue); - spin_lock_init(&info->empty_packet_queue_lock); - info->count_empty_packet_queue = 0; - init_waitqueue_head(&info->wait_receive_queues); for (i = 0; i < num_buf; i++) { - response = mempool_alloc(info->response_mempool, GFP_KERNEL); + response = mempool_alloc(sc->recv_io.mem.pool, GFP_KERNEL); if (!response) goto allocate_failed; - response->info = info; - list_add_tail(&response->list, &info->receive_queue); + response->socket = sc; + response->sge.length = 0; + list_add_tail(&response->list, &sc->recv_io.free.list); info->count_receive_queue++; } return 0; allocate_failed: - while (!list_empty(&info->receive_queue)) { + while (!list_empty(&sc->recv_io.free.list)) { response = list_first_entry( - &info->receive_queue, - struct smbd_response, list); + &sc->recv_io.free.list, + struct smbdirect_recv_io, list); list_del(&response->list); info->count_receive_queue--; - mempool_free(response, info->response_mempool); + mempool_free(response, sc->recv_io.mem.pool); } return -ENOMEM; } static void destroy_receive_buffers(struct smbd_connection *info) { - struct smbd_response *response; + struct smbdirect_socket *sc = &info->socket; + struct smbdirect_recv_io *response; while ((response = get_receive_buffer(info))) - mempool_free(response, info->response_mempool); - - while ((response = get_empty_queue_buffer(info))) - mempool_free(response, info->response_mempool); + mempool_free(response, sc->recv_io.mem.pool); } /* Implement idle connection timer [MS-SMBD] 3.1.6.2 */ @@ -1332,7 +1310,7 @@ void smbd_destroy(struct TCP_Server_Info *server) struct smbd_connection *info = server->smbd_conn; struct smbdirect_socket *sc; struct smbdirect_socket_parameters *sp; - struct smbd_response *response; + struct smbdirect_recv_io *response; unsigned long flags; if (!info) { @@ -1347,7 +1325,7 @@ void smbd_destroy(struct TCP_Server_Info *server) rdma_disconnect(sc->rdma.cm_id); log_rdma_event(INFO, "wait for transport being disconnected\n"); wait_event_interruptible( - info->disconn_wait, + info->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); } @@ -1366,23 +1344,22 @@ void smbd_destroy(struct TCP_Server_Info *server) /* It's not possible for upper layer to get to reassembly */ log_rdma_event(INFO, "drain the reassembly queue\n"); do { - spin_lock_irqsave(&info->reassembly_queue_lock, flags); + spin_lock_irqsave(&sc->recv_io.reassembly.lock, flags); response = _get_first_reassembly(info); if (response) { list_del(&response->list); spin_unlock_irqrestore( - &info->reassembly_queue_lock, flags); + &sc->recv_io.reassembly.lock, flags); put_receive_buffer(info, response); } else spin_unlock_irqrestore( - &info->reassembly_queue_lock, flags); + &sc->recv_io.reassembly.lock, flags); } while (response); - info->reassembly_data_length = 0; + sc->recv_io.reassembly.data_length = 0; log_rdma_event(INFO, "free receive buffers\n"); wait_event(info->wait_receive_queues, - info->count_receive_queue + info->count_empty_packet_queue - == sp->recv_credit_max); + info->count_receive_queue == sp->recv_credit_max); destroy_receive_buffers(info); /* @@ -1407,11 +1384,11 @@ void smbd_destroy(struct TCP_Server_Info *server) rdma_destroy_id(sc->rdma.cm_id); /* free mempools */ - mempool_destroy(info->request_mempool); - kmem_cache_destroy(info->request_cache); + mempool_destroy(sc->send_io.mem.pool); + kmem_cache_destroy(sc->send_io.mem.cache); - mempool_destroy(info->response_mempool); - kmem_cache_destroy(info->response_cache); + mempool_destroy(sc->recv_io.mem.pool); + kmem_cache_destroy(sc->recv_io.mem.cache); sc->status = SMBDIRECT_SOCKET_DESTROYED; @@ -1459,12 +1436,14 @@ create_conn: static void destroy_caches_and_workqueue(struct smbd_connection *info) { + struct smbdirect_socket *sc = &info->socket; + destroy_receive_buffers(info); destroy_workqueue(info->workqueue); - mempool_destroy(info->response_mempool); - kmem_cache_destroy(info->response_cache); - mempool_destroy(info->request_mempool); - kmem_cache_destroy(info->request_cache); + mempool_destroy(sc->recv_io.mem.pool); + kmem_cache_destroy(sc->recv_io.mem.cache); + mempool_destroy(sc->send_io.mem.pool); + kmem_cache_destroy(sc->send_io.mem.cache); } #define MAX_NAME_LEN 80 @@ -1478,41 +1457,41 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info) if (WARN_ON_ONCE(sp->max_recv_size < sizeof(struct smbdirect_data_transfer))) return -ENOMEM; - scnprintf(name, MAX_NAME_LEN, "smbd_request_%p", info); - info->request_cache = + scnprintf(name, MAX_NAME_LEN, "smbdirect_send_io_%p", info); + sc->send_io.mem.cache = kmem_cache_create( name, - sizeof(struct smbd_request) + + sizeof(struct smbdirect_send_io) + sizeof(struct smbdirect_data_transfer), 0, SLAB_HWCACHE_ALIGN, NULL); - if (!info->request_cache) + if (!sc->send_io.mem.cache) return -ENOMEM; - info->request_mempool = + sc->send_io.mem.pool = mempool_create(sp->send_credit_target, mempool_alloc_slab, - mempool_free_slab, info->request_cache); - if (!info->request_mempool) + mempool_free_slab, sc->send_io.mem.cache); + if (!sc->send_io.mem.pool) goto out1; - scnprintf(name, MAX_NAME_LEN, "smbd_response_%p", info); + scnprintf(name, MAX_NAME_LEN, "smbdirect_recv_io_%p", info); struct kmem_cache_args response_args = { - .align = __alignof__(struct smbd_response), - .useroffset = (offsetof(struct smbd_response, packet) + + .align = __alignof__(struct smbdirect_recv_io), + .useroffset = (offsetof(struct smbdirect_recv_io, packet) + sizeof(struct smbdirect_data_transfer)), .usersize = sp->max_recv_size - sizeof(struct smbdirect_data_transfer), }; - info->response_cache = + sc->recv_io.mem.cache = kmem_cache_create(name, - sizeof(struct smbd_response) + sp->max_recv_size, + sizeof(struct smbdirect_recv_io) + sp->max_recv_size, &response_args, SLAB_HWCACHE_ALIGN); - if (!info->response_cache) + if (!sc->recv_io.mem.cache) goto out2; - info->response_mempool = + sc->recv_io.mem.pool = mempool_create(sp->recv_credit_max, mempool_alloc_slab, - mempool_free_slab, info->response_cache); - if (!info->response_mempool) + mempool_free_slab, sc->recv_io.mem.cache); + if (!sc->recv_io.mem.pool) goto out3; scnprintf(name, MAX_NAME_LEN, "smbd_%p", info); @@ -1531,13 +1510,13 @@ static int allocate_caches_and_workqueue(struct smbd_connection *info) out5: destroy_workqueue(info->workqueue); out4: - mempool_destroy(info->response_mempool); + mempool_destroy(sc->recv_io.mem.pool); out3: - kmem_cache_destroy(info->response_cache); + kmem_cache_destroy(sc->recv_io.mem.cache); out2: - mempool_destroy(info->request_mempool); + mempool_destroy(sc->send_io.mem.pool); out1: - kmem_cache_destroy(info->request_cache); + kmem_cache_destroy(sc->send_io.mem.cache); return -ENOMEM; } @@ -1593,8 +1572,8 @@ static struct smbd_connection *_smbd_get_connection( sp->max_recv_size = smbd_max_receive_size; sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000; - if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_MAX_SEND_SGE || - sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_MAX_RECV_SGE) { + if (sc->ib.dev->attrs.max_send_sge < SMBDIRECT_SEND_IO_MAX_SGE || + sc->ib.dev->attrs.max_recv_sge < SMBDIRECT_RECV_IO_MAX_SGE) { log_rdma_event(ERR, "device %.*s max_send_sge/max_recv_sge = %d/%d too small\n", IB_DEVICE_NAME_MAX, @@ -1625,8 +1604,8 @@ static struct smbd_connection *_smbd_get_connection( qp_attr.qp_context = info; qp_attr.cap.max_send_wr = sp->send_credit_target; qp_attr.cap.max_recv_wr = sp->recv_credit_max; - qp_attr.cap.max_send_sge = SMBDIRECT_MAX_SEND_SGE; - qp_attr.cap.max_recv_sge = SMBDIRECT_MAX_RECV_SGE; + qp_attr.cap.max_send_sge = SMBDIRECT_SEND_IO_MAX_SGE; + qp_attr.cap.max_recv_sge = SMBDIRECT_RECV_IO_MAX_SGE; qp_attr.cap.max_inline_data = 0; qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR; qp_attr.qp_type = IB_QPT_RC; @@ -1671,17 +1650,18 @@ static struct smbd_connection *_smbd_get_connection( log_rdma_event(INFO, "connecting to IP %pI4 port %d\n", &addr_in->sin_addr, port); - init_waitqueue_head(&info->conn_wait); - init_waitqueue_head(&info->disconn_wait); - init_waitqueue_head(&info->wait_reassembly_queue); + init_waitqueue_head(&info->status_wait); + init_waitqueue_head(&sc->recv_io.reassembly.wait_queue); rc = rdma_connect(sc->rdma.cm_id, &conn_param); if (rc) { log_rdma_event(ERR, "rdma_connect() failed with %i\n", rc); goto rdma_connect_failed; } - wait_event_interruptible( - info->conn_wait, sc->status != SMBDIRECT_SOCKET_CONNECTING); + wait_event_interruptible_timeout( + info->status_wait, + sc->status != SMBDIRECT_SOCKET_CONNECTING, + msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT)); if (sc->status != SMBDIRECT_SOCKET_CONNECTED) { log_rdma_event(ERR, "rdma_connect failed port=%d\n", port); @@ -1735,9 +1715,8 @@ negotiation_failed: cancel_delayed_work_sync(&info->idle_timer_work); destroy_caches_and_workqueue(info); sc->status = SMBDIRECT_SOCKET_NEGOTIATE_FAILED; - init_waitqueue_head(&info->conn_wait); rdma_disconnect(sc->rdma.cm_id); - wait_event(info->conn_wait, + wait_event(info->status_wait, sc->status == SMBDIRECT_SOCKET_DISCONNECTED); allocate_cache_failed: @@ -1794,7 +1773,7 @@ try_again: int smbd_recv(struct smbd_connection *info, struct msghdr *msg) { struct smbdirect_socket *sc = &info->socket; - struct smbd_response *response; + struct smbdirect_recv_io *response; struct smbdirect_data_transfer *data_transfer; size_t size = iov_iter_count(&msg->msg_iter); int to_copy, to_read, data_read, offset; @@ -1810,9 +1789,9 @@ again: * the only one reading from the front of the queue. The transport * may add more entries to the back of the queue at the same time */ - log_read(INFO, "size=%zd info->reassembly_data_length=%d\n", size, - info->reassembly_data_length); - if (info->reassembly_data_length >= size) { + log_read(INFO, "size=%zd sc->recv_io.reassembly.data_length=%d\n", size, + sc->recv_io.reassembly.data_length); + if (sc->recv_io.reassembly.data_length >= size) { int queue_length; int queue_removed = 0; @@ -1824,13 +1803,13 @@ again: * updated in SOFTIRQ as more data is received */ virt_rmb(); - queue_length = info->reassembly_queue_length; + queue_length = sc->recv_io.reassembly.queue_length; data_read = 0; to_read = size; - offset = info->first_entry_offset; + offset = sc->recv_io.reassembly.first_entry_offset; while (data_read < size) { response = _get_first_reassembly(info); - data_transfer = smbd_response_payload(response); + data_transfer = smbdirect_recv_io_payload(response); data_length = le32_to_cpu(data_transfer->data_length); remaining_data_length = le32_to_cpu( @@ -1875,10 +1854,10 @@ again: list_del(&response->list); else { spin_lock_irq( - &info->reassembly_queue_lock); + &sc->recv_io.reassembly.lock); list_del(&response->list); spin_unlock_irq( - &info->reassembly_queue_lock); + &sc->recv_io.reassembly.lock); } queue_removed++; info->count_reassembly_queue--; @@ -1897,23 +1876,23 @@ again: to_read, data_read, offset); } - spin_lock_irq(&info->reassembly_queue_lock); - info->reassembly_data_length -= data_read; - info->reassembly_queue_length -= queue_removed; - spin_unlock_irq(&info->reassembly_queue_lock); + spin_lock_irq(&sc->recv_io.reassembly.lock); + sc->recv_io.reassembly.data_length -= data_read; + sc->recv_io.reassembly.queue_length -= queue_removed; + spin_unlock_irq(&sc->recv_io.reassembly.lock); - info->first_entry_offset = offset; + sc->recv_io.reassembly.first_entry_offset = offset; log_read(INFO, "returning to thread data_read=%d reassembly_data_length=%d first_entry_offset=%d\n", - data_read, info->reassembly_data_length, - info->first_entry_offset); + data_read, sc->recv_io.reassembly.data_length, + sc->recv_io.reassembly.first_entry_offset); read_rfc1002_done: return data_read; } log_read(INFO, "wait_event on more data\n"); rc = wait_event_interruptible( - info->wait_reassembly_queue, - info->reassembly_data_length >= size || + sc->recv_io.reassembly.wait_queue, + sc->recv_io.reassembly.data_length >= size || sc->status != SMBDIRECT_SOCKET_CONNECTED); /* Don't return any data if interrupted */ if (rc) diff --git a/fs/smb/client/smbdirect.h b/fs/smb/client/smbdirect.h index 75b3f491c3ad..e45aa9ddd71d 100644 --- a/fs/smb/client/smbdirect.h +++ b/fs/smb/client/smbdirect.h @@ -33,16 +33,6 @@ enum keep_alive_status { KEEP_ALIVE_SENT, }; -enum smbd_connection_status { - SMBD_CREATED, - SMBD_CONNECTING, - SMBD_CONNECTED, - SMBD_NEGOTIATE_FAILED, - SMBD_DISCONNECTING, - SMBD_DISCONNECTED, - SMBD_DESTROYED -}; - /* * The context for the SMBDirect transport * Everything related to the transport is here. It has several logical parts @@ -57,8 +47,7 @@ struct smbd_connection { int ri_rc; struct completion ri_done; - wait_queue_head_t conn_wait; - wait_queue_head_t disconn_wait; + wait_queue_head_t status_wait; struct completion negotiate_completion; bool negotiate_done; @@ -75,7 +64,6 @@ struct smbd_connection { atomic_t send_credits; atomic_t receive_credits; int receive_credit_target; - int fragment_reassembly_remaining; /* Memory registrations */ /* Maximum number of RDMA read/write outstanding on this connection */ @@ -106,52 +94,16 @@ struct smbd_connection { wait_queue_head_t wait_post_send; /* Receive queue */ - struct list_head receive_queue; int count_receive_queue; - spinlock_t receive_queue_lock; - - struct list_head empty_packet_queue; - int count_empty_packet_queue; - spinlock_t empty_packet_queue_lock; - wait_queue_head_t wait_receive_queues; - /* Reassembly queue */ - struct list_head reassembly_queue; - spinlock_t reassembly_queue_lock; - wait_queue_head_t wait_reassembly_queue; - - /* total data length of reassembly queue */ - int reassembly_data_length; - int reassembly_queue_length; - /* the offset to first buffer in reassembly queue */ - int first_entry_offset; - bool send_immediate; wait_queue_head_t wait_send_queue; - /* - * Indicate if we have received a full packet on the connection - * This is used to identify the first SMBD packet of a assembled - * payload (SMB packet) in reassembly queue so we can return a - * RFC1002 length to upper layer to indicate the length of the SMB - * packet received - */ - bool full_packet_received; - struct workqueue_struct *workqueue; struct delayed_work idle_timer_work; - /* Memory pool for preallocating buffers */ - /* request pool for RDMA send */ - struct kmem_cache *request_cache; - mempool_t *request_mempool; - - /* response pool for RDMA receive */ - struct kmem_cache *response_cache; - mempool_t *response_mempool; - /* for debug purposes */ unsigned int count_get_receive_buffer; unsigned int count_put_receive_buffer; @@ -161,48 +113,6 @@ struct smbd_connection { unsigned int count_send_empty; }; -enum smbd_message_type { - SMBD_NEGOTIATE_RESP, - SMBD_TRANSFER_DATA, -}; - -/* Maximum number of SGEs used by smbdirect.c in any send work request */ -#define SMBDIRECT_MAX_SEND_SGE 6 - -/* The context for a SMBD request */ -struct smbd_request { - struct smbd_connection *info; - struct ib_cqe cqe; - - /* the SGE entries for this work request */ - struct ib_sge sge[SMBDIRECT_MAX_SEND_SGE]; - int num_sge; - - /* SMBD packet header follows this structure */ - u8 packet[]; -}; - -/* Maximum number of SGEs used by smbdirect.c in any receive work request */ -#define SMBDIRECT_MAX_RECV_SGE 1 - -/* The context for a SMBD response */ -struct smbd_response { - struct smbd_connection *info; - struct ib_cqe cqe; - struct ib_sge sge; - - enum smbd_message_type type; - - /* Link to receive queue or reassembly queue */ - struct list_head list; - - /* Indicate if this is the 1st packet of a payload */ - bool first_segment; - - /* SMBD packet header and payload follows this structure */ - u8 packet[]; -}; - /* Create a SMBDirect session */ struct smbd_connection *smbd_get_connection( struct TCP_Server_Info *server, struct sockaddr *dstaddr); diff --git a/fs/smb/client/transport.c b/fs/smb/client/transport.c index 191783f553ce..32d528b4dd83 100644 --- a/fs/smb/client/transport.c +++ b/fs/smb/client/transport.c @@ -30,9 +30,6 @@ #include "smbdirect.h" #include "compress.h" -/* Max number of iovectors we can use off the stack when sending requests. */ -#define CIFS_MAX_IOV_SIZE 8 - void cifs_wake_up_task(struct mid_q_entry *mid) { @@ -41,42 +38,6 @@ cifs_wake_up_task(struct mid_q_entry *mid) wake_up_process(mid->callback_data); } -static struct mid_q_entry * -alloc_mid(const struct smb_hdr *smb_buffer, struct TCP_Server_Info *server) -{ - struct mid_q_entry *temp; - - if (server == NULL) { - cifs_dbg(VFS, "%s: null TCP session\n", __func__); - return NULL; - } - - temp = mempool_alloc(cifs_mid_poolp, GFP_NOFS); - memset(temp, 0, sizeof(struct mid_q_entry)); - kref_init(&temp->refcount); - temp->mid = get_mid(smb_buffer); - temp->pid = current->pid; - temp->command = cpu_to_le16(smb_buffer->Command); - cifs_dbg(FYI, "For smb_command %d\n", smb_buffer->Command); - /* easier to use jiffies */ - /* when mid allocated can be before when sent */ - temp->when_alloc = jiffies; - temp->server = server; - - /* - * The default is for the mid to be synchronous, so the - * default callback just wakes up the current task. - */ - get_task_struct(current); - temp->creator = current; - temp->callback = cifs_wake_up_task; - temp->callback_data = current; - - atomic_inc(&mid_count); - temp->mid_state = MID_REQUEST_ALLOCATED; - return temp; -} - void __release_mid(struct kref *refcount) { struct mid_q_entry *midEntry = @@ -89,7 +50,7 @@ void __release_mid(struct kref *refcount) #endif struct TCP_Server_Info *server = midEntry->server; - if (midEntry->resp_buf && (midEntry->mid_flags & MID_WAIT_CANCELLED) && + if (midEntry->resp_buf && (midEntry->wait_cancelled) && (midEntry->mid_state == MID_RESPONSE_RECEIVED || midEntry->mid_state == MID_RESPONSE_READY) && server->ops->handle_cancelled_mid) @@ -160,12 +121,12 @@ void __release_mid(struct kref *refcount) void delete_mid(struct mid_q_entry *mid) { - spin_lock(&mid->server->mid_lock); - if (!(mid->mid_flags & MID_DELETED)) { + spin_lock(&mid->server->mid_queue_lock); + if (mid->deleted_from_q == false) { list_del_init(&mid->qhead); - mid->mid_flags |= MID_DELETED; + mid->deleted_from_q = true; } - spin_unlock(&mid->server->mid_lock); + spin_unlock(&mid->server->mid_queue_lock); release_mid(mid); } @@ -269,9 +230,8 @@ smb_rqst_len(struct TCP_Server_Info *server, struct smb_rqst *rqst) return buflen; } -static int -__smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, - struct smb_rqst *rqst) +int __smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, + struct smb_rqst *rqst) { int rc; struct kvec *iov; @@ -397,7 +357,7 @@ unmask: * socket so the server throws away the partial SMB */ cifs_signal_cifsd_for_reconnect(server, false); - trace_smb3_partial_send_reconnect(server->CurrentMid, + trace_smb3_partial_send_reconnect(server->current_mid, server->conn_id, server->hostname); } smbd_done: @@ -456,22 +416,6 @@ smb_send_rqst(struct TCP_Server_Info *server, int num_rqst, return rc; } -int -smb_send(struct TCP_Server_Info *server, struct smb_hdr *smb_buffer, - unsigned int smb_buf_length) -{ - struct kvec iov[2]; - struct smb_rqst rqst = { .rq_iov = iov, - .rq_nvec = 2 }; - - iov[0].iov_base = smb_buffer; - iov[0].iov_len = 4; - iov[1].iov_base = (char *)smb_buffer + 4; - iov[1].iov_len = smb_buf_length; - - return __smb_send_rqst(server, 1, &rqst); -} - static int wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, const int timeout, const int flags, @@ -509,7 +453,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, in_flight = server->in_flight; spin_unlock(&server->req_lock); - trace_smb3_nblk_credits(server->CurrentMid, + trace_smb3_nblk_credits(server->current_mid, server->conn_id, server->hostname, scredits, -1, in_flight); cifs_dbg(FYI, "%s: remove %u credits total=%d\n", __func__, 1, scredits); @@ -542,7 +486,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, in_flight = server->in_flight; spin_unlock(&server->req_lock); - trace_smb3_credit_timeout(server->CurrentMid, + trace_smb3_credit_timeout(server->current_mid, server->conn_id, server->hostname, scredits, num_credits, in_flight); cifs_server_dbg(VFS, "wait timed out after %d ms\n", @@ -585,7 +529,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, spin_unlock(&server->req_lock); trace_smb3_credit_timeout( - server->CurrentMid, + server->current_mid, server->conn_id, server->hostname, scredits, num_credits, in_flight); cifs_server_dbg(VFS, "wait timed out after %d ms\n", @@ -615,7 +559,7 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, in_flight = server->in_flight; spin_unlock(&server->req_lock); - trace_smb3_waitff_credits(server->CurrentMid, + trace_smb3_waitff_credits(server->current_mid, server->conn_id, server->hostname, scredits, -(num_credits), in_flight); cifs_dbg(FYI, "%s: remove %u credits total=%d\n", @@ -626,9 +570,8 @@ wait_for_free_credits(struct TCP_Server_Info *server, const int num_credits, return 0; } -static int -wait_for_free_request(struct TCP_Server_Info *server, const int flags, - unsigned int *instance) +int wait_for_free_request(struct TCP_Server_Info *server, const int flags, + unsigned int *instance) { return wait_for_free_credits(server, 1, -1, flags, instance); @@ -666,7 +609,7 @@ wait_for_compound_request(struct TCP_Server_Info *server, int num, */ if (server->in_flight == 0) { spin_unlock(&server->req_lock); - trace_smb3_insufficient_credits(server->CurrentMid, + trace_smb3_insufficient_credits(server->current_mid, server->conn_id, server->hostname, scredits, num, in_flight); cifs_dbg(FYI, "%s: %d requests in flight, needed %d total=%d\n", @@ -690,40 +633,7 @@ cifs_wait_mtu_credits(struct TCP_Server_Info *server, size_t size, return 0; } -static int allocate_mid(struct cifs_ses *ses, struct smb_hdr *in_buf, - struct mid_q_entry **ppmidQ) -{ - spin_lock(&ses->ses_lock); - if (ses->ses_status == SES_NEW) { - if ((in_buf->Command != SMB_COM_SESSION_SETUP_ANDX) && - (in_buf->Command != SMB_COM_NEGOTIATE)) { - spin_unlock(&ses->ses_lock); - return -EAGAIN; - } - /* else ok - we are setting up session */ - } - - if (ses->ses_status == SES_EXITING) { - /* check if SMB session is bad because we are setting it up */ - if (in_buf->Command != SMB_COM_LOGOFF_ANDX) { - spin_unlock(&ses->ses_lock); - return -EAGAIN; - } - /* else ok - we are shutting down session */ - } - spin_unlock(&ses->ses_lock); - - *ppmidQ = alloc_mid(in_buf, ses->server); - if (*ppmidQ == NULL) - return -ENOMEM; - spin_lock(&ses->server->mid_lock); - list_add_tail(&(*ppmidQ)->qhead, &ses->server->pending_mid_q); - spin_unlock(&ses->server->mid_lock); - return 0; -} - -static int -wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *midQ) +int wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *midQ) { int error; @@ -737,34 +647,6 @@ wait_for_response(struct TCP_Server_Info *server, struct mid_q_entry *midQ) return 0; } -struct mid_q_entry * -cifs_setup_async_request(struct TCP_Server_Info *server, struct smb_rqst *rqst) -{ - int rc; - struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; - struct mid_q_entry *mid; - - if (rqst->rq_iov[0].iov_len != 4 || - rqst->rq_iov[0].iov_base + 4 != rqst->rq_iov[1].iov_base) - return ERR_PTR(-EIO); - - /* enable signing if server requires it */ - if (server->sign) - hdr->Flags2 |= SMBFLG2_SECURITY_SIGNATURE; - - mid = alloc_mid(hdr, server); - if (mid == NULL) - return ERR_PTR(-ENOMEM); - - rc = cifs_sign_rqst(rqst, server, &mid->sequence_number); - if (rc) { - release_mid(mid); - return ERR_PTR(rc); - } - - return mid; -} - /* * Send a SMB request and set the callback function in the mid to handle * the result. Caller is responsible for dealing with timeouts. @@ -819,9 +701,9 @@ cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst, mid->mid_state = MID_REQUEST_SUBMITTED; /* put it on the pending_mid_q */ - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); list_add_tail(&mid->qhead, &server->pending_mid_q); - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); /* * Need to store the time in mid before calling I/O. For call_async, @@ -845,45 +727,17 @@ cifs_call_async(struct TCP_Server_Info *server, struct smb_rqst *rqst, return rc; } -/* - * - * Send an SMB Request. No response info (other than return code) - * needs to be parsed. - * - * flags indicate the type of request buffer and how long to wait - * and whether to log NT STATUS code (error) before mapping it to POSIX error - * - */ -int -SendReceiveNoRsp(const unsigned int xid, struct cifs_ses *ses, - char *in_buf, int flags) -{ - int rc; - struct kvec iov[1]; - struct kvec rsp_iov; - int resp_buf_type; - - iov[0].iov_base = in_buf; - iov[0].iov_len = get_rfc1002_length(in_buf) + 4; - flags |= CIFS_NO_RSP_BUF; - rc = SendReceive2(xid, ses, iov, 1, &resp_buf_type, flags, &rsp_iov); - cifs_dbg(NOISY, "SendRcvNoRsp flags %d rc %d\n", flags, rc); - - return rc; -} - -static int -cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server) +int cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server) { int rc = 0; cifs_dbg(FYI, "%s: cmd=%d mid=%llu state=%d\n", __func__, le16_to_cpu(mid->command), mid->mid, mid->mid_state); - spin_lock(&server->mid_lock); + spin_lock(&server->mid_queue_lock); switch (mid->mid_state) { case MID_RESPONSE_READY: - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); return rc; case MID_RETRY_NEEDED: rc = -EAGAIN; @@ -898,85 +752,23 @@ cifs_sync_mid_result(struct mid_q_entry *mid, struct TCP_Server_Info *server) rc = mid->mid_rc; break; default: - if (!(mid->mid_flags & MID_DELETED)) { + if (mid->deleted_from_q == false) { list_del_init(&mid->qhead); - mid->mid_flags |= MID_DELETED; + mid->deleted_from_q = true; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); cifs_server_dbg(VFS, "%s: invalid mid state mid=%llu state=%d\n", __func__, mid->mid, mid->mid_state); rc = -EIO; goto sync_mid_done; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); sync_mid_done: release_mid(mid); return rc; } -static inline int -send_cancel(struct TCP_Server_Info *server, struct smb_rqst *rqst, - struct mid_q_entry *mid) -{ - return server->ops->send_cancel ? - server->ops->send_cancel(server, rqst, mid) : 0; -} - -int -cifs_check_receive(struct mid_q_entry *mid, struct TCP_Server_Info *server, - bool log_error) -{ - unsigned int len = get_rfc1002_length(mid->resp_buf) + 4; - - dump_smb(mid->resp_buf, min_t(u32, 92, len)); - - /* convert the length into a more usable form */ - if (server->sign) { - struct kvec iov[2]; - int rc = 0; - struct smb_rqst rqst = { .rq_iov = iov, - .rq_nvec = 2 }; - - iov[0].iov_base = mid->resp_buf; - iov[0].iov_len = 4; - iov[1].iov_base = (char *)mid->resp_buf + 4; - iov[1].iov_len = len - 4; - /* FIXME: add code to kill session */ - rc = cifs_verify_signature(&rqst, server, - mid->sequence_number); - if (rc) - cifs_server_dbg(VFS, "SMB signature verification returned error = %d\n", - rc); - } - - /* BB special case reconnect tid and uid here? */ - return map_and_check_smb_error(mid, log_error); -} - -struct mid_q_entry * -cifs_setup_request(struct cifs_ses *ses, struct TCP_Server_Info *ignored, - struct smb_rqst *rqst) -{ - int rc; - struct smb_hdr *hdr = (struct smb_hdr *)rqst->rq_iov[0].iov_base; - struct mid_q_entry *mid; - - if (rqst->rq_iov[0].iov_len != 4 || - rqst->rq_iov[0].iov_base + 4 != rqst->rq_iov[1].iov_base) - return ERR_PTR(-EIO); - - rc = allocate_mid(ses, hdr, &mid); - if (rc) - return ERR_PTR(rc); - rc = cifs_sign_rqst(rqst, ses->server, &mid->sequence_number); - if (rc) { - delete_mid(mid); - return ERR_PTR(rc); - } - return mid; -} - static void cifs_compound_callback(struct mid_q_entry *mid) { @@ -1213,15 +1005,15 @@ compound_send_recv(const unsigned int xid, struct cifs_ses *ses, cifs_server_dbg(FYI, "Cancelling wait for mid %llu cmd: %d\n", midQ[i]->mid, le16_to_cpu(midQ[i]->command)); send_cancel(server, &rqst[i], midQ[i]); - spin_lock(&server->mid_lock); - midQ[i]->mid_flags |= MID_WAIT_CANCELLED; + spin_lock(&server->mid_queue_lock); + midQ[i]->wait_cancelled = true; if (midQ[i]->mid_state == MID_REQUEST_SUBMITTED || midQ[i]->mid_state == MID_RESPONSE_RECEIVED) { midQ[i]->callback = cifs_cancelled_callback; cancelled_mid[i] = true; credits[i].value = 0; } - spin_unlock(&server->mid_lock); + spin_unlock(&server->mid_queue_lock); } } @@ -1304,344 +1096,6 @@ cifs_send_recv(const unsigned int xid, struct cifs_ses *ses, rqst, resp_buf_type, resp_iov); } -int -SendReceive2(const unsigned int xid, struct cifs_ses *ses, - struct kvec *iov, int n_vec, int *resp_buf_type /* ret */, - const int flags, struct kvec *resp_iov) -{ - struct smb_rqst rqst; - struct kvec s_iov[CIFS_MAX_IOV_SIZE], *new_iov; - int rc; - - if (n_vec + 1 > CIFS_MAX_IOV_SIZE) { - new_iov = kmalloc_array(n_vec + 1, sizeof(struct kvec), - GFP_KERNEL); - if (!new_iov) { - /* otherwise cifs_send_recv below sets resp_buf_type */ - *resp_buf_type = CIFS_NO_BUFFER; - return -ENOMEM; - } - } else - new_iov = s_iov; - - /* 1st iov is a RFC1001 length followed by the rest of the packet */ - memcpy(new_iov + 1, iov, (sizeof(struct kvec) * n_vec)); - - new_iov[0].iov_base = new_iov[1].iov_base; - new_iov[0].iov_len = 4; - new_iov[1].iov_base += 4; - new_iov[1].iov_len -= 4; - - memset(&rqst, 0, sizeof(struct smb_rqst)); - rqst.rq_iov = new_iov; - rqst.rq_nvec = n_vec + 1; - - rc = cifs_send_recv(xid, ses, ses->server, - &rqst, resp_buf_type, flags, resp_iov); - if (n_vec + 1 > CIFS_MAX_IOV_SIZE) - kfree(new_iov); - return rc; -} - -int -SendReceive(const unsigned int xid, struct cifs_ses *ses, - struct smb_hdr *in_buf, struct smb_hdr *out_buf, - int *pbytes_returned, const int flags) -{ - int rc = 0; - struct mid_q_entry *midQ; - unsigned int len = be32_to_cpu(in_buf->smb_buf_length); - struct kvec iov = { .iov_base = in_buf, .iov_len = len }; - struct smb_rqst rqst = { .rq_iov = &iov, .rq_nvec = 1 }; - struct cifs_credits credits = { .value = 1, .instance = 0 }; - struct TCP_Server_Info *server; - - if (ses == NULL) { - cifs_dbg(VFS, "Null smb session\n"); - return -EIO; - } - server = ses->server; - if (server == NULL) { - cifs_dbg(VFS, "Null tcp session\n"); - return -EIO; - } - - spin_lock(&server->srv_lock); - if (server->tcpStatus == CifsExiting) { - spin_unlock(&server->srv_lock); - return -ENOENT; - } - spin_unlock(&server->srv_lock); - - /* Ensure that we do not send more than 50 overlapping requests - to the same server. We may make this configurable later or - use ses->maxReq */ - - if (len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { - cifs_server_dbg(VFS, "Invalid length, greater than maximum frame, %d\n", - len); - return -EIO; - } - - rc = wait_for_free_request(server, flags, &credits.instance); - if (rc) - return rc; - - /* make sure that we sign in the same order that we send on this socket - and avoid races inside tcp sendmsg code that could cause corruption - of smb data */ - - cifs_server_lock(server); - - rc = allocate_mid(ses, in_buf, &midQ); - if (rc) { - cifs_server_unlock(server); - /* Update # of requests on wire to server */ - add_credits(server, &credits, 0); - return rc; - } - - rc = cifs_sign_smb(in_buf, server, &midQ->sequence_number); - if (rc) { - cifs_server_unlock(server); - goto out; - } - - midQ->mid_state = MID_REQUEST_SUBMITTED; - - rc = smb_send(server, in_buf, len); - cifs_save_when_sent(midQ); - - if (rc < 0) - server->sequence_number -= 2; - - cifs_server_unlock(server); - - if (rc < 0) - goto out; - - rc = wait_for_response(server, midQ); - if (rc != 0) { - send_cancel(server, &rqst, midQ); - spin_lock(&server->mid_lock); - if (midQ->mid_state == MID_REQUEST_SUBMITTED || - midQ->mid_state == MID_RESPONSE_RECEIVED) { - /* no longer considered to be "in-flight" */ - midQ->callback = release_mid; - spin_unlock(&server->mid_lock); - add_credits(server, &credits, 0); - return rc; - } - spin_unlock(&server->mid_lock); - } - - rc = cifs_sync_mid_result(midQ, server); - if (rc != 0) { - add_credits(server, &credits, 0); - return rc; - } - - if (!midQ->resp_buf || !out_buf || - midQ->mid_state != MID_RESPONSE_READY) { - rc = -EIO; - cifs_server_dbg(VFS, "Bad MID state?\n"); - goto out; - } - - *pbytes_returned = get_rfc1002_length(midQ->resp_buf); - memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); - rc = cifs_check_receive(midQ, server, 0); -out: - delete_mid(midQ); - add_credits(server, &credits, 0); - - return rc; -} - -/* We send a LOCKINGX_CANCEL_LOCK to cause the Windows - blocking lock to return. */ - -static int -send_lock_cancel(const unsigned int xid, struct cifs_tcon *tcon, - struct smb_hdr *in_buf, - struct smb_hdr *out_buf) -{ - int bytes_returned; - struct cifs_ses *ses = tcon->ses; - LOCK_REQ *pSMB = (LOCK_REQ *)in_buf; - - /* We just modify the current in_buf to change - the type of lock from LOCKING_ANDX_SHARED_LOCK - or LOCKING_ANDX_EXCLUSIVE_LOCK to - LOCKING_ANDX_CANCEL_LOCK. */ - - pSMB->LockType = LOCKING_ANDX_CANCEL_LOCK|LOCKING_ANDX_LARGE_FILES; - pSMB->Timeout = 0; - pSMB->hdr.Mid = get_next_mid(ses->server); - - return SendReceive(xid, ses, in_buf, out_buf, - &bytes_returned, 0); -} - -int -SendReceiveBlockingLock(const unsigned int xid, struct cifs_tcon *tcon, - struct smb_hdr *in_buf, struct smb_hdr *out_buf, - int *pbytes_returned) -{ - int rc = 0; - int rstart = 0; - struct mid_q_entry *midQ; - struct cifs_ses *ses; - unsigned int len = be32_to_cpu(in_buf->smb_buf_length); - struct kvec iov = { .iov_base = in_buf, .iov_len = len }; - struct smb_rqst rqst = { .rq_iov = &iov, .rq_nvec = 1 }; - unsigned int instance; - struct TCP_Server_Info *server; - - if (tcon == NULL || tcon->ses == NULL) { - cifs_dbg(VFS, "Null smb session\n"); - return -EIO; - } - ses = tcon->ses; - server = ses->server; - - if (server == NULL) { - cifs_dbg(VFS, "Null tcp session\n"); - return -EIO; - } - - spin_lock(&server->srv_lock); - if (server->tcpStatus == CifsExiting) { - spin_unlock(&server->srv_lock); - return -ENOENT; - } - spin_unlock(&server->srv_lock); - - /* Ensure that we do not send more than 50 overlapping requests - to the same server. We may make this configurable later or - use ses->maxReq */ - - if (len > CIFSMaxBufSize + MAX_CIFS_HDR_SIZE - 4) { - cifs_tcon_dbg(VFS, "Invalid length, greater than maximum frame, %d\n", - len); - return -EIO; - } - - rc = wait_for_free_request(server, CIFS_BLOCKING_OP, &instance); - if (rc) - return rc; - - /* make sure that we sign in the same order that we send on this socket - and avoid races inside tcp sendmsg code that could cause corruption - of smb data */ - - cifs_server_lock(server); - - rc = allocate_mid(ses, in_buf, &midQ); - if (rc) { - cifs_server_unlock(server); - return rc; - } - - rc = cifs_sign_smb(in_buf, server, &midQ->sequence_number); - if (rc) { - delete_mid(midQ); - cifs_server_unlock(server); - return rc; - } - - midQ->mid_state = MID_REQUEST_SUBMITTED; - rc = smb_send(server, in_buf, len); - cifs_save_when_sent(midQ); - - if (rc < 0) - server->sequence_number -= 2; - - cifs_server_unlock(server); - - if (rc < 0) { - delete_mid(midQ); - return rc; - } - - /* Wait for a reply - allow signals to interrupt. */ - rc = wait_event_interruptible(server->response_q, - (!(midQ->mid_state == MID_REQUEST_SUBMITTED || - midQ->mid_state == MID_RESPONSE_RECEIVED)) || - ((server->tcpStatus != CifsGood) && - (server->tcpStatus != CifsNew))); - - /* Were we interrupted by a signal ? */ - spin_lock(&server->srv_lock); - if ((rc == -ERESTARTSYS) && - (midQ->mid_state == MID_REQUEST_SUBMITTED || - midQ->mid_state == MID_RESPONSE_RECEIVED) && - ((server->tcpStatus == CifsGood) || - (server->tcpStatus == CifsNew))) { - spin_unlock(&server->srv_lock); - - if (in_buf->Command == SMB_COM_TRANSACTION2) { - /* POSIX lock. We send a NT_CANCEL SMB to cause the - blocking lock to return. */ - rc = send_cancel(server, &rqst, midQ); - if (rc) { - delete_mid(midQ); - return rc; - } - } else { - /* Windows lock. We send a LOCKINGX_CANCEL_LOCK - to cause the blocking lock to return. */ - - rc = send_lock_cancel(xid, tcon, in_buf, out_buf); - - /* If we get -ENOLCK back the lock may have - already been removed. Don't exit in this case. */ - if (rc && rc != -ENOLCK) { - delete_mid(midQ); - return rc; - } - } - - rc = wait_for_response(server, midQ); - if (rc) { - send_cancel(server, &rqst, midQ); - spin_lock(&server->mid_lock); - if (midQ->mid_state == MID_REQUEST_SUBMITTED || - midQ->mid_state == MID_RESPONSE_RECEIVED) { - /* no longer considered to be "in-flight" */ - midQ->callback = release_mid; - spin_unlock(&server->mid_lock); - return rc; - } - spin_unlock(&server->mid_lock); - } - - /* We got the response - restart system call. */ - rstart = 1; - spin_lock(&server->srv_lock); - } - spin_unlock(&server->srv_lock); - - rc = cifs_sync_mid_result(midQ, server); - if (rc != 0) - return rc; - - /* rcvd frame is ok */ - if (out_buf == NULL || midQ->mid_state != MID_RESPONSE_READY) { - rc = -EIO; - cifs_tcon_dbg(VFS, "Bad MID state?\n"); - goto out; - } - - *pbytes_returned = get_rfc1002_length(midQ->resp_buf); - memcpy(out_buf, midQ->resp_buf, *pbytes_returned + 4); - rc = cifs_check_receive(midQ, server, 0); -out: - delete_mid(midQ); - if (rstart && rc == -EACCES) - return -ERESTARTSYS; - return rc; -} /* * Discard any remaining data in the current SMB. To do this, we borrow the diff --git a/fs/smb/common/smbdirect/smbdirect_socket.h b/fs/smb/common/smbdirect/smbdirect_socket.h index e5b15cc44a7b..3c4a8d627aa3 100644 --- a/fs/smb/common/smbdirect/smbdirect_socket.h +++ b/fs/smb/common/smbdirect/smbdirect_socket.h @@ -38,6 +38,124 @@ struct smbdirect_socket { } ib; struct smbdirect_socket_parameters parameters; + + /* + * The state for posted send buffers + */ + struct { + /* + * Memory pools for preallocating + * smbdirect_send_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + } mem; + } send_io; + + /* + * The state for posted receive buffers + */ + struct { + /* + * The type of PDU we are expecting + */ + enum { + SMBDIRECT_EXPECT_NEGOTIATE_REQ = 1, + SMBDIRECT_EXPECT_NEGOTIATE_REP = 2, + SMBDIRECT_EXPECT_DATA_TRANSFER = 3, + } expected; + + /* + * Memory pools for preallocating + * smbdirect_recv_io buffers + */ + struct { + struct kmem_cache *cache; + mempool_t *pool; + } mem; + + /* + * The list of free smbdirect_recv_io + * structures + */ + struct { + struct list_head list; + spinlock_t lock; + } free; + + /* + * The list of arrived non-empty smbdirect_recv_io + * structures + * + * This represents the reassembly queue. + */ + struct { + struct list_head list; + spinlock_t lock; + wait_queue_head_t wait_queue; + /* total data length of reassembly queue */ + int data_length; + int queue_length; + /* the offset to first buffer in reassembly queue */ + int first_entry_offset; + /* + * Indicate if we have received a full packet on the + * connection This is used to identify the first SMBD + * packet of a assembled payload (SMB packet) in + * reassembly queue so we can return a RFC1002 length to + * upper layer to indicate the length of the SMB packet + * received + */ + bool full_packet_received; + } reassembly; + } recv_io; +}; + +struct smbdirect_send_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * The SGE entries for this work request + * + * The first points to the packet header + */ +#define SMBDIRECT_SEND_IO_MAX_SGE 6 + size_t num_sge; + struct ib_sge sge[SMBDIRECT_SEND_IO_MAX_SGE]; + + /* + * Link to the list of sibling smbdirect_send_io + * messages. + */ + struct list_head sibling_list; + struct ib_send_wr wr; + + /* SMBD packet header follows this structure */ + u8 packet[]; +}; + +struct smbdirect_recv_io { + struct smbdirect_socket *socket; + struct ib_cqe cqe; + + /* + * For now we only use a single SGE + * as we have just one large buffer + * per posted recv. + */ +#define SMBDIRECT_RECV_IO_MAX_SGE 1 + struct ib_sge sge; + + /* Link to free or reassembly list */ + struct list_head list; + + /* Indicate if this is the 1st packet of a payload */ + bool first_segment; + + /* SMBD packet header and payload follows this structure */ + u8 packet[]; }; #endif /* __FS_SMB_COMMON_SMBDIRECT_SMBDIRECT_SOCKET_H__ */ diff --git a/fs/smb/server/connection.h b/fs/smb/server/connection.h index dd3e0e3f7bf0..31dd1caac1e8 100644 --- a/fs/smb/server/connection.h +++ b/fs/smb/server/connection.h @@ -46,6 +46,7 @@ struct ksmbd_conn { struct mutex srv_mutex; int status; unsigned int cli_cap; + __be32 inet_addr; char *request_buf; struct ksmbd_transport *transport; struct nls_table *local_nls; diff --git a/fs/smb/server/smb_common.c b/fs/smb/server/smb_common.c index 425c756bcfb8..b23203a1c286 100644 --- a/fs/smb/server/smb_common.c +++ b/fs/smb/server/smb_common.c @@ -515,7 +515,7 @@ int ksmbd_extract_shortname(struct ksmbd_conn *conn, const char *longname, p = strrchr(longname, '.'); if (p == longname) { /*name starts with a dot*/ - strscpy(extension, "___", strlen("___")); + strscpy(extension, "___", sizeof(extension)); } else { if (p) { p++; diff --git a/fs/smb/server/transport_rdma.c b/fs/smb/server/transport_rdma.c index c6cbe0d56e32..8d366db5f605 100644 --- a/fs/smb/server/transport_rdma.c +++ b/fs/smb/server/transport_rdma.c @@ -129,9 +129,6 @@ struct smb_direct_transport { spinlock_t recvmsg_queue_lock; struct list_head recvmsg_queue; - spinlock_t empty_recvmsg_queue_lock; - struct list_head empty_recvmsg_queue; - int send_credit_target; atomic_t send_credits; spinlock_t lock_new_recv_credits; @@ -268,40 +265,19 @@ smb_direct_recvmsg *get_free_recvmsg(struct smb_direct_transport *t) static void put_recvmsg(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg) { - ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, - recvmsg->sge.length, DMA_FROM_DEVICE); + if (likely(recvmsg->sge.length != 0)) { + ib_dma_unmap_single(t->cm_id->device, + recvmsg->sge.addr, + recvmsg->sge.length, + DMA_FROM_DEVICE); + recvmsg->sge.length = 0; + } spin_lock(&t->recvmsg_queue_lock); list_add(&recvmsg->list, &t->recvmsg_queue); spin_unlock(&t->recvmsg_queue_lock); } -static struct -smb_direct_recvmsg *get_empty_recvmsg(struct smb_direct_transport *t) -{ - struct smb_direct_recvmsg *recvmsg = NULL; - - spin_lock(&t->empty_recvmsg_queue_lock); - if (!list_empty(&t->empty_recvmsg_queue)) { - recvmsg = list_first_entry(&t->empty_recvmsg_queue, - struct smb_direct_recvmsg, list); - list_del(&recvmsg->list); - } - spin_unlock(&t->empty_recvmsg_queue_lock); - return recvmsg; -} - -static void put_empty_recvmsg(struct smb_direct_transport *t, - struct smb_direct_recvmsg *recvmsg) -{ - ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, - recvmsg->sge.length, DMA_FROM_DEVICE); - - spin_lock(&t->empty_recvmsg_queue_lock); - list_add_tail(&recvmsg->list, &t->empty_recvmsg_queue); - spin_unlock(&t->empty_recvmsg_queue_lock); -} - static void enqueue_reassembly(struct smb_direct_transport *t, struct smb_direct_recvmsg *recvmsg, int data_length) @@ -386,9 +362,6 @@ static struct smb_direct_transport *alloc_transport(struct rdma_cm_id *cm_id) spin_lock_init(&t->recvmsg_queue_lock); INIT_LIST_HEAD(&t->recvmsg_queue); - spin_lock_init(&t->empty_recvmsg_queue_lock); - INIT_LIST_HEAD(&t->empty_recvmsg_queue); - init_waitqueue_head(&t->wait_send_pending); atomic_set(&t->send_pending, 0); @@ -548,13 +521,13 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) t = recvmsg->transport; if (wc->status != IB_WC_SUCCESS || wc->opcode != IB_WC_RECV) { + put_recvmsg(t, recvmsg); if (wc->status != IB_WC_WR_FLUSH_ERR) { pr_err("Recv error. status='%s (%d)' opcode=%d\n", ib_wc_status_msg(wc->status), wc->status, wc->opcode); smb_direct_disconnect_rdma_connection(t); } - put_empty_recvmsg(t, recvmsg); return; } @@ -568,7 +541,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) switch (recvmsg->type) { case SMB_DIRECT_MSG_NEGOTIATE_REQ: if (wc->byte_len < sizeof(struct smb_direct_negotiate_req)) { - put_empty_recvmsg(t, recvmsg); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); return; } t->negotiation_requested = true; @@ -576,7 +550,7 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) t->status = SMB_DIRECT_CS_CONNECTED; enqueue_reassembly(t, recvmsg, 0); wake_up_interruptible(&t->wait_status); - break; + return; case SMB_DIRECT_MSG_DATA_TRANSFER: { struct smb_direct_data_transfer *data_transfer = (struct smb_direct_data_transfer *)recvmsg->packet; @@ -585,7 +559,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) if (wc->byte_len < offsetof(struct smb_direct_data_transfer, padding)) { - put_empty_recvmsg(t, recvmsg); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); return; } @@ -593,7 +568,8 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) if (data_length) { if (wc->byte_len < sizeof(struct smb_direct_data_transfer) + (u64)data_length) { - put_empty_recvmsg(t, recvmsg); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); return; } @@ -605,16 +581,11 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) else t->full_packet_received = true; - enqueue_reassembly(t, recvmsg, (int)data_length); - wake_up_interruptible(&t->wait_reassembly_queue); - spin_lock(&t->receive_credit_lock); receive_credits = --(t->recv_credits); avail_recvmsg_count = t->count_avail_recvmsg; spin_unlock(&t->receive_credit_lock); } else { - put_empty_recvmsg(t, recvmsg); - spin_lock(&t->receive_credit_lock); receive_credits = --(t->recv_credits); avail_recvmsg_count = ++(t->count_avail_recvmsg); @@ -636,11 +607,23 @@ static void recv_done(struct ib_cq *cq, struct ib_wc *wc) if (is_receive_credit_post_required(receive_credits, avail_recvmsg_count)) mod_delayed_work(smb_direct_wq, &t->post_recv_credits_work, 0); - break; + + if (data_length) { + enqueue_reassembly(t, recvmsg, (int)data_length); + wake_up_interruptible(&t->wait_reassembly_queue); + } else + put_recvmsg(t, recvmsg); + + return; } - default: - break; } + + /* + * This is an internal error! + */ + WARN_ON_ONCE(recvmsg->type != SMB_DIRECT_MSG_DATA_TRANSFER); + put_recvmsg(t, recvmsg); + smb_direct_disconnect_rdma_connection(t); } static int smb_direct_post_recv(struct smb_direct_transport *t, @@ -670,6 +653,7 @@ static int smb_direct_post_recv(struct smb_direct_transport *t, ib_dma_unmap_single(t->cm_id->device, recvmsg->sge.addr, recvmsg->sge.length, DMA_FROM_DEVICE); + recvmsg->sge.length = 0; smb_direct_disconnect_rdma_connection(t); return ret; } @@ -811,7 +795,6 @@ static void smb_direct_post_recv_credits(struct work_struct *work) struct smb_direct_recvmsg *recvmsg; int receive_credits, credits = 0; int ret; - int use_free = 1; spin_lock(&t->receive_credit_lock); receive_credits = t->recv_credits; @@ -819,18 +802,9 @@ static void smb_direct_post_recv_credits(struct work_struct *work) if (receive_credits < t->recv_credit_target) { while (true) { - if (use_free) - recvmsg = get_free_recvmsg(t); - else - recvmsg = get_empty_recvmsg(t); - if (!recvmsg) { - if (use_free) { - use_free = 0; - continue; - } else { - break; - } - } + recvmsg = get_free_recvmsg(t); + if (!recvmsg) + break; recvmsg->type = SMB_DIRECT_MSG_DATA_TRANSFER; recvmsg->first_segment = false; @@ -1806,8 +1780,6 @@ static void smb_direct_destroy_pools(struct smb_direct_transport *t) while ((recvmsg = get_free_recvmsg(t))) mempool_free(recvmsg, t->recvmsg_mempool); - while ((recvmsg = get_empty_recvmsg(t))) - mempool_free(recvmsg, t->recvmsg_mempool); mempool_destroy(t->recvmsg_mempool); t->recvmsg_mempool = NULL; @@ -1863,6 +1835,7 @@ static int smb_direct_create_pools(struct smb_direct_transport *t) if (!recvmsg) goto err; recvmsg->transport = t; + recvmsg->sge.length = 0; list_add(&recvmsg->list, &t->recvmsg_queue); } t->count_avail_recvmsg = t->recv_credit_max; diff --git a/fs/smb/server/transport_tcp.c b/fs/smb/server/transport_tcp.c index f8c772a7cb43..b1df02e321b0 100644 --- a/fs/smb/server/transport_tcp.c +++ b/fs/smb/server/transport_tcp.c @@ -85,6 +85,7 @@ static struct tcp_transport *alloc_transport(struct socket *client_sk) return NULL; } + conn->inet_addr = inet_sk(client_sk->sk)->inet_daddr; conn->transport = KSMBD_TRANS(t); KSMBD_TRANS(t)->conn = conn; KSMBD_TRANS(t)->ops = &ksmbd_tcp_transport_ops; @@ -228,6 +229,8 @@ static int ksmbd_kthread_fn(void *p) { struct socket *client_sk = NULL; struct interface *iface = (struct interface *)p; + struct inet_sock *csk_inet; + struct ksmbd_conn *conn; int ret; while (!kthread_should_stop()) { @@ -246,6 +249,20 @@ static int ksmbd_kthread_fn(void *p) continue; } + /* + * Limits repeated connections from clients with the same IP. + */ + csk_inet = inet_sk(client_sk->sk); + down_read(&conn_list_lock); + list_for_each_entry(conn, &conn_list, conns_list) + if (csk_inet->inet_daddr == conn->inet_addr) { + ret = -EAGAIN; + break; + } + up_read(&conn_list_lock); + if (ret == -EAGAIN) + continue; + if (server_conf.max_connections && atomic_inc_return(&active_num_conn) >= server_conf.max_connections) { pr_info_ratelimited("Limit the maximum number of connections(%u)\n", diff --git a/include/acpi/pcc.h b/include/acpi/pcc.h index 840bfc95bae3..9af3b502f839 100644 --- a/include/acpi/pcc.h +++ b/include/acpi/pcc.h @@ -17,6 +17,35 @@ struct pcc_mbox_chan { u32 latency; u32 max_access_rate; u16 min_turnaround_time; + + /* Set to true to indicate that the mailbox should manage + * writing the dat to the shared buffer. This differs from + * the case where the drivesr are writing to the buffer and + * using send_data only to ring the doorbell. If this flag + * is set, then the void * data parameter of send_data must + * point to a kernel-memory buffer formatted in accordance with + * the PCC specification. + * + * The active buffer management will include reading the + * notify_on_completion flag, and will then + * call mbox_chan_txdone when the acknowledgment interrupt is + * received. + */ + bool manage_writes; + + /* Optional callback that allows the driver + * to allocate the memory used for receiving + * messages. The return value is the location + * inside the buffer where the mailbox should write the data. + */ + void *(*rx_alloc)(struct mbox_client *cl, int size); +}; + +struct pcc_header { + u32 signature; + u32 flags; + u32 length; + u32 command; }; /* Generic Communications Channel Shared Memory Region */ diff --git a/include/crypto/hash.h b/include/crypto/hash.h index db294d452e8c..bbaeae705ef0 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -184,7 +184,7 @@ struct shash_desc { * Worst case is hmac(sha3-224-s390). Its context is a nested 'shash_desc' * containing a 'struct s390_sha_ctx'. */ -#define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 360) +#define HASH_MAX_DESCSIZE (sizeof(struct shash_desc) + 361) #define MAX_SYNC_HASH_REQSIZE (sizeof(struct ahash_request) + \ HASH_MAX_DESCSIZE) diff --git a/include/linux/efi.h b/include/linux/efi.h index e3776d9cad07..a98cc39e7aaa 100644 --- a/include/linux/efi.h +++ b/include/linux/efi.h @@ -439,6 +439,7 @@ void efi_native_runtime_setup(void); /* OVMF protocol GUIDs */ #define OVMF_SEV_MEMORY_ACCEPTANCE_PROTOCOL_GUID EFI_GUID(0xc5a010fe, 0x38a7, 0x4531, 0x8a, 0x4a, 0x05, 0x00, 0xd2, 0xfd, 0x16, 0x49) +#define OVMF_MEMORY_LOG_TABLE_GUID EFI_GUID(0x95305139, 0xb20f, 0x4723, 0x84, 0x25, 0x62, 0x7c, 0x88, 0x8f, 0xf1, 0x21) typedef struct { efi_guid_t guid; @@ -642,6 +643,7 @@ extern struct efi { unsigned long esrt; /* ESRT table */ unsigned long tpm_log; /* TPM2 Event Log table */ unsigned long tpm_final_log; /* TPM2 Final Events Log table */ + unsigned long ovmf_debug_log; unsigned long mokvar_table; /* MOK variable config table */ unsigned long coco_secret; /* Confidential computing secret table */ unsigned long unaccepted; /* Unaccepted memory table */ @@ -1344,6 +1346,8 @@ bool efi_config_table_is_usable(const efi_guid_t *guid, unsigned long table) umode_t efi_attr_is_visible(struct kobject *kobj, struct attribute *attr, int n); +int ovmf_log_probe(unsigned long ovmf_debug_log_table); + /* * efivar ops event type */ diff --git a/include/linux/ioprio.h b/include/linux/ioprio.h index b25377b6ea98..5210e8371238 100644 --- a/include/linux/ioprio.h +++ b/include/linux/ioprio.h @@ -60,7 +60,8 @@ static inline int __get_task_ioprio(struct task_struct *p) int prio; if (!ioc) - return IOPRIO_DEFAULT; + return IOPRIO_PRIO_VALUE(task_nice_ioclass(p), + task_nice_ioprio(p)); if (p != current) lockdep_assert_held(&p->alloc_lock); diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 67ae2c3f41d2..c585939b6cd6 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -161,6 +161,12 @@ struct nfs_inode { unsigned long cache_validity; /* bit mask */ /* + * NFS Attributes not included in struct inode + */ + + struct timespec64 btime; + + /* * read_cache_jiffies is when we started read-caching this inode. * attrtimeo is for how long the cached information is assumed * to be valid. A successful attribute revalidation doubles @@ -316,10 +322,12 @@ struct nfs4_copy_state { #define NFS_INO_INVALID_XATTR BIT(15) /* xattrs are invalid */ #define NFS_INO_INVALID_NLINK BIT(16) /* cached nlinks is invalid */ #define NFS_INO_INVALID_MODE BIT(17) /* cached mode is invalid */ +#define NFS_INO_INVALID_BTIME BIT(18) /* cached btime is invalid */ #define NFS_INO_INVALID_ATTR (NFS_INO_INVALID_CHANGE \ | NFS_INO_INVALID_CTIME \ | NFS_INO_INVALID_MTIME \ + | NFS_INO_INVALID_BTIME \ | NFS_INO_INVALID_SIZE \ | NFS_INO_INVALID_NLINK \ | NFS_INO_INVALID_MODE \ diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h index 63141320c2a8..d30c0245031c 100644 --- a/include/linux/nfs_fs_sb.h +++ b/include/linux/nfs_fs_sb.h @@ -172,12 +172,11 @@ struct nfs_server { #define NFS_MOUNT_FORCE_RDIRPLUS 0x20000000 #define NFS_MOUNT_NETUNREACH_FATAL 0x40000000 - unsigned int fattr_valid; /* Valid attributes */ unsigned int caps; /* server capabilities */ + __u64 fattr_valid; /* Valid attributes */ unsigned int rsize; /* read size */ unsigned int rpages; /* read size (in pages) */ unsigned int wsize; /* write size */ - unsigned int wpages; /* write size (in pages) */ unsigned int wtmult; /* server disk block size */ unsigned int dtsize; /* readdir size */ unsigned short port; /* "port=" setting */ @@ -203,7 +202,6 @@ struct nfs_server { struct nfs_fsid fsid; int s_sysfs_id; /* sysfs dentry index */ __u64 maxfilesize; /* maximum file size */ - struct timespec64 time_delta; /* smallest time granularity */ unsigned long mount_time; /* when this fs was mounted */ struct super_block *super; /* VFS super block */ dev_t s_dev; /* superblock dev numbers */ @@ -248,7 +246,6 @@ struct nfs_server { filesystem */ struct pnfs_layoutdriver_type *pnfs_curr_ld; /* Active layout driver */ struct rpc_wait_queue roc_rpcwaitq; - void *pnfs_ld_data; /* per mount point data */ /* the following fields are protected by nfs_client->cl_lock */ struct rb_root state_owners; @@ -257,6 +254,9 @@ struct nfs_server { struct list_head state_owners_lru; struct list_head layouts; struct list_head delegations; + atomic_long_t nr_active_delegations; + unsigned int delegation_hash_mask; + struct hlist_head *delegation_hash_table; struct list_head ss_copies; struct list_head ss_src_copies; diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index 67f6632f723b..ac4bff6e9913 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -45,7 +45,7 @@ struct nfs4_threshold { }; struct nfs_fattr { - unsigned int valid; /* which fields are valid */ + __u64 valid; /* which fields are valid */ umode_t mode; __u32 nlink; kuid_t uid; @@ -67,6 +67,7 @@ struct nfs_fattr { struct timespec64 atime; struct timespec64 mtime; struct timespec64 ctime; + struct timespec64 btime; __u64 change_attr; /* NFSv4 change attribute */ __u64 pre_change_attr;/* pre-op NFSv4 change attribute */ __u64 pre_size; /* pre_op_attr.size */ @@ -80,32 +81,33 @@ struct nfs_fattr { struct nfs4_label *label; }; -#define NFS_ATTR_FATTR_TYPE (1U << 0) -#define NFS_ATTR_FATTR_MODE (1U << 1) -#define NFS_ATTR_FATTR_NLINK (1U << 2) -#define NFS_ATTR_FATTR_OWNER (1U << 3) -#define NFS_ATTR_FATTR_GROUP (1U << 4) -#define NFS_ATTR_FATTR_RDEV (1U << 5) -#define NFS_ATTR_FATTR_SIZE (1U << 6) -#define NFS_ATTR_FATTR_PRESIZE (1U << 7) -#define NFS_ATTR_FATTR_BLOCKS_USED (1U << 8) -#define NFS_ATTR_FATTR_SPACE_USED (1U << 9) -#define NFS_ATTR_FATTR_FSID (1U << 10) -#define NFS_ATTR_FATTR_FILEID (1U << 11) -#define NFS_ATTR_FATTR_ATIME (1U << 12) -#define NFS_ATTR_FATTR_MTIME (1U << 13) -#define NFS_ATTR_FATTR_CTIME (1U << 14) -#define NFS_ATTR_FATTR_PREMTIME (1U << 15) -#define NFS_ATTR_FATTR_PRECTIME (1U << 16) -#define NFS_ATTR_FATTR_CHANGE (1U << 17) -#define NFS_ATTR_FATTR_PRECHANGE (1U << 18) -#define NFS_ATTR_FATTR_V4_LOCATIONS (1U << 19) -#define NFS_ATTR_FATTR_V4_REFERRAL (1U << 20) -#define NFS_ATTR_FATTR_MOUNTPOINT (1U << 21) -#define NFS_ATTR_FATTR_MOUNTED_ON_FILEID (1U << 22) -#define NFS_ATTR_FATTR_OWNER_NAME (1U << 23) -#define NFS_ATTR_FATTR_GROUP_NAME (1U << 24) -#define NFS_ATTR_FATTR_V4_SECURITY_LABEL (1U << 25) +#define NFS_ATTR_FATTR_TYPE BIT_ULL(0) +#define NFS_ATTR_FATTR_MODE BIT_ULL(1) +#define NFS_ATTR_FATTR_NLINK BIT_ULL(2) +#define NFS_ATTR_FATTR_OWNER BIT_ULL(3) +#define NFS_ATTR_FATTR_GROUP BIT_ULL(4) +#define NFS_ATTR_FATTR_RDEV BIT_ULL(5) +#define NFS_ATTR_FATTR_SIZE BIT_ULL(6) +#define NFS_ATTR_FATTR_PRESIZE BIT_ULL(7) +#define NFS_ATTR_FATTR_BLOCKS_USED BIT_ULL(8) +#define NFS_ATTR_FATTR_SPACE_USED BIT_ULL(9) +#define NFS_ATTR_FATTR_FSID BIT_ULL(10) +#define NFS_ATTR_FATTR_FILEID BIT_ULL(11) +#define NFS_ATTR_FATTR_ATIME BIT_ULL(12) +#define NFS_ATTR_FATTR_MTIME BIT_ULL(13) +#define NFS_ATTR_FATTR_CTIME BIT_ULL(14) +#define NFS_ATTR_FATTR_PREMTIME BIT_ULL(15) +#define NFS_ATTR_FATTR_PRECTIME BIT_ULL(16) +#define NFS_ATTR_FATTR_CHANGE BIT_ULL(17) +#define NFS_ATTR_FATTR_PRECHANGE BIT_ULL(18) +#define NFS_ATTR_FATTR_V4_LOCATIONS BIT_ULL(19) +#define NFS_ATTR_FATTR_V4_REFERRAL BIT_ULL(20) +#define NFS_ATTR_FATTR_MOUNTPOINT BIT_ULL(21) +#define NFS_ATTR_FATTR_MOUNTED_ON_FILEID BIT_ULL(22) +#define NFS_ATTR_FATTR_OWNER_NAME BIT_ULL(23) +#define NFS_ATTR_FATTR_GROUP_NAME BIT_ULL(24) +#define NFS_ATTR_FATTR_V4_SECURITY_LABEL BIT_ULL(25) +#define NFS_ATTR_FATTR_BTIME BIT_ULL(26) #define NFS_ATTR_FATTR (NFS_ATTR_FATTR_TYPE \ | NFS_ATTR_FATTR_MODE \ @@ -126,6 +128,7 @@ struct nfs_fattr { | NFS_ATTR_FATTR_SPACE_USED) #define NFS_ATTR_FATTR_V4 (NFS_ATTR_FATTR \ | NFS_ATTR_FATTR_SPACE_USED \ + | NFS_ATTR_FATTR_BTIME \ | NFS_ATTR_FATTR_V4_SECURITY_LABEL) /* diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h index 189140bf11fc..ffb9907c7070 100644 --- a/include/linux/sbitmap.h +++ b/include/linux/sbitmap.h @@ -210,23 +210,6 @@ void sbitmap_resize(struct sbitmap *sb, unsigned int depth); int sbitmap_get(struct sbitmap *sb); /** - * sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap, - * limiting the depth used from each word. - * @sb: Bitmap to allocate from. - * @shallow_depth: The maximum number of bits to allocate from a single word. - * - * This rather specific operation allows for having multiple users with - * different allocation limits. E.g., there can be a high-priority class that - * uses sbitmap_get() and a low-priority class that uses sbitmap_get_shallow() - * with a @shallow_depth of (1 << (@sb->shift - 1)). Then, the low-priority - * class can only allocate half of the total bits in the bitmap, preventing it - * from starving out the high-priority class. - * - * Return: Non-negative allocated bit number if successful, -1 otherwise. - */ -int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth); - -/** * sbitmap_any_bit_set() - Check for a set bit in a &struct sbitmap. * @sb: Bitmap to check. * @@ -478,7 +461,7 @@ unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags, * sbitmap_queue, limiting the depth used from each word, with preemption * already disabled. * @sbq: Bitmap queue to allocate from. - * @shallow_depth: The maximum number of bits to allocate from a single word. + * @shallow_depth: The maximum number of bits to allocate from the queue. * See sbitmap_get_shallow(). * * If you call this, make sure to call sbitmap_queue_min_shallow_depth() after diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index b8b06e71b73e..14b923ddb6df 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -3033,6 +3033,29 @@ static inline void skb_reset_transport_header(struct sk_buff *skb) skb->transport_header = offset; } +/** + * skb_reset_transport_header_careful - conditionally reset transport header + * @skb: buffer to alter + * + * Hardened version of skb_reset_transport_header(). + * + * Returns: true if the operation was a success. + */ +static inline bool __must_check +skb_reset_transport_header_careful(struct sk_buff *skb) +{ + long offset = skb->data - skb->head; + + if (unlikely(offset != (typeof(skb->transport_header))offset)) + return false; + + if (unlikely(offset == (typeof(skb->transport_header))~0U)) + return false; + + skb->transport_header = offset; + return true; +} + static inline void skb_set_transport_header(struct sk_buff *skb, const int offset) { diff --git a/include/linux/sunrpc/xdr.h b/include/linux/sunrpc/xdr.h index e3358c630ba1..8a9ec617cf66 100644 --- a/include/linux/sunrpc/xdr.h +++ b/include/linux/sunrpc/xdr.h @@ -130,10 +130,7 @@ xdr_buf_init(struct xdr_buf *buf, void *start, size_t len) __be32 *xdr_encode_opaque_fixed(__be32 *p, const void *ptr, unsigned int len); __be32 *xdr_encode_opaque(__be32 *p, const void *ptr, unsigned int len); __be32 *xdr_encode_string(__be32 *p, const char *s); -__be32 *xdr_decode_string_inplace(__be32 *p, char **sp, unsigned int *lenp, - unsigned int maxlen); __be32 *xdr_encode_netobj(__be32 *p, const struct xdr_netobj *); -__be32 *xdr_decode_netobj(__be32 *p, struct xdr_netobj *); void xdr_inline_pages(struct xdr_buf *, unsigned int, struct page **, unsigned int, unsigned int); @@ -342,12 +339,6 @@ xdr_stream_remaining(const struct xdr_stream *xdr) return xdr->nwords << 2; } -ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, - size_t size); -ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr, - size_t maxlen, gfp_t gfp_flags); -ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, - size_t size); ssize_t xdr_stream_decode_string_dup(struct xdr_stream *xdr, char **str, size_t maxlen, gfp_t gfp_flags); ssize_t xdr_stream_decode_opaque_auth(struct xdr_stream *xdr, u32 *flavor, diff --git a/include/net/dst.h b/include/net/dst.h index 00467c1b5093..bab01363bb97 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -568,11 +568,23 @@ static inline struct net_device *dst_dev(const struct dst_entry *dst) return READ_ONCE(dst->dev); } +static inline struct net_device *dst_dev_rcu(const struct dst_entry *dst) +{ + /* In the future, use rcu_dereference(dst->dev) */ + WARN_ON_ONCE(!rcu_read_lock_held()); + return READ_ONCE(dst->dev); +} + static inline struct net_device *skb_dst_dev(const struct sk_buff *skb) { return dst_dev(skb_dst(skb)); } +static inline struct net_device *skb_dst_dev_rcu(const struct sk_buff *skb) +{ + return dst_dev_rcu(skb_dst(skb)); +} + static inline struct net *skb_dst_dev_net(const struct sk_buff *skb) { return dev_net(skb_dst_dev(skb)); diff --git a/include/net/udp.h b/include/net/udp.h index f8ae2c4ade14..e2af3bda90c9 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -586,6 +586,16 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, { netdev_features_t features = NETIF_F_SG; struct sk_buff *segs; + int drop_count; + + /* + * Segmentation in UDP receive path is only for UDP GRO, drop udp + * fragmentation offload (UFO) packets. + */ + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP) { + drop_count = 1; + goto drop; + } /* Avoid csum recalculation by skb_segment unless userspace explicitly * asks for the final checksum values @@ -609,16 +619,18 @@ static inline struct sk_buff *udp_rcv_segment(struct sock *sk, */ segs = __skb_gso_segment(skb, features, false); if (IS_ERR_OR_NULL(segs)) { - int segs_nr = skb_shinfo(skb)->gso_segs; - - atomic_add(segs_nr, &sk->sk_drops); - SNMP_ADD_STATS(__UDPX_MIB(sk, ipv4), UDP_MIB_INERRORS, segs_nr); - kfree_skb(skb); - return NULL; + drop_count = skb_shinfo(skb)->gso_segs; + goto drop; } consume_skb(skb); return segs; + +drop: + atomic_add(drop_count, &sk->sk_drops); + SNMP_ADD_STATS(__UDPX_MIB(sk, ipv4), UDP_MIB_INERRORS, drop_count); + kfree_skb(skb); + return NULL; } static inline void udp_post_segment_fix_csum(struct sk_buff *skb) diff --git a/include/sound/tas2781-tlv.h b/include/sound/tas2781-tlv.h index d87263e43fdb..ef9b9f19d212 100644 --- a/include/sound/tas2781-tlv.h +++ b/include/sound/tas2781-tlv.h @@ -15,7 +15,7 @@ #ifndef __TAS2781_TLV_H__ #define __TAS2781_TLV_H__ -static const __maybe_unused DECLARE_TLV_DB_SCALE(dvc_tlv, -10000, 100, 0); +static const __maybe_unused DECLARE_TLV_DB_SCALE(dvc_tlv, -10000, 50, 0); static const __maybe_unused DECLARE_TLV_DB_SCALE(amp_vol_tlv, 1100, 50, 0); #endif diff --git a/include/uapi/linux/io_uring.h b/include/uapi/linux/io_uring.h index b8a0e70ee2fd..6957dc539d83 100644 --- a/include/uapi/linux/io_uring.h +++ b/include/uapi/linux/io_uring.h @@ -392,12 +392,16 @@ enum io_uring_op { * the starting buffer ID in cqe->flags as per * usual for provided buffer usage. The buffers * will be contiguous from the starting buffer ID. + * + * IORING_SEND_VECTORIZED If set, SEND[_ZC] will take a pointer to a io_vec + * to allow vectorized send operations. */ #define IORING_RECVSEND_POLL_FIRST (1U << 0) #define IORING_RECV_MULTISHOT (1U << 1) #define IORING_RECVSEND_FIXED_BUF (1U << 2) #define IORING_SEND_ZC_REPORT_USAGE (1U << 3) #define IORING_RECVSEND_BUNDLE (1U << 4) +#define IORING_SEND_VECTORIZED (1U << 5) /* * cqe.res for IORING_CQE_F_NOTIF if diff --git a/include/uapi/linux/raid/md_p.h b/include/uapi/linux/raid/md_p.h index ff47b6f0ba0f..b13946287277 100644 --- a/include/uapi/linux/raid/md_p.h +++ b/include/uapi/linux/raid/md_p.h @@ -173,7 +173,7 @@ typedef struct mdp_superblock_s { #else #error unspecified endianness #endif - __u32 recovery_cp; /* 11 recovery checkpoint sector count */ + __u32 resync_offset; /* 11 resync checkpoint sector count */ /* There are only valid for minor_version > 90 */ __u64 reshape_position; /* 12,13 next address in array-space for reshape */ __u32 new_level; /* 14 new level we are reshaping to */ diff --git a/include/uapi/linux/vt.h b/include/uapi/linux/vt.h index b60fcdfb2746..714483d68c69 100644 --- a/include/uapi/linux/vt.h +++ b/include/uapi/linux/vt.h @@ -14,9 +14,9 @@ /* Note: the ioctl VT_GETSTATE does not work for consoles 16 and higher (since it returns a short) */ -/* 'V' to avoid collision with termios and kd */ +/* 0x56 is 'V', to avoid collision with termios and kd */ -#define VT_OPENQRY _IO('V', 0x00) /* find available vt */ +#define VT_OPENQRY 0x5600 /* find available vt */ struct vt_mode { __u8 mode; /* vt mode */ @@ -25,8 +25,8 @@ struct vt_mode { __s16 acqsig; /* signal to raise on acquisition */ __s16 frsig; /* unused (set to 0) */ }; -#define VT_GETMODE _IO('V', 0x01) /* get mode of active vt */ -#define VT_SETMODE _IO('V', 0x02) /* set mode of active vt */ +#define VT_GETMODE 0x5601 /* get mode of active vt */ +#define VT_SETMODE 0x5602 /* set mode of active vt */ #define VT_AUTO 0x00 /* auto vt switching */ #define VT_PROCESS 0x01 /* process controls switching */ #define VT_ACKACQ 0x02 /* acknowledge switch */ @@ -36,21 +36,21 @@ struct vt_stat { __u16 v_signal; /* signal to send */ __u16 v_state; /* vt bitmask */ }; -#define VT_GETSTATE _IO('V', 0x03) /* get global vt state info */ -#define VT_SENDSIG _IO('V', 0x04) /* signal to send to bitmask of vts */ +#define VT_GETSTATE 0x5603 /* get global vt state info */ +#define VT_SENDSIG 0x5604 /* signal to send to bitmask of vts */ -#define VT_RELDISP _IO('V', 0x05) /* release display */ +#define VT_RELDISP 0x5605 /* release display */ -#define VT_ACTIVATE _IO('V', 0x06) /* make vt active */ -#define VT_WAITACTIVE _IO('V', 0x07) /* wait for vt active */ -#define VT_DISALLOCATE _IO('V', 0x08) /* free memory associated to vt */ +#define VT_ACTIVATE 0x5606 /* make vt active */ +#define VT_WAITACTIVE 0x5607 /* wait for vt active */ +#define VT_DISALLOCATE 0x5608 /* free memory associated to vt */ struct vt_sizes { __u16 v_rows; /* number of rows */ __u16 v_cols; /* number of columns */ __u16 v_scrollsize; /* number of lines of scrollback */ }; -#define VT_RESIZE _IO('V', 0x09) /* set kernel's idea of screensize */ +#define VT_RESIZE 0x5609 /* set kernel's idea of screensize */ struct vt_consize { __u16 v_rows; /* number of rows */ @@ -60,10 +60,10 @@ struct vt_consize { __u16 v_vcol; /* number of pixel columns on screen */ __u16 v_ccol; /* number of pixel columns per character */ }; -#define VT_RESIZEX _IO('V', 0x0A) /* set kernel's idea of screensize + more */ -#define VT_LOCKSWITCH _IO('V', 0x0B) /* disallow vt switching */ -#define VT_UNLOCKSWITCH _IO('V', 0x0C) /* allow vt switching */ -#define VT_GETHIFONTMASK _IO('V', 0x0D) /* return hi font mask */ +#define VT_RESIZEX 0x560A /* set kernel's idea of screensize + more */ +#define VT_LOCKSWITCH 0x560B /* disallow vt switching */ +#define VT_UNLOCKSWITCH 0x560C /* allow vt switching */ +#define VT_GETHIFONTMASK 0x560D /* return hi font mask */ struct vt_event { __u32 event; @@ -77,14 +77,14 @@ struct vt_event { __u32 pad[4]; /* Padding for expansion */ }; -#define VT_WAITEVENT _IO('V', 0x0E) /* Wait for an event */ +#define VT_WAITEVENT 0x560E /* Wait for an event */ struct vt_setactivate { __u32 console; struct vt_mode mode; }; -#define VT_SETACTIVATE _IO('V', 0x0F) /* Activate and set the mode of a console */ +#define VT_SETACTIVATE 0x560F /* Activate and set the mode of a console */ /* get console size and cursor position */ struct vt_consizecsrpos { diff --git a/io_uring/memmap.c b/io_uring/memmap.c index 725dc0bec24c..2e99dffddfc5 100644 --- a/io_uring/memmap.c +++ b/io_uring/memmap.c @@ -156,7 +156,7 @@ static int io_region_allocate_pages(struct io_ring_ctx *ctx, unsigned long mmap_offset) { gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN; - unsigned long size = mr->nr_pages << PAGE_SHIFT; + size_t size = (size_t) mr->nr_pages << PAGE_SHIFT; unsigned long nr_allocated; struct page **pages; void *p; diff --git a/io_uring/net.c b/io_uring/net.c index 35585bdc59f3..dd96e355982f 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -382,6 +382,10 @@ static int io_send_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe) } if (req->flags & REQ_F_BUFFER_SELECT) return 0; + + if (sr->flags & IORING_SEND_VECTORIZED) + return io_net_import_vec(req, kmsg, sr->buf, sr->len, ITER_SOURCE); + return import_ubuf(ITER_SOURCE, sr->buf, sr->len, &kmsg->msg.msg_iter); } @@ -409,7 +413,7 @@ static int io_sendmsg_setup(struct io_kiocb *req, const struct io_uring_sqe *sqe return io_net_import_vec(req, kmsg, msg.msg_iov, msg.msg_iovlen, ITER_SOURCE); } -#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE) +#define SENDMSG_FLAGS (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_BUNDLE | IORING_SEND_VECTORIZED) int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { @@ -1318,7 +1322,8 @@ void io_send_zc_cleanup(struct io_kiocb *req) } #define IO_ZC_FLAGS_COMMON (IORING_RECVSEND_POLL_FIRST | IORING_RECVSEND_FIXED_BUF) -#define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE) +#define IO_ZC_FLAGS_VALID (IO_ZC_FLAGS_COMMON | IORING_SEND_ZC_REPORT_USAGE | \ + IORING_SEND_VECTORIZED) int io_send_zc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 0806295945e4..c4f69a9e9af6 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -23114,6 +23114,8 @@ static void free_states(struct bpf_verifier_env *env) for (i = 0; i < env->scc_cnt; ++i) { info = env->scc_info[i]; + if (!info) + continue; for (j = 0; j < info->num_visits; j++) free_backedges(&info->visits[j]); kvfree(info); @@ -24554,6 +24556,7 @@ dfs_continue: err = -ENOMEM; goto exit; } + env->scc_cnt = next_scc_id; exit: kvfree(stack); kvfree(pre); diff --git a/lib/sbitmap.c b/lib/sbitmap.c index d3412984170c..4d188d05db15 100644 --- a/lib/sbitmap.c +++ b/lib/sbitmap.c @@ -208,8 +208,28 @@ static int sbitmap_find_bit_in_word(struct sbitmap_word *map, return nr; } +static unsigned int __map_depth_with_shallow(const struct sbitmap *sb, + int index, + unsigned int shallow_depth) +{ + u64 shallow_word_depth; + unsigned int word_depth, reminder; + + word_depth = __map_depth(sb, index); + if (shallow_depth >= sb->depth) + return word_depth; + + shallow_word_depth = word_depth * shallow_depth; + reminder = do_div(shallow_word_depth, sb->depth); + + if (reminder >= (index + 1) * word_depth) + shallow_word_depth++; + + return (unsigned int)shallow_word_depth; +} + static int sbitmap_find_bit(struct sbitmap *sb, - unsigned int depth, + unsigned int shallow_depth, unsigned int index, unsigned int alloc_hint, bool wrap) @@ -218,12 +238,12 @@ static int sbitmap_find_bit(struct sbitmap *sb, int nr = -1; for (i = 0; i < sb->map_nr; i++) { - nr = sbitmap_find_bit_in_word(&sb->map[index], - min_t(unsigned int, - __map_depth(sb, index), - depth), - alloc_hint, wrap); + unsigned int depth = __map_depth_with_shallow(sb, index, + shallow_depth); + if (depth) + nr = sbitmap_find_bit_in_word(&sb->map[index], depth, + alloc_hint, wrap); if (nr != -1) { nr += index << sb->shift; break; @@ -287,7 +307,22 @@ static int __sbitmap_get_shallow(struct sbitmap *sb, return sbitmap_find_bit(sb, shallow_depth, index, alloc_hint, true); } -int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth) +/** + * sbitmap_get_shallow() - Try to allocate a free bit from a &struct sbitmap, + * limiting the depth used from each word. + * @sb: Bitmap to allocate from. + * @shallow_depth: The maximum number of bits to allocate from the bitmap. + * + * This rather specific operation allows for having multiple users with + * different allocation limits. E.g., there can be a high-priority class that + * uses sbitmap_get() and a low-priority class that uses sbitmap_get_shallow() + * with a @shallow_depth of (sb->depth >> 1). Then, the low-priority + * class can only allocate half of the total bits in the bitmap, preventing it + * from starving out the high-priority class. + * + * Return: Non-negative allocated bit number if successful, -1 otherwise. + */ +static int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth) { int nr; unsigned int hint, depth; @@ -302,7 +337,6 @@ int sbitmap_get_shallow(struct sbitmap *sb, unsigned long shallow_depth) return nr; } -EXPORT_SYMBOL_GPL(sbitmap_get_shallow); bool sbitmap_any_bit_set(const struct sbitmap *sb) { @@ -406,27 +440,9 @@ EXPORT_SYMBOL_GPL(sbitmap_bitmap_show); static unsigned int sbq_calc_wake_batch(struct sbitmap_queue *sbq, unsigned int depth) { - unsigned int wake_batch; - unsigned int shallow_depth; - - /* - * Each full word of the bitmap has bits_per_word bits, and there might - * be a partial word. There are depth / bits_per_word full words and - * depth % bits_per_word bits left over. In bitwise arithmetic: - * - * bits_per_word = 1 << shift - * depth / bits_per_word = depth >> shift - * depth % bits_per_word = depth & ((1 << shift) - 1) - * - * Each word can be limited to sbq->min_shallow_depth bits. - */ - shallow_depth = min(1U << sbq->sb.shift, sbq->min_shallow_depth); - depth = ((depth >> sbq->sb.shift) * shallow_depth + - min(depth & ((1U << sbq->sb.shift) - 1), shallow_depth)); - wake_batch = clamp_t(unsigned int, depth / SBQ_WAIT_QUEUES, 1, - SBQ_WAKE_BATCH); - - return wake_batch; + return clamp_t(unsigned int, + min(depth, sbq->min_shallow_depth) / SBQ_WAIT_QUEUES, + 1, SBQ_WAKE_BATCH); } int sbitmap_queue_init_node(struct sbitmap_queue *sbq, unsigned int depth, diff --git a/net/core/dev.c b/net/core/dev.c index b28ce68830b2..68dc47d7e700 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -6978,6 +6978,12 @@ int napi_set_threaded(struct napi_struct *napi, if (napi->config) napi->config->threaded = threaded; + /* Setting/unsetting threaded mode on a napi might not immediately + * take effect, if the current napi instance is actively being + * polled. In this case, the switch between threaded mode and + * softirq mode will happen in the next round of napi_schedule(). + * This should not cause hiccups/stalls to the live traffic. + */ if (!threaded && napi->thread) { napi_stop_kthread(napi); } else { @@ -7011,23 +7017,9 @@ int netif_set_threaded(struct net_device *dev, WRITE_ONCE(dev->threaded, threaded); - /* Make sure kthread is created before THREADED bit - * is set. - */ - smp_mb__before_atomic(); - - /* Setting/unsetting threaded mode on a napi might not immediately - * take effect, if the current napi instance is actively being - * polled. In this case, the switch between threaded mode and - * softirq mode will happen in the next round of napi_schedule(). - * This should not cause hiccups/stalls to the live traffic. - */ - list_for_each_entry(napi, &dev->napi_list, dev_list) { - if (!threaded && napi->thread) - napi_stop_kthread(napi); - else - assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); - } + /* The error should not occur as the kthreads are already created. */ + list_for_each_entry(napi, &dev->napi_list, dev_list) + WARN_ON_ONCE(napi_set_threaded(napi, threaded)); return err; } diff --git a/net/core/devmem.c b/net/core/devmem.c index b3a62ca0df65..24c591ab38ae 100644 --- a/net/core/devmem.c +++ b/net/core/devmem.c @@ -70,14 +70,13 @@ void __net_devmem_dmabuf_binding_free(struct work_struct *wq) gen_pool_destroy(binding->chunk_pool); dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, - DMA_FROM_DEVICE); + binding->direction); dma_buf_detach(binding->dmabuf, binding->attachment); dma_buf_put(binding->dmabuf); xa_destroy(&binding->bound_rxqs); kvfree(binding->tx_vec); kfree(binding); } -EXPORT_SYMBOL(__net_devmem_dmabuf_binding_free); struct net_iov * net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) @@ -208,6 +207,7 @@ net_devmem_bind_dmabuf(struct net_device *dev, mutex_init(&binding->lock); binding->dmabuf = dmabuf; + binding->direction = direction; binding->attachment = dma_buf_attach(binding->dmabuf, dev->dev.parent); if (IS_ERR(binding->attachment)) { @@ -312,7 +312,7 @@ err_tx_vec: kvfree(binding->tx_vec); err_unmap: dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, - DMA_FROM_DEVICE); + direction); err_detach: dma_buf_detach(dmabuf, binding->attachment); err_free_binding: diff --git a/net/core/devmem.h b/net/core/devmem.h index 0a3b28ba5c13..41cd6e1c9141 100644 --- a/net/core/devmem.h +++ b/net/core/devmem.h @@ -56,6 +56,9 @@ struct net_devmem_dmabuf_binding { */ u32 id; + /* DMA direction, FROM_DEVICE for Rx binding, TO_DEVICE for Tx. */ + enum dma_data_direction direction; + /* Array of net_iov pointers for this binding, sorted by virtual * address. This array is convenient to map the virtual addresses to * net_iovs in the TX path. @@ -165,10 +168,6 @@ static inline void net_devmem_put_net_iov(struct net_iov *niov) { } -static inline void __net_devmem_dmabuf_binding_free(struct work_struct *wq) -{ -} - static inline struct net_devmem_dmabuf_binding * net_devmem_bind_dmabuf(struct net_device *dev, enum dma_data_direction direction, diff --git a/net/core/netpoll.c b/net/core/netpoll.c index a1da97b5b30b..5f65b62346d4 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -768,6 +768,13 @@ int netpoll_setup(struct netpoll *np) if (err) goto flush; rtnl_unlock(); + + /* Make sure all NAPI polls which started before dev->npinfo + * was visible have exited before we start calling NAPI poll. + * NAPI skips locking if dev->npinfo is NULL. + */ + synchronize_rcu(); + return 0; flush: diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 10a1d182fd84..84e7f8a2f50f 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -425,15 +425,20 @@ int ip_mc_output(struct net *net, struct sock *sk, struct sk_buff *skb) int ip_output(struct net *net, struct sock *sk, struct sk_buff *skb) { - struct net_device *dev = skb_dst_dev(skb), *indev = skb->dev; + struct net_device *dev, *indev = skb->dev; + int ret_val; + rcu_read_lock(); + dev = skb_dst_dev_rcu(skb); skb->dev = dev; skb->protocol = htons(ETH_P_IP); - return NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, - net, sk, skb, indev, dev, - ip_finish_output, - !(IPCB(skb)->flags & IPSKB_REROUTED)); + ret_val = NF_HOOK_COND(NFPROTO_IPV4, NF_INET_POST_ROUTING, + net, sk, skb, indev, dev, + ip_finish_output, + !(IPCB(skb)->flags & IPSKB_REROUTED)); + rcu_read_unlock(); + return ret_val; } EXPORT_SYMBOL(ip_output); diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 9822163428b0..fce91183797a 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -148,7 +148,9 @@ static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb, ops = rcu_dereference(inet6_offloads[proto]); if (likely(ops && ops->callbacks.gso_segment)) { - skb_reset_transport_header(skb); + if (!skb_reset_transport_header_careful(skb)) + goto out; + segs = ops->callbacks.gso_segment(skb, features); if (!segs) skb->network_header = skb_mac_header(skb) + nhoff - skb->head; diff --git a/net/kcm/kcmsock.c b/net/kcm/kcmsock.c index a0be3896a934..a4971e6fa943 100644 --- a/net/kcm/kcmsock.c +++ b/net/kcm/kcmsock.c @@ -19,6 +19,7 @@ #include <linux/rculist.h> #include <linux/skbuff.h> #include <linux/socket.h> +#include <linux/splice.h> #include <linux/uaccess.h> #include <linux/workqueue.h> #include <linux/syscalls.h> @@ -1029,6 +1030,11 @@ static ssize_t kcm_splice_read(struct socket *sock, loff_t *ppos, ssize_t copied; struct sk_buff *skb; + if (sock->file->f_flags & O_NONBLOCK || flags & SPLICE_F_NONBLOCK) + flags = MSG_DONTWAIT; + else + flags = 0; + /* Only support splice for SOCKSEQPACKET */ skb = skb_recv_datagram(sk, flags, &err); diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 5949855fa29e..e2f7080dd5d7 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -1218,7 +1218,7 @@ int netlink_attachskb(struct sock *sk, struct sk_buff *skb, nlk = nlk_sk(sk); rmem = atomic_add_return(skb->truesize, &sk->sk_rmem_alloc); - if ((rmem == skb->truesize || rmem < READ_ONCE(sk->sk_rcvbuf)) && + if ((rmem == skb->truesize || rmem <= READ_ONCE(sk->sk_rcvbuf)) && !test_bit(NETLINK_S_CONGESTED, &nlk->state)) { netlink_skb_set_owner_r(skb, sk); return 0; diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index bc438d0d96a7..a7017d7f0927 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -4573,10 +4573,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, spin_lock(&po->bind_lock); was_running = packet_sock_flag(po, PACKET_SOCK_RUNNING); num = po->num; - if (was_running) { - WRITE_ONCE(po->num, 0); + WRITE_ONCE(po->num, 0); + if (was_running) __unregister_prot_hook(sk, false); - } + spin_unlock(&po->bind_lock); synchronize_net(); @@ -4608,10 +4608,10 @@ static int packet_set_ring(struct sock *sk, union tpacket_req_u *req_u, mutex_unlock(&po->pg_vec_lock); spin_lock(&po->bind_lock); - if (was_running) { - WRITE_ONCE(po->num, num); + WRITE_ONCE(po->num, num); + if (was_running) register_prot_hook(sk); - } + spin_unlock(&po->bind_lock); if (pg_vec && (po->tp_version > TPACKET_V2)) { /* Because we don't support block-based V3 on tx-ring */ diff --git a/net/sched/sch_mqprio.c b/net/sched/sch_mqprio.c index 51d4013b6121..f3e5ef9a9592 100644 --- a/net/sched/sch_mqprio.c +++ b/net/sched/sch_mqprio.c @@ -152,7 +152,7 @@ static int mqprio_parse_opt(struct net_device *dev, struct tc_mqprio_qopt *qopt, static const struct nla_policy mqprio_tc_entry_policy[TCA_MQPRIO_TC_ENTRY_MAX + 1] = { [TCA_MQPRIO_TC_ENTRY_INDEX] = NLA_POLICY_MAX(NLA_U32, - TC_QOPT_MAX_QUEUE), + TC_QOPT_MAX_QUEUE - 1), [TCA_MQPRIO_TC_ENTRY_FP] = NLA_POLICY_RANGE(NLA_U32, TC_FP_EXPRESS, TC_FP_PREEMPTIBLE), diff --git a/net/sched/sch_taprio.c b/net/sched/sch_taprio.c index e759e43ad27e..39b735386996 100644 --- a/net/sched/sch_taprio.c +++ b/net/sched/sch_taprio.c @@ -43,6 +43,11 @@ static struct static_key_false taprio_have_working_mqprio; #define TAPRIO_SUPPORTED_FLAGS \ (TCA_TAPRIO_ATTR_FLAG_TXTIME_ASSIST | TCA_TAPRIO_ATTR_FLAG_FULL_OFFLOAD) #define TAPRIO_FLAGS_INVALID U32_MAX +/* Minimum value for picos_per_byte to ensure non-zero duration + * for minimum-sized Ethernet frames (ETH_ZLEN = 60). + * 60 * 17 > PSEC_PER_NSEC (1000) + */ +#define TAPRIO_PICOS_PER_BYTE_MIN 17 struct sched_entry { /* Durations between this GCL entry and the GCL entry where the @@ -1284,7 +1289,8 @@ static void taprio_start_sched(struct Qdisc *sch, } static void taprio_set_picos_per_byte(struct net_device *dev, - struct taprio_sched *q) + struct taprio_sched *q, + struct netlink_ext_ack *extack) { struct ethtool_link_ksettings ecmd; int speed = SPEED_10; @@ -1300,6 +1306,15 @@ static void taprio_set_picos_per_byte(struct net_device *dev, skip: picos_per_byte = (USEC_PER_SEC * 8) / speed; + if (picos_per_byte < TAPRIO_PICOS_PER_BYTE_MIN) { + if (!extack) + pr_warn("Link speed %d is too high. Schedule may be inaccurate.\n", + speed); + NL_SET_ERR_MSG_FMT_MOD(extack, + "Link speed %d is too high. Schedule may be inaccurate.", + speed); + picos_per_byte = TAPRIO_PICOS_PER_BYTE_MIN; + } atomic64_set(&q->picos_per_byte, picos_per_byte); netdev_dbg(dev, "taprio: set %s's picos_per_byte to: %lld, linkspeed: %d\n", @@ -1324,7 +1339,7 @@ static int taprio_dev_notifier(struct notifier_block *nb, unsigned long event, if (dev != qdisc_dev(q->root)) continue; - taprio_set_picos_per_byte(dev, q); + taprio_set_picos_per_byte(dev, q, NULL); stab = rtnl_dereference(q->root->stab); @@ -1844,7 +1859,7 @@ static int taprio_change(struct Qdisc *sch, struct nlattr *opt, q->flags = taprio_flags; /* Needed for length_to_duration() during netlink attribute parsing */ - taprio_set_picos_per_byte(dev, q); + taprio_set_picos_per_byte(dev, q, extack); err = taprio_parse_mqprio_opt(dev, mqprio, extack, q->flags); if (err < 0) diff --git a/net/sunrpc/auth_gss/gss_krb5_crypto.c b/net/sunrpc/auth_gss/gss_krb5_crypto.c index 8f2d65c1e831..16dcf115de1e 100644 --- a/net/sunrpc/auth_gss/gss_krb5_crypto.c +++ b/net/sunrpc/auth_gss/gss_krb5_crypto.c @@ -875,8 +875,8 @@ out_err: * krb5_etm_decrypt - Decrypt using the RFC 8009 rules * @kctx: Kerberos context * @offset: starting offset of the ciphertext, in bytes - * @len: - * @buf: + * @len: size of ciphertext to unwrap + * @buf: ciphertext to unwrap * @headskip: OUT: the enctype's confounder length, in octets * @tailskip: OUT: the enctype's HMAC length, in octets * diff --git a/net/sunrpc/xdr.c b/net/sunrpc/xdr.c index 1346fdf33835..70efc727a9cd 100644 --- a/net/sunrpc/xdr.c +++ b/net/sunrpc/xdr.c @@ -37,19 +37,6 @@ xdr_encode_netobj(__be32 *p, const struct xdr_netobj *obj) } EXPORT_SYMBOL_GPL(xdr_encode_netobj); -__be32 * -xdr_decode_netobj(__be32 *p, struct xdr_netobj *obj) -{ - unsigned int len; - - if ((len = be32_to_cpu(*p++)) > XDR_MAX_NETOBJ) - return NULL; - obj->len = len; - obj->data = (u8 *) p; - return p + XDR_QUADLEN(len); -} -EXPORT_SYMBOL_GPL(xdr_decode_netobj); - /** * xdr_encode_opaque_fixed - Encode fixed length opaque data * @p: pointer to current position in XDR buffer. @@ -102,21 +89,6 @@ xdr_encode_string(__be32 *p, const char *string) } EXPORT_SYMBOL_GPL(xdr_encode_string); -__be32 * -xdr_decode_string_inplace(__be32 *p, char **sp, - unsigned int *lenp, unsigned int maxlen) -{ - u32 len; - - len = be32_to_cpu(*p++); - if (len > maxlen) - return NULL; - *lenp = len; - *sp = (char *) p; - return p + XDR_QUADLEN(len); -} -EXPORT_SYMBOL_GPL(xdr_decode_string_inplace); - /** * xdr_terminate_string - '\0'-terminate a string residing in an xdr_buf * @buf: XDR buffer where string resides @@ -2245,88 +2217,6 @@ out: EXPORT_SYMBOL_GPL(xdr_process_buf); /** - * xdr_stream_decode_opaque - Decode variable length opaque - * @xdr: pointer to xdr_stream - * @ptr: location to store opaque data - * @size: size of storage buffer @ptr - * - * Return values: - * On success, returns size of object stored in *@ptr - * %-EBADMSG on XDR buffer overflow - * %-EMSGSIZE on overflow of storage buffer @ptr - */ -ssize_t xdr_stream_decode_opaque(struct xdr_stream *xdr, void *ptr, size_t size) -{ - ssize_t ret; - void *p; - - ret = xdr_stream_decode_opaque_inline(xdr, &p, size); - if (ret <= 0) - return ret; - memcpy(ptr, p, ret); - return ret; -} -EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque); - -/** - * xdr_stream_decode_opaque_dup - Decode and duplicate variable length opaque - * @xdr: pointer to xdr_stream - * @ptr: location to store pointer to opaque data - * @maxlen: maximum acceptable object size - * @gfp_flags: GFP mask to use - * - * Return values: - * On success, returns size of object stored in *@ptr - * %-EBADMSG on XDR buffer overflow - * %-EMSGSIZE if the size of the object would exceed @maxlen - * %-ENOMEM on memory allocation failure - */ -ssize_t xdr_stream_decode_opaque_dup(struct xdr_stream *xdr, void **ptr, - size_t maxlen, gfp_t gfp_flags) -{ - ssize_t ret; - void *p; - - ret = xdr_stream_decode_opaque_inline(xdr, &p, maxlen); - if (ret > 0) { - *ptr = kmemdup(p, ret, gfp_flags); - if (*ptr != NULL) - return ret; - ret = -ENOMEM; - } - *ptr = NULL; - return ret; -} -EXPORT_SYMBOL_GPL(xdr_stream_decode_opaque_dup); - -/** - * xdr_stream_decode_string - Decode variable length string - * @xdr: pointer to xdr_stream - * @str: location to store string - * @size: size of storage buffer @str - * - * Return values: - * On success, returns length of NUL-terminated string stored in *@str - * %-EBADMSG on XDR buffer overflow - * %-EMSGSIZE on overflow of storage buffer @str - */ -ssize_t xdr_stream_decode_string(struct xdr_stream *xdr, char *str, size_t size) -{ - ssize_t ret; - void *p; - - ret = xdr_stream_decode_opaque_inline(xdr, &p, size); - if (ret > 0) { - memcpy(str, p, ret); - str[ret] = '\0'; - return strlen(str); - } - *str = '\0'; - return ret; -} -EXPORT_SYMBOL_GPL(xdr_stream_decode_string); - -/** * xdr_stream_decode_string_dup - Decode and duplicate variable length string * @xdr: pointer to xdr_stream * @str: location to store pointer to string diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 04ff66758fc3..c5f7bbf5775f 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -358,7 +358,7 @@ xs_alloc_sparse_pages(struct xdr_buf *buf, size_t want, gfp_t gfp) static int xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, - struct cmsghdr *cmsg, int ret) + unsigned int *msg_flags, struct cmsghdr *cmsg, int ret) { u8 content_type = tls_get_record_type(sock->sk, cmsg); u8 level, description; @@ -371,7 +371,7 @@ xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, * record, even though there might be more frames * waiting to be decrypted. */ - msg->msg_flags &= ~MSG_EOR; + *msg_flags &= ~MSG_EOR; break; case TLS_RECORD_TYPE_ALERT: tls_alert_recv(sock->sk, msg, &level, &description); @@ -386,19 +386,33 @@ xs_sock_process_cmsg(struct socket *sock, struct msghdr *msg, } static int -xs_sock_recv_cmsg(struct socket *sock, struct msghdr *msg, int flags) +xs_sock_recv_cmsg(struct socket *sock, unsigned int *msg_flags, int flags) { union { struct cmsghdr cmsg; u8 buf[CMSG_SPACE(sizeof(u8))]; } u; + u8 alert[2]; + struct kvec alert_kvec = { + .iov_base = alert, + .iov_len = sizeof(alert), + }; + struct msghdr msg = { + .msg_flags = *msg_flags, + .msg_control = &u, + .msg_controllen = sizeof(u), + }; int ret; - msg->msg_control = &u; - msg->msg_controllen = sizeof(u); - ret = sock_recvmsg(sock, msg, flags); - if (msg->msg_controllen != sizeof(u)) - ret = xs_sock_process_cmsg(sock, msg, &u.cmsg, ret); + iov_iter_kvec(&msg.msg_iter, ITER_DEST, &alert_kvec, 1, + alert_kvec.iov_len); + ret = sock_recvmsg(sock, &msg, flags); + if (ret > 0 && + tls_get_record_type(sock->sk, &u.cmsg) == TLS_RECORD_TYPE_ALERT) { + iov_iter_revert(&msg.msg_iter, ret); + ret = xs_sock_process_cmsg(sock, &msg, msg_flags, &u.cmsg, + -EAGAIN); + } return ret; } @@ -408,7 +422,13 @@ xs_sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags, size_t seek) ssize_t ret; if (seek != 0) iov_iter_advance(&msg->msg_iter, seek); - ret = xs_sock_recv_cmsg(sock, msg, flags); + ret = sock_recvmsg(sock, msg, flags); + /* Handle TLS inband control message lazily */ + if (msg->msg_flags & MSG_CTRUNC) { + msg->msg_flags &= ~(MSG_CTRUNC | MSG_EOR); + if (ret == 0 || ret == -EIO) + ret = xs_sock_recv_cmsg(sock, &msg->msg_flags, flags); + } return ret > 0 ? ret + seek : ret; } @@ -434,7 +454,7 @@ xs_read_discard(struct socket *sock, struct msghdr *msg, int flags, size_t count) { iov_iter_discard(&msg->msg_iter, ITER_DEST, count); - return xs_sock_recv_cmsg(sock, msg, flags); + return xs_sock_recvmsg(sock, msg, flags, 0); } #if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE diff --git a/sound/hda/codecs/ca0132.c b/sound/hda/codecs/ca0132.c index b716f721f25d..b7d456e16c93 100644 --- a/sound/hda/codecs/ca0132.c +++ b/sound/hda/codecs/ca0132.c @@ -4802,7 +4802,8 @@ static int ca0132_alt_select_out(struct hda_codec *codec) if (err < 0) goto exit; - if (ca0132_alt_select_out_quirk_set(codec) < 0) + err = ca0132_alt_select_out_quirk_set(codec); + if (err < 0) goto exit; switch (spec->cur_out_type) { @@ -4892,6 +4893,8 @@ static int ca0132_alt_select_out(struct hda_codec *codec) spec->bass_redirection_val); else err = ca0132_alt_surround_set_bass_redirection(codec, 0); + if (err < 0) + goto exit; /* Unmute DSP now that we're done with output selection. */ err = dspio_set_uint_param(codec, 0x96, diff --git a/sound/hda/codecs/cirrus/Kconfig b/sound/hda/codecs/cirrus/Kconfig index 33cfe52713bc..ec6cbcaf64f0 100644 --- a/sound/hda/codecs/cirrus/Kconfig +++ b/sound/hda/codecs/cirrus/Kconfig @@ -2,27 +2,31 @@ menuconfig SND_HDA_CODEC_CIRRUS tristate "Cirrus Logic HD-audio codec support" + help + Say Y or M here to include Cirrus Logic HD-audio codec support. + + This will enable both CS420x and CS421x HD-audio codec drivers + as default, but you can enable/disable each codec driver + individually, too (only when CONFIG_EXPERT is set). if SND_HDA_CODEC_CIRRUS config SND_HDA_CODEC_CS420X - tristate "Build Cirrus Logic CS420x codec support" + tristate "Build Cirrus Logic CS420x codec support" if EXPERT select SND_HDA_GENERIC default y help - Say Y or M here to include Cirrus Logic CS420x codec support in - snd-hda-intel driver + Say Y or M here to include Cirrus Logic CS420x codec support comment "Set to Y if you want auto-loading the codec driver" depends on SND_HDA=y && SND_HDA_CODEC_CS420X=m config SND_HDA_CODEC_CS421X - tristate "Build Cirrus Logic CS421x codec support" + tristate "Build Cirrus Logic CS421x codec support" if EXPERT select SND_HDA_GENERIC default y help - Say Y or M here to include Cirrus Logic CS421x codec support in - snd-hda-intel driver + Say Y or M here to include Cirrus Logic CS421x codec support comment "Set to Y if you want auto-loading the codec driver" depends on SND_HDA=y && SND_HDA_CODEC_CS421X=m @@ -31,8 +35,8 @@ config SND_HDA_CODEC_CS8409 tristate "Build Cirrus Logic HDA bridge support" select SND_HDA_GENERIC help - Say Y or M here to include Cirrus Logic HDA bridge support in - snd-hda-intel driver, such as CS8409. + Say Y or M here to include Cirrus Logic HDA bridge support + such as CS8409. comment "Set to Y if you want auto-loading the codec driver" depends on SND_HDA=y && SND_HDA_CODEC_CS8409=m diff --git a/sound/hda/codecs/hdmi/Kconfig b/sound/hda/codecs/hdmi/Kconfig index 973ca4ca077b..6ea3553ba9f8 100644 --- a/sound/hda/codecs/hdmi/Kconfig +++ b/sound/hda/codecs/hdmi/Kconfig @@ -2,11 +2,17 @@ menuconfig SND_HDA_CODEC_HDMI tristate "HD-audio HDMI codec support" + help + Say Y or M here to include HD-audio HDMI/DislayPort codec support. + + This will enable all HDMI/DP codec drivers as default, but you can + enable/disable each codec driver individually, too (only when + CONFIG_EXPERT is set). if SND_HDA_CODEC_HDMI config SND_HDA_CODEC_HDMI_GENERIC - tristate "Generic HDMI/DisplayPort HD-audio codec support" + tristate "Generic HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_DYNAMIC_MINORS select SND_PCM_ELD default y @@ -18,14 +24,14 @@ config SND_HDA_CODEC_HDMI_GENERIC to assure the multiple streams for DP-MST support. config SND_HDA_CODEC_HDMI_SIMPLE - tristate "Simple HDMI/DisplayPort HD-audio codec support" + tristate "Simple HDMI/DisplayPort HD-audio codec support" if EXPERT default y help Say Y or M here to include Simple HDMI and DisplayPort HD-audio codec support for VIA and other codecs. config SND_HDA_CODEC_HDMI_INTEL - tristate "Intel HDMI/DisplayPort HD-audio codec support" + tristate "Intel HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_HDA_CODEC_HDMI_GENERIC default y help @@ -48,7 +54,7 @@ config SND_HDA_INTEL_HDMI_SILENT_STREAM are kept reserved both at transmitter and receiver. config SND_HDA_CODEC_HDMI_ATI - tristate "AMD/ATI HDMI/DisplayPort HD-audio codec support" + tristate "AMD/ATI HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_HDA_CODEC_HDMI_GENERIC default y help @@ -56,7 +62,7 @@ config SND_HDA_CODEC_HDMI_ATI HD-audio codec support. config SND_HDA_CODEC_HDMI_NVIDIA - tristate "Nvidia HDMI/DisplayPort HD-audio codec support" + tristate "Nvidia HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_HDA_CODEC_HDMI_GENERIC default y help @@ -64,7 +70,7 @@ config SND_HDA_CODEC_HDMI_NVIDIA support for the recent Nvidia graphics cards. config SND_HDA_CODEC_HDMI_NVIDIA_MCP - tristate "Legacy Nvidia HDMI/DisplayPort HD-audio codec support" + tristate "Legacy Nvidia HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_HDA_CODEC_HDMI_SIMPLE default y help @@ -72,7 +78,7 @@ config SND_HDA_CODEC_HDMI_NVIDIA_MCP support for the legacy Nvidia graphics like MCP73, MCP67, MCP77/78. config SND_HDA_CODEC_HDMI_TEGRA - tristate "Nvidia Tegra HDMI/DisplayPort HD-audio codec support" + tristate "Nvidia Tegra HDMI/DisplayPort HD-audio codec support" if EXPERT select SND_HDA_CODEC_HDMI_GENERIC default y help diff --git a/sound/hda/codecs/realtek/Kconfig b/sound/hda/codecs/realtek/Kconfig index 20899f3fc051..cdc6d9509a01 100644 --- a/sound/hda/codecs/realtek/Kconfig +++ b/sound/hda/codecs/realtek/Kconfig @@ -2,6 +2,12 @@ menuconfig SND_HDA_CODEC_REALTEK tristate "Realtek HD-audio codec support" + help + Say Y or M here to include Realtek HD-audio codec support. + + This will enable all Realtek HD-audio codec drivers as default, + but you can enable/disable each codec driver individually, too + (only when CONFIG_EXPERT is set). if SND_HDA_CODEC_REALTEK @@ -12,7 +18,7 @@ config SND_HDA_CODEC_REALTEK_LIB select SND_HDA_SCODEC_COMPONENT config SND_HDA_CODEC_ALC260 - tristate "Build Realtek ALC260 HD-audio codec support" + tristate "Build Realtek ALC260 HD-audio codec support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -20,7 +26,7 @@ config SND_HDA_CODEC_ALC260 Say Y or M here to include Realtek ALC260 HD-audio codec support config SND_HDA_CODEC_ALC262 - tristate "Build Realtek ALC262 HD-audio codec support" + tristate "Build Realtek ALC262 HD-audio codec support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -28,7 +34,7 @@ config SND_HDA_CODEC_ALC262 Say Y or M here to include Realtek ALC262 HD-audio codec support config SND_HDA_CODEC_ALC268 - tristate "Build Realtek ALC268 HD-audio codec support" + tristate "Build Realtek ALC268 HD-audio codec support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -37,7 +43,7 @@ config SND_HDA_CODEC_ALC268 codec support config SND_HDA_CODEC_ALC269 - tristate "Build Realtek ALC269 HD-audio codecs support" + tristate "Build Realtek ALC269 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -46,7 +52,7 @@ config SND_HDA_CODEC_ALC269 codec support config SND_HDA_CODEC_ALC662 - tristate "Build Realtek ALC662 HD-audio codecs support" + tristate "Build Realtek ALC662 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -55,7 +61,7 @@ config SND_HDA_CODEC_ALC662 codec support config SND_HDA_CODEC_ALC680 - tristate "Build Realtek ALC680 HD-audio codecs support" + tristate "Build Realtek ALC680 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -63,7 +69,7 @@ config SND_HDA_CODEC_ALC680 Say Y or M here to include Realtek ALC680 HD-audio codec support config SND_HDA_CODEC_ALC861 - tristate "Build Realtek ALC861 HD-audio codecs support" + tristate "Build Realtek ALC861 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -71,7 +77,7 @@ config SND_HDA_CODEC_ALC861 Say Y or M here to include Realtek ALC861 HD-audio codec support config SND_HDA_CODEC_ALC861VD - tristate "Build Realtek ALC861-VD HD-audio codecs support" + tristate "Build Realtek ALC861-VD HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -79,7 +85,7 @@ config SND_HDA_CODEC_ALC861VD Say Y or M here to include Realtek ALC861-VD HD-audio codec support config SND_HDA_CODEC_ALC880 - tristate "Build Realtek ALC880 HD-audio codecs support" + tristate "Build Realtek ALC880 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -87,7 +93,7 @@ config SND_HDA_CODEC_ALC880 Say Y or M here to include Realtek ALC880 HD-audio codec support config SND_HDA_CODEC_ALC882 - tristate "Build Realtek ALC882 HD-audio codecs support" + tristate "Build Realtek ALC882 HD-audio codecs support" if EXPERT depends on INPUT select SND_HDA_CODEC_REALTEK_LIB default y @@ -96,5 +102,3 @@ config SND_HDA_CODEC_ALC882 codec support endif - - diff --git a/sound/hda/codecs/realtek/alc269.c b/sound/hda/codecs/realtek/alc269.c index 2554b42eeb0f..e27a36e4e92a 100644 --- a/sound/hda/codecs/realtek/alc269.c +++ b/sound/hda/codecs/realtek/alc269.c @@ -7110,6 +7110,7 @@ static const struct hda_quirk alc269_fixup_tbl[] = { SND_PCI_QUIRK(0x1854, 0x0440, "LG CQ6", ALC256_FIXUP_HEADPHONE_AMP_VOL), SND_PCI_QUIRK(0x1854, 0x0441, "LG CQ6 AIO", ALC256_FIXUP_HEADPHONE_AMP_VOL), SND_PCI_QUIRK(0x1854, 0x0488, "LG gram 16 (16Z90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), + SND_PCI_QUIRK(0x1854, 0x0489, "LG gram 16 (16Z90R-A)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x1854, 0x048a, "LG gram 17 (17ZD90R)", ALC298_FIXUP_SAMSUNG_AMP_V2_4_AMPS), SND_PCI_QUIRK(0x19e5, 0x3204, "Huawei MACH-WX9", ALC256_FIXUP_HUAWEI_MACH_WX9_PINS), SND_PCI_QUIRK(0x19e5, 0x320f, "Huawei WRT-WX9 ", ALC256_FIXUP_ASUS_MIC_NO_PRESENCE), diff --git a/sound/hda/codecs/side-codecs/tas2781_hda.c b/sound/hda/codecs/side-codecs/tas2781_hda.c index 34217ce9f28e..f46d2e06c64f 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda.c +++ b/sound/hda/codecs/side-codecs/tas2781_hda.c @@ -18,6 +18,8 @@ #include "tas2781_hda.h" +#define CALIBRATION_DATA_AREA_NUM 2 + const efi_guid_t tasdev_fct_efi_guid[] = { /* DELL */ EFI_GUID(0xcc92382d, 0x6337, 0x41cb, 0xa8, 0x8b, 0x8e, 0xce, 0x74, @@ -160,36 +162,51 @@ int tas2781_save_calibration(struct tas2781_hda *hda) * manufactory. */ efi_guid_t efi_guid = tasdev_fct_efi_guid[LENOVO]; - static efi_char16_t efi_name[] = TASDEVICE_CALIBRATION_DATA_NAME; + /* + * Some devices save the calibrated data into L"CALI_DATA", + * and others into L"SmartAmpCalibrationData". + */ + static efi_char16_t *efi_name[CALIBRATION_DATA_AREA_NUM] = { + L"CALI_DATA", + L"SmartAmpCalibrationData", + }; struct tasdevice_priv *p = hda->priv; struct calidata *cali_data = &p->cali_data; unsigned long total_sz = 0; unsigned int attr, size; unsigned char *data; efi_status_t status; + int i; if (hda->catlog_id < LENOVO) efi_guid = tasdev_fct_efi_guid[hda->catlog_id]; cali_data->cali_dat_sz_per_dev = 20; size = p->ndev * (cali_data->cali_dat_sz_per_dev + 1); - /* Get real size of UEFI variable */ - status = efi.get_variable(efi_name, &efi_guid, &attr, &total_sz, NULL); - cali_data->total_sz = total_sz > size ? total_sz : size; - if (status == EFI_BUFFER_TOO_SMALL) { - /* Allocate data buffer of data_size bytes */ - data = p->cali_data.data = devm_kzalloc(p->dev, - p->cali_data.total_sz, GFP_KERNEL); - if (!data) { - p->cali_data.total_sz = 0; - return -ENOMEM; + for (i = 0; i < CALIBRATION_DATA_AREA_NUM; i++) { + /* Get real size of UEFI variable */ + status = efi.get_variable(efi_name[i], &efi_guid, &attr, + &total_sz, NULL); + cali_data->total_sz = total_sz > size ? total_sz : size; + if (status == EFI_BUFFER_TOO_SMALL) { + /* Allocate data buffer of data_size bytes */ + data = cali_data->data = devm_kzalloc(p->dev, + cali_data->total_sz, GFP_KERNEL); + if (!data) { + status = -ENOMEM; + continue; + } + /* Get variable contents into buffer */ + status = efi.get_variable(efi_name[i], &efi_guid, + &attr, &cali_data->total_sz, data); } - /* Get variable contents into buffer */ - status = efi.get_variable(efi_name, &efi_guid, &attr, - &p->cali_data.total_sz, data); + /* Check whether get the calibrated data */ + if (status == EFI_SUCCESS) + break; } + if (status != EFI_SUCCESS) { - p->cali_data.total_sz = 0; + cali_data->total_sz = 0; return status; } diff --git a/sound/hda/codecs/side-codecs/tas2781_hda.h b/sound/hda/codecs/side-codecs/tas2781_hda.h index 575a701c8dfb..66188909a0bb 100644 --- a/sound/hda/codecs/side-codecs/tas2781_hda.h +++ b/sound/hda/codecs/side-codecs/tas2781_hda.h @@ -11,7 +11,7 @@ /* Flag of calibration registers address. */ #define TASDEV_UEFI_CALI_REG_ADDR_FLG BIT(7) -#define TASDEVICE_CALIBRATION_DATA_NAME L"CALI_DATA" + #define TASDEV_CALIB_N 5 /* diff --git a/sound/soc/amd/acp/acp-sdw-legacy-mach.c b/sound/soc/amd/acp/acp-sdw-legacy-mach.c index 6c24f9d8694e..c2197b75a7dd 100644 --- a/sound/soc/amd/acp/acp-sdw-legacy-mach.c +++ b/sound/soc/amd/acp/acp-sdw-legacy-mach.c @@ -158,6 +158,7 @@ static int create_sdw_dailink(struct snd_soc_card *card, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: ret = get_acp70_cpu_pin_id(ffs(soc_end->link_mask - 1), *be_id, &cpu_pin_id, dev); if (ret) @@ -264,6 +265,7 @@ static int create_sdw_dailinks(struct snd_soc_card *card, case ACP63_PCI_REV: case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: sdw_platform_component->name = "amd_ps_sdw_dma.0"; break; default: @@ -311,6 +313,7 @@ static int create_dmic_dailinks(struct snd_soc_card *card, case ACP63_PCI_REV: case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: pdm_cpu->name = "acp_ps_pdm_dma.0"; pdm_platform->name = "acp_ps_pdm_dma.0"; break; diff --git a/sound/soc/amd/acp/acp-sdw-sof-mach.c b/sound/soc/amd/acp/acp-sdw-sof-mach.c index 654fe78b2e2e..91d72d4bb9a2 100644 --- a/sound/soc/amd/acp/acp-sdw-sof-mach.c +++ b/sound/soc/amd/acp/acp-sdw-sof-mach.c @@ -130,6 +130,7 @@ static int create_sdw_dailink(struct snd_soc_card *card, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: ret = get_acp70_cpu_pin_id(ffs(sof_end->link_mask - 1), *be_id, &cpu_pin_id, dev); if (ret) diff --git a/sound/soc/amd/acp/soc_amd_sdw_common.h b/sound/soc/amd/acp/soc_amd_sdw_common.h index 1f24e0e06487..3930cc46fa58 100644 --- a/sound/soc/amd/acp/soc_amd_sdw_common.h +++ b/sound/soc/amd/acp/soc_amd_sdw_common.h @@ -21,6 +21,8 @@ #define ACP63_PCI_REV 0x63 #define ACP70_PCI_REV 0x70 #define ACP71_PCI_REV 0x71 +#define ACP72_PCI_REV 0x72 + #define SOC_JACK_JDSRC(quirk) ((quirk) & GENMASK(3, 0)) #define ASOC_SDW_FOUR_SPK BIT(4) #define ASOC_SDW_ACP_DMIC BIT(5) diff --git a/sound/soc/amd/ps/acp63.h b/sound/soc/amd/ps/acp63.h index d7c994e26e4d..90fc016dac0b 100644 --- a/sound/soc/amd/ps/acp63.h +++ b/sound/soc/amd/ps/acp63.h @@ -14,6 +14,7 @@ #define ACP63_PCI_REV 0x63 #define ACP70_PCI_REV 0x70 #define ACP71_PCI_REV 0x71 +#define ACP72_PCI_REV 0x72 #define ACP_SOFT_RESET_SOFTRESET_AUDDONE_MASK 0x00010001 #define ACP63_PGFSM_CNTL_POWER_ON_MASK 1 diff --git a/sound/soc/amd/ps/pci-ps.c b/sound/soc/amd/ps/pci-ps.c index 7936b3173632..c62299b29204 100644 --- a/sound/soc/amd/ps/pci-ps.c +++ b/sound/soc/amd/ps/pci-ps.c @@ -117,6 +117,7 @@ static short int check_and_handle_sdw_dma_irq(struct acp63_dev_data *adata, u32 break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: adata->acp70_sdw0_dma_intr_stat[stream_id] = 1; break; } @@ -141,6 +142,7 @@ static short int check_and_handle_sdw_dma_irq(struct acp63_dev_data *adata, u32 break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: if (ext_intr_stat1 & ACP70_P1_SDW_DMA_IRQ_MASK) { for (index = ACP70_P1_AUDIO2_RX_THRESHOLD; index <= ACP70_P1_AUDIO0_TX_THRESHOLD; index++) { @@ -552,6 +554,7 @@ static int acp_hw_init_ops(struct acp63_dev_data *adata, struct pci_dev *pci) break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: acp70_hw_init_ops(adata->hw_ops); break; default: @@ -581,6 +584,7 @@ static int snd_acp63_probe(struct pci_dev *pci, case ACP63_PCI_REV: case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: break; default: dev_dbg(&pci->dev, "acp63/acp70/acp71 pci device not found\n"); diff --git a/sound/soc/amd/ps/ps-sdw-dma.c b/sound/soc/amd/ps/ps-sdw-dma.c index 1b933a017c06..5449323e2728 100644 --- a/sound/soc/amd/ps/ps-sdw-dma.c +++ b/sound/soc/amd/ps/ps-sdw-dma.c @@ -269,6 +269,7 @@ static int acp63_configure_sdw_ringbuffer(void __iomem *acp_base, u32 stream_id, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: switch (manager_instance) { case ACP_SDW0: reg_dma_size = acp70_sdw0_dma_reg[stream_id].reg_dma_size; @@ -382,6 +383,7 @@ static int acp63_sdw_dma_hw_params(struct snd_soc_component *component, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: switch (stream->instance) { case ACP_SDW0: sdw_data->acp70_sdw0_dma_stream[stream_id] = substream; @@ -451,6 +453,7 @@ static u64 acp63_sdw_get_byte_count(struct acp_sdw_dma_stream *stream, void __io break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: switch (stream->instance) { case ACP_SDW0: pos_low_reg = acp70_sdw0_dma_reg[stream->stream_id].pos_low_reg; @@ -529,6 +532,7 @@ static int acp63_sdw_dma_close(struct snd_soc_component *component, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: switch (stream->instance) { case ACP_SDW0: sdw_data->acp70_sdw0_dma_stream[stream->stream_id] = NULL; @@ -574,6 +578,7 @@ static int acp63_sdw_dma_enable(struct snd_pcm_substream *substream, break; case ACP70_PCI_REV: case ACP71_PCI_REV: + case ACP72_PCI_REV: switch (stream->instance) { case ACP_SDW0: sdw_dma_en_reg = acp70_sdw0_dma_enable_reg[stream_id]; diff --git a/sound/soc/codecs/wm8962.c b/sound/soc/codecs/wm8962.c index 7bf6b88c056c..08c8ec3aeb44 100644 --- a/sound/soc/codecs/wm8962.c +++ b/sound/soc/codecs/wm8962.c @@ -82,6 +82,7 @@ struct wm8962_priv { #endif int irq; + bool master_flag; }; /* We can't use the same notifier block for more than one supply and @@ -2715,6 +2716,7 @@ static int wm8962_set_dai_sysclk(struct snd_soc_dai *dai, int clk_id, static int wm8962_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) { struct snd_soc_component *component = dai->component; + struct wm8962_priv *wm8962 = snd_soc_component_get_drvdata(component); int aif0 = 0; switch (fmt & SND_SOC_DAIFMT_FORMAT_MASK) { @@ -2761,9 +2763,11 @@ static int wm8962_set_dai_fmt(struct snd_soc_dai *dai, unsigned int fmt) return -EINVAL; } + wm8962->master_flag = false; switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { case SND_SOC_DAIFMT_CBP_CFP: aif0 |= WM8962_MSTR; + wm8962->master_flag = true; break; case SND_SOC_DAIFMT_CBC_CFC: break; @@ -3903,6 +3907,9 @@ static int wm8962_runtime_resume(struct device *dev) WM8962_BIAS_ENA | WM8962_VMID_SEL_MASK, WM8962_BIAS_ENA | 0x180); + if (wm8962->master_flag) + regmap_update_bits(wm8962->regmap, WM8962_AUDIO_INTERFACE_0, + WM8962_MSTR, WM8962_MSTR); msleep(5); return 0; @@ -3916,6 +3923,10 @@ static int wm8962_runtime_suspend(struct device *dev) { struct wm8962_priv *wm8962 = dev_get_drvdata(dev); + if (wm8962->master_flag) + regmap_update_bits(wm8962->regmap, WM8962_AUDIO_INTERFACE_0, + WM8962_MSTR, 0); + regmap_update_bits(wm8962->regmap, WM8962_PWR_MGMT_1, WM8962_VMID_SEL_MASK | WM8962_BIAS_ENA, 0); diff --git a/sound/soc/intel/avs/core.c b/sound/soc/intel/avs/core.c index 7af324753673..5ebadba07ecc 100644 --- a/sound/soc/intel/avs/core.c +++ b/sound/soc/intel/avs/core.c @@ -445,6 +445,8 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) adev = devm_kzalloc(dev, sizeof(*adev), GFP_KERNEL); if (!adev) return -ENOMEM; + bus = &adev->base.core; + ret = avs_bus_init(adev, pci, id); if (ret < 0) { dev_err(dev, "failed to init avs bus: %d\n", ret); @@ -455,7 +457,6 @@ static int avs_pci_probe(struct pci_dev *pci, const struct pci_device_id *id) if (ret < 0) return ret; - bus = &adev->base.core; bus->addr = pci_resource_start(pci, 0); bus->remap_addr = pci_ioremap_bar(pci, 0); if (!bus->remap_addr) { diff --git a/sound/soc/intel/boards/sof_sdw.c b/sound/soc/intel/boards/sof_sdw.c index c639df2cacdd..f997b2dc221b 100644 --- a/sound/soc/intel/boards/sof_sdw.c +++ b/sound/soc/intel/boards/sof_sdw.c @@ -741,6 +741,14 @@ static const struct dmi_system_id sof_sdw_quirk_table[] = { }, .driver_data = (void *)(SOC_SDW_CODEC_SPKR), }, + { + .callback = sof_sdw_quirk_cb, + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Alienware"), + DMI_EXACT_MATCH(DMI_PRODUCT_SKU, "0CCC") + }, + .driver_data = (void *)(SOC_SDW_CODEC_SPKR), + }, /* Pantherlake devices*/ { .callback = sof_sdw_quirk_cb, diff --git a/sound/soc/sof/amd/acp-loader.c b/sound/soc/sof/amd/acp-loader.c index ea105227227d..98324bbade15 100644 --- a/sound/soc/sof/amd/acp-loader.c +++ b/sound/soc/sof/amd/acp-loader.c @@ -65,7 +65,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t dma_size = page_count * ACP_PAGE_SIZE; adata->bin_buf = dma_alloc_coherent(&pci->dev, dma_size, &adata->sha_dma_addr, - GFP_ATOMIC); + GFP_KERNEL); if (!adata->bin_buf) return -ENOMEM; } @@ -77,7 +77,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t adata->data_buf = dma_alloc_coherent(&pci->dev, ACP_DEFAULT_DRAM_LENGTH, &adata->dma_addr, - GFP_ATOMIC); + GFP_KERNEL); if (!adata->data_buf) return -ENOMEM; } @@ -90,7 +90,7 @@ int acp_dsp_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_t adata->sram_data_buf = dma_alloc_coherent(&pci->dev, ACP_DEFAULT_SRAM_LENGTH, &adata->sram_dma_addr, - GFP_ATOMIC); + GFP_KERNEL); if (!adata->sram_data_buf) return -ENOMEM; } diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c index 7132916aa253..71a18f156de2 100644 --- a/sound/soc/sof/amd/acp.c +++ b/sound/soc/sof/amd/acp.c @@ -59,6 +59,7 @@ static void init_dma_descriptor(struct acp_dev_data *adata) switch (acp_data->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: acp_dma_desc_base_addr = ACP70_DMA_DESC_BASE_ADDR; acp_dma_desc_max_num_dscr = ACP70_DMA_DESC_MAX_NUM_DSCR; break; @@ -99,6 +100,7 @@ static int config_dma_channel(struct acp_dev_data *adata, unsigned int ch, switch (acp_data->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: acp_dma_cntl_0 = ACP70_DMA_CNTL_0; acp_dma_ch_rst_sts = ACP70_DMA_CH_RST_STS; acp_dma_dscr_err_sts_0 = ACP70_DMA_ERR_STS_0; @@ -339,6 +341,7 @@ int acp_dma_status(struct acp_dev_data *adata, unsigned char ch) switch (adata->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: acp_dma_ch_sts = ACP70_DMA_CH_STS; break; default: @@ -522,6 +525,7 @@ static irqreturn_t acp_irq_handler(int irq, void *dev_id) switch (adata->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: wake_irq_flag = amd_sof_check_and_handle_acp70_sdw_wake_irq(sdev); break; } @@ -559,6 +563,7 @@ static int acp_power_on(struct snd_sof_dev *sdev) break; case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: acp_pgfsm_status_mask = ACP70_PGFSM_STATUS_MASK; acp_pgfsm_cntl_mask = ACP70_PGFSM_CNTL_POWER_ON_MASK; break; @@ -661,6 +666,7 @@ static int acp_init(struct snd_sof_dev *sdev) switch (acp_data->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: sdw0_wake_en = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP70_SW0_WAKE_EN); sdw1_wake_en = snd_sof_dsp_read(sdev, ACP_DSP_BAR, ACP70_SW1_WAKE_EN); if (sdw0_wake_en || sdw1_wake_en) @@ -712,6 +718,7 @@ int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state) switch (acp_data->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: enable = true; break; } @@ -738,6 +745,7 @@ int amd_sof_acp_resume(struct snd_sof_dev *sdev) switch (acp_data->pci_rev) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP70_PME_EN, 1); break; } diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h index d3c5b2386cdf..2b7ea8c64106 100644 --- a/sound/soc/sof/amd/acp.h +++ b/sound/soc/sof/amd/acp.h @@ -75,6 +75,7 @@ #define ACP63_PCI_ID 0x63 #define ACP70_PCI_ID 0x70 #define ACP71_PCI_ID 0x71 +#define ACP72_PCI_ID 0x72 #define HOST_BRIDGE_CZN 0x1630 #define HOST_BRIDGE_VGH 0x1645 diff --git a/sound/soc/sof/amd/pci-acp70.c b/sound/soc/sof/amd/pci-acp70.c index 51d36d43c42b..3523c9a92a94 100644 --- a/sound/soc/sof/amd/pci-acp70.c +++ b/sound/soc/sof/amd/pci-acp70.c @@ -77,6 +77,7 @@ static int acp70_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_ switch (pci->revision) { case ACP70_PCI_ID: case ACP71_PCI_ID: + case ACP72_PCI_ID: break; default: return -ENODEV; diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index dc1d21de4ab7..4f27f8c8debf 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -266,9 +266,10 @@ config SND_SOC_SOF_METEORLAKE config SND_SOC_SOF_INTEL_LNL tristate + select SOUNDWIRE_INTEL if SND_SOC_SOF_INTEL_SOUNDWIRE != n select SND_SOC_SOF_HDA_GENERIC select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE - select SND_SOF_SOF_HDA_SDW_BPT if SND_SOC_SOF_INTEL_SOUNDWIRE + select SND_SOF_SOF_HDA_SDW_BPT if SND_SOC_SOF_INTEL_SOUNDWIRE != n select SND_SOC_SOF_IPC4 select SND_SOC_SOF_INTEL_MTL diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index b24ee38fad72..bff92505e408 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -1336,11 +1336,10 @@ static void retire_capture_urb(struct snd_usb_substream *subs, for (i = 0; i < urb->number_of_packets; i++) { cp = (unsigned char *)urb->transfer_buffer + urb->iso_frame_desc[i].offset + subs->pkt_offset_adj; - if (urb->iso_frame_desc[i].status && printk_ratelimit()) { - dev_dbg(&subs->dev->dev, "frame %d active: %d\n", - i, urb->iso_frame_desc[i].status); - // continue; - } + if (urb->iso_frame_desc[i].status) + dev_dbg_ratelimited(&subs->dev->dev, + "frame %d active: %d\n", i, + urb->iso_frame_desc[i].status); bytes = urb->iso_frame_desc[i].actual_length; if (subs->stream_offset_adj > 0) { unsigned int adj = min(subs->stream_offset_adj, bytes); diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c index cc54539c6030..01f49555c5f6 100644 --- a/sound/x86/intel_hdmi_audio.c +++ b/sound/x86/intel_hdmi_audio.c @@ -1765,7 +1765,7 @@ static int __hdmi_lpe_audio_probe(struct platform_device *pdev) /* setup private data which can be retrieved when required */ pcm->private_data = ctx; pcm->info_flags = 0; - strscpy(pcm->name, card->shortname, strlen(card->shortname)); + strscpy(pcm->name, card->shortname, sizeof(pcm->name)); /* setup the ops for playback */ snd_pcm_set_ops(pcm, SNDRV_PCM_STREAM_PLAYBACK, &had_pcm_ops); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index fb4d92c5c339..8f5a81b672e1 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -10965,11 +10965,14 @@ struct bpf_link *bpf_program__attach_perf_event_opts(const struct bpf_program *p } link->link.fd = pfd; } - if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { - err = -errno; - pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", - prog->name, pfd, errstr(err)); - goto err_out; + + if (!OPTS_GET(opts, dont_enable, false)) { + if (ioctl(pfd, PERF_EVENT_IOC_ENABLE, 0) < 0) { + err = -errno; + pr_warn("prog '%s': failed to enable perf_event FD %d: %s\n", + prog->name, pfd, errstr(err)); + goto err_out; + } } return &link->link; diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index d1cf813a057b..455a957cb702 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -499,9 +499,11 @@ struct bpf_perf_event_opts { __u64 bpf_cookie; /* don't use BPF link when attach BPF program */ bool force_ioctl_attach; + /* don't automatically enable the event */ + bool dont_enable; size_t :0; }; -#define bpf_perf_event_opts__last_field force_ioctl_attach +#define bpf_perf_event_opts__last_field dont_enable LIBBPF_API struct bpf_link * bpf_program__attach_perf_event(const struct bpf_program *prog, int pfd); diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c index d0e013eeb0f7..a0b11f35395f 100644 --- a/tools/perf/util/bpf-filter.c +++ b/tools/perf/util/bpf-filter.c @@ -451,6 +451,8 @@ int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target) struct bpf_link *link; struct perf_bpf_filter_entry *entry; bool needs_idx_hash = !target__has_cpu(target); + DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts, + .dont_enable = true); entry = calloc(MAX_FILTERS, sizeof(*entry)); if (entry == NULL) @@ -522,7 +524,8 @@ int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target) prog = skel->progs.perf_sample_filter; for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { - link = bpf_program__attach_perf_event(prog, FD(evsel, x, y)); + link = bpf_program__attach_perf_event_opts(prog, FD(evsel, x, y), + &pe_opts); if (IS_ERR(link)) { pr_err("Failed to attach perf sample-filter program\n"); ret = PTR_ERR(link); diff --git a/tools/testing/selftests/drivers/net/Makefile b/tools/testing/selftests/drivers/net/Makefile index 3556f3563e08..984ece05f7f9 100644 --- a/tools/testing/selftests/drivers/net/Makefile +++ b/tools/testing/selftests/drivers/net/Makefile @@ -11,6 +11,7 @@ TEST_GEN_FILES := \ TEST_PROGS := \ napi_id.py \ + napi_threaded.py \ netcons_basic.sh \ netcons_cmdline.sh \ netcons_fragmented_msg.sh \ diff --git a/tools/testing/selftests/drivers/net/napi_threaded.py b/tools/testing/selftests/drivers/net/napi_threaded.py new file mode 100755 index 000000000000..b2698db39817 --- /dev/null +++ b/tools/testing/selftests/drivers/net/napi_threaded.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 + +""" +Test napi threaded states. +""" + +from lib.py import ksft_run, ksft_exit +from lib.py import ksft_eq, ksft_ne, ksft_ge +from lib.py import NetDrvEnv, NetdevFamily +from lib.py import cmd, defer, ethtool + + +def _assert_napi_threaded_enabled(nl, napi_id) -> None: + napi = nl.napi_get({'id': napi_id}) + ksft_eq(napi['threaded'], 'enabled') + ksft_ne(napi.get('pid'), None) + + +def _assert_napi_threaded_disabled(nl, napi_id) -> None: + napi = nl.napi_get({'id': napi_id}) + ksft_eq(napi['threaded'], 'disabled') + ksft_eq(napi.get('pid'), None) + + +def _set_threaded_state(cfg, threaded) -> None: + cmd(f"echo {threaded} > /sys/class/net/{cfg.ifname}/threaded") + + +def _setup_deferred_cleanup(cfg) -> None: + combined = ethtool(f"-l {cfg.ifname}", json=True)[0].get("combined", 0) + ksft_ge(combined, 2) + defer(ethtool, f"-L {cfg.ifname} combined {combined}") + + threaded = cmd(f"cat /sys/class/net/{cfg.ifname}/threaded").stdout + defer(_set_threaded_state, cfg, threaded) + + +def enable_dev_threaded_disable_napi_threaded(cfg, nl) -> None: + """ + Test that when napi threaded is enabled at device level and + then disabled at napi level for one napi, the threaded state + of all napis is preserved after a change in number of queues. + """ + + napis = nl.napi_get({'ifindex': cfg.ifindex}, dump=True) + ksft_ge(len(napis), 2) + + napi0_id = napis[0]['id'] + napi1_id = napis[1]['id'] + + _setup_deferred_cleanup(cfg) + + # set threaded + _set_threaded_state(cfg, 1) + + # check napi threaded is set for both napis + _assert_napi_threaded_enabled(nl, napi0_id) + _assert_napi_threaded_enabled(nl, napi1_id) + + # disable threaded for napi1 + nl.napi_set({'id': napi1_id, 'threaded': 'disabled'}) + + cmd(f"ethtool -L {cfg.ifname} combined 1") + cmd(f"ethtool -L {cfg.ifname} combined 2") + _assert_napi_threaded_enabled(nl, napi0_id) + _assert_napi_threaded_disabled(nl, napi1_id) + + +def change_num_queues(cfg, nl) -> None: + """ + Test that when napi threaded is enabled at device level, + the napi threaded state is preserved after a change in + number of queues. + """ + + napis = nl.napi_get({'ifindex': cfg.ifindex}, dump=True) + ksft_ge(len(napis), 2) + + napi0_id = napis[0]['id'] + napi1_id = napis[1]['id'] + + _setup_deferred_cleanup(cfg) + + # set threaded + _set_threaded_state(cfg, 1) + + # check napi threaded is set for both napis + _assert_napi_threaded_enabled(nl, napi0_id) + _assert_napi_threaded_enabled(nl, napi1_id) + + cmd(f"ethtool -L {cfg.ifname} combined 1") + cmd(f"ethtool -L {cfg.ifname} combined 2") + + # check napi threaded is set for both napis + _assert_napi_threaded_enabled(nl, napi0_id) + _assert_napi_threaded_enabled(nl, napi1_id) + + +def main() -> None: + """ Ksft boiler plate main """ + + with NetDrvEnv(__file__, queue_count=2) as cfg: + ksft_run([change_num_queues, + enable_dev_threaded_disable_napi_threaded], + args=(cfg, NetdevFamily())) + ksft_exit() + + +if __name__ == "__main__": + main() diff --git a/tools/testing/selftests/drivers/net/netdevsim/nexthop.sh b/tools/testing/selftests/drivers/net/netdevsim/nexthop.sh index e8e0dc088d6a..01d0c044a5fc 100755 --- a/tools/testing/selftests/drivers/net/netdevsim/nexthop.sh +++ b/tools/testing/selftests/drivers/net/netdevsim/nexthop.sh @@ -1053,6 +1053,6 @@ trap cleanup EXIT setup_prepare -tests_run +xfail_on_slow tests_run exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/packetdrill/ksft_runner.sh b/tools/testing/selftests/net/packetdrill/ksft_runner.sh index c5b01e1bd4c7..a7e790af38ff 100755 --- a/tools/testing/selftests/net/packetdrill/ksft_runner.sh +++ b/tools/testing/selftests/net/packetdrill/ksft_runner.sh @@ -35,24 +35,7 @@ failfunc=ktap_test_fail if [[ -n "${KSFT_MACHINE_SLOW}" ]]; then optargs+=('--tolerance_usecs=14000') - - # xfail tests that are known flaky with dbg config, not fixable. - # still run them for coverage (and expect 100% pass without dbg). - declare -ar xfail_list=( - "tcp_blocking_blocking-connect.pkt" - "tcp_blocking_blocking-read.pkt" - "tcp_eor_no-coalesce-retrans.pkt" - "tcp_fast_recovery_prr-ss.*.pkt" - "tcp_sack_sack-route-refresh-ip-tos.pkt" - "tcp_slow_start_slow-start-after-win-update.pkt" - "tcp_timestamping.*.pkt" - "tcp_user_timeout_user-timeout-probe.pkt" - "tcp_zerocopy_cl.*.pkt" - "tcp_zerocopy_epoll_.*.pkt" - "tcp_tcp_info_tcp-info-.*-limited.pkt" - ) - readonly xfail_regex="^($(printf '%s|' "${xfail_list[@]}"))$" - [[ "$script" =~ ${xfail_regex} ]] && failfunc=ktap_test_xfail + failfunc=ktap_test_xfail fi ktap_print_header diff --git a/tools/testing/selftests/net/test_neigh.sh b/tools/testing/selftests/net/test_neigh.sh index 388056472b5b..7c594bf6ead0 100755 --- a/tools/testing/selftests/net/test_neigh.sh +++ b/tools/testing/selftests/net/test_neigh.sh @@ -289,11 +289,11 @@ extern_valid_common() orig_base_reachable=$(ip -j ntable show name "$tbl_name" | jq '.[] | select(has("thresh1")) | .["base_reachable"]') run_cmd "ip ntable change name $tbl_name thresh1 10 base_reachable 10000" orig_gc_stale=$(ip -n "$ns1" -j ntable show name "$tbl_name" dev veth0 | jq '.[]["gc_stale"]') - run_cmd "ip -n $ns1 ntable change name $tbl_name dev veth0 gc_stale 5000" - # Wait orig_base_reachable/2 for the new interval to take effect. - run_cmd "sleep $(((orig_base_reachable / 1000) / 2 + 2))" + run_cmd "ip -n $ns1 ntable change name $tbl_name dev veth0 gc_stale 1000" run_cmd "ip -n $ns1 neigh add $ip_addr lladdr $mac nud stale dev veth0 extern_valid" run_cmd "ip -n $ns1 neigh add ${subnet}3 lladdr $mac nud stale dev veth0" + # Wait orig_base_reachable/2 for the new interval to take effect. + run_cmd "sleep $(((orig_base_reachable / 1000) / 2 + 2))" for i in {1..20}; do run_cmd "ip -n $ns1 neigh add ${subnet}$((i + 4)) nud none dev veth0" done diff --git a/tools/testing/selftests/net/vlan_hw_filter.sh b/tools/testing/selftests/net/vlan_hw_filter.sh index 0fb56baf28e4..e195d5cab6f7 100755 --- a/tools/testing/selftests/net/vlan_hw_filter.sh +++ b/tools/testing/selftests/net/vlan_hw_filter.sh @@ -55,10 +55,10 @@ test_vlan0_del_crash_01() { ip netns exec ${NETNS} ip link add bond0 type bond mode 0 ip netns exec ${NETNS} ip link add link bond0 name vlan0 type vlan id 0 protocol 802.1q ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter off - ip netns exec ${NETNS} ifconfig bond0 up + ip netns exec ${NETNS} ip link set dev bond0 up ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter on - ip netns exec ${NETNS} ifconfig bond0 down - ip netns exec ${NETNS} ifconfig bond0 up + ip netns exec ${NETNS} ip link set dev bond0 down + ip netns exec ${NETNS} ip link set dev bond0 up ip netns exec ${NETNS} ip link del vlan0 || fail "Please check vlan HW filter function" cleanup } @@ -68,11 +68,11 @@ test_vlan0_del_crash_02() { setup ip netns exec ${NETNS} ip link add bond0 type bond mode 0 ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter off - ip netns exec ${NETNS} ifconfig bond0 up + ip netns exec ${NETNS} ip link set dev bond0 up ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter on ip netns exec ${NETNS} ip link add link bond0 name vlan0 type vlan id 0 protocol 802.1q - ip netns exec ${NETNS} ifconfig bond0 down - ip netns exec ${NETNS} ifconfig bond0 up + ip netns exec ${NETNS} ip link set dev bond0 down + ip netns exec ${NETNS} ip link set dev bond0 up ip netns exec ${NETNS} ip link del vlan0 || fail "Please check vlan HW filter function" cleanup } @@ -84,9 +84,9 @@ test_vlan0_del_crash_03() { ip netns exec ${NETNS} ip link add bond0 type bond mode 0 ip netns exec ${NETNS} ip link add link bond0 name vlan0 type vlan id 0 protocol 802.1q ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter off - ip netns exec ${NETNS} ifconfig bond0 up + ip netns exec ${NETNS} ip link set dev bond0 up ip netns exec ${NETNS} ethtool -K bond0 rx-vlan-filter on - ip netns exec ${NETNS} ifconfig bond0 down + ip netns exec ${NETNS} ip link set dev bond0 down ip netns exec ${NETNS} ip link del vlan0 || fail "Please check vlan HW filter function" cleanup } |