diff options
958 files changed, 50138 insertions, 19793 deletions
diff --git a/Documentation/bpf/btf.rst b/Documentation/bpf/btf.rst index 9ad4218a751f..d0ec40d00c28 100644 --- a/Documentation/bpf/btf.rst +++ b/Documentation/bpf/btf.rst @@ -86,6 +86,7 @@ sequentially and type id is assigned to each recognized type starting from id #define BTF_KIND_DATASEC 15 /* Section */ #define BTF_KIND_FLOAT 16 /* Floating point */ #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ + #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ Note that the type section encodes debug info, not just pure types. ``BTF_KIND_FUNC`` is not a type, and it represents a defined subprogram. @@ -107,7 +108,7 @@ Each type contains the following common data:: * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO and DECL_TAG. + * FUNC, FUNC_PROTO, DECL_TAG and TYPE_TAG. * "type" is a type_id referring to another type. */ union { @@ -492,6 +493,16 @@ the attribute is applied to a ``struct``/``union`` member or a ``func`` argument, and ``btf_decl_tag.component_idx`` should be a valid index (starting from 0) pointing to a member or an argument. +2.2.17 BTF_KIND_TYPE_TAG +~~~~~~~~~~~~~~~~~~~~~~~~ + +``struct btf_type`` encoding requirement: + * ``name_off``: offset to a non-empty string + * ``info.kind_flag``: 0 + * ``info.kind``: BTF_KIND_TYPE_TAG + * ``info.vlen``: 0 + * ``type``: the type with ``btf_type_tag`` attribute + 3. BTF Kernel API ***************** diff --git a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml index c435c9f369a4..9a785bbaafb7 100644 --- a/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml +++ b/Documentation/devicetree/bindings/firmware/intel,ixp4xx-network-processing-engine.yaml @@ -37,6 +37,20 @@ properties: should be named with the instance number of the NPE engine used for the crypto engine. + "#address-cells": + const: 1 + + "#size-cells": + const: 0 + +patternProperties: + hss@[0-9]+$: + $ref: /schemas/net/intel,ixp4xx-hss.yaml# + type: object + description: Optional node for the High Speed Serial link (HSS), the + node should be named with the instance number of the NPE engine + used for the HSS. + required: - compatible - reg @@ -45,9 +59,30 @@ additionalProperties: false examples: - | + #include <dt-bindings/gpio/gpio.h> + npe: npe@c8006000 { compatible = "intel,ixp4xx-network-processing-engine"; reg = <0xc8006000 0x1000>, <0xc8007000 0x1000>, <0xc8008000 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + + hss@0 { + compatible = "intel,ixp4xx-hss"; + reg = <0>; + intel,npe-handle = <&npe 0>; + intel,queue-chl-rxtrig = <&qmgr 12>; + intel,queue-chl-txready = <&qmgr 34>; + intel,queue-pkt-rx = <&qmgr 13>; + intel,queue-pkt-tx = <&qmgr 14>, <&qmgr 15>, <&qmgr 16>, <&qmgr 17>; + intel,queue-pkt-rxfree = <&qmgr 18>, <&qmgr 19>, <&qmgr 20>, <&qmgr 21>; + intel,queue-pkt-txdone = <&qmgr 22>; + cts-gpios = <&gpio0 10 GPIO_ACTIVE_LOW>; + rts-gpios = <&gpio0 14 GPIO_ACTIVE_LOW>; + dcd-gpios = <&gpio0 6 GPIO_ACTIVE_LOW>; + dtr-gpios = <&gpio_74 2 GPIO_ACTIVE_LOW>; + clk-internal-gpios = <&gpio_74 0 GPIO_ACTIVE_HIGH>; + }; crypto { compatible = "intel,ixp4xx-crypto"; diff --git a/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml b/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml index a95960ee3feb..c93fe9d3ea82 100644 --- a/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml +++ b/Documentation/devicetree/bindings/net/can/allwinner,sun4i-a10-can.yaml @@ -17,6 +17,7 @@ properties: - const: allwinner,sun7i-a20-can - const: allwinner,sun4i-a10-can - const: allwinner,sun4i-a10-can + - const: allwinner,sun8i-r40-can reg: maxItems: 1 @@ -27,6 +28,19 @@ properties: clocks: maxItems: 1 + resets: + maxItems: 1 + +if: + properties: + compatible: + contains: + const: allwinner,sun8i-r40-can + +then: + required: + - resets + required: - compatible - reg @@ -47,5 +61,15 @@ examples: interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>; clocks = <&ccu CLK_APB1_CAN>; }; + - | + #define RST_BUS_CAN 68 + #define CLK_BUS_CAN 91 + can1: can@1c2bc00 { + compatible = "allwinner,sun8i-r40-can"; + reg = <0x01c2bc00 0x400>; + interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_CAN>; + resets = <&ccu RST_BUS_CAN>; + }; ... diff --git a/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml new file mode 100644 index 000000000000..702df848a71d --- /dev/null +++ b/Documentation/devicetree/bindings/net/dsa/dsa-port.yaml @@ -0,0 +1,77 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/dsa/dsa-port.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Ethernet Switch port Device Tree Bindings + +maintainers: + - Andrew Lunn <andrew@lunn.ch> + - Florian Fainelli <f.fainelli@gmail.com> + - Vivien Didelot <vivien.didelot@gmail.com> + +description: + Ethernet switch port Description + +allOf: + - $ref: "http://devicetree.org/schemas/net/ethernet-controller.yaml#" + +properties: + reg: + description: Port number + + label: + description: + Describes the label associated with this port, which will become + the netdev name + $ref: /schemas/types.yaml#/definitions/string + + link: + description: + Should be a list of phandles to other switch's DSA port. This + port is used as the outgoing port towards the phandle ports. The + full routing information must be given, not just the one hop + routes to neighbouring switches + $ref: /schemas/types.yaml#/definitions/phandle-array + + ethernet: + description: + Should be a phandle to a valid Ethernet device node. This host + device is what the switch port is connected to + $ref: /schemas/types.yaml#/definitions/phandle + + dsa-tag-protocol: + description: + Instead of the default, the switch will use this tag protocol if + possible. Useful when a device supports multiple protocols and + the default is incompatible with the Ethernet device. + enum: + - dsa + - edsa + - ocelot + - ocelot-8021q + - seville + + phy-handle: true + + phy-mode: true + + fixed-link: true + + mac-address: true + + sfp: true + + managed: true + + rx-internal-delay-ps: true + + tx-internal-delay-ps: true + +required: + - reg + +additionalProperties: true + +... diff --git a/Documentation/devicetree/bindings/net/dsa/dsa.yaml b/Documentation/devicetree/bindings/net/dsa/dsa.yaml index 2ad7f79ad371..b9d48e357e77 100644 --- a/Documentation/devicetree/bindings/net/dsa/dsa.yaml +++ b/Documentation/devicetree/bindings/net/dsa/dsa.yaml @@ -46,65 +46,9 @@ patternProperties: type: object description: Ethernet switch ports - allOf: - - $ref: "http://devicetree.org/schemas/net/ethernet-controller.yaml#" + $ref: dsa-port.yaml# - properties: - reg: - description: Port number - - label: - description: - Describes the label associated with this port, which will become - the netdev name - $ref: /schemas/types.yaml#/definitions/string - - link: - description: - Should be a list of phandles to other switch's DSA port. This - port is used as the outgoing port towards the phandle ports. The - full routing information must be given, not just the one hop - routes to neighbouring switches - $ref: /schemas/types.yaml#/definitions/phandle-array - - ethernet: - description: - Should be a phandle to a valid Ethernet device node. This host - device is what the switch port is connected to - $ref: /schemas/types.yaml#/definitions/phandle - - dsa-tag-protocol: - description: - Instead of the default, the switch will use this tag protocol if - possible. Useful when a device supports multiple protocols and - the default is incompatible with the Ethernet device. - enum: - - dsa - - edsa - - ocelot - - ocelot-8021q - - seville - - phy-handle: true - - phy-mode: true - - fixed-link: true - - mac-address: true - - sfp: true - - managed: true - - rx-internal-delay-ps: true - - tx-internal-delay-ps: true - - required: - - reg - - additionalProperties: false + unevaluatedProperties: false oneOf: - required: diff --git a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml index 48de0ace265d..89c21b289447 100644 --- a/Documentation/devicetree/bindings/net/dsa/qca8k.yaml +++ b/Documentation/devicetree/bindings/net/dsa/qca8k.yaml @@ -99,40 +99,9 @@ patternProperties: type: object description: Ethernet switch ports - properties: - reg: - description: Port number - - label: - description: - Describes the label associated with this port, which will become - the netdev name - $ref: /schemas/types.yaml#/definitions/string - - link: - description: - Should be a list of phandles to other switch's DSA port. This - port is used as the outgoing port towards the phandle ports. The - full routing information must be given, not just the one hop - routes to neighbouring switches - $ref: /schemas/types.yaml#/definitions/phandle-array - - ethernet: - description: - Should be a phandle to a valid Ethernet device node. This host - device is what the switch port is connected to - $ref: /schemas/types.yaml#/definitions/phandle - - phy-handle: true - - phy-mode: true - - fixed-link: true - - mac-address: true - - sfp: true + $ref: dsa-port.yaml# + properties: qca,sgmii-rxclk-falling-edge: $ref: /schemas/types.yaml#/definitions/flag description: @@ -154,10 +123,7 @@ patternProperties: SGMII on the QCA8337, it is advised to set this unless a communication issue is observed. - required: - - reg - - additionalProperties: false + unevaluatedProperties: false oneOf: - required: diff --git a/Documentation/devicetree/bindings/net/engleder,tsnep.yaml b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml new file mode 100644 index 000000000000..d0e1476e15b5 --- /dev/null +++ b/Documentation/devicetree/bindings/net/engleder,tsnep.yaml @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/engleder,tsnep.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TSN endpoint Ethernet MAC binding + +maintainers: + - Gerhard Engleder <gerhard@engleder-embedded.com> + +allOf: + - $ref: ethernet-controller.yaml# + +properties: + compatible: + const: engleder,tsnep + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + local-mac-address: true + + mac-address: true + + nvmem-cells: true + + nvmem-cells-names: true + + phy-connection-type: + enum: + - mii + - gmii + - rgmii + - rgmii-id + + phy-mode: true + + phy-handle: true + + mdio: + type: object + $ref: "mdio.yaml#" + description: optional node for embedded MDIO controller + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + axi { + #address-cells = <2>; + #size-cells = <2>; + tnsep0: ethernet@a0000000 { + compatible = "engleder,tsnep"; + reg = <0x0 0xa0000000 0x0 0x10000>; + interrupts = <0 89 1>; + interrupt-parent = <&gic>; + local-mac-address = [00 00 00 00 00 00]; + phy-mode = "rgmii"; + phy-handle = <&phy0>; + mdio { + #address-cells = <1>; + #size-cells = <0>; + suppress-preamble; + phy0: ethernet-phy@1 { + reg = <1>; + rxc-skew-ps = <1080>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml b/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml new file mode 100644 index 000000000000..4dcd53c3e0b4 --- /dev/null +++ b/Documentation/devicetree/bindings/net/intel,ixp4xx-hss.yaml @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +# Copyright 2021 Linaro Ltd. +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/net/intel,ixp4xx-hss.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Intel IXP4xx V.35 WAN High Speed Serial Link (HSS) + +maintainers: + - Linus Walleij <linus.walleij@linaro.org> + +description: | + The Intel IXP4xx HSS makes use of the IXP4xx NPE (Network + Processing Engine) and the IXP4xx Queue Manager to process + V.35 Wideband Modem (WAN) links. + +properties: + compatible: + const: intel,ixp4xx-hss + + reg: + maxItems: 1 + description: The HSS instance + + intel,npe-handle: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the NPE this HSS instance is using + and the instance to use in the second cell + + intel,queue-chl-rxtrig: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the RX trigger queue on the NPE + + intel,queue-chl-txready: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the TX ready queue on the NPE + + intel,queue-pkt-rx: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the packet RX queue on the NPE + + intel,queue-pkt-tx: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 4 + description: phandle to the packet TX0, TX1, TX2 and TX3 queues on the NPE + + intel,queue-pkt-rxfree: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 4 + description: phandle to the packet RXFREE0, RXFREE1, RXFREE2 and + RXFREE3 queues on the NPE + + intel,queue-pkt-txdone: + $ref: '/schemas/types.yaml#/definitions/phandle-array' + maxItems: 1 + description: phandle to the packet TXDONE queue on the NPE + + cts-gpios: + maxItems: 1 + description: Clear To Send (CTS) GPIO line + + rts-gpios: + maxItems: 1 + description: Ready To Send (RTS) GPIO line + + dcd-gpios: + maxItems: 1 + description: Data Carrier Detect (DCD) GPIO line + + dtr-gpios: + maxItems: 1 + description: Data Terminal Ready (DTR) GPIO line + + clk-internal-gpios: + maxItems: 1 + description: Clock internal GPIO line, driving this high will make the HSS + use internal clocking as opposed to external clocking + +required: + - compatible + - reg + - intel,npe-handle + - intel,queue-chl-rxtrig + - intel,queue-chl-txready + - intel,queue-pkt-rx + - intel,queue-pkt-tx + - intel,queue-pkt-rxfree + - intel,queue-pkt-txdone + - cts-gpios + - rts-gpios + - dcd-gpios + - dtr-gpios + - clk-internal-gpios + +additionalProperties: false diff --git a/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml b/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml new file mode 100644 index 000000000000..5bee665d5fcf --- /dev/null +++ b/Documentation/devicetree/bindings/net/microchip,lan966x-switch.yaml @@ -0,0 +1,167 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/microchip,lan966x-switch.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Microchip Lan966x Ethernet switch controller + +maintainers: + - Horatiu Vultur <horatiu.vultur@microchip.com> + +description: | + The lan966x switch is a multi-port Gigabit AVB/TSN Ethernet Switch with + two integrated 10/100/1000Base-T PHYs. In addition to the integrated PHYs, + it supports up to 2RGMII/RMII, up to 3BASE-X/SERDES/2.5GBASE-X and up to + 2 Quad-SGMII/Quad-USGMII interfaces. + +properties: + $nodename: + pattern: "^switch@[0-9a-f]+$" + + compatible: + const: microchip,lan966x-switch + + reg: + items: + - description: cpu target + - description: general control block target + + reg-names: + items: + - const: cpu + - const: gcb + + interrupts: + minItems: 1 + items: + - description: register based extraction + - description: frame dma based extraction + + interrupt-names: + minItems: 1 + items: + - const: xtr + - const: fdma + + resets: + items: + - description: Reset controller used for switch core reset (soft reset) + - description: Reset controller used for releasing the phy from reset + + reset-names: + items: + - const: switch + - const: phy + + ethernet-ports: + type: object + + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + + additionalProperties: false + + patternProperties: + "^port@[0-9a-f]+$": + type: object + + $ref: "/schemas/net/ethernet-controller.yaml#" + unevaluatedProperties: false + + properties: + '#address-cells': + const: 1 + '#size-cells': + const: 0 + + reg: + description: + Switch port number + + phys: + description: + Phandle of a Ethernet SerDes PHY + + phy-mode: + description: + This specifies the interface used by the Ethernet SerDes towards + the PHY or SFP. + enum: + - gmii + - sgmii + - qsgmii + - 1000base-x + - 2500base-x + + phy-handle: + description: + Phandle of a Ethernet PHY. + + sfp: + description: + Phandle of an SFP. + + managed: true + + required: + - reg + - phys + - phy-mode + + oneOf: + - required: + - phy-handle + - required: + - sfp + - managed + +required: + - compatible + - reg + - reg-names + - interrupts + - interrupt-names + - resets + - reset-names + - ethernet-ports + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + switch: switch@e0000000 { + compatible = "microchip,lan966x-switch"; + reg = <0xe0000000 0x0100000>, + <0xe2000000 0x0800000>; + reg-names = "cpu", "gcb"; + interrupts = <GIC_SPI 30 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "xtr"; + resets = <&switch_reset 0>, <&phy_reset 0>; + reset-names = "switch", "phy"; + ethernet-ports { + #address-cells = <1>; + #size-cells = <0>; + + port0: port@0 { + reg = <0>; + phy-handle = <&phy0>; + phys = <&serdes 0 0>; + phy-mode = "gmii"; + }; + + port1: port@1 { + reg = <1>; + sfp = <&sfp_eth1>; + managed = "in-band-status"; + phys = <&serdes 2 4>; + phy-mode = "sgmii"; + }; + }; + }; + +... diff --git a/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml b/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml new file mode 100644 index 000000000000..b30544410d09 --- /dev/null +++ b/Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/net/qcom,bam-dmux.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm BAM Data Multiplexer + +maintainers: + - Stephan Gerhold <stephan@gerhold.net> + +description: | + The BAM Data Multiplexer provides access to the network data channels + of modems integrated into many older Qualcomm SoCs, e.g. Qualcomm MSM8916 + or MSM8974. It is built using a simple protocol layer on top of a DMA engine + (Qualcomm BAM DMA) and bidirectional interrupts to coordinate power control. + + Note that this schema does not directly describe a hardware block but rather + a firmware convention that combines several other hardware blocks (such as the + DMA engine). As such it is specific to a firmware version, not a particular + SoC or hardware version. + +properties: + compatible: + const: qcom,bam-dmux + + interrupts: + description: + Interrupts used by the modem to signal the AP. + Both interrupts must be declared as IRQ_TYPE_EDGE_BOTH. + items: + - description: Power control + - description: Power control acknowledgment + + interrupt-names: + items: + - const: pc + - const: pc-ack + + qcom,smem-states: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: State bits used by the AP to signal the modem. + items: + - description: Power control + - description: Power control acknowledgment + + qcom,smem-state-names: + description: Names for the state bits used by the AP to signal the modem. + items: + - const: pc + - const: pc-ack + + dmas: + items: + - description: TX DMA channel phandle + - description: RX DMA channel phandle + + dma-names: + items: + - const: tx + - const: rx + +required: + - compatible + - interrupts + - interrupt-names + - qcom,smem-states + - qcom,smem-state-names + - dmas + - dma-names + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + + mpss: remoteproc { + bam-dmux { + compatible = "qcom,bam-dmux"; + + interrupt-parent = <&modem_smsm>; + interrupts = <1 IRQ_TYPE_EDGE_BOTH>, <11 IRQ_TYPE_EDGE_BOTH>; + interrupt-names = "pc", "pc-ack"; + + qcom,smem-states = <&apps_smsm 1>, <&apps_smsm 11>; + qcom,smem-state-names = "pc", "pc-ack"; + + dmas = <&bam_dmux_dma 4>, <&bam_dmux_dma 5>; + dma-names = "tx", "rx"; + }; + }; diff --git a/Documentation/devicetree/bindings/vendor-prefixes.yaml b/Documentation/devicetree/bindings/vendor-prefixes.yaml index 66d6432fd781..5f4ca46bfb13 100644 --- a/Documentation/devicetree/bindings/vendor-prefixes.yaml +++ b/Documentation/devicetree/bindings/vendor-prefixes.yaml @@ -379,6 +379,8 @@ patternProperties: description: Silicon Laboratories (formerly Energy Micro AS) "^engicam,.*": description: Engicam S.r.l. + "^engleder,.*": + description: Engleder "^epcos,.*": description: EPCOS AG "^epfl,.*": diff --git a/Documentation/networking/bonding.rst b/Documentation/networking/bonding.rst index 31cfd7d674a6..e5a07cb48f68 100644 --- a/Documentation/networking/bonding.rst +++ b/Documentation/networking/bonding.rst @@ -421,6 +421,17 @@ arp_all_targets consider the slave up only when all of the arp_ip_targets are reachable +arp_missed_max + + Specifies the number of arp_interval monitor checks that must + fail in order for an interface to be marked down by the ARP monitor. + + In order to provide orderly failover semantics, backup interfaces + are permitted an extra monitor check (i.e., they must fail + arp_missed_max + 1 times before being marked down). + + The default value is 2, and the allowable range is 1 - 255. + downdelay Specifies the time, in milliseconds, to wait before disabling diff --git a/Documentation/networking/devlink/devlink-params.rst b/Documentation/networking/devlink/devlink-params.rst index 4878907e9232..b7dfe693a332 100644 --- a/Documentation/networking/devlink/devlink-params.rst +++ b/Documentation/networking/devlink/devlink-params.rst @@ -109,6 +109,9 @@ own name. - Boolean - When enabled, the device driver will instantiate VDPA networking specific auxiliary device of the devlink device. + * - ``enable_iwarp`` + - Boolean + - Enable handling of iWARP traffic in the device. * - ``internal_err_reset`` - Boolean - When enabled, the device driver will reset the device on internal diff --git a/Documentation/networking/ethtool-netlink.rst b/Documentation/networking/ethtool-netlink.rst index 7b598c7e3912..9d98e0511249 100644 --- a/Documentation/networking/ethtool-netlink.rst +++ b/Documentation/networking/ethtool-netlink.rst @@ -849,7 +849,7 @@ Request contents: Kernel response contents: - ==================================== ====== ========================== + ==================================== ====== =========================== ``ETHTOOL_A_RINGS_HEADER`` nested reply header ``ETHTOOL_A_RINGS_RX_MAX`` u32 max size of RX ring ``ETHTOOL_A_RINGS_RX_MINI_MAX`` u32 max size of RX mini ring @@ -859,7 +859,8 @@ Kernel response contents: ``ETHTOOL_A_RINGS_RX_MINI`` u32 size of RX mini ring ``ETHTOOL_A_RINGS_RX_JUMBO`` u32 size of RX jumbo ring ``ETHTOOL_A_RINGS_TX`` u32 size of TX ring - ==================================== ====== ========================== + ``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring + ==================================== ====== =========================== RINGS_SET @@ -869,13 +870,14 @@ Sets ring sizes like ``ETHTOOL_SRINGPARAM`` ioctl request. Request contents: - ==================================== ====== ========================== + ==================================== ====== =========================== ``ETHTOOL_A_RINGS_HEADER`` nested reply header ``ETHTOOL_A_RINGS_RX`` u32 size of RX ring ``ETHTOOL_A_RINGS_RX_MINI`` u32 size of RX mini ring ``ETHTOOL_A_RINGS_RX_JUMBO`` u32 size of RX jumbo ring ``ETHTOOL_A_RINGS_TX`` u32 size of TX ring - ==================================== ====== ========================== + ``ETHTOOL_A_RINGS_RX_BUF_LEN`` u32 size of buffers on the ring + ==================================== ====== =========================== Kernel checks that requested ring sizes do not exceed limits reported by driver. Driver may impose additional constraints and may not suspport all diff --git a/Documentation/networking/net_failover.rst b/Documentation/networking/net_failover.rst index e143ab79a960..3a662f2b4d6e 100644 --- a/Documentation/networking/net_failover.rst +++ b/Documentation/networking/net_failover.rst @@ -35,7 +35,7 @@ To support this, the hypervisor needs to enable VIRTIO_NET_F_STANDBY feature on the virtio-net interface and assign the same MAC address to both virtio-net and VF interfaces. -Here is an example XML snippet that shows such configuration. +Here is an example libvirt XML snippet that shows such configuration: :: <interface type='network'> @@ -45,18 +45,32 @@ Here is an example XML snippet that shows such configuration. <model type='virtio'/> <driver name='vhost' queues='4'/> <link state='down'/> - <address type='pci' domain='0x0000' bus='0x00' slot='0x0a' function='0x0'/> + <teaming type='persistent'/> + <alias name='ua-backup0'/> </interface> <interface type='hostdev' managed='yes'> <mac address='52:54:00:00:12:53'/> <source> <address type='pci' domain='0x0000' bus='0x42' slot='0x02' function='0x5'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/> + <teaming type='transient' persistent='ua-backup0'/> </interface> +In this configuration, the first device definition is for the virtio-net +interface and this acts as the 'persistent' device indicating that this +interface will always be plugged in. This is specified by the 'teaming' tag with +required attribute type having value 'persistent'. The link state for the +virtio-net device is set to 'down' to ensure that the 'failover' netdev prefers +the VF passthrough device for normal communication. The virtio-net device will +be brought UP during live migration to allow uninterrupted communication. + +The second device definition is for the VF passthrough interface. Here the +'teaming' tag is provided with type 'transient' indicating that this device may +periodically be unplugged. A second attribute - 'persistent' is provided and +points to the alias name declared for the virtio-net device. + Booting a VM with the above configuration will result in the following 3 -netdevs created in the VM. +interfaces created in the VM: :: 4: ens10: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000 @@ -65,13 +79,36 @@ netdevs created in the VM. valid_lft 42482sec preferred_lft 42482sec inet6 fe80::97d8:db2:8c10:b6d6/64 scope link valid_lft forever preferred_lft forever - 5: ens10nsby: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel master ens10 state UP group default qlen 1000 + 5: ens10nsby: <BROADCAST,MULTICAST> mtu 1500 qdisc fq_codel master ens10 state DOWN group default qlen 1000 link/ether 52:54:00:00:12:53 brd ff:ff:ff:ff:ff:ff 7: ens11: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq master ens10 state UP group default qlen 1000 link/ether 52:54:00:00:12:53 brd ff:ff:ff:ff:ff:ff -ens10 is the 'failover' master netdev, ens10nsby and ens11 are the slave -'standby' and 'primary' netdevs respectively. +Here, ens10 is the 'failover' master interface, ens10nsby is the slave 'standby' +virtio-net interface, and ens11 is the slave 'primary' VF passthrough interface. + +One point to note here is that some user space network configuration daemons +like systemd-networkd, ifupdown, etc, do not understand the 'net_failover' +device; and on the first boot, the VM might end up with both 'failover' device +and VF accquiring IP addresses (either same or different) from the DHCP server. +This will result in lack of connectivity to the VM. So some tweaks might be +needed to these network configuration daemons to make sure that an IP is +received only on the 'failover' device. + +Below is the patch snippet used with 'cloud-ifupdown-helper' script found on +Debian cloud images: + +:: + @@ -27,6 +27,8 @@ do_setup() { + local working="$cfgdir/.$INTERFACE" + local final="$cfgdir/$INTERFACE" + + + if [ -d "/sys/class/net/${INTERFACE}/master" ]; then exit 0; fi + + + if ifup --no-act "$INTERFACE" > /dev/null 2>&1; then + # interface is already known to ifupdown, no need to generate cfg + log "Skipping configuration generation for $INTERFACE" + Live Migration of a VM with SR-IOV VF & virtio-net in STANDBY mode ================================================================== @@ -80,40 +117,68 @@ net_failover also enables hypervisor controlled live migration to be supported with VMs that have direct attached SR-IOV VF devices by automatic failover to the paravirtual datapath when the VF is unplugged. -Here is a sample script that shows the steps to initiate live migration on -the source hypervisor. +Here is a sample script that shows the steps to initiate live migration from +the source hypervisor. Note: It is assumed that the VM is connected to a +software bridge 'br0' which has a single VF attached to it along with the vnet +device to the VM. This is not the VF that was passthrough'd to the VM (seen in +the vf.xml file). :: - # cat vf_xml + # cat vf.xml <interface type='hostdev' managed='yes'> <mac address='52:54:00:00:12:53'/> <source> <address type='pci' domain='0x0000' bus='0x42' slot='0x02' function='0x5'/> </source> - <address type='pci' domain='0x0000' bus='0x00' slot='0x0b' function='0x0'/> + <teaming type='transient' persistent='ua-backup0'/> </interface> - # Source Hypervisor + # Source Hypervisor migrate.sh #!/bin/bash - DOMAIN=fedora27-tap01 - PF=enp66s0f0 - VF_NUM=5 - TAP_IF=tap01 - VF_XML= + DOMAIN=vm-01 + PF=ens6np0 + VF=ens6v1 # VF attached to the bridge. + VF_NUM=1 + TAP_IF=vmtap01 # virtio-net interface in the VM. + VF_XML=vf.xml MAC=52:54:00:00:12:53 ZERO_MAC=00:00:00:00:00:00 + # Set the virtio-net interface up. virsh domif-setlink $DOMAIN $TAP_IF up - bridge fdb del $MAC dev $PF master - virsh detach-device $DOMAIN $VF_XML + + # Remove the VF that was passthrough'd to the VM. + virsh detach-device --live --config $DOMAIN $VF_XML + ip link set $PF vf $VF_NUM mac $ZERO_MAC - virsh migrate --live $DOMAIN qemu+ssh://$REMOTE_HOST/system + # Add FDB entry for traffic to continue going to the VM via + # the VF -> br0 -> vnet interface path. + bridge fdb add $MAC dev $VF + bridge fdb add $MAC dev $TAP_IF master + + # Migrate the VM + virsh migrate --live --persistent $DOMAIN qemu+ssh://$REMOTE_HOST/system + + # Clean up FDB entries after migration completes. + bridge fdb del $MAC dev $VF + bridge fdb del $MAC dev $TAP_IF master - # Destination Hypervisor +On the destination hypervisor, a shared bridge 'br0' is created before migration +starts, and a VF from the destination PF is added to the bridge. Similarly an +appropriate FDB entry is added. + +The following script is executed on the destination hypervisor once migration +completes, and it reattaches the VF to the VM and brings down the virtio-net +interface. + +:: + # reattach-vf.sh #!/bin/bash - virsh attach-device $DOMAIN $VF_XML - virsh domif-setlink $DOMAIN $TAP_IF down + bridge fdb del 52:54:00:00:12:53 dev ens36v0 + bridge fdb del 52:54:00:00:12:53 dev vmtap01 master + virsh attach-device --config --live vm01 vf.xml + virsh domif-setlink vm01 vmtap01 down diff --git a/Documentation/networking/phy.rst b/Documentation/networking/phy.rst index 571ba08386e7..d43da709bf40 100644 --- a/Documentation/networking/phy.rst +++ b/Documentation/networking/phy.rst @@ -237,6 +237,11 @@ negotiation results. Some of the interface modes are described below: +``PHY_INTERFACE_MODE_SMII`` + This is serial MII, clocked at 125MHz, supporting 100M and 10M speeds. + Some details can be found in + https://opencores.org/ocsvn/smii/smii/trunk/doc/SMII.pdf + ``PHY_INTERFACE_MODE_1000BASEX`` This defines the 1000BASE-X single-lane serdes link as defined by the 802.3 standard section 36. The link operates at a fixed bit rate of diff --git a/MAINTAINERS b/MAINTAINERS index 9eff63447569..9b6440e12607 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -12524,6 +12524,13 @@ L: netdev@vger.kernel.org S: Maintained F: drivers/net/ethernet/microchip/lan743x_* +MICROCHIP LAN966X ETHERNET DRIVER +M: Horatiu Vultur <horatiu.vultur@microchip.com> +M: UNGLinuxDriver@microchip.com +L: netdev@vger.kernel.org +S: Maintained +F: drivers/net/ethernet/microchip/lan966x/* + MICROCHIP LCDFB DRIVER M: Nicolas Ferre <nicolas.ferre@microchip.com> L: linux-fbdev@vger.kernel.org @@ -15727,6 +15734,14 @@ W: https://wireless.wiki.kernel.org/en/users/Drivers/ath9k F: Documentation/devicetree/bindings/net/wireless/qca,ath9k.yaml F: drivers/net/wireless/ath/ath9k/ +QUALCOMM BAM-DMUX WWAN NETWORK DRIVER +M: Stephan Gerhold <stephan@gerhold.net> +L: netdev@vger.kernel.org +L: linux-arm-msm@vger.kernel.org +S: Maintained +F: Documentation/devicetree/bindings/net/qcom,bam-dmux.yaml +F: drivers/net/wwan/qcom_bam_dmux.c + QUALCOMM CAMERA SUBSYSTEM DRIVER M: Robert Foss <robert.foss@linaro.org> M: Todor Tomov <todor.too@gmail.com> diff --git a/arch/arm/boot/dts/sun8i-r40.dtsi b/arch/arm/boot/dts/sun8i-r40.dtsi index 1d87fc0c24ee..c99c92f008a0 100644 --- a/arch/arm/boot/dts/sun8i-r40.dtsi +++ b/arch/arm/boot/dts/sun8i-r40.dtsi @@ -511,6 +511,16 @@ #interrupt-cells = <3>; #gpio-cells = <3>; + can_ph_pins: can-ph-pins { + pins = "PH20", "PH21"; + function = "can"; + }; + + can_pa_pins: can-pa-pins { + pins = "PA16", "PA17"; + function = "can"; + }; + clk_out_a_pin: clk-out-a-pin { pins = "PI12"; function = "clk_out_a"; @@ -926,6 +936,15 @@ #size-cells = <0>; }; + can0: can@1c2bc00 { + compatible = "allwinner,sun8i-r40-can"; + reg = <0x01c2bc00 0x400>; + interrupts = <GIC_SPI 26 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&ccu CLK_BUS_CAN>; + resets = <&ccu RST_BUS_CAN>; + status = "disabled"; + }; + i2c4: i2c@1c2c000 { compatible = "allwinner,sun6i-a31-i2c"; reg = <0x01c2c000 0x400>; diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index 8b300dd28def..72b0e71cc3de 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -33,15 +33,6 @@ do { \ (b)->data = (tmp).data; \ } while (0) -static inline bool in_bpf_jit(struct pt_regs *regs) -{ - if (!IS_ENABLED(CONFIG_BPF_JIT)) - return false; - - return regs->pc >= BPF_JIT_REGION_START && - regs->pc < BPF_JIT_REGION_END; -} - #ifdef CONFIG_BPF_JIT bool ex_handler_bpf(const struct exception_table_entry *ex, struct pt_regs *regs); diff --git a/arch/arm64/include/asm/memory.h b/arch/arm64/include/asm/memory.h index 1b9a1e242612..0af70d9abede 100644 --- a/arch/arm64/include/asm/memory.h +++ b/arch/arm64/include/asm/memory.h @@ -44,11 +44,8 @@ #define _PAGE_OFFSET(va) (-(UL(1) << (va))) #define PAGE_OFFSET (_PAGE_OFFSET(VA_BITS)) #define KIMAGE_VADDR (MODULES_END) -#define BPF_JIT_REGION_START (_PAGE_END(VA_BITS_MIN)) -#define BPF_JIT_REGION_SIZE (SZ_128M) -#define BPF_JIT_REGION_END (BPF_JIT_REGION_START + BPF_JIT_REGION_SIZE) #define MODULES_END (MODULES_VADDR + MODULES_VSIZE) -#define MODULES_VADDR (BPF_JIT_REGION_END) +#define MODULES_VADDR (_PAGE_END(VA_BITS_MIN)) #define MODULES_VSIZE (SZ_128M) #define VMEMMAP_START (-(UL(1) << (VA_BITS - VMEMMAP_SHIFT))) #define VMEMMAP_END (VMEMMAP_START + VMEMMAP_SIZE) diff --git a/arch/arm64/kernel/traps.c b/arch/arm64/kernel/traps.c index 7b21213a570f..e8986e6067a9 100644 --- a/arch/arm64/kernel/traps.c +++ b/arch/arm64/kernel/traps.c @@ -994,7 +994,7 @@ static struct break_hook bug_break_hook = { static int reserved_fault_handler(struct pt_regs *regs, unsigned int esr) { pr_err("%s generated an invalid instruction at %pS!\n", - in_bpf_jit(regs) ? "BPF JIT" : "Kernel text patching", + "Kernel text patching", (void *)instruction_pointer(regs)); /* We cannot handle this */ diff --git a/arch/arm64/mm/ptdump.c b/arch/arm64/mm/ptdump.c index 1c403536c9bb..9bc4066c5bf3 100644 --- a/arch/arm64/mm/ptdump.c +++ b/arch/arm64/mm/ptdump.c @@ -41,8 +41,6 @@ static struct addr_marker address_markers[] = { { 0 /* KASAN_SHADOW_START */, "Kasan shadow start" }, { KASAN_SHADOW_END, "Kasan shadow end" }, #endif - { BPF_JIT_REGION_START, "BPF start" }, - { BPF_JIT_REGION_END, "BPF end" }, { MODULES_VADDR, "Modules start" }, { MODULES_END, "Modules end" }, { VMALLOC_START, "vmalloc() area" }, diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index 3a8a7140a9bf..86c9dc0681cc 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -1141,15 +1141,12 @@ out: u64 bpf_jit_alloc_exec_limit(void) { - return BPF_JIT_REGION_SIZE; + return VMALLOC_END - VMALLOC_START; } void *bpf_jit_alloc_exec(unsigned long size) { - return __vmalloc_node_range(size, PAGE_SIZE, BPF_JIT_REGION_START, - BPF_JIT_REGION_END, GFP_KERNEL, - PAGE_KERNEL, 0, NUMA_NO_NODE, - __builtin_return_address(0)); + return vmalloc(size); } void bpf_jit_free_exec(void *addr) diff --git a/arch/um/drivers/vector_kern.c b/arch/um/drivers/vector_kern.c index cde6db184c26..4fc1a5d70dcf 100644 --- a/arch/um/drivers/vector_kern.c +++ b/arch/um/drivers/vector_kern.c @@ -1441,7 +1441,9 @@ flash_fail: } static void vector_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct vector_private *vp = netdev_priv(netdev); diff --git a/drivers/base/regmap/regmap.c b/drivers/base/regmap/regmap.c index 21a0c2562ec0..2d74f9f82aa9 100644 --- a/drivers/base/regmap/regmap.c +++ b/drivers/base/regmap/regmap.c @@ -876,6 +876,7 @@ struct regmap *__regmap_init(struct device *dev, if (!bus) { map->reg_read = config->reg_read; map->reg_write = config->reg_write; + map->reg_update_bits = config->reg_update_bits; map->defer_caching = false; goto skip_format_initialization; diff --git a/drivers/bluetooth/Kconfig b/drivers/bluetooth/Kconfig index 851842372c9b..36380e618ba4 100644 --- a/drivers/bluetooth/Kconfig +++ b/drivers/bluetooth/Kconfig @@ -19,6 +19,10 @@ config BT_QCA tristate select FW_LOADER +config BT_MTK + tristate + select FW_LOADER + config BT_HCIBTUSB tristate "HCI USB driver" depends on USB @@ -55,6 +59,7 @@ config BT_HCIBTUSB_BCM config BT_HCIBTUSB_MTK bool "MediaTek protocol support" depends on BT_HCIBTUSB + select BT_MTK default n help The MediaTek protocol support enables firmware download @@ -383,6 +388,7 @@ config BT_ATH3K config BT_MTKSDIO tristate "MediaTek HCI SDIO driver" depends on MMC + select BT_MTK help MediaTek Bluetooth HCI SDIO driver. This driver is required if you want to use MediaTek Bluetooth diff --git a/drivers/bluetooth/Makefile b/drivers/bluetooth/Makefile index 16286ea2655d..3321a8aea4a0 100644 --- a/drivers/bluetooth/Makefile +++ b/drivers/bluetooth/Makefile @@ -25,6 +25,7 @@ obj-$(CONFIG_BT_QCOMSMD) += btqcomsmd.o obj-$(CONFIG_BT_BCM) += btbcm.o obj-$(CONFIG_BT_RTL) += btrtl.o obj-$(CONFIG_BT_QCA) += btqca.o +obj-$(CONFIG_BT_MTK) += btmtk.o obj-$(CONFIG_BT_VIRTIO) += virtio_bt.o diff --git a/drivers/bluetooth/bfusb.c b/drivers/bluetooth/bfusb.c index 5a321b4076aa..cab93935cc7f 100644 --- a/drivers/bluetooth/bfusb.c +++ b/drivers/bluetooth/bfusb.c @@ -628,6 +628,9 @@ static int bfusb_probe(struct usb_interface *intf, const struct usb_device_id *i data->bulk_out_ep = bulk_out_ep->desc.bEndpointAddress; data->bulk_pkt_size = le16_to_cpu(bulk_out_ep->desc.wMaxPacketSize); + if (!data->bulk_pkt_size) + goto done; + rwlock_init(&data->lock); data->reassembly = NULL; diff --git a/drivers/bluetooth/btintel.c b/drivers/bluetooth/btintel.c index 9359bff47296..8f9109b40961 100644 --- a/drivers/bluetooth/btintel.c +++ b/drivers/bluetooth/btintel.c @@ -2081,14 +2081,16 @@ static int btintel_prepare_fw_download_tlv(struct hci_dev *hdev, if (ver->img_type == 0x03) { btintel_clear_flag(hdev, INTEL_BOOTLOADER); btintel_check_bdaddr(hdev); - } - - /* If the OTP has no valid Bluetooth device address, then there will - * also be no valid address for the operational firmware. - */ - if (!bacmp(&ver->otp_bd_addr, BDADDR_ANY)) { - bt_dev_info(hdev, "No device address configured"); - set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks); + } else { + /* + * Check for valid bd address in boot loader mode. Device + * will be marked as unconfigured if empty bd address is + * found. + */ + if (!bacmp(&ver->otp_bd_addr, BDADDR_ANY)) { + bt_dev_info(hdev, "No device address configured"); + set_bit(HCI_QUIRK_INVALID_BDADDR, &hdev->quirks); + } } btintel_get_fw_name_tlv(ver, fwname, sizeof(fwname), "sfi"); @@ -2466,6 +2468,10 @@ static int btintel_setup_combined(struct hci_dev *hdev) goto exit_error; } + /* memset ver_tlv to start with clean state as few fields are exclusive + * to bootloader mode and are not populated in operational mode + */ + memset(&ver_tlv, 0, sizeof(ver_tlv)); /* For TLV type device, parse the tlv data */ err = btintel_parse_version_tlv(hdev, &ver_tlv, skb); if (err) { diff --git a/drivers/bluetooth/btmrvl_main.c b/drivers/bluetooth/btmrvl_main.c index 5ccbe4d459d0..181338f60530 100644 --- a/drivers/bluetooth/btmrvl_main.c +++ b/drivers/bluetooth/btmrvl_main.c @@ -1,4 +1,4 @@ -/** +/* * Marvell Bluetooth driver * * Copyright (C) 2009, Marvell International Ltd. diff --git a/drivers/bluetooth/btmtk.c b/drivers/bluetooth/btmtk.c new file mode 100644 index 000000000000..c2ee5c4b975a --- /dev/null +++ b/drivers/bluetooth/btmtk.c @@ -0,0 +1,289 @@ +// SPDX-License-Identifier: ISC +/* Copyright (C) 2021 MediaTek Inc. + * + */ +#include <linux/module.h> +#include <linux/firmware.h> + +#include <net/bluetooth/bluetooth.h> +#include <net/bluetooth/hci_core.h> + +#include "btmtk.h" + +#define VERSION "0.1" + +/* It is for mt79xx download rom patch*/ +#define MTK_FW_ROM_PATCH_HEADER_SIZE 32 +#define MTK_FW_ROM_PATCH_GD_SIZE 64 +#define MTK_FW_ROM_PATCH_SEC_MAP_SIZE 64 +#define MTK_SEC_MAP_COMMON_SIZE 12 +#define MTK_SEC_MAP_NEED_SEND_SIZE 52 + +struct btmtk_patch_header { + u8 datetime[16]; + u8 platform[4]; + __le16 hwver; + __le16 swver; + __le32 magicnum; +} __packed; + +struct btmtk_global_desc { + __le32 patch_ver; + __le32 sub_sys; + __le32 feature_opt; + __le32 section_num; +} __packed; + +struct btmtk_section_map { + __le32 sectype; + __le32 secoffset; + __le32 secsize; + union { + __le32 u4SecSpec[13]; + struct { + __le32 dlAddr; + __le32 dlsize; + __le32 seckeyidx; + __le32 alignlen; + __le32 sectype; + __le32 dlmodecrctype; + __le32 crc; + __le32 reserved[6]; + } bin_info_spec; + }; +} __packed; + +int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync) +{ + struct btmtk_hci_wmt_params wmt_params; + struct btmtk_global_desc *globaldesc = NULL; + struct btmtk_section_map *sectionmap; + const struct firmware *fw; + const u8 *fw_ptr; + const u8 *fw_bin_ptr; + int err, dlen, i, status; + u8 flag, first_block, retry; + u32 section_num, dl_size, section_offset; + u8 cmd[64]; + + err = request_firmware(&fw, fwname, &hdev->dev); + if (err < 0) { + bt_dev_err(hdev, "Failed to load firmware file (%d)", err); + return err; + } + + fw_ptr = fw->data; + fw_bin_ptr = fw_ptr; + globaldesc = (struct btmtk_global_desc *)(fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE); + section_num = le32_to_cpu(globaldesc->section_num); + + for (i = 0; i < section_num; i++) { + first_block = 1; + fw_ptr = fw_bin_ptr; + sectionmap = (struct btmtk_section_map *)(fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE + + MTK_FW_ROM_PATCH_GD_SIZE + MTK_FW_ROM_PATCH_SEC_MAP_SIZE * i); + + section_offset = le32_to_cpu(sectionmap->secoffset); + dl_size = le32_to_cpu(sectionmap->bin_info_spec.dlsize); + + if (dl_size > 0) { + retry = 20; + while (retry > 0) { + cmd[0] = 0; /* 0 means legacy dl mode. */ + memcpy(cmd + 1, + fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE + + MTK_FW_ROM_PATCH_GD_SIZE + + MTK_FW_ROM_PATCH_SEC_MAP_SIZE * i + + MTK_SEC_MAP_COMMON_SIZE, + MTK_SEC_MAP_NEED_SEND_SIZE + 1); + + wmt_params.op = BTMTK_WMT_PATCH_DWNLD; + wmt_params.status = &status; + wmt_params.flag = 0; + wmt_params.dlen = MTK_SEC_MAP_NEED_SEND_SIZE + 1; + wmt_params.data = &cmd; + + err = wmt_cmd_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", + err); + goto err_release_fw; + } + + if (status == BTMTK_WMT_PATCH_UNDONE) { + break; + } else if (status == BTMTK_WMT_PATCH_PROGRESS) { + msleep(100); + retry--; + } else if (status == BTMTK_WMT_PATCH_DONE) { + goto next_section; + } else { + bt_dev_err(hdev, "Failed wmt patch dwnld status (%d)", + status); + goto err_release_fw; + } + } + + fw_ptr += section_offset; + wmt_params.op = BTMTK_WMT_PATCH_DWNLD; + wmt_params.status = NULL; + + while (dl_size > 0) { + dlen = min_t(int, 250, dl_size); + if (first_block == 1) { + flag = 1; + first_block = 0; + } else if (dl_size - dlen <= 0) { + flag = 3; + } else { + flag = 2; + } + + wmt_params.flag = flag; + wmt_params.dlen = dlen; + wmt_params.data = fw_ptr; + + err = wmt_cmd_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", + err); + goto err_release_fw; + } + + dl_size -= dlen; + fw_ptr += dlen; + } + } +next_section: + continue; + } + /* Wait a few moments for firmware activation done */ + usleep_range(100000, 120000); + +err_release_fw: + release_firmware(fw); + + return err; +} +EXPORT_SYMBOL_GPL(btmtk_setup_firmware_79xx); + +int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync) +{ + struct btmtk_hci_wmt_params wmt_params; + const struct firmware *fw; + const u8 *fw_ptr; + size_t fw_size; + int err, dlen; + u8 flag, param; + + err = request_firmware(&fw, fwname, &hdev->dev); + if (err < 0) { + bt_dev_err(hdev, "Failed to load firmware file (%d)", err); + return err; + } + + /* Power on data RAM the firmware relies on. */ + param = 1; + wmt_params.op = BTMTK_WMT_FUNC_CTRL; + wmt_params.flag = 3; + wmt_params.dlen = sizeof(param); + wmt_params.data = ¶m; + wmt_params.status = NULL; + + err = wmt_cmd_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to power on data RAM (%d)", err); + goto err_release_fw; + } + + fw_ptr = fw->data; + fw_size = fw->size; + + /* The size of patch header is 30 bytes, should be skip */ + if (fw_size < 30) { + err = -EINVAL; + goto err_release_fw; + } + + fw_size -= 30; + fw_ptr += 30; + flag = 1; + + wmt_params.op = BTMTK_WMT_PATCH_DWNLD; + wmt_params.status = NULL; + + while (fw_size > 0) { + dlen = min_t(int, 250, fw_size); + + /* Tell device the position in sequence */ + if (fw_size - dlen <= 0) + flag = 3; + else if (fw_size < fw->size - 30) + flag = 2; + + wmt_params.flag = flag; + wmt_params.dlen = dlen; + wmt_params.data = fw_ptr; + + err = wmt_cmd_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", + err); + goto err_release_fw; + } + + fw_size -= dlen; + fw_ptr += dlen; + } + + wmt_params.op = BTMTK_WMT_RST; + wmt_params.flag = 4; + wmt_params.dlen = 0; + wmt_params.data = NULL; + wmt_params.status = NULL; + + /* Activate funciton the firmware providing to */ + err = wmt_cmd_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to send wmt rst (%d)", err); + goto err_release_fw; + } + + /* Wait a few moments for firmware activation done */ + usleep_range(10000, 12000); + +err_release_fw: + release_firmware(fw); + + return err; +} +EXPORT_SYMBOL_GPL(btmtk_setup_firmware); + +int btmtk_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr) +{ + struct sk_buff *skb; + long ret; + + skb = __hci_cmd_sync(hdev, 0xfc1a, 6, bdaddr, HCI_INIT_TIMEOUT); + if (IS_ERR(skb)) { + ret = PTR_ERR(skb); + bt_dev_err(hdev, "changing Mediatek device address failed (%ld)", + ret); + return ret; + } + kfree_skb(skb); + + return 0; +} +EXPORT_SYMBOL_GPL(btmtk_set_bdaddr); + +MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>"); +MODULE_AUTHOR("Mark Chen <mark-yw.chen@mediatek.com>"); +MODULE_DESCRIPTION("Bluetooth support for MediaTek devices ver " VERSION); +MODULE_VERSION(VERSION); +MODULE_LICENSE("GPL"); +MODULE_FIRMWARE(FIRMWARE_MT7663); +MODULE_FIRMWARE(FIRMWARE_MT7668); +MODULE_FIRMWARE(FIRMWARE_MT7961); diff --git a/drivers/bluetooth/btmtk.h b/drivers/bluetooth/btmtk.h new file mode 100644 index 000000000000..6e7b0c7567c0 --- /dev/null +++ b/drivers/bluetooth/btmtk.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: ISC */ +/* Copyright (C) 2021 MediaTek Inc. */ + +#define FIRMWARE_MT7663 "mediatek/mt7663pr2h.bin" +#define FIRMWARE_MT7668 "mediatek/mt7668pr2h.bin" +#define FIRMWARE_MT7961 "mediatek/BT_RAM_CODE_MT7961_1_2_hdr.bin" + +#define HCI_WMT_MAX_EVENT_SIZE 64 + +#define BTMTK_WMT_REG_READ 0x2 + +enum { + BTMTK_WMT_PATCH_DWNLD = 0x1, + BTMTK_WMT_TEST = 0x2, + BTMTK_WMT_WAKEUP = 0x3, + BTMTK_WMT_HIF = 0x4, + BTMTK_WMT_FUNC_CTRL = 0x6, + BTMTK_WMT_RST = 0x7, + BTMTK_WMT_REGISTER = 0x8, + BTMTK_WMT_SEMAPHORE = 0x17, +}; + +enum { + BTMTK_WMT_INVALID, + BTMTK_WMT_PATCH_UNDONE, + BTMTK_WMT_PATCH_PROGRESS, + BTMTK_WMT_PATCH_DONE, + BTMTK_WMT_ON_UNDONE, + BTMTK_WMT_ON_DONE, + BTMTK_WMT_ON_PROGRESS, +}; + +struct btmtk_wmt_hdr { + u8 dir; + u8 op; + __le16 dlen; + u8 flag; +} __packed; + +struct btmtk_hci_wmt_cmd { + struct btmtk_wmt_hdr hdr; + u8 data[]; +} __packed; + +struct btmtk_hci_wmt_evt { + struct hci_event_hdr hhdr; + struct btmtk_wmt_hdr whdr; +} __packed; + +struct btmtk_hci_wmt_evt_funcc { + struct btmtk_hci_wmt_evt hwhdr; + __be16 status; +} __packed; + +struct btmtk_hci_wmt_evt_reg { + struct btmtk_hci_wmt_evt hwhdr; + u8 rsv[2]; + u8 num; + __le32 addr; + __le32 val; +} __packed; + +struct btmtk_tci_sleep { + u8 mode; + __le16 duration; + __le16 host_duration; + u8 host_wakeup_pin; + u8 time_compensation; +} __packed; + +struct btmtk_hci_wmt_params { + u8 op; + u8 flag; + u16 dlen; + const void *data; + u32 *status; +}; + +typedef int (*wmt_cmd_sync_func_t)(struct hci_dev *, + struct btmtk_hci_wmt_params *); + +#if IS_ENABLED(CONFIG_BT_MTK) + +int btmtk_set_bdaddr(struct hci_dev *hdev, const bdaddr_t *bdaddr); + +int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync); + +int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync); +#else + +static inline int btmtk_set_bdaddr(struct hci_dev *hdev, + const bdaddr_t *bdaddr) +{ + return -EOPNOTSUPP; +} + +static int btmtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync) +{ + return -EOPNOTSUPP; +} + +static int btmtk_setup_firmware(struct hci_dev *hdev, const char *fwname, + wmt_cmd_sync_func_t wmt_cmd_sync) +{ + return -EOPNOTSUPP; +} + +#endif diff --git a/drivers/bluetooth/btmtksdio.c b/drivers/bluetooth/btmtksdio.c index 9872ef18f9fe..ce6a6c00ff98 100644 --- a/drivers/bluetooth/btmtksdio.c +++ b/drivers/bluetooth/btmtksdio.c @@ -12,7 +12,6 @@ #include <asm/unaligned.h> #include <linux/atomic.h> -#include <linux/firmware.h> #include <linux/init.h> #include <linux/iopoll.h> #include <linux/kernel.h> @@ -28,26 +27,32 @@ #include <net/bluetooth/hci_core.h> #include "h4_recv.h" +#include "btmtk.h" #define VERSION "0.1" -#define FIRMWARE_MT7663 "mediatek/mt7663pr2h.bin" -#define FIRMWARE_MT7668 "mediatek/mt7668pr2h.bin" - #define MTKBTSDIO_AUTOSUSPEND_DELAY 8000 static bool enable_autosuspend; struct btmtksdio_data { const char *fwname; + u16 chipid; }; static const struct btmtksdio_data mt7663_data = { .fwname = FIRMWARE_MT7663, + .chipid = 0x7663, }; static const struct btmtksdio_data mt7668_data = { .fwname = FIRMWARE_MT7668, + .chipid = 0x7668, +}; + +static const struct btmtksdio_data mt7921_data = { + .fwname = FIRMWARE_MT7961, + .chipid = 0x7921, }; static const struct sdio_device_id btmtksdio_table[] = { @@ -55,6 +60,8 @@ static const struct sdio_device_id btmtksdio_table[] = { .driver_data = (kernel_ulong_t)&mt7663_data }, {SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7668), .driver_data = (kernel_ulong_t)&mt7668_data }, + {SDIO_DEVICE(SDIO_VENDOR_ID_MEDIATEK, SDIO_DEVICE_ID_MEDIATEK_MT7961), + .driver_data = (kernel_ulong_t)&mt7921_data }, { } /* Terminating entry */ }; MODULE_DEVICE_TABLE(sdio, btmtksdio_table); @@ -86,81 +93,27 @@ MODULE_DEVICE_TABLE(sdio, btmtksdio_table); #define MTK_REG_CRDR 0x1c +#define MTK_REG_CRPLR 0x24 + #define MTK_SDIO_BLOCK_SIZE 256 #define BTMTKSDIO_TX_WAIT_VND_EVT 1 -enum { - MTK_WMT_PATCH_DWNLD = 0x1, - MTK_WMT_TEST = 0x2, - MTK_WMT_WAKEUP = 0x3, - MTK_WMT_HIF = 0x4, - MTK_WMT_FUNC_CTRL = 0x6, - MTK_WMT_RST = 0x7, - MTK_WMT_SEMAPHORE = 0x17, -}; - -enum { - BTMTK_WMT_INVALID, - BTMTK_WMT_PATCH_UNDONE, - BTMTK_WMT_PATCH_DONE, - BTMTK_WMT_ON_UNDONE, - BTMTK_WMT_ON_DONE, - BTMTK_WMT_ON_PROGRESS, -}; - struct mtkbtsdio_hdr { __le16 len; __le16 reserved; u8 bt_type; } __packed; -struct mtk_wmt_hdr { - u8 dir; - u8 op; - __le16 dlen; - u8 flag; -} __packed; - -struct mtk_hci_wmt_cmd { - struct mtk_wmt_hdr hdr; - u8 data[256]; -} __packed; - -struct btmtk_hci_wmt_evt { - struct hci_event_hdr hhdr; - struct mtk_wmt_hdr whdr; -} __packed; - -struct btmtk_hci_wmt_evt_funcc { - struct btmtk_hci_wmt_evt hwhdr; - __be16 status; -} __packed; - -struct btmtk_tci_sleep { - u8 mode; - __le16 duration; - __le16 host_duration; - u8 host_wakeup_pin; - u8 time_compensation; -} __packed; - -struct btmtk_hci_wmt_params { - u8 op; - u8 flag; - u16 dlen; - const void *data; - u32 *status; -}; - struct btmtksdio_dev { struct hci_dev *hdev; struct sdio_func *func; struct device *dev; - struct work_struct tx_work; + struct work_struct txrx_work; unsigned long tx_state; struct sk_buff_head txq; + bool hw_tx_ready; struct sk_buff *evt_skb; @@ -172,29 +125,35 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, { struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); struct btmtk_hci_wmt_evt_funcc *wmt_evt_funcc; + struct btmtk_hci_wmt_evt_reg *wmt_evt_reg; u32 hlen, status = BTMTK_WMT_INVALID; struct btmtk_hci_wmt_evt *wmt_evt; - struct mtk_hci_wmt_cmd wc; - struct mtk_wmt_hdr *hdr; + struct btmtk_hci_wmt_cmd *wc; + struct btmtk_wmt_hdr *hdr; int err; + /* Send the WMT command and wait until the WMT event returns */ hlen = sizeof(*hdr) + wmt_params->dlen; if (hlen > 255) return -EINVAL; - hdr = (struct mtk_wmt_hdr *)&wc; + wc = kzalloc(hlen, GFP_KERNEL); + if (!wc) + return -ENOMEM; + + hdr = &wc->hdr; hdr->dir = 1; hdr->op = wmt_params->op; hdr->dlen = cpu_to_le16(wmt_params->dlen + 1); hdr->flag = wmt_params->flag; - memcpy(wc.data, wmt_params->data, wmt_params->dlen); + memcpy(wc->data, wmt_params->data, wmt_params->dlen); set_bit(BTMTKSDIO_TX_WAIT_VND_EVT, &bdev->tx_state); - err = __hci_cmd_send(hdev, 0xfc6f, hlen, &wc); + err = __hci_cmd_send(hdev, 0xfc6f, hlen, wc); if (err < 0) { clear_bit(BTMTKSDIO_TX_WAIT_VND_EVT, &bdev->tx_state); - return err; + goto err_free_wc; } /* The vendor specific WMT commands are all answered by a vendor @@ -211,13 +170,14 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, if (err == -EINTR) { bt_dev_err(hdev, "Execution of wmt command interrupted"); clear_bit(BTMTKSDIO_TX_WAIT_VND_EVT, &bdev->tx_state); - return err; + goto err_free_wc; } if (err) { bt_dev_err(hdev, "Execution of wmt command timed out"); clear_bit(BTMTKSDIO_TX_WAIT_VND_EVT, &bdev->tx_state); - return -ETIMEDOUT; + err = -ETIMEDOUT; + goto err_free_wc; } /* Parse and handle the return WMT event */ @@ -230,13 +190,13 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, } switch (wmt_evt->whdr.op) { - case MTK_WMT_SEMAPHORE: + case BTMTK_WMT_SEMAPHORE: if (wmt_evt->whdr.flag == 2) status = BTMTK_WMT_PATCH_UNDONE; else status = BTMTK_WMT_PATCH_DONE; break; - case MTK_WMT_FUNC_CTRL: + case BTMTK_WMT_FUNC_CTRL: wmt_evt_funcc = (struct btmtk_hci_wmt_evt_funcc *)wmt_evt; if (be16_to_cpu(wmt_evt_funcc->status) == 0x404) status = BTMTK_WMT_ON_DONE; @@ -245,6 +205,19 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, else status = BTMTK_WMT_ON_UNDONE; break; + case BTMTK_WMT_PATCH_DWNLD: + if (wmt_evt->whdr.flag == 2) + status = BTMTK_WMT_PATCH_DONE; + else if (wmt_evt->whdr.flag == 1) + status = BTMTK_WMT_PATCH_PROGRESS; + else + status = BTMTK_WMT_PATCH_UNDONE; + break; + case BTMTK_WMT_REGISTER: + wmt_evt_reg = (struct btmtk_hci_wmt_evt_reg *)wmt_evt; + if (le16_to_cpu(wmt_evt->whdr.dlen) == 12) + status = le32_to_cpu(wmt_evt_reg->val); + break; } if (wmt_params->status) @@ -253,6 +226,8 @@ static int mtk_hci_wmt_sync(struct hci_dev *hdev, err_free_skb: kfree_skb(bdev->evt_skb); bdev->evt_skb = NULL; +err_free_wc: + kfree(wc); return err; } @@ -279,6 +254,7 @@ static int btmtksdio_tx_packet(struct btmtksdio_dev *bdev, sdio_hdr->reserved = cpu_to_le16(0); sdio_hdr->bt_type = hci_skb_pkt_type(skb); + bdev->hw_tx_ready = false; err = sdio_writesb(bdev->func, MTK_REG_CTDR, skb->data, round_up(skb->len, MTK_SDIO_BLOCK_SIZE)); if (err < 0) @@ -301,32 +277,6 @@ static u32 btmtksdio_drv_own_query(struct btmtksdio_dev *bdev) return sdio_readl(bdev->func, MTK_REG_CHLPCR, NULL); } -static void btmtksdio_tx_work(struct work_struct *work) -{ - struct btmtksdio_dev *bdev = container_of(work, struct btmtksdio_dev, - tx_work); - struct sk_buff *skb; - int err; - - pm_runtime_get_sync(bdev->dev); - - sdio_claim_host(bdev->func); - - while ((skb = skb_dequeue(&bdev->txq))) { - err = btmtksdio_tx_packet(bdev, skb); - if (err < 0) { - bdev->hdev->stat.err_tx++; - skb_queue_head(&bdev->txq, skb); - break; - } - } - - sdio_release_host(bdev->func); - - pm_runtime_mark_last_busy(bdev->dev); - pm_runtime_put_autosuspend(bdev->dev); -} - static int btmtksdio_recv_event(struct hci_dev *hdev, struct sk_buff *skb) { struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); @@ -477,70 +427,89 @@ err_kfree_skb: return err; } -static void btmtksdio_interrupt(struct sdio_func *func) +static void btmtksdio_txrx_work(struct work_struct *work) { - struct btmtksdio_dev *bdev = sdio_get_drvdata(func); - u32 int_status; - u16 rx_size; - - /* It is required that the host gets ownership from the device before - * accessing any register, however, if SDIO host is not being released, - * a potential deadlock probably happens in a circular wait between SDIO - * IRQ work and PM runtime work. So, we have to explicitly release SDIO - * host here and claim again after the PM runtime work is all done. - */ - sdio_release_host(bdev->func); + struct btmtksdio_dev *bdev = container_of(work, struct btmtksdio_dev, + txrx_work); + unsigned long txrx_timeout; + u32 int_status, rx_size; + struct sk_buff *skb; + int err; pm_runtime_get_sync(bdev->dev); sdio_claim_host(bdev->func); /* Disable interrupt */ - sdio_writel(func, C_INT_EN_CLR, MTK_REG_CHLPCR, NULL); - - int_status = sdio_readl(func, MTK_REG_CHISR, NULL); - - /* Ack an interrupt as soon as possible before any operation on - * hardware. - * - * Note that we don't ack any status during operations to avoid race - * condition between the host and the device such as it's possible to - * mistakenly ack RX_DONE for the next packet and then cause interrupts - * not be raised again but there is still pending data in the hardware - * FIFO. - */ - sdio_writel(func, int_status, MTK_REG_CHISR, NULL); + sdio_writel(bdev->func, C_INT_EN_CLR, MTK_REG_CHLPCR, 0); + + txrx_timeout = jiffies + 5 * HZ; + + do { + int_status = sdio_readl(bdev->func, MTK_REG_CHISR, NULL); + + /* Ack an interrupt as soon as possible before any operation on + * hardware. + * + * Note that we don't ack any status during operations to avoid race + * condition between the host and the device such as it's possible to + * mistakenly ack RX_DONE for the next packet and then cause interrupts + * not be raised again but there is still pending data in the hardware + * FIFO. + */ + sdio_writel(bdev->func, int_status, MTK_REG_CHISR, NULL); + + if (int_status & FW_OWN_BACK_INT) + bt_dev_dbg(bdev->hdev, "Get fw own back"); + + if (int_status & TX_EMPTY) + bdev->hw_tx_ready = true; + else if (unlikely(int_status & TX_FIFO_OVERFLOW)) + bt_dev_warn(bdev->hdev, "Tx fifo overflow"); + + if (bdev->hw_tx_ready) { + skb = skb_dequeue(&bdev->txq); + if (skb) { + err = btmtksdio_tx_packet(bdev, skb); + if (err < 0) { + bdev->hdev->stat.err_tx++; + skb_queue_head(&bdev->txq, skb); + } + } + } - if (unlikely(!int_status)) - bt_dev_err(bdev->hdev, "CHISR is 0"); + if (int_status & RX_DONE_INT) { + rx_size = sdio_readl(bdev->func, MTK_REG_CRPLR, NULL); + rx_size = (rx_size & RX_PKT_LEN) >> 16; + if (btmtksdio_rx_packet(bdev, rx_size) < 0) + bdev->hdev->stat.err_rx++; + } + } while (int_status || time_is_before_jiffies(txrx_timeout)); - if (int_status & FW_OWN_BACK_INT) - bt_dev_dbg(bdev->hdev, "Get fw own back"); + /* Enable interrupt */ + sdio_writel(bdev->func, C_INT_EN_SET, MTK_REG_CHLPCR, 0); - if (int_status & TX_EMPTY) - schedule_work(&bdev->tx_work); - else if (unlikely(int_status & TX_FIFO_OVERFLOW)) - bt_dev_warn(bdev->hdev, "Tx fifo overflow"); + sdio_release_host(bdev->func); - if (int_status & RX_DONE_INT) { - rx_size = (int_status & RX_PKT_LEN) >> 16; + pm_runtime_mark_last_busy(bdev->dev); + pm_runtime_put_autosuspend(bdev->dev); +} - if (btmtksdio_rx_packet(bdev, rx_size) < 0) - bdev->hdev->stat.err_rx++; - } +static void btmtksdio_interrupt(struct sdio_func *func) +{ + struct btmtksdio_dev *bdev = sdio_get_drvdata(func); - /* Enable interrupt */ - sdio_writel(func, C_INT_EN_SET, MTK_REG_CHLPCR, NULL); + /* Disable interrupt */ + sdio_writel(bdev->func, C_INT_EN_CLR, MTK_REG_CHLPCR, 0); - pm_runtime_mark_last_busy(bdev->dev); - pm_runtime_put_autosuspend(bdev->dev); + schedule_work(&bdev->txrx_work); } static int btmtksdio_open(struct hci_dev *hdev) { struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); + u32 status, val; int err; - u32 status; sdio_claim_host(bdev->func); @@ -580,13 +549,22 @@ static int btmtksdio_open(struct hci_dev *hdev) /* SDIO CMD 5 allows the SDIO device back to idle state an * synchronous interrupt is supported in SDIO 4-bit mode */ - sdio_writel(bdev->func, SDIO_INT_CTL | SDIO_RE_INIT_EN, - MTK_REG_CSDIOCSR, &err); + val = sdio_readl(bdev->func, MTK_REG_CSDIOCSR, &err); + if (err < 0) + goto err_release_irq; + + val |= SDIO_INT_CTL; + sdio_writel(bdev->func, val, MTK_REG_CSDIOCSR, &err); + if (err < 0) + goto err_release_irq; + + /* Explitly set write-1-clear method */ + val = sdio_readl(bdev->func, MTK_REG_CHCR, &err); if (err < 0) goto err_release_irq; - /* Setup write-1-clear for CHISR register */ - sdio_writel(bdev->func, C_INT_CLR_CTRL, MTK_REG_CHCR, &err); + val |= C_INT_CLR_CTRL; + sdio_writel(bdev->func, val, MTK_REG_CHCR, &err); if (err < 0) goto err_release_irq; @@ -630,6 +608,8 @@ static int btmtksdio_close(struct hci_dev *hdev) sdio_release_irq(bdev->func); + cancel_work_sync(&bdev->txrx_work); + /* Return ownership to the device */ sdio_writel(bdev->func, C_FW_OWN_REQ_SET, MTK_REG_CHLPCR, NULL); @@ -651,7 +631,7 @@ static int btmtksdio_flush(struct hci_dev *hdev) skb_queue_purge(&bdev->txq); - cancel_work_sync(&bdev->tx_work); + cancel_work_sync(&bdev->txrx_work); return 0; } @@ -663,7 +643,7 @@ static int btmtksdio_func_query(struct hci_dev *hdev) u8 param = 0; /* Query whether the function is enabled */ - wmt_params.op = MTK_WMT_FUNC_CTRL; + wmt_params.op = BTMTK_WMT_FUNC_CTRL; wmt_params.flag = 4; wmt_params.dlen = sizeof(param); wmt_params.data = ¶m; @@ -678,111 +658,16 @@ static int btmtksdio_func_query(struct hci_dev *hdev) return status; } -static int mtk_setup_firmware(struct hci_dev *hdev, const char *fwname) +static int mt76xx_setup(struct hci_dev *hdev, const char *fwname) { struct btmtk_hci_wmt_params wmt_params; - const struct firmware *fw; - const u8 *fw_ptr; - size_t fw_size; - int err, dlen; - u8 flag, param; - - err = request_firmware(&fw, fwname, &hdev->dev); - if (err < 0) { - bt_dev_err(hdev, "Failed to load firmware file (%d)", err); - return err; - } - - /* Power on data RAM the firmware relies on. */ - param = 1; - wmt_params.op = MTK_WMT_FUNC_CTRL; - wmt_params.flag = 3; - wmt_params.dlen = sizeof(param); - wmt_params.data = ¶m; - wmt_params.status = NULL; - - err = mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to power on data RAM (%d)", err); - goto free_fw; - } - - fw_ptr = fw->data; - fw_size = fw->size; - - /* The size of patch header is 30 bytes, should be skip */ - if (fw_size < 30) { - err = -EINVAL; - goto free_fw; - } - - fw_size -= 30; - fw_ptr += 30; - flag = 1; - - wmt_params.op = MTK_WMT_PATCH_DWNLD; - wmt_params.status = NULL; - - while (fw_size > 0) { - dlen = min_t(int, 250, fw_size); - - /* Tell device the position in sequence */ - if (fw_size - dlen <= 0) - flag = 3; - else if (fw_size < fw->size - 30) - flag = 2; - - wmt_params.flag = flag; - wmt_params.dlen = dlen; - wmt_params.data = fw_ptr; - - err = mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", - err); - goto free_fw; - } - - fw_size -= dlen; - fw_ptr += dlen; - } - - wmt_params.op = MTK_WMT_RST; - wmt_params.flag = 4; - wmt_params.dlen = 0; - wmt_params.data = NULL; - wmt_params.status = NULL; - - /* Activate funciton the firmware providing to */ - err = mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt rst (%d)", err); - goto free_fw; - } - - /* Wait a few moments for firmware activation done */ - usleep_range(10000, 12000); - -free_fw: - release_firmware(fw); - return err; -} - -static int btmtksdio_setup(struct hci_dev *hdev) -{ - struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); - struct btmtk_hci_wmt_params wmt_params; - ktime_t calltime, delta, rettime; struct btmtk_tci_sleep tci_sleep; - unsigned long long duration; struct sk_buff *skb; int err, status; u8 param = 0x1; - calltime = ktime_get(); - /* Query whether the firmware is already download */ - wmt_params.op = MTK_WMT_SEMAPHORE; + wmt_params.op = BTMTK_WMT_SEMAPHORE; wmt_params.flag = 1; wmt_params.dlen = 0; wmt_params.data = NULL; @@ -800,7 +685,7 @@ static int btmtksdio_setup(struct hci_dev *hdev) } /* Setup a firmware which the device definitely requires */ - err = mtk_setup_firmware(hdev, bdev->data->fwname); + err = btmtk_setup_firmware(hdev, fwname, mtk_hci_wmt_sync); if (err < 0) return err; @@ -823,7 +708,7 @@ ignore_setup_fw: } /* Enable Bluetooth protocol */ - wmt_params.op = MTK_WMT_FUNC_CTRL; + wmt_params.op = BTMTK_WMT_FUNC_CTRL; wmt_params.flag = 0; wmt_params.dlen = sizeof(param); wmt_params.data = ¶m; @@ -852,6 +737,113 @@ ignore_func_on: } kfree_skb(skb); + return 0; +} + +static int mt79xx_setup(struct hci_dev *hdev, const char *fwname) +{ + struct btmtk_hci_wmt_params wmt_params; + u8 param = 0x1; + int err; + + err = btmtk_setup_firmware_79xx(hdev, fwname, mtk_hci_wmt_sync); + if (err < 0) { + bt_dev_err(hdev, "Failed to setup 79xx firmware (%d)", err); + return err; + } + + /* Enable Bluetooth protocol */ + wmt_params.op = BTMTK_WMT_FUNC_CTRL; + wmt_params.flag = 0; + wmt_params.dlen = sizeof(param); + wmt_params.data = ¶m; + wmt_params.status = NULL; + + err = mtk_hci_wmt_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to send wmt func ctrl (%d)", err); + return err; + } + + return err; +} + +static int btsdio_mtk_reg_read(struct hci_dev *hdev, u32 reg, u32 *val) +{ + struct btmtk_hci_wmt_params wmt_params; + struct reg_read_cmd { + u8 type; + u8 rsv; + u8 num; + __le32 addr; + } __packed reg_read = { + .type = 1, + .num = 1, + }; + u32 status; + int err; + + reg_read.addr = cpu_to_le32(reg); + wmt_params.op = BTMTK_WMT_REGISTER; + wmt_params.flag = BTMTK_WMT_REG_READ; + wmt_params.dlen = sizeof(reg_read); + wmt_params.data = ®_read; + wmt_params.status = &status; + + err = mtk_hci_wmt_sync(hdev, &wmt_params); + if (err < 0) { + bt_dev_err(hdev, "Failed to read reg(%d)", err); + return err; + } + + *val = status; + + return err; +} + +static int btmtksdio_setup(struct hci_dev *hdev) +{ + struct btmtksdio_dev *bdev = hci_get_drvdata(hdev); + ktime_t calltime, delta, rettime; + unsigned long long duration; + char fwname[64]; + int err, dev_id; + u32 fw_version = 0; + + calltime = ktime_get(); + bdev->hw_tx_ready = true; + + switch (bdev->data->chipid) { + case 0x7921: + err = btsdio_mtk_reg_read(hdev, 0x70010200, &dev_id); + if (err < 0) { + bt_dev_err(hdev, "Failed to get device id (%d)", err); + return err; + } + + err = btsdio_mtk_reg_read(hdev, 0x80021004, &fw_version); + if (err < 0) { + bt_dev_err(hdev, "Failed to get fw version (%d)", err); + return err; + } + + snprintf(fwname, sizeof(fwname), + "mediatek/BT_RAM_CODE_MT%04x_1_%x_hdr.bin", + dev_id & 0xffff, (fw_version & 0xff) + 1); + err = mt79xx_setup(hdev, fwname); + if (err < 0) + return err; + break; + case 0x7663: + case 0x7668: + err = mt76xx_setup(hdev, bdev->data->fwname); + if (err < 0) + return err; + break; + default: + return -ENODEV; + } + rettime = ktime_get(); delta = ktime_sub(rettime, calltime); duration = (unsigned long long)ktime_to_ns(delta) >> 10; @@ -891,7 +883,7 @@ static int btmtksdio_shutdown(struct hci_dev *hdev) pm_runtime_get_sync(bdev->dev); /* Disable the device */ - wmt_params.op = MTK_WMT_FUNC_CTRL; + wmt_params.op = BTMTK_WMT_FUNC_CTRL; wmt_params.flag = 0; wmt_params.dlen = sizeof(param); wmt_params.data = ¶m; @@ -932,7 +924,7 @@ static int btmtksdio_send_frame(struct hci_dev *hdev, struct sk_buff *skb) skb_queue_tail(&bdev->txq, skb); - schedule_work(&bdev->tx_work); + schedule_work(&bdev->txrx_work); return 0; } @@ -955,7 +947,7 @@ static int btmtksdio_probe(struct sdio_func *func, bdev->dev = &func->dev; bdev->func = func; - INIT_WORK(&bdev->tx_work, btmtksdio_tx_work); + INIT_WORK(&bdev->txrx_work, btmtksdio_txrx_work); skb_queue_head_init(&bdev->txq); /* Initialize and register HCI device */ @@ -976,6 +968,8 @@ static int btmtksdio_probe(struct sdio_func *func, hdev->setup = btmtksdio_setup; hdev->shutdown = btmtksdio_shutdown; hdev->send = btmtksdio_send_frame; + hdev->set_bdaddr = btmtk_set_bdaddr; + SET_HCIDEV_DEV(hdev, &func->dev); hdev->manufacturer = 70; @@ -1112,5 +1106,3 @@ MODULE_AUTHOR("Sean Wang <sean.wang@mediatek.com>"); MODULE_DESCRIPTION("MediaTek Bluetooth SDIO driver ver " VERSION); MODULE_VERSION(VERSION); MODULE_LICENSE("GPL"); -MODULE_FIRMWARE(FIRMWARE_MT7663); -MODULE_FIRMWARE(FIRMWARE_MT7668); diff --git a/drivers/bluetooth/btsdio.c b/drivers/bluetooth/btsdio.c index 199e8f7d426d..795be33f2892 100644 --- a/drivers/bluetooth/btsdio.c +++ b/drivers/bluetooth/btsdio.c @@ -295,6 +295,8 @@ static int btsdio_probe(struct sdio_func *func, switch (func->device) { case SDIO_DEVICE_ID_BROADCOM_43341: case SDIO_DEVICE_ID_BROADCOM_43430: + case SDIO_DEVICE_ID_BROADCOM_4345: + case SDIO_DEVICE_ID_BROADCOM_43455: case SDIO_DEVICE_ID_BROADCOM_4356: return -ENODEV; } diff --git a/drivers/bluetooth/btusb.c b/drivers/bluetooth/btusb.c index 75c83768c257..30a057e1d4e3 100644 --- a/drivers/bluetooth/btusb.c +++ b/drivers/bluetooth/btusb.c @@ -24,6 +24,7 @@ #include "btintel.h" #include "btbcm.h" #include "btrtl.h" +#include "btmtk.h" #define VERSION "0.8" @@ -2131,122 +2132,6 @@ static int btusb_send_frame_intel(struct hci_dev *hdev, struct sk_buff *skb) #define MTK_BT_RST_DONE 0x00000100 #define MTK_BT_RESET_WAIT_MS 100 #define MTK_BT_RESET_NUM_TRIES 10 -#define FIRMWARE_MT7663 "mediatek/mt7663pr2h.bin" -#define FIRMWARE_MT7668 "mediatek/mt7668pr2h.bin" - -#define HCI_WMT_MAX_EVENT_SIZE 64 -/* It is for mt79xx download rom patch*/ -#define MTK_FW_ROM_PATCH_HEADER_SIZE 32 -#define MTK_FW_ROM_PATCH_GD_SIZE 64 -#define MTK_FW_ROM_PATCH_SEC_MAP_SIZE 64 -#define MTK_SEC_MAP_COMMON_SIZE 12 -#define MTK_SEC_MAP_NEED_SEND_SIZE 52 - -enum { - BTMTK_WMT_PATCH_DWNLD = 0x1, - BTMTK_WMT_FUNC_CTRL = 0x6, - BTMTK_WMT_RST = 0x7, - BTMTK_WMT_SEMAPHORE = 0x17, -}; - -enum { - BTMTK_WMT_INVALID, - BTMTK_WMT_PATCH_UNDONE, - BTMTK_WMT_PATCH_PROGRESS, - BTMTK_WMT_PATCH_DONE, - BTMTK_WMT_ON_UNDONE, - BTMTK_WMT_ON_DONE, - BTMTK_WMT_ON_PROGRESS, -}; - -struct btmtk_wmt_hdr { - u8 dir; - u8 op; - __le16 dlen; - u8 flag; -} __packed; - -struct btmtk_hci_wmt_cmd { - struct btmtk_wmt_hdr hdr; - u8 data[]; -} __packed; - -struct btmtk_hci_wmt_evt { - struct hci_event_hdr hhdr; - struct btmtk_wmt_hdr whdr; -} __packed; - -struct btmtk_hci_wmt_evt_funcc { - struct btmtk_hci_wmt_evt hwhdr; - __be16 status; -} __packed; - -struct btmtk_tci_sleep { - u8 mode; - __le16 duration; - __le16 host_duration; - u8 host_wakeup_pin; - u8 time_compensation; -} __packed; - -struct btmtk_hci_wmt_params { - u8 op; - u8 flag; - u16 dlen; - const void *data; - u32 *status; -}; - -struct btmtk_patch_header { - u8 datetime[16]; - u8 platform[4]; - __le16 hwver; - __le16 swver; - __le32 magicnum; -} __packed; - -struct btmtk_global_desc { - __le32 patch_ver; - __le32 sub_sys; - __le32 feature_opt; - __le32 section_num; -} __packed; - -struct btmtk_section_map { - __le32 sectype; - __le32 secoffset; - __le32 secsize; - union { - __le32 u4SecSpec[13]; - struct { - __le32 dlAddr; - __le32 dlsize; - __le32 seckeyidx; - __le32 alignlen; - __le32 sectype; - __le32 dlmodecrctype; - __le32 crc; - __le32 reserved[6]; - } bin_info_spec; - }; -} __packed; - -static int btusb_set_bdaddr_mtk(struct hci_dev *hdev, const bdaddr_t *bdaddr) -{ - struct sk_buff *skb; - long ret; - - skb = __hci_cmd_sync(hdev, 0xfc1a, sizeof(bdaddr), bdaddr, HCI_INIT_TIMEOUT); - if (IS_ERR(skb)) { - ret = PTR_ERR(skb); - bt_dev_err(hdev, "changing Mediatek device address failed (%ld)", - ret); - return ret; - } - kfree_skb(skb); - - return 0; -} static void btusb_mtk_wmt_recv(struct urb *urb) { @@ -2265,6 +2150,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb) skb = bt_skb_alloc(HCI_WMT_MAX_EVENT_SIZE, GFP_ATOMIC); if (!skb) { hdev->stat.err_rx++; + kfree(urb->setup_packet); return; } @@ -2285,6 +2171,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb) data->evt_skb = skb_clone(skb, GFP_ATOMIC); if (!data->evt_skb) { kfree_skb(skb); + kfree(urb->setup_packet); return; } } @@ -2293,6 +2180,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb) if (err < 0) { kfree_skb(data->evt_skb); data->evt_skb = NULL; + kfree(urb->setup_packet); return; } @@ -2303,6 +2191,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb) wake_up_bit(&data->flags, BTUSB_TX_WAIT_VND_EVT); } + kfree(urb->setup_packet); return; } else if (urb->status == -ENOENT) { /* Avoid suspend failed when usb_kill_urb */ @@ -2323,6 +2212,7 @@ static void btusb_mtk_wmt_recv(struct urb *urb) usb_anchor_urb(urb, &data->ctrl_anchor); err = usb_submit_urb(urb, GFP_ATOMIC); if (err < 0) { + kfree(urb->setup_packet); /* -EPERM: urb is being killed; * -ENODEV: device got disconnected */ @@ -2497,209 +2387,6 @@ err_free_wc: return err; } -static int btusb_mtk_setup_firmware_79xx(struct hci_dev *hdev, const char *fwname) -{ - struct btmtk_hci_wmt_params wmt_params; - struct btmtk_global_desc *globaldesc = NULL; - struct btmtk_section_map *sectionmap; - const struct firmware *fw; - const u8 *fw_ptr; - const u8 *fw_bin_ptr; - int err, dlen, i, status; - u8 flag, first_block, retry; - u32 section_num, dl_size, section_offset; - u8 cmd[64]; - - err = request_firmware(&fw, fwname, &hdev->dev); - if (err < 0) { - bt_dev_err(hdev, "Failed to load firmware file (%d)", err); - return err; - } - - fw_ptr = fw->data; - fw_bin_ptr = fw_ptr; - globaldesc = (struct btmtk_global_desc *)(fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE); - section_num = le32_to_cpu(globaldesc->section_num); - - for (i = 0; i < section_num; i++) { - first_block = 1; - fw_ptr = fw_bin_ptr; - sectionmap = (struct btmtk_section_map *)(fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE + - MTK_FW_ROM_PATCH_GD_SIZE + MTK_FW_ROM_PATCH_SEC_MAP_SIZE * i); - - section_offset = le32_to_cpu(sectionmap->secoffset); - dl_size = le32_to_cpu(sectionmap->bin_info_spec.dlsize); - - if (dl_size > 0) { - retry = 20; - while (retry > 0) { - cmd[0] = 0; /* 0 means legacy dl mode. */ - memcpy(cmd + 1, - fw_ptr + MTK_FW_ROM_PATCH_HEADER_SIZE + - MTK_FW_ROM_PATCH_GD_SIZE + MTK_FW_ROM_PATCH_SEC_MAP_SIZE * i + - MTK_SEC_MAP_COMMON_SIZE, - MTK_SEC_MAP_NEED_SEND_SIZE + 1); - - wmt_params.op = BTMTK_WMT_PATCH_DWNLD; - wmt_params.status = &status; - wmt_params.flag = 0; - wmt_params.dlen = MTK_SEC_MAP_NEED_SEND_SIZE + 1; - wmt_params.data = &cmd; - - err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", - err); - goto err_release_fw; - } - - if (status == BTMTK_WMT_PATCH_UNDONE) { - break; - } else if (status == BTMTK_WMT_PATCH_PROGRESS) { - msleep(100); - retry--; - } else if (status == BTMTK_WMT_PATCH_DONE) { - goto next_section; - } else { - bt_dev_err(hdev, "Failed wmt patch dwnld status (%d)", - status); - goto err_release_fw; - } - } - - fw_ptr += section_offset; - wmt_params.op = BTMTK_WMT_PATCH_DWNLD; - wmt_params.status = NULL; - - while (dl_size > 0) { - dlen = min_t(int, 250, dl_size); - if (first_block == 1) { - flag = 1; - first_block = 0; - } else if (dl_size - dlen <= 0) { - flag = 3; - } else { - flag = 2; - } - - wmt_params.flag = flag; - wmt_params.dlen = dlen; - wmt_params.data = fw_ptr; - - err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", - err); - goto err_release_fw; - } - - dl_size -= dlen; - fw_ptr += dlen; - } - } -next_section: - continue; - } - /* Wait a few moments for firmware activation done */ - usleep_range(100000, 120000); - -err_release_fw: - release_firmware(fw); - - return err; -} - -static int btusb_mtk_setup_firmware(struct hci_dev *hdev, const char *fwname) -{ - struct btmtk_hci_wmt_params wmt_params; - const struct firmware *fw; - const u8 *fw_ptr; - size_t fw_size; - int err, dlen; - u8 flag, param; - - err = request_firmware(&fw, fwname, &hdev->dev); - if (err < 0) { - bt_dev_err(hdev, "Failed to load firmware file (%d)", err); - return err; - } - - /* Power on data RAM the firmware relies on. */ - param = 1; - wmt_params.op = BTMTK_WMT_FUNC_CTRL; - wmt_params.flag = 3; - wmt_params.dlen = sizeof(param); - wmt_params.data = ¶m; - wmt_params.status = NULL; - - err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to power on data RAM (%d)", err); - goto err_release_fw; - } - - fw_ptr = fw->data; - fw_size = fw->size; - - /* The size of patch header is 30 bytes, should be skip */ - if (fw_size < 30) { - err = -EINVAL; - goto err_release_fw; - } - - fw_size -= 30; - fw_ptr += 30; - flag = 1; - - wmt_params.op = BTMTK_WMT_PATCH_DWNLD; - wmt_params.status = NULL; - - while (fw_size > 0) { - dlen = min_t(int, 250, fw_size); - - /* Tell device the position in sequence */ - if (fw_size - dlen <= 0) - flag = 3; - else if (fw_size < fw->size - 30) - flag = 2; - - wmt_params.flag = flag; - wmt_params.dlen = dlen; - wmt_params.data = fw_ptr; - - err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt patch dwnld (%d)", - err); - goto err_release_fw; - } - - fw_size -= dlen; - fw_ptr += dlen; - } - - wmt_params.op = BTMTK_WMT_RST; - wmt_params.flag = 4; - wmt_params.dlen = 0; - wmt_params.data = NULL; - wmt_params.status = NULL; - - /* Activate funciton the firmware providing to */ - err = btusb_mtk_hci_wmt_sync(hdev, &wmt_params); - if (err < 0) { - bt_dev_err(hdev, "Failed to send wmt rst (%d)", err); - goto err_release_fw; - } - - /* Wait a few moments for firmware activation done */ - usleep_range(10000, 12000); - -err_release_fw: - release_firmware(fw); - - return err; -} - static int btusb_mtk_func_query(struct hci_dev *hdev) { struct btmtk_hci_wmt_params wmt_params; @@ -2857,7 +2544,8 @@ static int btusb_mtk_setup(struct hci_dev *hdev) snprintf(fw_bin_name, sizeof(fw_bin_name), "mediatek/BT_RAM_CODE_MT%04x_1_%x_hdr.bin", dev_id & 0xffff, (fw_version & 0xff) + 1); - err = btusb_mtk_setup_firmware_79xx(hdev, fw_bin_name); + err = btmtk_setup_firmware_79xx(hdev, fw_bin_name, + btusb_mtk_hci_wmt_sync); /* It's Device EndPoint Reset Option Register */ btusb_mtk_uhw_reg_write(data, MTK_EP_RST_OPT, MTK_EP_RST_IN_OUT_OPT); @@ -2877,6 +2565,7 @@ static int btusb_mtk_setup(struct hci_dev *hdev) } hci_set_msft_opcode(hdev, 0xFD30); + hci_set_aosp_capable(hdev); goto done; default: bt_dev_err(hdev, "Unsupported hardware variant (%08x)", @@ -2903,7 +2592,8 @@ static int btusb_mtk_setup(struct hci_dev *hdev) } /* Setup a firmware which the device definitely requires */ - err = btusb_mtk_setup_firmware(hdev, fwname); + err = btmtk_setup_firmware(hdev, fwname, + btusb_mtk_hci_wmt_sync); if (err < 0) return err; @@ -3064,9 +2754,6 @@ static int btusb_recv_acl_mtk(struct hci_dev *hdev, struct sk_buff *skb) return hci_recv_frame(hdev, skb); } -MODULE_FIRMWARE(FIRMWARE_MT7663); -MODULE_FIRMWARE(FIRMWARE_MT7668); - #ifdef CONFIG_PM /* Configure an out-of-band gpio as wake-up pin, if specified in device tree */ static int marvell_config_oob_wake(struct hci_dev *hdev) @@ -3190,6 +2877,9 @@ static int btusb_set_bdaddr_wcn6855(struct hci_dev *hdev, #define QCA_DFU_TIMEOUT 3000 #define QCA_FLAG_MULTI_NVM 0x80 +#define WCN6855_2_0_RAM_VERSION_GF 0x400c1200 +#define WCN6855_2_1_RAM_VERSION_GF 0x400c1211 + struct qca_version { __le32 rom_version; __le32 patch_version; @@ -3221,6 +2911,7 @@ static const struct qca_device_info qca_devices_table[] = { { 0x00000302, 28, 4, 16 }, /* Rome 3.2 */ { 0x00130100, 40, 4, 16 }, /* WCN6855 1.0 */ { 0x00130200, 40, 4, 16 }, /* WCN6855 2.0 */ + { 0x00130201, 40, 4, 16 }, /* WCN6855 2.1 */ }; static int btusb_qca_send_vendor_req(struct usb_device *udev, u8 request, @@ -3375,6 +3066,40 @@ done: return err; } +static void btusb_generate_qca_nvm_name(char *fwname, size_t max_size, + const struct qca_version *ver) +{ + u32 rom_version = le32_to_cpu(ver->rom_version); + u16 flag = le16_to_cpu(ver->flag); + + if (((flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) { + u16 board_id = le16_to_cpu(ver->board_id); + const char *variant; + + switch (le32_to_cpu(ver->ram_version)) { + case WCN6855_2_0_RAM_VERSION_GF: + case WCN6855_2_1_RAM_VERSION_GF: + variant = "_gf"; + break; + default: + variant = ""; + break; + } + + if (board_id == 0) { + snprintf(fwname, max_size, "qca/nvm_usb_%08x%s.bin", + rom_version, variant); + } else { + snprintf(fwname, max_size, "qca/nvm_usb_%08x%s_%04x.bin", + rom_version, variant, board_id); + } + } else { + snprintf(fwname, max_size, "qca/nvm_usb_%08x.bin", + rom_version); + } + +} + static int btusb_setup_qca_load_nvm(struct hci_dev *hdev, struct qca_version *ver, const struct qca_device_info *info) @@ -3383,20 +3108,7 @@ static int btusb_setup_qca_load_nvm(struct hci_dev *hdev, char fwname[64]; int err; - if (((ver->flag >> 8) & 0xff) == QCA_FLAG_MULTI_NVM) { - /* if boardid equal 0, use default nvm without surfix */ - if (le16_to_cpu(ver->board_id) == 0x0) { - snprintf(fwname, sizeof(fwname), "qca/nvm_usb_%08x.bin", - le32_to_cpu(ver->rom_version)); - } else { - snprintf(fwname, sizeof(fwname), "qca/nvm_usb_%08x_%04x.bin", - le32_to_cpu(ver->rom_version), - le16_to_cpu(ver->board_id)); - } - } else { - snprintf(fwname, sizeof(fwname), "qca/nvm_usb_%08x.bin", - le32_to_cpu(ver->rom_version)); - } + btusb_generate_qca_nvm_name(fwname, sizeof(fwname), ver); err = request_firmware(&fw, fwname, &hdev->dev); if (err) { @@ -3868,7 +3580,7 @@ static int btusb_probe(struct usb_interface *intf, hdev->shutdown = btusb_mtk_shutdown; hdev->manufacturer = 70; hdev->cmd_timeout = btusb_mtk_cmd_timeout; - hdev->set_bdaddr = btusb_set_bdaddr_mtk; + hdev->set_bdaddr = btmtk_set_bdaddr; set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks); data->recv_acl = btusb_recv_acl_mtk; } diff --git a/drivers/bluetooth/hci_bcm.c b/drivers/bluetooth/hci_bcm.c index ef54afa29357..7852abf15ddf 100644 --- a/drivers/bluetooth/hci_bcm.c +++ b/drivers/bluetooth/hci_bcm.c @@ -1508,7 +1508,6 @@ static const struct of_device_id bcm_bluetooth_of_match[] = { { .compatible = "brcm,bcm4330-bt" }, { .compatible = "brcm,bcm4334-bt" }, { .compatible = "brcm,bcm4345c5" }, - { .compatible = "brcm,bcm4330-bt" }, { .compatible = "brcm,bcm43438-bt", .data = &bcm43438_device_data }, { .compatible = "brcm,bcm43540-bt", .data = &bcm4354_device_data }, { .compatible = "brcm,bcm4335a0" }, diff --git a/drivers/bluetooth/hci_h4.c b/drivers/bluetooth/hci_h4.c index 4b3b14a34794..1d0cdf023243 100644 --- a/drivers/bluetooth/hci_h4.c +++ b/drivers/bluetooth/hci_h4.c @@ -252,7 +252,7 @@ struct sk_buff *h4_recv_buf(struct hci_dev *hdev, struct sk_buff *skb, } if (!dlen) { - hu->padding = (skb->len - 1) % alignment; + hu->padding = (skb->len + 1) % alignment; hu->padding = (alignment - hu->padding) % alignment; /* No more data, complete frame */ @@ -260,7 +260,7 @@ struct sk_buff *h4_recv_buf(struct hci_dev *hdev, struct sk_buff *skb, skb = NULL; } } else { - hu->padding = (skb->len - 1) % alignment; + hu->padding = (skb->len + 1) % alignment; hu->padding = (alignment - hu->padding) % alignment; /* Complete frame */ diff --git a/drivers/bluetooth/hci_vhci.c b/drivers/bluetooth/hci_vhci.c index b45db0db347c..49ac884d996e 100644 --- a/drivers/bluetooth/hci_vhci.c +++ b/drivers/bluetooth/hci_vhci.c @@ -38,9 +38,12 @@ struct vhci_data { struct mutex open_mutex; struct delayed_work open_timeout; + struct work_struct suspend_work; bool suspended; bool wakeup; + __u16 msft_opcode; + bool aosp_capable; }; static int vhci_open_dev(struct hci_dev *hdev) @@ -114,6 +117,17 @@ static ssize_t force_suspend_read(struct file *file, char __user *user_buf, return simple_read_from_buffer(user_buf, count, ppos, buf, 2); } +static void vhci_suspend_work(struct work_struct *work) +{ + struct vhci_data *data = container_of(work, struct vhci_data, + suspend_work); + + if (data->suspended) + hci_suspend_dev(data->hdev); + else + hci_resume_dev(data->hdev); +} + static ssize_t force_suspend_write(struct file *file, const char __user *user_buf, size_t count, loff_t *ppos) @@ -129,16 +143,10 @@ static ssize_t force_suspend_write(struct file *file, if (data->suspended == enable) return -EALREADY; - if (enable) - err = hci_suspend_dev(data->hdev); - else - err = hci_resume_dev(data->hdev); - - if (err) - return err; - data->suspended = enable; + schedule_work(&data->suspend_work); + return count; } @@ -176,6 +184,8 @@ static ssize_t force_wakeup_write(struct file *file, if (data->wakeup == enable) return -EALREADY; + data->wakeup = enable; + return count; } @@ -186,6 +196,88 @@ static const struct file_operations force_wakeup_fops = { .llseek = default_llseek, }; +static int msft_opcode_set(void *data, u64 val) +{ + struct vhci_data *vhci = data; + + if (val > 0xffff || hci_opcode_ogf(val) != 0x3f) + return -EINVAL; + + if (vhci->msft_opcode) + return -EALREADY; + + vhci->msft_opcode = val; + + return 0; +} + +static int msft_opcode_get(void *data, u64 *val) +{ + struct vhci_data *vhci = data; + + *val = vhci->msft_opcode; + + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(msft_opcode_fops, msft_opcode_get, msft_opcode_set, + "%llu\n"); + +static ssize_t aosp_capable_read(struct file *file, char __user *user_buf, + size_t count, loff_t *ppos) +{ + struct vhci_data *vhci = file->private_data; + char buf[3]; + + buf[0] = vhci->aosp_capable ? 'Y' : 'N'; + buf[1] = '\n'; + buf[2] = '\0'; + return simple_read_from_buffer(user_buf, count, ppos, buf, 2); +} + +static ssize_t aosp_capable_write(struct file *file, + const char __user *user_buf, size_t count, + loff_t *ppos) +{ + struct vhci_data *vhci = file->private_data; + bool enable; + int err; + + err = kstrtobool_from_user(user_buf, count, &enable); + if (err) + return err; + + if (!enable) + return -EINVAL; + + if (vhci->aosp_capable) + return -EALREADY; + + vhci->aosp_capable = enable; + + return count; +} + +static const struct file_operations aosp_capable_fops = { + .open = simple_open, + .read = aosp_capable_read, + .write = aosp_capable_write, + .llseek = default_llseek, +}; + +static int vhci_setup(struct hci_dev *hdev) +{ + struct vhci_data *vhci = hci_get_drvdata(hdev); + + if (vhci->msft_opcode) + hci_set_msft_opcode(hdev, vhci->msft_opcode); + + if (vhci->aosp_capable) + hci_set_aosp_capable(hdev); + + return 0; +} + static int __vhci_create_device(struct vhci_data *data, __u8 opcode) { struct hci_dev *hdev; @@ -228,6 +320,8 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode) hdev->get_data_path_id = vhci_get_data_path_id; hdev->get_codec_config_data = vhci_get_codec_config_data; hdev->wakeup = vhci_wakeup; + hdev->setup = vhci_setup; + set_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks); /* bit 6 is for external configuration */ if (opcode & 0x40) @@ -251,6 +345,14 @@ static int __vhci_create_device(struct vhci_data *data, __u8 opcode) debugfs_create_file("force_wakeup", 0644, hdev->debugfs, data, &force_wakeup_fops); + if (IS_ENABLED(CONFIG_BT_MSFTEXT)) + debugfs_create_file("msft_opcode", 0644, hdev->debugfs, data, + &msft_opcode_fops); + + if (IS_ENABLED(CONFIG_BT_AOSPEXT)) + debugfs_create_file("aosp_capable", 0644, hdev->debugfs, data, + &aosp_capable_fops); + hci_skb_pkt_type(skb) = HCI_VENDOR_PKT; skb_put_u8(skb, 0xff); @@ -440,6 +542,7 @@ static int vhci_open(struct inode *inode, struct file *file) mutex_init(&data->open_mutex); INIT_DELAYED_WORK(&data->open_timeout, vhci_open_timeout); + INIT_WORK(&data->suspend_work, vhci_suspend_work); file->private_data = data; nonseekable_open(inode, file); @@ -455,6 +558,7 @@ static int vhci_release(struct inode *inode, struct file *file) struct hci_dev *hdev; cancel_delayed_work_sync(&data->open_timeout); + flush_work(&data->suspend_work); hdev = data->hdev; diff --git a/drivers/bluetooth/virtio_bt.c b/drivers/bluetooth/virtio_bt.c index 57908ce4fae8..076e4942a3f0 100644 --- a/drivers/bluetooth/virtio_bt.c +++ b/drivers/bluetooth/virtio_bt.c @@ -202,6 +202,9 @@ static void virtbt_rx_handle(struct virtio_bluetooth *vbt, struct sk_buff *skb) hci_skb_pkt_type(skb) = pkt_type; hci_recv_frame(vbt->hdev, skb); break; + default: + kfree_skb(skb); + break; } } diff --git a/drivers/infiniband/hw/irdma/main.c b/drivers/infiniband/hw/irdma/main.c index 51a41359e0b4..3fda7b78a9af 100644 --- a/drivers/infiniband/hw/irdma/main.c +++ b/drivers/infiniband/hw/irdma/main.c @@ -228,7 +228,8 @@ static void irdma_fill_device_info(struct irdma_device *iwdev, struct ice_pf *pf rf->msix_count = pf->num_rdma_msix; rf->msix_entries = &pf->msix_entries[pf->rdma_base_vector]; rf->default_vsi.vsi_idx = vsi->vsi_num; - rf->protocol_used = IRDMA_ROCE_PROTOCOL_ONLY; + rf->protocol_used = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? + IRDMA_ROCE_PROTOCOL_ONLY : IRDMA_IWARP_PROTOCOL_ONLY; rf->rdma_ver = IRDMA_GEN_2; rf->rsrc_profile = IRDMA_HMC_PROFILE_DEFAULT; rf->rst_to = IRDMA_RST_TIMEOUT_HZ; diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 44bac4ad687c..46aa3554e97b 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -643,6 +643,64 @@ static void mei_cl_bus_vtag_free(struct mei_cl_device *cldev) kfree(cl_vtag); } +void *mei_cldev_dma_map(struct mei_cl_device *cldev, u8 buffer_id, size_t size) +{ + struct mei_device *bus; + struct mei_cl *cl; + int ret; + + if (!cldev || !buffer_id || !size) + return ERR_PTR(-EINVAL); + + if (!IS_ALIGNED(size, MEI_FW_PAGE_SIZE)) { + dev_err(&cldev->dev, "Map size should be aligned to %lu\n", + MEI_FW_PAGE_SIZE); + return ERR_PTR(-EINVAL); + } + + cl = cldev->cl; + bus = cldev->bus; + + mutex_lock(&bus->device_lock); + if (cl->state == MEI_FILE_UNINITIALIZED) { + ret = mei_cl_link(cl); + if (ret) + goto out; + /* update pointers */ + cl->cldev = cldev; + } + + ret = mei_cl_dma_alloc_and_map(cl, NULL, buffer_id, size); +out: + mutex_unlock(&bus->device_lock); + if (ret) + return ERR_PTR(ret); + return cl->dma.vaddr; +} +EXPORT_SYMBOL_GPL(mei_cldev_dma_map); + +int mei_cldev_dma_unmap(struct mei_cl_device *cldev) +{ + struct mei_device *bus; + struct mei_cl *cl; + int ret; + + if (!cldev) + return -EINVAL; + + cl = cldev->cl; + bus = cldev->bus; + + mutex_lock(&bus->device_lock); + ret = mei_cl_dma_unmap(cl, NULL); + + mei_cl_flush_queues(cl, NULL); + mei_cl_unlink(cl); + mutex_unlock(&bus->device_lock); + return ret; +} +EXPORT_SYMBOL_GPL(mei_cldev_dma_unmap); + /** * mei_cldev_enable - enable me client device * create connection with me client @@ -753,9 +811,11 @@ int mei_cldev_disable(struct mei_cl_device *cldev) dev_err(bus->dev, "Could not disconnect from the ME client\n"); out: - /* Flush queues and remove any pending read */ - mei_cl_flush_queues(cl, NULL); - mei_cl_unlink(cl); + /* Flush queues and remove any pending read unless we have mapped DMA */ + if (!cl->dma_mapped) { + mei_cl_flush_queues(cl, NULL); + mei_cl_unlink(cl); + } mutex_unlock(&bus->device_lock); return err; @@ -1052,6 +1112,7 @@ static void mei_cl_bus_dev_release(struct device *dev) if (!cldev) return; + mei_cl_flush_queues(cldev->cl, NULL); mei_me_cl_put(cldev->me_cl); mei_dev_bus_put(cldev->bus); mei_cl_unlink(cldev->cl); diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index 96f4e59c32a5..0e90591235a6 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -700,6 +700,9 @@ int mei_cl_unlink(struct mei_cl *cl) cl_dbg(dev, cl, "unlink client"); + if (cl->state == MEI_FILE_UNINITIALIZED) + return 0; + if (dev->open_handle_count > 0) dev->open_handle_count--; diff --git a/drivers/misc/mei/hw.h b/drivers/misc/mei/hw.h index dfd60c916da0..b46077b17114 100644 --- a/drivers/misc/mei/hw.h +++ b/drivers/misc/mei/hw.h @@ -23,6 +23,11 @@ #define MEI_HBM_TIMEOUT 1 /* 1 second */ /* + * FW page size for DMA allocations + */ +#define MEI_FW_PAGE_SIZE 4096UL + +/* * MEI Version */ #define HBM_MINOR_VERSION 2 diff --git a/drivers/net/bonding/bond_main.c b/drivers/net/bonding/bond_main.c index ff8da720a33a..0f39ad2af81c 100644 --- a/drivers/net/bonding/bond_main.c +++ b/drivers/net/bonding/bond_main.c @@ -71,6 +71,7 @@ #include <linux/ethtool.h> #include <linux/if_vlan.h> #include <linux/if_bonding.h> +#include <linux/phy.h> #include <linux/jiffies.h> #include <linux/preempt.h> #include <net/route.h> @@ -1460,7 +1461,7 @@ done: bond_dev->hw_enc_features |= xfrm_features; #endif /* CONFIG_XFRM_OFFLOAD */ bond_dev->mpls_features = mpls_features; - bond_dev->gso_max_segs = gso_max_segs; + netif_set_gso_max_segs(bond_dev, gso_max_segs); netif_set_gso_max_size(bond_dev, gso_max_size); bond_dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; @@ -3129,8 +3130,8 @@ static void bond_loadbalance_arp_mon(struct bonding *bond) * when the source ip is 0, so don't take the link down * if we don't know our ip yet */ - if (!bond_time_in_interval(bond, trans_start, 2) || - !bond_time_in_interval(bond, slave->last_rx, 2)) { + if (!bond_time_in_interval(bond, trans_start, bond->params.missed_max) || + !bond_time_in_interval(bond, slave->last_rx, bond->params.missed_max)) { bond_propose_link_state(slave, BOND_LINK_DOWN); slave_state_changed = 1; @@ -3224,7 +3225,7 @@ static int bond_ab_arp_inspect(struct bonding *bond) /* Backup slave is down if: * - No current_arp_slave AND - * - more than 3*delta since last receive AND + * - more than (missed_max+1)*delta since last receive AND * - the bond has an IP address * * Note: a non-null current_arp_slave indicates @@ -3236,20 +3237,20 @@ static int bond_ab_arp_inspect(struct bonding *bond) */ if (!bond_is_active_slave(slave) && !rcu_access_pointer(bond->current_arp_slave) && - !bond_time_in_interval(bond, last_rx, 3)) { + !bond_time_in_interval(bond, last_rx, bond->params.missed_max + 1)) { bond_propose_link_state(slave, BOND_LINK_DOWN); commit++; } /* Active slave is down if: - * - more than 2*delta since transmitting OR - * - (more than 2*delta since receive AND + * - more than missed_max*delta since transmitting OR + * - (more than missed_max*delta since receive AND * the bond has an IP address) */ trans_start = dev_trans_start(slave->dev); if (bond_is_active_slave(slave) && - (!bond_time_in_interval(bond, trans_start, 2) || - !bond_time_in_interval(bond, last_rx, 2))) { + (!bond_time_in_interval(bond, trans_start, bond->params.missed_max) || + !bond_time_in_interval(bond, last_rx, bond->params.missed_max))) { bond_propose_link_state(slave, BOND_LINK_DOWN); commit++; } @@ -4091,7 +4092,10 @@ static int bond_eth_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cm { struct bonding *bond = netdev_priv(bond_dev); struct mii_ioctl_data *mii = NULL; - int res; + const struct net_device_ops *ops; + struct net_device *real_dev; + struct ifreq ifrr; + int res = 0; netdev_dbg(bond_dev, "bond_eth_ioctl: cmd=%d\n", cmd); @@ -4117,7 +4121,24 @@ static int bond_eth_ioctl(struct net_device *bond_dev, struct ifreq *ifr, int cm mii->val_out = BMSR_LSTATUS; } - return 0; + break; + case SIOCSHWTSTAMP: + case SIOCGHWTSTAMP: + rcu_read_lock(); + real_dev = bond_option_active_slave_get_rcu(bond); + rcu_read_unlock(); + if (real_dev) { + strscpy_pad(ifrr.ifr_name, real_dev->name, IFNAMSIZ); + ifrr.ifr_ifru = ifr->ifr_ifru; + + ops = real_dev->netdev_ops; + if (netif_device_present(real_dev) && ops->ndo_eth_ioctl) + res = ops->ndo_eth_ioctl(real_dev, &ifrr, cmd); + + if (!res) + ifr->ifr_ifru = ifrr.ifr_ifru; + } + break; default: res = -EOPNOTSUPP; } @@ -5319,10 +5340,40 @@ static void bond_ethtool_get_drvinfo(struct net_device *bond_dev, BOND_ABI_VERSION); } +static int bond_ethtool_get_ts_info(struct net_device *bond_dev, + struct ethtool_ts_info *info) +{ + struct bonding *bond = netdev_priv(bond_dev); + const struct ethtool_ops *ops; + struct net_device *real_dev; + struct phy_device *phydev; + + rcu_read_lock(); + real_dev = bond_option_active_slave_get_rcu(bond); + rcu_read_unlock(); + if (real_dev) { + ops = real_dev->ethtool_ops; + phydev = real_dev->phydev; + + if (phy_has_tsinfo(phydev)) { + return phy_ts_info(phydev, info); + } else if (ops->get_ts_info) { + return ops->get_ts_info(real_dev, info); + } + } + + info->so_timestamping = SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE; + info->phc_index = -1; + + return 0; +} + static const struct ethtool_ops bond_ethtool_ops = { .get_drvinfo = bond_ethtool_get_drvinfo, .get_link = ethtool_op_get_link, .get_link_ksettings = bond_ethtool_get_link_ksettings, + .get_ts_info = bond_ethtool_get_ts_info, }; static const struct net_device_ops bond_netdev_ops = { @@ -5822,6 +5873,7 @@ static int bond_check_params(struct bond_params *params) params->arp_interval = arp_interval; params->arp_validate = arp_validate_value; params->arp_all_targets = arp_all_targets_value; + params->missed_max = 2; params->updelay = updelay; params->downdelay = downdelay; params->peer_notif_delay = 0; diff --git a/drivers/net/bonding/bond_netlink.c b/drivers/net/bonding/bond_netlink.c index 5d54e11d18fa..1007bf6d385d 100644 --- a/drivers/net/bonding/bond_netlink.c +++ b/drivers/net/bonding/bond_netlink.c @@ -110,6 +110,7 @@ static const struct nla_policy bond_policy[IFLA_BOND_MAX + 1] = { .len = ETH_ALEN }, [IFLA_BOND_TLB_DYNAMIC_LB] = { .type = NLA_U8 }, [IFLA_BOND_PEER_NOTIF_DELAY] = { .type = NLA_U32 }, + [IFLA_BOND_MISSED_MAX] = { .type = NLA_U8 }, }; static const struct nla_policy bond_slave_policy[IFLA_BOND_SLAVE_MAX + 1] = { @@ -453,6 +454,15 @@ static int bond_changelink(struct net_device *bond_dev, struct nlattr *tb[], return err; } + if (data[IFLA_BOND_MISSED_MAX]) { + int missed_max = nla_get_u8(data[IFLA_BOND_MISSED_MAX]); + + bond_opt_initval(&newval, missed_max); + err = __bond_opt_set(bond, BOND_OPT_MISSED_MAX, &newval); + if (err) + return err; + } + return 0; } @@ -515,6 +525,7 @@ static size_t bond_get_size(const struct net_device *bond_dev) nla_total_size(ETH_ALEN) + /* IFLA_BOND_AD_ACTOR_SYSTEM */ nla_total_size(sizeof(u8)) + /* IFLA_BOND_TLB_DYNAMIC_LB */ nla_total_size(sizeof(u32)) + /* IFLA_BOND_PEER_NOTIF_DELAY */ + nla_total_size(sizeof(u8)) + /* IFLA_BOND_MISSED_MAX */ 0; } @@ -650,6 +661,10 @@ static int bond_fill_info(struct sk_buff *skb, bond->params.tlb_dynamic_lb)) goto nla_put_failure; + if (nla_put_u8(skb, IFLA_BOND_MISSED_MAX, + bond->params.missed_max)) + goto nla_put_failure; + if (BOND_MODE(bond) == BOND_MODE_8023AD) { struct ad_info info; diff --git a/drivers/net/bonding/bond_options.c b/drivers/net/bonding/bond_options.c index a8fde3bc458f..0f48921c4f15 100644 --- a/drivers/net/bonding/bond_options.c +++ b/drivers/net/bonding/bond_options.c @@ -78,6 +78,8 @@ static int bond_option_ad_actor_system_set(struct bonding *bond, const struct bond_opt_value *newval); static int bond_option_ad_user_port_key_set(struct bonding *bond, const struct bond_opt_value *newval); +static int bond_option_missed_max_set(struct bonding *bond, + const struct bond_opt_value *newval); static const struct bond_opt_value bond_mode_tbl[] = { @@ -213,6 +215,13 @@ static const struct bond_opt_value bond_ad_user_port_key_tbl[] = { { NULL, -1, 0}, }; +static const struct bond_opt_value bond_missed_max_tbl[] = { + { "minval", 1, BOND_VALFLAG_MIN}, + { "maxval", 255, BOND_VALFLAG_MAX}, + { "default", 2, BOND_VALFLAG_DEFAULT}, + { NULL, -1, 0}, +}; + static const struct bond_option bond_opts[BOND_OPT_LAST] = { [BOND_OPT_MODE] = { .id = BOND_OPT_MODE, @@ -270,6 +279,15 @@ static const struct bond_option bond_opts[BOND_OPT_LAST] = { .values = bond_intmax_tbl, .set = bond_option_arp_interval_set }, + [BOND_OPT_MISSED_MAX] = { + .id = BOND_OPT_MISSED_MAX, + .name = "arp_missed_max", + .desc = "Maximum number of missed ARP interval", + .unsuppmodes = BIT(BOND_MODE_8023AD) | BIT(BOND_MODE_TLB) | + BIT(BOND_MODE_ALB), + .values = bond_missed_max_tbl, + .set = bond_option_missed_max_set + }, [BOND_OPT_ARP_TARGETS] = { .id = BOND_OPT_ARP_TARGETS, .name = "arp_ip_target", @@ -1186,6 +1204,16 @@ static int bond_option_arp_all_targets_set(struct bonding *bond, return 0; } +static int bond_option_missed_max_set(struct bonding *bond, + const struct bond_opt_value *newval) +{ + netdev_dbg(bond->dev, "Setting missed max to %s (%llu)\n", + newval->string, newval->value); + bond->params.missed_max = newval->value; + + return 0; +} + static int bond_option_primary_set(struct bonding *bond, const struct bond_opt_value *newval) { diff --git a/drivers/net/bonding/bond_procfs.c b/drivers/net/bonding/bond_procfs.c index f3e3bfd72556..2ec11af5f0cc 100644 --- a/drivers/net/bonding/bond_procfs.c +++ b/drivers/net/bonding/bond_procfs.c @@ -115,6 +115,8 @@ static void bond_info_show_master(struct seq_file *seq) seq_printf(seq, "ARP Polling Interval (ms): %d\n", bond->params.arp_interval); + seq_printf(seq, "ARP Missed Max: %u\n", + bond->params.missed_max); seq_printf(seq, "ARP IP target/s (n.n.n.n form):"); diff --git a/drivers/net/bonding/bond_sysfs.c b/drivers/net/bonding/bond_sysfs.c index c48b77167fab..9b5a5df23d21 100644 --- a/drivers/net/bonding/bond_sysfs.c +++ b/drivers/net/bonding/bond_sysfs.c @@ -303,6 +303,18 @@ static ssize_t bonding_show_arp_targets(struct device *d, static DEVICE_ATTR(arp_ip_target, 0644, bonding_show_arp_targets, bonding_sysfs_store_option); +/* Show the arp missed max. */ +static ssize_t bonding_show_missed_max(struct device *d, + struct device_attribute *attr, + char *buf) +{ + struct bonding *bond = to_bond(d); + + return sprintf(buf, "%u\n", bond->params.missed_max); +} +static DEVICE_ATTR(arp_missed_max, 0644, + bonding_show_missed_max, bonding_sysfs_store_option); + /* Show the up and down delays. */ static ssize_t bonding_show_downdelay(struct device *d, struct device_attribute *attr, @@ -779,6 +791,7 @@ static struct attribute *per_bond_attrs[] = { &dev_attr_ad_actor_sys_prio.attr, &dev_attr_ad_actor_system.attr, &dev_attr_ad_user_port_key.attr, + &dev_attr_arp_missed_max.attr, NULL, }; diff --git a/drivers/net/can/c_can/c_can_ethtool.c b/drivers/net/can/c_can/c_can_ethtool.c index 377c7d2e7612..6655146294fc 100644 --- a/drivers/net/can/c_can/c_can_ethtool.c +++ b/drivers/net/can/c_can/c_can_ethtool.c @@ -20,7 +20,9 @@ static void c_can_get_drvinfo(struct net_device *netdev, } static void c_can_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct c_can_priv *priv = netdev_priv(netdev); diff --git a/drivers/net/can/dev/bittiming.c b/drivers/net/can/dev/bittiming.c index 0509625c3082..d5fca3bfaf9a 100644 --- a/drivers/net/can/dev/bittiming.c +++ b/drivers/net/can/dev/bittiming.c @@ -4,6 +4,7 @@ * Copyright (C) 2008-2009 Wolfgang Grandegger <wg@grandegger.com> */ +#include <linux/units.h> #include <linux/can/dev.h> #ifdef CONFIG_CAN_CALC_BITTIMING @@ -81,9 +82,9 @@ int can_calc_bittiming(struct net_device *dev, struct can_bittiming *bt, if (bt->sample_point) { sample_point_nominal = bt->sample_point; } else { - if (bt->bitrate > 800 * CAN_KBPS) + if (bt->bitrate > 800 * KILO /* BPS */) sample_point_nominal = 750; - else if (bt->bitrate > 500 * CAN_KBPS) + else if (bt->bitrate > 500 * KILO /* BPS */) sample_point_nominal = 800; else sample_point_nominal = 875; diff --git a/drivers/net/can/spi/hi311x.c b/drivers/net/can/spi/hi311x.c index 89d9c986a229..a17641d36468 100644 --- a/drivers/net/can/spi/hi311x.c +++ b/drivers/net/can/spi/hi311x.c @@ -25,11 +25,11 @@ #include <linux/interrupt.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/mod_devicetable.h> #include <linux/module.h> #include <linux/netdevice.h> -#include <linux/of.h> -#include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/property.h> #include <linux/regulator/consumer.h> #include <linux/slab.h> #include <linux/spi/spi.h> @@ -828,19 +828,25 @@ MODULE_DEVICE_TABLE(spi, hi3110_id_table); static int hi3110_can_probe(struct spi_device *spi) { - const struct of_device_id *of_id = of_match_device(hi3110_of_match, - &spi->dev); + struct device *dev = &spi->dev; struct net_device *net; struct hi3110_priv *priv; + const void *match; struct clk *clk; - int freq, ret; + u32 freq; + int ret; + + clk = devm_clk_get_optional(&spi->dev, NULL); + if (IS_ERR(clk)) + return dev_err_probe(dev, PTR_ERR(clk), "no CAN clock source defined\n"); - clk = devm_clk_get(&spi->dev, NULL); - if (IS_ERR(clk)) { - dev_err(&spi->dev, "no CAN clock source defined\n"); - return PTR_ERR(clk); + if (clk) { + freq = clk_get_rate(clk); + } else { + ret = device_property_read_u32(dev, "clock-frequency", &freq); + if (ret) + return dev_err_probe(dev, ret, "Failed to get clock-frequency!\n"); } - freq = clk_get_rate(clk); /* Sanity check */ if (freq > 40000000) @@ -851,11 +857,9 @@ static int hi3110_can_probe(struct spi_device *spi) if (!net) return -ENOMEM; - if (!IS_ERR(clk)) { - ret = clk_prepare_enable(clk); - if (ret) - goto out_free; - } + ret = clk_prepare_enable(clk); + if (ret) + goto out_free; net->netdev_ops = &hi3110_netdev_ops; net->flags |= IFF_ECHO; @@ -870,8 +874,9 @@ static int hi3110_can_probe(struct spi_device *spi) CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_BERR_REPORTING; - if (of_id) - priv->model = (enum hi3110_model)(uintptr_t)of_id->data; + match = device_get_match_data(dev); + if (match) + priv->model = (enum hi3110_model)(uintptr_t)match; else priv->model = spi_get_device_id(spi)->driver_data; priv->net = net; @@ -918,9 +923,7 @@ static int hi3110_can_probe(struct spi_device *spi) ret = hi3110_hw_probe(spi); if (ret) { - if (ret == -ENODEV) - dev_err(&spi->dev, "Cannot initialize %x. Wrong wiring?\n", - priv->model); + dev_err_probe(dev, ret, "Cannot initialize %x. Wrong wiring?\n", priv->model); goto error_probe; } hi3110_hw_sleep(spi); @@ -938,14 +941,12 @@ static int hi3110_can_probe(struct spi_device *spi) hi3110_power_enable(priv->power, 0); out_clk: - if (!IS_ERR(clk)) - clk_disable_unprepare(clk); + clk_disable_unprepare(clk); out_free: free_candev(net); - dev_err(&spi->dev, "Probe failed, err=%d\n", -ret); - return ret; + return dev_err_probe(dev, ret, "Probe failed\n"); } static int hi3110_can_remove(struct spi_device *spi) @@ -957,8 +958,7 @@ static int hi3110_can_remove(struct spi_device *spi) hi3110_power_enable(priv->power, 0); - if (!IS_ERR(priv->clk)) - clk_disable_unprepare(priv->clk); + clk_disable_unprepare(priv->clk); free_candev(net); diff --git a/drivers/net/can/sun4i_can.c b/drivers/net/can/sun4i_can.c index 54aa7c25c4de..37862e7f6840 100644 --- a/drivers/net/can/sun4i_can.c +++ b/drivers/net/can/sun4i_can.c @@ -61,6 +61,7 @@ #include <linux/of.h> #include <linux/of_device.h> #include <linux/platform_device.h> +#include <linux/reset.h> #define DRV_NAME "sun4i_can" @@ -200,10 +201,20 @@ #define SUN4I_CAN_MAX_IRQ 20 #define SUN4I_MODE_MAX_RETRIES 100 +/** + * struct sun4ican_quirks - Differences between SoC variants. + * + * @has_reset: SoC needs reset deasserted. + */ +struct sun4ican_quirks { + bool has_reset; +}; + struct sun4ican_priv { struct can_priv can; void __iomem *base; struct clk *clk; + struct reset_control *reset; spinlock_t cmdreg_lock; /* lock for concurrent cmd register writes */ }; @@ -702,6 +713,13 @@ static int sun4ican_open(struct net_device *dev) goto exit_irq; } + /* software reset deassert */ + err = reset_control_deassert(priv->reset); + if (err) { + netdev_err(dev, "could not deassert CAN reset\n"); + goto exit_soft_reset; + } + /* turn on clocking for CAN peripheral block */ err = clk_prepare_enable(priv->clk); if (err) { @@ -723,6 +741,8 @@ static int sun4ican_open(struct net_device *dev) exit_can_start: clk_disable_unprepare(priv->clk); exit_clock: + reset_control_assert(priv->reset); +exit_soft_reset: free_irq(dev->irq, dev); exit_irq: close_candev(dev); @@ -736,6 +756,7 @@ static int sun4ican_close(struct net_device *dev) netif_stop_queue(dev); sun4i_can_stop(dev); clk_disable_unprepare(priv->clk); + reset_control_assert(priv->reset); free_irq(dev->irq, dev); close_candev(dev); @@ -750,9 +771,27 @@ static const struct net_device_ops sun4ican_netdev_ops = { .ndo_start_xmit = sun4ican_start_xmit, }; +static const struct sun4ican_quirks sun4ican_quirks_a10 = { + .has_reset = false, +}; + +static const struct sun4ican_quirks sun4ican_quirks_r40 = { + .has_reset = true, +}; + static const struct of_device_id sun4ican_of_match[] = { - {.compatible = "allwinner,sun4i-a10-can"}, - {}, + { + .compatible = "allwinner,sun4i-a10-can", + .data = &sun4ican_quirks_a10 + }, { + .compatible = "allwinner,sun7i-a20-can", + .data = &sun4ican_quirks_a10 + }, { + .compatible = "allwinner,sun8i-r40-can", + .data = &sun4ican_quirks_r40 + }, { + /* sentinel */ + }, }; MODULE_DEVICE_TABLE(of, sun4ican_of_match); @@ -771,10 +810,28 @@ static int sun4ican_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct clk *clk; + struct reset_control *reset = NULL; void __iomem *addr; int err, irq; struct net_device *dev; struct sun4ican_priv *priv; + const struct sun4ican_quirks *quirks; + + quirks = of_device_get_match_data(&pdev->dev); + if (!quirks) { + dev_err(&pdev->dev, "failed to determine the quirks to use\n"); + err = -ENODEV; + goto exit; + } + + if (quirks->has_reset) { + reset = devm_reset_control_get_exclusive(&pdev->dev, NULL); + if (IS_ERR(reset)) { + dev_err(&pdev->dev, "unable to request reset\n"); + err = PTR_ERR(reset); + goto exit; + } + } clk = of_clk_get(np, 0); if (IS_ERR(clk)) { @@ -818,6 +875,7 @@ static int sun4ican_probe(struct platform_device *pdev) CAN_CTRLMODE_3_SAMPLES; priv->base = addr; priv->clk = clk; + priv->reset = reset; spin_lock_init(&priv->cmdreg_lock); platform_set_drvdata(pdev, dev); diff --git a/drivers/net/can/usb/etas_es58x/es581_4.c b/drivers/net/can/usb/etas_es58x/es581_4.c index 14e360c9f2c9..1bcdcece5ec7 100644 --- a/drivers/net/can/usb/etas_es58x/es581_4.c +++ b/drivers/net/can/usb/etas_es58x/es581_4.c @@ -10,6 +10,7 @@ */ #include <linux/kernel.h> +#include <linux/units.h> #include <asm/unaligned.h> #include "es58x_core.h" @@ -469,8 +470,8 @@ const struct es58x_parameters es581_4_param = { .bittiming_const = &es581_4_bittiming_const, .data_bittiming_const = NULL, .tdc_const = NULL, - .bitrate_max = 1 * CAN_MBPS, - .clock = {.freq = 50 * CAN_MHZ}, + .bitrate_max = 1 * MEGA /* BPS */, + .clock = {.freq = 50 * MEGA /* Hz */}, .ctrlmode_supported = CAN_CTRLMODE_CC_LEN8_DLC, .tx_start_of_frame = 0xAFAF, .rx_start_of_frame = 0xFAFA, diff --git a/drivers/net/can/usb/etas_es58x/es58x_fd.c b/drivers/net/can/usb/etas_es58x/es58x_fd.c index 4f0cae29f4d8..ec87126e1a7d 100644 --- a/drivers/net/can/usb/etas_es58x/es58x_fd.c +++ b/drivers/net/can/usb/etas_es58x/es58x_fd.c @@ -12,6 +12,7 @@ */ #include <linux/kernel.h> +#include <linux/units.h> #include <asm/unaligned.h> #include "es58x_core.h" @@ -522,8 +523,8 @@ const struct es58x_parameters es58x_fd_param = { * Mbps work in an optimal environment but are not recommended * for production environment. */ - .bitrate_max = 8 * CAN_MBPS, - .clock = {.freq = 80 * CAN_MHZ}, + .bitrate_max = 8 * MEGA /* BPS */, + .clock = {.freq = 80 * MEGA /* Hz */}, .ctrlmode_supported = CAN_CTRLMODE_LOOPBACK | CAN_CTRLMODE_LISTENONLY | CAN_CTRLMODE_3_SAMPLES | CAN_CTRLMODE_FD | CAN_CTRLMODE_FD_NON_ISO | CAN_CTRLMODE_CC_LEN8_DLC | CAN_CTRLMODE_TDC_AUTO, diff --git a/drivers/net/dsa/b53/b53_common.c b/drivers/net/dsa/b53/b53_common.c index af4761968733..3867f3d4545f 100644 --- a/drivers/net/dsa/b53/b53_common.c +++ b/drivers/net/dsa/b53/b53_common.c @@ -1860,7 +1860,8 @@ int b53_mdb_del(struct dsa_switch *ds, int port, } EXPORT_SYMBOL(b53_mdb_del); -int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) +int b53_br_join(struct dsa_switch *ds, int port, struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct b53_device *dev = ds->priv; s8 cpu_port = dsa_to_port(ds, port)->cpu_dp->index; @@ -1887,7 +1888,7 @@ int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(port), &pvlan); b53_for_each_port(dev, i) { - if (dsa_to_port(ds, i)->bridge_dev != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Add this local port to the remote port VLAN control @@ -1911,7 +1912,7 @@ int b53_br_join(struct dsa_switch *ds, int port, struct net_device *br) } EXPORT_SYMBOL(b53_br_join); -void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *br) +void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge) { struct b53_device *dev = ds->priv; struct b53_vlan *vl = &dev->vlans[0]; @@ -1923,7 +1924,7 @@ void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *br) b53_for_each_port(dev, i) { /* Don't touch the remaining ports */ - if (dsa_to_port(ds, i)->bridge_dev != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; b53_read16(dev, B53_PVLAN_PAGE, B53_PVLAN_PORT_MASK(i), ®); diff --git a/drivers/net/dsa/b53/b53_priv.h b/drivers/net/dsa/b53/b53_priv.h index 579da74ada64..b41dc8ac2ca8 100644 --- a/drivers/net/dsa/b53/b53_priv.h +++ b/drivers/net/dsa/b53/b53_priv.h @@ -324,8 +324,9 @@ void b53_get_strings(struct dsa_switch *ds, int port, u32 stringset, void b53_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data); int b53_get_sset_count(struct dsa_switch *ds, int port, int sset); void b53_get_ethtool_phy_stats(struct dsa_switch *ds, int port, uint64_t *data); -int b53_br_join(struct dsa_switch *ds, int port, struct net_device *bridge); -void b53_br_leave(struct dsa_switch *ds, int port, struct net_device *bridge); +int b53_br_join(struct dsa_switch *ds, int port, struct dsa_bridge bridge, + bool *tx_fwd_offload); +void b53_br_leave(struct dsa_switch *ds, int port, struct dsa_bridge bridge); void b53_br_set_stp_state(struct dsa_switch *ds, int port, u8 state); void b53_br_fast_age(struct dsa_switch *ds, int port); int b53_br_flags_pre(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/dsa_loop.c b/drivers/net/dsa/dsa_loop.c index e638e3eea911..33daaf10c488 100644 --- a/drivers/net/dsa/dsa_loop.c +++ b/drivers/net/dsa/dsa_loop.c @@ -167,19 +167,20 @@ static int dsa_loop_phy_write(struct dsa_switch *ds, int port, } static int dsa_loop_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n", - __func__, port, bridge->name); + __func__, port, bridge.dev->name); return 0; } static void dsa_loop_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { dev_dbg(ds->dev, "%s: port: %d, bridge: %s\n", - __func__, port, bridge->name); + __func__, port, bridge.dev->name); } static void dsa_loop_port_stp_state_set(struct dsa_switch *ds, int port, diff --git a/drivers/net/dsa/hirschmann/hellcreek.c b/drivers/net/dsa/hirschmann/hellcreek.c index 4e0b53d94b52..9eecb7529573 100644 --- a/drivers/net/dsa/hirschmann/hellcreek.c +++ b/drivers/net/dsa/hirschmann/hellcreek.c @@ -674,7 +674,8 @@ static int hellcreek_bridge_flags(struct dsa_switch *ds, int port, } static int hellcreek_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct hellcreek *hellcreek = ds->priv; @@ -691,7 +692,7 @@ static int hellcreek_port_bridge_join(struct dsa_switch *ds, int port, } static void hellcreek_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct hellcreek *hellcreek = ds->priv; @@ -1384,14 +1385,19 @@ static void hellcreek_teardown(struct dsa_switch *ds) dsa_devlink_resources_unregister(ds); } -static void hellcreek_phylink_validate(struct dsa_switch *ds, int port, - unsigned long *supported, - struct phylink_link_state *state) +static void hellcreek_phylink_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) { - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; struct hellcreek *hellcreek = ds->priv; - dev_dbg(hellcreek->dev, "Phylink validate for port %d\n", port); + __set_bit(PHY_INTERFACE_MODE_MII, config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RGMII, config->supported_interfaces); + + /* Include GMII - the hardware does not support this interface + * mode, but it's the default interface mode for phylib, so we + * need it for compatibility with existing DT. + */ + __set_bit(PHY_INTERFACE_MODE_GMII, config->supported_interfaces); /* The MAC settings are a hardware configuration option and cannot be * changed at run time or by strapping. Therefore the attached PHYs @@ -1399,12 +1405,9 @@ static void hellcreek_phylink_validate(struct dsa_switch *ds, int port, * by the hardware. */ if (hellcreek->pdata->is_100_mbits) - phylink_set(mask, 100baseT_Full); + config->mac_capabilities = MAC_100FD; else - phylink_set(mask, 1000baseT_Full); - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); + config->mac_capabilities = MAC_1000FD; } static int @@ -1755,7 +1758,7 @@ static const struct dsa_switch_ops hellcreek_ds_ops = { .get_strings = hellcreek_get_strings, .get_tag_protocol = hellcreek_get_tag_protocol, .get_ts_info = hellcreek_get_ts_info, - .phylink_validate = hellcreek_phylink_validate, + .phylink_get_caps = hellcreek_phylink_get_caps, .port_bridge_flags = hellcreek_bridge_flags, .port_bridge_join = hellcreek_port_bridge_join, .port_bridge_leave = hellcreek_port_bridge_leave, diff --git a/drivers/net/dsa/lan9303-core.c b/drivers/net/dsa/lan9303-core.c index 89f920289ae2..d55784d19fa4 100644 --- a/drivers/net/dsa/lan9303-core.c +++ b/drivers/net/dsa/lan9303-core.c @@ -1103,12 +1103,13 @@ static void lan9303_port_disable(struct dsa_switch *ds, int port) } static int lan9303_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct lan9303 *chip = ds->priv; dev_dbg(chip->dev, "%s(port %d)\n", __func__, port); - if (dsa_to_port(ds, 1)->bridge_dev == dsa_to_port(ds, 2)->bridge_dev) { + if (dsa_port_bridge_same(dsa_to_port(ds, 1), dsa_to_port(ds, 2))) { lan9303_bridge_ports(chip); chip->is_bridged = true; /* unleash stp_state_set() */ } @@ -1117,7 +1118,7 @@ static int lan9303_port_bridge_join(struct dsa_switch *ds, int port, } static void lan9303_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct lan9303 *chip = ds->priv; diff --git a/drivers/net/dsa/lantiq_gswip.c b/drivers/net/dsa/lantiq_gswip.c index 7056d98d8177..46ed953e787e 100644 --- a/drivers/net/dsa/lantiq_gswip.c +++ b/drivers/net/dsa/lantiq_gswip.c @@ -759,7 +759,7 @@ static int gswip_port_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering, struct netlink_ext_ack *extack) { - struct net_device *bridge = dsa_to_port(ds, port)->bridge_dev; + struct net_device *bridge = dsa_port_bridge_dev_get(dsa_to_port(ds, port)); struct gswip_priv *priv = ds->priv; /* Do not allow changing the VLAN filtering options while in bridge */ @@ -1146,16 +1146,18 @@ static int gswip_vlan_remove(struct gswip_priv *priv, } static int gswip_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { + struct net_device *br = bridge.dev; struct gswip_priv *priv = ds->priv; int err; /* When the bridge uses VLAN filtering we have to configure VLAN * specific bridges. No bridge is configured here. */ - if (!br_vlan_enabled(bridge)) { - err = gswip_vlan_add_unaware(priv, bridge, port); + if (!br_vlan_enabled(br)) { + err = gswip_vlan_add_unaware(priv, br, port); if (err) return err; priv->port_vlan_filter &= ~BIT(port); @@ -1166,8 +1168,9 @@ static int gswip_port_bridge_join(struct dsa_switch *ds, int port, } static void gswip_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { + struct net_device *br = bridge.dev; struct gswip_priv *priv = ds->priv; gswip_add_single_port_br(priv, port, true); @@ -1175,16 +1178,16 @@ static void gswip_port_bridge_leave(struct dsa_switch *ds, int port, /* When the bridge uses VLAN filtering we have to configure VLAN * specific bridges. No bridge is configured here. */ - if (!br_vlan_enabled(bridge)) - gswip_vlan_remove(priv, bridge, port, 0, true, false); + if (!br_vlan_enabled(br)) + gswip_vlan_remove(priv, br, port, 0, true, false); } static int gswip_port_vlan_prepare(struct dsa_switch *ds, int port, const struct switchdev_obj_port_vlan *vlan, struct netlink_ext_ack *extack) { + struct net_device *bridge = dsa_port_bridge_dev_get(dsa_to_port(ds, port)); struct gswip_priv *priv = ds->priv; - struct net_device *bridge = dsa_to_port(ds, port)->bridge_dev; unsigned int max_ports = priv->hw_info->max_ports; int pos = max_ports; int i, idx = -1; @@ -1229,8 +1232,8 @@ static int gswip_port_vlan_add(struct dsa_switch *ds, int port, const struct switchdev_obj_port_vlan *vlan, struct netlink_ext_ack *extack) { + struct net_device *bridge = dsa_port_bridge_dev_get(dsa_to_port(ds, port)); struct gswip_priv *priv = ds->priv; - struct net_device *bridge = dsa_to_port(ds, port)->bridge_dev; bool untagged = vlan->flags & BRIDGE_VLAN_INFO_UNTAGGED; bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID; int err; @@ -1254,8 +1257,8 @@ static int gswip_port_vlan_add(struct dsa_switch *ds, int port, static int gswip_port_vlan_del(struct dsa_switch *ds, int port, const struct switchdev_obj_port_vlan *vlan) { + struct net_device *bridge = dsa_port_bridge_dev_get(dsa_to_port(ds, port)); struct gswip_priv *priv = ds->priv; - struct net_device *bridge = dsa_to_port(ds, port)->bridge_dev; bool pvid = vlan->flags & BRIDGE_VLAN_INFO_PVID; /* We have to receive all packets on the CPU port and should not @@ -1340,8 +1343,8 @@ static void gswip_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) static int gswip_port_fdb(struct dsa_switch *ds, int port, const unsigned char *addr, u16 vid, bool add) { + struct net_device *bridge = dsa_port_bridge_dev_get(dsa_to_port(ds, port)); struct gswip_priv *priv = ds->priv; - struct net_device *bridge = dsa_to_port(ds, port)->bridge_dev; struct gswip_pce_table_entry mac_bridge = {0,}; unsigned int cpu_port = priv->hw_info->cpu_port; int fid = -1; @@ -1438,114 +1441,70 @@ static int gswip_port_fdb_dump(struct dsa_switch *ds, int port, return 0; } -static void gswip_phylink_set_capab(unsigned long *supported, - struct phylink_link_state *state) -{ - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - /* Allow all the expected bits */ - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - - /* With the exclusion of MII, Reverse MII and Reduced MII, we - * support Gigabit, including Half duplex - */ - if (state->interface != PHY_INTERFACE_MODE_MII && - state->interface != PHY_INTERFACE_MODE_REVMII && - state->interface != PHY_INTERFACE_MODE_RMII) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseT_Half); - } - - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - -static void gswip_xrx200_phylink_validate(struct dsa_switch *ds, int port, - unsigned long *supported, - struct phylink_link_state *state) +static void gswip_xrx200_phylink_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) { switch (port) { case 0: case 1: - if (!phy_interface_mode_is_rgmii(state->interface) && - state->interface != PHY_INTERFACE_MODE_MII && - state->interface != PHY_INTERFACE_MODE_REVMII && - state->interface != PHY_INTERFACE_MODE_RMII) - goto unsupported; + phy_interface_set_rgmii(config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_MII, + config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_REVMII, + config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + config->supported_interfaces); break; + case 2: case 3: case 4: - if (state->interface != PHY_INTERFACE_MODE_INTERNAL) - goto unsupported; + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); break; + case 5: - if (!phy_interface_mode_is_rgmii(state->interface) && - state->interface != PHY_INTERFACE_MODE_INTERNAL) - goto unsupported; + phy_interface_set_rgmii(config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); break; - default: - linkmode_zero(supported); - dev_err(ds->dev, "Unsupported port: %i\n", port); - return; } - gswip_phylink_set_capab(supported, state); - - return; - -unsupported: - linkmode_zero(supported); - dev_err(ds->dev, "Unsupported interface '%s' for port %d\n", - phy_modes(state->interface), port); + config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000; } -static void gswip_xrx300_phylink_validate(struct dsa_switch *ds, int port, - unsigned long *supported, - struct phylink_link_state *state) +static void gswip_xrx300_phylink_get_caps(struct dsa_switch *ds, int port, + struct phylink_config *config) { switch (port) { case 0: - if (!phy_interface_mode_is_rgmii(state->interface) && - state->interface != PHY_INTERFACE_MODE_GMII && - state->interface != PHY_INTERFACE_MODE_RMII) - goto unsupported; + phy_interface_set_rgmii(config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + config->supported_interfaces); break; + case 1: case 2: case 3: case 4: - if (state->interface != PHY_INTERFACE_MODE_INTERNAL) - goto unsupported; + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); break; + case 5: - if (!phy_interface_mode_is_rgmii(state->interface) && - state->interface != PHY_INTERFACE_MODE_INTERNAL && - state->interface != PHY_INTERFACE_MODE_RMII) - goto unsupported; + phy_interface_set_rgmii(config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + config->supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + config->supported_interfaces); break; - default: - linkmode_zero(supported); - dev_err(ds->dev, "Unsupported port: %i\n", port); - return; } - gswip_phylink_set_capab(supported, state); - - return; - -unsupported: - linkmode_zero(supported); - dev_err(ds->dev, "Unsupported interface '%s' for port %d\n", - phy_modes(state->interface), port); + config->mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000; } static void gswip_port_set_link(struct gswip_priv *priv, int port, bool link) @@ -1827,7 +1786,7 @@ static const struct dsa_switch_ops gswip_xrx200_switch_ops = { .port_fdb_add = gswip_port_fdb_add, .port_fdb_del = gswip_port_fdb_del, .port_fdb_dump = gswip_port_fdb_dump, - .phylink_validate = gswip_xrx200_phylink_validate, + .phylink_get_caps = gswip_xrx200_phylink_get_caps, .phylink_mac_config = gswip_phylink_mac_config, .phylink_mac_link_down = gswip_phylink_mac_link_down, .phylink_mac_link_up = gswip_phylink_mac_link_up, @@ -1851,7 +1810,7 @@ static const struct dsa_switch_ops gswip_xrx300_switch_ops = { .port_fdb_add = gswip_port_fdb_add, .port_fdb_del = gswip_port_fdb_del, .port_fdb_dump = gswip_port_fdb_dump, - .phylink_validate = gswip_xrx300_phylink_validate, + .phylink_get_caps = gswip_xrx300_phylink_get_caps, .phylink_mac_config = gswip_phylink_mac_config, .phylink_mac_link_down = gswip_phylink_mac_link_down, .phylink_mac_link_up = gswip_phylink_mac_link_up, diff --git a/drivers/net/dsa/microchip/ksz_common.c b/drivers/net/dsa/microchip/ksz_common.c index 8a04302018dc..47a856533cff 100644 --- a/drivers/net/dsa/microchip/ksz_common.c +++ b/drivers/net/dsa/microchip/ksz_common.c @@ -43,7 +43,7 @@ void ksz_update_port_member(struct ksz_device *dev, int port) continue; if (port == i) continue; - if (!dp->bridge_dev || dp->bridge_dev != other_dp->bridge_dev) + if (!dsa_port_bridge_same(dp, other_dp)) continue; if (other_p->stp_state == BR_STATE_FORWARDING && @@ -192,7 +192,8 @@ void ksz_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *buf) EXPORT_SYMBOL_GPL(ksz_get_ethtool_stats); int ksz_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { /* port_stp_state_set() will be called after to put the port in * appropriate state so there is no need to do anything. @@ -203,7 +204,7 @@ int ksz_port_bridge_join(struct dsa_switch *ds, int port, EXPORT_SYMBOL_GPL(ksz_port_bridge_join); void ksz_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { /* port_stp_state_set() will be called after to put the port in * forwarding state so there is no need to do anything. diff --git a/drivers/net/dsa/microchip/ksz_common.h b/drivers/net/dsa/microchip/ksz_common.h index 54b456bc8972..df8ae59c8525 100644 --- a/drivers/net/dsa/microchip/ksz_common.h +++ b/drivers/net/dsa/microchip/ksz_common.h @@ -155,9 +155,9 @@ void ksz_mac_link_down(struct dsa_switch *ds, int port, unsigned int mode, int ksz_sset_count(struct dsa_switch *ds, int port, int sset); void ksz_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *buf); int ksz_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br); + struct dsa_bridge bridge, bool *tx_fwd_offload); void ksz_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br); + struct dsa_bridge bridge); void ksz_port_fast_age(struct dsa_switch *ds, int port); int ksz_port_fdb_dump(struct dsa_switch *ds, int port, dsa_fdb_dump_cb_t *cb, void *data); diff --git a/drivers/net/dsa/mt7530.c b/drivers/net/dsa/mt7530.c index 9890672a206d..b82512e5b33b 100644 --- a/drivers/net/dsa/mt7530.c +++ b/drivers/net/dsa/mt7530.c @@ -1186,29 +1186,33 @@ mt7530_port_bridge_flags(struct dsa_switch *ds, int port, static int mt7530_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge, bool *tx_fwd_offload) { - struct mt7530_priv *priv = ds->priv; + struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; u32 port_bitmap = BIT(MT7530_CPU_PORT); - int i; + struct mt7530_priv *priv = ds->priv; mutex_lock(&priv->reg_mutex); - for (i = 0; i < MT7530_NUM_PORTS; i++) { + dsa_switch_for_each_user_port(other_dp, ds) { + int other_port = other_dp->index; + + if (dp == other_dp) + continue; + /* Add this port to the port matrix of the other ports in the * same bridge. If the port is disabled, port matrix is kept * and not being setup until the port becomes enabled. */ - if (dsa_is_user_port(ds, i) && i != port) { - if (dsa_to_port(ds, i)->bridge_dev != bridge) - continue; - if (priv->ports[i].enable) - mt7530_set(priv, MT7530_PCR_P(i), - PCR_MATRIX(BIT(port))); - priv->ports[i].pm |= PCR_MATRIX(BIT(port)); + if (!dsa_port_offloads_bridge(other_dp, &bridge)) + continue; - port_bitmap |= BIT(i); - } + if (priv->ports[other_port].enable) + mt7530_set(priv, MT7530_PCR_P(other_port), + PCR_MATRIX(BIT(port))); + priv->ports[other_port].pm |= PCR_MATRIX(BIT(port)); + + port_bitmap |= BIT(other_port); } /* Add the all other ports to this port matrix. */ @@ -1236,7 +1240,7 @@ mt7530_port_set_vlan_unaware(struct dsa_switch *ds, int port) /* This is called after .port_bridge_leave when leaving a VLAN-aware * bridge. Don't set standalone ports to fallback mode. */ - if (dsa_to_port(ds, port)->bridge_dev) + if (dsa_port_bridge_dev_get(dsa_to_port(ds, port))) mt7530_rmw(priv, MT7530_PCR_P(port), PCR_PORT_VLAN_MASK, MT7530_PORT_FALLBACK_MODE); @@ -1299,26 +1303,30 @@ mt7530_port_set_vlan_aware(struct dsa_switch *ds, int port) static void mt7530_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { + struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; struct mt7530_priv *priv = ds->priv; - int i; mutex_lock(&priv->reg_mutex); - for (i = 0; i < MT7530_NUM_PORTS; i++) { + dsa_switch_for_each_user_port(other_dp, ds) { + int other_port = other_dp->index; + + if (dp == other_dp) + continue; + /* Remove this port from the port matrix of the other ports * in the same bridge. If the port is disabled, port matrix * is kept and not being setup until the port becomes enabled. */ - if (dsa_is_user_port(ds, i) && i != port) { - if (dsa_to_port(ds, i)->bridge_dev != bridge) - continue; - if (priv->ports[i].enable) - mt7530_clear(priv, MT7530_PCR_P(i), - PCR_MATRIX(BIT(port))); - priv->ports[i].pm &= ~PCR_MATRIX(BIT(port)); - } + if (!dsa_port_offloads_bridge(other_dp, &bridge)) + continue; + + if (priv->ports[other_port].enable) + mt7530_clear(priv, MT7530_PCR_P(other_port), + PCR_MATRIX(BIT(port))); + priv->ports[other_port].pm &= ~PCR_MATRIX(BIT(port)); } /* Set the cpu port to be the only one in the port matrix of diff --git a/drivers/net/dsa/mv88e6xxx/chip.c b/drivers/net/dsa/mv88e6xxx/chip.c index 14f87f6ac479..45aaa7333fb8 100644 --- a/drivers/net/dsa/mv88e6xxx/chip.c +++ b/drivers/net/dsa/mv88e6xxx/chip.c @@ -1237,8 +1237,7 @@ static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port) { struct dsa_switch *ds = chip->ds; struct dsa_switch_tree *dst = ds->dst; - struct net_device *br; - struct dsa_port *dp; + struct dsa_port *dp, *other_dp; bool found = false; u16 pvlan; @@ -1247,11 +1246,9 @@ static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port) list_for_each_entry(dp, &dst->ports, list) { if (dp->ds->index == dev && dp->index == port) { /* dp might be a DSA link or a user port, so it - * might or might not have a bridge_dev - * pointer. Use the "found" variable for both - * cases. + * might or might not have a bridge. + * Use the "found" variable for both cases. */ - br = dp->bridge_dev; found = true; break; } @@ -1259,13 +1256,14 @@ static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port) /* dev is a virtual bridge */ } else { list_for_each_entry(dp, &dst->ports, list) { - if (dp->bridge_num < 0) + unsigned int bridge_num = dsa_port_bridge_num_get(dp); + + if (!bridge_num) continue; - if (dp->bridge_num + 1 + dst->last_switch != dev) + if (bridge_num + dst->last_switch != dev) continue; - br = dp->bridge_dev; found = true; break; } @@ -1284,12 +1282,11 @@ static u16 mv88e6xxx_port_vlan(struct mv88e6xxx_chip *chip, int dev, int port) /* Frames from user ports can egress any local DSA links and CPU ports, * as well as any local member of their bridge group. */ - list_for_each_entry(dp, &dst->ports, list) - if (dp->ds == ds && - (dp->type == DSA_PORT_TYPE_CPU || - dp->type == DSA_PORT_TYPE_DSA || - (br && dp->bridge_dev == br))) - pvlan |= BIT(dp->index); + dsa_switch_for_each_port(other_dp, ds) + if (other_dp->type == DSA_PORT_TYPE_CPU || + other_dp->type == DSA_PORT_TYPE_DSA || + dsa_port_bridge_same(dp, other_dp)) + pvlan |= BIT(other_dp->index); return pvlan; } @@ -1656,12 +1653,13 @@ static int mv88e6xxx_atu_new(struct mv88e6xxx_chip *chip, u16 *fid) static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port, u16 vid) { + struct dsa_port *dp = dsa_to_port(ds, port), *other_dp; struct mv88e6xxx_chip *chip = ds->priv; struct mv88e6xxx_vtu_entry vlan; - int i, err; + int err; /* DSA and CPU ports have to be members of multiple vlans */ - if (dsa_is_dsa_port(ds, port) || dsa_is_cpu_port(ds, port)) + if (dsa_port_is_dsa(dp) || dsa_port_is_cpu(dp)) return 0; err = mv88e6xxx_vtu_get(chip, vid, &vlan); @@ -1671,27 +1669,22 @@ static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port, if (!vlan.valid) return 0; - for (i = 0; i < mv88e6xxx_num_ports(chip); ++i) { - if (dsa_is_dsa_port(ds, i) || dsa_is_cpu_port(ds, i)) - continue; + dsa_switch_for_each_user_port(other_dp, ds) { + struct net_device *other_br; - if (!dsa_to_port(ds, i)->slave) - continue; - - if (vlan.member[i] == + if (vlan.member[other_dp->index] == MV88E6XXX_G1_VTU_DATA_MEMBER_TAG_NON_MEMBER) continue; - if (dsa_to_port(ds, i)->bridge_dev == - dsa_to_port(ds, port)->bridge_dev) + if (dsa_port_bridge_same(dp, other_dp)) break; /* same bridge, check next VLAN */ - if (!dsa_to_port(ds, i)->bridge_dev) + other_br = dsa_port_bridge_dev_get(other_dp); + if (!other_br) continue; dev_err(ds->dev, "p%d: hw VLAN %d already used by port %d in %s\n", - port, vlan.vid, i, - netdev_name(dsa_to_port(ds, i)->bridge_dev)); + port, vlan.vid, other_dp->index, netdev_name(other_br)); return -EOPNOTSUPP; } @@ -1701,13 +1694,14 @@ static int mv88e6xxx_port_check_hw_vlan(struct dsa_switch *ds, int port, static int mv88e6xxx_port_commit_pvid(struct mv88e6xxx_chip *chip, int port) { struct dsa_port *dp = dsa_to_port(chip->ds, port); + struct net_device *br = dsa_port_bridge_dev_get(dp); struct mv88e6xxx_port *p = &chip->ports[port]; u16 pvid = MV88E6XXX_VID_STANDALONE; bool drop_untagged = false; int err; - if (dp->bridge_dev) { - if (br_vlan_enabled(dp->bridge_dev)) { + if (br) { + if (br_vlan_enabled(br)) { pvid = p->bridge_pvid.vid; drop_untagged = !p->bridge_pvid.valid; } else { @@ -2425,7 +2419,7 @@ static int mv88e6xxx_port_fdb_dump(struct dsa_switch *ds, int port, } static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, - struct net_device *br) + struct dsa_bridge bridge) { struct dsa_switch *ds = chip->ds; struct dsa_switch_tree *dst = ds->dst; @@ -2433,7 +2427,7 @@ static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, int err; list_for_each_entry(dp, &dst->ports, list) { - if (dp->bridge_dev == br) { + if (dsa_port_offloads_bridge(dp, &bridge)) { if (dp->ds == ds) { /* This is a local bridge group member, * remap its Port VLAN Map. @@ -2456,15 +2450,29 @@ static int mv88e6xxx_bridge_map(struct mv88e6xxx_chip *chip, return 0; } +/* Treat the software bridge as a virtual single-port switch behind the + * CPU and map in the PVT. First dst->last_switch elements are taken by + * physical switches, so start from beyond that range. + */ +static int mv88e6xxx_map_virtual_bridge_to_pvt(struct dsa_switch *ds, + unsigned int bridge_num) +{ + u8 dev = bridge_num + ds->dst->last_switch; + struct mv88e6xxx_chip *chip = ds->priv; + + return mv88e6xxx_pvt_map(chip, dev, 0); +} + static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct mv88e6xxx_chip *chip = ds->priv; int err; mv88e6xxx_reg_lock(chip); - err = mv88e6xxx_bridge_map(chip, br); + err = mv88e6xxx_bridge_map(chip, bridge); if (err) goto unlock; @@ -2472,6 +2480,14 @@ static int mv88e6xxx_port_bridge_join(struct dsa_switch *ds, int port, if (err) goto unlock; + if (mv88e6xxx_has_pvt(chip)) { + err = mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge.num); + if (err) + goto unlock; + + *tx_fwd_offload = true; + } + unlock: mv88e6xxx_reg_unlock(chip); @@ -2479,14 +2495,18 @@ unlock: } static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; int err; mv88e6xxx_reg_lock(chip); - if (mv88e6xxx_bridge_map(chip, br) || + if (bridge.tx_fwd_offload && + mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge.num)) + dev_err(ds->dev, "failed to remap cross-chip Port VLAN\n"); + + if (mv88e6xxx_bridge_map(chip, bridge) || mv88e6xxx_port_vlan_map(chip, port)) dev_err(ds->dev, "failed to remap in-chip Port VLAN\n"); @@ -2501,7 +2521,7 @@ static void mv88e6xxx_port_bridge_leave(struct dsa_switch *ds, int port, static int mv88e6xxx_crosschip_bridge_join(struct dsa_switch *ds, int tree_index, int sw_index, - int port, struct net_device *br) + int port, struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; int err; @@ -2518,7 +2538,7 @@ static int mv88e6xxx_crosschip_bridge_join(struct dsa_switch *ds, static void mv88e6xxx_crosschip_bridge_leave(struct dsa_switch *ds, int tree_index, int sw_index, - int port, struct net_device *br) + int port, struct dsa_bridge bridge) { struct mv88e6xxx_chip *chip = ds->priv; @@ -2531,44 +2551,6 @@ static void mv88e6xxx_crosschip_bridge_leave(struct dsa_switch *ds, mv88e6xxx_reg_unlock(chip); } -/* Treat the software bridge as a virtual single-port switch behind the - * CPU and map in the PVT. First dst->last_switch elements are taken by - * physical switches, so start from beyond that range. - */ -static int mv88e6xxx_map_virtual_bridge_to_pvt(struct dsa_switch *ds, - int bridge_num) -{ - u8 dev = bridge_num + ds->dst->last_switch + 1; - struct mv88e6xxx_chip *chip = ds->priv; - int err; - - mv88e6xxx_reg_lock(chip); - err = mv88e6xxx_pvt_map(chip, dev, 0); - mv88e6xxx_reg_unlock(chip); - - return err; -} - -static int mv88e6xxx_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num) -{ - return mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge_num); -} - -static void mv88e6xxx_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num) -{ - int err; - - err = mv88e6xxx_map_virtual_bridge_to_pvt(ds, bridge_num); - if (err) { - dev_err(ds->dev, "failed to remap cross-chip Port VLAN: %pe\n", - ERR_PTR(err)); - } -} - static int mv88e6xxx_software_reset(struct mv88e6xxx_chip *chip) { if (chip->info->ops->reset) @@ -3195,8 +3177,8 @@ static int mv88e6xxx_setup(struct dsa_switch *ds) * time. */ if (mv88e6xxx_has_pvt(chip)) - ds->num_fwd_offloading_bridges = MV88E6XXX_MAX_PVT_SWITCHES - - ds->dst->last_switch - 1; + ds->max_num_bridges = MV88E6XXX_MAX_PVT_SWITCHES - + ds->dst->last_switch - 1; mv88e6xxx_reg_lock(chip); @@ -6288,8 +6270,6 @@ static const struct dsa_switch_ops mv88e6xxx_switch_ops = { .crosschip_lag_change = mv88e6xxx_crosschip_lag_change, .crosschip_lag_join = mv88e6xxx_crosschip_lag_join, .crosschip_lag_leave = mv88e6xxx_crosschip_lag_leave, - .port_bridge_tx_fwd_offload = mv88e6xxx_bridge_tx_fwd_offload, - .port_bridge_tx_fwd_unoffload = mv88e6xxx_bridge_tx_fwd_unoffload, }; static int mv88e6xxx_register_switch(struct mv88e6xxx_chip *chip) diff --git a/drivers/net/dsa/ocelot/Kconfig b/drivers/net/dsa/ocelot/Kconfig index 9948544ba1c4..220b0b027b55 100644 --- a/drivers/net/dsa/ocelot/Kconfig +++ b/drivers/net/dsa/ocelot/Kconfig @@ -21,6 +21,7 @@ config NET_DSA_MSCC_SEVILLE depends on NET_VENDOR_MICROSEMI depends on HAS_IOMEM depends on PTP_1588_CLOCK_OPTIONAL + select MDIO_MSCC_MIIM select MSCC_OCELOT_SWITCH_LIB select NET_DSA_TAG_OCELOT_8021Q select NET_DSA_TAG_OCELOT diff --git a/drivers/net/dsa/ocelot/felix.c b/drivers/net/dsa/ocelot/felix.c index f1a05e7dc818..a02fec33552d 100644 --- a/drivers/net/dsa/ocelot/felix.c +++ b/drivers/net/dsa/ocelot/felix.c @@ -240,24 +240,32 @@ static int felix_tag_8021q_vlan_del(struct dsa_switch *ds, int port, u16 vid) */ static void felix_8021q_cpu_port_init(struct ocelot *ocelot, int port) { + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ports[port]->is_dsa_8021q_cpu = true; ocelot->npi = -1; /* Overwrite PGID_CPU with the non-tagging port */ ocelot_write_rix(ocelot, BIT(port), ANA_PGID_PGID, PGID_CPU); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, true); + + mutex_unlock(&ocelot->fwd_domain_lock); } static void felix_8021q_cpu_port_deinit(struct ocelot *ocelot, int port) { + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ports[port]->is_dsa_8021q_cpu = false; /* Restore PGID_CPU */ ocelot_write_rix(ocelot, BIT(ocelot->num_phys_ports), ANA_PGID_PGID, PGID_CPU); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, true); + + mutex_unlock(&ocelot->fwd_domain_lock); } /* Set up a VCAP IS2 rule for delivering PTP frames to the CPU port module. @@ -701,21 +709,21 @@ static int felix_bridge_flags(struct dsa_switch *ds, int port, } static int felix_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, bool *tx_fwd_offload) { struct ocelot *ocelot = ds->priv; - ocelot_port_bridge_join(ocelot, port, br); + ocelot_port_bridge_join(ocelot, port, bridge.dev); return 0; } static void felix_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { struct ocelot *ocelot = ds->priv; - ocelot_port_bridge_leave(ocelot, port, br); + ocelot_port_bridge_leave(ocelot, port, bridge.dev); } static int felix_lag_join(struct dsa_switch *ds, int port, @@ -823,7 +831,7 @@ static void felix_phylink_mac_config(struct dsa_switch *ds, int port, struct felix *felix = ocelot_to_felix(ocelot); struct dsa_port *dp = dsa_to_port(ds, port); - if (felix->pcs[port]) + if (felix->pcs && felix->pcs[port]) phylink_set_pcs(dp->pl, &felix->pcs[port]->pcs); } @@ -992,6 +1000,10 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) ocelot->num_stats = felix->info->num_stats; ocelot->num_mact_rows = felix->info->num_mact_rows; ocelot->vcap = felix->info->vcap; + ocelot->vcap_pol.base = felix->info->vcap_pol_base; + ocelot->vcap_pol.max = felix->info->vcap_pol_max; + ocelot->vcap_pol.base2 = felix->info->vcap_pol_base2; + ocelot->vcap_pol.max2 = felix->info->vcap_pol_max2; ocelot->ops = felix->info->ops; ocelot->npi_inj_prefix = OCELOT_TAG_PREFIX_SHORT; ocelot->npi_xtr_prefix = OCELOT_TAG_PREFIX_SHORT; @@ -1019,7 +1031,7 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) res.start += felix->switch_base; res.end += felix->switch_base; - target = ocelot_regmap_init(ocelot, &res); + target = felix->info->init_regmap(ocelot, &res); if (IS_ERR(target)) { dev_err(ocelot->dev, "Failed to map device memory space\n"); @@ -1056,7 +1068,7 @@ static int felix_init_structs(struct felix *felix, int num_phys_ports) res.start += felix->switch_base; res.end += felix->switch_base; - target = ocelot_regmap_init(ocelot, &res); + target = felix->info->init_regmap(ocelot, &res); if (IS_ERR(target)) { dev_err(ocelot->dev, "Failed to map memory space for port %d\n", diff --git a/drivers/net/dsa/ocelot/felix.h b/drivers/net/dsa/ocelot/felix.h index be3e42e135c0..515bddc012c0 100644 --- a/drivers/net/dsa/ocelot/felix.h +++ b/drivers/net/dsa/ocelot/felix.h @@ -21,8 +21,10 @@ struct felix_info { int num_ports; int num_tx_queues; struct vcap_props *vcap; - int switch_pci_bar; - int imdio_pci_bar; + u16 vcap_pol_base; + u16 vcap_pol_max; + u16 vcap_pol_base2; + u16 vcap_pol_max2; const struct ptp_clock_info *ptp_caps; /* Some Ocelot switches are integrated into the SoC without the @@ -48,6 +50,8 @@ struct felix_info { enum tc_setup_type type, void *type_data); void (*port_sched_speed_set)(struct ocelot *ocelot, int port, u32 speed); + struct regmap *(*init_regmap)(struct ocelot *ocelot, + struct resource *res); }; extern const struct dsa_switch_ops felix_switch_ops; diff --git a/drivers/net/dsa/ocelot/felix_vsc9959.c b/drivers/net/dsa/ocelot/felix_vsc9959.c index 45c5ec7a83ea..110d6c403bdd 100644 --- a/drivers/net/dsa/ocelot/felix_vsc9959.c +++ b/drivers/net/dsa/ocelot/felix_vsc9959.c @@ -5,8 +5,10 @@ #include <linux/fsl/enetc_mdio.h> #include <soc/mscc/ocelot_qsys.h> #include <soc/mscc/ocelot_vcap.h> +#include <soc/mscc/ocelot_ana.h> #include <soc/mscc/ocelot_ptp.h> #include <soc/mscc/ocelot_sys.h> +#include <net/tc_act/tc_gate.h> #include <soc/mscc/ocelot.h> #include <linux/dsa/ocelot.h> #include <linux/pcs-lynx.h> @@ -17,6 +19,10 @@ #include "felix.h" #define VSC9959_TAS_GCL_ENTRY_MAX 63 +#define VSC9959_VCAP_POLICER_BASE 63 +#define VSC9959_VCAP_POLICER_MAX 383 +#define VSC9959_SWITCH_PCI_BAR 4 +#define VSC9959_IMDIO_PCI_BAR 0 static const u32 vsc9959_ana_regmap[] = { REG(ANA_ADVLEARN, 0x0089a0), @@ -292,7 +298,7 @@ static const u32 vsc9959_sys_regmap[] = { REG_RESERVED(SYS_MMGT_FAST), REG_RESERVED(SYS_EVENTS_DIF), REG_RESERVED(SYS_EVENTS_CORE), - REG_RESERVED(SYS_CNT), + REG(SYS_CNT, 0x000000), REG(SYS_PTP_STATUS, 0x000f14), REG(SYS_PTP_TXSTAMP, 0x000f18), REG(SYS_PTP_NXT, 0x000f1c), @@ -1020,15 +1026,6 @@ static void vsc9959_wm_stat(u32 val, u32 *inuse, u32 *maxuse) *maxuse = val & GENMASK(11, 0); } -static const struct ocelot_ops vsc9959_ops = { - .reset = vsc9959_reset, - .wm_enc = vsc9959_wm_enc, - .wm_dec = vsc9959_wm_dec, - .wm_stat = vsc9959_wm_stat, - .port_to_netdev = felix_port_to_netdev, - .netdev_to_port = felix_netdev_to_port, -}; - static int vsc9959_mdio_bus_alloc(struct ocelot *ocelot) { struct felix *felix = ocelot_to_felix(ocelot); @@ -1344,6 +1341,877 @@ static int vsc9959_port_setup_tc(struct dsa_switch *ds, int port, } } +#define VSC9959_PSFP_SFID_MAX 175 +#define VSC9959_PSFP_GATE_ID_MAX 183 +#define VSC9959_PSFP_POLICER_BASE 63 +#define VSC9959_PSFP_POLICER_MAX 383 +#define VSC9959_PSFP_GATE_LIST_NUM 4 +#define VSC9959_PSFP_GATE_CYCLETIME_MIN 5000 + +struct felix_stream { + struct list_head list; + unsigned long id; + bool dummy; + int ports; + int port; + u8 dmac[ETH_ALEN]; + u16 vid; + s8 prio; + u8 sfid_valid; + u8 ssid_valid; + u32 sfid; + u32 ssid; +}; + +struct felix_stream_filter { + struct list_head list; + refcount_t refcount; + u32 index; + u8 enable; + int portmask; + u8 sg_valid; + u32 sgid; + u8 fm_valid; + u32 fmid; + u8 prio_valid; + u8 prio; + u32 maxsdu; +}; + +struct felix_stream_filter_counters { + u32 match; + u32 not_pass_gate; + u32 not_pass_sdu; + u32 red; +}; + +struct felix_stream_gate { + u32 index; + u8 enable; + u8 ipv_valid; + u8 init_ipv; + u64 basetime; + u64 cycletime; + u64 cycletime_ext; + u32 num_entries; + struct action_gate_entry entries[]; +}; + +struct felix_stream_gate_entry { + struct list_head list; + refcount_t refcount; + u32 index; +}; + +static int vsc9959_stream_identify(struct flow_cls_offload *f, + struct felix_stream *stream) +{ + struct flow_rule *rule = flow_cls_offload_flow_rule(f); + struct flow_dissector *dissector = rule->match.dissector; + + if (dissector->used_keys & + ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | + BIT(FLOW_DISSECTOR_KEY_BASIC) | + BIT(FLOW_DISSECTOR_KEY_VLAN) | + BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS))) + return -EOPNOTSUPP; + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { + struct flow_match_eth_addrs match; + + flow_rule_match_eth_addrs(rule, &match); + ether_addr_copy(stream->dmac, match.key->dst); + if (!is_zero_ether_addr(match.mask->src)) + return -EOPNOTSUPP; + } else { + return -EOPNOTSUPP; + } + + if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { + struct flow_match_vlan match; + + flow_rule_match_vlan(rule, &match); + if (match.mask->vlan_priority) + stream->prio = match.key->vlan_priority; + else + stream->prio = -1; + + if (!match.mask->vlan_id) + return -EOPNOTSUPP; + stream->vid = match.key->vlan_id; + } else { + return -EOPNOTSUPP; + } + + stream->id = f->cookie; + + return 0; +} + +static int vsc9959_mact_stream_set(struct ocelot *ocelot, + struct felix_stream *stream, + struct netlink_ext_ack *extack) +{ + enum macaccess_entry_type type; + int ret, sfid, ssid; + u32 vid, dst_idx; + u8 mac[ETH_ALEN]; + + ether_addr_copy(mac, stream->dmac); + vid = stream->vid; + + /* Stream identification desn't support to add a stream with non + * existent MAC (The MAC entry has not been learned in MAC table). + */ + ret = ocelot_mact_lookup(ocelot, &dst_idx, mac, vid, &type); + if (ret) { + if (extack) + NL_SET_ERR_MSG_MOD(extack, "Stream is not learned in MAC table"); + return -EOPNOTSUPP; + } + + if ((stream->sfid_valid || stream->ssid_valid) && + type == ENTRYTYPE_NORMAL) + type = ENTRYTYPE_LOCKED; + + sfid = stream->sfid_valid ? stream->sfid : -1; + ssid = stream->ssid_valid ? stream->ssid : -1; + + ret = ocelot_mact_learn_streamdata(ocelot, dst_idx, mac, vid, type, + sfid, ssid); + + return ret; +} + +static struct felix_stream * +vsc9959_stream_table_lookup(struct list_head *stream_list, + struct felix_stream *stream) +{ + struct felix_stream *tmp; + + list_for_each_entry(tmp, stream_list, list) + if (ether_addr_equal(tmp->dmac, stream->dmac) && + tmp->vid == stream->vid) + return tmp; + + return NULL; +} + +static int vsc9959_stream_table_add(struct ocelot *ocelot, + struct list_head *stream_list, + struct felix_stream *stream, + struct netlink_ext_ack *extack) +{ + struct felix_stream *stream_entry; + int ret; + + stream_entry = kmemdup(stream, sizeof(*stream_entry), GFP_KERNEL); + if (!stream_entry) + return -ENOMEM; + + if (!stream->dummy) { + ret = vsc9959_mact_stream_set(ocelot, stream_entry, extack); + if (ret) { + kfree(stream_entry); + return ret; + } + } + + list_add_tail(&stream_entry->list, stream_list); + + return 0; +} + +static struct felix_stream * +vsc9959_stream_table_get(struct list_head *stream_list, unsigned long id) +{ + struct felix_stream *tmp; + + list_for_each_entry(tmp, stream_list, list) + if (tmp->id == id) + return tmp; + + return NULL; +} + +static void vsc9959_stream_table_del(struct ocelot *ocelot, + struct felix_stream *stream) +{ + if (!stream->dummy) + vsc9959_mact_stream_set(ocelot, stream, NULL); + + list_del(&stream->list); + kfree(stream); +} + +static u32 vsc9959_sfi_access_status(struct ocelot *ocelot) +{ + return ocelot_read(ocelot, ANA_TABLES_SFIDACCESS); +} + +static int vsc9959_psfp_sfi_set(struct ocelot *ocelot, + struct felix_stream_filter *sfi) +{ + u32 val; + + if (sfi->index > VSC9959_PSFP_SFID_MAX) + return -EINVAL; + + if (!sfi->enable) { + ocelot_write(ocelot, ANA_TABLES_SFIDTIDX_SFID_INDEX(sfi->index), + ANA_TABLES_SFIDTIDX); + + val = ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(SFIDACCESS_CMD_WRITE); + ocelot_write(ocelot, val, ANA_TABLES_SFIDACCESS); + + return readx_poll_timeout(vsc9959_sfi_access_status, ocelot, val, + (!ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(val)), + 10, 100000); + } + + if (sfi->sgid > VSC9959_PSFP_GATE_ID_MAX || + sfi->fmid > VSC9959_PSFP_POLICER_MAX) + return -EINVAL; + + ocelot_write(ocelot, + (sfi->sg_valid ? ANA_TABLES_SFIDTIDX_SGID_VALID : 0) | + ANA_TABLES_SFIDTIDX_SGID(sfi->sgid) | + (sfi->fm_valid ? ANA_TABLES_SFIDTIDX_POL_ENA : 0) | + ANA_TABLES_SFIDTIDX_POL_IDX(sfi->fmid) | + ANA_TABLES_SFIDTIDX_SFID_INDEX(sfi->index), + ANA_TABLES_SFIDTIDX); + + ocelot_write(ocelot, + (sfi->prio_valid ? ANA_TABLES_SFIDACCESS_IGR_PRIO_MATCH_ENA : 0) | + ANA_TABLES_SFIDACCESS_IGR_PRIO(sfi->prio) | + ANA_TABLES_SFIDACCESS_MAX_SDU_LEN(sfi->maxsdu) | + ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(SFIDACCESS_CMD_WRITE), + ANA_TABLES_SFIDACCESS); + + return readx_poll_timeout(vsc9959_sfi_access_status, ocelot, val, + (!ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(val)), + 10, 100000); +} + +static int vsc9959_psfp_sfidmask_set(struct ocelot *ocelot, u32 sfid, int ports) +{ + u32 val; + + ocelot_rmw(ocelot, + ANA_TABLES_SFIDTIDX_SFID_INDEX(sfid), + ANA_TABLES_SFIDTIDX_SFID_INDEX_M, + ANA_TABLES_SFIDTIDX); + + ocelot_write(ocelot, + ANA_TABLES_SFID_MASK_IGR_PORT_MASK(ports) | + ANA_TABLES_SFID_MASK_IGR_SRCPORT_MATCH_ENA, + ANA_TABLES_SFID_MASK); + + ocelot_rmw(ocelot, + ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(SFIDACCESS_CMD_WRITE), + ANA_TABLES_SFIDACCESS_SFID_TBL_CMD_M, + ANA_TABLES_SFIDACCESS); + + return readx_poll_timeout(vsc9959_sfi_access_status, ocelot, val, + (!ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(val)), + 10, 100000); +} + +static int vsc9959_psfp_sfi_list_add(struct ocelot *ocelot, + struct felix_stream_filter *sfi, + struct list_head *pos) +{ + struct felix_stream_filter *sfi_entry; + int ret; + + sfi_entry = kmemdup(sfi, sizeof(*sfi_entry), GFP_KERNEL); + if (!sfi_entry) + return -ENOMEM; + + refcount_set(&sfi_entry->refcount, 1); + + ret = vsc9959_psfp_sfi_set(ocelot, sfi_entry); + if (ret) { + kfree(sfi_entry); + return ret; + } + + vsc9959_psfp_sfidmask_set(ocelot, sfi->index, sfi->portmask); + + list_add(&sfi_entry->list, pos); + + return 0; +} + +static int vsc9959_psfp_sfi_table_add(struct ocelot *ocelot, + struct felix_stream_filter *sfi) +{ + struct list_head *pos, *q, *last; + struct felix_stream_filter *tmp; + struct ocelot_psfp_list *psfp; + u32 insert = 0; + + psfp = &ocelot->psfp; + last = &psfp->sfi_list; + + list_for_each_safe(pos, q, &psfp->sfi_list) { + tmp = list_entry(pos, struct felix_stream_filter, list); + if (sfi->sg_valid == tmp->sg_valid && + sfi->fm_valid == tmp->fm_valid && + sfi->portmask == tmp->portmask && + tmp->sgid == sfi->sgid && + tmp->fmid == sfi->fmid) { + sfi->index = tmp->index; + refcount_inc(&tmp->refcount); + return 0; + } + /* Make sure that the index is increasing in order. */ + if (tmp->index == insert) { + last = pos; + insert++; + } + } + sfi->index = insert; + + return vsc9959_psfp_sfi_list_add(ocelot, sfi, last); +} + +static int vsc9959_psfp_sfi_table_add2(struct ocelot *ocelot, + struct felix_stream_filter *sfi, + struct felix_stream_filter *sfi2) +{ + struct felix_stream_filter *tmp; + struct list_head *pos, *q, *last; + struct ocelot_psfp_list *psfp; + u32 insert = 0; + int ret; + + psfp = &ocelot->psfp; + last = &psfp->sfi_list; + + list_for_each_safe(pos, q, &psfp->sfi_list) { + tmp = list_entry(pos, struct felix_stream_filter, list); + /* Make sure that the index is increasing in order. */ + if (tmp->index >= insert + 2) + break; + + insert = tmp->index + 1; + last = pos; + } + sfi->index = insert; + + ret = vsc9959_psfp_sfi_list_add(ocelot, sfi, last); + if (ret) + return ret; + + sfi2->index = insert + 1; + + return vsc9959_psfp_sfi_list_add(ocelot, sfi2, last->next); +} + +static struct felix_stream_filter * +vsc9959_psfp_sfi_table_get(struct list_head *sfi_list, u32 index) +{ + struct felix_stream_filter *tmp; + + list_for_each_entry(tmp, sfi_list, list) + if (tmp->index == index) + return tmp; + + return NULL; +} + +static void vsc9959_psfp_sfi_table_del(struct ocelot *ocelot, u32 index) +{ + struct felix_stream_filter *tmp, *n; + struct ocelot_psfp_list *psfp; + u8 z; + + psfp = &ocelot->psfp; + + list_for_each_entry_safe(tmp, n, &psfp->sfi_list, list) + if (tmp->index == index) { + z = refcount_dec_and_test(&tmp->refcount); + if (z) { + tmp->enable = 0; + vsc9959_psfp_sfi_set(ocelot, tmp); + list_del(&tmp->list); + kfree(tmp); + } + break; + } +} + +static void vsc9959_psfp_parse_gate(const struct flow_action_entry *entry, + struct felix_stream_gate *sgi) +{ + sgi->index = entry->gate.index; + sgi->ipv_valid = (entry->gate.prio < 0) ? 0 : 1; + sgi->init_ipv = (sgi->ipv_valid) ? entry->gate.prio : 0; + sgi->basetime = entry->gate.basetime; + sgi->cycletime = entry->gate.cycletime; + sgi->num_entries = entry->gate.num_entries; + sgi->enable = 1; + + memcpy(sgi->entries, entry->gate.entries, + entry->gate.num_entries * sizeof(struct action_gate_entry)); +} + +static u32 vsc9959_sgi_cfg_status(struct ocelot *ocelot) +{ + return ocelot_read(ocelot, ANA_SG_ACCESS_CTRL); +} + +static int vsc9959_psfp_sgi_set(struct ocelot *ocelot, + struct felix_stream_gate *sgi) +{ + struct action_gate_entry *e; + struct timespec64 base_ts; + u32 interval_sum = 0; + u32 val; + int i; + + if (sgi->index > VSC9959_PSFP_GATE_ID_MAX) + return -EINVAL; + + ocelot_write(ocelot, ANA_SG_ACCESS_CTRL_SGID(sgi->index), + ANA_SG_ACCESS_CTRL); + + if (!sgi->enable) { + ocelot_rmw(ocelot, ANA_SG_CONFIG_REG_3_INIT_GATE_STATE, + ANA_SG_CONFIG_REG_3_INIT_GATE_STATE | + ANA_SG_CONFIG_REG_3_GATE_ENABLE, + ANA_SG_CONFIG_REG_3); + + return 0; + } + + if (sgi->cycletime < VSC9959_PSFP_GATE_CYCLETIME_MIN || + sgi->cycletime > NSEC_PER_SEC) + return -EINVAL; + + if (sgi->num_entries > VSC9959_PSFP_GATE_LIST_NUM) + return -EINVAL; + + vsc9959_new_base_time(ocelot, sgi->basetime, sgi->cycletime, &base_ts); + ocelot_write(ocelot, base_ts.tv_nsec, ANA_SG_CONFIG_REG_1); + val = lower_32_bits(base_ts.tv_sec); + ocelot_write(ocelot, val, ANA_SG_CONFIG_REG_2); + + val = upper_32_bits(base_ts.tv_sec); + ocelot_write(ocelot, + (sgi->ipv_valid ? ANA_SG_CONFIG_REG_3_IPV_VALID : 0) | + ANA_SG_CONFIG_REG_3_INIT_IPV(sgi->init_ipv) | + ANA_SG_CONFIG_REG_3_GATE_ENABLE | + ANA_SG_CONFIG_REG_3_LIST_LENGTH(sgi->num_entries) | + ANA_SG_CONFIG_REG_3_INIT_GATE_STATE | + ANA_SG_CONFIG_REG_3_BASE_TIME_SEC_MSB(val), + ANA_SG_CONFIG_REG_3); + + ocelot_write(ocelot, sgi->cycletime, ANA_SG_CONFIG_REG_4); + + e = sgi->entries; + for (i = 0; i < sgi->num_entries; i++) { + u32 ips = (e[i].ipv < 0) ? 0 : (e[i].ipv + 8); + + ocelot_write_rix(ocelot, ANA_SG_GCL_GS_CONFIG_IPS(ips) | + (e[i].gate_state ? + ANA_SG_GCL_GS_CONFIG_GATE_STATE : 0), + ANA_SG_GCL_GS_CONFIG, i); + + interval_sum += e[i].interval; + ocelot_write_rix(ocelot, interval_sum, ANA_SG_GCL_TI_CONFIG, i); + } + + ocelot_rmw(ocelot, ANA_SG_ACCESS_CTRL_CONFIG_CHANGE, + ANA_SG_ACCESS_CTRL_CONFIG_CHANGE, + ANA_SG_ACCESS_CTRL); + + return readx_poll_timeout(vsc9959_sgi_cfg_status, ocelot, val, + (!(ANA_SG_ACCESS_CTRL_CONFIG_CHANGE & val)), + 10, 100000); +} + +static int vsc9959_psfp_sgi_table_add(struct ocelot *ocelot, + struct felix_stream_gate *sgi) +{ + struct felix_stream_gate_entry *tmp; + struct ocelot_psfp_list *psfp; + int ret; + + psfp = &ocelot->psfp; + + list_for_each_entry(tmp, &psfp->sgi_list, list) + if (tmp->index == sgi->index) { + refcount_inc(&tmp->refcount); + return 0; + } + + tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + ret = vsc9959_psfp_sgi_set(ocelot, sgi); + if (ret) { + kfree(tmp); + return ret; + } + + tmp->index = sgi->index; + refcount_set(&tmp->refcount, 1); + list_add_tail(&tmp->list, &psfp->sgi_list); + + return 0; +} + +static void vsc9959_psfp_sgi_table_del(struct ocelot *ocelot, + u32 index) +{ + struct felix_stream_gate_entry *tmp, *n; + struct felix_stream_gate sgi = {0}; + struct ocelot_psfp_list *psfp; + u8 z; + + psfp = &ocelot->psfp; + + list_for_each_entry_safe(tmp, n, &psfp->sgi_list, list) + if (tmp->index == index) { + z = refcount_dec_and_test(&tmp->refcount); + if (z) { + sgi.index = index; + sgi.enable = 0; + vsc9959_psfp_sgi_set(ocelot, &sgi); + list_del(&tmp->list); + kfree(tmp); + } + break; + } +} + +static void vsc9959_psfp_counters_get(struct ocelot *ocelot, u32 index, + struct felix_stream_filter_counters *counters) +{ + ocelot_rmw(ocelot, SYS_STAT_CFG_STAT_VIEW(index), + SYS_STAT_CFG_STAT_VIEW_M, + SYS_STAT_CFG); + + counters->match = ocelot_read_gix(ocelot, SYS_CNT, 0x200); + counters->not_pass_gate = ocelot_read_gix(ocelot, SYS_CNT, 0x201); + counters->not_pass_sdu = ocelot_read_gix(ocelot, SYS_CNT, 0x202); + counters->red = ocelot_read_gix(ocelot, SYS_CNT, 0x203); + + /* Clear the PSFP counter. */ + ocelot_write(ocelot, + SYS_STAT_CFG_STAT_VIEW(index) | + SYS_STAT_CFG_STAT_CLEAR_SHOT(0x10), + SYS_STAT_CFG); +} + +static int vsc9959_psfp_filter_add(struct ocelot *ocelot, int port, + struct flow_cls_offload *f) +{ + struct netlink_ext_ack *extack = f->common.extack; + struct felix_stream_filter old_sfi, *sfi_entry; + struct felix_stream_filter sfi = {0}; + const struct flow_action_entry *a; + struct felix_stream *stream_entry; + struct felix_stream stream = {0}; + struct felix_stream_gate *sgi; + struct ocelot_psfp_list *psfp; + struct ocelot_policer pol; + int ret, i, size; + u64 rate, burst; + u32 index; + + psfp = &ocelot->psfp; + + ret = vsc9959_stream_identify(f, &stream); + if (ret) { + NL_SET_ERR_MSG_MOD(extack, "Only can match on VID, PCP, and dest MAC"); + return ret; + } + + flow_action_for_each(i, a, &f->rule->action) { + switch (a->id) { + case FLOW_ACTION_GATE: + size = struct_size(sgi, entries, a->gate.num_entries); + sgi = kzalloc(size, GFP_KERNEL); + vsc9959_psfp_parse_gate(a, sgi); + ret = vsc9959_psfp_sgi_table_add(ocelot, sgi); + if (ret) { + kfree(sgi); + goto err; + } + sfi.sg_valid = 1; + sfi.sgid = sgi->index; + kfree(sgi); + break; + case FLOW_ACTION_POLICE: + index = a->police.index + VSC9959_PSFP_POLICER_BASE; + if (index > VSC9959_PSFP_POLICER_MAX) { + ret = -EINVAL; + goto err; + } + + rate = a->police.rate_bytes_ps; + burst = rate * PSCHED_NS2TICKS(a->police.burst); + pol = (struct ocelot_policer) { + .burst = div_u64(burst, PSCHED_TICKS_PER_SEC), + .rate = div_u64(rate, 1000) * 8, + }; + ret = ocelot_vcap_policer_add(ocelot, index, &pol); + if (ret) + goto err; + + sfi.fm_valid = 1; + sfi.fmid = index; + sfi.maxsdu = a->police.mtu; + break; + default: + return -EOPNOTSUPP; + } + } + + stream.ports = BIT(port); + stream.port = port; + + sfi.portmask = stream.ports; + sfi.prio_valid = (stream.prio < 0 ? 0 : 1); + sfi.prio = (sfi.prio_valid ? stream.prio : 0); + sfi.enable = 1; + + /* Check if stream is set. */ + stream_entry = vsc9959_stream_table_lookup(&psfp->stream_list, &stream); + if (stream_entry) { + if (stream_entry->ports & BIT(port)) { + NL_SET_ERR_MSG_MOD(extack, + "The stream is added on this port"); + ret = -EEXIST; + goto err; + } + + if (stream_entry->ports != BIT(stream_entry->port)) { + NL_SET_ERR_MSG_MOD(extack, + "The stream is added on two ports"); + ret = -EEXIST; + goto err; + } + + stream_entry->ports |= BIT(port); + stream.ports = stream_entry->ports; + + sfi_entry = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, + stream_entry->sfid); + memcpy(&old_sfi, sfi_entry, sizeof(old_sfi)); + + vsc9959_psfp_sfi_table_del(ocelot, stream_entry->sfid); + + old_sfi.portmask = stream_entry->ports; + sfi.portmask = stream.ports; + + if (stream_entry->port > port) { + ret = vsc9959_psfp_sfi_table_add2(ocelot, &sfi, + &old_sfi); + stream_entry->dummy = true; + } else { + ret = vsc9959_psfp_sfi_table_add2(ocelot, &old_sfi, + &sfi); + stream.dummy = true; + } + if (ret) + goto err; + + stream_entry->sfid = old_sfi.index; + } else { + ret = vsc9959_psfp_sfi_table_add(ocelot, &sfi); + if (ret) + goto err; + } + + stream.sfid = sfi.index; + stream.sfid_valid = 1; + ret = vsc9959_stream_table_add(ocelot, &psfp->stream_list, + &stream, extack); + if (ret) { + vsc9959_psfp_sfi_table_del(ocelot, stream.sfid); + goto err; + } + + return 0; + +err: + if (sfi.sg_valid) + vsc9959_psfp_sgi_table_del(ocelot, sfi.sgid); + + if (sfi.fm_valid) + ocelot_vcap_policer_del(ocelot, sfi.fmid); + + return ret; +} + +static int vsc9959_psfp_filter_del(struct ocelot *ocelot, + struct flow_cls_offload *f) +{ + struct felix_stream *stream, tmp, *stream_entry; + static struct felix_stream_filter *sfi; + struct ocelot_psfp_list *psfp; + + psfp = &ocelot->psfp; + + stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie); + if (!stream) + return -ENOMEM; + + sfi = vsc9959_psfp_sfi_table_get(&psfp->sfi_list, stream->sfid); + if (!sfi) + return -ENOMEM; + + if (sfi->sg_valid) + vsc9959_psfp_sgi_table_del(ocelot, sfi->sgid); + + if (sfi->fm_valid) + ocelot_vcap_policer_del(ocelot, sfi->fmid); + + vsc9959_psfp_sfi_table_del(ocelot, stream->sfid); + + memcpy(&tmp, stream, sizeof(tmp)); + + stream->sfid_valid = 0; + vsc9959_stream_table_del(ocelot, stream); + + stream_entry = vsc9959_stream_table_lookup(&psfp->stream_list, &tmp); + if (stream_entry) { + stream_entry->ports = BIT(stream_entry->port); + if (stream_entry->dummy) { + stream_entry->dummy = false; + vsc9959_mact_stream_set(ocelot, stream_entry, NULL); + } + vsc9959_psfp_sfidmask_set(ocelot, stream_entry->sfid, + stream_entry->ports); + } + + return 0; +} + +static int vsc9959_psfp_stats_get(struct ocelot *ocelot, + struct flow_cls_offload *f, + struct flow_stats *stats) +{ + struct felix_stream_filter_counters counters; + struct ocelot_psfp_list *psfp; + struct felix_stream *stream; + + psfp = &ocelot->psfp; + stream = vsc9959_stream_table_get(&psfp->stream_list, f->cookie); + if (!stream) + return -ENOMEM; + + vsc9959_psfp_counters_get(ocelot, stream->sfid, &counters); + + stats->pkts = counters.match; + stats->drops = counters.not_pass_gate + counters.not_pass_sdu + + counters.red; + + return 0; +} + +static void vsc9959_psfp_init(struct ocelot *ocelot) +{ + struct ocelot_psfp_list *psfp = &ocelot->psfp; + + INIT_LIST_HEAD(&psfp->stream_list); + INIT_LIST_HEAD(&psfp->sfi_list); + INIT_LIST_HEAD(&psfp->sgi_list); +} + +/* When using cut-through forwarding and the egress port runs at a higher data + * rate than the ingress port, the packet currently under transmission would + * suffer an underrun since it would be transmitted faster than it is received. + * The Felix switch implementation of cut-through forwarding does not check in + * hardware whether this condition is satisfied or not, so we must restrict the + * list of ports that have cut-through forwarding enabled on egress to only be + * the ports operating at the lowest link speed within their respective + * forwarding domain. + */ +static void vsc9959_cut_through_fwd(struct ocelot *ocelot) +{ + struct felix *felix = ocelot_to_felix(ocelot); + struct dsa_switch *ds = felix->ds; + int port, other_port; + + lockdep_assert_held(&ocelot->fwd_domain_lock); + + for (port = 0; port < ocelot->num_phys_ports; port++) { + struct ocelot_port *ocelot_port = ocelot->ports[port]; + int min_speed = ocelot_port->speed; + unsigned long mask = 0; + u32 tmp, val = 0; + + /* Disable cut-through on ports that are down */ + if (ocelot_port->speed <= 0) + goto set; + + if (dsa_is_cpu_port(ds, port)) { + /* Ocelot switches forward from the NPI port towards + * any port, regardless of it being in the NPI port's + * forwarding domain or not. + */ + mask = dsa_user_ports(ds); + } else { + mask = ocelot_get_bridge_fwd_mask(ocelot, port); + mask &= ~BIT(port); + if (ocelot->npi >= 0) + mask |= BIT(ocelot->npi); + else + mask |= ocelot_get_dsa_8021q_cpu_mask(ocelot); + } + + /* Calculate the minimum link speed, among the ports that are + * up, of this source port's forwarding domain. + */ + for_each_set_bit(other_port, &mask, ocelot->num_phys_ports) { + struct ocelot_port *other_ocelot_port; + + other_ocelot_port = ocelot->ports[other_port]; + if (other_ocelot_port->speed <= 0) + continue; + + if (min_speed > other_ocelot_port->speed) + min_speed = other_ocelot_port->speed; + } + + /* Enable cut-through forwarding for all traffic classes. */ + if (ocelot_port->speed == min_speed) + val = GENMASK(7, 0); + +set: + tmp = ocelot_read_rix(ocelot, ANA_CUT_THRU_CFG, port); + if (tmp == val) + continue; + + dev_dbg(ocelot->dev, + "port %d fwd mask 0x%lx speed %d min_speed %d, %s cut-through forwarding\n", + port, mask, ocelot_port->speed, min_speed, + val ? "enabling" : "disabling"); + + ocelot_write_rix(ocelot, val, ANA_CUT_THRU_CFG, port); + } +} + +static const struct ocelot_ops vsc9959_ops = { + .reset = vsc9959_reset, + .wm_enc = vsc9959_wm_enc, + .wm_dec = vsc9959_wm_dec, + .wm_stat = vsc9959_wm_stat, + .port_to_netdev = felix_port_to_netdev, + .netdev_to_port = felix_netdev_to_port, + .psfp_init = vsc9959_psfp_init, + .psfp_filter_add = vsc9959_psfp_filter_add, + .psfp_filter_del = vsc9959_psfp_filter_del, + .psfp_stats_get = vsc9959_psfp_stats_get, + .cut_through_fwd = vsc9959_cut_through_fwd, +}; + static const struct felix_info felix_info_vsc9959 = { .target_io_res = vsc9959_target_io_res, .port_io_res = vsc9959_port_io_res, @@ -1354,11 +2222,13 @@ static const struct felix_info felix_info_vsc9959 = { .stats_layout = vsc9959_stats_layout, .num_stats = ARRAY_SIZE(vsc9959_stats_layout), .vcap = vsc9959_vcap_props, + .vcap_pol_base = VSC9959_VCAP_POLICER_BASE, + .vcap_pol_max = VSC9959_VCAP_POLICER_MAX, + .vcap_pol_base2 = 0, + .vcap_pol_max2 = 0, .num_mact_rows = 2048, .num_ports = 6, .num_tx_queues = OCELOT_NUM_TC, - .switch_pci_bar = 4, - .imdio_pci_bar = 0, .quirk_no_xtr_irq = true, .ptp_caps = &vsc9959_ptp_caps, .mdio_bus_alloc = vsc9959_mdio_bus_alloc, @@ -1367,6 +2237,7 @@ static const struct felix_info felix_info_vsc9959 = { .prevalidate_phy_mode = vsc9959_prevalidate_phy_mode, .port_setup_tc = vsc9959_port_setup_tc, .port_sched_speed_set = vsc9959_sched_speed_set, + .init_regmap = ocelot_regmap_init, }; static irqreturn_t felix_irq_handler(int irq, void *data) @@ -1417,10 +2288,8 @@ static int felix_pci_probe(struct pci_dev *pdev, ocelot->dev = &pdev->dev; ocelot->num_flooding_pgids = OCELOT_NUM_TC; felix->info = &felix_info_vsc9959; - felix->switch_base = pci_resource_start(pdev, - felix->info->switch_pci_bar); - felix->imdio_base = pci_resource_start(pdev, - felix->info->imdio_pci_bar); + felix->switch_base = pci_resource_start(pdev, VSC9959_SWITCH_PCI_BAR); + felix->imdio_base = pci_resource_start(pdev, VSC9959_IMDIO_PCI_BAR); pci_set_master(pdev); diff --git a/drivers/net/dsa/ocelot/seville_vsc9953.c b/drivers/net/dsa/ocelot/seville_vsc9953.c index 92eae63150ea..e110550e3507 100644 --- a/drivers/net/dsa/ocelot/seville_vsc9953.c +++ b/drivers/net/dsa/ocelot/seville_vsc9953.c @@ -6,18 +6,18 @@ #include <soc/mscc/ocelot_vcap.h> #include <soc/mscc/ocelot_sys.h> #include <soc/mscc/ocelot.h> +#include <linux/mdio/mdio-mscc-miim.h> +#include <linux/of_mdio.h> #include <linux/of_platform.h> #include <linux/pcs-lynx.h> #include <linux/dsa/ocelot.h> #include <linux/iopoll.h> #include "felix.h" -#define MSCC_MIIM_CMD_OPR_WRITE BIT(1) -#define MSCC_MIIM_CMD_OPR_READ BIT(2) -#define MSCC_MIIM_CMD_WRDATA_SHIFT 4 -#define MSCC_MIIM_CMD_REGAD_SHIFT 20 -#define MSCC_MIIM_CMD_PHYAD_SHIFT 25 -#define MSCC_MIIM_CMD_VLD BIT(31) +#define VSC9953_VCAP_POLICER_BASE 11 +#define VSC9953_VCAP_POLICER_MAX 31 +#define VSC9953_VCAP_POLICER_BASE2 120 +#define VSC9953_VCAP_POLICER_MAX2 161 static const u32 vsc9953_ana_regmap[] = { REG(ANA_ADVLEARN, 0x00b500), @@ -857,7 +857,6 @@ static struct vcap_props vsc9953_vcap_props[] = { #define VSC9953_INIT_TIMEOUT 50000 #define VSC9953_GCB_RST_SLEEP 100 #define VSC9953_SYS_RAMINIT_SLEEP 80 -#define VCS9953_MII_TIMEOUT 10000 static int vsc9953_gcb_soft_rst_status(struct ocelot *ocelot) { @@ -877,82 +876,6 @@ static int vsc9953_sys_ram_init_status(struct ocelot *ocelot) return val; } -static int vsc9953_gcb_miim_pending_status(struct ocelot *ocelot) -{ - int val; - - ocelot_field_read(ocelot, GCB_MIIM_MII_STATUS_PENDING, &val); - - return val; -} - -static int vsc9953_gcb_miim_busy_status(struct ocelot *ocelot) -{ - int val; - - ocelot_field_read(ocelot, GCB_MIIM_MII_STATUS_BUSY, &val); - - return val; -} - -static int vsc9953_mdio_write(struct mii_bus *bus, int phy_id, int regnum, - u16 value) -{ - struct ocelot *ocelot = bus->priv; - int err, cmd, val; - - /* Wait while MIIM controller becomes idle */ - err = readx_poll_timeout(vsc9953_gcb_miim_pending_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO write: pending timeout\n"); - goto out; - } - - cmd = MSCC_MIIM_CMD_VLD | (phy_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | - (value << MSCC_MIIM_CMD_WRDATA_SHIFT) | - MSCC_MIIM_CMD_OPR_WRITE; - - ocelot_write(ocelot, cmd, GCB_MIIM_MII_CMD); - -out: - return err; -} - -static int vsc9953_mdio_read(struct mii_bus *bus, int phy_id, int regnum) -{ - struct ocelot *ocelot = bus->priv; - int err, cmd, val; - - /* Wait until MIIM controller becomes idle */ - err = readx_poll_timeout(vsc9953_gcb_miim_pending_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO read: pending timeout\n"); - goto out; - } - - /* Write the MIIM COMMAND register */ - cmd = MSCC_MIIM_CMD_VLD | (phy_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | MSCC_MIIM_CMD_OPR_READ; - - ocelot_write(ocelot, cmd, GCB_MIIM_MII_CMD); - - /* Wait while read operation via the MIIM controller is in progress */ - err = readx_poll_timeout(vsc9953_gcb_miim_busy_status, ocelot, - val, !val, 10, VCS9953_MII_TIMEOUT); - if (err) { - dev_err(ocelot->dev, "MDIO read: busy timeout\n"); - goto out; - } - - val = ocelot_read(ocelot, GCB_MIIM_MII_DATA); - - err = val & 0xFFFF; -out: - return err; -} /* CORE_ENA is in SYS:SYSTEM:RESET_CFG * MEM_INIT is in SYS:SYSTEM:RESET_CFG @@ -1096,19 +1019,17 @@ static int vsc9953_mdio_bus_alloc(struct ocelot *ocelot) return -ENOMEM; } - bus = devm_mdiobus_alloc(dev); - if (!bus) - return -ENOMEM; + rc = mscc_miim_setup(dev, &bus, "VSC9953 internal MDIO bus", + ocelot->targets[GCB], + ocelot->map[GCB][GCB_MIIM_MII_STATUS & REG_MASK]); - bus->name = "VSC9953 internal MDIO bus"; - bus->read = vsc9953_mdio_read; - bus->write = vsc9953_mdio_write; - bus->parent = dev; - bus->priv = ocelot; - snprintf(bus->id, MII_BUS_ID_SIZE, "%s-imdio", dev_name(dev)); + if (rc) { + dev_err(dev, "failed to setup MDIO bus\n"); + return rc; + } /* Needed in order to initialize the bus mutex lock */ - rc = mdiobus_register(bus); + rc = of_mdiobus_register(bus, NULL); if (rc < 0) { dev_err(dev, "failed to register MDIO bus\n"); return rc; @@ -1172,6 +1093,10 @@ static const struct felix_info seville_info_vsc9953 = { .stats_layout = vsc9953_stats_layout, .num_stats = ARRAY_SIZE(vsc9953_stats_layout), .vcap = vsc9953_vcap_props, + .vcap_pol_base = VSC9953_VCAP_POLICER_BASE, + .vcap_pol_max = VSC9953_VCAP_POLICER_MAX, + .vcap_pol_base2 = VSC9953_VCAP_POLICER_BASE2, + .vcap_pol_max2 = VSC9953_VCAP_POLICER_MAX2, .num_mact_rows = 2048, .num_ports = 10, .num_tx_queues = OCELOT_NUM_TC, @@ -1179,6 +1104,7 @@ static const struct felix_info seville_info_vsc9953 = { .mdio_bus_free = vsc9953_mdio_bus_free, .phylink_validate = vsc9953_phylink_validate, .prevalidate_phy_mode = vsc9953_prevalidate_phy_mode, + .init_regmap = ocelot_regmap_init, }; static int seville_probe(struct platform_device *pdev) diff --git a/drivers/net/dsa/qca8k.c b/drivers/net/dsa/qca8k.c index 147ca39531a3..039694518788 100644 --- a/drivers/net/dsa/qca8k.c +++ b/drivers/net/dsa/qca8k.c @@ -9,6 +9,8 @@ #include <linux/module.h> #include <linux/phy.h> #include <linux/netdevice.h> +#include <linux/bitfield.h> +#include <linux/regmap.h> #include <net/dsa.h> #include <linux/of_net.h> #include <linux/of_mdio.h> @@ -68,6 +70,8 @@ static const struct qca8k_mib_desc ar8327_mib[] = { MIB_DESC(1, 0x9c, "TxExcDefer"), MIB_DESC(1, 0xa0, "TxDefer"), MIB_DESC(1, 0xa4, "TxLateCol"), + MIB_DESC(1, 0xa8, "RXUnicast"), + MIB_DESC(1, 0xac, "TXUnicast"), }; /* The 32bit switch registers are accessed indirectly. To achieve this we need @@ -151,6 +155,25 @@ qca8k_set_page(struct mii_bus *bus, u16 page) static int qca8k_read(struct qca8k_priv *priv, u32 reg, u32 *val) { + return regmap_read(priv->regmap, reg, val); +} + +static int +qca8k_write(struct qca8k_priv *priv, u32 reg, u32 val) +{ + return regmap_write(priv->regmap, reg, val); +} + +static int +qca8k_rmw(struct qca8k_priv *priv, u32 reg, u32 mask, u32 write_val) +{ + return regmap_update_bits(priv->regmap, reg, mask, write_val); +} + +static int +qca8k_regmap_read(void *ctx, uint32_t reg, uint32_t *val) +{ + struct qca8k_priv *priv = (struct qca8k_priv *)ctx; struct mii_bus *bus = priv->bus; u16 r1, r2, page; int ret; @@ -171,8 +194,9 @@ exit: } static int -qca8k_write(struct qca8k_priv *priv, u32 reg, u32 val) +qca8k_regmap_write(void *ctx, uint32_t reg, uint32_t val) { + struct qca8k_priv *priv = (struct qca8k_priv *)ctx; struct mii_bus *bus = priv->bus; u16 r1, r2, page; int ret; @@ -193,8 +217,9 @@ exit: } static int -qca8k_rmw(struct qca8k_priv *priv, u32 reg, u32 mask, u32 write_val) +qca8k_regmap_update_bits(void *ctx, uint32_t reg, uint32_t mask, uint32_t write_val) { + struct qca8k_priv *priv = (struct qca8k_priv *)ctx; struct mii_bus *bus = priv->bus; u16 r1, r2, page; u32 val; @@ -222,34 +247,6 @@ exit: return ret; } -static int -qca8k_reg_set(struct qca8k_priv *priv, u32 reg, u32 val) -{ - return qca8k_rmw(priv, reg, 0, val); -} - -static int -qca8k_reg_clear(struct qca8k_priv *priv, u32 reg, u32 val) -{ - return qca8k_rmw(priv, reg, val, 0); -} - -static int -qca8k_regmap_read(void *ctx, uint32_t reg, uint32_t *val) -{ - struct qca8k_priv *priv = (struct qca8k_priv *)ctx; - - return qca8k_read(priv, reg, val); -} - -static int -qca8k_regmap_write(void *ctx, uint32_t reg, uint32_t val) -{ - struct qca8k_priv *priv = (struct qca8k_priv *)ctx; - - return qca8k_write(priv, reg, val); -} - static const struct regmap_range qca8k_readable_ranges[] = { regmap_reg_range(0x0000, 0x00e4), /* Global control */ regmap_reg_range(0x0100, 0x0168), /* EEE control */ @@ -281,26 +278,19 @@ static struct regmap_config qca8k_regmap_config = { .max_register = 0x16ac, /* end MIB - Port6 range */ .reg_read = qca8k_regmap_read, .reg_write = qca8k_regmap_write, + .reg_update_bits = qca8k_regmap_update_bits, .rd_table = &qca8k_readable_table, + .disable_locking = true, /* Locking is handled by qca8k read/write */ + .cache_type = REGCACHE_NONE, /* Explicitly disable CACHE */ }; static int qca8k_busy_wait(struct qca8k_priv *priv, u32 reg, u32 mask) { - int ret, ret1; u32 val; - ret = read_poll_timeout(qca8k_read, ret1, !(val & mask), - 0, QCA8K_BUSY_WAIT_TIMEOUT * USEC_PER_MSEC, false, - priv, reg, &val); - - /* Check if qca8k_read has failed for a different reason - * before returning -ETIMEDOUT - */ - if (ret < 0 && ret1 < 0) - return ret1; - - return ret; + return regmap_read_poll_timeout(priv->regmap, reg, val, !(val & mask), 0, + QCA8K_BUSY_WAIT_TIMEOUT * USEC_PER_MSEC); } static int @@ -319,18 +309,18 @@ qca8k_fdb_read(struct qca8k_priv *priv, struct qca8k_fdb *fdb) } /* vid - 83:72 */ - fdb->vid = (reg[2] >> QCA8K_ATU_VID_S) & QCA8K_ATU_VID_M; + fdb->vid = FIELD_GET(QCA8K_ATU_VID_MASK, reg[2]); /* aging - 67:64 */ - fdb->aging = reg[2] & QCA8K_ATU_STATUS_M; + fdb->aging = FIELD_GET(QCA8K_ATU_STATUS_MASK, reg[2]); /* portmask - 54:48 */ - fdb->port_mask = (reg[1] >> QCA8K_ATU_PORT_S) & QCA8K_ATU_PORT_M; + fdb->port_mask = FIELD_GET(QCA8K_ATU_PORT_MASK, reg[1]); /* mac - 47:0 */ - fdb->mac[0] = (reg[1] >> QCA8K_ATU_ADDR0_S) & 0xff; - fdb->mac[1] = reg[1] & 0xff; - fdb->mac[2] = (reg[0] >> QCA8K_ATU_ADDR2_S) & 0xff; - fdb->mac[3] = (reg[0] >> QCA8K_ATU_ADDR3_S) & 0xff; - fdb->mac[4] = (reg[0] >> QCA8K_ATU_ADDR4_S) & 0xff; - fdb->mac[5] = reg[0] & 0xff; + fdb->mac[0] = FIELD_GET(QCA8K_ATU_ADDR0_MASK, reg[1]); + fdb->mac[1] = FIELD_GET(QCA8K_ATU_ADDR1_MASK, reg[1]); + fdb->mac[2] = FIELD_GET(QCA8K_ATU_ADDR2_MASK, reg[0]); + fdb->mac[3] = FIELD_GET(QCA8K_ATU_ADDR3_MASK, reg[0]); + fdb->mac[4] = FIELD_GET(QCA8K_ATU_ADDR4_MASK, reg[0]); + fdb->mac[5] = FIELD_GET(QCA8K_ATU_ADDR5_MASK, reg[0]); return 0; } @@ -343,18 +333,18 @@ qca8k_fdb_write(struct qca8k_priv *priv, u16 vid, u8 port_mask, const u8 *mac, int i; /* vid - 83:72 */ - reg[2] = (vid & QCA8K_ATU_VID_M) << QCA8K_ATU_VID_S; + reg[2] = FIELD_PREP(QCA8K_ATU_VID_MASK, vid); /* aging - 67:64 */ - reg[2] |= aging & QCA8K_ATU_STATUS_M; + reg[2] |= FIELD_PREP(QCA8K_ATU_STATUS_MASK, aging); /* portmask - 54:48 */ - reg[1] = (port_mask & QCA8K_ATU_PORT_M) << QCA8K_ATU_PORT_S; + reg[1] = FIELD_PREP(QCA8K_ATU_PORT_MASK, port_mask); /* mac - 47:0 */ - reg[1] |= mac[0] << QCA8K_ATU_ADDR0_S; - reg[1] |= mac[1]; - reg[0] |= mac[2] << QCA8K_ATU_ADDR2_S; - reg[0] |= mac[3] << QCA8K_ATU_ADDR3_S; - reg[0] |= mac[4] << QCA8K_ATU_ADDR4_S; - reg[0] |= mac[5]; + reg[1] |= FIELD_PREP(QCA8K_ATU_ADDR0_MASK, mac[0]); + reg[1] |= FIELD_PREP(QCA8K_ATU_ADDR1_MASK, mac[1]); + reg[0] |= FIELD_PREP(QCA8K_ATU_ADDR2_MASK, mac[2]); + reg[0] |= FIELD_PREP(QCA8K_ATU_ADDR3_MASK, mac[3]); + reg[0] |= FIELD_PREP(QCA8K_ATU_ADDR4_MASK, mac[4]); + reg[0] |= FIELD_PREP(QCA8K_ATU_ADDR5_MASK, mac[5]); /* load the array into the ARL table */ for (i = 0; i < 3; i++) @@ -372,7 +362,7 @@ qca8k_fdb_access(struct qca8k_priv *priv, enum qca8k_fdb_cmd cmd, int port) reg |= cmd; if (port >= 0) { reg |= QCA8K_ATU_FUNC_PORT_EN; - reg |= (port & QCA8K_ATU_FUNC_PORT_M) << QCA8K_ATU_FUNC_PORT_S; + reg |= FIELD_PREP(QCA8K_ATU_FUNC_PORT_MASK, port); } /* Write the function register triggering the table access */ @@ -446,6 +436,81 @@ qca8k_fdb_flush(struct qca8k_priv *priv) } static int +qca8k_fdb_search_and_insert(struct qca8k_priv *priv, u8 port_mask, + const u8 *mac, u16 vid) +{ + struct qca8k_fdb fdb = { 0 }; + int ret; + + mutex_lock(&priv->reg_mutex); + + qca8k_fdb_write(priv, vid, 0, mac, 0); + ret = qca8k_fdb_access(priv, QCA8K_FDB_SEARCH, -1); + if (ret < 0) + goto exit; + + ret = qca8k_fdb_read(priv, &fdb); + if (ret < 0) + goto exit; + + /* Rule exist. Delete first */ + if (!fdb.aging) { + ret = qca8k_fdb_access(priv, QCA8K_FDB_PURGE, -1); + if (ret) + goto exit; + } + + /* Add port to fdb portmask */ + fdb.port_mask |= port_mask; + + qca8k_fdb_write(priv, vid, fdb.port_mask, mac, fdb.aging); + ret = qca8k_fdb_access(priv, QCA8K_FDB_LOAD, -1); + +exit: + mutex_unlock(&priv->reg_mutex); + return ret; +} + +static int +qca8k_fdb_search_and_del(struct qca8k_priv *priv, u8 port_mask, + const u8 *mac, u16 vid) +{ + struct qca8k_fdb fdb = { 0 }; + int ret; + + mutex_lock(&priv->reg_mutex); + + qca8k_fdb_write(priv, vid, 0, mac, 0); + ret = qca8k_fdb_access(priv, QCA8K_FDB_SEARCH, -1); + if (ret < 0) + goto exit; + + /* Rule doesn't exist. Why delete? */ + if (!fdb.aging) { + ret = -EINVAL; + goto exit; + } + + ret = qca8k_fdb_access(priv, QCA8K_FDB_PURGE, -1); + if (ret) + goto exit; + + /* Only port in the rule is this port. Don't re insert */ + if (fdb.port_mask == port_mask) + goto exit; + + /* Remove port from port mask */ + fdb.port_mask &= ~port_mask; + + qca8k_fdb_write(priv, vid, fdb.port_mask, mac, fdb.aging); + ret = qca8k_fdb_access(priv, QCA8K_FDB_LOAD, -1); + +exit: + mutex_unlock(&priv->reg_mutex); + return ret; +} + +static int qca8k_vlan_access(struct qca8k_priv *priv, enum qca8k_vlan_cmd cmd, u16 vid) { u32 reg; @@ -454,7 +519,7 @@ qca8k_vlan_access(struct qca8k_priv *priv, enum qca8k_vlan_cmd cmd, u16 vid) /* Set the command and VLAN index */ reg = QCA8K_VTU_FUNC1_BUSY; reg |= cmd; - reg |= vid << QCA8K_VTU_FUNC1_VID_S; + reg |= FIELD_PREP(QCA8K_VTU_FUNC1_VID_MASK, vid); /* Write the function register triggering the table access */ ret = qca8k_write(priv, QCA8K_REG_VTU_FUNC1, reg); @@ -500,13 +565,11 @@ qca8k_vlan_add(struct qca8k_priv *priv, u8 port, u16 vid, bool untagged) if (ret < 0) goto out; reg |= QCA8K_VTU_FUNC0_VALID | QCA8K_VTU_FUNC0_IVL_EN; - reg &= ~(QCA8K_VTU_FUNC0_EG_MODE_MASK << QCA8K_VTU_FUNC0_EG_MODE_S(port)); + reg &= ~QCA8K_VTU_FUNC0_EG_MODE_PORT_MASK(port); if (untagged) - reg |= QCA8K_VTU_FUNC0_EG_MODE_UNTAG << - QCA8K_VTU_FUNC0_EG_MODE_S(port); + reg |= QCA8K_VTU_FUNC0_EG_MODE_PORT_UNTAG(port); else - reg |= QCA8K_VTU_FUNC0_EG_MODE_TAG << - QCA8K_VTU_FUNC0_EG_MODE_S(port); + reg |= QCA8K_VTU_FUNC0_EG_MODE_PORT_TAG(port); ret = qca8k_write(priv, QCA8K_REG_VTU_FUNC0, reg); if (ret) @@ -534,15 +597,13 @@ qca8k_vlan_del(struct qca8k_priv *priv, u8 port, u16 vid) ret = qca8k_read(priv, QCA8K_REG_VTU_FUNC0, ®); if (ret < 0) goto out; - reg &= ~(3 << QCA8K_VTU_FUNC0_EG_MODE_S(port)); - reg |= QCA8K_VTU_FUNC0_EG_MODE_NOT << - QCA8K_VTU_FUNC0_EG_MODE_S(port); + reg &= ~QCA8K_VTU_FUNC0_EG_MODE_PORT_MASK(port); + reg |= QCA8K_VTU_FUNC0_EG_MODE_PORT_NOT(port); /* Check if we're the last member to be removed */ del = true; for (i = 0; i < QCA8K_NUM_PORTS; i++) { - mask = QCA8K_VTU_FUNC0_EG_MODE_NOT; - mask <<= QCA8K_VTU_FUNC0_EG_MODE_S(i); + mask = QCA8K_VTU_FUNC0_EG_MODE_PORT_NOT(i); if ((reg & mask) != mask) { del = false; @@ -571,7 +632,7 @@ qca8k_mib_init(struct qca8k_priv *priv) int ret; mutex_lock(&priv->reg_mutex); - ret = qca8k_reg_set(priv, QCA8K_REG_MIB, QCA8K_MIB_FLUSH | QCA8K_MIB_BUSY); + ret = regmap_set_bits(priv->regmap, QCA8K_REG_MIB, QCA8K_MIB_FLUSH | QCA8K_MIB_BUSY); if (ret) goto exit; @@ -579,7 +640,7 @@ qca8k_mib_init(struct qca8k_priv *priv) if (ret) goto exit; - ret = qca8k_reg_set(priv, QCA8K_REG_MIB, QCA8K_MIB_CPU_KEEP); + ret = regmap_set_bits(priv->regmap, QCA8K_REG_MIB, QCA8K_MIB_CPU_KEEP); if (ret) goto exit; @@ -600,9 +661,9 @@ qca8k_port_set_status(struct qca8k_priv *priv, int port, int enable) mask |= QCA8K_PORT_STATUS_LINK_AUTO; if (enable) - qca8k_reg_set(priv, QCA8K_REG_PORT_STATUS(port), mask); + regmap_set_bits(priv->regmap, QCA8K_REG_PORT_STATUS(port), mask); else - qca8k_reg_clear(priv, QCA8K_REG_PORT_STATUS(port), mask); + regmap_clear_bits(priv->regmap, QCA8K_REG_PORT_STATUS(port), mask); } static u32 @@ -864,8 +925,8 @@ qca8k_setup_mdio_bus(struct qca8k_priv *priv) * a dt-overlay and driver reload changed the configuration */ - return qca8k_reg_clear(priv, QCA8K_MDIO_MASTER_CTRL, - QCA8K_MDIO_MASTER_EN); + return regmap_clear_bits(priv->regmap, QCA8K_MDIO_MASTER_CTRL, + QCA8K_MDIO_MASTER_EN); } /* Check if the devicetree declare the port:phy mapping */ @@ -983,7 +1044,7 @@ qca8k_parse_port_config(struct qca8k_priv *priv) u32 delay; /* We have 2 CPU port. Check them */ - for (port = 0; port < QCA8K_NUM_PORTS && cpu_port_index < QCA8K_NUM_CPU_PORTS; port++) { + for (port = 0; port < QCA8K_NUM_PORTS; port++) { /* Skip every other port */ if (port != 0 && port != 6) continue; @@ -1014,7 +1075,7 @@ qca8k_parse_port_config(struct qca8k_priv *priv) mode == PHY_INTERFACE_MODE_RGMII_TXID) delay = 1; - if (delay > QCA8K_MAX_DELAY) { + if (!FIELD_FIT(QCA8K_PORT_PAD_RGMII_TX_DELAY_MASK, delay)) { dev_err(priv->dev, "rgmii tx delay is limited to a max value of 3ns, setting to the max value"); delay = 3; } @@ -1030,7 +1091,7 @@ qca8k_parse_port_config(struct qca8k_priv *priv) mode == PHY_INTERFACE_MODE_RGMII_RXID) delay = 2; - if (delay > QCA8K_MAX_DELAY) { + if (!FIELD_FIT(QCA8K_PORT_PAD_RGMII_RX_DELAY_MASK, delay)) { dev_err(priv->dev, "rgmii rx delay is limited to a max value of 3ns, setting to the max value"); delay = 3; } @@ -1089,14 +1150,6 @@ qca8k_setup(struct dsa_switch *ds) if (ret) return ret; - mutex_init(&priv->reg_mutex); - - /* Start by setting up the register mapping */ - priv->regmap = devm_regmap_init(ds->dev, NULL, priv, - &qca8k_regmap_config); - if (IS_ERR(priv->regmap)) - dev_warn(priv->dev, "regmap initialization failed"); - ret = qca8k_setup_mdio_bus(priv); if (ret) return ret; @@ -1110,16 +1163,16 @@ qca8k_setup(struct dsa_switch *ds) return ret; /* Make sure MAC06 is disabled */ - ret = qca8k_reg_clear(priv, QCA8K_REG_PORT0_PAD_CTRL, - QCA8K_PORT0_PAD_MAC06_EXCHANGE_EN); + ret = regmap_clear_bits(priv->regmap, QCA8K_REG_PORT0_PAD_CTRL, + QCA8K_PORT0_PAD_MAC06_EXCHANGE_EN); if (ret) { dev_err(priv->dev, "failed disabling MAC06 exchange"); return ret; } /* Enable CPU Port */ - ret = qca8k_reg_set(priv, QCA8K_REG_GLOBAL_FW_CTRL0, - QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN); + ret = regmap_set_bits(priv->regmap, QCA8K_REG_GLOBAL_FW_CTRL0, + QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN); if (ret) { dev_err(priv->dev, "failed enabling CPU port"); return ret; @@ -1141,8 +1194,8 @@ qca8k_setup(struct dsa_switch *ds) /* Enable QCA header mode on all cpu ports */ if (dsa_is_cpu_port(ds, i)) { ret = qca8k_write(priv, QCA8K_REG_PORT_HDR_CTRL(i), - QCA8K_PORT_HDR_CTRL_ALL << QCA8K_PORT_HDR_CTRL_TX_S | - QCA8K_PORT_HDR_CTRL_ALL << QCA8K_PORT_HDR_CTRL_RX_S); + FIELD_PREP(QCA8K_PORT_HDR_CTRL_TX_MASK, QCA8K_PORT_HDR_CTRL_ALL) | + FIELD_PREP(QCA8K_PORT_HDR_CTRL_RX_MASK, QCA8K_PORT_HDR_CTRL_ALL)); if (ret) { dev_err(priv->dev, "failed enabling QCA header mode"); return ret; @@ -1159,10 +1212,10 @@ qca8k_setup(struct dsa_switch *ds) * for igmp, unknown, multicast and broadcast packet */ ret = qca8k_write(priv, QCA8K_REG_GLOBAL_FW_CTRL1, - BIT(cpu_port) << QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_S | - BIT(cpu_port) << QCA8K_GLOBAL_FW_CTRL1_BC_DP_S | - BIT(cpu_port) << QCA8K_GLOBAL_FW_CTRL1_MC_DP_S | - BIT(cpu_port) << QCA8K_GLOBAL_FW_CTRL1_UC_DP_S); + FIELD_PREP(QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_MASK, BIT(cpu_port)) | + FIELD_PREP(QCA8K_GLOBAL_FW_CTRL1_BC_DP_MASK, BIT(cpu_port)) | + FIELD_PREP(QCA8K_GLOBAL_FW_CTRL1_MC_DP_MASK, BIT(cpu_port)) | + FIELD_PREP(QCA8K_GLOBAL_FW_CTRL1_UC_DP_MASK, BIT(cpu_port))); if (ret) return ret; @@ -1180,8 +1233,6 @@ qca8k_setup(struct dsa_switch *ds) /* Individual user ports get connected to CPU port only */ if (dsa_is_user_port(ds, i)) { - int shift = 16 * (i % 2); - ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(i), QCA8K_PORT_LOOKUP_MEMBER, BIT(cpu_port)); @@ -1189,8 +1240,8 @@ qca8k_setup(struct dsa_switch *ds) return ret; /* Enable ARP Auto-learning by default */ - ret = qca8k_reg_set(priv, QCA8K_PORT_LOOKUP_CTRL(i), - QCA8K_PORT_LOOKUP_LEARN); + ret = regmap_set_bits(priv->regmap, QCA8K_PORT_LOOKUP_CTRL(i), + QCA8K_PORT_LOOKUP_LEARN); if (ret) return ret; @@ -1198,8 +1249,8 @@ qca8k_setup(struct dsa_switch *ds) * default egress vid */ ret = qca8k_rmw(priv, QCA8K_EGRESS_VLAN(i), - 0xfff << shift, - QCA8K_PORT_VID_DEF << shift); + QCA8K_EGREES_VLAN_PORT_MASK(i), + QCA8K_EGREES_VLAN_PORT(i, QCA8K_PORT_VID_DEF)); if (ret) return ret; @@ -1246,7 +1297,7 @@ qca8k_setup(struct dsa_switch *ds) QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | QCA8K_PORT_HOL_CTRL1_WRED_EN; qca8k_rmw(priv, QCA8K_REG_PORT_HOL_CTRL1(i), - QCA8K_PORT_HOL_CTRL1_ING_BUF | + QCA8K_PORT_HOL_CTRL1_ING_BUF_MASK | QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN | QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN | QCA8K_PORT_HOL_CTRL1_WRED_EN, @@ -1269,8 +1320,8 @@ qca8k_setup(struct dsa_switch *ds) mask = QCA8K_GLOBAL_FC_GOL_XON_THRES(288) | QCA8K_GLOBAL_FC_GOL_XOFF_THRES(496); qca8k_rmw(priv, QCA8K_REG_GLOBAL_FC_THRESH, - QCA8K_GLOBAL_FC_GOL_XON_THRES_S | - QCA8K_GLOBAL_FC_GOL_XOFF_THRES_S, + QCA8K_GLOBAL_FC_GOL_XON_THRES_MASK | + QCA8K_GLOBAL_FC_GOL_XOFF_THRES_MASK, mask); } @@ -1285,6 +1336,13 @@ qca8k_setup(struct dsa_switch *ds) /* We don't have interrupts for link changes, so we need to poll */ ds->pcs_poll = true; + /* Set min a max ageing value supported */ + ds->ageing_time_min = 7000; + ds->ageing_time_max = 458745000; + + /* Set max number of LAGs supported */ + ds->num_lag_ids = QCA8K_NUM_LAGS; + return 0; } @@ -1631,12 +1689,16 @@ qca8k_phylink_mac_link_up(struct dsa_switch *ds, int port, unsigned int mode, static void qca8k_get_strings(struct dsa_switch *ds, int port, u32 stringset, uint8_t *data) { + const struct qca8k_match_data *match_data; + struct qca8k_priv *priv = ds->priv; int i; if (stringset != ETH_SS_STATS) return; - for (i = 0; i < ARRAY_SIZE(ar8327_mib); i++) + match_data = of_device_get_match_data(priv->dev); + + for (i = 0; i < match_data->mib_count; i++) strncpy(data + i * ETH_GSTRING_LEN, ar8327_mib[i].name, ETH_GSTRING_LEN); } @@ -1646,12 +1708,15 @@ qca8k_get_ethtool_stats(struct dsa_switch *ds, int port, uint64_t *data) { struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv; + const struct qca8k_match_data *match_data; const struct qca8k_mib_desc *mib; u32 reg, i, val; u32 hi = 0; int ret; - for (i = 0; i < ARRAY_SIZE(ar8327_mib); i++) { + match_data = of_device_get_match_data(priv->dev); + + for (i = 0; i < match_data->mib_count; i++) { mib = &ar8327_mib[i]; reg = QCA8K_PORT_MIB_COUNTER(port) + mib->offset; @@ -1674,10 +1739,15 @@ qca8k_get_ethtool_stats(struct dsa_switch *ds, int port, static int qca8k_get_sset_count(struct dsa_switch *ds, int port, int sset) { + const struct qca8k_match_data *match_data; + struct qca8k_priv *priv = ds->priv; + if (sset != ETH_SS_STATS) return 0; - return ARRAY_SIZE(ar8327_mib); + match_data = of_device_get_match_data(priv->dev); + + return match_data->mib_count; } static int @@ -1740,8 +1810,9 @@ qca8k_port_stp_state_set(struct dsa_switch *ds, int port, u8 state) QCA8K_PORT_LOOKUP_STATE_MASK, stp_state); } -static int -qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) +static int qca8k_port_bridge_join(struct dsa_switch *ds, int port, + struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv; int port_mask, cpu_port; @@ -1753,14 +1824,14 @@ qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) for (i = 0; i < QCA8K_NUM_PORTS; i++) { if (dsa_is_cpu_port(ds, i)) continue; - if (dsa_to_port(ds, i)->bridge_dev != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Add this port to the portvlan mask of the other ports * in the bridge */ - ret = qca8k_reg_set(priv, - QCA8K_PORT_LOOKUP_CTRL(i), - BIT(port)); + ret = regmap_set_bits(priv->regmap, + QCA8K_PORT_LOOKUP_CTRL(i), + BIT(port)); if (ret) return ret; if (i != port) @@ -1774,8 +1845,8 @@ qca8k_port_bridge_join(struct dsa_switch *ds, int port, struct net_device *br) return ret; } -static void -qca8k_port_bridge_leave(struct dsa_switch *ds, int port, struct net_device *br) +static void qca8k_port_bridge_leave(struct dsa_switch *ds, int port, + struct dsa_bridge bridge) { struct qca8k_priv *priv = (struct qca8k_priv *)ds->priv; int cpu_port, i; @@ -1785,14 +1856,14 @@ qca8k_port_bridge_leave(struct dsa_switch *ds, int port, struct net_device *br) for (i = 0; i < QCA8K_NUM_PORTS; i++) { if (dsa_is_cpu_port(ds, i)) continue; - if (dsa_to_port(ds, i)->bridge_dev != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Remove this port to the portvlan mask of the other ports * in the bridge */ - qca8k_reg_clear(priv, - QCA8K_PORT_LOOKUP_CTRL(i), - BIT(port)); + regmap_clear_bits(priv->regmap, + QCA8K_PORT_LOOKUP_CTRL(i), + BIT(port)); } /* Set the cpu port to be the only one in the portvlan mask of @@ -1802,6 +1873,36 @@ qca8k_port_bridge_leave(struct dsa_switch *ds, int port, struct net_device *br) QCA8K_PORT_LOOKUP_MEMBER, BIT(cpu_port)); } +static void +qca8k_port_fast_age(struct dsa_switch *ds, int port) +{ + struct qca8k_priv *priv = ds->priv; + + mutex_lock(&priv->reg_mutex); + qca8k_fdb_access(priv, QCA8K_FDB_FLUSH_PORT, port); + mutex_unlock(&priv->reg_mutex); +} + +static int +qca8k_set_ageing_time(struct dsa_switch *ds, unsigned int msecs) +{ + struct qca8k_priv *priv = ds->priv; + unsigned int secs = msecs / 1000; + u32 val; + + /* AGE_TIME reg is set in 7s step */ + val = secs / 7; + + /* Handle case with 0 as val to NOT disable + * learning + */ + if (!val) + val = 1; + + return regmap_update_bits(priv->regmap, QCA8K_REG_ATU_CTRL, QCA8K_ATU_AGE_TIME_MASK, + QCA8K_ATU_AGE_TIME(val)); +} + static int qca8k_port_enable(struct dsa_switch *ds, int port, struct phy_device *phy) @@ -1908,6 +2009,121 @@ qca8k_port_fdb_dump(struct dsa_switch *ds, int port, } static int +qca8k_port_mdb_add(struct dsa_switch *ds, int port, + const struct switchdev_obj_port_mdb *mdb) +{ + struct qca8k_priv *priv = ds->priv; + const u8 *addr = mdb->addr; + u16 vid = mdb->vid; + + return qca8k_fdb_search_and_insert(priv, BIT(port), addr, vid); +} + +static int +qca8k_port_mdb_del(struct dsa_switch *ds, int port, + const struct switchdev_obj_port_mdb *mdb) +{ + struct qca8k_priv *priv = ds->priv; + const u8 *addr = mdb->addr; + u16 vid = mdb->vid; + + return qca8k_fdb_search_and_del(priv, BIT(port), addr, vid); +} + +static int +qca8k_port_mirror_add(struct dsa_switch *ds, int port, + struct dsa_mall_mirror_tc_entry *mirror, + bool ingress) +{ + struct qca8k_priv *priv = ds->priv; + int monitor_port, ret; + u32 reg, val; + + /* Check for existent entry */ + if ((ingress ? priv->mirror_rx : priv->mirror_tx) & BIT(port)) + return -EEXIST; + + ret = regmap_read(priv->regmap, QCA8K_REG_GLOBAL_FW_CTRL0, &val); + if (ret) + return ret; + + /* QCA83xx can have only one port set to mirror mode. + * Check that the correct port is requested and return error otherwise. + * When no mirror port is set, the values is set to 0xF + */ + monitor_port = FIELD_GET(QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM, val); + if (monitor_port != 0xF && monitor_port != mirror->to_local_port) + return -EEXIST; + + /* Set the monitor port */ + val = FIELD_PREP(QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM, + mirror->to_local_port); + ret = regmap_update_bits(priv->regmap, QCA8K_REG_GLOBAL_FW_CTRL0, + QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM, val); + if (ret) + return ret; + + if (ingress) { + reg = QCA8K_PORT_LOOKUP_CTRL(port); + val = QCA8K_PORT_LOOKUP_ING_MIRROR_EN; + } else { + reg = QCA8K_REG_PORT_HOL_CTRL1(port); + val = QCA8K_PORT_HOL_CTRL1_EG_MIRROR_EN; + } + + ret = regmap_update_bits(priv->regmap, reg, val, val); + if (ret) + return ret; + + /* Track mirror port for tx and rx to decide when the + * mirror port has to be disabled. + */ + if (ingress) + priv->mirror_rx |= BIT(port); + else + priv->mirror_tx |= BIT(port); + + return 0; +} + +static void +qca8k_port_mirror_del(struct dsa_switch *ds, int port, + struct dsa_mall_mirror_tc_entry *mirror) +{ + struct qca8k_priv *priv = ds->priv; + u32 reg, val; + int ret; + + if (mirror->ingress) { + reg = QCA8K_PORT_LOOKUP_CTRL(port); + val = QCA8K_PORT_LOOKUP_ING_MIRROR_EN; + } else { + reg = QCA8K_REG_PORT_HOL_CTRL1(port); + val = QCA8K_PORT_HOL_CTRL1_EG_MIRROR_EN; + } + + ret = regmap_clear_bits(priv->regmap, reg, val); + if (ret) + goto err; + + if (mirror->ingress) + priv->mirror_rx &= ~BIT(port); + else + priv->mirror_tx &= ~BIT(port); + + /* No port set to send packet to mirror port. Disable mirror port */ + if (!priv->mirror_rx && !priv->mirror_tx) { + val = FIELD_PREP(QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM, 0xF); + ret = regmap_update_bits(priv->regmap, QCA8K_REG_GLOBAL_FW_CTRL0, + QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM, val); + if (ret) + goto err; + } +err: + dev_err(priv->dev, "Failed to del mirror port from %d", port); +} + +static int qca8k_port_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering, struct netlink_ext_ack *extack) { @@ -1916,11 +2132,11 @@ qca8k_port_vlan_filtering(struct dsa_switch *ds, int port, bool vlan_filtering, if (vlan_filtering) { ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port), - QCA8K_PORT_LOOKUP_VLAN_MODE, + QCA8K_PORT_LOOKUP_VLAN_MODE_MASK, QCA8K_PORT_LOOKUP_VLAN_MODE_SECURE); } else { ret = qca8k_rmw(priv, QCA8K_PORT_LOOKUP_CTRL(port), - QCA8K_PORT_LOOKUP_VLAN_MODE, + QCA8K_PORT_LOOKUP_VLAN_MODE_MASK, QCA8K_PORT_LOOKUP_VLAN_MODE_NONE); } @@ -1944,10 +2160,9 @@ qca8k_port_vlan_add(struct dsa_switch *ds, int port, } if (pvid) { - int shift = 16 * (port % 2); - ret = qca8k_rmw(priv, QCA8K_EGRESS_VLAN(port), - 0xfff << shift, vlan->vid << shift); + QCA8K_EGREES_VLAN_PORT_MASK(port), + QCA8K_EGREES_VLAN_PORT(port, vlan->vid)); if (ret) return ret; @@ -1996,12 +2211,185 @@ qca8k_get_tag_protocol(struct dsa_switch *ds, int port, return DSA_TAG_PROTO_QCA; } +static bool +qca8k_lag_can_offload(struct dsa_switch *ds, + struct net_device *lag, + struct netdev_lag_upper_info *info) +{ + struct dsa_port *dp; + int id, members = 0; + + id = dsa_lag_id(ds->dst, lag); + if (id < 0 || id >= ds->num_lag_ids) + return false; + + dsa_lag_foreach_port(dp, ds->dst, lag) + /* Includes the port joining the LAG */ + members++; + + if (members > QCA8K_NUM_PORTS_FOR_LAG) + return false; + + if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) + return false; + + if (info->hash_type != NETDEV_LAG_HASH_L2 && + info->hash_type != NETDEV_LAG_HASH_L23) + return false; + + return true; +} + +static int +qca8k_lag_setup_hash(struct dsa_switch *ds, + struct net_device *lag, + struct netdev_lag_upper_info *info) +{ + struct qca8k_priv *priv = ds->priv; + bool unique_lag = true; + u32 hash = 0; + int i, id; + + id = dsa_lag_id(ds->dst, lag); + + switch (info->hash_type) { + case NETDEV_LAG_HASH_L23: + hash |= QCA8K_TRUNK_HASH_SIP_EN; + hash |= QCA8K_TRUNK_HASH_DIP_EN; + fallthrough; + case NETDEV_LAG_HASH_L2: + hash |= QCA8K_TRUNK_HASH_SA_EN; + hash |= QCA8K_TRUNK_HASH_DA_EN; + break; + default: /* We should NEVER reach this */ + return -EOPNOTSUPP; + } + + /* Check if we are the unique configured LAG */ + dsa_lags_foreach_id(i, ds->dst) + if (i != id && dsa_lag_dev(ds->dst, i)) { + unique_lag = false; + break; + } + + /* Hash Mode is global. Make sure the same Hash Mode + * is set to all the 4 possible lag. + * If we are the unique LAG we can set whatever hash + * mode we want. + * To change hash mode it's needed to remove all LAG + * and change the mode with the latest. + */ + if (unique_lag) { + priv->lag_hash_mode = hash; + } else if (priv->lag_hash_mode != hash) { + netdev_err(lag, "Error: Mismatched Hash Mode across different lag is not supported\n"); + return -EOPNOTSUPP; + } + + return regmap_update_bits(priv->regmap, QCA8K_TRUNK_HASH_EN_CTRL, + QCA8K_TRUNK_HASH_MASK, hash); +} + +static int +qca8k_lag_refresh_portmap(struct dsa_switch *ds, int port, + struct net_device *lag, bool delete) +{ + struct qca8k_priv *priv = ds->priv; + int ret, id, i; + u32 val; + + id = dsa_lag_id(ds->dst, lag); + + /* Read current port member */ + ret = regmap_read(priv->regmap, QCA8K_REG_GOL_TRUNK_CTRL0, &val); + if (ret) + return ret; + + /* Shift val to the correct trunk */ + val >>= QCA8K_REG_GOL_TRUNK_SHIFT(id); + val &= QCA8K_REG_GOL_TRUNK_MEMBER_MASK; + if (delete) + val &= ~BIT(port); + else + val |= BIT(port); + + /* Update port member. With empty portmap disable trunk */ + ret = regmap_update_bits(priv->regmap, QCA8K_REG_GOL_TRUNK_CTRL0, + QCA8K_REG_GOL_TRUNK_MEMBER(id) | + QCA8K_REG_GOL_TRUNK_EN(id), + !val << QCA8K_REG_GOL_TRUNK_SHIFT(id) | + val << QCA8K_REG_GOL_TRUNK_SHIFT(id)); + + /* Search empty member if adding or port on deleting */ + for (i = 0; i < QCA8K_NUM_PORTS_FOR_LAG; i++) { + ret = regmap_read(priv->regmap, QCA8K_REG_GOL_TRUNK_CTRL(id), &val); + if (ret) + return ret; + + val >>= QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(id, i); + val &= QCA8K_REG_GOL_TRUNK_ID_MEM_ID_MASK; + + if (delete) { + /* If port flagged to be disabled assume this member is + * empty + */ + if (val != QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN_MASK) + continue; + + val &= QCA8K_REG_GOL_TRUNK_ID_MEM_ID_PORT_MASK; + if (val != port) + continue; + } else { + /* If port flagged to be enabled assume this member is + * already set + */ + if (val == QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN_MASK) + continue; + } + + /* We have found the member to add/remove */ + break; + } + + /* Set port in the correct port mask or disable port if in delete mode */ + return regmap_update_bits(priv->regmap, QCA8K_REG_GOL_TRUNK_CTRL(id), + QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN(id, i) | + QCA8K_REG_GOL_TRUNK_ID_MEM_ID_PORT(id, i), + !delete << QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(id, i) | + port << QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(id, i)); +} + +static int +qca8k_port_lag_join(struct dsa_switch *ds, int port, + struct net_device *lag, + struct netdev_lag_upper_info *info) +{ + int ret; + + if (!qca8k_lag_can_offload(ds, lag, info)) + return -EOPNOTSUPP; + + ret = qca8k_lag_setup_hash(ds, lag, info); + if (ret) + return ret; + + return qca8k_lag_refresh_portmap(ds, port, lag, false); +} + +static int +qca8k_port_lag_leave(struct dsa_switch *ds, int port, + struct net_device *lag) +{ + return qca8k_lag_refresh_portmap(ds, port, lag, true); +} + static const struct dsa_switch_ops qca8k_switch_ops = { .get_tag_protocol = qca8k_get_tag_protocol, .setup = qca8k_setup, .get_strings = qca8k_get_strings, .get_ethtool_stats = qca8k_get_ethtool_stats, .get_sset_count = qca8k_get_sset_count, + .set_ageing_time = qca8k_set_ageing_time, .get_mac_eee = qca8k_get_mac_eee, .set_mac_eee = qca8k_set_mac_eee, .port_enable = qca8k_port_enable, @@ -2011,9 +2399,14 @@ static const struct dsa_switch_ops qca8k_switch_ops = { .port_stp_state_set = qca8k_port_stp_state_set, .port_bridge_join = qca8k_port_bridge_join, .port_bridge_leave = qca8k_port_bridge_leave, + .port_fast_age = qca8k_port_fast_age, .port_fdb_add = qca8k_port_fdb_add, .port_fdb_del = qca8k_port_fdb_del, .port_fdb_dump = qca8k_port_fdb_dump, + .port_mdb_add = qca8k_port_mdb_add, + .port_mdb_del = qca8k_port_mdb_del, + .port_mirror_add = qca8k_port_mirror_add, + .port_mirror_del = qca8k_port_mirror_del, .port_vlan_filtering = qca8k_port_vlan_filtering, .port_vlan_add = qca8k_port_vlan_add, .port_vlan_del = qca8k_port_vlan_del, @@ -2023,6 +2416,8 @@ static const struct dsa_switch_ops qca8k_switch_ops = { .phylink_mac_link_down = qca8k_phylink_mac_link_down, .phylink_mac_link_up = qca8k_phylink_mac_link_up, .get_phy_flags = qca8k_get_phy_flags, + .port_lag_join = qca8k_port_lag_join, + .port_lag_leave = qca8k_port_lag_leave, }; static int qca8k_read_switch_id(struct qca8k_priv *priv) @@ -2041,7 +2436,7 @@ static int qca8k_read_switch_id(struct qca8k_priv *priv) if (ret < 0) return -ENODEV; - id = QCA8K_MASK_CTRL_DEVICE_ID(val & QCA8K_MASK_CTRL_DEVICE_ID_MASK); + id = QCA8K_MASK_CTRL_DEVICE_ID(val); if (id != data->id) { dev_err(priv->dev, "Switch id detected %x but expected %x", id, data->id); return -ENODEV; @@ -2050,7 +2445,7 @@ static int qca8k_read_switch_id(struct qca8k_priv *priv) priv->switch_id = id; /* Save revision to communicate to the internal PHY driver */ - priv->switch_revision = (val & QCA8K_MASK_CTRL_REV_ID_MASK); + priv->switch_revision = QCA8K_MASK_CTRL_REV_ID(val); return 0; } @@ -2085,6 +2480,14 @@ qca8k_sw_probe(struct mdio_device *mdiodev) gpiod_set_value_cansleep(priv->reset_gpio, 0); } + /* Start by setting up the register mapping */ + priv->regmap = devm_regmap_init(&mdiodev->dev, NULL, priv, + &qca8k_regmap_config); + if (IS_ERR(priv->regmap)) { + dev_err(priv->dev, "regmap initialization failed"); + return PTR_ERR(priv->regmap); + } + /* Check the detected switch id */ ret = qca8k_read_switch_id(priv); if (ret) @@ -2173,14 +2576,17 @@ static SIMPLE_DEV_PM_OPS(qca8k_pm_ops, static const struct qca8k_match_data qca8327 = { .id = QCA8K_ID_QCA8327, .reduced_package = true, + .mib_count = QCA8K_QCA832X_MIB_COUNT, }; static const struct qca8k_match_data qca8328 = { .id = QCA8K_ID_QCA8327, + .mib_count = QCA8K_QCA832X_MIB_COUNT, }; static const struct qca8k_match_data qca833x = { .id = QCA8K_ID_QCA8337, + .mib_count = QCA8K_QCA833X_MIB_COUNT, }; static const struct of_device_id qca8k_of_match[] = { diff --git a/drivers/net/dsa/qca8k.h b/drivers/net/dsa/qca8k.h index 128b8cf85e08..ab4a417b25a9 100644 --- a/drivers/net/dsa/qca8k.h +++ b/drivers/net/dsa/qca8k.h @@ -15,12 +15,17 @@ #define QCA8K_NUM_PORTS 7 #define QCA8K_NUM_CPU_PORTS 2 #define QCA8K_MAX_MTU 9000 +#define QCA8K_NUM_LAGS 4 +#define QCA8K_NUM_PORTS_FOR_LAG 4 #define PHY_ID_QCA8327 0x004dd034 #define QCA8K_ID_QCA8327 0x12 #define PHY_ID_QCA8337 0x004dd036 #define QCA8K_ID_QCA8337 0x13 +#define QCA8K_QCA832X_MIB_COUNT 39 +#define QCA8K_QCA833X_MIB_COUNT 41 + #define QCA8K_BUSY_WAIT_TIMEOUT 2000 #define QCA8K_NUM_FDB_RECORDS 2048 @@ -30,9 +35,9 @@ /* Global control registers */ #define QCA8K_REG_MASK_CTRL 0x000 #define QCA8K_MASK_CTRL_REV_ID_MASK GENMASK(7, 0) -#define QCA8K_MASK_CTRL_REV_ID(x) ((x) >> 0) +#define QCA8K_MASK_CTRL_REV_ID(x) FIELD_GET(QCA8K_MASK_CTRL_REV_ID_MASK, x) #define QCA8K_MASK_CTRL_DEVICE_ID_MASK GENMASK(15, 8) -#define QCA8K_MASK_CTRL_DEVICE_ID(x) ((x) >> 8) +#define QCA8K_MASK_CTRL_DEVICE_ID(x) FIELD_GET(QCA8K_MASK_CTRL_DEVICE_ID_MASK, x) #define QCA8K_REG_PORT0_PAD_CTRL 0x004 #define QCA8K_PORT0_PAD_MAC06_EXCHANGE_EN BIT(31) #define QCA8K_PORT0_PAD_SGMII_RXCLK_FALLING_EDGE BIT(19) @@ -41,12 +46,11 @@ #define QCA8K_REG_PORT6_PAD_CTRL 0x00c #define QCA8K_PORT_PAD_RGMII_EN BIT(26) #define QCA8K_PORT_PAD_RGMII_TX_DELAY_MASK GENMASK(23, 22) -#define QCA8K_PORT_PAD_RGMII_TX_DELAY(x) ((x) << 22) +#define QCA8K_PORT_PAD_RGMII_TX_DELAY(x) FIELD_PREP(QCA8K_PORT_PAD_RGMII_TX_DELAY_MASK, x) #define QCA8K_PORT_PAD_RGMII_RX_DELAY_MASK GENMASK(21, 20) -#define QCA8K_PORT_PAD_RGMII_RX_DELAY(x) ((x) << 20) +#define QCA8K_PORT_PAD_RGMII_RX_DELAY(x) FIELD_PREP(QCA8K_PORT_PAD_RGMII_RX_DELAY_MASK, x) #define QCA8K_PORT_PAD_RGMII_TX_DELAY_EN BIT(25) #define QCA8K_PORT_PAD_RGMII_RX_DELAY_EN BIT(24) -#define QCA8K_MAX_DELAY 3 #define QCA8K_PORT_PAD_SGMII_EN BIT(7) #define QCA8K_REG_PWS 0x010 #define QCA8K_PWS_POWER_ON_SEL BIT(31) @@ -68,10 +72,12 @@ #define QCA8K_MDIO_MASTER_READ BIT(27) #define QCA8K_MDIO_MASTER_WRITE 0 #define QCA8K_MDIO_MASTER_SUP_PRE BIT(26) -#define QCA8K_MDIO_MASTER_PHY_ADDR(x) ((x) << 21) -#define QCA8K_MDIO_MASTER_REG_ADDR(x) ((x) << 16) -#define QCA8K_MDIO_MASTER_DATA(x) (x) +#define QCA8K_MDIO_MASTER_PHY_ADDR_MASK GENMASK(25, 21) +#define QCA8K_MDIO_MASTER_PHY_ADDR(x) FIELD_PREP(QCA8K_MDIO_MASTER_PHY_ADDR_MASK, x) +#define QCA8K_MDIO_MASTER_REG_ADDR_MASK GENMASK(20, 16) +#define QCA8K_MDIO_MASTER_REG_ADDR(x) FIELD_PREP(QCA8K_MDIO_MASTER_REG_ADDR_MASK, x) #define QCA8K_MDIO_MASTER_DATA_MASK GENMASK(15, 0) +#define QCA8K_MDIO_MASTER_DATA(x) FIELD_PREP(QCA8K_MDIO_MASTER_DATA_MASK, x) #define QCA8K_MDIO_MASTER_MAX_PORTS 5 #define QCA8K_MDIO_MASTER_MAX_REG 32 #define QCA8K_GOL_MAC_ADDR0 0x60 @@ -93,9 +99,7 @@ #define QCA8K_PORT_STATUS_FLOW_AUTO BIT(12) #define QCA8K_REG_PORT_HDR_CTRL(_i) (0x9c + (_i * 4)) #define QCA8K_PORT_HDR_CTRL_RX_MASK GENMASK(3, 2) -#define QCA8K_PORT_HDR_CTRL_RX_S 2 #define QCA8K_PORT_HDR_CTRL_TX_MASK GENMASK(1, 0) -#define QCA8K_PORT_HDR_CTRL_TX_S 0 #define QCA8K_PORT_HDR_CTRL_ALL 2 #define QCA8K_PORT_HDR_CTRL_MGMT 1 #define QCA8K_PORT_HDR_CTRL_NONE 0 @@ -105,10 +109,11 @@ #define QCA8K_SGMII_EN_TX BIT(3) #define QCA8K_SGMII_EN_SD BIT(4) #define QCA8K_SGMII_CLK125M_DELAY BIT(7) -#define QCA8K_SGMII_MODE_CTRL_MASK (BIT(22) | BIT(23)) -#define QCA8K_SGMII_MODE_CTRL_BASEX (0 << 22) -#define QCA8K_SGMII_MODE_CTRL_PHY (1 << 22) -#define QCA8K_SGMII_MODE_CTRL_MAC (2 << 22) +#define QCA8K_SGMII_MODE_CTRL_MASK GENMASK(23, 22) +#define QCA8K_SGMII_MODE_CTRL(x) FIELD_PREP(QCA8K_SGMII_MODE_CTRL_MASK, x) +#define QCA8K_SGMII_MODE_CTRL_BASEX QCA8K_SGMII_MODE_CTRL(0x0) +#define QCA8K_SGMII_MODE_CTRL_PHY QCA8K_SGMII_MODE_CTRL(0x1) +#define QCA8K_SGMII_MODE_CTRL_MAC QCA8K_SGMII_MODE_CTRL(0x2) /* MAC_PWR_SEL registers */ #define QCA8K_REG_MAC_PWR_SEL 0x0e4 @@ -119,102 +124,152 @@ #define QCA8K_REG_EEE_CTRL 0x100 #define QCA8K_REG_EEE_CTRL_LPI_EN(_i) ((_i + 1) * 2) +/* TRUNK_HASH_EN registers */ +#define QCA8K_TRUNK_HASH_EN_CTRL 0x270 +#define QCA8K_TRUNK_HASH_SIP_EN BIT(3) +#define QCA8K_TRUNK_HASH_DIP_EN BIT(2) +#define QCA8K_TRUNK_HASH_SA_EN BIT(1) +#define QCA8K_TRUNK_HASH_DA_EN BIT(0) +#define QCA8K_TRUNK_HASH_MASK GENMASK(3, 0) + /* ACL registers */ #define QCA8K_REG_PORT_VLAN_CTRL0(_i) (0x420 + (_i * 8)) -#define QCA8K_PORT_VLAN_CVID(x) (x << 16) -#define QCA8K_PORT_VLAN_SVID(x) x +#define QCA8K_PORT_VLAN_CVID_MASK GENMASK(27, 16) +#define QCA8K_PORT_VLAN_CVID(x) FIELD_PREP(QCA8K_PORT_VLAN_CVID_MASK, x) +#define QCA8K_PORT_VLAN_SVID_MASK GENMASK(11, 0) +#define QCA8K_PORT_VLAN_SVID(x) FIELD_PREP(QCA8K_PORT_VLAN_SVID_MASK, x) #define QCA8K_REG_PORT_VLAN_CTRL1(_i) (0x424 + (_i * 8)) #define QCA8K_REG_IPV4_PRI_BASE_ADDR 0x470 #define QCA8K_REG_IPV4_PRI_ADDR_MASK 0x474 /* Lookup registers */ #define QCA8K_REG_ATU_DATA0 0x600 -#define QCA8K_ATU_ADDR2_S 24 -#define QCA8K_ATU_ADDR3_S 16 -#define QCA8K_ATU_ADDR4_S 8 +#define QCA8K_ATU_ADDR2_MASK GENMASK(31, 24) +#define QCA8K_ATU_ADDR3_MASK GENMASK(23, 16) +#define QCA8K_ATU_ADDR4_MASK GENMASK(15, 8) +#define QCA8K_ATU_ADDR5_MASK GENMASK(7, 0) #define QCA8K_REG_ATU_DATA1 0x604 -#define QCA8K_ATU_PORT_M 0x7f -#define QCA8K_ATU_PORT_S 16 -#define QCA8K_ATU_ADDR0_S 8 +#define QCA8K_ATU_PORT_MASK GENMASK(22, 16) +#define QCA8K_ATU_ADDR0_MASK GENMASK(15, 8) +#define QCA8K_ATU_ADDR1_MASK GENMASK(7, 0) #define QCA8K_REG_ATU_DATA2 0x608 -#define QCA8K_ATU_VID_M 0xfff -#define QCA8K_ATU_VID_S 8 -#define QCA8K_ATU_STATUS_M 0xf +#define QCA8K_ATU_VID_MASK GENMASK(19, 8) +#define QCA8K_ATU_STATUS_MASK GENMASK(3, 0) #define QCA8K_ATU_STATUS_STATIC 0xf #define QCA8K_REG_ATU_FUNC 0x60c #define QCA8K_ATU_FUNC_BUSY BIT(31) #define QCA8K_ATU_FUNC_PORT_EN BIT(14) #define QCA8K_ATU_FUNC_MULTI_EN BIT(13) #define QCA8K_ATU_FUNC_FULL BIT(12) -#define QCA8K_ATU_FUNC_PORT_M 0xf -#define QCA8K_ATU_FUNC_PORT_S 8 +#define QCA8K_ATU_FUNC_PORT_MASK GENMASK(11, 8) #define QCA8K_REG_VTU_FUNC0 0x610 #define QCA8K_VTU_FUNC0_VALID BIT(20) #define QCA8K_VTU_FUNC0_IVL_EN BIT(19) -#define QCA8K_VTU_FUNC0_EG_MODE_S(_i) (4 + (_i) * 2) -#define QCA8K_VTU_FUNC0_EG_MODE_MASK 3 -#define QCA8K_VTU_FUNC0_EG_MODE_UNMOD 0 -#define QCA8K_VTU_FUNC0_EG_MODE_UNTAG 1 -#define QCA8K_VTU_FUNC0_EG_MODE_TAG 2 -#define QCA8K_VTU_FUNC0_EG_MODE_NOT 3 +/* QCA8K_VTU_FUNC0_EG_MODE_MASK GENMASK(17, 4) + * It does contain VLAN_MODE for each port [5:4] for port0, + * [7:6] for port1 ... [17:16] for port6. Use virtual port + * define to handle this. + */ +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i) (4 + (_i) * 2) +#define QCA8K_VTU_FUNC0_EG_MODE_MASK GENMASK(1, 0) +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_MASK(_i) (GENMASK(1, 0) << QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i)) +#define QCA8K_VTU_FUNC0_EG_MODE_UNMOD FIELD_PREP(QCA8K_VTU_FUNC0_EG_MODE_MASK, 0x0) +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_UNMOD(_i) (QCA8K_VTU_FUNC0_EG_MODE_UNMOD << QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i)) +#define QCA8K_VTU_FUNC0_EG_MODE_UNTAG FIELD_PREP(QCA8K_VTU_FUNC0_EG_MODE_MASK, 0x1) +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_UNTAG(_i) (QCA8K_VTU_FUNC0_EG_MODE_UNTAG << QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i)) +#define QCA8K_VTU_FUNC0_EG_MODE_TAG FIELD_PREP(QCA8K_VTU_FUNC0_EG_MODE_MASK, 0x2) +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_TAG(_i) (QCA8K_VTU_FUNC0_EG_MODE_TAG << QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i)) +#define QCA8K_VTU_FUNC0_EG_MODE_NOT FIELD_PREP(QCA8K_VTU_FUNC0_EG_MODE_MASK, 0x3) +#define QCA8K_VTU_FUNC0_EG_MODE_PORT_NOT(_i) (QCA8K_VTU_FUNC0_EG_MODE_NOT << QCA8K_VTU_FUNC0_EG_MODE_PORT_SHIFT(_i)) #define QCA8K_REG_VTU_FUNC1 0x614 #define QCA8K_VTU_FUNC1_BUSY BIT(31) -#define QCA8K_VTU_FUNC1_VID_S 16 +#define QCA8K_VTU_FUNC1_VID_MASK GENMASK(27, 16) #define QCA8K_VTU_FUNC1_FULL BIT(4) +#define QCA8K_REG_ATU_CTRL 0x618 +#define QCA8K_ATU_AGE_TIME_MASK GENMASK(15, 0) +#define QCA8K_ATU_AGE_TIME(x) FIELD_PREP(QCA8K_ATU_AGE_TIME_MASK, (x)) #define QCA8K_REG_GLOBAL_FW_CTRL0 0x620 #define QCA8K_GLOBAL_FW_CTRL0_CPU_PORT_EN BIT(10) +#define QCA8K_GLOBAL_FW_CTRL0_MIRROR_PORT_NUM GENMASK(7, 4) #define QCA8K_REG_GLOBAL_FW_CTRL1 0x624 -#define QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_S 24 -#define QCA8K_GLOBAL_FW_CTRL1_BC_DP_S 16 -#define QCA8K_GLOBAL_FW_CTRL1_MC_DP_S 8 -#define QCA8K_GLOBAL_FW_CTRL1_UC_DP_S 0 +#define QCA8K_GLOBAL_FW_CTRL1_IGMP_DP_MASK GENMASK(30, 24) +#define QCA8K_GLOBAL_FW_CTRL1_BC_DP_MASK GENMASK(22, 16) +#define QCA8K_GLOBAL_FW_CTRL1_MC_DP_MASK GENMASK(14, 8) +#define QCA8K_GLOBAL_FW_CTRL1_UC_DP_MASK GENMASK(6, 0) #define QCA8K_PORT_LOOKUP_CTRL(_i) (0x660 + (_i) * 0xc) #define QCA8K_PORT_LOOKUP_MEMBER GENMASK(6, 0) -#define QCA8K_PORT_LOOKUP_VLAN_MODE GENMASK(9, 8) -#define QCA8K_PORT_LOOKUP_VLAN_MODE_NONE (0 << 8) -#define QCA8K_PORT_LOOKUP_VLAN_MODE_FALLBACK (1 << 8) -#define QCA8K_PORT_LOOKUP_VLAN_MODE_CHECK (2 << 8) -#define QCA8K_PORT_LOOKUP_VLAN_MODE_SECURE (3 << 8) +#define QCA8K_PORT_LOOKUP_VLAN_MODE_MASK GENMASK(9, 8) +#define QCA8K_PORT_LOOKUP_VLAN_MODE(x) FIELD_PREP(QCA8K_PORT_LOOKUP_VLAN_MODE_MASK, x) +#define QCA8K_PORT_LOOKUP_VLAN_MODE_NONE QCA8K_PORT_LOOKUP_VLAN_MODE(0x0) +#define QCA8K_PORT_LOOKUP_VLAN_MODE_FALLBACK QCA8K_PORT_LOOKUP_VLAN_MODE(0x1) +#define QCA8K_PORT_LOOKUP_VLAN_MODE_CHECK QCA8K_PORT_LOOKUP_VLAN_MODE(0x2) +#define QCA8K_PORT_LOOKUP_VLAN_MODE_SECURE QCA8K_PORT_LOOKUP_VLAN_MODE(0x3) #define QCA8K_PORT_LOOKUP_STATE_MASK GENMASK(18, 16) -#define QCA8K_PORT_LOOKUP_STATE_DISABLED (0 << 16) -#define QCA8K_PORT_LOOKUP_STATE_BLOCKING (1 << 16) -#define QCA8K_PORT_LOOKUP_STATE_LISTENING (2 << 16) -#define QCA8K_PORT_LOOKUP_STATE_LEARNING (3 << 16) -#define QCA8K_PORT_LOOKUP_STATE_FORWARD (4 << 16) -#define QCA8K_PORT_LOOKUP_STATE GENMASK(18, 16) +#define QCA8K_PORT_LOOKUP_STATE(x) FIELD_PREP(QCA8K_PORT_LOOKUP_STATE_MASK, x) +#define QCA8K_PORT_LOOKUP_STATE_DISABLED QCA8K_PORT_LOOKUP_STATE(0x0) +#define QCA8K_PORT_LOOKUP_STATE_BLOCKING QCA8K_PORT_LOOKUP_STATE(0x1) +#define QCA8K_PORT_LOOKUP_STATE_LISTENING QCA8K_PORT_LOOKUP_STATE(0x2) +#define QCA8K_PORT_LOOKUP_STATE_LEARNING QCA8K_PORT_LOOKUP_STATE(0x3) +#define QCA8K_PORT_LOOKUP_STATE_FORWARD QCA8K_PORT_LOOKUP_STATE(0x4) #define QCA8K_PORT_LOOKUP_LEARN BIT(20) +#define QCA8K_PORT_LOOKUP_ING_MIRROR_EN BIT(25) + +#define QCA8K_REG_GOL_TRUNK_CTRL0 0x700 +/* 4 max trunk first + * first 6 bit for member bitmap + * 7th bit is to enable trunk port + */ +#define QCA8K_REG_GOL_TRUNK_SHIFT(_i) ((_i) * 8) +#define QCA8K_REG_GOL_TRUNK_EN_MASK BIT(7) +#define QCA8K_REG_GOL_TRUNK_EN(_i) (QCA8K_REG_GOL_TRUNK_EN_MASK << QCA8K_REG_GOL_TRUNK_SHIFT(_i)) +#define QCA8K_REG_GOL_TRUNK_MEMBER_MASK GENMASK(6, 0) +#define QCA8K_REG_GOL_TRUNK_MEMBER(_i) (QCA8K_REG_GOL_TRUNK_MEMBER_MASK << QCA8K_REG_GOL_TRUNK_SHIFT(_i)) +/* 0x704 for TRUNK 0-1 --- 0x708 for TRUNK 2-3 */ +#define QCA8K_REG_GOL_TRUNK_CTRL(_i) (0x704 + (((_i) / 2) * 4)) +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_MASK GENMASK(3, 0) +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN_MASK BIT(3) +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_PORT_MASK GENMASK(2, 0) +#define QCA8K_REG_GOL_TRUNK_ID_SHIFT(_i) (((_i) / 2) * 16) +#define QCA8K_REG_GOL_MEM_ID_SHIFT(_i) ((_i) * 4) +/* Complex shift: FIRST shift for port THEN shift for trunk */ +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(_i, _j) (QCA8K_REG_GOL_MEM_ID_SHIFT(_j) + QCA8K_REG_GOL_TRUNK_ID_SHIFT(_i)) +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN(_i, _j) (QCA8K_REG_GOL_TRUNK_ID_MEM_ID_EN_MASK << QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(_i, _j)) +#define QCA8K_REG_GOL_TRUNK_ID_MEM_ID_PORT(_i, _j) (QCA8K_REG_GOL_TRUNK_ID_MEM_ID_PORT_MASK << QCA8K_REG_GOL_TRUNK_ID_MEM_ID_SHIFT(_i, _j)) #define QCA8K_REG_GLOBAL_FC_THRESH 0x800 -#define QCA8K_GLOBAL_FC_GOL_XON_THRES(x) ((x) << 16) -#define QCA8K_GLOBAL_FC_GOL_XON_THRES_S GENMASK(24, 16) -#define QCA8K_GLOBAL_FC_GOL_XOFF_THRES(x) ((x) << 0) -#define QCA8K_GLOBAL_FC_GOL_XOFF_THRES_S GENMASK(8, 0) +#define QCA8K_GLOBAL_FC_GOL_XON_THRES_MASK GENMASK(24, 16) +#define QCA8K_GLOBAL_FC_GOL_XON_THRES(x) FIELD_PREP(QCA8K_GLOBAL_FC_GOL_XON_THRES_MASK, x) +#define QCA8K_GLOBAL_FC_GOL_XOFF_THRES_MASK GENMASK(8, 0) +#define QCA8K_GLOBAL_FC_GOL_XOFF_THRES(x) FIELD_PREP(QCA8K_GLOBAL_FC_GOL_XOFF_THRES_MASK, x) #define QCA8K_REG_PORT_HOL_CTRL0(_i) (0x970 + (_i) * 0x8) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI0_BUF GENMASK(3, 0) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI0(x) ((x) << 0) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI1_BUF GENMASK(7, 4) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI1(x) ((x) << 4) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI2_BUF GENMASK(11, 8) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI2(x) ((x) << 8) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI3_BUF GENMASK(15, 12) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI3(x) ((x) << 12) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI4_BUF GENMASK(19, 16) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI4(x) ((x) << 16) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI5_BUF GENMASK(23, 20) -#define QCA8K_PORT_HOL_CTRL0_EG_PRI5(x) ((x) << 20) -#define QCA8K_PORT_HOL_CTRL0_EG_PORT_BUF GENMASK(29, 24) -#define QCA8K_PORT_HOL_CTRL0_EG_PORT(x) ((x) << 24) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI0_BUF_MASK GENMASK(3, 0) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI0(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI0_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI1_BUF_MASK GENMASK(7, 4) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI1(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI1_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI2_BUF_MASK GENMASK(11, 8) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI2(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI2_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI3_BUF_MASK GENMASK(15, 12) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI3(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI3_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI4_BUF_MASK GENMASK(19, 16) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI4(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI4_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI5_BUF_MASK GENMASK(23, 20) +#define QCA8K_PORT_HOL_CTRL0_EG_PRI5(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PRI5_BUF_MASK, x) +#define QCA8K_PORT_HOL_CTRL0_EG_PORT_BUF_MASK GENMASK(29, 24) +#define QCA8K_PORT_HOL_CTRL0_EG_PORT(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL0_EG_PORT_BUF_MASK, x) #define QCA8K_REG_PORT_HOL_CTRL1(_i) (0x974 + (_i) * 0x8) -#define QCA8K_PORT_HOL_CTRL1_ING_BUF GENMASK(3, 0) -#define QCA8K_PORT_HOL_CTRL1_ING(x) ((x) << 0) +#define QCA8K_PORT_HOL_CTRL1_ING_BUF_MASK GENMASK(3, 0) +#define QCA8K_PORT_HOL_CTRL1_ING(x) FIELD_PREP(QCA8K_PORT_HOL_CTRL1_ING_BUF_MASK, x) #define QCA8K_PORT_HOL_CTRL1_EG_PRI_BUF_EN BIT(6) #define QCA8K_PORT_HOL_CTRL1_EG_PORT_BUF_EN BIT(7) #define QCA8K_PORT_HOL_CTRL1_WRED_EN BIT(8) #define QCA8K_PORT_HOL_CTRL1_EG_MIRROR_EN BIT(16) /* Pkt edit registers */ +#define QCA8K_EGREES_VLAN_PORT_SHIFT(_i) (16 * ((_i) % 2)) +#define QCA8K_EGREES_VLAN_PORT_MASK(_i) (GENMASK(11, 0) << QCA8K_EGREES_VLAN_PORT_SHIFT(_i)) +#define QCA8K_EGREES_VLAN_PORT(_i, x) ((x) << QCA8K_EGREES_VLAN_PORT_SHIFT(_i)) #define QCA8K_EGRESS_VLAN(x) (0x0c70 + (4 * (x / 2))) /* L3 registers */ @@ -244,6 +299,7 @@ enum qca8k_fdb_cmd { QCA8K_FDB_FLUSH = 1, QCA8K_FDB_LOAD = 2, QCA8K_FDB_PURGE = 3, + QCA8K_FDB_FLUSH_PORT = 5, QCA8K_FDB_NEXT = 6, QCA8K_FDB_SEARCH = 7, }; @@ -264,6 +320,7 @@ struct ar8xxx_port_status { struct qca8k_match_data { u8 id; bool reduced_package; + u8 mib_count; }; enum { @@ -282,6 +339,9 @@ struct qca8k_ports_config { struct qca8k_priv { u8 switch_id; u8 switch_revision; + u8 mirror_rx; + u8 mirror_tx; + u8 lag_hash_mode; bool legacy_phy_port_mapping; struct qca8k_ports_config ports_config; struct regmap *regmap; diff --git a/drivers/net/dsa/realtek-smi-core.c b/drivers/net/dsa/realtek-smi-core.c index c66ebd0ee217..aae46ada8d83 100644 --- a/drivers/net/dsa/realtek-smi-core.c +++ b/drivers/net/dsa/realtek-smi-core.c @@ -456,7 +456,7 @@ static int realtek_smi_probe(struct platform_device *pdev) smi->ds->ops = var->ds_ops; ret = dsa_register_switch(smi->ds); if (ret) { - dev_err(dev, "unable to register switch ret = %d\n", ret); + dev_err_probe(dev, ret, "unable to register switch\n"); return ret; } return 0; diff --git a/drivers/net/dsa/rtl8365mb.c b/drivers/net/dsa/rtl8365mb.c index 078ca4cd7160..2ac68c867636 100644 --- a/drivers/net/dsa/rtl8365mb.c +++ b/drivers/net/dsa/rtl8365mb.c @@ -277,7 +277,7 @@ (RTL8365MB_PORT_ISOLATION_REG_BASE + (_physport)) #define RTL8365MB_PORT_ISOLATION_MASK 0x07FF -/* MSTP port state registers - indexed by tree instancrSTI (tree ine */ +/* MSTP port state registers - indexed by tree instance */ #define RTL8365MB_MSTI_CTRL_BASE 0x0A00 #define RTL8365MB_MSTI_CTRL_REG(_msti, _physport) \ (RTL8365MB_MSTI_CTRL_BASE + ((_msti) << 1) + ((_physport) >> 3)) @@ -767,7 +767,8 @@ static int rtl8365mb_ext_config_rgmii(struct realtek_smi *smi, int port, * 0 = no delay, 1 = 2 ns delay * RX delay: * 0 = no delay, 7 = maximum delay - * No units are specified, but there are a total of 8 steps. + * Each step is approximately 0.3 ns, so the maximum delay is about + * 2.1 ns. * * The vendor driver also states that this must be configured *before* * forcing the external interface into a particular mode, which is done @@ -778,10 +779,6 @@ static int rtl8365mb_ext_config_rgmii(struct realtek_smi *smi, int port, * specified. We ignore the detail of the RGMII interface mode * (RGMII_{RXID, TXID, etc.}), as this is considered to be a PHY-only * property. - * - * For the RX delay, we assume that a register value of 4 corresponds to - * 2 ns. But this is just an educated guess, so ignore all other values - * to avoid too much confusion. */ if (!of_property_read_u32(dn, "tx-internal-delay-ps", &val)) { val = val / 1000; /* convert to ns */ @@ -794,13 +791,13 @@ static int rtl8365mb_ext_config_rgmii(struct realtek_smi *smi, int port, } if (!of_property_read_u32(dn, "rx-internal-delay-ps", &val)) { - val = val / 1000; /* convert to ns */ + val = DIV_ROUND_CLOSEST(val, 300); /* convert to 0.3 ns step */ - if (val == 0 || val == 2) - rx_delay = val * 2; + if (val <= 7) + rx_delay = val; else dev_warn(smi->dev, - "EXT port RX delay must be 0 to 2 ns\n"); + "EXT port RX delay must be 0 to 2.1 ns\n"); } ret = regmap_update_bits( diff --git a/drivers/net/dsa/rtl8366rb.c b/drivers/net/dsa/rtl8366rb.c index 03deacd83e61..ecc19bd5115f 100644 --- a/drivers/net/dsa/rtl8366rb.c +++ b/drivers/net/dsa/rtl8366rb.c @@ -1186,7 +1186,8 @@ rtl8366rb_port_disable(struct dsa_switch *ds, int port) static int rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { struct realtek_smi *smi = ds->priv; unsigned int port_bitmap = 0; @@ -1198,7 +1199,7 @@ rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, if (i == port) continue; /* Not on this bridge */ - if (dsa_to_port(ds, i)->bridge_dev != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Join this port to each other port on the bridge */ ret = regmap_update_bits(smi->map, RTL8366RB_PORT_ISO(i), @@ -1218,7 +1219,7 @@ rtl8366rb_port_bridge_join(struct dsa_switch *ds, int port, static void rtl8366rb_port_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { struct realtek_smi *smi = ds->priv; unsigned int port_bitmap = 0; @@ -1230,7 +1231,7 @@ rtl8366rb_port_bridge_leave(struct dsa_switch *ds, int port, if (i == port) continue; /* Not on this bridge */ - if (dsa_to_port(ds, i)->bridge_dev != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* Remove this port from any other port on the bridge */ ret = regmap_update_bits(smi->map, RTL8366RB_PORT_ISO(i), diff --git a/drivers/net/dsa/sja1105/sja1105_main.c b/drivers/net/dsa/sja1105/sja1105_main.c index c343effe2e96..cefde41ce8d6 100644 --- a/drivers/net/dsa/sja1105/sja1105_main.c +++ b/drivers/net/dsa/sja1105/sja1105_main.c @@ -118,13 +118,14 @@ static int sja1105_pvid_apply(struct sja1105_private *priv, int port, u16 pvid) static int sja1105_commit_pvid(struct dsa_switch *ds, int port) { struct dsa_port *dp = dsa_to_port(ds, port); + struct net_device *br = dsa_port_bridge_dev_get(dp); struct sja1105_private *priv = ds->priv; struct sja1105_vlan_lookup_entry *vlan; bool drop_untagged = false; int match, rc; u16 pvid; - if (dp->bridge_dev && br_vlan_enabled(dp->bridge_dev)) + if (br && br_vlan_enabled(br)) pvid = priv->bridge_pvid[port]; else pvid = priv->tag_8021q_pvid[port]; @@ -1979,7 +1980,7 @@ static int sja1105_manage_flood_domains(struct sja1105_private *priv) } static int sja1105_bridge_member(struct dsa_switch *ds, int port, - struct net_device *br, bool member) + struct dsa_bridge bridge, bool member) { struct sja1105_l2_forwarding_entry *l2_fwd; struct sja1105_private *priv = ds->priv; @@ -2004,7 +2005,7 @@ static int sja1105_bridge_member(struct dsa_switch *ds, int port, */ if (i == port) continue; - if (dsa_to_port(ds, i)->bridge_dev != br) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; sja1105_port_allow_traffic(l2_fwd, i, port, member); sja1105_port_allow_traffic(l2_fwd, port, i, member); @@ -2073,15 +2074,31 @@ static void sja1105_bridge_stp_state_set(struct dsa_switch *ds, int port, } static int sja1105_bridge_join(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge, + bool *tx_fwd_offload) { - return sja1105_bridge_member(ds, port, br, true); + int rc; + + rc = sja1105_bridge_member(ds, port, bridge, true); + if (rc) + return rc; + + rc = dsa_tag_8021q_bridge_tx_fwd_offload(ds, port, bridge); + if (rc) { + sja1105_bridge_member(ds, port, bridge, false); + return rc; + } + + *tx_fwd_offload = true; + + return 0; } static void sja1105_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *br) + struct dsa_bridge bridge) { - sja1105_bridge_member(ds, port, br, false); + dsa_tag_8021q_bridge_tx_fwd_unoffload(ds, port, bridge); + sja1105_bridge_member(ds, port, bridge, false); } #define BYTES_PER_KBIT (1000LL / 8) @@ -2587,8 +2604,9 @@ static int sja1105_prechangeupper(struct dsa_switch *ds, int port, if (netif_is_bridge_master(upper)) { list_for_each_entry(dp, &dst->ports, list) { - if (dp->bridge_dev && dp->bridge_dev != upper && - br_vlan_enabled(dp->bridge_dev)) { + struct net_device *br = dsa_port_bridge_dev_get(dp); + + if (br && br != upper && br_vlan_enabled(br)) { NL_SET_ERR_MSG_MOD(extack, "Only one VLAN-aware bridge is supported"); return -EBUSY; @@ -3139,7 +3157,7 @@ static int sja1105_setup(struct dsa_switch *ds) ds->vlan_filtering_is_global = true; ds->untag_bridge_pvid = true; /* tag_8021q has 3 bits for the VBID, and the value 0 is reserved */ - ds->num_fwd_offloading_bridges = 7; + ds->max_num_bridges = 7; /* Advertise the 8 egress queues */ ds->num_tx_queues = SJA1105_NUM_TC; @@ -3228,8 +3246,6 @@ static const struct dsa_switch_ops sja1105_switch_ops = { .tag_8021q_vlan_add = sja1105_dsa_8021q_vlan_add, .tag_8021q_vlan_del = sja1105_dsa_8021q_vlan_del, .port_prechangeupper = sja1105_prechangeupper, - .port_bridge_tx_fwd_offload = dsa_tag_8021q_bridge_tx_fwd_offload, - .port_bridge_tx_fwd_unoffload = dsa_tag_8021q_bridge_tx_fwd_unoffload, }; static const struct of_device_id sja1105_dt_ids[]; diff --git a/drivers/net/dsa/vitesse-vsc73xx-core.c b/drivers/net/dsa/vitesse-vsc73xx-core.c index a4b1447ff055..ae55167ce0a6 100644 --- a/drivers/net/dsa/vitesse-vsc73xx-core.c +++ b/drivers/net/dsa/vitesse-vsc73xx-core.c @@ -1122,9 +1122,6 @@ static int vsc73xx_gpio_probe(struct vsc73xx *vsc) vsc->gc.ngpio = 4; vsc->gc.owner = THIS_MODULE; vsc->gc.parent = vsc->dev; -#if IS_ENABLED(CONFIG_OF_GPIO) - vsc->gc.of_node = vsc->dev->of_node; -#endif vsc->gc.base = -1; vsc->gc.get = vsc73xx_gpio_get; vsc->gc.set = vsc73xx_gpio_set; @@ -1216,12 +1213,10 @@ int vsc73xx_probe(struct vsc73xx *vsc) } EXPORT_SYMBOL(vsc73xx_probe); -int vsc73xx_remove(struct vsc73xx *vsc) +void vsc73xx_remove(struct vsc73xx *vsc) { dsa_unregister_switch(vsc->ds); gpiod_set_value(vsc->reset, 1); - - return 0; } EXPORT_SYMBOL(vsc73xx_remove); diff --git a/drivers/net/dsa/vitesse-vsc73xx.h b/drivers/net/dsa/vitesse-vsc73xx.h index 30b951504e65..30b1f0a36566 100644 --- a/drivers/net/dsa/vitesse-vsc73xx.h +++ b/drivers/net/dsa/vitesse-vsc73xx.h @@ -26,5 +26,5 @@ struct vsc73xx_ops { int vsc73xx_is_addr_valid(u8 block, u8 subblock); int vsc73xx_probe(struct vsc73xx *vsc); -int vsc73xx_remove(struct vsc73xx *vsc); +void vsc73xx_remove(struct vsc73xx *vsc); void vsc73xx_shutdown(struct vsc73xx *vsc); diff --git a/drivers/net/dsa/xrs700x/xrs700x.c b/drivers/net/dsa/xrs700x/xrs700x.c index 910fcb3b252b..35fa19ddaf19 100644 --- a/drivers/net/dsa/xrs700x/xrs700x.c +++ b/drivers/net/dsa/xrs700x/xrs700x.c @@ -501,7 +501,7 @@ static void xrs700x_mac_link_up(struct dsa_switch *ds, int port, } static int xrs700x_bridge_common(struct dsa_switch *ds, int port, - struct net_device *bridge, bool join) + struct dsa_bridge bridge, bool join) { unsigned int i, cpu_mask = 0, mask = 0; struct xrs700x *priv = ds->priv; @@ -513,14 +513,14 @@ static int xrs700x_bridge_common(struct dsa_switch *ds, int port, cpu_mask |= BIT(i); - if (dsa_to_port(ds, i)->bridge_dev == bridge) + if (dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; mask |= BIT(i); } for (i = 0; i < ds->num_ports; i++) { - if (dsa_to_port(ds, i)->bridge_dev != bridge) + if (!dsa_port_offloads_bridge(dsa_to_port(ds, i), &bridge)) continue; /* 1 = Disable forwarding to the port */ @@ -540,13 +540,13 @@ static int xrs700x_bridge_common(struct dsa_switch *ds, int port, } static int xrs700x_bridge_join(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge, bool *tx_fwd_offload) { return xrs700x_bridge_common(ds, port, bridge, true); } static void xrs700x_bridge_leave(struct dsa_switch *ds, int port, - struct net_device *bridge) + struct dsa_bridge bridge) { xrs700x_bridge_common(ds, port, bridge, false); } diff --git a/drivers/net/eql.c b/drivers/net/eql.c index 8ef34901c2d8..1111d1f33865 100644 --- a/drivers/net/eql.c +++ b/drivers/net/eql.c @@ -225,7 +225,7 @@ static void eql_kill_one_slave(slave_queue_t *queue, slave_t *slave) list_del(&slave->list); queue->num_slaves--; slave->dev->flags &= ~IFF_SLAVE; - dev_put(slave->dev); + dev_put_track(slave->dev, &slave->dev_tracker); kfree(slave); } @@ -399,7 +399,7 @@ static int __eql_insert_slave(slave_queue_t *queue, slave_t *slave) if (duplicate_slave) eql_kill_one_slave(queue, duplicate_slave); - dev_hold(slave->dev); + dev_hold_track(slave->dev, &slave->dev_tracker, GFP_ATOMIC); list_add(&slave->list, &queue->all_slaves); queue->num_slaves++; slave->dev->flags |= IFF_SLAVE; diff --git a/drivers/net/ethernet/3com/typhoon.c b/drivers/net/ethernet/3com/typhoon.c index 05e15b6e5e2c..481f1df3106c 100644 --- a/drivers/net/ethernet/3com/typhoon.c +++ b/drivers/net/ethernet/3com/typhoon.c @@ -1138,7 +1138,9 @@ typhoon_set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) } static void -typhoon_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +typhoon_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { ering->rx_max_pending = RXENT_ENTRIES; ering->tx_max_pending = TXLO_ENTRIES - 1; diff --git a/drivers/net/ethernet/8390/hydra.c b/drivers/net/ethernet/8390/hydra.c index 941754ea78ec..1df7601af86a 100644 --- a/drivers/net/ethernet/8390/hydra.c +++ b/drivers/net/ethernet/8390/hydra.c @@ -116,6 +116,7 @@ static int hydra_init(struct zorro_dev *z) unsigned long ioaddr = board+HYDRA_NIC_BASE; const char name[] = "NE2000"; int start_page, stop_page; + u8 macaddr[ETH_ALEN]; int j; int err; @@ -129,7 +130,8 @@ static int hydra_init(struct zorro_dev *z) return -ENOMEM; for (j = 0; j < ETH_ALEN; j++) - dev->dev_addr[j] = *((u8 *)(board + HYDRA_ADDRPROM + 2*j)); + macaddr[j] = *((u8 *)(board + HYDRA_ADDRPROM + 2*j)); + eth_hw_addr_set(dev, macaddr); /* We must set the 8390 for word mode. */ z_writeb(0x4b, ioaddr + NE_EN0_DCFG); diff --git a/drivers/net/ethernet/8390/mac8390.c b/drivers/net/ethernet/8390/mac8390.c index 91b04abfd687..7fb819b9b89a 100644 --- a/drivers/net/ethernet/8390/mac8390.c +++ b/drivers/net/ethernet/8390/mac8390.c @@ -292,6 +292,7 @@ static bool mac8390_rsrc_init(struct net_device *dev, struct nubus_dirent ent; int offset; volatile unsigned short *i; + u8 addr[ETH_ALEN]; dev->irq = SLOT2IRQ(board->slot); /* This is getting to be a habit */ @@ -314,7 +315,8 @@ static bool mac8390_rsrc_init(struct net_device *dev, return false; } - nubus_get_rsrc_mem(dev->dev_addr, &ent, 6); + nubus_get_rsrc_mem(addr, &ent, 6); + eth_hw_addr_set(dev, addr); if (useresources[cardtype] == 1) { nubus_rewinddir(&dir); diff --git a/drivers/net/ethernet/8390/smc-ultra.c b/drivers/net/ethernet/8390/smc-ultra.c index 0890fa493f70..6e62c37c9400 100644 --- a/drivers/net/ethernet/8390/smc-ultra.c +++ b/drivers/net/ethernet/8390/smc-ultra.c @@ -204,6 +204,7 @@ static int __init ultra_probe1(struct net_device *dev, int ioaddr) { int i, retval; int checksum = 0; + u8 macaddr[ETH_ALEN]; const char *model_name; unsigned char eeprom_irq = 0; static unsigned version_printed; @@ -239,7 +240,8 @@ static int __init ultra_probe1(struct net_device *dev, int ioaddr) model_name = (idreg & 0xF0) == 0x20 ? "SMC Ultra" : "SMC EtherEZ"; for (i = 0; i < 6; i++) - dev->dev_addr[i] = inb(ioaddr + 8 + i); + macaddr[i] = inb(ioaddr + 8 + i); + eth_hw_addr_set(dev, macaddr); netdev_info(dev, "%s at %#3x, %pM", model_name, ioaddr, dev->dev_addr); diff --git a/drivers/net/ethernet/8390/wd.c b/drivers/net/ethernet/8390/wd.c index 263a942d81fa..5b00c452bede 100644 --- a/drivers/net/ethernet/8390/wd.c +++ b/drivers/net/ethernet/8390/wd.c @@ -168,6 +168,7 @@ static int __init wd_probe1(struct net_device *dev, int ioaddr) int checksum = 0; int ancient = 0; /* An old card without config registers. */ int word16 = 0; /* 0 = 8 bit, 1 = 16 bit */ + u8 addr[ETH_ALEN]; const char *model_name; static unsigned version_printed; struct ei_device *ei_local = netdev_priv(dev); @@ -191,7 +192,8 @@ static int __init wd_probe1(struct net_device *dev, int ioaddr) netdev_info(dev, version); for (i = 0; i < 6; i++) - dev->dev_addr[i] = inb(ioaddr + 8 + i); + addr[i] = inb(ioaddr + 8 + i); + eth_hw_addr_set(dev, addr); netdev_info(dev, "WD80x3 at %#3x, %pM", ioaddr, dev->dev_addr); diff --git a/drivers/net/ethernet/Kconfig b/drivers/net/ethernet/Kconfig index 4601b38f532a..027cbacca1c9 100644 --- a/drivers/net/ethernet/Kconfig +++ b/drivers/net/ethernet/Kconfig @@ -73,6 +73,7 @@ config DNET source "drivers/net/ethernet/dec/Kconfig" source "drivers/net/ethernet/dlink/Kconfig" source "drivers/net/ethernet/emulex/Kconfig" +source "drivers/net/ethernet/engleder/Kconfig" source "drivers/net/ethernet/ezchip/Kconfig" source "drivers/net/ethernet/faraday/Kconfig" source "drivers/net/ethernet/freescale/Kconfig" diff --git a/drivers/net/ethernet/Makefile b/drivers/net/ethernet/Makefile index fdd8c6c17451..33d30b619e00 100644 --- a/drivers/net/ethernet/Makefile +++ b/drivers/net/ethernet/Makefile @@ -36,6 +36,7 @@ obj-$(CONFIG_DNET) += dnet.o obj-$(CONFIG_NET_VENDOR_DEC) += dec/ obj-$(CONFIG_NET_VENDOR_DLINK) += dlink/ obj-$(CONFIG_NET_VENDOR_EMULEX) += emulex/ +obj-$(CONFIG_NET_VENDOR_ENGLEDER) += engleder/ obj-$(CONFIG_NET_VENDOR_EZCHIP) += ezchip/ obj-$(CONFIG_NET_VENDOR_FARADAY) += faraday/ obj-$(CONFIG_NET_VENDOR_FREESCALE) += freescale/ diff --git a/drivers/net/ethernet/amazon/ena/ena_ethtool.c b/drivers/net/ethernet/amazon/ena/ena_ethtool.c index 13e745cf3781..6b9b43e422c1 100644 --- a/drivers/net/ethernet/amazon/ena/ena_ethtool.c +++ b/drivers/net/ethernet/amazon/ena/ena_ethtool.c @@ -465,7 +465,9 @@ static void ena_get_drvinfo(struct net_device *dev, } static void ena_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ena_adapter *adapter = netdev_priv(netdev); @@ -476,7 +478,9 @@ static void ena_get_ringparam(struct net_device *netdev, } static int ena_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ena_adapter *adapter = netdev_priv(netdev); u32 new_tx_size, new_rx_size; diff --git a/drivers/net/ethernet/amd/a2065.c b/drivers/net/ethernet/amd/a2065.c index 2f808dbc8b0e..3a351d3396bf 100644 --- a/drivers/net/ethernet/amd/a2065.c +++ b/drivers/net/ethernet/amd/a2065.c @@ -680,6 +680,7 @@ static int a2065_init_one(struct zorro_dev *z, unsigned long base_addr = board + A2065_LANCE; unsigned long mem_start = board + A2065_RAM; struct resource *r1, *r2; + u8 addr[ETH_ALEN]; u32 serial; int err; @@ -706,17 +707,18 @@ static int a2065_init_one(struct zorro_dev *z, r2->name = dev->name; serial = be32_to_cpu(z->rom.er_SerialNumber); - dev->dev_addr[0] = 0x00; + addr[0] = 0x00; if (z->id != ZORRO_PROD_AMERISTAR_A2065) { /* Commodore */ - dev->dev_addr[1] = 0x80; - dev->dev_addr[2] = 0x10; + addr[1] = 0x80; + addr[2] = 0x10; } else { /* Ameristar */ - dev->dev_addr[1] = 0x00; - dev->dev_addr[2] = 0x9f; + addr[1] = 0x00; + addr[2] = 0x9f; } - dev->dev_addr[3] = (serial >> 16) & 0xff; - dev->dev_addr[4] = (serial >> 8) & 0xff; - dev->dev_addr[5] = serial & 0xff; + addr[3] = (serial >> 16) & 0xff; + addr[4] = (serial >> 8) & 0xff; + addr[5] = serial & 0xff; + eth_hw_addr_set(dev, addr); dev->base_addr = (unsigned long)ZTWO_VADDR(base_addr); dev->mem_start = (unsigned long)ZTWO_VADDR(mem_start); dev->mem_end = dev->mem_start + A2065_RAM_SIZE; diff --git a/drivers/net/ethernet/amd/ariadne.c b/drivers/net/ethernet/amd/ariadne.c index 5e0f645f5bde..4ea7b9f3c424 100644 --- a/drivers/net/ethernet/amd/ariadne.c +++ b/drivers/net/ethernet/amd/ariadne.c @@ -441,11 +441,11 @@ static int ariadne_open(struct net_device *dev) /* Set the Ethernet Hardware Address */ lance->RAP = CSR12; /* Physical Address Register, PADR[15:0] */ - lance->RDP = ((u_short *)&dev->dev_addr[0])[0]; + lance->RDP = ((const u_short *)&dev->dev_addr[0])[0]; lance->RAP = CSR13; /* Physical Address Register, PADR[31:16] */ - lance->RDP = ((u_short *)&dev->dev_addr[0])[1]; + lance->RDP = ((const u_short *)&dev->dev_addr[0])[1]; lance->RAP = CSR14; /* Physical Address Register, PADR[47:32] */ - lance->RDP = ((u_short *)&dev->dev_addr[0])[2]; + lance->RDP = ((const u_short *)&dev->dev_addr[0])[2]; /* Set the Init Block Mode */ lance->RAP = CSR15; /* Mode Register */ @@ -717,6 +717,7 @@ static int ariadne_init_one(struct zorro_dev *z, unsigned long mem_start = board + ARIADNE_RAM; struct resource *r1, *r2; struct net_device *dev; + u8 addr[ETH_ALEN]; u32 serial; int err; @@ -740,12 +741,13 @@ static int ariadne_init_one(struct zorro_dev *z, r2->name = dev->name; serial = be32_to_cpu(z->rom.er_SerialNumber); - dev->dev_addr[0] = 0x00; - dev->dev_addr[1] = 0x60; - dev->dev_addr[2] = 0x30; - dev->dev_addr[3] = (serial >> 16) & 0xff; - dev->dev_addr[4] = (serial >> 8) & 0xff; - dev->dev_addr[5] = serial & 0xff; + addr[0] = 0x00; + addr[1] = 0x60; + addr[2] = 0x30; + addr[3] = (serial >> 16) & 0xff; + addr[4] = (serial >> 8) & 0xff; + addr[5] = serial & 0xff; + eth_hw_addr_set(dev, addr); dev->base_addr = (unsigned long)ZTWO_VADDR(base_addr); dev->mem_start = (unsigned long)ZTWO_VADDR(mem_start); dev->mem_end = dev->mem_start + ARIADNE_RAM_SIZE; diff --git a/drivers/net/ethernet/amd/atarilance.c b/drivers/net/ethernet/amd/atarilance.c index 9c7d9690d00c..27869164c6e6 100644 --- a/drivers/net/ethernet/amd/atarilance.c +++ b/drivers/net/ethernet/amd/atarilance.c @@ -471,6 +471,7 @@ static unsigned long __init lance_probe1( struct net_device *dev, int i; static int did_version; unsigned short save1, save2; + u8 addr[ETH_ALEN]; PROBE_PRINT(( "Probing for Lance card at mem %#lx io %#lx\n", (long)memaddr, (long)ioaddr )); @@ -585,14 +586,16 @@ static unsigned long __init lance_probe1( struct net_device *dev, eth_hw_addr_set(dev, OldRieblDefHwaddr); break; case NEW_RIEBL: - lp->memcpy_f(dev->dev_addr, RIEBL_HWADDR_ADDR, ETH_ALEN); + lp->memcpy_f(addr, RIEBL_HWADDR_ADDR, ETH_ALEN); + eth_hw_addr_set(dev, addr); break; case PAM_CARD: i = IO->eeprom; for( i = 0; i < 6; ++i ) - dev->dev_addr[i] = + addr[i] = ((((unsigned short *)MEM)[i*2] & 0x0f) << 4) | ((((unsigned short *)MEM)[i*2+1] & 0x0f)); + eth_hw_addr_set(dev, addr); i = IO->mem; break; } diff --git a/drivers/net/ethernet/amd/hplance.c b/drivers/net/ethernet/amd/hplance.c index 6784f8748638..055fda11c572 100644 --- a/drivers/net/ethernet/amd/hplance.c +++ b/drivers/net/ethernet/amd/hplance.c @@ -129,6 +129,7 @@ static void hplance_init(struct net_device *dev, struct dio_dev *d) { unsigned long va = (d->resource.start + DIO_VIRADDRBASE); struct hplance_private *lp; + u8 addr[ETH_ALEN]; int i; /* reset the board */ @@ -144,9 +145,10 @@ static void hplance_init(struct net_device *dev, struct dio_dev *d) /* The NVRAM holds our ethernet address, one nibble per byte, * at bytes NVRAMOFF+1,3,5,7,9... */ - dev->dev_addr[i] = ((in_8(va + HPLANCE_NVRAMOFF + i*4 + 1) & 0xF) << 4) + addr[i] = ((in_8(va + HPLANCE_NVRAMOFF + i*4 + 1) & 0xF) << 4) | (in_8(va + HPLANCE_NVRAMOFF + i*4 + 3) & 0xF); } + eth_hw_addr_set(dev, addr); lp = netdev_priv(dev); lp->lance.name = d->name; diff --git a/drivers/net/ethernet/amd/lance.c b/drivers/net/ethernet/amd/lance.c index 945bf1d87507..462016666752 100644 --- a/drivers/net/ethernet/amd/lance.c +++ b/drivers/net/ethernet/amd/lance.c @@ -480,6 +480,7 @@ static int __init lance_probe1(struct net_device *dev, int ioaddr, int irq, int unsigned long flags; int err = -ENOMEM; void __iomem *bios; + u8 addr[ETH_ALEN]; /* First we look for special cases. Check for HP's on-board ethernet by looking for 'HP' in the BIOS. @@ -541,7 +542,8 @@ static int __init lance_probe1(struct net_device *dev, int ioaddr, int irq, int /* There is a 16 byte station address PROM at the base address. The first six bytes are the station address. */ for (i = 0; i < 6; i++) - dev->dev_addr[i] = inb(ioaddr + i); + addr[i] = inb(ioaddr + i); + eth_hw_addr_set(dev, addr); printk("%pM", dev->dev_addr); dev->base_addr = ioaddr; diff --git a/drivers/net/ethernet/amd/mvme147.c b/drivers/net/ethernet/amd/mvme147.c index da97fccea9ea..410c7b67eba4 100644 --- a/drivers/net/ethernet/amd/mvme147.c +++ b/drivers/net/ethernet/amd/mvme147.c @@ -74,6 +74,7 @@ static struct net_device * __init mvme147lance_probe(void) static int called; static const char name[] = "MVME147 LANCE"; struct m147lance_private *lp; + u8 macaddr[ETH_ALEN]; u_long *addr; u_long address; int err; @@ -93,15 +94,16 @@ static struct net_device * __init mvme147lance_probe(void) addr = (u_long *)ETHERNET_ADDRESS; address = *addr; - dev->dev_addr[0] = 0x08; - dev->dev_addr[1] = 0x00; - dev->dev_addr[2] = 0x3e; + macaddr[0] = 0x08; + macaddr[1] = 0x00; + macaddr[2] = 0x3e; address = address >> 8; - dev->dev_addr[5] = address&0xff; + macaddr[5] = address&0xff; address = address >> 8; - dev->dev_addr[4] = address&0xff; + macaddr[4] = address&0xff; address = address >> 8; - dev->dev_addr[3] = address&0xff; + macaddr[3] = address&0xff; + eth_hw_addr_set(dev, macaddr); printk("%s: MVME147 at 0x%08lx, irq %d, Hardware Address %pM\n", dev->name, dev->base_addr, MVME147_LANCE_IRQ, diff --git a/drivers/net/ethernet/amd/ni65.c b/drivers/net/ethernet/amd/ni65.c index 032e8922b482..8ba579b89b75 100644 --- a/drivers/net/ethernet/amd/ni65.c +++ b/drivers/net/ethernet/amd/ni65.c @@ -251,7 +251,7 @@ static void ni65_recv_intr(struct net_device *dev,int); static void ni65_xmit_intr(struct net_device *dev,int); static int ni65_open(struct net_device *dev); static int ni65_lance_reinit(struct net_device *dev); -static void ni65_init_lance(struct priv *p,unsigned char*,int,int); +static void ni65_init_lance(struct priv *p,const unsigned char*,int,int); static netdev_tx_t ni65_send_packet(struct sk_buff *skb, struct net_device *dev); static void ni65_timeout(struct net_device *dev, unsigned int txqueue); @@ -418,6 +418,7 @@ static int __init ni65_probe1(struct net_device *dev,int ioaddr) { int i,j; struct priv *p; + u8 addr[ETH_ALEN]; unsigned long flags; dev->irq = irq; @@ -444,7 +445,8 @@ static int __init ni65_probe1(struct net_device *dev,int ioaddr) return -ENODEV; for(j=0;j<6;j++) - dev->dev_addr[j] = inb(ioaddr+cards[i].addr_offset+j); + addr[j] = inb(ioaddr+cards[i].addr_offset+j); + eth_hw_addr_set(dev, addr); if( (j=ni65_alloc_buffer(dev)) < 0) { release_region(ioaddr, cards[i].total_size); @@ -566,7 +568,7 @@ static int __init ni65_probe1(struct net_device *dev,int ioaddr) /* * set lance register and trigger init */ -static void ni65_init_lance(struct priv *p,unsigned char *daddr,int filter,int mode) +static void ni65_init_lance(struct priv *p,const unsigned char *daddr,int filter,int mode) { int i; u32 pib; diff --git a/drivers/net/ethernet/amd/pcnet32.c b/drivers/net/ethernet/amd/pcnet32.c index f5c50ff377ff..c20c369c7eb8 100644 --- a/drivers/net/ethernet/amd/pcnet32.c +++ b/drivers/net/ethernet/amd/pcnet32.c @@ -860,7 +860,9 @@ static int pcnet32_nway_reset(struct net_device *dev) } static void pcnet32_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pcnet32_private *lp = netdev_priv(dev); @@ -871,7 +873,9 @@ static void pcnet32_get_ringparam(struct net_device *dev, } static int pcnet32_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pcnet32_private *lp = netdev_priv(dev); unsigned long flags; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 94879cf8b420..6ceb1cdf6eba 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -619,8 +619,11 @@ static int xgbe_get_module_eeprom(struct net_device *netdev, return pdata->phy_if.module_eeprom(pdata, eeprom, data); } -static void xgbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) +static void +xgbe_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct xgbe_prv_data *pdata = netdev_priv(netdev); @@ -631,7 +634,9 @@ static void xgbe_get_ringparam(struct net_device *netdev, } static int xgbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct xgbe_prv_data *pdata = netdev_priv(netdev); unsigned int rx, tx; diff --git a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c index 220dc42af31a..ff2d099aab21 100644 --- a/drivers/net/ethernet/apm/xgene/xgene_enet_main.c +++ b/drivers/net/ethernet/apm/xgene/xgene_enet_main.c @@ -869,7 +869,7 @@ static void xgene_enet_timeout(struct net_device *ndev, unsigned int txqueue) for (i = 0; i < pdata->txq_cnt; i++) { txq = netdev_get_tx_queue(ndev, i); - txq->trans_start = jiffies; + txq_trans_cond_update(txq); netif_tx_start_queue(txq); } } diff --git a/drivers/net/ethernet/apple/macmace.c b/drivers/net/ethernet/apple/macmace.c index 95d3061c61be..8fcaf1639920 100644 --- a/drivers/net/ethernet/apple/macmace.c +++ b/drivers/net/ethernet/apple/macmace.c @@ -92,7 +92,7 @@ static void mace_reset(struct net_device *dev); static irqreturn_t mace_interrupt(int irq, void *dev_id); static irqreturn_t mace_dma_intr(int irq, void *dev_id); static void mace_tx_timeout(struct net_device *dev, unsigned int txqueue); -static void __mace_set_address(struct net_device *dev, void *addr); +static void __mace_set_address(struct net_device *dev, const void *addr); /* * Load a receive DMA channel with a base address and ring length @@ -197,6 +197,7 @@ static int mace_probe(struct platform_device *pdev) unsigned char *addr; struct net_device *dev; unsigned char checksum = 0; + u8 macaddr[ETH_ALEN]; int err; dev = alloc_etherdev(PRIV_BYTES); @@ -229,8 +230,9 @@ static int mace_probe(struct platform_device *pdev) for (j = 0; j < 6; ++j) { u8 v = bitrev8(addr[j<<4]); checksum ^= v; - dev->dev_addr[j] = v; + macaddr[j] = v; } + eth_hw_addr_set(dev, macaddr); for (; j < 8; ++j) { checksum ^= bitrev8(addr[j<<4]); } @@ -315,11 +317,12 @@ static void mace_reset(struct net_device *dev) * Load the address on a mace controller. */ -static void __mace_set_address(struct net_device *dev, void *addr) +static void __mace_set_address(struct net_device *dev, const void *addr) { struct mace_data *mp = netdev_priv(dev); volatile struct mace *mb = mp->mace; - unsigned char *p = addr; + const unsigned char *p = addr; + u8 macaddr[ETH_ALEN]; int i; /* load up the hardware address */ @@ -331,7 +334,8 @@ static void __mace_set_address(struct net_device *dev, void *addr) ; } for (i = 0; i < 6; ++i) - mb->padr = dev->dev_addr[i] = p[i]; + mb->padr = macaddr[i] = p[i]; + eth_hw_addr_set(dev, macaddr); if (mp->chipid != BROKEN_ADDRCHG_REV) mb->iac = 0; } diff --git a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c index a9ef0544e30f..a418238f6309 100644 --- a/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c +++ b/drivers/net/ethernet/aquantia/atlantic/aq_ethtool.c @@ -812,7 +812,9 @@ static int aq_ethtool_set_pauseparam(struct net_device *ndev, } static void aq_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct aq_nic_s *aq_nic = netdev_priv(ndev); struct aq_nic_cfg_s *cfg; @@ -827,7 +829,9 @@ static void aq_get_ringparam(struct net_device *ndev, } static int aq_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct aq_nic_s *aq_nic = netdev_priv(ndev); const struct aq_hw_caps_s *hw_caps; diff --git a/drivers/net/ethernet/asix/ax88796c_main.c b/drivers/net/ethernet/asix/ax88796c_main.c index e230d8d0ff73..e7a9f9863258 100644 --- a/drivers/net/ethernet/asix/ax88796c_main.c +++ b/drivers/net/ethernet/asix/ax88796c_main.c @@ -144,12 +144,13 @@ static void ax88796c_set_mac_addr(struct net_device *ndev) static void ax88796c_load_mac_addr(struct net_device *ndev) { struct ax88796c_device *ax_local = to_ax88796c_device(ndev); + u8 addr[ETH_ALEN]; u16 temp; lockdep_assert_held(&ax_local->spi_lock); /* Try the device tree first */ - if (!eth_platform_get_mac_address(&ax_local->spi->dev, ndev->dev_addr) && + if (!platform_get_ethdev_address(&ax_local->spi->dev, ndev) && is_valid_ether_addr(ndev->dev_addr)) { if (netif_msg_probe(ax_local)) dev_info(&ax_local->spi->dev, @@ -159,18 +160,19 @@ static void ax88796c_load_mac_addr(struct net_device *ndev) /* Read the MAC address from AX88796C */ temp = AX_READ(&ax_local->ax_spi, P3_MACASR0); - ndev->dev_addr[5] = (u8)temp; - ndev->dev_addr[4] = (u8)(temp >> 8); + addr[5] = (u8)temp; + addr[4] = (u8)(temp >> 8); temp = AX_READ(&ax_local->ax_spi, P3_MACASR1); - ndev->dev_addr[3] = (u8)temp; - ndev->dev_addr[2] = (u8)(temp >> 8); + addr[3] = (u8)temp; + addr[2] = (u8)(temp >> 8); temp = AX_READ(&ax_local->ax_spi, P3_MACASR2); - ndev->dev_addr[1] = (u8)temp; - ndev->dev_addr[0] = (u8)(temp >> 8); + addr[1] = (u8)temp; + addr[0] = (u8)(temp >> 8); - if (is_valid_ether_addr(ndev->dev_addr)) { + if (is_valid_ether_addr(addr)) { + eth_hw_addr_set(ndev, addr); if (netif_msg_probe(ax_local)) dev_info(&ax_local->spi->dev, "MAC address read from ASIX chip\n"); diff --git a/drivers/net/ethernet/atheros/ag71xx.c b/drivers/net/ethernet/atheros/ag71xx.c index 88d2ab748399..270c2935591b 100644 --- a/drivers/net/ethernet/atheros/ag71xx.c +++ b/drivers/net/ethernet/atheros/ag71xx.c @@ -766,7 +766,7 @@ static bool ag71xx_check_dma_stuck(struct ag71xx *ag) unsigned long timestamp; u32 rx_sm, tx_sm, rx_fd; - timestamp = netdev_get_tx_queue(ag->ndev, 0)->trans_start; + timestamp = READ_ONCE(netdev_get_tx_queue(ag->ndev, 0)->trans_start); if (likely(time_before(jiffies, timestamp + HZ / 10))) return false; @@ -1024,83 +1024,6 @@ static void ag71xx_mac_config(struct phylink_config *config, unsigned int mode, ag71xx_wr(ag, AG71XX_REG_FIFO_CFG3, ag->fifodata[2]); } -static void ag71xx_mac_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct ag71xx *ag = netdev_priv(to_net_dev(config->dev)); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - switch (state->interface) { - case PHY_INTERFACE_MODE_NA: - break; - case PHY_INTERFACE_MODE_MII: - if ((ag71xx_is(ag, AR9330) && ag->mac_idx == 0) || - ag71xx_is(ag, AR9340) || - ag71xx_is(ag, QCA9530) || - (ag71xx_is(ag, QCA9550) && ag->mac_idx == 1)) - break; - goto unsupported; - case PHY_INTERFACE_MODE_GMII: - if ((ag71xx_is(ag, AR9330) && ag->mac_idx == 1) || - (ag71xx_is(ag, AR9340) && ag->mac_idx == 1) || - (ag71xx_is(ag, QCA9530) && ag->mac_idx == 1)) - break; - goto unsupported; - case PHY_INTERFACE_MODE_SGMII: - if (ag71xx_is(ag, QCA9550) && ag->mac_idx == 0) - break; - goto unsupported; - case PHY_INTERFACE_MODE_RMII: - if (ag71xx_is(ag, AR9340) && ag->mac_idx == 0) - break; - goto unsupported; - case PHY_INTERFACE_MODE_RGMII: - if ((ag71xx_is(ag, AR9340) && ag->mac_idx == 0) || - (ag71xx_is(ag, QCA9550) && ag->mac_idx == 1)) - break; - goto unsupported; - default: - goto unsupported; - } - - phylink_set(mask, MII); - - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - phylink_set(mask, Autoneg); - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - - if (state->interface == PHY_INTERFACE_MODE_NA || - state->interface == PHY_INTERFACE_MODE_SGMII || - state->interface == PHY_INTERFACE_MODE_RGMII || - state->interface == PHY_INTERFACE_MODE_GMII) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); - - return; -unsupported: - linkmode_zero(supported); -} - -static void ag71xx_mac_pcs_get_state(struct phylink_config *config, - struct phylink_link_state *state) -{ - state->link = 0; -} - -static void ag71xx_mac_an_restart(struct phylink_config *config) -{ - /* Not Supported */ -} - static void ag71xx_mac_link_down(struct phylink_config *config, unsigned int mode, phy_interface_t interface) { @@ -1163,9 +1086,7 @@ static void ag71xx_mac_link_up(struct phylink_config *config, } static const struct phylink_mac_ops ag71xx_phylink_mac_ops = { - .validate = ag71xx_mac_validate, - .mac_pcs_get_state = ag71xx_mac_pcs_get_state, - .mac_an_restart = ag71xx_mac_an_restart, + .validate = phylink_generic_validate, .mac_config = ag71xx_mac_config, .mac_link_down = ag71xx_mac_link_down, .mac_link_up = ag71xx_mac_link_up, @@ -1177,6 +1098,34 @@ static int ag71xx_phylink_setup(struct ag71xx *ag) ag->phylink_config.dev = &ag->ndev->dev; ag->phylink_config.type = PHYLINK_NETDEV; + ag->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000FD; + + if ((ag71xx_is(ag, AR9330) && ag->mac_idx == 0) || + ag71xx_is(ag, AR9340) || + ag71xx_is(ag, QCA9530) || + (ag71xx_is(ag, QCA9550) && ag->mac_idx == 1)) + __set_bit(PHY_INTERFACE_MODE_MII, + ag->phylink_config.supported_interfaces); + + if ((ag71xx_is(ag, AR9330) && ag->mac_idx == 1) || + (ag71xx_is(ag, AR9340) && ag->mac_idx == 1) || + (ag71xx_is(ag, QCA9530) && ag->mac_idx == 1)) + __set_bit(PHY_INTERFACE_MODE_GMII, + ag->phylink_config.supported_interfaces); + + if (ag71xx_is(ag, QCA9550) && ag->mac_idx == 0) + __set_bit(PHY_INTERFACE_MODE_SGMII, + ag->phylink_config.supported_interfaces); + + if (ag71xx_is(ag, AR9340) && ag->mac_idx == 0) + __set_bit(PHY_INTERFACE_MODE_RMII, + ag->phylink_config.supported_interfaces); + + if ((ag71xx_is(ag, AR9340) && ag->mac_idx == 0) || + (ag71xx_is(ag, QCA9550) && ag->mac_idx == 1)) + __set_bit(PHY_INTERFACE_MODE_RGMII, + ag->phylink_config.supported_interfaces); phylink = phylink_create(&ag->phylink_config, ag->pdev->dev.fwnode, ag->phy_if_mode, &ag71xx_phylink_mac_ops); diff --git a/drivers/net/ethernet/atheros/atlx/atl1.c b/drivers/net/ethernet/atheros/atlx/atl1.c index b4c9e805e981..6a969969d221 100644 --- a/drivers/net/ethernet/atheros/atlx/atl1.c +++ b/drivers/net/ethernet/atheros/atlx/atl1.c @@ -3438,7 +3438,9 @@ static void atl1_get_regs(struct net_device *netdev, struct ethtool_regs *regs, } static void atl1_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct atl1_adapter *adapter = netdev_priv(netdev); struct atl1_tpd_ring *txdr = &adapter->tpd_ring; @@ -3451,7 +3453,9 @@ static void atl1_get_ringparam(struct net_device *netdev, } static int atl1_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct atl1_adapter *adapter = netdev_priv(netdev); struct atl1_tpd_ring *tpdr = &adapter->tpd_ring; diff --git a/drivers/net/ethernet/broadcom/b44.c b/drivers/net/ethernet/broadcom/b44.c index 969591bbc066..e5857e88c207 100644 --- a/drivers/net/ethernet/broadcom/b44.c +++ b/drivers/net/ethernet/broadcom/b44.c @@ -1961,7 +1961,9 @@ static int b44_set_link_ksettings(struct net_device *dev, } static void b44_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct b44 *bp = netdev_priv(dev); @@ -1972,7 +1974,9 @@ static void b44_get_ringparam(struct net_device *dev, } static int b44_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct b44 *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index a568994a03a6..b04e423c446a 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -1497,8 +1497,11 @@ static int bcm_enet_set_link_ksettings(struct net_device *dev, } } -static void bcm_enet_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +bcm_enet_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; @@ -1512,7 +1515,9 @@ static void bcm_enet_get_ringparam(struct net_device *dev, } static int bcm_enet_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; int was_running; @@ -2579,8 +2584,11 @@ static void bcm_enetsw_get_ethtool_stats(struct net_device *netdev, } } -static void bcm_enetsw_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +bcm_enetsw_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; @@ -2595,8 +2603,11 @@ static void bcm_enetsw_get_ringparam(struct net_device *dev, ering->tx_pending = priv->tx_ring_size; } -static int bcm_enetsw_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static int +bcm_enetsw_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bcm_enet_priv *priv; int was_running; diff --git a/drivers/net/ethernet/broadcom/bnx2.c b/drivers/net/ethernet/broadcom/bnx2.c index babc955ba64e..e20aafeb4ca9 100644 --- a/drivers/net/ethernet/broadcom/bnx2.c +++ b/drivers/net/ethernet/broadcom/bnx2.c @@ -7318,7 +7318,9 @@ static int bnx2_set_coalesce(struct net_device *dev, } static void -bnx2_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +bnx2_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2 *bp = netdev_priv(dev); @@ -7389,7 +7391,9 @@ bnx2_change_ring_size(struct bnx2 *bp, u32 rx, u32 tx, bool reset_irq) } static int -bnx2_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +bnx2_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2 *bp = netdev_priv(dev); int rc; diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index e8e8c2d593c5..54a2334dee56 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -25,6 +25,7 @@ #include <linux/ip.h> #include <linux/crash_dump.h> #include <net/tcp.h> +#include <net/gro.h> #include <net/ipv6.h> #include <net/ip6_checksum.h> #include <linux/prefetch.h> diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 472a3a478038..0e319ac7799f 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -1914,7 +1914,9 @@ static int bnx2x_set_coalesce(struct net_device *dev, } static void bnx2x_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2x *bp = netdev_priv(dev); @@ -1938,7 +1940,9 @@ static void bnx2x_get_ringparam(struct net_device *dev, } static int bnx2x_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnx2x *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h index 8c2cf5519787..2dac704dc346 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_sriov.h @@ -586,7 +586,7 @@ static inline int bnx2x_vfpf_release(struct bnx2x *bp) {return 0; } static inline int bnx2x_vfpf_init(struct bnx2x *bp) {return 0; } static inline void bnx2x_vfpf_close_vf(struct bnx2x *bp) {} static inline int bnx2x_vfpf_setup_q(struct bnx2x *bp, struct bnx2x_fastpath *fp, bool is_leading) {return 0; } -static inline int bnx2x_vfpf_config_mac(struct bnx2x *bp, u8 *addr, +static inline int bnx2x_vfpf_config_mac(struct bnx2x *bp, const u8 *addr, u8 vf_qid, bool set) {return 0; } static inline int bnx2x_vfpf_config_rss(struct bnx2x *bp, struct bnx2x_config_rss_params *params) {return 0; } diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.c index 0b193edb73b8..2bb133ae61c3 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.c @@ -849,7 +849,8 @@ static int bnx2x_hw_stats_update(struct bnx2x *bp) memcpy(old, new, sizeof(struct nig_stats)); - memcpy(&(estats->rx_stat_ifhcinbadoctets_hi), &(pstats->mac_stx[1]), + BUILD_BUG_ON(sizeof(estats->shared) != sizeof(pstats->mac_stx[1])); + memcpy(&(estats->shared), &(pstats->mac_stx[1]), sizeof(struct mac_stx)); estats->brb_drop_hi = pstats->brb_drop_hi; estats->brb_drop_lo = pstats->brb_drop_lo; @@ -1634,9 +1635,9 @@ void bnx2x_stats_init(struct bnx2x *bp) REG_RD(bp, NIG_REG_STAT0_BRB_TRUNCATE + port*0x38); if (!CHIP_IS_E3(bp)) { REG_RD_DMAE(bp, NIG_REG_STAT0_EGRESS_MAC_PKT0 + port*0x50, - &(bp->port.old_nig_stats.egress_mac_pkt0_lo), 2); + &(bp->port.old_nig_stats.egress_mac_pkt0), 2); REG_RD_DMAE(bp, NIG_REG_STAT0_EGRESS_MAC_PKT1 + port*0x50, - &(bp->port.old_nig_stats.egress_mac_pkt1_lo), 2); + &(bp->port.old_nig_stats.egress_mac_pkt1), 2); } /* Prepare statistics ramrod data */ diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h index d55e63692cf3..ae93c078707b 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h @@ -36,10 +36,14 @@ struct nig_stats { u32 pbf_octets; u32 pbf_packet; u32 safc_inp; - u32 egress_mac_pkt0_lo; - u32 egress_mac_pkt0_hi; - u32 egress_mac_pkt1_lo; - u32 egress_mac_pkt1_hi; + struct_group(egress_mac_pkt0, + u32 egress_mac_pkt0_lo; + u32 egress_mac_pkt0_hi; + ); + struct_group(egress_mac_pkt1, + u32 egress_mac_pkt1_lo; + u32 egress_mac_pkt1_hi; + ); }; enum bnx2x_stats_event { @@ -83,6 +87,7 @@ struct bnx2x_eth_stats { u32 no_buff_discard_hi; u32 no_buff_discard_lo; + struct_group(shared, u32 rx_stat_ifhcinbadoctets_hi; u32 rx_stat_ifhcinbadoctets_lo; u32 tx_stat_ifhcoutbadoctets_hi; @@ -159,6 +164,7 @@ struct bnx2x_eth_stats { u32 tx_stat_dot3statsinternalmactransmiterrors_lo; u32 tx_stat_bmac_ufl_hi; u32 tx_stat_bmac_ufl_lo; + ); u32 pause_frames_received_hi; u32 pause_frames_received_lo; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index c04ea83188e2..c057b1df86a9 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -37,6 +37,7 @@ #include <linux/if_bridge.h> #include <linux/rtc.h> #include <linux/bpf.h> +#include <net/gro.h> #include <net/ip.h> #include <net/tcp.h> #include <net/udp.h> diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index 8188d55722e4..15253396096a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -775,7 +775,9 @@ skip_tpa_stats: } static void bnxt_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnxt *bp = netdev_priv(dev); @@ -794,7 +796,9 @@ static void bnxt_get_ringparam(struct net_device *dev, } static int bnxt_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct bnxt *bp = netdev_priv(dev); diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 85ca3909859d..283f3c1f1195 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -12390,7 +12390,10 @@ static int tg3_nway_reset(struct net_device *dev) return r; } -static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +static void tg3_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct tg3 *tp = netdev_priv(dev); @@ -12411,7 +12414,10 @@ static void tg3_get_ringparam(struct net_device *dev, struct ethtool_ringparam * ering->tx_pending = tp->napi[0].tx_pending; } -static int tg3_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ering) +static int tg3_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct tg3 *tp = netdev_priv(dev); int i, irq_sync = 0, err = 0; diff --git a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c index 391b85f25141..4b4ad731897f 100644 --- a/drivers/net/ethernet/brocade/bna/bnad_ethtool.c +++ b/drivers/net/ethernet/brocade/bna/bnad_ethtool.c @@ -405,7 +405,9 @@ static int bnad_set_coalesce(struct net_device *netdev, static void bnad_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { struct bnad *bnad = netdev_priv(netdev); @@ -418,7 +420,9 @@ bnad_get_ringparam(struct net_device *netdev, static int bnad_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ringparam) + struct ethtool_ringparam *ringparam, + struct kernel_ethtool_ringparam *kernel_ringparam, + struct netlink_ext_ack *extack) { int i, current_err, err = 0; struct bnad *bnad = netdev_priv(netdev); diff --git a/drivers/net/ethernet/cadence/macb_main.c b/drivers/net/ethernet/cadence/macb_main.c index ffce528aa00e..d4da9adf6777 100644 --- a/drivers/net/ethernet/cadence/macb_main.c +++ b/drivers/net/ethernet/cadence/macb_main.c @@ -506,74 +506,6 @@ static void macb_set_tx_clk(struct macb *bp, int speed) netdev_err(bp->dev, "adjusting tx_clk failed.\n"); } -static void macb_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - struct macb *bp = netdev_priv(ndev); - - /* We only support MII, RMII, GMII, RGMII & SGMII. */ - if (state->interface != PHY_INTERFACE_MODE_NA && - state->interface != PHY_INTERFACE_MODE_MII && - state->interface != PHY_INTERFACE_MODE_RMII && - state->interface != PHY_INTERFACE_MODE_GMII && - state->interface != PHY_INTERFACE_MODE_SGMII && - state->interface != PHY_INTERFACE_MODE_10GBASER && - !phy_interface_mode_is_rgmii(state->interface)) { - linkmode_zero(supported); - return; - } - - if (!macb_is_gem(bp) && - (state->interface == PHY_INTERFACE_MODE_GMII || - phy_interface_mode_is_rgmii(state->interface))) { - linkmode_zero(supported); - return; - } - - if (state->interface == PHY_INTERFACE_MODE_10GBASER && - !(bp->caps & MACB_CAPS_HIGH_SPEED && - bp->caps & MACB_CAPS_PCS)) { - linkmode_zero(supported); - return; - } - - phylink_set_port_modes(mask); - phylink_set(mask, Autoneg); - phylink_set(mask, Asym_Pause); - - if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && - (state->interface == PHY_INTERFACE_MODE_NA || - state->interface == PHY_INTERFACE_MODE_10GBASER)) { - phylink_set_10g_modes(mask); - phylink_set(mask, 10000baseKR_Full); - if (state->interface != PHY_INTERFACE_MODE_NA) - goto out; - } - - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - - if (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE && - (state->interface == PHY_INTERFACE_MODE_NA || - state->interface == PHY_INTERFACE_MODE_GMII || - state->interface == PHY_INTERFACE_MODE_SGMII || - phy_interface_mode_is_rgmii(state->interface))) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - - if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) - phylink_set(mask, 1000baseT_Half); - } -out: - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void macb_usx_pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex) @@ -815,7 +747,7 @@ static int macb_mac_prepare(struct phylink_config *config, unsigned int mode, } static const struct phylink_mac_ops macb_phylink_ops = { - .validate = macb_validate, + .validate = phylink_generic_validate, .mac_prepare = macb_mac_prepare, .mac_config = macb_mac_config, .mac_link_down = macb_mac_link_down, @@ -882,6 +814,35 @@ static int macb_mii_probe(struct net_device *dev) bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state; } + bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_10 | MAC_100; + + __set_bit(PHY_INTERFACE_MODE_MII, + bp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_RMII, + bp->phylink_config.supported_interfaces); + + /* Determine what modes are supported */ + if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) { + bp->phylink_config.mac_capabilities |= MAC_1000FD; + if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF)) + bp->phylink_config.mac_capabilities |= MAC_1000HD; + + __set_bit(PHY_INTERFACE_MODE_GMII, + bp->phylink_config.supported_interfaces); + phy_interface_set_rgmii(bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_PCS) + __set_bit(PHY_INTERFACE_MODE_SGMII, + bp->phylink_config.supported_interfaces); + + if (bp->caps & MACB_CAPS_HIGH_SPEED) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + bp->phylink_config.supported_interfaces); + bp->phylink_config.mac_capabilities |= MAC_10000FD; + } + } + bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode, bp->phy_interface, &macb_phylink_ops); if (IS_ERR(bp->phylink)) { @@ -3101,7 +3062,9 @@ static int macb_set_link_ksettings(struct net_device *netdev, } static void macb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); @@ -3113,7 +3076,9 @@ static void macb_get_ringparam(struct net_device *netdev, } static int macb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct macb *bp = netdev_priv(netdev); u32 new_rx_size, new_tx_size; diff --git a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c index 2b9747867d4c..2c10ae3f7fc1 100644 --- a/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c +++ b/drivers/net/ethernet/cavium/liquidio/lio_ethtool.c @@ -947,7 +947,9 @@ static int lio_set_phys_id(struct net_device *netdev, static void lio_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct lio *lio = GET_LIO(netdev); struct octeon_device *oct = lio->oct_dev; @@ -1252,8 +1254,11 @@ static int lio_reset_queues(struct net_device *netdev, uint32_t num_qs) return 0; } -static int lio_ethtool_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static int +lio_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { u32 rx_count, tx_count, rx_count_old, tx_count_old; struct lio *lio = GET_LIO(netdev); diff --git a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c index 4e39d712e121..4b4ffdd1044d 100644 --- a/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c +++ b/drivers/net/ethernet/cavium/octeon/octeon_mgmt.c @@ -548,7 +548,7 @@ struct octeon_mgmt_cam_state { }; static void octeon_mgmt_cam_state_add(struct octeon_mgmt_cam_state *cs, - unsigned char *addr) + const unsigned char *addr) { int i; diff --git a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c index 7f2882109b16..5a9fad61e9ea 100644 --- a/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c +++ b/drivers/net/ethernet/cavium/thunder/nicvf_ethtool.c @@ -467,7 +467,9 @@ static int nicvf_get_coalesce(struct net_device *netdev, } static void nicvf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nicvf *nic = netdev_priv(netdev); struct queue_set *qs = nic->qs; @@ -479,7 +481,9 @@ static void nicvf_get_ringparam(struct net_device *netdev, } static int nicvf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nicvf *nic = netdev_priv(netdev); struct queue_set *qs = nic->qs; diff --git a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c index 609820e214a3..18acd7cf3d6d 100644 --- a/drivers/net/ethernet/chelsio/cxgb/cxgb2.c +++ b/drivers/net/ethernet/chelsio/cxgb/cxgb2.c @@ -710,7 +710,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct adapter *adapter = dev->ml_priv; int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; @@ -724,7 +726,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = adapter->params.sge.cmdQ_size[0]; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct adapter *adapter = dev->ml_priv; int jumbo_fl = t1_is_T1B(adapter) ? 1 : 0; diff --git a/drivers/net/ethernet/chelsio/cxgb/sge.c b/drivers/net/ethernet/chelsio/cxgb/sge.c index cda01f22c71c..12e76fd0ae91 100644 --- a/drivers/net/ethernet/chelsio/cxgb/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb/sge.c @@ -1359,7 +1359,7 @@ static void restart_sched(struct tasklet_struct *t) * @fl: the free list that contains the packet buffer * @len: the packet length * - * Process an ingress ethernet pakcet and deliver it to the stack. + * Process an ingress ethernet packet and deliver it to the stack. */ static void sge_rx(struct sge *sge, struct freelQ *fl, unsigned int len) { diff --git a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c index bfffcaeee624..e2637bd2f423 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c +++ b/drivers/net/ethernet/chelsio/cxgb3/cxgb3_main.c @@ -1948,7 +1948,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; @@ -1964,7 +1966,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = q->txq_size[0]; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; diff --git a/drivers/net/ethernet/chelsio/cxgb3/sge.c b/drivers/net/ethernet/chelsio/cxgb3/sge.c index c3afec1041f8..62dfbdd33365 100644 --- a/drivers/net/ethernet/chelsio/cxgb3/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb3/sge.c @@ -126,8 +126,10 @@ struct rsp_desc { /* response queue descriptor */ struct rss_header rss_hdr; __be32 flags; __be32 len_cq; - u8 imm_data[47]; - u8 intr_gen; + struct_group(immediate, + u8 imm_data[47]; + u8 intr_gen; + ); }; /* @@ -925,7 +927,8 @@ static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp) if (skb) { __skb_put(skb, IMMED_PKT_SIZE); - skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE); + BUILD_BUG_ON(IMMED_PKT_SIZE != sizeof(resp->immediate)); + skb_copy_to_linear_data(skb, &resp->immediate, IMMED_PKT_SIZE); } return skb; } @@ -1953,7 +1956,7 @@ static int ofld_poll(struct napi_struct *napi, int budget) * @rx_gather: a gather list of packets if we are building a bundle * @gather_idx: index of the next available slot in the bundle * - * Process an ingress offload pakcet and add it to the offload ingress + * Process an ingress offload packet and add it to the offload ingress * queue. Returns the index of the next available slot in the bundle. */ static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq, @@ -2079,7 +2082,7 @@ static void cxgb3_process_iscsi_prov_pack(struct port_info *pi, * @pad: padding * @lro: large receive offload * - * Process an ingress ethernet pakcet and deliver it to the stack. + * Process an ingress ethernet packet and deliver it to the stack. * The padding is 2 if the packet was delivered in an Rx buffer and 0 * if it was immediate data in a response. */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 129352bbe114..6c790af92170 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -890,7 +890,9 @@ static int set_pauseparam(struct net_device *dev, return 0; } -static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); const struct sge *s = &pi->adapter->sge; @@ -906,7 +908,9 @@ static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) e->tx_pending = s->ethtxq[pi->first_qset].q.size; } -static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) +static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e, + struct kernel_ethtool_ringparam *kernel_e, + struct netlink_ext_ack *extack) { int i; const struct port_info *pi = netdev_priv(dev); @@ -1989,6 +1993,15 @@ static int get_dump_data(struct net_device *dev, struct ethtool_dump *eth_dump, return 0; } +static bool cxgb4_fw_mod_type_info_available(unsigned int fw_mod_type) +{ + /* Read port module EEPROM as long as it is plugged-in and + * safe to read. + */ + return (fw_mod_type != FW_PORT_MOD_TYPE_NONE && + fw_mod_type != FW_PORT_MOD_TYPE_ERROR); +} + static int cxgb4_get_module_info(struct net_device *dev, struct ethtool_modinfo *modinfo) { @@ -1997,7 +2010,7 @@ static int cxgb4_get_module_info(struct net_device *dev, struct adapter *adapter = pi->adapter; int ret; - if (!t4_is_inserted_mod_type(pi->mod_type)) + if (!cxgb4_fw_mod_type_info_available(pi->mod_type)) return -EINVAL; switch (pi->port_type) { diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index fa5b596ff23a..f889f404305c 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1842,8 +1842,10 @@ static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb, * (including the VLAN tag) into the header so we reject anything * smaller than that ... */ - fw_hdr_copy_len = sizeof(wr->ethmacdst) + sizeof(wr->ethmacsrc) + - sizeof(wr->ethtype) + sizeof(wr->vlantci); + BUILD_BUG_ON(sizeof(wr->firmware) != + (sizeof(wr->ethmacdst) + sizeof(wr->ethmacsrc) + + sizeof(wr->ethtype) + sizeof(wr->vlantci))); + fw_hdr_copy_len = sizeof(wr->firmware); ret = cxgb4_validate_skb(skb, dev, fw_hdr_copy_len); if (ret) goto out_free; @@ -1924,7 +1926,7 @@ static netdev_tx_t cxgb4_vf_eth_xmit(struct sk_buff *skb, wr->equiq_to_len16 = cpu_to_be32(wr_mid); wr->r3[0] = cpu_to_be32(0); wr->r3[1] = cpu_to_be32(0); - skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len); + skb_copy_from_linear_data(skb, &wr->firmware, fw_hdr_copy_len); end = (u64 *)wr + flits; /* If this is a Large Send Offload packet we'll put in an LSO CPL diff --git a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h index 0a326c054707..2419459a0b85 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h +++ b/drivers/net/ethernet/chelsio/cxgb4/t4fw_api.h @@ -794,10 +794,12 @@ struct fw_eth_tx_pkt_vm_wr { __be32 op_immdlen; __be32 equiq_to_len16; __be32 r3[2]; - u8 ethmacdst[6]; - u8 ethmacsrc[6]; - __be16 ethtype; - __be16 vlantci; + struct_group(firmware, + u8 ethmacdst[ETH_ALEN]; + u8 ethmacsrc[ETH_ALEN]; + __be16 ethtype; + __be16 vlantci; + ); }; #define FW_CMD_MAX_TIMEOUT 10000 diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c index ae9cca768d74..acac2be0e3f0 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c @@ -1591,7 +1591,9 @@ static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel) * first Queue Set. */ static void cxgb4vf_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); const struct sge *s = &pi->adapter->sge; @@ -1614,7 +1616,9 @@ static void cxgb4vf_get_ringparam(struct net_device *dev, * device -- after vetting them of course! */ static int cxgb4vf_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { const struct port_info *pi = netdev_priv(dev); struct adapter *adapter = pi->adapter; diff --git a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c index 0295b2406646..43b2ceb6aa32 100644 --- a/drivers/net/ethernet/chelsio/cxgb4vf/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4vf/sge.c @@ -1167,10 +1167,7 @@ netdev_tx_t t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev) struct cpl_tx_pkt_core *cpl; const struct skb_shared_info *ssi; dma_addr_t addr[MAX_SKB_FRAGS + 1]; - const size_t fw_hdr_copy_len = (sizeof(wr->ethmacdst) + - sizeof(wr->ethmacsrc) + - sizeof(wr->ethtype) + - sizeof(wr->vlantci)); + const size_t fw_hdr_copy_len = sizeof(wr->firmware); /* * The chip minimum packet length is 10 octets but the firmware @@ -1267,7 +1264,7 @@ netdev_tx_t t4vf_eth_xmit(struct sk_buff *skb, struct net_device *dev) wr->equiq_to_len16 = cpu_to_be32(wr_mid); wr->r3[0] = cpu_to_be32(0); wr->r3[1] = cpu_to_be32(0); - skb_copy_from_linear_data(skb, (void *)wr->ethmacdst, fw_hdr_copy_len); + skb_copy_from_linear_data(skb, &wr->firmware, fw_hdr_copy_len); end = (u64 *)wr + flits; /* diff --git a/drivers/net/ethernet/cirrus/mac89x0.c b/drivers/net/ethernet/cirrus/mac89x0.c index 84251b85fc93..21a70b1f0ac5 100644 --- a/drivers/net/ethernet/cirrus/mac89x0.c +++ b/drivers/net/ethernet/cirrus/mac89x0.c @@ -242,12 +242,15 @@ static int mac89x0_device_probe(struct platform_device *pdev) pr_info("No EEPROM, giving up now.\n"); goto out1; } else { + u8 addr[ETH_ALEN]; + for (i = 0; i < ETH_ALEN; i += 2) { /* Big-endian (why??!) */ unsigned short s = readreg(dev, PP_IA + i); - dev->dev_addr[i] = s >> 8; - dev->dev_addr[i+1] = s & 0xff; + addr[i] = s >> 8; + addr[i+1] = s & 0xff; } + eth_hw_addr_set(dev, addr); } dev->irq = SLOT2IRQ(slot); diff --git a/drivers/net/ethernet/cisco/enic/enic_ethtool.c b/drivers/net/ethernet/cisco/enic/enic_ethtool.c index 6ded4d9fa32a..6c11f9d62526 100644 --- a/drivers/net/ethernet/cisco/enic/enic_ethtool.c +++ b/drivers/net/ethernet/cisco/enic/enic_ethtool.c @@ -177,7 +177,9 @@ static void enic_get_strings(struct net_device *netdev, u32 stringset, } static void enic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enic *enic = netdev_priv(netdev); struct vnic_enet_config *c = &enic->config; @@ -189,7 +191,9 @@ static void enic_get_ringparam(struct net_device *netdev, } static int enic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enic *enic = netdev_priv(netdev); struct vnic_enet_config *c = &enic->config; diff --git a/drivers/net/ethernet/cortina/gemini.c b/drivers/net/ethernet/cortina/gemini.c index 941f175fb911..07add311f65d 100644 --- a/drivers/net/ethernet/cortina/gemini.c +++ b/drivers/net/ethernet/cortina/gemini.c @@ -2105,7 +2105,9 @@ static void gmac_get_pauseparam(struct net_device *netdev, } static void gmac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct gemini_ethernet_port *port = netdev_priv(netdev); @@ -2123,7 +2125,9 @@ static void gmac_get_ringparam(struct net_device *netdev, } static int gmac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct gemini_ethernet_port *port = netdev_priv(netdev); int err = 0; diff --git a/drivers/net/ethernet/emulex/benet/be_ethtool.c b/drivers/net/ethernet/emulex/benet/be_ethtool.c index f9955308b93d..dfa784339781 100644 --- a/drivers/net/ethernet/emulex/benet/be_ethtool.c +++ b/drivers/net/ethernet/emulex/benet/be_ethtool.c @@ -683,7 +683,9 @@ static int be_get_link_ksettings(struct net_device *netdev, } static void be_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct be_adapter *adapter = netdev_priv(netdev); diff --git a/drivers/net/ethernet/engleder/Kconfig b/drivers/net/ethernet/engleder/Kconfig new file mode 100644 index 000000000000..614dcc65c634 --- /dev/null +++ b/drivers/net/ethernet/engleder/Kconfig @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Engleder network device configuration +# + +config NET_VENDOR_ENGLEDER + bool "Engleder devices" + default y + help + If you have a network (Ethernet) card belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all + the questions about Engleder devices. If you say Y, you will be asked + for your specific card in the following questions. + +if NET_VENDOR_ENGLEDER + +config TSNEP + tristate "TSN endpoint support" + depends on PTP_1588_CLOCK_OPTIONAL + select PHYLIB + help + Support for the Engleder TSN endpoint Ethernet MAC IP Core. + + To compile this driver as a module, choose M here. The module will be + called tsnep. + +config TSNEP_SELFTESTS + bool "TSN endpoint self test support" + default n + depends on TSNEP + help + This enables self test support within the TSN endpoint driver. + + If unsure, say N. + +endif # NET_VENDOR_ENGLEDER diff --git a/drivers/net/ethernet/engleder/Makefile b/drivers/net/ethernet/engleder/Makefile new file mode 100644 index 000000000000..cce2191cb889 --- /dev/null +++ b/drivers/net/ethernet/engleder/Makefile @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Engleder Ethernet drivers +# + +obj-$(CONFIG_TSNEP) += tsnep.o + +tsnep-objs := tsnep_main.o tsnep_ethtool.o tsnep_ptp.o tsnep_tc.o \ + $(tsnep-y) +tsnep-$(CONFIG_TSNEP_SELFTESTS) += tsnep_selftests.o diff --git a/drivers/net/ethernet/engleder/tsnep.h b/drivers/net/ethernet/engleder/tsnep.h new file mode 100644 index 000000000000..23bbece6b7de --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +#ifndef _TSNEP_H +#define _TSNEP_H + +#include "tsnep_hw.h" + +#include <linux/platform_device.h> +#include <linux/dma-mapping.h> +#include <linux/etherdevice.h> +#include <linux/phy.h> +#include <linux/ethtool.h> +#include <linux/net_tstamp.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/miscdevice.h> + +#define TSNEP "tsnep" + +#define TSNEP_RING_SIZE 256 +#define TSNEP_RING_ENTRIES_PER_PAGE (PAGE_SIZE / TSNEP_DESC_SIZE) +#define TSNEP_RING_PAGE_COUNT (TSNEP_RING_SIZE / TSNEP_RING_ENTRIES_PER_PAGE) + +#define TSNEP_QUEUES 1 + +struct tsnep_gcl { + void __iomem *addr; + + u64 base_time; + u64 cycle_time; + u64 cycle_time_extension; + + struct tsnep_gcl_operation operation[TSNEP_GCL_COUNT]; + int count; + + u64 change_limit; + + u64 start_time; + bool change; +}; + +struct tsnep_tx_entry { + struct tsnep_tx_desc *desc; + struct tsnep_tx_desc_wb *desc_wb; + dma_addr_t desc_dma; + bool owner_user_flag; + + u32 properties; + + struct sk_buff *skb; + size_t len; + DEFINE_DMA_UNMAP_ADDR(dma); +}; + +struct tsnep_tx { + struct tsnep_adapter *adapter; + void __iomem *addr; + + void *page[TSNEP_RING_PAGE_COUNT]; + dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT]; + + /* TX ring lock */ + spinlock_t lock; + struct tsnep_tx_entry entry[TSNEP_RING_SIZE]; + int write; + int read; + u32 owner_counter; + int increment_owner_counter; + + u32 packets; + u32 bytes; + u32 dropped; +}; + +struct tsnep_rx_entry { + struct tsnep_rx_desc *desc; + struct tsnep_rx_desc_wb *desc_wb; + dma_addr_t desc_dma; + + u32 properties; + + struct sk_buff *skb; + size_t len; + DEFINE_DMA_UNMAP_ADDR(dma); +}; + +struct tsnep_rx { + struct tsnep_adapter *adapter; + void __iomem *addr; + + void *page[TSNEP_RING_PAGE_COUNT]; + dma_addr_t page_dma[TSNEP_RING_PAGE_COUNT]; + + struct tsnep_rx_entry entry[TSNEP_RING_SIZE]; + int read; + u32 owner_counter; + int increment_owner_counter; + + u32 packets; + u32 bytes; + u32 dropped; + u32 multicast; +}; + +struct tsnep_queue { + struct tsnep_adapter *adapter; + + struct tsnep_tx *tx; + struct tsnep_rx *rx; + + struct napi_struct napi; + + u32 irq_mask; +}; + +struct tsnep_adapter { + struct net_device *netdev; + u8 mac_address[ETH_ALEN]; + struct mii_bus *mdiobus; + bool suppress_preamble; + phy_interface_t phy_mode; + struct phy_device *phydev; + int msg_enable; + + struct platform_device *pdev; + struct device *dmadev; + void __iomem *addr; + int irq; + + bool gate_control; + /* gate control lock */ + struct mutex gate_control_lock; + bool gate_control_active; + struct tsnep_gcl gcl[2]; + int next_gcl; + + struct hwtstamp_config hwtstamp_config; + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_clock_info; + /* ptp clock lock */ + spinlock_t ptp_lock; + + int num_tx_queues; + struct tsnep_tx tx[TSNEP_MAX_QUEUES]; + int num_rx_queues; + struct tsnep_rx rx[TSNEP_MAX_QUEUES]; + + int num_queues; + struct tsnep_queue queue[TSNEP_MAX_QUEUES]; +}; + +extern const struct ethtool_ops tsnep_ethtool_ops; + +int tsnep_ptp_init(struct tsnep_adapter *adapter); +void tsnep_ptp_cleanup(struct tsnep_adapter *adapter); +int tsnep_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd); + +int tsnep_tc_init(struct tsnep_adapter *adapter); +void tsnep_tc_cleanup(struct tsnep_adapter *adapter); +int tsnep_tc_setup(struct net_device *netdev, enum tc_setup_type type, + void *type_data); + +#if IS_ENABLED(CONFIG_TSNEP_SELFTESTS) +int tsnep_ethtool_get_test_count(void); +void tsnep_ethtool_get_test_strings(u8 *data); +void tsnep_ethtool_self_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data); +#else +static inline int tsnep_ethtool_get_test_count(void) +{ + return -EOPNOTSUPP; +} + +static inline void tsnep_ethtool_get_test_strings(u8 *data) +{ + /* not enabled */ +} + +static inline void tsnep_ethtool_self_test(struct net_device *dev, + struct ethtool_test *eth_test, + u64 *data) +{ + /* not enabled */ +} +#endif /* CONFIG_TSNEP_SELFTESTS */ + +void tsnep_get_system_time(struct tsnep_adapter *adapter, u64 *time); + +#endif /* _TSNEP_H */ diff --git a/drivers/net/ethernet/engleder/tsnep_ethtool.c b/drivers/net/ethernet/engleder/tsnep_ethtool.c new file mode 100644 index 000000000000..e6760dc68ddd --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_ethtool.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +#include "tsnep.h" + +static const char tsnep_stats_strings[][ETH_GSTRING_LEN] = { + "rx_packets", + "rx_bytes", + "rx_dropped", + "rx_multicast", + "rx_phy_errors", + "rx_forwarded_phy_errors", + "rx_invalid_frame_errors", + "tx_packets", + "tx_bytes", + "tx_dropped", +}; + +struct tsnep_stats { + u64 rx_packets; + u64 rx_bytes; + u64 rx_dropped; + u64 rx_multicast; + u64 rx_phy_errors; + u64 rx_forwarded_phy_errors; + u64 rx_invalid_frame_errors; + u64 tx_packets; + u64 tx_bytes; + u64 tx_dropped; +}; + +#define TSNEP_STATS_COUNT (sizeof(struct tsnep_stats) / sizeof(u64)) + +static const char tsnep_rx_queue_stats_strings[][ETH_GSTRING_LEN] = { + "rx_%d_packets", + "rx_%d_bytes", + "rx_%d_dropped", + "rx_%d_multicast", + "rx_%d_no_descriptor_errors", + "rx_%d_buffer_too_small_errors", + "rx_%d_fifo_overflow_errors", + "rx_%d_invalid_frame_errors", +}; + +struct tsnep_rx_queue_stats { + u64 rx_packets; + u64 rx_bytes; + u64 rx_dropped; + u64 rx_multicast; + u64 rx_no_descriptor_errors; + u64 rx_buffer_too_small_errors; + u64 rx_fifo_overflow_errors; + u64 rx_invalid_frame_errors; +}; + +#define TSNEP_RX_QUEUE_STATS_COUNT (sizeof(struct tsnep_rx_queue_stats) / \ + sizeof(u64)) + +static const char tsnep_tx_queue_stats_strings[][ETH_GSTRING_LEN] = { + "tx_%d_packets", + "tx_%d_bytes", + "tx_%d_dropped", +}; + +struct tsnep_tx_queue_stats { + u64 tx_packets; + u64 tx_bytes; + u64 tx_dropped; +}; + +#define TSNEP_TX_QUEUE_STATS_COUNT (sizeof(struct tsnep_tx_queue_stats) / \ + sizeof(u64)) + +static void tsnep_ethtool_get_drvinfo(struct net_device *netdev, + struct ethtool_drvinfo *drvinfo) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + strscpy(drvinfo->driver, TSNEP, sizeof(drvinfo->driver)); + strscpy(drvinfo->bus_info, dev_name(&adapter->pdev->dev), + sizeof(drvinfo->bus_info)); +} + +static int tsnep_ethtool_get_regs_len(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int len; + int num_additional_queues; + + len = TSNEP_MAC_SIZE; + + /* first queue pair is within TSNEP_MAC_SIZE, only queues additional to + * the first queue pair extend the register length by TSNEP_QUEUE_SIZE + */ + num_additional_queues = + max(adapter->num_tx_queues, adapter->num_rx_queues) - 1; + len += TSNEP_QUEUE_SIZE * num_additional_queues; + + return len; +} + +static void tsnep_ethtool_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, + void *p) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + regs->version = 1; + + memcpy_fromio(p, adapter->addr, regs->len); +} + +static u32 tsnep_ethtool_get_msglevel(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + return adapter->msg_enable; +} + +static void tsnep_ethtool_set_msglevel(struct net_device *netdev, u32 data) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + adapter->msg_enable = data; +} + +static void tsnep_ethtool_get_strings(struct net_device *netdev, u32 stringset, + u8 *data) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int rx_count = adapter->num_rx_queues; + int tx_count = adapter->num_tx_queues; + int i, j; + + switch (stringset) { + case ETH_SS_STATS: + memcpy(data, tsnep_stats_strings, sizeof(tsnep_stats_strings)); + data += sizeof(tsnep_stats_strings); + + for (i = 0; i < rx_count; i++) { + for (j = 0; j < TSNEP_RX_QUEUE_STATS_COUNT; j++) { + snprintf(data, ETH_GSTRING_LEN, + tsnep_rx_queue_stats_strings[j], i); + data += ETH_GSTRING_LEN; + } + } + + for (i = 0; i < tx_count; i++) { + for (j = 0; j < TSNEP_TX_QUEUE_STATS_COUNT; j++) { + snprintf(data, ETH_GSTRING_LEN, + tsnep_tx_queue_stats_strings[j], i); + data += ETH_GSTRING_LEN; + } + } + break; + case ETH_SS_TEST: + tsnep_ethtool_get_test_strings(data); + break; + } +} + +static void tsnep_ethtool_get_ethtool_stats(struct net_device *netdev, + struct ethtool_stats *stats, + u64 *data) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int rx_count = adapter->num_rx_queues; + int tx_count = adapter->num_tx_queues; + struct tsnep_stats tsnep_stats; + struct tsnep_rx_queue_stats tsnep_rx_queue_stats; + struct tsnep_tx_queue_stats tsnep_tx_queue_stats; + u32 reg; + int i; + + memset(&tsnep_stats, 0, sizeof(tsnep_stats)); + for (i = 0; i < adapter->num_rx_queues; i++) { + tsnep_stats.rx_packets += adapter->rx[i].packets; + tsnep_stats.rx_bytes += adapter->rx[i].bytes; + tsnep_stats.rx_dropped += adapter->rx[i].dropped; + tsnep_stats.rx_multicast += adapter->rx[i].multicast; + } + reg = ioread32(adapter->addr + ECM_STAT); + tsnep_stats.rx_phy_errors = + (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; + tsnep_stats.rx_forwarded_phy_errors = + (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; + tsnep_stats.rx_invalid_frame_errors = + (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; + for (i = 0; i < adapter->num_tx_queues; i++) { + tsnep_stats.tx_packets += adapter->tx[i].packets; + tsnep_stats.tx_bytes += adapter->tx[i].bytes; + tsnep_stats.tx_dropped += adapter->tx[i].dropped; + } + memcpy(data, &tsnep_stats, sizeof(tsnep_stats)); + data += TSNEP_STATS_COUNT; + + for (i = 0; i < rx_count; i++) { + memset(&tsnep_rx_queue_stats, 0, sizeof(tsnep_rx_queue_stats)); + tsnep_rx_queue_stats.rx_packets = adapter->rx[i].packets; + tsnep_rx_queue_stats.rx_bytes = adapter->rx[i].bytes; + tsnep_rx_queue_stats.rx_dropped = adapter->rx[i].dropped; + tsnep_rx_queue_stats.rx_multicast = adapter->rx[i].multicast; + reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + + TSNEP_RX_STATISTIC); + tsnep_rx_queue_stats.rx_no_descriptor_errors = + (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> + TSNEP_RX_STATISTIC_NO_DESC_SHIFT; + tsnep_rx_queue_stats.rx_buffer_too_small_errors = + (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> + TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; + tsnep_rx_queue_stats.rx_fifo_overflow_errors = + (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> + TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; + tsnep_rx_queue_stats.rx_invalid_frame_errors = + (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> + TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; + memcpy(data, &tsnep_rx_queue_stats, + sizeof(tsnep_rx_queue_stats)); + data += TSNEP_RX_QUEUE_STATS_COUNT; + } + + for (i = 0; i < tx_count; i++) { + memset(&tsnep_tx_queue_stats, 0, sizeof(tsnep_tx_queue_stats)); + tsnep_tx_queue_stats.tx_packets += adapter->tx[i].packets; + tsnep_tx_queue_stats.tx_bytes += adapter->tx[i].bytes; + tsnep_tx_queue_stats.tx_dropped += adapter->tx[i].dropped; + memcpy(data, &tsnep_tx_queue_stats, + sizeof(tsnep_tx_queue_stats)); + data += TSNEP_TX_QUEUE_STATS_COUNT; + } +} + +static int tsnep_ethtool_get_sset_count(struct net_device *netdev, int sset) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int rx_count; + int tx_count; + + switch (sset) { + case ETH_SS_STATS: + rx_count = adapter->num_rx_queues; + tx_count = adapter->num_tx_queues; + return TSNEP_STATS_COUNT + + TSNEP_RX_QUEUE_STATS_COUNT * rx_count + + TSNEP_TX_QUEUE_STATS_COUNT * tx_count; + case ETH_SS_TEST: + return tsnep_ethtool_get_test_count(); + default: + return -EOPNOTSUPP; + } +} + +static int tsnep_ethtool_get_ts_info(struct net_device *dev, + struct ethtool_ts_info *info) +{ + struct tsnep_adapter *adapter = netdev_priv(dev); + + info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_RX_SOFTWARE | + SOF_TIMESTAMPING_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + if (adapter->ptp_clock) + info->phc_index = ptp_clock_index(adapter->ptp_clock); + else + info->phc_index = -1; + + info->tx_types = BIT(HWTSTAMP_TX_OFF) | + BIT(HWTSTAMP_TX_ON); + info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_ALL); + + return 0; +} + +const struct ethtool_ops tsnep_ethtool_ops = { + .get_drvinfo = tsnep_ethtool_get_drvinfo, + .get_regs_len = tsnep_ethtool_get_regs_len, + .get_regs = tsnep_ethtool_get_regs, + .get_msglevel = tsnep_ethtool_get_msglevel, + .set_msglevel = tsnep_ethtool_set_msglevel, + .nway_reset = phy_ethtool_nway_reset, + .get_link = ethtool_op_get_link, + .self_test = tsnep_ethtool_self_test, + .get_strings = tsnep_ethtool_get_strings, + .get_ethtool_stats = tsnep_ethtool_get_ethtool_stats, + .get_sset_count = tsnep_ethtool_get_sset_count, + .get_ts_info = tsnep_ethtool_get_ts_info, + .get_link_ksettings = phy_ethtool_get_link_ksettings, + .set_link_ksettings = phy_ethtool_set_link_ksettings, +}; diff --git a/drivers/net/ethernet/engleder/tsnep_hw.h b/drivers/net/ethernet/engleder/tsnep_hw.h new file mode 100644 index 000000000000..71cc8577d640 --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_hw.h @@ -0,0 +1,230 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +/* Hardware definition of TSNEP and EtherCAT MAC device */ + +#ifndef _TSNEP_HW_H +#define _TSNEP_HW_H + +#include <linux/types.h> + +/* type */ +#define ECM_TYPE 0x0000 +#define ECM_REVISION_MASK 0x000000FF +#define ECM_REVISION_SHIFT 0 +#define ECM_VERSION_MASK 0x0000FF00 +#define ECM_VERSION_SHIFT 8 +#define ECM_QUEUE_COUNT_MASK 0x00070000 +#define ECM_QUEUE_COUNT_SHIFT 16 +#define ECM_GATE_CONTROL 0x02000000 + +/* system time */ +#define ECM_SYSTEM_TIME_LOW 0x0008 +#define ECM_SYSTEM_TIME_HIGH 0x000C + +/* clock */ +#define ECM_CLOCK_RATE 0x0010 +#define ECM_CLOCK_RATE_OFFSET_MASK 0x7FFFFFFF +#define ECM_CLOCK_RATE_OFFSET_SIGN 0x80000000 + +/* interrupt */ +#define ECM_INT_ENABLE 0x0018 +#define ECM_INT_ACTIVE 0x001C +#define ECM_INT_ACKNOWLEDGE 0x001C +#define ECM_INT_LINK 0x00000020 +#define ECM_INT_TX_0 0x00000100 +#define ECM_INT_RX_0 0x00000200 +#define ECM_INT_ALL 0x7FFFFFFF +#define ECM_INT_DISABLE 0x80000000 + +/* reset */ +#define ECM_RESET 0x0020 +#define ECM_RESET_COMMON 0x00000001 +#define ECM_RESET_CHANNEL 0x00000100 +#define ECM_RESET_TXRX 0x00010000 + +/* control and status */ +#define ECM_STATUS 0x0080 +#define ECM_LINK_MODE_OFF 0x01000000 +#define ECM_LINK_MODE_100 0x02000000 +#define ECM_LINK_MODE_1000 0x04000000 +#define ECM_NO_LINK 0x01000000 +#define ECM_LINK_MODE_MASK 0x06000000 + +/* management data */ +#define ECM_MD_CONTROL 0x0084 +#define ECM_MD_STATUS 0x0084 +#define ECM_MD_PREAMBLE 0x00000001 +#define ECM_MD_READ 0x00000004 +#define ECM_MD_WRITE 0x00000002 +#define ECM_MD_ADDR_MASK 0x000000F8 +#define ECM_MD_ADDR_SHIFT 3 +#define ECM_MD_PHY_ADDR_MASK 0x00001F00 +#define ECM_MD_PHY_ADDR_SHIFT 8 +#define ECM_MD_BUSY 0x00000001 +#define ECM_MD_DATA_MASK 0xFFFF0000 +#define ECM_MD_DATA_SHIFT 16 + +/* statistic */ +#define ECM_STAT 0x00B0 +#define ECM_STAT_RX_ERR_MASK 0x000000FF +#define ECM_STAT_RX_ERR_SHIFT 0 +#define ECM_STAT_INV_FRM_MASK 0x0000FF00 +#define ECM_STAT_INV_FRM_SHIFT 8 +#define ECM_STAT_FWD_RX_ERR_MASK 0x00FF0000 +#define ECM_STAT_FWD_RX_ERR_SHIFT 16 + +/* tsnep */ +#define TSNEP_MAC_SIZE 0x4000 +#define TSNEP_QUEUE_SIZE 0x1000 +#define TSNEP_QUEUE(n) ({ typeof(n) __n = (n); \ + (__n) == 0 ? \ + 0 : \ + TSNEP_MAC_SIZE + TSNEP_QUEUE_SIZE * ((__n) - 1); }) +#define TSNEP_MAX_QUEUES 8 +#define TSNEP_MAX_FRAME_SIZE (2 * 1024) /* hardware supports actually 16k */ +#define TSNEP_DESC_SIZE 256 +#define TSNEP_DESC_OFFSET 128 + +/* tsnep register */ +#define TSNEP_INFO 0x0100 +#define TSNEP_INFO_RX_ASSIGN 0x00010000 +#define TSNEP_INFO_TX_TIME 0x00020000 +#define TSNEP_CONTROL 0x0108 +#define TSNEP_CONTROL_TX_RESET 0x00000001 +#define TSNEP_CONTROL_TX_ENABLE 0x00000002 +#define TSNEP_CONTROL_TX_DMA_ERROR 0x00000010 +#define TSNEP_CONTROL_TX_DESC_ERROR 0x00000020 +#define TSNEP_CONTROL_RX_RESET 0x00000100 +#define TSNEP_CONTROL_RX_ENABLE 0x00000200 +#define TSNEP_CONTROL_RX_DISABLE 0x00000400 +#define TSNEP_CONTROL_RX_DMA_ERROR 0x00001000 +#define TSNEP_CONTROL_RX_DESC_ERROR 0x00002000 +#define TSNEP_TX_DESC_ADDR_LOW 0x0140 +#define TSNEP_TX_DESC_ADDR_HIGH 0x0144 +#define TSNEP_RX_DESC_ADDR_LOW 0x0180 +#define TSNEP_RX_DESC_ADDR_HIGH 0x0184 +#define TSNEP_RESET_OWNER_COUNTER 0x01 +#define TSNEP_RX_STATISTIC 0x0190 +#define TSNEP_RX_STATISTIC_NO_DESC_MASK 0x000000FF +#define TSNEP_RX_STATISTIC_NO_DESC_SHIFT 0 +#define TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK 0x0000FF00 +#define TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT 8 +#define TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK 0x00FF0000 +#define TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT 16 +#define TSNEP_RX_STATISTIC_INVALID_FRAME_MASK 0xFF000000 +#define TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT 24 +#define TSNEP_RX_STATISTIC_NO_DESC 0x0190 +#define TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL 0x0191 +#define TSNEP_RX_STATISTIC_FIFO_OVERFLOW 0x0192 +#define TSNEP_RX_STATISTIC_INVALID_FRAME 0x0193 +#define TSNEP_RX_ASSIGN 0x01A0 +#define TSNEP_RX_ASSIGN_ETHER_TYPE_ACTIVE 0x00000001 +#define TSNEP_RX_ASSIGN_ETHER_TYPE_MASK 0xFFFF0000 +#define TSNEP_RX_ASSIGN_ETHER_TYPE_SHIFT 16 +#define TSNEP_MAC_ADDRESS_LOW 0x0800 +#define TSNEP_MAC_ADDRESS_HIGH 0x0804 +#define TSNEP_RX_FILTER 0x0806 +#define TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS 0x0001 +#define TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS 0x0002 +#define TSNEP_GC 0x0808 +#define TSNEP_GC_ENABLE_A 0x00000002 +#define TSNEP_GC_ENABLE_B 0x00000004 +#define TSNEP_GC_DISABLE 0x00000008 +#define TSNEP_GC_ENABLE_TIMEOUT 0x00000010 +#define TSNEP_GC_ACTIVE_A 0x00000002 +#define TSNEP_GC_ACTIVE_B 0x00000004 +#define TSNEP_GC_CHANGE_AB 0x00000008 +#define TSNEP_GC_TIMEOUT_ACTIVE 0x00000010 +#define TSNEP_GC_TIMEOUT_SIGNAL 0x00000020 +#define TSNEP_GC_LIST_ERROR 0x00000080 +#define TSNEP_GC_OPEN 0x00FF0000 +#define TSNEP_GC_OPEN_SHIFT 16 +#define TSNEP_GC_NEXT_OPEN 0xFF000000 +#define TSNEP_GC_NEXT_OPEN_SHIFT 24 +#define TSNEP_GC_TIMEOUT 131072 +#define TSNEP_GC_TIME 0x080C +#define TSNEP_GC_CHANGE 0x0810 +#define TSNEP_GCL_A 0x2000 +#define TSNEP_GCL_B 0x2800 +#define TSNEP_GCL_SIZE SZ_2K + +/* tsnep gate control list operation */ +struct tsnep_gcl_operation { + u32 properties; + u32 interval; +}; + +#define TSNEP_GCL_COUNT (TSNEP_GCL_SIZE / sizeof(struct tsnep_gcl_operation)) +#define TSNEP_GCL_MASK 0x000000FF +#define TSNEP_GCL_INSERT 0x20000000 +#define TSNEP_GCL_CHANGE 0x40000000 +#define TSNEP_GCL_LAST 0x80000000 +#define TSNEP_GCL_MIN_INTERVAL 32 + +/* tsnep TX/RX descriptor */ +#define TSNEP_DESC_SIZE 256 +#define TSNEP_DESC_SIZE_DATA_AFTER 2048 +#define TSNEP_DESC_OFFSET 128 +#define TSNEP_DESC_OWNER_COUNTER_MASK 0xC0000000 +#define TSNEP_DESC_OWNER_COUNTER_SHIFT 30 +#define TSNEP_DESC_LENGTH_MASK 0x00003FFF +#define TSNEP_DESC_INTERRUPT_FLAG 0x00040000 +#define TSNEP_DESC_EXTENDED_WRITEBACK_FLAG 0x00080000 +#define TSNEP_DESC_NO_LINK_FLAG 0x01000000 + +/* tsnep TX descriptor */ +struct tsnep_tx_desc { + __le32 properties; + __le32 more_properties; + __le32 reserved[2]; + __le64 next; + __le64 tx; +}; + +#define TSNEP_TX_DESC_OWNER_MASK 0xE0000000 +#define TSNEP_TX_DESC_OWNER_USER_FLAG 0x20000000 +#define TSNEP_TX_DESC_LAST_FRAGMENT_FLAG 0x00010000 +#define TSNEP_TX_DESC_DATA_AFTER_DESC_FLAG 0x00020000 + +/* tsnep TX descriptor writeback */ +struct tsnep_tx_desc_wb { + __le32 properties; + __le32 reserved1[3]; + __le64 timestamp; + __le32 dma_delay; + __le32 reserved2; +}; + +#define TSNEP_TX_DESC_UNDERRUN_ERROR_FLAG 0x00010000 +#define TSNEP_TX_DESC_DMA_DELAY_FIRST_DATA_MASK 0x0000FFFC +#define TSNEP_TX_DESC_DMA_DELAY_FIRST_DATA_SHIFT 2 +#define TSNEP_TX_DESC_DMA_DELAY_LAST_DATA_MASK 0xFFFC0000 +#define TSNEP_TX_DESC_DMA_DELAY_LAST_DATA_SHIFT 18 +#define TSNEP_TX_DESC_DMA_DELAY_NS 64 + +/* tsnep RX descriptor */ +struct tsnep_rx_desc { + __le32 properties; + __le32 reserved[3]; + __le64 next; + __le64 rx; +}; + +#define TSNEP_RX_DESC_BUFFER_SIZE_MASK 0x00003FFC + +/* tsnep RX descriptor writeback */ +struct tsnep_rx_desc_wb { + __le32 properties; + __le32 reserved[7]; +}; + +/* tsnep RX inline meta */ +struct tsnep_rx_inline { + __le64 reserved; + __le64 timestamp; +}; + +#define TSNEP_RX_INLINE_METADATA_SIZE (sizeof(struct tsnep_rx_inline)) + +#endif /* _TSNEP_HW_H */ diff --git a/drivers/net/ethernet/engleder/tsnep_main.c b/drivers/net/ethernet/engleder/tsnep_main.c new file mode 100644 index 000000000000..904f3304727e --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_main.c @@ -0,0 +1,1272 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +/* TSN endpoint Ethernet MAC driver + * + * The TSN endpoint Ethernet MAC is a FPGA based network device for real-time + * communication. It is designed for endpoints within TSN (Time Sensitive + * Networking) networks; e.g., for PLCs in the industrial automation case. + * + * It supports multiple TX/RX queue pairs. The first TX/RX queue pair is used + * by the driver. + * + * More information can be found here: + * - www.embedded-experts.at/tsn + * - www.engleder-embedded.com + */ + +#include "tsnep.h" +#include "tsnep_hw.h" + +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_net.h> +#include <linux/of_mdio.h> +#include <linux/interrupt.h> +#include <linux/etherdevice.h> +#include <linux/phy.h> +#include <linux/iopoll.h> + +#define RX_SKB_LENGTH (round_up(TSNEP_RX_INLINE_METADATA_SIZE + ETH_HLEN + \ + TSNEP_MAX_FRAME_SIZE + ETH_FCS_LEN, 4)) +#define RX_SKB_RESERVE ((16 - TSNEP_RX_INLINE_METADATA_SIZE) + NET_IP_ALIGN) +#define RX_SKB_ALLOC_LENGTH (RX_SKB_RESERVE + RX_SKB_LENGTH) + +#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT +#define DMA_ADDR_HIGH(dma_addr) ((u32)(((dma_addr) >> 32) & 0xFFFFFFFF)) +#else +#define DMA_ADDR_HIGH(dma_addr) ((u32)(0)) +#endif +#define DMA_ADDR_LOW(dma_addr) ((u32)((dma_addr) & 0xFFFFFFFF)) + +static void tsnep_enable_irq(struct tsnep_adapter *adapter, u32 mask) +{ + iowrite32(mask, adapter->addr + ECM_INT_ENABLE); +} + +static void tsnep_disable_irq(struct tsnep_adapter *adapter, u32 mask) +{ + mask |= ECM_INT_DISABLE; + iowrite32(mask, adapter->addr + ECM_INT_ENABLE); +} + +static irqreturn_t tsnep_irq(int irq, void *arg) +{ + struct tsnep_adapter *adapter = arg; + u32 active = ioread32(adapter->addr + ECM_INT_ACTIVE); + + /* acknowledge interrupt */ + if (active != 0) + iowrite32(active, adapter->addr + ECM_INT_ACKNOWLEDGE); + + /* handle link interrupt */ + if ((active & ECM_INT_LINK) != 0) { + if (adapter->netdev->phydev) + phy_mac_interrupt(adapter->netdev->phydev); + } + + /* handle TX/RX queue 0 interrupt */ + if ((active & adapter->queue[0].irq_mask) != 0) { + if (adapter->netdev) { + tsnep_disable_irq(adapter, adapter->queue[0].irq_mask); + napi_schedule(&adapter->queue[0].napi); + } + } + + return IRQ_HANDLED; +} + +static int tsnep_mdiobus_read(struct mii_bus *bus, int addr, int regnum) +{ + struct tsnep_adapter *adapter = bus->priv; + u32 md; + int retval; + + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + + md = ECM_MD_READ; + if (!adapter->suppress_preamble) + md |= ECM_MD_PREAMBLE; + md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; + md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; + iowrite32(md, adapter->addr + ECM_MD_CONTROL); + retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, + !(md & ECM_MD_BUSY), 16, 1000); + if (retval != 0) + return retval; + + return (md & ECM_MD_DATA_MASK) >> ECM_MD_DATA_SHIFT; +} + +static int tsnep_mdiobus_write(struct mii_bus *bus, int addr, int regnum, + u16 val) +{ + struct tsnep_adapter *adapter = bus->priv; + u32 md; + int retval; + + if (regnum & MII_ADDR_C45) + return -EOPNOTSUPP; + + md = ECM_MD_WRITE; + if (!adapter->suppress_preamble) + md |= ECM_MD_PREAMBLE; + md |= (regnum << ECM_MD_ADDR_SHIFT) & ECM_MD_ADDR_MASK; + md |= (addr << ECM_MD_PHY_ADDR_SHIFT) & ECM_MD_PHY_ADDR_MASK; + md |= ((u32)val << ECM_MD_DATA_SHIFT) & ECM_MD_DATA_MASK; + iowrite32(md, adapter->addr + ECM_MD_CONTROL); + retval = readl_poll_timeout_atomic(adapter->addr + ECM_MD_STATUS, md, + !(md & ECM_MD_BUSY), 16, 1000); + if (retval != 0) + return retval; + + return 0; +} + +static void tsnep_phy_link_status_change(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct phy_device *phydev = netdev->phydev; + u32 mode; + + if (phydev->link) { + switch (phydev->speed) { + case SPEED_100: + mode = ECM_LINK_MODE_100; + break; + case SPEED_1000: + mode = ECM_LINK_MODE_1000; + break; + default: + mode = ECM_LINK_MODE_OFF; + break; + } + iowrite32(mode, adapter->addr + ECM_STATUS); + } + + phy_print_status(netdev->phydev); +} + +static int tsnep_phy_open(struct tsnep_adapter *adapter) +{ + struct phy_device *phydev; + struct ethtool_eee ethtool_eee; + int retval; + + retval = phy_connect_direct(adapter->netdev, adapter->phydev, + tsnep_phy_link_status_change, + adapter->phy_mode); + if (retval) + return retval; + phydev = adapter->netdev->phydev; + + /* MAC supports only 100Mbps|1000Mbps full duplex + * SPE (Single Pair Ethernet) is also an option but not implemented yet + */ + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Half_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_10baseT_Full_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_100baseT_Half_BIT); + phy_remove_link_mode(phydev, ETHTOOL_LINK_MODE_1000baseT_Half_BIT); + + /* disable EEE autoneg, EEE not supported by TSNEP */ + memset(ðtool_eee, 0, sizeof(ethtool_eee)); + phy_ethtool_set_eee(adapter->phydev, ðtool_eee); + + adapter->phydev->irq = PHY_MAC_INTERRUPT; + phy_start(adapter->phydev); + + return 0; +} + +static void tsnep_phy_close(struct tsnep_adapter *adapter) +{ + phy_stop(adapter->netdev->phydev); + phy_disconnect(adapter->netdev->phydev); + adapter->netdev->phydev = NULL; +} + +static void tsnep_tx_ring_cleanup(struct tsnep_tx *tx) +{ + struct device *dmadev = tx->adapter->dmadev; + int i; + + memset(tx->entry, 0, sizeof(tx->entry)); + + for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { + if (tx->page[i]) { + dma_free_coherent(dmadev, PAGE_SIZE, tx->page[i], + tx->page_dma[i]); + tx->page[i] = NULL; + tx->page_dma[i] = 0; + } + } +} + +static int tsnep_tx_ring_init(struct tsnep_tx *tx) +{ + struct device *dmadev = tx->adapter->dmadev; + struct tsnep_tx_entry *entry; + struct tsnep_tx_entry *next_entry; + int i, j; + int retval; + + for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { + tx->page[i] = + dma_alloc_coherent(dmadev, PAGE_SIZE, &tx->page_dma[i], + GFP_KERNEL); + if (!tx->page[i]) { + retval = -ENOMEM; + goto alloc_failed; + } + for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { + entry = &tx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; + entry->desc_wb = (struct tsnep_tx_desc_wb *) + (((u8 *)tx->page[i]) + TSNEP_DESC_SIZE * j); + entry->desc = (struct tsnep_tx_desc *) + (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); + entry->desc_dma = tx->page_dma[i] + TSNEP_DESC_SIZE * j; + } + } + for (i = 0; i < TSNEP_RING_SIZE; i++) { + entry = &tx->entry[i]; + next_entry = &tx->entry[(i + 1) % TSNEP_RING_SIZE]; + entry->desc->next = __cpu_to_le64(next_entry->desc_dma); + } + + return 0; + +alloc_failed: + tsnep_tx_ring_cleanup(tx); + return retval; +} + +static void tsnep_tx_activate(struct tsnep_tx *tx, int index, bool last) +{ + struct tsnep_tx_entry *entry = &tx->entry[index]; + + entry->properties = 0; + if (entry->skb) { + entry->properties = + skb_pagelen(entry->skb) & TSNEP_DESC_LENGTH_MASK; + entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; + if (skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) + entry->properties |= TSNEP_DESC_EXTENDED_WRITEBACK_FLAG; + + /* toggle user flag to prevent false acknowledge + * + * Only the first fragment is acknowledged. For all other + * fragments no acknowledge is done and the last written owner + * counter stays in the writeback descriptor. Therefore, it is + * possible that the last written owner counter is identical to + * the new incremented owner counter and a false acknowledge is + * detected before the real acknowledge has been done by + * hardware. + * + * The user flag is used to prevent this situation. The user + * flag is copied to the writeback descriptor by the hardware + * and is used as additional acknowledge data. By toggeling the + * user flag only for the first fragment (which is + * acknowledged), it is guaranteed that the last acknowledge + * done for this descriptor has used a different user flag and + * cannot be detected as false acknowledge. + */ + entry->owner_user_flag = !entry->owner_user_flag; + } + if (last) + entry->properties |= TSNEP_TX_DESC_LAST_FRAGMENT_FLAG; + if (index == tx->increment_owner_counter) { + tx->owner_counter++; + if (tx->owner_counter == 4) + tx->owner_counter = 1; + tx->increment_owner_counter--; + if (tx->increment_owner_counter < 0) + tx->increment_owner_counter = TSNEP_RING_SIZE - 1; + } + entry->properties |= + (tx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & + TSNEP_DESC_OWNER_COUNTER_MASK; + if (entry->owner_user_flag) + entry->properties |= TSNEP_TX_DESC_OWNER_USER_FLAG; + entry->desc->more_properties = + __cpu_to_le32(entry->len & TSNEP_DESC_LENGTH_MASK); + + /* descriptor properties shall be written last, because valid data is + * signaled there + */ + dma_wmb(); + + entry->desc->properties = __cpu_to_le32(entry->properties); +} + +static int tsnep_tx_desc_available(struct tsnep_tx *tx) +{ + if (tx->read <= tx->write) + return TSNEP_RING_SIZE - tx->write + tx->read - 1; + else + return tx->read - tx->write - 1; +} + +static int tsnep_tx_map(struct sk_buff *skb, struct tsnep_tx *tx, int count) +{ + struct device *dmadev = tx->adapter->dmadev; + struct tsnep_tx_entry *entry; + unsigned int len; + dma_addr_t dma; + int i; + + for (i = 0; i < count; i++) { + entry = &tx->entry[(tx->write + i) % TSNEP_RING_SIZE]; + + if (i == 0) { + len = skb_headlen(skb); + dma = dma_map_single(dmadev, skb->data, len, + DMA_TO_DEVICE); + } else { + len = skb_frag_size(&skb_shinfo(skb)->frags[i - 1]); + dma = skb_frag_dma_map(dmadev, + &skb_shinfo(skb)->frags[i - 1], + 0, len, DMA_TO_DEVICE); + } + if (dma_mapping_error(dmadev, dma)) + return -ENOMEM; + + entry->len = len; + dma_unmap_addr_set(entry, dma, dma); + + entry->desc->tx = __cpu_to_le64(dma); + } + + return 0; +} + +static void tsnep_tx_unmap(struct tsnep_tx *tx, int count) +{ + struct device *dmadev = tx->adapter->dmadev; + struct tsnep_tx_entry *entry; + int i; + + for (i = 0; i < count; i++) { + entry = &tx->entry[(tx->read + i) % TSNEP_RING_SIZE]; + + if (entry->len) { + if (i == 0) + dma_unmap_single(dmadev, + dma_unmap_addr(entry, dma), + dma_unmap_len(entry, len), + DMA_TO_DEVICE); + else + dma_unmap_page(dmadev, + dma_unmap_addr(entry, dma), + dma_unmap_len(entry, len), + DMA_TO_DEVICE); + entry->len = 0; + } + } +} + +static netdev_tx_t tsnep_xmit_frame_ring(struct sk_buff *skb, + struct tsnep_tx *tx) +{ + unsigned long flags; + int count = 1; + struct tsnep_tx_entry *entry; + int i; + int retval; + + if (skb_shinfo(skb)->nr_frags > 0) + count += skb_shinfo(skb)->nr_frags; + + spin_lock_irqsave(&tx->lock, flags); + + if (tsnep_tx_desc_available(tx) < count) { + /* ring full, shall not happen because queue is stopped if full + * below + */ + netif_stop_queue(tx->adapter->netdev); + + spin_unlock_irqrestore(&tx->lock, flags); + + return NETDEV_TX_BUSY; + } + + entry = &tx->entry[tx->write]; + entry->skb = skb; + + retval = tsnep_tx_map(skb, tx, count); + if (retval != 0) { + tsnep_tx_unmap(tx, count); + dev_kfree_skb_any(entry->skb); + entry->skb = NULL; + + tx->dropped++; + + spin_unlock_irqrestore(&tx->lock, flags); + + netdev_err(tx->adapter->netdev, "TX DMA map failed\n"); + + return NETDEV_TX_OK; + } + + if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + + for (i = 0; i < count; i++) + tsnep_tx_activate(tx, (tx->write + i) % TSNEP_RING_SIZE, + i == (count - 1)); + tx->write = (tx->write + count) % TSNEP_RING_SIZE; + + skb_tx_timestamp(skb); + + /* descriptor properties shall be valid before hardware is notified */ + dma_wmb(); + + iowrite32(TSNEP_CONTROL_TX_ENABLE, tx->addr + TSNEP_CONTROL); + + if (tsnep_tx_desc_available(tx) < (MAX_SKB_FRAGS + 1)) { + /* ring can get full with next frame */ + netif_stop_queue(tx->adapter->netdev); + } + + tx->packets++; + tx->bytes += skb_pagelen(entry->skb) + ETH_FCS_LEN; + + spin_unlock_irqrestore(&tx->lock, flags); + + return NETDEV_TX_OK; +} + +static bool tsnep_tx_poll(struct tsnep_tx *tx, int napi_budget) +{ + unsigned long flags; + int budget = 128; + struct tsnep_tx_entry *entry; + int count; + + spin_lock_irqsave(&tx->lock, flags); + + do { + if (tx->read == tx->write) + break; + + entry = &tx->entry[tx->read]; + if ((__le32_to_cpu(entry->desc_wb->properties) & + TSNEP_TX_DESC_OWNER_MASK) != + (entry->properties & TSNEP_TX_DESC_OWNER_MASK)) + break; + + /* descriptor properties shall be read first, because valid data + * is signaled there + */ + dma_rmb(); + + count = 1; + if (skb_shinfo(entry->skb)->nr_frags > 0) + count += skb_shinfo(entry->skb)->nr_frags; + + tsnep_tx_unmap(tx, count); + + if ((skb_shinfo(entry->skb)->tx_flags & SKBTX_IN_PROGRESS) && + (__le32_to_cpu(entry->desc_wb->properties) & + TSNEP_DESC_EXTENDED_WRITEBACK_FLAG)) { + struct skb_shared_hwtstamps hwtstamps; + u64 timestamp = + __le64_to_cpu(entry->desc_wb->timestamp); + + memset(&hwtstamps, 0, sizeof(hwtstamps)); + hwtstamps.hwtstamp = ns_to_ktime(timestamp); + + skb_tstamp_tx(entry->skb, &hwtstamps); + } + + napi_consume_skb(entry->skb, budget); + entry->skb = NULL; + + tx->read = (tx->read + count) % TSNEP_RING_SIZE; + + budget--; + } while (likely(budget)); + + if ((tsnep_tx_desc_available(tx) >= ((MAX_SKB_FRAGS + 1) * 2)) && + netif_queue_stopped(tx->adapter->netdev)) { + netif_wake_queue(tx->adapter->netdev); + } + + spin_unlock_irqrestore(&tx->lock, flags); + + return (budget != 0); +} + +static int tsnep_tx_open(struct tsnep_adapter *adapter, void __iomem *addr, + struct tsnep_tx *tx) +{ + dma_addr_t dma; + int retval; + + memset(tx, 0, sizeof(*tx)); + tx->adapter = adapter; + tx->addr = addr; + + retval = tsnep_tx_ring_init(tx); + if (retval) + return retval; + + dma = tx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; + iowrite32(DMA_ADDR_LOW(dma), tx->addr + TSNEP_TX_DESC_ADDR_LOW); + iowrite32(DMA_ADDR_HIGH(dma), tx->addr + TSNEP_TX_DESC_ADDR_HIGH); + tx->owner_counter = 1; + tx->increment_owner_counter = TSNEP_RING_SIZE - 1; + + spin_lock_init(&tx->lock); + + return 0; +} + +static void tsnep_tx_close(struct tsnep_tx *tx) +{ + u32 val; + + readx_poll_timeout(ioread32, tx->addr + TSNEP_CONTROL, val, + ((val & TSNEP_CONTROL_TX_ENABLE) == 0), 10000, + 1000000); + + tsnep_tx_ring_cleanup(tx); +} + +static void tsnep_rx_ring_cleanup(struct tsnep_rx *rx) +{ + struct device *dmadev = rx->adapter->dmadev; + struct tsnep_rx_entry *entry; + int i; + + for (i = 0; i < TSNEP_RING_SIZE; i++) { + entry = &rx->entry[i]; + if (dma_unmap_addr(entry, dma)) + dma_unmap_single(dmadev, dma_unmap_addr(entry, dma), + dma_unmap_len(entry, len), + DMA_FROM_DEVICE); + if (entry->skb) + dev_kfree_skb(entry->skb); + } + + memset(rx->entry, 0, sizeof(rx->entry)); + + for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { + if (rx->page[i]) { + dma_free_coherent(dmadev, PAGE_SIZE, rx->page[i], + rx->page_dma[i]); + rx->page[i] = NULL; + rx->page_dma[i] = 0; + } + } +} + +static int tsnep_rx_alloc_and_map_skb(struct tsnep_rx *rx, + struct tsnep_rx_entry *entry) +{ + struct device *dmadev = rx->adapter->dmadev; + struct sk_buff *skb; + dma_addr_t dma; + + skb = __netdev_alloc_skb(rx->adapter->netdev, RX_SKB_ALLOC_LENGTH, + GFP_ATOMIC | GFP_DMA); + if (!skb) + return -ENOMEM; + + skb_reserve(skb, RX_SKB_RESERVE); + + dma = dma_map_single(dmadev, skb->data, RX_SKB_LENGTH, + DMA_FROM_DEVICE); + if (dma_mapping_error(dmadev, dma)) { + dev_kfree_skb(skb); + return -ENOMEM; + } + + entry->skb = skb; + entry->len = RX_SKB_LENGTH; + dma_unmap_addr_set(entry, dma, dma); + entry->desc->rx = __cpu_to_le64(dma); + + return 0; +} + +static int tsnep_rx_ring_init(struct tsnep_rx *rx) +{ + struct device *dmadev = rx->adapter->dmadev; + struct tsnep_rx_entry *entry; + struct tsnep_rx_entry *next_entry; + int i, j; + int retval; + + for (i = 0; i < TSNEP_RING_PAGE_COUNT; i++) { + rx->page[i] = + dma_alloc_coherent(dmadev, PAGE_SIZE, &rx->page_dma[i], + GFP_KERNEL); + if (!rx->page[i]) { + retval = -ENOMEM; + goto failed; + } + for (j = 0; j < TSNEP_RING_ENTRIES_PER_PAGE; j++) { + entry = &rx->entry[TSNEP_RING_ENTRIES_PER_PAGE * i + j]; + entry->desc_wb = (struct tsnep_rx_desc_wb *) + (((u8 *)rx->page[i]) + TSNEP_DESC_SIZE * j); + entry->desc = (struct tsnep_rx_desc *) + (((u8 *)entry->desc_wb) + TSNEP_DESC_OFFSET); + entry->desc_dma = rx->page_dma[i] + TSNEP_DESC_SIZE * j; + } + } + for (i = 0; i < TSNEP_RING_SIZE; i++) { + entry = &rx->entry[i]; + next_entry = &rx->entry[(i + 1) % TSNEP_RING_SIZE]; + entry->desc->next = __cpu_to_le64(next_entry->desc_dma); + + retval = tsnep_rx_alloc_and_map_skb(rx, entry); + if (retval) + goto failed; + } + + return 0; + +failed: + tsnep_rx_ring_cleanup(rx); + return retval; +} + +static void tsnep_rx_activate(struct tsnep_rx *rx, int index) +{ + struct tsnep_rx_entry *entry = &rx->entry[index]; + + /* RX_SKB_LENGTH is a multiple of 4 */ + entry->properties = entry->len & TSNEP_DESC_LENGTH_MASK; + entry->properties |= TSNEP_DESC_INTERRUPT_FLAG; + if (index == rx->increment_owner_counter) { + rx->owner_counter++; + if (rx->owner_counter == 4) + rx->owner_counter = 1; + rx->increment_owner_counter--; + if (rx->increment_owner_counter < 0) + rx->increment_owner_counter = TSNEP_RING_SIZE - 1; + } + entry->properties |= + (rx->owner_counter << TSNEP_DESC_OWNER_COUNTER_SHIFT) & + TSNEP_DESC_OWNER_COUNTER_MASK; + + /* descriptor properties shall be written last, because valid data is + * signaled there + */ + dma_wmb(); + + entry->desc->properties = __cpu_to_le32(entry->properties); +} + +static int tsnep_rx_poll(struct tsnep_rx *rx, struct napi_struct *napi, + int budget) +{ + struct device *dmadev = rx->adapter->dmadev; + int done = 0; + struct tsnep_rx_entry *entry; + struct sk_buff *skb; + size_t len; + dma_addr_t dma; + int length; + bool enable = false; + int retval; + + while (likely(done < budget)) { + entry = &rx->entry[rx->read]; + if ((__le32_to_cpu(entry->desc_wb->properties) & + TSNEP_DESC_OWNER_COUNTER_MASK) != + (entry->properties & TSNEP_DESC_OWNER_COUNTER_MASK)) + break; + + /* descriptor properties shall be read first, because valid data + * is signaled there + */ + dma_rmb(); + + skb = entry->skb; + len = dma_unmap_len(entry, len); + dma = dma_unmap_addr(entry, dma); + + /* forward skb only if allocation is successful, otherwise + * skb is reused and frame dropped + */ + retval = tsnep_rx_alloc_and_map_skb(rx, entry); + if (!retval) { + dma_unmap_single(dmadev, dma, len, DMA_FROM_DEVICE); + + length = __le32_to_cpu(entry->desc_wb->properties) & + TSNEP_DESC_LENGTH_MASK; + skb_put(skb, length - ETH_FCS_LEN); + if (rx->adapter->hwtstamp_config.rx_filter == + HWTSTAMP_FILTER_ALL) { + struct skb_shared_hwtstamps *hwtstamps = + skb_hwtstamps(skb); + struct tsnep_rx_inline *rx_inline = + (struct tsnep_rx_inline *)skb->data; + u64 timestamp = + __le64_to_cpu(rx_inline->timestamp); + + memset(hwtstamps, 0, sizeof(*hwtstamps)); + hwtstamps->hwtstamp = ns_to_ktime(timestamp); + } + skb_pull(skb, TSNEP_RX_INLINE_METADATA_SIZE); + skb->protocol = eth_type_trans(skb, + rx->adapter->netdev); + + rx->packets++; + rx->bytes += length - TSNEP_RX_INLINE_METADATA_SIZE; + if (skb->pkt_type == PACKET_MULTICAST) + rx->multicast++; + + napi_gro_receive(napi, skb); + done++; + } else { + rx->dropped++; + } + + tsnep_rx_activate(rx, rx->read); + + enable = true; + + rx->read = (rx->read + 1) % TSNEP_RING_SIZE; + } + + if (enable) { + /* descriptor properties shall be valid before hardware is + * notified + */ + dma_wmb(); + + iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); + } + + return done; +} + +static int tsnep_rx_open(struct tsnep_adapter *adapter, void __iomem *addr, + struct tsnep_rx *rx) +{ + dma_addr_t dma; + int i; + int retval; + + memset(rx, 0, sizeof(*rx)); + rx->adapter = adapter; + rx->addr = addr; + + retval = tsnep_rx_ring_init(rx); + if (retval) + return retval; + + dma = rx->entry[0].desc_dma | TSNEP_RESET_OWNER_COUNTER; + iowrite32(DMA_ADDR_LOW(dma), rx->addr + TSNEP_RX_DESC_ADDR_LOW); + iowrite32(DMA_ADDR_HIGH(dma), rx->addr + TSNEP_RX_DESC_ADDR_HIGH); + rx->owner_counter = 1; + rx->increment_owner_counter = TSNEP_RING_SIZE - 1; + + for (i = 0; i < TSNEP_RING_SIZE; i++) + tsnep_rx_activate(rx, i); + + /* descriptor properties shall be valid before hardware is notified */ + dma_wmb(); + + iowrite32(TSNEP_CONTROL_RX_ENABLE, rx->addr + TSNEP_CONTROL); + + return 0; +} + +static void tsnep_rx_close(struct tsnep_rx *rx) +{ + u32 val; + + iowrite32(TSNEP_CONTROL_RX_DISABLE, rx->addr + TSNEP_CONTROL); + readx_poll_timeout(ioread32, rx->addr + TSNEP_CONTROL, val, + ((val & TSNEP_CONTROL_RX_ENABLE) == 0), 10000, + 1000000); + + tsnep_rx_ring_cleanup(rx); +} + +static int tsnep_poll(struct napi_struct *napi, int budget) +{ + struct tsnep_queue *queue = container_of(napi, struct tsnep_queue, + napi); + bool complete = true; + int done = 0; + + if (queue->tx) + complete = tsnep_tx_poll(queue->tx, budget); + + if (queue->rx) { + done = tsnep_rx_poll(queue->rx, napi, budget); + if (done >= budget) + complete = false; + } + + /* if all work not completed, return budget and keep polling */ + if (!complete) + return budget; + + if (likely(napi_complete_done(napi, done))) + tsnep_enable_irq(queue->adapter, queue->irq_mask); + + return min(done, budget - 1); +} + +static int tsnep_netdev_open(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int i; + void __iomem *addr; + int tx_queue_index = 0; + int rx_queue_index = 0; + int retval; + + retval = tsnep_phy_open(adapter); + if (retval) + return retval; + + for (i = 0; i < adapter->num_queues; i++) { + adapter->queue[i].adapter = adapter; + if (adapter->queue[i].tx) { + addr = adapter->addr + TSNEP_QUEUE(tx_queue_index); + retval = tsnep_tx_open(adapter, addr, + adapter->queue[i].tx); + if (retval) + goto failed; + tx_queue_index++; + } + if (adapter->queue[i].rx) { + addr = adapter->addr + TSNEP_QUEUE(rx_queue_index); + retval = tsnep_rx_open(adapter, addr, + adapter->queue[i].rx); + if (retval) + goto failed; + rx_queue_index++; + } + } + + retval = netif_set_real_num_tx_queues(adapter->netdev, + adapter->num_tx_queues); + if (retval) + goto failed; + retval = netif_set_real_num_rx_queues(adapter->netdev, + adapter->num_rx_queues); + if (retval) + goto failed; + + for (i = 0; i < adapter->num_queues; i++) { + netif_napi_add(adapter->netdev, &adapter->queue[i].napi, + tsnep_poll, 64); + napi_enable(&adapter->queue[i].napi); + + tsnep_enable_irq(adapter, adapter->queue[i].irq_mask); + } + + return 0; + +failed: + for (i = 0; i < adapter->num_queues; i++) { + if (adapter->queue[i].rx) + tsnep_rx_close(adapter->queue[i].rx); + if (adapter->queue[i].tx) + tsnep_tx_close(adapter->queue[i].tx); + } + tsnep_phy_close(adapter); + return retval; +} + +static int tsnep_netdev_close(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + int i; + + for (i = 0; i < adapter->num_queues; i++) { + tsnep_disable_irq(adapter, adapter->queue[i].irq_mask); + + napi_disable(&adapter->queue[i].napi); + netif_napi_del(&adapter->queue[i].napi); + + if (adapter->queue[i].rx) + tsnep_rx_close(adapter->queue[i].rx); + if (adapter->queue[i].tx) + tsnep_tx_close(adapter->queue[i].tx); + } + + tsnep_phy_close(adapter); + + return 0; +} + +static netdev_tx_t tsnep_netdev_xmit_frame(struct sk_buff *skb, + struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + u16 queue_mapping = skb_get_queue_mapping(skb); + + if (queue_mapping >= adapter->num_tx_queues) + queue_mapping = 0; + + return tsnep_xmit_frame_ring(skb, &adapter->tx[queue_mapping]); +} + +static int tsnep_netdev_ioctl(struct net_device *netdev, struct ifreq *ifr, + int cmd) +{ + if (!netif_running(netdev)) + return -EINVAL; + if (cmd == SIOCSHWTSTAMP || cmd == SIOCGHWTSTAMP) + return tsnep_ptp_ioctl(netdev, ifr, cmd); + return phy_mii_ioctl(netdev->phydev, ifr, cmd); +} + +static void tsnep_netdev_set_multicast(struct net_device *netdev) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + u16 rx_filter = 0; + + /* configured MAC address and broadcasts are never filtered */ + if (netdev->flags & IFF_PROMISC) { + rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; + rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_UNICASTS; + } else if (!netdev_mc_empty(netdev) || (netdev->flags & IFF_ALLMULTI)) { + rx_filter |= TSNEP_RX_FILTER_ACCEPT_ALL_MULTICASTS; + } + iowrite16(rx_filter, adapter->addr + TSNEP_RX_FILTER); +} + +static void tsnep_netdev_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + u32 reg; + u32 val; + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) { + stats->tx_packets += adapter->tx[i].packets; + stats->tx_bytes += adapter->tx[i].bytes; + stats->tx_dropped += adapter->tx[i].dropped; + } + for (i = 0; i < adapter->num_rx_queues; i++) { + stats->rx_packets += adapter->rx[i].packets; + stats->rx_bytes += adapter->rx[i].bytes; + stats->rx_dropped += adapter->rx[i].dropped; + stats->multicast += adapter->rx[i].multicast; + + reg = ioread32(adapter->addr + TSNEP_QUEUE(i) + + TSNEP_RX_STATISTIC); + val = (reg & TSNEP_RX_STATISTIC_NO_DESC_MASK) >> + TSNEP_RX_STATISTIC_NO_DESC_SHIFT; + stats->rx_dropped += val; + val = (reg & TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_MASK) >> + TSNEP_RX_STATISTIC_BUFFER_TOO_SMALL_SHIFT; + stats->rx_dropped += val; + val = (reg & TSNEP_RX_STATISTIC_FIFO_OVERFLOW_MASK) >> + TSNEP_RX_STATISTIC_FIFO_OVERFLOW_SHIFT; + stats->rx_errors += val; + stats->rx_fifo_errors += val; + val = (reg & TSNEP_RX_STATISTIC_INVALID_FRAME_MASK) >> + TSNEP_RX_STATISTIC_INVALID_FRAME_SHIFT; + stats->rx_errors += val; + stats->rx_frame_errors += val; + } + + reg = ioread32(adapter->addr + ECM_STAT); + val = (reg & ECM_STAT_RX_ERR_MASK) >> ECM_STAT_RX_ERR_SHIFT; + stats->rx_errors += val; + val = (reg & ECM_STAT_INV_FRM_MASK) >> ECM_STAT_INV_FRM_SHIFT; + stats->rx_errors += val; + stats->rx_crc_errors += val; + val = (reg & ECM_STAT_FWD_RX_ERR_MASK) >> ECM_STAT_FWD_RX_ERR_SHIFT; + stats->rx_errors += val; +} + +static void tsnep_mac_set_address(struct tsnep_adapter *adapter, u8 *addr) +{ + iowrite32(*(u32 *)addr, adapter->addr + TSNEP_MAC_ADDRESS_LOW); + iowrite16(*(u16 *)(addr + sizeof(u32)), + adapter->addr + TSNEP_MAC_ADDRESS_HIGH); + + ether_addr_copy(adapter->mac_address, addr); + netif_info(adapter, drv, adapter->netdev, "MAC address set to %pM\n", + addr); +} + +static int tsnep_netdev_set_mac_address(struct net_device *netdev, void *addr) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct sockaddr *sock_addr = addr; + int retval; + + retval = eth_prepare_mac_addr_change(netdev, sock_addr); + if (retval) + return retval; + eth_hw_addr_set(netdev, sock_addr->sa_data); + tsnep_mac_set_address(adapter, sock_addr->sa_data); + + return 0; +} + +static const struct net_device_ops tsnep_netdev_ops = { + .ndo_open = tsnep_netdev_open, + .ndo_stop = tsnep_netdev_close, + .ndo_start_xmit = tsnep_netdev_xmit_frame, + .ndo_eth_ioctl = tsnep_netdev_ioctl, + .ndo_set_rx_mode = tsnep_netdev_set_multicast, + + .ndo_get_stats64 = tsnep_netdev_get_stats64, + .ndo_set_mac_address = tsnep_netdev_set_mac_address, + .ndo_setup_tc = tsnep_tc_setup, +}; + +static int tsnep_mac_init(struct tsnep_adapter *adapter) +{ + int retval; + + /* initialize RX filtering, at least configured MAC address and + * broadcast are not filtered + */ + iowrite16(0, adapter->addr + TSNEP_RX_FILTER); + + /* try to get MAC address in the following order: + * - device tree + * - valid MAC address already set + * - MAC address register if valid + * - random MAC address + */ + retval = of_get_mac_address(adapter->pdev->dev.of_node, + adapter->mac_address); + if (retval == -EPROBE_DEFER) + return retval; + if (retval && !is_valid_ether_addr(adapter->mac_address)) { + *(u32 *)adapter->mac_address = + ioread32(adapter->addr + TSNEP_MAC_ADDRESS_LOW); + *(u16 *)(adapter->mac_address + sizeof(u32)) = + ioread16(adapter->addr + TSNEP_MAC_ADDRESS_HIGH); + if (!is_valid_ether_addr(adapter->mac_address)) + eth_random_addr(adapter->mac_address); + } + + tsnep_mac_set_address(adapter, adapter->mac_address); + eth_hw_addr_set(adapter->netdev, adapter->mac_address); + + return 0; +} + +static int tsnep_mdio_init(struct tsnep_adapter *adapter) +{ + struct device_node *np = adapter->pdev->dev.of_node; + int retval; + + if (np) { + np = of_get_child_by_name(np, "mdio"); + if (!np) + return 0; + + adapter->suppress_preamble = + of_property_read_bool(np, "suppress-preamble"); + } + + adapter->mdiobus = devm_mdiobus_alloc(&adapter->pdev->dev); + if (!adapter->mdiobus) { + retval = -ENOMEM; + + goto out; + } + + adapter->mdiobus->priv = (void *)adapter; + adapter->mdiobus->parent = &adapter->pdev->dev; + adapter->mdiobus->read = tsnep_mdiobus_read; + adapter->mdiobus->write = tsnep_mdiobus_write; + adapter->mdiobus->name = TSNEP "-mdiobus"; + snprintf(adapter->mdiobus->id, MII_BUS_ID_SIZE, "%s", + adapter->pdev->name); + + /* do not scan broadcast address */ + adapter->mdiobus->phy_mask = 0x0000001; + + retval = of_mdiobus_register(adapter->mdiobus, np); + +out: + if (np) + of_node_put(np); + + return retval; +} + +static int tsnep_phy_init(struct tsnep_adapter *adapter) +{ + struct device_node *phy_node; + int retval; + + retval = of_get_phy_mode(adapter->pdev->dev.of_node, + &adapter->phy_mode); + if (retval) + adapter->phy_mode = PHY_INTERFACE_MODE_GMII; + + phy_node = of_parse_phandle(adapter->pdev->dev.of_node, "phy-handle", + 0); + adapter->phydev = of_phy_find_device(phy_node); + of_node_put(phy_node); + if (!adapter->phydev && adapter->mdiobus) + adapter->phydev = phy_find_first(adapter->mdiobus); + if (!adapter->phydev) + return -EIO; + + return 0; +} + +static int tsnep_probe(struct platform_device *pdev) +{ + struct tsnep_adapter *adapter; + struct net_device *netdev; + struct resource *io; + u32 type; + int revision; + int version; + int retval; + + netdev = devm_alloc_etherdev_mqs(&pdev->dev, + sizeof(struct tsnep_adapter), + TSNEP_MAX_QUEUES, TSNEP_MAX_QUEUES); + if (!netdev) + return -ENODEV; + SET_NETDEV_DEV(netdev, &pdev->dev); + adapter = netdev_priv(netdev); + platform_set_drvdata(pdev, adapter); + adapter->pdev = pdev; + adapter->dmadev = &pdev->dev; + adapter->netdev = netdev; + adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE | + NETIF_MSG_LINK | NETIF_MSG_IFUP | + NETIF_MSG_IFDOWN | NETIF_MSG_TX_QUEUED; + + netdev->min_mtu = ETH_MIN_MTU; + netdev->max_mtu = TSNEP_MAX_FRAME_SIZE; + + mutex_init(&adapter->gate_control_lock); + + io = platform_get_resource(pdev, IORESOURCE_MEM, 0); + adapter->addr = devm_ioremap_resource(&pdev->dev, io); + if (IS_ERR(adapter->addr)) + return PTR_ERR(adapter->addr); + adapter->irq = platform_get_irq(pdev, 0); + netdev->mem_start = io->start; + netdev->mem_end = io->end; + netdev->irq = adapter->irq; + + type = ioread32(adapter->addr + ECM_TYPE); + revision = (type & ECM_REVISION_MASK) >> ECM_REVISION_SHIFT; + version = (type & ECM_VERSION_MASK) >> ECM_VERSION_SHIFT; + adapter->gate_control = type & ECM_GATE_CONTROL; + + adapter->num_tx_queues = TSNEP_QUEUES; + adapter->num_rx_queues = TSNEP_QUEUES; + adapter->num_queues = TSNEP_QUEUES; + adapter->queue[0].tx = &adapter->tx[0]; + adapter->queue[0].rx = &adapter->rx[0]; + adapter->queue[0].irq_mask = ECM_INT_TX_0 | ECM_INT_RX_0; + + tsnep_disable_irq(adapter, ECM_INT_ALL); + retval = devm_request_irq(&adapter->pdev->dev, adapter->irq, tsnep_irq, + 0, TSNEP, adapter); + if (retval != 0) { + dev_err(&adapter->pdev->dev, "can't get assigned irq %d.\n", + adapter->irq); + return retval; + } + tsnep_enable_irq(adapter, ECM_INT_LINK); + + retval = tsnep_mac_init(adapter); + if (retval) + goto mac_init_failed; + + retval = tsnep_mdio_init(adapter); + if (retval) + goto mdio_init_failed; + + retval = tsnep_phy_init(adapter); + if (retval) + goto phy_init_failed; + + retval = tsnep_ptp_init(adapter); + if (retval) + goto ptp_init_failed; + + retval = tsnep_tc_init(adapter); + if (retval) + goto tc_init_failed; + + netdev->netdev_ops = &tsnep_netdev_ops; + netdev->ethtool_ops = &tsnep_ethtool_ops; + netdev->features = NETIF_F_SG; + netdev->hw_features = netdev->features; + + /* carrier off reporting is important to ethtool even BEFORE open */ + netif_carrier_off(netdev); + + retval = register_netdev(netdev); + if (retval) + goto register_failed; + + dev_info(&adapter->pdev->dev, "device version %d.%02d\n", version, + revision); + if (adapter->gate_control) + dev_info(&adapter->pdev->dev, "gate control detected\n"); + + return 0; + +register_failed: + tsnep_tc_cleanup(adapter); +tc_init_failed: + tsnep_ptp_cleanup(adapter); +ptp_init_failed: +phy_init_failed: + if (adapter->mdiobus) + mdiobus_unregister(adapter->mdiobus); +mdio_init_failed: +mac_init_failed: + tsnep_disable_irq(adapter, ECM_INT_ALL); + return retval; +} + +static int tsnep_remove(struct platform_device *pdev) +{ + struct tsnep_adapter *adapter = platform_get_drvdata(pdev); + + unregister_netdev(adapter->netdev); + + tsnep_tc_cleanup(adapter); + + tsnep_ptp_cleanup(adapter); + + if (adapter->mdiobus) + mdiobus_unregister(adapter->mdiobus); + + tsnep_disable_irq(adapter, ECM_INT_ALL); + + return 0; +} + +static const struct of_device_id tsnep_of_match[] = { + { .compatible = "engleder,tsnep", }, +{ }, +}; +MODULE_DEVICE_TABLE(of, tsnep_of_match); + +static struct platform_driver tsnep_driver = { + .driver = { + .name = TSNEP, + .of_match_table = of_match_ptr(tsnep_of_match), + }, + .probe = tsnep_probe, + .remove = tsnep_remove, +}; +module_platform_driver(tsnep_driver); + +MODULE_AUTHOR("Gerhard Engleder <gerhard@engleder-embedded.com>"); +MODULE_DESCRIPTION("TSN endpoint Ethernet MAC driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/net/ethernet/engleder/tsnep_ptp.c b/drivers/net/ethernet/engleder/tsnep_ptp.c new file mode 100644 index 000000000000..4bfb4d8508f5 --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_ptp.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +#include "tsnep.h" + +void tsnep_get_system_time(struct tsnep_adapter *adapter, u64 *time) +{ + u32 high_before; + u32 low; + u32 high; + + /* read high dword twice to detect overrun */ + high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + do { + low = ioread32(adapter->addr + ECM_SYSTEM_TIME_LOW); + high_before = high; + high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + } while (high != high_before); + *time = (((u64)high) << 32) | ((u64)low); +} + +int tsnep_ptp_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + struct hwtstamp_config config; + + if (!ifr) + return -EINVAL; + + if (cmd == SIOCSHWTSTAMP) { + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + if (config.flags) + return -EINVAL; + + switch (config.tx_type) { + case HWTSTAMP_TX_OFF: + case HWTSTAMP_TX_ON: + break; + default: + return -ERANGE; + } + + switch (config.rx_filter) { + case HWTSTAMP_FILTER_NONE: + break; + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_NTP_ALL: + config.rx_filter = HWTSTAMP_FILTER_ALL; + break; + default: + return -ERANGE; + } + + memcpy(&adapter->hwtstamp_config, &config, + sizeof(adapter->hwtstamp_config)); + } + + if (copy_to_user(ifr->ifr_data, &adapter->hwtstamp_config, + sizeof(adapter->hwtstamp_config))) + return -EFAULT; + + return 0; +} + +static int tsnep_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct tsnep_adapter *adapter = container_of(ptp, struct tsnep_adapter, + ptp_clock_info); + bool negative = false; + u64 rate_offset; + + if (scaled_ppm < 0) { + scaled_ppm = -scaled_ppm; + negative = true; + } + + /* convert from 16 bit to 32 bit binary fractional, divide by 1000000 to + * eliminate ppm, multiply with 8 to compensate 8ns clock cycle time, + * simplify calculation because 15625 * 8 = 1000000 / 8 + */ + rate_offset = scaled_ppm; + rate_offset <<= 16 - 3; + rate_offset = div_u64(rate_offset, 15625); + + rate_offset &= ECM_CLOCK_RATE_OFFSET_MASK; + if (negative) + rate_offset |= ECM_CLOCK_RATE_OFFSET_SIGN; + iowrite32(rate_offset & 0xFFFFFFFF, adapter->addr + ECM_CLOCK_RATE); + + return 0; +} + +static int tsnep_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct tsnep_adapter *adapter = container_of(ptp, struct tsnep_adapter, + ptp_clock_info); + u64 system_time; + unsigned long flags; + + spin_lock_irqsave(&adapter->ptp_lock, flags); + + tsnep_get_system_time(adapter, &system_time); + + system_time += delta; + + /* high dword is buffered in hardware and synchronously written to + * system time when low dword is written + */ + iowrite32(system_time >> 32, adapter->addr + ECM_SYSTEM_TIME_HIGH); + iowrite32(system_time & 0xFFFFFFFF, + adapter->addr + ECM_SYSTEM_TIME_LOW); + + spin_unlock_irqrestore(&adapter->ptp_lock, flags); + + return 0; +} + +static int tsnep_ptp_gettimex64(struct ptp_clock_info *ptp, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct tsnep_adapter *adapter = container_of(ptp, struct tsnep_adapter, + ptp_clock_info); + u32 high_before; + u32 low; + u32 high; + u64 system_time; + + /* read high dword twice to detect overrun */ + high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + do { + ptp_read_system_prets(sts); + low = ioread32(adapter->addr + ECM_SYSTEM_TIME_LOW); + ptp_read_system_postts(sts); + high_before = high; + high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + } while (high != high_before); + system_time = (((u64)high) << 32) | ((u64)low); + + *ts = ns_to_timespec64(system_time); + + return 0; +} + +static int tsnep_ptp_settime64(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct tsnep_adapter *adapter = container_of(ptp, struct tsnep_adapter, + ptp_clock_info); + u64 system_time = timespec64_to_ns(ts); + unsigned long flags; + + spin_lock_irqsave(&adapter->ptp_lock, flags); + + /* high dword is buffered in hardware and synchronously written to + * system time when low dword is written + */ + iowrite32(system_time >> 32, adapter->addr + ECM_SYSTEM_TIME_HIGH); + iowrite32(system_time & 0xFFFFFFFF, + adapter->addr + ECM_SYSTEM_TIME_LOW); + + spin_unlock_irqrestore(&adapter->ptp_lock, flags); + + return 0; +} + +int tsnep_ptp_init(struct tsnep_adapter *adapter) +{ + int retval = 0; + + adapter->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; + adapter->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; + + snprintf(adapter->ptp_clock_info.name, 16, "%s", TSNEP); + adapter->ptp_clock_info.owner = THIS_MODULE; + /* at most 2^-1ns adjustment every clock cycle for 8ns clock cycle time, + * stay slightly below because only bits below 2^-1ns are supported + */ + adapter->ptp_clock_info.max_adj = (500000000 / 8 - 1); + adapter->ptp_clock_info.adjfine = tsnep_ptp_adjfine; + adapter->ptp_clock_info.adjtime = tsnep_ptp_adjtime; + adapter->ptp_clock_info.gettimex64 = tsnep_ptp_gettimex64; + adapter->ptp_clock_info.settime64 = tsnep_ptp_settime64; + + spin_lock_init(&adapter->ptp_lock); + + adapter->ptp_clock = ptp_clock_register(&adapter->ptp_clock_info, + &adapter->pdev->dev); + if (IS_ERR(adapter->ptp_clock)) { + netdev_err(adapter->netdev, "ptp_clock_register failed\n"); + + retval = PTR_ERR(adapter->ptp_clock); + adapter->ptp_clock = NULL; + } else if (adapter->ptp_clock) { + netdev_info(adapter->netdev, "PHC added\n"); + } + + return retval; +} + +void tsnep_ptp_cleanup(struct tsnep_adapter *adapter) +{ + if (adapter->ptp_clock) { + ptp_clock_unregister(adapter->ptp_clock); + netdev_info(adapter->netdev, "PHC removed\n"); + } +} diff --git a/drivers/net/ethernet/engleder/tsnep_selftests.c b/drivers/net/ethernet/engleder/tsnep_selftests.c new file mode 100644 index 000000000000..1581d6b22232 --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_selftests.c @@ -0,0 +1,811 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +#include "tsnep.h" + +#include <net/pkt_sched.h> + +enum tsnep_test { + TSNEP_TEST_ENABLE = 0, + TSNEP_TEST_TAPRIO, + TSNEP_TEST_TAPRIO_CHANGE, + TSNEP_TEST_TAPRIO_EXTENSION, +}; + +static const char tsnep_test_strings[][ETH_GSTRING_LEN] = { + "Enable timeout (offline)", + "TAPRIO (offline)", + "TAPRIO change (offline)", + "TAPRIO extension (offline)", +}; + +#define TSNEP_TEST_COUNT (sizeof(tsnep_test_strings) / ETH_GSTRING_LEN) + +static bool enable_gc_timeout(struct tsnep_adapter *adapter) +{ + iowrite8(TSNEP_GC_ENABLE_TIMEOUT, adapter->addr + TSNEP_GC); + if (!(ioread32(adapter->addr + TSNEP_GC) & TSNEP_GC_TIMEOUT_ACTIVE)) + return false; + + return true; +} + +static bool gc_timeout_signaled(struct tsnep_adapter *adapter) +{ + if (ioread32(adapter->addr + TSNEP_GC) & TSNEP_GC_TIMEOUT_SIGNAL) + return true; + + return false; +} + +static bool ack_gc_timeout(struct tsnep_adapter *adapter) +{ + iowrite8(TSNEP_GC_ENABLE_TIMEOUT, adapter->addr + TSNEP_GC); + if (ioread32(adapter->addr + TSNEP_GC) & + (TSNEP_GC_TIMEOUT_ACTIVE | TSNEP_GC_TIMEOUT_SIGNAL)) + return false; + return true; +} + +static bool enable_gc(struct tsnep_adapter *adapter, bool a) +{ + u8 enable; + u8 active; + + if (a) { + enable = TSNEP_GC_ENABLE_A; + active = TSNEP_GC_ACTIVE_A; + } else { + enable = TSNEP_GC_ENABLE_B; + active = TSNEP_GC_ACTIVE_B; + } + + iowrite8(enable, adapter->addr + TSNEP_GC); + if (!(ioread32(adapter->addr + TSNEP_GC) & active)) + return false; + + return true; +} + +static bool disable_gc(struct tsnep_adapter *adapter) +{ + iowrite8(TSNEP_GC_DISABLE, adapter->addr + TSNEP_GC); + if (ioread32(adapter->addr + TSNEP_GC) & + (TSNEP_GC_ACTIVE_A | TSNEP_GC_ACTIVE_B)) + return false; + + return true; +} + +static bool gc_delayed_enable(struct tsnep_adapter *adapter, bool a, int delay) +{ + u64 before, after; + u32 time; + bool enabled; + + if (!disable_gc(adapter)) + return false; + + before = ktime_get_ns(); + + if (!enable_gc_timeout(adapter)) + return false; + + /* for start time after timeout, the timeout can guarantee, that enable + * is blocked if too late + */ + time = ioread32(adapter->addr + ECM_SYSTEM_TIME_LOW); + time += TSNEP_GC_TIMEOUT; + iowrite32(time, adapter->addr + TSNEP_GC_TIME); + + ndelay(delay); + + enabled = enable_gc(adapter, a); + after = ktime_get_ns(); + + if (delay > TSNEP_GC_TIMEOUT) { + /* timeout must have blocked enable */ + if (enabled) + return false; + } else if ((after - before) < TSNEP_GC_TIMEOUT * 14 / 16) { + /* timeout must not have blocked enable */ + if (!enabled) + return false; + } + + if (enabled) { + if (gc_timeout_signaled(adapter)) + return false; + } else { + if (!gc_timeout_signaled(adapter)) + return false; + if (!ack_gc_timeout(adapter)) + return false; + } + + if (!disable_gc(adapter)) + return false; + + return true; +} + +static bool tsnep_test_gc_enable(struct tsnep_adapter *adapter) +{ + int i; + + iowrite32(0x80000001, adapter->addr + TSNEP_GCL_A + 0); + iowrite32(100000, adapter->addr + TSNEP_GCL_A + 4); + + for (i = 0; i < 200000; i += 100) { + if (!gc_delayed_enable(adapter, true, i)) + return false; + } + + iowrite32(0x80000001, adapter->addr + TSNEP_GCL_B + 0); + iowrite32(100000, adapter->addr + TSNEP_GCL_B + 4); + + for (i = 0; i < 200000; i += 100) { + if (!gc_delayed_enable(adapter, false, i)) + return false; + } + + return true; +} + +static void delay_base_time(struct tsnep_adapter *adapter, + struct tc_taprio_qopt_offload *qopt, s64 ms) +{ + u64 system_time; + u64 base_time = ktime_to_ns(qopt->base_time); + u64 n; + + tsnep_get_system_time(adapter, &system_time); + system_time += ms * 1000000; + n = div64_u64(system_time - base_time, qopt->cycle_time); + + qopt->base_time = ktime_add_ns(qopt->base_time, + (n + 1) * qopt->cycle_time); +} + +static void get_gate_state(struct tsnep_adapter *adapter, u32 *gc, u32 *gc_time, + u64 *system_time) +{ + u32 time_high_before; + u32 time_low; + u32 time_high; + u32 gc_time_before; + + time_high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + *gc_time = ioread32(adapter->addr + TSNEP_GC_TIME); + do { + time_low = ioread32(adapter->addr + ECM_SYSTEM_TIME_LOW); + *gc = ioread32(adapter->addr + TSNEP_GC); + + gc_time_before = *gc_time; + *gc_time = ioread32(adapter->addr + TSNEP_GC_TIME); + time_high_before = time_high; + time_high = ioread32(adapter->addr + ECM_SYSTEM_TIME_HIGH); + } while ((time_high != time_high_before) || + (*gc_time != gc_time_before)); + + *system_time = (((u64)time_high) << 32) | ((u64)time_low); +} + +static int get_operation(struct tsnep_gcl *gcl, u64 system_time, u64 *next) +{ + u64 n = div64_u64(system_time - gcl->base_time, gcl->cycle_time); + u64 cycle_start = gcl->base_time + gcl->cycle_time * n; + int i; + + *next = cycle_start; + for (i = 0; i < gcl->count; i++) { + *next += gcl->operation[i].interval; + if (*next > system_time) + break; + } + + return i; +} + +static bool check_gate(struct tsnep_adapter *adapter) +{ + u32 gc_time; + u32 gc; + u64 system_time; + struct tsnep_gcl *curr; + struct tsnep_gcl *prev; + u64 next_time; + u8 gate_open; + u8 next_gate_open; + + get_gate_state(adapter, &gc, &gc_time, &system_time); + + if (gc & TSNEP_GC_ACTIVE_A) { + curr = &adapter->gcl[0]; + prev = &adapter->gcl[1]; + } else if (gc & TSNEP_GC_ACTIVE_B) { + curr = &adapter->gcl[1]; + prev = &adapter->gcl[0]; + } else { + return false; + } + if (curr->start_time <= system_time) { + /* GCL is already active */ + int index; + + index = get_operation(curr, system_time, &next_time); + gate_open = curr->operation[index].properties & TSNEP_GCL_MASK; + if (index == curr->count - 1) + index = 0; + else + index++; + next_gate_open = + curr->operation[index].properties & TSNEP_GCL_MASK; + } else if (curr->change) { + /* operation of previous GCL is active */ + int index; + u64 start_before; + u64 n; + + index = get_operation(prev, system_time, &next_time); + next_time = curr->start_time; + start_before = prev->base_time; + n = div64_u64(curr->start_time - start_before, + prev->cycle_time); + start_before += n * prev->cycle_time; + if (curr->start_time == start_before) + start_before -= prev->cycle_time; + if (((start_before + prev->cycle_time_extension) >= + curr->start_time) && + (curr->start_time - prev->cycle_time_extension <= + system_time)) { + /* extend */ + index = prev->count - 1; + } + gate_open = prev->operation[index].properties & TSNEP_GCL_MASK; + next_gate_open = + curr->operation[0].properties & TSNEP_GCL_MASK; + } else { + /* GCL is waiting for start */ + next_time = curr->start_time; + gate_open = 0xFF; + next_gate_open = curr->operation[0].properties & TSNEP_GCL_MASK; + } + + if (gc_time != (next_time & 0xFFFFFFFF)) { + dev_err(&adapter->pdev->dev, "gate control time 0x%x!=0x%llx\n", + gc_time, next_time); + return false; + } + if (((gc & TSNEP_GC_OPEN) >> TSNEP_GC_OPEN_SHIFT) != gate_open) { + dev_err(&adapter->pdev->dev, + "gate control open 0x%02x!=0x%02x\n", + ((gc & TSNEP_GC_OPEN) >> TSNEP_GC_OPEN_SHIFT), + gate_open); + return false; + } + if (((gc & TSNEP_GC_NEXT_OPEN) >> TSNEP_GC_NEXT_OPEN_SHIFT) != + next_gate_open) { + dev_err(&adapter->pdev->dev, + "gate control next open 0x%02x!=0x%02x\n", + ((gc & TSNEP_GC_NEXT_OPEN) >> TSNEP_GC_NEXT_OPEN_SHIFT), + next_gate_open); + return false; + } + + return true; +} + +static bool check_gate_duration(struct tsnep_adapter *adapter, s64 ms) +{ + ktime_t start = ktime_get(); + + do { + if (!check_gate(adapter)) + return false; + } while (ktime_ms_delta(ktime_get(), start) < ms); + + return true; +} + +static bool enable_check_taprio(struct tsnep_adapter *adapter, + struct tc_taprio_qopt_offload *qopt, s64 ms) +{ + int retval; + + retval = tsnep_tc_setup(adapter->netdev, TC_SETUP_QDISC_TAPRIO, qopt); + if (retval) + return false; + + if (!check_gate_duration(adapter, ms)) + return false; + + return true; +} + +static bool disable_taprio(struct tsnep_adapter *adapter) +{ + struct tc_taprio_qopt_offload qopt; + int retval; + + memset(&qopt, 0, sizeof(qopt)); + qopt.enable = 0; + retval = tsnep_tc_setup(adapter->netdev, TC_SETUP_QDISC_TAPRIO, &qopt); + if (retval) + return false; + + return true; +} + +static bool run_taprio(struct tsnep_adapter *adapter, + struct tc_taprio_qopt_offload *qopt, s64 ms) +{ + if (!enable_check_taprio(adapter, qopt, ms)) + return false; + + if (!disable_taprio(adapter)) + return false; + + return true; +} + +static bool tsnep_test_taprio(struct tsnep_adapter *adapter) +{ + struct tc_taprio_qopt_offload *qopt; + int i; + + qopt = kzalloc(struct_size(qopt, entries, 255), GFP_KERNEL); + if (!qopt) + return false; + for (i = 0; i < 255; i++) + qopt->entries[i].command = TC_TAPRIO_CMD_SET_GATES; + + qopt->enable = 1; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1500000; + qopt->cycle_time_extension = 0; + qopt->entries[0].gate_mask = 0x02; + qopt->entries[0].interval = 200000; + qopt->entries[1].gate_mask = 0x03; + qopt->entries[1].interval = 800000; + qopt->entries[2].gate_mask = 0x07; + qopt->entries[2].interval = 240000; + qopt->entries[3].gate_mask = 0x01; + qopt->entries[3].interval = 80000; + qopt->entries[4].gate_mask = 0x04; + qopt->entries[4].interval = 70000; + qopt->entries[5].gate_mask = 0x06; + qopt->entries[5].interval = 60000; + qopt->entries[6].gate_mask = 0x0F; + qopt->entries[6].interval = 50000; + qopt->num_entries = 7; + if (!run_taprio(adapter, qopt, 100)) + goto failed; + + qopt->enable = 1; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 411854; + qopt->cycle_time_extension = 0; + qopt->entries[0].gate_mask = 0x17; + qopt->entries[0].interval = 23842; + qopt->entries[1].gate_mask = 0x16; + qopt->entries[1].interval = 13482; + qopt->entries[2].gate_mask = 0x15; + qopt->entries[2].interval = 49428; + qopt->entries[3].gate_mask = 0x14; + qopt->entries[3].interval = 38189; + qopt->entries[4].gate_mask = 0x13; + qopt->entries[4].interval = 92321; + qopt->entries[5].gate_mask = 0x12; + qopt->entries[5].interval = 71239; + qopt->entries[6].gate_mask = 0x11; + qopt->entries[6].interval = 69932; + qopt->entries[7].gate_mask = 0x10; + qopt->entries[7].interval = 53421; + qopt->num_entries = 8; + if (!run_taprio(adapter, qopt, 100)) + goto failed; + + qopt->enable = 1; + qopt->base_time = ktime_set(0, 0); + delay_base_time(adapter, qopt, 12); + qopt->cycle_time = 125000; + qopt->cycle_time_extension = 0; + qopt->entries[0].gate_mask = 0x27; + qopt->entries[0].interval = 15000; + qopt->entries[1].gate_mask = 0x26; + qopt->entries[1].interval = 15000; + qopt->entries[2].gate_mask = 0x25; + qopt->entries[2].interval = 12500; + qopt->entries[3].gate_mask = 0x24; + qopt->entries[3].interval = 17500; + qopt->entries[4].gate_mask = 0x23; + qopt->entries[4].interval = 10000; + qopt->entries[5].gate_mask = 0x22; + qopt->entries[5].interval = 11000; + qopt->entries[6].gate_mask = 0x21; + qopt->entries[6].interval = 9000; + qopt->entries[7].gate_mask = 0x20; + qopt->entries[7].interval = 10000; + qopt->entries[8].gate_mask = 0x20; + qopt->entries[8].interval = 12500; + qopt->entries[9].gate_mask = 0x20; + qopt->entries[9].interval = 12500; + qopt->num_entries = 10; + if (!run_taprio(adapter, qopt, 100)) + goto failed; + + kfree(qopt); + + return true; + +failed: + disable_taprio(adapter); + kfree(qopt); + + return false; +} + +static bool tsnep_test_taprio_change(struct tsnep_adapter *adapter) +{ + struct tc_taprio_qopt_offload *qopt; + int i; + + qopt = kzalloc(struct_size(qopt, entries, 255), GFP_KERNEL); + if (!qopt) + return false; + for (i = 0; i < 255; i++) + qopt->entries[i].command = TC_TAPRIO_CMD_SET_GATES; + + qopt->enable = 1; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 100000; + qopt->cycle_time_extension = 0; + qopt->entries[0].gate_mask = 0x30; + qopt->entries[0].interval = 20000; + qopt->entries[1].gate_mask = 0x31; + qopt->entries[1].interval = 80000; + qopt->num_entries = 2; + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to identical */ + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + delay_base_time(adapter, qopt, 17); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to same cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->entries[0].gate_mask = 0x42; + qopt->entries[1].gate_mask = 0x43; + delay_base_time(adapter, qopt, 2); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->entries[0].gate_mask = 0x54; + qopt->entries[0].interval = 33333; + qopt->entries[1].gate_mask = 0x55; + qopt->entries[1].interval = 66667; + delay_base_time(adapter, qopt, 23); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->entries[0].gate_mask = 0x66; + qopt->entries[0].interval = 50000; + qopt->entries[1].gate_mask = 0x67; + qopt->entries[1].interval = 25000; + qopt->entries[2].gate_mask = 0x68; + qopt->entries[2].interval = 25000; + qopt->num_entries = 3; + delay_base_time(adapter, qopt, 11); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to multiple of cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 200000; + qopt->entries[0].gate_mask = 0x79; + qopt->entries[0].interval = 50000; + qopt->entries[1].gate_mask = 0x7A; + qopt->entries[1].interval = 150000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 11); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1000000; + qopt->entries[0].gate_mask = 0x7B; + qopt->entries[0].interval = 125000; + qopt->entries[1].gate_mask = 0x7C; + qopt->entries[1].interval = 250000; + qopt->entries[2].gate_mask = 0x7D; + qopt->entries[2].interval = 375000; + qopt->entries[3].gate_mask = 0x7E; + qopt->entries[3].interval = 250000; + qopt->num_entries = 4; + delay_base_time(adapter, qopt, 3); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to shorter cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 333333; + qopt->entries[0].gate_mask = 0x8F; + qopt->entries[0].interval = 166666; + qopt->entries[1].gate_mask = 0x80; + qopt->entries[1].interval = 166667; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 11); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 62500; + qopt->entries[0].gate_mask = 0x81; + qopt->entries[0].interval = 31250; + qopt->entries[1].gate_mask = 0x82; + qopt->entries[1].interval = 15625; + qopt->entries[2].gate_mask = 0x83; + qopt->entries[2].interval = 15625; + qopt->num_entries = 3; + delay_base_time(adapter, qopt, 1); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to longer cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 400000; + qopt->entries[0].gate_mask = 0x84; + qopt->entries[0].interval = 100000; + qopt->entries[1].gate_mask = 0x85; + qopt->entries[1].interval = 100000; + qopt->entries[2].gate_mask = 0x86; + qopt->entries[2].interval = 100000; + qopt->entries[3].gate_mask = 0x87; + qopt->entries[3].interval = 100000; + qopt->num_entries = 4; + delay_base_time(adapter, qopt, 7); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1700000; + qopt->entries[0].gate_mask = 0x88; + qopt->entries[0].interval = 200000; + qopt->entries[1].gate_mask = 0x89; + qopt->entries[1].interval = 300000; + qopt->entries[2].gate_mask = 0x8A; + qopt->entries[2].interval = 600000; + qopt->entries[3].gate_mask = 0x8B; + qopt->entries[3].interval = 100000; + qopt->entries[4].gate_mask = 0x8C; + qopt->entries[4].interval = 500000; + qopt->num_entries = 5; + delay_base_time(adapter, qopt, 6); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + if (!disable_taprio(adapter)) + goto failed; + + kfree(qopt); + + return true; + +failed: + disable_taprio(adapter); + kfree(qopt); + + return false; +} + +static bool tsnep_test_taprio_extension(struct tsnep_adapter *adapter) +{ + struct tc_taprio_qopt_offload *qopt; + int i; + + qopt = kzalloc(struct_size(qopt, entries, 255), GFP_KERNEL); + if (!qopt) + return false; + for (i = 0; i < 255; i++) + qopt->entries[i].command = TC_TAPRIO_CMD_SET_GATES; + + qopt->enable = 1; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 100000; + qopt->cycle_time_extension = 50000; + qopt->entries[0].gate_mask = 0x90; + qopt->entries[0].interval = 20000; + qopt->entries[1].gate_mask = 0x91; + qopt->entries[1].interval = 80000; + qopt->num_entries = 2; + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to different phase */ + qopt->base_time = ktime_set(0, 50000); + qopt->entries[0].gate_mask = 0x92; + qopt->entries[0].interval = 33000; + qopt->entries[1].gate_mask = 0x93; + qopt->entries[1].interval = 67000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 2); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to different phase and longer cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1000000; + qopt->cycle_time_extension = 700000; + qopt->entries[0].gate_mask = 0x94; + qopt->entries[0].interval = 400000; + qopt->entries[1].gate_mask = 0x95; + qopt->entries[1].interval = 600000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 7); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 700000); + qopt->cycle_time = 2000000; + qopt->cycle_time_extension = 1900000; + qopt->entries[0].gate_mask = 0x96; + qopt->entries[0].interval = 400000; + qopt->entries[1].gate_mask = 0x97; + qopt->entries[1].interval = 1600000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 9); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to different phase and shorter cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1500000; + qopt->cycle_time_extension = 700000; + qopt->entries[0].gate_mask = 0x98; + qopt->entries[0].interval = 400000; + qopt->entries[1].gate_mask = 0x99; + qopt->entries[1].interval = 600000; + qopt->entries[2].gate_mask = 0x9A; + qopt->entries[2].interval = 500000; + qopt->num_entries = 3; + delay_base_time(adapter, qopt, 3); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 100000); + qopt->cycle_time = 500000; + qopt->cycle_time_extension = 300000; + qopt->entries[0].gate_mask = 0x9B; + qopt->entries[0].interval = 150000; + qopt->entries[1].gate_mask = 0x9C; + qopt->entries[1].interval = 350000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 9); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + /* change to different cycle time */ + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 1000000; + qopt->cycle_time_extension = 700000; + qopt->entries[0].gate_mask = 0xAD; + qopt->entries[0].interval = 400000; + qopt->entries[1].gate_mask = 0xAE; + qopt->entries[1].interval = 300000; + qopt->entries[2].gate_mask = 0xAF; + qopt->entries[2].interval = 300000; + qopt->num_entries = 3; + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 400000; + qopt->cycle_time_extension = 100000; + qopt->entries[0].gate_mask = 0xA0; + qopt->entries[0].interval = 200000; + qopt->entries[1].gate_mask = 0xA1; + qopt->entries[1].interval = 200000; + qopt->num_entries = 2; + delay_base_time(adapter, qopt, 19); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 500000; + qopt->cycle_time_extension = 499999; + qopt->entries[0].gate_mask = 0xB2; + qopt->entries[0].interval = 100000; + qopt->entries[1].gate_mask = 0xB3; + qopt->entries[1].interval = 100000; + qopt->entries[2].gate_mask = 0xB4; + qopt->entries[2].interval = 100000; + qopt->entries[3].gate_mask = 0xB5; + qopt->entries[3].interval = 200000; + qopt->num_entries = 4; + delay_base_time(adapter, qopt, 19); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + qopt->base_time = ktime_set(0, 0); + qopt->cycle_time = 6000000; + qopt->cycle_time_extension = 5999999; + qopt->entries[0].gate_mask = 0xC6; + qopt->entries[0].interval = 1000000; + qopt->entries[1].gate_mask = 0xC7; + qopt->entries[1].interval = 1000000; + qopt->entries[2].gate_mask = 0xC8; + qopt->entries[2].interval = 1000000; + qopt->entries[3].gate_mask = 0xC9; + qopt->entries[3].interval = 1500000; + qopt->entries[4].gate_mask = 0xCA; + qopt->entries[4].interval = 1500000; + qopt->num_entries = 5; + delay_base_time(adapter, qopt, 1); + if (!enable_check_taprio(adapter, qopt, 100)) + goto failed; + + if (!disable_taprio(adapter)) + goto failed; + + kfree(qopt); + + return true; + +failed: + disable_taprio(adapter); + kfree(qopt); + + return false; +} + +int tsnep_ethtool_get_test_count(void) +{ + return TSNEP_TEST_COUNT; +} + +void tsnep_ethtool_get_test_strings(u8 *data) +{ + memcpy(data, tsnep_test_strings, sizeof(tsnep_test_strings)); +} + +void tsnep_ethtool_self_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + eth_test->len = TSNEP_TEST_COUNT; + + if (eth_test->flags != ETH_TEST_FL_OFFLINE) { + /* no tests are done online */ + data[TSNEP_TEST_ENABLE] = 0; + data[TSNEP_TEST_TAPRIO] = 0; + data[TSNEP_TEST_TAPRIO_CHANGE] = 0; + data[TSNEP_TEST_TAPRIO_EXTENSION] = 0; + + return; + } + + if (tsnep_test_gc_enable(adapter)) { + data[TSNEP_TEST_ENABLE] = 0; + } else { + eth_test->flags |= ETH_TEST_FL_FAILED; + data[TSNEP_TEST_ENABLE] = 1; + } + + if (tsnep_test_taprio(adapter)) { + data[TSNEP_TEST_TAPRIO] = 0; + } else { + eth_test->flags |= ETH_TEST_FL_FAILED; + data[TSNEP_TEST_TAPRIO] = 1; + } + + if (tsnep_test_taprio_change(adapter)) { + data[TSNEP_TEST_TAPRIO_CHANGE] = 0; + } else { + eth_test->flags |= ETH_TEST_FL_FAILED; + data[TSNEP_TEST_TAPRIO_CHANGE] = 1; + } + + if (tsnep_test_taprio_extension(adapter)) { + data[TSNEP_TEST_TAPRIO_EXTENSION] = 0; + } else { + eth_test->flags |= ETH_TEST_FL_FAILED; + data[TSNEP_TEST_TAPRIO_EXTENSION] = 1; + } +} diff --git a/drivers/net/ethernet/engleder/tsnep_tc.c b/drivers/net/ethernet/engleder/tsnep_tc.c new file mode 100644 index 000000000000..c4c6e1357317 --- /dev/null +++ b/drivers/net/ethernet/engleder/tsnep_tc.c @@ -0,0 +1,443 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2021 Gerhard Engleder <gerhard@engleder-embedded.com> */ + +#include "tsnep.h" + +#include <net/pkt_sched.h> + +/* save one operation at the end for additional operation at list change */ +#define TSNEP_MAX_GCL_NUM (TSNEP_GCL_COUNT - 1) + +static int tsnep_validate_gcl(struct tc_taprio_qopt_offload *qopt) +{ + int i; + u64 cycle_time; + + if (!qopt->cycle_time) + return -ERANGE; + if (qopt->num_entries > TSNEP_MAX_GCL_NUM) + return -EINVAL; + cycle_time = 0; + for (i = 0; i < qopt->num_entries; i++) { + if (qopt->entries[i].command != TC_TAPRIO_CMD_SET_GATES) + return -EINVAL; + if (qopt->entries[i].gate_mask & ~TSNEP_GCL_MASK) + return -EINVAL; + if (qopt->entries[i].interval < TSNEP_GCL_MIN_INTERVAL) + return -EINVAL; + cycle_time += qopt->entries[i].interval; + } + if (qopt->cycle_time != cycle_time) + return -EINVAL; + if (qopt->cycle_time_extension >= qopt->cycle_time) + return -EINVAL; + + return 0; +} + +static void tsnep_write_gcl_operation(struct tsnep_gcl *gcl, int index, + u32 properties, u32 interval, bool flush) +{ + void __iomem *addr = gcl->addr + + sizeof(struct tsnep_gcl_operation) * index; + + gcl->operation[index].properties = properties; + gcl->operation[index].interval = interval; + + iowrite32(properties, addr); + iowrite32(interval, addr + sizeof(u32)); + + if (flush) { + /* flush write with read access */ + ioread32(addr); + } +} + +static u64 tsnep_change_duration(struct tsnep_gcl *gcl, int index) +{ + u64 duration; + int count; + + /* change needs to be triggered one or two operations before start of + * new gate control list + * - change is triggered at start of operation (minimum one operation) + * - operation with adjusted interval is inserted on demand to exactly + * meet the start of the new gate control list (optional) + * + * additionally properties are read directly after start of previous + * operation + * + * therefore, three operations needs to be considered for the limit + */ + duration = 0; + count = 3; + while (count) { + duration += gcl->operation[index].interval; + + index--; + if (index < 0) + index = gcl->count - 1; + + count--; + } + + return duration; +} + +static void tsnep_write_gcl(struct tsnep_gcl *gcl, + struct tc_taprio_qopt_offload *qopt) +{ + int i; + u32 properties; + u64 extend; + u64 cut; + + gcl->base_time = ktime_to_ns(qopt->base_time); + gcl->cycle_time = qopt->cycle_time; + gcl->cycle_time_extension = qopt->cycle_time_extension; + + for (i = 0; i < qopt->num_entries; i++) { + properties = qopt->entries[i].gate_mask; + if (i == (qopt->num_entries - 1)) + properties |= TSNEP_GCL_LAST; + + tsnep_write_gcl_operation(gcl, i, properties, + qopt->entries[i].interval, true); + } + gcl->count = qopt->num_entries; + + /* calculate change limit; i.e., the time needed between enable and + * start of new gate control list + */ + + /* case 1: extend cycle time for change + * - change duration of last operation + * - cycle time extension + */ + extend = tsnep_change_duration(gcl, gcl->count - 1); + extend += gcl->cycle_time_extension; + + /* case 2: cut cycle time for change + * - maximum change duration + */ + cut = 0; + for (i = 0; i < gcl->count; i++) + cut = max(cut, tsnep_change_duration(gcl, i)); + + /* use maximum, because the actual case (extend or cut) can be + * determined only after limit is known (chicken-and-egg problem) + */ + gcl->change_limit = max(extend, cut); +} + +static u64 tsnep_gcl_start_after(struct tsnep_gcl *gcl, u64 limit) +{ + u64 start = gcl->base_time; + u64 n; + + if (start <= limit) { + n = div64_u64(limit - start, gcl->cycle_time); + start += (n + 1) * gcl->cycle_time; + } + + return start; +} + +static u64 tsnep_gcl_start_before(struct tsnep_gcl *gcl, u64 limit) +{ + u64 start = gcl->base_time; + u64 n; + + n = div64_u64(limit - start, gcl->cycle_time); + start += n * gcl->cycle_time; + if (start == limit) + start -= gcl->cycle_time; + + return start; +} + +static u64 tsnep_set_gcl_change(struct tsnep_gcl *gcl, int index, u64 change, + bool insert) +{ + /* previous operation triggers change and properties are evaluated at + * start of operation + */ + if (index == 0) + index = gcl->count - 1; + else + index = index - 1; + change -= gcl->operation[index].interval; + + /* optionally change to new list with additional operation in between */ + if (insert) { + void __iomem *addr = gcl->addr + + sizeof(struct tsnep_gcl_operation) * index; + + gcl->operation[index].properties |= TSNEP_GCL_INSERT; + iowrite32(gcl->operation[index].properties, addr); + } + + return change; +} + +static void tsnep_clean_gcl(struct tsnep_gcl *gcl) +{ + int i; + u32 mask = TSNEP_GCL_LAST | TSNEP_GCL_MASK; + void __iomem *addr; + + /* search for insert operation and reset properties */ + for (i = 0; i < gcl->count; i++) { + if (gcl->operation[i].properties & ~mask) { + addr = gcl->addr + + sizeof(struct tsnep_gcl_operation) * i; + + gcl->operation[i].properties &= mask; + iowrite32(gcl->operation[i].properties, addr); + + break; + } + } +} + +static u64 tsnep_insert_gcl_operation(struct tsnep_gcl *gcl, int ref, + u64 change, u32 interval) +{ + u32 properties; + + properties = gcl->operation[ref].properties & TSNEP_GCL_MASK; + /* change to new list directly after inserted operation */ + properties |= TSNEP_GCL_CHANGE; + + /* last operation of list is reserved to insert operation */ + tsnep_write_gcl_operation(gcl, TSNEP_GCL_COUNT - 1, properties, + interval, false); + + return tsnep_set_gcl_change(gcl, ref, change, true); +} + +static u64 tsnep_extend_gcl(struct tsnep_gcl *gcl, u64 start, u32 extension) +{ + int ref = gcl->count - 1; + u32 interval = gcl->operation[ref].interval + extension; + + start -= gcl->operation[ref].interval; + + return tsnep_insert_gcl_operation(gcl, ref, start, interval); +} + +static u64 tsnep_cut_gcl(struct tsnep_gcl *gcl, u64 start, u64 cycle_time) +{ + u64 sum = 0; + int i; + + /* find operation which shall be cutted */ + for (i = 0; i < gcl->count; i++) { + u64 sum_tmp = sum + gcl->operation[i].interval; + u64 interval; + + /* sum up operations as long as cycle time is not exceeded */ + if (sum_tmp > cycle_time) + break; + + /* remaining interval must be big enough for hardware */ + interval = cycle_time - sum_tmp; + if (interval > 0 && interval < TSNEP_GCL_MIN_INTERVAL) + break; + + sum = sum_tmp; + } + if (sum == cycle_time) { + /* no need to cut operation itself or whole cycle + * => change exactly at operation + */ + return tsnep_set_gcl_change(gcl, i, start + sum, false); + } + return tsnep_insert_gcl_operation(gcl, i, start + sum, + cycle_time - sum); +} + +static int tsnep_enable_gcl(struct tsnep_adapter *adapter, + struct tsnep_gcl *gcl, struct tsnep_gcl *curr) +{ + u64 system_time; + u64 timeout; + u64 limit; + + /* estimate timeout limit after timeout enable, actually timeout limit + * in hardware will be earlier than estimate so we are on the safe side + */ + tsnep_get_system_time(adapter, &system_time); + timeout = system_time + TSNEP_GC_TIMEOUT; + + if (curr) + limit = timeout + curr->change_limit; + else + limit = timeout; + + gcl->start_time = tsnep_gcl_start_after(gcl, limit); + + /* gate control time register is only 32bit => time shall be in the near + * future (no driver support for far future implemented) + */ + if ((gcl->start_time - system_time) >= U32_MAX) + return -EAGAIN; + + if (curr) { + /* change gate control list */ + u64 last; + u64 change; + + last = tsnep_gcl_start_before(curr, gcl->start_time); + if ((last + curr->cycle_time) == gcl->start_time) + change = tsnep_cut_gcl(curr, last, + gcl->start_time - last); + else if (((gcl->start_time - last) <= + curr->cycle_time_extension) || + ((gcl->start_time - last) <= TSNEP_GCL_MIN_INTERVAL)) + change = tsnep_extend_gcl(curr, last, + gcl->start_time - last); + else + change = tsnep_cut_gcl(curr, last, + gcl->start_time - last); + + WARN_ON(change <= timeout); + gcl->change = true; + iowrite32(change & 0xFFFFFFFF, adapter->addr + TSNEP_GC_CHANGE); + } else { + /* start gate control list */ + WARN_ON(gcl->start_time <= timeout); + gcl->change = false; + iowrite32(gcl->start_time & 0xFFFFFFFF, + adapter->addr + TSNEP_GC_TIME); + } + + return 0; +} + +static int tsnep_taprio(struct tsnep_adapter *adapter, + struct tc_taprio_qopt_offload *qopt) +{ + struct tsnep_gcl *gcl; + struct tsnep_gcl *curr; + int retval; + + if (!adapter->gate_control) + return -EOPNOTSUPP; + + if (!qopt->enable) { + /* disable gate control if active */ + mutex_lock(&adapter->gate_control_lock); + + if (adapter->gate_control_active) { + iowrite8(TSNEP_GC_DISABLE, adapter->addr + TSNEP_GC); + adapter->gate_control_active = false; + } + + mutex_unlock(&adapter->gate_control_lock); + + return 0; + } + + retval = tsnep_validate_gcl(qopt); + if (retval) + return retval; + + mutex_lock(&adapter->gate_control_lock); + + gcl = &adapter->gcl[adapter->next_gcl]; + tsnep_write_gcl(gcl, qopt); + + /* select current gate control list if active */ + if (adapter->gate_control_active) { + if (adapter->next_gcl == 0) + curr = &adapter->gcl[1]; + else + curr = &adapter->gcl[0]; + } else { + curr = NULL; + } + + for (;;) { + /* start timeout which discards late enable, this helps ensuring + * that start/change time are in the future at enable + */ + iowrite8(TSNEP_GC_ENABLE_TIMEOUT, adapter->addr + TSNEP_GC); + + retval = tsnep_enable_gcl(adapter, gcl, curr); + if (retval) { + mutex_unlock(&adapter->gate_control_lock); + + return retval; + } + + /* enable gate control list */ + if (adapter->next_gcl == 0) + iowrite8(TSNEP_GC_ENABLE_A, adapter->addr + TSNEP_GC); + else + iowrite8(TSNEP_GC_ENABLE_B, adapter->addr + TSNEP_GC); + + /* done if timeout did not happen */ + if (!(ioread32(adapter->addr + TSNEP_GC) & + TSNEP_GC_TIMEOUT_SIGNAL)) + break; + + /* timeout is acknowledged with any enable */ + iowrite8(TSNEP_GC_ENABLE_A, adapter->addr + TSNEP_GC); + + if (curr) + tsnep_clean_gcl(curr); + + /* retry because of timeout */ + } + + adapter->gate_control_active = true; + + if (adapter->next_gcl == 0) + adapter->next_gcl = 1; + else + adapter->next_gcl = 0; + + mutex_unlock(&adapter->gate_control_lock); + + return 0; +} + +int tsnep_tc_setup(struct net_device *netdev, enum tc_setup_type type, + void *type_data) +{ + struct tsnep_adapter *adapter = netdev_priv(netdev); + + switch (type) { + case TC_SETUP_QDISC_TAPRIO: + return tsnep_taprio(adapter, type_data); + default: + return -EOPNOTSUPP; + } +} + +int tsnep_tc_init(struct tsnep_adapter *adapter) +{ + if (!adapter->gate_control) + return 0; + + /* open all gates */ + iowrite8(TSNEP_GC_DISABLE, adapter->addr + TSNEP_GC); + iowrite32(TSNEP_GC_OPEN | TSNEP_GC_NEXT_OPEN, adapter->addr + TSNEP_GC); + + adapter->gcl[0].addr = adapter->addr + TSNEP_GCL_A; + adapter->gcl[1].addr = adapter->addr + TSNEP_GCL_B; + + return 0; +} + +void tsnep_tc_cleanup(struct tsnep_adapter *adapter) +{ + if (!adapter->gate_control) + return; + + if (adapter->gate_control_active) { + iowrite8(TSNEP_GC_DISABLE, adapter->addr + TSNEP_GC); + adapter->gate_control_active = false; + } +} diff --git a/drivers/net/ethernet/ethoc.c b/drivers/net/ethernet/ethoc.c index b1c8ffea6ad2..d618a8b785b0 100644 --- a/drivers/net/ethernet/ethoc.c +++ b/drivers/net/ethernet/ethoc.c @@ -945,7 +945,9 @@ static void ethoc_get_regs(struct net_device *dev, struct ethtool_regs *regs, } static void ethoc_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ethoc *priv = netdev_priv(dev); @@ -961,7 +963,9 @@ static void ethoc_get_ringparam(struct net_device *dev, } static int ethoc_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ethoc *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/faraday/ftgmac100.c b/drivers/net/ethernet/faraday/ftgmac100.c index 97c5d70de76e..691605c15265 100644 --- a/drivers/net/ethernet/faraday/ftgmac100.c +++ b/drivers/net/ethernet/faraday/ftgmac100.c @@ -1178,8 +1178,11 @@ static void ftgmac100_get_drvinfo(struct net_device *netdev, strlcpy(info->bus_info, dev_name(&netdev->dev), sizeof(info->bus_info)); } -static void ftgmac100_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static void +ftgmac100_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct ftgmac100 *priv = netdev_priv(netdev); @@ -1190,8 +1193,11 @@ static void ftgmac100_get_ringparam(struct net_device *netdev, ering->tx_pending = priv->tx_q_entries; } -static int ftgmac100_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static int +ftgmac100_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct ftgmac100 *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c index 6b2927d863e2..d6871437d951 100644 --- a/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c +++ b/drivers/net/ethernet/freescale/dpaa/dpaa_eth.c @@ -2325,7 +2325,7 @@ dpaa_start_xmit(struct sk_buff *skb, struct net_device *net_dev) txq = netdev_get_tx_queue(net_dev, queue_mapping); /* LLTX requires to do our own update of trans_start */ - txq->trans_start = jiffies; + txq_trans_cond_update(txq); if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) { fd.cmd |= cpu_to_be32(FM_FD_CMD_UPD); @@ -2531,7 +2531,7 @@ static int dpaa_xdp_xmit_frame(struct net_device *net_dev, /* Bump the trans_start */ txq = netdev_get_tx_queue(net_dev, smp_processor_id()); - txq->trans_start = jiffies; + txq_trans_cond_update(txq); err = dpaa_xmit(priv, percpu_stats, smp_processor_id(), &fd); if (err) { diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c index ef8f0a055024..34b2a73c347f 100644 --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-mac.c @@ -90,88 +90,6 @@ static int dpaa2_mac_get_if_mode(struct fwnode_handle *dpmac_node, return err; } -static bool dpaa2_mac_phy_mode_mismatch(struct dpaa2_mac *mac, - phy_interface_t interface) -{ - switch (interface) { - /* We can switch between SGMII and 1000BASE-X at runtime with - * pcs-lynx - */ - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_1000BASEX: - if (mac->pcs && - (mac->if_mode == PHY_INTERFACE_MODE_SGMII || - mac->if_mode == PHY_INTERFACE_MODE_1000BASEX)) - return false; - return interface != mac->if_mode; - - case PHY_INTERFACE_MODE_10GBASER: - case PHY_INTERFACE_MODE_USXGMII: - case PHY_INTERFACE_MODE_QSGMII: - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - return (interface != mac->if_mode); - default: - return true; - } -} - -static void dpaa2_mac_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct dpaa2_mac *mac = phylink_to_dpaa2_mac(config); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - if (state->interface != PHY_INTERFACE_MODE_NA && - dpaa2_mac_phy_mode_mismatch(mac, state->interface)) { - goto empty_set; - } - - phylink_set_port_modes(mask); - phylink_set(mask, Autoneg); - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - - switch (state->interface) { - case PHY_INTERFACE_MODE_NA: - case PHY_INTERFACE_MODE_10GBASER: - case PHY_INTERFACE_MODE_USXGMII: - phylink_set_10g_modes(mask); - if (state->interface == PHY_INTERFACE_MODE_10GBASER) - break; - phylink_set(mask, 5000baseT_Full); - phylink_set(mask, 2500baseT_Full); - fallthrough; - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_QSGMII: - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - phylink_set(mask, 1000baseX_Full); - phylink_set(mask, 1000baseT_Full); - if (state->interface == PHY_INTERFACE_MODE_1000BASEX) - break; - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 10baseT_Full); - break; - default: - goto empty_set; - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); - - return; - -empty_set: - linkmode_zero(supported); -} - static void dpaa2_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) { @@ -243,7 +161,7 @@ static void dpaa2_mac_link_down(struct phylink_config *config, } static const struct phylink_mac_ops dpaa2_mac_phylink_ops = { - .validate = dpaa2_mac_validate, + .validate = phylink_generic_validate, .mac_config = dpaa2_mac_config, .mac_link_up = dpaa2_mac_link_up, .mac_link_down = dpaa2_mac_link_down, @@ -336,9 +254,34 @@ int dpaa2_mac_connect(struct dpaa2_mac *mac) return err; } + memset(&mac->phylink_config, 0, sizeof(mac->phylink_config)); mac->phylink_config.dev = &net_dev->dev; mac->phylink_config.type = PHYLINK_NETDEV; + mac->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | + MAC_10FD | MAC_100FD | MAC_1000FD | MAC_2500FD | MAC_5000FD | + MAC_10000FD; + + /* We support the current interface mode, and if we have a PCS + * similar interface modes that do not require the PLLs to be + * reconfigured. + */ + __set_bit(mac->if_mode, mac->phylink_config.supported_interfaces); + if (mac->pcs) { + switch (mac->if_mode) { + case PHY_INTERFACE_MODE_1000BASEX: + case PHY_INTERFACE_MODE_SGMII: + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + mac->phylink_config.supported_interfaces); + break; + + default: + break; + } + } + phylink = phylink_create(&mac->phylink_config, dpmac_node, mac->if_mode, &dpaa2_mac_phylink_ops); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c index 910b9f722504..fa5b4f885b17 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_ethtool.c @@ -562,7 +562,9 @@ static int enetc_set_rxfh(struct net_device *ndev, const u32 *indir, } static void enetc_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct enetc_ndev_priv *priv = netdev_priv(ndev); diff --git a/drivers/net/ethernet/freescale/enetc/enetc_pf.c b/drivers/net/ethernet/freescale/enetc/enetc_pf.c index 0e87c7043b77..fe6a544f37f0 100644 --- a/drivers/net/ethernet/freescale/enetc/enetc_pf.c +++ b/drivers/net/ethernet/freescale/enetc/enetc_pf.c @@ -930,45 +930,6 @@ static void enetc_mdiobus_destroy(struct enetc_pf *pf) enetc_imdio_remove(pf); } -static void enetc_pl_mac_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - if (state->interface != PHY_INTERFACE_MODE_NA && - state->interface != PHY_INTERFACE_MODE_INTERNAL && - state->interface != PHY_INTERFACE_MODE_SGMII && - state->interface != PHY_INTERFACE_MODE_2500BASEX && - state->interface != PHY_INTERFACE_MODE_USXGMII && - !phy_interface_mode_is_rgmii(state->interface)) { - linkmode_zero(supported); - return; - } - - phylink_set_port_modes(mask); - phylink_set(mask, Autoneg); - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 1000baseT_Half); - phylink_set(mask, 1000baseT_Full); - - if (state->interface == PHY_INTERFACE_MODE_INTERNAL || - state->interface == PHY_INTERFACE_MODE_2500BASEX || - state->interface == PHY_INTERFACE_MODE_USXGMII) { - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void enetc_pl_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) @@ -1096,7 +1057,7 @@ static void enetc_pl_mac_link_down(struct phylink_config *config, } static const struct phylink_mac_ops enetc_mac_phylink_ops = { - .validate = enetc_pl_mac_validate, + .validate = phylink_generic_validate, .mac_config = enetc_pl_mac_config, .mac_link_up = enetc_pl_mac_link_up, .mac_link_down = enetc_pl_mac_link_down, @@ -1111,6 +1072,18 @@ static int enetc_phylink_create(struct enetc_ndev_priv *priv, pf->phylink_config.dev = &priv->ndev->dev; pf->phylink_config.type = PHYLINK_NETDEV; + pf->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD; + + __set_bit(PHY_INTERFACE_MODE_INTERNAL, + pf->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + pf->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + pf->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_USXGMII, + pf->phylink_config.supported_interfaces); + phy_interface_set_rgmii(pf->phylink_config.supported_interfaces); phylink = phylink_create(&pf->phylink_config, of_fwnode_handle(node), pf->if_mode, &enetc_mac_phylink_ops); diff --git a/drivers/net/ethernet/freescale/fec_main.c b/drivers/net/ethernet/freescale/fec_main.c index 1b1f7f2a6130..613b8180a1bd 100644 --- a/drivers/net/ethernet/freescale/fec_main.c +++ b/drivers/net/ethernet/freescale/fec_main.c @@ -3558,7 +3558,7 @@ static int fec_enet_init(struct net_device *ndev) ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; if (fep->quirks & FEC_QUIRK_HAS_CSUM) { - ndev->gso_max_segs = FEC_MAX_TSO_SEGS; + netif_set_gso_max_segs(ndev, FEC_MAX_TSO_SEGS); /* enable hw accelerator */ ndev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM diff --git a/drivers/net/ethernet/freescale/gianfar_ethtool.c b/drivers/net/ethernet/freescale/gianfar_ethtool.c index 7b32ed29bf4c..ff756265d58f 100644 --- a/drivers/net/ethernet/freescale/gianfar_ethtool.c +++ b/drivers/net/ethernet/freescale/gianfar_ethtool.c @@ -372,7 +372,9 @@ static int gfar_scoalesce(struct net_device *dev, * rx, rx_mini, and rx_jumbo rings are the same size, as mini and * jumbo are ignored by the driver */ static void gfar_gringparam(struct net_device *dev, - struct ethtool_ringparam *rvals) + struct ethtool_ringparam *rvals, + struct kernel_ethtool_ringparam *kernel_rvals, + struct netlink_ext_ack *extack) { struct gfar_private *priv = netdev_priv(dev); struct gfar_priv_tx_q *tx_queue = NULL; @@ -399,7 +401,9 @@ static void gfar_gringparam(struct net_device *dev, * necessary so that we don't mess things up while we're in motion. */ static int gfar_sringparam(struct net_device *dev, - struct ethtool_ringparam *rvals) + struct ethtool_ringparam *rvals, + struct kernel_ethtool_ringparam *kernel_rvals, + struct netlink_ext_ack *extack) { struct gfar_private *priv = netdev_priv(dev); int err = 0, i; diff --git a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c index 14c08a868190..69b2b98b1525 100644 --- a/drivers/net/ethernet/freescale/ucc_geth_ethtool.c +++ b/drivers/net/ethernet/freescale/ucc_geth_ethtool.c @@ -207,7 +207,9 @@ uec_get_regs(struct net_device *netdev, static void uec_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ucc_geth_private *ugeth = netdev_priv(netdev); struct ucc_geth_info *ug_info = ugeth->ug_info; @@ -226,7 +228,9 @@ uec_get_ringparam(struct net_device *netdev, static int uec_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ucc_geth_private *ugeth = netdev_priv(netdev); struct ucc_geth_info *ug_info = ugeth->ug_info; diff --git a/drivers/net/ethernet/google/gve/gve_ethtool.c b/drivers/net/ethernet/google/gve/gve_ethtool.c index c8df47a97fa4..fd2d2c705391 100644 --- a/drivers/net/ethernet/google/gve/gve_ethtool.c +++ b/drivers/net/ethernet/google/gve/gve_ethtool.c @@ -419,7 +419,9 @@ static int gve_set_channels(struct net_device *netdev, } static void gve_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *cmd) + struct ethtool_ringparam *cmd, + struct kernel_ethtool_ringparam *kernel_cmd, + struct netlink_ext_ack *extack) { struct gve_priv *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c index ab7390225942..d7a27c244d48 100644 --- a/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns/hns_ethtool.c @@ -663,9 +663,13 @@ static void hns_nic_get_drvinfo(struct net_device *net_dev, * hns_get_ringparam - get ring parameter * @net_dev: net device * @param: ethtool parameter + * @kernel_param: ethtool external parameter + * @extack: netlink extended ACK report struct */ static void hns_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns_nic_priv *priv = netdev_priv(net_dev); struct hnae_ae_ops *ops; diff --git a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h index c2bd2584201f..b668df6193be 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h +++ b/drivers/net/ethernet/hisilicon/hns3/hclge_mbx.h @@ -80,6 +80,9 @@ enum hclge_mbx_tbl_cfg_subcode { #define HCLGE_MBX_MAX_RESP_DATA_SIZE 8U #define HCLGE_MBX_MAX_RING_CHAIN_PARAM_NUM 4 +#define HCLGE_RESET_SCHED_TIMEOUT (3 * HZ) +#define HCLGE_MBX_SCHED_TIMEOUT (HZ / 2) + struct hclge_ring_chain_param { u8 ring_type; u8 tqp_index; diff --git a/drivers/net/ethernet/hisilicon/hns3/hnae3.h b/drivers/net/ethernet/hisilicon/hns3/hnae3.h index 3f7a9a4c59d5..54caf0dd1204 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hnae3.h +++ b/drivers/net/ethernet/hisilicon/hns3/hnae3.h @@ -859,6 +859,20 @@ struct hnae3_handle { #define hnae3_get_bit(origin, shift) \ hnae3_get_field(origin, 0x1 << (shift), shift) +#define HNAE3_FORMAT_MAC_ADDR_LEN 18 +#define HNAE3_FORMAT_MAC_ADDR_OFFSET_0 0 +#define HNAE3_FORMAT_MAC_ADDR_OFFSET_4 4 +#define HNAE3_FORMAT_MAC_ADDR_OFFSET_5 5 + +static inline void hnae3_format_mac_addr(char *format_mac_addr, + const u8 *mac_addr) +{ + snprintf(format_mac_addr, HNAE3_FORMAT_MAC_ADDR_LEN, "%02x:**:**:**:%02x:%02x", + mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_0], + mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_4], + mac_addr[HNAE3_FORMAT_MAC_ADDR_OFFSET_5]); +} + int hnae3_register_ae_dev(struct hnae3_ae_dev *ae_dev); void hnae3_unregister_ae_dev(struct hnae3_ae_dev *ae_dev); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c index 081295bff765..817e2e8a7287 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.c @@ -1083,7 +1083,7 @@ static void hns3_dump_page_pool_info(struct hns3_enet_ring *ring, sprintf(result[j++], "%u", index); sprintf(result[j++], "%u", READ_ONCE(ring->page_pool->pages_state_hold_cnt)); - sprintf(result[j++], "%u", + sprintf(result[j++], "%d", atomic_read(&ring->page_pool->pages_state_release_cnt)); sprintf(result[j++], "%u", ring->page_pool->p.pool_size); sprintf(result[j++], "%u", ring->page_pool->p.order); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h index bd8801065e02..83aa1450ab9f 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_debugfs.h @@ -4,6 +4,8 @@ #ifndef __HNS3_DEBUGFS_H #define __HNS3_DEBUGFS_H +#include "hnae3.h" + #define HNS3_DBG_READ_LEN 65536 #define HNS3_DBG_READ_LEN_128KB 0x20000 #define HNS3_DBG_READ_LEN_1MB 0x100000 diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c index 9ccebbaa0d69..babc5d7a3b52 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.c @@ -17,6 +17,7 @@ #include <linux/skbuff.h> #include <linux/sctp.h> #include <net/gre.h> +#include <net/gro.h> #include <net/ip6_checksum.h> #include <net/pkt_cls.h> #include <net/tcp.h> @@ -53,10 +54,6 @@ static int debug = -1; module_param(debug, int, 0); MODULE_PARM_DESC(debug, " Network interface message level setting"); -static unsigned int tx_spare_buf_size; -module_param(tx_spare_buf_size, uint, 0400); -MODULE_PARM_DESC(tx_spare_buf_size, "Size used to allocate tx spare buffer"); - static unsigned int tx_sgl = 1; module_param(tx_sgl, uint, 0600); MODULE_PARM_DESC(tx_sgl, "Minimum number of frags when using dma_map_sg() to optimize the IOMMU mapping"); @@ -1005,9 +1002,7 @@ static bool hns3_can_use_tx_bounce(struct hns3_enet_ring *ring, return false; if (ALIGN(len, dma_get_cache_alignment()) > space) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_spare_full++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_spare_full); return false; } @@ -1024,9 +1019,7 @@ static bool hns3_can_use_tx_sgl(struct hns3_enet_ring *ring, return false; if (space < HNS3_MAX_SGL_SIZE) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_spare_full++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_spare_full); return false; } @@ -1041,8 +1034,7 @@ static void hns3_init_tx_spare_buffer(struct hns3_enet_ring *ring) dma_addr_t dma; int order; - alloc_size = tx_spare_buf_size ? tx_spare_buf_size : - ring->tqp->handle->kinfo.tx_spare_buf_size; + alloc_size = ring->tqp->handle->kinfo.tx_spare_buf_size; if (!alloc_size) return; @@ -1306,7 +1298,7 @@ static bool hns3_tunnel_csum_bug(struct sk_buff *skb) if (!(!skb->encapsulation && (l4.udp->dest == htons(IANA_VXLAN_UDP_PORT) || l4.udp->dest == htons(GENEVE_UDP_PORT) || - l4.udp->dest == htons(4790)))) + l4.udp->dest == htons(IANA_VXLAN_GPE_UDP_PORT)))) return false; return true; @@ -1359,44 +1351,9 @@ static void hns3_set_outer_l2l3l4(struct sk_buff *skb, u8 ol4_proto, HNS3_TUN_NVGRE); } -static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, - u8 il4_proto, u32 *type_cs_vlan_tso, - u32 *ol_type_vlan_len_msec) +static void hns3_set_l3_type(struct sk_buff *skb, union l3_hdr_info l3, + u32 *type_cs_vlan_tso) { - unsigned char *l2_hdr = skb->data; - u32 l4_proto = ol4_proto; - union l4_hdr_info l4; - union l3_hdr_info l3; - u32 l2_len, l3_len; - - l4.hdr = skb_transport_header(skb); - l3.hdr = skb_network_header(skb); - - /* handle encapsulation skb */ - if (skb->encapsulation) { - /* If this is a not UDP/GRE encapsulation skb */ - if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) { - /* drop the skb tunnel packet if hardware don't support, - * because hardware can't calculate csum when TSO. - */ - if (skb_is_gso(skb)) - return -EDOM; - - /* the stack computes the IP header already, - * driver calculate l4 checksum when not TSO. - */ - return skb_checksum_help(skb); - } - - hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec); - - /* switch to inner header */ - l2_hdr = skb_inner_mac_header(skb); - l3.hdr = skb_inner_network_header(skb); - l4.hdr = skb_inner_transport_header(skb); - l4_proto = il4_proto; - } - if (l3.v4->version == 4) { hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S, HNS3_L3T_IPV4); @@ -1410,15 +1367,11 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3T_S, HNS3_L3T_IPV6); } +} - /* compute inner(/normal) L2 header size, defined in 2 Bytes */ - l2_len = l3.hdr - l2_hdr; - hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1); - - /* compute inner(/normal) L3 header size, defined in 4 Bytes */ - l3_len = l4.hdr - l3.hdr; - hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2); - +static int hns3_set_l4_csum_length(struct sk_buff *skb, union l4_hdr_info l4, + u32 l4_proto, u32 *type_cs_vlan_tso) +{ /* compute inner(/normal) L4 header size, defined in 4 Bytes */ switch (l4_proto) { case IPPROTO_TCP: @@ -1464,6 +1417,57 @@ static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, return 0; } +static int hns3_set_l2l3l4(struct sk_buff *skb, u8 ol4_proto, + u8 il4_proto, u32 *type_cs_vlan_tso, + u32 *ol_type_vlan_len_msec) +{ + unsigned char *l2_hdr = skb->data; + u32 l4_proto = ol4_proto; + union l4_hdr_info l4; + union l3_hdr_info l3; + u32 l2_len, l3_len; + + l4.hdr = skb_transport_header(skb); + l3.hdr = skb_network_header(skb); + + /* handle encapsulation skb */ + if (skb->encapsulation) { + /* If this is a not UDP/GRE encapsulation skb */ + if (!(ol4_proto == IPPROTO_UDP || ol4_proto == IPPROTO_GRE)) { + /* drop the skb tunnel packet if hardware don't support, + * because hardware can't calculate csum when TSO. + */ + if (skb_is_gso(skb)) + return -EDOM; + + /* the stack computes the IP header already, + * driver calculate l4 checksum when not TSO. + */ + return skb_checksum_help(skb); + } + + hns3_set_outer_l2l3l4(skb, ol4_proto, ol_type_vlan_len_msec); + + /* switch to inner header */ + l2_hdr = skb_inner_mac_header(skb); + l3.hdr = skb_inner_network_header(skb); + l4.hdr = skb_inner_transport_header(skb); + l4_proto = il4_proto; + } + + hns3_set_l3_type(skb, l3, type_cs_vlan_tso); + + /* compute inner(/normal) L2 header size, defined in 2 Bytes */ + l2_len = l3.hdr - l2_hdr; + hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L2LEN_S, l2_len >> 1); + + /* compute inner(/normal) L3 header size, defined in 4 Bytes */ + l3_len = l4.hdr - l3.hdr; + hns3_set_field(*type_cs_vlan_tso, HNS3_TXD_L3LEN_S, l3_len >> 2); + + return hns3_set_l4_csum_length(skb, l4, l4_proto, type_cs_vlan_tso); +} + static int hns3_handle_vtags(struct hns3_enet_ring *tx_ring, struct sk_buff *skb) { @@ -1540,92 +1544,122 @@ static bool hns3_check_hw_tx_csum(struct sk_buff *skb) return true; } -static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, - struct sk_buff *skb, struct hns3_desc *desc, - struct hns3_desc_cb *desc_cb) +struct hns3_desc_param { + u32 paylen_ol4cs; + u32 ol_type_vlan_len_msec; + u32 type_cs_vlan_tso; + u16 mss_hw_csum; + u16 inner_vtag; + u16 out_vtag; +}; + +static void hns3_init_desc_data(struct sk_buff *skb, struct hns3_desc_param *pa) +{ + pa->paylen_ol4cs = skb->len; + pa->ol_type_vlan_len_msec = 0; + pa->type_cs_vlan_tso = 0; + pa->mss_hw_csum = 0; + pa->inner_vtag = 0; + pa->out_vtag = 0; +} + +static int hns3_handle_vlan_info(struct hns3_enet_ring *ring, + struct sk_buff *skb, + struct hns3_desc_param *param) { - u32 ol_type_vlan_len_msec = 0; - u32 paylen_ol4cs = skb->len; - u32 type_cs_vlan_tso = 0; - u16 mss_hw_csum = 0; - u16 inner_vtag = 0; - u16 out_vtag = 0; int ret; ret = hns3_handle_vtags(ring, skb); if (unlikely(ret < 0)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_vlan_err++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_vlan_err); return ret; } else if (ret == HNS3_INNER_VLAN_TAG) { - inner_vtag = skb_vlan_tag_get(skb); - inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & + param->inner_vtag = skb_vlan_tag_get(skb); + param->inner_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK; - hns3_set_field(type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1); + hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_VLAN_B, 1); } else if (ret == HNS3_OUTER_VLAN_TAG) { - out_vtag = skb_vlan_tag_get(skb); - out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & + param->out_vtag = skb_vlan_tag_get(skb); + param->out_vtag |= (skb->priority << VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK; - hns3_set_field(ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B, + hns3_set_field(param->ol_type_vlan_len_msec, HNS3_TXD_OVLAN_B, 1); } + return 0; +} - desc_cb->send_bytes = skb->len; +static int hns3_handle_csum_partial(struct hns3_enet_ring *ring, + struct sk_buff *skb, + struct hns3_desc_cb *desc_cb, + struct hns3_desc_param *param) +{ + u8 ol4_proto, il4_proto; + int ret; - if (skb->ip_summed == CHECKSUM_PARTIAL) { - u8 ol4_proto, il4_proto; - - if (hns3_check_hw_tx_csum(skb)) { - /* set checksum start and offset, defined in 2 Bytes */ - hns3_set_field(type_cs_vlan_tso, HNS3_TXD_CSUM_START_S, - skb_checksum_start_offset(skb) >> 1); - hns3_set_field(ol_type_vlan_len_msec, - HNS3_TXD_CSUM_OFFSET_S, - skb->csum_offset >> 1); - mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B); - goto out_hw_tx_csum; - } + if (hns3_check_hw_tx_csum(skb)) { + /* set checksum start and offset, defined in 2 Bytes */ + hns3_set_field(param->type_cs_vlan_tso, HNS3_TXD_CSUM_START_S, + skb_checksum_start_offset(skb) >> 1); + hns3_set_field(param->ol_type_vlan_len_msec, + HNS3_TXD_CSUM_OFFSET_S, + skb->csum_offset >> 1); + param->mss_hw_csum |= BIT(HNS3_TXD_HW_CS_B); + return 0; + } - skb_reset_mac_len(skb); + skb_reset_mac_len(skb); - ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); - if (unlikely(ret < 0)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_l4_proto_err++; - u64_stats_update_end(&ring->syncp); - return ret; - } + ret = hns3_get_l4_protocol(skb, &ol4_proto, &il4_proto); + if (unlikely(ret < 0)) { + hns3_ring_stats_update(ring, tx_l4_proto_err); + return ret; + } - ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto, - &type_cs_vlan_tso, - &ol_type_vlan_len_msec); - if (unlikely(ret < 0)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_l2l3l4_err++; - u64_stats_update_end(&ring->syncp); - return ret; - } + ret = hns3_set_l2l3l4(skb, ol4_proto, il4_proto, + ¶m->type_cs_vlan_tso, + ¶m->ol_type_vlan_len_msec); + if (unlikely(ret < 0)) { + hns3_ring_stats_update(ring, tx_l2l3l4_err); + return ret; + } + + ret = hns3_set_tso(skb, ¶m->paylen_ol4cs, ¶m->mss_hw_csum, + ¶m->type_cs_vlan_tso, &desc_cb->send_bytes); + if (unlikely(ret < 0)) { + hns3_ring_stats_update(ring, tx_tso_err); + return ret; + } + return 0; +} + +static int hns3_fill_skb_desc(struct hns3_enet_ring *ring, + struct sk_buff *skb, struct hns3_desc *desc, + struct hns3_desc_cb *desc_cb) +{ + struct hns3_desc_param param; + int ret; + + hns3_init_desc_data(skb, ¶m); + ret = hns3_handle_vlan_info(ring, skb, ¶m); + if (unlikely(ret < 0)) + return ret; + + desc_cb->send_bytes = skb->len; - ret = hns3_set_tso(skb, &paylen_ol4cs, &mss_hw_csum, - &type_cs_vlan_tso, &desc_cb->send_bytes); - if (unlikely(ret < 0)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_tso_err++; - u64_stats_update_end(&ring->syncp); + if (skb->ip_summed == CHECKSUM_PARTIAL) { + ret = hns3_handle_csum_partial(ring, skb, desc_cb, ¶m); + if (ret) return ret; - } } -out_hw_tx_csum: /* Set txbd */ desc->tx.ol_type_vlan_len_msec = - cpu_to_le32(ol_type_vlan_len_msec); - desc->tx.type_cs_vlan_tso_len = cpu_to_le32(type_cs_vlan_tso); - desc->tx.paylen_ol4cs = cpu_to_le32(paylen_ol4cs); - desc->tx.mss_hw_csum = cpu_to_le16(mss_hw_csum); - desc->tx.vlan_tag = cpu_to_le16(inner_vtag); - desc->tx.outer_vlan_tag = cpu_to_le16(out_vtag); + cpu_to_le32(param.ol_type_vlan_len_msec); + desc->tx.type_cs_vlan_tso_len = cpu_to_le32(param.type_cs_vlan_tso); + desc->tx.paylen_ol4cs = cpu_to_le32(param.paylen_ol4cs); + desc->tx.mss_hw_csum = cpu_to_le16(param.mss_hw_csum); + desc->tx.vlan_tag = cpu_to_le16(param.inner_vtag); + desc->tx.outer_vlan_tag = cpu_to_le16(param.out_vtag); return 0; } @@ -1705,9 +1739,7 @@ static int hns3_map_and_fill_desc(struct hns3_enet_ring *ring, void *priv, } if (unlikely(dma_mapping_error(dev, dma))) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); return -ENOMEM; } @@ -1853,9 +1885,7 @@ static int hns3_skb_linearize(struct hns3_enet_ring *ring, * recursion level of over HNS3_MAX_RECURSION_LEVEL. */ if (bd_num == UINT_MAX) { - u64_stats_update_begin(&ring->syncp); - ring->stats.over_max_recursion++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, over_max_recursion); return -ENOMEM; } @@ -1864,16 +1894,12 @@ static int hns3_skb_linearize(struct hns3_enet_ring *ring, */ if (skb->len > HNS3_MAX_TSO_SIZE || (!skb_is_gso(skb) && skb->len > HNS3_MAX_NON_TSO_SIZE)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.hw_limitation++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, hw_limitation); return -ENOMEM; } if (__skb_linearize(skb)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); return -ENOMEM; } @@ -1903,9 +1929,7 @@ static int hns3_nic_maybe_stop_tx(struct hns3_enet_ring *ring, bd_num = hns3_tx_bd_count(skb->len); - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_copy++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_copy); } out: @@ -1925,9 +1949,7 @@ out: return bd_num; } - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_busy++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_busy); return -EBUSY; } @@ -2012,9 +2034,7 @@ static void hns3_tx_doorbell(struct hns3_enet_ring *ring, int num, ring->pending_buf += num; if (!doorbell) { - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_more++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_more); return; } @@ -2064,9 +2084,7 @@ static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring, ret = skb_copy_bits(skb, 0, buf, size); if (unlikely(ret < 0)) { hns3_tx_spare_rollback(ring, cb_len); - u64_stats_update_begin(&ring->syncp); - ring->stats.copy_bits_err++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, copy_bits_err); return ret; } @@ -2089,9 +2107,8 @@ static int hns3_handle_tx_bounce(struct hns3_enet_ring *ring, dma_sync_single_for_device(ring_to_dev(ring), dma, size, DMA_TO_DEVICE); - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_bounce++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_bounce); + return bd_num; } @@ -2121,9 +2138,7 @@ static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring, nents = skb_to_sgvec(skb, sgt->sgl, 0, skb->len); if (unlikely(nents < 0)) { hns3_tx_spare_rollback(ring, cb_len); - u64_stats_update_begin(&ring->syncp); - ring->stats.skb2sgl_err++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, skb2sgl_err); return -ENOMEM; } @@ -2132,9 +2147,7 @@ static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring, DMA_TO_DEVICE); if (unlikely(!sgt->nents)) { hns3_tx_spare_rollback(ring, cb_len); - u64_stats_update_begin(&ring->syncp); - ring->stats.map_sg_err++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, map_sg_err); return -ENOMEM; } @@ -2146,10 +2159,7 @@ static int hns3_handle_tx_sgl(struct hns3_enet_ring *ring, for (i = 0; i < sgt->nents; i++) bd_num += hns3_fill_desc(ring, sg_dma_address(sgt->sgl + i), sg_dma_len(sgt->sgl + i)); - - u64_stats_update_begin(&ring->syncp); - ring->stats.tx_sgl++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, tx_sgl); return bd_num; } @@ -2174,23 +2184,45 @@ out: return hns3_fill_skb_to_desc(ring, skb, DESC_TYPE_SKB); } +static int hns3_handle_skb_desc(struct hns3_enet_ring *ring, + struct sk_buff *skb, + struct hns3_desc_cb *desc_cb, + int next_to_use_head) +{ + int ret; + + ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use], + desc_cb); + if (unlikely(ret < 0)) + goto fill_err; + + /* 'ret < 0' means filling error, 'ret == 0' means skb->len is + * zero, which is unlikely, and 'ret > 0' means how many tx desc + * need to be notified to the hw. + */ + ret = hns3_handle_desc_filling(ring, skb); + if (likely(ret > 0)) + return ret; + +fill_err: + hns3_clear_desc(ring, next_to_use_head); + return ret; +} + netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); struct hns3_enet_ring *ring = &priv->ring[skb->queue_mapping]; struct hns3_desc_cb *desc_cb = &ring->desc_cb[ring->next_to_use]; struct netdev_queue *dev_queue; - int pre_ntu, next_to_use_head; + int pre_ntu, ret; bool doorbell; - int ret; /* Hardware can only handle short frames above 32 bytes */ if (skb_put_padto(skb, HNS3_MIN_TX_LEN)) { hns3_tx_doorbell(ring, 0, !netdev_xmit_more()); - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); return NETDEV_TX_OK; } @@ -2209,20 +2241,9 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) goto out_err_tx_ok; } - next_to_use_head = ring->next_to_use; - - ret = hns3_fill_skb_desc(ring, skb, &ring->desc[ring->next_to_use], - desc_cb); - if (unlikely(ret < 0)) - goto fill_err; - - /* 'ret < 0' means filling error, 'ret == 0' means skb->len is - * zero, which is unlikely, and 'ret > 0' means how many tx desc - * need to be notified to the hw. - */ - ret = hns3_handle_desc_filling(ring, skb); + ret = hns3_handle_skb_desc(ring, skb, desc_cb, ring->next_to_use); if (unlikely(ret <= 0)) - goto fill_err; + goto out_err_tx_ok; pre_ntu = ring->next_to_use ? (ring->next_to_use - 1) : (ring->desc_num - 1); @@ -2244,9 +2265,6 @@ netdev_tx_t hns3_nic_net_xmit(struct sk_buff *skb, struct net_device *netdev) return NETDEV_TX_OK; -fill_err: - hns3_clear_desc(ring, next_to_use_head); - out_err_tx_ok: dev_kfree_skb_any(skb); hns3_tx_doorbell(ring, 0, !netdev_xmit_more()); @@ -2255,6 +2273,8 @@ out_err_tx_ok: static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) { + char format_mac_addr_perm[HNAE3_FORMAT_MAC_ADDR_LEN]; + char format_mac_addr_sa[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hnae3_handle *h = hns3_get_handle(netdev); struct sockaddr *mac_addr = p; int ret; @@ -2263,8 +2283,9 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) return -EADDRNOTAVAIL; if (ether_addr_equal(netdev->dev_addr, mac_addr->sa_data)) { - netdev_info(netdev, "already using mac address %pM\n", - mac_addr->sa_data); + hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data); + netdev_info(netdev, "already using mac address %s\n", + format_mac_addr_sa); return 0; } @@ -2273,8 +2294,10 @@ static int hns3_nic_net_set_mac_address(struct net_device *netdev, void *p) */ if (!hns3_is_phys_func(h->pdev) && !is_zero_ether_addr(netdev->perm_addr)) { - netdev_err(netdev, "has permanent MAC %pM, user MAC %pM not allow\n", - netdev->perm_addr, mac_addr->sa_data); + hnae3_format_mac_addr(format_mac_addr_perm, netdev->perm_addr); + hnae3_format_mac_addr(format_mac_addr_sa, mac_addr->sa_data); + netdev_err(netdev, "has permanent MAC %s, user MAC %s not allow\n", + format_mac_addr_perm, format_mac_addr_sa); return -EPERM; } @@ -2382,90 +2405,89 @@ static netdev_features_t hns3_features_check(struct sk_buff *skb, return features; } +static void hns3_fetch_stats(struct rtnl_link_stats64 *stats, + struct hns3_enet_ring *ring, bool is_tx) +{ + unsigned int start; + + do { + start = u64_stats_fetch_begin_irq(&ring->syncp); + if (is_tx) { + stats->tx_bytes += ring->stats.tx_bytes; + stats->tx_packets += ring->stats.tx_pkts; + stats->tx_dropped += ring->stats.sw_err_cnt; + stats->tx_dropped += ring->stats.tx_vlan_err; + stats->tx_dropped += ring->stats.tx_l4_proto_err; + stats->tx_dropped += ring->stats.tx_l2l3l4_err; + stats->tx_dropped += ring->stats.tx_tso_err; + stats->tx_dropped += ring->stats.over_max_recursion; + stats->tx_dropped += ring->stats.hw_limitation; + stats->tx_dropped += ring->stats.copy_bits_err; + stats->tx_dropped += ring->stats.skb2sgl_err; + stats->tx_dropped += ring->stats.map_sg_err; + stats->tx_errors += ring->stats.sw_err_cnt; + stats->tx_errors += ring->stats.tx_vlan_err; + stats->tx_errors += ring->stats.tx_l4_proto_err; + stats->tx_errors += ring->stats.tx_l2l3l4_err; + stats->tx_errors += ring->stats.tx_tso_err; + stats->tx_errors += ring->stats.over_max_recursion; + stats->tx_errors += ring->stats.hw_limitation; + stats->tx_errors += ring->stats.copy_bits_err; + stats->tx_errors += ring->stats.skb2sgl_err; + stats->tx_errors += ring->stats.map_sg_err; + } else { + stats->rx_bytes += ring->stats.rx_bytes; + stats->rx_packets += ring->stats.rx_pkts; + stats->rx_dropped += ring->stats.l2_err; + stats->rx_errors += ring->stats.l2_err; + stats->rx_errors += ring->stats.l3l4_csum_err; + stats->rx_crc_errors += ring->stats.l2_err; + stats->multicast += ring->stats.rx_multicast; + stats->rx_length_errors += ring->stats.err_pkt_len; + } + } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); +} + static void hns3_nic_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { struct hns3_nic_priv *priv = netdev_priv(netdev); int queue_num = priv->ae_handle->kinfo.num_tqps; struct hnae3_handle *handle = priv->ae_handle; + struct rtnl_link_stats64 ring_total_stats; struct hns3_enet_ring *ring; - u64 rx_length_errors = 0; - u64 rx_crc_errors = 0; - u64 rx_multicast = 0; - unsigned int start; - u64 tx_errors = 0; - u64 rx_errors = 0; unsigned int idx; - u64 tx_bytes = 0; - u64 rx_bytes = 0; - u64 tx_pkts = 0; - u64 rx_pkts = 0; - u64 tx_drop = 0; - u64 rx_drop = 0; if (test_bit(HNS3_NIC_STATE_DOWN, &priv->state)) return; handle->ae_algo->ops->update_stats(handle, &netdev->stats); + memset(&ring_total_stats, 0, sizeof(ring_total_stats)); for (idx = 0; idx < queue_num; idx++) { /* fetch the tx stats */ ring = &priv->ring[idx]; - do { - start = u64_stats_fetch_begin_irq(&ring->syncp); - tx_bytes += ring->stats.tx_bytes; - tx_pkts += ring->stats.tx_pkts; - tx_drop += ring->stats.sw_err_cnt; - tx_drop += ring->stats.tx_vlan_err; - tx_drop += ring->stats.tx_l4_proto_err; - tx_drop += ring->stats.tx_l2l3l4_err; - tx_drop += ring->stats.tx_tso_err; - tx_drop += ring->stats.over_max_recursion; - tx_drop += ring->stats.hw_limitation; - tx_drop += ring->stats.copy_bits_err; - tx_drop += ring->stats.skb2sgl_err; - tx_drop += ring->stats.map_sg_err; - tx_errors += ring->stats.sw_err_cnt; - tx_errors += ring->stats.tx_vlan_err; - tx_errors += ring->stats.tx_l4_proto_err; - tx_errors += ring->stats.tx_l2l3l4_err; - tx_errors += ring->stats.tx_tso_err; - tx_errors += ring->stats.over_max_recursion; - tx_errors += ring->stats.hw_limitation; - tx_errors += ring->stats.copy_bits_err; - tx_errors += ring->stats.skb2sgl_err; - tx_errors += ring->stats.map_sg_err; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); + hns3_fetch_stats(&ring_total_stats, ring, true); /* fetch the rx stats */ ring = &priv->ring[idx + queue_num]; - do { - start = u64_stats_fetch_begin_irq(&ring->syncp); - rx_bytes += ring->stats.rx_bytes; - rx_pkts += ring->stats.rx_pkts; - rx_drop += ring->stats.l2_err; - rx_errors += ring->stats.l2_err; - rx_errors += ring->stats.l3l4_csum_err; - rx_crc_errors += ring->stats.l2_err; - rx_multicast += ring->stats.rx_multicast; - rx_length_errors += ring->stats.err_pkt_len; - } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); - } - - stats->tx_bytes = tx_bytes; - stats->tx_packets = tx_pkts; - stats->rx_bytes = rx_bytes; - stats->rx_packets = rx_pkts; - - stats->rx_errors = rx_errors; - stats->multicast = rx_multicast; - stats->rx_length_errors = rx_length_errors; - stats->rx_crc_errors = rx_crc_errors; + hns3_fetch_stats(&ring_total_stats, ring, false); + } + + stats->tx_bytes = ring_total_stats.tx_bytes; + stats->tx_packets = ring_total_stats.tx_packets; + stats->rx_bytes = ring_total_stats.rx_bytes; + stats->rx_packets = ring_total_stats.rx_packets; + + stats->rx_errors = ring_total_stats.rx_errors; + stats->multicast = ring_total_stats.multicast; + stats->rx_length_errors = ring_total_stats.rx_length_errors; + stats->rx_crc_errors = ring_total_stats.rx_crc_errors; stats->rx_missed_errors = netdev->stats.rx_missed_errors; - stats->tx_errors = tx_errors; - stats->rx_dropped = rx_drop; - stats->tx_dropped = tx_drop; + stats->tx_errors = ring_total_stats.tx_errors; + stats->rx_dropped = ring_total_stats.rx_dropped; + stats->tx_dropped = ring_total_stats.tx_dropped; stats->collisions = netdev->stats.collisions; stats->rx_over_errors = netdev->stats.rx_over_errors; stats->rx_frame_errors = netdev->stats.rx_frame_errors; @@ -2658,18 +2680,8 @@ static int hns3_nic_change_mtu(struct net_device *netdev, int new_mtu) return ret; } -static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) +static int hns3_get_timeout_queue(struct net_device *ndev) { - struct hns3_nic_priv *priv = netdev_priv(ndev); - struct hnae3_handle *h = hns3_get_handle(ndev); - struct hns3_enet_ring *tx_ring; - struct napi_struct *napi; - int timeout_queue = 0; - int hw_head, hw_tail; - int fbd_num, fbd_oft; - int ebd_num, ebd_oft; - int bd_num, bd_err; - int ring_en, tc; int i; /* Find the stopped queue the same way the stack does */ @@ -2678,11 +2690,17 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) unsigned long trans_start; q = netdev_get_tx_queue(ndev, i); - trans_start = q->trans_start; + trans_start = READ_ONCE(q->trans_start); if (netif_xmit_stopped(q) && time_after(jiffies, (trans_start + ndev->watchdog_timeo))) { - timeout_queue = i; +#ifdef CONFIG_BQL + struct dql *dql = &q->dql; + + netdev_info(ndev, "DQL info last_cnt: %u, queued: %u, adj_limit: %u, completed: %u\n", + dql->last_obj_cnt, dql->num_queued, + dql->adj_limit, dql->num_completed); +#endif netdev_info(ndev, "queue state: 0x%lx, delta msecs: %u\n", q->state, jiffies_to_msecs(jiffies - trans_start)); @@ -2690,17 +2708,15 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) } } - if (i == ndev->num_tx_queues) { - netdev_info(ndev, - "no netdev TX timeout queue found, timeout count: %llu\n", - priv->tx_timeout_count); - return false; - } - - priv->tx_timeout_count++; + return i; +} - tx_ring = &priv->ring[timeout_queue]; - napi = &tx_ring->tqp_vector->napi; +static void hns3_dump_queue_stats(struct net_device *ndev, + struct hns3_enet_ring *tx_ring, + int timeout_queue) +{ + struct napi_struct *napi = &tx_ring->tqp_vector->napi; + struct hns3_nic_priv *priv = netdev_priv(ndev); netdev_info(ndev, "tx_timeout count: %llu, queue id: %d, SW_NTU: 0x%x, SW_NTC: 0x%x, napi state: %lu\n", @@ -2716,6 +2732,48 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) "seg_pkt_cnt: %llu, tx_more: %llu, restart_queue: %llu, tx_busy: %llu\n", tx_ring->stats.seg_pkt_cnt, tx_ring->stats.tx_more, tx_ring->stats.restart_queue, tx_ring->stats.tx_busy); +} + +static void hns3_dump_queue_reg(struct net_device *ndev, + struct hns3_enet_ring *tx_ring) +{ + netdev_info(ndev, + "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n", + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_NUM_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_HEAD_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TAIL_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_BD_ERR_REG), + readl(tx_ring->tqp_vector->mask_addr)); + netdev_info(ndev, + "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n", + hns3_tqp_read_reg(tx_ring, HNS3_RING_EN_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_TC_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_FBDNUM_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_OFFSET_REG), + hns3_tqp_read_reg(tx_ring, HNS3_RING_TX_RING_EBDNUM_REG), + hns3_tqp_read_reg(tx_ring, + HNS3_RING_TX_RING_EBD_OFFSET_REG)); +} + +static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) +{ + struct hns3_nic_priv *priv = netdev_priv(ndev); + struct hnae3_handle *h = hns3_get_handle(ndev); + struct hns3_enet_ring *tx_ring; + int timeout_queue; + + timeout_queue = hns3_get_timeout_queue(ndev); + if (timeout_queue >= ndev->num_tx_queues) { + netdev_info(ndev, + "no netdev TX timeout queue found, timeout count: %llu\n", + priv->tx_timeout_count); + return false; + } + + priv->tx_timeout_count++; + + tx_ring = &priv->ring[timeout_queue]; + hns3_dump_queue_stats(ndev, tx_ring, timeout_queue); /* When mac received many pause frames continuous, it's unable to send * packets, which may cause tx timeout @@ -2728,32 +2786,7 @@ static bool hns3_get_tx_timeo_queue_info(struct net_device *ndev) mac_stats.tx_pause_cnt, mac_stats.rx_pause_cnt); } - hw_head = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_HEAD_REG); - hw_tail = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_TAIL_REG); - fbd_num = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_FBDNUM_REG); - fbd_oft = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_OFFSET_REG); - ebd_num = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_EBDNUM_REG); - ebd_oft = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_EBD_OFFSET_REG); - bd_num = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_BD_NUM_REG); - bd_err = readl_relaxed(tx_ring->tqp->io_base + - HNS3_RING_TX_RING_BD_ERR_REG); - ring_en = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_EN_REG); - tc = readl_relaxed(tx_ring->tqp->io_base + HNS3_RING_TX_RING_TC_REG); - - netdev_info(ndev, - "BD_NUM: 0x%x HW_HEAD: 0x%x, HW_TAIL: 0x%x, BD_ERR: 0x%x, INT: 0x%x\n", - bd_num, hw_head, hw_tail, bd_err, - readl(tx_ring->tqp_vector->mask_addr)); - netdev_info(ndev, - "RING_EN: 0x%x, TC: 0x%x, FBD_NUM: 0x%x FBD_OFT: 0x%x, EBD_NUM: 0x%x, EBD_OFT: 0x%x\n", - ring_en, tc, fbd_num, fbd_oft, ebd_num, ebd_oft); + hns3_dump_queue_reg(ndev, tx_ring); return true; } @@ -2836,14 +2869,16 @@ static int hns3_nic_set_vf_rate(struct net_device *ndev, int vf, static int hns3_nic_set_vf_mac(struct net_device *netdev, int vf_id, u8 *mac) { struct hnae3_handle *h = hns3_get_handle(netdev); + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; if (!h->ae_algo->ops->set_vf_mac) return -EOPNOTSUPP; if (is_multicast_ether_addr(mac)) { + hnae3_format_mac_addr(format_mac_addr, mac); netdev_err(netdev, - "Invalid MAC:%pM specified. Could not set MAC\n", - mac); + "Invalid MAC:%s specified. Could not set MAC\n", + format_mac_addr); return -EINVAL; } @@ -3497,17 +3532,13 @@ static bool hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, for (i = 0; i < cleand_count; i++) { desc_cb = &ring->desc_cb[ring->next_to_use]; if (desc_cb->reuse_flag) { - u64_stats_update_begin(&ring->syncp); - ring->stats.reuse_pg_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, reuse_pg_cnt); hns3_reuse_buffer(ring, ring->next_to_use); } else { ret = hns3_alloc_and_map_buffer(ring, &res_cbs); if (ret) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); hns3_rl_err(ring_to_netdev(ring), "alloc rx buffer failed: %d\n", @@ -3519,9 +3550,7 @@ static bool hns3_nic_alloc_rx_buffers(struct hns3_enet_ring *ring, } hns3_replace_buffer(ring, ring->next_to_use, &res_cbs); - u64_stats_update_begin(&ring->syncp); - ring->stats.non_reuse_pg++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, non_reuse_pg); } ring_ptr_move_fw(ring, next_to_use); @@ -3536,6 +3565,34 @@ static bool hns3_can_reuse_page(struct hns3_desc_cb *cb) return page_count(cb->priv) == cb->pagecnt_bias; } +static int hns3_handle_rx_copybreak(struct sk_buff *skb, int i, + struct hns3_enet_ring *ring, + int pull_len, + struct hns3_desc_cb *desc_cb) +{ + struct hns3_desc *desc = &ring->desc[ring->next_to_clean]; + u32 frag_offset = desc_cb->page_offset + pull_len; + int size = le16_to_cpu(desc->rx.size); + u32 frag_size = size - pull_len; + void *frag = napi_alloc_frag(frag_size); + + if (unlikely(!frag)) { + hns3_ring_stats_update(ring, frag_alloc_err); + + hns3_rl_err(ring_to_netdev(ring), + "failed to allocate rx frag\n"); + return -ENOMEM; + } + + desc_cb->reuse_flag = 1; + memcpy(frag, desc_cb->buf + frag_offset, frag_size); + skb_add_rx_frag(skb, i, virt_to_page(frag), + offset_in_page(frag), frag_size, frag_size); + + hns3_ring_stats_update(ring, frag_alloc); + return 0; +} + static void hns3_nic_reuse_page(struct sk_buff *skb, int i, struct hns3_enet_ring *ring, int pull_len, struct hns3_desc_cb *desc_cb) @@ -3545,6 +3602,7 @@ static void hns3_nic_reuse_page(struct sk_buff *skb, int i, int size = le16_to_cpu(desc->rx.size); u32 truesize = hns3_buf_size(ring); u32 frag_size = size - pull_len; + int ret = 0; bool reused; if (ring->page_pool) { @@ -3579,27 +3637,9 @@ static void hns3_nic_reuse_page(struct sk_buff *skb, int i, desc_cb->page_offset = 0; desc_cb->reuse_flag = 1; } else if (frag_size <= ring->rx_copybreak) { - void *frag = napi_alloc_frag(frag_size); - - if (unlikely(!frag)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.frag_alloc_err++; - u64_stats_update_end(&ring->syncp); - - hns3_rl_err(ring_to_netdev(ring), - "failed to allocate rx frag\n"); + ret = hns3_handle_rx_copybreak(skb, i, ring, pull_len, desc_cb); + if (ret) goto out; - } - - desc_cb->reuse_flag = 1; - memcpy(frag, desc_cb->buf + frag_offset, frag_size); - skb_add_rx_frag(skb, i, virt_to_page(frag), - offset_in_page(frag), frag_size, frag_size); - - u64_stats_update_begin(&ring->syncp); - ring->stats.frag_alloc++; - u64_stats_update_end(&ring->syncp); - return; } out: @@ -3682,9 +3722,7 @@ static bool hns3_checksum_complete(struct hns3_enet_ring *ring, hns3_rx_ptype_tbl[ptype].ip_summed != CHECKSUM_COMPLETE) return false; - u64_stats_update_begin(&ring->syncp); - ring->stats.csum_complete++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, csum_complete); skb->ip_summed = CHECKSUM_COMPLETE; skb->csum = csum_unfold((__force __sum16)csum); @@ -3758,9 +3796,7 @@ static void hns3_rx_checksum(struct hns3_enet_ring *ring, struct sk_buff *skb, if (unlikely(l234info & (BIT(HNS3_RXD_L3E_B) | BIT(HNS3_RXD_L4E_B) | BIT(HNS3_RXD_OL3E_B) | BIT(HNS3_RXD_OL4E_B)))) { - u64_stats_update_begin(&ring->syncp); - ring->stats.l3l4_csum_err++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, l3l4_csum_err); return; } @@ -3851,10 +3887,7 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, skb = ring->skb; if (unlikely(!skb)) { hns3_rl_err(netdev, "alloc rx skb fail\n"); - - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); return -ENOMEM; } @@ -3885,9 +3918,7 @@ static int hns3_alloc_skb(struct hns3_enet_ring *ring, unsigned int length, if (ring->page_pool) skb_mark_for_recycle(skb); - u64_stats_update_begin(&ring->syncp); - ring->stats.seg_pkt_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, seg_pkt_cnt); ring->pull_len = eth_get_headlen(netdev, va, HNS3_RX_HEAD_SIZE); __skb_put(skb, ring->pull_len); @@ -4015,6 +4046,39 @@ static void hns3_set_rx_skb_rss_type(struct hns3_enet_ring *ring, skb_set_hash(skb, rss_hash, rss_type); } +static void hns3_handle_rx_ts_info(struct net_device *netdev, + struct hns3_desc *desc, struct sk_buff *skb, + u32 bd_base_info) +{ + if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) { + struct hnae3_handle *h = hns3_get_handle(netdev); + u32 nsec = le32_to_cpu(desc->ts_nsec); + u32 sec = le32_to_cpu(desc->ts_sec); + + if (h->ae_algo->ops->get_rx_hwts) + h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec); + } +} + +static void hns3_handle_rx_vlan_tag(struct hns3_enet_ring *ring, + struct hns3_desc *desc, struct sk_buff *skb, + u32 l234info) +{ + struct net_device *netdev = ring_to_netdev(ring); + + /* Based on hw strategy, the tag offloaded will be stored at + * ot_vlan_tag in two layer tag case, and stored at vlan_tag + * in one layer tag case. + */ + if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) { + u16 vlan_tag; + + if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag)) + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), + vlan_tag); + } +} + static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) { struct net_device *netdev = ring_to_netdev(ring); @@ -4037,26 +4101,9 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) ol_info = le32_to_cpu(desc->rx.ol_info); csum = le16_to_cpu(desc->csum); - if (unlikely(bd_base_info & BIT(HNS3_RXD_TS_VLD_B))) { - struct hnae3_handle *h = hns3_get_handle(netdev); - u32 nsec = le32_to_cpu(desc->ts_nsec); - u32 sec = le32_to_cpu(desc->ts_sec); - - if (h->ae_algo->ops->get_rx_hwts) - h->ae_algo->ops->get_rx_hwts(h, skb, nsec, sec); - } - - /* Based on hw strategy, the tag offloaded will be stored at - * ot_vlan_tag in two layer tag case, and stored at vlan_tag - * in one layer tag case. - */ - if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) { - u16 vlan_tag; + hns3_handle_rx_ts_info(netdev, desc, skb, bd_base_info); - if (hns3_parse_vlan_tag(ring, desc, l234info, &vlan_tag)) - __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), - vlan_tag); - } + hns3_handle_rx_vlan_tag(ring, desc, skb, l234info); if (unlikely(!desc->rx.pkt_len || (l234info & (BIT(HNS3_RXD_TRUNCAT_B) | BIT(HNS3_RXD_L2E_B))))) { @@ -4079,9 +4126,7 @@ static int hns3_handle_bdinfo(struct hns3_enet_ring *ring, struct sk_buff *skb) ret = hns3_set_gro_and_checksum(ring, skb, l234info, bd_base_info, ol_info, csum); if (unlikely(ret)) { - u64_stats_update_begin(&ring->syncp); - ring->stats.rx_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, rx_err_cnt); return ret; } @@ -4297,87 +4342,70 @@ static int hns3_nic_common_poll(struct napi_struct *napi, int budget) return rx_pkt_total; } -static int hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, - struct hnae3_ring_chain_node *head) +static int hns3_create_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, + struct hnae3_ring_chain_node **head, + bool is_tx) { + u32 bit_value = is_tx ? HNAE3_RING_TYPE_TX : HNAE3_RING_TYPE_RX; + u32 field_value = is_tx ? HNAE3_RING_GL_TX : HNAE3_RING_GL_RX; + struct hnae3_ring_chain_node *cur_chain = *head; struct pci_dev *pdev = tqp_vector->handle->pdev; - struct hnae3_ring_chain_node *cur_chain = head; struct hnae3_ring_chain_node *chain; - struct hns3_enet_ring *tx_ring; - struct hns3_enet_ring *rx_ring; - - tx_ring = tqp_vector->tx_group.ring; - if (tx_ring) { - cur_chain->tqp_index = tx_ring->tqp->tqp_index; - hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B, - HNAE3_RING_TYPE_TX); - hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M, - HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_TX); - - cur_chain->next = NULL; - - while (tx_ring->next) { - tx_ring = tx_ring->next; - - chain = devm_kzalloc(&pdev->dev, sizeof(*chain), - GFP_KERNEL); - if (!chain) - goto err_free_chain; - - cur_chain->next = chain; - chain->tqp_index = tx_ring->tqp->tqp_index; - hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B, - HNAE3_RING_TYPE_TX); - hnae3_set_field(chain->int_gl_idx, - HNAE3_RING_GL_IDX_M, - HNAE3_RING_GL_IDX_S, - HNAE3_RING_GL_TX); - - cur_chain = chain; - } - } + struct hns3_enet_ring *ring; - rx_ring = tqp_vector->rx_group.ring; - if (!tx_ring && rx_ring) { - cur_chain->next = NULL; - cur_chain->tqp_index = rx_ring->tqp->tqp_index; - hnae3_set_bit(cur_chain->flag, HNAE3_RING_TYPE_B, - HNAE3_RING_TYPE_RX); - hnae3_set_field(cur_chain->int_gl_idx, HNAE3_RING_GL_IDX_M, - HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX); + ring = is_tx ? tqp_vector->tx_group.ring : tqp_vector->rx_group.ring; - rx_ring = rx_ring->next; + if (cur_chain) { + while (cur_chain->next) + cur_chain = cur_chain->next; } - while (rx_ring) { + while (ring) { chain = devm_kzalloc(&pdev->dev, sizeof(*chain), GFP_KERNEL); if (!chain) - goto err_free_chain; - - cur_chain->next = chain; - chain->tqp_index = rx_ring->tqp->tqp_index; + return -ENOMEM; + if (cur_chain) + cur_chain->next = chain; + else + *head = chain; + chain->tqp_index = ring->tqp->tqp_index; hnae3_set_bit(chain->flag, HNAE3_RING_TYPE_B, - HNAE3_RING_TYPE_RX); - hnae3_set_field(chain->int_gl_idx, HNAE3_RING_GL_IDX_M, - HNAE3_RING_GL_IDX_S, HNAE3_RING_GL_RX); + bit_value); + hnae3_set_field(chain->int_gl_idx, + HNAE3_RING_GL_IDX_M, + HNAE3_RING_GL_IDX_S, field_value); cur_chain = chain; - rx_ring = rx_ring->next; + ring = ring->next; } return 0; +} + +static struct hnae3_ring_chain_node * +hns3_get_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector) +{ + struct pci_dev *pdev = tqp_vector->handle->pdev; + struct hnae3_ring_chain_node *cur_chain = NULL; + struct hnae3_ring_chain_node *chain; + + if (hns3_create_ring_chain(tqp_vector, &cur_chain, true)) + goto err_free_chain; + + if (hns3_create_ring_chain(tqp_vector, &cur_chain, false)) + goto err_free_chain; + + return cur_chain; err_free_chain: - cur_chain = head->next; while (cur_chain) { chain = cur_chain->next; devm_kfree(&pdev->dev, cur_chain); cur_chain = chain; } - head->next = NULL; - return -ENOMEM; + return NULL; } static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, @@ -4386,7 +4414,7 @@ static void hns3_free_vector_ring_chain(struct hns3_enet_tqp_vector *tqp_vector, struct pci_dev *pdev = tqp_vector->handle->pdev; struct hnae3_ring_chain_node *chain_tmp, *chain; - chain = head->next; + chain = head; while (chain) { chain_tmp = chain->next; @@ -4501,7 +4529,7 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) } for (i = 0; i < priv->vector_num; i++) { - struct hnae3_ring_chain_node vector_ring_chain; + struct hnae3_ring_chain_node *vector_ring_chain; tqp_vector = &priv->tqp_vector[i]; @@ -4511,15 +4539,16 @@ static int hns3_nic_init_vector_data(struct hns3_nic_priv *priv) tqp_vector->tx_group.total_packets = 0; tqp_vector->handle = h; - ret = hns3_get_vector_ring_chain(tqp_vector, - &vector_ring_chain); - if (ret) + vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector); + if (!vector_ring_chain) { + ret = -ENOMEM; goto map_ring_fail; + } ret = h->ae_algo->ops->map_ring_to_vector(h, - tqp_vector->vector_irq, &vector_ring_chain); + tqp_vector->vector_irq, vector_ring_chain); - hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain); + hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain); if (ret) goto map_ring_fail; @@ -4618,7 +4647,7 @@ static void hns3_clear_ring_group(struct hns3_enet_ring_group *group) static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv) { - struct hnae3_ring_chain_node vector_ring_chain; + struct hnae3_ring_chain_node *vector_ring_chain; struct hnae3_handle *h = priv->ae_handle; struct hns3_enet_tqp_vector *tqp_vector; int i; @@ -4633,13 +4662,14 @@ static void hns3_nic_uninit_vector_data(struct hns3_nic_priv *priv) * chain between vector and ring, we should go on to deal with * the remaining options. */ - if (hns3_get_vector_ring_chain(tqp_vector, &vector_ring_chain)) + vector_ring_chain = hns3_get_vector_ring_chain(tqp_vector); + if (!vector_ring_chain) dev_warn(priv->dev, "failed to get ring chain\n"); h->ae_algo->ops->unmap_ring_from_vector(h, - tqp_vector->vector_irq, &vector_ring_chain); + tqp_vector->vector_irq, vector_ring_chain); - hns3_free_vector_ring_chain(tqp_vector, &vector_ring_chain); + hns3_free_vector_ring_chain(tqp_vector, vector_ring_chain); hns3_clear_ring_group(&tqp_vector->rx_group); hns3_clear_ring_group(&tqp_vector->tx_group); @@ -4934,6 +4964,7 @@ static void hns3_uninit_all_ring(struct hns3_nic_priv *priv) static int hns3_init_mac_addr(struct net_device *netdev) { struct hns3_nic_priv *priv = netdev_priv(netdev); + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hnae3_handle *h = priv->ae_handle; u8 mac_addr_temp[ETH_ALEN]; int ret = 0; @@ -4944,8 +4975,9 @@ static int hns3_init_mac_addr(struct net_device *netdev) /* Check if the MAC address is valid, if not get a random one */ if (!is_valid_ether_addr(mac_addr_temp)) { eth_hw_addr_random(netdev); - dev_warn(priv->dev, "using random MAC address %pM\n", - netdev->dev_addr); + hnae3_format_mac_addr(format_mac_addr, netdev->dev_addr); + dev_warn(priv->dev, "using random MAC address %s\n", + format_mac_addr); } else if (!ether_addr_equal(netdev->dev_addr, mac_addr_temp)) { eth_hw_addr_set(netdev, mac_addr_temp); ether_addr_copy(netdev->perm_addr, mac_addr_temp); @@ -4997,8 +5029,10 @@ static void hns3_client_stop(struct hnae3_handle *handle) static void hns3_info_show(struct hns3_nic_priv *priv) { struct hnae3_knic_private_info *kinfo = &priv->ae_handle->kinfo; + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; - dev_info(priv->dev, "MAC address: %pM\n", priv->netdev->dev_addr); + hnae3_format_mac_addr(format_mac_addr, priv->netdev->dev_addr); + dev_info(priv->dev, "MAC address: %s\n", format_mac_addr); dev_info(priv->dev, "Task queue pairs numbers: %u\n", kinfo->num_tqps); dev_info(priv->dev, "RSS size: %u\n", kinfo->rss_size); dev_info(priv->dev, "Allocated RSS size: %u\n", kinfo->req_rss_size); @@ -5287,9 +5321,7 @@ static int hns3_clear_rx_ring(struct hns3_enet_ring *ring) if (!ring->desc_cb[ring->next_to_use].reuse_flag) { ret = hns3_alloc_and_map_buffer(ring, &res_cbs); if (ret) { - u64_stats_update_begin(&ring->syncp); - ring->stats.sw_err_cnt++; - u64_stats_update_end(&ring->syncp); + hns3_ring_stats_update(ring, sw_err_cnt); /* if alloc new buffer fail, exit directly * and reclear in up flow. */ @@ -5531,8 +5563,8 @@ static int hns3_reset_notify_uninit_enet(struct hnae3_handle *handle) return 0; } -static int hns3_reset_notify(struct hnae3_handle *handle, - enum hnae3_reset_notify_type type) +int hns3_reset_notify(struct hnae3_handle *handle, + enum hnae3_reset_notify_type type) { int ret = 0; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h index 1715c98d906d..a05a0c7423ce 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_enet.h @@ -10,6 +10,9 @@ #include "hnae3.h" +struct iphdr; +struct ipv6hdr; + enum hns3_nic_state { HNS3_NIC_STATE_TESTING, HNS3_NIC_STATE_RESETTING, @@ -621,6 +624,11 @@ static inline int ring_space(struct hns3_enet_ring *ring) (begin - end)) - 1; } +static inline u32 hns3_tqp_read_reg(struct hns3_enet_ring *ring, u32 reg) +{ + return readl_relaxed(ring->tqp->io_base + reg); +} + static inline u32 hns3_read_reg(void __iomem *base, u32 reg) { return readl(base + reg); @@ -655,6 +663,13 @@ static inline bool hns3_nic_resetting(struct net_device *netdev) #define hns3_buf_size(_ring) ((_ring)->buf_size) +#define hns3_ring_stats_update(ring, cnt) do { \ + typeof(ring) (tmp) = (ring); \ + u64_stats_update_begin(&(tmp)->syncp); \ + ((tmp)->stats.cnt)++; \ + u64_stats_update_end(&(tmp)->syncp); \ +} while (0) \ + static inline unsigned int hns3_page_order(struct hns3_enet_ring *ring) { #if (PAGE_SIZE < 8192) @@ -705,6 +720,8 @@ void hns3_set_vector_coalesce_tx_ql(struct hns3_enet_tqp_vector *tqp_vector, u32 ql_value); void hns3_request_update_promisc_mode(struct hnae3_handle *handle); +int hns3_reset_notify(struct hnae3_handle *handle, + enum hnae3_reset_notify_type type); #ifdef CONFIG_HNS3_DCB void hns3_dcbnl_setup(struct hnae3_handle *handle); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c index c9b4568d7a8d..c06c39ece80d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3_ethtool.c @@ -643,11 +643,13 @@ static u32 hns3_get_link(struct net_device *netdev) } static void hns3_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns3_nic_priv *priv = netdev_priv(netdev); struct hnae3_handle *h = priv->ae_handle; - int queue_num = h->kinfo.num_tqps; + int rx_queue_index = h->kinfo.num_tqps; if (hns3_nic_resetting(netdev)) { netdev_err(netdev, "dev resetting!"); @@ -658,7 +660,8 @@ static void hns3_get_ringparam(struct net_device *netdev, param->rx_max_pending = HNS3_RING_MAX_PENDING; param->tx_pending = priv->ring[0].desc_num; - param->rx_pending = priv->ring[queue_num].desc_num; + param->rx_pending = priv->ring[rx_queue_index].desc_num; + kernel_param->rx_buf_len = priv->ring[rx_queue_index].buf_size; } static void hns3_get_pauseparam(struct net_device *netdev, @@ -1064,14 +1067,23 @@ static struct hns3_enet_ring *hns3_backup_ringparam(struct hns3_nic_priv *priv) } static int hns3_check_ringparam(struct net_device *ndev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param) { +#define RX_BUF_LEN_2K 2048 +#define RX_BUF_LEN_4K 4096 if (hns3_nic_resetting(ndev)) return -EBUSY; if (param->rx_mini_pending || param->rx_jumbo_pending) return -EINVAL; + if (kernel_param->rx_buf_len != RX_BUF_LEN_2K && + kernel_param->rx_buf_len != RX_BUF_LEN_4K) { + netdev_err(ndev, "Rx buf len only support 2048 and 4096\n"); + return -EINVAL; + } + if (param->tx_pending > HNS3_RING_MAX_PENDING || param->tx_pending < HNS3_RING_MIN_PENDING || param->rx_pending > HNS3_RING_MAX_PENDING || @@ -1084,8 +1096,26 @@ static int hns3_check_ringparam(struct net_device *ndev, return 0; } +static int hns3_change_rx_buf_len(struct net_device *ndev, u32 rx_buf_len) +{ + struct hns3_nic_priv *priv = netdev_priv(ndev); + struct hnae3_handle *h = priv->ae_handle; + int i; + + h->kinfo.rx_buf_len = rx_buf_len; + + for (i = 0; i < h->kinfo.num_tqps; i++) { + h->kinfo.tqp[i]->buf_size = rx_buf_len; + priv->ring[i + h->kinfo.num_tqps].buf_size = rx_buf_len; + } + + return 0; +} + static int hns3_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct hns3_nic_priv *priv = netdev_priv(ndev); struct hnae3_handle *h = priv->ae_handle; @@ -1094,9 +1124,10 @@ static int hns3_set_ringparam(struct net_device *ndev, u32 old_tx_desc_num, new_tx_desc_num; u32 old_rx_desc_num, new_rx_desc_num; u16 queue_num = h->kinfo.num_tqps; + u32 old_rx_buf_len; int ret, i; - ret = hns3_check_ringparam(ndev, param); + ret = hns3_check_ringparam(ndev, param, kernel_param); if (ret) return ret; @@ -1105,8 +1136,10 @@ static int hns3_set_ringparam(struct net_device *ndev, new_rx_desc_num = ALIGN(param->rx_pending, HNS3_RING_BD_MULTIPLE); old_tx_desc_num = priv->ring[0].desc_num; old_rx_desc_num = priv->ring[queue_num].desc_num; + old_rx_buf_len = priv->ring[queue_num].buf_size; if (old_tx_desc_num == new_tx_desc_num && - old_rx_desc_num == new_rx_desc_num) + old_rx_desc_num == new_rx_desc_num && + kernel_param->rx_buf_len == old_rx_buf_len) return 0; tmp_rings = hns3_backup_ringparam(priv); @@ -1117,19 +1150,22 @@ static int hns3_set_ringparam(struct net_device *ndev, } netdev_info(ndev, - "Changing Tx/Rx ring depth from %u/%u to %u/%u\n", + "Changing Tx/Rx ring depth from %u/%u to %u/%u, Changing rx buffer len from %d to %d\n", old_tx_desc_num, old_rx_desc_num, - new_tx_desc_num, new_rx_desc_num); + new_tx_desc_num, new_rx_desc_num, + old_rx_buf_len, kernel_param->rx_buf_len); if (if_running) ndev->netdev_ops->ndo_stop(ndev); hns3_change_all_ring_bd_num(priv, new_tx_desc_num, new_rx_desc_num); + hns3_change_rx_buf_len(ndev, kernel_param->rx_buf_len); ret = hns3_init_all_ring(priv); if (ret) { - netdev_err(ndev, "Change bd num fail, revert to old value(%d)\n", + netdev_err(ndev, "set ringparam fail, revert to old value(%d)\n", ret); + hns3_change_rx_buf_len(ndev, old_rx_buf_len); hns3_change_all_ring_bd_num(priv, old_tx_desc_num, old_rx_desc_num); for (i = 0; i < h->kinfo.num_tqps * 2; i++) @@ -1699,6 +1735,7 @@ static int hns3_get_tunable(struct net_device *netdev, void *data) { struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = priv->ae_handle; int ret = 0; switch (tuna->id) { @@ -1709,6 +1746,9 @@ static int hns3_get_tunable(struct net_device *netdev, case ETHTOOL_RX_COPYBREAK: *(u32 *)data = priv->rx_copybreak; break; + case ETHTOOL_TX_COPYBREAK_BUF_SIZE: + *(u32 *)data = h->kinfo.tx_spare_buf_size; + break; default: ret = -EOPNOTSUPP; break; @@ -1717,11 +1757,43 @@ static int hns3_get_tunable(struct net_device *netdev, return ret; } +static int hns3_set_tx_spare_buf_size(struct net_device *netdev, + u32 data) +{ + struct hns3_nic_priv *priv = netdev_priv(netdev); + struct hnae3_handle *h = priv->ae_handle; + int ret; + + if (hns3_nic_resetting(netdev)) + return -EBUSY; + + h->kinfo.tx_spare_buf_size = data; + + ret = hns3_reset_notify(h, HNAE3_DOWN_CLIENT); + if (ret) + return ret; + + ret = hns3_reset_notify(h, HNAE3_UNINIT_CLIENT); + if (ret) + return ret; + + ret = hns3_reset_notify(h, HNAE3_INIT_CLIENT); + if (ret) + return ret; + + ret = hns3_reset_notify(h, HNAE3_UP_CLIENT); + if (ret) + hns3_reset_notify(h, HNAE3_UNINIT_CLIENT); + + return ret; +} + static int hns3_set_tunable(struct net_device *netdev, const struct ethtool_tunable *tuna, const void *data) { struct hns3_nic_priv *priv = netdev_priv(netdev); + u32 old_tx_spare_buf_size, new_tx_spare_buf_size; struct hnae3_handle *h = priv->ae_handle; int i, ret = 0; @@ -1740,6 +1812,26 @@ static int hns3_set_tunable(struct net_device *netdev, priv->ring[i].rx_copybreak = priv->rx_copybreak; break; + case ETHTOOL_TX_COPYBREAK_BUF_SIZE: + old_tx_spare_buf_size = h->kinfo.tx_spare_buf_size; + new_tx_spare_buf_size = *(u32 *)data; + ret = hns3_set_tx_spare_buf_size(netdev, new_tx_spare_buf_size); + if (ret) { + int ret1; + + netdev_warn(netdev, + "change tx spare buf size fail, revert to old value\n"); + ret1 = hns3_set_tx_spare_buf_size(netdev, + old_tx_spare_buf_size); + if (ret1) { + netdev_err(netdev, + "revert to old tx spare buf size fail\n"); + return ret1; + } + + return ret; + } + break; default: ret = -EOPNOTSUPP; break; @@ -1755,6 +1847,8 @@ static int hns3_set_tunable(struct net_device *netdev, ETHTOOL_COALESCE_MAX_FRAMES | \ ETHTOOL_COALESCE_USE_CQE) +#define HNS3_ETHTOOL_RING ETHTOOL_RING_USE_RX_BUF_LEN + static int hns3_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *info) { @@ -1833,6 +1927,7 @@ static int hns3_get_link_ext_state(struct net_device *netdev, static const struct ethtool_ops hns3vf_ethtool_ops = { .supported_coalesce_params = HNS3_ETHTOOL_COALESCE, + .supported_ring_params = HNS3_ETHTOOL_RING, .get_drvinfo = hns3_get_drvinfo, .get_ringparam = hns3_get_ringparam, .set_ringparam = hns3_set_ringparam, @@ -1864,6 +1959,7 @@ static const struct ethtool_ops hns3vf_ethtool_ops = { static const struct ethtool_ops hns3_ethtool_ops = { .supported_coalesce_params = HNS3_ETHTOOL_COALESCE, + .supported_ring_params = HNS3_ETHTOOL_RING, .self_test = hns3_self_test, .get_drvinfo = hns3_get_drvinfo, .get_link = hns3_get_link, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c index 4e0a8c2f7c05..c287be8bc48d 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.c @@ -77,6 +77,10 @@ static const struct hclge_dbg_reg_type_info hclge_dbg_reg_info[] = { .cmd = HCLGE_OPC_DFX_TQP_REG } }, }; +/* make sure: len(name) + interval >= maxlen(item data) + 2, + * for example, name = "pkt_num"(len: 7), the prototype of item data is u32, + * and print as "%u"(maxlen: 10), so the interval should be at least 5. + */ static void hclge_dbg_fill_content(char *content, u16 len, const struct hclge_dbg_item *items, const char **result, u16 size) @@ -99,7 +103,7 @@ static void hclge_dbg_fill_content(char *content, u16 len, static char *hclge_dbg_get_func_id_str(char *buf, u8 id) { if (id) - sprintf(buf, "vf%u", id - 1); + sprintf(buf, "vf%u", id - 1U); else sprintf(buf, "pf"); @@ -258,12 +262,29 @@ hclge_dbg_dump_reg_common(struct hclge_dev *hdev, return 0; } +static const struct hclge_dbg_status_dfx_info hclge_dbg_mac_en_status[] = { + {HCLGE_MAC_TX_EN_B, "mac_trans_en"}, + {HCLGE_MAC_RX_EN_B, "mac_rcv_en"}, + {HCLGE_MAC_PAD_TX_B, "pad_trans_en"}, + {HCLGE_MAC_PAD_RX_B, "pad_rcv_en"}, + {HCLGE_MAC_1588_TX_B, "1588_trans_en"}, + {HCLGE_MAC_1588_RX_B, "1588_rcv_en"}, + {HCLGE_MAC_APP_LP_B, "mac_app_loop_en"}, + {HCLGE_MAC_LINE_LP_B, "mac_line_loop_en"}, + {HCLGE_MAC_FCS_TX_B, "mac_fcs_tx_en"}, + {HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B, "mac_rx_oversize_truncate_en"}, + {HCLGE_MAC_RX_FCS_STRIP_B, "mac_rx_fcs_strip_en"}, + {HCLGE_MAC_RX_FCS_B, "mac_rx_fcs_en"}, + {HCLGE_MAC_TX_UNDER_MIN_ERR_B, "mac_tx_under_min_err_en"}, + {HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B, "mac_tx_oversize_truncate_en"} +}; + static int hclge_dbg_dump_mac_enable_status(struct hclge_dev *hdev, char *buf, int len, int *pos) { struct hclge_config_mac_mode_cmd *req; struct hclge_desc desc; - u32 loop_en; + u32 loop_en, i, offset; int ret; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_CONFIG_MAC_MODE, true); @@ -278,39 +299,12 @@ static int hclge_dbg_dump_mac_enable_status(struct hclge_dev *hdev, char *buf, req = (struct hclge_config_mac_mode_cmd *)desc.data; loop_en = le32_to_cpu(req->txrx_pad_fcs_loop_en); - *pos += scnprintf(buf + *pos, len - *pos, "mac_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_TX_EN_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_EN_B)); - *pos += scnprintf(buf + *pos, len - *pos, "pad_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_PAD_TX_B)); - *pos += scnprintf(buf + *pos, len - *pos, "pad_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_PAD_RX_B)); - *pos += scnprintf(buf + *pos, len - *pos, "1588_trans_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_1588_TX_B)); - *pos += scnprintf(buf + *pos, len - *pos, "1588_rcv_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_1588_RX_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_app_loop_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_APP_LP_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_line_loop_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_LINE_LP_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_fcs_tx_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_FCS_TX_B)); - *pos += scnprintf(buf + *pos, len - *pos, - "mac_rx_oversize_truncate_en: %#x\n", - hnae3_get_bit(loop_en, - HCLGE_MAC_RX_OVERSIZE_TRUNCATE_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_rx_fcs_strip_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_STRIP_B)); - *pos += scnprintf(buf + *pos, len - *pos, "mac_rx_fcs_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_RX_FCS_B)); - *pos += scnprintf(buf + *pos, len - *pos, - "mac_tx_under_min_err_en: %#x\n", - hnae3_get_bit(loop_en, HCLGE_MAC_TX_UNDER_MIN_ERR_B)); - *pos += scnprintf(buf + *pos, len - *pos, - "mac_tx_oversize_truncate_en: %#x\n", - hnae3_get_bit(loop_en, - HCLGE_MAC_TX_OVERSIZE_TRUNCATE_B)); + for (i = 0; i < ARRAY_SIZE(hclge_dbg_mac_en_status); i++) { + offset = hclge_dbg_mac_en_status[i].offset; + *pos += scnprintf(buf + *pos, len - *pos, "%s: %#x\n", + hclge_dbg_mac_en_status[i].message, + hnae3_get_bit(loop_en, offset)); + } return 0; } @@ -788,7 +782,6 @@ static int hclge_dbg_dump_tm_pg(struct hclge_dev *hdev, char *buf, int len) data_str = kcalloc(ARRAY_SIZE(tm_pg_items), HCLGE_DBG_DATA_STR_LEN, GFP_KERNEL); - if (!data_str) return -ENOMEM; @@ -1614,8 +1607,19 @@ static int hclge_dbg_dump_fd_counter(struct hclge_dev *hdev, char *buf, int len) return 0; } +static const struct hclge_dbg_status_dfx_info hclge_dbg_rst_info[] = { + {HCLGE_MISC_VECTOR_REG_BASE, "vector0 interrupt enable status"}, + {HCLGE_MISC_RESET_STS_REG, "reset interrupt source"}, + {HCLGE_MISC_VECTOR_INT_STS, "reset interrupt status"}, + {HCLGE_RAS_PF_OTHER_INT_STS_REG, "RAS interrupt status"}, + {HCLGE_GLOBAL_RESET_REG, "hardware reset status"}, + {HCLGE_NIC_CSQ_DEPTH_REG, "handshake status"}, + {HCLGE_FUN_RST_ING, "function reset status"} +}; + int hclge_dbg_dump_rst_info(struct hclge_dev *hdev, char *buf, int len) { + u32 i, offset; int pos = 0; pos += scnprintf(buf + pos, len - pos, "PF reset count: %u\n", @@ -1634,22 +1638,14 @@ int hclge_dbg_dump_rst_info(struct hclge_dev *hdev, char *buf, int len) hdev->rst_stats.reset_cnt); pos += scnprintf(buf + pos, len - pos, "reset fail count: %u\n", hdev->rst_stats.reset_fail_cnt); - pos += scnprintf(buf + pos, len - pos, - "vector0 interrupt enable status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_REG_BASE)); - pos += scnprintf(buf + pos, len - pos, "reset interrupt source: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_RESET_STS_REG)); - pos += scnprintf(buf + pos, len - pos, "reset interrupt status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_MISC_VECTOR_INT_STS)); - pos += scnprintf(buf + pos, len - pos, "RAS interrupt status: 0x%x\n", - hclge_read_dev(&hdev->hw, - HCLGE_RAS_PF_OTHER_INT_STS_REG)); - pos += scnprintf(buf + pos, len - pos, "hardware reset status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_GLOBAL_RESET_REG)); - pos += scnprintf(buf + pos, len - pos, "handshake status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_NIC_CSQ_DEPTH_REG)); - pos += scnprintf(buf + pos, len - pos, "function reset status: 0x%x\n", - hclge_read_dev(&hdev->hw, HCLGE_FUN_RST_ING)); + + for (i = 0; i < ARRAY_SIZE(hclge_dbg_rst_info); i++) { + offset = hclge_dbg_rst_info[i].offset; + pos += scnprintf(buf + pos, len - pos, "%s: 0x%x\n", + hclge_dbg_rst_info[i].message, + hclge_read_dev(&hdev->hw, offset)); + } + pos += scnprintf(buf + pos, len - pos, "hdev state: 0x%lx\n", hdev->state); @@ -1771,7 +1767,7 @@ hclge_dbg_get_imp_stats_info(struct hclge_dev *hdev, char *buf, int len) #define HCLGE_MAX_NCL_CONFIG_LENGTH 16384 static void hclge_ncl_config_data_print(struct hclge_desc *desc, int *index, - char *buf, int *len, int *pos) + char *buf, int len, int *pos) { #define HCLGE_CMD_DATA_NUM 6 @@ -1783,7 +1779,7 @@ static void hclge_ncl_config_data_print(struct hclge_desc *desc, int *index, if (i == 0 && j == 0) continue; - *pos += scnprintf(buf + *pos, *len - *pos, + *pos += scnprintf(buf + *pos, len - *pos, "0x%04x | 0x%08x\n", offset, le32_to_cpu(desc[i].data[j])); @@ -1821,7 +1817,7 @@ hclge_dbg_dump_ncl_config(struct hclge_dev *hdev, char *buf, int len) if (ret) return ret; - hclge_ncl_config_data_print(desc, &index, buf, &len, &pos); + hclge_ncl_config_data_print(desc, &index, buf, len, &pos); } return 0; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h index c526591a7240..724052928b88 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_debugfs.h @@ -94,6 +94,11 @@ struct hclge_dbg_func { char *buf, int len); }; +struct hclge_dbg_status_dfx_info { + u32 offset; + char message[HCLGE_DBG_MAX_DFX_MSG_LEN]; +}; + static const struct hclge_dbg_dfx_message hclge_dbg_bios_common_reg[] = { {false, "Reserved"}, {true, "BP_CPU_STATE"}, @@ -321,10 +326,10 @@ static const struct hclge_dbg_dfx_message hclge_dbg_igu_egu_reg[] = { {true, "IGU_RX_OUT_UDP0_PKT"}, {true, "IGU_RX_IN_UDP0_PKT"}, - {false, "Reserved"}, - {false, "Reserved"}, - {false, "Reserved"}, - {false, "Reserved"}, + {true, "IGU_MC_CAR_DROP_PKT_L"}, + {true, "IGU_MC_CAR_DROP_PKT_H"}, + {true, "IGU_BC_CAR_DROP_PKT_L"}, + {true, "IGU_BC_CAR_DROP_PKT_H"}, {false, "Reserved"}, {true, "IGU_RX_OVERSIZE_PKT_L"}, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c index c2a58101144e..1d1c4514aac2 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.c @@ -1613,12 +1613,39 @@ static void hclge_init_kdump_kernel_config(struct hclge_dev *hdev) hdev->num_rx_desc = HCLGE_MIN_RX_DESC; } +static void hclge_init_tc_config(struct hclge_dev *hdev) +{ + unsigned int i; + + if (hdev->tc_max > HNAE3_MAX_TC || + hdev->tc_max < 1) { + dev_warn(&hdev->pdev->dev, "TC num = %u.\n", + hdev->tc_max); + hdev->tc_max = 1; + } + + /* Dev does not support DCB */ + if (!hnae3_dev_dcb_supported(hdev)) { + hdev->tc_max = 1; + hdev->pfc_max = 0; + } else { + hdev->pfc_max = hdev->tc_max; + } + + hdev->tm_info.num_tc = 1; + + /* Currently not support uncontiuous tc */ + for (i = 0; i < hdev->tm_info.num_tc; i++) + hnae3_set_bit(hdev->hw_tc_map, i, 1); + + hdev->tx_sch_mode = HCLGE_FLAG_TC_BASE_SCH_MODE; +} + static int hclge_configure(struct hclge_dev *hdev) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(hdev->pdev); const struct cpumask *cpumask = cpu_online_mask; struct hclge_cfg cfg; - unsigned int i; int node, ret; ret = hclge_get_cfg(hdev, &cfg); @@ -1662,29 +1689,7 @@ static int hclge_configure(struct hclge_dev *hdev) hdev->hw.mac.max_speed = hclge_get_max_speed(cfg.speed_ability); - if ((hdev->tc_max > HNAE3_MAX_TC) || - (hdev->tc_max < 1)) { - dev_warn(&hdev->pdev->dev, "TC num = %u.\n", - hdev->tc_max); - hdev->tc_max = 1; - } - - /* Dev does not support DCB */ - if (!hnae3_dev_dcb_supported(hdev)) { - hdev->tc_max = 1; - hdev->pfc_max = 0; - } else { - hdev->pfc_max = hdev->tc_max; - } - - hdev->tm_info.num_tc = 1; - - /* Currently not support uncontiuous tc */ - for (i = 0; i < hdev->tm_info.num_tc; i++) - hnae3_set_bit(hdev->hw_tc_map, i, 1); - - hdev->tx_sch_mode = HCLGE_FLAG_TC_BASE_SCH_MODE; - + hclge_init_tc_config(hdev); hclge_init_kdump_kernel_config(hdev); /* Set the affinity based on numa node */ @@ -1885,7 +1890,7 @@ static int hclge_map_tqp(struct hclge_dev *hdev) u16 i, num_vport; num_vport = hdev->num_req_vfs + 1; - for (i = 0; i < num_vport; i++) { + for (i = 0; i < num_vport; i++) { int ret; ret = hclge_map_tqp_to_vport(hdev, vport); @@ -2653,11 +2658,38 @@ static u8 hclge_check_speed_dup(u8 duplex, int speed) return duplex; } +static struct hclge_mac_speed_map hclge_mac_speed_map_to_fw[] = { + {HCLGE_MAC_SPEED_10M, HCLGE_FW_MAC_SPEED_10M}, + {HCLGE_MAC_SPEED_100M, HCLGE_FW_MAC_SPEED_100M}, + {HCLGE_MAC_SPEED_1G, HCLGE_FW_MAC_SPEED_1G}, + {HCLGE_MAC_SPEED_10G, HCLGE_FW_MAC_SPEED_10G}, + {HCLGE_MAC_SPEED_25G, HCLGE_FW_MAC_SPEED_25G}, + {HCLGE_MAC_SPEED_40G, HCLGE_FW_MAC_SPEED_40G}, + {HCLGE_MAC_SPEED_50G, HCLGE_FW_MAC_SPEED_50G}, + {HCLGE_MAC_SPEED_100G, HCLGE_FW_MAC_SPEED_100G}, + {HCLGE_MAC_SPEED_200G, HCLGE_FW_MAC_SPEED_200G}, +}; + +static int hclge_convert_to_fw_speed(u32 speed_drv, u32 *speed_fw) +{ + u16 i; + + for (i = 0; i < ARRAY_SIZE(hclge_mac_speed_map_to_fw); i++) { + if (hclge_mac_speed_map_to_fw[i].speed_drv == speed_drv) { + *speed_fw = hclge_mac_speed_map_to_fw[i].speed_fw; + return 0; + } + } + + return -EINVAL; +} + static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, u8 duplex) { struct hclge_config_mac_speed_dup_cmd *req; struct hclge_desc desc; + u32 speed_fw; int ret; req = (struct hclge_config_mac_speed_dup_cmd *)desc.data; @@ -2667,48 +2699,14 @@ static int hclge_cfg_mac_speed_dup_hw(struct hclge_dev *hdev, int speed, if (duplex) hnae3_set_bit(req->speed_dup, HCLGE_CFG_DUPLEX_B, 1); - switch (speed) { - case HCLGE_MAC_SPEED_10M: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_10M); - break; - case HCLGE_MAC_SPEED_100M: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_100M); - break; - case HCLGE_MAC_SPEED_1G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_1G); - break; - case HCLGE_MAC_SPEED_10G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_10G); - break; - case HCLGE_MAC_SPEED_25G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_25G); - break; - case HCLGE_MAC_SPEED_40G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_40G); - break; - case HCLGE_MAC_SPEED_50G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_50G); - break; - case HCLGE_MAC_SPEED_100G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_100G); - break; - case HCLGE_MAC_SPEED_200G: - hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, - HCLGE_CFG_SPEED_S, HCLGE_FW_MAC_SPEED_200G); - break; - default: + ret = hclge_convert_to_fw_speed(speed, &speed_fw); + if (ret) { dev_err(&hdev->pdev->dev, "invalid speed (%d)\n", speed); - return -EINVAL; + return ret; } + hnae3_set_field(req->speed_dup, HCLGE_CFG_SPEED_M, HCLGE_CFG_SPEED_S, + speed_fw); hnae3_set_bit(req->mac_change_fec_en, HCLGE_CFG_MAC_SPEED_CHANGE_EN_B, 1); @@ -2933,16 +2931,20 @@ static int hclge_mac_init(struct hclge_dev *hdev) static void hclge_mbx_task_schedule(struct hclge_dev *hdev) { if (!test_bit(HCLGE_STATE_REMOVING, &hdev->state) && - !test_and_set_bit(HCLGE_STATE_MBX_SERVICE_SCHED, &hdev->state)) + !test_and_set_bit(HCLGE_STATE_MBX_SERVICE_SCHED, &hdev->state)) { + hdev->last_mbx_scheduled = jiffies; mod_delayed_work(hclge_wq, &hdev->service_task, 0); + } } static void hclge_reset_task_schedule(struct hclge_dev *hdev) { if (!test_bit(HCLGE_STATE_REMOVING, &hdev->state) && test_bit(HCLGE_STATE_SERVICE_INITED, &hdev->state) && - !test_and_set_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state)) + !test_and_set_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state)) { + hdev->last_rst_scheduled = jiffies; mod_delayed_work(hclge_wq, &hdev->service_task, 0); + } } static void hclge_errhand_task_schedule(struct hclge_dev *hdev) @@ -3831,6 +3833,13 @@ static void hclge_mailbox_service_task(struct hclge_dev *hdev) test_and_set_bit(HCLGE_STATE_MBX_HANDLING, &hdev->state)) return; + if (time_is_before_jiffies(hdev->last_mbx_scheduled + + HCLGE_MBX_SCHED_TIMEOUT)) + dev_warn(&hdev->pdev->dev, + "mbx service task is scheduled after %ums on cpu%u!\n", + jiffies_to_msecs(jiffies - hdev->last_mbx_scheduled), + smp_processor_id()); + hclge_mbx_handler(hdev); clear_bit(HCLGE_STATE_MBX_HANDLING, &hdev->state); @@ -4480,6 +4489,13 @@ static void hclge_reset_service_task(struct hclge_dev *hdev) if (!test_and_clear_bit(HCLGE_STATE_RST_SERVICE_SCHED, &hdev->state)) return; + if (time_is_before_jiffies(hdev->last_rst_scheduled + + HCLGE_RESET_SCHED_TIMEOUT)) + dev_warn(&hdev->pdev->dev, + "reset service task is scheduled after %ums on cpu%u!\n", + jiffies_to_msecs(jiffies - hdev->last_rst_scheduled), + smp_processor_id()); + down(&hdev->reset_sem); set_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); @@ -6790,7 +6806,7 @@ static int hclge_fd_parse_ring_cookie(struct hclge_dev *hdev, u64 ring_cookie, if (vf > hdev->num_req_vfs) { dev_err(&hdev->pdev->dev, "Error: vf id (%u) should be less than %u\n", - vf - 1, hdev->num_req_vfs); + vf - 1U, hdev->num_req_vfs); return -EINVAL; } @@ -6800,7 +6816,7 @@ static int hclge_fd_parse_ring_cookie(struct hclge_dev *hdev, u64 ring_cookie, if (ring >= tqps) { dev_err(&hdev->pdev->dev, "Error: queue id (%u) > max tqp num (%u)\n", - ring, tqps - 1); + ring, tqps - 1U); return -EINVAL; } @@ -7161,6 +7177,37 @@ static void hclge_fd_get_ext_info(struct ethtool_rx_flow_spec *fs, } } +static struct hclge_fd_rule *hclge_get_fd_rule(struct hclge_dev *hdev, + u16 location) +{ + struct hclge_fd_rule *rule = NULL; + struct hlist_node *node2; + + hlist_for_each_entry_safe(rule, node2, &hdev->fd_rule_list, rule_node) { + if (rule->location == location) + return rule; + else if (rule->location > location) + return NULL; + } + + return NULL; +} + +static void hclge_fd_get_ring_cookie(struct ethtool_rx_flow_spec *fs, + struct hclge_fd_rule *rule) +{ + if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { + fs->ring_cookie = RX_CLS_FLOW_DISC; + } else { + u64 vf_id; + + fs->ring_cookie = rule->queue_id; + vf_id = rule->vf_id; + vf_id <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF; + fs->ring_cookie |= vf_id; + } +} + static int hclge_get_fd_rule_info(struct hnae3_handle *handle, struct ethtool_rxnfc *cmd) { @@ -7168,7 +7215,6 @@ static int hclge_get_fd_rule_info(struct hnae3_handle *handle, struct hclge_fd_rule *rule = NULL; struct hclge_dev *hdev = vport->back; struct ethtool_rx_flow_spec *fs; - struct hlist_node *node2; if (!hnae3_dev_fd_supported(hdev)) return -EOPNOTSUPP; @@ -7177,14 +7223,9 @@ static int hclge_get_fd_rule_info(struct hnae3_handle *handle, spin_lock_bh(&hdev->fd_rule_lock); - hlist_for_each_entry_safe(rule, node2, &hdev->fd_rule_list, rule_node) { - if (rule->location >= fs->location) - break; - } - - if (!rule || fs->location != rule->location) { + rule = hclge_get_fd_rule(hdev, fs->location); + if (!rule) { spin_unlock_bh(&hdev->fd_rule_lock); - return -ENOENT; } @@ -7222,16 +7263,7 @@ static int hclge_get_fd_rule_info(struct hnae3_handle *handle, hclge_fd_get_ext_info(fs, rule); - if (rule->action == HCLGE_FD_ACTION_DROP_PACKET) { - fs->ring_cookie = RX_CLS_FLOW_DISC; - } else { - u64 vf_id; - - fs->ring_cookie = rule->queue_id; - vf_id = rule->vf_id; - vf_id <<= ETHTOOL_RX_FLOW_SPEC_RING_VF_OFF; - fs->ring_cookie |= vf_id; - } + hclge_fd_get_ring_cookie(fs, rule); spin_unlock_bh(&hdev->fd_rule_lock); @@ -7968,16 +8000,13 @@ static int hclge_set_app_loopback(struct hclge_dev *hdev, bool en) return ret; } -static int hclge_cfg_common_loopback(struct hclge_dev *hdev, bool en, - enum hnae3_loop loop_mode) +static int hclge_cfg_common_loopback_cmd_send(struct hclge_dev *hdev, bool en, + enum hnae3_loop loop_mode) { -#define HCLGE_COMMON_LB_RETRY_MS 10 -#define HCLGE_COMMON_LB_RETRY_NUM 100 - struct hclge_common_lb_cmd *req; struct hclge_desc desc; - int ret, i = 0; u8 loop_mode_b; + int ret; req = (struct hclge_common_lb_cmd *)desc.data; hclge_cmd_setup_basic_desc(&desc, HCLGE_OPC_COMMON_LOOPBACK, false); @@ -7994,23 +8023,34 @@ static int hclge_cfg_common_loopback(struct hclge_dev *hdev, bool en, break; default: dev_err(&hdev->pdev->dev, - "unsupported common loopback mode %d\n", loop_mode); + "unsupported loopback mode %d\n", loop_mode); return -ENOTSUPP; } - if (en) { + req->mask = loop_mode_b; + if (en) req->enable = loop_mode_b; - req->mask = loop_mode_b; - } else { - req->mask = loop_mode_b; - } ret = hclge_cmd_send(&hdev->hw, &desc, 1); - if (ret) { + if (ret) dev_err(&hdev->pdev->dev, - "common loopback set fail, ret = %d\n", ret); - return ret; - } + "failed to send loopback cmd, loop_mode = %d, ret = %d\n", + loop_mode, ret); + + return ret; +} + +static int hclge_cfg_common_loopback_wait(struct hclge_dev *hdev) +{ +#define HCLGE_COMMON_LB_RETRY_MS 10 +#define HCLGE_COMMON_LB_RETRY_NUM 100 + + struct hclge_common_lb_cmd *req; + struct hclge_desc desc; + u32 i = 0; + int ret; + + req = (struct hclge_common_lb_cmd *)desc.data; do { msleep(HCLGE_COMMON_LB_RETRY_MS); @@ -8019,20 +8059,34 @@ static int hclge_cfg_common_loopback(struct hclge_dev *hdev, bool en, ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { dev_err(&hdev->pdev->dev, - "common loopback get, ret = %d\n", ret); + "failed to get loopback done status, ret = %d\n", + ret); return ret; } } while (++i < HCLGE_COMMON_LB_RETRY_NUM && !(req->result & HCLGE_CMD_COMMON_LB_DONE_B)); if (!(req->result & HCLGE_CMD_COMMON_LB_DONE_B)) { - dev_err(&hdev->pdev->dev, "common loopback set timeout\n"); + dev_err(&hdev->pdev->dev, "wait loopback timeout\n"); return -EBUSY; } else if (!(req->result & HCLGE_CMD_COMMON_LB_SUCCESS_B)) { - dev_err(&hdev->pdev->dev, "common loopback set failed in fw\n"); + dev_err(&hdev->pdev->dev, "failed to do loopback test\n"); return -EIO; } - return ret; + + return 0; +} + +static int hclge_cfg_common_loopback(struct hclge_dev *hdev, bool en, + enum hnae3_loop loop_mode) +{ + int ret; + + ret = hclge_cfg_common_loopback_cmd_send(hdev, en, loop_mode); + if (ret) + return ret; + + return hclge_cfg_common_loopback_wait(hdev); } static int hclge_set_common_loopback(struct hclge_dev *hdev, bool en, @@ -8743,6 +8797,7 @@ int hclge_update_mac_list(struct hclge_vport *vport, enum HCLGE_MAC_ADDR_TYPE mac_type, const unsigned char *addr) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; struct hclge_mac_node *mac_node; struct list_head *list; @@ -8767,9 +8822,10 @@ int hclge_update_mac_list(struct hclge_vport *vport, /* if this address is never added, unnecessary to delete */ if (state == HCLGE_MAC_TO_DEL) { spin_unlock_bh(&vport->mac_list_lock); + hnae3_format_mac_addr(format_mac_addr, addr); dev_err(&hdev->pdev->dev, - "failed to delete address %pM from mac list\n", - addr); + "failed to delete address %s from mac list\n", + format_mac_addr); return -ENOENT; } @@ -8802,6 +8858,7 @@ static int hclge_add_uc_addr(struct hnae3_handle *handle, int hclge_add_uc_addr_common(struct hclge_vport *vport, const unsigned char *addr) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; struct hclge_mac_vlan_tbl_entry_cmd req; struct hclge_desc desc; @@ -8812,9 +8869,10 @@ int hclge_add_uc_addr_common(struct hclge_vport *vport, if (is_zero_ether_addr(addr) || is_broadcast_ether_addr(addr) || is_multicast_ether_addr(addr)) { + hnae3_format_mac_addr(format_mac_addr, addr); dev_err(&hdev->pdev->dev, - "Set_uc mac err! invalid mac:%pM. is_zero:%d,is_br=%d,is_mul=%d\n", - addr, is_zero_ether_addr(addr), + "Set_uc mac err! invalid mac:%s. is_zero:%d,is_br=%d,is_mul=%d\n", + format_mac_addr, is_zero_ether_addr(addr), is_broadcast_ether_addr(addr), is_multicast_ether_addr(addr)); return -EINVAL; @@ -8871,6 +8929,7 @@ static int hclge_rm_uc_addr(struct hnae3_handle *handle, int hclge_rm_uc_addr_common(struct hclge_vport *vport, const unsigned char *addr) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; struct hclge_mac_vlan_tbl_entry_cmd req; int ret; @@ -8879,8 +8938,9 @@ int hclge_rm_uc_addr_common(struct hclge_vport *vport, if (is_zero_ether_addr(addr) || is_broadcast_ether_addr(addr) || is_multicast_ether_addr(addr)) { - dev_dbg(&hdev->pdev->dev, "Remove mac err! invalid mac:%pM.\n", - addr); + hnae3_format_mac_addr(format_mac_addr, addr); + dev_dbg(&hdev->pdev->dev, "Remove mac err! invalid mac:%s.\n", + format_mac_addr); return -EINVAL; } @@ -8911,6 +8971,7 @@ static int hclge_add_mc_addr(struct hnae3_handle *handle, int hclge_add_mc_addr_common(struct hclge_vport *vport, const unsigned char *addr) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; struct hclge_mac_vlan_tbl_entry_cmd req; struct hclge_desc desc[3]; @@ -8919,9 +8980,10 @@ int hclge_add_mc_addr_common(struct hclge_vport *vport, /* mac addr check */ if (!is_multicast_ether_addr(addr)) { + hnae3_format_mac_addr(format_mac_addr, addr); dev_err(&hdev->pdev->dev, - "Add mc mac err! invalid mac:%pM.\n", - addr); + "Add mc mac err! invalid mac:%s.\n", + format_mac_addr); return -EINVAL; } memset(&req, 0, sizeof(req)); @@ -8973,6 +9035,7 @@ static int hclge_rm_mc_addr(struct hnae3_handle *handle, int hclge_rm_mc_addr_common(struct hclge_vport *vport, const unsigned char *addr) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; struct hclge_mac_vlan_tbl_entry_cmd req; enum hclge_cmd_status status; @@ -8980,9 +9043,10 @@ int hclge_rm_mc_addr_common(struct hclge_vport *vport, /* mac addr check */ if (!is_multicast_ether_addr(addr)) { + hnae3_format_mac_addr(format_mac_addr, addr); dev_dbg(&hdev->pdev->dev, - "Remove mc mac err! invalid mac:%pM.\n", - addr); + "Remove mc mac err! invalid mac:%s.\n", + format_mac_addr); return -EINVAL; } @@ -9422,16 +9486,18 @@ static int hclge_set_vf_mac(struct hnae3_handle *handle, int vf, u8 *mac_addr) { struct hclge_vport *vport = hclge_get_vport(handle); + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; vport = hclge_get_vf_vport(hdev, vf); if (!vport) return -EINVAL; + hnae3_format_mac_addr(format_mac_addr, mac_addr); if (ether_addr_equal(mac_addr, vport->vf_info.mac)) { dev_info(&hdev->pdev->dev, - "Specified MAC(=%pM) is same as before, no change committed!\n", - mac_addr); + "Specified MAC(=%s) is same as before, no change committed!\n", + format_mac_addr); return 0; } @@ -9439,13 +9505,13 @@ static int hclge_set_vf_mac(struct hnae3_handle *handle, int vf, if (test_bit(HCLGE_VPORT_STATE_ALIVE, &vport->state)) { dev_info(&hdev->pdev->dev, - "MAC of VF %d has been set to %pM, and it will be reinitialized!\n", - vf, mac_addr); + "MAC of VF %d has been set to %s, and it will be reinitialized!\n", + vf, format_mac_addr); return hclge_inform_reset_assert_to_vf(vport); } - dev_info(&hdev->pdev->dev, "MAC of VF %d has been set to %pM\n", - vf, mac_addr); + dev_info(&hdev->pdev->dev, "MAC of VF %d has been set to %s\n", + vf, format_mac_addr); return 0; } @@ -9549,6 +9615,7 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, const void *p, { const unsigned char *new_addr = (const unsigned char *)p; struct hclge_vport *vport = hclge_get_vport(handle); + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclge_dev *hdev = vport->back; unsigned char *old_addr = NULL; int ret; @@ -9557,9 +9624,10 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, const void *p, if (is_zero_ether_addr(new_addr) || is_broadcast_ether_addr(new_addr) || is_multicast_ether_addr(new_addr)) { + hnae3_format_mac_addr(format_mac_addr, new_addr); dev_err(&hdev->pdev->dev, - "change uc mac err! invalid mac: %pM.\n", - new_addr); + "change uc mac err! invalid mac: %s.\n", + format_mac_addr); return -EINVAL; } @@ -9577,9 +9645,10 @@ static int hclge_set_mac_addr(struct hnae3_handle *handle, const void *p, spin_lock_bh(&vport->mac_list_lock); ret = hclge_update_mac_node_for_dev_addr(vport, old_addr, new_addr); if (ret) { + hnae3_format_mac_addr(format_mac_addr, new_addr); dev_err(&hdev->pdev->dev, - "failed to change the mac addr:%pM, ret = %d\n", - new_addr, ret); + "failed to change the mac addr:%s, ret = %d\n", + format_mac_addr, ret); spin_unlock_bh(&vport->mac_list_lock); if (!is_first) @@ -9677,8 +9746,8 @@ static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type, ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) { - dev_err(&hdev->pdev->dev, - "failed to get vlan filter config, ret = %d.\n", ret); + dev_err(&hdev->pdev->dev, "failed to get vport%u vlan filter config, ret = %d.\n", + vf_id, ret); return ret; } @@ -9689,8 +9758,8 @@ static int hclge_set_vlan_filter_ctrl(struct hclge_dev *hdev, u8 vlan_type, ret = hclge_cmd_send(&hdev->hw, &desc, 1); if (ret) - dev_err(&hdev->pdev->dev, "failed to set vlan filter, ret = %d.\n", - ret); + dev_err(&hdev->pdev->dev, "failed to set vport%u vlan filter, ret = %d.\n", + vf_id, ret); return ret; } @@ -9936,6 +10005,32 @@ static int hclge_set_port_vlan_filter(struct hclge_dev *hdev, __be16 proto, return ret; } +static bool hclge_need_update_port_vlan(struct hclge_dev *hdev, u16 vport_id, + u16 vlan_id, bool is_kill) +{ + /* vlan 0 may be added twice when 8021q module is enabled */ + if (!is_kill && !vlan_id && + test_bit(vport_id, hdev->vlan_table[vlan_id])) + return false; + + if (!is_kill && test_and_set_bit(vport_id, hdev->vlan_table[vlan_id])) { + dev_warn(&hdev->pdev->dev, + "Add port vlan failed, vport %u is already in vlan %u\n", + vport_id, vlan_id); + return false; + } + + if (is_kill && + !test_and_clear_bit(vport_id, hdev->vlan_table[vlan_id])) { + dev_warn(&hdev->pdev->dev, + "Delete port vlan failed, vport %u is not in vlan %u\n", + vport_id, vlan_id); + return false; + } + + return true; +} + static int hclge_set_vlan_filter_hw(struct hclge_dev *hdev, __be16 proto, u16 vport_id, u16 vlan_id, bool is_kill) @@ -9957,26 +10052,9 @@ static int hclge_set_vlan_filter_hw(struct hclge_dev *hdev, __be16 proto, return ret; } - /* vlan 0 may be added twice when 8021q module is enabled */ - if (!is_kill && !vlan_id && - test_bit(vport_id, hdev->vlan_table[vlan_id])) + if (!hclge_need_update_port_vlan(hdev, vport_id, vlan_id, is_kill)) return 0; - if (!is_kill && test_and_set_bit(vport_id, hdev->vlan_table[vlan_id])) { - dev_err(&hdev->pdev->dev, - "Add port vlan failed, vport %u is already in vlan %u\n", - vport_id, vlan_id); - return -EINVAL; - } - - if (is_kill && - !test_and_clear_bit(vport_id, hdev->vlan_table[vlan_id])) { - dev_err(&hdev->pdev->dev, - "Delete port vlan failed, vport %u is not in vlan %u\n", - vport_id, vlan_id); - return -EINVAL; - } - for_each_set_bit(vport_idx, hdev->vlan_table[vlan_id], HCLGE_VPORT_NUM) vport_num++; @@ -10168,67 +10246,80 @@ static int hclge_set_vlan_protocol_type(struct hclge_dev *hdev) return status; } -static int hclge_init_vlan_config(struct hclge_dev *hdev) +static int hclge_init_vlan_filter(struct hclge_dev *hdev) { -#define HCLGE_DEF_VLAN_TYPE 0x8100 - - struct hnae3_handle *handle = &hdev->vport[0].nic; struct hclge_vport *vport; int ret; int i; - if (hdev->ae_dev->dev_version >= HNAE3_DEVICE_VERSION_V2) { - /* for revision 0x21, vf vlan filter is per function */ - for (i = 0; i < hdev->num_alloc_vport; i++) { - vport = &hdev->vport[i]; - ret = hclge_set_vlan_filter_ctrl(hdev, - HCLGE_FILTER_TYPE_VF, - HCLGE_FILTER_FE_EGRESS, - true, - vport->vport_id); - if (ret) - return ret; - vport->cur_vlan_fltr_en = true; - } + if (hdev->ae_dev->dev_version < HNAE3_DEVICE_VERSION_V2) + return hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, + HCLGE_FILTER_FE_EGRESS_V1_B, + true, 0); - ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, - HCLGE_FILTER_FE_INGRESS, true, - 0); - if (ret) - return ret; - } else { + /* for revision 0x21, vf vlan filter is per function */ + for (i = 0; i < hdev->num_alloc_vport; i++) { + vport = &hdev->vport[i]; ret = hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_VF, - HCLGE_FILTER_FE_EGRESS_V1_B, - true, 0); + HCLGE_FILTER_FE_EGRESS, true, + vport->vport_id); if (ret) return ret; + vport->cur_vlan_fltr_en = true; } - hdev->vlan_type_cfg.rx_in_fst_vlan_type = HCLGE_DEF_VLAN_TYPE; - hdev->vlan_type_cfg.rx_in_sec_vlan_type = HCLGE_DEF_VLAN_TYPE; - hdev->vlan_type_cfg.rx_ot_fst_vlan_type = HCLGE_DEF_VLAN_TYPE; - hdev->vlan_type_cfg.rx_ot_sec_vlan_type = HCLGE_DEF_VLAN_TYPE; - hdev->vlan_type_cfg.tx_ot_vlan_type = HCLGE_DEF_VLAN_TYPE; - hdev->vlan_type_cfg.tx_in_vlan_type = HCLGE_DEF_VLAN_TYPE; + return hclge_set_vlan_filter_ctrl(hdev, HCLGE_FILTER_TYPE_PORT, + HCLGE_FILTER_FE_INGRESS, true, 0); +} - ret = hclge_set_vlan_protocol_type(hdev); - if (ret) - return ret; +static int hclge_init_vlan_type(struct hclge_dev *hdev) +{ + hdev->vlan_type_cfg.rx_in_fst_vlan_type = ETH_P_8021Q; + hdev->vlan_type_cfg.rx_in_sec_vlan_type = ETH_P_8021Q; + hdev->vlan_type_cfg.rx_ot_fst_vlan_type = ETH_P_8021Q; + hdev->vlan_type_cfg.rx_ot_sec_vlan_type = ETH_P_8021Q; + hdev->vlan_type_cfg.tx_ot_vlan_type = ETH_P_8021Q; + hdev->vlan_type_cfg.tx_in_vlan_type = ETH_P_8021Q; - for (i = 0; i < hdev->num_alloc_vport; i++) { - u16 vlan_tag; - u8 qos; + return hclge_set_vlan_protocol_type(hdev); +} + +static int hclge_init_vport_vlan_offload(struct hclge_dev *hdev) +{ + struct hclge_port_base_vlan_config *cfg; + struct hclge_vport *vport; + int ret; + int i; + for (i = 0; i < hdev->num_alloc_vport; i++) { vport = &hdev->vport[i]; - vlan_tag = vport->port_base_vlan_cfg.vlan_info.vlan_tag; - qos = vport->port_base_vlan_cfg.vlan_info.qos; + cfg = &vport->port_base_vlan_cfg; - ret = hclge_vlan_offload_cfg(vport, - vport->port_base_vlan_cfg.state, - vlan_tag, qos); + ret = hclge_vlan_offload_cfg(vport, cfg->state, + cfg->vlan_info.vlan_tag, + cfg->vlan_info.qos); if (ret) return ret; } + return 0; +} + +static int hclge_init_vlan_config(struct hclge_dev *hdev) +{ + struct hnae3_handle *handle = &hdev->vport[0].nic; + int ret; + + ret = hclge_init_vlan_filter(hdev); + if (ret) + return ret; + + ret = hclge_init_vlan_type(hdev); + if (ret) + return ret; + + ret = hclge_init_vport_vlan_offload(hdev); + if (ret) + return ret; return hclge_set_vlan_filter(handle, htons(ETH_P_8021Q), 0, false); } @@ -10485,12 +10576,41 @@ static bool hclge_need_update_vlan_filter(const struct hclge_vlan_info *new_cfg, return false; } +static int hclge_modify_port_base_vlan_tag(struct hclge_vport *vport, + struct hclge_vlan_info *new_info, + struct hclge_vlan_info *old_info) +{ + struct hclge_dev *hdev = vport->back; + int ret; + + /* add new VLAN tag */ + ret = hclge_set_vlan_filter_hw(hdev, htons(new_info->vlan_proto), + vport->vport_id, new_info->vlan_tag, + false); + if (ret) + return ret; + + /* remove old VLAN tag */ + if (old_info->vlan_tag == 0) + ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, + true, 0); + else + ret = hclge_set_vlan_filter_hw(hdev, htons(ETH_P_8021Q), + vport->vport_id, + old_info->vlan_tag, true); + if (ret) + dev_err(&hdev->pdev->dev, + "failed to clear vport%u port base vlan %u, ret = %d.\n", + vport->vport_id, old_info->vlan_tag, ret); + + return ret; +} + int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, struct hclge_vlan_info *vlan_info) { struct hnae3_handle *nic = &vport->nic; struct hclge_vlan_info *old_vlan_info; - struct hclge_dev *hdev = vport->back; int ret; old_vlan_info = &vport->port_base_vlan_cfg.vlan_info; @@ -10503,38 +10623,12 @@ int hclge_update_port_base_vlan_cfg(struct hclge_vport *vport, u16 state, if (!hclge_need_update_vlan_filter(vlan_info, old_vlan_info)) goto out; - if (state == HNAE3_PORT_BASE_VLAN_MODIFY) { - /* add new VLAN tag */ - ret = hclge_set_vlan_filter_hw(hdev, - htons(vlan_info->vlan_proto), - vport->vport_id, - vlan_info->vlan_tag, - false); - if (ret) - return ret; - - /* remove old VLAN tag */ - if (old_vlan_info->vlan_tag == 0) - ret = hclge_set_vf_vlan_common(hdev, vport->vport_id, - true, 0); - else - ret = hclge_set_vlan_filter_hw(hdev, - htons(ETH_P_8021Q), - vport->vport_id, - old_vlan_info->vlan_tag, - true); - if (ret) { - dev_err(&hdev->pdev->dev, - "failed to clear vport%u port base vlan %u, ret = %d.\n", - vport->vport_id, old_vlan_info->vlan_tag, ret); - return ret; - } - - goto out; - } - - ret = hclge_update_vlan_filter_entries(vport, state, vlan_info, - old_vlan_info); + if (state == HNAE3_PORT_BASE_VLAN_MODIFY) + ret = hclge_modify_port_base_vlan_tag(vport, vlan_info, + old_vlan_info); + else + ret = hclge_update_vlan_filter_entries(vport, state, vlan_info, + old_vlan_info); if (ret) return ret; @@ -11556,24 +11650,20 @@ static void hclge_reset_prepare_general(struct hnae3_ae_dev *ae_dev, int retry_cnt = 0; int ret; -retry: - down(&hdev->reset_sem); - set_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); - hdev->reset_type = rst_type; - ret = hclge_reset_prepare(hdev); - if (ret || hdev->reset_pending) { - dev_err(&hdev->pdev->dev, "fail to prepare to reset, ret=%d\n", - ret); - if (hdev->reset_pending || - retry_cnt++ < HCLGE_RESET_RETRY_CNT) { - dev_err(&hdev->pdev->dev, - "reset_pending:0x%lx, retry_cnt:%d\n", - hdev->reset_pending, retry_cnt); - clear_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); - up(&hdev->reset_sem); - msleep(HCLGE_RESET_RETRY_WAIT_MS); - goto retry; - } + while (retry_cnt++ < HCLGE_RESET_RETRY_CNT) { + down(&hdev->reset_sem); + set_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); + hdev->reset_type = rst_type; + ret = hclge_reset_prepare(hdev); + if (!ret && !hdev->reset_pending) + break; + + dev_err(&hdev->pdev->dev, + "failed to prepare to reset, ret=%d, reset_pending:0x%lx, retry_cnt:%d\n", + ret, hdev->reset_pending, retry_cnt); + clear_bit(HCLGE_STATE_RST_HANDLING, &hdev->state); + up(&hdev->reset_sem); + msleep(HCLGE_RESET_RETRY_WAIT_MS); } /* disable misc vector before reset done */ @@ -12288,19 +12378,42 @@ static void hclge_get_tqps_and_rss_info(struct hnae3_handle *handle, *max_rss_size = hdev->pf_rss_size_max; } +static int hclge_set_rss_tc_mode_cfg(struct hnae3_handle *handle) +{ + struct hclge_vport *vport = hclge_get_vport(handle); + u16 tc_offset[HCLGE_MAX_TC_NUM] = {0}; + struct hclge_dev *hdev = vport->back; + u16 tc_size[HCLGE_MAX_TC_NUM] = {0}; + u16 tc_valid[HCLGE_MAX_TC_NUM]; + u16 roundup_size; + unsigned int i; + + roundup_size = roundup_pow_of_two(vport->nic.kinfo.rss_size); + roundup_size = ilog2(roundup_size); + /* Set the RSS TC mode according to the new RSS size */ + for (i = 0; i < HCLGE_MAX_TC_NUM; i++) { + tc_valid[i] = 0; + + if (!(hdev->hw_tc_map & BIT(i))) + continue; + + tc_valid[i] = 1; + tc_size[i] = roundup_size; + tc_offset[i] = vport->nic.kinfo.rss_size * i; + } + + return hclge_set_rss_tc_mode(hdev, tc_valid, tc_size, tc_offset); +} + static int hclge_set_channels(struct hnae3_handle *handle, u32 new_tqps_num, bool rxfh_configured) { struct hnae3_ae_dev *ae_dev = pci_get_drvdata(handle->pdev); struct hclge_vport *vport = hclge_get_vport(handle); struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo; - u16 tc_offset[HCLGE_MAX_TC_NUM] = {0}; struct hclge_dev *hdev = vport->back; - u16 tc_size[HCLGE_MAX_TC_NUM] = {0}; u16 cur_rss_size = kinfo->rss_size; u16 cur_tqps = kinfo->num_tqps; - u16 tc_valid[HCLGE_MAX_TC_NUM]; - u16 roundup_size; u32 *rss_indir; unsigned int i; int ret; @@ -12313,20 +12426,7 @@ static int hclge_set_channels(struct hnae3_handle *handle, u32 new_tqps_num, return ret; } - roundup_size = roundup_pow_of_two(kinfo->rss_size); - roundup_size = ilog2(roundup_size); - /* Set the RSS TC mode according to the new RSS size */ - for (i = 0; i < HCLGE_MAX_TC_NUM; i++) { - tc_valid[i] = 0; - - if (!(hdev->hw_tc_map & BIT(i))) - continue; - - tc_valid[i] = 1; - tc_size[i] = roundup_size; - tc_offset[i] = kinfo->rss_size * i; - } - ret = hclge_set_rss_tc_mode(hdev, tc_valid, tc_size, tc_offset); + ret = hclge_set_rss_tc_mode_cfg(handle); if (ret) return ret; diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h index ebba603483a0..a51418fdbb24 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_main.h @@ -955,6 +955,8 @@ struct hclge_dev { u16 hclge_fd_rule_num; unsigned long serv_processed_cnt; unsigned long last_serv_processed; + unsigned long last_rst_scheduled; + unsigned long last_mbx_scheduled; unsigned long fd_bmap[BITS_TO_LONGS(MAX_FD_FILTER_NUM)]; enum HCLGE_FD_ACTIVE_RULE_TYPE fd_active_type; u8 fd_en; @@ -1093,6 +1095,11 @@ struct hclge_speed_bit_map { u32 speed_bit; }; +struct hclge_mac_speed_map { + u32 speed_drv; /* speed defined in driver */ + u32 speed_fw; /* speed defined in firmware */ +}; + int hclge_set_vport_promisc_mode(struct hclge_vport *vport, bool en_uc_pmc, bool en_mc_pmc, bool en_bc_pmc); int hclge_add_uc_addr_common(struct hclge_vport *vport, diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c index 65d78ee4d65a..8b3954b39147 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mbx.c @@ -181,7 +181,7 @@ static int hclge_get_ring_chain_from_mbx( if (req->msg.param[i].tqp_index >= vport->nic.kinfo.rss_size) { dev_err(&hdev->pdev->dev, "tqp index(%u) is out of range(0-%u)\n", req->msg.param[i].tqp_index, - vport->nic.kinfo.rss_size - 1); + vport->nic.kinfo.rss_size - 1U); return -EINVAL; } } @@ -848,6 +848,14 @@ void hclge_mbx_handler(struct hclge_dev *hdev) if (hnae3_get_bit(req->mbx_need_resp, HCLGE_MBX_NEED_RESP_B) && req->msg.code < HCLGE_MBX_GET_VF_FLR_STATUS) { resp_msg.status = ret; + if (time_is_before_jiffies(hdev->last_mbx_scheduled + + HCLGE_MBX_SCHED_TIMEOUT)) + dev_warn(&hdev->pdev->dev, + "resp vport%u mbx(%u,%u) late\n", + req->mbx_src_vfid, + req->msg.code, + req->msg.subcode); + hclge_gen_resp_to_vf(vport, req, &resp_msg); } diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h index fd0e20190b90..4200d0b6d931 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_mdio.h @@ -4,6 +4,10 @@ #ifndef __HCLGE_MDIO_H #define __HCLGE_MDIO_H +#include "hnae3.h" + +struct hclge_dev; + int hclge_mac_mdio_config(struct hclge_dev *hdev); int hclge_mac_connect_phy(struct hnae3_handle *handle); void hclge_mac_disconnect_phy(struct hnae3_handle *handle); diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h index 7a9b77de632a..bbee74cd8404 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_ptp.h @@ -8,6 +8,9 @@ #include <linux/net_tstamp.h> #include <linux/types.h> +struct hclge_dev; +struct ifreq; + #define HCLGE_PTP_REG_OFFSET 0x29000 #define HCLGE_PTP_TX_TS_SEQID_REG 0x0 diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c index 429652a8cde1..3edbfc8d17e8 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.c @@ -916,38 +916,63 @@ static int hclge_vport_q_to_qs_map(struct hclge_dev *hdev, return 0; } -static int hclge_tm_pri_q_qs_cfg(struct hclge_dev *hdev) +static int hclge_tm_pri_q_qs_cfg_tc_base(struct hclge_dev *hdev) { struct hclge_vport *vport = hdev->vport; + u16 i, k; int ret; - u32 i, k; - if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) { - /* Cfg qs -> pri mapping, one by one mapping */ - for (k = 0; k < hdev->num_alloc_vport; k++) { - struct hnae3_knic_private_info *kinfo = - &vport[k].nic.kinfo; - - for (i = 0; i < kinfo->tc_info.num_tc; i++) { - ret = hclge_tm_qs_to_pri_map_cfg( - hdev, vport[k].qs_offset + i, i); - if (ret) - return ret; - } + /* Cfg qs -> pri mapping, one by one mapping */ + for (k = 0; k < hdev->num_alloc_vport; k++) { + struct hnae3_knic_private_info *kinfo = &vport[k].nic.kinfo; + + for (i = 0; i < kinfo->tc_info.num_tc; i++) { + ret = hclge_tm_qs_to_pri_map_cfg(hdev, + vport[k].qs_offset + i, + i); + if (ret) + return ret; } - } else if (hdev->tx_sch_mode == HCLGE_FLAG_VNET_BASE_SCH_MODE) { - /* Cfg qs -> pri mapping, qs = tc, pri = vf, 8 qs -> 1 pri */ - for (k = 0; k < hdev->num_alloc_vport; k++) - for (i = 0; i < HNAE3_MAX_TC; i++) { - ret = hclge_tm_qs_to_pri_map_cfg( - hdev, vport[k].qs_offset + i, k); - if (ret) - return ret; - } - } else { - return -EINVAL; } + return 0; +} + +static int hclge_tm_pri_q_qs_cfg_vnet_base(struct hclge_dev *hdev) +{ + struct hclge_vport *vport = hdev->vport; + u16 i, k; + int ret; + + /* Cfg qs -> pri mapping, qs = tc, pri = vf, 8 qs -> 1 pri */ + for (k = 0; k < hdev->num_alloc_vport; k++) + for (i = 0; i < HNAE3_MAX_TC; i++) { + ret = hclge_tm_qs_to_pri_map_cfg(hdev, + vport[k].qs_offset + i, + k); + if (ret) + return ret; + } + + return 0; +} + +static int hclge_tm_pri_q_qs_cfg(struct hclge_dev *hdev) +{ + struct hclge_vport *vport = hdev->vport; + int ret; + u32 i; + + if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) + ret = hclge_tm_pri_q_qs_cfg_tc_base(hdev); + else if (hdev->tx_sch_mode == HCLGE_FLAG_VNET_BASE_SCH_MODE) + ret = hclge_tm_pri_q_qs_cfg_vnet_base(hdev); + else + return -EINVAL; + + if (ret) + return ret; + /* Cfg q -> qs mapping */ for (i = 0; i < hdev->num_alloc_vport; i++) { ret = hclge_vport_q_to_qs_map(hdev, vport); @@ -1274,6 +1299,27 @@ static int hclge_tm_lvl2_schd_mode_cfg(struct hclge_dev *hdev) return 0; } +static int hclge_tm_schd_mode_tc_base_cfg(struct hclge_dev *hdev, u8 pri_id) +{ + struct hclge_vport *vport = hdev->vport; + int ret; + u16 i; + + ret = hclge_tm_pri_schd_mode_cfg(hdev, pri_id); + if (ret) + return ret; + + for (i = 0; i < hdev->num_alloc_vport; i++) { + ret = hclge_tm_qs_schd_mode_cfg(hdev, + vport[i].qs_offset + pri_id, + HCLGE_SCH_MODE_DWRR); + if (ret) + return ret; + } + + return 0; +} + static int hclge_tm_schd_mode_vnet_base_cfg(struct hclge_vport *vport) { struct hnae3_knic_private_info *kinfo = &vport->nic.kinfo; @@ -1304,21 +1350,13 @@ static int hclge_tm_lvl34_schd_mode_cfg(struct hclge_dev *hdev) { struct hclge_vport *vport = hdev->vport; int ret; - u8 i, k; + u8 i; if (hdev->tx_sch_mode == HCLGE_FLAG_TC_BASE_SCH_MODE) { for (i = 0; i < hdev->tm_info.num_tc; i++) { - ret = hclge_tm_pri_schd_mode_cfg(hdev, i); + ret = hclge_tm_schd_mode_tc_base_cfg(hdev, i); if (ret) return ret; - - for (k = 0; k < hdev->num_alloc_vport; k++) { - ret = hclge_tm_qs_schd_mode_cfg( - hdev, vport[k].qs_offset + i, - HCLGE_SCH_MODE_DWRR); - if (ret) - return ret; - } } } else { for (i = 0; i < hdev->num_alloc_vport; i++) { diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h index 1db7f40b4525..619cc30a2dfc 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h +++ b/drivers/net/ethernet/hisilicon/hns3/hns3pf/hclge_tm.h @@ -6,6 +6,12 @@ #include <linux/types.h> +#include "hnae3.h" + +struct hclge_dev; +struct hclge_vport; +enum hclge_opcode_type; + /* MAC Pause */ #define HCLGE_TX_MAC_PAUSE_EN_MSK BIT(0) #define HCLGE_RX_MAC_PAUSE_EN_MSK BIT(1) diff --git a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c index 41afaeea881b..0568cc31d391 100644 --- a/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c +++ b/drivers/net/ethernet/hisilicon/hns3/hns3vf/hclgevf_main.c @@ -1514,15 +1514,18 @@ static void hclgevf_config_mac_list(struct hclgevf_dev *hdev, struct list_head *list, enum HCLGEVF_MAC_ADDR_TYPE mac_type) { + char format_mac_addr[HNAE3_FORMAT_MAC_ADDR_LEN]; struct hclgevf_mac_addr_node *mac_node, *tmp; int ret; list_for_each_entry_safe(mac_node, tmp, list, node) { ret = hclgevf_add_del_mac_addr(hdev, mac_node, mac_type); if (ret) { + hnae3_format_mac_addr(format_mac_addr, + mac_node->mac_addr); dev_err(&hdev->pdev->dev, - "failed to configure mac %pM, state = %d, ret = %d\n", - mac_node->mac_addr, mac_node->state, ret); + "failed to configure mac %s, state = %d, ret = %d\n", + format_mac_addr, mac_node->state, ret); return; } if (mac_node->state == HCLGEVF_MAC_TO_ADD) { @@ -2163,24 +2166,20 @@ static void hclgevf_reset_prepare_general(struct hnae3_ae_dev *ae_dev, int retry_cnt = 0; int ret; -retry: - down(&hdev->reset_sem); - set_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state); - hdev->reset_type = rst_type; - ret = hclgevf_reset_prepare(hdev); - if (ret) { - dev_err(&hdev->pdev->dev, "fail to prepare to reset, ret=%d\n", - ret); - if (hdev->reset_pending || - retry_cnt++ < HCLGEVF_RESET_RETRY_CNT) { - dev_err(&hdev->pdev->dev, - "reset_pending:0x%lx, retry_cnt:%d\n", - hdev->reset_pending, retry_cnt); - clear_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state); - up(&hdev->reset_sem); - msleep(HCLGEVF_RESET_RETRY_WAIT_MS); - goto retry; - } + while (retry_cnt++ < HCLGEVF_RESET_RETRY_CNT) { + down(&hdev->reset_sem); + set_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state); + hdev->reset_type = rst_type; + ret = hclgevf_reset_prepare(hdev); + if (!ret && !hdev->reset_pending) + break; + + dev_err(&hdev->pdev->dev, + "failed to prepare to reset, ret=%d, reset_pending:0x%lx, retry_cnt:%d\n", + ret, hdev->reset_pending, retry_cnt); + clear_bit(HCLGEVF_STATE_RST_HANDLING, &hdev->state); + up(&hdev->reset_sem); + msleep(HCLGEVF_RESET_RETRY_WAIT_MS); } /* disable misc vector before reset done */ diff --git a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c index a85667078b72..93192f58ac88 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_ethtool.c @@ -547,7 +547,9 @@ static void hinic_get_drvinfo(struct net_device *netdev, } static void hinic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct hinic_dev *nic_dev = netdev_priv(netdev); @@ -580,7 +582,9 @@ static int check_ringparam_valid(struct hinic_dev *nic_dev, } static int hinic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct hinic_dev *nic_dev = netdev_priv(netdev); u16 new_sq_depth, new_rq_depth; @@ -1205,8 +1209,6 @@ static u32 hinic_get_rxfh_indir_size(struct net_device *netdev) return HINIC_RSS_INDIR_SIZE; } -#define ARRAY_LEN(arr) ((int)((int)sizeof(arr) / (int)sizeof(arr[0]))) - #define HINIC_FUNC_STAT(_stat_item) { \ .name = #_stat_item, \ .size = sizeof_field(struct hinic_vport_stats, _stat_item), \ @@ -1374,7 +1376,7 @@ static void get_drv_queue_stats(struct hinic_dev *nic_dev, u64 *data) break; hinic_txq_get_stats(&nic_dev->txqs[qid], &txq_stats); - for (j = 0; j < ARRAY_LEN(hinic_tx_queue_stats); j++, i++) { + for (j = 0; j < ARRAY_SIZE(hinic_tx_queue_stats); j++, i++) { p = (char *)&txq_stats + hinic_tx_queue_stats[j].offset; data[i] = (hinic_tx_queue_stats[j].size == @@ -1387,7 +1389,7 @@ static void get_drv_queue_stats(struct hinic_dev *nic_dev, u64 *data) break; hinic_rxq_get_stats(&nic_dev->rxqs[qid], &rxq_stats); - for (j = 0; j < ARRAY_LEN(hinic_rx_queue_stats); j++, i++) { + for (j = 0; j < ARRAY_SIZE(hinic_rx_queue_stats); j++, i++) { p = (char *)&rxq_stats + hinic_rx_queue_stats[j].offset; data[i] = (hinic_rx_queue_stats[j].size == @@ -1411,7 +1413,7 @@ static void hinic_get_ethtool_stats(struct net_device *netdev, netif_err(nic_dev, drv, netdev, "Failed to get vport stats from firmware\n"); - for (j = 0; j < ARRAY_LEN(hinic_function_stats); j++, i++) { + for (j = 0; j < ARRAY_SIZE(hinic_function_stats); j++, i++) { p = (char *)&vport_stats + hinic_function_stats[j].offset; data[i] = (hinic_function_stats[j].size == sizeof(u64)) ? *(u64 *)p : *(u32 *)p; @@ -1420,8 +1422,8 @@ static void hinic_get_ethtool_stats(struct net_device *netdev, port_stats = kzalloc(sizeof(*port_stats), GFP_KERNEL); if (!port_stats) { memset(&data[i], 0, - ARRAY_LEN(hinic_port_stats) * sizeof(*data)); - i += ARRAY_LEN(hinic_port_stats); + ARRAY_SIZE(hinic_port_stats) * sizeof(*data)); + i += ARRAY_SIZE(hinic_port_stats); goto get_drv_stats; } @@ -1430,7 +1432,7 @@ static void hinic_get_ethtool_stats(struct net_device *netdev, netif_err(nic_dev, drv, netdev, "Failed to get port stats from firmware\n"); - for (j = 0; j < ARRAY_LEN(hinic_port_stats); j++, i++) { + for (j = 0; j < ARRAY_SIZE(hinic_port_stats); j++, i++) { p = (char *)port_stats + hinic_port_stats[j].offset; data[i] = (hinic_port_stats[j].size == sizeof(u64)) ? *(u64 *)p : *(u32 *)p; @@ -1449,14 +1451,14 @@ static int hinic_get_sset_count(struct net_device *netdev, int sset) switch (sset) { case ETH_SS_TEST: - return ARRAY_LEN(hinic_test_strings); + return ARRAY_SIZE(hinic_test_strings); case ETH_SS_STATS: q_num = nic_dev->num_qps; - count = ARRAY_LEN(hinic_function_stats) + - (ARRAY_LEN(hinic_tx_queue_stats) + - ARRAY_LEN(hinic_rx_queue_stats)) * q_num; + count = ARRAY_SIZE(hinic_function_stats) + + (ARRAY_SIZE(hinic_tx_queue_stats) + + ARRAY_SIZE(hinic_rx_queue_stats)) * q_num; - count += ARRAY_LEN(hinic_port_stats); + count += ARRAY_SIZE(hinic_port_stats); return count; default: @@ -1476,27 +1478,27 @@ static void hinic_get_strings(struct net_device *netdev, memcpy(data, *hinic_test_strings, sizeof(hinic_test_strings)); return; case ETH_SS_STATS: - for (i = 0; i < ARRAY_LEN(hinic_function_stats); i++) { + for (i = 0; i < ARRAY_SIZE(hinic_function_stats); i++) { memcpy(p, hinic_function_stats[i].name, ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } - for (i = 0; i < ARRAY_LEN(hinic_port_stats); i++) { + for (i = 0; i < ARRAY_SIZE(hinic_port_stats); i++) { memcpy(p, hinic_port_stats[i].name, ETH_GSTRING_LEN); p += ETH_GSTRING_LEN; } for (i = 0; i < nic_dev->num_qps; i++) { - for (j = 0; j < ARRAY_LEN(hinic_tx_queue_stats); j++) { + for (j = 0; j < ARRAY_SIZE(hinic_tx_queue_stats); j++) { sprintf(p, hinic_tx_queue_stats[j].name, i); p += ETH_GSTRING_LEN; } } for (i = 0; i < nic_dev->num_qps; i++) { - for (j = 0; j < ARRAY_LEN(hinic_rx_queue_stats); j++) { + for (j = 0; j < ARRAY_SIZE(hinic_rx_queue_stats); j++) { sprintf(p, hinic_rx_queue_stats[j].name, i); p += ETH_GSTRING_LEN; } diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c index 06586173add7..998717f02136 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_api_cmd.c @@ -814,7 +814,6 @@ static int api_chain_init(struct hinic_api_cmd_chain *chain, { struct hinic_hwif *hwif = attr->hwif; struct pci_dev *pdev = hwif->pdev; - size_t cell_ctxt_size; chain->hwif = hwif; chain->chain_type = attr->chain_type; @@ -826,8 +825,8 @@ static int api_chain_init(struct hinic_api_cmd_chain *chain, sema_init(&chain->sem, 1); - cell_ctxt_size = chain->num_cells * sizeof(*chain->cell_ctxt); - chain->cell_ctxt = devm_kzalloc(&pdev->dev, cell_ctxt_size, GFP_KERNEL); + chain->cell_ctxt = devm_kcalloc(&pdev->dev, chain->num_cells, + sizeof(*chain->cell_ctxt), GFP_KERNEL); if (!chain->cell_ctxt) return -ENOMEM; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c index 307a6d4af993..a627237f694b 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_cmdq.c @@ -796,11 +796,10 @@ static int init_cmdqs_ctxt(struct hinic_hwdev *hwdev, struct hinic_cmdq_ctxt *cmdq_ctxts; struct pci_dev *pdev = hwif->pdev; struct hinic_pfhwdev *pfhwdev; - size_t cmdq_ctxts_size; int err; - cmdq_ctxts_size = HINIC_MAX_CMDQ_TYPES * sizeof(*cmdq_ctxts); - cmdq_ctxts = devm_kzalloc(&pdev->dev, cmdq_ctxts_size, GFP_KERNEL); + cmdq_ctxts = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES, + sizeof(*cmdq_ctxts), GFP_KERNEL); if (!cmdq_ctxts) return -ENOMEM; @@ -884,7 +883,6 @@ int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, struct hinic_func_to_io *func_to_io = cmdqs_to_func_to_io(cmdqs); struct pci_dev *pdev = hwif->pdev; struct hinic_hwdev *hwdev; - size_t saved_wqs_size; u16 max_wqe_size; int err; @@ -895,8 +893,8 @@ int hinic_init_cmdqs(struct hinic_cmdqs *cmdqs, struct hinic_hwif *hwif, if (!cmdqs->cmdq_buf_pool) return -ENOMEM; - saved_wqs_size = HINIC_MAX_CMDQ_TYPES * sizeof(struct hinic_wq); - cmdqs->saved_wqs = devm_kzalloc(&pdev->dev, saved_wqs_size, GFP_KERNEL); + cmdqs->saved_wqs = devm_kcalloc(&pdev->dev, HINIC_MAX_CMDQ_TYPES, + sizeof(*cmdqs->saved_wqs), GFP_KERNEL); if (!cmdqs->saved_wqs) { err = -ENOMEM; goto err_saved_wqs; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c index 657a15447bd0..2127a48749a8 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_dev.c @@ -162,7 +162,6 @@ static int init_msix(struct hinic_hwdev *hwdev) struct hinic_hwif *hwif = hwdev->hwif; struct pci_dev *pdev = hwif->pdev; int nr_irqs, num_aeqs, num_ceqs; - size_t msix_entries_size; int i, err; num_aeqs = HINIC_HWIF_NUM_AEQS(hwif); @@ -171,8 +170,8 @@ static int init_msix(struct hinic_hwdev *hwdev) if (nr_irqs > HINIC_HWIF_NUM_IRQS(hwif)) nr_irqs = HINIC_HWIF_NUM_IRQS(hwif); - msix_entries_size = nr_irqs * sizeof(*hwdev->msix_entries); - hwdev->msix_entries = devm_kzalloc(&pdev->dev, msix_entries_size, + hwdev->msix_entries = devm_kcalloc(&pdev->dev, nr_irqs, + sizeof(*hwdev->msix_entries), GFP_KERNEL); if (!hwdev->msix_entries) return -ENOMEM; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c index d3fc05a07fdb..045c47786a04 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_eqs.c @@ -631,16 +631,15 @@ static int alloc_eq_pages(struct hinic_eq *eq) struct hinic_hwif *hwif = eq->hwif; struct pci_dev *pdev = hwif->pdev; u32 init_val, addr, val; - size_t addr_size; int err, pg; - addr_size = eq->num_pages * sizeof(*eq->dma_addr); - eq->dma_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL); + eq->dma_addr = devm_kcalloc(&pdev->dev, eq->num_pages, + sizeof(*eq->dma_addr), GFP_KERNEL); if (!eq->dma_addr) return -ENOMEM; - addr_size = eq->num_pages * sizeof(*eq->virt_addr); - eq->virt_addr = devm_kzalloc(&pdev->dev, addr_size, GFP_KERNEL); + eq->virt_addr = devm_kcalloc(&pdev->dev, eq->num_pages, + sizeof(*eq->virt_addr), GFP_KERNEL); if (!eq->virt_addr) { err = -ENOMEM; goto err_virt_addr_alloc; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c index a6e43d686293..c4a0ba6e183a 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_io.c @@ -375,31 +375,30 @@ int hinic_io_create_qps(struct hinic_func_to_io *func_to_io, { struct hinic_hwif *hwif = func_to_io->hwif; struct pci_dev *pdev = hwif->pdev; - size_t qps_size, wq_size, db_size; void *ci_addr_base; int i, j, err; - qps_size = num_qps * sizeof(*func_to_io->qps); - func_to_io->qps = devm_kzalloc(&pdev->dev, qps_size, GFP_KERNEL); + func_to_io->qps = devm_kcalloc(&pdev->dev, num_qps, + sizeof(*func_to_io->qps), GFP_KERNEL); if (!func_to_io->qps) return -ENOMEM; - wq_size = num_qps * sizeof(*func_to_io->sq_wq); - func_to_io->sq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL); + func_to_io->sq_wq = devm_kcalloc(&pdev->dev, num_qps, + sizeof(*func_to_io->sq_wq), GFP_KERNEL); if (!func_to_io->sq_wq) { err = -ENOMEM; goto err_sq_wq; } - wq_size = num_qps * sizeof(*func_to_io->rq_wq); - func_to_io->rq_wq = devm_kzalloc(&pdev->dev, wq_size, GFP_KERNEL); + func_to_io->rq_wq = devm_kcalloc(&pdev->dev, num_qps, + sizeof(*func_to_io->rq_wq), GFP_KERNEL); if (!func_to_io->rq_wq) { err = -ENOMEM; goto err_rq_wq; } - db_size = num_qps * sizeof(*func_to_io->sq_db); - func_to_io->sq_db = devm_kzalloc(&pdev->dev, db_size, GFP_KERNEL); + func_to_io->sq_db = devm_kcalloc(&pdev->dev, num_qps, + sizeof(*func_to_io->sq_db), GFP_KERNEL); if (!func_to_io->sq_db) { err = -ENOMEM; goto err_sq_db; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c index 7f0f1aa3cedd..2d9b06d7caad 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_hw_wq.c @@ -193,20 +193,20 @@ static int alloc_page_arrays(struct hinic_wqs *wqs) { struct hinic_hwif *hwif = wqs->hwif; struct pci_dev *pdev = hwif->pdev; - size_t size; - size = wqs->num_pages * sizeof(*wqs->page_paddr); - wqs->page_paddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + wqs->page_paddr = devm_kcalloc(&pdev->dev, wqs->num_pages, + sizeof(*wqs->page_paddr), GFP_KERNEL); if (!wqs->page_paddr) return -ENOMEM; - size = wqs->num_pages * sizeof(*wqs->page_vaddr); - wqs->page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + wqs->page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages, + sizeof(*wqs->page_vaddr), GFP_KERNEL); if (!wqs->page_vaddr) goto err_page_vaddr; - size = wqs->num_pages * sizeof(*wqs->shadow_page_vaddr); - wqs->shadow_page_vaddr = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + wqs->shadow_page_vaddr = devm_kcalloc(&pdev->dev, wqs->num_pages, + sizeof(*wqs->shadow_page_vaddr), + GFP_KERNEL); if (!wqs->shadow_page_vaddr) goto err_page_shadow_vaddr; @@ -379,15 +379,14 @@ static int alloc_wqes_shadow(struct hinic_wq *wq) { struct hinic_hwif *hwif = wq->hwif; struct pci_dev *pdev = hwif->pdev; - size_t size; - size = wq->num_q_pages * wq->max_wqe_size; - wq->shadow_wqe = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + wq->shadow_wqe = devm_kcalloc(&pdev->dev, wq->num_q_pages, + wq->max_wqe_size, GFP_KERNEL); if (!wq->shadow_wqe) return -ENOMEM; - size = wq->num_q_pages * sizeof(wq->prod_idx); - wq->shadow_idx = devm_kzalloc(&pdev->dev, size, GFP_KERNEL); + wq->shadow_idx = devm_kcalloc(&pdev->dev, wq->num_q_pages, + sizeof(wq->prod_idx), GFP_KERNEL); if (!wq->shadow_idx) goto err_shadow_idx; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_main.c b/drivers/net/ethernet/huawei/hinic/hinic_main.c index f9a766b8ac43..1e1b1be86174 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_main.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_main.c @@ -144,13 +144,12 @@ static int create_txqs(struct hinic_dev *nic_dev) { int err, i, j, num_txqs = hinic_hwdev_num_qps(nic_dev->hwdev); struct net_device *netdev = nic_dev->netdev; - size_t txq_size; if (nic_dev->txqs) return -EINVAL; - txq_size = num_txqs * sizeof(*nic_dev->txqs); - nic_dev->txqs = devm_kzalloc(&netdev->dev, txq_size, GFP_KERNEL); + nic_dev->txqs = devm_kcalloc(&netdev->dev, num_txqs, + sizeof(*nic_dev->txqs), GFP_KERNEL); if (!nic_dev->txqs) return -ENOMEM; @@ -241,13 +240,12 @@ static int create_rxqs(struct hinic_dev *nic_dev) { int err, i, j, num_rxqs = hinic_hwdev_num_qps(nic_dev->hwdev); struct net_device *netdev = nic_dev->netdev; - size_t rxq_size; if (nic_dev->rxqs) return -EINVAL; - rxq_size = num_rxqs * sizeof(*nic_dev->rxqs); - nic_dev->rxqs = devm_kzalloc(&netdev->dev, rxq_size, GFP_KERNEL); + nic_dev->rxqs = devm_kcalloc(&netdev->dev, num_rxqs, + sizeof(*nic_dev->rxqs), GFP_KERNEL); if (!nic_dev->rxqs) return -ENOMEM; diff --git a/drivers/net/ethernet/huawei/hinic/hinic_tx.c b/drivers/net/ethernet/huawei/hinic/hinic_tx.c index c5bdb0d374ef..a984a7a6dd2e 100644 --- a/drivers/net/ethernet/huawei/hinic/hinic_tx.c +++ b/drivers/net/ethernet/huawei/hinic/hinic_tx.c @@ -862,7 +862,6 @@ int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq, struct hinic_dev *nic_dev = netdev_priv(netdev); struct hinic_hwdev *hwdev = nic_dev->hwdev; int err, irqname_len; - size_t sges_size; txq->netdev = netdev; txq->sq = sq; @@ -871,13 +870,13 @@ int hinic_init_txq(struct hinic_txq *txq, struct hinic_sq *sq, txq->max_sges = HINIC_MAX_SQ_BUFDESCS; - sges_size = txq->max_sges * sizeof(*txq->sges); - txq->sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL); + txq->sges = devm_kcalloc(&netdev->dev, txq->max_sges, + sizeof(*txq->sges), GFP_KERNEL); if (!txq->sges) return -ENOMEM; - sges_size = txq->max_sges * sizeof(*txq->free_sges); - txq->free_sges = devm_kzalloc(&netdev->dev, sges_size, GFP_KERNEL); + txq->free_sges = devm_kcalloc(&netdev->dev, txq->max_sges, + sizeof(*txq->free_sges), GFP_KERNEL); if (!txq->free_sges) { err = -ENOMEM; goto err_alloc_free_sges; diff --git a/drivers/net/ethernet/i825xx/82596.c b/drivers/net/ethernet/i825xx/82596.c index b482f6f633bd..3ee89ae496d0 100644 --- a/drivers/net/ethernet/i825xx/82596.c +++ b/drivers/net/ethernet/i825xx/82596.c @@ -1178,7 +1178,8 @@ found: DEB(DEB_PROBE,printk(KERN_INFO "%s: 82596 at %#3lx,", dev->name, dev->base_addr)); for (i = 0; i < 6; i++) - DEB(DEB_PROBE,printk(" %2.2X", dev->dev_addr[i] = eth_addr[i])); + DEB(DEB_PROBE,printk(" %2.2X", eth_addr[i])); + eth_hw_addr_set(dev, eth_addr); DEB(DEB_PROBE,printk(" IRQ %d.\n", dev->irq)); diff --git a/drivers/net/ethernet/i825xx/lasi_82596.c b/drivers/net/ethernet/i825xx/lasi_82596.c index 48e001881c75..0af70094aba3 100644 --- a/drivers/net/ethernet/i825xx/lasi_82596.c +++ b/drivers/net/ethernet/i825xx/lasi_82596.c @@ -147,6 +147,7 @@ lan_init_chip(struct parisc_device *dev) struct net_device *netdevice; struct i596_private *lp; int retval = -ENOMEM; + u8 addr[ETH_ALEN]; int i; if (!dev->irq) { @@ -167,13 +168,14 @@ lan_init_chip(struct parisc_device *dev) netdevice->base_addr = dev->hpa.start; netdevice->irq = dev->irq; - if (pdc_lan_station_id(netdevice->dev_addr, netdevice->base_addr)) { + if (pdc_lan_station_id(addr, netdevice->base_addr)) { for (i = 0; i < 6; i++) { - netdevice->dev_addr[i] = gsc_readb(LAN_PROM_ADDR + i); + addr[i] = gsc_readb(LAN_PROM_ADDR + i); } printk(KERN_INFO "%s: MAC of HP700 LAN read from EEPROM\n", __FILE__); } + eth_hw_addr_set(netdevice, addr); lp = netdev_priv(netdevice); lp->options = dev->id.sversion == 0x72 ? OPT_SWAP_PORT : 0; diff --git a/drivers/net/ethernet/ibm/emac/core.c b/drivers/net/ethernet/ibm/emac/core.c index 6b3fc8823c54..fbea9f7efe8c 100644 --- a/drivers/net/ethernet/ibm/emac/core.c +++ b/drivers/net/ethernet/ibm/emac/core.c @@ -2137,8 +2137,11 @@ emac_ethtool_set_link_ksettings(struct net_device *ndev, return 0; } -static void emac_ethtool_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *rp) +static void +emac_ethtool_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { rp->rx_max_pending = rp->rx_pending = NUM_RX_BUFF; rp->tx_max_pending = rp->tx_pending = NUM_TX_BUFF; diff --git a/drivers/net/ethernet/ibm/ibmvnic.c b/drivers/net/ethernet/ibm/ibmvnic.c index 0bb3911dd014..4ba4c4f3be4f 100644 --- a/drivers/net/ethernet/ibm/ibmvnic.c +++ b/drivers/net/ethernet/ibm/ibmvnic.c @@ -2042,7 +2042,7 @@ static netdev_tx_t ibmvnic_xmit(struct sk_buff *skb, struct net_device *netdev) tx_packets++; tx_bytes += skb->len; - txq->trans_start = jiffies; + txq_trans_cond_update(txq); ret = NETDEV_TX_OK; goto out; @@ -3072,7 +3072,9 @@ static u32 ibmvnic_get_link(struct net_device *netdev) } static void ibmvnic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); @@ -3092,7 +3094,9 @@ static void ibmvnic_get_ringparam(struct net_device *netdev, } static int ibmvnic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ibmvnic_adapter *adapter = netdev_priv(netdev); int ret; diff --git a/drivers/net/ethernet/intel/e100.c b/drivers/net/ethernet/intel/e100.c index 0bf3d47bb90d..4a8013f20152 100644 --- a/drivers/net/ethernet/intel/e100.c +++ b/drivers/net/ethernet/intel/e100.c @@ -2557,7 +2557,9 @@ static int e100_set_eeprom(struct net_device *netdev, } static void e100_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nic *nic = netdev_priv(netdev); struct param_range *rfds = &nic->params.rfds; @@ -2570,7 +2572,9 @@ static void e100_get_ringparam(struct net_device *netdev, } static int e100_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nic *nic = netdev_priv(netdev); struct param_range *rfds = &nic->params.rfds; diff --git a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c index 0a57172dfcbc..32803b0cf1e8 100644 --- a/drivers/net/ethernet/intel/e1000/e1000_ethtool.c +++ b/drivers/net/ethernet/intel/e1000/e1000_ethtool.c @@ -539,7 +539,9 @@ static void e1000_get_drvinfo(struct net_device *netdev, } static void e1000_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; @@ -556,7 +558,9 @@ static void e1000_get_ringparam(struct net_device *netdev, } static int e1000_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_hw *hw = &adapter->hw; diff --git a/drivers/net/ethernet/intel/e1000e/ethtool.c b/drivers/net/ethernet/intel/e1000e/ethtool.c index 8515e00d1b40..b80ae9a82224 100644 --- a/drivers/net/ethernet/intel/e1000e/ethtool.c +++ b/drivers/net/ethernet/intel/e1000e/ethtool.c @@ -655,7 +655,9 @@ static void e1000_get_drvinfo(struct net_device *netdev, } static void e1000_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); @@ -666,7 +668,9 @@ static void e1000_get_ringparam(struct net_device *netdev, } static int e1000_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct e1000_adapter *adapter = netdev_priv(netdev); struct e1000_ring *temp_tx = NULL, *temp_rx = NULL; diff --git a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c index 0d37f011d0ce..d53369e30040 100644 --- a/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c +++ b/drivers/net/ethernet/intel/fm10k/fm10k_ethtool.c @@ -502,7 +502,9 @@ static void fm10k_set_msglevel(struct net_device *netdev, u32 data) } static void fm10k_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fm10k_intfc *interface = netdev_priv(netdev); @@ -517,7 +519,9 @@ static void fm10k_get_ringparam(struct net_device *netdev, } static int fm10k_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fm10k_intfc *interface = netdev_priv(netdev); struct fm10k_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 513ba6974355..091f36adbbe1 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -1916,7 +1916,9 @@ static void i40e_get_drvinfo(struct net_device *netdev, } static void i40e_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct i40e_netdev_priv *np = netdev_priv(netdev); struct i40e_pf *pf = np->vsi->back; @@ -1944,7 +1946,9 @@ static bool i40e_active_tx_ring_index(struct i40e_vsi *vsi, u16 index) } static int i40e_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct i40e_ring *tx_rings = NULL, *rx_rings = NULL; struct i40e_netdev_priv *np = netdev_priv(netdev); diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 3789269ce741..b5728bdbcf33 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -137,9 +137,13 @@ struct iavf_q_vector { struct iavf_mac_filter { struct list_head list; u8 macaddr[ETH_ALEN]; - bool is_new_mac; /* filter is new, wait for PF decision */ - bool remove; /* filter needs to be removed */ - bool add; /* filter needs to be added */ + struct { + u8 is_new_mac:1; /* filter is new, wait for PF decision */ + u8 remove:1; /* filter needs to be removed */ + u8 add:1; /* filter needs to be added */ + u8 is_primary:1; /* filter is a default VF MAC */ + u8 padding:4; + }; }; struct iavf_vlan_filter { diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 461f5237a2f8..3bb56714beb0 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -331,9 +331,16 @@ static int iavf_get_link_ksettings(struct net_device *netdev, **/ static int iavf_get_sset_count(struct net_device *netdev, int sset) { + /* Report the maximum number queues, even if not every queue is + * currently configured. Since allocation of queues is in pairs, + * use netdev->real_num_tx_queues * 2. The real_num_tx_queues is set + * at device creation and never changes. + */ + if (sset == ETH_SS_STATS) return IAVF_STATS_LEN + - (IAVF_QUEUE_STATS_LEN * 2 * IAVF_MAX_REQ_QUEUES); + (IAVF_QUEUE_STATS_LEN * 2 * + netdev->real_num_tx_queues); else if (sset == ETH_SS_PRIV_FLAGS) return IAVF_PRIV_FLAGS_STR_LEN; else @@ -360,17 +367,18 @@ static void iavf_get_ethtool_stats(struct net_device *netdev, iavf_add_ethtool_stats(&data, adapter, iavf_gstrings_stats); rcu_read_lock(); - for (i = 0; i < IAVF_MAX_REQ_QUEUES; i++) { + /* As num_active_queues describe both tx and rx queues, we can use + * it to iterate over rings' stats. + */ + for (i = 0; i < adapter->num_active_queues; i++) { struct iavf_ring *ring; - /* Avoid accessing un-allocated queues */ - ring = (i < adapter->num_active_queues ? - &adapter->tx_rings[i] : NULL); + /* Tx rings stats */ + ring = &adapter->tx_rings[i]; iavf_add_queue_stats(&data, ring); - /* Avoid accessing un-allocated queues */ - ring = (i < adapter->num_active_queues ? - &adapter->rx_rings[i] : NULL); + /* Rx rings stats */ + ring = &adapter->rx_rings[i]; iavf_add_queue_stats(&data, ring); } rcu_read_unlock(); @@ -407,10 +415,10 @@ static void iavf_get_stat_strings(struct net_device *netdev, u8 *data) iavf_add_stat_strings(&data, iavf_gstrings_stats); - /* Queues are always allocated in pairs, so we just use num_tx_queues - * for both Tx and Rx queues. + /* Queues are always allocated in pairs, so we just use + * real_num_tx_queues for both Tx and Rx queues. */ - for (i = 0; i < netdev->num_tx_queues; i++) { + for (i = 0; i < netdev->real_num_tx_queues; i++) { iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, "tx", i); iavf_add_stat_strings(&data, iavf_gstrings_queue_stats, @@ -583,12 +591,16 @@ static void iavf_get_drvinfo(struct net_device *netdev, * iavf_get_ringparam - Get ring parameters * @netdev: network interface device structure * @ring: ethtool ringparam structure + * @kernel_ring: ethtool extenal ringparam structure + * @extack: netlink extended ACK report struct * * Returns current ring parameters. TX and RX rings are reported separately, * but the number of rings is not reported. **/ static void iavf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct iavf_adapter *adapter = netdev_priv(netdev); @@ -602,12 +614,16 @@ static void iavf_get_ringparam(struct net_device *netdev, * iavf_set_ringparam - Set ring parameters * @netdev: network interface device structure * @ring: ethtool ringparam structure + * @kernel_ring: ethtool external ringparam structure + * @extack: netlink extended ACK report struct * * Sets ring parameters. TX and RX rings are controlled separately, but the * number of rings is not specified, so all rings get the same settings. **/ static int iavf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct iavf_adapter *adapter = netdev_priv(netdev); u32 new_rx_count, new_tx_count; @@ -1923,7 +1939,7 @@ static int iavf_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, * @key: hash key * @hfunc: hash function to use * - * Returns -EINVAL if the table specifies an inavlid queue id, otherwise + * Returns -EINVAL if the table specifies an invalid queue id, otherwise * returns 0 after programming the table. **/ static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, @@ -1932,19 +1948,21 @@ static int iavf_set_rxfh(struct net_device *netdev, const u32 *indir, struct iavf_adapter *adapter = netdev_priv(netdev); u16 i; - /* We do not allow change in unsupported parameters */ - if (key || - (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) + /* Only support toeplitz hash function */ + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) return -EOPNOTSUPP; - if (!indir) + + if (!key && !indir) return 0; if (key) memcpy(adapter->rss_key, key, adapter->rss_key_size); - /* Each 32 bits pointed by 'indir' is stored with a lut entry */ - for (i = 0; i < adapter->rss_lut_size; i++) - adapter->rss_lut[i] = (u8)(indir[i]); + if (indir) { + /* Each 32 bits pointed by 'indir' is stored with a lut entry */ + for (i = 0; i < adapter->rss_lut_size; i++) + adapter->rss_lut[i] = (u8)(indir[i]); + } return iavf_config_rss(adapter); } diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index cfdbf8c08d18..3a820b3ff466 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -463,14 +463,14 @@ iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) if (q_vector->tx.ring && q_vector->rx.ring) { snprintf(q_vector->name, sizeof(q_vector->name), - "iavf-%s-TxRx-%d", basename, rx_int_idx++); + "iavf-%s-TxRx-%u", basename, rx_int_idx++); tx_int_idx++; } else if (q_vector->rx.ring) { snprintf(q_vector->name, sizeof(q_vector->name), - "iavf-%s-rx-%d", basename, rx_int_idx++); + "iavf-%s-rx-%u", basename, rx_int_idx++); } else if (q_vector->tx.ring) { snprintf(q_vector->name, sizeof(q_vector->name), - "iavf-%s-tx-%d", basename, tx_int_idx++); + "iavf-%s-tx-%u", basename, tx_int_idx++); } else { /* skip this unused q_vector */ continue; @@ -2911,7 +2911,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", match.mask->dst); - return IAVF_ERR_CONFIG; + return -EINVAL; } } @@ -2921,7 +2921,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", match.mask->src); - return IAVF_ERR_CONFIG; + return -EINVAL; } } @@ -2956,7 +2956,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", match.mask->vlan_id); - return IAVF_ERR_CONFIG; + return -EINVAL; } } vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); @@ -2980,7 +2980,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", be32_to_cpu(match.mask->dst)); - return IAVF_ERR_CONFIG; + return -EINVAL; } } @@ -2990,13 +2990,13 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", be32_to_cpu(match.mask->dst)); - return IAVF_ERR_CONFIG; + return -EINVAL; } } if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); - return IAVF_ERR_CONFIG; + return -EINVAL; } if (match.key->dst) { vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); @@ -3017,7 +3017,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, if (ipv6_addr_any(&match.mask->dst)) { dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", IPV6_ADDR_ANY); - return IAVF_ERR_CONFIG; + return -EINVAL; } /* src and dest IPv6 address should not be LOOPBACK @@ -3027,7 +3027,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, ipv6_addr_loopback(&match.key->src)) { dev_err(&adapter->pdev->dev, "ipv6 addr should not be loopback\n"); - return IAVF_ERR_CONFIG; + return -EINVAL; } if (!ipv6_addr_any(&match.mask->dst) || !ipv6_addr_any(&match.mask->src)) @@ -3052,7 +3052,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", be16_to_cpu(match.mask->src)); - return IAVF_ERR_CONFIG; + return -EINVAL; } } @@ -3062,7 +3062,7 @@ static int iavf_parse_cls_flower(struct iavf_adapter *adapter, } else { dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", be16_to_cpu(match.mask->dst)); - return IAVF_ERR_CONFIG; + return -EINVAL; } } if (match.key->dst) { @@ -3429,6 +3429,8 @@ static int iavf_change_mtu(struct net_device *netdev, int new_mtu) { struct iavf_adapter *adapter = netdev_priv(netdev); + netdev_dbg(netdev, "changing MTU from %d to %d\n", + netdev->mtu, new_mtu); netdev->mtu = new_mtu; if (CLIENT_ENABLED(adapter)) { iavf_notify_client_l2_params(&adapter->vsi); @@ -3999,6 +4001,7 @@ static void iavf_remove(struct pci_dev *pdev) if (iavf_lock_timeout(&adapter->crit_lock, 5000)) dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__); + dev_info(&adapter->pdev->dev, "Removing device\n"); /* Shut down all the garbage mashers on the detention level */ iavf_change_state(adapter, __IAVF_REMOVE); adapter->aq_required = 0; diff --git a/drivers/net/ethernet/intel/iavf/iavf_txrx.c b/drivers/net/ethernet/intel/iavf/iavf_txrx.c index 3525eab8e9f9..42c9f9dc235c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_txrx.c +++ b/drivers/net/ethernet/intel/iavf/iavf_txrx.c @@ -1766,7 +1766,7 @@ tx_only: if (likely(napi_complete_done(napi, work_done))) iavf_update_enable_itr(vsi, q_vector); - return min(work_done, budget - 1); + return min_t(int, work_done, budget - 1); } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index d60bf7c21200..cfa1f0e0e2fe 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -762,15 +762,23 @@ void iavf_set_promiscuous(struct iavf_adapter *adapter, int flags) if (flags & FLAG_VF_MULTICAST_PROMISC) { adapter->flags |= IAVF_FLAG_ALLMULTI_ON; adapter->aq_required &= ~IAVF_FLAG_AQ_REQUEST_ALLMULTI; - dev_info(&adapter->pdev->dev, "Entering multicast promiscuous mode\n"); + dev_info(&adapter->pdev->dev, "%s is entering multicast promiscuous mode\n", + adapter->netdev->name); } if (!flags) { - adapter->flags &= ~(IAVF_FLAG_PROMISC_ON | - IAVF_FLAG_ALLMULTI_ON); - adapter->aq_required &= ~(IAVF_FLAG_AQ_RELEASE_PROMISC | - IAVF_FLAG_AQ_RELEASE_ALLMULTI); - dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n"); + if (adapter->flags & IAVF_FLAG_PROMISC_ON) { + adapter->flags &= ~IAVF_FLAG_PROMISC_ON; + adapter->aq_required &= ~IAVF_FLAG_AQ_RELEASE_PROMISC; + dev_info(&adapter->pdev->dev, "Leaving promiscuous mode\n"); + } + + if (adapter->flags & IAVF_FLAG_ALLMULTI_ON) { + adapter->flags &= ~IAVF_FLAG_ALLMULTI_ON; + adapter->aq_required &= ~IAVF_FLAG_AQ_RELEASE_ALLMULTI; + dev_info(&adapter->pdev->dev, "%s is leaving multicast promiscuous mode\n", + adapter->netdev->name); + } } adapter->current_op = VIRTCHNL_OP_CONFIG_PROMISCUOUS_MODE; @@ -1017,7 +1025,7 @@ print_link_msg: } else if (link_speed_mbps == SPEED_UNKNOWN) { snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%s", "Unknown Mbps"); } else { - snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%u %s", + snprintf(speed, IAVF_MAX_SPEED_STRLEN, "%d %s", link_speed_mbps, "Mbps"); } @@ -1522,7 +1530,7 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, iavf_print_link_message(adapter); break; case VIRTCHNL_EVENT_RESET_IMPENDING: - dev_info(&adapter->pdev->dev, "Reset warning received from the PF\n"); + dev_info(&adapter->pdev->dev, "Reset indication received from the PF\n"); if (!(adapter->flags & IAVF_FLAG_RESET_PENDING)) { adapter->flags |= IAVF_FLAG_RESET_PENDING; dev_info(&adapter->pdev->dev, "Scheduling reset task\n"); diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index b2db39ee5f85..b67ad51cbcc9 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -576,6 +576,7 @@ struct ice_pf { struct ice_hw_port_stats stats_prev; struct ice_hw hw; u8 stat_prev_loaded:1; /* has previous stats been loaded */ + u8 rdma_mode; u16 dcbx_cap; u32 tx_timeout_count; unsigned long tx_timeout_last_recovery; diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.c b/drivers/net/ethernet/intel/ice/ice_devlink.c index b9bd9f9472f6..1cfe918db8b9 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.c +++ b/drivers/net/ethernet/intel/ice/ice_devlink.c @@ -430,6 +430,120 @@ static const struct devlink_ops ice_devlink_ops = { .flash_update = ice_devlink_flash_update, }; +static int +ice_devlink_enable_roce_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + + ctx->val.vbool = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? true : false; + + return 0; +} + +static int +ice_devlink_enable_roce_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + bool roce_ena = ctx->val.vbool; + int ret; + + if (!roce_ena) { + ice_unplug_aux_dev(pf); + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_ROCEV2; + return 0; + } + + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_ROCEV2; + ret = ice_plug_aux_dev(pf); + if (ret) + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_ROCEV2; + + return ret; +} + +static int +ice_devlink_enable_roce_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct ice_pf *pf = devlink_priv(devlink); + + if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + return -EOPNOTSUPP; + + if (pf->rdma_mode & IIDC_RDMA_PROTOCOL_IWARP) { + NL_SET_ERR_MSG_MOD(extack, "iWARP is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int +ice_devlink_enable_iw_get(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + + ctx->val.vbool = pf->rdma_mode & IIDC_RDMA_PROTOCOL_IWARP; + + return 0; +} + +static int +ice_devlink_enable_iw_set(struct devlink *devlink, u32 id, + struct devlink_param_gset_ctx *ctx) +{ + struct ice_pf *pf = devlink_priv(devlink); + bool iw_ena = ctx->val.vbool; + int ret; + + if (!iw_ena) { + ice_unplug_aux_dev(pf); + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_IWARP; + return 0; + } + + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_IWARP; + ret = ice_plug_aux_dev(pf); + if (ret) + pf->rdma_mode &= ~IIDC_RDMA_PROTOCOL_IWARP; + + return ret; +} + +static int +ice_devlink_enable_iw_validate(struct devlink *devlink, u32 id, + union devlink_param_value val, + struct netlink_ext_ack *extack) +{ + struct ice_pf *pf = devlink_priv(devlink); + + if (!test_bit(ICE_FLAG_RDMA_ENA, pf->flags)) + return -EOPNOTSUPP; + + if (pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2) { + NL_SET_ERR_MSG_MOD(extack, "RoCEv2 is currently enabled. This device cannot enable iWARP and RoCEv2 simultaneously"); + return -EOPNOTSUPP; + } + + return 0; +} + +static const struct devlink_param ice_devlink_params[] = { + DEVLINK_PARAM_GENERIC(ENABLE_ROCE, BIT(DEVLINK_PARAM_CMODE_RUNTIME), + ice_devlink_enable_roce_get, + ice_devlink_enable_roce_set, + ice_devlink_enable_roce_validate), + DEVLINK_PARAM_GENERIC(ENABLE_IWARP, BIT(DEVLINK_PARAM_CMODE_RUNTIME), + ice_devlink_enable_iw_get, + ice_devlink_enable_iw_set, + ice_devlink_enable_iw_validate), + +}; + static void ice_devlink_free(void *devlink_ptr) { devlink_free((struct devlink *)devlink_ptr); @@ -484,6 +598,36 @@ void ice_devlink_unregister(struct ice_pf *pf) devlink_unregister(priv_to_devlink(pf)); } +int ice_devlink_register_params(struct ice_pf *pf) +{ + struct devlink *devlink = priv_to_devlink(pf); + union devlink_param_value value; + int err; + + err = devlink_params_register(devlink, ice_devlink_params, + ARRAY_SIZE(ice_devlink_params)); + if (err) + return err; + + value.vbool = false; + devlink_param_driverinit_value_set(devlink, + DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP, + value); + + value.vbool = test_bit(ICE_FLAG_RDMA_ENA, pf->flags) ? true : false; + devlink_param_driverinit_value_set(devlink, + DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE, + value); + + return 0; +} + +void ice_devlink_unregister_params(struct ice_pf *pf) +{ + devlink_params_unregister(priv_to_devlink(pf), ice_devlink_params, + ARRAY_SIZE(ice_devlink_params)); +} + /** * ice_devlink_create_pf_port - Create a devlink port for this PF * @pf: the PF to create a devlink port for diff --git a/drivers/net/ethernet/intel/ice/ice_devlink.h b/drivers/net/ethernet/intel/ice/ice_devlink.h index b7f9551e4fc4..fe006d9946f8 100644 --- a/drivers/net/ethernet/intel/ice/ice_devlink.h +++ b/drivers/net/ethernet/intel/ice/ice_devlink.h @@ -8,6 +8,8 @@ struct ice_pf *ice_allocate_pf(struct device *dev); void ice_devlink_register(struct ice_pf *pf); void ice_devlink_unregister(struct ice_pf *pf); +int ice_devlink_register_params(struct ice_pf *pf); +void ice_devlink_unregister_params(struct ice_pf *pf); int ice_devlink_create_pf_port(struct ice_pf *pf); void ice_devlink_destroy_pf_port(struct ice_pf *pf); int ice_devlink_create_vf_port(struct ice_vf *vf); diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index 572519e402f4..5af2faaa21e1 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -2686,7 +2686,9 @@ ice_get_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd, } static void -ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; @@ -2704,7 +2706,9 @@ ice_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) } static int -ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +ice_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_tx_ring *xdp_rings = NULL; diff --git a/drivers/net/ethernet/intel/ice/ice_idc.c b/drivers/net/ethernet/intel/ice/ice_idc.c index adcc9a251595..fc3580167e7b 100644 --- a/drivers/net/ethernet/intel/ice/ice_idc.c +++ b/drivers/net/ethernet/intel/ice/ice_idc.c @@ -288,7 +288,7 @@ int ice_plug_aux_dev(struct ice_pf *pf) adev->id = pf->aux_idx; adev->dev.release = ice_adev_release; adev->dev.parent = &pf->pdev->dev; - adev->name = IIDC_RDMA_ROCE_NAME; + adev->name = pf->rdma_mode & IIDC_RDMA_PROTOCOL_ROCEV2 ? "roce" : "iwarp"; ret = auxiliary_device_init(adev); if (ret) { @@ -335,6 +335,6 @@ int ice_init_rdma(struct ice_pf *pf) dev_err(dev, "failed to reserve vectors for RDMA\n"); return ret; } - + pf->rdma_mode |= IIDC_RDMA_PROTOCOL_ROCEV2; return ice_plug_aux_dev(pf); } diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 73c61cdb036f..a3ce54a78859 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -4721,6 +4721,10 @@ probe_done: if (err) goto err_netdev_reg; + err = ice_devlink_register_params(pf); + if (err) + goto err_netdev_reg; + /* ready to go, so clear down state bit */ clear_bit(ICE_DOWN, pf->state); if (ice_is_aux_ena(pf)) { @@ -4728,7 +4732,7 @@ probe_done: if (pf->aux_idx < 0) { dev_err(dev, "Failed to allocate device ID for AUX driver\n"); err = -ENOMEM; - goto err_netdev_reg; + goto err_devlink_reg_param; } err = ice_init_rdma(pf); @@ -4747,6 +4751,8 @@ probe_done: err_init_aux_unroll: pf->adev = NULL; ida_free(&ice_aux_ida, pf->aux_idx); +err_devlink_reg_param: + ice_devlink_unregister_params(pf); err_netdev_reg: err_send_version_unroll: ice_vsi_release_all(pf); @@ -4861,6 +4867,7 @@ static void ice_remove(struct pci_dev *pdev) ice_unplug_aux_dev(pf); if (pf->aux_idx >= 0) ida_free(&ice_aux_ida, pf->aux_idx); + ice_devlink_unregister_params(pf); set_bit(ICE_DOWN, pf->state); mutex_destroy(&(&pf->hw)->fdir_fltr_lock); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index fb1029352c3e..51a2dcaf553d 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -864,7 +864,9 @@ static void igb_get_drvinfo(struct net_device *netdev, } static void igb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igb_adapter *adapter = netdev_priv(netdev); @@ -875,7 +877,9 @@ static void igb_get_ringparam(struct net_device *netdev, } static int igb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igb_adapter *adapter = netdev_priv(netdev); struct igb_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index fd54d3ef890b..dd208930fbe4 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -2927,7 +2927,7 @@ static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp) nq = txring_txq(tx_ring); __netif_tx_lock(nq, cpu); /* Avoid transmit queue timeout since we share it with the slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf); __netif_tx_unlock(nq); @@ -2961,7 +2961,7 @@ static int igb_xdp_xmit(struct net_device *dev, int n, __netif_tx_lock(nq, cpu); /* Avoid transmit queue timeout since we share it with the slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); for (i = 0; i < n; i++) { struct xdp_frame *xdpf = frames[i]; diff --git a/drivers/net/ethernet/intel/igbvf/ethtool.c b/drivers/net/ethernet/intel/igbvf/ethtool.c index 06e5bd646a0e..9d4322b74163 100644 --- a/drivers/net/ethernet/intel/igbvf/ethtool.c +++ b/drivers/net/ethernet/intel/igbvf/ethtool.c @@ -175,7 +175,9 @@ static void igbvf_get_drvinfo(struct net_device *netdev, } static void igbvf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igbvf_adapter *adapter = netdev_priv(netdev); struct igbvf_ring *tx_ring = adapter->tx_ring; @@ -188,7 +190,9 @@ static void igbvf_get_ringparam(struct net_device *netdev, } static int igbvf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct igbvf_adapter *adapter = netdev_priv(netdev); struct igbvf_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index e0a76ac1bbbc..8cc077b712ad 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -567,8 +567,11 @@ static int igc_ethtool_set_eeprom(struct net_device *netdev, return ret_val; } -static void igc_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +igc_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); @@ -578,8 +581,11 @@ static void igc_ethtool_get_ringparam(struct net_device *netdev, ring->tx_pending = adapter->tx_ring_count; } -static int igc_ethtool_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static int +igc_ethtool_set_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 8e448288ee26..142c57b7a451 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1718,24 +1718,26 @@ static void igc_add_rx_frag(struct igc_ring *rx_ring, static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring, struct igc_rx_buffer *rx_buffer, - union igc_adv_rx_desc *rx_desc, - unsigned int size) + struct xdp_buff *xdp) { - void *va = page_address(rx_buffer->page) + rx_buffer->page_offset; + unsigned int size = xdp->data_end - xdp->data; unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); + unsigned int metasize = xdp->data - xdp->data_meta; struct sk_buff *skb; /* prefetch first cache line of first page */ - net_prefetch(va); + net_prefetch(xdp->data_meta); /* build an skb around the page buffer */ - skb = build_skb(va - IGC_SKB_PAD, truesize); + skb = build_skb(xdp->data_hard_start, truesize); if (unlikely(!skb)) return NULL; /* update pointers within the skb to store the data */ - skb_reserve(skb, IGC_SKB_PAD); + skb_reserve(skb, xdp->data - xdp->data_hard_start); __skb_put(skb, size); + if (metasize) + skb_metadata_set(skb, metasize); igc_rx_buffer_flip(rx_buffer, truesize); return skb; @@ -1746,6 +1748,7 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, struct xdp_buff *xdp, ktime_t timestamp) { + unsigned int metasize = xdp->data - xdp->data_meta; unsigned int size = xdp->data_end - xdp->data; unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size); void *va = xdp->data; @@ -1753,10 +1756,11 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, struct sk_buff *skb; /* prefetch first cache line of first page */ - net_prefetch(va); + net_prefetch(xdp->data_meta); /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGC_RX_HDR_LEN); + skb = napi_alloc_skb(&rx_ring->q_vector->napi, + IGC_RX_HDR_LEN + metasize); if (unlikely(!skb)) return NULL; @@ -1769,7 +1773,13 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN); /* align pull length to size of long to optimize memcpy performance */ - memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long))); + memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta, + ALIGN(headlen + metasize, sizeof(long))); + + if (metasize) { + skb_metadata_set(skb, metasize); + __skb_pull(skb, metasize); + } /* update all of the pointers */ size -= headlen; @@ -2354,7 +2364,8 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) if (!skb) { xdp_init_buff(&xdp, truesize, &rx_ring->xdp_rxq); xdp_prepare_buff(&xdp, pktbuf - igc_rx_offset(rx_ring), - igc_rx_offset(rx_ring) + pkt_offset, size, false); + igc_rx_offset(rx_ring) + pkt_offset, + size, true); skb = igc_xdp_run_prog(adapter, &xdp); } @@ -2378,7 +2389,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) } else if (skb) igc_add_rx_frag(rx_ring, rx_buffer, skb, size); else if (ring_uses_build_skb(rx_ring)) - skb = igc_build_skb(rx_ring, rx_buffer, rx_desc, size); + skb = igc_build_skb(rx_ring, rx_buffer, &xdp); else skb = igc_construct_skb(rx_ring, rx_buffer, &xdp, timestamp); @@ -2448,8 +2459,10 @@ static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring, skb_reserve(skb, xdp->data_meta - xdp->data_hard_start); memcpy(__skb_put(skb, totalsize), xdp->data_meta, totalsize); - if (metasize) + if (metasize) { skb_metadata_set(skb, metasize); + __skb_pull(skb, metasize); + } return skb; } diff --git a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c index 582099a5ad41..46efcfab7234 100644 --- a/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c +++ b/drivers/net/ethernet/intel/ixgb/ixgb_ethtool.c @@ -464,7 +464,9 @@ ixgb_get_drvinfo(struct net_device *netdev, static void ixgb_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgb_adapter *adapter = netdev_priv(netdev); struct ixgb_desc_ring *txdr = &adapter->tx_ring; @@ -478,7 +480,9 @@ ixgb_get_ringparam(struct net_device *netdev, static int ixgb_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgb_adapter *adapter = netdev_priv(netdev); struct ixgb_desc_ring *txdr = &adapter->tx_ring; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 8362822316a9..f70967c32116 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1118,7 +1118,9 @@ static void ixgbe_get_drvinfo(struct net_device *netdev, } static void ixgbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_ring *tx_ring = adapter->tx_ring[0]; @@ -1131,7 +1133,9 @@ static void ixgbe_get_ringparam(struct net_device *netdev, } static int ixgbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = netdev_priv(netdev); struct ixgbe_ring *temp_ring; diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index 6bace746eaac..5f08779c0e4e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -281,6 +281,10 @@ struct ixgbe_adv_tx_context_desc { #define IXGBE_ERR_INVALID_MAC_ADDR -1 #define IXGBE_ERR_RESET_FAILED -2 #define IXGBE_ERR_INVALID_ARGUMENT -3 +#define IXGBE_ERR_CONFIG -4 +#define IXGBE_ERR_MBX -5 +#define IXGBE_ERR_TIMEOUT -6 +#define IXGBE_ERR_PARAM -7 /* Transmit Config masks */ #define IXGBE_TXDCTL_ENABLE 0x02000000 /* Ena specific Tx Queue */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ethtool.c b/drivers/net/ethernet/intel/ixgbevf/ethtool.c index 8380f905e708..3b41f83c8dff 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ethtool.c +++ b/drivers/net/ethernet/intel/ixgbevf/ethtool.c @@ -225,7 +225,9 @@ static void ixgbevf_get_drvinfo(struct net_device *netdev, } static void ixgbevf_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); @@ -236,7 +238,9 @@ static void ixgbevf_get_ringparam(struct net_device *netdev, } static int ixgbevf_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ixgbevf_adapter *adapter = netdev_priv(netdev); struct ixgbevf_ring *tx_ring = NULL, *rx_ring = NULL; diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index e3e4676af9e4..e763cee0695e 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -40,16 +40,16 @@ static int ixgbevf_ipsec_set_pf_sa(struct ixgbevf_adapter *adapter, spin_lock_bh(&adapter->mbx_lock); - ret = hw->mbx.ops.write_posted(hw, msgbuf, IXGBE_VFMAILBOX_SIZE); + ret = ixgbevf_write_mbx(hw, msgbuf, IXGBE_VFMAILBOX_SIZE); if (ret) goto out; - ret = hw->mbx.ops.read_posted(hw, msgbuf, 2); + ret = ixgbevf_poll_mbx(hw, msgbuf, 2); if (ret) goto out; ret = (int)msgbuf[1]; - if (msgbuf[0] & IXGBE_VT_MSGTYPE_NACK && ret >= 0) + if (msgbuf[0] & IXGBE_VT_MSGTYPE_FAILURE && ret >= 0) ret = -1; out: @@ -77,11 +77,11 @@ static int ixgbevf_ipsec_del_pf_sa(struct ixgbevf_adapter *adapter, int pfsa) spin_lock_bh(&adapter->mbx_lock); - err = hw->mbx.ops.write_posted(hw, msgbuf, 2); + err = ixgbevf_write_mbx(hw, msgbuf, 2); if (err) goto out; - err = hw->mbx.ops.read_posted(hw, msgbuf, 2); + err = ixgbevf_poll_mbx(hw, msgbuf, 2); if (err) goto out; @@ -623,6 +623,7 @@ void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter) switch (adapter->hw.api_version) { case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: break; default: return; diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index a0e325774819..e257390a4f6a 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -430,6 +430,7 @@ extern const struct ixgbevf_info ixgbevf_X540_vf_info; extern const struct ixgbevf_info ixgbevf_X550_vf_info; extern const struct ixgbevf_info ixgbevf_X550EM_x_vf_info; extern const struct ixgbe_mbx_operations ixgbevf_mbx_ops; +extern const struct ixgbe_mbx_operations ixgbevf_mbx_ops_legacy; extern const struct ixgbevf_info ixgbevf_x550em_a_vf_info; extern const struct ixgbevf_info ixgbevf_82599_vf_hv_info; @@ -491,4 +492,8 @@ void ixgbe_napi_del_all(struct ixgbevf_adapter *adapter); #define hw_dbg(hw, format, arg...) \ netdev_dbg(ixgbevf_hw_to_netdev(hw), format, ## arg) + +s32 ixgbevf_poll_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size); +s32 ixgbevf_write_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size); + #endif /* _IXGBEVF_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index d81811ab4ec4..b1dfbaff8b31 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -2266,6 +2266,7 @@ static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; static const int api[] = { + ixgbe_mbox_api_15, ixgbe_mbox_api_14, ixgbe_mbox_api_13, ixgbe_mbox_api_12, @@ -2284,6 +2285,12 @@ static void ixgbevf_negotiate_api(struct ixgbevf_adapter *adapter) idx++; } + if (hw->api_version >= ixgbe_mbox_api_15) { + hw->mbx.ops.init_params(hw); + memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, + sizeof(struct ixgbe_mbx_operations)); + } + spin_unlock_bh(&adapter->mbx_lock); } @@ -2627,6 +2634,7 @@ static void ixgbevf_set_num_queues(struct ixgbevf_adapter *adapter) case ixgbe_mbox_api_12: case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: if (adapter->xdp_prog && hw->mac.max_tx_queues == rss) rss = rss > 3 ? 2 : 1; @@ -4565,7 +4573,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) memcpy(&hw->mac.ops, ii->mac_ops, sizeof(hw->mac.ops)); hw->mac.type = ii->mac; - memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops, + memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops_legacy, sizeof(struct ixgbe_mbx_operations)); /* setup the private structure */ @@ -4625,6 +4633,7 @@ static int ixgbevf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) case ixgbe_mbox_api_12: case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: netdev->max_mtu = IXGBE_MAX_JUMBO_FRAME_SIZE - (ETH_HLEN + ETH_FCS_LEN); break; diff --git a/drivers/net/ethernet/intel/ixgbevf/mbx.c b/drivers/net/ethernet/intel/ixgbevf/mbx.c index 6bc1953263b9..a55dd978f7ca 100644 --- a/drivers/net/ethernet/intel/ixgbevf/mbx.c +++ b/drivers/net/ethernet/intel/ixgbevf/mbx.c @@ -15,16 +15,15 @@ static s32 ixgbevf_poll_for_msg(struct ixgbe_hw *hw) struct ixgbe_mbx_info *mbx = &hw->mbx; int countdown = mbx->timeout; + if (!countdown || !mbx->ops.check_for_msg) + return IXGBE_ERR_CONFIG; + while (countdown && mbx->ops.check_for_msg(hw)) { countdown--; udelay(mbx->udelay); } - /* if we failed, all future posted messages fail until reset */ - if (!countdown) - mbx->timeout = 0; - - return countdown ? 0 : IXGBE_ERR_MBX; + return countdown ? 0 : IXGBE_ERR_TIMEOUT; } /** @@ -38,87 +37,82 @@ static s32 ixgbevf_poll_for_ack(struct ixgbe_hw *hw) struct ixgbe_mbx_info *mbx = &hw->mbx; int countdown = mbx->timeout; + if (!countdown || !mbx->ops.check_for_ack) + return IXGBE_ERR_CONFIG; + while (countdown && mbx->ops.check_for_ack(hw)) { countdown--; udelay(mbx->udelay); } - /* if we failed, all future posted messages fail until reset */ - if (!countdown) - mbx->timeout = 0; - - return countdown ? 0 : IXGBE_ERR_MBX; + return countdown ? 0 : IXGBE_ERR_TIMEOUT; } /** - * ixgbevf_read_posted_mbx - Wait for message notification and receive message - * @hw: pointer to the HW structure - * @msg: The message buffer - * @size: Length of buffer + * ixgbevf_read_mailbox_vf - read VF's mailbox register + * @hw: pointer to the HW structure * - * returns 0 if it successfully received a message notification and - * copied it into the receive buffer. + * This function is used to read the mailbox register dedicated for VF without + * losing the read to clear status bits. **/ -static s32 ixgbevf_read_posted_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size) +static u32 ixgbevf_read_mailbox_vf(struct ixgbe_hw *hw) { - struct ixgbe_mbx_info *mbx = &hw->mbx; - s32 ret_val = IXGBE_ERR_MBX; + u32 vf_mailbox = IXGBE_READ_REG(hw, IXGBE_VFMAILBOX); - if (!mbx->ops.read) - goto out; + vf_mailbox |= hw->mbx.vf_mailbox; + hw->mbx.vf_mailbox |= vf_mailbox & IXGBE_VFMAILBOX_R2C_BITS; - ret_val = ixgbevf_poll_for_msg(hw); - - /* if ack received read message, otherwise we timed out */ - if (!ret_val) - ret_val = mbx->ops.read(hw, msg, size); -out: - return ret_val; + return vf_mailbox; } /** - * ixgbevf_write_posted_mbx - Write a message to the mailbox, wait for ack - * @hw: pointer to the HW structure - * @msg: The message buffer - * @size: Length of buffer + * ixgbevf_clear_msg_vf - clear PF status bit + * @hw: pointer to the HW structure * - * returns 0 if it successfully copied message into the buffer and - * received an ack to that message within delay * timeout period + * This function is used to clear PFSTS bit in the VFMAILBOX register **/ -static s32 ixgbevf_write_posted_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size) +static void ixgbevf_clear_msg_vf(struct ixgbe_hw *hw) { - struct ixgbe_mbx_info *mbx = &hw->mbx; - s32 ret_val = IXGBE_ERR_MBX; + u32 vf_mailbox = ixgbevf_read_mailbox_vf(hw); - /* exit if either we can't write or there isn't a defined timeout */ - if (!mbx->ops.write || !mbx->timeout) - goto out; + if (vf_mailbox & IXGBE_VFMAILBOX_PFSTS) { + hw->mbx.stats.reqs++; + hw->mbx.vf_mailbox &= ~IXGBE_VFMAILBOX_PFSTS; + } +} - /* send msg */ - ret_val = mbx->ops.write(hw, msg, size); +/** + * ixgbevf_clear_ack_vf - clear PF ACK bit + * @hw: pointer to the HW structure + * + * This function is used to clear PFACK bit in the VFMAILBOX register + **/ +static void ixgbevf_clear_ack_vf(struct ixgbe_hw *hw) +{ + u32 vf_mailbox = ixgbevf_read_mailbox_vf(hw); - /* if msg sent wait until we receive an ack */ - if (!ret_val) - ret_val = ixgbevf_poll_for_ack(hw); -out: - return ret_val; + if (vf_mailbox & IXGBE_VFMAILBOX_PFACK) { + hw->mbx.stats.acks++; + hw->mbx.vf_mailbox &= ~IXGBE_VFMAILBOX_PFACK; + } } /** - * ixgbevf_read_v2p_mailbox - read v2p mailbox - * @hw: pointer to the HW structure + * ixgbevf_clear_rst_vf - clear PF reset bit + * @hw: pointer to the HW structure * - * This function is used to read the v2p mailbox without losing the read to - * clear status bits. + * This function is used to clear reset indication and reset done bit in + * VFMAILBOX register after reset the shared resources and the reset sequence. **/ -static u32 ixgbevf_read_v2p_mailbox(struct ixgbe_hw *hw) +static void ixgbevf_clear_rst_vf(struct ixgbe_hw *hw) { - u32 v2p_mailbox = IXGBE_READ_REG(hw, IXGBE_VFMAILBOX); - - v2p_mailbox |= hw->mbx.v2p_mailbox; - hw->mbx.v2p_mailbox |= v2p_mailbox & IXGBE_VFMAILBOX_R2C_BITS; + u32 vf_mailbox = ixgbevf_read_mailbox_vf(hw); - return v2p_mailbox; + if (vf_mailbox & (IXGBE_VFMAILBOX_RSTI | IXGBE_VFMAILBOX_RSTD)) { + hw->mbx.stats.rsts++; + hw->mbx.vf_mailbox &= ~(IXGBE_VFMAILBOX_RSTI | + IXGBE_VFMAILBOX_RSTD); + } } /** @@ -131,14 +125,12 @@ static u32 ixgbevf_read_v2p_mailbox(struct ixgbe_hw *hw) **/ static s32 ixgbevf_check_for_bit_vf(struct ixgbe_hw *hw, u32 mask) { - u32 v2p_mailbox = ixgbevf_read_v2p_mailbox(hw); + u32 vf_mailbox = ixgbevf_read_mailbox_vf(hw); s32 ret_val = IXGBE_ERR_MBX; - if (v2p_mailbox & mask) + if (vf_mailbox & mask) ret_val = 0; - hw->mbx.v2p_mailbox &= ~mask; - return ret_val; } @@ -172,6 +164,7 @@ static s32 ixgbevf_check_for_ack_vf(struct ixgbe_hw *hw) if (!ixgbevf_check_for_bit_vf(hw, IXGBE_VFMAILBOX_PFACK)) { ret_val = 0; + ixgbevf_clear_ack_vf(hw); hw->mbx.stats.acks++; } @@ -191,6 +184,7 @@ static s32 ixgbevf_check_for_rst_vf(struct ixgbe_hw *hw) if (!ixgbevf_check_for_bit_vf(hw, (IXGBE_VFMAILBOX_RSTD | IXGBE_VFMAILBOX_RSTI))) { ret_val = 0; + ixgbevf_clear_rst_vf(hw); hw->mbx.stats.rsts++; } @@ -205,19 +199,59 @@ static s32 ixgbevf_check_for_rst_vf(struct ixgbe_hw *hw) **/ static s32 ixgbevf_obtain_mbx_lock_vf(struct ixgbe_hw *hw) { - s32 ret_val = IXGBE_ERR_MBX; + struct ixgbe_mbx_info *mbx = &hw->mbx; + s32 ret_val = IXGBE_ERR_CONFIG; + int countdown = mbx->timeout; + u32 vf_mailbox; - /* Take ownership of the buffer */ - IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, IXGBE_VFMAILBOX_VFU); + if (!mbx->timeout) + return ret_val; - /* reserve mailbox for VF use */ - if (ixgbevf_read_v2p_mailbox(hw) & IXGBE_VFMAILBOX_VFU) - ret_val = 0; + while (countdown--) { + /* Reserve mailbox for VF use */ + vf_mailbox = ixgbevf_read_mailbox_vf(hw); + vf_mailbox |= IXGBE_VFMAILBOX_VFU; + IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, vf_mailbox); + + /* Verify that VF is the owner of the lock */ + if (ixgbevf_read_mailbox_vf(hw) & IXGBE_VFMAILBOX_VFU) { + ret_val = 0; + break; + } + + /* Wait a bit before trying again */ + udelay(mbx->udelay); + } + + if (ret_val) + ret_val = IXGBE_ERR_TIMEOUT; return ret_val; } /** + * ixgbevf_release_mbx_lock_vf - release mailbox lock + * @hw: pointer to the HW structure + **/ +static void ixgbevf_release_mbx_lock_vf(struct ixgbe_hw *hw) +{ + u32 vf_mailbox; + + /* Return ownership of the buffer */ + vf_mailbox = ixgbevf_read_mailbox_vf(hw); + vf_mailbox &= ~IXGBE_VFMAILBOX_VFU; + IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, vf_mailbox); +} + +/** + * ixgbevf_release_mbx_lock_vf_legacy - release mailbox lock + * @hw: pointer to the HW structure + **/ +static void ixgbevf_release_mbx_lock_vf_legacy(struct ixgbe_hw *__always_unused hw) +{ +} + +/** * ixgbevf_write_mbx_vf - Write a message to the mailbox * @hw: pointer to the HW structure * @msg: The message buffer @@ -227,6 +261,50 @@ static s32 ixgbevf_obtain_mbx_lock_vf(struct ixgbe_hw *hw) **/ static s32 ixgbevf_write_mbx_vf(struct ixgbe_hw *hw, u32 *msg, u16 size) { + u32 vf_mailbox; + s32 ret_val; + u16 i; + + /* lock the mailbox to prevent PF/VF race condition */ + ret_val = ixgbevf_obtain_mbx_lock_vf(hw); + if (ret_val) + goto out_no_write; + + /* flush msg and acks as we are overwriting the message buffer */ + ixgbevf_clear_msg_vf(hw); + ixgbevf_clear_ack_vf(hw); + + /* copy the caller specified message to the mailbox memory buffer */ + for (i = 0; i < size; i++) + IXGBE_WRITE_REG_ARRAY(hw, IXGBE_VFMBMEM, i, msg[i]); + + /* update stats */ + hw->mbx.stats.msgs_tx++; + + /* interrupt the PF to tell it a message has been sent */ + vf_mailbox = ixgbevf_read_mailbox_vf(hw); + vf_mailbox |= IXGBE_VFMAILBOX_REQ; + IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, vf_mailbox); + + /* if msg sent wait until we receive an ack */ + ret_val = ixgbevf_poll_for_ack(hw); + +out_no_write: + hw->mbx.ops.release(hw); + + return ret_val; +} + +/** + * ixgbevf_write_mbx_vf_legacy - Write a message to the mailbox + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully copied message into the buffer + **/ +static s32 ixgbevf_write_mbx_vf_legacy(struct ixgbe_hw *hw, u32 *msg, u16 size) +{ s32 ret_val; u16 i; @@ -237,7 +315,9 @@ static s32 ixgbevf_write_mbx_vf(struct ixgbe_hw *hw, u32 *msg, u16 size) /* flush msg and acks as we are overwriting the message buffer */ ixgbevf_check_for_msg_vf(hw); + ixgbevf_clear_msg_vf(hw); ixgbevf_check_for_ack_vf(hw); + ixgbevf_clear_ack_vf(hw); /* copy the caller specified message to the mailbox memory buffer */ for (i = 0; i < size; i++) @@ -263,6 +343,42 @@ out_no_write: **/ static s32 ixgbevf_read_mbx_vf(struct ixgbe_hw *hw, u32 *msg, u16 size) { + u32 vf_mailbox; + s32 ret_val; + u16 i; + + /* check if there is a message from PF */ + ret_val = ixgbevf_check_for_msg_vf(hw); + if (ret_val) + return ret_val; + + ixgbevf_clear_msg_vf(hw); + + /* copy the message from the mailbox memory buffer */ + for (i = 0; i < size; i++) + msg[i] = IXGBE_READ_REG_ARRAY(hw, IXGBE_VFMBMEM, i); + + /* Acknowledge receipt */ + vf_mailbox = ixgbevf_read_mailbox_vf(hw); + vf_mailbox |= IXGBE_VFMAILBOX_ACK; + IXGBE_WRITE_REG(hw, IXGBE_VFMAILBOX, vf_mailbox); + + /* update stats */ + hw->mbx.stats.msgs_rx++; + + return ret_val; +} + +/** + * ixgbevf_read_mbx_vf_legacy - Reads a message from the inbox intended for VF + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully read message from buffer + **/ +static s32 ixgbevf_read_mbx_vf_legacy(struct ixgbe_hw *hw, u32 *msg, u16 size) +{ s32 ret_val = 0; u16 i; @@ -298,7 +414,7 @@ static s32 ixgbevf_init_mbx_params_vf(struct ixgbe_hw *hw) /* start mailbox as timed out and let the reset_hw call set the timeout * value to begin communications */ - mbx->timeout = 0; + mbx->timeout = IXGBE_VF_MBX_INIT_TIMEOUT; mbx->udelay = IXGBE_VF_MBX_INIT_DELAY; mbx->size = IXGBE_VFMAILBOX_SIZE; @@ -312,12 +428,79 @@ static s32 ixgbevf_init_mbx_params_vf(struct ixgbe_hw *hw) return 0; } +/** + * ixgbevf_poll_mbx - Wait for message and read it from the mailbox + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully read message from buffer + **/ +s32 ixgbevf_poll_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size) +{ + struct ixgbe_mbx_info *mbx = &hw->mbx; + s32 ret_val = IXGBE_ERR_CONFIG; + + if (!mbx->ops.read || !mbx->ops.check_for_msg || !mbx->timeout) + return ret_val; + + /* limit read to size of mailbox */ + if (size > mbx->size) + size = mbx->size; + + ret_val = ixgbevf_poll_for_msg(hw); + /* if ack received read message, otherwise we timed out */ + if (!ret_val) + ret_val = mbx->ops.read(hw, msg, size); + + return ret_val; +} + +/** + * ixgbevf_write_mbx - Write a message to the mailbox and wait for ACK + * @hw: pointer to the HW structure + * @msg: The message buffer + * @size: Length of buffer + * + * returns 0 if it successfully copied message into the buffer and + * received an ACK to that message within specified period + **/ +s32 ixgbevf_write_mbx(struct ixgbe_hw *hw, u32 *msg, u16 size) +{ + struct ixgbe_mbx_info *mbx = &hw->mbx; + s32 ret_val = IXGBE_ERR_CONFIG; + + /** + * exit if either we can't write, release + * or there is no timeout defined + */ + if (!mbx->ops.write || !mbx->ops.check_for_ack || !mbx->ops.release || + !mbx->timeout) + return ret_val; + + if (size > mbx->size) + ret_val = IXGBE_ERR_PARAM; + else + ret_val = mbx->ops.write(hw, msg, size); + + return ret_val; +} + const struct ixgbe_mbx_operations ixgbevf_mbx_ops = { .init_params = ixgbevf_init_mbx_params_vf, + .release = ixgbevf_release_mbx_lock_vf, .read = ixgbevf_read_mbx_vf, .write = ixgbevf_write_mbx_vf, - .read_posted = ixgbevf_read_posted_mbx, - .write_posted = ixgbevf_write_posted_mbx, + .check_for_msg = ixgbevf_check_for_msg_vf, + .check_for_ack = ixgbevf_check_for_ack_vf, + .check_for_rst = ixgbevf_check_for_rst_vf, +}; + +const struct ixgbe_mbx_operations ixgbevf_mbx_ops_legacy = { + .init_params = ixgbevf_init_mbx_params_vf, + .release = ixgbevf_release_mbx_lock_vf_legacy, + .read = ixgbevf_read_mbx_vf_legacy, + .write = ixgbevf_write_mbx_vf_legacy, .check_for_msg = ixgbevf_check_for_msg_vf, .check_for_ack = ixgbevf_check_for_ack_vf, .check_for_rst = ixgbevf_check_for_rst_vf, diff --git a/drivers/net/ethernet/intel/ixgbevf/mbx.h b/drivers/net/ethernet/intel/ixgbevf/mbx.h index 853796c8ef0e..7346ccf014a5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/mbx.h +++ b/drivers/net/ethernet/intel/ixgbevf/mbx.h @@ -7,7 +7,6 @@ #include "vf.h" #define IXGBE_VFMAILBOX_SIZE 16 /* 16 32 bit words - 64 bytes */ -#define IXGBE_ERR_MBX -100 #define IXGBE_VFMAILBOX 0x002FC #define IXGBE_VFMBMEM 0x00200 @@ -39,14 +38,17 @@ /* If it's a IXGBE_VF_* msg then it originates in the VF and is sent to the * PF. The reverse is true if it is IXGBE_PF_*. - * Message ACK's are the value or'd with 0xF0000000 + * Message results are the value or'd with 0xF0000000 */ -/* Messages below or'd with this are the ACK */ -#define IXGBE_VT_MSGTYPE_ACK 0x80000000 -/* Messages below or'd with this are the NACK */ -#define IXGBE_VT_MSGTYPE_NACK 0x40000000 -/* Indicates that VF is still clear to send requests */ -#define IXGBE_VT_MSGTYPE_CTS 0x20000000 +#define IXGBE_VT_MSGTYPE_SUCCESS 0x80000000 /* Messages or'd with this + * have succeeded + */ +#define IXGBE_VT_MSGTYPE_FAILURE 0x40000000 /* Messages or'd with this + * have failed + */ +#define IXGBE_VT_MSGTYPE_CTS 0x20000000 /* Indicates that VF is still + * clear to send requests + */ #define IXGBE_VT_MSGINFO_SHIFT 16 /* bits 23:16 are used for exra info for certain messages */ #define IXGBE_VT_MSGINFO_MASK (0xFF << IXGBE_VT_MSGINFO_SHIFT) @@ -63,6 +65,7 @@ enum ixgbe_pfvf_api_rev { ixgbe_mbox_api_12, /* API version 1.2, linux/freebsd VF driver */ ixgbe_mbox_api_13, /* API version 1.3, linux/freebsd VF driver */ ixgbe_mbox_api_14, /* API version 1.4, linux/freebsd VF driver */ + ixgbe_mbox_api_15, /* API version 1.5, linux/freebsd VF driver */ /* This value should always be last */ ixgbe_mbox_api_unknown, /* indicates that API version is not known */ }; diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.c b/drivers/net/ethernet/intel/ixgbevf/vf.c index d459f5c8e98f..61d8970c6d1d 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.c +++ b/drivers/net/ethernet/intel/ixgbevf/vf.c @@ -13,13 +13,12 @@ static inline s32 ixgbevf_write_msg_read_ack(struct ixgbe_hw *hw, u32 *msg, u32 *retmsg, u16 size) { - struct ixgbe_mbx_info *mbx = &hw->mbx; - s32 retval = mbx->ops.write_posted(hw, msg, size); + s32 retval = ixgbevf_write_mbx(hw, msg, size); if (retval) return retval; - return mbx->ops.read_posted(hw, retmsg, size); + return ixgbevf_poll_mbx(hw, retmsg, size); } /** @@ -75,6 +74,9 @@ static s32 ixgbevf_reset_hw_vf(struct ixgbe_hw *hw) /* reset the api version */ hw->api_version = ixgbe_mbox_api_10; + hw->mbx.ops.init_params(hw); + memcpy(&hw->mbx.ops, &ixgbevf_mbx_ops_legacy, + sizeof(struct ixgbe_mbx_operations)); IXGBE_WRITE_REG(hw, IXGBE_VFCTRL, IXGBE_CTRL_RST); IXGBE_WRITE_FLUSH(hw); @@ -92,7 +94,7 @@ static s32 ixgbevf_reset_hw_vf(struct ixgbe_hw *hw) mbx->timeout = IXGBE_VF_MBX_INIT_TIMEOUT; msgbuf[0] = IXGBE_VF_RESET; - mbx->ops.write_posted(hw, msgbuf, 1); + ixgbevf_write_mbx(hw, msgbuf, 1); mdelay(10); @@ -100,7 +102,7 @@ static s32 ixgbevf_reset_hw_vf(struct ixgbe_hw *hw) * also set up the mc_filter_type which is piggy backed * on the mac address in word 3 */ - ret_val = mbx->ops.read_posted(hw, msgbuf, IXGBE_VF_PERMADDR_MSG_LEN); + ret_val = ixgbevf_poll_mbx(hw, msgbuf, IXGBE_VF_PERMADDR_MSG_LEN); if (ret_val) return ret_val; @@ -108,11 +110,11 @@ static s32 ixgbevf_reset_hw_vf(struct ixgbe_hw *hw) * to indicate that no MAC address has yet been assigned for * the VF. */ - if (msgbuf[0] != (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_ACK) && - msgbuf[0] != (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_NACK)) + if (msgbuf[0] != (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_SUCCESS) && + msgbuf[0] != (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_FAILURE)) return IXGBE_ERR_INVALID_MAC_ADDR; - if (msgbuf[0] == (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_ACK)) + if (msgbuf[0] == (IXGBE_VF_RESET | IXGBE_VT_MSGTYPE_SUCCESS)) ether_addr_copy(hw->mac.perm_addr, addr); hw->mac.mc_filter_type = msgbuf[IXGBE_VF_MC_TYPE_WORD]; @@ -269,7 +271,7 @@ static s32 ixgbevf_set_uc_addr_vf(struct ixgbe_hw *hw, u32 index, u8 *addr) if (!ret_val) { msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; - if (msgbuf[0] == (msgbuf_chk | IXGBE_VT_MSGTYPE_NACK)) + if (msgbuf[0] == (msgbuf_chk | IXGBE_VT_MSGTYPE_FAILURE)) return -ENOMEM; } @@ -311,6 +313,7 @@ int ixgbevf_get_reta_locked(struct ixgbe_hw *hw, u32 *reta, int num_rx_queues) * is not supported for this device type. */ switch (hw->api_version) { + case ixgbe_mbox_api_15: case ixgbe_mbox_api_14: case ixgbe_mbox_api_13: case ixgbe_mbox_api_12: @@ -323,12 +326,12 @@ int ixgbevf_get_reta_locked(struct ixgbe_hw *hw, u32 *reta, int num_rx_queues) msgbuf[0] = IXGBE_VF_GET_RETA; - err = hw->mbx.ops.write_posted(hw, msgbuf, 1); + err = ixgbevf_write_mbx(hw, msgbuf, 1); if (err) return err; - err = hw->mbx.ops.read_posted(hw, msgbuf, dwords + 1); + err = ixgbevf_poll_mbx(hw, msgbuf, dwords + 1); if (err) return err; @@ -336,14 +339,14 @@ int ixgbevf_get_reta_locked(struct ixgbe_hw *hw, u32 *reta, int num_rx_queues) msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; /* If the operation has been refused by a PF return -EPERM */ - if (msgbuf[0] == (IXGBE_VF_GET_RETA | IXGBE_VT_MSGTYPE_NACK)) + if (msgbuf[0] == (IXGBE_VF_GET_RETA | IXGBE_VT_MSGTYPE_FAILURE)) return -EPERM; /* If we didn't get an ACK there must have been * some sort of mailbox error so we should treat it * as such. */ - if (msgbuf[0] != (IXGBE_VF_GET_RETA | IXGBE_VT_MSGTYPE_ACK)) + if (msgbuf[0] != (IXGBE_VF_GET_RETA | IXGBE_VT_MSGTYPE_SUCCESS)) return IXGBE_ERR_MBX; /* ixgbevf doesn't support more than 2 queues at the moment */ @@ -379,6 +382,7 @@ int ixgbevf_get_rss_key_locked(struct ixgbe_hw *hw, u8 *rss_key) * or if the operation is not supported for this device type. */ switch (hw->api_version) { + case ixgbe_mbox_api_15: case ixgbe_mbox_api_14: case ixgbe_mbox_api_13: case ixgbe_mbox_api_12: @@ -390,12 +394,12 @@ int ixgbevf_get_rss_key_locked(struct ixgbe_hw *hw, u8 *rss_key) } msgbuf[0] = IXGBE_VF_GET_RSS_KEY; - err = hw->mbx.ops.write_posted(hw, msgbuf, 1); + err = ixgbevf_write_mbx(hw, msgbuf, 1); if (err) return err; - err = hw->mbx.ops.read_posted(hw, msgbuf, 11); + err = ixgbevf_poll_mbx(hw, msgbuf, 11); if (err) return err; @@ -403,14 +407,14 @@ int ixgbevf_get_rss_key_locked(struct ixgbe_hw *hw, u8 *rss_key) msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; /* If the operation has been refused by a PF return -EPERM */ - if (msgbuf[0] == (IXGBE_VF_GET_RSS_KEY | IXGBE_VT_MSGTYPE_NACK)) + if (msgbuf[0] == (IXGBE_VF_GET_RSS_KEY | IXGBE_VT_MSGTYPE_FAILURE)) return -EPERM; /* If we didn't get an ACK there must have been * some sort of mailbox error so we should treat it * as such. */ - if (msgbuf[0] != (IXGBE_VF_GET_RSS_KEY | IXGBE_VT_MSGTYPE_ACK)) + if (msgbuf[0] != (IXGBE_VF_GET_RSS_KEY | IXGBE_VT_MSGTYPE_SUCCESS)) return IXGBE_ERR_MBX; memcpy(rss_key, msgbuf + 1, IXGBEVF_RSS_HASH_KEY_SIZE); @@ -442,7 +446,7 @@ static s32 ixgbevf_set_rar_vf(struct ixgbe_hw *hw, u32 index, u8 *addr, /* if nacked the address was rejected, use "perm_addr" */ if (!ret_val && - (msgbuf[0] == (IXGBE_VF_SET_MAC_ADDR | IXGBE_VT_MSGTYPE_NACK))) { + (msgbuf[0] == (IXGBE_VF_SET_MAC_ADDR | IXGBE_VT_MSGTYPE_FAILURE))) { ixgbevf_get_mac_addr_vf(hw, hw->mac.addr); return IXGBE_ERR_MBX; } @@ -545,8 +549,9 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) if (xcast_mode == IXGBEVF_XCAST_MODE_PROMISC) return -EOPNOTSUPP; fallthrough; - case ixgbe_mbox_api_14: case ixgbe_mbox_api_13: + case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: break; default: return -EOPNOTSUPP; @@ -561,7 +566,7 @@ static s32 ixgbevf_update_xcast_mode(struct ixgbe_hw *hw, int xcast_mode) return err; msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; - if (msgbuf[0] == (IXGBE_VF_UPDATE_XCAST_MODE | IXGBE_VT_MSGTYPE_NACK)) + if (msgbuf[0] == (IXGBE_VF_UPDATE_XCAST_MODE | IXGBE_VT_MSGTYPE_FAILURE)) return -EPERM; return 0; @@ -606,7 +611,7 @@ static s32 ixgbevf_set_vfta_vf(struct ixgbe_hw *hw, u32 vlan, u32 vind, msgbuf[0] &= ~IXGBE_VT_MSGTYPE_CTS; msgbuf[0] &= ~(0xFF << IXGBE_VT_MSGINFO_SHIFT); - if (msgbuf[0] != (IXGBE_VF_SET_VLAN | IXGBE_VT_MSGTYPE_ACK)) + if (msgbuf[0] != (IXGBE_VF_SET_VLAN | IXGBE_VT_MSGTYPE_SUCCESS)) err = IXGBE_ERR_INVALID_ARGUMENT; mbx_err: @@ -705,12 +710,15 @@ static s32 ixgbevf_check_mac_link_vf(struct ixgbe_hw *hw, /* if the read failed it could just be a mailbox collision, best wait * until we are called again and don't report an error */ - if (mbx->ops.read(hw, &in_msg, 1)) + if (mbx->ops.read(hw, &in_msg, 1)) { + if (hw->api_version >= ixgbe_mbox_api_15) + mac->get_link_status = false; goto out; + } if (!(in_msg & IXGBE_VT_MSGTYPE_CTS)) { /* msg is not CTS and is NACK we must have lost CTS status */ - if (in_msg & IXGBE_VT_MSGTYPE_NACK) + if (in_msg & IXGBE_VT_MSGTYPE_FAILURE) ret_val = -1; goto out; } @@ -816,7 +824,7 @@ static s32 ixgbevf_set_rlpml_vf(struct ixgbe_hw *hw, u16 max_size) if (ret_val) return ret_val; if ((msgbuf[0] & IXGBE_VF_SET_LPE) && - (msgbuf[0] & IXGBE_VT_MSGTYPE_NACK)) + (msgbuf[0] & IXGBE_VT_MSGTYPE_FAILURE)) return IXGBE_ERR_MBX; return 0; @@ -863,7 +871,8 @@ static int ixgbevf_negotiate_api_version_vf(struct ixgbe_hw *hw, int api) msg[0] &= ~IXGBE_VT_MSGTYPE_CTS; /* Store value and return 0 on success */ - if (msg[0] == (IXGBE_VF_API_NEGOTIATE | IXGBE_VT_MSGTYPE_ACK)) { + if (msg[0] == (IXGBE_VF_API_NEGOTIATE | + IXGBE_VT_MSGTYPE_SUCCESS)) { hw->api_version = api; return 0; } @@ -901,6 +910,7 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, case ixgbe_mbox_api_12: case ixgbe_mbox_api_13: case ixgbe_mbox_api_14: + case ixgbe_mbox_api_15: break; default: return 0; @@ -918,7 +928,7 @@ int ixgbevf_get_queues(struct ixgbe_hw *hw, unsigned int *num_tcs, * some sort of mailbox error so we should treat it * as such */ - if (msg[0] != (IXGBE_VF_GET_QUEUE | IXGBE_VT_MSGTYPE_ACK)) + if (msg[0] != (IXGBE_VF_GET_QUEUE | IXGBE_VT_MSGTYPE_SUCCESS)) return IXGBE_ERR_MBX; /* record and validate values from message */ diff --git a/drivers/net/ethernet/intel/ixgbevf/vf.h b/drivers/net/ethernet/intel/ixgbevf/vf.h index 1d8209df4162..54158dac8707 100644 --- a/drivers/net/ethernet/intel/ixgbevf/vf.h +++ b/drivers/net/ethernet/intel/ixgbevf/vf.h @@ -73,10 +73,9 @@ struct ixgbe_mac_info { struct ixgbe_mbx_operations { s32 (*init_params)(struct ixgbe_hw *hw); + void (*release)(struct ixgbe_hw *hw); s32 (*read)(struct ixgbe_hw *, u32 *, u16); s32 (*write)(struct ixgbe_hw *, u32 *, u16); - s32 (*read_posted)(struct ixgbe_hw *, u32 *, u16); - s32 (*write_posted)(struct ixgbe_hw *, u32 *, u16); s32 (*check_for_msg)(struct ixgbe_hw *); s32 (*check_for_ack)(struct ixgbe_hw *); s32 (*check_for_rst)(struct ixgbe_hw *); @@ -96,7 +95,7 @@ struct ixgbe_mbx_info { struct ixgbe_mbx_stats stats; u32 timeout; u32 udelay; - u32 v2p_mailbox; + u32 vf_mailbox; u16 size; }; diff --git a/drivers/net/ethernet/lantiq_etop.c b/drivers/net/ethernet/lantiq_etop.c index 072391c494ce..14059e11710a 100644 --- a/drivers/net/ethernet/lantiq_etop.c +++ b/drivers/net/ethernet/lantiq_etop.c @@ -687,13 +687,13 @@ ltq_etop_probe(struct platform_device *pdev) err = device_property_read_u32(&pdev->dev, "lantiq,tx-burst-length", &priv->tx_burst_len); if (err < 0) { dev_err(&pdev->dev, "unable to read tx-burst-length property\n"); - return err; + goto err_free; } err = device_property_read_u32(&pdev->dev, "lantiq,rx-burst-length", &priv->rx_burst_len); if (err < 0) { dev_err(&pdev->dev, "unable to read rx-burst-length property\n"); - return err; + goto err_free; } for (i = 0; i < MAX_DMA_CHAN; i++) { diff --git a/drivers/net/ethernet/marvell/mv643xx_eth.c b/drivers/net/ethernet/marvell/mv643xx_eth.c index bb14fa2241a3..105247582684 100644 --- a/drivers/net/ethernet/marvell/mv643xx_eth.c +++ b/drivers/net/ethernet/marvell/mv643xx_eth.c @@ -1638,7 +1638,9 @@ static int mv643xx_eth_set_coalesce(struct net_device *dev, } static void -mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); @@ -1650,7 +1652,9 @@ mv643xx_eth_get_ringparam(struct net_device *dev, struct ethtool_ringparam *er) } static int -mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er) +mv643xx_eth_set_ringparam(struct net_device *dev, struct ethtool_ringparam *er, + struct kernel_ethtool_ringparam *kernel_er, + struct netlink_ext_ack *extack) { struct mv643xx_eth_private *mp = netdev_priv(dev); @@ -3197,7 +3201,7 @@ static int mv643xx_eth_probe(struct platform_device *pdev) dev->hw_features = dev->features; dev->priv_flags |= IFF_UNICAST_FLT; - dev->gso_max_segs = MV643XX_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MV643XX_MAX_TSO_SEGS); /* MTU range: 64 - 9500 */ dev->min_mtu = 64; diff --git a/drivers/net/ethernet/marvell/mvneta.c b/drivers/net/ethernet/marvell/mvneta.c index 5a7bdca22a63..ce810fc3c1a2 100644 --- a/drivers/net/ethernet/marvell/mvneta.c +++ b/drivers/net/ethernet/marvell/mvneta.c @@ -38,6 +38,7 @@ #include <net/ipv6.h> #include <net/tso.h> #include <net/page_pool.h> +#include <net/pkt_cls.h> #include <linux/bpf_trace.h> /* Registers */ @@ -247,12 +248,39 @@ #define MVNETA_TXQ_SENT_DESC_MASK 0x3fff0000 #define MVNETA_PORT_TX_RESET 0x3cf0 #define MVNETA_PORT_TX_DMA_RESET BIT(0) +#define MVNETA_TXQ_CMD1_REG 0x3e00 +#define MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 BIT(3) +#define MVNETA_TXQ_CMD1_BW_LIM_EN BIT(0) +#define MVNETA_REFILL_NUM_CLK_REG 0x3e08 +#define MVNETA_REFILL_MAX_NUM_CLK 0x0000ffff #define MVNETA_TX_MTU 0x3e0c #define MVNETA_TX_TOKEN_SIZE 0x3e14 #define MVNETA_TX_TOKEN_SIZE_MAX 0xffffffff +#define MVNETA_TXQ_BUCKET_REFILL_REG(q) (0x3e20 + ((q) << 2)) +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD_MASK 0x3ff00000 +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT 20 +#define MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX 0x0007ffff #define MVNETA_TXQ_TOKEN_SIZE_REG(q) (0x3e40 + ((q) << 2)) #define MVNETA_TXQ_TOKEN_SIZE_MAX 0x7fffffff +/* The values of the bucket refill base period and refill period are taken from + * the reference manual, and adds up to a base resolution of 10Kbps. This allows + * to cover all rate-limit values from 10Kbps up to 5Gbps + */ + +/* Base period for the rate limit algorithm */ +#define MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS 100 + +/* Number of Base Period to wait between each bucket refill */ +#define MVNETA_TXQ_BUCKET_REFILL_PERIOD 1000 + +/* The base resolution for rate limiting, in bps. Any max_rate value should be + * a multiple of that value. + */ +#define MVNETA_TXQ_RATE_LIMIT_RESOLUTION (NSEC_PER_SEC / \ + (MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS * \ + MVNETA_TXQ_BUCKET_REFILL_PERIOD)) + #define MVNETA_LPI_CTRL_0 0x2cc0 #define MVNETA_LPI_CTRL_1 0x2cc4 #define MVNETA_LPI_REQUEST_ENABLE BIT(0) @@ -492,7 +520,6 @@ struct mvneta_port { u8 mcast_count[256]; u16 tx_ring_size; u16 rx_ring_size; - u8 prio_tc_map[8]; phy_interface_t phy_interface; struct device_node *dn; @@ -3823,8 +3850,6 @@ static void mvneta_validate(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state) { - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - /* We only support QSGMII, SGMII, 802.3z and RGMII modes. * When in 802.3z mode, we must have AN enabled: * "Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... @@ -3836,34 +3861,7 @@ static void mvneta_validate(struct phylink_config *config, return; } - /* Allow all the expected bits */ - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - - /* Asymmetric pause is unsupported */ - phylink_set(mask, Pause); - - /* Half-duplex at speeds higher than 100Mbit is unsupported */ - if (state->interface != PHY_INTERFACE_MODE_2500BASEX) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - } - - if (state->interface == PHY_INTERFACE_MODE_2500BASEX) { - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - } - - if (!phy_interface_mode_is_8023z(state->interface)) { - /* 10M and 100M are only supported in non-802.3z mode */ - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); + phylink_generic_validate(config, supported, state); } static void mvneta_mac_pcs_get_state(struct phylink_config *config, @@ -3950,7 +3948,7 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { /* SGMII mode receives the state from the PHY */ new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE; - new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; + new_clk = MVNETA_GMAC_1MS_CLOCK_ENABLE; new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | MVNETA_GMAC_FORCE_LINK_PASS | MVNETA_GMAC_CONFIG_MII_SPEED | @@ -3962,7 +3960,7 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, } else { /* 802.3z negotiation - only 1000base-X */ new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X; - new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; + new_clk = MVNETA_GMAC_1MS_CLOCK_ENABLE; new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | MVNETA_GMAC_FORCE_LINK_PASS | MVNETA_GMAC_CONFIG_MII_SPEED)) | @@ -3975,6 +3973,10 @@ static void mvneta_mac_config(struct phylink_config *config, unsigned int mode, new_an |= MVNETA_GMAC_AN_FLOW_CTRL_EN; } + /* Set the 1ms clock divisor */ + if (new_clk == MVNETA_GMAC_1MS_CLOCK_ENABLE) + new_clk |= clk_get_rate(pp->clk) / 1000; + /* Armada 370 documentation says we can only change the port mode * and in-band enable when the link is down, so force it down * while making these changes. We also do this for GMAC_CTRL2 @@ -4539,8 +4541,11 @@ static void mvneta_ethtool_get_drvinfo(struct net_device *dev, } -static void mvneta_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) +static void +mvneta_ethtool_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(netdev); @@ -4550,8 +4555,11 @@ static void mvneta_ethtool_get_ringparam(struct net_device *netdev, ring->tx_pending = pp->tx_ring_size; } -static int mvneta_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvneta_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvneta_port *pp = netdev_priv(dev); @@ -4919,43 +4927,144 @@ static void mvneta_clear_rx_prio_map(struct mvneta_port *pp) mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, 0); } -static void mvneta_setup_rx_prio_map(struct mvneta_port *pp) +static void mvneta_map_vlan_prio_to_rxq(struct mvneta_port *pp, u8 pri, u8 rxq) { - u32 val = 0; - int i; + u32 val = mvreg_read(pp, MVNETA_VLAN_PRIO_TO_RXQ); - for (i = 0; i < rxq_number; i++) - val |= MVNETA_VLAN_PRIO_RXQ_MAP(i, pp->prio_tc_map[i]); + val &= ~MVNETA_VLAN_PRIO_RXQ_MAP(pri, 0x7); + val |= MVNETA_VLAN_PRIO_RXQ_MAP(pri, rxq); mvreg_write(pp, MVNETA_VLAN_PRIO_TO_RXQ, val); } +static int mvneta_enable_per_queue_rate_limit(struct mvneta_port *pp) +{ + unsigned long core_clk_rate; + u32 refill_cycles; + u32 val; + + core_clk_rate = clk_get_rate(pp->clk); + if (!core_clk_rate) + return -EINVAL; + + refill_cycles = MVNETA_TXQ_BUCKET_REFILL_BASE_PERIOD_NS / + (NSEC_PER_SEC / core_clk_rate); + + if (refill_cycles > MVNETA_REFILL_MAX_NUM_CLK) + return -EINVAL; + + /* Enable bw limit algorithm version 3 */ + val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG); + val &= ~(MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN); + mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val); + + /* Set the base refill rate */ + mvreg_write(pp, MVNETA_REFILL_NUM_CLK_REG, refill_cycles); + + return 0; +} + +static void mvneta_disable_per_queue_rate_limit(struct mvneta_port *pp) +{ + u32 val = mvreg_read(pp, MVNETA_TXQ_CMD1_REG); + + val |= (MVNETA_TXQ_CMD1_BW_LIM_SEL_V1 | MVNETA_TXQ_CMD1_BW_LIM_EN); + mvreg_write(pp, MVNETA_TXQ_CMD1_REG, val); +} + +static int mvneta_setup_queue_rates(struct mvneta_port *pp, int queue, + u64 min_rate, u64 max_rate) +{ + u32 refill_val, rem; + u32 val = 0; + + /* Convert to from Bps to bps */ + max_rate *= 8; + + if (min_rate) + return -EINVAL; + + refill_val = div_u64_rem(max_rate, MVNETA_TXQ_RATE_LIMIT_RESOLUTION, + &rem); + + if (rem || !refill_val || + refill_val > MVNETA_TXQ_BUCKET_REFILL_VALUE_MAX) + return -EINVAL; + + val = refill_val; + val |= (MVNETA_TXQ_BUCKET_REFILL_PERIOD << + MVNETA_TXQ_BUCKET_REFILL_PERIOD_SHIFT); + + mvreg_write(pp, MVNETA_TXQ_BUCKET_REFILL_REG(queue), val); + + return 0; +} + static int mvneta_setup_mqprio(struct net_device *dev, - struct tc_mqprio_qopt *qopt) + struct tc_mqprio_qopt_offload *mqprio) { struct mvneta_port *pp = netdev_priv(dev); + int rxq, txq, tc, ret; u8 num_tc; - int i; - qopt->hw = TC_MQPRIO_HW_OFFLOAD_TCS; - num_tc = qopt->num_tc; + if (mqprio->qopt.hw != TC_MQPRIO_HW_OFFLOAD_TCS) + return 0; + + num_tc = mqprio->qopt.num_tc; if (num_tc > rxq_number) return -EINVAL; + mvneta_clear_rx_prio_map(pp); + if (!num_tc) { - mvneta_clear_rx_prio_map(pp); + mvneta_disable_per_queue_rate_limit(pp); netdev_reset_tc(dev); return 0; } - memcpy(pp->prio_tc_map, qopt->prio_tc_map, sizeof(pp->prio_tc_map)); + netdev_set_num_tc(dev, mqprio->qopt.num_tc); + + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + netdev_set_tc_queue(dev, tc, mqprio->qopt.count[tc], + mqprio->qopt.offset[tc]); - mvneta_setup_rx_prio_map(pp); + for (rxq = mqprio->qopt.offset[tc]; + rxq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc]; + rxq++) { + if (rxq >= rxq_number) + return -EINVAL; - netdev_set_num_tc(dev, qopt->num_tc); - for (i = 0; i < qopt->num_tc; i++) - netdev_set_tc_queue(dev, i, qopt->count[i], qopt->offset[i]); + mvneta_map_vlan_prio_to_rxq(pp, tc, rxq); + } + } + + if (mqprio->shaper != TC_MQPRIO_SHAPER_BW_RATE) { + mvneta_disable_per_queue_rate_limit(pp); + return 0; + } + + if (mqprio->qopt.num_tc > txq_number) + return -EINVAL; + + ret = mvneta_enable_per_queue_rate_limit(pp); + if (ret) + return ret; + + for (tc = 0; tc < mqprio->qopt.num_tc; tc++) { + for (txq = mqprio->qopt.offset[tc]; + txq < mqprio->qopt.count[tc] + mqprio->qopt.offset[tc]; + txq++) { + if (txq >= txq_number) + return -EINVAL; + + ret = mvneta_setup_queue_rates(pp, txq, + mqprio->min_rate[tc], + mqprio->max_rate[tc]); + if (ret) + return ret; + } + } return 0; } @@ -5166,6 +5275,9 @@ static int mvneta_probe(struct platform_device *pdev) pp->phylink_config.dev = &dev->dev; pp->phylink_config.type = PHYLINK_NETDEV; + pp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_10 | + MAC_100 | MAC_1000FD | MAC_2500FD; + phy_interface_set_rgmii(pp->phylink_config.supported_interfaces); __set_bit(PHY_INTERFACE_MODE_QSGMII, pp->phylink_config.supported_interfaces); @@ -5355,7 +5467,7 @@ static int mvneta_probe(struct platform_device *pdev) dev->hw_features |= dev->features; dev->vlan_features |= dev->features; dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; - dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MVNETA_MAX_TSO_SEGS); /* MTU range: 68 - 9676 */ dev->min_mtu = ETH_MIN_MTU; diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c index 6da8a595026b..8e5820d12362 100644 --- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c +++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c @@ -1488,6 +1488,7 @@ static bool mvpp2_port_supports_rgmii(struct mvpp2_port *port) static bool mvpp2_is_xlg(phy_interface_t interface) { return interface == PHY_INTERFACE_MODE_10GBASER || + interface == PHY_INTERFACE_MODE_5GBASER || interface == PHY_INTERFACE_MODE_XAUI; } @@ -1627,6 +1628,7 @@ static int mvpp22_gop_init(struct mvpp2_port *port, phy_interface_t interface) case PHY_INTERFACE_MODE_2500BASEX: mvpp22_gop_init_sgmii(port); break; + case PHY_INTERFACE_MODE_5GBASER: case PHY_INTERFACE_MODE_10GBASER: if (!mvpp2_port_supports_xlg(port)) goto invalid_conf; @@ -2186,6 +2188,7 @@ static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port, xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); switch (interface) { + case PHY_INTERFACE_MODE_5GBASER: case PHY_INTERFACE_MODE_10GBASER: val = readl(mpcs + MVPP22_MPCS_CLK_RESET); val |= MAC_CLK_RESET_MAC | MAC_CLK_RESET_SD_RX | @@ -5433,8 +5436,11 @@ static void mvpp2_ethtool_get_drvinfo(struct net_device *dev, sizeof(drvinfo->bus_info)); } -static void mvpp2_ethtool_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static void +mvpp2_ethtool_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); @@ -5444,8 +5450,11 @@ static void mvpp2_ethtool_get_ringparam(struct net_device *dev, ring->tx_pending = port->tx_ring_size; } -static int mvpp2_ethtool_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) +static int +mvpp2_ethtool_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct mvpp2_port *port = netdev_priv(dev); u16 prev_rx_ring_size = port->rx_ring_size; @@ -6120,7 +6129,10 @@ static void mvpp2_xlg_pcs_get_state(struct phylink_pcs *pcs, struct mvpp2_port *port = mvpp2_pcs_to_port(pcs); u32 val; - state->speed = SPEED_10000; + if (port->phy_interface == PHY_INTERFACE_MODE_5GBASER) + state->speed = SPEED_5000; + else + state->speed = SPEED_10000; state->duplex = 1; state->an_complete = 1; @@ -6262,9 +6274,6 @@ static void mvpp2_phylink_validate(struct phylink_config *config, unsigned long *supported, struct phylink_link_state *state) { - struct mvpp2_port *port = mvpp2_phylink_to_port(config); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - /* When in 802.3z mode, we must have AN enabled: * Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... * When <PortType> = 1 (1000BASE-X) this field must be set to 1. @@ -6273,52 +6282,7 @@ static void mvpp2_phylink_validate(struct phylink_config *config, !phylink_test(state->advertising, Autoneg)) goto empty_set; - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - - if (port->priv->global_tx_fc) { - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - } - - switch (state->interface) { - case PHY_INTERFACE_MODE_10GBASER: - case PHY_INTERFACE_MODE_XAUI: - if (mvpp2_port_supports_xlg(port)) { - phylink_set_10g_modes(mask); - phylink_set(mask, 10000baseKR_Full); - } - break; - - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - case PHY_INTERFACE_MODE_SGMII: - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - break; - - case PHY_INTERFACE_MODE_1000BASEX: - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - break; - - case PHY_INTERFACE_MODE_2500BASEX: - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - break; - - default: - goto empty_set; - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); + phylink_generic_validate(config, supported, state); return; empty_set: @@ -6901,7 +6865,7 @@ static int mvpp2_port_probe(struct platform_device *pdev, mvpp2_set_hw_csum(port, port->pool_long->id); dev->vlan_features |= features; - dev->gso_max_segs = MVPP2_MAX_TSO_SEGS; + netif_set_gso_max_segs(dev, MVPP2_MAX_TSO_SEGS); dev->priv_flags |= IFF_UNICAST_FLT; /* MTU range: 68 - 9704 */ @@ -6913,12 +6877,44 @@ static int mvpp2_port_probe(struct platform_device *pdev, if (!mvpp2_use_acpi_compat_mode(port_fwnode)) { port->phylink_config.dev = &dev->dev; port->phylink_config.type = PHYLINK_NETDEV; + port->phylink_config.mac_capabilities = + MAC_2500FD | MAC_1000FD | MAC_100 | MAC_10; + + if (port->priv->global_tx_fc) + port->phylink_config.mac_capabilities |= + MAC_SYM_PAUSE | MAC_ASYM_PAUSE; if (mvpp2_port_supports_xlg(port)) { - __set_bit(PHY_INTERFACE_MODE_10GBASER, - port->phylink_config.supported_interfaces); - __set_bit(PHY_INTERFACE_MODE_XAUI, - port->phylink_config.supported_interfaces); + /* If a COMPHY is present, we can support any of + * the serdes modes and switch between them. + */ + if (comphy) { + __set_bit(PHY_INTERFACE_MODE_5GBASER, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_XAUI, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_5GBASER) { + __set_bit(PHY_INTERFACE_MODE_5GBASER, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_10GBASER) { + __set_bit(PHY_INTERFACE_MODE_10GBASER, + port->phylink_config.supported_interfaces); + } else if (phy_mode == PHY_INTERFACE_MODE_XAUI) { + __set_bit(PHY_INTERFACE_MODE_XAUI, + port->phylink_config.supported_interfaces); + } + + if (comphy) + port->phylink_config.mac_capabilities |= + MAC_10000FD | MAC_5000FD; + else if (phy_mode == PHY_INTERFACE_MODE_5GBASER) + port->phylink_config.mac_capabilities |= + MAC_5000FD; + else + port->phylink_config.mac_capabilities |= + MAC_10000FD; } if (mvpp2_port_supports_rgmii(port)) diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c index 80d4ce61f442..d85db90632d6 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_ethtool.c @@ -360,7 +360,9 @@ static int otx2_set_pauseparam(struct net_device *netdev, } static void otx2_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); struct otx2_qset *qs = &pfvf->qset; @@ -372,7 +374,9 @@ static void otx2_get_ringparam(struct net_device *netdev, } static int otx2_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct otx2_nic *pfvf = netdev_priv(netdev); bool if_up = netif_running(netdev); diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c index 1e0d0c9c1dac..1333edf1c361 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_pf.c @@ -2741,7 +2741,7 @@ static int otx2_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_LOOPBACK | NETIF_F_RXALL; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_gso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2_netdev_ops; diff --git a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c index 78944ad3492f..254bebffe8c1 100644 --- a/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c +++ b/drivers/net/ethernet/marvell/octeontx2/nic/otx2_vf.c @@ -663,7 +663,7 @@ static int otx2vf_probe(struct pci_dev *pdev, const struct pci_device_id *id) netdev->hw_features |= NETIF_F_NTUPLE; netdev->hw_features |= NETIF_F_RXALL; - netdev->gso_max_segs = OTX2_MAX_GSO_SEGS; + netif_set_gso_max_segs(netdev, OTX2_MAX_GSO_SEGS); netdev->watchdog_timeo = OTX2_TX_TIMEOUT; netdev->netdev_ops = &otx2vf_netdev_ops; diff --git a/drivers/net/ethernet/marvell/prestera/Makefile b/drivers/net/ethernet/marvell/prestera/Makefile index 0609df8b913d..48dbcb2baf8f 100644 --- a/drivers/net/ethernet/marvell/prestera/Makefile +++ b/drivers/net/ethernet/marvell/prestera/Makefile @@ -3,6 +3,6 @@ obj-$(CONFIG_PRESTERA) += prestera.o prestera-objs := prestera_main.o prestera_hw.o prestera_dsa.o \ prestera_rxtx.o prestera_devlink.o prestera_ethtool.o \ prestera_switchdev.o prestera_acl.o prestera_flow.o \ - prestera_flower.o prestera_span.o + prestera_flower.o prestera_span.o prestera_counter.o obj-$(CONFIG_PRESTERA_PCI) += prestera_pci.o diff --git a/drivers/net/ethernet/marvell/prestera/prestera.h b/drivers/net/ethernet/marvell/prestera/prestera.h index 2a4c14c704c0..797b2e4d3551 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera.h +++ b/drivers/net/ethernet/marvell/prestera/prestera.h @@ -248,6 +248,7 @@ struct prestera_switch { u32 mtu_max; u8 id; struct prestera_lag *lags; + struct prestera_counter *counter; u8 lag_member_max; u8 lag_max; }; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.c b/drivers/net/ethernet/marvell/prestera/prestera_acl.c index 83c75ffb1a1c..fc7f2fedafd7 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.c @@ -1,35 +1,72 @@ // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved */ #include <linux/rhashtable.h> -#include "prestera.h" -#include "prestera_hw.h" #include "prestera_acl.h" -#include "prestera_span.h" +#include "prestera_flow.h" +#include "prestera_hw.h" +#include "prestera.h" + +#define ACL_KEYMASK_SIZE \ + (sizeof(__be32) * __PRESTERA_ACL_RULE_MATCH_TYPE_MAX) struct prestera_acl { struct prestera_switch *sw; + struct list_head vtcam_list; struct list_head rules; + struct rhashtable ruleset_ht; + struct rhashtable acl_rule_entry_ht; + struct idr uid; +}; + +struct prestera_acl_ruleset_ht_key { + struct prestera_flow_block *block; +}; + +struct prestera_acl_rule_entry { + struct rhash_head ht_node; + struct prestera_acl_rule_entry_key key; + u32 hw_id; + u32 vtcam_id; + struct { + struct { + u8 valid:1; + } accept, drop, trap; + struct { + u32 id; + struct prestera_counter_block *block; + } counter; + }; }; struct prestera_acl_ruleset { + struct rhash_head ht_node; /* Member of acl HT */ + struct prestera_acl_ruleset_ht_key ht_key; struct rhashtable rule_ht; - struct prestera_switch *sw; - u16 id; + struct prestera_acl *acl; + unsigned long rule_count; + refcount_t refcount; + void *keymask; + u32 vtcam_id; + u16 pcl_id; + bool offload; }; -struct prestera_acl_rule { - struct rhash_head ht_node; +struct prestera_acl_vtcam { struct list_head list; - struct list_head match_list; - struct list_head action_list; - struct prestera_flow_block *block; - unsigned long cookie; - u32 priority; - u8 n_actions; - u8 n_matches; + __be32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + refcount_t refcount; u32 id; + bool is_keymask_set; + u8 lookup; +}; + +static const struct rhashtable_params prestera_acl_ruleset_ht_params = { + .key_len = sizeof(struct prestera_acl_ruleset_ht_key), + .key_offset = offsetof(struct prestera_acl_ruleset, ht_key), + .head_offset = offsetof(struct prestera_acl_ruleset, ht_node), + .automatic_shrinking = true, }; static const struct rhashtable_params prestera_acl_rule_ht_params = { @@ -39,28 +76,48 @@ static const struct rhashtable_params prestera_acl_rule_ht_params = { .automatic_shrinking = true, }; +static const struct rhashtable_params __prestera_acl_rule_entry_ht_params = { + .key_offset = offsetof(struct prestera_acl_rule_entry, key), + .head_offset = offsetof(struct prestera_acl_rule_entry, ht_node), + .key_len = sizeof(struct prestera_acl_rule_entry_key), + .automatic_shrinking = true, +}; + static struct prestera_acl_ruleset * -prestera_acl_ruleset_create(struct prestera_switch *sw) +prestera_acl_ruleset_create(struct prestera_acl *acl, + struct prestera_flow_block *block) { struct prestera_acl_ruleset *ruleset; int err; + u32 uid; ruleset = kzalloc(sizeof(*ruleset), GFP_KERNEL); if (!ruleset) return ERR_PTR(-ENOMEM); + ruleset->acl = acl; + ruleset->ht_key.block = block; + refcount_set(&ruleset->refcount, 1); + err = rhashtable_init(&ruleset->rule_ht, &prestera_acl_rule_ht_params); if (err) goto err_rhashtable_init; - err = prestera_hw_acl_ruleset_create(sw, &ruleset->id); + err = idr_alloc_u32(&acl->uid, NULL, &uid, U8_MAX, GFP_KERNEL); if (err) goto err_ruleset_create; - ruleset->sw = sw; + /* make pcl-id based on uid */ + ruleset->pcl_id = (u8)uid; + err = rhashtable_insert_fast(&acl->ruleset_ht, &ruleset->ht_node, + prestera_acl_ruleset_ht_params); + if (err) + goto err_ruleset_ht_insert; return ruleset; +err_ruleset_ht_insert: + idr_remove(&acl->uid, uid); err_ruleset_create: rhashtable_destroy(&ruleset->rule_ht); err_rhashtable_init: @@ -68,117 +125,158 @@ err_rhashtable_init: return ERR_PTR(err); } +int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset) +{ + u32 vtcam_id; + int err; + + if (ruleset->offload) + return -EEXIST; + + err = prestera_acl_vtcam_id_get(ruleset->acl, 0, + ruleset->keymask, &vtcam_id); + if (err) + return err; + + ruleset->vtcam_id = vtcam_id; + ruleset->offload = true; + return 0; +} + static void prestera_acl_ruleset_destroy(struct prestera_acl_ruleset *ruleset) { - prestera_hw_acl_ruleset_del(ruleset->sw, ruleset->id); + struct prestera_acl *acl = ruleset->acl; + u8 uid = ruleset->pcl_id & PRESTERA_ACL_KEYMASK_PCL_ID_USER; + + rhashtable_remove_fast(&acl->ruleset_ht, &ruleset->ht_node, + prestera_acl_ruleset_ht_params); + + if (ruleset->offload) + WARN_ON(prestera_acl_vtcam_id_put(acl, ruleset->vtcam_id)); + + idr_remove(&acl->uid, uid); + rhashtable_destroy(&ruleset->rule_ht); + kfree(ruleset->keymask); kfree(ruleset); } -struct prestera_flow_block * -prestera_acl_block_create(struct prestera_switch *sw, struct net *net) +static struct prestera_acl_ruleset * +__prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block) { - struct prestera_flow_block *block; + struct prestera_acl_ruleset_ht_key ht_key; - block = kzalloc(sizeof(*block), GFP_KERNEL); - if (!block) - return NULL; - INIT_LIST_HEAD(&block->binding_list); - block->net = net; - block->sw = sw; - - block->ruleset = prestera_acl_ruleset_create(sw); - if (IS_ERR(block->ruleset)) { - kfree(block); - return NULL; - } - - return block; + memset(&ht_key, 0, sizeof(ht_key)); + ht_key.block = block; + return rhashtable_lookup_fast(&acl->ruleset_ht, &ht_key, + prestera_acl_ruleset_ht_params); } -void prestera_acl_block_destroy(struct prestera_flow_block *block) +struct prestera_acl_ruleset * +prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block) { - prestera_acl_ruleset_destroy(block->ruleset); - WARN_ON(!list_empty(&block->binding_list)); - kfree(block); + struct prestera_acl_ruleset *ruleset; + + ruleset = __prestera_acl_ruleset_lookup(acl, block); + if (!ruleset) + return ERR_PTR(-ENOENT); + + refcount_inc(&ruleset->refcount); + return ruleset; } -static struct prestera_flow_block_binding * -prestera_acl_block_lookup(struct prestera_flow_block *block, - struct prestera_port *port) +struct prestera_acl_ruleset * +prestera_acl_ruleset_get(struct prestera_acl *acl, + struct prestera_flow_block *block) { - struct prestera_flow_block_binding *binding; + struct prestera_acl_ruleset *ruleset; - list_for_each_entry(binding, &block->binding_list, list) - if (binding->port == port) - return binding; + ruleset = __prestera_acl_ruleset_lookup(acl, block); + if (ruleset) { + refcount_inc(&ruleset->refcount); + return ruleset; + } - return NULL; + return prestera_acl_ruleset_create(acl, block); } -int prestera_acl_block_bind(struct prestera_flow_block *block, - struct prestera_port *port) +void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset) { - struct prestera_flow_block_binding *binding; - int err; + if (!refcount_dec_and_test(&ruleset->refcount)) + return; - if (WARN_ON(prestera_acl_block_lookup(block, port))) - return -EEXIST; + prestera_acl_ruleset_destroy(ruleset); +} - binding = kzalloc(sizeof(*binding), GFP_KERNEL); - if (!binding) - return -ENOMEM; - binding->span_id = PRESTERA_SPAN_INVALID_ID; - binding->port = port; +int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port) +{ + struct prestera_acl_iface iface = { + .type = PRESTERA_ACL_IFACE_TYPE_PORT, + .port = port + }; - err = prestera_hw_acl_port_bind(port, block->ruleset->id); - if (err) - goto err_rules_bind; + return prestera_hw_vtcam_iface_bind(port->sw, &iface, ruleset->vtcam_id, + ruleset->pcl_id); +} - list_add(&binding->list, &block->binding_list); - return 0; +int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port) +{ + struct prestera_acl_iface iface = { + .type = PRESTERA_ACL_IFACE_TYPE_PORT, + .port = port + }; -err_rules_bind: - kfree(binding); - return err; + return prestera_hw_vtcam_iface_unbind(port->sw, &iface, + ruleset->vtcam_id); } -int prestera_acl_block_unbind(struct prestera_flow_block *block, - struct prestera_port *port) +static int prestera_acl_ruleset_block_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_flow_block *block) { struct prestera_flow_block_binding *binding; + int err; - binding = prestera_acl_block_lookup(block, port); - if (!binding) - return -ENOENT; - - list_del(&binding->list); + block->ruleset_zero = ruleset; + list_for_each_entry(binding, &block->binding_list, list) { + err = prestera_acl_ruleset_bind(ruleset, binding->port); + if (err) + goto rollback; + } + return 0; - prestera_hw_acl_port_unbind(port, block->ruleset->id); +rollback: + list_for_each_entry_continue_reverse(binding, &block->binding_list, + list) + err = prestera_acl_ruleset_unbind(ruleset, binding->port); + block->ruleset_zero = NULL; - kfree(binding); - return 0; + return err; } -struct prestera_acl_ruleset * -prestera_acl_block_ruleset_get(struct prestera_flow_block *block) +static void +prestera_acl_ruleset_block_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_flow_block *block) { - return block->ruleset; -} + struct prestera_flow_block_binding *binding; -u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule) -{ - return rule->block->ruleset->id; + list_for_each_entry(binding, &block->binding_list, list) + prestera_acl_ruleset_unbind(ruleset, binding->port); + block->ruleset_zero = NULL; } -struct net *prestera_acl_block_net(struct prestera_flow_block *block) +void +prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, u16 pcl_id) { - return block->net; -} + struct prestera_acl_match *r_match = &rule->re_key.match; + __be16 pcl_id_mask = htons(PRESTERA_ACL_KEYMASK_PCL_ID); + __be16 pcl_id_key = htons(pcl_id); -struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block) -{ - return block->sw; + rule_match_set(r_match->key, PCL_ID, pcl_id_key); + rule_match_set(r_match->mask, PCL_ID, pcl_id_mask); } struct prestera_acl_rule * @@ -189,8 +287,13 @@ prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, prestera_acl_rule_ht_params); } +bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset) +{ + return ruleset->offload; +} + struct prestera_acl_rule * -prestera_acl_rule_create(struct prestera_flow_block *block, +prestera_acl_rule_create(struct prestera_acl_ruleset *ruleset, unsigned long cookie) { struct prestera_acl_rule *rule; @@ -199,178 +302,386 @@ prestera_acl_rule_create(struct prestera_flow_block *block, if (!rule) return ERR_PTR(-ENOMEM); - INIT_LIST_HEAD(&rule->match_list); - INIT_LIST_HEAD(&rule->action_list); + rule->ruleset = ruleset; rule->cookie = cookie; - rule->block = block; + + refcount_inc(&ruleset->refcount); return rule; } -struct list_head * -prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule) +void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, + u32 priority) { - return &rule->match_list; + rule->priority = priority; } -struct list_head * -prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule) +void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) { - return &rule->action_list; + prestera_acl_ruleset_put(rule->ruleset); + kfree(rule); } -int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_action_entry *entry) +int prestera_acl_rule_add(struct prestera_switch *sw, + struct prestera_acl_rule *rule) { - struct prestera_acl_rule_action_entry *a_entry; + int err; + struct prestera_acl_ruleset *ruleset = rule->ruleset; + struct prestera_flow_block *block = ruleset->ht_key.block; - a_entry = kmalloc(sizeof(*a_entry), GFP_KERNEL); - if (!a_entry) - return -ENOMEM; + /* try to add rule to hash table first */ + err = rhashtable_insert_fast(&ruleset->rule_ht, &rule->ht_node, + prestera_acl_rule_ht_params); + if (err) + goto err_ht_insert; + + prestera_acl_rule_keymask_pcl_id_set(rule, ruleset->pcl_id); + rule->re_arg.vtcam_id = ruleset->vtcam_id; + rule->re_key.prio = rule->priority; - memcpy(a_entry, entry, sizeof(*entry)); - list_add(&a_entry->list, &rule->action_list); + /* setup counter */ + rule->re_arg.count.valid = true; + rule->re_arg.count.client = PRESTERA_HW_COUNTER_CLIENT_LOOKUP_0; + + rule->re = prestera_acl_rule_entry_find(sw->acl, &rule->re_key); + err = WARN_ON(rule->re) ? -EEXIST : 0; + if (err) + goto err_rule_add; - rule->n_actions++; + rule->re = prestera_acl_rule_entry_create(sw->acl, &rule->re_key, + &rule->re_arg); + err = !rule->re ? -EINVAL : 0; + if (err) + goto err_rule_add; + + /* bind the block (all ports) to chain index 0 */ + if (!ruleset->rule_count) { + err = prestera_acl_ruleset_block_bind(ruleset, block); + if (err) + goto err_acl_block_bind; + } + + list_add_tail(&rule->list, &sw->acl->rules); + ruleset->rule_count++; return 0; + +err_acl_block_bind: + prestera_acl_rule_entry_destroy(sw->acl, rule->re); +err_rule_add: + rule->re = NULL; + rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node, + prestera_acl_rule_ht_params); +err_ht_insert: + return err; } -u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule) +void prestera_acl_rule_del(struct prestera_switch *sw, + struct prestera_acl_rule *rule) { - return rule->n_actions; + struct prestera_acl_ruleset *ruleset = rule->ruleset; + struct prestera_flow_block *block = ruleset->ht_key.block; + + rhashtable_remove_fast(&ruleset->rule_ht, &rule->ht_node, + prestera_acl_rule_ht_params); + ruleset->rule_count--; + list_del(&rule->list); + + prestera_acl_rule_entry_destroy(sw->acl, rule->re); + + /* unbind block (all ports) */ + if (!ruleset->rule_count) + prestera_acl_ruleset_block_unbind(ruleset, block); } -u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule) +int prestera_acl_rule_get_stats(struct prestera_acl *acl, + struct prestera_acl_rule *rule, + u64 *packets, u64 *bytes, u64 *last_use) { - return rule->priority; + u64 current_packets; + u64 current_bytes; + int err; + + err = prestera_counter_stats_get(acl->sw->counter, + rule->re->counter.block, + rule->re->counter.id, + ¤t_packets, ¤t_bytes); + if (err) + return err; + + *packets = current_packets; + *bytes = current_bytes; + *last_use = jiffies; + + return 0; } -void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, - u32 priority) +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_find(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key) { - rule->priority = priority; + return rhashtable_lookup_fast(&acl->acl_rule_entry_ht, key, + __prestera_acl_rule_entry_ht_params); } -int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_match_entry *entry) +static int __prestera_acl_rule_entry2hw_del(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) { - struct prestera_acl_rule_match_entry *m_entry; + return prestera_hw_vtcam_rule_del(sw, e->vtcam_id, e->hw_id); +} - m_entry = kmalloc(sizeof(*m_entry), GFP_KERNEL); - if (!m_entry) - return -ENOMEM; +static int __prestera_acl_rule_entry2hw_add(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) +{ + struct prestera_acl_hw_action_info act_hw[PRESTERA_ACL_RULE_ACTION_MAX]; + int act_num; - memcpy(m_entry, entry, sizeof(*entry)); - list_add(&m_entry->list, &rule->match_list); + memset(&act_hw, 0, sizeof(act_hw)); + act_num = 0; - rule->n_matches++; - return 0; + /* accept */ + if (e->accept.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_ACCEPT; + act_num++; + } + /* drop */ + if (e->drop.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_DROP; + act_num++; + } + /* trap */ + if (e->trap.valid) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_TRAP; + act_num++; + } + /* counter */ + if (e->counter.block) { + act_hw[act_num].id = PRESTERA_ACL_RULE_ACTION_COUNT; + act_hw[act_num].count.id = e->counter.id; + act_num++; + } + + return prestera_hw_vtcam_rule_add(sw, e->vtcam_id, e->key.prio, + e->key.match.key, e->key.match.mask, + act_hw, act_num, &e->hw_id); } -u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule) +static void +__prestera_acl_rule_entry_act_destruct(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e) { - return rule->n_matches; + /* counter */ + prestera_counter_put(sw->counter, e->counter.block, e->counter.id); } -void prestera_acl_rule_destroy(struct prestera_acl_rule *rule) +void prestera_acl_rule_entry_destroy(struct prestera_acl *acl, + struct prestera_acl_rule_entry *e) { - struct prestera_acl_rule_action_entry *a_entry; - struct prestera_acl_rule_match_entry *m_entry; - struct list_head *pos, *n; - - list_for_each_safe(pos, n, &rule->match_list) { - m_entry = list_entry(pos, typeof(*m_entry), list); - list_del(pos); - kfree(m_entry); - } + int ret; + + rhashtable_remove_fast(&acl->acl_rule_entry_ht, &e->ht_node, + __prestera_acl_rule_entry_ht_params); - list_for_each_safe(pos, n, &rule->action_list) { - a_entry = list_entry(pos, typeof(*a_entry), list); - list_del(pos); - kfree(a_entry); + ret = __prestera_acl_rule_entry2hw_del(acl->sw, e); + WARN_ON(ret && ret != -ENODEV); + + __prestera_acl_rule_entry_act_destruct(acl->sw, e); + kfree(e); +} + +static int +__prestera_acl_rule_entry_act_construct(struct prestera_switch *sw, + struct prestera_acl_rule_entry *e, + struct prestera_acl_rule_entry_arg *arg) +{ + /* accept */ + e->accept.valid = arg->accept.valid; + /* drop */ + e->drop.valid = arg->drop.valid; + /* trap */ + e->trap.valid = arg->trap.valid; + /* counter */ + if (arg->count.valid) { + int err; + + err = prestera_counter_get(sw->counter, arg->count.client, + &e->counter.block, + &e->counter.id); + if (err) + goto err_out; } - kfree(rule); + return 0; + +err_out: + __prestera_acl_rule_entry_act_destruct(sw, e); + return -EINVAL; } -int prestera_acl_rule_add(struct prestera_switch *sw, - struct prestera_acl_rule *rule) +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_create(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key, + struct prestera_acl_rule_entry_arg *arg) { - u32 rule_id; + struct prestera_acl_rule_entry *e; int err; - /* try to add rule to hash table first */ - err = rhashtable_insert_fast(&rule->block->ruleset->rule_ht, - &rule->ht_node, - prestera_acl_rule_ht_params); - if (err) - return err; + e = kzalloc(sizeof(*e), GFP_KERNEL); + if (!e) + goto err_kzalloc; - /* add rule to hw */ - err = prestera_hw_acl_rule_add(sw, rule, &rule_id); + memcpy(&e->key, key, sizeof(*key)); + e->vtcam_id = arg->vtcam_id; + err = __prestera_acl_rule_entry_act_construct(acl->sw, e, arg); if (err) - goto err_rule_add; + goto err_act_construct; - rule->id = rule_id; + err = __prestera_acl_rule_entry2hw_add(acl->sw, e); + if (err) + goto err_hw_add; - list_add_tail(&rule->list, &sw->acl->rules); + err = rhashtable_insert_fast(&acl->acl_rule_entry_ht, &e->ht_node, + __prestera_acl_rule_entry_ht_params); + if (err) + goto err_ht_insert; - return 0; + return e; -err_rule_add: - rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, - prestera_acl_rule_ht_params); - return err; +err_ht_insert: + WARN_ON(__prestera_acl_rule_entry2hw_del(acl->sw, e)); +err_hw_add: + __prestera_acl_rule_entry_act_destruct(acl->sw, e); +err_act_construct: + kfree(e); +err_kzalloc: + return NULL; } -void prestera_acl_rule_del(struct prestera_switch *sw, - struct prestera_acl_rule *rule) +int prestera_acl_vtcam_id_get(struct prestera_acl *acl, u8 lookup, + void *keymask, u32 *vtcam_id) { - rhashtable_remove_fast(&rule->block->ruleset->rule_ht, &rule->ht_node, - prestera_acl_rule_ht_params); - list_del(&rule->list); - prestera_hw_acl_rule_del(sw, rule->id); + struct prestera_acl_vtcam *vtcam; + u32 new_vtcam_id; + int err; + + /* find the vtcam that suits keymask. We do not expect to have + * a big number of vtcams, so, the list type for vtcam list is + * fine for now + */ + list_for_each_entry(vtcam, &acl->vtcam_list, list) { + if (lookup != vtcam->lookup) + continue; + + if (!keymask && !vtcam->is_keymask_set) { + refcount_inc(&vtcam->refcount); + goto vtcam_found; + } + + if (keymask && vtcam->is_keymask_set && + !memcmp(keymask, vtcam->keymask, sizeof(vtcam->keymask))) { + refcount_inc(&vtcam->refcount); + goto vtcam_found; + } + } + + /* vtcam not found, try to create new one */ + vtcam = kzalloc(sizeof(*vtcam), GFP_KERNEL); + if (!vtcam) + return -ENOMEM; + + err = prestera_hw_vtcam_create(acl->sw, lookup, keymask, &new_vtcam_id, + PRESTERA_HW_VTCAM_DIR_INGRESS); + if (err) { + kfree(vtcam); + return err; + } + + vtcam->id = new_vtcam_id; + vtcam->lookup = lookup; + if (keymask) { + memcpy(vtcam->keymask, keymask, sizeof(vtcam->keymask)); + vtcam->is_keymask_set = true; + } + refcount_set(&vtcam->refcount, 1); + list_add_rcu(&vtcam->list, &acl->vtcam_list); + +vtcam_found: + *vtcam_id = vtcam->id; + return 0; } -int prestera_acl_rule_get_stats(struct prestera_switch *sw, - struct prestera_acl_rule *rule, - u64 *packets, u64 *bytes, u64 *last_use) +int prestera_acl_vtcam_id_put(struct prestera_acl *acl, u32 vtcam_id) { - u64 current_packets; - u64 current_bytes; + struct prestera_acl_vtcam *vtcam; int err; - err = prestera_hw_acl_rule_stats_get(sw, rule->id, ¤t_packets, - ¤t_bytes); - if (err) - return err; + list_for_each_entry(vtcam, &acl->vtcam_list, list) { + if (vtcam_id != vtcam->id) + continue; - *packets = current_packets; - *bytes = current_bytes; - *last_use = jiffies; + if (!refcount_dec_and_test(&vtcam->refcount)) + return 0; - return 0; + err = prestera_hw_vtcam_destroy(acl->sw, vtcam->id); + if (err && err != -ENODEV) { + refcount_set(&vtcam->refcount, 1); + return err; + } + + list_del(&vtcam->list); + kfree(vtcam); + return 0; + } + + return -ENOENT; } int prestera_acl_init(struct prestera_switch *sw) { struct prestera_acl *acl; + int err; acl = kzalloc(sizeof(*acl), GFP_KERNEL); if (!acl) return -ENOMEM; + acl->sw = sw; INIT_LIST_HEAD(&acl->rules); + INIT_LIST_HEAD(&acl->vtcam_list); + idr_init(&acl->uid); + + err = rhashtable_init(&acl->acl_rule_entry_ht, + &__prestera_acl_rule_entry_ht_params); + if (err) + goto err_acl_rule_entry_ht_init; + + err = rhashtable_init(&acl->ruleset_ht, + &prestera_acl_ruleset_ht_params); + if (err) + goto err_ruleset_ht_init; + sw->acl = acl; - acl->sw = sw; return 0; + +err_ruleset_ht_init: + rhashtable_destroy(&acl->acl_rule_entry_ht); +err_acl_rule_entry_ht_init: + kfree(acl); + return err; } void prestera_acl_fini(struct prestera_switch *sw) { struct prestera_acl *acl = sw->acl; + WARN_ON(!idr_is_empty(&acl->uid)); + idr_destroy(&acl->uid); + + WARN_ON(!list_empty(&acl->vtcam_list)); WARN_ON(!list_empty(&acl->rules)); + + rhashtable_destroy(&acl->ruleset_ht); + rhashtable_destroy(&acl->acl_rule_entry_ht); + kfree(acl); } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_acl.h b/drivers/net/ethernet/marvell/prestera/prestera_acl.h index 39b7869be659..4e6006b4531f 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_acl.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_acl.h @@ -1,114 +1,130 @@ /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved. */ #ifndef _PRESTERA_ACL_H_ #define _PRESTERA_ACL_H_ -enum prestera_acl_rule_match_entry_type { - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE = 1, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE, - PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE +#include <linux/types.h> +#include "prestera_counter.h" + +#define PRESTERA_ACL_KEYMASK_PCL_ID 0x3FF +#define PRESTERA_ACL_KEYMASK_PCL_ID_USER \ + (PRESTERA_ACL_KEYMASK_PCL_ID & 0x00FF) + +#define rule_match_set_n(match_p, type, val_p, size) \ + memcpy(&(match_p)[PRESTERA_ACL_RULE_MATCH_TYPE_##type], \ + val_p, size) +#define rule_match_set(match_p, type, val) \ + memcpy(&(match_p)[PRESTERA_ACL_RULE_MATCH_TYPE_##type], \ + &(val), sizeof(val)) + +enum prestera_acl_match_type { + PRESTERA_ACL_RULE_MATCH_TYPE_PCL_ID, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_TYPE, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_DMAC_0, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_DMAC_1, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_SMAC_0, + PRESTERA_ACL_RULE_MATCH_TYPE_ETH_SMAC_1, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_PROTO, + PRESTERA_ACL_RULE_MATCH_TYPE_SYS_PORT, + PRESTERA_ACL_RULE_MATCH_TYPE_SYS_DEV, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_IP_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_RANGE_SRC, + PRESTERA_ACL_RULE_MATCH_TYPE_L4_PORT_RANGE_DST, + PRESTERA_ACL_RULE_MATCH_TYPE_VLAN_ID, + PRESTERA_ACL_RULE_MATCH_TYPE_VLAN_TPID, + PRESTERA_ACL_RULE_MATCH_TYPE_ICMP_TYPE, + PRESTERA_ACL_RULE_MATCH_TYPE_ICMP_CODE, + + __PRESTERA_ACL_RULE_MATCH_TYPE_MAX }; enum prestera_acl_rule_action { - PRESTERA_ACL_RULE_ACTION_ACCEPT, - PRESTERA_ACL_RULE_ACTION_DROP, - PRESTERA_ACL_RULE_ACTION_TRAP + PRESTERA_ACL_RULE_ACTION_ACCEPT = 0, + PRESTERA_ACL_RULE_ACTION_DROP = 1, + PRESTERA_ACL_RULE_ACTION_TRAP = 2, + PRESTERA_ACL_RULE_ACTION_COUNT = 7, + + PRESTERA_ACL_RULE_ACTION_MAX }; -struct prestera_switch; -struct prestera_port; -struct prestera_acl_rule; -struct prestera_acl_ruleset; +enum { + PRESTERA_ACL_IFACE_TYPE_PORT, + PRESTERA_ACL_IFACE_TYPE_INDEX +}; -struct prestera_flow_block_binding { - struct list_head list; - struct prestera_port *port; - int span_id; +struct prestera_acl_match { + __be32 key[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __be32 mask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; }; -struct prestera_flow_block { - struct list_head binding_list; - struct prestera_switch *sw; - struct net *net; - struct prestera_acl_ruleset *ruleset; - struct flow_block_cb *block_cb; +struct prestera_acl_action_count { + u32 id; }; -struct prestera_acl_rule_action_entry { - struct list_head list; - enum prestera_acl_rule_action id; +struct prestera_acl_rule_entry_key { + u32 prio; + struct prestera_acl_match match; }; -struct prestera_acl_rule_match_entry { - struct list_head list; - enum prestera_acl_rule_match_entry_type type; +struct prestera_acl_hw_action_info { + enum prestera_acl_rule_action id; union { + struct prestera_acl_action_count count; + }; +}; + +/* This struct (arg) used only to be passed as parameter for + * acl_rule_entry_create. Must be flat. Can contain object keys, which will be + * resolved to object links, before saving to acl_rule_entry struct + */ +struct prestera_acl_rule_entry_arg { + u32 vtcam_id; + struct { struct { - u8 key; - u8 mask; - } u8; - struct { - u16 key; - u16 mask; - } u16; - struct { - u32 key; - u32 mask; - } u32; - struct { - u64 key; - u64 mask; - } u64; + u8 valid:1; + } accept, drop, trap; struct { - u8 key[ETH_ALEN]; - u8 mask[ETH_ALEN]; - } mac; - } keymask; + u8 valid:1; + u32 client; + } count; + }; +}; + +struct prestera_acl_rule { + struct rhash_head ht_node; /* Member of acl HT */ + struct list_head list; + struct prestera_acl_ruleset *ruleset; + unsigned long cookie; + u32 priority; + struct prestera_acl_rule_entry_key re_key; + struct prestera_acl_rule_entry_arg re_arg; + struct prestera_acl_rule_entry *re; }; +struct prestera_acl_iface { + union { + struct prestera_port *port; + u32 index; + }; + u8 type; +}; + +struct prestera_acl; +struct prestera_switch; +struct prestera_flow_block; + int prestera_acl_init(struct prestera_switch *sw); void prestera_acl_fini(struct prestera_switch *sw); -struct prestera_flow_block * -prestera_acl_block_create(struct prestera_switch *sw, struct net *net); -void prestera_acl_block_destroy(struct prestera_flow_block *block); -struct net *prestera_acl_block_net(struct prestera_flow_block *block); -struct prestera_switch *prestera_acl_block_sw(struct prestera_flow_block *block); -int prestera_acl_block_bind(struct prestera_flow_block *block, - struct prestera_port *port); -int prestera_acl_block_unbind(struct prestera_flow_block *block, - struct prestera_port *port); -struct prestera_acl_ruleset * -prestera_acl_block_ruleset_get(struct prestera_flow_block *block); + struct prestera_acl_rule * -prestera_acl_rule_create(struct prestera_flow_block *block, +prestera_acl_rule_create(struct prestera_acl_ruleset *ruleset, unsigned long cookie); -u32 prestera_acl_rule_priority_get(struct prestera_acl_rule *rule); void prestera_acl_rule_priority_set(struct prestera_acl_rule *rule, u32 priority); -u16 prestera_acl_rule_ruleset_id_get(const struct prestera_acl_rule *rule); -struct list_head * -prestera_acl_rule_action_list_get(struct prestera_acl_rule *rule); -u8 prestera_acl_rule_action_len(struct prestera_acl_rule *rule); -u8 prestera_acl_rule_match_len(struct prestera_acl_rule *rule); -int prestera_acl_rule_action_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_action_entry *entry); -struct list_head * -prestera_acl_rule_match_list_get(struct prestera_acl_rule *rule); -int prestera_acl_rule_match_add(struct prestera_acl_rule *rule, - struct prestera_acl_rule_match_entry *entry); void prestera_acl_rule_destroy(struct prestera_acl_rule *rule); struct prestera_acl_rule * prestera_acl_rule_lookup(struct prestera_acl_ruleset *ruleset, @@ -117,8 +133,37 @@ int prestera_acl_rule_add(struct prestera_switch *sw, struct prestera_acl_rule *rule); void prestera_acl_rule_del(struct prestera_switch *sw, struct prestera_acl_rule *rule); -int prestera_acl_rule_get_stats(struct prestera_switch *sw, +int prestera_acl_rule_get_stats(struct prestera_acl *acl, struct prestera_acl_rule *rule, u64 *packets, u64 *bytes, u64 *last_use); +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_find(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key); +void prestera_acl_rule_entry_destroy(struct prestera_acl *acl, + struct prestera_acl_rule_entry *e); +struct prestera_acl_rule_entry * +prestera_acl_rule_entry_create(struct prestera_acl *acl, + struct prestera_acl_rule_entry_key *key, + struct prestera_acl_rule_entry_arg *arg); +struct prestera_acl_ruleset * +prestera_acl_ruleset_get(struct prestera_acl *acl, + struct prestera_flow_block *block); +struct prestera_acl_ruleset * +prestera_acl_ruleset_lookup(struct prestera_acl *acl, + struct prestera_flow_block *block); +bool prestera_acl_ruleset_is_offload(struct prestera_acl_ruleset *ruleset); +int prestera_acl_ruleset_offload(struct prestera_acl_ruleset *ruleset); +void prestera_acl_ruleset_put(struct prestera_acl_ruleset *ruleset); +int prestera_acl_ruleset_bind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port); +int prestera_acl_ruleset_unbind(struct prestera_acl_ruleset *ruleset, + struct prestera_port *port); +void +prestera_acl_rule_keymask_pcl_id_set(struct prestera_acl_rule *rule, + u16 pcl_id); + +int prestera_acl_vtcam_id_get(struct prestera_acl *acl, u8 lookup, + void *keymask, u32 *vtcam_id); +int prestera_acl_vtcam_id_put(struct prestera_acl *acl, u32 vtcam_id); #endif /* _PRESTERA_ACL_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_counter.c b/drivers/net/ethernet/marvell/prestera/prestera_counter.c new file mode 100644 index 000000000000..4cd53a2dae46 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_counter.c @@ -0,0 +1,475 @@ +// SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 +/* Copyright (c) 2021 Marvell International Ltd. All rights reserved */ + +#include "prestera.h" +#include "prestera_hw.h" +#include "prestera_acl.h" +#include "prestera_counter.h" + +#define COUNTER_POLL_TIME (msecs_to_jiffies(1000)) +#define COUNTER_RESCHED_TIME (msecs_to_jiffies(50)) +#define COUNTER_BULK_SIZE (256) + +struct prestera_counter { + struct prestera_switch *sw; + struct delayed_work stats_dw; + struct mutex mtx; /* protect block_list */ + struct prestera_counter_block **block_list; + u32 total_read; + u32 block_list_len; + u32 curr_idx; + bool is_fetching; +}; + +struct prestera_counter_block { + struct list_head list; + u32 id; + u32 offset; + u32 num_counters; + u32 client; + struct idr counter_idr; + refcount_t refcnt; + struct mutex mtx; /* protect stats and counter_idr */ + struct prestera_counter_stats *stats; + u8 *counter_flag; + bool is_updating; + bool full; +}; + +enum { + COUNTER_FLAG_READY = 0, + COUNTER_FLAG_INVALID = 1 +}; + +static bool +prestera_counter_is_ready(struct prestera_counter_block *block, u32 id) +{ + return block->counter_flag[id - block->offset] == COUNTER_FLAG_READY; +} + +static void prestera_counter_lock(struct prestera_counter *counter) +{ + mutex_lock(&counter->mtx); +} + +static void prestera_counter_unlock(struct prestera_counter *counter) +{ + mutex_unlock(&counter->mtx); +} + +static void prestera_counter_block_lock(struct prestera_counter_block *block) +{ + mutex_lock(&block->mtx); +} + +static void prestera_counter_block_unlock(struct prestera_counter_block *block) +{ + mutex_unlock(&block->mtx); +} + +static bool prestera_counter_block_incref(struct prestera_counter_block *block) +{ + return refcount_inc_not_zero(&block->refcnt); +} + +static bool prestera_counter_block_decref(struct prestera_counter_block *block) +{ + return refcount_dec_and_test(&block->refcnt); +} + +/* must be called with prestera_counter_block_lock() */ +static void prestera_counter_stats_clear(struct prestera_counter_block *block, + u32 counter_id) +{ + memset(&block->stats[counter_id - block->offset], 0, + sizeof(*block->stats)); +} + +static struct prestera_counter_block * +prestera_counter_block_lookup_not_full(struct prestera_counter *counter, + u32 client) +{ + u32 i; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i] && + counter->block_list[i]->client == client && + !counter->block_list[i]->full && + prestera_counter_block_incref(counter->block_list[i])) { + prestera_counter_unlock(counter); + return counter->block_list[i]; + } + } + prestera_counter_unlock(counter); + + return NULL; +} + +static int prestera_counter_block_list_add(struct prestera_counter *counter, + struct prestera_counter_block *block) +{ + struct prestera_counter_block **arr; + u32 i; + + prestera_counter_lock(counter); + + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i]) + continue; + + counter->block_list[i] = block; + prestera_counter_unlock(counter); + return 0; + } + + arr = krealloc(counter->block_list, (counter->block_list_len + 1) * + sizeof(*counter->block_list), GFP_KERNEL); + if (!arr) { + prestera_counter_unlock(counter); + return -ENOMEM; + } + + counter->block_list = arr; + counter->block_list[counter->block_list_len] = block; + counter->block_list_len++; + prestera_counter_unlock(counter); + return 0; +} + +static struct prestera_counter_block * +prestera_counter_block_get(struct prestera_counter *counter, u32 client) +{ + struct prestera_counter_block *block; + int err; + + block = prestera_counter_block_lookup_not_full(counter, client); + if (block) + return block; + + block = kzalloc(sizeof(*block), GFP_KERNEL); + if (!block) + return ERR_PTR(-ENOMEM); + + err = prestera_hw_counter_block_get(counter->sw, client, + &block->id, &block->offset, + &block->num_counters); + if (err) + goto err_block; + + block->stats = kcalloc(block->num_counters, + sizeof(*block->stats), GFP_KERNEL); + if (!block->stats) { + err = -ENOMEM; + goto err_stats; + } + + block->counter_flag = kcalloc(block->num_counters, + sizeof(*block->counter_flag), + GFP_KERNEL); + if (!block->counter_flag) { + err = -ENOMEM; + goto err_flag; + } + + block->client = client; + mutex_init(&block->mtx); + refcount_set(&block->refcnt, 1); + idr_init_base(&block->counter_idr, block->offset); + + err = prestera_counter_block_list_add(counter, block); + if (err) + goto err_list_add; + + return block; + +err_list_add: + idr_destroy(&block->counter_idr); + mutex_destroy(&block->mtx); + kfree(block->counter_flag); +err_flag: + kfree(block->stats); +err_stats: + prestera_hw_counter_block_release(counter->sw, block->id); +err_block: + kfree(block); + return ERR_PTR(err); +} + +static void prestera_counter_block_put(struct prestera_counter *counter, + struct prestera_counter_block *block) +{ + u32 i; + + if (!prestera_counter_block_decref(block)) + return; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + if (counter->block_list[i] && + counter->block_list[i]->id == block->id) { + counter->block_list[i] = NULL; + break; + } + } + prestera_counter_unlock(counter); + + WARN_ON(!idr_is_empty(&block->counter_idr)); + + prestera_hw_counter_block_release(counter->sw, block->id); + idr_destroy(&block->counter_idr); + mutex_destroy(&block->mtx); + kfree(block->stats); + kfree(block); +} + +static int prestera_counter_get_vacant(struct prestera_counter_block *block, + u32 *id) +{ + int free_id; + + if (block->full) + return -ENOSPC; + + prestera_counter_block_lock(block); + free_id = idr_alloc_cyclic(&block->counter_idr, NULL, block->offset, + block->offset + block->num_counters, + GFP_KERNEL); + if (free_id < 0) { + if (free_id == -ENOSPC) + block->full = true; + + prestera_counter_block_unlock(block); + return free_id; + } + *id = free_id; + prestera_counter_block_unlock(block); + + return 0; +} + +int prestera_counter_get(struct prestera_counter *counter, u32 client, + struct prestera_counter_block **bl, u32 *counter_id) +{ + struct prestera_counter_block *block; + int err; + u32 id; + +get_next_block: + block = prestera_counter_block_get(counter, client); + if (IS_ERR(block)) + return PTR_ERR(block); + + err = prestera_counter_get_vacant(block, &id); + if (err) { + prestera_counter_block_put(counter, block); + + if (err == -ENOSPC) + goto get_next_block; + + return err; + } + + prestera_counter_block_lock(block); + if (block->is_updating) + block->counter_flag[id - block->offset] = COUNTER_FLAG_INVALID; + prestera_counter_block_unlock(block); + + *counter_id = id; + *bl = block; + + return 0; +} + +void prestera_counter_put(struct prestera_counter *counter, + struct prestera_counter_block *block, u32 counter_id) +{ + if (!block) + return; + + prestera_counter_block_lock(block); + idr_remove(&block->counter_idr, counter_id); + block->full = false; + prestera_counter_stats_clear(block, counter_id); + prestera_counter_block_unlock(block); + + prestera_hw_counter_clear(counter->sw, block->id, counter_id); + prestera_counter_block_put(counter, block); +} + +static u32 prestera_counter_block_idx_next(struct prestera_counter *counter, + u32 curr_idx) +{ + u32 idx, i, start = curr_idx + 1; + + prestera_counter_lock(counter); + for (i = 0; i < counter->block_list_len; i++) { + idx = (start + i) % counter->block_list_len; + if (!counter->block_list[idx]) + continue; + + prestera_counter_unlock(counter); + return idx; + } + prestera_counter_unlock(counter); + + return 0; +} + +static struct prestera_counter_block * +prestera_counter_block_get_by_idx(struct prestera_counter *counter, u32 idx) +{ + if (idx >= counter->block_list_len) + return NULL; + + prestera_counter_lock(counter); + + if (!counter->block_list[idx] || + !prestera_counter_block_incref(counter->block_list[idx])) { + prestera_counter_unlock(counter); + return NULL; + } + + prestera_counter_unlock(counter); + return counter->block_list[idx]; +} + +static void prestera_counter_stats_work(struct work_struct *work) +{ + struct delayed_work *dl_work = + container_of(work, struct delayed_work, work); + struct prestera_counter *counter = + container_of(dl_work, struct prestera_counter, stats_dw); + struct prestera_counter_block *block; + u32 resched_time = COUNTER_POLL_TIME; + u32 count = COUNTER_BULK_SIZE; + bool done = false; + int err; + u32 i; + + block = prestera_counter_block_get_by_idx(counter, counter->curr_idx); + if (!block) { + if (counter->is_fetching) + goto abort; + + goto next; + } + + if (!counter->is_fetching) { + err = prestera_hw_counter_trigger(counter->sw, block->id); + if (err) + goto abort; + + prestera_counter_block_lock(block); + block->is_updating = true; + prestera_counter_block_unlock(block); + + counter->is_fetching = true; + counter->total_read = 0; + resched_time = COUNTER_RESCHED_TIME; + goto resched; + } + + prestera_counter_block_lock(block); + err = prestera_hw_counters_get(counter->sw, counter->total_read, + &count, &done, + &block->stats[counter->total_read]); + prestera_counter_block_unlock(block); + if (err) + goto abort; + + counter->total_read += count; + if (!done || counter->total_read < block->num_counters) { + resched_time = COUNTER_RESCHED_TIME; + goto resched; + } + + for (i = 0; i < block->num_counters; i++) { + if (block->counter_flag[i] == COUNTER_FLAG_INVALID) { + prestera_counter_block_lock(block); + block->counter_flag[i] = COUNTER_FLAG_READY; + memset(&block->stats[i], 0, sizeof(*block->stats)); + prestera_counter_block_unlock(block); + } + } + + prestera_counter_block_lock(block); + block->is_updating = false; + prestera_counter_block_unlock(block); + + goto next; +abort: + prestera_hw_counter_abort(counter->sw); +next: + counter->is_fetching = false; + counter->curr_idx = + prestera_counter_block_idx_next(counter, counter->curr_idx); +resched: + if (block) + prestera_counter_block_put(counter, block); + + schedule_delayed_work(&counter->stats_dw, resched_time); +} + +/* Can be executed without rtnl_lock(). + * So pay attention when something changing. + */ +int prestera_counter_stats_get(struct prestera_counter *counter, + struct prestera_counter_block *block, + u32 counter_id, u64 *packets, u64 *bytes) +{ + if (!block || !prestera_counter_is_ready(block, counter_id)) { + *packets = 0; + *bytes = 0; + return 0; + } + + prestera_counter_block_lock(block); + *packets = block->stats[counter_id - block->offset].packets; + *bytes = block->stats[counter_id - block->offset].bytes; + + prestera_counter_stats_clear(block, counter_id); + prestera_counter_block_unlock(block); + + return 0; +} + +int prestera_counter_init(struct prestera_switch *sw) +{ + struct prestera_counter *counter; + + counter = kzalloc(sizeof(*counter), GFP_KERNEL); + if (!counter) + return -ENOMEM; + + counter->block_list = kzalloc(sizeof(*counter->block_list), GFP_KERNEL); + if (!counter->block_list) { + kfree(counter); + return -ENOMEM; + } + + mutex_init(&counter->mtx); + counter->block_list_len = 1; + counter->sw = sw; + sw->counter = counter; + + INIT_DELAYED_WORK(&counter->stats_dw, prestera_counter_stats_work); + schedule_delayed_work(&counter->stats_dw, COUNTER_POLL_TIME); + + return 0; +} + +void prestera_counter_fini(struct prestera_switch *sw) +{ + struct prestera_counter *counter = sw->counter; + u32 i; + + cancel_delayed_work_sync(&counter->stats_dw); + + for (i = 0; i < counter->block_list_len; i++) + WARN_ON(counter->block_list[i]); + + mutex_destroy(&counter->mtx); + kfree(counter->block_list); + kfree(counter); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_counter.h b/drivers/net/ethernet/marvell/prestera/prestera_counter.h new file mode 100644 index 000000000000..ad6b73907799 --- /dev/null +++ b/drivers/net/ethernet/marvell/prestera/prestera_counter.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ +/* Copyright (c) 2021 Marvell International Ltd. All rights reserved. */ + +#ifndef _PRESTERA_COUNTER_H_ +#define _PRESTERA_COUNTER_H_ + +#include <linux/types.h> + +struct prestera_counter_stats { + u64 packets; + u64 bytes; +}; + +struct prestera_switch; +struct prestera_counter; +struct prestera_counter_block; + +int prestera_counter_init(struct prestera_switch *sw); +void prestera_counter_fini(struct prestera_switch *sw); + +int prestera_counter_get(struct prestera_counter *counter, u32 client, + struct prestera_counter_block **block, + u32 *counter_id); +void prestera_counter_put(struct prestera_counter *counter, + struct prestera_counter_block *block, u32 counter_id); +int prestera_counter_stats_get(struct prestera_counter *counter, + struct prestera_counter_block *block, + u32 counter_id, u64 *packets, u64 *bytes); + +#endif /* _PRESTERA_COUNTER_H_ */ diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.c b/drivers/net/ethernet/marvell/prestera/prestera_flow.c index c9891e968259..94a1feb3d9e1 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.c @@ -60,11 +60,100 @@ static int prestera_flow_block_cb(enum tc_setup_type type, } } +static void prestera_flow_block_destroy(void *cb_priv) +{ + struct prestera_flow_block *block = cb_priv; + + WARN_ON(!list_empty(&block->binding_list)); + + kfree(block); +} + +static struct prestera_flow_block * +prestera_flow_block_create(struct prestera_switch *sw, struct net *net) +{ + struct prestera_flow_block *block; + + block = kzalloc(sizeof(*block), GFP_KERNEL); + if (!block) + return NULL; + + INIT_LIST_HEAD(&block->binding_list); + block->net = net; + block->sw = sw; + + return block; +} + static void prestera_flow_block_release(void *cb_priv) { struct prestera_flow_block *block = cb_priv; - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); +} + +static bool +prestera_flow_block_is_bound(const struct prestera_flow_block *block) +{ + return block->ruleset_zero; +} + +static struct prestera_flow_block_binding * +prestera_flow_block_lookup(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + list_for_each_entry(binding, &block->binding_list, list) + if (binding->port == port) + return binding; + + return NULL; +} + +static int prestera_flow_block_bind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + int err; + + binding = kzalloc(sizeof(*binding), GFP_KERNEL); + if (!binding) + return -ENOMEM; + + binding->span_id = PRESTERA_SPAN_INVALID_ID; + binding->port = port; + + if (prestera_flow_block_is_bound(block)) { + err = prestera_acl_ruleset_bind(block->ruleset_zero, port); + if (err) + goto err_ruleset_bind; + } + + list_add(&binding->list, &block->binding_list); + return 0; + +err_ruleset_bind: + kfree(binding); + return err; +} + +static int prestera_flow_block_unbind(struct prestera_flow_block *block, + struct prestera_port *port) +{ + struct prestera_flow_block_binding *binding; + + binding = prestera_flow_block_lookup(block, port); + if (!binding) + return -ENOENT; + + list_del(&binding->list); + + if (prestera_flow_block_is_bound(block)) + prestera_acl_ruleset_unbind(block->ruleset_zero, port); + + kfree(binding); + return 0; } static struct prestera_flow_block * @@ -78,7 +167,7 @@ prestera_flow_block_get(struct prestera_switch *sw, block_cb = flow_block_cb_lookup(f->block, prestera_flow_block_cb, sw); if (!block_cb) { - block = prestera_acl_block_create(sw, f->net); + block = prestera_flow_block_create(sw, f->net); if (!block) return ERR_PTR(-ENOMEM); @@ -86,7 +175,7 @@ prestera_flow_block_get(struct prestera_switch *sw, sw, block, prestera_flow_block_release); if (IS_ERR(block_cb)) { - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); return ERR_CAST(block_cb); } @@ -110,7 +199,7 @@ static void prestera_flow_block_put(struct prestera_flow_block *block) return; flow_block_cb_free(block_cb); - prestera_acl_block_destroy(block); + prestera_flow_block_destroy(block); } static int prestera_setup_flow_block_bind(struct prestera_port *port, @@ -128,7 +217,7 @@ static int prestera_setup_flow_block_bind(struct prestera_port *port, block_cb = block->block_cb; - err = prestera_acl_block_bind(block, port); + err = prestera_flow_block_bind(block, port); if (err) goto err_block_bind; @@ -162,7 +251,7 @@ static void prestera_setup_flow_block_unbind(struct prestera_port *port, prestera_span_destroy(block); - err = prestera_acl_block_unbind(block, port); + err = prestera_flow_block_unbind(block, port); if (err) goto error; diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flow.h b/drivers/net/ethernet/marvell/prestera/prestera_flow.h index 467c7038cace..5863acf06005 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flow.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flow.h @@ -7,6 +7,22 @@ #include <net/flow_offload.h> struct prestera_port; +struct prestera_switch; + +struct prestera_flow_block_binding { + struct list_head list; + struct prestera_port *port; + int span_id; +}; + +struct prestera_flow_block { + struct list_head binding_list; + struct prestera_switch *sw; + unsigned int rule_count; + struct net *net; + struct prestera_acl_ruleset *ruleset_zero; + struct flow_block_cb *block_cb; +}; int prestera_flow_block_setup(struct prestera_port *port, struct flow_block_offload *f); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.c b/drivers/net/ethernet/marvell/prestera/prestera_flower.c index e571ba09ec08..c1dc4e49b07f 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.c @@ -3,6 +3,7 @@ #include "prestera.h" #include "prestera_acl.h" +#include "prestera_flow.h" #include "prestera_flower.h" static int prestera_flower_parse_actions(struct prestera_flow_block *block, @@ -10,35 +11,38 @@ static int prestera_flower_parse_actions(struct prestera_flow_block *block, struct flow_action *flow_action, struct netlink_ext_ack *extack) { - struct prestera_acl_rule_action_entry a_entry; const struct flow_action_entry *act; - int err, i; + int i; + /* whole struct (rule->re_arg) must be initialized with 0 */ if (!flow_action_has_entries(flow_action)) return 0; flow_action_for_each(i, act, flow_action) { - memset(&a_entry, 0, sizeof(a_entry)); - switch (act->id) { case FLOW_ACTION_ACCEPT: - a_entry.id = PRESTERA_ACL_RULE_ACTION_ACCEPT; + if (rule->re_arg.accept.valid) + return -EEXIST; + + rule->re_arg.accept.valid = 1; break; case FLOW_ACTION_DROP: - a_entry.id = PRESTERA_ACL_RULE_ACTION_DROP; + if (rule->re_arg.drop.valid) + return -EEXIST; + + rule->re_arg.drop.valid = 1; break; case FLOW_ACTION_TRAP: - a_entry.id = PRESTERA_ACL_RULE_ACTION_TRAP; + if (rule->re_arg.trap.valid) + return -EEXIST; + + rule->re_arg.trap.valid = 1; break; default: NL_SET_ERR_MSG_MOD(extack, "Unsupported action"); pr_err("Unsupported action\n"); return -EOPNOTSUPP; } - - err = prestera_acl_rule_action_add(rule, &a_entry); - if (err) - return err; } return 0; @@ -47,12 +51,12 @@ static int prestera_flower_parse_actions(struct prestera_flow_block *block, static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, struct flow_cls_offload *f, struct prestera_flow_block *block) -{ - struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); - struct prestera_acl_rule_match_entry m_entry = {0}; +{ struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); + struct prestera_acl_match *r_match = &rule->re_key.match; + struct prestera_port *port; struct net_device *ingress_dev; struct flow_match_meta match; - struct prestera_port *port; + __be16 key, mask; flow_rule_match_meta(f_rule, &match); if (match.mask->ingress_ifindex != 0xFFFFFFFF) { @@ -61,7 +65,7 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, return -EINVAL; } - ingress_dev = __dev_get_by_index(prestera_acl_block_net(block), + ingress_dev = __dev_get_by_index(block->net, match.key->ingress_ifindex); if (!ingress_dev) { NL_SET_ERR_MSG_MOD(f->common.extack, @@ -76,22 +80,28 @@ static int prestera_flower_parse_meta(struct prestera_acl_rule *rule, } port = netdev_priv(ingress_dev); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT; - m_entry.keymask.u64.key = port->hw_id | ((u64)port->dev_id << 32); - m_entry.keymask.u64.mask = ~(u64)0; + mask = htons(0x1FFF); + key = htons(port->hw_id); + rule_match_set(r_match->key, SYS_PORT, key); + rule_match_set(r_match->mask, SYS_PORT, mask); + + mask = htons(0x1FF); + key = htons(port->dev_id); + rule_match_set(r_match->key, SYS_DEV, key); + rule_match_set(r_match->mask, SYS_DEV, mask); + + return 0; - return prestera_acl_rule_match_add(rule, &m_entry); } static int prestera_flower_parse(struct prestera_flow_block *block, struct prestera_acl_rule *rule, struct flow_cls_offload *f) -{ - struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); +{ struct flow_rule *f_rule = flow_cls_offload_flow_rule(f); struct flow_dissector *dissector = f_rule->match.dissector; - struct prestera_acl_rule_match_entry m_entry; - u16 n_proto_mask = 0; - u16 n_proto_key = 0; + struct prestera_acl_match *r_match = &rule->re_key.match; + __be16 n_proto_mask = 0; + __be16 n_proto_key = 0; u16 addr_type = 0; u8 ip_proto = 0; int err; @@ -129,32 +139,19 @@ static int prestera_flower_parse(struct prestera_flow_block *block, struct flow_match_basic match; flow_rule_match_basic(f_rule, &match); - n_proto_key = ntohs(match.key->n_proto); - n_proto_mask = ntohs(match.mask->n_proto); + n_proto_key = match.key->n_proto; + n_proto_mask = match.mask->n_proto; - if (n_proto_key == ETH_P_ALL) { + if (ntohs(match.key->n_proto) == ETH_P_ALL) { n_proto_key = 0; n_proto_mask = 0; } - /* add eth type key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE; - m_entry.keymask.u16.key = n_proto_key; - m_entry.keymask.u16.mask = n_proto_mask; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; - - /* add ip proto key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO; - m_entry.keymask.u8.key = match.key->ip_proto; - m_entry.keymask.u8.mask = match.mask->ip_proto; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ETH_TYPE, n_proto_key); + rule_match_set(r_match->mask, ETH_TYPE, n_proto_mask); + rule_match_set(r_match->key, IP_PROTO, match.key->ip_proto); + rule_match_set(r_match->mask, IP_PROTO, match.mask->ip_proto); ip_proto = match.key->ip_proto; } @@ -163,27 +160,27 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_eth_addrs(f_rule, &match); - /* add ethernet dst key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC; - memcpy(&m_entry.keymask.mac.key, - &match.key->dst, sizeof(match.key->dst)); - memcpy(&m_entry.keymask.mac.mask, - &match.mask->dst, sizeof(match.mask->dst)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; - - /* add ethernet src key,mask */ - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC; - memcpy(&m_entry.keymask.mac.key, - &match.key->src, sizeof(match.key->src)); - memcpy(&m_entry.keymask.mac.mask, - &match.mask->src, sizeof(match.mask->src)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + /* DA key, mask */ + rule_match_set_n(r_match->key, + ETH_DMAC_0, &match.key->dst[0], 4); + rule_match_set_n(r_match->key, + ETH_DMAC_1, &match.key->dst[4], 2); + + rule_match_set_n(r_match->mask, + ETH_DMAC_0, &match.mask->dst[0], 4); + rule_match_set_n(r_match->mask, + ETH_DMAC_1, &match.mask->dst[4], 2); + + /* SA key, mask */ + rule_match_set_n(r_match->key, + ETH_SMAC_0, &match.key->src[0], 4); + rule_match_set_n(r_match->key, + ETH_SMAC_1, &match.key->src[4], 2); + + rule_match_set_n(r_match->mask, + ETH_SMAC_0, &match.mask->src[0], 4); + rule_match_set_n(r_match->mask, + ETH_SMAC_1, &match.mask->src[4], 2); } if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { @@ -191,25 +188,11 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_ipv4_addrs(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC; - memcpy(&m_entry.keymask.u32.key, - &match.key->src, sizeof(match.key->src)); - memcpy(&m_entry.keymask.u32.mask, - &match.mask->src, sizeof(match.mask->src)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, IP_SRC, match.key->src); + rule_match_set(r_match->mask, IP_SRC, match.mask->src); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST; - memcpy(&m_entry.keymask.u32.key, - &match.key->dst, sizeof(match.key->dst)); - memcpy(&m_entry.keymask.u32.mask, - &match.mask->dst, sizeof(match.mask->dst)); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, IP_DST, match.key->dst); + rule_match_set(r_match->mask, IP_DST, match.mask->dst); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_PORTS)) { @@ -224,21 +207,11 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_ports(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC; - m_entry.keymask.u16.key = ntohs(match.key->src); - m_entry.keymask.u16.mask = ntohs(match.mask->src); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, L4_PORT_SRC, match.key->src); + rule_match_set(r_match->mask, L4_PORT_SRC, match.mask->src); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST; - m_entry.keymask.u16.key = ntohs(match.key->dst); - m_entry.keymask.u16.mask = ntohs(match.mask->dst); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, L4_PORT_DST, match.key->dst); + rule_match_set(r_match->mask, L4_PORT_DST, match.mask->dst); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_VLAN)) { @@ -247,22 +220,15 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_vlan(f_rule, &match); if (match.mask->vlan_id != 0) { - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID; - m_entry.keymask.u16.key = match.key->vlan_id; - m_entry.keymask.u16.mask = match.mask->vlan_id; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + __be16 key = cpu_to_be16(match.key->vlan_id); + __be16 mask = cpu_to_be16(match.mask->vlan_id); + + rule_match_set(r_match->key, VLAN_ID, key); + rule_match_set(r_match->mask, VLAN_ID, mask); } - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID; - m_entry.keymask.u16.key = ntohs(match.key->vlan_tpid); - m_entry.keymask.u16.mask = ntohs(match.mask->vlan_tpid); - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, VLAN_TPID, match.key->vlan_tpid); + rule_match_set(r_match->mask, VLAN_TPID, match.mask->vlan_tpid); } if (flow_rule_match_key(f_rule, FLOW_DISSECTOR_KEY_ICMP)) { @@ -270,90 +236,109 @@ static int prestera_flower_parse(struct prestera_flow_block *block, flow_rule_match_icmp(f_rule, &match); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE; - m_entry.keymask.u8.key = match.key->type; - m_entry.keymask.u8.mask = match.mask->type; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ICMP_TYPE, match.key->type); + rule_match_set(r_match->mask, ICMP_TYPE, match.mask->type); - memset(&m_entry, 0, sizeof(m_entry)); - m_entry.type = PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE; - m_entry.keymask.u8.key = match.key->code; - m_entry.keymask.u8.mask = match.mask->code; - err = prestera_acl_rule_match_add(rule, &m_entry); - if (err) - return err; + rule_match_set(r_match->key, ICMP_CODE, match.key->code); + rule_match_set(r_match->mask, ICMP_CODE, match.mask->code); } - return prestera_flower_parse_actions(block, rule, - &f->rule->action, + return prestera_flower_parse_actions(block, rule, &f->rule->action, f->common.extack); } int prestera_flower_replace(struct prestera_flow_block *block, struct flow_cls_offload *f) { - struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_ruleset *ruleset; + struct prestera_acl *acl = block->sw->acl; struct prestera_acl_rule *rule; int err; - rule = prestera_acl_rule_create(block, f->cookie); - if (IS_ERR(rule)) - return PTR_ERR(rule); + ruleset = prestera_acl_ruleset_get(acl, block); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + /* increments the ruleset reference */ + rule = prestera_acl_rule_create(ruleset, f->cookie); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + goto err_rule_create; + } err = prestera_flower_parse(block, rule, f); if (err) - goto err_flower_parse; + goto err_rule_add; - err = prestera_acl_rule_add(sw, rule); + if (!prestera_acl_ruleset_is_offload(ruleset)) { + err = prestera_acl_ruleset_offload(ruleset); + if (err) + goto err_ruleset_offload; + } + + err = prestera_acl_rule_add(block->sw, rule); if (err) goto err_rule_add; + prestera_acl_ruleset_put(ruleset); return 0; +err_ruleset_offload: err_rule_add: -err_flower_parse: prestera_acl_rule_destroy(rule); +err_rule_create: + prestera_acl_ruleset_put(ruleset); return err; } void prestera_flower_destroy(struct prestera_flow_block *block, struct flow_cls_offload *f) { + struct prestera_acl_ruleset *ruleset; struct prestera_acl_rule *rule; - struct prestera_switch *sw; - rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), - f->cookie); + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block); + if (IS_ERR(ruleset)) + return; + + rule = prestera_acl_rule_lookup(ruleset, f->cookie); if (rule) { - sw = prestera_acl_block_sw(block); - prestera_acl_rule_del(sw, rule); + prestera_acl_rule_del(block->sw, rule); prestera_acl_rule_destroy(rule); } + prestera_acl_ruleset_put(ruleset); + } int prestera_flower_stats(struct prestera_flow_block *block, struct flow_cls_offload *f) { - struct prestera_switch *sw = prestera_acl_block_sw(block); + struct prestera_acl_ruleset *ruleset; struct prestera_acl_rule *rule; u64 packets; u64 lastuse; u64 bytes; int err; - rule = prestera_acl_rule_lookup(prestera_acl_block_ruleset_get(block), - f->cookie); - if (!rule) - return -EINVAL; + ruleset = prestera_acl_ruleset_lookup(block->sw->acl, block); + if (IS_ERR(ruleset)) + return PTR_ERR(ruleset); + + rule = prestera_acl_rule_lookup(ruleset, f->cookie); + if (!rule) { + err = -EINVAL; + goto err_rule_get_stats; + } - err = prestera_acl_rule_get_stats(sw, rule, &packets, &bytes, &lastuse); + err = prestera_acl_rule_get_stats(block->sw->acl, rule, &packets, + &bytes, &lastuse); if (err) - return err; + goto err_rule_get_stats; flow_stats_update(&f->stats, bytes, packets, 0, lastuse, - FLOW_ACTION_HW_STATS_IMMEDIATE); - return 0; + FLOW_ACTION_HW_STATS_DELAYED); + +err_rule_get_stats: + prestera_acl_ruleset_put(ruleset); + return err; } diff --git a/drivers/net/ethernet/marvell/prestera/prestera_flower.h b/drivers/net/ethernet/marvell/prestera/prestera_flower.h index 91e045eec58b..c6182473efa5 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_flower.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_flower.h @@ -1,11 +1,12 @@ /* SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 */ -/* Copyright (c) 2020 Marvell International Ltd. All rights reserved. */ +/* Copyright (c) 2020-2021 Marvell International Ltd. All rights reserved. */ #ifndef _PRESTERA_FLOWER_H_ #define _PRESTERA_FLOWER_H_ #include <net/pkt_cls.h> +struct prestera_switch; struct prestera_flow_block; int prestera_flower_replace(struct prestera_flow_block *block, diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.c b/drivers/net/ethernet/marvell/prestera/prestera_hw.c index 9b8b1ed474fc..6282c9822e2b 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.c @@ -9,6 +9,7 @@ #include "prestera.h" #include "prestera_hw.h" #include "prestera_acl.h" +#include "prestera_counter.h" #define PRESTERA_SWITCH_INIT_TIMEOUT_MS (30 * 1000) @@ -38,13 +39,19 @@ enum prestera_cmd_type_t { PRESTERA_CMD_TYPE_BRIDGE_PORT_ADD = 0x402, PRESTERA_CMD_TYPE_BRIDGE_PORT_DELETE = 0x403, - PRESTERA_CMD_TYPE_ACL_RULE_ADD = 0x500, - PRESTERA_CMD_TYPE_ACL_RULE_DELETE = 0x501, - PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET = 0x510, - PRESTERA_CMD_TYPE_ACL_RULESET_CREATE = 0x520, - PRESTERA_CMD_TYPE_ACL_RULESET_DELETE = 0x521, - PRESTERA_CMD_TYPE_ACL_PORT_BIND = 0x530, - PRESTERA_CMD_TYPE_ACL_PORT_UNBIND = 0x531, + PRESTERA_CMD_TYPE_COUNTER_GET = 0x510, + PRESTERA_CMD_TYPE_COUNTER_ABORT = 0x511, + PRESTERA_CMD_TYPE_COUNTER_TRIGGER = 0x512, + PRESTERA_CMD_TYPE_COUNTER_BLOCK_GET = 0x513, + PRESTERA_CMD_TYPE_COUNTER_BLOCK_RELEASE = 0x514, + PRESTERA_CMD_TYPE_COUNTER_CLEAR = 0x515, + + PRESTERA_CMD_TYPE_VTCAM_CREATE = 0x540, + PRESTERA_CMD_TYPE_VTCAM_DESTROY = 0x541, + PRESTERA_CMD_TYPE_VTCAM_RULE_ADD = 0x550, + PRESTERA_CMD_TYPE_VTCAM_RULE_DELETE = 0x551, + PRESTERA_CMD_TYPE_VTCAM_IFACE_BIND = 0x560, + PRESTERA_CMD_TYPE_VTCAM_IFACE_UNBIND = 0x561, PRESTERA_CMD_TYPE_RXTX_INIT = 0x800, @@ -359,76 +366,84 @@ struct prestera_msg_bridge_resp { u8 pad[2]; }; -struct prestera_msg_acl_action { - __le32 id; - __le32 reserved[5]; +struct prestera_msg_vtcam_create_req { + struct prestera_msg_cmd cmd; + __le32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + u8 direction; + u8 lookup; + u8 pad[2]; }; -struct prestera_msg_acl_match { - __le32 type; - __le32 __reserved; - union { - struct { - u8 key; - u8 mask; - } u8; - struct { - __le16 key; - __le16 mask; - } u16; - struct { - __le32 key; - __le32 mask; - } u32; - struct { - __le64 key; - __le64 mask; - } u64; - struct { - u8 key[ETH_ALEN]; - u8 mask[ETH_ALEN]; - } mac; - } keymask; +struct prestera_msg_vtcam_destroy_req { + struct prestera_msg_cmd cmd; + __le32 vtcam_id; }; -struct prestera_msg_acl_rule_req { +struct prestera_msg_vtcam_rule_add_req { struct prestera_msg_cmd cmd; - __le32 id; - __le32 priority; - __le16 ruleset_id; - u8 n_actions; - u8 n_matches; + __le32 key[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __le32 keymask[__PRESTERA_ACL_RULE_MATCH_TYPE_MAX]; + __le32 vtcam_id; + __le32 prio; + __le32 n_act; }; -struct prestera_msg_acl_rule_resp { - struct prestera_msg_ret ret; +struct prestera_msg_vtcam_rule_del_req { + struct prestera_msg_cmd cmd; + __le32 vtcam_id; __le32 id; }; -struct prestera_msg_acl_rule_stats_resp { +struct prestera_msg_vtcam_bind_req { + struct prestera_msg_cmd cmd; + union { + struct { + __le32 hw_id; + __le32 dev_id; + } port; + __le32 index; + }; + __le32 vtcam_id; + __le16 pcl_id; + __le16 type; +}; + +struct prestera_msg_vtcam_resp { struct prestera_msg_ret ret; - __le64 packets; - __le64 bytes; + __le32 vtcam_id; + __le32 rule_id; }; -struct prestera_msg_acl_ruleset_bind_req { - struct prestera_msg_cmd cmd; - __le32 port; - __le32 dev; - __le16 ruleset_id; - u8 pad[2]; +struct prestera_msg_acl_action { + __le32 id; + __le32 __reserved; + union { + struct { + __le32 id; + } count; + __le32 reserved[6]; + }; }; -struct prestera_msg_acl_ruleset_req { +struct prestera_msg_counter_req { struct prestera_msg_cmd cmd; - __le16 id; - u8 pad[2]; + __le32 client; + __le32 block_id; + __le32 num_counters; }; -struct prestera_msg_acl_ruleset_resp { +struct prestera_msg_counter_stats { + __le64 packets; + __le64 bytes; +}; + +struct prestera_msg_counter_resp { struct prestera_msg_ret ret; - __le16 id; - u8 pad[2]; + __le32 block_id; + __le32 offset; + __le32 num_counters; + __le32 done; + struct prestera_msg_counter_stats stats[]; }; struct prestera_msg_span_req { @@ -521,14 +536,19 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_vlan_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_fdb_req) != 28); BUILD_BUG_ON(sizeof(struct prestera_msg_bridge_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_bind_req) != 16); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_span_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_stp_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_rxtx_req) != 8); BUILD_BUG_ON(sizeof(struct prestera_msg_lag_req) != 16); BUILD_BUG_ON(sizeof(struct prestera_msg_cpu_code_counter_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_create_req) != 84); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_destroy_req) != 8); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_rule_add_req) != 168); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_rule_del_req) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_bind_req) != 20); + BUILD_BUG_ON(sizeof(struct prestera_msg_acl_action) != 32); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_req) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_stats) != 16); /* check responses */ BUILD_BUG_ON(sizeof(struct prestera_msg_common_resp) != 8); @@ -537,11 +557,10 @@ static void prestera_hw_build_tests(void) BUILD_BUG_ON(sizeof(struct prestera_msg_port_stats_resp) != 248); BUILD_BUG_ON(sizeof(struct prestera_msg_port_info_resp) != 20); BUILD_BUG_ON(sizeof(struct prestera_msg_bridge_resp) != 12); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_resp) != 12); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_rule_stats_resp) != 24); - BUILD_BUG_ON(sizeof(struct prestera_msg_acl_ruleset_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_span_resp) != 12); BUILD_BUG_ON(sizeof(struct prestera_msg_rxtx_resp) != 12); + BUILD_BUG_ON(sizeof(struct prestera_msg_vtcam_resp) != 16); + BUILD_BUG_ON(sizeof(struct prestera_msg_counter_resp) != 24); /* check events */ BUILD_BUG_ON(sizeof(struct prestera_msg_event_port) != 20); @@ -1044,225 +1063,162 @@ static void prestera_hw_remote_fc_to_eth(u8 fc, bool *pause, bool *asym_pause) } } -int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, u16 *ruleset_id) +int prestera_hw_vtcam_create(struct prestera_switch *sw, + u8 lookup, const u32 *keymask, u32 *vtcam_id, + enum prestera_hw_vtcam_direction_t dir) { - struct prestera_msg_acl_ruleset_resp resp; - struct prestera_msg_acl_ruleset_req req; int err; + struct prestera_msg_vtcam_resp resp; + struct prestera_msg_vtcam_create_req req = { + .lookup = lookup, + .direction = dir, + }; - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULESET_CREATE, + if (keymask) + memcpy(req.keymask, keymask, sizeof(req.keymask)); + else + memset(req.keymask, 0, sizeof(req.keymask)); + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_VTCAM_CREATE, &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); if (err) return err; - *ruleset_id = __le16_to_cpu(resp.id); - + *vtcam_id = __le32_to_cpu(resp.vtcam_id); return 0; } -int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, u16 ruleset_id) +int prestera_hw_vtcam_destroy(struct prestera_switch *sw, u32 vtcam_id) { - struct prestera_msg_acl_ruleset_req req = { - .id = __cpu_to_le16(ruleset_id), + struct prestera_msg_vtcam_destroy_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), }; - return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULESET_DELETE, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_DESTROY, &req.cmd, sizeof(req)); } -static int prestera_hw_acl_actions_put(struct prestera_msg_acl_action *action, - struct prestera_acl_rule *rule) +static int +prestera_acl_rule_add_put_action(struct prestera_msg_acl_action *action, + struct prestera_acl_hw_action_info *info) { - struct list_head *a_list = prestera_acl_rule_action_list_get(rule); - struct prestera_acl_rule_action_entry *a_entry; - int i = 0; - - list_for_each_entry(a_entry, a_list, list) { - action[i].id = __cpu_to_le32(a_entry->id); - - switch (a_entry->id) { - case PRESTERA_ACL_RULE_ACTION_ACCEPT: - case PRESTERA_ACL_RULE_ACTION_DROP: - case PRESTERA_ACL_RULE_ACTION_TRAP: - /* just rule action id, no specific data */ - break; - default: - return -EINVAL; - } - - i++; - } - - return 0; -} + action->id = __cpu_to_le32(info->id); -static int prestera_hw_acl_matches_put(struct prestera_msg_acl_match *match, - struct prestera_acl_rule *rule) -{ - struct list_head *m_list = prestera_acl_rule_match_list_get(rule); - struct prestera_acl_rule_match_entry *m_entry; - int i = 0; - - list_for_each_entry(m_entry, m_list, list) { - match[i].type = __cpu_to_le32(m_entry->type); - - switch (m_entry->type) { - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_TYPE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_DST: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_ID: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_VLAN_TPID: - match[i].keymask.u16.key = - __cpu_to_le16(m_entry->keymask.u16.key); - match[i].keymask.u16.mask = - __cpu_to_le16(m_entry->keymask.u16.mask); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_TYPE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ICMP_CODE: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_PROTO: - match[i].keymask.u8.key = m_entry->keymask.u8.key; - match[i].keymask.u8.mask = m_entry->keymask.u8.mask; - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_SMAC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_ETH_DMAC: - memcpy(match[i].keymask.mac.key, - m_entry->keymask.mac.key, - sizeof(match[i].keymask.mac.key)); - memcpy(match[i].keymask.mac.mask, - m_entry->keymask.mac.mask, - sizeof(match[i].keymask.mac.mask)); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_IP_DST: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_SRC: - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_L4_PORT_RANGE_DST: - match[i].keymask.u32.key = - __cpu_to_le32(m_entry->keymask.u32.key); - match[i].keymask.u32.mask = - __cpu_to_le32(m_entry->keymask.u32.mask); - break; - case PRESTERA_ACL_RULE_MATCH_ENTRY_TYPE_PORT: - match[i].keymask.u64.key = - __cpu_to_le64(m_entry->keymask.u64.key); - match[i].keymask.u64.mask = - __cpu_to_le64(m_entry->keymask.u64.mask); - break; - default: - return -EINVAL; - } - - i++; + switch (info->id) { + case PRESTERA_ACL_RULE_ACTION_ACCEPT: + case PRESTERA_ACL_RULE_ACTION_DROP: + case PRESTERA_ACL_RULE_ACTION_TRAP: + /* just rule action id, no specific data */ + break; + case PRESTERA_ACL_RULE_ACTION_COUNT: + action->count.id = __cpu_to_le32(info->count.id); + break; + default: + return -EINVAL; } return 0; } -int prestera_hw_acl_rule_add(struct prestera_switch *sw, - struct prestera_acl_rule *rule, - u32 *rule_id) +int prestera_hw_vtcam_rule_add(struct prestera_switch *sw, + u32 vtcam_id, u32 prio, void *key, void *keymask, + struct prestera_acl_hw_action_info *act, + u8 n_act, u32 *rule_id) { - struct prestera_msg_acl_action *actions; - struct prestera_msg_acl_match *matches; - struct prestera_msg_acl_rule_resp resp; - struct prestera_msg_acl_rule_req *req; - u8 n_actions; - u8 n_matches; + struct prestera_msg_acl_action *actions_msg; + struct prestera_msg_vtcam_rule_add_req *req; + struct prestera_msg_vtcam_resp resp; void *buff; u32 size; int err; + u8 i; - n_actions = prestera_acl_rule_action_len(rule); - n_matches = prestera_acl_rule_match_len(rule); - - size = sizeof(*req) + sizeof(*actions) * n_actions + - sizeof(*matches) * n_matches; + size = sizeof(*req) + sizeof(*actions_msg) * n_act; buff = kzalloc(size, GFP_KERNEL); if (!buff) return -ENOMEM; req = buff; - actions = buff + sizeof(*req); - matches = buff + sizeof(*req) + sizeof(*actions) * n_actions; - - /* put acl actions into the message */ - err = prestera_hw_acl_actions_put(actions, rule); - if (err) - goto free_buff; + req->n_act = __cpu_to_le32(n_act); + actions_msg = buff + sizeof(*req); /* put acl matches into the message */ - err = prestera_hw_acl_matches_put(matches, rule); - if (err) - goto free_buff; + memcpy(req->key, key, sizeof(req->key)); + memcpy(req->keymask, keymask, sizeof(req->keymask)); - req->ruleset_id = __cpu_to_le16(prestera_acl_rule_ruleset_id_get(rule)); - req->priority = __cpu_to_le32(prestera_acl_rule_priority_get(rule)); - req->n_actions = prestera_acl_rule_action_len(rule); - req->n_matches = prestera_acl_rule_match_len(rule); + /* put acl actions into the message */ + for (i = 0; i < n_act; i++) { + err = prestera_acl_rule_add_put_action(&actions_msg[i], + &act[i]); + if (err) + goto free_buff; + } - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_ADD, + req->vtcam_id = __cpu_to_le32(vtcam_id); + req->prio = __cpu_to_le32(prio); + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_VTCAM_RULE_ADD, &req->cmd, size, &resp.ret, sizeof(resp)); if (err) goto free_buff; - *rule_id = __le32_to_cpu(resp.id); + *rule_id = __le32_to_cpu(resp.rule_id); free_buff: kfree(buff); return err; } -int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id) +int prestera_hw_vtcam_rule_del(struct prestera_switch *sw, + u32 vtcam_id, u32 rule_id) { - struct prestera_msg_acl_rule_req req = { + struct prestera_msg_vtcam_rule_del_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), .id = __cpu_to_le32(rule_id) }; - return prestera_cmd(sw, PRESTERA_CMD_TYPE_ACL_RULE_DELETE, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_RULE_DELETE, &req.cmd, sizeof(req)); } -int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, u32 rule_id, - u64 *packets, u64 *bytes) +int prestera_hw_vtcam_iface_bind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id, u16 pcl_id) { - struct prestera_msg_acl_rule_stats_resp resp; - struct prestera_msg_acl_rule_req req = { - .id = __cpu_to_le32(rule_id) + struct prestera_msg_vtcam_bind_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), + .type = __cpu_to_le16(iface->type), + .pcl_id = __cpu_to_le16(pcl_id) }; - int err; - - err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_ACL_RULE_STATS_GET, - &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); - if (err) - return err; - - *packets = __le64_to_cpu(resp.packets); - *bytes = __le64_to_cpu(resp.bytes); - - return 0; -} -int prestera_hw_acl_port_bind(const struct prestera_port *port, u16 ruleset_id) -{ - struct prestera_msg_acl_ruleset_bind_req req = { - .port = __cpu_to_le32(port->hw_id), - .dev = __cpu_to_le32(port->dev_id), - .ruleset_id = __cpu_to_le16(ruleset_id), - }; + if (iface->type == PRESTERA_ACL_IFACE_TYPE_PORT) { + req.port.dev_id = __cpu_to_le32(iface->port->dev_id); + req.port.hw_id = __cpu_to_le32(iface->port->hw_id); + } else { + req.index = __cpu_to_le32(iface->index); + } - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_BIND, + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_IFACE_BIND, &req.cmd, sizeof(req)); } -int prestera_hw_acl_port_unbind(const struct prestera_port *port, - u16 ruleset_id) +int prestera_hw_vtcam_iface_unbind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id) { - struct prestera_msg_acl_ruleset_bind_req req = { - .port = __cpu_to_le32(port->hw_id), - .dev = __cpu_to_le32(port->dev_id), - .ruleset_id = __cpu_to_le16(ruleset_id), + struct prestera_msg_vtcam_bind_req req = { + .vtcam_id = __cpu_to_le32(vtcam_id), + .type = __cpu_to_le16(iface->type) }; - return prestera_cmd(port->sw, PRESTERA_CMD_TYPE_ACL_PORT_UNBIND, + if (iface->type == PRESTERA_ACL_IFACE_TYPE_PORT) { + req.port.dev_id = __cpu_to_le32(iface->port->dev_id); + req.port.hw_id = __cpu_to_le32(iface->port->hw_id); + } else { + req.index = __cpu_to_le32(iface->index); + } + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_VTCAM_IFACE_UNBIND, &req.cmd, sizeof(req)); } @@ -1916,3 +1872,100 @@ void prestera_hw_event_handler_unregister(struct prestera_switch *sw, list_del_rcu(&eh->list); kfree_rcu(eh, rcu); } + +int prestera_hw_counter_trigger(struct prestera_switch *sw, u32 block_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_TRIGGER, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counter_abort(struct prestera_switch *sw) +{ + struct prestera_msg_counter_req req; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_ABORT, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counters_get(struct prestera_switch *sw, u32 idx, + u32 *len, bool *done, + struct prestera_counter_stats *stats) +{ + struct prestera_msg_counter_resp *resp; + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(idx), + .num_counters = __cpu_to_le32(*len), + }; + size_t size = struct_size(resp, stats, *len); + int err, i; + + resp = kmalloc(size, GFP_KERNEL); + if (!resp) + return -ENOMEM; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_COUNTER_GET, + &req.cmd, sizeof(req), &resp->ret, size); + if (err) + goto free_buff; + + for (i = 0; i < __le32_to_cpu(resp->num_counters); i++) { + stats[i].packets += __le64_to_cpu(resp->stats[i].packets); + stats[i].bytes += __le64_to_cpu(resp->stats[i].bytes); + } + + *len = __le32_to_cpu(resp->num_counters); + *done = __le32_to_cpu(resp->done); + +free_buff: + kfree(resp); + return err; +} + +int prestera_hw_counter_block_get(struct prestera_switch *sw, + u32 client, u32 *block_id, u32 *offset, + u32 *num_counters) +{ + struct prestera_msg_counter_resp resp; + struct prestera_msg_counter_req req = { + .client = __cpu_to_le32(client) + }; + int err; + + err = prestera_cmd_ret(sw, PRESTERA_CMD_TYPE_COUNTER_BLOCK_GET, + &req.cmd, sizeof(req), &resp.ret, sizeof(resp)); + if (err) + return err; + + *block_id = __le32_to_cpu(resp.block_id); + *offset = __le32_to_cpu(resp.offset); + *num_counters = __le32_to_cpu(resp.num_counters); + + return 0; +} + +int prestera_hw_counter_block_release(struct prestera_switch *sw, + u32 block_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_BLOCK_RELEASE, + &req.cmd, sizeof(req)); +} + +int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, + u32 counter_id) +{ + struct prestera_msg_counter_req req = { + .block_id = __cpu_to_le32(block_id), + .num_counters = __cpu_to_le32(counter_id) + }; + + return prestera_cmd(sw, PRESTERA_CMD_TYPE_COUNTER_CLEAR, + &req.cmd, sizeof(req)); +} diff --git a/drivers/net/ethernet/marvell/prestera/prestera_hw.h b/drivers/net/ethernet/marvell/prestera/prestera_hw.h index 57a3c2e5b112..0496e454e148 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_hw.h +++ b/drivers/net/ethernet/marvell/prestera/prestera_hw.h @@ -5,6 +5,7 @@ #define _PRESTERA_HW_H_ #include <linux/types.h> +#include "prestera_acl.h" enum prestera_accept_frm_type { PRESTERA_ACCEPT_FRAME_TYPE_TAGGED, @@ -111,18 +112,31 @@ enum prestera_hw_cpu_code_cnt_t { PRESTERA_HW_CPU_CODE_CNT_TYPE_TRAP = 1, }; +enum prestera_hw_vtcam_direction_t { + PRESTERA_HW_VTCAM_DIR_INGRESS = 0, + PRESTERA_HW_VTCAM_DIR_EGRESS = 1, +}; + +enum { + PRESTERA_HW_COUNTER_CLIENT_LOOKUP_0 = 0, + PRESTERA_HW_COUNTER_CLIENT_LOOKUP_1 = 1, + PRESTERA_HW_COUNTER_CLIENT_LOOKUP_2 = 2, +}; + struct prestera_switch; struct prestera_port; struct prestera_port_stats; struct prestera_port_caps; enum prestera_event_type; struct prestera_event; -struct prestera_acl_rule; typedef void (*prestera_event_cb_t) (struct prestera_switch *sw, struct prestera_event *evt, void *arg); struct prestera_rxtx_params; +struct prestera_acl_hw_action_info; +struct prestera_acl_iface; +struct prestera_counter_stats; /* Switch API */ int prestera_hw_switch_init(struct prestera_switch *sw); @@ -186,21 +200,37 @@ int prestera_hw_bridge_delete(struct prestera_switch *sw, u16 bridge_id); int prestera_hw_bridge_port_add(struct prestera_port *port, u16 bridge_id); int prestera_hw_bridge_port_delete(struct prestera_port *port, u16 bridge_id); -/* ACL API */ -int prestera_hw_acl_ruleset_create(struct prestera_switch *sw, - u16 *ruleset_id); -int prestera_hw_acl_ruleset_del(struct prestera_switch *sw, - u16 ruleset_id); -int prestera_hw_acl_rule_add(struct prestera_switch *sw, - struct prestera_acl_rule *rule, - u32 *rule_id); -int prestera_hw_acl_rule_del(struct prestera_switch *sw, u32 rule_id); -int prestera_hw_acl_rule_stats_get(struct prestera_switch *sw, - u32 rule_id, u64 *packets, u64 *bytes); -int prestera_hw_acl_port_bind(const struct prestera_port *port, - u16 ruleset_id); -int prestera_hw_acl_port_unbind(const struct prestera_port *port, - u16 ruleset_id); +/* vTCAM API */ +int prestera_hw_vtcam_create(struct prestera_switch *sw, + u8 lookup, const u32 *keymask, u32 *vtcam_id, + enum prestera_hw_vtcam_direction_t direction); +int prestera_hw_vtcam_rule_add(struct prestera_switch *sw, u32 vtcam_id, + u32 prio, void *key, void *keymask, + struct prestera_acl_hw_action_info *act, + u8 n_act, u32 *rule_id); +int prestera_hw_vtcam_rule_del(struct prestera_switch *sw, + u32 vtcam_id, u32 rule_id); +int prestera_hw_vtcam_destroy(struct prestera_switch *sw, u32 vtcam_id); +int prestera_hw_vtcam_iface_bind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id, u16 pcl_id); +int prestera_hw_vtcam_iface_unbind(struct prestera_switch *sw, + struct prestera_acl_iface *iface, + u32 vtcam_id); + +/* Counter API */ +int prestera_hw_counter_trigger(struct prestera_switch *sw, u32 block_id); +int prestera_hw_counter_abort(struct prestera_switch *sw); +int prestera_hw_counters_get(struct prestera_switch *sw, u32 idx, + u32 *len, bool *done, + struct prestera_counter_stats *stats); +int prestera_hw_counter_block_get(struct prestera_switch *sw, + u32 client, u32 *block_id, u32 *offset, + u32 *num_counters); +int prestera_hw_counter_block_release(struct prestera_switch *sw, + u32 block_id); +int prestera_hw_counter_clear(struct prestera_switch *sw, u32 block_id, + u32 counter_id); /* SPAN API */ int prestera_hw_span_get(const struct prestera_port *port, u8 *span_id); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_main.c b/drivers/net/ethernet/marvell/prestera/prestera_main.c index 4369a3ffad45..a0dbad5cb88d 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_main.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_main.c @@ -18,6 +18,7 @@ #include "prestera_rxtx.h" #include "prestera_devlink.h" #include "prestera_ethtool.h" +#include "prestera_counter.h" #include "prestera_switchdev.h" #define PRESTERA_MTU_DEFAULT 1536 @@ -904,6 +905,10 @@ static int prestera_switch_init(struct prestera_switch *sw) if (err) goto err_handlers_register; + err = prestera_counter_init(sw); + if (err) + goto err_counter_init; + err = prestera_acl_init(sw); if (err) goto err_acl_init; @@ -936,6 +941,8 @@ err_dl_register: err_span_init: prestera_acl_fini(sw); err_acl_init: + prestera_counter_fini(sw); +err_counter_init: prestera_event_handlers_unregister(sw); err_handlers_register: prestera_rxtx_switch_fini(sw); @@ -956,6 +963,7 @@ static void prestera_switch_fini(struct prestera_switch *sw) prestera_devlink_traps_unregister(sw); prestera_span_fini(sw); prestera_acl_fini(sw); + prestera_counter_fini(sw); prestera_event_handlers_unregister(sw); prestera_rxtx_switch_fini(sw); prestera_switchdev_fini(sw); diff --git a/drivers/net/ethernet/marvell/prestera/prestera_span.c b/drivers/net/ethernet/marvell/prestera/prestera_span.c index 3cafca827bb7..845e9d8c8cc7 100644 --- a/drivers/net/ethernet/marvell/prestera/prestera_span.c +++ b/drivers/net/ethernet/marvell/prestera/prestera_span.c @@ -7,6 +7,7 @@ #include "prestera.h" #include "prestera_hw.h" #include "prestera_acl.h" +#include "prestera_flow.h" #include "prestera_span.h" struct prestera_span_entry { diff --git a/drivers/net/ethernet/marvell/skge.c b/drivers/net/ethernet/marvell/skge.c index 0c864e5bf0a6..cf03c67fbf40 100644 --- a/drivers/net/ethernet/marvell/skge.c +++ b/drivers/net/ethernet/marvell/skge.c @@ -492,7 +492,9 @@ static void skge_get_strings(struct net_device *dev, u32 stringset, u8 *data) } static void skge_get_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); @@ -504,7 +506,9 @@ static void skge_get_ring_param(struct net_device *dev, } static int skge_set_ring_param(struct net_device *dev, - struct ethtool_ringparam *p) + struct ethtool_ringparam *p, + struct kernel_ethtool_ringparam *kernel_p, + struct netlink_ext_ack *extack) { struct skge_port *skge = netdev_priv(dev); int err = 0; diff --git a/drivers/net/ethernet/marvell/sky2.c b/drivers/net/ethernet/marvell/sky2.c index 28b5b9341145..ea16b1dd6a98 100644 --- a/drivers/net/ethernet/marvell/sky2.c +++ b/drivers/net/ethernet/marvell/sky2.c @@ -4149,7 +4149,9 @@ static unsigned long roundup_ring_size(unsigned long pending) } static void sky2_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); @@ -4161,7 +4163,9 @@ static void sky2_get_ringparam(struct net_device *dev, } static int sky2_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct sky2_port *sky2 = netdev_priv(dev); @@ -4266,96 +4270,36 @@ static int sky2_get_eeprom_len(struct net_device *dev) return 1 << ( ((reg2 & PCI_VPD_ROM_SZ) >> 14) + 8); } -static int sky2_vpd_wait(const struct sky2_hw *hw, int cap, u16 busy) -{ - unsigned long start = jiffies; - - while ( (sky2_pci_read16(hw, cap + PCI_VPD_ADDR) & PCI_VPD_ADDR_F) == busy) { - /* Can take up to 10.6 ms for write */ - if (time_after(jiffies, start + HZ/4)) { - dev_err(&hw->pdev->dev, "VPD cycle timed out\n"); - return -ETIMEDOUT; - } - msleep(1); - } - - return 0; -} - -static int sky2_vpd_read(struct sky2_hw *hw, int cap, void *data, - u16 offset, size_t length) -{ - int rc = 0; - - while (length > 0) { - u32 val; - - sky2_pci_write16(hw, cap + PCI_VPD_ADDR, offset); - rc = sky2_vpd_wait(hw, cap, 0); - if (rc) - break; - - val = sky2_pci_read32(hw, cap + PCI_VPD_DATA); - - memcpy(data, &val, min(sizeof(val), length)); - offset += sizeof(u32); - data += sizeof(u32); - length -= sizeof(u32); - } - - return rc; -} - -static int sky2_vpd_write(struct sky2_hw *hw, int cap, const void *data, - u16 offset, unsigned int length) -{ - unsigned int i; - int rc = 0; - - for (i = 0; i < length; i += sizeof(u32)) { - u32 val = *(u32 *)(data + i); - - sky2_pci_write32(hw, cap + PCI_VPD_DATA, val); - sky2_pci_write32(hw, cap + PCI_VPD_ADDR, offset | PCI_VPD_ADDR_F); - - rc = sky2_vpd_wait(hw, cap, PCI_VPD_ADDR_F); - if (rc) - break; - } - return rc; -} - static int sky2_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data) { struct sky2_port *sky2 = netdev_priv(dev); - int cap = pci_find_capability(sky2->hw->pdev, PCI_CAP_ID_VPD); - - if (!cap) - return -EINVAL; + int rc; eeprom->magic = SKY2_EEPROM_MAGIC; + rc = pci_read_vpd_any(sky2->hw->pdev, eeprom->offset, eeprom->len, + data); + if (rc < 0) + return rc; + + eeprom->len = rc; - return sky2_vpd_read(sky2->hw, cap, data, eeprom->offset, eeprom->len); + return 0; } static int sky2_set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, u8 *data) { struct sky2_port *sky2 = netdev_priv(dev); - int cap = pci_find_capability(sky2->hw->pdev, PCI_CAP_ID_VPD); - - if (!cap) - return -EINVAL; + int rc; if (eeprom->magic != SKY2_EEPROM_MAGIC) return -EINVAL; - /* Partial writes not supported */ - if ((eeprom->offset & 3) || (eeprom->len & 3)) - return -EINVAL; + rc = pci_write_vpd_any(sky2->hw->pdev, eeprom->offset, eeprom->len, + data); - return sky2_vpd_write(sky2->hw, cap, data, eeprom->offset, eeprom->len); + return rc < 0 ? rc : 0; } static netdev_features_t sky2_fix_features(struct net_device *dev, diff --git a/drivers/net/ethernet/mediatek/mtk_eth_soc.c b/drivers/net/ethernet/mediatek/mtk_eth_soc.c index 75d67d1b5f6b..a068cf5c970f 100644 --- a/drivers/net/ethernet/mediatek/mtk_eth_soc.c +++ b/drivers/net/ethernet/mediatek/mtk_eth_soc.c @@ -463,94 +463,8 @@ static void mtk_mac_link_up(struct phylink_config *config, mtk_w32(mac->hw, mcr, MTK_MAC_MCR(mac->id)); } -static void mtk_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct mtk_mac *mac = container_of(config, struct mtk_mac, - phylink_config); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - if (state->interface != PHY_INTERFACE_MODE_NA && - state->interface != PHY_INTERFACE_MODE_MII && - state->interface != PHY_INTERFACE_MODE_GMII && - !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII) && - phy_interface_mode_is_rgmii(state->interface)) && - !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) && - !mac->id && state->interface == PHY_INTERFACE_MODE_TRGMII) && - !(MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII) && - (state->interface == PHY_INTERFACE_MODE_SGMII || - phy_interface_mode_is_8023z(state->interface)))) { - linkmode_zero(supported); - return; - } - - phylink_set_port_modes(mask); - phylink_set(mask, Autoneg); - - switch (state->interface) { - case PHY_INTERFACE_MODE_TRGMII: - phylink_set(mask, 1000baseT_Full); - break; - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_2500BASEX: - phylink_set(mask, 1000baseX_Full); - phylink_set(mask, 2500baseX_Full); - break; - case PHY_INTERFACE_MODE_GMII: - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - phylink_set(mask, 1000baseT_Half); - fallthrough; - case PHY_INTERFACE_MODE_SGMII: - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - fallthrough; - case PHY_INTERFACE_MODE_MII: - case PHY_INTERFACE_MODE_RMII: - case PHY_INTERFACE_MODE_REVMII: - case PHY_INTERFACE_MODE_NA: - default: - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - break; - } - - if (state->interface == PHY_INTERFACE_MODE_NA) { - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - phylink_set(mask, 2500baseX_Full); - } - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII)) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseT_Half); - phylink_set(mask, 1000baseX_Full); - } - if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_GEPHY)) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseT_Half); - } - } - - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); - - /* We can only operate at 2500BaseX or 1000BaseX. If requested - * to advertise both, only report advertising at 2500BaseX. - */ - phylink_helper_basex_speed(state); -} - static const struct phylink_mac_ops mtk_phylink_ops = { - .validate = mtk_validate, + .validate = phylink_generic_validate, .mac_pcs_get_state = mtk_mac_pcs_get_state, .mac_an_restart = mtk_mac_an_restart, .mac_config = mtk_mac_config, @@ -3009,6 +2923,33 @@ static int mtk_add_mac(struct mtk_eth *eth, struct device_node *np) mac->phylink_config.dev = ð->netdev[id]->dev; mac->phylink_config.type = PHYLINK_NETDEV; + /* This driver makes use of state->speed/state->duplex in + * mac_config + */ + mac->phylink_config.legacy_pre_march2020 = true; + mac->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000 | MAC_2500FD; + + __set_bit(PHY_INTERFACE_MODE_MII, + mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + mac->phylink_config.supported_interfaces); + + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_RGMII)) + phy_interface_set_rgmii(mac->phylink_config.supported_interfaces); + + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_TRGMII) && !mac->id) + __set_bit(PHY_INTERFACE_MODE_TRGMII, + mac->phylink_config.supported_interfaces); + + if (MTK_HAS_CAPS(mac->hw->soc->caps, MTK_SGMII)) { + __set_bit(PHY_INTERFACE_MODE_SGMII, + mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + mac->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + mac->phylink_config.supported_interfaces); + } phylink = phylink_create(&mac->phylink_config, of_fwnode_handle(mac->of_node), diff --git a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c index 10238bedd694..ed5038d98ef6 100644 --- a/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx4/en_ethtool.c @@ -1141,7 +1141,9 @@ static void mlx4_en_get_pauseparam(struct net_device *dev, } static int mlx4_en_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx4_en_priv *priv = netdev_priv(dev); struct mlx4_en_dev *mdev = priv->mdev; @@ -1208,7 +1210,9 @@ out: } static void mlx4_en_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx4_en_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig index 92056452a9e3..4ba1a78c6515 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Kconfig +++ b/drivers/net/ethernet/mellanox/mlx5/core/Kconfig @@ -115,6 +115,7 @@ config MLX5_TC_CT config MLX5_TC_SAMPLE bool "MLX5 TC sample offload support" depends on MLX5_CLS_ACT + depends on PSAMPLE=y || PSAMPLE=n || MLX5_CORE=m default y help Say Y here if you want to support offloading sample rules via tc diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index f0ac6b0d9653..e77c4159713f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -173,7 +173,7 @@ struct page_pool; #define MLX5E_KLM_ENTRIES_PER_WQE(wqe_size)\ ALIGN_DOWN(MLX5E_KLM_MAX_ENTRIES_PER_WQE(wqe_size), MLX5_UMR_KLM_ALIGNMENT) -#define MLX5E_MAX_KLM_PER_WQE(mdev) \ +#define MLX5E_MAX_KLM_PER_WQE \ MLX5E_KLM_ENTRIES_PER_WQE(MLX5E_TX_MPW_MAX_NUM_DS << MLX5_MKEY_BSF_OCTO_SIZE) #define MLX5E_MSG_LEVEL NETIF_MSG_LINK @@ -1057,7 +1057,6 @@ int mlx5e_safe_switch_params(struct mlx5e_priv *priv, mlx5e_fp_preactivate preactivate, void *context, bool reset); int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv); -int mlx5e_num_channels_changed(struct mlx5e_priv *priv); int mlx5e_num_channels_changed_ctx(struct mlx5e_priv *priv, void *context); void mlx5e_activate_priv_channels(struct mlx5e_priv *priv); void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv); @@ -1148,9 +1147,12 @@ void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv, int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, struct ethtool_channels *ch); int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, - struct ethtool_coalesce *coal); + struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal); int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, - struct ethtool_coalesce *coal); + struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack); int mlx5e_ethtool_get_link_ksettings(struct mlx5e_priv *priv, struct ethtool_link_ksettings *link_ksettings); int mlx5e_ethtool_set_link_ksettings(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c index 7edde4d536fd..17325c5d6516 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.c @@ -155,3 +155,61 @@ struct mlx5_modify_hdr *mlx5e_mod_hdr_get(struct mlx5e_mod_hdr_handle *mh) return mh->modify_hdr; } +char * +mlx5e_mod_hdr_alloc(struct mlx5_core_dev *mdev, int namespace, + struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts) +{ + int new_num_actions, max_hw_actions; + size_t new_sz, old_sz; + void *ret; + + if (mod_hdr_acts->num_actions < mod_hdr_acts->max_actions) + goto out; + + max_hw_actions = mlx5e_mod_hdr_max_actions(mdev, namespace); + new_num_actions = min(max_hw_actions, + mod_hdr_acts->actions ? + mod_hdr_acts->max_actions * 2 : 1); + if (mod_hdr_acts->max_actions == new_num_actions) + return ERR_PTR(-ENOSPC); + + new_sz = MLX5_MH_ACT_SZ * new_num_actions; + old_sz = mod_hdr_acts->max_actions * MLX5_MH_ACT_SZ; + + if (mod_hdr_acts->is_static) { + ret = kzalloc(new_sz, GFP_KERNEL); + if (ret) { + memcpy(ret, mod_hdr_acts->actions, old_sz); + mod_hdr_acts->is_static = false; + } + } else { + ret = krealloc(mod_hdr_acts->actions, new_sz, GFP_KERNEL); + if (ret) + memset(ret + old_sz, 0, new_sz - old_sz); + } + if (!ret) + return ERR_PTR(-ENOMEM); + + mod_hdr_acts->actions = ret; + mod_hdr_acts->max_actions = new_num_actions; + +out: + return mod_hdr_acts->actions + (mod_hdr_acts->num_actions * MLX5_MH_ACT_SZ); +} + +void +mlx5e_mod_hdr_dealloc(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts) +{ + if (!mod_hdr_acts->is_static) + kfree(mod_hdr_acts->actions); + + mod_hdr_acts->actions = NULL; + mod_hdr_acts->num_actions = 0; + mod_hdr_acts->max_actions = 0; +} + +char * +mlx5e_mod_hdr_get_item(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, int pos) +{ + return mod_hdr_acts->actions + (pos * MLX5_MH_ACT_SZ); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h index 33b23d8f9182..b8dac418d0a5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/mod_hdr.h @@ -7,14 +7,32 @@ #include <linux/hashtable.h> #include <linux/mlx5/fs.h> +#define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) + struct mlx5e_mod_hdr_handle; struct mlx5e_tc_mod_hdr_acts { int num_actions; int max_actions; + bool is_static; void *actions; }; +#define DECLARE_MOD_HDR_ACTS_ACTIONS(name, len) \ + u8 name[len][MLX5_MH_ACT_SZ] = {} + +#define DECLARE_MOD_HDR_ACTS(name, acts_arr) \ + struct mlx5e_tc_mod_hdr_acts name = { \ + .max_actions = ARRAY_SIZE(acts_arr), \ + .is_static = true, \ + .actions = acts_arr, \ + } + +char *mlx5e_mod_hdr_alloc(struct mlx5_core_dev *mdev, int namespace, + struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); +void mlx5e_mod_hdr_dealloc(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); +char *mlx5e_mod_hdr_get_item(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts, int pos); + struct mlx5e_mod_hdr_handle * mlx5e_mod_hdr_attach(struct mlx5_core_dev *mdev, struct mod_hdr_tbl *tbl, @@ -28,4 +46,12 @@ struct mlx5_modify_hdr *mlx5e_mod_hdr_get(struct mlx5e_mod_hdr_handle *mh); void mlx5e_mod_hdr_tbl_init(struct mod_hdr_tbl *tbl); void mlx5e_mod_hdr_tbl_destroy(struct mod_hdr_tbl *tbl); +static inline int mlx5e_mod_hdr_max_actions(struct mlx5_core_dev *mdev, int namespace) +{ + if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */ + return MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, max_modify_header_actions); + else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */ + return MLX5_CAP_FLOWTABLE_NIC_RX(mdev, max_modify_header_actions); +} + #endif /* __MLX5E_EN_MOD_HDR_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c index f8c29022dbb2..66180ffb4606 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/params.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/params.c @@ -717,7 +717,7 @@ static u32 mlx5e_shampo_icosq_sz(struct mlx5_core_dev *mdev, int wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz)); u32 wqebbs; - max_klm_per_umr = MLX5E_MAX_KLM_PER_WQE(mdev); + max_klm_per_umr = MLX5E_MAX_KLM_PER_WQE; max_hd_per_wqe = mlx5e_shampo_hd_per_wqe(mdev, params, rq_param); max_num_of_umr_per_wqe = max_hd_per_wqe / max_klm_per_umr; rest = max_hd_per_wqe % max_klm_per_umr; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c index 4f4bc8726ec4..860605133287 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/reporter_tx.c @@ -565,7 +565,7 @@ int mlx5e_reporter_tx_timeout(struct mlx5e_txqsq *sq) snprintf(err_str, sizeof(err_str), "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u", sq->ch_ix, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc, - jiffies_to_usecs(jiffies - sq->txq->trans_start)); + jiffies_to_usecs(jiffies - READ_ONCE(sq->txq->trans_start))); mlx5e_health_report(priv, priv->tx_reporter, err_str, &err_ctx); return to_ctx.status; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c index df6888c4793c..ff4b4f8a5a9d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/sample.c @@ -5,6 +5,7 @@ #include <net/psample.h> #include "en/mapping.h" #include "en/tc/post_act.h" +#include "en/mod_hdr.h" #include "sample.h" #include "eswitch.h" #include "en_tc.h" @@ -255,12 +256,12 @@ sample_modify_hdr_get(struct mlx5_core_dev *mdev, u32 obj_id, goto err_modify_hdr; } - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); return modify_hdr; err_modify_hdr: err_post_act: - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); err_set_regc0: return ERR_PTR(err); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index 2445e2ae3324..9f33729e7fc4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -36,6 +36,12 @@ #define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen) #define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0) +/* Statically allocate modify actions for + * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10. + * This will be increased dynamically if needed (for the ipv6 snat + dnat). + */ +#define MLX5_CT_MIN_MOD_ACTS 10 + #define ct_dbg(fmt, args...)\ netdev_dbg(ct_priv->netdev, "ct_debug: " fmt "\n", ##args) @@ -609,22 +615,15 @@ mlx5_tc_ct_entry_create_nat(struct mlx5_tc_ct_priv *ct_priv, struct flow_action *flow_action = &flow_rule->action; struct mlx5_core_dev *mdev = ct_priv->dev; struct flow_action_entry *act; - size_t action_size; char *modact; int err, i; - action_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto); - flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_MANGLE: { - err = alloc_mod_hdr_actions(mdev, ct_priv->ns_type, - mod_acts); - if (err) - return err; - - modact = mod_acts->actions + - mod_acts->num_actions * action_size; + modact = mlx5e_mod_hdr_alloc(mdev, ct_priv->ns_type, mod_acts); + if (IS_ERR(modact)) + return PTR_ERR(modact); err = mlx5_tc_ct_parse_mangle_to_mod_act(act, modact); if (err) @@ -652,7 +651,8 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, struct mlx5e_mod_hdr_handle **mh, u8 zone_restore_id, bool nat) { - struct mlx5e_tc_mod_hdr_acts mod_acts = {}; + DECLARE_MOD_HDR_ACTS_ACTIONS(actions_arr, MLX5_CT_MIN_MOD_ACTS); + DECLARE_MOD_HDR_ACTS(mod_acts, actions_arr); struct flow_action_entry *meta; u16 ct_state = 0; int err; @@ -706,11 +706,11 @@ mlx5_tc_ct_entry_create_mod_hdr(struct mlx5_tc_ct_priv *ct_priv, attr->modify_hdr = mlx5e_mod_hdr_get(*mh); } - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); return 0; err_mapping: - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id); return err; } @@ -907,12 +907,9 @@ mlx5_tc_ct_shared_counter_get(struct mlx5_tc_ct_priv *ct_priv, struct mlx5_ct_tuple rev_tuple = entry->tuple; struct mlx5_ct_counter *shared_counter; struct mlx5_ct_entry *rev_entry; - __be16 tmp_port; /* get the reversed tuple */ - tmp_port = rev_tuple.port.src; - rev_tuple.port.src = rev_tuple.port.dst; - rev_tuple.port.dst = tmp_port; + swap(rev_tuple.port.src, rev_tuple.port.dst); if (rev_tuple.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { __be32 tmp_addr = rev_tuple.ip.src_v4; @@ -1460,7 +1457,7 @@ static int tc_ct_pre_ct_add_rules(struct mlx5_ct_ft *ct_ft, } pre_ct->miss_rule = rule; - dealloc_mod_hdr_actions(&pre_mod_acts); + mlx5e_mod_hdr_dealloc(&pre_mod_acts); kvfree(spec); return 0; @@ -1469,7 +1466,7 @@ err_miss_rule: err_flow_rule: mlx5_modify_header_dealloc(dev, pre_ct->modify_hdr); err_mapping: - dealloc_mod_hdr_actions(&pre_mod_acts); + mlx5e_mod_hdr_dealloc(&pre_mod_acts); kvfree(spec); return err; } @@ -1865,14 +1862,14 @@ __mlx5_tc_ct_flow_offload(struct mlx5_tc_ct_priv *ct_priv, } attr->ct_attr.ct_flow = ct_flow; - dealloc_mod_hdr_actions(&pre_mod_acts); + mlx5e_mod_hdr_dealloc(&pre_mod_acts); return ct_flow->pre_ct_rule; err_insert_orig: mlx5_modify_header_dealloc(priv->mdev, pre_ct_attr->modify_hdr); err_mapping: - dealloc_mod_hdr_actions(&pre_mod_acts); + mlx5e_mod_hdr_dealloc(&pre_mod_acts); mlx5_chains_put_chain_mapping(ct_priv->chains, ct_flow->chain_mapping); err_get_chain: kfree(ct_flow->pre_ct_attr); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c index a5e450973225..33815246fead 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.c @@ -103,7 +103,7 @@ static int get_route_and_out_devs(struct mlx5e_priv *priv, } static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv, - struct net_device *mirred_dev, + struct net_device *dev, struct mlx5e_tc_tun_route_attr *attr) { struct net_device *route_dev; @@ -122,13 +122,13 @@ static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv, uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH); attr->fl.fl4.flowi4_oif = uplink_dev->ifindex; } else { - struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev); + struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev); if (tunnel && tunnel->get_remote_ifindex) - attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(mirred_dev); + attr->fl.fl4.flowi4_oif = tunnel->get_remote_ifindex(dev); } - rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4); + rt = ip_route_output_key(dev_net(dev), &attr->fl.fl4); if (IS_ERR(rt)) return PTR_ERR(rt); @@ -440,10 +440,10 @@ release_neigh: #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, - struct net_device *mirred_dev, + struct net_device *dev, struct mlx5e_tc_tun_route_attr *attr) { - struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(mirred_dev); + struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(dev); struct net_device *route_dev; struct net_device *out_dev; struct dst_entry *dst; @@ -451,8 +451,8 @@ static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv, int ret; if (tunnel && tunnel->get_remote_ifindex) - attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(mirred_dev); - dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6, + attr->fl.fl6.flowi6_oif = tunnel->get_remote_ifindex(dev); + dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(dev), NULL, &attr->fl.fl6, NULL); if (IS_ERR(dst)) return PTR_ERR(dst); @@ -708,7 +708,8 @@ release_neigh: int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, - struct mlx5_flow_attr *flow_attr) + struct mlx5_flow_attr *flow_attr, + struct net_device *filter_dev) { struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr; struct mlx5e_tc_int_port *int_port; @@ -720,14 +721,14 @@ int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, /* Addresses are swapped for decap */ attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4; attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4; - err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr); + err = mlx5e_route_lookup_ipv4_get(priv, filter_dev, &attr); } #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6) else if (flow_attr->tun_ip_version == 6) { /* Addresses are swapped for decap */ attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6; attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6; - err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr); + err = mlx5e_route_lookup_ipv6_get(priv, filter_dev, &attr); } #endif else diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h index aa092eaeaec3..b38f693bbb52 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun.h @@ -94,7 +94,8 @@ mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv, #endif int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, - struct mlx5_flow_attr *attr); + struct mlx5_flow_attr *attr, + struct net_device *filter_dev); bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv, struct net_device *netdev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c index 042b1abe1437..3c8851ee8172 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_tun_encap.c @@ -1159,7 +1159,7 @@ int mlx5e_attach_decap_route(struct mlx5e_priv *priv, tbl_time_before = mlx5e_route_tbl_get_last_update(priv); tbl_time_after = tbl_time_before; - err = mlx5e_tc_tun_route_lookup(priv, &parse_attr->spec, attr); + err = mlx5e_tc_tun_route_lookup(priv, &parse_attr->spec, attr, parse_attr->filter_dev); if (err || !esw_attr->rx_tun_attr->decap_vport) goto out; @@ -1480,7 +1480,7 @@ static void mlx5e_reoffload_decap(struct mlx5e_priv *priv, parse_attr = attr->parse_attr; spec = &parse_attr->spec; - err = mlx5e_tc_tun_route_lookup(priv, spec, attr); + err = mlx5e_tc_tun_route_lookup(priv, spec, attr, parse_attr->filter_dev); if (err) { mlx5_core_warn(priv->mdev, "Failed to lookup route for flow, %d\n", err); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index c2ea5fad48dd..c8757c5a812b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -314,7 +314,9 @@ void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, } static void mlx5e_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -380,7 +382,9 @@ unlock: } static int mlx5e_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -511,7 +515,8 @@ static int mlx5e_set_channels(struct net_device *dev, } int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, - struct ethtool_coalesce *coal) + struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal) { struct dim_cq_moder *rx_moder, *tx_moder; @@ -528,6 +533,11 @@ int mlx5e_ethtool_get_coalesce(struct mlx5e_priv *priv, coal->tx_max_coalesced_frames = tx_moder->pkts; coal->use_adaptive_tx_coalesce = priv->channels.params.tx_dim_enabled; + kernel_coal->use_cqe_mode_rx = + MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_BASED_MODER); + kernel_coal->use_cqe_mode_tx = + MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_CQE_BASED_MODER); + return 0; } @@ -538,7 +548,7 @@ static int mlx5e_get_coalesce(struct net_device *netdev, { struct mlx5e_priv *priv = netdev_priv(netdev); - return mlx5e_ethtool_get_coalesce(priv, coal); + return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal); } #define MLX5E_MAX_COAL_TIME MLX5_MAX_CQ_PERIOD @@ -578,14 +588,26 @@ mlx5e_set_priv_channels_rx_coalesce(struct mlx5e_priv *priv, struct ethtool_coal } } +/* convert a boolean value of cq_mode to mlx5 period mode + * true : MLX5_CQ_PERIOD_MODE_START_FROM_CQE + * false : MLX5_CQ_PERIOD_MODE_START_FROM_EQE + */ +static int cqe_mode_to_period_mode(bool val) +{ + return val ? MLX5_CQ_PERIOD_MODE_START_FROM_CQE : MLX5_CQ_PERIOD_MODE_START_FROM_EQE; +} + int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, - struct ethtool_coalesce *coal) + struct ethtool_coalesce *coal, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) { struct dim_cq_moder *rx_moder, *tx_moder; struct mlx5_core_dev *mdev = priv->mdev; struct mlx5e_params new_params; bool reset_rx, reset_tx; bool reset = true; + u8 cq_period_mode; int err = 0; if (!MLX5_CAP_GEN(mdev, cq_moderation)) @@ -605,6 +627,12 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, return -ERANGE; } + if ((kernel_coal->use_cqe_mode_rx || kernel_coal->use_cqe_mode_tx) && + !MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe)) { + NL_SET_ERR_MSG_MOD(extack, "cqe_mode_rx/tx is not supported on this device"); + return -EOPNOTSUPP; + } + mutex_lock(&priv->state_lock); new_params = priv->channels.params; @@ -621,6 +649,18 @@ int mlx5e_ethtool_set_coalesce(struct mlx5e_priv *priv, reset_rx = !!coal->use_adaptive_rx_coalesce != priv->channels.params.rx_dim_enabled; reset_tx = !!coal->use_adaptive_tx_coalesce != priv->channels.params.tx_dim_enabled; + cq_period_mode = cqe_mode_to_period_mode(kernel_coal->use_cqe_mode_rx); + if (cq_period_mode != rx_moder->cq_period_mode) { + mlx5e_set_rx_cq_mode_params(&new_params, cq_period_mode); + reset_rx = true; + } + + cq_period_mode = cqe_mode_to_period_mode(kernel_coal->use_cqe_mode_tx); + if (cq_period_mode != tx_moder->cq_period_mode) { + mlx5e_set_tx_cq_mode_params(&new_params, cq_period_mode); + reset_tx = true; + } + if (reset_rx) { u8 mode = MLX5E_GET_PFLAG(&new_params, MLX5E_PFLAG_RX_CQE_BASED_MODER); @@ -656,9 +696,9 @@ static int mlx5e_set_coalesce(struct net_device *netdev, struct kernel_ethtool_coalesce *kernel_coal, struct netlink_ext_ack *extack) { - struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_priv *priv = netdev_priv(netdev); - return mlx5e_ethtool_set_coalesce(priv, coal); + return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack); } static void ptys2ethtool_supported_link(struct mlx5_core_dev *mdev, @@ -2358,7 +2398,8 @@ static void mlx5e_get_rmon_stats(struct net_device *netdev, const struct ethtool_ops mlx5e_ethtool_ops = { .supported_coalesce_params = ETHTOOL_COALESCE_USECS | ETHTOOL_COALESCE_MAX_FRAMES | - ETHTOOL_COALESCE_USE_ADAPTIVE, + ETHTOOL_COALESCE_USE_ADAPTIVE | + ETHTOOL_COALESCE_USE_CQE, .get_drvinfo = mlx5e_get_drvinfo, .get_link = ethtool_op_get_link, .get_link_ext_state = mlx5e_get_link_ext_state, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 65571593ec5c..496977e7406e 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -2598,7 +2598,7 @@ static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv, } } -int mlx5e_num_channels_changed(struct mlx5e_priv *priv) +static int mlx5e_num_channels_changed(struct mlx5e_priv *priv) { u16 count = priv->channels.params.num_channels; int err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index 48895d79796a..6e0f88ea3701 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -219,16 +219,22 @@ static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset) } } -static void mlx5e_rep_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) +static void +mlx5e_rep_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); mlx5e_ethtool_get_ringparam(priv, param); } -static int mlx5e_rep_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) +static int +mlx5e_rep_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = netdev_priv(dev); @@ -258,7 +264,7 @@ static int mlx5e_rep_get_coalesce(struct net_device *netdev, { struct mlx5e_priv *priv = netdev_priv(netdev); - return mlx5e_ethtool_get_coalesce(priv, coal); + return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal); } static int mlx5e_rep_set_coalesce(struct net_device *netdev, @@ -268,7 +274,7 @@ static int mlx5e_rep_set_coalesce(struct net_device *netdev, { struct mlx5e_priv *priv = netdev_priv(netdev); - return mlx5e_ethtool_set_coalesce(priv, coal); + return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack); } static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c index 793511d5ee4c..7e05d7592bce 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c @@ -37,6 +37,7 @@ #include <net/ip6_checksum.h> #include <net/page_pool.h> #include <net/inet_ecn.h> +#include <net/gro.h> #include <net/udp.h> #include <net/tcp.h> #include "en.h" @@ -618,7 +619,7 @@ static int mlx5e_alloc_rx_hd_mpwqe(struct mlx5e_rq *rq) struct mlx5e_icosq *sq = rq->icosq; int i, err, max_klm_entries, len; - max_klm_entries = MLX5E_MAX_KLM_PER_WQE(rq->mdev); + max_klm_entries = MLX5E_MAX_KLM_PER_WQE; klm_entries = bitmap_find_window(shampo->bitmap, shampo->hd_per_wqe, shampo->hd_per_wq, shampo->pi); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 2a9bfc3ffa2e..3c91a11e27ad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -2076,7 +2076,7 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(ptp) for (i = 0; i < NUM_PTP_CH_STATS; i++) sprintf(data + (idx++) * ETH_GSTRING_LEN, - ptp_ch_stats_desc[i].format); + "%s", ptp_ch_stats_desc[i].format); if (priv->tx_ptp_opened) { for (tc = 0; tc < priv->max_opened_tc; tc++) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 3d45f4ae80c0..3e3419190c55 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -71,7 +71,6 @@ #include "lag/mp.h" #define nic_chains(priv) ((priv)->fs.tc.chains) -#define MLX5_MH_ACT_SZ MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto) #define MLX5E_TC_TABLE_NUM_GROUPS 4 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(18) @@ -209,12 +208,9 @@ mlx5e_tc_match_to_reg_set_and_get_id(struct mlx5_core_dev *mdev, char *modact; int err; - err = alloc_mod_hdr_actions(mdev, ns, mod_hdr_acts); - if (err) - return err; - - modact = mod_hdr_acts->actions + - (mod_hdr_acts->num_actions * MLX5_MH_ACT_SZ); + modact = mlx5e_mod_hdr_alloc(mdev, ns, mod_hdr_acts); + if (IS_ERR(modact)) + return PTR_ERR(modact); /* Firmware has 5bit length field and 0 means 32bits */ if (mlen == 32) @@ -333,7 +329,7 @@ void mlx5e_tc_match_to_reg_mod_hdr_change(struct mlx5_core_dev *mdev, int mlen = mlx5e_tc_attr_to_reg_mappings[type].mlen; char *modact; - modact = mod_hdr_acts->actions + (act_id * MLX5_MH_ACT_SZ); + modact = mlx5e_mod_hdr_get_item(mod_hdr_acts, act_id); /* Firmware has 5bit length field and 0 means 32bits */ if (mlen == 32) @@ -1076,7 +1072,7 @@ mlx5e_tc_add_nic_flow(struct mlx5e_priv *priv, if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { err = mlx5e_attach_mod_hdr(priv, flow, parse_attr); - dealloc_mod_hdr_actions(&parse_attr->mod_hdr_acts); + mlx5e_mod_hdr_dealloc(&parse_attr->mod_hdr_acts); if (err) return err; } @@ -1132,16 +1128,16 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, } mutex_unlock(&priv->fs.tc.t_lock); - kvfree(attr->parse_attr); - if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) mlx5e_detach_mod_hdr(priv, flow); - mlx5_fc_destroy(priv->mdev, attr->counter); + if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) + mlx5_fc_destroy(priv->mdev, attr->counter); if (flow_flag_test(flow, HAIRPIN)) mlx5e_hairpin_flow_del(priv, flow); + kvfree(attr->parse_attr); kfree(flow->attr); } @@ -1445,7 +1441,7 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, MLX5_FLOW_NAMESPACE_FDB, VPORT_TO_REG, metadata); if (err) - return err; + goto err_out; } } @@ -1461,13 +1457,15 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, if (attr->chain) { NL_SET_ERR_MSG_MOD(extack, "Internal port rule is only supported on chain 0"); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto err_out; } if (attr->dest_chain) { NL_SET_ERR_MSG_MOD(extack, "Internal port rule offload doesn't support goto action"); - return -EOPNOTSUPP; + err = -EOPNOTSUPP; + goto err_out; } int_port = mlx5e_tc_int_port_get(mlx5e_get_int_port_priv(priv), @@ -1475,8 +1473,10 @@ mlx5e_tc_add_fdb_flow(struct mlx5e_priv *priv, flow_flag_test(flow, EGRESS) ? MLX5E_TC_INT_PORT_EGRESS : MLX5E_TC_INT_PORT_INGRESS); - if (IS_ERR(int_port)) - return PTR_ERR(int_port); + if (IS_ERR(int_port)) { + err = PTR_ERR(int_port); + goto err_out; + } esw_attr->int_port = int_port; } @@ -1624,15 +1624,12 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, mlx5_tc_ct_match_del(get_ct_priv(priv), &flow->attr->ct_attr); if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) { - dealloc_mod_hdr_actions(&attr->parse_attr->mod_hdr_acts); + mlx5e_mod_hdr_dealloc(&attr->parse_attr->mod_hdr_acts); if (vf_tun && attr->modify_hdr) mlx5_modify_header_dealloc(priv->mdev, attr->modify_hdr); else mlx5e_detach_mod_hdr(priv, flow); } - kfree(attr->sample_attr); - kvfree(attr->parse_attr); - kvfree(attr->esw_attr->rx_tun_attr); if (attr->action & MLX5_FLOW_CONTEXT_ACTION_COUNT) mlx5_fc_destroy(esw_attr->counter_dev, attr->counter); @@ -1646,6 +1643,9 @@ static void mlx5e_tc_del_fdb_flow(struct mlx5e_priv *priv, if (flow_flag_test(flow, L3_TO_L2_DECAP)) mlx5e_detach_decap(priv, flow); + kfree(attr->sample_attr); + kvfree(attr->esw_attr->rx_tun_attr); + kvfree(attr->parse_attr); kfree(flow->attr); } @@ -2767,13 +2767,12 @@ static int offload_pedit_fields(struct mlx5e_priv *priv, struct netlink_ext_ack *extack) { struct pedit_headers *set_masks, *add_masks, *set_vals, *add_vals; - int i, action_size, first, last, next_z; void *headers_c, *headers_v, *action, *vals_p; u32 *s_masks_p, *a_masks_p, s_mask, a_mask; struct mlx5e_tc_mod_hdr_acts *mod_acts; - struct mlx5_fields *f; unsigned long mask, field_mask; - int err; + int i, first, last, next_z; + struct mlx5_fields *f; u8 cmd; mod_acts = &parse_attr->mod_hdr_acts; @@ -2785,8 +2784,6 @@ static int offload_pedit_fields(struct mlx5e_priv *priv, set_vals = &hdrs[0].vals; add_vals = &hdrs[1].vals; - action_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto); - for (i = 0; i < ARRAY_SIZE(fields); i++) { bool skip; @@ -2854,18 +2851,16 @@ static int offload_pedit_fields(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - err = alloc_mod_hdr_actions(priv->mdev, namespace, mod_acts); - if (err) { + action = mlx5e_mod_hdr_alloc(priv->mdev, namespace, mod_acts); + if (IS_ERR(action)) { NL_SET_ERR_MSG_MOD(extack, "too many pedit actions, can't offload"); mlx5_core_warn(priv->mdev, "mlx5: parsed %d pedit actions, can't do more\n", mod_acts->num_actions); - return err; + return PTR_ERR(action); } - action = mod_acts->actions + - (mod_acts->num_actions * action_size); MLX5_SET(set_action_in, action, action_type, cmd); MLX5_SET(set_action_in, action, field, f->field); @@ -2895,57 +2890,6 @@ static int offload_pedit_fields(struct mlx5e_priv *priv, return 0; } -static int mlx5e_flow_namespace_max_modify_action(struct mlx5_core_dev *mdev, - int namespace) -{ - if (namespace == MLX5_FLOW_NAMESPACE_FDB) /* FDB offloading */ - return MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, max_modify_header_actions); - else /* namespace is MLX5_FLOW_NAMESPACE_KERNEL - NIC offloading */ - return MLX5_CAP_FLOWTABLE_NIC_RX(mdev, max_modify_header_actions); -} - -int alloc_mod_hdr_actions(struct mlx5_core_dev *mdev, - int namespace, - struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts) -{ - int action_size, new_num_actions, max_hw_actions; - size_t new_sz, old_sz; - void *ret; - - if (mod_hdr_acts->num_actions < mod_hdr_acts->max_actions) - return 0; - - action_size = MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto); - - max_hw_actions = mlx5e_flow_namespace_max_modify_action(mdev, - namespace); - new_num_actions = min(max_hw_actions, - mod_hdr_acts->actions ? - mod_hdr_acts->max_actions * 2 : 1); - if (mod_hdr_acts->max_actions == new_num_actions) - return -ENOSPC; - - new_sz = action_size * new_num_actions; - old_sz = mod_hdr_acts->max_actions * action_size; - ret = krealloc(mod_hdr_acts->actions, new_sz, GFP_KERNEL); - if (!ret) - return -ENOMEM; - - memset(ret + old_sz, 0, new_sz - old_sz); - mod_hdr_acts->actions = ret; - mod_hdr_acts->max_actions = new_num_actions; - - return 0; -} - -void dealloc_mod_hdr_actions(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts) -{ - kfree(mod_hdr_acts->actions); - mod_hdr_acts->actions = NULL; - mod_hdr_acts->num_actions = 0; - mod_hdr_acts->max_actions = 0; -} - static const struct pedit_headers zero_masks = {}; static int @@ -2968,7 +2912,7 @@ parse_pedit_to_modify_hdr(struct mlx5e_priv *priv, goto out_err; } - if (!mlx5e_flow_namespace_max_modify_action(priv->mdev, namespace)) { + if (!mlx5e_mod_hdr_max_actions(priv->mdev, namespace)) { NL_SET_ERR_MSG_MOD(extack, "The pedit offload action is not supported"); goto out_err; @@ -3061,7 +3005,7 @@ static int alloc_tc_pedit_action(struct mlx5e_priv *priv, int namespace, return 0; out_dealloc_parsed_actions: - dealloc_mod_hdr_actions(&parse_attr->mod_hdr_acts); + mlx5e_mod_hdr_dealloc(&parse_attr->mod_hdr_acts); return err; } @@ -3415,11 +3359,9 @@ add_vlan_prio_tag_rewrite_action(struct mlx5e_priv *priv, static int validate_goto_chain(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow, const struct flow_action_entry *act, - u32 actions, struct netlink_ext_ack *extack) { bool is_esw = mlx5e_is_eswitch_flow(flow); - struct mlx5_flow_attr *attr = flow->attr; bool ft_flow = mlx5e_is_ft_flow(flow); u32 dest_chain = act->chain_index; struct mlx5_fs_chains *chains; @@ -3440,7 +3382,7 @@ static int validate_goto_chain(struct mlx5e_priv *priv, } if (!mlx5_chains_backwards_supported(chains) && - dest_chain <= attr->chain) { + dest_chain <= flow->attr->chain) { NL_SET_ERR_MSG_MOD(extack, "Goto lower numbered chain isn't supported"); return -EOPNOTSUPP; @@ -3452,8 +3394,8 @@ static int validate_goto_chain(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - if (actions & (MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | - MLX5_FLOW_CONTEXT_ACTION_DECAP) && + if (flow->attr->action & (MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | + MLX5_FLOW_CONTEXT_ACTION_DECAP) && !reformat_and_fwd) { NL_SET_ERR_MSG_MOD(extack, "Goto chain is not allowed if action has reformat or decap"); @@ -3485,12 +3427,12 @@ actions_prepare_mod_hdr_actions(struct mlx5e_priv *priv, if (err) return err; - /* In case all pedit actions are skipped, remove the MOD_HDR flag. */ if (parse_attr->mod_hdr_acts.num_actions > 0) return 0; + /* In case all pedit actions are skipped, remove the MOD_HDR flag. */ attr->action &= ~MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; - dealloc_mod_hdr_actions(&parse_attr->mod_hdr_acts); + mlx5e_mod_hdr_dealloc(&parse_attr->mod_hdr_acts); if (ns_type != MLX5_FLOW_NAMESPACE_FDB) return 0; @@ -3503,6 +3445,24 @@ actions_prepare_mod_hdr_actions(struct mlx5e_priv *priv, } static int +flow_action_supported(struct flow_action *flow_action, + struct netlink_ext_ack *extack) +{ + if (!flow_action_has_entries(flow_action)) { + NL_SET_ERR_MSG_MOD(extack, "Flow action doesn't have any entries"); + return -EINVAL; + } + + if (!flow_action_hw_stats_check(flow_action, extack, + FLOW_ACTION_HW_STATS_DELAYED_BIT)) { + NL_SET_ERR_MSG_MOD(extack, "Flow action HW stats type is not supported"); + return -EOPNOTSUPP; + } + + return 0; +} + +static int parse_tc_nic_actions(struct mlx5e_priv *priv, struct flow_action *flow_action, struct mlx5e_tc_flow *flow, @@ -3513,19 +3473,11 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, struct pedit_headers_action hdrs[2] = {}; const struct flow_action_entry *act; struct mlx5_nic_flow_attr *nic_attr; - u32 action = 0; int err, i; - if (!flow_action_has_entries(flow_action)) { - NL_SET_ERR_MSG_MOD(extack, "Flow action doesn't have any entries"); - return -EINVAL; - } - - if (!flow_action_hw_stats_check(flow_action, extack, - FLOW_ACTION_HW_STATS_DELAYED_BIT)) { - NL_SET_ERR_MSG_MOD(extack, "Flow action HW stats type is not supported"); - return -EOPNOTSUPP; - } + err = flow_action_supported(flow_action, extack); + if (err) + return err; nic_attr = attr->nic_attr; nic_attr->flow_tag = MLX5_FS_DEFAULT_FLOW_TAG; @@ -3534,12 +3486,12 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_ACCEPT: - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; break; case FLOW_ACTION_DROP: - action |= MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; break; case FLOW_ACTION_MANGLE: case FLOW_ACTION_ADD: @@ -3548,19 +3500,19 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, if (err) return err; - action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; break; case FLOW_ACTION_VLAN_MANGLE: err = add_vlan_rewrite_action(priv, MLX5_FLOW_NAMESPACE_KERNEL, act, parse_attr, hdrs, - &action, extack); + &attr->action, extack); if (err) return err; break; case FLOW_ACTION_CSUM: - if (csum_offload_supported(priv, action, + if (csum_offload_supported(priv, attr->action, act->csum_flags, extack)) break; @@ -3573,8 +3525,8 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, same_hw_devs(priv, netdev_priv(peer_dev))) { parse_attr->mirred_ifindex[0] = peer_dev->ifindex; flow_flag_set(flow, HAIRPIN); - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; } else { NL_SET_ERR_MSG_MOD(extack, "device is not on same HW, can't offload"); @@ -3594,17 +3546,16 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, } nic_attr->flow_tag = mark; - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; } break; case FLOW_ACTION_GOTO: - err = validate_goto_chain(priv, flow, act, action, - extack); + err = validate_goto_chain(priv, flow, act, extack); if (err) return err; - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; attr->dest_chain = act->chain_index; break; case FLOW_ACTION_CT: @@ -3623,8 +3574,6 @@ parse_tc_nic_actions(struct mlx5e_priv *priv, } } - attr->action = action; - if (attr->dest_chain && parse_attr->mirred_ifindex[0]) { NL_SET_ERR_MSG(extack, "Mirroring goto chain rules isn't supported"); return -EOPNOTSUPP; @@ -3744,7 +3693,6 @@ static struct net_device *get_fdb_out_dev(struct net_device *uplink_dev, static int add_vlan_push_action(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr, struct net_device **out_dev, - u32 *action, struct netlink_ext_ack *extack) { struct net_device *vlan_dev = *out_dev; @@ -3756,7 +3704,7 @@ static int add_vlan_push_action(struct mlx5e_priv *priv, }; int err; - err = parse_tc_vlan_action(priv, &vlan_act, attr->esw_attr, action, extack); + err = parse_tc_vlan_action(priv, &vlan_act, attr->esw_attr, &attr->action, extack); if (err) return err; @@ -3767,14 +3715,13 @@ static int add_vlan_push_action(struct mlx5e_priv *priv, return -ENODEV; if (is_vlan_dev(*out_dev)) - err = add_vlan_push_action(priv, attr, out_dev, action, extack); + err = add_vlan_push_action(priv, attr, out_dev, extack); return err; } static int add_vlan_pop_action(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr, - u32 *action, struct netlink_ext_ack *extack) { struct flow_action_entry vlan_act = { @@ -3785,7 +3732,8 @@ static int add_vlan_pop_action(struct mlx5e_priv *priv, nest_level = attr->parse_attr->filter_dev->lower_level - priv->netdev->lower_level; while (nest_level--) { - err = parse_tc_vlan_action(priv, &vlan_act, attr->esw_attr, action, extack); + err = parse_tc_vlan_action(priv, &vlan_act, attr->esw_attr, + &attr->action, extack); if (err) return err; } @@ -3942,21 +3890,13 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, const struct flow_action_entry *act; struct mlx5_esw_flow_attr *esw_attr; bool encap = false, decap = false; - u32 action = attr->action; int err, i, if_count = 0; bool ptype_host = false; bool mpls_push = false; - if (!flow_action_has_entries(flow_action)) { - NL_SET_ERR_MSG_MOD(extack, "Flow action doesn't have any entries"); - return -EINVAL; - } - - if (!flow_action_hw_stats_check(flow_action, extack, - FLOW_ACTION_HW_STATS_DELAYED_BIT)) { - NL_SET_ERR_MSG_MOD(extack, "Flow action HW stats type is not supported"); - return -EOPNOTSUPP; - } + err = flow_action_supported(flow_action, extack); + if (err) + return err; esw_attr = attr->esw_attr; parse_attr = attr->parse_attr; @@ -3964,8 +3904,8 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, flow_action_for_each(i, act, flow_action) { switch (act->id) { case FLOW_ACTION_ACCEPT: - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; attr->flags |= MLX5_ESW_ATTR_FLAG_ACCEPT; break; case FLOW_ACTION_PTYPE: @@ -3978,8 +3918,8 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, ptype_host = true; break; case FLOW_ACTION_DROP: - action |= MLX5_FLOW_CONTEXT_ACTION_DROP | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; break; case FLOW_ACTION_TRAP: if (!flow_offload_has_one_action(flow_action)) { @@ -3987,8 +3927,8 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, "action trap is supported as a sole action only"); return -EOPNOTSUPP; } - action |= (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT); + attr->action |= (MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT); attr->flags |= MLX5_ESW_ATTR_FLAG_SLOW_PATH; break; case FLOW_ACTION_MPLS_PUSH: @@ -4019,7 +3959,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, } parse_attr->eth.h_proto = act->mpls_pop.proto; - action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; flow_flag_set(flow, L3_TO_L2_DECAP); break; case FLOW_ACTION_MANGLE: @@ -4030,12 +3970,12 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, return err; if (!flow_flag_test(flow, L3_TO_L2_DECAP)) { - action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; esw_attr->split_count = esw_attr->out_count; } break; case FLOW_ACTION_CSUM: - if (csum_offload_supported(priv, action, + if (csum_offload_supported(priv, attr->action, act->csum_flags, extack)) break; @@ -4071,12 +4011,12 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; err = mlx5e_set_fwd_to_int_port_actions(priv, attr, out_dev->ifindex, MLX5E_TC_INT_PORT_INGRESS, - &action, esw_attr->out_count); + &attr->action, esw_attr->out_count); if (err) return err; @@ -4121,8 +4061,8 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; if (encap) { parse_attr->mirred_ifindex[esw_attr->out_count] = out_dev->ifindex; @@ -4156,16 +4096,13 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, return -ENODEV; if (is_vlan_dev(out_dev)) { - err = add_vlan_push_action(priv, attr, - &out_dev, - &action, extack); + err = add_vlan_push_action(priv, attr, &out_dev, extack); if (err) return err; } if (is_vlan_dev(parse_attr->filter_dev)) { - err = add_vlan_pop_action(priv, attr, - &action, extack); + err = add_vlan_pop_action(priv, attr, extack); if (err) return err; } @@ -4198,7 +4135,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, err = mlx5e_set_fwd_to_int_port_actions(priv, attr, out_dev->ifindex, MLX5E_TC_INT_PORT_EGRESS, - &action, + &attr->action, esw_attr->out_count); if (err) return err; @@ -4236,15 +4173,16 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, case FLOW_ACTION_VLAN_PUSH: case FLOW_ACTION_VLAN_POP: if (act->id == FLOW_ACTION_VLAN_PUSH && - (action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP)) { + (attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP)) { /* Replace vlan pop+push with vlan modify */ - action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; + attr->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; err = add_vlan_rewrite_action(priv, MLX5_FLOW_NAMESPACE_FDB, act, parse_attr, hdrs, - &action, extack); + &attr->action, extack); } else { - err = parse_tc_vlan_action(priv, act, esw_attr, &action, extack); + err = parse_tc_vlan_action(priv, act, esw_attr, &attr->action, + extack); } if (err) return err; @@ -4255,7 +4193,7 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, err = add_vlan_rewrite_action(priv, MLX5_FLOW_NAMESPACE_FDB, act, parse_attr, hdrs, - &action, extack); + &attr->action, extack); if (err) return err; @@ -4265,13 +4203,12 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, decap = true; break; case FLOW_ACTION_GOTO: - err = validate_goto_chain(priv, flow, act, action, - extack); + err = validate_goto_chain(priv, flow, act, extack); if (err) return err; - action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_COUNT; + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; attr->dest_chain = act->chain_index; break; case FLOW_ACTION_CT: @@ -4314,23 +4251,18 @@ static int parse_tc_fdb_actions(struct mlx5e_priv *priv, return -EOPNOTSUPP; } - /* always set IP version for indirect table handling */ - attr->ip_version = mlx5e_tc_get_ip_version(&parse_attr->spec, true); - if (MLX5_CAP_GEN(esw->dev, prio_tag_required) && - action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP) { + attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP) { /* For prio tag mode, replace vlan pop with rewrite vlan prio * tag rewrite. */ - action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; + attr->action &= ~MLX5_FLOW_CONTEXT_ACTION_VLAN_POP; err = add_vlan_prio_tag_rewrite_action(priv, parse_attr, hdrs, - &action, extack); + &attr->action, extack); if (err) return err; } - attr->action = action; - err = actions_prepare_mod_hdr_actions(priv, flow, attr, hdrs, extack); if (err) return err; @@ -4553,6 +4485,9 @@ __mlx5e_add_fdb_flow(struct mlx5e_priv *priv, if (err) goto err_free; + /* always set IP version for indirect table handling */ + flow->attr->ip_version = mlx5e_tc_get_ip_version(&parse_attr->spec, true); + err = parse_tc_fdb_actions(priv, &rule->action, flow, extack); if (err) goto err_free; @@ -4714,7 +4649,7 @@ mlx5e_add_nic_flow(struct mlx5e_priv *priv, err_free: flow_flag_set(flow, FAILED); - dealloc_mod_hdr_actions(&parse_attr->mod_hdr_acts); + mlx5e_mod_hdr_dealloc(&parse_attr->mod_hdr_acts); mlx5e_flow_put(priv, flow); out: return err; @@ -5014,14 +4949,8 @@ static int scan_tc_matchall_fdb_actions(struct mlx5e_priv *priv, int mlx5e_tc_configure_matchall(struct mlx5e_priv *priv, struct tc_cls_matchall_offload *ma) { - struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; struct netlink_ext_ack *extack = ma->common.extack; - if (!mlx5_esw_qos_enabled(esw)) { - NL_SET_ERR_MSG_MOD(extack, "QoS is not supported on this device"); - return -EOPNOTSUPP; - } - if (ma->common.prio != 1) { NL_SET_ERR_MSG_MOD(extack, "only priority 1 is supported"); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index fdb222793027..eb042f0f5a41 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -247,11 +247,6 @@ int mlx5e_tc_add_flow_mod_hdr(struct mlx5e_priv *priv, struct mlx5e_tc_flow_parse_attr *parse_attr, struct mlx5e_tc_flow *flow); -int alloc_mod_hdr_actions(struct mlx5_core_dev *mdev, - int namespace, - struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); -void dealloc_mod_hdr_actions(struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts); - struct mlx5e_tc_flow; u32 mlx5e_tc_get_flow_tun_id(struct mlx5e_tc_flow *flow); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c index 425c91814b34..c275fe028b6d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/indir_table.c @@ -14,6 +14,7 @@ #include "fs_core.h" #include "esw/indir_table.h" #include "lib/fs_chains.h" +#include "en/mod_hdr.h" #define MLX5_ESW_INDIR_TABLE_SIZE 128 #define MLX5_ESW_INDIR_TABLE_RECIRC_IDX_MAX (MLX5_ESW_INDIR_TABLE_SIZE - 2) @@ -226,7 +227,7 @@ static int mlx5_esw_indir_table_rule_get(struct mlx5_eswitch *esw, goto err_handle; } - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); rule->handle = handle; rule->vni = esw_attr->rx_tun_attr->vni; rule->mh = flow_act.modify_hdr; @@ -243,7 +244,7 @@ err_table: mlx5_modify_header_dealloc(esw->dev, flow_act.modify_hdr); err_mod_hdr_alloc: err_mod_hdr_regc1: - dealloc_mod_hdr_actions(&mod_acts); + mlx5e_mod_hdr_dealloc(&mod_acts); err_mod_hdr_regc0: err_ethertype: kfree(rule); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c index df277a6cddc0..2b52f7c09152 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/legacy.c @@ -522,9 +522,7 @@ int mlx5_eswitch_set_vport_rate(struct mlx5_eswitch *esw, u16 vport, return PTR_ERR(evport); mutex_lock(&esw->state_lock); - err = mlx5_esw_qos_set_vport_min_rate(esw, evport, min_rate, NULL); - if (!err) - err = mlx5_esw_qos_set_vport_max_rate(esw, evport, max_rate, NULL); + err = mlx5_esw_qos_set_vport_rate(esw, evport, max_rate, min_rate); mutex_unlock(&esw->state_lock); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c index d377ddc70fc7..11bbcd5f5b8b 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.c @@ -204,10 +204,8 @@ static int esw_qos_normalize_groups_min_rate(struct mlx5_eswitch *esw, u32 divid return 0; } -int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, - struct mlx5_vport *evport, - u32 min_rate, - struct netlink_ext_ack *extack) +static int esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport, + u32 min_rate, struct netlink_ext_ack *extack) { u32 fw_max_bw_share, previous_min_rate; bool min_rate_supported; @@ -231,10 +229,8 @@ int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, return err; } -int mlx5_esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw, - struct mlx5_vport *evport, - u32 max_rate, - struct netlink_ext_ack *extack) +static int esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport, + u32 max_rate, struct netlink_ext_ack *extack) { u32 act_max_rate = max_rate; bool max_rate_supported; @@ -432,16 +428,13 @@ static int esw_qos_vport_update_group(struct mlx5_eswitch *esw, } static struct mlx5_esw_rate_group * -esw_qos_create_rate_group(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) +__esw_qos_create_rate_group(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) { u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; struct mlx5_esw_rate_group *group; u32 divider; int err; - if (!MLX5_CAP_QOS(esw->dev, log_esw_max_sched_depth)) - return ERR_PTR(-EOPNOTSUPP); - group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) return ERR_PTR(-ENOMEM); @@ -482,9 +475,32 @@ err_sched_elem: return ERR_PTR(err); } -static int esw_qos_destroy_rate_group(struct mlx5_eswitch *esw, - struct mlx5_esw_rate_group *group, - struct netlink_ext_ack *extack) +static int esw_qos_get(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack); +static void esw_qos_put(struct mlx5_eswitch *esw); + +static struct mlx5_esw_rate_group * +esw_qos_create_rate_group(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) +{ + struct mlx5_esw_rate_group *group; + int err; + + if (!MLX5_CAP_QOS(esw->dev, log_esw_max_sched_depth)) + return ERR_PTR(-EOPNOTSUPP); + + err = esw_qos_get(esw, extack); + if (err) + return ERR_PTR(err); + + group = __esw_qos_create_rate_group(esw, extack); + if (IS_ERR(group)) + esw_qos_put(esw); + + return group; +} + +static int __esw_qos_destroy_rate_group(struct mlx5_eswitch *esw, + struct mlx5_esw_rate_group *group, + struct netlink_ext_ack *extack) { u32 divider; int err; @@ -503,7 +519,21 @@ static int esw_qos_destroy_rate_group(struct mlx5_eswitch *esw, NL_SET_ERR_MSG_MOD(extack, "E-Switch destroy TSAR_ID failed"); trace_mlx5_esw_group_qos_destroy(esw->dev, group, group->tsar_ix); + kfree(group); + + return err; +} + +static int esw_qos_destroy_rate_group(struct mlx5_eswitch *esw, + struct mlx5_esw_rate_group *group, + struct netlink_ext_ack *extack) +{ + int err; + + err = __esw_qos_destroy_rate_group(esw, group, extack); + esw_qos_put(esw); + return err; } @@ -526,7 +556,7 @@ static bool esw_qos_element_type_supported(struct mlx5_core_dev *dev, int type) return false; } -void mlx5_esw_qos_create(struct mlx5_eswitch *esw) +static int esw_qos_create(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) { u32 tsar_ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; struct mlx5_core_dev *dev = esw->dev; @@ -534,14 +564,10 @@ void mlx5_esw_qos_create(struct mlx5_eswitch *esw) int err; if (!MLX5_CAP_GEN(dev, qos) || !MLX5_CAP_QOS(dev, esw_scheduling)) - return; + return -EOPNOTSUPP; if (!esw_qos_element_type_supported(dev, SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR)) - return; - - mutex_lock(&esw->state_lock); - if (esw->qos.enabled) - goto unlock; + return -EOPNOTSUPP; MLX5_SET(scheduling_context, tsar_ctx, element_type, SCHEDULING_CONTEXT_ELEMENT_TYPE_TSAR); @@ -555,75 +581,94 @@ void mlx5_esw_qos_create(struct mlx5_eswitch *esw) &esw->qos.root_tsar_ix); if (err) { esw_warn(dev, "E-Switch create root TSAR failed (%d)\n", err); - goto unlock; + return err; } INIT_LIST_HEAD(&esw->qos.groups); if (MLX5_CAP_QOS(dev, log_esw_max_sched_depth)) { - esw->qos.group0 = esw_qos_create_rate_group(esw, NULL); + esw->qos.group0 = __esw_qos_create_rate_group(esw, extack); if (IS_ERR(esw->qos.group0)) { esw_warn(dev, "E-Switch create rate group 0 failed (%ld)\n", PTR_ERR(esw->qos.group0)); + err = PTR_ERR(esw->qos.group0); goto err_group0; } } - esw->qos.enabled = true; -unlock: - mutex_unlock(&esw->state_lock); - return; + refcount_set(&esw->qos.refcnt, 1); + + return 0; err_group0: - err = mlx5_destroy_scheduling_element_cmd(esw->dev, - SCHEDULING_HIERARCHY_E_SWITCH, - esw->qos.root_tsar_ix); - if (err) - esw_warn(esw->dev, "E-Switch destroy root TSAR failed (%d)\n", err); - mutex_unlock(&esw->state_lock); + if (mlx5_destroy_scheduling_element_cmd(esw->dev, SCHEDULING_HIERARCHY_E_SWITCH, + esw->qos.root_tsar_ix)) + esw_warn(esw->dev, "E-Switch destroy root TSAR failed.\n"); + + return err; } -void mlx5_esw_qos_destroy(struct mlx5_eswitch *esw) +static void esw_qos_destroy(struct mlx5_eswitch *esw) { - struct devlink *devlink = priv_to_devlink(esw->dev); int err; - devlink_rate_nodes_destroy(devlink); - mutex_lock(&esw->state_lock); - if (!esw->qos.enabled) - goto unlock; - if (esw->qos.group0) - esw_qos_destroy_rate_group(esw, esw->qos.group0, NULL); + __esw_qos_destroy_rate_group(esw, esw->qos.group0, NULL); err = mlx5_destroy_scheduling_element_cmd(esw->dev, SCHEDULING_HIERARCHY_E_SWITCH, esw->qos.root_tsar_ix); if (err) esw_warn(esw->dev, "E-Switch destroy root TSAR failed (%d)\n", err); +} - esw->qos.enabled = false; -unlock: - mutex_unlock(&esw->state_lock); +static int esw_qos_get(struct mlx5_eswitch *esw, struct netlink_ext_ack *extack) +{ + int err = 0; + + lockdep_assert_held(&esw->state_lock); + + if (!refcount_inc_not_zero(&esw->qos.refcnt)) { + /* esw_qos_create() set refcount to 1 only on success. + * No need to decrement on failure. + */ + err = esw_qos_create(esw, extack); + } + + return err; } -int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, - u32 max_rate, u32 bw_share) +static void esw_qos_put(struct mlx5_eswitch *esw) +{ + lockdep_assert_held(&esw->state_lock); + if (refcount_dec_and_test(&esw->qos.refcnt)) + esw_qos_destroy(esw); +} + +static int esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, + u32 max_rate, u32 bw_share, struct netlink_ext_ack *extack) { int err; lockdep_assert_held(&esw->state_lock); - if (!esw->qos.enabled) + if (vport->qos.enabled) return 0; - if (vport->qos.enabled) - return -EEXIST; + err = esw_qos_get(esw, extack); + if (err) + return err; vport->qos.group = esw->qos.group0; err = esw_qos_vport_create_sched_element(esw, vport, max_rate, bw_share); - if (!err) { - vport->qos.enabled = true; - trace_mlx5_esw_vport_qos_create(vport, bw_share, max_rate); - } + if (err) + goto err_out; + + vport->qos.enabled = true; + trace_mlx5_esw_vport_qos_create(vport, bw_share, max_rate); + + return 0; + +err_out: + esw_qos_put(esw); return err; } @@ -633,7 +678,7 @@ void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vpo int err; lockdep_assert_held(&esw->state_lock); - if (!esw->qos.enabled || !vport->qos.enabled) + if (!vport->qos.enabled) return; WARN(vport->qos.group && vport->qos.group != esw->qos.group0, "Disabling QoS on port before detaching it from group"); @@ -645,8 +690,27 @@ void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vpo esw_warn(esw->dev, "E-Switch destroy TSAR vport element failed (vport=%d,err=%d)\n", vport->vport, err); - vport->qos.enabled = false; + memset(&vport->qos, 0, sizeof(vport->qos)); trace_mlx5_esw_vport_qos_destroy(vport); + + esw_qos_put(esw); +} + +int mlx5_esw_qos_set_vport_rate(struct mlx5_eswitch *esw, struct mlx5_vport *vport, + u32 min_rate, u32 max_rate) +{ + int err; + + lockdep_assert_held(&esw->state_lock); + err = esw_qos_vport_enable(esw, vport, 0, 0, NULL); + if (err) + return err; + + err = esw_qos_set_vport_min_rate(esw, vport, min_rate, NULL); + if (!err) + err = esw_qos_set_vport_max_rate(esw, vport, max_rate, NULL); + + return err; } int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 rate_mbps) @@ -654,22 +718,29 @@ int mlx5_esw_qos_modify_vport_rate(struct mlx5_eswitch *esw, u16 vport_num, u32 u32 ctx[MLX5_ST_SZ_DW(scheduling_context)] = {}; struct mlx5_vport *vport; u32 bitmask; + int err; vport = mlx5_eswitch_get_vport(esw, vport_num); if (IS_ERR(vport)) return PTR_ERR(vport); - if (!vport->qos.enabled) - return -EOPNOTSUPP; - - MLX5_SET(scheduling_context, ctx, max_average_bw, rate_mbps); - bitmask = MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW; + mutex_lock(&esw->state_lock); + if (!vport->qos.enabled) { + /* Eswitch QoS wasn't enabled yet. Enable it and vport QoS. */ + err = esw_qos_vport_enable(esw, vport, rate_mbps, vport->qos.bw_share, NULL); + } else { + MLX5_SET(scheduling_context, ctx, max_average_bw, rate_mbps); + + bitmask = MODIFY_SCHEDULING_ELEMENT_IN_MODIFY_BITMASK_MAX_AVERAGE_BW; + err = mlx5_modify_scheduling_element_cmd(esw->dev, + SCHEDULING_HIERARCHY_E_SWITCH, + ctx, + vport->qos.esw_tsar_ix, + bitmask); + } + mutex_unlock(&esw->state_lock); - return mlx5_modify_scheduling_element_cmd(esw->dev, - SCHEDULING_HIERARCHY_E_SWITCH, - ctx, - vport->qos.esw_tsar_ix, - bitmask); + return err; } #define MLX5_LINKSPEED_UNIT 125000 /* 1Mbps in Bps */ @@ -728,7 +799,12 @@ int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void return err; mutex_lock(&esw->state_lock); - err = mlx5_esw_qos_set_vport_min_rate(esw, vport, tx_share, extack); + err = esw_qos_vport_enable(esw, vport, 0, 0, extack); + if (err) + goto unlock; + + err = esw_qos_set_vport_min_rate(esw, vport, tx_share, extack); +unlock: mutex_unlock(&esw->state_lock); return err; } @@ -749,7 +825,12 @@ int mlx5_esw_devlink_rate_leaf_tx_max_set(struct devlink_rate *rate_leaf, void * return err; mutex_lock(&esw->state_lock); - err = mlx5_esw_qos_set_vport_max_rate(esw, vport, tx_max, extack); + err = esw_qos_vport_enable(esw, vport, 0, 0, extack); + if (err) + goto unlock; + + err = esw_qos_set_vport_max_rate(esw, vport, tx_max, extack); +unlock: mutex_unlock(&esw->state_lock); return err; } @@ -846,7 +927,9 @@ int mlx5_esw_qos_vport_update_group(struct mlx5_eswitch *esw, int err; mutex_lock(&esw->state_lock); - err = esw_qos_vport_update_group(esw, vport, group, extack); + err = esw_qos_vport_enable(esw, vport, 0, 0, extack); + if (!err) + err = esw_qos_vport_update_group(esw, vport, group, extack); mutex_unlock(&esw->state_lock); return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h index 28451abe2d2f..0141e9d52037 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/qos.h @@ -6,18 +6,8 @@ #ifdef CONFIG_MLX5_ESWITCH -int mlx5_esw_qos_set_vport_min_rate(struct mlx5_eswitch *esw, - struct mlx5_vport *evport, - u32 min_rate, - struct netlink_ext_ack *extack); -int mlx5_esw_qos_set_vport_max_rate(struct mlx5_eswitch *esw, - struct mlx5_vport *evport, - u32 max_rate, - struct netlink_ext_ack *extack); -void mlx5_esw_qos_create(struct mlx5_eswitch *esw); -void mlx5_esw_qos_destroy(struct mlx5_eswitch *esw); -int mlx5_esw_qos_vport_enable(struct mlx5_eswitch *esw, struct mlx5_vport *vport, - u32 max_rate, u32 bw_share); +int mlx5_esw_qos_set_vport_rate(struct mlx5_eswitch *esw, struct mlx5_vport *evport, + u32 max_rate, u32 min_rate); void mlx5_esw_qos_vport_disable(struct mlx5_eswitch *esw, struct mlx5_vport *vport); int mlx5_esw_devlink_rate_leaf_tx_share_set(struct devlink_rate *rate_leaf, void *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 51a8cecc4a7c..458ec0bca1b8 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -781,9 +781,6 @@ static int esw_vport_setup(struct mlx5_eswitch *esw, struct mlx5_vport *vport) if (err) return err; - /* Attach vport to the eswitch rate limiter */ - mlx5_esw_qos_vport_enable(esw, vport, vport->qos.max_rate, vport->qos.bw_share); - if (mlx5_esw_is_manager_vport(esw, vport_num)) return 0; @@ -1260,8 +1257,6 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) mlx5_eswitch_update_num_of_vfs(esw, num_vfs); - mlx5_esw_qos_create(esw); - esw->mode = mode; if (mode == MLX5_ESWITCH_LEGACY) { @@ -1290,7 +1285,6 @@ abort: if (mode == MLX5_ESWITCH_OFFLOADS) mlx5_rescan_drivers(esw->dev); - mlx5_esw_qos_destroy(esw); mlx5_esw_acls_ns_cleanup(esw); return err; } @@ -1338,6 +1332,7 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) { + struct devlink *devlink = priv_to_devlink(esw->dev); int old_mode; lockdep_assert_held_write(&esw->mode_lock); @@ -1367,7 +1362,8 @@ void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) if (old_mode == MLX5_ESWITCH_OFFLOADS) mlx5_rescan_drivers(esw->dev); - mlx5_esw_qos_destroy(esw); + devlink_rate_nodes_destroy(devlink); + mlx5_esw_acls_ns_cleanup(esw); if (clear_vf) @@ -1576,6 +1572,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) lockdep_register_key(&esw->mode_lock_key); init_rwsem(&esw->mode_lock); lockdep_set_class(&esw->mode_lock, &esw->mode_lock_key); + refcount_set(&esw->qos.refcnt, 0); esw->enabled_vports = 0; esw->mode = MLX5_ESWITCH_NONE; @@ -1614,6 +1611,7 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) esw->dev->priv.eswitch = NULL; destroy_workqueue(esw->work_queue); + WARN_ON(refcount_read(&esw->qos.refcnt)); lockdep_unregister_key(&esw->mode_lock_key); mutex_destroy(&esw->state_lock); WARN_ON(!xa_empty(&esw->offloads.vhca_map)); @@ -1703,82 +1701,6 @@ bool mlx5_esw_is_sf_vport(struct mlx5_eswitch *esw, u16 vport_num) return mlx5_esw_check_port_type(esw, vport_num, MLX5_ESW_VPT_SF); } -static bool -is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num) -{ - return vport_num == MLX5_VPORT_PF || - mlx5_eswitch_is_vf_vport(esw, vport_num) || - mlx5_esw_is_sf_vport(esw, vport_num); -} - -int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, - u8 *hw_addr, int *hw_addr_len, - struct netlink_ext_ack *extack) -{ - struct mlx5_eswitch *esw; - struct mlx5_vport *vport; - int err = -EOPNOTSUPP; - u16 vport_num; - - esw = mlx5_devlink_eswitch_get(port->devlink); - if (IS_ERR(esw)) - return PTR_ERR(esw); - - vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); - if (!is_port_function_supported(esw, vport_num)) - return -EOPNOTSUPP; - - vport = mlx5_eswitch_get_vport(esw, vport_num); - if (IS_ERR(vport)) { - NL_SET_ERR_MSG_MOD(extack, "Invalid port"); - return PTR_ERR(vport); - } - - mutex_lock(&esw->state_lock); - if (vport->enabled) { - ether_addr_copy(hw_addr, vport->info.mac); - *hw_addr_len = ETH_ALEN; - err = 0; - } - mutex_unlock(&esw->state_lock); - return err; -} - -int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, - const u8 *hw_addr, int hw_addr_len, - struct netlink_ext_ack *extack) -{ - struct mlx5_eswitch *esw; - struct mlx5_vport *vport; - int err = -EOPNOTSUPP; - u16 vport_num; - - esw = mlx5_devlink_eswitch_get(port->devlink); - if (IS_ERR(esw)) { - NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); - return PTR_ERR(esw); - } - - vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); - if (!is_port_function_supported(esw, vport_num)) { - NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); - return -EINVAL; - } - vport = mlx5_eswitch_get_vport(esw, vport_num); - if (IS_ERR(vport)) { - NL_SET_ERR_MSG_MOD(extack, "Invalid port"); - return PTR_ERR(vport); - } - - mutex_lock(&esw->state_lock); - if (vport->enabled) - err = mlx5_esw_set_vport_mac_locked(esw, vport, hw_addr); - else - NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); - mutex_unlock(&esw->state_lock); - return err; -} - int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { @@ -1835,8 +1757,10 @@ int mlx5_eswitch_get_vport_config(struct mlx5_eswitch *esw, ivi->qos = evport->info.qos; ivi->spoofchk = evport->info.spoofchk; ivi->trusted = evport->info.trusted; - ivi->min_tx_rate = evport->qos.min_rate; - ivi->max_tx_rate = evport->qos.max_rate; + if (evport->qos.enabled) { + ivi->min_tx_rate = evport->qos.min_rate; + ivi->max_tx_rate = evport->qos.max_rate; + } mutex_unlock(&esw->state_lock); return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 42f8ee2e5d9f..513f741d16c7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -308,10 +308,14 @@ struct mlx5_eswitch { atomic64_t user_count; struct { - bool enabled; u32 root_tsar_ix; struct mlx5_esw_rate_group *group0; struct list_head groups; /* Protected by esw->state_lock */ + + /* Protected by esw->state_lock. + * Initially 0, meaning no QoS users and QoS is disabled. + */ + refcount_t refcnt; } qos; struct mlx5_esw_bridge_offloads *br_offloads; @@ -516,11 +520,6 @@ int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw, int __mlx5_eswitch_set_vport_vlan(struct mlx5_eswitch *esw, u16 vport, u16 vlan, u8 qos, u8 set_flags); -static inline bool mlx5_esw_qos_enabled(struct mlx5_eswitch *esw) -{ - return esw->qos.enabled; -} - static inline bool mlx5_eswitch_vlan_actions_supported(struct mlx5_core_dev *dev, u8 vlan_depth) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index 32bc08a39925..3bd5ae6ecd04 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -3867,3 +3867,62 @@ u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, return vport->metadata; } EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); + +static bool +is_port_function_supported(struct mlx5_eswitch *esw, u16 vport_num) +{ + return vport_num == MLX5_VPORT_PF || + mlx5_eswitch_is_vf_vport(esw, vport_num) || + mlx5_esw_is_sf_vport(esw, vport_num); +} + +int mlx5_devlink_port_function_hw_addr_get(struct devlink_port *port, + u8 *hw_addr, int *hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + struct mlx5_vport *vport; + u16 vport_num; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) + return PTR_ERR(esw); + + vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); + if (!is_port_function_supported(esw, vport_num)) + return -EOPNOTSUPP; + + vport = mlx5_eswitch_get_vport(esw, vport_num); + if (IS_ERR(vport)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid port"); + return PTR_ERR(vport); + } + + mutex_lock(&esw->state_lock); + ether_addr_copy(hw_addr, vport->info.mac); + *hw_addr_len = ETH_ALEN; + mutex_unlock(&esw->state_lock); + return 0; +} + +int mlx5_devlink_port_function_hw_addr_set(struct devlink_port *port, + const u8 *hw_addr, int hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct mlx5_eswitch *esw; + u16 vport_num; + + esw = mlx5_devlink_eswitch_get(port->devlink); + if (IS_ERR(esw)) { + NL_SET_ERR_MSG_MOD(extack, "Eswitch doesn't support set hw_addr"); + return PTR_ERR(esw); + } + + vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); + if (!is_port_function_supported(esw, vport_num)) { + NL_SET_ERR_MSG_MOD(extack, "Port doesn't support set hw_addr"); + return -EINVAL; + } + + return mlx5_eswitch_set_vport_mac(esw, vport_num, hw_addr); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c index 7e0e04cf26f8..b406e0367af6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_counters.c @@ -38,9 +38,10 @@ #include "fs_cmd.h" #define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000) +#define MLX5_FC_BULK_QUERY_ALLOC_PERIOD msecs_to_jiffies(180 * 1000) /* Max number of counters to query in bulk read is 32K */ #define MLX5_SW_MAX_COUNTERS_BULK BIT(15) -#define MLX5_SF_NUM_COUNTERS_BULK 8 +#define MLX5_INIT_COUNTERS_BULK 8 #define MLX5_FC_POOL_MAX_THRESHOLD BIT(18) #define MLX5_FC_POOL_USED_BUFF_RATIO 10 @@ -145,13 +146,15 @@ static void mlx5_fc_stats_remove(struct mlx5_core_dev *dev, spin_unlock(&fc_stats->counters_idr_lock); } -static int get_max_bulk_query_len(struct mlx5_core_dev *dev) +static int get_init_bulk_query_len(struct mlx5_core_dev *dev) { - int num_counters_bulk = mlx5_core_is_sf(dev) ? - MLX5_SF_NUM_COUNTERS_BULK : - MLX5_SW_MAX_COUNTERS_BULK; + return min_t(int, MLX5_INIT_COUNTERS_BULK, + (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk))); +} - return min_t(int, num_counters_bulk, +static int get_max_bulk_query_len(struct mlx5_core_dev *dev) +{ + return min_t(int, MLX5_SW_MAX_COUNTERS_BULK, (1 << MLX5_CAP_GEN(dev, log_max_flow_counter_bulk))); } @@ -177,7 +180,7 @@ static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, { struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; bool query_more_counters = (first->id <= last_id); - int max_bulk_len = get_max_bulk_query_len(dev); + int cur_bulk_len = fc_stats->bulk_query_len; u32 *data = fc_stats->bulk_query_out; struct mlx5_fc *counter = first; u32 bulk_base_id; @@ -189,7 +192,7 @@ static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev, bulk_base_id = counter->id & ~0x3; /* number of counters to query inc. the last counter */ - bulk_len = min_t(int, max_bulk_len, + bulk_len = min_t(int, cur_bulk_len, ALIGN(last_id - bulk_base_id + 1, 4)); err = mlx5_cmd_fc_bulk_query(dev, bulk_base_id, bulk_len, @@ -230,6 +233,41 @@ static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter) mlx5_fc_free(dev, counter); } +static void mlx5_fc_stats_bulk_query_size_increase(struct mlx5_core_dev *dev) +{ + struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; + int max_bulk_len = get_max_bulk_query_len(dev); + unsigned long now = jiffies; + u32 *bulk_query_out_tmp; + int max_out_len; + + if (fc_stats->bulk_query_alloc_failed && + time_before(now, fc_stats->next_bulk_query_alloc)) + return; + + max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); + bulk_query_out_tmp = kzalloc(max_out_len, GFP_KERNEL); + if (!bulk_query_out_tmp) { + mlx5_core_warn_once(dev, + "Can't increase flow counters bulk query buffer size, insufficient memory, bulk_size(%d)\n", + max_bulk_len); + fc_stats->bulk_query_alloc_failed = true; + fc_stats->next_bulk_query_alloc = + now + MLX5_FC_BULK_QUERY_ALLOC_PERIOD; + return; + } + + kfree(fc_stats->bulk_query_out); + fc_stats->bulk_query_out = bulk_query_out_tmp; + fc_stats->bulk_query_len = max_bulk_len; + if (fc_stats->bulk_query_alloc_failed) { + mlx5_core_info(dev, + "Flow counters bulk query buffer size increased, bulk_size(%d)\n", + max_bulk_len); + fc_stats->bulk_query_alloc_failed = false; + } +} + static void mlx5_fc_stats_work(struct work_struct *work) { struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev, @@ -247,15 +285,22 @@ static void mlx5_fc_stats_work(struct work_struct *work) queue_delayed_work(fc_stats->wq, &fc_stats->work, fc_stats->sampling_interval); - llist_for_each_entry(counter, addlist, addlist) + llist_for_each_entry(counter, addlist, addlist) { mlx5_fc_stats_insert(dev, counter); + fc_stats->num_counters++; + } llist_for_each_entry_safe(counter, tmp, dellist, dellist) { mlx5_fc_stats_remove(dev, counter); mlx5_fc_release(dev, counter); + fc_stats->num_counters--; } + if (fc_stats->bulk_query_len < get_max_bulk_query_len(dev) && + fc_stats->num_counters > get_init_bulk_query_len(dev)) + mlx5_fc_stats_bulk_query_size_increase(dev); + if (time_before(now, fc_stats->next_query) || list_empty(&fc_stats->counters)) return; @@ -378,8 +423,8 @@ EXPORT_SYMBOL(mlx5_fc_destroy); int mlx5_init_fc_stats(struct mlx5_core_dev *dev) { struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats; - int max_bulk_len; - int max_out_len; + int init_bulk_len; + int init_out_len; spin_lock_init(&fc_stats->counters_idr_lock); idr_init(&fc_stats->counters_idr); @@ -387,11 +432,12 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev) init_llist_head(&fc_stats->addlist); init_llist_head(&fc_stats->dellist); - max_bulk_len = get_max_bulk_query_len(dev); - max_out_len = mlx5_cmd_fc_get_bulk_query_out_len(max_bulk_len); - fc_stats->bulk_query_out = kzalloc(max_out_len, GFP_KERNEL); + init_bulk_len = get_init_bulk_query_len(dev); + init_out_len = mlx5_cmd_fc_get_bulk_query_out_len(init_bulk_len); + fc_stats->bulk_query_out = kzalloc(init_out_len, GFP_KERNEL); if (!fc_stats->bulk_query_out) return -ENOMEM; + fc_stats->bulk_query_len = init_bulk_len; fc_stats->wq = create_singlethread_workqueue("mlx5_fc"); if (!fc_stats->wq) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 3ca998874c50..737df402c927 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -420,6 +420,11 @@ static void print_health_info(struct mlx5_core_dev *dev) if (!ioread8(&h->synd)) return; + if (ioread32be(&h->fw_ver) == 0xFFFFFFFF) { + mlx5_log(dev, LOGLEVEL_ERR, "PCI slot is unavailable\n"); + return; + } + rfr_severity = ioread8(&h->rfr_severity); severity = mlx5_health_get_severity(rfr_severity); mlx5_log(dev, severity, "Health issue observed, %s, severity(%d) %s:\n", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 962d41418ce7..f4f7eaf16446 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -67,7 +67,9 @@ static void mlx5i_get_ethtool_stats(struct net_device *dev, } static int mlx5i_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = mlx5i_epriv(dev); @@ -75,7 +77,9 @@ static int mlx5i_set_ringparam(struct net_device *dev, } static void mlx5i_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct mlx5e_priv *priv = mlx5i_epriv(dev); @@ -105,7 +109,7 @@ static int mlx5i_set_coalesce(struct net_device *netdev, { struct mlx5e_priv *priv = mlx5i_epriv(netdev); - return mlx5e_ethtool_set_coalesce(priv, coal); + return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack); } static int mlx5i_get_coalesce(struct net_device *netdev, @@ -115,7 +119,7 @@ static int mlx5i_get_coalesce(struct net_device *netdev, { struct mlx5e_priv *priv = mlx5i_epriv(netdev); - return mlx5e_ethtool_get_coalesce(priv, coal); + return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal); } static int mlx5i_get_ts_info(struct net_device *netdev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index ea1efdecc88c..051b20ec7bdb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -110,7 +110,7 @@ void mlx5i_cleanup(struct mlx5e_priv *priv) static void mlx5i_grp_sw_update_stats(struct mlx5e_priv *priv) { - struct mlx5e_sw_stats s = { 0 }; + struct rtnl_link_stats64 s = {}; int i, j; for (i = 0; i < priv->stats_nch; i++) { @@ -128,11 +128,17 @@ static void mlx5i_grp_sw_update_stats(struct mlx5e_priv *priv) s.tx_packets += sq_stats->packets; s.tx_bytes += sq_stats->bytes; - s.tx_queue_dropped += sq_stats->dropped; + s.tx_dropped += sq_stats->dropped; } } - memcpy(&priv->stats.sw, &s, sizeof(s)); + memset(&priv->stats.sw, 0, sizeof(s)); + + priv->stats.sw.rx_packets = s.rx_packets; + priv->stats.sw.rx_bytes = s.rx_bytes; + priv->stats.sw.tx_packets = s.tx_packets; + priv->stats.sw.tx_bytes = s.tx_bytes; + priv->stats.sw.tx_queue_dropped = s.tx_dropped; } void mlx5i_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index 7df9c7f8d9c8..d97c9e86d7b3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -1604,12 +1604,28 @@ static void remove_one(struct pci_dev *pdev) mlx5_devlink_free(devlink); } +#define mlx5_pci_trace(dev, fmt, ...) ({ \ + struct mlx5_core_dev *__dev = (dev); \ + mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \ + __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \ + __dev->pci_status, ##__VA_ARGS__); \ +}) + +static const char *result2str(enum pci_ers_result result) +{ + return result == PCI_ERS_RESULT_NEED_RESET ? "need reset" : + result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" : + result == PCI_ERS_RESULT_RECOVERED ? "recovered" : + "unknown"; +} + static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + enum pci_ers_result res; - mlx5_core_info(dev, "%s was called\n", __func__); + mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state); mlx5_enter_error_state(dev, false); mlx5_error_sw_reset(dev); @@ -1617,8 +1633,11 @@ static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev, mlx5_drain_health_wq(dev); mlx5_pci_disable_device(dev); - return state == pci_channel_io_perm_failure ? + res = state == pci_channel_io_perm_failure ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; + + mlx5_pci_trace(dev, "Exit, result = %d, %s\n", res, result2str(res)); + return res; } /* wait for the device to show vital signs by waiting @@ -1652,28 +1671,34 @@ static int wait_vital(struct pci_dev *pdev) static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev) { + enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT; struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err; - mlx5_core_info(dev, "%s was called\n", __func__); + mlx5_pci_trace(dev, "Enter\n"); err = mlx5_pci_enable_device(dev); if (err) { mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n", __func__, err); - return PCI_ERS_RESULT_DISCONNECT; + goto out; } pci_set_master(pdev); pci_restore_state(pdev); pci_save_state(pdev); - if (wait_vital(pdev)) { - mlx5_core_err(dev, "%s: wait_vital timed out\n", __func__); - return PCI_ERS_RESULT_DISCONNECT; + err = wait_vital(pdev); + if (err) { + mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n", + __func__, err); + goto out; } - return PCI_ERS_RESULT_RECOVERED; + res = PCI_ERS_RESULT_RECOVERED; +out: + mlx5_pci_trace(dev, "Exit, err = %d, result = %d, %s\n", err, res, result2str(res)); + return res; } static void mlx5_pci_resume(struct pci_dev *pdev) @@ -1681,14 +1706,12 @@ static void mlx5_pci_resume(struct pci_dev *pdev) struct mlx5_core_dev *dev = pci_get_drvdata(pdev); int err; - mlx5_core_info(dev, "%s was called\n", __func__); + mlx5_pci_trace(dev, "Enter, loading driver..\n"); err = mlx5_load_one(dev); - if (err) - mlx5_core_err(dev, "%s: mlx5_load_one failed with error code: %d\n", - __func__, err); - else - mlx5_core_info(dev, "%s: device recovered\n", __func__); + + mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err, + !err ? "recovered" : "Failed"); } static const struct pci_error_handlers mlx5_err_handler = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c index 830444f927d4..19bf2b66707d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/pci_irq.c @@ -479,7 +479,7 @@ irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name, pool->xa_num_irqs.max = start + size - 1; if (name) snprintf(pool->name, MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS, - name); + "%s", name); pool->min_threshold = min_threshold * MLX5_EQ_REFS_PER_IRQ; pool->max_threshold = max_threshold * MLX5_EQ_REFS_PER_IRQ; mlx5_core_dbg(dev, "pool->name = %s, pool->size = %d, pool->start = %d", diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c index 252d6017387d..17aa348989cb 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/hw_table.c @@ -247,7 +247,7 @@ int mlx5_sf_hw_table_init(struct mlx5_core_dev *dev) { struct mlx5_sf_hw_table *table; u16 max_ext_fn = 0; - u16 ext_base_id; + u16 ext_base_id = 0; u16 max_fn = 0; u16 base_id; int err; diff --git a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c index 92b798f8e73a..ceeb7f4c3f6c 100644 --- a/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxbf_gige/mlxbf_gige_ethtool.c @@ -33,8 +33,11 @@ static void mlxbf_gige_get_regs(struct net_device *netdev, memcpy_fromio(p, priv->base, MLXBF_GIGE_MMIO_REG_SZ); } -static void mlxbf_gige_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) +static void +mlxbf_gige_get_ringparam(struct net_device *netdev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct mlxbf_gige *priv = netdev_priv(netdev); diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.c b/drivers/net/ethernet/mellanox/mlxsw/core.c index 3fd3812b8f31..7e89d5980d5e 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core.c @@ -47,7 +47,7 @@ static struct workqueue_struct *mlxsw_owq; struct mlxsw_core_port { struct devlink_port devlink_port; void *port_driver_priv; - u8 local_port; + u16 local_port; }; void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port) @@ -77,7 +77,7 @@ struct mlxsw_core { bool enable_string_tlv; } emad; struct { - u8 *mapping; /* lag_id+port_index to local_port mapping */ + u16 *mapping; /* lag_id+port_index to local_port mapping */ } lag; struct mlxsw_res res; struct mlxsw_hwmon *hwmon; @@ -160,7 +160,7 @@ static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core, bool reload) devlink_resource_occ_get_unregister(devlink, MLXSW_CORE_RESOURCE_PORTS); if (!reload) - devlink_resources_unregister(priv_to_devlink(mlxsw_core), NULL); + devlink_resources_unregister(priv_to_devlink(mlxsw_core)); kfree(mlxsw_core->ports); } @@ -718,7 +718,7 @@ static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core, } /* called with rcu read lock held */ -static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port, +static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u16 local_port, void *priv) { struct mlxsw_core *mlxsw_core = priv; @@ -1959,7 +1959,7 @@ __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) && MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) { - alloc_size = sizeof(u8) * + alloc_size = sizeof(*mlxsw_core->lag.mapping) * MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) * MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL); @@ -2033,7 +2033,7 @@ err_alloc_lag_mapping: mlxsw_ports_fini(mlxsw_core, reload); err_ports_init: if (!reload) - devlink_resources_unregister(devlink, NULL); + devlink_resources_unregister(devlink); err_register_resources: mlxsw_bus->fini(bus_priv); err_bus_init: @@ -2099,7 +2099,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, kfree(mlxsw_core->lag.mapping); mlxsw_ports_fini(mlxsw_core, reload); if (!reload) - devlink_resources_unregister(devlink, NULL); + devlink_resources_unregister(devlink); mlxsw_core->bus->fini(mlxsw_core->bus_priv); if (!reload) devlink_free(devlink); @@ -2108,7 +2108,7 @@ void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, reload_fail_deinit: mlxsw_core_params_unregister(mlxsw_core); - devlink_resources_unregister(devlink, NULL); + devlink_resources_unregister(devlink); devlink_free(devlink); } EXPORT_SYMBOL(mlxsw_core_bus_device_unregister); @@ -2130,7 +2130,7 @@ int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, EXPORT_SYMBOL(mlxsw_core_skb_transmit); void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { if (mlxsw_core->driver->ptp_transmitted) mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb, @@ -2208,7 +2208,7 @@ mlxsw_core_rx_listener_state_set(struct mlxsw_core *mlxsw_core, rxl_item->enabled = enabled; } -static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port, +static void mlxsw_core_event_listener_func(struct sk_buff *skb, u16 local_port, void *priv) { struct mlxsw_event_listener_item *event_listener_item = priv; @@ -2641,7 +2641,7 @@ void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, { struct mlxsw_rx_listener_item *rxl_item; const struct mlxsw_rx_listener *rxl; - u8 local_port; + u16 local_port; bool found = false; if (rx_info->is_lag) { @@ -2699,7 +2699,7 @@ static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core, } void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 port_index, u8 local_port) + u16 lag_id, u8 port_index, u16 local_port) { int index = mlxsw_core_lag_mapping_index(mlxsw_core, lag_id, port_index); @@ -2708,8 +2708,8 @@ void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core, } EXPORT_SYMBOL(mlxsw_core_lag_mapping_set); -u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 port_index) +u16 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core, + u16 lag_id, u8 port_index) { int index = mlxsw_core_lag_mapping_index(mlxsw_core, lag_id, port_index); @@ -2719,7 +2719,7 @@ u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core, EXPORT_SYMBOL(mlxsw_core_lag_mapping_get); void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 local_port) + u16 lag_id, u16 local_port) { int i; @@ -2747,7 +2747,7 @@ u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core, } EXPORT_SYMBOL(mlxsw_core_res_get); -static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, +static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u16 local_port, enum devlink_port_flavour flavour, u32 port_number, bool split, u32 split_port_subnumber, @@ -2778,7 +2778,7 @@ static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, return err; } -static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port) +static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u16 local_port) { struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; @@ -2788,7 +2788,7 @@ static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port) memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port)); } -int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, +int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u16 local_port, u32 port_number, bool split, u32 split_port_subnumber, bool splittable, u32 lanes, @@ -2810,7 +2810,7 @@ int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, } EXPORT_SYMBOL(mlxsw_core_port_init); -void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port) +void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u16 local_port) { atomic_dec(&mlxsw_core->active_ports_count); @@ -2845,7 +2845,7 @@ void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core) } EXPORT_SYMBOL(mlxsw_core_cpu_port_fini); -void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv, struct net_device *dev) { struct mlxsw_core_port *mlxsw_core_port = @@ -2857,7 +2857,7 @@ void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port, } EXPORT_SYMBOL(mlxsw_core_port_eth_set); -void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv) { struct mlxsw_core_port *mlxsw_core_port = @@ -2869,7 +2869,7 @@ void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port, } EXPORT_SYMBOL(mlxsw_core_port_ib_set); -void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv) { struct mlxsw_core_port *mlxsw_core_port = @@ -2882,7 +2882,7 @@ void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port, EXPORT_SYMBOL(mlxsw_core_port_clear); enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, - u8 local_port) + u16 local_port) { struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; @@ -2895,7 +2895,7 @@ EXPORT_SYMBOL(mlxsw_core_port_type_get); struct devlink_port * mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, - u8 local_port) + u16 local_port) { struct mlxsw_core_port *mlxsw_core_port = &mlxsw_core->ports[local_port]; @@ -2905,7 +2905,7 @@ mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, } EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get); -bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u8 local_port) +bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u16 local_port) { const struct mlxsw_bus_info *bus_info = mlxsw_core->bus_info; int i; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core.h b/drivers/net/ethernet/mellanox/mlxsw/core.h index 12023a550007..f30bb8614e69 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core.h @@ -54,7 +54,7 @@ int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info, void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core, bool reload); struct mlxsw_tx_info { - u8 local_port; + u16 local_port; bool is_emad; }; @@ -67,7 +67,7 @@ struct mlxsw_rx_md_info { u16 tx_sys_port; u16 tx_lag_id; }; - u8 tx_lag_port_index; /* Valid when 'tx_port_is_lag' is set. */ + u16 tx_lag_port_index; /* Valid when 'tx_port_is_lag' is set. */ u8 tx_tc; u8 latency_valid:1, tx_congestion_valid:1, @@ -82,11 +82,11 @@ bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core, int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, const struct mlxsw_tx_info *tx_info); void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core, - struct sk_buff *skb, u8 local_port); + struct sk_buff *skb, u16 local_port); struct mlxsw_rx_listener { - void (*func)(struct sk_buff *skb, u8 local_port, void *priv); - u8 local_port; + void (*func)(struct sk_buff *skb, u16 local_port, void *priv); + u16 local_port; u8 mirror_reason; u16 trap_id; }; @@ -209,7 +209,7 @@ struct mlxsw_rx_info { u16 sys_port; u16 lag_id; } u; - u8 lag_port_index; + u16 lag_port_index; u8 mirror_reason; int trap_id; }; @@ -218,36 +218,36 @@ void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb, struct mlxsw_rx_info *rx_info); void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 port_index, u8 local_port); -u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 port_index); + u16 lag_id, u8 port_index, u16 local_port); +u16 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core, + u16 lag_id, u8 port_index); void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core, - u16 lag_id, u8 local_port); + u16 lag_id, u16 local_port); void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port); -int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port, +int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u16 local_port, u32 port_number, bool split, u32 split_port_subnumber, bool splittable, u32 lanes, const unsigned char *switch_id, unsigned char switch_id_len); -void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port); +void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u16 local_port); int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core, void *port_driver_priv, const unsigned char *switch_id, unsigned char switch_id_len); void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core); -void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv, struct net_device *dev); -void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv); -void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port, +void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u16 local_port, void *port_driver_priv); enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core, - u8 local_port); + u16 local_port); struct devlink_port * mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core, - u8 local_port); -bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u8 local_port); + u16 local_port); +bool mlxsw_core_port_is_xm(const struct mlxsw_core *mlxsw_core, u16 local_port); struct mlxsw_env *mlxsw_core_env(const struct mlxsw_core *mlxsw_core); int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay); @@ -316,11 +316,11 @@ struct mlxsw_driver { struct netlink_ext_ack *extack); void (*fini)(struct mlxsw_core *mlxsw_core); int (*basic_trap_groups_set)(struct mlxsw_core *mlxsw_core); - int (*port_type_set)(struct mlxsw_core *mlxsw_core, u8 local_port, + int (*port_type_set)(struct mlxsw_core *mlxsw_core, u16 local_port, enum devlink_port_type new_type); - int (*port_split)(struct mlxsw_core *mlxsw_core, u8 local_port, + int (*port_split)(struct mlxsw_core *mlxsw_core, u16 local_port, unsigned int count, struct netlink_ext_ack *extack); - int (*port_unsplit)(struct mlxsw_core *mlxsw_core, u8 local_port, + int (*port_unsplit)(struct mlxsw_core *mlxsw_core, u16 local_port, struct netlink_ext_ack *extack); int (*sb_pool_get)(struct mlxsw_core *mlxsw_core, unsigned int sb_index, u16 pool_index, @@ -394,7 +394,7 @@ struct mlxsw_driver { * is responsible for freeing the passed-in SKB. */ void (*ptp_transmitted)(struct mlxsw_core *mlxsw_core, - struct sk_buff *skb, u8 local_port); + struct sk_buff *skb, u16 local_port); u8 txhdr_len; const struct mlxsw_config_profile *profile; diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c index 78d9c0196f2b..77e82e6cf6e8 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.c @@ -113,7 +113,7 @@ static const struct rhashtable_params mlxsw_afa_set_ht_params = { }; struct mlxsw_afa_fwd_entry_ht_key { - u8 local_port; + u16 local_port; }; struct mlxsw_afa_fwd_entry { @@ -555,7 +555,7 @@ int mlxsw_afa_block_terminate(struct mlxsw_afa_block *block) EXPORT_SYMBOL(mlxsw_afa_block_terminate); static struct mlxsw_afa_fwd_entry * -mlxsw_afa_fwd_entry_create(struct mlxsw_afa *mlxsw_afa, u8 local_port) +mlxsw_afa_fwd_entry_create(struct mlxsw_afa *mlxsw_afa, u16 local_port) { struct mlxsw_afa_fwd_entry *fwd_entry; int err; @@ -598,7 +598,7 @@ static void mlxsw_afa_fwd_entry_destroy(struct mlxsw_afa *mlxsw_afa, } static struct mlxsw_afa_fwd_entry * -mlxsw_afa_fwd_entry_get(struct mlxsw_afa *mlxsw_afa, u8 local_port) +mlxsw_afa_fwd_entry_get(struct mlxsw_afa *mlxsw_afa, u16 local_port) { struct mlxsw_afa_fwd_entry_ht_key ht_key = {0}; struct mlxsw_afa_fwd_entry *fwd_entry; @@ -647,7 +647,7 @@ mlxsw_afa_fwd_entry_ref_destructor(struct mlxsw_afa_block *block, } static struct mlxsw_afa_fwd_entry_ref * -mlxsw_afa_fwd_entry_ref_create(struct mlxsw_afa_block *block, u8 local_port) +mlxsw_afa_fwd_entry_ref_create(struct mlxsw_afa_block *block, u16 local_port) { struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref; struct mlxsw_afa_fwd_entry *fwd_entry; @@ -1352,7 +1352,7 @@ EXPORT_SYMBOL(mlxsw_afa_block_append_trap_and_forward); struct mlxsw_afa_mirror { struct mlxsw_afa_resource resource; int span_id; - u8 local_in_port; + u16 local_in_port; bool ingress; }; @@ -1379,7 +1379,7 @@ mlxsw_afa_mirror_destructor(struct mlxsw_afa_block *block, } static struct mlxsw_afa_mirror * -mlxsw_afa_mirror_create(struct mlxsw_afa_block *block, u8 local_in_port, +mlxsw_afa_mirror_create(struct mlxsw_afa_block *block, u16 local_in_port, const struct net_device *out_dev, bool ingress) { struct mlxsw_afa_mirror *mirror; @@ -1423,7 +1423,7 @@ mlxsw_afa_block_append_allocated_mirror(struct mlxsw_afa_block *block, } int -mlxsw_afa_block_append_mirror(struct mlxsw_afa_block *block, u8 local_in_port, +mlxsw_afa_block_append_mirror(struct mlxsw_afa_block *block, u16 local_in_port, const struct net_device *out_dev, bool ingress, struct netlink_ext_ack *extack) { @@ -1663,7 +1663,7 @@ mlxsw_afa_forward_pack(char *payload, enum mlxsw_afa_forward_type type, } int mlxsw_afa_block_append_fwd(struct mlxsw_afa_block *block, - u8 local_port, bool in_port, + u16 local_port, bool in_port, struct netlink_ext_ack *extack) { struct mlxsw_afa_fwd_entry_ref *fwd_entry_ref; @@ -2038,7 +2038,7 @@ static void mlxsw_afa_sampler_pack(char *payload, u8 mirror_agent, u32 rate) struct mlxsw_afa_sampler { struct mlxsw_afa_resource resource; int span_id; - u8 local_port; + u16 local_port; bool ingress; }; @@ -2061,7 +2061,7 @@ static void mlxsw_afa_sampler_destructor(struct mlxsw_afa_block *block, } static struct mlxsw_afa_sampler * -mlxsw_afa_sampler_create(struct mlxsw_afa_block *block, u8 local_port, +mlxsw_afa_sampler_create(struct mlxsw_afa_block *block, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, struct netlink_ext_ack *extack) @@ -2104,7 +2104,7 @@ mlxsw_afa_block_append_allocated_sampler(struct mlxsw_afa_block *block, return 0; } -int mlxsw_afa_block_append_sampler(struct mlxsw_afa_block *block, u8 local_port, +int mlxsw_afa_block_append_sampler(struct mlxsw_afa_block *block, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, diff --git a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h index b65bf98eb5ab..16cbd6acbb01 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h +++ b/drivers/net/ethernet/mellanox/mlxsw/core_acl_flex_actions.h @@ -17,24 +17,24 @@ struct mlxsw_afa_ops { void (*kvdl_set_del)(void *priv, u32 kvdl_index, bool is_first); int (*kvdl_set_activity_get)(void *priv, u32 kvdl_index, bool *activity); - int (*kvdl_fwd_entry_add)(void *priv, u32 *p_kvdl_index, u8 local_port); + int (*kvdl_fwd_entry_add)(void *priv, u32 *p_kvdl_index, u16 local_port); void (*kvdl_fwd_entry_del)(void *priv, u32 kvdl_index); int (*counter_index_get)(void *priv, unsigned int *p_counter_index); void (*counter_index_put)(void *priv, unsigned int counter_index); - int (*mirror_add)(void *priv, u8 local_in_port, + int (*mirror_add)(void *priv, u16 local_in_port, const struct net_device *out_dev, bool ingress, int *p_span_id); - void (*mirror_del)(void *priv, u8 local_in_port, int span_id, + void (*mirror_del)(void *priv, u16 local_in_port, int span_id, bool ingress); int (*policer_add)(void *priv, u64 rate_bytes_ps, u32 burst, u16 *p_policer_index, struct netlink_ext_ack *extack); void (*policer_del)(void *priv, u16 policer_index); - int (*sampler_add)(void *priv, u8 local_port, + int (*sampler_add)(void *priv, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, int *p_span_id, struct netlink_ext_ack *extack); - void (*sampler_del)(void *priv, u8 local_port, int span_id, + void (*sampler_del)(void *priv, u16 local_port, int span_id, bool ingress); bool dummy_first_set; }; @@ -62,12 +62,12 @@ int mlxsw_afa_block_append_trap(struct mlxsw_afa_block *block, u16 trap_id); int mlxsw_afa_block_append_trap_and_forward(struct mlxsw_afa_block *block, u16 trap_id); int mlxsw_afa_block_append_mirror(struct mlxsw_afa_block *block, - u8 local_in_port, + u16 local_in_port, const struct net_device *out_dev, bool ingress, struct netlink_ext_ack *extack); int mlxsw_afa_block_append_fwd(struct mlxsw_afa_block *block, - u8 local_port, bool in_port, + u16 local_port, bool in_port, struct netlink_ext_ack *extack); int mlxsw_afa_block_append_vlan_modify(struct mlxsw_afa_block *block, u16 vid, u8 pcp, u8 et, @@ -98,7 +98,7 @@ int mlxsw_afa_block_append_police(struct mlxsw_afa_block *block, u32 fa_index, u64 rate_bytes_ps, u32 burst, u16 *p_policer_index, struct netlink_ext_ack *extack); -int mlxsw_afa_block_append_sampler(struct mlxsw_afa_block *block, u8 local_port, +int mlxsw_afa_block_append_sampler(struct mlxsw_afa_block *block, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, diff --git a/drivers/net/ethernet/mellanox/mlxsw/item.h b/drivers/net/ethernet/mellanox/mlxsw/item.h index ab70a873a01a..cfafbeb42586 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/item.h +++ b/drivers/net/ethernet/mellanox/mlxsw/item.h @@ -367,6 +367,42 @@ mlxsw_##_type##_##_cname##_##_iname##_set(char *buf, u32 val) \ __mlxsw_item_set32(buf, &__ITEM_NAME(_type, _cname, _iname), 0, val); \ } +#define LOCAL_PORT_LSB_SIZE 8 +#define LOCAL_PORT_MSB_SIZE 2 + +#define MLXSW_ITEM32_LP(_type, _cname, _offset1, _shift1, _offset2, _shift2) \ +static struct mlxsw_item __ITEM_NAME(_type, _cname, local_port) = { \ + .offset = _offset1, \ + .shift = _shift1, \ + .size = {.bits = LOCAL_PORT_LSB_SIZE,}, \ + .name = #_type "_" #_cname "_local_port", \ +}; \ +static struct mlxsw_item __ITEM_NAME(_type, _cname, lp_msb) = { \ + .offset = _offset2, \ + .shift = _shift2, \ + .size = {.bits = LOCAL_PORT_MSB_SIZE,}, \ + .name = #_type "_" #_cname "_lp_msb", \ +}; \ +static inline u32 __maybe_unused \ +mlxsw_##_type##_##_cname##_local_port_get(const char *buf) \ +{ \ + u32 local_port, lp_msb; \ + \ + local_port = __mlxsw_item_get32(buf, &__ITEM_NAME(_type, _cname, \ + local_port), 0); \ + lp_msb = __mlxsw_item_get32(buf, &__ITEM_NAME(_type, _cname, lp_msb), \ + 0); \ + return (lp_msb << LOCAL_PORT_LSB_SIZE) + local_port; \ +} \ +static inline void __maybe_unused \ +mlxsw_##_type##_##_cname##_local_port_set(char *buf, u32 val) \ +{ \ + __mlxsw_item_set32(buf, &__ITEM_NAME(_type, _cname, local_port), 0, \ + val & ((1 << LOCAL_PORT_LSB_SIZE) - 1)); \ + __mlxsw_item_set32(buf, &__ITEM_NAME(_type, _cname, lp_msb), 0, \ + val >> LOCAL_PORT_LSB_SIZE); \ +} + #define MLXSW_ITEM32_INDEXED(_type, _cname, _iname, _offset, _shift, _sizebits, \ _step, _instepoffset, _norealshift) \ static struct mlxsw_item __ITEM_NAME(_type, _cname, _iname) = { \ diff --git a/drivers/net/ethernet/mellanox/mlxsw/minimal.c b/drivers/net/ethernet/mellanox/mlxsw/minimal.c index 5d4dfa5ddbb5..10d13f5f9c7d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/minimal.c +++ b/drivers/net/ethernet/mellanox/mlxsw/minimal.c @@ -38,7 +38,7 @@ struct mlxsw_m { struct mlxsw_m_port { struct net_device *dev; struct mlxsw_m *mlxsw_m; - u8 local_port; + u16 local_port; u8 module; }; @@ -180,7 +180,7 @@ static const struct ethtool_ops mlxsw_m_port_ethtool_ops = { }; static int -mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u8 local_port, +mlxsw_m_port_module_info_get(struct mlxsw_m *mlxsw_m, u16 local_port, u8 *p_module, u8 *p_width) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; @@ -214,7 +214,7 @@ mlxsw_m_port_dev_addr_get(struct mlxsw_m_port *mlxsw_m_port) } static int -mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u8 local_port, u8 module) +mlxsw_m_port_create(struct mlxsw_m *mlxsw_m, u16 local_port, u8 module) { struct mlxsw_m_port *mlxsw_m_port; struct net_device *dev; @@ -277,7 +277,7 @@ err_alloc_etherdev: return err; } -static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u8 local_port) +static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u16 local_port) { struct mlxsw_m_port *mlxsw_m_port = mlxsw_m->ports[local_port]; @@ -288,7 +288,7 @@ static void mlxsw_m_port_remove(struct mlxsw_m *mlxsw_m, u8 local_port) mlxsw_core_port_fini(mlxsw_m->core, local_port); } -static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u8 local_port, +static int mlxsw_m_port_module_map(struct mlxsw_m *mlxsw_m, u16 local_port, u8 *last_module) { unsigned int max_ports = mlxsw_core_max_ports(mlxsw_m->core); diff --git a/drivers/net/ethernet/mellanox/mlxsw/pci.c b/drivers/net/ethernet/mellanox/mlxsw/pci.c index a15c95a10bae..cd3331a077bb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/pci.c +++ b/drivers/net/ethernet/mellanox/mlxsw/pci.c @@ -1973,6 +1973,7 @@ int mlxsw_pci_driver_register(struct pci_driver *pci_driver) { pci_driver->probe = mlxsw_pci_probe; pci_driver->remove = mlxsw_pci_remove; + pci_driver->shutdown = mlxsw_pci_remove; return pci_register_driver(pci_driver); } EXPORT_SYMBOL(mlxsw_pci_driver_register); diff --git a/drivers/net/ethernet/mellanox/mlxsw/reg.h b/drivers/net/ethernet/mellanox/mlxsw/reg.h index 8d420eb8ade2..5eaba2abf212 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/reg.h +++ b/drivers/net/ethernet/mellanox/mlxsw/reg.h @@ -69,52 +69,6 @@ MLXSW_REG_DEFINE(spad, MLXSW_REG_SPAD_ID, MLXSW_REG_SPAD_LEN); */ MLXSW_ITEM_BUF(reg, spad, base_mac, 0x02, 6); -/* SMID - Switch Multicast ID - * -------------------------- - * The MID record maps from a MID (Multicast ID), which is a unique identifier - * of the multicast group within the stacking domain, into a list of local - * ports into which the packet is replicated. - */ -#define MLXSW_REG_SMID_ID 0x2007 -#define MLXSW_REG_SMID_LEN 0x240 - -MLXSW_REG_DEFINE(smid, MLXSW_REG_SMID_ID, MLXSW_REG_SMID_LEN); - -/* reg_smid_swid - * Switch partition ID. - * Access: Index - */ -MLXSW_ITEM32(reg, smid, swid, 0x00, 24, 8); - -/* reg_smid_mid - * Multicast identifier - global identifier that represents the multicast group - * across all devices. - * Access: Index - */ -MLXSW_ITEM32(reg, smid, mid, 0x00, 0, 16); - -/* reg_smid_port - * Local port memebership (1 bit per port). - * Access: RW - */ -MLXSW_ITEM_BIT_ARRAY(reg, smid, port, 0x20, 0x20, 1); - -/* reg_smid_port_mask - * Local port mask (1 bit per port). - * Access: W - */ -MLXSW_ITEM_BIT_ARRAY(reg, smid, port_mask, 0x220, 0x20, 1); - -static inline void mlxsw_reg_smid_pack(char *payload, u16 mid, - u8 port, bool set) -{ - MLXSW_REG_ZERO(smid, payload); - mlxsw_reg_smid_swid_set(payload, 0); - mlxsw_reg_smid_mid_set(payload, mid); - mlxsw_reg_smid_port_set(payload, port, set); - mlxsw_reg_smid_port_mask_set(payload, port, 1); -} - /* SSPR - Switch System Port Record Register * ----------------------------------------- * Configures the system port to local port mapping. @@ -141,7 +95,7 @@ MLXSW_ITEM32(reg, sspr, m, 0x00, 31, 1); * * Access: RW */ -MLXSW_ITEM32(reg, sspr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, sspr, 0x00, 16, 0x00, 12); /* reg_sspr_sub_port * Virtual port within the physical port. @@ -161,7 +115,7 @@ MLXSW_ITEM32(reg, sspr, sub_port, 0x00, 8, 8); */ MLXSW_ITEM32(reg, sspr, system_port, 0x04, 0, 16); -static inline void mlxsw_reg_sspr_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_sspr_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(sspr, payload); mlxsw_reg_sspr_m_set(payload, 1); @@ -407,7 +361,7 @@ static inline void mlxsw_reg_sfd_uc_pack(char *payload, int rec_index, enum mlxsw_reg_sfd_rec_policy policy, const char *mac, u16 fid_vid, enum mlxsw_reg_sfd_rec_action action, - u8 local_port) + u16 local_port) { mlxsw_reg_sfd_rec_pack(payload, rec_index, MLXSW_REG_SFD_REC_TYPE_UNICAST, mac, action); @@ -417,15 +371,6 @@ static inline void mlxsw_reg_sfd_uc_pack(char *payload, int rec_index, mlxsw_reg_sfd_uc_system_port_set(payload, rec_index, local_port); } -static inline void mlxsw_reg_sfd_uc_unpack(char *payload, int rec_index, - char *mac, u16 *p_fid_vid, - u8 *p_local_port) -{ - mlxsw_reg_sfd_rec_mac_memcpy_from(payload, rec_index, mac); - *p_fid_vid = mlxsw_reg_sfd_uc_fid_vid_get(payload, rec_index); - *p_local_port = mlxsw_reg_sfd_uc_system_port_get(payload, rec_index); -} - /* reg_sfd_uc_lag_sub_port * LAG sub port. * Must be 0 if multichannel VEPA is not enabled. @@ -478,15 +423,6 @@ mlxsw_reg_sfd_uc_lag_pack(char *payload, int rec_index, mlxsw_reg_sfd_uc_lag_lag_id_set(payload, rec_index, lag_id); } -static inline void mlxsw_reg_sfd_uc_lag_unpack(char *payload, int rec_index, - char *mac, u16 *p_vid, - u16 *p_lag_id) -{ - mlxsw_reg_sfd_rec_mac_memcpy_from(payload, rec_index, mac); - *p_vid = mlxsw_reg_sfd_uc_lag_fid_vid_get(payload, rec_index); - *p_lag_id = mlxsw_reg_sfd_uc_lag_lag_id_get(payload, rec_index); -} - /* reg_sfd_mc_pgi * * Multicast port group index - index into the port group table. @@ -692,7 +628,7 @@ MLXSW_ITEM32_INDEXED(reg, sfn, mac_system_port, MLXSW_REG_SFN_BASE_LEN, 0, 16, static inline void mlxsw_reg_sfn_mac_unpack(char *payload, int rec_index, char *mac, u16 *p_vid, - u8 *p_local_port) + u16 *p_local_port) { mlxsw_reg_sfn_rec_mac_memcpy_from(payload, rec_index, mac); *p_vid = mlxsw_reg_sfn_mac_fid_get(payload, rec_index); @@ -781,7 +717,7 @@ MLXSW_REG_DEFINE(spms, MLXSW_REG_SPMS_ID, MLXSW_REG_SPMS_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, spms, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spms, 0x00, 16, 0x00, 12); enum mlxsw_reg_spms_state { MLXSW_REG_SPMS_STATE_NO_CHANGE, @@ -800,7 +736,7 @@ enum mlxsw_reg_spms_state { */ MLXSW_ITEM_BIT_ARRAY(reg, spms, state, 0x04, 0x400, 2); -static inline void mlxsw_reg_spms_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_spms_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(spms, payload); mlxsw_reg_spms_local_port_set(payload, local_port); @@ -833,7 +769,7 @@ MLXSW_ITEM32(reg, spvid, tport, 0x00, 24, 1); * When tport = 1: Tunnel port. * Access: Index */ -MLXSW_ITEM32(reg, spvid, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spvid, 0x00, 16, 0x00, 12); /* reg_spvid_sub_port * Virtual port within the physical port. @@ -868,7 +804,7 @@ MLXSW_ITEM32(reg, spvid, et_vlan, 0x04, 16, 2); */ MLXSW_ITEM32(reg, spvid, pvid, 0x04, 0, 12); -static inline void mlxsw_reg_spvid_pack(char *payload, u8 local_port, u16 pvid, +static inline void mlxsw_reg_spvid_pack(char *payload, u16 local_port, u16 pvid, u8 et_vlan) { MLXSW_REG_ZERO(spvid, payload); @@ -911,7 +847,7 @@ MLXSW_ITEM32(reg, spvm, pte, 0x00, 30, 1); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, spvm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spvm, 0x00, 16, 0x00, 12); /* reg_spvm_sub_port * Virtual port within the physical port. @@ -959,7 +895,7 @@ MLXSW_ITEM32_INDEXED(reg, spvm, rec_vid, MLXSW_REG_SPVM_BASE_LEN, 0, 12, MLXSW_REG_SPVM_REC_LEN, 0, false); -static inline void mlxsw_reg_spvm_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_spvm_pack(char *payload, u16 local_port, u16 vid_begin, u16 vid_end, bool is_member, bool untagged) { @@ -994,7 +930,7 @@ MLXSW_REG_DEFINE(spaft, MLXSW_REG_SPAFT_ID, MLXSW_REG_SPAFT_LEN); * * Note: CPU port is not supported (all tag types are allowed). */ -MLXSW_ITEM32(reg, spaft, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spaft, 0x00, 16, 0x00, 12); /* reg_spaft_sub_port * Virtual port within the physical port. @@ -1021,7 +957,7 @@ MLXSW_ITEM32(reg, spaft, allow_prio_tagged, 0x04, 30, 1); */ MLXSW_ITEM32(reg, spaft, allow_tagged, 0x04, 29, 1); -static inline void mlxsw_reg_spaft_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_spaft_pack(char *payload, u16 local_port, bool allow_untagged) { MLXSW_REG_ZERO(spaft, payload); @@ -1126,76 +1062,6 @@ mlxsw_reg_sfgc_pack(char *payload, enum mlxsw_reg_sfgc_type type, mlxsw_reg_sfgc_mid_set(payload, MLXSW_PORT_MID); } -/* SFTR - Switch Flooding Table Register - * ------------------------------------- - * The switch flooding table is used for flooding packet replication. The table - * defines a bit mask of ports for packet replication. - */ -#define MLXSW_REG_SFTR_ID 0x2012 -#define MLXSW_REG_SFTR_LEN 0x420 - -MLXSW_REG_DEFINE(sftr, MLXSW_REG_SFTR_ID, MLXSW_REG_SFTR_LEN); - -/* reg_sftr_swid - * Switch partition ID with which to associate the port. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr, swid, 0x00, 24, 8); - -/* reg_sftr_flood_table - * Flooding table index to associate with the specific type on the specific - * switch partition. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr, flood_table, 0x00, 16, 6); - -/* reg_sftr_index - * Index. Used as an index into the Flooding Table in case the table is - * configured to use VID / FID or FID Offset. - * Access: Index - */ -MLXSW_ITEM32(reg, sftr, index, 0x00, 0, 16); - -/* reg_sftr_table_type - * See mlxsw_flood_table_type - * Access: RW - */ -MLXSW_ITEM32(reg, sftr, table_type, 0x04, 16, 3); - -/* reg_sftr_range - * Range of entries to update - * Access: Index - */ -MLXSW_ITEM32(reg, sftr, range, 0x04, 0, 16); - -/* reg_sftr_port - * Local port membership (1 bit per port). - * Access: RW - */ -MLXSW_ITEM_BIT_ARRAY(reg, sftr, port, 0x20, 0x20, 1); - -/* reg_sftr_cpu_port_mask - * CPU port mask (1 bit per port). - * Access: W - */ -MLXSW_ITEM_BIT_ARRAY(reg, sftr, port_mask, 0x220, 0x20, 1); - -static inline void mlxsw_reg_sftr_pack(char *payload, - unsigned int flood_table, - unsigned int index, - enum mlxsw_flood_table_type table_type, - unsigned int range, u8 port, bool set) -{ - MLXSW_REG_ZERO(sftr, payload); - mlxsw_reg_sftr_swid_set(payload, 0); - mlxsw_reg_sftr_flood_table_set(payload, flood_table); - mlxsw_reg_sftr_index_set(payload, index); - mlxsw_reg_sftr_table_type_set(payload, table_type); - mlxsw_reg_sftr_range_set(payload, range); - mlxsw_reg_sftr_port_set(payload, port, set); - mlxsw_reg_sftr_port_mask_set(payload, port, 1); -} - /* SFDF - Switch Filtering DB Flush * -------------------------------- * The switch filtering DB flush register is used to flush the FDB. @@ -1347,7 +1213,7 @@ MLXSW_ITEM32(reg, sldr, num_ports, 0x04, 24, 8); MLXSW_ITEM32_INDEXED(reg, sldr, system_port, 0x08, 0, 16, 4, 0, false); static inline void mlxsw_reg_sldr_lag_add_port_pack(char *payload, u8 lag_id, - u8 local_port) + u16 local_port) { MLXSW_REG_ZERO(sldr, payload); mlxsw_reg_sldr_op_set(payload, MLXSW_REG_SLDR_OP_LAG_ADD_PORT_LIST); @@ -1357,7 +1223,7 @@ static inline void mlxsw_reg_sldr_lag_add_port_pack(char *payload, u8 lag_id, } static inline void mlxsw_reg_sldr_lag_remove_port_pack(char *payload, u8 lag_id, - u8 local_port) + u16 local_port) { MLXSW_REG_ZERO(sldr, payload); mlxsw_reg_sldr_op_set(payload, MLXSW_REG_SLDR_OP_LAG_REMOVE_PORT_LIST); @@ -1397,7 +1263,7 @@ MLXSW_ITEM32(reg, slcr, pp, 0x00, 24, 1); * Reserved when pp = Global Configuration * Access: Index */ -MLXSW_ITEM32(reg, slcr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, slcr, 0x00, 16, 0x00, 12); enum mlxsw_reg_slcr_type { MLXSW_REG_SLCR_TYPE_CRC, /* default */ @@ -1515,7 +1381,7 @@ MLXSW_ITEM32(reg, slcor, col, 0x00, 30, 2); * Not supported for CPU port * Access: Index */ -MLXSW_ITEM32(reg, slcor, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, slcor, 0x00, 16, 0x00, 12); /* reg_slcor_lag_id * LAG Identifier. Index into the LAG descriptor table. @@ -1531,7 +1397,7 @@ MLXSW_ITEM32(reg, slcor, lag_id, 0x00, 0, 10); MLXSW_ITEM32(reg, slcor, port_index, 0x04, 0, 10); static inline void mlxsw_reg_slcor_pack(char *payload, - u8 local_port, u16 lag_id, + u16 local_port, u16 lag_id, enum mlxsw_reg_slcor_col col) { MLXSW_REG_ZERO(slcor, payload); @@ -1541,7 +1407,7 @@ static inline void mlxsw_reg_slcor_pack(char *payload, } static inline void mlxsw_reg_slcor_port_add_pack(char *payload, - u8 local_port, u16 lag_id, + u16 local_port, u16 lag_id, u8 port_index) { mlxsw_reg_slcor_pack(payload, local_port, lag_id, @@ -1550,21 +1416,21 @@ static inline void mlxsw_reg_slcor_port_add_pack(char *payload, } static inline void mlxsw_reg_slcor_port_remove_pack(char *payload, - u8 local_port, u16 lag_id) + u16 local_port, u16 lag_id) { mlxsw_reg_slcor_pack(payload, local_port, lag_id, MLXSW_REG_SLCOR_COL_LAG_REMOVE_PORT); } static inline void mlxsw_reg_slcor_col_enable_pack(char *payload, - u8 local_port, u16 lag_id) + u16 local_port, u16 lag_id) { mlxsw_reg_slcor_pack(payload, local_port, lag_id, MLXSW_REG_SLCOR_COL_LAG_COLLECTOR_ENABLED); } static inline void mlxsw_reg_slcor_col_disable_pack(char *payload, - u8 local_port, u16 lag_id) + u16 local_port, u16 lag_id) { mlxsw_reg_slcor_pack(payload, local_port, lag_id, MLXSW_REG_SLCOR_COL_LAG_COLLECTOR_ENABLED); @@ -1583,7 +1449,7 @@ MLXSW_REG_DEFINE(spmlr, MLXSW_REG_SPMLR_ID, MLXSW_REG_SPMLR_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, spmlr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spmlr, 0x00, 16, 0x00, 12); /* reg_spmlr_sub_port * Virtual port within the physical port. @@ -1611,7 +1477,7 @@ enum mlxsw_reg_spmlr_learn_mode { */ MLXSW_ITEM32(reg, spmlr, learn_mode, 0x04, 30, 2); -static inline void mlxsw_reg_spmlr_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_spmlr_pack(char *payload, u16 local_port, enum mlxsw_reg_spmlr_learn_mode mode) { MLXSW_REG_ZERO(spmlr, payload); @@ -1642,7 +1508,7 @@ MLXSW_ITEM32(reg, svfa, swid, 0x00, 24, 8); * * Note: Reserved for 802.1Q FIDs. */ -MLXSW_ITEM32(reg, svfa, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, svfa, 0x00, 16, 0x00, 12); enum mlxsw_reg_svfa_mt { MLXSW_REG_SVFA_MT_VID_TO_FID, @@ -1696,7 +1562,7 @@ MLXSW_ITEM32(reg, svfa, counter_set_type, 0x08, 24, 8); */ MLXSW_ITEM32(reg, svfa, counter_index, 0x08, 0, 24); -static inline void mlxsw_reg_svfa_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_svfa_pack(char *payload, u16 local_port, enum mlxsw_reg_svfa_mt mt, bool valid, u16 fid, u16 vid) { @@ -1733,7 +1599,7 @@ MLXSW_ITEM32(reg, spvtr, tport, 0x00, 24, 1); * When tport = 1: tunnel port. * Access: Index */ -MLXSW_ITEM32(reg, spvtr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spvtr, 0x00, 16, 0x00, 12); /* reg_spvtr_ippe * Ingress Port Prio Mode Update Enable. @@ -1803,7 +1669,7 @@ enum mlxsw_reg_spvtr_epvid_mode { MLXSW_ITEM32(reg, spvtr, epvid_mode, 0x04, 0, 4); static inline void mlxsw_reg_spvtr_pack(char *payload, bool tport, - u8 local_port, + u16 local_port, enum mlxsw_reg_spvtr_ipvid_mode ipvid_mode) { MLXSW_REG_ZERO(spvtr, payload); @@ -1828,7 +1694,7 @@ MLXSW_REG_DEFINE(svpe, MLXSW_REG_SVPE_ID, MLXSW_REG_SVPE_LEN); * * Note: CPU port is not supported (uses VLAN mode only). */ -MLXSW_ITEM32(reg, svpe, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, svpe, 0x00, 16, 0x00, 12); /* reg_svpe_vp_en * Virtual port enable. @@ -1838,7 +1704,7 @@ MLXSW_ITEM32(reg, svpe, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, svpe, vp_en, 0x00, 8, 1); -static inline void mlxsw_reg_svpe_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_svpe_pack(char *payload, u16 local_port, bool enable) { MLXSW_REG_ZERO(svpe, payload); @@ -1948,7 +1814,7 @@ MLXSW_REG_DEFINE(spvmlr, MLXSW_REG_SPVMLR_ID, MLXSW_REG_SPVMLR_LEN); * * Note: CPU port is not supported. */ -MLXSW_ITEM32(reg, spvmlr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spvmlr, 0x00, 16, 0x00, 12); /* reg_spvmlr_num_rec * Number of records to update. @@ -1971,7 +1837,7 @@ MLXSW_ITEM32_INDEXED(reg, spvmlr, rec_learn_enable, MLXSW_REG_SPVMLR_BASE_LEN, MLXSW_ITEM32_INDEXED(reg, spvmlr, rec_vid, MLXSW_REG_SPVMLR_BASE_LEN, 0, 12, MLXSW_REG_SPVMLR_REC_LEN, 0x00, false); -static inline void mlxsw_reg_spvmlr_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_spvmlr_pack(char *payload, u16 local_port, u16 vid_begin, u16 vid_end, bool learn_enable) { @@ -2009,7 +1875,7 @@ MLXSW_REG_DEFINE(spvc, MLXSW_REG_SPVC_ID, MLXSW_REG_SPVC_LEN); * through Rx port i and a Tx port j then port i and port j must have the * same configuration. */ -MLXSW_ITEM32(reg, spvc, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spvc, 0x00, 16, 0x00, 12); /* reg_spvc_inner_et2 * Vlan Tag1 EtherType2 enable. @@ -2074,7 +1940,7 @@ MLXSW_ITEM32(reg, spvc, inner_et0, 0x08, 1, 1); */ MLXSW_ITEM32(reg, spvc, et0, 0x08, 0, 1); -static inline void mlxsw_reg_spvc_pack(char *payload, u8 local_port, bool et1, +static inline void mlxsw_reg_spvc_pack(char *payload, u16 local_port, bool et1, bool et0) { MLXSW_REG_ZERO(spvc, payload); @@ -2104,7 +1970,7 @@ MLXSW_REG_DEFINE(spevet, MLXSW_REG_SPEVET_ID, MLXSW_REG_SPEVET_LEN); * Not supported to CPU port. * Access: Index */ -MLXSW_ITEM32(reg, spevet, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, spevet, 0x00, 16, 0x00, 12); /* reg_spevet_et_vlan * Egress EtherType VLAN to push when SPVID.egr_et_set field set for the packet: @@ -2115,7 +1981,7 @@ MLXSW_ITEM32(reg, spevet, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, spevet, et_vlan, 0x04, 16, 2); -static inline void mlxsw_reg_spevet_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_spevet_pack(char *payload, u16 local_port, u8 et_vlan) { MLXSW_REG_ZERO(spevet, payload); @@ -2123,6 +1989,122 @@ static inline void mlxsw_reg_spevet_pack(char *payload, u8 local_port, mlxsw_reg_spevet_et_vlan_set(payload, et_vlan); } +/* SFTR-V2 - Switch Flooding Table Version 2 Register + * -------------------------------------------------- + * The switch flooding table is used for flooding packet replication. The table + * defines a bit mask of ports for packet replication. + */ +#define MLXSW_REG_SFTR2_ID 0x202F +#define MLXSW_REG_SFTR2_LEN 0x120 + +MLXSW_REG_DEFINE(sftr2, MLXSW_REG_SFTR2_ID, MLXSW_REG_SFTR2_LEN); + +/* reg_sftr2_swid + * Switch partition ID with which to associate the port. + * Access: Index + */ +MLXSW_ITEM32(reg, sftr2, swid, 0x00, 24, 8); + +/* reg_sftr2_flood_table + * Flooding table index to associate with the specific type on the specific + * switch partition. + * Access: Index + */ +MLXSW_ITEM32(reg, sftr2, flood_table, 0x00, 16, 6); + +/* reg_sftr2_index + * Index. Used as an index into the Flooding Table in case the table is + * configured to use VID / FID or FID Offset. + * Access: Index + */ +MLXSW_ITEM32(reg, sftr2, index, 0x00, 0, 16); + +/* reg_sftr2_table_type + * See mlxsw_flood_table_type + * Access: RW + */ +MLXSW_ITEM32(reg, sftr2, table_type, 0x04, 16, 3); + +/* reg_sftr2_range + * Range of entries to update + * Access: Index + */ +MLXSW_ITEM32(reg, sftr2, range, 0x04, 0, 16); + +/* reg_sftr2_port + * Local port membership (1 bit per port). + * Access: RW + */ +MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port, 0x20, 0x80, 1); + +/* reg_sftr2_port_mask + * Local port mask (1 bit per port). + * Access: WO + */ +MLXSW_ITEM_BIT_ARRAY(reg, sftr2, port_mask, 0xA0, 0x80, 1); + +static inline void mlxsw_reg_sftr2_pack(char *payload, + unsigned int flood_table, + unsigned int index, + enum mlxsw_flood_table_type table_type, + unsigned int range, u16 port, bool set) +{ + MLXSW_REG_ZERO(sftr2, payload); + mlxsw_reg_sftr2_swid_set(payload, 0); + mlxsw_reg_sftr2_flood_table_set(payload, flood_table); + mlxsw_reg_sftr2_index_set(payload, index); + mlxsw_reg_sftr2_table_type_set(payload, table_type); + mlxsw_reg_sftr2_range_set(payload, range); + mlxsw_reg_sftr2_port_set(payload, port, set); + mlxsw_reg_sftr2_port_mask_set(payload, port, 1); +} + +/* SMID-V2 - Switch Multicast ID Version 2 Register + * ------------------------------------------------ + * The MID record maps from a MID (Multicast ID), which is a unique identifier + * of the multicast group within the stacking domain, into a list of local + * ports into which the packet is replicated. + */ +#define MLXSW_REG_SMID2_ID 0x2034 +#define MLXSW_REG_SMID2_LEN 0x120 + +MLXSW_REG_DEFINE(smid2, MLXSW_REG_SMID2_ID, MLXSW_REG_SMID2_LEN); + +/* reg_smid2_swid + * Switch partition ID. + * Access: Index + */ +MLXSW_ITEM32(reg, smid2, swid, 0x00, 24, 8); + +/* reg_smid2_mid + * Multicast identifier - global identifier that represents the multicast group + * across all devices. + * Access: Index + */ +MLXSW_ITEM32(reg, smid2, mid, 0x00, 0, 16); + +/* reg_smid2_port + * Local port memebership (1 bit per port). + * Access: RW + */ +MLXSW_ITEM_BIT_ARRAY(reg, smid2, port, 0x20, 0x80, 1); + +/* reg_smid2_port_mask + * Local port mask (1 bit per port). + * Access: WO + */ +MLXSW_ITEM_BIT_ARRAY(reg, smid2, port_mask, 0xA0, 0x80, 1); + +static inline void mlxsw_reg_smid2_pack(char *payload, u16 mid, u16 port, + bool set) +{ + MLXSW_REG_ZERO(smid2, payload); + mlxsw_reg_smid2_swid_set(payload, 0); + mlxsw_reg_smid2_mid_set(payload, mid); + mlxsw_reg_smid2_port_set(payload, port, set); + mlxsw_reg_smid2_port_mask_set(payload, port, 1); +} + /* CWTP - Congetion WRED ECN TClass Profile * ---------------------------------------- * Configures the profiles for queues of egress port and traffic class @@ -2139,7 +2121,7 @@ MLXSW_REG_DEFINE(cwtp, MLXSW_REG_CWTP_ID, MLXSW_REG_CWTP_LEN); * Not supported for CPU port * Access: Index */ -MLXSW_ITEM32(reg, cwtp, local_port, 0, 16, 8); +MLXSW_ITEM32_LP(reg, cwtp, 0x00, 16, 0x00, 12); /* reg_cwtp_traffic_class * Traffic Class to configure @@ -2173,7 +2155,7 @@ MLXSW_ITEM32_INDEXED(reg, cwtp, profile_max, MLXSW_REG_CWTP_BASE_LEN, #define MLXSW_REG_CWTP_MAX_PROFILE 2 #define MLXSW_REG_CWTP_DEFAULT_PROFILE 1 -static inline void mlxsw_reg_cwtp_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_cwtp_pack(char *payload, u16 local_port, u8 traffic_class) { int i; @@ -2217,7 +2199,7 @@ MLXSW_REG_DEFINE(cwtpm, MLXSW_REG_CWTPM_ID, MLXSW_REG_CWTPM_LEN); * Not supported for CPU port * Access: Index */ -MLXSW_ITEM32(reg, cwtpm, local_port, 0, 16, 8); +MLXSW_ITEM32_LP(reg, cwtpm, 0x00, 16, 0x00, 12); /* reg_cwtpm_traffic_class * Traffic Class to configure @@ -2291,7 +2273,7 @@ MLXSW_ITEM32(reg, cwtpm, ntcp_r, 64, 0, 2); #define MLXSW_REG_CWTPM_RESET_PROFILE 0 -static inline void mlxsw_reg_cwtpm_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_cwtpm_pack(char *payload, u16 local_port, u8 traffic_class, u8 profile, bool wred, bool ecn) { @@ -2363,7 +2345,7 @@ MLXSW_ITEM32(reg, ppbt, op, 0x00, 28, 3); * Local port. Not including CPU port. * Access: Index */ -MLXSW_ITEM32(reg, ppbt, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, ppbt, 0x00, 16, 0x00, 12); /* reg_ppbt_g * group - When set, the binding is of an ACL group. When cleared, @@ -2382,7 +2364,7 @@ MLXSW_ITEM32(reg, ppbt, acl_info, 0x10, 0, 16); static inline void mlxsw_reg_ppbt_pack(char *payload, enum mlxsw_reg_pxbt_e e, enum mlxsw_reg_pxbt_op op, - u8 local_port, u16 acl_info) + u16 local_port, u16 acl_info) { MLXSW_REG_ZERO(ppbt, payload); mlxsw_reg_ppbt_e_set(payload, e); @@ -3513,7 +3495,7 @@ MLXSW_REG_DEFINE(qpts, MLXSW_REG_QPTS_ID, MLXSW_REG_QPTS_LEN); * * Note: CPU port is supported. */ -MLXSW_ITEM32(reg, qpts, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qpts, 0x00, 16, 0x00, 12); enum mlxsw_reg_qpts_trust_state { MLXSW_REG_QPTS_TRUST_STATE_PCP = 1, @@ -3526,7 +3508,7 @@ enum mlxsw_reg_qpts_trust_state { */ MLXSW_ITEM32(reg, qpts, trust_state, 0x04, 0, 3); -static inline void mlxsw_reg_qpts_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qpts_pack(char *payload, u16 local_port, enum mlxsw_reg_qpts_trust_state ts) { MLXSW_REG_ZERO(qpts, payload); @@ -3717,7 +3699,7 @@ MLXSW_REG_DEFINE(qtct, MLXSW_REG_QTCT_ID, MLXSW_REG_QTCT_LEN); * * Note: CPU port is not supported. */ -MLXSW_ITEM32(reg, qtct, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qtct, 0x00, 16, 0x00, 12); /* reg_qtct_sub_port * Virtual port within the physical port. @@ -3742,7 +3724,7 @@ MLXSW_ITEM32(reg, qtct, switch_prio, 0x00, 0, 4); */ MLXSW_ITEM32(reg, qtct, tclass, 0x04, 0, 4); -static inline void mlxsw_reg_qtct_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qtct_pack(char *payload, u16 local_port, u8 switch_prio, u8 tclass) { MLXSW_REG_ZERO(qtct, payload); @@ -3766,7 +3748,7 @@ MLXSW_REG_DEFINE(qeec, MLXSW_REG_QEEC_ID, MLXSW_REG_QEEC_LEN); * * Note: CPU port is supported. */ -MLXSW_ITEM32(reg, qeec, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qeec, 0x00, 16, 0x00, 12); enum mlxsw_reg_qeec_hr { MLXSW_REG_QEEC_HR_PORT, @@ -3909,7 +3891,7 @@ MLXSW_ITEM32(reg, qeec, max_shaper_bs, 0x1C, 0, 6); #define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP2 11 #define MLXSW_REG_QEEC_LOWEST_SHAPER_BS_SP3 11 -static inline void mlxsw_reg_qeec_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qeec_pack(char *payload, u16 local_port, enum mlxsw_reg_qeec_hr hr, u8 index, u8 next_index) { @@ -3920,7 +3902,7 @@ static inline void mlxsw_reg_qeec_pack(char *payload, u8 local_port, mlxsw_reg_qeec_next_element_index_set(payload, next_index); } -static inline void mlxsw_reg_qeec_ptps_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qeec_ptps_pack(char *payload, u16 local_port, bool ptps) { MLXSW_REG_ZERO(qeec, payload); @@ -3944,7 +3926,7 @@ MLXSW_REG_DEFINE(qrwe, MLXSW_REG_QRWE_ID, MLXSW_REG_QRWE_LEN); * * Note: CPU port is supported. No support for router port. */ -MLXSW_ITEM32(reg, qrwe, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qrwe, 0x00, 16, 0x00, 12); /* reg_qrwe_dscp * Whether to enable DSCP rewrite (default is 0, don't rewrite). @@ -3958,7 +3940,7 @@ MLXSW_ITEM32(reg, qrwe, dscp, 0x04, 1, 1); */ MLXSW_ITEM32(reg, qrwe, pcp, 0x04, 0, 1); -static inline void mlxsw_reg_qrwe_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qrwe_pack(char *payload, u16 local_port, bool rewrite_pcp, bool rewrite_dscp) { MLXSW_REG_ZERO(qrwe, payload); @@ -3985,7 +3967,7 @@ MLXSW_REG_DEFINE(qpdsm, MLXSW_REG_QPDSM_ID, MLXSW_REG_QPDSM_LEN); * Local Port. Supported for data packets from CPU port. * Access: Index */ -MLXSW_ITEM32(reg, qpdsm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qpdsm, 0x00, 16, 0x00, 12); /* reg_qpdsm_prio_entry_color0_e * Enable update of the entry for color 0 and a given port. @@ -4038,7 +4020,7 @@ MLXSW_ITEM32_INDEXED(reg, qpdsm, prio_entry_color2_dscp, MLXSW_REG_QPDSM_BASE_LEN, 8, 6, MLXSW_REG_QPDSM_PRIO_ENTRY_REC_LEN, 0x00, false); -static inline void mlxsw_reg_qpdsm_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_qpdsm_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(qpdsm, payload); mlxsw_reg_qpdsm_local_port_set(payload, local_port); @@ -4071,7 +4053,7 @@ MLXSW_REG_DEFINE(qpdp, MLXSW_REG_QPDP_ID, MLXSW_REG_QPDP_LEN); * Local Port. Supported for data packets from CPU port. * Access: Index */ -MLXSW_ITEM32(reg, qpdp, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qpdp, 0x00, 16, 0x00, 12); /* reg_qpdp_switch_prio * Default port Switch Priority (default 0) @@ -4079,7 +4061,7 @@ MLXSW_ITEM32(reg, qpdp, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, qpdp, switch_prio, 0x04, 0, 4); -static inline void mlxsw_reg_qpdp_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_qpdp_pack(char *payload, u16 local_port, u8 switch_prio) { MLXSW_REG_ZERO(qpdp, payload); @@ -4106,7 +4088,7 @@ MLXSW_REG_DEFINE(qpdpm, MLXSW_REG_QPDPM_ID, MLXSW_REG_QPDPM_LEN); * Local Port. Supported for data packets from CPU port. * Access: Index */ -MLXSW_ITEM32(reg, qpdpm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qpdpm, 0x00, 16, 0x00, 12); /* reg_qpdpm_dscp_e * Enable update of the specific entry. When cleared, the switch_prio and color @@ -4125,7 +4107,7 @@ MLXSW_ITEM16_INDEXED(reg, qpdpm, dscp_entry_prio, MLXSW_REG_QPDPM_BASE_LEN, 0, 4, MLXSW_REG_QPDPM_DSCP_ENTRY_REC_LEN, 0x00, false); -static inline void mlxsw_reg_qpdpm_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_qpdpm_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(qpdpm, payload); mlxsw_reg_qpdpm_local_port_set(payload, local_port); @@ -4157,7 +4139,7 @@ MLXSW_REG_DEFINE(qtctm, MLXSW_REG_QTCTM_ID, MLXSW_REG_QTCTM_LEN); * No support for CPU port. * Access: Index */ -MLXSW_ITEM32(reg, qtctm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, qtctm, 0x00, 16, 0x00, 12); /* reg_qtctm_mc * Multicast Mode @@ -4167,7 +4149,7 @@ MLXSW_ITEM32(reg, qtctm, local_port, 0x00, 16, 8); MLXSW_ITEM32(reg, qtctm, mc, 0x04, 0, 1); static inline void -mlxsw_reg_qtctm_pack(char *payload, u8 local_port, bool mc) +mlxsw_reg_qtctm_pack(char *payload, u16 local_port, bool mc) { MLXSW_REG_ZERO(qtctm, payload); mlxsw_reg_qtctm_local_port_set(payload, local_port); @@ -4300,7 +4282,7 @@ MLXSW_ITEM32(reg, pmlp, rxtx, 0x00, 31, 1); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pmlp, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pmlp, 0x00, 16, 0x00, 12); /* reg_pmlp_width * 0 - Unmap local port. @@ -4331,7 +4313,7 @@ MLXSW_ITEM32_INDEXED(reg, pmlp, tx_lane, 0x04, 16, 4, 0x04, 0x00, false); */ MLXSW_ITEM32_INDEXED(reg, pmlp, rx_lane, 0x04, 24, 4, 0x04, 0x00, false); -static inline void mlxsw_reg_pmlp_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_pmlp_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(pmlp, payload); mlxsw_reg_pmlp_local_port_set(payload, local_port); @@ -4350,7 +4332,7 @@ MLXSW_REG_DEFINE(pmtu, MLXSW_REG_PMTU_ID, MLXSW_REG_PMTU_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pmtu, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pmtu, 0x00, 16, 0x00, 12); /* reg_pmtu_max_mtu * Maximum MTU. @@ -4378,7 +4360,7 @@ MLXSW_ITEM32(reg, pmtu, admin_mtu, 0x08, 16, 16); */ MLXSW_ITEM32(reg, pmtu, oper_mtu, 0x0C, 16, 16); -static inline void mlxsw_reg_pmtu_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_pmtu_pack(char *payload, u16 local_port, u16 new_mtu) { MLXSW_REG_ZERO(pmtu, payload); @@ -4412,7 +4394,7 @@ MLXSW_ITEM32(reg, ptys, an_disable_admin, 0x00, 30, 1); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, ptys, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, ptys, 0x00, 16, 0x00, 12); #define MLXSW_REG_PTYS_PROTO_MASK_IB BIT(0) #define MLXSW_REG_PTYS_PROTO_MASK_ETH BIT(2) @@ -4572,7 +4554,7 @@ enum mlxsw_reg_ptys_connector_type { */ MLXSW_ITEM32(reg, ptys, connector_type, 0x2C, 0, 4); -static inline void mlxsw_reg_ptys_eth_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_ptys_eth_pack(char *payload, u16 local_port, u32 proto_admin, bool autoneg) { MLXSW_REG_ZERO(ptys, payload); @@ -4582,7 +4564,7 @@ static inline void mlxsw_reg_ptys_eth_pack(char *payload, u8 local_port, mlxsw_reg_ptys_an_disable_admin_set(payload, !autoneg); } -static inline void mlxsw_reg_ptys_ext_eth_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_ptys_ext_eth_pack(char *payload, u16 local_port, u32 proto_admin, bool autoneg) { MLXSW_REG_ZERO(ptys, payload); @@ -4624,7 +4606,7 @@ static inline void mlxsw_reg_ptys_ext_eth_unpack(char *payload, mlxsw_reg_ptys_ext_eth_proto_oper_get(payload); } -static inline void mlxsw_reg_ptys_ib_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_ptys_ib_pack(char *payload, u16 local_port, u16 proto_admin, u16 link_width) { MLXSW_REG_ZERO(ptys, payload); @@ -4672,7 +4654,7 @@ MLXSW_ITEM32(reg, ppad, single_base_mac, 0x00, 28, 1); * port number, if single_base_mac = 0 then local_port is reserved * Access: RW */ -MLXSW_ITEM32(reg, ppad, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, ppad, 0x00, 16, 0x00, 24); /* reg_ppad_mac * If single_base_mac = 0 - base MAC address, mac[7:0] is reserved. @@ -4682,7 +4664,7 @@ MLXSW_ITEM32(reg, ppad, local_port, 0x00, 16, 8); MLXSW_ITEM_BUF(reg, ppad, mac, 0x02, 6); static inline void mlxsw_reg_ppad_pack(char *payload, bool single_base_mac, - u8 local_port) + u16 local_port) { MLXSW_REG_ZERO(ppad, payload); mlxsw_reg_ppad_single_base_mac_set(payload, !!single_base_mac); @@ -4711,7 +4693,7 @@ MLXSW_ITEM32(reg, paos, swid, 0x00, 24, 8); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, paos, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, paos, 0x00, 16, 0x00, 12); /* reg_paos_admin_status * Port administrative state (the desired state of the port): @@ -4756,7 +4738,7 @@ MLXSW_ITEM32(reg, paos, ee, 0x04, 30, 1); */ MLXSW_ITEM32(reg, paos, e, 0x04, 0, 2); -static inline void mlxsw_reg_paos_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_paos_pack(char *payload, u16 local_port, enum mlxsw_port_admin_status status) { MLXSW_REG_ZERO(paos, payload); @@ -4782,7 +4764,7 @@ MLXSW_REG_DEFINE(pfcc, MLXSW_REG_PFCC_ID, MLXSW_REG_PFCC_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pfcc, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pfcc, 0x00, 16, 0x00, 12); /* reg_pfcc_pnat * Port number access type. Determines the way local_port is interpreted: @@ -4899,7 +4881,7 @@ static inline void mlxsw_reg_pfcc_prio_pack(char *payload, u8 pfc_en) mlxsw_reg_pfcc_pfcrx_set(payload, pfc_en); } -static inline void mlxsw_reg_pfcc_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_pfcc_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(pfcc, payload); mlxsw_reg_pfcc_local_port_set(payload, local_port); @@ -4928,11 +4910,9 @@ MLXSW_ITEM32(reg, ppcnt, swid, 0x00, 24, 8); /* reg_ppcnt_local_port * Local port number. - * 255 indicates all ports on the device, and is only allowed - * for Set() operation. * Access: Index */ -MLXSW_ITEM32(reg, ppcnt, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, ppcnt, 0x00, 16, 0x00, 12); /* reg_ppcnt_pnat * Port number access type: @@ -4981,6 +4961,14 @@ MLXSW_ITEM32(reg, ppcnt, grp, 0x00, 0, 6); */ MLXSW_ITEM32(reg, ppcnt, clr, 0x04, 31, 1); +/* reg_ppcnt_lp_gl + * Local port global variable. + * 0: local_port 255 = all ports of the device. + * 1: local_port indicates local port number for all ports. + * Access: OP + */ +MLXSW_ITEM32(reg, ppcnt, lp_gl, 0x04, 30, 1); + /* reg_ppcnt_prio_tc * Priority for counter set that support per priority, valid values: 0-7. * Traffic class for counter set that support per traffic class, @@ -5404,7 +5392,7 @@ MLXSW_ITEM64(reg, ppcnt, wred_discard, MLXSW_ITEM64(reg, ppcnt, ecn_marked_tc, MLXSW_REG_PPCNT_COUNTERS_OFFSET + 0x08, 0, 64); -static inline void mlxsw_reg_ppcnt_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_ppcnt_pack(char *payload, u16 local_port, enum mlxsw_reg_ppcnt_grp grp, u8 prio_tc) { @@ -5414,6 +5402,7 @@ static inline void mlxsw_reg_ppcnt_pack(char *payload, u8 local_port, mlxsw_reg_ppcnt_pnat_set(payload, 0); mlxsw_reg_ppcnt_grp_set(payload, grp); mlxsw_reg_ppcnt_clr_set(payload, 0); + mlxsw_reg_ppcnt_lp_gl_set(payload, 1); mlxsw_reg_ppcnt_prio_tc_set(payload, prio_tc); } @@ -5430,7 +5419,7 @@ MLXSW_REG_DEFINE(plib, MLXSW_REG_PLIB_ID, MLXSW_REG_PLIB_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, plib, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, plib, 0x00, 16, 0x00, 12); /* reg_plib_ib_port * InfiniBand port remapping for local_port. @@ -5468,7 +5457,7 @@ MLXSW_ITEM32(reg, pptb, mm, 0x00, 28, 2); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pptb, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pptb, 0x00, 16, 0x00, 12); /* reg_pptb_um * Enables the update of the untagged_buf field. @@ -5515,7 +5504,7 @@ MLXSW_ITEM_BIT_ARRAY(reg, pptb, prio_to_buff_msb, 0x0C, 0x04, 4); #define MLXSW_REG_PPTB_ALL_PRIO 0xFF -static inline void mlxsw_reg_pptb_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_pptb_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(pptb, payload); mlxsw_reg_pptb_mm_set(payload, MLXSW_REG_PPTB_MM_UM); @@ -5545,7 +5534,7 @@ MLXSW_REG_DEFINE(pbmc, MLXSW_REG_PBMC_ID, MLXSW_REG_PBMC_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pbmc, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pbmc, 0x00, 16, 0x00, 12); /* reg_pbmc_xoff_timer_value * When device generates a pause frame, it uses this value as the pause @@ -5612,7 +5601,7 @@ MLXSW_ITEM32_INDEXED(reg, pbmc, buf_xoff_threshold, 0x0C, 16, 16, MLXSW_ITEM32_INDEXED(reg, pbmc, buf_xon_threshold, 0x0C, 0, 16, 0x08, 0x04, false); -static inline void mlxsw_reg_pbmc_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_pbmc_pack(char *payload, u16 local_port, u16 xoff_timer_value, u16 xoff_refresh) { MLXSW_REG_ZERO(pbmc, payload); @@ -5661,7 +5650,7 @@ MLXSW_ITEM32(reg, pspa, swid, 0x00, 24, 8); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pspa, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pspa, 0x00, 16, 0x00, 0); /* reg_pspa_sub_port * Virtual port within the local port. Set to 0 when virtual ports are @@ -5670,7 +5659,7 @@ MLXSW_ITEM32(reg, pspa, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, pspa, sub_port, 0x00, 8, 8); -static inline void mlxsw_reg_pspa_pack(char *payload, u8 swid, u8 local_port) +static inline void mlxsw_reg_pspa_pack(char *payload, u8 swid, u16 local_port) { MLXSW_REG_ZERO(pspa, payload); mlxsw_reg_pspa_swid_set(payload, swid); @@ -5772,7 +5761,7 @@ MLXSW_REG_DEFINE(pplr, MLXSW_REG_PPLR_ID, MLXSW_REG_PPLR_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pplr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pplr, 0x00, 16, 0x00, 12); /* Phy local loopback. When set the port's egress traffic is looped back * to the receiver and the port transmitter is disabled. @@ -5785,7 +5774,7 @@ MLXSW_ITEM32(reg, pplr, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, pplr, lb_en, 0x04, 0, 8); -static inline void mlxsw_reg_pplr_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_pplr_pack(char *payload, u16 local_port, bool phy_local) { MLXSW_REG_ZERO(pplr, payload); @@ -5846,7 +5835,7 @@ MLXSW_ITEM32(reg, pmtdb, status, 0x00, 0, 4); * the module. * Access: RO */ -MLXSW_ITEM16_INDEXED(reg, pmtdb, port_num, 0x04, 0, 8, 0x02, 0x00, false); +MLXSW_ITEM16_INDEXED(reg, pmtdb, port_num, 0x04, 0, 10, 0x02, 0x00, false); static inline void mlxsw_reg_pmtdb_pack(char *payload, u8 slot_index, u8 module, u8 ports_width, u8 num_ports) @@ -5915,7 +5904,7 @@ MLXSW_REG_DEFINE(pddr, MLXSW_REG_PDDR_ID, MLXSW_REG_PDDR_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pddr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pddr, 0x00, 16, 0x00, 12); enum mlxsw_reg_pddr_page_select { MLXSW_REG_PDDR_PAGE_SELECT_TROUBLESHOOTING_INFO = 1, @@ -5944,7 +5933,7 @@ MLXSW_ITEM32(reg, pddr, trblsh_group_opcode, 0x08, 0, 16); */ MLXSW_ITEM32(reg, pddr, trblsh_status_opcode, 0x0C, 0, 16); -static inline void mlxsw_reg_pddr_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_pddr_pack(char *payload, u16 local_port, u8 page_select) { MLXSW_REG_ZERO(pddr, payload); @@ -6014,7 +6003,7 @@ MLXSW_REG_DEFINE(pllp, MLXSW_REG_PLLP_ID, MLXSW_REG_PLLP_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pllp, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pllp, 0x00, 16, 0x00, 12); /* reg_pllp_label_port * Front panel label of the port. @@ -6034,7 +6023,7 @@ MLXSW_ITEM32(reg, pllp, split_num, 0x04, 0, 4); */ MLXSW_ITEM32(reg, pllp, slot_index, 0x08, 0, 4); -static inline void mlxsw_reg_pllp_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_pllp_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(pllp, payload); mlxsw_reg_pllp_local_port_set(payload, local_port); @@ -10245,7 +10234,7 @@ MLXSW_REG_DEFINE(mpar, MLXSW_REG_MPAR_ID, MLXSW_REG_MPAR_LEN); * The local port to mirror the packets from. * Access: Index */ -MLXSW_ITEM32(reg, mpar, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, mpar, 0x00, 16, 0x00, 4); enum mlxsw_reg_mpar_i_e { MLXSW_REG_MPAR_TYPE_EGRESS, @@ -10282,7 +10271,7 @@ MLXSW_ITEM32(reg, mpar, pa_id, 0x04, 0, 4); */ MLXSW_ITEM32(reg, mpar, probability_rate, 0x08, 0, 32); -static inline void mlxsw_reg_mpar_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_mpar_pack(char *payload, u16 local_port, enum mlxsw_reg_mpar_i_e i_e, bool enable, u8 pa_id, u32 probability_rate) @@ -10386,7 +10375,7 @@ MLXSW_REG_DEFINE(mlcr, MLXSW_REG_MLCR_ID, MLXSW_REG_MLCR_LEN); * Local port number. * Access: RW */ -MLXSW_ITEM32(reg, mlcr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, mlcr, 0x00, 16, 0x00, 24); #define MLXSW_REG_MLCR_DURATION_MAX 0xFFFF @@ -10405,7 +10394,7 @@ MLXSW_ITEM32(reg, mlcr, beacon_duration, 0x04, 0, 16); */ MLXSW_ITEM32(reg, mlcr, beacon_remain, 0x08, 0, 16); -static inline void mlxsw_reg_mlcr_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_mlcr_pack(char *payload, u16 local_port, bool active) { MLXSW_REG_ZERO(mlcr, payload); @@ -10778,7 +10767,7 @@ MLXSW_REG_DEFINE(mpsc, MLXSW_REG_MPSC_ID, MLXSW_REG_MPSC_LEN); * Not supported for CPU port * Access: Index */ -MLXSW_ITEM32(reg, mpsc, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, mpsc, 0x00, 16, 0x00, 12); /* reg_mpsc_e * Enable sampling on port local_port @@ -10795,7 +10784,7 @@ MLXSW_ITEM32(reg, mpsc, e, 0x04, 30, 1); */ MLXSW_ITEM32(reg, mpsc, rate, 0x08, 0, 32); -static inline void mlxsw_reg_mpsc_pack(char *payload, u8 local_port, bool e, +static inline void mlxsw_reg_mpsc_pack(char *payload, u16 local_port, bool e, u32 rate) { MLXSW_REG_ZERO(mpsc, payload); @@ -11003,7 +10992,7 @@ MLXSW_REG_DEFINE(momte, MLXSW_REG_MOMTE_ID, MLXSW_REG_MOMTE_LEN); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, momte, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, momte, 0x00, 16, 0x00, 12); enum mlxsw_reg_momte_type { MLXSW_REG_MOMTE_TYPE_WRED = 0x20, @@ -11030,7 +11019,7 @@ MLXSW_ITEM32(reg, momte, type, 0x04, 0, 8); */ MLXSW_ITEM_BIT_ARRAY(reg, momte, tclass_en, 0x08, 0x08, 1); -static inline void mlxsw_reg_momte_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_momte_pack(char *payload, u16 local_port, enum mlxsw_reg_momte_type type) { MLXSW_REG_ZERO(momte, payload); @@ -11098,7 +11087,7 @@ MLXSW_REG_DEFINE(mtpptr, MLXSW_REG_MTPPTR_ID, MLXSW_REG_MTPPTR_LEN); * Not supported for CPU port. * Access: Index */ -MLXSW_ITEM32(reg, mtpptr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, mtpptr, 0x00, 16, 0x00, 12); enum mlxsw_reg_mtpptr_dir { MLXSW_REG_MTPPTR_DIR_INGRESS, @@ -11692,7 +11681,7 @@ MLXSW_REG_DEFINE(tnqdr, MLXSW_REG_TNQDR_ID, MLXSW_REG_TNQDR_LEN); * Local port number (receive port). CPU port is supported. * Access: Index */ -MLXSW_ITEM32(reg, tnqdr, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, tnqdr, 0x00, 16, 0x00, 12); /* reg_tnqdr_dscp * For encapsulation, the default DSCP. @@ -11700,7 +11689,7 @@ MLXSW_ITEM32(reg, tnqdr, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, tnqdr, dscp, 0x04, 0, 6); -static inline void mlxsw_reg_tnqdr_pack(char *payload, u8 local_port) +static inline void mlxsw_reg_tnqdr_pack(char *payload, u16 local_port) { MLXSW_REG_ZERO(tnqdr, payload); mlxsw_reg_tnqdr_local_port_set(payload, local_port); @@ -12028,7 +12017,7 @@ MLXSW_REG_DEFINE(sbcm, MLXSW_REG_SBCM_ID, MLXSW_REG_SBCM_LEN); * For Egress: excludes IP Router * Access: Index */ -MLXSW_ITEM32(reg, sbcm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, sbcm, 0x00, 16, 0x00, 4); /* reg_sbcm_pg_buff * PG buffer - Port PG (dir=ingress) / traffic class (dir=egress) @@ -12082,7 +12071,7 @@ MLXSW_ITEM32(reg, sbcm, max_buff, 0x1C, 0, 24); */ MLXSW_ITEM32(reg, sbcm, pool, 0x24, 0, 4); -static inline void mlxsw_reg_sbcm_pack(char *payload, u8 local_port, u8 pg_buff, +static inline void mlxsw_reg_sbcm_pack(char *payload, u16 local_port, u8 pg_buff, enum mlxsw_reg_sbxx_dir dir, u32 min_buff, u32 max_buff, bool infi_max, u8 pool) @@ -12114,7 +12103,7 @@ MLXSW_REG_DEFINE(sbpm, MLXSW_REG_SBPM_ID, MLXSW_REG_SBPM_LEN); * For Egress: excludes IP Router * Access: Index */ -MLXSW_ITEM32(reg, sbpm, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, sbpm, 0x00, 16, 0x00, 12); /* reg_sbpm_pool * The pool associated to quota counting on the local_port. @@ -12168,7 +12157,7 @@ MLXSW_ITEM32(reg, sbpm, min_buff, 0x18, 0, 24); */ MLXSW_ITEM32(reg, sbpm, max_buff, 0x1C, 0, 24); -static inline void mlxsw_reg_sbpm_pack(char *payload, u8 local_port, u8 pool, +static inline void mlxsw_reg_sbpm_pack(char *payload, u16 local_port, u8 pool, enum mlxsw_reg_sbxx_dir dir, bool clr, u32 min_buff, u32 max_buff) { @@ -12266,6 +12255,16 @@ MLXSW_REG_DEFINE(sbsr, MLXSW_REG_SBSR_ID, MLXSW_REG_SBSR_LEN); */ MLXSW_ITEM32(reg, sbsr, clr, 0x00, 31, 1); +#define MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE 256 + +/* reg_sbsr_port_page + * Determines the range of the ports specified in the 'ingress_port_mask' + * and 'egress_port_mask' bit masks. + * {ingress,egress}_port_mask[x] is (256 * port_page) + x + * Access: Index + */ +MLXSW_ITEM32(reg, sbsr, port_page, 0x04, 0, 4); + /* reg_sbsr_ingress_port_mask * Bit vector for all ingress network ports. * Indicates which of the ports (for which the relevant bit is set) @@ -12353,7 +12352,7 @@ MLXSW_REG_DEFINE(sbib, MLXSW_REG_SBIB_ID, MLXSW_REG_SBIB_LEN); * Not supported for CPU port and router port * Access: Index */ -MLXSW_ITEM32(reg, sbib, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, sbib, 0x00, 16, 0x00, 12); /* reg_sbib_buff_size * Units represented in cells @@ -12363,7 +12362,7 @@ MLXSW_ITEM32(reg, sbib, local_port, 0x00, 16, 8); */ MLXSW_ITEM32(reg, sbib, buff_size, 0x08, 0, 24); -static inline void mlxsw_reg_sbib_pack(char *payload, u8 local_port, +static inline void mlxsw_reg_sbib_pack(char *payload, u16 local_port, u32 buff_size) { MLXSW_REG_ZERO(sbib, payload); @@ -12374,7 +12373,6 @@ static inline void mlxsw_reg_sbib_pack(char *payload, u8 local_port, static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(sgcr), MLXSW_REG(spad), - MLXSW_REG(smid), MLXSW_REG(sspr), MLXSW_REG(sfdat), MLXSW_REG(sfd), @@ -12384,7 +12382,6 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(spvm), MLXSW_REG(spaft), MLXSW_REG(sfgc), - MLXSW_REG(sftr), MLXSW_REG(sfdf), MLXSW_REG(sldr), MLXSW_REG(slcr), @@ -12397,6 +12394,8 @@ static const struct mlxsw_reg_info *mlxsw_reg_infos[] = { MLXSW_REG(spvmlr), MLXSW_REG(spvc), MLXSW_REG(spevet), + MLXSW_REG(sftr2), + MLXSW_REG(smid2), MLXSW_REG(cwtp), MLXSW_REG(cwtpm), MLXSW_REG(pgcr), @@ -12556,7 +12555,7 @@ MLXSW_ITEM32(reg, pude, swid, 0x00, 24, 8); * Local port number. * Access: Index */ -MLXSW_ITEM32(reg, pude, local_port, 0x00, 16, 8); +MLXSW_ITEM32_LP(reg, pude, 0x00, 16, 0x00, 12); /* reg_pude_admin_status * Port administrative state (the desired state). diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c index 03e5bad4e405..fb06b2ddfd6d 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.c @@ -46,8 +46,8 @@ #include "spectrum_trap.h" #define MLXSW_SP1_FWREV_MAJOR 13 -#define MLXSW_SP1_FWREV_MINOR 2008 -#define MLXSW_SP1_FWREV_SUBMINOR 3326 +#define MLXSW_SP1_FWREV_MINOR 2010 +#define MLXSW_SP1_FWREV_SUBMINOR 1006 #define MLXSW_SP1_FWREV_CAN_RESET_MINOR 1702 static const struct mlxsw_fw_rev mlxsw_sp1_fw_rev = { @@ -63,8 +63,8 @@ static const struct mlxsw_fw_rev mlxsw_sp1_fw_rev = { "." __stringify(MLXSW_SP1_FWREV_SUBMINOR) ".mfa2" #define MLXSW_SP2_FWREV_MAJOR 29 -#define MLXSW_SP2_FWREV_MINOR 2008 -#define MLXSW_SP2_FWREV_SUBMINOR 3326 +#define MLXSW_SP2_FWREV_MINOR 2010 +#define MLXSW_SP2_FWREV_SUBMINOR 1006 static const struct mlxsw_fw_rev mlxsw_sp2_fw_rev = { .major = MLXSW_SP2_FWREV_MAJOR, @@ -78,8 +78,8 @@ static const struct mlxsw_fw_rev mlxsw_sp2_fw_rev = { "." __stringify(MLXSW_SP2_FWREV_SUBMINOR) ".mfa2" #define MLXSW_SP3_FWREV_MAJOR 30 -#define MLXSW_SP3_FWREV_MINOR 2008 -#define MLXSW_SP3_FWREV_SUBMINOR 3326 +#define MLXSW_SP3_FWREV_MINOR 2010 +#define MLXSW_SP3_FWREV_SUBMINOR 1006 static const struct mlxsw_fw_rev mlxsw_sp3_fw_rev = { .major = MLXSW_SP3_FWREV_MAJOR, @@ -303,7 +303,7 @@ int mlxsw_sp_port_admin_status_set(struct mlxsw_sp_port *mlxsw_sp_port, } static int mlxsw_sp_port_dev_addr_set(struct mlxsw_sp_port *mlxsw_sp_port, - unsigned char *addr) + const unsigned char *addr) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; char ppad_pl[MLXSW_REG_PPAD_LEN]; @@ -352,7 +352,7 @@ static int mlxsw_sp_port_mtu_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 mtu) } static int mlxsw_sp_port_swid_set(struct mlxsw_sp *mlxsw_sp, - u8 local_port, u8 swid) + u16 local_port, u8 swid) { char pspa_pl[MLXSW_REG_PSPA_LEN]; @@ -483,7 +483,7 @@ mlxsw_sp_port_system_port_mapping_set(struct mlxsw_sp_port *mlxsw_sp_port) } static int -mlxsw_sp_port_module_info_get(struct mlxsw_sp *mlxsw_sp, u8 local_port, +mlxsw_sp_port_module_info_get(struct mlxsw_sp *mlxsw_sp, u16 local_port, struct mlxsw_sp_port_mapping *port_mapping) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; @@ -535,7 +535,7 @@ mlxsw_sp_port_module_info_get(struct mlxsw_sp *mlxsw_sp, u8 local_port, } static int -mlxsw_sp_port_module_map(struct mlxsw_sp *mlxsw_sp, u8 local_port, +mlxsw_sp_port_module_map(struct mlxsw_sp *mlxsw_sp, u16 local_port, const struct mlxsw_sp_port_mapping *port_mapping) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; @@ -560,7 +560,7 @@ err_pmlp_write: return err; } -static void mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static void mlxsw_sp_port_module_unmap(struct mlxsw_sp *mlxsw_sp, u16 local_port, u8 module) { char pmlp_pl[MLXSW_REG_PMLP_LEN]; @@ -1474,7 +1474,7 @@ mlxsw_sp_port_vlan_classification_set(struct mlxsw_sp_port *mlxsw_sp_port, } static int mlxsw_sp_port_label_info_get(struct mlxsw_sp *mlxsw_sp, - u8 local_port, u8 *port_number, + u16 local_port, u8 *port_number, u8 *split_port_subnumber, u8 *slot_index) { @@ -1490,7 +1490,7 @@ static int mlxsw_sp_port_label_info_get(struct mlxsw_sp *mlxsw_sp, return 0; } -static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_port_create(struct mlxsw_sp *mlxsw_sp, u16 local_port, bool split, struct mlxsw_sp_port_mapping *port_mapping) { @@ -1781,7 +1781,7 @@ err_port_swid_set: return err; } -static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u8 local_port) +static void mlxsw_sp_port_remove(struct mlxsw_sp *mlxsw_sp, u16 local_port) { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp->ports[local_port]; u8 module = mlxsw_sp_port->mapping.module; @@ -1848,12 +1848,12 @@ static void mlxsw_sp_cpu_port_remove(struct mlxsw_sp *mlxsw_sp) kfree(mlxsw_sp_port); } -static bool mlxsw_sp_local_port_valid(u8 local_port) +static bool mlxsw_sp_local_port_valid(u16 local_port) { return local_port != MLXSW_PORT_CPU_PORT; } -static bool mlxsw_sp_port_created(struct mlxsw_sp *mlxsw_sp, u8 local_port) +static bool mlxsw_sp_port_created(struct mlxsw_sp *mlxsw_sp, u16 local_port) { if (!mlxsw_sp_local_port_valid(local_port)) return false; @@ -1971,7 +1971,7 @@ mlxsw_sp_port_split_create(struct mlxsw_sp *mlxsw_sp, split_port_mapping = *port_mapping; split_port_mapping.width /= count; for (i = 0; i < count; i++) { - u8 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); + u16 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); if (!mlxsw_sp_local_port_valid(s_local_port)) continue; @@ -1987,7 +1987,7 @@ mlxsw_sp_port_split_create(struct mlxsw_sp *mlxsw_sp, err_port_create: for (i--; i >= 0; i--) { - u8 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); + u16 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); if (mlxsw_sp_port_created(mlxsw_sp, s_local_port)) mlxsw_sp_port_remove(mlxsw_sp, s_local_port); @@ -2004,7 +2004,7 @@ static void mlxsw_sp_port_unsplit_create(struct mlxsw_sp *mlxsw_sp, /* Go over original unsplit ports in the gap and recreate them. */ for (i = 0; i < count; i++) { - u8 local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); + u16 local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); port_mapping = mlxsw_sp->port_mapping[local_port]; if (!port_mapping || !mlxsw_sp_local_port_valid(local_port)) @@ -2015,14 +2015,14 @@ static void mlxsw_sp_port_unsplit_create(struct mlxsw_sp *mlxsw_sp, } static struct mlxsw_sp_port * -mlxsw_sp_port_get_by_local_port(struct mlxsw_sp *mlxsw_sp, u8 local_port) +mlxsw_sp_port_get_by_local_port(struct mlxsw_sp *mlxsw_sp, u16 local_port) { if (mlxsw_sp->ports && mlxsw_sp->ports[local_port]) return mlxsw_sp->ports[local_port]; return NULL; } -static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, +static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u16 local_port, unsigned int count, struct netlink_ext_ack *extack) { @@ -2065,7 +2065,7 @@ static int mlxsw_sp_port_split(struct mlxsw_core *mlxsw_core, u8 local_port, port_mapping = mlxsw_sp_port->mapping; for (i = 0; i < count; i++) { - u8 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); + u16 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); if (mlxsw_sp_port_created(mlxsw_sp, s_local_port)) mlxsw_sp_port_remove(mlxsw_sp, s_local_port); @@ -2085,7 +2085,7 @@ err_port_split_create: return err; } -static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u8 local_port, +static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u16 local_port, struct netlink_ext_ack *extack) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); @@ -2121,7 +2121,7 @@ static int mlxsw_sp_port_unsplit(struct mlxsw_core *mlxsw_core, u8 local_port, } for (i = 0; i < count; i++) { - u8 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); + u16 s_local_port = mlxsw_reg_pmtdb_port_num_get(pmtdb_pl, i); if (mlxsw_sp_port_created(mlxsw_sp, s_local_port)) mlxsw_sp_port_remove(mlxsw_sp, s_local_port); @@ -2148,7 +2148,7 @@ static void mlxsw_sp_pude_event_func(const struct mlxsw_reg_info *reg, struct mlxsw_sp_port *mlxsw_sp_port; enum mlxsw_reg_pude_oper_status status; unsigned int max_ports; - u8 local_port; + u16 local_port; max_ports = mlxsw_core_max_ports(mlxsw_sp->core); local_port = mlxsw_reg_pude_local_port_get(pude_pl); @@ -2174,7 +2174,7 @@ static void mlxsw_sp_pude_event_func(const struct mlxsw_reg_info *reg, static void mlxsw_sp1_ptp_fifo_event_func(struct mlxsw_sp *mlxsw_sp, char *mtpptr_pl, bool ingress) { - u8 local_port; + u16 local_port; u8 num_rec; int i; @@ -2212,7 +2212,7 @@ static void mlxsw_sp1_ptp_egr_fifo_event_func(const struct mlxsw_reg_info *reg, } void mlxsw_sp_rx_listener_no_mark_func(struct sk_buff *skb, - u8 local_port, void *priv) + u16 local_port, void *priv) { struct mlxsw_sp *mlxsw_sp = priv; struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp->ports[local_port]; @@ -2236,7 +2236,7 @@ void mlxsw_sp_rx_listener_no_mark_func(struct sk_buff *skb, netif_receive_skb(skb); } -static void mlxsw_sp_rx_listener_mark_func(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_listener_mark_func(struct sk_buff *skb, u16 local_port, void *priv) { skb->offload_fwd_mark = 1; @@ -2244,7 +2244,7 @@ static void mlxsw_sp_rx_listener_mark_func(struct sk_buff *skb, u8 local_port, } static void mlxsw_sp_rx_listener_l3_mark_func(struct sk_buff *skb, - u8 local_port, void *priv) + u16 local_port, void *priv) { skb->offload_l3_fwd_mark = 1; skb->offload_fwd_mark = 1; @@ -2252,7 +2252,7 @@ static void mlxsw_sp_rx_listener_l3_mark_func(struct sk_buff *skb, } void mlxsw_sp_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port) + u16 local_port) { mlxsw_sp->ptp_ops->receive(mlxsw_sp, skb, local_port); } @@ -3336,7 +3336,7 @@ err_resources_rif_mac_profile_register: err_policer_resources_register: err_resources_counter_register: err_resources_span_register: - devlink_resources_unregister(priv_to_devlink(mlxsw_core), NULL); + devlink_resources_unregister(priv_to_devlink(mlxsw_core)); return err; } @@ -3370,7 +3370,7 @@ err_resources_rif_mac_profile_register: err_policer_resources_register: err_resources_counter_register: err_resources_span_register: - devlink_resources_unregister(priv_to_devlink(mlxsw_core), NULL); + devlink_resources_unregister(priv_to_devlink(mlxsw_core)); return err; } @@ -3486,7 +3486,7 @@ static void mlxsw_sp2_params_unregister(struct mlxsw_core *mlxsw_core) } static void mlxsw_sp_ptp_transmitted(struct mlxsw_core *mlxsw_core, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h index 32fdd37657dd..ef4188e203a0 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum.h @@ -217,13 +217,13 @@ struct mlxsw_sp_ptp_ops { * is responsible for freeing the passed-in SKB. */ void (*receive)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); + u16 local_port); /* Notify a driver that a timestamped packet was transmitted. Driver * is responsible for freeing the passed-in SKB. */ void (*transmitted)(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); + u16 local_port); int (*hwtstamp_get)(struct mlxsw_sp_port *mlxsw_sp_port, struct hwtstamp_config *config); @@ -261,7 +261,7 @@ enum mlxsw_sp_sample_trigger_type { struct mlxsw_sp_sample_trigger { enum mlxsw_sp_sample_trigger_type type; - u8 local_port; /* Reserved when trigger type is not ingress / egress. */ + u16 local_port; /* Reserved when trigger type is not ingress / egress. */ }; struct mlxsw_sp_sample_params { @@ -308,7 +308,7 @@ struct mlxsw_sp_port { struct net_device *dev; struct mlxsw_sp_port_pcpu_stats __percpu *pcpu_stats; struct mlxsw_sp *mlxsw_sp; - u8 local_port; + u16 local_port; u8 lagged:1, split:1; u16 pvid; @@ -370,7 +370,7 @@ struct mlxsw_sp_port_type_speed_ops { u32 (*to_ptys_speed_lanes)(struct mlxsw_sp *mlxsw_sp, u8 width, const struct ethtool_link_ksettings *cmd); void (*reg_ptys_eth_pack)(struct mlxsw_sp *mlxsw_sp, char *payload, - u8 local_port, u32 proto_admin, bool autoneg); + u16 local_port, u32 proto_admin, bool autoneg); void (*reg_ptys_eth_unpack)(struct mlxsw_sp *mlxsw_sp, char *payload, u32 *p_eth_proto_cap, u32 *p_eth_proto_admin, @@ -441,7 +441,7 @@ static inline struct mlxsw_sp_port * mlxsw_sp_port_lagged_get(struct mlxsw_sp *mlxsw_sp, u16 lag_id, u8 port_index) { struct mlxsw_sp_port *mlxsw_sp_port; - u8 local_port; + u16 local_port; local_port = mlxsw_core_lag_mapping_get(mlxsw_sp->core, lag_id, port_index); @@ -621,9 +621,9 @@ extern struct notifier_block mlxsw_sp_switchdev_notifier; /* spectrum.c */ void mlxsw_sp_rx_listener_no_mark_func(struct sk_buff *skb, - u8 local_port, void *priv); + u16 local_port, void *priv); void mlxsw_sp_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); + u16 local_port); int mlxsw_sp_port_speed_get(struct mlxsw_sp_port *mlxsw_sp_port, u32 *speed); int mlxsw_sp_port_ets_set(struct mlxsw_sp_port *mlxsw_sp_port, enum mlxsw_reg_qeec_hr hr, u8 index, u8 next_index, @@ -729,7 +729,7 @@ void mlxsw_sp_rif_destroy_by_dev(struct mlxsw_sp *mlxsw_sp, bool mlxsw_sp_rif_exists(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev); u16 mlxsw_sp_rif_vid(struct mlxsw_sp *mlxsw_sp, const struct net_device *dev); -u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp); +u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp); int mlxsw_sp_router_nve_promote_decap(struct mlxsw_sp *mlxsw_sp, u32 ul_tb_id, enum mlxsw_sp_l3proto ul_proto, const union mlxsw_sp_l3addr *ul_sip, @@ -1222,7 +1222,7 @@ bool mlxsw_sp_fid_vni_is_set(const struct mlxsw_sp_fid *fid); void mlxsw_sp_fid_fdb_clear_offload(const struct mlxsw_sp_fid *fid, const struct net_device *nve_dev); int mlxsw_sp_fid_flood_set(struct mlxsw_sp_fid *fid, - enum mlxsw_sp_flood_type packet_type, u8 local_port, + enum mlxsw_sp_flood_type packet_type, u16 local_port, bool member); int mlxsw_sp_fid_port_vid_map(struct mlxsw_sp_fid *fid, struct mlxsw_sp_port *mlxsw_sp_port, u16 vid); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c index 67cedfa76f78..70c11bfac08f 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl.c @@ -406,7 +406,7 @@ int mlxsw_sp_acl_rulei_act_fwd(struct mlxsw_sp *mlxsw_sp, struct netlink_ext_ack *extack) { struct mlxsw_sp_port *mlxsw_sp_port; - u8 local_port; + u16 local_port; bool in_port; if (out_dev) { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c index c72aa38424dc..50806594d977 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_acl_flex_actions.c @@ -83,7 +83,7 @@ static int mlxsw_sp2_act_kvdl_set_activity_get(void *priv, u32 kvdl_index, } static int mlxsw_sp_act_kvdl_fwd_entry_add(void *priv, u32 *p_kvdl_index, - u8 local_port) + u16 local_port) { struct mlxsw_sp *mlxsw_sp = priv; char ppbs_pl[MLXSW_REG_PPBS_LEN]; @@ -132,7 +132,7 @@ mlxsw_sp_act_counter_index_put(void *priv, unsigned int counter_index) } static int -mlxsw_sp_act_mirror_add(void *priv, u8 local_in_port, +mlxsw_sp_act_mirror_add(void *priv, u16 local_in_port, const struct net_device *out_dev, bool ingress, int *p_span_id) { @@ -159,7 +159,7 @@ err_analyzed_port_get: } static void -mlxsw_sp_act_mirror_del(void *priv, u8 local_in_port, int span_id, bool ingress) +mlxsw_sp_act_mirror_del(void *priv, u16 local_in_port, int span_id, bool ingress) { struct mlxsw_sp_port *mlxsw_sp_port; struct mlxsw_sp *mlxsw_sp = priv; @@ -192,7 +192,7 @@ static void mlxsw_sp_act_policer_del(void *priv, u16 policer_index) policer_index); } -static int mlxsw_sp1_act_sampler_add(void *priv, u8 local_port, +static int mlxsw_sp1_act_sampler_add(void *priv, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, int *p_span_id, @@ -202,7 +202,7 @@ static int mlxsw_sp1_act_sampler_add(void *priv, u8 local_port, return -EOPNOTSUPP; } -static void mlxsw_sp1_act_sampler_del(void *priv, u8 local_port, int span_id, +static void mlxsw_sp1_act_sampler_del(void *priv, u16 local_port, int span_id, bool ingress) { WARN_ON_ONCE(1); @@ -224,7 +224,7 @@ const struct mlxsw_afa_ops mlxsw_sp1_act_afa_ops = { .sampler_del = mlxsw_sp1_act_sampler_del, }; -static int mlxsw_sp2_act_sampler_add(void *priv, u8 local_port, +static int mlxsw_sp2_act_sampler_add(void *priv, u16 local_port, struct psample_group *psample_group, u32 rate, u32 trunc_size, bool truncate, bool ingress, int *p_span_id, @@ -272,7 +272,7 @@ err_span_agent_get: return err; } -static void mlxsw_sp2_act_sampler_del(void *priv, u8 local_port, int span_id, +static void mlxsw_sp2_act_sampler_del(void *priv, u16 local_port, int span_id, bool ingress) { struct mlxsw_sp_sample_trigger trigger = { diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index d78cf5a7220a..98f26f596e30 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -160,7 +160,7 @@ static bool mlxsw_sp_sb_cm_exists(u8 pg_buff, enum mlxsw_reg_sbxx_dir dir) } static struct mlxsw_sp_sb_cm *mlxsw_sp_sb_cm_get(struct mlxsw_sp *mlxsw_sp, - u8 local_port, u8 pg_buff, + u16 local_port, u8 pg_buff, enum mlxsw_reg_sbxx_dir dir) { struct mlxsw_sp_sb_port *sb_port = &mlxsw_sp->sb->ports[local_port]; @@ -173,7 +173,7 @@ static struct mlxsw_sp_sb_cm *mlxsw_sp_sb_cm_get(struct mlxsw_sp *mlxsw_sp, } static struct mlxsw_sp_sb_pm *mlxsw_sp_sb_pm_get(struct mlxsw_sp *mlxsw_sp, - u8 local_port, u16 pool_index) + u16 local_port, u16 pool_index) { return &mlxsw_sp->sb->ports[local_port].pms[pool_index]; } @@ -202,7 +202,7 @@ static int mlxsw_sp_sb_pr_write(struct mlxsw_sp *mlxsw_sp, u16 pool_index, return 0; } -static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port, u8 pg_buff, u32 min_buff, u32 max_buff, bool infi_max, u16 pool_index) { @@ -232,7 +232,7 @@ static int mlxsw_sp_sb_cm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port, return 0; } -static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u16 local_port, u16 pool_index, u32 min_buff, u32 max_buff) { const struct mlxsw_sp_sb_pool_des *des = @@ -253,7 +253,7 @@ static int mlxsw_sp_sb_pm_write(struct mlxsw_sp *mlxsw_sp, u8 local_port, return 0; } -static int mlxsw_sp_sb_pm_occ_clear(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_sb_pm_occ_clear(struct mlxsw_sp *mlxsw_sp, u16 local_port, u16 pool_index, struct list_head *bulk_list) { const struct mlxsw_sp_sb_pool_des *des = @@ -279,7 +279,7 @@ static void mlxsw_sp_sb_pm_occ_query_cb(struct mlxsw_core *mlxsw_core, mlxsw_reg_sbpm_unpack(sbpm_pl, &pm->occ.cur, &pm->occ.max); } -static int mlxsw_sp_sb_pm_occ_query(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_sb_pm_occ_query(struct mlxsw_sp *mlxsw_sp, u16 local_port, u16 pool_index, struct list_head *bulk_list) { const struct mlxsw_sp_sb_pool_des *des = @@ -919,7 +919,7 @@ mlxsw_sp_sb_pool_is_static(struct mlxsw_sp *mlxsw_sp, u16 pool_index) return pr->mode == MLXSW_REG_SBPR_MODE_STATIC; } -static int __mlxsw_sp_sb_cms_init(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int __mlxsw_sp_sb_cms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port, enum mlxsw_reg_sbxx_dir dir, const struct mlxsw_sp_sb_cm *cms, size_t cms_len) @@ -1037,7 +1037,7 @@ static const struct mlxsw_sp_sb_pm mlxsw_sp_cpu_port_sb_pms[] = { MLXSW_SP_SB_PM(0, MLXSW_REG_SBXX_DYN_MAX_BUFF_MAX), }; -static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_sb_pms_init(struct mlxsw_sp *mlxsw_sp, u16 local_port, const struct mlxsw_sp_sb_pm *pms, bool skip_ingress) { @@ -1416,7 +1416,7 @@ int mlxsw_sp_sb_port_pool_get(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index); @@ -1432,7 +1432,7 @@ int mlxsw_sp_sb_port_pool_set(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; u32 max_buff; int err; @@ -1458,7 +1458,7 @@ int mlxsw_sp_sb_tc_pool_bind_get(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; u8 pg_buff = tc_index; enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type; struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, @@ -1479,7 +1479,7 @@ int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; const struct mlxsw_sp_sb_cm *cm; u8 pg_buff = tc_index; enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type; @@ -1526,7 +1526,7 @@ int mlxsw_sp_sb_tc_pool_bind_set(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_sb_sr_occ_query_cb_ctx { u8 masked_count; - u8 local_port_1; + u16 local_port_1; }; static void mlxsw_sp_sb_sr_occ_query_cb(struct mlxsw_core *mlxsw_core, @@ -1536,7 +1536,7 @@ static void mlxsw_sp_sb_sr_occ_query_cb(struct mlxsw_core *mlxsw_core, struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx; u8 masked_count; - u8 local_port; + u16 local_port; int rec_index = 0; struct mlxsw_sp_sb_cm *cm; int i; @@ -1582,13 +1582,12 @@ int mlxsw_sp_sb_occ_snapshot(struct mlxsw_core *mlxsw_core, unsigned int sb_index) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + u16 local_port, local_port_1, last_local_port; struct mlxsw_sp_sb_sr_occ_query_cb_ctx cb_ctx; + u8 masked_count, current_page = 0; unsigned long cb_priv = 0; LIST_HEAD(bulk_list); char *sbsr_pl; - u8 masked_count; - u8 local_port_1; - u8 local_port; int i; int err; int err2; @@ -1602,6 +1601,10 @@ next_batch: local_port_1 = local_port; masked_count = 0; mlxsw_reg_sbsr_pack(sbsr_pl, false); + mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page); + last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE + + MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1; + for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++) mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1); for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++) @@ -1609,6 +1612,10 @@ next_batch: for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) { if (!mlxsw_sp->ports[local_port]) continue; + if (local_port > last_local_port) { + current_page++; + goto do_query; + } if (local_port != MLXSW_PORT_CPU_PORT) { /* Ingress quotas are not supported for the CPU port */ mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl, @@ -1651,10 +1658,11 @@ int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core, unsigned int sb_index) { struct mlxsw_sp *mlxsw_sp = mlxsw_core_driver_priv(mlxsw_core); + u16 local_port, last_local_port; LIST_HEAD(bulk_list); - char *sbsr_pl; unsigned int masked_count; - u8 local_port; + u8 current_page = 0; + char *sbsr_pl; int i; int err; int err2; @@ -1667,6 +1675,10 @@ int mlxsw_sp_sb_occ_max_clear(struct mlxsw_core *mlxsw_core, next_batch: masked_count = 0; mlxsw_reg_sbsr_pack(sbsr_pl, true); + mlxsw_reg_sbsr_port_page_set(sbsr_pl, current_page); + last_local_port = current_page * MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE + + MLXSW_REG_SBSR_NUM_PORTS_IN_PAGE - 1; + for (i = 0; i < MLXSW_SP_SB_ING_TC_COUNT; i++) mlxsw_reg_sbsr_pg_buff_mask_set(sbsr_pl, i, 1); for (i = 0; i < MLXSW_SP_SB_EG_TC_COUNT; i++) @@ -1674,6 +1686,10 @@ next_batch: for (; local_port < mlxsw_core_max_ports(mlxsw_core); local_port++) { if (!mlxsw_sp->ports[local_port]) continue; + if (local_port > last_local_port) { + current_page++; + goto do_query; + } if (local_port != MLXSW_PORT_CPU_PORT) { /* Ingress quotas are not supported for the CPU port */ mlxsw_reg_sbsr_ingress_port_mask_set(sbsr_pl, @@ -1715,7 +1731,7 @@ int mlxsw_sp_sb_occ_port_pool_get(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; struct mlxsw_sp_sb_pm *pm = mlxsw_sp_sb_pm_get(mlxsw_sp, local_port, pool_index); @@ -1732,7 +1748,7 @@ int mlxsw_sp_sb_occ_tc_port_bind_get(struct mlxsw_core_port *mlxsw_core_port, struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_core_port_driver_priv(mlxsw_core_port); struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; u8 pg_buff = tc_index; enum mlxsw_reg_sbxx_dir dir = (enum mlxsw_reg_sbxx_dir) pool_type; struct mlxsw_sp_sb_cm *cm = mlxsw_sp_sb_cm_get(mlxsw_sp, local_port, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c index 84d4460f3dcd..20530712eadb 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ethtool.c @@ -1491,7 +1491,7 @@ static u32 mlxsw_sp1_to_ptys_speed_lanes(struct mlxsw_sp *mlxsw_sp, u8 width, static void mlxsw_sp1_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, - u8 local_port, u32 proto_admin, bool autoneg) + u16 local_port, u32 proto_admin, bool autoneg) { mlxsw_reg_ptys_eth_pack(payload, local_port, proto_admin, autoneg); } @@ -1969,7 +1969,7 @@ static u32 mlxsw_sp2_to_ptys_speed_lanes(struct mlxsw_sp *mlxsw_sp, u8 width, static void mlxsw_sp2_reg_ptys_eth_pack(struct mlxsw_sp *mlxsw_sp, char *payload, - u8 local_port, u32 proto_admin, + u16 local_port, u32 proto_admin, bool autoneg) { mlxsw_reg_ptys_ext_eth_pack(payload, local_port, proto_admin, autoneg); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c index 004c42274e48..ce80931f0402 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_fid.c @@ -317,13 +317,13 @@ mlxsw_sp_fid_flood_table_lookup(const struct mlxsw_sp_fid *fid, } int mlxsw_sp_fid_flood_set(struct mlxsw_sp_fid *fid, - enum mlxsw_sp_flood_type packet_type, u8 local_port, + enum mlxsw_sp_flood_type packet_type, u16 local_port, bool member) { struct mlxsw_sp_fid_family *fid_family = fid->fid_family; const struct mlxsw_sp_fid_ops *ops = fid_family->ops; const struct mlxsw_sp_flood_table *flood_table; - char *sftr_pl; + char *sftr2_pl; int err; if (WARN_ON(!fid_family->flood_tables || !ops->flood_index)) @@ -333,16 +333,16 @@ int mlxsw_sp_fid_flood_set(struct mlxsw_sp_fid *fid, if (!flood_table) return -ESRCH; - sftr_pl = kmalloc(MLXSW_REG_SFTR_LEN, GFP_KERNEL); - if (!sftr_pl) + sftr2_pl = kmalloc(MLXSW_REG_SFTR2_LEN, GFP_KERNEL); + if (!sftr2_pl) return -ENOMEM; - mlxsw_reg_sftr_pack(sftr_pl, flood_table->table_index, - ops->flood_index(fid), flood_table->table_type, 1, - local_port, member); - err = mlxsw_reg_write(fid_family->mlxsw_sp->core, MLXSW_REG(sftr), - sftr_pl); - kfree(sftr_pl); + mlxsw_reg_sftr2_pack(sftr2_pl, flood_table->table_index, + ops->flood_index(fid), flood_table->table_type, 1, + local_port, member); + err = mlxsw_reg_write(fid_family->mlxsw_sp->core, MLXSW_REG(sftr2), + sftr2_pl); + kfree(sftr2_pl); return err; } @@ -439,7 +439,7 @@ static int mlxsw_sp_fid_vni_op(struct mlxsw_sp *mlxsw_sp, u16 fid_index, } static int __mlxsw_sp_fid_port_vid_map(struct mlxsw_sp *mlxsw_sp, u16 fid_index, - u8 local_port, u16 vid, bool valid) + u16 local_port, u16 vid, bool valid) { enum mlxsw_reg_svfa_mt mt = MLXSW_REG_SVFA_MT_PORT_VID_TO_FID; char svfa_pl[MLXSW_REG_SVFA_LEN]; @@ -573,7 +573,7 @@ static int mlxsw_sp_fid_8021d_port_vid_map(struct mlxsw_sp_fid *fid, u16 vid) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; int err; err = __mlxsw_sp_fid_port_vid_map(mlxsw_sp, fid->fid_index, @@ -601,7 +601,7 @@ mlxsw_sp_fid_8021d_port_vid_unmap(struct mlxsw_sp_fid *fid, struct mlxsw_sp_port *mlxsw_sp_port, u16 vid) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; if (mlxsw_sp->fid_core->port_fid_mappings[local_port] == 1) mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port); @@ -784,7 +784,7 @@ static int mlxsw_sp_fid_rfid_port_vid_map(struct mlxsw_sp_fid *fid, u16 vid) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; int err; /* We only need to transition the port to virtual mode since @@ -808,7 +808,7 @@ mlxsw_sp_fid_rfid_port_vid_unmap(struct mlxsw_sp_fid *fid, struct mlxsw_sp_port *mlxsw_sp_port, u16 vid) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; if (mlxsw_sp->fid_core->port_fid_mappings[local_port] == 1) mlxsw_sp_port_vlan_mode_trans(mlxsw_sp_port); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c index 1a180384e7e8..0ff163fbc775 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.c @@ -36,7 +36,7 @@ struct mlxsw_sp_ptp_state { }; struct mlxsw_sp1_ptp_key { - u8 local_port; + u16 local_port; u8 message_type; u16 sequence_id; u8 domain_number; @@ -406,7 +406,7 @@ mlxsw_sp1_ptp_unmatched_remove(struct mlxsw_sp *mlxsw_sp, * This case is similar to 2) above. */ static void mlxsw_sp1_ptp_packet_finish(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port, + struct sk_buff *skb, u16 local_port, bool ingress, struct skb_shared_hwtstamps *hwtstamps) { @@ -524,7 +524,7 @@ static void mlxsw_sp1_ptp_got_piece(struct mlxsw_sp *mlxsw_sp, } static void mlxsw_sp1_ptp_got_packet(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port, + struct sk_buff *skb, u16 local_port, bool ingress) { struct mlxsw_sp_port *mlxsw_sp_port; @@ -564,7 +564,7 @@ immediate: } void mlxsw_sp1_ptp_got_timestamp(struct mlxsw_sp *mlxsw_sp, bool ingress, - u8 local_port, u8 message_type, + u16 local_port, u8 message_type, u8 domain_number, u16 sequence_id, u64 timestamp) { @@ -599,14 +599,14 @@ void mlxsw_sp1_ptp_got_timestamp(struct mlxsw_sp *mlxsw_sp, bool ingress, } void mlxsw_sp1_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port) + u16 local_port) { skb_reset_mac_header(skb); mlxsw_sp1_ptp_got_packet(mlxsw_sp, skb, local_port, true); } void mlxsw_sp1_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { mlxsw_sp1_ptp_got_packet(mlxsw_sp, skb, local_port, false); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h index 1d43a3755285..c06cd1384bca 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_ptp.h @@ -31,13 +31,13 @@ struct mlxsw_sp_ptp_state *mlxsw_sp1_ptp_init(struct mlxsw_sp *mlxsw_sp); void mlxsw_sp1_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state); void mlxsw_sp1_ptp_receive(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port); + u16 local_port); void mlxsw_sp1_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port); + struct sk_buff *skb, u16 local_port); void mlxsw_sp1_ptp_got_timestamp(struct mlxsw_sp *mlxsw_sp, bool ingress, - u8 local_port, u8 message_type, + u16 local_port, u8 message_type, u8 domain_number, u16 sequence_id, u64 timestamp); @@ -80,20 +80,20 @@ static inline void mlxsw_sp1_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state) } static inline void mlxsw_sp1_ptp_receive(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { mlxsw_sp_rx_listener_no_mark_func(skb, local_port, mlxsw_sp); } static inline void mlxsw_sp1_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { dev_kfree_skb_any(skb); } static inline void mlxsw_sp1_ptp_got_timestamp(struct mlxsw_sp *mlxsw_sp, bool ingress, - u8 local_port, u8 message_type, + u16 local_port, u8 message_type, u8 domain_number, u16 sequence_id, u64 timestamp) { @@ -159,13 +159,13 @@ static inline void mlxsw_sp2_ptp_fini(struct mlxsw_sp_ptp_state *ptp_state) } static inline void mlxsw_sp2_ptp_receive(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { mlxsw_sp_rx_listener_no_mark_func(skb, local_port, mlxsw_sp); } static inline void mlxsw_sp2_ptp_transmitted(struct mlxsw_sp *mlxsw_sp, - struct sk_buff *skb, u8 local_port) + struct sk_buff *skb, u16 local_port) { dev_kfree_skb_any(skb); } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c index 217e3b351dfe..2af0c6382609 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_router.c @@ -8369,9 +8369,6 @@ mlxsw_sp_rif_mac_profile_find(const struct mlxsw_sp *mlxsw_sp, const char *mac) int id; idr_for_each_entry(&router->rif_mac_profiles_idr, profile, id) { - if (!profile) - continue; - if (ether_addr_equal_masked(profile->mac_prefix, mac, mlxsw_sp->mac_mask)) return profile; @@ -9343,7 +9340,7 @@ static int mlxsw_sp_rif_vlan_fid_op(struct mlxsw_sp_rif *rif, return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(ritr), ritr_pl); } -u8 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) +u16 mlxsw_sp_router_port(const struct mlxsw_sp *mlxsw_sp) { return mlxsw_core_max_ports(mlxsw_sp->core) + 1; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c index f5f819aa9a65..f9671cc53002 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_span.c @@ -37,7 +37,7 @@ struct mlxsw_sp_span { struct mlxsw_sp_span_analyzed_port { struct list_head list; /* Member of analyzed_ports_list */ refcount_t ref_count; - u8 local_port; + u16 local_port; bool ingress; }; @@ -46,7 +46,7 @@ struct mlxsw_sp_span_trigger_entry { struct mlxsw_sp_span *span; const struct mlxsw_sp_span_trigger_ops *ops; refcount_t ref_count; - u8 local_port; + u16 local_port; enum mlxsw_sp_span_trigger trigger; struct mlxsw_sp_span_trigger_parms parms; }; @@ -179,7 +179,7 @@ mlxsw_sp_span_entry_phys_configure(struct mlxsw_sp_span_entry *span_entry, { struct mlxsw_sp_port *dest_port = sparms.dest_port; struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp; - u8 local_port = dest_port->local_port; + u16 local_port = dest_port->local_port; char mpat_pl[MLXSW_REG_MPAT_LEN]; int pa_id = span_entry->id; @@ -199,7 +199,7 @@ mlxsw_sp_span_entry_deconfigure_common(struct mlxsw_sp_span_entry *span_entry, { struct mlxsw_sp_port *dest_port = span_entry->parms.dest_port; struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp; - u8 local_port = dest_port->local_port; + u16 local_port = dest_port->local_port; char mpat_pl[MLXSW_REG_MPAT_LEN]; int pa_id = span_entry->id; @@ -480,7 +480,7 @@ mlxsw_sp_span_entry_gretap4_configure(struct mlxsw_sp_span_entry *span_entry, { struct mlxsw_sp_port *dest_port = sparms.dest_port; struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp; - u8 local_port = dest_port->local_port; + u16 local_port = dest_port->local_port; char mpat_pl[MLXSW_REG_MPAT_LEN]; int pa_id = span_entry->id; @@ -584,7 +584,7 @@ mlxsw_sp_span_entry_gretap6_configure(struct mlxsw_sp_span_entry *span_entry, { struct mlxsw_sp_port *dest_port = sparms.dest_port; struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp; - u8 local_port = dest_port->local_port; + u16 local_port = dest_port->local_port; char mpat_pl[MLXSW_REG_MPAT_LEN]; int pa_id = span_entry->id; @@ -650,7 +650,7 @@ mlxsw_sp_span_entry_vlan_configure(struct mlxsw_sp_span_entry *span_entry, { struct mlxsw_sp_port *dest_port = sparms.dest_port; struct mlxsw_sp *mlxsw_sp = dest_port->mlxsw_sp; - u8 local_port = dest_port->local_port; + u16 local_port = dest_port->local_port; char mpat_pl[MLXSW_REG_MPAT_LEN]; int pa_id = span_entry->id; @@ -997,7 +997,7 @@ static void mlxsw_sp_span_port_buffer_disable(struct mlxsw_sp_port *mlxsw_sp_por } static struct mlxsw_sp_span_analyzed_port * -mlxsw_sp_span_analyzed_port_find(struct mlxsw_sp_span *span, u8 local_port, +mlxsw_sp_span_analyzed_port_find(struct mlxsw_sp_span *span, u16 local_port, bool ingress) { struct mlxsw_sp_span_analyzed_port *analyzed_port; @@ -1165,7 +1165,7 @@ int mlxsw_sp_span_analyzed_port_get(struct mlxsw_sp_port *mlxsw_sp_port, { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; struct mlxsw_sp_span_analyzed_port *analyzed_port; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; int err = 0; mutex_lock(&mlxsw_sp->span->analyzed_ports_lock); @@ -1193,7 +1193,7 @@ void mlxsw_sp_span_analyzed_port_put(struct mlxsw_sp_port *mlxsw_sp_port, { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; struct mlxsw_sp_span_analyzed_port *analyzed_port; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; mutex_lock(&mlxsw_sp->span->analyzed_ports_lock); diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c index 81c7e8a7fcf5..c5fd69a6bedd 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_switchdev.c @@ -865,17 +865,17 @@ static int mlxsw_sp_port_mc_disabled_set(struct mlxsw_sp_port *mlxsw_sp_port, static int mlxsw_sp_smid_router_port_set(struct mlxsw_sp *mlxsw_sp, u16 mid_idx, bool add) { - char *smid_pl; + char *smid2_pl; int err; - smid_pl = kmalloc(MLXSW_REG_SMID_LEN, GFP_KERNEL); - if (!smid_pl) + smid2_pl = kmalloc(MLXSW_REG_SMID2_LEN, GFP_KERNEL); + if (!smid2_pl) return -ENOMEM; - mlxsw_reg_smid_pack(smid_pl, mid_idx, - mlxsw_sp_router_port(mlxsw_sp), add); - err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid), smid_pl); - kfree(smid_pl); + mlxsw_reg_smid2_pack(smid2_pl, mid_idx, + mlxsw_sp_router_port(mlxsw_sp), add); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid2), smid2_pl); + kfree(smid2_pl); return err; } @@ -980,7 +980,7 @@ mlxsw_sp_port_vlan_fid_join(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan, { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; struct mlxsw_sp_bridge_device *bridge_device; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; u16 vid = mlxsw_sp_port_vlan->vid; struct mlxsw_sp_fid *fid; int err; @@ -1029,7 +1029,7 @@ mlxsw_sp_port_vlan_fid_leave(struct mlxsw_sp_port_vlan *mlxsw_sp_port_vlan) { struct mlxsw_sp_port *mlxsw_sp_port = mlxsw_sp_port_vlan->mlxsw_sp_port; struct mlxsw_sp_fid *fid = mlxsw_sp_port_vlan->fid; - u8 local_port = mlxsw_sp_port->local_port; + u16 local_port = mlxsw_sp_port->local_port; u16 vid = mlxsw_sp_port_vlan->vid; mlxsw_sp_port_vlan->fid = NULL; @@ -1335,7 +1335,7 @@ out: return err; } -static int __mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int __mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u16 local_port, const char *mac, u16 fid, bool adding, enum mlxsw_reg_sfd_rec_action action, enum mlxsw_reg_sfd_rec_policy policy) @@ -1363,7 +1363,7 @@ out: return err; } -static int mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u8 local_port, +static int mlxsw_sp_port_fdb_uc_op(struct mlxsw_sp *mlxsw_sp, u16 local_port, const char *mac, u16 fid, bool adding, bool dynamic) { @@ -1477,30 +1477,30 @@ static int mlxsw_sp_port_smid_full_entry(struct mlxsw_sp *mlxsw_sp, u16 mid_idx, long *ports_bitmap, bool set_router_port) { - char *smid_pl; + char *smid2_pl; int err, i; - smid_pl = kmalloc(MLXSW_REG_SMID_LEN, GFP_KERNEL); - if (!smid_pl) + smid2_pl = kmalloc(MLXSW_REG_SMID2_LEN, GFP_KERNEL); + if (!smid2_pl) return -ENOMEM; - mlxsw_reg_smid_pack(smid_pl, mid_idx, 0, false); + mlxsw_reg_smid2_pack(smid2_pl, mid_idx, 0, false); for (i = 1; i < mlxsw_core_max_ports(mlxsw_sp->core); i++) { if (mlxsw_sp->ports[i]) - mlxsw_reg_smid_port_mask_set(smid_pl, i, 1); + mlxsw_reg_smid2_port_mask_set(smid2_pl, i, 1); } - mlxsw_reg_smid_port_mask_set(smid_pl, - mlxsw_sp_router_port(mlxsw_sp), 1); + mlxsw_reg_smid2_port_mask_set(smid2_pl, + mlxsw_sp_router_port(mlxsw_sp), 1); for_each_set_bit(i, ports_bitmap, mlxsw_core_max_ports(mlxsw_sp->core)) - mlxsw_reg_smid_port_set(smid_pl, i, 1); + mlxsw_reg_smid2_port_set(smid2_pl, i, 1); - mlxsw_reg_smid_port_set(smid_pl, mlxsw_sp_router_port(mlxsw_sp), - set_router_port); + mlxsw_reg_smid2_port_set(smid2_pl, mlxsw_sp_router_port(mlxsw_sp), + set_router_port); - err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid), smid_pl); - kfree(smid_pl); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid2), smid2_pl); + kfree(smid2_pl); return err; } @@ -1508,16 +1508,16 @@ static int mlxsw_sp_port_smid_set(struct mlxsw_sp_port *mlxsw_sp_port, u16 mid_idx, bool add) { struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; - char *smid_pl; + char *smid2_pl; int err; - smid_pl = kmalloc(MLXSW_REG_SMID_LEN, GFP_KERNEL); - if (!smid_pl) + smid2_pl = kmalloc(MLXSW_REG_SMID2_LEN, GFP_KERNEL); + if (!smid2_pl) return -ENOMEM; - mlxsw_reg_smid_pack(smid_pl, mid_idx, mlxsw_sp_port->local_port, add); - err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid), smid_pl); - kfree(smid_pl); + mlxsw_reg_smid2_pack(smid2_pl, mid_idx, mlxsw_sp_port->local_port, add); + err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(smid2), smid2_pl); + kfree(smid2_pl); return err; } @@ -2536,7 +2536,7 @@ static void mlxsw_sp_fdb_notify_mac_process(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_port *mlxsw_sp_port; enum switchdev_notifier_type type; char mac[ETH_ALEN]; - u8 local_port; + u16 local_port; u16 vid, fid; bool do_notification = true; int err; diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c index 26d01adbedad..47b061b99160 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_trap.c @@ -60,7 +60,7 @@ enum { }; static int mlxsw_sp_rx_listener(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, - u8 local_port, + u16 local_port, struct mlxsw_sp_port *mlxsw_sp_port) { struct mlxsw_sp_port_pcpu_stats *pcpu_stats; @@ -85,7 +85,7 @@ static int mlxsw_sp_rx_listener(struct mlxsw_sp *mlxsw_sp, struct sk_buff *skb, return 0; } -static void mlxsw_sp_rx_drop_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_drop_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct devlink_port *in_devlink_port; @@ -109,7 +109,7 @@ static void mlxsw_sp_rx_drop_listener(struct sk_buff *skb, u8 local_port, consume_skb(skb); } -static void mlxsw_sp_rx_acl_drop_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_acl_drop_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { u32 cookie_index = mlxsw_skb_cb(skb)->rx_md_info.cookie_index; @@ -138,7 +138,7 @@ static void mlxsw_sp_rx_acl_drop_listener(struct sk_buff *skb, u8 local_port, consume_skb(skb); } -static int __mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u8 local_port, +static int __mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct devlink_port *in_devlink_port; @@ -164,7 +164,7 @@ static int __mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u8 local_port, return 0; } -static void mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { int err; @@ -176,14 +176,14 @@ static void mlxsw_sp_rx_no_mark_listener(struct sk_buff *skb, u8 local_port, netif_receive_skb(skb); } -static void mlxsw_sp_rx_mark_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_mark_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { skb->offload_fwd_mark = 1; mlxsw_sp_rx_no_mark_listener(skb, local_port, trap_ctx); } -static void mlxsw_sp_rx_l3_mark_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_l3_mark_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { skb->offload_l3_fwd_mark = 1; @@ -191,7 +191,7 @@ static void mlxsw_sp_rx_l3_mark_listener(struct sk_buff *skb, u8 local_port, mlxsw_sp_rx_no_mark_listener(skb, local_port, trap_ctx); } -static void mlxsw_sp_rx_ptp_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_ptp_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct mlxsw_sp *mlxsw_sp = devlink_trap_ctx_priv(trap_ctx); @@ -212,7 +212,7 @@ static struct mlxsw_sp_port * mlxsw_sp_sample_tx_port_get(struct mlxsw_sp *mlxsw_sp, const struct mlxsw_rx_md_info *rx_md_info) { - u8 local_port; + u16 local_port; if (!rx_md_info->tx_port_valid) return NULL; @@ -257,7 +257,7 @@ static void mlxsw_sp_psample_md_init(struct mlxsw_sp *mlxsw_sp, md->latency <<= MLXSW_SP_MIRROR_LATENCY_SHIFT; } -static void mlxsw_sp_rx_sample_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_sample_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct mlxsw_sp *mlxsw_sp = devlink_trap_ctx_priv(trap_ctx); @@ -293,7 +293,7 @@ out: consume_skb(skb); } -static void mlxsw_sp_rx_sample_tx_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_sample_tx_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct mlxsw_rx_md_info *rx_md_info = &mlxsw_skb_cb(skb)->rx_md_info; @@ -343,7 +343,7 @@ out: consume_skb(skb); } -static void mlxsw_sp_rx_sample_acl_listener(struct sk_buff *skb, u8 local_port, +static void mlxsw_sp_rx_sample_acl_listener(struct sk_buff *skb, u16 local_port, void *trap_ctx) { struct mlxsw_sp *mlxsw_sp = devlink_trap_ctx_priv(trap_ctx); diff --git a/drivers/net/ethernet/micrel/ksz884x.c b/drivers/net/ethernet/micrel/ksz884x.c index 99c0c1491af2..d024983815da 100644 --- a/drivers/net/ethernet/micrel/ksz884x.c +++ b/drivers/net/ethernet/micrel/ksz884x.c @@ -6317,11 +6317,15 @@ static int netdev_set_pauseparam(struct net_device *dev, * netdev_get_ringparam - get tx/rx ring parameters * @dev: Network device. * @ring: Ethtool RING settings data structure. + * @kernel_ring: Ethtool external RING settings data structure. + * @extack: Netlink handle. * * This procedure returns the TX/RX ring settings. */ static void netdev_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct dev_priv *priv = netdev_priv(dev); struct dev_info *hw_priv = priv->adapter; diff --git a/drivers/net/ethernet/microchip/Kconfig b/drivers/net/ethernet/microchip/Kconfig index 735eea1dacf1..ed7a35c3ceac 100644 --- a/drivers/net/ethernet/microchip/Kconfig +++ b/drivers/net/ethernet/microchip/Kconfig @@ -55,6 +55,7 @@ config LAN743X To compile this driver as a module, choose M here. The module will be called lan743x. +source "drivers/net/ethernet/microchip/lan966x/Kconfig" source "drivers/net/ethernet/microchip/sparx5/Kconfig" endif # NET_VENDOR_MICROCHIP diff --git a/drivers/net/ethernet/microchip/Makefile b/drivers/net/ethernet/microchip/Makefile index c77dc0379bfd..9faa41436198 100644 --- a/drivers/net/ethernet/microchip/Makefile +++ b/drivers/net/ethernet/microchip/Makefile @@ -9,4 +9,5 @@ obj-$(CONFIG_LAN743X) += lan743x.o lan743x-objs := lan743x_main.o lan743x_ethtool.o lan743x_ptp.o +obj-$(CONFIG_LAN966X_SWITCH) += lan966x/ obj-$(CONFIG_SPARX5_SWITCH) += sparx5/ diff --git a/drivers/net/ethernet/microchip/lan966x/Kconfig b/drivers/net/ethernet/microchip/lan966x/Kconfig new file mode 100644 index 000000000000..2860a8c9923d --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/Kconfig @@ -0,0 +1,8 @@ +config LAN966X_SWITCH + tristate "Lan966x switch driver" + depends on HAS_IOMEM + depends on OF + select PHYLINK + select PACKING + help + This driver supports the Lan966x network switch device. diff --git a/drivers/net/ethernet/microchip/lan966x/Makefile b/drivers/net/ethernet/microchip/lan966x/Makefile new file mode 100644 index 000000000000..2989ba528236 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Makefile for the Microchip Lan966x network device drivers. +# + +obj-$(CONFIG_LAN966X_SWITCH) += lan966x-switch.o + +lan966x-switch-objs := lan966x_main.o lan966x_phylink.o lan966x_port.o \ + lan966x_mac.o lan966x_ethtool.o diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c new file mode 100644 index 000000000000..614f12c2fe6a --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ethtool.c @@ -0,0 +1,682 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/netdevice.h> + +#include "lan966x_main.h" + +/* Number of traffic classes */ +#define LAN966X_NUM_TC 8 +#define LAN966X_STATS_CHECK_DELAY (2 * HZ) + +static const struct lan966x_stat_layout lan966x_stats_layout[] = { + { .name = "rx_octets", .offset = 0x00, }, + { .name = "rx_unicast", .offset = 0x01, }, + { .name = "rx_multicast", .offset = 0x02 }, + { .name = "rx_broadcast", .offset = 0x03 }, + { .name = "rx_short", .offset = 0x04 }, + { .name = "rx_frag", .offset = 0x05 }, + { .name = "rx_jabber", .offset = 0x06 }, + { .name = "rx_crc", .offset = 0x07 }, + { .name = "rx_symbol_err", .offset = 0x08 }, + { .name = "rx_sz_64", .offset = 0x09 }, + { .name = "rx_sz_65_127", .offset = 0x0a}, + { .name = "rx_sz_128_255", .offset = 0x0b}, + { .name = "rx_sz_256_511", .offset = 0x0c }, + { .name = "rx_sz_512_1023", .offset = 0x0d }, + { .name = "rx_sz_1024_1526", .offset = 0x0e }, + { .name = "rx_sz_jumbo", .offset = 0x0f }, + { .name = "rx_pause", .offset = 0x10 }, + { .name = "rx_control", .offset = 0x11 }, + { .name = "rx_long", .offset = 0x12 }, + { .name = "rx_cat_drop", .offset = 0x13 }, + { .name = "rx_red_prio_0", .offset = 0x14 }, + { .name = "rx_red_prio_1", .offset = 0x15 }, + { .name = "rx_red_prio_2", .offset = 0x16 }, + { .name = "rx_red_prio_3", .offset = 0x17 }, + { .name = "rx_red_prio_4", .offset = 0x18 }, + { .name = "rx_red_prio_5", .offset = 0x19 }, + { .name = "rx_red_prio_6", .offset = 0x1a }, + { .name = "rx_red_prio_7", .offset = 0x1b }, + { .name = "rx_yellow_prio_0", .offset = 0x1c }, + { .name = "rx_yellow_prio_1", .offset = 0x1d }, + { .name = "rx_yellow_prio_2", .offset = 0x1e }, + { .name = "rx_yellow_prio_3", .offset = 0x1f }, + { .name = "rx_yellow_prio_4", .offset = 0x20 }, + { .name = "rx_yellow_prio_5", .offset = 0x21 }, + { .name = "rx_yellow_prio_6", .offset = 0x22 }, + { .name = "rx_yellow_prio_7", .offset = 0x23 }, + { .name = "rx_green_prio_0", .offset = 0x24 }, + { .name = "rx_green_prio_1", .offset = 0x25 }, + { .name = "rx_green_prio_2", .offset = 0x26 }, + { .name = "rx_green_prio_3", .offset = 0x27 }, + { .name = "rx_green_prio_4", .offset = 0x28 }, + { .name = "rx_green_prio_5", .offset = 0x29 }, + { .name = "rx_green_prio_6", .offset = 0x2a }, + { .name = "rx_green_prio_7", .offset = 0x2b }, + { .name = "rx_assembly_err", .offset = 0x2c }, + { .name = "rx_smd_err", .offset = 0x2d }, + { .name = "rx_assembly_ok", .offset = 0x2e }, + { .name = "rx_merge_frag", .offset = 0x2f }, + { .name = "rx_pmac_octets", .offset = 0x30, }, + { .name = "rx_pmac_unicast", .offset = 0x31, }, + { .name = "rx_pmac_multicast", .offset = 0x32 }, + { .name = "rx_pmac_broadcast", .offset = 0x33 }, + { .name = "rx_pmac_short", .offset = 0x34 }, + { .name = "rx_pmac_frag", .offset = 0x35 }, + { .name = "rx_pmac_jabber", .offset = 0x36 }, + { .name = "rx_pmac_crc", .offset = 0x37 }, + { .name = "rx_pmac_symbol_err", .offset = 0x38 }, + { .name = "rx_pmac_sz_64", .offset = 0x39 }, + { .name = "rx_pmac_sz_65_127", .offset = 0x3a }, + { .name = "rx_pmac_sz_128_255", .offset = 0x3b }, + { .name = "rx_pmac_sz_256_511", .offset = 0x3c }, + { .name = "rx_pmac_sz_512_1023", .offset = 0x3d }, + { .name = "rx_pmac_sz_1024_1526", .offset = 0x3e }, + { .name = "rx_pmac_sz_jumbo", .offset = 0x3f }, + { .name = "rx_pmac_pause", .offset = 0x40 }, + { .name = "rx_pmac_control", .offset = 0x41 }, + { .name = "rx_pmac_long", .offset = 0x42 }, + + { .name = "tx_octets", .offset = 0x80, }, + { .name = "tx_unicast", .offset = 0x81, }, + { .name = "tx_multicast", .offset = 0x82 }, + { .name = "tx_broadcast", .offset = 0x83 }, + { .name = "tx_col", .offset = 0x84 }, + { .name = "tx_drop", .offset = 0x85 }, + { .name = "tx_pause", .offset = 0x86 }, + { .name = "tx_sz_64", .offset = 0x87 }, + { .name = "tx_sz_65_127", .offset = 0x88 }, + { .name = "tx_sz_128_255", .offset = 0x89 }, + { .name = "tx_sz_256_511", .offset = 0x8a }, + { .name = "tx_sz_512_1023", .offset = 0x8b }, + { .name = "tx_sz_1024_1526", .offset = 0x8c }, + { .name = "tx_sz_jumbo", .offset = 0x8d }, + { .name = "tx_yellow_prio_0", .offset = 0x8e }, + { .name = "tx_yellow_prio_1", .offset = 0x8f }, + { .name = "tx_yellow_prio_2", .offset = 0x90 }, + { .name = "tx_yellow_prio_3", .offset = 0x91 }, + { .name = "tx_yellow_prio_4", .offset = 0x92 }, + { .name = "tx_yellow_prio_5", .offset = 0x93 }, + { .name = "tx_yellow_prio_6", .offset = 0x94 }, + { .name = "tx_yellow_prio_7", .offset = 0x95 }, + { .name = "tx_green_prio_0", .offset = 0x96 }, + { .name = "tx_green_prio_1", .offset = 0x97 }, + { .name = "tx_green_prio_2", .offset = 0x98 }, + { .name = "tx_green_prio_3", .offset = 0x99 }, + { .name = "tx_green_prio_4", .offset = 0x9a }, + { .name = "tx_green_prio_5", .offset = 0x9b }, + { .name = "tx_green_prio_6", .offset = 0x9c }, + { .name = "tx_green_prio_7", .offset = 0x9d }, + { .name = "tx_aged", .offset = 0x9e }, + { .name = "tx_llct", .offset = 0x9f }, + { .name = "tx_ct", .offset = 0xa0 }, + { .name = "tx_mm_hold", .offset = 0xa1 }, + { .name = "tx_merge_frag", .offset = 0xa2 }, + { .name = "tx_pmac_octets", .offset = 0xa3, }, + { .name = "tx_pmac_unicast", .offset = 0xa4, }, + { .name = "tx_pmac_multicast", .offset = 0xa5 }, + { .name = "tx_pmac_broadcast", .offset = 0xa6 }, + { .name = "tx_pmac_pause", .offset = 0xa7 }, + { .name = "tx_pmac_sz_64", .offset = 0xa8 }, + { .name = "tx_pmac_sz_65_127", .offset = 0xa9 }, + { .name = "tx_pmac_sz_128_255", .offset = 0xaa }, + { .name = "tx_pmac_sz_256_511", .offset = 0xab }, + { .name = "tx_pmac_sz_512_1023", .offset = 0xac }, + { .name = "tx_pmac_sz_1024_1526", .offset = 0xad }, + { .name = "tx_pmac_sz_jumbo", .offset = 0xae }, + + { .name = "dr_local", .offset = 0x100 }, + { .name = "dr_tail", .offset = 0x101 }, + { .name = "dr_yellow_prio_0", .offset = 0x102 }, + { .name = "dr_yellow_prio_1", .offset = 0x103 }, + { .name = "dr_yellow_prio_2", .offset = 0x104 }, + { .name = "dr_yellow_prio_3", .offset = 0x105 }, + { .name = "dr_yellow_prio_4", .offset = 0x106 }, + { .name = "dr_yellow_prio_5", .offset = 0x107 }, + { .name = "dr_yellow_prio_6", .offset = 0x108 }, + { .name = "dr_yellow_prio_7", .offset = 0x109 }, + { .name = "dr_green_prio_0", .offset = 0x10a }, + { .name = "dr_green_prio_1", .offset = 0x10b }, + { .name = "dr_green_prio_2", .offset = 0x10c }, + { .name = "dr_green_prio_3", .offset = 0x10d }, + { .name = "dr_green_prio_4", .offset = 0x10e }, + { .name = "dr_green_prio_5", .offset = 0x10f }, + { .name = "dr_green_prio_6", .offset = 0x110 }, + { .name = "dr_green_prio_7", .offset = 0x111 }, +}; + +/* The following numbers are indexes into lan966x_stats_layout[] */ +#define SYS_COUNT_RX_OCT 0 +#define SYS_COUNT_RX_UC 1 +#define SYS_COUNT_RX_MC 2 +#define SYS_COUNT_RX_BC 3 +#define SYS_COUNT_RX_SHORT 4 +#define SYS_COUNT_RX_FRAG 5 +#define SYS_COUNT_RX_JABBER 6 +#define SYS_COUNT_RX_CRC 7 +#define SYS_COUNT_RX_SYMBOL_ERR 8 +#define SYS_COUNT_RX_SZ_64 9 +#define SYS_COUNT_RX_SZ_65_127 10 +#define SYS_COUNT_RX_SZ_128_255 11 +#define SYS_COUNT_RX_SZ_256_511 12 +#define SYS_COUNT_RX_SZ_512_1023 13 +#define SYS_COUNT_RX_SZ_1024_1526 14 +#define SYS_COUNT_RX_SZ_JUMBO 15 +#define SYS_COUNT_RX_PAUSE 16 +#define SYS_COUNT_RX_CONTROL 17 +#define SYS_COUNT_RX_LONG 18 +#define SYS_COUNT_RX_CAT_DROP 19 +#define SYS_COUNT_RX_RED_PRIO_0 20 +#define SYS_COUNT_RX_RED_PRIO_1 21 +#define SYS_COUNT_RX_RED_PRIO_2 22 +#define SYS_COUNT_RX_RED_PRIO_3 23 +#define SYS_COUNT_RX_RED_PRIO_4 24 +#define SYS_COUNT_RX_RED_PRIO_5 25 +#define SYS_COUNT_RX_RED_PRIO_6 26 +#define SYS_COUNT_RX_RED_PRIO_7 27 +#define SYS_COUNT_RX_YELLOW_PRIO_0 28 +#define SYS_COUNT_RX_YELLOW_PRIO_1 29 +#define SYS_COUNT_RX_YELLOW_PRIO_2 30 +#define SYS_COUNT_RX_YELLOW_PRIO_3 31 +#define SYS_COUNT_RX_YELLOW_PRIO_4 32 +#define SYS_COUNT_RX_YELLOW_PRIO_5 33 +#define SYS_COUNT_RX_YELLOW_PRIO_6 34 +#define SYS_COUNT_RX_YELLOW_PRIO_7 35 +#define SYS_COUNT_RX_GREEN_PRIO_0 36 +#define SYS_COUNT_RX_GREEN_PRIO_1 37 +#define SYS_COUNT_RX_GREEN_PRIO_2 38 +#define SYS_COUNT_RX_GREEN_PRIO_3 39 +#define SYS_COUNT_RX_GREEN_PRIO_4 40 +#define SYS_COUNT_RX_GREEN_PRIO_5 41 +#define SYS_COUNT_RX_GREEN_PRIO_6 42 +#define SYS_COUNT_RX_GREEN_PRIO_7 43 +#define SYS_COUNT_RX_ASSEMBLY_ERR 44 +#define SYS_COUNT_RX_SMD_ERR 45 +#define SYS_COUNT_RX_ASSEMBLY_OK 46 +#define SYS_COUNT_RX_MERGE_FRAG 47 +#define SYS_COUNT_RX_PMAC_OCT 48 +#define SYS_COUNT_RX_PMAC_UC 49 +#define SYS_COUNT_RX_PMAC_MC 50 +#define SYS_COUNT_RX_PMAC_BC 51 +#define SYS_COUNT_RX_PMAC_SHORT 52 +#define SYS_COUNT_RX_PMAC_FRAG 53 +#define SYS_COUNT_RX_PMAC_JABBER 54 +#define SYS_COUNT_RX_PMAC_CRC 55 +#define SYS_COUNT_RX_PMAC_SYMBOL_ERR 56 +#define SYS_COUNT_RX_PMAC_SZ_64 57 +#define SYS_COUNT_RX_PMAC_SZ_65_127 58 +#define SYS_COUNT_RX_PMAC_SZ_128_255 59 +#define SYS_COUNT_RX_PMAC_SZ_256_511 60 +#define SYS_COUNT_RX_PMAC_SZ_512_1023 61 +#define SYS_COUNT_RX_PMAC_SZ_1024_1526 62 +#define SYS_COUNT_RX_PMAC_SZ_JUMBO 63 +#define SYS_COUNT_RX_PMAC_PAUSE 64 +#define SYS_COUNT_RX_PMAC_CONTROL 65 +#define SYS_COUNT_RX_PMAC_LONG 66 + +#define SYS_COUNT_TX_OCT 67 +#define SYS_COUNT_TX_UC 68 +#define SYS_COUNT_TX_MC 69 +#define SYS_COUNT_TX_BC 70 +#define SYS_COUNT_TX_COL 71 +#define SYS_COUNT_TX_DROP 72 +#define SYS_COUNT_TX_PAUSE 73 +#define SYS_COUNT_TX_SZ_64 74 +#define SYS_COUNT_TX_SZ_65_127 75 +#define SYS_COUNT_TX_SZ_128_255 76 +#define SYS_COUNT_TX_SZ_256_511 77 +#define SYS_COUNT_TX_SZ_512_1023 78 +#define SYS_COUNT_TX_SZ_1024_1526 79 +#define SYS_COUNT_TX_SZ_JUMBO 80 +#define SYS_COUNT_TX_YELLOW_PRIO_0 81 +#define SYS_COUNT_TX_YELLOW_PRIO_1 82 +#define SYS_COUNT_TX_YELLOW_PRIO_2 83 +#define SYS_COUNT_TX_YELLOW_PRIO_3 84 +#define SYS_COUNT_TX_YELLOW_PRIO_4 85 +#define SYS_COUNT_TX_YELLOW_PRIO_5 86 +#define SYS_COUNT_TX_YELLOW_PRIO_6 87 +#define SYS_COUNT_TX_YELLOW_PRIO_7 88 +#define SYS_COUNT_TX_GREEN_PRIO_0 89 +#define SYS_COUNT_TX_GREEN_PRIO_1 90 +#define SYS_COUNT_TX_GREEN_PRIO_2 91 +#define SYS_COUNT_TX_GREEN_PRIO_3 92 +#define SYS_COUNT_TX_GREEN_PRIO_4 93 +#define SYS_COUNT_TX_GREEN_PRIO_5 94 +#define SYS_COUNT_TX_GREEN_PRIO_6 95 +#define SYS_COUNT_TX_GREEN_PRIO_7 96 +#define SYS_COUNT_TX_AGED 97 +#define SYS_COUNT_TX_LLCT 98 +#define SYS_COUNT_TX_CT 99 +#define SYS_COUNT_TX_MM_HOLD 100 +#define SYS_COUNT_TX_MERGE_FRAG 101 +#define SYS_COUNT_TX_PMAC_OCT 102 +#define SYS_COUNT_TX_PMAC_UC 103 +#define SYS_COUNT_TX_PMAC_MC 104 +#define SYS_COUNT_TX_PMAC_BC 105 +#define SYS_COUNT_TX_PMAC_PAUSE 106 +#define SYS_COUNT_TX_PMAC_SZ_64 107 +#define SYS_COUNT_TX_PMAC_SZ_65_127 108 +#define SYS_COUNT_TX_PMAC_SZ_128_255 109 +#define SYS_COUNT_TX_PMAC_SZ_256_511 110 +#define SYS_COUNT_TX_PMAC_SZ_512_1023 111 +#define SYS_COUNT_TX_PMAC_SZ_1024_1526 112 +#define SYS_COUNT_TX_PMAC_SZ_JUMBO 113 + +#define SYS_COUNT_DR_LOCAL 114 +#define SYS_COUNT_DR_TAIL 115 +#define SYS_COUNT_DR_YELLOW_PRIO_0 116 +#define SYS_COUNT_DR_YELLOW_PRIO_1 117 +#define SYS_COUNT_DR_YELLOW_PRIO_2 118 +#define SYS_COUNT_DR_YELLOW_PRIO_3 119 +#define SYS_COUNT_DR_YELLOW_PRIO_4 120 +#define SYS_COUNT_DR_YELLOW_PRIO_5 121 +#define SYS_COUNT_DR_YELLOW_PRIO_6 122 +#define SYS_COUNT_DR_YELLOW_PRIO_7 123 +#define SYS_COUNT_DR_GREEN_PRIO_0 124 +#define SYS_COUNT_DR_GREEN_PRIO_1 125 +#define SYS_COUNT_DR_GREEN_PRIO_2 126 +#define SYS_COUNT_DR_GREEN_PRIO_3 127 +#define SYS_COUNT_DR_GREEN_PRIO_4 128 +#define SYS_COUNT_DR_GREEN_PRIO_5 129 +#define SYS_COUNT_DR_GREEN_PRIO_6 130 +#define SYS_COUNT_DR_GREEN_PRIO_7 131 + +/* Add a possibly wrapping 32 bit value to a 64 bit counter */ +static void lan966x_add_cnt(u64 *cnt, u32 val) +{ + if (val < (*cnt & U32_MAX)) + *cnt += (u64)1 << 32; /* value has wrapped */ + + *cnt = (*cnt & ~(u64)U32_MAX) + val; +} + +static void lan966x_stats_update(struct lan966x *lan966x) +{ + int i, j; + + mutex_lock(&lan966x->stats_lock); + + for (i = 0; i < lan966x->num_phys_ports; i++) { + uint idx = i * lan966x->num_stats; + + lan_wr(SYS_STAT_CFG_STAT_VIEW_SET(i), + lan966x, SYS_STAT_CFG); + + for (j = 0; j < lan966x->num_stats; j++) { + u32 offset = lan966x->stats_layout[j].offset; + + lan966x_add_cnt(&lan966x->stats[idx++], + lan_rd(lan966x, SYS_CNT(offset))); + } + } + + mutex_unlock(&lan966x->stats_lock); +} + +static int lan966x_get_sset_count(struct net_device *dev, int sset) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + + if (sset != ETH_SS_STATS) + return -EOPNOTSUPP; + + return lan966x->num_stats; +} + +static void lan966x_get_strings(struct net_device *netdev, u32 sset, u8 *data) +{ + struct lan966x_port *port = netdev_priv(netdev); + struct lan966x *lan966x = port->lan966x; + int i; + + if (sset != ETH_SS_STATS) + return; + + for (i = 0; i < lan966x->num_stats; i++) + memcpy(data + i * ETH_GSTRING_LEN, + lan966x->stats_layout[i].name, ETH_GSTRING_LEN); +} + +static void lan966x_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + int i; + + /* check and update now */ + lan966x_stats_update(lan966x); + + /* Copy all counters */ + for (i = 0; i < lan966x->num_stats; i++) + *data++ = lan966x->stats[port->chip_port * + lan966x->num_stats + i]; +} + +static void lan966x_get_eth_mac_stats(struct net_device *dev, + struct ethtool_eth_mac_stats *mac_stats) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + u32 idx; + + lan966x_stats_update(lan966x); + + idx = port->chip_port * lan966x->num_stats; + + mutex_lock(&lan966x->stats_lock); + + mac_stats->FramesTransmittedOK = + lan966x->stats[idx + SYS_COUNT_TX_UC] + + lan966x->stats[idx + SYS_COUNT_TX_MC] + + lan966x->stats[idx + SYS_COUNT_TX_BC] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_UC] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_MC] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_BC]; + mac_stats->SingleCollisionFrames = + lan966x->stats[idx + SYS_COUNT_TX_COL]; + mac_stats->MultipleCollisionFrames = 0; + mac_stats->FramesReceivedOK = + lan966x->stats[idx + SYS_COUNT_RX_UC] + + lan966x->stats[idx + SYS_COUNT_RX_MC] + + lan966x->stats[idx + SYS_COUNT_RX_BC]; + mac_stats->FrameCheckSequenceErrors = + lan966x->stats[idx + SYS_COUNT_RX_CRC] + + lan966x->stats[idx + SYS_COUNT_RX_CRC]; + mac_stats->AlignmentErrors = 0; + mac_stats->OctetsTransmittedOK = + lan966x->stats[idx + SYS_COUNT_TX_OCT] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_OCT]; + mac_stats->FramesWithDeferredXmissions = + lan966x->stats[idx + SYS_COUNT_TX_MM_HOLD]; + mac_stats->LateCollisions = 0; + mac_stats->FramesAbortedDueToXSColls = 0; + mac_stats->FramesLostDueToIntMACXmitError = 0; + mac_stats->CarrierSenseErrors = 0; + mac_stats->OctetsReceivedOK = + lan966x->stats[idx + SYS_COUNT_RX_OCT]; + mac_stats->FramesLostDueToIntMACRcvError = 0; + mac_stats->MulticastFramesXmittedOK = + lan966x->stats[idx + SYS_COUNT_TX_MC] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_MC]; + mac_stats->BroadcastFramesXmittedOK = + lan966x->stats[idx + SYS_COUNT_TX_BC] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_BC]; + mac_stats->FramesWithExcessiveDeferral = 0; + mac_stats->MulticastFramesReceivedOK = + lan966x->stats[idx + SYS_COUNT_RX_MC]; + mac_stats->BroadcastFramesReceivedOK = + lan966x->stats[idx + SYS_COUNT_RX_BC]; + mac_stats->InRangeLengthErrors = + lan966x->stats[idx + SYS_COUNT_RX_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_CRC] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_CRC]; + mac_stats->OutOfRangeLengthField = + lan966x->stats[idx + SYS_COUNT_RX_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_LONG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_LONG]; + mac_stats->FrameTooLongErrors = + lan966x->stats[idx + SYS_COUNT_RX_LONG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_LONG]; + + mutex_unlock(&lan966x->stats_lock); +} + +static const struct ethtool_rmon_hist_range lan966x_rmon_ranges[] = { + { 0, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 10239 }, + {} +}; + +static void lan966x_get_eth_rmon_stats(struct net_device *dev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + u32 idx; + + lan966x_stats_update(lan966x); + + idx = port->chip_port * lan966x->num_stats; + + mutex_lock(&lan966x->stats_lock); + + rmon_stats->undersize_pkts = + lan966x->stats[idx + SYS_COUNT_RX_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SHORT]; + rmon_stats->oversize_pkts = + lan966x->stats[idx + SYS_COUNT_RX_LONG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_LONG]; + rmon_stats->fragments = + lan966x->stats[idx + SYS_COUNT_RX_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_FRAG]; + rmon_stats->jabbers = + lan966x->stats[idx + SYS_COUNT_RX_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_JABBER]; + rmon_stats->hist[0] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_64] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_64]; + rmon_stats->hist[1] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_65_127]; + rmon_stats->hist[2] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_128_255]; + rmon_stats->hist[3] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_256_511]; + rmon_stats->hist[4] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_512_1023]; + rmon_stats->hist[5] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_1024_1526]; + rmon_stats->hist[6] = + lan966x->stats[idx + SYS_COUNT_RX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_1024_1526]; + + rmon_stats->hist_tx[0] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_64] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_64]; + rmon_stats->hist_tx[1] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_65_127]; + rmon_stats->hist_tx[2] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_128_255]; + rmon_stats->hist_tx[3] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_256_511]; + rmon_stats->hist_tx[4] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_512_1023]; + rmon_stats->hist_tx[5] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_1024_1526]; + rmon_stats->hist_tx[6] = + lan966x->stats[idx + SYS_COUNT_TX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_1024_1526]; + + mutex_unlock(&lan966x->stats_lock); + + *ranges = lan966x_rmon_ranges; +} + +static int lan966x_get_link_ksettings(struct net_device *ndev, + struct ethtool_link_ksettings *cmd) +{ + struct lan966x_port *port = netdev_priv(ndev); + + return phylink_ethtool_ksettings_get(port->phylink, cmd); +} + +static int lan966x_set_link_ksettings(struct net_device *ndev, + const struct ethtool_link_ksettings *cmd) +{ + struct lan966x_port *port = netdev_priv(ndev); + + return phylink_ethtool_ksettings_set(port->phylink, cmd); +} + +static void lan966x_get_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct lan966x_port *port = netdev_priv(dev); + + phylink_ethtool_get_pauseparam(port->phylink, pause); +} + +static int lan966x_set_pauseparam(struct net_device *dev, + struct ethtool_pauseparam *pause) +{ + struct lan966x_port *port = netdev_priv(dev); + + return phylink_ethtool_set_pauseparam(port->phylink, pause); +} + +const struct ethtool_ops lan966x_ethtool_ops = { + .get_link_ksettings = lan966x_get_link_ksettings, + .set_link_ksettings = lan966x_set_link_ksettings, + .get_pauseparam = lan966x_get_pauseparam, + .set_pauseparam = lan966x_set_pauseparam, + .get_sset_count = lan966x_get_sset_count, + .get_strings = lan966x_get_strings, + .get_ethtool_stats = lan966x_get_ethtool_stats, + .get_eth_mac_stats = lan966x_get_eth_mac_stats, + .get_rmon_stats = lan966x_get_eth_rmon_stats, + .get_link = ethtool_op_get_link, +}; + +static void lan966x_check_stats_work(struct work_struct *work) +{ + struct delayed_work *del_work = to_delayed_work(work); + struct lan966x *lan966x = container_of(del_work, struct lan966x, + stats_work); + + lan966x_stats_update(lan966x); + + queue_delayed_work(lan966x->stats_queue, &lan966x->stats_work, + LAN966X_STATS_CHECK_DELAY); +} + +void lan966x_stats_get(struct net_device *dev, + struct rtnl_link_stats64 *stats) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + u32 idx; + int i; + + idx = port->chip_port * lan966x->num_stats; + + mutex_lock(&lan966x->stats_lock); + + stats->rx_bytes = lan966x->stats[idx + SYS_COUNT_RX_OCT] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_OCT]; + + stats->rx_packets = lan966x->stats[idx + SYS_COUNT_RX_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_CRC] + + lan966x->stats[idx + SYS_COUNT_RX_SYMBOL_ERR] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_64] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_RX_SZ_JUMBO] + + lan966x->stats[idx + SYS_COUNT_RX_LONG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_64] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_SZ_JUMBO]; + + stats->multicast = lan966x->stats[idx + SYS_COUNT_RX_MC] + + lan966x->stats[idx + SYS_COUNT_RX_PMAC_MC]; + + stats->rx_errors = lan966x->stats[idx + SYS_COUNT_RX_SHORT] + + lan966x->stats[idx + SYS_COUNT_RX_FRAG] + + lan966x->stats[idx + SYS_COUNT_RX_JABBER] + + lan966x->stats[idx + SYS_COUNT_RX_CRC] + + lan966x->stats[idx + SYS_COUNT_RX_SYMBOL_ERR] + + lan966x->stats[idx + SYS_COUNT_RX_LONG]; + + stats->rx_dropped = dev->stats.rx_dropped + + lan966x->stats[idx + SYS_COUNT_RX_LONG] + + lan966x->stats[idx + SYS_COUNT_DR_LOCAL] + + lan966x->stats[idx + SYS_COUNT_DR_TAIL]; + + for (i = 0; i < LAN966X_NUM_TC; i++) { + stats->rx_dropped += + (lan966x->stats[idx + SYS_COUNT_DR_YELLOW_PRIO_0 + i] + + lan966x->stats[idx + SYS_COUNT_DR_GREEN_PRIO_0 + i]); + } + + /* Get Tx stats */ + stats->tx_bytes = lan966x->stats[idx + SYS_COUNT_TX_OCT] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_OCT]; + + stats->tx_packets = lan966x->stats[idx + SYS_COUNT_TX_SZ_64] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_TX_SZ_JUMBO] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_64] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_65_127] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_128_255] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_256_511] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_512_1023] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_1024_1526] + + lan966x->stats[idx + SYS_COUNT_TX_PMAC_SZ_JUMBO]; + + stats->tx_dropped = lan966x->stats[idx + SYS_COUNT_TX_DROP] + + lan966x->stats[idx + SYS_COUNT_TX_AGED]; + + stats->collisions = lan966x->stats[idx + SYS_COUNT_TX_COL]; + + mutex_unlock(&lan966x->stats_lock); +} + +int lan966x_stats_init(struct lan966x *lan966x) +{ + char queue_name[32]; + + lan966x->stats_layout = lan966x_stats_layout; + lan966x->num_stats = ARRAY_SIZE(lan966x_stats_layout); + lan966x->stats = devm_kcalloc(lan966x->dev, lan966x->num_phys_ports * + lan966x->num_stats, + sizeof(u64), GFP_KERNEL); + if (!lan966x->stats) + return -ENOMEM; + + /* Init stats worker */ + mutex_init(&lan966x->stats_lock); + snprintf(queue_name, sizeof(queue_name), "%s-stats", + dev_name(lan966x->dev)); + lan966x->stats_queue = create_singlethread_workqueue(queue_name); + INIT_DELAYED_WORK(&lan966x->stats_work, lan966x_check_stats_work); + queue_delayed_work(lan966x->stats_queue, &lan966x->stats_work, + LAN966X_STATS_CHECK_DELAY); + + return 0; +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h b/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h new file mode 100644 index 000000000000..ca3314789d18 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_ifh.h @@ -0,0 +1,173 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef __LAN966X_IFH_H__ +#define __LAN966X_IFH_H__ + +/* Fields with description (*) should just be cleared upon injection + * IFH is transmitted MSByte first (Highest bit pos sent as MSB of first byte) + */ + +#define IFH_LEN 7 + +/* Timestamp for frame */ +#define IFH_POS_TIMESTAMP 192 + +/* Bypass analyzer with a prefilled IFH */ +#define IFH_POS_BYPASS 191 + +/* Masqueraded injection with masq_port defining logical source port */ +#define IFH_POS_MASQ 190 + +/* Masqueraded port number for injection */ +#define IFH_POS_MASQ_PORT 186 + +/* Frame length (*) */ +#define IFH_POS_LEN 178 + +/* Cell filling mode. Full(0),Etype(1), LlctOpt(2), Llct(3) */ +#define IFH_POS_WRDMODE 176 + +/* Frame has 16 bits rtag removed compared to line data */ +#define IFH_POS_RTAG48 175 + +/* Frame has a redundancy tag */ +#define IFH_POS_HAS_RED_TAG 174 + +/* Frame has been cut through forwarded (*) */ +#define IFH_POS_CUTTHRU 173 + +/* Rewriter command */ +#define IFH_POS_REW_CMD 163 + +/* Enable OAM-related rewriting. PDU_TYPE encodes OAM type. */ +#define IFH_POS_REW_OAM 162 + +/* PDU type. Encoding: (0-NONE, 1-Y1731_CCM, 2-MRP_TST, 3-MRP_ITST, 4-DLR_BCN, + * 5-DLR_ADV, 6-RTE_NULL_INJ, 7-IPV4, 8-IPV6, 9-Y1731_NON_CCM). + */ +#define IFH_POS_PDU_TYPE 158 + +/* Update FCS before transmission */ +#define IFH_POS_FCS_UPD 157 + +/* Classified DSCP value of frame */ +#define IFH_POS_DSCP 151 + +/* Yellow indication */ +#define IFH_POS_DP 150 + +/* Process in RTE/inbound */ +#define IFH_POS_RTE_INB_UPDATE 149 + +/* Number of tags to pop from frame */ +#define IFH_POS_POP_CNT 147 + +/* Number of tags in front of the ethertype */ +#define IFH_POS_ETYPE_OFS 145 + +/* Logical source port of frame (*) */ +#define IFH_POS_SRCPORT 141 + +/* Sequence number in redundancy tag */ +#define IFH_POS_SEQ_NUM 120 + +/* Stagd flag and classified TCI of frame (PCP/DEI/VID) */ +#define IFH_POS_TCI 103 + +/* Classified internal priority for queuing */ +#define IFH_POS_QOS_CLASS 100 + +/* Bit mask with eight cpu copy classses */ +#define IFH_POS_CPUQ 92 + +/* Relearn + learn flags (*) */ +#define IFH_POS_LEARN_FLAGS 90 + +/* SFLOW identifier for frame (0-8: Tx port, 9: Rx sampling, 15: No sampling) */ +#define IFH_POS_SFLOW_ID 86 + +/* Set if an ACL/S2 rule was hit (*). + * Super priority: acl_hit=0 and acl_hit(4)=1. + */ +#define IFH_POS_ACL_HIT 85 + +/* S2 rule index hit (*) */ +#define IFH_POS_ACL_IDX 79 + +/* ISDX as classified by S1 */ +#define IFH_POS_ISDX 71 + +/* Destination ports for frame */ +#define IFH_POS_DSTS 62 + +/* Storm policer to be applied: None/Uni/Multi/Broad (*) */ +#define IFH_POS_FLOOD 60 + +/* Redundancy tag operation */ +#define IFH_POS_SEQ_OP 58 + +/* Classified internal priority for resourcemgt, tagging etc */ +#define IFH_POS_IPV 55 + +/* Frame is for AFI use */ +#define IFH_POS_AFI 54 + +/* Internal aging value (*) */ +#define IFH_POS_AGED 52 + +/* RTP Identifier */ +#define IFH_POS_RTP_ID 42 + +/* RTP MRPD flow */ +#define IFH_POS_RTP_SUBID 41 + +/* Profinet DataStatus or opcua GroupVersion MSB */ +#define IFH_POS_PN_DATA_STATUS 33 + +/* Profinet transfer status (1 iff the status is 0) */ +#define IFH_POS_PN_TRANSF_STATUS_ZERO 32 + +/* Profinet cycle counter or opcua NetworkMessageNumber */ +#define IFH_POS_PN_CC 16 + +#define IFH_WID_TIMESTAMP 32 +#define IFH_WID_BYPASS 1 +#define IFH_WID_MASQ 1 +#define IFH_WID_MASQ_PORT 4 +#define IFH_WID_LEN 14 +#define IFH_WID_WRDMODE 2 +#define IFH_WID_RTAG48 1 +#define IFH_WID_HAS_RED_TAG 1 +#define IFH_WID_CUTTHRU 1 +#define IFH_WID_REW_CMD 10 +#define IFH_WID_REW_OAM 1 +#define IFH_WID_PDU_TYPE 4 +#define IFH_WID_FCS_UPD 1 +#define IFH_WID_DSCP 6 +#define IFH_WID_DP 1 +#define IFH_WID_RTE_INB_UPDATE 1 +#define IFH_WID_POP_CNT 2 +#define IFH_WID_ETYPE_OFS 2 +#define IFH_WID_SRCPORT 4 +#define IFH_WID_SEQ_NUM 16 +#define IFH_WID_TCI 17 +#define IFH_WID_QOS_CLASS 3 +#define IFH_WID_CPUQ 8 +#define IFH_WID_LEARN_FLAGS 2 +#define IFH_WID_SFLOW_ID 4 +#define IFH_WID_ACL_HIT 1 +#define IFH_WID_ACL_IDX 6 +#define IFH_WID_ISDX 8 +#define IFH_WID_DSTS 9 +#define IFH_WID_FLOOD 2 +#define IFH_WID_SEQ_OP 2 +#define IFH_WID_IPV 3 +#define IFH_WID_AFI 1 +#define IFH_WID_AGED 2 +#define IFH_WID_RTP_ID 10 +#define IFH_WID_RTP_SUBID 1 +#define IFH_WID_PN_DATA_STATUS 8 +#define IFH_WID_PN_TRANSF_STATUS_ZERO 1 +#define IFH_WID_PN_CC 16 + +#endif /* __LAN966X_IFH_H__ */ diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c new file mode 100644 index 000000000000..f6878b9f57ef --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_mac.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include "lan966x_main.h" + +#define LAN966X_MAC_COLUMNS 4 +#define MACACCESS_CMD_IDLE 0 +#define MACACCESS_CMD_LEARN 1 +#define MACACCESS_CMD_FORGET 2 +#define MACACCESS_CMD_AGE 3 +#define MACACCESS_CMD_GET_NEXT 4 +#define MACACCESS_CMD_INIT 5 +#define MACACCESS_CMD_READ 6 +#define MACACCESS_CMD_WRITE 7 +#define MACACCESS_CMD_SYNC_GET_NEXT 8 + +static int lan966x_mac_get_status(struct lan966x *lan966x) +{ + return lan_rd(lan966x, ANA_MACACCESS); +} + +static int lan966x_mac_wait_for_completion(struct lan966x *lan966x) +{ + u32 val; + + return readx_poll_timeout(lan966x_mac_get_status, + lan966x, val, + (ANA_MACACCESS_MAC_TABLE_CMD_GET(val)) == + MACACCESS_CMD_IDLE, + TABLE_UPDATE_SLEEP_US, TABLE_UPDATE_TIMEOUT_US); +} + +static void lan966x_mac_select(struct lan966x *lan966x, + const unsigned char mac[ETH_ALEN], + unsigned int vid) +{ + u32 macl = 0, mach = 0; + + /* Set the MAC address to handle and the vlan associated in a format + * understood by the hardware. + */ + mach |= vid << 16; + mach |= mac[0] << 8; + mach |= mac[1] << 0; + macl |= mac[2] << 24; + macl |= mac[3] << 16; + macl |= mac[4] << 8; + macl |= mac[5] << 0; + + lan_wr(macl, lan966x, ANA_MACLDATA); + lan_wr(mach, lan966x, ANA_MACHDATA); +} + +int lan966x_mac_learn(struct lan966x *lan966x, int port, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) +{ + lan966x_mac_select(lan966x, mac, vid); + + /* Issue a write command */ + lan_wr(ANA_MACACCESS_VALID_SET(1) | + ANA_MACACCESS_CHANGE2SW_SET(0) | + ANA_MACACCESS_DEST_IDX_SET(port) | + ANA_MACACCESS_ENTRYTYPE_SET(type) | + ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_LEARN), + lan966x, ANA_MACACCESS); + + return lan966x_mac_wait_for_completion(lan966x); +} + +int lan966x_mac_forget(struct lan966x *lan966x, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type) +{ + lan966x_mac_select(lan966x, mac, vid); + + /* Issue a forget command */ + lan_wr(ANA_MACACCESS_ENTRYTYPE_SET(type) | + ANA_MACACCESS_MAC_TABLE_CMD_SET(MACACCESS_CMD_FORGET), + lan966x, ANA_MACACCESS); + + return lan966x_mac_wait_for_completion(lan966x); +} + +int lan966x_mac_cpu_learn(struct lan966x *lan966x, const char *addr, u16 vid) +{ + return lan966x_mac_learn(lan966x, PGID_CPU, addr, vid, ENTRYTYPE_LOCKED); +} + +int lan966x_mac_cpu_forget(struct lan966x *lan966x, const char *addr, u16 vid) +{ + return lan966x_mac_forget(lan966x, addr, vid, ENTRYTYPE_LOCKED); +} + +void lan966x_mac_init(struct lan966x *lan966x) +{ + /* Clear the MAC table */ + lan_wr(MACACCESS_CMD_INIT, lan966x, ANA_MACACCESS); + lan966x_mac_wait_for_completion(lan966x); +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.c b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c new file mode 100644 index 000000000000..101c1f005baf --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.c @@ -0,0 +1,941 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/module.h> +#include <linux/if_bridge.h> +#include <linux/if_vlan.h> +#include <linux/iopoll.h> +#include <linux/of_platform.h> +#include <linux/of_net.h> +#include <linux/packing.h> +#include <linux/phy/phy.h> +#include <linux/reset.h> + +#include "lan966x_main.h" + +#define XTR_EOF_0 0x00000080U +#define XTR_EOF_1 0x01000080U +#define XTR_EOF_2 0x02000080U +#define XTR_EOF_3 0x03000080U +#define XTR_PRUNED 0x04000080U +#define XTR_ABORT 0x05000080U +#define XTR_ESCAPE 0x06000080U +#define XTR_NOT_READY 0x07000080U +#define XTR_VALID_BYTES(x) (4 - (((x) >> 24) & 3)) + +#define READL_SLEEP_US 10 +#define READL_TIMEOUT_US 100000000 + +#define IO_RANGES 2 + +static const struct of_device_id lan966x_match[] = { + { .compatible = "microchip,lan966x-switch" }, + { } +}; +MODULE_DEVICE_TABLE(of, lan966x_match); + +struct lan966x_main_io_resource { + enum lan966x_target id; + phys_addr_t offset; + int range; +}; + +static const struct lan966x_main_io_resource lan966x_main_iomap[] = { + { TARGET_CPU, 0xc0000, 0 }, /* 0xe00c0000 */ + { TARGET_ORG, 0, 1 }, /* 0xe2000000 */ + { TARGET_GCB, 0x4000, 1 }, /* 0xe2004000 */ + { TARGET_QS, 0x8000, 1 }, /* 0xe2008000 */ + { TARGET_CHIP_TOP, 0x10000, 1 }, /* 0xe2010000 */ + { TARGET_REW, 0x14000, 1 }, /* 0xe2014000 */ + { TARGET_SYS, 0x28000, 1 }, /* 0xe2028000 */ + { TARGET_DEV, 0x34000, 1 }, /* 0xe2034000 */ + { TARGET_DEV + 1, 0x38000, 1 }, /* 0xe2038000 */ + { TARGET_DEV + 2, 0x3c000, 1 }, /* 0xe203c000 */ + { TARGET_DEV + 3, 0x40000, 1 }, /* 0xe2040000 */ + { TARGET_DEV + 4, 0x44000, 1 }, /* 0xe2044000 */ + { TARGET_DEV + 5, 0x48000, 1 }, /* 0xe2048000 */ + { TARGET_DEV + 6, 0x4c000, 1 }, /* 0xe204c000 */ + { TARGET_DEV + 7, 0x50000, 1 }, /* 0xe2050000 */ + { TARGET_QSYS, 0x100000, 1 }, /* 0xe2100000 */ + { TARGET_AFI, 0x120000, 1 }, /* 0xe2120000 */ + { TARGET_ANA, 0x140000, 1 }, /* 0xe2140000 */ +}; + +static int lan966x_create_targets(struct platform_device *pdev, + struct lan966x *lan966x) +{ + struct resource *iores[IO_RANGES]; + void __iomem *begin[IO_RANGES]; + int idx; + + /* Initially map the entire range and after that update each target to + * point inside the region at the correct offset. It is possible that + * other devices access the same region so don't add any checks about + * this. + */ + for (idx = 0; idx < IO_RANGES; idx++) { + iores[idx] = platform_get_resource(pdev, IORESOURCE_MEM, + idx); + if (!iores[idx]) { + dev_err(&pdev->dev, "Invalid resource\n"); + return -EINVAL; + } + + begin[idx] = devm_ioremap(&pdev->dev, + iores[idx]->start, + resource_size(iores[idx])); + if (!begin[idx]) { + dev_err(&pdev->dev, "Unable to get registers: %s\n", + iores[idx]->name); + return -ENOMEM; + } + } + + for (idx = 0; idx < ARRAY_SIZE(lan966x_main_iomap); idx++) { + const struct lan966x_main_io_resource *iomap = + &lan966x_main_iomap[idx]; + + lan966x->regs[iomap->id] = begin[iomap->range] + iomap->offset; + } + + return 0; +} + +static int lan966x_port_set_mac_address(struct net_device *dev, void *p) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + const struct sockaddr *addr = p; + int ret; + + /* Learn the new net device MAC address in the mac table. */ + ret = lan966x_mac_cpu_learn(lan966x, addr->sa_data, port->pvid); + if (ret) + return ret; + + /* Then forget the previous one. */ + ret = lan966x_mac_cpu_forget(lan966x, dev->dev_addr, port->pvid); + if (ret) + return ret; + + eth_hw_addr_set(dev, addr->sa_data); + return ret; +} + +static int lan966x_port_get_phys_port_name(struct net_device *dev, + char *buf, size_t len) +{ + struct lan966x_port *port = netdev_priv(dev); + int ret; + + ret = snprintf(buf, len, "p%d", port->chip_port); + if (ret >= len) + return -EINVAL; + + return 0; +} + +static int lan966x_port_open(struct net_device *dev) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + int err; + + /* Enable receiving frames on the port, and activate auto-learning of + * MAC addresses. + */ + lan_rmw(ANA_PORT_CFG_LEARNAUTO_SET(1) | + ANA_PORT_CFG_RECV_ENA_SET(1) | + ANA_PORT_CFG_PORTID_VAL_SET(port->chip_port), + ANA_PORT_CFG_LEARNAUTO | + ANA_PORT_CFG_RECV_ENA | + ANA_PORT_CFG_PORTID_VAL, + lan966x, ANA_PORT_CFG(port->chip_port)); + + err = phylink_fwnode_phy_connect(port->phylink, port->fwnode, 0); + if (err) { + netdev_err(dev, "Could not attach to PHY\n"); + return err; + } + + phylink_start(port->phylink); + + return 0; +} + +static int lan966x_port_stop(struct net_device *dev) +{ + struct lan966x_port *port = netdev_priv(dev); + + lan966x_port_config_down(port); + phylink_stop(port->phylink); + phylink_disconnect_phy(port->phylink); + + return 0; +} + +static int lan966x_port_inj_status(struct lan966x *lan966x) +{ + return lan_rd(lan966x, QS_INJ_STATUS); +} + +static int lan966x_port_inj_ready(struct lan966x *lan966x, u8 grp) +{ + u32 val; + + return readx_poll_timeout(lan966x_port_inj_status, lan966x, val, + QS_INJ_STATUS_FIFO_RDY_GET(val) & BIT(grp), + READL_SLEEP_US, READL_TIMEOUT_US); +} + +static int lan966x_port_ifh_xmit(struct sk_buff *skb, + __be32 *ifh, + struct net_device *dev) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + u32 i, count, last; + u8 grp = 0; + u32 val; + int err; + + val = lan_rd(lan966x, QS_INJ_STATUS); + if (!(QS_INJ_STATUS_FIFO_RDY_GET(val) & BIT(grp)) || + (QS_INJ_STATUS_WMARK_REACHED_GET(val) & BIT(grp))) + return NETDEV_TX_BUSY; + + /* Write start of frame */ + lan_wr(QS_INJ_CTRL_GAP_SIZE_SET(1) | + QS_INJ_CTRL_SOF_SET(1), + lan966x, QS_INJ_CTRL(grp)); + + /* Write IFH header */ + for (i = 0; i < IFH_LEN; ++i) { + /* Wait until the fifo is ready */ + err = lan966x_port_inj_ready(lan966x, grp); + if (err) + return NETDEV_TX_BUSY; + + lan_wr((__force u32)ifh[i], lan966x, QS_INJ_WR(grp)); + } + + /* Write frame */ + count = DIV_ROUND_UP(skb->len, 4); + last = skb->len % 4; + for (i = 0; i < count; ++i) { + /* Wait until the fifo is ready */ + err = lan966x_port_inj_ready(lan966x, grp); + if (err) + return NETDEV_TX_BUSY; + + lan_wr(((u32 *)skb->data)[i], lan966x, QS_INJ_WR(grp)); + } + + /* Add padding */ + while (i < (LAN966X_BUFFER_MIN_SZ / 4)) { + /* Wait until the fifo is ready */ + err = lan966x_port_inj_ready(lan966x, grp); + if (err) + return NETDEV_TX_BUSY; + + lan_wr(0, lan966x, QS_INJ_WR(grp)); + ++i; + } + + /* Inidcate EOF and valid bytes in the last word */ + lan_wr(QS_INJ_CTRL_GAP_SIZE_SET(1) | + QS_INJ_CTRL_VLD_BYTES_SET(skb->len < LAN966X_BUFFER_MIN_SZ ? + 0 : last) | + QS_INJ_CTRL_EOF_SET(1), + lan966x, QS_INJ_CTRL(grp)); + + /* Add dummy CRC */ + lan_wr(0, lan966x, QS_INJ_WR(grp)); + skb_tx_timestamp(skb); + + dev->stats.tx_packets++; + dev->stats.tx_bytes += skb->len; + + dev_consume_skb_any(skb); + return NETDEV_TX_OK; +} + +static void lan966x_ifh_set_bypass(void *ifh, u64 bypass) +{ + packing(ifh, &bypass, IFH_POS_BYPASS + IFH_WID_BYPASS - 1, + IFH_POS_BYPASS, IFH_LEN * 4, PACK, 0); +} + +static void lan966x_ifh_set_port(void *ifh, u64 bypass) +{ + packing(ifh, &bypass, IFH_POS_DSTS + IFH_WID_DSTS - 1, + IFH_POS_DSTS, IFH_LEN * 4, PACK, 0); +} + +static void lan966x_ifh_set_qos_class(void *ifh, u64 bypass) +{ + packing(ifh, &bypass, IFH_POS_QOS_CLASS + IFH_WID_QOS_CLASS - 1, + IFH_POS_QOS_CLASS, IFH_LEN * 4, PACK, 0); +} + +static void lan966x_ifh_set_ipv(void *ifh, u64 bypass) +{ + packing(ifh, &bypass, IFH_POS_IPV + IFH_WID_IPV - 1, + IFH_POS_IPV, IFH_LEN * 4, PACK, 0); +} + +static int lan966x_port_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct lan966x_port *port = netdev_priv(dev); + __be32 ifh[IFH_LEN]; + + memset(ifh, 0x0, sizeof(__be32) * IFH_LEN); + + lan966x_ifh_set_bypass(ifh, 1); + lan966x_ifh_set_port(ifh, BIT_ULL(port->chip_port)); + lan966x_ifh_set_qos_class(ifh, skb->priority >= 7 ? 0x7 : skb->priority); + lan966x_ifh_set_ipv(ifh, skb->priority >= 7 ? 0x7 : skb->priority); + + return lan966x_port_ifh_xmit(skb, ifh, dev); +} + +static void lan966x_set_promisc(struct lan966x_port *port, bool enable) +{ + struct lan966x *lan966x = port->lan966x; + + lan_rmw(ANA_CPU_FWD_CFG_SRC_COPY_ENA_SET(enable), + ANA_CPU_FWD_CFG_SRC_COPY_ENA, + lan966x, ANA_CPU_FWD_CFG(port->chip_port)); +} + +static void lan966x_port_change_rx_flags(struct net_device *dev, int flags) +{ + struct lan966x_port *port = netdev_priv(dev); + + if (!(flags & IFF_PROMISC)) + return; + + if (dev->flags & IFF_PROMISC) + lan966x_set_promisc(port, true); + else + lan966x_set_promisc(port, false); +} + +static int lan966x_port_change_mtu(struct net_device *dev, int new_mtu) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + + lan_wr(DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(new_mtu), + lan966x, DEV_MAC_MAXLEN_CFG(port->chip_port)); + dev->mtu = new_mtu; + + return 0; +} + +static int lan966x_mc_unsync(struct net_device *dev, const unsigned char *addr) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + + return lan966x_mac_forget(lan966x, addr, port->pvid, ENTRYTYPE_LOCKED); +} + +static int lan966x_mc_sync(struct net_device *dev, const unsigned char *addr) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + + return lan966x_mac_cpu_learn(lan966x, addr, port->pvid); +} + +static void lan966x_port_set_rx_mode(struct net_device *dev) +{ + __dev_mc_sync(dev, lan966x_mc_sync, lan966x_mc_unsync); +} + +static int lan966x_port_get_parent_id(struct net_device *dev, + struct netdev_phys_item_id *ppid) +{ + struct lan966x_port *port = netdev_priv(dev); + struct lan966x *lan966x = port->lan966x; + + ppid->id_len = sizeof(lan966x->base_mac); + memcpy(&ppid->id, &lan966x->base_mac, ppid->id_len); + + return 0; +} + +static const struct net_device_ops lan966x_port_netdev_ops = { + .ndo_open = lan966x_port_open, + .ndo_stop = lan966x_port_stop, + .ndo_start_xmit = lan966x_port_xmit, + .ndo_change_rx_flags = lan966x_port_change_rx_flags, + .ndo_change_mtu = lan966x_port_change_mtu, + .ndo_set_rx_mode = lan966x_port_set_rx_mode, + .ndo_get_phys_port_name = lan966x_port_get_phys_port_name, + .ndo_get_stats64 = lan966x_stats_get, + .ndo_set_mac_address = lan966x_port_set_mac_address, + .ndo_get_port_parent_id = lan966x_port_get_parent_id, +}; + +static int lan966x_port_xtr_status(struct lan966x *lan966x, u8 grp) +{ + return lan_rd(lan966x, QS_XTR_RD(grp)); +} + +static int lan966x_port_xtr_ready(struct lan966x *lan966x, u8 grp) +{ + u32 val; + + return read_poll_timeout(lan966x_port_xtr_status, val, + val != XTR_NOT_READY, + READL_SLEEP_US, READL_TIMEOUT_US, false, + lan966x, grp); +} + +static int lan966x_rx_frame_word(struct lan966x *lan966x, u8 grp, u32 *rval) +{ + u32 bytes_valid; + u32 val; + int err; + + val = lan_rd(lan966x, QS_XTR_RD(grp)); + if (val == XTR_NOT_READY) { + err = lan966x_port_xtr_ready(lan966x, grp); + if (err) + return -EIO; + } + + switch (val) { + case XTR_ABORT: + return -EIO; + case XTR_EOF_0: + case XTR_EOF_1: + case XTR_EOF_2: + case XTR_EOF_3: + case XTR_PRUNED: + bytes_valid = XTR_VALID_BYTES(val); + val = lan_rd(lan966x, QS_XTR_RD(grp)); + if (val == XTR_ESCAPE) + *rval = lan_rd(lan966x, QS_XTR_RD(grp)); + else + *rval = val; + + return bytes_valid; + case XTR_ESCAPE: + *rval = lan_rd(lan966x, QS_XTR_RD(grp)); + + return 4; + default: + *rval = val; + + return 4; + } +} + +static void lan966x_ifh_get_src_port(void *ifh, u64 *src_port) +{ + packing(ifh, src_port, IFH_POS_SRCPORT + IFH_WID_SRCPORT - 1, + IFH_POS_SRCPORT, IFH_LEN * 4, UNPACK, 0); +} + +static void lan966x_ifh_get_len(void *ifh, u64 *len) +{ + packing(ifh, len, IFH_POS_LEN + IFH_WID_LEN - 1, + IFH_POS_LEN, IFH_LEN * 4, UNPACK, 0); +} + +static irqreturn_t lan966x_xtr_irq_handler(int irq, void *args) +{ + struct lan966x *lan966x = args; + int i, grp = 0, err = 0; + + if (!(lan_rd(lan966x, QS_XTR_DATA_PRESENT) & BIT(grp))) + return IRQ_NONE; + + do { + struct net_device *dev; + struct sk_buff *skb; + int sz = 0, buf_len; + u64 src_port, len; + u32 ifh[IFH_LEN]; + u32 *buf; + u32 val; + + for (i = 0; i < IFH_LEN; i++) { + err = lan966x_rx_frame_word(lan966x, grp, &ifh[i]); + if (err != 4) + goto recover; + } + + err = 0; + + lan966x_ifh_get_src_port(ifh, &src_port); + lan966x_ifh_get_len(ifh, &len); + + WARN_ON(src_port >= lan966x->num_phys_ports); + + dev = lan966x->ports[src_port]->dev; + skb = netdev_alloc_skb(dev, len); + if (unlikely(!skb)) { + netdev_err(dev, "Unable to allocate sk_buff\n"); + err = -ENOMEM; + break; + } + buf_len = len - ETH_FCS_LEN; + buf = (u32 *)skb_put(skb, buf_len); + + len = 0; + do { + sz = lan966x_rx_frame_word(lan966x, grp, &val); + if (sz < 0) { + kfree_skb(skb); + goto recover; + } + + *buf++ = val; + len += sz; + } while (len < buf_len); + + /* Read the FCS */ + sz = lan966x_rx_frame_word(lan966x, grp, &val); + if (sz < 0) { + kfree_skb(skb); + goto recover; + } + + /* Update the statistics if part of the FCS was read before */ + len -= ETH_FCS_LEN - sz; + + if (unlikely(dev->features & NETIF_F_RXFCS)) { + buf = (u32 *)skb_put(skb, ETH_FCS_LEN); + *buf = val; + } + + skb->protocol = eth_type_trans(skb, dev); + + netif_rx_ni(skb); + dev->stats.rx_bytes += len; + dev->stats.rx_packets++; + +recover: + if (sz < 0 || err) + lan_rd(lan966x, QS_XTR_RD(grp)); + + } while (lan_rd(lan966x, QS_XTR_DATA_PRESENT) & BIT(grp)); + + return IRQ_HANDLED; +} + +static void lan966x_cleanup_ports(struct lan966x *lan966x) +{ + struct lan966x_port *port; + int p; + + for (p = 0; p < lan966x->num_phys_ports; p++) { + port = lan966x->ports[p]; + if (!port) + continue; + + if (port->dev) + unregister_netdev(port->dev); + + if (port->phylink) { + rtnl_lock(); + lan966x_port_stop(port->dev); + rtnl_unlock(); + phylink_destroy(port->phylink); + port->phylink = NULL; + } + + if (port->fwnode) + fwnode_handle_put(port->fwnode); + } + + disable_irq(lan966x->xtr_irq); + lan966x->xtr_irq = -ENXIO; +} + +static int lan966x_probe_port(struct lan966x *lan966x, u32 p, + phy_interface_t phy_mode, + struct fwnode_handle *portnp) +{ + struct lan966x_port *port; + struct phylink *phylink; + struct net_device *dev; + int err; + + if (p >= lan966x->num_phys_ports) + return -EINVAL; + + dev = devm_alloc_etherdev_mqs(lan966x->dev, + sizeof(struct lan966x_port), 8, 1); + if (!dev) + return -ENOMEM; + + SET_NETDEV_DEV(dev, lan966x->dev); + port = netdev_priv(dev); + port->dev = dev; + port->lan966x = lan966x; + port->chip_port = p; + port->pvid = PORT_PVID; + lan966x->ports[p] = port; + + dev->max_mtu = ETH_MAX_MTU; + + dev->netdev_ops = &lan966x_port_netdev_ops; + dev->ethtool_ops = &lan966x_ethtool_ops; + dev->needed_headroom = IFH_LEN * sizeof(u32); + + eth_hw_addr_gen(dev, lan966x->base_mac, p + 1); + + lan966x_mac_learn(lan966x, PGID_CPU, dev->dev_addr, port->pvid, + ENTRYTYPE_LOCKED); + + port->phylink_config.dev = &port->dev->dev; + port->phylink_config.type = PHYLINK_NETDEV; + port->phylink_pcs.poll = true; + port->phylink_pcs.ops = &lan966x_phylink_pcs_ops; + + port->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000FD | MAC_2500FD; + + __set_bit(PHY_INTERFACE_MODE_MII, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_GMII, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_QSGMII, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + port->phylink_config.supported_interfaces); + + phylink = phylink_create(&port->phylink_config, + portnp, + phy_mode, + &lan966x_phylink_mac_ops); + if (IS_ERR(phylink)) { + port->dev = NULL; + return PTR_ERR(phylink); + } + + port->phylink = phylink; + phylink_set_pcs(phylink, &port->phylink_pcs); + + err = register_netdev(dev); + if (err) { + dev_err(lan966x->dev, "register_netdev failed\n"); + return err; + } + + return 0; +} + +static void lan966x_init(struct lan966x *lan966x) +{ + u32 p, i; + + /* MAC table initialization */ + lan966x_mac_init(lan966x); + + /* Flush queues */ + lan_wr(lan_rd(lan966x, QS_XTR_FLUSH) | + GENMASK(1, 0), + lan966x, QS_XTR_FLUSH); + + /* Allow to drain */ + mdelay(1); + + /* All Queues normal */ + lan_wr(lan_rd(lan966x, QS_XTR_FLUSH) & + ~(GENMASK(1, 0)), + lan966x, QS_XTR_FLUSH); + + /* Set MAC age time to default value, the entry is aged after + * 2 * AGE_PERIOD + */ + lan_wr(ANA_AUTOAGE_AGE_PERIOD_SET(BR_DEFAULT_AGEING_TIME / 2 / HZ), + lan966x, ANA_AUTOAGE); + + /* Disable learning for frames discarded by VLAN ingress filtering */ + lan_rmw(ANA_ADVLEARN_VLAN_CHK_SET(1), + ANA_ADVLEARN_VLAN_CHK, + lan966x, ANA_ADVLEARN); + + /* Setup frame ageing - "2 sec" - The unit is 6.5 us on lan966x */ + lan_wr(SYS_FRM_AGING_AGE_TX_ENA_SET(1) | + (20000000 / 65), + lan966x, SYS_FRM_AGING); + + /* Map the 8 CPU extraction queues to CPU port */ + lan_wr(0, lan966x, QSYS_CPU_GROUP_MAP); + + /* Do byte-swap and expect status after last data word + * Extraction: Mode: manual extraction) | Byte_swap + */ + lan_wr(QS_XTR_GRP_CFG_MODE_SET(1) | + QS_XTR_GRP_CFG_BYTE_SWAP_SET(1), + lan966x, QS_XTR_GRP_CFG(0)); + + /* Injection: Mode: manual injection | Byte_swap */ + lan_wr(QS_INJ_GRP_CFG_MODE_SET(1) | + QS_INJ_GRP_CFG_BYTE_SWAP_SET(1), + lan966x, QS_INJ_GRP_CFG(0)); + + lan_rmw(QS_INJ_CTRL_GAP_SIZE_SET(0), + QS_INJ_CTRL_GAP_SIZE, + lan966x, QS_INJ_CTRL(0)); + + /* Enable IFH insertion/parsing on CPU ports */ + lan_wr(SYS_PORT_MODE_INCL_INJ_HDR_SET(1) | + SYS_PORT_MODE_INCL_XTR_HDR_SET(1), + lan966x, SYS_PORT_MODE(CPU_PORT)); + + /* Setup flooding PGIDs */ + lan_wr(ANA_FLOODING_IPMC_FLD_MC4_DATA_SET(PGID_MCIPV4) | + ANA_FLOODING_IPMC_FLD_MC4_CTRL_SET(PGID_MC) | + ANA_FLOODING_IPMC_FLD_MC6_DATA_SET(PGID_MC) | + ANA_FLOODING_IPMC_FLD_MC6_CTRL_SET(PGID_MC), + lan966x, ANA_FLOODING_IPMC); + + /* There are 8 priorities */ + for (i = 0; i < 8; ++i) + lan_rmw(ANA_FLOODING_FLD_MULTICAST_SET(PGID_MC) | + ANA_FLOODING_FLD_BROADCAST_SET(PGID_BC), + ANA_FLOODING_FLD_MULTICAST | + ANA_FLOODING_FLD_BROADCAST, + lan966x, ANA_FLOODING(i)); + + for (i = 0; i < PGID_ENTRIES; ++i) + /* Set all the entries to obey VLAN_VLAN */ + lan_rmw(ANA_PGID_CFG_OBEY_VLAN_SET(1), + ANA_PGID_CFG_OBEY_VLAN, + lan966x, ANA_PGID_CFG(i)); + + for (p = 0; p < lan966x->num_phys_ports; p++) { + /* Disable bridging by default */ + lan_rmw(ANA_PGID_PGID_SET(0x0), + ANA_PGID_PGID, + lan966x, ANA_PGID(p + PGID_SRC)); + + /* Do not forward BPDU frames to the front ports and copy them + * to CPU + */ + lan_wr(0xffff, lan966x, ANA_CPU_FWD_BPDU_CFG(p)); + } + + /* Set source buffer size for each priority and each port to 1500 bytes */ + for (i = 0; i <= QSYS_Q_RSRV; ++i) { + lan_wr(1500 / 64, lan966x, QSYS_RES_CFG(i)); + lan_wr(1500 / 64, lan966x, QSYS_RES_CFG(512 + i)); + } + + /* Enable switching to/from cpu port */ + lan_wr(QSYS_SW_PORT_MODE_PORT_ENA_SET(1) | + QSYS_SW_PORT_MODE_SCH_NEXT_CFG_SET(1) | + QSYS_SW_PORT_MODE_INGRESS_DROP_MODE_SET(1), + lan966x, QSYS_SW_PORT_MODE(CPU_PORT)); + + /* Configure and enable the CPU port */ + lan_rmw(ANA_PGID_PGID_SET(0), + ANA_PGID_PGID, + lan966x, ANA_PGID(CPU_PORT)); + lan_rmw(ANA_PGID_PGID_SET(BIT(CPU_PORT)), + ANA_PGID_PGID, + lan966x, ANA_PGID(PGID_CPU)); + + /* Multicast to all other ports */ + lan_rmw(GENMASK(lan966x->num_phys_ports - 1, 0), + ANA_PGID_PGID, + lan966x, ANA_PGID(PGID_MC)); + + /* This will be controlled by mrouter ports */ + lan_rmw(GENMASK(lan966x->num_phys_ports - 1, 0), + ANA_PGID_PGID, + lan966x, ANA_PGID(PGID_MCIPV4)); + + /* Broadcast to the CPU port and to other ports */ + lan_rmw(ANA_PGID_PGID_SET(BIT(CPU_PORT) | GENMASK(lan966x->num_phys_ports - 1, 0)), + ANA_PGID_PGID, + lan966x, ANA_PGID(PGID_BC)); + + lan_wr(REW_PORT_CFG_NO_REWRITE_SET(1), + lan966x, REW_PORT_CFG(CPU_PORT)); + + lan_rmw(ANA_ANAINTR_INTR_ENA_SET(1), + ANA_ANAINTR_INTR_ENA, + lan966x, ANA_ANAINTR); +} + +static int lan966x_ram_init(struct lan966x *lan966x) +{ + return lan_rd(lan966x, SYS_RAM_INIT); +} + +static int lan966x_reset_switch(struct lan966x *lan966x) +{ + struct reset_control *switch_reset, *phy_reset; + int val = 0; + int ret; + + switch_reset = devm_reset_control_get_shared(lan966x->dev, "switch"); + if (IS_ERR(switch_reset)) + return dev_err_probe(lan966x->dev, PTR_ERR(switch_reset), + "Could not obtain switch reset"); + + phy_reset = devm_reset_control_get_shared(lan966x->dev, "phy"); + if (IS_ERR(phy_reset)) + return dev_err_probe(lan966x->dev, PTR_ERR(phy_reset), + "Could not obtain phy reset\n"); + + reset_control_reset(switch_reset); + reset_control_reset(phy_reset); + + lan_wr(SYS_RESET_CFG_CORE_ENA_SET(0), lan966x, SYS_RESET_CFG); + lan_wr(SYS_RAM_INIT_RAM_INIT_SET(1), lan966x, SYS_RAM_INIT); + ret = readx_poll_timeout(lan966x_ram_init, lan966x, + val, (val & BIT(1)) == 0, READL_SLEEP_US, + READL_TIMEOUT_US); + if (ret) + return ret; + + lan_wr(SYS_RESET_CFG_CORE_ENA_SET(1), lan966x, SYS_RESET_CFG); + + return 0; +} + +static int lan966x_probe(struct platform_device *pdev) +{ + struct fwnode_handle *ports, *portnp; + struct lan966x *lan966x; + u8 mac_addr[ETH_ALEN]; + int err, i; + + lan966x = devm_kzalloc(&pdev->dev, sizeof(*lan966x), GFP_KERNEL); + if (!lan966x) + return -ENOMEM; + + platform_set_drvdata(pdev, lan966x); + lan966x->dev = &pdev->dev; + + if (!device_get_mac_address(&pdev->dev, mac_addr)) { + ether_addr_copy(lan966x->base_mac, mac_addr); + } else { + pr_info("MAC addr was not set, use random MAC\n"); + eth_random_addr(lan966x->base_mac); + lan966x->base_mac[5] &= 0xf0; + } + + ports = device_get_named_child_node(&pdev->dev, "ethernet-ports"); + if (!ports) + return dev_err_probe(&pdev->dev, -ENODEV, + "no ethernet-ports child found\n"); + + err = lan966x_create_targets(pdev, lan966x); + if (err) + return dev_err_probe(&pdev->dev, err, + "Failed to create targets"); + + err = lan966x_reset_switch(lan966x); + if (err) + return dev_err_probe(&pdev->dev, err, "Reset failed"); + + i = 0; + fwnode_for_each_available_child_node(ports, portnp) + ++i; + + lan966x->num_phys_ports = i; + lan966x->ports = devm_kcalloc(&pdev->dev, lan966x->num_phys_ports, + sizeof(struct lan966x_port *), + GFP_KERNEL); + if (!lan966x->ports) + return -ENOMEM; + + /* There QS system has 32KB of memory */ + lan966x->shared_queue_sz = LAN966X_BUFFER_MEMORY; + + /* set irq */ + lan966x->xtr_irq = platform_get_irq_byname(pdev, "xtr"); + if (lan966x->xtr_irq <= 0) + return -EINVAL; + + err = devm_request_threaded_irq(&pdev->dev, lan966x->xtr_irq, NULL, + lan966x_xtr_irq_handler, IRQF_ONESHOT, + "frame extraction", lan966x); + if (err) { + pr_err("Unable to use xtr irq"); + return -ENODEV; + } + + /* init switch */ + lan966x_init(lan966x); + lan966x_stats_init(lan966x); + + /* go over the child nodes */ + fwnode_for_each_available_child_node(ports, portnp) { + phy_interface_t phy_mode; + struct phy *serdes; + u32 p; + + if (fwnode_property_read_u32(portnp, "reg", &p)) + continue; + + phy_mode = fwnode_get_phy_mode(portnp); + err = lan966x_probe_port(lan966x, p, phy_mode, portnp); + if (err) + goto cleanup_ports; + + /* Read needed configuration */ + lan966x->ports[p]->config.portmode = phy_mode; + lan966x->ports[p]->fwnode = fwnode_handle_get(portnp); + + serdes = devm_of_phy_get(lan966x->dev, to_of_node(portnp), NULL); + if (!IS_ERR(serdes)) + lan966x->ports[p]->serdes = serdes; + + lan966x_port_init(lan966x->ports[p]); + } + + return 0; + +cleanup_ports: + fwnode_handle_put(portnp); + + lan966x_cleanup_ports(lan966x); + + cancel_delayed_work_sync(&lan966x->stats_work); + destroy_workqueue(lan966x->stats_queue); + mutex_destroy(&lan966x->stats_lock); + + return err; +} + +static int lan966x_remove(struct platform_device *pdev) +{ + struct lan966x *lan966x = platform_get_drvdata(pdev); + + lan966x_cleanup_ports(lan966x); + + cancel_delayed_work_sync(&lan966x->stats_work); + destroy_workqueue(lan966x->stats_queue); + mutex_destroy(&lan966x->stats_lock); + + return 0; +} + +static struct platform_driver lan966x_driver = { + .probe = lan966x_probe, + .remove = lan966x_remove, + .driver = { + .name = "lan966x-switch", + .of_match_table = lan966x_match, + }, +}; +module_platform_driver(lan966x_driver); + +MODULE_DESCRIPTION("Microchip LAN966X switch driver"); +MODULE_AUTHOR("Horatiu Vultur <horatiu.vultur@microchip.com>"); +MODULE_LICENSE("Dual MIT/GPL"); diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_main.h b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h new file mode 100644 index 000000000000..7e5a3b6f168d --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_main.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ + +#ifndef __LAN966X_MAIN_H__ +#define __LAN966X_MAIN_H__ + +#include <linux/etherdevice.h> +#include <linux/jiffies.h> +#include <linux/phy.h> +#include <linux/phylink.h> + +#include "lan966x_regs.h" +#include "lan966x_ifh.h" + +#define TABLE_UPDATE_SLEEP_US 10 +#define TABLE_UPDATE_TIMEOUT_US 100000 + +#define LAN966X_BUFFER_CELL_SZ 64 +#define LAN966X_BUFFER_MEMORY (160 * 1024) +#define LAN966X_BUFFER_MIN_SZ 60 + +#define PGID_AGGR 64 +#define PGID_SRC 80 +#define PGID_ENTRIES 89 + +#define PORT_PVID 0 + +/* Reserved amount for (SRC, PRIO) at index 8*SRC + PRIO */ +#define QSYS_Q_RSRV 95 + +/* Reserved PGIDs */ +#define PGID_CPU (PGID_AGGR - 6) +#define PGID_UC (PGID_AGGR - 5) +#define PGID_BC (PGID_AGGR - 4) +#define PGID_MC (PGID_AGGR - 3) +#define PGID_MCIPV4 (PGID_AGGR - 2) +#define PGID_MCIPV6 (PGID_AGGR - 1) + +#define LAN966X_SPEED_NONE 0 +#define LAN966X_SPEED_2500 1 +#define LAN966X_SPEED_1000 1 +#define LAN966X_SPEED_100 2 +#define LAN966X_SPEED_10 3 + +#define CPU_PORT 8 + +/* MAC table entry types. + * ENTRYTYPE_NORMAL is subject to aging. + * ENTRYTYPE_LOCKED is not subject to aging. + * ENTRYTYPE_MACv4 is not subject to aging. For IPv4 multicast. + * ENTRYTYPE_MACv6 is not subject to aging. For IPv6 multicast. + */ +enum macaccess_entry_type { + ENTRYTYPE_NORMAL = 0, + ENTRYTYPE_LOCKED, + ENTRYTYPE_MACV4, + ENTRYTYPE_MACV6, +}; + +struct lan966x_port; + +struct lan966x_stat_layout { + u32 offset; + char name[ETH_GSTRING_LEN]; +}; + +struct lan966x { + struct device *dev; + + u8 num_phys_ports; + struct lan966x_port **ports; + + void __iomem *regs[NUM_TARGETS]; + + int shared_queue_sz; + + u8 base_mac[ETH_ALEN]; + + /* stats */ + const struct lan966x_stat_layout *stats_layout; + u32 num_stats; + + /* workqueue for reading stats */ + struct mutex stats_lock; + u64 *stats; + struct delayed_work stats_work; + struct workqueue_struct *stats_queue; + + /* interrupts */ + int xtr_irq; +}; + +struct lan966x_port_config { + phy_interface_t portmode; + const unsigned long *advertising; + int speed; + int duplex; + u32 pause; + bool inband; + bool autoneg; +}; + +struct lan966x_port { + struct net_device *dev; + struct lan966x *lan966x; + + u8 chip_port; + u16 pvid; + + struct phylink_config phylink_config; + struct phylink_pcs phylink_pcs; + struct lan966x_port_config config; + struct phylink *phylink; + struct phy *serdes; + struct fwnode_handle *fwnode; +}; + +extern const struct phylink_mac_ops lan966x_phylink_mac_ops; +extern const struct phylink_pcs_ops lan966x_phylink_pcs_ops; +extern const struct ethtool_ops lan966x_ethtool_ops; + +void lan966x_stats_get(struct net_device *dev, + struct rtnl_link_stats64 *stats); +int lan966x_stats_init(struct lan966x *lan966x); + +void lan966x_port_config_down(struct lan966x_port *port); +void lan966x_port_config_up(struct lan966x_port *port); +void lan966x_port_status_get(struct lan966x_port *port, + struct phylink_link_state *state); +int lan966x_port_pcs_set(struct lan966x_port *port, + struct lan966x_port_config *config); +void lan966x_port_init(struct lan966x_port *port); + +int lan966x_mac_learn(struct lan966x *lan966x, int port, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type); +int lan966x_mac_forget(struct lan966x *lan966x, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type); +int lan966x_mac_cpu_learn(struct lan966x *lan966x, const char *addr, u16 vid); +int lan966x_mac_cpu_forget(struct lan966x *lan966x, const char *addr, u16 vid); +void lan966x_mac_init(struct lan966x *lan966x); + +static inline void __iomem *lan_addr(void __iomem *base[], + int id, int tinst, int tcnt, + int gbase, int ginst, + int gcnt, int gwidth, + int raddr, int rinst, + int rcnt, int rwidth) +{ + WARN_ON((tinst) >= tcnt); + WARN_ON((ginst) >= gcnt); + WARN_ON((rinst) >= rcnt); + return base[id + (tinst)] + + gbase + ((ginst) * gwidth) + + raddr + ((rinst) * rwidth); +} + +static inline u32 lan_rd(struct lan966x *lan966x, int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + return readl(lan_addr(lan966x->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); +} + +static inline void lan_wr(u32 val, struct lan966x *lan966x, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + writel(val, lan_addr(lan966x->regs, id, tinst, tcnt, + gbase, ginst, gcnt, gwidth, + raddr, rinst, rcnt, rwidth)); +} + +static inline void lan_rmw(u32 val, u32 mask, struct lan966x *lan966x, + int id, int tinst, int tcnt, + int gbase, int ginst, int gcnt, int gwidth, + int raddr, int rinst, int rcnt, int rwidth) +{ + u32 nval; + + nval = readl(lan_addr(lan966x->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); + nval = (nval & ~mask) | (val & mask); + writel(nval, lan_addr(lan966x->regs, id, tinst, tcnt, gbase, ginst, + gcnt, gwidth, raddr, rinst, rcnt, rwidth)); +} + +#endif /* __LAN966X_MAIN_H__ */ diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c new file mode 100644 index 000000000000..b66a9aa00ea4 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_phylink.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/module.h> +#include <linux/phylink.h> +#include <linux/device.h> +#include <linux/netdevice.h> +#include <linux/phy/phy.h> +#include <linux/sfp.h> + +#include "lan966x_main.h" + +static void lan966x_phylink_mac_config(struct phylink_config *config, + unsigned int mode, + const struct phylink_link_state *state) +{ +} + +static int lan966x_phylink_mac_prepare(struct phylink_config *config, + unsigned int mode, + phy_interface_t iface) +{ + struct lan966x_port *port = netdev_priv(to_net_dev(config->dev)); + int err; + + if (port->serdes) { + err = phy_set_mode_ext(port->serdes, PHY_MODE_ETHERNET, + iface); + if (err) { + netdev_err(to_net_dev(config->dev), + "Could not set mode of SerDes\n"); + return err; + } + } + + return 0; +} + +static void lan966x_phylink_mac_link_up(struct phylink_config *config, + struct phy_device *phy, + unsigned int mode, + phy_interface_t interface, + int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct lan966x_port *port = netdev_priv(to_net_dev(config->dev)); + struct lan966x_port_config *port_config = &port->config; + + port_config->duplex = duplex; + port_config->speed = speed; + port_config->pause = 0; + port_config->pause |= tx_pause ? MLO_PAUSE_TX : 0; + port_config->pause |= rx_pause ? MLO_PAUSE_RX : 0; + + lan966x_port_config_up(port); +} + +static void lan966x_phylink_mac_link_down(struct phylink_config *config, + unsigned int mode, + phy_interface_t interface) +{ + struct lan966x_port *port = netdev_priv(to_net_dev(config->dev)); + struct lan966x *lan966x = port->lan966x; + + lan966x_port_config_down(port); + + /* Take PCS out of reset */ + lan_rmw(DEV_CLOCK_CFG_PCS_RX_RST_SET(0) | + DEV_CLOCK_CFG_PCS_TX_RST_SET(0), + DEV_CLOCK_CFG_PCS_RX_RST | + DEV_CLOCK_CFG_PCS_TX_RST, + lan966x, DEV_CLOCK_CFG(port->chip_port)); +} + +static struct lan966x_port *lan966x_pcs_to_port(struct phylink_pcs *pcs) +{ + return container_of(pcs, struct lan966x_port, phylink_pcs); +} + +static void lan966x_pcs_get_state(struct phylink_pcs *pcs, + struct phylink_link_state *state) +{ + struct lan966x_port *port = lan966x_pcs_to_port(pcs); + + lan966x_port_status_get(port, state); +} + +static int lan966x_pcs_config(struct phylink_pcs *pcs, + unsigned int mode, + phy_interface_t interface, + const unsigned long *advertising, + bool permit_pause_to_mac) +{ + struct lan966x_port *port = lan966x_pcs_to_port(pcs); + struct lan966x_port_config config; + int ret; + + config = port->config; + config.portmode = interface; + config.inband = phylink_autoneg_inband(mode); + config.autoneg = phylink_test(advertising, Autoneg); + config.advertising = advertising; + + ret = lan966x_port_pcs_set(port, &config); + if (ret) + netdev_err(port->dev, "port PCS config failed: %d\n", ret); + + return ret; +} + +static void lan966x_pcs_aneg_restart(struct phylink_pcs *pcs) +{ + /* Currently not used */ +} + +const struct phylink_mac_ops lan966x_phylink_mac_ops = { + .validate = phylink_generic_validate, + .mac_config = lan966x_phylink_mac_config, + .mac_prepare = lan966x_phylink_mac_prepare, + .mac_link_down = lan966x_phylink_mac_link_down, + .mac_link_up = lan966x_phylink_mac_link_up, +}; + +const struct phylink_pcs_ops lan966x_phylink_pcs_ops = { + .pcs_get_state = lan966x_pcs_get_state, + .pcs_config = lan966x_pcs_config, + .pcs_an_restart = lan966x_pcs_aneg_restart, +}; diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_port.c b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c new file mode 100644 index 000000000000..2ddb20585d40 --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_port.c @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-2.0+ + +#include <linux/netdevice.h> +#include <linux/phy/phy.h> + +#include "lan966x_main.h" + +/* Watermark encode */ +#define MULTIPLIER_BIT BIT(8) +static u32 lan966x_wm_enc(u32 value) +{ + value /= LAN966X_BUFFER_CELL_SZ; + + if (value >= MULTIPLIER_BIT) { + value /= 16; + if (value >= MULTIPLIER_BIT) + value = (MULTIPLIER_BIT - 1); + + value |= MULTIPLIER_BIT; + } + + return value; +} + +static void lan966x_port_link_down(struct lan966x_port *port) +{ + struct lan966x *lan966x = port->lan966x; + u32 val, delay = 0; + + /* 0.5: Disable any AFI */ + lan_rmw(AFI_PORT_CFG_FC_SKIP_TTI_INJ_SET(1) | + AFI_PORT_CFG_FRM_OUT_MAX_SET(0), + AFI_PORT_CFG_FC_SKIP_TTI_INJ | + AFI_PORT_CFG_FRM_OUT_MAX, + lan966x, AFI_PORT_CFG(port->chip_port)); + + /* wait for reg afi_port_frm_out to become 0 for the port */ + while (true) { + val = lan_rd(lan966x, AFI_PORT_FRM_OUT(port->chip_port)); + if (!AFI_PORT_FRM_OUT_FRM_OUT_CNT_GET(val)) + break; + + usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); + delay++; + if (delay == 2000) { + pr_err("AFI timeout chip port %u", port->chip_port); + break; + } + } + + delay = 0; + + /* 1: Reset the PCS Rx clock domain */ + lan_rmw(DEV_CLOCK_CFG_PCS_RX_RST_SET(1), + DEV_CLOCK_CFG_PCS_RX_RST, + lan966x, DEV_CLOCK_CFG(port->chip_port)); + + /* 2: Disable MAC frame reception */ + lan_rmw(DEV_MAC_ENA_CFG_RX_ENA_SET(0), + DEV_MAC_ENA_CFG_RX_ENA, + lan966x, DEV_MAC_ENA_CFG(port->chip_port)); + + /* 3: Disable traffic being sent to or from switch port */ + lan_rmw(QSYS_SW_PORT_MODE_PORT_ENA_SET(0), + QSYS_SW_PORT_MODE_PORT_ENA, + lan966x, QSYS_SW_PORT_MODE(port->chip_port)); + + /* 4: Disable dequeuing from the egress queues */ + lan_rmw(QSYS_PORT_MODE_DEQUEUE_DIS_SET(1), + QSYS_PORT_MODE_DEQUEUE_DIS, + lan966x, QSYS_PORT_MODE(port->chip_port)); + + /* 5: Disable Flowcontrol */ + lan_rmw(SYS_PAUSE_CFG_PAUSE_ENA_SET(0), + SYS_PAUSE_CFG_PAUSE_ENA, + lan966x, SYS_PAUSE_CFG(port->chip_port)); + + /* 5.1: Disable PFC */ + lan_rmw(QSYS_SW_PORT_MODE_TX_PFC_ENA_SET(0), + QSYS_SW_PORT_MODE_TX_PFC_ENA, + lan966x, QSYS_SW_PORT_MODE(port->chip_port)); + + /* 6: Wait a worst case time 8ms (jumbo/10Mbit) */ + usleep_range(8 * USEC_PER_MSEC, 9 * USEC_PER_MSEC); + + /* 7: Disable HDX backpressure */ + lan_rmw(SYS_FRONT_PORT_MODE_HDX_MODE_SET(0), + SYS_FRONT_PORT_MODE_HDX_MODE, + lan966x, SYS_FRONT_PORT_MODE(port->chip_port)); + + /* 8: Flush the queues accociated with the port */ + lan_rmw(QSYS_SW_PORT_MODE_AGING_MODE_SET(3), + QSYS_SW_PORT_MODE_AGING_MODE, + lan966x, QSYS_SW_PORT_MODE(port->chip_port)); + + /* 9: Enable dequeuing from the egress queues */ + lan_rmw(QSYS_PORT_MODE_DEQUEUE_DIS_SET(0), + QSYS_PORT_MODE_DEQUEUE_DIS, + lan966x, QSYS_PORT_MODE(port->chip_port)); + + /* 10: Wait until flushing is complete */ + while (true) { + val = lan_rd(lan966x, QSYS_SW_STATUS(port->chip_port)); + if (!QSYS_SW_STATUS_EQ_AVAIL_GET(val)) + break; + + usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); + delay++; + if (delay == 2000) { + pr_err("Flush timeout chip port %u", port->chip_port); + break; + } + } + + /* 11: Reset the Port and MAC clock domains */ + lan_rmw(DEV_MAC_ENA_CFG_TX_ENA_SET(0), + DEV_MAC_ENA_CFG_TX_ENA, + lan966x, DEV_MAC_ENA_CFG(port->chip_port)); + + lan_rmw(DEV_CLOCK_CFG_PORT_RST_SET(1), + DEV_CLOCK_CFG_PORT_RST, + lan966x, DEV_CLOCK_CFG(port->chip_port)); + + usleep_range(USEC_PER_MSEC, 2 * USEC_PER_MSEC); + + lan_rmw(DEV_CLOCK_CFG_MAC_TX_RST_SET(1) | + DEV_CLOCK_CFG_MAC_RX_RST_SET(1) | + DEV_CLOCK_CFG_PORT_RST_SET(1), + DEV_CLOCK_CFG_MAC_TX_RST | + DEV_CLOCK_CFG_MAC_RX_RST | + DEV_CLOCK_CFG_PORT_RST, + lan966x, DEV_CLOCK_CFG(port->chip_port)); + + /* 12: Clear flushing */ + lan_rmw(QSYS_SW_PORT_MODE_AGING_MODE_SET(2), + QSYS_SW_PORT_MODE_AGING_MODE, + lan966x, QSYS_SW_PORT_MODE(port->chip_port)); + + /* The port is disabled and flushed, now set up the port in the + * new operating mode + */ +} + +static void lan966x_port_link_up(struct lan966x_port *port) +{ + struct lan966x_port_config *config = &port->config; + struct lan966x *lan966x = port->lan966x; + int speed = 0, mode = 0; + int atop_wm = 0; + + switch (config->speed) { + case SPEED_10: + speed = LAN966X_SPEED_10; + break; + case SPEED_100: + speed = LAN966X_SPEED_100; + break; + case SPEED_1000: + speed = LAN966X_SPEED_1000; + mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(1); + break; + case SPEED_2500: + speed = LAN966X_SPEED_2500; + mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(1); + break; + } + + /* Also the GIGA_MODE_ENA(1) needs to be set regardless of the + * port speed for QSGMII ports. + */ + if (config->portmode == PHY_INTERFACE_MODE_QSGMII) + mode = DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(1); + + lan_wr(config->duplex | mode, + lan966x, DEV_MAC_MODE_CFG(port->chip_port)); + + lan_rmw(DEV_MAC_IFG_CFG_TX_IFG_SET(config->duplex ? 6 : 5) | + DEV_MAC_IFG_CFG_RX_IFG1_SET(config->speed == SPEED_10 ? 2 : 1) | + DEV_MAC_IFG_CFG_RX_IFG2_SET(2), + DEV_MAC_IFG_CFG_TX_IFG | + DEV_MAC_IFG_CFG_RX_IFG1 | + DEV_MAC_IFG_CFG_RX_IFG2, + lan966x, DEV_MAC_IFG_CFG(port->chip_port)); + + lan_rmw(DEV_MAC_HDX_CFG_SEED_SET(4) | + DEV_MAC_HDX_CFG_SEED_LOAD_SET(1), + DEV_MAC_HDX_CFG_SEED | + DEV_MAC_HDX_CFG_SEED_LOAD, + lan966x, DEV_MAC_HDX_CFG(port->chip_port)); + + if (config->portmode == PHY_INTERFACE_MODE_GMII) { + if (config->speed == SPEED_1000) + lan_rmw(CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA_SET(1), + CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA, + lan966x, + CHIP_TOP_CUPHY_PORT_CFG(port->chip_port)); + else + lan_rmw(CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA_SET(0), + CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA, + lan966x, + CHIP_TOP_CUPHY_PORT_CFG(port->chip_port)); + } + + /* No PFC */ + lan_wr(ANA_PFC_CFG_FC_LINK_SPEED_SET(speed), + lan966x, ANA_PFC_CFG(port->chip_port)); + + lan_rmw(DEV_PCS1G_CFG_PCS_ENA_SET(1), + DEV_PCS1G_CFG_PCS_ENA, + lan966x, DEV_PCS1G_CFG(port->chip_port)); + + lan_rmw(DEV_PCS1G_SD_CFG_SD_ENA_SET(0), + DEV_PCS1G_SD_CFG_SD_ENA, + lan966x, DEV_PCS1G_SD_CFG(port->chip_port)); + + /* Set Pause WM hysteresis, start/stop are in 1518 byte units */ + lan_wr(SYS_PAUSE_CFG_PAUSE_ENA_SET(1) | + SYS_PAUSE_CFG_PAUSE_STOP_SET(lan966x_wm_enc(4 * 1518)) | + SYS_PAUSE_CFG_PAUSE_START_SET(lan966x_wm_enc(6 * 1518)), + lan966x, SYS_PAUSE_CFG(port->chip_port)); + + /* Set SMAC of Pause frame (00:00:00:00:00:00) */ + lan_wr(0, lan966x, DEV_FC_MAC_LOW_CFG(port->chip_port)); + lan_wr(0, lan966x, DEV_FC_MAC_HIGH_CFG(port->chip_port)); + + /* Flow control */ + lan_rmw(SYS_MAC_FC_CFG_FC_LINK_SPEED_SET(speed) | + SYS_MAC_FC_CFG_FC_LATENCY_CFG_SET(7) | + SYS_MAC_FC_CFG_ZERO_PAUSE_ENA_SET(1) | + SYS_MAC_FC_CFG_PAUSE_VAL_CFG_SET(0xffff) | + SYS_MAC_FC_CFG_RX_FC_ENA_SET(config->pause & MLO_PAUSE_RX ? 1 : 0) | + SYS_MAC_FC_CFG_TX_FC_ENA_SET(config->pause & MLO_PAUSE_TX ? 1 : 0), + SYS_MAC_FC_CFG_FC_LINK_SPEED | + SYS_MAC_FC_CFG_FC_LATENCY_CFG | + SYS_MAC_FC_CFG_ZERO_PAUSE_ENA | + SYS_MAC_FC_CFG_PAUSE_VAL_CFG | + SYS_MAC_FC_CFG_RX_FC_ENA | + SYS_MAC_FC_CFG_TX_FC_ENA, + lan966x, SYS_MAC_FC_CFG(port->chip_port)); + + /* Tail dropping watermark */ + atop_wm = lan966x->shared_queue_sz; + + /* The total memory size is diveded by number of front ports plus CPU + * port + */ + lan_wr(lan966x_wm_enc(atop_wm / lan966x->num_phys_ports + 1), lan966x, + SYS_ATOP(port->chip_port)); + lan_wr(lan966x_wm_enc(atop_wm), lan966x, SYS_ATOP_TOT_CFG); + + /* This needs to be at the end */ + /* Enable MAC module */ + lan_wr(DEV_MAC_ENA_CFG_RX_ENA_SET(1) | + DEV_MAC_ENA_CFG_TX_ENA_SET(1), + lan966x, DEV_MAC_ENA_CFG(port->chip_port)); + + /* Take out the clock from reset */ + lan_wr(DEV_CLOCK_CFG_LINK_SPEED_SET(speed), + lan966x, DEV_CLOCK_CFG(port->chip_port)); + + /* Core: Enable port for frame transfer */ + lan_wr(QSYS_SW_PORT_MODE_PORT_ENA_SET(1) | + QSYS_SW_PORT_MODE_SCH_NEXT_CFG_SET(1) | + QSYS_SW_PORT_MODE_INGRESS_DROP_MODE_SET(1), + lan966x, QSYS_SW_PORT_MODE(port->chip_port)); + + lan_rmw(AFI_PORT_CFG_FC_SKIP_TTI_INJ_SET(0) | + AFI_PORT_CFG_FRM_OUT_MAX_SET(16), + AFI_PORT_CFG_FC_SKIP_TTI_INJ | + AFI_PORT_CFG_FRM_OUT_MAX, + lan966x, AFI_PORT_CFG(port->chip_port)); +} + +void lan966x_port_config_down(struct lan966x_port *port) +{ + lan966x_port_link_down(port); +} + +void lan966x_port_config_up(struct lan966x_port *port) +{ + lan966x_port_link_up(port); +} + +void lan966x_port_status_get(struct lan966x_port *port, + struct phylink_link_state *state) +{ + struct lan966x *lan966x = port->lan966x; + bool link_down; + u16 bmsr = 0; + u16 lp_adv; + u32 val; + + val = lan_rd(lan966x, DEV_PCS1G_STICKY(port->chip_port)); + link_down = DEV_PCS1G_STICKY_LINK_DOWN_STICKY_GET(val); + if (link_down) + lan_wr(val, lan966x, DEV_PCS1G_STICKY(port->chip_port)); + + /* Get both current Link and Sync status */ + val = lan_rd(lan966x, DEV_PCS1G_LINK_STATUS(port->chip_port)); + state->link = DEV_PCS1G_LINK_STATUS_LINK_STATUS_GET(val) && + DEV_PCS1G_LINK_STATUS_SYNC_STATUS_GET(val); + state->link &= !link_down; + + /* Get PCS ANEG status register */ + val = lan_rd(lan966x, DEV_PCS1G_ANEG_STATUS(port->chip_port)); + /* Aneg complete provides more information */ + if (DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE_GET(val)) { + state->an_complete = true; + + bmsr |= state->link ? BMSR_LSTATUS : 0; + bmsr |= BMSR_ANEGCOMPLETE; + + lp_adv = DEV_PCS1G_ANEG_STATUS_LP_ADV_GET(val); + phylink_mii_c22_pcs_decode_state(state, bmsr, lp_adv); + } else { + if (!state->link) + return; + + if (state->interface == PHY_INTERFACE_MODE_1000BASEX) + state->speed = SPEED_1000; + else if (state->interface == PHY_INTERFACE_MODE_2500BASEX) + state->speed = SPEED_2500; + + state->duplex = DUPLEX_FULL; + } +} + +int lan966x_port_pcs_set(struct lan966x_port *port, + struct lan966x_port_config *config) +{ + struct lan966x *lan966x = port->lan966x; + bool inband_aneg = false; + bool outband; + int err; + + if (config->inband) { + if (config->portmode == PHY_INTERFACE_MODE_SGMII || + config->portmode == PHY_INTERFACE_MODE_QSGMII) + inband_aneg = true; /* Cisco-SGMII in-band-aneg */ + else if (config->portmode == PHY_INTERFACE_MODE_1000BASEX && + config->autoneg) + inband_aneg = true; /* Clause-37 in-band-aneg */ + + if (config->speed > 0) { + err = phy_set_speed(port->serdes, config->speed); + if (err) + return err; + } + outband = false; + } else { + outband = true; + } + + /* Disable or enable inband */ + lan_rmw(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(outband), + DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, + lan966x, DEV_PCS1G_MODE_CFG(port->chip_port)); + + /* Enable PCS */ + lan_wr(DEV_PCS1G_CFG_PCS_ENA_SET(1), + lan966x, DEV_PCS1G_CFG(port->chip_port)); + + if (inband_aneg) { + int adv = phylink_mii_c22_pcs_encode_advertisement(config->portmode, + config->advertising); + if (adv >= 0) + /* Enable in-band aneg */ + lan_wr(DEV_PCS1G_ANEG_CFG_ADV_ABILITY_SET(adv) | + DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_SET(1) | + DEV_PCS1G_ANEG_CFG_ENA_SET(1) | + DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT_SET(1), + lan966x, DEV_PCS1G_ANEG_CFG(port->chip_port)); + } else { + lan_wr(0, lan966x, DEV_PCS1G_ANEG_CFG(port->chip_port)); + } + + /* Take PCS out of reset */ + lan_rmw(DEV_CLOCK_CFG_LINK_SPEED_SET(2) | + DEV_CLOCK_CFG_PCS_RX_RST_SET(0) | + DEV_CLOCK_CFG_PCS_TX_RST_SET(0), + DEV_CLOCK_CFG_LINK_SPEED | + DEV_CLOCK_CFG_PCS_RX_RST | + DEV_CLOCK_CFG_PCS_TX_RST, + lan966x, DEV_CLOCK_CFG(port->chip_port)); + + port->config = *config; + + return 0; +} + +void lan966x_port_init(struct lan966x_port *port) +{ + struct lan966x_port_config *config = &port->config; + struct lan966x *lan966x = port->lan966x; + + lan_rmw(ANA_PORT_CFG_LEARN_ENA_SET(0), + ANA_PORT_CFG_LEARN_ENA, + lan966x, ANA_PORT_CFG(port->chip_port)); + + lan966x_port_config_down(port); + + if (config->portmode != PHY_INTERFACE_MODE_QSGMII) + return; + + lan_rmw(DEV_CLOCK_CFG_PCS_RX_RST_SET(0) | + DEV_CLOCK_CFG_PCS_TX_RST_SET(0) | + DEV_CLOCK_CFG_LINK_SPEED_SET(LAN966X_SPEED_1000), + DEV_CLOCK_CFG_PCS_RX_RST | + DEV_CLOCK_CFG_PCS_TX_RST | + DEV_CLOCK_CFG_LINK_SPEED, + lan966x, DEV_CLOCK_CFG(port->chip_port)); +} diff --git a/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h new file mode 100644 index 000000000000..879dcd807dec --- /dev/null +++ b/drivers/net/ethernet/microchip/lan966x/lan966x_regs.h @@ -0,0 +1,730 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ + +/* This file is autogenerated by cml-utils 2021-10-10 13:25:08 +0200. + * Commit ID: 26db2002924973d36a30b369c94f025a678fe9ea (dirty) + */ + +#ifndef _LAN966X_REGS_H_ +#define _LAN966X_REGS_H_ + +#include <linux/bitfield.h> +#include <linux/types.h> +#include <linux/bug.h> + +enum lan966x_target { + TARGET_AFI = 2, + TARGET_ANA = 3, + TARGET_CHIP_TOP = 5, + TARGET_CPU = 6, + TARGET_DEV = 13, + TARGET_GCB = 27, + TARGET_ORG = 36, + TARGET_QS = 42, + TARGET_QSYS = 46, + TARGET_REW = 47, + TARGET_SYS = 52, + NUM_TARGETS = 66 +}; + +#define __REG(...) __VA_ARGS__ + +/* AFI:PORT_TBL:PORT_FRM_OUT */ +#define AFI_PORT_FRM_OUT(g) __REG(TARGET_AFI, 0, 1, 98816, g, 10, 8, 0, 0, 1, 4) + +#define AFI_PORT_FRM_OUT_FRM_OUT_CNT GENMASK(26, 16) +#define AFI_PORT_FRM_OUT_FRM_OUT_CNT_SET(x)\ + FIELD_PREP(AFI_PORT_FRM_OUT_FRM_OUT_CNT, x) +#define AFI_PORT_FRM_OUT_FRM_OUT_CNT_GET(x)\ + FIELD_GET(AFI_PORT_FRM_OUT_FRM_OUT_CNT, x) + +/* AFI:PORT_TBL:PORT_CFG */ +#define AFI_PORT_CFG(g) __REG(TARGET_AFI, 0, 1, 98816, g, 10, 8, 4, 0, 1, 4) + +#define AFI_PORT_CFG_FC_SKIP_TTI_INJ BIT(16) +#define AFI_PORT_CFG_FC_SKIP_TTI_INJ_SET(x)\ + FIELD_PREP(AFI_PORT_CFG_FC_SKIP_TTI_INJ, x) +#define AFI_PORT_CFG_FC_SKIP_TTI_INJ_GET(x)\ + FIELD_GET(AFI_PORT_CFG_FC_SKIP_TTI_INJ, x) + +#define AFI_PORT_CFG_FRM_OUT_MAX GENMASK(9, 0) +#define AFI_PORT_CFG_FRM_OUT_MAX_SET(x)\ + FIELD_PREP(AFI_PORT_CFG_FRM_OUT_MAX, x) +#define AFI_PORT_CFG_FRM_OUT_MAX_GET(x)\ + FIELD_GET(AFI_PORT_CFG_FRM_OUT_MAX, x) + +/* ANA:ANA:ADVLEARN */ +#define ANA_ADVLEARN __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 0, 0, 1, 4) + +#define ANA_ADVLEARN_VLAN_CHK BIT(0) +#define ANA_ADVLEARN_VLAN_CHK_SET(x)\ + FIELD_PREP(ANA_ADVLEARN_VLAN_CHK, x) +#define ANA_ADVLEARN_VLAN_CHK_GET(x)\ + FIELD_GET(ANA_ADVLEARN_VLAN_CHK, x) + +/* ANA:ANA:ANAINTR */ +#define ANA_ANAINTR __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 16, 0, 1, 4) + +#define ANA_ANAINTR_INTR BIT(1) +#define ANA_ANAINTR_INTR_SET(x)\ + FIELD_PREP(ANA_ANAINTR_INTR, x) +#define ANA_ANAINTR_INTR_GET(x)\ + FIELD_GET(ANA_ANAINTR_INTR, x) + +#define ANA_ANAINTR_INTR_ENA BIT(0) +#define ANA_ANAINTR_INTR_ENA_SET(x)\ + FIELD_PREP(ANA_ANAINTR_INTR_ENA, x) +#define ANA_ANAINTR_INTR_ENA_GET(x)\ + FIELD_GET(ANA_ANAINTR_INTR_ENA, x) + +/* ANA:ANA:AUTOAGE */ +#define ANA_AUTOAGE __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 44, 0, 1, 4) + +#define ANA_AUTOAGE_AGE_PERIOD GENMASK(20, 1) +#define ANA_AUTOAGE_AGE_PERIOD_SET(x)\ + FIELD_PREP(ANA_AUTOAGE_AGE_PERIOD, x) +#define ANA_AUTOAGE_AGE_PERIOD_GET(x)\ + FIELD_GET(ANA_AUTOAGE_AGE_PERIOD, x) + +/* ANA:ANA:FLOODING */ +#define ANA_FLOODING(r) __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 68, r, 8, 4) + +#define ANA_FLOODING_FLD_BROADCAST GENMASK(11, 6) +#define ANA_FLOODING_FLD_BROADCAST_SET(x)\ + FIELD_PREP(ANA_FLOODING_FLD_BROADCAST, x) +#define ANA_FLOODING_FLD_BROADCAST_GET(x)\ + FIELD_GET(ANA_FLOODING_FLD_BROADCAST, x) + +#define ANA_FLOODING_FLD_MULTICAST GENMASK(5, 0) +#define ANA_FLOODING_FLD_MULTICAST_SET(x)\ + FIELD_PREP(ANA_FLOODING_FLD_MULTICAST, x) +#define ANA_FLOODING_FLD_MULTICAST_GET(x)\ + FIELD_GET(ANA_FLOODING_FLD_MULTICAST, x) + +/* ANA:ANA:FLOODING_IPMC */ +#define ANA_FLOODING_IPMC __REG(TARGET_ANA, 0, 1, 29824, 0, 1, 244, 100, 0, 1, 4) + +#define ANA_FLOODING_IPMC_FLD_MC4_CTRL GENMASK(23, 18) +#define ANA_FLOODING_IPMC_FLD_MC4_CTRL_SET(x)\ + FIELD_PREP(ANA_FLOODING_IPMC_FLD_MC4_CTRL, x) +#define ANA_FLOODING_IPMC_FLD_MC4_CTRL_GET(x)\ + FIELD_GET(ANA_FLOODING_IPMC_FLD_MC4_CTRL, x) + +#define ANA_FLOODING_IPMC_FLD_MC4_DATA GENMASK(17, 12) +#define ANA_FLOODING_IPMC_FLD_MC4_DATA_SET(x)\ + FIELD_PREP(ANA_FLOODING_IPMC_FLD_MC4_DATA, x) +#define ANA_FLOODING_IPMC_FLD_MC4_DATA_GET(x)\ + FIELD_GET(ANA_FLOODING_IPMC_FLD_MC4_DATA, x) + +#define ANA_FLOODING_IPMC_FLD_MC6_CTRL GENMASK(11, 6) +#define ANA_FLOODING_IPMC_FLD_MC6_CTRL_SET(x)\ + FIELD_PREP(ANA_FLOODING_IPMC_FLD_MC6_CTRL, x) +#define ANA_FLOODING_IPMC_FLD_MC6_CTRL_GET(x)\ + FIELD_GET(ANA_FLOODING_IPMC_FLD_MC6_CTRL, x) + +#define ANA_FLOODING_IPMC_FLD_MC6_DATA GENMASK(5, 0) +#define ANA_FLOODING_IPMC_FLD_MC6_DATA_SET(x)\ + FIELD_PREP(ANA_FLOODING_IPMC_FLD_MC6_DATA, x) +#define ANA_FLOODING_IPMC_FLD_MC6_DATA_GET(x)\ + FIELD_GET(ANA_FLOODING_IPMC_FLD_MC6_DATA, x) + +/* ANA:PGID:PGID */ +#define ANA_PGID(g) __REG(TARGET_ANA, 0, 1, 27648, g, 89, 8, 0, 0, 1, 4) + +#define ANA_PGID_PGID GENMASK(8, 0) +#define ANA_PGID_PGID_SET(x)\ + FIELD_PREP(ANA_PGID_PGID, x) +#define ANA_PGID_PGID_GET(x)\ + FIELD_GET(ANA_PGID_PGID, x) + +/* ANA:PGID:PGID_CFG */ +#define ANA_PGID_CFG(g) __REG(TARGET_ANA, 0, 1, 27648, g, 89, 8, 4, 0, 1, 4) + +#define ANA_PGID_CFG_OBEY_VLAN BIT(0) +#define ANA_PGID_CFG_OBEY_VLAN_SET(x)\ + FIELD_PREP(ANA_PGID_CFG_OBEY_VLAN, x) +#define ANA_PGID_CFG_OBEY_VLAN_GET(x)\ + FIELD_GET(ANA_PGID_CFG_OBEY_VLAN, x) + +/* ANA:ANA_TABLES:MACHDATA */ +#define ANA_MACHDATA __REG(TARGET_ANA, 0, 1, 27520, 0, 1, 128, 40, 0, 1, 4) + +/* ANA:ANA_TABLES:MACLDATA */ +#define ANA_MACLDATA __REG(TARGET_ANA, 0, 1, 27520, 0, 1, 128, 44, 0, 1, 4) + +/* ANA:ANA_TABLES:MACACCESS */ +#define ANA_MACACCESS __REG(TARGET_ANA, 0, 1, 27520, 0, 1, 128, 48, 0, 1, 4) + +#define ANA_MACACCESS_CHANGE2SW BIT(17) +#define ANA_MACACCESS_CHANGE2SW_SET(x)\ + FIELD_PREP(ANA_MACACCESS_CHANGE2SW, x) +#define ANA_MACACCESS_CHANGE2SW_GET(x)\ + FIELD_GET(ANA_MACACCESS_CHANGE2SW, x) + +#define ANA_MACACCESS_VALID BIT(12) +#define ANA_MACACCESS_VALID_SET(x)\ + FIELD_PREP(ANA_MACACCESS_VALID, x) +#define ANA_MACACCESS_VALID_GET(x)\ + FIELD_GET(ANA_MACACCESS_VALID, x) + +#define ANA_MACACCESS_ENTRYTYPE GENMASK(11, 10) +#define ANA_MACACCESS_ENTRYTYPE_SET(x)\ + FIELD_PREP(ANA_MACACCESS_ENTRYTYPE, x) +#define ANA_MACACCESS_ENTRYTYPE_GET(x)\ + FIELD_GET(ANA_MACACCESS_ENTRYTYPE, x) + +#define ANA_MACACCESS_DEST_IDX GENMASK(9, 4) +#define ANA_MACACCESS_DEST_IDX_SET(x)\ + FIELD_PREP(ANA_MACACCESS_DEST_IDX, x) +#define ANA_MACACCESS_DEST_IDX_GET(x)\ + FIELD_GET(ANA_MACACCESS_DEST_IDX, x) + +#define ANA_MACACCESS_MAC_TABLE_CMD GENMASK(3, 0) +#define ANA_MACACCESS_MAC_TABLE_CMD_SET(x)\ + FIELD_PREP(ANA_MACACCESS_MAC_TABLE_CMD, x) +#define ANA_MACACCESS_MAC_TABLE_CMD_GET(x)\ + FIELD_GET(ANA_MACACCESS_MAC_TABLE_CMD, x) + +/* ANA:PORT:CPU_FWD_CFG */ +#define ANA_CPU_FWD_CFG(g) __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 96, 0, 1, 4) + +#define ANA_CPU_FWD_CFG_SRC_COPY_ENA BIT(3) +#define ANA_CPU_FWD_CFG_SRC_COPY_ENA_SET(x)\ + FIELD_PREP(ANA_CPU_FWD_CFG_SRC_COPY_ENA, x) +#define ANA_CPU_FWD_CFG_SRC_COPY_ENA_GET(x)\ + FIELD_GET(ANA_CPU_FWD_CFG_SRC_COPY_ENA, x) + +/* ANA:PORT:CPU_FWD_BPDU_CFG */ +#define ANA_CPU_FWD_BPDU_CFG(g) __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 100, 0, 1, 4) + +/* ANA:PORT:PORT_CFG */ +#define ANA_PORT_CFG(g) __REG(TARGET_ANA, 0, 1, 28672, g, 9, 128, 112, 0, 1, 4) + +#define ANA_PORT_CFG_LEARNAUTO BIT(6) +#define ANA_PORT_CFG_LEARNAUTO_SET(x)\ + FIELD_PREP(ANA_PORT_CFG_LEARNAUTO, x) +#define ANA_PORT_CFG_LEARNAUTO_GET(x)\ + FIELD_GET(ANA_PORT_CFG_LEARNAUTO, x) + +#define ANA_PORT_CFG_LEARN_ENA BIT(5) +#define ANA_PORT_CFG_LEARN_ENA_SET(x)\ + FIELD_PREP(ANA_PORT_CFG_LEARN_ENA, x) +#define ANA_PORT_CFG_LEARN_ENA_GET(x)\ + FIELD_GET(ANA_PORT_CFG_LEARN_ENA, x) + +#define ANA_PORT_CFG_RECV_ENA BIT(4) +#define ANA_PORT_CFG_RECV_ENA_SET(x)\ + FIELD_PREP(ANA_PORT_CFG_RECV_ENA, x) +#define ANA_PORT_CFG_RECV_ENA_GET(x)\ + FIELD_GET(ANA_PORT_CFG_RECV_ENA, x) + +#define ANA_PORT_CFG_PORTID_VAL GENMASK(3, 0) +#define ANA_PORT_CFG_PORTID_VAL_SET(x)\ + FIELD_PREP(ANA_PORT_CFG_PORTID_VAL, x) +#define ANA_PORT_CFG_PORTID_VAL_GET(x)\ + FIELD_GET(ANA_PORT_CFG_PORTID_VAL, x) + +/* ANA:PFC:PFC_CFG */ +#define ANA_PFC_CFG(g) __REG(TARGET_ANA, 0, 1, 30720, g, 8, 64, 0, 0, 1, 4) + +#define ANA_PFC_CFG_FC_LINK_SPEED GENMASK(1, 0) +#define ANA_PFC_CFG_FC_LINK_SPEED_SET(x)\ + FIELD_PREP(ANA_PFC_CFG_FC_LINK_SPEED, x) +#define ANA_PFC_CFG_FC_LINK_SPEED_GET(x)\ + FIELD_GET(ANA_PFC_CFG_FC_LINK_SPEED, x) + +/* CHIP_TOP:CUPHY_CFG:CUPHY_PORT_CFG */ +#define CHIP_TOP_CUPHY_PORT_CFG(r) __REG(TARGET_CHIP_TOP, 0, 1, 16, 0, 1, 20, 8, r, 2, 4) + +#define CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA BIT(0) +#define CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA_SET(x)\ + FIELD_PREP(CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA, x) +#define CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA_GET(x)\ + FIELD_GET(CHIP_TOP_CUPHY_PORT_CFG_GTX_CLK_ENA, x) + +/* DEV:PORT_MODE:CLOCK_CFG */ +#define DEV_CLOCK_CFG(t) __REG(TARGET_DEV, t, 8, 0, 0, 1, 28, 0, 0, 1, 4) + +#define DEV_CLOCK_CFG_MAC_TX_RST BIT(7) +#define DEV_CLOCK_CFG_MAC_TX_RST_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_MAC_TX_RST, x) +#define DEV_CLOCK_CFG_MAC_TX_RST_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_MAC_TX_RST, x) + +#define DEV_CLOCK_CFG_MAC_RX_RST BIT(6) +#define DEV_CLOCK_CFG_MAC_RX_RST_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_MAC_RX_RST, x) +#define DEV_CLOCK_CFG_MAC_RX_RST_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_MAC_RX_RST, x) + +#define DEV_CLOCK_CFG_PCS_TX_RST BIT(5) +#define DEV_CLOCK_CFG_PCS_TX_RST_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_PCS_TX_RST, x) +#define DEV_CLOCK_CFG_PCS_TX_RST_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_PCS_TX_RST, x) + +#define DEV_CLOCK_CFG_PCS_RX_RST BIT(4) +#define DEV_CLOCK_CFG_PCS_RX_RST_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_PCS_RX_RST, x) +#define DEV_CLOCK_CFG_PCS_RX_RST_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_PCS_RX_RST, x) + +#define DEV_CLOCK_CFG_PORT_RST BIT(3) +#define DEV_CLOCK_CFG_PORT_RST_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_PORT_RST, x) +#define DEV_CLOCK_CFG_PORT_RST_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_PORT_RST, x) + +#define DEV_CLOCK_CFG_LINK_SPEED GENMASK(1, 0) +#define DEV_CLOCK_CFG_LINK_SPEED_SET(x)\ + FIELD_PREP(DEV_CLOCK_CFG_LINK_SPEED, x) +#define DEV_CLOCK_CFG_LINK_SPEED_GET(x)\ + FIELD_GET(DEV_CLOCK_CFG_LINK_SPEED, x) + +/* DEV:MAC_CFG_STATUS:MAC_ENA_CFG */ +#define DEV_MAC_ENA_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 0, 0, 1, 4) + +#define DEV_MAC_ENA_CFG_RX_ENA BIT(4) +#define DEV_MAC_ENA_CFG_RX_ENA_SET(x)\ + FIELD_PREP(DEV_MAC_ENA_CFG_RX_ENA, x) +#define DEV_MAC_ENA_CFG_RX_ENA_GET(x)\ + FIELD_GET(DEV_MAC_ENA_CFG_RX_ENA, x) + +#define DEV_MAC_ENA_CFG_TX_ENA BIT(0) +#define DEV_MAC_ENA_CFG_TX_ENA_SET(x)\ + FIELD_PREP(DEV_MAC_ENA_CFG_TX_ENA, x) +#define DEV_MAC_ENA_CFG_TX_ENA_GET(x)\ + FIELD_GET(DEV_MAC_ENA_CFG_TX_ENA, x) + +/* DEV:MAC_CFG_STATUS:MAC_MODE_CFG */ +#define DEV_MAC_MODE_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 4, 0, 1, 4) + +#define DEV_MAC_MODE_CFG_GIGA_MODE_ENA BIT(4) +#define DEV_MAC_MODE_CFG_GIGA_MODE_ENA_SET(x)\ + FIELD_PREP(DEV_MAC_MODE_CFG_GIGA_MODE_ENA, x) +#define DEV_MAC_MODE_CFG_GIGA_MODE_ENA_GET(x)\ + FIELD_GET(DEV_MAC_MODE_CFG_GIGA_MODE_ENA, x) + +/* DEV:MAC_CFG_STATUS:MAC_MAXLEN_CFG */ +#define DEV_MAC_MAXLEN_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 8, 0, 1, 4) + +#define DEV_MAC_MAXLEN_CFG_MAX_LEN GENMASK(15, 0) +#define DEV_MAC_MAXLEN_CFG_MAX_LEN_SET(x)\ + FIELD_PREP(DEV_MAC_MAXLEN_CFG_MAX_LEN, x) +#define DEV_MAC_MAXLEN_CFG_MAX_LEN_GET(x)\ + FIELD_GET(DEV_MAC_MAXLEN_CFG_MAX_LEN, x) + +/* DEV:MAC_CFG_STATUS:MAC_IFG_CFG */ +#define DEV_MAC_IFG_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 20, 0, 1, 4) + +#define DEV_MAC_IFG_CFG_TX_IFG GENMASK(12, 8) +#define DEV_MAC_IFG_CFG_TX_IFG_SET(x)\ + FIELD_PREP(DEV_MAC_IFG_CFG_TX_IFG, x) +#define DEV_MAC_IFG_CFG_TX_IFG_GET(x)\ + FIELD_GET(DEV_MAC_IFG_CFG_TX_IFG, x) + +#define DEV_MAC_IFG_CFG_RX_IFG2 GENMASK(7, 4) +#define DEV_MAC_IFG_CFG_RX_IFG2_SET(x)\ + FIELD_PREP(DEV_MAC_IFG_CFG_RX_IFG2, x) +#define DEV_MAC_IFG_CFG_RX_IFG2_GET(x)\ + FIELD_GET(DEV_MAC_IFG_CFG_RX_IFG2, x) + +#define DEV_MAC_IFG_CFG_RX_IFG1 GENMASK(3, 0) +#define DEV_MAC_IFG_CFG_RX_IFG1_SET(x)\ + FIELD_PREP(DEV_MAC_IFG_CFG_RX_IFG1, x) +#define DEV_MAC_IFG_CFG_RX_IFG1_GET(x)\ + FIELD_GET(DEV_MAC_IFG_CFG_RX_IFG1, x) + +/* DEV:MAC_CFG_STATUS:MAC_HDX_CFG */ +#define DEV_MAC_HDX_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 24, 0, 1, 4) + +#define DEV_MAC_HDX_CFG_SEED GENMASK(23, 16) +#define DEV_MAC_HDX_CFG_SEED_SET(x)\ + FIELD_PREP(DEV_MAC_HDX_CFG_SEED, x) +#define DEV_MAC_HDX_CFG_SEED_GET(x)\ + FIELD_GET(DEV_MAC_HDX_CFG_SEED, x) + +#define DEV_MAC_HDX_CFG_SEED_LOAD BIT(12) +#define DEV_MAC_HDX_CFG_SEED_LOAD_SET(x)\ + FIELD_PREP(DEV_MAC_HDX_CFG_SEED_LOAD, x) +#define DEV_MAC_HDX_CFG_SEED_LOAD_GET(x)\ + FIELD_GET(DEV_MAC_HDX_CFG_SEED_LOAD, x) + +/* DEV:MAC_CFG_STATUS:MAC_FC_MAC_LOW_CFG */ +#define DEV_FC_MAC_LOW_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 32, 0, 1, 4) + +/* DEV:MAC_CFG_STATUS:MAC_FC_MAC_HIGH_CFG */ +#define DEV_FC_MAC_HIGH_CFG(t) __REG(TARGET_DEV, t, 8, 28, 0, 1, 44, 36, 0, 1, 4) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_CFG */ +#define DEV_PCS1G_CFG(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 0, 0, 1, 4) + +#define DEV_PCS1G_CFG_PCS_ENA BIT(0) +#define DEV_PCS1G_CFG_PCS_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_CFG_PCS_ENA, x) +#define DEV_PCS1G_CFG_PCS_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_CFG_PCS_ENA, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_MODE_CFG */ +#define DEV_PCS1G_MODE_CFG(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 4, 0, 1, 4) + +#define DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA BIT(0) +#define DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, x) +#define DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_MODE_CFG_SGMII_MODE_ENA, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_SD_CFG */ +#define DEV_PCS1G_SD_CFG(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 8, 0, 1, 4) + +#define DEV_PCS1G_SD_CFG_SD_ENA BIT(0) +#define DEV_PCS1G_SD_CFG_SD_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_SD_CFG_SD_ENA, x) +#define DEV_PCS1G_SD_CFG_SD_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_SD_CFG_SD_ENA, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_ANEG_CFG */ +#define DEV_PCS1G_ANEG_CFG(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 12, 0, 1, 4) + +#define DEV_PCS1G_ANEG_CFG_ADV_ABILITY GENMASK(31, 16) +#define DEV_PCS1G_ANEG_CFG_ADV_ABILITY_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_CFG_ADV_ABILITY, x) +#define DEV_PCS1G_ANEG_CFG_ADV_ABILITY_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_CFG_ADV_ABILITY, x) + +#define DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA BIT(8) +#define DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA, x) +#define DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_CFG_SW_RESOLVE_ENA, x) + +#define DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT BIT(1) +#define DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT, x) +#define DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_CFG_RESTART_ONE_SHOT, x) + +#define DEV_PCS1G_ANEG_CFG_ENA BIT(0) +#define DEV_PCS1G_ANEG_CFG_ENA_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_CFG_ENA, x) +#define DEV_PCS1G_ANEG_CFG_ENA_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_CFG_ENA, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_ANEG_STATUS */ +#define DEV_PCS1G_ANEG_STATUS(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 32, 0, 1, 4) + +#define DEV_PCS1G_ANEG_STATUS_LP_ADV GENMASK(31, 16) +#define DEV_PCS1G_ANEG_STATUS_LP_ADV_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_STATUS_LP_ADV, x) +#define DEV_PCS1G_ANEG_STATUS_LP_ADV_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_STATUS_LP_ADV, x) + +#define DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE BIT(0) +#define DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE_SET(x)\ + FIELD_PREP(DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE, x) +#define DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE_GET(x)\ + FIELD_GET(DEV_PCS1G_ANEG_STATUS_ANEG_COMPLETE, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_LINK_STATUS */ +#define DEV_PCS1G_LINK_STATUS(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 40, 0, 1, 4) + +#define DEV_PCS1G_LINK_STATUS_LINK_STATUS BIT(4) +#define DEV_PCS1G_LINK_STATUS_LINK_STATUS_SET(x)\ + FIELD_PREP(DEV_PCS1G_LINK_STATUS_LINK_STATUS, x) +#define DEV_PCS1G_LINK_STATUS_LINK_STATUS_GET(x)\ + FIELD_GET(DEV_PCS1G_LINK_STATUS_LINK_STATUS, x) + +#define DEV_PCS1G_LINK_STATUS_SYNC_STATUS BIT(0) +#define DEV_PCS1G_LINK_STATUS_SYNC_STATUS_SET(x)\ + FIELD_PREP(DEV_PCS1G_LINK_STATUS_SYNC_STATUS, x) +#define DEV_PCS1G_LINK_STATUS_SYNC_STATUS_GET(x)\ + FIELD_GET(DEV_PCS1G_LINK_STATUS_SYNC_STATUS, x) + +/* DEV:PCS1G_CFG_STATUS:PCS1G_STICKY */ +#define DEV_PCS1G_STICKY(t) __REG(TARGET_DEV, t, 8, 72, 0, 1, 68, 48, 0, 1, 4) + +#define DEV_PCS1G_STICKY_LINK_DOWN_STICKY BIT(4) +#define DEV_PCS1G_STICKY_LINK_DOWN_STICKY_SET(x)\ + FIELD_PREP(DEV_PCS1G_STICKY_LINK_DOWN_STICKY, x) +#define DEV_PCS1G_STICKY_LINK_DOWN_STICKY_GET(x)\ + FIELD_GET(DEV_PCS1G_STICKY_LINK_DOWN_STICKY, x) + +/* DEVCPU_QS:XTR:XTR_GRP_CFG */ +#define QS_XTR_GRP_CFG(r) __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 0, r, 2, 4) + +#define QS_XTR_GRP_CFG_MODE GENMASK(3, 2) +#define QS_XTR_GRP_CFG_MODE_SET(x)\ + FIELD_PREP(QS_XTR_GRP_CFG_MODE, x) +#define QS_XTR_GRP_CFG_MODE_GET(x)\ + FIELD_GET(QS_XTR_GRP_CFG_MODE, x) + +#define QS_XTR_GRP_CFG_BYTE_SWAP BIT(0) +#define QS_XTR_GRP_CFG_BYTE_SWAP_SET(x)\ + FIELD_PREP(QS_XTR_GRP_CFG_BYTE_SWAP, x) +#define QS_XTR_GRP_CFG_BYTE_SWAP_GET(x)\ + FIELD_GET(QS_XTR_GRP_CFG_BYTE_SWAP, x) + +/* DEVCPU_QS:XTR:XTR_RD */ +#define QS_XTR_RD(r) __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 8, r, 2, 4) + +/* DEVCPU_QS:XTR:XTR_FLUSH */ +#define QS_XTR_FLUSH __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 24, 0, 1, 4) + +/* DEVCPU_QS:XTR:XTR_DATA_PRESENT */ +#define QS_XTR_DATA_PRESENT __REG(TARGET_QS, 0, 1, 0, 0, 1, 36, 28, 0, 1, 4) + +/* DEVCPU_QS:INJ:INJ_GRP_CFG */ +#define QS_INJ_GRP_CFG(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 0, r, 2, 4) + +#define QS_INJ_GRP_CFG_MODE GENMASK(3, 2) +#define QS_INJ_GRP_CFG_MODE_SET(x)\ + FIELD_PREP(QS_INJ_GRP_CFG_MODE, x) +#define QS_INJ_GRP_CFG_MODE_GET(x)\ + FIELD_GET(QS_INJ_GRP_CFG_MODE, x) + +#define QS_INJ_GRP_CFG_BYTE_SWAP BIT(0) +#define QS_INJ_GRP_CFG_BYTE_SWAP_SET(x)\ + FIELD_PREP(QS_INJ_GRP_CFG_BYTE_SWAP, x) +#define QS_INJ_GRP_CFG_BYTE_SWAP_GET(x)\ + FIELD_GET(QS_INJ_GRP_CFG_BYTE_SWAP, x) + +/* DEVCPU_QS:INJ:INJ_WR */ +#define QS_INJ_WR(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 8, r, 2, 4) + +/* DEVCPU_QS:INJ:INJ_CTRL */ +#define QS_INJ_CTRL(r) __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 16, r, 2, 4) + +#define QS_INJ_CTRL_GAP_SIZE GENMASK(24, 21) +#define QS_INJ_CTRL_GAP_SIZE_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_GAP_SIZE, x) +#define QS_INJ_CTRL_GAP_SIZE_GET(x)\ + FIELD_GET(QS_INJ_CTRL_GAP_SIZE, x) + +#define QS_INJ_CTRL_EOF BIT(19) +#define QS_INJ_CTRL_EOF_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_EOF, x) +#define QS_INJ_CTRL_EOF_GET(x)\ + FIELD_GET(QS_INJ_CTRL_EOF, x) + +#define QS_INJ_CTRL_SOF BIT(18) +#define QS_INJ_CTRL_SOF_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_SOF, x) +#define QS_INJ_CTRL_SOF_GET(x)\ + FIELD_GET(QS_INJ_CTRL_SOF, x) + +#define QS_INJ_CTRL_VLD_BYTES GENMASK(17, 16) +#define QS_INJ_CTRL_VLD_BYTES_SET(x)\ + FIELD_PREP(QS_INJ_CTRL_VLD_BYTES, x) +#define QS_INJ_CTRL_VLD_BYTES_GET(x)\ + FIELD_GET(QS_INJ_CTRL_VLD_BYTES, x) + +/* DEVCPU_QS:INJ:INJ_STATUS */ +#define QS_INJ_STATUS __REG(TARGET_QS, 0, 1, 36, 0, 1, 40, 24, 0, 1, 4) + +#define QS_INJ_STATUS_WMARK_REACHED GENMASK(5, 4) +#define QS_INJ_STATUS_WMARK_REACHED_SET(x)\ + FIELD_PREP(QS_INJ_STATUS_WMARK_REACHED, x) +#define QS_INJ_STATUS_WMARK_REACHED_GET(x)\ + FIELD_GET(QS_INJ_STATUS_WMARK_REACHED, x) + +#define QS_INJ_STATUS_FIFO_RDY GENMASK(3, 2) +#define QS_INJ_STATUS_FIFO_RDY_SET(x)\ + FIELD_PREP(QS_INJ_STATUS_FIFO_RDY, x) +#define QS_INJ_STATUS_FIFO_RDY_GET(x)\ + FIELD_GET(QS_INJ_STATUS_FIFO_RDY, x) + +/* QSYS:SYSTEM:PORT_MODE */ +#define QSYS_PORT_MODE(r) __REG(TARGET_QSYS, 0, 1, 28008, 0, 1, 216, 0, r, 10, 4) + +#define QSYS_PORT_MODE_DEQUEUE_DIS BIT(1) +#define QSYS_PORT_MODE_DEQUEUE_DIS_SET(x)\ + FIELD_PREP(QSYS_PORT_MODE_DEQUEUE_DIS, x) +#define QSYS_PORT_MODE_DEQUEUE_DIS_GET(x)\ + FIELD_GET(QSYS_PORT_MODE_DEQUEUE_DIS, x) + +/* QSYS:SYSTEM:SWITCH_PORT_MODE */ +#define QSYS_SW_PORT_MODE(r) __REG(TARGET_QSYS, 0, 1, 28008, 0, 1, 216, 80, r, 9, 4) + +#define QSYS_SW_PORT_MODE_PORT_ENA BIT(18) +#define QSYS_SW_PORT_MODE_PORT_ENA_SET(x)\ + FIELD_PREP(QSYS_SW_PORT_MODE_PORT_ENA, x) +#define QSYS_SW_PORT_MODE_PORT_ENA_GET(x)\ + FIELD_GET(QSYS_SW_PORT_MODE_PORT_ENA, x) + +#define QSYS_SW_PORT_MODE_SCH_NEXT_CFG GENMASK(16, 14) +#define QSYS_SW_PORT_MODE_SCH_NEXT_CFG_SET(x)\ + FIELD_PREP(QSYS_SW_PORT_MODE_SCH_NEXT_CFG, x) +#define QSYS_SW_PORT_MODE_SCH_NEXT_CFG_GET(x)\ + FIELD_GET(QSYS_SW_PORT_MODE_SCH_NEXT_CFG, x) + +#define QSYS_SW_PORT_MODE_INGRESS_DROP_MODE BIT(12) +#define QSYS_SW_PORT_MODE_INGRESS_DROP_MODE_SET(x)\ + FIELD_PREP(QSYS_SW_PORT_MODE_INGRESS_DROP_MODE, x) +#define QSYS_SW_PORT_MODE_INGRESS_DROP_MODE_GET(x)\ + FIELD_GET(QSYS_SW_PORT_MODE_INGRESS_DROP_MODE, x) + +#define QSYS_SW_PORT_MODE_TX_PFC_ENA GENMASK(11, 4) +#define QSYS_SW_PORT_MODE_TX_PFC_ENA_SET(x)\ + FIELD_PREP(QSYS_SW_PORT_MODE_TX_PFC_ENA, x) +#define QSYS_SW_PORT_MODE_TX_PFC_ENA_GET(x)\ + FIELD_GET(QSYS_SW_PORT_MODE_TX_PFC_ENA, x) + +#define QSYS_SW_PORT_MODE_AGING_MODE GENMASK(1, 0) +#define QSYS_SW_PORT_MODE_AGING_MODE_SET(x)\ + FIELD_PREP(QSYS_SW_PORT_MODE_AGING_MODE, x) +#define QSYS_SW_PORT_MODE_AGING_MODE_GET(x)\ + FIELD_GET(QSYS_SW_PORT_MODE_AGING_MODE, x) + +/* QSYS:SYSTEM:SW_STATUS */ +#define QSYS_SW_STATUS(r) __REG(TARGET_QSYS, 0, 1, 28008, 0, 1, 216, 164, r, 9, 4) + +#define QSYS_SW_STATUS_EQ_AVAIL GENMASK(7, 0) +#define QSYS_SW_STATUS_EQ_AVAIL_SET(x)\ + FIELD_PREP(QSYS_SW_STATUS_EQ_AVAIL, x) +#define QSYS_SW_STATUS_EQ_AVAIL_GET(x)\ + FIELD_GET(QSYS_SW_STATUS_EQ_AVAIL, x) + +/* QSYS:SYSTEM:CPU_GROUP_MAP */ +#define QSYS_CPU_GROUP_MAP __REG(TARGET_QSYS, 0, 1, 28008, 0, 1, 216, 204, 0, 1, 4) + +/* QSYS:RES_CTRL:RES_CFG */ +#define QSYS_RES_CFG(g) __REG(TARGET_QSYS, 0, 1, 32768, g, 1024, 8, 0, 0, 1, 4) + +/* REW:PORT:PORT_CFG */ +#define REW_PORT_CFG(g) __REG(TARGET_REW, 0, 1, 0, g, 10, 128, 8, 0, 1, 4) + +#define REW_PORT_CFG_NO_REWRITE BIT(0) +#define REW_PORT_CFG_NO_REWRITE_SET(x)\ + FIELD_PREP(REW_PORT_CFG_NO_REWRITE, x) +#define REW_PORT_CFG_NO_REWRITE_GET(x)\ + FIELD_GET(REW_PORT_CFG_NO_REWRITE, x) + +/* SYS:SYSTEM:RESET_CFG */ +#define SYS_RESET_CFG __REG(TARGET_SYS, 0, 1, 4128, 0, 1, 168, 0, 0, 1, 4) + +#define SYS_RESET_CFG_CORE_ENA BIT(0) +#define SYS_RESET_CFG_CORE_ENA_SET(x)\ + FIELD_PREP(SYS_RESET_CFG_CORE_ENA, x) +#define SYS_RESET_CFG_CORE_ENA_GET(x)\ + FIELD_GET(SYS_RESET_CFG_CORE_ENA, x) + +/* SYS:SYSTEM:PORT_MODE */ +#define SYS_PORT_MODE(r) __REG(TARGET_SYS, 0, 1, 4128, 0, 1, 168, 44, r, 10, 4) + +#define SYS_PORT_MODE_INCL_INJ_HDR GENMASK(5, 4) +#define SYS_PORT_MODE_INCL_INJ_HDR_SET(x)\ + FIELD_PREP(SYS_PORT_MODE_INCL_INJ_HDR, x) +#define SYS_PORT_MODE_INCL_INJ_HDR_GET(x)\ + FIELD_GET(SYS_PORT_MODE_INCL_INJ_HDR, x) + +#define SYS_PORT_MODE_INCL_XTR_HDR GENMASK(3, 2) +#define SYS_PORT_MODE_INCL_XTR_HDR_SET(x)\ + FIELD_PREP(SYS_PORT_MODE_INCL_XTR_HDR, x) +#define SYS_PORT_MODE_INCL_XTR_HDR_GET(x)\ + FIELD_GET(SYS_PORT_MODE_INCL_XTR_HDR, x) + +/* SYS:SYSTEM:FRONT_PORT_MODE */ +#define SYS_FRONT_PORT_MODE(r) __REG(TARGET_SYS, 0, 1, 4128, 0, 1, 168, 84, r, 8, 4) + +#define SYS_FRONT_PORT_MODE_HDX_MODE BIT(1) +#define SYS_FRONT_PORT_MODE_HDX_MODE_SET(x)\ + FIELD_PREP(SYS_FRONT_PORT_MODE_HDX_MODE, x) +#define SYS_FRONT_PORT_MODE_HDX_MODE_GET(x)\ + FIELD_GET(SYS_FRONT_PORT_MODE_HDX_MODE, x) + +/* SYS:SYSTEM:FRM_AGING */ +#define SYS_FRM_AGING __REG(TARGET_SYS, 0, 1, 4128, 0, 1, 168, 116, 0, 1, 4) + +#define SYS_FRM_AGING_AGE_TX_ENA BIT(20) +#define SYS_FRM_AGING_AGE_TX_ENA_SET(x)\ + FIELD_PREP(SYS_FRM_AGING_AGE_TX_ENA, x) +#define SYS_FRM_AGING_AGE_TX_ENA_GET(x)\ + FIELD_GET(SYS_FRM_AGING_AGE_TX_ENA, x) + +/* SYS:SYSTEM:STAT_CFG */ +#define SYS_STAT_CFG __REG(TARGET_SYS, 0, 1, 4128, 0, 1, 168, 120, 0, 1, 4) + +#define SYS_STAT_CFG_STAT_VIEW GENMASK(9, 0) +#define SYS_STAT_CFG_STAT_VIEW_SET(x)\ + FIELD_PREP(SYS_STAT_CFG_STAT_VIEW, x) +#define SYS_STAT_CFG_STAT_VIEW_GET(x)\ + FIELD_GET(SYS_STAT_CFG_STAT_VIEW, x) + +/* SYS:PAUSE_CFG:PAUSE_CFG */ +#define SYS_PAUSE_CFG(r) __REG(TARGET_SYS, 0, 1, 4296, 0, 1, 112, 0, r, 9, 4) + +#define SYS_PAUSE_CFG_PAUSE_START GENMASK(18, 10) +#define SYS_PAUSE_CFG_PAUSE_START_SET(x)\ + FIELD_PREP(SYS_PAUSE_CFG_PAUSE_START, x) +#define SYS_PAUSE_CFG_PAUSE_START_GET(x)\ + FIELD_GET(SYS_PAUSE_CFG_PAUSE_START, x) + +#define SYS_PAUSE_CFG_PAUSE_STOP GENMASK(9, 1) +#define SYS_PAUSE_CFG_PAUSE_STOP_SET(x)\ + FIELD_PREP(SYS_PAUSE_CFG_PAUSE_STOP, x) +#define SYS_PAUSE_CFG_PAUSE_STOP_GET(x)\ + FIELD_GET(SYS_PAUSE_CFG_PAUSE_STOP, x) + +#define SYS_PAUSE_CFG_PAUSE_ENA BIT(0) +#define SYS_PAUSE_CFG_PAUSE_ENA_SET(x)\ + FIELD_PREP(SYS_PAUSE_CFG_PAUSE_ENA, x) +#define SYS_PAUSE_CFG_PAUSE_ENA_GET(x)\ + FIELD_GET(SYS_PAUSE_CFG_PAUSE_ENA, x) + +/* SYS:PAUSE_CFG:ATOP */ +#define SYS_ATOP(r) __REG(TARGET_SYS, 0, 1, 4296, 0, 1, 112, 40, r, 9, 4) + +/* SYS:PAUSE_CFG:ATOP_TOT_CFG */ +#define SYS_ATOP_TOT_CFG __REG(TARGET_SYS, 0, 1, 4296, 0, 1, 112, 76, 0, 1, 4) + +/* SYS:PAUSE_CFG:MAC_FC_CFG */ +#define SYS_MAC_FC_CFG(r) __REG(TARGET_SYS, 0, 1, 4296, 0, 1, 112, 80, r, 8, 4) + +#define SYS_MAC_FC_CFG_FC_LINK_SPEED GENMASK(27, 26) +#define SYS_MAC_FC_CFG_FC_LINK_SPEED_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_FC_LINK_SPEED, x) +#define SYS_MAC_FC_CFG_FC_LINK_SPEED_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_FC_LINK_SPEED, x) + +#define SYS_MAC_FC_CFG_FC_LATENCY_CFG GENMASK(25, 20) +#define SYS_MAC_FC_CFG_FC_LATENCY_CFG_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_FC_LATENCY_CFG, x) +#define SYS_MAC_FC_CFG_FC_LATENCY_CFG_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_FC_LATENCY_CFG, x) + +#define SYS_MAC_FC_CFG_ZERO_PAUSE_ENA BIT(18) +#define SYS_MAC_FC_CFG_ZERO_PAUSE_ENA_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_ZERO_PAUSE_ENA, x) +#define SYS_MAC_FC_CFG_ZERO_PAUSE_ENA_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_ZERO_PAUSE_ENA, x) + +#define SYS_MAC_FC_CFG_TX_FC_ENA BIT(17) +#define SYS_MAC_FC_CFG_TX_FC_ENA_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_TX_FC_ENA, x) +#define SYS_MAC_FC_CFG_TX_FC_ENA_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_TX_FC_ENA, x) + +#define SYS_MAC_FC_CFG_RX_FC_ENA BIT(16) +#define SYS_MAC_FC_CFG_RX_FC_ENA_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_RX_FC_ENA, x) +#define SYS_MAC_FC_CFG_RX_FC_ENA_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_RX_FC_ENA, x) + +#define SYS_MAC_FC_CFG_PAUSE_VAL_CFG GENMASK(15, 0) +#define SYS_MAC_FC_CFG_PAUSE_VAL_CFG_SET(x)\ + FIELD_PREP(SYS_MAC_FC_CFG_PAUSE_VAL_CFG, x) +#define SYS_MAC_FC_CFG_PAUSE_VAL_CFG_GET(x)\ + FIELD_GET(SYS_MAC_FC_CFG_PAUSE_VAL_CFG, x) + +/* SYS:STAT:CNT */ +#define SYS_CNT(g) __REG(TARGET_SYS, 0, 1, 0, g, 896, 4, 0, 0, 1, 4) + +/* SYS:RAM_CTRL:RAM_INIT */ +#define SYS_RAM_INIT __REG(TARGET_SYS, 0, 1, 4432, 0, 1, 4, 0, 0, 1, 4) + +#define SYS_RAM_INIT_RAM_INIT BIT(1) +#define SYS_RAM_INIT_RAM_INIT_SET(x)\ + FIELD_PREP(SYS_RAM_INIT_RAM_INIT, x) +#define SYS_RAM_INIT_RAM_INIT_GET(x)\ + FIELD_GET(SYS_RAM_INIT_RAM_INIT, x) + +#endif /* _LAN966X_REGS_H_ */ diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c index 4625d4fb4cde..16266275dd36 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_main.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_main.c @@ -292,6 +292,33 @@ static int sparx5_create_port(struct sparx5 *sparx5, spx5_port->phylink_config.dev = &spx5_port->ndev->dev; spx5_port->phylink_config.type = PHYLINK_NETDEV; spx5_port->phylink_config.pcs_poll = true; + spx5_port->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | + MAC_SYM_PAUSE | MAC_10 | MAC_100 | MAC_1000FD | + MAC_2500FD | MAC_5000FD | MAC_10000FD | MAC_25000FD; + + __set_bit(PHY_INTERFACE_MODE_SGMII, + spx5_port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_QSGMII, + spx5_port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + spx5_port->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_2500BASEX, + spx5_port->phylink_config.supported_interfaces); + + if (spx5_port->conf.bandwidth == SPEED_5000 || + spx5_port->conf.bandwidth == SPEED_10000 || + spx5_port->conf.bandwidth == SPEED_25000) + __set_bit(PHY_INTERFACE_MODE_5GBASER, + spx5_port->phylink_config.supported_interfaces); + + if (spx5_port->conf.bandwidth == SPEED_10000 || + spx5_port->conf.bandwidth == SPEED_25000) + __set_bit(PHY_INTERFACE_MODE_10GBASER, + spx5_port->phylink_config.supported_interfaces); + + if (spx5_port->conf.bandwidth == SPEED_25000) + __set_bit(PHY_INTERFACE_MODE_25GBASER, + spx5_port->phylink_config.supported_interfaces); phylink = phylink_create(&spx5_port->phylink_config, of_fwnode_handle(config->node), diff --git a/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c b/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c index fb74752de0ca..8ba33bc1a001 100644 --- a/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c +++ b/drivers/net/ethernet/microchip/sparx5/sparx5_phylink.c @@ -26,79 +26,6 @@ static bool port_conf_has_changed(struct sparx5_port_config *a, struct sparx5_po return false; } -static void sparx5_phylink_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct sparx5_port *port = netdev_priv(to_net_dev(config->dev)); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - phylink_set(mask, Pause); - phylink_set(mask, Asym_Pause); - - switch (state->interface) { - case PHY_INTERFACE_MODE_5GBASER: - case PHY_INTERFACE_MODE_10GBASER: - case PHY_INTERFACE_MODE_25GBASER: - case PHY_INTERFACE_MODE_NA: - if (port->conf.bandwidth == SPEED_5000) - phylink_set(mask, 5000baseT_Full); - if (port->conf.bandwidth == SPEED_10000) { - phylink_set(mask, 5000baseT_Full); - phylink_set(mask, 10000baseT_Full); - phylink_set(mask, 10000baseCR_Full); - phylink_set(mask, 10000baseSR_Full); - phylink_set(mask, 10000baseLR_Full); - phylink_set(mask, 10000baseLRM_Full); - phylink_set(mask, 10000baseER_Full); - } - if (port->conf.bandwidth == SPEED_25000) { - phylink_set(mask, 5000baseT_Full); - phylink_set(mask, 10000baseT_Full); - phylink_set(mask, 10000baseCR_Full); - phylink_set(mask, 10000baseSR_Full); - phylink_set(mask, 10000baseLR_Full); - phylink_set(mask, 10000baseLRM_Full); - phylink_set(mask, 10000baseER_Full); - phylink_set(mask, 25000baseCR_Full); - phylink_set(mask, 25000baseSR_Full); - } - if (state->interface != PHY_INTERFACE_MODE_NA) - break; - fallthrough; - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_QSGMII: - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - if (state->interface != PHY_INTERFACE_MODE_NA) - break; - fallthrough; - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_2500BASEX: - if (state->interface != PHY_INTERFACE_MODE_2500BASEX) { - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - } - if (state->interface == PHY_INTERFACE_MODE_2500BASEX || - state->interface == PHY_INTERFACE_MODE_NA) { - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - } - break; - default: - linkmode_zero(supported); - return; - } - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void sparx5_phylink_mac_config(struct phylink_config *config, unsigned int mode, const struct phylink_link_state *state) @@ -202,7 +129,7 @@ const struct phylink_pcs_ops sparx5_phylink_pcs_ops = { }; const struct phylink_mac_ops sparx5_phylink_mac_ops = { - .validate = sparx5_phylink_validate, + .validate = phylink_generic_validate, .mac_config = sparx5_phylink_mac_config, .mac_link_down = sparx5_phylink_mac_link_down, .mac_link_up = sparx5_phylink_mac_link_up, diff --git a/drivers/net/ethernet/microsoft/mana/Makefile b/drivers/net/ethernet/microsoft/mana/Makefile index 0edd5bb685f3..e16a4221f571 100644 --- a/drivers/net/ethernet/microsoft/mana/Makefile +++ b/drivers/net/ethernet/microsoft/mana/Makefile @@ -3,4 +3,4 @@ # Makefile for the Microsoft Azure Network Adapter driver obj-$(CONFIG_MICROSOFT_MANA) += mana.o -mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o +mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o mana_bpf.o diff --git a/drivers/net/ethernet/microsoft/mana/mana.h b/drivers/net/ethernet/microsoft/mana/mana.h index d047ee876f12..0c5553887b75 100644 --- a/drivers/net/ethernet/microsoft/mana/mana.h +++ b/drivers/net/ethernet/microsoft/mana/mana.h @@ -298,6 +298,9 @@ struct mana_rxq { struct mana_stats stats; + struct bpf_prog __rcu *bpf_prog; + struct xdp_rxq_info xdp_rxq; + /* MUST BE THE LAST MEMBER: * Each receive buffer has an associated mana_recv_buf_oob. */ @@ -353,6 +356,8 @@ struct mana_port_context { /* This points to an array of num_queues of RQ pointers. */ struct mana_rxq **rxqs; + struct bpf_prog *bpf_prog; + /* Create num_queues EQs, SQs, SQ-CQs, RQs and RQ-CQs, respectively. */ unsigned int max_queues; unsigned int num_queues; @@ -367,6 +372,7 @@ struct mana_port_context { struct mana_ethtool_stats eth_stats; }; +int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev); int mana_config_rss(struct mana_port_context *ac, enum TRI_STATE rx, bool update_hash, bool update_tab); @@ -377,6 +383,13 @@ int mana_detach(struct net_device *ndev, bool from_close); int mana_probe(struct gdma_dev *gd, bool resuming); void mana_remove(struct gdma_dev *gd, bool suspending); +void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev); +u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, + struct xdp_buff *xdp, void *buf_va, uint pkt_len); +struct bpf_prog *mana_xdp_get(struct mana_port_context *apc); +void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog); +int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf); + extern const struct ethtool_ops mana_ethtool_ops; struct mana_obj_spec { diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c new file mode 100644 index 000000000000..1bc8ff388341 --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include <linux/inetdevice.h> +#include <linux/etherdevice.h> +#include <linux/mm.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> +#include <net/xdp.h> + +#include "mana.h" + +void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev) +{ + u16 txq_idx = skb_get_queue_mapping(skb); + struct netdev_queue *ndevtxq; + int rc; + + __skb_push(skb, ETH_HLEN); + + ndevtxq = netdev_get_tx_queue(ndev, txq_idx); + __netif_tx_lock(ndevtxq, smp_processor_id()); + + rc = mana_start_xmit(skb, ndev); + + __netif_tx_unlock(ndevtxq); + + if (dev_xmit_complete(rc)) + return; + + dev_kfree_skb_any(skb); + ndev->stats.tx_dropped++; +} + +u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, + struct xdp_buff *xdp, void *buf_va, uint pkt_len) +{ + struct bpf_prog *prog; + u32 act = XDP_PASS; + + rcu_read_lock(); + prog = rcu_dereference(rxq->bpf_prog); + + if (!prog) + goto out; + + xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq); + xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, false); + + act = bpf_prog_run_xdp(prog, xdp); + + switch (act) { + case XDP_PASS: + case XDP_TX: + case XDP_DROP: + break; + + case XDP_ABORTED: + trace_xdp_exception(ndev, prog, act); + break; + + default: + bpf_warn_invalid_xdp_action(act); + } + +out: + rcu_read_unlock(); + + return act; +} + +static unsigned int mana_xdp_fraglen(unsigned int len) +{ + return SKB_DATA_ALIGN(len) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); +} + +struct bpf_prog *mana_xdp_get(struct mana_port_context *apc) +{ + ASSERT_RTNL(); + + return apc->bpf_prog; +} + +static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc) +{ + return rtnl_dereference(apc->rxqs[0]->bpf_prog); +} + +/* Set xdp program on channels */ +void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog) +{ + struct bpf_prog *old_prog = mana_chn_xdp_get(apc); + unsigned int num_queues = apc->num_queues; + int i; + + ASSERT_RTNL(); + + if (old_prog == prog) + return; + + if (prog) + bpf_prog_add(prog, num_queues); + + for (i = 0; i < num_queues; i++) + rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog); + + if (old_prog) + for (i = 0; i < num_queues; i++) + bpf_prog_put(old_prog); +} + +static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct bpf_prog *old_prog; + int buf_max; + + old_prog = mana_xdp_get(apc); + + if (!old_prog && !prog) + return 0; + + buf_max = XDP_PACKET_HEADROOM + mana_xdp_fraglen(ndev->mtu + ETH_HLEN); + if (prog && buf_max > PAGE_SIZE) { + netdev_err(ndev, "XDP: mtu:%u too large, buf_max:%u\n", + ndev->mtu, buf_max); + NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large"); + + return -EOPNOTSUPP; + } + + /* One refcnt of the prog is hold by the caller already, so + * don't increase refcnt for this one. + */ + apc->bpf_prog = prog; + + if (old_prog) + bpf_prog_put(old_prog); + + if (apc->port_is_up) + mana_chn_setxdp(apc, prog); + + return 0; +} + +int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf) +{ + struct netlink_ext_ack *extack = bpf->extack; + int ret; + + switch (bpf->command) { + case XDP_SETUP_PROG: + return mana_xdp_set(ndev, bpf->prog, extack); + + default: + return -EOPNOTSUPP; + } + + return ret; +} diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c index 72cbf45c42d8..c1d5a374b967 100644 --- a/drivers/net/ethernet/microsoft/mana/mana_en.c +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -125,7 +125,7 @@ frag_err: return -ENOMEM; } -static int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) +int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) { enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; struct mana_port_context *apc = netdev_priv(ndev); @@ -378,6 +378,7 @@ static const struct net_device_ops mana_devops = { .ndo_start_xmit = mana_start_xmit, .ndo_validate_addr = eth_validate_addr, .ndo_get_stats64 = mana_get_stats64, + .ndo_bpf = mana_bpf, }; static void mana_cleanup_port_context(struct mana_port_context *apc) @@ -906,6 +907,25 @@ static void mana_post_pkt_rxq(struct mana_rxq *rxq) WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); } +static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len, + struct xdp_buff *xdp) +{ + struct sk_buff *skb = build_skb(buf_va, PAGE_SIZE); + + if (!skb) + return NULL; + + if (xdp->data_hard_start) { + skb_reserve(skb, xdp->data - xdp->data_hard_start); + skb_put(skb, xdp->data_end - xdp->data); + } else { + skb_reserve(skb, XDP_PACKET_HEADROOM); + skb_put(skb, pkt_len); + } + + return skb; +} + static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq) { @@ -914,8 +934,10 @@ static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe, uint pkt_len = cqe->ppi[0].pkt_len; u16 rxq_idx = rxq->rxq_idx; struct napi_struct *napi; + struct xdp_buff xdp = {}; struct sk_buff *skb; u32 hash_value; + u32 act; rxq->rx_cq.work_done++; napi = &rxq->rx_cq.napi; @@ -925,15 +947,16 @@ static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe, return; } - skb = build_skb(buf_va, PAGE_SIZE); + act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); - if (!skb) { - free_page((unsigned long)buf_va); - ++ndev->stats.rx_dropped; - return; - } + if (act != XDP_PASS && act != XDP_TX) + goto drop; + + skb = mana_build_skb(buf_va, pkt_len, &xdp); + + if (!skb) + goto drop; - skb_put(skb, pkt_len); skb->dev = napi->dev; skb->protocol = eth_type_trans(skb, ndev); @@ -954,12 +977,24 @@ static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe, skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3); } + if (act == XDP_TX) { + skb_set_queue_mapping(skb, rxq_idx); + mana_xdp_tx(skb, ndev); + return; + } + napi_gro_receive(napi, skb); u64_stats_update_begin(&rx_stats->syncp); rx_stats->packets++; rx_stats->bytes += pkt_len; u64_stats_update_end(&rx_stats->syncp); + return; + +drop: + free_page((unsigned long)buf_va); + ++ndev->stats.rx_dropped; + return; } static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, @@ -1016,7 +1051,7 @@ static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, new_page = alloc_page(GFP_ATOMIC); if (new_page) { - da = dma_map_page(dev, new_page, 0, rxq->datasize, + da = dma_map_page(dev, new_page, XDP_PACKET_HEADROOM, rxq->datasize, DMA_FROM_DEVICE); if (dma_mapping_error(dev, da)) { @@ -1291,6 +1326,9 @@ static void mana_destroy_rxq(struct mana_port_context *apc, napi_synchronize(napi); napi_disable(napi); + + xdp_rxq_info_unreg(&rxq->xdp_rxq); + netif_napi_del(napi); mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); @@ -1342,7 +1380,8 @@ static int mana_alloc_rx_wqe(struct mana_port_context *apc, if (!page) return -ENOMEM; - da = dma_map_page(dev, page, 0, rxq->datasize, DMA_FROM_DEVICE); + da = dma_map_page(dev, page, XDP_PACKET_HEADROOM, rxq->datasize, + DMA_FROM_DEVICE); if (dma_mapping_error(dev, da)) { __free_page(page); @@ -1485,6 +1524,12 @@ static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, gc->cq_table[cq->gdma_id] = cq->gdma_cq; netif_napi_add(ndev, &cq->napi, mana_poll, 1); + + WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, + cq->napi.napi_id)); + WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, + MEM_TYPE_PAGE_SHARED, NULL)); + napi_enable(&cq->napi); mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); @@ -1650,6 +1695,8 @@ int mana_alloc_queues(struct net_device *ndev) if (err) goto destroy_vport; + mana_chn_setxdp(apc, mana_xdp_get(apc)); + return 0; destroy_vport: @@ -1698,6 +1745,8 @@ static int mana_dealloc_queues(struct net_device *ndev) if (apc->port_is_up) return -EINVAL; + mana_chn_setxdp(apc, NULL); + /* No packet can be transmitted now since apc->port_is_up is false. * There is still a tiny chance that mana_poll_tx_cq() can re-enable * a txq because it may not timely see apc->port_is_up being cleared diff --git a/drivers/net/ethernet/mscc/Makefile b/drivers/net/ethernet/mscc/Makefile index 722c27694b21..dfa939376d6c 100644 --- a/drivers/net/ethernet/mscc/Makefile +++ b/drivers/net/ethernet/mscc/Makefile @@ -7,7 +7,8 @@ mscc_ocelot_switch_lib-y := \ ocelot_vcap.o \ ocelot_flower.o \ ocelot_ptp.o \ - ocelot_devlink.o + ocelot_devlink.o \ + vsc7514_regs.o mscc_ocelot_switch_lib-$(CONFIG_BRIDGE_MRP) += ocelot_mrp.o obj-$(CONFIG_MSCC_OCELOT_SWITCH) += mscc_ocelot.o mscc_ocelot-y := \ diff --git a/drivers/net/ethernet/mscc/ocelot.c b/drivers/net/ethernet/mscc/ocelot.c index 1e4ad953cffb..b1856d8c944b 100644 --- a/drivers/net/ethernet/mscc/ocelot.c +++ b/drivers/net/ethernet/mscc/ocelot.c @@ -61,9 +61,9 @@ static void ocelot_mact_select(struct ocelot *ocelot, } -int ocelot_mact_learn(struct ocelot *ocelot, int port, - const unsigned char mac[ETH_ALEN], - unsigned int vid, enum macaccess_entry_type type) +static int __ocelot_mact_learn(struct ocelot *ocelot, int port, + const unsigned char mac[ETH_ALEN], + unsigned int vid, enum macaccess_entry_type type) { u32 cmd = ANA_TABLES_MACACCESS_VALID | ANA_TABLES_MACACCESS_DEST_IDX(port) | @@ -83,8 +83,6 @@ int ocelot_mact_learn(struct ocelot *ocelot, int port, if (mc_ports & BIT(ocelot->num_phys_ports)) cmd |= ANA_TABLES_MACACCESS_MAC_CPU_COPY; - mutex_lock(&ocelot->mact_lock); - ocelot_mact_select(ocelot, mac, vid); /* Issue a write command */ @@ -92,9 +90,20 @@ int ocelot_mact_learn(struct ocelot *ocelot, int port, err = ocelot_mact_wait_for_completion(ocelot); + return err; +} + +int ocelot_mact_learn(struct ocelot *ocelot, int port, + const unsigned char mac[ETH_ALEN], + unsigned int vid, enum macaccess_entry_type type) +{ + int ret; + + mutex_lock(&ocelot->mact_lock); + ret = __ocelot_mact_learn(ocelot, port, mac, vid, type); mutex_unlock(&ocelot->mact_lock); - return err; + return ret; } EXPORT_SYMBOL(ocelot_mact_learn); @@ -120,6 +129,66 @@ int ocelot_mact_forget(struct ocelot *ocelot, } EXPORT_SYMBOL(ocelot_mact_forget); +int ocelot_mact_lookup(struct ocelot *ocelot, int *dst_idx, + const unsigned char mac[ETH_ALEN], + unsigned int vid, enum macaccess_entry_type *type) +{ + int val; + + mutex_lock(&ocelot->mact_lock); + + ocelot_mact_select(ocelot, mac, vid); + + /* Issue a read command with MACACCESS_VALID=1. */ + ocelot_write(ocelot, ANA_TABLES_MACACCESS_VALID | + ANA_TABLES_MACACCESS_MAC_TABLE_CMD(MACACCESS_CMD_READ), + ANA_TABLES_MACACCESS); + + if (ocelot_mact_wait_for_completion(ocelot)) { + mutex_unlock(&ocelot->mact_lock); + return -ETIMEDOUT; + } + + /* Read back the entry flags */ + val = ocelot_read(ocelot, ANA_TABLES_MACACCESS); + + mutex_unlock(&ocelot->mact_lock); + + if (!(val & ANA_TABLES_MACACCESS_VALID)) + return -ENOENT; + + *dst_idx = ANA_TABLES_MACACCESS_DEST_IDX_X(val); + *type = ANA_TABLES_MACACCESS_ENTRYTYPE_X(val); + + return 0; +} +EXPORT_SYMBOL(ocelot_mact_lookup); + +int ocelot_mact_learn_streamdata(struct ocelot *ocelot, int dst_idx, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type, + int sfid, int ssid) +{ + int ret; + + mutex_lock(&ocelot->mact_lock); + + ocelot_write(ocelot, + (sfid < 0 ? 0 : ANA_TABLES_STREAMDATA_SFID_VALID) | + ANA_TABLES_STREAMDATA_SFID(sfid) | + (ssid < 0 ? 0 : ANA_TABLES_STREAMDATA_SSID_VALID) | + ANA_TABLES_STREAMDATA_SSID(ssid), + ANA_TABLES_STREAMDATA); + + ret = __ocelot_mact_learn(ocelot, dst_idx, mac, vid, type); + + mutex_unlock(&ocelot->mact_lock); + + return ret; +} +EXPORT_SYMBOL(ocelot_mact_learn_streamdata); + static void ocelot_mact_init(struct ocelot *ocelot) { /* Configure the learning mode entries attributes: @@ -594,9 +663,17 @@ void ocelot_phylink_mac_link_down(struct ocelot *ocelot, int port, struct ocelot_port *ocelot_port = ocelot->ports[port]; int err; + ocelot_port->speed = SPEED_UNKNOWN; + ocelot_port_rmwl(ocelot_port, 0, DEV_MAC_ENA_CFG_RX_ENA, DEV_MAC_ENA_CFG); + if (ocelot->ops->cut_through_fwd) { + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ops->cut_through_fwd(ocelot); + mutex_unlock(&ocelot->fwd_domain_lock); + } + ocelot_fields_write(ocelot, port, QSYS_SWITCH_PORT_MODE_PORT_ENA, 0); err = ocelot_port_flush(ocelot, port); @@ -628,6 +705,8 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port, int mac_speed, mode = 0; u32 mac_fc_cfg; + ocelot_port->speed = speed; + /* The MAC might be integrated in systems where the MAC speed is fixed * and it's the PCS who is performing the rate adaptation, so we have * to write "1000Mbps" into the LINK_SPEED field of DEV_CLOCK_CFG @@ -700,6 +779,15 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port, ocelot_port_writel(ocelot_port, DEV_MAC_ENA_CFG_RX_ENA | DEV_MAC_ENA_CFG_TX_ENA, DEV_MAC_ENA_CFG); + /* If the port supports cut-through forwarding, update the masks before + * enabling forwarding on the port. + */ + if (ocelot->ops->cut_through_fwd) { + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ops->cut_through_fwd(ocelot); + mutex_unlock(&ocelot->fwd_domain_lock); + } + /* Core: Enable port for frame transfer */ ocelot_fields_write(ocelot, port, QSYS_SWITCH_PORT_MODE_PORT_ENA, 1); @@ -1711,15 +1799,18 @@ static u32 ocelot_get_bond_mask(struct ocelot *ocelot, struct net_device *bond, return mask; } -static u32 ocelot_get_bridge_fwd_mask(struct ocelot *ocelot, int src_port, - struct net_device *bridge) +u32 ocelot_get_bridge_fwd_mask(struct ocelot *ocelot, int src_port) { struct ocelot_port *ocelot_port = ocelot->ports[src_port]; + const struct net_device *bridge; u32 mask = 0; int port; - if (!ocelot_port || ocelot_port->bridge != bridge || - ocelot_port->stp_state != BR_STATE_FORWARDING) + if (!ocelot_port || ocelot_port->stp_state != BR_STATE_FORWARDING) + return 0; + + bridge = ocelot_port->bridge; + if (!bridge) return 0; for (port = 0; port < ocelot->num_phys_ports; port++) { @@ -1735,8 +1826,9 @@ static u32 ocelot_get_bridge_fwd_mask(struct ocelot *ocelot, int src_port, return mask; } +EXPORT_SYMBOL_GPL(ocelot_get_bridge_fwd_mask); -static u32 ocelot_get_dsa_8021q_cpu_mask(struct ocelot *ocelot) +u32 ocelot_get_dsa_8021q_cpu_mask(struct ocelot *ocelot) { u32 mask = 0; int port; @@ -1753,12 +1845,22 @@ static u32 ocelot_get_dsa_8021q_cpu_mask(struct ocelot *ocelot) return mask; } +EXPORT_SYMBOL_GPL(ocelot_get_dsa_8021q_cpu_mask); -void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot) +void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot, bool joining) { unsigned long cpu_fwd_mask; int port; + lockdep_assert_held(&ocelot->fwd_domain_lock); + + /* If cut-through forwarding is supported, update the masks before a + * port joins the forwarding domain, to avoid potential underruns if it + * has the highest speed from the new domain. + */ + if (joining && ocelot->ops->cut_through_fwd) + ocelot->ops->cut_through_fwd(ocelot); + /* If a DSA tag_8021q CPU exists, it needs to be included in the * regular forwarding path of the front ports regardless of whether * those are bridged or standalone. @@ -1786,10 +1888,9 @@ void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot) mask = GENMASK(ocelot->num_phys_ports - 1, 0); mask &= ~cpu_fwd_mask; } else if (ocelot_port->bridge) { - struct net_device *bridge = ocelot_port->bridge; struct net_device *bond = ocelot_port->bond; - mask = ocelot_get_bridge_fwd_mask(ocelot, port, bridge); + mask = ocelot_get_bridge_fwd_mask(ocelot, port); mask |= cpu_fwd_mask; mask &= ~BIT(port); if (bond) { @@ -1806,6 +1907,16 @@ void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot) ocelot_write_rix(ocelot, mask, ANA_PGID_PGID, PGID_SRC + port); } + + /* If cut-through forwarding is supported and a port is leaving, there + * is a chance that cut-through was disabled on the other ports due to + * the port which is leaving (it has a higher link speed). We need to + * update the cut-through masks of the remaining ports no earlier than + * after the port has left, to prevent underruns from happening between + * the cut-through update and the forwarding domain update. + */ + if (!joining && ocelot->ops->cut_through_fwd) + ocelot->ops->cut_through_fwd(ocelot); } EXPORT_SYMBOL(ocelot_apply_bridge_fwd_mask); @@ -1814,6 +1925,8 @@ void ocelot_bridge_stp_state_set(struct ocelot *ocelot, int port, u8 state) struct ocelot_port *ocelot_port = ocelot->ports[port]; u32 learn_ena = 0; + mutex_lock(&ocelot->fwd_domain_lock); + ocelot_port->stp_state = state; if ((state == BR_STATE_LEARNING || state == BR_STATE_FORWARDING) && @@ -1823,7 +1936,9 @@ void ocelot_bridge_stp_state_set(struct ocelot *ocelot, int port, u8 state) ocelot_rmw_gix(ocelot, learn_ena, ANA_PORT_PORT_CFG_LEARN_ENA, ANA_PORT_PORT_CFG, port); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, state == BR_STATE_FORWARDING); + + mutex_unlock(&ocelot->fwd_domain_lock); } EXPORT_SYMBOL(ocelot_bridge_stp_state_set); @@ -2053,9 +2168,13 @@ void ocelot_port_bridge_join(struct ocelot *ocelot, int port, { struct ocelot_port *ocelot_port = ocelot->ports[port]; + mutex_lock(&ocelot->fwd_domain_lock); + ocelot_port->bridge = bridge; - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, true); + + mutex_unlock(&ocelot->fwd_domain_lock); } EXPORT_SYMBOL(ocelot_port_bridge_join); @@ -2064,11 +2183,15 @@ void ocelot_port_bridge_leave(struct ocelot *ocelot, int port, { struct ocelot_port *ocelot_port = ocelot->ports[port]; + mutex_lock(&ocelot->fwd_domain_lock); + ocelot_port->bridge = NULL; ocelot_port_set_pvid(ocelot, port, NULL); ocelot_port_manage_port_tag(ocelot, port); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, false); + + mutex_unlock(&ocelot->fwd_domain_lock); } EXPORT_SYMBOL(ocelot_port_bridge_leave); @@ -2190,12 +2313,16 @@ int ocelot_port_lag_join(struct ocelot *ocelot, int port, if (info->tx_type != NETDEV_LAG_TX_TYPE_HASH) return -EOPNOTSUPP; + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ports[port]->bond = bond; ocelot_setup_logical_port_ids(ocelot); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, true); ocelot_set_aggr_pgids(ocelot); + mutex_unlock(&ocelot->fwd_domain_lock); + return 0; } EXPORT_SYMBOL(ocelot_port_lag_join); @@ -2203,11 +2330,15 @@ EXPORT_SYMBOL(ocelot_port_lag_join); void ocelot_port_lag_leave(struct ocelot *ocelot, int port, struct net_device *bond) { + mutex_lock(&ocelot->fwd_domain_lock); + ocelot->ports[port]->bond = NULL; ocelot_setup_logical_port_ids(ocelot); - ocelot_apply_bridge_fwd_mask(ocelot); + ocelot_apply_bridge_fwd_mask(ocelot, false); ocelot_set_aggr_pgids(ocelot); + + mutex_unlock(&ocelot->fwd_domain_lock); } EXPORT_SYMBOL(ocelot_port_lag_leave); @@ -2498,6 +2629,7 @@ int ocelot_init(struct ocelot *ocelot) mutex_init(&ocelot->stats_lock); mutex_init(&ocelot->ptp_lock); mutex_init(&ocelot->mact_lock); + mutex_init(&ocelot->fwd_domain_lock); spin_lock_init(&ocelot->ptp_clock_lock); spin_lock_init(&ocelot->ts_id_lock); snprintf(queue_name, sizeof(queue_name), "%s-stats", @@ -2521,6 +2653,9 @@ int ocelot_init(struct ocelot *ocelot) ocelot_vcap_init(ocelot); ocelot_cpu_port_init(ocelot); + if (ocelot->ops->psfp_init) + ocelot->ops->psfp_init(ocelot); + for (port = 0; port < ocelot->num_phys_ports; port++) { /* Clear all counters (5 groups) */ ocelot_write(ocelot, SYS_STAT_CFG_STAT_VIEW(port) | diff --git a/drivers/net/ethernet/mscc/ocelot.h b/drivers/net/ethernet/mscc/ocelot.h index e43da09b8f91..1eb0b5ad51e9 100644 --- a/drivers/net/ethernet/mscc/ocelot.h +++ b/drivers/net/ethernet/mscc/ocelot.h @@ -55,19 +55,6 @@ struct ocelot_dump_ctx { int idx; }; -/* MAC table entry types. - * ENTRYTYPE_NORMAL is subject to aging. - * ENTRYTYPE_LOCKED is not subject to aging. - * ENTRYTYPE_MACv4 is not subject to aging. For IPv4 multicast. - * ENTRYTYPE_MACv6 is not subject to aging. For IPv6 multicast. - */ -enum macaccess_entry_type { - ENTRYTYPE_NORMAL = 0, - ENTRYTYPE_LOCKED, - ENTRYTYPE_MACv4, - ENTRYTYPE_MACv6, -}; - /* A (PGID) port mask structure, encoding the 2^ocelot->num_phys_ports * possibilities of egress port masks for L2 multicast traffic. * For a switch with 9 user ports, there are 512 possible port masks, but the diff --git a/drivers/net/ethernet/mscc/ocelot_flower.c b/drivers/net/ethernet/mscc/ocelot_flower.c index 769a8159373e..58fce173f95b 100644 --- a/drivers/net/ethernet/mscc/ocelot_flower.c +++ b/drivers/net/ethernet/mscc/ocelot_flower.c @@ -20,6 +20,9 @@ (1 * VCAP_BLOCK + (lookup) * VCAP_LOOKUP) #define VCAP_IS2_CHAIN(lookup, pag) \ (2 * VCAP_BLOCK + (lookup) * VCAP_LOOKUP + (pag)) +/* PSFP chain and block ID */ +#define PSFP_BLOCK_ID OCELOT_NUM_VCAP_BLOCKS +#define OCELOT_PSFP_CHAIN (3 * VCAP_BLOCK) static int ocelot_chain_to_block(int chain, bool ingress) { @@ -46,6 +49,9 @@ static int ocelot_chain_to_block(int chain, bool ingress) if (chain == VCAP_IS2_CHAIN(lookup, pag)) return VCAP_IS2; + if (chain == OCELOT_PSFP_CHAIN) + return PSFP_BLOCK_ID; + return -EOPNOTSUPP; } @@ -84,7 +90,8 @@ static bool ocelot_is_goto_target_valid(int goto_target, int chain, goto_target == VCAP_IS1_CHAIN(1) || goto_target == VCAP_IS1_CHAIN(2) || goto_target == VCAP_IS2_CHAIN(0, 0) || - goto_target == VCAP_IS2_CHAIN(1, 0)); + goto_target == VCAP_IS2_CHAIN(1, 0) || + goto_target == OCELOT_PSFP_CHAIN); if (chain == VCAP_IS1_CHAIN(0)) return (goto_target == VCAP_IS1_CHAIN(1)); @@ -111,7 +118,11 @@ static bool ocelot_is_goto_target_valid(int goto_target, int chain, if (chain == VCAP_IS2_CHAIN(0, pag)) return (goto_target == VCAP_IS2_CHAIN(1, pag)); - /* VCAP IS2 lookup 1 cannot jump anywhere */ + /* VCAP IS2 lookup 1 can goto to PSFP block if hardware support */ + for (pag = 0; pag < VCAP_IS2_NUM_PAG; pag++) + if (chain == VCAP_IS2_CHAIN(1, pag)) + return (goto_target == OCELOT_PSFP_CHAIN); + return false; } @@ -211,6 +222,7 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, const struct flow_action_entry *a; enum ocelot_tag_tpid_sel tpid; int i, chain, egress_port; + u32 pol_ix, pol_max; u64 rate; int err; @@ -269,10 +281,14 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; case FLOW_ACTION_POLICE: + if (filter->block_id == PSFP_BLOCK_ID) { + filter->type = OCELOT_PSFP_FILTER_OFFLOAD; + break; + } if (filter->block_id != VCAP_IS2 || filter->lookup != 0) { NL_SET_ERR_MSG_MOD(extack, - "Police action can only be offloaded to VCAP IS2 lookup 0"); + "Police action can only be offloaded to VCAP IS2 lookup 0 or PSFP"); return -EOPNOTSUPP; } if (filter->goto_target != -1) { @@ -286,6 +302,20 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, return -EOPNOTSUPP; } filter->action.police_ena = true; + + pol_ix = a->police.index + ocelot->vcap_pol.base; + pol_max = ocelot->vcap_pol.max; + + if (ocelot->vcap_pol.max2 && pol_ix > pol_max) { + pol_ix += ocelot->vcap_pol.base2 - pol_max - 1; + pol_max = ocelot->vcap_pol.max2; + } + + if (pol_ix >= pol_max) + return -EINVAL; + + filter->action.pol_ix = pol_ix; + rate = a->police.rate_bytes_ps; filter->action.pol.rate = div_u64(rate, 1000) * 8; filter->action.pol.burst = a->police.burst; @@ -399,6 +429,14 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, filter->action.pcp_a_val = a->vlan.prio; filter->type = OCELOT_VCAP_FILTER_OFFLOAD; break; + case FLOW_ACTION_GATE: + if (filter->block_id != PSFP_BLOCK_ID) { + NL_SET_ERR_MSG_MOD(extack, + "Gate action can only be offloaded to PSFP chain"); + return -EOPNOTSUPP; + } + filter->type = OCELOT_PSFP_FILTER_OFFLOAD; + break; default: NL_SET_ERR_MSG_MOD(extack, "Cannot offload action"); return -EOPNOTSUPP; @@ -407,7 +445,7 @@ static int ocelot_flower_parse_action(struct ocelot *ocelot, int port, if (filter->goto_target == -1) { if ((filter->block_id == VCAP_IS2 && filter->lookup == 1) || - chain == 0) { + chain == 0 || filter->block_id == PSFP_BLOCK_ID) { allow_missing_goto_target = true; } else { NL_SET_ERR_MSG_MOD(extack, "Missing GOTO action"); @@ -689,6 +727,10 @@ static int ocelot_flower_parse(struct ocelot *ocelot, int port, bool ingress, if (ret) return ret; + /* PSFP filter need to parse key by stream identification function. */ + if (filter->type == OCELOT_PSFP_FILTER_OFFLOAD) + return 0; + return ocelot_flower_parse_key(ocelot, port, ingress, f, filter); } @@ -792,6 +834,15 @@ int ocelot_cls_flower_replace(struct ocelot *ocelot, int port, if (filter->type == OCELOT_VCAP_FILTER_DUMMY) return ocelot_vcap_dummy_filter_add(ocelot, filter); + if (filter->type == OCELOT_PSFP_FILTER_OFFLOAD) { + kfree(filter); + if (ocelot->ops->psfp_filter_add) + return ocelot->ops->psfp_filter_add(ocelot, port, f); + + NL_SET_ERR_MSG_MOD(extack, "PSFP chain is not supported in HW"); + return -EOPNOTSUPP; + } + return ocelot_vcap_filter_add(ocelot, filter, f->common.extack); } EXPORT_SYMBOL_GPL(ocelot_cls_flower_replace); @@ -807,6 +858,13 @@ int ocelot_cls_flower_destroy(struct ocelot *ocelot, int port, if (block_id < 0) return 0; + if (block_id == PSFP_BLOCK_ID) { + if (ocelot->ops->psfp_filter_del) + return ocelot->ops->psfp_filter_del(ocelot, f); + + return -EOPNOTSUPP; + } + block = &ocelot->block[block_id]; filter = ocelot_vcap_block_find_filter_by_id(block, f->cookie, true); @@ -825,12 +883,25 @@ int ocelot_cls_flower_stats(struct ocelot *ocelot, int port, { struct ocelot_vcap_filter *filter; struct ocelot_vcap_block *block; + struct flow_stats stats = {0}; int block_id, ret; block_id = ocelot_chain_to_block(f->common.chain_index, ingress); if (block_id < 0) return 0; + if (block_id == PSFP_BLOCK_ID) { + if (ocelot->ops->psfp_stats_get) { + ret = ocelot->ops->psfp_stats_get(ocelot, f, &stats); + if (ret) + return ret; + + goto stats_update; + } + + return -EOPNOTSUPP; + } + block = &ocelot->block[block_id]; filter = ocelot_vcap_block_find_filter_by_id(block, f->cookie, true); @@ -841,7 +912,10 @@ int ocelot_cls_flower_stats(struct ocelot *ocelot, int port, if (ret) return ret; - flow_stats_update(&f->stats, 0x0, filter->stats.pkts, 0, 0x0, + stats.pkts = filter->stats.pkts; + +stats_update: + flow_stats_update(&f->stats, 0x0, stats.pkts, stats.drops, 0x0, FLOW_ACTION_HW_STATS_IMMEDIATE); return 0; } diff --git a/drivers/net/ethernet/mscc/ocelot_net.c b/drivers/net/ethernet/mscc/ocelot_net.c index eaeba60b1bba..0fcf359a6975 100644 --- a/drivers/net/ethernet/mscc/ocelot_net.c +++ b/drivers/net/ethernet/mscc/ocelot_net.c @@ -1498,40 +1498,6 @@ struct notifier_block ocelot_switchdev_blocking_nb __read_mostly = { .notifier_call = ocelot_switchdev_blocking_event, }; -static void vsc7514_phylink_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct ocelot_port_private *priv = netdev_priv(ndev); - struct ocelot_port *ocelot_port = &priv->port; - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = {}; - - if (state->interface != PHY_INTERFACE_MODE_NA && - state->interface != ocelot_port->phy_mode) { - linkmode_zero(supported); - return; - } - - phylink_set_port_modes(mask); - - phylink_set(mask, Pause); - phylink_set(mask, Autoneg); - phylink_set(mask, Asym_Pause); - phylink_set(mask, 10baseT_Half); - phylink_set(mask, 10baseT_Full); - phylink_set(mask, 100baseT_Half); - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 1000baseT_Half); - phylink_set(mask, 1000baseT_Full); - phylink_set(mask, 1000baseX_Full); - phylink_set(mask, 2500baseT_Full); - phylink_set(mask, 2500baseX_Full); - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void vsc7514_phylink_mac_config(struct phylink_config *config, unsigned int link_an_mode, const struct phylink_link_state *state) @@ -1590,7 +1556,7 @@ static void vsc7514_phylink_mac_link_up(struct phylink_config *config, } static const struct phylink_mac_ops ocelot_phylink_ops = { - .validate = vsc7514_phylink_validate, + .validate = phylink_generic_validate, .mac_config = vsc7514_phylink_mac_config, .mac_link_down = vsc7514_phylink_mac_link_down, .mac_link_up = vsc7514_phylink_mac_link_up, @@ -1654,6 +1620,11 @@ static int ocelot_port_phylink_create(struct ocelot *ocelot, int port, priv->phylink_config.dev = &priv->dev->dev; priv->phylink_config.type = PHYLINK_NETDEV; + priv->phylink_config.mac_capabilities = MAC_ASYM_PAUSE | MAC_SYM_PAUSE | + MAC_10 | MAC_100 | MAC_1000FD | MAC_2500FD; + + __set_bit(ocelot_port->phy_mode, + priv->phylink_config.supported_interfaces); phylink = phylink_create(&priv->phylink_config, of_fwnode_handle(portnp), diff --git a/drivers/net/ethernet/mscc/ocelot_vcap.c b/drivers/net/ethernet/mscc/ocelot_vcap.c index 337cd08b1a54..d3544413a8a4 100644 --- a/drivers/net/ethernet/mscc/ocelot_vcap.c +++ b/drivers/net/ethernet/mscc/ocelot_vcap.c @@ -887,10 +887,18 @@ static void vcap_entry_set(struct ocelot *ocelot, int ix, return es0_entry_set(ocelot, ix, filter); } -static int ocelot_vcap_policer_add(struct ocelot *ocelot, u32 pol_ix, - struct ocelot_policer *pol) +struct vcap_policer_entry { + struct list_head list; + refcount_t refcount; + u32 pol_ix; +}; + +int ocelot_vcap_policer_add(struct ocelot *ocelot, u32 pol_ix, + struct ocelot_policer *pol) { struct qos_policer_conf pp = { 0 }; + struct vcap_policer_entry *tmp; + int ret; if (!pol) return -EINVAL; @@ -899,57 +907,74 @@ static int ocelot_vcap_policer_add(struct ocelot *ocelot, u32 pol_ix, pp.pir = pol->rate; pp.pbs = pol->burst; - return qos_policer_conf_set(ocelot, 0, pol_ix, &pp); + list_for_each_entry(tmp, &ocelot->vcap_pol.pol_list, list) + if (tmp->pol_ix == pol_ix) { + refcount_inc(&tmp->refcount); + return 0; + } + + tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); + if (!tmp) + return -ENOMEM; + + ret = qos_policer_conf_set(ocelot, 0, pol_ix, &pp); + if (ret) { + kfree(tmp); + return ret; + } + + tmp->pol_ix = pol_ix; + refcount_set(&tmp->refcount, 1); + list_add_tail(&tmp->list, &ocelot->vcap_pol.pol_list); + + return 0; } +EXPORT_SYMBOL(ocelot_vcap_policer_add); -static void ocelot_vcap_policer_del(struct ocelot *ocelot, - struct ocelot_vcap_block *block, - u32 pol_ix) +int ocelot_vcap_policer_del(struct ocelot *ocelot, u32 pol_ix) { - struct ocelot_vcap_filter *filter; struct qos_policer_conf pp = {0}; - int index = -1; - - if (pol_ix < block->pol_lpr) - return; - - list_for_each_entry(filter, &block->rules, list) { - index++; - if (filter->block_id == VCAP_IS2 && - filter->action.police_ena && - filter->action.pol_ix < pol_ix) { - filter->action.pol_ix += 1; - ocelot_vcap_policer_add(ocelot, filter->action.pol_ix, - &filter->action.pol); - is2_entry_set(ocelot, index, filter); + struct vcap_policer_entry *tmp, *n; + u8 z = 0; + + list_for_each_entry_safe(tmp, n, &ocelot->vcap_pol.pol_list, list) + if (tmp->pol_ix == pol_ix) { + z = refcount_dec_and_test(&tmp->refcount); + if (z) { + list_del(&tmp->list); + kfree(tmp); + } } - } - pp.mode = MSCC_QOS_RATE_MODE_DISABLED; - qos_policer_conf_set(ocelot, 0, pol_ix, &pp); + if (z) { + pp.mode = MSCC_QOS_RATE_MODE_DISABLED; + return qos_policer_conf_set(ocelot, 0, pol_ix, &pp); + } - block->pol_lpr++; + return 0; } +EXPORT_SYMBOL(ocelot_vcap_policer_del); -static void ocelot_vcap_filter_add_to_block(struct ocelot *ocelot, - struct ocelot_vcap_block *block, - struct ocelot_vcap_filter *filter) +static int ocelot_vcap_filter_add_to_block(struct ocelot *ocelot, + struct ocelot_vcap_block *block, + struct ocelot_vcap_filter *filter) { struct ocelot_vcap_filter *tmp; struct list_head *pos, *n; + int ret; if (filter->block_id == VCAP_IS2 && filter->action.police_ena) { - block->pol_lpr--; - filter->action.pol_ix = block->pol_lpr; - ocelot_vcap_policer_add(ocelot, filter->action.pol_ix, - &filter->action.pol); + ret = ocelot_vcap_policer_add(ocelot, filter->action.pol_ix, + &filter->action.pol); + if (ret) + return ret; } block->count++; if (list_empty(&block->rules)) { list_add(&filter->list, &block->rules); - return; + return 0; } list_for_each_safe(pos, n, &block->rules) { @@ -958,6 +983,8 @@ static void ocelot_vcap_filter_add_to_block(struct ocelot *ocelot, break; } list_add(&filter->list, pos->prev); + + return 0; } static bool ocelot_vcap_filter_equal(const struct ocelot_vcap_filter *a, @@ -1132,7 +1159,7 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot, struct netlink_ext_ack *extack) { struct ocelot_vcap_block *block = &ocelot->block[filter->block_id]; - int i, index; + int i, index, ret; if (!ocelot_exclusive_mac_etype_filter_rules(ocelot, filter)) { NL_SET_ERR_MSG_MOD(extack, @@ -1141,7 +1168,9 @@ int ocelot_vcap_filter_add(struct ocelot *ocelot, } /* Add filter to the linked list */ - ocelot_vcap_filter_add_to_block(ocelot, block, filter); + ret = ocelot_vcap_filter_add_to_block(ocelot, block, filter); + if (ret) + return ret; /* Get the index of the inserted filter */ index = ocelot_vcap_block_get_filter_index(block, filter); @@ -1174,7 +1203,7 @@ static void ocelot_vcap_block_remove_filter(struct ocelot *ocelot, if (ocelot_vcap_filter_equal(filter, tmp)) { if (tmp->block_id == VCAP_IS2 && tmp->action.police_ena) - ocelot_vcap_policer_del(ocelot, block, + ocelot_vcap_policer_del(ocelot, tmp->action.pol_ix); list_del(pos); @@ -1366,13 +1395,13 @@ int ocelot_vcap_init(struct ocelot *ocelot) struct vcap_props *vcap = &ocelot->vcap[i]; INIT_LIST_HEAD(&block->rules); - block->pol_lpr = OCELOT_POLICER_DISCARD - 1; ocelot_vcap_detect_constants(ocelot, vcap); ocelot_vcap_init_one(ocelot, vcap); } INIT_LIST_HEAD(&ocelot->dummy_rules); + INIT_LIST_HEAD(&ocelot->vcap_pol.pol_list); return 0; } diff --git a/drivers/net/ethernet/mscc/ocelot_vsc7514.c b/drivers/net/ethernet/mscc/ocelot_vsc7514.c index 38103b0255b0..2db59060f5ab 100644 --- a/drivers/net/ethernet/mscc/ocelot_vsc7514.c +++ b/drivers/net/ethernet/mscc/ocelot_vsc7514.c @@ -18,313 +18,23 @@ #include <soc/mscc/ocelot_vcap.h> #include <soc/mscc/ocelot_hsio.h> +#include <soc/mscc/vsc7514_regs.h> #include "ocelot.h" -static const u32 ocelot_ana_regmap[] = { - REG(ANA_ADVLEARN, 0x009000), - REG(ANA_VLANMASK, 0x009004), - REG(ANA_PORT_B_DOMAIN, 0x009008), - REG(ANA_ANAGEFIL, 0x00900c), - REG(ANA_ANEVENTS, 0x009010), - REG(ANA_STORMLIMIT_BURST, 0x009014), - REG(ANA_STORMLIMIT_CFG, 0x009018), - REG(ANA_ISOLATED_PORTS, 0x009028), - REG(ANA_COMMUNITY_PORTS, 0x00902c), - REG(ANA_AUTOAGE, 0x009030), - REG(ANA_MACTOPTIONS, 0x009034), - REG(ANA_LEARNDISC, 0x009038), - REG(ANA_AGENCTRL, 0x00903c), - REG(ANA_MIRRORPORTS, 0x009040), - REG(ANA_EMIRRORPORTS, 0x009044), - REG(ANA_FLOODING, 0x009048), - REG(ANA_FLOODING_IPMC, 0x00904c), - REG(ANA_SFLOW_CFG, 0x009050), - REG(ANA_PORT_MODE, 0x009080), - REG(ANA_PGID_PGID, 0x008c00), - REG(ANA_TABLES_ANMOVED, 0x008b30), - REG(ANA_TABLES_MACHDATA, 0x008b34), - REG(ANA_TABLES_MACLDATA, 0x008b38), - REG(ANA_TABLES_MACACCESS, 0x008b3c), - REG(ANA_TABLES_MACTINDX, 0x008b40), - REG(ANA_TABLES_VLANACCESS, 0x008b44), - REG(ANA_TABLES_VLANTIDX, 0x008b48), - REG(ANA_TABLES_ISDXACCESS, 0x008b4c), - REG(ANA_TABLES_ISDXTIDX, 0x008b50), - REG(ANA_TABLES_ENTRYLIM, 0x008b00), - REG(ANA_TABLES_PTP_ID_HIGH, 0x008b54), - REG(ANA_TABLES_PTP_ID_LOW, 0x008b58), - REG(ANA_MSTI_STATE, 0x008e00), - REG(ANA_PORT_VLAN_CFG, 0x007000), - REG(ANA_PORT_DROP_CFG, 0x007004), - REG(ANA_PORT_QOS_CFG, 0x007008), - REG(ANA_PORT_VCAP_CFG, 0x00700c), - REG(ANA_PORT_VCAP_S1_KEY_CFG, 0x007010), - REG(ANA_PORT_VCAP_S2_CFG, 0x00701c), - REG(ANA_PORT_PCP_DEI_MAP, 0x007020), - REG(ANA_PORT_CPU_FWD_CFG, 0x007060), - REG(ANA_PORT_CPU_FWD_BPDU_CFG, 0x007064), - REG(ANA_PORT_CPU_FWD_GARP_CFG, 0x007068), - REG(ANA_PORT_CPU_FWD_CCM_CFG, 0x00706c), - REG(ANA_PORT_PORT_CFG, 0x007070), - REG(ANA_PORT_POL_CFG, 0x007074), - REG(ANA_PORT_PTP_CFG, 0x007078), - REG(ANA_PORT_PTP_DLY1_CFG, 0x00707c), - REG(ANA_OAM_UPM_LM_CNT, 0x007c00), - REG(ANA_PORT_PTP_DLY2_CFG, 0x007080), - REG(ANA_PFC_PFC_CFG, 0x008800), - REG(ANA_PFC_PFC_TIMER, 0x008804), - REG(ANA_IPT_OAM_MEP_CFG, 0x008000), - REG(ANA_IPT_IPT, 0x008004), - REG(ANA_PPT_PPT, 0x008ac0), - REG(ANA_FID_MAP_FID_MAP, 0x000000), - REG(ANA_AGGR_CFG, 0x0090b4), - REG(ANA_CPUQ_CFG, 0x0090b8), - REG(ANA_CPUQ_CFG2, 0x0090bc), - REG(ANA_CPUQ_8021_CFG, 0x0090c0), - REG(ANA_DSCP_CFG, 0x009100), - REG(ANA_DSCP_REWR_CFG, 0x009200), - REG(ANA_VCAP_RNG_TYPE_CFG, 0x009240), - REG(ANA_VCAP_RNG_VAL_CFG, 0x009260), - REG(ANA_VRAP_CFG, 0x009280), - REG(ANA_VRAP_HDR_DATA, 0x009284), - REG(ANA_VRAP_HDR_MASK, 0x009288), - REG(ANA_DISCARD_CFG, 0x00928c), - REG(ANA_FID_CFG, 0x009290), - REG(ANA_POL_PIR_CFG, 0x004000), - REG(ANA_POL_CIR_CFG, 0x004004), - REG(ANA_POL_MODE_CFG, 0x004008), - REG(ANA_POL_PIR_STATE, 0x00400c), - REG(ANA_POL_CIR_STATE, 0x004010), - REG(ANA_POL_STATE, 0x004014), - REG(ANA_POL_FLOWC, 0x008b80), - REG(ANA_POL_HYST, 0x008bec), - REG(ANA_POL_MISC_CFG, 0x008bf0), -}; - -static const u32 ocelot_qs_regmap[] = { - REG(QS_XTR_GRP_CFG, 0x000000), - REG(QS_XTR_RD, 0x000008), - REG(QS_XTR_FRM_PRUNING, 0x000010), - REG(QS_XTR_FLUSH, 0x000018), - REG(QS_XTR_DATA_PRESENT, 0x00001c), - REG(QS_XTR_CFG, 0x000020), - REG(QS_INJ_GRP_CFG, 0x000024), - REG(QS_INJ_WR, 0x00002c), - REG(QS_INJ_CTRL, 0x000034), - REG(QS_INJ_STATUS, 0x00003c), - REG(QS_INJ_ERR, 0x000040), - REG(QS_INH_DBG, 0x000048), -}; - -static const u32 ocelot_qsys_regmap[] = { - REG(QSYS_PORT_MODE, 0x011200), - REG(QSYS_SWITCH_PORT_MODE, 0x011234), - REG(QSYS_STAT_CNT_CFG, 0x011264), - REG(QSYS_EEE_CFG, 0x011268), - REG(QSYS_EEE_THRES, 0x011294), - REG(QSYS_IGR_NO_SHARING, 0x011298), - REG(QSYS_EGR_NO_SHARING, 0x01129c), - REG(QSYS_SW_STATUS, 0x0112a0), - REG(QSYS_EXT_CPU_CFG, 0x0112d0), - REG(QSYS_PAD_CFG, 0x0112d4), - REG(QSYS_CPU_GROUP_MAP, 0x0112d8), - REG(QSYS_QMAP, 0x0112dc), - REG(QSYS_ISDX_SGRP, 0x011400), - REG(QSYS_TIMED_FRAME_ENTRY, 0x014000), - REG(QSYS_TFRM_MISC, 0x011310), - REG(QSYS_TFRM_PORT_DLY, 0x011314), - REG(QSYS_TFRM_TIMER_CFG_1, 0x011318), - REG(QSYS_TFRM_TIMER_CFG_2, 0x01131c), - REG(QSYS_TFRM_TIMER_CFG_3, 0x011320), - REG(QSYS_TFRM_TIMER_CFG_4, 0x011324), - REG(QSYS_TFRM_TIMER_CFG_5, 0x011328), - REG(QSYS_TFRM_TIMER_CFG_6, 0x01132c), - REG(QSYS_TFRM_TIMER_CFG_7, 0x011330), - REG(QSYS_TFRM_TIMER_CFG_8, 0x011334), - REG(QSYS_RED_PROFILE, 0x011338), - REG(QSYS_RES_QOS_MODE, 0x011378), - REG(QSYS_RES_CFG, 0x012000), - REG(QSYS_RES_STAT, 0x012004), - REG(QSYS_EGR_DROP_MODE, 0x01137c), - REG(QSYS_EQ_CTRL, 0x011380), - REG(QSYS_EVENTS_CORE, 0x011384), - REG(QSYS_CIR_CFG, 0x000000), - REG(QSYS_EIR_CFG, 0x000004), - REG(QSYS_SE_CFG, 0x000008), - REG(QSYS_SE_DWRR_CFG, 0x00000c), - REG(QSYS_SE_CONNECT, 0x00003c), - REG(QSYS_SE_DLB_SENSE, 0x000040), - REG(QSYS_CIR_STATE, 0x000044), - REG(QSYS_EIR_STATE, 0x000048), - REG(QSYS_SE_STATE, 0x00004c), - REG(QSYS_HSCH_MISC_CFG, 0x011388), -}; - -static const u32 ocelot_rew_regmap[] = { - REG(REW_PORT_VLAN_CFG, 0x000000), - REG(REW_TAG_CFG, 0x000004), - REG(REW_PORT_CFG, 0x000008), - REG(REW_DSCP_CFG, 0x00000c), - REG(REW_PCP_DEI_QOS_MAP_CFG, 0x000010), - REG(REW_PTP_CFG, 0x000050), - REG(REW_PTP_DLY1_CFG, 0x000054), - REG(REW_DSCP_REMAP_DP1_CFG, 0x000690), - REG(REW_DSCP_REMAP_CFG, 0x000790), - REG(REW_STAT_CFG, 0x000890), - REG(REW_PPT, 0x000680), -}; - -static const u32 ocelot_sys_regmap[] = { - REG(SYS_COUNT_RX_OCTETS, 0x000000), - REG(SYS_COUNT_RX_UNICAST, 0x000004), - REG(SYS_COUNT_RX_MULTICAST, 0x000008), - REG(SYS_COUNT_RX_BROADCAST, 0x00000c), - REG(SYS_COUNT_RX_SHORTS, 0x000010), - REG(SYS_COUNT_RX_FRAGMENTS, 0x000014), - REG(SYS_COUNT_RX_JABBERS, 0x000018), - REG(SYS_COUNT_RX_CRC_ALIGN_ERRS, 0x00001c), - REG(SYS_COUNT_RX_SYM_ERRS, 0x000020), - REG(SYS_COUNT_RX_64, 0x000024), - REG(SYS_COUNT_RX_65_127, 0x000028), - REG(SYS_COUNT_RX_128_255, 0x00002c), - REG(SYS_COUNT_RX_256_1023, 0x000030), - REG(SYS_COUNT_RX_1024_1526, 0x000034), - REG(SYS_COUNT_RX_1527_MAX, 0x000038), - REG(SYS_COUNT_RX_PAUSE, 0x00003c), - REG(SYS_COUNT_RX_CONTROL, 0x000040), - REG(SYS_COUNT_RX_LONGS, 0x000044), - REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x000048), - REG(SYS_COUNT_TX_OCTETS, 0x000100), - REG(SYS_COUNT_TX_UNICAST, 0x000104), - REG(SYS_COUNT_TX_MULTICAST, 0x000108), - REG(SYS_COUNT_TX_BROADCAST, 0x00010c), - REG(SYS_COUNT_TX_COLLISION, 0x000110), - REG(SYS_COUNT_TX_DROPS, 0x000114), - REG(SYS_COUNT_TX_PAUSE, 0x000118), - REG(SYS_COUNT_TX_64, 0x00011c), - REG(SYS_COUNT_TX_65_127, 0x000120), - REG(SYS_COUNT_TX_128_511, 0x000124), - REG(SYS_COUNT_TX_512_1023, 0x000128), - REG(SYS_COUNT_TX_1024_1526, 0x00012c), - REG(SYS_COUNT_TX_1527_MAX, 0x000130), - REG(SYS_COUNT_TX_AGING, 0x000170), - REG(SYS_RESET_CFG, 0x000508), - REG(SYS_CMID, 0x00050c), - REG(SYS_VLAN_ETYPE_CFG, 0x000510), - REG(SYS_PORT_MODE, 0x000514), - REG(SYS_FRONT_PORT_MODE, 0x000548), - REG(SYS_FRM_AGING, 0x000574), - REG(SYS_STAT_CFG, 0x000578), - REG(SYS_SW_STATUS, 0x00057c), - REG(SYS_MISC_CFG, 0x0005ac), - REG(SYS_REW_MAC_HIGH_CFG, 0x0005b0), - REG(SYS_REW_MAC_LOW_CFG, 0x0005dc), - REG(SYS_CM_ADDR, 0x000500), - REG(SYS_CM_DATA, 0x000504), - REG(SYS_PAUSE_CFG, 0x000608), - REG(SYS_PAUSE_TOT_CFG, 0x000638), - REG(SYS_ATOP, 0x00063c), - REG(SYS_ATOP_TOT_CFG, 0x00066c), - REG(SYS_MAC_FC_CFG, 0x000670), - REG(SYS_MMGT, 0x00069c), - REG(SYS_MMGT_FAST, 0x0006a0), - REG(SYS_EVENTS_DIF, 0x0006a4), - REG(SYS_EVENTS_CORE, 0x0006b4), - REG(SYS_CNT, 0x000000), - REG(SYS_PTP_STATUS, 0x0006b8), - REG(SYS_PTP_TXSTAMP, 0x0006bc), - REG(SYS_PTP_NXT, 0x0006c0), - REG(SYS_PTP_CFG, 0x0006c4), -}; - -static const u32 ocelot_vcap_regmap[] = { - /* VCAP_CORE_CFG */ - REG(VCAP_CORE_UPDATE_CTRL, 0x000000), - REG(VCAP_CORE_MV_CFG, 0x000004), - /* VCAP_CORE_CACHE */ - REG(VCAP_CACHE_ENTRY_DAT, 0x000008), - REG(VCAP_CACHE_MASK_DAT, 0x000108), - REG(VCAP_CACHE_ACTION_DAT, 0x000208), - REG(VCAP_CACHE_CNT_DAT, 0x000308), - REG(VCAP_CACHE_TG_DAT, 0x000388), - /* VCAP_CONST */ - REG(VCAP_CONST_VCAP_VER, 0x000398), - REG(VCAP_CONST_ENTRY_WIDTH, 0x00039c), - REG(VCAP_CONST_ENTRY_CNT, 0x0003a0), - REG(VCAP_CONST_ENTRY_SWCNT, 0x0003a4), - REG(VCAP_CONST_ENTRY_TG_WIDTH, 0x0003a8), - REG(VCAP_CONST_ACTION_DEF_CNT, 0x0003ac), - REG(VCAP_CONST_ACTION_WIDTH, 0x0003b0), - REG(VCAP_CONST_CNT_WIDTH, 0x0003b4), - REG(VCAP_CONST_CORE_CNT, 0x0003b8), - REG(VCAP_CONST_IF_CNT, 0x0003bc), -}; - -static const u32 ocelot_ptp_regmap[] = { - REG(PTP_PIN_CFG, 0x000000), - REG(PTP_PIN_TOD_SEC_MSB, 0x000004), - REG(PTP_PIN_TOD_SEC_LSB, 0x000008), - REG(PTP_PIN_TOD_NSEC, 0x00000c), - REG(PTP_PIN_WF_HIGH_PERIOD, 0x000014), - REG(PTP_PIN_WF_LOW_PERIOD, 0x000018), - REG(PTP_CFG_MISC, 0x0000a0), - REG(PTP_CLK_CFG_ADJ_CFG, 0x0000a4), - REG(PTP_CLK_CFG_ADJ_FREQ, 0x0000a8), -}; - -static const u32 ocelot_dev_gmii_regmap[] = { - REG(DEV_CLOCK_CFG, 0x0), - REG(DEV_PORT_MISC, 0x4), - REG(DEV_EVENTS, 0x8), - REG(DEV_EEE_CFG, 0xc), - REG(DEV_RX_PATH_DELAY, 0x10), - REG(DEV_TX_PATH_DELAY, 0x14), - REG(DEV_PTP_PREDICT_CFG, 0x18), - REG(DEV_MAC_ENA_CFG, 0x1c), - REG(DEV_MAC_MODE_CFG, 0x20), - REG(DEV_MAC_MAXLEN_CFG, 0x24), - REG(DEV_MAC_TAGS_CFG, 0x28), - REG(DEV_MAC_ADV_CHK_CFG, 0x2c), - REG(DEV_MAC_IFG_CFG, 0x30), - REG(DEV_MAC_HDX_CFG, 0x34), - REG(DEV_MAC_DBG_CFG, 0x38), - REG(DEV_MAC_FC_MAC_LOW_CFG, 0x3c), - REG(DEV_MAC_FC_MAC_HIGH_CFG, 0x40), - REG(DEV_MAC_STICKY, 0x44), - REG(PCS1G_CFG, 0x48), - REG(PCS1G_MODE_CFG, 0x4c), - REG(PCS1G_SD_CFG, 0x50), - REG(PCS1G_ANEG_CFG, 0x54), - REG(PCS1G_ANEG_NP_CFG, 0x58), - REG(PCS1G_LB_CFG, 0x5c), - REG(PCS1G_DBG_CFG, 0x60), - REG(PCS1G_CDET_CFG, 0x64), - REG(PCS1G_ANEG_STATUS, 0x68), - REG(PCS1G_ANEG_NP_STATUS, 0x6c), - REG(PCS1G_LINK_STATUS, 0x70), - REG(PCS1G_LINK_DOWN_CNT, 0x74), - REG(PCS1G_STICKY, 0x78), - REG(PCS1G_DEBUG_STATUS, 0x7c), - REG(PCS1G_LPI_CFG, 0x80), - REG(PCS1G_LPI_WAKE_ERROR_CNT, 0x84), - REG(PCS1G_LPI_STATUS, 0x88), - REG(PCS1G_TSTPAT_MODE_CFG, 0x8c), - REG(PCS1G_TSTPAT_STATUS, 0x90), - REG(DEV_PCS_FX100_CFG, 0x94), - REG(DEV_PCS_FX100_STATUS, 0x98), -}; +#define VSC7514_VCAP_POLICER_BASE 128 +#define VSC7514_VCAP_POLICER_MAX 191 static const u32 *ocelot_regmap[TARGET_MAX] = { - [ANA] = ocelot_ana_regmap, - [QS] = ocelot_qs_regmap, - [QSYS] = ocelot_qsys_regmap, - [REW] = ocelot_rew_regmap, - [SYS] = ocelot_sys_regmap, - [S0] = ocelot_vcap_regmap, - [S1] = ocelot_vcap_regmap, - [S2] = ocelot_vcap_regmap, - [PTP] = ocelot_ptp_regmap, - [DEV_GMII] = ocelot_dev_gmii_regmap, + [ANA] = vsc7514_ana_regmap, + [QS] = vsc7514_qs_regmap, + [QSYS] = vsc7514_qsys_regmap, + [REW] = vsc7514_rew_regmap, + [SYS] = vsc7514_sys_regmap, + [S0] = vsc7514_vcap_regmap, + [S1] = vsc7514_vcap_regmap, + [S2] = vsc7514_vcap_regmap, + [PTP] = vsc7514_ptp_regmap, + [DEV_GMII] = vsc7514_dev_gmii_regmap, }; static const struct reg_field ocelot_regfields[REGFIELD_MAX] = { @@ -633,211 +343,6 @@ static const struct ocelot_ops ocelot_ops = { .netdev_to_port = ocelot_netdev_to_port, }; -static const struct vcap_field vsc7514_vcap_es0_keys[] = { - [VCAP_ES0_EGR_PORT] = { 0, 4}, - [VCAP_ES0_IGR_PORT] = { 4, 4}, - [VCAP_ES0_RSV] = { 8, 2}, - [VCAP_ES0_L2_MC] = { 10, 1}, - [VCAP_ES0_L2_BC] = { 11, 1}, - [VCAP_ES0_VID] = { 12, 12}, - [VCAP_ES0_DP] = { 24, 1}, - [VCAP_ES0_PCP] = { 25, 3}, -}; - -static const struct vcap_field vsc7514_vcap_es0_actions[] = { - [VCAP_ES0_ACT_PUSH_OUTER_TAG] = { 0, 2}, - [VCAP_ES0_ACT_PUSH_INNER_TAG] = { 2, 1}, - [VCAP_ES0_ACT_TAG_A_TPID_SEL] = { 3, 2}, - [VCAP_ES0_ACT_TAG_A_VID_SEL] = { 5, 1}, - [VCAP_ES0_ACT_TAG_A_PCP_SEL] = { 6, 2}, - [VCAP_ES0_ACT_TAG_A_DEI_SEL] = { 8, 2}, - [VCAP_ES0_ACT_TAG_B_TPID_SEL] = { 10, 2}, - [VCAP_ES0_ACT_TAG_B_VID_SEL] = { 12, 1}, - [VCAP_ES0_ACT_TAG_B_PCP_SEL] = { 13, 2}, - [VCAP_ES0_ACT_TAG_B_DEI_SEL] = { 15, 2}, - [VCAP_ES0_ACT_VID_A_VAL] = { 17, 12}, - [VCAP_ES0_ACT_PCP_A_VAL] = { 29, 3}, - [VCAP_ES0_ACT_DEI_A_VAL] = { 32, 1}, - [VCAP_ES0_ACT_VID_B_VAL] = { 33, 12}, - [VCAP_ES0_ACT_PCP_B_VAL] = { 45, 3}, - [VCAP_ES0_ACT_DEI_B_VAL] = { 48, 1}, - [VCAP_ES0_ACT_RSV] = { 49, 24}, - [VCAP_ES0_ACT_HIT_STICKY] = { 73, 1}, -}; - -static const struct vcap_field vsc7514_vcap_is1_keys[] = { - [VCAP_IS1_HK_TYPE] = { 0, 1}, - [VCAP_IS1_HK_LOOKUP] = { 1, 2}, - [VCAP_IS1_HK_IGR_PORT_MASK] = { 3, 12}, - [VCAP_IS1_HK_RSV] = { 15, 9}, - [VCAP_IS1_HK_OAM_Y1731] = { 24, 1}, - [VCAP_IS1_HK_L2_MC] = { 25, 1}, - [VCAP_IS1_HK_L2_BC] = { 26, 1}, - [VCAP_IS1_HK_IP_MC] = { 27, 1}, - [VCAP_IS1_HK_VLAN_TAGGED] = { 28, 1}, - [VCAP_IS1_HK_VLAN_DBL_TAGGED] = { 29, 1}, - [VCAP_IS1_HK_TPID] = { 30, 1}, - [VCAP_IS1_HK_VID] = { 31, 12}, - [VCAP_IS1_HK_DEI] = { 43, 1}, - [VCAP_IS1_HK_PCP] = { 44, 3}, - /* Specific Fields for IS1 Half Key S1_NORMAL */ - [VCAP_IS1_HK_L2_SMAC] = { 47, 48}, - [VCAP_IS1_HK_ETYPE_LEN] = { 95, 1}, - [VCAP_IS1_HK_ETYPE] = { 96, 16}, - [VCAP_IS1_HK_IP_SNAP] = {112, 1}, - [VCAP_IS1_HK_IP4] = {113, 1}, - /* Layer-3 Information */ - [VCAP_IS1_HK_L3_FRAGMENT] = {114, 1}, - [VCAP_IS1_HK_L3_FRAG_OFS_GT0] = {115, 1}, - [VCAP_IS1_HK_L3_OPTIONS] = {116, 1}, - [VCAP_IS1_HK_L3_DSCP] = {117, 6}, - [VCAP_IS1_HK_L3_IP4_SIP] = {123, 32}, - /* Layer-4 Information */ - [VCAP_IS1_HK_TCP_UDP] = {155, 1}, - [VCAP_IS1_HK_TCP] = {156, 1}, - [VCAP_IS1_HK_L4_SPORT] = {157, 16}, - [VCAP_IS1_HK_L4_RNG] = {173, 8}, - /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ - [VCAP_IS1_HK_IP4_INNER_TPID] = { 47, 1}, - [VCAP_IS1_HK_IP4_INNER_VID] = { 48, 12}, - [VCAP_IS1_HK_IP4_INNER_DEI] = { 60, 1}, - [VCAP_IS1_HK_IP4_INNER_PCP] = { 61, 3}, - [VCAP_IS1_HK_IP4_IP4] = { 64, 1}, - [VCAP_IS1_HK_IP4_L3_FRAGMENT] = { 65, 1}, - [VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0] = { 66, 1}, - [VCAP_IS1_HK_IP4_L3_OPTIONS] = { 67, 1}, - [VCAP_IS1_HK_IP4_L3_DSCP] = { 68, 6}, - [VCAP_IS1_HK_IP4_L3_IP4_DIP] = { 74, 32}, - [VCAP_IS1_HK_IP4_L3_IP4_SIP] = {106, 32}, - [VCAP_IS1_HK_IP4_L3_PROTO] = {138, 8}, - [VCAP_IS1_HK_IP4_TCP_UDP] = {146, 1}, - [VCAP_IS1_HK_IP4_TCP] = {147, 1}, - [VCAP_IS1_HK_IP4_L4_RNG] = {148, 8}, - [VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE] = {156, 32}, -}; - -static const struct vcap_field vsc7514_vcap_is1_actions[] = { - [VCAP_IS1_ACT_DSCP_ENA] = { 0, 1}, - [VCAP_IS1_ACT_DSCP_VAL] = { 1, 6}, - [VCAP_IS1_ACT_QOS_ENA] = { 7, 1}, - [VCAP_IS1_ACT_QOS_VAL] = { 8, 3}, - [VCAP_IS1_ACT_DP_ENA] = { 11, 1}, - [VCAP_IS1_ACT_DP_VAL] = { 12, 1}, - [VCAP_IS1_ACT_PAG_OVERRIDE_MASK] = { 13, 8}, - [VCAP_IS1_ACT_PAG_VAL] = { 21, 8}, - [VCAP_IS1_ACT_RSV] = { 29, 9}, - /* The fields below are incorrectly shifted by 2 in the manual */ - [VCAP_IS1_ACT_VID_REPLACE_ENA] = { 38, 1}, - [VCAP_IS1_ACT_VID_ADD_VAL] = { 39, 12}, - [VCAP_IS1_ACT_FID_SEL] = { 51, 2}, - [VCAP_IS1_ACT_FID_VAL] = { 53, 13}, - [VCAP_IS1_ACT_PCP_DEI_ENA] = { 66, 1}, - [VCAP_IS1_ACT_PCP_VAL] = { 67, 3}, - [VCAP_IS1_ACT_DEI_VAL] = { 70, 1}, - [VCAP_IS1_ACT_VLAN_POP_CNT_ENA] = { 71, 1}, - [VCAP_IS1_ACT_VLAN_POP_CNT] = { 72, 2}, - [VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA] = { 74, 4}, - [VCAP_IS1_ACT_HIT_STICKY] = { 78, 1}, -}; - -static const struct vcap_field vsc7514_vcap_is2_keys[] = { - /* Common: 46 bits */ - [VCAP_IS2_TYPE] = { 0, 4}, - [VCAP_IS2_HK_FIRST] = { 4, 1}, - [VCAP_IS2_HK_PAG] = { 5, 8}, - [VCAP_IS2_HK_IGR_PORT_MASK] = { 13, 12}, - [VCAP_IS2_HK_RSV2] = { 25, 1}, - [VCAP_IS2_HK_HOST_MATCH] = { 26, 1}, - [VCAP_IS2_HK_L2_MC] = { 27, 1}, - [VCAP_IS2_HK_L2_BC] = { 28, 1}, - [VCAP_IS2_HK_VLAN_TAGGED] = { 29, 1}, - [VCAP_IS2_HK_VID] = { 30, 12}, - [VCAP_IS2_HK_DEI] = { 42, 1}, - [VCAP_IS2_HK_PCP] = { 43, 3}, - /* MAC_ETYPE / MAC_LLC / MAC_SNAP / OAM common */ - [VCAP_IS2_HK_L2_DMAC] = { 46, 48}, - [VCAP_IS2_HK_L2_SMAC] = { 94, 48}, - /* MAC_ETYPE (TYPE=000) */ - [VCAP_IS2_HK_MAC_ETYPE_ETYPE] = {142, 16}, - [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD0] = {158, 16}, - [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD1] = {174, 8}, - [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD2] = {182, 3}, - /* MAC_LLC (TYPE=001) */ - [VCAP_IS2_HK_MAC_LLC_L2_LLC] = {142, 40}, - /* MAC_SNAP (TYPE=010) */ - [VCAP_IS2_HK_MAC_SNAP_L2_SNAP] = {142, 40}, - /* MAC_ARP (TYPE=011) */ - [VCAP_IS2_HK_MAC_ARP_SMAC] = { 46, 48}, - [VCAP_IS2_HK_MAC_ARP_ADDR_SPACE_OK] = { 94, 1}, - [VCAP_IS2_HK_MAC_ARP_PROTO_SPACE_OK] = { 95, 1}, - [VCAP_IS2_HK_MAC_ARP_LEN_OK] = { 96, 1}, - [VCAP_IS2_HK_MAC_ARP_TARGET_MATCH] = { 97, 1}, - [VCAP_IS2_HK_MAC_ARP_SENDER_MATCH] = { 98, 1}, - [VCAP_IS2_HK_MAC_ARP_OPCODE_UNKNOWN] = { 99, 1}, - [VCAP_IS2_HK_MAC_ARP_OPCODE] = {100, 2}, - [VCAP_IS2_HK_MAC_ARP_L3_IP4_DIP] = {102, 32}, - [VCAP_IS2_HK_MAC_ARP_L3_IP4_SIP] = {134, 32}, - [VCAP_IS2_HK_MAC_ARP_DIP_EQ_SIP] = {166, 1}, - /* IP4_TCP_UDP / IP4_OTHER common */ - [VCAP_IS2_HK_IP4] = { 46, 1}, - [VCAP_IS2_HK_L3_FRAGMENT] = { 47, 1}, - [VCAP_IS2_HK_L3_FRAG_OFS_GT0] = { 48, 1}, - [VCAP_IS2_HK_L3_OPTIONS] = { 49, 1}, - [VCAP_IS2_HK_IP4_L3_TTL_GT0] = { 50, 1}, - [VCAP_IS2_HK_L3_TOS] = { 51, 8}, - [VCAP_IS2_HK_L3_IP4_DIP] = { 59, 32}, - [VCAP_IS2_HK_L3_IP4_SIP] = { 91, 32}, - [VCAP_IS2_HK_DIP_EQ_SIP] = {123, 1}, - /* IP4_TCP_UDP (TYPE=100) */ - [VCAP_IS2_HK_TCP] = {124, 1}, - [VCAP_IS2_HK_L4_DPORT] = {125, 16}, - [VCAP_IS2_HK_L4_SPORT] = {141, 16}, - [VCAP_IS2_HK_L4_RNG] = {157, 8}, - [VCAP_IS2_HK_L4_SPORT_EQ_DPORT] = {165, 1}, - [VCAP_IS2_HK_L4_SEQUENCE_EQ0] = {166, 1}, - [VCAP_IS2_HK_L4_FIN] = {167, 1}, - [VCAP_IS2_HK_L4_SYN] = {168, 1}, - [VCAP_IS2_HK_L4_RST] = {169, 1}, - [VCAP_IS2_HK_L4_PSH] = {170, 1}, - [VCAP_IS2_HK_L4_ACK] = {171, 1}, - [VCAP_IS2_HK_L4_URG] = {172, 1}, - [VCAP_IS2_HK_L4_1588_DOM] = {173, 8}, - [VCAP_IS2_HK_L4_1588_VER] = {181, 4}, - /* IP4_OTHER (TYPE=101) */ - [VCAP_IS2_HK_IP4_L3_PROTO] = {124, 8}, - [VCAP_IS2_HK_L3_PAYLOAD] = {132, 56}, - /* IP6_STD (TYPE=110) */ - [VCAP_IS2_HK_IP6_L3_TTL_GT0] = { 46, 1}, - [VCAP_IS2_HK_L3_IP6_SIP] = { 47, 128}, - [VCAP_IS2_HK_IP6_L3_PROTO] = {175, 8}, - /* OAM (TYPE=111) */ - [VCAP_IS2_HK_OAM_MEL_FLAGS] = {142, 7}, - [VCAP_IS2_HK_OAM_VER] = {149, 5}, - [VCAP_IS2_HK_OAM_OPCODE] = {154, 8}, - [VCAP_IS2_HK_OAM_FLAGS] = {162, 8}, - [VCAP_IS2_HK_OAM_MEPID] = {170, 16}, - [VCAP_IS2_HK_OAM_CCM_CNTS_EQ0] = {186, 1}, - [VCAP_IS2_HK_OAM_IS_Y1731] = {187, 1}, -}; - -static const struct vcap_field vsc7514_vcap_is2_actions[] = { - [VCAP_IS2_ACT_HIT_ME_ONCE] = { 0, 1}, - [VCAP_IS2_ACT_CPU_COPY_ENA] = { 1, 1}, - [VCAP_IS2_ACT_CPU_QU_NUM] = { 2, 3}, - [VCAP_IS2_ACT_MASK_MODE] = { 5, 2}, - [VCAP_IS2_ACT_MIRROR_ENA] = { 7, 1}, - [VCAP_IS2_ACT_LRN_DIS] = { 8, 1}, - [VCAP_IS2_ACT_POLICE_ENA] = { 9, 1}, - [VCAP_IS2_ACT_POLICE_IDX] = { 10, 9}, - [VCAP_IS2_ACT_POLICE_VCAP_ONLY] = { 19, 1}, - [VCAP_IS2_ACT_PORT_MASK] = { 20, 11}, - [VCAP_IS2_ACT_REW_OP] = { 31, 9}, - [VCAP_IS2_ACT_SMAC_REPLACE_ENA] = { 40, 1}, - [VCAP_IS2_ACT_RSV] = { 41, 2}, - [VCAP_IS2_ACT_ACL_ID] = { 43, 6}, - [VCAP_IS2_ACT_HIT_CNT] = { 49, 32}, -}; - static struct vcap_props vsc7514_vcap_props[] = { [VCAP_ES0] = { .action_type_width = 0, @@ -1129,6 +634,10 @@ static int mscc_ocelot_probe(struct platform_device *pdev) ocelot->num_flooding_pgids = 1; ocelot->vcap = vsc7514_vcap_props; + + ocelot->vcap_pol.base = VSC7514_VCAP_POLICER_BASE; + ocelot->vcap_pol.max = VSC7514_VCAP_POLICER_MAX; + ocelot->npi = -1; err = ocelot_init(ocelot); diff --git a/drivers/net/ethernet/mscc/vsc7514_regs.c b/drivers/net/ethernet/mscc/vsc7514_regs.c new file mode 100644 index 000000000000..c2af4eb8ca5d --- /dev/null +++ b/drivers/net/ethernet/mscc/vsc7514_regs.c @@ -0,0 +1,523 @@ +// SPDX-License-Identifier: (GPL-2.0 OR MIT) +/* + * Microsemi Ocelot Switch driver + * + * Copyright (c) 2017 Microsemi Corporation + * Copyright (c) 2021 Innovative Advantage + */ +#include <soc/mscc/ocelot_vcap.h> +#include <soc/mscc/vsc7514_regs.h> +#include "ocelot.h" + +const u32 vsc7514_ana_regmap[] = { + REG(ANA_ADVLEARN, 0x009000), + REG(ANA_VLANMASK, 0x009004), + REG(ANA_PORT_B_DOMAIN, 0x009008), + REG(ANA_ANAGEFIL, 0x00900c), + REG(ANA_ANEVENTS, 0x009010), + REG(ANA_STORMLIMIT_BURST, 0x009014), + REG(ANA_STORMLIMIT_CFG, 0x009018), + REG(ANA_ISOLATED_PORTS, 0x009028), + REG(ANA_COMMUNITY_PORTS, 0x00902c), + REG(ANA_AUTOAGE, 0x009030), + REG(ANA_MACTOPTIONS, 0x009034), + REG(ANA_LEARNDISC, 0x009038), + REG(ANA_AGENCTRL, 0x00903c), + REG(ANA_MIRRORPORTS, 0x009040), + REG(ANA_EMIRRORPORTS, 0x009044), + REG(ANA_FLOODING, 0x009048), + REG(ANA_FLOODING_IPMC, 0x00904c), + REG(ANA_SFLOW_CFG, 0x009050), + REG(ANA_PORT_MODE, 0x009080), + REG(ANA_PGID_PGID, 0x008c00), + REG(ANA_TABLES_ANMOVED, 0x008b30), + REG(ANA_TABLES_MACHDATA, 0x008b34), + REG(ANA_TABLES_MACLDATA, 0x008b38), + REG(ANA_TABLES_MACACCESS, 0x008b3c), + REG(ANA_TABLES_MACTINDX, 0x008b40), + REG(ANA_TABLES_VLANACCESS, 0x008b44), + REG(ANA_TABLES_VLANTIDX, 0x008b48), + REG(ANA_TABLES_ISDXACCESS, 0x008b4c), + REG(ANA_TABLES_ISDXTIDX, 0x008b50), + REG(ANA_TABLES_ENTRYLIM, 0x008b00), + REG(ANA_TABLES_PTP_ID_HIGH, 0x008b54), + REG(ANA_TABLES_PTP_ID_LOW, 0x008b58), + REG(ANA_MSTI_STATE, 0x008e00), + REG(ANA_PORT_VLAN_CFG, 0x007000), + REG(ANA_PORT_DROP_CFG, 0x007004), + REG(ANA_PORT_QOS_CFG, 0x007008), + REG(ANA_PORT_VCAP_CFG, 0x00700c), + REG(ANA_PORT_VCAP_S1_KEY_CFG, 0x007010), + REG(ANA_PORT_VCAP_S2_CFG, 0x00701c), + REG(ANA_PORT_PCP_DEI_MAP, 0x007020), + REG(ANA_PORT_CPU_FWD_CFG, 0x007060), + REG(ANA_PORT_CPU_FWD_BPDU_CFG, 0x007064), + REG(ANA_PORT_CPU_FWD_GARP_CFG, 0x007068), + REG(ANA_PORT_CPU_FWD_CCM_CFG, 0x00706c), + REG(ANA_PORT_PORT_CFG, 0x007070), + REG(ANA_PORT_POL_CFG, 0x007074), + REG(ANA_PORT_PTP_CFG, 0x007078), + REG(ANA_PORT_PTP_DLY1_CFG, 0x00707c), + REG(ANA_OAM_UPM_LM_CNT, 0x007c00), + REG(ANA_PORT_PTP_DLY2_CFG, 0x007080), + REG(ANA_PFC_PFC_CFG, 0x008800), + REG(ANA_PFC_PFC_TIMER, 0x008804), + REG(ANA_IPT_OAM_MEP_CFG, 0x008000), + REG(ANA_IPT_IPT, 0x008004), + REG(ANA_PPT_PPT, 0x008ac0), + REG(ANA_FID_MAP_FID_MAP, 0x000000), + REG(ANA_AGGR_CFG, 0x0090b4), + REG(ANA_CPUQ_CFG, 0x0090b8), + REG(ANA_CPUQ_CFG2, 0x0090bc), + REG(ANA_CPUQ_8021_CFG, 0x0090c0), + REG(ANA_DSCP_CFG, 0x009100), + REG(ANA_DSCP_REWR_CFG, 0x009200), + REG(ANA_VCAP_RNG_TYPE_CFG, 0x009240), + REG(ANA_VCAP_RNG_VAL_CFG, 0x009260), + REG(ANA_VRAP_CFG, 0x009280), + REG(ANA_VRAP_HDR_DATA, 0x009284), + REG(ANA_VRAP_HDR_MASK, 0x009288), + REG(ANA_DISCARD_CFG, 0x00928c), + REG(ANA_FID_CFG, 0x009290), + REG(ANA_POL_PIR_CFG, 0x004000), + REG(ANA_POL_CIR_CFG, 0x004004), + REG(ANA_POL_MODE_CFG, 0x004008), + REG(ANA_POL_PIR_STATE, 0x00400c), + REG(ANA_POL_CIR_STATE, 0x004010), + REG(ANA_POL_STATE, 0x004014), + REG(ANA_POL_FLOWC, 0x008b80), + REG(ANA_POL_HYST, 0x008bec), + REG(ANA_POL_MISC_CFG, 0x008bf0), +}; +EXPORT_SYMBOL(vsc7514_ana_regmap); + +const u32 vsc7514_qs_regmap[] = { + REG(QS_XTR_GRP_CFG, 0x000000), + REG(QS_XTR_RD, 0x000008), + REG(QS_XTR_FRM_PRUNING, 0x000010), + REG(QS_XTR_FLUSH, 0x000018), + REG(QS_XTR_DATA_PRESENT, 0x00001c), + REG(QS_XTR_CFG, 0x000020), + REG(QS_INJ_GRP_CFG, 0x000024), + REG(QS_INJ_WR, 0x00002c), + REG(QS_INJ_CTRL, 0x000034), + REG(QS_INJ_STATUS, 0x00003c), + REG(QS_INJ_ERR, 0x000040), + REG(QS_INH_DBG, 0x000048), +}; +EXPORT_SYMBOL(vsc7514_qs_regmap); + +const u32 vsc7514_qsys_regmap[] = { + REG(QSYS_PORT_MODE, 0x011200), + REG(QSYS_SWITCH_PORT_MODE, 0x011234), + REG(QSYS_STAT_CNT_CFG, 0x011264), + REG(QSYS_EEE_CFG, 0x011268), + REG(QSYS_EEE_THRES, 0x011294), + REG(QSYS_IGR_NO_SHARING, 0x011298), + REG(QSYS_EGR_NO_SHARING, 0x01129c), + REG(QSYS_SW_STATUS, 0x0112a0), + REG(QSYS_EXT_CPU_CFG, 0x0112d0), + REG(QSYS_PAD_CFG, 0x0112d4), + REG(QSYS_CPU_GROUP_MAP, 0x0112d8), + REG(QSYS_QMAP, 0x0112dc), + REG(QSYS_ISDX_SGRP, 0x011400), + REG(QSYS_TIMED_FRAME_ENTRY, 0x014000), + REG(QSYS_TFRM_MISC, 0x011310), + REG(QSYS_TFRM_PORT_DLY, 0x011314), + REG(QSYS_TFRM_TIMER_CFG_1, 0x011318), + REG(QSYS_TFRM_TIMER_CFG_2, 0x01131c), + REG(QSYS_TFRM_TIMER_CFG_3, 0x011320), + REG(QSYS_TFRM_TIMER_CFG_4, 0x011324), + REG(QSYS_TFRM_TIMER_CFG_5, 0x011328), + REG(QSYS_TFRM_TIMER_CFG_6, 0x01132c), + REG(QSYS_TFRM_TIMER_CFG_7, 0x011330), + REG(QSYS_TFRM_TIMER_CFG_8, 0x011334), + REG(QSYS_RED_PROFILE, 0x011338), + REG(QSYS_RES_QOS_MODE, 0x011378), + REG(QSYS_RES_CFG, 0x012000), + REG(QSYS_RES_STAT, 0x012004), + REG(QSYS_EGR_DROP_MODE, 0x01137c), + REG(QSYS_EQ_CTRL, 0x011380), + REG(QSYS_EVENTS_CORE, 0x011384), + REG(QSYS_CIR_CFG, 0x000000), + REG(QSYS_EIR_CFG, 0x000004), + REG(QSYS_SE_CFG, 0x000008), + REG(QSYS_SE_DWRR_CFG, 0x00000c), + REG(QSYS_SE_CONNECT, 0x00003c), + REG(QSYS_SE_DLB_SENSE, 0x000040), + REG(QSYS_CIR_STATE, 0x000044), + REG(QSYS_EIR_STATE, 0x000048), + REG(QSYS_SE_STATE, 0x00004c), + REG(QSYS_HSCH_MISC_CFG, 0x011388), +}; +EXPORT_SYMBOL(vsc7514_qsys_regmap); + +const u32 vsc7514_rew_regmap[] = { + REG(REW_PORT_VLAN_CFG, 0x000000), + REG(REW_TAG_CFG, 0x000004), + REG(REW_PORT_CFG, 0x000008), + REG(REW_DSCP_CFG, 0x00000c), + REG(REW_PCP_DEI_QOS_MAP_CFG, 0x000010), + REG(REW_PTP_CFG, 0x000050), + REG(REW_PTP_DLY1_CFG, 0x000054), + REG(REW_DSCP_REMAP_DP1_CFG, 0x000690), + REG(REW_DSCP_REMAP_CFG, 0x000790), + REG(REW_STAT_CFG, 0x000890), + REG(REW_PPT, 0x000680), +}; +EXPORT_SYMBOL(vsc7514_rew_regmap); + +const u32 vsc7514_sys_regmap[] = { + REG(SYS_COUNT_RX_OCTETS, 0x000000), + REG(SYS_COUNT_RX_UNICAST, 0x000004), + REG(SYS_COUNT_RX_MULTICAST, 0x000008), + REG(SYS_COUNT_RX_BROADCAST, 0x00000c), + REG(SYS_COUNT_RX_SHORTS, 0x000010), + REG(SYS_COUNT_RX_FRAGMENTS, 0x000014), + REG(SYS_COUNT_RX_JABBERS, 0x000018), + REG(SYS_COUNT_RX_CRC_ALIGN_ERRS, 0x00001c), + REG(SYS_COUNT_RX_SYM_ERRS, 0x000020), + REG(SYS_COUNT_RX_64, 0x000024), + REG(SYS_COUNT_RX_65_127, 0x000028), + REG(SYS_COUNT_RX_128_255, 0x00002c), + REG(SYS_COUNT_RX_256_1023, 0x000030), + REG(SYS_COUNT_RX_1024_1526, 0x000034), + REG(SYS_COUNT_RX_1527_MAX, 0x000038), + REG(SYS_COUNT_RX_PAUSE, 0x00003c), + REG(SYS_COUNT_RX_CONTROL, 0x000040), + REG(SYS_COUNT_RX_LONGS, 0x000044), + REG(SYS_COUNT_RX_CLASSIFIED_DROPS, 0x000048), + REG(SYS_COUNT_TX_OCTETS, 0x000100), + REG(SYS_COUNT_TX_UNICAST, 0x000104), + REG(SYS_COUNT_TX_MULTICAST, 0x000108), + REG(SYS_COUNT_TX_BROADCAST, 0x00010c), + REG(SYS_COUNT_TX_COLLISION, 0x000110), + REG(SYS_COUNT_TX_DROPS, 0x000114), + REG(SYS_COUNT_TX_PAUSE, 0x000118), + REG(SYS_COUNT_TX_64, 0x00011c), + REG(SYS_COUNT_TX_65_127, 0x000120), + REG(SYS_COUNT_TX_128_511, 0x000124), + REG(SYS_COUNT_TX_512_1023, 0x000128), + REG(SYS_COUNT_TX_1024_1526, 0x00012c), + REG(SYS_COUNT_TX_1527_MAX, 0x000130), + REG(SYS_COUNT_TX_AGING, 0x000170), + REG(SYS_RESET_CFG, 0x000508), + REG(SYS_CMID, 0x00050c), + REG(SYS_VLAN_ETYPE_CFG, 0x000510), + REG(SYS_PORT_MODE, 0x000514), + REG(SYS_FRONT_PORT_MODE, 0x000548), + REG(SYS_FRM_AGING, 0x000574), + REG(SYS_STAT_CFG, 0x000578), + REG(SYS_SW_STATUS, 0x00057c), + REG(SYS_MISC_CFG, 0x0005ac), + REG(SYS_REW_MAC_HIGH_CFG, 0x0005b0), + REG(SYS_REW_MAC_LOW_CFG, 0x0005dc), + REG(SYS_CM_ADDR, 0x000500), + REG(SYS_CM_DATA, 0x000504), + REG(SYS_PAUSE_CFG, 0x000608), + REG(SYS_PAUSE_TOT_CFG, 0x000638), + REG(SYS_ATOP, 0x00063c), + REG(SYS_ATOP_TOT_CFG, 0x00066c), + REG(SYS_MAC_FC_CFG, 0x000670), + REG(SYS_MMGT, 0x00069c), + REG(SYS_MMGT_FAST, 0x0006a0), + REG(SYS_EVENTS_DIF, 0x0006a4), + REG(SYS_EVENTS_CORE, 0x0006b4), + REG(SYS_CNT, 0x000000), + REG(SYS_PTP_STATUS, 0x0006b8), + REG(SYS_PTP_TXSTAMP, 0x0006bc), + REG(SYS_PTP_NXT, 0x0006c0), + REG(SYS_PTP_CFG, 0x0006c4), +}; +EXPORT_SYMBOL(vsc7514_sys_regmap); + +const u32 vsc7514_vcap_regmap[] = { + /* VCAP_CORE_CFG */ + REG(VCAP_CORE_UPDATE_CTRL, 0x000000), + REG(VCAP_CORE_MV_CFG, 0x000004), + /* VCAP_CORE_CACHE */ + REG(VCAP_CACHE_ENTRY_DAT, 0x000008), + REG(VCAP_CACHE_MASK_DAT, 0x000108), + REG(VCAP_CACHE_ACTION_DAT, 0x000208), + REG(VCAP_CACHE_CNT_DAT, 0x000308), + REG(VCAP_CACHE_TG_DAT, 0x000388), + /* VCAP_CONST */ + REG(VCAP_CONST_VCAP_VER, 0x000398), + REG(VCAP_CONST_ENTRY_WIDTH, 0x00039c), + REG(VCAP_CONST_ENTRY_CNT, 0x0003a0), + REG(VCAP_CONST_ENTRY_SWCNT, 0x0003a4), + REG(VCAP_CONST_ENTRY_TG_WIDTH, 0x0003a8), + REG(VCAP_CONST_ACTION_DEF_CNT, 0x0003ac), + REG(VCAP_CONST_ACTION_WIDTH, 0x0003b0), + REG(VCAP_CONST_CNT_WIDTH, 0x0003b4), + REG(VCAP_CONST_CORE_CNT, 0x0003b8), + REG(VCAP_CONST_IF_CNT, 0x0003bc), +}; +EXPORT_SYMBOL(vsc7514_vcap_regmap); + +const u32 vsc7514_ptp_regmap[] = { + REG(PTP_PIN_CFG, 0x000000), + REG(PTP_PIN_TOD_SEC_MSB, 0x000004), + REG(PTP_PIN_TOD_SEC_LSB, 0x000008), + REG(PTP_PIN_TOD_NSEC, 0x00000c), + REG(PTP_PIN_WF_HIGH_PERIOD, 0x000014), + REG(PTP_PIN_WF_LOW_PERIOD, 0x000018), + REG(PTP_CFG_MISC, 0x0000a0), + REG(PTP_CLK_CFG_ADJ_CFG, 0x0000a4), + REG(PTP_CLK_CFG_ADJ_FREQ, 0x0000a8), +}; +EXPORT_SYMBOL(vsc7514_ptp_regmap); + +const u32 vsc7514_dev_gmii_regmap[] = { + REG(DEV_CLOCK_CFG, 0x0), + REG(DEV_PORT_MISC, 0x4), + REG(DEV_EVENTS, 0x8), + REG(DEV_EEE_CFG, 0xc), + REG(DEV_RX_PATH_DELAY, 0x10), + REG(DEV_TX_PATH_DELAY, 0x14), + REG(DEV_PTP_PREDICT_CFG, 0x18), + REG(DEV_MAC_ENA_CFG, 0x1c), + REG(DEV_MAC_MODE_CFG, 0x20), + REG(DEV_MAC_MAXLEN_CFG, 0x24), + REG(DEV_MAC_TAGS_CFG, 0x28), + REG(DEV_MAC_ADV_CHK_CFG, 0x2c), + REG(DEV_MAC_IFG_CFG, 0x30), + REG(DEV_MAC_HDX_CFG, 0x34), + REG(DEV_MAC_DBG_CFG, 0x38), + REG(DEV_MAC_FC_MAC_LOW_CFG, 0x3c), + REG(DEV_MAC_FC_MAC_HIGH_CFG, 0x40), + REG(DEV_MAC_STICKY, 0x44), + REG(PCS1G_CFG, 0x48), + REG(PCS1G_MODE_CFG, 0x4c), + REG(PCS1G_SD_CFG, 0x50), + REG(PCS1G_ANEG_CFG, 0x54), + REG(PCS1G_ANEG_NP_CFG, 0x58), + REG(PCS1G_LB_CFG, 0x5c), + REG(PCS1G_DBG_CFG, 0x60), + REG(PCS1G_CDET_CFG, 0x64), + REG(PCS1G_ANEG_STATUS, 0x68), + REG(PCS1G_ANEG_NP_STATUS, 0x6c), + REG(PCS1G_LINK_STATUS, 0x70), + REG(PCS1G_LINK_DOWN_CNT, 0x74), + REG(PCS1G_STICKY, 0x78), + REG(PCS1G_DEBUG_STATUS, 0x7c), + REG(PCS1G_LPI_CFG, 0x80), + REG(PCS1G_LPI_WAKE_ERROR_CNT, 0x84), + REG(PCS1G_LPI_STATUS, 0x88), + REG(PCS1G_TSTPAT_MODE_CFG, 0x8c), + REG(PCS1G_TSTPAT_STATUS, 0x90), + REG(DEV_PCS_FX100_CFG, 0x94), + REG(DEV_PCS_FX100_STATUS, 0x98), +}; +EXPORT_SYMBOL(vsc7514_dev_gmii_regmap); + +const struct vcap_field vsc7514_vcap_es0_keys[] = { + [VCAP_ES0_EGR_PORT] = { 0, 4 }, + [VCAP_ES0_IGR_PORT] = { 4, 4 }, + [VCAP_ES0_RSV] = { 8, 2 }, + [VCAP_ES0_L2_MC] = { 10, 1 }, + [VCAP_ES0_L2_BC] = { 11, 1 }, + [VCAP_ES0_VID] = { 12, 12 }, + [VCAP_ES0_DP] = { 24, 1 }, + [VCAP_ES0_PCP] = { 25, 3 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_es0_keys); + +const struct vcap_field vsc7514_vcap_es0_actions[] = { + [VCAP_ES0_ACT_PUSH_OUTER_TAG] = { 0, 2 }, + [VCAP_ES0_ACT_PUSH_INNER_TAG] = { 2, 1 }, + [VCAP_ES0_ACT_TAG_A_TPID_SEL] = { 3, 2 }, + [VCAP_ES0_ACT_TAG_A_VID_SEL] = { 5, 1 }, + [VCAP_ES0_ACT_TAG_A_PCP_SEL] = { 6, 2 }, + [VCAP_ES0_ACT_TAG_A_DEI_SEL] = { 8, 2 }, + [VCAP_ES0_ACT_TAG_B_TPID_SEL] = { 10, 2 }, + [VCAP_ES0_ACT_TAG_B_VID_SEL] = { 12, 1 }, + [VCAP_ES0_ACT_TAG_B_PCP_SEL] = { 13, 2 }, + [VCAP_ES0_ACT_TAG_B_DEI_SEL] = { 15, 2 }, + [VCAP_ES0_ACT_VID_A_VAL] = { 17, 12 }, + [VCAP_ES0_ACT_PCP_A_VAL] = { 29, 3 }, + [VCAP_ES0_ACT_DEI_A_VAL] = { 32, 1 }, + [VCAP_ES0_ACT_VID_B_VAL] = { 33, 12 }, + [VCAP_ES0_ACT_PCP_B_VAL] = { 45, 3 }, + [VCAP_ES0_ACT_DEI_B_VAL] = { 48, 1 }, + [VCAP_ES0_ACT_RSV] = { 49, 24 }, + [VCAP_ES0_ACT_HIT_STICKY] = { 73, 1 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_es0_actions); + +const struct vcap_field vsc7514_vcap_is1_keys[] = { + [VCAP_IS1_HK_TYPE] = { 0, 1 }, + [VCAP_IS1_HK_LOOKUP] = { 1, 2 }, + [VCAP_IS1_HK_IGR_PORT_MASK] = { 3, 12 }, + [VCAP_IS1_HK_RSV] = { 15, 9 }, + [VCAP_IS1_HK_OAM_Y1731] = { 24, 1 }, + [VCAP_IS1_HK_L2_MC] = { 25, 1 }, + [VCAP_IS1_HK_L2_BC] = { 26, 1 }, + [VCAP_IS1_HK_IP_MC] = { 27, 1 }, + [VCAP_IS1_HK_VLAN_TAGGED] = { 28, 1 }, + [VCAP_IS1_HK_VLAN_DBL_TAGGED] = { 29, 1 }, + [VCAP_IS1_HK_TPID] = { 30, 1 }, + [VCAP_IS1_HK_VID] = { 31, 12 }, + [VCAP_IS1_HK_DEI] = { 43, 1 }, + [VCAP_IS1_HK_PCP] = { 44, 3 }, + /* Specific Fields for IS1 Half Key S1_NORMAL */ + [VCAP_IS1_HK_L2_SMAC] = { 47, 48 }, + [VCAP_IS1_HK_ETYPE_LEN] = { 95, 1 }, + [VCAP_IS1_HK_ETYPE] = { 96, 16 }, + [VCAP_IS1_HK_IP_SNAP] = { 112, 1 }, + [VCAP_IS1_HK_IP4] = { 113, 1 }, + /* Layer-3 Information */ + [VCAP_IS1_HK_L3_FRAGMENT] = { 114, 1 }, + [VCAP_IS1_HK_L3_FRAG_OFS_GT0] = { 115, 1 }, + [VCAP_IS1_HK_L3_OPTIONS] = { 116, 1 }, + [VCAP_IS1_HK_L3_DSCP] = { 117, 6 }, + [VCAP_IS1_HK_L3_IP4_SIP] = { 123, 32 }, + /* Layer-4 Information */ + [VCAP_IS1_HK_TCP_UDP] = { 155, 1 }, + [VCAP_IS1_HK_TCP] = { 156, 1 }, + [VCAP_IS1_HK_L4_SPORT] = { 157, 16 }, + [VCAP_IS1_HK_L4_RNG] = { 173, 8 }, + /* Specific Fields for IS1 Half Key S1_5TUPLE_IP4 */ + [VCAP_IS1_HK_IP4_INNER_TPID] = { 47, 1 }, + [VCAP_IS1_HK_IP4_INNER_VID] = { 48, 12 }, + [VCAP_IS1_HK_IP4_INNER_DEI] = { 60, 1 }, + [VCAP_IS1_HK_IP4_INNER_PCP] = { 61, 3 }, + [VCAP_IS1_HK_IP4_IP4] = { 64, 1 }, + [VCAP_IS1_HK_IP4_L3_FRAGMENT] = { 65, 1 }, + [VCAP_IS1_HK_IP4_L3_FRAG_OFS_GT0] = { 66, 1 }, + [VCAP_IS1_HK_IP4_L3_OPTIONS] = { 67, 1 }, + [VCAP_IS1_HK_IP4_L3_DSCP] = { 68, 6 }, + [VCAP_IS1_HK_IP4_L3_IP4_DIP] = { 74, 32 }, + [VCAP_IS1_HK_IP4_L3_IP4_SIP] = { 106, 32 }, + [VCAP_IS1_HK_IP4_L3_PROTO] = { 138, 8 }, + [VCAP_IS1_HK_IP4_TCP_UDP] = { 146, 1 }, + [VCAP_IS1_HK_IP4_TCP] = { 147, 1 }, + [VCAP_IS1_HK_IP4_L4_RNG] = { 148, 8 }, + [VCAP_IS1_HK_IP4_IP_PAYLOAD_S1_5TUPLE] = { 156, 32 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_is1_keys); + +const struct vcap_field vsc7514_vcap_is1_actions[] = { + [VCAP_IS1_ACT_DSCP_ENA] = { 0, 1 }, + [VCAP_IS1_ACT_DSCP_VAL] = { 1, 6 }, + [VCAP_IS1_ACT_QOS_ENA] = { 7, 1 }, + [VCAP_IS1_ACT_QOS_VAL] = { 8, 3 }, + [VCAP_IS1_ACT_DP_ENA] = { 11, 1 }, + [VCAP_IS1_ACT_DP_VAL] = { 12, 1 }, + [VCAP_IS1_ACT_PAG_OVERRIDE_MASK] = { 13, 8 }, + [VCAP_IS1_ACT_PAG_VAL] = { 21, 8 }, + [VCAP_IS1_ACT_RSV] = { 29, 9 }, + /* The fields below are incorrectly shifted by 2 in the manual */ + [VCAP_IS1_ACT_VID_REPLACE_ENA] = { 38, 1 }, + [VCAP_IS1_ACT_VID_ADD_VAL] = { 39, 12 }, + [VCAP_IS1_ACT_FID_SEL] = { 51, 2 }, + [VCAP_IS1_ACT_FID_VAL] = { 53, 13 }, + [VCAP_IS1_ACT_PCP_DEI_ENA] = { 66, 1 }, + [VCAP_IS1_ACT_PCP_VAL] = { 67, 3 }, + [VCAP_IS1_ACT_DEI_VAL] = { 70, 1 }, + [VCAP_IS1_ACT_VLAN_POP_CNT_ENA] = { 71, 1 }, + [VCAP_IS1_ACT_VLAN_POP_CNT] = { 72, 2 }, + [VCAP_IS1_ACT_CUSTOM_ACE_TYPE_ENA] = { 74, 4 }, + [VCAP_IS1_ACT_HIT_STICKY] = { 78, 1 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_is1_actions); + +const struct vcap_field vsc7514_vcap_is2_keys[] = { + /* Common: 46 bits */ + [VCAP_IS2_TYPE] = { 0, 4 }, + [VCAP_IS2_HK_FIRST] = { 4, 1 }, + [VCAP_IS2_HK_PAG] = { 5, 8 }, + [VCAP_IS2_HK_IGR_PORT_MASK] = { 13, 12 }, + [VCAP_IS2_HK_RSV2] = { 25, 1 }, + [VCAP_IS2_HK_HOST_MATCH] = { 26, 1 }, + [VCAP_IS2_HK_L2_MC] = { 27, 1 }, + [VCAP_IS2_HK_L2_BC] = { 28, 1 }, + [VCAP_IS2_HK_VLAN_TAGGED] = { 29, 1 }, + [VCAP_IS2_HK_VID] = { 30, 12 }, + [VCAP_IS2_HK_DEI] = { 42, 1 }, + [VCAP_IS2_HK_PCP] = { 43, 3 }, + /* MAC_ETYPE / MAC_LLC / MAC_SNAP / OAM common */ + [VCAP_IS2_HK_L2_DMAC] = { 46, 48 }, + [VCAP_IS2_HK_L2_SMAC] = { 94, 48 }, + /* MAC_ETYPE (TYPE=000) */ + [VCAP_IS2_HK_MAC_ETYPE_ETYPE] = { 142, 16 }, + [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD0] = { 158, 16 }, + [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD1] = { 174, 8 }, + [VCAP_IS2_HK_MAC_ETYPE_L2_PAYLOAD2] = { 182, 3 }, + /* MAC_LLC (TYPE=001) */ + [VCAP_IS2_HK_MAC_LLC_L2_LLC] = { 142, 40 }, + /* MAC_SNAP (TYPE=010) */ + [VCAP_IS2_HK_MAC_SNAP_L2_SNAP] = { 142, 40 }, + /* MAC_ARP (TYPE=011) */ + [VCAP_IS2_HK_MAC_ARP_SMAC] = { 46, 48 }, + [VCAP_IS2_HK_MAC_ARP_ADDR_SPACE_OK] = { 94, 1 }, + [VCAP_IS2_HK_MAC_ARP_PROTO_SPACE_OK] = { 95, 1 }, + [VCAP_IS2_HK_MAC_ARP_LEN_OK] = { 96, 1 }, + [VCAP_IS2_HK_MAC_ARP_TARGET_MATCH] = { 97, 1 }, + [VCAP_IS2_HK_MAC_ARP_SENDER_MATCH] = { 98, 1 }, + [VCAP_IS2_HK_MAC_ARP_OPCODE_UNKNOWN] = { 99, 1 }, + [VCAP_IS2_HK_MAC_ARP_OPCODE] = { 100, 2 }, + [VCAP_IS2_HK_MAC_ARP_L3_IP4_DIP] = { 102, 32 }, + [VCAP_IS2_HK_MAC_ARP_L3_IP4_SIP] = { 134, 32 }, + [VCAP_IS2_HK_MAC_ARP_DIP_EQ_SIP] = { 166, 1 }, + /* IP4_TCP_UDP / IP4_OTHER common */ + [VCAP_IS2_HK_IP4] = { 46, 1 }, + [VCAP_IS2_HK_L3_FRAGMENT] = { 47, 1 }, + [VCAP_IS2_HK_L3_FRAG_OFS_GT0] = { 48, 1 }, + [VCAP_IS2_HK_L3_OPTIONS] = { 49, 1 }, + [VCAP_IS2_HK_IP4_L3_TTL_GT0] = { 50, 1 }, + [VCAP_IS2_HK_L3_TOS] = { 51, 8 }, + [VCAP_IS2_HK_L3_IP4_DIP] = { 59, 32 }, + [VCAP_IS2_HK_L3_IP4_SIP] = { 91, 32 }, + [VCAP_IS2_HK_DIP_EQ_SIP] = { 123, 1 }, + /* IP4_TCP_UDP (TYPE=100) */ + [VCAP_IS2_HK_TCP] = { 124, 1 }, + [VCAP_IS2_HK_L4_DPORT] = { 125, 16 }, + [VCAP_IS2_HK_L4_SPORT] = { 141, 16 }, + [VCAP_IS2_HK_L4_RNG] = { 157, 8 }, + [VCAP_IS2_HK_L4_SPORT_EQ_DPORT] = { 165, 1 }, + [VCAP_IS2_HK_L4_SEQUENCE_EQ0] = { 166, 1 }, + [VCAP_IS2_HK_L4_FIN] = { 167, 1 }, + [VCAP_IS2_HK_L4_SYN] = { 168, 1 }, + [VCAP_IS2_HK_L4_RST] = { 169, 1 }, + [VCAP_IS2_HK_L4_PSH] = { 170, 1 }, + [VCAP_IS2_HK_L4_ACK] = { 171, 1 }, + [VCAP_IS2_HK_L4_URG] = { 172, 1 }, + [VCAP_IS2_HK_L4_1588_DOM] = { 173, 8 }, + [VCAP_IS2_HK_L4_1588_VER] = { 181, 4 }, + /* IP4_OTHER (TYPE=101) */ + [VCAP_IS2_HK_IP4_L3_PROTO] = { 124, 8 }, + [VCAP_IS2_HK_L3_PAYLOAD] = { 132, 56 }, + /* IP6_STD (TYPE=110) */ + [VCAP_IS2_HK_IP6_L3_TTL_GT0] = { 46, 1 }, + [VCAP_IS2_HK_L3_IP6_SIP] = { 47, 128 }, + [VCAP_IS2_HK_IP6_L3_PROTO] = { 175, 8 }, + /* OAM (TYPE=111) */ + [VCAP_IS2_HK_OAM_MEL_FLAGS] = { 142, 7 }, + [VCAP_IS2_HK_OAM_VER] = { 149, 5 }, + [VCAP_IS2_HK_OAM_OPCODE] = { 154, 8 }, + [VCAP_IS2_HK_OAM_FLAGS] = { 162, 8 }, + [VCAP_IS2_HK_OAM_MEPID] = { 170, 16 }, + [VCAP_IS2_HK_OAM_CCM_CNTS_EQ0] = { 186, 1 }, + [VCAP_IS2_HK_OAM_IS_Y1731] = { 187, 1 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_is2_keys); + +const struct vcap_field vsc7514_vcap_is2_actions[] = { + [VCAP_IS2_ACT_HIT_ME_ONCE] = { 0, 1 }, + [VCAP_IS2_ACT_CPU_COPY_ENA] = { 1, 1 }, + [VCAP_IS2_ACT_CPU_QU_NUM] = { 2, 3 }, + [VCAP_IS2_ACT_MASK_MODE] = { 5, 2 }, + [VCAP_IS2_ACT_MIRROR_ENA] = { 7, 1 }, + [VCAP_IS2_ACT_LRN_DIS] = { 8, 1 }, + [VCAP_IS2_ACT_POLICE_ENA] = { 9, 1 }, + [VCAP_IS2_ACT_POLICE_IDX] = { 10, 9 }, + [VCAP_IS2_ACT_POLICE_VCAP_ONLY] = { 19, 1 }, + [VCAP_IS2_ACT_PORT_MASK] = { 20, 11 }, + [VCAP_IS2_ACT_REW_OP] = { 31, 9 }, + [VCAP_IS2_ACT_SMAC_REPLACE_ENA] = { 40, 1 }, + [VCAP_IS2_ACT_RSV] = { 41, 2 }, + [VCAP_IS2_ACT_ACL_ID] = { 43, 6 }, + [VCAP_IS2_ACT_HIT_CNT] = { 49, 32 }, +}; +EXPORT_SYMBOL(vsc7514_vcap_is2_actions); diff --git a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c index 5736fcdafd7a..83a5e29c836a 100644 --- a/drivers/net/ethernet/myricom/myri10ge/myri10ge.c +++ b/drivers/net/ethernet/myricom/myri10ge/myri10ge.c @@ -1704,7 +1704,9 @@ myri10ge_set_pauseparam(struct net_device *netdev, static void myri10ge_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct myri10ge_priv *mgp = netdev_priv(netdev); diff --git a/drivers/net/ethernet/natsemi/jazzsonic.c b/drivers/net/ethernet/natsemi/jazzsonic.c index d74a80f010c5..3f371faeb6d0 100644 --- a/drivers/net/ethernet/natsemi/jazzsonic.c +++ b/drivers/net/ethernet/natsemi/jazzsonic.c @@ -114,6 +114,7 @@ static int sonic_probe1(struct net_device *dev) struct sonic_local *lp = netdev_priv(dev); int err = -ENODEV; int i; + unsigned char addr[ETH_ALEN]; if (!request_mem_region(dev->base_addr, SONIC_MEM_SIZE, jazz_sonic_string)) return -EBUSY; @@ -143,9 +144,10 @@ static int sonic_probe1(struct net_device *dev) SONIC_WRITE(SONIC_CEP,0); for (i=0; i<3; i++) { val = SONIC_READ(SONIC_CAP0-i); - dev->dev_addr[i*2] = val; - dev->dev_addr[i*2+1] = val >> 8; + addr[i*2] = val; + addr[i*2+1] = val >> 8; } + eth_hw_addr_set(dev, addr); lp->dma_bitmode = SONIC_BITMODE32; diff --git a/drivers/net/ethernet/natsemi/macsonic.c b/drivers/net/ethernet/natsemi/macsonic.c index 8709d700e15a..b16f7c830f9b 100644 --- a/drivers/net/ethernet/natsemi/macsonic.c +++ b/drivers/net/ethernet/natsemi/macsonic.c @@ -203,6 +203,7 @@ static void mac_onboard_sonic_ethernet_addr(struct net_device *dev) struct sonic_local *lp = netdev_priv(dev); const int prom_addr = ONBOARD_SONIC_PROM_BASE; unsigned short val; + u8 addr[ETH_ALEN]; /* * On NuBus boards we can sometimes look in the ROM resources. @@ -213,7 +214,8 @@ static void mac_onboard_sonic_ethernet_addr(struct net_device *dev) int i; for (i = 0; i < 6; i++) - dev->dev_addr[i] = SONIC_READ_PROM(i); + addr[i] = SONIC_READ_PROM(i); + eth_hw_addr_set(dev, addr); if (!INVALID_MAC(dev->dev_addr)) return; @@ -222,7 +224,8 @@ static void mac_onboard_sonic_ethernet_addr(struct net_device *dev) * source has a rather long and detailed historical account of * why this is so. */ - bit_reverse_addr(dev->dev_addr); + bit_reverse_addr(addr); + eth_hw_addr_set(dev, addr); if (!INVALID_MAC(dev->dev_addr)) return; @@ -243,14 +246,15 @@ static void mac_onboard_sonic_ethernet_addr(struct net_device *dev) SONIC_WRITE(SONIC_CEP, 15); val = SONIC_READ(SONIC_CAP2); - dev->dev_addr[5] = val >> 8; - dev->dev_addr[4] = val & 0xff; + addr[5] = val >> 8; + addr[4] = val & 0xff; val = SONIC_READ(SONIC_CAP1); - dev->dev_addr[3] = val >> 8; - dev->dev_addr[2] = val & 0xff; + addr[3] = val >> 8; + addr[2] = val & 0xff; val = SONIC_READ(SONIC_CAP0); - dev->dev_addr[1] = val >> 8; - dev->dev_addr[0] = val & 0xff; + addr[1] = val >> 8; + addr[0] = val & 0xff; + eth_hw_addr_set(dev, addr); if (!INVALID_MAC(dev->dev_addr)) return; @@ -355,13 +359,16 @@ static int mac_onboard_sonic_probe(struct net_device *dev) static int mac_sonic_nubus_ethernet_addr(struct net_device *dev, unsigned long prom_addr, int id) { + u8 addr[ETH_ALEN]; int i; + for(i = 0; i < 6; i++) - dev->dev_addr[i] = SONIC_READ_PROM(i); + addr[i] = SONIC_READ_PROM(i); /* Some of the addresses are bit-reversed */ if (id != MACSONIC_DAYNA) - bit_reverse_addr(dev->dev_addr); + bit_reverse_addr(addr); + eth_hw_addr_set(dev, addr); return 0; } diff --git a/drivers/net/ethernet/natsemi/xtsonic.c b/drivers/net/ethernet/natsemi/xtsonic.c index 0a02d8bd0a3e..52fef34d43f9 100644 --- a/drivers/net/ethernet/natsemi/xtsonic.c +++ b/drivers/net/ethernet/natsemi/xtsonic.c @@ -127,6 +127,7 @@ static int sonic_probe1(struct net_device *dev) unsigned int base_addr = dev->base_addr; int i; int err = 0; + unsigned char addr[ETH_ALEN]; if (!request_mem_region(base_addr, 0x100, xtsonic_string)) return -EBUSY; @@ -163,9 +164,10 @@ static int sonic_probe1(struct net_device *dev) for (i=0; i<3; i++) { unsigned int val = SONIC_READ(SONIC_CAP0-i); - dev->dev_addr[i*2] = val; - dev->dev_addr[i*2+1] = val >> 8; + addr[i*2] = val; + addr[i*2+1] = val >> 8; } + eth_hw_addr_set(dev, addr); lp->dma_bitmode = SONIC_BITMODE32; diff --git a/drivers/net/ethernet/neterion/s2io.c b/drivers/net/ethernet/neterion/s2io.c index d1c32c65db05..d6ae44f164a9 100644 --- a/drivers/net/ethernet/neterion/s2io.c +++ b/drivers/net/ethernet/neterion/s2io.c @@ -5461,8 +5461,11 @@ static int s2io_ethtool_set_led(struct net_device *dev, return 0; } -static void s2io_ethtool_gringparam(struct net_device *dev, - struct ethtool_ringparam *ering) +static void +s2io_ethtool_gringparam(struct net_device *dev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct s2io_nic *sp = netdev_priv(dev); int i, tx_desc_count = 0, rx_desc_count = 0; diff --git a/drivers/net/ethernet/netronome/nfp/flower/metadata.c b/drivers/net/ethernet/netronome/nfp/flower/metadata.c index 2af9faee96c5..0c60a436a8f2 100644 --- a/drivers/net/ethernet/netronome/nfp/flower/metadata.c +++ b/drivers/net/ethernet/netronome/nfp/flower/metadata.c @@ -338,11 +338,6 @@ int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie, nfp_flow->meta.mask_len, &nfp_flow->meta.flags, &new_mask_id)) { NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot allocate a new mask id"); - if (nfp_release_stats_entry(app, stats_cxt)) { - NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release stats context"); - err = -EINVAL; - goto err_remove_rhash; - } err = -ENOENT; goto err_remove_rhash; } @@ -359,21 +354,6 @@ int nfp_compile_flow_metadata(struct nfp_app *app, u32 cookie, check_entry = nfp_flower_search_fl_table(app, cookie, netdev); if (check_entry) { NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot offload duplicate flow entry"); - if (nfp_release_stats_entry(app, stats_cxt)) { - NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release stats context"); - err = -EINVAL; - goto err_remove_mask; - } - - if (!nfp_flow->pre_tun_rule.dev && - !nfp_check_mask_remove(app, nfp_flow->mask_data, - nfp_flow->meta.mask_len, - NULL, &new_mask_id)) { - NL_SET_ERR_MSG_MOD(extack, "invalid entry: cannot release mask id"); - err = -EINVAL; - goto err_remove_mask; - } - err = -EEXIST; goto err_remove_mask; } diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c index 850bfdf83d0a..6e38da4ad554 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_common.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_common.c @@ -4097,7 +4097,7 @@ static void nfp_net_netdev_init(struct nfp_net *nn) netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = nn->max_mtu; - netdev->gso_max_segs = NFP_NET_LSO_MAX_SEGS; + netif_set_gso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS); netif_carrier_off(netdev); diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c index cf7882933993..e0c27471bcdb 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_ethtool.c @@ -381,7 +381,9 @@ err_bad_set: } static void nfp_net_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nfp_net *nn = netdev_priv(netdev); @@ -406,7 +408,9 @@ static int nfp_net_set_ring_size(struct nfp_net *nn, u32 rxd_cnt, u32 txd_cnt) } static int nfp_net_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct nfp_net *nn = netdev_priv(netdev); u32 rxd_cnt, txd_cnt; diff --git a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c index 369f6ae700c7..181ac8e789a3 100644 --- a/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c +++ b/drivers/net/ethernet/netronome/nfp/nfp_net_repr.c @@ -286,8 +286,8 @@ nfp_repr_transfer_features(struct net_device *netdev, struct net_device *lower) if (repr->dst->u.port_info.lower_dev != lower) return; - netdev->gso_max_size = lower->gso_max_size; - netdev->gso_max_segs = lower->gso_max_segs; + netif_set_gso_max_size(netdev, lower->gso_max_size); + netif_set_gso_max_segs(netdev, lower->gso_max_segs); netdev_update_features(netdev); } @@ -381,7 +381,7 @@ int nfp_repr_init(struct nfp_app *app, struct net_device *netdev, /* Advertise but disable TSO by default. */ netdev->features &= ~(NETIF_F_TSO | NETIF_F_TSO6); - netdev->gso_max_segs = NFP_NET_LSO_MAX_SEGS; + netif_set_gso_max_segs(netdev, NFP_NET_LSO_MAX_SEGS); netdev->priv_flags |= IFF_NO_QUEUE | IFF_DISABLE_NETPOLL; netdev->features |= NETIF_F_LLTX; diff --git a/drivers/net/ethernet/nvidia/forcedeth.c b/drivers/net/ethernet/nvidia/forcedeth.c index 9b530d7509a4..660013f716d4 100644 --- a/drivers/net/ethernet/nvidia/forcedeth.c +++ b/drivers/net/ethernet/nvidia/forcedeth.c @@ -4651,7 +4651,10 @@ static int nv_nway_reset(struct net_device *dev) return ret; } -static void nv_get_ringparam(struct net_device *dev, struct ethtool_ringparam* ring) +static void nv_get_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fe_priv *np = netdev_priv(dev); @@ -4662,7 +4665,10 @@ static void nv_get_ringparam(struct net_device *dev, struct ethtool_ringparam* r ring->tx_pending = np->tx_ring_size; } -static int nv_set_ringparam(struct net_device *dev, struct ethtool_ringparam* ring) +static int nv_set_ringparam(struct net_device *dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct fe_priv *np = netdev_priv(dev); u8 __iomem *base = get_hwbase(dev); diff --git a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c index 660b07cb5b92..84cc79e928c8 100644 --- a/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c +++ b/drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_ethtool.c @@ -270,9 +270,13 @@ static int pch_gbe_nway_reset(struct net_device *netdev) * pch_gbe_get_ringparam - Report ring sizes * @netdev: Network interface device structure * @ring: Ring param structure + * @kernel_ring: Ring external param structure + * @extack: netlink handle */ static void pch_gbe_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct pch_gbe_adapter *adapter = netdev_priv(netdev); struct pch_gbe_tx_ring *txdr = adapter->tx_ring; @@ -288,12 +292,16 @@ static void pch_gbe_get_ringparam(struct net_device *netdev, * pch_gbe_set_ringparam - Set ring sizes * @netdev: Network interface device structure * @ring: Ring param structure + * @kernel_ring: Ring external param structure + * @extack: netlink handle * Returns * 0: Successful. * Negative value: Failed. */ static int pch_gbe_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct pch_gbe_adapter *adapter = netdev_priv(netdev); struct pch_gbe_tx_ring *txdr, *tx_old; diff --git a/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c b/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c index e1a304886a3c..4c7e0c991105 100644 --- a/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c +++ b/drivers/net/ethernet/pasemi/pasemi_mac_ethtool.c @@ -69,7 +69,9 @@ pasemi_mac_ethtool_set_msglevel(struct net_device *netdev, static void pasemi_mac_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct pasemi_mac *mac = netdev_priv(netdev); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index c54d735b9e2e..386a5cf1e224 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -512,7 +512,9 @@ static int ionic_set_coalesce(struct net_device *netdev, } static void ionic_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ionic_lif *lif = netdev_priv(netdev); @@ -523,7 +525,9 @@ static void ionic_get_ringparam(struct net_device *netdev, } static int ionic_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ionic_lif *lif = netdev_priv(netdev); struct ionic_queue_params qparam; diff --git a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c index a075643f5826..3c4a84ea6321 100644 --- a/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c +++ b/drivers/net/ethernet/qlogic/netxen/netxen_nic_ethtool.c @@ -392,7 +392,9 @@ netxen_nic_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, static void netxen_nic_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netxen_adapter *adapter = netdev_priv(dev); @@ -430,7 +432,9 @@ netxen_validate_ringparam(u32 val, u32 min, u32 max, char *r_name) static int netxen_nic_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netxen_adapter *adapter = netdev_priv(dev); u16 max_rcv_desc = MAX_RCV_DESCRIPTORS_10G; diff --git a/drivers/net/ethernet/qlogic/qed/qed_cxt.c b/drivers/net/ethernet/qlogic/qed/qed_cxt.c index 452494f8c298..65e20693c549 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_cxt.c +++ b/drivers/net/ethernet/qlogic/qed/qed_cxt.c @@ -1036,12 +1036,12 @@ static void qed_cid_map_free(struct qed_hwfn *p_hwfn) u32 type, vf; for (type = 0; type < MAX_CONN_TYPES; type++) { - kfree(p_mngr->acquired[type].cid_map); + bitmap_free(p_mngr->acquired[type].cid_map); p_mngr->acquired[type].max_count = 0; p_mngr->acquired[type].start_cid = 0; for (vf = 0; vf < MAX_NUM_VFS; vf++) { - kfree(p_mngr->acquired_vf[type][vf].cid_map); + bitmap_free(p_mngr->acquired_vf[type][vf].cid_map); p_mngr->acquired_vf[type][vf].max_count = 0; p_mngr->acquired_vf[type][vf].start_cid = 0; } @@ -1054,15 +1054,10 @@ qed_cid_map_alloc_single(struct qed_hwfn *p_hwfn, u32 cid_start, u32 cid_count, struct qed_cid_acquired_map *p_map) { - u32 size; - if (!cid_count) return 0; - size = DIV_ROUND_UP(cid_count, - sizeof(unsigned long) * BITS_PER_BYTE) * - sizeof(unsigned long); - p_map->cid_map = kzalloc(size, GFP_KERNEL); + p_map->cid_map = bitmap_zalloc(cid_count, GFP_KERNEL); if (!p_map->cid_map) return -ENOMEM; @@ -1216,7 +1211,6 @@ void qed_cxt_mngr_setup(struct qed_hwfn *p_hwfn) struct qed_cid_acquired_map *p_map; struct qed_conn_type_cfg *p_cfg; int type; - u32 len; /* Reset acquired cids */ for (type = 0; type < MAX_CONN_TYPES; type++) { @@ -1225,11 +1219,7 @@ void qed_cxt_mngr_setup(struct qed_hwfn *p_hwfn) p_cfg = &p_mngr->conn_cfg[type]; if (p_cfg->cid_count) { p_map = &p_mngr->acquired[type]; - len = DIV_ROUND_UP(p_map->max_count, - sizeof(unsigned long) * - BITS_PER_BYTE) * - sizeof(unsigned long); - memset(p_map->cid_map, 0, len); + bitmap_zero(p_map->cid_map, p_map->max_count); } if (!p_cfg->cids_per_vf) @@ -1237,11 +1227,7 @@ void qed_cxt_mngr_setup(struct qed_hwfn *p_hwfn) for (vf = 0; vf < MAX_NUM_VFS; vf++) { p_map = &p_mngr->acquired_vf[type][vf]; - len = DIV_ROUND_UP(p_map->max_count, - sizeof(unsigned long) * - BITS_PER_BYTE) * - sizeof(unsigned long); - memset(p_map->cid_map, 0, len); + bitmap_zero(p_map->cid_map, p_map->max_count); } } } diff --git a/drivers/net/ethernet/qlogic/qed/qed_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_hsi.h index f2cedbd9489c..ed1a84542ad2 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_hsi.h +++ b/drivers/net/ethernet/qlogic/qed/qed_hsi.h @@ -2721,6 +2721,25 @@ void qed_memset_task_ctx(void *p_ctx_mem, u32 ctx_size, u8 ctx_type); #define NUM_STORMS 6 /** + * qed_get_protocol_type_str(): Get a string for Protocol type. + * + * @protocol_type: Protocol type (using enum protocol_type). + * + * Return: String. + */ +const char *qed_get_protocol_type_str(u32 protocol_type); + +/** + * qed_get_ramrod_cmd_id_str(): Get a string for Ramrod command ID. + * + * @protocol_type: Protocol type (using enum protocol_type). + * @ramrod_cmd_id: Ramrod command ID (using per-protocol enum <protocol>_ramrod_cmd_id). + * + * Return: String. + */ +const char *qed_get_ramrod_cmd_id_str(u32 protocol_type, u32 ramrod_cmd_id); + +/** * qed_set_rdma_error_level(): Sets the RDMA assert level. * If the severity of the error will be * above the level, the FW will assert. diff --git a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c index 321c43408153..0ce37f2460a4 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c +++ b/drivers/net/ethernet/qlogic/qed/qed_init_fw_funcs.c @@ -210,6 +210,82 @@ static u16 task_region_offsets[1][NUM_OF_CONNECTION_TYPES] = { (XSEM_REG_FAST_MEMORY + SEM_FAST_REG_INT_RAM + \ XSTORM_PQ_INFO_OFFSET(pq_id)) +static const char * const s_protocol_types[] = { + "PROTOCOLID_ISCSI", "PROTOCOLID_FCOE", "PROTOCOLID_ROCE", + "PROTOCOLID_CORE", "PROTOCOLID_ETH", "PROTOCOLID_IWARP", + "PROTOCOLID_TOE", "PROTOCOLID_PREROCE", "PROTOCOLID_COMMON", + "PROTOCOLID_TCP", "PROTOCOLID_RDMA", "PROTOCOLID_SCSI", +}; + +static const char *s_ramrod_cmd_ids[][28] = { + { + "ISCSI_RAMROD_CMD_ID_UNUSED", "ISCSI_RAMROD_CMD_ID_INIT_FUNC", + "ISCSI_RAMROD_CMD_ID_DESTROY_FUNC", + "ISCSI_RAMROD_CMD_ID_OFFLOAD_CONN", + "ISCSI_RAMROD_CMD_ID_UPDATE_CONN", + "ISCSI_RAMROD_CMD_ID_TERMINATION_CONN", + "ISCSI_RAMROD_CMD_ID_CLEAR_SQ", "ISCSI_RAMROD_CMD_ID_MAC_UPDATE", + "ISCSI_RAMROD_CMD_ID_CONN_STATS", }, + { "FCOE_RAMROD_CMD_ID_INIT_FUNC", "FCOE_RAMROD_CMD_ID_DESTROY_FUNC", + "FCOE_RAMROD_CMD_ID_STAT_FUNC", + "FCOE_RAMROD_CMD_ID_OFFLOAD_CONN", + "FCOE_RAMROD_CMD_ID_TERMINATE_CONN", }, + { "RDMA_RAMROD_UNUSED", "RDMA_RAMROD_FUNC_INIT", + "RDMA_RAMROD_FUNC_CLOSE", "RDMA_RAMROD_REGISTER_MR", + "RDMA_RAMROD_DEREGISTER_MR", "RDMA_RAMROD_CREATE_CQ", + "RDMA_RAMROD_RESIZE_CQ", "RDMA_RAMROD_DESTROY_CQ", + "RDMA_RAMROD_CREATE_SRQ", "RDMA_RAMROD_MODIFY_SRQ", + "RDMA_RAMROD_DESTROY_SRQ", "RDMA_RAMROD_START_NS_TRACKING", + "RDMA_RAMROD_STOP_NS_TRACKING", "ROCE_RAMROD_CREATE_QP", + "ROCE_RAMROD_MODIFY_QP", "ROCE_RAMROD_QUERY_QP", + "ROCE_RAMROD_DESTROY_QP", "ROCE_RAMROD_CREATE_UD_QP", + "ROCE_RAMROD_DESTROY_UD_QP", "ROCE_RAMROD_FUNC_UPDATE", + "ROCE_RAMROD_SUSPEND_QP", "ROCE_RAMROD_QUERY_SUSPENDED_QP", + "ROCE_RAMROD_CREATE_SUSPENDED_QP", "ROCE_RAMROD_RESUME_QP", + "ROCE_RAMROD_SUSPEND_UD_QP", "ROCE_RAMROD_RESUME_UD_QP", + "ROCE_RAMROD_CREATE_SUSPENDED_UD_QP", "ROCE_RAMROD_FLUSH_DPT_QP", }, + { "CORE_RAMROD_UNUSED", "CORE_RAMROD_RX_QUEUE_START", + "CORE_RAMROD_TX_QUEUE_START", "CORE_RAMROD_RX_QUEUE_STOP", + "CORE_RAMROD_TX_QUEUE_STOP", + "CORE_RAMROD_RX_QUEUE_FLUSH", + "CORE_RAMROD_TX_QUEUE_UPDATE", "CORE_RAMROD_QUEUE_STATS_QUERY", }, + { "ETH_RAMROD_UNUSED", "ETH_RAMROD_VPORT_START", + "ETH_RAMROD_VPORT_UPDATE", "ETH_RAMROD_VPORT_STOP", + "ETH_RAMROD_RX_QUEUE_START", "ETH_RAMROD_RX_QUEUE_STOP", + "ETH_RAMROD_TX_QUEUE_START", "ETH_RAMROD_TX_QUEUE_STOP", + "ETH_RAMROD_FILTERS_UPDATE", "ETH_RAMROD_RX_QUEUE_UPDATE", + "ETH_RAMROD_RX_CREATE_OPENFLOW_ACTION", + "ETH_RAMROD_RX_ADD_OPENFLOW_FILTER", + "ETH_RAMROD_RX_DELETE_OPENFLOW_FILTER", + "ETH_RAMROD_RX_ADD_UDP_FILTER", + "ETH_RAMROD_RX_DELETE_UDP_FILTER", + "ETH_RAMROD_RX_CREATE_GFT_ACTION", + "ETH_RAMROD_RX_UPDATE_GFT_FILTER", "ETH_RAMROD_TX_QUEUE_UPDATE", + "ETH_RAMROD_RGFS_FILTER_ADD", "ETH_RAMROD_RGFS_FILTER_DEL", + "ETH_RAMROD_TGFS_FILTER_ADD", "ETH_RAMROD_TGFS_FILTER_DEL", + "ETH_RAMROD_GFS_COUNTERS_REPORT_REQUEST", }, + { "RDMA_RAMROD_UNUSED", "RDMA_RAMROD_FUNC_INIT", + "RDMA_RAMROD_FUNC_CLOSE", "RDMA_RAMROD_REGISTER_MR", + "RDMA_RAMROD_DEREGISTER_MR", "RDMA_RAMROD_CREATE_CQ", + "RDMA_RAMROD_RESIZE_CQ", "RDMA_RAMROD_DESTROY_CQ", + "RDMA_RAMROD_CREATE_SRQ", "RDMA_RAMROD_MODIFY_SRQ", + "RDMA_RAMROD_DESTROY_SRQ", "RDMA_RAMROD_START_NS_TRACKING", + "RDMA_RAMROD_STOP_NS_TRACKING", + "IWARP_RAMROD_CMD_ID_TCP_OFFLOAD", + "IWARP_RAMROD_CMD_ID_MPA_OFFLOAD", + "IWARP_RAMROD_CMD_ID_MPA_OFFLOAD_SEND_RTR", + "IWARP_RAMROD_CMD_ID_CREATE_QP", "IWARP_RAMROD_CMD_ID_QUERY_QP", + "IWARP_RAMROD_CMD_ID_MODIFY_QP", + "IWARP_RAMROD_CMD_ID_DESTROY_QP", + "IWARP_RAMROD_CMD_ID_ABORT_TCP_OFFLOAD", }, + { NULL }, /*TOE*/ + { NULL }, /*PREROCE*/ + { "COMMON_RAMROD_UNUSED", "COMMON_RAMROD_PF_START", + "COMMON_RAMROD_PF_STOP", "COMMON_RAMROD_VF_START", + "COMMON_RAMROD_VF_STOP", "COMMON_RAMROD_PF_UPDATE", + "COMMON_RAMROD_RL_UPDATE", "COMMON_RAMROD_EMPTY", } +}; + /******************** INTERNAL IMPLEMENTATION *********************/ /* Returns the external VOQ number */ @@ -1647,6 +1723,32 @@ void qed_enable_context_validation(struct qed_hwfn *p_hwfn, qed_wr(p_hwfn, p_ptt, CDU_REG_TCFC_CTX_VALID0, ctx_validation); } +const char *qed_get_protocol_type_str(u32 protocol_type) +{ + if (protocol_type >= ARRAY_SIZE(s_protocol_types)) + return "Invalid protocol type"; + + return s_protocol_types[protocol_type]; +} + +const char *qed_get_ramrod_cmd_id_str(u32 protocol_type, u32 ramrod_cmd_id) +{ + const char *ramrod_cmd_id_str; + + if (protocol_type >= ARRAY_SIZE(s_ramrod_cmd_ids)) + return "Invalid protocol type"; + + if (ramrod_cmd_id >= ARRAY_SIZE(s_ramrod_cmd_ids[0])) + return "Invalid Ramrod command ID"; + + ramrod_cmd_id_str = s_ramrod_cmd_ids[protocol_type][ramrod_cmd_id]; + + if (!ramrod_cmd_id_str) + return "Invalid Ramrod command ID"; + + return ramrod_cmd_id_str; +} + static u32 qed_get_rdma_assert_ram_addr(struct qed_hwfn *p_hwfn, u8 storm_id) { switch (storm_id) { diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.c b/drivers/net/ethernet/qlogic/qed/qed_int.c index 6958adeca86d..82e74f62b677 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.c +++ b/drivers/net/ethernet/qlogic/qed/qed_int.c @@ -2399,3 +2399,25 @@ int qed_int_set_timer_res(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, return rc; } + +int qed_int_get_sb_dbg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, + struct qed_sb_info *p_sb, struct qed_sb_info_dbg *p_info) +{ + u16 sbid = p_sb->igu_sb_id; + u32 i; + + if (IS_VF(p_hwfn->cdev)) + return -EINVAL; + + if (sbid >= NUM_OF_SBS(p_hwfn->cdev)) + return -EINVAL; + + p_info->igu_prod = qed_rd(p_hwfn, p_ptt, IGU_REG_PRODUCER_MEMORY + sbid * 4); + p_info->igu_cons = qed_rd(p_hwfn, p_ptt, IGU_REG_CONSUMER_MEM + sbid * 4); + + for (i = 0; i < PIS_PER_SB; i++) + p_info->pi[i] = (u16)qed_rd(p_hwfn, p_ptt, + CAU_REG_PI_MEMORY + sbid * 4 * PIS_PER_SB + i * 4); + + return 0; +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_int.h b/drivers/net/ethernet/qlogic/qed/qed_int.h index 84c17e97f569..7e5127f61744 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_int.h +++ b/drivers/net/ethernet/qlogic/qed/qed_int.h @@ -186,6 +186,19 @@ void qed_int_disable_post_isr_release(struct qed_dev *cdev); void qed_int_attn_clr_enable(struct qed_dev *cdev, bool clr_enable); /** + * qed_int_get_sb_dbg: Read debug information regarding a given SB + * + * @p_hwfn: hw function pointer + * @p_ptt: ptt resource + * @p_sb: pointer to status block for which we want to get info + * @p_info: pointer to struct to fill with information regarding SB + * + * Return: 0 with status block info filled on success, otherwise return error + */ +int qed_int_get_sb_dbg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, + struct qed_sb_info *p_sb, struct qed_sb_info_dbg *p_info); + +/** * qed_db_rec_handler(): Doorbell Recovery handler. * Run doorbell recovery in case of PF overflow (and flush DORQ if * needed). diff --git a/drivers/net/ethernet/qlogic/qed/qed_main.c b/drivers/net/ethernet/qlogic/qed/qed_main.c index 7673b3e07736..46d4207f22a3 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_main.c +++ b/drivers/net/ethernet/qlogic/qed/qed_main.c @@ -447,7 +447,7 @@ int qed_fill_dev_info(struct qed_dev *cdev, dev_info->wol_support = true; dev_info->smart_an = qed_mcp_is_smart_an_supported(p_hwfn); - + dev_info->esl = qed_mcp_is_esl_supported(p_hwfn); dev_info->abs_pf_id = QED_LEADING_HWFN(cdev)->abs_pf_id; } else { qed_vf_get_fw_version(&cdev->hwfns[0], &dev_info->fw_major, @@ -2936,6 +2936,30 @@ out: return status; } +static int +qed_get_sb_info(struct qed_dev *cdev, struct qed_sb_info *sb, + u16 qid, struct qed_sb_info_dbg *sb_dbg) +{ + struct qed_hwfn *hwfn = &cdev->hwfns[qid % cdev->num_hwfns]; + struct qed_ptt *ptt; + int rc; + + if (IS_VF(cdev)) + return -EINVAL; + + ptt = qed_ptt_acquire(hwfn); + if (!ptt) { + DP_NOTICE(hwfn, "Can't acquire PTT\n"); + return -EAGAIN; + } + + memset(sb_dbg, 0, sizeof(*sb_dbg)); + rc = qed_int_get_sb_dbg(hwfn, ptt, sb, sb_dbg); + + qed_ptt_release(hwfn, ptt); + return rc; +} + static int qed_read_module_eeprom(struct qed_dev *cdev, char *buf, u8 dev_addr, u32 offset, u32 len) { @@ -2978,11 +3002,54 @@ static int qed_set_grc_config(struct qed_dev *cdev, u32 cfg_id, u32 val) return rc; } +static __printf(2, 3) void qed_mfw_report(struct qed_dev *cdev, char *fmt, ...) +{ + char buf[QED_MFW_REPORT_STR_SIZE]; + struct qed_hwfn *p_hwfn; + struct qed_ptt *p_ptt; + va_list vl; + + va_start(vl, fmt); + vsnprintf(buf, QED_MFW_REPORT_STR_SIZE, fmt, vl); + va_end(vl); + + if (IS_PF(cdev)) { + p_hwfn = QED_LEADING_HWFN(cdev); + p_ptt = qed_ptt_acquire(p_hwfn); + if (p_ptt) { + qed_mcp_send_raw_debug_data(p_hwfn, p_ptt, buf, strlen(buf)); + qed_ptt_release(p_hwfn, p_ptt); + } + } +} + static u8 qed_get_affin_hwfn_idx(struct qed_dev *cdev) { return QED_AFFIN_HWFN_IDX(cdev); } +static int qed_get_esl_status(struct qed_dev *cdev, bool *esl_active) +{ + struct qed_hwfn *hwfn = QED_LEADING_HWFN(cdev); + struct qed_ptt *ptt; + int rc = 0; + + *esl_active = false; + + if (IS_VF(cdev)) + return 0; + + ptt = qed_ptt_acquire(hwfn); + if (!ptt) + return -EAGAIN; + + rc = qed_mcp_get_esl_status(hwfn, ptt, esl_active); + + qed_ptt_release(hwfn, ptt); + + return rc; +} + static struct qed_selftest_ops qed_selftest_ops_pass = { .selftest_memory = &qed_selftest_memory, .selftest_interrupt = &qed_selftest_interrupt, @@ -3038,6 +3105,9 @@ const struct qed_common_ops qed_common_ops_pass = { .read_nvm_cfg = &qed_nvm_flash_cfg_read, .read_nvm_cfg_len = &qed_nvm_flash_cfg_len, .set_grc_config = &qed_set_grc_config, + .mfw_report = &qed_mfw_report, + .get_sb_info = &qed_get_sb_info, + .get_esl_status = &qed_get_esl_status, }; void qed_get_protocol_stats(struct qed_dev *cdev, diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.c b/drivers/net/ethernet/qlogic/qed/qed_mcp.c index 64678a256f3b..da1eadabcb41 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.c +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.c @@ -4158,3 +4158,25 @@ qed_mcp_send_raw_debug_data(struct qed_hwfn *p_hwfn, return qed_mcp_send_debug_data(p_hwfn, p_ptt, QED_MCP_DBG_DATA_TYPE_RAW, p_buf, size); } + +bool qed_mcp_is_esl_supported(struct qed_hwfn *p_hwfn) +{ + return !!(p_hwfn->mcp_info->capabilities & + FW_MB_PARAM_FEATURE_SUPPORT_ENHANCED_SYS_LCK); +} + +int qed_mcp_get_esl_status(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool *active) +{ + u32 resp = 0, param = 0; + int rc; + + rc = qed_mcp_cmd(p_hwfn, p_ptt, DRV_MSG_CODE_GET_MANAGEMENT_STATUS, 0, &resp, ¶m); + if (rc) { + DP_NOTICE(p_hwfn, "Failed to send ESL command, rc = %d\n", rc); + return rc; + } + + *active = !!(param & FW_MB_PARAM_MANAGEMENT_STATUS_LOCKDOWN_ENABLED); + + return 0; +} diff --git a/drivers/net/ethernet/qlogic/qed/qed_mcp.h b/drivers/net/ethernet/qlogic/qed/qed_mcp.h index 564723800d15..369e1892450a 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mcp.h +++ b/drivers/net/ethernet/qlogic/qed/qed_mcp.h @@ -15,6 +15,8 @@ #include "qed_hsi.h" #include "qed_dev_api.h" +#define QED_MFW_REPORT_STR_SIZE 256 + struct qed_mcp_link_speed_params { bool autoneg; @@ -1337,4 +1339,24 @@ int qed_mcp_nvm_get_cfg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, int qed_mcp_nvm_set_cfg(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, u16 option_id, u8 entity_id, u16 flags, u8 *p_buf, u32 len); + +/** + * qed_mcp_is_esl_supported(): Return whether management firmware support ESL or not. + * + * @p_hwfn: hw function pointer + * + * Return: true if esl is supported, otherwise return false + */ +bool qed_mcp_is_esl_supported(struct qed_hwfn *p_hwfn); + +/** + * qed_mcp_get_esl_status(): Get enhanced system lockdown status + * + * @p_hwfn: hw function pointer + * @p_ptt: ptt resource pointer + * @active: ESL active status data pointer + * + * Return: 0 with esl status info on success, otherwise return error + */ +int qed_mcp_get_esl_status(struct qed_hwfn *p_hwfn, struct qed_ptt *p_ptt, bool *active); #endif diff --git a/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h b/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h index 8a0e3c5d4bda..b70ee8200e15 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h +++ b/drivers/net/ethernet/qlogic/qed/qed_mfw_hsi.h @@ -1191,6 +1191,7 @@ enum drv_msg_code_enum { DRV_MSG_CODE_CFG_VF_MSIX = DRV_MSG_CODE(0xc001), DRV_MSG_CODE_CFG_PF_VFS_MSIX = DRV_MSG_CODE(0xc002), DRV_MSG_CODE_DEBUG_DATA_SEND = DRV_MSG_CODE(0xc004), + DRV_MSG_CODE_GET_MANAGEMENT_STATUS = DRV_MSG_CODE(0xc007), }; #define DRV_MSG_CODE_VMAC_TYPE_SHIFT 4 diff --git a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h index 6f1a52e6beb2..b5e35f433a20 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h +++ b/drivers/net/ethernet/qlogic/qed/qed_reg_addr.h @@ -550,6 +550,8 @@ 0x1 << 1) #define IGU_REG_BLOCK_CONFIGURATION_PXP_TPH_INTERFACE_EN ( \ 0x1 << 0) +#define IGU_REG_PRODUCER_MEMORY 0x182000UL +#define IGU_REG_CONSUMER_MEM 0x183000UL #define IGU_REG_MAPPING_MEMORY \ 0x184000UL #define IGU_REG_STATISTIC_NUM_VF_MSG_SENT \ diff --git a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c index 648176dfb871..3b54da963554 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c +++ b/drivers/net/ethernet/qlogic/qed/qed_sp_commands.c @@ -85,10 +85,12 @@ int qed_sp_init_request(struct qed_hwfn *p_hwfn, goto err; } - DP_VERBOSE(p_hwfn, QED_MSG_SPQ, - "Initialized: CID %08x cmd %02x protocol %02x data_addr %lu comp_mode [%s]\n", - opaque_cid, cmd, protocol, - (unsigned long)&p_ent->ramrod, + DP_VERBOSE(p_hwfn, + QED_MSG_SPQ, + "Initialized: CID %08x %s:[%02x] %s:%02x data_addr %llx comp_mode [%s]\n", + opaque_cid, qed_get_ramrod_cmd_id_str(protocol, cmd), + cmd, qed_get_protocol_type_str(protocol), protocol, + (unsigned long long)(uintptr_t)&p_ent->ramrod, D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK, QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK", "MODE_CB")); diff --git a/drivers/net/ethernet/qlogic/qed/qed_spq.c b/drivers/net/ethernet/qlogic/qed/qed_spq.c index e0473729b161..d01b9245f811 100644 --- a/drivers/net/ethernet/qlogic/qed/qed_spq.c +++ b/drivers/net/ethernet/qlogic/qed/qed_spq.c @@ -139,10 +139,13 @@ err: if (!p_ptt) return -EBUSY; qed_hw_err_notify(p_hwfn, p_ptt, QED_HW_ERR_RAMROD_FAIL, - "Ramrod is stuck [CID %08x cmd %02x protocol %02x echo %04x]\n", + "Ramrod is stuck [CID %08x %s:%02x %s:%02x echo %04x]\n", le32_to_cpu(p_ent->elem.hdr.cid), + qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, + p_ent->elem.hdr.cmd_id), p_ent->elem.hdr.cmd_id, - p_ent->elem.hdr.protocol_id, + qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), + p_ent->elem.hdr.protocol_id, le16_to_cpu(p_ent->elem.hdr.echo)); qed_ptt_release(p_hwfn, p_ptt); @@ -170,13 +173,16 @@ static int qed_spq_fill_entry(struct qed_hwfn *p_hwfn, return -EINVAL; } - DP_VERBOSE(p_hwfn, QED_MSG_SPQ, - "Ramrod header: [CID 0x%08x CMD 0x%02x protocol 0x%02x] Data pointer: [%08x:%08x] Completion Mode: %s\n", + DP_VERBOSE(p_hwfn, + QED_MSG_SPQ, + "Ramrod hdr: [CID 0x%08x %s:0x%02x %s:0x%02x] Data ptr: [%08x:%08x] Cmpltion Mode: %s\n", p_ent->elem.hdr.cid, + qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, + p_ent->elem.hdr.cmd_id), p_ent->elem.hdr.cmd_id, - p_ent->elem.hdr.protocol_id, - p_ent->elem.data_ptr.hi, - p_ent->elem.data_ptr.lo, + qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), + p_ent->elem.hdr.protocol_id, + p_ent->elem.data_ptr.hi, p_ent->elem.data_ptr.lo, D_TRINE(p_ent->comp_mode, QED_SPQ_MODE_EBLOCK, QED_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK", "MODE_CB")); @@ -271,17 +277,27 @@ qed_async_event_completion(struct qed_hwfn *p_hwfn, { qed_spq_async_comp_cb cb; - if (!p_hwfn->p_spq || (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE)) + if (!p_hwfn->p_spq) return -EINVAL; + if (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE) { + DP_ERR(p_hwfn, "Wrong protocol: %s:%d\n", + qed_get_protocol_type_str(p_eqe->protocol_id), + p_eqe->protocol_id); + + return -EINVAL; + } + cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id]; if (cb) { return cb(p_hwfn, p_eqe->opcode, p_eqe->echo, &p_eqe->data, p_eqe->fw_return_code); } else { DP_NOTICE(p_hwfn, - "Unknown Async completion for protocol: %d\n", + "Unknown Async completion for %s:%d\n", + qed_get_protocol_type_str(p_eqe->protocol_id), p_eqe->protocol_id); + return -EINVAL; } } @@ -830,8 +846,12 @@ int qed_spq_post(struct qed_hwfn *p_hwfn, if (p_hwfn->cdev->recov_in_prog) { DP_VERBOSE(p_hwfn, QED_MSG_SPQ, - "Recovery is in progress. Skip spq post [cmd %02x protocol %02x]\n", - p_ent->elem.hdr.cmd_id, p_ent->elem.hdr.protocol_id); + "Recovery is in progress. Skip spq post [%s:%02x %s:%02x]\n", + qed_get_ramrod_cmd_id_str(p_ent->elem.hdr.protocol_id, + p_ent->elem.hdr.cmd_id), + p_ent->elem.hdr.cmd_id, + qed_get_protocol_type_str(p_ent->elem.hdr.protocol_id), + p_ent->elem.hdr.protocol_id); /* Let the flow complete w/o any error handling */ qed_spq_recov_set_ret_code(p_ent, fw_return_code); diff --git a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c index 8284c4c1528f..97a7ab0826ed 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_ethtool.c +++ b/drivers/net/ethernet/qlogic/qede/qede_ethtool.c @@ -168,6 +168,8 @@ enum { QEDE_PRI_FLAG_CMT, QEDE_PRI_FLAG_SMART_AN_SUPPORT, /* MFW supports SmartAN */ QEDE_PRI_FLAG_RECOVER_ON_ERROR, + QEDE_PRI_FLAG_ESL_SUPPORT, /* MFW supports Enhanced System Lockdown */ + QEDE_PRI_FLAG_ESL_ACTIVE, /* Enhanced System Lockdown Active status */ QEDE_PRI_FLAG_LEN, }; @@ -175,6 +177,8 @@ static const char qede_private_arr[QEDE_PRI_FLAG_LEN][ETH_GSTRING_LEN] = { "Coupled-Function", "SmartAN capable", "Recover on error", + "ESL capable", + "ESL active", }; enum qede_ethtool_tests { @@ -478,6 +482,7 @@ static int qede_get_sset_count(struct net_device *dev, int stringset) static u32 qede_get_priv_flags(struct net_device *dev) { struct qede_dev *edev = netdev_priv(dev); + bool esl_active; u32 flags = 0; if (edev->dev_info.common.num_hwfns > 1) @@ -489,6 +494,14 @@ static u32 qede_get_priv_flags(struct net_device *dev) if (edev->err_flags & BIT(QEDE_ERR_IS_RECOVERABLE)) flags |= BIT(QEDE_PRI_FLAG_RECOVER_ON_ERROR); + if (edev->dev_info.common.esl) + flags |= BIT(QEDE_PRI_FLAG_ESL_SUPPORT); + + edev->ops->common->get_esl_status(edev->cdev, &esl_active); + + if (esl_active) + flags |= BIT(QEDE_PRI_FLAG_ESL_ACTIVE); + return flags; } @@ -888,7 +901,9 @@ int qede_set_coalesce(struct net_device *dev, struct ethtool_coalesce *coal, } static void qede_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct qede_dev *edev = netdev_priv(dev); @@ -899,7 +914,9 @@ static void qede_get_ringparam(struct net_device *dev, } static int qede_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct qede_dev *edev = netdev_priv(dev); diff --git a/drivers/net/ethernet/qlogic/qede/qede_fp.c b/drivers/net/ethernet/qlogic/qede/qede_fp.c index 999abcfe3310..5ea9cb4311a1 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_fp.c +++ b/drivers/net/ethernet/qlogic/qede/qede_fp.c @@ -10,6 +10,7 @@ #include <linux/bpf_trace.h> #include <net/udp_tunnel.h> #include <linux/ip.h> +#include <net/gro.h> #include <net/ipv6.h> #include <net/tcp.h> #include <linux/if_ether.h> diff --git a/drivers/net/ethernet/qlogic/qede/qede_main.c b/drivers/net/ethernet/qlogic/qede/qede_main.c index 06c6a5813606..b4e5a15e308b 100644 --- a/drivers/net/ethernet/qlogic/qede/qede_main.c +++ b/drivers/net/ethernet/qlogic/qede/qede_main.c @@ -509,34 +509,95 @@ static int qede_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) return 0; } -static void qede_tx_log_print(struct qede_dev *edev, struct qede_tx_queue *txq) +static void qede_fp_sb_dump(struct qede_dev *edev, struct qede_fastpath *fp) { + char *p_sb = (char *)fp->sb_info->sb_virt; + u32 sb_size, i; + + sb_size = sizeof(struct status_block); + + for (i = 0; i < sb_size; i += 8) + DP_NOTICE(edev, + "%02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX %02hhX\n", + p_sb[i], p_sb[i + 1], p_sb[i + 2], p_sb[i + 3], + p_sb[i + 4], p_sb[i + 5], p_sb[i + 6], p_sb[i + 7]); +} + +static void +qede_txq_fp_log_metadata(struct qede_dev *edev, + struct qede_fastpath *fp, struct qede_tx_queue *txq) +{ + struct qed_chain *p_chain = &txq->tx_pbl; + + /* Dump txq/fp/sb ids etc. other metadata */ DP_NOTICE(edev, - "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n", - txq->index, le16_to_cpu(*txq->hw_cons_ptr), - qed_chain_get_cons_idx(&txq->tx_pbl), - qed_chain_get_prod_idx(&txq->tx_pbl), - jiffies); + "fpid 0x%x sbid 0x%x txqid [0x%x] ndev_qid [0x%x] cos [0x%x] p_chain %p cap %d size %d jiffies %lu HZ 0x%x\n", + fp->id, fp->sb_info->igu_sb_id, txq->index, txq->ndev_txq_id, txq->cos, + p_chain, p_chain->capacity, p_chain->size, jiffies, HZ); + + /* Dump all the relevant prod/cons indexes */ + DP_NOTICE(edev, + "hw cons %04x sw_tx_prod=0x%x, sw_tx_cons=0x%x, bd_prod 0x%x bd_cons 0x%x\n", + le16_to_cpu(*txq->hw_cons_ptr), txq->sw_tx_prod, txq->sw_tx_cons, + qed_chain_get_prod_idx(p_chain), qed_chain_get_cons_idx(p_chain)); +} + +static void +qede_tx_log_print(struct qede_dev *edev, struct qede_fastpath *fp, struct qede_tx_queue *txq) +{ + struct qed_sb_info_dbg sb_dbg; + int rc; + + /* sb info */ + qede_fp_sb_dump(edev, fp); + + memset(&sb_dbg, 0, sizeof(sb_dbg)); + rc = edev->ops->common->get_sb_info(edev->cdev, fp->sb_info, (u16)fp->id, &sb_dbg); + + DP_NOTICE(edev, "IGU: prod %08x cons %08x CAU Tx %04x\n", + sb_dbg.igu_prod, sb_dbg.igu_cons, sb_dbg.pi[TX_PI(txq->cos)]); + + /* report to mfw */ + edev->ops->common->mfw_report(edev->cdev, + "Txq[%d]: FW cons [host] %04x, SW cons %04x, SW prod %04x [Jiffies %lu]\n", + txq->index, le16_to_cpu(*txq->hw_cons_ptr), + qed_chain_get_cons_idx(&txq->tx_pbl), + qed_chain_get_prod_idx(&txq->tx_pbl), jiffies); + if (!rc) + edev->ops->common->mfw_report(edev->cdev, + "Txq[%d]: SB[0x%04x] - IGU: prod %08x cons %08x CAU Tx %04x\n", + txq->index, fp->sb_info->igu_sb_id, + sb_dbg.igu_prod, sb_dbg.igu_cons, + sb_dbg.pi[TX_PI(txq->cos)]); } static void qede_tx_timeout(struct net_device *dev, unsigned int txqueue) { struct qede_dev *edev = netdev_priv(dev); - struct qede_tx_queue *txq; - int cos; + int i; netif_carrier_off(dev); DP_NOTICE(edev, "TX timeout on queue %u!\n", txqueue); - if (!(edev->fp_array[txqueue].type & QEDE_FASTPATH_TX)) - return; + for_each_queue(i) { + struct qede_tx_queue *txq; + struct qede_fastpath *fp; + int cos; - for_each_cos_in_txq(edev, cos) { - txq = &edev->fp_array[txqueue].txq[cos]; + fp = &edev->fp_array[i]; + if (!(fp->type & QEDE_FASTPATH_TX)) + continue; - if (qed_chain_get_cons_idx(&txq->tx_pbl) != - qed_chain_get_prod_idx(&txq->tx_pbl)) - qede_tx_log_print(edev, txq); + for_each_cos_in_txq(edev, cos) { + txq = &fp->txq[cos]; + + /* Dump basic metadata for all queues */ + qede_txq_fp_log_metadata(edev, fp, txq); + + if (qed_chain_get_cons_idx(&txq->tx_pbl) != + qed_chain_get_prod_idx(&txq->tx_pbl)) + qede_tx_log_print(edev, fp, txq); + } } if (IS_VF(edev)) diff --git a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c index fc364b4ab6eb..e10fe071a40f 100644 --- a/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c +++ b/drivers/net/ethernet/qlogic/qlcnic/qlcnic_ethtool.c @@ -632,7 +632,9 @@ qlcnic_get_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, static void qlcnic_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qlcnic_adapter *adapter = netdev_priv(dev); @@ -663,7 +665,9 @@ qlcnic_validate_ringparam(u32 val, u32 min, u32 max, char *r_name) static int qlcnic_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qlcnic_adapter *adapter = netdev_priv(dev); u16 num_rxd, num_jumbo_rxd, num_txd; diff --git a/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c b/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c index f72e13b83869..f502db9cdea9 100644 --- a/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c +++ b/drivers/net/ethernet/qualcomm/emac/emac-ethtool.c @@ -133,7 +133,9 @@ static int emac_nway_reset(struct net_device *netdev) } static void emac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct emac_adapter *adpt = netdev_priv(netdev); @@ -144,7 +146,9 @@ static void emac_get_ringparam(struct net_device *netdev, } static int emac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct emac_adapter *adpt = netdev_priv(netdev); diff --git a/drivers/net/ethernet/qualcomm/qca_debug.c b/drivers/net/ethernet/qualcomm/qca_debug.c index d59fff2fbcc6..792ce9a323cd 100644 --- a/drivers/net/ethernet/qualcomm/qca_debug.c +++ b/drivers/net/ethernet/qualcomm/qca_debug.c @@ -246,7 +246,9 @@ qcaspi_get_regs(struct net_device *dev, struct ethtool_regs *regs, void *p) } static void -qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) +qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct qcaspi *qca = netdev_priv(dev); @@ -257,7 +259,9 @@ qcaspi_get_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) } static int -qcaspi_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ring) +qcaspi_set_ringparam(struct net_device *dev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { const struct net_device_ops *ops = dev->netdev_ops; struct qcaspi *qca = netdev_priv(dev); diff --git a/drivers/net/ethernet/realtek/8139cp.c b/drivers/net/ethernet/realtek/8139cp.c index 4f39f843bb3a..ad7b9e9d7f95 100644 --- a/drivers/net/ethernet/realtek/8139cp.c +++ b/drivers/net/ethernet/realtek/8139cp.c @@ -1388,7 +1388,9 @@ static void cp_get_drvinfo (struct net_device *dev, struct ethtool_drvinfo *info } static void cp_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { ring->rx_max_pending = CP_RX_RING_SIZE; ring->tx_max_pending = CP_TX_RING_SIZE; diff --git a/drivers/net/ethernet/realtek/r8169_main.c b/drivers/net/ethernet/realtek/r8169_main.c index 86c44bc5f73f..3d6843332c77 100644 --- a/drivers/net/ethernet/realtek/r8169_main.c +++ b/drivers/net/ethernet/realtek/r8169_main.c @@ -1873,7 +1873,9 @@ static int rtl8169_set_eee(struct net_device *dev, struct ethtool_eee *data) } static void rtl8169_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *data) + struct ethtool_ringparam *data, + struct kernel_ethtool_ringparam *kernel_data, + struct netlink_ext_ack *extack) { data->rx_max_pending = NUM_RX_DESC; data->rx_pending = NUM_RX_DESC; @@ -1969,8 +1971,11 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) { 0x7cf, 0x641, RTL_GIGA_MAC_VER_63 }, /* 8125A family. */ - { 0x7cf, 0x608, RTL_GIGA_MAC_VER_60 }, - { 0x7c8, 0x608, RTL_GIGA_MAC_VER_61 }, + { 0x7cf, 0x609, RTL_GIGA_MAC_VER_61 }, + /* It seems only XID 609 made it to the mass market. + * { 0x7cf, 0x608, RTL_GIGA_MAC_VER_60 }, + * { 0x7c8, 0x608, RTL_GIGA_MAC_VER_61 }, + */ /* RTL8117 */ { 0x7cf, 0x54b, RTL_GIGA_MAC_VER_53 }, @@ -1978,17 +1983,26 @@ static enum mac_version rtl8169_get_mac_version(u16 xid, bool gmii) /* 8168EP family. */ { 0x7cf, 0x502, RTL_GIGA_MAC_VER_51 }, - { 0x7cf, 0x501, RTL_GIGA_MAC_VER_50 }, - { 0x7cf, 0x500, RTL_GIGA_MAC_VER_49 }, + /* It seems this chip version never made it to + * the wild. Let's disable detection. + * { 0x7cf, 0x501, RTL_GIGA_MAC_VER_50 }, + * { 0x7cf, 0x500, RTL_GIGA_MAC_VER_49 }, + */ /* 8168H family. */ { 0x7cf, 0x541, RTL_GIGA_MAC_VER_46 }, - { 0x7cf, 0x540, RTL_GIGA_MAC_VER_45 }, + /* It seems this chip version never made it to + * the wild. Let's disable detection. + * { 0x7cf, 0x540, RTL_GIGA_MAC_VER_45 }, + */ /* 8168G family. */ { 0x7cf, 0x5c8, RTL_GIGA_MAC_VER_44 }, { 0x7cf, 0x509, RTL_GIGA_MAC_VER_42 }, - { 0x7cf, 0x4c1, RTL_GIGA_MAC_VER_41 }, + /* It seems this chip version never made it to + * the wild. Let's disable detection. + * { 0x7cf, 0x4c1, RTL_GIGA_MAC_VER_41 }, + */ { 0x7cf, 0x4c0, RTL_GIGA_MAC_VER_40 }, /* 8168F family. */ @@ -5272,12 +5286,6 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) if (rc) return rc; - /* Disable ASPM L1 as that cause random device stop working - * problems as well as full system hangs for some PCIe devices users. - */ - rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1); - tp->aspm_manageable = !rc; - /* enable device (incl. PCI PM wakeup and hotplug setup) */ rc = pcim_enable_device(pdev); if (rc < 0) { @@ -5320,6 +5328,17 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) tp->mac_version = chipset; + /* Disable ASPM L1 as that cause random device stop working + * problems as well as full system hangs for some PCIe devices users. + * Chips from RTL8168h partially have issues with L1.2, but seem + * to work fine with L1 and L1.1. + */ + if (tp->mac_version >= RTL_GIGA_MAC_VER_45) + rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1_2); + else + rc = pci_disable_link_state(pdev, PCIE_LINK_STATE_L1); + tp->aspm_manageable = !rc; + tp->dash_type = rtl_check_dash(tp); tp->cp_cmd = RTL_R16(tp, CPlusCmd) & CPCMD_MASK; @@ -5375,12 +5394,12 @@ static int rtl_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (rtl_chip_supports_csum_v2(tp)) { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6; - dev->gso_max_size = RTL_GSO_MAX_SIZE_V2; - dev->gso_max_segs = RTL_GSO_MAX_SEGS_V2; + netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V2); + netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V2); } else { dev->hw_features |= NETIF_F_SG | NETIF_F_TSO; - dev->gso_max_size = RTL_GSO_MAX_SIZE_V1; - dev->gso_max_segs = RTL_GSO_MAX_SEGS_V1; + netif_set_gso_max_size(dev, RTL_GSO_MAX_SIZE_V1); + netif_set_gso_max_segs(dev, RTL_GSO_MAX_SEGS_V1); } dev->hw_features |= NETIF_F_RXALL; diff --git a/drivers/net/ethernet/renesas/ravb_main.c b/drivers/net/ethernet/renesas/ravb_main.c index b4c597f4040c..ce09bd45527e 100644 --- a/drivers/net/ethernet/renesas/ravb_main.c +++ b/drivers/net/ethernet/renesas/ravb_main.c @@ -30,8 +30,7 @@ #include <linux/spinlock.h> #include <linux/sys_soc.h> #include <linux/reset.h> - -#include <asm/div64.h> +#include <linux/math64.h> #include "ravb.h" @@ -1605,7 +1604,9 @@ static void ravb_get_strings(struct net_device *ndev, u32 stringset, u8 *data) } static void ravb_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ravb_private *priv = netdev_priv(ndev); @@ -1616,7 +1617,9 @@ static void ravb_get_ringparam(struct net_device *ndev, } static int ravb_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ravb_private *priv = netdev_priv(ndev); const struct ravb_hw_info *info = priv->info; @@ -2488,8 +2491,7 @@ static int ravb_set_gti(struct net_device *ndev) if (!rate) return -EINVAL; - inc = 1000000000ULL << 20; - do_div(inc, rate); + inc = div64_ul(1000000000ULL << 20, rate); if (inc < GTI_TIV_MIN || inc > GTI_TIV_MAX) { dev_err(dev, "gti.tiv increment 0x%llx is outside the range 0x%x - 0x%x\n", diff --git a/drivers/net/ethernet/renesas/sh_eth.c b/drivers/net/ethernet/renesas/sh_eth.c index a3fbb2221c9a..223626290ce0 100644 --- a/drivers/net/ethernet/renesas/sh_eth.c +++ b/drivers/net/ethernet/renesas/sh_eth.c @@ -2296,7 +2296,9 @@ static void sh_eth_get_strings(struct net_device *ndev, u32 stringset, u8 *data) } static void sh_eth_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct sh_eth_private *mdp = netdev_priv(ndev); @@ -2307,7 +2309,9 @@ static void sh_eth_get_ringparam(struct net_device *ndev, } static int sh_eth_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct sh_eth_private *mdp = netdev_priv(ndev); int ret; diff --git a/drivers/net/ethernet/rocker/rocker_main.c b/drivers/net/ethernet/rocker/rocker_main.c index ba4062881eed..b620470c7905 100644 --- a/drivers/net/ethernet/rocker/rocker_main.c +++ b/drivers/net/ethernet/rocker/rocker_main.c @@ -1995,17 +1995,6 @@ static int rocker_port_get_phys_port_name(struct net_device *dev, return err ? -EOPNOTSUPP : 0; } -static int rocker_port_change_proto_down(struct net_device *dev, - bool proto_down) -{ - struct rocker_port *rocker_port = netdev_priv(dev); - - if (rocker_port->dev->flags & IFF_UP) - rocker_port_set_enable(rocker_port, !proto_down); - rocker_port->dev->proto_down = proto_down; - return 0; -} - static void rocker_port_neigh_destroy(struct net_device *dev, struct neighbour *n) { @@ -2037,7 +2026,6 @@ static const struct net_device_ops rocker_port_netdev_ops = { .ndo_set_mac_address = rocker_port_set_mac_address, .ndo_change_mtu = rocker_port_change_mtu, .ndo_get_phys_port_name = rocker_port_get_phys_port_name, - .ndo_change_proto_down = rocker_port_change_proto_down, .ndo_neigh_destroy = rocker_port_neigh_destroy, .ndo_get_port_parent_id = rocker_port_get_port_parent_id, }; diff --git a/drivers/net/ethernet/sfc/ef100_ethtool.c b/drivers/net/ethernet/sfc/ef100_ethtool.c index 835c838b7dfa..5dba4125d953 100644 --- a/drivers/net/ethernet/sfc/ef100_ethtool.c +++ b/drivers/net/ethernet/sfc/ef100_ethtool.c @@ -20,8 +20,11 @@ /* This is the maximum number of descriptor rings supported by the QDMA */ #define EFX_EF100_MAX_DMAQ_SIZE 16384UL -static void ef100_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +ef100_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); diff --git a/drivers/net/ethernet/sfc/ef100_nic.c b/drivers/net/ethernet/sfc/ef100_nic.c index 6aa81229b68a..5a6fed33a1d7 100644 --- a/drivers/net/ethernet/sfc/ef100_nic.c +++ b/drivers/net/ethernet/sfc/ef100_nic.c @@ -993,11 +993,11 @@ static int ef100_process_design_param(struct efx_nic *efx, return 0; case ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_LEN: nic_data->tso_max_payload_len = min_t(u64, reader->value, GSO_MAX_SIZE); - efx->net_dev->gso_max_size = nic_data->tso_max_payload_len; + netif_set_gso_max_size(efx->net_dev, nic_data->tso_max_payload_len); return 0; case ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_NUM_SEGS: nic_data->tso_max_payload_num_segs = min_t(u64, reader->value, 0xffff); - efx->net_dev->gso_max_segs = nic_data->tso_max_payload_num_segs; + netif_set_gso_max_segs(efx->net_dev, nic_data->tso_max_payload_num_segs); return 0; case ESE_EF100_DP_GZ_TSO_MAX_NUM_FRAMES: nic_data->tso_max_frames = min_t(u64, reader->value, 0xffff); @@ -1122,7 +1122,7 @@ static int ef100_probe_main(struct efx_nic *efx) nic_data->tso_max_frames = ESE_EF100_DP_GZ_TSO_MAX_NUM_FRAMES_DEFAULT; nic_data->tso_max_payload_num_segs = ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_NUM_SEGS_DEFAULT; nic_data->tso_max_payload_len = ESE_EF100_DP_GZ_TSO_MAX_PAYLOAD_LEN_DEFAULT; - net_dev->gso_max_segs = ESE_EF100_DP_GZ_TSO_MAX_HDR_NUM_SEGS_DEFAULT; + netif_set_gso_max_segs(net_dev, ESE_EF100_DP_GZ_TSO_MAX_HDR_NUM_SEGS_DEFAULT); /* Read design parameters */ rc = ef100_check_design_params(efx); if (rc) { diff --git a/drivers/net/ethernet/sfc/efx.c b/drivers/net/ethernet/sfc/efx.c index 6960a2fe2b53..a8c252e2b252 100644 --- a/drivers/net/ethernet/sfc/efx.c +++ b/drivers/net/ethernet/sfc/efx.c @@ -709,7 +709,7 @@ static int efx_register_netdev(struct efx_nic *efx) if (efx_nic_rev(efx) >= EFX_REV_HUNT_A0) net_dev->priv_flags |= IFF_UNICAST_FLT; net_dev->ethtool_ops = &efx_ethtool_ops; - net_dev->gso_max_segs = EFX_TSO_MAX_SEGS; + netif_set_gso_max_segs(net_dev, EFX_TSO_MAX_SEGS); net_dev->min_mtu = EFX_MIN_MTU; net_dev->max_mtu = EFX_MAX_MTU; diff --git a/drivers/net/ethernet/sfc/ethtool.c b/drivers/net/ethernet/sfc/ethtool.c index e002ce21788d..48506373721a 100644 --- a/drivers/net/ethernet/sfc/ethtool.c +++ b/drivers/net/ethernet/sfc/ethtool.c @@ -157,8 +157,11 @@ static int efx_ethtool_set_coalesce(struct net_device *net_dev, return 0; } -static void efx_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +efx_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); @@ -168,8 +171,11 @@ static void efx_ethtool_get_ringparam(struct net_device *net_dev, ring->tx_pending = efx->txq_entries; } -static int efx_ethtool_set_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static int +efx_ethtool_set_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct efx_nic *efx = netdev_priv(net_dev); u32 txq_entries; diff --git a/drivers/net/ethernet/sfc/falcon/efx.c b/drivers/net/ethernet/sfc/falcon/efx.c index 314c9c69eb0e..60c595ef7589 100644 --- a/drivers/net/ethernet/sfc/falcon/efx.c +++ b/drivers/net/ethernet/sfc/falcon/efx.c @@ -2267,7 +2267,7 @@ static int ef4_register_netdev(struct ef4_nic *efx) net_dev->irq = efx->pci_dev->irq; net_dev->netdev_ops = &ef4_netdev_ops; net_dev->ethtool_ops = &ef4_ethtool_ops; - net_dev->gso_max_segs = EF4_TSO_MAX_SEGS; + netif_set_gso_max_segs(net_dev, EF4_TSO_MAX_SEGS); net_dev->min_mtu = EF4_MIN_MTU; net_dev->max_mtu = EF4_MAX_MTU; diff --git a/drivers/net/ethernet/sfc/falcon/ethtool.c b/drivers/net/ethernet/sfc/falcon/ethtool.c index 137e8a7aeaa1..907254b36663 100644 --- a/drivers/net/ethernet/sfc/falcon/ethtool.c +++ b/drivers/net/ethernet/sfc/falcon/ethtool.c @@ -637,8 +637,11 @@ static int ef4_ethtool_set_coalesce(struct net_device *net_dev, return 0; } -static void ef4_ethtool_get_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static void +ef4_ethtool_get_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ef4_nic *efx = netdev_priv(net_dev); @@ -648,8 +651,11 @@ static void ef4_ethtool_get_ringparam(struct net_device *net_dev, ring->tx_pending = efx->txq_entries; } -static int ef4_ethtool_set_ringparam(struct net_device *net_dev, - struct ethtool_ringparam *ring) +static int +ef4_ethtool_set_ringparam(struct net_device *net_dev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct ef4_nic *efx = netdev_priv(net_dev); u32 txq_entries; diff --git a/drivers/net/ethernet/smsc/smc9194.c b/drivers/net/ethernet/smsc/smc9194.c index 0ce403fa5f1a..af661c65ffe2 100644 --- a/drivers/net/ethernet/smsc/smc9194.c +++ b/drivers/net/ethernet/smsc/smc9194.c @@ -856,6 +856,7 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) word configuration_register; word memory_info_register; word memory_cfg_register; + u8 addr[ETH_ALEN]; /* Grab the region so that no one else tries to probe our ioports. */ if (!request_region(ioaddr, SMC_IO_EXTENT, DRV_NAME)) @@ -924,9 +925,10 @@ static int __init smc_probe(struct net_device *dev, int ioaddr) word address; address = inw( ioaddr + ADDR0 + i ); - dev->dev_addr[ i + 1] = address >> 8; - dev->dev_addr[ i ] = address & 0xFF; + addr[i + 1] = address >> 8; + addr[i] = address & 0xFF; } + eth_hw_addr_set(dev, addr); /* get the memory information */ diff --git a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c index 5c74b6279d69..2ffa0a11eea5 100644 --- a/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c +++ b/drivers/net/ethernet/stmicro/stmmac/dwmac-qcom-ethqos.c @@ -113,8 +113,10 @@ static void rgmii_updatel(struct qcom_ethqos *ethqos, rgmii_writel(ethqos, temp, offset); } -static void rgmii_dump(struct qcom_ethqos *ethqos) +static void rgmii_dump(void *priv) { + struct qcom_ethqos *ethqos = priv; + dev_dbg(ðqos->pdev->dev, "Rgmii register dump\n"); dev_dbg(ðqos->pdev->dev, "RGMII_IO_MACRO_CONFIG: %x\n", rgmii_readl(ethqos, RGMII_IO_MACRO_CONFIG)); @@ -447,6 +449,24 @@ static void ethqos_fix_mac_speed(void *priv, unsigned int speed) ethqos_configure(ethqos); } +static int ethqos_clks_config(void *priv, bool enabled) +{ + struct qcom_ethqos *ethqos = priv; + int ret = 0; + + if (enabled) { + ret = clk_prepare_enable(ethqos->rgmii_clk); + if (ret) { + dev_err(ðqos->pdev->dev, "rgmii_clk enable failed\n"); + return ret; + } + } else { + clk_disable_unprepare(ethqos->rgmii_clk); + } + + return ret; +} + static int qcom_ethqos_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; @@ -466,6 +486,8 @@ static int qcom_ethqos_probe(struct platform_device *pdev) return PTR_ERR(plat_dat); } + plat_dat->clks_config = ethqos_clks_config; + ethqos = devm_kzalloc(&pdev->dev, sizeof(*ethqos), GFP_KERNEL); if (!ethqos) { ret = -ENOMEM; @@ -489,7 +511,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) goto err_mem; } - ret = clk_prepare_enable(ethqos->rgmii_clk); + ret = ethqos_clks_config(ethqos, true); if (ret) goto err_mem; @@ -499,6 +521,7 @@ static int qcom_ethqos_probe(struct platform_device *pdev) plat_dat->bsp_priv = ethqos; plat_dat->fix_mac_speed = ethqos_fix_mac_speed; + plat_dat->dump_debug_regs = rgmii_dump; plat_dat->has_gmac4 = 1; plat_dat->pmt = 1; plat_dat->tso_en = of_property_read_bool(np, "snps,tso"); @@ -507,12 +530,10 @@ static int qcom_ethqos_probe(struct platform_device *pdev) if (ret) goto err_clk; - rgmii_dump(ethqos); - return ret; err_clk: - clk_disable_unprepare(ethqos->rgmii_clk); + ethqos_clks_config(ethqos, false); err_mem: stmmac_remove_config_dt(pdev, plat_dat); @@ -530,7 +551,7 @@ static int qcom_ethqos_remove(struct platform_device *pdev) return -ENODEV; ret = stmmac_pltfr_remove(pdev); - clk_disable_unprepare(ethqos->rgmii_clk); + ethqos_clks_config(ethqos, false); return ret; } diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac.h b/drivers/net/ethernet/stmicro/stmmac/stmmac.h index 5f129733aabd..4f5292cadf54 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac.h +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac.h @@ -10,7 +10,6 @@ #define __STMMAC_H__ #define STMMAC_RESOURCE_NAME "stmmaceth" -#define DRV_MODULE_VERSION "Jan_2016" #include <linux/clk.h> #include <linux/hrtimer.h> @@ -317,8 +316,8 @@ void stmmac_set_ethtool_ops(struct net_device *netdev); int stmmac_init_tstamp_counter(struct stmmac_priv *priv, u32 systime_flags); void stmmac_ptp_register(struct stmmac_priv *priv); void stmmac_ptp_unregister(struct stmmac_priv *priv); -int stmmac_open(struct net_device *dev); -int stmmac_release(struct net_device *dev); +int stmmac_xdp_open(struct net_device *dev); +void stmmac_xdp_release(struct net_device *dev); int stmmac_resume(struct device *dev); int stmmac_suspend(struct device *dev); int stmmac_dvr_remove(struct device *dev); diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c index d89455803bed..164dff5ec32e 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ethtool.c @@ -290,7 +290,6 @@ static void stmmac_ethtool_getdrvinfo(struct net_device *dev, strlcpy(info->bus_info, pci_name(priv->plat->pdev), sizeof(info->bus_info)); } - strlcpy(info->version, DRV_MODULE_VERSION, sizeof(info->version)); } static int stmmac_ethtool_get_link_ksettings(struct net_device *dev, @@ -463,7 +462,9 @@ static int stmmac_nway_reset(struct net_device *dev) } static void stmmac_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct stmmac_priv *priv = netdev_priv(netdev); @@ -474,7 +475,9 @@ static void stmmac_get_ringparam(struct net_device *netdev, } static int stmmac_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { if (ring->rx_mini_pending || ring->rx_jumbo_pending || ring->rx_pending < DMA_MIN_RX_SIZE || diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c index da8306f60730..4e05c1d92935 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_main.c @@ -518,14 +518,6 @@ bool stmmac_eee_init(struct stmmac_priv *priv) return true; } -static inline u32 stmmac_cdc_adjust(struct stmmac_priv *priv) -{ - /* Correct the clk domain crossing(CDC) error */ - if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) - return (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; - return 0; -} - /* stmmac_get_tx_hwtstamp - get HW TX timestamps * @priv: driver private structure * @p : descriptor pointer @@ -557,7 +549,7 @@ static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv, } if (found) { - ns -= stmmac_cdc_adjust(priv); + ns -= priv->plat->cdc_error_adj; memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps)); shhwtstamp.hwtstamp = ns_to_ktime(ns); @@ -594,7 +586,7 @@ static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p, if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) { stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns); - ns -= stmmac_cdc_adjust(priv); + ns -= priv->plat->cdc_error_adj; netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns); shhwtstamp = skb_hwtstamps(skb); @@ -2390,7 +2382,7 @@ static bool stmmac_xdp_xmit_zc(struct stmmac_priv *priv, u32 queue, u32 budget) bool work_done = true; /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); budget = min(budget, stmmac_tx_avail(priv, queue)); @@ -3673,7 +3665,7 @@ static int stmmac_request_irq(struct net_device *dev) * 0 on success and an appropriate (-)ve integer as defined in errno.h * file on failure. */ -int stmmac_open(struct net_device *dev) +static int stmmac_open(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); int mode = priv->plat->phy_interface; @@ -3797,7 +3789,7 @@ static void stmmac_fpe_stop_wq(struct stmmac_priv *priv) * Description: * This is the stop entry point of the driver. */ -int stmmac_release(struct net_device *dev) +static int stmmac_release(struct net_device *dev) { struct stmmac_priv *priv = netdev_priv(dev); u32 chan; @@ -4689,7 +4681,7 @@ static int stmmac_xdp_xmit_back(struct stmmac_priv *priv, __netif_tx_lock(nq, cpu); /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); res = stmmac_xdp_xmit_xdpf(priv, queue, xdpf, false); if (res == STMMAC_XDP_TX) @@ -6327,7 +6319,7 @@ static int stmmac_xdp_xmit(struct net_device *dev, int num_frames, __netif_tx_lock(nq, cpu); /* Avoids TX time-out as we are sharing with slow path */ - nq->trans_start = jiffies; + txq_trans_cond_update(nq); for (i = 0; i < num_frames; i++) { int res; @@ -6463,6 +6455,140 @@ void stmmac_enable_tx_queue(struct stmmac_priv *priv, u32 queue) spin_unlock_irqrestore(&ch->lock, flags); } +void stmmac_xdp_release(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 chan; + + /* Disable NAPI process */ + stmmac_disable_all_queues(priv); + + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + hrtimer_cancel(&priv->tx_queue[chan].txtimer); + + /* Free the IRQ lines */ + stmmac_free_irq(dev, REQ_IRQ_ERR_ALL, 0); + + /* Stop TX/RX DMA channels */ + stmmac_stop_all_dma(priv); + + /* Release and free the Rx/Tx resources */ + free_dma_desc_resources(priv); + + /* Disable the MAC Rx/Tx */ + stmmac_mac_set(priv, priv->ioaddr, false); + + /* set trans_start so we don't get spurious + * watchdogs during reset + */ + netif_trans_update(dev); + netif_carrier_off(dev); +} + +int stmmac_xdp_open(struct net_device *dev) +{ + struct stmmac_priv *priv = netdev_priv(dev); + u32 rx_cnt = priv->plat->rx_queues_to_use; + u32 tx_cnt = priv->plat->tx_queues_to_use; + u32 dma_csr_ch = max(rx_cnt, tx_cnt); + struct stmmac_rx_queue *rx_q; + struct stmmac_tx_queue *tx_q; + u32 buf_size; + bool sph_en; + u32 chan; + int ret; + + ret = alloc_dma_desc_resources(priv); + if (ret < 0) { + netdev_err(dev, "%s: DMA descriptors allocation failed\n", + __func__); + goto dma_desc_error; + } + + ret = init_dma_desc_rings(dev, GFP_KERNEL); + if (ret < 0) { + netdev_err(dev, "%s: DMA descriptors initialization failed\n", + __func__); + goto init_error; + } + + /* DMA CSR Channel configuration */ + for (chan = 0; chan < dma_csr_ch; chan++) + stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan); + + /* Adjust Split header */ + sph_en = (priv->hw->rx_csum > 0) && priv->sph; + + /* DMA RX Channel Configuration */ + for (chan = 0; chan < rx_cnt; chan++) { + rx_q = &priv->rx_queue[chan]; + + stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + rx_q->dma_rx_phy, chan); + + rx_q->rx_tail_addr = rx_q->dma_rx_phy + + (rx_q->buf_alloc_num * + sizeof(struct dma_desc)); + stmmac_set_rx_tail_ptr(priv, priv->ioaddr, + rx_q->rx_tail_addr, chan); + + if (rx_q->xsk_pool && rx_q->buf_alloc_num) { + buf_size = xsk_pool_get_rx_frame_size(rx_q->xsk_pool); + stmmac_set_dma_bfsize(priv, priv->ioaddr, + buf_size, + rx_q->queue_index); + } else { + stmmac_set_dma_bfsize(priv, priv->ioaddr, + priv->dma_buf_sz, + rx_q->queue_index); + } + + stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan); + } + + /* DMA TX Channel Configuration */ + for (chan = 0; chan < tx_cnt; chan++) { + tx_q = &priv->tx_queue[chan]; + + stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg, + tx_q->dma_tx_phy, chan); + + tx_q->tx_tail_addr = tx_q->dma_tx_phy; + stmmac_set_tx_tail_ptr(priv, priv->ioaddr, + tx_q->tx_tail_addr, chan); + + hrtimer_init(&tx_q->txtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); + tx_q->txtimer.function = stmmac_tx_timer; + } + + /* Enable the MAC Rx/Tx */ + stmmac_mac_set(priv, priv->ioaddr, true); + + /* Start Rx & Tx DMA Channels */ + stmmac_start_all_dma(priv); + + ret = stmmac_request_irq(dev); + if (ret) + goto irq_error; + + /* Enable NAPI process*/ + stmmac_enable_all_queues(priv); + netif_carrier_on(dev); + netif_tx_start_all_queues(dev); + + return 0; + +irq_error: + for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++) + hrtimer_cancel(&priv->tx_queue[chan].txtimer); + + stmmac_hw_teardown(dev); +init_error: + free_dma_desc_resources(priv); +dma_desc_error: + return ret; +} + int stmmac_xsk_wakeup(struct net_device *dev, u32 queue, u32 flags) { struct stmmac_priv *priv = netdev_priv(dev); @@ -7086,6 +7212,9 @@ int stmmac_dvr_probe(struct device *device, stmmac_init_fs(ndev); #endif + if (priv->plat->dump_debug_regs) + priv->plat->dump_debug_regs(priv->plat->bsp_priv); + /* Let pm_runtime_put() disable the clocks. * If CONFIG_PM is not enabled, the clocks will stay powered. */ diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c index 580cc035536b..e14c97c04317 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_ptp.c @@ -309,6 +309,11 @@ void stmmac_ptp_register(struct stmmac_priv *priv) if (priv->plat->ptp_max_adj) stmmac_ptp_clock_ops.max_adj = priv->plat->ptp_max_adj; + /* Calculate the clock domain crossing (CDC) error if necessary */ + priv->plat->cdc_error_adj = 0; + if (priv->plat->has_gmac4 && priv->plat->clk_ptp_rate) + priv->plat->cdc_error_adj = (2 * NSEC_PER_SEC) / priv->plat->clk_ptp_rate; + stmmac_ptp_clock_ops.n_per_out = priv->dma_cap.pps_out_num; stmmac_ptp_clock_ops.n_ext_ts = priv->dma_cap.aux_snapshot_n; diff --git a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c index 2a616c6f7cd0..9d4d8c3dad0a 100644 --- a/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c +++ b/drivers/net/ethernet/stmicro/stmmac/stmmac_xdp.c @@ -119,7 +119,7 @@ int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog, need_update = !!priv->xdp_prog != !!prog; if (if_running && need_update) - stmmac_release(dev); + stmmac_xdp_release(dev); old_prog = xchg(&priv->xdp_prog, prog); if (old_prog) @@ -129,7 +129,7 @@ int stmmac_xdp_set_prog(struct stmmac_priv *priv, struct bpf_prog *prog, priv->sph = priv->sph_cap && !stmmac_xdp_is_enabled(priv); if (if_running && need_update) - stmmac_open(dev); + stmmac_xdp_open(dev); return 0; } diff --git a/drivers/net/ethernet/tehuti/tehuti.c b/drivers/net/ethernet/tehuti/tehuti.c index 0775a5542f2f..89bc1602661c 100644 --- a/drivers/net/ethernet/tehuti/tehuti.c +++ b/drivers/net/ethernet/tehuti/tehuti.c @@ -2245,9 +2245,13 @@ static inline int bdx_tx_fifo_size_to_packets(int tx_size) * bdx_get_ringparam - report ring sizes * @netdev * @ring + * @kernel_ring + * @extack */ static void -bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct bdx_priv *priv = netdev_priv(netdev); @@ -2262,9 +2266,13 @@ bdx_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) * bdx_set_ringparam - set ring sizes * @netdev * @ring + * @kernel_ring + * @extack */ static int -bdx_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring) +bdx_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct bdx_priv *priv = netdev_priv(netdev); int rx_size = 0; diff --git a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c index b05de9b61ad6..d45b6bb86f0b 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-ethtool.c +++ b/drivers/net/ethernet/ti/am65-cpsw-ethtool.c @@ -453,8 +453,11 @@ static int am65_cpsw_set_channels(struct net_device *ndev, return am65_cpsw_nuss_update_tx_chns(common, chs->tx_count); } -static void am65_cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +am65_cpsw_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct am65_cpsw_common *common = am65_ndev_to_common(ndev); diff --git a/drivers/net/ethernet/ti/am65-cpsw-nuss.c b/drivers/net/ethernet/ti/am65-cpsw-nuss.c index c092cb61416a..750cea23e9cd 100644 --- a/drivers/net/ethernet/ti/am65-cpsw-nuss.c +++ b/drivers/net/ethernet/ti/am65-cpsw-nuss.c @@ -345,7 +345,7 @@ static void am65_cpsw_nuss_ndo_host_tx_timeout(struct net_device *ndev, netif_txq = netdev_get_tx_queue(ndev, txqueue); tx_chn = &common->tx_chns[txqueue]; - trans_start = netif_txq->trans_start; + trans_start = READ_ONCE(netif_txq->trans_start); netdev_err(ndev, "txq:%d DRV_XOFF:%d tmo:%u dql_avail:%d free_desc:%zu\n", txqueue, diff --git a/drivers/net/ethernet/ti/cpmac.c b/drivers/net/ethernet/ti/cpmac.c index 7449436fc87c..bef5e68dac31 100644 --- a/drivers/net/ethernet/ti/cpmac.c +++ b/drivers/net/ethernet/ti/cpmac.c @@ -817,7 +817,9 @@ static void cpmac_tx_timeout(struct net_device *dev, unsigned int txqueue) } static void cpmac_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct cpmac_priv *priv = netdev_priv(dev); @@ -833,7 +835,9 @@ static void cpmac_get_ringparam(struct net_device *dev, } static int cpmac_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct cpmac_priv *priv = netdev_priv(dev); diff --git a/drivers/net/ethernet/ti/cpsw_ethtool.c b/drivers/net/ethernet/ti/cpsw_ethtool.c index 158c8d3793f4..aa42141be3c0 100644 --- a/drivers/net/ethernet/ti/cpsw_ethtool.c +++ b/drivers/net/ethernet/ti/cpsw_ethtool.c @@ -658,7 +658,9 @@ err: } void cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct cpsw_priv *priv = netdev_priv(ndev); struct cpsw_common *cpsw = priv->cpsw; @@ -671,7 +673,9 @@ void cpsw_get_ringparam(struct net_device *ndev, } int cpsw_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct cpsw_common *cpsw = ndev_to_cpsw(ndev); int descs_num, ret; diff --git a/drivers/net/ethernet/ti/cpsw_priv.c b/drivers/net/ethernet/ti/cpsw_priv.c index ecc2a6b7e28f..c99dd9735087 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.c +++ b/drivers/net/ethernet/ti/cpsw_priv.c @@ -710,20 +710,26 @@ int cpsw_ndo_ioctl(struct net_device *dev, struct ifreq *req, int cmd) struct cpsw_priv *priv = netdev_priv(dev); struct cpsw_common *cpsw = priv->cpsw; int slave_no = cpsw_slave_index(cpsw, priv); + struct phy_device *phy; if (!netif_running(dev)) return -EINVAL; - switch (cmd) { - case SIOCSHWTSTAMP: - return cpsw_hwtstamp_set(dev, req); - case SIOCGHWTSTAMP: - return cpsw_hwtstamp_get(dev, req); + phy = cpsw->slaves[slave_no].phy; + + if (!phy_has_hwtstamp(phy)) { + switch (cmd) { + case SIOCSHWTSTAMP: + return cpsw_hwtstamp_set(dev, req); + case SIOCGHWTSTAMP: + return cpsw_hwtstamp_get(dev, req); + } } - if (!cpsw->slaves[slave_no].phy) - return -EOPNOTSUPP; - return phy_mii_ioctl(cpsw->slaves[slave_no].phy, req, cmd); + if (phy) + return phy_mii_ioctl(phy, req, cmd); + + return -EOPNOTSUPP; } int cpsw_ndo_set_tx_maxrate(struct net_device *ndev, int queue, u32 rate) diff --git a/drivers/net/ethernet/ti/cpsw_priv.h b/drivers/net/ethernet/ti/cpsw_priv.h index 435668ee542d..f33c882eb70e 100644 --- a/drivers/net/ethernet/ti/cpsw_priv.h +++ b/drivers/net/ethernet/ti/cpsw_priv.h @@ -491,9 +491,13 @@ int cpsw_get_eee(struct net_device *ndev, struct ethtool_eee *edata); int cpsw_set_eee(struct net_device *ndev, struct ethtool_eee *edata); int cpsw_nway_reset(struct net_device *ndev); void cpsw_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering); + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack); int cpsw_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering); + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack); int cpsw_set_channels_common(struct net_device *ndev, struct ethtool_channels *chs, cpdma_handler_fn rx_handler); diff --git a/drivers/net/ethernet/toshiba/spider_net.c b/drivers/net/ethernet/toshiba/spider_net.c index f50f9a43d3ea..f47b8358669d 100644 --- a/drivers/net/ethernet/toshiba/spider_net.c +++ b/drivers/net/ethernet/toshiba/spider_net.c @@ -595,24 +595,24 @@ spider_net_set_multi(struct net_device *netdev) int i; u32 reg; struct spider_net_card *card = netdev_priv(netdev); - DECLARE_BITMAP(bitmask, SPIDER_NET_MULTICAST_HASHES) = {}; + DECLARE_BITMAP(bitmask, SPIDER_NET_MULTICAST_HASHES); spider_net_set_promisc(card); if (netdev->flags & IFF_ALLMULTI) { - for (i = 0; i < SPIDER_NET_MULTICAST_HASHES; i++) { - set_bit(i, bitmask); - } + bitmap_fill(bitmask, SPIDER_NET_MULTICAST_HASHES); goto write_hash; } + bitmap_zero(bitmask, SPIDER_NET_MULTICAST_HASHES); + /* well, we know, what the broadcast hash value is: it's xfd hash = spider_net_get_multicast_hash(netdev, netdev->broadcast); */ - set_bit(0xfd, bitmask); + __set_bit(0xfd, bitmask); netdev_for_each_mc_addr(ha, netdev) { hash = spider_net_get_multicast_hash(netdev, ha->addr); - set_bit(hash, bitmask); + __set_bit(hash, bitmask); } write_hash: diff --git a/drivers/net/ethernet/toshiba/spider_net_ethtool.c b/drivers/net/ethernet/toshiba/spider_net_ethtool.c index 54f655a68148..93110dba0bfa 100644 --- a/drivers/net/ethernet/toshiba/spider_net_ethtool.c +++ b/drivers/net/ethernet/toshiba/spider_net_ethtool.c @@ -110,7 +110,9 @@ spider_net_ethtool_nway_reset(struct net_device *netdev) static void spider_net_ethtool_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ering) + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct spider_net_card *card = netdev_priv(netdev); diff --git a/drivers/net/ethernet/xilinx/ll_temac_main.c b/drivers/net/ethernet/xilinx/ll_temac_main.c index e7065c9a8e38..b900ab5aef2a 100644 --- a/drivers/net/ethernet/xilinx/ll_temac_main.c +++ b/drivers/net/ethernet/xilinx/ll_temac_main.c @@ -1276,8 +1276,11 @@ static const struct attribute_group temac_attr_group = { * ethtool support */ -static void ll_temac_ethtools_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +ll_temac_ethtools_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct temac_local *lp = netdev_priv(ndev); @@ -1291,8 +1294,11 @@ static void ll_temac_ethtools_get_ringparam(struct net_device *ndev, ering->tx_pending = lp->tx_bd_num; } -static int ll_temac_ethtools_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static int +ll_temac_ethtools_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct temac_local *lp = netdev_priv(ndev); diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c index 9b068b81ae09..e39356364f33 100644 --- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c +++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c @@ -1323,8 +1323,11 @@ static void axienet_ethtools_get_regs(struct net_device *ndev, data[39] = axienet_dma_in32(lp, XAXIDMA_RX_TDESC_OFFSET); } -static void axienet_ethtools_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static void +axienet_ethtools_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct axienet_local *lp = netdev_priv(ndev); @@ -1338,8 +1341,11 @@ static void axienet_ethtools_get_ringparam(struct net_device *ndev, ering->tx_pending = lp->tx_bd_num; } -static int axienet_ethtools_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ering) +static int +axienet_ethtools_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ering, + struct kernel_ethtool_ringparam *kernel_ering, + struct netlink_ext_ack *extack) { struct axienet_local *lp = netdev_priv(ndev); @@ -1503,65 +1509,6 @@ static const struct ethtool_ops axienet_ethtool_ops = { .nway_reset = axienet_ethtools_nway_reset, }; -static void axienet_validate(struct phylink_config *config, - unsigned long *supported, - struct phylink_link_state *state) -{ - struct net_device *ndev = to_net_dev(config->dev); - struct axienet_local *lp = netdev_priv(ndev); - __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; - - /* Only support the mode we are configured for */ - switch (state->interface) { - case PHY_INTERFACE_MODE_NA: - break; - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_SGMII: - if (lp->switch_x_sgmii) - break; - fallthrough; - default: - if (state->interface != lp->phy_mode) { - netdev_warn(ndev, "Cannot use PHY mode %s, supported: %s\n", - phy_modes(state->interface), - phy_modes(lp->phy_mode)); - linkmode_zero(supported); - return; - } - } - - phylink_set(mask, Autoneg); - phylink_set_port_modes(mask); - - phylink_set(mask, Asym_Pause); - phylink_set(mask, Pause); - - switch (state->interface) { - case PHY_INTERFACE_MODE_NA: - case PHY_INTERFACE_MODE_1000BASEX: - case PHY_INTERFACE_MODE_SGMII: - case PHY_INTERFACE_MODE_GMII: - case PHY_INTERFACE_MODE_RGMII: - case PHY_INTERFACE_MODE_RGMII_ID: - case PHY_INTERFACE_MODE_RGMII_RXID: - case PHY_INTERFACE_MODE_RGMII_TXID: - phylink_set(mask, 1000baseX_Full); - phylink_set(mask, 1000baseT_Full); - if (state->interface == PHY_INTERFACE_MODE_1000BASEX) - break; - fallthrough; - case PHY_INTERFACE_MODE_MII: - phylink_set(mask, 100baseT_Full); - phylink_set(mask, 10baseT_Full); - fallthrough; - default: - break; - } - - linkmode_and(supported, supported, mask); - linkmode_and(state->advertising, state->advertising, mask); -} - static void axienet_mac_pcs_get_state(struct phylink_config *config, struct phylink_link_state *state) { @@ -1687,7 +1634,7 @@ static void axienet_mac_link_up(struct phylink_config *config, } static const struct phylink_mac_ops axienet_phylink_ops = { - .validate = axienet_validate, + .validate = phylink_generic_validate, .mac_pcs_get_state = axienet_mac_pcs_get_state, .mac_an_restart = axienet_mac_an_restart, .mac_prepare = axienet_mac_prepare, @@ -2104,6 +2051,16 @@ static int axienet_probe(struct platform_device *pdev) lp->phylink_config.dev = &ndev->dev; lp->phylink_config.type = PHYLINK_NETDEV; + lp->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | + MAC_10FD | MAC_100FD | MAC_1000FD; + + __set_bit(lp->phy_mode, lp->phylink_config.supported_interfaces); + if (lp->switch_x_sgmii) { + __set_bit(PHY_INTERFACE_MODE_1000BASEX, + lp->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_SGMII, + lp->phylink_config.supported_interfaces); + } lp->phylink = phylink_create(&lp->phylink_config, pdev->dev.fwnode, lp->phy_mode, diff --git a/drivers/net/fddi/skfp/smt.c b/drivers/net/fddi/skfp/smt.c index 6b68a53f1b38..72c31f0013ad 100644 --- a/drivers/net/fddi/skfp/smt.c +++ b/drivers/net/fddi/skfp/smt.c @@ -1846,10 +1846,10 @@ void smt_swap_para(struct smt_header *sm, int len, int direction) } } + static void smt_string_swap(char *data, const char *format, int len) { const char *open_paren = NULL ; - int x ; while (len > 0 && *format) { switch (*format) { @@ -1876,19 +1876,13 @@ static void smt_string_swap(char *data, const char *format, int len) len-- ; break ; case 's' : - x = data[0] ; - data[0] = data[1] ; - data[1] = x ; + swap(data[0], data[1]) ; data += 2 ; len -= 2 ; break ; case 'l' : - x = data[0] ; - data[0] = data[3] ; - data[3] = x ; - x = data[1] ; - data[1] = data[2] ; - data[2] = x ; + swap(data[0], data[3]) ; + swap(data[1], data[2]) ; data += 4 ; len -= 4 ; break ; diff --git a/drivers/net/geneve.c b/drivers/net/geneve.c index 1ab94b5f9bbf..c1fdd721a730 100644 --- a/drivers/net/geneve.c +++ b/drivers/net/geneve.c @@ -17,6 +17,7 @@ #include <net/gro_cells.h> #include <net/rtnetlink.h> #include <net/geneve.h> +#include <net/gro.h> #include <net/protocol.h> #define GENEVE_NETDEV_VER "0.6" @@ -516,18 +517,15 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk, type = gh->proto_type; - rcu_read_lock(); ptype = gro_find_receive_by_type(type); if (!ptype) - goto out_unlock; + goto out; skb_gro_pull(skb, gh_len); skb_gro_postpull_rcsum(skb, gh, gh_len); pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); flush = 0; -out_unlock: - rcu_read_unlock(); out: skb_gro_flush_final(skb, pp, flush); @@ -547,13 +545,10 @@ static int geneve_gro_complete(struct sock *sk, struct sk_buff *skb, gh_len = geneve_hlen(gh); type = gh->proto_type; - rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype) err = ptype->callbacks.gro_complete(skb, nhoff + gh_len); - rcu_read_unlock(); - skb_set_inner_mac_header(skb, nhoff + gh_len); return err; diff --git a/drivers/net/hyperv/netvsc.c b/drivers/net/hyperv/netvsc.c index 396bc1c204e6..5086cd07d1ed 100644 --- a/drivers/net/hyperv/netvsc.c +++ b/drivers/net/hyperv/netvsc.c @@ -155,7 +155,7 @@ static void free_netvsc_device(struct rcu_head *head) kfree(nvdev->extension); vfree(nvdev->recv_buf); vfree(nvdev->send_buf); - kfree(nvdev->send_section_map); + bitmap_free(nvdev->send_section_map); for (i = 0; i < VRSS_CHANNEL_MAX; i++) { xdp_rxq_info_unreg(&nvdev->chan_table[i].xdp_rxq); @@ -336,7 +336,6 @@ static int netvsc_init_buf(struct hv_device *device, struct net_device *ndev = hv_get_drvdata(device); struct nvsp_message *init_packet; unsigned int buf_size; - size_t map_words; int i, ret = 0; /* Get receive buffer area. */ @@ -528,10 +527,9 @@ static int netvsc_init_buf(struct hv_device *device, net_device->send_section_size, net_device->send_section_cnt); /* Setup state for managing the send buffer. */ - map_words = DIV_ROUND_UP(net_device->send_section_cnt, BITS_PER_LONG); - - net_device->send_section_map = kcalloc(map_words, sizeof(ulong), GFP_KERNEL); - if (net_device->send_section_map == NULL) { + net_device->send_section_map = bitmap_zalloc(net_device->send_section_cnt, + GFP_KERNEL); + if (!net_device->send_section_map) { ret = -ENOMEM; goto cleanup; } diff --git a/drivers/net/hyperv/netvsc_drv.c b/drivers/net/hyperv/netvsc_drv.c index 7e66ae1d2a59..efa963b7af54 100644 --- a/drivers/net/hyperv/netvsc_drv.c +++ b/drivers/net/hyperv/netvsc_drv.c @@ -1858,7 +1858,9 @@ static void __netvsc_get_ringparam(struct netvsc_device *nvdev, } static void netvsc_get_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct net_device_context *ndevctx = netdev_priv(ndev); struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); @@ -1870,7 +1872,9 @@ static void netvsc_get_ringparam(struct net_device *ndev, } static int netvsc_set_ringparam(struct net_device *ndev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct net_device_context *ndevctx = netdev_priv(ndev); struct netvsc_device *nvdev = rtnl_dereference(ndevctx->nvdev); diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c index 31f522b8e54e..1c64d5347b8e 100644 --- a/drivers/net/ifb.c +++ b/drivers/net/ifb.c @@ -27,6 +27,7 @@ #include <linux/module.h> #include <linux/kernel.h> #include <linux/netdevice.h> +#include <linux/ethtool.h> #include <linux/etherdevice.h> #include <linux/init.h> #include <linux/interrupt.h> @@ -36,30 +37,55 @@ #include <net/net_namespace.h> #define TX_Q_LIMIT 32 + +struct ifb_q_stats { + u64 packets; + u64 bytes; + struct u64_stats_sync sync; +}; + struct ifb_q_private { struct net_device *dev; struct tasklet_struct ifb_tasklet; int tasklet_pending; int txqnum; struct sk_buff_head rq; - u64 rx_packets; - u64 rx_bytes; - struct u64_stats_sync rsync; - - struct u64_stats_sync tsync; - u64 tx_packets; - u64 tx_bytes; struct sk_buff_head tq; + struct ifb_q_stats rx_stats; + struct ifb_q_stats tx_stats; } ____cacheline_aligned_in_smp; struct ifb_dev_private { struct ifb_q_private *tx_private; }; +/* For ethtools stats. */ +struct ifb_q_stats_desc { + char desc[ETH_GSTRING_LEN]; + size_t offset; +}; + +#define IFB_Q_STAT(m) offsetof(struct ifb_q_stats, m) + +static const struct ifb_q_stats_desc ifb_q_stats_desc[] = { + { "packets", IFB_Q_STAT(packets) }, + { "bytes", IFB_Q_STAT(bytes) }, +}; + +#define IFB_Q_STATS_LEN ARRAY_SIZE(ifb_q_stats_desc) + static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev); static int ifb_open(struct net_device *dev); static int ifb_close(struct net_device *dev); +static void ifb_update_q_stats(struct ifb_q_stats *stats, int len) +{ + u64_stats_update_begin(&stats->sync); + stats->packets++; + stats->bytes += len; + u64_stats_update_end(&stats->sync); +} + static void ifb_ri_tasklet(struct tasklet_struct *t) { struct ifb_q_private *txp = from_tasklet(txp, t, ifb_tasklet); @@ -83,10 +109,7 @@ static void ifb_ri_tasklet(struct tasklet_struct *t) #endif nf_skip_egress(skb, true); - u64_stats_update_begin(&txp->tsync); - txp->tx_packets++; - txp->tx_bytes += skb->len; - u64_stats_update_end(&txp->tsync); + ifb_update_q_stats(&txp->tx_stats, skb->len); rcu_read_lock(); skb->dev = dev_get_by_index_rcu(dev_net(txp->dev), skb->skb_iif); @@ -139,18 +162,18 @@ static void ifb_stats64(struct net_device *dev, for (i = 0; i < dev->num_tx_queues; i++,txp++) { do { - start = u64_stats_fetch_begin_irq(&txp->rsync); - packets = txp->rx_packets; - bytes = txp->rx_bytes; - } while (u64_stats_fetch_retry_irq(&txp->rsync, start)); + start = u64_stats_fetch_begin_irq(&txp->rx_stats.sync); + packets = txp->rx_stats.packets; + bytes = txp->rx_stats.bytes; + } while (u64_stats_fetch_retry_irq(&txp->rx_stats.sync, start)); stats->rx_packets += packets; stats->rx_bytes += bytes; do { - start = u64_stats_fetch_begin_irq(&txp->tsync); - packets = txp->tx_packets; - bytes = txp->tx_bytes; - } while (u64_stats_fetch_retry_irq(&txp->tsync, start)); + start = u64_stats_fetch_begin_irq(&txp->tx_stats.sync); + packets = txp->tx_stats.packets; + bytes = txp->tx_stats.bytes; + } while (u64_stats_fetch_retry_irq(&txp->tx_stats.sync, start)); stats->tx_packets += packets; stats->tx_bytes += bytes; } @@ -173,14 +196,83 @@ static int ifb_dev_init(struct net_device *dev) txp->dev = dev; __skb_queue_head_init(&txp->rq); __skb_queue_head_init(&txp->tq); - u64_stats_init(&txp->rsync); - u64_stats_init(&txp->tsync); + u64_stats_init(&txp->rx_stats.sync); + u64_stats_init(&txp->tx_stats.sync); tasklet_setup(&txp->ifb_tasklet, ifb_ri_tasklet); netif_tx_start_queue(netdev_get_tx_queue(dev, i)); } return 0; } +static void ifb_get_strings(struct net_device *dev, u32 stringset, u8 *buf) +{ + u8 *p = buf; + int i, j; + + switch (stringset) { + case ETH_SS_STATS: + for (i = 0; i < dev->real_num_rx_queues; i++) + for (j = 0; j < IFB_Q_STATS_LEN; j++) + ethtool_sprintf(&p, "rx_queue_%u_%.18s", + i, ifb_q_stats_desc[j].desc); + + for (i = 0; i < dev->real_num_tx_queues; i++) + for (j = 0; j < IFB_Q_STATS_LEN; j++) + ethtool_sprintf(&p, "tx_queue_%u_%.18s", + i, ifb_q_stats_desc[j].desc); + + break; + } +} + +static int ifb_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return IFB_Q_STATS_LEN * (dev->real_num_rx_queues + + dev->real_num_tx_queues); + default: + return -EOPNOTSUPP; + } +} + +static void ifb_fill_stats_data(u64 **data, + struct ifb_q_stats *q_stats) +{ + void *stats_base = (void *)q_stats; + unsigned int start; + size_t offset; + int j; + + do { + start = u64_stats_fetch_begin_irq(&q_stats->sync); + for (j = 0; j < IFB_Q_STATS_LEN; j++) { + offset = ifb_q_stats_desc[j].offset; + (*data)[j] = *(u64 *)(stats_base + offset); + } + } while (u64_stats_fetch_retry_irq(&q_stats->sync, start)); + + *data += IFB_Q_STATS_LEN; +} + +static void ifb_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct ifb_dev_private *dp = netdev_priv(dev); + struct ifb_q_private *txp; + int i; + + for (i = 0; i < dev->real_num_rx_queues; i++) { + txp = dp->tx_private + i; + ifb_fill_stats_data(&data, &txp->rx_stats); + } + + for (i = 0; i < dev->real_num_tx_queues; i++) { + txp = dp->tx_private + i; + ifb_fill_stats_data(&data, &txp->tx_stats); + } +} + static const struct net_device_ops ifb_netdev_ops = { .ndo_open = ifb_open, .ndo_stop = ifb_close, @@ -190,6 +282,12 @@ static const struct net_device_ops ifb_netdev_ops = { .ndo_init = ifb_dev_init, }; +static const struct ethtool_ops ifb_ethtool_ops = { + .get_strings = ifb_get_strings, + .get_sset_count = ifb_get_sset_count, + .get_ethtool_stats = ifb_get_ethtool_stats, +}; + #define IFB_FEATURES (NETIF_F_HW_CSUM | NETIF_F_SG | NETIF_F_FRAGLIST | \ NETIF_F_GSO_SOFTWARE | NETIF_F_GSO_ENCAP_ALL | \ NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX | \ @@ -213,6 +311,7 @@ static void ifb_setup(struct net_device *dev) { /* Initialize the device structure. */ dev->netdev_ops = &ifb_netdev_ops; + dev->ethtool_ops = &ifb_ethtool_ops; /* Fill in device structure with ethernet-generic values. */ ether_setup(dev); @@ -241,10 +340,7 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev) struct ifb_dev_private *dp = netdev_priv(dev); struct ifb_q_private *txp = dp->tx_private + skb_get_queue_mapping(skb); - u64_stats_update_begin(&txp->rsync); - txp->rx_packets++; - txp->rx_bytes += skb->len; - u64_stats_update_end(&txp->rsync); + ifb_update_q_stats(&txp->rx_stats, skb->len); if (!skb->redirected || !skb->skb_iif) { dev_kfree_skb(skb); diff --git a/drivers/net/ipa/gsi.c b/drivers/net/ipa/gsi.c index a2fcdb1abdb9..bc981043cc80 100644 --- a/drivers/net/ipa/gsi.c +++ b/drivers/net/ipa/gsi.c @@ -93,6 +93,7 @@ #define GSI_CHANNEL_STOP_RETRIES 10 #define GSI_CHANNEL_MODEM_HALT_RETRIES 10 +#define GSI_CHANNEL_MODEM_FLOW_RETRIES 5 /* disable flow control only */ #define GSI_MHI_EVENT_ID_START 10 /* 1st reserved event id */ #define GSI_MHI_EVENT_ID_END 16 /* Last reserved event id */ @@ -339,10 +340,10 @@ static u32 gsi_ring_index(struct gsi_ring *ring, u32 offset) * completion to be signaled. Returns true if the command completes * or false if it times out. */ -static bool -gsi_command(struct gsi *gsi, u32 reg, u32 val, struct completion *completion) +static bool gsi_command(struct gsi *gsi, u32 reg, u32 val) { unsigned long timeout = msecs_to_jiffies(GSI_CMD_TIMEOUT); + struct completion *completion = &gsi->completion; reinit_completion(completion); @@ -366,8 +367,6 @@ gsi_evt_ring_state(struct gsi *gsi, u32 evt_ring_id) static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, enum gsi_evt_cmd_opcode opcode) { - struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; - struct completion *completion = &evt_ring->completion; struct device *dev = gsi->dev; bool timeout; u32 val; @@ -378,7 +377,7 @@ static void gsi_evt_ring_command(struct gsi *gsi, u32 evt_ring_id, val = u32_encode_bits(evt_ring_id, EV_CHID_FMASK); val |= u32_encode_bits(opcode, EV_OPCODE_FMASK); - timeout = !gsi_command(gsi, GSI_EV_CH_CMD_OFFSET, val, completion); + timeout = !gsi_command(gsi, GSI_EV_CH_CMD_OFFSET, val); gsi_irq_ev_ctrl_disable(gsi); @@ -478,7 +477,6 @@ static enum gsi_channel_state gsi_channel_state(struct gsi_channel *channel) static void gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode) { - struct completion *completion = &channel->completion; u32 channel_id = gsi_channel_id(channel); struct gsi *gsi = channel->gsi; struct device *dev = gsi->dev; @@ -490,7 +488,7 @@ gsi_channel_command(struct gsi_channel *channel, enum gsi_ch_cmd_opcode opcode) val = u32_encode_bits(channel_id, CH_CHID_FMASK); val |= u32_encode_bits(opcode, CH_OPCODE_FMASK); - timeout = !gsi_command(gsi, GSI_CH_CMD_OFFSET, val, completion); + timeout = !gsi_command(gsi, GSI_CH_CMD_OFFSET, val); gsi_irq_ch_ctrl_disable(gsi); @@ -1074,13 +1072,10 @@ static void gsi_isr_chan_ctrl(struct gsi *gsi) while (channel_mask) { u32 channel_id = __ffs(channel_mask); - struct gsi_channel *channel; channel_mask ^= BIT(channel_id); - channel = &gsi->channel[channel_id]; - - complete(&channel->completion); + complete(&gsi->completion); } } @@ -1094,13 +1089,10 @@ static void gsi_isr_evt_ctrl(struct gsi *gsi) while (event_mask) { u32 evt_ring_id = __ffs(event_mask); - struct gsi_evt_ring *evt_ring; event_mask ^= BIT(evt_ring_id); - evt_ring = &gsi->evt_ring[evt_ring_id]; - - complete(&evt_ring->completion); + complete(&gsi->completion); } } @@ -1110,7 +1102,7 @@ gsi_isr_glob_chan_err(struct gsi *gsi, u32 err_ee, u32 channel_id, u32 code) { if (code == GSI_OUT_OF_RESOURCES) { dev_err(gsi->dev, "channel %u out of resources\n", channel_id); - complete(&gsi->channel[channel_id].completion); + complete(&gsi->completion); return; } @@ -1127,7 +1119,7 @@ gsi_isr_glob_evt_err(struct gsi *gsi, u32 err_ee, u32 evt_ring_id, u32 code) struct gsi_evt_ring *evt_ring = &gsi->evt_ring[evt_ring_id]; u32 channel_id = gsi_channel_id(evt_ring->channel); - complete(&evt_ring->completion); + complete(&gsi->completion); dev_err(gsi->dev, "evt_ring for channel %u out of resources\n", channel_id); return; @@ -1171,18 +1163,23 @@ static void gsi_isr_gp_int1(struct gsi *gsi) u32 result; u32 val; - /* This interrupt is used to handle completions of the two GENERIC - * GSI commands. We use these to allocate and halt channels on - * the modem's behalf due to a hardware quirk on IPA v4.2. Once - * allocated, the modem "owns" these channels, and as a result we - * have no way of knowing the channel's state at any given time. + /* This interrupt is used to handle completions of GENERIC GSI + * commands. We use these to allocate and halt channels on the + * modem's behalf due to a hardware quirk on IPA v4.2. The modem + * "owns" channels even when the AP allocates them, and have no + * way of knowing whether a modem channel's state has been changed. + * + * We also use GENERIC commands to enable/disable channel flow + * control for IPA v4.2+. * * It is recommended that we halt the modem channels we allocated * when shutting down, but it's possible the channel isn't running * at the time we issue the HALT command. We'll get an error in * that case, but it's harmless (the channel is already halted). + * Similarly, we could get an error back when updating flow control + * on a channel because it's not in the proper state. * - * For this reason, we silently ignore a CHANNEL_NOT_RUNNING error + * In either case, we silently ignore a CHANNEL_NOT_RUNNING error * if we receive it. */ val = ioread32(gsi->virt + GSI_CNTXT_SCRATCH_0_OFFSET); @@ -1648,19 +1645,25 @@ static void gsi_channel_teardown_one(struct gsi *gsi, u32 channel_id) gsi_evt_ring_de_alloc_command(gsi, evt_ring_id); } +/* We use generic commands only to operate on modem channels. We don't have + * the ability to determine channel state for a modem channel, so we simply + * issue the command and wait for it to complete. + */ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, - enum gsi_generic_cmd_opcode opcode) + enum gsi_generic_cmd_opcode opcode, + u8 params) { - struct completion *completion = &gsi->completion; bool timeout; u32 val; - /* The error global interrupt type is always enabled (until we - * teardown), so we won't change that. A generic EE command - * completes with a GSI global interrupt of type GP_INT1. We - * only perform one generic command at a time (to allocate or - * halt a modem channel) and only from this function. So we - * enable the GP_INT1 IRQ type here while we're expecting it. + /* The error global interrupt type is always enabled (until we tear + * down), so we will keep it enabled. + * + * A generic EE command completes with a GSI global interrupt of + * type GP_INT1. We only perform one generic command at a time + * (to allocate, halt, or enable/disable flow control on a modem + * channel), and only from this function. So we enable the GP_INT1 + * IRQ type here, and disable it again after the command completes. */ val = BIT(ERROR_INT) | BIT(GP_INT1); iowrite32(val, gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); @@ -1674,8 +1677,9 @@ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, val = u32_encode_bits(opcode, GENERIC_OPCODE_FMASK); val |= u32_encode_bits(channel_id, GENERIC_CHID_FMASK); val |= u32_encode_bits(GSI_EE_MODEM, GENERIC_EE_FMASK); + val |= u32_encode_bits(params, GENERIC_PARAMS_FMASK); - timeout = !gsi_command(gsi, GSI_GENERIC_CMD_OFFSET, val, completion); + timeout = !gsi_command(gsi, GSI_GENERIC_CMD_OFFSET, val); /* Disable the GP_INT1 IRQ type again */ iowrite32(BIT(ERROR_INT), gsi->virt + GSI_CNTXT_GLOB_IRQ_EN_OFFSET); @@ -1692,7 +1696,7 @@ static int gsi_generic_command(struct gsi *gsi, u32 channel_id, static int gsi_modem_channel_alloc(struct gsi *gsi, u32 channel_id) { return gsi_generic_command(gsi, channel_id, - GSI_GENERIC_ALLOCATE_CHANNEL); + GSI_GENERIC_ALLOCATE_CHANNEL, 0); } static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) @@ -1702,7 +1706,7 @@ static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) do ret = gsi_generic_command(gsi, channel_id, - GSI_GENERIC_HALT_CHANNEL); + GSI_GENERIC_HALT_CHANNEL, 0); while (ret == -EAGAIN && retries--); if (ret) @@ -1710,6 +1714,32 @@ static void gsi_modem_channel_halt(struct gsi *gsi, u32 channel_id) ret, channel_id); } +/* Enable or disable flow control for a modem GSI TX channel (IPA v4.2+) */ +void +gsi_modem_channel_flow_control(struct gsi *gsi, u32 channel_id, bool enable) +{ + u32 retries = 0; + u32 command; + int ret; + + command = enable ? GSI_GENERIC_ENABLE_FLOW_CONTROL + : GSI_GENERIC_DISABLE_FLOW_CONTROL; + /* Disabling flow control on IPA v4.11+ can return -EAGAIN if enable + * is underway. In this case we need to retry the command. + */ + if (!enable && gsi->version >= IPA_VERSION_4_11) + retries = GSI_CHANNEL_MODEM_FLOW_RETRIES; + + do + ret = gsi_generic_command(gsi, channel_id, command, 0); + while (ret == -EAGAIN && retries--); + + if (ret) + dev_err(gsi->dev, + "error %d %sabling mode channel %u flow control\n", + ret, enable ? "en" : "dis", channel_id); +} + /* Setup function for channels */ static int gsi_channel_setup(struct gsi *gsi) { @@ -1975,18 +2005,6 @@ static void gsi_channel_evt_ring_exit(struct gsi_channel *channel) gsi_evt_ring_id_free(gsi, evt_ring_id); } -/* Init function for event rings; there is no gsi_evt_ring_exit() */ -static void gsi_evt_ring_init(struct gsi *gsi) -{ - u32 evt_ring_id = 0; - - gsi->event_bitmap = gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX); - gsi->ieob_enabled_bitmap = 0; - do - init_completion(&gsi->evt_ring[evt_ring_id].completion); - while (++evt_ring_id < GSI_EVT_RING_COUNT_MAX); -} - static bool gsi_channel_data_valid(struct gsi *gsi, const struct ipa_gsi_endpoint_data *data) { @@ -2069,7 +2087,6 @@ static int gsi_channel_init_one(struct gsi *gsi, channel->tlv_count = data->channel.tlv_count; channel->tre_count = tre_count; channel->event_count = data->channel.event_count; - init_completion(&channel->completion); ret = gsi_channel_evt_ring_init(channel); if (ret) @@ -2129,7 +2146,8 @@ static int gsi_channel_init(struct gsi *gsi, u32 count, /* IPA v4.2 requires the AP to allocate channels for the modem */ modem_alloc = gsi->version == IPA_VERSION_4_2; - gsi_evt_ring_init(gsi); /* No matching exit required */ + gsi->event_bitmap = gsi_event_bitmap_init(GSI_EVT_RING_COUNT_MAX); + gsi->ieob_enabled_bitmap = 0; /* The endpoint data array is indexed by endpoint name */ for (i = 0; i < count; i++) { diff --git a/drivers/net/ipa/gsi.h b/drivers/net/ipa/gsi.h index 88b80dc3db79..9cc657658811 100644 --- a/drivers/net/ipa/gsi.h +++ b/drivers/net/ipa/gsi.h @@ -101,6 +101,7 @@ enum gsi_channel_state { GSI_CHANNEL_STATE_STARTED = 0x2, GSI_CHANNEL_STATE_STOPPED = 0x3, GSI_CHANNEL_STATE_STOP_IN_PROC = 0x4, + GSI_CHANNEL_STATE_FLOW_CONTROLLED = 0x5, /* IPA v4.2-v4.9 */ GSI_CHANNEL_STATE_ERROR = 0xf, }; @@ -114,8 +115,6 @@ struct gsi_channel { u16 tre_count; u16 event_count; - struct completion completion; /* signals channel command completion */ - struct gsi_ring tre_ring; u32 evt_ring_id; @@ -141,28 +140,27 @@ enum gsi_evt_ring_state { struct gsi_evt_ring { struct gsi_channel *channel; - struct completion completion; /* signals event ring state changes */ struct gsi_ring ring; }; struct gsi { struct device *dev; /* Same as IPA device */ enum ipa_version version; - struct net_device dummy_dev; /* needed for NAPI */ void __iomem *virt_raw; /* I/O mapped address range */ void __iomem *virt; /* Adjusted for most registers */ u32 irq; u32 channel_count; u32 evt_ring_count; - struct gsi_channel channel[GSI_CHANNEL_COUNT_MAX]; - struct gsi_evt_ring evt_ring[GSI_EVT_RING_COUNT_MAX]; u32 event_bitmap; /* allocated event rings */ u32 modem_channel_bitmap; /* modem channels to allocate */ u32 type_enabled_bitmap; /* GSI IRQ types enabled */ u32 ieob_enabled_bitmap; /* IEOB IRQ enabled (event rings) */ - struct completion completion; /* for global EE commands */ int result; /* Negative errno (generic commands) */ + struct completion completion; /* Signals GSI command completion */ struct mutex mutex; /* protects commands, programming */ + struct gsi_channel channel[GSI_CHANNEL_COUNT_MAX]; + struct gsi_evt_ring evt_ring[GSI_EVT_RING_COUNT_MAX]; + struct net_device dummy_dev; /* needed for NAPI */ }; /** @@ -219,6 +217,15 @@ int gsi_channel_start(struct gsi *gsi, u32 channel_id); int gsi_channel_stop(struct gsi *gsi, u32 channel_id); /** + * gsi_modem_channel_flow_control() - Set channel flow control state (IPA v4.2+) + * @gsi: GSI pointer returned by gsi_setup() + * @channel_id: Modem TX channel to control + * @enable: Whether to enable flow control (i.e., prevent flow) + */ +void gsi_modem_channel_flow_control(struct gsi *gsi, u32 channel_id, + bool enable); + +/** * gsi_channel_reset() - Reset an allocated GSI channel * @gsi: GSI pointer * @channel_id: Channel to be reset diff --git a/drivers/net/ipa/gsi_reg.h b/drivers/net/ipa/gsi_reg.h index bf9593d9eaea..8906f4381032 100644 --- a/drivers/net/ipa/gsi_reg.h +++ b/drivers/net/ipa/gsi_reg.h @@ -313,11 +313,15 @@ enum gsi_evt_cmd_opcode { #define GENERIC_OPCODE_FMASK GENMASK(4, 0) #define GENERIC_CHID_FMASK GENMASK(9, 5) #define GENERIC_EE_FMASK GENMASK(13, 10) +#define GENERIC_PARAMS_FMASK GENMASK(31, 24) /* IPA v4.11+ */ /** enum gsi_generic_cmd_opcode - GENERIC_OPCODE field values in GENERIC_CMD */ enum gsi_generic_cmd_opcode { GSI_GENERIC_HALT_CHANNEL = 0x1, GSI_GENERIC_ALLOCATE_CHANNEL = 0x2, + GSI_GENERIC_ENABLE_FLOW_CONTROL = 0x3, /* IPA v4.2+ */ + GSI_GENERIC_DISABLE_FLOW_CONTROL = 0x4, /* IPA v4.2+ */ + GSI_GENERIC_QUERY_FLOW_CONTROL = 0x5, /* IPA v4.11+ */ }; /* The next register is present for IPA v3.5.1 and above */ diff --git a/drivers/net/ipa/ipa_endpoint.c b/drivers/net/ipa/ipa_endpoint.c index 03a170993420..49d9a077d037 100644 --- a/drivers/net/ipa/ipa_endpoint.c +++ b/drivers/net/ipa/ipa_endpoint.c @@ -237,7 +237,8 @@ static struct gsi_trans *ipa_endpoint_trans_alloc(struct ipa_endpoint *endpoint, } /* suspend_delay represents suspend for RX, delay for TX endpoints. - * Note that suspend is not supported starting with IPA v4.0. + * Note that suspend is not supported starting with IPA v4.0, and + * delay mode should not be used starting with IPA v4.2. */ static bool ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) @@ -248,11 +249,8 @@ ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) u32 mask; u32 val; - /* Suspend is not supported for IPA v4.0+. Delay doesn't work - * correctly on IPA v4.2. - */ if (endpoint->toward_ipa) - WARN_ON(ipa->version == IPA_VERSION_4_2); + WARN_ON(ipa->version >= IPA_VERSION_4_2); else WARN_ON(ipa->version >= IPA_VERSION_4_0); @@ -270,15 +268,15 @@ ipa_endpoint_init_ctrl(struct ipa_endpoint *endpoint, bool suspend_delay) return state; } -/* We currently don't care what the previous state was for delay mode */ +/* We don't care what the previous state was for delay mode */ static void ipa_endpoint_program_delay(struct ipa_endpoint *endpoint, bool enable) { + /* Delay mode should not be used for IPA v4.2+ */ + WARN_ON(endpoint->ipa->version >= IPA_VERSION_4_2); WARN_ON(!endpoint->toward_ipa); - /* Delay mode doesn't work properly for IPA v4.2 */ - if (endpoint->ipa->version != IPA_VERSION_4_2) - (void)ipa_endpoint_init_ctrl(endpoint, enable); + (void)ipa_endpoint_init_ctrl(endpoint, enable); } static bool ipa_endpoint_aggr_active(struct ipa_endpoint *endpoint) @@ -355,26 +353,29 @@ ipa_endpoint_program_suspend(struct ipa_endpoint *endpoint, bool enable) return suspended; } -/* Enable or disable delay or suspend mode on all modem endpoints */ +/* Put all modem RX endpoints into suspend mode, and stop transmission + * on all modem TX endpoints. Prior to IPA v4.2, endpoint DELAY mode is + * used for TX endpoints; starting with IPA v4.2 we use GSI channel flow + * control instead. + */ void ipa_endpoint_modem_pause_all(struct ipa *ipa, bool enable) { u32 endpoint_id; - /* DELAY mode doesn't work correctly on IPA v4.2 */ - if (ipa->version == IPA_VERSION_4_2) - return; - for (endpoint_id = 0; endpoint_id < IPA_ENDPOINT_MAX; endpoint_id++) { struct ipa_endpoint *endpoint = &ipa->endpoint[endpoint_id]; if (endpoint->ee_id != GSI_EE_MODEM) continue; - /* Set TX delay mode or RX suspend mode */ - if (endpoint->toward_ipa) + if (!endpoint->toward_ipa) + (void)ipa_endpoint_program_suspend(endpoint, enable); + else if (ipa->version < IPA_VERSION_4_2) ipa_endpoint_program_delay(endpoint, enable); else - (void)ipa_endpoint_program_suspend(endpoint, enable); + gsi_modem_channel_flow_control(&ipa->gsi, + endpoint->channel_id, + enable); } } @@ -860,7 +861,7 @@ static void ipa_endpoint_init_hol_block_timer(struct ipa_endpoint *endpoint, } static void -ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, bool enable) +ipa_endpoint_init_hol_block_en(struct ipa_endpoint *endpoint, bool enable) { u32 endpoint_id = endpoint->endpoint_id; u32 offset; @@ -874,6 +875,19 @@ ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, bool enable) iowrite32(val, endpoint->ipa->reg_virt + offset); } +/* Assumes HOL_BLOCK is in disabled state */ +static void ipa_endpoint_init_hol_block_enable(struct ipa_endpoint *endpoint, + u32 microseconds) +{ + ipa_endpoint_init_hol_block_timer(endpoint, microseconds); + ipa_endpoint_init_hol_block_en(endpoint, true); +} + +static void ipa_endpoint_init_hol_block_disable(struct ipa_endpoint *endpoint) +{ + ipa_endpoint_init_hol_block_en(endpoint, false); +} + void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) { u32 i; @@ -884,9 +898,8 @@ void ipa_endpoint_modem_hol_block_clear_all(struct ipa *ipa) if (endpoint->toward_ipa || endpoint->ee_id != GSI_EE_MODEM) continue; - ipa_endpoint_init_hol_block_enable(endpoint, false); - ipa_endpoint_init_hol_block_timer(endpoint, 0); - ipa_endpoint_init_hol_block_enable(endpoint, true); + ipa_endpoint_init_hol_block_disable(endpoint); + ipa_endpoint_init_hol_block_enable(endpoint, 0); } } @@ -1141,18 +1154,19 @@ static void ipa_endpoint_skb_copy(struct ipa_endpoint *endpoint, { struct sk_buff *skb; + if (!endpoint->netdev) + return; + skb = __dev_alloc_skb(len, GFP_ATOMIC); - if (skb) { - skb_put(skb, len); - memcpy(skb->data, data, len); - skb->truesize += extra; - } + if (!skb) + return; + + /* Copy the data into the socket buffer and receive it */ + skb_put(skb, len); + memcpy(skb->data, data, len); + skb->truesize += extra; - /* Now receive it, or drop it if there's no netdev */ - if (endpoint->netdev) - ipa_modem_skb_rx(endpoint->netdev, skb); - else if (skb) - dev_kfree_skb_any(skb); + ipa_modem_skb_rx(endpoint->netdev, skb); } static bool ipa_endpoint_skb_build(struct ipa_endpoint *endpoint, @@ -1519,10 +1533,19 @@ static void ipa_endpoint_reset(struct ipa_endpoint *endpoint) static void ipa_endpoint_program(struct ipa_endpoint *endpoint) { - if (endpoint->toward_ipa) - ipa_endpoint_program_delay(endpoint, false); - else + if (endpoint->toward_ipa) { + /* Newer versions of IPA use GSI channel flow control + * instead of endpoint DELAY mode to prevent sending data. + * Flow control is disabled for newly-allocated channels, + * and we can assume flow control is not (ever) enabled + * for AP TX channels. + */ + if (endpoint->ipa->version < IPA_VERSION_4_2) + ipa_endpoint_program_delay(endpoint, false); + } else { + /* Ensure suspend mode is off on all AP RX endpoints */ (void)ipa_endpoint_program_suspend(endpoint, false); + } ipa_endpoint_init_cfg(endpoint); ipa_endpoint_init_nat(endpoint); ipa_endpoint_init_hdr(endpoint); @@ -1530,6 +1553,8 @@ static void ipa_endpoint_program(struct ipa_endpoint *endpoint) ipa_endpoint_init_hdr_metadata_mask(endpoint); ipa_endpoint_init_mode(endpoint); ipa_endpoint_init_aggr(endpoint); + if (!endpoint->toward_ipa) + ipa_endpoint_init_hol_block_disable(endpoint); ipa_endpoint_init_deaggr(endpoint); ipa_endpoint_init_rsrc_grp(endpoint); ipa_endpoint_init_seq(endpoint); diff --git a/drivers/net/ipa/ipa_main.c b/drivers/net/ipa/ipa_main.c index a448ec198bee..3757ce3de2c5 100644 --- a/drivers/net/ipa/ipa_main.c +++ b/drivers/net/ipa/ipa_main.c @@ -734,7 +734,7 @@ static int ipa_probe(struct platform_device *pdev) if (ret) goto err_endpoint_exit; - ret = ipa_modem_init(ipa, modem_init); + ret = ipa_smp2p_init(ipa, modem_init); if (ret) goto err_table_exit; @@ -776,7 +776,7 @@ err_deconfig: ipa_deconfig(ipa); err_power_put: pm_runtime_put_noidle(dev); - ipa_modem_exit(ipa); + ipa_smp2p_exit(ipa); err_table_exit: ipa_table_exit(ipa); err_endpoint_exit: @@ -827,7 +827,7 @@ static int ipa_remove(struct platform_device *pdev) ipa_deconfig(ipa); out_power_put: pm_runtime_put_noidle(dev); - ipa_modem_exit(ipa); + ipa_smp2p_exit(ipa); ipa_table_exit(ipa); ipa_endpoint_exit(ipa); gsi_exit(&ipa->gsi); diff --git a/drivers/net/ipa/ipa_mem.c b/drivers/net/ipa/ipa_mem.c index 4337b0920d3d..1e9eae208e44 100644 --- a/drivers/net/ipa/ipa_mem.c +++ b/drivers/net/ipa/ipa_mem.c @@ -266,9 +266,7 @@ static bool ipa_mem_valid(struct ipa *ipa, const struct ipa_mem_data *mem_data) } /* Now see if any required regions are not defined */ - for (mem_id = find_first_zero_bit(regions, IPA_MEM_COUNT); - mem_id < IPA_MEM_COUNT; - mem_id = find_next_zero_bit(regions, IPA_MEM_COUNT, mem_id + 1)) { + for_each_clear_bit(mem_id, regions, IPA_MEM_COUNT) { if (ipa_mem_id_required(ipa, mem_id)) dev_err(dev, "required memory region %u missing\n", mem_id); diff --git a/drivers/net/ipa/ipa_modem.c b/drivers/net/ipa/ipa_modem.c index d0ab4d70c303..27d87097433f 100644 --- a/drivers/net/ipa/ipa_modem.c +++ b/drivers/net/ipa/ipa_modem.c @@ -442,16 +442,6 @@ static int ipa_modem_notify(struct notifier_block *nb, unsigned long action, return NOTIFY_OK; } -int ipa_modem_init(struct ipa *ipa, bool modem_init) -{ - return ipa_smp2p_init(ipa, modem_init); -} - -void ipa_modem_exit(struct ipa *ipa) -{ - ipa_smp2p_exit(ipa); -} - int ipa_modem_config(struct ipa *ipa) { void *notifier; diff --git a/drivers/net/ipa/ipa_modem.h b/drivers/net/ipa/ipa_modem.h index 5e6e3d234454..e64ccc2402e9 100644 --- a/drivers/net/ipa/ipa_modem.h +++ b/drivers/net/ipa/ipa_modem.h @@ -18,9 +18,6 @@ void ipa_modem_skb_rx(struct net_device *netdev, struct sk_buff *skb); void ipa_modem_suspend(struct net_device *netdev); void ipa_modem_resume(struct net_device *netdev); -int ipa_modem_init(struct ipa *ipa, bool modem_init); -void ipa_modem_exit(struct ipa *ipa); - int ipa_modem_config(struct ipa *ipa); void ipa_modem_deconfig(struct ipa *ipa); diff --git a/drivers/net/ipa/ipa_table.c b/drivers/net/ipa/ipa_table.c index 1da334f54944..2f5a58bfc529 100644 --- a/drivers/net/ipa/ipa_table.c +++ b/drivers/net/ipa/ipa_table.c @@ -419,21 +419,26 @@ static void ipa_table_init_add(struct gsi_trans *trans, bool filter, const struct ipa_mem *mem = ipa_mem_find(ipa, mem_id); dma_addr_t hash_addr; dma_addr_t addr; + u32 zero_offset; u16 hash_count; + u32 zero_size; u16 hash_size; u16 count; u16 size; - /* The number of filtering endpoints determines number of entries - * in the filter table. The hashed and non-hashed filter table - * will have the same number of entries. The size of the route - * table region determines the number of entries it has. - */ + /* Compute the number of table entries to initialize */ if (filter) { - /* Include one extra "slot" to hold the filter map itself */ + /* The number of filtering endpoints determines number of + * entries in the filter table; we also add one more "slot" + * to hold the bitmap itself. The size of the hashed filter + * table is either the same as the non-hashed one, or zero. + */ count = 1 + hweight32(ipa->filter_map); hash_count = hash_mem->size ? count : 0; } else { + /* The size of a route table region determines the number + * of entries it has. + */ count = mem->size / sizeof(__le64); hash_count = hash_mem->size / sizeof(__le64); } @@ -445,13 +450,42 @@ static void ipa_table_init_add(struct gsi_trans *trans, bool filter, ipa_cmd_table_init_add(trans, opcode, size, mem->offset, addr, hash_size, hash_mem->offset, hash_addr); + if (!filter) + return; + + /* Zero the unused space in the filter table */ + zero_offset = mem->offset + size; + zero_size = mem->size - size; + ipa_cmd_dma_shared_mem_add(trans, zero_offset, zero_size, + ipa->zero_addr, true); + if (!hash_size) + return; + + /* Zero the unused space in the hashed filter table */ + zero_offset = hash_mem->offset + hash_size; + zero_size = hash_mem->size - hash_size; + ipa_cmd_dma_shared_mem_add(trans, zero_offset, zero_size, + ipa->zero_addr, true); } int ipa_table_setup(struct ipa *ipa) { struct gsi_trans *trans; - trans = ipa_cmd_trans_alloc(ipa, 4); + /* We will need at most 8 TREs: + * - IPv4: + * - One for route table initialization (non-hashed and hashed) + * - One for filter table initialization (non-hashed and hashed) + * - One to zero unused entries in the non-hashed filter table + * - One to zero unused entries in the hashed filter table + * - IPv6: + * - One for route table initialization (non-hashed and hashed) + * - One for filter table initialization (non-hashed and hashed) + * - One to zero unused entries in the non-hashed filter table + * - One to zero unused entries in the hashed filter table + * All platforms support at least 8 TREs in a transaction. + */ + trans = ipa_cmd_trans_alloc(ipa, 8); if (!trans) { dev_err(&ipa->pdev->dev, "no transaction for table setup\n"); return -EBUSY; diff --git a/drivers/net/ipvlan/ipvlan_core.c b/drivers/net/ipvlan/ipvlan_core.c index 6cd50106e611..c613900c3811 100644 --- a/drivers/net/ipvlan/ipvlan_core.c +++ b/drivers/net/ipvlan/ipvlan_core.c @@ -291,8 +291,7 @@ void ipvlan_process_multicast(struct work_struct *work) else kfree_skb(skb); } - if (dev) - dev_put(dev); + dev_put(dev); cond_resched(); } } diff --git a/drivers/net/ipvlan/ipvlan_main.c b/drivers/net/ipvlan/ipvlan_main.c index 1d2f4e7d7324..696e245f6d00 100644 --- a/drivers/net/ipvlan/ipvlan_main.c +++ b/drivers/net/ipvlan/ipvlan_main.c @@ -100,8 +100,7 @@ static void ipvlan_port_destroy(struct net_device *dev) netdev_rx_handler_unregister(dev); cancel_work_sync(&port->wq); while ((skb = __skb_dequeue(&port->backlog)) != NULL) { - if (skb->dev) - dev_put(skb->dev); + dev_put(skb->dev); kfree_skb(skb); } ida_destroy(&port->ida); @@ -140,8 +139,8 @@ static int ipvlan_init(struct net_device *dev) dev->vlan_features = phy_dev->vlan_features & IPVLAN_FEATURES; dev->vlan_features |= IPVLAN_ALWAYS_ON_OFLOADS; dev->hw_enc_features |= dev->features; - dev->gso_max_size = phy_dev->gso_max_size; - dev->gso_max_segs = phy_dev->gso_max_segs; + netif_set_gso_max_size(dev, phy_dev->gso_max_size); + netif_set_gso_max_segs(dev, phy_dev->gso_max_segs); dev->hard_header_len = phy_dev->hard_header_len; netdev_lockdep_set_classes(dev); @@ -763,8 +762,8 @@ static int ipvlan_device_event(struct notifier_block *unused, case NETDEV_FEAT_CHANGE: list_for_each_entry(ipvlan, &port->ipvlans, pnode) { - ipvlan->dev->gso_max_size = dev->gso_max_size; - ipvlan->dev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_size(ipvlan->dev, dev->gso_max_size); + netif_set_gso_max_segs(ipvlan->dev, dev->gso_max_segs); netdev_update_features(ipvlan->dev); } break; diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index d2f830ec2969..6ef5f77be4d0 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -900,8 +900,8 @@ static int macvlan_init(struct net_device *dev) dev->vlan_features = lowerdev->vlan_features & MACVLAN_FEATURES; dev->vlan_features |= ALWAYS_ON_OFFLOADS; dev->hw_enc_features |= dev->features; - dev->gso_max_size = lowerdev->gso_max_size; - dev->gso_max_segs = lowerdev->gso_max_segs; + netif_set_gso_max_size(dev, lowerdev->gso_max_size); + netif_set_gso_max_segs(dev, lowerdev->gso_max_segs); dev->hard_header_len = lowerdev->hard_header_len; macvlan_set_lockdep_class(dev); @@ -1171,7 +1171,6 @@ static const struct net_device_ops macvlan_netdev_ops = { #endif .ndo_get_iflink = macvlan_dev_get_iflink, .ndo_features_check = passthru_features_check, - .ndo_change_proto_down = dev_change_proto_down_generic, }; void macvlan_common_setup(struct net_device *dev) @@ -1182,7 +1181,7 @@ void macvlan_common_setup(struct net_device *dev) dev->max_mtu = ETH_MAX_MTU; dev->priv_flags &= ~IFF_TX_SKB_SHARING; netif_keep_dst(dev); - dev->priv_flags |= IFF_UNICAST_FLT; + dev->priv_flags |= IFF_UNICAST_FLT | IFF_CHANGE_PROTO_DOWN; dev->netdev_ops = &macvlan_netdev_ops; dev->needs_free_netdev = true; dev->header_ops = &macvlan_hard_header_ops; @@ -1748,8 +1747,8 @@ static int macvlan_device_event(struct notifier_block *unused, break; case NETDEV_FEAT_CHANGE: list_for_each_entry(vlan, &port->vlans, list) { - vlan->dev->gso_max_size = dev->gso_max_size; - vlan->dev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_size(vlan->dev, dev->gso_max_size); + netif_set_gso_max_segs(vlan->dev, dev->gso_max_segs); netdev_update_features(vlan->dev); } break; diff --git a/drivers/net/mctp/Kconfig b/drivers/net/mctp/Kconfig index d8f966cedc89..2929471395ae 100644 --- a/drivers/net/mctp/Kconfig +++ b/drivers/net/mctp/Kconfig @@ -3,6 +3,24 @@ if MCTP menu "MCTP Device Drivers" +config MCTP_SERIAL + tristate "MCTP serial transport" + depends on TTY + select CRC_CCITT + help + This driver provides an MCTP-over-serial interface, through a + serial line-discipline, as defined by DMTF specification "DSP0253 - + MCTP Serial Transport Binding". By attaching the ldisc to a serial + device, we get a new net device to transport MCTP packets. + + This allows communication with external MCTP endpoints which use + serial as their transport. It can also be used as an easy way to + provide MCTP connectivity between virtual machines, by forwarding + data between simple virtual serial devices. + + Say y here if you need to connect to MCTP endpoints over serial. To + compile as a module, use m; the module will be called mctp-serial. + endmenu endif diff --git a/drivers/net/mctp/Makefile b/drivers/net/mctp/Makefile index e69de29bb2d1..d32622613ce4 100644 --- a/drivers/net/mctp/Makefile +++ b/drivers/net/mctp/Makefile @@ -0,0 +1 @@ +obj-$(CONFIG_MCTP_SERIAL) += mctp-serial.o diff --git a/drivers/net/mctp/mctp-serial.c b/drivers/net/mctp/mctp-serial.c new file mode 100644 index 000000000000..eaa6fb3224bc --- /dev/null +++ b/drivers/net/mctp/mctp-serial.c @@ -0,0 +1,515 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Management Component Transport Protocol (MCTP) - serial transport + * binding. This driver is an implementation of the DMTF specificiation + * "DSP0253 - Management Component Transport Protocol (MCTP) Serial Transport + * Binding", available at: + * + * https://www.dmtf.org/sites/default/files/standards/documents/DSP0253_1.0.0.pdf + * + * This driver provides DSP0253-type MCTP-over-serial transport using a Linux + * tty device, by setting the N_MCTP line discipline on the tty. + * + * Copyright (c) 2021 Code Construct + */ + +#include <linux/idr.h> +#include <linux/if_arp.h> +#include <linux/module.h> +#include <linux/skbuff.h> +#include <linux/tty.h> +#include <linux/workqueue.h> +#include <linux/crc-ccitt.h> + +#include <linux/mctp.h> +#include <net/mctp.h> +#include <net/pkt_sched.h> + +#define MCTP_SERIAL_MTU 68 /* base mtu (64) + mctp header */ +#define MCTP_SERIAL_FRAME_MTU (MCTP_SERIAL_MTU + 6) /* + serial framing */ + +#define MCTP_SERIAL_VERSION 0x1 /* DSP0253 defines a single version: 1 */ + +#define BUFSIZE MCTP_SERIAL_FRAME_MTU + +#define BYTE_FRAME 0x7e +#define BYTE_ESC 0x7d + +static DEFINE_IDA(mctp_serial_ida); + +enum mctp_serial_state { + STATE_IDLE, + STATE_START, + STATE_HEADER, + STATE_DATA, + STATE_ESCAPE, + STATE_TRAILER, + STATE_DONE, + STATE_ERR, +}; + +struct mctp_serial { + struct net_device *netdev; + struct tty_struct *tty; + + int idx; + + /* protects our rx & tx state machines; held during both paths */ + spinlock_t lock; + + struct work_struct tx_work; + enum mctp_serial_state txstate, rxstate; + u16 txfcs, rxfcs, rxfcs_rcvd; + unsigned int txlen, rxlen; + unsigned int txpos, rxpos; + unsigned char txbuf[BUFSIZE], + rxbuf[BUFSIZE]; +}; + +static bool needs_escape(unsigned char c) +{ + return c == BYTE_ESC || c == BYTE_FRAME; +} + +static int next_chunk_len(struct mctp_serial *dev) +{ + int i; + + /* either we have no bytes to send ... */ + if (dev->txpos == dev->txlen) + return 0; + + /* ... or the next byte to send is an escaped byte; requiring a + * single-byte chunk... + */ + if (needs_escape(dev->txbuf[dev->txpos])) + return 1; + + /* ... or we have one or more bytes up to the next escape - this chunk + * will be those non-escaped bytes, and does not include the escaped + * byte. + */ + for (i = 1; i + dev->txpos + 1 < dev->txlen; i++) { + if (needs_escape(dev->txbuf[dev->txpos + i + 1])) + break; + } + + return i; +} + +static int write_chunk(struct mctp_serial *dev, unsigned char *buf, int len) +{ + return dev->tty->ops->write(dev->tty, buf, len); +} + +static void mctp_serial_tx_work(struct work_struct *work) +{ + struct mctp_serial *dev = container_of(work, struct mctp_serial, + tx_work); + unsigned char c, buf[3]; + unsigned long flags; + int len, txlen; + + spin_lock_irqsave(&dev->lock, flags); + + /* txstate represents the next thing to send */ + switch (dev->txstate) { + case STATE_START: + dev->txpos = 0; + fallthrough; + case STATE_HEADER: + buf[0] = BYTE_FRAME; + buf[1] = MCTP_SERIAL_VERSION; + buf[2] = dev->txlen; + + if (!dev->txpos) + dev->txfcs = crc_ccitt(0, buf + 1, 2); + + txlen = write_chunk(dev, buf + dev->txpos, 3 - dev->txpos); + if (txlen <= 0) { + dev->txstate = STATE_ERR; + } else { + dev->txpos += txlen; + if (dev->txpos == 3) { + dev->txstate = STATE_DATA; + dev->txpos = 0; + } + } + break; + + case STATE_ESCAPE: + buf[0] = dev->txbuf[dev->txpos] & ~0x20; + txlen = write_chunk(dev, buf, 1); + if (txlen <= 0) { + dev->txstate = STATE_ERR; + } else { + dev->txpos += txlen; + if (dev->txpos == dev->txlen) { + dev->txstate = STATE_TRAILER; + dev->txpos = 0; + } + } + + break; + + case STATE_DATA: + len = next_chunk_len(dev); + if (len) { + c = dev->txbuf[dev->txpos]; + if (len == 1 && needs_escape(c)) { + buf[0] = BYTE_ESC; + buf[1] = c & ~0x20; + dev->txfcs = crc_ccitt_byte(dev->txfcs, c); + txlen = write_chunk(dev, buf, 2); + if (txlen == 2) + dev->txpos++; + else if (txlen == 1) + dev->txstate = STATE_ESCAPE; + else + dev->txstate = STATE_ERR; + } else { + txlen = write_chunk(dev, + dev->txbuf + dev->txpos, + len); + if (txlen <= 0) { + dev->txstate = STATE_ERR; + } else { + dev->txfcs = crc_ccitt(dev->txfcs, + dev->txbuf + + dev->txpos, + txlen); + dev->txpos += txlen; + } + } + if (dev->txstate == STATE_DATA && + dev->txpos == dev->txlen) { + dev->txstate = STATE_TRAILER; + dev->txpos = 0; + } + break; + } + dev->txstate = STATE_TRAILER; + dev->txpos = 0; + fallthrough; + + case STATE_TRAILER: + buf[0] = dev->txfcs >> 8; + buf[1] = dev->txfcs & 0xff; + buf[2] = BYTE_FRAME; + txlen = write_chunk(dev, buf + dev->txpos, 3 - dev->txpos); + if (txlen <= 0) { + dev->txstate = STATE_ERR; + } else { + dev->txpos += txlen; + if (dev->txpos == 3) { + dev->txstate = STATE_DONE; + dev->txpos = 0; + } + } + break; + default: + netdev_err_once(dev->netdev, "invalid tx state %d\n", + dev->txstate); + } + + if (dev->txstate == STATE_DONE) { + dev->netdev->stats.tx_packets++; + dev->netdev->stats.tx_bytes += dev->txlen; + dev->txlen = 0; + dev->txpos = 0; + clear_bit(TTY_DO_WRITE_WAKEUP, &dev->tty->flags); + dev->txstate = STATE_IDLE; + spin_unlock_irqrestore(&dev->lock, flags); + + netif_wake_queue(dev->netdev); + } else { + spin_unlock_irqrestore(&dev->lock, flags); + } +} + +static netdev_tx_t mctp_serial_tx(struct sk_buff *skb, struct net_device *ndev) +{ + struct mctp_serial *dev = netdev_priv(ndev); + unsigned long flags; + + WARN_ON(dev->txstate != STATE_IDLE); + + if (skb->len > MCTP_SERIAL_MTU) { + dev->netdev->stats.tx_dropped++; + goto out; + } + + spin_lock_irqsave(&dev->lock, flags); + netif_stop_queue(dev->netdev); + skb_copy_bits(skb, 0, dev->txbuf, skb->len); + dev->txpos = 0; + dev->txlen = skb->len; + dev->txstate = STATE_START; + spin_unlock_irqrestore(&dev->lock, flags); + + set_bit(TTY_DO_WRITE_WAKEUP, &dev->tty->flags); + schedule_work(&dev->tx_work); + +out: + kfree_skb(skb); + return NETDEV_TX_OK; +} + +static void mctp_serial_tty_write_wakeup(struct tty_struct *tty) +{ + struct mctp_serial *dev = tty->disc_data; + + schedule_work(&dev->tx_work); +} + +static void mctp_serial_rx(struct mctp_serial *dev) +{ + struct mctp_skb_cb *cb; + struct sk_buff *skb; + + if (dev->rxfcs != dev->rxfcs_rcvd) { + dev->netdev->stats.rx_dropped++; + dev->netdev->stats.rx_crc_errors++; + return; + } + + skb = netdev_alloc_skb(dev->netdev, dev->rxlen); + if (!skb) { + dev->netdev->stats.rx_dropped++; + return; + } + + skb->protocol = htons(ETH_P_MCTP); + skb_put_data(skb, dev->rxbuf, dev->rxlen); + skb_reset_network_header(skb); + + cb = __mctp_cb(skb); + cb->halen = 0; + + netif_rx_ni(skb); + dev->netdev->stats.rx_packets++; + dev->netdev->stats.rx_bytes += dev->rxlen; +} + +static void mctp_serial_push_header(struct mctp_serial *dev, unsigned char c) +{ + switch (dev->rxpos) { + case 0: + if (c == BYTE_FRAME) + dev->rxpos++; + else + dev->rxstate = STATE_ERR; + break; + case 1: + if (c == MCTP_SERIAL_VERSION) { + dev->rxpos++; + dev->rxfcs = crc_ccitt_byte(0, c); + } else { + dev->rxstate = STATE_ERR; + } + break; + case 2: + if (c > MCTP_SERIAL_FRAME_MTU) { + dev->rxstate = STATE_ERR; + } else { + dev->rxlen = c; + dev->rxpos = 0; + dev->rxstate = STATE_DATA; + dev->rxfcs = crc_ccitt_byte(dev->rxfcs, c); + } + break; + } +} + +static void mctp_serial_push_trailer(struct mctp_serial *dev, unsigned char c) +{ + switch (dev->rxpos) { + case 0: + dev->rxfcs_rcvd = c << 8; + dev->rxpos++; + break; + case 1: + dev->rxfcs_rcvd |= c; + dev->rxpos++; + break; + case 2: + if (c != BYTE_FRAME) { + dev->rxstate = STATE_ERR; + } else { + mctp_serial_rx(dev); + dev->rxlen = 0; + dev->rxpos = 0; + dev->rxstate = STATE_IDLE; + } + break; + } +} + +static void mctp_serial_push(struct mctp_serial *dev, unsigned char c) +{ + switch (dev->rxstate) { + case STATE_IDLE: + dev->rxstate = STATE_HEADER; + fallthrough; + case STATE_HEADER: + mctp_serial_push_header(dev, c); + break; + + case STATE_ESCAPE: + c |= 0x20; + fallthrough; + case STATE_DATA: + if (dev->rxstate != STATE_ESCAPE && c == BYTE_ESC) { + dev->rxstate = STATE_ESCAPE; + } else { + dev->rxfcs = crc_ccitt_byte(dev->rxfcs, c); + dev->rxbuf[dev->rxpos] = c; + dev->rxpos++; + dev->rxstate = STATE_DATA; + if (dev->rxpos == dev->rxlen) { + dev->rxpos = 0; + dev->rxstate = STATE_TRAILER; + } + } + break; + + case STATE_TRAILER: + mctp_serial_push_trailer(dev, c); + break; + + case STATE_ERR: + if (c == BYTE_FRAME) + dev->rxstate = STATE_IDLE; + break; + + default: + netdev_err_once(dev->netdev, "invalid rx state %d\n", + dev->rxstate); + } +} + +static void mctp_serial_tty_receive_buf(struct tty_struct *tty, + const unsigned char *c, + const char *f, int len) +{ + struct mctp_serial *dev = tty->disc_data; + int i; + + if (!netif_running(dev->netdev)) + return; + + /* we don't (currently) use the flag bytes, just data. */ + for (i = 0; i < len; i++) + mctp_serial_push(dev, c[i]); +} + +static const struct net_device_ops mctp_serial_netdev_ops = { + .ndo_start_xmit = mctp_serial_tx, +}; + +static void mctp_serial_setup(struct net_device *ndev) +{ + ndev->type = ARPHRD_MCTP; + + /* we limit at the fixed MTU, which is also the MCTP-standard + * baseline MTU, so is also our minimum + */ + ndev->mtu = MCTP_SERIAL_MTU; + ndev->max_mtu = MCTP_SERIAL_MTU; + ndev->min_mtu = MCTP_SERIAL_MTU; + + ndev->hard_header_len = 0; + ndev->addr_len = 0; + ndev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; + ndev->flags = IFF_NOARP; + ndev->netdev_ops = &mctp_serial_netdev_ops; + ndev->needs_free_netdev = true; +} + +static int mctp_serial_open(struct tty_struct *tty) +{ + struct mctp_serial *dev; + struct net_device *ndev; + char name[32]; + int idx, rc; + + if (!capable(CAP_NET_ADMIN)) + return -EPERM; + + if (!tty->ops->write) + return -EOPNOTSUPP; + + idx = ida_alloc(&mctp_serial_ida, GFP_KERNEL); + if (idx < 0) + return idx; + + snprintf(name, sizeof(name), "mctpserial%d", idx); + ndev = alloc_netdev(sizeof(*dev), name, NET_NAME_ENUM, + mctp_serial_setup); + if (!ndev) { + rc = -ENOMEM; + goto free_ida; + } + + dev = netdev_priv(ndev); + dev->idx = idx; + dev->tty = tty; + dev->netdev = ndev; + dev->txstate = STATE_IDLE; + dev->rxstate = STATE_IDLE; + spin_lock_init(&dev->lock); + INIT_WORK(&dev->tx_work, mctp_serial_tx_work); + + rc = register_netdev(ndev); + if (rc) + goto free_netdev; + + tty->receive_room = 64 * 1024; + tty->disc_data = dev; + + return 0; + +free_netdev: + free_netdev(ndev); + +free_ida: + ida_free(&mctp_serial_ida, idx); + return rc; +} + +static void mctp_serial_close(struct tty_struct *tty) +{ + struct mctp_serial *dev = tty->disc_data; + int idx = dev->idx; + + unregister_netdev(dev->netdev); + cancel_work_sync(&dev->tx_work); + ida_free(&mctp_serial_ida, idx); +} + +static struct tty_ldisc_ops mctp_ldisc = { + .owner = THIS_MODULE, + .num = N_MCTP, + .name = "mctp", + .open = mctp_serial_open, + .close = mctp_serial_close, + .receive_buf = mctp_serial_tty_receive_buf, + .write_wakeup = mctp_serial_tty_write_wakeup, +}; + +static int __init mctp_serial_init(void) +{ + return tty_register_ldisc(&mctp_ldisc); +} + +static void __exit mctp_serial_exit(void) +{ + tty_unregister_ldisc(&mctp_ldisc); +} + +module_init(mctp_serial_init); +module_exit(mctp_serial_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Jeremy Kerr <jk@codeconstruct.com.au>"); +MODULE_DESCRIPTION("MCTP Serial transport"); diff --git a/drivers/net/mdio/Kconfig b/drivers/net/mdio/Kconfig index 6da1fcb25847..bfa16826a6e1 100644 --- a/drivers/net/mdio/Kconfig +++ b/drivers/net/mdio/Kconfig @@ -141,7 +141,7 @@ config MDIO_MVUSB config MDIO_MSCC_MIIM tristate "Microsemi MIIM interface support" - depends on HAS_IOMEM + depends on HAS_IOMEM && REGMAP_MMIO select MDIO_DEVRES help This driver supports the MIIM (MDIO) interface found in the network diff --git a/drivers/net/mdio/mdio-ipq8064.c b/drivers/net/mdio/mdio-ipq8064.c index bd1aea2d5a26..37e0d8b6da07 100644 --- a/drivers/net/mdio/mdio-ipq8064.c +++ b/drivers/net/mdio/mdio-ipq8064.c @@ -127,7 +127,7 @@ ipq8064_mdio_probe(struct platform_device *pdev) if (of_address_to_resource(np, 0, &res)) return -ENOMEM; - base = ioremap(res.start, resource_size(&res)); + base = devm_ioremap(&pdev->dev, res.start, resource_size(&res)); if (!base) return -ENOMEM; diff --git a/drivers/net/mdio/mdio-mscc-miim.c b/drivers/net/mdio/mdio-mscc-miim.c index 17f98f609ec8..7d2abaf2b2c9 100644 --- a/drivers/net/mdio/mdio-mscc-miim.c +++ b/drivers/net/mdio/mdio-mscc-miim.c @@ -10,10 +10,12 @@ #include <linux/io.h> #include <linux/iopoll.h> #include <linux/kernel.h> +#include <linux/mdio/mdio-mscc-miim.h> #include <linux/module.h> #include <linux/of_mdio.h> #include <linux/phy.h> #include <linux/platform_device.h> +#include <linux/regmap.h> #define MSCC_MIIM_REG_STATUS 0x0 #define MSCC_MIIM_STATUS_STAT_PENDING BIT(2) @@ -35,37 +37,52 @@ #define MSCC_PHY_REG_PHY_STATUS 0x4 struct mscc_miim_dev { - void __iomem *regs; - void __iomem *phy_regs; + struct regmap *regs; + int mii_status_offset; + struct regmap *phy_regs; + int phy_reset_offset; }; /* When high resolution timers aren't built-in: we can't use usleep_range() as * we would sleep way too long. Use udelay() instead. */ -#define mscc_readl_poll_timeout(addr, val, cond, delay_us, timeout_us) \ -({ \ - if (!IS_ENABLED(CONFIG_HIGH_RES_TIMERS)) \ - readl_poll_timeout_atomic(addr, val, cond, delay_us, \ - timeout_us); \ - readl_poll_timeout(addr, val, cond, delay_us, timeout_us); \ +#define mscc_readx_poll_timeout(op, addr, val, cond, delay_us, timeout_us)\ +({ \ + if (!IS_ENABLED(CONFIG_HIGH_RES_TIMERS)) \ + readx_poll_timeout_atomic(op, addr, val, cond, delay_us, \ + timeout_us); \ + readx_poll_timeout(op, addr, val, cond, delay_us, timeout_us); \ }) -static int mscc_miim_wait_ready(struct mii_bus *bus) +static int mscc_miim_status(struct mii_bus *bus) { struct mscc_miim_dev *miim = bus->priv; + int val, ret; + + ret = regmap_read(miim->regs, + MSCC_MIIM_REG_STATUS + miim->mii_status_offset, &val); + if (ret < 0) { + WARN_ONCE(1, "mscc miim status read error %d\n", ret); + return ret; + } + + return val; +} + +static int mscc_miim_wait_ready(struct mii_bus *bus) +{ u32 val; - return mscc_readl_poll_timeout(miim->regs + MSCC_MIIM_REG_STATUS, val, + return mscc_readx_poll_timeout(mscc_miim_status, bus, val, !(val & MSCC_MIIM_STATUS_STAT_BUSY), 50, 10000); } static int mscc_miim_wait_pending(struct mii_bus *bus) { - struct mscc_miim_dev *miim = bus->priv; u32 val; - return mscc_readl_poll_timeout(miim->regs + MSCC_MIIM_REG_STATUS, val, + return mscc_readx_poll_timeout(mscc_miim_status, bus, val, !(val & MSCC_MIIM_STATUS_STAT_PENDING), 50, 10000); } @@ -80,15 +97,29 @@ static int mscc_miim_read(struct mii_bus *bus, int mii_id, int regnum) if (ret) goto out; - writel(MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | MSCC_MIIM_CMD_OPR_READ, - miim->regs + MSCC_MIIM_REG_CMD); + ret = regmap_write(miim->regs, + MSCC_MIIM_REG_CMD + miim->mii_status_offset, + MSCC_MIIM_CMD_VLD | + (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | + (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | + MSCC_MIIM_CMD_OPR_READ); + + if (ret < 0) { + WARN_ONCE(1, "mscc miim write cmd reg error %d\n", ret); + goto out; + } ret = mscc_miim_wait_ready(bus); if (ret) goto out; - val = readl(miim->regs + MSCC_MIIM_REG_DATA); + ret = regmap_read(miim->regs, + MSCC_MIIM_REG_DATA + miim->mii_status_offset, &val); + if (ret < 0) { + WARN_ONCE(1, "mscc miim read data reg error %d\n", ret); + goto out; + } + if (val & MSCC_MIIM_DATA_ERROR) { ret = -EIO; goto out; @@ -109,12 +140,16 @@ static int mscc_miim_write(struct mii_bus *bus, int mii_id, if (ret < 0) goto out; - writel(MSCC_MIIM_CMD_VLD | (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | - (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | - (value << MSCC_MIIM_CMD_WRDATA_SHIFT) | - MSCC_MIIM_CMD_OPR_WRITE, - miim->regs + MSCC_MIIM_REG_CMD); + ret = regmap_write(miim->regs, + MSCC_MIIM_REG_CMD + miim->mii_status_offset, + MSCC_MIIM_CMD_VLD | + (mii_id << MSCC_MIIM_CMD_PHYAD_SHIFT) | + (regnum << MSCC_MIIM_CMD_REGAD_SHIFT) | + (value << MSCC_MIIM_CMD_WRDATA_SHIFT) | + MSCC_MIIM_CMD_OPR_WRITE); + if (ret < 0) + WARN_ONCE(1, "mscc miim write error %d\n", ret); out: return ret; } @@ -122,51 +157,116 @@ out: static int mscc_miim_reset(struct mii_bus *bus) { struct mscc_miim_dev *miim = bus->priv; + int offset = miim->phy_reset_offset; + int ret; if (miim->phy_regs) { - writel(0, miim->phy_regs + MSCC_PHY_REG_PHY_CFG); - writel(0x1ff, miim->phy_regs + MSCC_PHY_REG_PHY_CFG); + ret = regmap_write(miim->phy_regs, + MSCC_PHY_REG_PHY_CFG + offset, 0); + if (ret < 0) { + WARN_ONCE(1, "mscc reset set error %d\n", ret); + return ret; + } + + ret = regmap_write(miim->phy_regs, + MSCC_PHY_REG_PHY_CFG + offset, 0x1ff); + if (ret < 0) { + WARN_ONCE(1, "mscc reset clear error %d\n", ret); + return ret; + } + mdelay(500); } return 0; } -static int mscc_miim_probe(struct platform_device *pdev) +static const struct regmap_config mscc_miim_regmap_config = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +int mscc_miim_setup(struct device *dev, struct mii_bus **pbus, const char *name, + struct regmap *mii_regmap, int status_offset) { - struct mscc_miim_dev *dev; - struct resource *res; + struct mscc_miim_dev *miim; struct mii_bus *bus; - int ret; - bus = devm_mdiobus_alloc_size(&pdev->dev, sizeof(*dev)); + bus = devm_mdiobus_alloc_size(dev, sizeof(*miim)); if (!bus) return -ENOMEM; - bus->name = "mscc_miim"; + bus->name = name; bus->read = mscc_miim_read; bus->write = mscc_miim_write; bus->reset = mscc_miim_reset; - snprintf(bus->id, MII_BUS_ID_SIZE, "%s-mii", dev_name(&pdev->dev)); - bus->parent = &pdev->dev; + snprintf(bus->id, MII_BUS_ID_SIZE, "%s-mii", dev_name(dev)); + bus->parent = dev; + + miim = bus->priv; + + *pbus = bus; + + miim->regs = mii_regmap; + miim->mii_status_offset = status_offset; + + *pbus = bus; + + return 0; +} +EXPORT_SYMBOL(mscc_miim_setup); + +static int mscc_miim_probe(struct platform_device *pdev) +{ + struct regmap *mii_regmap, *phy_regmap = NULL; + void __iomem *regs, *phy_regs; + struct mscc_miim_dev *miim; + struct resource *res; + struct mii_bus *bus; + int ret; - dev = bus->priv; - dev->regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); - if (IS_ERR(dev->regs)) { + regs = devm_platform_get_and_ioremap_resource(pdev, 0, NULL); + if (IS_ERR(regs)) { dev_err(&pdev->dev, "Unable to map MIIM registers\n"); - return PTR_ERR(dev->regs); + return PTR_ERR(regs); + } + + mii_regmap = devm_regmap_init_mmio(&pdev->dev, regs, + &mscc_miim_regmap_config); + + if (IS_ERR(mii_regmap)) { + dev_err(&pdev->dev, "Unable to create MIIM regmap\n"); + return PTR_ERR(mii_regmap); } /* This resource is optional */ res = platform_get_resource(pdev, IORESOURCE_MEM, 1); if (res) { - dev->phy_regs = devm_ioremap_resource(&pdev->dev, res); - if (IS_ERR(dev->phy_regs)) { + phy_regs = devm_ioremap_resource(&pdev->dev, res); + if (IS_ERR(phy_regs)) { dev_err(&pdev->dev, "Unable to map internal phy registers\n"); - return PTR_ERR(dev->phy_regs); + return PTR_ERR(phy_regs); } + + phy_regmap = devm_regmap_init_mmio(&pdev->dev, phy_regs, + &mscc_miim_regmap_config); + if (IS_ERR(phy_regmap)) { + dev_err(&pdev->dev, "Unable to create phy register regmap\n"); + return PTR_ERR(phy_regmap); + } + } + + ret = mscc_miim_setup(&pdev->dev, &bus, "mscc_miim", mii_regmap, 0); + if (ret < 0) { + dev_err(&pdev->dev, "Unable to setup the MDIO bus\n"); + return ret; } + miim = bus->priv; + miim->phy_regs = phy_regmap; + miim->phy_reset_offset = 0; + ret = of_mdiobus_register(bus, pdev->dev.of_node); if (ret < 0) { dev_err(&pdev->dev, "Cannot register MDIO bus (%d)\n", ret); diff --git a/drivers/net/netconsole.c b/drivers/net/netconsole.c index ccecba908ded..ab8cd5551020 100644 --- a/drivers/net/netconsole.c +++ b/drivers/net/netconsole.c @@ -721,7 +721,7 @@ restart: __netpoll_cleanup(&nt->np); spin_lock_irqsave(&target_list_lock, flags); - dev_put(nt->np.dev); + dev_put_track(nt->np.dev, &nt->np.dev_tracker); nt->np.dev = NULL; nt->enabled = false; stopped = true; diff --git a/drivers/net/netdevsim/dev.c b/drivers/net/netdevsim/dev.c index 54345c096a16..08d7b465a0de 100644 --- a/drivers/net/netdevsim/dev.c +++ b/drivers/net/netdevsim/dev.c @@ -1622,7 +1622,7 @@ err_params_unregister: devlink_params_unregister(devlink, nsim_devlink_params, ARRAY_SIZE(nsim_devlink_params)); err_dl_unregister: - devlink_resources_unregister(devlink, NULL); + devlink_resources_unregister(devlink); err_vfc_free: kfree(nsim_dev->vfconfigs); err_devlink_free: @@ -1668,7 +1668,7 @@ void nsim_drv_remove(struct nsim_bus_dev *nsim_bus_dev) nsim_dev_debugfs_exit(nsim_dev); devlink_params_unregister(devlink, nsim_devlink_params, ARRAY_SIZE(nsim_devlink_params)); - devlink_resources_unregister(devlink, NULL); + devlink_resources_unregister(devlink); kfree(nsim_dev->vfconfigs); devlink_free(devlink); dev_set_drvdata(&nsim_bus_dev->dev, NULL); diff --git a/drivers/net/netdevsim/ethtool.c b/drivers/net/netdevsim/ethtool.c index 0ab6a40be611..2b84169bf3a2 100644 --- a/drivers/net/netdevsim/ethtool.c +++ b/drivers/net/netdevsim/ethtool.c @@ -65,7 +65,9 @@ static int nsim_set_coalesce(struct net_device *dev, } static void nsim_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netdevsim *ns = netdev_priv(dev); @@ -73,7 +75,9 @@ static void nsim_get_ringparam(struct net_device *dev, } static int nsim_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct netdevsim *ns = netdev_priv(dev); diff --git a/drivers/net/phy/dp83869.c b/drivers/net/phy/dp83869.c index 7113925606f7..b4ff9c5073a3 100644 --- a/drivers/net/phy/dp83869.c +++ b/drivers/net/phy/dp83869.c @@ -16,6 +16,7 @@ #include <dt-bindings/net/ti-dp83869.h> #define DP83869_PHY_ID 0x2000a0f1 +#define DP83561_PHY_ID 0x2000a1a4 #define DP83869_DEVADDR 0x1f #define MII_DP83869_PHYCTRL 0x10 @@ -878,34 +879,35 @@ static int dp83869_phy_reset(struct phy_device *phydev) return dp83869_config_init(phydev); } -static struct phy_driver dp83869_driver[] = { - { - PHY_ID_MATCH_MODEL(DP83869_PHY_ID), - .name = "TI DP83869", - - .probe = dp83869_probe, - .config_init = dp83869_config_init, - .soft_reset = dp83869_phy_reset, - - /* IRQ related */ - .config_intr = dp83869_config_intr, - .handle_interrupt = dp83869_handle_interrupt, - .read_status = dp83869_read_status, - .get_tunable = dp83869_get_tunable, - .set_tunable = dp83869_set_tunable, +#define DP83869_PHY_DRIVER(_id, _name) \ +{ \ + PHY_ID_MATCH_MODEL(_id), \ + .name = (_name), \ + .probe = dp83869_probe, \ + .config_init = dp83869_config_init, \ + .soft_reset = dp83869_phy_reset, \ + .config_intr = dp83869_config_intr, \ + .handle_interrupt = dp83869_handle_interrupt, \ + .read_status = dp83869_read_status, \ + .get_tunable = dp83869_get_tunable, \ + .set_tunable = dp83869_set_tunable, \ + .get_wol = dp83869_get_wol, \ + .set_wol = dp83869_set_wol, \ + .suspend = genphy_suspend, \ + .resume = genphy_resume, \ +} - .get_wol = dp83869_get_wol, - .set_wol = dp83869_set_wol, +static struct phy_driver dp83869_driver[] = { + DP83869_PHY_DRIVER(DP83869_PHY_ID, "TI DP83869"), + DP83869_PHY_DRIVER(DP83561_PHY_ID, "TI DP83561-SP"), - .suspend = genphy_suspend, - .resume = genphy_resume, - }, }; module_phy_driver(dp83869_driver); static struct mdio_device_id __maybe_unused dp83869_tbl[] = { { PHY_ID_MATCH_MODEL(DP83869_PHY_ID) }, + { PHY_ID_MATCH_MODEL(DP83561_PHY_ID) }, { } }; MODULE_DEVICE_TABLE(mdio, dp83869_tbl); diff --git a/drivers/net/phy/mdio_bus.c b/drivers/net/phy/mdio_bus.c index c204067f1890..9b6f2df07211 100644 --- a/drivers/net/phy/mdio_bus.c +++ b/drivers/net/phy/mdio_bus.c @@ -176,9 +176,11 @@ static void mdiobus_release(struct device *d) { struct mii_bus *bus = to_mii_bus(d); - BUG_ON(bus->state != MDIOBUS_RELEASED && - /* for compatibility with error handling in drivers */ - bus->state != MDIOBUS_ALLOCATED); + WARN(bus->state != MDIOBUS_RELEASED && + /* for compatibility with error handling in drivers */ + bus->state != MDIOBUS_ALLOCATED, + "%s: not in RELEASED or ALLOCATED state\n", + bus->id); kfree(bus); } @@ -529,8 +531,9 @@ int __mdiobus_register(struct mii_bus *bus, struct module *owner) bus->parent->of_node->fwnode.flags |= FWNODE_FLAG_NEEDS_CHILD_BOUND_ON_ADD; - BUG_ON(bus->state != MDIOBUS_ALLOCATED && - bus->state != MDIOBUS_UNREGISTERED); + WARN(bus->state != MDIOBUS_ALLOCATED && + bus->state != MDIOBUS_UNREGISTERED, + "%s: not in ALLOCATED or UNREGISTERED state\n", bus->id); bus->owner = owner; bus->dev.parent = bus->parent; @@ -658,7 +661,8 @@ void mdiobus_free(struct mii_bus *bus) return; } - BUG_ON(bus->state != MDIOBUS_UNREGISTERED); + WARN(bus->state != MDIOBUS_UNREGISTERED, + "%s: not in UNREGISTERED state\n", bus->id); bus->state = MDIOBUS_RELEASED; put_device(&bus->dev); diff --git a/drivers/net/phy/phy-core.c b/drivers/net/phy/phy-core.c index 2870c33b8975..271fc01f7f7f 100644 --- a/drivers/net/phy/phy-core.c +++ b/drivers/net/phy/phy-core.c @@ -162,11 +162,11 @@ static const struct phy_setting settings[] = { PHY_SETTING( 2500, FULL, 2500baseT_Full ), PHY_SETTING( 2500, FULL, 2500baseX_Full ), /* 1G */ - PHY_SETTING( 1000, FULL, 1000baseKX_Full ), PHY_SETTING( 1000, FULL, 1000baseT_Full ), PHY_SETTING( 1000, HALF, 1000baseT_Half ), PHY_SETTING( 1000, FULL, 1000baseT1_Full ), PHY_SETTING( 1000, FULL, 1000baseX_Full ), + PHY_SETTING( 1000, FULL, 1000baseKX_Full ), /* 100M */ PHY_SETTING( 100, FULL, 100baseT_Full ), PHY_SETTING( 100, FULL, 100baseT1_Full ), diff --git a/drivers/net/phy/phylink.c b/drivers/net/phy/phylink.c index ea82ea5660e7..20df8af3e201 100644 --- a/drivers/net/phy/phylink.c +++ b/drivers/net/phy/phylink.c @@ -166,6 +166,259 @@ static const char *phylink_an_mode_str(unsigned int mode) return mode < ARRAY_SIZE(modestr) ? modestr[mode] : "unknown"; } +static void phylink_caps_to_linkmodes(unsigned long *linkmodes, + unsigned long caps) +{ + if (caps & MAC_SYM_PAUSE) + __set_bit(ETHTOOL_LINK_MODE_Pause_BIT, linkmodes); + + if (caps & MAC_ASYM_PAUSE) + __set_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, linkmodes); + + if (caps & MAC_10HD) + __set_bit(ETHTOOL_LINK_MODE_10baseT_Half_BIT, linkmodes); + + if (caps & MAC_10FD) + __set_bit(ETHTOOL_LINK_MODE_10baseT_Full_BIT, linkmodes); + + if (caps & MAC_100HD) { + __set_bit(ETHTOOL_LINK_MODE_100baseT_Half_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseFX_Half_BIT, linkmodes); + } + + if (caps & MAC_100FD) { + __set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseT1_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100baseFX_Full_BIT, linkmodes); + } + + if (caps & MAC_1000HD) + __set_bit(ETHTOOL_LINK_MODE_1000baseT_Half_BIT, linkmodes); + + if (caps & MAC_1000FD) { + __set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_1000baseX_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_1000baseT1_Full_BIT, linkmodes); + } + + if (caps & MAC_2500FD) { + __set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_2500baseX_Full_BIT, linkmodes); + } + + if (caps & MAC_5000FD) + __set_bit(ETHTOOL_LINK_MODE_5000baseT_Full_BIT, linkmodes); + + if (caps & MAC_10000FD) { + __set_bit(ETHTOOL_LINK_MODE_10000baseT_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseR_FEC_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseLR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseLRM_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_10000baseER_Full_BIT, linkmodes); + } + + if (caps & MAC_25000FD) { + __set_bit(ETHTOOL_LINK_MODE_25000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_25000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_25000baseSR_Full_BIT, linkmodes); + } + + if (caps & MAC_40000FD) { + __set_bit(ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT, linkmodes); + } + + if (caps & MAC_50000FD) { + __set_bit(ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseLR_ER_FR_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_50000baseDR_Full_BIT, linkmodes); + } + + if (caps & MAC_56000FD) { + __set_bit(ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT, linkmodes); + } + + if (caps & MAC_100000FD) { + __set_bit(ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR2_ER2_FR2_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseDR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseKR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseSR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseLR_ER_FR_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseCR_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_100000baseDR_Full_BIT, linkmodes); + } + + if (caps & MAC_200000FD) { + __set_bit(ETHTOOL_LINK_MODE_200000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseLR4_ER4_FR4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseDR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseCR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseKR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseSR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseLR2_ER2_FR2_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseDR2_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_200000baseCR2_Full_BIT, linkmodes); + } + + if (caps & MAC_400000FD) { + __set_bit(ETHTOOL_LINK_MODE_400000baseKR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseSR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseLR8_ER8_FR8_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseDR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseCR8_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseKR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseSR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseLR4_ER4_FR4_Full_BIT, + linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseDR4_Full_BIT, linkmodes); + __set_bit(ETHTOOL_LINK_MODE_400000baseCR4_Full_BIT, linkmodes); + } +} + +/** + * phylink_get_linkmodes() - get acceptable link modes + * @linkmodes: ethtool linkmode mask (must be already initialised) + * @interface: phy interface mode defined by &typedef phy_interface_t + * @mac_capabilities: bitmask of MAC capabilities + * + * Set all possible pause, speed and duplex linkmodes in @linkmodes that + * are supported by the @interface mode and @mac_capabilities. @linkmodes + * must have been initialised previously. + */ +void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, + unsigned long mac_capabilities) +{ + unsigned long caps = MAC_SYM_PAUSE | MAC_ASYM_PAUSE; + + switch (interface) { + case PHY_INTERFACE_MODE_USXGMII: + caps |= MAC_10000FD | MAC_5000FD | MAC_2500FD; + fallthrough; + + case PHY_INTERFACE_MODE_RGMII_TXID: + case PHY_INTERFACE_MODE_RGMII_RXID: + case PHY_INTERFACE_MODE_RGMII_ID: + case PHY_INTERFACE_MODE_RGMII: + case PHY_INTERFACE_MODE_QSGMII: + case PHY_INTERFACE_MODE_SGMII: + case PHY_INTERFACE_MODE_GMII: + caps |= MAC_1000HD | MAC_1000FD; + fallthrough; + + case PHY_INTERFACE_MODE_REVRMII: + case PHY_INTERFACE_MODE_RMII: + case PHY_INTERFACE_MODE_SMII: + case PHY_INTERFACE_MODE_REVMII: + case PHY_INTERFACE_MODE_MII: + caps |= MAC_10HD | MAC_10FD; + fallthrough; + + case PHY_INTERFACE_MODE_100BASEX: + caps |= MAC_100HD | MAC_100FD; + break; + + case PHY_INTERFACE_MODE_TBI: + case PHY_INTERFACE_MODE_MOCA: + case PHY_INTERFACE_MODE_RTBI: + case PHY_INTERFACE_MODE_1000BASEX: + caps |= MAC_1000HD; + fallthrough; + case PHY_INTERFACE_MODE_TRGMII: + caps |= MAC_1000FD; + break; + + case PHY_INTERFACE_MODE_2500BASEX: + caps |= MAC_2500FD; + break; + + case PHY_INTERFACE_MODE_5GBASER: + caps |= MAC_5000FD; + break; + + case PHY_INTERFACE_MODE_XGMII: + case PHY_INTERFACE_MODE_RXAUI: + case PHY_INTERFACE_MODE_XAUI: + case PHY_INTERFACE_MODE_10GBASER: + case PHY_INTERFACE_MODE_10GKR: + caps |= MAC_10000FD; + break; + + case PHY_INTERFACE_MODE_25GBASER: + caps |= MAC_25000FD; + break; + + case PHY_INTERFACE_MODE_XLGMII: + caps |= MAC_40000FD; + break; + + case PHY_INTERFACE_MODE_INTERNAL: + caps |= ~0; + break; + + case PHY_INTERFACE_MODE_NA: + case PHY_INTERFACE_MODE_MAX: + break; + } + + phylink_caps_to_linkmodes(linkmodes, caps & mac_capabilities); +} +EXPORT_SYMBOL_GPL(phylink_get_linkmodes); + +/** + * phylink_generic_validate() - generic validate() callback implementation + * @config: a pointer to a &struct phylink_config. + * @supported: ethtool bitmask for supported link modes. + * @state: a pointer to a &struct phylink_link_state. + * + * Generic implementation of the validate() callback that MAC drivers can + * use when they pass the range of supported interfaces and MAC capabilities. + * This makes use of phylink_get_linkmodes(). + */ +void phylink_generic_validate(struct phylink_config *config, + unsigned long *supported, + struct phylink_link_state *state) +{ + __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; + + phylink_set_port_modes(mask); + phylink_set(mask, Autoneg); + phylink_get_linkmodes(mask, state->interface, config->mac_capabilities); + + linkmode_and(supported, supported, mask); + linkmode_and(state->advertising, state->advertising, mask); +} +EXPORT_SYMBOL_GPL(phylink_generic_validate); + static int phylink_validate_any(struct phylink *pl, unsigned long *supported, struct phylink_link_state *state) { @@ -489,7 +742,7 @@ static void phylink_mac_pcs_an_restart(struct phylink *pl) phylink_autoneg_inband(pl->cur_link_an_mode)) { if (pl->pcs_ops) pl->pcs_ops->pcs_an_restart(pl->pcs); - else + else if (pl->config->legacy_pre_march2020) pl->mac_ops->mac_an_restart(pl->config); } } @@ -550,7 +803,7 @@ static int phylink_change_inband_advert(struct phylink *pl) if (test_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state)) return 0; - if (!pl->pcs_ops) { + if (!pl->pcs_ops && pl->config->legacy_pre_march2020) { /* Legacy method */ phylink_mac_config(pl, &pl->link_config); phylink_mac_pcs_an_restart(pl); @@ -601,7 +854,8 @@ static void phylink_mac_pcs_get_state(struct phylink *pl, if (pl->pcs_ops) pl->pcs_ops->pcs_get_state(pl->pcs, state); - else if (pl->mac_ops->mac_pcs_get_state) + else if (pl->mac_ops->mac_pcs_get_state && + pl->config->legacy_pre_march2020) pl->mac_ops->mac_pcs_get_state(pl->config, state); else state->link = 0; @@ -795,12 +1049,11 @@ static void phylink_resolve(struct work_struct *w) } phylink_major_config(pl, false, &link_state); pl->link_config.interface = link_state.interface; - } else if (!pl->pcs_ops) { + } else if (!pl->pcs_ops && pl->config->legacy_pre_march2020) { /* The interface remains unchanged, only the speed, * duplex or pause settings have changed. Call the * old mac_config() method to configure the MAC/PCS - * only if we do not have a PCS installed (an - * unconverted user.) + * only if we do not have a legacy MAC driver. */ phylink_mac_config(pl, &link_state); } @@ -837,6 +1090,12 @@ static void phylink_run_resolve_and_disable(struct phylink *pl, int bit) } } +static void phylink_enable_and_run_resolve(struct phylink *pl, int bit) +{ + clear_bit(bit, &pl->phylink_disable_state); + phylink_run_resolve(pl); +} + static void phylink_fixed_poll(struct timer_list *t) { struct phylink *pl = container_of(t, struct phylink, link_poll); @@ -1096,7 +1355,8 @@ static int phylink_bringup_phy(struct phylink *pl, struct phy_device *phy, mutex_unlock(&phy->lock); phylink_dbg(pl, - "phy: setting supported %*pb advertising %*pb\n", + "phy: %s setting supported %*pb advertising %*pb\n", + phy_modes(interface), __ETHTOOL_LINK_MODE_MASK_NBITS, pl->supported, __ETHTOOL_LINK_MODE_MASK_NBITS, phy->advertising); @@ -1214,6 +1474,12 @@ int phylink_fwnode_phy_connect(struct phylink *pl, if (!phy_dev) return -ENODEV; + /* Use PHY device/driver interface */ + if (pl->link_interface == PHY_INTERFACE_MODE_NA) { + pl->link_interface = phy_dev->interface; + pl->link_config.interface = pl->link_interface; + } + ret = phy_attach_direct(pl->netdev, phy_dev, flags, pl->link_interface); if (ret) { @@ -1314,8 +1580,7 @@ void phylink_start(struct phylink *pl) */ phylink_mac_initial_config(pl, true); - clear_bit(PHYLINK_DISABLE_STOPPED, &pl->phylink_disable_state); - phylink_run_resolve(pl); + phylink_enable_and_run_resolve(pl, PHYLINK_DISABLE_STOPPED); if (pl->cfg_link_an_mode == MLO_AN_FIXED && pl->link_gpio) { int irq = gpiod_to_irq(pl->link_gpio); @@ -1456,8 +1721,7 @@ void phylink_resume(struct phylink *pl) phylink_mac_initial_config(pl, true); /* Re-enable and re-resolve the link parameters */ - clear_bit(PHYLINK_DISABLE_MAC_WOL, &pl->phylink_disable_state); - phylink_run_resolve(pl); + phylink_enable_and_run_resolve(pl, PHYLINK_DISABLE_MAC_WOL); } else { phylink_start(pl); } @@ -2386,8 +2650,7 @@ static void phylink_sfp_link_up(void *upstream) ASSERT_RTNL(); - clear_bit(PHYLINK_DISABLE_LINK, &pl->phylink_disable_state); - phylink_run_resolve(pl); + phylink_enable_and_run_resolve(pl, PHYLINK_DISABLE_LINK); } /* The Broadcom BCM84881 in the Methode DM7052 is unable to provide a SGMII @@ -2587,31 +2850,22 @@ void phylink_decode_usxgmii_word(struct phylink_link_state *state, EXPORT_SYMBOL_GPL(phylink_decode_usxgmii_word); /** - * phylink_mii_c22_pcs_get_state() - read the MAC PCS state - * @pcs: a pointer to a &struct mdio_device. + * phylink_mii_c22_pcs_decode_state() - Decode MAC PCS state from MII registers * @state: a pointer to a &struct phylink_link_state. + * @bmsr: The value of the %MII_BMSR register + * @lpa: The value of the %MII_LPA register * * Helper for MAC PCS supporting the 802.3 clause 22 register set for * clause 37 negotiation and/or SGMII control. * - * Read the MAC PCS state from the MII device configured in @config and - * parse the Clause 37 or Cisco SGMII link partner negotiation word into - * the phylink @state structure. This is suitable to be directly plugged - * into the mac_pcs_get_state() member of the struct phylink_mac_ops - * structure. + * Parse the Clause 37 or Cisco SGMII link partner negotiation word into + * the phylink @state structure. This is suitable to be used for implementing + * the mac_pcs_get_state() member of the struct phylink_mac_ops structure if + * accessing @bmsr and @lpa cannot be done with MDIO directly. */ -void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, - struct phylink_link_state *state) +void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, + u16 bmsr, u16 lpa) { - int bmsr, lpa; - - bmsr = mdiodev_read(pcs, MII_BMSR); - lpa = mdiodev_read(pcs, MII_LPA); - if (bmsr < 0 || lpa < 0) { - state->link = false; - return; - } - state->link = !!(bmsr & BMSR_LSTATUS); state->an_complete = !!(bmsr & BMSR_ANEGCOMPLETE); /* If there is no link or autonegotiation is disabled, the LP advertisement @@ -2639,28 +2893,54 @@ void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, break; } } +EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_decode_state); + +/** + * phylink_mii_c22_pcs_get_state() - read the MAC PCS state + * @pcs: a pointer to a &struct mdio_device. + * @state: a pointer to a &struct phylink_link_state. + * + * Helper for MAC PCS supporting the 802.3 clause 22 register set for + * clause 37 negotiation and/or SGMII control. + * + * Read the MAC PCS state from the MII device configured in @config and + * parse the Clause 37 or Cisco SGMII link partner negotiation word into + * the phylink @state structure. This is suitable to be directly plugged + * into the mac_pcs_get_state() member of the struct phylink_mac_ops + * structure. + */ +void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, + struct phylink_link_state *state) +{ + int bmsr, lpa; + + bmsr = mdiodev_read(pcs, MII_BMSR); + lpa = mdiodev_read(pcs, MII_LPA); + if (bmsr < 0 || lpa < 0) { + state->link = false; + return; + } + + phylink_mii_c22_pcs_decode_state(state, bmsr, lpa); +} EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_get_state); /** - * phylink_mii_c22_pcs_set_advertisement() - configure the clause 37 PCS + * phylink_mii_c22_pcs_encode_advertisement() - configure the clause 37 PCS * advertisement - * @pcs: a pointer to a &struct mdio_device. * @interface: the PHY interface mode being configured * @advertising: the ethtool advertisement mask * * Helper for MAC PCS supporting the 802.3 clause 22 register set for * clause 37 negotiation and/or SGMII control. * - * Configure the clause 37 PCS advertisement as specified by @state. This - * does not trigger a renegotiation; phylink will do that via the - * mac_an_restart() method of the struct phylink_mac_ops structure. + * Encode the clause 37 PCS advertisement as specified by @interface and + * @advertising. * - * Returns negative error code on failure to configure the advertisement, - * zero if no change has been made, or one if the advertisement has changed. + * Return: The new value for @adv, or ``-EINVAL`` if it should not be changed. */ -int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, - phy_interface_t interface, - const unsigned long *advertising) +int phylink_mii_c22_pcs_encode_advertisement(phy_interface_t interface, + const unsigned long *advertising) { u16 adv; @@ -2674,18 +2954,15 @@ int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, if (linkmode_test_bit(ETHTOOL_LINK_MODE_Asym_Pause_BIT, advertising)) adv |= ADVERTISE_1000XPSE_ASYM; - - return mdiodev_modify_changed(pcs, MII_ADVERTISE, 0xffff, adv); - + return adv; case PHY_INTERFACE_MODE_SGMII: - return mdiodev_modify_changed(pcs, MII_ADVERTISE, 0xffff, 0x0001); - + return 0x0001; default: /* Nothing to do for other modes */ - return 0; + return -EINVAL; } } -EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_set_advertisement); +EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_encode_advertisement); /** * phylink_mii_c22_pcs_config() - configure clause 22 PCS @@ -2703,16 +2980,18 @@ int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising) { - bool changed; + bool changed = 0; u16 bmcr; - int ret; + int ret, adv; - ret = phylink_mii_c22_pcs_set_advertisement(pcs, interface, - advertising); - if (ret < 0) - return ret; - - changed = ret > 0; + adv = phylink_mii_c22_pcs_encode_advertisement(interface, advertising); + if (adv >= 0) { + ret = mdiobus_modify_changed(pcs->bus, pcs->addr, + MII_ADVERTISE, 0xffff, adv); + if (ret < 0) + return ret; + changed = ret; + } /* Ensure ISOLATE bit is disabled */ if (mode == MLO_AN_INBAND && @@ -2725,7 +3004,7 @@ int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, if (ret < 0) return ret; - return changed ? 1 : 0; + return changed; } EXPORT_SYMBOL_GPL(phylink_mii_c22_pcs_config); diff --git a/drivers/net/usb/ax88179_178a.c b/drivers/net/usb/ax88179_178a.c index ea8aa8c33241..1a627ba4b850 100644 --- a/drivers/net/usb/ax88179_178a.c +++ b/drivers/net/usb/ax88179_178a.c @@ -1377,11 +1377,12 @@ static int ax88179_bind(struct usbnet *dev, struct usb_interface *intf) dev->mii.phy_id = 0x03; dev->mii.supports_gmii = 1; - dev->net->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_RXCSUM; + dev->net->features |= NETIF_F_SG | NETIF_F_IP_CSUM | + NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | NETIF_F_TSO; - dev->net->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | - NETIF_F_RXCSUM; + dev->net->hw_features |= dev->net->features; + + netif_set_gso_max_size(dev->net, 16384); /* Enable checksum offload */ *tmp = AX_RXCOE_IP | AX_RXCOE_TCP | AX_RXCOE_UDP | @@ -1526,17 +1527,19 @@ ax88179_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) { u32 tx_hdr1, tx_hdr2; int frame_size = dev->maxpacket; - int mss = skb_shinfo(skb)->gso_size; int headroom; void *ptr; tx_hdr1 = skb->len; - tx_hdr2 = mss; + tx_hdr2 = skb_shinfo(skb)->gso_size; /* Set TSO mss */ if (((skb->len + 8) % frame_size) == 0) tx_hdr2 |= 0x80008000; /* Enable padding */ headroom = skb_headroom(skb) - 8; + if ((dev->net->features & NETIF_F_SG) && skb_linearize(skb)) + return NULL; + if ((skb_header_cloned(skb) || headroom < 0) && pskb_expand_head(skb, headroom < 0 ? 8 : 0, 0, GFP_ATOMIC)) { dev_kfree_skb_any(skb); @@ -1547,6 +1550,8 @@ ax88179_tx_fixup(struct usbnet *dev, struct sk_buff *skb, gfp_t flags) put_unaligned_le32(tx_hdr1, ptr); put_unaligned_le32(tx_hdr2, ptr + 4); + usbnet_set_skb_tx_stats(skb, (skb_shinfo(skb)->gso_segs ?: 1), 0); + return skb; } diff --git a/drivers/net/usb/lan78xx.c b/drivers/net/usb/lan78xx.c index 8cd265fc1fd9..185e08c1af31 100644 --- a/drivers/net/usb/lan78xx.c +++ b/drivers/net/usb/lan78xx.c @@ -67,7 +67,7 @@ #define DEFAULT_TSO_CSUM_ENABLE (true) #define DEFAULT_VLAN_FILTER_ENABLE (true) #define DEFAULT_VLAN_RX_OFFLOAD (true) -#define TX_OVERHEAD (8) +#define TX_ALIGNMENT (4) #define RXW_PADDING 2 #define LAN78XX_USB_VENDOR_ID (0x0424) @@ -90,6 +90,41 @@ WAKE_MCAST | WAKE_BCAST | \ WAKE_ARP | WAKE_MAGIC) +#define LAN78XX_NAPI_WEIGHT 64 + +#define TX_URB_NUM 10 +#define TX_SS_URB_NUM TX_URB_NUM +#define TX_HS_URB_NUM TX_URB_NUM +#define TX_FS_URB_NUM TX_URB_NUM + +/* A single URB buffer must be large enough to hold a complete jumbo packet + */ +#define TX_SS_URB_SIZE (32 * 1024) +#define TX_HS_URB_SIZE (16 * 1024) +#define TX_FS_URB_SIZE (10 * 1024) + +#define RX_SS_URB_NUM 30 +#define RX_HS_URB_NUM 10 +#define RX_FS_URB_NUM 10 +#define RX_SS_URB_SIZE TX_SS_URB_SIZE +#define RX_HS_URB_SIZE TX_HS_URB_SIZE +#define RX_FS_URB_SIZE TX_FS_URB_SIZE + +#define SS_BURST_CAP_SIZE RX_SS_URB_SIZE +#define SS_BULK_IN_DELAY 0x2000 +#define HS_BURST_CAP_SIZE RX_HS_URB_SIZE +#define HS_BULK_IN_DELAY 0x2000 +#define FS_BURST_CAP_SIZE RX_FS_URB_SIZE +#define FS_BULK_IN_DELAY 0x2000 + +#define TX_CMD_LEN 8 +#define TX_SKB_MIN_LEN (TX_CMD_LEN + ETH_HLEN) +#define LAN78XX_TSO_SIZE(dev) ((dev)->tx_urb_size - TX_SKB_MIN_LEN) + +#define RX_CMD_LEN 10 +#define RX_SKB_MIN_LEN (RX_CMD_LEN + ETH_HLEN) +#define RX_MAX_FRAME_LEN(mtu) ((mtu) + ETH_HLEN + VLAN_HLEN) + /* USB related defines */ #define BULK_IN_PIPE 1 #define BULK_OUT_PIPE 2 @@ -385,14 +420,22 @@ struct lan78xx_net { struct usb_interface *intf; void *driver_priv; - int rx_qlen; - int tx_qlen; + unsigned int tx_pend_data_len; + size_t n_tx_urbs; + size_t n_rx_urbs; + size_t tx_urb_size; + size_t rx_urb_size; + + struct sk_buff_head rxq_free; struct sk_buff_head rxq; + struct sk_buff_head rxq_done; + struct sk_buff_head rxq_overflow; + struct sk_buff_head txq_free; struct sk_buff_head txq; - struct sk_buff_head done; struct sk_buff_head txq_pend; - struct tasklet_struct bh; + struct napi_struct napi; + struct delayed_work wq; int msg_enable; @@ -404,8 +447,8 @@ struct lan78xx_net { struct mutex phy_mutex; /* for phy access */ unsigned int pipe_in, pipe_out, pipe_intr; - u32 hard_mtu; /* count any extra framing */ - size_t rx_urb_size; /* size for rx urbs */ + unsigned int bulk_in_delay; + unsigned int burst_cap; unsigned long flags; @@ -443,6 +486,129 @@ static int msg_level = -1; module_param(msg_level, int, 0); MODULE_PARM_DESC(msg_level, "Override default message level"); +static struct sk_buff *lan78xx_get_buf(struct sk_buff_head *buf_pool) +{ + if (skb_queue_empty(buf_pool)) + return NULL; + + return skb_dequeue(buf_pool); +} + +static void lan78xx_release_buf(struct sk_buff_head *buf_pool, + struct sk_buff *buf) +{ + buf->data = buf->head; + skb_reset_tail_pointer(buf); + + buf->len = 0; + buf->data_len = 0; + + skb_queue_tail(buf_pool, buf); +} + +static void lan78xx_free_buf_pool(struct sk_buff_head *buf_pool) +{ + struct skb_data *entry; + struct sk_buff *buf; + + while (!skb_queue_empty(buf_pool)) { + buf = skb_dequeue(buf_pool); + if (buf) { + entry = (struct skb_data *)buf->cb; + usb_free_urb(entry->urb); + dev_kfree_skb_any(buf); + } + } +} + +static int lan78xx_alloc_buf_pool(struct sk_buff_head *buf_pool, + size_t n_urbs, size_t urb_size, + struct lan78xx_net *dev) +{ + struct skb_data *entry; + struct sk_buff *buf; + struct urb *urb; + int i; + + skb_queue_head_init(buf_pool); + + for (i = 0; i < n_urbs; i++) { + buf = alloc_skb(urb_size, GFP_ATOMIC); + if (!buf) + goto error; + + if (skb_linearize(buf) != 0) { + dev_kfree_skb_any(buf); + goto error; + } + + urb = usb_alloc_urb(0, GFP_ATOMIC); + if (!urb) { + dev_kfree_skb_any(buf); + goto error; + } + + entry = (struct skb_data *)buf->cb; + entry->urb = urb; + entry->dev = dev; + entry->length = 0; + entry->num_of_packet = 0; + + skb_queue_tail(buf_pool, buf); + } + + return 0; + +error: + lan78xx_free_buf_pool(buf_pool); + + return -ENOMEM; +} + +static struct sk_buff *lan78xx_get_rx_buf(struct lan78xx_net *dev) +{ + return lan78xx_get_buf(&dev->rxq_free); +} + +static void lan78xx_release_rx_buf(struct lan78xx_net *dev, + struct sk_buff *rx_buf) +{ + lan78xx_release_buf(&dev->rxq_free, rx_buf); +} + +static void lan78xx_free_rx_resources(struct lan78xx_net *dev) +{ + lan78xx_free_buf_pool(&dev->rxq_free); +} + +static int lan78xx_alloc_rx_resources(struct lan78xx_net *dev) +{ + return lan78xx_alloc_buf_pool(&dev->rxq_free, + dev->n_rx_urbs, dev->rx_urb_size, dev); +} + +static struct sk_buff *lan78xx_get_tx_buf(struct lan78xx_net *dev) +{ + return lan78xx_get_buf(&dev->txq_free); +} + +static void lan78xx_release_tx_buf(struct lan78xx_net *dev, + struct sk_buff *tx_buf) +{ + lan78xx_release_buf(&dev->txq_free, tx_buf); +} + +static void lan78xx_free_tx_resources(struct lan78xx_net *dev) +{ + lan78xx_free_buf_pool(&dev->txq_free); +} + +static int lan78xx_alloc_tx_resources(struct lan78xx_net *dev) +{ + return lan78xx_alloc_buf_pool(&dev->txq_free, + dev->n_tx_urbs, dev->tx_urb_size, dev); +} + static int lan78xx_read_reg(struct lan78xx_net *dev, u32 index, u32 *data) { u32 *buf; @@ -1200,6 +1366,8 @@ static int lan78xx_update_flowcontrol(struct lan78xx_net *dev, u8 duplex, return 0; } +static void lan78xx_rx_urb_submit_all(struct lan78xx_net *dev); + static int lan78xx_mac_reset(struct lan78xx_net *dev) { unsigned long start_time = jiffies; @@ -1331,7 +1499,9 @@ static int lan78xx_link_reset(struct lan78xx_net *dev) jiffies + STAT_UPDATE_TIMER); } - tasklet_schedule(&dev->bh); + lan78xx_rx_urb_submit_all(dev); + + napi_schedule(&dev->napi); } return 0; @@ -2371,37 +2541,24 @@ found: static int lan78xx_change_mtu(struct net_device *netdev, int new_mtu) { struct lan78xx_net *dev = netdev_priv(netdev); - int ll_mtu = new_mtu + netdev->hard_header_len; - int old_hard_mtu = dev->hard_mtu; - int old_rx_urb_size = dev->rx_urb_size; + int max_frame_len = RX_MAX_FRAME_LEN(new_mtu); int ret; /* no second zero-length packet read wanted after mtu-sized packets */ - if ((ll_mtu % dev->maxpacket) == 0) + if ((max_frame_len % dev->maxpacket) == 0) return -EDOM; ret = usb_autopm_get_interface(dev->intf); if (ret < 0) return ret; - lan78xx_set_rx_max_frame_length(dev, new_mtu + VLAN_ETH_HLEN); - - netdev->mtu = new_mtu; - - dev->hard_mtu = netdev->mtu + netdev->hard_header_len; - if (dev->rx_urb_size == old_hard_mtu) { - dev->rx_urb_size = dev->hard_mtu; - if (dev->rx_urb_size > old_rx_urb_size) { - if (netif_running(dev->net)) { - unlink_urbs(dev, &dev->rxq); - tasklet_schedule(&dev->bh); - } - } - } + ret = lan78xx_set_rx_max_frame_length(dev, max_frame_len); + if (!ret) + netdev->mtu = new_mtu; usb_autopm_put_interface(dev->intf); - return 0; + return ret; } static int lan78xx_set_mac_addr(struct net_device *netdev, void *p) @@ -2557,6 +2714,44 @@ static void lan78xx_init_ltm(struct lan78xx_net *dev) lan78xx_write_reg(dev, LTM_INACTIVE1, regs[5]); } +static int lan78xx_urb_config_init(struct lan78xx_net *dev) +{ + int result = 0; + + switch (dev->udev->speed) { + case USB_SPEED_SUPER: + dev->rx_urb_size = RX_SS_URB_SIZE; + dev->tx_urb_size = TX_SS_URB_SIZE; + dev->n_rx_urbs = RX_SS_URB_NUM; + dev->n_tx_urbs = TX_SS_URB_NUM; + dev->bulk_in_delay = SS_BULK_IN_DELAY; + dev->burst_cap = SS_BURST_CAP_SIZE / SS_USB_PKT_SIZE; + break; + case USB_SPEED_HIGH: + dev->rx_urb_size = RX_HS_URB_SIZE; + dev->tx_urb_size = TX_HS_URB_SIZE; + dev->n_rx_urbs = RX_HS_URB_NUM; + dev->n_tx_urbs = TX_HS_URB_NUM; + dev->bulk_in_delay = HS_BULK_IN_DELAY; + dev->burst_cap = HS_BURST_CAP_SIZE / HS_USB_PKT_SIZE; + break; + case USB_SPEED_FULL: + dev->rx_urb_size = RX_FS_URB_SIZE; + dev->tx_urb_size = TX_FS_URB_SIZE; + dev->n_rx_urbs = RX_FS_URB_NUM; + dev->n_tx_urbs = TX_FS_URB_NUM; + dev->bulk_in_delay = FS_BULK_IN_DELAY; + dev->burst_cap = FS_BURST_CAP_SIZE / FS_USB_PKT_SIZE; + break; + default: + netdev_warn(dev->net, "USB bus speed not supported\n"); + result = -EIO; + break; + } + + return result; +} + static int lan78xx_start_hw(struct lan78xx_net *dev, u32 reg, u32 hw_enable) { return lan78xx_update_reg(dev, reg, hw_enable, hw_enable); @@ -2764,28 +2959,11 @@ static int lan78xx_reset(struct lan78xx_net *dev) /* Init LTM */ lan78xx_init_ltm(dev); - if (dev->udev->speed == USB_SPEED_SUPER) { - buf = DEFAULT_BURST_CAP_SIZE / SS_USB_PKT_SIZE; - dev->rx_urb_size = DEFAULT_BURST_CAP_SIZE; - dev->rx_qlen = 4; - dev->tx_qlen = 4; - } else if (dev->udev->speed == USB_SPEED_HIGH) { - buf = DEFAULT_BURST_CAP_SIZE / HS_USB_PKT_SIZE; - dev->rx_urb_size = DEFAULT_BURST_CAP_SIZE; - dev->rx_qlen = RX_MAX_QUEUE_MEMORY / dev->rx_urb_size; - dev->tx_qlen = RX_MAX_QUEUE_MEMORY / dev->hard_mtu; - } else { - buf = DEFAULT_BURST_CAP_SIZE / FS_USB_PKT_SIZE; - dev->rx_urb_size = DEFAULT_BURST_CAP_SIZE; - dev->rx_qlen = 4; - dev->tx_qlen = 4; - } - - ret = lan78xx_write_reg(dev, BURST_CAP, buf); + ret = lan78xx_write_reg(dev, BURST_CAP, dev->burst_cap); if (ret < 0) return ret; - ret = lan78xx_write_reg(dev, BULK_IN_DLY, DEFAULT_BULK_IN_DELAY); + ret = lan78xx_write_reg(dev, BULK_IN_DLY, dev->bulk_in_delay); if (ret < 0) return ret; @@ -2898,7 +3076,7 @@ static int lan78xx_reset(struct lan78xx_net *dev) return ret; ret = lan78xx_set_rx_max_frame_length(dev, - dev->net->mtu + VLAN_ETH_HLEN); + RX_MAX_FRAME_LEN(dev->net->mtu)); return ret; } @@ -2978,6 +3156,8 @@ static int lan78xx_open(struct net_device *net) dev->link_on = false; + napi_enable(&dev->napi); + lan78xx_defer_kevent(dev, EVENT_LINK_RESET); done: mutex_unlock(&dev->dev_mutex); @@ -3011,15 +3191,16 @@ static void lan78xx_terminate_urbs(struct lan78xx_net *dev) dev->wait = NULL; remove_wait_queue(&unlink_wakeup, &wait); - while (!skb_queue_empty(&dev->done)) { - struct skb_data *entry; - struct sk_buff *skb; + /* empty Rx done, Rx overflow and Tx pend queues + */ + while (!skb_queue_empty(&dev->rxq_done)) { + struct sk_buff *skb = skb_dequeue(&dev->rxq_done); - skb = skb_dequeue(&dev->done); - entry = (struct skb_data *)(skb->cb); - usb_free_urb(entry->urb); - dev_kfree_skb(skb); + lan78xx_release_rx_buf(dev, skb); } + + skb_queue_purge(&dev->rxq_overflow); + skb_queue_purge(&dev->txq_pend); } static int lan78xx_stop(struct net_device *net) @@ -3035,7 +3216,7 @@ static int lan78xx_stop(struct net_device *net) clear_bit(EVENT_DEV_OPEN, &dev->flags); netif_stop_queue(net); - tasklet_kill(&dev->bh); + napi_disable(&dev->napi); lan78xx_terminate_urbs(dev); @@ -3071,48 +3252,6 @@ static int lan78xx_stop(struct net_device *net) return 0; } -static struct sk_buff *lan78xx_tx_prep(struct lan78xx_net *dev, - struct sk_buff *skb, gfp_t flags) -{ - u32 tx_cmd_a, tx_cmd_b; - void *ptr; - - if (skb_cow_head(skb, TX_OVERHEAD)) { - dev_kfree_skb_any(skb); - return NULL; - } - - if (skb_linearize(skb)) { - dev_kfree_skb_any(skb); - return NULL; - } - - tx_cmd_a = (u32)(skb->len & TX_CMD_A_LEN_MASK_) | TX_CMD_A_FCS_; - - if (skb->ip_summed == CHECKSUM_PARTIAL) - tx_cmd_a |= TX_CMD_A_IPE_ | TX_CMD_A_TPE_; - - tx_cmd_b = 0; - if (skb_is_gso(skb)) { - u16 mss = max(skb_shinfo(skb)->gso_size, TX_CMD_B_MSS_MIN_); - - tx_cmd_b = (mss << TX_CMD_B_MSS_SHIFT_) & TX_CMD_B_MSS_MASK_; - - tx_cmd_a |= TX_CMD_A_LSO_; - } - - if (skb_vlan_tag_present(skb)) { - tx_cmd_a |= TX_CMD_A_IVTG_; - tx_cmd_b |= skb_vlan_tag_get(skb) & TX_CMD_B_VTAG_MASK_; - } - - ptr = skb_push(skb, 8); - put_unaligned_le32(tx_cmd_a, ptr); - put_unaligned_le32(tx_cmd_b, ptr + 4); - - return skb; -} - static enum skb_state defer_bh(struct lan78xx_net *dev, struct sk_buff *skb, struct sk_buff_head *list, enum skb_state state) { @@ -3126,12 +3265,13 @@ static enum skb_state defer_bh(struct lan78xx_net *dev, struct sk_buff *skb, __skb_unlink(skb, list); spin_unlock(&list->lock); - spin_lock(&dev->done.lock); + spin_lock(&dev->rxq_done.lock); + + __skb_queue_tail(&dev->rxq_done, skb); + if (skb_queue_len(&dev->rxq_done) == 1) + napi_schedule(&dev->napi); - __skb_queue_tail(&dev->done, skb); - if (skb_queue_len(&dev->done) == 1) - tasklet_schedule(&dev->bh); - spin_unlock_irqrestore(&dev->done.lock, flags); + spin_unlock_irqrestore(&dev->rxq_done.lock, flags); return old_state; } @@ -3146,7 +3286,7 @@ static void tx_complete(struct urb *urb) dev->net->stats.tx_packets += entry->num_of_packet; dev->net->stats.tx_bytes += entry->length; } else { - dev->net->stats.tx_errors++; + dev->net->stats.tx_errors += entry->num_of_packet; switch (urb->status) { case -EPIPE: @@ -3179,7 +3319,15 @@ static void tx_complete(struct urb *urb) usb_autopm_put_interface_async(dev->intf); - defer_bh(dev, skb, &dev->txq, tx_done); + skb_unlink(skb, &dev->txq); + + lan78xx_release_tx_buf(dev, skb); + + /* Re-schedule NAPI if Tx data pending but no URBs in progress. + */ + if (skb_queue_empty(&dev->txq) && + !skb_queue_empty(&dev->txq_pend)) + napi_schedule(&dev->napi); } static void lan78xx_queue_skb(struct sk_buff_head *list, @@ -3191,35 +3339,96 @@ static void lan78xx_queue_skb(struct sk_buff_head *list, entry->state = state; } +static unsigned int lan78xx_tx_urb_space(struct lan78xx_net *dev) +{ + return skb_queue_len(&dev->txq_free) * dev->tx_urb_size; +} + +static unsigned int lan78xx_tx_pend_data_len(struct lan78xx_net *dev) +{ + return dev->tx_pend_data_len; +} + +static void lan78xx_tx_pend_skb_add(struct lan78xx_net *dev, + struct sk_buff *skb, + unsigned int *tx_pend_data_len) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->txq_pend.lock, flags); + + __skb_queue_tail(&dev->txq_pend, skb); + + dev->tx_pend_data_len += skb->len; + *tx_pend_data_len = dev->tx_pend_data_len; + + spin_unlock_irqrestore(&dev->txq_pend.lock, flags); +} + +static void lan78xx_tx_pend_skb_head_add(struct lan78xx_net *dev, + struct sk_buff *skb, + unsigned int *tx_pend_data_len) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->txq_pend.lock, flags); + + __skb_queue_head(&dev->txq_pend, skb); + + dev->tx_pend_data_len += skb->len; + *tx_pend_data_len = dev->tx_pend_data_len; + + spin_unlock_irqrestore(&dev->txq_pend.lock, flags); +} + +static void lan78xx_tx_pend_skb_get(struct lan78xx_net *dev, + struct sk_buff **skb, + unsigned int *tx_pend_data_len) +{ + unsigned long flags; + + spin_lock_irqsave(&dev->txq_pend.lock, flags); + + *skb = __skb_dequeue(&dev->txq_pend); + if (*skb) + dev->tx_pend_data_len -= (*skb)->len; + *tx_pend_data_len = dev->tx_pend_data_len; + + spin_unlock_irqrestore(&dev->txq_pend.lock, flags); +} + static netdev_tx_t lan78xx_start_xmit(struct sk_buff *skb, struct net_device *net) { struct lan78xx_net *dev = netdev_priv(net); - struct sk_buff *skb2 = NULL; + unsigned int tx_pend_data_len; if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) schedule_delayed_work(&dev->wq, 0); - if (skb) { - skb_tx_timestamp(skb); - skb2 = lan78xx_tx_prep(dev, skb, GFP_ATOMIC); - } + skb_tx_timestamp(skb); - if (skb2) { - skb_queue_tail(&dev->txq_pend, skb2); + lan78xx_tx_pend_skb_add(dev, skb, &tx_pend_data_len); - /* throttle TX patch at slower than SUPER SPEED USB */ - if ((dev->udev->speed < USB_SPEED_SUPER) && - (skb_queue_len(&dev->txq_pend) > 10)) - netif_stop_queue(net); - } else { - netif_dbg(dev, tx_err, dev->net, - "lan78xx_tx_prep return NULL\n"); - dev->net->stats.tx_errors++; - dev->net->stats.tx_dropped++; - } + /* Set up a Tx URB if none is in progress */ + + if (skb_queue_empty(&dev->txq)) + napi_schedule(&dev->napi); + + /* Stop stack Tx queue if we have enough data to fill + * all the free Tx URBs. + */ + if (tx_pend_data_len > lan78xx_tx_urb_space(dev)) { + netif_stop_queue(net); + + netif_dbg(dev, hw, dev->net, "tx data len: %u, urb space %u", + tx_pend_data_len, lan78xx_tx_urb_space(dev)); - tasklet_schedule(&dev->bh); + /* Kick off transmission of pending data */ + + if (!skb_queue_empty(&dev->txq_free)) + napi_schedule(&dev->napi); + } return NETDEV_TX_OK; } @@ -3276,9 +3485,6 @@ static int lan78xx_bind(struct lan78xx_net *dev, struct usb_interface *intf) goto out1; } - dev->net->hard_header_len += TX_OVERHEAD; - dev->hard_mtu = dev->net->mtu + dev->net->hard_header_len; - /* Init all registers */ ret = lan78xx_reset(dev); if (ret) { @@ -3357,8 +3563,6 @@ static void lan78xx_rx_vlan_offload(struct lan78xx_net *dev, static void lan78xx_skb_return(struct lan78xx_net *dev, struct sk_buff *skb) { - int status; - dev->net->stats.rx_packets++; dev->net->stats.rx_bytes += skb->len; @@ -3371,21 +3575,21 @@ static void lan78xx_skb_return(struct lan78xx_net *dev, struct sk_buff *skb) if (skb_defer_rx_timestamp(skb)) return; - status = netif_rx(skb); - if (status != NET_RX_SUCCESS) - netif_dbg(dev, rx_err, dev->net, - "netif_rx status %d\n", status); + napi_gro_receive(&dev->napi, skb); } -static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb) +static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb, + int budget, int *work_done) { - if (skb->len < dev->net->hard_header_len) + if (skb->len < RX_SKB_MIN_LEN) return 0; + /* Extract frames from the URB buffer and pass each one to + * the stack in a new NAPI SKB. + */ while (skb->len > 0) { u32 rx_cmd_a, rx_cmd_b, align_count, size; u16 rx_cmd_c; - struct sk_buff *skb2; unsigned char *packet; rx_cmd_a = get_unaligned_le32(skb->data); @@ -3407,41 +3611,36 @@ static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb) netif_dbg(dev, rx_err, dev->net, "Error rx_cmd_a=0x%08x", rx_cmd_a); } else { - /* last frame in this batch */ - if (skb->len == size) { - lan78xx_rx_csum_offload(dev, skb, - rx_cmd_a, rx_cmd_b); - lan78xx_rx_vlan_offload(dev, skb, - rx_cmd_a, rx_cmd_b); - - skb_trim(skb, skb->len - 4); /* remove fcs */ - skb->truesize = size + sizeof(struct sk_buff); - - return 1; - } + u32 frame_len = size - ETH_FCS_LEN; + struct sk_buff *skb2; - skb2 = skb_clone(skb, GFP_ATOMIC); - if (unlikely(!skb2)) { - netdev_warn(dev->net, "Error allocating skb"); + skb2 = napi_alloc_skb(&dev->napi, frame_len); + if (!skb2) return 0; - } - skb2->len = size; - skb2->data = packet; - skb_set_tail_pointer(skb2, size); + memcpy(skb2->data, packet, frame_len); + + skb_put(skb2, frame_len); lan78xx_rx_csum_offload(dev, skb2, rx_cmd_a, rx_cmd_b); lan78xx_rx_vlan_offload(dev, skb2, rx_cmd_a, rx_cmd_b); - skb_trim(skb2, skb2->len - 4); /* remove fcs */ - skb2->truesize = size + sizeof(struct sk_buff); - - lan78xx_skb_return(dev, skb2); + /* Processing of the URB buffer must complete once + * it has started. If the NAPI work budget is exhausted + * while frames remain they are added to the overflow + * queue for delivery in the next NAPI polling cycle. + */ + if (*work_done < budget) { + lan78xx_skb_return(dev, skb2); + ++(*work_done); + } else { + skb_queue_tail(&dev->rxq_overflow, skb2); + } } skb_pull(skb, size); - /* padding bytes before the next frame starts */ + /* skip padding bytes before the next frame starts */ if (skb->len) skb_pull(skb, align_count); } @@ -3449,85 +3648,13 @@ static int lan78xx_rx(struct lan78xx_net *dev, struct sk_buff *skb) return 1; } -static inline void rx_process(struct lan78xx_net *dev, struct sk_buff *skb) +static inline void rx_process(struct lan78xx_net *dev, struct sk_buff *skb, + int budget, int *work_done) { - if (!lan78xx_rx(dev, skb)) { + if (!lan78xx_rx(dev, skb, budget, work_done)) { + netif_dbg(dev, rx_err, dev->net, "drop\n"); dev->net->stats.rx_errors++; - goto done; - } - - if (skb->len) { - lan78xx_skb_return(dev, skb); - return; } - - netif_dbg(dev, rx_err, dev->net, "drop\n"); - dev->net->stats.rx_errors++; -done: - skb_queue_tail(&dev->done, skb); -} - -static void rx_complete(struct urb *urb); - -static int rx_submit(struct lan78xx_net *dev, struct urb *urb, gfp_t flags) -{ - struct sk_buff *skb; - struct skb_data *entry; - unsigned long lockflags; - size_t size = dev->rx_urb_size; - int ret = 0; - - skb = netdev_alloc_skb_ip_align(dev->net, size); - if (!skb) { - usb_free_urb(urb); - return -ENOMEM; - } - - entry = (struct skb_data *)skb->cb; - entry->urb = urb; - entry->dev = dev; - entry->length = 0; - - usb_fill_bulk_urb(urb, dev->udev, dev->pipe_in, - skb->data, size, rx_complete, skb); - - spin_lock_irqsave(&dev->rxq.lock, lockflags); - - if (netif_device_present(dev->net) && - netif_running(dev->net) && - !test_bit(EVENT_RX_HALT, &dev->flags) && - !test_bit(EVENT_DEV_ASLEEP, &dev->flags)) { - ret = usb_submit_urb(urb, GFP_ATOMIC); - switch (ret) { - case 0: - lan78xx_queue_skb(&dev->rxq, skb, rx_start); - break; - case -EPIPE: - lan78xx_defer_kevent(dev, EVENT_RX_HALT); - break; - case -ENODEV: - case -ENOENT: - netif_dbg(dev, ifdown, dev->net, "device gone\n"); - netif_device_detach(dev->net); - break; - case -EHOSTUNREACH: - ret = -ENOLINK; - break; - default: - netif_dbg(dev, rx_err, dev->net, - "rx submit, %d\n", ret); - tasklet_schedule(&dev->bh); - } - } else { - netif_dbg(dev, ifdown, dev->net, "rx: stopped\n"); - ret = -ENOLINK; - } - spin_unlock_irqrestore(&dev->rxq.lock, lockflags); - if (ret) { - dev_kfree_skb_any(skb); - usb_free_urb(urb); - } - return ret; } static void rx_complete(struct urb *urb) @@ -3538,13 +3665,18 @@ static void rx_complete(struct urb *urb) int urb_status = urb->status; enum skb_state state; + netif_dbg(dev, rx_status, dev->net, + "rx done: status %d", urb->status); + skb_put(skb, urb->actual_length); state = rx_done; - entry->urb = NULL; + + if (urb != entry->urb) + netif_warn(dev, rx_err, dev->net, "URB pointer mismatch"); switch (urb_status) { case 0: - if (skb->len < dev->net->hard_header_len) { + if (skb->len < RX_SKB_MIN_LEN) { state = rx_cleanup; dev->net->stats.rx_errors++; dev->net->stats.rx_length_errors++; @@ -3562,16 +3694,12 @@ static void rx_complete(struct urb *urb) netif_dbg(dev, ifdown, dev->net, "rx shutdown, code %d\n", urb_status); state = rx_cleanup; - entry->urb = urb; - urb = NULL; break; case -EPROTO: case -ETIME: case -EILSEQ: dev->net->stats.rx_errors++; state = rx_cleanup; - entry->urb = urb; - urb = NULL; break; /* data overrun ... flush fifo? */ @@ -3587,203 +3715,327 @@ static void rx_complete(struct urb *urb) } state = defer_bh(dev, skb, &dev->rxq, state); +} - if (urb) { - if (netif_running(dev->net) && - !test_bit(EVENT_RX_HALT, &dev->flags) && - state != unlink_start) { - rx_submit(dev, urb, GFP_ATOMIC); - return; +static int rx_submit(struct lan78xx_net *dev, struct sk_buff *skb, gfp_t flags) +{ + struct skb_data *entry = (struct skb_data *)skb->cb; + size_t size = dev->rx_urb_size; + struct urb *urb = entry->urb; + unsigned long lockflags; + int ret = 0; + + usb_fill_bulk_urb(urb, dev->udev, dev->pipe_in, + skb->data, size, rx_complete, skb); + + spin_lock_irqsave(&dev->rxq.lock, lockflags); + + if (netif_device_present(dev->net) && + netif_running(dev->net) && + !test_bit(EVENT_RX_HALT, &dev->flags) && + !test_bit(EVENT_DEV_ASLEEP, &dev->flags)) { + ret = usb_submit_urb(urb, flags); + switch (ret) { + case 0: + lan78xx_queue_skb(&dev->rxq, skb, rx_start); + break; + case -EPIPE: + lan78xx_defer_kevent(dev, EVENT_RX_HALT); + break; + case -ENODEV: + case -ENOENT: + netif_dbg(dev, ifdown, dev->net, "device gone\n"); + netif_device_detach(dev->net); + break; + case -EHOSTUNREACH: + ret = -ENOLINK; + napi_schedule(&dev->napi); + break; + default: + netif_dbg(dev, rx_err, dev->net, + "rx submit, %d\n", ret); + napi_schedule(&dev->napi); + break; } - usb_free_urb(urb); + } else { + netif_dbg(dev, ifdown, dev->net, "rx: stopped\n"); + ret = -ENOLINK; } - netif_dbg(dev, rx_err, dev->net, "no read resubmitted\n"); + spin_unlock_irqrestore(&dev->rxq.lock, lockflags); + + if (ret) + lan78xx_release_rx_buf(dev, skb); + + return ret; } -static void lan78xx_tx_bh(struct lan78xx_net *dev) +static void lan78xx_rx_urb_submit_all(struct lan78xx_net *dev) { - int length; - struct urb *urb = NULL; - struct skb_data *entry; - unsigned long flags; - struct sk_buff_head *tqp = &dev->txq_pend; - struct sk_buff *skb, *skb2; - int ret; - int count, pos; - int skb_totallen, pkt_cnt; - - skb_totallen = 0; - pkt_cnt = 0; - count = 0; - length = 0; - spin_lock_irqsave(&tqp->lock, flags); - skb_queue_walk(tqp, skb) { - if (skb_is_gso(skb)) { - if (!skb_queue_is_first(tqp, skb)) { - /* handle previous packets first */ - break; - } - count = 1; - length = skb->len - TX_OVERHEAD; - __skb_unlink(skb, tqp); - spin_unlock_irqrestore(&tqp->lock, flags); - goto gso_skb; - } + struct sk_buff *rx_buf; - if ((skb_totallen + skb->len) > MAX_SINGLE_PACKET_SIZE) + /* Ensure the maximum number of Rx URBs is submitted + */ + while ((rx_buf = lan78xx_get_rx_buf(dev)) != NULL) { + if (rx_submit(dev, rx_buf, GFP_ATOMIC) != 0) break; - skb_totallen = skb->len + roundup(skb_totallen, sizeof(u32)); - pkt_cnt++; - } - spin_unlock_irqrestore(&tqp->lock, flags); - - /* copy to a single skb */ - skb = alloc_skb(skb_totallen, GFP_ATOMIC); - if (!skb) - goto drop; - - skb_put(skb, skb_totallen); - - for (count = pos = 0; count < pkt_cnt; count++) { - skb2 = skb_dequeue(tqp); - if (skb2) { - length += (skb2->len - TX_OVERHEAD); - memcpy(skb->data + pos, skb2->data, skb2->len); - pos += roundup(skb2->len, sizeof(u32)); - dev_kfree_skb(skb2); - } } +} -gso_skb: - urb = usb_alloc_urb(0, GFP_ATOMIC); - if (!urb) - goto drop; +static void lan78xx_rx_urb_resubmit(struct lan78xx_net *dev, + struct sk_buff *rx_buf) +{ + /* reset SKB data pointers */ - entry = (struct skb_data *)skb->cb; - entry->urb = urb; - entry->dev = dev; - entry->length = length; - entry->num_of_packet = count; + rx_buf->data = rx_buf->head; + skb_reset_tail_pointer(rx_buf); + rx_buf->len = 0; + rx_buf->data_len = 0; - spin_lock_irqsave(&dev->txq.lock, flags); - ret = usb_autopm_get_interface_async(dev->intf); - if (ret < 0) { - spin_unlock_irqrestore(&dev->txq.lock, flags); - goto drop; - } + rx_submit(dev, rx_buf, GFP_ATOMIC); +} - usb_fill_bulk_urb(urb, dev->udev, dev->pipe_out, - skb->data, skb->len, tx_complete, skb); +static void lan78xx_fill_tx_cmd_words(struct sk_buff *skb, u8 *buffer) +{ + u32 tx_cmd_a; + u32 tx_cmd_b; - if (length % dev->maxpacket == 0) { - /* send USB_ZERO_PACKET */ - urb->transfer_flags |= URB_ZERO_PACKET; - } + tx_cmd_a = (u32)(skb->len & TX_CMD_A_LEN_MASK_) | TX_CMD_A_FCS_; -#ifdef CONFIG_PM - /* if this triggers the device is still a sleep */ - if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) { - /* transmission will be done in resume */ - usb_anchor_urb(urb, &dev->deferred); - /* no use to process more packets */ - netif_stop_queue(dev->net); - usb_put_urb(urb); - spin_unlock_irqrestore(&dev->txq.lock, flags); - netdev_dbg(dev->net, "Delaying transmission for resumption\n"); - return; + if (skb->ip_summed == CHECKSUM_PARTIAL) + tx_cmd_a |= TX_CMD_A_IPE_ | TX_CMD_A_TPE_; + + tx_cmd_b = 0; + if (skb_is_gso(skb)) { + u16 mss = max(skb_shinfo(skb)->gso_size, TX_CMD_B_MSS_MIN_); + + tx_cmd_b = (mss << TX_CMD_B_MSS_SHIFT_) & TX_CMD_B_MSS_MASK_; + + tx_cmd_a |= TX_CMD_A_LSO_; } -#endif - ret = usb_submit_urb(urb, GFP_ATOMIC); - switch (ret) { - case 0: - netif_trans_update(dev->net); - lan78xx_queue_skb(&dev->txq, skb, tx_start); - if (skb_queue_len(&dev->txq) >= dev->tx_qlen) - netif_stop_queue(dev->net); - break; - case -EPIPE: - netif_stop_queue(dev->net); - lan78xx_defer_kevent(dev, EVENT_TX_HALT); - usb_autopm_put_interface_async(dev->intf); - break; - case -ENODEV: - case -ENOENT: - netif_dbg(dev, tx_err, dev->net, - "tx: submit urb err %d (disconnected?)", ret); - netif_device_detach(dev->net); - break; - default: - usb_autopm_put_interface_async(dev->intf); - netif_dbg(dev, tx_err, dev->net, - "tx: submit urb err %d\n", ret); - break; + if (skb_vlan_tag_present(skb)) { + tx_cmd_a |= TX_CMD_A_IVTG_; + tx_cmd_b |= skb_vlan_tag_get(skb) & TX_CMD_B_VTAG_MASK_; } - spin_unlock_irqrestore(&dev->txq.lock, flags); + put_unaligned_le32(tx_cmd_a, buffer); + put_unaligned_le32(tx_cmd_b, buffer + 4); +} - if (ret) { - netif_dbg(dev, tx_err, dev->net, "drop, code %d\n", ret); -drop: - dev->net->stats.tx_dropped++; - if (skb) +static struct skb_data *lan78xx_tx_buf_fill(struct lan78xx_net *dev, + struct sk_buff *tx_buf) +{ + struct skb_data *entry = (struct skb_data *)tx_buf->cb; + int remain = dev->tx_urb_size; + u8 *tx_data = tx_buf->data; + u32 urb_len = 0; + + entry->num_of_packet = 0; + entry->length = 0; + + /* Work through the pending SKBs and copy the data of each SKB into + * the URB buffer if there room for all the SKB data. + * + * There must be at least DST+SRC+TYPE in the SKB (with padding enabled) + */ + while (remain >= TX_SKB_MIN_LEN) { + unsigned int pending_bytes; + unsigned int align_bytes; + struct sk_buff *skb; + unsigned int len; + + lan78xx_tx_pend_skb_get(dev, &skb, &pending_bytes); + + if (!skb) + break; + + align_bytes = (TX_ALIGNMENT - (urb_len % TX_ALIGNMENT)) % + TX_ALIGNMENT; + len = align_bytes + TX_CMD_LEN + skb->len; + if (len > remain) { + lan78xx_tx_pend_skb_head_add(dev, skb, &pending_bytes); + break; + } + + tx_data += align_bytes; + + lan78xx_fill_tx_cmd_words(skb, tx_data); + tx_data += TX_CMD_LEN; + + len = skb->len; + if (skb_copy_bits(skb, 0, tx_data, len) < 0) { + struct net_device_stats *stats = &dev->net->stats; + + stats->tx_dropped++; dev_kfree_skb_any(skb); - usb_free_urb(urb); - } else { - netif_dbg(dev, tx_queued, dev->net, - "> tx, len %d, type 0x%x\n", length, skb->protocol); + tx_data -= TX_CMD_LEN; + continue; + } + + tx_data += len; + entry->length += len; + entry->num_of_packet += skb_shinfo(skb)->gso_segs ?: 1; + + dev_kfree_skb_any(skb); + + urb_len = (u32)(tx_data - (u8 *)tx_buf->data); + + remain = dev->tx_urb_size - urb_len; } + + skb_put(tx_buf, urb_len); + + return entry; } -static void lan78xx_rx_bh(struct lan78xx_net *dev) +static void lan78xx_tx_bh(struct lan78xx_net *dev) { - struct urb *urb; - int i; + int ret; - if (skb_queue_len(&dev->rxq) < dev->rx_qlen) { - for (i = 0; i < 10; i++) { - if (skb_queue_len(&dev->rxq) >= dev->rx_qlen) - break; - urb = usb_alloc_urb(0, GFP_ATOMIC); - if (urb) - if (rx_submit(dev, urb, GFP_ATOMIC) == -ENOLINK) - return; + /* Start the stack Tx queue if it was stopped + */ + netif_tx_lock(dev->net); + if (netif_queue_stopped(dev->net)) { + if (lan78xx_tx_pend_data_len(dev) < lan78xx_tx_urb_space(dev)) + netif_wake_queue(dev->net); + } + netif_tx_unlock(dev->net); + + /* Go through the Tx pending queue and set up URBs to transfer + * the data to the device. Stop if no more pending data or URBs, + * or if an error occurs when a URB is submitted. + */ + do { + struct skb_data *entry; + struct sk_buff *tx_buf; + unsigned long flags; + + if (skb_queue_empty(&dev->txq_pend)) + break; + + tx_buf = lan78xx_get_tx_buf(dev); + if (!tx_buf) + break; + + entry = lan78xx_tx_buf_fill(dev, tx_buf); + + spin_lock_irqsave(&dev->txq.lock, flags); + ret = usb_autopm_get_interface_async(dev->intf); + if (ret < 0) { + spin_unlock_irqrestore(&dev->txq.lock, flags); + goto out; } - if (skb_queue_len(&dev->rxq) < dev->rx_qlen) - tasklet_schedule(&dev->bh); - } - if (skb_queue_len(&dev->txq) < dev->tx_qlen) - netif_wake_queue(dev->net); + usb_fill_bulk_urb(entry->urb, dev->udev, dev->pipe_out, + tx_buf->data, tx_buf->len, tx_complete, + tx_buf); + + if (tx_buf->len % dev->maxpacket == 0) { + /* send USB_ZERO_PACKET */ + entry->urb->transfer_flags |= URB_ZERO_PACKET; + } + +#ifdef CONFIG_PM + /* if device is asleep stop outgoing packet processing */ + if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) { + usb_anchor_urb(entry->urb, &dev->deferred); + netif_stop_queue(dev->net); + spin_unlock_irqrestore(&dev->txq.lock, flags); + netdev_dbg(dev->net, + "Delaying transmission for resumption\n"); + return; + } +#endif + ret = usb_submit_urb(entry->urb, GFP_ATOMIC); + switch (ret) { + case 0: + netif_trans_update(dev->net); + lan78xx_queue_skb(&dev->txq, tx_buf, tx_start); + break; + case -EPIPE: + netif_stop_queue(dev->net); + lan78xx_defer_kevent(dev, EVENT_TX_HALT); + usb_autopm_put_interface_async(dev->intf); + break; + case -ENODEV: + case -ENOENT: + netif_dbg(dev, tx_err, dev->net, + "tx submit urb err %d (disconnected?)", ret); + netif_device_detach(dev->net); + break; + default: + usb_autopm_put_interface_async(dev->intf); + netif_dbg(dev, tx_err, dev->net, + "tx submit urb err %d\n", ret); + break; + } + + spin_unlock_irqrestore(&dev->txq.lock, flags); + + if (ret) { + netdev_warn(dev->net, "failed to tx urb %d\n", ret); +out: + dev->net->stats.tx_dropped += entry->num_of_packet; + lan78xx_release_tx_buf(dev, tx_buf); + } + } while (ret == 0); } -static void lan78xx_bh(struct tasklet_struct *t) +static int lan78xx_bh(struct lan78xx_net *dev, int budget) { - struct lan78xx_net *dev = from_tasklet(dev, t, bh); - struct sk_buff *skb; + struct sk_buff_head done; + struct sk_buff *rx_buf; struct skb_data *entry; + unsigned long flags; + int work_done = 0; + + /* Pass frames received in the last NAPI cycle before + * working on newly completed URBs. + */ + while (!skb_queue_empty(&dev->rxq_overflow)) { + lan78xx_skb_return(dev, skb_dequeue(&dev->rxq_overflow)); + ++work_done; + } + + /* Take a snapshot of the done queue and move items to a + * temporary queue. Rx URB completions will continue to add + * to the done queue. + */ + __skb_queue_head_init(&done); - while ((skb = skb_dequeue(&dev->done))) { - entry = (struct skb_data *)(skb->cb); + spin_lock_irqsave(&dev->rxq_done.lock, flags); + skb_queue_splice_init(&dev->rxq_done, &done); + spin_unlock_irqrestore(&dev->rxq_done.lock, flags); + + /* Extract receive frames from completed URBs and + * pass them to the stack. Re-submit each completed URB. + */ + while ((work_done < budget) && + (rx_buf = __skb_dequeue(&done))) { + entry = (struct skb_data *)(rx_buf->cb); switch (entry->state) { case rx_done: - entry->state = rx_cleanup; - rx_process(dev, skb); - continue; - case tx_done: - usb_free_urb(entry->urb); - dev_kfree_skb(skb); - continue; + rx_process(dev, rx_buf, budget, &work_done); + break; case rx_cleanup: - usb_free_urb(entry->urb); - dev_kfree_skb(skb); - continue; + break; default: - netdev_dbg(dev->net, "skb state %d\n", entry->state); - return; + netdev_dbg(dev->net, "rx buf state %d\n", + entry->state); + break; } + + lan78xx_rx_urb_resubmit(dev, rx_buf); } + /* If budget was consumed before processing all the URBs put them + * back on the front of the done queue. They will be first to be + * processed in the next NAPI cycle. + */ + spin_lock_irqsave(&dev->rxq_done.lock, flags); + skb_queue_splice(&done, &dev->rxq_done); + spin_unlock_irqrestore(&dev->rxq_done.lock, flags); + if (netif_device_present(dev->net) && netif_running(dev->net)) { /* reset update timer delta */ if (timer_pending(&dev->stat_monitor) && (dev->delta != 1)) { @@ -3792,12 +4044,61 @@ static void lan78xx_bh(struct tasklet_struct *t) jiffies + STAT_UPDATE_TIMER); } - if (!skb_queue_empty(&dev->txq_pend)) - lan78xx_tx_bh(dev); + /* Submit all free Rx URBs */ if (!test_bit(EVENT_RX_HALT, &dev->flags)) - lan78xx_rx_bh(dev); + lan78xx_rx_urb_submit_all(dev); + + /* Submit new Tx URBs */ + + lan78xx_tx_bh(dev); } + + return work_done; +} + +static int lan78xx_poll(struct napi_struct *napi, int budget) +{ + struct lan78xx_net *dev = container_of(napi, struct lan78xx_net, napi); + int result = budget; + int work_done; + + /* Don't do any work if the device is suspended */ + + if (test_bit(EVENT_DEV_ASLEEP, &dev->flags)) { + napi_complete_done(napi, 0); + return 0; + } + + /* Process completed URBs and submit new URBs */ + + work_done = lan78xx_bh(dev, budget); + + if (work_done < budget) { + napi_complete_done(napi, work_done); + + /* Start a new polling cycle if data was received or + * data is waiting to be transmitted. + */ + if (!skb_queue_empty(&dev->rxq_done)) { + napi_schedule(napi); + } else if (netif_carrier_ok(dev->net)) { + if (skb_queue_empty(&dev->txq) && + !skb_queue_empty(&dev->txq_pend)) { + napi_schedule(napi); + } else { + netif_tx_lock(dev->net); + if (netif_queue_stopped(dev->net)) { + netif_wake_queue(dev->net); + napi_schedule(napi); + } + netif_tx_unlock(dev->net); + } + } + result = work_done; + } + + return result; } static void lan78xx_delayedwork(struct work_struct *work) @@ -3843,7 +4144,7 @@ static void lan78xx_delayedwork(struct work_struct *work) status); } else { clear_bit(EVENT_RX_HALT, &dev->flags); - tasklet_schedule(&dev->bh); + napi_schedule(&dev->napi); } } @@ -3937,6 +4238,8 @@ static void lan78xx_disconnect(struct usb_interface *intf) set_bit(EVENT_DEV_DISCONNECT, &dev->flags); + netif_napi_del(&dev->napi); + udev = interface_to_usbdev(intf); net = dev->net; @@ -3961,6 +4264,9 @@ static void lan78xx_disconnect(struct usb_interface *intf) lan78xx_unbind(dev, intf); + lan78xx_free_tx_resources(dev); + lan78xx_free_rx_resources(dev); + usb_kill_urb(dev->urb_intr); usb_free_urb(dev->urb_intr); @@ -3973,14 +4279,16 @@ static void lan78xx_tx_timeout(struct net_device *net, unsigned int txqueue) struct lan78xx_net *dev = netdev_priv(net); unlink_urbs(dev, &dev->txq); - tasklet_schedule(&dev->bh); + napi_schedule(&dev->napi); } static netdev_features_t lan78xx_features_check(struct sk_buff *skb, struct net_device *netdev, netdev_features_t features) { - if (skb->len + TX_OVERHEAD > MAX_SINGLE_PACKET_SIZE) + struct lan78xx_net *dev = netdev_priv(netdev); + + if (skb->len > LAN78XX_TSO_SIZE(dev)) features &= ~NETIF_F_GSO_MASK; features = vlan_features_check(skb, features); @@ -4046,12 +4354,31 @@ static int lan78xx_probe(struct usb_interface *intf, skb_queue_head_init(&dev->rxq); skb_queue_head_init(&dev->txq); - skb_queue_head_init(&dev->done); + skb_queue_head_init(&dev->rxq_done); skb_queue_head_init(&dev->txq_pend); + skb_queue_head_init(&dev->rxq_overflow); mutex_init(&dev->phy_mutex); mutex_init(&dev->dev_mutex); - tasklet_setup(&dev->bh, lan78xx_bh); + ret = lan78xx_urb_config_init(dev); + if (ret < 0) + goto out2; + + ret = lan78xx_alloc_tx_resources(dev); + if (ret < 0) + goto out2; + + ret = lan78xx_alloc_rx_resources(dev); + if (ret < 0) + goto out3; + + /* MTU range: 68 - 9000 */ + netdev->max_mtu = MAX_SINGLE_PACKET_SIZE; + + netif_set_gso_max_size(netdev, LAN78XX_TSO_SIZE(dev)); + + netif_napi_add(netdev, &dev->napi, lan78xx_poll, LAN78XX_NAPI_WEIGHT); + INIT_DELAYED_WORK(&dev->wq, lan78xx_delayedwork); init_usb_anchor(&dev->deferred); @@ -4066,27 +4393,27 @@ static int lan78xx_probe(struct usb_interface *intf, if (intf->cur_altsetting->desc.bNumEndpoints < 3) { ret = -ENODEV; - goto out2; + goto out4; } dev->pipe_in = usb_rcvbulkpipe(udev, BULK_IN_PIPE); ep_blkin = usb_pipe_endpoint(udev, dev->pipe_in); if (!ep_blkin || !usb_endpoint_is_bulk_in(&ep_blkin->desc)) { ret = -ENODEV; - goto out2; + goto out4; } dev->pipe_out = usb_sndbulkpipe(udev, BULK_OUT_PIPE); ep_blkout = usb_pipe_endpoint(udev, dev->pipe_out); if (!ep_blkout || !usb_endpoint_is_bulk_out(&ep_blkout->desc)) { ret = -ENODEV; - goto out2; + goto out4; } ep_intr = &intf->cur_altsetting->endpoint[2]; if (!usb_endpoint_is_int_in(&ep_intr->desc)) { ret = -ENODEV; - goto out2; + goto out4; } dev->pipe_intr = usb_rcvintpipe(dev->udev, @@ -4094,30 +4421,25 @@ static int lan78xx_probe(struct usb_interface *intf, ret = lan78xx_bind(dev, intf); if (ret < 0) - goto out2; - - if (netdev->mtu > (dev->hard_mtu - netdev->hard_header_len)) - netdev->mtu = dev->hard_mtu - netdev->hard_header_len; - - /* MTU range: 68 - 9000 */ - netdev->max_mtu = MAX_SINGLE_PACKET_SIZE; - netif_set_gso_max_size(netdev, MAX_SINGLE_PACKET_SIZE - MAX_HEADER); + goto out4; period = ep_intr->desc.bInterval; maxp = usb_maxpacket(dev->udev, dev->pipe_intr, 0); buf = kmalloc(maxp, GFP_KERNEL); - if (buf) { - dev->urb_intr = usb_alloc_urb(0, GFP_KERNEL); - if (!dev->urb_intr) { - ret = -ENOMEM; - kfree(buf); - goto out3; - } else { - usb_fill_int_urb(dev->urb_intr, dev->udev, - dev->pipe_intr, buf, maxp, - intr_complete, dev, period); - dev->urb_intr->transfer_flags |= URB_FREE_BUFFER; - } + if (!buf) { + ret = -ENOMEM; + goto out5; + } + + dev->urb_intr = usb_alloc_urb(0, GFP_KERNEL); + if (!dev->urb_intr) { + ret = -ENOMEM; + goto out6; + } else { + usb_fill_int_urb(dev->urb_intr, dev->udev, + dev->pipe_intr, buf, maxp, + intr_complete, dev, period); + dev->urb_intr->transfer_flags |= URB_FREE_BUFFER; } dev->maxpacket = usb_maxpacket(dev->udev, dev->pipe_out, 1); @@ -4125,7 +4447,7 @@ static int lan78xx_probe(struct usb_interface *intf, /* Reject broken descriptors. */ if (dev->maxpacket == 0) { ret = -ENODEV; - goto out4; + goto out6; } /* driver requires remote-wakeup capability during autosuspend. */ @@ -4133,12 +4455,12 @@ static int lan78xx_probe(struct usb_interface *intf, ret = lan78xx_phy_init(dev); if (ret < 0) - goto out4; + goto out7; ret = register_netdev(netdev); if (ret != 0) { netif_err(dev, probe, netdev, "couldn't register the device\n"); - goto out5; + goto out8; } usb_set_intfdata(intf, dev); @@ -4153,12 +4475,19 @@ static int lan78xx_probe(struct usb_interface *intf, return 0; -out5: +out8: phy_disconnect(netdev->phydev); -out4: +out7: usb_free_urb(dev->urb_intr); -out3: +out6: + kfree(buf); +out5: lan78xx_unbind(dev, intf); +out4: + netif_napi_del(&dev->napi); + lan78xx_free_rx_resources(dev); +out3: + lan78xx_free_tx_resources(dev); out2: free_netdev(netdev); out1: @@ -4579,8 +4908,7 @@ static bool lan78xx_submit_deferred_urbs(struct lan78xx_net *dev) if (!netif_device_present(dev->net) || !netif_carrier_ok(dev->net) || pipe_halted) { - usb_free_urb(urb); - dev_kfree_skb(skb); + lan78xx_release_tx_buf(dev, skb); continue; } @@ -4590,15 +4918,14 @@ static bool lan78xx_submit_deferred_urbs(struct lan78xx_net *dev) netif_trans_update(dev->net); lan78xx_queue_skb(&dev->txq, skb, tx_start); } else { - usb_free_urb(urb); - dev_kfree_skb(skb); - if (ret == -EPIPE) { netif_stop_queue(dev->net); pipe_halted = true; } else if (ret == -ENODEV) { netif_device_detach(dev->net); } + + lan78xx_release_tx_buf(dev, skb); } } @@ -4630,8 +4957,7 @@ static int lan78xx_resume(struct usb_interface *intf) if (ret < 0) { if (ret == -ENODEV) netif_device_detach(dev->net); - - netdev_warn(dev->net, "Failed to submit intr URB"); + netdev_warn(dev->net, "Failed to submit intr URB"); } } @@ -4650,14 +4976,14 @@ static int lan78xx_resume(struct usb_interface *intf) if (!pipe_halted && netif_device_present(dev->net) && - (skb_queue_len(&dev->txq) < dev->tx_qlen)) + (lan78xx_tx_pend_data_len(dev) < lan78xx_tx_urb_space(dev))) netif_start_queue(dev->net); ret = lan78xx_start_tx_path(dev); if (ret < 0) goto out; - tasklet_schedule(&dev->bh); + napi_schedule(&dev->napi); if (!timer_pending(&dev->stat_monitor)) { dev->delta = 1; diff --git a/drivers/net/usb/r8152.c b/drivers/net/usb/r8152.c index f9877a3e83ac..b88f07c77a6b 100644 --- a/drivers/net/usb/r8152.c +++ b/drivers/net/usb/r8152.c @@ -8983,7 +8983,9 @@ static int rtl8152_set_tunable(struct net_device *netdev, } static void rtl8152_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct r8152 *tp = netdev_priv(netdev); @@ -8992,7 +8994,9 @@ static void rtl8152_get_ringparam(struct net_device *netdev, } static int rtl8152_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct r8152 *tp = netdev_priv(netdev); diff --git a/drivers/net/veth.c b/drivers/net/veth.c index 50eb43e5bf45..38f6da24f460 100644 --- a/drivers/net/veth.c +++ b/drivers/net/veth.c @@ -134,29 +134,22 @@ static void veth_get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *inf static void veth_get_strings(struct net_device *dev, u32 stringset, u8 *buf) { - char *p = (char *)buf; + u8 *p = buf; int i, j; switch(stringset) { case ETH_SS_STATS: memcpy(p, ðtool_stats_keys, sizeof(ethtool_stats_keys)); p += sizeof(ethtool_stats_keys); - for (i = 0; i < dev->real_num_rx_queues; i++) { - for (j = 0; j < VETH_RQ_STATS_LEN; j++) { - snprintf(p, ETH_GSTRING_LEN, - "rx_queue_%u_%.18s", - i, veth_rq_stats_desc[j].desc); - p += ETH_GSTRING_LEN; - } - } - for (i = 0; i < dev->real_num_tx_queues; i++) { - for (j = 0; j < VETH_TQ_STATS_LEN; j++) { - snprintf(p, ETH_GSTRING_LEN, - "tx_queue_%u_%.18s", - i, veth_tq_stats_desc[j].desc); - p += ETH_GSTRING_LEN; - } - } + for (i = 0; i < dev->real_num_rx_queues; i++) + for (j = 0; j < VETH_RQ_STATS_LEN; j++) + ethtool_sprintf(&p, "rx_queue_%u_%.18s", + i, veth_rq_stats_desc[j].desc); + + for (i = 0; i < dev->real_num_tx_queues; i++) + for (j = 0; j < VETH_TQ_STATS_LEN; j++) + ethtool_sprintf(&p, "tx_queue_%u_%.18s", + i, veth_tq_stats_desc[j].desc); break; } } @@ -1689,8 +1682,8 @@ static int veth_newlink(struct net *src_net, struct net_device *dev, if (ifmp && (dev->ifindex != 0)) peer->ifindex = ifmp->ifi_index; - peer->gso_max_size = dev->gso_max_size; - peer->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_size(peer, dev->gso_max_size); + netif_set_gso_max_segs(peer, dev->gso_max_segs); err = register_netdevice(peer); put_net(net); diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c index 55db6a336f7e..088d9404c93d 100644 --- a/drivers/net/virtio_net.c +++ b/drivers/net/virtio_net.c @@ -2174,7 +2174,9 @@ static void virtnet_cpu_notif_remove(struct virtnet_info *vi) } static void virtnet_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *ring) + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) { struct virtnet_info *vi = netdev_priv(dev); @@ -2694,7 +2696,7 @@ static void virtnet_tx_timeout(struct net_device *dev, unsigned int txqueue) netdev_err(dev, "TX timeout on queue: %u, sq: %s, vq: 0x%x, name: %s, %u usecs ago\n", txqueue, sq->name, sq->vq->index, sq->vq->name, - jiffies_to_usecs(jiffies - txq->trans_start)); + jiffies_to_usecs(jiffies - READ_ONCE(txq->trans_start))); } static const struct net_device_ops virtnet_netdev = { diff --git a/drivers/net/vmxnet3/vmxnet3_ethtool.c b/drivers/net/vmxnet3/vmxnet3_ethtool.c index 16f3a2057b90..3172d46c0335 100644 --- a/drivers/net/vmxnet3/vmxnet3_ethtool.c +++ b/drivers/net/vmxnet3/vmxnet3_ethtool.c @@ -575,10 +575,11 @@ vmxnet3_get_link_ksettings(struct net_device *netdev, return 0; } - static void vmxnet3_get_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); @@ -595,10 +596,11 @@ vmxnet3_get_ringparam(struct net_device *netdev, param->rx_jumbo_pending = adapter->rx_ring2_size; } - static int vmxnet3_set_ringparam(struct net_device *netdev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct vmxnet3_adapter *adapter = netdev_priv(netdev); u32 new_tx_ring_size, new_rx_ring_size, new_rx_ring2_size; diff --git a/drivers/net/vrf.c b/drivers/net/vrf.c index b2242a082431..b4c64226b7ca 100644 --- a/drivers/net/vrf.c +++ b/drivers/net/vrf.c @@ -814,9 +814,9 @@ static void vrf_rt6_release(struct net_device *dev, struct net_vrf *vrf) */ if (rt6) { dst = &rt6->dst; - dev_put(dst->dev); + dev_replace_track(dst->dev, net->loopback_dev, + &dst->dev_tracker, GFP_KERNEL); dst->dev = net->loopback_dev; - dev_hold(dst->dev); dst_release(dst); } } @@ -1061,9 +1061,9 @@ static void vrf_rtable_release(struct net_device *dev, struct net_vrf *vrf) */ if (rth) { dst = &rth->dst; - dev_put(dst->dev); + dev_replace_track(dst->dev, net->loopback_dev, + &dst->dev_tracker, GFP_KERNEL); dst->dev = net->loopback_dev; - dev_hold(dst->dev); dst_release(dst); } } diff --git a/drivers/net/vxlan.c b/drivers/net/vxlan.c index 141635a35c28..359d16780dbb 100644 --- a/drivers/net/vxlan.c +++ b/drivers/net/vxlan.c @@ -17,6 +17,7 @@ #include <linux/ethtool.h> #include <net/arp.h> #include <net/ndisc.h> +#include <net/gro.h> #include <net/ipv6_stubs.h> #include <net/ip.h> #include <net/icmp.h> @@ -3233,7 +3234,6 @@ static const struct net_device_ops vxlan_netdev_ether_ops = { .ndo_fdb_dump = vxlan_fdb_dump, .ndo_fdb_get = vxlan_fdb_get, .ndo_fill_metadata_dst = vxlan_fill_metadata_dst, - .ndo_change_proto_down = dev_change_proto_down_generic, }; static const struct net_device_ops vxlan_netdev_raw_ops = { @@ -3304,7 +3304,7 @@ static void vxlan_setup(struct net_device *dev) dev->hw_features |= NETIF_F_RXCSUM; dev->hw_features |= NETIF_F_GSO_SOFTWARE; netif_keep_dst(dev); - dev->priv_flags |= IFF_NO_QUEUE; + dev->priv_flags |= IFF_NO_QUEUE | IFF_CHANGE_PROTO_DOWN; /* MTU range: 68 - 65535 */ dev->min_mtu = ETH_MIN_MTU; @@ -3747,7 +3747,7 @@ static int vxlan_config_validate(struct net *src_net, struct vxlan_config *conf, if (!conf->dst_port) { if (conf->flags & VXLAN_F_GPE) - conf->dst_port = htons(4790); /* IANA VXLAN-GPE port */ + conf->dst_port = htons(IANA_VXLAN_GPE_UDP_PORT); else conf->dst_port = htons(vxlan_port); } @@ -3810,8 +3810,8 @@ static void vxlan_config_apply(struct net_device *dev, if (lowerdev) { dst->remote_ifindex = conf->remote_ifindex; - dev->gso_max_size = lowerdev->gso_max_size; - dev->gso_max_segs = lowerdev->gso_max_segs; + netif_set_gso_max_size(dev, lowerdev->gso_max_size); + netif_set_gso_max_segs(dev, lowerdev->gso_max_segs); needed_headroom = lowerdev->hard_header_len; needed_headroom += lowerdev->needed_headroom; diff --git a/drivers/net/wan/fsl_ucc_hdlc.c b/drivers/net/wan/fsl_ucc_hdlc.c index cda1b4ce6b21..5ae2d27b5da9 100644 --- a/drivers/net/wan/fsl_ucc_hdlc.c +++ b/drivers/net/wan/fsl_ucc_hdlc.c @@ -306,9 +306,8 @@ static int uhdlc_init(struct ucc_hdlc_private *priv) else bd_status = R_E_S | R_I_S | R_W_S; - iowrite16be(bd_status, &priv->rx_bd_base[i].status); - iowrite32be(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH, - &priv->rx_bd_base[i].buf); + priv->rx_bd_base[i].status = cpu_to_be16(bd_status); + priv->rx_bd_base[i].buf = cpu_to_be32(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH); } for (i = 0; i < TX_BD_RING_LEN; i++) { @@ -317,10 +316,10 @@ static int uhdlc_init(struct ucc_hdlc_private *priv) else bd_status = T_I_S | T_TC_S | T_W_S; - iowrite16be(bd_status, &priv->tx_bd_base[i].status); - iowrite32be(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH, - &priv->tx_bd_base[i].buf); + priv->tx_bd_base[i].status = cpu_to_be16(bd_status); + priv->tx_bd_base[i].buf = cpu_to_be32(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH); } + dma_wmb(); return 0; @@ -352,10 +351,10 @@ static netdev_tx_t ucc_hdlc_tx(struct sk_buff *skb, struct net_device *dev) { hdlc_device *hdlc = dev_to_hdlc(dev); struct ucc_hdlc_private *priv = (struct ucc_hdlc_private *)hdlc->priv; - struct qe_bd __iomem *bd; + struct qe_bd *bd; u16 bd_status; unsigned long flags; - u16 *proto_head; + __be16 *proto_head; switch (dev->type) { case ARPHRD_RAWHDLC: @@ -368,14 +367,14 @@ static netdev_tx_t ucc_hdlc_tx(struct sk_buff *skb, struct net_device *dev) skb_push(skb, HDLC_HEAD_LEN); - proto_head = (u16 *)skb->data; + proto_head = (__be16 *)skb->data; *proto_head = htons(DEFAULT_HDLC_HEAD); dev->stats.tx_bytes += skb->len; break; case ARPHRD_PPP: - proto_head = (u16 *)skb->data; + proto_head = (__be16 *)skb->data; if (*proto_head != htons(DEFAULT_PPP_HEAD)) { dev->stats.tx_dropped++; dev_kfree_skb(skb); @@ -398,9 +397,10 @@ static netdev_tx_t ucc_hdlc_tx(struct sk_buff *skb, struct net_device *dev) netdev_sent_queue(dev, skb->len); spin_lock_irqsave(&priv->lock, flags); + dma_rmb(); /* Start from the next BD that should be filled */ bd = priv->curtx_bd; - bd_status = ioread16be(&bd->status); + bd_status = be16_to_cpu(bd->status); /* Save the skb pointer so we can free it later */ priv->tx_skbuff[priv->skb_curtx] = skb; @@ -415,8 +415,8 @@ static netdev_tx_t ucc_hdlc_tx(struct sk_buff *skb, struct net_device *dev) /* set bd status and length */ bd_status = (bd_status & T_W_S) | T_R_S | T_I_S | T_L_S | T_TC_S; - iowrite16be(skb->len, &bd->length); - iowrite16be(bd_status, &bd->status); + bd->length = cpu_to_be16(skb->len); + bd->status = cpu_to_be16(bd_status); /* Move to next BD in the ring */ if (!(bd_status & T_W_S)) @@ -458,8 +458,9 @@ static int hdlc_tx_done(struct ucc_hdlc_private *priv) u16 bd_status; int tx_restart = 0; + dma_rmb(); bd = priv->dirty_tx; - bd_status = ioread16be(&bd->status); + bd_status = be16_to_cpu(bd->status); /* Normal processing. */ while ((bd_status & T_R_S) == 0) { @@ -503,7 +504,7 @@ static int hdlc_tx_done(struct ucc_hdlc_private *priv) bd += 1; else bd = priv->tx_bd_base; - bd_status = ioread16be(&bd->status); + bd_status = be16_to_cpu(bd->status); } priv->dirty_tx = bd; @@ -524,8 +525,9 @@ static int hdlc_rx_done(struct ucc_hdlc_private *priv, int rx_work_limit) u16 length, howmany = 0; u8 *bdbuffer; + dma_rmb(); bd = priv->currx_bd; - bd_status = ioread16be(&bd->status); + bd_status = be16_to_cpu(bd->status); /* while there are received buffers and BD is full (~R_E) */ while (!((bd_status & (R_E_S)) || (--rx_work_limit < 0))) { @@ -549,7 +551,7 @@ static int hdlc_rx_done(struct ucc_hdlc_private *priv, int rx_work_limit) } bdbuffer = priv->rx_buffer + (priv->currx_bdnum * MAX_RX_BUF_LENGTH); - length = ioread16be(&bd->length); + length = be16_to_cpu(bd->length); switch (dev->type) { case ARPHRD_RAWHDLC: @@ -593,7 +595,7 @@ static int hdlc_rx_done(struct ucc_hdlc_private *priv, int rx_work_limit) netif_receive_skb(skb); recycle: - iowrite16be((bd_status & R_W_S) | R_E_S | R_I_S, &bd->status); + bd->status = cpu_to_be16((bd_status & R_W_S) | R_E_S | R_I_S); /* update to point at the next bd */ if (bd_status & R_W_S) { @@ -608,8 +610,9 @@ recycle: bd += 1; } - bd_status = ioread16be(&bd->status); + bd_status = be16_to_cpu(bd->status); } + dma_rmb(); priv->currx_bd = bd; return howmany; @@ -721,7 +724,7 @@ static int uhdlc_open(struct net_device *dev) /* Enable the TDM port */ if (priv->tsa) - utdm->si_regs->siglmr1_h |= (0x1 << utdm->tdm_port); + qe_setbits_8(&utdm->si_regs->siglmr1_h, 0x1 << utdm->tdm_port); priv->hdlc_busy = 1; netif_device_attach(priv->ndev); @@ -812,7 +815,7 @@ static int uhdlc_close(struct net_device *dev) (u8)QE_CR_PROTOCOL_UNSPECIFIED, 0); if (priv->tsa) - utdm->si_regs->siglmr1_h &= ~(0x1 << utdm->tdm_port); + qe_clrbits_8(&utdm->si_regs->siglmr1_h, 0x1 << utdm->tdm_port); ucc_fast_disable(priv->uccf, COMM_DIR_RX | COMM_DIR_TX); @@ -848,7 +851,7 @@ static int ucc_hdlc_attach(struct net_device *dev, unsigned short encoding, #ifdef CONFIG_PM static void store_clk_config(struct ucc_hdlc_private *priv) { - struct qe_mux *qe_mux_reg = &qe_immr->qmx; + struct qe_mux __iomem *qe_mux_reg = &qe_immr->qmx; /* store si clk */ priv->cmxsi1cr_h = ioread32be(&qe_mux_reg->cmxsi1cr_h); @@ -863,7 +866,7 @@ static void store_clk_config(struct ucc_hdlc_private *priv) static void resume_clk_config(struct ucc_hdlc_private *priv) { - struct qe_mux *qe_mux_reg = &qe_immr->qmx; + struct qe_mux __iomem *qe_mux_reg = &qe_immr->qmx; memcpy_toio(qe_mux_reg->cmxucr, priv->cmxucr, 4 * sizeof(u32)); @@ -990,9 +993,8 @@ static int uhdlc_resume(struct device *dev) else bd_status = R_E_S | R_I_S | R_W_S; - iowrite16be(bd_status, &priv->rx_bd_base[i].status); - iowrite32be(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH, - &priv->rx_bd_base[i].buf); + priv->rx_bd_base[i].status = cpu_to_be16(bd_status); + priv->rx_bd_base[i].buf = cpu_to_be32(priv->dma_rx_addr + i * MAX_RX_BUF_LENGTH); } for (i = 0; i < TX_BD_RING_LEN; i++) { @@ -1001,10 +1003,10 @@ static int uhdlc_resume(struct device *dev) else bd_status = T_I_S | T_TC_S | T_W_S; - iowrite16be(bd_status, &priv->tx_bd_base[i].status); - iowrite32be(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH, - &priv->tx_bd_base[i].buf); + priv->tx_bd_base[i].status = cpu_to_be16(bd_status); + priv->tx_bd_base[i].buf = cpu_to_be32(priv->dma_tx_addr + i * MAX_RX_BUF_LENGTH); } + dma_wmb(); /* if hdlc is busy enable TX and RX */ if (priv->hdlc_busy == 1) { @@ -1018,7 +1020,7 @@ static int uhdlc_resume(struct device *dev) /* Enable the TDM port */ if (priv->tsa) - utdm->si_regs->siglmr1_h |= (0x1 << utdm->tdm_port); + qe_setbits_8(&utdm->si_regs->siglmr1_h, 0x1 << utdm->tdm_port); } napi_enable(&priv->napi); diff --git a/drivers/net/wan/ixp4xx_hss.c b/drivers/net/wan/ixp4xx_hss.c index 88a36a069311..0b7d9f2f2b8b 100644 --- a/drivers/net/wan/ixp4xx_hss.c +++ b/drivers/net/wan/ixp4xx_hss.c @@ -17,13 +17,19 @@ #include <linux/io.h> #include <linux/kernel.h> #include <linux/platform_device.h> -#include <linux/platform_data/wan_ixp4xx_hss.h> #include <linux/poll.h> #include <linux/slab.h> +#include <linux/gpio/consumer.h> +#include <linux/of.h> #include <linux/soc/ixp4xx/npe.h> #include <linux/soc/ixp4xx/qmgr.h> #include <linux/soc/ixp4xx/cpu.h> +/* This is what all IXP4xx platforms we know uses, if more frequencies + * are needed, we need to migrate to the clock framework. + */ +#define IXP4XX_TIMER_FREQ 66666000 + #define DEBUG_DESC 0 #define DEBUG_RX 0 #define DEBUG_TX 0 @@ -50,7 +56,6 @@ #define NAPI_WEIGHT 16 /* Queue IDs */ -#define HSS0_CHL_RXTRIG_QUEUE 12 /* orig size = 32 dwords */ #define HSS0_PKT_RX_QUEUE 13 /* orig size = 32 dwords */ #define HSS0_PKT_TX0_QUEUE 14 /* orig size = 16 dwords */ #define HSS0_PKT_TX1_QUEUE 15 @@ -62,7 +67,6 @@ #define HSS0_PKT_RXFREE3_QUEUE 21 #define HSS0_PKT_TXDONE_QUEUE 22 /* orig size = 64 dwords */ -#define HSS1_CHL_RXTRIG_QUEUE 10 #define HSS1_PKT_RX_QUEUE 0 #define HSS1_PKT_TX0_QUEUE 5 #define HSS1_PKT_TX1_QUEUE 6 @@ -252,9 +256,19 @@ typedef void buffer_t; struct port { struct device *dev; struct npe *npe; + unsigned int txreadyq; + unsigned int rxtrigq; + unsigned int rxfreeq; + unsigned int rxq; + unsigned int txq; + unsigned int txdoneq; + struct gpio_desc *cts; + struct gpio_desc *rts; + struct gpio_desc *dcd; + struct gpio_desc *dtr; + struct gpio_desc *clk_internal; struct net_device *netdev; struct napi_struct napi; - struct hss_plat_info *plat; buffer_t *rx_buff_tab[RX_DESCS], *tx_buff_tab[TX_DESCS]; struct desc *desc_tab; /* coherent */ dma_addr_t desc_tab_phys; @@ -322,14 +336,6 @@ static int ports_open; static struct dma_pool *dma_pool; static DEFINE_SPINLOCK(npe_lock); -static const struct { - int tx, txdone, rx, rxfree; -} queue_ids[2] = {{HSS0_PKT_TX0_QUEUE, HSS0_PKT_TXDONE_QUEUE, HSS0_PKT_RX_QUEUE, - HSS0_PKT_RXFREE0_QUEUE}, - {HSS1_PKT_TX0_QUEUE, HSS1_PKT_TXDONE_QUEUE, HSS1_PKT_RX_QUEUE, - HSS1_PKT_RXFREE0_QUEUE}, -}; - /***************************************************************************** * utility functions ****************************************************************************/ @@ -645,7 +651,7 @@ static void hss_hdlc_rx_irq(void *pdev) #if DEBUG_RX printk(KERN_DEBUG "%s: hss_hdlc_rx_irq\n", dev->name); #endif - qmgr_disable_irq(queue_ids[port->id].rx); + qmgr_disable_irq(port->rxq); napi_schedule(&port->napi); } @@ -653,8 +659,8 @@ static int hss_hdlc_poll(struct napi_struct *napi, int budget) { struct port *port = container_of(napi, struct port, napi); struct net_device *dev = port->netdev; - unsigned int rxq = queue_ids[port->id].rx; - unsigned int rxfreeq = queue_ids[port->id].rxfree; + unsigned int rxq = port->rxq; + unsigned int rxfreeq = port->rxfreeq; int received = 0; #if DEBUG_RX @@ -795,7 +801,7 @@ static void hss_hdlc_txdone_irq(void *pdev) #if DEBUG_TX printk(KERN_DEBUG DRV_NAME ": hss_hdlc_txdone_irq\n"); #endif - while ((n_desc = queue_get_desc(queue_ids[port->id].txdone, + while ((n_desc = queue_get_desc(port->txdoneq, port, 1)) >= 0) { struct desc *desc; int start; @@ -813,8 +819,8 @@ static void hss_hdlc_txdone_irq(void *pdev) free_buffer_irq(port->tx_buff_tab[n_desc]); port->tx_buff_tab[n_desc] = NULL; - start = qmgr_stat_below_low_watermark(port->plat->txreadyq); - queue_put_desc(port->plat->txreadyq, + start = qmgr_stat_below_low_watermark(port->txreadyq); + queue_put_desc(port->txreadyq, tx_desc_phys(port, n_desc), desc); if (start) { /* TX-ready queue was empty */ #if DEBUG_TX @@ -829,7 +835,7 @@ static void hss_hdlc_txdone_irq(void *pdev) static int hss_hdlc_xmit(struct sk_buff *skb, struct net_device *dev) { struct port *port = dev_to_port(dev); - unsigned int txreadyq = port->plat->txreadyq; + unsigned int txreadyq = port->txreadyq; int len, offset, bytes, n; void *mem; u32 phys; @@ -889,7 +895,7 @@ static int hss_hdlc_xmit(struct sk_buff *skb, struct net_device *dev) desc->buf_len = desc->pkt_len = len; wmb(); - queue_put_desc(queue_ids[port->id].tx, tx_desc_phys(port, n), desc); + queue_put_desc(port->txq, tx_desc_phys(port, n), desc); if (qmgr_stat_below_low_watermark(txreadyq)) { /* empty */ #if DEBUG_TX @@ -916,40 +922,40 @@ static int request_hdlc_queues(struct port *port) { int err; - err = qmgr_request_queue(queue_ids[port->id].rxfree, RX_DESCS, 0, 0, + err = qmgr_request_queue(port->rxfreeq, RX_DESCS, 0, 0, "%s:RX-free", port->netdev->name); if (err) return err; - err = qmgr_request_queue(queue_ids[port->id].rx, RX_DESCS, 0, 0, + err = qmgr_request_queue(port->rxq, RX_DESCS, 0, 0, "%s:RX", port->netdev->name); if (err) goto rel_rxfree; - err = qmgr_request_queue(queue_ids[port->id].tx, TX_DESCS, 0, 0, + err = qmgr_request_queue(port->txq, TX_DESCS, 0, 0, "%s:TX", port->netdev->name); if (err) goto rel_rx; - err = qmgr_request_queue(port->plat->txreadyq, TX_DESCS, 0, 0, + err = qmgr_request_queue(port->txreadyq, TX_DESCS, 0, 0, "%s:TX-ready", port->netdev->name); if (err) goto rel_tx; - err = qmgr_request_queue(queue_ids[port->id].txdone, TX_DESCS, 0, 0, + err = qmgr_request_queue(port->txdoneq, TX_DESCS, 0, 0, "%s:TX-done", port->netdev->name); if (err) goto rel_txready; return 0; rel_txready: - qmgr_release_queue(port->plat->txreadyq); + qmgr_release_queue(port->txreadyq); rel_tx: - qmgr_release_queue(queue_ids[port->id].tx); + qmgr_release_queue(port->txq); rel_rx: - qmgr_release_queue(queue_ids[port->id].rx); + qmgr_release_queue(port->rxq); rel_rxfree: - qmgr_release_queue(queue_ids[port->id].rxfree); + qmgr_release_queue(port->rxfreeq); printk(KERN_DEBUG "%s: unable to request hardware queues\n", port->netdev->name); return err; @@ -957,11 +963,11 @@ rel_rxfree: static void release_hdlc_queues(struct port *port) { - qmgr_release_queue(queue_ids[port->id].rxfree); - qmgr_release_queue(queue_ids[port->id].rx); - qmgr_release_queue(queue_ids[port->id].txdone); - qmgr_release_queue(queue_ids[port->id].tx); - qmgr_release_queue(port->plat->txreadyq); + qmgr_release_queue(port->rxfreeq); + qmgr_release_queue(port->rxq); + qmgr_release_queue(port->txdoneq); + qmgr_release_queue(port->txq); + qmgr_release_queue(port->txreadyq); } static int init_hdlc_queues(struct port *port) @@ -1046,11 +1052,24 @@ static void destroy_hdlc_queues(struct port *port) } } +static irqreturn_t hss_hdlc_dcd_irq(int irq, void *data) +{ + struct net_device *dev = data; + struct port *port = dev_to_port(dev); + int val; + + val = gpiod_get_value(port->dcd); + hss_hdlc_set_carrier(dev, val); + + return IRQ_HANDLED; +} + static int hss_hdlc_open(struct net_device *dev) { struct port *port = dev_to_port(dev); unsigned long flags; int i, err = 0; + int val; err = hdlc_open(dev); if (err) @@ -1069,32 +1088,44 @@ static int hss_hdlc_open(struct net_device *dev) goto err_destroy_queues; spin_lock_irqsave(&npe_lock, flags); - if (port->plat->open) { - err = port->plat->open(port->id, dev, hss_hdlc_set_carrier); - if (err) - goto err_unlock; + + /* Set the carrier, the GPIO is flagged active low so this will return + * 1 if DCD is asserted. + */ + val = gpiod_get_value(port->dcd); + hss_hdlc_set_carrier(dev, val); + + /* Set up an IRQ for DCD */ + err = request_irq(gpiod_to_irq(port->dcd), hss_hdlc_dcd_irq, 0, "IXP4xx HSS", dev); + if (err) { + dev_err(&dev->dev, "ixp4xx_hss: failed to request DCD IRQ (%i)\n", err); + goto err_unlock; } + /* GPIOs are flagged active low so this asserts DTR and RTS */ + gpiod_set_value(port->dtr, 1); + gpiod_set_value(port->rts, 1); + spin_unlock_irqrestore(&npe_lock, flags); /* Populate queues with buffers, no failure after this point */ for (i = 0; i < TX_DESCS; i++) - queue_put_desc(port->plat->txreadyq, + queue_put_desc(port->txreadyq, tx_desc_phys(port, i), tx_desc_ptr(port, i)); for (i = 0; i < RX_DESCS; i++) - queue_put_desc(queue_ids[port->id].rxfree, + queue_put_desc(port->rxfreeq, rx_desc_phys(port, i), rx_desc_ptr(port, i)); napi_enable(&port->napi); netif_start_queue(dev); - qmgr_set_irq(queue_ids[port->id].rx, QUEUE_IRQ_SRC_NOT_EMPTY, + qmgr_set_irq(port->rxq, QUEUE_IRQ_SRC_NOT_EMPTY, hss_hdlc_rx_irq, dev); - qmgr_set_irq(queue_ids[port->id].txdone, QUEUE_IRQ_SRC_NOT_EMPTY, + qmgr_set_irq(port->txdoneq, QUEUE_IRQ_SRC_NOT_EMPTY, hss_hdlc_txdone_irq, dev); - qmgr_enable_irq(queue_ids[port->id].txdone); + qmgr_enable_irq(port->txdoneq); ports_open++; @@ -1125,15 +1156,15 @@ static int hss_hdlc_close(struct net_device *dev) spin_lock_irqsave(&npe_lock, flags); ports_open--; - qmgr_disable_irq(queue_ids[port->id].rx); + qmgr_disable_irq(port->rxq); netif_stop_queue(dev); napi_disable(&port->napi); hss_stop_hdlc(port); - while (queue_get_desc(queue_ids[port->id].rxfree, port, 0) >= 0) + while (queue_get_desc(port->rxfreeq, port, 0) >= 0) buffs--; - while (queue_get_desc(queue_ids[port->id].rx, port, 0) >= 0) + while (queue_get_desc(port->rxq, port, 0) >= 0) buffs--; if (buffs) @@ -1141,12 +1172,12 @@ static int hss_hdlc_close(struct net_device *dev) buffs); buffs = TX_DESCS; - while (queue_get_desc(queue_ids[port->id].tx, port, 1) >= 0) + while (queue_get_desc(port->txq, port, 1) >= 0) buffs--; /* cancel TX */ i = 0; do { - while (queue_get_desc(port->plat->txreadyq, port, 1) >= 0) + while (queue_get_desc(port->txreadyq, port, 1) >= 0) buffs--; if (!buffs) break; @@ -1159,10 +1190,12 @@ static int hss_hdlc_close(struct net_device *dev) if (!buffs) printk(KERN_DEBUG "Draining TX queues took %i cycles\n", i); #endif - qmgr_disable_irq(queue_ids[port->id].txdone); + qmgr_disable_irq(port->txdoneq); - if (port->plat->close) - port->plat->close(port->id, dev); + free_irq(gpiod_to_irq(port->dcd), dev); + /* GPIOs are flagged active low so this de-asserts DTR and RTS */ + gpiod_set_value(port->dtr, 0); + gpiod_set_value(port->rts, 0); spin_unlock_irqrestore(&npe_lock, flags); destroy_hdlc_queues(port); @@ -1254,6 +1287,21 @@ static void find_best_clock(u32 timer_freq, u32 rate, u32 *best, u32 *reg) } } +static int hss_hdlc_set_clock(struct port *port, unsigned int clock_type) +{ + switch (clock_type) { + case CLOCK_DEFAULT: + case CLOCK_EXT: + gpiod_set_value(port->clk_internal, 0); + return CLOCK_EXT; + case CLOCK_INT: + gpiod_set_value(port->clk_internal, 1); + return CLOCK_INT; + default: + return -EINVAL; + } +} + static int hss_hdlc_ioctl(struct net_device *dev, struct if_settings *ifs) { const size_t size = sizeof(sync_serial_settings); @@ -1286,8 +1334,7 @@ static int hss_hdlc_ioctl(struct net_device *dev, struct if_settings *ifs) return -EFAULT; clk = new_line.clock_type; - if (port->plat->set_clock) - clk = port->plat->set_clock(port->id, clk); + hss_hdlc_set_clock(port, clk); if (clk != CLOCK_EXT && clk != CLOCK_INT) return -EINVAL; /* No such clock setting */ @@ -1297,7 +1344,7 @@ static int hss_hdlc_ioctl(struct net_device *dev, struct if_settings *ifs) port->clock_type = clk; /* Update settings */ if (clk == CLOCK_INT) { - find_best_clock(port->plat->timer_freq, + find_best_clock(IXP4XX_TIMER_FREQ, new_line.clock_rate, &port->clock_rate, &port->clock_reg); } else { @@ -1335,77 +1382,139 @@ static const struct net_device_ops hss_hdlc_ops = { .ndo_siocwandev = hss_hdlc_ioctl, }; -static int hss_init_one(struct platform_device *pdev) +static int ixp4xx_hss_probe(struct platform_device *pdev) { + struct of_phandle_args queue_spec; + struct of_phandle_args npe_spec; + struct device *dev = &pdev->dev; + struct net_device *ndev; + struct device_node *np; struct port *port; - struct net_device *dev; hdlc_device *hdlc; int err; - port = kzalloc(sizeof(*port), GFP_KERNEL); + np = dev->of_node; + + port = devm_kzalloc(dev, sizeof(*port), GFP_KERNEL); if (!port) return -ENOMEM; - port->npe = npe_request(0); + err = of_parse_phandle_with_fixed_args(np, "intel,npe-handle", 1, 0, + &npe_spec); + if (err) + return dev_err_probe(dev, err, "no NPE engine specified\n"); + /* NPE ID 0x00, 0x10, 0x20... */ + port->npe = npe_request(npe_spec.args[0] << 4); if (!port->npe) { - err = -ENODEV; - goto err_free; + dev_err(dev, "unable to obtain NPE instance\n"); + return -ENODEV; } - dev = alloc_hdlcdev(port); + /* Get the TX ready queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-chl-txready", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no txready queue phandle\n"); + port->txreadyq = queue_spec.args[0]; + /* Get the RX trig queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-chl-rxtrig", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no rxtrig queue phandle\n"); + port->rxtrigq = queue_spec.args[0]; + /* Get the RX queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-pkt-rx", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no RX queue phandle\n"); + port->rxq = queue_spec.args[0]; + /* Get the TX queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-pkt-tx", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no RX queue phandle\n"); + port->txq = queue_spec.args[0]; + /* Get the RX free queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-pkt-rxfree", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no RX free queue phandle\n"); + port->rxfreeq = queue_spec.args[0]; + /* Get the TX done queue as resource from queue manager */ + err = of_parse_phandle_with_fixed_args(np, "intek,queue-pkt-txdone", 1, 0, + &queue_spec); + if (err) + return dev_err_probe(dev, err, "no TX done queue phandle\n"); + port->txdoneq = queue_spec.args[0]; + + /* Obtain all the line control GPIOs */ + port->cts = devm_gpiod_get(dev, "cts", GPIOD_OUT_LOW); + if (IS_ERR(port->cts)) + return dev_err_probe(dev, PTR_ERR(port->cts), "unable to get CTS GPIO\n"); + port->rts = devm_gpiod_get(dev, "rts", GPIOD_OUT_LOW); + if (IS_ERR(port->rts)) + return dev_err_probe(dev, PTR_ERR(port->rts), "unable to get RTS GPIO\n"); + port->dcd = devm_gpiod_get(dev, "dcd", GPIOD_IN); + if (IS_ERR(port->dcd)) + return dev_err_probe(dev, PTR_ERR(port->dcd), "unable to get DCD GPIO\n"); + port->dtr = devm_gpiod_get(dev, "dtr", GPIOD_OUT_LOW); + if (IS_ERR(port->dtr)) + return dev_err_probe(dev, PTR_ERR(port->dtr), "unable to get DTR GPIO\n"); + port->clk_internal = devm_gpiod_get(dev, "clk-internal", GPIOD_OUT_LOW); + if (IS_ERR(port->clk_internal)) + return dev_err_probe(dev, PTR_ERR(port->clk_internal), + "unable to get CLK internal GPIO\n"); + + ndev = alloc_hdlcdev(port); port->netdev = alloc_hdlcdev(port); if (!port->netdev) { err = -ENOMEM; goto err_plat; } - SET_NETDEV_DEV(dev, &pdev->dev); - hdlc = dev_to_hdlc(dev); + SET_NETDEV_DEV(ndev, &pdev->dev); + hdlc = dev_to_hdlc(ndev); hdlc->attach = hss_hdlc_attach; hdlc->xmit = hss_hdlc_xmit; - dev->netdev_ops = &hss_hdlc_ops; - dev->tx_queue_len = 100; + ndev->netdev_ops = &hss_hdlc_ops; + ndev->tx_queue_len = 100; port->clock_type = CLOCK_EXT; port->clock_rate = 0; port->clock_reg = CLK42X_SPEED_2048KHZ; port->id = pdev->id; port->dev = &pdev->dev; - port->plat = pdev->dev.platform_data; - netif_napi_add(dev, &port->napi, hss_hdlc_poll, NAPI_WEIGHT); + netif_napi_add(ndev, &port->napi, hss_hdlc_poll, NAPI_WEIGHT); - err = register_hdlc_device(dev); + err = register_hdlc_device(ndev); if (err) goto err_free_netdev; platform_set_drvdata(pdev, port); - netdev_info(dev, "initialized\n"); + netdev_info(ndev, "initialized\n"); return 0; err_free_netdev: - free_netdev(dev); + free_netdev(ndev); err_plat: npe_release(port->npe); -err_free: - kfree(port); return err; } -static int hss_remove_one(struct platform_device *pdev) +static int ixp4xx_hss_remove(struct platform_device *pdev) { struct port *port = platform_get_drvdata(pdev); unregister_hdlc_device(port->netdev); free_netdev(port->netdev); npe_release(port->npe); - kfree(port); return 0; } static struct platform_driver ixp4xx_hss_driver = { .driver.name = DRV_NAME, - .probe = hss_init_one, - .remove = hss_remove_one, + .probe = ixp4xx_hss_probe, + .remove = ixp4xx_hss_remove, }; static int __init hss_init_module(void) diff --git a/drivers/net/wireguard/queueing.h b/drivers/net/wireguard/queueing.h index e2388107f7fd..583adb37ee1e 100644 --- a/drivers/net/wireguard/queueing.h +++ b/drivers/net/wireguard/queueing.h @@ -79,9 +79,7 @@ static inline void wg_reset_packet(struct sk_buff *skb, bool encapsulating) u8 sw_hash = skb->sw_hash; u32 hash = skb->hash; skb_scrub_packet(skb, true); - memset(&skb->headers_start, 0, - offsetof(struct sk_buff, headers_end) - - offsetof(struct sk_buff, headers_start)); + memset(&skb->headers, 0, sizeof(skb->headers)); if (encapsulating) { skb->l4_hash = l4_hash; skb->sw_hash = sw_hash; diff --git a/drivers/net/wireless/ath/ar5523/ar5523.c b/drivers/net/wireless/ath/ar5523/ar5523.c index 0e9bad33fac8..141c1b5a7b1f 100644 --- a/drivers/net/wireless/ath/ar5523/ar5523.c +++ b/drivers/net/wireless/ath/ar5523/ar5523.c @@ -153,6 +153,10 @@ static void ar5523_cmd_rx_cb(struct urb *urb) ar5523_err(ar, "Invalid reply to WDCMSG_TARGET_START"); return; } + if (!cmd->odata) { + ar5523_err(ar, "Unexpected WDCMSG_TARGET_START reply"); + return; + } memcpy(cmd->odata, hdr + 1, sizeof(u32)); cmd->olen = sizeof(u32); cmd->res = 0; diff --git a/drivers/net/wireless/ath/ath10k/core.c b/drivers/net/wireless/ath/ath10k/core.c index 5935e0973d14..72a366aa9f60 100644 --- a/drivers/net/wireless/ath/ath10k/core.c +++ b/drivers/net/wireless/ath/ath10k/core.c @@ -12,6 +12,7 @@ #include <linux/dmi.h> #include <linux/ctype.h> #include <linux/pm_qos.h> +#include <linux/nvmem-consumer.h> #include <asm/byteorder.h> #include "core.h" @@ -935,7 +936,8 @@ static int ath10k_core_get_board_id_from_otp(struct ath10k *ar) } if (ar->cal_mode == ATH10K_PRE_CAL_MODE_DT || - ar->cal_mode == ATH10K_PRE_CAL_MODE_FILE) + ar->cal_mode == ATH10K_PRE_CAL_MODE_FILE || + ar->cal_mode == ATH10K_PRE_CAL_MODE_NVMEM) bmi_board_id_param = BMI_PARAM_GET_FLASH_BOARD_ID; else bmi_board_id_param = BMI_PARAM_GET_EEPROM_BOARD_ID; @@ -1726,7 +1728,8 @@ static int ath10k_download_and_run_otp(struct ath10k *ar) /* As of now pre-cal is valid for 10_4 variants */ if (ar->cal_mode == ATH10K_PRE_CAL_MODE_DT || - ar->cal_mode == ATH10K_PRE_CAL_MODE_FILE) + ar->cal_mode == ATH10K_PRE_CAL_MODE_FILE || + ar->cal_mode == ATH10K_PRE_CAL_MODE_NVMEM) bmi_otp_exe_param = BMI_PARAM_FLASH_SECTION_ALL; ret = ath10k_bmi_execute(ar, address, bmi_otp_exe_param, &result); @@ -1853,6 +1856,39 @@ out_free: return ret; } +static int ath10k_download_cal_nvmem(struct ath10k *ar, const char *cell_name) +{ + struct nvmem_cell *cell; + void *buf; + size_t len; + int ret; + + cell = devm_nvmem_cell_get(ar->dev, cell_name); + if (IS_ERR(cell)) { + ret = PTR_ERR(cell); + return ret; + } + + buf = nvmem_cell_read(cell, &len); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + if (ar->hw_params.cal_data_len != len) { + kfree(buf); + ath10k_warn(ar, "invalid calibration data length in nvmem-cell '%s': %zu != %u\n", + cell_name, len, ar->hw_params.cal_data_len); + return -EMSGSIZE; + } + + ret = ath10k_download_board_data(ar, buf, len); + kfree(buf); + if (ret) + ath10k_warn(ar, "failed to download calibration data from nvmem-cell '%s': %d\n", + cell_name, ret); + + return ret; +} + int ath10k_core_fetch_firmware_api_n(struct ath10k *ar, const char *name, struct ath10k_fw_file *fw_file) { @@ -2087,6 +2123,18 @@ static int ath10k_core_pre_cal_download(struct ath10k *ar) { int ret; + ret = ath10k_download_cal_nvmem(ar, "pre-calibration"); + if (ret == 0) { + ar->cal_mode = ATH10K_PRE_CAL_MODE_NVMEM; + goto success; + } else if (ret == -EPROBE_DEFER) { + return ret; + } + + ath10k_dbg(ar, ATH10K_DBG_BOOT, + "boot did not find a pre-calibration nvmem-cell, try file next: %d\n", + ret); + ret = ath10k_download_cal_file(ar, ar->pre_cal_file); if (ret == 0) { ar->cal_mode = ATH10K_PRE_CAL_MODE_FILE; @@ -2153,6 +2201,18 @@ static int ath10k_download_cal_data(struct ath10k *ar) "pre cal download procedure failed, try cal file: %d\n", ret); + ret = ath10k_download_cal_nvmem(ar, "calibration"); + if (ret == 0) { + ar->cal_mode = ATH10K_CAL_MODE_NVMEM; + goto done; + } else if (ret == -EPROBE_DEFER) { + return ret; + } + + ath10k_dbg(ar, ATH10K_DBG_BOOT, + "boot did not find a calibration nvmem-cell, try file next: %d\n", + ret); + ret = ath10k_download_cal_file(ar, ar->cal_file); if (ret == 0) { ar->cal_mode = ATH10K_CAL_MODE_FILE; diff --git a/drivers/net/wireless/ath/ath10k/core.h b/drivers/net/wireless/ath/ath10k/core.h index 5aeff2d9f6cf..9f6680b3be0a 100644 --- a/drivers/net/wireless/ath/ath10k/core.h +++ b/drivers/net/wireless/ath/ath10k/core.h @@ -877,8 +877,10 @@ enum ath10k_cal_mode { ATH10K_CAL_MODE_FILE, ATH10K_CAL_MODE_OTP, ATH10K_CAL_MODE_DT, + ATH10K_CAL_MODE_NVMEM, ATH10K_PRE_CAL_MODE_FILE, ATH10K_PRE_CAL_MODE_DT, + ATH10K_PRE_CAL_MODE_NVMEM, ATH10K_CAL_MODE_EEPROM, }; @@ -898,10 +900,14 @@ static inline const char *ath10k_cal_mode_str(enum ath10k_cal_mode mode) return "otp"; case ATH10K_CAL_MODE_DT: return "dt"; + case ATH10K_CAL_MODE_NVMEM: + return "nvmem"; case ATH10K_PRE_CAL_MODE_FILE: return "pre-cal-file"; case ATH10K_PRE_CAL_MODE_DT: return "pre-cal-dt"; + case ATH10K_PRE_CAL_MODE_NVMEM: + return "pre-cal-nvmem"; case ATH10K_CAL_MODE_EEPROM: return "eeprom"; } diff --git a/drivers/net/wireless/ath/ath11k/ce.c b/drivers/net/wireless/ath/ath11k/ce.c index de8b632b058c..aaa7b05ff49d 100644 --- a/drivers/net/wireless/ath/ath11k/ce.c +++ b/drivers/net/wireless/ath/ath11k/ce.c @@ -14,6 +14,7 @@ const struct ce_attr ath11k_host_ce_config_ipq8074[] = { .src_nentries = 16, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE1: target->host HTT + HTC control */ @@ -40,6 +41,7 @@ const struct ce_attr ath11k_host_ce_config_ipq8074[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE4: host->target HTT */ @@ -73,11 +75,12 @@ const struct ce_attr ath11k_host_ce_config_ipq8074[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE8: target autonomous hif_memcpy */ { - .flags = CE_ATTR_FLAGS, + .flags = CE_ATTR_FLAGS | CE_ATTR_DIS_INTR, .src_nentries = 0, .src_sz_max = 0, .dest_nentries = 0, @@ -89,6 +92,7 @@ const struct ce_attr ath11k_host_ce_config_ipq8074[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE10: target->host HTT */ @@ -142,6 +146,7 @@ const struct ce_attr ath11k_host_ce_config_qca6390[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE4: host->target HTT */ @@ -175,6 +180,7 @@ const struct ce_attr ath11k_host_ce_config_qca6390[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE8: target autonomous hif_memcpy */ @@ -220,6 +226,7 @@ const struct ce_attr ath11k_host_ce_config_qcn9074[] = { .src_nentries = 32, .src_sz_max = 2048, .dest_nentries = 0, + .send_cb = ath11k_htc_tx_completion_handler, }, /* CE4: host->target HTT */ @@ -489,18 +496,32 @@ err_unlock: return skb; } -static void ath11k_ce_send_done_cb(struct ath11k_ce_pipe *pipe) +static void ath11k_ce_tx_process_cb(struct ath11k_ce_pipe *pipe) { struct ath11k_base *ab = pipe->ab; struct sk_buff *skb; + struct sk_buff_head list; + __skb_queue_head_init(&list); while (!IS_ERR(skb = ath11k_ce_completed_send_next(pipe))) { if (!skb) continue; dma_unmap_single(ab->dev, ATH11K_SKB_CB(skb)->paddr, skb->len, DMA_TO_DEVICE); - dev_kfree_skb_any(skb); + + if ((!pipe->send_cb) || ab->hw_params.credit_flow) { + dev_kfree_skb_any(skb); + continue; + } + + __skb_queue_tail(&list, skb); + } + + while ((skb = __skb_dequeue(&list))) { + ath11k_dbg(ab, ATH11K_DBG_AHB, "tx ce pipe %d len %d\n", + pipe->pipe_num, skb->len); + pipe->send_cb(ab, skb); } } @@ -636,7 +657,7 @@ static int ath11k_ce_alloc_pipe(struct ath11k_base *ab, int ce_id) pipe->attr_flags = attr->flags; if (attr->src_nentries) { - pipe->send_cb = ath11k_ce_send_done_cb; + pipe->send_cb = attr->send_cb; nentries = roundup_pow_of_two(attr->src_nentries); desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_SRC); ring = ath11k_ce_alloc_ring(ab, nentries, desc_sz); @@ -667,9 +688,10 @@ static int ath11k_ce_alloc_pipe(struct ath11k_base *ab, int ce_id) void ath11k_ce_per_engine_service(struct ath11k_base *ab, u16 ce_id) { struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[ce_id]; + const struct ce_attr *attr = &ab->hw_params.host_ce_config[ce_id]; - if (pipe->send_cb) - pipe->send_cb(pipe); + if (attr->src_nentries) + ath11k_ce_tx_process_cb(pipe); if (pipe->recv_cb) ath11k_ce_recv_process_cb(pipe); @@ -678,9 +700,10 @@ void ath11k_ce_per_engine_service(struct ath11k_base *ab, u16 ce_id) void ath11k_ce_poll_send_completed(struct ath11k_base *ab, u8 pipe_id) { struct ath11k_ce_pipe *pipe = &ab->ce.ce_pipe[pipe_id]; + const struct ce_attr *attr = &ab->hw_params.host_ce_config[pipe_id]; - if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && pipe->send_cb) - pipe->send_cb(pipe); + if ((pipe->attr_flags & CE_ATTR_DIS_INTR) && attr->src_nentries) + ath11k_ce_tx_process_cb(pipe); } EXPORT_SYMBOL(ath11k_ce_per_engine_service); @@ -953,6 +976,7 @@ int ath11k_ce_init_pipes(struct ath11k_base *ab) void ath11k_ce_free_pipes(struct ath11k_base *ab) { struct ath11k_ce_pipe *pipe; + struct ath11k_ce_ring *ce_ring; int desc_sz; int i; @@ -964,22 +988,24 @@ void ath11k_ce_free_pipes(struct ath11k_base *ab) if (pipe->src_ring) { desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_SRC); + ce_ring = pipe->src_ring; dma_free_coherent(ab->dev, pipe->src_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->src_ring->base_addr_owner_space, - pipe->src_ring->base_addr_ce_space); + ce_ring->base_addr_owner_space_unaligned, + ce_ring->base_addr_ce_space_unaligned); kfree(pipe->src_ring); pipe->src_ring = NULL; } if (pipe->dest_ring) { desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST); + ce_ring = pipe->dest_ring; dma_free_coherent(ab->dev, pipe->dest_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->dest_ring->base_addr_owner_space, - pipe->dest_ring->base_addr_ce_space); + ce_ring->base_addr_owner_space_unaligned, + ce_ring->base_addr_ce_space_unaligned); kfree(pipe->dest_ring); pipe->dest_ring = NULL; } @@ -987,11 +1013,12 @@ void ath11k_ce_free_pipes(struct ath11k_base *ab) if (pipe->status_ring) { desc_sz = ath11k_hal_ce_get_desc_size(HAL_CE_DESC_DST_STATUS); + ce_ring = pipe->status_ring; dma_free_coherent(ab->dev, pipe->status_ring->nentries * desc_sz + CE_DESC_RING_ALIGN, - pipe->status_ring->base_addr_owner_space, - pipe->status_ring->base_addr_ce_space); + ce_ring->base_addr_owner_space_unaligned, + ce_ring->base_addr_ce_space_unaligned); kfree(pipe->status_ring); pipe->status_ring = NULL; } diff --git a/drivers/net/wireless/ath/ath11k/ce.h b/drivers/net/wireless/ath/ath11k/ce.h index 713f766cac22..8255b6cfab0c 100644 --- a/drivers/net/wireless/ath/ath11k/ce.h +++ b/drivers/net/wireless/ath/ath11k/ce.h @@ -101,6 +101,7 @@ struct ce_attr { unsigned int dest_nentries; void (*recv_cb)(struct ath11k_base *, struct sk_buff *); + void (*send_cb)(struct ath11k_base *, struct sk_buff *); }; #define CE_DESC_RING_ALIGN 8 @@ -154,7 +155,7 @@ struct ath11k_ce_pipe { unsigned int buf_sz; unsigned int rx_buf_needed; - void (*send_cb)(struct ath11k_ce_pipe *); + void (*send_cb)(struct ath11k_base *, struct sk_buff *); void (*recv_cb)(struct ath11k_base *, struct sk_buff *); struct tasklet_struct intr_tq; diff --git a/drivers/net/wireless/ath/ath11k/core.c b/drivers/net/wireless/ath/ath11k/core.c index b5a2af3ffc3e..dd1a1bb078c3 100644 --- a/drivers/net/wireless/ath/ath11k/core.c +++ b/drivers/net/wireless/ath/ath11k/core.c @@ -76,12 +76,17 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_monitor = true, .supports_shadow_regs = false, .idle_ps = false, + .supports_sta_ps = false, .cold_boot_calib = true, .supports_suspend = false, .hal_desc_sz = sizeof(struct hal_rx_desc_ipq8074), .fix_l1ss = true, + .credit_flow = false, .max_tx_ring = DP_TCL_NUM_RING_MAX, .hal_params = &ath11k_hw_hal_params_ipq8074, + .supports_dynamic_smps_6ghz = false, + .alloc_cacheable_memory = true, + .wakeup_mhi = false, }, { .hw_rev = ATH11K_HW_IPQ6018_HW10, @@ -125,12 +130,17 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_monitor = true, .supports_shadow_regs = false, .idle_ps = false, + .supports_sta_ps = false, .cold_boot_calib = true, .supports_suspend = false, .hal_desc_sz = sizeof(struct hal_rx_desc_ipq8074), .fix_l1ss = true, + .credit_flow = false, .max_tx_ring = DP_TCL_NUM_RING_MAX, .hal_params = &ath11k_hw_hal_params_ipq8074, + .supports_dynamic_smps_6ghz = false, + .alloc_cacheable_memory = true, + .wakeup_mhi = false, }, { .name = "qca6390 hw2.0", @@ -173,12 +183,17 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_monitor = false, .supports_shadow_regs = true, .idle_ps = true, + .supports_sta_ps = true, .cold_boot_calib = false, .supports_suspend = true, .hal_desc_sz = sizeof(struct hal_rx_desc_ipq8074), .fix_l1ss = true, + .credit_flow = true, .max_tx_ring = DP_TCL_NUM_RING_MAX_QCA6390, .hal_params = &ath11k_hw_hal_params_qca6390, + .supports_dynamic_smps_6ghz = false, + .alloc_cacheable_memory = false, + .wakeup_mhi = true, }, { .name = "qcn9074 hw1.0", @@ -221,12 +236,17 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_monitor = true, .supports_shadow_regs = false, .idle_ps = false, + .supports_sta_ps = false, .cold_boot_calib = false, .supports_suspend = false, .hal_desc_sz = sizeof(struct hal_rx_desc_qcn9074), .fix_l1ss = true, + .credit_flow = false, .max_tx_ring = DP_TCL_NUM_RING_MAX, .hal_params = &ath11k_hw_hal_params_ipq8074, + .supports_dynamic_smps_6ghz = true, + .alloc_cacheable_memory = true, + .wakeup_mhi = false, }, { .name = "wcn6855 hw2.0", @@ -269,12 +289,17 @@ static const struct ath11k_hw_params ath11k_hw_params[] = { .supports_monitor = false, .supports_shadow_regs = true, .idle_ps = true, + .supports_sta_ps = true, .cold_boot_calib = false, .supports_suspend = true, .hal_desc_sz = sizeof(struct hal_rx_desc_wcn6855), .fix_l1ss = false, + .credit_flow = true, .max_tx_ring = DP_TCL_NUM_RING_MAX_QCA6390, .hal_params = &ath11k_hw_hal_params_qca6390, + .supports_dynamic_smps_6ghz = false, + .alloc_cacheable_memory = false, + .wakeup_mhi = true, }, }; @@ -392,11 +417,26 @@ static int ath11k_core_create_board_name(struct ath11k_base *ab, char *name, scnprintf(variant, sizeof(variant), ",variant=%s", ab->qmi.target.bdf_ext); - scnprintf(name, name_len, - "bus=%s,qmi-chip-id=%d,qmi-board-id=%d%s", - ath11k_bus_str(ab->hif.bus), - ab->qmi.target.chip_id, - ab->qmi.target.board_id, variant); + switch (ab->id.bdf_search) { + case ATH11K_BDF_SEARCH_BUS_AND_BOARD: + scnprintf(name, name_len, + "bus=%s,vendor=%04x,device=%04x,subsystem-vendor=%04x,subsystem-device=%04x,qmi-chip-id=%d,qmi-board-id=%d%s", + ath11k_bus_str(ab->hif.bus), + ab->id.vendor, ab->id.device, + ab->id.subsystem_vendor, + ab->id.subsystem_device, + ab->qmi.target.chip_id, + ab->qmi.target.board_id, + variant); + break; + default: + scnprintf(name, name_len, + "bus=%s,qmi-chip-id=%d,qmi-board-id=%d%s", + ath11k_bus_str(ab->hif.bus), + ab->qmi.target.chip_id, + ab->qmi.target.board_id, variant); + break; + } ath11k_dbg(ab, ATH11K_DBG_BOOT, "boot using board name '%s'\n", name); @@ -633,7 +673,7 @@ static int ath11k_core_fetch_board_data_api_1(struct ath11k_base *ab, return 0; } -#define BOARD_NAME_SIZE 100 +#define BOARD_NAME_SIZE 200 int ath11k_core_fetch_bdf(struct ath11k_base *ab, struct ath11k_board_data *bd) { char boardname[BOARD_NAME_SIZE]; diff --git a/drivers/net/wireless/ath/ath11k/core.h b/drivers/net/wireless/ath/ath11k/core.h index 31d234a51c79..bbfc10fd5c6d 100644 --- a/drivers/net/wireless/ath/ath11k/core.h +++ b/drivers/net/wireless/ath/ath11k/core.h @@ -47,6 +47,11 @@ enum ath11k_supported_bw { ATH11K_BW_160 = 3, }; +enum ath11k_bdf_search { + ATH11K_BDF_SEARCH_DEFAULT, + ATH11K_BDF_SEARCH_BUS_AND_BOARD, +}; + enum wme_ac { WME_AC_BE, WME_AC_BK, @@ -240,6 +245,7 @@ struct ath11k_vif { bool is_started; bool is_up; bool spectral_enabled; + bool ps; u32 aid; u8 bssid[ETH_ALEN]; struct cfg80211_bitrate_mask bitrate_mask; @@ -249,6 +255,8 @@ struct ath11k_vif { int txpower; bool rsnie_present; bool wpaie_present; + bool bcca_zero_sent; + bool do_not_send_tmpl; struct ieee80211_chanctx_conf chanctx; }; @@ -759,6 +767,14 @@ struct ath11k_base { struct completion htc_suspend; + struct { + enum ath11k_bdf_search bdf_search; + u32 vendor; + u32 device; + u32 subsystem_vendor; + u32 subsystem_device; + } id; + /* must be last */ u8 drv_priv[0] __aligned(sizeof(void *)); }; diff --git a/drivers/net/wireless/ath/ath11k/dbring.c b/drivers/net/wireless/ath/ath11k/dbring.c index fd98ba5b1130..de220a10bce3 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.c +++ b/drivers/net/wireless/ath/ath11k/dbring.c @@ -87,17 +87,23 @@ static int ath11k_dbring_fill_bufs(struct ath11k *ar, req_entries = min(num_free, ring->bufs_max); num_remain = req_entries; align = ring->buf_align; - size = sizeof(*buff) + ring->buf_sz + align - 1; + size = ring->buf_sz + align - 1; while (num_remain > 0) { - buff = kzalloc(size, GFP_ATOMIC); + buff = kzalloc(sizeof(*buff), GFP_ATOMIC); if (!buff) break; + buff->payload = kzalloc(size, GFP_ATOMIC); + if (!buff->payload) { + kfree(buff); + break; + } ret = ath11k_dbring_bufs_replenish(ar, ring, buff); if (ret) { ath11k_warn(ar->ab, "failed to replenish db ring num_remain %d req_ent %d\n", num_remain, req_entries); + kfree(buff->payload); kfree(buff); break; } @@ -282,7 +288,7 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab, srng = &ab->hal.srng_list[ring->refill_srng.ring_id]; num_entry = ev->fixed.num_buf_release_entry; - size = sizeof(*buff) + ring->buf_sz + ring->buf_align - 1; + size = ring->buf_sz + ring->buf_align - 1; num_buff_reaped = 0; spin_lock_bh(&srng->lock); @@ -319,7 +325,8 @@ int ath11k_dbring_buffer_release_event(struct ath11k_base *ab, ring->handler(ar, &handler_data); } - memset(buff, 0, size); + buff->paddr = 0; + memset(buff->payload, 0, size); ath11k_dbring_bufs_replenish(ar, ring, buff); } @@ -346,6 +353,7 @@ void ath11k_dbring_buf_cleanup(struct ath11k *ar, struct ath11k_dbring *ring) idr_remove(&ring->bufs_idr, buf_id); dma_unmap_single(ar->ab->dev, buff->paddr, ring->buf_sz, DMA_FROM_DEVICE); + kfree(buff->payload); kfree(buff); } diff --git a/drivers/net/wireless/ath/ath11k/dbring.h b/drivers/net/wireless/ath/ath11k/dbring.h index f7fce9ef9c36..78a985faa0a1 100644 --- a/drivers/net/wireless/ath/ath11k/dbring.h +++ b/drivers/net/wireless/ath/ath11k/dbring.h @@ -13,7 +13,7 @@ struct ath11k_dbring_element { dma_addr_t paddr; - u8 payload[0]; + u8 *payload; }; struct ath11k_dbring_data { diff --git a/drivers/net/wireless/ath/ath11k/debug.c b/drivers/net/wireless/ath/ath11k/debug.c index c86de95fbdc5..958d87429062 100644 --- a/drivers/net/wireless/ath/ath11k/debug.c +++ b/drivers/net/wireless/ath/ath11k/debug.c @@ -17,7 +17,7 @@ void ath11k_info(struct ath11k_base *ab, const char *fmt, ...) va_start(args, fmt); vaf.va = &args; dev_info(ab->dev, "%pV", &vaf); - /* TODO: Trace the log */ + trace_ath11k_log_info(ab, &vaf); va_end(args); } EXPORT_SYMBOL(ath11k_info); @@ -32,7 +32,7 @@ void ath11k_err(struct ath11k_base *ab, const char *fmt, ...) va_start(args, fmt); vaf.va = &args; dev_err(ab->dev, "%pV", &vaf); - /* TODO: Trace the log */ + trace_ath11k_log_err(ab, &vaf); va_end(args); } EXPORT_SYMBOL(ath11k_err); @@ -47,7 +47,7 @@ void ath11k_warn(struct ath11k_base *ab, const char *fmt, ...) va_start(args, fmt); vaf.va = &args; dev_warn_ratelimited(ab->dev, "%pV", &vaf); - /* TODO: Trace the log */ + trace_ath11k_log_warn(ab, &vaf); va_end(args); } EXPORT_SYMBOL(ath11k_warn); @@ -68,7 +68,7 @@ void __ath11k_dbg(struct ath11k_base *ab, enum ath11k_debug_mask mask, if (ath11k_debug_mask & mask) dev_printk(KERN_DEBUG, ab->dev, "%pV", &vaf); - /* TODO: trace log */ + trace_ath11k_log_dbg(ab, mask, &vaf); va_end(args); } @@ -100,6 +100,10 @@ void ath11k_dbg_dump(struct ath11k_base *ab, dev_printk(KERN_DEBUG, ab->dev, "%s\n", linebuf); } } + + /* tracing code doesn't like null strings */ + trace_ath11k_log_dbg_dump(ab, msg ? msg : "", prefix ? prefix : "", + buf, len); } EXPORT_SYMBOL(ath11k_dbg_dump); diff --git a/drivers/net/wireless/ath/ath11k/debug.h b/drivers/net/wireless/ath/ath11k/debug.h index 659a275e2eb3..fbbd5fe02aa8 100644 --- a/drivers/net/wireless/ath/ath11k/debug.h +++ b/drivers/net/wireless/ath/ath11k/debug.h @@ -60,7 +60,8 @@ static inline void ath11k_dbg_dump(struct ath11k_base *ab, #define ath11k_dbg(ar, dbg_mask, fmt, ...) \ do { \ - if (ath11k_debug_mask & dbg_mask) \ + if ((ath11k_debug_mask & dbg_mask) || \ + trace_ath11k_log_dbg_enabled()) \ __ath11k_dbg(ar, dbg_mask, fmt, ##__VA_ARGS__); \ } while (0) diff --git a/drivers/net/wireless/ath/ath11k/debugfs.c b/drivers/net/wireless/ath/ath11k/debugfs.c index 80afd35337a1..dba055d085be 100644 --- a/drivers/net/wireless/ath/ath11k/debugfs.c +++ b/drivers/net/wireless/ath/ath11k/debugfs.c @@ -195,7 +195,7 @@ static int ath11k_debugfs_fw_stats_request(struct ath11k *ar, * received 'update stats' event, we keep a 3 seconds timeout in case, * fw_stats_done is not marked yet */ - timeout = jiffies + msecs_to_jiffies(3 * HZ); + timeout = jiffies + msecs_to_jiffies(3 * 1000); ath11k_debugfs_fw_stats_reset(ar); diff --git a/drivers/net/wireless/ath/ath11k/dp.c b/drivers/net/wireless/ath/ath11k/dp.c index 8baaeeb8cf82..81b0b2baa461 100644 --- a/drivers/net/wireless/ath/ath11k/dp.c +++ b/drivers/net/wireless/ath/ath11k/dp.c @@ -101,8 +101,11 @@ void ath11k_dp_srng_cleanup(struct ath11k_base *ab, struct dp_srng *ring) if (!ring->vaddr_unaligned) return; - dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, - ring->paddr_unaligned); + if (ring->cached) + kfree(ring->vaddr_unaligned); + else + dma_free_coherent(ab->dev, ring->size, ring->vaddr_unaligned, + ring->paddr_unaligned); ring->vaddr_unaligned = NULL; } @@ -222,6 +225,7 @@ int ath11k_dp_srng_setup(struct ath11k_base *ab, struct dp_srng *ring, int entry_sz = ath11k_hal_srng_get_entrysize(ab, type); int max_entries = ath11k_hal_srng_get_max_entries(ab, type); int ret; + bool cached = false; if (max_entries < 0 || entry_sz < 0) return -EINVAL; @@ -230,9 +234,29 @@ int ath11k_dp_srng_setup(struct ath11k_base *ab, struct dp_srng *ring, num_entries = max_entries; ring->size = (num_entries * entry_sz) + HAL_RING_BASE_ALIGN - 1; - ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, - &ring->paddr_unaligned, - GFP_KERNEL); + + if (ab->hw_params.alloc_cacheable_memory) { + /* Allocate the reo dst and tx completion rings from cacheable memory */ + switch (type) { + case HAL_REO_DST: + case HAL_WBM2SW_RELEASE: + cached = true; + break; + default: + cached = false; + } + + if (cached) { + ring->vaddr_unaligned = kzalloc(ring->size, GFP_KERNEL); + ring->paddr_unaligned = virt_to_phys(ring->vaddr_unaligned); + } + } + + if (!cached) + ring->vaddr_unaligned = dma_alloc_coherent(ab->dev, ring->size, + &ring->paddr_unaligned, + GFP_KERNEL); + if (!ring->vaddr_unaligned) return -ENOMEM; @@ -292,6 +316,11 @@ int ath11k_dp_srng_setup(struct ath11k_base *ab, struct dp_srng *ring, return -EINVAL; } + if (cached) { + params.flags |= HAL_SRNG_FLAGS_CACHED; + ring->cached = 1; + } + ret = ath11k_hal_srng_setup(ab, type, ring_num, mac_id, ¶ms); if (ret < 0) { ath11k_warn(ab, "failed to setup srng: %d ring_id %d\n", @@ -742,13 +771,12 @@ int ath11k_dp_service_srng(struct ath11k_base *ab, const struct ath11k_hw_hal_params *hal_params; int grp_id = irq_grp->grp_id; int work_done = 0; - int i = 0, j; + int i, j; int tot_work_done = 0; - while (ab->hw_params.ring_mask->tx[grp_id] >> i) { - if (ab->hw_params.ring_mask->tx[grp_id] & BIT(i)) - ath11k_dp_tx_completion_handler(ab, i); - i++; + if (ab->hw_params.ring_mask->tx[grp_id]) { + i = __fls(ab->hw_params.ring_mask->tx[grp_id]); + ath11k_dp_tx_completion_handler(ab, i); } if (ab->hw_params.ring_mask->rx_err[grp_id]) { diff --git a/drivers/net/wireless/ath/ath11k/dp.h b/drivers/net/wireless/ath/ath11k/dp.h index 4794ca04f213..a4c36a9be338 100644 --- a/drivers/net/wireless/ath/ath11k/dp.h +++ b/drivers/net/wireless/ath/ath11k/dp.h @@ -64,6 +64,7 @@ struct dp_srng { dma_addr_t paddr; int size; u32 ring_id; + u8 cached; }; struct dp_rxdma_ring { @@ -517,7 +518,8 @@ struct htt_ppdu_stats_cfg_cmd { } __packed; #define HTT_PPDU_STATS_CFG_MSG_TYPE GENMASK(7, 0) -#define HTT_PPDU_STATS_CFG_PDEV_ID GENMASK(15, 8) +#define HTT_PPDU_STATS_CFG_SOC_STATS BIT(8) +#define HTT_PPDU_STATS_CFG_PDEV_ID GENMASK(15, 9) #define HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK GENMASK(31, 16) enum htt_ppdu_stats_tag_type { diff --git a/drivers/net/wireless/ath/ath11k/dp_rx.c b/drivers/net/wireless/ath/ath11k/dp_rx.c index c5320847b80a..fcd7a6d27d12 100644 --- a/drivers/net/wireless/ath/ath11k/dp_rx.c +++ b/drivers/net/wireless/ath/ath11k/dp_rx.c @@ -20,13 +20,15 @@ #define ATH11K_DP_RX_FRAGMENT_TIMEOUT_MS (2 * HZ) -static u8 *ath11k_dp_rx_h_80211_hdr(struct ath11k_base *ab, struct hal_rx_desc *desc) +static inline +u8 *ath11k_dp_rx_h_80211_hdr(struct ath11k_base *ab, struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_hdr_status(desc); } -static enum hal_encrypt_type ath11k_dp_rx_h_mpdu_start_enctype(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline +enum hal_encrypt_type ath11k_dp_rx_h_mpdu_start_enctype(struct ath11k_base *ab, + struct hal_rx_desc *desc) { if (!ab->hw_params.hw_ops->rx_desc_encrypt_valid(desc)) return HAL_ENCRYPT_TYPE_OPEN; @@ -34,32 +36,34 @@ static enum hal_encrypt_type ath11k_dp_rx_h_mpdu_start_enctype(struct ath11k_bas return ab->hw_params.hw_ops->rx_desc_get_encrypt_type(desc); } -static u8 ath11k_dp_rx_h_msdu_start_decap_type(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_decap_type(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_decap_type(desc); } -static u8 ath11k_dp_rx_h_msdu_start_mesh_ctl_present(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline +u8 ath11k_dp_rx_h_msdu_start_mesh_ctl_present(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mesh_ctl(desc); } -static bool ath11k_dp_rx_h_mpdu_start_seq_ctrl_valid(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline +bool ath11k_dp_rx_h_mpdu_start_seq_ctrl_valid(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_seq_ctl_vld(desc); } -static bool ath11k_dp_rx_h_mpdu_start_fc_valid(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline bool ath11k_dp_rx_h_mpdu_start_fc_valid(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_fc_valid(desc); } -static bool ath11k_dp_rx_h_mpdu_start_more_frags(struct ath11k_base *ab, - struct sk_buff *skb) +static inline bool ath11k_dp_rx_h_mpdu_start_more_frags(struct ath11k_base *ab, + struct sk_buff *skb) { struct ieee80211_hdr *hdr; @@ -67,8 +71,8 @@ static bool ath11k_dp_rx_h_mpdu_start_more_frags(struct ath11k_base *ab, return ieee80211_has_morefrags(hdr->frame_control); } -static u16 ath11k_dp_rx_h_mpdu_start_frag_no(struct ath11k_base *ab, - struct sk_buff *skb) +static inline u16 ath11k_dp_rx_h_mpdu_start_frag_no(struct ath11k_base *ab, + struct sk_buff *skb) { struct ieee80211_hdr *hdr; @@ -76,37 +80,37 @@ static u16 ath11k_dp_rx_h_mpdu_start_frag_no(struct ath11k_base *ab, return le16_to_cpu(hdr->seq_ctrl) & IEEE80211_SCTL_FRAG; } -static u16 ath11k_dp_rx_h_mpdu_start_seq_no(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u16 ath11k_dp_rx_h_mpdu_start_seq_no(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_start_seq_no(desc); } -static void *ath11k_dp_rx_get_attention(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline void *ath11k_dp_rx_get_attention(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_attention(desc); } -static bool ath11k_dp_rx_h_attn_msdu_done(struct rx_attention *attn) +static inline bool ath11k_dp_rx_h_attn_msdu_done(struct rx_attention *attn) { return !!FIELD_GET(RX_ATTENTION_INFO2_MSDU_DONE, __le32_to_cpu(attn->info2)); } -static bool ath11k_dp_rx_h_attn_l4_cksum_fail(struct rx_attention *attn) +static inline bool ath11k_dp_rx_h_attn_l4_cksum_fail(struct rx_attention *attn) { return !!FIELD_GET(RX_ATTENTION_INFO1_TCP_UDP_CKSUM_FAIL, __le32_to_cpu(attn->info1)); } -static bool ath11k_dp_rx_h_attn_ip_cksum_fail(struct rx_attention *attn) +static inline bool ath11k_dp_rx_h_attn_ip_cksum_fail(struct rx_attention *attn) { return !!FIELD_GET(RX_ATTENTION_INFO1_IP_CKSUM_FAIL, __le32_to_cpu(attn->info1)); } -static bool ath11k_dp_rx_h_attn_is_decrypted(struct rx_attention *attn) +static inline bool ath11k_dp_rx_h_attn_is_decrypted(struct rx_attention *attn) { return (FIELD_GET(RX_ATTENTION_INFO2_DCRYPT_STATUS_CODE, __le32_to_cpu(attn->info2)) == @@ -154,68 +158,68 @@ static bool ath11k_dp_rx_h_attn_msdu_len_err(struct ath11k_base *ab, return errmap & DP_RX_MPDU_ERR_MSDU_LEN; } -static u16 ath11k_dp_rx_h_msdu_start_msdu_len(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u16 ath11k_dp_rx_h_msdu_start_msdu_len(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_len(desc); } -static u8 ath11k_dp_rx_h_msdu_start_sgi(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_sgi(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_sgi(desc); } -static u8 ath11k_dp_rx_h_msdu_start_rate_mcs(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_rate_mcs(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_rate_mcs(desc); } -static u8 ath11k_dp_rx_h_msdu_start_rx_bw(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_rx_bw(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_rx_bw(desc); } -static u32 ath11k_dp_rx_h_msdu_start_freq(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u32 ath11k_dp_rx_h_msdu_start_freq(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_freq(desc); } -static u8 ath11k_dp_rx_h_msdu_start_pkt_type(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_pkt_type(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_msdu_pkt_type(desc); } -static u8 ath11k_dp_rx_h_msdu_start_nss(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_start_nss(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return hweight8(ab->hw_params.hw_ops->rx_desc_get_msdu_nss(desc)); } -static u8 ath11k_dp_rx_h_mpdu_start_tid(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_mpdu_start_tid(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_tid(desc); } -static u16 ath11k_dp_rx_h_mpdu_start_peer_id(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u16 ath11k_dp_rx_h_mpdu_start_peer_id(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_peer_id(desc); } -static u8 ath11k_dp_rx_h_msdu_end_l3pad(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline u8 ath11k_dp_rx_h_msdu_end_l3pad(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_l3_pad_bytes(desc); } -static bool ath11k_dp_rx_h_msdu_end_first_msdu(struct ath11k_base *ab, - struct hal_rx_desc *desc) +static inline bool ath11k_dp_rx_h_msdu_end_first_msdu(struct ath11k_base *ab, + struct hal_rx_desc *desc) { return ab->hw_params.hw_ops->rx_desc_get_first_msdu(desc); } @@ -233,14 +237,14 @@ static void ath11k_dp_rx_desc_end_tlv_copy(struct ath11k_base *ab, ab->hw_params.hw_ops->rx_desc_copy_attn_end_tlv(fdesc, ldesc); } -static u32 ath11k_dp_rxdesc_get_mpdulen_err(struct rx_attention *attn) +static inline u32 ath11k_dp_rxdesc_get_mpdulen_err(struct rx_attention *attn) { return FIELD_GET(RX_ATTENTION_INFO1_MPDU_LEN_ERR, __le32_to_cpu(attn->info1)); } -static u8 *ath11k_dp_rxdesc_get_80211hdr(struct ath11k_base *ab, - struct hal_rx_desc *rx_desc) +static inline u8 *ath11k_dp_rxdesc_get_80211hdr(struct ath11k_base *ab, + struct hal_rx_desc *rx_desc) { u8 *rx_pkt_hdr; @@ -249,8 +253,8 @@ static u8 *ath11k_dp_rxdesc_get_80211hdr(struct ath11k_base *ab, return rx_pkt_hdr; } -static bool ath11k_dp_rxdesc_mpdu_valid(struct ath11k_base *ab, - struct hal_rx_desc *rx_desc) +static inline bool ath11k_dp_rxdesc_mpdu_valid(struct ath11k_base *ab, + struct hal_rx_desc *rx_desc) { u32 tlv_tag; @@ -259,15 +263,15 @@ static bool ath11k_dp_rxdesc_mpdu_valid(struct ath11k_base *ab, return tlv_tag == HAL_RX_MPDU_START; } -static u32 ath11k_dp_rxdesc_get_ppduid(struct ath11k_base *ab, - struct hal_rx_desc *rx_desc) +static inline u32 ath11k_dp_rxdesc_get_ppduid(struct ath11k_base *ab, + struct hal_rx_desc *rx_desc) { return ab->hw_params.hw_ops->rx_desc_get_mpdu_ppdu_id(rx_desc); } -static void ath11k_dp_rxdesc_set_msdu_len(struct ath11k_base *ab, - struct hal_rx_desc *desc, - u16 len) +static inline void ath11k_dp_rxdesc_set_msdu_len(struct ath11k_base *ab, + struct hal_rx_desc *desc, + u16 len) { ab->hw_params.hw_ops->rx_desc_set_msdu_len(desc, len); } @@ -2596,36 +2600,30 @@ free_out: static void ath11k_dp_rx_process_received_packets(struct ath11k_base *ab, struct napi_struct *napi, struct sk_buff_head *msdu_list, - int *quota, int ring_id) + int mac_id) { - struct ath11k_skb_rxcb *rxcb; struct sk_buff *msdu; struct ath11k *ar; struct ieee80211_rx_status rx_status = {0}; - u8 mac_id; int ret; if (skb_queue_empty(msdu_list)) return; - rcu_read_lock(); - - while (*quota && (msdu = __skb_dequeue(msdu_list))) { - rxcb = ATH11K_SKB_RXCB(msdu); - mac_id = rxcb->mac_id; - ar = ab->pdevs[mac_id].ar; - if (!rcu_dereference(ab->pdevs_active[mac_id])) { - dev_kfree_skb_any(msdu); - continue; - } + if (unlikely(!rcu_access_pointer(ab->pdevs_active[mac_id]))) { + __skb_queue_purge(msdu_list); + return; + } - if (test_bit(ATH11K_CAC_RUNNING, &ar->dev_flags)) { - dev_kfree_skb_any(msdu); - continue; - } + ar = ab->pdevs[mac_id].ar; + if (unlikely(test_bit(ATH11K_CAC_RUNNING, &ar->dev_flags))) { + __skb_queue_purge(msdu_list); + return; + } + while ((msdu = __skb_dequeue(msdu_list))) { ret = ath11k_dp_rx_process_msdu(ar, msdu, msdu_list, &rx_status); - if (ret) { + if (unlikely(ret)) { ath11k_dbg(ab, ATH11K_DBG_DATA, "Unable to process msdu %d", ret); dev_kfree_skb_any(msdu); @@ -2633,10 +2631,7 @@ static void ath11k_dp_rx_process_received_packets(struct ath11k_base *ab, } ath11k_dp_rx_deliver_msdu(ar, napi, msdu, &rx_status); - (*quota)--; } - - rcu_read_unlock(); } int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, @@ -2645,19 +2640,21 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, struct ath11k_dp *dp = &ab->dp; struct dp_rxdma_ring *rx_ring; int num_buffs_reaped[MAX_RADIOS] = {0}; - struct sk_buff_head msdu_list; + struct sk_buff_head msdu_list[MAX_RADIOS]; struct ath11k_skb_rxcb *rxcb; int total_msdu_reaped = 0; struct hal_srng *srng; struct sk_buff *msdu; - int quota = budget; bool done = false; int buf_id, mac_id; struct ath11k *ar; - u32 *rx_desc; + struct hal_reo_dest_ring *desc; + enum hal_reo_dest_ring_push_reason push_reason; + u32 cookie; int i; - __skb_queue_head_init(&msdu_list); + for (i = 0; i < MAX_RADIOS; i++) + __skb_queue_head_init(&msdu_list[i]); srng = &ab->hal.srng_list[dp->reo_dst_ring[ring_id].ring_id]; @@ -2666,13 +2663,11 @@ int ath11k_dp_process_rx(struct ath11k_base *ab, int ring_id, ath11k_hal_srng_access_begin(ab, srng); try_again: - while ((rx_desc = ath11k_hal_srng_dst_get_next_entry(ab, srng))) { - struct hal_reo_dest_ring desc = *(struct hal_reo_dest_ring *)rx_desc; - enum hal_reo_dest_ring_push_reason push_reason; - u32 cookie; - + while (likely(desc = + (struct hal_reo_dest_ring *)ath11k_hal_srng_dst_get_next_entry(ab, + srng))) { cookie = FIELD_GET(BUFFER_ADDR_INFO1_SW_COOKIE, - desc.buf_addr_info.info1); + desc->buf_addr_info.info1); buf_id = FIELD_GET(DP_RXDMA_BUF_COOKIE_BUF_ID, cookie); mac_id = FIELD_GET(DP_RXDMA_BUF_COOKIE_PDEV_ID, cookie); @@ -2681,7 +2676,7 @@ try_again: rx_ring = &ar->dp.rx_refill_buf_ring; spin_lock_bh(&rx_ring->idr_lock); msdu = idr_find(&rx_ring->bufs_idr, buf_id); - if (!msdu) { + if (unlikely(!msdu)) { ath11k_warn(ab, "frame rx with invalid buf_id %d\n", buf_id); spin_unlock_bh(&rx_ring->idr_lock); @@ -2697,37 +2692,41 @@ try_again: DMA_FROM_DEVICE); num_buffs_reaped[mac_id]++; - total_msdu_reaped++; push_reason = FIELD_GET(HAL_REO_DEST_RING_INFO0_PUSH_REASON, - desc.info0); - if (push_reason != - HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION) { + desc->info0); + if (unlikely(push_reason != + HAL_REO_DEST_RING_PUSH_REASON_ROUTING_INSTRUCTION)) { dev_kfree_skb_any(msdu); ab->soc_stats.hal_reo_error[dp->reo_dst_ring[ring_id].ring_id]++; continue; } - rxcb->is_first_msdu = !!(desc.rx_msdu_info.info0 & + rxcb->is_first_msdu = !!(desc->rx_msdu_info.info0 & RX_MSDU_DESC_INFO0_FIRST_MSDU_IN_MPDU); - rxcb->is_last_msdu = !!(desc.rx_msdu_info.info0 & + rxcb->is_last_msdu = !!(desc->rx_msdu_info.info0 & RX_MSDU_DESC_INFO0_LAST_MSDU_IN_MPDU); - rxcb->is_continuation = !!(desc.rx_msdu_info.info0 & + rxcb->is_continuation = !!(desc->rx_msdu_info.info0 & RX_MSDU_DESC_INFO0_MSDU_CONTINUATION); rxcb->peer_id = FIELD_GET(RX_MPDU_DESC_META_DATA_PEER_ID, - desc.rx_mpdu_info.meta_data); + desc->rx_mpdu_info.meta_data); rxcb->seq_no = FIELD_GET(RX_MPDU_DESC_INFO0_SEQ_NUM, - desc.rx_mpdu_info.info0); + desc->rx_mpdu_info.info0); rxcb->tid = FIELD_GET(HAL_REO_DEST_RING_INFO0_RX_QUEUE_NUM, - desc.info0); + desc->info0); rxcb->mac_id = mac_id; - __skb_queue_tail(&msdu_list, msdu); + __skb_queue_tail(&msdu_list[mac_id], msdu); - if (total_msdu_reaped >= quota && !rxcb->is_continuation) { + if (rxcb->is_continuation) { + done = false; + } else { + total_msdu_reaped++; done = true; - break; } + + if (total_msdu_reaped >= budget) + break; } /* Hw might have updated the head pointer after we cached it. @@ -2736,7 +2735,7 @@ try_again: * head pointer so that we can reap complete MPDU in the current * rx processing. */ - if (!done && ath11k_hal_srng_dst_num_free(ab, srng, true)) { + if (unlikely(!done && ath11k_hal_srng_dst_num_free(ab, srng, true))) { ath11k_hal_srng_access_end(ab, srng); goto try_again; } @@ -2745,25 +2744,23 @@ try_again: spin_unlock_bh(&srng->lock); - if (!total_msdu_reaped) + if (unlikely(!total_msdu_reaped)) goto exit; for (i = 0; i < ab->num_radios; i++) { if (!num_buffs_reaped[i]) continue; + ath11k_dp_rx_process_received_packets(ab, napi, &msdu_list[i], i); + ar = ab->pdevs[i].ar; rx_ring = &ar->dp.rx_refill_buf_ring; ath11k_dp_rxbufs_replenish(ab, i, rx_ring, num_buffs_reaped[i], ab->hw_params.hal_params->rx_buf_rbm); } - - ath11k_dp_rx_process_received_packets(ab, napi, &msdu_list, - "a, ring_id); - exit: - return budget - quota; + return total_msdu_reaped; } static void ath11k_dp_rx_update_peer_stats(struct ath11k_sta *arsta, @@ -4829,7 +4826,7 @@ static struct sk_buff * ath11k_dp_rx_mon_merg_msdus(struct ath11k *ar, u32 mac_id, struct sk_buff *head_msdu, struct sk_buff *last_msdu, - struct ieee80211_rx_status *rxs) + struct ieee80211_rx_status *rxs, bool *fcs_err) { struct ath11k_base *ab = ar->ab; struct sk_buff *msdu, *prev_buf; @@ -4839,12 +4836,17 @@ ath11k_dp_rx_mon_merg_msdus(struct ath11k *ar, u8 *dest, decap_format; struct ieee80211_hdr_3addr *wh; struct rx_attention *rx_attention; + u32 err_bitmap; if (!head_msdu) goto err_merge_fail; rx_desc = (struct hal_rx_desc *)head_msdu->data; rx_attention = ath11k_dp_rx_get_attention(ab, rx_desc); + err_bitmap = ath11k_dp_rx_h_attn_mpdu_err(rx_attention); + + if (err_bitmap & DP_RX_MPDU_ERR_FCS) + *fcs_err = true; if (ath11k_dp_rxdesc_get_mpdulen_err(rx_attention)) return NULL; @@ -4933,9 +4935,10 @@ static int ath11k_dp_rx_mon_deliver(struct ath11k *ar, u32 mac_id, struct ath11k_pdev_dp *dp = &ar->dp; struct sk_buff *mon_skb, *skb_next, *header; struct ieee80211_rx_status *rxs = &dp->rx_status; + bool fcs_err = false; mon_skb = ath11k_dp_rx_mon_merg_msdus(ar, mac_id, head_msdu, - tail_msdu, rxs); + tail_msdu, rxs, &fcs_err); if (!mon_skb) goto mon_deliver_fail; @@ -4943,6 +4946,10 @@ static int ath11k_dp_rx_mon_deliver(struct ath11k *ar, u32 mac_id, header = mon_skb; rxs->flag = 0; + + if (fcs_err) + rxs->flag = RX_FLAG_FAILED_FCS_CRC; + do { skb_next = mon_skb->next; if (!skb_next) diff --git a/drivers/net/wireless/ath/ath11k/dp_tx.c b/drivers/net/wireless/ath/ath11k/dp_tx.c index 879fb2a9dc0c..88abd64e9047 100644 --- a/drivers/net/wireless/ath/ath11k/dp_tx.c +++ b/drivers/net/wireless/ath/ath11k/dp_tx.c @@ -95,11 +95,11 @@ int ath11k_dp_tx(struct ath11k *ar, struct ath11k_vif *arvif, u8 ring_selector = 0, ring_map = 0; bool tcl_ring_retry; - if (test_bit(ATH11K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags)) + if (unlikely(test_bit(ATH11K_FLAG_CRASH_FLUSH, &ar->ab->dev_flags))) return -ESHUTDOWN; - if (!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && - !ieee80211_is_data(hdr->frame_control)) + if (unlikely(!(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) && + !ieee80211_is_data(hdr->frame_control))) return -ENOTSUPP; pool_id = skb_get_queue_mapping(skb) & (ATH11K_HW_MAX_QUEUES - 1); @@ -127,7 +127,7 @@ tcl_ring_sel: DP_TX_IDR_SIZE - 1, GFP_ATOMIC); spin_unlock_bh(&tx_ring->tx_idr_lock); - if (ret < 0) { + if (unlikely(ret < 0)) { if (ring_map == (BIT(ab->hw_params.max_tx_ring) - 1)) { atomic_inc(&ab->soc_stats.tx_err.misc_fail); return -ENOSPC; @@ -152,7 +152,7 @@ tcl_ring_sel: ti.meta_data_flags = arvif->tcl_metadata; } - if (ti.encap_type == HAL_TCL_ENCAP_TYPE_RAW) { + if (unlikely(ti.encap_type == HAL_TCL_ENCAP_TYPE_RAW)) { if (skb_cb->flags & ATH11K_SKB_CIPHER_SET) { ti.encrypt_type = ath11k_dp_tx_get_encrypt_type(skb_cb->cipher); @@ -173,8 +173,8 @@ tcl_ring_sel: ti.bss_ast_idx = arvif->ast_idx; ti.dscp_tid_tbl_idx = 0; - if (skb->ip_summed == CHECKSUM_PARTIAL && - ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW) { + if (likely(skb->ip_summed == CHECKSUM_PARTIAL && + ti.encap_type != HAL_TCL_ENCAP_TYPE_RAW)) { ti.flags0 |= FIELD_PREP(HAL_TCL_DATA_CMD_INFO1_IP4_CKSUM_EN, 1) | FIELD_PREP(HAL_TCL_DATA_CMD_INFO1_UDP4_CKSUM_EN, 1) | FIELD_PREP(HAL_TCL_DATA_CMD_INFO1_UDP6_CKSUM_EN, 1) | @@ -211,7 +211,7 @@ tcl_ring_sel: } ti.paddr = dma_map_single(ab->dev, skb->data, skb->len, DMA_TO_DEVICE); - if (dma_mapping_error(ab->dev, ti.paddr)) { + if (unlikely(dma_mapping_error(ab->dev, ti.paddr))) { atomic_inc(&ab->soc_stats.tx_err.misc_fail); ath11k_warn(ab, "failed to DMA map data Tx buffer\n"); ret = -ENOMEM; @@ -231,7 +231,7 @@ tcl_ring_sel: ath11k_hal_srng_access_begin(ab, tcl_ring); hal_tcl_desc = (void *)ath11k_hal_srng_src_get_next_entry(ab, tcl_ring); - if (!hal_tcl_desc) { + if (unlikely(!hal_tcl_desc)) { /* NOTE: It is highly unlikely we'll be running out of tcl_ring * desc because the desc is directly enqueued onto hw queue. */ @@ -245,7 +245,7 @@ tcl_ring_sel: * checking this ring earlier for each pkt tx. * Restart ring selection if some rings are not checked yet. */ - if (ring_map != (BIT(ab->hw_params.max_tx_ring) - 1) && + if (unlikely(ring_map != (BIT(ab->hw_params.max_tx_ring)) - 1) && ab->hw_params.max_tx_ring > 1) { tcl_ring_retry = true; ring_selector++; @@ -293,20 +293,18 @@ static void ath11k_dp_tx_free_txbuf(struct ath11k_base *ab, u8 mac_id, struct sk_buff *msdu; struct ath11k_skb_cb *skb_cb; - spin_lock_bh(&tx_ring->tx_idr_lock); - msdu = idr_find(&tx_ring->txbuf_idr, msdu_id); - if (!msdu) { + spin_lock(&tx_ring->tx_idr_lock); + msdu = idr_remove(&tx_ring->txbuf_idr, msdu_id); + spin_unlock(&tx_ring->tx_idr_lock); + + if (unlikely(!msdu)) { ath11k_warn(ab, "tx completion for unknown msdu_id %d\n", msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); return; } skb_cb = ATH11K_SKB_CB(msdu); - idr_remove(&tx_ring->txbuf_idr, msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); - dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); dev_kfree_skb_any(msdu); @@ -325,12 +323,13 @@ ath11k_dp_tx_htt_tx_complete_buf(struct ath11k_base *ab, struct ath11k_skb_cb *skb_cb; struct ath11k *ar; - spin_lock_bh(&tx_ring->tx_idr_lock); - msdu = idr_find(&tx_ring->txbuf_idr, ts->msdu_id); - if (!msdu) { + spin_lock(&tx_ring->tx_idr_lock); + msdu = idr_remove(&tx_ring->txbuf_idr, ts->msdu_id); + spin_unlock(&tx_ring->tx_idr_lock); + + if (unlikely(!msdu)) { ath11k_warn(ab, "htt tx completion for unknown msdu_id %d\n", ts->msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); return; } @@ -339,9 +338,6 @@ ath11k_dp_tx_htt_tx_complete_buf(struct ath11k_base *ab, ar = skb_cb->ar; - idr_remove(&tx_ring->txbuf_idr, ts->msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); - if (atomic_dec_and_test(&ar->dp.num_tx_pending)) wake_up(&ar->dp.tx_empty_waitq); @@ -435,16 +431,14 @@ static void ath11k_dp_tx_complete_msdu(struct ath11k *ar, dma_unmap_single(ab->dev, skb_cb->paddr, msdu->len, DMA_TO_DEVICE); - rcu_read_lock(); - - if (!rcu_dereference(ab->pdevs_active[ar->pdev_idx])) { + if (unlikely(!rcu_access_pointer(ab->pdevs_active[ar->pdev_idx]))) { dev_kfree_skb_any(msdu); - goto exit; + return; } - if (!skb_cb->vif) { + if (unlikely(!skb_cb->vif)) { dev_kfree_skb_any(msdu); - goto exit; + return; } info = IEEE80211_SKB_CB(msdu); @@ -465,7 +459,7 @@ static void ath11k_dp_tx_complete_msdu(struct ath11k *ar, (info->flags & IEEE80211_TX_CTL_NO_ACK)) info->flags |= IEEE80211_TX_STAT_NOACK_TRANSMITTED; - if (ath11k_debugfs_is_extd_tx_stats_enabled(ar)) { + if (unlikely(ath11k_debugfs_is_extd_tx_stats_enabled(ar))) { if (ts->flags & HAL_TX_STATUS_FLAGS_FIRST_MSDU) { if (ar->last_ppdu_id == 0) { ar->last_ppdu_id = ts->ppdu_id; @@ -494,9 +488,6 @@ static void ath11k_dp_tx_complete_msdu(struct ath11k *ar, */ ieee80211_tx_status(ar->hw, msdu); - -exit: - rcu_read_unlock(); } static inline void ath11k_dp_tx_status_parse(struct ath11k_base *ab, @@ -505,11 +496,11 @@ static inline void ath11k_dp_tx_status_parse(struct ath11k_base *ab, { ts->buf_rel_source = FIELD_GET(HAL_WBM_RELEASE_INFO0_REL_SRC_MODULE, desc->info0); - if (ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW && - ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM) + if (unlikely(ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_FW && + ts->buf_rel_source != HAL_WBM_REL_SRC_MODULE_TQM)) return; - if (ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) + if (unlikely(ts->buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW)) return; ts->status = FIELD_GET(HAL_WBM_RELEASE_INFO0_TQM_RELEASE_REASON, @@ -556,8 +547,9 @@ void ath11k_dp_tx_completion_handler(struct ath11k_base *ab, int ring_id) ATH11K_TX_COMPL_NEXT(tx_ring->tx_status_head); } - if ((ath11k_hal_srng_dst_peek(ab, status_ring) != NULL) && - (ATH11K_TX_COMPL_NEXT(tx_ring->tx_status_head) == tx_ring->tx_status_tail)) { + if (unlikely((ath11k_hal_srng_dst_peek(ab, status_ring) != NULL) && + (ATH11K_TX_COMPL_NEXT(tx_ring->tx_status_head) == + tx_ring->tx_status_tail))) { /* TODO: Process pending tx_status messages when kfifo_is_full() */ ath11k_warn(ab, "Unable to process some of the tx_status ring desc because status_fifo is full\n"); } @@ -580,7 +572,7 @@ void ath11k_dp_tx_completion_handler(struct ath11k_base *ab, int ring_id) mac_id = FIELD_GET(DP_TX_DESC_ID_MAC_ID, desc_id); msdu_id = FIELD_GET(DP_TX_DESC_ID_MSDU_ID, desc_id); - if (ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW) { + if (unlikely(ts.buf_rel_source == HAL_WBM_REL_SRC_MODULE_FW)) { ath11k_dp_tx_process_htt_tx_complete(ab, (void *)tx_status, mac_id, msdu_id, @@ -588,16 +580,16 @@ void ath11k_dp_tx_completion_handler(struct ath11k_base *ab, int ring_id) continue; } - spin_lock_bh(&tx_ring->tx_idr_lock); - msdu = idr_find(&tx_ring->txbuf_idr, msdu_id); - if (!msdu) { + spin_lock(&tx_ring->tx_idr_lock); + msdu = idr_remove(&tx_ring->txbuf_idr, msdu_id); + if (unlikely(!msdu)) { ath11k_warn(ab, "tx completion for unknown msdu_id %d\n", msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); + spin_unlock(&tx_ring->tx_idr_lock); continue; } - idr_remove(&tx_ring->txbuf_idr, msdu_id); - spin_unlock_bh(&tx_ring->tx_idr_lock); + + spin_unlock(&tx_ring->tx_idr_lock); ar = ab->pdevs[mac_id].ar; @@ -903,7 +895,7 @@ int ath11k_dp_tx_htt_h2t_ppdu_stats_req(struct ath11k *ar, u32 mask) cmd->msg = FIELD_PREP(HTT_PPDU_STATS_CFG_MSG_TYPE, HTT_H2T_MSG_TYPE_PPDU_STATS_CFG); - pdev_mask = 1 << (i + 1); + pdev_mask = 1 << (ar->pdev_idx + i); cmd->msg |= FIELD_PREP(HTT_PPDU_STATS_CFG_PDEV_ID, pdev_mask); cmd->msg |= FIELD_PREP(HTT_PPDU_STATS_CFG_TLV_TYPE_BITMASK, mask); diff --git a/drivers/net/wireless/ath/ath11k/hal.c b/drivers/net/wireless/ath/ath11k/hal.c index eaa0edca5576..7cf9e23cb922 100644 --- a/drivers/net/wireless/ath/ath11k/hal.c +++ b/drivers/net/wireless/ath/ath11k/hal.c @@ -627,6 +627,21 @@ u32 *ath11k_hal_srng_dst_peek(struct ath11k_base *ab, struct hal_srng *srng) return NULL; } +static void ath11k_hal_srng_prefetch_desc(struct ath11k_base *ab, + struct hal_srng *srng) +{ + u32 *desc; + + /* prefetch only if desc is available */ + desc = ath11k_hal_srng_dst_peek(ab, srng); + if (likely(desc)) { + dma_sync_single_for_cpu(ab->dev, virt_to_phys(desc), + (srng->entry_size * sizeof(u32)), + DMA_FROM_DEVICE); + prefetch(desc); + } +} + u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab, struct hal_srng *srng) { @@ -639,8 +654,15 @@ u32 *ath11k_hal_srng_dst_get_next_entry(struct ath11k_base *ab, desc = srng->ring_base_vaddr + srng->u.dst_ring.tp; - srng->u.dst_ring.tp = (srng->u.dst_ring.tp + srng->entry_size) % - srng->ring_size; + srng->u.dst_ring.tp += srng->entry_size; + + /* wrap around to start of ring*/ + if (srng->u.dst_ring.tp == srng->ring_size) + srng->u.dst_ring.tp = 0; + + /* Try to prefetch the next descriptor in the ring */ + if (srng->flags & HAL_SRNG_FLAGS_CACHED) + ath11k_hal_srng_prefetch_desc(ab, srng); return desc; } @@ -775,11 +797,16 @@ void ath11k_hal_srng_access_begin(struct ath11k_base *ab, struct hal_srng *srng) { lockdep_assert_held(&srng->lock); - if (srng->ring_dir == HAL_SRNG_DIR_SRC) + if (srng->ring_dir == HAL_SRNG_DIR_SRC) { srng->u.src_ring.cached_tp = *(volatile u32 *)srng->u.src_ring.tp_addr; - else + } else { srng->u.dst_ring.cached_hp = *srng->u.dst_ring.hp_addr; + + /* Try to prefetch the next descriptor in the ring */ + if (srng->flags & HAL_SRNG_FLAGS_CACHED) + ath11k_hal_srng_prefetch_desc(ab, srng); + } } /* Update cached ring head/tail pointers to HW. ath11k_hal_srng_access_begin() diff --git a/drivers/net/wireless/ath/ath11k/hal.h b/drivers/net/wireless/ath/ath11k/hal.h index 35ed3a14e200..0f4f9ce74354 100644 --- a/drivers/net/wireless/ath/ath11k/hal.h +++ b/drivers/net/wireless/ath/ath11k/hal.h @@ -513,6 +513,7 @@ enum hal_srng_dir { #define HAL_SRNG_FLAGS_DATA_TLV_SWAP 0x00000020 #define HAL_SRNG_FLAGS_LOW_THRESH_INTR_EN 0x00010000 #define HAL_SRNG_FLAGS_MSI_INTR 0x00020000 +#define HAL_SRNG_FLAGS_CACHED 0x20000000 #define HAL_SRNG_FLAGS_LMAC_RING 0x80000000 #define HAL_SRNG_TLV_HDR_TAG GENMASK(9, 1) diff --git a/drivers/net/wireless/ath/ath11k/htc.c b/drivers/net/wireless/ath/ath11k/htc.c index 54b1d34724d7..6913b7494b9b 100644 --- a/drivers/net/wireless/ath/ath11k/htc.c +++ b/drivers/net/wireless/ath/ath11k/htc.c @@ -81,6 +81,8 @@ int ath11k_htc_send(struct ath11k_htc *htc, struct ath11k_base *ab = htc->ab; int credits = 0; int ret; + bool credit_flow_enabled = (ab->hw_params.credit_flow && + ep->tx_credit_flow_enabled); if (eid >= ATH11K_HTC_EP_COUNT) { ath11k_warn(ab, "Invalid endpoint id: %d\n", eid); @@ -89,7 +91,7 @@ int ath11k_htc_send(struct ath11k_htc *htc, skb_push(skb, sizeof(struct ath11k_htc_hdr)); - if (ep->tx_credit_flow_enabled) { + if (credit_flow_enabled) { credits = DIV_ROUND_UP(skb->len, htc->target_credit_size); spin_lock_bh(&htc->tx_lock); if (ep->tx_credits < credits) { @@ -126,7 +128,7 @@ int ath11k_htc_send(struct ath11k_htc *htc, err_unmap: dma_unmap_single(dev, skb_cb->paddr, skb->len, DMA_TO_DEVICE); err_credits: - if (ep->tx_credit_flow_enabled) { + if (credit_flow_enabled) { spin_lock_bh(&htc->tx_lock); ep->tx_credits += credits; ath11k_dbg(ab, ATH11K_DBG_HTC, @@ -203,23 +205,25 @@ static int ath11k_htc_process_trailer(struct ath11k_htc *htc, break; } - switch (record->hdr.id) { - case ATH11K_HTC_RECORD_CREDITS: - len = sizeof(struct ath11k_htc_credit_report); - if (record->hdr.len < len) { - ath11k_warn(ab, "Credit report too long\n"); - status = -EINVAL; + if (ab->hw_params.credit_flow) { + switch (record->hdr.id) { + case ATH11K_HTC_RECORD_CREDITS: + len = sizeof(struct ath11k_htc_credit_report); + if (record->hdr.len < len) { + ath11k_warn(ab, "Credit report too long\n"); + status = -EINVAL; + break; + } + ath11k_htc_process_credit_report(htc, + record->credit_report, + record->hdr.len, + src_eid); + break; + default: + ath11k_warn(ab, "Unhandled record: id:%d length:%d\n", + record->hdr.id, record->hdr.len); break; } - ath11k_htc_process_credit_report(htc, - record->credit_report, - record->hdr.len, - src_eid); - break; - default: - ath11k_warn(ab, "Unhandled record: id:%d length:%d\n", - record->hdr.id, record->hdr.len); - break; } if (status) @@ -245,6 +249,29 @@ static void ath11k_htc_suspend_complete(struct ath11k_base *ab, bool ack) complete(&ab->htc_suspend); } +void ath11k_htc_tx_completion_handler(struct ath11k_base *ab, + struct sk_buff *skb) +{ + struct ath11k_htc *htc = &ab->htc; + struct ath11k_htc_ep *ep; + void (*ep_tx_complete)(struct ath11k_base *, struct sk_buff *); + u8 eid; + + eid = ATH11K_SKB_CB(skb)->eid; + if (eid >= ATH11K_HTC_EP_COUNT) + return; + + ep = &htc->endpoint[eid]; + spin_lock_bh(&htc->tx_lock); + ep_tx_complete = ep->ep_ops.ep_tx_complete; + spin_unlock_bh(&htc->tx_lock); + if (!ep_tx_complete) { + dev_kfree_skb_any(skb); + return; + } + ep_tx_complete(htc->ab, skb); +} + void ath11k_htc_rx_completion_handler(struct ath11k_base *ab, struct sk_buff *skb) { @@ -607,6 +634,11 @@ int ath11k_htc_connect_service(struct ath11k_htc *htc, disable_credit_flow_ctrl = true; } + if (!ab->hw_params.credit_flow) { + flags |= ATH11K_HTC_CONN_FLAGS_DISABLE_CREDIT_FLOW_CTRL; + disable_credit_flow_ctrl = true; + } + req_msg->flags_len = FIELD_PREP(HTC_SVC_MSG_CONNECTIONFLAGS, flags); req_msg->msg_svc_id |= FIELD_PREP(HTC_SVC_MSG_SERVICE_ID, conn_req->service_id); @@ -732,7 +764,10 @@ int ath11k_htc_start(struct ath11k_htc *htc) msg->msg_id = FIELD_PREP(HTC_MSG_MESSAGEID, ATH11K_HTC_MSG_SETUP_COMPLETE_EX_ID); - ath11k_dbg(ab, ATH11K_DBG_HTC, "HTC is using TX credit flow control\n"); + if (ab->hw_params.credit_flow) + ath11k_dbg(ab, ATH11K_DBG_HTC, "HTC is using TX credit flow control\n"); + else + msg->flags |= ATH11K_GLOBAL_DISABLE_CREDIT_FLOW; status = ath11k_htc_send(htc, ATH11K_HTC_EP_0, skb); if (status) { diff --git a/drivers/net/wireless/ath/ath11k/htc.h b/drivers/net/wireless/ath/ath11k/htc.h index 6c8a469d7f9d..f429b37cfdf7 100644 --- a/drivers/net/wireless/ath/ath11k/htc.h +++ b/drivers/net/wireless/ath/ath11k/htc.h @@ -83,8 +83,8 @@ enum ath11k_htc_conn_flags { ATH11K_HTC_CONN_FLAGS_THRESHOLD_LEVEL_ONE_HALF = 0x1, ATH11K_HTC_CONN_FLAGS_THRESHOLD_LEVEL_THREE_FOURTHS = 0x2, ATH11K_HTC_CONN_FLAGS_THRESHOLD_LEVEL_UNITY = 0x3, - ATH11K_HTC_CONN_FLAGS_REDUCE_CREDIT_DRIBBLE = 1 << 2, - ATH11K_HTC_CONN_FLAGS_DISABLE_CREDIT_FLOW_CTRL = 1 << 3 + ATH11K_HTC_CONN_FLAGS_REDUCE_CREDIT_DRIBBLE = 0x4, + ATH11K_HTC_CONN_FLAGS_DISABLE_CREDIT_FLOW_CTRL = 0x8, }; enum ath11k_htc_conn_svc_status { @@ -116,6 +116,8 @@ struct ath11k_htc_conn_svc_resp { u32 svc_meta_pad; } __packed; +#define ATH11K_GLOBAL_DISABLE_CREDIT_FLOW BIT(1) + struct ath11k_htc_setup_complete_extended { u32 msg_id; u32 flags; @@ -305,5 +307,6 @@ int ath11k_htc_send(struct ath11k_htc *htc, enum ath11k_htc_ep_id eid, struct sk_buff *ath11k_htc_alloc_skb(struct ath11k_base *ar, int size); void ath11k_htc_rx_completion_handler(struct ath11k_base *ar, struct sk_buff *skb); - +void ath11k_htc_tx_completion_handler(struct ath11k_base *ab, + struct sk_buff *skb); #endif diff --git a/drivers/net/wireless/ath/ath11k/hw.c b/drivers/net/wireless/ath/ath11k/hw.c index da35fcf5bc56..2f0b526188e4 100644 --- a/drivers/net/wireless/ath/ath11k/hw.c +++ b/drivers/net/wireless/ath/ath11k/hw.c @@ -1061,8 +1061,6 @@ const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_ipq8074 = { const struct ath11k_hw_ring_mask ath11k_hw_ring_mask_qca6390 = { .tx = { ATH11K_TX_RING_MASK_0, - ATH11K_TX_RING_MASK_1, - ATH11K_TX_RING_MASK_2, }, .rx_mon_status = { 0, 0, 0, 0, diff --git a/drivers/net/wireless/ath/ath11k/hw.h b/drivers/net/wireless/ath/ath11k/hw.h index 19223d36846e..2c9d232ebfed 100644 --- a/drivers/net/wireless/ath/ath11k/hw.h +++ b/drivers/net/wireless/ath/ath11k/hw.h @@ -170,12 +170,17 @@ struct ath11k_hw_params { bool supports_monitor; bool supports_shadow_regs; bool idle_ps; + bool supports_sta_ps; bool cold_boot_calib; bool supports_suspend; u32 hal_desc_sz; bool fix_l1ss; + bool credit_flow; u8 max_tx_ring; const struct ath11k_hw_hal_params *hal_params; + bool supports_dynamic_smps_6ghz; + bool alloc_cacheable_memory; + bool wakeup_mhi; }; struct ath11k_hw_ops { diff --git a/drivers/net/wireless/ath/ath11k/mac.c b/drivers/net/wireless/ath/ath11k/mac.c index 1cc55602787b..292b2b7eab11 100644 --- a/drivers/net/wireless/ath/ath11k/mac.c +++ b/drivers/net/wireless/ath/ath11k/mac.c @@ -775,9 +775,9 @@ static int ath11k_mac_monitor_vdev_start(struct ath11k *ar, int vdev_id, arg.channel.chan_radar = !!(channel->flags & IEEE80211_CHAN_RADAR); arg.channel.min_power = 0; - arg.channel.max_power = channel->max_power * 2; - arg.channel.max_reg_power = channel->max_reg_power * 2; - arg.channel.max_antenna_gain = channel->max_antenna_gain * 2; + arg.channel.max_power = channel->max_power; + arg.channel.max_reg_power = channel->max_reg_power; + arg.channel.max_antenna_gain = channel->max_antenna_gain; arg.pref_tx_streams = ar->num_tx_chains; arg.pref_rx_streams = ar->num_rx_chains; @@ -1049,6 +1049,83 @@ static int ath11k_mac_monitor_stop(struct ath11k *ar) return 0; } +static int ath11k_mac_vif_setup_ps(struct ath11k_vif *arvif) +{ + struct ath11k *ar = arvif->ar; + struct ieee80211_vif *vif = arvif->vif; + struct ieee80211_conf *conf = &ar->hw->conf; + enum wmi_sta_powersave_param param; + enum wmi_sta_ps_mode psmode; + int ret; + int timeout; + bool enable_ps; + + lockdep_assert_held(&arvif->ar->conf_mutex); + + if (arvif->vif->type != NL80211_IFTYPE_STATION) + return 0; + + enable_ps = arvif->ps; + + if (!arvif->is_started) { + /* mac80211 can update vif powersave state while disconnected. + * Firmware doesn't behave nicely and consumes more power than + * necessary if PS is disabled on a non-started vdev. Hence + * force-enable PS for non-running vdevs. + */ + psmode = WMI_STA_PS_MODE_ENABLED; + } else if (enable_ps) { + psmode = WMI_STA_PS_MODE_ENABLED; + param = WMI_STA_PS_PARAM_INACTIVITY_TIME; + + timeout = conf->dynamic_ps_timeout; + if (timeout == 0) { + /* firmware doesn't like 0 */ + timeout = ieee80211_tu_to_usec(vif->bss_conf.beacon_int) / 1000; + } + + ret = ath11k_wmi_set_sta_ps_param(ar, arvif->vdev_id, param, + timeout); + if (ret) { + ath11k_warn(ar->ab, "failed to set inactivity time for vdev %d: %i\n", + arvif->vdev_id, ret); + return ret; + } + } else { + psmode = WMI_STA_PS_MODE_DISABLED; + } + + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, "mac vdev %d psmode %s\n", + arvif->vdev_id, psmode ? "enable" : "disable"); + + ret = ath11k_wmi_pdev_set_ps_mode(ar, arvif->vdev_id, psmode); + if (ret) { + ath11k_warn(ar->ab, "failed to set sta power save mode %d for vdev %d: %d\n", + psmode, arvif->vdev_id, ret); + return ret; + } + + return 0; +} + +static int ath11k_mac_config_ps(struct ath11k *ar) +{ + struct ath11k_vif *arvif; + int ret = 0; + + lockdep_assert_held(&ar->conf_mutex); + + list_for_each_entry(arvif, &ar->arvifs, list) { + ret = ath11k_mac_vif_setup_ps(arvif); + if (ret) { + ath11k_warn(ar->ab, "failed to setup powersave: %d\n", ret); + break; + } + } + + return ret; +} + static int ath11k_mac_op_config(struct ieee80211_hw *hw, u32 changed) { struct ath11k *ar = hw->priv; @@ -1137,11 +1214,15 @@ static int ath11k_mac_setup_bcn_tmpl(struct ath11k_vif *arvif) if (cfg80211_find_ie(WLAN_EID_RSN, ies, (skb_tail_pointer(bcn) - ies))) arvif->rsnie_present = true; + else + arvif->rsnie_present = false; if (cfg80211_find_vendor_ie(WLAN_OUI_MICROSOFT, WLAN_OUI_TYPE_MICROSOFT_WPA, ies, (skb_tail_pointer(bcn) - ies))) arvif->wpaie_present = true; + else + arvif->wpaie_present = false; ret = ath11k_wmi_bcn_tmpl(ar, arvif->vdev_id, &offs, bcn); @@ -1154,6 +1235,26 @@ static int ath11k_mac_setup_bcn_tmpl(struct ath11k_vif *arvif) return ret; } +void ath11k_mac_bcn_tx_event(struct ath11k_vif *arvif) +{ + struct ieee80211_vif *vif = arvif->vif; + + if (!vif->color_change_active && !arvif->bcca_zero_sent) + return; + + if (vif->color_change_active && ieee80211_beacon_cntdwn_is_complete(vif)) { + arvif->bcca_zero_sent = true; + ieee80211_color_change_finish(vif); + return; + } + + arvif->bcca_zero_sent = false; + + if (vif->color_change_active) + ieee80211_beacon_update_cntdwn(vif); + ath11k_mac_setup_bcn_tmpl(arvif); +} + static void ath11k_control_beaconing(struct ath11k_vif *arvif, struct ieee80211_bss_conf *info) { @@ -2397,6 +2498,8 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw, struct ath11k_vif *arvif = (void *)vif->drv_priv; struct peer_assoc_params peer_arg; struct ieee80211_sta *ap_sta; + struct ath11k_peer *peer; + bool is_auth = false; int ret; lockdep_assert_held(&ar->conf_mutex); @@ -2418,6 +2521,7 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw, rcu_read_unlock(); + peer_arg.is_assoc = true; ret = ath11k_wmi_send_peer_assoc_cmd(ar, &peer_arg); if (ret) { ath11k_warn(ar->ab, "failed to run peer assoc for %pM vdev %i: %d\n", @@ -2458,13 +2562,22 @@ static void ath11k_bss_assoc(struct ieee80211_hw *hw, "mac vdev %d up (associated) bssid %pM aid %d\n", arvif->vdev_id, bss_conf->bssid, bss_conf->aid); - /* Authorize BSS Peer */ - ret = ath11k_wmi_set_peer_param(ar, arvif->bssid, - arvif->vdev_id, - WMI_PEER_AUTHORIZE, - 1); - if (ret) - ath11k_warn(ar->ab, "Unable to authorize BSS peer: %d\n", ret); + spin_lock_bh(&ar->ab->base_lock); + + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, arvif->bssid); + if (peer && peer->is_authorized) + is_auth = true; + + spin_unlock_bh(&ar->ab->base_lock); + + if (is_auth) { + ret = ath11k_wmi_set_peer_param(ar, arvif->bssid, + arvif->vdev_id, + WMI_PEER_AUTHORIZE, + 1); + if (ret) + ath11k_warn(ar->ab, "Unable to authorize BSS peer: %d\n", ret); + } ret = ath11k_wmi_send_obss_spr_cmd(ar, arvif->vdev_id, &bss_conf->he_obss_pd); @@ -2805,10 +2918,17 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw, "Set staggered beacon mode for VDEV: %d\n", arvif->vdev_id); - ret = ath11k_mac_setup_bcn_tmpl(arvif); - if (ret) - ath11k_warn(ar->ab, "failed to update bcn template: %d\n", - ret); + if (!arvif->do_not_send_tmpl || !arvif->bcca_zero_sent) { + ret = ath11k_mac_setup_bcn_tmpl(arvif); + if (ret) + ath11k_warn(ar->ab, "failed to update bcn template: %d\n", + ret); + } + + if (arvif->bcca_zero_sent) + arvif->do_not_send_tmpl = true; + else + arvif->do_not_send_tmpl = false; } if (changed & (BSS_CHANGED_BEACON_INFO | BSS_CHANGED_BEACON)) { @@ -2942,6 +3062,16 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw, ath11k_mac_txpower_recalc(ar); } + if (changed & BSS_CHANGED_PS && + ar->ab->hw_params.supports_sta_ps) { + arvif->ps = vif->bss_conf.ps; + + ret = ath11k_mac_config_ps(ar); + if (ret) + ath11k_warn(ar->ab, "failed to setup ps on vdev %i: %d\n", + arvif->vdev_id, ret); + } + if (changed & BSS_CHANGED_MCAST_RATE && !ath11k_mac_vif_chan(arvif->vif, &def)) { band = def.chan->band; @@ -3009,6 +3139,25 @@ static void ath11k_mac_op_bss_info_changed(struct ieee80211_hw *hw, if (ret) ath11k_warn(ar->ab, "failed to set bss color collision on vdev %i: %d\n", arvif->vdev_id, ret); + + param_id = WMI_VDEV_PARAM_BSS_COLOR; + if (info->he_bss_color.enabled) + param_value = info->he_bss_color.color << + IEEE80211_HE_OPERATION_BSS_COLOR_OFFSET; + else + param_value = IEEE80211_HE_OPERATION_BSS_COLOR_DISABLED; + + ret = ath11k_wmi_vdev_set_param_cmd(ar, arvif->vdev_id, + param_id, + param_value); + if (ret) + ath11k_warn(ar->ab, + "failed to set bss color param on vdev %i: %d\n", + arvif->vdev_id, ret); + + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, + "bss color param 0x%x set on vdev %i\n", + param_value, arvif->vdev_id); } else if (vif->type == NL80211_IFTYPE_STATION) { ret = ath11k_wmi_send_bss_color_change_enable_cmd(ar, arvif->vdev_id, @@ -3316,9 +3465,7 @@ static int ath11k_install_key(struct ath11k_vif *arvif, return 0; if (cmd == DISABLE_KEY) { - /* TODO: Check if FW expects value other than NONE for del */ - /* arg.key_cipher = WMI_CIPHER_NONE; */ - arg.key_len = 0; + arg.key_cipher = WMI_CIPHER_NONE; arg.key_data = NULL; goto install; } @@ -3685,6 +3832,7 @@ static int ath11k_station_assoc(struct ath11k *ar, ath11k_peer_assoc_prepare(ar, vif, sta, &peer_arg, reassoc); + peer_arg.is_assoc = true; ret = ath11k_wmi_send_peer_assoc_cmd(ar, &peer_arg); if (ret) { ath11k_warn(ar->ab, "failed to run peer assoc for STA %pM vdev %i: %d\n", @@ -3824,11 +3972,27 @@ static void ath11k_sta_rc_update_wk(struct work_struct *wk) ath11k_mac_max_he_nss(he_mcs_mask))); if (changed & IEEE80211_RC_BW_CHANGED) { - err = ath11k_wmi_set_peer_param(ar, sta->addr, arvif->vdev_id, - WMI_PEER_CHWIDTH, bw); - if (err) - ath11k_warn(ar->ab, "failed to update STA %pM peer bw %d: %d\n", - sta->addr, bw, err); + /* Send peer assoc command before set peer bandwidth param to + * avoid the mismatch between the peer phymode and the peer + * bandwidth. + */ + ath11k_peer_assoc_prepare(ar, arvif->vif, sta, &peer_arg, true); + + peer_arg.is_assoc = false; + err = ath11k_wmi_send_peer_assoc_cmd(ar, &peer_arg); + if (err) { + ath11k_warn(ar->ab, "failed to send peer assoc for STA %pM vdev %i: %d\n", + sta->addr, arvif->vdev_id, err); + } else if (wait_for_completion_timeout(&ar->peer_assoc_done, 1 * HZ)) { + err = ath11k_wmi_set_peer_param(ar, sta->addr, arvif->vdev_id, + WMI_PEER_CHWIDTH, bw); + if (err) + ath11k_warn(ar->ab, "failed to update STA %pM peer bw %d: %d\n", + sta->addr, bw, err); + } else { + ath11k_warn(ar->ab, "failed to get peer assoc conf event for %pM vdev %i\n", + sta->addr, arvif->vdev_id); + } } if (changed & IEEE80211_RC_NSS_CHANGED) { @@ -3896,6 +4060,7 @@ static void ath11k_sta_rc_update_wk(struct work_struct *wk) ath11k_peer_assoc_prepare(ar, arvif->vif, sta, &peer_arg, true); + peer_arg.is_assoc = false; err = ath11k_wmi_send_peer_assoc_cmd(ar, &peer_arg); if (err) ath11k_warn(ar->ab, "failed to run peer assoc for STA %pM vdev %i: %d\n", @@ -4095,6 +4260,10 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, new_state == IEEE80211_STA_NOTEXIST)) { ath11k_dp_peer_cleanup(ar, arvif->vdev_id, sta->addr); + if (ar->ab->hw_params.vdev_start_delay && + vif->type == NL80211_IFTYPE_STATION) + goto free; + ret = ath11k_peer_delete(ar, arvif->vdev_id, sta->addr); if (ret) ath11k_warn(ar->ab, "Failed to delete peer: %pM for VDEV: %d\n", @@ -4116,6 +4285,7 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, } spin_unlock_bh(&ar->ab->base_lock); +free: kfree(arsta->tx_stats); arsta->tx_stats = NULL; @@ -4131,6 +4301,34 @@ static int ath11k_mac_op_sta_state(struct ieee80211_hw *hw, ath11k_warn(ar->ab, "Failed to associate station: %pM\n", sta->addr); } else if (old_state == IEEE80211_STA_ASSOC && + new_state == IEEE80211_STA_AUTHORIZED) { + spin_lock_bh(&ar->ab->base_lock); + + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); + if (peer) + peer->is_authorized = true; + + spin_unlock_bh(&ar->ab->base_lock); + + if (vif->type == NL80211_IFTYPE_STATION && arvif->is_up) { + ret = ath11k_wmi_set_peer_param(ar, sta->addr, + arvif->vdev_id, + WMI_PEER_AUTHORIZE, + 1); + if (ret) + ath11k_warn(ar->ab, "Unable to authorize peer %pM vdev %d: %d\n", + sta->addr, arvif->vdev_id, ret); + } + } else if (old_state == IEEE80211_STA_AUTHORIZED && + new_state == IEEE80211_STA_ASSOC) { + spin_lock_bh(&ar->ab->base_lock); + + peer = ath11k_peer_find(ar->ab, arvif->vdev_id, sta->addr); + if (peer) + peer->is_authorized = false; + + spin_unlock_bh(&ar->ab->base_lock); + } else if (old_state == IEEE80211_STA_ASSOC && new_state == IEEE80211_STA_AUTH && (vif->type == NL80211_IFTYPE_AP || vif->type == NL80211_IFTYPE_MESH_POINT || @@ -4561,6 +4759,10 @@ ath11k_create_vht_cap(struct ath11k *ar, u32 rate_cap_tx_chainmask, vht_cap.vht_supported = 1; vht_cap.cap = ar->pdev->cap.vht_cap; + if (ar->pdev->cap.nss_ratio_enabled) + vht_cap.vht_mcs.tx_highest |= + cpu_to_le16(IEEE80211_VHT_EXT_NSS_BW_CAPABLE); + ath11k_set_vht_txbf_cap(ar, &vht_cap.cap); rxmcs_map = 0; @@ -5048,13 +5250,15 @@ static void ath11k_mgmt_over_wmi_tx_work(struct work_struct *work) arvif = ath11k_vif_to_arvif(skb_cb->vif); if (ar->allocated_vdev_map & (1LL << arvif->vdev_id) && arvif->is_started) { + atomic_inc(&ar->num_pending_mgmt_tx); ret = ath11k_mac_mgmt_tx_wmi(ar, arvif, skb); if (ret) { + if (atomic_dec_if_positive(&ar->num_pending_mgmt_tx) < 0) + WARN_ON_ONCE(1); + ath11k_warn(ar->ab, "failed to tx mgmt frame, vdev_id %d :%d\n", arvif->vdev_id, ret); ieee80211_free_txskb(ar->hw, skb); - } else { - atomic_inc(&ar->num_pending_mgmt_tx); } } else { ath11k_warn(ar->ab, @@ -5138,7 +5342,7 @@ static void ath11k_mac_op_tx(struct ieee80211_hw *hw, arsta = (struct ath11k_sta *)control->sta->drv_priv; ret = ath11k_dp_tx(ar, arvif, arsta, skb); - if (ret) { + if (unlikely(ret)) { ath11k_warn(ar->ab, "failed to transmit frame %d\n", ret); ieee80211_free_txskb(ar->hw, skb); } @@ -5484,7 +5688,7 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw, u32 param_id, param_value; u16 nss; int i; - int ret; + int ret, fbret; int bit; vif->driver_flags |= IEEE80211_VIF_SUPPORTS_UAPSD; @@ -5638,7 +5842,8 @@ static int ath11k_mac_op_add_interface(struct ieee80211_hw *hw, goto err_peer_del; } - ret = ath11k_wmi_pdev_set_ps_mode(ar, arvif->vdev_id, false); + ret = ath11k_wmi_pdev_set_ps_mode(ar, arvif->vdev_id, + WMI_STA_PS_MODE_DISABLED); if (ret) { ath11k_warn(ar->ab, "failed to disable vdev %d ps mode: %d\n", arvif->vdev_id, ret); @@ -5686,17 +5891,17 @@ err_peer_del: if (arvif->vdev_type == WMI_VDEV_TYPE_AP) { reinit_completion(&ar->peer_delete_done); - ret = ath11k_wmi_send_peer_delete_cmd(ar, vif->addr, - arvif->vdev_id); - if (ret) { + fbret = ath11k_wmi_send_peer_delete_cmd(ar, vif->addr, + arvif->vdev_id); + if (fbret) { ath11k_warn(ar->ab, "failed to delete peer vdev_id %d addr %pM\n", arvif->vdev_id, vif->addr); goto err; } - ret = ath11k_wait_for_peer_delete_done(ar, arvif->vdev_id, - vif->addr); - if (ret) + fbret = ath11k_wait_for_peer_delete_done(ar, arvif->vdev_id, + vif->addr); + if (fbret) goto err; ar->num_peers--; @@ -5831,7 +6036,6 @@ static void ath11k_mac_op_configure_filter(struct ieee80211_hw *hw, mutex_lock(&ar->conf_mutex); - changed_flags &= SUPPORTED_FILTERS; *total_flags &= SUPPORTED_FILTERS; ar->filter_flags = *total_flags; @@ -5969,9 +6173,9 @@ ath11k_mac_vdev_start_restart(struct ath11k_vif *arvif, ath11k_phymodes[chandef->chan->band][chandef->width]; arg.channel.min_power = 0; - arg.channel.max_power = chandef->chan->max_power * 2; - arg.channel.max_reg_power = chandef->chan->max_reg_power * 2; - arg.channel.max_antenna_gain = chandef->chan->max_antenna_gain * 2; + arg.channel.max_power = chandef->chan->max_power; + arg.channel.max_reg_power = chandef->chan->max_reg_power; + arg.channel.max_antenna_gain = chandef->chan->max_antenna_gain; arg.pref_tx_streams = ar->num_tx_chains; arg.pref_rx_streams = ar->num_rx_chains; @@ -6468,6 +6672,19 @@ ath11k_mac_op_unassign_vif_chanctx(struct ieee80211_hw *hw, arvif->is_started = false; if (ab->hw_params.vdev_start_delay && + arvif->vdev_type == WMI_VDEV_TYPE_STA) { + ret = ath11k_peer_delete(ar, arvif->vdev_id, arvif->bssid); + if (ret) + ath11k_warn(ar->ab, + "failed to delete peer %pM for vdev %d: %d\n", + arvif->bssid, arvif->vdev_id, ret); + else + ath11k_dbg(ar->ab, ATH11K_DBG_MAC, + "mac removed peer %pM vdev %d after vdev stop\n", + arvif->bssid, arvif->vdev_id); + } + + if (ab->hw_params.vdev_start_delay && arvif->vdev_type == WMI_VDEV_TYPE_MONITOR) ath11k_wmi_vdev_down(ar, arvif->vdev_id); @@ -7277,21 +7494,20 @@ static void ath11k_mac_op_sta_statistics(struct ieee80211_hw *hw, sinfo->tx_duration = arsta->tx_duration; sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_DURATION); - if (!arsta->txrate.legacy && !arsta->txrate.nss) - return; - - if (arsta->txrate.legacy) { - sinfo->txrate.legacy = arsta->txrate.legacy; - } else { - sinfo->txrate.mcs = arsta->txrate.mcs; - sinfo->txrate.nss = arsta->txrate.nss; - sinfo->txrate.bw = arsta->txrate.bw; - sinfo->txrate.he_gi = arsta->txrate.he_gi; - sinfo->txrate.he_dcm = arsta->txrate.he_dcm; - sinfo->txrate.he_ru_alloc = arsta->txrate.he_ru_alloc; + if (arsta->txrate.legacy || arsta->txrate.nss) { + if (arsta->txrate.legacy) { + sinfo->txrate.legacy = arsta->txrate.legacy; + } else { + sinfo->txrate.mcs = arsta->txrate.mcs; + sinfo->txrate.nss = arsta->txrate.nss; + sinfo->txrate.bw = arsta->txrate.bw; + sinfo->txrate.he_gi = arsta->txrate.he_gi; + sinfo->txrate.he_dcm = arsta->txrate.he_dcm; + sinfo->txrate.he_ru_alloc = arsta->txrate.he_ru_alloc; + } + sinfo->txrate.flags = arsta->txrate.flags; + sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); } - sinfo->txrate.flags = arsta->txrate.flags; - sinfo->filled |= BIT_ULL(NL80211_STA_INFO_TX_BITRATE); /* TODO: Use real NF instead of default one. */ sinfo->signal = arsta->rssi_comb + ATH11K_DEFAULT_NOISE_FLOOR; @@ -7672,7 +7888,8 @@ static int __ath11k_mac_register(struct ath11k *ar) * for each band for a dual band capable radio. It will be tricky to * handle it when the ht capability different for each band. */ - if (ht_cap & WMI_HT_CAP_DYNAMIC_SMPS || ar->supports_6ghz) + if (ht_cap & WMI_HT_CAP_DYNAMIC_SMPS || + (ar->supports_6ghz && ab->hw_params.supports_dynamic_smps_6ghz)) ar->hw->wiphy->features |= NL80211_FEATURE_DYNAMIC_SMPS; ar->hw->wiphy->max_scan_ssids = WLAN_SCAN_PARAMS_MAX_SSID; @@ -7703,6 +7920,9 @@ static int __ath11k_mac_register(struct ath11k *ar) wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_CQM_RSSI_LIST); wiphy_ext_feature_set(ar->hw->wiphy, NL80211_EXT_FEATURE_STA_TX_PWR); + if (test_bit(WMI_TLV_SERVICE_BSS_COLOR_OFFLOAD, ar->ab->wmi_ab.svc_map)) + wiphy_ext_feature_set(ar->hw->wiphy, + NL80211_EXT_FEATURE_BSS_COLOR); ar->hw->wiphy->cipher_suites = cipher_suites; ar->hw->wiphy->n_cipher_suites = ARRAY_SIZE(cipher_suites); diff --git a/drivers/net/wireless/ath/ath11k/mac.h b/drivers/net/wireless/ath/ath11k/mac.h index 254ca4acc8e8..f6f37e8c8c6a 100644 --- a/drivers/net/wireless/ath/ath11k/mac.h +++ b/drivers/net/wireless/ath/ath11k/mac.h @@ -155,4 +155,5 @@ enum ath11k_supported_bw ath11k_mac_mac80211_bw_to_ath11k_bw(enum rate_info_bw b enum hal_encrypt_type ath11k_dp_tx_get_encrypt_type(u32 cipher); void ath11k_mac_handle_beacon(struct ath11k *ar, struct sk_buff *skb); void ath11k_mac_handle_beacon_miss(struct ath11k *ar, u32 vdev_id); +void ath11k_mac_bcn_tx_event(struct ath11k_vif *arvif); #endif diff --git a/drivers/net/wireless/ath/ath11k/pci.c b/drivers/net/wireless/ath/ath11k/pci.c index 3d353e7c9d5c..c2af8184e4a2 100644 --- a/drivers/net/wireless/ath/ath11k/pci.c +++ b/drivers/net/wireless/ath/ath11k/pci.c @@ -182,7 +182,8 @@ void ath11k_pci_write32(struct ath11k_base *ab, u32 offset, u32 value) /* for offset beyond BAR + 4K - 32, may * need to wakeup MHI to access. */ - if (test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && + if (ab->hw_params.wakeup_mhi && + test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && offset >= ACCESS_ALWAYS_OFF) mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev); @@ -206,7 +207,8 @@ void ath11k_pci_write32(struct ath11k_base *ab, u32 offset, u32 value) } } - if (test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && + if (ab->hw_params.wakeup_mhi && + test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && offset >= ACCESS_ALWAYS_OFF) mhi_device_put(ab_pci->mhi_ctrl->mhi_dev); } @@ -219,7 +221,8 @@ u32 ath11k_pci_read32(struct ath11k_base *ab, u32 offset) /* for offset beyond BAR + 4K - 32, may * need to wakeup MHI to access. */ - if (test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && + if (ab->hw_params.wakeup_mhi && + test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && offset >= ACCESS_ALWAYS_OFF) mhi_device_get_sync(ab_pci->mhi_ctrl->mhi_dev); @@ -243,7 +246,8 @@ u32 ath11k_pci_read32(struct ath11k_base *ab, u32 offset) } } - if (test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && + if (ab->hw_params.wakeup_mhi && + test_bit(ATH11K_PCI_FLAG_INIT_DONE, &ab_pci->flags) && offset >= ACCESS_ALWAYS_OFF) mhi_device_put(ab_pci->mhi_ctrl->mhi_dev); @@ -1251,6 +1255,15 @@ static int ath11k_pci_probe(struct pci_dev *pdev, goto err_free_core; } + ath11k_dbg(ab, ATH11K_DBG_BOOT, "pci probe %04x:%04x %04x:%04x\n", + pdev->vendor, pdev->device, + pdev->subsystem_vendor, pdev->subsystem_device); + + ab->id.vendor = pdev->vendor; + ab->id.device = pdev->device; + ab->id.subsystem_vendor = pdev->subsystem_vendor; + ab->id.subsystem_device = pdev->subsystem_device; + switch (pci_dev->device) { case QCA6390_DEVICE_ID: ath11k_pci_read_hw_version(ab, &soc_hw_version_major, @@ -1273,6 +1286,7 @@ static int ath11k_pci_probe(struct pci_dev *pdev, ab->hw_rev = ATH11K_HW_QCN9074_HW10; break; case WCN6855_DEVICE_ID: + ab->id.bdf_search = ATH11K_BDF_SEARCH_BUS_AND_BOARD; ath11k_pci_read_hw_version(ab, &soc_hw_version_major, &soc_hw_version_minor); switch (soc_hw_version_major) { diff --git a/drivers/net/wireless/ath/ath11k/peer.h b/drivers/net/wireless/ath/ath11k/peer.h index 619db001be8e..63fe5665badf 100644 --- a/drivers/net/wireless/ath/ath11k/peer.h +++ b/drivers/net/wireless/ath/ath11k/peer.h @@ -28,6 +28,7 @@ struct ath11k_peer { u8 ucast_keyidx; u16 sec_type; u16 sec_type_grp; + bool is_authorized; }; void ath11k_peer_unmap_event(struct ath11k_base *ab, u16 peer_id); diff --git a/drivers/net/wireless/ath/ath11k/qmi.c b/drivers/net/wireless/ath/ath11k/qmi.c index fa73118de6db..25eb22cbeaeb 100644 --- a/drivers/net/wireless/ath/ath11k/qmi.c +++ b/drivers/net/wireless/ath/ath11k/qmi.c @@ -1586,7 +1586,7 @@ static int ath11k_qmi_host_cap_send(struct ath11k_base *ab) { struct qmi_wlanfw_host_cap_req_msg_v01 req; struct qmi_wlanfw_host_cap_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0; memset(&req, 0, sizeof(req)); @@ -1640,6 +1640,7 @@ static int ath11k_qmi_host_cap_send(struct ath11k_base *ab) QMI_WLANFW_HOST_CAP_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_host_cap_req_msg_v01_ei, &req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send host capability request: %d\n", ret); goto out; } @@ -1705,6 +1706,7 @@ static int ath11k_qmi_fw_ind_register_send(struct ath11k_base *ab) QMI_WLANFW_IND_REGISTER_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_ind_register_req_msg_v01_ei, req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send indication register request: %d\n", ret); goto out; @@ -1734,7 +1736,7 @@ static int ath11k_qmi_respond_fw_mem_request(struct ath11k_base *ab) { struct qmi_wlanfw_respond_mem_req_msg_v01 *req; struct qmi_wlanfw_respond_mem_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0, i; bool delayed; @@ -1783,6 +1785,7 @@ static int ath11k_qmi_respond_fw_mem_request(struct ath11k_base *ab) QMI_WLANFW_RESPOND_MEM_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_respond_mem_req_msg_v01_ei, req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to respond qmi memory request: %d\n", ret); goto out; @@ -1911,7 +1914,7 @@ static int ath11k_qmi_request_target_cap(struct ath11k_base *ab) { struct qmi_wlanfw_cap_req_msg_v01 req; struct qmi_wlanfw_cap_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0; int r; @@ -1930,6 +1933,7 @@ static int ath11k_qmi_request_target_cap(struct ath11k_base *ab) QMI_WLANFW_CAP_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_cap_req_msg_v01_ei, &req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send qmi cap request: %d\n", ret); goto out; @@ -2000,7 +2004,7 @@ static int ath11k_qmi_load_file_target_mem(struct ath11k_base *ab, { struct qmi_wlanfw_bdf_download_req_msg_v01 *req; struct qmi_wlanfw_bdf_download_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; const u8 *temp = data; void __iomem *bdf_addr = NULL; int ret; @@ -2245,7 +2249,7 @@ static int ath11k_qmi_wlanfw_m3_info_send(struct ath11k_base *ab) struct m3_mem_region *m3_mem = &ab->qmi.m3_mem; struct qmi_wlanfw_m3_info_req_msg_v01 req; struct qmi_wlanfw_m3_info_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0; memset(&req, 0, sizeof(req)); @@ -2277,6 +2281,7 @@ static int ath11k_qmi_wlanfw_m3_info_send(struct ath11k_base *ab) QMI_WLANFW_M3_INFO_REQ_MSG_V01_MAX_MSG_LEN, qmi_wlanfw_m3_info_req_msg_v01_ei, &req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send m3 information request: %d\n", ret); goto out; @@ -2303,7 +2308,7 @@ static int ath11k_qmi_wlanfw_mode_send(struct ath11k_base *ab, { struct qmi_wlanfw_wlan_mode_req_msg_v01 req; struct qmi_wlanfw_wlan_mode_resp_msg_v01 resp; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0; memset(&req, 0, sizeof(req)); @@ -2325,6 +2330,7 @@ static int ath11k_qmi_wlanfw_mode_send(struct ath11k_base *ab, QMI_WLANFW_WLAN_MODE_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_wlan_mode_req_msg_v01_ei, &req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send wlan mode request (mode %d): %d\n", mode, ret); goto out; @@ -2358,7 +2364,7 @@ static int ath11k_qmi_wlanfw_wlan_cfg_send(struct ath11k_base *ab) struct qmi_wlanfw_wlan_cfg_resp_msg_v01 resp; struct ce_pipe_config *ce_cfg; struct service_to_pipe *svc_cfg; - struct qmi_txn txn = {}; + struct qmi_txn txn; int ret = 0, pipe_num; ce_cfg = (struct ce_pipe_config *)ab->qmi.ce_cfg.tgt_ce; @@ -2419,6 +2425,7 @@ static int ath11k_qmi_wlanfw_wlan_cfg_send(struct ath11k_base *ab) QMI_WLANFW_WLAN_CFG_REQ_MSG_V01_MAX_LEN, qmi_wlanfw_wlan_cfg_req_msg_v01_ei, req); if (ret < 0) { + qmi_txn_cancel(&txn); ath11k_warn(ab, "failed to send wlan config request: %d\n", ret); goto out; diff --git a/drivers/net/wireless/ath/ath11k/reg.c b/drivers/net/wireless/ath/ath11k/reg.c index a66b5bdd2167..8606170ba80d 100644 --- a/drivers/net/wireless/ath/ath11k/reg.c +++ b/drivers/net/wireless/ath/ath11k/reg.c @@ -456,6 +456,9 @@ ath11k_reg_adjust_bw(u16 start_freq, u16 end_freq, u16 max_bw) { u16 bw; + if (end_freq <= start_freq) + return 0; + bw = end_freq - start_freq; bw = min_t(u16, bw, max_bw); @@ -463,8 +466,10 @@ ath11k_reg_adjust_bw(u16 start_freq, u16 end_freq, u16 max_bw) bw = 80; else if (bw >= 40 && bw < 80) bw = 40; - else if (bw < 40) + else if (bw >= 20 && bw < 40) bw = 20; + else + bw = 0; return bw; } @@ -488,73 +493,77 @@ ath11k_reg_update_weather_radar_band(struct ath11k_base *ab, struct cur_reg_rule *reg_rule, u8 *rule_idx, u32 flags, u16 max_bw) { + u32 start_freq; u32 end_freq; u16 bw; u8 i; i = *rule_idx; + /* there might be situations when even the input rule must be dropped */ + i--; + + /* frequencies below weather radar */ bw = ath11k_reg_adjust_bw(reg_rule->start_freq, ETSI_WEATHER_RADAR_BAND_LOW, max_bw); + if (bw > 0) { + i++; - ath11k_reg_update_rule(regd->reg_rules + i, reg_rule->start_freq, - ETSI_WEATHER_RADAR_BAND_LOW, bw, - reg_rule->ant_gain, reg_rule->reg_power, - flags); + ath11k_reg_update_rule(regd->reg_rules + i, + reg_rule->start_freq, + ETSI_WEATHER_RADAR_BAND_LOW, bw, + reg_rule->ant_gain, reg_rule->reg_power, + flags); - ath11k_dbg(ab, ATH11K_DBG_REG, - "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", - i + 1, reg_rule->start_freq, ETSI_WEATHER_RADAR_BAND_LOW, - bw, reg_rule->ant_gain, reg_rule->reg_power, - regd->reg_rules[i].dfs_cac_ms, - flags); - - if (reg_rule->end_freq > ETSI_WEATHER_RADAR_BAND_HIGH) - end_freq = ETSI_WEATHER_RADAR_BAND_HIGH; - else - end_freq = reg_rule->end_freq; + ath11k_dbg(ab, ATH11K_DBG_REG, + "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", + i + 1, reg_rule->start_freq, + ETSI_WEATHER_RADAR_BAND_LOW, bw, reg_rule->ant_gain, + reg_rule->reg_power, regd->reg_rules[i].dfs_cac_ms, + flags); + } - bw = ath11k_reg_adjust_bw(ETSI_WEATHER_RADAR_BAND_LOW, end_freq, - max_bw); + /* weather radar frequencies */ + start_freq = max_t(u32, reg_rule->start_freq, + ETSI_WEATHER_RADAR_BAND_LOW); + end_freq = min_t(u32, reg_rule->end_freq, ETSI_WEATHER_RADAR_BAND_HIGH); - i++; + bw = ath11k_reg_adjust_bw(start_freq, end_freq, max_bw); + if (bw > 0) { + i++; - ath11k_reg_update_rule(regd->reg_rules + i, - ETSI_WEATHER_RADAR_BAND_LOW, end_freq, bw, - reg_rule->ant_gain, reg_rule->reg_power, - flags); + ath11k_reg_update_rule(regd->reg_rules + i, start_freq, + end_freq, bw, reg_rule->ant_gain, + reg_rule->reg_power, flags); - regd->reg_rules[i].dfs_cac_ms = ETSI_WEATHER_RADAR_BAND_CAC_TIMEOUT; + regd->reg_rules[i].dfs_cac_ms = ETSI_WEATHER_RADAR_BAND_CAC_TIMEOUT; - ath11k_dbg(ab, ATH11K_DBG_REG, - "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", - i + 1, ETSI_WEATHER_RADAR_BAND_LOW, end_freq, - bw, reg_rule->ant_gain, reg_rule->reg_power, - regd->reg_rules[i].dfs_cac_ms, - flags); - - if (end_freq == reg_rule->end_freq) { - regd->n_reg_rules--; - *rule_idx = i; - return; + ath11k_dbg(ab, ATH11K_DBG_REG, + "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", + i + 1, start_freq, end_freq, bw, + reg_rule->ant_gain, reg_rule->reg_power, + regd->reg_rules[i].dfs_cac_ms, flags); } + /* frequencies above weather radar */ bw = ath11k_reg_adjust_bw(ETSI_WEATHER_RADAR_BAND_HIGH, reg_rule->end_freq, max_bw); + if (bw > 0) { + i++; - i++; - - ath11k_reg_update_rule(regd->reg_rules + i, ETSI_WEATHER_RADAR_BAND_HIGH, - reg_rule->end_freq, bw, - reg_rule->ant_gain, reg_rule->reg_power, - flags); + ath11k_reg_update_rule(regd->reg_rules + i, + ETSI_WEATHER_RADAR_BAND_HIGH, + reg_rule->end_freq, bw, + reg_rule->ant_gain, reg_rule->reg_power, + flags); - ath11k_dbg(ab, ATH11K_DBG_REG, - "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", - i + 1, ETSI_WEATHER_RADAR_BAND_HIGH, reg_rule->end_freq, - bw, reg_rule->ant_gain, reg_rule->reg_power, - regd->reg_rules[i].dfs_cac_ms, - flags); + ath11k_dbg(ab, ATH11K_DBG_REG, + "\t%d. (%d - %d @ %d) (%d, %d) (%d ms) (FLAGS %d)\n", + i + 1, ETSI_WEATHER_RADAR_BAND_HIGH, + reg_rule->end_freq, bw, reg_rule->ant_gain, + reg_rule->reg_power, regd->reg_rules[i].dfs_cac_ms, + flags); + } *rule_idx = i; } diff --git a/drivers/net/wireless/ath/ath11k/trace.c b/drivers/net/wireless/ath/ath11k/trace.c index f0cc49ba0387..6620650d7845 100644 --- a/drivers/net/wireless/ath/ath11k/trace.c +++ b/drivers/net/wireless/ath/ath11k/trace.c @@ -7,3 +7,4 @@ #define CREATE_TRACE_POINTS #include "trace.h" +EXPORT_SYMBOL(__tracepoint_ath11k_log_dbg); diff --git a/drivers/net/wireless/ath/ath11k/trace.h b/drivers/net/wireless/ath/ath11k/trace.h index 25d18e9d5b0b..02003dc4207d 100644 --- a/drivers/net/wireless/ath/ath11k/trace.h +++ b/drivers/net/wireless/ath/ath11k/trace.h @@ -14,12 +14,24 @@ #if !defined(CONFIG_ATH11K_TRACING) #undef TRACE_EVENT #define TRACE_EVENT(name, proto, ...) \ +static inline void trace_ ## name(proto) {} \ +static inline bool trace_##name##_enabled(void) \ +{ \ + return false; \ +} + +#undef DECLARE_EVENT_CLASS +#define DECLARE_EVENT_CLASS(...) +#undef DEFINE_EVENT +#define DEFINE_EVENT(evt_class, name, proto, ...) \ static inline void trace_ ## name(proto) {} #endif /* !CONFIG_ATH11K_TRACING || __CHECKER__ */ #undef TRACE_SYSTEM #define TRACE_SYSTEM ath11k +#define ATH11K_MSG_MAX 400 + TRACE_EVENT(ath11k_htt_pktlog, TP_PROTO(struct ath11k *ar, const void *buf, u16 buf_len, u32 pktlog_checksum), @@ -108,6 +120,166 @@ TRACE_EVENT(ath11k_htt_rxdesc, ) ); +DECLARE_EVENT_CLASS(ath11k_log_event, + TP_PROTO(struct ath11k_base *ab, struct va_format *vaf), + TP_ARGS(ab, vaf), + TP_STRUCT__entry( + __string(device, dev_name(ab->dev)) + __string(driver, dev_driver_string(ab->dev)) + __dynamic_array(char, msg, ATH11K_MSG_MAX) + ), + TP_fast_assign( + __assign_str(device, dev_name(ab->dev)); + __assign_str(driver, dev_driver_string(ab->dev)); + WARN_ON_ONCE(vsnprintf(__get_dynamic_array(msg), + ATH11K_MSG_MAX, + vaf->fmt, + *vaf->va) >= ATH11K_MSG_MAX); + ), + TP_printk( + "%s %s %s", + __get_str(driver), + __get_str(device), + __get_str(msg) + ) +); + +DEFINE_EVENT(ath11k_log_event, ath11k_log_err, + TP_PROTO(struct ath11k_base *ab, struct va_format *vaf), + TP_ARGS(ab, vaf) +); + +DEFINE_EVENT(ath11k_log_event, ath11k_log_warn, + TP_PROTO(struct ath11k_base *ab, struct va_format *vaf), + TP_ARGS(ab, vaf) +); + +DEFINE_EVENT(ath11k_log_event, ath11k_log_info, + TP_PROTO(struct ath11k_base *ab, struct va_format *vaf), + TP_ARGS(ab, vaf) +); + +TRACE_EVENT(ath11k_wmi_cmd, + TP_PROTO(struct ath11k_base *ab, int id, const void *buf, size_t buf_len), + + TP_ARGS(ab, id, buf, buf_len), + + TP_STRUCT__entry( + __string(device, dev_name(ab->dev)) + __string(driver, dev_driver_string(ab->dev)) + __field(unsigned int, id) + __field(size_t, buf_len) + __dynamic_array(u8, buf, buf_len) + ), + + TP_fast_assign( + __assign_str(device, dev_name(ab->dev)); + __assign_str(driver, dev_driver_string(ab->dev)); + __entry->id = id; + __entry->buf_len = buf_len; + memcpy(__get_dynamic_array(buf), buf, buf_len); + ), + + TP_printk( + "%s %s id %d len %zu", + __get_str(driver), + __get_str(device), + __entry->id, + __entry->buf_len + ) +); + +TRACE_EVENT(ath11k_wmi_event, + TP_PROTO(struct ath11k_base *ab, int id, const void *buf, size_t buf_len), + + TP_ARGS(ab, id, buf, buf_len), + + TP_STRUCT__entry( + __string(device, dev_name(ab->dev)) + __string(driver, dev_driver_string(ab->dev)) + __field(unsigned int, id) + __field(size_t, buf_len) + __dynamic_array(u8, buf, buf_len) + ), + + TP_fast_assign( + __assign_str(device, dev_name(ab->dev)); + __assign_str(driver, dev_driver_string(ab->dev)); + __entry->id = id; + __entry->buf_len = buf_len; + memcpy(__get_dynamic_array(buf), buf, buf_len); + ), + + TP_printk( + "%s %s id %d len %zu", + __get_str(driver), + __get_str(device), + __entry->id, + __entry->buf_len + ) +); + +TRACE_EVENT(ath11k_log_dbg, + TP_PROTO(struct ath11k_base *ab, unsigned int level, struct va_format *vaf), + + TP_ARGS(ab, level, vaf), + + TP_STRUCT__entry( + __string(device, dev_name(ab->dev)) + __string(driver, dev_driver_string(ab->dev)) + __field(unsigned int, level) + __dynamic_array(char, msg, ATH11K_MSG_MAX) + ), + + TP_fast_assign( + __assign_str(device, dev_name(ab->dev)); + __assign_str(driver, dev_driver_string(ab->dev)); + __entry->level = level; + WARN_ON_ONCE(vsnprintf(__get_dynamic_array(msg), + ATH11K_MSG_MAX, vaf->fmt, + *vaf->va) >= ATH11K_MSG_MAX); + ), + + TP_printk( + "%s %s %s", + __get_str(driver), + __get_str(device), + __get_str(msg) + ) +); + +TRACE_EVENT(ath11k_log_dbg_dump, + TP_PROTO(struct ath11k_base *ab, const char *msg, const char *prefix, + const void *buf, size_t buf_len), + + TP_ARGS(ab, msg, prefix, buf, buf_len), + + TP_STRUCT__entry( + __string(device, dev_name(ab->dev)) + __string(driver, dev_driver_string(ab->dev)) + __string(msg, msg) + __string(prefix, prefix) + __field(size_t, buf_len) + __dynamic_array(u8, buf, buf_len) + ), + + TP_fast_assign( + __assign_str(device, dev_name(ab->dev)); + __assign_str(driver, dev_driver_string(ab->dev)); + __assign_str(msg, msg); + __assign_str(prefix, prefix); + __entry->buf_len = buf_len; + memcpy(__get_dynamic_array(buf), buf, buf_len); + ), + + TP_printk( + "%s %s %s/%s\n", + __get_str(driver), + __get_str(device), + __get_str(prefix), + __get_str(msg) + ) +); #endif /* _TRACE_H_ || TRACE_HEADER_MULTI_READ*/ /* we don't want to use include/trace/events */ diff --git a/drivers/net/wireless/ath/ath11k/wmi.c b/drivers/net/wireless/ath/ath11k/wmi.c index 5ae2ef4680d6..614b2f6bcc8e 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.c +++ b/drivers/net/wireless/ath/ath11k/wmi.c @@ -128,6 +128,8 @@ static const struct wmi_tlv_policy wmi_tlv_policies[] = { .min_len = sizeof(struct wmi_probe_resp_tx_status_event) }, [WMI_TAG_VDEV_DELETE_RESP_EVENT] = { .min_len = sizeof(struct wmi_vdev_delete_resp_event) }, + [WMI_TAG_OBSS_COLOR_COLLISION_EVT] = { + .min_len = sizeof(struct wmi_obss_color_collision_event) }, }; #define PRIMAP(_hw_mode_) \ @@ -249,6 +251,8 @@ static int ath11k_wmi_cmd_send_nowait(struct ath11k_pdev_wmi *wmi, struct sk_buf cmd_hdr = (struct wmi_cmd_hdr *)skb->data; cmd_hdr->cmd_id = cmd; + trace_ath11k_wmi_cmd(ab, cmd_id, skb->data, skb->len); + memset(skb_cb, 0, sizeof(*skb_cb)); ret = ath11k_htc_send(&ab->htc, wmi->eid, skb); @@ -267,21 +271,39 @@ int ath11k_wmi_cmd_send(struct ath11k_pdev_wmi *wmi, struct sk_buff *skb, { struct ath11k_wmi_base *wmi_sc = wmi->wmi_ab; int ret = -EOPNOTSUPP; + struct ath11k_base *ab = wmi_sc->ab; might_sleep(); - wait_event_timeout(wmi_sc->tx_credits_wq, ({ - ret = ath11k_wmi_cmd_send_nowait(wmi, skb, cmd_id); + if (ab->hw_params.credit_flow) { + wait_event_timeout(wmi_sc->tx_credits_wq, ({ + ret = ath11k_wmi_cmd_send_nowait(wmi, skb, cmd_id); + + if (ret && test_bit(ATH11K_FLAG_CRASH_FLUSH, + &wmi_sc->ab->dev_flags)) + ret = -ESHUTDOWN; + + (ret != -EAGAIN); + }), WMI_SEND_TIMEOUT_HZ); + } else { + wait_event_timeout(wmi->tx_ce_desc_wq, ({ + ret = ath11k_wmi_cmd_send_nowait(wmi, skb, cmd_id); - if (ret && test_bit(ATH11K_FLAG_CRASH_FLUSH, &wmi_sc->ab->dev_flags)) - ret = -ESHUTDOWN; + if (ret && test_bit(ATH11K_FLAG_CRASH_FLUSH, + &wmi_sc->ab->dev_flags)) + ret = -ESHUTDOWN; - (ret != -EAGAIN); - }), WMI_SEND_TIMEOUT_HZ); + (ret != -ENOBUFS); + }), WMI_SEND_TIMEOUT_HZ); + } if (ret == -EAGAIN) ath11k_warn(wmi_sc->ab, "wmi command %d timeout\n", cmd_id); + if (ret == -ENOBUFS) + ath11k_warn(wmi_sc->ab, "ce desc not available for wmi command %d\n", + cmd_id); + return ret; } @@ -1244,7 +1266,8 @@ int ath11k_wmi_pdev_set_param(struct ath11k *ar, u32 param_id, return ret; } -int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, u32 enable) +int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, + enum wmi_sta_ps_mode psmode) { struct ath11k_pdev_wmi *wmi = ar->wmi; struct wmi_pdev_set_ps_mode_cmd *cmd; @@ -1259,7 +1282,7 @@ int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, u32 enable) cmd->tlv_header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_STA_POWERSAVE_MODE_CMD) | FIELD_PREP(WMI_TLV_LEN, sizeof(*cmd) - TLV_HDR_SIZE); cmd->vdev_id = vdev_id; - cmd->sta_ps_mode = enable; + cmd->sta_ps_mode = psmode; ret = ath11k_wmi_cmd_send(wmi, skb, WMI_STA_POWERSAVE_MODE_CMDID); if (ret) { @@ -1269,7 +1292,7 @@ int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, u32 enable) ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "WMI vdev set psmode %d vdev id %d\n", - enable, vdev_id); + psmode, vdev_id); return ret; } @@ -1612,6 +1635,15 @@ int ath11k_wmi_bcn_tmpl(struct ath11k *ar, u32 vdev_id, void *ptr; int ret, len; size_t aligned_len = roundup(bcn->len, 4); + struct ieee80211_vif *vif; + struct ath11k_vif *arvif = ath11k_mac_get_arvif(ar, vdev_id); + + if (!arvif) { + ath11k_warn(ar->ab, "failed to find arvif with vdev id %d\n", vdev_id); + return -EINVAL; + } + + vif = arvif->vif; len = sizeof(*cmd) + sizeof(*bcn_prb_info) + TLV_HDR_SIZE + aligned_len; @@ -1624,8 +1656,12 @@ int ath11k_wmi_bcn_tmpl(struct ath11k *ar, u32 vdev_id, FIELD_PREP(WMI_TLV_LEN, sizeof(*cmd) - TLV_HDR_SIZE); cmd->vdev_id = vdev_id; cmd->tim_ie_offset = offs->tim_offset; - cmd->csa_switch_count_offset = offs->cntdwn_counter_offs[0]; - cmd->ext_csa_switch_count_offset = offs->cntdwn_counter_offs[1]; + + if (vif->csa_active) { + cmd->csa_switch_count_offset = offs->cntdwn_counter_offs[0]; + cmd->ext_csa_switch_count_offset = offs->cntdwn_counter_offs[1]; + } + cmd->buf_len = bcn->len; ptr = skb->data + sizeof(*cmd); @@ -1689,7 +1725,8 @@ int ath11k_wmi_vdev_install_key(struct ath11k *ar, tlv = (struct wmi_tlv *)(skb->data + sizeof(*cmd)); tlv->header = FIELD_PREP(WMI_TLV_TAG, WMI_TAG_ARRAY_BYTE) | FIELD_PREP(WMI_TLV_LEN, key_len_aligned); - memcpy(tlv->value, (u8 *)arg->key_data, key_len_aligned); + if (arg->key_data) + memcpy(tlv->value, (u8 *)arg->key_data, key_len_aligned); ret = ath11k_wmi_cmd_send(wmi, skb, WMI_VDEV_INSTALL_KEY_CMDID); if (ret) { @@ -1762,7 +1799,7 @@ ath11k_wmi_copy_peer_flags(struct wmi_peer_assoc_complete_cmd *cmd, cmd->peer_flags |= WMI_PEER_AUTH; if (param->need_ptk_4_way) { cmd->peer_flags |= WMI_PEER_NEED_PTK_4_WAY; - if (!hw_crypto_disabled) + if (!hw_crypto_disabled && param->is_assoc) cmd->peer_flags &= ~WMI_PEER_AUTH; } if (param->need_gtk_2_way) @@ -2386,6 +2423,8 @@ int ath11k_wmi_send_scan_chan_list_cmd(struct ath11k *ar, tchan_info->reg_class_id); *reg2 |= FIELD_PREP(WMI_CHAN_REG_INFO2_ANT_MAX, tchan_info->antennamax); + *reg2 |= FIELD_PREP(WMI_CHAN_REG_INFO2_MAX_TX_PWR, + tchan_info->maxregpower); ath11k_dbg(ar->ab, ATH11K_DBG_WMI, "WMI chan scan list chan[%d] = %u, chan_info->info %8x\n", @@ -3428,6 +3467,53 @@ int ath11k_wmi_fils_discovery(struct ath11k *ar, u32 vdev_id, u32 interval, } static void +ath11k_wmi_obss_color_collision_event(struct ath11k_base *ab, struct sk_buff *skb) +{ + const void **tb; + const struct wmi_obss_color_collision_event *ev; + struct ath11k_vif *arvif; + int ret; + + tb = ath11k_wmi_tlv_parse_alloc(ab, skb->data, skb->len, GFP_ATOMIC); + if (IS_ERR(tb)) { + ret = PTR_ERR(tb); + ath11k_warn(ab, "failed to parse tlv: %d\n", ret); + return; + } + + ev = tb[WMI_TAG_OBSS_COLOR_COLLISION_EVT]; + if (!ev) { + ath11k_warn(ab, "failed to fetch obss color collision ev"); + goto exit; + } + + arvif = ath11k_mac_get_arvif_by_vdev_id(ab, ev->vdev_id); + if (!arvif) { + ath11k_warn(ab, "failed to find arvif with vedv id %d in obss_color_collision_event\n", + ev->vdev_id); + goto exit; + } + + switch (ev->evt_type) { + case WMI_BSS_COLOR_COLLISION_DETECTION: + ieeee80211_obss_color_collision_notify(arvif->vif, ev->obss_color_bitmap); + ath11k_dbg(ab, ATH11K_DBG_WMI, + "OBSS color collision detected vdev:%d, event:%d, bitmap:%08llx\n", + ev->vdev_id, ev->evt_type, ev->obss_color_bitmap); + break; + case WMI_BSS_COLOR_COLLISION_DISABLE: + case WMI_BSS_COLOR_FREE_SLOT_TIMER_EXPIRY: + case WMI_BSS_COLOR_FREE_SLOT_AVAILABLE: + break; + default: + ath11k_warn(ab, "received unknown obss color collision detetction event\n"); + } + +exit: + kfree(tb); +} + +static void ath11k_fill_band_to_mac_param(struct ath11k_base *soc, struct wmi_host_pdev_band_to_mac *band_to_mac) { @@ -5813,7 +5899,30 @@ static void ath11k_wmi_op_ep_tx_credits(struct ath11k_base *ab) static void ath11k_wmi_htc_tx_complete(struct ath11k_base *ab, struct sk_buff *skb) { + struct ath11k_pdev_wmi *wmi = NULL; + u32 i; + u8 wmi_ep_count; + u8 eid; + + eid = ATH11K_SKB_CB(skb)->eid; dev_kfree_skb(skb); + + if (eid >= ATH11K_HTC_EP_COUNT) + return; + + wmi_ep_count = ab->htc.wmi_ep_count; + if (wmi_ep_count > ab->hw_params.max_radios) + return; + + for (i = 0; i < ab->htc.wmi_ep_count; i++) { + if (ab->wmi_ab.wmi[i].eid == eid) { + wmi = &ab->wmi_ab.wmi[i]; + break; + } + } + + if (wmi) + wake_up(&wmi->tx_ce_desc_wq); } static bool ath11k_reg_is_world_alpha(char *alpha) @@ -6111,6 +6220,7 @@ static void ath11k_vdev_start_resp_event(struct ath11k_base *ab, struct sk_buff static void ath11k_bcn_tx_status_event(struct ath11k_base *ab, struct sk_buff *skb) { + struct ath11k_vif *arvif; u32 vdev_id, tx_status; if (ath11k_pull_bcn_tx_status_ev(ab, skb->data, skb->len, @@ -6118,6 +6228,14 @@ static void ath11k_bcn_tx_status_event(struct ath11k_base *ab, struct sk_buff *s ath11k_warn(ab, "failed to extract bcn tx status"); return; } + + arvif = ath11k_mac_get_arvif_by_vdev_id(ab, vdev_id); + if (!arvif) { + ath11k_warn(ab, "invalid vdev id %d in bcn_tx_status", + vdev_id); + return; + } + ath11k_mac_bcn_tx_event(arvif); } static void ath11k_vdev_stopped_event(struct ath11k_base *ab, struct sk_buff *skb) @@ -6398,6 +6516,7 @@ static void ath11k_peer_sta_kickout_event(struct ath11k_base *ab, struct sk_buff struct ieee80211_sta *sta; struct ath11k_peer *peer; struct ath11k *ar; + u32 vdev_id; if (ath11k_pull_peer_sta_kickout_ev(ab, skb, &arg) != 0) { ath11k_warn(ab, "failed to extract peer sta kickout event"); @@ -6413,10 +6532,15 @@ static void ath11k_peer_sta_kickout_event(struct ath11k_base *ab, struct sk_buff if (!peer) { ath11k_warn(ab, "peer not found %pM\n", arg.mac_addr); + spin_unlock_bh(&ab->base_lock); goto exit; } - ar = ath11k_mac_get_ar_by_vdev_id(ab, peer->vdev_id); + vdev_id = peer->vdev_id; + + spin_unlock_bh(&ab->base_lock); + + ar = ath11k_mac_get_ar_by_vdev_id(ab, vdev_id); if (!ar) { ath11k_warn(ab, "invalid vdev id in peer sta kickout ev %d", peer->vdev_id); @@ -6437,7 +6561,6 @@ static void ath11k_peer_sta_kickout_event(struct ath11k_base *ab, struct sk_buff ieee80211_report_low_ack(sta, 10); exit: - spin_unlock_bh(&ab->base_lock); rcu_read_unlock(); } @@ -7054,6 +7177,8 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) cmd_hdr = (struct wmi_cmd_hdr *)skb->data; id = FIELD_GET(WMI_CMD_HDR_CMD_ID, (cmd_hdr->cmd_id)); + trace_ath11k_wmi_event(ab, id, skb->data, skb->len); + if (skb_pull(skb, sizeof(struct wmi_cmd_hdr)) == NULL) goto out; @@ -7138,6 +7263,9 @@ static void ath11k_wmi_tlv_op_rx(struct ath11k_base *ab, struct sk_buff *skb) case WMI_OFFLOAD_PROB_RESP_TX_STATUS_EVENTID: ath11k_probe_resp_tx_status_event(ab, skb); break; + case WMI_OBSS_COLOR_COLLISION_DETECTION_EVENTID: + ath11k_wmi_obss_color_collision_event(ab, skb); + break; /* add Unsupported events here */ case WMI_TBTTOFFSET_EXT_UPDATE_EVENTID: case WMI_PEER_OPER_MODE_CHANGE_EVENTID: @@ -7199,6 +7327,7 @@ static int ath11k_connect_pdev_htc_service(struct ath11k_base *ab, ab->wmi_ab.wmi_endpoint_id[pdev_idx] = conn_resp.eid; ab->wmi_ab.wmi[pdev_idx].eid = conn_resp.eid; ab->wmi_ab.max_msg_len[pdev_idx] = conn_resp.max_msg_len; + init_waitqueue_head(&ab->wmi_ab.wmi[pdev_idx].tx_ce_desc_wq); return 0; } diff --git a/drivers/net/wireless/ath/ath11k/wmi.h b/drivers/net/wireless/ath/ath11k/wmi.h index 0584e68e7593..4eb06cb7f883 100644 --- a/drivers/net/wireless/ath/ath11k/wmi.h +++ b/drivers/net/wireless/ath/ath11k/wmi.h @@ -774,6 +774,8 @@ enum wmi_tlv_event_id { WMI_MDNS_STATS_EVENTID = WMI_TLV_CMD(WMI_GRP_MDNS_OFL), WMI_SAP_OFL_ADD_STA_EVENTID = WMI_TLV_CMD(WMI_GRP_SAP_OFL), WMI_SAP_OFL_DEL_STA_EVENTID, + WMI_OBSS_COLOR_COLLISION_DETECTION_EVENTID = + WMI_EVT_GRP_START_ID(WMI_GRP_OBSS_OFL), WMI_OCB_SET_CONFIG_RESP_EVENTID = WMI_TLV_CMD(WMI_GRP_OCB), WMI_OCB_GET_TSF_TIMER_RESP_EVENTID, WMI_DCC_GET_STATS_RESP_EVENTID, @@ -2522,6 +2524,7 @@ struct ath11k_pdev_wmi { enum ath11k_htc_ep_id eid; const struct wmi_peer_flags_map *peer_flags; u32 rx_decap_mode; + wait_queue_head_t tx_ce_desc_wq; }; struct vdev_create_params { @@ -3617,6 +3620,7 @@ struct peer_assoc_params { u32 peer_he_tx_mcs_set[WMI_HOST_MAX_HE_RATE_SET]; bool twt_responder; bool twt_requester; + bool is_assoc; struct ath11k_ppe_threshold peer_ppet; }; @@ -4914,6 +4918,13 @@ struct wmi_pdev_obss_pd_bitmap_cmd { #define ATH11K_BSS_COLOR_COLLISION_DETECTION_STA_PERIOD_MS 10000 #define ATH11K_BSS_COLOR_COLLISION_DETECTION_AP_PERIOD_MS 5000 +enum wmi_bss_color_collision { + WMI_BSS_COLOR_COLLISION_DISABLE = 0, + WMI_BSS_COLOR_COLLISION_DETECTION, + WMI_BSS_COLOR_FREE_SLOT_TIMER_EXPIRY, + WMI_BSS_COLOR_FREE_SLOT_AVAILABLE, +}; + struct wmi_obss_color_collision_cfg_params_cmd { u32 tlv_header; u32 vdev_id; @@ -4931,6 +4942,12 @@ struct wmi_bss_color_change_enable_params_cmd { u32 enable; } __packed; +struct wmi_obss_color_collision_event { + u32 vdev_id; + u32 evt_type; + u64 obss_color_bitmap; +} __packed; + #define ATH11K_IPV4_TH_SEED_SIZE 5 #define ATH11K_IPV6_TH_SEED_SIZE 11 @@ -5351,7 +5368,8 @@ int ath11k_wmi_set_peer_param(struct ath11k *ar, const u8 *peer_addr, u32 vdev_id, u32 param_id, u32 param_val); int ath11k_wmi_pdev_set_param(struct ath11k *ar, u32 param_id, u32 param_value, u8 pdev_id); -int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, u32 enable); +int ath11k_wmi_pdev_set_ps_mode(struct ath11k *ar, int vdev_id, + enum wmi_sta_ps_mode psmode); int ath11k_wmi_wait_for_unified_ready(struct ath11k_base *ab); int ath11k_wmi_cmd_init(struct ath11k_base *ab); int ath11k_wmi_wait_for_service_ready(struct ath11k_base *ab); diff --git a/drivers/net/wireless/ath/ath9k/ar9002_mac.c b/drivers/net/wireless/ath/ath9k/ar9002_mac.c index ce9a0a53771e..fba5a847c3bb 100644 --- a/drivers/net/wireless/ath/ath9k/ar9002_mac.c +++ b/drivers/net/wireless/ath/ath9k/ar9002_mac.c @@ -120,7 +120,7 @@ static bool ar9002_hw_get_isr(struct ath_hw *ah, enum ath9k_int *masked, AR_ISR_TXEOL); } - ah->intr_txqs |= MS(s0_s, AR_ISR_S0_QCU_TXOK); + ah->intr_txqs = MS(s0_s, AR_ISR_S0_QCU_TXOK); ah->intr_txqs |= MS(s0_s, AR_ISR_S0_QCU_TXDESC); ah->intr_txqs |= MS(s1_s, AR_ISR_S1_QCU_TXERR); ah->intr_txqs |= MS(s1_s, AR_ISR_S1_QCU_TXEOL); diff --git a/drivers/net/wireless/ath/ath9k/ar9003_calib.c b/drivers/net/wireless/ath/ath9k/ar9003_calib.c index 7e27a06e5df1..dc24da1ff00b 100644 --- a/drivers/net/wireless/ath/ath9k/ar9003_calib.c +++ b/drivers/net/wireless/ath/ath9k/ar9003_calib.c @@ -1005,24 +1005,20 @@ static void __ar955x_tx_iq_cal_sort(struct ath_hw *ah, int i, int nmeasurement) { struct ath_common *common = ath9k_hw_common(ah); - int im, ix, iy, temp; + int im, ix, iy; for (im = 0; im < nmeasurement; im++) { for (ix = 0; ix < MAXIQCAL - 1; ix++) { for (iy = ix + 1; iy <= MAXIQCAL - 1; iy++) { if (coeff->mag_coeff[i][im][iy] < coeff->mag_coeff[i][im][ix]) { - temp = coeff->mag_coeff[i][im][ix]; - coeff->mag_coeff[i][im][ix] = - coeff->mag_coeff[i][im][iy]; - coeff->mag_coeff[i][im][iy] = temp; + swap(coeff->mag_coeff[i][im][ix], + coeff->mag_coeff[i][im][iy]); } if (coeff->phs_coeff[i][im][iy] < coeff->phs_coeff[i][im][ix]) { - temp = coeff->phs_coeff[i][im][ix]; - coeff->phs_coeff[i][im][ix] = - coeff->phs_coeff[i][im][iy]; - coeff->phs_coeff[i][im][iy] = temp; + swap(coeff->phs_coeff[i][im][ix], + coeff->phs_coeff[i][im][iy]); } } } diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.c b/drivers/net/wireless/ath/wcn36xx/dxe.c index aff04ef66266..4e9e13941c8f 100644 --- a/drivers/net/wireless/ath/wcn36xx/dxe.c +++ b/drivers/net/wireless/ath/wcn36xx/dxe.c @@ -272,6 +272,21 @@ static int wcn36xx_dxe_enable_ch_int(struct wcn36xx *wcn, u16 wcn_ch) return 0; } +static void wcn36xx_dxe_disable_ch_int(struct wcn36xx *wcn, u16 wcn_ch) +{ + int reg_data = 0; + + wcn36xx_dxe_read_register(wcn, + WCN36XX_DXE_INT_MASK_REG, + ®_data); + + reg_data &= ~wcn_ch; + + wcn36xx_dxe_write_register(wcn, + WCN36XX_DXE_INT_MASK_REG, + (int)reg_data); +} + static int wcn36xx_dxe_fill_skb(struct device *dev, struct wcn36xx_dxe_ctl *ctl, gfp_t gfp) @@ -834,6 +849,53 @@ unlock: return ret; } +static bool _wcn36xx_dxe_tx_channel_is_empty(struct wcn36xx_dxe_ch *ch) +{ + unsigned long flags; + struct wcn36xx_dxe_ctl *ctl_bd_start, *ctl_skb_start; + struct wcn36xx_dxe_ctl *ctl_bd, *ctl_skb; + bool ret = true; + + spin_lock_irqsave(&ch->lock, flags); + + /* Loop through ring buffer looking for nonempty entries. */ + ctl_bd_start = ch->head_blk_ctl; + ctl_bd = ctl_bd_start; + ctl_skb_start = ctl_bd_start->next; + ctl_skb = ctl_skb_start; + do { + if (ctl_skb->skb) { + ret = false; + goto unlock; + } + ctl_bd = ctl_skb->next; + ctl_skb = ctl_bd->next; + } while (ctl_skb != ctl_skb_start); + +unlock: + spin_unlock_irqrestore(&ch->lock, flags); + return ret; +} + +int wcn36xx_dxe_tx_flush(struct wcn36xx *wcn) +{ + int i = 0; + + /* Called with mac80211 queues stopped. Wait for empty HW queues. */ + do { + if (_wcn36xx_dxe_tx_channel_is_empty(&wcn->dxe_tx_l_ch) && + _wcn36xx_dxe_tx_channel_is_empty(&wcn->dxe_tx_h_ch)) { + return 0; + } + /* This ieee80211_ops callback is specifically allowed to + * sleep. + */ + usleep_range(1000, 1100); + } while (++i < 100); + + return -EBUSY; +} + int wcn36xx_dxe_init(struct wcn36xx *wcn) { int reg_data = 0, ret; @@ -869,7 +931,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn) WCN36XX_DXE_WQ_TX_L); wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, ®_data); - wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L); /***************************************/ /* Init descriptors for TX HIGH channel */ @@ -893,9 +954,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn) wcn36xx_dxe_read_register(wcn, WCN36XX_DXE_REG_CH_EN, ®_data); - /* Enable channel interrupts */ - wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H); - /***************************************/ /* Init descriptors for RX LOW channel */ /***************************************/ @@ -905,7 +963,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn) goto out_err_rxl_ch; } - /* For RX we need to preallocated buffers */ wcn36xx_dxe_ch_alloc_skb(wcn, &wcn->dxe_rx_l_ch); @@ -928,9 +985,6 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn) WCN36XX_DXE_REG_CTL_RX_L, WCN36XX_DXE_CH_DEFAULT_CTL_RX_L); - /* Enable channel interrupts */ - wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L); - /***************************************/ /* Init descriptors for RX HIGH channel */ /***************************************/ @@ -962,15 +1016,18 @@ int wcn36xx_dxe_init(struct wcn36xx *wcn) WCN36XX_DXE_REG_CTL_RX_H, WCN36XX_DXE_CH_DEFAULT_CTL_RX_H); - /* Enable channel interrupts */ - wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H); - ret = wcn36xx_dxe_request_irqs(wcn); if (ret < 0) goto out_err_irq; timer_setup(&wcn->tx_ack_timer, wcn36xx_dxe_tx_timer, 0); + /* Enable channel interrupts */ + wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L); + wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H); + wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L); + wcn36xx_dxe_enable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H); + return 0; out_err_irq: @@ -987,6 +1044,14 @@ out_err_txh_ch: void wcn36xx_dxe_deinit(struct wcn36xx *wcn) { + int reg_data = 0; + + /* Disable channel interrupts */ + wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_H); + wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_RX_L); + wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_H); + wcn36xx_dxe_disable_ch_int(wcn, WCN36XX_INT_MASK_CHAN_TX_L); + free_irq(wcn->tx_irq, wcn); free_irq(wcn->rx_irq, wcn); del_timer(&wcn->tx_ack_timer); @@ -996,6 +1061,15 @@ void wcn36xx_dxe_deinit(struct wcn36xx *wcn) wcn->tx_ack_skb = NULL; } + /* Put the DXE block into reset before freeing memory */ + reg_data = WCN36XX_DXE_REG_RESET; + wcn36xx_dxe_write_register(wcn, WCN36XX_DXE_REG_CSR_RESET, reg_data); + wcn36xx_dxe_ch_free_skbs(wcn, &wcn->dxe_rx_l_ch); wcn36xx_dxe_ch_free_skbs(wcn, &wcn->dxe_rx_h_ch); + + wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_l_ch); + wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_tx_h_ch); + wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_l_ch); + wcn36xx_dxe_deinit_descs(wcn->dev, &wcn->dxe_rx_h_ch); } diff --git a/drivers/net/wireless/ath/wcn36xx/dxe.h b/drivers/net/wireless/ath/wcn36xx/dxe.h index 31b81b7547a3..26a31edf52e9 100644 --- a/drivers/net/wireless/ath/wcn36xx/dxe.h +++ b/drivers/net/wireless/ath/wcn36xx/dxe.h @@ -466,5 +466,6 @@ int wcn36xx_dxe_tx_frame(struct wcn36xx *wcn, struct wcn36xx_tx_bd *bd, struct sk_buff *skb, bool is_low); +int wcn36xx_dxe_tx_flush(struct wcn36xx *wcn); void wcn36xx_dxe_tx_ack_ind(struct wcn36xx *wcn, u32 status); #endif /* _DXE_H_ */ diff --git a/drivers/net/wireless/ath/wcn36xx/main.c b/drivers/net/wireless/ath/wcn36xx/main.c index b04533bbc3a4..0110f5062296 100644 --- a/drivers/net/wireless/ath/wcn36xx/main.c +++ b/drivers/net/wireless/ath/wcn36xx/main.c @@ -402,6 +402,7 @@ static void wcn36xx_change_opchannel(struct wcn36xx *wcn, int ch) static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed) { struct wcn36xx *wcn = hw->priv; + int ret; wcn36xx_dbg(WCN36XX_DBG_MAC, "mac config changed 0x%08x\n", changed); @@ -417,17 +418,31 @@ static int wcn36xx_config(struct ieee80211_hw *hw, u32 changed) * want to receive/transmit regular data packets, then * simply stop the scan session and exit PS mode. */ - wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN, - wcn->sw_scan_vif); - wcn->sw_scan_channel = 0; + if (wcn->sw_scan_channel) + wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel); + if (wcn->sw_scan_init) { + wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN, + wcn->sw_scan_vif); + } } else if (wcn->sw_scan) { /* A scan is ongoing, do not change the operating * channel, but start a scan session on the channel. */ - wcn36xx_smd_init_scan(wcn, HAL_SYS_MODE_SCAN, - wcn->sw_scan_vif); + if (wcn->sw_scan_channel) + wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel); + if (!wcn->sw_scan_init) { + /* This can fail if we are unable to notify the + * operating channel. + */ + ret = wcn36xx_smd_init_scan(wcn, + HAL_SYS_MODE_SCAN, + wcn->sw_scan_vif); + if (ret) { + mutex_unlock(&wcn->conf_mutex); + return -EIO; + } + } wcn36xx_smd_start_scan(wcn, ch); - wcn->sw_scan_channel = ch; } else { wcn36xx_change_opchannel(wcn, ch); } @@ -707,6 +722,8 @@ static void wcn36xx_sw_scan_start(struct ieee80211_hw *hw, struct wcn36xx *wcn = hw->priv; struct wcn36xx_vif *vif_priv = wcn36xx_vif_to_priv(vif); + wcn36xx_dbg(WCN36XX_DBG_MAC, "sw_scan_start"); + wcn->sw_scan = true; wcn->sw_scan_vif = vif; wcn->sw_scan_channel = 0; @@ -721,8 +738,15 @@ static void wcn36xx_sw_scan_complete(struct ieee80211_hw *hw, { struct wcn36xx *wcn = hw->priv; + wcn36xx_dbg(WCN36XX_DBG_MAC, "sw_scan_complete"); + /* ensure that any scan session is finished */ - wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN, wcn->sw_scan_vif); + if (wcn->sw_scan_channel) + wcn36xx_smd_end_scan(wcn, wcn->sw_scan_channel); + if (wcn->sw_scan_init) { + wcn36xx_smd_finish_scan(wcn, HAL_SYS_MODE_SCAN, + wcn->sw_scan_vif); + } wcn->sw_scan = false; wcn->sw_scan_opchannel = 0; } @@ -1277,6 +1301,16 @@ static void wcn36xx_ipv6_addr_change(struct ieee80211_hw *hw, } #endif +static void wcn36xx_flush(struct ieee80211_hw *hw, struct ieee80211_vif *vif, + u32 queues, bool drop) +{ + struct wcn36xx *wcn = hw->priv; + + if (wcn36xx_dxe_tx_flush(wcn)) { + wcn36xx_err("Failed to flush hardware tx queues\n"); + } +} + static const struct ieee80211_ops wcn36xx_ops = { .start = wcn36xx_start, .stop = wcn36xx_stop, @@ -1304,6 +1338,7 @@ static const struct ieee80211_ops wcn36xx_ops = { #if IS_ENABLED(CONFIG_IPV6) .ipv6_addr_change = wcn36xx_ipv6_addr_change, #endif + .flush = wcn36xx_flush, CFG80211_TESTMODE_CMD(wcn36xx_tm_cmd) }; diff --git a/drivers/net/wireless/ath/wcn36xx/smd.c b/drivers/net/wireless/ath/wcn36xx/smd.c index ed45e2cf039b..d3285a504429 100644 --- a/drivers/net/wireless/ath/wcn36xx/smd.c +++ b/drivers/net/wireless/ath/wcn36xx/smd.c @@ -722,6 +722,7 @@ int wcn36xx_smd_init_scan(struct wcn36xx *wcn, enum wcn36xx_hal_sys_mode mode, wcn36xx_err("hal_init_scan response failed err=%d\n", ret); goto out; } + wcn->sw_scan_init = true; out: mutex_unlock(&wcn->hal_mutex); return ret; @@ -752,6 +753,7 @@ int wcn36xx_smd_start_scan(struct wcn36xx *wcn, u8 scan_channel) wcn36xx_err("hal_start_scan response failed err=%d\n", ret); goto out; } + wcn->sw_scan_channel = scan_channel; out: mutex_unlock(&wcn->hal_mutex); return ret; @@ -782,6 +784,7 @@ int wcn36xx_smd_end_scan(struct wcn36xx *wcn, u8 scan_channel) wcn36xx_err("hal_end_scan response failed err=%d\n", ret); goto out; } + wcn->sw_scan_channel = 0; out: mutex_unlock(&wcn->hal_mutex); return ret; @@ -823,6 +826,7 @@ int wcn36xx_smd_finish_scan(struct wcn36xx *wcn, wcn36xx_err("hal_finish_scan response failed err=%d\n", ret); goto out; } + wcn->sw_scan_init = false; out: mutex_unlock(&wcn->hal_mutex); return ret; @@ -2732,7 +2736,7 @@ static int wcn36xx_smd_missed_beacon_ind(struct wcn36xx *wcn, wcn36xx_dbg(WCN36XX_DBG_HAL, "beacon missed bss_index %d\n", tmp->bss_index); vif = wcn36xx_priv_to_vif(tmp); - ieee80211_connection_loss(vif); + ieee80211_beacon_loss(vif); } return 0; } @@ -2747,7 +2751,7 @@ static int wcn36xx_smd_missed_beacon_ind(struct wcn36xx *wcn, wcn36xx_dbg(WCN36XX_DBG_HAL, "beacon missed bss_index %d\n", rsp->bss_index); vif = wcn36xx_priv_to_vif(tmp); - ieee80211_connection_loss(vif); + ieee80211_beacon_loss(vif); return 0; } } diff --git a/drivers/net/wireless/ath/wcn36xx/txrx.c b/drivers/net/wireless/ath/wcn36xx/txrx.c index 75951ccbc840..dd58dde8c836 100644 --- a/drivers/net/wireless/ath/wcn36xx/txrx.c +++ b/drivers/net/wireless/ath/wcn36xx/txrx.c @@ -272,7 +272,6 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb) const struct wcn36xx_rate *rate; struct ieee80211_hdr *hdr; struct wcn36xx_rx_bd *bd; - struct ieee80211_supported_band *sband; u16 fc, sn; /* @@ -314,8 +313,6 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb) fc = __le16_to_cpu(hdr->frame_control); sn = IEEE80211_SEQ_TO_SN(__le16_to_cpu(hdr->seq_ctrl)); - status.freq = WCN36XX_CENTER_FREQ(wcn); - status.band = WCN36XX_BAND(wcn); status.mactime = 10; status.signal = -get_rssi0(bd); status.antenna = 1; @@ -327,18 +324,36 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb) wcn36xx_dbg(WCN36XX_DBG_RX, "status.flags=%x\n", status.flag); + if (bd->scan_learn) { + /* If packet originate from hardware scanning, extract the + * band/channel from bd descriptor. + */ + u8 hwch = (bd->reserved0 << 4) + bd->rx_ch; + + if (bd->rf_band != 1 && hwch <= sizeof(ab_rx_ch_map) && hwch >= 1) { + status.band = NL80211_BAND_5GHZ; + status.freq = ieee80211_channel_to_frequency(ab_rx_ch_map[hwch - 1], + status.band); + } else { + status.band = NL80211_BAND_2GHZ; + status.freq = ieee80211_channel_to_frequency(hwch, status.band); + } + } else { + status.band = WCN36XX_BAND(wcn); + status.freq = WCN36XX_CENTER_FREQ(wcn); + } + if (bd->rate_id < ARRAY_SIZE(wcn36xx_rate_table)) { rate = &wcn36xx_rate_table[bd->rate_id]; status.encoding = rate->encoding; status.enc_flags = rate->encoding_flags; status.bw = rate->bw; status.rate_idx = rate->mcs_or_legacy_index; - sband = wcn->hw->wiphy->bands[status.band]; status.nss = 1; if (status.band == NL80211_BAND_5GHZ && status.encoding == RX_ENC_LEGACY && - status.rate_idx >= sband->n_bitrates) { + status.rate_idx >= 4) { /* no dsss rates in 5Ghz rates table */ status.rate_idx -= 4; } @@ -353,22 +368,6 @@ int wcn36xx_rx_skb(struct wcn36xx *wcn, struct sk_buff *skb) ieee80211_is_probe_resp(hdr->frame_control)) status.boottime_ns = ktime_get_boottime_ns(); - if (bd->scan_learn) { - /* If packet originates from hardware scanning, extract the - * band/channel from bd descriptor. - */ - u8 hwch = (bd->reserved0 << 4) + bd->rx_ch; - - if (bd->rf_band != 1 && hwch <= sizeof(ab_rx_ch_map) && hwch >= 1) { - status.band = NL80211_BAND_5GHZ; - status.freq = ieee80211_channel_to_frequency(ab_rx_ch_map[hwch - 1], - status.band); - } else { - status.band = NL80211_BAND_2GHZ; - status.freq = ieee80211_channel_to_frequency(hwch, status.band); - } - } - memcpy(IEEE80211_SKB_RXCB(skb), &status, sizeof(status)); if (ieee80211_is_beacon(hdr->frame_control)) { diff --git a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h index 1c8d918137da..fbd0558c2c19 100644 --- a/drivers/net/wireless/ath/wcn36xx/wcn36xx.h +++ b/drivers/net/wireless/ath/wcn36xx/wcn36xx.h @@ -248,6 +248,7 @@ struct wcn36xx { struct cfg80211_scan_request *scan_req; bool sw_scan; u8 sw_scan_opchannel; + bool sw_scan_init; u8 sw_scan_channel; struct ieee80211_vif *sw_scan_vif; struct mutex scan_lock; diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c index fb727778312c..1679361f187b 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/cfg80211.c @@ -3901,6 +3901,24 @@ static void brcmf_configure_wowl(struct brcmf_cfg80211_info *cfg, cfg->wowl.active = true; } +static int brcmf_keepalive_start(struct brcmf_if *ifp, unsigned int interval) +{ + struct brcmf_mkeep_alive_pkt_le kalive = {0}; + int ret = 0; + + /* Configure Null function/data keepalive */ + kalive.version = cpu_to_le16(1); + kalive.period_msec = cpu_to_le16(interval * MSEC_PER_SEC); + kalive.len_bytes = cpu_to_le16(0); + kalive.keep_alive_id = cpu_to_le16(0); + + ret = brcmf_fil_iovar_data_set(ifp, "mkeep_alive", &kalive, sizeof(kalive)); + if (ret) + brcmf_err("keep-alive packet config failed, ret=%d\n", ret); + + return ret; +} + static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, struct cfg80211_wowlan *wowl) { @@ -3947,6 +3965,9 @@ static s32 brcmf_cfg80211_suspend(struct wiphy *wiphy, } else { /* Configure WOWL paramaters */ brcmf_configure_wowl(cfg, ifp, wowl); + + /* Prevent disassociation due to inactivity with keep-alive */ + brcmf_keepalive_start(ifp, 30); } exit: diff --git a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h index ff2ef557f0ea..e69d1e56996f 100644 --- a/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h +++ b/drivers/net/wireless/broadcom/brcm80211/brcmfmac/fwil_types.h @@ -1052,4 +1052,23 @@ struct brcmf_gscan_config { struct brcmf_gscan_bucket_config bucket[1]; }; +/** + * struct brcmf_mkeep_alive_pkt_le - configuration data for keep-alive frame. + * + * @version: version for mkeep_alive + * @length: length of fixed parameters in the structure. + * @period_msec: keep-alive period in milliseconds. + * @len_bytes: size of the data. + * @keep_alive_id: ID (0 - 3). + * @data: keep-alive frame data. + */ +struct brcmf_mkeep_alive_pkt_le { + __le16 version; + __le16 length; + __le32 period_msec; + __le16 len_bytes; + u8 keep_alive_id; + u8 data[0]; +} __packed; + #endif /* FWIL_TYPES_H_ */ diff --git a/drivers/net/wireless/intel/ipw2x00/ipw2200.c b/drivers/net/wireless/intel/ipw2x00/ipw2200.c index 23037bfc9e4c..5727c7c00a28 100644 --- a/drivers/net/wireless/intel/ipw2x00/ipw2200.c +++ b/drivers/net/wireless/intel/ipw2x00/ipw2200.c @@ -2303,7 +2303,7 @@ static int ipw_send_ssid(struct ipw_priv *priv, u8 * ssid, int len) ssid); } -static int ipw_send_adapter_address(struct ipw_priv *priv, u8 * mac) +static int ipw_send_adapter_address(struct ipw_priv *priv, const u8 * mac) { if (!priv || !mac) { IPW_ERROR("Invalid args\n"); diff --git a/drivers/net/wireless/intel/iwlwifi/Kconfig b/drivers/net/wireless/intel/iwlwifi/Kconfig index 1085afbefba8..337428ef2e67 100644 --- a/drivers/net/wireless/intel/iwlwifi/Kconfig +++ b/drivers/net/wireless/intel/iwlwifi/Kconfig @@ -92,6 +92,32 @@ config IWLWIFI_BCAST_FILTERING If unsure, don't enable this option, as some programs might expect incoming broadcasts for their normal operations. +config IWLMEI + tristate "Intel Management Engine communication over WLAN" + depends on INTEL_MEI + depends on PM + depends on IWLMVM + help + Enables the iwlmei kernel module. + + CSME stands for Converged Security and Management Engine. It is a CPU + on the chipset and runs a dedicated firmware. AMT (Active Management + Technology) is one of the applications that run on that CPU. AMT + allows to control the platform remotely. + + This kernel module allows to communicate with the Intel Management + Engine over Wifi. This is supported starting from Tiger Lake + platforms and has been tested on 9260 devices only. + If AMT is configured not to use the wireless device, this module is + harmless (and useless). + Enabling this option on a platform that has a different device and + has Wireless enabled on AMT can prevent WiFi from working correctly. + + For more information see + <https://software.intel.com/en-us/manageability/> + + If unsure, say N. + menu "Debugging Options" config IWLWIFI_DEBUG diff --git a/drivers/net/wireless/intel/iwlwifi/Makefile b/drivers/net/wireless/intel/iwlwifi/Makefile index 0d4656efe908..75a703eb1bdf 100644 --- a/drivers/net/wireless/intel/iwlwifi/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/Makefile @@ -30,5 +30,6 @@ ccflags-y += -I$(src) obj-$(CONFIG_IWLDVM) += dvm/ obj-$(CONFIG_IWLMVM) += mvm/ +obj-$(CONFIG_IWLMEI) += mei/ CFLAGS_iwl-devtrace.o := -I$(src) diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c index f470f9aea50f..0cd4718372f5 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.c @@ -22,6 +22,7 @@ #include "fw/api/commands.h" #include "fw/api/cmdhdr.h" #include "fw/img.h" +#include "mei/iwl-mei.h" /* NVM offsets (in words) definitions */ enum nvm_offsets { @@ -1115,6 +1116,66 @@ iwl_nvm_no_wide_in_5ghz(struct iwl_trans *trans, const struct iwl_cfg *cfg, } struct iwl_nvm_data * +iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg, + const struct iwl_mei_nvm *mei_nvm, + const struct iwl_fw *fw) +{ + struct iwl_nvm_data *data; + u32 sbands_flags = 0; + u8 rx_chains = fw->valid_rx_ant; + u8 tx_chains = fw->valid_rx_ant; + + if (cfg->uhb_supported) + data = kzalloc(struct_size(data, channels, + IWL_NVM_NUM_CHANNELS_UHB), + GFP_KERNEL); + else + data = kzalloc(struct_size(data, channels, + IWL_NVM_NUM_CHANNELS_EXT), + GFP_KERNEL); + if (!data) + return NULL; + + BUILD_BUG_ON(ARRAY_SIZE(mei_nvm->channels) != + IWL_NVM_NUM_CHANNELS_UHB); + data->nvm_version = mei_nvm->nvm_version; + + iwl_set_radio_cfg(cfg, data, mei_nvm->radio_cfg); + if (data->valid_tx_ant) + tx_chains &= data->valid_tx_ant; + if (data->valid_rx_ant) + rx_chains &= data->valid_rx_ant; + + data->sku_cap_mimo_disabled = false; + data->sku_cap_band_24ghz_enable = true; + data->sku_cap_band_52ghz_enable = true; + data->sku_cap_11n_enable = + !(iwlwifi_mod_params.disable_11n & IWL_DISABLE_HT_ALL); + data->sku_cap_11ac_enable = true; + data->sku_cap_11ax_enable = + mei_nvm->caps & MEI_NVM_CAPS_11AX_SUPPORT; + + data->lar_enabled = mei_nvm->caps & MEI_NVM_CAPS_LARI_SUPPORT; + + data->n_hw_addrs = mei_nvm->n_hw_addrs; + /* If no valid mac address was found - bail out */ + if (iwl_set_hw_address(trans, cfg, data, NULL, NULL)) { + kfree(data); + return NULL; + } + + if (data->lar_enabled && + fw_has_capa(&fw->ucode_capa, IWL_UCODE_TLV_CAPA_LAR_SUPPORT)) + sbands_flags |= IWL_NVM_SBANDS_FLAGS_LAR; + + iwl_init_sbands(trans, data, mei_nvm->channels, tx_chains, rx_chains, + sbands_flags, true, fw); + + return data; +} +IWL_EXPORT_SYMBOL(iwl_parse_mei_nvm_data); + +struct iwl_nvm_data * iwl_parse_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg, const struct iwl_fw *fw, const __be16 *nvm_hw, const __le16 *nvm_sw, diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h index e1f5a9741850..e01f7751cf11 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-nvm-parse.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2005-2015, 2018-2020 Intel Corporation + * Copyright (C) 2005-2015, 2018-2021 Intel Corporation * Copyright (C) 2016-2017 Intel Deutschland GmbH */ #ifndef __iwl_nvm_parse_h__ @@ -8,6 +8,7 @@ #include <net/cfg80211.h> #include "iwl-eeprom-parse.h" +#include "mei/iwl-mei.h" /** * enum iwl_nvm_sbands_flags - modification flags for the channel profiles @@ -81,4 +82,12 @@ void iwl_nvm_fixups(u32 hw_id, unsigned int section, u8 *data, struct iwl_nvm_data *iwl_get_nvm(struct iwl_trans *trans, const struct iwl_fw *fw); +/** + * iwl_parse_mei_nvm_data - parse the mei_nvm_data and get an iwl_nvm_data + */ +struct iwl_nvm_data * +iwl_parse_mei_nvm_data(struct iwl_trans *trans, const struct iwl_cfg *cfg, + const struct iwl_mei_nvm *mei_nvm, + const struct iwl_fw *fw); + #endif /* __iwl_nvm_parse_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h index 4ebb1871bd1f..a4060af50201 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.h +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.h @@ -924,6 +924,7 @@ struct iwl_trans_txqs { /** * struct iwl_trans - transport common data * + * @csme_own - true if we couldn't get ownership on the device * @ops - pointer to iwl_trans_ops * @op_mode - pointer to the op_mode * @trans_cfg: the trans-specific configuration part @@ -958,6 +959,7 @@ struct iwl_trans_txqs { * @iwl_trans_txqs: transport tx queues data. */ struct iwl_trans { + bool csme_own; const struct iwl_trans_ops *ops; struct iwl_op_mode *op_mode; const struct iwl_cfg_trans_params *trans_cfg; diff --git a/drivers/net/wireless/intel/iwlwifi/mei/Makefile b/drivers/net/wireless/intel/iwlwifi/mei/Makefile new file mode 100644 index 000000000000..8e3ef0347db7 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_IWLMEI) += iwlmei.o +iwlmei-y += main.o +iwlmei-y += net.o +iwlmei-$(CONFIG_IWLWIFI_DEVICE_TRACING) += trace.o +CFLAGS_trace.o := -I$(src) + +ccflags-y += -I $(srctree)/$(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/internal.h b/drivers/net/wireless/intel/iwlwifi/mei/internal.h new file mode 100644 index 000000000000..92fea7dd71e2 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/internal.h @@ -0,0 +1,20 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#ifndef __IWLMEI_INTERNAL_H_ +#define __IWLMEI_INTERNAL_H_ + +#include <uapi/linux/if_ether.h> +#include <linux/netdevice.h> + +#include "sap.h" + +rx_handler_result_t iwl_mei_rx_filter(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + bool *pass_to_csme); + +void iwl_mei_add_data_to_ring(struct sk_buff *skb, bool cb_tx); + +#endif /* __IWLMEI_INTERNAL_H_ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h b/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h new file mode 100644 index 000000000000..67122cfa2292 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/iwl-mei.h @@ -0,0 +1,505 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#ifndef __iwl_mei_h__ +#define __iwl_mei_h__ + +#include <linux/if_ether.h> +#include <linux/skbuff.h> +#include <linux/ieee80211.h> + +/** + * DOC: Introduction + * + * iwlmei is the kernel module that is in charge of the commnunication between + * the iwlwifi driver and the CSME firmware's WLAN driver. This communication + * uses the SAP protocol defined in another file. + * iwlwifi can request or release ownership on the WiFi device through iwlmei. + * iwlmei may notify iwlwifi about certain events: what filter iwlwifi should + * use to passthrough inbound packets to the CSME firmware for example. iwlmei + * may also use iwlwifi to send traffic. This means that we need communication + * from iwlmei to iwlwifi and the other way around. + */ + +/** + * DOC: Life cycle + * + * iwlmei exports symbols that are needed by iwlwifi so that iwlmei will always + * be loaded when iwlwifi is alive. iwlwifi registers itself to iwlmei and + * provides the pointers to the functions that iwlmei calls whenever needed. + * iwlwifi calls iwlmei through direct and context-free function calls. + * It is assumed that only one device is accessible to the CSME firmware and + * under the scope of iwlmei so that it is valid not to have any context passed + * to iwlmei's functions. + * + * There are cases in which iwlmei can't access the CSME firmware, because the + * CSME firmware is undergoing a reset, or the mei bus decided to unbind the + * device. In those cases, iwlmei will need not to send requests over the mei + * bus. Instead, it needs to cache the requests from iwlwifi and fulfill them + * when the mei bus is available again. + * + * iwlmei can call iwlwifi as long as iwlwifi is registered to iwlmei. When + * iwlwifi goes down (the PCI device is unbound, or the iwlwifi is unloaded) + * iwlwifi needs to unregister from iwlmei. + */ + +/** + * DOC: Memory layout + * + * Since iwlwifi calls iwlmei without any context, iwlmei needs to hold a + * global pointer to its data (which is in the mei client device's private + * data area). If there was no bind on the mei bus, this pointer is NULL and + * iwlmei knows not access to the CSME firmware upon requests from iwlwifi. + * + * iwlmei needs to cache requests from iwlwifi when there is no mei client + * device available (when iwlmei has been removed from the mei bus). In this + * case, all iwlmei's data that resides in the mei client device's private data + * area is unavailable. For this specific case, a separate caching area is + * needed. + */ + +/** + * DOC: Concurrency + * + * iwlwifi can call iwlmei at any time. iwlmei will take care to synchronize + * the calls from iwlwifi with its internal flows. iwlwifi must not call iwlmei + * in flows that cannot sleep. Moreover, iwlwifi must not call iwlmei in flows + * that originated from iwlmei. + */ + +/** + * DOC: Probe and remove from mei bus driver + * + * When the mei bus driver enumerates its devices, it calls the iwlmei's probe + * function which will send the %SAP_ME_MSG_START message. The probe completes + * before the response (%SAP_ME_MSG_START_OK) is received. This response will + * be handle by the Rx path. Once it arrives, the connection to the CSME + * firmware is considered established and iwlwifi's requests can be treated + * against the CSME firmware. + * + * When the mei bus driver removes the device, iwlmei loses all the data that + * was attached to the mei client device. It clears the global pointer to the + * mei client device since it is not available anymore. This will cause all the + * requests coming from iwlwifi to be cached. This flow takes the global mutex + * to be synchronized with all the requests coming from iwlwifi. + */ + +/** + * DOC: Driver load when CSME owns the device + * + * When the driver (iwlwifi) is loaded while CSME owns the device, + * it'll ask CSME to release the device through HW registers. CSME + * will release the device only in the case that there is no connection + * through the mei bus. If there is a mei bus connection, CSME will refuse + * to release the ownership on the device through the HW registers. In that + * case, iwlwifi must first request ownership using the SAP protocol. + * + * Once iwlwifi will request ownership through the SAP protocol, CSME will + * grant the ownership on the device through the HW registers as well. + * In order to request ownership over SAP, we first need to have an interface + * which means that we need to register to mac80211. + * This can't happen before we get the NVM that contains all the capabilities + * of the device. Reading the NVM usually requires the load the firmware, but + * this is impossible as long as we don't have ownership on the device. + * In order to solve this chicken and egg problem, the host driver can get + * the NVM through CSME which owns the device. It can send + * %SAP_MSG_NOTIF_GET_NVM, which will be replied by %SAP_MSG_NOTIF_NVM with + * the NVM's content that the host driver needs. + */ + +/** + * DOC: CSME behavior regarding the ownership requests + * + * The ownership requests from the host can come in two different ways: + * - the HW registers in iwl_pcie_set_hw_ready + * - using the Software Arbitration Protocol (SAP) + * + * The host can ask CSME who owns the device with %SAP_MSG_NOTIF_WHO_OWNS_NIC, + * and it can request ownership with %SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP. + * The host will first use %SAP_MSG_NOTIF_WHO_OWNS_NIC to know what state + * CSME is in. In case CSME thinks it owns the device, the host can ask for + * ownership with %SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP. + * + * Here the table that describes CSME's behavior upon ownership request: + * + * +-------------------+------------+--------------+-----------------------------+------------+ + * | State | HW reg bit | Reply for | Event | HW reg bit | + * | | before | WHO_OWNS_NIC | | after | + * +===================+============+==============+=============================+============+ + * | WiAMT not | 0 | Host | HW register or | 0 | + * | operational | Host owner | | HOST_ASKS_FOR_NIC_OWNERSHIP | Host owner | + * +-------------------+------------+--------------+-----------------------------+------------+ + * | Operational & | 1 | N/A | HW register | 0 | + * | SAP down & | CSME owner | | | Host owner | + * | no session active | | | | | + * +-------------------+------------+--------------+-----------------------------+------------+ + * | Operational & | 1 | CSME | HW register | 1 | + * | SAP up | CSME owner | | | CSME owner | + * +-------------------+------------+--------------+-----------------------------+------------+ + * | Operational & | 1 | CSME | HOST_ASKS_FOR_NIC_OWNERSHIP | 0 | + * | SAP up | CSME owner | | | Host owner | + * +-------------------+------------+--------------+-----------------------------+------------+ + */ + +/** + * DOC: Driver load when CSME is associated and a session is active + * + * A "session" is active when CSME is associated to an access point and the + * link is used to attach a remote driver or to control the system remotely. + * When a session is active, we want to make sure it won't disconnect when we + * take ownership on the device. + * In this case, the driver can get the device, but it'll need to make + * sure that it'll connect to the exact same AP (same BSSID). + * In order to do so, CSME will send the connection parameters through + * SAP and then the host can check if it can connect to this same AP. + * If yes, it can request ownership through SAP and connect quickly without + * scanning all the channels, but just probing the AP on the channel that + * CSME was connected to. + * In order to signal this specific scenario to iwlwifi, iwlmei will + * immediately require iwlwifi to report RF-Kill to the network stack. This + * RF-Kill will prevent the stack from getting the device, and it has a reason + * that tells the userspace that the device is in RF-Kill because it is not + * owned by the host. Once the userspace has configured the right profile, + * it'll be able to let iwlmei know that it can request ownership over SAP + * which will remove the RF-Kill, and finally allow the host to connect. + * The host has then 3 seconds to connect (including DHCP). Had the host + * failed to connect within those 3 seconds, CSME will take the device back. + */ + +/** + * DOC: Datapath + * + * CSME can transmit packets, through the netdev that it gets from the wifi + * driver. It'll send packet in the 802.3 format and simply call + * dev_queue_xmit. + * + * For Rx, iwlmei registers a Rx handler that it attaches to the netdev. iwlmei + * may catch packets and send them to CSME, it can then either drop them so + * that they are invisible to user space, or let them go the user space. + * + * Packets transmitted by the user space do not need to be forwarded to CSME + * with the exception of the DHCP request. In order to know what IP is used + * by the user space, CSME needs to get the DHCP request. See + * iwl_mei_tx_copy_to_csme(). + */ + +/** + * enum iwl_mei_nvm_caps - capabilities for MEI NVM + * @MEI_NVM_CAPS_LARI_SUPPORT: Lari is supported + * @MEI_NVM_CAPS_11AX_SUPPORT: 11AX is supported + */ +enum iwl_mei_nvm_caps { + MEI_NVM_CAPS_LARI_SUPPORT = BIT(0), + MEI_NVM_CAPS_11AX_SUPPORT = BIT(1), +}; + +/** + * struct iwl_mei_nvm - used to pass the NVM from CSME + * @hw_addr: The MAC address + * @n_hw_addrs: The number of MAC addresses + * @reserved: For alignment. + * @radio_cfg: The radio configuration. + * @caps: See &enum iwl_mei_nvm_caps. + * @nvm_version: The version of the NVM. + * @channels: The data for each channel. + * + * If a field is added, it must correspond to the SAP structure. + */ +struct iwl_mei_nvm { + u8 hw_addr[ETH_ALEN]; + u8 n_hw_addrs; + u8 reserved; + u32 radio_cfg; + u32 caps; + u32 nvm_version; + u32 channels[110]; +}; + +/** + * enum iwl_mei_pairwise_cipher - cipher for UCAST key + * @IWL_MEI_CIPHER_NONE: none + * @IWL_MEI_CIPHER_CCMP: ccmp + * @IWL_MEI_CIPHER_GCMP: gcmp + * @IWL_MEI_CIPHER_GCMP_256: gcmp 256 + * + * Note that those values are dictated by the CSME firmware API (see sap.h) + */ +enum iwl_mei_pairwise_cipher { + IWL_MEI_CIPHER_NONE = 0, + IWL_MEI_CIPHER_CCMP = 4, + IWL_MEI_CIPHER_GCMP = 8, + IWL_MEI_CIPHER_GCMP_256 = 9, +}; + +/** + * enum iwl_mei_akm_auth - a combination of AKM and AUTH method + * @IWL_MEI_AKM_AUTH_OPEN: No encryption + * @IWL_MEI_AKM_AUTH_RSNA: 1X profile + * @IWL_MEI_AKM_AUTH_RSNA_PSK: PSK profile + * @IWL_MEI_AKM_AUTH_SAE: SAE profile + * + * Note that those values are dictated by the CSME firmware API (see sap.h) + */ +enum iwl_mei_akm_auth { + IWL_MEI_AKM_AUTH_OPEN = 0, + IWL_MEI_AKM_AUTH_RSNA = 6, + IWL_MEI_AKM_AUTH_RSNA_PSK = 7, + IWL_MEI_AKM_AUTH_SAE = 9, +}; + +/** + * struct iwl_mei_conn_info - connection info + * @lp_state: link protection state + * @auth_mode: authentication mode + * @ssid_len: the length of SSID + * @ssid: the SSID + * @pairwise_cipher: the cipher used for unicast packets + * @channel: the associated channel + * @band: the associated band + * @bssid: the BSSID + */ +struct iwl_mei_conn_info { + u8 lp_state; + u8 auth_mode; + u8 ssid_len; + u8 channel; + u8 band; + u8 pairwise_cipher; + u8 bssid[ETH_ALEN]; + u8 ssid[IEEE80211_MAX_SSID_LEN]; +}; + +/** + * struct iwl_mei_colloc_info - collocated AP info + * @channel: the channel of the collocated AP + * @bssid: the BSSID of the collocated AP + */ +struct iwl_mei_colloc_info { + u8 channel; + u8 bssid[ETH_ALEN]; +}; + +/* + * struct iwl_mei_ops - driver's operations called by iwlmei + * Operations will not be called more than once concurrently. + * It's not allowed to call iwlmei functions from this context. + * + * @me_conn_status: provide information about CSME's current connection. + * @rfkill: called when the wifi driver should report a change in the rfkill + * status. + * @roaming_forbidden: indicates whether roaming is forbidden. + * @sap_connected: indicate that SAP is now connected. Will be called in case + * the wifi driver registered to iwlmei before SAP connection succeeded or + * when the SAP connection is re-established. + * @nic_stolen: this means that device is no longer available. The device can + * still be used until the callback returns. + */ +struct iwl_mei_ops { + void (*me_conn_status)(void *priv, + const struct iwl_mei_conn_info *conn_info); + void (*rfkill)(void *priv, bool blocked); + void (*roaming_forbidden)(void *priv, bool forbidden); + void (*sap_connected)(void *priv); + void (*nic_stolen)(void *priv); +}; + +#if IS_ENABLED(CONFIG_IWLMEI) + +/** + * iwl_mei_is_connected() - is the connection to the CSME firmware established? + * + * Return: true if we have a SAP connection + */ +bool iwl_mei_is_connected(void); + +/** + * iwl_mei_get_nvm() - returns the NVM for the device + * + * It is the caller's responsibility to free the memory returned + * by this function. + * This function blocks (sleeps) until the NVM is ready. + * + * Return: the NVM as received from CSME + */ +struct iwl_mei_nvm *iwl_mei_get_nvm(void); + +/** + * iwl_mei_get_ownership() - request ownership + * + * This function blocks until ownership is granted or timeout expired. + * + * Return: 0 in case we could get ownership on the device + */ +int iwl_mei_get_ownership(void); + +/** + * iwl_mei_set_rfkill_state() - set SW and HW RF kill states + * @hw_rfkill: HW RF kill state. + * @sw_rfkill: SW RF kill state. + * + * This function must be called when SW RF kill is issued by the user. + */ +void iwl_mei_set_rfkill_state(bool hw_rfkill, bool sw_rfkill); + +/** + * iwl_mei_set_nic_info() - set mac address + * @mac_address: mac address to set + * @nvm_address: NVM mac adsress to set + * + * This function must be called upon mac address change. + */ +void iwl_mei_set_nic_info(const u8 *mac_address, const u8 *nvm_address); + +/** + * iwl_mei_set_country_code() - set new country code + * @mcc: the new applied MCC + * + * This function must be called upon country code update + */ +void iwl_mei_set_country_code(u16 mcc); + +/** + * iwl_mei_set_power_limit() - set TX power limit + * @power_limit: pointer to an array of 10 elements (le16) represents the power + * restrictions per chain. + * + * This function must be called upon power restrictions change + */ +void iwl_mei_set_power_limit(const __le16 *power_limit); + +/** + * iwl_mei_register() - register the wifi driver to iwlmei + * @priv: a pointer to the wifi driver's context. Cannot be NULL. + * @ops: the ops structure. + * + * Return: 0 unless something went wrong. It is illegal to call any + * other API function before this function is called and succeeds. + * + * Only one wifi driver instance (wifi device instance really) + * can register at a time. + */ +int iwl_mei_register(void *priv, const struct iwl_mei_ops *ops); + +/** + * iwl_mei_start_unregister() - unregister the wifi driver from iwlmei + * + * From this point on, iwlmei will not used the callbacks provided by + * the driver, but the device is still usable. + */ +void iwl_mei_start_unregister(void); + +/** + * iwl_mei_unregister_complete() - complete the unregistration + * + * Must be called after iwl_mei_start_unregister. When this function returns, + * the device is owned by CSME. + */ +void iwl_mei_unregister_complete(void); + +/** + * iwl_mei_set_netdev() - sets the netdev for Tx / Rx. + * @netdev: the net_device + * + * The caller should set the netdev to a non-NULL value when the + * interface is added. Packets might be sent to the driver immediately + * afterwards. + * The caller should set the netdev to NULL when the interface is removed. + * This function will call synchronize_net() after setting the netdev to NULL. + * Only when this function returns, can the caller assume that iwlmei will + * no longer inject packets into the netdev's Tx path. + * + * Context: This function can sleep and assumes rtnl_lock is taken. + * The netdev must be set to NULL before iwl_mei_start_unregister() is called. + */ +void iwl_mei_set_netdev(struct net_device *netdev); + +/** + * iwl_mei_tx_copy_to_csme() - must be called for each packet sent by + * the wifi driver. + * @skb: the skb sent + * @ivlen: the size of the IV that needs to be skipped after the MAC and + * before the SNAP header. + * + * This function doesn't take any lock, it simply tries to catch DHCP + * packets sent by the wifi driver. If the packet is a DHCP packet, it + * will send it to CSME. This function must not be called for virtual + * interfaces that are not monitored by CSME, meaning it must be called + * only for packets transmitted by the netdevice that was registered + * with iwl_mei_set_netdev(). + */ +void iwl_mei_tx_copy_to_csme(struct sk_buff *skb, unsigned int ivlen); + +/** + * iwl_mei_host_associated() - must be called when iwlwifi associated. + * @conn_info: pointer to the connection info structure. + * @colloc_info: pointer to the collocated AP info. This is relevant only in + * case of UHB associated AP, otherwise set to NULL. + */ +void iwl_mei_host_associated(const struct iwl_mei_conn_info *conn_info, + const struct iwl_mei_colloc_info *colloc_info); + +/** + * iwl_mei_host_disassociated() - must be called when iwlwifi disassociated. + */ +void iwl_mei_host_disassociated(void); + +/** + * iwl_mei_device_down() - must be called when the device is down + */ +void iwl_mei_device_down(void); + +#else + +static inline bool iwl_mei_is_connected(void) +{ return false; } + +static inline struct iwl_mei_nvm *iwl_mei_get_nvm(void) +{ return NULL; } + +static inline int iwl_mei_get_ownership(void) +{ return 0; } + +static inline void iwl_mei_set_rfkill_state(bool hw_rfkill, bool sw_rfkill) +{} + +static inline void iwl_mei_set_nic_info(const u8 *mac_address, const u8 *nvm_address) +{} + +static inline void iwl_mei_set_country_code(u16 mcc) +{} + +static inline void iwl_mei_set_power_limit(__le16 *power_limit) +{} + +static inline int iwl_mei_register(void *priv, + const struct iwl_mei_ops *ops) +{ return 0; } + +static inline void iwl_mei_start_unregister(void) +{} + +static inline void iwl_mei_unregister_complete(void) +{} + +static inline void iwl_mei_set_netdev(struct net_device *netdev) +{} + +static inline void iwl_mei_tx_copy_to_csme(struct sk_buff *skb, + unsigned int ivlen) +{} + +static inline void iwl_mei_host_associated(const struct iwl_mei_conn_info *conn_info, + const struct iwl_mei_colloc_info *colloc_info) +{} + +static inline void iwl_mei_host_disassociated(void) +{} + +static inline void iwl_mei_device_down(void) +{} + +#endif /* CONFIG_IWLMEI */ + +#endif /* __iwl_mei_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/main.c b/drivers/net/wireless/intel/iwlwifi/mei/main.c new file mode 100644 index 000000000000..112cc362e8e7 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/main.c @@ -0,0 +1,1982 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#include <linux/etherdevice.h> +#include <linux/netdevice.h> +#include <linux/ieee80211.h> +#include <linux/rtnetlink.h> +#include <linux/module.h> +#include <linux/moduleparam.h> +#include <linux/mei_cl_bus.h> +#include <linux/rcupdate.h> +#include <linux/debugfs.h> +#include <linux/skbuff.h> +#include <linux/wait.h> +#include <linux/slab.h> +#include <linux/mm.h> + +#include <net/cfg80211.h> + +#include "internal.h" +#include "iwl-mei.h" +#include "trace.h" +#include "trace-data.h" +#include "sap.h" + +MODULE_DESCRIPTION("The Intel(R) wireless / CSME firmware interface"); +MODULE_LICENSE("GPL"); + +#define MEI_WLAN_UUID UUID_LE(0x13280904, 0x7792, 0x4fcb, \ + 0xa1, 0xaa, 0x5e, 0x70, 0xcb, 0xb1, 0xe8, 0x65) + +/* + * Since iwlwifi calls iwlmei without any context, hold a pointer to the + * mei_cl_device structure here. + * Define a mutex that will synchronize all the flows between iwlwifi and + * iwlmei. + * Note that iwlmei can't have several instances, so it ok to have static + * variables here. + */ +static struct mei_cl_device *iwl_mei_global_cldev; +static DEFINE_MUTEX(iwl_mei_mutex); +static unsigned long iwl_mei_status; + +enum iwl_mei_status_bits { + IWL_MEI_STATUS_SAP_CONNECTED, +}; + +bool iwl_mei_is_connected(void) +{ + return test_bit(IWL_MEI_STATUS_SAP_CONNECTED, &iwl_mei_status); +} +EXPORT_SYMBOL_GPL(iwl_mei_is_connected); + +#define SAP_VERSION 3 +#define SAP_CONTROL_BLOCK_ID 0x21504153 /* SAP! in ASCII */ + +struct iwl_sap_q_ctrl_blk { + __le32 wr_ptr; + __le32 rd_ptr; + __le32 size; +}; + +enum iwl_sap_q_idx { + SAP_QUEUE_IDX_NOTIF = 0, + SAP_QUEUE_IDX_DATA, + SAP_QUEUE_IDX_MAX, +}; + +struct iwl_sap_dir { + __le32 reserved; + struct iwl_sap_q_ctrl_blk q_ctrl_blk[SAP_QUEUE_IDX_MAX]; +}; + +enum iwl_sap_dir_idx { + SAP_DIRECTION_HOST_TO_ME = 0, + SAP_DIRECTION_ME_TO_HOST, + SAP_DIRECTION_MAX, +}; + +struct iwl_sap_shared_mem_ctrl_blk { + __le32 sap_id; + __le32 size; + struct iwl_sap_dir dir[SAP_DIRECTION_MAX]; +}; + +/* + * The shared area has the following layout: + * + * +-----------------------------------+ + * |struct iwl_sap_shared_mem_ctrl_blk | + * +-----------------------------------+ + * |Host -> ME data queue | + * +-----------------------------------+ + * |Host -> ME notif queue | + * +-----------------------------------+ + * |ME -> Host data queue | + * +-----------------------------------+ + * |ME -> host notif queue | + * +-----------------------------------+ + * |SAP control block id (SAP!) | + * +-----------------------------------+ + */ + +#define SAP_H2M_DATA_Q_SZ 48256 +#define SAP_M2H_DATA_Q_SZ 24128 +#define SAP_H2M_NOTIF_Q_SZ 2240 +#define SAP_M2H_NOTIF_Q_SZ 62720 + +#define _IWL_MEI_SAP_SHARED_MEM_SZ \ + (sizeof(struct iwl_sap_shared_mem_ctrl_blk) + \ + SAP_H2M_DATA_Q_SZ + SAP_H2M_NOTIF_Q_SZ + \ + SAP_M2H_DATA_Q_SZ + SAP_M2H_NOTIF_Q_SZ + 4) + +#define IWL_MEI_SAP_SHARED_MEM_SZ \ + (roundup(_IWL_MEI_SAP_SHARED_MEM_SZ, PAGE_SIZE)) + +struct iwl_mei_shared_mem_ptrs { + struct iwl_sap_shared_mem_ctrl_blk *ctrl; + void *q_head[SAP_DIRECTION_MAX][SAP_QUEUE_IDX_MAX]; +}; + +struct iwl_mei_filters { + struct rcu_head rcu_head; + struct iwl_sap_oob_filters filters; +}; + +/** + * struct iwl_mei - holds the private date for iwl_mei + * + * @get_nvm_wq: the wait queue for the get_nvm flow + * @send_csa_msg_wk: used to defer the transmission of the CHECK_SHARED_AREA + * message. Used so that we can send CHECK_SHARED_AREA from atomic + * contexts. + * @get_ownership_wq: the wait queue for the get_ownership_flow + * @shared_mem: the memory that is shared between CSME and the host + * @cldev: the pointer to the MEI client device + * @nvm: the data returned by the CSME for the NVM + * @filters: the filters sent by CSME + * @got_ownership: true if we own the device + * @amt_enabled: true if CSME has wireless enabled + * @csa_throttled: when true, we can't send CHECK_SHARED_AREA over the MEI + * bus, but rather need to wait until send_csa_msg_wk runs + * @csme_taking_ownership: true when CSME is taking ownership. Used to remember + * to send CSME_OWNERSHIP_CONFIRMED when the driver completes its down + * flow. + * @csa_throttle_end_wk: used when &csa_throttled is true + * @data_q_lock: protects the access to the data queues which are + * accessed without the mutex. + * @sap_seq_no: the sequence number for the SAP messages + * @seq_no: the sequence number for the SAP messages + * @dbgfs_dir: the debugfs dir entry + */ +struct iwl_mei { + wait_queue_head_t get_nvm_wq; + struct work_struct send_csa_msg_wk; + wait_queue_head_t get_ownership_wq; + struct iwl_mei_shared_mem_ptrs shared_mem; + struct mei_cl_device *cldev; + struct iwl_mei_nvm *nvm; + struct iwl_mei_filters __rcu *filters; + bool got_ownership; + bool amt_enabled; + bool csa_throttled; + bool csme_taking_ownership; + struct delayed_work csa_throttle_end_wk; + spinlock_t data_q_lock; + + atomic_t sap_seq_no; + atomic_t seq_no; + + struct dentry *dbgfs_dir; +}; + +/** + * iwl_mei_cache - cache for the parameters from iwlwifi + * @ops: Callbacks to iwlwifi. + * @netdev: The netdev that will be used to transmit / receive packets. + * @conn_info: The connection info message triggered by iwlwifi's association. + * @power_limit: pointer to an array of 10 elements (le16) represents the power + * restrictions per chain. + * @rf_kill: rf kill state. + * @mcc: MCC info + * @mac_address: interface MAC address. + * @nvm_address: NVM MAC address. + * @priv: A pointer to iwlwifi. + * + * This used to cache the configurations coming from iwlwifi's way. The data + * is cached here so that we can buffer the configuration even if we don't have + * a bind from the mei bus and hence, on iwl_mei structure. + */ +static struct { + const struct iwl_mei_ops *ops; + struct net_device __rcu *netdev; + const struct iwl_sap_notif_connection_info *conn_info; + const __le16 *power_limit; + u32 rf_kill; + u16 mcc; + u8 mac_address[6]; + u8 nvm_address[6]; + void *priv; +} iwl_mei_cache = { + .rf_kill = SAP_HW_RFKILL_DEASSERTED | SAP_SW_RFKILL_DEASSERTED +}; + +static void iwl_mei_free_shared_mem(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + + if (mei_cldev_dma_unmap(cldev)) + dev_err(&cldev->dev, "Coudln't unmap the shared mem properly\n"); + memset(&mei->shared_mem, 0, sizeof(mei->shared_mem)); +} + +#define HBM_DMA_BUF_ID_WLAN 1 + +static int iwl_mei_alloc_shared_mem(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_mei_shared_mem_ptrs *mem = &mei->shared_mem; + + mem->ctrl = mei_cldev_dma_map(cldev, HBM_DMA_BUF_ID_WLAN, + IWL_MEI_SAP_SHARED_MEM_SZ); + + if (IS_ERR(mem->ctrl)) { + int ret = PTR_ERR(mem->ctrl); + + dev_err(&cldev->dev, "Couldn't allocate the shared memory: %d\n", + ret); + mem->ctrl = NULL; + + return ret; + } + + memset(mem->ctrl, 0, IWL_MEI_SAP_SHARED_MEM_SZ); + + return 0; +} + +static void iwl_mei_init_shared_mem(struct iwl_mei *mei) +{ + struct iwl_mei_shared_mem_ptrs *mem = &mei->shared_mem; + struct iwl_sap_dir *h2m; + struct iwl_sap_dir *m2h; + int dir, queue; + u8 *q_head; + + mem->ctrl->sap_id = cpu_to_le32(SAP_CONTROL_BLOCK_ID); + + mem->ctrl->size = cpu_to_le32(sizeof(*mem->ctrl)); + + h2m = &mem->ctrl->dir[SAP_DIRECTION_HOST_TO_ME]; + m2h = &mem->ctrl->dir[SAP_DIRECTION_ME_TO_HOST]; + + h2m->q_ctrl_blk[SAP_QUEUE_IDX_DATA].size = + cpu_to_le32(SAP_H2M_DATA_Q_SZ); + h2m->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF].size = + cpu_to_le32(SAP_H2M_NOTIF_Q_SZ); + m2h->q_ctrl_blk[SAP_QUEUE_IDX_DATA].size = + cpu_to_le32(SAP_M2H_DATA_Q_SZ); + m2h->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF].size = + cpu_to_le32(SAP_M2H_NOTIF_Q_SZ); + + /* q_head points to the start of the first queue */ + q_head = (void *)(mem->ctrl + 1); + + /* Initialize the queue heads */ + for (dir = 0; dir < SAP_DIRECTION_MAX; dir++) { + for (queue = 0; queue < SAP_QUEUE_IDX_MAX; queue++) { + mem->q_head[dir][queue] = q_head; + q_head += + le32_to_cpu(mem->ctrl->dir[dir].q_ctrl_blk[queue].size); + } + } + + *(__le32 *)q_head = cpu_to_le32(SAP_CONTROL_BLOCK_ID); +} + +static ssize_t iwl_mei_write_cyclic_buf(struct mei_cl_device *cldev, + struct iwl_sap_q_ctrl_blk *notif_q, + u8 *q_head, + const struct iwl_sap_hdr *hdr) +{ + u32 rd = le32_to_cpu(READ_ONCE(notif_q->rd_ptr)); + u32 wr = le32_to_cpu(READ_ONCE(notif_q->wr_ptr)); + u32 q_sz = le32_to_cpu(notif_q->size); + size_t room_in_buf; + size_t tx_sz = sizeof(*hdr) + le16_to_cpu(hdr->len); + + if (rd > q_sz || wr > q_sz) { + dev_err(&cldev->dev, + "Pointers are past the end of the buffer\n"); + return -EINVAL; + } + + room_in_buf = wr >= rd ? q_sz - wr + rd : rd - wr; + + /* we don't have enough room for the data to write */ + if (room_in_buf < tx_sz) { + dev_err(&cldev->dev, + "Not enough room in the buffer\n"); + return -ENOSPC; + } + + if (wr + tx_sz <= q_sz) { + memcpy(q_head + wr, hdr, tx_sz); + } else { + memcpy(q_head + wr, hdr, q_sz - wr); + memcpy(q_head, (u8 *)hdr + q_sz - wr, tx_sz - (q_sz - wr)); + } + + WRITE_ONCE(notif_q->wr_ptr, cpu_to_le32((wr + tx_sz) % q_sz)); + return 0; +} + +static bool iwl_mei_host_to_me_data_pending(const struct iwl_mei *mei) +{ + struct iwl_sap_q_ctrl_blk *notif_q; + struct iwl_sap_dir *dir; + + dir = &mei->shared_mem.ctrl->dir[SAP_DIRECTION_HOST_TO_ME]; + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_DATA]; + + if (READ_ONCE(notif_q->wr_ptr) != READ_ONCE(notif_q->rd_ptr)) + return true; + + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; + return READ_ONCE(notif_q->wr_ptr) != READ_ONCE(notif_q->rd_ptr); +} + +static int iwl_mei_send_check_shared_area(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_sap_me_msg_start msg = { + .hdr.type = cpu_to_le32(SAP_ME_MSG_CHECK_SHARED_AREA), + .hdr.seq_num = cpu_to_le32(atomic_inc_return(&mei->seq_no)), + }; + int ret; + + lockdep_assert_held(&iwl_mei_mutex); + + if (mei->csa_throttled) + return 0; + + trace_iwlmei_me_msg(&msg.hdr, true); + ret = mei_cldev_send(cldev, (void *)&msg, sizeof(msg)); + if (ret != sizeof(msg)) { + dev_err(&cldev->dev, + "failed to send the SAP_ME_MSG_CHECK_SHARED_AREA message %d\n", + ret); + return ret; + } + + mei->csa_throttled = true; + + schedule_delayed_work(&mei->csa_throttle_end_wk, + msecs_to_jiffies(100)); + + return 0; +} + +static void iwl_mei_csa_throttle_end_wk(struct work_struct *wk) +{ + struct iwl_mei *mei = + container_of(wk, struct iwl_mei, csa_throttle_end_wk.work); + + mutex_lock(&iwl_mei_mutex); + + mei->csa_throttled = false; + + if (iwl_mei_host_to_me_data_pending(mei)) + iwl_mei_send_check_shared_area(mei->cldev); + + mutex_unlock(&iwl_mei_mutex); +} + +static int iwl_mei_send_sap_msg_payload(struct mei_cl_device *cldev, + struct iwl_sap_hdr *hdr) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_sap_q_ctrl_blk *notif_q; + struct iwl_sap_dir *dir; + void *q_head; + int ret; + + lockdep_assert_held(&iwl_mei_mutex); + + if (!mei->shared_mem.ctrl) { + dev_err(&cldev->dev, + "No shared memory, can't send any SAP message\n"); + return -EINVAL; + } + + if (!iwl_mei_is_connected()) { + dev_err(&cldev->dev, + "Can't send a SAP message if we're not connected\n"); + return -ENODEV; + } + + hdr->seq_num = cpu_to_le32(atomic_inc_return(&mei->sap_seq_no)); + dev_dbg(&cldev->dev, "Sending %d\n", hdr->type); + + dir = &mei->shared_mem.ctrl->dir[SAP_DIRECTION_HOST_TO_ME]; + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; + q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_NOTIF]; + ret = iwl_mei_write_cyclic_buf(q_head, notif_q, q_head, hdr); + + if (ret < 0) + return ret; + + trace_iwlmei_sap_cmd(hdr, true); + + return iwl_mei_send_check_shared_area(cldev); +} + +void iwl_mei_add_data_to_ring(struct sk_buff *skb, bool cb_tx) +{ + struct iwl_sap_q_ctrl_blk *notif_q; + struct iwl_sap_dir *dir; + struct iwl_mei *mei; + size_t room_in_buf; + size_t tx_sz; + size_t hdr_sz; + u32 q_sz; + u32 rd; + u32 wr; + void *q_head; + + if (!iwl_mei_global_cldev) + return; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + /* + * We access this path for Rx packets (the more common case) + * and from Tx path when we send DHCP packets, the latter is + * very unlikely. + * Take the lock already here to make sure we see that remove() + * might have cleared the IWL_MEI_STATUS_SAP_CONNECTED bit. + */ + spin_lock_bh(&mei->data_q_lock); + + if (!iwl_mei_is_connected()) { + spin_unlock_bh(&mei->data_q_lock); + return; + } + + /* + * We are in a RCU critical section and the remove from the CSME bus + * which would free this memory waits for the readers to complete (this + * is done in netdev_rx_handler_unregister). + */ + dir = &mei->shared_mem.ctrl->dir[SAP_DIRECTION_HOST_TO_ME]; + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_DATA]; + q_head = mei->shared_mem.q_head[SAP_DIRECTION_HOST_TO_ME][SAP_QUEUE_IDX_DATA]; + + rd = le32_to_cpu(READ_ONCE(notif_q->rd_ptr)); + wr = le32_to_cpu(READ_ONCE(notif_q->wr_ptr)); + q_sz = le32_to_cpu(notif_q->size); + hdr_sz = cb_tx ? sizeof(struct iwl_sap_cb_data) : + sizeof(struct iwl_sap_hdr); + tx_sz = skb->len + hdr_sz; + + if (rd > q_sz || wr > q_sz) { + dev_err(&mei->cldev->dev, + "can't write the data: pointers are past the end of the buffer\n"); + goto out; + } + + room_in_buf = wr >= rd ? q_sz - wr + rd : rd - wr; + + /* we don't have enough room for the data to write */ + if (room_in_buf < tx_sz) { + dev_err(&mei->cldev->dev, + "Not enough room in the buffer for this data\n"); + goto out; + } + + if (skb_headroom(skb) < hdr_sz) { + dev_err(&mei->cldev->dev, + "Not enough headroom in the skb to write the SAP header\n"); + goto out; + } + + if (cb_tx) { + struct iwl_sap_cb_data *cb_hdr = skb_push(skb, sizeof(*cb_hdr)); + + cb_hdr->hdr.type = cpu_to_le16(SAP_MSG_CB_DATA_PACKET); + cb_hdr->hdr.len = cpu_to_le16(skb->len - sizeof(cb_hdr->hdr)); + cb_hdr->hdr.seq_num = cpu_to_le32(atomic_inc_return(&mei->sap_seq_no)); + cb_hdr->to_me_filt_status = cpu_to_le32(BIT(CB_TX_DHCP_FILT_IDX)); + cb_hdr->data_len = cpu_to_le32(skb->len - sizeof(*cb_hdr)); + trace_iwlmei_sap_data(skb, IWL_SAP_TX_DHCP); + } else { + struct iwl_sap_hdr *hdr = skb_push(skb, sizeof(*hdr)); + + hdr->type = cpu_to_le16(SAP_MSG_DATA_PACKET); + hdr->len = cpu_to_le16(skb->len - sizeof(*hdr)); + hdr->seq_num = cpu_to_le32(atomic_inc_return(&mei->sap_seq_no)); + trace_iwlmei_sap_data(skb, IWL_SAP_TX_DATA_FROM_AIR); + } + + if (wr + tx_sz <= q_sz) { + skb_copy_bits(skb, 0, q_head + wr, tx_sz); + } else { + skb_copy_bits(skb, 0, q_head + wr, q_sz - wr); + skb_copy_bits(skb, q_sz - wr, q_head, tx_sz - (q_sz - wr)); + } + + WRITE_ONCE(notif_q->wr_ptr, cpu_to_le32((wr + tx_sz) % q_sz)); + +out: + spin_unlock_bh(&mei->data_q_lock); +} + +static int +iwl_mei_send_sap_msg(struct mei_cl_device *cldev, u16 type) +{ + struct iwl_sap_hdr msg = { + .type = cpu_to_le16(type), + }; + + return iwl_mei_send_sap_msg_payload(cldev, &msg); +} + +static void iwl_mei_send_csa_msg_wk(struct work_struct *wk) +{ + struct iwl_mei *mei = + container_of(wk, struct iwl_mei, send_csa_msg_wk); + + if (!iwl_mei_is_connected()) + return; + + mutex_lock(&iwl_mei_mutex); + + iwl_mei_send_check_shared_area(mei->cldev); + + mutex_unlock(&iwl_mei_mutex); +} + +/* Called in a RCU read critical section from netif_receive_skb */ +static rx_handler_result_t iwl_mei_rx_handler(struct sk_buff **pskb) +{ + struct sk_buff *skb = *pskb; + struct iwl_mei *mei = + rcu_dereference(skb->dev->rx_handler_data); + struct iwl_mei_filters *filters = rcu_dereference(mei->filters); + bool rx_for_csme = false; + rx_handler_result_t res; + + /* + * remove() unregisters this handler and synchronize_net, so this + * should never happen. + */ + if (!iwl_mei_is_connected()) { + dev_err(&mei->cldev->dev, + "Got an Rx packet, but we're not connected to SAP?\n"); + return RX_HANDLER_PASS; + } + + if (filters) + res = iwl_mei_rx_filter(skb, &filters->filters, &rx_for_csme); + else + res = RX_HANDLER_PASS; + + /* + * The data is already on the ring of the shared area, all we + * need to do is to tell the CSME firmware to check what we have + * there. + */ + if (rx_for_csme) + schedule_work(&mei->send_csa_msg_wk); + + if (res != RX_HANDLER_PASS) { + trace_iwlmei_sap_data(skb, IWL_SAP_RX_DATA_DROPPED_FROM_AIR); + dev_kfree_skb(skb); + } + + return res; +} + +static void +iwl_mei_handle_rx_start_ok(struct mei_cl_device *cldev, + const struct iwl_sap_me_msg_start_ok *rsp, + ssize_t len) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + + if (len != sizeof(*rsp)) { + dev_err(&cldev->dev, + "got invalid SAP_ME_MSG_START_OK from CSME firmware\n"); + dev_err(&cldev->dev, + "size is incorrect: %zd instead of %zu\n", + len, sizeof(*rsp)); + return; + } + + if (rsp->supported_version != SAP_VERSION) { + dev_err(&cldev->dev, + "didn't get the expected version: got %d\n", + rsp->supported_version); + return; + } + + mutex_lock(&iwl_mei_mutex); + set_bit(IWL_MEI_STATUS_SAP_CONNECTED, &iwl_mei_status); + /* wifi driver has registered already */ + if (iwl_mei_cache.ops) { + iwl_mei_send_sap_msg(mei->cldev, + SAP_MSG_NOTIF_WIFIDR_UP); + iwl_mei_cache.ops->sap_connected(iwl_mei_cache.priv); + } + + mutex_unlock(&iwl_mei_mutex); +} + +static void iwl_mei_handle_csme_filters(struct mei_cl_device *cldev, + const struct iwl_sap_csme_filters *filters) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + struct iwl_mei_filters *new_filters; + struct iwl_mei_filters *old_filters; + + old_filters = + rcu_dereference_protected(mei->filters, + lockdep_is_held(&iwl_mei_mutex)); + + new_filters = kzalloc(sizeof(*new_filters), GFP_KERNEL); + + /* Copy the OOB filters */ + new_filters->filters = filters->filters; + + rcu_assign_pointer(mei->filters, new_filters); + + if (old_filters) + kfree_rcu(old_filters, rcu_head); +} + +static void +iwl_mei_handle_conn_status(struct mei_cl_device *cldev, + const struct iwl_sap_notif_conn_status *status) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_mei_conn_info conn_info = { + .lp_state = le32_to_cpu(status->link_prot_state), + .ssid_len = le32_to_cpu(status->conn_info.ssid_len), + .channel = status->conn_info.channel, + .band = status->conn_info.band, + .auth_mode = le32_to_cpu(status->conn_info.auth_mode), + .pairwise_cipher = le32_to_cpu(status->conn_info.pairwise_cipher), + }; + + if (!iwl_mei_cache.ops || + conn_info.ssid_len > ARRAY_SIZE(conn_info.ssid)) + return; + + memcpy(conn_info.ssid, status->conn_info.ssid, conn_info.ssid_len); + ether_addr_copy(conn_info.bssid, status->conn_info.bssid); + + iwl_mei_cache.ops->me_conn_status(iwl_mei_cache.priv, &conn_info); + + /* + * Update the Rfkill state in case the host does not own the device: + * if we are in Link Protection, ask to not touch the device, else, + * unblock rfkill. + * If the host owns the device, inform the user space whether it can + * roam. + */ + if (mei->got_ownership) + iwl_mei_cache.ops->roaming_forbidden(iwl_mei_cache.priv, + status->link_prot_state); + else + iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, + status->link_prot_state); +} + +static void iwl_mei_set_init_conf(struct iwl_mei *mei) +{ + struct iwl_sap_notif_host_link_up link_msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_HOST_LINK_UP), + .hdr.len = cpu_to_le16(sizeof(link_msg) - sizeof(link_msg.hdr)), + }; + struct iwl_sap_notif_country_code mcc_msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_COUNTRY_CODE), + .hdr.len = cpu_to_le16(sizeof(mcc_msg) - sizeof(mcc_msg.hdr)), + .mcc = cpu_to_le16(iwl_mei_cache.mcc), + }; + struct iwl_sap_notif_sar_limits sar_msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_SAR_LIMITS), + .hdr.len = cpu_to_le16(sizeof(sar_msg) - sizeof(sar_msg.hdr)), + }; + struct iwl_sap_notif_host_nic_info nic_info_msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_NIC_INFO), + .hdr.len = cpu_to_le16(sizeof(nic_info_msg) - sizeof(nic_info_msg.hdr)), + }; + struct iwl_sap_msg_dw rfkill_msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_RADIO_STATE), + .hdr.len = cpu_to_le16(sizeof(rfkill_msg) - sizeof(rfkill_msg.hdr)), + .val = cpu_to_le32(iwl_mei_cache.rf_kill), + }; + + iwl_mei_send_sap_msg(mei->cldev, SAP_MSG_NOTIF_WHO_OWNS_NIC); + + if (iwl_mei_cache.conn_info) { + link_msg.conn_info = *iwl_mei_cache.conn_info; + iwl_mei_send_sap_msg_payload(mei->cldev, &link_msg.hdr); + } + + iwl_mei_send_sap_msg_payload(mei->cldev, &mcc_msg.hdr); + + if (iwl_mei_cache.power_limit) { + memcpy(sar_msg.sar_chain_info_table, iwl_mei_cache.power_limit, + sizeof(sar_msg.sar_chain_info_table)); + iwl_mei_send_sap_msg_payload(mei->cldev, &sar_msg.hdr); + } + + ether_addr_copy(nic_info_msg.mac_address, iwl_mei_cache.mac_address); + ether_addr_copy(nic_info_msg.nvm_address, iwl_mei_cache.nvm_address); + iwl_mei_send_sap_msg_payload(mei->cldev, &nic_info_msg.hdr); + + iwl_mei_send_sap_msg_payload(mei->cldev, &rfkill_msg.hdr); +} + +static void iwl_mei_handle_amt_state(struct mei_cl_device *cldev, + const struct iwl_sap_msg_dw *dw) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct net_device *netdev; + + /* + * First take rtnl and only then the mutex to avoid an ABBA + * with iwl_mei_set_netdev() + */ + rtnl_lock(); + mutex_lock(&iwl_mei_mutex); + + netdev = rcu_dereference_protected(iwl_mei_cache.netdev, + lockdep_is_held(&iwl_mei_mutex)); + + if (mei->amt_enabled == !!le32_to_cpu(dw->val)) + goto out; + + mei->amt_enabled = dw->val; + + if (mei->amt_enabled) { + if (netdev) + netdev_rx_handler_register(netdev, iwl_mei_rx_handler, mei); + + iwl_mei_set_init_conf(mei); + } else { + if (iwl_mei_cache.ops) + iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false); + if (netdev) + netdev_rx_handler_unregister(netdev); + } + +out: + mutex_unlock(&iwl_mei_mutex); + rtnl_unlock(); +} + +static void iwl_mei_handle_nic_owner(struct mei_cl_device *cldev, + const struct iwl_sap_msg_dw *dw) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + + mei->got_ownership = dw->val != cpu_to_le32(SAP_NIC_OWNER_ME); +} + +static void iwl_mei_handle_can_release_ownership(struct mei_cl_device *cldev, + const void *payload) +{ + /* We can get ownership and driver is registered, go ahead */ + if (iwl_mei_cache.ops) + iwl_mei_send_sap_msg(cldev, + SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP); +} + +static void iwl_mei_handle_csme_taking_ownership(struct mei_cl_device *cldev, + const void *payload) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + + dev_info(&cldev->dev, "CSME takes ownership\n"); + + mei->got_ownership = false; + + /* + * Remember to send CSME_OWNERSHIP_CONFIRMED when the wifi driver + * is finished taking the device down. + */ + mei->csme_taking_ownership = true; + + if (iwl_mei_cache.ops) + iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, true); +} + +static void iwl_mei_handle_nvm(struct mei_cl_device *cldev, + const struct iwl_sap_nvm *sap_nvm) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + const struct iwl_mei_nvm *mei_nvm = (const void *)sap_nvm; + int i; + + kfree(mei->nvm); + mei->nvm = kzalloc(sizeof(*mei_nvm), GFP_KERNEL); + if (!mei->nvm) + return; + + ether_addr_copy(mei->nvm->hw_addr, sap_nvm->hw_addr); + mei->nvm->n_hw_addrs = sap_nvm->n_hw_addrs; + mei->nvm->radio_cfg = le32_to_cpu(sap_nvm->radio_cfg); + mei->nvm->caps = le32_to_cpu(sap_nvm->caps); + mei->nvm->nvm_version = le32_to_cpu(sap_nvm->nvm_version); + + for (i = 0; i < ARRAY_SIZE(mei->nvm->channels); i++) + mei->nvm->channels[i] = le32_to_cpu(sap_nvm->channels[i]); + + wake_up_all(&mei->get_nvm_wq); +} + +static void iwl_mei_handle_rx_host_own_req(struct mei_cl_device *cldev, + const struct iwl_sap_msg_dw *dw) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + + /* + * This means that we can't use the wifi device right now, CSME is not + * ready to let us use it. + */ + if (!dw->val) { + dev_info(&cldev->dev, "Ownership req denied\n"); + return; + } + + mei->got_ownership = true; + wake_up_all(&mei->get_ownership_wq); + + iwl_mei_send_sap_msg(cldev, + SAP_MSG_NOTIF_HOST_OWNERSHIP_CONFIRMED); + + /* We can now start the connection, unblock rfkill */ + if (iwl_mei_cache.ops) + iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false); +} + +static void iwl_mei_handle_ping(struct mei_cl_device *cldev, + const struct iwl_sap_hdr *hdr) +{ + iwl_mei_send_sap_msg(cldev, SAP_MSG_NOTIF_PONG); +} + +static void iwl_mei_handle_sap_msg(struct mei_cl_device *cldev, + const struct iwl_sap_hdr *hdr) +{ + u16 len = le16_to_cpu(hdr->len) + sizeof(*hdr); + u16 type = le16_to_cpu(hdr->type); + + dev_dbg(&cldev->dev, + "Got a new SAP message: type %d, len %d, seq %d\n", + le16_to_cpu(hdr->type), len, + le32_to_cpu(hdr->seq_num)); + +#define SAP_MSG_HANDLER(_cmd, _handler, _sz) \ + case SAP_MSG_NOTIF_ ## _cmd: \ + if (len < _sz) { \ + dev_err(&cldev->dev, \ + "Bad size for %d: %u < %u\n", \ + le16_to_cpu(hdr->type), \ + (unsigned int)len, \ + (unsigned int)_sz); \ + break; \ + } \ + mutex_lock(&iwl_mei_mutex); \ + _handler(cldev, (const void *)hdr); \ + mutex_unlock(&iwl_mei_mutex); \ + break + +#define SAP_MSG_HANDLER_NO_LOCK(_cmd, _handler, _sz) \ + case SAP_MSG_NOTIF_ ## _cmd: \ + if (len < _sz) { \ + dev_err(&cldev->dev, \ + "Bad size for %d: %u < %u\n", \ + le16_to_cpu(hdr->type), \ + (unsigned int)len, \ + (unsigned int)_sz); \ + break; \ + } \ + _handler(cldev, (const void *)hdr); \ + break + +#define SAP_MSG_HANDLER_NO_HANDLER(_cmd, _sz) \ + case SAP_MSG_NOTIF_ ## _cmd: \ + if (len < _sz) { \ + dev_err(&cldev->dev, \ + "Bad size for %d: %u < %u\n", \ + le16_to_cpu(hdr->type), \ + (unsigned int)len, \ + (unsigned int)_sz); \ + break; \ + } \ + break + + switch (type) { + SAP_MSG_HANDLER(PING, iwl_mei_handle_ping, 0); + SAP_MSG_HANDLER(CSME_FILTERS, + iwl_mei_handle_csme_filters, + sizeof(struct iwl_sap_csme_filters)); + SAP_MSG_HANDLER(CSME_CONN_STATUS, + iwl_mei_handle_conn_status, + sizeof(struct iwl_sap_notif_conn_status)); + SAP_MSG_HANDLER_NO_LOCK(AMT_STATE, + iwl_mei_handle_amt_state, + sizeof(struct iwl_sap_msg_dw)); + SAP_MSG_HANDLER_NO_HANDLER(PONG, 0); + SAP_MSG_HANDLER(NVM, iwl_mei_handle_nvm, + sizeof(struct iwl_sap_nvm)); + SAP_MSG_HANDLER(CSME_REPLY_TO_HOST_OWNERSHIP_REQ, + iwl_mei_handle_rx_host_own_req, + sizeof(struct iwl_sap_msg_dw)); + SAP_MSG_HANDLER(NIC_OWNER, iwl_mei_handle_nic_owner, + sizeof(struct iwl_sap_msg_dw)); + SAP_MSG_HANDLER(CSME_CAN_RELEASE_OWNERSHIP, + iwl_mei_handle_can_release_ownership, 0); + SAP_MSG_HANDLER(CSME_TAKING_OWNERSHIP, + iwl_mei_handle_csme_taking_ownership, 0); + default: + /* + * This is not really an error, there are message that we decided + * to ignore, yet, it is useful to be able to leave a note if debug + * is enabled. + */ + dev_dbg(&cldev->dev, "Unsupported message: type %d, len %d\n", + le16_to_cpu(hdr->type), len); + } + +#undef SAP_MSG_HANDLER +#undef SAP_MSG_HANDLER_NO_LOCK +} + +static void iwl_mei_read_from_q(const u8 *q_head, u32 q_sz, + u32 *_rd, u32 wr, + void *_buf, u32 len) +{ + u8 *buf = _buf; + u32 rd = *_rd; + + if (rd + len <= q_sz) { + memcpy(buf, q_head + rd, len); + rd += len; + } else { + memcpy(buf, q_head + rd, q_sz - rd); + memcpy(buf + q_sz - rd, q_head, len - (q_sz - rd)); + rd = len - (q_sz - rd); + } + + *_rd = rd; +} + +#define QOS_HDR_IV_SNAP_LEN (sizeof(struct ieee80211_qos_hdr) + \ + IEEE80211_TKIP_IV_LEN + \ + sizeof(rfc1042_header) + ETH_TLEN) + +static void iwl_mei_handle_sap_data(struct mei_cl_device *cldev, + const u8 *q_head, u32 q_sz, + u32 rd, u32 wr, ssize_t valid_rx_sz, + struct sk_buff_head *tx_skbs) +{ + struct iwl_sap_hdr hdr; + struct net_device *netdev = + rcu_dereference_protected(iwl_mei_cache.netdev, + lockdep_is_held(&iwl_mei_mutex)); + + if (!netdev) + return; + + while (valid_rx_sz >= sizeof(hdr)) { + struct ethhdr *ethhdr; + unsigned char *data; + struct sk_buff *skb; + u16 len; + + iwl_mei_read_from_q(q_head, q_sz, &rd, wr, &hdr, sizeof(hdr)); + valid_rx_sz -= sizeof(hdr); + len = le16_to_cpu(hdr.len); + + if (valid_rx_sz < len) { + dev_err(&cldev->dev, + "Data queue is corrupted: valid data len %zd, len %d\n", + valid_rx_sz, len); + break; + } + + if (len < sizeof(*ethhdr)) { + dev_err(&cldev->dev, + "Data len is smaller than an ethernet header? len = %d\n", + len); + } + + valid_rx_sz -= len; + + if (le16_to_cpu(hdr.type) != SAP_MSG_DATA_PACKET) { + dev_err(&cldev->dev, "Unsupported Rx data: type %d, len %d\n", + le16_to_cpu(hdr.type), len); + continue; + } + + /* We need enough room for the WiFi header + SNAP + IV */ + skb = netdev_alloc_skb(netdev, len + QOS_HDR_IV_SNAP_LEN); + + skb_reserve(skb, QOS_HDR_IV_SNAP_LEN); + ethhdr = skb_push(skb, sizeof(*ethhdr)); + + iwl_mei_read_from_q(q_head, q_sz, &rd, wr, + ethhdr, sizeof(*ethhdr)); + len -= sizeof(*ethhdr); + + skb_reset_mac_header(skb); + skb_reset_network_header(skb); + skb->protocol = ethhdr->h_proto; + + data = skb_put(skb, len); + iwl_mei_read_from_q(q_head, q_sz, &rd, wr, data, len); + + /* + * Enqueue the skb here so that it can be sent later when we + * do not hold the mutex. TX'ing a packet with a mutex held is + * possible, but it wouldn't be nice to forbid the TX path to + * call any of iwlmei's functions, since every API from iwlmei + * needs the mutex. + */ + __skb_queue_tail(tx_skbs, skb); + } +} + +static void iwl_mei_handle_sap_rx_cmd(struct mei_cl_device *cldev, + const u8 *q_head, u32 q_sz, + u32 rd, u32 wr, ssize_t valid_rx_sz) +{ + struct page *p = alloc_page(GFP_KERNEL); + struct iwl_sap_hdr *hdr; + + if (!p) + return; + + hdr = page_address(p); + + while (valid_rx_sz >= sizeof(*hdr)) { + u16 len; + + iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr, sizeof(*hdr)); + valid_rx_sz -= sizeof(*hdr); + len = le16_to_cpu(hdr->len); + + if (valid_rx_sz < len) + break; + + iwl_mei_read_from_q(q_head, q_sz, &rd, wr, hdr + 1, len); + + trace_iwlmei_sap_cmd(hdr, false); + iwl_mei_handle_sap_msg(cldev, hdr); + valid_rx_sz -= len; + } + + /* valid_rx_sz must be 0 now... */ + if (valid_rx_sz) + dev_err(&cldev->dev, + "More data in the buffer although we read it all\n"); + + __free_page(p); +} + +static void iwl_mei_handle_sap_rx(struct mei_cl_device *cldev, + struct iwl_sap_q_ctrl_blk *notif_q, + const u8 *q_head, + struct sk_buff_head *skbs) +{ + u32 rd = le32_to_cpu(READ_ONCE(notif_q->rd_ptr)); + u32 wr = le32_to_cpu(READ_ONCE(notif_q->wr_ptr)); + u32 q_sz = le32_to_cpu(notif_q->size); + ssize_t valid_rx_sz; + + if (rd > q_sz || wr > q_sz) { + dev_err(&cldev->dev, + "Pointers are past the buffer limit\n"); + return; + } + + if (rd == wr) + return; + + valid_rx_sz = wr > rd ? wr - rd : q_sz - rd + wr; + + if (skbs) + iwl_mei_handle_sap_data(cldev, q_head, q_sz, rd, wr, + valid_rx_sz, skbs); + else + iwl_mei_handle_sap_rx_cmd(cldev, q_head, q_sz, rd, wr, + valid_rx_sz); + + /* Increment the read pointer to point to the write pointer */ + WRITE_ONCE(notif_q->rd_ptr, cpu_to_le32(wr)); +} + +static void iwl_mei_handle_check_shared_area(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_sap_q_ctrl_blk *notif_q; + struct sk_buff_head tx_skbs; + struct iwl_sap_dir *dir; + void *q_head; + + if (!mei->shared_mem.ctrl) + return; + + dir = &mei->shared_mem.ctrl->dir[SAP_DIRECTION_ME_TO_HOST]; + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_NOTIF]; + q_head = mei->shared_mem.q_head[SAP_DIRECTION_ME_TO_HOST][SAP_QUEUE_IDX_NOTIF]; + + /* + * Do not hold the mutex here, but rather each and every message + * handler takes it. + * This allows message handlers to take it at a certain time. + */ + iwl_mei_handle_sap_rx(cldev, notif_q, q_head, NULL); + + mutex_lock(&iwl_mei_mutex); + dir = &mei->shared_mem.ctrl->dir[SAP_DIRECTION_ME_TO_HOST]; + notif_q = &dir->q_ctrl_blk[SAP_QUEUE_IDX_DATA]; + q_head = mei->shared_mem.q_head[SAP_DIRECTION_ME_TO_HOST][SAP_QUEUE_IDX_DATA]; + + __skb_queue_head_init(&tx_skbs); + + iwl_mei_handle_sap_rx(cldev, notif_q, q_head, &tx_skbs); + + if (skb_queue_empty(&tx_skbs)) { + mutex_unlock(&iwl_mei_mutex); + return; + } + + /* + * Take the RCU read lock before we unlock the mutex to make sure that + * even if the netdev is replaced by another non-NULL netdev right after + * we unlock the mutex, the old netdev will still be valid when we + * transmit the frames. We can't allow to replace the netdev here because + * the skbs hold a pointer to the netdev. + */ + rcu_read_lock(); + + mutex_unlock(&iwl_mei_mutex); + + if (!rcu_access_pointer(iwl_mei_cache.netdev)) { + dev_err(&cldev->dev, "Can't Tx without a netdev\n"); + skb_queue_purge(&tx_skbs); + goto out; + } + + while (!skb_queue_empty(&tx_skbs)) { + struct sk_buff *skb = __skb_dequeue(&tx_skbs); + + trace_iwlmei_sap_data(skb, IWL_SAP_RX_DATA_TO_AIR); + dev_queue_xmit(skb); + } + +out: + rcu_read_unlock(); +} + +static void iwl_mei_rx(struct mei_cl_device *cldev) +{ + struct iwl_sap_me_msg_hdr *hdr; + u8 msg[100]; + ssize_t ret; + + ret = mei_cldev_recv(cldev, (u8 *)&msg, sizeof(msg)); + if (ret < 0) { + dev_err(&cldev->dev, "failed to receive data: %zd\n", ret); + return; + } + + if (ret == 0) { + dev_err(&cldev->dev, "got an empty response\n"); + return; + } + + hdr = (void *)msg; + trace_iwlmei_me_msg(hdr, false); + + switch (le32_to_cpu(hdr->type)) { + case SAP_ME_MSG_START_OK: + BUILD_BUG_ON(sizeof(struct iwl_sap_me_msg_start_ok) > + sizeof(msg)); + + iwl_mei_handle_rx_start_ok(cldev, (void *)msg, ret); + break; + case SAP_ME_MSG_CHECK_SHARED_AREA: + iwl_mei_handle_check_shared_area(cldev); + break; + default: + dev_err(&cldev->dev, "got a RX notification: %d\n", + le32_to_cpu(hdr->type)); + break; + } +} + +static int iwl_mei_send_start(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + struct iwl_sap_me_msg_start msg = { + .hdr.type = cpu_to_le32(SAP_ME_MSG_START), + .hdr.seq_num = cpu_to_le32(atomic_inc_return(&mei->seq_no)), + .hdr.len = cpu_to_le32(sizeof(msg)), + .supported_versions[0] = SAP_VERSION, + .init_data_seq_num = cpu_to_le16(0x100), + .init_notif_seq_num = cpu_to_le16(0x800), + }; + int ret; + + trace_iwlmei_me_msg(&msg.hdr, true); + ret = mei_cldev_send(cldev, (void *)&msg, sizeof(msg)); + if (ret != sizeof(msg)) { + dev_err(&cldev->dev, + "failed to send the SAP_ME_MSG_START message %d\n", + ret); + return ret; + } + + return 0; +} + +static int iwl_mei_enable(struct mei_cl_device *cldev) +{ + int ret; + + ret = mei_cldev_enable(cldev); + if (ret < 0) { + dev_err(&cldev->dev, "failed to enable the device: %d\n", ret); + return ret; + } + + ret = mei_cldev_register_rx_cb(cldev, iwl_mei_rx); + if (ret) { + dev_err(&cldev->dev, + "failed to register to the rx cb: %d\n", ret); + mei_cldev_disable(cldev); + return ret; + } + + return 0; +} + +struct iwl_mei_nvm *iwl_mei_get_nvm(void) +{ + struct iwl_mei_nvm *nvm = NULL; + struct iwl_mei *mei; + int ret; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + ret = iwl_mei_send_sap_msg(iwl_mei_global_cldev, + SAP_MSG_NOTIF_GET_NVM); + if (ret) + goto out; + + mutex_unlock(&iwl_mei_mutex); + + ret = wait_event_timeout(mei->get_nvm_wq, mei->nvm, 2 * HZ); + if (!ret) + return NULL; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + if (mei->nvm) + nvm = kmemdup(mei->nvm, sizeof(*mei->nvm), GFP_KERNEL); + +out: + mutex_unlock(&iwl_mei_mutex); + return nvm; +} +EXPORT_SYMBOL_GPL(iwl_mei_get_nvm); + +int iwl_mei_get_ownership(void) +{ + struct iwl_mei *mei; + int ret; + + mutex_lock(&iwl_mei_mutex); + + /* In case we didn't have a bind */ + if (!iwl_mei_is_connected()) { + ret = 0; + goto out; + } + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) { + ret = -ENODEV; + goto out; + } + + if (!mei->amt_enabled) { + ret = 0; + goto out; + } + + if (mei->got_ownership) { + ret = 0; + goto out; + } + + ret = iwl_mei_send_sap_msg(mei->cldev, + SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP); + if (ret) + goto out; + + mutex_unlock(&iwl_mei_mutex); + + ret = wait_event_timeout(mei->get_ownership_wq, + mei->got_ownership, HZ / 2); + if (!ret) + return -ETIMEDOUT; + + mutex_lock(&iwl_mei_mutex); + + /* In case we didn't have a bind */ + if (!iwl_mei_is_connected()) { + ret = 0; + goto out; + } + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) { + ret = -ENODEV; + goto out; + } + + ret = !mei->got_ownership; + +out: + mutex_unlock(&iwl_mei_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(iwl_mei_get_ownership); + +void iwl_mei_host_associated(const struct iwl_mei_conn_info *conn_info, + const struct iwl_mei_colloc_info *colloc_info) +{ + struct iwl_sap_notif_host_link_up msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_HOST_LINK_UP), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + .conn_info = { + .ssid_len = cpu_to_le32(conn_info->ssid_len), + .channel = conn_info->channel, + .band = conn_info->band, + .pairwise_cipher = cpu_to_le32(conn_info->pairwise_cipher), + .auth_mode = cpu_to_le32(conn_info->auth_mode), + }, + }; + struct iwl_mei *mei; + + if (conn_info->ssid_len > ARRAY_SIZE(msg.conn_info.ssid)) + return; + + memcpy(msg.conn_info.ssid, conn_info->ssid, conn_info->ssid_len); + memcpy(msg.conn_info.bssid, conn_info->bssid, ETH_ALEN); + + if (colloc_info) { + msg.colloc_channel = colloc_info->channel; + msg.colloc_band = colloc_info->channel <= 14 ? 0 : 1; + memcpy(msg.colloc_bssid, colloc_info->bssid, ETH_ALEN); + } + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + if (!mei->amt_enabled) + goto out; + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + kfree(iwl_mei_cache.conn_info); + iwl_mei_cache.conn_info = + kmemdup(&msg.conn_info, sizeof(msg.conn_info), GFP_KERNEL); + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_host_associated); + +void iwl_mei_host_disassociated(void) +{ + struct iwl_mei *mei; + struct iwl_sap_notif_host_link_down msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_HOST_LINK_DOWN), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + .type = HOST_LINK_DOWN_TYPE_LONG, + }; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + kfree(iwl_mei_cache.conn_info); + iwl_mei_cache.conn_info = NULL; + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_host_disassociated); + +void iwl_mei_set_rfkill_state(bool hw_rfkill, bool sw_rfkill) +{ + struct iwl_mei *mei; + u32 rfkill_state = 0; + struct iwl_sap_msg_dw msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_RADIO_STATE), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + }; + + if (!sw_rfkill) + rfkill_state |= SAP_SW_RFKILL_DEASSERTED; + + if (!hw_rfkill) + rfkill_state |= SAP_HW_RFKILL_DEASSERTED; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + msg.val = cpu_to_le32(rfkill_state); + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + iwl_mei_cache.rf_kill = rfkill_state; + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_set_rfkill_state); + +void iwl_mei_set_nic_info(const u8 *mac_address, const u8 *nvm_address) +{ + struct iwl_mei *mei; + struct iwl_sap_notif_host_nic_info msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_NIC_INFO), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + }; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + ether_addr_copy(msg.mac_address, mac_address); + ether_addr_copy(msg.nvm_address, nvm_address); + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + ether_addr_copy(iwl_mei_cache.mac_address, mac_address); + ether_addr_copy(iwl_mei_cache.nvm_address, nvm_address); + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_set_nic_info); + +void iwl_mei_set_country_code(u16 mcc) +{ + struct iwl_mei *mei; + struct iwl_sap_notif_country_code msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_COUNTRY_CODE), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + .mcc = cpu_to_le16(mcc), + }; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + iwl_mei_cache.mcc = mcc; + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_set_country_code); + +void iwl_mei_set_power_limit(const __le16 *power_limit) +{ + struct iwl_mei *mei; + struct iwl_sap_notif_sar_limits msg = { + .hdr.type = cpu_to_le16(SAP_MSG_NOTIF_SAR_LIMITS), + .hdr.len = cpu_to_le16(sizeof(msg) - sizeof(msg.hdr)), + }; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + memcpy(msg.sar_chain_info_table, power_limit, sizeof(msg.sar_chain_info_table)); + + iwl_mei_send_sap_msg_payload(mei->cldev, &msg.hdr); + +out: + kfree(iwl_mei_cache.power_limit); + iwl_mei_cache.power_limit = kmemdup(power_limit, + sizeof(msg.sar_chain_info_table), GFP_KERNEL); + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_set_power_limit); + +void iwl_mei_set_netdev(struct net_device *netdev) +{ + struct iwl_mei *mei; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) { + rcu_assign_pointer(iwl_mei_cache.netdev, netdev); + goto out; + } + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + if (!netdev) { + struct net_device *dev = + rcu_dereference_protected(iwl_mei_cache.netdev, + lockdep_is_held(&iwl_mei_mutex)); + + if (!dev) + goto out; + + netdev_rx_handler_unregister(dev); + } + + rcu_assign_pointer(iwl_mei_cache.netdev, netdev); + + if (netdev && mei->amt_enabled) + netdev_rx_handler_register(netdev, iwl_mei_rx_handler, mei); + +out: + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_set_netdev); + +void iwl_mei_device_down(void) +{ + struct iwl_mei *mei; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_is_connected()) + goto out; + + mei = mei_cldev_get_drvdata(iwl_mei_global_cldev); + + if (!mei) + goto out; + + if (!mei->csme_taking_ownership) + goto out; + + iwl_mei_send_sap_msg(mei->cldev, + SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED); + mei->csme_taking_ownership = false; +out: + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_device_down); + +int iwl_mei_register(void *priv, const struct iwl_mei_ops *ops) +{ + int ret; + + /* + * We must have a non-NULL priv pointer to not crash when there are + * multiple WiFi devices. + */ + if (!priv) + return -EINVAL; + + mutex_lock(&iwl_mei_mutex); + + /* do not allow registration if someone else already registered */ + if (iwl_mei_cache.priv || iwl_mei_cache.ops) { + ret = -EBUSY; + goto out; + } + + iwl_mei_cache.priv = priv; + iwl_mei_cache.ops = ops; + + if (iwl_mei_global_cldev) { + struct iwl_mei *mei = + mei_cldev_get_drvdata(iwl_mei_global_cldev); + + /* we have already a SAP connection */ + if (iwl_mei_is_connected()) + iwl_mei_send_sap_msg(mei->cldev, + SAP_MSG_NOTIF_WIFIDR_UP); + } + ret = 0; + +out: + mutex_unlock(&iwl_mei_mutex); + return ret; +} +EXPORT_SYMBOL_GPL(iwl_mei_register); + +void iwl_mei_start_unregister(void) +{ + mutex_lock(&iwl_mei_mutex); + + /* At this point, the wifi driver should have removed the netdev */ + if (rcu_access_pointer(iwl_mei_cache.netdev)) + pr_err("Still had a netdev pointer set upon unregister\n"); + + kfree(iwl_mei_cache.conn_info); + iwl_mei_cache.conn_info = NULL; + kfree(iwl_mei_cache.power_limit); + iwl_mei_cache.power_limit = NULL; + iwl_mei_cache.ops = NULL; + /* leave iwl_mei_cache.priv non-NULL to prevent any new registration */ + + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_start_unregister); + +void iwl_mei_unregister_complete(void) +{ + mutex_lock(&iwl_mei_mutex); + + iwl_mei_cache.priv = NULL; + + if (iwl_mei_global_cldev) { + struct iwl_mei *mei = + mei_cldev_get_drvdata(iwl_mei_global_cldev); + + iwl_mei_send_sap_msg(mei->cldev, SAP_MSG_NOTIF_WIFIDR_DOWN); + } + + mutex_unlock(&iwl_mei_mutex); +} +EXPORT_SYMBOL_GPL(iwl_mei_unregister_complete); + +#if IS_ENABLED(CONFIG_DEBUG_FS) + +static ssize_t +iwl_mei_dbgfs_send_start_message_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + int ret; + + mutex_lock(&iwl_mei_mutex); + + if (!iwl_mei_global_cldev) { + ret = -ENODEV; + goto out; + } + + ret = iwl_mei_send_start(iwl_mei_global_cldev); + +out: + mutex_unlock(&iwl_mei_mutex); + return ret ?: count; +} + +static const struct file_operations iwl_mei_dbgfs_send_start_message_ops = { + .write = iwl_mei_dbgfs_send_start_message_write, + .open = simple_open, + .llseek = default_llseek, +}; + +static ssize_t iwl_mei_dbgfs_req_ownership_write(struct file *file, + const char __user *user_buf, + size_t count, loff_t *ppos) +{ + iwl_mei_get_ownership(); + + return count; +} + +static const struct file_operations iwl_mei_dbgfs_req_ownership_ops = { + .write = iwl_mei_dbgfs_req_ownership_write, + .open = simple_open, + .llseek = default_llseek, +}; + +static void iwl_mei_dbgfs_register(struct iwl_mei *mei) +{ + mei->dbgfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL); + + if (!mei->dbgfs_dir) + return; + + debugfs_create_ulong("status", S_IRUSR, + mei->dbgfs_dir, &iwl_mei_status); + debugfs_create_file("send_start_message", S_IWUSR, mei->dbgfs_dir, + mei, &iwl_mei_dbgfs_send_start_message_ops); + debugfs_create_file("req_ownserhip", S_IWUSR, mei->dbgfs_dir, + mei, &iwl_mei_dbgfs_req_ownership_ops); +} + +static void iwl_mei_dbgfs_unregister(struct iwl_mei *mei) +{ + debugfs_remove_recursive(mei->dbgfs_dir); + mei->dbgfs_dir = NULL; +} + +#else + +static void iwl_mei_dbgfs_register(struct iwl_mei *mei) {} +static void iwl_mei_dbgfs_unregister(struct iwl_mei *mei) {} + +#endif /* CONFIG_DEBUG_FS */ + +/** + * iwl_mei_probe - the probe function called by the mei bus enumeration + * + * This allocates the data needed by iwlmei and sets a pointer to this data + * into the mei_cl_device's drvdata. + * It starts the SAP protocol by sending the SAP_ME_MSG_START without + * waiting for the answer. The answer will be caught later by the Rx callback. + */ +static int iwl_mei_probe(struct mei_cl_device *cldev, + const struct mei_cl_device_id *id) +{ + struct iwl_mei *mei; + int ret; + + mei = devm_kzalloc(&cldev->dev, sizeof(*mei), GFP_KERNEL); + if (!mei) + return -ENOMEM; + + init_waitqueue_head(&mei->get_nvm_wq); + INIT_WORK(&mei->send_csa_msg_wk, iwl_mei_send_csa_msg_wk); + INIT_DELAYED_WORK(&mei->csa_throttle_end_wk, + iwl_mei_csa_throttle_end_wk); + init_waitqueue_head(&mei->get_ownership_wq); + spin_lock_init(&mei->data_q_lock); + + mei_cldev_set_drvdata(cldev, mei); + mei->cldev = cldev; + + ret = iwl_mei_alloc_shared_mem(cldev); + if (ret) + goto free; + + iwl_mei_init_shared_mem(mei); + + ret = iwl_mei_enable(cldev); + if (ret) + goto free_shared_mem; + + iwl_mei_dbgfs_register(mei); + + /* + * We now have a Rx function in place, start the SAP procotol + * we expect to get the SAP_ME_MSG_START_OK response later on. + */ + mutex_lock(&iwl_mei_mutex); + ret = iwl_mei_send_start(cldev); + mutex_unlock(&iwl_mei_mutex); + if (ret) + goto debugfs_unregister; + + /* must be last */ + iwl_mei_global_cldev = cldev; + + return 0; + +debugfs_unregister: + iwl_mei_dbgfs_unregister(mei); + mei_cldev_disable(cldev); +free_shared_mem: + iwl_mei_free_shared_mem(cldev); +free: + mei_cldev_set_drvdata(cldev, NULL); + devm_kfree(&cldev->dev, mei); + + return ret; +} + +#define SEND_SAP_MAX_WAIT_ITERATION 10 + +static void iwl_mei_remove(struct mei_cl_device *cldev) +{ + struct iwl_mei *mei = mei_cldev_get_drvdata(cldev); + int i; + + /* + * We are being removed while the bus is active, it means we are + * going to suspend/ shutdown, so the NIC will disappear. + */ + if (mei_cldev_enabled(cldev) && iwl_mei_cache.ops) + iwl_mei_cache.ops->nic_stolen(iwl_mei_cache.priv); + + if (rcu_access_pointer(iwl_mei_cache.netdev)) { + struct net_device *dev; + + /* + * First take rtnl and only then the mutex to avoid an ABBA + * with iwl_mei_set_netdev() + */ + rtnl_lock(); + mutex_lock(&iwl_mei_mutex); + + /* + * If we are suspending and the wifi driver hasn't removed it's netdev + * yet, do it now. In any case, don't change the cache.netdev pointer. + */ + dev = rcu_dereference_protected(iwl_mei_cache.netdev, + lockdep_is_held(&iwl_mei_mutex)); + + netdev_rx_handler_unregister(dev); + mutex_unlock(&iwl_mei_mutex); + rtnl_unlock(); + } + + mutex_lock(&iwl_mei_mutex); + + /* + * Tell CSME that we are going down so that it won't access the + * memory anymore, make sure this message goes through immediately. + */ + mei->csa_throttled = false; + iwl_mei_send_sap_msg(mei->cldev, + SAP_MSG_NOTIF_HOST_GOES_DOWN); + + for (i = 0; i < SEND_SAP_MAX_WAIT_ITERATION; i++) { + if (!iwl_mei_host_to_me_data_pending(mei)) + break; + + msleep(5); + } + + /* + * If we couldn't make sure that CSME saw the HOST_GOES_DOWN message, + * it means that it will probably keep reading memory that we are going + * to unmap and free, expect IOMMU error messages. + */ + if (i == SEND_SAP_MAX_WAIT_ITERATION) + dev_err(&mei->cldev->dev, + "Couldn't get ACK from CSME on HOST_GOES_DOWN message\n"); + + mutex_unlock(&iwl_mei_mutex); + + /* + * This looks strange, but this lock is taken here to make sure that + * iwl_mei_add_data_to_ring called from the Tx path sees that we + * clear the IWL_MEI_STATUS_SAP_CONNECTED bit. + * Rx isn't a problem because the rx_handler can't be called after + * having been unregistered. + */ + spin_lock_bh(&mei->data_q_lock); + clear_bit(IWL_MEI_STATUS_SAP_CONNECTED, &iwl_mei_status); + spin_unlock_bh(&mei->data_q_lock); + + if (iwl_mei_cache.ops) + iwl_mei_cache.ops->rfkill(iwl_mei_cache.priv, false); + + /* + * mei_cldev_disable will return only after all the MEI Rx is done. + * It must be called when iwl_mei_mutex is *not* held, since it waits + * for our Rx handler to complete. + * After it returns, no new Rx will start. + */ + mei_cldev_disable(cldev); + + /* + * Since the netdev was already removed and the netdev's removal + * includes a call to synchronize_net() so that we know there won't be + * any new Rx that will trigger the following workers. + */ + cancel_work_sync(&mei->send_csa_msg_wk); + cancel_delayed_work_sync(&mei->csa_throttle_end_wk); + + /* + * If someone waits for the ownership, let him know that we are going + * down and that we are not connected anymore. He'll be able to take + * the device. + */ + wake_up_all(&mei->get_ownership_wq); + + mutex_lock(&iwl_mei_mutex); + + iwl_mei_global_cldev = NULL; + + wake_up_all(&mei->get_nvm_wq); + + iwl_mei_free_shared_mem(cldev); + + iwl_mei_dbgfs_unregister(mei); + + mei_cldev_set_drvdata(cldev, NULL); + + kfree(mei->nvm); + + kfree(rcu_access_pointer(mei->filters)); + + devm_kfree(&cldev->dev, mei); + + mutex_unlock(&iwl_mei_mutex); +} + +static const struct mei_cl_device_id iwl_mei_tbl[] = { + { KBUILD_MODNAME, MEI_WLAN_UUID, MEI_CL_VERSION_ANY}, + + /* required last entry */ + { } +}; + +/* + * Do not export the device table because this module is loaded by + * iwlwifi's dependency. + */ + +static struct mei_cl_driver iwl_mei_cl_driver = { + .id_table = iwl_mei_tbl, + .name = KBUILD_MODNAME, + .probe = iwl_mei_probe, + .remove = iwl_mei_remove, +}; + +module_mei_cl_driver(iwl_mei_cl_driver); diff --git a/drivers/net/wireless/intel/iwlwifi/mei/net.c b/drivers/net/wireless/intel/iwlwifi/mei/net.c new file mode 100644 index 000000000000..5f966af69720 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/net.c @@ -0,0 +1,409 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#include <uapi/linux/if_ether.h> +#include <uapi/linux/if_arp.h> +#include <uapi/linux/icmp.h> + +#include <linux/etherdevice.h> +#include <linux/netdevice.h> +#include <linux/skbuff.h> +#include <linux/ieee80211.h> + +#include <net/cfg80211.h> +#include <net/ip.h> + +#include <linux/if_arp.h> +#include <linux/icmp.h> +#include <linux/udp.h> +#include <linux/ip.h> +#include <linux/mm.h> + +#include "internal.h" +#include "sap.h" +#include "iwl-mei.h" + +/* + * Returns true if further filtering should be stopped. Only in that case + * pass_to_csme and rx_handler_res are set. Otherwise, next level of filters + * should be checked. + */ +static bool iwl_mei_rx_filter_eth(const struct ethhdr *ethhdr, + const struct iwl_sap_oob_filters *filters, + bool *pass_to_csme, + rx_handler_result_t *rx_handler_res) +{ + const struct iwl_sap_eth_filter *filt; + + /* This filter is not relevant for UCAST packet */ + if (!is_multicast_ether_addr(ethhdr->h_dest) || + is_broadcast_ether_addr(ethhdr->h_dest)) + return false; + + for (filt = &filters->eth_filters[0]; + filt < &filters->eth_filters[0] + ARRAY_SIZE(filters->eth_filters); + filt++) { + /* Assume there are no enabled filter after a disabled one */ + if (!(filt->flags & SAP_ETH_FILTER_ENABLED)) + break; + + if (compare_ether_header(filt->mac_address, ethhdr->h_dest)) + continue; + + /* Packet needs to reach the host's stack */ + if (filt->flags & SAP_ETH_FILTER_COPY) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + /* We have an authoritative answer, stop filtering */ + if (filt->flags & SAP_ETH_FILTER_STOP) { + *pass_to_csme = true; + return true; + } + + return false; + } + + /* MCAST frames that don't match layer 2 filters are not sent to ME */ + *pass_to_csme = false; + + return true; +} + +/* + * Returns true iff the frame should be passed to CSME in which case + * rx_handler_res is set. + */ +static bool iwl_mei_rx_filter_arp(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + rx_handler_result_t *rx_handler_res) +{ + const struct iwl_sap_ipv4_filter *filt = &filters->ipv4_filter; + const struct arphdr *arp; + const __be32 *target_ip; + u32 flags = le32_to_cpu(filt->flags); + + if (!pskb_may_pull(skb, arp_hdr_len(skb->dev))) + return false; + + arp = arp_hdr(skb); + + /* Handle only IPv4 over ethernet ARP frames */ + if (arp->ar_hrd != htons(ARPHRD_ETHER) || + arp->ar_pro != htons(ETH_P_IP)) + return false; + + /* + * After the ARP header, we have: + * src MAC address - 6 bytes + * src IP address - 4 bytes + * target MAC addess - 6 bytes + */ + target_ip = (void *)((u8 *)(arp + 1) + + ETH_ALEN + sizeof(__be32) + ETH_ALEN); + + /* + * ARP request is forwarded to ME only if IP address match in the + * ARP request's target ip field. + */ + if (arp->ar_op == htons(ARPOP_REQUEST) && + (filt->flags & cpu_to_le32(SAP_IPV4_FILTER_ARP_REQ_PASS)) && + (filt->ipv4_addr == 0 || filt->ipv4_addr == *target_ip)) { + if (flags & SAP_IPV4_FILTER_ARP_REQ_COPY) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + return true; + } + + /* ARP reply is always forwarded to ME regardless of the IP */ + if (flags & SAP_IPV4_FILTER_ARP_RESP_PASS && + arp->ar_op == htons(ARPOP_REPLY)) { + if (flags & SAP_IPV4_FILTER_ARP_RESP_COPY) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + return true; + } + + return false; +} + +static bool +iwl_mei_rx_filter_tcp_udp(struct sk_buff *skb, bool ip_match, + const struct iwl_sap_oob_filters *filters, + rx_handler_result_t *rx_handler_res) +{ + const struct iwl_sap_flex_filter *filt; + + for (filt = &filters->flex_filters[0]; + filt < &filters->flex_filters[0] + ARRAY_SIZE(filters->flex_filters); + filt++) { + if (!(filt->flags & SAP_FLEX_FILTER_ENABLED)) + break; + + /* + * We are required to have a match on the IP level and we didn't + * have such match. + */ + if ((filt->flags & + (SAP_FLEX_FILTER_IPV4 | SAP_FLEX_FILTER_IPV6)) && + !ip_match) + continue; + + if ((filt->flags & SAP_FLEX_FILTER_UDP) && + ip_hdr(skb)->protocol != IPPROTO_UDP) + continue; + + if ((filt->flags & SAP_FLEX_FILTER_TCP) && + ip_hdr(skb)->protocol != IPPROTO_TCP) + continue; + + /* + * We must have either a TCP header or a UDP header, both + * starts with a source port and then a destination port. + * Both are big endian words. + * Use a UDP header and that will work for TCP as well. + */ + if ((filt->src_port && filt->src_port != udp_hdr(skb)->source) || + (filt->dst_port && filt->dst_port != udp_hdr(skb)->dest)) + continue; + + if (filt->flags & SAP_FLEX_FILTER_COPY) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + return true; + } + + return false; +} + +static bool iwl_mei_rx_filter_ipv4(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + rx_handler_result_t *rx_handler_res) +{ + const struct iwl_sap_ipv4_filter *filt = &filters->ipv4_filter; + const struct iphdr *iphdr; + unsigned int iphdrlen; + bool match; + + if (!pskb_may_pull(skb, skb_network_offset(skb) + sizeof(*iphdr)) || + !pskb_may_pull(skb, skb_network_offset(skb) + + sizeof(ip_hdrlen(skb) - sizeof(*iphdr)))) + return false; + + iphdrlen = ip_hdrlen(skb); + iphdr = ip_hdr(skb); + match = !filters->ipv4_filter.ipv4_addr || + filters->ipv4_filter.ipv4_addr == iphdr->daddr; + + skb_set_transport_header(skb, skb_network_offset(skb) + iphdrlen); + + switch (ip_hdr(skb)->protocol) { + case IPPROTO_UDP: + case IPPROTO_TCP: + /* + * UDP header is shorter than TCP header and we look at the first bytes + * of the header anyway (see below). + * If we have a truncated TCP packet, let CSME handle this. + */ + if (!pskb_may_pull(skb, skb_transport_offset(skb) + + sizeof(struct udphdr))) + return false; + + return iwl_mei_rx_filter_tcp_udp(skb, match, + filters, rx_handler_res); + + case IPPROTO_ICMP: { + struct icmphdr *icmp; + + if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(*icmp))) + return false; + + icmp = icmp_hdr(skb); + + /* + * Don't pass echo requests to ME even if it wants it as we + * want the host to answer. + */ + if ((filt->flags & cpu_to_le32(SAP_IPV4_FILTER_ICMP_PASS)) && + match && (icmp->type != ICMP_ECHO || icmp->code != 0)) { + if (filt->flags & cpu_to_le32(SAP_IPV4_FILTER_ICMP_COPY)) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + return true; + } + break; + } + case IPPROTO_ICMPV6: + /* TODO: Should we have the same ICMP request logic here too? */ + if ((filters->icmpv6_flags & cpu_to_le32(SAP_ICMPV6_FILTER_ENABLED) && + match)) { + if (filters->icmpv6_flags & + cpu_to_le32(SAP_ICMPV6_FILTER_COPY)) + *rx_handler_res = RX_HANDLER_PASS; + else + *rx_handler_res = RX_HANDLER_CONSUMED; + + return true; + } + break; + default: + return false; + } + + return false; +} + +static bool iwl_mei_rx_filter_ipv6(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + rx_handler_result_t *rx_handler_res) +{ + *rx_handler_res = RX_HANDLER_PASS; + + /* TODO */ + + return false; +} + +static rx_handler_result_t +iwl_mei_rx_pass_to_csme(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + bool *pass_to_csme) +{ + const struct ethhdr *ethhdr = (void *)skb_mac_header(skb); + rx_handler_result_t rx_handler_res = RX_HANDLER_PASS; + bool (*filt_handler)(struct sk_buff *skb, + const struct iwl_sap_oob_filters *filters, + rx_handler_result_t *rx_handler_res); + + /* + * skb->data points the IP header / ARP header and the ETH header + * is in the headroom. + */ + skb_reset_network_header(skb); + + /* + * MCAST IP packets sent by us are received again here without + * an ETH header. Drop them here. + */ + if (!skb_mac_offset(skb)) + return RX_HANDLER_PASS; + + if (skb_headroom(skb) < sizeof(*ethhdr)) + return RX_HANDLER_PASS; + + if (iwl_mei_rx_filter_eth(ethhdr, filters, + pass_to_csme, &rx_handler_res)) + return rx_handler_res; + + switch (skb->protocol) { + case htons(ETH_P_IP): + filt_handler = iwl_mei_rx_filter_ipv4; + break; + case htons(ETH_P_ARP): + filt_handler = iwl_mei_rx_filter_arp; + break; + case htons(ETH_P_IPV6): + filt_handler = iwl_mei_rx_filter_ipv6; + break; + default: + *pass_to_csme = false; + return rx_handler_res; + } + + *pass_to_csme = filt_handler(skb, filters, &rx_handler_res); + + return rx_handler_res; +} + +rx_handler_result_t iwl_mei_rx_filter(struct sk_buff *orig_skb, + const struct iwl_sap_oob_filters *filters, + bool *pass_to_csme) +{ + rx_handler_result_t ret; + struct sk_buff *skb; + + ret = iwl_mei_rx_pass_to_csme(orig_skb, filters, pass_to_csme); + + if (!*pass_to_csme) + return RX_HANDLER_PASS; + + if (ret == RX_HANDLER_PASS) + skb = skb_copy(orig_skb, GFP_ATOMIC); + else + skb = orig_skb; + + /* CSME wants the MAC header as well, push it back */ + skb_push(skb, skb->data - skb_mac_header(skb)); + + /* + * Add the packet that CSME wants to get to the ring. Don't send the + * Check Shared Area HECI message since this is not possible from the + * Rx context. The caller will schedule a worker to do just that. + */ + iwl_mei_add_data_to_ring(skb, false); + + /* + * In case we drop the packet, don't free it, the caller will do that + * for us + */ + if (ret == RX_HANDLER_PASS) + dev_kfree_skb(skb); + + return ret; +} + +#define DHCP_SERVER_PORT 67 +#define DHCP_CLIENT_PORT 68 +void iwl_mei_tx_copy_to_csme(struct sk_buff *origskb, unsigned int ivlen) +{ + struct ieee80211_hdr *hdr; + struct sk_buff *skb; + struct ethhdr ethhdr; + struct ethhdr *eth; + + /* Catch DHCP packets */ + if (origskb->protocol != htons(ETH_P_IP) || + ip_hdr(origskb)->protocol != IPPROTO_UDP || + udp_hdr(origskb)->source != htons(DHCP_CLIENT_PORT) || + udp_hdr(origskb)->dest != htons(DHCP_SERVER_PORT)) + return; + + /* + * We could be a bit less aggressive here and not copy everything, but + * this is very rare anyway, do don't bother much. + */ + skb = skb_copy(origskb, GFP_ATOMIC); + if (!skb) + return; + + skb->protocol = origskb->protocol; + + hdr = (void *)skb->data; + + memcpy(ethhdr.h_dest, ieee80211_get_DA(hdr), ETH_ALEN); + memcpy(ethhdr.h_source, ieee80211_get_SA(hdr), ETH_ALEN); + + /* + * Remove the ieee80211 header + IV + SNAP but leave the ethertype + * We still have enough headroom for the sap header. + */ + pskb_pull(skb, ieee80211_hdrlen(hdr->frame_control) + ivlen + 6); + eth = skb_push(skb, sizeof(ethhdr.h_dest) + sizeof(ethhdr.h_source)); + memcpy(eth, ðhdr, sizeof(ethhdr.h_dest) + sizeof(ethhdr.h_source)); + + iwl_mei_add_data_to_ring(skb, true); + + dev_kfree_skb(skb); +} +EXPORT_SYMBOL_GPL(iwl_mei_tx_copy_to_csme); diff --git a/drivers/net/wireless/intel/iwlwifi/mei/sap.h b/drivers/net/wireless/intel/iwlwifi/mei/sap.h new file mode 100644 index 000000000000..11e3009121cc --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/sap.h @@ -0,0 +1,733 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#ifndef __sap_h__ +#define __sap_h__ + +#include "mei/iwl-mei.h" + +/** + * DOC: Introduction + * + * SAP is the protocol used by the Intel Wireless driver (iwlwifi) + * and the wireless driver implemented in the CSME firmware. + * It allows to do several things: + * 1) Decide who is the owner of the device: CSME or the host + * 2) When the host is the owner of the device, CSME can still + * send and receive packets through iwlwifi. + * + * The protocol uses the ME interface (mei driver) to send + * messages to the CSME firmware. Those messages have a header + * &struct iwl_sap_me_msg_hdr and this header is followed + * by a payload. + * + * Since this messaging system cannot support high amounts of + * traffic, iwlwifi and the CSME firmware's WLAN driver have an + * addtional communication pipe to exchange information. The body + * of the message is copied to a shared area and the message that + * goes over the ME interface just signals the other side + * that a new message is waiting in the shared area. The ME + * interface is used only for signaling and not to transfer + * the payload. + * + * This shared area of memory is DMA'able mapped to be + * writable by both the CSME firmware and iwlwifi. It is + * mapped to address space of the device that controls the ME + * interface's DMA engine. Any data that iwlwifi needs to + * send to the CSME firmware needs to be copied to there. + */ + +/** + * DOC: Initial Handshake + * + * Once we get a link to the CMSE's WLAN driver we start the handshake + * to establish the shared memory that will allow the communication between + * the CSME's WLAN driver and the host. + * + * 1) Host sends %SAP_ME_MSG_START message with the physical address + * of the shared area. + * 2) CSME replies with %SAP_ME_MSG_START_OK which includes the versions + * protocol versions supported by CSME. + */ + +/** + * DOC: Host and driver state messages + * + * In order to let CSME konw about the host state and the host driver state, + * the host sends messages that let CSME know about the host's state. + * When the host driver is loaded, the host sends %SAP_MSG_NOTIF_WIFIDR_UP. + * When the host driver is unloaded, the host sends %SAP_MSG_NOTIF_WIFIDR_DOWN. + * When the iwlmei is unloaded, %SAP_MSG_NOTIF_HOST_GOES_DOWN is sent to let + * CSME know not to access the shared memory anymore since it'll be freed. + * + * CSME will reply to SAP_MSG_NOTIF_WIFIDR_UP by + * %SAP_MSG_NOTIF_AMT_STATE to let the host driver whether CSME can use the + * WiFi device or not followed by %SAP_MSG_NOTIF_CSME_CONN_STATUS to inform + * the host driver on the connection state of CSME. + * + * When host is associated to an AP, it must send %SAP_MSG_NOTIF_HOST_LINK_UP + * and when it disconnect from the AP, it must send + * %SAP_MSG_NOTIF_HOST_LINK_DOWN. + */ + +/** + * DOC: Ownership + * + * The device can be controlled either by the CSME firmware or + * by the host driver: iwlwifi. There is a negotiaion between + * those two entities to determine who controls (or owns) the + * device. Since the CSME can control the device even when the + * OS is not working or even missing, the CSME can request the + * device if it comes to the conclusion that the OS's host driver + * is not operational. This is why the host driver needs to + * signal CSME that it is up and running. If the driver is + * unloaded, it'll signal CSME that it is going down so that + * CSME can take ownership. + */ + +/** + * DOC: Ownership transfer + * + * When the host driver needs the device, it'll send the + * %SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP that will be replied by + * %SAP_MSG_NOTIF_CSME_REPLY_TO_HOST_OWNERSHIP_REQ which will let the + * host know whether the ownership is granted or no. If the ownership is + * granted, the hosts sends %SAP_MSG_NOTIF_HOST_OWNERSHIP_CONFIRMED. + * + * When CSME requests ownership, it'll send the + * %SAP_MSG_NOTIF_CSME_TAKING_OWNERSHIP and give some time to host to stop + * accessing the device. The host needs to send + * %SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED to confirm that it won't access + * the device anymore. If the host failed to send this message fast enough, + * CSME will take ownership on the device anyway. + * When CSME is willing to release the ownership, it'll send + * %SAP_MSG_NOTIF_CSME_CAN_RELEASE_OWNERSHIP. + */ + +/** + * DOC: Data messages + * + * Data messages must be sent and receives on a separate queue in the shared + * memory. Almost all the data messages use the %SAP_MSG_DATA_PACKET for both + * packets sent by CSME to the host to be sent to the AP or for packets + * received from the AP and sent by the host to CSME. + * CSME sends filters to the host to let the host what inbound packets it must + * send to CSME. Those filters are received by the host as a + * %SAP_MSG_NOTIF_CSME_FILTERS command. + * The only outbound packets that must be sent to CSME are the DHCP packets. + * Those packets must use the %SAP_MSG_CB_DATA_PACKET message. + */ + +/** + * enum iwl_sap_me_msg_id - the ID of the ME message + * @SAP_ME_MSG_START: See &struct iwl_sap_me_msg_start. + * @SAP_ME_MSG_START_OK: See &struct iwl_sap_me_msg_start_ok. + * @SAP_ME_MSG_CHECK_SHARED_AREA: This message has no payload. + */ +enum iwl_sap_me_msg_id { + SAP_ME_MSG_START = 1, + SAP_ME_MSG_START_OK, + SAP_ME_MSG_CHECK_SHARED_AREA, +}; + +/** + * struct iwl_sap_me_msg_hdr - the header of the ME message + * @type: the type of the message, see &enum iwl_sap_me_msg_id. + * @seq_num: a sequence number used for debug only. + * @len: the length of the mssage. + */ +struct iwl_sap_me_msg_hdr { + __le32 type; + __le32 seq_num; + __le32 len; +} __packed; + +/** + * struct iwl_sap_me_msg_start - used for the %SAP_ME_MSG_START message + * @hdr: See &struct iwl_sap_me_msg_hdr. + * @shared_mem: physical address of SAP shared memory area. + * @init_data_seq_num: seq_num of the first data packet HOST -> CSME. + * @init_notif_seq_num: seq_num of the first notification HOST -> CSME. + * @supported_versions: The host sends to the CSME a zero-terminated array + * of versions its supports. + * + * This message is sent by the host to CSME and will responded by the + * %SAP_ME_MSG_START_OK message. + */ +struct iwl_sap_me_msg_start { + struct iwl_sap_me_msg_hdr hdr; + __le64 shared_mem; + __le16 init_data_seq_num; + __le16 init_notif_seq_num; + u8 supported_versions[64]; +} __packed; + +/** + * struct iwl_sap_me_msg_start_ok - used for the %SAP_ME_MSG_START_OK + * @hdr: See &struct iwl_sap_me_msg_hdr + * @init_data_seq_num: Not used. + * @init_notif_seq_num: Not used + * @supported_version: The version that will be used. + * @reserved: For alignment. + * + * This message is sent by CSME to the host in response to the + * %SAP_ME_MSG_START message. + */ +struct iwl_sap_me_msg_start_ok { + struct iwl_sap_me_msg_hdr hdr; + __le16 init_data_seq_num; + __le16 init_notif_seq_num; + u8 supported_version; + u8 reserved[3]; +} __packed; + +/** + * enum iwl_sap_msg - SAP messages + * @SAP_MSG_NOTIF_BOTH_WAYS_MIN: Not used. + * @SAP_MSG_NOTIF_PING: No payload. Solicitate a response message (check-alive). + * @SAP_MSG_NOTIF_PONG: No payload. The response message. + * @SAP_MSG_NOTIF_BOTH_WAYS_MAX: Not used. + * + * @SAP_MSG_NOTIF_FROM_CSME_MIN: Not used. + * @SAP_MSG_NOTIF_CSME_FILTERS: TODO + * @SAP_MSG_NOTIF_AMT_STATE: Payload is a DW. Any non-zero value means + * that CSME is enabled. + * @SAP_MSG_NOTIF_CSME_REPLY_TO_HOST_OWNERSHIP_REQ: Payload is a DW. 0 means + * the host will not get ownership. Any other value means the host is + * the owner. + * @SAP_MSG_NOTIF_CSME_TAKING_OWNERSHIP: No payload. + * @SAP_MSG_NOTIF_TRIGGER_IP_REFRESH: No payload. + * @SAP_MSG_NOTIF_CSME_CAN_RELEASE_OWNERSHIP: No payload. + * @SAP_MSG_NOTIF_NIC_OWNER: Payload is a DW. See &enum iwl_sap_nic_owner. + * @SAP_MSG_NOTIF_CSME_CONN_STATUS: See &struct iwl_sap_notif_conn_status. + * @SAP_MSG_NOTIF_NVM: See &struct iwl_sap_nvm. + * @SAP_MSG_NOTIF_FROM_CSME_MAX: Not used. + * + * @SAP_MSG_NOTIF_FROM_HOST_MIN: Not used. + * @SAP_MSG_NOTIF_BAND_SELECTION: TODO + * @SAP_MSG_NOTIF_RADIO_STATE: Payload is a DW. + * See &enum iwl_sap_radio_state_bitmap. + * @SAP_MSG_NOTIF_NIC_INFO: See &struct iwl_sap_notif_host_nic_info. + * @SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP: No payload. + * @SAP_MSG_NOTIF_HOST_SUSPENDS: Payload is a DW. Bitmap described in + * &enum iwl_sap_notif_host_suspends_bitmap. + * @SAP_MSG_NOTIF_HOST_RESUMES: Payload is a DW. 0 or 1. 1 says that + * the CSME should re-initialize the init control block. + * @SAP_MSG_NOTIF_HOST_GOES_DOWN: No payload. + * @SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED: No payload. + * @SAP_MSG_NOTIF_COUNTRY_CODE: See &struct iwl_sap_notif_country_code. + * @SAP_MSG_NOTIF_HOST_LINK_UP: See &struct iwl_sap_notif_host_link_up. + * @SAP_MSG_NOTIF_HOST_LINK_DOWN: See &struct iwl_sap_notif_host_link_down. + * @SAP_MSG_NOTIF_WHO_OWNS_NIC: No payload. + * @SAP_MSG_NOTIF_WIFIDR_DOWN: No payload. + * @SAP_MSG_NOTIF_WIFIDR_UP: No payload. + * @SAP_MSG_NOTIF_HOST_OWNERSHIP_CONFIRMED: No payload. + * @SAP_MSG_NOTIF_SAR_LIMITS: See &struct iwl_sap_notif_sar_limits. + * @SAP_MSG_NOTIF_GET_NVM: No payload. Triggers %SAP_MSG_NOTIF_NVM. + * @SAP_MSG_NOTIF_FROM_HOST_MAX: Not used. + * + * @SAP_MSG_DATA_MIN: Not used. + * @SAP_MSG_DATA_PACKET: Packets that passed the filters defined by + * %SAP_MSG_NOTIF_CSME_FILTERS. The payload is &struct iwl_sap_hdr with + * the payload of the packet immediately afterwards. + * @SAP_MSG_CB_DATA_PACKET: Indicates to CSME that we transmitted a specific + * packet. Used only for DHCP transmitted packets. See + * &struct iwl_sap_cb_data. + * @SAP_MSG_DATA_MAX: Not used. + */ +enum iwl_sap_msg { + SAP_MSG_NOTIF_BOTH_WAYS_MIN = 0, + SAP_MSG_NOTIF_PING = 1, + SAP_MSG_NOTIF_PONG = 2, + SAP_MSG_NOTIF_BOTH_WAYS_MAX, + + SAP_MSG_NOTIF_FROM_CSME_MIN = 500, + SAP_MSG_NOTIF_CSME_FILTERS = SAP_MSG_NOTIF_FROM_CSME_MIN, + /* 501 is deprecated */ + SAP_MSG_NOTIF_AMT_STATE = 502, + SAP_MSG_NOTIF_CSME_REPLY_TO_HOST_OWNERSHIP_REQ = 503, + SAP_MSG_NOTIF_CSME_TAKING_OWNERSHIP = 504, + SAP_MSG_NOTIF_TRIGGER_IP_REFRESH = 505, + SAP_MSG_NOTIF_CSME_CAN_RELEASE_OWNERSHIP = 506, + /* 507 is deprecated */ + /* 508 is deprecated */ + /* 509 is deprecated */ + /* 510 is deprecated */ + SAP_MSG_NOTIF_NIC_OWNER = 511, + SAP_MSG_NOTIF_CSME_CONN_STATUS = 512, + SAP_MSG_NOTIF_NVM = 513, + SAP_MSG_NOTIF_FROM_CSME_MAX, + + SAP_MSG_NOTIF_FROM_HOST_MIN = 1000, + SAP_MSG_NOTIF_BAND_SELECTION = SAP_MSG_NOTIF_FROM_HOST_MIN, + SAP_MSG_NOTIF_RADIO_STATE = 1001, + SAP_MSG_NOTIF_NIC_INFO = 1002, + SAP_MSG_NOTIF_HOST_ASKS_FOR_NIC_OWNERSHIP = 1003, + SAP_MSG_NOTIF_HOST_SUSPENDS = 1004, + SAP_MSG_NOTIF_HOST_RESUMES = 1005, + SAP_MSG_NOTIF_HOST_GOES_DOWN = 1006, + SAP_MSG_NOTIF_CSME_OWNERSHIP_CONFIRMED = 1007, + SAP_MSG_NOTIF_COUNTRY_CODE = 1008, + SAP_MSG_NOTIF_HOST_LINK_UP = 1009, + SAP_MSG_NOTIF_HOST_LINK_DOWN = 1010, + SAP_MSG_NOTIF_WHO_OWNS_NIC = 1011, + SAP_MSG_NOTIF_WIFIDR_DOWN = 1012, + SAP_MSG_NOTIF_WIFIDR_UP = 1013, + /* 1014 is deprecated */ + SAP_MSG_NOTIF_HOST_OWNERSHIP_CONFIRMED = 1015, + SAP_MSG_NOTIF_SAR_LIMITS = 1016, + SAP_MSG_NOTIF_GET_NVM = 1017, + SAP_MSG_NOTIF_FROM_HOST_MAX, + + SAP_MSG_DATA_MIN = 2000, + SAP_MSG_DATA_PACKET = SAP_MSG_DATA_MIN, + SAP_MSG_CB_DATA_PACKET = 2001, + SAP_MSG_DATA_MAX, +}; + +/** + * struct iwl_sap_hdr - prefixes any SAP message + * @type: See &enum iwl_sap_msg. + * @len: The length of the message (header not included). + * @seq_num: For debug. + * @payload: The payload of the message. + */ +struct iwl_sap_hdr { + __le16 type; + __le16 len; + __le32 seq_num; + u8 payload[0]; +}; + +/** + * struct iwl_sap_msg_dw - suits any DW long SAP message + * @hdr: The SAP header + * @val: The value of the DW. + */ +struct iwl_sap_msg_dw { + struct iwl_sap_hdr hdr; + __le32 val; +}; + +/** + * enum iwl_sap_nic_owner - used by %SAP_MSG_NOTIF_NIC_OWNER + * @SAP_NIC_OWNER_UNKNOWN: Not used. + * @SAP_NIC_OWNER_HOST: The host owns the NIC. + * @SAP_NIC_OWNER_ME: CSME owns the NIC. + */ +enum iwl_sap_nic_owner { + SAP_NIC_OWNER_UNKNOWN, + SAP_NIC_OWNER_HOST, + SAP_NIC_OWNER_ME, +}; + +enum iwl_sap_wifi_auth_type { + SAP_WIFI_AUTH_TYPE_OPEN = IWL_MEI_AKM_AUTH_OPEN, + SAP_WIFI_AUTH_TYPE_RSNA = IWL_MEI_AKM_AUTH_RSNA, + SAP_WIFI_AUTH_TYPE_RSNA_PSK = IWL_MEI_AKM_AUTH_RSNA_PSK, + SAP_WIFI_AUTH_TYPE_SAE = IWL_MEI_AKM_AUTH_SAE, + SAP_WIFI_AUTH_TYPE_MAX, +}; + +/** + * enum iwl_sap_wifi_cipher_alg + * @SAP_WIFI_CIPHER_ALG_NONE: TBD + * @SAP_WIFI_CIPHER_ALG_CCMP: TBD + * @SAP_WIFI_CIPHER_ALG_GCMP: TBD + * @SAP_WIFI_CIPHER_ALG_GCMP_256: TBD + */ +enum iwl_sap_wifi_cipher_alg { + SAP_WIFI_CIPHER_ALG_NONE = IWL_MEI_CIPHER_NONE, + SAP_WIFI_CIPHER_ALG_CCMP = IWL_MEI_CIPHER_CCMP, + SAP_WIFI_CIPHER_ALG_GCMP = IWL_MEI_CIPHER_GCMP, + SAP_WIFI_CIPHER_ALG_GCMP_256 = IWL_MEI_CIPHER_GCMP_256, +}; + +/** + * struct iwl_sap_notif_connection_info - nested in other structures + * @ssid_len: The length of the SSID. + * @ssid: The SSID. + * @auth_mode: The authentication mode. See &enum iwl_sap_wifi_auth_type. + * @pairwise_cipher: The cipher used for unicast packets. + * See &enum iwl_sap_wifi_cipher_alg. + * @channel: The channel on which we are associated. + * @band: The band on which we are associated. + * @reserved: For alignment. + * @bssid: The BSSID. + * @reserved1: For alignment. + */ +struct iwl_sap_notif_connection_info { + __le32 ssid_len; + u8 ssid[32]; + __le32 auth_mode; + __le32 pairwise_cipher; + u8 channel; + u8 band; + __le16 reserved; + u8 bssid[6]; + __le16 reserved1; +} __packed; + +/** + * enum iwl_sap_scan_request - for the scan_request field + * @SCAN_REQUEST_FILTERING: Filtering is requested. + * @SCAN_REQUEST_FAST: Fast scan is requested. + */ +enum iwl_sap_scan_request { + SCAN_REQUEST_FILTERING = 1 << 0, + SCAN_REQUEST_FAST = 1 << 1, +}; + +/** + * struct iwl_sap_notif_conn_status - payload of %SAP_MSG_NOTIF_CSME_CONN_STATUS + * @hdr: The SAP header + * @link_prot_state: Non-zero if link protection is active. + * @scan_request: See &enum iwl_sap_scan_request. + * @conn_info: Information about the connection. + */ +struct iwl_sap_notif_conn_status { + struct iwl_sap_hdr hdr; + __le32 link_prot_state; + __le32 scan_request; + struct iwl_sap_notif_connection_info conn_info; +} __packed; + +/** + * enum iwl_sap_radio_state_bitmap - used for %SAP_MSG_NOTIF_RADIO_STATE + * @SAP_SW_RFKILL_DEASSERTED: If set, SW RfKill is de-asserted + * @SAP_HW_RFKILL_DEASSERTED: If set, HW RfKill is de-asserted + * + * If both bits are set, then the radio is on. + */ +enum iwl_sap_radio_state_bitmap { + SAP_SW_RFKILL_DEASSERTED = 1 << 0, + SAP_HW_RFKILL_DEASSERTED = 1 << 1, +}; + +/** + * enum iwl_sap_notif_host_suspends_bitmap - used for %SAP_MSG_NOTIF_HOST_SUSPENDS + * @SAP_OFFER_NIC: TBD + * @SAP_FILTER_CONFIGURED: TBD + * @SAP_NLO_CONFIGURED: TBD + * @SAP_HOST_OWNS_NIC: TBD + * @SAP_LINK_PROTECTED: TBD + */ +enum iwl_sap_notif_host_suspends_bitmap { + SAP_OFFER_NIC = 1 << 0, + SAP_FILTER_CONFIGURED = 1 << 1, + SAP_NLO_CONFIGURED = 1 << 2, + SAP_HOST_OWNS_NIC = 1 << 3, + SAP_LINK_PROTECTED = 1 << 4, +}; + +/** + * struct iwl_sap_notif_country_code - payload of %SAP_MSG_NOTIF_COUNTRY_CODE + * @hdr: The SAP header + * @mcc: The country code. + * @source_id: TBD + * @reserved: For alignment. + * @diff_time: TBD + */ +struct iwl_sap_notif_country_code { + struct iwl_sap_hdr hdr; + __le16 mcc; + u8 source_id; + u8 reserved; + __le32 diff_time; +} __packed; + +/** + * struct iwl_sap_notif_host_link_up - payload of %SAP_MSG_NOTIF_HOST_LINK_UP + * @hdr: The SAP header + * @conn_info: Information about the connection. + * @colloc_channel: The collocated channel + * @colloc_band: The band of the collocated channel. + * @reserved: For alignment. + * @colloc_bssid: The collocated BSSID. + * @reserved1: For alignment. + */ +struct iwl_sap_notif_host_link_up { + struct iwl_sap_hdr hdr; + struct iwl_sap_notif_connection_info conn_info; + u8 colloc_channel; + u8 colloc_band; + __le16 reserved; + u8 colloc_bssid[6]; + __le16 reserved1; +} __packed; + +/** + * enum iwl_sap_notif_link_down_type - used in &struct iwl_sap_notif_host_link_down + * @HOST_LINK_DOWN_TYPE_NONE: TBD + * @HOST_LINK_DOWN_TYPE_TEMPORARY: TBD + * @HOST_LINK_DOWN_TYPE_LONG: TBD + */ +enum iwl_sap_notif_link_down_type { + HOST_LINK_DOWN_TYPE_NONE, + HOST_LINK_DOWN_TYPE_TEMPORARY, + HOST_LINK_DOWN_TYPE_LONG, +}; + +/** + * struct iwl_sap_notif_host_link_down - payload for %SAP_MSG_NOTIF_HOST_LINK_DOWN + * @hdr: The SAP header + * @type: See &enum iwl_sap_notif_link_down_type. + * @reserved: For alignment. + * @reason_valid: If 0, ignore the next field. + * @reason: The reason of the disconnection. + */ +struct iwl_sap_notif_host_link_down { + struct iwl_sap_hdr hdr; + u8 type; + u8 reserved[2]; + u8 reason_valid; + __le32 reason; +} __packed; + +/** + * struct iwl_sap_notif_host_nic_info - payload for %SAP_MSG_NOTIF_NIC_INFO + * @hdr: The SAP header + * @mac_address: The MAC address as configured to the interface. + * @nvm_address: The MAC address as configured in the NVM. + */ +struct iwl_sap_notif_host_nic_info { + struct iwl_sap_hdr hdr; + u8 mac_address[6]; + u8 nvm_address[6]; +} __packed; + +/** + * struct iwl_sap_notif_dw - payload is a dw + * @hdr: The SAP header. + * @dw: The payload. + */ +struct iwl_sap_notif_dw { + struct iwl_sap_hdr hdr; + __le32 dw; +} __packed; + +/** + * struct iwl_sap_notif_sar_limits - payload for %SAP_MSG_NOTIF_SAR_LIMITS + * @hdr: The SAP header + * @sar_chain_info_table: Tx power limits. + */ +struct iwl_sap_notif_sar_limits { + struct iwl_sap_hdr hdr; + __le16 sar_chain_info_table[2][5]; +} __packed; + +/** + * enum iwl_sap_nvm_caps - capabilities for NVM SAP + * @SAP_NVM_CAPS_LARI_SUPPORT: Lari is supported + * @SAP_NVM_CAPS_11AX_SUPPORT: 11AX is supported + */ +enum iwl_sap_nvm_caps { + SAP_NVM_CAPS_LARI_SUPPORT = BIT(0), + SAP_NVM_CAPS_11AX_SUPPORT = BIT(1), +}; + +/** + * struct iwl_sap_nvm - payload for %SAP_MSG_NOTIF_NVM + * @hdr: The SAP header. + * @hw_addr: The MAC address + * @n_hw_addrs: The number of MAC addresses + * @reserved: For alignment. + * @radio_cfg: The radio configuration. + * @caps: See &enum iwl_sap_nvm_caps. + * @nvm_version: The version of the NVM. + * @channels: The data for each channel. + */ +struct iwl_sap_nvm { + struct iwl_sap_hdr hdr; + u8 hw_addr[6]; + u8 n_hw_addrs; + u8 reserved; + __le32 radio_cfg; + __le32 caps; + __le32 nvm_version; + __le32 channels[110]; +} __packed; + +/** + * enum iwl_sap_eth_filter_flags - used in &struct iwl_sap_eth_filter + * @SAP_ETH_FILTER_STOP: Do not process further filters. + * @SAP_ETH_FILTER_COPY: Copy the packet to the CSME. + * @SAP_ETH_FILTER_ENABLED: If false, the filter should be ignored. + */ +enum iwl_sap_eth_filter_flags { + SAP_ETH_FILTER_STOP = BIT(0), + SAP_ETH_FILTER_COPY = BIT(1), + SAP_ETH_FILTER_ENABLED = BIT(2), +}; + +/** + * struct iwl_sap_eth_filter - a L2 filter + * @mac_address: Address to filter. + * @flags: See &enum iwl_sap_eth_filter_flags. + */ +struct iwl_sap_eth_filter { + u8 mac_address[6]; + u8 flags; +} __packed; + +/** + * enum iwl_sap_flex_filter_flags - used in &struct iwl_sap_flex_filter + * @SAP_FLEX_FILTER_COPY: Pass UDP / TCP packets to CSME. + * @SAP_FLEX_FILTER_ENABLED: If false, the filter should be ignored. + * @SAP_FLEX_FILTER_IPV4: Filter requires match on the IP address as well. + * @SAP_FLEX_FILTER_IPV6: Filter requires match on the IP address as well. + * @SAP_FLEX_FILTER_TCP: Filter should be applied on TCP packets. + * @SAP_FLEX_FILTER_UDP: Filter should be applied on UDP packets. + */ +enum iwl_sap_flex_filter_flags { + SAP_FLEX_FILTER_COPY = BIT(0), + SAP_FLEX_FILTER_ENABLED = BIT(1), + SAP_FLEX_FILTER_IPV6 = BIT(2), + SAP_FLEX_FILTER_IPV4 = BIT(3), + SAP_FLEX_FILTER_TCP = BIT(4), + SAP_FLEX_FILTER_UDP = BIT(5), +}; + +/** + * struct iwl_sap_flex_filter - + * @src_port: Source port in network format. + * @dst_port: Destination port in network format. + * @flags: Flags and protocol, see &enum iwl_sap_flex_filter_flags. + * @reserved: For alignment. + */ +struct iwl_sap_flex_filter { + __be16 src_port; + __be16 dst_port; + u8 flags; + u8 reserved; +} __packed; + +/** + * enum iwl_sap_ipv4_filter_flags - used in &struct iwl_sap_ipv4_filter + * @SAP_IPV4_FILTER_ICMP_PASS: Pass ICMP packets to CSME. + * @SAP_IPV4_FILTER_ICMP_COPY: Pass ICMP packets to host. + * @SAP_IPV4_FILTER_ARP_REQ_PASS: Pass ARP requests to CSME. + * @SAP_IPV4_FILTER_ARP_REQ_COPY: Pass ARP requests to host. + * @SAP_IPV4_FILTER_ARP_RESP_PASS: Pass ARP responses to CSME. + * @SAP_IPV4_FILTER_ARP_RESP_COPY: Pass ARP responses to host. + */ +enum iwl_sap_ipv4_filter_flags { + SAP_IPV4_FILTER_ICMP_PASS = BIT(0), + SAP_IPV4_FILTER_ICMP_COPY = BIT(1), + SAP_IPV4_FILTER_ARP_REQ_PASS = BIT(2), + SAP_IPV4_FILTER_ARP_REQ_COPY = BIT(3), + SAP_IPV4_FILTER_ARP_RESP_PASS = BIT(4), + SAP_IPV4_FILTER_ARP_RESP_COPY = BIT(5), +}; + +/** + * struct iwl_sap_ipv4_filter- + * @ipv4_addr: The IP address to filer. + * @flags: See &enum iwl_sap_ipv4_filter_flags. + */ +struct iwl_sap_ipv4_filter { + __be32 ipv4_addr; + __le32 flags; +} __packed; + +/** + * enum iwl_sap_ipv6_filter_flags - + * @SAP_IPV6_ADDR_FILTER_COPY: Pass packets to the host. + * @SAP_IPV6_ADDR_FILTER_ENABLED: If false, the filter should be ignored. + */ +enum iwl_sap_ipv6_filter_flags { + SAP_IPV6_ADDR_FILTER_COPY = BIT(0), + SAP_IPV6_ADDR_FILTER_ENABLED = BIT(1), +}; + +/** + * struct iwl_sap_ipv6_filter - + * @addr_lo24: Lowest 24 bits of the IPv6 address. + * @flags: See &enum iwl_sap_ipv6_filter_flags. + */ +struct iwl_sap_ipv6_filter { + u8 addr_lo24[3]; + u8 flags; +} __packed; + +/** + * enum iwl_sap_icmpv6_filter_flags - + * @SAP_ICMPV6_FILTER_ENABLED: If false, the filter should be ignored. + * @SAP_ICMPV6_FILTER_COPY: Pass packets to the host. + */ +enum iwl_sap_icmpv6_filter_flags { + SAP_ICMPV6_FILTER_ENABLED = BIT(0), + SAP_ICMPV6_FILTER_COPY = BIT(1), +}; + +/** + * enum iwl_sap_vlan_filter_flags - + * @SAP_VLAN_FILTER_VLAN_ID_MSK: TBD + * @SAP_VLAN_FILTER_ENABLED: If false, the filter should be ignored. + */ +enum iwl_sap_vlan_filter_flags { + SAP_VLAN_FILTER_VLAN_ID_MSK = 0x0FFF, + SAP_VLAN_FILTER_ENABLED = BIT(15), +}; + +/** + * struct iwl_sap_oob_filters - Out of band filters (for RX only) + * @flex_filters: Array of &struct iwl_sap_flex_filter. + * @icmpv6_flags: See &enum iwl_sap_icmpv6_filter_flags. + * @ipv6_filters: Array of &struct iwl_sap_ipv6_filter. + * @eth_filters: Array of &struct iwl_sap_eth_filter. + * @reserved: For alignment. + * @ipv4_filter: &struct iwl_sap_ipv4_filter. + * @vlan: See &enum iwl_sap_vlan_filter_flags. + */ +struct iwl_sap_oob_filters { + struct iwl_sap_flex_filter flex_filters[14]; + __le32 icmpv6_flags; + struct iwl_sap_ipv6_filter ipv6_filters[4]; + struct iwl_sap_eth_filter eth_filters[5]; + u8 reserved; + struct iwl_sap_ipv4_filter ipv4_filter; + __le16 vlan[4]; +} __packed; + +/** + * struct iwl_sap_csme_filters - payload of %SAP_MSG_NOTIF_CSME_FILTERS + * @hdr: The SAP header. + * @mode: Not used. + * @mac_address: Not used. + * @reserved: For alignment. + * @cbfilters: Not used. + * @filters: Out of band filters. + */ +struct iwl_sap_csme_filters { + struct iwl_sap_hdr hdr; + __le32 mode; + u8 mac_address[6]; + __le16 reserved; + u8 cbfilters[1728]; + struct iwl_sap_oob_filters filters; +} __packed; + +#define CB_TX_DHCP_FILT_IDX 30 +/** + * struct iwl_sap_cb_data - header to be added for transmitted packets. + * @hdr: The SAP header. + * @reserved: Not used. + * @to_me_filt_status: The filter that matches. Bit %CB_TX_DHCP_FILT_IDX should + * be set for DHCP (the only packet that uses this header). + * @reserved2: Not used. + * @data_len: The length of the payload. + * @payload: The payload of the transmitted packet. + */ +struct iwl_sap_cb_data { + struct iwl_sap_hdr hdr; + __le32 reserved[7]; + __le32 to_me_filt_status; + __le32 reserved2; + __le32 data_len; + u8 payload[]; +}; + +#endif /* __sap_h__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/trace-data.h b/drivers/net/wireless/intel/iwlwifi/mei/trace-data.h new file mode 100644 index 000000000000..83639c6225ca --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/trace-data.h @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright(c) 2021 Intel Corporation + */ + +#if !defined(CONFIG_IWLWIFI_DEVICE_TRACING) + +#define trace_iwlmei_sap_data(...) + +#else + +#if !defined(__IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_DATA) || defined(TRACE_HEADER_MULTI_READ) + +#ifndef __IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_DATA +enum iwl_sap_data_trace_type { + IWL_SAP_RX_DATA_TO_AIR, + IWL_SAP_TX_DATA_FROM_AIR, + IWL_SAP_RX_DATA_DROPPED_FROM_AIR, + IWL_SAP_TX_DHCP, +}; + +static inline size_t +iwlmei_sap_data_offset(enum iwl_sap_data_trace_type trace_type) +{ + switch (trace_type) { + case IWL_SAP_RX_DATA_TO_AIR: + return 0; + case IWL_SAP_TX_DATA_FROM_AIR: + case IWL_SAP_RX_DATA_DROPPED_FROM_AIR: + return sizeof(struct iwl_sap_hdr); + case IWL_SAP_TX_DHCP: + return sizeof(struct iwl_sap_cb_data); + default: + WARN_ON_ONCE(1); + } + + return 0; +} +#endif + +#define __IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_DATA + +#include <linux/tracepoint.h> +#include <linux/skbuff.h> +#include "sap.h" + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM iwlmei_sap_data + +TRACE_EVENT(iwlmei_sap_data, + TP_PROTO(const struct sk_buff *skb, + enum iwl_sap_data_trace_type trace_type), + TP_ARGS(skb, trace_type), + TP_STRUCT__entry( + __dynamic_array(u8, data, + skb->len - iwlmei_sap_data_offset(trace_type)) + __field(u32, trace_type) + ), + TP_fast_assign( + size_t offset = iwlmei_sap_data_offset(trace_type); + __entry->trace_type = trace_type; + skb_copy_bits(skb, offset, __get_dynamic_array(data), + skb->len - offset); + ), + TP_printk("sap_data:trace_type %d len %d", + __entry->trace_type, __get_dynamic_array_len(data)) +); + +/* + * If you add something here, add a stub in case + * !defined(CONFIG_IWLWIFI_DEVICE_TRACING) + */ + +#endif /* __IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_DATA */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace-data +#include <trace/define_trace.h> + +#endif /* CONFIG_IWLWIFI_DEVICE_TRACING */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/trace.c b/drivers/net/wireless/intel/iwlwifi/mei/trace.c new file mode 100644 index 000000000000..47ac32ef9f69 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/trace.c @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ + +#include <linux/module.h> + +/* sparse doesn't like tracepoint macros */ +#ifndef __CHECKER__ + +#define CREATE_TRACE_POINTS +#include "trace.h" +#include "trace-data.h" + +#endif /* __CHECKER__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mei/trace.h b/drivers/net/wireless/intel/iwlwifi/mei/trace.h new file mode 100644 index 000000000000..45ecb22ec84a --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mei/trace.h @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright(c) 2021 Intel Corporation + */ + +#if !defined(CONFIG_IWLWIFI_DEVICE_TRACING) + +#define trace_iwlmei_sap_cmd(...) +#define trace_iwlmei_me_msg(...) + +#else + +#if !defined(__IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_CMD) || defined(TRACE_HEADER_MULTI_READ) +#define __IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_CMD + +#include <linux/tracepoint.h> + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM iwlmei_sap_cmd + +#include "mei/sap.h" + +TRACE_EVENT(iwlmei_sap_cmd, + TP_PROTO(const struct iwl_sap_hdr *sap_cmd, bool tx), + TP_ARGS(sap_cmd, tx), + TP_STRUCT__entry( + __dynamic_array(u8, cmd, + le16_to_cpu(sap_cmd->len) + sizeof(*sap_cmd)) + __field(u8, tx) + __field(u16, type) + __field(u16, len) + __field(u32, seq) + ), + TP_fast_assign( + memcpy(__get_dynamic_array(cmd), sap_cmd, + le16_to_cpu(sap_cmd->len) + sizeof(*sap_cmd)); + __entry->tx = tx; + __entry->type = le16_to_cpu(sap_cmd->type); + __entry->len = le16_to_cpu(sap_cmd->len); + __entry->seq = le32_to_cpu(sap_cmd->seq_num); + ), + TP_printk("sap_cmd %s: type %d len %d seq %d", __entry->tx ? "Tx" : "Rx", + __entry->type, __entry->len, __entry->seq) +); + +TRACE_EVENT(iwlmei_me_msg, + TP_PROTO(const struct iwl_sap_me_msg_hdr *hdr, bool tx), + TP_ARGS(hdr, tx), + TP_STRUCT__entry( + __field(u8, type) + __field(u8, tx) + __field(u32, seq_num) + ), + TP_fast_assign( + __entry->type = le32_to_cpu(hdr->type); + __entry->seq_num = le32_to_cpu(hdr->seq_num); + __entry->tx = tx; + ), + TP_printk("ME message: %s: type %d seq %d", __entry->tx ? "Tx" : "Rx", + __entry->type, __entry->seq_num) +); + +/* + * If you add something here, add a stub in case + * !defined(CONFIG_IWLWIFI_DEVICE_TRACING) + */ + +#endif /* __IWLWIFI_DEVICE_TRACE_IWLWIFI_SAP_CMD */ + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE trace +#include <trace/define_trace.h> + +#endif /* CONFIG_IWLWIFI_DEVICE_TRACING */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile index 75fc2d935e5d..11e814b7cad0 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/Makefile +++ b/drivers/net/wireless/intel/iwlwifi/mvm/Makefile @@ -10,5 +10,6 @@ iwlmvm-y += rfi.o iwlmvm-$(CONFIG_IWLWIFI_DEBUGFS) += debugfs.o debugfs-vif.o iwlmvm-$(CONFIG_IWLWIFI_LEDS) += led.o iwlmvm-$(CONFIG_PM) += d3.o +iwlmvm-$(CONFIG_IWLMEI) += vendor-cmd.o ccflags-y += -I $(srctree)/$(src)/../ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c index 863fec150e53..6ce78c03e51f 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/fw.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/fw.c @@ -757,6 +757,8 @@ int iwl_mvm_sar_select_profile(struct iwl_mvm *mvm, int prof_a, int prof_b) if (ret) return ret; + iwl_mei_set_power_limit(per_chain); + IWL_DEBUG_RADIO(mvm, "Sending REDUCE_TX_POWER_CMD per chain\n"); return iwl_mvm_send_cmd_pdu(mvm, REDUCE_TX_POWER_CMD, 0, len, &cmd); } @@ -1401,7 +1403,6 @@ static int iwl_mvm_load_rt_fw(struct iwl_mvm *mvm) if (iwl_mvm_has_unified_ucode(mvm)) return iwl_run_unified_mvm_ucode(mvm); - WARN_ON(!mvm->nvm_data); ret = iwl_run_init_mvm_ucode(mvm); if (ret) { diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c index 897e3b91ddb2..ede9fb1ebf2e 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mac80211.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2020 Intel Corporation + * Copyright (C) 2012-2014, 2018-2021 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -191,6 +191,7 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy, if (IS_ERR_OR_NULL(resp)) { IWL_DEBUG_LAR(mvm, "Could not get update from FW %d\n", PTR_ERR_OR_ZERO(resp)); + resp = NULL; goto out; } @@ -212,7 +213,6 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy, __le16_to_cpu(resp->cap), resp_ver); /* Store the return source id */ src_id = resp->source_id; - kfree(resp); if (IS_ERR_OR_NULL(regd)) { IWL_DEBUG_LAR(mvm, "Could not get parse update from FW %d\n", PTR_ERR_OR_ZERO(regd)); @@ -224,7 +224,10 @@ struct ieee80211_regdomain *iwl_mvm_get_regdomain(struct wiphy *wiphy, mvm->lar_regdom_set = true; mvm->mcc_src = src_id; + iwl_mei_set_country_code(__le16_to_cpu(resp->mcc)); + out: + kfree(resp); return regd; } @@ -718,6 +721,8 @@ int iwl_mvm_mac_setup_register(struct iwl_mvm *mvm) wiphy_ext_feature_set(hw->wiphy, NL80211_EXT_FEATURE_PROTECTED_TWT); + iwl_mvm_vendor_cmds_register(mvm); + hw->wiphy->available_antennas_tx = iwl_mvm_get_valid_tx_ant(mvm); hw->wiphy->available_antennas_rx = iwl_mvm_get_valid_rx_ant(mvm); @@ -1084,6 +1089,27 @@ int __iwl_mvm_mac_start(struct iwl_mvm *mvm) lockdep_assert_held(&mvm->mutex); + ret = iwl_mvm_mei_get_ownership(mvm); + if (ret) + return ret; + + if (mvm->mei_nvm_data) { + /* We got the NIC, we can now free the MEI NVM data */ + kfree(mvm->mei_nvm_data); + mvm->mei_nvm_data = NULL; + + /* + * We can't free the nvm_data we allocated based on the SAP + * data because we registered to cfg80211 with the channels + * allocated on mvm->nvm_data. Keep a pointer in temp_nvm_data + * just in order to be able free it later. + * NULLify nvm_data so that we will read the NVM from the + * firmware this time. + */ + mvm->temp_nvm_data = mvm->nvm_data; + mvm->nvm_data = NULL; + } + if (test_bit(IWL_MVM_STATUS_HW_RESTART_REQUESTED, &mvm->status)) { /* * Now convert the HW_RESTART_REQUESTED flag to IN_HW_RESTART @@ -1144,6 +1170,8 @@ static int iwl_mvm_mac_start(struct ieee80211_hw *hw) mutex_unlock(&mvm->mutex); + iwl_mvm_mei_set_sw_rfkill_state(mvm); + return ret; } @@ -1261,6 +1289,8 @@ static void iwl_mvm_mac_stop(struct ieee80211_hw *hw) */ flush_work(&mvm->roc_done_wk); + iwl_mvm_mei_set_sw_rfkill_state(mvm); + mutex_lock(&mvm->mutex); __iwl_mvm_mac_stop(mvm); mutex_unlock(&mvm->mutex); @@ -1531,6 +1561,15 @@ static int iwl_mvm_mac_add_interface(struct ieee80211_hw *hw, mvm->monitor_on = true; iwl_mvm_vif_dbgfs_register(mvm, vif); + + if (!test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status) && + vif->type == NL80211_IFTYPE_STATION && !vif->p2p && + !mvm->csme_vif && mvm->mei_registered) { + iwl_mei_set_nic_info(vif->addr, mvm->nvm_data->hw_addr); + iwl_mei_set_netdev(ieee80211_vif_to_wdev(vif)->netdev); + mvm->csme_vif = vif; + } + goto out_unlock; out_unbind: @@ -1583,6 +1622,11 @@ static void iwl_mvm_mac_remove_interface(struct ieee80211_hw *hw, mutex_lock(&mvm->mutex); + if (vif == mvm->csme_vif) { + iwl_mei_set_netdev(NULL); + mvm->csme_vif = NULL; + } + probe_data = rcu_dereference_protected(mvmvif->probe_resp_data, lockdep_is_held(&mvm->mutex)); RCU_INIT_POINTER(mvmvif->probe_resp_data, NULL); @@ -2393,6 +2437,7 @@ static void iwl_mvm_bss_info_changed_station(struct iwl_mvm *mvm, IEEE80211_SMPS_DYNAMIC); } } else if (mvmvif->ap_sta_id != IWL_MVM_INVALID_STA) { + iwl_mvm_mei_host_disassociated(mvm); /* * If update fails - SF might be running in associated * mode while disassociated - which is forbidden. @@ -3129,6 +3174,69 @@ static void iwl_mvm_reset_cca_40mhz_workaround(struct iwl_mvm *mvm, } } +static void iwl_mvm_mei_host_associated(struct iwl_mvm *mvm, + struct ieee80211_vif *vif, + struct iwl_mvm_sta *mvm_sta) +{ +#if IS_ENABLED(CONFIG_IWLMEI) + struct iwl_mvm_vif *mvmvif = iwl_mvm_vif_from_mac80211(vif); + struct iwl_mei_conn_info conn_info = { + .ssid_len = vif->bss_conf.ssid_len, + .channel = vif->bss_conf.chandef.chan->hw_value, + }; + + if (test_bit(IWL_MVM_STATUS_IN_HW_RESTART, &mvm->status)) + return; + + if (!mvm->mei_registered) + return; + + switch (mvm_sta->pairwise_cipher) { + case WLAN_CIPHER_SUITE_CCMP: + conn_info.pairwise_cipher = IWL_MEI_CIPHER_CCMP; + break; + case WLAN_CIPHER_SUITE_GCMP: + conn_info.pairwise_cipher = IWL_MEI_CIPHER_GCMP; + break; + case WLAN_CIPHER_SUITE_GCMP_256: + conn_info.pairwise_cipher = IWL_MEI_CIPHER_GCMP_256; + break; + case 0: + /* open profile */ + break; + default: + /* cipher not supported, don't send anything to iwlmei */ + return; + } + + switch (mvmvif->rekey_data.akm) { + case WLAN_AKM_SUITE_SAE & 0xff: + conn_info.auth_mode = IWL_MEI_AKM_AUTH_SAE; + break; + case WLAN_AKM_SUITE_PSK & 0xff: + conn_info.auth_mode = IWL_MEI_AKM_AUTH_RSNA_PSK; + break; + case WLAN_AKM_SUITE_8021X & 0xff: + conn_info.auth_mode = IWL_MEI_AKM_AUTH_RSNA; + break; + case 0: + /* open profile */ + conn_info.auth_mode = IWL_MEI_AKM_AUTH_OPEN; + break; + default: + /* auth method / AKM not supported */ + /* TODO: All the FT vesions of these? */ + return; + } + + memcpy(conn_info.ssid, vif->bss_conf.ssid, vif->bss_conf.ssid_len); + memcpy(conn_info.bssid, vif->bss_conf.bssid, ETH_ALEN); + + /* TODO: add support for collocated AP data */ + iwl_mei_host_associated(&conn_info, NULL); +#endif +} + static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw, struct ieee80211_vif *vif, struct ieee80211_sta *sta, @@ -3273,6 +3381,7 @@ static int iwl_mvm_mac_sta_state(struct ieee80211_hw *hw, * multicast data frames can be forwarded to the driver */ iwl_mvm_mac_ctxt_changed(mvm, vif, false, NULL); + iwl_mvm_mei_host_associated(mvm, vif, mvm_sta); } iwl_mvm_rs_rate_init(mvm, sta, mvmvif->phy_ctxt->channel->band, @@ -3482,6 +3591,8 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw, int ret, i; u8 key_offset; + mvmsta = iwl_mvm_sta_from_mac80211(sta); + switch (key->cipher) { case WLAN_CIPHER_SUITE_TKIP: if (!mvm->trans->trans_cfg->gen2) { @@ -3590,7 +3701,6 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw, struct ieee80211_key_seq seq; int tid, q; - mvmsta = iwl_mvm_sta_from_mac80211(sta); WARN_ON(rcu_access_pointer(mvmsta->ptk_pn[keyidx])); ptk_pn = kzalloc(struct_size(ptk_pn, q, mvm->trans->num_rx_queues), @@ -3617,6 +3727,9 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw, else key_offset = STA_KEY_IDX_INVALID; + if (key->flags & IEEE80211_KEY_FLAG_PAIRWISE) + mvmsta->pairwise_cipher = key->cipher; + IWL_DEBUG_MAC80211(mvm, "set hwcrypto key\n"); ret = iwl_mvm_set_sta_key(mvm, vif, sta, key, key_offset); if (ret) { @@ -3662,7 +3775,6 @@ static int __iwl_mvm_mac_set_key(struct ieee80211_hw *hw, (key->cipher == WLAN_CIPHER_SUITE_CCMP || key->cipher == WLAN_CIPHER_SUITE_GCMP || key->cipher == WLAN_CIPHER_SUITE_GCMP_256)) { - mvmsta = iwl_mvm_sta_from_mac80211(sta); ptk_pn = rcu_dereference_protected( mvmsta->ptk_pn[keyidx], lockdep_is_held(&mvm->mutex)); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h index a72d85086fe3..8cd0ed096010 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/mvm.h @@ -30,6 +30,7 @@ #include "fw/runtime.h" #include "fw/dbg.h" #include "fw/acpi.h" +#include "mei/iwl-mei.h" #include "iwl-nvm-parse.h" #include <linux/average.h> @@ -830,6 +831,18 @@ struct iwl_mvm { const char *nvm_file_name; struct iwl_nvm_data *nvm_data; + struct iwl_mei_nvm *mei_nvm_data; + struct iwl_mvm_csme_conn_info __rcu *csme_conn_info; + bool mei_rfkill_blocked; + bool mei_registered; + struct work_struct sap_connected_wk; + + /* + * NVM built based on the SAP data but that we can't free even after + * we get ownership because it contains the cfg80211's channel. + */ + struct iwl_nvm_data *temp_nvm_data; + /* NVM sections */ struct iwl_nvm_section nvm_sections[NVM_MAX_NUM_SECTIONS]; @@ -1021,6 +1034,8 @@ struct iwl_mvm { /* Indicate if 32Khz external clock is valid */ u32 ext_clock_valid; + /* This vif used by CSME to send / receive traffic */ + struct ieee80211_vif *csme_vif; struct ieee80211_vif __rcu *csa_vif; struct ieee80211_vif __rcu *csa_tx_blocked_vif; u8 csa_tx_block_bcn_timeout; @@ -1139,6 +1154,11 @@ enum iwl_mvm_status { IWL_MVM_STATUS_STARTING, }; +struct iwl_mvm_csme_conn_info { + struct rcu_head rcu_head; + struct iwl_mei_conn_info conn_info; +}; + /* Keep track of completed init configuration */ enum iwl_mvm_init_status { IWL_MVM_INIT_STATUS_THERMAL_INIT_COMPLETE = BIT(0), @@ -1942,6 +1962,17 @@ void iwl_mvm_enter_ctkill(struct iwl_mvm *mvm); int iwl_mvm_send_temp_report_ths_cmd(struct iwl_mvm *mvm); int iwl_mvm_ctdp_command(struct iwl_mvm *mvm, u32 op, u32 budget); +#if IS_ENABLED(CONFIG_IWLMEI) + +/* vendor commands */ +void iwl_mvm_vendor_cmds_register(struct iwl_mvm *mvm); + +#else + +static inline void iwl_mvm_vendor_cmds_register(struct iwl_mvm *mvm) {} + +#endif + /* Location Aware Regulatory */ struct iwl_mcc_update_resp * iwl_mvm_update_mcc(struct iwl_mvm *mvm, const char *alpha2, @@ -2161,4 +2192,47 @@ enum iwl_location_cipher iwl_mvm_cipher_to_location_cipher(u32 cipher) return IWL_LOCATION_CIPHER_INVALID; } } + +struct iwl_mvm_csme_conn_info *iwl_mvm_get_csme_conn_info(struct iwl_mvm *mvm); +static inline int iwl_mvm_mei_get_ownership(struct iwl_mvm *mvm) +{ + if (mvm->mei_registered) + return iwl_mei_get_ownership(); + return 0; +} + +static inline void iwl_mvm_mei_tx_copy_to_csme(struct iwl_mvm *mvm, + struct sk_buff *skb, + unsigned int ivlen) +{ + if (mvm->mei_registered) + iwl_mei_tx_copy_to_csme(skb, ivlen); +} + +static inline void iwl_mvm_mei_host_disassociated(struct iwl_mvm *mvm) +{ + if (mvm->mei_registered) + iwl_mei_host_disassociated(); +} + +static inline void iwl_mvm_mei_device_down(struct iwl_mvm *mvm) +{ + if (mvm->mei_registered) + iwl_mei_device_down(); +} + +static inline void iwl_mvm_mei_set_sw_rfkill_state(struct iwl_mvm *mvm) +{ + bool sw_rfkill = + mvm->hw_registered ? rfkill_blocked(mvm->hw->wiphy->rfkill) : false; + + if (mvm->mei_registered) + iwl_mei_set_rfkill_state(iwl_mvm_is_radio_killed(mvm), + sw_rfkill); +} + +void iwl_mvm_send_roaming_forbidden_event(struct iwl_mvm *mvm, + struct ieee80211_vif *vif, + bool forbidden); + #endif /* __IWL_MVM_H__ */ diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c index cd08e289cd9a..0cd92b3915f1 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/ops.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/ops.c @@ -683,14 +683,43 @@ static const struct iwl_fw_runtime_ops iwl_mvm_fwrt_ops = { static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm) { + struct iwl_trans *trans = mvm->trans; int ret; + if (trans->csme_own) { + if (WARN(!mvm->mei_registered, + "csme is owner, but we aren't registered to iwlmei\n")) + goto get_nvm_from_fw; + + mvm->mei_nvm_data = iwl_mei_get_nvm(); + if (mvm->mei_nvm_data) { + /* + * mvm->mei_nvm_data is set and because of that, + * we'll load the NVM from the FW when we'll get + * ownership. + */ + mvm->nvm_data = + iwl_parse_mei_nvm_data(trans, trans->cfg, + mvm->mei_nvm_data, mvm->fw); + return 0; + } + + IWL_ERR(mvm, + "Got a NULL NVM from CSME, trying to get it from the device\n"); + } + +get_nvm_from_fw: rtnl_lock(); wiphy_lock(mvm->hw->wiphy); mutex_lock(&mvm->mutex); - ret = iwl_run_init_mvm_ucode(mvm); + ret = iwl_trans_start_hw(mvm->trans); + if (ret) { + mutex_unlock(&mvm->mutex); + return ret; + } + ret = iwl_run_init_mvm_ucode(mvm); if (ret && ret != -ERFKILL) iwl_fw_dbg_error_collect(&mvm->fwrt, FW_DBG_TRIGGER_DRIVER); if (!ret && iwl_mvm_is_lar_supported(mvm)) { @@ -705,7 +734,7 @@ static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm) wiphy_unlock(mvm->hw->wiphy); rtnl_unlock(); - if (ret < 0) + if (ret) IWL_ERR(mvm, "Failed to run INIT ucode: %d\n", ret); return ret; @@ -713,6 +742,7 @@ static int iwl_mvm_start_get_nvm(struct iwl_mvm *mvm) static int iwl_mvm_start_post_nvm(struct iwl_mvm *mvm) { + struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused; int ret; iwl_mvm_toggle_tx_ant(mvm, &mvm->mgmt_last_antenna_idx); @@ -720,10 +750,17 @@ static int iwl_mvm_start_post_nvm(struct iwl_mvm *mvm) ret = iwl_mvm_mac_setup_register(mvm); if (ret) return ret; + mvm->hw_registered = true; iwl_mvm_dbgfs_register(mvm); + wiphy_rfkill_set_hw_state_reason(mvm->hw->wiphy, + mvm->mei_rfkill_blocked, + RFKILL_HARD_BLOCK_NOT_OWNER); + + iwl_mvm_mei_set_sw_rfkill_state(mvm); + return 0; } @@ -904,6 +941,109 @@ static const struct iwl_dump_sanitize_ops iwl_mvm_sanitize_ops = { .frob_mem = iwl_mvm_frob_mem, }; +static void iwl_mvm_me_conn_status(void *priv, const struct iwl_mei_conn_info *conn_info) +{ + struct iwl_mvm *mvm = priv; + struct iwl_mvm_csme_conn_info *prev_conn_info, *curr_conn_info; + + /* + * This is protected by the guarantee that this function will not be + * called twice on two different threads + */ + prev_conn_info = rcu_dereference_protected(mvm->csme_conn_info, true); + + curr_conn_info = kzalloc(sizeof(*curr_conn_info), GFP_KERNEL); + if (!curr_conn_info) + return; + + curr_conn_info->conn_info = *conn_info; + + rcu_assign_pointer(mvm->csme_conn_info, curr_conn_info); + + if (prev_conn_info) + kfree_rcu(prev_conn_info, rcu_head); +} + +static void iwl_mvm_mei_rfkill(void *priv, bool blocked) +{ + struct iwl_mvm *mvm = priv; + + mvm->mei_rfkill_blocked = blocked; + if (!mvm->hw_registered) + return; + + wiphy_rfkill_set_hw_state_reason(mvm->hw->wiphy, + mvm->mei_rfkill_blocked, + RFKILL_HARD_BLOCK_NOT_OWNER); +} + +static void iwl_mvm_mei_roaming_forbidden(void *priv, bool forbidden) +{ + struct iwl_mvm *mvm = priv; + + if (!mvm->hw_registered || !mvm->csme_vif) + return; + + iwl_mvm_send_roaming_forbidden_event(mvm, mvm->csme_vif, forbidden); +} + +static void iwl_mvm_sap_connected_wk(struct work_struct *wk) +{ + struct iwl_mvm *mvm = + container_of(wk, struct iwl_mvm, sap_connected_wk); + int ret; + + ret = iwl_mvm_start_get_nvm(mvm); + if (ret) + goto out_free; + + ret = iwl_mvm_start_post_nvm(mvm); + if (ret) + goto out_free; + + return; + +out_free: + IWL_ERR(mvm, "Couldn't get started...\n"); + iwl_mei_start_unregister(); + iwl_mei_unregister_complete(); + iwl_fw_flush_dumps(&mvm->fwrt); + iwl_mvm_thermal_exit(mvm); + iwl_fw_runtime_free(&mvm->fwrt); + iwl_phy_db_free(mvm->phy_db); + kfree(mvm->scan_cmd); + iwl_trans_op_mode_leave(mvm->trans); + kfree(mvm->nvm_data); + kfree(mvm->mei_nvm_data); + + ieee80211_free_hw(mvm->hw); +} + +static void iwl_mvm_mei_sap_connected(void *priv) +{ + struct iwl_mvm *mvm = priv; + + if (!mvm->hw_registered) + schedule_work(&mvm->sap_connected_wk); +} + +static void iwl_mvm_mei_nic_stolen(void *priv) +{ + struct iwl_mvm *mvm = priv; + + rtnl_lock(); + cfg80211_shutdown_all_interfaces(mvm->hw->wiphy); + rtnl_unlock(); +} + +static const struct iwl_mei_ops mei_ops = { + .me_conn_status = iwl_mvm_me_conn_status, + .rfkill = iwl_mvm_mei_rfkill, + .roaming_forbidden = iwl_mvm_mei_roaming_forbidden, + .sap_connected = iwl_mvm_mei_sap_connected, + .nic_stolen = iwl_mvm_mei_nic_stolen, +}; + static struct iwl_op_mode * iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, const struct iwl_fw *fw, struct dentry *dbgfs_dir) @@ -915,9 +1055,10 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, static const u8 no_reclaim_cmds[] = { TX_CMD, }; - int err, scan_size; + int scan_size; u32 min_backoff; enum iwl_amsdu_size rb_size_default; + struct iwl_mvm_csme_conn_info *csme_conn_info __maybe_unused; /* * We use IWL_MVM_STATION_COUNT_MAX to check the validity of the station @@ -1017,6 +1158,7 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, INIT_WORK(&mvm->async_handlers_wk, iwl_mvm_async_handlers_wk); INIT_WORK(&mvm->roc_done_wk, iwl_mvm_roc_done_wk); + INIT_WORK(&mvm->sap_connected_wk, iwl_mvm_sap_connected_wk); INIT_DELAYED_WORK(&mvm->tdls_cs.dwork, iwl_mvm_tdls_ch_switch_work); INIT_DELAYED_WORK(&mvm->scan_timeout_dwork, iwl_mvm_scan_timeout_wk); INIT_WORK(&mvm->add_stream_wk, iwl_mvm_add_new_dqa_stream_wk); @@ -1139,10 +1281,6 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, IWL_DEBUG_EEPROM(mvm->trans->dev, "working without external nvm file\n"); - err = iwl_trans_start_hw(mvm->trans); - if (err) - goto out_free; - scan_size = iwl_mvm_scan_size(mvm); mvm->scan_cmd = kmalloc(scan_size, GFP_KERNEL); @@ -1167,8 +1305,14 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, mvm->debugfs_dir = dbgfs_dir; - if (iwl_mvm_start_get_nvm(mvm)) - goto out_thermal_exit; + mvm->mei_registered = !iwl_mei_register(mvm, &mei_ops); + + /* + * Get NVM failed, but we are registered to MEI, we'll get + * the NVM later when it'll be possible to get it from CSME. + */ + if (iwl_mvm_start_get_nvm(mvm) && mvm->mei_registered) + return op_mode; if (iwl_mvm_start_post_nvm(mvm)) goto out_thermal_exit; @@ -1177,6 +1321,10 @@ iwl_op_mode_mvm_start(struct iwl_trans *trans, const struct iwl_cfg *cfg, out_thermal_exit: iwl_mvm_thermal_exit(mvm); + if (mvm->mei_registered) { + iwl_mei_start_unregister(); + iwl_mei_unregister_complete(); + } out_free: iwl_fw_flush_dumps(&mvm->fwrt); iwl_fw_runtime_free(&mvm->fwrt); @@ -1203,6 +1351,7 @@ void iwl_mvm_stop_device(struct iwl_mvm *mvm) iwl_trans_stop_device(mvm->trans); iwl_free_fw_paging(&mvm->fwrt); iwl_fw_dump_conf_clear(&mvm->fwrt); + iwl_mvm_mei_device_down(mvm); } static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode) @@ -1210,11 +1359,33 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode) struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode); int i; + if (mvm->mei_registered) { + rtnl_lock(); + iwl_mei_set_netdev(NULL); + rtnl_unlock(); + iwl_mei_start_unregister(); + } + + /* + * After we unregister from mei, the worker can't be scheduled + * anymore. + */ + cancel_work_sync(&mvm->sap_connected_wk); + iwl_mvm_leds_exit(mvm); iwl_mvm_thermal_exit(mvm); - ieee80211_unregister_hw(mvm->hw); + /* + * If we couldn't get ownership on the device and we couldn't + * get the NVM from CSME, we haven't registered to mac80211. + * In that case, we didn't fail op_mode_start, because we are + * waiting for CSME to allow us to get the NVM to register to + * mac80211. If that didn't happen, we haven't registered to + * mac80211, hence the if below. + */ + if (mvm->hw_registered) + ieee80211_unregister_hw(mvm->hw); kfree(mvm->scan_cmd); kfree(mvm->mcast_filter_cmd); @@ -1229,6 +1400,9 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode) mvm->phy_db = NULL; kfree(mvm->nvm_data); + kfree(mvm->mei_nvm_data); + kfree(rcu_access_pointer(mvm->csme_conn_info)); + kfree(mvm->temp_nvm_data); for (i = 0; i < NVM_MAX_NUM_SECTIONS; i++) kfree(mvm->nvm_sections[i].data); @@ -1237,6 +1411,9 @@ static void iwl_op_mode_mvm_stop(struct iwl_op_mode *op_mode) iwl_fw_runtime_free(&mvm->fwrt); mutex_destroy(&mvm->mutex); + if (mvm->mei_registered) + iwl_mei_unregister_complete(); + ieee80211_free_hw(mvm->hw); } @@ -1519,6 +1696,12 @@ void iwl_mvm_set_hw_ctkill_state(struct iwl_mvm *mvm, bool state) iwl_mvm_set_rfkill_state(mvm); } +struct iwl_mvm_csme_conn_info *iwl_mvm_get_csme_conn_info(struct iwl_mvm *mvm) +{ + return rcu_dereference_protected(mvm->csme_conn_info, + lockdep_is_held(&mvm->mutex)); +} + static bool iwl_mvm_set_hw_rfkill_state(struct iwl_op_mode *op_mode, bool state) { struct iwl_mvm *mvm = IWL_OP_MODE_GET_MVM(op_mode); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h index 32b4d1935788..e34b82b2a288 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/sta.h +++ b/drivers/net/wireless/intel/iwlwifi/mvm/sta.h @@ -1,6 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause */ /* - * Copyright (C) 2012-2014, 2018-2020 Intel Corporation + * Copyright (C) 2012-2014, 2018-2021 Intel Corporation * Copyright (C) 2013-2014 Intel Mobile Communications GmbH * Copyright (C) 2015-2016 Intel Deutschland GmbH */ @@ -373,6 +373,7 @@ struct iwl_mvm_rxq_dup_data { * @tx_ant: the index of the antenna to use for data tx to this station. Only * used during connection establishment (e.g. for the 4 way handshake * exchange). + * @pairwise_cipher: used to feed iwlmei upon authorization * * When mac80211 creates a station it reserves some space (hw->sta_data_size) * in the structure for use by driver. This structure is placed in that @@ -415,6 +416,7 @@ struct iwl_mvm_sta { u8 sleep_tx_count; u8 avg_energy; u8 tx_ant; + u32 pairwise_cipher; }; u16 iwl_mvm_tid_queued(struct iwl_mvm *mvm, struct iwl_mvm_tid_data *tid_data); diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c index bdd4ee432548..1883d98abb2f 100644 --- a/drivers/net/wireless/intel/iwlwifi/mvm/tx.c +++ b/drivers/net/wireless/intel/iwlwifi/mvm/tx.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause /* - * Copyright (C) 2012-2014, 2018-2020 Intel Corporation + * Copyright (C) 2012-2014, 2018-2021 Intel Corporation * Copyright (C) 2013-2015 Intel Mobile Communications GmbH * Copyright (C) 2016-2017 Intel Deutschland GmbH */ @@ -1128,6 +1128,11 @@ static int iwl_mvm_tx_mpdu(struct iwl_mvm *mvm, struct sk_buff *skb, /* From now on, we cannot access info->control */ iwl_mvm_skb_prepare_status(skb, dev_cmd); + if (ieee80211_is_data(fc)) + iwl_mvm_mei_tx_copy_to_csme(mvm, skb, + info->control.hw_key ? + info->control.hw_key->iv_len : 0); + if (iwl_trans_tx(mvm->trans, skb, dev_cmd, txq_id)) goto drop_unlock_sta; diff --git a/drivers/net/wireless/intel/iwlwifi/mvm/vendor-cmd.c b/drivers/net/wireless/intel/iwlwifi/mvm/vendor-cmd.c new file mode 100644 index 000000000000..f702ad85e609 --- /dev/null +++ b/drivers/net/wireless/intel/iwlwifi/mvm/vendor-cmd.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2021 Intel Corporation + */ +#include "mvm.h" +#include <linux/nl80211-vnd-intel.h> + +static const struct nla_policy +iwl_mvm_vendor_attr_policy[NUM_IWL_MVM_VENDOR_ATTR] = { + [IWL_MVM_VENDOR_ATTR_ROAMING_FORBIDDEN] = { .type = NLA_U8 }, + [IWL_MVM_VENDOR_ATTR_AUTH_MODE] = { .type = NLA_U32 }, + [IWL_MVM_VENDOR_ATTR_CHANNEL_NUM] = { .type = NLA_U8 }, + [IWL_MVM_VENDOR_ATTR_SSID] = { .type = NLA_BINARY, + .len = IEEE80211_MAX_SSID_LEN }, + [IWL_MVM_VENDOR_ATTR_BAND] = { .type = NLA_U8 }, + [IWL_MVM_VENDOR_ATTR_COLLOC_CHANNEL] = { .type = NLA_U8 }, + [IWL_MVM_VENDOR_ATTR_COLLOC_ADDR] = { .type = NLA_BINARY, .len = ETH_ALEN }, +}; + +static int iwl_mvm_vendor_get_csme_conn_info(struct wiphy *wiphy, + struct wireless_dev *wdev, + const void *data, int data_len) +{ + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw); + struct iwl_mvm_csme_conn_info *csme_conn_info; + struct sk_buff *skb; + int err = 0; + + mutex_lock(&mvm->mutex); + csme_conn_info = iwl_mvm_get_csme_conn_info(mvm); + + if (!csme_conn_info) { + err = -EINVAL; + goto out_unlock; + } + + skb = cfg80211_vendor_cmd_alloc_reply_skb(wiphy, 200); + if (!skb) { + err = -ENOMEM; + goto out_unlock; + } + + if (nla_put_u32(skb, IWL_MVM_VENDOR_ATTR_AUTH_MODE, + csme_conn_info->conn_info.auth_mode) || + nla_put(skb, IWL_MVM_VENDOR_ATTR_SSID, + csme_conn_info->conn_info.ssid_len, + csme_conn_info->conn_info.ssid) || + nla_put_u32(skb, IWL_MVM_VENDOR_ATTR_STA_CIPHER, + csme_conn_info->conn_info.pairwise_cipher) || + nla_put_u8(skb, IWL_MVM_VENDOR_ATTR_CHANNEL_NUM, + csme_conn_info->conn_info.channel) || + nla_put(skb, IWL_MVM_VENDOR_ATTR_ADDR, ETH_ALEN, + csme_conn_info->conn_info.bssid)) { + kfree_skb(skb); + err = -ENOBUFS; + } + +out_unlock: + mutex_unlock(&mvm->mutex); + if (err) + return err; + + return cfg80211_vendor_cmd_reply(skb); +} + +static int iwl_mvm_vendor_host_get_ownership(struct wiphy *wiphy, + struct wireless_dev *wdev, + const void *data, int data_len) +{ + struct ieee80211_hw *hw = wiphy_to_ieee80211_hw(wiphy); + struct iwl_mvm *mvm = IWL_MAC80211_GET_MVM(hw); + + mutex_lock(&mvm->mutex); + iwl_mvm_mei_get_ownership(mvm); + mutex_unlock(&mvm->mutex); + + return 0; +} + +static const struct wiphy_vendor_command iwl_mvm_vendor_commands[] = { + { + .info = { + .vendor_id = INTEL_OUI, + .subcmd = IWL_MVM_VENDOR_CMD_GET_CSME_CONN_INFO, + }, + .doit = iwl_mvm_vendor_get_csme_conn_info, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV, + .policy = iwl_mvm_vendor_attr_policy, + .maxattr = MAX_IWL_MVM_VENDOR_ATTR, + }, + { + .info = { + .vendor_id = INTEL_OUI, + .subcmd = IWL_MVM_VENDOR_CMD_HOST_GET_OWNERSHIP, + }, + .doit = iwl_mvm_vendor_host_get_ownership, + .flags = WIPHY_VENDOR_CMD_NEED_WDEV, + .policy = iwl_mvm_vendor_attr_policy, + .maxattr = MAX_IWL_MVM_VENDOR_ATTR, + }, +}; + +enum iwl_mvm_vendor_events_idx { + /* 0x0 - 0x3 are deprecated */ + IWL_MVM_VENDOR_EVENT_IDX_ROAMING_FORBIDDEN = 4, + NUM_IWL_MVM_VENDOR_EVENT_IDX +}; + +static const struct nl80211_vendor_cmd_info +iwl_mvm_vendor_events[NUM_IWL_MVM_VENDOR_EVENT_IDX] = { + [IWL_MVM_VENDOR_EVENT_IDX_ROAMING_FORBIDDEN] = { + .vendor_id = INTEL_OUI, + .subcmd = IWL_MVM_VENDOR_CMD_ROAMING_FORBIDDEN_EVENT, + }, +}; + +void iwl_mvm_vendor_cmds_register(struct iwl_mvm *mvm) +{ + mvm->hw->wiphy->vendor_commands = iwl_mvm_vendor_commands; + mvm->hw->wiphy->n_vendor_commands = ARRAY_SIZE(iwl_mvm_vendor_commands); + mvm->hw->wiphy->vendor_events = iwl_mvm_vendor_events; + mvm->hw->wiphy->n_vendor_events = ARRAY_SIZE(iwl_mvm_vendor_events); +} + +void iwl_mvm_send_roaming_forbidden_event(struct iwl_mvm *mvm, + struct ieee80211_vif *vif, + bool forbidden) +{ + struct sk_buff *msg = + cfg80211_vendor_event_alloc(mvm->hw->wiphy, + ieee80211_vif_to_wdev(vif), + 200, IWL_MVM_VENDOR_EVENT_IDX_ROAMING_FORBIDDEN, + GFP_ATOMIC); + if (!msg) + return; + + if (WARN_ON(!vif)) + return; + + if (nla_put(msg, IWL_MVM_VENDOR_ATTR_VIF_ADDR, + ETH_ALEN, vif->addr) || + nla_put_u8(msg, IWL_MVM_VENDOR_ATTR_ROAMING_FORBIDDEN, forbidden)) + goto nla_put_failure; + + cfg80211_vendor_event(msg, GFP_ATOMIC); + return; + + nla_put_failure: + kfree_skb(msg); +} diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c index 5ce07f28e7c3..c2d1e899477c 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/drv.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/drv.c @@ -1426,15 +1426,18 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) * first trying to load the firmware etc. and potentially only * detecting any problems when the first interface is brought up. */ - ret = iwl_finish_nic_init(iwl_trans); - if (ret) - goto out_free_trans; - if (iwl_trans_grab_nic_access(iwl_trans)) { - /* all good */ - iwl_trans_release_nic_access(iwl_trans); - } else { - ret = -EIO; - goto out_free_trans; + ret = iwl_pcie_prepare_card_hw(iwl_trans); + if (!ret) { + ret = iwl_finish_nic_init(iwl_trans); + if (ret) + goto out_free_trans; + if (iwl_trans_grab_nic_access(iwl_trans)) { + /* all good */ + iwl_trans_release_nic_access(iwl_trans); + } else { + ret = -EIO; + goto out_free_trans; + } } iwl_trans->hw_rf_id = iwl_read32(iwl_trans, CSR_HW_RF_ID); @@ -1569,6 +1572,10 @@ static int iwl_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_free_trans; pci_set_drvdata(pdev, iwl_trans); + + /* try to get ownership so that we'll know if we don't own it */ + iwl_pcie_prepare_card_hw(iwl_trans); + iwl_trans->drv = iwl_drv_start(iwl_trans); if (IS_ERR(iwl_trans->drv)) { diff --git a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c index 1efb53f78a62..451b28de1392 100644 --- a/drivers/net/wireless/intel/iwlwifi/pcie/trans.c +++ b/drivers/net/wireless/intel/iwlwifi/pcie/trans.c @@ -24,6 +24,7 @@ #include "fw/error-dump.h" #include "fw/dbg.h" #include "fw/api/tx.h" +#include "mei/iwl-mei.h" #include "internal.h" #include "iwl-fh.h" #include "iwl-context-info-gen3.h" @@ -594,8 +595,10 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans) ret = iwl_pcie_set_hw_ready(trans); /* If the card is ready, exit 0 */ - if (ret >= 0) + if (ret >= 0) { + trans->csme_own = false; return 0; + } iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG, CSR_RESET_LINK_PWR_MGMT_DISABLED); @@ -608,8 +611,22 @@ int iwl_pcie_prepare_card_hw(struct iwl_trans *trans) do { ret = iwl_pcie_set_hw_ready(trans); - if (ret >= 0) + if (ret >= 0) { + trans->csme_own = false; return 0; + } + + if (iwl_mei_is_connected()) { + IWL_DEBUG_INFO(trans, + "Couldn't prepare the card but SAP is connected\n"); + trans->csme_own = true; + if (trans->trans_cfg->device_family != + IWL_DEVICE_FAMILY_9000) + IWL_ERR(trans, + "SAP not supported for this NIC family\n"); + + return -EBUSY; + } usleep_range(200, 1000); t += 200; diff --git a/drivers/net/wireless/intersil/hostap/hostap_hw.c b/drivers/net/wireless/intersil/hostap/hostap_hw.c index e459e7192ae9..b74f4cb5d6d3 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_hw.c +++ b/drivers/net/wireless/intersil/hostap/hostap_hw.c @@ -1815,8 +1815,9 @@ static int prism2_tx_80211(struct sk_buff *skb, struct net_device *dev) memset(&txdesc, 0, sizeof(txdesc)); /* skb->data starts with txdesc->frame_control */ - hdr_len = 24; - skb_copy_from_linear_data(skb, &txdesc.frame_control, hdr_len); + hdr_len = sizeof(txdesc.header); + BUILD_BUG_ON(hdr_len != 24); + skb_copy_from_linear_data(skb, &txdesc.header, hdr_len); if (ieee80211_is_data(txdesc.frame_control) && ieee80211_has_a4(txdesc.frame_control) && skb->len >= 30) { diff --git a/drivers/net/wireless/intersil/hostap/hostap_wlan.h b/drivers/net/wireless/intersil/hostap/hostap_wlan.h index dd2603d9b5d3..c25cd21d18bd 100644 --- a/drivers/net/wireless/intersil/hostap/hostap_wlan.h +++ b/drivers/net/wireless/intersil/hostap/hostap_wlan.h @@ -115,12 +115,14 @@ struct hfa384x_tx_frame { __le16 tx_control; /* HFA384X_TX_CTRL_ flags */ /* 802.11 */ - __le16 frame_control; /* parts not used */ - __le16 duration_id; - u8 addr1[ETH_ALEN]; - u8 addr2[ETH_ALEN]; /* filled by firmware */ - u8 addr3[ETH_ALEN]; - __le16 seq_ctrl; /* filled by firmware */ + struct_group(header, + __le16 frame_control; /* parts not used */ + __le16 duration_id; + u8 addr1[ETH_ALEN]; + u8 addr2[ETH_ALEN]; /* filled by firmware */ + u8 addr3[ETH_ALEN]; + __le16 seq_ctrl; /* filled by firmware */ + ); u8 addr4[ETH_ALEN]; __le16 data_len; diff --git a/drivers/net/wireless/marvell/libertas/host.h b/drivers/net/wireless/marvell/libertas/host.h index dfa22468b14a..af96bdba3b2b 100644 --- a/drivers/net/wireless/marvell/libertas/host.h +++ b/drivers/net/wireless/marvell/libertas/host.h @@ -308,10 +308,12 @@ struct txpd { __le32 tx_packet_location; /* Tx packet length */ __le16 tx_packet_length; - /* First 2 byte of destination MAC address */ - u8 tx_dest_addr_high[2]; - /* Last 4 byte of destination MAC address */ - u8 tx_dest_addr_low[4]; + struct_group(tx_dest_addr, + /* First 2 byte of destination MAC address */ + u8 tx_dest_addr_high[2]; + /* Last 4 byte of destination MAC address */ + u8 tx_dest_addr_low[4]; + ); /* Pkt Priority */ u8 priority; /* Pkt Trasnit Power control */ diff --git a/drivers/net/wireless/marvell/libertas/tx.c b/drivers/net/wireless/marvell/libertas/tx.c index aeb481740df6..27304a98787d 100644 --- a/drivers/net/wireless/marvell/libertas/tx.c +++ b/drivers/net/wireless/marvell/libertas/tx.c @@ -113,6 +113,7 @@ netdev_tx_t lbs_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) p802x_hdr = skb->data; pkt_len = skb->len; + BUILD_BUG_ON(sizeof(txpd->tx_dest_addr) != ETH_ALEN); if (priv->wdev->iftype == NL80211_IFTYPE_MONITOR) { struct tx_radiotap_hdr *rtap_hdr = (void *)skb->data; @@ -124,10 +125,10 @@ netdev_tx_t lbs_hard_start_xmit(struct sk_buff *skb, struct net_device *dev) pkt_len -= sizeof(*rtap_hdr); /* copy destination address from 802.11 header */ - memcpy(txpd->tx_dest_addr_high, p802x_hdr + 4, ETH_ALEN); + memcpy(&txpd->tx_dest_addr, p802x_hdr + 4, ETH_ALEN); } else { /* copy destination address from 802.3 header */ - memcpy(txpd->tx_dest_addr_high, p802x_hdr, ETH_ALEN); + memcpy(&txpd->tx_dest_addr, p802x_hdr, ETH_ALEN); } txpd->tx_packet_length = cpu_to_le16(pkt_len); diff --git a/drivers/net/wireless/marvell/libertas_tf/libertas_tf.h b/drivers/net/wireless/marvell/libertas_tf/libertas_tf.h index 5d726545d987..b2af2ddb6bc4 100644 --- a/drivers/net/wireless/marvell/libertas_tf/libertas_tf.h +++ b/drivers/net/wireless/marvell/libertas_tf/libertas_tf.h @@ -268,10 +268,12 @@ struct txpd { __le32 tx_packet_location; /* Tx packet length */ __le16 tx_packet_length; - /* First 2 byte of destination MAC address */ - u8 tx_dest_addr_high[2]; - /* Last 4 byte of destination MAC address */ - u8 tx_dest_addr_low[4]; + struct_group(tx_dest_addr, + /* First 2 byte of destination MAC address */ + u8 tx_dest_addr_high[2]; + /* Last 4 byte of destination MAC address */ + u8 tx_dest_addr_low[4]; + ); /* Pkt Priority */ u8 priority; /* Pkt Trasnit Power control */ diff --git a/drivers/net/wireless/marvell/libertas_tf/main.c b/drivers/net/wireless/marvell/libertas_tf/main.c index 71492211904b..02a1e1f547d8 100644 --- a/drivers/net/wireless/marvell/libertas_tf/main.c +++ b/drivers/net/wireless/marvell/libertas_tf/main.c @@ -232,7 +232,8 @@ static void lbtf_tx_work(struct work_struct *work) ieee80211_get_tx_rate(priv->hw, info)->hw_value); /* copy destination address from 802.11 header */ - memcpy(txpd->tx_dest_addr_high, skb->data + sizeof(struct txpd) + 4, + BUILD_BUG_ON(sizeof(txpd->tx_dest_addr) != ETH_ALEN); + memcpy(&txpd->tx_dest_addr, skb->data + sizeof(struct txpd) + 4, ETH_ALEN); txpd->tx_packet_length = cpu_to_le16(len); txpd->tx_packet_location = cpu_to_le32(sizeof(struct txpd)); diff --git a/drivers/net/wireless/marvell/mwifiex/fw.h b/drivers/net/wireless/marvell/mwifiex/fw.h index 2ff23ab259ab..63c25c69ed2b 100644 --- a/drivers/net/wireless/marvell/mwifiex/fw.h +++ b/drivers/net/wireless/marvell/mwifiex/fw.h @@ -2071,9 +2071,11 @@ struct mwifiex_ie_types_robust_coex { __le32 mode; } __packed; +#define MWIFIEX_VERSION_STR_LENGTH 128 + struct host_cmd_ds_version_ext { u8 version_str_sel; - char version_str[128]; + char version_str[MWIFIEX_VERSION_STR_LENGTH]; } __packed; struct host_cmd_ds_mgmt_frame_reg { diff --git a/drivers/net/wireless/marvell/mwifiex/init.c b/drivers/net/wireless/marvell/mwifiex/init.c index f006a3d72b40..88c72d1827a0 100644 --- a/drivers/net/wireless/marvell/mwifiex/init.c +++ b/drivers/net/wireless/marvell/mwifiex/init.c @@ -332,7 +332,7 @@ void mwifiex_set_trans_start(struct net_device *dev) int i; for (i = 0; i < dev->num_tx_queues; i++) - netdev_get_tx_queue(dev, i)->trans_start = jiffies; + txq_trans_cond_update(netdev_get_tx_queue(dev, i)); netif_trans_update(dev); } diff --git a/drivers/net/wireless/marvell/mwifiex/main.c b/drivers/net/wireless/marvell/mwifiex/main.c index 19b996c6a260..ace7371c4773 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.c +++ b/drivers/net/wireless/marvell/mwifiex/main.c @@ -226,6 +226,23 @@ exit_rx_proc: return 0; } +static void maybe_quirk_fw_disable_ds(struct mwifiex_adapter *adapter) +{ + struct mwifiex_private *priv = mwifiex_get_priv(adapter, MWIFIEX_BSS_ROLE_STA); + struct mwifiex_ver_ext ver_ext; + + if (test_and_set_bit(MWIFIEX_IS_REQUESTING_FW_VEREXT, &adapter->work_flags)) + return; + + memset(&ver_ext, 0, sizeof(ver_ext)); + ver_ext.version_str_sel = 1; + if (mwifiex_send_cmd(priv, HostCmd_CMD_VERSION_EXT, + HostCmd_ACT_GEN_GET, 0, &ver_ext, false)) { + mwifiex_dbg(priv->adapter, MSG, + "Checking hardware revision failed.\n"); + } +} + /* * The main process. * @@ -356,6 +373,7 @@ process_start: if (adapter->hw_status == MWIFIEX_HW_STATUS_INIT_DONE) { adapter->hw_status = MWIFIEX_HW_STATUS_READY; mwifiex_init_fw_complete(adapter); + maybe_quirk_fw_disable_ds(adapter); } } diff --git a/drivers/net/wireless/marvell/mwifiex/main.h b/drivers/net/wireless/marvell/mwifiex/main.h index 90012cbcfd15..332dd1c8db35 100644 --- a/drivers/net/wireless/marvell/mwifiex/main.h +++ b/drivers/net/wireless/marvell/mwifiex/main.h @@ -524,6 +524,7 @@ enum mwifiex_adapter_work_flags { MWIFIEX_IS_SUSPENDED, MWIFIEX_IS_HS_CONFIGURED, MWIFIEX_IS_HS_ENABLING, + MWIFIEX_IS_REQUESTING_FW_VEREXT, }; struct mwifiex_band_config { @@ -646,7 +647,7 @@ struct mwifiex_private { struct wireless_dev wdev; struct mwifiex_chan_freq_power cfp; u32 versionstrsel; - char version_str[128]; + char version_str[MWIFIEX_VERSION_STR_LENGTH]; #ifdef CONFIG_DEBUG_FS struct dentry *dfs_dev_dir; #endif @@ -1055,6 +1056,8 @@ struct mwifiex_adapter { void *devdump_data; int devdump_len; struct timer_list devdump_timer; + + bool ignore_btcoex_events; }; void mwifiex_process_tx_queue(struct mwifiex_adapter *adapter); diff --git a/drivers/net/wireless/marvell/mwifiex/pcie.c b/drivers/net/wireless/marvell/mwifiex/pcie.c index c3f5583ea70d..d5fb29400bad 100644 --- a/drivers/net/wireless/marvell/mwifiex/pcie.c +++ b/drivers/net/wireless/marvell/mwifiex/pcie.c @@ -3152,6 +3152,9 @@ static int mwifiex_init_pcie(struct mwifiex_adapter *adapter) if (ret) goto err_alloc_buffers; + if (pdev->device == PCIE_DEVICE_ID_MARVELL_88W8897) + adapter->ignore_btcoex_events = true; + return 0; err_alloc_buffers: diff --git a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c index 6b5d35d9e69f..1a4ae8a42a31 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_cmdresp.c @@ -708,11 +708,35 @@ static int mwifiex_ret_ver_ext(struct mwifiex_private *priv, { struct host_cmd_ds_version_ext *ver_ext = &resp->params.verext; + if (test_and_clear_bit(MWIFIEX_IS_REQUESTING_FW_VEREXT, &priv->adapter->work_flags)) { + if (strncmp(ver_ext->version_str, "ChipRev:20, BB:9b(10.00), RF:40(21)", + MWIFIEX_VERSION_STR_LENGTH) == 0) { + struct mwifiex_ds_auto_ds auto_ds = { + .auto_ds = DEEP_SLEEP_OFF, + }; + + mwifiex_dbg(priv->adapter, MSG, + "Bad HW revision detected, disabling deep sleep\n"); + + if (mwifiex_send_cmd(priv, HostCmd_CMD_802_11_PS_MODE_ENH, + DIS_AUTO_PS, BITMAP_AUTO_DS, &auto_ds, false)) { + mwifiex_dbg(priv->adapter, MSG, + "Disabling deep sleep failed.\n"); + } + } + + return 0; + } + if (version_ext) { version_ext->version_str_sel = ver_ext->version_str_sel; memcpy(version_ext->version_str, ver_ext->version_str, - sizeof(char) * 128); - memcpy(priv->version_str, ver_ext->version_str, 128); + MWIFIEX_VERSION_STR_LENGTH); + memcpy(priv->version_str, ver_ext->version_str, + MWIFIEX_VERSION_STR_LENGTH); + + /* Ensure the version string from the firmware is 0-terminated */ + priv->version_str[MWIFIEX_VERSION_STR_LENGTH - 1] = '\0'; } return 0; } diff --git a/drivers/net/wireless/marvell/mwifiex/sta_event.c b/drivers/net/wireless/marvell/mwifiex/sta_event.c index 68c63268e2e6..80e5d44bad9d 100644 --- a/drivers/net/wireless/marvell/mwifiex/sta_event.c +++ b/drivers/net/wireless/marvell/mwifiex/sta_event.c @@ -1058,6 +1058,9 @@ int mwifiex_process_sta_event(struct mwifiex_private *priv) break; case EVENT_BT_COEX_WLAN_PARA_CHANGE: dev_dbg(adapter->dev, "EVENT: BT coex wlan param update\n"); + if (adapter->ignore_btcoex_events) + break; + mwifiex_bt_coex_wlan_param_update_event(priv, adapter->event_skb); break; diff --git a/drivers/net/wireless/marvell/mwifiex/usb.c b/drivers/net/wireless/marvell/mwifiex/usb.c index 9736aa0ab7fd..8f01fcbe9396 100644 --- a/drivers/net/wireless/marvell/mwifiex/usb.c +++ b/drivers/net/wireless/marvell/mwifiex/usb.c @@ -130,7 +130,8 @@ static int mwifiex_usb_recv(struct mwifiex_adapter *adapter, default: mwifiex_dbg(adapter, ERROR, "unknown recv_type %#x\n", recv_type); - return -1; + ret = -1; + goto exit_restore_skb; } break; case MWIFIEX_USB_EP_DATA: diff --git a/drivers/net/wireless/marvell/mwl8k.c b/drivers/net/wireless/marvell/mwl8k.c index 529e325498cd..864a2ba9efee 100644 --- a/drivers/net/wireless/marvell/mwl8k.c +++ b/drivers/net/wireless/marvell/mwl8k.c @@ -4225,9 +4225,11 @@ struct mwl8k_cmd_set_key { __le32 key_info; __le32 key_id; __le16 key_len; - __u8 key_material[MAX_ENCR_KEY_LENGTH]; - __u8 tkip_tx_mic_key[MIC_KEY_LENGTH]; - __u8 tkip_rx_mic_key[MIC_KEY_LENGTH]; + struct { + __u8 key_material[MAX_ENCR_KEY_LENGTH]; + __u8 tkip_tx_mic_key[MIC_KEY_LENGTH]; + __u8 tkip_rx_mic_key[MIC_KEY_LENGTH]; + } tkip; __le16 tkip_rsc_low; __le32 tkip_rsc_high; __le16 tkip_tsc_low; @@ -4375,7 +4377,7 @@ static int mwl8k_cmd_encryption_set_key(struct ieee80211_hw *hw, goto done; } - memcpy(cmd->key_material, key->key, keymlen); + memcpy(&cmd->tkip, key->key, keymlen); cmd->action = cpu_to_le32(action); rc = mwl8k_post_pervif_cmd(hw, vif, &cmd->header); diff --git a/drivers/net/wireless/microchip/wilc1000/netdev.c b/drivers/net/wireless/microchip/wilc1000/netdev.c index 690572e01a2a..4712cd7dff9f 100644 --- a/drivers/net/wireless/microchip/wilc1000/netdev.c +++ b/drivers/net/wireless/microchip/wilc1000/netdev.c @@ -574,6 +574,7 @@ static int wilc_mac_open(struct net_device *ndev) struct wilc *wl = vif->wilc; int ret = 0; struct mgmt_frame_regs mgmt_regs = {}; + u8 addr[ETH_ALEN] __aligned(2); if (!wl || !wl->dev) { netdev_err(ndev, "device not ready\n"); @@ -596,10 +597,9 @@ static int wilc_mac_open(struct net_device *ndev) vif->idx); if (is_valid_ether_addr(ndev->dev_addr)) { - wilc_set_mac_address(vif, ndev->dev_addr); + ether_addr_copy(addr, ndev->dev_addr); + wilc_set_mac_address(vif, addr); } else { - u8 addr[ETH_ALEN]; - wilc_get_mac_address(vif, addr); eth_hw_addr_set(ndev, addr); } diff --git a/drivers/net/wireless/microchip/wilc1000/wlan.c b/drivers/net/wireless/microchip/wilc1000/wlan.c index ea81ef120fd1..82566544419a 100644 --- a/drivers/net/wireless/microchip/wilc1000/wlan.c +++ b/drivers/net/wireless/microchip/wilc1000/wlan.c @@ -626,7 +626,6 @@ void chip_wakeup(struct wilc *wilc) u32 clk_status_val = 0, trials = 0; u32 wakeup_reg, wakeup_bit; u32 clk_status_reg, clk_status_bit; - u32 to_host_from_fw_reg, to_host_from_fw_bit; u32 from_host_to_fw_reg, from_host_to_fw_bit; const struct wilc_hif_func *hif_func = wilc->hif_func; @@ -637,8 +636,6 @@ void chip_wakeup(struct wilc *wilc) clk_status_bit = WILC_SDIO_CLK_STATUS_BIT; from_host_to_fw_reg = WILC_SDIO_HOST_TO_FW_REG; from_host_to_fw_bit = WILC_SDIO_HOST_TO_FW_BIT; - to_host_from_fw_reg = WILC_SDIO_FW_TO_HOST_REG; - to_host_from_fw_bit = WILC_SDIO_FW_TO_HOST_BIT; } else { wakeup_reg = WILC_SPI_WAKEUP_REG; wakeup_bit = WILC_SPI_WAKEUP_BIT; @@ -646,8 +643,6 @@ void chip_wakeup(struct wilc *wilc) clk_status_bit = WILC_SPI_CLK_STATUS_BIT; from_host_to_fw_reg = WILC_SPI_HOST_TO_FW_REG; from_host_to_fw_bit = WILC_SPI_HOST_TO_FW_BIT; - to_host_from_fw_reg = WILC_SPI_FW_TO_HOST_REG; - to_host_from_fw_bit = WILC_SPI_FW_TO_HOST_BIT; } /* indicate host wakeup */ diff --git a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c index 9b83c710c9b8..51fe51bb0504 100644 --- a/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c +++ b/drivers/net/wireless/realtek/rtlwifi/rtl8192de/phy.c @@ -899,7 +899,7 @@ static u8 _rtl92c_phy_get_rightchnlplace(u8 chnl) u8 place = chnl; if (chnl > 14) { - for (place = 14; place < sizeof(channel5g); place++) { + for (place = 14; place < ARRAY_SIZE(channel5g); place++) { if (channel5g[place] == chnl) { place++; break; @@ -1366,7 +1366,7 @@ u8 rtl92d_get_rightchnlplace_for_iqk(u8 chnl) u8 place; if (chnl > 14) { - for (place = 14; place < sizeof(channel_all); place++) { + for (place = 14; place < ARRAY_SIZE(channel_all); place++) { if (channel_all[place] == chnl) return place - 13; } @@ -2428,7 +2428,7 @@ static bool _rtl92d_is_legal_5g_channel(struct ieee80211_hw *hw, u8 channel) int i; - for (i = 0; i < sizeof(channel5g); i++) + for (i = 0; i < ARRAY_SIZE(channel5g); i++) if (channel == channel5g[i]) return true; return false; @@ -2692,9 +2692,8 @@ void rtl92d_phy_reset_iqk_result(struct ieee80211_hw *hw) u8 i; rtl_dbg(rtlpriv, COMP_INIT, DBG_LOUD, - "settings regs %d default regs %d\n", - (int)(sizeof(rtlphy->iqk_matrix) / - sizeof(struct iqk_matrix_regs)), + "settings regs %zu default regs %d\n", + ARRAY_SIZE(rtlphy->iqk_matrix), IQK_MATRIX_REG_NUM); /* 0xe94, 0xe9c, 0xea4, 0xeac, 0xeb4, 0xebc, 0xec4, 0xecc */ for (i = 0; i < IQK_MATRIX_SETTINGS_NUM; i++) { @@ -2861,16 +2860,14 @@ u8 rtl92d_phy_sw_chnl(struct ieee80211_hw *hw) case BAND_ON_5G: /* Get first channel error when change between * 5G and 2.4G band. */ - if (channel <= 14) + if (WARN_ONCE(channel <= 14, "rtl8192de: 5G but channel<=14\n")) return 0; - WARN_ONCE((channel <= 14), "rtl8192de: 5G but channel<=14\n"); break; case BAND_ON_2_4G: /* Get first channel error when change between * 5G and 2.4G band. */ - if (channel > 14) + if (WARN_ONCE(channel > 14, "rtl8192de: 2G but channel>14\n")) return 0; - WARN_ONCE((channel > 14), "rtl8192de: 2G but channel>14\n"); break; default: WARN_ONCE(true, "rtl8192de: Invalid WirelessMode(%#x)!!\n", diff --git a/drivers/net/wireless/realtek/rtlwifi/wifi.h b/drivers/net/wireless/realtek/rtlwifi/wifi.h index aa07856411b1..31f9e9e5c680 100644 --- a/drivers/net/wireless/realtek/rtlwifi/wifi.h +++ b/drivers/net/wireless/realtek/rtlwifi/wifi.h @@ -108,7 +108,6 @@ #define CHANNEL_GROUP_IDX_5GM 6 #define CHANNEL_GROUP_IDX_5GH 9 #define CHANNEL_GROUP_MAX_5G 9 -#define CHANNEL_MAX_NUMBER_2G 14 #define AVG_THERMAL_NUM 8 #define AVG_THERMAL_NUM_88E 4 #define AVG_THERMAL_NUM_8723BE 4 diff --git a/drivers/net/wireless/realtek/rtw88/debug.c b/drivers/net/wireless/realtek/rtw88/debug.c index 682b23502e6e..37cf1439a8a0 100644 --- a/drivers/net/wireless/realtek/rtw88/debug.c +++ b/drivers/net/wireless/realtek/rtw88/debug.c @@ -904,6 +904,39 @@ static int rtw_debugfs_get_fw_crash(struct seq_file *m, void *v) return 0; } +static ssize_t rtw_debugfs_set_force_lowest_basic_rate(struct file *filp, + const char __user *buffer, + size_t count, loff_t *loff) +{ + struct seq_file *seqpriv = (struct seq_file *)filp->private_data; + struct rtw_debugfs_priv *debugfs_priv = seqpriv->private; + struct rtw_dev *rtwdev = debugfs_priv->rtwdev; + bool input; + int err; + + err = kstrtobool_from_user(buffer, count, &input); + if (err) + return err; + + if (input) + set_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags); + else + clear_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags); + + return count; +} + +static int rtw_debugfs_get_force_lowest_basic_rate(struct seq_file *m, void *v) +{ + struct rtw_debugfs_priv *debugfs_priv = m->private; + struct rtw_dev *rtwdev = debugfs_priv->rtwdev; + + seq_printf(m, "force lowest basic rate: %d\n", + test_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags)); + + return 0; +} + static ssize_t rtw_debugfs_set_dm_cap(struct file *filp, const char __user *buffer, size_t count, loff_t *loff) @@ -1094,6 +1127,11 @@ static struct rtw_debugfs_priv rtw_debug_priv_fw_crash = { .cb_read = rtw_debugfs_get_fw_crash, }; +static struct rtw_debugfs_priv rtw_debug_priv_force_lowest_basic_rate = { + .cb_write = rtw_debugfs_set_force_lowest_basic_rate, + .cb_read = rtw_debugfs_get_force_lowest_basic_rate, +}; + static struct rtw_debugfs_priv rtw_debug_priv_dm_cap = { .cb_write = rtw_debugfs_set_dm_cap, .cb_read = rtw_debugfs_get_dm_cap, @@ -1174,6 +1212,7 @@ void rtw_debugfs_init(struct rtw_dev *rtwdev) rtw_debugfs_add_r(tx_pwr_tbl); rtw_debugfs_add_rw(edcca_enable); rtw_debugfs_add_rw(fw_crash); + rtw_debugfs_add_rw(force_lowest_basic_rate); rtw_debugfs_add_rw(dm_cap); } diff --git a/drivers/net/wireless/realtek/rtw88/main.h b/drivers/net/wireless/realtek/rtw88/main.h index bbdd535b64e7..c37f1d508882 100644 --- a/drivers/net/wireless/realtek/rtw88/main.h +++ b/drivers/net/wireless/realtek/rtw88/main.h @@ -364,6 +364,7 @@ enum rtw_flags { RTW_FLAG_WOWLAN, RTW_FLAG_RESTARTING, RTW_FLAG_RESTART_TRIGGERING, + RTW_FLAG_FORCE_LOWEST_RATE, NUM_OF_RTW_FLAGS, }; diff --git a/drivers/net/wireless/realtek/rtw88/pci.c b/drivers/net/wireless/realtek/rtw88/pci.c index a7a6ebfaa203..3b367c9085eb 100644 --- a/drivers/net/wireless/realtek/rtw88/pci.c +++ b/drivers/net/wireless/realtek/rtw88/pci.c @@ -1738,6 +1738,15 @@ static const struct dmi_system_id rtw88_pci_quirks[] = { }, .driver_data = (void *)BIT(QUIRK_DIS_PCI_CAP_ASPM), }, + { + .callback = disable_pci_caps, + .ident = "HP HP 250 G7 Notebook PC", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "HP"), + DMI_MATCH(DMI_PRODUCT_NAME, "HP 250 G7 Notebook PC"), + }, + .driver_data = (void *)BIT(QUIRK_DIS_PCI_CAP_ASPM), + }, {} }; diff --git a/drivers/net/wireless/realtek/rtw88/tx.c b/drivers/net/wireless/realtek/rtw88/tx.c index 3a101aa139ed..f9332fa463ca 100644 --- a/drivers/net/wireless/realtek/rtw88/tx.c +++ b/drivers/net/wireless/realtek/rtw88/tx.c @@ -233,17 +233,34 @@ void rtw_tx_report_handle(struct rtw_dev *rtwdev, struct sk_buff *skb, int src) spin_unlock_irqrestore(&tx_report->q_lock, flags); } +static u8 rtw_get_mgmt_rate(struct rtw_dev *rtwdev, struct sk_buff *skb, + u8 lowest_rate, bool ignore_rate) +{ + struct ieee80211_tx_info *tx_info = IEEE80211_SKB_CB(skb); + struct ieee80211_vif *vif = tx_info->control.vif; + bool force_lowest = test_bit(RTW_FLAG_FORCE_LOWEST_RATE, rtwdev->flags); + + if (!vif || !vif->bss_conf.basic_rates || ignore_rate || force_lowest) + return lowest_rate; + + return __ffs(vif->bss_conf.basic_rates) + lowest_rate; +} + static void rtw_tx_pkt_info_update_rate(struct rtw_dev *rtwdev, struct rtw_tx_pkt_info *pkt_info, - struct sk_buff *skb) + struct sk_buff *skb, + bool ignore_rate) { if (rtwdev->hal.current_band_type == RTW_BAND_2G) { pkt_info->rate_id = RTW_RATEID_B_20M; - pkt_info->rate = DESC_RATE1M; + pkt_info->rate = rtw_get_mgmt_rate(rtwdev, skb, DESC_RATE1M, + ignore_rate); } else { pkt_info->rate_id = RTW_RATEID_G; - pkt_info->rate = DESC_RATE6M; + pkt_info->rate = rtw_get_mgmt_rate(rtwdev, skb, DESC_RATE6M, + ignore_rate); } + pkt_info->use_rate = true; pkt_info->dis_rate_fallback = true; } @@ -280,7 +297,7 @@ static void rtw_tx_mgmt_pkt_info_update(struct rtw_dev *rtwdev, struct ieee80211_sta *sta, struct sk_buff *skb) { - rtw_tx_pkt_info_update_rate(rtwdev, pkt_info, skb); + rtw_tx_pkt_info_update_rate(rtwdev, pkt_info, skb, false); pkt_info->dis_qselseq = true; pkt_info->en_hwseq = true; pkt_info->hw_ssn_sel = 0; @@ -404,7 +421,7 @@ void rtw_tx_rsvd_page_pkt_info_update(struct rtw_dev *rtwdev, if (type != RSVD_BEACON && type != RSVD_DUMMY) pkt_info->qsel = TX_DESC_QSEL_MGMT; - rtw_tx_pkt_info_update_rate(rtwdev, pkt_info, skb); + rtw_tx_pkt_info_update_rate(rtwdev, pkt_info, skb, true); bmc = is_broadcast_ether_addr(hdr->addr1) || is_multicast_ether_addr(hdr->addr1); diff --git a/drivers/net/wireless/realtek/rtw89/core.h b/drivers/net/wireless/realtek/rtw89/core.h index c2885e4dd882..3729abda04f9 100644 --- a/drivers/net/wireless/realtek/rtw89/core.h +++ b/drivers/net/wireless/realtek/rtw89/core.h @@ -411,12 +411,13 @@ enum rtw89_regulation_type { RTW89_NA = 4, RTW89_IC = 5, RTW89_KCC = 6, - RTW89_NCC = 7, - RTW89_CHILE = 8, - RTW89_ACMA = 9, - RTW89_MEXICO = 10, + RTW89_ACMA = 7, + RTW89_NCC = 8, + RTW89_MEXICO = 9, + RTW89_CHILE = 10, RTW89_UKRAINE = 11, RTW89_CN = 12, + RTW89_QATAR = 13, RTW89_REGD_NUM, }; diff --git a/drivers/net/wireless/realtek/rtw89/debug.c b/drivers/net/wireless/realtek/rtw89/debug.c index 29eb188c888c..1e85808aaf4b 100644 --- a/drivers/net/wireless/realtek/rtw89/debug.c +++ b/drivers/net/wireless/realtek/rtw89/debug.c @@ -723,6 +723,7 @@ rtw89_debug_priv_mac_mem_dump_select(struct file *filp, } static const u32 mac_mem_base_addr_table[RTW89_MAC_MEM_MAX] = { + [RTW89_MAC_MEM_AXIDMA] = AXIDMA_BASE_ADDR, [RTW89_MAC_MEM_SHARED_BUF] = SHARED_BUF_BASE_ADDR, [RTW89_MAC_MEM_DMAC_TBL] = DMAC_TBL_BASE_ADDR, [RTW89_MAC_MEM_SHCUT_MACHDR] = SHCUT_MACHDR_BASE_ADDR, @@ -735,6 +736,10 @@ static const u32 mac_mem_base_addr_table[RTW89_MAC_MEM_MAX] = { [RTW89_MAC_MEM_BA_CAM] = BA_CAM_BASE_ADDR, [RTW89_MAC_MEM_BCN_IE_CAM0] = BCN_IE_CAM0_BASE_ADDR, [RTW89_MAC_MEM_BCN_IE_CAM1] = BCN_IE_CAM1_BASE_ADDR, + [RTW89_MAC_MEM_TXD_FIFO_0] = TXD_FIFO_0_BASE_ADDR, + [RTW89_MAC_MEM_TXD_FIFO_1] = TXD_FIFO_1_BASE_ADDR, + [RTW89_MAC_MEM_TXDATA_FIFO_0] = TXDATA_FIFO_0_BASE_ADDR, + [RTW89_MAC_MEM_TXDATA_FIFO_1] = TXDATA_FIFO_1_BASE_ADDR, }; static void rtw89_debug_dump_mac_mem(struct seq_file *m, @@ -814,7 +819,7 @@ rtw89_debug_priv_mac_dbg_port_dump_select(struct file *filp, return -EINVAL; } - enable = set == 0 ? false : true; + enable = set != 0; switch (sel) { case 0: debugfs_priv->dbgpkg_en.ss_dbg = enable; diff --git a/drivers/net/wireless/realtek/rtw89/mac.c b/drivers/net/wireless/realtek/rtw89/mac.c index afcd07ab1de7..f8389e849c67 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.c +++ b/drivers/net/wireless/realtek/rtw89/mac.c @@ -1093,7 +1093,6 @@ static int cmac_func_en(struct rtw89_dev *rtwdev, u8 mac_idx, bool en) static int dmac_func_en(struct rtw89_dev *rtwdev) { u32 val32; - u32 ret = 0; val32 = (B_AX_MAC_FUNC_EN | B_AX_DMAC_FUNC_EN | B_AX_MAC_SEC_EN | B_AX_DISPATCHER_EN | B_AX_DLE_CPUIO_EN | B_AX_PKT_IN_EN | @@ -1107,7 +1106,7 @@ static int dmac_func_en(struct rtw89_dev *rtwdev) B_AX_WD_RLS_CLK_EN); rtw89_write32(rtwdev, R_AX_DMAC_CLK_EN, val32); - return ret; + return 0; } static int chip_func_en(struct rtw89_dev *rtwdev) @@ -3695,7 +3694,7 @@ void _rtw89_mac_bf_monitor_track(struct rtw89_dev *rtwdev) { struct rtw89_traffic_stats *stats = &rtwdev->stats; struct rtw89_vif *rtwvif; - bool en = stats->tx_tfc_lv > stats->rx_tfc_lv ? false : true; + bool en = stats->tx_tfc_lv <= stats->rx_tfc_lv; bool old = test_bit(RTW89_FLAG_BFEE_EN, rtwdev->flags); if (en == old) diff --git a/drivers/net/wireless/realtek/rtw89/mac.h b/drivers/net/wireless/realtek/rtw89/mac.h index 6f3db8a2a9c2..94cd29bd83d7 100644 --- a/drivers/net/wireless/realtek/rtw89/mac.h +++ b/drivers/net/wireless/realtek/rtw89/mac.h @@ -227,6 +227,7 @@ enum rtw89_mac_dbg_port_sel { /* SRAM mem dump */ #define R_AX_INDIR_ACCESS_ENTRY 0x40000 +#define AXIDMA_BASE_ADDR 0x18006000 #define STA_SCHED_BASE_ADDR 0x18808000 #define RXPLD_FLTR_CAM_BASE_ADDR 0x18813000 #define SECURITY_CAM_BASE_ADDR 0x18814000 @@ -240,10 +241,15 @@ enum rtw89_mac_dbg_port_sel { #define DMAC_TBL_BASE_ADDR 0x18800000 #define SHCUT_MACHDR_BASE_ADDR 0x18800800 #define BCN_IE_CAM1_BASE_ADDR 0x188A0000 +#define TXD_FIFO_0_BASE_ADDR 0x18856200 +#define TXD_FIFO_1_BASE_ADDR 0x188A1080 +#define TXDATA_FIFO_0_BASE_ADDR 0x18856000 +#define TXDATA_FIFO_1_BASE_ADDR 0x188A1000 #define CCTL_INFO_SIZE 32 enum rtw89_mac_mem_sel { + RTW89_MAC_MEM_AXIDMA, RTW89_MAC_MEM_SHARED_BUF, RTW89_MAC_MEM_DMAC_TBL, RTW89_MAC_MEM_SHCUT_MACHDR, @@ -256,6 +262,10 @@ enum rtw89_mac_mem_sel { RTW89_MAC_MEM_BA_CAM, RTW89_MAC_MEM_BCN_IE_CAM0, RTW89_MAC_MEM_BCN_IE_CAM1, + RTW89_MAC_MEM_TXD_FIFO_0, + RTW89_MAC_MEM_TXD_FIFO_1, + RTW89_MAC_MEM_TXDATA_FIFO_0, + RTW89_MAC_MEM_TXDATA_FIFO_1, /* keep last */ RTW89_MAC_MEM_LAST, diff --git a/drivers/net/wireless/realtek/rtw89/phy.c b/drivers/net/wireless/realtek/rtw89/phy.c index ab134856baac..312d9a07599d 100644 --- a/drivers/net/wireless/realtek/rtw89/phy.c +++ b/drivers/net/wireless/realtek/rtw89/phy.c @@ -654,6 +654,12 @@ rtw89_phy_cofig_rf_reg_store(struct rtw89_dev *rtwdev, u16 idx = info->curr_idx % RTW89_H2C_RF_PAGE_SIZE; u8 page = info->curr_idx / RTW89_H2C_RF_PAGE_SIZE; + if (page >= RTW89_H2C_RF_PAGE_NUM) { + rtw89_warn(rtwdev, "RF parameters exceed size. path=%d, idx=%d", + rf_path, info->curr_idx); + return; + } + info->rtw89_phy_config_rf_h2c[page][idx] = cpu_to_le32((reg->addr << 20) | reg->data); info->curr_idx++; @@ -662,30 +668,29 @@ rtw89_phy_cofig_rf_reg_store(struct rtw89_dev *rtwdev, static int rtw89_phy_config_rf_reg_fw(struct rtw89_dev *rtwdev, struct rtw89_fw_h2c_rf_reg_info *info) { - u16 page = info->curr_idx / RTW89_H2C_RF_PAGE_SIZE; - u16 len = (info->curr_idx % RTW89_H2C_RF_PAGE_SIZE) * 4; + u16 remain = info->curr_idx; + u16 len = 0; u8 i; int ret = 0; - if (page > RTW89_H2C_RF_PAGE_NUM) { + if (remain > RTW89_H2C_RF_PAGE_NUM * RTW89_H2C_RF_PAGE_SIZE) { rtw89_warn(rtwdev, - "rf reg h2c total page num %d larger than %d (RTW89_H2C_RF_PAGE_NUM)\n", - page, RTW89_H2C_RF_PAGE_NUM); - return -EINVAL; + "rf reg h2c total len %d larger than %d\n", + remain, RTW89_H2C_RF_PAGE_NUM * RTW89_H2C_RF_PAGE_SIZE); + ret = -EINVAL; + goto out; } - for (i = 0; i < page; i++) { - ret = rtw89_fw_h2c_rf_reg(rtwdev, info, - RTW89_H2C_RF_PAGE_SIZE * 4, i); + for (i = 0; i < RTW89_H2C_RF_PAGE_NUM && remain; i++, remain -= len) { + len = remain > RTW89_H2C_RF_PAGE_SIZE ? RTW89_H2C_RF_PAGE_SIZE : remain; + ret = rtw89_fw_h2c_rf_reg(rtwdev, info, len * 4, i); if (ret) - return ret; + goto out; } - ret = rtw89_fw_h2c_rf_reg(rtwdev, info, len, i); - if (ret) - return ret; +out: info->curr_idx = 0; - return 0; + return ret; } static void rtw89_phy_config_rf_reg(struct rtw89_dev *rtwdev, @@ -1099,9 +1104,15 @@ s8 rtw89_phy_read_txpwr_limit(struct rtw89_dev *rtwdev, switch (band) { case RTW89_BAND_2G: lmt = (*chip->txpwr_lmt_2g)[bw][ntx][rs][bf][regd][ch_idx]; + if (!lmt) + lmt = (*chip->txpwr_lmt_2g)[bw][ntx][rs][bf] + [RTW89_WW][ch_idx]; break; case RTW89_BAND_5G: lmt = (*chip->txpwr_lmt_5g)[bw][ntx][rs][bf][regd][ch_idx]; + if (!lmt) + lmt = (*chip->txpwr_lmt_5g)[bw][ntx][rs][bf] + [RTW89_WW][ch_idx]; break; default: rtw89_warn(rtwdev, "unknown band type: %d\n", band); @@ -1224,9 +1235,15 @@ static s8 rtw89_phy_read_txpwr_limit_ru(struct rtw89_dev *rtwdev, switch (band) { case RTW89_BAND_2G: lmt_ru = (*chip->txpwr_lmt_ru_2g)[ru][ntx][regd][ch_idx]; + if (!lmt_ru) + lmt_ru = (*chip->txpwr_lmt_ru_2g)[ru][ntx] + [RTW89_WW][ch_idx]; break; case RTW89_BAND_5G: lmt_ru = (*chip->txpwr_lmt_ru_5g)[ru][ntx][regd][ch_idx]; + if (!lmt_ru) + lmt_ru = (*chip->txpwr_lmt_ru_5g)[ru][ntx] + [RTW89_WW][ch_idx]; break; default: rtw89_warn(rtwdev, "unknown band type: %d\n", band); @@ -1767,7 +1784,7 @@ static void rtw89_phy_cfo_dm(struct rtw89_dev *rtwdev) } rtw89_phy_cfo_crystal_cap_adjust(rtwdev, new_cfo); cfo->cfo_avg_pre = new_cfo; - x_cap_update = cfo->crystal_cap == pre_x_cap ? false : true; + x_cap_update = cfo->crystal_cap != pre_x_cap; rtw89_debug(rtwdev, RTW89_DBG_CFO, "Xcap_up=%d\n", x_cap_update); rtw89_debug(rtwdev, RTW89_DBG_CFO, "Xcap: D:%x C:%x->%x, ofst=%d\n", cfo->def_x_cap, pre_x_cap, cfo->crystal_cap, diff --git a/drivers/net/wireless/realtek/rtw89/regd.c b/drivers/net/wireless/realtek/rtw89/regd.c index f00b94ecfff4..4c37e590e43c 100644 --- a/drivers/net/wireless/realtek/rtw89/regd.c +++ b/drivers/net/wireless/realtek/rtw89/regd.c @@ -15,243 +15,244 @@ static const struct rtw89_regulatory rtw89_ww_regd = COUNTRY_REGD("00", RTW89_WW, RTW89_WW); static const struct rtw89_regulatory rtw89_regd_map[] = { - COUNTRY_REGD("AR", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("BO", RTW89_WW, RTW89_FCC), + COUNTRY_REGD("AR", RTW89_MEXICO, RTW89_MEXICO), + COUNTRY_REGD("BO", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("BR", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("CL", RTW89_WW, RTW89_CHILE), + COUNTRY_REGD("CL", RTW89_CHILE, RTW89_CHILE), COUNTRY_REGD("CO", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("CR", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("EC", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("SV", RTW89_WW, RTW89_FCC), + COUNTRY_REGD("SV", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("GT", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("HN", RTW89_WW, RTW89_FCC), - COUNTRY_REGD("MX", RTW89_FCC, RTW89_MEXICO), + COUNTRY_REGD("HN", RTW89_FCC, RTW89_FCC), + COUNTRY_REGD("MX", RTW89_MEXICO, RTW89_MEXICO), COUNTRY_REGD("NI", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("PA", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("PY", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("PE", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("US", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("UY", RTW89_WW, RTW89_FCC), - COUNTRY_REGD("VE", RTW89_WW, RTW89_FCC), + COUNTRY_REGD("UY", RTW89_FCC, RTW89_FCC), + COUNTRY_REGD("VE", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("PR", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("DO", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("AT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CY", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("DK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("EE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("FI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("FR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("DE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("HU", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IS", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LV", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LU", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MC", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NL", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("PL", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("PT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ES", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GB", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AL", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("HR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("EG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IQ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IL", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("JO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LB", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LS", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("OM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("QA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("RO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("RU", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("RS", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ME", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ZA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("UA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("YE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ZW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BD", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("HK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IN", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("AT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CY", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("DK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("EE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("FI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("FR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("DE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("HU", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IS", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LV", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LU", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MC", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NL", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PL", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ES", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GB", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AL", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("HR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("EG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IQ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IL", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("JO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LB", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LS", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("OM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("QA", RTW89_QATAR, RTW89_QATAR), + COUNTRY_REGD("RO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("RU", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SN", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("RS", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ME", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ZA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("UA", RTW89_UKRAINE, RTW89_UKRAINE), + COUNTRY_REGD("AE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("YE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ZW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BD", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CN", RTW89_CN, RTW89_CN), + COUNTRY_REGD("HK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IN", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("ID", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("KR", RTW89_KCC, RTW89_KCC), - COUNTRY_REGD("MY", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("PK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("PH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LK", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("MY", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LK", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("TW", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("TH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("VN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AU", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("NZ", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("PG", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("TH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("VN", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AU", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("NZ", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("PG", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("CA", RTW89_IC, RTW89_IC), COUNTRY_REGD("JP", RTW89_MKK, RTW89_MKK), - COUNTRY_REGD("JM", RTW89_WW, RTW89_FCC), + COUNTRY_REGD("JM", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("AN", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("TT", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("TN", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("TN", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("AF", RTW89_ETSI, RTW89_ETSI), - COUNTRY_REGD("DZ", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("DZ", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("AS", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("AD", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("AQ", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("AD", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("AQ", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("AG", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("AM", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("AM", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("AW", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("BS", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("BB", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("BY", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("BY", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("BZ", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("BJ", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("BJ", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("BM", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("BT", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BV", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IO", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("BT", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BV", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IO", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("VG", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("BN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("BI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CV", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("BN", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("BI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CV", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("KY", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("CF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TD", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CX", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("CC", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CD", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("CK", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("CF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TD", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CX", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("CC", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CD", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("CK", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("CI", RTW89_ETSI, RTW89_ETSI), - COUNTRY_REGD("DJ", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("DJ", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("DM", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("GQ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ER", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ET", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("FK", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("FO", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("GQ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ER", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ET", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("FK", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("FO", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("FJ", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("GF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("PF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GL", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("GF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GL", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("GD", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("GP", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("GP", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("GU", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("GG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GY", RTW89_FCC, RTW89_NCC), + COUNTRY_REGD("GG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GN", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GY", RTW89_NCC, RTW89_NCC), COUNTRY_REGD("HT", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("HM", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("VA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("JE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("KI", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LA", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("LY", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MV", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ML", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("HM", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("VA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("JE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("KI", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LA", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("LY", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MV", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ML", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("MH", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("MQ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MU", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("YT", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("MQ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MU", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("YT", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("FM", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("MD", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MN", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("MS", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NR", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NP", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NC", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("NU", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("NF", RTW89_WW, RTW89_ACMA), + COUNTRY_REGD("MD", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MN", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("MS", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NP", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NC", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("NU", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("NF", RTW89_ACMA, RTW89_ACMA), COUNTRY_REGD("MP", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("PW", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("RE", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("RW", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SH", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("RE", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("RW", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SH", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("KN", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("LC", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("MF", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("SX", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("PM", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("PM", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("VC", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("WS", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("SM", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("SM", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("ST", RTW89_FCC, RTW89_FCC), COUNTRY_REGD("SC", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("SL", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SB", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("GS", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("SL", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SB", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("GS", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("SR", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("SJ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("SZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TJ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TG", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TK", RTW89_WW, RTW89_ACMA), - COUNTRY_REGD("TO", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("TC", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("SJ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("SZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TJ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TG", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TK", RTW89_ACMA, RTW89_ACMA), + COUNTRY_REGD("TO", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("TC", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("TV", RTW89_ETSI, RTW89_NA), - COUNTRY_REGD("UG", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("UG", RTW89_ETSI, RTW89_ETSI), COUNTRY_REGD("VI", RTW89_FCC, RTW89_FCC), - COUNTRY_REGD("UZ", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("VU", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("WF", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("EH", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("ZM", RTW89_WW, RTW89_ETSI), - COUNTRY_REGD("IR", RTW89_WW, RTW89_ETSI), + COUNTRY_REGD("UZ", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("VU", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("WF", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("EH", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("ZM", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("IR", RTW89_ETSI, RTW89_ETSI), + COUNTRY_REGD("PS", RTW89_ETSI, RTW89_ETSI), }; static const struct rtw89_regulatory *rtw89_regd_find_reg_by_name(char *alpha2) diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a.c b/drivers/net/wireless/realtek/rtw89/rtw8852a.c index 5c6ffca3a324..9e25e53f6c4a 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a.c @@ -1053,10 +1053,10 @@ static void rtw8852a_set_channel_bb(struct rtw89_dev *rtwdev, struct rtw89_channel_params *param, enum rtw89_phy_idx phy_idx) { - bool cck_en = param->center_chan > 14 ? false : true; + bool cck_en = param->center_chan <= 14; u8 pri_ch_idx = param->pri_ch_idx; - if (param->center_chan <= 14) + if (cck_en) rtw8852a_ctrl_sco_cck(rtwdev, param->center_chan, param->primary_chan, param->bandwidth); diff --git a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c index 3a4fe7207420..253b5f8fc4f9 100644 --- a/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c +++ b/drivers/net/wireless/realtek/rtw89/rtw8852a_table.c @@ -43384,5248 +43384,6991 @@ static const u8 _txpwr_track_delta_swingidx_2g_cck_a_p[] = { const s8 rtw89_8852a_txpwr_lmt_2g[RTW89_2G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = { - [0][0][0][0][0][0] = 56, - [0][0][0][0][0][1] = 56, - [0][0][0][0][0][2] = 56, - [0][0][0][0][0][3] = 56, - [0][0][0][0][0][4] = 56, - [0][0][0][0][0][5] = 56, - [0][0][0][0][0][6] = 56, - [0][0][0][0][0][7] = 56, - [0][0][0][0][0][8] = 56, - [0][0][0][0][0][9] = 56, - [0][0][0][0][0][10] = 56, - [0][0][0][0][0][11] = 56, - [0][0][0][0][0][12] = 48, - [0][0][0][0][0][13] = 76, - [0][1][0][0][0][0] = 44, - [0][1][0][0][0][1] = 44, - [0][1][0][0][0][2] = 44, - [0][1][0][0][0][3] = 44, - [0][1][0][0][0][4] = 44, - [0][1][0][0][0][5] = 44, - [0][1][0][0][0][6] = 44, - [0][1][0][0][0][7] = 44, - [0][1][0][0][0][8] = 44, - [0][1][0][0][0][9] = 44, - [0][1][0][0][0][10] = 44, - [0][1][0][0][0][11] = 44, - [0][1][0][0][0][12] = 38, - [0][1][0][0][0][13] = 64, - [1][0][0][0][0][0] = 0, - [1][0][0][0][0][1] = 0, - [1][0][0][0][0][2] = 58, - [1][0][0][0][0][3] = 58, - [1][0][0][0][0][4] = 58, - [1][0][0][0][0][5] = 58, - [1][0][0][0][0][6] = 46, - [1][0][0][0][0][7] = 46, - [1][0][0][0][0][8] = 46, - [1][0][0][0][0][9] = 32, - [1][0][0][0][0][10] = 32, - [1][0][0][0][0][11] = 0, - [1][0][0][0][0][12] = 0, - [1][0][0][0][0][13] = 0, - [1][1][0][0][0][0] = 0, - [1][1][0][0][0][1] = 0, - [1][1][0][0][0][2] = 46, - [1][1][0][0][0][3] = 46, - [1][1][0][0][0][4] = 46, - [1][1][0][0][0][5] = 46, - [1][1][0][0][0][6] = 46, - [1][1][0][0][0][7] = 46, - [1][1][0][0][0][8] = 46, - [1][1][0][0][0][9] = 24, - [1][1][0][0][0][10] = 24, - [1][1][0][0][0][11] = 0, - [1][1][0][0][0][12] = 0, - [1][1][0][0][0][13] = 0, - [0][0][1][0][0][0] = 58, - [0][0][1][0][0][1] = 58, - [0][0][1][0][0][2] = 58, - [0][0][1][0][0][3] = 58, - [0][0][1][0][0][4] = 58, - [0][0][1][0][0][5] = 58, - [0][0][1][0][0][6] = 58, - [0][0][1][0][0][7] = 58, - [0][0][1][0][0][8] = 58, - [0][0][1][0][0][9] = 58, - [0][0][1][0][0][10] = 58, - [0][0][1][0][0][11] = 56, - [0][0][1][0][0][12] = 52, - [0][0][1][0][0][13] = 0, - [0][1][1][0][0][0] = 46, - [0][1][1][0][0][1] = 46, - [0][1][1][0][0][2] = 46, - [0][1][1][0][0][3] = 46, - [0][1][1][0][0][4] = 46, - [0][1][1][0][0][5] = 46, - [0][1][1][0][0][6] = 46, - [0][1][1][0][0][7] = 46, - [0][1][1][0][0][8] = 46, - [0][1][1][0][0][9] = 46, - [0][1][1][0][0][10] = 46, - [0][1][1][0][0][11] = 42, - [0][1][1][0][0][12] = 40, - [0][1][1][0][0][13] = 0, - [0][0][2][0][0][0] = 58, - [0][0][2][0][0][1] = 58, - [0][0][2][0][0][2] = 58, - [0][0][2][0][0][3] = 58, - [0][0][2][0][0][4] = 58, - [0][0][2][0][0][5] = 58, - [0][0][2][0][0][6] = 58, - [0][0][2][0][0][7] = 58, - [0][0][2][0][0][8] = 58, - [0][0][2][0][0][9] = 58, - [0][0][2][0][0][10] = 58, - [0][0][2][0][0][11] = 54, - [0][0][2][0][0][12] = 50, - [0][0][2][0][0][13] = 0, - [0][1][2][0][0][0] = 46, - [0][1][2][0][0][1] = 46, - [0][1][2][0][0][2] = 46, - [0][1][2][0][0][3] = 46, - [0][1][2][0][0][4] = 46, - [0][1][2][0][0][5] = 46, - [0][1][2][0][0][6] = 46, - [0][1][2][0][0][7] = 46, - [0][1][2][0][0][8] = 46, - [0][1][2][0][0][9] = 46, - [0][1][2][0][0][10] = 46, - [0][1][2][0][0][11] = 42, - [0][1][2][0][0][12] = 40, - [0][1][2][0][0][13] = 0, - [0][1][2][1][0][0] = 34, - [0][1][2][1][0][1] = 34, - [0][1][2][1][0][2] = 34, - [0][1][2][1][0][3] = 34, - [0][1][2][1][0][4] = 34, - [0][1][2][1][0][5] = 34, - [0][1][2][1][0][6] = 34, - [0][1][2][1][0][7] = 34, - [0][1][2][1][0][8] = 34, - [0][1][2][1][0][9] = 34, - [0][1][2][1][0][10] = 34, - [0][1][2][1][0][11] = 34, - [0][1][2][1][0][12] = 34, - [0][1][2][1][0][13] = 0, - [1][0][2][0][0][0] = 0, - [1][0][2][0][0][1] = 0, - [1][0][2][0][0][2] = 56, - [1][0][2][0][0][3] = 56, - [1][0][2][0][0][4] = 58, - [1][0][2][0][0][5] = 58, - [1][0][2][0][0][6] = 54, - [1][0][2][0][0][7] = 50, - [1][0][2][0][0][8] = 50, - [1][0][2][0][0][9] = 42, - [1][0][2][0][0][10] = 40, - [1][0][2][0][0][11] = 0, - [1][0][2][0][0][12] = 0, - [1][0][2][0][0][13] = 0, - [1][1][2][0][0][0] = 0, - [1][1][2][0][0][1] = 0, - [1][1][2][0][0][2] = 46, - [1][1][2][0][0][3] = 46, - [1][1][2][0][0][4] = 46, - [1][1][2][0][0][5] = 46, - [1][1][2][0][0][6] = 46, - [1][1][2][0][0][7] = 46, - [1][1][2][0][0][8] = 46, - [1][1][2][0][0][9] = 38, - [1][1][2][0][0][10] = 36, - [1][1][2][0][0][11] = 0, - [1][1][2][0][0][12] = 0, - [1][1][2][0][0][13] = 0, - [1][1][2][1][0][0] = 0, - [1][1][2][1][0][1] = 0, - [1][1][2][1][0][2] = 34, - [1][1][2][1][0][3] = 34, - [1][1][2][1][0][4] = 34, - [1][1][2][1][0][5] = 34, - [1][1][2][1][0][6] = 34, - [1][1][2][1][0][7] = 34, - [1][1][2][1][0][8] = 34, - [1][1][2][1][0][9] = 34, - [1][1][2][1][0][10] = 34, - [1][1][2][1][0][11] = 0, - [1][1][2][1][0][12] = 0, - [1][1][2][1][0][13] = 0, - [0][0][0][0][2][0] = 76, - [0][0][0][0][1][0] = 56, - [0][0][0][0][3][0] = 68, - [0][0][0][0][5][0] = 76, - [0][0][0][0][6][0] = 56, - [0][0][0][0][9][0] = 56, - [0][0][0][0][8][0] = 60, - [0][0][0][0][11][0] = 56, - [0][0][0][0][2][1] = 76, - [0][0][0][0][1][1] = 56, - [0][0][0][0][3][1] = 68, - [0][0][0][0][5][1] = 76, - [0][0][0][0][6][1] = 56, - [0][0][0][0][9][1] = 56, - [0][0][0][0][8][1] = 60, - [0][0][0][0][11][1] = 56, - [0][0][0][0][2][2] = 76, - [0][0][0][0][1][2] = 56, - [0][0][0][0][3][2] = 68, - [0][0][0][0][5][2] = 76, - [0][0][0][0][6][2] = 56, - [0][0][0][0][9][2] = 56, - [0][0][0][0][8][2] = 60, - [0][0][0][0][11][2] = 56, - [0][0][0][0][2][3] = 76, - [0][0][0][0][1][3] = 56, - [0][0][0][0][3][3] = 68, - [0][0][0][0][5][3] = 76, - [0][0][0][0][6][3] = 56, - [0][0][0][0][9][3] = 56, - [0][0][0][0][8][3] = 60, - [0][0][0][0][11][3] = 56, - [0][0][0][0][2][4] = 76, - [0][0][0][0][1][4] = 56, - [0][0][0][0][3][4] = 68, - [0][0][0][0][5][4] = 76, - [0][0][0][0][6][4] = 56, - [0][0][0][0][9][4] = 56, - [0][0][0][0][8][4] = 60, - [0][0][0][0][11][4] = 56, - [0][0][0][0][2][5] = 76, - [0][0][0][0][1][5] = 56, - [0][0][0][0][3][5] = 68, - [0][0][0][0][5][5] = 76, - [0][0][0][0][6][5] = 56, - [0][0][0][0][9][5] = 56, - [0][0][0][0][8][5] = 60, - [0][0][0][0][11][5] = 56, - [0][0][0][0][2][6] = 76, - [0][0][0][0][1][6] = 56, - [0][0][0][0][3][6] = 68, - [0][0][0][0][5][6] = 76, - [0][0][0][0][6][6] = 56, - [0][0][0][0][9][6] = 56, - [0][0][0][0][8][6] = 60, - [0][0][0][0][11][6] = 56, - [0][0][0][0][2][7] = 76, - [0][0][0][0][1][7] = 56, - [0][0][0][0][3][7] = 68, - [0][0][0][0][5][7] = 76, - [0][0][0][0][6][7] = 56, - [0][0][0][0][9][7] = 56, - [0][0][0][0][8][7] = 60, - [0][0][0][0][11][7] = 56, - [0][0][0][0][2][8] = 76, - [0][0][0][0][1][8] = 56, - [0][0][0][0][3][8] = 68, - [0][0][0][0][5][8] = 76, - [0][0][0][0][6][8] = 56, - [0][0][0][0][9][8] = 56, - [0][0][0][0][8][8] = 60, - [0][0][0][0][11][8] = 56, - [0][0][0][0][2][9] = 76, - [0][0][0][0][1][9] = 56, - [0][0][0][0][3][9] = 68, - [0][0][0][0][5][9] = 76, - [0][0][0][0][6][9] = 56, - [0][0][0][0][9][9] = 56, - [0][0][0][0][8][9] = 60, - [0][0][0][0][11][9] = 56, - [0][0][0][0][2][10] = 76, - [0][0][0][0][1][10] = 56, - [0][0][0][0][3][10] = 68, - [0][0][0][0][5][10] = 76, - [0][0][0][0][6][10] = 56, - [0][0][0][0][9][10] = 56, - [0][0][0][0][8][10] = 60, - [0][0][0][0][11][10] = 56, - [0][0][0][0][2][11] = 68, - [0][0][0][0][1][11] = 56, - [0][0][0][0][3][11] = 68, - [0][0][0][0][5][11] = 68, - [0][0][0][0][6][11] = 56, - [0][0][0][0][9][11] = 56, - [0][0][0][0][8][11] = 60, - [0][0][0][0][11][11] = 56, - [0][0][0][0][2][12] = 48, - [0][0][0][0][1][12] = 56, - [0][0][0][0][3][12] = 68, - [0][0][0][0][5][12] = 48, - [0][0][0][0][6][12] = 56, - [0][0][0][0][9][12] = 56, - [0][0][0][0][8][12] = 60, - [0][0][0][0][11][12] = 56, - [0][0][0][0][2][13] = 127, - [0][0][0][0][1][13] = 127, - [0][0][0][0][3][13] = 76, - [0][0][0][0][5][13] = 127, - [0][0][0][0][6][13] = 127, - [0][0][0][0][9][13] = 127, - [0][0][0][0][8][13] = 127, - [0][0][0][0][11][13] = 127, - [0][1][0][0][2][0] = 74, - [0][1][0][0][1][0] = 44, - [0][1][0][0][3][0] = 56, - [0][1][0][0][5][0] = 74, - [0][1][0][0][6][0] = 44, - [0][1][0][0][9][0] = 44, - [0][1][0][0][8][0] = 48, - [0][1][0][0][11][0] = 44, - [0][1][0][0][2][1] = 76, - [0][1][0][0][1][1] = 44, - [0][1][0][0][3][1] = 56, - [0][1][0][0][5][1] = 76, - [0][1][0][0][6][1] = 44, - [0][1][0][0][9][1] = 44, - [0][1][0][0][8][1] = 48, - [0][1][0][0][11][1] = 44, - [0][1][0][0][2][2] = 76, - [0][1][0][0][1][2] = 44, - [0][1][0][0][3][2] = 56, - [0][1][0][0][5][2] = 76, - [0][1][0][0][6][2] = 44, - [0][1][0][0][9][2] = 44, - [0][1][0][0][8][2] = 48, - [0][1][0][0][11][2] = 44, - [0][1][0][0][2][3] = 76, - [0][1][0][0][1][3] = 44, - [0][1][0][0][3][3] = 56, - [0][1][0][0][5][3] = 76, - [0][1][0][0][6][3] = 44, - [0][1][0][0][9][3] = 44, - [0][1][0][0][8][3] = 48, - [0][1][0][0][11][3] = 44, - [0][1][0][0][2][4] = 76, - [0][1][0][0][1][4] = 44, - [0][1][0][0][3][4] = 56, - [0][1][0][0][5][4] = 76, - [0][1][0][0][6][4] = 44, - [0][1][0][0][9][4] = 44, - [0][1][0][0][8][4] = 48, - [0][1][0][0][11][4] = 44, - [0][1][0][0][2][5] = 76, - [0][1][0][0][1][5] = 44, - [0][1][0][0][3][5] = 56, - [0][1][0][0][5][5] = 76, - [0][1][0][0][6][5] = 44, - [0][1][0][0][9][5] = 44, - [0][1][0][0][8][5] = 48, - [0][1][0][0][11][5] = 44, - [0][1][0][0][2][6] = 76, - [0][1][0][0][1][6] = 44, - [0][1][0][0][3][6] = 56, - [0][1][0][0][5][6] = 76, - [0][1][0][0][6][6] = 44, - [0][1][0][0][9][6] = 44, - [0][1][0][0][8][6] = 48, - [0][1][0][0][11][6] = 44, - [0][1][0][0][2][7] = 76, - [0][1][0][0][1][7] = 44, - [0][1][0][0][3][7] = 56, - [0][1][0][0][5][7] = 76, - [0][1][0][0][6][7] = 44, - [0][1][0][0][9][7] = 44, - [0][1][0][0][8][7] = 48, - [0][1][0][0][11][7] = 44, - [0][1][0][0][2][8] = 76, - [0][1][0][0][1][8] = 44, - [0][1][0][0][3][8] = 56, - [0][1][0][0][5][8] = 76, - [0][1][0][0][6][8] = 44, - [0][1][0][0][9][8] = 44, - [0][1][0][0][8][8] = 48, - [0][1][0][0][11][8] = 44, - [0][1][0][0][2][9] = 76, - [0][1][0][0][1][9] = 44, - [0][1][0][0][3][9] = 56, - [0][1][0][0][5][9] = 76, - [0][1][0][0][6][9] = 44, - [0][1][0][0][9][9] = 44, - [0][1][0][0][8][9] = 48, - [0][1][0][0][11][9] = 44, - [0][1][0][0][2][10] = 62, - [0][1][0][0][1][10] = 44, - [0][1][0][0][3][10] = 56, - [0][1][0][0][5][10] = 62, - [0][1][0][0][6][10] = 44, - [0][1][0][0][9][10] = 44, - [0][1][0][0][8][10] = 48, - [0][1][0][0][11][10] = 44, - [0][1][0][0][2][11] = 52, - [0][1][0][0][1][11] = 44, - [0][1][0][0][3][11] = 56, - [0][1][0][0][5][11] = 52, - [0][1][0][0][6][11] = 44, - [0][1][0][0][9][11] = 44, - [0][1][0][0][8][11] = 48, - [0][1][0][0][11][11] = 44, - [0][1][0][0][2][12] = 38, - [0][1][0][0][1][12] = 44, - [0][1][0][0][3][12] = 56, - [0][1][0][0][5][12] = 38, - [0][1][0][0][6][12] = 44, - [0][1][0][0][9][12] = 44, - [0][1][0][0][8][12] = 48, - [0][1][0][0][11][12] = 44, - [0][1][0][0][2][13] = 127, - [0][1][0][0][1][13] = 127, - [0][1][0][0][3][13] = 64, - [0][1][0][0][5][13] = 127, - [0][1][0][0][6][13] = 127, - [0][1][0][0][9][13] = 127, - [0][1][0][0][8][13] = 127, - [0][1][0][0][11][13] = 127, - [1][0][0][0][2][0] = 127, - [1][0][0][0][1][0] = 127, - [1][0][0][0][3][0] = 127, - [1][0][0][0][5][0] = 127, - [1][0][0][0][6][0] = 127, - [1][0][0][0][9][0] = 127, - [1][0][0][0][8][0] = 127, - [1][0][0][0][11][0] = 127, - [1][0][0][0][2][1] = 127, - [1][0][0][0][1][1] = 127, - [1][0][0][0][3][1] = 127, - [1][0][0][0][5][1] = 127, - [1][0][0][0][6][1] = 127, - [1][0][0][0][9][1] = 127, - [1][0][0][0][8][1] = 127, - [1][0][0][0][11][1] = 127, - [1][0][0][0][2][2] = 60, - [1][0][0][0][1][2] = 58, - [1][0][0][0][3][2] = 68, - [1][0][0][0][5][2] = 60, - [1][0][0][0][6][2] = 58, - [1][0][0][0][9][2] = 58, - [1][0][0][0][8][2] = 60, - [1][0][0][0][11][2] = 58, - [1][0][0][0][2][3] = 60, - [1][0][0][0][1][3] = 58, - [1][0][0][0][3][3] = 68, - [1][0][0][0][5][3] = 60, - [1][0][0][0][6][3] = 58, - [1][0][0][0][9][3] = 58, - [1][0][0][0][8][3] = 60, - [1][0][0][0][11][3] = 58, - [1][0][0][0][2][4] = 60, - [1][0][0][0][1][4] = 58, - [1][0][0][0][3][4] = 68, - [1][0][0][0][5][4] = 60, - [1][0][0][0][6][4] = 58, - [1][0][0][0][9][4] = 58, - [1][0][0][0][8][4] = 60, - [1][0][0][0][11][4] = 58, - [1][0][0][0][2][5] = 60, - [1][0][0][0][1][5] = 58, - [1][0][0][0][3][5] = 68, - [1][0][0][0][5][5] = 60, - [1][0][0][0][6][5] = 58, - [1][0][0][0][9][5] = 58, - [1][0][0][0][8][5] = 60, - [1][0][0][0][11][5] = 58, - [1][0][0][0][2][6] = 46, - [1][0][0][0][1][6] = 58, - [1][0][0][0][3][6] = 68, - [1][0][0][0][5][6] = 46, - [1][0][0][0][6][6] = 58, - [1][0][0][0][9][6] = 58, - [1][0][0][0][8][6] = 60, - [1][0][0][0][11][6] = 58, - [1][0][0][0][2][7] = 46, - [1][0][0][0][1][7] = 58, - [1][0][0][0][3][7] = 68, - [1][0][0][0][5][7] = 46, - [1][0][0][0][6][7] = 58, - [1][0][0][0][9][7] = 58, - [1][0][0][0][8][7] = 60, - [1][0][0][0][11][7] = 58, - [1][0][0][0][2][8] = 46, - [1][0][0][0][1][8] = 58, - [1][0][0][0][3][8] = 68, - [1][0][0][0][5][8] = 46, - [1][0][0][0][6][8] = 58, - [1][0][0][0][9][8] = 58, - [1][0][0][0][8][8] = 60, - [1][0][0][0][11][8] = 58, - [1][0][0][0][2][9] = 32, - [1][0][0][0][1][9] = 58, - [1][0][0][0][3][9] = 68, - [1][0][0][0][5][9] = 32, - [1][0][0][0][6][9] = 58, - [1][0][0][0][9][9] = 58, - [1][0][0][0][8][9] = 60, - [1][0][0][0][11][9] = 58, - [1][0][0][0][2][10] = 32, - [1][0][0][0][1][10] = 58, - [1][0][0][0][3][10] = 68, - [1][0][0][0][5][10] = 32, - [1][0][0][0][6][10] = 58, - [1][0][0][0][9][10] = 58, - [1][0][0][0][8][10] = 60, - [1][0][0][0][11][10] = 58, - [1][0][0][0][2][11] = 127, - [1][0][0][0][1][11] = 127, - [1][0][0][0][3][11] = 127, - [1][0][0][0][5][11] = 127, - [1][0][0][0][6][11] = 127, - [1][0][0][0][9][11] = 127, - [1][0][0][0][8][11] = 127, - [1][0][0][0][11][11] = 127, - [1][0][0][0][2][12] = 127, - [1][0][0][0][1][12] = 127, - [1][0][0][0][3][12] = 127, - [1][0][0][0][5][12] = 127, - [1][0][0][0][6][12] = 127, - [1][0][0][0][9][12] = 127, - [1][0][0][0][8][12] = 127, - [1][0][0][0][11][12] = 127, - [1][0][0][0][2][13] = 127, - [1][0][0][0][1][13] = 127, - [1][0][0][0][3][13] = 127, - [1][0][0][0][5][13] = 127, - [1][0][0][0][6][13] = 127, - [1][0][0][0][9][13] = 127, - [1][0][0][0][8][13] = 127, - [1][0][0][0][11][13] = 127, - [1][1][0][0][2][0] = 127, - [1][1][0][0][1][0] = 127, - [1][1][0][0][3][0] = 127, - [1][1][0][0][5][0] = 127, - [1][1][0][0][6][0] = 127, - [1][1][0][0][9][0] = 127, - [1][1][0][0][8][0] = 127, - [1][1][0][0][11][0] = 127, - [1][1][0][0][2][1] = 127, - [1][1][0][0][1][1] = 127, - [1][1][0][0][3][1] = 127, - [1][1][0][0][5][1] = 127, - [1][1][0][0][6][1] = 127, - [1][1][0][0][9][1] = 127, - [1][1][0][0][8][1] = 127, - [1][1][0][0][11][1] = 127, - [1][1][0][0][2][2] = 48, - [1][1][0][0][1][2] = 46, - [1][1][0][0][3][2] = 56, - [1][1][0][0][5][2] = 48, - [1][1][0][0][6][2] = 46, - [1][1][0][0][9][2] = 46, - [1][1][0][0][8][2] = 48, - [1][1][0][0][11][2] = 46, - [1][1][0][0][2][3] = 48, - [1][1][0][0][1][3] = 46, - [1][1][0][0][3][3] = 56, - [1][1][0][0][5][3] = 48, - [1][1][0][0][6][3] = 46, - [1][1][0][0][9][3] = 46, - [1][1][0][0][8][3] = 48, - [1][1][0][0][11][3] = 46, - [1][1][0][0][2][4] = 48, - [1][1][0][0][1][4] = 46, - [1][1][0][0][3][4] = 56, - [1][1][0][0][5][4] = 48, - [1][1][0][0][6][4] = 46, - [1][1][0][0][9][4] = 46, - [1][1][0][0][8][4] = 48, - [1][1][0][0][11][4] = 46, - [1][1][0][0][2][5] = 58, - [1][1][0][0][1][5] = 46, - [1][1][0][0][3][5] = 56, - [1][1][0][0][5][5] = 58, - [1][1][0][0][6][5] = 46, - [1][1][0][0][9][5] = 46, - [1][1][0][0][8][5] = 48, - [1][1][0][0][11][5] = 46, - [1][1][0][0][2][6] = 46, - [1][1][0][0][1][6] = 46, - [1][1][0][0][3][6] = 56, - [1][1][0][0][5][6] = 46, - [1][1][0][0][6][6] = 46, - [1][1][0][0][9][6] = 46, - [1][1][0][0][8][6] = 48, - [1][1][0][0][11][6] = 46, - [1][1][0][0][2][7] = 46, - [1][1][0][0][1][7] = 46, - [1][1][0][0][3][7] = 56, - [1][1][0][0][5][7] = 46, - [1][1][0][0][6][7] = 46, - [1][1][0][0][9][7] = 46, - [1][1][0][0][8][7] = 48, - [1][1][0][0][11][7] = 46, - [1][1][0][0][2][8] = 46, - [1][1][0][0][1][8] = 46, - [1][1][0][0][3][8] = 56, - [1][1][0][0][5][8] = 46, - [1][1][0][0][6][8] = 46, - [1][1][0][0][9][8] = 46, - [1][1][0][0][8][8] = 48, - [1][1][0][0][11][8] = 46, - [1][1][0][0][2][9] = 24, - [1][1][0][0][1][9] = 46, - [1][1][0][0][3][9] = 56, - [1][1][0][0][5][9] = 24, - [1][1][0][0][6][9] = 46, - [1][1][0][0][9][9] = 46, - [1][1][0][0][8][9] = 48, - [1][1][0][0][11][9] = 46, - [1][1][0][0][2][10] = 24, - [1][1][0][0][1][10] = 46, - [1][1][0][0][3][10] = 56, - [1][1][0][0][5][10] = 24, - [1][1][0][0][6][10] = 46, - [1][1][0][0][9][10] = 46, - [1][1][0][0][8][10] = 48, - [1][1][0][0][11][10] = 46, - [1][1][0][0][2][11] = 127, - [1][1][0][0][1][11] = 127, - [1][1][0][0][3][11] = 127, - [1][1][0][0][5][11] = 127, - [1][1][0][0][6][11] = 127, - [1][1][0][0][9][11] = 127, - [1][1][0][0][8][11] = 127, - [1][1][0][0][11][11] = 127, - [1][1][0][0][2][12] = 127, - [1][1][0][0][1][12] = 127, - [1][1][0][0][3][12] = 127, - [1][1][0][0][5][12] = 127, - [1][1][0][0][6][12] = 127, - [1][1][0][0][9][12] = 127, - [1][1][0][0][8][12] = 127, - [1][1][0][0][11][12] = 127, - [1][1][0][0][2][13] = 127, - [1][1][0][0][1][13] = 127, - [1][1][0][0][3][13] = 127, - [1][1][0][0][5][13] = 127, - [1][1][0][0][6][13] = 127, - [1][1][0][0][9][13] = 127, - [1][1][0][0][8][13] = 127, - [1][1][0][0][11][13] = 127, - [0][0][1][0][2][0] = 66, - [0][0][1][0][1][0] = 58, - [0][0][1][0][3][0] = 76, - [0][0][1][0][5][0] = 66, - [0][0][1][0][6][0] = 58, - [0][0][1][0][9][0] = 58, - [0][0][1][0][8][0] = 60, - [0][0][1][0][11][0] = 58, - [0][0][1][0][2][1] = 66, - [0][0][1][0][1][1] = 58, - [0][0][1][0][3][1] = 76, - [0][0][1][0][5][1] = 66, - [0][0][1][0][6][1] = 58, - [0][0][1][0][9][1] = 58, - [0][0][1][0][8][1] = 60, - [0][0][1][0][11][1] = 58, - [0][0][1][0][2][2] = 70, - [0][0][1][0][1][2] = 58, - [0][0][1][0][3][2] = 76, - [0][0][1][0][5][2] = 70, - [0][0][1][0][6][2] = 58, - [0][0][1][0][9][2] = 58, - [0][0][1][0][8][2] = 60, - [0][0][1][0][11][2] = 58, - [0][0][1][0][2][3] = 74, - [0][0][1][0][1][3] = 58, - [0][0][1][0][3][3] = 76, - [0][0][1][0][5][3] = 74, - [0][0][1][0][6][3] = 58, - [0][0][1][0][9][3] = 58, - [0][0][1][0][8][3] = 60, - [0][0][1][0][11][3] = 58, - [0][0][1][0][2][4] = 78, - [0][0][1][0][1][4] = 58, - [0][0][1][0][3][4] = 76, - [0][0][1][0][5][4] = 78, - [0][0][1][0][6][4] = 58, - [0][0][1][0][9][4] = 58, - [0][0][1][0][8][4] = 60, - [0][0][1][0][11][4] = 58, - [0][0][1][0][2][5] = 78, - [0][0][1][0][1][5] = 58, - [0][0][1][0][3][5] = 76, - [0][0][1][0][5][5] = 78, - [0][0][1][0][6][5] = 58, - [0][0][1][0][9][5] = 58, - [0][0][1][0][8][5] = 60, - [0][0][1][0][11][5] = 58, - [0][0][1][0][2][6] = 78, - [0][0][1][0][1][6] = 58, - [0][0][1][0][3][6] = 76, - [0][0][1][0][5][6] = 78, - [0][0][1][0][6][6] = 58, - [0][0][1][0][9][6] = 58, - [0][0][1][0][8][6] = 60, - [0][0][1][0][11][6] = 58, - [0][0][1][0][2][7] = 74, - [0][0][1][0][1][7] = 58, - [0][0][1][0][3][7] = 76, - [0][0][1][0][5][7] = 74, - [0][0][1][0][6][7] = 58, - [0][0][1][0][9][7] = 58, - [0][0][1][0][8][7] = 60, - [0][0][1][0][11][7] = 58, - [0][0][1][0][2][8] = 70, - [0][0][1][0][1][8] = 58, - [0][0][1][0][3][8] = 76, - [0][0][1][0][5][8] = 70, - [0][0][1][0][6][8] = 58, - [0][0][1][0][9][8] = 58, - [0][0][1][0][8][8] = 60, - [0][0][1][0][11][8] = 58, - [0][0][1][0][2][9] = 66, - [0][0][1][0][1][9] = 58, - [0][0][1][0][3][9] = 76, - [0][0][1][0][5][9] = 66, - [0][0][1][0][6][9] = 58, - [0][0][1][0][9][9] = 58, - [0][0][1][0][8][9] = 60, - [0][0][1][0][11][9] = 58, - [0][0][1][0][2][10] = 66, - [0][0][1][0][1][10] = 58, - [0][0][1][0][3][10] = 76, - [0][0][1][0][5][10] = 66, - [0][0][1][0][6][10] = 58, - [0][0][1][0][9][10] = 58, - [0][0][1][0][8][10] = 60, - [0][0][1][0][11][10] = 58, - [0][0][1][0][2][11] = 56, - [0][0][1][0][1][11] = 58, - [0][0][1][0][3][11] = 76, - [0][0][1][0][5][11] = 56, - [0][0][1][0][6][11] = 58, - [0][0][1][0][9][11] = 58, - [0][0][1][0][8][11] = 60, - [0][0][1][0][11][11] = 58, - [0][0][1][0][2][12] = 52, - [0][0][1][0][1][12] = 58, - [0][0][1][0][3][12] = 76, - [0][0][1][0][5][12] = 52, - [0][0][1][0][6][12] = 58, - [0][0][1][0][9][12] = 58, - [0][0][1][0][8][12] = 60, - [0][0][1][0][11][12] = 58, - [0][0][1][0][2][13] = 127, - [0][0][1][0][1][13] = 127, - [0][0][1][0][3][13] = 127, - [0][0][1][0][5][13] = 127, - [0][0][1][0][6][13] = 127, - [0][0][1][0][9][13] = 127, - [0][0][1][0][8][13] = 127, - [0][0][1][0][11][13] = 127, - [0][1][1][0][2][0] = 62, - [0][1][1][0][1][0] = 46, - [0][1][1][0][3][0] = 64, - [0][1][1][0][5][0] = 62, - [0][1][1][0][6][0] = 46, - [0][1][1][0][9][0] = 46, - [0][1][1][0][8][0] = 48, - [0][1][1][0][11][0] = 46, - [0][1][1][0][2][1] = 62, - [0][1][1][0][1][1] = 46, - [0][1][1][0][3][1] = 64, - [0][1][1][0][5][1] = 62, - [0][1][1][0][6][1] = 46, - [0][1][1][0][9][1] = 46, - [0][1][1][0][8][1] = 48, - [0][1][1][0][11][1] = 46, - [0][1][1][0][2][2] = 66, - [0][1][1][0][1][2] = 46, - [0][1][1][0][3][2] = 64, - [0][1][1][0][5][2] = 66, - [0][1][1][0][6][2] = 46, - [0][1][1][0][9][2] = 46, - [0][1][1][0][8][2] = 48, - [0][1][1][0][11][2] = 46, - [0][1][1][0][2][3] = 70, - [0][1][1][0][1][3] = 46, - [0][1][1][0][3][3] = 64, - [0][1][1][0][5][3] = 70, - [0][1][1][0][6][3] = 46, - [0][1][1][0][9][3] = 46, - [0][1][1][0][8][3] = 48, - [0][1][1][0][11][3] = 46, - [0][1][1][0][2][4] = 78, - [0][1][1][0][1][4] = 46, - [0][1][1][0][3][4] = 64, - [0][1][1][0][5][4] = 78, - [0][1][1][0][6][4] = 46, - [0][1][1][0][9][4] = 46, - [0][1][1][0][8][4] = 48, - [0][1][1][0][11][4] = 46, - [0][1][1][0][2][5] = 78, - [0][1][1][0][1][5] = 46, - [0][1][1][0][3][5] = 64, - [0][1][1][0][5][5] = 78, - [0][1][1][0][6][5] = 46, - [0][1][1][0][9][5] = 46, - [0][1][1][0][8][5] = 48, - [0][1][1][0][11][5] = 46, - [0][1][1][0][2][6] = 78, - [0][1][1][0][1][6] = 46, - [0][1][1][0][3][6] = 64, - [0][1][1][0][5][6] = 78, - [0][1][1][0][6][6] = 46, - [0][1][1][0][9][6] = 46, - [0][1][1][0][8][6] = 48, - [0][1][1][0][11][6] = 46, - [0][1][1][0][2][7] = 70, - [0][1][1][0][1][7] = 46, - [0][1][1][0][3][7] = 64, - [0][1][1][0][5][7] = 70, - [0][1][1][0][6][7] = 46, - [0][1][1][0][9][7] = 46, - [0][1][1][0][8][7] = 48, - [0][1][1][0][11][7] = 46, - [0][1][1][0][2][8] = 66, - [0][1][1][0][1][8] = 46, - [0][1][1][0][3][8] = 64, - [0][1][1][0][5][8] = 66, - [0][1][1][0][6][8] = 46, - [0][1][1][0][9][8] = 46, - [0][1][1][0][8][8] = 48, - [0][1][1][0][11][8] = 46, - [0][1][1][0][2][9] = 62, - [0][1][1][0][1][9] = 46, - [0][1][1][0][3][9] = 64, - [0][1][1][0][5][9] = 62, - [0][1][1][0][6][9] = 46, - [0][1][1][0][9][9] = 46, - [0][1][1][0][8][9] = 48, - [0][1][1][0][11][9] = 46, - [0][1][1][0][2][10] = 62, - [0][1][1][0][1][10] = 46, - [0][1][1][0][3][10] = 64, - [0][1][1][0][5][10] = 62, - [0][1][1][0][6][10] = 46, - [0][1][1][0][9][10] = 46, - [0][1][1][0][8][10] = 48, - [0][1][1][0][11][10] = 46, - [0][1][1][0][2][11] = 42, - [0][1][1][0][1][11] = 46, - [0][1][1][0][3][11] = 64, - [0][1][1][0][5][11] = 42, - [0][1][1][0][6][11] = 46, - [0][1][1][0][9][11] = 46, - [0][1][1][0][8][11] = 48, - [0][1][1][0][11][11] = 46, - [0][1][1][0][2][12] = 40, - [0][1][1][0][1][12] = 46, - [0][1][1][0][3][12] = 64, - [0][1][1][0][5][12] = 40, - [0][1][1][0][6][12] = 46, - [0][1][1][0][9][12] = 46, - [0][1][1][0][8][12] = 48, - [0][1][1][0][11][12] = 46, - [0][1][1][0][2][13] = 127, - [0][1][1][0][1][13] = 127, - [0][1][1][0][3][13] = 127, - [0][1][1][0][5][13] = 127, - [0][1][1][0][6][13] = 127, - [0][1][1][0][9][13] = 127, - [0][1][1][0][8][13] = 127, - [0][1][1][0][11][13] = 127, - [0][0][2][0][2][0] = 66, - [0][0][2][0][1][0] = 58, - [0][0][2][0][3][0] = 76, - [0][0][2][0][5][0] = 66, - [0][0][2][0][6][0] = 58, - [0][0][2][0][9][0] = 58, - [0][0][2][0][8][0] = 60, - [0][0][2][0][11][0] = 58, - [0][0][2][0][2][1] = 66, - [0][0][2][0][1][1] = 58, - [0][0][2][0][3][1] = 76, - [0][0][2][0][5][1] = 66, - [0][0][2][0][6][1] = 58, - [0][0][2][0][9][1] = 58, - [0][0][2][0][8][1] = 60, - [0][0][2][0][11][1] = 58, - [0][0][2][0][2][2] = 70, - [0][0][2][0][1][2] = 58, - [0][0][2][0][3][2] = 76, - [0][0][2][0][5][2] = 70, - [0][0][2][0][6][2] = 58, - [0][0][2][0][9][2] = 58, - [0][0][2][0][8][2] = 60, - [0][0][2][0][11][2] = 58, - [0][0][2][0][2][3] = 74, - [0][0][2][0][1][3] = 58, - [0][0][2][0][3][3] = 76, - [0][0][2][0][5][3] = 74, - [0][0][2][0][6][3] = 58, - [0][0][2][0][9][3] = 58, - [0][0][2][0][8][3] = 60, - [0][0][2][0][11][3] = 58, - [0][0][2][0][2][4] = 76, - [0][0][2][0][1][4] = 58, - [0][0][2][0][3][4] = 76, - [0][0][2][0][5][4] = 76, - [0][0][2][0][6][4] = 58, - [0][0][2][0][9][4] = 58, - [0][0][2][0][8][4] = 60, - [0][0][2][0][11][4] = 58, - [0][0][2][0][2][5] = 76, - [0][0][2][0][1][5] = 58, - [0][0][2][0][3][5] = 76, - [0][0][2][0][5][5] = 76, - [0][0][2][0][6][5] = 58, - [0][0][2][0][9][5] = 58, - [0][0][2][0][8][5] = 60, - [0][0][2][0][11][5] = 58, - [0][0][2][0][2][6] = 76, - [0][0][2][0][1][6] = 58, - [0][0][2][0][3][6] = 76, - [0][0][2][0][5][6] = 76, - [0][0][2][0][6][6] = 58, - [0][0][2][0][9][6] = 58, - [0][0][2][0][8][6] = 60, - [0][0][2][0][11][6] = 58, - [0][0][2][0][2][7] = 74, - [0][0][2][0][1][7] = 58, - [0][0][2][0][3][7] = 76, - [0][0][2][0][5][7] = 74, - [0][0][2][0][6][7] = 58, - [0][0][2][0][9][7] = 58, - [0][0][2][0][8][7] = 60, - [0][0][2][0][11][7] = 58, - [0][0][2][0][2][8] = 70, - [0][0][2][0][1][8] = 58, - [0][0][2][0][3][8] = 76, - [0][0][2][0][5][8] = 70, - [0][0][2][0][6][8] = 58, - [0][0][2][0][9][8] = 58, - [0][0][2][0][8][8] = 60, - [0][0][2][0][11][8] = 58, - [0][0][2][0][2][9] = 66, - [0][0][2][0][1][9] = 58, - [0][0][2][0][3][9] = 76, - [0][0][2][0][5][9] = 66, - [0][0][2][0][6][9] = 58, - [0][0][2][0][9][9] = 58, - [0][0][2][0][8][9] = 60, - [0][0][2][0][11][9] = 58, - [0][0][2][0][2][10] = 66, - [0][0][2][0][1][10] = 58, - [0][0][2][0][3][10] = 76, - [0][0][2][0][5][10] = 66, - [0][0][2][0][6][10] = 58, - [0][0][2][0][9][10] = 58, - [0][0][2][0][8][10] = 60, - [0][0][2][0][11][10] = 58, - [0][0][2][0][2][11] = 54, - [0][0][2][0][1][11] = 58, - [0][0][2][0][3][11] = 76, - [0][0][2][0][5][11] = 54, - [0][0][2][0][6][11] = 58, - [0][0][2][0][9][11] = 58, - [0][0][2][0][8][11] = 60, - [0][0][2][0][11][11] = 58, - [0][0][2][0][2][12] = 50, - [0][0][2][0][1][12] = 58, - [0][0][2][0][3][12] = 76, - [0][0][2][0][5][12] = 50, - [0][0][2][0][6][12] = 58, - [0][0][2][0][9][12] = 58, - [0][0][2][0][8][12] = 60, - [0][0][2][0][11][12] = 58, - [0][0][2][0][2][13] = 127, - [0][0][2][0][1][13] = 127, - [0][0][2][0][3][13] = 127, - [0][0][2][0][5][13] = 127, - [0][0][2][0][6][13] = 127, - [0][0][2][0][9][13] = 127, - [0][0][2][0][8][13] = 127, - [0][0][2][0][11][13] = 127, - [0][1][2][0][2][0] = 62, - [0][1][2][0][1][0] = 46, - [0][1][2][0][3][0] = 64, - [0][1][2][0][5][0] = 62, - [0][1][2][0][6][0] = 46, - [0][1][2][0][9][0] = 46, - [0][1][2][0][8][0] = 48, - [0][1][2][0][11][0] = 46, - [0][1][2][0][2][1] = 62, - [0][1][2][0][1][1] = 46, - [0][1][2][0][3][1] = 64, - [0][1][2][0][5][1] = 62, - [0][1][2][0][6][1] = 46, - [0][1][2][0][9][1] = 46, - [0][1][2][0][8][1] = 48, - [0][1][2][0][11][1] = 46, - [0][1][2][0][2][2] = 66, - [0][1][2][0][1][2] = 46, - [0][1][2][0][3][2] = 64, - [0][1][2][0][5][2] = 66, - [0][1][2][0][6][2] = 46, - [0][1][2][0][9][2] = 46, - [0][1][2][0][8][2] = 48, - [0][1][2][0][11][2] = 46, - [0][1][2][0][2][3] = 70, - [0][1][2][0][1][3] = 46, - [0][1][2][0][3][3] = 64, - [0][1][2][0][5][3] = 70, - [0][1][2][0][6][3] = 46, - [0][1][2][0][9][3] = 46, - [0][1][2][0][8][3] = 48, - [0][1][2][0][11][3] = 46, - [0][1][2][0][2][4] = 76, - [0][1][2][0][1][4] = 46, - [0][1][2][0][3][4] = 64, - [0][1][2][0][5][4] = 76, - [0][1][2][0][6][4] = 46, - [0][1][2][0][9][4] = 46, - [0][1][2][0][8][4] = 48, - [0][1][2][0][11][4] = 46, - [0][1][2][0][2][5] = 76, - [0][1][2][0][1][5] = 46, - [0][1][2][0][3][5] = 64, - [0][1][2][0][5][5] = 76, - [0][1][2][0][6][5] = 46, - [0][1][2][0][9][5] = 46, - [0][1][2][0][8][5] = 48, - [0][1][2][0][11][5] = 46, - [0][1][2][0][2][6] = 76, - [0][1][2][0][1][6] = 46, - [0][1][2][0][3][6] = 64, - [0][1][2][0][5][6] = 76, - [0][1][2][0][6][6] = 46, - [0][1][2][0][9][6] = 46, - [0][1][2][0][8][6] = 48, - [0][1][2][0][11][6] = 46, - [0][1][2][0][2][7] = 68, - [0][1][2][0][1][7] = 46, - [0][1][2][0][3][7] = 64, - [0][1][2][0][5][7] = 68, - [0][1][2][0][6][7] = 46, - [0][1][2][0][9][7] = 46, - [0][1][2][0][8][7] = 48, - [0][1][2][0][11][7] = 46, - [0][1][2][0][2][8] = 64, - [0][1][2][0][1][8] = 46, - [0][1][2][0][3][8] = 64, - [0][1][2][0][5][8] = 64, - [0][1][2][0][6][8] = 46, - [0][1][2][0][9][8] = 46, - [0][1][2][0][8][8] = 48, - [0][1][2][0][11][8] = 46, - [0][1][2][0][2][9] = 60, - [0][1][2][0][1][9] = 46, - [0][1][2][0][3][9] = 64, - [0][1][2][0][5][9] = 60, - [0][1][2][0][6][9] = 46, - [0][1][2][0][9][9] = 46, - [0][1][2][0][8][9] = 48, - [0][1][2][0][11][9] = 46, - [0][1][2][0][2][10] = 60, - [0][1][2][0][1][10] = 46, - [0][1][2][0][3][10] = 64, - [0][1][2][0][5][10] = 60, - [0][1][2][0][6][10] = 46, - [0][1][2][0][9][10] = 46, - [0][1][2][0][8][10] = 48, - [0][1][2][0][11][10] = 46, - [0][1][2][0][2][11] = 42, - [0][1][2][0][1][11] = 46, - [0][1][2][0][3][11] = 64, - [0][1][2][0][5][11] = 42, - [0][1][2][0][6][11] = 46, - [0][1][2][0][9][11] = 46, - [0][1][2][0][8][11] = 48, - [0][1][2][0][11][11] = 46, - [0][1][2][0][2][12] = 40, - [0][1][2][0][1][12] = 46, - [0][1][2][0][3][12] = 64, - [0][1][2][0][5][12] = 40, - [0][1][2][0][6][12] = 46, - [0][1][2][0][9][12] = 46, - [0][1][2][0][8][12] = 48, - [0][1][2][0][11][12] = 46, - [0][1][2][0][2][13] = 127, - [0][1][2][0][1][13] = 127, - [0][1][2][0][3][13] = 127, - [0][1][2][0][5][13] = 127, - [0][1][2][0][6][13] = 127, - [0][1][2][0][9][13] = 127, - [0][1][2][0][8][13] = 127, - [0][1][2][0][11][13] = 127, - [0][1][2][1][2][0] = 62, - [0][1][2][1][1][0] = 34, - [0][1][2][1][3][0] = 64, - [0][1][2][1][5][0] = 62, - [0][1][2][1][6][0] = 34, - [0][1][2][1][9][0] = 34, - [0][1][2][1][8][0] = 36, - [0][1][2][1][11][0] = 34, - [0][1][2][1][2][1] = 62, - [0][1][2][1][1][1] = 34, - [0][1][2][1][3][1] = 64, - [0][1][2][1][5][1] = 62, - [0][1][2][1][6][1] = 34, - [0][1][2][1][9][1] = 34, - [0][1][2][1][8][1] = 36, - [0][1][2][1][11][1] = 34, - [0][1][2][1][2][2] = 66, - [0][1][2][1][1][2] = 34, - [0][1][2][1][3][2] = 64, - [0][1][2][1][5][2] = 66, - [0][1][2][1][6][2] = 34, - [0][1][2][1][9][2] = 34, - [0][1][2][1][8][2] = 36, - [0][1][2][1][11][2] = 34, - [0][1][2][1][2][3] = 70, - [0][1][2][1][1][3] = 34, - [0][1][2][1][3][3] = 64, - [0][1][2][1][5][3] = 70, - [0][1][2][1][6][3] = 34, - [0][1][2][1][9][3] = 34, - [0][1][2][1][8][3] = 36, - [0][1][2][1][11][3] = 34, - [0][1][2][1][2][4] = 76, - [0][1][2][1][1][4] = 34, - [0][1][2][1][3][4] = 64, - [0][1][2][1][5][4] = 76, - [0][1][2][1][6][4] = 34, - [0][1][2][1][9][4] = 34, - [0][1][2][1][8][4] = 36, - [0][1][2][1][11][4] = 34, - [0][1][2][1][2][5] = 76, - [0][1][2][1][1][5] = 34, - [0][1][2][1][3][5] = 64, - [0][1][2][1][5][5] = 76, - [0][1][2][1][6][5] = 34, - [0][1][2][1][9][5] = 34, - [0][1][2][1][8][5] = 36, - [0][1][2][1][11][5] = 34, - [0][1][2][1][2][6] = 76, - [0][1][2][1][1][6] = 34, - [0][1][2][1][3][6] = 64, - [0][1][2][1][5][6] = 76, - [0][1][2][1][6][6] = 34, - [0][1][2][1][9][6] = 34, - [0][1][2][1][8][6] = 36, - [0][1][2][1][11][6] = 34, - [0][1][2][1][2][7] = 68, - [0][1][2][1][1][7] = 34, - [0][1][2][1][3][7] = 64, - [0][1][2][1][5][7] = 68, - [0][1][2][1][6][7] = 34, - [0][1][2][1][9][7] = 34, - [0][1][2][1][8][7] = 36, - [0][1][2][1][11][7] = 34, - [0][1][2][1][2][8] = 64, - [0][1][2][1][1][8] = 34, - [0][1][2][1][3][8] = 64, - [0][1][2][1][5][8] = 64, - [0][1][2][1][6][8] = 34, - [0][1][2][1][9][8] = 34, - [0][1][2][1][8][8] = 36, - [0][1][2][1][11][8] = 34, - [0][1][2][1][2][9] = 60, - [0][1][2][1][1][9] = 34, - [0][1][2][1][3][9] = 64, - [0][1][2][1][5][9] = 60, - [0][1][2][1][6][9] = 34, - [0][1][2][1][9][9] = 34, - [0][1][2][1][8][9] = 36, - [0][1][2][1][11][9] = 34, - [0][1][2][1][2][10] = 60, - [0][1][2][1][1][10] = 34, - [0][1][2][1][3][10] = 64, - [0][1][2][1][5][10] = 60, - [0][1][2][1][6][10] = 34, - [0][1][2][1][9][10] = 34, - [0][1][2][1][8][10] = 36, - [0][1][2][1][11][10] = 34, - [0][1][2][1][2][11] = 42, - [0][1][2][1][1][11] = 34, - [0][1][2][1][3][11] = 64, - [0][1][2][1][5][11] = 42, - [0][1][2][1][6][11] = 34, - [0][1][2][1][9][11] = 34, - [0][1][2][1][8][11] = 36, - [0][1][2][1][11][11] = 34, - [0][1][2][1][2][12] = 40, - [0][1][2][1][1][12] = 34, - [0][1][2][1][3][12] = 64, - [0][1][2][1][5][12] = 40, - [0][1][2][1][6][12] = 34, - [0][1][2][1][9][12] = 34, - [0][1][2][1][8][12] = 36, - [0][1][2][1][11][12] = 34, - [0][1][2][1][2][13] = 127, - [0][1][2][1][1][13] = 127, - [0][1][2][1][3][13] = 127, - [0][1][2][1][5][13] = 127, - [0][1][2][1][6][13] = 127, - [0][1][2][1][9][13] = 127, - [0][1][2][1][8][13] = 127, - [0][1][2][1][11][13] = 127, - [1][0][2][0][2][0] = 127, - [1][0][2][0][1][0] = 127, - [1][0][2][0][3][0] = 127, - [1][0][2][0][5][0] = 127, - [1][0][2][0][6][0] = 127, - [1][0][2][0][9][0] = 127, - [1][0][2][0][8][0] = 127, - [1][0][2][0][11][0] = 127, - [1][0][2][0][2][1] = 127, - [1][0][2][0][1][1] = 127, - [1][0][2][0][3][1] = 127, - [1][0][2][0][5][1] = 127, - [1][0][2][0][6][1] = 127, - [1][0][2][0][9][1] = 127, - [1][0][2][0][8][1] = 127, - [1][0][2][0][11][1] = 127, - [1][0][2][0][2][2] = 56, - [1][0][2][0][1][2] = 58, - [1][0][2][0][3][2] = 76, - [1][0][2][0][5][2] = 56, - [1][0][2][0][6][2] = 58, - [1][0][2][0][9][2] = 58, - [1][0][2][0][8][2] = 60, - [1][0][2][0][11][2] = 58, - [1][0][2][0][2][3] = 56, - [1][0][2][0][1][3] = 58, - [1][0][2][0][3][3] = 76, - [1][0][2][0][5][3] = 56, - [1][0][2][0][6][3] = 58, - [1][0][2][0][9][3] = 58, - [1][0][2][0][8][3] = 60, - [1][0][2][0][11][3] = 58, - [1][0][2][0][2][4] = 60, - [1][0][2][0][1][4] = 58, - [1][0][2][0][3][4] = 76, - [1][0][2][0][5][4] = 60, - [1][0][2][0][6][4] = 58, - [1][0][2][0][9][4] = 58, - [1][0][2][0][8][4] = 60, - [1][0][2][0][11][4] = 58, - [1][0][2][0][2][5] = 64, - [1][0][2][0][1][5] = 58, - [1][0][2][0][3][5] = 76, - [1][0][2][0][5][5] = 64, - [1][0][2][0][6][5] = 58, - [1][0][2][0][9][5] = 58, - [1][0][2][0][8][5] = 60, - [1][0][2][0][11][5] = 58, - [1][0][2][0][2][6] = 54, - [1][0][2][0][1][6] = 58, - [1][0][2][0][3][6] = 76, - [1][0][2][0][5][6] = 54, - [1][0][2][0][6][6] = 58, - [1][0][2][0][9][6] = 58, - [1][0][2][0][8][6] = 60, - [1][0][2][0][11][6] = 58, - [1][0][2][0][2][7] = 50, - [1][0][2][0][1][7] = 58, - [1][0][2][0][3][7] = 76, - [1][0][2][0][5][7] = 50, - [1][0][2][0][6][7] = 58, - [1][0][2][0][9][7] = 58, - [1][0][2][0][8][7] = 60, - [1][0][2][0][11][7] = 58, - [1][0][2][0][2][8] = 50, - [1][0][2][0][1][8] = 58, - [1][0][2][0][3][8] = 76, - [1][0][2][0][5][8] = 50, - [1][0][2][0][6][8] = 58, - [1][0][2][0][9][8] = 58, - [1][0][2][0][8][8] = 60, - [1][0][2][0][11][8] = 58, - [1][0][2][0][2][9] = 42, - [1][0][2][0][1][9] = 58, - [1][0][2][0][3][9] = 76, - [1][0][2][0][5][9] = 42, - [1][0][2][0][6][9] = 58, - [1][0][2][0][9][9] = 58, - [1][0][2][0][8][9] = 60, - [1][0][2][0][11][9] = 58, - [1][0][2][0][2][10] = 40, - [1][0][2][0][1][10] = 58, - [1][0][2][0][3][10] = 76, - [1][0][2][0][5][10] = 40, - [1][0][2][0][6][10] = 58, - [1][0][2][0][9][10] = 58, - [1][0][2][0][8][10] = 60, - [1][0][2][0][11][10] = 58, - [1][0][2][0][2][11] = 127, - [1][0][2][0][1][11] = 127, - [1][0][2][0][3][11] = 127, - [1][0][2][0][5][11] = 127, - [1][0][2][0][6][11] = 127, - [1][0][2][0][9][11] = 127, - [1][0][2][0][8][11] = 127, - [1][0][2][0][11][11] = 127, - [1][0][2][0][2][12] = 127, - [1][0][2][0][1][12] = 127, - [1][0][2][0][3][12] = 127, - [1][0][2][0][5][12] = 127, - [1][0][2][0][6][12] = 127, - [1][0][2][0][9][12] = 127, - [1][0][2][0][8][12] = 127, - [1][0][2][0][11][12] = 127, - [1][0][2][0][2][13] = 127, - [1][0][2][0][1][13] = 127, - [1][0][2][0][3][13] = 127, - [1][0][2][0][5][13] = 127, - [1][0][2][0][6][13] = 127, - [1][0][2][0][9][13] = 127, - [1][0][2][0][8][13] = 127, - [1][0][2][0][11][13] = 127, - [1][1][2][0][2][0] = 127, - [1][1][2][0][1][0] = 127, - [1][1][2][0][3][0] = 127, - [1][1][2][0][5][0] = 127, - [1][1][2][0][6][0] = 127, - [1][1][2][0][9][0] = 127, - [1][1][2][0][8][0] = 127, - [1][1][2][0][11][0] = 127, - [1][1][2][0][2][1] = 127, - [1][1][2][0][1][1] = 127, - [1][1][2][0][3][1] = 127, - [1][1][2][0][5][1] = 127, - [1][1][2][0][6][1] = 127, - [1][1][2][0][9][1] = 127, - [1][1][2][0][8][1] = 127, - [1][1][2][0][11][1] = 127, - [1][1][2][0][2][2] = 52, - [1][1][2][0][1][2] = 46, - [1][1][2][0][3][2] = 64, - [1][1][2][0][5][2] = 52, - [1][1][2][0][6][2] = 46, - [1][1][2][0][9][2] = 46, - [1][1][2][0][8][2] = 48, - [1][1][2][0][11][2] = 46, - [1][1][2][0][2][3] = 52, - [1][1][2][0][1][3] = 46, - [1][1][2][0][3][3] = 64, - [1][1][2][0][5][3] = 52, - [1][1][2][0][6][3] = 46, - [1][1][2][0][9][3] = 46, - [1][1][2][0][8][3] = 48, - [1][1][2][0][11][3] = 46, - [1][1][2][0][2][4] = 56, - [1][1][2][0][1][4] = 46, - [1][1][2][0][3][4] = 64, - [1][1][2][0][5][4] = 56, - [1][1][2][0][6][4] = 46, - [1][1][2][0][9][4] = 46, - [1][1][2][0][8][4] = 48, - [1][1][2][0][11][4] = 46, - [1][1][2][0][2][5] = 60, - [1][1][2][0][1][5] = 46, - [1][1][2][0][3][5] = 64, - [1][1][2][0][5][5] = 60, - [1][1][2][0][6][5] = 46, - [1][1][2][0][9][5] = 46, - [1][1][2][0][8][5] = 48, - [1][1][2][0][11][5] = 46, - [1][1][2][0][2][6] = 54, - [1][1][2][0][1][6] = 46, - [1][1][2][0][3][6] = 64, - [1][1][2][0][5][6] = 52, - [1][1][2][0][6][6] = 46, - [1][1][2][0][9][6] = 46, - [1][1][2][0][8][6] = 48, - [1][1][2][0][11][6] = 46, - [1][1][2][0][2][7] = 50, - [1][1][2][0][1][7] = 46, - [1][1][2][0][3][7] = 64, - [1][1][2][0][5][7] = 48, - [1][1][2][0][6][7] = 46, - [1][1][2][0][9][7] = 46, - [1][1][2][0][8][7] = 48, - [1][1][2][0][11][7] = 46, - [1][1][2][0][2][8] = 50, - [1][1][2][0][1][8] = 46, - [1][1][2][0][3][8] = 64, - [1][1][2][0][5][8] = 48, - [1][1][2][0][6][8] = 46, - [1][1][2][0][9][8] = 46, - [1][1][2][0][8][8] = 48, - [1][1][2][0][11][8] = 46, - [1][1][2][0][2][9] = 38, - [1][1][2][0][1][9] = 46, - [1][1][2][0][3][9] = 64, - [1][1][2][0][5][9] = 38, - [1][1][2][0][6][9] = 46, - [1][1][2][0][9][9] = 46, - [1][1][2][0][8][9] = 48, - [1][1][2][0][11][9] = 46, - [1][1][2][0][2][10] = 36, - [1][1][2][0][1][10] = 46, - [1][1][2][0][3][10] = 64, - [1][1][2][0][5][10] = 36, - [1][1][2][0][6][10] = 46, - [1][1][2][0][9][10] = 46, - [1][1][2][0][8][10] = 48, - [1][1][2][0][11][10] = 46, - [1][1][2][0][2][11] = 127, - [1][1][2][0][1][11] = 127, - [1][1][2][0][3][11] = 127, - [1][1][2][0][5][11] = 127, - [1][1][2][0][6][11] = 127, - [1][1][2][0][9][11] = 127, - [1][1][2][0][8][11] = 127, - [1][1][2][0][11][11] = 127, - [1][1][2][0][2][12] = 127, - [1][1][2][0][1][12] = 127, - [1][1][2][0][3][12] = 127, - [1][1][2][0][5][12] = 127, - [1][1][2][0][6][12] = 127, - [1][1][2][0][9][12] = 127, - [1][1][2][0][8][12] = 127, - [1][1][2][0][11][12] = 127, - [1][1][2][0][2][13] = 127, - [1][1][2][0][1][13] = 127, - [1][1][2][0][3][13] = 127, - [1][1][2][0][5][13] = 127, - [1][1][2][0][6][13] = 127, - [1][1][2][0][9][13] = 127, - [1][1][2][0][8][13] = 127, - [1][1][2][0][11][13] = 127, - [1][1][2][1][2][0] = 127, - [1][1][2][1][1][0] = 127, - [1][1][2][1][3][0] = 127, - [1][1][2][1][5][0] = 127, - [1][1][2][1][6][0] = 127, - [1][1][2][1][9][0] = 127, - [1][1][2][1][8][0] = 127, - [1][1][2][1][11][0] = 127, - [1][1][2][1][2][1] = 127, - [1][1][2][1][1][1] = 127, - [1][1][2][1][3][1] = 127, - [1][1][2][1][5][1] = 127, - [1][1][2][1][6][1] = 127, - [1][1][2][1][9][1] = 127, - [1][1][2][1][8][1] = 127, - [1][1][2][1][11][1] = 127, - [1][1][2][1][2][2] = 52, - [1][1][2][1][1][2] = 34, - [1][1][2][1][3][2] = 64, - [1][1][2][1][5][2] = 52, - [1][1][2][1][6][2] = 34, - [1][1][2][1][9][2] = 34, - [1][1][2][1][8][2] = 36, - [1][1][2][1][11][2] = 34, - [1][1][2][1][2][3] = 52, - [1][1][2][1][1][3] = 34, - [1][1][2][1][3][3] = 64, - [1][1][2][1][5][3] = 52, - [1][1][2][1][6][3] = 34, - [1][1][2][1][9][3] = 34, - [1][1][2][1][8][3] = 36, - [1][1][2][1][11][3] = 34, - [1][1][2][1][2][4] = 56, - [1][1][2][1][1][4] = 34, - [1][1][2][1][3][4] = 64, - [1][1][2][1][5][4] = 56, - [1][1][2][1][6][4] = 34, - [1][1][2][1][9][4] = 34, - [1][1][2][1][8][4] = 36, - [1][1][2][1][11][4] = 34, - [1][1][2][1][2][5] = 60, - [1][1][2][1][1][5] = 34, - [1][1][2][1][3][5] = 64, - [1][1][2][1][5][5] = 60, - [1][1][2][1][6][5] = 34, - [1][1][2][1][9][5] = 34, - [1][1][2][1][8][5] = 36, - [1][1][2][1][11][5] = 34, - [1][1][2][1][2][6] = 54, - [1][1][2][1][1][6] = 34, - [1][1][2][1][3][6] = 64, - [1][1][2][1][5][6] = 52, - [1][1][2][1][6][6] = 34, - [1][1][2][1][9][6] = 34, - [1][1][2][1][8][6] = 36, - [1][1][2][1][11][6] = 34, - [1][1][2][1][2][7] = 50, - [1][1][2][1][1][7] = 34, - [1][1][2][1][3][7] = 64, - [1][1][2][1][5][7] = 48, - [1][1][2][1][6][7] = 34, - [1][1][2][1][9][7] = 34, - [1][1][2][1][8][7] = 36, - [1][1][2][1][11][7] = 34, - [1][1][2][1][2][8] = 50, - [1][1][2][1][1][8] = 34, - [1][1][2][1][3][8] = 64, - [1][1][2][1][5][8] = 48, - [1][1][2][1][6][8] = 34, - [1][1][2][1][9][8] = 34, - [1][1][2][1][8][8] = 36, - [1][1][2][1][11][8] = 34, - [1][1][2][1][2][9] = 38, - [1][1][2][1][1][9] = 34, - [1][1][2][1][3][9] = 64, - [1][1][2][1][5][9] = 38, - [1][1][2][1][6][9] = 34, - [1][1][2][1][9][9] = 34, - [1][1][2][1][8][9] = 36, - [1][1][2][1][11][9] = 34, - [1][1][2][1][2][10] = 36, - [1][1][2][1][1][10] = 34, - [1][1][2][1][3][10] = 64, - [1][1][2][1][5][10] = 36, - [1][1][2][1][6][10] = 34, - [1][1][2][1][9][10] = 34, - [1][1][2][1][8][10] = 36, - [1][1][2][1][11][10] = 34, - [1][1][2][1][2][11] = 127, - [1][1][2][1][1][11] = 127, - [1][1][2][1][3][11] = 127, - [1][1][2][1][5][11] = 127, - [1][1][2][1][6][11] = 127, - [1][1][2][1][9][11] = 127, - [1][1][2][1][8][11] = 127, - [1][1][2][1][11][11] = 127, - [1][1][2][1][2][12] = 127, - [1][1][2][1][1][12] = 127, - [1][1][2][1][3][12] = 127, - [1][1][2][1][5][12] = 127, - [1][1][2][1][6][12] = 127, - [1][1][2][1][9][12] = 127, - [1][1][2][1][8][12] = 127, - [1][1][2][1][11][12] = 127, - [1][1][2][1][2][13] = 127, - [1][1][2][1][1][13] = 127, - [1][1][2][1][3][13] = 127, - [1][1][2][1][5][13] = 127, - [1][1][2][1][6][13] = 127, - [1][1][2][1][9][13] = 127, - [1][1][2][1][8][13] = 127, - [1][1][2][1][11][13] = 127, + [0][0][0][0][RTW89_WW][0] = 56, + [0][0][0][0][RTW89_WW][1] = 56, + [0][0][0][0][RTW89_WW][2] = 56, + [0][0][0][0][RTW89_WW][3] = 56, + [0][0][0][0][RTW89_WW][4] = 56, + [0][0][0][0][RTW89_WW][5] = 56, + [0][0][0][0][RTW89_WW][6] = 56, + [0][0][0][0][RTW89_WW][7] = 56, + [0][0][0][0][RTW89_WW][8] = 56, + [0][0][0][0][RTW89_WW][9] = 56, + [0][0][0][0][RTW89_WW][10] = 56, + [0][0][0][0][RTW89_WW][11] = 56, + [0][0][0][0][RTW89_WW][12] = 48, + [0][0][0][0][RTW89_WW][13] = 76, + [0][1][0][0][RTW89_WW][0] = 44, + [0][1][0][0][RTW89_WW][1] = 44, + [0][1][0][0][RTW89_WW][2] = 44, + [0][1][0][0][RTW89_WW][3] = 44, + [0][1][0][0][RTW89_WW][4] = 44, + [0][1][0][0][RTW89_WW][5] = 44, + [0][1][0][0][RTW89_WW][6] = 44, + [0][1][0][0][RTW89_WW][7] = 44, + [0][1][0][0][RTW89_WW][8] = 44, + [0][1][0][0][RTW89_WW][9] = 44, + [0][1][0][0][RTW89_WW][10] = 44, + [0][1][0][0][RTW89_WW][11] = 44, + [0][1][0][0][RTW89_WW][12] = 38, + [0][1][0][0][RTW89_WW][13] = 64, + [1][0][0][0][RTW89_WW][0] = 0, + [1][0][0][0][RTW89_WW][1] = 0, + [1][0][0][0][RTW89_WW][2] = 58, + [1][0][0][0][RTW89_WW][3] = 58, + [1][0][0][0][RTW89_WW][4] = 58, + [1][0][0][0][RTW89_WW][5] = 58, + [1][0][0][0][RTW89_WW][6] = 46, + [1][0][0][0][RTW89_WW][7] = 46, + [1][0][0][0][RTW89_WW][8] = 46, + [1][0][0][0][RTW89_WW][9] = 32, + [1][0][0][0][RTW89_WW][10] = 32, + [1][0][0][0][RTW89_WW][11] = 0, + [1][0][0][0][RTW89_WW][12] = 0, + [1][0][0][0][RTW89_WW][13] = 0, + [1][1][0][0][RTW89_WW][0] = 0, + [1][1][0][0][RTW89_WW][1] = 0, + [1][1][0][0][RTW89_WW][2] = 46, + [1][1][0][0][RTW89_WW][3] = 46, + [1][1][0][0][RTW89_WW][4] = 46, + [1][1][0][0][RTW89_WW][5] = 46, + [1][1][0][0][RTW89_WW][6] = 46, + [1][1][0][0][RTW89_WW][7] = 46, + [1][1][0][0][RTW89_WW][8] = 46, + [1][1][0][0][RTW89_WW][9] = 24, + [1][1][0][0][RTW89_WW][10] = 24, + [1][1][0][0][RTW89_WW][11] = 0, + [1][1][0][0][RTW89_WW][12] = 0, + [1][1][0][0][RTW89_WW][13] = 0, + [0][0][1][0][RTW89_WW][0] = 58, + [0][0][1][0][RTW89_WW][1] = 58, + [0][0][1][0][RTW89_WW][2] = 58, + [0][0][1][0][RTW89_WW][3] = 58, + [0][0][1][0][RTW89_WW][4] = 58, + [0][0][1][0][RTW89_WW][5] = 58, + [0][0][1][0][RTW89_WW][6] = 58, + [0][0][1][0][RTW89_WW][7] = 58, + [0][0][1][0][RTW89_WW][8] = 58, + [0][0][1][0][RTW89_WW][9] = 58, + [0][0][1][0][RTW89_WW][10] = 58, + [0][0][1][0][RTW89_WW][11] = 56, + [0][0][1][0][RTW89_WW][12] = 52, + [0][0][1][0][RTW89_WW][13] = 0, + [0][1][1][0][RTW89_WW][0] = 46, + [0][1][1][0][RTW89_WW][1] = 46, + [0][1][1][0][RTW89_WW][2] = 46, + [0][1][1][0][RTW89_WW][3] = 46, + [0][1][1][0][RTW89_WW][4] = 46, + [0][1][1][0][RTW89_WW][5] = 46, + [0][1][1][0][RTW89_WW][6] = 46, + [0][1][1][0][RTW89_WW][7] = 46, + [0][1][1][0][RTW89_WW][8] = 46, + [0][1][1][0][RTW89_WW][9] = 46, + [0][1][1][0][RTW89_WW][10] = 46, + [0][1][1][0][RTW89_WW][11] = 42, + [0][1][1][0][RTW89_WW][12] = 40, + [0][1][1][0][RTW89_WW][13] = 0, + [0][0][2][0][RTW89_WW][0] = 58, + [0][0][2][0][RTW89_WW][1] = 58, + [0][0][2][0][RTW89_WW][2] = 58, + [0][0][2][0][RTW89_WW][3] = 58, + [0][0][2][0][RTW89_WW][4] = 58, + [0][0][2][0][RTW89_WW][5] = 58, + [0][0][2][0][RTW89_WW][6] = 58, + [0][0][2][0][RTW89_WW][7] = 58, + [0][0][2][0][RTW89_WW][8] = 58, + [0][0][2][0][RTW89_WW][9] = 58, + [0][0][2][0][RTW89_WW][10] = 58, + [0][0][2][0][RTW89_WW][11] = 54, + [0][0][2][0][RTW89_WW][12] = 50, + [0][0][2][0][RTW89_WW][13] = 0, + [0][1][2][0][RTW89_WW][0] = 46, + [0][1][2][0][RTW89_WW][1] = 46, + [0][1][2][0][RTW89_WW][2] = 46, + [0][1][2][0][RTW89_WW][3] = 46, + [0][1][2][0][RTW89_WW][4] = 46, + [0][1][2][0][RTW89_WW][5] = 46, + [0][1][2][0][RTW89_WW][6] = 46, + [0][1][2][0][RTW89_WW][7] = 46, + [0][1][2][0][RTW89_WW][8] = 46, + [0][1][2][0][RTW89_WW][9] = 46, + [0][1][2][0][RTW89_WW][10] = 46, + [0][1][2][0][RTW89_WW][11] = 42, + [0][1][2][0][RTW89_WW][12] = 40, + [0][1][2][0][RTW89_WW][13] = 0, + [0][1][2][1][RTW89_WW][0] = 34, + [0][1][2][1][RTW89_WW][1] = 34, + [0][1][2][1][RTW89_WW][2] = 34, + [0][1][2][1][RTW89_WW][3] = 34, + [0][1][2][1][RTW89_WW][4] = 34, + [0][1][2][1][RTW89_WW][5] = 34, + [0][1][2][1][RTW89_WW][6] = 34, + [0][1][2][1][RTW89_WW][7] = 34, + [0][1][2][1][RTW89_WW][8] = 34, + [0][1][2][1][RTW89_WW][9] = 34, + [0][1][2][1][RTW89_WW][10] = 34, + [0][1][2][1][RTW89_WW][11] = 34, + [0][1][2][1][RTW89_WW][12] = 34, + [0][1][2][1][RTW89_WW][13] = 0, + [1][0][2][0][RTW89_WW][0] = 0, + [1][0][2][0][RTW89_WW][1] = 0, + [1][0][2][0][RTW89_WW][2] = 56, + [1][0][2][0][RTW89_WW][3] = 56, + [1][0][2][0][RTW89_WW][4] = 58, + [1][0][2][0][RTW89_WW][5] = 58, + [1][0][2][0][RTW89_WW][6] = 54, + [1][0][2][0][RTW89_WW][7] = 50, + [1][0][2][0][RTW89_WW][8] = 50, + [1][0][2][0][RTW89_WW][9] = 42, + [1][0][2][0][RTW89_WW][10] = 40, + [1][0][2][0][RTW89_WW][11] = 0, + [1][0][2][0][RTW89_WW][12] = 0, + [1][0][2][0][RTW89_WW][13] = 0, + [1][1][2][0][RTW89_WW][0] = 0, + [1][1][2][0][RTW89_WW][1] = 0, + [1][1][2][0][RTW89_WW][2] = 46, + [1][1][2][0][RTW89_WW][3] = 46, + [1][1][2][0][RTW89_WW][4] = 46, + [1][1][2][0][RTW89_WW][5] = 46, + [1][1][2][0][RTW89_WW][6] = 46, + [1][1][2][0][RTW89_WW][7] = 46, + [1][1][2][0][RTW89_WW][8] = 46, + [1][1][2][0][RTW89_WW][9] = 38, + [1][1][2][0][RTW89_WW][10] = 36, + [1][1][2][0][RTW89_WW][11] = 0, + [1][1][2][0][RTW89_WW][12] = 0, + [1][1][2][0][RTW89_WW][13] = 0, + [1][1][2][1][RTW89_WW][0] = 0, + [1][1][2][1][RTW89_WW][1] = 0, + [1][1][2][1][RTW89_WW][2] = 34, + [1][1][2][1][RTW89_WW][3] = 34, + [1][1][2][1][RTW89_WW][4] = 34, + [1][1][2][1][RTW89_WW][5] = 34, + [1][1][2][1][RTW89_WW][6] = 34, + [1][1][2][1][RTW89_WW][7] = 34, + [1][1][2][1][RTW89_WW][8] = 34, + [1][1][2][1][RTW89_WW][9] = 34, + [1][1][2][1][RTW89_WW][10] = 34, + [1][1][2][1][RTW89_WW][11] = 0, + [1][1][2][1][RTW89_WW][12] = 0, + [1][1][2][1][RTW89_WW][13] = 0, + [0][0][0][0][RTW89_FCC][0] = 76, + [0][0][0][0][RTW89_ETSI][0] = 56, + [0][0][0][0][RTW89_MKK][0] = 68, + [0][0][0][0][RTW89_IC][0] = 76, + [0][0][0][0][RTW89_KCC][0] = 76, + [0][0][0][0][RTW89_ACMA][0] = 56, + [0][0][0][0][RTW89_CHILE][0] = 60, + [0][0][0][0][RTW89_UKRAINE][0] = 56, + [0][0][0][0][RTW89_MEXICO][0] = 76, + [0][0][0][0][RTW89_CN][0] = 56, + [0][0][0][0][RTW89_QATAR][0] = 56, + [0][0][0][0][RTW89_FCC][1] = 76, + [0][0][0][0][RTW89_ETSI][1] = 56, + [0][0][0][0][RTW89_MKK][1] = 68, + [0][0][0][0][RTW89_IC][1] = 76, + [0][0][0][0][RTW89_KCC][1] = 76, + [0][0][0][0][RTW89_ACMA][1] = 56, + [0][0][0][0][RTW89_CHILE][1] = 60, + [0][0][0][0][RTW89_UKRAINE][1] = 56, + [0][0][0][0][RTW89_MEXICO][1] = 76, + [0][0][0][0][RTW89_CN][1] = 56, + [0][0][0][0][RTW89_QATAR][1] = 56, + [0][0][0][0][RTW89_FCC][2] = 76, + [0][0][0][0][RTW89_ETSI][2] = 56, + [0][0][0][0][RTW89_MKK][2] = 68, + [0][0][0][0][RTW89_IC][2] = 76, + [0][0][0][0][RTW89_KCC][2] = 76, + [0][0][0][0][RTW89_ACMA][2] = 56, + [0][0][0][0][RTW89_CHILE][2] = 60, + [0][0][0][0][RTW89_UKRAINE][2] = 56, + [0][0][0][0][RTW89_MEXICO][2] = 76, + [0][0][0][0][RTW89_CN][2] = 56, + [0][0][0][0][RTW89_QATAR][2] = 56, + [0][0][0][0][RTW89_FCC][3] = 76, + [0][0][0][0][RTW89_ETSI][3] = 56, + [0][0][0][0][RTW89_MKK][3] = 68, + [0][0][0][0][RTW89_IC][3] = 76, + [0][0][0][0][RTW89_KCC][3] = 76, + [0][0][0][0][RTW89_ACMA][3] = 56, + [0][0][0][0][RTW89_CHILE][3] = 60, + [0][0][0][0][RTW89_UKRAINE][3] = 56, + [0][0][0][0][RTW89_MEXICO][3] = 76, + [0][0][0][0][RTW89_CN][3] = 56, + [0][0][0][0][RTW89_QATAR][3] = 56, + [0][0][0][0][RTW89_FCC][4] = 76, + [0][0][0][0][RTW89_ETSI][4] = 56, + [0][0][0][0][RTW89_MKK][4] = 68, + [0][0][0][0][RTW89_IC][4] = 76, + [0][0][0][0][RTW89_KCC][4] = 76, + [0][0][0][0][RTW89_ACMA][4] = 56, + [0][0][0][0][RTW89_CHILE][4] = 60, + [0][0][0][0][RTW89_UKRAINE][4] = 56, + [0][0][0][0][RTW89_MEXICO][4] = 76, + [0][0][0][0][RTW89_CN][4] = 56, + [0][0][0][0][RTW89_QATAR][4] = 56, + [0][0][0][0][RTW89_FCC][5] = 76, + [0][0][0][0][RTW89_ETSI][5] = 56, + [0][0][0][0][RTW89_MKK][5] = 68, + [0][0][0][0][RTW89_IC][5] = 76, + [0][0][0][0][RTW89_KCC][5] = 76, + [0][0][0][0][RTW89_ACMA][5] = 56, + [0][0][0][0][RTW89_CHILE][5] = 60, + [0][0][0][0][RTW89_UKRAINE][5] = 56, + [0][0][0][0][RTW89_MEXICO][5] = 76, + [0][0][0][0][RTW89_CN][5] = 56, + [0][0][0][0][RTW89_QATAR][5] = 56, + [0][0][0][0][RTW89_FCC][6] = 76, + [0][0][0][0][RTW89_ETSI][6] = 56, + [0][0][0][0][RTW89_MKK][6] = 68, + [0][0][0][0][RTW89_IC][6] = 76, + [0][0][0][0][RTW89_KCC][6] = 76, + [0][0][0][0][RTW89_ACMA][6] = 56, + [0][0][0][0][RTW89_CHILE][6] = 60, + [0][0][0][0][RTW89_UKRAINE][6] = 56, + [0][0][0][0][RTW89_MEXICO][6] = 76, + [0][0][0][0][RTW89_CN][6] = 56, + [0][0][0][0][RTW89_QATAR][6] = 56, + [0][0][0][0][RTW89_FCC][7] = 76, + [0][0][0][0][RTW89_ETSI][7] = 56, + [0][0][0][0][RTW89_MKK][7] = 68, + [0][0][0][0][RTW89_IC][7] = 76, + [0][0][0][0][RTW89_KCC][7] = 76, + [0][0][0][0][RTW89_ACMA][7] = 56, + [0][0][0][0][RTW89_CHILE][7] = 60, + [0][0][0][0][RTW89_UKRAINE][7] = 56, + [0][0][0][0][RTW89_MEXICO][7] = 76, + [0][0][0][0][RTW89_CN][7] = 56, + [0][0][0][0][RTW89_QATAR][7] = 56, + [0][0][0][0][RTW89_FCC][8] = 76, + [0][0][0][0][RTW89_ETSI][8] = 56, + [0][0][0][0][RTW89_MKK][8] = 68, + [0][0][0][0][RTW89_IC][8] = 76, + [0][0][0][0][RTW89_KCC][8] = 76, + [0][0][0][0][RTW89_ACMA][8] = 56, + [0][0][0][0][RTW89_CHILE][8] = 60, + [0][0][0][0][RTW89_UKRAINE][8] = 56, + [0][0][0][0][RTW89_MEXICO][8] = 76, + [0][0][0][0][RTW89_CN][8] = 56, + [0][0][0][0][RTW89_QATAR][8] = 56, + [0][0][0][0][RTW89_FCC][9] = 76, + [0][0][0][0][RTW89_ETSI][9] = 56, + [0][0][0][0][RTW89_MKK][9] = 68, + [0][0][0][0][RTW89_IC][9] = 76, + [0][0][0][0][RTW89_KCC][9] = 76, + [0][0][0][0][RTW89_ACMA][9] = 56, + [0][0][0][0][RTW89_CHILE][9] = 60, + [0][0][0][0][RTW89_UKRAINE][9] = 56, + [0][0][0][0][RTW89_MEXICO][9] = 76, + [0][0][0][0][RTW89_CN][9] = 56, + [0][0][0][0][RTW89_QATAR][9] = 56, + [0][0][0][0][RTW89_FCC][10] = 76, + [0][0][0][0][RTW89_ETSI][10] = 56, + [0][0][0][0][RTW89_MKK][10] = 68, + [0][0][0][0][RTW89_IC][10] = 76, + [0][0][0][0][RTW89_KCC][10] = 76, + [0][0][0][0][RTW89_ACMA][10] = 56, + [0][0][0][0][RTW89_CHILE][10] = 60, + [0][0][0][0][RTW89_UKRAINE][10] = 56, + [0][0][0][0][RTW89_MEXICO][10] = 76, + [0][0][0][0][RTW89_CN][10] = 56, + [0][0][0][0][RTW89_QATAR][10] = 56, + [0][0][0][0][RTW89_FCC][11] = 68, + [0][0][0][0][RTW89_ETSI][11] = 56, + [0][0][0][0][RTW89_MKK][11] = 68, + [0][0][0][0][RTW89_IC][11] = 68, + [0][0][0][0][RTW89_KCC][11] = 76, + [0][0][0][0][RTW89_ACMA][11] = 56, + [0][0][0][0][RTW89_CHILE][11] = 60, + [0][0][0][0][RTW89_UKRAINE][11] = 56, + [0][0][0][0][RTW89_MEXICO][11] = 68, + [0][0][0][0][RTW89_CN][11] = 56, + [0][0][0][0][RTW89_QATAR][11] = 56, + [0][0][0][0][RTW89_FCC][12] = 48, + [0][0][0][0][RTW89_ETSI][12] = 56, + [0][0][0][0][RTW89_MKK][12] = 68, + [0][0][0][0][RTW89_IC][12] = 48, + [0][0][0][0][RTW89_KCC][12] = 76, + [0][0][0][0][RTW89_ACMA][12] = 56, + [0][0][0][0][RTW89_CHILE][12] = 48, + [0][0][0][0][RTW89_UKRAINE][12] = 56, + [0][0][0][0][RTW89_MEXICO][12] = 48, + [0][0][0][0][RTW89_CN][12] = 56, + [0][0][0][0][RTW89_QATAR][12] = 56, + [0][0][0][0][RTW89_FCC][13] = 127, + [0][0][0][0][RTW89_ETSI][13] = 127, + [0][0][0][0][RTW89_MKK][13] = 76, + [0][0][0][0][RTW89_IC][13] = 127, + [0][0][0][0][RTW89_KCC][13] = 127, + [0][0][0][0][RTW89_ACMA][13] = 127, + [0][0][0][0][RTW89_CHILE][13] = 127, + [0][0][0][0][RTW89_UKRAINE][13] = 127, + [0][0][0][0][RTW89_MEXICO][13] = 127, + [0][0][0][0][RTW89_CN][13] = 127, + [0][0][0][0][RTW89_QATAR][13] = 127, + [0][1][0][0][RTW89_FCC][0] = 74, + [0][1][0][0][RTW89_ETSI][0] = 44, + [0][1][0][0][RTW89_MKK][0] = 56, + [0][1][0][0][RTW89_IC][0] = 74, + [0][1][0][0][RTW89_KCC][0] = 68, + [0][1][0][0][RTW89_ACMA][0] = 44, + [0][1][0][0][RTW89_CHILE][0] = 48, + [0][1][0][0][RTW89_UKRAINE][0] = 44, + [0][1][0][0][RTW89_MEXICO][0] = 74, + [0][1][0][0][RTW89_CN][0] = 44, + [0][1][0][0][RTW89_QATAR][0] = 44, + [0][1][0][0][RTW89_FCC][1] = 76, + [0][1][0][0][RTW89_ETSI][1] = 44, + [0][1][0][0][RTW89_MKK][1] = 56, + [0][1][0][0][RTW89_IC][1] = 76, + [0][1][0][0][RTW89_KCC][1] = 68, + [0][1][0][0][RTW89_ACMA][1] = 44, + [0][1][0][0][RTW89_CHILE][1] = 48, + [0][1][0][0][RTW89_UKRAINE][1] = 44, + [0][1][0][0][RTW89_MEXICO][1] = 76, + [0][1][0][0][RTW89_CN][1] = 44, + [0][1][0][0][RTW89_QATAR][1] = 44, + [0][1][0][0][RTW89_FCC][2] = 76, + [0][1][0][0][RTW89_ETSI][2] = 44, + [0][1][0][0][RTW89_MKK][2] = 56, + [0][1][0][0][RTW89_IC][2] = 76, + [0][1][0][0][RTW89_KCC][2] = 68, + [0][1][0][0][RTW89_ACMA][2] = 44, + [0][1][0][0][RTW89_CHILE][2] = 48, + [0][1][0][0][RTW89_UKRAINE][2] = 44, + [0][1][0][0][RTW89_MEXICO][2] = 76, + [0][1][0][0][RTW89_CN][2] = 44, + [0][1][0][0][RTW89_QATAR][2] = 44, + [0][1][0][0][RTW89_FCC][3] = 76, + [0][1][0][0][RTW89_ETSI][3] = 44, + [0][1][0][0][RTW89_MKK][3] = 56, + [0][1][0][0][RTW89_IC][3] = 76, + [0][1][0][0][RTW89_KCC][3] = 68, + [0][1][0][0][RTW89_ACMA][3] = 44, + [0][1][0][0][RTW89_CHILE][3] = 48, + [0][1][0][0][RTW89_UKRAINE][3] = 44, + [0][1][0][0][RTW89_MEXICO][3] = 76, + [0][1][0][0][RTW89_CN][3] = 44, + [0][1][0][0][RTW89_QATAR][3] = 44, + [0][1][0][0][RTW89_FCC][4] = 76, + [0][1][0][0][RTW89_ETSI][4] = 44, + [0][1][0][0][RTW89_MKK][4] = 56, + [0][1][0][0][RTW89_IC][4] = 76, + [0][1][0][0][RTW89_KCC][4] = 68, + [0][1][0][0][RTW89_ACMA][4] = 44, + [0][1][0][0][RTW89_CHILE][4] = 48, + [0][1][0][0][RTW89_UKRAINE][4] = 44, + [0][1][0][0][RTW89_MEXICO][4] = 76, + [0][1][0][0][RTW89_CN][4] = 44, + [0][1][0][0][RTW89_QATAR][4] = 44, + [0][1][0][0][RTW89_FCC][5] = 76, + [0][1][0][0][RTW89_ETSI][5] = 44, + [0][1][0][0][RTW89_MKK][5] = 56, + [0][1][0][0][RTW89_IC][5] = 76, + [0][1][0][0][RTW89_KCC][5] = 68, + [0][1][0][0][RTW89_ACMA][5] = 44, + [0][1][0][0][RTW89_CHILE][5] = 48, + [0][1][0][0][RTW89_UKRAINE][5] = 44, + [0][1][0][0][RTW89_MEXICO][5] = 76, + [0][1][0][0][RTW89_CN][5] = 44, + [0][1][0][0][RTW89_QATAR][5] = 44, + [0][1][0][0][RTW89_FCC][6] = 76, + [0][1][0][0][RTW89_ETSI][6] = 44, + [0][1][0][0][RTW89_MKK][6] = 56, + [0][1][0][0][RTW89_IC][6] = 76, + [0][1][0][0][RTW89_KCC][6] = 68, + [0][1][0][0][RTW89_ACMA][6] = 44, + [0][1][0][0][RTW89_CHILE][6] = 48, + [0][1][0][0][RTW89_UKRAINE][6] = 44, + [0][1][0][0][RTW89_MEXICO][6] = 76, + [0][1][0][0][RTW89_CN][6] = 44, + [0][1][0][0][RTW89_QATAR][6] = 44, + [0][1][0][0][RTW89_FCC][7] = 76, + [0][1][0][0][RTW89_ETSI][7] = 44, + [0][1][0][0][RTW89_MKK][7] = 56, + [0][1][0][0][RTW89_IC][7] = 76, + [0][1][0][0][RTW89_KCC][7] = 68, + [0][1][0][0][RTW89_ACMA][7] = 44, + [0][1][0][0][RTW89_CHILE][7] = 48, + [0][1][0][0][RTW89_UKRAINE][7] = 44, + [0][1][0][0][RTW89_MEXICO][7] = 76, + [0][1][0][0][RTW89_CN][7] = 44, + [0][1][0][0][RTW89_QATAR][7] = 44, + [0][1][0][0][RTW89_FCC][8] = 76, + [0][1][0][0][RTW89_ETSI][8] = 44, + [0][1][0][0][RTW89_MKK][8] = 56, + [0][1][0][0][RTW89_IC][8] = 76, + [0][1][0][0][RTW89_KCC][8] = 68, + [0][1][0][0][RTW89_ACMA][8] = 44, + [0][1][0][0][RTW89_CHILE][8] = 48, + [0][1][0][0][RTW89_UKRAINE][8] = 44, + [0][1][0][0][RTW89_MEXICO][8] = 76, + [0][1][0][0][RTW89_CN][8] = 44, + [0][1][0][0][RTW89_QATAR][8] = 44, + [0][1][0][0][RTW89_FCC][9] = 76, + [0][1][0][0][RTW89_ETSI][9] = 44, + [0][1][0][0][RTW89_MKK][9] = 56, + [0][1][0][0][RTW89_IC][9] = 76, + [0][1][0][0][RTW89_KCC][9] = 68, + [0][1][0][0][RTW89_ACMA][9] = 44, + [0][1][0][0][RTW89_CHILE][9] = 48, + [0][1][0][0][RTW89_UKRAINE][9] = 44, + [0][1][0][0][RTW89_MEXICO][9] = 76, + [0][1][0][0][RTW89_CN][9] = 44, + [0][1][0][0][RTW89_QATAR][9] = 44, + [0][1][0][0][RTW89_FCC][10] = 62, + [0][1][0][0][RTW89_ETSI][10] = 44, + [0][1][0][0][RTW89_MKK][10] = 56, + [0][1][0][0][RTW89_IC][10] = 62, + [0][1][0][0][RTW89_KCC][10] = 68, + [0][1][0][0][RTW89_ACMA][10] = 44, + [0][1][0][0][RTW89_CHILE][10] = 48, + [0][1][0][0][RTW89_UKRAINE][10] = 44, + [0][1][0][0][RTW89_MEXICO][10] = 62, + [0][1][0][0][RTW89_CN][10] = 44, + [0][1][0][0][RTW89_QATAR][10] = 44, + [0][1][0][0][RTW89_FCC][11] = 52, + [0][1][0][0][RTW89_ETSI][11] = 44, + [0][1][0][0][RTW89_MKK][11] = 56, + [0][1][0][0][RTW89_IC][11] = 52, + [0][1][0][0][RTW89_KCC][11] = 68, + [0][1][0][0][RTW89_ACMA][11] = 44, + [0][1][0][0][RTW89_CHILE][11] = 48, + [0][1][0][0][RTW89_UKRAINE][11] = 44, + [0][1][0][0][RTW89_MEXICO][11] = 52, + [0][1][0][0][RTW89_CN][11] = 44, + [0][1][0][0][RTW89_QATAR][11] = 44, + [0][1][0][0][RTW89_FCC][12] = 38, + [0][1][0][0][RTW89_ETSI][12] = 44, + [0][1][0][0][RTW89_MKK][12] = 56, + [0][1][0][0][RTW89_IC][12] = 38, + [0][1][0][0][RTW89_KCC][12] = 68, + [0][1][0][0][RTW89_ACMA][12] = 44, + [0][1][0][0][RTW89_CHILE][12] = 38, + [0][1][0][0][RTW89_UKRAINE][12] = 44, + [0][1][0][0][RTW89_MEXICO][12] = 38, + [0][1][0][0][RTW89_CN][12] = 44, + [0][1][0][0][RTW89_QATAR][12] = 44, + [0][1][0][0][RTW89_FCC][13] = 127, + [0][1][0][0][RTW89_ETSI][13] = 127, + [0][1][0][0][RTW89_MKK][13] = 64, + [0][1][0][0][RTW89_IC][13] = 127, + [0][1][0][0][RTW89_KCC][13] = 127, + [0][1][0][0][RTW89_ACMA][13] = 127, + [0][1][0][0][RTW89_CHILE][13] = 127, + [0][1][0][0][RTW89_UKRAINE][13] = 127, + [0][1][0][0][RTW89_MEXICO][13] = 127, + [0][1][0][0][RTW89_CN][13] = 127, + [0][1][0][0][RTW89_QATAR][13] = 127, + [1][0][0][0][RTW89_FCC][0] = 127, + [1][0][0][0][RTW89_ETSI][0] = 127, + [1][0][0][0][RTW89_MKK][0] = 127, + [1][0][0][0][RTW89_IC][0] = 127, + [1][0][0][0][RTW89_KCC][0] = 127, + [1][0][0][0][RTW89_ACMA][0] = 127, + [1][0][0][0][RTW89_CHILE][0] = 127, + [1][0][0][0][RTW89_UKRAINE][0] = 127, + [1][0][0][0][RTW89_MEXICO][0] = 127, + [1][0][0][0][RTW89_CN][0] = 127, + [1][0][0][0][RTW89_QATAR][0] = 127, + [1][0][0][0][RTW89_FCC][1] = 127, + [1][0][0][0][RTW89_ETSI][1] = 127, + [1][0][0][0][RTW89_MKK][1] = 127, + [1][0][0][0][RTW89_IC][1] = 127, + [1][0][0][0][RTW89_KCC][1] = 127, + [1][0][0][0][RTW89_ACMA][1] = 127, + [1][0][0][0][RTW89_CHILE][1] = 127, + [1][0][0][0][RTW89_UKRAINE][1] = 127, + [1][0][0][0][RTW89_MEXICO][1] = 127, + [1][0][0][0][RTW89_CN][1] = 127, + [1][0][0][0][RTW89_QATAR][1] = 127, + [1][0][0][0][RTW89_FCC][2] = 60, + [1][0][0][0][RTW89_ETSI][2] = 58, + [1][0][0][0][RTW89_MKK][2] = 68, + [1][0][0][0][RTW89_IC][2] = 60, + [1][0][0][0][RTW89_KCC][2] = 70, + [1][0][0][0][RTW89_ACMA][2] = 58, + [1][0][0][0][RTW89_CHILE][2] = 60, + [1][0][0][0][RTW89_UKRAINE][2] = 58, + [1][0][0][0][RTW89_MEXICO][2] = 60, + [1][0][0][0][RTW89_CN][2] = 58, + [1][0][0][0][RTW89_QATAR][2] = 58, + [1][0][0][0][RTW89_FCC][3] = 60, + [1][0][0][0][RTW89_ETSI][3] = 58, + [1][0][0][0][RTW89_MKK][3] = 68, + [1][0][0][0][RTW89_IC][3] = 60, + [1][0][0][0][RTW89_KCC][3] = 70, + [1][0][0][0][RTW89_ACMA][3] = 58, + [1][0][0][0][RTW89_CHILE][3] = 60, + [1][0][0][0][RTW89_UKRAINE][3] = 58, + [1][0][0][0][RTW89_MEXICO][3] = 60, + [1][0][0][0][RTW89_CN][3] = 58, + [1][0][0][0][RTW89_QATAR][3] = 58, + [1][0][0][0][RTW89_FCC][4] = 60, + [1][0][0][0][RTW89_ETSI][4] = 58, + [1][0][0][0][RTW89_MKK][4] = 68, + [1][0][0][0][RTW89_IC][4] = 60, + [1][0][0][0][RTW89_KCC][4] = 70, + [1][0][0][0][RTW89_ACMA][4] = 58, + [1][0][0][0][RTW89_CHILE][4] = 60, + [1][0][0][0][RTW89_UKRAINE][4] = 58, + [1][0][0][0][RTW89_MEXICO][4] = 60, + [1][0][0][0][RTW89_CN][4] = 58, + [1][0][0][0][RTW89_QATAR][4] = 58, + [1][0][0][0][RTW89_FCC][5] = 60, + [1][0][0][0][RTW89_ETSI][5] = 58, + [1][0][0][0][RTW89_MKK][5] = 68, + [1][0][0][0][RTW89_IC][5] = 60, + [1][0][0][0][RTW89_KCC][5] = 70, + [1][0][0][0][RTW89_ACMA][5] = 58, + [1][0][0][0][RTW89_CHILE][5] = 60, + [1][0][0][0][RTW89_UKRAINE][5] = 58, + [1][0][0][0][RTW89_MEXICO][5] = 60, + [1][0][0][0][RTW89_CN][5] = 58, + [1][0][0][0][RTW89_QATAR][5] = 58, + [1][0][0][0][RTW89_FCC][6] = 46, + [1][0][0][0][RTW89_ETSI][6] = 58, + [1][0][0][0][RTW89_MKK][6] = 68, + [1][0][0][0][RTW89_IC][6] = 46, + [1][0][0][0][RTW89_KCC][6] = 70, + [1][0][0][0][RTW89_ACMA][6] = 58, + [1][0][0][0][RTW89_CHILE][6] = 46, + [1][0][0][0][RTW89_UKRAINE][6] = 58, + [1][0][0][0][RTW89_MEXICO][6] = 46, + [1][0][0][0][RTW89_CN][6] = 58, + [1][0][0][0][RTW89_QATAR][6] = 58, + [1][0][0][0][RTW89_FCC][7] = 46, + [1][0][0][0][RTW89_ETSI][7] = 58, + [1][0][0][0][RTW89_MKK][7] = 68, + [1][0][0][0][RTW89_IC][7] = 46, + [1][0][0][0][RTW89_KCC][7] = 70, + [1][0][0][0][RTW89_ACMA][7] = 58, + [1][0][0][0][RTW89_CHILE][7] = 46, + [1][0][0][0][RTW89_UKRAINE][7] = 58, + [1][0][0][0][RTW89_MEXICO][7] = 46, + [1][0][0][0][RTW89_CN][7] = 58, + [1][0][0][0][RTW89_QATAR][7] = 58, + [1][0][0][0][RTW89_FCC][8] = 46, + [1][0][0][0][RTW89_ETSI][8] = 58, + [1][0][0][0][RTW89_MKK][8] = 68, + [1][0][0][0][RTW89_IC][8] = 46, + [1][0][0][0][RTW89_KCC][8] = 70, + [1][0][0][0][RTW89_ACMA][8] = 58, + [1][0][0][0][RTW89_CHILE][8] = 46, + [1][0][0][0][RTW89_UKRAINE][8] = 58, + [1][0][0][0][RTW89_MEXICO][8] = 46, + [1][0][0][0][RTW89_CN][8] = 58, + [1][0][0][0][RTW89_QATAR][8] = 58, + [1][0][0][0][RTW89_FCC][9] = 32, + [1][0][0][0][RTW89_ETSI][9] = 58, + [1][0][0][0][RTW89_MKK][9] = 68, + [1][0][0][0][RTW89_IC][9] = 32, + [1][0][0][0][RTW89_KCC][9] = 70, + [1][0][0][0][RTW89_ACMA][9] = 58, + [1][0][0][0][RTW89_CHILE][9] = 32, + [1][0][0][0][RTW89_UKRAINE][9] = 58, + [1][0][0][0][RTW89_MEXICO][9] = 32, + [1][0][0][0][RTW89_CN][9] = 58, + [1][0][0][0][RTW89_QATAR][9] = 58, + [1][0][0][0][RTW89_FCC][10] = 32, + [1][0][0][0][RTW89_ETSI][10] = 58, + [1][0][0][0][RTW89_MKK][10] = 68, + [1][0][0][0][RTW89_IC][10] = 32, + [1][0][0][0][RTW89_KCC][10] = 70, + [1][0][0][0][RTW89_ACMA][10] = 58, + [1][0][0][0][RTW89_CHILE][10] = 32, + [1][0][0][0][RTW89_UKRAINE][10] = 58, + [1][0][0][0][RTW89_MEXICO][10] = 32, + [1][0][0][0][RTW89_CN][10] = 58, + [1][0][0][0][RTW89_QATAR][10] = 58, + [1][0][0][0][RTW89_FCC][11] = 127, + [1][0][0][0][RTW89_ETSI][11] = 127, + [1][0][0][0][RTW89_MKK][11] = 127, + [1][0][0][0][RTW89_IC][11] = 127, + [1][0][0][0][RTW89_KCC][11] = 127, + [1][0][0][0][RTW89_ACMA][11] = 127, + [1][0][0][0][RTW89_CHILE][11] = 127, + [1][0][0][0][RTW89_UKRAINE][11] = 127, + [1][0][0][0][RTW89_MEXICO][11] = 127, + [1][0][0][0][RTW89_CN][11] = 127, + [1][0][0][0][RTW89_QATAR][11] = 127, + [1][0][0][0][RTW89_FCC][12] = 127, + [1][0][0][0][RTW89_ETSI][12] = 127, + [1][0][0][0][RTW89_MKK][12] = 127, + [1][0][0][0][RTW89_IC][12] = 127, + [1][0][0][0][RTW89_KCC][12] = 127, + [1][0][0][0][RTW89_ACMA][12] = 127, + [1][0][0][0][RTW89_CHILE][12] = 127, + [1][0][0][0][RTW89_UKRAINE][12] = 127, + [1][0][0][0][RTW89_MEXICO][12] = 127, + [1][0][0][0][RTW89_CN][12] = 127, + [1][0][0][0][RTW89_QATAR][12] = 127, + [1][0][0][0][RTW89_FCC][13] = 127, + [1][0][0][0][RTW89_ETSI][13] = 127, + [1][0][0][0][RTW89_MKK][13] = 127, + [1][0][0][0][RTW89_IC][13] = 127, + [1][0][0][0][RTW89_KCC][13] = 127, + [1][0][0][0][RTW89_ACMA][13] = 127, + [1][0][0][0][RTW89_CHILE][13] = 127, + [1][0][0][0][RTW89_UKRAINE][13] = 127, + [1][0][0][0][RTW89_MEXICO][13] = 127, + [1][0][0][0][RTW89_CN][13] = 127, + [1][0][0][0][RTW89_QATAR][13] = 127, + [1][1][0][0][RTW89_FCC][0] = 127, + [1][1][0][0][RTW89_ETSI][0] = 127, + [1][1][0][0][RTW89_MKK][0] = 127, + [1][1][0][0][RTW89_IC][0] = 127, + [1][1][0][0][RTW89_KCC][0] = 127, + [1][1][0][0][RTW89_ACMA][0] = 127, + [1][1][0][0][RTW89_CHILE][0] = 127, + [1][1][0][0][RTW89_UKRAINE][0] = 127, + [1][1][0][0][RTW89_MEXICO][0] = 127, + [1][1][0][0][RTW89_CN][0] = 127, + [1][1][0][0][RTW89_QATAR][0] = 127, + [1][1][0][0][RTW89_FCC][1] = 127, + [1][1][0][0][RTW89_ETSI][1] = 127, + [1][1][0][0][RTW89_MKK][1] = 127, + [1][1][0][0][RTW89_IC][1] = 127, + [1][1][0][0][RTW89_KCC][1] = 127, + [1][1][0][0][RTW89_ACMA][1] = 127, + [1][1][0][0][RTW89_CHILE][1] = 127, + [1][1][0][0][RTW89_UKRAINE][1] = 127, + [1][1][0][0][RTW89_MEXICO][1] = 127, + [1][1][0][0][RTW89_CN][1] = 127, + [1][1][0][0][RTW89_QATAR][1] = 127, + [1][1][0][0][RTW89_FCC][2] = 48, + [1][1][0][0][RTW89_ETSI][2] = 46, + [1][1][0][0][RTW89_MKK][2] = 56, + [1][1][0][0][RTW89_IC][2] = 48, + [1][1][0][0][RTW89_KCC][2] = 58, + [1][1][0][0][RTW89_ACMA][2] = 46, + [1][1][0][0][RTW89_CHILE][2] = 48, + [1][1][0][0][RTW89_UKRAINE][2] = 46, + [1][1][0][0][RTW89_MEXICO][2] = 48, + [1][1][0][0][RTW89_CN][2] = 46, + [1][1][0][0][RTW89_QATAR][2] = 46, + [1][1][0][0][RTW89_FCC][3] = 48, + [1][1][0][0][RTW89_ETSI][3] = 46, + [1][1][0][0][RTW89_MKK][3] = 56, + [1][1][0][0][RTW89_IC][3] = 48, + [1][1][0][0][RTW89_KCC][3] = 58, + [1][1][0][0][RTW89_ACMA][3] = 46, + [1][1][0][0][RTW89_CHILE][3] = 48, + [1][1][0][0][RTW89_UKRAINE][3] = 46, + [1][1][0][0][RTW89_MEXICO][3] = 48, + [1][1][0][0][RTW89_CN][3] = 46, + [1][1][0][0][RTW89_QATAR][3] = 46, + [1][1][0][0][RTW89_FCC][4] = 48, + [1][1][0][0][RTW89_ETSI][4] = 46, + [1][1][0][0][RTW89_MKK][4] = 56, + [1][1][0][0][RTW89_IC][4] = 48, + [1][1][0][0][RTW89_KCC][4] = 58, + [1][1][0][0][RTW89_ACMA][4] = 46, + [1][1][0][0][RTW89_CHILE][4] = 48, + [1][1][0][0][RTW89_UKRAINE][4] = 46, + [1][1][0][0][RTW89_MEXICO][4] = 48, + [1][1][0][0][RTW89_CN][4] = 46, + [1][1][0][0][RTW89_QATAR][4] = 46, + [1][1][0][0][RTW89_FCC][5] = 58, + [1][1][0][0][RTW89_ETSI][5] = 46, + [1][1][0][0][RTW89_MKK][5] = 56, + [1][1][0][0][RTW89_IC][5] = 58, + [1][1][0][0][RTW89_KCC][5] = 58, + [1][1][0][0][RTW89_ACMA][5] = 46, + [1][1][0][0][RTW89_CHILE][5] = 48, + [1][1][0][0][RTW89_UKRAINE][5] = 46, + [1][1][0][0][RTW89_MEXICO][5] = 58, + [1][1][0][0][RTW89_CN][5] = 46, + [1][1][0][0][RTW89_QATAR][5] = 46, + [1][1][0][0][RTW89_FCC][6] = 46, + [1][1][0][0][RTW89_ETSI][6] = 46, + [1][1][0][0][RTW89_MKK][6] = 56, + [1][1][0][0][RTW89_IC][6] = 46, + [1][1][0][0][RTW89_KCC][6] = 58, + [1][1][0][0][RTW89_ACMA][6] = 46, + [1][1][0][0][RTW89_CHILE][6] = 46, + [1][1][0][0][RTW89_UKRAINE][6] = 46, + [1][1][0][0][RTW89_MEXICO][6] = 46, + [1][1][0][0][RTW89_CN][6] = 46, + [1][1][0][0][RTW89_QATAR][6] = 46, + [1][1][0][0][RTW89_FCC][7] = 46, + [1][1][0][0][RTW89_ETSI][7] = 46, + [1][1][0][0][RTW89_MKK][7] = 56, + [1][1][0][0][RTW89_IC][7] = 46, + [1][1][0][0][RTW89_KCC][7] = 58, + [1][1][0][0][RTW89_ACMA][7] = 46, + [1][1][0][0][RTW89_CHILE][7] = 46, + [1][1][0][0][RTW89_UKRAINE][7] = 46, + [1][1][0][0][RTW89_MEXICO][7] = 46, + [1][1][0][0][RTW89_CN][7] = 46, + [1][1][0][0][RTW89_QATAR][7] = 46, + [1][1][0][0][RTW89_FCC][8] = 46, + [1][1][0][0][RTW89_ETSI][8] = 46, + [1][1][0][0][RTW89_MKK][8] = 56, + [1][1][0][0][RTW89_IC][8] = 46, + [1][1][0][0][RTW89_KCC][8] = 58, + [1][1][0][0][RTW89_ACMA][8] = 46, + [1][1][0][0][RTW89_CHILE][8] = 46, + [1][1][0][0][RTW89_UKRAINE][8] = 46, + [1][1][0][0][RTW89_MEXICO][8] = 46, + [1][1][0][0][RTW89_CN][8] = 46, + [1][1][0][0][RTW89_QATAR][8] = 46, + [1][1][0][0][RTW89_FCC][9] = 24, + [1][1][0][0][RTW89_ETSI][9] = 46, + [1][1][0][0][RTW89_MKK][9] = 56, + [1][1][0][0][RTW89_IC][9] = 24, + [1][1][0][0][RTW89_KCC][9] = 58, + [1][1][0][0][RTW89_ACMA][9] = 46, + [1][1][0][0][RTW89_CHILE][9] = 24, + [1][1][0][0][RTW89_UKRAINE][9] = 46, + [1][1][0][0][RTW89_MEXICO][9] = 24, + [1][1][0][0][RTW89_CN][9] = 46, + [1][1][0][0][RTW89_QATAR][9] = 46, + [1][1][0][0][RTW89_FCC][10] = 24, + [1][1][0][0][RTW89_ETSI][10] = 46, + [1][1][0][0][RTW89_MKK][10] = 56, + [1][1][0][0][RTW89_IC][10] = 24, + [1][1][0][0][RTW89_KCC][10] = 58, + [1][1][0][0][RTW89_ACMA][10] = 46, + [1][1][0][0][RTW89_CHILE][10] = 24, + [1][1][0][0][RTW89_UKRAINE][10] = 46, + [1][1][0][0][RTW89_MEXICO][10] = 24, + [1][1][0][0][RTW89_CN][10] = 46, + [1][1][0][0][RTW89_QATAR][10] = 46, + [1][1][0][0][RTW89_FCC][11] = 127, + [1][1][0][0][RTW89_ETSI][11] = 127, + [1][1][0][0][RTW89_MKK][11] = 127, + [1][1][0][0][RTW89_IC][11] = 127, + [1][1][0][0][RTW89_KCC][11] = 127, + [1][1][0][0][RTW89_ACMA][11] = 127, + [1][1][0][0][RTW89_CHILE][11] = 127, + [1][1][0][0][RTW89_UKRAINE][11] = 127, + [1][1][0][0][RTW89_MEXICO][11] = 127, + [1][1][0][0][RTW89_CN][11] = 127, + [1][1][0][0][RTW89_QATAR][11] = 127, + [1][1][0][0][RTW89_FCC][12] = 127, + [1][1][0][0][RTW89_ETSI][12] = 127, + [1][1][0][0][RTW89_MKK][12] = 127, + [1][1][0][0][RTW89_IC][12] = 127, + [1][1][0][0][RTW89_KCC][12] = 127, + [1][1][0][0][RTW89_ACMA][12] = 127, + [1][1][0][0][RTW89_CHILE][12] = 127, + [1][1][0][0][RTW89_UKRAINE][12] = 127, + [1][1][0][0][RTW89_MEXICO][12] = 127, + [1][1][0][0][RTW89_CN][12] = 127, + [1][1][0][0][RTW89_QATAR][12] = 127, + [1][1][0][0][RTW89_FCC][13] = 127, + [1][1][0][0][RTW89_ETSI][13] = 127, + [1][1][0][0][RTW89_MKK][13] = 127, + [1][1][0][0][RTW89_IC][13] = 127, + [1][1][0][0][RTW89_KCC][13] = 127, + [1][1][0][0][RTW89_ACMA][13] = 127, + [1][1][0][0][RTW89_CHILE][13] = 127, + [1][1][0][0][RTW89_UKRAINE][13] = 127, + [1][1][0][0][RTW89_MEXICO][13] = 127, + [1][1][0][0][RTW89_CN][13] = 127, + [1][1][0][0][RTW89_QATAR][13] = 127, + [0][0][1][0][RTW89_FCC][0] = 66, + [0][0][1][0][RTW89_ETSI][0] = 58, + [0][0][1][0][RTW89_MKK][0] = 76, + [0][0][1][0][RTW89_IC][0] = 66, + [0][0][1][0][RTW89_KCC][0] = 76, + [0][0][1][0][RTW89_ACMA][0] = 58, + [0][0][1][0][RTW89_CHILE][0] = 60, + [0][0][1][0][RTW89_UKRAINE][0] = 58, + [0][0][1][0][RTW89_MEXICO][0] = 66, + [0][0][1][0][RTW89_CN][0] = 58, + [0][0][1][0][RTW89_QATAR][0] = 58, + [0][0][1][0][RTW89_FCC][1] = 66, + [0][0][1][0][RTW89_ETSI][1] = 58, + [0][0][1][0][RTW89_MKK][1] = 76, + [0][0][1][0][RTW89_IC][1] = 66, + [0][0][1][0][RTW89_KCC][1] = 76, + [0][0][1][0][RTW89_ACMA][1] = 58, + [0][0][1][0][RTW89_CHILE][1] = 60, + [0][0][1][0][RTW89_UKRAINE][1] = 58, + [0][0][1][0][RTW89_MEXICO][1] = 66, + [0][0][1][0][RTW89_CN][1] = 58, + [0][0][1][0][RTW89_QATAR][1] = 58, + [0][0][1][0][RTW89_FCC][2] = 70, + [0][0][1][0][RTW89_ETSI][2] = 58, + [0][0][1][0][RTW89_MKK][2] = 76, + [0][0][1][0][RTW89_IC][2] = 70, + [0][0][1][0][RTW89_KCC][2] = 76, + [0][0][1][0][RTW89_ACMA][2] = 58, + [0][0][1][0][RTW89_CHILE][2] = 60, + [0][0][1][0][RTW89_UKRAINE][2] = 58, + [0][0][1][0][RTW89_MEXICO][2] = 70, + [0][0][1][0][RTW89_CN][2] = 58, + [0][0][1][0][RTW89_QATAR][2] = 58, + [0][0][1][0][RTW89_FCC][3] = 74, + [0][0][1][0][RTW89_ETSI][3] = 58, + [0][0][1][0][RTW89_MKK][3] = 76, + [0][0][1][0][RTW89_IC][3] = 74, + [0][0][1][0][RTW89_KCC][3] = 76, + [0][0][1][0][RTW89_ACMA][3] = 58, + [0][0][1][0][RTW89_CHILE][3] = 60, + [0][0][1][0][RTW89_UKRAINE][3] = 58, + [0][0][1][0][RTW89_MEXICO][3] = 74, + [0][0][1][0][RTW89_CN][3] = 58, + [0][0][1][0][RTW89_QATAR][3] = 58, + [0][0][1][0][RTW89_FCC][4] = 78, + [0][0][1][0][RTW89_ETSI][4] = 58, + [0][0][1][0][RTW89_MKK][4] = 76, + [0][0][1][0][RTW89_IC][4] = 78, + [0][0][1][0][RTW89_KCC][4] = 76, + [0][0][1][0][RTW89_ACMA][4] = 58, + [0][0][1][0][RTW89_CHILE][4] = 60, + [0][0][1][0][RTW89_UKRAINE][4] = 58, + [0][0][1][0][RTW89_MEXICO][4] = 78, + [0][0][1][0][RTW89_CN][4] = 58, + [0][0][1][0][RTW89_QATAR][4] = 58, + [0][0][1][0][RTW89_FCC][5] = 78, + [0][0][1][0][RTW89_ETSI][5] = 58, + [0][0][1][0][RTW89_MKK][5] = 76, + [0][0][1][0][RTW89_IC][5] = 78, + [0][0][1][0][RTW89_KCC][5] = 76, + [0][0][1][0][RTW89_ACMA][5] = 58, + [0][0][1][0][RTW89_CHILE][5] = 60, + [0][0][1][0][RTW89_UKRAINE][5] = 58, + [0][0][1][0][RTW89_MEXICO][5] = 78, + [0][0][1][0][RTW89_CN][5] = 58, + [0][0][1][0][RTW89_QATAR][5] = 58, + [0][0][1][0][RTW89_FCC][6] = 78, + [0][0][1][0][RTW89_ETSI][6] = 58, + [0][0][1][0][RTW89_MKK][6] = 76, + [0][0][1][0][RTW89_IC][6] = 78, + [0][0][1][0][RTW89_KCC][6] = 76, + [0][0][1][0][RTW89_ACMA][6] = 58, + [0][0][1][0][RTW89_CHILE][6] = 60, + [0][0][1][0][RTW89_UKRAINE][6] = 58, + [0][0][1][0][RTW89_MEXICO][6] = 78, + [0][0][1][0][RTW89_CN][6] = 58, + [0][0][1][0][RTW89_QATAR][6] = 58, + [0][0][1][0][RTW89_FCC][7] = 74, + [0][0][1][0][RTW89_ETSI][7] = 58, + [0][0][1][0][RTW89_MKK][7] = 76, + [0][0][1][0][RTW89_IC][7] = 74, + [0][0][1][0][RTW89_KCC][7] = 76, + [0][0][1][0][RTW89_ACMA][7] = 58, + [0][0][1][0][RTW89_CHILE][7] = 60, + [0][0][1][0][RTW89_UKRAINE][7] = 58, + [0][0][1][0][RTW89_MEXICO][7] = 74, + [0][0][1][0][RTW89_CN][7] = 58, + [0][0][1][0][RTW89_QATAR][7] = 58, + [0][0][1][0][RTW89_FCC][8] = 70, + [0][0][1][0][RTW89_ETSI][8] = 58, + [0][0][1][0][RTW89_MKK][8] = 76, + [0][0][1][0][RTW89_IC][8] = 70, + [0][0][1][0][RTW89_KCC][8] = 76, + [0][0][1][0][RTW89_ACMA][8] = 58, + [0][0][1][0][RTW89_CHILE][8] = 60, + [0][0][1][0][RTW89_UKRAINE][8] = 58, + [0][0][1][0][RTW89_MEXICO][8] = 70, + [0][0][1][0][RTW89_CN][8] = 58, + [0][0][1][0][RTW89_QATAR][8] = 58, + [0][0][1][0][RTW89_FCC][9] = 66, + [0][0][1][0][RTW89_ETSI][9] = 58, + [0][0][1][0][RTW89_MKK][9] = 76, + [0][0][1][0][RTW89_IC][9] = 66, + [0][0][1][0][RTW89_KCC][9] = 76, + [0][0][1][0][RTW89_ACMA][9] = 58, + [0][0][1][0][RTW89_CHILE][9] = 60, + [0][0][1][0][RTW89_UKRAINE][9] = 58, + [0][0][1][0][RTW89_MEXICO][9] = 66, + [0][0][1][0][RTW89_CN][9] = 58, + [0][0][1][0][RTW89_QATAR][9] = 58, + [0][0][1][0][RTW89_FCC][10] = 66, + [0][0][1][0][RTW89_ETSI][10] = 58, + [0][0][1][0][RTW89_MKK][10] = 76, + [0][0][1][0][RTW89_IC][10] = 66, + [0][0][1][0][RTW89_KCC][10] = 76, + [0][0][1][0][RTW89_ACMA][10] = 58, + [0][0][1][0][RTW89_CHILE][10] = 60, + [0][0][1][0][RTW89_UKRAINE][10] = 58, + [0][0][1][0][RTW89_MEXICO][10] = 66, + [0][0][1][0][RTW89_CN][10] = 58, + [0][0][1][0][RTW89_QATAR][10] = 58, + [0][0][1][0][RTW89_FCC][11] = 56, + [0][0][1][0][RTW89_ETSI][11] = 58, + [0][0][1][0][RTW89_MKK][11] = 76, + [0][0][1][0][RTW89_IC][11] = 56, + [0][0][1][0][RTW89_KCC][11] = 76, + [0][0][1][0][RTW89_ACMA][11] = 58, + [0][0][1][0][RTW89_CHILE][11] = 56, + [0][0][1][0][RTW89_UKRAINE][11] = 58, + [0][0][1][0][RTW89_MEXICO][11] = 56, + [0][0][1][0][RTW89_CN][11] = 58, + [0][0][1][0][RTW89_QATAR][11] = 58, + [0][0][1][0][RTW89_FCC][12] = 52, + [0][0][1][0][RTW89_ETSI][12] = 58, + [0][0][1][0][RTW89_MKK][12] = 76, + [0][0][1][0][RTW89_IC][12] = 52, + [0][0][1][0][RTW89_KCC][12] = 76, + [0][0][1][0][RTW89_ACMA][12] = 58, + [0][0][1][0][RTW89_CHILE][12] = 52, + [0][0][1][0][RTW89_UKRAINE][12] = 58, + [0][0][1][0][RTW89_MEXICO][12] = 52, + [0][0][1][0][RTW89_CN][12] = 58, + [0][0][1][0][RTW89_QATAR][12] = 58, + [0][0][1][0][RTW89_FCC][13] = 127, + [0][0][1][0][RTW89_ETSI][13] = 127, + [0][0][1][0][RTW89_MKK][13] = 127, + [0][0][1][0][RTW89_IC][13] = 127, + [0][0][1][0][RTW89_KCC][13] = 127, + [0][0][1][0][RTW89_ACMA][13] = 127, + [0][0][1][0][RTW89_CHILE][13] = 127, + [0][0][1][0][RTW89_UKRAINE][13] = 127, + [0][0][1][0][RTW89_MEXICO][13] = 127, + [0][0][1][0][RTW89_CN][13] = 127, + [0][0][1][0][RTW89_QATAR][13] = 127, + [0][1][1][0][RTW89_FCC][0] = 62, + [0][1][1][0][RTW89_ETSI][0] = 46, + [0][1][1][0][RTW89_MKK][0] = 64, + [0][1][1][0][RTW89_IC][0] = 62, + [0][1][1][0][RTW89_KCC][0] = 70, + [0][1][1][0][RTW89_ACMA][0] = 46, + [0][1][1][0][RTW89_CHILE][0] = 48, + [0][1][1][0][RTW89_UKRAINE][0] = 46, + [0][1][1][0][RTW89_MEXICO][0] = 62, + [0][1][1][0][RTW89_CN][0] = 46, + [0][1][1][0][RTW89_QATAR][0] = 46, + [0][1][1][0][RTW89_FCC][1] = 62, + [0][1][1][0][RTW89_ETSI][1] = 46, + [0][1][1][0][RTW89_MKK][1] = 64, + [0][1][1][0][RTW89_IC][1] = 62, + [0][1][1][0][RTW89_KCC][1] = 70, + [0][1][1][0][RTW89_ACMA][1] = 46, + [0][1][1][0][RTW89_CHILE][1] = 48, + [0][1][1][0][RTW89_UKRAINE][1] = 46, + [0][1][1][0][RTW89_MEXICO][1] = 62, + [0][1][1][0][RTW89_CN][1] = 46, + [0][1][1][0][RTW89_QATAR][1] = 46, + [0][1][1][0][RTW89_FCC][2] = 66, + [0][1][1][0][RTW89_ETSI][2] = 46, + [0][1][1][0][RTW89_MKK][2] = 64, + [0][1][1][0][RTW89_IC][2] = 66, + [0][1][1][0][RTW89_KCC][2] = 70, + [0][1][1][0][RTW89_ACMA][2] = 46, + [0][1][1][0][RTW89_CHILE][2] = 48, + [0][1][1][0][RTW89_UKRAINE][2] = 46, + [0][1][1][0][RTW89_MEXICO][2] = 66, + [0][1][1][0][RTW89_CN][2] = 46, + [0][1][1][0][RTW89_QATAR][2] = 46, + [0][1][1][0][RTW89_FCC][3] = 70, + [0][1][1][0][RTW89_ETSI][3] = 46, + [0][1][1][0][RTW89_MKK][3] = 64, + [0][1][1][0][RTW89_IC][3] = 70, + [0][1][1][0][RTW89_KCC][3] = 70, + [0][1][1][0][RTW89_ACMA][3] = 46, + [0][1][1][0][RTW89_CHILE][3] = 48, + [0][1][1][0][RTW89_UKRAINE][3] = 46, + [0][1][1][0][RTW89_MEXICO][3] = 70, + [0][1][1][0][RTW89_CN][3] = 46, + [0][1][1][0][RTW89_QATAR][3] = 46, + [0][1][1][0][RTW89_FCC][4] = 78, + [0][1][1][0][RTW89_ETSI][4] = 46, + [0][1][1][0][RTW89_MKK][4] = 64, + [0][1][1][0][RTW89_IC][4] = 78, + [0][1][1][0][RTW89_KCC][4] = 70, + [0][1][1][0][RTW89_ACMA][4] = 46, + [0][1][1][0][RTW89_CHILE][4] = 48, + [0][1][1][0][RTW89_UKRAINE][4] = 46, + [0][1][1][0][RTW89_MEXICO][4] = 78, + [0][1][1][0][RTW89_CN][4] = 46, + [0][1][1][0][RTW89_QATAR][4] = 46, + [0][1][1][0][RTW89_FCC][5] = 78, + [0][1][1][0][RTW89_ETSI][5] = 46, + [0][1][1][0][RTW89_MKK][5] = 64, + [0][1][1][0][RTW89_IC][5] = 78, + [0][1][1][0][RTW89_KCC][5] = 70, + [0][1][1][0][RTW89_ACMA][5] = 46, + [0][1][1][0][RTW89_CHILE][5] = 48, + [0][1][1][0][RTW89_UKRAINE][5] = 46, + [0][1][1][0][RTW89_MEXICO][5] = 78, + [0][1][1][0][RTW89_CN][5] = 46, + [0][1][1][0][RTW89_QATAR][5] = 46, + [0][1][1][0][RTW89_FCC][6] = 78, + [0][1][1][0][RTW89_ETSI][6] = 46, + [0][1][1][0][RTW89_MKK][6] = 64, + [0][1][1][0][RTW89_IC][6] = 78, + [0][1][1][0][RTW89_KCC][6] = 70, + [0][1][1][0][RTW89_ACMA][6] = 46, + [0][1][1][0][RTW89_CHILE][6] = 48, + [0][1][1][0][RTW89_UKRAINE][6] = 46, + [0][1][1][0][RTW89_MEXICO][6] = 78, + [0][1][1][0][RTW89_CN][6] = 46, + [0][1][1][0][RTW89_QATAR][6] = 46, + [0][1][1][0][RTW89_FCC][7] = 70, + [0][1][1][0][RTW89_ETSI][7] = 46, + [0][1][1][0][RTW89_MKK][7] = 64, + [0][1][1][0][RTW89_IC][7] = 70, + [0][1][1][0][RTW89_KCC][7] = 70, + [0][1][1][0][RTW89_ACMA][7] = 46, + [0][1][1][0][RTW89_CHILE][7] = 48, + [0][1][1][0][RTW89_UKRAINE][7] = 46, + [0][1][1][0][RTW89_MEXICO][7] = 70, + [0][1][1][0][RTW89_CN][7] = 46, + [0][1][1][0][RTW89_QATAR][7] = 46, + [0][1][1][0][RTW89_FCC][8] = 66, + [0][1][1][0][RTW89_ETSI][8] = 46, + [0][1][1][0][RTW89_MKK][8] = 64, + [0][1][1][0][RTW89_IC][8] = 66, + [0][1][1][0][RTW89_KCC][8] = 70, + [0][1][1][0][RTW89_ACMA][8] = 46, + [0][1][1][0][RTW89_CHILE][8] = 48, + [0][1][1][0][RTW89_UKRAINE][8] = 46, + [0][1][1][0][RTW89_MEXICO][8] = 66, + [0][1][1][0][RTW89_CN][8] = 46, + [0][1][1][0][RTW89_QATAR][8] = 46, + [0][1][1][0][RTW89_FCC][9] = 62, + [0][1][1][0][RTW89_ETSI][9] = 46, + [0][1][1][0][RTW89_MKK][9] = 64, + [0][1][1][0][RTW89_IC][9] = 62, + [0][1][1][0][RTW89_KCC][9] = 70, + [0][1][1][0][RTW89_ACMA][9] = 46, + [0][1][1][0][RTW89_CHILE][9] = 48, + [0][1][1][0][RTW89_UKRAINE][9] = 46, + [0][1][1][0][RTW89_MEXICO][9] = 62, + [0][1][1][0][RTW89_CN][9] = 46, + [0][1][1][0][RTW89_QATAR][9] = 46, + [0][1][1][0][RTW89_FCC][10] = 62, + [0][1][1][0][RTW89_ETSI][10] = 46, + [0][1][1][0][RTW89_MKK][10] = 64, + [0][1][1][0][RTW89_IC][10] = 62, + [0][1][1][0][RTW89_KCC][10] = 70, + [0][1][1][0][RTW89_ACMA][10] = 46, + [0][1][1][0][RTW89_CHILE][10] = 48, + [0][1][1][0][RTW89_UKRAINE][10] = 46, + [0][1][1][0][RTW89_MEXICO][10] = 62, + [0][1][1][0][RTW89_CN][10] = 46, + [0][1][1][0][RTW89_QATAR][10] = 46, + [0][1][1][0][RTW89_FCC][11] = 42, + [0][1][1][0][RTW89_ETSI][11] = 46, + [0][1][1][0][RTW89_MKK][11] = 64, + [0][1][1][0][RTW89_IC][11] = 42, + [0][1][1][0][RTW89_KCC][11] = 70, + [0][1][1][0][RTW89_ACMA][11] = 46, + [0][1][1][0][RTW89_CHILE][11] = 42, + [0][1][1][0][RTW89_UKRAINE][11] = 46, + [0][1][1][0][RTW89_MEXICO][11] = 42, + [0][1][1][0][RTW89_CN][11] = 46, + [0][1][1][0][RTW89_QATAR][11] = 46, + [0][1][1][0][RTW89_FCC][12] = 40, + [0][1][1][0][RTW89_ETSI][12] = 46, + [0][1][1][0][RTW89_MKK][12] = 64, + [0][1][1][0][RTW89_IC][12] = 40, + [0][1][1][0][RTW89_KCC][12] = 70, + [0][1][1][0][RTW89_ACMA][12] = 46, + [0][1][1][0][RTW89_CHILE][12] = 40, + [0][1][1][0][RTW89_UKRAINE][12] = 46, + [0][1][1][0][RTW89_MEXICO][12] = 40, + [0][1][1][0][RTW89_CN][12] = 46, + [0][1][1][0][RTW89_QATAR][12] = 46, + [0][1][1][0][RTW89_FCC][13] = 127, + [0][1][1][0][RTW89_ETSI][13] = 127, + [0][1][1][0][RTW89_MKK][13] = 127, + [0][1][1][0][RTW89_IC][13] = 127, + [0][1][1][0][RTW89_KCC][13] = 127, + [0][1][1][0][RTW89_ACMA][13] = 127, + [0][1][1][0][RTW89_CHILE][13] = 127, + [0][1][1][0][RTW89_UKRAINE][13] = 127, + [0][1][1][0][RTW89_MEXICO][13] = 127, + [0][1][1][0][RTW89_CN][13] = 127, + [0][1][1][0][RTW89_QATAR][13] = 127, + [0][0][2][0][RTW89_FCC][0] = 66, + [0][0][2][0][RTW89_ETSI][0] = 58, + [0][0][2][0][RTW89_MKK][0] = 76, + [0][0][2][0][RTW89_IC][0] = 66, + [0][0][2][0][RTW89_KCC][0] = 76, + [0][0][2][0][RTW89_ACMA][0] = 58, + [0][0][2][0][RTW89_CHILE][0] = 60, + [0][0][2][0][RTW89_UKRAINE][0] = 58, + [0][0][2][0][RTW89_MEXICO][0] = 66, + [0][0][2][0][RTW89_CN][0] = 58, + [0][0][2][0][RTW89_QATAR][0] = 58, + [0][0][2][0][RTW89_FCC][1] = 66, + [0][0][2][0][RTW89_ETSI][1] = 58, + [0][0][2][0][RTW89_MKK][1] = 76, + [0][0][2][0][RTW89_IC][1] = 66, + [0][0][2][0][RTW89_KCC][1] = 76, + [0][0][2][0][RTW89_ACMA][1] = 58, + [0][0][2][0][RTW89_CHILE][1] = 60, + [0][0][2][0][RTW89_UKRAINE][1] = 58, + [0][0][2][0][RTW89_MEXICO][1] = 66, + [0][0][2][0][RTW89_CN][1] = 58, + [0][0][2][0][RTW89_QATAR][1] = 58, + [0][0][2][0][RTW89_FCC][2] = 70, + [0][0][2][0][RTW89_ETSI][2] = 58, + [0][0][2][0][RTW89_MKK][2] = 76, + [0][0][2][0][RTW89_IC][2] = 70, + [0][0][2][0][RTW89_KCC][2] = 76, + [0][0][2][0][RTW89_ACMA][2] = 58, + [0][0][2][0][RTW89_CHILE][2] = 60, + [0][0][2][0][RTW89_UKRAINE][2] = 58, + [0][0][2][0][RTW89_MEXICO][2] = 70, + [0][0][2][0][RTW89_CN][2] = 58, + [0][0][2][0][RTW89_QATAR][2] = 58, + [0][0][2][0][RTW89_FCC][3] = 74, + [0][0][2][0][RTW89_ETSI][3] = 58, + [0][0][2][0][RTW89_MKK][3] = 76, + [0][0][2][0][RTW89_IC][3] = 74, + [0][0][2][0][RTW89_KCC][3] = 76, + [0][0][2][0][RTW89_ACMA][3] = 58, + [0][0][2][0][RTW89_CHILE][3] = 60, + [0][0][2][0][RTW89_UKRAINE][3] = 58, + [0][0][2][0][RTW89_MEXICO][3] = 74, + [0][0][2][0][RTW89_CN][3] = 58, + [0][0][2][0][RTW89_QATAR][3] = 58, + [0][0][2][0][RTW89_FCC][4] = 76, + [0][0][2][0][RTW89_ETSI][4] = 58, + [0][0][2][0][RTW89_MKK][4] = 76, + [0][0][2][0][RTW89_IC][4] = 76, + [0][0][2][0][RTW89_KCC][4] = 76, + [0][0][2][0][RTW89_ACMA][4] = 58, + [0][0][2][0][RTW89_CHILE][4] = 60, + [0][0][2][0][RTW89_UKRAINE][4] = 58, + [0][0][2][0][RTW89_MEXICO][4] = 76, + [0][0][2][0][RTW89_CN][4] = 58, + [0][0][2][0][RTW89_QATAR][4] = 58, + [0][0][2][0][RTW89_FCC][5] = 76, + [0][0][2][0][RTW89_ETSI][5] = 58, + [0][0][2][0][RTW89_MKK][5] = 76, + [0][0][2][0][RTW89_IC][5] = 76, + [0][0][2][0][RTW89_KCC][5] = 76, + [0][0][2][0][RTW89_ACMA][5] = 58, + [0][0][2][0][RTW89_CHILE][5] = 60, + [0][0][2][0][RTW89_UKRAINE][5] = 58, + [0][0][2][0][RTW89_MEXICO][5] = 76, + [0][0][2][0][RTW89_CN][5] = 58, + [0][0][2][0][RTW89_QATAR][5] = 58, + [0][0][2][0][RTW89_FCC][6] = 76, + [0][0][2][0][RTW89_ETSI][6] = 58, + [0][0][2][0][RTW89_MKK][6] = 76, + [0][0][2][0][RTW89_IC][6] = 76, + [0][0][2][0][RTW89_KCC][6] = 76, + [0][0][2][0][RTW89_ACMA][6] = 58, + [0][0][2][0][RTW89_CHILE][6] = 60, + [0][0][2][0][RTW89_UKRAINE][6] = 58, + [0][0][2][0][RTW89_MEXICO][6] = 76, + [0][0][2][0][RTW89_CN][6] = 58, + [0][0][2][0][RTW89_QATAR][6] = 58, + [0][0][2][0][RTW89_FCC][7] = 74, + [0][0][2][0][RTW89_ETSI][7] = 58, + [0][0][2][0][RTW89_MKK][7] = 76, + [0][0][2][0][RTW89_IC][7] = 74, + [0][0][2][0][RTW89_KCC][7] = 76, + [0][0][2][0][RTW89_ACMA][7] = 58, + [0][0][2][0][RTW89_CHILE][7] = 60, + [0][0][2][0][RTW89_UKRAINE][7] = 58, + [0][0][2][0][RTW89_MEXICO][7] = 74, + [0][0][2][0][RTW89_CN][7] = 58, + [0][0][2][0][RTW89_QATAR][7] = 58, + [0][0][2][0][RTW89_FCC][8] = 70, + [0][0][2][0][RTW89_ETSI][8] = 58, + [0][0][2][0][RTW89_MKK][8] = 76, + [0][0][2][0][RTW89_IC][8] = 70, + [0][0][2][0][RTW89_KCC][8] = 76, + [0][0][2][0][RTW89_ACMA][8] = 58, + [0][0][2][0][RTW89_CHILE][8] = 60, + [0][0][2][0][RTW89_UKRAINE][8] = 58, + [0][0][2][0][RTW89_MEXICO][8] = 70, + [0][0][2][0][RTW89_CN][8] = 58, + [0][0][2][0][RTW89_QATAR][8] = 58, + [0][0][2][0][RTW89_FCC][9] = 66, + [0][0][2][0][RTW89_ETSI][9] = 58, + [0][0][2][0][RTW89_MKK][9] = 76, + [0][0][2][0][RTW89_IC][9] = 66, + [0][0][2][0][RTW89_KCC][9] = 76, + [0][0][2][0][RTW89_ACMA][9] = 58, + [0][0][2][0][RTW89_CHILE][9] = 60, + [0][0][2][0][RTW89_UKRAINE][9] = 58, + [0][0][2][0][RTW89_MEXICO][9] = 66, + [0][0][2][0][RTW89_CN][9] = 58, + [0][0][2][0][RTW89_QATAR][9] = 58, + [0][0][2][0][RTW89_FCC][10] = 66, + [0][0][2][0][RTW89_ETSI][10] = 58, + [0][0][2][0][RTW89_MKK][10] = 76, + [0][0][2][0][RTW89_IC][10] = 66, + [0][0][2][0][RTW89_KCC][10] = 76, + [0][0][2][0][RTW89_ACMA][10] = 58, + [0][0][2][0][RTW89_CHILE][10] = 60, + [0][0][2][0][RTW89_UKRAINE][10] = 58, + [0][0][2][0][RTW89_MEXICO][10] = 66, + [0][0][2][0][RTW89_CN][10] = 58, + [0][0][2][0][RTW89_QATAR][10] = 58, + [0][0][2][0][RTW89_FCC][11] = 54, + [0][0][2][0][RTW89_ETSI][11] = 58, + [0][0][2][0][RTW89_MKK][11] = 76, + [0][0][2][0][RTW89_IC][11] = 54, + [0][0][2][0][RTW89_KCC][11] = 76, + [0][0][2][0][RTW89_ACMA][11] = 58, + [0][0][2][0][RTW89_CHILE][11] = 54, + [0][0][2][0][RTW89_UKRAINE][11] = 58, + [0][0][2][0][RTW89_MEXICO][11] = 54, + [0][0][2][0][RTW89_CN][11] = 58, + [0][0][2][0][RTW89_QATAR][11] = 58, + [0][0][2][0][RTW89_FCC][12] = 50, + [0][0][2][0][RTW89_ETSI][12] = 58, + [0][0][2][0][RTW89_MKK][12] = 76, + [0][0][2][0][RTW89_IC][12] = 50, + [0][0][2][0][RTW89_KCC][12] = 74, + [0][0][2][0][RTW89_ACMA][12] = 58, + [0][0][2][0][RTW89_CHILE][12] = 50, + [0][0][2][0][RTW89_UKRAINE][12] = 58, + [0][0][2][0][RTW89_MEXICO][12] = 50, + [0][0][2][0][RTW89_CN][12] = 58, + [0][0][2][0][RTW89_QATAR][12] = 58, + [0][0][2][0][RTW89_FCC][13] = 127, + [0][0][2][0][RTW89_ETSI][13] = 127, + [0][0][2][0][RTW89_MKK][13] = 127, + [0][0][2][0][RTW89_IC][13] = 127, + [0][0][2][0][RTW89_KCC][13] = 127, + [0][0][2][0][RTW89_ACMA][13] = 127, + [0][0][2][0][RTW89_CHILE][13] = 127, + [0][0][2][0][RTW89_UKRAINE][13] = 127, + [0][0][2][0][RTW89_MEXICO][13] = 127, + [0][0][2][0][RTW89_CN][13] = 127, + [0][0][2][0][RTW89_QATAR][13] = 127, + [0][1][2][0][RTW89_FCC][0] = 62, + [0][1][2][0][RTW89_ETSI][0] = 46, + [0][1][2][0][RTW89_MKK][0] = 64, + [0][1][2][0][RTW89_IC][0] = 62, + [0][1][2][0][RTW89_KCC][0] = 68, + [0][1][2][0][RTW89_ACMA][0] = 46, + [0][1][2][0][RTW89_CHILE][0] = 48, + [0][1][2][0][RTW89_UKRAINE][0] = 46, + [0][1][2][0][RTW89_MEXICO][0] = 62, + [0][1][2][0][RTW89_CN][0] = 46, + [0][1][2][0][RTW89_QATAR][0] = 46, + [0][1][2][0][RTW89_FCC][1] = 62, + [0][1][2][0][RTW89_ETSI][1] = 46, + [0][1][2][0][RTW89_MKK][1] = 64, + [0][1][2][0][RTW89_IC][1] = 62, + [0][1][2][0][RTW89_KCC][1] = 70, + [0][1][2][0][RTW89_ACMA][1] = 46, + [0][1][2][0][RTW89_CHILE][1] = 48, + [0][1][2][0][RTW89_UKRAINE][1] = 46, + [0][1][2][0][RTW89_MEXICO][1] = 62, + [0][1][2][0][RTW89_CN][1] = 46, + [0][1][2][0][RTW89_QATAR][1] = 46, + [0][1][2][0][RTW89_FCC][2] = 66, + [0][1][2][0][RTW89_ETSI][2] = 46, + [0][1][2][0][RTW89_MKK][2] = 64, + [0][1][2][0][RTW89_IC][2] = 66, + [0][1][2][0][RTW89_KCC][2] = 70, + [0][1][2][0][RTW89_ACMA][2] = 46, + [0][1][2][0][RTW89_CHILE][2] = 48, + [0][1][2][0][RTW89_UKRAINE][2] = 46, + [0][1][2][0][RTW89_MEXICO][2] = 66, + [0][1][2][0][RTW89_CN][2] = 46, + [0][1][2][0][RTW89_QATAR][2] = 46, + [0][1][2][0][RTW89_FCC][3] = 70, + [0][1][2][0][RTW89_ETSI][3] = 46, + [0][1][2][0][RTW89_MKK][3] = 64, + [0][1][2][0][RTW89_IC][3] = 70, + [0][1][2][0][RTW89_KCC][3] = 70, + [0][1][2][0][RTW89_ACMA][3] = 46, + [0][1][2][0][RTW89_CHILE][3] = 48, + [0][1][2][0][RTW89_UKRAINE][3] = 46, + [0][1][2][0][RTW89_MEXICO][3] = 70, + [0][1][2][0][RTW89_CN][3] = 46, + [0][1][2][0][RTW89_QATAR][3] = 46, + [0][1][2][0][RTW89_FCC][4] = 76, + [0][1][2][0][RTW89_ETSI][4] = 46, + [0][1][2][0][RTW89_MKK][4] = 64, + [0][1][2][0][RTW89_IC][4] = 76, + [0][1][2][0][RTW89_KCC][4] = 70, + [0][1][2][0][RTW89_ACMA][4] = 46, + [0][1][2][0][RTW89_CHILE][4] = 48, + [0][1][2][0][RTW89_UKRAINE][4] = 46, + [0][1][2][0][RTW89_MEXICO][4] = 76, + [0][1][2][0][RTW89_CN][4] = 46, + [0][1][2][0][RTW89_QATAR][4] = 46, + [0][1][2][0][RTW89_FCC][5] = 76, + [0][1][2][0][RTW89_ETSI][5] = 46, + [0][1][2][0][RTW89_MKK][5] = 64, + [0][1][2][0][RTW89_IC][5] = 76, + [0][1][2][0][RTW89_KCC][5] = 70, + [0][1][2][0][RTW89_ACMA][5] = 46, + [0][1][2][0][RTW89_CHILE][5] = 48, + [0][1][2][0][RTW89_UKRAINE][5] = 46, + [0][1][2][0][RTW89_MEXICO][5] = 76, + [0][1][2][0][RTW89_CN][5] = 46, + [0][1][2][0][RTW89_QATAR][5] = 46, + [0][1][2][0][RTW89_FCC][6] = 76, + [0][1][2][0][RTW89_ETSI][6] = 46, + [0][1][2][0][RTW89_MKK][6] = 64, + [0][1][2][0][RTW89_IC][6] = 76, + [0][1][2][0][RTW89_KCC][6] = 70, + [0][1][2][0][RTW89_ACMA][6] = 46, + [0][1][2][0][RTW89_CHILE][6] = 48, + [0][1][2][0][RTW89_UKRAINE][6] = 46, + [0][1][2][0][RTW89_MEXICO][6] = 76, + [0][1][2][0][RTW89_CN][6] = 46, + [0][1][2][0][RTW89_QATAR][6] = 46, + [0][1][2][0][RTW89_FCC][7] = 68, + [0][1][2][0][RTW89_ETSI][7] = 46, + [0][1][2][0][RTW89_MKK][7] = 64, + [0][1][2][0][RTW89_IC][7] = 68, + [0][1][2][0][RTW89_KCC][7] = 70, + [0][1][2][0][RTW89_ACMA][7] = 46, + [0][1][2][0][RTW89_CHILE][7] = 48, + [0][1][2][0][RTW89_UKRAINE][7] = 46, + [0][1][2][0][RTW89_MEXICO][7] = 68, + [0][1][2][0][RTW89_CN][7] = 46, + [0][1][2][0][RTW89_QATAR][7] = 46, + [0][1][2][0][RTW89_FCC][8] = 64, + [0][1][2][0][RTW89_ETSI][8] = 46, + [0][1][2][0][RTW89_MKK][8] = 64, + [0][1][2][0][RTW89_IC][8] = 64, + [0][1][2][0][RTW89_KCC][8] = 70, + [0][1][2][0][RTW89_ACMA][8] = 46, + [0][1][2][0][RTW89_CHILE][8] = 48, + [0][1][2][0][RTW89_UKRAINE][8] = 46, + [0][1][2][0][RTW89_MEXICO][8] = 64, + [0][1][2][0][RTW89_CN][8] = 46, + [0][1][2][0][RTW89_QATAR][8] = 46, + [0][1][2][0][RTW89_FCC][9] = 60, + [0][1][2][0][RTW89_ETSI][9] = 46, + [0][1][2][0][RTW89_MKK][9] = 64, + [0][1][2][0][RTW89_IC][9] = 60, + [0][1][2][0][RTW89_KCC][9] = 70, + [0][1][2][0][RTW89_ACMA][9] = 46, + [0][1][2][0][RTW89_CHILE][9] = 48, + [0][1][2][0][RTW89_UKRAINE][9] = 46, + [0][1][2][0][RTW89_MEXICO][9] = 60, + [0][1][2][0][RTW89_CN][9] = 46, + [0][1][2][0][RTW89_QATAR][9] = 46, + [0][1][2][0][RTW89_FCC][10] = 60, + [0][1][2][0][RTW89_ETSI][10] = 46, + [0][1][2][0][RTW89_MKK][10] = 64, + [0][1][2][0][RTW89_IC][10] = 60, + [0][1][2][0][RTW89_KCC][10] = 70, + [0][1][2][0][RTW89_ACMA][10] = 46, + [0][1][2][0][RTW89_CHILE][10] = 48, + [0][1][2][0][RTW89_UKRAINE][10] = 46, + [0][1][2][0][RTW89_MEXICO][10] = 60, + [0][1][2][0][RTW89_CN][10] = 46, + [0][1][2][0][RTW89_QATAR][10] = 46, + [0][1][2][0][RTW89_FCC][11] = 42, + [0][1][2][0][RTW89_ETSI][11] = 46, + [0][1][2][0][RTW89_MKK][11] = 64, + [0][1][2][0][RTW89_IC][11] = 42, + [0][1][2][0][RTW89_KCC][11] = 70, + [0][1][2][0][RTW89_ACMA][11] = 46, + [0][1][2][0][RTW89_CHILE][11] = 42, + [0][1][2][0][RTW89_UKRAINE][11] = 46, + [0][1][2][0][RTW89_MEXICO][11] = 42, + [0][1][2][0][RTW89_CN][11] = 46, + [0][1][2][0][RTW89_QATAR][11] = 46, + [0][1][2][0][RTW89_FCC][12] = 40, + [0][1][2][0][RTW89_ETSI][12] = 46, + [0][1][2][0][RTW89_MKK][12] = 64, + [0][1][2][0][RTW89_IC][12] = 40, + [0][1][2][0][RTW89_KCC][12] = 68, + [0][1][2][0][RTW89_ACMA][12] = 46, + [0][1][2][0][RTW89_CHILE][12] = 40, + [0][1][2][0][RTW89_UKRAINE][12] = 46, + [0][1][2][0][RTW89_MEXICO][12] = 40, + [0][1][2][0][RTW89_CN][12] = 46, + [0][1][2][0][RTW89_QATAR][12] = 46, + [0][1][2][0][RTW89_FCC][13] = 127, + [0][1][2][0][RTW89_ETSI][13] = 127, + [0][1][2][0][RTW89_MKK][13] = 127, + [0][1][2][0][RTW89_IC][13] = 127, + [0][1][2][0][RTW89_KCC][13] = 127, + [0][1][2][0][RTW89_ACMA][13] = 127, + [0][1][2][0][RTW89_CHILE][13] = 127, + [0][1][2][0][RTW89_UKRAINE][13] = 127, + [0][1][2][0][RTW89_MEXICO][13] = 127, + [0][1][2][0][RTW89_CN][13] = 127, + [0][1][2][0][RTW89_QATAR][13] = 127, + [0][1][2][1][RTW89_FCC][0] = 62, + [0][1][2][1][RTW89_ETSI][0] = 34, + [0][1][2][1][RTW89_MKK][0] = 64, + [0][1][2][1][RTW89_IC][0] = 62, + [0][1][2][1][RTW89_KCC][0] = 68, + [0][1][2][1][RTW89_ACMA][0] = 34, + [0][1][2][1][RTW89_CHILE][0] = 36, + [0][1][2][1][RTW89_UKRAINE][0] = 34, + [0][1][2][1][RTW89_MEXICO][0] = 62, + [0][1][2][1][RTW89_CN][0] = 34, + [0][1][2][1][RTW89_QATAR][0] = 34, + [0][1][2][1][RTW89_FCC][1] = 62, + [0][1][2][1][RTW89_ETSI][1] = 34, + [0][1][2][1][RTW89_MKK][1] = 64, + [0][1][2][1][RTW89_IC][1] = 62, + [0][1][2][1][RTW89_KCC][1] = 70, + [0][1][2][1][RTW89_ACMA][1] = 34, + [0][1][2][1][RTW89_CHILE][1] = 36, + [0][1][2][1][RTW89_UKRAINE][1] = 34, + [0][1][2][1][RTW89_MEXICO][1] = 62, + [0][1][2][1][RTW89_CN][1] = 34, + [0][1][2][1][RTW89_QATAR][1] = 34, + [0][1][2][1][RTW89_FCC][2] = 66, + [0][1][2][1][RTW89_ETSI][2] = 34, + [0][1][2][1][RTW89_MKK][2] = 64, + [0][1][2][1][RTW89_IC][2] = 66, + [0][1][2][1][RTW89_KCC][2] = 70, + [0][1][2][1][RTW89_ACMA][2] = 34, + [0][1][2][1][RTW89_CHILE][2] = 36, + [0][1][2][1][RTW89_UKRAINE][2] = 34, + [0][1][2][1][RTW89_MEXICO][2] = 66, + [0][1][2][1][RTW89_CN][2] = 34, + [0][1][2][1][RTW89_QATAR][2] = 34, + [0][1][2][1][RTW89_FCC][3] = 70, + [0][1][2][1][RTW89_ETSI][3] = 34, + [0][1][2][1][RTW89_MKK][3] = 64, + [0][1][2][1][RTW89_IC][3] = 70, + [0][1][2][1][RTW89_KCC][3] = 70, + [0][1][2][1][RTW89_ACMA][3] = 34, + [0][1][2][1][RTW89_CHILE][3] = 36, + [0][1][2][1][RTW89_UKRAINE][3] = 34, + [0][1][2][1][RTW89_MEXICO][3] = 70, + [0][1][2][1][RTW89_CN][3] = 34, + [0][1][2][1][RTW89_QATAR][3] = 34, + [0][1][2][1][RTW89_FCC][4] = 76, + [0][1][2][1][RTW89_ETSI][4] = 34, + [0][1][2][1][RTW89_MKK][4] = 64, + [0][1][2][1][RTW89_IC][4] = 76, + [0][1][2][1][RTW89_KCC][4] = 70, + [0][1][2][1][RTW89_ACMA][4] = 34, + [0][1][2][1][RTW89_CHILE][4] = 36, + [0][1][2][1][RTW89_UKRAINE][4] = 34, + [0][1][2][1][RTW89_MEXICO][4] = 76, + [0][1][2][1][RTW89_CN][4] = 34, + [0][1][2][1][RTW89_QATAR][4] = 34, + [0][1][2][1][RTW89_FCC][5] = 76, + [0][1][2][1][RTW89_ETSI][5] = 34, + [0][1][2][1][RTW89_MKK][5] = 64, + [0][1][2][1][RTW89_IC][5] = 76, + [0][1][2][1][RTW89_KCC][5] = 70, + [0][1][2][1][RTW89_ACMA][5] = 34, + [0][1][2][1][RTW89_CHILE][5] = 36, + [0][1][2][1][RTW89_UKRAINE][5] = 34, + [0][1][2][1][RTW89_MEXICO][5] = 76, + [0][1][2][1][RTW89_CN][5] = 34, + [0][1][2][1][RTW89_QATAR][5] = 34, + [0][1][2][1][RTW89_FCC][6] = 76, + [0][1][2][1][RTW89_ETSI][6] = 34, + [0][1][2][1][RTW89_MKK][6] = 64, + [0][1][2][1][RTW89_IC][6] = 76, + [0][1][2][1][RTW89_KCC][6] = 70, + [0][1][2][1][RTW89_ACMA][6] = 34, + [0][1][2][1][RTW89_CHILE][6] = 36, + [0][1][2][1][RTW89_UKRAINE][6] = 34, + [0][1][2][1][RTW89_MEXICO][6] = 76, + [0][1][2][1][RTW89_CN][6] = 34, + [0][1][2][1][RTW89_QATAR][6] = 34, + [0][1][2][1][RTW89_FCC][7] = 68, + [0][1][2][1][RTW89_ETSI][7] = 34, + [0][1][2][1][RTW89_MKK][7] = 64, + [0][1][2][1][RTW89_IC][7] = 68, + [0][1][2][1][RTW89_KCC][7] = 70, + [0][1][2][1][RTW89_ACMA][7] = 34, + [0][1][2][1][RTW89_CHILE][7] = 36, + [0][1][2][1][RTW89_UKRAINE][7] = 34, + [0][1][2][1][RTW89_MEXICO][7] = 68, + [0][1][2][1][RTW89_CN][7] = 34, + [0][1][2][1][RTW89_QATAR][7] = 34, + [0][1][2][1][RTW89_FCC][8] = 64, + [0][1][2][1][RTW89_ETSI][8] = 34, + [0][1][2][1][RTW89_MKK][8] = 64, + [0][1][2][1][RTW89_IC][8] = 64, + [0][1][2][1][RTW89_KCC][8] = 70, + [0][1][2][1][RTW89_ACMA][8] = 34, + [0][1][2][1][RTW89_CHILE][8] = 36, + [0][1][2][1][RTW89_UKRAINE][8] = 34, + [0][1][2][1][RTW89_MEXICO][8] = 64, + [0][1][2][1][RTW89_CN][8] = 34, + [0][1][2][1][RTW89_QATAR][8] = 34, + [0][1][2][1][RTW89_FCC][9] = 60, + [0][1][2][1][RTW89_ETSI][9] = 34, + [0][1][2][1][RTW89_MKK][9] = 64, + [0][1][2][1][RTW89_IC][9] = 60, + [0][1][2][1][RTW89_KCC][9] = 70, + [0][1][2][1][RTW89_ACMA][9] = 34, + [0][1][2][1][RTW89_CHILE][9] = 36, + [0][1][2][1][RTW89_UKRAINE][9] = 34, + [0][1][2][1][RTW89_MEXICO][9] = 60, + [0][1][2][1][RTW89_CN][9] = 34, + [0][1][2][1][RTW89_QATAR][9] = 34, + [0][1][2][1][RTW89_FCC][10] = 60, + [0][1][2][1][RTW89_ETSI][10] = 34, + [0][1][2][1][RTW89_MKK][10] = 64, + [0][1][2][1][RTW89_IC][10] = 60, + [0][1][2][1][RTW89_KCC][10] = 70, + [0][1][2][1][RTW89_ACMA][10] = 34, + [0][1][2][1][RTW89_CHILE][10] = 36, + [0][1][2][1][RTW89_UKRAINE][10] = 34, + [0][1][2][1][RTW89_MEXICO][10] = 60, + [0][1][2][1][RTW89_CN][10] = 34, + [0][1][2][1][RTW89_QATAR][10] = 34, + [0][1][2][1][RTW89_FCC][11] = 42, + [0][1][2][1][RTW89_ETSI][11] = 34, + [0][1][2][1][RTW89_MKK][11] = 64, + [0][1][2][1][RTW89_IC][11] = 42, + [0][1][2][1][RTW89_KCC][11] = 70, + [0][1][2][1][RTW89_ACMA][11] = 34, + [0][1][2][1][RTW89_CHILE][11] = 36, + [0][1][2][1][RTW89_UKRAINE][11] = 34, + [0][1][2][1][RTW89_MEXICO][11] = 42, + [0][1][2][1][RTW89_CN][11] = 34, + [0][1][2][1][RTW89_QATAR][11] = 34, + [0][1][2][1][RTW89_FCC][12] = 40, + [0][1][2][1][RTW89_ETSI][12] = 34, + [0][1][2][1][RTW89_MKK][12] = 64, + [0][1][2][1][RTW89_IC][12] = 40, + [0][1][2][1][RTW89_KCC][12] = 68, + [0][1][2][1][RTW89_ACMA][12] = 34, + [0][1][2][1][RTW89_CHILE][12] = 36, + [0][1][2][1][RTW89_UKRAINE][12] = 34, + [0][1][2][1][RTW89_MEXICO][12] = 40, + [0][1][2][1][RTW89_CN][12] = 34, + [0][1][2][1][RTW89_QATAR][12] = 34, + [0][1][2][1][RTW89_FCC][13] = 127, + [0][1][2][1][RTW89_ETSI][13] = 127, + [0][1][2][1][RTW89_MKK][13] = 127, + [0][1][2][1][RTW89_IC][13] = 127, + [0][1][2][1][RTW89_KCC][13] = 127, + [0][1][2][1][RTW89_ACMA][13] = 127, + [0][1][2][1][RTW89_CHILE][13] = 127, + [0][1][2][1][RTW89_UKRAINE][13] = 127, + [0][1][2][1][RTW89_MEXICO][13] = 127, + [0][1][2][1][RTW89_CN][13] = 127, + [0][1][2][1][RTW89_QATAR][13] = 127, + [1][0][2][0][RTW89_FCC][0] = 127, + [1][0][2][0][RTW89_ETSI][0] = 127, + [1][0][2][0][RTW89_MKK][0] = 127, + [1][0][2][0][RTW89_IC][0] = 127, + [1][0][2][0][RTW89_KCC][0] = 127, + [1][0][2][0][RTW89_ACMA][0] = 127, + [1][0][2][0][RTW89_CHILE][0] = 127, + [1][0][2][0][RTW89_UKRAINE][0] = 127, + [1][0][2][0][RTW89_MEXICO][0] = 127, + [1][0][2][0][RTW89_CN][0] = 127, + [1][0][2][0][RTW89_QATAR][0] = 127, + [1][0][2][0][RTW89_FCC][1] = 127, + [1][0][2][0][RTW89_ETSI][1] = 127, + [1][0][2][0][RTW89_MKK][1] = 127, + [1][0][2][0][RTW89_IC][1] = 127, + [1][0][2][0][RTW89_KCC][1] = 127, + [1][0][2][0][RTW89_ACMA][1] = 127, + [1][0][2][0][RTW89_CHILE][1] = 127, + [1][0][2][0][RTW89_UKRAINE][1] = 127, + [1][0][2][0][RTW89_MEXICO][1] = 127, + [1][0][2][0][RTW89_CN][1] = 127, + [1][0][2][0][RTW89_QATAR][1] = 127, + [1][0][2][0][RTW89_FCC][2] = 56, + [1][0][2][0][RTW89_ETSI][2] = 58, + [1][0][2][0][RTW89_MKK][2] = 68, + [1][0][2][0][RTW89_IC][2] = 56, + [1][0][2][0][RTW89_KCC][2] = 68, + [1][0][2][0][RTW89_ACMA][2] = 58, + [1][0][2][0][RTW89_CHILE][2] = 56, + [1][0][2][0][RTW89_UKRAINE][2] = 58, + [1][0][2][0][RTW89_MEXICO][2] = 56, + [1][0][2][0][RTW89_CN][2] = 58, + [1][0][2][0][RTW89_QATAR][2] = 58, + [1][0][2][0][RTW89_FCC][3] = 56, + [1][0][2][0][RTW89_ETSI][3] = 58, + [1][0][2][0][RTW89_MKK][3] = 68, + [1][0][2][0][RTW89_IC][3] = 56, + [1][0][2][0][RTW89_KCC][3] = 68, + [1][0][2][0][RTW89_ACMA][3] = 58, + [1][0][2][0][RTW89_CHILE][3] = 56, + [1][0][2][0][RTW89_UKRAINE][3] = 58, + [1][0][2][0][RTW89_MEXICO][3] = 56, + [1][0][2][0][RTW89_CN][3] = 58, + [1][0][2][0][RTW89_QATAR][3] = 58, + [1][0][2][0][RTW89_FCC][4] = 60, + [1][0][2][0][RTW89_ETSI][4] = 58, + [1][0][2][0][RTW89_MKK][4] = 68, + [1][0][2][0][RTW89_IC][4] = 60, + [1][0][2][0][RTW89_KCC][4] = 68, + [1][0][2][0][RTW89_ACMA][4] = 58, + [1][0][2][0][RTW89_CHILE][4] = 60, + [1][0][2][0][RTW89_UKRAINE][4] = 58, + [1][0][2][0][RTW89_MEXICO][4] = 60, + [1][0][2][0][RTW89_CN][4] = 58, + [1][0][2][0][RTW89_QATAR][4] = 58, + [1][0][2][0][RTW89_FCC][5] = 64, + [1][0][2][0][RTW89_ETSI][5] = 58, + [1][0][2][0][RTW89_MKK][5] = 68, + [1][0][2][0][RTW89_IC][5] = 64, + [1][0][2][0][RTW89_KCC][5] = 68, + [1][0][2][0][RTW89_ACMA][5] = 58, + [1][0][2][0][RTW89_CHILE][5] = 60, + [1][0][2][0][RTW89_UKRAINE][5] = 58, + [1][0][2][0][RTW89_MEXICO][5] = 64, + [1][0][2][0][RTW89_CN][5] = 58, + [1][0][2][0][RTW89_QATAR][5] = 58, + [1][0][2][0][RTW89_FCC][6] = 54, + [1][0][2][0][RTW89_ETSI][6] = 58, + [1][0][2][0][RTW89_MKK][6] = 68, + [1][0][2][0][RTW89_IC][6] = 54, + [1][0][2][0][RTW89_KCC][6] = 68, + [1][0][2][0][RTW89_ACMA][6] = 58, + [1][0][2][0][RTW89_CHILE][6] = 54, + [1][0][2][0][RTW89_UKRAINE][6] = 58, + [1][0][2][0][RTW89_MEXICO][6] = 54, + [1][0][2][0][RTW89_CN][6] = 58, + [1][0][2][0][RTW89_QATAR][6] = 58, + [1][0][2][0][RTW89_FCC][7] = 50, + [1][0][2][0][RTW89_ETSI][7] = 58, + [1][0][2][0][RTW89_MKK][7] = 68, + [1][0][2][0][RTW89_IC][7] = 50, + [1][0][2][0][RTW89_KCC][7] = 68, + [1][0][2][0][RTW89_ACMA][7] = 58, + [1][0][2][0][RTW89_CHILE][7] = 50, + [1][0][2][0][RTW89_UKRAINE][7] = 58, + [1][0][2][0][RTW89_MEXICO][7] = 50, + [1][0][2][0][RTW89_CN][7] = 58, + [1][0][2][0][RTW89_QATAR][7] = 58, + [1][0][2][0][RTW89_FCC][8] = 50, + [1][0][2][0][RTW89_ETSI][8] = 58, + [1][0][2][0][RTW89_MKK][8] = 68, + [1][0][2][0][RTW89_IC][8] = 50, + [1][0][2][0][RTW89_KCC][8] = 68, + [1][0][2][0][RTW89_ACMA][8] = 58, + [1][0][2][0][RTW89_CHILE][8] = 50, + [1][0][2][0][RTW89_UKRAINE][8] = 58, + [1][0][2][0][RTW89_MEXICO][8] = 50, + [1][0][2][0][RTW89_CN][8] = 58, + [1][0][2][0][RTW89_QATAR][8] = 58, + [1][0][2][0][RTW89_FCC][9] = 42, + [1][0][2][0][RTW89_ETSI][9] = 58, + [1][0][2][0][RTW89_MKK][9] = 68, + [1][0][2][0][RTW89_IC][9] = 42, + [1][0][2][0][RTW89_KCC][9] = 68, + [1][0][2][0][RTW89_ACMA][9] = 58, + [1][0][2][0][RTW89_CHILE][9] = 42, + [1][0][2][0][RTW89_UKRAINE][9] = 58, + [1][0][2][0][RTW89_MEXICO][9] = 42, + [1][0][2][0][RTW89_CN][9] = 58, + [1][0][2][0][RTW89_QATAR][9] = 58, + [1][0][2][0][RTW89_FCC][10] = 40, + [1][0][2][0][RTW89_ETSI][10] = 58, + [1][0][2][0][RTW89_MKK][10] = 68, + [1][0][2][0][RTW89_IC][10] = 40, + [1][0][2][0][RTW89_KCC][10] = 68, + [1][0][2][0][RTW89_ACMA][10] = 58, + [1][0][2][0][RTW89_CHILE][10] = 40, + [1][0][2][0][RTW89_UKRAINE][10] = 58, + [1][0][2][0][RTW89_MEXICO][10] = 40, + [1][0][2][0][RTW89_CN][10] = 58, + [1][0][2][0][RTW89_QATAR][10] = 58, + [1][0][2][0][RTW89_FCC][11] = 127, + [1][0][2][0][RTW89_ETSI][11] = 127, + [1][0][2][0][RTW89_MKK][11] = 127, + [1][0][2][0][RTW89_IC][11] = 127, + [1][0][2][0][RTW89_KCC][11] = 127, + [1][0][2][0][RTW89_ACMA][11] = 127, + [1][0][2][0][RTW89_CHILE][11] = 127, + [1][0][2][0][RTW89_UKRAINE][11] = 127, + [1][0][2][0][RTW89_MEXICO][11] = 127, + [1][0][2][0][RTW89_CN][11] = 127, + [1][0][2][0][RTW89_QATAR][11] = 127, + [1][0][2][0][RTW89_FCC][12] = 127, + [1][0][2][0][RTW89_ETSI][12] = 127, + [1][0][2][0][RTW89_MKK][12] = 127, + [1][0][2][0][RTW89_IC][12] = 127, + [1][0][2][0][RTW89_KCC][12] = 127, + [1][0][2][0][RTW89_ACMA][12] = 127, + [1][0][2][0][RTW89_CHILE][12] = 127, + [1][0][2][0][RTW89_UKRAINE][12] = 127, + [1][0][2][0][RTW89_MEXICO][12] = 127, + [1][0][2][0][RTW89_CN][12] = 127, + [1][0][2][0][RTW89_QATAR][12] = 127, + [1][0][2][0][RTW89_FCC][13] = 127, + [1][0][2][0][RTW89_ETSI][13] = 127, + [1][0][2][0][RTW89_MKK][13] = 127, + [1][0][2][0][RTW89_IC][13] = 127, + [1][0][2][0][RTW89_KCC][13] = 127, + [1][0][2][0][RTW89_ACMA][13] = 127, + [1][0][2][0][RTW89_CHILE][13] = 127, + [1][0][2][0][RTW89_UKRAINE][13] = 127, + [1][0][2][0][RTW89_MEXICO][13] = 127, + [1][0][2][0][RTW89_CN][13] = 127, + [1][0][2][0][RTW89_QATAR][13] = 127, + [1][1][2][0][RTW89_FCC][0] = 127, + [1][1][2][0][RTW89_ETSI][0] = 127, + [1][1][2][0][RTW89_MKK][0] = 127, + [1][1][2][0][RTW89_IC][0] = 127, + [1][1][2][0][RTW89_KCC][0] = 127, + [1][1][2][0][RTW89_ACMA][0] = 127, + [1][1][2][0][RTW89_CHILE][0] = 127, + [1][1][2][0][RTW89_UKRAINE][0] = 127, + [1][1][2][0][RTW89_MEXICO][0] = 127, + [1][1][2][0][RTW89_CN][0] = 127, + [1][1][2][0][RTW89_QATAR][0] = 127, + [1][1][2][0][RTW89_FCC][1] = 127, + [1][1][2][0][RTW89_ETSI][1] = 127, + [1][1][2][0][RTW89_MKK][1] = 127, + [1][1][2][0][RTW89_IC][1] = 127, + [1][1][2][0][RTW89_KCC][1] = 127, + [1][1][2][0][RTW89_ACMA][1] = 127, + [1][1][2][0][RTW89_CHILE][1] = 127, + [1][1][2][0][RTW89_UKRAINE][1] = 127, + [1][1][2][0][RTW89_MEXICO][1] = 127, + [1][1][2][0][RTW89_CN][1] = 127, + [1][1][2][0][RTW89_QATAR][1] = 127, + [1][1][2][0][RTW89_FCC][2] = 52, + [1][1][2][0][RTW89_ETSI][2] = 46, + [1][1][2][0][RTW89_MKK][2] = 64, + [1][1][2][0][RTW89_IC][2] = 52, + [1][1][2][0][RTW89_KCC][2] = 66, + [1][1][2][0][RTW89_ACMA][2] = 46, + [1][1][2][0][RTW89_CHILE][2] = 48, + [1][1][2][0][RTW89_UKRAINE][2] = 46, + [1][1][2][0][RTW89_MEXICO][2] = 52, + [1][1][2][0][RTW89_CN][2] = 46, + [1][1][2][0][RTW89_QATAR][2] = 46, + [1][1][2][0][RTW89_FCC][3] = 52, + [1][1][2][0][RTW89_ETSI][3] = 46, + [1][1][2][0][RTW89_MKK][3] = 64, + [1][1][2][0][RTW89_IC][3] = 52, + [1][1][2][0][RTW89_KCC][3] = 68, + [1][1][2][0][RTW89_ACMA][3] = 46, + [1][1][2][0][RTW89_CHILE][3] = 48, + [1][1][2][0][RTW89_UKRAINE][3] = 46, + [1][1][2][0][RTW89_MEXICO][3] = 52, + [1][1][2][0][RTW89_CN][3] = 46, + [1][1][2][0][RTW89_QATAR][3] = 46, + [1][1][2][0][RTW89_FCC][4] = 56, + [1][1][2][0][RTW89_ETSI][4] = 46, + [1][1][2][0][RTW89_MKK][4] = 64, + [1][1][2][0][RTW89_IC][4] = 56, + [1][1][2][0][RTW89_KCC][4] = 68, + [1][1][2][0][RTW89_ACMA][4] = 46, + [1][1][2][0][RTW89_CHILE][4] = 48, + [1][1][2][0][RTW89_UKRAINE][4] = 46, + [1][1][2][0][RTW89_MEXICO][4] = 56, + [1][1][2][0][RTW89_CN][4] = 46, + [1][1][2][0][RTW89_QATAR][4] = 46, + [1][1][2][0][RTW89_FCC][5] = 60, + [1][1][2][0][RTW89_ETSI][5] = 46, + [1][1][2][0][RTW89_MKK][5] = 64, + [1][1][2][0][RTW89_IC][5] = 60, + [1][1][2][0][RTW89_KCC][5] = 68, + [1][1][2][0][RTW89_ACMA][5] = 46, + [1][1][2][0][RTW89_CHILE][5] = 48, + [1][1][2][0][RTW89_UKRAINE][5] = 46, + [1][1][2][0][RTW89_MEXICO][5] = 60, + [1][1][2][0][RTW89_CN][5] = 46, + [1][1][2][0][RTW89_QATAR][5] = 46, + [1][1][2][0][RTW89_FCC][6] = 54, + [1][1][2][0][RTW89_ETSI][6] = 46, + [1][1][2][0][RTW89_MKK][6] = 64, + [1][1][2][0][RTW89_IC][6] = 52, + [1][1][2][0][RTW89_KCC][6] = 68, + [1][1][2][0][RTW89_ACMA][6] = 46, + [1][1][2][0][RTW89_CHILE][6] = 48, + [1][1][2][0][RTW89_UKRAINE][6] = 46, + [1][1][2][0][RTW89_MEXICO][6] = 54, + [1][1][2][0][RTW89_CN][6] = 46, + [1][1][2][0][RTW89_QATAR][6] = 46, + [1][1][2][0][RTW89_FCC][7] = 50, + [1][1][2][0][RTW89_ETSI][7] = 46, + [1][1][2][0][RTW89_MKK][7] = 64, + [1][1][2][0][RTW89_IC][7] = 48, + [1][1][2][0][RTW89_KCC][7] = 68, + [1][1][2][0][RTW89_ACMA][7] = 46, + [1][1][2][0][RTW89_CHILE][7] = 48, + [1][1][2][0][RTW89_UKRAINE][7] = 46, + [1][1][2][0][RTW89_MEXICO][7] = 50, + [1][1][2][0][RTW89_CN][7] = 46, + [1][1][2][0][RTW89_QATAR][7] = 46, + [1][1][2][0][RTW89_FCC][8] = 50, + [1][1][2][0][RTW89_ETSI][8] = 46, + [1][1][2][0][RTW89_MKK][8] = 64, + [1][1][2][0][RTW89_IC][8] = 48, + [1][1][2][0][RTW89_KCC][8] = 68, + [1][1][2][0][RTW89_ACMA][8] = 46, + [1][1][2][0][RTW89_CHILE][8] = 48, + [1][1][2][0][RTW89_UKRAINE][8] = 46, + [1][1][2][0][RTW89_MEXICO][8] = 50, + [1][1][2][0][RTW89_CN][8] = 46, + [1][1][2][0][RTW89_QATAR][8] = 46, + [1][1][2][0][RTW89_FCC][9] = 38, + [1][1][2][0][RTW89_ETSI][9] = 46, + [1][1][2][0][RTW89_MKK][9] = 64, + [1][1][2][0][RTW89_IC][9] = 38, + [1][1][2][0][RTW89_KCC][9] = 68, + [1][1][2][0][RTW89_ACMA][9] = 46, + [1][1][2][0][RTW89_CHILE][9] = 38, + [1][1][2][0][RTW89_UKRAINE][9] = 46, + [1][1][2][0][RTW89_MEXICO][9] = 38, + [1][1][2][0][RTW89_CN][9] = 46, + [1][1][2][0][RTW89_QATAR][9] = 46, + [1][1][2][0][RTW89_FCC][10] = 36, + [1][1][2][0][RTW89_ETSI][10] = 46, + [1][1][2][0][RTW89_MKK][10] = 64, + [1][1][2][0][RTW89_IC][10] = 36, + [1][1][2][0][RTW89_KCC][10] = 66, + [1][1][2][0][RTW89_ACMA][10] = 46, + [1][1][2][0][RTW89_CHILE][10] = 36, + [1][1][2][0][RTW89_UKRAINE][10] = 46, + [1][1][2][0][RTW89_MEXICO][10] = 36, + [1][1][2][0][RTW89_CN][10] = 46, + [1][1][2][0][RTW89_QATAR][10] = 46, + [1][1][2][0][RTW89_FCC][11] = 127, + [1][1][2][0][RTW89_ETSI][11] = 127, + [1][1][2][0][RTW89_MKK][11] = 127, + [1][1][2][0][RTW89_IC][11] = 127, + [1][1][2][0][RTW89_KCC][11] = 127, + [1][1][2][0][RTW89_ACMA][11] = 127, + [1][1][2][0][RTW89_CHILE][11] = 127, + [1][1][2][0][RTW89_UKRAINE][11] = 127, + [1][1][2][0][RTW89_MEXICO][11] = 127, + [1][1][2][0][RTW89_CN][11] = 127, + [1][1][2][0][RTW89_QATAR][11] = 127, + [1][1][2][0][RTW89_FCC][12] = 127, + [1][1][2][0][RTW89_ETSI][12] = 127, + [1][1][2][0][RTW89_MKK][12] = 127, + [1][1][2][0][RTW89_IC][12] = 127, + [1][1][2][0][RTW89_KCC][12] = 127, + [1][1][2][0][RTW89_ACMA][12] = 127, + [1][1][2][0][RTW89_CHILE][12] = 127, + [1][1][2][0][RTW89_UKRAINE][12] = 127, + [1][1][2][0][RTW89_MEXICO][12] = 127, + [1][1][2][0][RTW89_CN][12] = 127, + [1][1][2][0][RTW89_QATAR][12] = 127, + [1][1][2][0][RTW89_FCC][13] = 127, + [1][1][2][0][RTW89_ETSI][13] = 127, + [1][1][2][0][RTW89_MKK][13] = 127, + [1][1][2][0][RTW89_IC][13] = 127, + [1][1][2][0][RTW89_KCC][13] = 127, + [1][1][2][0][RTW89_ACMA][13] = 127, + [1][1][2][0][RTW89_CHILE][13] = 127, + [1][1][2][0][RTW89_UKRAINE][13] = 127, + [1][1][2][0][RTW89_MEXICO][13] = 127, + [1][1][2][0][RTW89_CN][13] = 127, + [1][1][2][0][RTW89_QATAR][13] = 127, + [1][1][2][1][RTW89_FCC][0] = 127, + [1][1][2][1][RTW89_ETSI][0] = 127, + [1][1][2][1][RTW89_MKK][0] = 127, + [1][1][2][1][RTW89_IC][0] = 127, + [1][1][2][1][RTW89_KCC][0] = 127, + [1][1][2][1][RTW89_ACMA][0] = 127, + [1][1][2][1][RTW89_CHILE][0] = 127, + [1][1][2][1][RTW89_UKRAINE][0] = 127, + [1][1][2][1][RTW89_MEXICO][0] = 127, + [1][1][2][1][RTW89_CN][0] = 127, + [1][1][2][1][RTW89_QATAR][0] = 127, + [1][1][2][1][RTW89_FCC][1] = 127, + [1][1][2][1][RTW89_ETSI][1] = 127, + [1][1][2][1][RTW89_MKK][1] = 127, + [1][1][2][1][RTW89_IC][1] = 127, + [1][1][2][1][RTW89_KCC][1] = 127, + [1][1][2][1][RTW89_ACMA][1] = 127, + [1][1][2][1][RTW89_CHILE][1] = 127, + [1][1][2][1][RTW89_UKRAINE][1] = 127, + [1][1][2][1][RTW89_MEXICO][1] = 127, + [1][1][2][1][RTW89_CN][1] = 127, + [1][1][2][1][RTW89_QATAR][1] = 127, + [1][1][2][1][RTW89_FCC][2] = 52, + [1][1][2][1][RTW89_ETSI][2] = 34, + [1][1][2][1][RTW89_MKK][2] = 64, + [1][1][2][1][RTW89_IC][2] = 52, + [1][1][2][1][RTW89_KCC][2] = 66, + [1][1][2][1][RTW89_ACMA][2] = 34, + [1][1][2][1][RTW89_CHILE][2] = 36, + [1][1][2][1][RTW89_UKRAINE][2] = 34, + [1][1][2][1][RTW89_MEXICO][2] = 52, + [1][1][2][1][RTW89_CN][2] = 34, + [1][1][2][1][RTW89_QATAR][2] = 34, + [1][1][2][1][RTW89_FCC][3] = 52, + [1][1][2][1][RTW89_ETSI][3] = 34, + [1][1][2][1][RTW89_MKK][3] = 64, + [1][1][2][1][RTW89_IC][3] = 52, + [1][1][2][1][RTW89_KCC][3] = 68, + [1][1][2][1][RTW89_ACMA][3] = 34, + [1][1][2][1][RTW89_CHILE][3] = 36, + [1][1][2][1][RTW89_UKRAINE][3] = 34, + [1][1][2][1][RTW89_MEXICO][3] = 52, + [1][1][2][1][RTW89_CN][3] = 34, + [1][1][2][1][RTW89_QATAR][3] = 34, + [1][1][2][1][RTW89_FCC][4] = 56, + [1][1][2][1][RTW89_ETSI][4] = 34, + [1][1][2][1][RTW89_MKK][4] = 64, + [1][1][2][1][RTW89_IC][4] = 56, + [1][1][2][1][RTW89_KCC][4] = 68, + [1][1][2][1][RTW89_ACMA][4] = 34, + [1][1][2][1][RTW89_CHILE][4] = 36, + [1][1][2][1][RTW89_UKRAINE][4] = 34, + [1][1][2][1][RTW89_MEXICO][4] = 56, + [1][1][2][1][RTW89_CN][4] = 34, + [1][1][2][1][RTW89_QATAR][4] = 34, + [1][1][2][1][RTW89_FCC][5] = 60, + [1][1][2][1][RTW89_ETSI][5] = 34, + [1][1][2][1][RTW89_MKK][5] = 64, + [1][1][2][1][RTW89_IC][5] = 60, + [1][1][2][1][RTW89_KCC][5] = 68, + [1][1][2][1][RTW89_ACMA][5] = 34, + [1][1][2][1][RTW89_CHILE][5] = 36, + [1][1][2][1][RTW89_UKRAINE][5] = 34, + [1][1][2][1][RTW89_MEXICO][5] = 60, + [1][1][2][1][RTW89_CN][5] = 34, + [1][1][2][1][RTW89_QATAR][5] = 34, + [1][1][2][1][RTW89_FCC][6] = 54, + [1][1][2][1][RTW89_ETSI][6] = 34, + [1][1][2][1][RTW89_MKK][6] = 64, + [1][1][2][1][RTW89_IC][6] = 52, + [1][1][2][1][RTW89_KCC][6] = 68, + [1][1][2][1][RTW89_ACMA][6] = 34, + [1][1][2][1][RTW89_CHILE][6] = 36, + [1][1][2][1][RTW89_UKRAINE][6] = 34, + [1][1][2][1][RTW89_MEXICO][6] = 54, + [1][1][2][1][RTW89_CN][6] = 34, + [1][1][2][1][RTW89_QATAR][6] = 34, + [1][1][2][1][RTW89_FCC][7] = 50, + [1][1][2][1][RTW89_ETSI][7] = 34, + [1][1][2][1][RTW89_MKK][7] = 64, + [1][1][2][1][RTW89_IC][7] = 48, + [1][1][2][1][RTW89_KCC][7] = 68, + [1][1][2][1][RTW89_ACMA][7] = 34, + [1][1][2][1][RTW89_CHILE][7] = 36, + [1][1][2][1][RTW89_UKRAINE][7] = 34, + [1][1][2][1][RTW89_MEXICO][7] = 50, + [1][1][2][1][RTW89_CN][7] = 34, + [1][1][2][1][RTW89_QATAR][7] = 34, + [1][1][2][1][RTW89_FCC][8] = 50, + [1][1][2][1][RTW89_ETSI][8] = 34, + [1][1][2][1][RTW89_MKK][8] = 64, + [1][1][2][1][RTW89_IC][8] = 48, + [1][1][2][1][RTW89_KCC][8] = 68, + [1][1][2][1][RTW89_ACMA][8] = 34, + [1][1][2][1][RTW89_CHILE][8] = 36, + [1][1][2][1][RTW89_UKRAINE][8] = 34, + [1][1][2][1][RTW89_MEXICO][8] = 50, + [1][1][2][1][RTW89_CN][8] = 34, + [1][1][2][1][RTW89_QATAR][8] = 34, + [1][1][2][1][RTW89_FCC][9] = 38, + [1][1][2][1][RTW89_ETSI][9] = 34, + [1][1][2][1][RTW89_MKK][9] = 64, + [1][1][2][1][RTW89_IC][9] = 38, + [1][1][2][1][RTW89_KCC][9] = 68, + [1][1][2][1][RTW89_ACMA][9] = 34, + [1][1][2][1][RTW89_CHILE][9] = 36, + [1][1][2][1][RTW89_UKRAINE][9] = 34, + [1][1][2][1][RTW89_MEXICO][9] = 38, + [1][1][2][1][RTW89_CN][9] = 34, + [1][1][2][1][RTW89_QATAR][9] = 34, + [1][1][2][1][RTW89_FCC][10] = 36, + [1][1][2][1][RTW89_ETSI][10] = 34, + [1][1][2][1][RTW89_MKK][10] = 64, + [1][1][2][1][RTW89_IC][10] = 36, + [1][1][2][1][RTW89_KCC][10] = 66, + [1][1][2][1][RTW89_ACMA][10] = 34, + [1][1][2][1][RTW89_CHILE][10] = 36, + [1][1][2][1][RTW89_UKRAINE][10] = 34, + [1][1][2][1][RTW89_MEXICO][10] = 36, + [1][1][2][1][RTW89_CN][10] = 34, + [1][1][2][1][RTW89_QATAR][10] = 34, + [1][1][2][1][RTW89_FCC][11] = 127, + [1][1][2][1][RTW89_ETSI][11] = 127, + [1][1][2][1][RTW89_MKK][11] = 127, + [1][1][2][1][RTW89_IC][11] = 127, + [1][1][2][1][RTW89_KCC][11] = 127, + [1][1][2][1][RTW89_ACMA][11] = 127, + [1][1][2][1][RTW89_CHILE][11] = 127, + [1][1][2][1][RTW89_UKRAINE][11] = 127, + [1][1][2][1][RTW89_MEXICO][11] = 127, + [1][1][2][1][RTW89_CN][11] = 127, + [1][1][2][1][RTW89_QATAR][11] = 127, + [1][1][2][1][RTW89_FCC][12] = 127, + [1][1][2][1][RTW89_ETSI][12] = 127, + [1][1][2][1][RTW89_MKK][12] = 127, + [1][1][2][1][RTW89_IC][12] = 127, + [1][1][2][1][RTW89_KCC][12] = 127, + [1][1][2][1][RTW89_ACMA][12] = 127, + [1][1][2][1][RTW89_CHILE][12] = 127, + [1][1][2][1][RTW89_UKRAINE][12] = 127, + [1][1][2][1][RTW89_MEXICO][12] = 127, + [1][1][2][1][RTW89_CN][12] = 127, + [1][1][2][1][RTW89_QATAR][12] = 127, + [1][1][2][1][RTW89_FCC][13] = 127, + [1][1][2][1][RTW89_ETSI][13] = 127, + [1][1][2][1][RTW89_MKK][13] = 127, + [1][1][2][1][RTW89_IC][13] = 127, + [1][1][2][1][RTW89_KCC][13] = 127, + [1][1][2][1][RTW89_ACMA][13] = 127, + [1][1][2][1][RTW89_CHILE][13] = 127, + [1][1][2][1][RTW89_UKRAINE][13] = 127, + [1][1][2][1][RTW89_MEXICO][13] = 127, + [1][1][2][1][RTW89_CN][13] = 127, + [1][1][2][1][RTW89_QATAR][13] = 127, }; const s8 rtw89_8852a_txpwr_lmt_5g[RTW89_5G_BW_NUM][RTW89_NTX_NUM] [RTW89_RS_LMT_NUM][RTW89_BF_NUM] [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = { - [0][0][1][0][0][0] = 30, - [0][0][1][0][0][2] = 30, - [0][0][1][0][0][4] = 30, - [0][0][1][0][0][6] = 30, - [0][0][1][0][0][8] = 52, - [0][0][1][0][0][10] = 52, - [0][0][1][0][0][12] = 52, - [0][0][1][0][0][14] = 52, - [0][0][1][0][0][15] = 52, - [0][0][1][0][0][17] = 52, - [0][0][1][0][0][19] = 52, - [0][0][1][0][0][21] = 52, - [0][0][1][0][0][23] = 52, - [0][0][1][0][0][25] = 52, - [0][0][1][0][0][27] = 52, - [0][0][1][0][0][29] = 52, - [0][0][1][0][0][31] = 52, - [0][0][1][0][0][33] = 52, - [0][0][1][0][0][35] = 52, - [0][0][1][0][0][37] = 54, - [0][0][1][0][0][38] = 28, - [0][0][1][0][0][40] = 28, - [0][0][1][0][0][42] = 28, - [0][0][1][0][0][44] = 28, - [0][0][1][0][0][46] = 28, - [0][1][1][0][0][0] = 18, - [0][1][1][0][0][2] = 18, - [0][1][1][0][0][4] = 18, - [0][1][1][0][0][6] = 18, - [0][1][1][0][0][8] = 40, - [0][1][1][0][0][10] = 40, - [0][1][1][0][0][12] = 40, - [0][1][1][0][0][14] = 40, - [0][1][1][0][0][15] = 40, - [0][1][1][0][0][17] = 40, - [0][1][1][0][0][19] = 40, - [0][1][1][0][0][21] = 40, - [0][1][1][0][0][23] = 40, - [0][1][1][0][0][25] = 40, - [0][1][1][0][0][27] = 40, - [0][1][1][0][0][29] = 40, - [0][1][1][0][0][31] = 40, - [0][1][1][0][0][33] = 40, - [0][1][1][0][0][35] = 40, - [0][1][1][0][0][37] = 42, - [0][1][1][0][0][38] = 16, - [0][1][1][0][0][40] = 16, - [0][1][1][0][0][42] = 16, - [0][1][1][0][0][44] = 16, - [0][1][1][0][0][46] = 16, - [0][0][2][0][0][0] = 30, - [0][0][2][0][0][2] = 30, - [0][0][2][0][0][4] = 30, - [0][0][2][0][0][6] = 30, - [0][0][2][0][0][8] = 52, - [0][0][2][0][0][10] = 52, - [0][0][2][0][0][12] = 52, - [0][0][2][0][0][14] = 52, - [0][0][2][0][0][15] = 52, - [0][0][2][0][0][17] = 52, - [0][0][2][0][0][19] = 52, - [0][0][2][0][0][21] = 52, - [0][0][2][0][0][23] = 52, - [0][0][2][0][0][25] = 52, - [0][0][2][0][0][27] = 52, - [0][0][2][0][0][29] = 52, - [0][0][2][0][0][31] = 52, - [0][0][2][0][0][33] = 52, - [0][0][2][0][0][35] = 52, - [0][0][2][0][0][37] = 54, - [0][0][2][0][0][38] = 28, - [0][0][2][0][0][40] = 28, - [0][0][2][0][0][42] = 28, - [0][0][2][0][0][44] = 28, - [0][0][2][0][0][46] = 28, - [0][1][2][0][0][0] = 18, - [0][1][2][0][0][2] = 18, - [0][1][2][0][0][4] = 18, - [0][1][2][0][0][6] = 18, - [0][1][2][0][0][8] = 40, - [0][1][2][0][0][10] = 40, - [0][1][2][0][0][12] = 40, - [0][1][2][0][0][14] = 40, - [0][1][2][0][0][15] = 40, - [0][1][2][0][0][17] = 40, - [0][1][2][0][0][19] = 40, - [0][1][2][0][0][21] = 40, - [0][1][2][0][0][23] = 40, - [0][1][2][0][0][25] = 40, - [0][1][2][0][0][27] = 40, - [0][1][2][0][0][29] = 40, - [0][1][2][0][0][31] = 40, - [0][1][2][0][0][33] = 40, - [0][1][2][0][0][35] = 40, - [0][1][2][0][0][37] = 42, - [0][1][2][0][0][38] = 16, - [0][1][2][0][0][40] = 16, - [0][1][2][0][0][42] = 16, - [0][1][2][0][0][44] = 16, - [0][1][2][0][0][46] = 16, - [0][1][2][1][0][0] = 6, - [0][1][2][1][0][2] = 6, - [0][1][2][1][0][4] = 6, - [0][1][2][1][0][6] = 6, - [0][1][2][1][0][8] = 28, - [0][1][2][1][0][10] = 28, - [0][1][2][1][0][12] = 28, - [0][1][2][1][0][14] = 28, - [0][1][2][1][0][15] = 28, - [0][1][2][1][0][17] = 28, - [0][1][2][1][0][19] = 28, - [0][1][2][1][0][21] = 28, - [0][1][2][1][0][23] = 28, - [0][1][2][1][0][25] = 28, - [0][1][2][1][0][27] = 28, - [0][1][2][1][0][29] = 28, - [0][1][2][1][0][31] = 28, - [0][1][2][1][0][33] = 28, - [0][1][2][1][0][35] = 28, - [0][1][2][1][0][37] = 30, - [0][1][2][1][0][38] = 4, - [0][1][2][1][0][40] = 4, - [0][1][2][1][0][42] = 4, - [0][1][2][1][0][44] = 4, - [0][1][2][1][0][46] = 4, - [1][0][2][0][0][1] = 30, - [1][0][2][0][0][5] = 30, - [1][0][2][0][0][9] = 52, - [1][0][2][0][0][13] = 52, - [1][0][2][0][0][16] = 52, - [1][0][2][0][0][20] = 52, - [1][0][2][0][0][24] = 52, - [1][0][2][0][0][28] = 52, - [1][0][2][0][0][32] = 52, - [1][0][2][0][0][36] = 54, - [1][0][2][0][0][39] = 28, - [1][0][2][0][0][43] = 28, - [1][1][2][0][0][1] = 18, - [1][1][2][0][0][5] = 18, - [1][1][2][0][0][9] = 40, - [1][1][2][0][0][13] = 40, - [1][1][2][0][0][16] = 40, - [1][1][2][0][0][20] = 40, - [1][1][2][0][0][24] = 40, - [1][1][2][0][0][28] = 40, - [1][1][2][0][0][32] = 40, - [1][1][2][0][0][36] = 42, - [1][1][2][0][0][39] = 16, - [1][1][2][0][0][43] = 16, - [1][1][2][1][0][1] = 6, - [1][1][2][1][0][5] = 6, - [1][1][2][1][0][9] = 28, - [1][1][2][1][0][13] = 28, - [1][1][2][1][0][16] = 28, - [1][1][2][1][0][20] = 28, - [1][1][2][1][0][24] = 28, - [1][1][2][1][0][28] = 28, - [1][1][2][1][0][32] = 28, - [1][1][2][1][0][36] = 30, - [1][1][2][1][0][39] = 4, - [1][1][2][1][0][43] = 4, - [2][0][2][0][0][3] = 30, - [2][0][2][0][0][11] = 52, - [2][0][2][0][0][18] = 52, - [2][0][2][0][0][26] = 52, - [2][0][2][0][0][34] = 54, - [2][0][2][0][0][41] = 28, - [2][1][2][0][0][3] = 18, - [2][1][2][0][0][11] = 40, - [2][1][2][0][0][18] = 40, - [2][1][2][0][0][26] = 40, - [2][1][2][0][0][34] = 42, - [2][1][2][0][0][41] = 16, - [2][1][2][1][0][3] = 6, - [2][1][2][1][0][11] = 28, - [2][1][2][1][0][18] = 28, - [2][1][2][1][0][26] = 28, - [2][1][2][1][0][34] = 30, - [2][1][2][1][0][41] = 4, - [0][0][1][0][2][0] = 76, - [0][0][1][0][1][0] = 58, - [0][0][1][0][3][0] = 62, - [0][0][1][0][5][0] = 62, - [0][0][1][0][6][0] = 58, - [0][0][1][0][9][0] = 58, - [0][0][1][0][8][0] = 30, - [0][0][1][0][11][0] = 52, - [0][0][1][0][2][2] = 76, - [0][0][1][0][1][2] = 58, - [0][0][1][0][3][2] = 62, - [0][0][1][0][5][2] = 62, - [0][0][1][0][6][2] = 58, - [0][0][1][0][9][2] = 58, - [0][0][1][0][8][2] = 30, - [0][0][1][0][11][2] = 52, - [0][0][1][0][2][4] = 76, - [0][0][1][0][1][4] = 58, - [0][0][1][0][3][4] = 62, - [0][0][1][0][5][4] = 62, - [0][0][1][0][6][4] = 58, - [0][0][1][0][9][4] = 58, - [0][0][1][0][8][4] = 30, - [0][0][1][0][11][4] = 52, - [0][0][1][0][2][6] = 76, - [0][0][1][0][1][6] = 58, - [0][0][1][0][3][6] = 62, - [0][0][1][0][5][6] = 62, - [0][0][1][0][6][6] = 54, - [0][0][1][0][9][6] = 58, - [0][0][1][0][8][6] = 30, - [0][0][1][0][11][6] = 52, - [0][0][1][0][2][8] = 76, - [0][0][1][0][1][8] = 58, - [0][0][1][0][3][8] = 62, - [0][0][1][0][5][8] = 64, - [0][0][1][0][6][8] = 58, - [0][0][1][0][9][8] = 58, - [0][0][1][0][8][8] = 54, - [0][0][1][0][11][8] = 52, - [0][0][1][0][2][10] = 76, - [0][0][1][0][1][10] = 58, - [0][0][1][0][3][10] = 62, - [0][0][1][0][5][10] = 64, - [0][0][1][0][6][10] = 58, - [0][0][1][0][9][10] = 58, - [0][0][1][0][8][10] = 54, - [0][0][1][0][11][10] = 52, - [0][0][1][0][2][12] = 76, - [0][0][1][0][1][12] = 58, - [0][0][1][0][3][12] = 62, - [0][0][1][0][5][12] = 64, - [0][0][1][0][6][12] = 58, - [0][0][1][0][9][12] = 58, - [0][0][1][0][8][12] = 54, - [0][0][1][0][11][12] = 52, - [0][0][1][0][2][14] = 76, - [0][0][1][0][1][14] = 58, - [0][0][1][0][3][14] = 62, - [0][0][1][0][5][14] = 64, - [0][0][1][0][6][14] = 58, - [0][0][1][0][9][14] = 58, - [0][0][1][0][8][14] = 54, - [0][0][1][0][11][14] = 52, - [0][0][1][0][2][15] = 76, - [0][0][1][0][1][15] = 58, - [0][0][1][0][3][15] = 76, - [0][0][1][0][5][15] = 76, - [0][0][1][0][6][15] = 58, - [0][0][1][0][9][15] = 58, - [0][0][1][0][8][15] = 54, - [0][0][1][0][11][15] = 52, - [0][0][1][0][2][17] = 76, - [0][0][1][0][1][17] = 58, - [0][0][1][0][3][17] = 76, - [0][0][1][0][5][17] = 76, - [0][0][1][0][6][17] = 58, - [0][0][1][0][9][17] = 58, - [0][0][1][0][8][17] = 54, - [0][0][1][0][11][17] = 52, - [0][0][1][0][2][19] = 76, - [0][0][1][0][1][19] = 58, - [0][0][1][0][3][19] = 76, - [0][0][1][0][5][19] = 76, - [0][0][1][0][6][19] = 58, - [0][0][1][0][9][19] = 58, - [0][0][1][0][8][19] = 54, - [0][0][1][0][11][19] = 52, - [0][0][1][0][2][21] = 76, - [0][0][1][0][1][21] = 58, - [0][0][1][0][3][21] = 76, - [0][0][1][0][5][21] = 76, - [0][0][1][0][6][21] = 58, - [0][0][1][0][9][21] = 58, - [0][0][1][0][8][21] = 54, - [0][0][1][0][11][21] = 52, - [0][0][1][0][2][23] = 76, - [0][0][1][0][1][23] = 58, - [0][0][1][0][3][23] = 76, - [0][0][1][0][5][23] = 76, - [0][0][1][0][6][23] = 58, - [0][0][1][0][9][23] = 58, - [0][0][1][0][8][23] = 54, - [0][0][1][0][11][23] = 52, - [0][0][1][0][2][25] = 76, - [0][0][1][0][1][25] = 58, - [0][0][1][0][3][25] = 76, - [0][0][1][0][5][25] = 127, - [0][0][1][0][6][25] = 58, - [0][0][1][0][9][25] = 127, - [0][0][1][0][8][25] = 54, - [0][0][1][0][11][25] = 52, - [0][0][1][0][2][27] = 76, - [0][0][1][0][1][27] = 58, - [0][0][1][0][3][27] = 76, - [0][0][1][0][5][27] = 127, - [0][0][1][0][6][27] = 58, - [0][0][1][0][9][27] = 127, - [0][0][1][0][8][27] = 54, - [0][0][1][0][11][27] = 52, - [0][0][1][0][2][29] = 76, - [0][0][1][0][1][29] = 58, - [0][0][1][0][3][29] = 76, - [0][0][1][0][5][29] = 127, - [0][0][1][0][6][29] = 58, - [0][0][1][0][9][29] = 127, - [0][0][1][0][8][29] = 54, - [0][0][1][0][11][29] = 52, - [0][0][1][0][2][31] = 76, - [0][0][1][0][1][31] = 58, - [0][0][1][0][3][31] = 76, - [0][0][1][0][5][31] = 76, - [0][0][1][0][6][31] = 58, - [0][0][1][0][9][31] = 58, - [0][0][1][0][8][31] = 54, - [0][0][1][0][11][31] = 52, - [0][0][1][0][2][33] = 76, - [0][0][1][0][1][33] = 58, - [0][0][1][0][3][33] = 76, - [0][0][1][0][5][33] = 76, - [0][0][1][0][6][33] = 58, - [0][0][1][0][9][33] = 58, - [0][0][1][0][8][33] = 54, - [0][0][1][0][11][33] = 52, - [0][0][1][0][2][35] = 74, - [0][0][1][0][1][35] = 58, - [0][0][1][0][3][35] = 76, - [0][0][1][0][5][35] = 74, - [0][0][1][0][6][35] = 58, - [0][0][1][0][9][35] = 58, - [0][0][1][0][8][35] = 54, - [0][0][1][0][11][35] = 52, - [0][0][1][0][2][37] = 76, - [0][0][1][0][1][37] = 127, - [0][0][1][0][3][37] = 76, - [0][0][1][0][5][37] = 76, - [0][0][1][0][6][37] = 58, - [0][0][1][0][9][37] = 76, - [0][0][1][0][8][37] = 54, - [0][0][1][0][11][37] = 127, - [0][0][1][0][2][38] = 76, - [0][0][1][0][1][38] = 28, - [0][0][1][0][3][38] = 127, - [0][0][1][0][5][38] = 76, - [0][0][1][0][6][38] = 28, - [0][0][1][0][9][38] = 76, - [0][0][1][0][8][38] = 54, - [0][0][1][0][11][38] = 52, - [0][0][1][0][2][40] = 76, - [0][0][1][0][1][40] = 28, - [0][0][1][0][3][40] = 127, - [0][0][1][0][5][40] = 76, - [0][0][1][0][6][40] = 28, - [0][0][1][0][9][40] = 76, - [0][0][1][0][8][40] = 54, - [0][0][1][0][11][40] = 52, - [0][0][1][0][2][42] = 76, - [0][0][1][0][1][42] = 28, - [0][0][1][0][3][42] = 127, - [0][0][1][0][5][42] = 76, - [0][0][1][0][6][42] = 28, - [0][0][1][0][9][42] = 76, - [0][0][1][0][8][42] = 54, - [0][0][1][0][11][42] = 52, - [0][0][1][0][2][44] = 76, - [0][0][1][0][1][44] = 28, - [0][0][1][0][3][44] = 127, - [0][0][1][0][5][44] = 76, - [0][0][1][0][6][44] = 28, - [0][0][1][0][9][44] = 76, - [0][0][1][0][8][44] = 54, - [0][0][1][0][11][44] = 52, - [0][0][1][0][2][46] = 76, - [0][0][1][0][1][46] = 28, - [0][0][1][0][3][46] = 127, - [0][0][1][0][5][46] = 76, - [0][0][1][0][6][46] = 28, - [0][0][1][0][9][46] = 76, - [0][0][1][0][8][46] = 54, - [0][0][1][0][11][46] = 52, - [0][1][1][0][2][0] = 68, - [0][1][1][0][1][0] = 46, - [0][1][1][0][3][0] = 50, - [0][1][1][0][5][0] = 40, - [0][1][1][0][6][0] = 46, - [0][1][1][0][9][0] = 46, - [0][1][1][0][8][0] = 18, - [0][1][1][0][11][0] = 40, - [0][1][1][0][2][2] = 68, - [0][1][1][0][1][2] = 46, - [0][1][1][0][3][2] = 50, - [0][1][1][0][5][2] = 40, - [0][1][1][0][6][2] = 46, - [0][1][1][0][9][2] = 46, - [0][1][1][0][8][2] = 18, - [0][1][1][0][11][2] = 40, - [0][1][1][0][2][4] = 68, - [0][1][1][0][1][4] = 46, - [0][1][1][0][3][4] = 50, - [0][1][1][0][5][4] = 40, - [0][1][1][0][6][4] = 46, - [0][1][1][0][9][4] = 46, - [0][1][1][0][8][4] = 18, - [0][1][1][0][11][4] = 40, - [0][1][1][0][2][6] = 68, - [0][1][1][0][1][6] = 46, - [0][1][1][0][3][6] = 50, - [0][1][1][0][5][6] = 40, - [0][1][1][0][6][6] = 36, - [0][1][1][0][9][6] = 46, - [0][1][1][0][8][6] = 18, - [0][1][1][0][11][6] = 40, - [0][1][1][0][2][8] = 68, - [0][1][1][0][1][8] = 46, - [0][1][1][0][3][8] = 50, - [0][1][1][0][5][8] = 52, - [0][1][1][0][6][8] = 46, - [0][1][1][0][9][8] = 46, - [0][1][1][0][8][8] = 42, - [0][1][1][0][11][8] = 40, - [0][1][1][0][2][10] = 68, - [0][1][1][0][1][10] = 46, - [0][1][1][0][3][10] = 50, - [0][1][1][0][5][10] = 52, - [0][1][1][0][6][10] = 46, - [0][1][1][0][9][10] = 46, - [0][1][1][0][8][10] = 42, - [0][1][1][0][11][10] = 40, - [0][1][1][0][2][12] = 68, - [0][1][1][0][1][12] = 46, - [0][1][1][0][3][12] = 50, - [0][1][1][0][5][12] = 52, - [0][1][1][0][6][12] = 46, - [0][1][1][0][9][12] = 46, - [0][1][1][0][8][12] = 42, - [0][1][1][0][11][12] = 40, - [0][1][1][0][2][14] = 68, - [0][1][1][0][1][14] = 46, - [0][1][1][0][3][14] = 50, - [0][1][1][0][5][14] = 52, - [0][1][1][0][6][14] = 46, - [0][1][1][0][9][14] = 46, - [0][1][1][0][8][14] = 42, - [0][1][1][0][11][14] = 40, - [0][1][1][0][2][15] = 68, - [0][1][1][0][1][15] = 46, - [0][1][1][0][3][15] = 70, - [0][1][1][0][5][15] = 68, - [0][1][1][0][6][15] = 46, - [0][1][1][0][9][15] = 46, - [0][1][1][0][8][15] = 42, - [0][1][1][0][11][15] = 40, - [0][1][1][0][2][17] = 68, - [0][1][1][0][1][17] = 46, - [0][1][1][0][3][17] = 70, - [0][1][1][0][5][17] = 68, - [0][1][1][0][6][17] = 46, - [0][1][1][0][9][17] = 46, - [0][1][1][0][8][17] = 42, - [0][1][1][0][11][17] = 40, - [0][1][1][0][2][19] = 68, - [0][1][1][0][1][19] = 46, - [0][1][1][0][3][19] = 70, - [0][1][1][0][5][19] = 68, - [0][1][1][0][6][19] = 46, - [0][1][1][0][9][19] = 46, - [0][1][1][0][8][19] = 42, - [0][1][1][0][11][19] = 40, - [0][1][1][0][2][21] = 68, - [0][1][1][0][1][21] = 46, - [0][1][1][0][3][21] = 70, - [0][1][1][0][5][21] = 68, - [0][1][1][0][6][21] = 46, - [0][1][1][0][9][21] = 46, - [0][1][1][0][8][21] = 42, - [0][1][1][0][11][21] = 40, - [0][1][1][0][2][23] = 68, - [0][1][1][0][1][23] = 46, - [0][1][1][0][3][23] = 70, - [0][1][1][0][5][23] = 68, - [0][1][1][0][6][23] = 46, - [0][1][1][0][9][23] = 46, - [0][1][1][0][8][23] = 42, - [0][1][1][0][11][23] = 40, - [0][1][1][0][2][25] = 68, - [0][1][1][0][1][25] = 46, - [0][1][1][0][3][25] = 70, - [0][1][1][0][5][25] = 127, - [0][1][1][0][6][25] = 46, - [0][1][1][0][9][25] = 127, - [0][1][1][0][8][25] = 42, - [0][1][1][0][11][25] = 40, - [0][1][1][0][2][27] = 68, - [0][1][1][0][1][27] = 46, - [0][1][1][0][3][27] = 70, - [0][1][1][0][5][27] = 127, - [0][1][1][0][6][27] = 46, - [0][1][1][0][9][27] = 127, - [0][1][1][0][8][27] = 42, - [0][1][1][0][11][27] = 40, - [0][1][1][0][2][29] = 68, - [0][1][1][0][1][29] = 46, - [0][1][1][0][3][29] = 70, - [0][1][1][0][5][29] = 127, - [0][1][1][0][6][29] = 46, - [0][1][1][0][9][29] = 127, - [0][1][1][0][8][29] = 42, - [0][1][1][0][11][29] = 40, - [0][1][1][0][2][31] = 68, - [0][1][1][0][1][31] = 46, - [0][1][1][0][3][31] = 70, - [0][1][1][0][5][31] = 68, - [0][1][1][0][6][31] = 46, - [0][1][1][0][9][31] = 46, - [0][1][1][0][8][31] = 42, - [0][1][1][0][11][31] = 40, - [0][1][1][0][2][33] = 68, - [0][1][1][0][1][33] = 46, - [0][1][1][0][3][33] = 70, - [0][1][1][0][5][33] = 68, - [0][1][1][0][6][33] = 46, - [0][1][1][0][9][33] = 46, - [0][1][1][0][8][33] = 42, - [0][1][1][0][11][33] = 40, - [0][1][1][0][2][35] = 66, - [0][1][1][0][1][35] = 46, - [0][1][1][0][3][35] = 70, - [0][1][1][0][5][35] = 66, - [0][1][1][0][6][35] = 46, - [0][1][1][0][9][35] = 46, - [0][1][1][0][8][35] = 42, - [0][1][1][0][11][35] = 40, - [0][1][1][0][2][37] = 68, - [0][1][1][0][1][37] = 127, - [0][1][1][0][3][37] = 70, - [0][1][1][0][5][37] = 68, - [0][1][1][0][6][37] = 46, - [0][1][1][0][9][37] = 68, - [0][1][1][0][8][37] = 42, - [0][1][1][0][11][37] = 127, - [0][1][1][0][2][38] = 76, - [0][1][1][0][1][38] = 16, - [0][1][1][0][3][38] = 127, - [0][1][1][0][5][38] = 76, - [0][1][1][0][6][38] = 16, - [0][1][1][0][9][38] = 76, - [0][1][1][0][8][38] = 42, - [0][1][1][0][11][38] = 40, - [0][1][1][0][2][40] = 76, - [0][1][1][0][1][40] = 16, - [0][1][1][0][3][40] = 127, - [0][1][1][0][5][40] = 76, - [0][1][1][0][6][40] = 16, - [0][1][1][0][9][40] = 76, - [0][1][1][0][8][40] = 42, - [0][1][1][0][11][40] = 40, - [0][1][1][0][2][42] = 76, - [0][1][1][0][1][42] = 16, - [0][1][1][0][3][42] = 127, - [0][1][1][0][5][42] = 76, - [0][1][1][0][6][42] = 16, - [0][1][1][0][9][42] = 76, - [0][1][1][0][8][42] = 42, - [0][1][1][0][11][42] = 40, - [0][1][1][0][2][44] = 76, - [0][1][1][0][1][44] = 16, - [0][1][1][0][3][44] = 127, - [0][1][1][0][5][44] = 76, - [0][1][1][0][6][44] = 16, - [0][1][1][0][9][44] = 76, - [0][1][1][0][8][44] = 42, - [0][1][1][0][11][44] = 40, - [0][1][1][0][2][46] = 76, - [0][1][1][0][1][46] = 16, - [0][1][1][0][3][46] = 127, - [0][1][1][0][5][46] = 76, - [0][1][1][0][6][46] = 16, - [0][1][1][0][9][46] = 76, - [0][1][1][0][8][46] = 42, - [0][1][1][0][11][46] = 40, - [0][0][2][0][2][0] = 76, - [0][0][2][0][1][0] = 58, - [0][0][2][0][3][0] = 62, - [0][0][2][0][5][0] = 62, - [0][0][2][0][6][0] = 58, - [0][0][2][0][9][0] = 58, - [0][0][2][0][8][0] = 30, - [0][0][2][0][11][0] = 52, - [0][0][2][0][2][2] = 76, - [0][0][2][0][1][2] = 58, - [0][0][2][0][3][2] = 62, - [0][0][2][0][5][2] = 62, - [0][0][2][0][6][2] = 58, - [0][0][2][0][9][2] = 58, - [0][0][2][0][8][2] = 30, - [0][0][2][0][11][2] = 52, - [0][0][2][0][2][4] = 76, - [0][0][2][0][1][4] = 58, - [0][0][2][0][3][4] = 62, - [0][0][2][0][5][4] = 62, - [0][0][2][0][6][4] = 58, - [0][0][2][0][9][4] = 58, - [0][0][2][0][8][4] = 30, - [0][0][2][0][11][4] = 52, - [0][0][2][0][2][6] = 76, - [0][0][2][0][1][6] = 58, - [0][0][2][0][3][6] = 62, - [0][0][2][0][5][6] = 62, - [0][0][2][0][6][6] = 54, - [0][0][2][0][9][6] = 58, - [0][0][2][0][8][6] = 30, - [0][0][2][0][11][6] = 52, - [0][0][2][0][2][8] = 76, - [0][0][2][0][1][8] = 58, - [0][0][2][0][3][8] = 62, - [0][0][2][0][5][8] = 64, - [0][0][2][0][6][8] = 58, - [0][0][2][0][9][8] = 58, - [0][0][2][0][8][8] = 54, - [0][0][2][0][11][8] = 52, - [0][0][2][0][2][10] = 76, - [0][0][2][0][1][10] = 58, - [0][0][2][0][3][10] = 62, - [0][0][2][0][5][10] = 64, - [0][0][2][0][6][10] = 58, - [0][0][2][0][9][10] = 58, - [0][0][2][0][8][10] = 54, - [0][0][2][0][11][10] = 52, - [0][0][2][0][2][12] = 76, - [0][0][2][0][1][12] = 58, - [0][0][2][0][3][12] = 62, - [0][0][2][0][5][12] = 64, - [0][0][2][0][6][12] = 58, - [0][0][2][0][9][12] = 58, - [0][0][2][0][8][12] = 54, - [0][0][2][0][11][12] = 52, - [0][0][2][0][2][14] = 76, - [0][0][2][0][1][14] = 58, - [0][0][2][0][3][14] = 62, - [0][0][2][0][5][14] = 64, - [0][0][2][0][6][14] = 58, - [0][0][2][0][9][14] = 58, - [0][0][2][0][8][14] = 54, - [0][0][2][0][11][14] = 52, - [0][0][2][0][2][15] = 74, - [0][0][2][0][1][15] = 58, - [0][0][2][0][3][15] = 76, - [0][0][2][0][5][15] = 74, - [0][0][2][0][6][15] = 58, - [0][0][2][0][9][15] = 58, - [0][0][2][0][8][15] = 54, - [0][0][2][0][11][15] = 52, - [0][0][2][0][2][17] = 76, - [0][0][2][0][1][17] = 58, - [0][0][2][0][3][17] = 76, - [0][0][2][0][5][17] = 76, - [0][0][2][0][6][17] = 58, - [0][0][2][0][9][17] = 58, - [0][0][2][0][8][17] = 54, - [0][0][2][0][11][17] = 52, - [0][0][2][0][2][19] = 76, - [0][0][2][0][1][19] = 58, - [0][0][2][0][3][19] = 76, - [0][0][2][0][5][19] = 76, - [0][0][2][0][6][19] = 58, - [0][0][2][0][9][19] = 58, - [0][0][2][0][8][19] = 54, - [0][0][2][0][11][19] = 52, - [0][0][2][0][2][21] = 76, - [0][0][2][0][1][21] = 58, - [0][0][2][0][3][21] = 76, - [0][0][2][0][5][21] = 76, - [0][0][2][0][6][21] = 58, - [0][0][2][0][9][21] = 58, - [0][0][2][0][8][21] = 54, - [0][0][2][0][11][21] = 52, - [0][0][2][0][2][23] = 76, - [0][0][2][0][1][23] = 58, - [0][0][2][0][3][23] = 76, - [0][0][2][0][5][23] = 76, - [0][0][2][0][6][23] = 58, - [0][0][2][0][9][23] = 58, - [0][0][2][0][8][23] = 54, - [0][0][2][0][11][23] = 52, - [0][0][2][0][2][25] = 76, - [0][0][2][0][1][25] = 58, - [0][0][2][0][3][25] = 76, - [0][0][2][0][5][25] = 127, - [0][0][2][0][6][25] = 58, - [0][0][2][0][9][25] = 127, - [0][0][2][0][8][25] = 54, - [0][0][2][0][11][25] = 52, - [0][0][2][0][2][27] = 76, - [0][0][2][0][1][27] = 58, - [0][0][2][0][3][27] = 76, - [0][0][2][0][5][27] = 127, - [0][0][2][0][6][27] = 58, - [0][0][2][0][9][27] = 127, - [0][0][2][0][8][27] = 54, - [0][0][2][0][11][27] = 52, - [0][0][2][0][2][29] = 76, - [0][0][2][0][1][29] = 58, - [0][0][2][0][3][29] = 76, - [0][0][2][0][5][29] = 127, - [0][0][2][0][6][29] = 58, - [0][0][2][0][9][29] = 127, - [0][0][2][0][8][29] = 54, - [0][0][2][0][11][29] = 52, - [0][0][2][0][2][31] = 76, - [0][0][2][0][1][31] = 58, - [0][0][2][0][3][31] = 76, - [0][0][2][0][5][31] = 76, - [0][0][2][0][6][31] = 58, - [0][0][2][0][9][31] = 58, - [0][0][2][0][8][31] = 54, - [0][0][2][0][11][31] = 52, - [0][0][2][0][2][33] = 76, - [0][0][2][0][1][33] = 58, - [0][0][2][0][3][33] = 76, - [0][0][2][0][5][33] = 76, - [0][0][2][0][6][33] = 58, - [0][0][2][0][9][33] = 58, - [0][0][2][0][8][33] = 54, - [0][0][2][0][11][33] = 52, - [0][0][2][0][2][35] = 70, - [0][0][2][0][1][35] = 58, - [0][0][2][0][3][35] = 76, - [0][0][2][0][5][35] = 70, - [0][0][2][0][6][35] = 58, - [0][0][2][0][9][35] = 58, - [0][0][2][0][8][35] = 54, - [0][0][2][0][11][35] = 52, - [0][0][2][0][2][37] = 76, - [0][0][2][0][1][37] = 127, - [0][0][2][0][3][37] = 76, - [0][0][2][0][5][37] = 76, - [0][0][2][0][6][37] = 58, - [0][0][2][0][9][37] = 76, - [0][0][2][0][8][37] = 54, - [0][0][2][0][11][37] = 127, - [0][0][2][0][2][38] = 76, - [0][0][2][0][1][38] = 28, - [0][0][2][0][3][38] = 127, - [0][0][2][0][5][38] = 76, - [0][0][2][0][6][38] = 28, - [0][0][2][0][9][38] = 76, - [0][0][2][0][8][38] = 54, - [0][0][2][0][11][38] = 52, - [0][0][2][0][2][40] = 76, - [0][0][2][0][1][40] = 28, - [0][0][2][0][3][40] = 127, - [0][0][2][0][5][40] = 76, - [0][0][2][0][6][40] = 28, - [0][0][2][0][9][40] = 76, - [0][0][2][0][8][40] = 54, - [0][0][2][0][11][40] = 52, - [0][0][2][0][2][42] = 76, - [0][0][2][0][1][42] = 28, - [0][0][2][0][3][42] = 127, - [0][0][2][0][5][42] = 76, - [0][0][2][0][6][42] = 28, - [0][0][2][0][9][42] = 76, - [0][0][2][0][8][42] = 54, - [0][0][2][0][11][42] = 52, - [0][0][2][0][2][44] = 76, - [0][0][2][0][1][44] = 28, - [0][0][2][0][3][44] = 127, - [0][0][2][0][5][44] = 76, - [0][0][2][0][6][44] = 28, - [0][0][2][0][9][44] = 76, - [0][0][2][0][8][44] = 54, - [0][0][2][0][11][44] = 52, - [0][0][2][0][2][46] = 76, - [0][0][2][0][1][46] = 28, - [0][0][2][0][3][46] = 127, - [0][0][2][0][5][46] = 76, - [0][0][2][0][6][46] = 28, - [0][0][2][0][9][46] = 76, - [0][0][2][0][8][46] = 54, - [0][0][2][0][11][46] = 52, - [0][1][2][0][2][0] = 68, - [0][1][2][0][1][0] = 46, - [0][1][2][0][3][0] = 50, - [0][1][2][0][5][0] = 40, - [0][1][2][0][6][0] = 46, - [0][1][2][0][9][0] = 46, - [0][1][2][0][8][0] = 18, - [0][1][2][0][11][0] = 40, - [0][1][2][0][2][2] = 68, - [0][1][2][0][1][2] = 46, - [0][1][2][0][3][2] = 50, - [0][1][2][0][5][2] = 40, - [0][1][2][0][6][2] = 46, - [0][1][2][0][9][2] = 46, - [0][1][2][0][8][2] = 18, - [0][1][2][0][11][2] = 40, - [0][1][2][0][2][4] = 68, - [0][1][2][0][1][4] = 46, - [0][1][2][0][3][4] = 50, - [0][1][2][0][5][4] = 40, - [0][1][2][0][6][4] = 46, - [0][1][2][0][9][4] = 46, - [0][1][2][0][8][4] = 18, - [0][1][2][0][11][4] = 40, - [0][1][2][0][2][6] = 68, - [0][1][2][0][1][6] = 46, - [0][1][2][0][3][6] = 50, - [0][1][2][0][5][6] = 40, - [0][1][2][0][6][6] = 36, - [0][1][2][0][9][6] = 46, - [0][1][2][0][8][6] = 18, - [0][1][2][0][11][6] = 40, - [0][1][2][0][2][8] = 68, - [0][1][2][0][1][8] = 46, - [0][1][2][0][3][8] = 50, - [0][1][2][0][5][8] = 52, - [0][1][2][0][6][8] = 46, - [0][1][2][0][9][8] = 46, - [0][1][2][0][8][8] = 42, - [0][1][2][0][11][8] = 40, - [0][1][2][0][2][10] = 68, - [0][1][2][0][1][10] = 46, - [0][1][2][0][3][10] = 50, - [0][1][2][0][5][10] = 52, - [0][1][2][0][6][10] = 46, - [0][1][2][0][9][10] = 46, - [0][1][2][0][8][10] = 42, - [0][1][2][0][11][10] = 40, - [0][1][2][0][2][12] = 68, - [0][1][2][0][1][12] = 46, - [0][1][2][0][3][12] = 50, - [0][1][2][0][5][12] = 52, - [0][1][2][0][6][12] = 46, - [0][1][2][0][9][12] = 46, - [0][1][2][0][8][12] = 42, - [0][1][2][0][11][12] = 40, - [0][1][2][0][2][14] = 68, - [0][1][2][0][1][14] = 46, - [0][1][2][0][3][14] = 50, - [0][1][2][0][5][14] = 52, - [0][1][2][0][6][14] = 46, - [0][1][2][0][9][14] = 46, - [0][1][2][0][8][14] = 42, - [0][1][2][0][11][14] = 40, - [0][1][2][0][2][15] = 68, - [0][1][2][0][1][15] = 46, - [0][1][2][0][3][15] = 70, - [0][1][2][0][5][15] = 68, - [0][1][2][0][6][15] = 46, - [0][1][2][0][9][15] = 46, - [0][1][2][0][8][15] = 42, - [0][1][2][0][11][15] = 40, - [0][1][2][0][2][17] = 68, - [0][1][2][0][1][17] = 46, - [0][1][2][0][3][17] = 70, - [0][1][2][0][5][17] = 68, - [0][1][2][0][6][17] = 46, - [0][1][2][0][9][17] = 46, - [0][1][2][0][8][17] = 42, - [0][1][2][0][11][17] = 40, - [0][1][2][0][2][19] = 68, - [0][1][2][0][1][19] = 46, - [0][1][2][0][3][19] = 70, - [0][1][2][0][5][19] = 68, - [0][1][2][0][6][19] = 46, - [0][1][2][0][9][19] = 46, - [0][1][2][0][8][19] = 42, - [0][1][2][0][11][19] = 40, - [0][1][2][0][2][21] = 68, - [0][1][2][0][1][21] = 46, - [0][1][2][0][3][21] = 70, - [0][1][2][0][5][21] = 68, - [0][1][2][0][6][21] = 46, - [0][1][2][0][9][21] = 46, - [0][1][2][0][8][21] = 42, - [0][1][2][0][11][21] = 40, - [0][1][2][0][2][23] = 68, - [0][1][2][0][1][23] = 46, - [0][1][2][0][3][23] = 70, - [0][1][2][0][5][23] = 68, - [0][1][2][0][6][23] = 46, - [0][1][2][0][9][23] = 46, - [0][1][2][0][8][23] = 42, - [0][1][2][0][11][23] = 40, - [0][1][2][0][2][25] = 68, - [0][1][2][0][1][25] = 46, - [0][1][2][0][3][25] = 70, - [0][1][2][0][5][25] = 127, - [0][1][2][0][6][25] = 46, - [0][1][2][0][9][25] = 127, - [0][1][2][0][8][25] = 42, - [0][1][2][0][11][25] = 40, - [0][1][2][0][2][27] = 68, - [0][1][2][0][1][27] = 46, - [0][1][2][0][3][27] = 70, - [0][1][2][0][5][27] = 127, - [0][1][2][0][6][27] = 46, - [0][1][2][0][9][27] = 127, - [0][1][2][0][8][27] = 42, - [0][1][2][0][11][27] = 40, - [0][1][2][0][2][29] = 68, - [0][1][2][0][1][29] = 46, - [0][1][2][0][3][29] = 70, - [0][1][2][0][5][29] = 127, - [0][1][2][0][6][29] = 46, - [0][1][2][0][9][29] = 127, - [0][1][2][0][8][29] = 42, - [0][1][2][0][11][29] = 40, - [0][1][2][0][2][31] = 68, - [0][1][2][0][1][31] = 46, - [0][1][2][0][3][31] = 70, - [0][1][2][0][5][31] = 68, - [0][1][2][0][6][31] = 46, - [0][1][2][0][9][31] = 46, - [0][1][2][0][8][31] = 42, - [0][1][2][0][11][31] = 40, - [0][1][2][0][2][33] = 68, - [0][1][2][0][1][33] = 46, - [0][1][2][0][3][33] = 70, - [0][1][2][0][5][33] = 68, - [0][1][2][0][6][33] = 46, - [0][1][2][0][9][33] = 46, - [0][1][2][0][8][33] = 42, - [0][1][2][0][11][33] = 40, - [0][1][2][0][2][35] = 64, - [0][1][2][0][1][35] = 46, - [0][1][2][0][3][35] = 70, - [0][1][2][0][5][35] = 64, - [0][1][2][0][6][35] = 46, - [0][1][2][0][9][35] = 46, - [0][1][2][0][8][35] = 42, - [0][1][2][0][11][35] = 40, - [0][1][2][0][2][37] = 68, - [0][1][2][0][1][37] = 127, - [0][1][2][0][3][37] = 70, - [0][1][2][0][5][37] = 68, - [0][1][2][0][6][37] = 46, - [0][1][2][0][9][37] = 68, - [0][1][2][0][8][37] = 42, - [0][1][2][0][11][37] = 127, - [0][1][2][0][2][38] = 76, - [0][1][2][0][1][38] = 16, - [0][1][2][0][3][38] = 127, - [0][1][2][0][5][38] = 76, - [0][1][2][0][6][38] = 16, - [0][1][2][0][9][38] = 76, - [0][1][2][0][8][38] = 42, - [0][1][2][0][11][38] = 40, - [0][1][2][0][2][40] = 76, - [0][1][2][0][1][40] = 16, - [0][1][2][0][3][40] = 127, - [0][1][2][0][5][40] = 76, - [0][1][2][0][6][40] = 16, - [0][1][2][0][9][40] = 76, - [0][1][2][0][8][40] = 42, - [0][1][2][0][11][40] = 40, - [0][1][2][0][2][42] = 76, - [0][1][2][0][1][42] = 16, - [0][1][2][0][3][42] = 127, - [0][1][2][0][5][42] = 76, - [0][1][2][0][6][42] = 16, - [0][1][2][0][9][42] = 76, - [0][1][2][0][8][42] = 42, - [0][1][2][0][11][42] = 40, - [0][1][2][0][2][44] = 76, - [0][1][2][0][1][44] = 16, - [0][1][2][0][3][44] = 127, - [0][1][2][0][5][44] = 76, - [0][1][2][0][6][44] = 16, - [0][1][2][0][9][44] = 76, - [0][1][2][0][8][44] = 42, - [0][1][2][0][11][44] = 40, - [0][1][2][0][2][46] = 76, - [0][1][2][0][1][46] = 16, - [0][1][2][0][3][46] = 127, - [0][1][2][0][5][46] = 76, - [0][1][2][0][6][46] = 16, - [0][1][2][0][9][46] = 76, - [0][1][2][0][8][46] = 42, - [0][1][2][0][11][46] = 40, - [0][1][2][1][2][0] = 68, - [0][1][2][1][1][0] = 34, - [0][1][2][1][3][0] = 50, - [0][1][2][1][5][0] = 38, - [0][1][2][1][6][0] = 34, - [0][1][2][1][9][0] = 34, - [0][1][2][1][8][0] = 6, - [0][1][2][1][11][0] = 28, - [0][1][2][1][2][2] = 68, - [0][1][2][1][1][2] = 34, - [0][1][2][1][3][2] = 50, - [0][1][2][1][5][2] = 38, - [0][1][2][1][6][2] = 34, - [0][1][2][1][9][2] = 34, - [0][1][2][1][8][2] = 6, - [0][1][2][1][11][2] = 28, - [0][1][2][1][2][4] = 68, - [0][1][2][1][1][4] = 34, - [0][1][2][1][3][4] = 50, - [0][1][2][1][5][4] = 38, - [0][1][2][1][6][4] = 34, - [0][1][2][1][9][4] = 34, - [0][1][2][1][8][4] = 6, - [0][1][2][1][11][4] = 28, - [0][1][2][1][2][6] = 68, - [0][1][2][1][1][6] = 34, - [0][1][2][1][3][6] = 50, - [0][1][2][1][5][6] = 38, - [0][1][2][1][6][6] = 34, - [0][1][2][1][9][6] = 34, - [0][1][2][1][8][6] = 6, - [0][1][2][1][11][6] = 28, - [0][1][2][1][2][8] = 68, - [0][1][2][1][1][8] = 34, - [0][1][2][1][3][8] = 50, - [0][1][2][1][5][8] = 38, - [0][1][2][1][6][8] = 34, - [0][1][2][1][9][8] = 34, - [0][1][2][1][8][8] = 30, - [0][1][2][1][11][8] = 28, - [0][1][2][1][2][10] = 68, - [0][1][2][1][1][10] = 34, - [0][1][2][1][3][10] = 50, - [0][1][2][1][5][10] = 38, - [0][1][2][1][6][10] = 34, - [0][1][2][1][9][10] = 34, - [0][1][2][1][8][10] = 30, - [0][1][2][1][11][10] = 28, - [0][1][2][1][2][12] = 68, - [0][1][2][1][1][12] = 34, - [0][1][2][1][3][12] = 50, - [0][1][2][1][5][12] = 38, - [0][1][2][1][6][12] = 34, - [0][1][2][1][9][12] = 34, - [0][1][2][1][8][12] = 30, - [0][1][2][1][11][12] = 28, - [0][1][2][1][2][14] = 68, - [0][1][2][1][1][14] = 34, - [0][1][2][1][3][14] = 50, - [0][1][2][1][5][14] = 38, - [0][1][2][1][6][14] = 34, - [0][1][2][1][9][14] = 34, - [0][1][2][1][8][14] = 30, - [0][1][2][1][11][14] = 28, - [0][1][2][1][2][15] = 68, - [0][1][2][1][1][15] = 34, - [0][1][2][1][3][15] = 70, - [0][1][2][1][5][15] = 62, - [0][1][2][1][6][15] = 34, - [0][1][2][1][9][15] = 34, - [0][1][2][1][8][15] = 30, - [0][1][2][1][11][15] = 28, - [0][1][2][1][2][17] = 68, - [0][1][2][1][1][17] = 34, - [0][1][2][1][3][17] = 70, - [0][1][2][1][5][17] = 62, - [0][1][2][1][6][17] = 34, - [0][1][2][1][9][17] = 34, - [0][1][2][1][8][17] = 30, - [0][1][2][1][11][17] = 28, - [0][1][2][1][2][19] = 68, - [0][1][2][1][1][19] = 34, - [0][1][2][1][3][19] = 70, - [0][1][2][1][5][19] = 62, - [0][1][2][1][6][19] = 34, - [0][1][2][1][9][19] = 34, - [0][1][2][1][8][19] = 30, - [0][1][2][1][11][19] = 28, - [0][1][2][1][2][21] = 68, - [0][1][2][1][1][21] = 34, - [0][1][2][1][3][21] = 70, - [0][1][2][1][5][21] = 62, - [0][1][2][1][6][21] = 34, - [0][1][2][1][9][21] = 34, - [0][1][2][1][8][21] = 30, - [0][1][2][1][11][21] = 28, - [0][1][2][1][2][23] = 68, - [0][1][2][1][1][23] = 34, - [0][1][2][1][3][23] = 70, - [0][1][2][1][5][23] = 62, - [0][1][2][1][6][23] = 34, - [0][1][2][1][9][23] = 34, - [0][1][2][1][8][23] = 30, - [0][1][2][1][11][23] = 28, - [0][1][2][1][2][25] = 68, - [0][1][2][1][1][25] = 34, - [0][1][2][1][3][25] = 70, - [0][1][2][1][5][25] = 127, - [0][1][2][1][6][25] = 34, - [0][1][2][1][9][25] = 127, - [0][1][2][1][8][25] = 30, - [0][1][2][1][11][25] = 28, - [0][1][2][1][2][27] = 68, - [0][1][2][1][1][27] = 34, - [0][1][2][1][3][27] = 70, - [0][1][2][1][5][27] = 127, - [0][1][2][1][6][27] = 34, - [0][1][2][1][9][27] = 127, - [0][1][2][1][8][27] = 30, - [0][1][2][1][11][27] = 28, - [0][1][2][1][2][29] = 68, - [0][1][2][1][1][29] = 34, - [0][1][2][1][3][29] = 70, - [0][1][2][1][5][29] = 127, - [0][1][2][1][6][29] = 34, - [0][1][2][1][9][29] = 127, - [0][1][2][1][8][29] = 30, - [0][1][2][1][11][29] = 28, - [0][1][2][1][2][31] = 68, - [0][1][2][1][1][31] = 34, - [0][1][2][1][3][31] = 70, - [0][1][2][1][5][31] = 62, - [0][1][2][1][6][31] = 34, - [0][1][2][1][9][31] = 34, - [0][1][2][1][8][31] = 30, - [0][1][2][1][11][31] = 28, - [0][1][2][1][2][33] = 68, - [0][1][2][1][1][33] = 34, - [0][1][2][1][3][33] = 70, - [0][1][2][1][5][33] = 62, - [0][1][2][1][6][33] = 34, - [0][1][2][1][9][33] = 34, - [0][1][2][1][8][33] = 30, - [0][1][2][1][11][33] = 28, - [0][1][2][1][2][35] = 64, - [0][1][2][1][1][35] = 34, - [0][1][2][1][3][35] = 70, - [0][1][2][1][5][35] = 62, - [0][1][2][1][6][35] = 34, - [0][1][2][1][9][35] = 34, - [0][1][2][1][8][35] = 30, - [0][1][2][1][11][35] = 28, - [0][1][2][1][2][37] = 68, - [0][1][2][1][1][37] = 127, - [0][1][2][1][3][37] = 70, - [0][1][2][1][5][37] = 62, - [0][1][2][1][6][37] = 34, - [0][1][2][1][9][37] = 68, - [0][1][2][1][8][37] = 30, - [0][1][2][1][11][37] = 127, - [0][1][2][1][2][38] = 76, - [0][1][2][1][1][38] = 4, - [0][1][2][1][3][38] = 127, - [0][1][2][1][5][38] = 76, - [0][1][2][1][6][38] = 4, - [0][1][2][1][9][38] = 76, - [0][1][2][1][8][38] = 30, - [0][1][2][1][11][38] = 28, - [0][1][2][1][2][40] = 76, - [0][1][2][1][1][40] = 4, - [0][1][2][1][3][40] = 127, - [0][1][2][1][5][40] = 76, - [0][1][2][1][6][40] = 4, - [0][1][2][1][9][40] = 76, - [0][1][2][1][8][40] = 30, - [0][1][2][1][11][40] = 28, - [0][1][2][1][2][42] = 76, - [0][1][2][1][1][42] = 4, - [0][1][2][1][3][42] = 127, - [0][1][2][1][5][42] = 76, - [0][1][2][1][6][42] = 4, - [0][1][2][1][9][42] = 76, - [0][1][2][1][8][42] = 30, - [0][1][2][1][11][42] = 28, - [0][1][2][1][2][44] = 76, - [0][1][2][1][1][44] = 4, - [0][1][2][1][3][44] = 127, - [0][1][2][1][5][44] = 76, - [0][1][2][1][6][44] = 4, - [0][1][2][1][9][44] = 76, - [0][1][2][1][8][44] = 30, - [0][1][2][1][11][44] = 28, - [0][1][2][1][2][46] = 76, - [0][1][2][1][1][46] = 4, - [0][1][2][1][3][46] = 127, - [0][1][2][1][5][46] = 76, - [0][1][2][1][6][46] = 4, - [0][1][2][1][9][46] = 76, - [0][1][2][1][8][46] = 30, - [0][1][2][1][11][46] = 28, - [1][0][2][0][2][1] = 68, - [1][0][2][0][1][1] = 64, - [1][0][2][0][3][1] = 62, - [1][0][2][0][5][1] = 64, - [1][0][2][0][6][1] = 64, - [1][0][2][0][9][1] = 64, - [1][0][2][0][8][1] = 30, - [1][0][2][0][11][1] = 52, - [1][0][2][0][2][5] = 72, - [1][0][2][0][1][5] = 64, - [1][0][2][0][3][5] = 62, - [1][0][2][0][5][5] = 64, - [1][0][2][0][6][5] = 60, - [1][0][2][0][9][5] = 64, - [1][0][2][0][8][5] = 30, - [1][0][2][0][11][5] = 52, - [1][0][2][0][2][9] = 72, - [1][0][2][0][1][9] = 64, - [1][0][2][0][3][9] = 62, - [1][0][2][0][5][9] = 64, - [1][0][2][0][6][9] = 64, - [1][0][2][0][9][9] = 64, - [1][0][2][0][8][9] = 54, - [1][0][2][0][11][9] = 52, - [1][0][2][0][2][13] = 66, - [1][0][2][0][1][13] = 64, - [1][0][2][0][3][13] = 62, - [1][0][2][0][5][13] = 64, - [1][0][2][0][6][13] = 64, - [1][0][2][0][9][13] = 64, - [1][0][2][0][8][13] = 54, - [1][0][2][0][11][13] = 52, - [1][0][2][0][2][16] = 62, - [1][0][2][0][1][16] = 64, - [1][0][2][0][3][16] = 72, - [1][0][2][0][5][16] = 62, - [1][0][2][0][6][16] = 64, - [1][0][2][0][9][16] = 64, - [1][0][2][0][8][16] = 54, - [1][0][2][0][11][16] = 52, - [1][0][2][0][2][20] = 72, - [1][0][2][0][1][20] = 64, - [1][0][2][0][3][20] = 72, - [1][0][2][0][5][20] = 72, - [1][0][2][0][6][20] = 64, - [1][0][2][0][9][20] = 64, - [1][0][2][0][8][20] = 54, - [1][0][2][0][11][20] = 52, - [1][0][2][0][2][24] = 72, - [1][0][2][0][1][24] = 64, - [1][0][2][0][3][24] = 72, - [1][0][2][0][5][24] = 127, - [1][0][2][0][6][24] = 64, - [1][0][2][0][9][24] = 127, - [1][0][2][0][8][24] = 54, - [1][0][2][0][11][24] = 52, - [1][0][2][0][2][28] = 72, - [1][0][2][0][1][28] = 64, - [1][0][2][0][3][28] = 72, - [1][0][2][0][5][28] = 127, - [1][0][2][0][6][28] = 64, - [1][0][2][0][9][28] = 127, - [1][0][2][0][8][28] = 54, - [1][0][2][0][11][28] = 52, - [1][0][2][0][2][32] = 72, - [1][0][2][0][1][32] = 64, - [1][0][2][0][3][32] = 72, - [1][0][2][0][5][32] = 72, - [1][0][2][0][6][32] = 64, - [1][0][2][0][9][32] = 64, - [1][0][2][0][8][32] = 54, - [1][0][2][0][11][32] = 52, - [1][0][2][0][2][36] = 72, - [1][0][2][0][1][36] = 127, - [1][0][2][0][3][36] = 72, - [1][0][2][0][5][36] = 72, - [1][0][2][0][6][36] = 64, - [1][0][2][0][9][36] = 72, - [1][0][2][0][8][36] = 54, - [1][0][2][0][11][36] = 127, - [1][0][2][0][2][39] = 72, - [1][0][2][0][1][39] = 28, - [1][0][2][0][3][39] = 127, - [1][0][2][0][5][39] = 72, - [1][0][2][0][6][39] = 28, - [1][0][2][0][9][39] = 72, - [1][0][2][0][8][39] = 54, - [1][0][2][0][11][39] = 52, - [1][0][2][0][2][43] = 72, - [1][0][2][0][1][43] = 28, - [1][0][2][0][3][43] = 127, - [1][0][2][0][5][43] = 72, - [1][0][2][0][6][43] = 28, - [1][0][2][0][9][43] = 72, - [1][0][2][0][8][43] = 54, - [1][0][2][0][11][43] = 52, - [1][1][2][0][2][1] = 58, - [1][1][2][0][1][1] = 52, - [1][1][2][0][3][1] = 50, - [1][1][2][0][5][1] = 52, - [1][1][2][0][6][1] = 52, - [1][1][2][0][9][1] = 52, - [1][1][2][0][8][1] = 18, - [1][1][2][0][11][1] = 40, - [1][1][2][0][2][5] = 72, - [1][1][2][0][1][5] = 52, - [1][1][2][0][3][5] = 50, - [1][1][2][0][5][5] = 52, - [1][1][2][0][6][5] = 46, - [1][1][2][0][9][5] = 52, - [1][1][2][0][8][5] = 18, - [1][1][2][0][11][5] = 40, - [1][1][2][0][2][9] = 72, - [1][1][2][0][1][9] = 52, - [1][1][2][0][3][9] = 50, - [1][1][2][0][5][9] = 52, - [1][1][2][0][6][9] = 52, - [1][1][2][0][9][9] = 52, - [1][1][2][0][8][9] = 42, - [1][1][2][0][11][9] = 40, - [1][1][2][0][2][13] = 58, - [1][1][2][0][1][13] = 52, - [1][1][2][0][3][13] = 50, - [1][1][2][0][5][13] = 52, - [1][1][2][0][6][13] = 52, - [1][1][2][0][9][13] = 52, - [1][1][2][0][8][13] = 42, - [1][1][2][0][11][13] = 40, - [1][1][2][0][2][16] = 56, - [1][1][2][0][1][16] = 52, - [1][1][2][0][3][16] = 72, - [1][1][2][0][5][16] = 56, - [1][1][2][0][6][16] = 52, - [1][1][2][0][9][16] = 52, - [1][1][2][0][8][16] = 42, - [1][1][2][0][11][16] = 40, - [1][1][2][0][2][20] = 72, - [1][1][2][0][1][20] = 52, - [1][1][2][0][3][20] = 72, - [1][1][2][0][5][20] = 72, - [1][1][2][0][6][20] = 52, - [1][1][2][0][9][20] = 52, - [1][1][2][0][8][20] = 42, - [1][1][2][0][11][20] = 40, - [1][1][2][0][2][24] = 72, - [1][1][2][0][1][24] = 52, - [1][1][2][0][3][24] = 72, - [1][1][2][0][5][24] = 127, - [1][1][2][0][6][24] = 52, - [1][1][2][0][9][24] = 127, - [1][1][2][0][8][24] = 42, - [1][1][2][0][11][24] = 40, - [1][1][2][0][2][28] = 72, - [1][1][2][0][1][28] = 52, - [1][1][2][0][3][28] = 72, - [1][1][2][0][5][28] = 127, - [1][1][2][0][6][28] = 52, - [1][1][2][0][9][28] = 127, - [1][1][2][0][8][28] = 42, - [1][1][2][0][11][28] = 40, - [1][1][2][0][2][32] = 68, - [1][1][2][0][1][32] = 52, - [1][1][2][0][3][32] = 72, - [1][1][2][0][5][32] = 68, - [1][1][2][0][6][32] = 52, - [1][1][2][0][9][32] = 52, - [1][1][2][0][8][32] = 42, - [1][1][2][0][11][32] = 40, - [1][1][2][0][2][36] = 72, - [1][1][2][0][1][36] = 127, - [1][1][2][0][3][36] = 72, - [1][1][2][0][5][36] = 72, - [1][1][2][0][6][36] = 52, - [1][1][2][0][9][36] = 72, - [1][1][2][0][8][36] = 42, - [1][1][2][0][11][36] = 127, - [1][1][2][0][2][39] = 72, - [1][1][2][0][1][39] = 16, - [1][1][2][0][3][39] = 127, - [1][1][2][0][5][39] = 72, - [1][1][2][0][6][39] = 16, - [1][1][2][0][9][39] = 72, - [1][1][2][0][8][39] = 42, - [1][1][2][0][11][39] = 40, - [1][1][2][0][2][43] = 72, - [1][1][2][0][1][43] = 16, - [1][1][2][0][3][43] = 127, - [1][1][2][0][5][43] = 72, - [1][1][2][0][6][43] = 16, - [1][1][2][0][9][43] = 72, - [1][1][2][0][8][43] = 42, - [1][1][2][0][11][43] = 40, - [1][1][2][1][2][1] = 58, - [1][1][2][1][1][1] = 40, - [1][1][2][1][3][1] = 50, - [1][1][2][1][5][1] = 40, - [1][1][2][1][6][1] = 40, - [1][1][2][1][9][1] = 40, - [1][1][2][1][8][1] = 6, - [1][1][2][1][11][1] = 28, - [1][1][2][1][2][5] = 68, - [1][1][2][1][1][5] = 40, - [1][1][2][1][3][5] = 50, - [1][1][2][1][5][5] = 40, - [1][1][2][1][6][5] = 40, - [1][1][2][1][9][5] = 40, - [1][1][2][1][8][5] = 6, - [1][1][2][1][11][5] = 28, - [1][1][2][1][2][9] = 68, - [1][1][2][1][1][9] = 40, - [1][1][2][1][3][9] = 50, - [1][1][2][1][5][9] = 40, - [1][1][2][1][6][9] = 40, - [1][1][2][1][9][9] = 40, - [1][1][2][1][8][9] = 30, - [1][1][2][1][11][9] = 28, - [1][1][2][1][2][13] = 58, - [1][1][2][1][1][13] = 40, - [1][1][2][1][3][13] = 50, - [1][1][2][1][5][13] = 40, - [1][1][2][1][6][13] = 40, - [1][1][2][1][9][13] = 40, - [1][1][2][1][8][13] = 30, - [1][1][2][1][11][13] = 28, - [1][1][2][1][2][16] = 56, - [1][1][2][1][1][16] = 40, - [1][1][2][1][3][16] = 72, - [1][1][2][1][5][16] = 56, - [1][1][2][1][6][16] = 40, - [1][1][2][1][9][16] = 40, - [1][1][2][1][8][16] = 30, - [1][1][2][1][11][16] = 28, - [1][1][2][1][2][20] = 68, - [1][1][2][1][1][20] = 40, - [1][1][2][1][3][20] = 72, - [1][1][2][1][5][20] = 68, - [1][1][2][1][6][20] = 40, - [1][1][2][1][9][20] = 40, - [1][1][2][1][8][20] = 30, - [1][1][2][1][11][20] = 28, - [1][1][2][1][2][24] = 68, - [1][1][2][1][1][24] = 40, - [1][1][2][1][3][24] = 72, - [1][1][2][1][5][24] = 127, - [1][1][2][1][6][24] = 40, - [1][1][2][1][9][24] = 127, - [1][1][2][1][8][24] = 30, - [1][1][2][1][11][24] = 28, - [1][1][2][1][2][28] = 68, - [1][1][2][1][1][28] = 40, - [1][1][2][1][3][28] = 72, - [1][1][2][1][5][28] = 127, - [1][1][2][1][6][28] = 40, - [1][1][2][1][9][28] = 127, - [1][1][2][1][8][28] = 30, - [1][1][2][1][11][28] = 28, - [1][1][2][1][2][32] = 68, - [1][1][2][1][1][32] = 40, - [1][1][2][1][3][32] = 72, - [1][1][2][1][5][32] = 68, - [1][1][2][1][6][32] = 40, - [1][1][2][1][9][32] = 40, - [1][1][2][1][8][32] = 30, - [1][1][2][1][11][32] = 28, - [1][1][2][1][2][36] = 68, - [1][1][2][1][1][36] = 127, - [1][1][2][1][3][36] = 72, - [1][1][2][1][5][36] = 68, - [1][1][2][1][6][36] = 40, - [1][1][2][1][9][36] = 68, - [1][1][2][1][8][36] = 30, - [1][1][2][1][11][36] = 127, - [1][1][2][1][2][39] = 72, - [1][1][2][1][1][39] = 4, - [1][1][2][1][3][39] = 127, - [1][1][2][1][5][39] = 72, - [1][1][2][1][6][39] = 4, - [1][1][2][1][9][39] = 72, - [1][1][2][1][8][39] = 30, - [1][1][2][1][11][39] = 28, - [1][1][2][1][2][43] = 72, - [1][1][2][1][1][43] = 4, - [1][1][2][1][3][43] = 127, - [1][1][2][1][5][43] = 72, - [1][1][2][1][6][43] = 4, - [1][1][2][1][9][43] = 72, - [1][1][2][1][8][43] = 30, - [1][1][2][1][11][43] = 28, - [2][0][2][0][2][3] = 64, - [2][0][2][0][1][3] = 64, - [2][0][2][0][3][3] = 64, - [2][0][2][0][5][3] = 62, - [2][0][2][0][6][3] = 64, - [2][0][2][0][9][3] = 64, - [2][0][2][0][8][3] = 30, - [2][0][2][0][11][3] = 52, - [2][0][2][0][2][11] = 64, - [2][0][2][0][1][11] = 64, - [2][0][2][0][3][11] = 64, - [2][0][2][0][5][11] = 62, - [2][0][2][0][6][11] = 64, - [2][0][2][0][9][11] = 64, - [2][0][2][0][8][11] = 54, - [2][0][2][0][11][11] = 52, - [2][0][2][0][2][18] = 62, - [2][0][2][0][1][18] = 64, - [2][0][2][0][3][18] = 72, - [2][0][2][0][5][18] = 66, - [2][0][2][0][6][18] = 64, - [2][0][2][0][9][18] = 64, - [2][0][2][0][8][18] = 54, - [2][0][2][0][11][18] = 52, - [2][0][2][0][2][26] = 72, - [2][0][2][0][1][26] = 64, - [2][0][2][0][3][26] = 72, - [2][0][2][0][5][26] = 127, - [2][0][2][0][6][26] = 64, - [2][0][2][0][9][26] = 127, - [2][0][2][0][8][26] = 54, - [2][0][2][0][11][26] = 52, - [2][0][2][0][2][34] = 72, - [2][0][2][0][1][34] = 127, - [2][0][2][0][3][34] = 72, - [2][0][2][0][5][34] = 72, - [2][0][2][0][6][34] = 64, - [2][0][2][0][9][34] = 72, - [2][0][2][0][8][34] = 54, - [2][0][2][0][11][34] = 127, - [2][0][2][0][2][41] = 72, - [2][0][2][0][1][41] = 28, - [2][0][2][0][3][41] = 127, - [2][0][2][0][5][41] = 72, - [2][0][2][0][6][41] = 28, - [2][0][2][0][9][41] = 72, - [2][0][2][0][8][41] = 54, - [2][0][2][0][11][41] = 52, - [2][1][2][0][2][3] = 56, - [2][1][2][0][1][3] = 52, - [2][1][2][0][3][3] = 52, - [2][1][2][0][5][3] = 52, - [2][1][2][0][6][3] = 52, - [2][1][2][0][9][3] = 52, - [2][1][2][0][8][3] = 18, - [2][1][2][0][11][3] = 40, - [2][1][2][0][2][11] = 56, - [2][1][2][0][1][11] = 52, - [2][1][2][0][3][11] = 52, - [2][1][2][0][5][11] = 52, - [2][1][2][0][6][11] = 52, - [2][1][2][0][9][11] = 52, - [2][1][2][0][8][11] = 42, - [2][1][2][0][11][11] = 40, - [2][1][2][0][2][18] = 56, - [2][1][2][0][1][18] = 52, - [2][1][2][0][3][18] = 72, - [2][1][2][0][5][18] = 56, - [2][1][2][0][6][18] = 52, - [2][1][2][0][9][18] = 52, - [2][1][2][0][8][18] = 42, - [2][1][2][0][11][18] = 40, - [2][1][2][0][2][26] = 72, - [2][1][2][0][1][26] = 52, - [2][1][2][0][3][26] = 72, - [2][1][2][0][5][26] = 127, - [2][1][2][0][6][26] = 52, - [2][1][2][0][9][26] = 127, - [2][1][2][0][8][26] = 42, - [2][1][2][0][11][26] = 40, - [2][1][2][0][2][34] = 72, - [2][1][2][0][1][34] = 127, - [2][1][2][0][3][34] = 72, - [2][1][2][0][5][34] = 72, - [2][1][2][0][6][34] = 52, - [2][1][2][0][9][34] = 72, - [2][1][2][0][8][34] = 42, - [2][1][2][0][11][34] = 127, - [2][1][2][0][2][41] = 72, - [2][1][2][0][1][41] = 16, - [2][1][2][0][3][41] = 127, - [2][1][2][0][5][41] = 72, - [2][1][2][0][6][41] = 16, - [2][1][2][0][9][41] = 72, - [2][1][2][0][8][41] = 42, - [2][1][2][0][11][41] = 40, - [2][1][2][1][2][3] = 56, - [2][1][2][1][1][3] = 40, - [2][1][2][1][3][3] = 52, - [2][1][2][1][5][3] = 40, - [2][1][2][1][6][3] = 40, - [2][1][2][1][9][3] = 40, - [2][1][2][1][8][3] = 6, - [2][1][2][1][11][3] = 28, - [2][1][2][1][2][11] = 56, - [2][1][2][1][1][11] = 40, - [2][1][2][1][3][11] = 52, - [2][1][2][1][5][11] = 40, - [2][1][2][1][6][11] = 40, - [2][1][2][1][9][11] = 40, - [2][1][2][1][8][11] = 30, - [2][1][2][1][11][11] = 28, - [2][1][2][1][2][18] = 56, - [2][1][2][1][1][18] = 40, - [2][1][2][1][3][18] = 72, - [2][1][2][1][5][18] = 56, - [2][1][2][1][6][18] = 40, - [2][1][2][1][9][18] = 40, - [2][1][2][1][8][18] = 30, - [2][1][2][1][11][18] = 28, - [2][1][2][1][2][26] = 68, - [2][1][2][1][1][26] = 40, - [2][1][2][1][3][26] = 72, - [2][1][2][1][5][26] = 127, - [2][1][2][1][6][26] = 40, - [2][1][2][1][9][26] = 127, - [2][1][2][1][8][26] = 30, - [2][1][2][1][11][26] = 28, - [2][1][2][1][2][34] = 68, - [2][1][2][1][1][34] = 127, - [2][1][2][1][3][34] = 72, - [2][1][2][1][5][34] = 68, - [2][1][2][1][6][34] = 40, - [2][1][2][1][9][34] = 68, - [2][1][2][1][8][34] = 30, - [2][1][2][1][11][34] = 127, - [2][1][2][1][2][41] = 72, - [2][1][2][1][1][41] = 4, - [2][1][2][1][3][41] = 127, - [2][1][2][1][5][41] = 72, - [2][1][2][1][6][41] = 4, - [2][1][2][1][9][41] = 72, - [2][1][2][1][8][41] = 30, - [2][1][2][1][11][41] = 28, + [0][0][1][0][RTW89_WW][0] = 30, + [0][0][1][0][RTW89_WW][2] = 30, + [0][0][1][0][RTW89_WW][4] = 30, + [0][0][1][0][RTW89_WW][6] = 30, + [0][0][1][0][RTW89_WW][8] = 52, + [0][0][1][0][RTW89_WW][10] = 52, + [0][0][1][0][RTW89_WW][12] = 52, + [0][0][1][0][RTW89_WW][14] = 52, + [0][0][1][0][RTW89_WW][15] = 52, + [0][0][1][0][RTW89_WW][17] = 52, + [0][0][1][0][RTW89_WW][19] = 52, + [0][0][1][0][RTW89_WW][21] = 52, + [0][0][1][0][RTW89_WW][23] = 52, + [0][0][1][0][RTW89_WW][25] = 52, + [0][0][1][0][RTW89_WW][27] = 52, + [0][0][1][0][RTW89_WW][29] = 52, + [0][0][1][0][RTW89_WW][31] = 52, + [0][0][1][0][RTW89_WW][33] = 52, + [0][0][1][0][RTW89_WW][35] = 52, + [0][0][1][0][RTW89_WW][37] = 54, + [0][0][1][0][RTW89_WW][38] = 28, + [0][0][1][0][RTW89_WW][40] = 28, + [0][0][1][0][RTW89_WW][42] = 28, + [0][0][1][0][RTW89_WW][44] = 28, + [0][0][1][0][RTW89_WW][46] = 28, + [0][1][1][0][RTW89_WW][0] = 18, + [0][1][1][0][RTW89_WW][2] = 18, + [0][1][1][0][RTW89_WW][4] = 18, + [0][1][1][0][RTW89_WW][6] = 18, + [0][1][1][0][RTW89_WW][8] = 40, + [0][1][1][0][RTW89_WW][10] = 40, + [0][1][1][0][RTW89_WW][12] = 40, + [0][1][1][0][RTW89_WW][14] = 40, + [0][1][1][0][RTW89_WW][15] = 40, + [0][1][1][0][RTW89_WW][17] = 40, + [0][1][1][0][RTW89_WW][19] = 40, + [0][1][1][0][RTW89_WW][21] = 40, + [0][1][1][0][RTW89_WW][23] = 40, + [0][1][1][0][RTW89_WW][25] = 40, + [0][1][1][0][RTW89_WW][27] = 40, + [0][1][1][0][RTW89_WW][29] = 40, + [0][1][1][0][RTW89_WW][31] = 40, + [0][1][1][0][RTW89_WW][33] = 40, + [0][1][1][0][RTW89_WW][35] = 40, + [0][1][1][0][RTW89_WW][37] = 42, + [0][1][1][0][RTW89_WW][38] = 16, + [0][1][1][0][RTW89_WW][40] = 16, + [0][1][1][0][RTW89_WW][42] = 16, + [0][1][1][0][RTW89_WW][44] = 16, + [0][1][1][0][RTW89_WW][46] = 16, + [0][0][2][0][RTW89_WW][0] = 30, + [0][0][2][0][RTW89_WW][2] = 30, + [0][0][2][0][RTW89_WW][4] = 30, + [0][0][2][0][RTW89_WW][6] = 30, + [0][0][2][0][RTW89_WW][8] = 52, + [0][0][2][0][RTW89_WW][10] = 52, + [0][0][2][0][RTW89_WW][12] = 52, + [0][0][2][0][RTW89_WW][14] = 52, + [0][0][2][0][RTW89_WW][15] = 52, + [0][0][2][0][RTW89_WW][17] = 52, + [0][0][2][0][RTW89_WW][19] = 52, + [0][0][2][0][RTW89_WW][21] = 52, + [0][0][2][0][RTW89_WW][23] = 52, + [0][0][2][0][RTW89_WW][25] = 52, + [0][0][2][0][RTW89_WW][27] = 52, + [0][0][2][0][RTW89_WW][29] = 52, + [0][0][2][0][RTW89_WW][31] = 52, + [0][0][2][0][RTW89_WW][33] = 52, + [0][0][2][0][RTW89_WW][35] = 52, + [0][0][2][0][RTW89_WW][37] = 54, + [0][0][2][0][RTW89_WW][38] = 28, + [0][0][2][0][RTW89_WW][40] = 28, + [0][0][2][0][RTW89_WW][42] = 28, + [0][0][2][0][RTW89_WW][44] = 28, + [0][0][2][0][RTW89_WW][46] = 28, + [0][1][2][0][RTW89_WW][0] = 18, + [0][1][2][0][RTW89_WW][2] = 18, + [0][1][2][0][RTW89_WW][4] = 18, + [0][1][2][0][RTW89_WW][6] = 18, + [0][1][2][0][RTW89_WW][8] = 40, + [0][1][2][0][RTW89_WW][10] = 40, + [0][1][2][0][RTW89_WW][12] = 40, + [0][1][2][0][RTW89_WW][14] = 40, + [0][1][2][0][RTW89_WW][15] = 40, + [0][1][2][0][RTW89_WW][17] = 40, + [0][1][2][0][RTW89_WW][19] = 40, + [0][1][2][0][RTW89_WW][21] = 40, + [0][1][2][0][RTW89_WW][23] = 40, + [0][1][2][0][RTW89_WW][25] = 40, + [0][1][2][0][RTW89_WW][27] = 40, + [0][1][2][0][RTW89_WW][29] = 40, + [0][1][2][0][RTW89_WW][31] = 40, + [0][1][2][0][RTW89_WW][33] = 40, + [0][1][2][0][RTW89_WW][35] = 40, + [0][1][2][0][RTW89_WW][37] = 42, + [0][1][2][0][RTW89_WW][38] = 16, + [0][1][2][0][RTW89_WW][40] = 16, + [0][1][2][0][RTW89_WW][42] = 16, + [0][1][2][0][RTW89_WW][44] = 16, + [0][1][2][0][RTW89_WW][46] = 16, + [0][1][2][1][RTW89_WW][0] = 6, + [0][1][2][1][RTW89_WW][2] = 6, + [0][1][2][1][RTW89_WW][4] = 6, + [0][1][2][1][RTW89_WW][6] = 6, + [0][1][2][1][RTW89_WW][8] = 28, + [0][1][2][1][RTW89_WW][10] = 28, + [0][1][2][1][RTW89_WW][12] = 28, + [0][1][2][1][RTW89_WW][14] = 28, + [0][1][2][1][RTW89_WW][15] = 28, + [0][1][2][1][RTW89_WW][17] = 28, + [0][1][2][1][RTW89_WW][19] = 28, + [0][1][2][1][RTW89_WW][21] = 28, + [0][1][2][1][RTW89_WW][23] = 28, + [0][1][2][1][RTW89_WW][25] = 28, + [0][1][2][1][RTW89_WW][27] = 28, + [0][1][2][1][RTW89_WW][29] = 28, + [0][1][2][1][RTW89_WW][31] = 28, + [0][1][2][1][RTW89_WW][33] = 28, + [0][1][2][1][RTW89_WW][35] = 28, + [0][1][2][1][RTW89_WW][37] = 30, + [0][1][2][1][RTW89_WW][38] = 4, + [0][1][2][1][RTW89_WW][40] = 4, + [0][1][2][1][RTW89_WW][42] = 4, + [0][1][2][1][RTW89_WW][44] = 4, + [0][1][2][1][RTW89_WW][46] = 4, + [1][0][2][0][RTW89_WW][1] = 30, + [1][0][2][0][RTW89_WW][5] = 30, + [1][0][2][0][RTW89_WW][9] = 52, + [1][0][2][0][RTW89_WW][13] = 52, + [1][0][2][0][RTW89_WW][16] = 52, + [1][0][2][0][RTW89_WW][20] = 52, + [1][0][2][0][RTW89_WW][24] = 52, + [1][0][2][0][RTW89_WW][28] = 52, + [1][0][2][0][RTW89_WW][32] = 52, + [1][0][2][0][RTW89_WW][36] = 54, + [1][0][2][0][RTW89_WW][39] = 28, + [1][0][2][0][RTW89_WW][43] = 28, + [1][1][2][0][RTW89_WW][1] = 18, + [1][1][2][0][RTW89_WW][5] = 18, + [1][1][2][0][RTW89_WW][9] = 40, + [1][1][2][0][RTW89_WW][13] = 40, + [1][1][2][0][RTW89_WW][16] = 40, + [1][1][2][0][RTW89_WW][20] = 40, + [1][1][2][0][RTW89_WW][24] = 40, + [1][1][2][0][RTW89_WW][28] = 40, + [1][1][2][0][RTW89_WW][32] = 40, + [1][1][2][0][RTW89_WW][36] = 42, + [1][1][2][0][RTW89_WW][39] = 16, + [1][1][2][0][RTW89_WW][43] = 16, + [1][1][2][1][RTW89_WW][1] = 6, + [1][1][2][1][RTW89_WW][5] = 6, + [1][1][2][1][RTW89_WW][9] = 28, + [1][1][2][1][RTW89_WW][13] = 28, + [1][1][2][1][RTW89_WW][16] = 28, + [1][1][2][1][RTW89_WW][20] = 28, + [1][1][2][1][RTW89_WW][24] = 28, + [1][1][2][1][RTW89_WW][28] = 28, + [1][1][2][1][RTW89_WW][32] = 28, + [1][1][2][1][RTW89_WW][36] = 30, + [1][1][2][1][RTW89_WW][39] = 4, + [1][1][2][1][RTW89_WW][43] = 4, + [2][0][2][0][RTW89_WW][3] = 30, + [2][0][2][0][RTW89_WW][11] = 52, + [2][0][2][0][RTW89_WW][18] = 52, + [2][0][2][0][RTW89_WW][26] = 52, + [2][0][2][0][RTW89_WW][34] = 54, + [2][0][2][0][RTW89_WW][41] = 28, + [2][1][2][0][RTW89_WW][3] = 18, + [2][1][2][0][RTW89_WW][11] = 40, + [2][1][2][0][RTW89_WW][18] = 40, + [2][1][2][0][RTW89_WW][26] = 40, + [2][1][2][0][RTW89_WW][34] = 42, + [2][1][2][0][RTW89_WW][41] = 16, + [2][1][2][1][RTW89_WW][3] = 6, + [2][1][2][1][RTW89_WW][11] = 28, + [2][1][2][1][RTW89_WW][18] = 28, + [2][1][2][1][RTW89_WW][26] = 28, + [2][1][2][1][RTW89_WW][34] = 30, + [2][1][2][1][RTW89_WW][41] = 4, + [0][0][1][0][RTW89_FCC][0] = 76, + [0][0][1][0][RTW89_ETSI][0] = 58, + [0][0][1][0][RTW89_MKK][0] = 62, + [0][0][1][0][RTW89_IC][0] = 62, + [0][0][1][0][RTW89_KCC][0] = 76, + [0][0][1][0][RTW89_ACMA][0] = 58, + [0][0][1][0][RTW89_CHILE][0] = 30, + [0][0][1][0][RTW89_UKRAINE][0] = 52, + [0][0][1][0][RTW89_MEXICO][0] = 62, + [0][0][1][0][RTW89_CN][0] = 58, + [0][0][1][0][RTW89_QATAR][0] = 58, + [0][0][1][0][RTW89_FCC][2] = 76, + [0][0][1][0][RTW89_ETSI][2] = 58, + [0][0][1][0][RTW89_MKK][2] = 62, + [0][0][1][0][RTW89_IC][2] = 62, + [0][0][1][0][RTW89_KCC][2] = 76, + [0][0][1][0][RTW89_ACMA][2] = 58, + [0][0][1][0][RTW89_CHILE][2] = 30, + [0][0][1][0][RTW89_UKRAINE][2] = 52, + [0][0][1][0][RTW89_MEXICO][2] = 62, + [0][0][1][0][RTW89_CN][2] = 58, + [0][0][1][0][RTW89_QATAR][2] = 58, + [0][0][1][0][RTW89_FCC][4] = 76, + [0][0][1][0][RTW89_ETSI][4] = 58, + [0][0][1][0][RTW89_MKK][4] = 62, + [0][0][1][0][RTW89_IC][4] = 62, + [0][0][1][0][RTW89_KCC][4] = 76, + [0][0][1][0][RTW89_ACMA][4] = 58, + [0][0][1][0][RTW89_CHILE][4] = 30, + [0][0][1][0][RTW89_UKRAINE][4] = 52, + [0][0][1][0][RTW89_MEXICO][4] = 62, + [0][0][1][0][RTW89_CN][4] = 58, + [0][0][1][0][RTW89_QATAR][4] = 58, + [0][0][1][0][RTW89_FCC][6] = 76, + [0][0][1][0][RTW89_ETSI][6] = 58, + [0][0][1][0][RTW89_MKK][6] = 62, + [0][0][1][0][RTW89_IC][6] = 62, + [0][0][1][0][RTW89_KCC][6] = 58, + [0][0][1][0][RTW89_ACMA][6] = 58, + [0][0][1][0][RTW89_CHILE][6] = 30, + [0][0][1][0][RTW89_UKRAINE][6] = 52, + [0][0][1][0][RTW89_MEXICO][6] = 62, + [0][0][1][0][RTW89_CN][6] = 58, + [0][0][1][0][RTW89_QATAR][6] = 58, + [0][0][1][0][RTW89_FCC][8] = 76, + [0][0][1][0][RTW89_ETSI][8] = 58, + [0][0][1][0][RTW89_MKK][8] = 62, + [0][0][1][0][RTW89_IC][8] = 64, + [0][0][1][0][RTW89_KCC][8] = 76, + [0][0][1][0][RTW89_ACMA][8] = 58, + [0][0][1][0][RTW89_CHILE][8] = 54, + [0][0][1][0][RTW89_UKRAINE][8] = 52, + [0][0][1][0][RTW89_MEXICO][8] = 76, + [0][0][1][0][RTW89_CN][8] = 58, + [0][0][1][0][RTW89_QATAR][8] = 58, + [0][0][1][0][RTW89_FCC][10] = 76, + [0][0][1][0][RTW89_ETSI][10] = 58, + [0][0][1][0][RTW89_MKK][10] = 62, + [0][0][1][0][RTW89_IC][10] = 64, + [0][0][1][0][RTW89_KCC][10] = 76, + [0][0][1][0][RTW89_ACMA][10] = 58, + [0][0][1][0][RTW89_CHILE][10] = 54, + [0][0][1][0][RTW89_UKRAINE][10] = 52, + [0][0][1][0][RTW89_MEXICO][10] = 76, + [0][0][1][0][RTW89_CN][10] = 58, + [0][0][1][0][RTW89_QATAR][10] = 58, + [0][0][1][0][RTW89_FCC][12] = 76, + [0][0][1][0][RTW89_ETSI][12] = 58, + [0][0][1][0][RTW89_MKK][12] = 62, + [0][0][1][0][RTW89_IC][12] = 64, + [0][0][1][0][RTW89_KCC][12] = 76, + [0][0][1][0][RTW89_ACMA][12] = 58, + [0][0][1][0][RTW89_CHILE][12] = 54, + [0][0][1][0][RTW89_UKRAINE][12] = 52, + [0][0][1][0][RTW89_MEXICO][12] = 76, + [0][0][1][0][RTW89_CN][12] = 58, + [0][0][1][0][RTW89_QATAR][12] = 58, + [0][0][1][0][RTW89_FCC][14] = 76, + [0][0][1][0][RTW89_ETSI][14] = 58, + [0][0][1][0][RTW89_MKK][14] = 62, + [0][0][1][0][RTW89_IC][14] = 64, + [0][0][1][0][RTW89_KCC][14] = 76, + [0][0][1][0][RTW89_ACMA][14] = 58, + [0][0][1][0][RTW89_CHILE][14] = 54, + [0][0][1][0][RTW89_UKRAINE][14] = 52, + [0][0][1][0][RTW89_MEXICO][14] = 76, + [0][0][1][0][RTW89_CN][14] = 58, + [0][0][1][0][RTW89_QATAR][14] = 58, + [0][0][1][0][RTW89_FCC][15] = 76, + [0][0][1][0][RTW89_ETSI][15] = 58, + [0][0][1][0][RTW89_MKK][15] = 76, + [0][0][1][0][RTW89_IC][15] = 76, + [0][0][1][0][RTW89_KCC][15] = 76, + [0][0][1][0][RTW89_ACMA][15] = 58, + [0][0][1][0][RTW89_CHILE][15] = 54, + [0][0][1][0][RTW89_UKRAINE][15] = 52, + [0][0][1][0][RTW89_MEXICO][15] = 76, + [0][0][1][0][RTW89_CN][15] = 127, + [0][0][1][0][RTW89_QATAR][15] = 52, + [0][0][1][0][RTW89_FCC][17] = 76, + [0][0][1][0][RTW89_ETSI][17] = 58, + [0][0][1][0][RTW89_MKK][17] = 76, + [0][0][1][0][RTW89_IC][17] = 76, + [0][0][1][0][RTW89_KCC][17] = 76, + [0][0][1][0][RTW89_ACMA][17] = 58, + [0][0][1][0][RTW89_CHILE][17] = 54, + [0][0][1][0][RTW89_UKRAINE][17] = 52, + [0][0][1][0][RTW89_MEXICO][17] = 76, + [0][0][1][0][RTW89_CN][17] = 127, + [0][0][1][0][RTW89_QATAR][17] = 52, + [0][0][1][0][RTW89_FCC][19] = 76, + [0][0][1][0][RTW89_ETSI][19] = 58, + [0][0][1][0][RTW89_MKK][19] = 76, + [0][0][1][0][RTW89_IC][19] = 76, + [0][0][1][0][RTW89_KCC][19] = 76, + [0][0][1][0][RTW89_ACMA][19] = 58, + [0][0][1][0][RTW89_CHILE][19] = 54, + [0][0][1][0][RTW89_UKRAINE][19] = 52, + [0][0][1][0][RTW89_MEXICO][19] = 76, + [0][0][1][0][RTW89_CN][19] = 127, + [0][0][1][0][RTW89_QATAR][19] = 52, + [0][0][1][0][RTW89_FCC][21] = 76, + [0][0][1][0][RTW89_ETSI][21] = 58, + [0][0][1][0][RTW89_MKK][21] = 76, + [0][0][1][0][RTW89_IC][21] = 76, + [0][0][1][0][RTW89_KCC][21] = 76, + [0][0][1][0][RTW89_ACMA][21] = 58, + [0][0][1][0][RTW89_CHILE][21] = 54, + [0][0][1][0][RTW89_UKRAINE][21] = 52, + [0][0][1][0][RTW89_MEXICO][21] = 76, + [0][0][1][0][RTW89_CN][21] = 127, + [0][0][1][0][RTW89_QATAR][21] = 52, + [0][0][1][0][RTW89_FCC][23] = 76, + [0][0][1][0][RTW89_ETSI][23] = 58, + [0][0][1][0][RTW89_MKK][23] = 76, + [0][0][1][0][RTW89_IC][23] = 76, + [0][0][1][0][RTW89_KCC][23] = 76, + [0][0][1][0][RTW89_ACMA][23] = 58, + [0][0][1][0][RTW89_CHILE][23] = 54, + [0][0][1][0][RTW89_UKRAINE][23] = 52, + [0][0][1][0][RTW89_MEXICO][23] = 76, + [0][0][1][0][RTW89_CN][23] = 127, + [0][0][1][0][RTW89_QATAR][23] = 52, + [0][0][1][0][RTW89_FCC][25] = 76, + [0][0][1][0][RTW89_ETSI][25] = 58, + [0][0][1][0][RTW89_MKK][25] = 76, + [0][0][1][0][RTW89_IC][25] = 127, + [0][0][1][0][RTW89_KCC][25] = 76, + [0][0][1][0][RTW89_ACMA][25] = 127, + [0][0][1][0][RTW89_CHILE][25] = 54, + [0][0][1][0][RTW89_UKRAINE][25] = 52, + [0][0][1][0][RTW89_MEXICO][25] = 76, + [0][0][1][0][RTW89_CN][25] = 127, + [0][0][1][0][RTW89_QATAR][25] = 52, + [0][0][1][0][RTW89_FCC][27] = 76, + [0][0][1][0][RTW89_ETSI][27] = 58, + [0][0][1][0][RTW89_MKK][27] = 76, + [0][0][1][0][RTW89_IC][27] = 127, + [0][0][1][0][RTW89_KCC][27] = 76, + [0][0][1][0][RTW89_ACMA][27] = 127, + [0][0][1][0][RTW89_CHILE][27] = 54, + [0][0][1][0][RTW89_UKRAINE][27] = 52, + [0][0][1][0][RTW89_MEXICO][27] = 76, + [0][0][1][0][RTW89_CN][27] = 127, + [0][0][1][0][RTW89_QATAR][27] = 52, + [0][0][1][0][RTW89_FCC][29] = 76, + [0][0][1][0][RTW89_ETSI][29] = 58, + [0][0][1][0][RTW89_MKK][29] = 76, + [0][0][1][0][RTW89_IC][29] = 127, + [0][0][1][0][RTW89_KCC][29] = 76, + [0][0][1][0][RTW89_ACMA][29] = 127, + [0][0][1][0][RTW89_CHILE][29] = 54, + [0][0][1][0][RTW89_UKRAINE][29] = 52, + [0][0][1][0][RTW89_MEXICO][29] = 76, + [0][0][1][0][RTW89_CN][29] = 127, + [0][0][1][0][RTW89_QATAR][29] = 52, + [0][0][1][0][RTW89_FCC][31] = 76, + [0][0][1][0][RTW89_ETSI][31] = 58, + [0][0][1][0][RTW89_MKK][31] = 76, + [0][0][1][0][RTW89_IC][31] = 76, + [0][0][1][0][RTW89_KCC][31] = 76, + [0][0][1][0][RTW89_ACMA][31] = 58, + [0][0][1][0][RTW89_CHILE][31] = 54, + [0][0][1][0][RTW89_UKRAINE][31] = 52, + [0][0][1][0][RTW89_MEXICO][31] = 76, + [0][0][1][0][RTW89_CN][31] = 127, + [0][0][1][0][RTW89_QATAR][31] = 52, + [0][0][1][0][RTW89_FCC][33] = 76, + [0][0][1][0][RTW89_ETSI][33] = 58, + [0][0][1][0][RTW89_MKK][33] = 76, + [0][0][1][0][RTW89_IC][33] = 76, + [0][0][1][0][RTW89_KCC][33] = 76, + [0][0][1][0][RTW89_ACMA][33] = 58, + [0][0][1][0][RTW89_CHILE][33] = 54, + [0][0][1][0][RTW89_UKRAINE][33] = 52, + [0][0][1][0][RTW89_MEXICO][33] = 76, + [0][0][1][0][RTW89_CN][33] = 127, + [0][0][1][0][RTW89_QATAR][33] = 52, + [0][0][1][0][RTW89_FCC][35] = 74, + [0][0][1][0][RTW89_ETSI][35] = 58, + [0][0][1][0][RTW89_MKK][35] = 76, + [0][0][1][0][RTW89_IC][35] = 74, + [0][0][1][0][RTW89_KCC][35] = 76, + [0][0][1][0][RTW89_ACMA][35] = 58, + [0][0][1][0][RTW89_CHILE][35] = 54, + [0][0][1][0][RTW89_UKRAINE][35] = 52, + [0][0][1][0][RTW89_MEXICO][35] = 74, + [0][0][1][0][RTW89_CN][35] = 127, + [0][0][1][0][RTW89_QATAR][35] = 52, + [0][0][1][0][RTW89_FCC][37] = 76, + [0][0][1][0][RTW89_ETSI][37] = 127, + [0][0][1][0][RTW89_MKK][37] = 76, + [0][0][1][0][RTW89_IC][37] = 76, + [0][0][1][0][RTW89_KCC][37] = 76, + [0][0][1][0][RTW89_ACMA][37] = 76, + [0][0][1][0][RTW89_CHILE][37] = 54, + [0][0][1][0][RTW89_UKRAINE][37] = 127, + [0][0][1][0][RTW89_MEXICO][37] = 76, + [0][0][1][0][RTW89_CN][37] = 127, + [0][0][1][0][RTW89_QATAR][37] = 127, + [0][0][1][0][RTW89_FCC][38] = 76, + [0][0][1][0][RTW89_ETSI][38] = 28, + [0][0][1][0][RTW89_MKK][38] = 127, + [0][0][1][0][RTW89_IC][38] = 76, + [0][0][1][0][RTW89_KCC][38] = 76, + [0][0][1][0][RTW89_ACMA][38] = 76, + [0][0][1][0][RTW89_CHILE][38] = 54, + [0][0][1][0][RTW89_UKRAINE][38] = 28, + [0][0][1][0][RTW89_MEXICO][38] = 76, + [0][0][1][0][RTW89_CN][38] = 72, + [0][0][1][0][RTW89_QATAR][38] = 28, + [0][0][1][0][RTW89_FCC][40] = 76, + [0][0][1][0][RTW89_ETSI][40] = 28, + [0][0][1][0][RTW89_MKK][40] = 127, + [0][0][1][0][RTW89_IC][40] = 76, + [0][0][1][0][RTW89_KCC][40] = 76, + [0][0][1][0][RTW89_ACMA][40] = 76, + [0][0][1][0][RTW89_CHILE][40] = 54, + [0][0][1][0][RTW89_UKRAINE][40] = 28, + [0][0][1][0][RTW89_MEXICO][40] = 76, + [0][0][1][0][RTW89_CN][40] = 76, + [0][0][1][0][RTW89_QATAR][40] = 28, + [0][0][1][0][RTW89_FCC][42] = 76, + [0][0][1][0][RTW89_ETSI][42] = 28, + [0][0][1][0][RTW89_MKK][42] = 127, + [0][0][1][0][RTW89_IC][42] = 76, + [0][0][1][0][RTW89_KCC][42] = 76, + [0][0][1][0][RTW89_ACMA][42] = 76, + [0][0][1][0][RTW89_CHILE][42] = 54, + [0][0][1][0][RTW89_UKRAINE][42] = 28, + [0][0][1][0][RTW89_MEXICO][42] = 76, + [0][0][1][0][RTW89_CN][42] = 76, + [0][0][1][0][RTW89_QATAR][42] = 28, + [0][0][1][0][RTW89_FCC][44] = 76, + [0][0][1][0][RTW89_ETSI][44] = 28, + [0][0][1][0][RTW89_MKK][44] = 127, + [0][0][1][0][RTW89_IC][44] = 76, + [0][0][1][0][RTW89_KCC][44] = 76, + [0][0][1][0][RTW89_ACMA][44] = 76, + [0][0][1][0][RTW89_CHILE][44] = 54, + [0][0][1][0][RTW89_UKRAINE][44] = 28, + [0][0][1][0][RTW89_MEXICO][44] = 76, + [0][0][1][0][RTW89_CN][44] = 76, + [0][0][1][0][RTW89_QATAR][44] = 28, + [0][0][1][0][RTW89_FCC][46] = 76, + [0][0][1][0][RTW89_ETSI][46] = 28, + [0][0][1][0][RTW89_MKK][46] = 127, + [0][0][1][0][RTW89_IC][46] = 76, + [0][0][1][0][RTW89_KCC][46] = 76, + [0][0][1][0][RTW89_ACMA][46] = 76, + [0][0][1][0][RTW89_CHILE][46] = 54, + [0][0][1][0][RTW89_UKRAINE][46] = 28, + [0][0][1][0][RTW89_MEXICO][46] = 76, + [0][0][1][0][RTW89_CN][46] = 76, + [0][0][1][0][RTW89_QATAR][46] = 28, + [0][1][1][0][RTW89_FCC][0] = 68, + [0][1][1][0][RTW89_ETSI][0] = 46, + [0][1][1][0][RTW89_MKK][0] = 50, + [0][1][1][0][RTW89_IC][0] = 40, + [0][1][1][0][RTW89_KCC][0] = 72, + [0][1][1][0][RTW89_ACMA][0] = 46, + [0][1][1][0][RTW89_CHILE][0] = 18, + [0][1][1][0][RTW89_UKRAINE][0] = 40, + [0][1][1][0][RTW89_MEXICO][0] = 50, + [0][1][1][0][RTW89_CN][0] = 46, + [0][1][1][0][RTW89_QATAR][0] = 46, + [0][1][1][0][RTW89_FCC][2] = 68, + [0][1][1][0][RTW89_ETSI][2] = 46, + [0][1][1][0][RTW89_MKK][2] = 50, + [0][1][1][0][RTW89_IC][2] = 40, + [0][1][1][0][RTW89_KCC][2] = 72, + [0][1][1][0][RTW89_ACMA][2] = 46, + [0][1][1][0][RTW89_CHILE][2] = 18, + [0][1][1][0][RTW89_UKRAINE][2] = 40, + [0][1][1][0][RTW89_MEXICO][2] = 50, + [0][1][1][0][RTW89_CN][2] = 46, + [0][1][1][0][RTW89_QATAR][2] = 46, + [0][1][1][0][RTW89_FCC][4] = 68, + [0][1][1][0][RTW89_ETSI][4] = 46, + [0][1][1][0][RTW89_MKK][4] = 50, + [0][1][1][0][RTW89_IC][4] = 40, + [0][1][1][0][RTW89_KCC][4] = 72, + [0][1][1][0][RTW89_ACMA][4] = 46, + [0][1][1][0][RTW89_CHILE][4] = 18, + [0][1][1][0][RTW89_UKRAINE][4] = 40, + [0][1][1][0][RTW89_MEXICO][4] = 50, + [0][1][1][0][RTW89_CN][4] = 46, + [0][1][1][0][RTW89_QATAR][4] = 46, + [0][1][1][0][RTW89_FCC][6] = 68, + [0][1][1][0][RTW89_ETSI][6] = 46, + [0][1][1][0][RTW89_MKK][6] = 50, + [0][1][1][0][RTW89_IC][6] = 40, + [0][1][1][0][RTW89_KCC][6] = 44, + [0][1][1][0][RTW89_ACMA][6] = 46, + [0][1][1][0][RTW89_CHILE][6] = 18, + [0][1][1][0][RTW89_UKRAINE][6] = 40, + [0][1][1][0][RTW89_MEXICO][6] = 50, + [0][1][1][0][RTW89_CN][6] = 46, + [0][1][1][0][RTW89_QATAR][6] = 46, + [0][1][1][0][RTW89_FCC][8] = 68, + [0][1][1][0][RTW89_ETSI][8] = 46, + [0][1][1][0][RTW89_MKK][8] = 50, + [0][1][1][0][RTW89_IC][8] = 52, + [0][1][1][0][RTW89_KCC][8] = 72, + [0][1][1][0][RTW89_ACMA][8] = 46, + [0][1][1][0][RTW89_CHILE][8] = 42, + [0][1][1][0][RTW89_UKRAINE][8] = 40, + [0][1][1][0][RTW89_MEXICO][8] = 68, + [0][1][1][0][RTW89_CN][8] = 46, + [0][1][1][0][RTW89_QATAR][8] = 46, + [0][1][1][0][RTW89_FCC][10] = 68, + [0][1][1][0][RTW89_ETSI][10] = 46, + [0][1][1][0][RTW89_MKK][10] = 50, + [0][1][1][0][RTW89_IC][10] = 52, + [0][1][1][0][RTW89_KCC][10] = 72, + [0][1][1][0][RTW89_ACMA][10] = 46, + [0][1][1][0][RTW89_CHILE][10] = 42, + [0][1][1][0][RTW89_UKRAINE][10] = 40, + [0][1][1][0][RTW89_MEXICO][10] = 68, + [0][1][1][0][RTW89_CN][10] = 46, + [0][1][1][0][RTW89_QATAR][10] = 46, + [0][1][1][0][RTW89_FCC][12] = 68, + [0][1][1][0][RTW89_ETSI][12] = 46, + [0][1][1][0][RTW89_MKK][12] = 50, + [0][1][1][0][RTW89_IC][12] = 52, + [0][1][1][0][RTW89_KCC][12] = 72, + [0][1][1][0][RTW89_ACMA][12] = 46, + [0][1][1][0][RTW89_CHILE][12] = 42, + [0][1][1][0][RTW89_UKRAINE][12] = 40, + [0][1][1][0][RTW89_MEXICO][12] = 68, + [0][1][1][0][RTW89_CN][12] = 46, + [0][1][1][0][RTW89_QATAR][12] = 46, + [0][1][1][0][RTW89_FCC][14] = 68, + [0][1][1][0][RTW89_ETSI][14] = 46, + [0][1][1][0][RTW89_MKK][14] = 50, + [0][1][1][0][RTW89_IC][14] = 52, + [0][1][1][0][RTW89_KCC][14] = 72, + [0][1][1][0][RTW89_ACMA][14] = 46, + [0][1][1][0][RTW89_CHILE][14] = 42, + [0][1][1][0][RTW89_UKRAINE][14] = 40, + [0][1][1][0][RTW89_MEXICO][14] = 68, + [0][1][1][0][RTW89_CN][14] = 46, + [0][1][1][0][RTW89_QATAR][14] = 46, + [0][1][1][0][RTW89_FCC][15] = 68, + [0][1][1][0][RTW89_ETSI][15] = 46, + [0][1][1][0][RTW89_MKK][15] = 70, + [0][1][1][0][RTW89_IC][15] = 68, + [0][1][1][0][RTW89_KCC][15] = 72, + [0][1][1][0][RTW89_ACMA][15] = 46, + [0][1][1][0][RTW89_CHILE][15] = 42, + [0][1][1][0][RTW89_UKRAINE][15] = 40, + [0][1][1][0][RTW89_MEXICO][15] = 68, + [0][1][1][0][RTW89_CN][15] = 127, + [0][1][1][0][RTW89_QATAR][15] = 40, + [0][1][1][0][RTW89_FCC][17] = 68, + [0][1][1][0][RTW89_ETSI][17] = 46, + [0][1][1][0][RTW89_MKK][17] = 70, + [0][1][1][0][RTW89_IC][17] = 68, + [0][1][1][0][RTW89_KCC][17] = 72, + [0][1][1][0][RTW89_ACMA][17] = 46, + [0][1][1][0][RTW89_CHILE][17] = 42, + [0][1][1][0][RTW89_UKRAINE][17] = 40, + [0][1][1][0][RTW89_MEXICO][17] = 68, + [0][1][1][0][RTW89_CN][17] = 127, + [0][1][1][0][RTW89_QATAR][17] = 40, + [0][1][1][0][RTW89_FCC][19] = 68, + [0][1][1][0][RTW89_ETSI][19] = 46, + [0][1][1][0][RTW89_MKK][19] = 70, + [0][1][1][0][RTW89_IC][19] = 68, + [0][1][1][0][RTW89_KCC][19] = 72, + [0][1][1][0][RTW89_ACMA][19] = 46, + [0][1][1][0][RTW89_CHILE][19] = 42, + [0][1][1][0][RTW89_UKRAINE][19] = 40, + [0][1][1][0][RTW89_MEXICO][19] = 68, + [0][1][1][0][RTW89_CN][19] = 127, + [0][1][1][0][RTW89_QATAR][19] = 40, + [0][1][1][0][RTW89_FCC][21] = 68, + [0][1][1][0][RTW89_ETSI][21] = 46, + [0][1][1][0][RTW89_MKK][21] = 70, + [0][1][1][0][RTW89_IC][21] = 68, + [0][1][1][0][RTW89_KCC][21] = 72, + [0][1][1][0][RTW89_ACMA][21] = 46, + [0][1][1][0][RTW89_CHILE][21] = 42, + [0][1][1][0][RTW89_UKRAINE][21] = 40, + [0][1][1][0][RTW89_MEXICO][21] = 68, + [0][1][1][0][RTW89_CN][21] = 127, + [0][1][1][0][RTW89_QATAR][21] = 40, + [0][1][1][0][RTW89_FCC][23] = 68, + [0][1][1][0][RTW89_ETSI][23] = 46, + [0][1][1][0][RTW89_MKK][23] = 70, + [0][1][1][0][RTW89_IC][23] = 68, + [0][1][1][0][RTW89_KCC][23] = 72, + [0][1][1][0][RTW89_ACMA][23] = 46, + [0][1][1][0][RTW89_CHILE][23] = 42, + [0][1][1][0][RTW89_UKRAINE][23] = 40, + [0][1][1][0][RTW89_MEXICO][23] = 68, + [0][1][1][0][RTW89_CN][23] = 127, + [0][1][1][0][RTW89_QATAR][23] = 40, + [0][1][1][0][RTW89_FCC][25] = 68, + [0][1][1][0][RTW89_ETSI][25] = 46, + [0][1][1][0][RTW89_MKK][25] = 70, + [0][1][1][0][RTW89_IC][25] = 127, + [0][1][1][0][RTW89_KCC][25] = 72, + [0][1][1][0][RTW89_ACMA][25] = 127, + [0][1][1][0][RTW89_CHILE][25] = 42, + [0][1][1][0][RTW89_UKRAINE][25] = 40, + [0][1][1][0][RTW89_MEXICO][25] = 68, + [0][1][1][0][RTW89_CN][25] = 127, + [0][1][1][0][RTW89_QATAR][25] = 40, + [0][1][1][0][RTW89_FCC][27] = 68, + [0][1][1][0][RTW89_ETSI][27] = 46, + [0][1][1][0][RTW89_MKK][27] = 70, + [0][1][1][0][RTW89_IC][27] = 127, + [0][1][1][0][RTW89_KCC][27] = 72, + [0][1][1][0][RTW89_ACMA][27] = 127, + [0][1][1][0][RTW89_CHILE][27] = 42, + [0][1][1][0][RTW89_UKRAINE][27] = 40, + [0][1][1][0][RTW89_MEXICO][27] = 68, + [0][1][1][0][RTW89_CN][27] = 127, + [0][1][1][0][RTW89_QATAR][27] = 40, + [0][1][1][0][RTW89_FCC][29] = 68, + [0][1][1][0][RTW89_ETSI][29] = 46, + [0][1][1][0][RTW89_MKK][29] = 70, + [0][1][1][0][RTW89_IC][29] = 127, + [0][1][1][0][RTW89_KCC][29] = 72, + [0][1][1][0][RTW89_ACMA][29] = 127, + [0][1][1][0][RTW89_CHILE][29] = 42, + [0][1][1][0][RTW89_UKRAINE][29] = 40, + [0][1][1][0][RTW89_MEXICO][29] = 68, + [0][1][1][0][RTW89_CN][29] = 127, + [0][1][1][0][RTW89_QATAR][29] = 40, + [0][1][1][0][RTW89_FCC][31] = 68, + [0][1][1][0][RTW89_ETSI][31] = 46, + [0][1][1][0][RTW89_MKK][31] = 70, + [0][1][1][0][RTW89_IC][31] = 68, + [0][1][1][0][RTW89_KCC][31] = 72, + [0][1][1][0][RTW89_ACMA][31] = 46, + [0][1][1][0][RTW89_CHILE][31] = 42, + [0][1][1][0][RTW89_UKRAINE][31] = 40, + [0][1][1][0][RTW89_MEXICO][31] = 68, + [0][1][1][0][RTW89_CN][31] = 127, + [0][1][1][0][RTW89_QATAR][31] = 40, + [0][1][1][0][RTW89_FCC][33] = 68, + [0][1][1][0][RTW89_ETSI][33] = 46, + [0][1][1][0][RTW89_MKK][33] = 70, + [0][1][1][0][RTW89_IC][33] = 68, + [0][1][1][0][RTW89_KCC][33] = 72, + [0][1][1][0][RTW89_ACMA][33] = 46, + [0][1][1][0][RTW89_CHILE][33] = 42, + [0][1][1][0][RTW89_UKRAINE][33] = 40, + [0][1][1][0][RTW89_MEXICO][33] = 68, + [0][1][1][0][RTW89_CN][33] = 127, + [0][1][1][0][RTW89_QATAR][33] = 40, + [0][1][1][0][RTW89_FCC][35] = 66, + [0][1][1][0][RTW89_ETSI][35] = 46, + [0][1][1][0][RTW89_MKK][35] = 70, + [0][1][1][0][RTW89_IC][35] = 66, + [0][1][1][0][RTW89_KCC][35] = 72, + [0][1][1][0][RTW89_ACMA][35] = 46, + [0][1][1][0][RTW89_CHILE][35] = 42, + [0][1][1][0][RTW89_UKRAINE][35] = 40, + [0][1][1][0][RTW89_MEXICO][35] = 66, + [0][1][1][0][RTW89_CN][35] = 127, + [0][1][1][0][RTW89_QATAR][35] = 40, + [0][1][1][0][RTW89_FCC][37] = 68, + [0][1][1][0][RTW89_ETSI][37] = 127, + [0][1][1][0][RTW89_MKK][37] = 70, + [0][1][1][0][RTW89_IC][37] = 68, + [0][1][1][0][RTW89_KCC][37] = 72, + [0][1][1][0][RTW89_ACMA][37] = 68, + [0][1][1][0][RTW89_CHILE][37] = 42, + [0][1][1][0][RTW89_UKRAINE][37] = 127, + [0][1][1][0][RTW89_MEXICO][37] = 68, + [0][1][1][0][RTW89_CN][37] = 127, + [0][1][1][0][RTW89_QATAR][37] = 127, + [0][1][1][0][RTW89_FCC][38] = 76, + [0][1][1][0][RTW89_ETSI][38] = 16, + [0][1][1][0][RTW89_MKK][38] = 127, + [0][1][1][0][RTW89_IC][38] = 76, + [0][1][1][0][RTW89_KCC][38] = 72, + [0][1][1][0][RTW89_ACMA][38] = 76, + [0][1][1][0][RTW89_CHILE][38] = 42, + [0][1][1][0][RTW89_UKRAINE][38] = 16, + [0][1][1][0][RTW89_MEXICO][38] = 76, + [0][1][1][0][RTW89_CN][38] = 72, + [0][1][1][0][RTW89_QATAR][38] = 16, + [0][1][1][0][RTW89_FCC][40] = 76, + [0][1][1][0][RTW89_ETSI][40] = 16, + [0][1][1][0][RTW89_MKK][40] = 127, + [0][1][1][0][RTW89_IC][40] = 76, + [0][1][1][0][RTW89_KCC][40] = 72, + [0][1][1][0][RTW89_ACMA][40] = 76, + [0][1][1][0][RTW89_CHILE][40] = 42, + [0][1][1][0][RTW89_UKRAINE][40] = 16, + [0][1][1][0][RTW89_MEXICO][40] = 76, + [0][1][1][0][RTW89_CN][40] = 76, + [0][1][1][0][RTW89_QATAR][40] = 16, + [0][1][1][0][RTW89_FCC][42] = 76, + [0][1][1][0][RTW89_ETSI][42] = 16, + [0][1][1][0][RTW89_MKK][42] = 127, + [0][1][1][0][RTW89_IC][42] = 76, + [0][1][1][0][RTW89_KCC][42] = 72, + [0][1][1][0][RTW89_ACMA][42] = 76, + [0][1][1][0][RTW89_CHILE][42] = 42, + [0][1][1][0][RTW89_UKRAINE][42] = 16, + [0][1][1][0][RTW89_MEXICO][42] = 76, + [0][1][1][0][RTW89_CN][42] = 76, + [0][1][1][0][RTW89_QATAR][42] = 16, + [0][1][1][0][RTW89_FCC][44] = 76, + [0][1][1][0][RTW89_ETSI][44] = 16, + [0][1][1][0][RTW89_MKK][44] = 127, + [0][1][1][0][RTW89_IC][44] = 76, + [0][1][1][0][RTW89_KCC][44] = 72, + [0][1][1][0][RTW89_ACMA][44] = 76, + [0][1][1][0][RTW89_CHILE][44] = 42, + [0][1][1][0][RTW89_UKRAINE][44] = 16, + [0][1][1][0][RTW89_MEXICO][44] = 76, + [0][1][1][0][RTW89_CN][44] = 76, + [0][1][1][0][RTW89_QATAR][44] = 16, + [0][1][1][0][RTW89_FCC][46] = 76, + [0][1][1][0][RTW89_ETSI][46] = 16, + [0][1][1][0][RTW89_MKK][46] = 127, + [0][1][1][0][RTW89_IC][46] = 76, + [0][1][1][0][RTW89_KCC][46] = 72, + [0][1][1][0][RTW89_ACMA][46] = 76, + [0][1][1][0][RTW89_CHILE][46] = 42, + [0][1][1][0][RTW89_UKRAINE][46] = 16, + [0][1][1][0][RTW89_MEXICO][46] = 76, + [0][1][1][0][RTW89_CN][46] = 76, + [0][1][1][0][RTW89_QATAR][46] = 16, + [0][0][2][0][RTW89_FCC][0] = 76, + [0][0][2][0][RTW89_ETSI][0] = 58, + [0][0][2][0][RTW89_MKK][0] = 62, + [0][0][2][0][RTW89_IC][0] = 62, + [0][0][2][0][RTW89_KCC][0] = 76, + [0][0][2][0][RTW89_ACMA][0] = 58, + [0][0][2][0][RTW89_CHILE][0] = 30, + [0][0][2][0][RTW89_UKRAINE][0] = 52, + [0][0][2][0][RTW89_MEXICO][0] = 62, + [0][0][2][0][RTW89_CN][0] = 58, + [0][0][2][0][RTW89_QATAR][0] = 58, + [0][0][2][0][RTW89_FCC][2] = 76, + [0][0][2][0][RTW89_ETSI][2] = 58, + [0][0][2][0][RTW89_MKK][2] = 62, + [0][0][2][0][RTW89_IC][2] = 62, + [0][0][2][0][RTW89_KCC][2] = 76, + [0][0][2][0][RTW89_ACMA][2] = 58, + [0][0][2][0][RTW89_CHILE][2] = 30, + [0][0][2][0][RTW89_UKRAINE][2] = 52, + [0][0][2][0][RTW89_MEXICO][2] = 62, + [0][0][2][0][RTW89_CN][2] = 58, + [0][0][2][0][RTW89_QATAR][2] = 58, + [0][0][2][0][RTW89_FCC][4] = 76, + [0][0][2][0][RTW89_ETSI][4] = 58, + [0][0][2][0][RTW89_MKK][4] = 62, + [0][0][2][0][RTW89_IC][4] = 62, + [0][0][2][0][RTW89_KCC][4] = 76, + [0][0][2][0][RTW89_ACMA][4] = 58, + [0][0][2][0][RTW89_CHILE][4] = 30, + [0][0][2][0][RTW89_UKRAINE][4] = 52, + [0][0][2][0][RTW89_MEXICO][4] = 62, + [0][0][2][0][RTW89_CN][4] = 58, + [0][0][2][0][RTW89_QATAR][4] = 58, + [0][0][2][0][RTW89_FCC][6] = 76, + [0][0][2][0][RTW89_ETSI][6] = 58, + [0][0][2][0][RTW89_MKK][6] = 62, + [0][0][2][0][RTW89_IC][6] = 62, + [0][0][2][0][RTW89_KCC][6] = 58, + [0][0][2][0][RTW89_ACMA][6] = 58, + [0][0][2][0][RTW89_CHILE][6] = 30, + [0][0][2][0][RTW89_UKRAINE][6] = 52, + [0][0][2][0][RTW89_MEXICO][6] = 62, + [0][0][2][0][RTW89_CN][6] = 58, + [0][0][2][0][RTW89_QATAR][6] = 58, + [0][0][2][0][RTW89_FCC][8] = 76, + [0][0][2][0][RTW89_ETSI][8] = 58, + [0][0][2][0][RTW89_MKK][8] = 62, + [0][0][2][0][RTW89_IC][8] = 64, + [0][0][2][0][RTW89_KCC][8] = 76, + [0][0][2][0][RTW89_ACMA][8] = 58, + [0][0][2][0][RTW89_CHILE][8] = 54, + [0][0][2][0][RTW89_UKRAINE][8] = 52, + [0][0][2][0][RTW89_MEXICO][8] = 76, + [0][0][2][0][RTW89_CN][8] = 58, + [0][0][2][0][RTW89_QATAR][8] = 58, + [0][0][2][0][RTW89_FCC][10] = 76, + [0][0][2][0][RTW89_ETSI][10] = 58, + [0][0][2][0][RTW89_MKK][10] = 62, + [0][0][2][0][RTW89_IC][10] = 64, + [0][0][2][0][RTW89_KCC][10] = 76, + [0][0][2][0][RTW89_ACMA][10] = 58, + [0][0][2][0][RTW89_CHILE][10] = 54, + [0][0][2][0][RTW89_UKRAINE][10] = 52, + [0][0][2][0][RTW89_MEXICO][10] = 76, + [0][0][2][0][RTW89_CN][10] = 58, + [0][0][2][0][RTW89_QATAR][10] = 58, + [0][0][2][0][RTW89_FCC][12] = 76, + [0][0][2][0][RTW89_ETSI][12] = 58, + [0][0][2][0][RTW89_MKK][12] = 62, + [0][0][2][0][RTW89_IC][12] = 64, + [0][0][2][0][RTW89_KCC][12] = 76, + [0][0][2][0][RTW89_ACMA][12] = 58, + [0][0][2][0][RTW89_CHILE][12] = 54, + [0][0][2][0][RTW89_UKRAINE][12] = 52, + [0][0][2][0][RTW89_MEXICO][12] = 76, + [0][0][2][0][RTW89_CN][12] = 58, + [0][0][2][0][RTW89_QATAR][12] = 58, + [0][0][2][0][RTW89_FCC][14] = 76, + [0][0][2][0][RTW89_ETSI][14] = 58, + [0][0][2][0][RTW89_MKK][14] = 62, + [0][0][2][0][RTW89_IC][14] = 64, + [0][0][2][0][RTW89_KCC][14] = 76, + [0][0][2][0][RTW89_ACMA][14] = 58, + [0][0][2][0][RTW89_CHILE][14] = 54, + [0][0][2][0][RTW89_UKRAINE][14] = 52, + [0][0][2][0][RTW89_MEXICO][14] = 76, + [0][0][2][0][RTW89_CN][14] = 58, + [0][0][2][0][RTW89_QATAR][14] = 58, + [0][0][2][0][RTW89_FCC][15] = 74, + [0][0][2][0][RTW89_ETSI][15] = 58, + [0][0][2][0][RTW89_MKK][15] = 76, + [0][0][2][0][RTW89_IC][15] = 74, + [0][0][2][0][RTW89_KCC][15] = 76, + [0][0][2][0][RTW89_ACMA][15] = 58, + [0][0][2][0][RTW89_CHILE][15] = 54, + [0][0][2][0][RTW89_UKRAINE][15] = 52, + [0][0][2][0][RTW89_MEXICO][15] = 74, + [0][0][2][0][RTW89_CN][15] = 127, + [0][0][2][0][RTW89_QATAR][15] = 52, + [0][0][2][0][RTW89_FCC][17] = 76, + [0][0][2][0][RTW89_ETSI][17] = 58, + [0][0][2][0][RTW89_MKK][17] = 76, + [0][0][2][0][RTW89_IC][17] = 76, + [0][0][2][0][RTW89_KCC][17] = 76, + [0][0][2][0][RTW89_ACMA][17] = 58, + [0][0][2][0][RTW89_CHILE][17] = 54, + [0][0][2][0][RTW89_UKRAINE][17] = 52, + [0][0][2][0][RTW89_MEXICO][17] = 76, + [0][0][2][0][RTW89_CN][17] = 127, + [0][0][2][0][RTW89_QATAR][17] = 52, + [0][0][2][0][RTW89_FCC][19] = 76, + [0][0][2][0][RTW89_ETSI][19] = 58, + [0][0][2][0][RTW89_MKK][19] = 76, + [0][0][2][0][RTW89_IC][19] = 76, + [0][0][2][0][RTW89_KCC][19] = 76, + [0][0][2][0][RTW89_ACMA][19] = 58, + [0][0][2][0][RTW89_CHILE][19] = 54, + [0][0][2][0][RTW89_UKRAINE][19] = 52, + [0][0][2][0][RTW89_MEXICO][19] = 76, + [0][0][2][0][RTW89_CN][19] = 127, + [0][0][2][0][RTW89_QATAR][19] = 52, + [0][0][2][0][RTW89_FCC][21] = 76, + [0][0][2][0][RTW89_ETSI][21] = 58, + [0][0][2][0][RTW89_MKK][21] = 76, + [0][0][2][0][RTW89_IC][21] = 76, + [0][0][2][0][RTW89_KCC][21] = 76, + [0][0][2][0][RTW89_ACMA][21] = 58, + [0][0][2][0][RTW89_CHILE][21] = 54, + [0][0][2][0][RTW89_UKRAINE][21] = 52, + [0][0][2][0][RTW89_MEXICO][21] = 76, + [0][0][2][0][RTW89_CN][21] = 127, + [0][0][2][0][RTW89_QATAR][21] = 52, + [0][0][2][0][RTW89_FCC][23] = 76, + [0][0][2][0][RTW89_ETSI][23] = 58, + [0][0][2][0][RTW89_MKK][23] = 76, + [0][0][2][0][RTW89_IC][23] = 76, + [0][0][2][0][RTW89_KCC][23] = 76, + [0][0][2][0][RTW89_ACMA][23] = 58, + [0][0][2][0][RTW89_CHILE][23] = 54, + [0][0][2][0][RTW89_UKRAINE][23] = 52, + [0][0][2][0][RTW89_MEXICO][23] = 76, + [0][0][2][0][RTW89_CN][23] = 127, + [0][0][2][0][RTW89_QATAR][23] = 52, + [0][0][2][0][RTW89_FCC][25] = 76, + [0][0][2][0][RTW89_ETSI][25] = 58, + [0][0][2][0][RTW89_MKK][25] = 76, + [0][0][2][0][RTW89_IC][25] = 127, + [0][0][2][0][RTW89_KCC][25] = 76, + [0][0][2][0][RTW89_ACMA][25] = 127, + [0][0][2][0][RTW89_CHILE][25] = 54, + [0][0][2][0][RTW89_UKRAINE][25] = 52, + [0][0][2][0][RTW89_MEXICO][25] = 76, + [0][0][2][0][RTW89_CN][25] = 127, + [0][0][2][0][RTW89_QATAR][25] = 52, + [0][0][2][0][RTW89_FCC][27] = 76, + [0][0][2][0][RTW89_ETSI][27] = 58, + [0][0][2][0][RTW89_MKK][27] = 76, + [0][0][2][0][RTW89_IC][27] = 127, + [0][0][2][0][RTW89_KCC][27] = 76, + [0][0][2][0][RTW89_ACMA][27] = 127, + [0][0][2][0][RTW89_CHILE][27] = 54, + [0][0][2][0][RTW89_UKRAINE][27] = 52, + [0][0][2][0][RTW89_MEXICO][27] = 76, + [0][0][2][0][RTW89_CN][27] = 127, + [0][0][2][0][RTW89_QATAR][27] = 52, + [0][0][2][0][RTW89_FCC][29] = 76, + [0][0][2][0][RTW89_ETSI][29] = 58, + [0][0][2][0][RTW89_MKK][29] = 76, + [0][0][2][0][RTW89_IC][29] = 127, + [0][0][2][0][RTW89_KCC][29] = 76, + [0][0][2][0][RTW89_ACMA][29] = 127, + [0][0][2][0][RTW89_CHILE][29] = 54, + [0][0][2][0][RTW89_UKRAINE][29] = 52, + [0][0][2][0][RTW89_MEXICO][29] = 76, + [0][0][2][0][RTW89_CN][29] = 127, + [0][0][2][0][RTW89_QATAR][29] = 52, + [0][0][2][0][RTW89_FCC][31] = 76, + [0][0][2][0][RTW89_ETSI][31] = 58, + [0][0][2][0][RTW89_MKK][31] = 76, + [0][0][2][0][RTW89_IC][31] = 76, + [0][0][2][0][RTW89_KCC][31] = 76, + [0][0][2][0][RTW89_ACMA][31] = 58, + [0][0][2][0][RTW89_CHILE][31] = 54, + [0][0][2][0][RTW89_UKRAINE][31] = 52, + [0][0][2][0][RTW89_MEXICO][31] = 76, + [0][0][2][0][RTW89_CN][31] = 127, + [0][0][2][0][RTW89_QATAR][31] = 52, + [0][0][2][0][RTW89_FCC][33] = 76, + [0][0][2][0][RTW89_ETSI][33] = 58, + [0][0][2][0][RTW89_MKK][33] = 76, + [0][0][2][0][RTW89_IC][33] = 76, + [0][0][2][0][RTW89_KCC][33] = 76, + [0][0][2][0][RTW89_ACMA][33] = 58, + [0][0][2][0][RTW89_CHILE][33] = 54, + [0][0][2][0][RTW89_UKRAINE][33] = 52, + [0][0][2][0][RTW89_MEXICO][33] = 76, + [0][0][2][0][RTW89_CN][33] = 127, + [0][0][2][0][RTW89_QATAR][33] = 52, + [0][0][2][0][RTW89_FCC][35] = 70, + [0][0][2][0][RTW89_ETSI][35] = 58, + [0][0][2][0][RTW89_MKK][35] = 76, + [0][0][2][0][RTW89_IC][35] = 70, + [0][0][2][0][RTW89_KCC][35] = 76, + [0][0][2][0][RTW89_ACMA][35] = 58, + [0][0][2][0][RTW89_CHILE][35] = 54, + [0][0][2][0][RTW89_UKRAINE][35] = 52, + [0][0][2][0][RTW89_MEXICO][35] = 70, + [0][0][2][0][RTW89_CN][35] = 127, + [0][0][2][0][RTW89_QATAR][35] = 52, + [0][0][2][0][RTW89_FCC][37] = 76, + [0][0][2][0][RTW89_ETSI][37] = 127, + [0][0][2][0][RTW89_MKK][37] = 76, + [0][0][2][0][RTW89_IC][37] = 76, + [0][0][2][0][RTW89_KCC][37] = 76, + [0][0][2][0][RTW89_ACMA][37] = 76, + [0][0][2][0][RTW89_CHILE][37] = 54, + [0][0][2][0][RTW89_UKRAINE][37] = 127, + [0][0][2][0][RTW89_MEXICO][37] = 76, + [0][0][2][0][RTW89_CN][37] = 127, + [0][0][2][0][RTW89_QATAR][37] = 127, + [0][0][2][0][RTW89_FCC][38] = 76, + [0][0][2][0][RTW89_ETSI][38] = 28, + [0][0][2][0][RTW89_MKK][38] = 127, + [0][0][2][0][RTW89_IC][38] = 76, + [0][0][2][0][RTW89_KCC][38] = 76, + [0][0][2][0][RTW89_ACMA][38] = 76, + [0][0][2][0][RTW89_CHILE][38] = 54, + [0][0][2][0][RTW89_UKRAINE][38] = 28, + [0][0][2][0][RTW89_MEXICO][38] = 76, + [0][0][2][0][RTW89_CN][38] = 68, + [0][0][2][0][RTW89_QATAR][38] = 28, + [0][0][2][0][RTW89_FCC][40] = 76, + [0][0][2][0][RTW89_ETSI][40] = 28, + [0][0][2][0][RTW89_MKK][40] = 127, + [0][0][2][0][RTW89_IC][40] = 76, + [0][0][2][0][RTW89_KCC][40] = 76, + [0][0][2][0][RTW89_ACMA][40] = 76, + [0][0][2][0][RTW89_CHILE][40] = 54, + [0][0][2][0][RTW89_UKRAINE][40] = 28, + [0][0][2][0][RTW89_MEXICO][40] = 76, + [0][0][2][0][RTW89_CN][40] = 76, + [0][0][2][0][RTW89_QATAR][40] = 28, + [0][0][2][0][RTW89_FCC][42] = 76, + [0][0][2][0][RTW89_ETSI][42] = 28, + [0][0][2][0][RTW89_MKK][42] = 127, + [0][0][2][0][RTW89_IC][42] = 76, + [0][0][2][0][RTW89_KCC][42] = 76, + [0][0][2][0][RTW89_ACMA][42] = 76, + [0][0][2][0][RTW89_CHILE][42] = 54, + [0][0][2][0][RTW89_UKRAINE][42] = 28, + [0][0][2][0][RTW89_MEXICO][42] = 76, + [0][0][2][0][RTW89_CN][42] = 76, + [0][0][2][0][RTW89_QATAR][42] = 28, + [0][0][2][0][RTW89_FCC][44] = 76, + [0][0][2][0][RTW89_ETSI][44] = 28, + [0][0][2][0][RTW89_MKK][44] = 127, + [0][0][2][0][RTW89_IC][44] = 76, + [0][0][2][0][RTW89_KCC][44] = 76, + [0][0][2][0][RTW89_ACMA][44] = 76, + [0][0][2][0][RTW89_CHILE][44] = 54, + [0][0][2][0][RTW89_UKRAINE][44] = 28, + [0][0][2][0][RTW89_MEXICO][44] = 76, + [0][0][2][0][RTW89_CN][44] = 76, + [0][0][2][0][RTW89_QATAR][44] = 28, + [0][0][2][0][RTW89_FCC][46] = 76, + [0][0][2][0][RTW89_ETSI][46] = 28, + [0][0][2][0][RTW89_MKK][46] = 127, + [0][0][2][0][RTW89_IC][46] = 76, + [0][0][2][0][RTW89_KCC][46] = 76, + [0][0][2][0][RTW89_ACMA][46] = 76, + [0][0][2][0][RTW89_CHILE][46] = 54, + [0][0][2][0][RTW89_UKRAINE][46] = 28, + [0][0][2][0][RTW89_MEXICO][46] = 76, + [0][0][2][0][RTW89_CN][46] = 76, + [0][0][2][0][RTW89_QATAR][46] = 28, + [0][1][2][0][RTW89_FCC][0] = 68, + [0][1][2][0][RTW89_ETSI][0] = 46, + [0][1][2][0][RTW89_MKK][0] = 50, + [0][1][2][0][RTW89_IC][0] = 40, + [0][1][2][0][RTW89_KCC][0] = 68, + [0][1][2][0][RTW89_ACMA][0] = 46, + [0][1][2][0][RTW89_CHILE][0] = 18, + [0][1][2][0][RTW89_UKRAINE][0] = 40, + [0][1][2][0][RTW89_MEXICO][0] = 50, + [0][1][2][0][RTW89_CN][0] = 46, + [0][1][2][0][RTW89_QATAR][0] = 46, + [0][1][2][0][RTW89_FCC][2] = 68, + [0][1][2][0][RTW89_ETSI][2] = 46, + [0][1][2][0][RTW89_MKK][2] = 50, + [0][1][2][0][RTW89_IC][2] = 40, + [0][1][2][0][RTW89_KCC][2] = 68, + [0][1][2][0][RTW89_ACMA][2] = 46, + [0][1][2][0][RTW89_CHILE][2] = 18, + [0][1][2][0][RTW89_UKRAINE][2] = 40, + [0][1][2][0][RTW89_MEXICO][2] = 50, + [0][1][2][0][RTW89_CN][2] = 46, + [0][1][2][0][RTW89_QATAR][2] = 46, + [0][1][2][0][RTW89_FCC][4] = 68, + [0][1][2][0][RTW89_ETSI][4] = 46, + [0][1][2][0][RTW89_MKK][4] = 50, + [0][1][2][0][RTW89_IC][4] = 40, + [0][1][2][0][RTW89_KCC][4] = 68, + [0][1][2][0][RTW89_ACMA][4] = 46, + [0][1][2][0][RTW89_CHILE][4] = 18, + [0][1][2][0][RTW89_UKRAINE][4] = 40, + [0][1][2][0][RTW89_MEXICO][4] = 50, + [0][1][2][0][RTW89_CN][4] = 46, + [0][1][2][0][RTW89_QATAR][4] = 46, + [0][1][2][0][RTW89_FCC][6] = 68, + [0][1][2][0][RTW89_ETSI][6] = 46, + [0][1][2][0][RTW89_MKK][6] = 50, + [0][1][2][0][RTW89_IC][6] = 40, + [0][1][2][0][RTW89_KCC][6] = 38, + [0][1][2][0][RTW89_ACMA][6] = 46, + [0][1][2][0][RTW89_CHILE][6] = 18, + [0][1][2][0][RTW89_UKRAINE][6] = 40, + [0][1][2][0][RTW89_MEXICO][6] = 50, + [0][1][2][0][RTW89_CN][6] = 46, + [0][1][2][0][RTW89_QATAR][6] = 46, + [0][1][2][0][RTW89_FCC][8] = 68, + [0][1][2][0][RTW89_ETSI][8] = 46, + [0][1][2][0][RTW89_MKK][8] = 50, + [0][1][2][0][RTW89_IC][8] = 52, + [0][1][2][0][RTW89_KCC][8] = 68, + [0][1][2][0][RTW89_ACMA][8] = 46, + [0][1][2][0][RTW89_CHILE][8] = 42, + [0][1][2][0][RTW89_UKRAINE][8] = 40, + [0][1][2][0][RTW89_MEXICO][8] = 68, + [0][1][2][0][RTW89_CN][8] = 46, + [0][1][2][0][RTW89_QATAR][8] = 46, + [0][1][2][0][RTW89_FCC][10] = 68, + [0][1][2][0][RTW89_ETSI][10] = 46, + [0][1][2][0][RTW89_MKK][10] = 50, + [0][1][2][0][RTW89_IC][10] = 52, + [0][1][2][0][RTW89_KCC][10] = 68, + [0][1][2][0][RTW89_ACMA][10] = 46, + [0][1][2][0][RTW89_CHILE][10] = 42, + [0][1][2][0][RTW89_UKRAINE][10] = 40, + [0][1][2][0][RTW89_MEXICO][10] = 68, + [0][1][2][0][RTW89_CN][10] = 46, + [0][1][2][0][RTW89_QATAR][10] = 46, + [0][1][2][0][RTW89_FCC][12] = 68, + [0][1][2][0][RTW89_ETSI][12] = 46, + [0][1][2][0][RTW89_MKK][12] = 50, + [0][1][2][0][RTW89_IC][12] = 52, + [0][1][2][0][RTW89_KCC][12] = 68, + [0][1][2][0][RTW89_ACMA][12] = 46, + [0][1][2][0][RTW89_CHILE][12] = 42, + [0][1][2][0][RTW89_UKRAINE][12] = 40, + [0][1][2][0][RTW89_MEXICO][12] = 68, + [0][1][2][0][RTW89_CN][12] = 46, + [0][1][2][0][RTW89_QATAR][12] = 46, + [0][1][2][0][RTW89_FCC][14] = 68, + [0][1][2][0][RTW89_ETSI][14] = 46, + [0][1][2][0][RTW89_MKK][14] = 50, + [0][1][2][0][RTW89_IC][14] = 52, + [0][1][2][0][RTW89_KCC][14] = 68, + [0][1][2][0][RTW89_ACMA][14] = 46, + [0][1][2][0][RTW89_CHILE][14] = 42, + [0][1][2][0][RTW89_UKRAINE][14] = 40, + [0][1][2][0][RTW89_MEXICO][14] = 68, + [0][1][2][0][RTW89_CN][14] = 46, + [0][1][2][0][RTW89_QATAR][14] = 46, + [0][1][2][0][RTW89_FCC][15] = 68, + [0][1][2][0][RTW89_ETSI][15] = 46, + [0][1][2][0][RTW89_MKK][15] = 70, + [0][1][2][0][RTW89_IC][15] = 68, + [0][1][2][0][RTW89_KCC][15] = 66, + [0][1][2][0][RTW89_ACMA][15] = 46, + [0][1][2][0][RTW89_CHILE][15] = 42, + [0][1][2][0][RTW89_UKRAINE][15] = 40, + [0][1][2][0][RTW89_MEXICO][15] = 68, + [0][1][2][0][RTW89_CN][15] = 127, + [0][1][2][0][RTW89_QATAR][15] = 40, + [0][1][2][0][RTW89_FCC][17] = 68, + [0][1][2][0][RTW89_ETSI][17] = 46, + [0][1][2][0][RTW89_MKK][17] = 70, + [0][1][2][0][RTW89_IC][17] = 68, + [0][1][2][0][RTW89_KCC][17] = 66, + [0][1][2][0][RTW89_ACMA][17] = 46, + [0][1][2][0][RTW89_CHILE][17] = 42, + [0][1][2][0][RTW89_UKRAINE][17] = 40, + [0][1][2][0][RTW89_MEXICO][17] = 68, + [0][1][2][0][RTW89_CN][17] = 127, + [0][1][2][0][RTW89_QATAR][17] = 40, + [0][1][2][0][RTW89_FCC][19] = 68, + [0][1][2][0][RTW89_ETSI][19] = 46, + [0][1][2][0][RTW89_MKK][19] = 70, + [0][1][2][0][RTW89_IC][19] = 68, + [0][1][2][0][RTW89_KCC][19] = 66, + [0][1][2][0][RTW89_ACMA][19] = 46, + [0][1][2][0][RTW89_CHILE][19] = 42, + [0][1][2][0][RTW89_UKRAINE][19] = 40, + [0][1][2][0][RTW89_MEXICO][19] = 68, + [0][1][2][0][RTW89_CN][19] = 127, + [0][1][2][0][RTW89_QATAR][19] = 40, + [0][1][2][0][RTW89_FCC][21] = 68, + [0][1][2][0][RTW89_ETSI][21] = 46, + [0][1][2][0][RTW89_MKK][21] = 70, + [0][1][2][0][RTW89_IC][21] = 68, + [0][1][2][0][RTW89_KCC][21] = 66, + [0][1][2][0][RTW89_ACMA][21] = 46, + [0][1][2][0][RTW89_CHILE][21] = 42, + [0][1][2][0][RTW89_UKRAINE][21] = 40, + [0][1][2][0][RTW89_MEXICO][21] = 68, + [0][1][2][0][RTW89_CN][21] = 127, + [0][1][2][0][RTW89_QATAR][21] = 40, + [0][1][2][0][RTW89_FCC][23] = 68, + [0][1][2][0][RTW89_ETSI][23] = 46, + [0][1][2][0][RTW89_MKK][23] = 70, + [0][1][2][0][RTW89_IC][23] = 68, + [0][1][2][0][RTW89_KCC][23] = 66, + [0][1][2][0][RTW89_ACMA][23] = 46, + [0][1][2][0][RTW89_CHILE][23] = 42, + [0][1][2][0][RTW89_UKRAINE][23] = 40, + [0][1][2][0][RTW89_MEXICO][23] = 68, + [0][1][2][0][RTW89_CN][23] = 127, + [0][1][2][0][RTW89_QATAR][23] = 40, + [0][1][2][0][RTW89_FCC][25] = 68, + [0][1][2][0][RTW89_ETSI][25] = 46, + [0][1][2][0][RTW89_MKK][25] = 70, + [0][1][2][0][RTW89_IC][25] = 127, + [0][1][2][0][RTW89_KCC][25] = 66, + [0][1][2][0][RTW89_ACMA][25] = 127, + [0][1][2][0][RTW89_CHILE][25] = 42, + [0][1][2][0][RTW89_UKRAINE][25] = 40, + [0][1][2][0][RTW89_MEXICO][25] = 68, + [0][1][2][0][RTW89_CN][25] = 127, + [0][1][2][0][RTW89_QATAR][25] = 40, + [0][1][2][0][RTW89_FCC][27] = 68, + [0][1][2][0][RTW89_ETSI][27] = 46, + [0][1][2][0][RTW89_MKK][27] = 70, + [0][1][2][0][RTW89_IC][27] = 127, + [0][1][2][0][RTW89_KCC][27] = 66, + [0][1][2][0][RTW89_ACMA][27] = 127, + [0][1][2][0][RTW89_CHILE][27] = 42, + [0][1][2][0][RTW89_UKRAINE][27] = 40, + [0][1][2][0][RTW89_MEXICO][27] = 68, + [0][1][2][0][RTW89_CN][27] = 127, + [0][1][2][0][RTW89_QATAR][27] = 40, + [0][1][2][0][RTW89_FCC][29] = 68, + [0][1][2][0][RTW89_ETSI][29] = 46, + [0][1][2][0][RTW89_MKK][29] = 70, + [0][1][2][0][RTW89_IC][29] = 127, + [0][1][2][0][RTW89_KCC][29] = 66, + [0][1][2][0][RTW89_ACMA][29] = 127, + [0][1][2][0][RTW89_CHILE][29] = 42, + [0][1][2][0][RTW89_UKRAINE][29] = 40, + [0][1][2][0][RTW89_MEXICO][29] = 68, + [0][1][2][0][RTW89_CN][29] = 127, + [0][1][2][0][RTW89_QATAR][29] = 40, + [0][1][2][0][RTW89_FCC][31] = 68, + [0][1][2][0][RTW89_ETSI][31] = 46, + [0][1][2][0][RTW89_MKK][31] = 70, + [0][1][2][0][RTW89_IC][31] = 68, + [0][1][2][0][RTW89_KCC][31] = 66, + [0][1][2][0][RTW89_ACMA][31] = 46, + [0][1][2][0][RTW89_CHILE][31] = 42, + [0][1][2][0][RTW89_UKRAINE][31] = 40, + [0][1][2][0][RTW89_MEXICO][31] = 68, + [0][1][2][0][RTW89_CN][31] = 127, + [0][1][2][0][RTW89_QATAR][31] = 40, + [0][1][2][0][RTW89_FCC][33] = 68, + [0][1][2][0][RTW89_ETSI][33] = 46, + [0][1][2][0][RTW89_MKK][33] = 70, + [0][1][2][0][RTW89_IC][33] = 68, + [0][1][2][0][RTW89_KCC][33] = 66, + [0][1][2][0][RTW89_ACMA][33] = 46, + [0][1][2][0][RTW89_CHILE][33] = 42, + [0][1][2][0][RTW89_UKRAINE][33] = 40, + [0][1][2][0][RTW89_MEXICO][33] = 68, + [0][1][2][0][RTW89_CN][33] = 127, + [0][1][2][0][RTW89_QATAR][33] = 40, + [0][1][2][0][RTW89_FCC][35] = 64, + [0][1][2][0][RTW89_ETSI][35] = 46, + [0][1][2][0][RTW89_MKK][35] = 70, + [0][1][2][0][RTW89_IC][35] = 64, + [0][1][2][0][RTW89_KCC][35] = 66, + [0][1][2][0][RTW89_ACMA][35] = 46, + [0][1][2][0][RTW89_CHILE][35] = 42, + [0][1][2][0][RTW89_UKRAINE][35] = 40, + [0][1][2][0][RTW89_MEXICO][35] = 64, + [0][1][2][0][RTW89_CN][35] = 127, + [0][1][2][0][RTW89_QATAR][35] = 40, + [0][1][2][0][RTW89_FCC][37] = 68, + [0][1][2][0][RTW89_ETSI][37] = 127, + [0][1][2][0][RTW89_MKK][37] = 70, + [0][1][2][0][RTW89_IC][37] = 68, + [0][1][2][0][RTW89_KCC][37] = 66, + [0][1][2][0][RTW89_ACMA][37] = 68, + [0][1][2][0][RTW89_CHILE][37] = 42, + [0][1][2][0][RTW89_UKRAINE][37] = 127, + [0][1][2][0][RTW89_MEXICO][37] = 68, + [0][1][2][0][RTW89_CN][37] = 127, + [0][1][2][0][RTW89_QATAR][37] = 127, + [0][1][2][0][RTW89_FCC][38] = 76, + [0][1][2][0][RTW89_ETSI][38] = 16, + [0][1][2][0][RTW89_MKK][38] = 127, + [0][1][2][0][RTW89_IC][38] = 76, + [0][1][2][0][RTW89_KCC][38] = 66, + [0][1][2][0][RTW89_ACMA][38] = 76, + [0][1][2][0][RTW89_CHILE][38] = 42, + [0][1][2][0][RTW89_UKRAINE][38] = 16, + [0][1][2][0][RTW89_MEXICO][38] = 76, + [0][1][2][0][RTW89_CN][38] = 68, + [0][1][2][0][RTW89_QATAR][38] = 16, + [0][1][2][0][RTW89_FCC][40] = 76, + [0][1][2][0][RTW89_ETSI][40] = 16, + [0][1][2][0][RTW89_MKK][40] = 127, + [0][1][2][0][RTW89_IC][40] = 76, + [0][1][2][0][RTW89_KCC][40] = 66, + [0][1][2][0][RTW89_ACMA][40] = 76, + [0][1][2][0][RTW89_CHILE][40] = 42, + [0][1][2][0][RTW89_UKRAINE][40] = 16, + [0][1][2][0][RTW89_MEXICO][40] = 76, + [0][1][2][0][RTW89_CN][40] = 76, + [0][1][2][0][RTW89_QATAR][40] = 16, + [0][1][2][0][RTW89_FCC][42] = 76, + [0][1][2][0][RTW89_ETSI][42] = 16, + [0][1][2][0][RTW89_MKK][42] = 127, + [0][1][2][0][RTW89_IC][42] = 76, + [0][1][2][0][RTW89_KCC][42] = 66, + [0][1][2][0][RTW89_ACMA][42] = 76, + [0][1][2][0][RTW89_CHILE][42] = 42, + [0][1][2][0][RTW89_UKRAINE][42] = 16, + [0][1][2][0][RTW89_MEXICO][42] = 76, + [0][1][2][0][RTW89_CN][42] = 76, + [0][1][2][0][RTW89_QATAR][42] = 16, + [0][1][2][0][RTW89_FCC][44] = 76, + [0][1][2][0][RTW89_ETSI][44] = 16, + [0][1][2][0][RTW89_MKK][44] = 127, + [0][1][2][0][RTW89_IC][44] = 76, + [0][1][2][0][RTW89_KCC][44] = 66, + [0][1][2][0][RTW89_ACMA][44] = 76, + [0][1][2][0][RTW89_CHILE][44] = 42, + [0][1][2][0][RTW89_UKRAINE][44] = 16, + [0][1][2][0][RTW89_MEXICO][44] = 76, + [0][1][2][0][RTW89_CN][44] = 76, + [0][1][2][0][RTW89_QATAR][44] = 16, + [0][1][2][0][RTW89_FCC][46] = 76, + [0][1][2][0][RTW89_ETSI][46] = 16, + [0][1][2][0][RTW89_MKK][46] = 127, + [0][1][2][0][RTW89_IC][46] = 76, + [0][1][2][0][RTW89_KCC][46] = 66, + [0][1][2][0][RTW89_ACMA][46] = 76, + [0][1][2][0][RTW89_CHILE][46] = 42, + [0][1][2][0][RTW89_UKRAINE][46] = 16, + [0][1][2][0][RTW89_MEXICO][46] = 76, + [0][1][2][0][RTW89_CN][46] = 76, + [0][1][2][0][RTW89_QATAR][46] = 16, + [0][1][2][1][RTW89_FCC][0] = 68, + [0][1][2][1][RTW89_ETSI][0] = 34, + [0][1][2][1][RTW89_MKK][0] = 50, + [0][1][2][1][RTW89_IC][0] = 38, + [0][1][2][1][RTW89_KCC][0] = 68, + [0][1][2][1][RTW89_ACMA][0] = 34, + [0][1][2][1][RTW89_CHILE][0] = 6, + [0][1][2][1][RTW89_UKRAINE][0] = 28, + [0][1][2][1][RTW89_MEXICO][0] = 50, + [0][1][2][1][RTW89_CN][0] = 34, + [0][1][2][1][RTW89_QATAR][0] = 34, + [0][1][2][1][RTW89_FCC][2] = 68, + [0][1][2][1][RTW89_ETSI][2] = 34, + [0][1][2][1][RTW89_MKK][2] = 50, + [0][1][2][1][RTW89_IC][2] = 38, + [0][1][2][1][RTW89_KCC][2] = 68, + [0][1][2][1][RTW89_ACMA][2] = 34, + [0][1][2][1][RTW89_CHILE][2] = 6, + [0][1][2][1][RTW89_UKRAINE][2] = 28, + [0][1][2][1][RTW89_MEXICO][2] = 50, + [0][1][2][1][RTW89_CN][2] = 34, + [0][1][2][1][RTW89_QATAR][2] = 34, + [0][1][2][1][RTW89_FCC][4] = 68, + [0][1][2][1][RTW89_ETSI][4] = 34, + [0][1][2][1][RTW89_MKK][4] = 50, + [0][1][2][1][RTW89_IC][4] = 38, + [0][1][2][1][RTW89_KCC][4] = 68, + [0][1][2][1][RTW89_ACMA][4] = 34, + [0][1][2][1][RTW89_CHILE][4] = 6, + [0][1][2][1][RTW89_UKRAINE][4] = 28, + [0][1][2][1][RTW89_MEXICO][4] = 50, + [0][1][2][1][RTW89_CN][4] = 34, + [0][1][2][1][RTW89_QATAR][4] = 34, + [0][1][2][1][RTW89_FCC][6] = 68, + [0][1][2][1][RTW89_ETSI][6] = 34, + [0][1][2][1][RTW89_MKK][6] = 50, + [0][1][2][1][RTW89_IC][6] = 38, + [0][1][2][1][RTW89_KCC][6] = 38, + [0][1][2][1][RTW89_ACMA][6] = 34, + [0][1][2][1][RTW89_CHILE][6] = 6, + [0][1][2][1][RTW89_UKRAINE][6] = 28, + [0][1][2][1][RTW89_MEXICO][6] = 50, + [0][1][2][1][RTW89_CN][6] = 34, + [0][1][2][1][RTW89_QATAR][6] = 34, + [0][1][2][1][RTW89_FCC][8] = 68, + [0][1][2][1][RTW89_ETSI][8] = 34, + [0][1][2][1][RTW89_MKK][8] = 50, + [0][1][2][1][RTW89_IC][8] = 38, + [0][1][2][1][RTW89_KCC][8] = 68, + [0][1][2][1][RTW89_ACMA][8] = 34, + [0][1][2][1][RTW89_CHILE][8] = 30, + [0][1][2][1][RTW89_UKRAINE][8] = 28, + [0][1][2][1][RTW89_MEXICO][8] = 68, + [0][1][2][1][RTW89_CN][8] = 34, + [0][1][2][1][RTW89_QATAR][8] = 34, + [0][1][2][1][RTW89_FCC][10] = 68, + [0][1][2][1][RTW89_ETSI][10] = 34, + [0][1][2][1][RTW89_MKK][10] = 50, + [0][1][2][1][RTW89_IC][10] = 38, + [0][1][2][1][RTW89_KCC][10] = 68, + [0][1][2][1][RTW89_ACMA][10] = 34, + [0][1][2][1][RTW89_CHILE][10] = 30, + [0][1][2][1][RTW89_UKRAINE][10] = 28, + [0][1][2][1][RTW89_MEXICO][10] = 68, + [0][1][2][1][RTW89_CN][10] = 34, + [0][1][2][1][RTW89_QATAR][10] = 34, + [0][1][2][1][RTW89_FCC][12] = 68, + [0][1][2][1][RTW89_ETSI][12] = 34, + [0][1][2][1][RTW89_MKK][12] = 50, + [0][1][2][1][RTW89_IC][12] = 38, + [0][1][2][1][RTW89_KCC][12] = 68, + [0][1][2][1][RTW89_ACMA][12] = 34, + [0][1][2][1][RTW89_CHILE][12] = 30, + [0][1][2][1][RTW89_UKRAINE][12] = 28, + [0][1][2][1][RTW89_MEXICO][12] = 68, + [0][1][2][1][RTW89_CN][12] = 34, + [0][1][2][1][RTW89_QATAR][12] = 34, + [0][1][2][1][RTW89_FCC][14] = 68, + [0][1][2][1][RTW89_ETSI][14] = 34, + [0][1][2][1][RTW89_MKK][14] = 50, + [0][1][2][1][RTW89_IC][14] = 38, + [0][1][2][1][RTW89_KCC][14] = 68, + [0][1][2][1][RTW89_ACMA][14] = 34, + [0][1][2][1][RTW89_CHILE][14] = 30, + [0][1][2][1][RTW89_UKRAINE][14] = 28, + [0][1][2][1][RTW89_MEXICO][14] = 68, + [0][1][2][1][RTW89_CN][14] = 34, + [0][1][2][1][RTW89_QATAR][14] = 34, + [0][1][2][1][RTW89_FCC][15] = 68, + [0][1][2][1][RTW89_ETSI][15] = 34, + [0][1][2][1][RTW89_MKK][15] = 70, + [0][1][2][1][RTW89_IC][15] = 62, + [0][1][2][1][RTW89_KCC][15] = 66, + [0][1][2][1][RTW89_ACMA][15] = 34, + [0][1][2][1][RTW89_CHILE][15] = 30, + [0][1][2][1][RTW89_UKRAINE][15] = 28, + [0][1][2][1][RTW89_MEXICO][15] = 68, + [0][1][2][1][RTW89_CN][15] = 127, + [0][1][2][1][RTW89_QATAR][15] = 28, + [0][1][2][1][RTW89_FCC][17] = 68, + [0][1][2][1][RTW89_ETSI][17] = 34, + [0][1][2][1][RTW89_MKK][17] = 70, + [0][1][2][1][RTW89_IC][17] = 62, + [0][1][2][1][RTW89_KCC][17] = 66, + [0][1][2][1][RTW89_ACMA][17] = 34, + [0][1][2][1][RTW89_CHILE][17] = 30, + [0][1][2][1][RTW89_UKRAINE][17] = 28, + [0][1][2][1][RTW89_MEXICO][17] = 68, + [0][1][2][1][RTW89_CN][17] = 127, + [0][1][2][1][RTW89_QATAR][17] = 28, + [0][1][2][1][RTW89_FCC][19] = 68, + [0][1][2][1][RTW89_ETSI][19] = 34, + [0][1][2][1][RTW89_MKK][19] = 70, + [0][1][2][1][RTW89_IC][19] = 62, + [0][1][2][1][RTW89_KCC][19] = 66, + [0][1][2][1][RTW89_ACMA][19] = 34, + [0][1][2][1][RTW89_CHILE][19] = 30, + [0][1][2][1][RTW89_UKRAINE][19] = 28, + [0][1][2][1][RTW89_MEXICO][19] = 68, + [0][1][2][1][RTW89_CN][19] = 127, + [0][1][2][1][RTW89_QATAR][19] = 28, + [0][1][2][1][RTW89_FCC][21] = 68, + [0][1][2][1][RTW89_ETSI][21] = 34, + [0][1][2][1][RTW89_MKK][21] = 70, + [0][1][2][1][RTW89_IC][21] = 62, + [0][1][2][1][RTW89_KCC][21] = 66, + [0][1][2][1][RTW89_ACMA][21] = 34, + [0][1][2][1][RTW89_CHILE][21] = 30, + [0][1][2][1][RTW89_UKRAINE][21] = 28, + [0][1][2][1][RTW89_MEXICO][21] = 68, + [0][1][2][1][RTW89_CN][21] = 127, + [0][1][2][1][RTW89_QATAR][21] = 28, + [0][1][2][1][RTW89_FCC][23] = 68, + [0][1][2][1][RTW89_ETSI][23] = 34, + [0][1][2][1][RTW89_MKK][23] = 70, + [0][1][2][1][RTW89_IC][23] = 62, + [0][1][2][1][RTW89_KCC][23] = 66, + [0][1][2][1][RTW89_ACMA][23] = 34, + [0][1][2][1][RTW89_CHILE][23] = 30, + [0][1][2][1][RTW89_UKRAINE][23] = 28, + [0][1][2][1][RTW89_MEXICO][23] = 68, + [0][1][2][1][RTW89_CN][23] = 127, + [0][1][2][1][RTW89_QATAR][23] = 28, + [0][1][2][1][RTW89_FCC][25] = 68, + [0][1][2][1][RTW89_ETSI][25] = 34, + [0][1][2][1][RTW89_MKK][25] = 70, + [0][1][2][1][RTW89_IC][25] = 127, + [0][1][2][1][RTW89_KCC][25] = 66, + [0][1][2][1][RTW89_ACMA][25] = 127, + [0][1][2][1][RTW89_CHILE][25] = 30, + [0][1][2][1][RTW89_UKRAINE][25] = 28, + [0][1][2][1][RTW89_MEXICO][25] = 68, + [0][1][2][1][RTW89_CN][25] = 127, + [0][1][2][1][RTW89_QATAR][25] = 28, + [0][1][2][1][RTW89_FCC][27] = 68, + [0][1][2][1][RTW89_ETSI][27] = 34, + [0][1][2][1][RTW89_MKK][27] = 70, + [0][1][2][1][RTW89_IC][27] = 127, + [0][1][2][1][RTW89_KCC][27] = 66, + [0][1][2][1][RTW89_ACMA][27] = 127, + [0][1][2][1][RTW89_CHILE][27] = 30, + [0][1][2][1][RTW89_UKRAINE][27] = 28, + [0][1][2][1][RTW89_MEXICO][27] = 68, + [0][1][2][1][RTW89_CN][27] = 127, + [0][1][2][1][RTW89_QATAR][27] = 28, + [0][1][2][1][RTW89_FCC][29] = 68, + [0][1][2][1][RTW89_ETSI][29] = 34, + [0][1][2][1][RTW89_MKK][29] = 70, + [0][1][2][1][RTW89_IC][29] = 127, + [0][1][2][1][RTW89_KCC][29] = 66, + [0][1][2][1][RTW89_ACMA][29] = 127, + [0][1][2][1][RTW89_CHILE][29] = 30, + [0][1][2][1][RTW89_UKRAINE][29] = 28, + [0][1][2][1][RTW89_MEXICO][29] = 68, + [0][1][2][1][RTW89_CN][29] = 127, + [0][1][2][1][RTW89_QATAR][29] = 28, + [0][1][2][1][RTW89_FCC][31] = 68, + [0][1][2][1][RTW89_ETSI][31] = 34, + [0][1][2][1][RTW89_MKK][31] = 70, + [0][1][2][1][RTW89_IC][31] = 62, + [0][1][2][1][RTW89_KCC][31] = 66, + [0][1][2][1][RTW89_ACMA][31] = 34, + [0][1][2][1][RTW89_CHILE][31] = 30, + [0][1][2][1][RTW89_UKRAINE][31] = 28, + [0][1][2][1][RTW89_MEXICO][31] = 68, + [0][1][2][1][RTW89_CN][31] = 127, + [0][1][2][1][RTW89_QATAR][31] = 28, + [0][1][2][1][RTW89_FCC][33] = 68, + [0][1][2][1][RTW89_ETSI][33] = 34, + [0][1][2][1][RTW89_MKK][33] = 70, + [0][1][2][1][RTW89_IC][33] = 62, + [0][1][2][1][RTW89_KCC][33] = 66, + [0][1][2][1][RTW89_ACMA][33] = 34, + [0][1][2][1][RTW89_CHILE][33] = 30, + [0][1][2][1][RTW89_UKRAINE][33] = 28, + [0][1][2][1][RTW89_MEXICO][33] = 68, + [0][1][2][1][RTW89_CN][33] = 127, + [0][1][2][1][RTW89_QATAR][33] = 28, + [0][1][2][1][RTW89_FCC][35] = 64, + [0][1][2][1][RTW89_ETSI][35] = 34, + [0][1][2][1][RTW89_MKK][35] = 70, + [0][1][2][1][RTW89_IC][35] = 62, + [0][1][2][1][RTW89_KCC][35] = 66, + [0][1][2][1][RTW89_ACMA][35] = 34, + [0][1][2][1][RTW89_CHILE][35] = 30, + [0][1][2][1][RTW89_UKRAINE][35] = 28, + [0][1][2][1][RTW89_MEXICO][35] = 64, + [0][1][2][1][RTW89_CN][35] = 127, + [0][1][2][1][RTW89_QATAR][35] = 28, + [0][1][2][1][RTW89_FCC][37] = 68, + [0][1][2][1][RTW89_ETSI][37] = 127, + [0][1][2][1][RTW89_MKK][37] = 70, + [0][1][2][1][RTW89_IC][37] = 62, + [0][1][2][1][RTW89_KCC][37] = 66, + [0][1][2][1][RTW89_ACMA][37] = 68, + [0][1][2][1][RTW89_CHILE][37] = 30, + [0][1][2][1][RTW89_UKRAINE][37] = 127, + [0][1][2][1][RTW89_MEXICO][37] = 68, + [0][1][2][1][RTW89_CN][37] = 127, + [0][1][2][1][RTW89_QATAR][37] = 127, + [0][1][2][1][RTW89_FCC][38] = 76, + [0][1][2][1][RTW89_ETSI][38] = 4, + [0][1][2][1][RTW89_MKK][38] = 127, + [0][1][2][1][RTW89_IC][38] = 76, + [0][1][2][1][RTW89_KCC][38] = 66, + [0][1][2][1][RTW89_ACMA][38] = 76, + [0][1][2][1][RTW89_CHILE][38] = 30, + [0][1][2][1][RTW89_UKRAINE][38] = 4, + [0][1][2][1][RTW89_MEXICO][38] = 76, + [0][1][2][1][RTW89_CN][38] = 68, + [0][1][2][1][RTW89_QATAR][38] = 4, + [0][1][2][1][RTW89_FCC][40] = 76, + [0][1][2][1][RTW89_ETSI][40] = 4, + [0][1][2][1][RTW89_MKK][40] = 127, + [0][1][2][1][RTW89_IC][40] = 76, + [0][1][2][1][RTW89_KCC][40] = 66, + [0][1][2][1][RTW89_ACMA][40] = 76, + [0][1][2][1][RTW89_CHILE][40] = 30, + [0][1][2][1][RTW89_UKRAINE][40] = 4, + [0][1][2][1][RTW89_MEXICO][40] = 76, + [0][1][2][1][RTW89_CN][40] = 70, + [0][1][2][1][RTW89_QATAR][40] = 4, + [0][1][2][1][RTW89_FCC][42] = 76, + [0][1][2][1][RTW89_ETSI][42] = 4, + [0][1][2][1][RTW89_MKK][42] = 127, + [0][1][2][1][RTW89_IC][42] = 76, + [0][1][2][1][RTW89_KCC][42] = 66, + [0][1][2][1][RTW89_ACMA][42] = 76, + [0][1][2][1][RTW89_CHILE][42] = 30, + [0][1][2][1][RTW89_UKRAINE][42] = 4, + [0][1][2][1][RTW89_MEXICO][42] = 76, + [0][1][2][1][RTW89_CN][42] = 70, + [0][1][2][1][RTW89_QATAR][42] = 4, + [0][1][2][1][RTW89_FCC][44] = 76, + [0][1][2][1][RTW89_ETSI][44] = 4, + [0][1][2][1][RTW89_MKK][44] = 127, + [0][1][2][1][RTW89_IC][44] = 76, + [0][1][2][1][RTW89_KCC][44] = 66, + [0][1][2][1][RTW89_ACMA][44] = 76, + [0][1][2][1][RTW89_CHILE][44] = 30, + [0][1][2][1][RTW89_UKRAINE][44] = 4, + [0][1][2][1][RTW89_MEXICO][44] = 76, + [0][1][2][1][RTW89_CN][44] = 70, + [0][1][2][1][RTW89_QATAR][44] = 4, + [0][1][2][1][RTW89_FCC][46] = 76, + [0][1][2][1][RTW89_ETSI][46] = 4, + [0][1][2][1][RTW89_MKK][46] = 127, + [0][1][2][1][RTW89_IC][46] = 76, + [0][1][2][1][RTW89_KCC][46] = 66, + [0][1][2][1][RTW89_ACMA][46] = 76, + [0][1][2][1][RTW89_CHILE][46] = 30, + [0][1][2][1][RTW89_UKRAINE][46] = 4, + [0][1][2][1][RTW89_MEXICO][46] = 76, + [0][1][2][1][RTW89_CN][46] = 70, + [0][1][2][1][RTW89_QATAR][46] = 4, + [1][0][2][0][RTW89_FCC][1] = 68, + [1][0][2][0][RTW89_ETSI][1] = 64, + [1][0][2][0][RTW89_MKK][1] = 62, + [1][0][2][0][RTW89_IC][1] = 64, + [1][0][2][0][RTW89_KCC][1] = 72, + [1][0][2][0][RTW89_ACMA][1] = 64, + [1][0][2][0][RTW89_CHILE][1] = 30, + [1][0][2][0][RTW89_UKRAINE][1] = 52, + [1][0][2][0][RTW89_MEXICO][1] = 62, + [1][0][2][0][RTW89_CN][1] = 64, + [1][0][2][0][RTW89_QATAR][1] = 64, + [1][0][2][0][RTW89_FCC][5] = 72, + [1][0][2][0][RTW89_ETSI][5] = 64, + [1][0][2][0][RTW89_MKK][5] = 62, + [1][0][2][0][RTW89_IC][5] = 64, + [1][0][2][0][RTW89_KCC][5] = 72, + [1][0][2][0][RTW89_ACMA][5] = 64, + [1][0][2][0][RTW89_CHILE][5] = 30, + [1][0][2][0][RTW89_UKRAINE][5] = 52, + [1][0][2][0][RTW89_MEXICO][5] = 62, + [1][0][2][0][RTW89_CN][5] = 64, + [1][0][2][0][RTW89_QATAR][5] = 64, + [1][0][2][0][RTW89_FCC][9] = 72, + [1][0][2][0][RTW89_ETSI][9] = 64, + [1][0][2][0][RTW89_MKK][9] = 62, + [1][0][2][0][RTW89_IC][9] = 64, + [1][0][2][0][RTW89_KCC][9] = 72, + [1][0][2][0][RTW89_ACMA][9] = 64, + [1][0][2][0][RTW89_CHILE][9] = 54, + [1][0][2][0][RTW89_UKRAINE][9] = 52, + [1][0][2][0][RTW89_MEXICO][9] = 72, + [1][0][2][0][RTW89_CN][9] = 64, + [1][0][2][0][RTW89_QATAR][9] = 64, + [1][0][2][0][RTW89_FCC][13] = 66, + [1][0][2][0][RTW89_ETSI][13] = 64, + [1][0][2][0][RTW89_MKK][13] = 62, + [1][0][2][0][RTW89_IC][13] = 64, + [1][0][2][0][RTW89_KCC][13] = 68, + [1][0][2][0][RTW89_ACMA][13] = 64, + [1][0][2][0][RTW89_CHILE][13] = 54, + [1][0][2][0][RTW89_UKRAINE][13] = 52, + [1][0][2][0][RTW89_MEXICO][13] = 66, + [1][0][2][0][RTW89_CN][13] = 64, + [1][0][2][0][RTW89_QATAR][13] = 64, + [1][0][2][0][RTW89_FCC][16] = 62, + [1][0][2][0][RTW89_ETSI][16] = 64, + [1][0][2][0][RTW89_MKK][16] = 72, + [1][0][2][0][RTW89_IC][16] = 62, + [1][0][2][0][RTW89_KCC][16] = 72, + [1][0][2][0][RTW89_ACMA][16] = 64, + [1][0][2][0][RTW89_CHILE][16] = 54, + [1][0][2][0][RTW89_UKRAINE][16] = 52, + [1][0][2][0][RTW89_MEXICO][16] = 62, + [1][0][2][0][RTW89_CN][16] = 127, + [1][0][2][0][RTW89_QATAR][16] = 52, + [1][0][2][0][RTW89_FCC][20] = 72, + [1][0][2][0][RTW89_ETSI][20] = 64, + [1][0][2][0][RTW89_MKK][20] = 72, + [1][0][2][0][RTW89_IC][20] = 72, + [1][0][2][0][RTW89_KCC][20] = 72, + [1][0][2][0][RTW89_ACMA][20] = 64, + [1][0][2][0][RTW89_CHILE][20] = 54, + [1][0][2][0][RTW89_UKRAINE][20] = 52, + [1][0][2][0][RTW89_MEXICO][20] = 72, + [1][0][2][0][RTW89_CN][20] = 127, + [1][0][2][0][RTW89_QATAR][20] = 52, + [1][0][2][0][RTW89_FCC][24] = 72, + [1][0][2][0][RTW89_ETSI][24] = 64, + [1][0][2][0][RTW89_MKK][24] = 72, + [1][0][2][0][RTW89_IC][24] = 127, + [1][0][2][0][RTW89_KCC][24] = 72, + [1][0][2][0][RTW89_ACMA][24] = 127, + [1][0][2][0][RTW89_CHILE][24] = 54, + [1][0][2][0][RTW89_UKRAINE][24] = 52, + [1][0][2][0][RTW89_MEXICO][24] = 72, + [1][0][2][0][RTW89_CN][24] = 127, + [1][0][2][0][RTW89_QATAR][24] = 52, + [1][0][2][0][RTW89_FCC][28] = 72, + [1][0][2][0][RTW89_ETSI][28] = 64, + [1][0][2][0][RTW89_MKK][28] = 72, + [1][0][2][0][RTW89_IC][28] = 127, + [1][0][2][0][RTW89_KCC][28] = 72, + [1][0][2][0][RTW89_ACMA][28] = 127, + [1][0][2][0][RTW89_CHILE][28] = 54, + [1][0][2][0][RTW89_UKRAINE][28] = 52, + [1][0][2][0][RTW89_MEXICO][28] = 72, + [1][0][2][0][RTW89_CN][28] = 127, + [1][0][2][0][RTW89_QATAR][28] = 52, + [1][0][2][0][RTW89_FCC][32] = 72, + [1][0][2][0][RTW89_ETSI][32] = 64, + [1][0][2][0][RTW89_MKK][32] = 72, + [1][0][2][0][RTW89_IC][32] = 72, + [1][0][2][0][RTW89_KCC][32] = 72, + [1][0][2][0][RTW89_ACMA][32] = 64, + [1][0][2][0][RTW89_CHILE][32] = 54, + [1][0][2][0][RTW89_UKRAINE][32] = 52, + [1][0][2][0][RTW89_MEXICO][32] = 72, + [1][0][2][0][RTW89_CN][32] = 127, + [1][0][2][0][RTW89_QATAR][32] = 52, + [1][0][2][0][RTW89_FCC][36] = 72, + [1][0][2][0][RTW89_ETSI][36] = 127, + [1][0][2][0][RTW89_MKK][36] = 72, + [1][0][2][0][RTW89_IC][36] = 72, + [1][0][2][0][RTW89_KCC][36] = 72, + [1][0][2][0][RTW89_ACMA][36] = 72, + [1][0][2][0][RTW89_CHILE][36] = 54, + [1][0][2][0][RTW89_UKRAINE][36] = 127, + [1][0][2][0][RTW89_MEXICO][36] = 72, + [1][0][2][0][RTW89_CN][36] = 127, + [1][0][2][0][RTW89_QATAR][36] = 127, + [1][0][2][0][RTW89_FCC][39] = 72, + [1][0][2][0][RTW89_ETSI][39] = 28, + [1][0][2][0][RTW89_MKK][39] = 127, + [1][0][2][0][RTW89_IC][39] = 72, + [1][0][2][0][RTW89_KCC][39] = 72, + [1][0][2][0][RTW89_ACMA][39] = 72, + [1][0][2][0][RTW89_CHILE][39] = 54, + [1][0][2][0][RTW89_UKRAINE][39] = 28, + [1][0][2][0][RTW89_MEXICO][39] = 72, + [1][0][2][0][RTW89_CN][39] = 68, + [1][0][2][0][RTW89_QATAR][39] = 28, + [1][0][2][0][RTW89_FCC][43] = 72, + [1][0][2][0][RTW89_ETSI][43] = 28, + [1][0][2][0][RTW89_MKK][43] = 127, + [1][0][2][0][RTW89_IC][43] = 72, + [1][0][2][0][RTW89_KCC][43] = 72, + [1][0][2][0][RTW89_ACMA][43] = 72, + [1][0][2][0][RTW89_CHILE][43] = 54, + [1][0][2][0][RTW89_UKRAINE][43] = 28, + [1][0][2][0][RTW89_MEXICO][43] = 72, + [1][0][2][0][RTW89_CN][43] = 72, + [1][0][2][0][RTW89_QATAR][43] = 28, + [1][1][2][0][RTW89_FCC][1] = 58, + [1][1][2][0][RTW89_ETSI][1] = 52, + [1][1][2][0][RTW89_MKK][1] = 50, + [1][1][2][0][RTW89_IC][1] = 52, + [1][1][2][0][RTW89_KCC][1] = 66, + [1][1][2][0][RTW89_ACMA][1] = 52, + [1][1][2][0][RTW89_CHILE][1] = 18, + [1][1][2][0][RTW89_UKRAINE][1] = 40, + [1][1][2][0][RTW89_MEXICO][1] = 50, + [1][1][2][0][RTW89_CN][1] = 52, + [1][1][2][0][RTW89_QATAR][1] = 52, + [1][1][2][0][RTW89_FCC][5] = 72, + [1][1][2][0][RTW89_ETSI][5] = 52, + [1][1][2][0][RTW89_MKK][5] = 50, + [1][1][2][0][RTW89_IC][5] = 52, + [1][1][2][0][RTW89_KCC][5] = 50, + [1][1][2][0][RTW89_ACMA][5] = 52, + [1][1][2][0][RTW89_CHILE][5] = 18, + [1][1][2][0][RTW89_UKRAINE][5] = 40, + [1][1][2][0][RTW89_MEXICO][5] = 50, + [1][1][2][0][RTW89_CN][5] = 52, + [1][1][2][0][RTW89_QATAR][5] = 52, + [1][1][2][0][RTW89_FCC][9] = 72, + [1][1][2][0][RTW89_ETSI][9] = 52, + [1][1][2][0][RTW89_MKK][9] = 50, + [1][1][2][0][RTW89_IC][9] = 52, + [1][1][2][0][RTW89_KCC][9] = 66, + [1][1][2][0][RTW89_ACMA][9] = 52, + [1][1][2][0][RTW89_CHILE][9] = 42, + [1][1][2][0][RTW89_UKRAINE][9] = 40, + [1][1][2][0][RTW89_MEXICO][9] = 72, + [1][1][2][0][RTW89_CN][9] = 52, + [1][1][2][0][RTW89_QATAR][9] = 52, + [1][1][2][0][RTW89_FCC][13] = 58, + [1][1][2][0][RTW89_ETSI][13] = 52, + [1][1][2][0][RTW89_MKK][13] = 50, + [1][1][2][0][RTW89_IC][13] = 52, + [1][1][2][0][RTW89_KCC][13] = 66, + [1][1][2][0][RTW89_ACMA][13] = 52, + [1][1][2][0][RTW89_CHILE][13] = 42, + [1][1][2][0][RTW89_UKRAINE][13] = 40, + [1][1][2][0][RTW89_MEXICO][13] = 58, + [1][1][2][0][RTW89_CN][13] = 52, + [1][1][2][0][RTW89_QATAR][13] = 52, + [1][1][2][0][RTW89_FCC][16] = 56, + [1][1][2][0][RTW89_ETSI][16] = 52, + [1][1][2][0][RTW89_MKK][16] = 72, + [1][1][2][0][RTW89_IC][16] = 56, + [1][1][2][0][RTW89_KCC][16] = 64, + [1][1][2][0][RTW89_ACMA][16] = 52, + [1][1][2][0][RTW89_CHILE][16] = 42, + [1][1][2][0][RTW89_UKRAINE][16] = 40, + [1][1][2][0][RTW89_MEXICO][16] = 56, + [1][1][2][0][RTW89_CN][16] = 127, + [1][1][2][0][RTW89_QATAR][16] = 40, + [1][1][2][0][RTW89_FCC][20] = 72, + [1][1][2][0][RTW89_ETSI][20] = 52, + [1][1][2][0][RTW89_MKK][20] = 72, + [1][1][2][0][RTW89_IC][20] = 72, + [1][1][2][0][RTW89_KCC][20] = 66, + [1][1][2][0][RTW89_ACMA][20] = 52, + [1][1][2][0][RTW89_CHILE][20] = 42, + [1][1][2][0][RTW89_UKRAINE][20] = 40, + [1][1][2][0][RTW89_MEXICO][20] = 72, + [1][1][2][0][RTW89_CN][20] = 127, + [1][1][2][0][RTW89_QATAR][20] = 40, + [1][1][2][0][RTW89_FCC][24] = 72, + [1][1][2][0][RTW89_ETSI][24] = 52, + [1][1][2][0][RTW89_MKK][24] = 72, + [1][1][2][0][RTW89_IC][24] = 127, + [1][1][2][0][RTW89_KCC][24] = 66, + [1][1][2][0][RTW89_ACMA][24] = 127, + [1][1][2][0][RTW89_CHILE][24] = 42, + [1][1][2][0][RTW89_UKRAINE][24] = 40, + [1][1][2][0][RTW89_MEXICO][24] = 72, + [1][1][2][0][RTW89_CN][24] = 127, + [1][1][2][0][RTW89_QATAR][24] = 40, + [1][1][2][0][RTW89_FCC][28] = 72, + [1][1][2][0][RTW89_ETSI][28] = 52, + [1][1][2][0][RTW89_MKK][28] = 72, + [1][1][2][0][RTW89_IC][28] = 127, + [1][1][2][0][RTW89_KCC][28] = 66, + [1][1][2][0][RTW89_ACMA][28] = 127, + [1][1][2][0][RTW89_CHILE][28] = 42, + [1][1][2][0][RTW89_UKRAINE][28] = 40, + [1][1][2][0][RTW89_MEXICO][28] = 72, + [1][1][2][0][RTW89_CN][28] = 127, + [1][1][2][0][RTW89_QATAR][28] = 40, + [1][1][2][0][RTW89_FCC][32] = 68, + [1][1][2][0][RTW89_ETSI][32] = 52, + [1][1][2][0][RTW89_MKK][32] = 72, + [1][1][2][0][RTW89_IC][32] = 68, + [1][1][2][0][RTW89_KCC][32] = 66, + [1][1][2][0][RTW89_ACMA][32] = 52, + [1][1][2][0][RTW89_CHILE][32] = 42, + [1][1][2][0][RTW89_UKRAINE][32] = 40, + [1][1][2][0][RTW89_MEXICO][32] = 68, + [1][1][2][0][RTW89_CN][32] = 127, + [1][1][2][0][RTW89_QATAR][32] = 40, + [1][1][2][0][RTW89_FCC][36] = 72, + [1][1][2][0][RTW89_ETSI][36] = 127, + [1][1][2][0][RTW89_MKK][36] = 72, + [1][1][2][0][RTW89_IC][36] = 72, + [1][1][2][0][RTW89_KCC][36] = 66, + [1][1][2][0][RTW89_ACMA][36] = 72, + [1][1][2][0][RTW89_CHILE][36] = 42, + [1][1][2][0][RTW89_UKRAINE][36] = 127, + [1][1][2][0][RTW89_MEXICO][36] = 72, + [1][1][2][0][RTW89_CN][36] = 127, + [1][1][2][0][RTW89_QATAR][36] = 127, + [1][1][2][0][RTW89_FCC][39] = 72, + [1][1][2][0][RTW89_ETSI][39] = 16, + [1][1][2][0][RTW89_MKK][39] = 127, + [1][1][2][0][RTW89_IC][39] = 72, + [1][1][2][0][RTW89_KCC][39] = 66, + [1][1][2][0][RTW89_ACMA][39] = 72, + [1][1][2][0][RTW89_CHILE][39] = 42, + [1][1][2][0][RTW89_UKRAINE][39] = 16, + [1][1][2][0][RTW89_MEXICO][39] = 72, + [1][1][2][0][RTW89_CN][39] = 68, + [1][1][2][0][RTW89_QATAR][39] = 16, + [1][1][2][0][RTW89_FCC][43] = 72, + [1][1][2][0][RTW89_ETSI][43] = 16, + [1][1][2][0][RTW89_MKK][43] = 127, + [1][1][2][0][RTW89_IC][43] = 72, + [1][1][2][0][RTW89_KCC][43] = 66, + [1][1][2][0][RTW89_ACMA][43] = 72, + [1][1][2][0][RTW89_CHILE][43] = 42, + [1][1][2][0][RTW89_UKRAINE][43] = 16, + [1][1][2][0][RTW89_MEXICO][43] = 72, + [1][1][2][0][RTW89_CN][43] = 72, + [1][1][2][0][RTW89_QATAR][43] = 16, + [1][1][2][1][RTW89_FCC][1] = 58, + [1][1][2][1][RTW89_ETSI][1] = 40, + [1][1][2][1][RTW89_MKK][1] = 50, + [1][1][2][1][RTW89_IC][1] = 40, + [1][1][2][1][RTW89_KCC][1] = 66, + [1][1][2][1][RTW89_ACMA][1] = 40, + [1][1][2][1][RTW89_CHILE][1] = 6, + [1][1][2][1][RTW89_UKRAINE][1] = 28, + [1][1][2][1][RTW89_MEXICO][1] = 50, + [1][1][2][1][RTW89_CN][1] = 40, + [1][1][2][1][RTW89_QATAR][1] = 40, + [1][1][2][1][RTW89_FCC][5] = 68, + [1][1][2][1][RTW89_ETSI][5] = 40, + [1][1][2][1][RTW89_MKK][5] = 50, + [1][1][2][1][RTW89_IC][5] = 40, + [1][1][2][1][RTW89_KCC][5] = 50, + [1][1][2][1][RTW89_ACMA][5] = 40, + [1][1][2][1][RTW89_CHILE][5] = 6, + [1][1][2][1][RTW89_UKRAINE][5] = 28, + [1][1][2][1][RTW89_MEXICO][5] = 50, + [1][1][2][1][RTW89_CN][5] = 40, + [1][1][2][1][RTW89_QATAR][5] = 40, + [1][1][2][1][RTW89_FCC][9] = 68, + [1][1][2][1][RTW89_ETSI][9] = 40, + [1][1][2][1][RTW89_MKK][9] = 50, + [1][1][2][1][RTW89_IC][9] = 40, + [1][1][2][1][RTW89_KCC][9] = 66, + [1][1][2][1][RTW89_ACMA][9] = 40, + [1][1][2][1][RTW89_CHILE][9] = 30, + [1][1][2][1][RTW89_UKRAINE][9] = 28, + [1][1][2][1][RTW89_MEXICO][9] = 68, + [1][1][2][1][RTW89_CN][9] = 40, + [1][1][2][1][RTW89_QATAR][9] = 40, + [1][1][2][1][RTW89_FCC][13] = 58, + [1][1][2][1][RTW89_ETSI][13] = 40, + [1][1][2][1][RTW89_MKK][13] = 50, + [1][1][2][1][RTW89_IC][13] = 40, + [1][1][2][1][RTW89_KCC][13] = 66, + [1][1][2][1][RTW89_ACMA][13] = 40, + [1][1][2][1][RTW89_CHILE][13] = 30, + [1][1][2][1][RTW89_UKRAINE][13] = 28, + [1][1][2][1][RTW89_MEXICO][13] = 58, + [1][1][2][1][RTW89_CN][13] = 40, + [1][1][2][1][RTW89_QATAR][13] = 40, + [1][1][2][1][RTW89_FCC][16] = 56, + [1][1][2][1][RTW89_ETSI][16] = 40, + [1][1][2][1][RTW89_MKK][16] = 72, + [1][1][2][1][RTW89_IC][16] = 56, + [1][1][2][1][RTW89_KCC][16] = 64, + [1][1][2][1][RTW89_ACMA][16] = 40, + [1][1][2][1][RTW89_CHILE][16] = 30, + [1][1][2][1][RTW89_UKRAINE][16] = 28, + [1][1][2][1][RTW89_MEXICO][16] = 56, + [1][1][2][1][RTW89_CN][16] = 127, + [1][1][2][1][RTW89_QATAR][16] = 28, + [1][1][2][1][RTW89_FCC][20] = 68, + [1][1][2][1][RTW89_ETSI][20] = 40, + [1][1][2][1][RTW89_MKK][20] = 72, + [1][1][2][1][RTW89_IC][20] = 68, + [1][1][2][1][RTW89_KCC][20] = 66, + [1][1][2][1][RTW89_ACMA][20] = 40, + [1][1][2][1][RTW89_CHILE][20] = 30, + [1][1][2][1][RTW89_UKRAINE][20] = 28, + [1][1][2][1][RTW89_MEXICO][20] = 68, + [1][1][2][1][RTW89_CN][20] = 127, + [1][1][2][1][RTW89_QATAR][20] = 28, + [1][1][2][1][RTW89_FCC][24] = 68, + [1][1][2][1][RTW89_ETSI][24] = 40, + [1][1][2][1][RTW89_MKK][24] = 72, + [1][1][2][1][RTW89_IC][24] = 127, + [1][1][2][1][RTW89_KCC][24] = 66, + [1][1][2][1][RTW89_ACMA][24] = 127, + [1][1][2][1][RTW89_CHILE][24] = 30, + [1][1][2][1][RTW89_UKRAINE][24] = 28, + [1][1][2][1][RTW89_MEXICO][24] = 68, + [1][1][2][1][RTW89_CN][24] = 127, + [1][1][2][1][RTW89_QATAR][24] = 28, + [1][1][2][1][RTW89_FCC][28] = 68, + [1][1][2][1][RTW89_ETSI][28] = 40, + [1][1][2][1][RTW89_MKK][28] = 72, + [1][1][2][1][RTW89_IC][28] = 127, + [1][1][2][1][RTW89_KCC][28] = 66, + [1][1][2][1][RTW89_ACMA][28] = 127, + [1][1][2][1][RTW89_CHILE][28] = 30, + [1][1][2][1][RTW89_UKRAINE][28] = 28, + [1][1][2][1][RTW89_MEXICO][28] = 68, + [1][1][2][1][RTW89_CN][28] = 127, + [1][1][2][1][RTW89_QATAR][28] = 28, + [1][1][2][1][RTW89_FCC][32] = 68, + [1][1][2][1][RTW89_ETSI][32] = 40, + [1][1][2][1][RTW89_MKK][32] = 72, + [1][1][2][1][RTW89_IC][32] = 68, + [1][1][2][1][RTW89_KCC][32] = 66, + [1][1][2][1][RTW89_ACMA][32] = 40, + [1][1][2][1][RTW89_CHILE][32] = 30, + [1][1][2][1][RTW89_UKRAINE][32] = 28, + [1][1][2][1][RTW89_MEXICO][32] = 68, + [1][1][2][1][RTW89_CN][32] = 127, + [1][1][2][1][RTW89_QATAR][32] = 28, + [1][1][2][1][RTW89_FCC][36] = 68, + [1][1][2][1][RTW89_ETSI][36] = 127, + [1][1][2][1][RTW89_MKK][36] = 72, + [1][1][2][1][RTW89_IC][36] = 68, + [1][1][2][1][RTW89_KCC][36] = 66, + [1][1][2][1][RTW89_ACMA][36] = 68, + [1][1][2][1][RTW89_CHILE][36] = 30, + [1][1][2][1][RTW89_UKRAINE][36] = 127, + [1][1][2][1][RTW89_MEXICO][36] = 68, + [1][1][2][1][RTW89_CN][36] = 127, + [1][1][2][1][RTW89_QATAR][36] = 127, + [1][1][2][1][RTW89_FCC][39] = 72, + [1][1][2][1][RTW89_ETSI][39] = 4, + [1][1][2][1][RTW89_MKK][39] = 127, + [1][1][2][1][RTW89_IC][39] = 72, + [1][1][2][1][RTW89_KCC][39] = 66, + [1][1][2][1][RTW89_ACMA][39] = 72, + [1][1][2][1][RTW89_CHILE][39] = 30, + [1][1][2][1][RTW89_UKRAINE][39] = 4, + [1][1][2][1][RTW89_MEXICO][39] = 72, + [1][1][2][1][RTW89_CN][39] = 62, + [1][1][2][1][RTW89_QATAR][39] = 4, + [1][1][2][1][RTW89_FCC][43] = 72, + [1][1][2][1][RTW89_ETSI][43] = 4, + [1][1][2][1][RTW89_MKK][43] = 127, + [1][1][2][1][RTW89_IC][43] = 72, + [1][1][2][1][RTW89_KCC][43] = 66, + [1][1][2][1][RTW89_ACMA][43] = 72, + [1][1][2][1][RTW89_CHILE][43] = 30, + [1][1][2][1][RTW89_UKRAINE][43] = 4, + [1][1][2][1][RTW89_MEXICO][43] = 72, + [1][1][2][1][RTW89_CN][43] = 72, + [1][1][2][1][RTW89_QATAR][43] = 4, + [2][0][2][0][RTW89_FCC][3] = 64, + [2][0][2][0][RTW89_ETSI][3] = 64, + [2][0][2][0][RTW89_MKK][3] = 64, + [2][0][2][0][RTW89_IC][3] = 62, + [2][0][2][0][RTW89_KCC][3] = 72, + [2][0][2][0][RTW89_ACMA][3] = 64, + [2][0][2][0][RTW89_CHILE][3] = 30, + [2][0][2][0][RTW89_UKRAINE][3] = 52, + [2][0][2][0][RTW89_MEXICO][3] = 62, + [2][0][2][0][RTW89_CN][3] = 64, + [2][0][2][0][RTW89_QATAR][3] = 64, + [2][0][2][0][RTW89_FCC][11] = 64, + [2][0][2][0][RTW89_ETSI][11] = 64, + [2][0][2][0][RTW89_MKK][11] = 64, + [2][0][2][0][RTW89_IC][11] = 62, + [2][0][2][0][RTW89_KCC][11] = 72, + [2][0][2][0][RTW89_ACMA][11] = 64, + [2][0][2][0][RTW89_CHILE][11] = 54, + [2][0][2][0][RTW89_UKRAINE][11] = 52, + [2][0][2][0][RTW89_MEXICO][11] = 64, + [2][0][2][0][RTW89_CN][11] = 64, + [2][0][2][0][RTW89_QATAR][11] = 64, + [2][0][2][0][RTW89_FCC][18] = 62, + [2][0][2][0][RTW89_ETSI][18] = 64, + [2][0][2][0][RTW89_MKK][18] = 72, + [2][0][2][0][RTW89_IC][18] = 66, + [2][0][2][0][RTW89_KCC][18] = 70, + [2][0][2][0][RTW89_ACMA][18] = 64, + [2][0][2][0][RTW89_CHILE][18] = 54, + [2][0][2][0][RTW89_UKRAINE][18] = 52, + [2][0][2][0][RTW89_MEXICO][18] = 62, + [2][0][2][0][RTW89_CN][18] = 127, + [2][0][2][0][RTW89_QATAR][18] = 52, + [2][0][2][0][RTW89_FCC][26] = 72, + [2][0][2][0][RTW89_ETSI][26] = 64, + [2][0][2][0][RTW89_MKK][26] = 72, + [2][0][2][0][RTW89_IC][26] = 127, + [2][0][2][0][RTW89_KCC][26] = 72, + [2][0][2][0][RTW89_ACMA][26] = 127, + [2][0][2][0][RTW89_CHILE][26] = 54, + [2][0][2][0][RTW89_UKRAINE][26] = 52, + [2][0][2][0][RTW89_MEXICO][26] = 72, + [2][0][2][0][RTW89_CN][26] = 127, + [2][0][2][0][RTW89_QATAR][26] = 52, + [2][0][2][0][RTW89_FCC][34] = 72, + [2][0][2][0][RTW89_ETSI][34] = 127, + [2][0][2][0][RTW89_MKK][34] = 72, + [2][0][2][0][RTW89_IC][34] = 72, + [2][0][2][0][RTW89_KCC][34] = 72, + [2][0][2][0][RTW89_ACMA][34] = 72, + [2][0][2][0][RTW89_CHILE][34] = 54, + [2][0][2][0][RTW89_UKRAINE][34] = 127, + [2][0][2][0][RTW89_MEXICO][34] = 72, + [2][0][2][0][RTW89_CN][34] = 127, + [2][0][2][0][RTW89_QATAR][34] = 127, + [2][0][2][0][RTW89_FCC][41] = 72, + [2][0][2][0][RTW89_ETSI][41] = 28, + [2][0][2][0][RTW89_MKK][41] = 127, + [2][0][2][0][RTW89_IC][41] = 72, + [2][0][2][0][RTW89_KCC][41] = 68, + [2][0][2][0][RTW89_ACMA][41] = 72, + [2][0][2][0][RTW89_CHILE][41] = 54, + [2][0][2][0][RTW89_UKRAINE][41] = 28, + [2][0][2][0][RTW89_MEXICO][41] = 72, + [2][0][2][0][RTW89_CN][41] = 68, + [2][0][2][0][RTW89_QATAR][41] = 28, + [2][1][2][0][RTW89_FCC][3] = 56, + [2][1][2][0][RTW89_ETSI][3] = 52, + [2][1][2][0][RTW89_MKK][3] = 52, + [2][1][2][0][RTW89_IC][3] = 52, + [2][1][2][0][RTW89_KCC][3] = 66, + [2][1][2][0][RTW89_ACMA][3] = 52, + [2][1][2][0][RTW89_CHILE][3] = 18, + [2][1][2][0][RTW89_UKRAINE][3] = 40, + [2][1][2][0][RTW89_MEXICO][3] = 50, + [2][1][2][0][RTW89_CN][3] = 52, + [2][1][2][0][RTW89_QATAR][3] = 52, + [2][1][2][0][RTW89_FCC][11] = 56, + [2][1][2][0][RTW89_ETSI][11] = 52, + [2][1][2][0][RTW89_MKK][11] = 52, + [2][1][2][0][RTW89_IC][11] = 52, + [2][1][2][0][RTW89_KCC][11] = 64, + [2][1][2][0][RTW89_ACMA][11] = 52, + [2][1][2][0][RTW89_CHILE][11] = 42, + [2][1][2][0][RTW89_UKRAINE][11] = 40, + [2][1][2][0][RTW89_MEXICO][11] = 56, + [2][1][2][0][RTW89_CN][11] = 52, + [2][1][2][0][RTW89_QATAR][11] = 52, + [2][1][2][0][RTW89_FCC][18] = 56, + [2][1][2][0][RTW89_ETSI][18] = 52, + [2][1][2][0][RTW89_MKK][18] = 72, + [2][1][2][0][RTW89_IC][18] = 56, + [2][1][2][0][RTW89_KCC][18] = 58, + [2][1][2][0][RTW89_ACMA][18] = 52, + [2][1][2][0][RTW89_CHILE][18] = 42, + [2][1][2][0][RTW89_UKRAINE][18] = 40, + [2][1][2][0][RTW89_MEXICO][18] = 56, + [2][1][2][0][RTW89_CN][18] = 127, + [2][1][2][0][RTW89_QATAR][18] = 40, + [2][1][2][0][RTW89_FCC][26] = 72, + [2][1][2][0][RTW89_ETSI][26] = 52, + [2][1][2][0][RTW89_MKK][26] = 72, + [2][1][2][0][RTW89_IC][26] = 127, + [2][1][2][0][RTW89_KCC][26] = 64, + [2][1][2][0][RTW89_ACMA][26] = 127, + [2][1][2][0][RTW89_CHILE][26] = 42, + [2][1][2][0][RTW89_UKRAINE][26] = 40, + [2][1][2][0][RTW89_MEXICO][26] = 72, + [2][1][2][0][RTW89_CN][26] = 127, + [2][1][2][0][RTW89_QATAR][26] = 40, + [2][1][2][0][RTW89_FCC][34] = 72, + [2][1][2][0][RTW89_ETSI][34] = 127, + [2][1][2][0][RTW89_MKK][34] = 72, + [2][1][2][0][RTW89_IC][34] = 72, + [2][1][2][0][RTW89_KCC][34] = 64, + [2][1][2][0][RTW89_ACMA][34] = 72, + [2][1][2][0][RTW89_CHILE][34] = 42, + [2][1][2][0][RTW89_UKRAINE][34] = 127, + [2][1][2][0][RTW89_MEXICO][34] = 72, + [2][1][2][0][RTW89_CN][34] = 127, + [2][1][2][0][RTW89_QATAR][34] = 127, + [2][1][2][0][RTW89_FCC][41] = 72, + [2][1][2][0][RTW89_ETSI][41] = 16, + [2][1][2][0][RTW89_MKK][41] = 127, + [2][1][2][0][RTW89_IC][41] = 72, + [2][1][2][0][RTW89_KCC][41] = 58, + [2][1][2][0][RTW89_ACMA][41] = 72, + [2][1][2][0][RTW89_CHILE][41] = 42, + [2][1][2][0][RTW89_UKRAINE][41] = 16, + [2][1][2][0][RTW89_MEXICO][41] = 72, + [2][1][2][0][RTW89_CN][41] = 68, + [2][1][2][0][RTW89_QATAR][41] = 16, + [2][1][2][1][RTW89_FCC][3] = 56, + [2][1][2][1][RTW89_ETSI][3] = 40, + [2][1][2][1][RTW89_MKK][3] = 52, + [2][1][2][1][RTW89_IC][3] = 40, + [2][1][2][1][RTW89_KCC][3] = 66, + [2][1][2][1][RTW89_ACMA][3] = 40, + [2][1][2][1][RTW89_CHILE][3] = 6, + [2][1][2][1][RTW89_UKRAINE][3] = 28, + [2][1][2][1][RTW89_MEXICO][3] = 50, + [2][1][2][1][RTW89_CN][3] = 40, + [2][1][2][1][RTW89_QATAR][3] = 40, + [2][1][2][1][RTW89_FCC][11] = 56, + [2][1][2][1][RTW89_ETSI][11] = 40, + [2][1][2][1][RTW89_MKK][11] = 52, + [2][1][2][1][RTW89_IC][11] = 40, + [2][1][2][1][RTW89_KCC][11] = 64, + [2][1][2][1][RTW89_ACMA][11] = 40, + [2][1][2][1][RTW89_CHILE][11] = 30, + [2][1][2][1][RTW89_UKRAINE][11] = 28, + [2][1][2][1][RTW89_MEXICO][11] = 56, + [2][1][2][1][RTW89_CN][11] = 40, + [2][1][2][1][RTW89_QATAR][11] = 40, + [2][1][2][1][RTW89_FCC][18] = 56, + [2][1][2][1][RTW89_ETSI][18] = 40, + [2][1][2][1][RTW89_MKK][18] = 72, + [2][1][2][1][RTW89_IC][18] = 56, + [2][1][2][1][RTW89_KCC][18] = 58, + [2][1][2][1][RTW89_ACMA][18] = 40, + [2][1][2][1][RTW89_CHILE][18] = 30, + [2][1][2][1][RTW89_UKRAINE][18] = 28, + [2][1][2][1][RTW89_MEXICO][18] = 56, + [2][1][2][1][RTW89_CN][18] = 127, + [2][1][2][1][RTW89_QATAR][18] = 28, + [2][1][2][1][RTW89_FCC][26] = 68, + [2][1][2][1][RTW89_ETSI][26] = 40, + [2][1][2][1][RTW89_MKK][26] = 72, + [2][1][2][1][RTW89_IC][26] = 127, + [2][1][2][1][RTW89_KCC][26] = 64, + [2][1][2][1][RTW89_ACMA][26] = 127, + [2][1][2][1][RTW89_CHILE][26] = 30, + [2][1][2][1][RTW89_UKRAINE][26] = 28, + [2][1][2][1][RTW89_MEXICO][26] = 68, + [2][1][2][1][RTW89_CN][26] = 127, + [2][1][2][1][RTW89_QATAR][26] = 28, + [2][1][2][1][RTW89_FCC][34] = 68, + [2][1][2][1][RTW89_ETSI][34] = 127, + [2][1][2][1][RTW89_MKK][34] = 72, + [2][1][2][1][RTW89_IC][34] = 68, + [2][1][2][1][RTW89_KCC][34] = 64, + [2][1][2][1][RTW89_ACMA][34] = 68, + [2][1][2][1][RTW89_CHILE][34] = 30, + [2][1][2][1][RTW89_UKRAINE][34] = 127, + [2][1][2][1][RTW89_MEXICO][34] = 68, + [2][1][2][1][RTW89_CN][34] = 127, + [2][1][2][1][RTW89_QATAR][34] = 127, + [2][1][2][1][RTW89_FCC][41] = 72, + [2][1][2][1][RTW89_ETSI][41] = 4, + [2][1][2][1][RTW89_MKK][41] = 127, + [2][1][2][1][RTW89_IC][41] = 72, + [2][1][2][1][RTW89_KCC][41] = 58, + [2][1][2][1][RTW89_ACMA][41] = 72, + [2][1][2][1][RTW89_CHILE][41] = 30, + [2][1][2][1][RTW89_UKRAINE][41] = 4, + [2][1][2][1][RTW89_MEXICO][41] = 72, + [2][1][2][1][RTW89_CN][41] = 64, + [2][1][2][1][RTW89_QATAR][41] = 4, }; const s8 rtw89_8852a_txpwr_lmt_ru_2g[RTW89_RU_NUM][RTW89_NTX_NUM] [RTW89_REGD_NUM][RTW89_2G_CH_NUM] = { - [0][0][0][0] = 32, - [0][0][0][1] = 32, - [0][0][0][2] = 32, - [0][0][0][3] = 32, - [0][0][0][4] = 32, - [0][0][0][5] = 32, - [0][0][0][6] = 32, - [0][0][0][7] = 32, - [0][0][0][8] = 32, - [0][0][0][9] = 32, - [0][0][0][10] = 32, - [0][0][0][11] = 32, - [0][0][0][12] = 32, - [0][0][0][13] = 0, - [0][1][0][0] = 20, - [0][1][0][1] = 20, - [0][1][0][2] = 20, - [0][1][0][3] = 20, - [0][1][0][4] = 20, - [0][1][0][5] = 20, - [0][1][0][6] = 20, - [0][1][0][7] = 20, - [0][1][0][8] = 20, - [0][1][0][9] = 20, - [0][1][0][10] = 20, - [0][1][0][11] = 20, - [0][1][0][12] = 20, - [0][1][0][13] = 0, - [1][0][0][0] = 42, - [1][0][0][1] = 42, - [1][0][0][2] = 42, - [1][0][0][3] = 42, - [1][0][0][4] = 42, - [1][0][0][5] = 42, - [1][0][0][6] = 42, - [1][0][0][7] = 42, - [1][0][0][8] = 42, - [1][0][0][9] = 42, - [1][0][0][10] = 42, - [1][0][0][11] = 42, - [1][0][0][12] = 36, - [1][0][0][13] = 0, - [1][1][0][0] = 30, - [1][1][0][1] = 30, - [1][1][0][2] = 30, - [1][1][0][3] = 30, - [1][1][0][4] = 30, - [1][1][0][5] = 30, - [1][1][0][6] = 30, - [1][1][0][7] = 30, - [1][1][0][8] = 30, - [1][1][0][9] = 30, - [1][1][0][10] = 30, - [1][1][0][11] = 30, - [1][1][0][12] = 30, - [1][1][0][13] = 0, - [2][0][0][0] = 52, - [2][0][0][1] = 52, - [2][0][0][2] = 52, - [2][0][0][3] = 52, - [2][0][0][4] = 52, - [2][0][0][5] = 52, - [2][0][0][6] = 52, - [2][0][0][7] = 52, - [2][0][0][8] = 52, - [2][0][0][9] = 52, - [2][0][0][10] = 52, - [2][0][0][11] = 52, - [2][0][0][12] = 40, - [2][0][0][13] = 0, - [2][1][0][0] = 40, - [2][1][0][1] = 40, - [2][1][0][2] = 40, - [2][1][0][3] = 40, - [2][1][0][4] = 40, - [2][1][0][5] = 40, - [2][1][0][6] = 40, - [2][1][0][7] = 40, - [2][1][0][8] = 40, - [2][1][0][9] = 40, - [2][1][0][10] = 40, - [2][1][0][11] = 40, - [2][1][0][12] = 26, - [2][1][0][13] = 0, - [0][0][2][0] = 70, - [0][0][1][0] = 32, - [0][0][3][0] = 40, - [0][0][5][0] = 70, - [0][0][6][0] = 32, - [0][0][9][0] = 32, - [0][0][8][0] = 60, - [0][0][11][0] = 32, - [0][0][2][1] = 70, - [0][0][1][1] = 32, - [0][0][3][1] = 40, - [0][0][5][1] = 70, - [0][0][6][1] = 32, - [0][0][9][1] = 32, - [0][0][8][1] = 60, - [0][0][11][1] = 32, - [0][0][2][2] = 74, - [0][0][1][2] = 32, - [0][0][3][2] = 40, - [0][0][5][2] = 74, - [0][0][6][2] = 32, - [0][0][9][2] = 32, - [0][0][8][2] = 60, - [0][0][11][2] = 32, - [0][0][2][3] = 78, - [0][0][1][3] = 32, - [0][0][3][3] = 40, - [0][0][5][3] = 78, - [0][0][6][3] = 32, - [0][0][9][3] = 32, - [0][0][8][3] = 60, - [0][0][11][3] = 32, - [0][0][2][4] = 78, - [0][0][1][4] = 32, - [0][0][3][4] = 40, - [0][0][5][4] = 78, - [0][0][6][4] = 32, - [0][0][9][4] = 32, - [0][0][8][4] = 60, - [0][0][11][4] = 32, - [0][0][2][5] = 78, - [0][0][1][5] = 32, - [0][0][3][5] = 40, - [0][0][5][5] = 78, - [0][0][6][5] = 32, - [0][0][9][5] = 32, - [0][0][8][5] = 60, - [0][0][11][5] = 32, - [0][0][2][6] = 78, - [0][0][1][6] = 32, - [0][0][3][6] = 40, - [0][0][5][6] = 78, - [0][0][6][6] = 32, - [0][0][9][6] = 32, - [0][0][8][6] = 60, - [0][0][11][6] = 32, - [0][0][2][7] = 78, - [0][0][1][7] = 32, - [0][0][3][7] = 40, - [0][0][5][7] = 78, - [0][0][6][7] = 32, - [0][0][9][7] = 32, - [0][0][8][7] = 60, - [0][0][11][7] = 32, - [0][0][2][8] = 74, - [0][0][1][8] = 32, - [0][0][3][8] = 40, - [0][0][5][8] = 74, - [0][0][6][8] = 32, - [0][0][9][8] = 32, - [0][0][8][8] = 60, - [0][0][11][8] = 32, - [0][0][2][9] = 70, - [0][0][1][9] = 32, - [0][0][3][9] = 40, - [0][0][5][9] = 70, - [0][0][6][9] = 32, - [0][0][9][9] = 32, - [0][0][8][9] = 60, - [0][0][11][9] = 32, - [0][0][2][10] = 70, - [0][0][1][10] = 32, - [0][0][3][10] = 40, - [0][0][5][10] = 70, - [0][0][6][10] = 32, - [0][0][9][10] = 32, - [0][0][8][10] = 60, - [0][0][11][10] = 32, - [0][0][2][11] = 58, - [0][0][1][11] = 32, - [0][0][3][11] = 40, - [0][0][5][11] = 58, - [0][0][6][11] = 32, - [0][0][9][11] = 32, - [0][0][8][11] = 60, - [0][0][11][11] = 32, - [0][0][2][12] = 34, - [0][0][1][12] = 32, - [0][0][3][12] = 40, - [0][0][5][12] = 34, - [0][0][6][12] = 32, - [0][0][9][12] = 32, - [0][0][8][12] = 60, - [0][0][11][12] = 32, - [0][0][2][13] = 127, - [0][0][1][13] = 127, - [0][0][3][13] = 127, - [0][0][5][13] = 127, - [0][0][6][13] = 127, - [0][0][9][13] = 127, - [0][0][8][13] = 127, - [0][0][11][13] = 127, - [0][1][2][0] = 64, - [0][1][1][0] = 20, - [0][1][3][0] = 28, - [0][1][5][0] = 64, - [0][1][6][0] = 20, - [0][1][9][0] = 20, - [0][1][8][0] = 48, - [0][1][11][0] = 20, - [0][1][2][1] = 64, - [0][1][1][1] = 20, - [0][1][3][1] = 28, - [0][1][5][1] = 64, - [0][1][6][1] = 20, - [0][1][9][1] = 20, - [0][1][8][1] = 48, - [0][1][11][1] = 20, - [0][1][2][2] = 68, - [0][1][1][2] = 20, - [0][1][3][2] = 28, - [0][1][5][2] = 68, - [0][1][6][2] = 20, - [0][1][9][2] = 20, - [0][1][8][2] = 48, - [0][1][11][2] = 20, - [0][1][2][3] = 72, - [0][1][1][3] = 20, - [0][1][3][3] = 28, - [0][1][5][3] = 72, - [0][1][6][3] = 20, - [0][1][9][3] = 20, - [0][1][8][3] = 48, - [0][1][11][3] = 20, - [0][1][2][4] = 76, - [0][1][1][4] = 20, - [0][1][3][4] = 28, - [0][1][5][4] = 76, - [0][1][6][4] = 20, - [0][1][9][4] = 20, - [0][1][8][4] = 48, - [0][1][11][4] = 20, - [0][1][2][5] = 78, - [0][1][1][5] = 20, - [0][1][3][5] = 28, - [0][1][5][5] = 78, - [0][1][6][5] = 20, - [0][1][9][5] = 20, - [0][1][8][5] = 48, - [0][1][11][5] = 20, - [0][1][2][6] = 76, - [0][1][1][6] = 20, - [0][1][3][6] = 28, - [0][1][5][6] = 76, - [0][1][6][6] = 20, - [0][1][9][6] = 20, - [0][1][8][6] = 48, - [0][1][11][6] = 20, - [0][1][2][7] = 72, - [0][1][1][7] = 20, - [0][1][3][7] = 28, - [0][1][5][7] = 72, - [0][1][6][7] = 20, - [0][1][9][7] = 20, - [0][1][8][7] = 48, - [0][1][11][7] = 20, - [0][1][2][8] = 68, - [0][1][1][8] = 20, - [0][1][3][8] = 28, - [0][1][5][8] = 68, - [0][1][6][8] = 20, - [0][1][9][8] = 20, - [0][1][8][8] = 48, - [0][1][11][8] = 20, - [0][1][2][9] = 64, - [0][1][1][9] = 20, - [0][1][3][9] = 28, - [0][1][5][9] = 64, - [0][1][6][9] = 20, - [0][1][9][9] = 20, - [0][1][8][9] = 48, - [0][1][11][9] = 20, - [0][1][2][10] = 64, - [0][1][1][10] = 20, - [0][1][3][10] = 28, - [0][1][5][10] = 64, - [0][1][6][10] = 20, - [0][1][9][10] = 20, - [0][1][8][10] = 48, - [0][1][11][10] = 20, - [0][1][2][11] = 54, - [0][1][1][11] = 20, - [0][1][3][11] = 28, - [0][1][5][11] = 54, - [0][1][6][11] = 20, - [0][1][9][11] = 20, - [0][1][8][11] = 48, - [0][1][11][11] = 20, - [0][1][2][12] = 32, - [0][1][1][12] = 20, - [0][1][3][12] = 28, - [0][1][5][12] = 32, - [0][1][6][12] = 20, - [0][1][9][12] = 20, - [0][1][8][12] = 48, - [0][1][11][12] = 20, - [0][1][2][13] = 127, - [0][1][1][13] = 127, - [0][1][3][13] = 127, - [0][1][5][13] = 127, - [0][1][6][13] = 127, - [0][1][9][13] = 127, - [0][1][8][13] = 127, - [0][1][11][13] = 127, - [1][0][2][0] = 72, - [1][0][1][0] = 42, - [1][0][3][0] = 50, - [1][0][5][0] = 72, - [1][0][6][0] = 42, - [1][0][9][0] = 42, - [1][0][8][0] = 60, - [1][0][11][0] = 42, - [1][0][2][1] = 72, - [1][0][1][1] = 42, - [1][0][3][1] = 50, - [1][0][5][1] = 72, - [1][0][6][1] = 42, - [1][0][9][1] = 42, - [1][0][8][1] = 60, - [1][0][11][1] = 42, - [1][0][2][2] = 76, - [1][0][1][2] = 42, - [1][0][3][2] = 50, - [1][0][5][2] = 76, - [1][0][6][2] = 42, - [1][0][9][2] = 42, - [1][0][8][2] = 60, - [1][0][11][2] = 42, - [1][0][2][3] = 78, - [1][0][1][3] = 42, - [1][0][3][3] = 50, - [1][0][5][3] = 78, - [1][0][6][3] = 42, - [1][0][9][3] = 42, - [1][0][8][3] = 60, - [1][0][11][3] = 42, - [1][0][2][4] = 78, - [1][0][1][4] = 42, - [1][0][3][4] = 50, - [1][0][5][4] = 78, - [1][0][6][4] = 42, - [1][0][9][4] = 42, - [1][0][8][4] = 60, - [1][0][11][4] = 42, - [1][0][2][5] = 78, - [1][0][1][5] = 42, - [1][0][3][5] = 50, - [1][0][5][5] = 78, - [1][0][6][5] = 42, - [1][0][9][5] = 42, - [1][0][8][5] = 60, - [1][0][11][5] = 42, - [1][0][2][6] = 78, - [1][0][1][6] = 42, - [1][0][3][6] = 50, - [1][0][5][6] = 78, - [1][0][6][6] = 42, - [1][0][9][6] = 42, - [1][0][8][6] = 60, - [1][0][11][6] = 42, - [1][0][2][7] = 78, - [1][0][1][7] = 42, - [1][0][3][7] = 50, - [1][0][5][7] = 78, - [1][0][6][7] = 42, - [1][0][9][7] = 42, - [1][0][8][7] = 60, - [1][0][11][7] = 42, - [1][0][2][8] = 78, - [1][0][1][8] = 42, - [1][0][3][8] = 50, - [1][0][5][8] = 78, - [1][0][6][8] = 42, - [1][0][9][8] = 42, - [1][0][8][8] = 60, - [1][0][11][8] = 42, - [1][0][2][9] = 74, - [1][0][1][9] = 42, - [1][0][3][9] = 50, - [1][0][5][9] = 74, - [1][0][6][9] = 42, - [1][0][9][9] = 42, - [1][0][8][9] = 60, - [1][0][11][9] = 42, - [1][0][2][10] = 74, - [1][0][1][10] = 42, - [1][0][3][10] = 50, - [1][0][5][10] = 74, - [1][0][6][10] = 42, - [1][0][9][10] = 42, - [1][0][8][10] = 60, - [1][0][11][10] = 42, - [1][0][2][11] = 64, - [1][0][1][11] = 42, - [1][0][3][11] = 50, - [1][0][5][11] = 64, - [1][0][6][11] = 42, - [1][0][9][11] = 42, - [1][0][8][11] = 60, - [1][0][11][11] = 42, - [1][0][2][12] = 36, - [1][0][1][12] = 42, - [1][0][3][12] = 50, - [1][0][5][12] = 36, - [1][0][6][12] = 42, - [1][0][9][12] = 42, - [1][0][8][12] = 60, - [1][0][11][12] = 42, - [1][0][2][13] = 127, - [1][0][1][13] = 127, - [1][0][3][13] = 127, - [1][0][5][13] = 127, - [1][0][6][13] = 127, - [1][0][9][13] = 127, - [1][0][8][13] = 127, - [1][0][11][13] = 127, - [1][1][2][0] = 66, - [1][1][1][0] = 30, - [1][1][3][0] = 38, - [1][1][5][0] = 66, - [1][1][6][0] = 30, - [1][1][9][0] = 30, - [1][1][8][0] = 48, - [1][1][11][0] = 30, - [1][1][2][1] = 66, - [1][1][1][1] = 30, - [1][1][3][1] = 38, - [1][1][5][1] = 66, - [1][1][6][1] = 30, - [1][1][9][1] = 30, - [1][1][8][1] = 48, - [1][1][11][1] = 30, - [1][1][2][2] = 70, - [1][1][1][2] = 30, - [1][1][3][2] = 38, - [1][1][5][2] = 70, - [1][1][6][2] = 30, - [1][1][9][2] = 30, - [1][1][8][2] = 48, - [1][1][11][2] = 30, - [1][1][2][3] = 74, - [1][1][1][3] = 30, - [1][1][3][3] = 38, - [1][1][5][3] = 74, - [1][1][6][3] = 30, - [1][1][9][3] = 30, - [1][1][8][3] = 48, - [1][1][11][3] = 30, - [1][1][2][4] = 78, - [1][1][1][4] = 30, - [1][1][3][4] = 38, - [1][1][5][4] = 78, - [1][1][6][4] = 30, - [1][1][9][4] = 30, - [1][1][8][4] = 48, - [1][1][11][4] = 30, - [1][1][2][5] = 78, - [1][1][1][5] = 30, - [1][1][3][5] = 38, - [1][1][5][5] = 78, - [1][1][6][5] = 30, - [1][1][9][5] = 30, - [1][1][8][5] = 48, - [1][1][11][5] = 30, - [1][1][2][6] = 78, - [1][1][1][6] = 30, - [1][1][3][6] = 38, - [1][1][5][6] = 78, - [1][1][6][6] = 30, - [1][1][9][6] = 30, - [1][1][8][6] = 48, - [1][1][11][6] = 30, - [1][1][2][7] = 74, - [1][1][1][7] = 30, - [1][1][3][7] = 38, - [1][1][5][7] = 74, - [1][1][6][7] = 30, - [1][1][9][7] = 30, - [1][1][8][7] = 48, - [1][1][11][7] = 30, - [1][1][2][8] = 70, - [1][1][1][8] = 30, - [1][1][3][8] = 38, - [1][1][5][8] = 70, - [1][1][6][8] = 30, - [1][1][9][8] = 30, - [1][1][8][8] = 48, - [1][1][11][8] = 30, - [1][1][2][9] = 66, - [1][1][1][9] = 30, - [1][1][3][9] = 38, - [1][1][5][9] = 66, - [1][1][6][9] = 30, - [1][1][9][9] = 30, - [1][1][8][9] = 48, - [1][1][11][9] = 30, - [1][1][2][10] = 66, - [1][1][1][10] = 30, - [1][1][3][10] = 38, - [1][1][5][10] = 66, - [1][1][6][10] = 30, - [1][1][9][10] = 30, - [1][1][8][10] = 48, - [1][1][11][10] = 30, - [1][1][2][11] = 60, - [1][1][1][11] = 30, - [1][1][3][11] = 38, - [1][1][5][11] = 60, - [1][1][6][11] = 30, - [1][1][9][11] = 30, - [1][1][8][11] = 48, - [1][1][11][11] = 30, - [1][1][2][12] = 32, - [1][1][1][12] = 30, - [1][1][3][12] = 38, - [1][1][5][12] = 32, - [1][1][6][12] = 30, - [1][1][9][12] = 30, - [1][1][8][12] = 48, - [1][1][11][12] = 30, - [1][1][2][13] = 127, - [1][1][1][13] = 127, - [1][1][3][13] = 127, - [1][1][5][13] = 127, - [1][1][6][13] = 127, - [1][1][9][13] = 127, - [1][1][8][13] = 127, - [1][1][11][13] = 127, - [2][0][2][0] = 76, - [2][0][1][0] = 52, - [2][0][3][0] = 64, - [2][0][5][0] = 76, - [2][0][6][0] = 52, - [2][0][9][0] = 52, - [2][0][8][0] = 60, - [2][0][11][0] = 52, - [2][0][2][1] = 76, - [2][0][1][1] = 52, - [2][0][3][1] = 64, - [2][0][5][1] = 76, - [2][0][6][1] = 52, - [2][0][9][1] = 52, - [2][0][8][1] = 60, - [2][0][11][1] = 52, - [2][0][2][2] = 78, - [2][0][1][2] = 52, - [2][0][3][2] = 64, - [2][0][5][2] = 78, - [2][0][6][2] = 52, - [2][0][9][2] = 52, - [2][0][8][2] = 60, - [2][0][11][2] = 52, - [2][0][2][3] = 78, - [2][0][1][3] = 52, - [2][0][3][3] = 64, - [2][0][5][3] = 78, - [2][0][6][3] = 52, - [2][0][9][3] = 52, - [2][0][8][3] = 60, - [2][0][11][3] = 52, - [2][0][2][4] = 78, - [2][0][1][4] = 52, - [2][0][3][4] = 64, - [2][0][5][4] = 78, - [2][0][6][4] = 52, - [2][0][9][4] = 52, - [2][0][8][4] = 60, - [2][0][11][4] = 52, - [2][0][2][5] = 78, - [2][0][1][5] = 52, - [2][0][3][5] = 64, - [2][0][5][5] = 78, - [2][0][6][5] = 52, - [2][0][9][5] = 52, - [2][0][8][5] = 60, - [2][0][11][5] = 52, - [2][0][2][6] = 78, - [2][0][1][6] = 52, - [2][0][3][6] = 64, - [2][0][5][6] = 78, - [2][0][6][6] = 52, - [2][0][9][6] = 52, - [2][0][8][6] = 60, - [2][0][11][6] = 52, - [2][0][2][7] = 78, - [2][0][1][7] = 52, - [2][0][3][7] = 64, - [2][0][5][7] = 78, - [2][0][6][7] = 52, - [2][0][9][7] = 52, - [2][0][8][7] = 60, - [2][0][11][7] = 52, - [2][0][2][8] = 78, - [2][0][1][8] = 52, - [2][0][3][8] = 64, - [2][0][5][8] = 78, - [2][0][6][8] = 52, - [2][0][9][8] = 52, - [2][0][8][8] = 60, - [2][0][11][8] = 52, - [2][0][2][9] = 76, - [2][0][1][9] = 52, - [2][0][3][9] = 64, - [2][0][5][9] = 76, - [2][0][6][9] = 52, - [2][0][9][9] = 52, - [2][0][8][9] = 60, - [2][0][11][9] = 52, - [2][0][2][10] = 76, - [2][0][1][10] = 52, - [2][0][3][10] = 64, - [2][0][5][10] = 76, - [2][0][6][10] = 52, - [2][0][9][10] = 52, - [2][0][8][10] = 60, - [2][0][11][10] = 52, - [2][0][2][11] = 68, - [2][0][1][11] = 52, - [2][0][3][11] = 64, - [2][0][5][11] = 68, - [2][0][6][11] = 52, - [2][0][9][11] = 52, - [2][0][8][11] = 60, - [2][0][11][11] = 52, - [2][0][2][12] = 40, - [2][0][1][12] = 52, - [2][0][3][12] = 64, - [2][0][5][12] = 40, - [2][0][6][12] = 52, - [2][0][9][12] = 52, - [2][0][8][12] = 60, - [2][0][11][12] = 52, - [2][0][2][13] = 127, - [2][0][1][13] = 127, - [2][0][3][13] = 127, - [2][0][5][13] = 127, - [2][0][6][13] = 127, - [2][0][9][13] = 127, - [2][0][8][13] = 127, - [2][0][11][13] = 127, - [2][1][2][0] = 68, - [2][1][1][0] = 40, - [2][1][3][0] = 52, - [2][1][5][0] = 68, - [2][1][6][0] = 40, - [2][1][9][0] = 40, - [2][1][8][0] = 48, - [2][1][11][0] = 40, - [2][1][2][1] = 68, - [2][1][1][1] = 40, - [2][1][3][1] = 52, - [2][1][5][1] = 68, - [2][1][6][1] = 40, - [2][1][9][1] = 40, - [2][1][8][1] = 48, - [2][1][11][1] = 40, - [2][1][2][2] = 72, - [2][1][1][2] = 40, - [2][1][3][2] = 52, - [2][1][5][2] = 72, - [2][1][6][2] = 40, - [2][1][9][2] = 40, - [2][1][8][2] = 48, - [2][1][11][2] = 40, - [2][1][2][3] = 76, - [2][1][1][3] = 40, - [2][1][3][3] = 52, - [2][1][5][3] = 76, - [2][1][6][3] = 40, - [2][1][9][3] = 40, - [2][1][8][3] = 48, - [2][1][11][3] = 40, - [2][1][2][4] = 78, - [2][1][1][4] = 40, - [2][1][3][4] = 52, - [2][1][5][4] = 78, - [2][1][6][4] = 40, - [2][1][9][4] = 40, - [2][1][8][4] = 48, - [2][1][11][4] = 40, - [2][1][2][5] = 78, - [2][1][1][5] = 40, - [2][1][3][5] = 52, - [2][1][5][5] = 78, - [2][1][6][5] = 40, - [2][1][9][5] = 40, - [2][1][8][5] = 48, - [2][1][11][5] = 40, - [2][1][2][6] = 78, - [2][1][1][6] = 40, - [2][1][3][6] = 52, - [2][1][5][6] = 78, - [2][1][6][6] = 40, - [2][1][9][6] = 40, - [2][1][8][6] = 48, - [2][1][11][6] = 40, - [2][1][2][7] = 78, - [2][1][1][7] = 40, - [2][1][3][7] = 52, - [2][1][5][7] = 78, - [2][1][6][7] = 40, - [2][1][9][7] = 40, - [2][1][8][7] = 48, - [2][1][11][7] = 40, - [2][1][2][8] = 74, - [2][1][1][8] = 40, - [2][1][3][8] = 52, - [2][1][5][8] = 74, - [2][1][6][8] = 40, - [2][1][9][8] = 40, - [2][1][8][8] = 48, - [2][1][11][8] = 40, - [2][1][2][9] = 70, - [2][1][1][9] = 40, - [2][1][3][9] = 52, - [2][1][5][9] = 70, - [2][1][6][9] = 40, - [2][1][9][9] = 40, - [2][1][8][9] = 48, - [2][1][11][9] = 40, - [2][1][2][10] = 70, - [2][1][1][10] = 40, - [2][1][3][10] = 52, - [2][1][5][10] = 70, - [2][1][6][10] = 40, - [2][1][9][10] = 40, - [2][1][8][10] = 48, - [2][1][11][10] = 40, - [2][1][2][11] = 48, - [2][1][1][11] = 40, - [2][1][3][11] = 52, - [2][1][5][11] = 48, - [2][1][6][11] = 40, - [2][1][9][11] = 40, - [2][1][8][11] = 48, - [2][1][11][11] = 40, - [2][1][2][12] = 26, - [2][1][1][12] = 40, - [2][1][3][12] = 52, - [2][1][5][12] = 26, - [2][1][6][12] = 40, - [2][1][9][12] = 40, - [2][1][8][12] = 48, - [2][1][11][12] = 40, - [2][1][2][13] = 127, - [2][1][1][13] = 127, - [2][1][3][13] = 127, - [2][1][5][13] = 127, - [2][1][6][13] = 127, - [2][1][9][13] = 127, - [2][1][8][13] = 127, - [2][1][11][13] = 127, + [0][0][RTW89_WW][0] = 32, + [0][0][RTW89_WW][1] = 32, + [0][0][RTW89_WW][2] = 32, + [0][0][RTW89_WW][3] = 32, + [0][0][RTW89_WW][4] = 32, + [0][0][RTW89_WW][5] = 32, + [0][0][RTW89_WW][6] = 32, + [0][0][RTW89_WW][7] = 32, + [0][0][RTW89_WW][8] = 32, + [0][0][RTW89_WW][9] = 32, + [0][0][RTW89_WW][10] = 32, + [0][0][RTW89_WW][11] = 32, + [0][0][RTW89_WW][12] = 32, + [0][0][RTW89_WW][13] = 0, + [0][1][RTW89_WW][0] = 20, + [0][1][RTW89_WW][1] = 20, + [0][1][RTW89_WW][2] = 20, + [0][1][RTW89_WW][3] = 20, + [0][1][RTW89_WW][4] = 20, + [0][1][RTW89_WW][5] = 20, + [0][1][RTW89_WW][6] = 20, + [0][1][RTW89_WW][7] = 20, + [0][1][RTW89_WW][8] = 20, + [0][1][RTW89_WW][9] = 20, + [0][1][RTW89_WW][10] = 20, + [0][1][RTW89_WW][11] = 20, + [0][1][RTW89_WW][12] = 20, + [0][1][RTW89_WW][13] = 0, + [1][0][RTW89_WW][0] = 42, + [1][0][RTW89_WW][1] = 42, + [1][0][RTW89_WW][2] = 42, + [1][0][RTW89_WW][3] = 42, + [1][0][RTW89_WW][4] = 42, + [1][0][RTW89_WW][5] = 42, + [1][0][RTW89_WW][6] = 42, + [1][0][RTW89_WW][7] = 42, + [1][0][RTW89_WW][8] = 42, + [1][0][RTW89_WW][9] = 42, + [1][0][RTW89_WW][10] = 42, + [1][0][RTW89_WW][11] = 42, + [1][0][RTW89_WW][12] = 36, + [1][0][RTW89_WW][13] = 0, + [1][1][RTW89_WW][0] = 30, + [1][1][RTW89_WW][1] = 30, + [1][1][RTW89_WW][2] = 30, + [1][1][RTW89_WW][3] = 30, + [1][1][RTW89_WW][4] = 30, + [1][1][RTW89_WW][5] = 30, + [1][1][RTW89_WW][6] = 30, + [1][1][RTW89_WW][7] = 30, + [1][1][RTW89_WW][8] = 30, + [1][1][RTW89_WW][9] = 30, + [1][1][RTW89_WW][10] = 30, + [1][1][RTW89_WW][11] = 30, + [1][1][RTW89_WW][12] = 30, + [1][1][RTW89_WW][13] = 0, + [2][0][RTW89_WW][0] = 52, + [2][0][RTW89_WW][1] = 52, + [2][0][RTW89_WW][2] = 52, + [2][0][RTW89_WW][3] = 52, + [2][0][RTW89_WW][4] = 52, + [2][0][RTW89_WW][5] = 52, + [2][0][RTW89_WW][6] = 52, + [2][0][RTW89_WW][7] = 52, + [2][0][RTW89_WW][8] = 52, + [2][0][RTW89_WW][9] = 52, + [2][0][RTW89_WW][10] = 52, + [2][0][RTW89_WW][11] = 52, + [2][0][RTW89_WW][12] = 40, + [2][0][RTW89_WW][13] = 0, + [2][1][RTW89_WW][0] = 40, + [2][1][RTW89_WW][1] = 40, + [2][1][RTW89_WW][2] = 40, + [2][1][RTW89_WW][3] = 40, + [2][1][RTW89_WW][4] = 40, + [2][1][RTW89_WW][5] = 40, + [2][1][RTW89_WW][6] = 40, + [2][1][RTW89_WW][7] = 40, + [2][1][RTW89_WW][8] = 40, + [2][1][RTW89_WW][9] = 40, + [2][1][RTW89_WW][10] = 40, + [2][1][RTW89_WW][11] = 40, + [2][1][RTW89_WW][12] = 26, + [2][1][RTW89_WW][13] = 0, + [0][0][RTW89_FCC][0] = 70, + [0][0][RTW89_ETSI][0] = 32, + [0][0][RTW89_MKK][0] = 40, + [0][0][RTW89_IC][0] = 70, + [0][0][RTW89_KCC][0] = 46, + [0][0][RTW89_ACMA][0] = 32, + [0][0][RTW89_CHILE][0] = 60, + [0][0][RTW89_UKRAINE][0] = 32, + [0][0][RTW89_MEXICO][0] = 70, + [0][0][RTW89_CN][0] = 32, + [0][0][RTW89_QATAR][0] = 32, + [0][0][RTW89_FCC][1] = 70, + [0][0][RTW89_ETSI][1] = 32, + [0][0][RTW89_MKK][1] = 40, + [0][0][RTW89_IC][1] = 70, + [0][0][RTW89_KCC][1] = 46, + [0][0][RTW89_ACMA][1] = 32, + [0][0][RTW89_CHILE][1] = 60, + [0][0][RTW89_UKRAINE][1] = 32, + [0][0][RTW89_MEXICO][1] = 70, + [0][0][RTW89_CN][1] = 32, + [0][0][RTW89_QATAR][1] = 32, + [0][0][RTW89_FCC][2] = 74, + [0][0][RTW89_ETSI][2] = 32, + [0][0][RTW89_MKK][2] = 40, + [0][0][RTW89_IC][2] = 74, + [0][0][RTW89_KCC][2] = 46, + [0][0][RTW89_ACMA][2] = 32, + [0][0][RTW89_CHILE][2] = 60, + [0][0][RTW89_UKRAINE][2] = 32, + [0][0][RTW89_MEXICO][2] = 74, + [0][0][RTW89_CN][2] = 32, + [0][0][RTW89_QATAR][2] = 32, + [0][0][RTW89_FCC][3] = 78, + [0][0][RTW89_ETSI][3] = 32, + [0][0][RTW89_MKK][3] = 40, + [0][0][RTW89_IC][3] = 78, + [0][0][RTW89_KCC][3] = 46, + [0][0][RTW89_ACMA][3] = 32, + [0][0][RTW89_CHILE][3] = 60, + [0][0][RTW89_UKRAINE][3] = 32, + [0][0][RTW89_MEXICO][3] = 78, + [0][0][RTW89_CN][3] = 32, + [0][0][RTW89_QATAR][3] = 32, + [0][0][RTW89_FCC][4] = 78, + [0][0][RTW89_ETSI][4] = 32, + [0][0][RTW89_MKK][4] = 40, + [0][0][RTW89_IC][4] = 78, + [0][0][RTW89_KCC][4] = 46, + [0][0][RTW89_ACMA][4] = 32, + [0][0][RTW89_CHILE][4] = 60, + [0][0][RTW89_UKRAINE][4] = 32, + [0][0][RTW89_MEXICO][4] = 78, + [0][0][RTW89_CN][4] = 32, + [0][0][RTW89_QATAR][4] = 32, + [0][0][RTW89_FCC][5] = 78, + [0][0][RTW89_ETSI][5] = 32, + [0][0][RTW89_MKK][5] = 40, + [0][0][RTW89_IC][5] = 78, + [0][0][RTW89_KCC][5] = 46, + [0][0][RTW89_ACMA][5] = 32, + [0][0][RTW89_CHILE][5] = 60, + [0][0][RTW89_UKRAINE][5] = 32, + [0][0][RTW89_MEXICO][5] = 78, + [0][0][RTW89_CN][5] = 32, + [0][0][RTW89_QATAR][5] = 32, + [0][0][RTW89_FCC][6] = 78, + [0][0][RTW89_ETSI][6] = 32, + [0][0][RTW89_MKK][6] = 40, + [0][0][RTW89_IC][6] = 78, + [0][0][RTW89_KCC][6] = 46, + [0][0][RTW89_ACMA][6] = 32, + [0][0][RTW89_CHILE][6] = 60, + [0][0][RTW89_UKRAINE][6] = 32, + [0][0][RTW89_MEXICO][6] = 78, + [0][0][RTW89_CN][6] = 32, + [0][0][RTW89_QATAR][6] = 32, + [0][0][RTW89_FCC][7] = 78, + [0][0][RTW89_ETSI][7] = 32, + [0][0][RTW89_MKK][7] = 40, + [0][0][RTW89_IC][7] = 78, + [0][0][RTW89_KCC][7] = 46, + [0][0][RTW89_ACMA][7] = 32, + [0][0][RTW89_CHILE][7] = 60, + [0][0][RTW89_UKRAINE][7] = 32, + [0][0][RTW89_MEXICO][7] = 78, + [0][0][RTW89_CN][7] = 32, + [0][0][RTW89_QATAR][7] = 32, + [0][0][RTW89_FCC][8] = 74, + [0][0][RTW89_ETSI][8] = 32, + [0][0][RTW89_MKK][8] = 40, + [0][0][RTW89_IC][8] = 74, + [0][0][RTW89_KCC][8] = 46, + [0][0][RTW89_ACMA][8] = 32, + [0][0][RTW89_CHILE][8] = 60, + [0][0][RTW89_UKRAINE][8] = 32, + [0][0][RTW89_MEXICO][8] = 74, + [0][0][RTW89_CN][8] = 32, + [0][0][RTW89_QATAR][8] = 32, + [0][0][RTW89_FCC][9] = 70, + [0][0][RTW89_ETSI][9] = 32, + [0][0][RTW89_MKK][9] = 40, + [0][0][RTW89_IC][9] = 70, + [0][0][RTW89_KCC][9] = 46, + [0][0][RTW89_ACMA][9] = 32, + [0][0][RTW89_CHILE][9] = 60, + [0][0][RTW89_UKRAINE][9] = 32, + [0][0][RTW89_MEXICO][9] = 70, + [0][0][RTW89_CN][9] = 32, + [0][0][RTW89_QATAR][9] = 32, + [0][0][RTW89_FCC][10] = 70, + [0][0][RTW89_ETSI][10] = 32, + [0][0][RTW89_MKK][10] = 40, + [0][0][RTW89_IC][10] = 70, + [0][0][RTW89_KCC][10] = 46, + [0][0][RTW89_ACMA][10] = 32, + [0][0][RTW89_CHILE][10] = 60, + [0][0][RTW89_UKRAINE][10] = 32, + [0][0][RTW89_MEXICO][10] = 70, + [0][0][RTW89_CN][10] = 32, + [0][0][RTW89_QATAR][10] = 32, + [0][0][RTW89_FCC][11] = 58, + [0][0][RTW89_ETSI][11] = 32, + [0][0][RTW89_MKK][11] = 40, + [0][0][RTW89_IC][11] = 58, + [0][0][RTW89_KCC][11] = 46, + [0][0][RTW89_ACMA][11] = 32, + [0][0][RTW89_CHILE][11] = 58, + [0][0][RTW89_UKRAINE][11] = 32, + [0][0][RTW89_MEXICO][11] = 58, + [0][0][RTW89_CN][11] = 32, + [0][0][RTW89_QATAR][11] = 32, + [0][0][RTW89_FCC][12] = 34, + [0][0][RTW89_ETSI][12] = 32, + [0][0][RTW89_MKK][12] = 40, + [0][0][RTW89_IC][12] = 34, + [0][0][RTW89_KCC][12] = 46, + [0][0][RTW89_ACMA][12] = 32, + [0][0][RTW89_CHILE][12] = 34, + [0][0][RTW89_UKRAINE][12] = 32, + [0][0][RTW89_MEXICO][12] = 34, + [0][0][RTW89_CN][12] = 32, + [0][0][RTW89_QATAR][12] = 32, + [0][0][RTW89_FCC][13] = 127, + [0][0][RTW89_ETSI][13] = 127, + [0][0][RTW89_MKK][13] = 127, + [0][0][RTW89_IC][13] = 127, + [0][0][RTW89_KCC][13] = 127, + [0][0][RTW89_ACMA][13] = 127, + [0][0][RTW89_CHILE][13] = 127, + [0][0][RTW89_UKRAINE][13] = 127, + [0][0][RTW89_MEXICO][13] = 127, + [0][0][RTW89_CN][13] = 127, + [0][0][RTW89_QATAR][13] = 127, + [0][1][RTW89_FCC][0] = 64, + [0][1][RTW89_ETSI][0] = 20, + [0][1][RTW89_MKK][0] = 28, + [0][1][RTW89_IC][0] = 64, + [0][1][RTW89_KCC][0] = 32, + [0][1][RTW89_ACMA][0] = 20, + [0][1][RTW89_CHILE][0] = 48, + [0][1][RTW89_UKRAINE][0] = 20, + [0][1][RTW89_MEXICO][0] = 64, + [0][1][RTW89_CN][0] = 20, + [0][1][RTW89_QATAR][0] = 20, + [0][1][RTW89_FCC][1] = 64, + [0][1][RTW89_ETSI][1] = 20, + [0][1][RTW89_MKK][1] = 28, + [0][1][RTW89_IC][1] = 64, + [0][1][RTW89_KCC][1] = 32, + [0][1][RTW89_ACMA][1] = 20, + [0][1][RTW89_CHILE][1] = 48, + [0][1][RTW89_UKRAINE][1] = 20, + [0][1][RTW89_MEXICO][1] = 64, + [0][1][RTW89_CN][1] = 20, + [0][1][RTW89_QATAR][1] = 20, + [0][1][RTW89_FCC][2] = 68, + [0][1][RTW89_ETSI][2] = 20, + [0][1][RTW89_MKK][2] = 28, + [0][1][RTW89_IC][2] = 68, + [0][1][RTW89_KCC][2] = 32, + [0][1][RTW89_ACMA][2] = 20, + [0][1][RTW89_CHILE][2] = 48, + [0][1][RTW89_UKRAINE][2] = 20, + [0][1][RTW89_MEXICO][2] = 68, + [0][1][RTW89_CN][2] = 20, + [0][1][RTW89_QATAR][2] = 20, + [0][1][RTW89_FCC][3] = 72, + [0][1][RTW89_ETSI][3] = 20, + [0][1][RTW89_MKK][3] = 28, + [0][1][RTW89_IC][3] = 72, + [0][1][RTW89_KCC][3] = 32, + [0][1][RTW89_ACMA][3] = 20, + [0][1][RTW89_CHILE][3] = 48, + [0][1][RTW89_UKRAINE][3] = 20, + [0][1][RTW89_MEXICO][3] = 72, + [0][1][RTW89_CN][3] = 20, + [0][1][RTW89_QATAR][3] = 20, + [0][1][RTW89_FCC][4] = 76, + [0][1][RTW89_ETSI][4] = 20, + [0][1][RTW89_MKK][4] = 28, + [0][1][RTW89_IC][4] = 76, + [0][1][RTW89_KCC][4] = 32, + [0][1][RTW89_ACMA][4] = 20, + [0][1][RTW89_CHILE][4] = 48, + [0][1][RTW89_UKRAINE][4] = 20, + [0][1][RTW89_MEXICO][4] = 76, + [0][1][RTW89_CN][4] = 20, + [0][1][RTW89_QATAR][4] = 20, + [0][1][RTW89_FCC][5] = 78, + [0][1][RTW89_ETSI][5] = 20, + [0][1][RTW89_MKK][5] = 28, + [0][1][RTW89_IC][5] = 78, + [0][1][RTW89_KCC][5] = 32, + [0][1][RTW89_ACMA][5] = 20, + [0][1][RTW89_CHILE][5] = 48, + [0][1][RTW89_UKRAINE][5] = 20, + [0][1][RTW89_MEXICO][5] = 78, + [0][1][RTW89_CN][5] = 20, + [0][1][RTW89_QATAR][5] = 20, + [0][1][RTW89_FCC][6] = 76, + [0][1][RTW89_ETSI][6] = 20, + [0][1][RTW89_MKK][6] = 28, + [0][1][RTW89_IC][6] = 76, + [0][1][RTW89_KCC][6] = 32, + [0][1][RTW89_ACMA][6] = 20, + [0][1][RTW89_CHILE][6] = 48, + [0][1][RTW89_UKRAINE][6] = 20, + [0][1][RTW89_MEXICO][6] = 76, + [0][1][RTW89_CN][6] = 20, + [0][1][RTW89_QATAR][6] = 20, + [0][1][RTW89_FCC][7] = 72, + [0][1][RTW89_ETSI][7] = 20, + [0][1][RTW89_MKK][7] = 28, + [0][1][RTW89_IC][7] = 72, + [0][1][RTW89_KCC][7] = 32, + [0][1][RTW89_ACMA][7] = 20, + [0][1][RTW89_CHILE][7] = 48, + [0][1][RTW89_UKRAINE][7] = 20, + [0][1][RTW89_MEXICO][7] = 72, + [0][1][RTW89_CN][7] = 20, + [0][1][RTW89_QATAR][7] = 20, + [0][1][RTW89_FCC][8] = 68, + [0][1][RTW89_ETSI][8] = 20, + [0][1][RTW89_MKK][8] = 28, + [0][1][RTW89_IC][8] = 68, + [0][1][RTW89_KCC][8] = 32, + [0][1][RTW89_ACMA][8] = 20, + [0][1][RTW89_CHILE][8] = 48, + [0][1][RTW89_UKRAINE][8] = 20, + [0][1][RTW89_MEXICO][8] = 68, + [0][1][RTW89_CN][8] = 20, + [0][1][RTW89_QATAR][8] = 20, + [0][1][RTW89_FCC][9] = 64, + [0][1][RTW89_ETSI][9] = 20, + [0][1][RTW89_MKK][9] = 28, + [0][1][RTW89_IC][9] = 64, + [0][1][RTW89_KCC][9] = 32, + [0][1][RTW89_ACMA][9] = 20, + [0][1][RTW89_CHILE][9] = 48, + [0][1][RTW89_UKRAINE][9] = 20, + [0][1][RTW89_MEXICO][9] = 64, + [0][1][RTW89_CN][9] = 20, + [0][1][RTW89_QATAR][9] = 20, + [0][1][RTW89_FCC][10] = 64, + [0][1][RTW89_ETSI][10] = 20, + [0][1][RTW89_MKK][10] = 28, + [0][1][RTW89_IC][10] = 64, + [0][1][RTW89_KCC][10] = 32, + [0][1][RTW89_ACMA][10] = 20, + [0][1][RTW89_CHILE][10] = 48, + [0][1][RTW89_UKRAINE][10] = 20, + [0][1][RTW89_MEXICO][10] = 64, + [0][1][RTW89_CN][10] = 20, + [0][1][RTW89_QATAR][10] = 20, + [0][1][RTW89_FCC][11] = 54, + [0][1][RTW89_ETSI][11] = 20, + [0][1][RTW89_MKK][11] = 28, + [0][1][RTW89_IC][11] = 54, + [0][1][RTW89_KCC][11] = 32, + [0][1][RTW89_ACMA][11] = 20, + [0][1][RTW89_CHILE][11] = 48, + [0][1][RTW89_UKRAINE][11] = 20, + [0][1][RTW89_MEXICO][11] = 54, + [0][1][RTW89_CN][11] = 20, + [0][1][RTW89_QATAR][11] = 20, + [0][1][RTW89_FCC][12] = 32, + [0][1][RTW89_ETSI][12] = 20, + [0][1][RTW89_MKK][12] = 28, + [0][1][RTW89_IC][12] = 32, + [0][1][RTW89_KCC][12] = 32, + [0][1][RTW89_ACMA][12] = 20, + [0][1][RTW89_CHILE][12] = 32, + [0][1][RTW89_UKRAINE][12] = 20, + [0][1][RTW89_MEXICO][12] = 32, + [0][1][RTW89_CN][12] = 20, + [0][1][RTW89_QATAR][12] = 20, + [0][1][RTW89_FCC][13] = 127, + [0][1][RTW89_ETSI][13] = 127, + [0][1][RTW89_MKK][13] = 127, + [0][1][RTW89_IC][13] = 127, + [0][1][RTW89_KCC][13] = 127, + [0][1][RTW89_ACMA][13] = 127, + [0][1][RTW89_CHILE][13] = 127, + [0][1][RTW89_UKRAINE][13] = 127, + [0][1][RTW89_MEXICO][13] = 127, + [0][1][RTW89_CN][13] = 127, + [0][1][RTW89_QATAR][13] = 127, + [1][0][RTW89_FCC][0] = 72, + [1][0][RTW89_ETSI][0] = 42, + [1][0][RTW89_MKK][0] = 50, + [1][0][RTW89_IC][0] = 72, + [1][0][RTW89_KCC][0] = 58, + [1][0][RTW89_ACMA][0] = 42, + [1][0][RTW89_CHILE][0] = 60, + [1][0][RTW89_UKRAINE][0] = 42, + [1][0][RTW89_MEXICO][0] = 72, + [1][0][RTW89_CN][0] = 42, + [1][0][RTW89_QATAR][0] = 42, + [1][0][RTW89_FCC][1] = 72, + [1][0][RTW89_ETSI][1] = 42, + [1][0][RTW89_MKK][1] = 50, + [1][0][RTW89_IC][1] = 72, + [1][0][RTW89_KCC][1] = 58, + [1][0][RTW89_ACMA][1] = 42, + [1][0][RTW89_CHILE][1] = 60, + [1][0][RTW89_UKRAINE][1] = 42, + [1][0][RTW89_MEXICO][1] = 72, + [1][0][RTW89_CN][1] = 42, + [1][0][RTW89_QATAR][1] = 42, + [1][0][RTW89_FCC][2] = 76, + [1][0][RTW89_ETSI][2] = 42, + [1][0][RTW89_MKK][2] = 50, + [1][0][RTW89_IC][2] = 76, + [1][0][RTW89_KCC][2] = 58, + [1][0][RTW89_ACMA][2] = 42, + [1][0][RTW89_CHILE][2] = 60, + [1][0][RTW89_UKRAINE][2] = 42, + [1][0][RTW89_MEXICO][2] = 76, + [1][0][RTW89_CN][2] = 42, + [1][0][RTW89_QATAR][2] = 42, + [1][0][RTW89_FCC][3] = 78, + [1][0][RTW89_ETSI][3] = 42, + [1][0][RTW89_MKK][3] = 50, + [1][0][RTW89_IC][3] = 78, + [1][0][RTW89_KCC][3] = 58, + [1][0][RTW89_ACMA][3] = 42, + [1][0][RTW89_CHILE][3] = 60, + [1][0][RTW89_UKRAINE][3] = 42, + [1][0][RTW89_MEXICO][3] = 78, + [1][0][RTW89_CN][3] = 42, + [1][0][RTW89_QATAR][3] = 42, + [1][0][RTW89_FCC][4] = 78, + [1][0][RTW89_ETSI][4] = 42, + [1][0][RTW89_MKK][4] = 50, + [1][0][RTW89_IC][4] = 78, + [1][0][RTW89_KCC][4] = 58, + [1][0][RTW89_ACMA][4] = 42, + [1][0][RTW89_CHILE][4] = 60, + [1][0][RTW89_UKRAINE][4] = 42, + [1][0][RTW89_MEXICO][4] = 78, + [1][0][RTW89_CN][4] = 42, + [1][0][RTW89_QATAR][4] = 42, + [1][0][RTW89_FCC][5] = 78, + [1][0][RTW89_ETSI][5] = 42, + [1][0][RTW89_MKK][5] = 50, + [1][0][RTW89_IC][5] = 78, + [1][0][RTW89_KCC][5] = 58, + [1][0][RTW89_ACMA][5] = 42, + [1][0][RTW89_CHILE][5] = 60, + [1][0][RTW89_UKRAINE][5] = 42, + [1][0][RTW89_MEXICO][5] = 78, + [1][0][RTW89_CN][5] = 42, + [1][0][RTW89_QATAR][5] = 42, + [1][0][RTW89_FCC][6] = 78, + [1][0][RTW89_ETSI][6] = 42, + [1][0][RTW89_MKK][6] = 50, + [1][0][RTW89_IC][6] = 78, + [1][0][RTW89_KCC][6] = 58, + [1][0][RTW89_ACMA][6] = 42, + [1][0][RTW89_CHILE][6] = 60, + [1][0][RTW89_UKRAINE][6] = 42, + [1][0][RTW89_MEXICO][6] = 78, + [1][0][RTW89_CN][6] = 42, + [1][0][RTW89_QATAR][6] = 42, + [1][0][RTW89_FCC][7] = 78, + [1][0][RTW89_ETSI][7] = 42, + [1][0][RTW89_MKK][7] = 50, + [1][0][RTW89_IC][7] = 78, + [1][0][RTW89_KCC][7] = 58, + [1][0][RTW89_ACMA][7] = 42, + [1][0][RTW89_CHILE][7] = 60, + [1][0][RTW89_UKRAINE][7] = 42, + [1][0][RTW89_MEXICO][7] = 78, + [1][0][RTW89_CN][7] = 42, + [1][0][RTW89_QATAR][7] = 42, + [1][0][RTW89_FCC][8] = 78, + [1][0][RTW89_ETSI][8] = 42, + [1][0][RTW89_MKK][8] = 50, + [1][0][RTW89_IC][8] = 78, + [1][0][RTW89_KCC][8] = 58, + [1][0][RTW89_ACMA][8] = 42, + [1][0][RTW89_CHILE][8] = 60, + [1][0][RTW89_UKRAINE][8] = 42, + [1][0][RTW89_MEXICO][8] = 78, + [1][0][RTW89_CN][8] = 42, + [1][0][RTW89_QATAR][8] = 42, + [1][0][RTW89_FCC][9] = 74, + [1][0][RTW89_ETSI][9] = 42, + [1][0][RTW89_MKK][9] = 50, + [1][0][RTW89_IC][9] = 74, + [1][0][RTW89_KCC][9] = 58, + [1][0][RTW89_ACMA][9] = 42, + [1][0][RTW89_CHILE][9] = 60, + [1][0][RTW89_UKRAINE][9] = 42, + [1][0][RTW89_MEXICO][9] = 74, + [1][0][RTW89_CN][9] = 42, + [1][0][RTW89_QATAR][9] = 42, + [1][0][RTW89_FCC][10] = 74, + [1][0][RTW89_ETSI][10] = 42, + [1][0][RTW89_MKK][10] = 50, + [1][0][RTW89_IC][10] = 74, + [1][0][RTW89_KCC][10] = 58, + [1][0][RTW89_ACMA][10] = 42, + [1][0][RTW89_CHILE][10] = 60, + [1][0][RTW89_UKRAINE][10] = 42, + [1][0][RTW89_MEXICO][10] = 74, + [1][0][RTW89_CN][10] = 42, + [1][0][RTW89_QATAR][10] = 42, + [1][0][RTW89_FCC][11] = 64, + [1][0][RTW89_ETSI][11] = 42, + [1][0][RTW89_MKK][11] = 50, + [1][0][RTW89_IC][11] = 64, + [1][0][RTW89_KCC][11] = 58, + [1][0][RTW89_ACMA][11] = 42, + [1][0][RTW89_CHILE][11] = 60, + [1][0][RTW89_UKRAINE][11] = 42, + [1][0][RTW89_MEXICO][11] = 64, + [1][0][RTW89_CN][11] = 42, + [1][0][RTW89_QATAR][11] = 42, + [1][0][RTW89_FCC][12] = 36, + [1][0][RTW89_ETSI][12] = 42, + [1][0][RTW89_MKK][12] = 50, + [1][0][RTW89_IC][12] = 36, + [1][0][RTW89_KCC][12] = 58, + [1][0][RTW89_ACMA][12] = 42, + [1][0][RTW89_CHILE][12] = 36, + [1][0][RTW89_UKRAINE][12] = 42, + [1][0][RTW89_MEXICO][12] = 36, + [1][0][RTW89_CN][12] = 42, + [1][0][RTW89_QATAR][12] = 42, + [1][0][RTW89_FCC][13] = 127, + [1][0][RTW89_ETSI][13] = 127, + [1][0][RTW89_MKK][13] = 127, + [1][0][RTW89_IC][13] = 127, + [1][0][RTW89_KCC][13] = 127, + [1][0][RTW89_ACMA][13] = 127, + [1][0][RTW89_CHILE][13] = 127, + [1][0][RTW89_UKRAINE][13] = 127, + [1][0][RTW89_MEXICO][13] = 127, + [1][0][RTW89_CN][13] = 127, + [1][0][RTW89_QATAR][13] = 127, + [1][1][RTW89_FCC][0] = 66, + [1][1][RTW89_ETSI][0] = 30, + [1][1][RTW89_MKK][0] = 38, + [1][1][RTW89_IC][0] = 66, + [1][1][RTW89_KCC][0] = 44, + [1][1][RTW89_ACMA][0] = 30, + [1][1][RTW89_CHILE][0] = 48, + [1][1][RTW89_UKRAINE][0] = 30, + [1][1][RTW89_MEXICO][0] = 66, + [1][1][RTW89_CN][0] = 30, + [1][1][RTW89_QATAR][0] = 30, + [1][1][RTW89_FCC][1] = 66, + [1][1][RTW89_ETSI][1] = 30, + [1][1][RTW89_MKK][1] = 38, + [1][1][RTW89_IC][1] = 66, + [1][1][RTW89_KCC][1] = 44, + [1][1][RTW89_ACMA][1] = 30, + [1][1][RTW89_CHILE][1] = 48, + [1][1][RTW89_UKRAINE][1] = 30, + [1][1][RTW89_MEXICO][1] = 66, + [1][1][RTW89_CN][1] = 30, + [1][1][RTW89_QATAR][1] = 30, + [1][1][RTW89_FCC][2] = 70, + [1][1][RTW89_ETSI][2] = 30, + [1][1][RTW89_MKK][2] = 38, + [1][1][RTW89_IC][2] = 70, + [1][1][RTW89_KCC][2] = 44, + [1][1][RTW89_ACMA][2] = 30, + [1][1][RTW89_CHILE][2] = 48, + [1][1][RTW89_UKRAINE][2] = 30, + [1][1][RTW89_MEXICO][2] = 70, + [1][1][RTW89_CN][2] = 30, + [1][1][RTW89_QATAR][2] = 30, + [1][1][RTW89_FCC][3] = 74, + [1][1][RTW89_ETSI][3] = 30, + [1][1][RTW89_MKK][3] = 38, + [1][1][RTW89_IC][3] = 74, + [1][1][RTW89_KCC][3] = 44, + [1][1][RTW89_ACMA][3] = 30, + [1][1][RTW89_CHILE][3] = 48, + [1][1][RTW89_UKRAINE][3] = 30, + [1][1][RTW89_MEXICO][3] = 74, + [1][1][RTW89_CN][3] = 30, + [1][1][RTW89_QATAR][3] = 30, + [1][1][RTW89_FCC][4] = 78, + [1][1][RTW89_ETSI][4] = 30, + [1][1][RTW89_MKK][4] = 38, + [1][1][RTW89_IC][4] = 78, + [1][1][RTW89_KCC][4] = 44, + [1][1][RTW89_ACMA][4] = 30, + [1][1][RTW89_CHILE][4] = 48, + [1][1][RTW89_UKRAINE][4] = 30, + [1][1][RTW89_MEXICO][4] = 78, + [1][1][RTW89_CN][4] = 30, + [1][1][RTW89_QATAR][4] = 30, + [1][1][RTW89_FCC][5] = 78, + [1][1][RTW89_ETSI][5] = 30, + [1][1][RTW89_MKK][5] = 38, + [1][1][RTW89_IC][5] = 78, + [1][1][RTW89_KCC][5] = 44, + [1][1][RTW89_ACMA][5] = 30, + [1][1][RTW89_CHILE][5] = 48, + [1][1][RTW89_UKRAINE][5] = 30, + [1][1][RTW89_MEXICO][5] = 78, + [1][1][RTW89_CN][5] = 30, + [1][1][RTW89_QATAR][5] = 30, + [1][1][RTW89_FCC][6] = 78, + [1][1][RTW89_ETSI][6] = 30, + [1][1][RTW89_MKK][6] = 38, + [1][1][RTW89_IC][6] = 78, + [1][1][RTW89_KCC][6] = 44, + [1][1][RTW89_ACMA][6] = 30, + [1][1][RTW89_CHILE][6] = 48, + [1][1][RTW89_UKRAINE][6] = 30, + [1][1][RTW89_MEXICO][6] = 78, + [1][1][RTW89_CN][6] = 30, + [1][1][RTW89_QATAR][6] = 30, + [1][1][RTW89_FCC][7] = 74, + [1][1][RTW89_ETSI][7] = 30, + [1][1][RTW89_MKK][7] = 38, + [1][1][RTW89_IC][7] = 74, + [1][1][RTW89_KCC][7] = 44, + [1][1][RTW89_ACMA][7] = 30, + [1][1][RTW89_CHILE][7] = 48, + [1][1][RTW89_UKRAINE][7] = 30, + [1][1][RTW89_MEXICO][7] = 74, + [1][1][RTW89_CN][7] = 30, + [1][1][RTW89_QATAR][7] = 30, + [1][1][RTW89_FCC][8] = 70, + [1][1][RTW89_ETSI][8] = 30, + [1][1][RTW89_MKK][8] = 38, + [1][1][RTW89_IC][8] = 70, + [1][1][RTW89_KCC][8] = 44, + [1][1][RTW89_ACMA][8] = 30, + [1][1][RTW89_CHILE][8] = 48, + [1][1][RTW89_UKRAINE][8] = 30, + [1][1][RTW89_MEXICO][8] = 70, + [1][1][RTW89_CN][8] = 30, + [1][1][RTW89_QATAR][8] = 30, + [1][1][RTW89_FCC][9] = 66, + [1][1][RTW89_ETSI][9] = 30, + [1][1][RTW89_MKK][9] = 38, + [1][1][RTW89_IC][9] = 66, + [1][1][RTW89_KCC][9] = 44, + [1][1][RTW89_ACMA][9] = 30, + [1][1][RTW89_CHILE][9] = 48, + [1][1][RTW89_UKRAINE][9] = 30, + [1][1][RTW89_MEXICO][9] = 66, + [1][1][RTW89_CN][9] = 30, + [1][1][RTW89_QATAR][9] = 30, + [1][1][RTW89_FCC][10] = 66, + [1][1][RTW89_ETSI][10] = 30, + [1][1][RTW89_MKK][10] = 38, + [1][1][RTW89_IC][10] = 66, + [1][1][RTW89_KCC][10] = 44, + [1][1][RTW89_ACMA][10] = 30, + [1][1][RTW89_CHILE][10] = 48, + [1][1][RTW89_UKRAINE][10] = 30, + [1][1][RTW89_MEXICO][10] = 66, + [1][1][RTW89_CN][10] = 30, + [1][1][RTW89_QATAR][10] = 30, + [1][1][RTW89_FCC][11] = 60, + [1][1][RTW89_ETSI][11] = 30, + [1][1][RTW89_MKK][11] = 38, + [1][1][RTW89_IC][11] = 60, + [1][1][RTW89_KCC][11] = 44, + [1][1][RTW89_ACMA][11] = 30, + [1][1][RTW89_CHILE][11] = 48, + [1][1][RTW89_UKRAINE][11] = 30, + [1][1][RTW89_MEXICO][11] = 60, + [1][1][RTW89_CN][11] = 30, + [1][1][RTW89_QATAR][11] = 30, + [1][1][RTW89_FCC][12] = 32, + [1][1][RTW89_ETSI][12] = 30, + [1][1][RTW89_MKK][12] = 38, + [1][1][RTW89_IC][12] = 32, + [1][1][RTW89_KCC][12] = 44, + [1][1][RTW89_ACMA][12] = 30, + [1][1][RTW89_CHILE][12] = 32, + [1][1][RTW89_UKRAINE][12] = 30, + [1][1][RTW89_MEXICO][12] = 32, + [1][1][RTW89_CN][12] = 30, + [1][1][RTW89_QATAR][12] = 30, + [1][1][RTW89_FCC][13] = 127, + [1][1][RTW89_ETSI][13] = 127, + [1][1][RTW89_MKK][13] = 127, + [1][1][RTW89_IC][13] = 127, + [1][1][RTW89_KCC][13] = 127, + [1][1][RTW89_ACMA][13] = 127, + [1][1][RTW89_CHILE][13] = 127, + [1][1][RTW89_UKRAINE][13] = 127, + [1][1][RTW89_MEXICO][13] = 127, + [1][1][RTW89_CN][13] = 127, + [1][1][RTW89_QATAR][13] = 127, + [2][0][RTW89_FCC][0] = 76, + [2][0][RTW89_ETSI][0] = 52, + [2][0][RTW89_MKK][0] = 64, + [2][0][RTW89_IC][0] = 76, + [2][0][RTW89_KCC][0] = 70, + [2][0][RTW89_ACMA][0] = 52, + [2][0][RTW89_CHILE][0] = 60, + [2][0][RTW89_UKRAINE][0] = 52, + [2][0][RTW89_MEXICO][0] = 76, + [2][0][RTW89_CN][0] = 52, + [2][0][RTW89_QATAR][0] = 52, + [2][0][RTW89_FCC][1] = 76, + [2][0][RTW89_ETSI][1] = 52, + [2][0][RTW89_MKK][1] = 64, + [2][0][RTW89_IC][1] = 76, + [2][0][RTW89_KCC][1] = 70, + [2][0][RTW89_ACMA][1] = 52, + [2][0][RTW89_CHILE][1] = 60, + [2][0][RTW89_UKRAINE][1] = 52, + [2][0][RTW89_MEXICO][1] = 76, + [2][0][RTW89_CN][1] = 52, + [2][0][RTW89_QATAR][1] = 52, + [2][0][RTW89_FCC][2] = 78, + [2][0][RTW89_ETSI][2] = 52, + [2][0][RTW89_MKK][2] = 64, + [2][0][RTW89_IC][2] = 78, + [2][0][RTW89_KCC][2] = 70, + [2][0][RTW89_ACMA][2] = 52, + [2][0][RTW89_CHILE][2] = 60, + [2][0][RTW89_UKRAINE][2] = 52, + [2][0][RTW89_MEXICO][2] = 78, + [2][0][RTW89_CN][2] = 52, + [2][0][RTW89_QATAR][2] = 52, + [2][0][RTW89_FCC][3] = 78, + [2][0][RTW89_ETSI][3] = 52, + [2][0][RTW89_MKK][3] = 64, + [2][0][RTW89_IC][3] = 78, + [2][0][RTW89_KCC][3] = 70, + [2][0][RTW89_ACMA][3] = 52, + [2][0][RTW89_CHILE][3] = 60, + [2][0][RTW89_UKRAINE][3] = 52, + [2][0][RTW89_MEXICO][3] = 78, + [2][0][RTW89_CN][3] = 52, + [2][0][RTW89_QATAR][3] = 52, + [2][0][RTW89_FCC][4] = 78, + [2][0][RTW89_ETSI][4] = 52, + [2][0][RTW89_MKK][4] = 64, + [2][0][RTW89_IC][4] = 78, + [2][0][RTW89_KCC][4] = 70, + [2][0][RTW89_ACMA][4] = 52, + [2][0][RTW89_CHILE][4] = 60, + [2][0][RTW89_UKRAINE][4] = 52, + [2][0][RTW89_MEXICO][4] = 78, + [2][0][RTW89_CN][4] = 52, + [2][0][RTW89_QATAR][4] = 52, + [2][0][RTW89_FCC][5] = 78, + [2][0][RTW89_ETSI][5] = 52, + [2][0][RTW89_MKK][5] = 64, + [2][0][RTW89_IC][5] = 78, + [2][0][RTW89_KCC][5] = 70, + [2][0][RTW89_ACMA][5] = 52, + [2][0][RTW89_CHILE][5] = 60, + [2][0][RTW89_UKRAINE][5] = 52, + [2][0][RTW89_MEXICO][5] = 78, + [2][0][RTW89_CN][5] = 52, + [2][0][RTW89_QATAR][5] = 52, + [2][0][RTW89_FCC][6] = 78, + [2][0][RTW89_ETSI][6] = 52, + [2][0][RTW89_MKK][6] = 64, + [2][0][RTW89_IC][6] = 78, + [2][0][RTW89_KCC][6] = 70, + [2][0][RTW89_ACMA][6] = 52, + [2][0][RTW89_CHILE][6] = 60, + [2][0][RTW89_UKRAINE][6] = 52, + [2][0][RTW89_MEXICO][6] = 78, + [2][0][RTW89_CN][6] = 52, + [2][0][RTW89_QATAR][6] = 52, + [2][0][RTW89_FCC][7] = 78, + [2][0][RTW89_ETSI][7] = 52, + [2][0][RTW89_MKK][7] = 64, + [2][0][RTW89_IC][7] = 78, + [2][0][RTW89_KCC][7] = 70, + [2][0][RTW89_ACMA][7] = 52, + [2][0][RTW89_CHILE][7] = 60, + [2][0][RTW89_UKRAINE][7] = 52, + [2][0][RTW89_MEXICO][7] = 78, + [2][0][RTW89_CN][7] = 52, + [2][0][RTW89_QATAR][7] = 52, + [2][0][RTW89_FCC][8] = 78, + [2][0][RTW89_ETSI][8] = 52, + [2][0][RTW89_MKK][8] = 64, + [2][0][RTW89_IC][8] = 78, + [2][0][RTW89_KCC][8] = 70, + [2][0][RTW89_ACMA][8] = 52, + [2][0][RTW89_CHILE][8] = 60, + [2][0][RTW89_UKRAINE][8] = 52, + [2][0][RTW89_MEXICO][8] = 78, + [2][0][RTW89_CN][8] = 52, + [2][0][RTW89_QATAR][8] = 52, + [2][0][RTW89_FCC][9] = 76, + [2][0][RTW89_ETSI][9] = 52, + [2][0][RTW89_MKK][9] = 64, + [2][0][RTW89_IC][9] = 76, + [2][0][RTW89_KCC][9] = 70, + [2][0][RTW89_ACMA][9] = 52, + [2][0][RTW89_CHILE][9] = 60, + [2][0][RTW89_UKRAINE][9] = 52, + [2][0][RTW89_MEXICO][9] = 76, + [2][0][RTW89_CN][9] = 52, + [2][0][RTW89_QATAR][9] = 52, + [2][0][RTW89_FCC][10] = 76, + [2][0][RTW89_ETSI][10] = 52, + [2][0][RTW89_MKK][10] = 64, + [2][0][RTW89_IC][10] = 76, + [2][0][RTW89_KCC][10] = 70, + [2][0][RTW89_ACMA][10] = 52, + [2][0][RTW89_CHILE][10] = 60, + [2][0][RTW89_UKRAINE][10] = 52, + [2][0][RTW89_MEXICO][10] = 76, + [2][0][RTW89_CN][10] = 52, + [2][0][RTW89_QATAR][10] = 52, + [2][0][RTW89_FCC][11] = 68, + [2][0][RTW89_ETSI][11] = 52, + [2][0][RTW89_MKK][11] = 64, + [2][0][RTW89_IC][11] = 68, + [2][0][RTW89_KCC][11] = 70, + [2][0][RTW89_ACMA][11] = 52, + [2][0][RTW89_CHILE][11] = 60, + [2][0][RTW89_UKRAINE][11] = 52, + [2][0][RTW89_MEXICO][11] = 68, + [2][0][RTW89_CN][11] = 52, + [2][0][RTW89_QATAR][11] = 52, + [2][0][RTW89_FCC][12] = 40, + [2][0][RTW89_ETSI][12] = 52, + [2][0][RTW89_MKK][12] = 64, + [2][0][RTW89_IC][12] = 40, + [2][0][RTW89_KCC][12] = 70, + [2][0][RTW89_ACMA][12] = 52, + [2][0][RTW89_CHILE][12] = 40, + [2][0][RTW89_UKRAINE][12] = 52, + [2][0][RTW89_MEXICO][12] = 40, + [2][0][RTW89_CN][12] = 52, + [2][0][RTW89_QATAR][12] = 52, + [2][0][RTW89_FCC][13] = 127, + [2][0][RTW89_ETSI][13] = 127, + [2][0][RTW89_MKK][13] = 127, + [2][0][RTW89_IC][13] = 127, + [2][0][RTW89_KCC][13] = 127, + [2][0][RTW89_ACMA][13] = 127, + [2][0][RTW89_CHILE][13] = 127, + [2][0][RTW89_UKRAINE][13] = 127, + [2][0][RTW89_MEXICO][13] = 127, + [2][0][RTW89_CN][13] = 127, + [2][0][RTW89_QATAR][13] = 127, + [2][1][RTW89_FCC][0] = 68, + [2][1][RTW89_ETSI][0] = 40, + [2][1][RTW89_MKK][0] = 52, + [2][1][RTW89_IC][0] = 68, + [2][1][RTW89_KCC][0] = 56, + [2][1][RTW89_ACMA][0] = 40, + [2][1][RTW89_CHILE][0] = 48, + [2][1][RTW89_UKRAINE][0] = 40, + [2][1][RTW89_MEXICO][0] = 68, + [2][1][RTW89_CN][0] = 40, + [2][1][RTW89_QATAR][0] = 40, + [2][1][RTW89_FCC][1] = 68, + [2][1][RTW89_ETSI][1] = 40, + [2][1][RTW89_MKK][1] = 52, + [2][1][RTW89_IC][1] = 68, + [2][1][RTW89_KCC][1] = 56, + [2][1][RTW89_ACMA][1] = 40, + [2][1][RTW89_CHILE][1] = 48, + [2][1][RTW89_UKRAINE][1] = 40, + [2][1][RTW89_MEXICO][1] = 68, + [2][1][RTW89_CN][1] = 40, + [2][1][RTW89_QATAR][1] = 40, + [2][1][RTW89_FCC][2] = 72, + [2][1][RTW89_ETSI][2] = 40, + [2][1][RTW89_MKK][2] = 52, + [2][1][RTW89_IC][2] = 72, + [2][1][RTW89_KCC][2] = 56, + [2][1][RTW89_ACMA][2] = 40, + [2][1][RTW89_CHILE][2] = 48, + [2][1][RTW89_UKRAINE][2] = 40, + [2][1][RTW89_MEXICO][2] = 72, + [2][1][RTW89_CN][2] = 40, + [2][1][RTW89_QATAR][2] = 40, + [2][1][RTW89_FCC][3] = 76, + [2][1][RTW89_ETSI][3] = 40, + [2][1][RTW89_MKK][3] = 52, + [2][1][RTW89_IC][3] = 76, + [2][1][RTW89_KCC][3] = 56, + [2][1][RTW89_ACMA][3] = 40, + [2][1][RTW89_CHILE][3] = 48, + [2][1][RTW89_UKRAINE][3] = 40, + [2][1][RTW89_MEXICO][3] = 76, + [2][1][RTW89_CN][3] = 40, + [2][1][RTW89_QATAR][3] = 40, + [2][1][RTW89_FCC][4] = 78, + [2][1][RTW89_ETSI][4] = 40, + [2][1][RTW89_MKK][4] = 52, + [2][1][RTW89_IC][4] = 78, + [2][1][RTW89_KCC][4] = 56, + [2][1][RTW89_ACMA][4] = 40, + [2][1][RTW89_CHILE][4] = 48, + [2][1][RTW89_UKRAINE][4] = 40, + [2][1][RTW89_MEXICO][4] = 78, + [2][1][RTW89_CN][4] = 40, + [2][1][RTW89_QATAR][4] = 40, + [2][1][RTW89_FCC][5] = 78, + [2][1][RTW89_ETSI][5] = 40, + [2][1][RTW89_MKK][5] = 52, + [2][1][RTW89_IC][5] = 78, + [2][1][RTW89_KCC][5] = 56, + [2][1][RTW89_ACMA][5] = 40, + [2][1][RTW89_CHILE][5] = 48, + [2][1][RTW89_UKRAINE][5] = 40, + [2][1][RTW89_MEXICO][5] = 78, + [2][1][RTW89_CN][5] = 40, + [2][1][RTW89_QATAR][5] = 40, + [2][1][RTW89_FCC][6] = 78, + [2][1][RTW89_ETSI][6] = 40, + [2][1][RTW89_MKK][6] = 52, + [2][1][RTW89_IC][6] = 78, + [2][1][RTW89_KCC][6] = 56, + [2][1][RTW89_ACMA][6] = 40, + [2][1][RTW89_CHILE][6] = 48, + [2][1][RTW89_UKRAINE][6] = 40, + [2][1][RTW89_MEXICO][6] = 78, + [2][1][RTW89_CN][6] = 40, + [2][1][RTW89_QATAR][6] = 40, + [2][1][RTW89_FCC][7] = 78, + [2][1][RTW89_ETSI][7] = 40, + [2][1][RTW89_MKK][7] = 52, + [2][1][RTW89_IC][7] = 78, + [2][1][RTW89_KCC][7] = 56, + [2][1][RTW89_ACMA][7] = 40, + [2][1][RTW89_CHILE][7] = 48, + [2][1][RTW89_UKRAINE][7] = 40, + [2][1][RTW89_MEXICO][7] = 78, + [2][1][RTW89_CN][7] = 40, + [2][1][RTW89_QATAR][7] = 40, + [2][1][RTW89_FCC][8] = 74, + [2][1][RTW89_ETSI][8] = 40, + [2][1][RTW89_MKK][8] = 52, + [2][1][RTW89_IC][8] = 74, + [2][1][RTW89_KCC][8] = 56, + [2][1][RTW89_ACMA][8] = 40, + [2][1][RTW89_CHILE][8] = 48, + [2][1][RTW89_UKRAINE][8] = 40, + [2][1][RTW89_MEXICO][8] = 74, + [2][1][RTW89_CN][8] = 40, + [2][1][RTW89_QATAR][8] = 40, + [2][1][RTW89_FCC][9] = 70, + [2][1][RTW89_ETSI][9] = 40, + [2][1][RTW89_MKK][9] = 52, + [2][1][RTW89_IC][9] = 70, + [2][1][RTW89_KCC][9] = 56, + [2][1][RTW89_ACMA][9] = 40, + [2][1][RTW89_CHILE][9] = 48, + [2][1][RTW89_UKRAINE][9] = 40, + [2][1][RTW89_MEXICO][9] = 70, + [2][1][RTW89_CN][9] = 40, + [2][1][RTW89_QATAR][9] = 40, + [2][1][RTW89_FCC][10] = 70, + [2][1][RTW89_ETSI][10] = 40, + [2][1][RTW89_MKK][10] = 52, + [2][1][RTW89_IC][10] = 70, + [2][1][RTW89_KCC][10] = 56, + [2][1][RTW89_ACMA][10] = 40, + [2][1][RTW89_CHILE][10] = 48, + [2][1][RTW89_UKRAINE][10] = 40, + [2][1][RTW89_MEXICO][10] = 70, + [2][1][RTW89_CN][10] = 40, + [2][1][RTW89_QATAR][10] = 40, + [2][1][RTW89_FCC][11] = 48, + [2][1][RTW89_ETSI][11] = 40, + [2][1][RTW89_MKK][11] = 52, + [2][1][RTW89_IC][11] = 48, + [2][1][RTW89_KCC][11] = 56, + [2][1][RTW89_ACMA][11] = 40, + [2][1][RTW89_CHILE][11] = 48, + [2][1][RTW89_UKRAINE][11] = 40, + [2][1][RTW89_MEXICO][11] = 48, + [2][1][RTW89_CN][11] = 40, + [2][1][RTW89_QATAR][11] = 40, + [2][1][RTW89_FCC][12] = 26, + [2][1][RTW89_ETSI][12] = 40, + [2][1][RTW89_MKK][12] = 52, + [2][1][RTW89_IC][12] = 26, + [2][1][RTW89_KCC][12] = 56, + [2][1][RTW89_ACMA][12] = 40, + [2][1][RTW89_CHILE][12] = 26, + [2][1][RTW89_UKRAINE][12] = 40, + [2][1][RTW89_MEXICO][12] = 26, + [2][1][RTW89_CN][12] = 40, + [2][1][RTW89_QATAR][12] = 40, + [2][1][RTW89_FCC][13] = 127, + [2][1][RTW89_ETSI][13] = 127, + [2][1][RTW89_MKK][13] = 127, + [2][1][RTW89_IC][13] = 127, + [2][1][RTW89_KCC][13] = 127, + [2][1][RTW89_ACMA][13] = 127, + [2][1][RTW89_CHILE][13] = 127, + [2][1][RTW89_UKRAINE][13] = 127, + [2][1][RTW89_MEXICO][13] = 127, + [2][1][RTW89_CN][13] = 127, + [2][1][RTW89_QATAR][13] = 127, }; const s8 rtw89_8852a_txpwr_lmt_ru_5g[RTW89_RU_NUM][RTW89_NTX_NUM] [RTW89_REGD_NUM][RTW89_5G_CH_NUM] = { - [0][0][0][0] = 22, - [0][0][0][2] = 22, - [0][0][0][4] = 22, - [0][0][0][6] = 22, - [0][0][0][8] = 24, - [0][0][0][10] = 24, - [0][0][0][12] = 24, - [0][0][0][14] = 24, - [0][0][0][15] = 24, - [0][0][0][17] = 24, - [0][0][0][19] = 24, - [0][0][0][21] = 24, - [0][0][0][23] = 24, - [0][0][0][25] = 24, - [0][0][0][27] = 24, - [0][0][0][29] = 24, - [0][0][0][31] = 24, - [0][0][0][33] = 24, - [0][0][0][35] = 24, - [0][0][0][37] = 24, - [0][0][0][38] = 28, - [0][0][0][40] = 28, - [0][0][0][42] = 28, - [0][0][0][44] = 28, - [0][0][0][46] = 28, - [0][1][0][0] = 8, - [0][1][0][2] = 8, - [0][1][0][4] = 8, - [0][1][0][6] = 8, - [0][1][0][8] = 12, - [0][1][0][10] = 12, - [0][1][0][12] = 12, - [0][1][0][14] = 12, - [0][1][0][15] = 12, - [0][1][0][17] = 12, - [0][1][0][19] = 12, - [0][1][0][21] = 12, - [0][1][0][23] = 12, - [0][1][0][25] = 12, - [0][1][0][27] = 12, - [0][1][0][29] = 12, - [0][1][0][31] = 12, - [0][1][0][33] = 12, - [0][1][0][35] = 12, - [0][1][0][37] = 12, - [0][1][0][38] = 16, - [0][1][0][40] = 16, - [0][1][0][42] = 16, - [0][1][0][44] = 16, - [0][1][0][46] = 16, - [1][0][0][0] = 30, - [1][0][0][2] = 30, - [1][0][0][4] = 30, - [1][0][0][6] = 30, - [1][0][0][8] = 36, - [1][0][0][10] = 36, - [1][0][0][12] = 36, - [1][0][0][14] = 36, - [1][0][0][15] = 36, - [1][0][0][17] = 36, - [1][0][0][19] = 36, - [1][0][0][21] = 36, - [1][0][0][23] = 36, - [1][0][0][25] = 36, - [1][0][0][27] = 36, - [1][0][0][29] = 36, - [1][0][0][31] = 36, - [1][0][0][33] = 36, - [1][0][0][35] = 36, - [1][0][0][37] = 36, - [1][0][0][38] = 28, - [1][0][0][40] = 28, - [1][0][0][42] = 28, - [1][0][0][44] = 28, - [1][0][0][46] = 28, - [1][1][0][0] = 18, - [1][1][0][2] = 18, - [1][1][0][4] = 18, - [1][1][0][6] = 18, - [1][1][0][8] = 22, - [1][1][0][10] = 22, - [1][1][0][12] = 22, - [1][1][0][14] = 22, - [1][1][0][15] = 22, - [1][1][0][17] = 22, - [1][1][0][19] = 22, - [1][1][0][21] = 22, - [1][1][0][23] = 22, - [1][1][0][25] = 22, - [1][1][0][27] = 22, - [1][1][0][29] = 22, - [1][1][0][31] = 22, - [1][1][0][33] = 22, - [1][1][0][35] = 22, - [1][1][0][37] = 22, - [1][1][0][38] = 16, - [1][1][0][40] = 16, - [1][1][0][42] = 16, - [1][1][0][44] = 16, - [1][1][0][46] = 16, - [2][0][0][0] = 30, - [2][0][0][2] = 30, - [2][0][0][4] = 30, - [2][0][0][6] = 30, - [2][0][0][8] = 46, - [2][0][0][10] = 46, - [2][0][0][12] = 46, - [2][0][0][14] = 46, - [2][0][0][15] = 46, - [2][0][0][17] = 46, - [2][0][0][19] = 46, - [2][0][0][21] = 46, - [2][0][0][23] = 46, - [2][0][0][25] = 46, - [2][0][0][27] = 46, - [2][0][0][29] = 46, - [2][0][0][31] = 46, - [2][0][0][33] = 46, - [2][0][0][35] = 46, - [2][0][0][37] = 46, - [2][0][0][38] = 28, - [2][0][0][40] = 28, - [2][0][0][42] = 28, - [2][0][0][44] = 28, - [2][0][0][46] = 28, - [2][1][0][0] = 18, - [2][1][0][2] = 18, - [2][1][0][4] = 18, - [2][1][0][6] = 18, - [2][1][0][8] = 32, - [2][1][0][10] = 32, - [2][1][0][12] = 32, - [2][1][0][14] = 32, - [2][1][0][15] = 32, - [2][1][0][17] = 32, - [2][1][0][19] = 32, - [2][1][0][21] = 32, - [2][1][0][23] = 32, - [2][1][0][25] = 32, - [2][1][0][27] = 32, - [2][1][0][29] = 32, - [2][1][0][31] = 32, - [2][1][0][33] = 32, - [2][1][0][35] = 32, - [2][1][0][37] = 32, - [2][1][0][38] = 16, - [2][1][0][40] = 16, - [2][1][0][42] = 16, - [2][1][0][44] = 16, - [2][1][0][46] = 16, - [0][0][2][0] = 48, - [0][0][1][0] = 24, - [0][0][3][0] = 26, - [0][0][5][0] = 22, - [0][0][6][0] = 24, - [0][0][9][0] = 24, - [0][0][8][0] = 30, - [0][0][11][0] = 24, - [0][0][2][2] = 48, - [0][0][1][2] = 24, - [0][0][3][2] = 26, - [0][0][5][2] = 22, - [0][0][6][2] = 24, - [0][0][9][2] = 24, - [0][0][8][2] = 30, - [0][0][11][2] = 24, - [0][0][2][4] = 48, - [0][0][1][4] = 24, - [0][0][3][4] = 26, - [0][0][5][4] = 22, - [0][0][6][4] = 24, - [0][0][9][4] = 24, - [0][0][8][4] = 30, - [0][0][11][4] = 24, - [0][0][2][6] = 48, - [0][0][1][6] = 24, - [0][0][3][6] = 26, - [0][0][5][6] = 22, - [0][0][6][6] = 24, - [0][0][9][6] = 24, - [0][0][8][6] = 30, - [0][0][11][6] = 24, - [0][0][2][8] = 48, - [0][0][1][8] = 24, - [0][0][3][8] = 26, - [0][0][5][8] = 48, - [0][0][6][8] = 24, - [0][0][9][8] = 24, - [0][0][8][8] = 54, - [0][0][11][8] = 24, - [0][0][2][10] = 48, - [0][0][1][10] = 24, - [0][0][3][10] = 26, - [0][0][5][10] = 48, - [0][0][6][10] = 24, - [0][0][9][10] = 24, - [0][0][8][10] = 54, - [0][0][11][10] = 24, - [0][0][2][12] = 48, - [0][0][1][12] = 24, - [0][0][3][12] = 26, - [0][0][5][12] = 48, - [0][0][6][12] = 24, - [0][0][9][12] = 24, - [0][0][8][12] = 54, - [0][0][11][12] = 24, - [0][0][2][14] = 48, - [0][0][1][14] = 24, - [0][0][3][14] = 26, - [0][0][5][14] = 48, - [0][0][6][14] = 24, - [0][0][9][14] = 24, - [0][0][8][14] = 54, - [0][0][11][14] = 24, - [0][0][2][15] = 48, - [0][0][1][15] = 24, - [0][0][3][15] = 44, - [0][0][5][15] = 48, - [0][0][6][15] = 24, - [0][0][9][15] = 24, - [0][0][8][15] = 54, - [0][0][11][15] = 24, - [0][0][2][17] = 48, - [0][0][1][17] = 24, - [0][0][3][17] = 44, - [0][0][5][17] = 48, - [0][0][6][17] = 24, - [0][0][9][17] = 24, - [0][0][8][17] = 54, - [0][0][11][17] = 24, - [0][0][2][19] = 48, - [0][0][1][19] = 24, - [0][0][3][19] = 44, - [0][0][5][19] = 48, - [0][0][6][19] = 24, - [0][0][9][19] = 24, - [0][0][8][19] = 54, - [0][0][11][19] = 24, - [0][0][2][21] = 48, - [0][0][1][21] = 24, - [0][0][3][21] = 44, - [0][0][5][21] = 48, - [0][0][6][21] = 24, - [0][0][9][21] = 24, - [0][0][8][21] = 54, - [0][0][11][21] = 24, - [0][0][2][23] = 48, - [0][0][1][23] = 24, - [0][0][3][23] = 44, - [0][0][5][23] = 48, - [0][0][6][23] = 24, - [0][0][9][23] = 24, - [0][0][8][23] = 54, - [0][0][11][23] = 24, - [0][0][2][25] = 48, - [0][0][1][25] = 24, - [0][0][3][25] = 44, - [0][0][5][25] = 127, - [0][0][6][25] = 24, - [0][0][9][25] = 127, - [0][0][8][25] = 54, - [0][0][11][25] = 24, - [0][0][2][27] = 48, - [0][0][1][27] = 24, - [0][0][3][27] = 44, - [0][0][5][27] = 127, - [0][0][6][27] = 24, - [0][0][9][27] = 127, - [0][0][8][27] = 54, - [0][0][11][27] = 24, - [0][0][2][29] = 48, - [0][0][1][29] = 24, - [0][0][3][29] = 44, - [0][0][5][29] = 127, - [0][0][6][29] = 24, - [0][0][9][29] = 127, - [0][0][8][29] = 54, - [0][0][11][29] = 24, - [0][0][2][31] = 48, - [0][0][1][31] = 24, - [0][0][3][31] = 44, - [0][0][5][31] = 48, - [0][0][6][31] = 24, - [0][0][9][31] = 24, - [0][0][8][31] = 54, - [0][0][11][31] = 24, - [0][0][2][33] = 48, - [0][0][1][33] = 24, - [0][0][3][33] = 44, - [0][0][5][33] = 48, - [0][0][6][33] = 24, - [0][0][9][33] = 24, - [0][0][8][33] = 54, - [0][0][11][33] = 24, - [0][0][2][35] = 48, - [0][0][1][35] = 24, - [0][0][3][35] = 44, - [0][0][5][35] = 48, - [0][0][6][35] = 24, - [0][0][9][35] = 24, - [0][0][8][35] = 54, - [0][0][11][35] = 24, - [0][0][2][37] = 48, - [0][0][1][37] = 127, - [0][0][3][37] = 44, - [0][0][5][37] = 48, - [0][0][6][37] = 24, - [0][0][9][37] = 48, - [0][0][8][37] = 54, - [0][0][11][37] = 127, - [0][0][2][38] = 76, - [0][0][1][38] = 28, - [0][0][3][38] = 127, - [0][0][5][38] = 76, - [0][0][6][38] = 28, - [0][0][9][38] = 76, - [0][0][8][38] = 54, - [0][0][11][38] = 28, - [0][0][2][40] = 76, - [0][0][1][40] = 28, - [0][0][3][40] = 127, - [0][0][5][40] = 76, - [0][0][6][40] = 28, - [0][0][9][40] = 76, - [0][0][8][40] = 54, - [0][0][11][40] = 28, - [0][0][2][42] = 76, - [0][0][1][42] = 28, - [0][0][3][42] = 127, - [0][0][5][42] = 76, - [0][0][6][42] = 28, - [0][0][9][42] = 76, - [0][0][8][42] = 54, - [0][0][11][42] = 28, - [0][0][2][44] = 76, - [0][0][1][44] = 28, - [0][0][3][44] = 127, - [0][0][5][44] = 76, - [0][0][6][44] = 28, - [0][0][9][44] = 76, - [0][0][8][44] = 54, - [0][0][11][44] = 28, - [0][0][2][46] = 76, - [0][0][1][46] = 28, - [0][0][3][46] = 127, - [0][0][5][46] = 76, - [0][0][6][46] = 28, - [0][0][9][46] = 76, - [0][0][8][46] = 54, - [0][0][11][46] = 28, - [0][1][2][0] = 36, - [0][1][1][0] = 12, - [0][1][3][0] = 14, - [0][1][5][0] = 8, - [0][1][6][0] = 12, - [0][1][9][0] = 12, - [0][1][8][0] = 18, - [0][1][11][0] = 12, - [0][1][2][2] = 36, - [0][1][1][2] = 12, - [0][1][3][2] = 14, - [0][1][5][2] = 8, - [0][1][6][2] = 12, - [0][1][9][2] = 12, - [0][1][8][2] = 18, - [0][1][11][2] = 12, - [0][1][2][4] = 36, - [0][1][1][4] = 12, - [0][1][3][4] = 14, - [0][1][5][4] = 8, - [0][1][6][4] = 12, - [0][1][9][4] = 12, - [0][1][8][4] = 18, - [0][1][11][4] = 12, - [0][1][2][6] = 36, - [0][1][1][6] = 12, - [0][1][3][6] = 14, - [0][1][5][6] = 8, - [0][1][6][6] = 12, - [0][1][9][6] = 12, - [0][1][8][6] = 18, - [0][1][11][6] = 12, - [0][1][2][8] = 36, - [0][1][1][8] = 12, - [0][1][3][8] = 14, - [0][1][5][8] = 36, - [0][1][6][8] = 12, - [0][1][9][8] = 12, - [0][1][8][8] = 42, - [0][1][11][8] = 12, - [0][1][2][10] = 36, - [0][1][1][10] = 12, - [0][1][3][10] = 14, - [0][1][5][10] = 36, - [0][1][6][10] = 12, - [0][1][9][10] = 12, - [0][1][8][10] = 42, - [0][1][11][10] = 12, - [0][1][2][12] = 36, - [0][1][1][12] = 12, - [0][1][3][12] = 14, - [0][1][5][12] = 36, - [0][1][6][12] = 12, - [0][1][9][12] = 12, - [0][1][8][12] = 42, - [0][1][11][12] = 12, - [0][1][2][14] = 36, - [0][1][1][14] = 12, - [0][1][3][14] = 14, - [0][1][5][14] = 36, - [0][1][6][14] = 12, - [0][1][9][14] = 12, - [0][1][8][14] = 42, - [0][1][11][14] = 12, - [0][1][2][15] = 36, - [0][1][1][15] = 12, - [0][1][3][15] = 32, - [0][1][5][15] = 36, - [0][1][6][15] = 12, - [0][1][9][15] = 12, - [0][1][8][15] = 42, - [0][1][11][15] = 12, - [0][1][2][17] = 36, - [0][1][1][17] = 12, - [0][1][3][17] = 32, - [0][1][5][17] = 36, - [0][1][6][17] = 12, - [0][1][9][17] = 12, - [0][1][8][17] = 42, - [0][1][11][17] = 12, - [0][1][2][19] = 36, - [0][1][1][19] = 12, - [0][1][3][19] = 32, - [0][1][5][19] = 36, - [0][1][6][19] = 12, - [0][1][9][19] = 12, - [0][1][8][19] = 42, - [0][1][11][19] = 12, - [0][1][2][21] = 36, - [0][1][1][21] = 12, - [0][1][3][21] = 32, - [0][1][5][21] = 36, - [0][1][6][21] = 12, - [0][1][9][21] = 12, - [0][1][8][21] = 42, - [0][1][11][21] = 12, - [0][1][2][23] = 36, - [0][1][1][23] = 12, - [0][1][3][23] = 32, - [0][1][5][23] = 36, - [0][1][6][23] = 12, - [0][1][9][23] = 12, - [0][1][8][23] = 42, - [0][1][11][23] = 12, - [0][1][2][25] = 36, - [0][1][1][25] = 12, - [0][1][3][25] = 32, - [0][1][5][25] = 127, - [0][1][6][25] = 12, - [0][1][9][25] = 127, - [0][1][8][25] = 42, - [0][1][11][25] = 12, - [0][1][2][27] = 36, - [0][1][1][27] = 12, - [0][1][3][27] = 32, - [0][1][5][27] = 127, - [0][1][6][27] = 12, - [0][1][9][27] = 127, - [0][1][8][27] = 42, - [0][1][11][27] = 12, - [0][1][2][29] = 36, - [0][1][1][29] = 12, - [0][1][3][29] = 32, - [0][1][5][29] = 127, - [0][1][6][29] = 12, - [0][1][9][29] = 127, - [0][1][8][29] = 42, - [0][1][11][29] = 12, - [0][1][2][31] = 36, - [0][1][1][31] = 12, - [0][1][3][31] = 32, - [0][1][5][31] = 36, - [0][1][6][31] = 12, - [0][1][9][31] = 12, - [0][1][8][31] = 42, - [0][1][11][31] = 12, - [0][1][2][33] = 36, - [0][1][1][33] = 12, - [0][1][3][33] = 32, - [0][1][5][33] = 36, - [0][1][6][33] = 12, - [0][1][9][33] = 12, - [0][1][8][33] = 42, - [0][1][11][33] = 12, - [0][1][2][35] = 36, - [0][1][1][35] = 12, - [0][1][3][35] = 32, - [0][1][5][35] = 36, - [0][1][6][35] = 12, - [0][1][9][35] = 12, - [0][1][8][35] = 42, - [0][1][11][35] = 12, - [0][1][2][37] = 36, - [0][1][1][37] = 127, - [0][1][3][37] = 32, - [0][1][5][37] = 36, - [0][1][6][37] = 12, - [0][1][9][37] = 36, - [0][1][8][37] = 42, - [0][1][11][37] = 127, - [0][1][2][38] = 72, - [0][1][1][38] = 16, - [0][1][3][38] = 127, - [0][1][5][38] = 72, - [0][1][6][38] = 16, - [0][1][9][38] = 76, - [0][1][8][38] = 42, - [0][1][11][38] = 16, - [0][1][2][40] = 76, - [0][1][1][40] = 16, - [0][1][3][40] = 127, - [0][1][5][40] = 76, - [0][1][6][40] = 16, - [0][1][9][40] = 76, - [0][1][8][40] = 42, - [0][1][11][40] = 16, - [0][1][2][42] = 76, - [0][1][1][42] = 16, - [0][1][3][42] = 127, - [0][1][5][42] = 76, - [0][1][6][42] = 16, - [0][1][9][42] = 76, - [0][1][8][42] = 42, - [0][1][11][42] = 16, - [0][1][2][44] = 76, - [0][1][1][44] = 16, - [0][1][3][44] = 127, - [0][1][5][44] = 76, - [0][1][6][44] = 16, - [0][1][9][44] = 76, - [0][1][8][44] = 42, - [0][1][11][44] = 16, - [0][1][2][46] = 76, - [0][1][1][46] = 16, - [0][1][3][46] = 127, - [0][1][5][46] = 76, - [0][1][6][46] = 16, - [0][1][9][46] = 76, - [0][1][8][46] = 42, - [0][1][11][46] = 16, - [1][0][2][0] = 62, - [1][0][1][0] = 36, - [1][0][3][0] = 36, - [1][0][5][0] = 34, - [1][0][6][0] = 36, - [1][0][9][0] = 36, - [1][0][8][0] = 30, - [1][0][11][0] = 36, - [1][0][2][2] = 62, - [1][0][1][2] = 36, - [1][0][3][2] = 36, - [1][0][5][2] = 34, - [1][0][6][2] = 36, - [1][0][9][2] = 36, - [1][0][8][2] = 30, - [1][0][11][2] = 36, - [1][0][2][4] = 62, - [1][0][1][4] = 36, - [1][0][3][4] = 36, - [1][0][5][4] = 34, - [1][0][6][4] = 36, - [1][0][9][4] = 36, - [1][0][8][4] = 30, - [1][0][11][4] = 36, - [1][0][2][6] = 62, - [1][0][1][6] = 36, - [1][0][3][6] = 36, - [1][0][5][6] = 34, - [1][0][6][6] = 36, - [1][0][9][6] = 36, - [1][0][8][6] = 30, - [1][0][11][6] = 36, - [1][0][2][8] = 62, - [1][0][1][8] = 36, - [1][0][3][8] = 36, - [1][0][5][8] = 62, - [1][0][6][8] = 36, - [1][0][9][8] = 36, - [1][0][8][8] = 54, - [1][0][11][8] = 36, - [1][0][2][10] = 62, - [1][0][1][10] = 36, - [1][0][3][10] = 36, - [1][0][5][10] = 62, - [1][0][6][10] = 36, - [1][0][9][10] = 36, - [1][0][8][10] = 54, - [1][0][11][10] = 36, - [1][0][2][12] = 62, - [1][0][1][12] = 36, - [1][0][3][12] = 36, - [1][0][5][12] = 62, - [1][0][6][12] = 36, - [1][0][9][12] = 36, - [1][0][8][12] = 54, - [1][0][11][12] = 36, - [1][0][2][14] = 62, - [1][0][1][14] = 36, - [1][0][3][14] = 36, - [1][0][5][14] = 62, - [1][0][6][14] = 36, - [1][0][9][14] = 36, - [1][0][8][14] = 54, - [1][0][11][14] = 36, - [1][0][2][15] = 62, - [1][0][1][15] = 36, - [1][0][3][15] = 58, - [1][0][5][15] = 62, - [1][0][6][15] = 36, - [1][0][9][15] = 36, - [1][0][8][15] = 54, - [1][0][11][15] = 36, - [1][0][2][17] = 62, - [1][0][1][17] = 36, - [1][0][3][17] = 58, - [1][0][5][17] = 62, - [1][0][6][17] = 36, - [1][0][9][17] = 36, - [1][0][8][17] = 54, - [1][0][11][17] = 36, - [1][0][2][19] = 62, - [1][0][1][19] = 36, - [1][0][3][19] = 58, - [1][0][5][19] = 62, - [1][0][6][19] = 36, - [1][0][9][19] = 36, - [1][0][8][19] = 54, - [1][0][11][19] = 36, - [1][0][2][21] = 62, - [1][0][1][21] = 36, - [1][0][3][21] = 58, - [1][0][5][21] = 62, - [1][0][6][21] = 36, - [1][0][9][21] = 36, - [1][0][8][21] = 54, - [1][0][11][21] = 36, - [1][0][2][23] = 62, - [1][0][1][23] = 36, - [1][0][3][23] = 58, - [1][0][5][23] = 62, - [1][0][6][23] = 36, - [1][0][9][23] = 36, - [1][0][8][23] = 54, - [1][0][11][23] = 36, - [1][0][2][25] = 62, - [1][0][1][25] = 36, - [1][0][3][25] = 58, - [1][0][5][25] = 127, - [1][0][6][25] = 36, - [1][0][9][25] = 127, - [1][0][8][25] = 54, - [1][0][11][25] = 36, - [1][0][2][27] = 62, - [1][0][1][27] = 36, - [1][0][3][27] = 58, - [1][0][5][27] = 127, - [1][0][6][27] = 36, - [1][0][9][27] = 127, - [1][0][8][27] = 54, - [1][0][11][27] = 36, - [1][0][2][29] = 62, - [1][0][1][29] = 36, - [1][0][3][29] = 58, - [1][0][5][29] = 127, - [1][0][6][29] = 36, - [1][0][9][29] = 127, - [1][0][8][29] = 54, - [1][0][11][29] = 36, - [1][0][2][31] = 62, - [1][0][1][31] = 36, - [1][0][3][31] = 58, - [1][0][5][31] = 62, - [1][0][6][31] = 36, - [1][0][9][31] = 36, - [1][0][8][31] = 54, - [1][0][11][31] = 36, - [1][0][2][33] = 62, - [1][0][1][33] = 36, - [1][0][3][33] = 58, - [1][0][5][33] = 62, - [1][0][6][33] = 36, - [1][0][9][33] = 36, - [1][0][8][33] = 54, - [1][0][11][33] = 36, - [1][0][2][35] = 62, - [1][0][1][35] = 36, - [1][0][3][35] = 58, - [1][0][5][35] = 62, - [1][0][6][35] = 36, - [1][0][9][35] = 36, - [1][0][8][35] = 54, - [1][0][11][35] = 36, - [1][0][2][37] = 56, - [1][0][1][37] = 62, - [1][0][3][37] = 127, - [1][0][5][37] = 58, - [1][0][6][37] = 62, - [1][0][9][37] = 36, - [1][0][8][37] = 62, - [1][0][11][37] = 54, - [1][0][2][38] = 76, - [1][0][1][38] = 28, - [1][0][3][38] = 127, - [1][0][5][38] = 76, - [1][0][6][38] = 28, - [1][0][9][38] = 76, - [1][0][8][38] = 54, - [1][0][11][38] = 28, - [1][0][2][40] = 76, - [1][0][1][40] = 28, - [1][0][3][40] = 127, - [1][0][5][40] = 76, - [1][0][6][40] = 28, - [1][0][9][40] = 76, - [1][0][8][40] = 54, - [1][0][11][40] = 28, - [1][0][2][42] = 76, - [1][0][1][42] = 28, - [1][0][3][42] = 127, - [1][0][5][42] = 76, - [1][0][6][42] = 28, - [1][0][9][42] = 76, - [1][0][8][42] = 54, - [1][0][11][42] = 28, - [1][0][2][44] = 76, - [1][0][1][44] = 28, - [1][0][3][44] = 127, - [1][0][5][44] = 76, - [1][0][6][44] = 28, - [1][0][9][44] = 76, - [1][0][8][44] = 54, - [1][0][11][44] = 28, - [1][0][2][46] = 76, - [1][0][1][46] = 28, - [1][0][3][46] = 127, - [1][0][5][46] = 76, - [1][0][6][46] = 28, - [1][0][9][46] = 76, - [1][0][8][46] = 54, - [1][0][11][46] = 28, - [1][1][2][0] = 46, - [1][1][1][0] = 22, - [1][1][3][0] = 24, - [1][1][5][0] = 18, - [1][1][6][0] = 22, - [1][1][9][0] = 22, - [1][1][8][0] = 18, - [1][1][11][0] = 22, - [1][1][2][2] = 46, - [1][1][1][2] = 22, - [1][1][3][2] = 24, - [1][1][5][2] = 18, - [1][1][6][2] = 22, - [1][1][9][2] = 22, - [1][1][8][2] = 18, - [1][1][11][2] = 22, - [1][1][2][4] = 46, - [1][1][1][4] = 22, - [1][1][3][4] = 24, - [1][1][5][4] = 18, - [1][1][6][4] = 22, - [1][1][9][4] = 22, - [1][1][8][4] = 18, - [1][1][11][4] = 22, - [1][1][2][6] = 46, - [1][1][1][6] = 22, - [1][1][3][6] = 24, - [1][1][5][6] = 18, - [1][1][6][6] = 22, - [1][1][9][6] = 22, - [1][1][8][6] = 18, - [1][1][11][6] = 22, - [1][1][2][8] = 46, - [1][1][1][8] = 22, - [1][1][3][8] = 24, - [1][1][5][8] = 46, - [1][1][6][8] = 22, - [1][1][9][8] = 22, - [1][1][8][8] = 42, - [1][1][11][8] = 22, - [1][1][2][10] = 46, - [1][1][1][10] = 22, - [1][1][3][10] = 24, - [1][1][5][10] = 46, - [1][1][6][10] = 22, - [1][1][9][10] = 22, - [1][1][8][10] = 42, - [1][1][11][10] = 22, - [1][1][2][12] = 46, - [1][1][1][12] = 22, - [1][1][3][12] = 24, - [1][1][5][12] = 46, - [1][1][6][12] = 22, - [1][1][9][12] = 22, - [1][1][8][12] = 42, - [1][1][11][12] = 22, - [1][1][2][14] = 46, - [1][1][1][14] = 22, - [1][1][3][14] = 24, - [1][1][5][14] = 46, - [1][1][6][14] = 22, - [1][1][9][14] = 22, - [1][1][8][14] = 42, - [1][1][11][14] = 22, - [1][1][2][15] = 46, - [1][1][1][15] = 22, - [1][1][3][15] = 46, - [1][1][5][15] = 46, - [1][1][6][15] = 22, - [1][1][9][15] = 22, - [1][1][8][15] = 42, - [1][1][11][15] = 22, - [1][1][2][17] = 46, - [1][1][1][17] = 22, - [1][1][3][17] = 46, - [1][1][5][17] = 46, - [1][1][6][17] = 22, - [1][1][9][17] = 22, - [1][1][8][17] = 42, - [1][1][11][17] = 22, - [1][1][2][19] = 46, - [1][1][1][19] = 22, - [1][1][3][19] = 46, - [1][1][5][19] = 46, - [1][1][6][19] = 22, - [1][1][9][19] = 22, - [1][1][8][19] = 42, - [1][1][11][19] = 22, - [1][1][2][21] = 46, - [1][1][1][21] = 22, - [1][1][3][21] = 46, - [1][1][5][21] = 46, - [1][1][6][21] = 22, - [1][1][9][21] = 22, - [1][1][8][21] = 42, - [1][1][11][21] = 22, - [1][1][2][23] = 46, - [1][1][1][23] = 22, - [1][1][3][23] = 46, - [1][1][5][23] = 46, - [1][1][6][23] = 22, - [1][1][9][23] = 22, - [1][1][8][23] = 42, - [1][1][11][23] = 22, - [1][1][2][25] = 46, - [1][1][1][25] = 22, - [1][1][3][25] = 46, - [1][1][5][25] = 127, - [1][1][6][25] = 22, - [1][1][9][25] = 127, - [1][1][8][25] = 42, - [1][1][11][25] = 22, - [1][1][2][27] = 46, - [1][1][1][27] = 22, - [1][1][3][27] = 46, - [1][1][5][27] = 127, - [1][1][6][27] = 22, - [1][1][9][27] = 127, - [1][1][8][27] = 42, - [1][1][11][27] = 22, - [1][1][2][29] = 46, - [1][1][1][29] = 22, - [1][1][3][29] = 46, - [1][1][5][29] = 127, - [1][1][6][29] = 22, - [1][1][9][29] = 127, - [1][1][8][29] = 42, - [1][1][11][29] = 22, - [1][1][2][31] = 46, - [1][1][1][31] = 22, - [1][1][3][31] = 46, - [1][1][5][31] = 46, - [1][1][6][31] = 22, - [1][1][9][31] = 22, - [1][1][8][31] = 42, - [1][1][11][31] = 22, - [1][1][2][33] = 46, - [1][1][1][33] = 22, - [1][1][3][33] = 46, - [1][1][5][33] = 46, - [1][1][6][33] = 22, - [1][1][9][33] = 22, - [1][1][8][33] = 42, - [1][1][11][33] = 22, - [1][1][2][35] = 46, - [1][1][1][35] = 22, - [1][1][3][35] = 46, - [1][1][5][35] = 46, - [1][1][6][35] = 22, - [1][1][9][35] = 22, - [1][1][8][35] = 42, - [1][1][11][35] = 22, - [1][1][2][37] = 46, - [1][1][1][37] = 127, - [1][1][3][37] = 46, - [1][1][5][37] = 46, - [1][1][6][37] = 22, - [1][1][9][37] = 50, - [1][1][8][37] = 42, - [1][1][11][37] = 127, - [1][1][2][38] = 74, - [1][1][1][38] = 16, - [1][1][3][38] = 127, - [1][1][5][38] = 74, - [1][1][6][38] = 16, - [1][1][9][38] = 76, - [1][1][8][38] = 42, - [1][1][11][38] = 16, - [1][1][2][40] = 76, - [1][1][1][40] = 16, - [1][1][3][40] = 127, - [1][1][5][40] = 76, - [1][1][6][40] = 16, - [1][1][9][40] = 76, - [1][1][8][40] = 42, - [1][1][11][40] = 16, - [1][1][2][42] = 76, - [1][1][1][42] = 16, - [1][1][3][42] = 127, - [1][1][5][42] = 76, - [1][1][6][42] = 16, - [1][1][9][42] = 76, - [1][1][8][42] = 42, - [1][1][11][42] = 16, - [1][1][2][44] = 76, - [1][1][1][44] = 16, - [1][1][3][44] = 127, - [1][1][5][44] = 76, - [1][1][6][44] = 16, - [1][1][9][44] = 76, - [1][1][8][44] = 42, - [1][1][11][44] = 16, - [1][1][2][46] = 76, - [1][1][1][46] = 16, - [1][1][3][46] = 127, - [1][1][5][46] = 76, - [1][1][6][46] = 16, - [1][1][9][46] = 76, - [1][1][8][46] = 42, - [1][1][11][46] = 16, - [2][0][2][0] = 74, - [2][0][1][0] = 46, - [2][0][3][0] = 50, - [2][0][5][0] = 46, - [2][0][6][0] = 46, - [2][0][9][0] = 46, - [2][0][8][0] = 30, - [2][0][11][0] = 46, - [2][0][2][2] = 74, - [2][0][1][2] = 46, - [2][0][3][2] = 50, - [2][0][5][2] = 46, - [2][0][6][2] = 46, - [2][0][9][2] = 46, - [2][0][8][2] = 30, - [2][0][11][2] = 46, - [2][0][2][4] = 74, - [2][0][1][4] = 46, - [2][0][3][4] = 50, - [2][0][5][4] = 46, - [2][0][6][4] = 46, - [2][0][9][4] = 46, - [2][0][8][4] = 30, - [2][0][11][4] = 46, - [2][0][2][6] = 74, - [2][0][1][6] = 46, - [2][0][3][6] = 50, - [2][0][5][6] = 46, - [2][0][6][6] = 46, - [2][0][9][6] = 46, - [2][0][8][6] = 30, - [2][0][11][6] = 46, - [2][0][2][8] = 74, - [2][0][1][8] = 46, - [2][0][3][8] = 50, - [2][0][5][8] = 66, - [2][0][6][8] = 46, - [2][0][9][8] = 46, - [2][0][8][8] = 54, - [2][0][11][8] = 46, - [2][0][2][10] = 74, - [2][0][1][10] = 46, - [2][0][3][10] = 50, - [2][0][5][10] = 66, - [2][0][6][10] = 46, - [2][0][9][10] = 46, - [2][0][8][10] = 54, - [2][0][11][10] = 46, - [2][0][2][12] = 74, - [2][0][1][12] = 46, - [2][0][3][12] = 50, - [2][0][5][12] = 66, - [2][0][6][12] = 46, - [2][0][9][12] = 46, - [2][0][8][12] = 54, - [2][0][11][12] = 46, - [2][0][2][14] = 74, - [2][0][1][14] = 46, - [2][0][3][14] = 50, - [2][0][5][14] = 66, - [2][0][6][14] = 46, - [2][0][9][14] = 46, - [2][0][8][14] = 54, - [2][0][11][14] = 46, - [2][0][2][15] = 74, - [2][0][1][15] = 46, - [2][0][3][15] = 70, - [2][0][5][15] = 74, - [2][0][6][15] = 46, - [2][0][9][15] = 46, - [2][0][8][15] = 54, - [2][0][11][15] = 46, - [2][0][2][17] = 74, - [2][0][1][17] = 46, - [2][0][3][17] = 70, - [2][0][5][17] = 74, - [2][0][6][17] = 46, - [2][0][9][17] = 46, - [2][0][8][17] = 54, - [2][0][11][17] = 46, - [2][0][2][19] = 74, - [2][0][1][19] = 46, - [2][0][3][19] = 70, - [2][0][5][19] = 74, - [2][0][6][19] = 46, - [2][0][9][19] = 46, - [2][0][8][19] = 54, - [2][0][11][19] = 46, - [2][0][2][21] = 74, - [2][0][1][21] = 46, - [2][0][3][21] = 70, - [2][0][5][21] = 74, - [2][0][6][21] = 46, - [2][0][9][21] = 46, - [2][0][8][21] = 54, - [2][0][11][21] = 46, - [2][0][2][23] = 74, - [2][0][1][23] = 46, - [2][0][3][23] = 70, - [2][0][5][23] = 74, - [2][0][6][23] = 46, - [2][0][9][23] = 46, - [2][0][8][23] = 54, - [2][0][11][23] = 46, - [2][0][2][25] = 74, - [2][0][1][25] = 46, - [2][0][3][25] = 70, - [2][0][5][25] = 127, - [2][0][6][25] = 46, - [2][0][9][25] = 127, - [2][0][8][25] = 54, - [2][0][11][25] = 46, - [2][0][2][27] = 74, - [2][0][1][27] = 46, - [2][0][3][27] = 70, - [2][0][5][27] = 127, - [2][0][6][27] = 46, - [2][0][9][27] = 127, - [2][0][8][27] = 54, - [2][0][11][27] = 46, - [2][0][2][29] = 74, - [2][0][1][29] = 46, - [2][0][3][29] = 70, - [2][0][5][29] = 127, - [2][0][6][29] = 46, - [2][0][9][29] = 127, - [2][0][8][29] = 54, - [2][0][11][29] = 46, - [2][0][2][31] = 74, - [2][0][1][31] = 46, - [2][0][3][31] = 70, - [2][0][5][31] = 74, - [2][0][6][31] = 46, - [2][0][9][31] = 46, - [2][0][8][31] = 54, - [2][0][11][31] = 46, - [2][0][2][33] = 74, - [2][0][1][33] = 46, - [2][0][3][33] = 70, - [2][0][5][33] = 74, - [2][0][6][33] = 46, - [2][0][9][33] = 46, - [2][0][8][33] = 54, - [2][0][11][33] = 46, - [2][0][2][35] = 74, - [2][0][1][35] = 46, - [2][0][3][35] = 70, - [2][0][5][35] = 74, - [2][0][6][35] = 46, - [2][0][9][35] = 46, - [2][0][8][35] = 54, - [2][0][11][35] = 46, - [2][0][2][37] = 74, - [2][0][1][37] = 127, - [2][0][3][37] = 70, - [2][0][5][37] = 74, - [2][0][6][37] = 46, - [2][0][9][37] = 74, - [2][0][8][37] = 54, - [2][0][11][37] = 127, - [2][0][2][38] = 76, - [2][0][1][38] = 28, - [2][0][3][38] = 127, - [2][0][5][38] = 76, - [2][0][6][38] = 28, - [2][0][9][38] = 76, - [2][0][8][38] = 54, - [2][0][11][38] = 28, - [2][0][2][40] = 76, - [2][0][1][40] = 28, - [2][0][3][40] = 127, - [2][0][5][40] = 76, - [2][0][6][40] = 28, - [2][0][9][40] = 76, - [2][0][8][40] = 54, - [2][0][11][40] = 28, - [2][0][2][42] = 76, - [2][0][1][42] = 28, - [2][0][3][42] = 127, - [2][0][5][42] = 76, - [2][0][6][42] = 28, - [2][0][9][42] = 76, - [2][0][8][42] = 54, - [2][0][11][42] = 28, - [2][0][2][44] = 76, - [2][0][1][44] = 28, - [2][0][3][44] = 127, - [2][0][5][44] = 76, - [2][0][6][44] = 28, - [2][0][9][44] = 76, - [2][0][8][44] = 54, - [2][0][11][44] = 28, - [2][0][2][46] = 76, - [2][0][1][46] = 28, - [2][0][3][46] = 127, - [2][0][5][46] = 76, - [2][0][6][46] = 28, - [2][0][9][46] = 76, - [2][0][8][46] = 54, - [2][0][11][46] = 28, - [2][1][2][0] = 58, - [2][1][1][0] = 32, - [2][1][3][0] = 38, - [2][1][5][0] = 30, - [2][1][6][0] = 32, - [2][1][9][0] = 32, - [2][1][8][0] = 18, - [2][1][11][0] = 32, - [2][1][2][2] = 58, - [2][1][1][2] = 32, - [2][1][3][2] = 38, - [2][1][5][2] = 30, - [2][1][6][2] = 32, - [2][1][9][2] = 32, - [2][1][8][2] = 18, - [2][1][11][2] = 32, - [2][1][2][4] = 58, - [2][1][1][4] = 32, - [2][1][3][4] = 38, - [2][1][5][4] = 30, - [2][1][6][4] = 32, - [2][1][9][4] = 32, - [2][1][8][4] = 18, - [2][1][11][4] = 32, - [2][1][2][6] = 58, - [2][1][1][6] = 32, - [2][1][3][6] = 38, - [2][1][5][6] = 30, - [2][1][6][6] = 32, - [2][1][9][6] = 32, - [2][1][8][6] = 18, - [2][1][11][6] = 32, - [2][1][2][8] = 58, - [2][1][1][8] = 32, - [2][1][3][8] = 38, - [2][1][5][8] = 52, - [2][1][6][8] = 32, - [2][1][9][8] = 32, - [2][1][8][8] = 42, - [2][1][11][8] = 32, - [2][1][2][10] = 58, - [2][1][1][10] = 32, - [2][1][3][10] = 38, - [2][1][5][10] = 52, - [2][1][6][10] = 32, - [2][1][9][10] = 32, - [2][1][8][10] = 42, - [2][1][11][10] = 32, - [2][1][2][12] = 58, - [2][1][1][12] = 32, - [2][1][3][12] = 38, - [2][1][5][12] = 52, - [2][1][6][12] = 32, - [2][1][9][12] = 32, - [2][1][8][12] = 42, - [2][1][11][12] = 32, - [2][1][2][14] = 58, - [2][1][1][14] = 32, - [2][1][3][14] = 38, - [2][1][5][14] = 52, - [2][1][6][14] = 32, - [2][1][9][14] = 32, - [2][1][8][14] = 42, - [2][1][11][14] = 32, - [2][1][2][15] = 58, - [2][1][1][15] = 32, - [2][1][3][15] = 58, - [2][1][5][15] = 58, - [2][1][6][15] = 32, - [2][1][9][15] = 32, - [2][1][8][15] = 42, - [2][1][11][15] = 32, - [2][1][2][17] = 58, - [2][1][1][17] = 32, - [2][1][3][17] = 58, - [2][1][5][17] = 58, - [2][1][6][17] = 32, - [2][1][9][17] = 32, - [2][1][8][17] = 42, - [2][1][11][17] = 32, - [2][1][2][19] = 58, - [2][1][1][19] = 32, - [2][1][3][19] = 58, - [2][1][5][19] = 58, - [2][1][6][19] = 32, - [2][1][9][19] = 32, - [2][1][8][19] = 42, - [2][1][11][19] = 32, - [2][1][2][21] = 58, - [2][1][1][21] = 32, - [2][1][3][21] = 58, - [2][1][5][21] = 58, - [2][1][6][21] = 32, - [2][1][9][21] = 32, - [2][1][8][21] = 42, - [2][1][11][21] = 32, - [2][1][2][23] = 58, - [2][1][1][23] = 32, - [2][1][3][23] = 58, - [2][1][5][23] = 58, - [2][1][6][23] = 32, - [2][1][9][23] = 32, - [2][1][8][23] = 42, - [2][1][11][23] = 32, - [2][1][2][25] = 58, - [2][1][1][25] = 32, - [2][1][3][25] = 58, - [2][1][5][25] = 127, - [2][1][6][25] = 32, - [2][1][9][25] = 127, - [2][1][8][25] = 42, - [2][1][11][25] = 32, - [2][1][2][27] = 58, - [2][1][1][27] = 32, - [2][1][3][27] = 58, - [2][1][5][27] = 127, - [2][1][6][27] = 32, - [2][1][9][27] = 127, - [2][1][8][27] = 42, - [2][1][11][27] = 32, - [2][1][2][29] = 58, - [2][1][1][29] = 32, - [2][1][3][29] = 58, - [2][1][5][29] = 127, - [2][1][6][29] = 32, - [2][1][9][29] = 127, - [2][1][8][29] = 42, - [2][1][11][29] = 32, - [2][1][2][31] = 58, - [2][1][1][31] = 32, - [2][1][3][31] = 58, - [2][1][5][31] = 58, - [2][1][6][31] = 32, - [2][1][9][31] = 32, - [2][1][8][31] = 42, - [2][1][11][31] = 32, - [2][1][2][33] = 58, - [2][1][1][33] = 32, - [2][1][3][33] = 58, - [2][1][5][33] = 58, - [2][1][6][33] = 32, - [2][1][9][33] = 32, - [2][1][8][33] = 42, - [2][1][11][33] = 32, - [2][1][2][35] = 58, - [2][1][1][35] = 32, - [2][1][3][35] = 58, - [2][1][5][35] = 58, - [2][1][6][35] = 32, - [2][1][9][35] = 32, - [2][1][8][35] = 42, - [2][1][11][35] = 32, - [2][1][2][37] = 58, - [2][1][1][37] = 127, - [2][1][3][37] = 58, - [2][1][5][37] = 58, - [2][1][6][37] = 32, - [2][1][9][37] = 62, - [2][1][8][37] = 42, - [2][1][11][37] = 127, - [2][1][2][38] = 76, - [2][1][1][38] = 16, - [2][1][3][38] = 127, - [2][1][5][38] = 76, - [2][1][6][38] = 16, - [2][1][9][38] = 76, - [2][1][8][38] = 42, - [2][1][11][38] = 16, - [2][1][2][40] = 76, - [2][1][1][40] = 16, - [2][1][3][40] = 127, - [2][1][5][40] = 76, - [2][1][6][40] = 16, - [2][1][9][40] = 76, - [2][1][8][40] = 42, - [2][1][11][40] = 16, - [2][1][2][42] = 76, - [2][1][1][42] = 16, - [2][1][3][42] = 127, - [2][1][5][42] = 76, - [2][1][6][42] = 16, - [2][1][9][42] = 76, - [2][1][8][42] = 42, - [2][1][11][42] = 16, - [2][1][2][44] = 76, - [2][1][1][44] = 16, - [2][1][3][44] = 127, - [2][1][5][44] = 76, - [2][1][6][44] = 16, - [2][1][9][44] = 76, - [2][1][8][44] = 42, - [2][1][11][44] = 16, - [2][1][2][46] = 76, - [2][1][1][46] = 16, - [2][1][3][46] = 127, - [2][1][5][46] = 76, - [2][1][6][46] = 16, - [2][1][9][46] = 76, - [2][1][8][46] = 42, - [2][1][11][46] = 16, + [0][0][RTW89_WW][0] = 22, + [0][0][RTW89_WW][2] = 22, + [0][0][RTW89_WW][4] = 22, + [0][0][RTW89_WW][6] = 20, + [0][0][RTW89_WW][8] = 24, + [0][0][RTW89_WW][10] = 24, + [0][0][RTW89_WW][12] = 24, + [0][0][RTW89_WW][14] = 24, + [0][0][RTW89_WW][15] = 24, + [0][0][RTW89_WW][17] = 24, + [0][0][RTW89_WW][19] = 24, + [0][0][RTW89_WW][21] = 24, + [0][0][RTW89_WW][23] = 24, + [0][0][RTW89_WW][25] = 24, + [0][0][RTW89_WW][27] = 24, + [0][0][RTW89_WW][29] = 24, + [0][0][RTW89_WW][31] = 24, + [0][0][RTW89_WW][33] = 24, + [0][0][RTW89_WW][35] = 24, + [0][0][RTW89_WW][37] = 44, + [0][0][RTW89_WW][38] = 28, + [0][0][RTW89_WW][40] = 28, + [0][0][RTW89_WW][42] = 28, + [0][0][RTW89_WW][44] = 28, + [0][0][RTW89_WW][46] = 28, + [0][1][RTW89_WW][0] = 8, + [0][1][RTW89_WW][2] = 8, + [0][1][RTW89_WW][4] = 8, + [0][1][RTW89_WW][6] = 4, + [0][1][RTW89_WW][8] = 12, + [0][1][RTW89_WW][10] = 12, + [0][1][RTW89_WW][12] = 12, + [0][1][RTW89_WW][14] = 12, + [0][1][RTW89_WW][15] = 12, + [0][1][RTW89_WW][17] = 12, + [0][1][RTW89_WW][19] = 12, + [0][1][RTW89_WW][21] = 12, + [0][1][RTW89_WW][23] = 12, + [0][1][RTW89_WW][25] = 12, + [0][1][RTW89_WW][27] = 12, + [0][1][RTW89_WW][29] = 12, + [0][1][RTW89_WW][31] = 12, + [0][1][RTW89_WW][33] = 12, + [0][1][RTW89_WW][35] = 12, + [0][1][RTW89_WW][37] = 32, + [0][1][RTW89_WW][38] = 16, + [0][1][RTW89_WW][40] = 16, + [0][1][RTW89_WW][42] = 16, + [0][1][RTW89_WW][44] = 16, + [0][1][RTW89_WW][46] = 16, + [1][0][RTW89_WW][0] = 30, + [1][0][RTW89_WW][2] = 30, + [1][0][RTW89_WW][4] = 30, + [1][0][RTW89_WW][6] = 30, + [1][0][RTW89_WW][8] = 36, + [1][0][RTW89_WW][10] = 36, + [1][0][RTW89_WW][12] = 36, + [1][0][RTW89_WW][14] = 36, + [1][0][RTW89_WW][15] = 36, + [1][0][RTW89_WW][17] = 36, + [1][0][RTW89_WW][19] = 36, + [1][0][RTW89_WW][21] = 36, + [1][0][RTW89_WW][23] = 36, + [1][0][RTW89_WW][25] = 36, + [1][0][RTW89_WW][27] = 36, + [1][0][RTW89_WW][29] = 36, + [1][0][RTW89_WW][31] = 36, + [1][0][RTW89_WW][33] = 36, + [1][0][RTW89_WW][35] = 36, + [1][0][RTW89_WW][37] = 54, + [1][0][RTW89_WW][38] = 28, + [1][0][RTW89_WW][40] = 28, + [1][0][RTW89_WW][42] = 28, + [1][0][RTW89_WW][44] = 28, + [1][0][RTW89_WW][46] = 28, + [1][1][RTW89_WW][0] = 18, + [1][1][RTW89_WW][2] = 18, + [1][1][RTW89_WW][4] = 18, + [1][1][RTW89_WW][6] = 16, + [1][1][RTW89_WW][8] = 22, + [1][1][RTW89_WW][10] = 22, + [1][1][RTW89_WW][12] = 22, + [1][1][RTW89_WW][14] = 22, + [1][1][RTW89_WW][15] = 22, + [1][1][RTW89_WW][17] = 22, + [1][1][RTW89_WW][19] = 22, + [1][1][RTW89_WW][21] = 22, + [1][1][RTW89_WW][23] = 22, + [1][1][RTW89_WW][25] = 22, + [1][1][RTW89_WW][27] = 22, + [1][1][RTW89_WW][29] = 22, + [1][1][RTW89_WW][31] = 22, + [1][1][RTW89_WW][33] = 22, + [1][1][RTW89_WW][35] = 22, + [1][1][RTW89_WW][37] = 42, + [1][1][RTW89_WW][38] = 16, + [1][1][RTW89_WW][40] = 16, + [1][1][RTW89_WW][42] = 16, + [1][1][RTW89_WW][44] = 16, + [1][1][RTW89_WW][46] = 16, + [2][0][RTW89_WW][0] = 30, + [2][0][RTW89_WW][2] = 30, + [2][0][RTW89_WW][4] = 30, + [2][0][RTW89_WW][6] = 30, + [2][0][RTW89_WW][8] = 46, + [2][0][RTW89_WW][10] = 46, + [2][0][RTW89_WW][12] = 46, + [2][0][RTW89_WW][14] = 46, + [2][0][RTW89_WW][15] = 46, + [2][0][RTW89_WW][17] = 46, + [2][0][RTW89_WW][19] = 46, + [2][0][RTW89_WW][21] = 46, + [2][0][RTW89_WW][23] = 46, + [2][0][RTW89_WW][25] = 46, + [2][0][RTW89_WW][27] = 46, + [2][0][RTW89_WW][29] = 46, + [2][0][RTW89_WW][31] = 46, + [2][0][RTW89_WW][33] = 46, + [2][0][RTW89_WW][35] = 46, + [2][0][RTW89_WW][37] = 54, + [2][0][RTW89_WW][38] = 28, + [2][0][RTW89_WW][40] = 28, + [2][0][RTW89_WW][42] = 28, + [2][0][RTW89_WW][44] = 28, + [2][0][RTW89_WW][46] = 28, + [2][1][RTW89_WW][0] = 18, + [2][1][RTW89_WW][2] = 18, + [2][1][RTW89_WW][4] = 18, + [2][1][RTW89_WW][6] = 18, + [2][1][RTW89_WW][8] = 32, + [2][1][RTW89_WW][10] = 32, + [2][1][RTW89_WW][12] = 32, + [2][1][RTW89_WW][14] = 32, + [2][1][RTW89_WW][15] = 32, + [2][1][RTW89_WW][17] = 32, + [2][1][RTW89_WW][19] = 32, + [2][1][RTW89_WW][21] = 32, + [2][1][RTW89_WW][23] = 32, + [2][1][RTW89_WW][25] = 32, + [2][1][RTW89_WW][27] = 32, + [2][1][RTW89_WW][29] = 32, + [2][1][RTW89_WW][31] = 32, + [2][1][RTW89_WW][33] = 32, + [2][1][RTW89_WW][35] = 32, + [2][1][RTW89_WW][37] = 42, + [2][1][RTW89_WW][38] = 16, + [2][1][RTW89_WW][40] = 16, + [2][1][RTW89_WW][42] = 16, + [2][1][RTW89_WW][44] = 16, + [2][1][RTW89_WW][46] = 16, + [0][0][RTW89_FCC][0] = 48, + [0][0][RTW89_ETSI][0] = 24, + [0][0][RTW89_MKK][0] = 26, + [0][0][RTW89_IC][0] = 22, + [0][0][RTW89_KCC][0] = 46, + [0][0][RTW89_ACMA][0] = 24, + [0][0][RTW89_CHILE][0] = 30, + [0][0][RTW89_UKRAINE][0] = 24, + [0][0][RTW89_MEXICO][0] = 48, + [0][0][RTW89_CN][0] = 24, + [0][0][RTW89_QATAR][0] = 24, + [0][0][RTW89_FCC][2] = 48, + [0][0][RTW89_ETSI][2] = 24, + [0][0][RTW89_MKK][2] = 26, + [0][0][RTW89_IC][2] = 22, + [0][0][RTW89_KCC][2] = 46, + [0][0][RTW89_ACMA][2] = 24, + [0][0][RTW89_CHILE][2] = 30, + [0][0][RTW89_UKRAINE][2] = 24, + [0][0][RTW89_MEXICO][2] = 48, + [0][0][RTW89_CN][2] = 24, + [0][0][RTW89_QATAR][2] = 24, + [0][0][RTW89_FCC][4] = 48, + [0][0][RTW89_ETSI][4] = 24, + [0][0][RTW89_MKK][4] = 26, + [0][0][RTW89_IC][4] = 22, + [0][0][RTW89_KCC][4] = 46, + [0][0][RTW89_ACMA][4] = 24, + [0][0][RTW89_CHILE][4] = 30, + [0][0][RTW89_UKRAINE][4] = 24, + [0][0][RTW89_MEXICO][4] = 48, + [0][0][RTW89_CN][4] = 24, + [0][0][RTW89_QATAR][4] = 24, + [0][0][RTW89_FCC][6] = 48, + [0][0][RTW89_ETSI][6] = 24, + [0][0][RTW89_MKK][6] = 26, + [0][0][RTW89_IC][6] = 22, + [0][0][RTW89_KCC][6] = 20, + [0][0][RTW89_ACMA][6] = 24, + [0][0][RTW89_CHILE][6] = 30, + [0][0][RTW89_UKRAINE][6] = 24, + [0][0][RTW89_MEXICO][6] = 48, + [0][0][RTW89_CN][6] = 24, + [0][0][RTW89_QATAR][6] = 24, + [0][0][RTW89_FCC][8] = 48, + [0][0][RTW89_ETSI][8] = 24, + [0][0][RTW89_MKK][8] = 26, + [0][0][RTW89_IC][8] = 48, + [0][0][RTW89_KCC][8] = 46, + [0][0][RTW89_ACMA][8] = 24, + [0][0][RTW89_CHILE][8] = 48, + [0][0][RTW89_UKRAINE][8] = 24, + [0][0][RTW89_MEXICO][8] = 48, + [0][0][RTW89_CN][8] = 24, + [0][0][RTW89_QATAR][8] = 24, + [0][0][RTW89_FCC][10] = 48, + [0][0][RTW89_ETSI][10] = 24, + [0][0][RTW89_MKK][10] = 26, + [0][0][RTW89_IC][10] = 48, + [0][0][RTW89_KCC][10] = 46, + [0][0][RTW89_ACMA][10] = 24, + [0][0][RTW89_CHILE][10] = 48, + [0][0][RTW89_UKRAINE][10] = 24, + [0][0][RTW89_MEXICO][10] = 48, + [0][0][RTW89_CN][10] = 24, + [0][0][RTW89_QATAR][10] = 24, + [0][0][RTW89_FCC][12] = 48, + [0][0][RTW89_ETSI][12] = 24, + [0][0][RTW89_MKK][12] = 26, + [0][0][RTW89_IC][12] = 48, + [0][0][RTW89_KCC][12] = 46, + [0][0][RTW89_ACMA][12] = 24, + [0][0][RTW89_CHILE][12] = 48, + [0][0][RTW89_UKRAINE][12] = 24, + [0][0][RTW89_MEXICO][12] = 48, + [0][0][RTW89_CN][12] = 24, + [0][0][RTW89_QATAR][12] = 24, + [0][0][RTW89_FCC][14] = 48, + [0][0][RTW89_ETSI][14] = 24, + [0][0][RTW89_MKK][14] = 26, + [0][0][RTW89_IC][14] = 48, + [0][0][RTW89_KCC][14] = 46, + [0][0][RTW89_ACMA][14] = 24, + [0][0][RTW89_CHILE][14] = 48, + [0][0][RTW89_UKRAINE][14] = 24, + [0][0][RTW89_MEXICO][14] = 48, + [0][0][RTW89_CN][14] = 24, + [0][0][RTW89_QATAR][14] = 24, + [0][0][RTW89_FCC][15] = 48, + [0][0][RTW89_ETSI][15] = 24, + [0][0][RTW89_MKK][15] = 44, + [0][0][RTW89_IC][15] = 48, + [0][0][RTW89_KCC][15] = 46, + [0][0][RTW89_ACMA][15] = 24, + [0][0][RTW89_CHILE][15] = 48, + [0][0][RTW89_UKRAINE][15] = 24, + [0][0][RTW89_MEXICO][15] = 48, + [0][0][RTW89_CN][15] = 127, + [0][0][RTW89_QATAR][15] = 24, + [0][0][RTW89_FCC][17] = 48, + [0][0][RTW89_ETSI][17] = 24, + [0][0][RTW89_MKK][17] = 44, + [0][0][RTW89_IC][17] = 48, + [0][0][RTW89_KCC][17] = 46, + [0][0][RTW89_ACMA][17] = 24, + [0][0][RTW89_CHILE][17] = 48, + [0][0][RTW89_UKRAINE][17] = 24, + [0][0][RTW89_MEXICO][17] = 48, + [0][0][RTW89_CN][17] = 127, + [0][0][RTW89_QATAR][17] = 24, + [0][0][RTW89_FCC][19] = 48, + [0][0][RTW89_ETSI][19] = 24, + [0][0][RTW89_MKK][19] = 44, + [0][0][RTW89_IC][19] = 48, + [0][0][RTW89_KCC][19] = 46, + [0][0][RTW89_ACMA][19] = 24, + [0][0][RTW89_CHILE][19] = 48, + [0][0][RTW89_UKRAINE][19] = 24, + [0][0][RTW89_MEXICO][19] = 48, + [0][0][RTW89_CN][19] = 127, + [0][0][RTW89_QATAR][19] = 24, + [0][0][RTW89_FCC][21] = 48, + [0][0][RTW89_ETSI][21] = 24, + [0][0][RTW89_MKK][21] = 44, + [0][0][RTW89_IC][21] = 48, + [0][0][RTW89_KCC][21] = 46, + [0][0][RTW89_ACMA][21] = 24, + [0][0][RTW89_CHILE][21] = 48, + [0][0][RTW89_UKRAINE][21] = 24, + [0][0][RTW89_MEXICO][21] = 48, + [0][0][RTW89_CN][21] = 127, + [0][0][RTW89_QATAR][21] = 24, + [0][0][RTW89_FCC][23] = 48, + [0][0][RTW89_ETSI][23] = 24, + [0][0][RTW89_MKK][23] = 44, + [0][0][RTW89_IC][23] = 48, + [0][0][RTW89_KCC][23] = 46, + [0][0][RTW89_ACMA][23] = 24, + [0][0][RTW89_CHILE][23] = 48, + [0][0][RTW89_UKRAINE][23] = 24, + [0][0][RTW89_MEXICO][23] = 48, + [0][0][RTW89_CN][23] = 127, + [0][0][RTW89_QATAR][23] = 24, + [0][0][RTW89_FCC][25] = 48, + [0][0][RTW89_ETSI][25] = 24, + [0][0][RTW89_MKK][25] = 44, + [0][0][RTW89_IC][25] = 127, + [0][0][RTW89_KCC][25] = 46, + [0][0][RTW89_ACMA][25] = 127, + [0][0][RTW89_CHILE][25] = 48, + [0][0][RTW89_UKRAINE][25] = 24, + [0][0][RTW89_MEXICO][25] = 48, + [0][0][RTW89_CN][25] = 127, + [0][0][RTW89_QATAR][25] = 24, + [0][0][RTW89_FCC][27] = 48, + [0][0][RTW89_ETSI][27] = 24, + [0][0][RTW89_MKK][27] = 44, + [0][0][RTW89_IC][27] = 127, + [0][0][RTW89_KCC][27] = 46, + [0][0][RTW89_ACMA][27] = 127, + [0][0][RTW89_CHILE][27] = 48, + [0][0][RTW89_UKRAINE][27] = 24, + [0][0][RTW89_MEXICO][27] = 48, + [0][0][RTW89_CN][27] = 127, + [0][0][RTW89_QATAR][27] = 24, + [0][0][RTW89_FCC][29] = 48, + [0][0][RTW89_ETSI][29] = 24, + [0][0][RTW89_MKK][29] = 44, + [0][0][RTW89_IC][29] = 127, + [0][0][RTW89_KCC][29] = 46, + [0][0][RTW89_ACMA][29] = 127, + [0][0][RTW89_CHILE][29] = 48, + [0][0][RTW89_UKRAINE][29] = 24, + [0][0][RTW89_MEXICO][29] = 48, + [0][0][RTW89_CN][29] = 127, + [0][0][RTW89_QATAR][29] = 24, + [0][0][RTW89_FCC][31] = 48, + [0][0][RTW89_ETSI][31] = 24, + [0][0][RTW89_MKK][31] = 44, + [0][0][RTW89_IC][31] = 48, + [0][0][RTW89_KCC][31] = 46, + [0][0][RTW89_ACMA][31] = 24, + [0][0][RTW89_CHILE][31] = 48, + [0][0][RTW89_UKRAINE][31] = 24, + [0][0][RTW89_MEXICO][31] = 48, + [0][0][RTW89_CN][31] = 127, + [0][0][RTW89_QATAR][31] = 24, + [0][0][RTW89_FCC][33] = 48, + [0][0][RTW89_ETSI][33] = 24, + [0][0][RTW89_MKK][33] = 44, + [0][0][RTW89_IC][33] = 48, + [0][0][RTW89_KCC][33] = 46, + [0][0][RTW89_ACMA][33] = 24, + [0][0][RTW89_CHILE][33] = 48, + [0][0][RTW89_UKRAINE][33] = 24, + [0][0][RTW89_MEXICO][33] = 48, + [0][0][RTW89_CN][33] = 127, + [0][0][RTW89_QATAR][33] = 24, + [0][0][RTW89_FCC][35] = 48, + [0][0][RTW89_ETSI][35] = 24, + [0][0][RTW89_MKK][35] = 44, + [0][0][RTW89_IC][35] = 48, + [0][0][RTW89_KCC][35] = 46, + [0][0][RTW89_ACMA][35] = 24, + [0][0][RTW89_CHILE][35] = 48, + [0][0][RTW89_UKRAINE][35] = 24, + [0][0][RTW89_MEXICO][35] = 48, + [0][0][RTW89_CN][35] = 127, + [0][0][RTW89_QATAR][35] = 24, + [0][0][RTW89_FCC][37] = 48, + [0][0][RTW89_ETSI][37] = 127, + [0][0][RTW89_MKK][37] = 44, + [0][0][RTW89_IC][37] = 48, + [0][0][RTW89_KCC][37] = 46, + [0][0][RTW89_ACMA][37] = 48, + [0][0][RTW89_CHILE][37] = 48, + [0][0][RTW89_UKRAINE][37] = 127, + [0][0][RTW89_MEXICO][37] = 48, + [0][0][RTW89_CN][37] = 127, + [0][0][RTW89_QATAR][37] = 127, + [0][0][RTW89_FCC][38] = 76, + [0][0][RTW89_ETSI][38] = 28, + [0][0][RTW89_MKK][38] = 127, + [0][0][RTW89_IC][38] = 76, + [0][0][RTW89_KCC][38] = 46, + [0][0][RTW89_ACMA][38] = 76, + [0][0][RTW89_CHILE][38] = 54, + [0][0][RTW89_UKRAINE][38] = 28, + [0][0][RTW89_MEXICO][38] = 76, + [0][0][RTW89_CN][38] = 62, + [0][0][RTW89_QATAR][38] = 28, + [0][0][RTW89_FCC][40] = 76, + [0][0][RTW89_ETSI][40] = 28, + [0][0][RTW89_MKK][40] = 127, + [0][0][RTW89_IC][40] = 76, + [0][0][RTW89_KCC][40] = 46, + [0][0][RTW89_ACMA][40] = 76, + [0][0][RTW89_CHILE][40] = 54, + [0][0][RTW89_UKRAINE][40] = 28, + [0][0][RTW89_MEXICO][40] = 76, + [0][0][RTW89_CN][40] = 62, + [0][0][RTW89_QATAR][40] = 28, + [0][0][RTW89_FCC][42] = 76, + [0][0][RTW89_ETSI][42] = 28, + [0][0][RTW89_MKK][42] = 127, + [0][0][RTW89_IC][42] = 76, + [0][0][RTW89_KCC][42] = 46, + [0][0][RTW89_ACMA][42] = 76, + [0][0][RTW89_CHILE][42] = 54, + [0][0][RTW89_UKRAINE][42] = 28, + [0][0][RTW89_MEXICO][42] = 76, + [0][0][RTW89_CN][42] = 62, + [0][0][RTW89_QATAR][42] = 28, + [0][0][RTW89_FCC][44] = 76, + [0][0][RTW89_ETSI][44] = 28, + [0][0][RTW89_MKK][44] = 127, + [0][0][RTW89_IC][44] = 76, + [0][0][RTW89_KCC][44] = 46, + [0][0][RTW89_ACMA][44] = 76, + [0][0][RTW89_CHILE][44] = 54, + [0][0][RTW89_UKRAINE][44] = 28, + [0][0][RTW89_MEXICO][44] = 76, + [0][0][RTW89_CN][44] = 62, + [0][0][RTW89_QATAR][44] = 28, + [0][0][RTW89_FCC][46] = 76, + [0][0][RTW89_ETSI][46] = 28, + [0][0][RTW89_MKK][46] = 127, + [0][0][RTW89_IC][46] = 76, + [0][0][RTW89_KCC][46] = 46, + [0][0][RTW89_ACMA][46] = 76, + [0][0][RTW89_CHILE][46] = 54, + [0][0][RTW89_UKRAINE][46] = 28, + [0][0][RTW89_MEXICO][46] = 76, + [0][0][RTW89_CN][46] = 62, + [0][0][RTW89_QATAR][46] = 28, + [0][1][RTW89_FCC][0] = 36, + [0][1][RTW89_ETSI][0] = 12, + [0][1][RTW89_MKK][0] = 14, + [0][1][RTW89_IC][0] = 8, + [0][1][RTW89_KCC][0] = 32, + [0][1][RTW89_ACMA][0] = 12, + [0][1][RTW89_CHILE][0] = 18, + [0][1][RTW89_UKRAINE][0] = 12, + [0][1][RTW89_MEXICO][0] = 36, + [0][1][RTW89_CN][0] = 12, + [0][1][RTW89_QATAR][0] = 12, + [0][1][RTW89_FCC][2] = 36, + [0][1][RTW89_ETSI][2] = 12, + [0][1][RTW89_MKK][2] = 14, + [0][1][RTW89_IC][2] = 8, + [0][1][RTW89_KCC][2] = 32, + [0][1][RTW89_ACMA][2] = 12, + [0][1][RTW89_CHILE][2] = 18, + [0][1][RTW89_UKRAINE][2] = 12, + [0][1][RTW89_MEXICO][2] = 36, + [0][1][RTW89_CN][2] = 12, + [0][1][RTW89_QATAR][2] = 12, + [0][1][RTW89_FCC][4] = 36, + [0][1][RTW89_ETSI][4] = 12, + [0][1][RTW89_MKK][4] = 14, + [0][1][RTW89_IC][4] = 8, + [0][1][RTW89_KCC][4] = 32, + [0][1][RTW89_ACMA][4] = 12, + [0][1][RTW89_CHILE][4] = 18, + [0][1][RTW89_UKRAINE][4] = 12, + [0][1][RTW89_MEXICO][4] = 36, + [0][1][RTW89_CN][4] = 12, + [0][1][RTW89_QATAR][4] = 12, + [0][1][RTW89_FCC][6] = 36, + [0][1][RTW89_ETSI][6] = 12, + [0][1][RTW89_MKK][6] = 14, + [0][1][RTW89_IC][6] = 8, + [0][1][RTW89_KCC][6] = 4, + [0][1][RTW89_ACMA][6] = 12, + [0][1][RTW89_CHILE][6] = 18, + [0][1][RTW89_UKRAINE][6] = 12, + [0][1][RTW89_MEXICO][6] = 36, + [0][1][RTW89_CN][6] = 12, + [0][1][RTW89_QATAR][6] = 12, + [0][1][RTW89_FCC][8] = 36, + [0][1][RTW89_ETSI][8] = 12, + [0][1][RTW89_MKK][8] = 14, + [0][1][RTW89_IC][8] = 36, + [0][1][RTW89_KCC][8] = 32, + [0][1][RTW89_ACMA][8] = 12, + [0][1][RTW89_CHILE][8] = 36, + [0][1][RTW89_UKRAINE][8] = 12, + [0][1][RTW89_MEXICO][8] = 36, + [0][1][RTW89_CN][8] = 12, + [0][1][RTW89_QATAR][8] = 12, + [0][1][RTW89_FCC][10] = 36, + [0][1][RTW89_ETSI][10] = 12, + [0][1][RTW89_MKK][10] = 14, + [0][1][RTW89_IC][10] = 36, + [0][1][RTW89_KCC][10] = 32, + [0][1][RTW89_ACMA][10] = 12, + [0][1][RTW89_CHILE][10] = 36, + [0][1][RTW89_UKRAINE][10] = 12, + [0][1][RTW89_MEXICO][10] = 36, + [0][1][RTW89_CN][10] = 12, + [0][1][RTW89_QATAR][10] = 12, + [0][1][RTW89_FCC][12] = 36, + [0][1][RTW89_ETSI][12] = 12, + [0][1][RTW89_MKK][12] = 14, + [0][1][RTW89_IC][12] = 36, + [0][1][RTW89_KCC][12] = 32, + [0][1][RTW89_ACMA][12] = 12, + [0][1][RTW89_CHILE][12] = 36, + [0][1][RTW89_UKRAINE][12] = 12, + [0][1][RTW89_MEXICO][12] = 36, + [0][1][RTW89_CN][12] = 12, + [0][1][RTW89_QATAR][12] = 12, + [0][1][RTW89_FCC][14] = 36, + [0][1][RTW89_ETSI][14] = 12, + [0][1][RTW89_MKK][14] = 14, + [0][1][RTW89_IC][14] = 36, + [0][1][RTW89_KCC][14] = 32, + [0][1][RTW89_ACMA][14] = 12, + [0][1][RTW89_CHILE][14] = 36, + [0][1][RTW89_UKRAINE][14] = 12, + [0][1][RTW89_MEXICO][14] = 36, + [0][1][RTW89_CN][14] = 12, + [0][1][RTW89_QATAR][14] = 12, + [0][1][RTW89_FCC][15] = 36, + [0][1][RTW89_ETSI][15] = 12, + [0][1][RTW89_MKK][15] = 32, + [0][1][RTW89_IC][15] = 36, + [0][1][RTW89_KCC][15] = 32, + [0][1][RTW89_ACMA][15] = 12, + [0][1][RTW89_CHILE][15] = 36, + [0][1][RTW89_UKRAINE][15] = 12, + [0][1][RTW89_MEXICO][15] = 36, + [0][1][RTW89_CN][15] = 127, + [0][1][RTW89_QATAR][15] = 12, + [0][1][RTW89_FCC][17] = 36, + [0][1][RTW89_ETSI][17] = 12, + [0][1][RTW89_MKK][17] = 32, + [0][1][RTW89_IC][17] = 36, + [0][1][RTW89_KCC][17] = 32, + [0][1][RTW89_ACMA][17] = 12, + [0][1][RTW89_CHILE][17] = 36, + [0][1][RTW89_UKRAINE][17] = 12, + [0][1][RTW89_MEXICO][17] = 36, + [0][1][RTW89_CN][17] = 127, + [0][1][RTW89_QATAR][17] = 12, + [0][1][RTW89_FCC][19] = 36, + [0][1][RTW89_ETSI][19] = 12, + [0][1][RTW89_MKK][19] = 32, + [0][1][RTW89_IC][19] = 36, + [0][1][RTW89_KCC][19] = 32, + [0][1][RTW89_ACMA][19] = 12, + [0][1][RTW89_CHILE][19] = 36, + [0][1][RTW89_UKRAINE][19] = 12, + [0][1][RTW89_MEXICO][19] = 36, + [0][1][RTW89_CN][19] = 127, + [0][1][RTW89_QATAR][19] = 12, + [0][1][RTW89_FCC][21] = 36, + [0][1][RTW89_ETSI][21] = 12, + [0][1][RTW89_MKK][21] = 32, + [0][1][RTW89_IC][21] = 36, + [0][1][RTW89_KCC][21] = 32, + [0][1][RTW89_ACMA][21] = 12, + [0][1][RTW89_CHILE][21] = 36, + [0][1][RTW89_UKRAINE][21] = 12, + [0][1][RTW89_MEXICO][21] = 36, + [0][1][RTW89_CN][21] = 127, + [0][1][RTW89_QATAR][21] = 12, + [0][1][RTW89_FCC][23] = 36, + [0][1][RTW89_ETSI][23] = 12, + [0][1][RTW89_MKK][23] = 32, + [0][1][RTW89_IC][23] = 36, + [0][1][RTW89_KCC][23] = 32, + [0][1][RTW89_ACMA][23] = 12, + [0][1][RTW89_CHILE][23] = 36, + [0][1][RTW89_UKRAINE][23] = 12, + [0][1][RTW89_MEXICO][23] = 36, + [0][1][RTW89_CN][23] = 127, + [0][1][RTW89_QATAR][23] = 12, + [0][1][RTW89_FCC][25] = 36, + [0][1][RTW89_ETSI][25] = 12, + [0][1][RTW89_MKK][25] = 32, + [0][1][RTW89_IC][25] = 127, + [0][1][RTW89_KCC][25] = 32, + [0][1][RTW89_ACMA][25] = 127, + [0][1][RTW89_CHILE][25] = 36, + [0][1][RTW89_UKRAINE][25] = 12, + [0][1][RTW89_MEXICO][25] = 36, + [0][1][RTW89_CN][25] = 127, + [0][1][RTW89_QATAR][25] = 12, + [0][1][RTW89_FCC][27] = 36, + [0][1][RTW89_ETSI][27] = 12, + [0][1][RTW89_MKK][27] = 32, + [0][1][RTW89_IC][27] = 127, + [0][1][RTW89_KCC][27] = 32, + [0][1][RTW89_ACMA][27] = 127, + [0][1][RTW89_CHILE][27] = 36, + [0][1][RTW89_UKRAINE][27] = 12, + [0][1][RTW89_MEXICO][27] = 36, + [0][1][RTW89_CN][27] = 127, + [0][1][RTW89_QATAR][27] = 12, + [0][1][RTW89_FCC][29] = 36, + [0][1][RTW89_ETSI][29] = 12, + [0][1][RTW89_MKK][29] = 32, + [0][1][RTW89_IC][29] = 127, + [0][1][RTW89_KCC][29] = 32, + [0][1][RTW89_ACMA][29] = 127, + [0][1][RTW89_CHILE][29] = 36, + [0][1][RTW89_UKRAINE][29] = 12, + [0][1][RTW89_MEXICO][29] = 36, + [0][1][RTW89_CN][29] = 127, + [0][1][RTW89_QATAR][29] = 12, + [0][1][RTW89_FCC][31] = 36, + [0][1][RTW89_ETSI][31] = 12, + [0][1][RTW89_MKK][31] = 32, + [0][1][RTW89_IC][31] = 36, + [0][1][RTW89_KCC][31] = 32, + [0][1][RTW89_ACMA][31] = 12, + [0][1][RTW89_CHILE][31] = 36, + [0][1][RTW89_UKRAINE][31] = 12, + [0][1][RTW89_MEXICO][31] = 36, + [0][1][RTW89_CN][31] = 127, + [0][1][RTW89_QATAR][31] = 12, + [0][1][RTW89_FCC][33] = 36, + [0][1][RTW89_ETSI][33] = 12, + [0][1][RTW89_MKK][33] = 32, + [0][1][RTW89_IC][33] = 36, + [0][1][RTW89_KCC][33] = 32, + [0][1][RTW89_ACMA][33] = 12, + [0][1][RTW89_CHILE][33] = 36, + [0][1][RTW89_UKRAINE][33] = 12, + [0][1][RTW89_MEXICO][33] = 36, + [0][1][RTW89_CN][33] = 127, + [0][1][RTW89_QATAR][33] = 12, + [0][1][RTW89_FCC][35] = 36, + [0][1][RTW89_ETSI][35] = 12, + [0][1][RTW89_MKK][35] = 32, + [0][1][RTW89_IC][35] = 36, + [0][1][RTW89_KCC][35] = 32, + [0][1][RTW89_ACMA][35] = 12, + [0][1][RTW89_CHILE][35] = 36, + [0][1][RTW89_UKRAINE][35] = 12, + [0][1][RTW89_MEXICO][35] = 36, + [0][1][RTW89_CN][35] = 127, + [0][1][RTW89_QATAR][35] = 12, + [0][1][RTW89_FCC][37] = 36, + [0][1][RTW89_ETSI][37] = 127, + [0][1][RTW89_MKK][37] = 32, + [0][1][RTW89_IC][37] = 36, + [0][1][RTW89_KCC][37] = 32, + [0][1][RTW89_ACMA][37] = 36, + [0][1][RTW89_CHILE][37] = 36, + [0][1][RTW89_UKRAINE][37] = 127, + [0][1][RTW89_MEXICO][37] = 36, + [0][1][RTW89_CN][37] = 127, + [0][1][RTW89_QATAR][37] = 127, + [0][1][RTW89_FCC][38] = 72, + [0][1][RTW89_ETSI][38] = 16, + [0][1][RTW89_MKK][38] = 127, + [0][1][RTW89_IC][38] = 72, + [0][1][RTW89_KCC][38] = 32, + [0][1][RTW89_ACMA][38] = 76, + [0][1][RTW89_CHILE][38] = 42, + [0][1][RTW89_UKRAINE][38] = 16, + [0][1][RTW89_MEXICO][38] = 72, + [0][1][RTW89_CN][38] = 50, + [0][1][RTW89_QATAR][38] = 16, + [0][1][RTW89_FCC][40] = 76, + [0][1][RTW89_ETSI][40] = 16, + [0][1][RTW89_MKK][40] = 127, + [0][1][RTW89_IC][40] = 76, + [0][1][RTW89_KCC][40] = 32, + [0][1][RTW89_ACMA][40] = 76, + [0][1][RTW89_CHILE][40] = 42, + [0][1][RTW89_UKRAINE][40] = 16, + [0][1][RTW89_MEXICO][40] = 76, + [0][1][RTW89_CN][40] = 50, + [0][1][RTW89_QATAR][40] = 16, + [0][1][RTW89_FCC][42] = 76, + [0][1][RTW89_ETSI][42] = 16, + [0][1][RTW89_MKK][42] = 127, + [0][1][RTW89_IC][42] = 76, + [0][1][RTW89_KCC][42] = 32, + [0][1][RTW89_ACMA][42] = 76, + [0][1][RTW89_CHILE][42] = 42, + [0][1][RTW89_UKRAINE][42] = 16, + [0][1][RTW89_MEXICO][42] = 76, + [0][1][RTW89_CN][42] = 50, + [0][1][RTW89_QATAR][42] = 16, + [0][1][RTW89_FCC][44] = 76, + [0][1][RTW89_ETSI][44] = 16, + [0][1][RTW89_MKK][44] = 127, + [0][1][RTW89_IC][44] = 76, + [0][1][RTW89_KCC][44] = 32, + [0][1][RTW89_ACMA][44] = 76, + [0][1][RTW89_CHILE][44] = 42, + [0][1][RTW89_UKRAINE][44] = 16, + [0][1][RTW89_MEXICO][44] = 76, + [0][1][RTW89_CN][44] = 50, + [0][1][RTW89_QATAR][44] = 16, + [0][1][RTW89_FCC][46] = 76, + [0][1][RTW89_ETSI][46] = 16, + [0][1][RTW89_MKK][46] = 127, + [0][1][RTW89_IC][46] = 76, + [0][1][RTW89_KCC][46] = 32, + [0][1][RTW89_ACMA][46] = 76, + [0][1][RTW89_CHILE][46] = 42, + [0][1][RTW89_UKRAINE][46] = 16, + [0][1][RTW89_MEXICO][46] = 76, + [0][1][RTW89_CN][46] = 50, + [0][1][RTW89_QATAR][46] = 16, + [1][0][RTW89_FCC][0] = 62, + [1][0][RTW89_ETSI][0] = 36, + [1][0][RTW89_MKK][0] = 36, + [1][0][RTW89_IC][0] = 34, + [1][0][RTW89_KCC][0] = 58, + [1][0][RTW89_ACMA][0] = 36, + [1][0][RTW89_CHILE][0] = 30, + [1][0][RTW89_UKRAINE][0] = 36, + [1][0][RTW89_MEXICO][0] = 62, + [1][0][RTW89_CN][0] = 36, + [1][0][RTW89_QATAR][0] = 36, + [1][0][RTW89_FCC][2] = 62, + [1][0][RTW89_ETSI][2] = 36, + [1][0][RTW89_MKK][2] = 36, + [1][0][RTW89_IC][2] = 34, + [1][0][RTW89_KCC][2] = 58, + [1][0][RTW89_ACMA][2] = 36, + [1][0][RTW89_CHILE][2] = 30, + [1][0][RTW89_UKRAINE][2] = 36, + [1][0][RTW89_MEXICO][2] = 62, + [1][0][RTW89_CN][2] = 36, + [1][0][RTW89_QATAR][2] = 36, + [1][0][RTW89_FCC][4] = 62, + [1][0][RTW89_ETSI][4] = 36, + [1][0][RTW89_MKK][4] = 36, + [1][0][RTW89_IC][4] = 34, + [1][0][RTW89_KCC][4] = 58, + [1][0][RTW89_ACMA][4] = 36, + [1][0][RTW89_CHILE][4] = 30, + [1][0][RTW89_UKRAINE][4] = 36, + [1][0][RTW89_MEXICO][4] = 62, + [1][0][RTW89_CN][4] = 36, + [1][0][RTW89_QATAR][4] = 36, + [1][0][RTW89_FCC][6] = 62, + [1][0][RTW89_ETSI][6] = 36, + [1][0][RTW89_MKK][6] = 36, + [1][0][RTW89_IC][6] = 34, + [1][0][RTW89_KCC][6] = 32, + [1][0][RTW89_ACMA][6] = 36, + [1][0][RTW89_CHILE][6] = 30, + [1][0][RTW89_UKRAINE][6] = 36, + [1][0][RTW89_MEXICO][6] = 62, + [1][0][RTW89_CN][6] = 36, + [1][0][RTW89_QATAR][6] = 36, + [1][0][RTW89_FCC][8] = 62, + [1][0][RTW89_ETSI][8] = 36, + [1][0][RTW89_MKK][8] = 36, + [1][0][RTW89_IC][8] = 62, + [1][0][RTW89_KCC][8] = 58, + [1][0][RTW89_ACMA][8] = 36, + [1][0][RTW89_CHILE][8] = 54, + [1][0][RTW89_UKRAINE][8] = 36, + [1][0][RTW89_MEXICO][8] = 62, + [1][0][RTW89_CN][8] = 36, + [1][0][RTW89_QATAR][8] = 36, + [1][0][RTW89_FCC][10] = 62, + [1][0][RTW89_ETSI][10] = 36, + [1][0][RTW89_MKK][10] = 36, + [1][0][RTW89_IC][10] = 62, + [1][0][RTW89_KCC][10] = 58, + [1][0][RTW89_ACMA][10] = 36, + [1][0][RTW89_CHILE][10] = 54, + [1][0][RTW89_UKRAINE][10] = 36, + [1][0][RTW89_MEXICO][10] = 62, + [1][0][RTW89_CN][10] = 36, + [1][0][RTW89_QATAR][10] = 36, + [1][0][RTW89_FCC][12] = 62, + [1][0][RTW89_ETSI][12] = 36, + [1][0][RTW89_MKK][12] = 36, + [1][0][RTW89_IC][12] = 62, + [1][0][RTW89_KCC][12] = 58, + [1][0][RTW89_ACMA][12] = 36, + [1][0][RTW89_CHILE][12] = 54, + [1][0][RTW89_UKRAINE][12] = 36, + [1][0][RTW89_MEXICO][12] = 62, + [1][0][RTW89_CN][12] = 36, + [1][0][RTW89_QATAR][12] = 36, + [1][0][RTW89_FCC][14] = 62, + [1][0][RTW89_ETSI][14] = 36, + [1][0][RTW89_MKK][14] = 36, + [1][0][RTW89_IC][14] = 62, + [1][0][RTW89_KCC][14] = 58, + [1][0][RTW89_ACMA][14] = 36, + [1][0][RTW89_CHILE][14] = 54, + [1][0][RTW89_UKRAINE][14] = 36, + [1][0][RTW89_MEXICO][14] = 62, + [1][0][RTW89_CN][14] = 36, + [1][0][RTW89_QATAR][14] = 36, + [1][0][RTW89_FCC][15] = 62, + [1][0][RTW89_ETSI][15] = 36, + [1][0][RTW89_MKK][15] = 58, + [1][0][RTW89_IC][15] = 62, + [1][0][RTW89_KCC][15] = 58, + [1][0][RTW89_ACMA][15] = 36, + [1][0][RTW89_CHILE][15] = 54, + [1][0][RTW89_UKRAINE][15] = 36, + [1][0][RTW89_MEXICO][15] = 62, + [1][0][RTW89_CN][15] = 127, + [1][0][RTW89_QATAR][15] = 36, + [1][0][RTW89_FCC][17] = 62, + [1][0][RTW89_ETSI][17] = 36, + [1][0][RTW89_MKK][17] = 58, + [1][0][RTW89_IC][17] = 62, + [1][0][RTW89_KCC][17] = 58, + [1][0][RTW89_ACMA][17] = 36, + [1][0][RTW89_CHILE][17] = 54, + [1][0][RTW89_UKRAINE][17] = 36, + [1][0][RTW89_MEXICO][17] = 62, + [1][0][RTW89_CN][17] = 127, + [1][0][RTW89_QATAR][17] = 36, + [1][0][RTW89_FCC][19] = 62, + [1][0][RTW89_ETSI][19] = 36, + [1][0][RTW89_MKK][19] = 58, + [1][0][RTW89_IC][19] = 62, + [1][0][RTW89_KCC][19] = 58, + [1][0][RTW89_ACMA][19] = 36, + [1][0][RTW89_CHILE][19] = 54, + [1][0][RTW89_UKRAINE][19] = 36, + [1][0][RTW89_MEXICO][19] = 62, + [1][0][RTW89_CN][19] = 127, + [1][0][RTW89_QATAR][19] = 36, + [1][0][RTW89_FCC][21] = 62, + [1][0][RTW89_ETSI][21] = 36, + [1][0][RTW89_MKK][21] = 58, + [1][0][RTW89_IC][21] = 62, + [1][0][RTW89_KCC][21] = 58, + [1][0][RTW89_ACMA][21] = 36, + [1][0][RTW89_CHILE][21] = 54, + [1][0][RTW89_UKRAINE][21] = 36, + [1][0][RTW89_MEXICO][21] = 62, + [1][0][RTW89_CN][21] = 127, + [1][0][RTW89_QATAR][21] = 36, + [1][0][RTW89_FCC][23] = 62, + [1][0][RTW89_ETSI][23] = 36, + [1][0][RTW89_MKK][23] = 58, + [1][0][RTW89_IC][23] = 62, + [1][0][RTW89_KCC][23] = 58, + [1][0][RTW89_ACMA][23] = 36, + [1][0][RTW89_CHILE][23] = 54, + [1][0][RTW89_UKRAINE][23] = 36, + [1][0][RTW89_MEXICO][23] = 62, + [1][0][RTW89_CN][23] = 127, + [1][0][RTW89_QATAR][23] = 36, + [1][0][RTW89_FCC][25] = 62, + [1][0][RTW89_ETSI][25] = 36, + [1][0][RTW89_MKK][25] = 58, + [1][0][RTW89_IC][25] = 127, + [1][0][RTW89_KCC][25] = 58, + [1][0][RTW89_ACMA][25] = 127, + [1][0][RTW89_CHILE][25] = 54, + [1][0][RTW89_UKRAINE][25] = 36, + [1][0][RTW89_MEXICO][25] = 62, + [1][0][RTW89_CN][25] = 127, + [1][0][RTW89_QATAR][25] = 36, + [1][0][RTW89_FCC][27] = 62, + [1][0][RTW89_ETSI][27] = 36, + [1][0][RTW89_MKK][27] = 58, + [1][0][RTW89_IC][27] = 127, + [1][0][RTW89_KCC][27] = 58, + [1][0][RTW89_ACMA][27] = 127, + [1][0][RTW89_CHILE][27] = 54, + [1][0][RTW89_UKRAINE][27] = 36, + [1][0][RTW89_MEXICO][27] = 62, + [1][0][RTW89_CN][27] = 127, + [1][0][RTW89_QATAR][27] = 36, + [1][0][RTW89_FCC][29] = 62, + [1][0][RTW89_ETSI][29] = 36, + [1][0][RTW89_MKK][29] = 58, + [1][0][RTW89_IC][29] = 127, + [1][0][RTW89_KCC][29] = 58, + [1][0][RTW89_ACMA][29] = 127, + [1][0][RTW89_CHILE][29] = 54, + [1][0][RTW89_UKRAINE][29] = 36, + [1][0][RTW89_MEXICO][29] = 62, + [1][0][RTW89_CN][29] = 127, + [1][0][RTW89_QATAR][29] = 36, + [1][0][RTW89_FCC][31] = 62, + [1][0][RTW89_ETSI][31] = 36, + [1][0][RTW89_MKK][31] = 58, + [1][0][RTW89_IC][31] = 62, + [1][0][RTW89_KCC][31] = 58, + [1][0][RTW89_ACMA][31] = 36, + [1][0][RTW89_CHILE][31] = 54, + [1][0][RTW89_UKRAINE][31] = 36, + [1][0][RTW89_MEXICO][31] = 62, + [1][0][RTW89_CN][31] = 127, + [1][0][RTW89_QATAR][31] = 36, + [1][0][RTW89_FCC][33] = 62, + [1][0][RTW89_ETSI][33] = 36, + [1][0][RTW89_MKK][33] = 58, + [1][0][RTW89_IC][33] = 62, + [1][0][RTW89_KCC][33] = 58, + [1][0][RTW89_ACMA][33] = 36, + [1][0][RTW89_CHILE][33] = 54, + [1][0][RTW89_UKRAINE][33] = 36, + [1][0][RTW89_MEXICO][33] = 62, + [1][0][RTW89_CN][33] = 127, + [1][0][RTW89_QATAR][33] = 36, + [1][0][RTW89_FCC][35] = 62, + [1][0][RTW89_ETSI][35] = 36, + [1][0][RTW89_MKK][35] = 58, + [1][0][RTW89_IC][35] = 62, + [1][0][RTW89_KCC][35] = 58, + [1][0][RTW89_ACMA][35] = 36, + [1][0][RTW89_CHILE][35] = 54, + [1][0][RTW89_UKRAINE][35] = 36, + [1][0][RTW89_MEXICO][35] = 62, + [1][0][RTW89_CN][35] = 127, + [1][0][RTW89_QATAR][35] = 36, + [1][0][RTW89_FCC][37] = 62, + [1][0][RTW89_ETSI][37] = 127, + [1][0][RTW89_MKK][37] = 58, + [1][0][RTW89_IC][37] = 62, + [1][0][RTW89_KCC][37] = 58, + [1][0][RTW89_ACMA][37] = 62, + [1][0][RTW89_CHILE][37] = 54, + [1][0][RTW89_UKRAINE][37] = 127, + [1][0][RTW89_MEXICO][37] = 62, + [1][0][RTW89_CN][37] = 127, + [1][0][RTW89_QATAR][37] = 127, + [1][0][RTW89_FCC][38] = 76, + [1][0][RTW89_ETSI][38] = 28, + [1][0][RTW89_MKK][38] = 127, + [1][0][RTW89_IC][38] = 76, + [1][0][RTW89_KCC][38] = 58, + [1][0][RTW89_ACMA][38] = 76, + [1][0][RTW89_CHILE][38] = 54, + [1][0][RTW89_UKRAINE][38] = 28, + [1][0][RTW89_MEXICO][38] = 76, + [1][0][RTW89_CN][38] = 74, + [1][0][RTW89_QATAR][38] = 28, + [1][0][RTW89_FCC][40] = 76, + [1][0][RTW89_ETSI][40] = 28, + [1][0][RTW89_MKK][40] = 127, + [1][0][RTW89_IC][40] = 76, + [1][0][RTW89_KCC][40] = 58, + [1][0][RTW89_ACMA][40] = 76, + [1][0][RTW89_CHILE][40] = 54, + [1][0][RTW89_UKRAINE][40] = 28, + [1][0][RTW89_MEXICO][40] = 76, + [1][0][RTW89_CN][40] = 74, + [1][0][RTW89_QATAR][40] = 28, + [1][0][RTW89_FCC][42] = 76, + [1][0][RTW89_ETSI][42] = 28, + [1][0][RTW89_MKK][42] = 127, + [1][0][RTW89_IC][42] = 76, + [1][0][RTW89_KCC][42] = 58, + [1][0][RTW89_ACMA][42] = 76, + [1][0][RTW89_CHILE][42] = 54, + [1][0][RTW89_UKRAINE][42] = 28, + [1][0][RTW89_MEXICO][42] = 76, + [1][0][RTW89_CN][42] = 74, + [1][0][RTW89_QATAR][42] = 28, + [1][0][RTW89_FCC][44] = 76, + [1][0][RTW89_ETSI][44] = 28, + [1][0][RTW89_MKK][44] = 127, + [1][0][RTW89_IC][44] = 76, + [1][0][RTW89_KCC][44] = 58, + [1][0][RTW89_ACMA][44] = 76, + [1][0][RTW89_CHILE][44] = 54, + [1][0][RTW89_UKRAINE][44] = 28, + [1][0][RTW89_MEXICO][44] = 76, + [1][0][RTW89_CN][44] = 74, + [1][0][RTW89_QATAR][44] = 28, + [1][0][RTW89_FCC][46] = 76, + [1][0][RTW89_ETSI][46] = 28, + [1][0][RTW89_MKK][46] = 127, + [1][0][RTW89_IC][46] = 76, + [1][0][RTW89_KCC][46] = 58, + [1][0][RTW89_ACMA][46] = 76, + [1][0][RTW89_CHILE][46] = 54, + [1][0][RTW89_UKRAINE][46] = 28, + [1][0][RTW89_MEXICO][46] = 76, + [1][0][RTW89_CN][46] = 74, + [1][0][RTW89_QATAR][46] = 28, + [1][1][RTW89_FCC][0] = 46, + [1][1][RTW89_ETSI][0] = 22, + [1][1][RTW89_MKK][0] = 24, + [1][1][RTW89_IC][0] = 18, + [1][1][RTW89_KCC][0] = 44, + [1][1][RTW89_ACMA][0] = 22, + [1][1][RTW89_CHILE][0] = 18, + [1][1][RTW89_UKRAINE][0] = 22, + [1][1][RTW89_MEXICO][0] = 46, + [1][1][RTW89_CN][0] = 22, + [1][1][RTW89_QATAR][0] = 22, + [1][1][RTW89_FCC][2] = 46, + [1][1][RTW89_ETSI][2] = 22, + [1][1][RTW89_MKK][2] = 24, + [1][1][RTW89_IC][2] = 18, + [1][1][RTW89_KCC][2] = 44, + [1][1][RTW89_ACMA][2] = 22, + [1][1][RTW89_CHILE][2] = 18, + [1][1][RTW89_UKRAINE][2] = 22, + [1][1][RTW89_MEXICO][2] = 46, + [1][1][RTW89_CN][2] = 22, + [1][1][RTW89_QATAR][2] = 22, + [1][1][RTW89_FCC][4] = 46, + [1][1][RTW89_ETSI][4] = 22, + [1][1][RTW89_MKK][4] = 24, + [1][1][RTW89_IC][4] = 18, + [1][1][RTW89_KCC][4] = 44, + [1][1][RTW89_ACMA][4] = 22, + [1][1][RTW89_CHILE][4] = 18, + [1][1][RTW89_UKRAINE][4] = 22, + [1][1][RTW89_MEXICO][4] = 46, + [1][1][RTW89_CN][4] = 22, + [1][1][RTW89_QATAR][4] = 22, + [1][1][RTW89_FCC][6] = 46, + [1][1][RTW89_ETSI][6] = 22, + [1][1][RTW89_MKK][6] = 24, + [1][1][RTW89_IC][6] = 18, + [1][1][RTW89_KCC][6] = 16, + [1][1][RTW89_ACMA][6] = 22, + [1][1][RTW89_CHILE][6] = 18, + [1][1][RTW89_UKRAINE][6] = 22, + [1][1][RTW89_MEXICO][6] = 46, + [1][1][RTW89_CN][6] = 22, + [1][1][RTW89_QATAR][6] = 22, + [1][1][RTW89_FCC][8] = 46, + [1][1][RTW89_ETSI][8] = 22, + [1][1][RTW89_MKK][8] = 24, + [1][1][RTW89_IC][8] = 46, + [1][1][RTW89_KCC][8] = 44, + [1][1][RTW89_ACMA][8] = 22, + [1][1][RTW89_CHILE][8] = 42, + [1][1][RTW89_UKRAINE][8] = 22, + [1][1][RTW89_MEXICO][8] = 46, + [1][1][RTW89_CN][8] = 22, + [1][1][RTW89_QATAR][8] = 22, + [1][1][RTW89_FCC][10] = 46, + [1][1][RTW89_ETSI][10] = 22, + [1][1][RTW89_MKK][10] = 24, + [1][1][RTW89_IC][10] = 46, + [1][1][RTW89_KCC][10] = 44, + [1][1][RTW89_ACMA][10] = 22, + [1][1][RTW89_CHILE][10] = 42, + [1][1][RTW89_UKRAINE][10] = 22, + [1][1][RTW89_MEXICO][10] = 46, + [1][1][RTW89_CN][10] = 22, + [1][1][RTW89_QATAR][10] = 22, + [1][1][RTW89_FCC][12] = 46, + [1][1][RTW89_ETSI][12] = 22, + [1][1][RTW89_MKK][12] = 24, + [1][1][RTW89_IC][12] = 46, + [1][1][RTW89_KCC][12] = 44, + [1][1][RTW89_ACMA][12] = 22, + [1][1][RTW89_CHILE][12] = 42, + [1][1][RTW89_UKRAINE][12] = 22, + [1][1][RTW89_MEXICO][12] = 46, + [1][1][RTW89_CN][12] = 22, + [1][1][RTW89_QATAR][12] = 22, + [1][1][RTW89_FCC][14] = 46, + [1][1][RTW89_ETSI][14] = 22, + [1][1][RTW89_MKK][14] = 24, + [1][1][RTW89_IC][14] = 46, + [1][1][RTW89_KCC][14] = 44, + [1][1][RTW89_ACMA][14] = 22, + [1][1][RTW89_CHILE][14] = 42, + [1][1][RTW89_UKRAINE][14] = 22, + [1][1][RTW89_MEXICO][14] = 46, + [1][1][RTW89_CN][14] = 22, + [1][1][RTW89_QATAR][14] = 22, + [1][1][RTW89_FCC][15] = 46, + [1][1][RTW89_ETSI][15] = 22, + [1][1][RTW89_MKK][15] = 46, + [1][1][RTW89_IC][15] = 46, + [1][1][RTW89_KCC][15] = 44, + [1][1][RTW89_ACMA][15] = 22, + [1][1][RTW89_CHILE][15] = 42, + [1][1][RTW89_UKRAINE][15] = 22, + [1][1][RTW89_MEXICO][15] = 46, + [1][1][RTW89_CN][15] = 127, + [1][1][RTW89_QATAR][15] = 22, + [1][1][RTW89_FCC][17] = 46, + [1][1][RTW89_ETSI][17] = 22, + [1][1][RTW89_MKK][17] = 46, + [1][1][RTW89_IC][17] = 46, + [1][1][RTW89_KCC][17] = 44, + [1][1][RTW89_ACMA][17] = 22, + [1][1][RTW89_CHILE][17] = 42, + [1][1][RTW89_UKRAINE][17] = 22, + [1][1][RTW89_MEXICO][17] = 46, + [1][1][RTW89_CN][17] = 127, + [1][1][RTW89_QATAR][17] = 22, + [1][1][RTW89_FCC][19] = 46, + [1][1][RTW89_ETSI][19] = 22, + [1][1][RTW89_MKK][19] = 46, + [1][1][RTW89_IC][19] = 46, + [1][1][RTW89_KCC][19] = 44, + [1][1][RTW89_ACMA][19] = 22, + [1][1][RTW89_CHILE][19] = 42, + [1][1][RTW89_UKRAINE][19] = 22, + [1][1][RTW89_MEXICO][19] = 46, + [1][1][RTW89_CN][19] = 127, + [1][1][RTW89_QATAR][19] = 22, + [1][1][RTW89_FCC][21] = 46, + [1][1][RTW89_ETSI][21] = 22, + [1][1][RTW89_MKK][21] = 46, + [1][1][RTW89_IC][21] = 46, + [1][1][RTW89_KCC][21] = 44, + [1][1][RTW89_ACMA][21] = 22, + [1][1][RTW89_CHILE][21] = 42, + [1][1][RTW89_UKRAINE][21] = 22, + [1][1][RTW89_MEXICO][21] = 46, + [1][1][RTW89_CN][21] = 127, + [1][1][RTW89_QATAR][21] = 22, + [1][1][RTW89_FCC][23] = 46, + [1][1][RTW89_ETSI][23] = 22, + [1][1][RTW89_MKK][23] = 46, + [1][1][RTW89_IC][23] = 46, + [1][1][RTW89_KCC][23] = 44, + [1][1][RTW89_ACMA][23] = 22, + [1][1][RTW89_CHILE][23] = 42, + [1][1][RTW89_UKRAINE][23] = 22, + [1][1][RTW89_MEXICO][23] = 46, + [1][1][RTW89_CN][23] = 127, + [1][1][RTW89_QATAR][23] = 22, + [1][1][RTW89_FCC][25] = 46, + [1][1][RTW89_ETSI][25] = 22, + [1][1][RTW89_MKK][25] = 46, + [1][1][RTW89_IC][25] = 127, + [1][1][RTW89_KCC][25] = 44, + [1][1][RTW89_ACMA][25] = 127, + [1][1][RTW89_CHILE][25] = 42, + [1][1][RTW89_UKRAINE][25] = 22, + [1][1][RTW89_MEXICO][25] = 46, + [1][1][RTW89_CN][25] = 127, + [1][1][RTW89_QATAR][25] = 22, + [1][1][RTW89_FCC][27] = 46, + [1][1][RTW89_ETSI][27] = 22, + [1][1][RTW89_MKK][27] = 46, + [1][1][RTW89_IC][27] = 127, + [1][1][RTW89_KCC][27] = 44, + [1][1][RTW89_ACMA][27] = 127, + [1][1][RTW89_CHILE][27] = 42, + [1][1][RTW89_UKRAINE][27] = 22, + [1][1][RTW89_MEXICO][27] = 46, + [1][1][RTW89_CN][27] = 127, + [1][1][RTW89_QATAR][27] = 22, + [1][1][RTW89_FCC][29] = 46, + [1][1][RTW89_ETSI][29] = 22, + [1][1][RTW89_MKK][29] = 46, + [1][1][RTW89_IC][29] = 127, + [1][1][RTW89_KCC][29] = 44, + [1][1][RTW89_ACMA][29] = 127, + [1][1][RTW89_CHILE][29] = 42, + [1][1][RTW89_UKRAINE][29] = 22, + [1][1][RTW89_MEXICO][29] = 46, + [1][1][RTW89_CN][29] = 127, + [1][1][RTW89_QATAR][29] = 22, + [1][1][RTW89_FCC][31] = 46, + [1][1][RTW89_ETSI][31] = 22, + [1][1][RTW89_MKK][31] = 46, + [1][1][RTW89_IC][31] = 46, + [1][1][RTW89_KCC][31] = 44, + [1][1][RTW89_ACMA][31] = 22, + [1][1][RTW89_CHILE][31] = 42, + [1][1][RTW89_UKRAINE][31] = 22, + [1][1][RTW89_MEXICO][31] = 46, + [1][1][RTW89_CN][31] = 127, + [1][1][RTW89_QATAR][31] = 22, + [1][1][RTW89_FCC][33] = 46, + [1][1][RTW89_ETSI][33] = 22, + [1][1][RTW89_MKK][33] = 46, + [1][1][RTW89_IC][33] = 46, + [1][1][RTW89_KCC][33] = 44, + [1][1][RTW89_ACMA][33] = 22, + [1][1][RTW89_CHILE][33] = 42, + [1][1][RTW89_UKRAINE][33] = 22, + [1][1][RTW89_MEXICO][33] = 46, + [1][1][RTW89_CN][33] = 127, + [1][1][RTW89_QATAR][33] = 22, + [1][1][RTW89_FCC][35] = 46, + [1][1][RTW89_ETSI][35] = 22, + [1][1][RTW89_MKK][35] = 46, + [1][1][RTW89_IC][35] = 46, + [1][1][RTW89_KCC][35] = 44, + [1][1][RTW89_ACMA][35] = 22, + [1][1][RTW89_CHILE][35] = 42, + [1][1][RTW89_UKRAINE][35] = 22, + [1][1][RTW89_MEXICO][35] = 46, + [1][1][RTW89_CN][35] = 127, + [1][1][RTW89_QATAR][35] = 22, + [1][1][RTW89_FCC][37] = 46, + [1][1][RTW89_ETSI][37] = 127, + [1][1][RTW89_MKK][37] = 46, + [1][1][RTW89_IC][37] = 46, + [1][1][RTW89_KCC][37] = 44, + [1][1][RTW89_ACMA][37] = 50, + [1][1][RTW89_CHILE][37] = 42, + [1][1][RTW89_UKRAINE][37] = 127, + [1][1][RTW89_MEXICO][37] = 46, + [1][1][RTW89_CN][37] = 127, + [1][1][RTW89_QATAR][37] = 127, + [1][1][RTW89_FCC][38] = 74, + [1][1][RTW89_ETSI][38] = 16, + [1][1][RTW89_MKK][38] = 127, + [1][1][RTW89_IC][38] = 74, + [1][1][RTW89_KCC][38] = 44, + [1][1][RTW89_ACMA][38] = 76, + [1][1][RTW89_CHILE][38] = 42, + [1][1][RTW89_UKRAINE][38] = 16, + [1][1][RTW89_MEXICO][38] = 74, + [1][1][RTW89_CN][38] = 62, + [1][1][RTW89_QATAR][38] = 16, + [1][1][RTW89_FCC][40] = 76, + [1][1][RTW89_ETSI][40] = 16, + [1][1][RTW89_MKK][40] = 127, + [1][1][RTW89_IC][40] = 76, + [1][1][RTW89_KCC][40] = 44, + [1][1][RTW89_ACMA][40] = 76, + [1][1][RTW89_CHILE][40] = 42, + [1][1][RTW89_UKRAINE][40] = 16, + [1][1][RTW89_MEXICO][40] = 76, + [1][1][RTW89_CN][40] = 62, + [1][1][RTW89_QATAR][40] = 16, + [1][1][RTW89_FCC][42] = 76, + [1][1][RTW89_ETSI][42] = 16, + [1][1][RTW89_MKK][42] = 127, + [1][1][RTW89_IC][42] = 76, + [1][1][RTW89_KCC][42] = 44, + [1][1][RTW89_ACMA][42] = 76, + [1][1][RTW89_CHILE][42] = 42, + [1][1][RTW89_UKRAINE][42] = 16, + [1][1][RTW89_MEXICO][42] = 76, + [1][1][RTW89_CN][42] = 62, + [1][1][RTW89_QATAR][42] = 16, + [1][1][RTW89_FCC][44] = 76, + [1][1][RTW89_ETSI][44] = 16, + [1][1][RTW89_MKK][44] = 127, + [1][1][RTW89_IC][44] = 76, + [1][1][RTW89_KCC][44] = 44, + [1][1][RTW89_ACMA][44] = 76, + [1][1][RTW89_CHILE][44] = 42, + [1][1][RTW89_UKRAINE][44] = 16, + [1][1][RTW89_MEXICO][44] = 76, + [1][1][RTW89_CN][44] = 62, + [1][1][RTW89_QATAR][44] = 16, + [1][1][RTW89_FCC][46] = 76, + [1][1][RTW89_ETSI][46] = 16, + [1][1][RTW89_MKK][46] = 127, + [1][1][RTW89_IC][46] = 76, + [1][1][RTW89_KCC][46] = 44, + [1][1][RTW89_ACMA][46] = 76, + [1][1][RTW89_CHILE][46] = 42, + [1][1][RTW89_UKRAINE][46] = 16, + [1][1][RTW89_MEXICO][46] = 76, + [1][1][RTW89_CN][46] = 62, + [1][1][RTW89_QATAR][46] = 16, + [2][0][RTW89_FCC][0] = 74, + [2][0][RTW89_ETSI][0] = 46, + [2][0][RTW89_MKK][0] = 50, + [2][0][RTW89_IC][0] = 46, + [2][0][RTW89_KCC][0] = 70, + [2][0][RTW89_ACMA][0] = 46, + [2][0][RTW89_CHILE][0] = 30, + [2][0][RTW89_UKRAINE][0] = 46, + [2][0][RTW89_MEXICO][0] = 62, + [2][0][RTW89_CN][0] = 46, + [2][0][RTW89_QATAR][0] = 46, + [2][0][RTW89_FCC][2] = 74, + [2][0][RTW89_ETSI][2] = 46, + [2][0][RTW89_MKK][2] = 50, + [2][0][RTW89_IC][2] = 46, + [2][0][RTW89_KCC][2] = 70, + [2][0][RTW89_ACMA][2] = 46, + [2][0][RTW89_CHILE][2] = 30, + [2][0][RTW89_UKRAINE][2] = 46, + [2][0][RTW89_MEXICO][2] = 62, + [2][0][RTW89_CN][2] = 46, + [2][0][RTW89_QATAR][2] = 46, + [2][0][RTW89_FCC][4] = 74, + [2][0][RTW89_ETSI][4] = 46, + [2][0][RTW89_MKK][4] = 50, + [2][0][RTW89_IC][4] = 46, + [2][0][RTW89_KCC][4] = 70, + [2][0][RTW89_ACMA][4] = 46, + [2][0][RTW89_CHILE][4] = 30, + [2][0][RTW89_UKRAINE][4] = 46, + [2][0][RTW89_MEXICO][4] = 62, + [2][0][RTW89_CN][4] = 46, + [2][0][RTW89_QATAR][4] = 46, + [2][0][RTW89_FCC][6] = 74, + [2][0][RTW89_ETSI][6] = 46, + [2][0][RTW89_MKK][6] = 50, + [2][0][RTW89_IC][6] = 46, + [2][0][RTW89_KCC][6] = 44, + [2][0][RTW89_ACMA][6] = 46, + [2][0][RTW89_CHILE][6] = 30, + [2][0][RTW89_UKRAINE][6] = 46, + [2][0][RTW89_MEXICO][6] = 62, + [2][0][RTW89_CN][6] = 46, + [2][0][RTW89_QATAR][6] = 46, + [2][0][RTW89_FCC][8] = 74, + [2][0][RTW89_ETSI][8] = 46, + [2][0][RTW89_MKK][8] = 50, + [2][0][RTW89_IC][8] = 66, + [2][0][RTW89_KCC][8] = 70, + [2][0][RTW89_ACMA][8] = 46, + [2][0][RTW89_CHILE][8] = 54, + [2][0][RTW89_UKRAINE][8] = 46, + [2][0][RTW89_MEXICO][8] = 74, + [2][0][RTW89_CN][8] = 46, + [2][0][RTW89_QATAR][8] = 46, + [2][0][RTW89_FCC][10] = 74, + [2][0][RTW89_ETSI][10] = 46, + [2][0][RTW89_MKK][10] = 50, + [2][0][RTW89_IC][10] = 66, + [2][0][RTW89_KCC][10] = 70, + [2][0][RTW89_ACMA][10] = 46, + [2][0][RTW89_CHILE][10] = 54, + [2][0][RTW89_UKRAINE][10] = 46, + [2][0][RTW89_MEXICO][10] = 74, + [2][0][RTW89_CN][10] = 46, + [2][0][RTW89_QATAR][10] = 46, + [2][0][RTW89_FCC][12] = 74, + [2][0][RTW89_ETSI][12] = 46, + [2][0][RTW89_MKK][12] = 50, + [2][0][RTW89_IC][12] = 66, + [2][0][RTW89_KCC][12] = 70, + [2][0][RTW89_ACMA][12] = 46, + [2][0][RTW89_CHILE][12] = 54, + [2][0][RTW89_UKRAINE][12] = 46, + [2][0][RTW89_MEXICO][12] = 74, + [2][0][RTW89_CN][12] = 46, + [2][0][RTW89_QATAR][12] = 46, + [2][0][RTW89_FCC][14] = 74, + [2][0][RTW89_ETSI][14] = 46, + [2][0][RTW89_MKK][14] = 50, + [2][0][RTW89_IC][14] = 66, + [2][0][RTW89_KCC][14] = 70, + [2][0][RTW89_ACMA][14] = 46, + [2][0][RTW89_CHILE][14] = 54, + [2][0][RTW89_UKRAINE][14] = 46, + [2][0][RTW89_MEXICO][14] = 74, + [2][0][RTW89_CN][14] = 46, + [2][0][RTW89_QATAR][14] = 46, + [2][0][RTW89_FCC][15] = 74, + [2][0][RTW89_ETSI][15] = 46, + [2][0][RTW89_MKK][15] = 70, + [2][0][RTW89_IC][15] = 74, + [2][0][RTW89_KCC][15] = 70, + [2][0][RTW89_ACMA][15] = 46, + [2][0][RTW89_CHILE][15] = 54, + [2][0][RTW89_UKRAINE][15] = 46, + [2][0][RTW89_MEXICO][15] = 74, + [2][0][RTW89_CN][15] = 127, + [2][0][RTW89_QATAR][15] = 46, + [2][0][RTW89_FCC][17] = 74, + [2][0][RTW89_ETSI][17] = 46, + [2][0][RTW89_MKK][17] = 70, + [2][0][RTW89_IC][17] = 74, + [2][0][RTW89_KCC][17] = 70, + [2][0][RTW89_ACMA][17] = 46, + [2][0][RTW89_CHILE][17] = 54, + [2][0][RTW89_UKRAINE][17] = 46, + [2][0][RTW89_MEXICO][17] = 74, + [2][0][RTW89_CN][17] = 127, + [2][0][RTW89_QATAR][17] = 46, + [2][0][RTW89_FCC][19] = 74, + [2][0][RTW89_ETSI][19] = 46, + [2][0][RTW89_MKK][19] = 70, + [2][0][RTW89_IC][19] = 74, + [2][0][RTW89_KCC][19] = 70, + [2][0][RTW89_ACMA][19] = 46, + [2][0][RTW89_CHILE][19] = 54, + [2][0][RTW89_UKRAINE][19] = 46, + [2][0][RTW89_MEXICO][19] = 74, + [2][0][RTW89_CN][19] = 127, + [2][0][RTW89_QATAR][19] = 46, + [2][0][RTW89_FCC][21] = 74, + [2][0][RTW89_ETSI][21] = 46, + [2][0][RTW89_MKK][21] = 70, + [2][0][RTW89_IC][21] = 74, + [2][0][RTW89_KCC][21] = 70, + [2][0][RTW89_ACMA][21] = 46, + [2][0][RTW89_CHILE][21] = 54, + [2][0][RTW89_UKRAINE][21] = 46, + [2][0][RTW89_MEXICO][21] = 74, + [2][0][RTW89_CN][21] = 127, + [2][0][RTW89_QATAR][21] = 46, + [2][0][RTW89_FCC][23] = 74, + [2][0][RTW89_ETSI][23] = 46, + [2][0][RTW89_MKK][23] = 70, + [2][0][RTW89_IC][23] = 74, + [2][0][RTW89_KCC][23] = 70, + [2][0][RTW89_ACMA][23] = 46, + [2][0][RTW89_CHILE][23] = 54, + [2][0][RTW89_UKRAINE][23] = 46, + [2][0][RTW89_MEXICO][23] = 74, + [2][0][RTW89_CN][23] = 127, + [2][0][RTW89_QATAR][23] = 46, + [2][0][RTW89_FCC][25] = 74, + [2][0][RTW89_ETSI][25] = 46, + [2][0][RTW89_MKK][25] = 70, + [2][0][RTW89_IC][25] = 127, + [2][0][RTW89_KCC][25] = 70, + [2][0][RTW89_ACMA][25] = 127, + [2][0][RTW89_CHILE][25] = 54, + [2][0][RTW89_UKRAINE][25] = 46, + [2][0][RTW89_MEXICO][25] = 74, + [2][0][RTW89_CN][25] = 127, + [2][0][RTW89_QATAR][25] = 46, + [2][0][RTW89_FCC][27] = 74, + [2][0][RTW89_ETSI][27] = 46, + [2][0][RTW89_MKK][27] = 70, + [2][0][RTW89_IC][27] = 127, + [2][0][RTW89_KCC][27] = 70, + [2][0][RTW89_ACMA][27] = 127, + [2][0][RTW89_CHILE][27] = 54, + [2][0][RTW89_UKRAINE][27] = 46, + [2][0][RTW89_MEXICO][27] = 74, + [2][0][RTW89_CN][27] = 127, + [2][0][RTW89_QATAR][27] = 46, + [2][0][RTW89_FCC][29] = 74, + [2][0][RTW89_ETSI][29] = 46, + [2][0][RTW89_MKK][29] = 70, + [2][0][RTW89_IC][29] = 127, + [2][0][RTW89_KCC][29] = 70, + [2][0][RTW89_ACMA][29] = 127, + [2][0][RTW89_CHILE][29] = 54, + [2][0][RTW89_UKRAINE][29] = 46, + [2][0][RTW89_MEXICO][29] = 74, + [2][0][RTW89_CN][29] = 127, + [2][0][RTW89_QATAR][29] = 46, + [2][0][RTW89_FCC][31] = 74, + [2][0][RTW89_ETSI][31] = 46, + [2][0][RTW89_MKK][31] = 70, + [2][0][RTW89_IC][31] = 74, + [2][0][RTW89_KCC][31] = 70, + [2][0][RTW89_ACMA][31] = 46, + [2][0][RTW89_CHILE][31] = 54, + [2][0][RTW89_UKRAINE][31] = 46, + [2][0][RTW89_MEXICO][31] = 74, + [2][0][RTW89_CN][31] = 127, + [2][0][RTW89_QATAR][31] = 46, + [2][0][RTW89_FCC][33] = 74, + [2][0][RTW89_ETSI][33] = 46, + [2][0][RTW89_MKK][33] = 70, + [2][0][RTW89_IC][33] = 74, + [2][0][RTW89_KCC][33] = 70, + [2][0][RTW89_ACMA][33] = 46, + [2][0][RTW89_CHILE][33] = 54, + [2][0][RTW89_UKRAINE][33] = 46, + [2][0][RTW89_MEXICO][33] = 74, + [2][0][RTW89_CN][33] = 127, + [2][0][RTW89_QATAR][33] = 46, + [2][0][RTW89_FCC][35] = 74, + [2][0][RTW89_ETSI][35] = 46, + [2][0][RTW89_MKK][35] = 70, + [2][0][RTW89_IC][35] = 74, + [2][0][RTW89_KCC][35] = 70, + [2][0][RTW89_ACMA][35] = 46, + [2][0][RTW89_CHILE][35] = 54, + [2][0][RTW89_UKRAINE][35] = 46, + [2][0][RTW89_MEXICO][35] = 74, + [2][0][RTW89_CN][35] = 127, + [2][0][RTW89_QATAR][35] = 46, + [2][0][RTW89_FCC][37] = 74, + [2][0][RTW89_ETSI][37] = 127, + [2][0][RTW89_MKK][37] = 70, + [2][0][RTW89_IC][37] = 74, + [2][0][RTW89_KCC][37] = 70, + [2][0][RTW89_ACMA][37] = 74, + [2][0][RTW89_CHILE][37] = 54, + [2][0][RTW89_UKRAINE][37] = 127, + [2][0][RTW89_MEXICO][37] = 74, + [2][0][RTW89_CN][37] = 127, + [2][0][RTW89_QATAR][37] = 127, + [2][0][RTW89_FCC][38] = 76, + [2][0][RTW89_ETSI][38] = 28, + [2][0][RTW89_MKK][38] = 127, + [2][0][RTW89_IC][38] = 76, + [2][0][RTW89_KCC][38] = 70, + [2][0][RTW89_ACMA][38] = 76, + [2][0][RTW89_CHILE][38] = 54, + [2][0][RTW89_UKRAINE][38] = 28, + [2][0][RTW89_MEXICO][38] = 76, + [2][0][RTW89_CN][38] = 76, + [2][0][RTW89_QATAR][38] = 28, + [2][0][RTW89_FCC][40] = 76, + [2][0][RTW89_ETSI][40] = 28, + [2][0][RTW89_MKK][40] = 127, + [2][0][RTW89_IC][40] = 76, + [2][0][RTW89_KCC][40] = 70, + [2][0][RTW89_ACMA][40] = 76, + [2][0][RTW89_CHILE][40] = 54, + [2][0][RTW89_UKRAINE][40] = 28, + [2][0][RTW89_MEXICO][40] = 76, + [2][0][RTW89_CN][40] = 76, + [2][0][RTW89_QATAR][40] = 28, + [2][0][RTW89_FCC][42] = 76, + [2][0][RTW89_ETSI][42] = 28, + [2][0][RTW89_MKK][42] = 127, + [2][0][RTW89_IC][42] = 76, + [2][0][RTW89_KCC][42] = 70, + [2][0][RTW89_ACMA][42] = 76, + [2][0][RTW89_CHILE][42] = 54, + [2][0][RTW89_UKRAINE][42] = 28, + [2][0][RTW89_MEXICO][42] = 76, + [2][0][RTW89_CN][42] = 76, + [2][0][RTW89_QATAR][42] = 28, + [2][0][RTW89_FCC][44] = 76, + [2][0][RTW89_ETSI][44] = 28, + [2][0][RTW89_MKK][44] = 127, + [2][0][RTW89_IC][44] = 76, + [2][0][RTW89_KCC][44] = 70, + [2][0][RTW89_ACMA][44] = 76, + [2][0][RTW89_CHILE][44] = 54, + [2][0][RTW89_UKRAINE][44] = 28, + [2][0][RTW89_MEXICO][44] = 76, + [2][0][RTW89_CN][44] = 76, + [2][0][RTW89_QATAR][44] = 28, + [2][0][RTW89_FCC][46] = 76, + [2][0][RTW89_ETSI][46] = 28, + [2][0][RTW89_MKK][46] = 127, + [2][0][RTW89_IC][46] = 76, + [2][0][RTW89_KCC][46] = 70, + [2][0][RTW89_ACMA][46] = 76, + [2][0][RTW89_CHILE][46] = 54, + [2][0][RTW89_UKRAINE][46] = 28, + [2][0][RTW89_MEXICO][46] = 76, + [2][0][RTW89_CN][46] = 76, + [2][0][RTW89_QATAR][46] = 28, + [2][1][RTW89_FCC][0] = 58, + [2][1][RTW89_ETSI][0] = 32, + [2][1][RTW89_MKK][0] = 38, + [2][1][RTW89_IC][0] = 30, + [2][1][RTW89_KCC][0] = 54, + [2][1][RTW89_ACMA][0] = 32, + [2][1][RTW89_CHILE][0] = 18, + [2][1][RTW89_UKRAINE][0] = 32, + [2][1][RTW89_MEXICO][0] = 50, + [2][1][RTW89_CN][0] = 32, + [2][1][RTW89_QATAR][0] = 32, + [2][1][RTW89_FCC][2] = 58, + [2][1][RTW89_ETSI][2] = 32, + [2][1][RTW89_MKK][2] = 38, + [2][1][RTW89_IC][2] = 30, + [2][1][RTW89_KCC][2] = 54, + [2][1][RTW89_ACMA][2] = 32, + [2][1][RTW89_CHILE][2] = 18, + [2][1][RTW89_UKRAINE][2] = 32, + [2][1][RTW89_MEXICO][2] = 50, + [2][1][RTW89_CN][2] = 32, + [2][1][RTW89_QATAR][2] = 32, + [2][1][RTW89_FCC][4] = 58, + [2][1][RTW89_ETSI][4] = 32, + [2][1][RTW89_MKK][4] = 38, + [2][1][RTW89_IC][4] = 30, + [2][1][RTW89_KCC][4] = 54, + [2][1][RTW89_ACMA][4] = 32, + [2][1][RTW89_CHILE][4] = 18, + [2][1][RTW89_UKRAINE][4] = 32, + [2][1][RTW89_MEXICO][4] = 50, + [2][1][RTW89_CN][4] = 32, + [2][1][RTW89_QATAR][4] = 32, + [2][1][RTW89_FCC][6] = 58, + [2][1][RTW89_ETSI][6] = 32, + [2][1][RTW89_MKK][6] = 38, + [2][1][RTW89_IC][6] = 30, + [2][1][RTW89_KCC][6] = 26, + [2][1][RTW89_ACMA][6] = 32, + [2][1][RTW89_CHILE][6] = 18, + [2][1][RTW89_UKRAINE][6] = 32, + [2][1][RTW89_MEXICO][6] = 50, + [2][1][RTW89_CN][6] = 32, + [2][1][RTW89_QATAR][6] = 32, + [2][1][RTW89_FCC][8] = 58, + [2][1][RTW89_ETSI][8] = 32, + [2][1][RTW89_MKK][8] = 38, + [2][1][RTW89_IC][8] = 52, + [2][1][RTW89_KCC][8] = 54, + [2][1][RTW89_ACMA][8] = 32, + [2][1][RTW89_CHILE][8] = 42, + [2][1][RTW89_UKRAINE][8] = 32, + [2][1][RTW89_MEXICO][8] = 58, + [2][1][RTW89_CN][8] = 32, + [2][1][RTW89_QATAR][8] = 32, + [2][1][RTW89_FCC][10] = 58, + [2][1][RTW89_ETSI][10] = 32, + [2][1][RTW89_MKK][10] = 38, + [2][1][RTW89_IC][10] = 52, + [2][1][RTW89_KCC][10] = 54, + [2][1][RTW89_ACMA][10] = 32, + [2][1][RTW89_CHILE][10] = 42, + [2][1][RTW89_UKRAINE][10] = 32, + [2][1][RTW89_MEXICO][10] = 58, + [2][1][RTW89_CN][10] = 32, + [2][1][RTW89_QATAR][10] = 32, + [2][1][RTW89_FCC][12] = 58, + [2][1][RTW89_ETSI][12] = 32, + [2][1][RTW89_MKK][12] = 38, + [2][1][RTW89_IC][12] = 52, + [2][1][RTW89_KCC][12] = 54, + [2][1][RTW89_ACMA][12] = 32, + [2][1][RTW89_CHILE][12] = 42, + [2][1][RTW89_UKRAINE][12] = 32, + [2][1][RTW89_MEXICO][12] = 58, + [2][1][RTW89_CN][12] = 32, + [2][1][RTW89_QATAR][12] = 32, + [2][1][RTW89_FCC][14] = 58, + [2][1][RTW89_ETSI][14] = 32, + [2][1][RTW89_MKK][14] = 38, + [2][1][RTW89_IC][14] = 52, + [2][1][RTW89_KCC][14] = 54, + [2][1][RTW89_ACMA][14] = 32, + [2][1][RTW89_CHILE][14] = 42, + [2][1][RTW89_UKRAINE][14] = 32, + [2][1][RTW89_MEXICO][14] = 58, + [2][1][RTW89_CN][14] = 32, + [2][1][RTW89_QATAR][14] = 32, + [2][1][RTW89_FCC][15] = 58, + [2][1][RTW89_ETSI][15] = 32, + [2][1][RTW89_MKK][15] = 58, + [2][1][RTW89_IC][15] = 58, + [2][1][RTW89_KCC][15] = 54, + [2][1][RTW89_ACMA][15] = 32, + [2][1][RTW89_CHILE][15] = 42, + [2][1][RTW89_UKRAINE][15] = 32, + [2][1][RTW89_MEXICO][15] = 58, + [2][1][RTW89_CN][15] = 127, + [2][1][RTW89_QATAR][15] = 32, + [2][1][RTW89_FCC][17] = 58, + [2][1][RTW89_ETSI][17] = 32, + [2][1][RTW89_MKK][17] = 58, + [2][1][RTW89_IC][17] = 58, + [2][1][RTW89_KCC][17] = 54, + [2][1][RTW89_ACMA][17] = 32, + [2][1][RTW89_CHILE][17] = 42, + [2][1][RTW89_UKRAINE][17] = 32, + [2][1][RTW89_MEXICO][17] = 58, + [2][1][RTW89_CN][17] = 127, + [2][1][RTW89_QATAR][17] = 32, + [2][1][RTW89_FCC][19] = 58, + [2][1][RTW89_ETSI][19] = 32, + [2][1][RTW89_MKK][19] = 58, + [2][1][RTW89_IC][19] = 58, + [2][1][RTW89_KCC][19] = 54, + [2][1][RTW89_ACMA][19] = 32, + [2][1][RTW89_CHILE][19] = 42, + [2][1][RTW89_UKRAINE][19] = 32, + [2][1][RTW89_MEXICO][19] = 58, + [2][1][RTW89_CN][19] = 127, + [2][1][RTW89_QATAR][19] = 32, + [2][1][RTW89_FCC][21] = 58, + [2][1][RTW89_ETSI][21] = 32, + [2][1][RTW89_MKK][21] = 58, + [2][1][RTW89_IC][21] = 58, + [2][1][RTW89_KCC][21] = 54, + [2][1][RTW89_ACMA][21] = 32, + [2][1][RTW89_CHILE][21] = 42, + [2][1][RTW89_UKRAINE][21] = 32, + [2][1][RTW89_MEXICO][21] = 58, + [2][1][RTW89_CN][21] = 127, + [2][1][RTW89_QATAR][21] = 32, + [2][1][RTW89_FCC][23] = 58, + [2][1][RTW89_ETSI][23] = 32, + [2][1][RTW89_MKK][23] = 58, + [2][1][RTW89_IC][23] = 58, + [2][1][RTW89_KCC][23] = 54, + [2][1][RTW89_ACMA][23] = 32, + [2][1][RTW89_CHILE][23] = 42, + [2][1][RTW89_UKRAINE][23] = 32, + [2][1][RTW89_MEXICO][23] = 58, + [2][1][RTW89_CN][23] = 127, + [2][1][RTW89_QATAR][23] = 32, + [2][1][RTW89_FCC][25] = 58, + [2][1][RTW89_ETSI][25] = 32, + [2][1][RTW89_MKK][25] = 58, + [2][1][RTW89_IC][25] = 127, + [2][1][RTW89_KCC][25] = 54, + [2][1][RTW89_ACMA][25] = 127, + [2][1][RTW89_CHILE][25] = 42, + [2][1][RTW89_UKRAINE][25] = 32, + [2][1][RTW89_MEXICO][25] = 58, + [2][1][RTW89_CN][25] = 127, + [2][1][RTW89_QATAR][25] = 32, + [2][1][RTW89_FCC][27] = 58, + [2][1][RTW89_ETSI][27] = 32, + [2][1][RTW89_MKK][27] = 58, + [2][1][RTW89_IC][27] = 127, + [2][1][RTW89_KCC][27] = 54, + [2][1][RTW89_ACMA][27] = 127, + [2][1][RTW89_CHILE][27] = 42, + [2][1][RTW89_UKRAINE][27] = 32, + [2][1][RTW89_MEXICO][27] = 58, + [2][1][RTW89_CN][27] = 127, + [2][1][RTW89_QATAR][27] = 32, + [2][1][RTW89_FCC][29] = 58, + [2][1][RTW89_ETSI][29] = 32, + [2][1][RTW89_MKK][29] = 58, + [2][1][RTW89_IC][29] = 127, + [2][1][RTW89_KCC][29] = 54, + [2][1][RTW89_ACMA][29] = 127, + [2][1][RTW89_CHILE][29] = 42, + [2][1][RTW89_UKRAINE][29] = 32, + [2][1][RTW89_MEXICO][29] = 58, + [2][1][RTW89_CN][29] = 127, + [2][1][RTW89_QATAR][29] = 32, + [2][1][RTW89_FCC][31] = 58, + [2][1][RTW89_ETSI][31] = 32, + [2][1][RTW89_MKK][31] = 58, + [2][1][RTW89_IC][31] = 58, + [2][1][RTW89_KCC][31] = 54, + [2][1][RTW89_ACMA][31] = 32, + [2][1][RTW89_CHILE][31] = 42, + [2][1][RTW89_UKRAINE][31] = 32, + [2][1][RTW89_MEXICO][31] = 58, + [2][1][RTW89_CN][31] = 127, + [2][1][RTW89_QATAR][31] = 32, + [2][1][RTW89_FCC][33] = 58, + [2][1][RTW89_ETSI][33] = 32, + [2][1][RTW89_MKK][33] = 58, + [2][1][RTW89_IC][33] = 58, + [2][1][RTW89_KCC][33] = 54, + [2][1][RTW89_ACMA][33] = 32, + [2][1][RTW89_CHILE][33] = 42, + [2][1][RTW89_UKRAINE][33] = 32, + [2][1][RTW89_MEXICO][33] = 58, + [2][1][RTW89_CN][33] = 127, + [2][1][RTW89_QATAR][33] = 32, + [2][1][RTW89_FCC][35] = 58, + [2][1][RTW89_ETSI][35] = 32, + [2][1][RTW89_MKK][35] = 58, + [2][1][RTW89_IC][35] = 58, + [2][1][RTW89_KCC][35] = 54, + [2][1][RTW89_ACMA][35] = 32, + [2][1][RTW89_CHILE][35] = 42, + [2][1][RTW89_UKRAINE][35] = 32, + [2][1][RTW89_MEXICO][35] = 58, + [2][1][RTW89_CN][35] = 127, + [2][1][RTW89_QATAR][35] = 32, + [2][1][RTW89_FCC][37] = 58, + [2][1][RTW89_ETSI][37] = 127, + [2][1][RTW89_MKK][37] = 58, + [2][1][RTW89_IC][37] = 58, + [2][1][RTW89_KCC][37] = 54, + [2][1][RTW89_ACMA][37] = 62, + [2][1][RTW89_CHILE][37] = 42, + [2][1][RTW89_UKRAINE][37] = 127, + [2][1][RTW89_MEXICO][37] = 58, + [2][1][RTW89_CN][37] = 127, + [2][1][RTW89_QATAR][37] = 127, + [2][1][RTW89_FCC][38] = 76, + [2][1][RTW89_ETSI][38] = 16, + [2][1][RTW89_MKK][38] = 127, + [2][1][RTW89_IC][38] = 76, + [2][1][RTW89_KCC][38] = 54, + [2][1][RTW89_ACMA][38] = 76, + [2][1][RTW89_CHILE][38] = 42, + [2][1][RTW89_UKRAINE][38] = 16, + [2][1][RTW89_MEXICO][38] = 76, + [2][1][RTW89_CN][38] = 64, + [2][1][RTW89_QATAR][38] = 16, + [2][1][RTW89_FCC][40] = 76, + [2][1][RTW89_ETSI][40] = 16, + [2][1][RTW89_MKK][40] = 127, + [2][1][RTW89_IC][40] = 76, + [2][1][RTW89_KCC][40] = 54, + [2][1][RTW89_ACMA][40] = 76, + [2][1][RTW89_CHILE][40] = 42, + [2][1][RTW89_UKRAINE][40] = 16, + [2][1][RTW89_MEXICO][40] = 76, + [2][1][RTW89_CN][40] = 64, + [2][1][RTW89_QATAR][40] = 16, + [2][1][RTW89_FCC][42] = 76, + [2][1][RTW89_ETSI][42] = 16, + [2][1][RTW89_MKK][42] = 127, + [2][1][RTW89_IC][42] = 76, + [2][1][RTW89_KCC][42] = 54, + [2][1][RTW89_ACMA][42] = 76, + [2][1][RTW89_CHILE][42] = 42, + [2][1][RTW89_UKRAINE][42] = 16, + [2][1][RTW89_MEXICO][42] = 76, + [2][1][RTW89_CN][42] = 64, + [2][1][RTW89_QATAR][42] = 16, + [2][1][RTW89_FCC][44] = 76, + [2][1][RTW89_ETSI][44] = 16, + [2][1][RTW89_MKK][44] = 127, + [2][1][RTW89_IC][44] = 76, + [2][1][RTW89_KCC][44] = 54, + [2][1][RTW89_ACMA][44] = 76, + [2][1][RTW89_CHILE][44] = 42, + [2][1][RTW89_UKRAINE][44] = 16, + [2][1][RTW89_MEXICO][44] = 76, + [2][1][RTW89_CN][44] = 64, + [2][1][RTW89_QATAR][44] = 16, + [2][1][RTW89_FCC][46] = 76, + [2][1][RTW89_ETSI][46] = 16, + [2][1][RTW89_MKK][46] = 127, + [2][1][RTW89_IC][46] = 76, + [2][1][RTW89_KCC][46] = 54, + [2][1][RTW89_ACMA][46] = 76, + [2][1][RTW89_CHILE][46] = 42, + [2][1][RTW89_UKRAINE][46] = 16, + [2][1][RTW89_MEXICO][46] = 76, + [2][1][RTW89_CN][46] = 64, + [2][1][RTW89_QATAR][46] = 16, }; #define DECLARE_DIG_TABLE(name) \ diff --git a/drivers/net/wireless/rsi/rsi_91x_main.c b/drivers/net/wireless/rsi/rsi_91x_main.c index f1bf71e6c608..5d1490fc32db 100644 --- a/drivers/net/wireless/rsi/rsi_91x_main.c +++ b/drivers/net/wireless/rsi/rsi_91x_main.c @@ -23,6 +23,7 @@ #include "rsi_common.h" #include "rsi_coex.h" #include "rsi_hal.h" +#include "rsi_usb.h" u32 rsi_zone_enabled = /* INFO_ZONE | INIT_ZONE | @@ -168,6 +169,9 @@ int rsi_read_pkt(struct rsi_common *common, u8 *rx_pkt, s32 rcv_pkt_len) frame_desc = &rx_pkt[index]; actual_length = *(u16 *)&frame_desc[0]; offset = *(u16 *)&frame_desc[2]; + if (!rcv_pkt_len && offset > + RSI_MAX_RX_USB_PKT_SIZE - FRAME_DESC_SZ) + goto fail; queueno = rsi_get_queueno(frame_desc, offset); length = rsi_get_length(frame_desc, offset); diff --git a/drivers/net/wireless/rsi/rsi_91x_usb.c b/drivers/net/wireless/rsi/rsi_91x_usb.c index 6821ea991895..66fe386ec9cc 100644 --- a/drivers/net/wireless/rsi/rsi_91x_usb.c +++ b/drivers/net/wireless/rsi/rsi_91x_usb.c @@ -269,8 +269,12 @@ static void rsi_rx_done_handler(struct urb *urb) struct rsi_91x_usbdev *dev = (struct rsi_91x_usbdev *)rx_cb->data; int status = -EINVAL; + if (!rx_cb->rx_skb) + return; + if (urb->status) { dev_kfree_skb(rx_cb->rx_skb); + rx_cb->rx_skb = NULL; return; } @@ -294,8 +298,10 @@ out: if (rsi_rx_urb_submit(dev->priv, rx_cb->ep_num, GFP_ATOMIC)) rsi_dbg(ERR_ZONE, "%s: Failed in urb submission", __func__); - if (status) + if (status) { dev_kfree_skb(rx_cb->rx_skb); + rx_cb->rx_skb = NULL; + } } static void rsi_rx_urb_kill(struct rsi_hw *adapter, u8 ep_num) @@ -324,7 +330,6 @@ static int rsi_rx_urb_submit(struct rsi_hw *adapter, u8 ep_num, gfp_t mem_flags) struct sk_buff *skb; u8 dword_align_bytes = 0; -#define RSI_MAX_RX_USB_PKT_SIZE 3000 skb = dev_alloc_skb(RSI_MAX_RX_USB_PKT_SIZE); if (!skb) return -ENOMEM; diff --git a/drivers/net/wireless/rsi/rsi_usb.h b/drivers/net/wireless/rsi/rsi_usb.h index 254d19b66412..961851748bc4 100644 --- a/drivers/net/wireless/rsi/rsi_usb.h +++ b/drivers/net/wireless/rsi/rsi_usb.h @@ -44,6 +44,8 @@ #define RSI_USB_BUF_SIZE 4096 #define RSI_USB_CTRL_BUF_SIZE 0x04 +#define RSI_MAX_RX_USB_PKT_SIZE 3000 + struct rx_usb_ctrl_block { u8 *data; struct urb *rx_urb; diff --git a/drivers/net/wireless/ti/wlcore/sdio.c b/drivers/net/wireless/ti/wlcore/sdio.c index 9fd8cf2d270c..72fc41ac83c0 100644 --- a/drivers/net/wireless/ti/wlcore/sdio.c +++ b/drivers/net/wireless/ti/wlcore/sdio.c @@ -26,7 +26,7 @@ #include "wl12xx_80211.h" #include "io.h" -static bool dump = false; +static bool dump; struct wl12xx_sdio_glue { struct device *dev; diff --git a/drivers/net/wwan/Kconfig b/drivers/net/wwan/Kconfig index 17543be14665..609fd4a2c865 100644 --- a/drivers/net/wwan/Kconfig +++ b/drivers/net/wwan/Kconfig @@ -16,6 +16,17 @@ config WWAN if WWAN +config WWAN_DEBUGFS + bool "WWAN devices debugfs interface" if EXPERT + depends on DEBUG_FS + default y + help + Enables debugfs infrastructure for the WWAN core and device drivers. + + If this option is selected, then you can find the debug interface + elements for each WWAN device in a directory that is corresponding to + the device name: debugfs/wwan/wwanX. + config WWAN_HWSIM tristate "Simulated WWAN device" help @@ -50,6 +61,19 @@ config MHI_WWAN_MBIM To compile this driver as a module, choose M here: the module will be called mhi_wwan_mbim. +config QCOM_BAM_DMUX + tristate "Qualcomm BAM-DMUX WWAN network driver" + depends on (DMA_ENGINE && PM && QCOM_SMEM_STATE) || COMPILE_TEST + help + The BAM Data Multiplexer provides access to the network data channels + of modems integrated into many older Qualcomm SoCs, e.g. Qualcomm + MSM8916 or MSM8974. The connection can be established via QMI/AT from + userspace with control ports available through the WWAN subsystem + (CONFIG_RPMSG_WWAN_CTRL) or QRTR network sockets (CONFIG_QRTR). + + To compile this driver as a module, choose M here: the module will be + called qcom_bam_dmux. + config RPMSG_WWAN_CTRL tristate "RPMSG WWAN control driver" depends on RPMSG @@ -72,6 +96,7 @@ config IOSM tristate "IOSM Driver for Intel M.2 WWAN Device" depends on INTEL_IOMMU select NET_DEVLINK + select RELAY if WWAN_DEBUGFS help This driver enables Intel M.2 WWAN Device communication. diff --git a/drivers/net/wwan/Makefile b/drivers/net/wwan/Makefile index fe51feedac21..e722650bebea 100644 --- a/drivers/net/wwan/Makefile +++ b/drivers/net/wwan/Makefile @@ -10,5 +10,6 @@ obj-$(CONFIG_WWAN_HWSIM) += wwan_hwsim.o obj-$(CONFIG_MHI_WWAN_CTRL) += mhi_wwan_ctrl.o obj-$(CONFIG_MHI_WWAN_MBIM) += mhi_wwan_mbim.o +obj-$(CONFIG_QCOM_BAM_DMUX) += qcom_bam_dmux.o obj-$(CONFIG_RPMSG_WWAN_CTRL) += rpmsg_wwan_ctrl.o obj-$(CONFIG_IOSM) += iosm/ diff --git a/drivers/net/wwan/iosm/Makefile b/drivers/net/wwan/iosm/Makefile index b838034bb120..fa8d6afd18e1 100644 --- a/drivers/net/wwan/iosm/Makefile +++ b/drivers/net/wwan/iosm/Makefile @@ -23,4 +23,8 @@ iosm-y = \ iosm_ipc_flash.o \ iosm_ipc_coredump.o +iosm-$(CONFIG_WWAN_DEBUGFS) += \ + iosm_ipc_debugfs.o \ + iosm_ipc_trace.o + obj-$(CONFIG_IOSM) := iosm.o diff --git a/drivers/net/wwan/iosm/iosm_ipc_debugfs.c b/drivers/net/wwan/iosm/iosm_ipc_debugfs.c new file mode 100644 index 000000000000..f2f57751a7d2 --- /dev/null +++ b/drivers/net/wwan/iosm/iosm_ipc_debugfs.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2021 Intel Corporation. + */ + +#include <linux/debugfs.h> +#include <linux/wwan.h> + +#include "iosm_ipc_imem.h" +#include "iosm_ipc_trace.h" +#include "iosm_ipc_debugfs.h" + +void ipc_debugfs_init(struct iosm_imem *ipc_imem) +{ + struct dentry *debugfs_pdev = wwan_get_debugfs_dir(ipc_imem->dev); + + ipc_imem->debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, + debugfs_pdev); + + ipc_imem->trace = ipc_trace_init(ipc_imem); + if (!ipc_imem->trace) + dev_warn(ipc_imem->dev, "trace channel init failed"); +} + +void ipc_debugfs_deinit(struct iosm_imem *ipc_imem) +{ + ipc_trace_deinit(ipc_imem->trace); + debugfs_remove_recursive(ipc_imem->debugfs_dir); +} diff --git a/drivers/net/wwan/iosm/iosm_ipc_debugfs.h b/drivers/net/wwan/iosm/iosm_ipc_debugfs.h new file mode 100644 index 000000000000..8a84bfa2c14a --- /dev/null +++ b/drivers/net/wwan/iosm/iosm_ipc_debugfs.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * + * Copyright (C) 2020-2021 Intel Corporation. + */ + +#ifndef IOSM_IPC_DEBUGFS_H +#define IOSM_IPC_DEBUGFS_H + +#ifdef CONFIG_WWAN_DEBUGFS +void ipc_debugfs_init(struct iosm_imem *ipc_imem); +void ipc_debugfs_deinit(struct iosm_imem *ipc_imem); +#else +static inline void ipc_debugfs_init(struct iosm_imem *ipc_imem) {} +static inline void ipc_debugfs_deinit(struct iosm_imem *ipc_imem) {} +#endif + +#endif diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem.c b/drivers/net/wwan/iosm/iosm_ipc_imem.c index 12c03dacb5dd..fb4e58beb366 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem.c +++ b/drivers/net/wwan/iosm/iosm_ipc_imem.c @@ -10,6 +10,8 @@ #include "iosm_ipc_flash.h" #include "iosm_ipc_imem.h" #include "iosm_ipc_port.h" +#include "iosm_ipc_trace.h" +#include "iosm_ipc_debugfs.h" /* Check the wwan ips if it is valid with Channel as input. */ static int ipc_imem_check_wwan_ips(struct ipc_mem_channel *chnl) @@ -269,9 +271,14 @@ static void ipc_imem_dl_skb_process(struct iosm_imem *ipc_imem, switch (pipe->channel->ctype) { case IPC_CTYPE_CTRL: port_id = pipe->channel->channel_id; + ipc_pcie_addr_unmap(ipc_imem->pcie, IPC_CB(skb)->len, + IPC_CB(skb)->mapping, + IPC_CB(skb)->direction); if (port_id == IPC_MEM_CTRL_CHL_ID_7) ipc_imem_sys_devlink_notify_rx(ipc_imem->ipc_devlink, skb); + else if (ipc_is_trace_channel(ipc_imem, port_id)) + ipc_trace_port_rx(ipc_imem, skb); else wwan_port_rx(ipc_imem->ipc_port[port_id]->iosm_port, skb); @@ -555,6 +562,8 @@ static void ipc_imem_run_state_worker(struct work_struct *instance) ctrl_chl_idx++; } + ipc_debugfs_init(ipc_imem); + ipc_task_queue_send_task(ipc_imem, ipc_imem_send_mdm_rdy_cb, 0, NULL, 0, false); @@ -1170,6 +1179,7 @@ void ipc_imem_cleanup(struct iosm_imem *ipc_imem) if (test_and_clear_bit(FULLY_FUNCTIONAL, &ipc_imem->flag)) { ipc_mux_deinit(ipc_imem->mux); + ipc_debugfs_deinit(ipc_imem); ipc_wwan_deinit(ipc_imem->wwan); ipc_port_deinit(ipc_imem->ipc_port); } diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem.h b/drivers/net/wwan/iosm/iosm_ipc_imem.h index 6b8a837faef2..98554e9beb01 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem.h +++ b/drivers/net/wwan/iosm/iosm_ipc_imem.h @@ -305,6 +305,7 @@ enum ipc_phase { * @sio: IPC SIO data structure pointer * @ipc_port: IPC PORT data structure pointer * @pcie: IPC PCIe + * @trace: IPC trace data structure pointer * @dev: Pointer to device structure * @ipc_requested_state: Expected IPC state on CP. * @channels: Channel list with UL/DL pipe pairs. @@ -339,6 +340,7 @@ enum ipc_phase { * @ev_mux_net_transmit_pending:0 means inform the IPC tasklet to pass * @reset_det_n: Reset detect flag * @pcie_wake_n: Pcie wake flag + * @debugfs_dir: Debug FS directory for driver-specific entries */ struct iosm_imem { struct iosm_mmio *mmio; @@ -348,6 +350,9 @@ struct iosm_imem { struct iosm_mux *mux; struct iosm_cdev *ipc_port[IPC_MEM_MAX_CHANNELS]; struct iosm_pcie *pcie; +#ifdef CONFIG_WWAN_DEBUGFS + struct iosm_trace *trace; +#endif struct device *dev; enum ipc_mem_device_ipc_state ipc_requested_state; struct ipc_mem_channel channels[IPC_MEM_MAX_CHANNELS]; @@ -376,6 +381,9 @@ struct iosm_imem { ev_mux_net_transmit_pending:1, reset_det_n:1, pcie_wake_n:1; +#ifdef CONFIG_WWAN_DEBUGFS + struct dentry *debugfs_dir; +#endif }; /** diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c index 831cdae28e8a..57304a5adf68 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c +++ b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.c @@ -176,11 +176,14 @@ channel_unavailable: return false; } -/* Release a sio link to CP. */ -void ipc_imem_sys_cdev_close(struct iosm_cdev *ipc_cdev) +/** + * ipc_imem_sys_port_close - Release a sio link to CP. + * @ipc_imem: Imem instance. + * @channel: Channel instance. + */ +void ipc_imem_sys_port_close(struct iosm_imem *ipc_imem, + struct ipc_mem_channel *channel) { - struct iosm_imem *ipc_imem = ipc_cdev->ipc_imem; - struct ipc_mem_channel *channel = ipc_cdev->channel; enum ipc_phase curr_phase; int status = 0; u32 tail = 0; @@ -638,6 +641,6 @@ int ipc_imem_sys_devlink_read(struct iosm_devlink *devlink, u8 *data, memcpy(data, skb->data, skb->len); devlink_read_fail: - ipc_pcie_kfree_skb(devlink->pcie, skb); + dev_kfree_skb(skb); return rc; } diff --git a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.h b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.h index f0c88ac5643c..f8afb217d9e2 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_imem_ops.h +++ b/drivers/net/wwan/iosm/iosm_ipc_imem_ops.h @@ -43,12 +43,8 @@ */ struct ipc_mem_channel *ipc_imem_sys_port_open(struct iosm_imem *ipc_imem, int chl_id, int hp_id); - -/** - * ipc_imem_sys_cdev_close - Release a sio link to CP. - * @ipc_cdev: iosm sio instance. - */ -void ipc_imem_sys_cdev_close(struct iosm_cdev *ipc_cdev); +void ipc_imem_sys_port_close(struct iosm_imem *ipc_imem, + struct ipc_mem_channel *channel); /** * ipc_imem_sys_cdev_write - Route the uplink buffer to CP. @@ -145,4 +141,5 @@ int ipc_imem_sys_devlink_read(struct iosm_devlink *ipc_devlink, u8 *data, */ int ipc_imem_sys_devlink_write(struct iosm_devlink *ipc_devlink, unsigned char *buf, int count); + #endif diff --git a/drivers/net/wwan/iosm/iosm_ipc_port.c b/drivers/net/wwan/iosm/iosm_ipc_port.c index beb944847398..b6d81c627277 100644 --- a/drivers/net/wwan/iosm/iosm_ipc_port.c +++ b/drivers/net/wwan/iosm/iosm_ipc_port.c @@ -27,7 +27,7 @@ static void ipc_port_ctrl_stop(struct wwan_port *port) { struct iosm_cdev *ipc_port = wwan_port_get_drvdata(port); - ipc_imem_sys_cdev_close(ipc_port); + ipc_imem_sys_port_close(ipc_port->ipc_imem, ipc_port->channel); } /* transfer control data to modem */ diff --git a/drivers/net/wwan/iosm/iosm_ipc_trace.c b/drivers/net/wwan/iosm/iosm_ipc_trace.c new file mode 100644 index 000000000000..eeecfa3d10c5 --- /dev/null +++ b/drivers/net/wwan/iosm/iosm_ipc_trace.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2020-2021 Intel Corporation. + */ + +#include <linux/wwan.h> +#include "iosm_ipc_trace.h" + +/* sub buffer size and number of sub buffer */ +#define IOSM_TRC_SUB_BUFF_SIZE 131072 +#define IOSM_TRC_N_SUB_BUFF 32 + +#define IOSM_TRC_FILE_PERM 0600 + +#define IOSM_TRC_DEBUGFS_TRACE "trace" +#define IOSM_TRC_DEBUGFS_TRACE_CTRL "trace_ctrl" + +/** + * ipc_trace_port_rx - Receive trace packet from cp and write to relay buffer + * @ipc_imem: Pointer to iosm_imem structure + * @skb: Pointer to struct sk_buff + */ +void ipc_trace_port_rx(struct iosm_imem *ipc_imem, struct sk_buff *skb) +{ + struct iosm_trace *ipc_trace = ipc_imem->trace; + + if (ipc_trace->ipc_rchan) + relay_write(ipc_trace->ipc_rchan, skb->data, skb->len); + + dev_kfree_skb(skb); +} + +/* Creates relay file in debugfs. */ +static struct dentry * +ipc_trace_create_buf_file_handler(const char *filename, + struct dentry *parent, + umode_t mode, + struct rchan_buf *buf, + int *is_global) +{ + *is_global = 1; + return debugfs_create_file(filename, mode, parent, buf, + &relay_file_operations); +} + +/* Removes relay file from debugfs. */ +static int ipc_trace_remove_buf_file_handler(struct dentry *dentry) +{ + debugfs_remove(dentry); + return 0; +} + +static int ipc_trace_subbuf_start_handler(struct rchan_buf *buf, void *subbuf, + void *prev_subbuf, + size_t prev_padding) +{ + if (relay_buf_full(buf)) { + pr_err_ratelimited("Relay_buf full dropping traces"); + return 0; + } + + return 1; +} + +/* Relay interface callbacks */ +static struct rchan_callbacks relay_callbacks = { + .subbuf_start = ipc_trace_subbuf_start_handler, + .create_buf_file = ipc_trace_create_buf_file_handler, + .remove_buf_file = ipc_trace_remove_buf_file_handler, +}; + +/* Copy the trace control mode to user buffer */ +static ssize_t ipc_trace_ctrl_file_read(struct file *filp, char __user *buffer, + size_t count, loff_t *ppos) +{ + struct iosm_trace *ipc_trace = filp->private_data; + char buf[16]; + int len; + + mutex_lock(&ipc_trace->trc_mutex); + len = snprintf(buf, sizeof(buf), "%d\n", ipc_trace->mode); + mutex_unlock(&ipc_trace->trc_mutex); + + return simple_read_from_buffer(buffer, count, ppos, buf, len); +} + +/* Open and close the trace channel depending on user input */ +static ssize_t ipc_trace_ctrl_file_write(struct file *filp, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct iosm_trace *ipc_trace = filp->private_data; + unsigned long val; + int ret; + + ret = kstrtoul_from_user(buffer, count, 10, &val); + if (ret) + return ret; + + mutex_lock(&ipc_trace->trc_mutex); + if (val == TRACE_ENABLE && ipc_trace->mode != TRACE_ENABLE) { + ipc_trace->channel = ipc_imem_sys_port_open(ipc_trace->ipc_imem, + ipc_trace->chl_id, + IPC_HP_CDEV_OPEN); + if (!ipc_trace->channel) { + ret = -EIO; + goto unlock; + } + ipc_trace->mode = TRACE_ENABLE; + } else if (val == TRACE_DISABLE && ipc_trace->mode != TRACE_DISABLE) { + ipc_trace->mode = TRACE_DISABLE; + /* close trace channel */ + ipc_imem_sys_port_close(ipc_trace->ipc_imem, + ipc_trace->channel); + relay_flush(ipc_trace->ipc_rchan); + } + ret = count; +unlock: + mutex_unlock(&ipc_trace->trc_mutex); + return ret; +} + +static const struct file_operations ipc_trace_fops = { + .open = simple_open, + .write = ipc_trace_ctrl_file_write, + .read = ipc_trace_ctrl_file_read, +}; + +/** + * ipc_trace_init - Create trace interface & debugfs entries + * @ipc_imem: Pointer to iosm_imem structure + * + * Returns: Pointer to trace instance on success else NULL + */ +struct iosm_trace *ipc_trace_init(struct iosm_imem *ipc_imem) +{ + struct ipc_chnl_cfg chnl_cfg = { 0 }; + struct iosm_trace *ipc_trace; + + ipc_chnl_cfg_get(&chnl_cfg, IPC_MEM_CTRL_CHL_ID_3); + ipc_imem_channel_init(ipc_imem, IPC_CTYPE_CTRL, chnl_cfg, + IRQ_MOD_OFF); + + ipc_trace = kzalloc(sizeof(*ipc_trace), GFP_KERNEL); + if (!ipc_trace) + return NULL; + + ipc_trace->mode = TRACE_DISABLE; + ipc_trace->dev = ipc_imem->dev; + ipc_trace->ipc_imem = ipc_imem; + ipc_trace->chl_id = IPC_MEM_CTRL_CHL_ID_3; + + mutex_init(&ipc_trace->trc_mutex); + + ipc_trace->ctrl_file = debugfs_create_file(IOSM_TRC_DEBUGFS_TRACE_CTRL, + IOSM_TRC_FILE_PERM, + ipc_imem->debugfs_dir, + ipc_trace, &ipc_trace_fops); + + ipc_trace->ipc_rchan = relay_open(IOSM_TRC_DEBUGFS_TRACE, + ipc_imem->debugfs_dir, + IOSM_TRC_SUB_BUFF_SIZE, + IOSM_TRC_N_SUB_BUFF, + &relay_callbacks, NULL); + + return ipc_trace; +} + +/** + * ipc_trace_deinit - Closing relayfs, removing debugfs entries + * @ipc_trace: Pointer to the iosm_trace data struct + */ +void ipc_trace_deinit(struct iosm_trace *ipc_trace) +{ + if (!ipc_trace) + return; + + debugfs_remove(ipc_trace->ctrl_file); + relay_close(ipc_trace->ipc_rchan); + mutex_destroy(&ipc_trace->trc_mutex); + kfree(ipc_trace); +} diff --git a/drivers/net/wwan/iosm/iosm_ipc_trace.h b/drivers/net/wwan/iosm/iosm_ipc_trace.h new file mode 100644 index 000000000000..5ebe7790585c --- /dev/null +++ b/drivers/net/wwan/iosm/iosm_ipc_trace.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0-only + * + * Copyright (C) 2020-2021 Intel Corporation. + */ + +#ifndef IOSM_IPC_TRACE_H +#define IOSM_IPC_TRACE_H + +#include <linux/debugfs.h> +#include <linux/relay.h> + +#include "iosm_ipc_chnl_cfg.h" +#include "iosm_ipc_imem_ops.h" + +/** + * enum trace_ctrl_mode - State of trace channel + * @TRACE_DISABLE: mode for disable trace + * @TRACE_ENABLE: mode for enable trace + */ +enum trace_ctrl_mode { + TRACE_DISABLE = 0, + TRACE_ENABLE, +}; + +/** + * struct iosm_trace - Struct for trace interface + * @ipc_rchan: Pointer to relay channel + * @ctrl_file: Pointer to trace control file + * @ipc_imem: Imem instance + * @dev: Pointer to device struct + * @channel: Channel instance + * @chl_id: Channel Indentifier + * @trc_mutex: Mutex used for read and write mode + * @mode: Mode for enable and disable trace + */ + +struct iosm_trace { + struct rchan *ipc_rchan; + struct dentry *ctrl_file; + struct iosm_imem *ipc_imem; + struct device *dev; + struct ipc_mem_channel *channel; + enum ipc_channel_id chl_id; + struct mutex trc_mutex; /* Mutex used for read and write mode */ + enum trace_ctrl_mode mode; +}; + +#ifdef CONFIG_WWAN_DEBUGFS + +static inline bool ipc_is_trace_channel(struct iosm_imem *ipc_mem, u16 chl_id) +{ + return ipc_mem->trace && ipc_mem->trace->chl_id == chl_id; +} + +struct iosm_trace *ipc_trace_init(struct iosm_imem *ipc_imem); +void ipc_trace_deinit(struct iosm_trace *ipc_trace); +void ipc_trace_port_rx(struct iosm_imem *ipc_imem, struct sk_buff *skb); + +#else + +static inline bool ipc_is_trace_channel(struct iosm_imem *ipc_mem, u16 chl_id) +{ + return false; +} + +static inline void ipc_trace_port_rx(struct iosm_imem *ipc_imem, + struct sk_buff *skb) +{ + dev_kfree_skb(skb); +} + +#endif + +#endif diff --git a/drivers/net/wwan/qcom_bam_dmux.c b/drivers/net/wwan/qcom_bam_dmux.c new file mode 100644 index 000000000000..5dfa2eba6014 --- /dev/null +++ b/drivers/net/wwan/qcom_bam_dmux.c @@ -0,0 +1,907 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Qualcomm BAM-DMUX WWAN network driver + * Copyright (c) 2020, Stephan Gerhold <stephan@gerhold.net> + */ + +#include <linux/atomic.h> +#include <linux/bitops.h> +#include <linux/completion.h> +#include <linux/dma-mapping.h> +#include <linux/dmaengine.h> +#include <linux/if_arp.h> +#include <linux/interrupt.h> +#include <linux/mod_devicetable.h> +#include <linux/module.h> +#include <linux/netdevice.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/soc/qcom/smem_state.h> +#include <linux/spinlock.h> +#include <linux/wait.h> +#include <linux/workqueue.h> +#include <net/pkt_sched.h> + +#define BAM_DMUX_BUFFER_SIZE SZ_2K +#define BAM_DMUX_HDR_SIZE sizeof(struct bam_dmux_hdr) +#define BAM_DMUX_MAX_DATA_SIZE (BAM_DMUX_BUFFER_SIZE - BAM_DMUX_HDR_SIZE) +#define BAM_DMUX_NUM_SKB 32 + +#define BAM_DMUX_HDR_MAGIC 0x33fc + +#define BAM_DMUX_AUTOSUSPEND_DELAY 1000 +#define BAM_DMUX_REMOTE_TIMEOUT msecs_to_jiffies(2000) + +enum { + BAM_DMUX_CMD_DATA, + BAM_DMUX_CMD_OPEN, + BAM_DMUX_CMD_CLOSE, +}; + +enum { + BAM_DMUX_CH_DATA_0, + BAM_DMUX_CH_DATA_1, + BAM_DMUX_CH_DATA_2, + BAM_DMUX_CH_DATA_3, + BAM_DMUX_CH_DATA_4, + BAM_DMUX_CH_DATA_5, + BAM_DMUX_CH_DATA_6, + BAM_DMUX_CH_DATA_7, + BAM_DMUX_NUM_CH +}; + +struct bam_dmux_hdr { + u16 magic; + u8 signal; + u8 cmd; + u8 pad; + u8 ch; + u16 len; +}; + +struct bam_dmux_skb_dma { + struct bam_dmux *dmux; + struct sk_buff *skb; + dma_addr_t addr; +}; + +struct bam_dmux { + struct device *dev; + + int pc_irq; + bool pc_state, pc_ack_state; + struct qcom_smem_state *pc, *pc_ack; + u32 pc_mask, pc_ack_mask; + wait_queue_head_t pc_wait; + struct completion pc_ack_completion; + + struct dma_chan *rx, *tx; + struct bam_dmux_skb_dma rx_skbs[BAM_DMUX_NUM_SKB]; + struct bam_dmux_skb_dma tx_skbs[BAM_DMUX_NUM_SKB]; + spinlock_t tx_lock; /* Protect tx_skbs, tx_next_skb */ + unsigned int tx_next_skb; + atomic_long_t tx_deferred_skb; + struct work_struct tx_wakeup_work; + + DECLARE_BITMAP(remote_channels, BAM_DMUX_NUM_CH); + struct work_struct register_netdev_work; + struct net_device *netdevs[BAM_DMUX_NUM_CH]; +}; + +struct bam_dmux_netdev { + struct bam_dmux *dmux; + u8 ch; +}; + +static void bam_dmux_pc_vote(struct bam_dmux *dmux, bool enable) +{ + reinit_completion(&dmux->pc_ack_completion); + qcom_smem_state_update_bits(dmux->pc, dmux->pc_mask, + enable ? dmux->pc_mask : 0); +} + +static void bam_dmux_pc_ack(struct bam_dmux *dmux) +{ + qcom_smem_state_update_bits(dmux->pc_ack, dmux->pc_ack_mask, + dmux->pc_ack_state ? 0 : dmux->pc_ack_mask); + dmux->pc_ack_state = !dmux->pc_ack_state; +} + +static bool bam_dmux_skb_dma_map(struct bam_dmux_skb_dma *skb_dma, + enum dma_data_direction dir) +{ + struct device *dev = skb_dma->dmux->dev; + + skb_dma->addr = dma_map_single(dev, skb_dma->skb->data, skb_dma->skb->len, dir); + if (dma_mapping_error(dev, skb_dma->addr)) { + dev_err(dev, "Failed to DMA map buffer\n"); + skb_dma->addr = 0; + return false; + } + + return true; +} + +static void bam_dmux_skb_dma_unmap(struct bam_dmux_skb_dma *skb_dma, + enum dma_data_direction dir) +{ + dma_unmap_single(skb_dma->dmux->dev, skb_dma->addr, skb_dma->skb->len, dir); + skb_dma->addr = 0; +} + +static void bam_dmux_tx_wake_queues(struct bam_dmux *dmux) +{ + int i; + + dev_dbg(dmux->dev, "wake queues\n"); + + for (i = 0; i < BAM_DMUX_NUM_CH; ++i) { + struct net_device *netdev = dmux->netdevs[i]; + + if (netdev && netif_running(netdev)) + netif_wake_queue(netdev); + } +} + +static void bam_dmux_tx_stop_queues(struct bam_dmux *dmux) +{ + int i; + + dev_dbg(dmux->dev, "stop queues\n"); + + for (i = 0; i < BAM_DMUX_NUM_CH; ++i) { + struct net_device *netdev = dmux->netdevs[i]; + + if (netdev) + netif_stop_queue(netdev); + } +} + +static void bam_dmux_tx_done(struct bam_dmux_skb_dma *skb_dma) +{ + struct bam_dmux *dmux = skb_dma->dmux; + unsigned long flags; + + pm_runtime_mark_last_busy(dmux->dev); + pm_runtime_put_autosuspend(dmux->dev); + + if (skb_dma->addr) + bam_dmux_skb_dma_unmap(skb_dma, DMA_TO_DEVICE); + + spin_lock_irqsave(&dmux->tx_lock, flags); + skb_dma->skb = NULL; + if (skb_dma == &dmux->tx_skbs[dmux->tx_next_skb % BAM_DMUX_NUM_SKB]) + bam_dmux_tx_wake_queues(dmux); + spin_unlock_irqrestore(&dmux->tx_lock, flags); +} + +static void bam_dmux_tx_callback(void *data) +{ + struct bam_dmux_skb_dma *skb_dma = data; + struct sk_buff *skb = skb_dma->skb; + + bam_dmux_tx_done(skb_dma); + dev_consume_skb_any(skb); +} + +static bool bam_dmux_skb_dma_submit_tx(struct bam_dmux_skb_dma *skb_dma) +{ + struct bam_dmux *dmux = skb_dma->dmux; + struct dma_async_tx_descriptor *desc; + + desc = dmaengine_prep_slave_single(dmux->tx, skb_dma->addr, + skb_dma->skb->len, DMA_MEM_TO_DEV, + DMA_PREP_INTERRUPT); + if (!desc) { + dev_err(dmux->dev, "Failed to prepare TX DMA buffer\n"); + return false; + } + + desc->callback = bam_dmux_tx_callback; + desc->callback_param = skb_dma; + desc->cookie = dmaengine_submit(desc); + return true; +} + +static struct bam_dmux_skb_dma * +bam_dmux_tx_queue(struct bam_dmux *dmux, struct sk_buff *skb) +{ + struct bam_dmux_skb_dma *skb_dma; + unsigned long flags; + + spin_lock_irqsave(&dmux->tx_lock, flags); + + skb_dma = &dmux->tx_skbs[dmux->tx_next_skb % BAM_DMUX_NUM_SKB]; + if (skb_dma->skb) { + bam_dmux_tx_stop_queues(dmux); + spin_unlock_irqrestore(&dmux->tx_lock, flags); + return NULL; + } + skb_dma->skb = skb; + + dmux->tx_next_skb++; + if (dmux->tx_skbs[dmux->tx_next_skb % BAM_DMUX_NUM_SKB].skb) + bam_dmux_tx_stop_queues(dmux); + + spin_unlock_irqrestore(&dmux->tx_lock, flags); + return skb_dma; +} + +static int bam_dmux_send_cmd(struct bam_dmux_netdev *bndev, u8 cmd) +{ + struct bam_dmux *dmux = bndev->dmux; + struct bam_dmux_skb_dma *skb_dma; + struct bam_dmux_hdr *hdr; + struct sk_buff *skb; + int ret; + + skb = alloc_skb(sizeof(*hdr), GFP_KERNEL); + if (!skb) + return -ENOMEM; + + hdr = skb_put_zero(skb, sizeof(*hdr)); + hdr->magic = BAM_DMUX_HDR_MAGIC; + hdr->cmd = cmd; + hdr->ch = bndev->ch; + + skb_dma = bam_dmux_tx_queue(dmux, skb); + if (!skb_dma) { + ret = -EAGAIN; + goto free_skb; + } + + ret = pm_runtime_get_sync(dmux->dev); + if (ret < 0) + goto tx_fail; + + if (!bam_dmux_skb_dma_map(skb_dma, DMA_TO_DEVICE)) { + ret = -ENOMEM; + goto tx_fail; + } + + if (!bam_dmux_skb_dma_submit_tx(skb_dma)) { + ret = -EIO; + goto tx_fail; + } + + dma_async_issue_pending(dmux->tx); + return 0; + +tx_fail: + bam_dmux_tx_done(skb_dma); +free_skb: + dev_kfree_skb(skb); + return ret; +} + +static int bam_dmux_netdev_open(struct net_device *netdev) +{ + struct bam_dmux_netdev *bndev = netdev_priv(netdev); + int ret; + + ret = bam_dmux_send_cmd(bndev, BAM_DMUX_CMD_OPEN); + if (ret) + return ret; + + netif_start_queue(netdev); + return 0; +} + +static int bam_dmux_netdev_stop(struct net_device *netdev) +{ + struct bam_dmux_netdev *bndev = netdev_priv(netdev); + + netif_stop_queue(netdev); + bam_dmux_send_cmd(bndev, BAM_DMUX_CMD_CLOSE); + return 0; +} + +static unsigned int needed_room(unsigned int avail, unsigned int needed) +{ + if (avail >= needed) + return 0; + return needed - avail; +} + +static int bam_dmux_tx_prepare_skb(struct bam_dmux_netdev *bndev, + struct sk_buff *skb) +{ + unsigned int head = needed_room(skb_headroom(skb), BAM_DMUX_HDR_SIZE); + unsigned int pad = sizeof(u32) - skb->len % sizeof(u32); + unsigned int tail = needed_room(skb_tailroom(skb), pad); + struct bam_dmux_hdr *hdr; + int ret; + + if (head || tail || skb_cloned(skb)) { + ret = pskb_expand_head(skb, head, tail, GFP_ATOMIC); + if (ret) + return ret; + } + + hdr = skb_push(skb, sizeof(*hdr)); + hdr->magic = BAM_DMUX_HDR_MAGIC; + hdr->signal = 0; + hdr->cmd = BAM_DMUX_CMD_DATA; + hdr->pad = pad; + hdr->ch = bndev->ch; + hdr->len = skb->len - sizeof(*hdr); + if (pad) + skb_put_zero(skb, pad); + + return 0; +} + +static netdev_tx_t bam_dmux_netdev_start_xmit(struct sk_buff *skb, + struct net_device *netdev) +{ + struct bam_dmux_netdev *bndev = netdev_priv(netdev); + struct bam_dmux *dmux = bndev->dmux; + struct bam_dmux_skb_dma *skb_dma; + int active, ret; + + skb_dma = bam_dmux_tx_queue(dmux, skb); + if (!skb_dma) + return NETDEV_TX_BUSY; + + active = pm_runtime_get(dmux->dev); + if (active < 0 && active != -EINPROGRESS) + goto drop; + + ret = bam_dmux_tx_prepare_skb(bndev, skb); + if (ret) + goto drop; + + if (!bam_dmux_skb_dma_map(skb_dma, DMA_TO_DEVICE)) + goto drop; + + if (active <= 0) { + /* Cannot sleep here so mark skb for wakeup handler and return */ + if (!atomic_long_fetch_or(BIT(skb_dma - dmux->tx_skbs), + &dmux->tx_deferred_skb)) + queue_pm_work(&dmux->tx_wakeup_work); + return NETDEV_TX_OK; + } + + if (!bam_dmux_skb_dma_submit_tx(skb_dma)) + goto drop; + + dma_async_issue_pending(dmux->tx); + return NETDEV_TX_OK; + +drop: + bam_dmux_tx_done(skb_dma); + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +static void bam_dmux_tx_wakeup_work(struct work_struct *work) +{ + struct bam_dmux *dmux = container_of(work, struct bam_dmux, tx_wakeup_work); + unsigned long pending; + int ret, i; + + ret = pm_runtime_resume_and_get(dmux->dev); + if (ret < 0) { + dev_err(dmux->dev, "Failed to resume: %d\n", ret); + return; + } + + pending = atomic_long_xchg(&dmux->tx_deferred_skb, 0); + if (!pending) + goto out; + + dev_dbg(dmux->dev, "pending skbs after wakeup: %#lx\n", pending); + for_each_set_bit(i, &pending, BAM_DMUX_NUM_SKB) { + bam_dmux_skb_dma_submit_tx(&dmux->tx_skbs[i]); + } + dma_async_issue_pending(dmux->tx); + +out: + pm_runtime_mark_last_busy(dmux->dev); + pm_runtime_put_autosuspend(dmux->dev); +} + +static const struct net_device_ops bam_dmux_ops = { + .ndo_open = bam_dmux_netdev_open, + .ndo_stop = bam_dmux_netdev_stop, + .ndo_start_xmit = bam_dmux_netdev_start_xmit, +}; + +static const struct device_type wwan_type = { + .name = "wwan", +}; + +static void bam_dmux_netdev_setup(struct net_device *dev) +{ + dev->netdev_ops = &bam_dmux_ops; + + dev->type = ARPHRD_RAWIP; + SET_NETDEV_DEVTYPE(dev, &wwan_type); + dev->flags = IFF_POINTOPOINT | IFF_NOARP; + + dev->mtu = ETH_DATA_LEN; + dev->max_mtu = BAM_DMUX_MAX_DATA_SIZE; + dev->needed_headroom = sizeof(struct bam_dmux_hdr); + dev->needed_tailroom = sizeof(u32); /* word-aligned */ + dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; + + /* This perm addr will be used as interface identifier by IPv6 */ + dev->addr_assign_type = NET_ADDR_RANDOM; + eth_random_addr(dev->perm_addr); +} + +static void bam_dmux_register_netdev_work(struct work_struct *work) +{ + struct bam_dmux *dmux = container_of(work, struct bam_dmux, register_netdev_work); + struct bam_dmux_netdev *bndev; + struct net_device *netdev; + int ch, ret; + + for_each_set_bit(ch, dmux->remote_channels, BAM_DMUX_NUM_CH) { + if (dmux->netdevs[ch]) + continue; + + netdev = alloc_netdev(sizeof(*bndev), "wwan%d", NET_NAME_ENUM, + bam_dmux_netdev_setup); + if (!netdev) + return; + + SET_NETDEV_DEV(netdev, dmux->dev); + netdev->dev_port = ch; + + bndev = netdev_priv(netdev); + bndev->dmux = dmux; + bndev->ch = ch; + + ret = register_netdev(netdev); + if (ret) { + dev_err(dmux->dev, "Failed to register netdev for channel %u: %d\n", + ch, ret); + free_netdev(netdev); + return; + } + + dmux->netdevs[ch] = netdev; + } +} + +static void bam_dmux_rx_callback(void *data); + +static bool bam_dmux_skb_dma_submit_rx(struct bam_dmux_skb_dma *skb_dma) +{ + struct bam_dmux *dmux = skb_dma->dmux; + struct dma_async_tx_descriptor *desc; + + desc = dmaengine_prep_slave_single(dmux->rx, skb_dma->addr, + skb_dma->skb->len, DMA_DEV_TO_MEM, + DMA_PREP_INTERRUPT); + if (!desc) { + dev_err(dmux->dev, "Failed to prepare RX DMA buffer\n"); + return false; + } + + desc->callback = bam_dmux_rx_callback; + desc->callback_param = skb_dma; + desc->cookie = dmaengine_submit(desc); + return true; +} + +static bool bam_dmux_skb_dma_queue_rx(struct bam_dmux_skb_dma *skb_dma, gfp_t gfp) +{ + if (!skb_dma->skb) { + skb_dma->skb = __netdev_alloc_skb(NULL, BAM_DMUX_BUFFER_SIZE, gfp); + if (!skb_dma->skb) + return false; + skb_put(skb_dma->skb, BAM_DMUX_BUFFER_SIZE); + } + + return bam_dmux_skb_dma_map(skb_dma, DMA_FROM_DEVICE) && + bam_dmux_skb_dma_submit_rx(skb_dma); +} + +static void bam_dmux_cmd_data(struct bam_dmux_skb_dma *skb_dma) +{ + struct bam_dmux *dmux = skb_dma->dmux; + struct sk_buff *skb = skb_dma->skb; + struct bam_dmux_hdr *hdr = (struct bam_dmux_hdr *)skb->data; + struct net_device *netdev = dmux->netdevs[hdr->ch]; + + if (!netdev || !netif_running(netdev)) { + dev_warn(dmux->dev, "Data for inactive channel %u\n", hdr->ch); + return; + } + + if (hdr->len > BAM_DMUX_MAX_DATA_SIZE) { + dev_err(dmux->dev, "Data larger than buffer? (%u > %u)\n", + hdr->len, (u16)BAM_DMUX_MAX_DATA_SIZE); + return; + } + + skb_dma->skb = NULL; /* Hand over to network stack */ + + skb_pull(skb, sizeof(*hdr)); + skb_trim(skb, hdr->len); + skb->dev = netdev; + + /* Only Raw-IP/QMAP is supported by this driver */ + switch (skb->data[0] & 0xf0) { + case 0x40: + skb->protocol = htons(ETH_P_IP); + break; + case 0x60: + skb->protocol = htons(ETH_P_IPV6); + break; + default: + skb->protocol = htons(ETH_P_MAP); + break; + } + + netif_receive_skb(skb); +} + +static void bam_dmux_cmd_open(struct bam_dmux *dmux, struct bam_dmux_hdr *hdr) +{ + struct net_device *netdev = dmux->netdevs[hdr->ch]; + + dev_dbg(dmux->dev, "open channel: %u\n", hdr->ch); + + if (__test_and_set_bit(hdr->ch, dmux->remote_channels)) { + dev_warn(dmux->dev, "Channel already open: %u\n", hdr->ch); + return; + } + + if (netdev) { + netif_device_attach(netdev); + } else { + /* Cannot sleep here, schedule work to register the netdev */ + schedule_work(&dmux->register_netdev_work); + } +} + +static void bam_dmux_cmd_close(struct bam_dmux *dmux, struct bam_dmux_hdr *hdr) +{ + struct net_device *netdev = dmux->netdevs[hdr->ch]; + + dev_dbg(dmux->dev, "close channel: %u\n", hdr->ch); + + if (!__test_and_clear_bit(hdr->ch, dmux->remote_channels)) { + dev_err(dmux->dev, "Channel not open: %u\n", hdr->ch); + return; + } + + if (netdev) + netif_device_detach(netdev); +} + +static void bam_dmux_rx_callback(void *data) +{ + struct bam_dmux_skb_dma *skb_dma = data; + struct bam_dmux *dmux = skb_dma->dmux; + struct sk_buff *skb = skb_dma->skb; + struct bam_dmux_hdr *hdr = (struct bam_dmux_hdr *)skb->data; + + bam_dmux_skb_dma_unmap(skb_dma, DMA_FROM_DEVICE); + + if (hdr->magic != BAM_DMUX_HDR_MAGIC) { + dev_err(dmux->dev, "Invalid magic in header: %#x\n", hdr->magic); + goto out; + } + + if (hdr->ch >= BAM_DMUX_NUM_CH) { + dev_dbg(dmux->dev, "Unsupported channel: %u\n", hdr->ch); + goto out; + } + + switch (hdr->cmd) { + case BAM_DMUX_CMD_DATA: + bam_dmux_cmd_data(skb_dma); + break; + case BAM_DMUX_CMD_OPEN: + bam_dmux_cmd_open(dmux, hdr); + break; + case BAM_DMUX_CMD_CLOSE: + bam_dmux_cmd_close(dmux, hdr); + break; + default: + dev_err(dmux->dev, "Unsupported command %u on channel %u\n", + hdr->cmd, hdr->ch); + break; + } + +out: + if (bam_dmux_skb_dma_queue_rx(skb_dma, GFP_ATOMIC)) + dma_async_issue_pending(dmux->rx); +} + +static bool bam_dmux_power_on(struct bam_dmux *dmux) +{ + struct device *dev = dmux->dev; + struct dma_slave_config dma_rx_conf = { + .direction = DMA_DEV_TO_MEM, + .src_maxburst = BAM_DMUX_BUFFER_SIZE, + }; + int i; + + dmux->rx = dma_request_chan(dev, "rx"); + if (IS_ERR(dmux->rx)) { + dev_err(dev, "Failed to request RX DMA channel: %pe\n", dmux->rx); + dmux->rx = NULL; + return false; + } + dmaengine_slave_config(dmux->rx, &dma_rx_conf); + + for (i = 0; i < BAM_DMUX_NUM_SKB; i++) { + if (!bam_dmux_skb_dma_queue_rx(&dmux->rx_skbs[i], GFP_KERNEL)) + return false; + } + dma_async_issue_pending(dmux->rx); + + return true; +} + +static void bam_dmux_free_skbs(struct bam_dmux_skb_dma skbs[], + enum dma_data_direction dir) +{ + int i; + + for (i = 0; i < BAM_DMUX_NUM_SKB; i++) { + struct bam_dmux_skb_dma *skb_dma = &skbs[i]; + + if (skb_dma->addr) + bam_dmux_skb_dma_unmap(skb_dma, dir); + if (skb_dma->skb) { + dev_kfree_skb(skb_dma->skb); + skb_dma->skb = NULL; + } + } +} + +static void bam_dmux_power_off(struct bam_dmux *dmux) +{ + if (dmux->tx) { + dmaengine_terminate_sync(dmux->tx); + dma_release_channel(dmux->tx); + dmux->tx = NULL; + } + + if (dmux->rx) { + dmaengine_terminate_sync(dmux->rx); + dma_release_channel(dmux->rx); + dmux->rx = NULL; + } + + bam_dmux_free_skbs(dmux->rx_skbs, DMA_FROM_DEVICE); +} + +static irqreturn_t bam_dmux_pc_irq(int irq, void *data) +{ + struct bam_dmux *dmux = data; + bool new_state = !dmux->pc_state; + + dev_dbg(dmux->dev, "pc: %u\n", new_state); + + if (new_state) { + if (bam_dmux_power_on(dmux)) + bam_dmux_pc_ack(dmux); + else + bam_dmux_power_off(dmux); + } else { + bam_dmux_power_off(dmux); + bam_dmux_pc_ack(dmux); + } + + dmux->pc_state = new_state; + wake_up_all(&dmux->pc_wait); + + return IRQ_HANDLED; +} + +static irqreturn_t bam_dmux_pc_ack_irq(int irq, void *data) +{ + struct bam_dmux *dmux = data; + + dev_dbg(dmux->dev, "pc ack\n"); + complete_all(&dmux->pc_ack_completion); + + return IRQ_HANDLED; +} + +static int bam_dmux_runtime_suspend(struct device *dev) +{ + struct bam_dmux *dmux = dev_get_drvdata(dev); + + dev_dbg(dev, "runtime suspend\n"); + bam_dmux_pc_vote(dmux, false); + + return 0; +} + +static int __maybe_unused bam_dmux_runtime_resume(struct device *dev) +{ + struct bam_dmux *dmux = dev_get_drvdata(dev); + + dev_dbg(dev, "runtime resume\n"); + + /* Wait until previous power down was acked */ + if (!wait_for_completion_timeout(&dmux->pc_ack_completion, + BAM_DMUX_REMOTE_TIMEOUT)) + return -ETIMEDOUT; + + /* Vote for power state */ + bam_dmux_pc_vote(dmux, true); + + /* Wait for ack */ + if (!wait_for_completion_timeout(&dmux->pc_ack_completion, + BAM_DMUX_REMOTE_TIMEOUT)) { + bam_dmux_pc_vote(dmux, false); + return -ETIMEDOUT; + } + + /* Wait until we're up */ + if (!wait_event_timeout(dmux->pc_wait, dmux->pc_state, + BAM_DMUX_REMOTE_TIMEOUT)) { + bam_dmux_pc_vote(dmux, false); + return -ETIMEDOUT; + } + + /* Ensure that we actually initialized successfully */ + if (!dmux->rx) { + bam_dmux_pc_vote(dmux, false); + return -ENXIO; + } + + /* Request TX channel if necessary */ + if (dmux->tx) + return 0; + + dmux->tx = dma_request_chan(dev, "tx"); + if (IS_ERR(dmux->rx)) { + dev_err(dev, "Failed to request TX DMA channel: %pe\n", dmux->tx); + dmux->tx = NULL; + bam_dmux_runtime_suspend(dev); + return -ENXIO; + } + + return 0; +} + +static int bam_dmux_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct bam_dmux *dmux; + int ret, pc_ack_irq, i; + unsigned int bit; + + dmux = devm_kzalloc(dev, sizeof(*dmux), GFP_KERNEL); + if (!dmux) + return -ENOMEM; + + dmux->dev = dev; + platform_set_drvdata(pdev, dmux); + + dmux->pc_irq = platform_get_irq_byname(pdev, "pc"); + if (dmux->pc_irq < 0) + return dmux->pc_irq; + + pc_ack_irq = platform_get_irq_byname(pdev, "pc-ack"); + if (pc_ack_irq < 0) + return pc_ack_irq; + + dmux->pc = devm_qcom_smem_state_get(dev, "pc", &bit); + if (IS_ERR(dmux->pc)) + return dev_err_probe(dev, PTR_ERR(dmux->pc), + "Failed to get pc state\n"); + dmux->pc_mask = BIT(bit); + + dmux->pc_ack = devm_qcom_smem_state_get(dev, "pc-ack", &bit); + if (IS_ERR(dmux->pc_ack)) + return dev_err_probe(dev, PTR_ERR(dmux->pc_ack), + "Failed to get pc-ack state\n"); + dmux->pc_ack_mask = BIT(bit); + + init_waitqueue_head(&dmux->pc_wait); + init_completion(&dmux->pc_ack_completion); + complete_all(&dmux->pc_ack_completion); + + spin_lock_init(&dmux->tx_lock); + INIT_WORK(&dmux->tx_wakeup_work, bam_dmux_tx_wakeup_work); + INIT_WORK(&dmux->register_netdev_work, bam_dmux_register_netdev_work); + + for (i = 0; i < BAM_DMUX_NUM_SKB; i++) { + dmux->rx_skbs[i].dmux = dmux; + dmux->tx_skbs[i].dmux = dmux; + } + + /* Runtime PM manages our own power vote. + * Note that the RX path may be active even if we are runtime suspended, + * since it is controlled by the remote side. + */ + pm_runtime_set_autosuspend_delay(dev, BAM_DMUX_AUTOSUSPEND_DELAY); + pm_runtime_use_autosuspend(dev); + pm_runtime_enable(dev); + + ret = devm_request_threaded_irq(dev, pc_ack_irq, NULL, bam_dmux_pc_ack_irq, + IRQF_ONESHOT, NULL, dmux); + if (ret) + return ret; + + ret = devm_request_threaded_irq(dev, dmux->pc_irq, NULL, bam_dmux_pc_irq, + IRQF_ONESHOT, NULL, dmux); + if (ret) + return ret; + + ret = irq_get_irqchip_state(dmux->pc_irq, IRQCHIP_STATE_LINE_LEVEL, + &dmux->pc_state); + if (ret) + return ret; + + /* Check if remote finished initialization before us */ + if (dmux->pc_state) { + if (bam_dmux_power_on(dmux)) + bam_dmux_pc_ack(dmux); + else + bam_dmux_power_off(dmux); + } + + return 0; +} + +static int bam_dmux_remove(struct platform_device *pdev) +{ + struct bam_dmux *dmux = platform_get_drvdata(pdev); + struct device *dev = dmux->dev; + LIST_HEAD(list); + int i; + + /* Unregister network interfaces */ + cancel_work_sync(&dmux->register_netdev_work); + rtnl_lock(); + for (i = 0; i < BAM_DMUX_NUM_CH; ++i) + if (dmux->netdevs[i]) + unregister_netdevice_queue(dmux->netdevs[i], &list); + unregister_netdevice_many(&list); + rtnl_unlock(); + cancel_work_sync(&dmux->tx_wakeup_work); + + /* Drop our own power vote */ + pm_runtime_disable(dev); + pm_runtime_dont_use_autosuspend(dev); + bam_dmux_runtime_suspend(dev); + pm_runtime_set_suspended(dev); + + /* Try to wait for remote side to drop power vote */ + if (!wait_event_timeout(dmux->pc_wait, !dmux->rx, BAM_DMUX_REMOTE_TIMEOUT)) + dev_err(dev, "Timed out waiting for remote side to suspend\n"); + + /* Make sure everything is cleaned up before we return */ + disable_irq(dmux->pc_irq); + bam_dmux_power_off(dmux); + bam_dmux_free_skbs(dmux->tx_skbs, DMA_TO_DEVICE); + + return 0; +} + +static const struct dev_pm_ops bam_dmux_pm_ops = { + SET_RUNTIME_PM_OPS(bam_dmux_runtime_suspend, bam_dmux_runtime_resume, NULL) +}; + +static const struct of_device_id bam_dmux_of_match[] = { + { .compatible = "qcom,bam-dmux" }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, bam_dmux_of_match); + +static struct platform_driver bam_dmux_driver = { + .probe = bam_dmux_probe, + .remove = bam_dmux_remove, + .driver = { + .name = "bam-dmux", + .pm = &bam_dmux_pm_ops, + .of_match_table = bam_dmux_of_match, + }, +}; +module_platform_driver(bam_dmux_driver); + +MODULE_LICENSE("GPL v2"); +MODULE_DESCRIPTION("Qualcomm BAM-DMUX WWAN Network Driver"); +MODULE_AUTHOR("Stephan Gerhold <stephan@gerhold.net>"); diff --git a/drivers/net/wwan/wwan_core.c b/drivers/net/wwan/wwan_core.c index d293ab688044..1508dc2a497b 100644 --- a/drivers/net/wwan/wwan_core.c +++ b/drivers/net/wwan/wwan_core.c @@ -3,6 +3,7 @@ #include <linux/err.h> #include <linux/errno.h> +#include <linux/debugfs.h> #include <linux/fs.h> #include <linux/init.h> #include <linux/idr.h> @@ -25,6 +26,7 @@ static DEFINE_IDA(minors); /* minors for WWAN port chardevs */ static DEFINE_IDA(wwan_dev_ids); /* for unique WWAN device IDs */ static struct class *wwan_class; static int wwan_major; +static struct dentry *wwan_debugfs_dir; #define to_wwan_dev(d) container_of(d, struct wwan_device, dev) #define to_wwan_port(d) container_of(d, struct wwan_port, dev) @@ -40,6 +42,7 @@ static int wwan_major; * @port_id: Current available port ID to pick. * @ops: wwan device ops * @ops_ctxt: context to pass to ops + * @debugfs_dir: WWAN device debugfs dir */ struct wwan_device { unsigned int id; @@ -47,6 +50,9 @@ struct wwan_device { atomic_t port_id; const struct wwan_ops *ops; void *ops_ctxt; +#ifdef CONFIG_WWAN_DEBUGFS + struct dentry *debugfs_dir; +#endif }; /** @@ -142,6 +148,20 @@ static struct wwan_device *wwan_dev_get_by_name(const char *name) return to_wwan_dev(dev); } +#ifdef CONFIG_WWAN_DEBUGFS +struct dentry *wwan_get_debugfs_dir(struct device *parent) +{ + struct wwan_device *wwandev; + + wwandev = wwan_dev_get_by_parent(parent); + if (IS_ERR(wwandev)) + return ERR_CAST(wwandev); + + return wwandev->debugfs_dir; +} +EXPORT_SYMBOL_GPL(wwan_get_debugfs_dir); +#endif + /* This function allocates and registers a new WWAN device OR if a WWAN device * already exist for the given parent, it gets a reference and return it. * This function is not exported (for now), it is called indirectly via @@ -189,6 +209,12 @@ static struct wwan_device *wwan_create_dev(struct device *parent) goto done_unlock; } +#ifdef CONFIG_WWAN_DEBUGFS + wwandev->debugfs_dir = + debugfs_create_dir(kobject_name(&wwandev->dev.kobj), + wwan_debugfs_dir); +#endif + done_unlock: mutex_unlock(&wwan_register_lock); @@ -218,10 +244,14 @@ static void wwan_remove_dev(struct wwan_device *wwandev) else ret = device_for_each_child(&wwandev->dev, NULL, is_wwan_child); - if (!ret) + if (!ret) { +#ifdef CONFIG_WWAN_DEBUGFS + debugfs_remove_recursive(wwandev->debugfs_dir); +#endif device_unregister(&wwandev->dev); - else + } else { put_device(&wwandev->dev); + } mutex_unlock(&wwan_register_lock); } @@ -1117,6 +1147,10 @@ static int __init wwan_init(void) goto destroy; } +#ifdef CONFIG_WWAN_DEBUGFS + wwan_debugfs_dir = debugfs_create_dir("wwan", NULL); +#endif + return 0; destroy: @@ -1128,6 +1162,7 @@ unregister: static void __exit wwan_exit(void) { + debugfs_remove_recursive(wwan_debugfs_dir); __unregister_chrdev(wwan_major, 0, WWAN_MAX_MINORS, "wwan_port"); rtnl_link_unregister(&wwan_rtnl_link_ops); class_destroy(wwan_class); diff --git a/drivers/nfc/fdp/i2c.c b/drivers/nfc/fdp/i2c.c index f78670bf41e0..28a9e1eb9bcf 100644 --- a/drivers/nfc/fdp/i2c.c +++ b/drivers/nfc/fdp/i2c.c @@ -205,9 +205,7 @@ static irqreturn_t fdp_nci_i2c_irq_thread_fn(int irq, void *phy_id) r = fdp_nci_i2c_read(phy, &skb); - if (r == -EREMOTEIO) - return IRQ_HANDLED; - else if (r == -ENOMEM || r == -EBADMSG) + if (r == -EREMOTEIO || r == -ENOMEM || r == -EBADMSG) return IRQ_HANDLED; if (skb != NULL) diff --git a/drivers/pcmcia/pcmcia_cis.c b/drivers/pcmcia/pcmcia_cis.c index f650e19a315c..6bc0bc24d357 100644 --- a/drivers/pcmcia/pcmcia_cis.c +++ b/drivers/pcmcia/pcmcia_cis.c @@ -386,7 +386,7 @@ size_t pcmcia_get_tuple(struct pcmcia_device *p_dev, cisdata_t code, } EXPORT_SYMBOL(pcmcia_get_tuple); - +#ifdef CONFIG_NET /* * pcmcia_do_get_mac() - internal helper for pcmcia_get_mac_from_cis() * @@ -431,3 +431,4 @@ int pcmcia_get_mac_from_cis(struct pcmcia_device *p_dev, struct net_device *dev) } EXPORT_SYMBOL(pcmcia_get_mac_from_cis); +#endif /* CONFIG_NET */ diff --git a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c index bbd6f2ad6f24..34672e868a1e 100644 --- a/drivers/phy/marvell/phy-mvebu-cp110-comphy.c +++ b/drivers/phy/marvell/phy-mvebu-cp110-comphy.c @@ -141,6 +141,7 @@ #define COMPHY_FW_SPEED_1250 0 #define COMPHY_FW_SPEED_3125 2 #define COMPHY_FW_SPEED_5000 3 +#define COMPHY_FW_SPEED_515625 4 #define COMPHY_FW_SPEED_103125 6 #define COMPHY_FW_PORT_OFFSET 8 #define COMPHY_FW_PORT_MASK GENMASK(11, 8) @@ -220,6 +221,7 @@ static const struct mvebu_comphy_conf mvebu_comphy_cp110_modes[] = { ETH_CONF(2, 0, PHY_INTERFACE_MODE_SGMII, 0x1, COMPHY_FW_MODE_SGMII), ETH_CONF(2, 0, PHY_INTERFACE_MODE_2500BASEX, 0x1, COMPHY_FW_MODE_2500BASEX), ETH_CONF(2, 0, PHY_INTERFACE_MODE_RXAUI, 0x1, COMPHY_FW_MODE_RXAUI), + ETH_CONF(2, 0, PHY_INTERFACE_MODE_5GBASER, 0x1, COMPHY_FW_MODE_XFI), ETH_CONF(2, 0, PHY_INTERFACE_MODE_10GBASER, 0x1, COMPHY_FW_MODE_XFI), GEN_CONF(2, 0, PHY_MODE_USB_HOST_SS, COMPHY_FW_MODE_USB3H), GEN_CONF(2, 0, PHY_MODE_SATA, COMPHY_FW_MODE_SATA), @@ -234,6 +236,7 @@ static const struct mvebu_comphy_conf mvebu_comphy_cp110_modes[] = { /* lane 4 */ ETH_CONF(4, 0, PHY_INTERFACE_MODE_SGMII, 0x2, COMPHY_FW_MODE_SGMII), ETH_CONF(4, 0, PHY_INTERFACE_MODE_2500BASEX, 0x2, COMPHY_FW_MODE_2500BASEX), + ETH_CONF(4, 0, PHY_INTERFACE_MODE_5GBASER, 0x2, COMPHY_FW_MODE_XFI), ETH_CONF(4, 0, PHY_INTERFACE_MODE_10GBASER, 0x2, COMPHY_FW_MODE_XFI), ETH_CONF(4, 0, PHY_INTERFACE_MODE_RXAUI, 0x2, COMPHY_FW_MODE_RXAUI), GEN_CONF(4, 0, PHY_MODE_USB_DEVICE_SS, COMPHY_FW_MODE_USB3D), @@ -241,6 +244,7 @@ static const struct mvebu_comphy_conf mvebu_comphy_cp110_modes[] = { GEN_CONF(4, 1, PHY_MODE_PCIE, COMPHY_FW_MODE_PCIE), ETH_CONF(4, 1, PHY_INTERFACE_MODE_SGMII, 0x1, COMPHY_FW_MODE_SGMII), ETH_CONF(4, 1, PHY_INTERFACE_MODE_2500BASEX, -1, COMPHY_FW_MODE_2500BASEX), + ETH_CONF(4, 1, PHY_INTERFACE_MODE_5GBASER, -1, COMPHY_FW_MODE_XFI), ETH_CONF(4, 1, PHY_INTERFACE_MODE_10GBASER, -1, COMPHY_FW_MODE_XFI), /* lane 5 */ ETH_CONF(5, 1, PHY_INTERFACE_MODE_RXAUI, 0x2, COMPHY_FW_MODE_RXAUI), @@ -790,6 +794,11 @@ static int mvebu_comphy_power_on(struct phy *phy) lane->id); fw_speed = COMPHY_FW_SPEED_3125; break; + case PHY_INTERFACE_MODE_5GBASER: + dev_dbg(priv->dev, "set lane %d to 5GBASE-R mode\n", + lane->id); + fw_speed = COMPHY_FW_SPEED_515625; + break; case PHY_INTERFACE_MODE_10GBASER: dev_dbg(priv->dev, "set lane %d to 10GBASE-R mode\n", lane->id); diff --git a/drivers/s390/net/ctcm_dbug.h b/drivers/s390/net/ctcm_dbug.h index 675575ef162e..cce11daf3245 100644 --- a/drivers/s390/net/ctcm_dbug.h +++ b/drivers/s390/net/ctcm_dbug.h @@ -65,6 +65,7 @@ extern struct ctcm_dbf_info ctcm_dbf[CTCM_DBF_INFOS]; int ctcm_register_dbf_views(void); void ctcm_unregister_dbf_views(void); +__printf(3, 4) void ctcm_dbf_longtext(enum ctcm_dbf_names dbf_nix, int level, char *text, ...); static inline const char *strtail(const char *s, int n) diff --git a/drivers/s390/net/ctcm_fsms.c b/drivers/s390/net/ctcm_fsms.c index de2423c72b02..5db591cf7215 100644 --- a/drivers/s390/net/ctcm_fsms.c +++ b/drivers/s390/net/ctcm_fsms.c @@ -1406,7 +1406,7 @@ static void ctcmpc_chx_rx(fsm_instance *fi, int event, void *arg) if (new_skb == NULL) { CTCM_DBF_TEXT_(MPC_ERROR, CTC_DBF_ERROR, - "%s(%d): skb allocation failed", + "%s(%s): skb allocation failed", CTCM_FUNTAIL, dev->name); fsm_event(priv->mpcg->fsm, MPCG_EVENT_INOP, dev); goto again; diff --git a/drivers/s390/net/lcs.c b/drivers/s390/net/lcs.c index 2a6479740600..a61d38a1b4ed 100644 --- a/drivers/s390/net/lcs.c +++ b/drivers/s390/net/lcs.c @@ -1808,19 +1808,20 @@ lcs_get_frames_cb(struct lcs_channel *channel, struct lcs_buffer *buffer) return; } /* What kind of frame is it? */ - if (lcs_hdr->type == LCS_FRAME_TYPE_CONTROL) + if (lcs_hdr->type == LCS_FRAME_TYPE_CONTROL) { /* Control frame. */ lcs_get_control(card, (struct lcs_cmd *) lcs_hdr); - else if (lcs_hdr->type == LCS_FRAME_TYPE_ENET || - lcs_hdr->type == LCS_FRAME_TYPE_TR || - lcs_hdr->type == LCS_FRAME_TYPE_FDDI) + } else if (lcs_hdr->type == LCS_FRAME_TYPE_ENET || + lcs_hdr->type == LCS_FRAME_TYPE_TR || + lcs_hdr->type == LCS_FRAME_TYPE_FDDI) { /* Normal network packet. */ lcs_get_skb(card, (char *)(lcs_hdr + 1), lcs_hdr->offset - offset - sizeof(struct lcs_header)); - else + } else { /* Unknown frame type. */ ; // FIXME: error message ? + } /* Proceed to next frame. */ offset = lcs_hdr->offset; lcs_hdr->offset = LCS_ILLEGAL_OFFSET; diff --git a/drivers/s390/net/qeth_core.h b/drivers/s390/net/qeth_core.h index 20dca4c0384a..de25d7ac41da 100644 --- a/drivers/s390/net/qeth_core.h +++ b/drivers/s390/net/qeth_core.h @@ -1030,8 +1030,6 @@ static inline int qeth_send_simple_setassparms_v6(struct qeth_card *card, data, QETH_PROT_IPV6); } -int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb); - extern const struct qeth_discipline qeth_l2_discipline; extern const struct qeth_discipline qeth_l3_discipline; extern const struct ethtool_ops qeth_ethtool_ops; @@ -1099,6 +1097,8 @@ void qeth_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats); int qeth_set_real_num_tx_queues(struct qeth_card *card, unsigned int count); u16 qeth_iqd_select_queue(struct net_device *dev, struct sk_buff *skb, u8 cast_type, struct net_device *sb_dev); +u16 qeth_osa_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev); int qeth_open(struct net_device *dev); int qeth_stop(struct net_device *dev); diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c index 26c55f67289f..c296e0556f49 100644 --- a/drivers/s390/net/qeth_core_main.c +++ b/drivers/s390/net/qeth_core_main.c @@ -194,9 +194,6 @@ static void qeth_clear_working_pool_list(struct qeth_card *card) &card->qdio.in_buf_pool.entry_list, list) list_del(&pool_entry->list); - if (!queue) - return; - for (i = 0; i < ARRAY_SIZE(queue->bufs); i++) queue->bufs[i].pool_entry = NULL; } @@ -275,8 +272,8 @@ int qeth_resize_buffer_pool(struct qeth_card *card, unsigned int count) QETH_CARD_TEXT(card, 2, "realcbp"); - /* Defer until queue is allocated: */ - if (!card->qdio.in_q) + /* Defer until pool is allocated: */ + if (list_empty(&pool->entry_list)) goto out; /* Remove entries from the pool: */ @@ -2557,14 +2554,9 @@ static int qeth_alloc_qdio_queues(struct qeth_card *card) QETH_QDIO_ALLOCATED) != QETH_QDIO_UNINITIALIZED) return 0; - QETH_CARD_TEXT(card, 2, "inq"); - card->qdio.in_q = qeth_alloc_qdio_queue(); - if (!card->qdio.in_q) - goto out_nomem; - /* inbound buffer pool */ if (qeth_alloc_buffer_pool(card)) - goto out_freeinq; + goto out_buffer_pool; /* outbound */ for (i = 0; i < card->qdio.no_out_queues; ++i) { @@ -2605,10 +2597,7 @@ out_freeoutq: card->qdio.out_qs[i] = NULL; } qeth_free_buffer_pool(card); -out_freeinq: - qeth_free_qdio_queue(card->qdio.in_q); - card->qdio.in_q = NULL; -out_nomem: +out_buffer_pool: atomic_set(&card->qdio.state, QETH_QDIO_UNINITIALIZED); return -ENOMEM; } @@ -2623,11 +2612,12 @@ static void qeth_free_qdio_queues(struct qeth_card *card) qeth_free_cq(card); for (j = 0; j < QDIO_MAX_BUFFERS_PER_Q; ++j) { - if (card->qdio.in_q->bufs[j].rx_skb) + if (card->qdio.in_q->bufs[j].rx_skb) { consume_skb(card->qdio.in_q->bufs[j].rx_skb); + card->qdio.in_q->bufs[j].rx_skb = NULL; + } } - qeth_free_qdio_queue(card->qdio.in_q); - card->qdio.in_q = NULL; + /* inbound buffer pool */ qeth_free_buffer_pool(card); /* free outbound qdio_qs */ @@ -3645,12 +3635,10 @@ static void qeth_check_outbound_queue(struct qeth_qdio_out_q *queue) if ((atomic_read(&queue->used_buffers) <= QETH_LOW_WATERMARK_PACK) || !atomic_read(&queue->set_pci_flags_count)) { unsigned int index, flush_cnt; - bool q_was_packing; spin_lock(&queue->lock); index = queue->next_buf_to_fill; - q_was_packing = queue->do_pack; flush_cnt = qeth_switch_to_nonpacking_if_needed(queue); if (!flush_cnt && !atomic_read(&queue->set_pci_flags_count)) @@ -3658,8 +3646,7 @@ static void qeth_check_outbound_queue(struct qeth_qdio_out_q *queue) if (flush_cnt) { qeth_flush_buffers(queue, index, flush_cnt); - if (q_was_packing) - QETH_TXQ_STAT_ADD(queue, bufs_pack, flush_cnt); + QETH_TXQ_STAT_ADD(queue, bufs_pack, flush_cnt); } spin_unlock(&queue->lock); @@ -3779,7 +3766,7 @@ static void qeth_qdio_output_handler(struct ccw_device *ccwdev, /* * Note: Function assumes that we have 4 outbound queues. */ -int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb) +static int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb) { struct vlan_ethhdr *veth = vlan_eth_hdr(skb); u8 tos; @@ -3824,7 +3811,6 @@ int qeth_get_priority_queue(struct qeth_card *card, struct sk_buff *skb) } return card->qdio.default_out_queue; } -EXPORT_SYMBOL_GPL(qeth_get_priority_queue); /** * qeth_get_elements_for_frags() - find number of SBALEs for skb frags. @@ -5585,29 +5571,9 @@ static void qeth_l3_rebuild_skb(struct qeth_card *card, struct sk_buff *skb, #endif static void qeth_receive_skb(struct qeth_card *card, struct sk_buff *skb, - struct qeth_hdr *hdr, bool uses_frags) + bool uses_frags, bool is_cso) { struct napi_struct *napi = &card->napi; - bool is_cso; - - switch (hdr->hdr.l2.id) { -#if IS_ENABLED(CONFIG_QETH_L3) - case QETH_HEADER_TYPE_LAYER3: - qeth_l3_rebuild_skb(card, skb, hdr); - is_cso = hdr->hdr.l3.ext_flags & QETH_HDR_EXT_CSUM_TRANSP_REQ; - break; -#endif - case QETH_HEADER_TYPE_LAYER2: - is_cso = hdr->hdr.l2.flags[1] & QETH_HDR_EXT_CSUM_TRANSP_REQ; - break; - default: - /* never happens */ - if (uses_frags) - napi_free_frags(napi); - else - kfree_skb(skb); - return; - } if (is_cso && (card->dev->features & NETIF_F_RXCSUM)) { skb->ip_summed = CHECKSUM_UNNECESSARY; @@ -5664,6 +5630,7 @@ static int qeth_extract_skb(struct qeth_card *card, struct qeth_hdr *hdr; struct sk_buff *skb; int skb_len = 0; + bool is_cso; element = &buffer->element[*element_no]; @@ -5683,11 +5650,15 @@ next_packet: switch (hdr->hdr.l2.id) { case QETH_HEADER_TYPE_LAYER2: skb_len = hdr->hdr.l2.pkt_length; + is_cso = hdr->hdr.l2.flags[1] & QETH_HDR_EXT_CSUM_TRANSP_REQ; + linear_len = ETH_HLEN; headroom = 0; break; case QETH_HEADER_TYPE_LAYER3: skb_len = hdr->hdr.l3.length; + is_cso = hdr->hdr.l3.ext_flags & QETH_HDR_EXT_CSUM_TRANSP_REQ; + if (!IS_LAYER3(card)) { QETH_CARD_STAT_INC(card, rx_dropped_notsupp); goto walk_packet; @@ -5814,7 +5785,12 @@ walk_packet: *element_no = element - &buffer->element[0]; *__offset = offset; - qeth_receive_skb(card, skb, hdr, uses_frags); +#if IS_ENABLED(CONFIG_QETH_L3) + if (hdr->hdr.l2.id == QETH_HEADER_TYPE_LAYER3) + qeth_l3_rebuild_skb(card, skb, hdr); +#endif + + qeth_receive_skb(card, skb, uses_frags, is_cso); return 0; } @@ -6447,6 +6423,12 @@ static int qeth_core_probe_device(struct ccwgroup_device *gdev) qeth_determine_capabilities(card); qeth_set_blkt_defaults(card); + card->qdio.in_q = qeth_alloc_qdio_queue(); + if (!card->qdio.in_q) { + rc = -ENOMEM; + goto err_rx_queue; + } + card->qdio.no_out_queues = card->dev->num_tx_queues; rc = qeth_update_from_chp_desc(card); if (rc) @@ -6473,6 +6455,8 @@ static int qeth_core_probe_device(struct ccwgroup_device *gdev) err_setup_disc: err_chp_desc: + qeth_free_qdio_queue(card->qdio.in_q); +err_rx_queue: free_netdev(card->dev); err_card: qeth_core_free_card(card); @@ -6494,6 +6478,7 @@ static void qeth_core_remove_device(struct ccwgroup_device *gdev) qeth_free_qdio_queues(card); + qeth_free_qdio_queue(card->qdio.in_q); free_netdev(card->dev); qeth_core_free_card(card); put_device(&gdev->dev); @@ -7089,6 +7074,18 @@ u16 qeth_iqd_select_queue(struct net_device *dev, struct sk_buff *skb, } EXPORT_SYMBOL_GPL(qeth_iqd_select_queue); +u16 qeth_osa_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + struct qeth_card *card = dev->ml_priv; + + if (qeth_uses_tx_prio_queueing(card)) + return qeth_get_priority_queue(card, skb); + + return netdev_pick_tx(dev, skb, sb_dev); +} +EXPORT_SYMBOL_GPL(qeth_osa_select_queue); + int qeth_open(struct net_device *dev) { struct qeth_card *card = dev->ml_priv; diff --git a/drivers/s390/net/qeth_ethtool.c b/drivers/s390/net/qeth_ethtool.c index 46d0fe0d0e8a..b0b36b2132fe 100644 --- a/drivers/s390/net/qeth_ethtool.c +++ b/drivers/s390/net/qeth_ethtool.c @@ -144,7 +144,9 @@ static int qeth_set_coalesce(struct net_device *dev, } static void qeth_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param, + struct netlink_ext_ack *extack) { struct qeth_card *card = dev->ml_priv; diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index 0347fc184786..303461d70af3 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -519,19 +519,11 @@ static netdev_tx_t qeth_l2_hard_start_xmit(struct sk_buff *skb, return NETDEV_TX_OK; } -static u16 qeth_l2_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) +static u16 qeth_l2_iqd_select_queue(struct net_device *dev, struct sk_buff *skb, + struct net_device *sb_dev) { - struct qeth_card *card = dev->ml_priv; - - if (IS_IQD(card)) - return qeth_iqd_select_queue(dev, skb, - qeth_get_ether_cast_type(skb), - sb_dev); - if (qeth_uses_tx_prio_queueing(card)) - return qeth_get_priority_queue(card, skb); - - return netdev_pick_tx(dev, skb, sb_dev); + return qeth_iqd_select_queue(dev, skb, qeth_get_ether_cast_type(skb), + sb_dev); } static void qeth_l2_set_rx_mode(struct net_device *dev) @@ -726,7 +718,8 @@ struct qeth_l2_br2dev_event_work { unsigned char addr[ETH_ALEN]; }; -static const struct net_device_ops qeth_l2_netdev_ops; +static const struct net_device_ops qeth_l2_iqd_netdev_ops; +static const struct net_device_ops qeth_l2_osa_netdev_ops; static bool qeth_l2_must_learn(struct net_device *netdev, struct net_device *dstdev) @@ -738,7 +731,8 @@ static bool qeth_l2_must_learn(struct net_device *netdev, (priv->brport_features & BR_LEARNING_SYNC) && !(br_port_flag_is_set(netdev, BR_ISOLATED) && br_port_flag_is_set(dstdev, BR_ISOLATED)) && - netdev->netdev_ops == &qeth_l2_netdev_ops); + (netdev->netdev_ops == &qeth_l2_iqd_netdev_ops || + netdev->netdev_ops == &qeth_l2_osa_netdev_ops)); } /** @@ -1051,20 +1045,20 @@ static int qeth_l2_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, return rc; } -static const struct net_device_ops qeth_l2_netdev_ops = { +static const struct net_device_ops qeth_l2_iqd_netdev_ops = { .ndo_open = qeth_open, .ndo_stop = qeth_stop, .ndo_get_stats64 = qeth_get_stats64, .ndo_start_xmit = qeth_l2_hard_start_xmit, .ndo_features_check = qeth_features_check, - .ndo_select_queue = qeth_l2_select_queue, + .ndo_select_queue = qeth_l2_iqd_select_queue, .ndo_validate_addr = qeth_l2_validate_addr, .ndo_set_rx_mode = qeth_l2_set_rx_mode, .ndo_eth_ioctl = qeth_do_ioctl, .ndo_siocdevprivate = qeth_siocdevprivate, - .ndo_set_mac_address = qeth_l2_set_mac_address, + .ndo_set_mac_address = qeth_l2_set_mac_address, .ndo_vlan_rx_add_vid = qeth_l2_vlan_rx_add_vid, - .ndo_vlan_rx_kill_vid = qeth_l2_vlan_rx_kill_vid, + .ndo_vlan_rx_kill_vid = qeth_l2_vlan_rx_kill_vid, .ndo_tx_timeout = qeth_tx_timeout, .ndo_fix_features = qeth_fix_features, .ndo_set_features = qeth_set_features, @@ -1072,10 +1066,30 @@ static const struct net_device_ops qeth_l2_netdev_ops = { .ndo_bridge_setlink = qeth_l2_bridge_setlink, }; +static const struct net_device_ops qeth_l2_osa_netdev_ops = { + .ndo_open = qeth_open, + .ndo_stop = qeth_stop, + .ndo_get_stats64 = qeth_get_stats64, + .ndo_start_xmit = qeth_l2_hard_start_xmit, + .ndo_features_check = qeth_features_check, + .ndo_select_queue = qeth_osa_select_queue, + .ndo_validate_addr = qeth_l2_validate_addr, + .ndo_set_rx_mode = qeth_l2_set_rx_mode, + .ndo_eth_ioctl = qeth_do_ioctl, + .ndo_siocdevprivate = qeth_siocdevprivate, + .ndo_set_mac_address = qeth_l2_set_mac_address, + .ndo_vlan_rx_add_vid = qeth_l2_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = qeth_l2_vlan_rx_kill_vid, + .ndo_tx_timeout = qeth_tx_timeout, + .ndo_fix_features = qeth_fix_features, + .ndo_set_features = qeth_set_features, +}; + static int qeth_l2_setup_netdev(struct qeth_card *card) { + card->dev->netdev_ops = IS_IQD(card) ? &qeth_l2_iqd_netdev_ops : + &qeth_l2_osa_netdev_ops; card->dev->needed_headroom = sizeof(struct qeth_hdr); - card->dev->netdev_ops = &qeth_l2_netdev_ops; card->dev->priv_flags |= IFF_UNICAST_FLT; if (IS_OSM(card)) { diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c index 48a886f7af62..9251ad276ee8 100644 --- a/drivers/s390/net/qeth_l3_main.c +++ b/drivers/s390/net/qeth_l3_main.c @@ -1822,17 +1822,6 @@ static u16 qeth_l3_iqd_select_queue(struct net_device *dev, struct sk_buff *skb, qeth_l3_get_cast_type(skb, proto), sb_dev); } -static u16 qeth_l3_osa_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) -{ - struct qeth_card *card = dev->ml_priv; - - if (qeth_uses_tx_prio_queueing(card)) - return qeth_get_priority_queue(card, skb); - - return netdev_pick_tx(dev, skb, sb_dev); -} - static const struct net_device_ops qeth_l3_netdev_ops = { .ndo_open = qeth_open, .ndo_stop = qeth_stop, @@ -1854,7 +1843,7 @@ static const struct net_device_ops qeth_l3_osa_netdev_ops = { .ndo_get_stats64 = qeth_get_stats64, .ndo_start_xmit = qeth_l3_hard_start_xmit, .ndo_features_check = qeth_l3_osa_features_check, - .ndo_select_queue = qeth_l3_osa_select_queue, + .ndo_select_queue = qeth_osa_select_queue, .ndo_validate_addr = eth_validate_addr, .ndo_set_rx_mode = qeth_l3_set_rx_mode, .ndo_eth_ioctl = qeth_do_ioctl, diff --git a/drivers/staging/rtl8192e/rtllib_softmac.c b/drivers/staging/rtl8192e/rtllib_softmac.c index d2726d01c757..aabbea48223d 100644 --- a/drivers/staging/rtl8192e/rtllib_softmac.c +++ b/drivers/staging/rtl8192e/rtllib_softmac.c @@ -2515,7 +2515,7 @@ void rtllib_stop_all_queues(struct rtllib_device *ieee) unsigned int i; for (i = 0; i < ieee->dev->num_tx_queues; i++) - netdev_get_tx_queue(ieee->dev, i)->trans_start = jiffies; + txq_trans_cond_update(netdev_get_tx_queue(ieee->dev, i)); netif_tx_stop_all_queues(ieee->dev); } diff --git a/include/asm-generic/sections.h b/include/asm-generic/sections.h index 1dfadb2e878d..76a0f16e56cf 100644 --- a/include/asm-generic/sections.h +++ b/include/asm-generic/sections.h @@ -130,18 +130,24 @@ static inline bool init_section_intersects(void *virt, size_t size) /** * is_kernel_core_data - checks if the pointer address is located in the - * .data section + * .data or .bss section * * @addr: address to check * - * Returns: true if the address is located in .data, false otherwise. + * Returns: true if the address is located in .data or .bss, false otherwise. * Note: On some archs it may return true for core RODATA, and false * for others. But will always be true for core RW data. */ static inline bool is_kernel_core_data(unsigned long addr) { - return addr >= (unsigned long)_sdata && - addr < (unsigned long)_edata; + if (addr >= (unsigned long)_sdata && addr < (unsigned long)_edata) + return true; + + if (addr >= (unsigned long)__bss_start && + addr < (unsigned long)__bss_stop) + return true; + + return false; } /** diff --git a/include/linux/bpf.h b/include/linux/bpf.h index 755f38e893be..72270f9c588e 100644 --- a/include/linux/bpf.h +++ b/include/linux/bpf.h @@ -2153,6 +2153,7 @@ extern const struct bpf_func_proto bpf_btf_find_by_name_kind_proto; extern const struct bpf_func_proto bpf_sk_setsockopt_proto; extern const struct bpf_func_proto bpf_sk_getsockopt_proto; extern const struct bpf_func_proto bpf_kallsyms_lookup_name_proto; +extern const struct bpf_func_proto bpf_find_vma_proto; const struct bpf_func_proto *tracing_prog_func_proto( enum bpf_func_id func_id, const struct bpf_prog *prog); diff --git a/include/linux/btf_ids.h b/include/linux/btf_ids.h index 47d9abfbdb55..919c0fde1c51 100644 --- a/include/linux/btf_ids.h +++ b/include/linux/btf_ids.h @@ -73,7 +73,7 @@ asm( \ __BTF_ID_LIST(name, local) \ extern u32 name[]; -#define BTF_ID_LIST_GLOBAL(name) \ +#define BTF_ID_LIST_GLOBAL(name, n) \ __BTF_ID_LIST(name, globl) /* The BTF_ID_LIST_SINGLE macro defines a BTF_ID_LIST with @@ -83,7 +83,7 @@ __BTF_ID_LIST(name, globl) BTF_ID_LIST(name) \ BTF_ID(prefix, typename) #define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) \ - BTF_ID_LIST_GLOBAL(name) \ + BTF_ID_LIST_GLOBAL(name, 1) \ BTF_ID(prefix, typename) /* @@ -149,7 +149,7 @@ extern struct btf_id_set name; #define BTF_ID_LIST(name) static u32 name[5]; #define BTF_ID(prefix, name) #define BTF_ID_UNUSED -#define BTF_ID_LIST_GLOBAL(name) u32 name[1]; +#define BTF_ID_LIST_GLOBAL(name, n) u32 name[n]; #define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 name[1]; #define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) u32 name[1]; #define BTF_SET_START(name) static struct btf_id_set name = { 0 }; @@ -189,6 +189,18 @@ MAX_BTF_SOCK_TYPE, extern u32 btf_sock_ids[]; #endif -extern u32 btf_task_struct_ids[]; +#define BTF_TRACING_TYPE_xxx \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_TASK, task_struct) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_FILE, file) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_VMA, vm_area_struct) + +enum { +#define BTF_TRACING_TYPE(name, type) name, +BTF_TRACING_TYPE_xxx +#undef BTF_TRACING_TYPE +MAX_BTF_TRACING_TYPE, +}; + +extern u32 btf_tracing_ids[]; #endif diff --git a/include/linux/can/bittiming.h b/include/linux/can/bittiming.h index 20b50baf3a02..a81652d1c6f3 100644 --- a/include/linux/can/bittiming.h +++ b/include/linux/can/bittiming.h @@ -12,13 +12,6 @@ #define CAN_SYNC_SEG 1 -/* Kilobits and Megabits per second */ -#define CAN_KBPS 1000UL -#define CAN_MBPS 1000000UL - -/* Megahertz */ -#define CAN_MHZ 1000000UL - #define CAN_CTRLMODE_TDC_MASK \ (CAN_CTRLMODE_TDC_AUTO | CAN_CTRLMODE_TDC_MANUAL) diff --git a/include/linux/dsa/8021q.h b/include/linux/dsa/8021q.h index 254b165f2b44..939a1beaddf7 100644 --- a/include/linux/dsa/8021q.h +++ b/include/linux/dsa/8021q.h @@ -7,6 +7,7 @@ #include <linux/refcount.h> #include <linux/types.h> +#include <net/dsa.h> struct dsa_switch; struct dsa_port; @@ -37,14 +38,12 @@ struct sk_buff *dsa_8021q_xmit(struct sk_buff *skb, struct net_device *netdev, void dsa_8021q_rcv(struct sk_buff *skb, int *source_port, int *switch_id); int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num); + struct dsa_bridge bridge); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num); + struct dsa_bridge bridge); -u16 dsa_8021q_bridge_tx_fwd_offload_vid(int bridge_num); +u16 dsa_8021q_bridge_tx_fwd_offload_vid(unsigned int bridge_num); u16 dsa_tag_8021q_tx_vid(const struct dsa_port *dp); diff --git a/include/linux/ethtool.h b/include/linux/ethtool.h index 845a0ffc16ee..a26f37a27167 100644 --- a/include/linux/ethtool.h +++ b/include/linux/ethtool.h @@ -67,6 +67,22 @@ enum { ETH_RSS_HASH_FUNCS_COUNT }; +/** + * struct kernel_ethtool_ringparam - RX/TX ring configuration + * @rx_buf_len: Current length of buffers on the rx ring. + */ +struct kernel_ethtool_ringparam { + u32 rx_buf_len; +}; + +/** + * enum ethtool_supported_ring_param - indicator caps for setting ring params + * @ETHTOOL_RING_USE_RX_BUF_LEN: capture for setting rx_buf_len + */ +enum ethtool_supported_ring_param { + ETHTOOL_RING_USE_RX_BUF_LEN = BIT(0), +}; + #define __ETH_RSS_HASH_BIT(bit) ((u32)1 << (bit)) #define __ETH_RSS_HASH(name) __ETH_RSS_HASH_BIT(ETH_RSS_HASH_##name##_BIT) @@ -432,6 +448,7 @@ struct ethtool_module_power_mode_params { * @cap_link_lanes_supported: indicates if the driver supports lanes * parameter. * @supported_coalesce_params: supported types of interrupt coalescing. + * @supported_ring_params: supported ring params. * @get_drvinfo: Report driver/device information. Should only set the * @driver, @version, @fw_version and @bus_info fields. If not * implemented, the @driver and @bus_info fields will be filled in @@ -613,6 +630,7 @@ struct ethtool_module_power_mode_params { struct ethtool_ops { u32 cap_link_lanes_supported:1; u32 supported_coalesce_params; + u32 supported_ring_params; void (*get_drvinfo)(struct net_device *, struct ethtool_drvinfo *); int (*get_regs_len)(struct net_device *); void (*get_regs)(struct net_device *, struct ethtool_regs *, void *); @@ -638,9 +656,13 @@ struct ethtool_ops { struct kernel_ethtool_coalesce *, struct netlink_ext_ack *); void (*get_ringparam)(struct net_device *, - struct ethtool_ringparam *); + struct ethtool_ringparam *, + struct kernel_ethtool_ringparam *, + struct netlink_ext_ack *); int (*set_ringparam)(struct net_device *, - struct ethtool_ringparam *); + struct ethtool_ringparam *, + struct kernel_ethtool_ringparam *, + struct netlink_ext_ack *); void (*get_pause_stats)(struct net_device *dev, struct ethtool_pause_stats *pause_stats); void (*get_pauseparam)(struct net_device *, diff --git a/include/linux/filter.h b/include/linux/filter.h index 7f1e88e3e2b5..68c6b5c208e7 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1371,6 +1371,7 @@ struct bpf_sk_lookup_kern { const struct in6_addr *daddr; } v6; struct sock *selected_sk; + u32 ingress_ifindex; bool no_reuseport; }; @@ -1433,7 +1434,7 @@ extern struct static_key_false bpf_sk_lookup_enabled; static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol, const __be32 saddr, const __be16 sport, const __be32 daddr, const u16 dport, - struct sock **psk) + const int ifindex, struct sock **psk) { struct bpf_prog_array *run_array; struct sock *selected_sk = NULL; @@ -1449,6 +1450,7 @@ static inline bool bpf_sk_lookup_run_v4(struct net *net, int protocol, .v4.daddr = daddr, .sport = sport, .dport = dport, + .ingress_ifindex = ifindex, }; u32 act; @@ -1471,7 +1473,7 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol, const __be16 sport, const struct in6_addr *daddr, const u16 dport, - struct sock **psk) + const int ifindex, struct sock **psk) { struct bpf_prog_array *run_array; struct sock *selected_sk = NULL; @@ -1487,6 +1489,7 @@ static inline bool bpf_sk_lookup_run_v6(struct net *net, int protocol, .v6.daddr = daddr, .sport = sport, .dport = dport, + .ingress_ifindex = ifindex, }; u32 act; diff --git a/include/linux/if_eql.h b/include/linux/if_eql.h index d984694c384d..d75601d613cc 100644 --- a/include/linux/if_eql.h +++ b/include/linux/if_eql.h @@ -26,6 +26,7 @@ typedef struct slave { struct list_head list; struct net_device *dev; + netdevice_tracker dev_tracker; long priority; long priority_bps; long priority_Bps; diff --git a/include/linux/if_vlan.h b/include/linux/if_vlan.h index 41a518336673..8420fe504927 100644 --- a/include/linux/if_vlan.h +++ b/include/linux/if_vlan.h @@ -162,6 +162,7 @@ struct netpoll; * @vlan_id: VLAN identifier * @flags: device flags * @real_dev: underlying netdevice + * @dev_tracker: refcount tracker for @real_dev reference * @real_dev_addr: address of underlying netdevice * @dent: proc dir entry * @vlan_pcpu_stats: ptr to percpu rx stats @@ -177,6 +178,8 @@ struct vlan_dev_priv { u16 flags; struct net_device *real_dev; + netdevice_tracker dev_tracker; + unsigned char real_dev_addr[ETH_ALEN]; struct proc_dir_entry *dent; diff --git a/include/linux/inetdevice.h b/include/linux/inetdevice.h index 518b484a7f07..674aeead6260 100644 --- a/include/linux/inetdevice.h +++ b/include/linux/inetdevice.h @@ -24,6 +24,8 @@ struct ipv4_devconf { struct in_device { struct net_device *dev; + netdevice_tracker dev_tracker; + refcount_t refcnt; int dead; struct in_ifaddr __rcu *ifa_list;/* IP ifaddr chain */ diff --git a/include/linux/mdio/mdio-mscc-miim.h b/include/linux/mdio/mdio-mscc-miim.h new file mode 100644 index 000000000000..5b4ed2c3cbb9 --- /dev/null +++ b/include/linux/mdio/mdio-mscc-miim.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Driver for the MDIO interface of Microsemi network switches. + * + * Author: Colin Foster <colin.foster@in-advantage.com> + * Copyright (C) 2021 Innovative Advantage + */ +#ifndef MDIO_MSCC_MIIM_H +#define MDIO_MSCC_MIIM_H + +#include <linux/device.h> +#include <linux/phy.h> +#include <linux/regmap.h> + +int mscc_miim_setup(struct device *device, struct mii_bus **bus, + const char *name, struct regmap *mii_regmap, + int status_offset); + +#endif diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index c6786c12b207..df1fab44ea5c 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -117,4 +117,7 @@ int mei_cldev_enable(struct mei_cl_device *cldev); int mei_cldev_disable(struct mei_cl_device *cldev); bool mei_cldev_enabled(const struct mei_cl_device *cldev); +void *mei_cldev_dma_map(struct mei_cl_device *cldev, u8 buffer_id, size_t size); +int mei_cldev_dma_unmap(struct mei_cl_device *cldev); + #endif /* _LINUX_MEI_CL_BUS_H */ diff --git a/include/linux/mlx5/driver.h b/include/linux/mlx5/driver.h index a623ec635947..78655d8d13a7 100644 --- a/include/linux/mlx5/driver.h +++ b/include/linux/mlx5/driver.h @@ -478,6 +478,10 @@ struct mlx5_fc_stats { unsigned long next_query; unsigned long sampling_interval; /* jiffies */ u32 *bulk_query_out; + int bulk_query_len; + size_t num_counters; + bool bulk_query_alloc_failed; + unsigned long next_bulk_query_alloc; struct mlx5_fc_pool fc_pool; }; diff --git a/include/linux/mmc/sdio_ids.h b/include/linux/mmc/sdio_ids.h index a85c9f0bd470..53f0efa0bccf 100644 --- a/include/linux/mmc/sdio_ids.h +++ b/include/linux/mmc/sdio_ids.h @@ -105,6 +105,7 @@ #define SDIO_VENDOR_ID_MEDIATEK 0x037a #define SDIO_DEVICE_ID_MEDIATEK_MT7663 0x7663 #define SDIO_DEVICE_ID_MEDIATEK_MT7668 0x7668 +#define SDIO_DEVICE_ID_MEDIATEK_MT7961 0x7961 #define SDIO_VENDOR_ID_MICROCHIP_WILC 0x0296 #define SDIO_DEVICE_ID_MICROCHIP_WILC1000 0x5347 diff --git a/include/linux/mroute_base.h b/include/linux/mroute_base.h index 8071148f29a6..e05ee9f001ff 100644 --- a/include/linux/mroute_base.h +++ b/include/linux/mroute_base.h @@ -12,6 +12,7 @@ /** * struct vif_device - interface representor for multicast routing * @dev: network device being used + * @dev_tracker: refcount tracker for @dev reference * @bytes_in: statistic; bytes ingressing * @bytes_out: statistic; bytes egresing * @pkt_in: statistic; packets ingressing @@ -26,6 +27,7 @@ */ struct vif_device { struct net_device *dev; + netdevice_tracker dev_tracker; unsigned long bytes_in, bytes_out; unsigned long pkt_in, pkt_out; unsigned long rate_limit; diff --git a/include/linux/net/intel/iidc.h b/include/linux/net/intel/iidc.h index e32f6712aee0..1289593411d3 100644 --- a/include/linux/net/intel/iidc.h +++ b/include/linux/net/intel/iidc.h @@ -26,6 +26,11 @@ enum iidc_reset_type { IIDC_GLOBR, }; +enum iidc_rdma_protocol { + IIDC_RDMA_PROTOCOL_IWARP = BIT(0), + IIDC_RDMA_PROTOCOL_ROCEV2 = BIT(1), +}; + #define IIDC_MAX_USER_PRIORITY 8 /* Struct to hold per RDMA Qset info */ @@ -70,8 +75,6 @@ int ice_rdma_request_reset(struct ice_pf *pf, enum iidc_reset_type reset_type); int ice_rdma_update_vsi_filter(struct ice_pf *pf, u16 vsi_id, bool enable); void ice_get_qos_params(struct ice_pf *pf, struct iidc_qos_params *qos); -#define IIDC_RDMA_ROCE_NAME "roce" - /* Structure representing auxiliary driver tailored information about the core * PCI dev, each auxiliary driver using the IIDC interface will have an * instance of this struct dedicated to it. diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index be5cb3360b94..1a748ee9a421 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -48,6 +48,7 @@ #include <uapi/linux/pkt_cls.h> #include <linux/hashtable.h> #include <linux/rbtree.h> +#include <linux/ref_tracker.h> struct netpoll_info; struct device; @@ -300,6 +301,12 @@ enum netdev_state_t { }; +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER +typedef struct ref_tracker *netdevice_tracker; +#else +typedef struct {} netdevice_tracker; +#endif + struct gro_list { struct list_head list; int count; @@ -579,6 +586,8 @@ struct netdev_queue { * read-mostly part */ struct net_device *dev; + netdevice_tracker dev_tracker; + struct Qdisc __rcu *qdisc; struct Qdisc *qdisc_sleeping; #ifdef CONFIG_SYSFS @@ -592,7 +601,7 @@ struct netdev_queue { * Number of TX timeouts for this queue * (/sys/class/net/DEV/Q/trans_timeout) */ - unsigned long trans_timeout; + atomic_long_t trans_timeout; /* Subordinate device that the queue has been assigned to */ struct net_device *sb_dev; @@ -734,6 +743,8 @@ struct netdev_rx_queue { #endif struct kobject kobj; struct net_device *dev; + netdevice_tracker dev_tracker; + #ifdef CONFIG_XDP_SOCKETS struct xsk_buff_pool *pool; #endif @@ -1297,11 +1308,6 @@ struct netdev_net_notifier { * TX queue. * int (*ndo_get_iflink)(const struct net_device *dev); * Called to get the iflink value of this device. - * void (*ndo_change_proto_down)(struct net_device *dev, - * bool proto_down); - * This function is used to pass protocol port error state information - * to the switch driver. The switch driver can react to the proto_down - * by doing a phys down on the associated switch port. * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb); * This function is used to get egress tunnel information for given skb. * This is useful for retrieving outer tunnel header parameters while @@ -1542,8 +1548,6 @@ struct net_device_ops { int queue_index, u32 maxrate); int (*ndo_get_iflink)(const struct net_device *dev); - int (*ndo_change_proto_down)(struct net_device *dev, - bool proto_down); int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb); void (*ndo_set_rx_headroom)(struct net_device *dev, @@ -1612,6 +1616,7 @@ struct net_device_ops { * @IFF_LIVE_RENAME_OK: rename is allowed while device is up and running * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with * skb_headlen(skb) == 0 (data starts from frag0) + * @IFF_CHANGE_PROTO_DOWN: device supports setting carrier via IFLA_PROTO_DOWN */ enum netdev_priv_flags { IFF_802_1Q_VLAN = 1<<0, @@ -1646,6 +1651,7 @@ enum netdev_priv_flags { IFF_L3MDEV_RX_HANDLER = 1<<29, IFF_LIVE_RENAME_OK = 1<<30, IFF_TX_SKB_NO_LINEAR = 1<<31, + IFF_CHANGE_PROTO_DOWN = BIT_ULL(32), }; #define IFF_802_1Q_VLAN IFF_802_1Q_VLAN @@ -1870,6 +1876,7 @@ enum netdev_ml_priv_type { * @proto_down_reason: reason a netdev interface is held down * @pcpu_refcnt: Number of references to this device * @dev_refcnt: Number of references to this device + * @refcnt_tracker: Tracker directory for tracked references to this device * @todo_list: Delayed register/unregister * @link_watch_list: XXX: need comments on this one * @@ -1942,6 +1949,10 @@ enum netdev_ml_priv_type { * @unlink_list: As netif_addr_lock() can be called recursively, * keep a list of interfaces to be deleted. * + * @dev_addr_shadow: Copy of @dev_addr to catch direct writes. + * @linkwatch_dev_tracker: refcount tracker used by linkwatch. + * @watchdog_dev_tracker: refcount tracker used by watchdog. + * * FIXME: cleanup struct net_device such that network protocol info * moves out. */ @@ -1980,7 +1991,7 @@ struct net_device { /* Read-mostly cache-line for fast-path access */ unsigned int flags; - unsigned int priv_flags; + unsigned long long priv_flags; const struct net_device_ops *netdev_ops; int ifindex; unsigned short gflags; @@ -2117,7 +2128,7 @@ struct net_device { * Cache lines mostly used on receive path (including eth_type_trans()) */ /* Interface address info used in eth_type_trans() */ - unsigned char *dev_addr; + const unsigned char *dev_addr; struct netdev_rx_queue *_rx; unsigned int num_rx_queues; @@ -2181,6 +2192,7 @@ struct net_device { #else refcount_t dev_refcnt; #endif + struct ref_tracker_dir refcnt_tracker; struct list_head link_watch_list; @@ -2268,6 +2280,10 @@ struct net_device { /* protected by rtnl_lock */ struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE]; + + u8 dev_addr_shadow[MAX_ADDR_LEN]; + netdevice_tracker linkwatch_dev_tracker; + netdevice_tracker watchdog_dev_tracker; }; #define to_net_dev(d) container_of(d, struct net_device, dev) @@ -2520,109 +2536,6 @@ static inline void netif_napi_del(struct napi_struct *napi) synchronize_net(); } -struct napi_gro_cb { - /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ - void *frag0; - - /* Length of frag0. */ - unsigned int frag0_len; - - /* This indicates where we are processing relative to skb->data. */ - int data_offset; - - /* This is non-zero if the packet cannot be merged with the new skb. */ - u16 flush; - - /* Save the IP ID here and check when we get to the transport layer */ - u16 flush_id; - - /* Number of segments aggregated. */ - u16 count; - - /* Start offset for remote checksum offload */ - u16 gro_remcsum_start; - - /* jiffies when first packet was created/queued */ - unsigned long age; - - /* Used in ipv6_gro_receive() and foo-over-udp */ - u16 proto; - - /* This is non-zero if the packet may be of the same flow. */ - u8 same_flow:1; - - /* Used in tunnel GRO receive */ - u8 encap_mark:1; - - /* GRO checksum is valid */ - u8 csum_valid:1; - - /* Number of checksums via CHECKSUM_UNNECESSARY */ - u8 csum_cnt:3; - - /* Free the skb? */ - u8 free:2; -#define NAPI_GRO_FREE 1 -#define NAPI_GRO_FREE_STOLEN_HEAD 2 - - /* Used in foo-over-udp, set in udp[46]_gro_receive */ - u8 is_ipv6:1; - - /* Used in GRE, set in fou/gue_gro_receive */ - u8 is_fou:1; - - /* Used to determine if flush_id can be ignored */ - u8 is_atomic:1; - - /* Number of gro_receive callbacks this packet already went through */ - u8 recursion_counter:4; - - /* GRO is done by frag_list pointer chaining. */ - u8 is_flist:1; - - /* used to support CHECKSUM_COMPLETE for tunneling protocols */ - __wsum csum; - - /* used in skb_gro_receive() slow path */ - struct sk_buff *last; -}; - -#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) - -#define GRO_RECURSION_LIMIT 15 -static inline int gro_recursion_inc_test(struct sk_buff *skb) -{ - return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; -} - -typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); -static inline struct sk_buff *call_gro_receive(gro_receive_t cb, - struct list_head *head, - struct sk_buff *skb) -{ - if (unlikely(gro_recursion_inc_test(skb))) { - NAPI_GRO_CB(skb)->flush |= 1; - return NULL; - } - - return cb(head, skb); -} - -typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, - struct sk_buff *); -static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, - struct sock *sk, - struct list_head *head, - struct sk_buff *skb) -{ - if (unlikely(gro_recursion_inc_test(skb))) { - NAPI_GRO_CB(skb)->flush |= 1; - return NULL; - } - - return cb(sk, head, skb); -} - struct packet_type { __be16 type; /* This is really htons(ether_type). */ bool ignore_outgoing; @@ -3005,254 +2918,7 @@ struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex); struct net_device *dev_get_by_napi_id(unsigned int napi_id); int netdev_get_name(struct net *net, char *name, int ifindex); int dev_restart(struct net_device *dev); -int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); -int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb); - -static inline unsigned int skb_gro_offset(const struct sk_buff *skb) -{ - return NAPI_GRO_CB(skb)->data_offset; -} - -static inline unsigned int skb_gro_len(const struct sk_buff *skb) -{ - return skb->len - NAPI_GRO_CB(skb)->data_offset; -} -static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) -{ - NAPI_GRO_CB(skb)->data_offset += len; -} - -static inline void *skb_gro_header_fast(struct sk_buff *skb, - unsigned int offset) -{ - return NAPI_GRO_CB(skb)->frag0 + offset; -} - -static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) -{ - return NAPI_GRO_CB(skb)->frag0_len < hlen; -} - -static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) -{ - NAPI_GRO_CB(skb)->frag0 = NULL; - NAPI_GRO_CB(skb)->frag0_len = 0; -} - -static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, - unsigned int offset) -{ - if (!pskb_may_pull(skb, hlen)) - return NULL; - - skb_gro_frag0_invalidate(skb); - return skb->data + offset; -} - -static inline void *skb_gro_network_header(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + - skb_network_offset(skb); -} - -static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, - const void *start, unsigned int len) -{ - if (NAPI_GRO_CB(skb)->csum_valid) - NAPI_GRO_CB(skb)->csum = csum_sub(NAPI_GRO_CB(skb)->csum, - csum_partial(start, len, 0)); -} - -/* GRO checksum functions. These are logical equivalents of the normal - * checksum functions (in skbuff.h) except that they operate on the GRO - * offsets and fields in sk_buff. - */ - -__sum16 __skb_gro_checksum_complete(struct sk_buff *skb); - -static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); -} - -static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, - bool zero_okay, - __sum16 check) -{ - return ((skb->ip_summed != CHECKSUM_PARTIAL || - skb_checksum_start_offset(skb) < - skb_gro_offset(skb)) && - !skb_at_gro_remcsum_start(skb) && - NAPI_GRO_CB(skb)->csum_cnt == 0 && - (!zero_okay || check)); -} - -static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, - __wsum psum) -{ - if (NAPI_GRO_CB(skb)->csum_valid && - !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) - return 0; - - NAPI_GRO_CB(skb)->csum = psum; - - return __skb_gro_checksum_complete(skb); -} - -static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) -{ - if (NAPI_GRO_CB(skb)->csum_cnt > 0) { - /* Consume a checksum from CHECKSUM_UNNECESSARY */ - NAPI_GRO_CB(skb)->csum_cnt--; - } else { - /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we - * verified a new top level checksum or an encapsulated one - * during GRO. This saves work if we fallback to normal path. - */ - __skb_incr_checksum_unnecessary(skb); - } -} - -#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ - compute_pseudo) \ -({ \ - __sum16 __ret = 0; \ - if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ - __ret = __skb_gro_checksum_validate_complete(skb, \ - compute_pseudo(skb, proto)); \ - if (!__ret) \ - skb_gro_incr_csum_unnecessary(skb); \ - __ret; \ -}) - -#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ - __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) - -#define skb_gro_checksum_validate_zero_check(skb, proto, check, \ - compute_pseudo) \ - __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) - -#define skb_gro_checksum_simple_validate(skb) \ - __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) - -static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) -{ - return (NAPI_GRO_CB(skb)->csum_cnt == 0 && - !NAPI_GRO_CB(skb)->csum_valid); -} - -static inline void __skb_gro_checksum_convert(struct sk_buff *skb, - __wsum pseudo) -{ - NAPI_GRO_CB(skb)->csum = ~pseudo; - NAPI_GRO_CB(skb)->csum_valid = 1; -} - -#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ -do { \ - if (__skb_gro_checksum_convert_check(skb)) \ - __skb_gro_checksum_convert(skb, \ - compute_pseudo(skb, proto)); \ -} while (0) - -struct gro_remcsum { - int offset; - __wsum delta; -}; - -static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) -{ - grc->offset = 0; - grc->delta = 0; -} - -static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, - unsigned int off, size_t hdrlen, - int start, int offset, - struct gro_remcsum *grc, - bool nopartial) -{ - __wsum delta; - size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); - - BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); - - if (!nopartial) { - NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; - return ptr; - } - - ptr = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, off + plen)) { - ptr = skb_gro_header_slow(skb, off + plen, off); - if (!ptr) - return NULL; - } - - delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, - start, offset); - - /* Adjust skb->csum since we changed the packet */ - NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); - - grc->offset = off + hdrlen + offset; - grc->delta = delta; - - return ptr; -} - -static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, - struct gro_remcsum *grc) -{ - void *ptr; - size_t plen = grc->offset + sizeof(u16); - - if (!grc->delta) - return; - - ptr = skb_gro_header_fast(skb, grc->offset); - if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { - ptr = skb_gro_header_slow(skb, plen, grc->offset); - if (!ptr) - return; - } - - remcsum_unadjust((__sum16 *)ptr, grc->delta); -} - -#ifdef CONFIG_XFRM_OFFLOAD -static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) -{ - if (PTR_ERR(pp) != -EINPROGRESS) - NAPI_GRO_CB(skb)->flush |= flush; -} -static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, - struct sk_buff *pp, - int flush, - struct gro_remcsum *grc) -{ - if (PTR_ERR(pp) != -EINPROGRESS) { - NAPI_GRO_CB(skb)->flush |= flush; - skb_gro_remcsum_cleanup(skb, grc); - skb->remcsum_offload = 0; - } -} -#else -static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) -{ - NAPI_GRO_CB(skb)->flush |= flush; -} -static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, - struct sk_buff *pp, - int flush, - struct gro_remcsum *grc) -{ - NAPI_GRO_CB(skb)->flush |= flush; - skb_gro_remcsum_cleanup(skb, grc); - skb->remcsum_offload = 0; -} -#endif static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev, unsigned short type, @@ -4007,6 +3673,7 @@ int netif_rx_ni(struct sk_buff *skb); int netif_rx_any_context(struct sk_buff *skb); int netif_receive_skb(struct sk_buff *skb); int netif_receive_skb_core(struct sk_buff *skb); +void netif_receive_skb_list_internal(struct list_head *head); void netif_receive_skb_list(struct list_head *head); gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb); void napi_gro_flush(struct napi_struct *napi, bool flush_old); @@ -4080,7 +3747,6 @@ int dev_get_port_parent_id(struct net_device *dev, struct netdev_phys_item_id *ppid, bool recurse); bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b); int dev_change_proto_down(struct net_device *dev, bool proto_down); -int dev_change_proto_down_generic(struct net_device *dev, bool proto_down); void dev_change_proto_down_reason(struct net_device *dev, unsigned long mask, u32 value); struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again); @@ -4156,6 +3822,7 @@ void netdev_run_todo(void); * @dev: network device * * Release reference to device to allow it to be freed. + * Try using dev_put_track() instead. */ static inline void dev_put(struct net_device *dev) { @@ -4173,6 +3840,7 @@ static inline void dev_put(struct net_device *dev) * @dev: network device * * Hold reference to device to keep it from being freed. + * Try using dev_hold_track() instead. */ static inline void dev_hold(struct net_device *dev) { @@ -4185,6 +3853,57 @@ static inline void dev_hold(struct net_device *dev) } } +static inline void netdev_tracker_alloc(struct net_device *dev, + netdevice_tracker *tracker, gfp_t gfp) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp); +#endif +} + +static inline void netdev_tracker_free(struct net_device *dev, + netdevice_tracker *tracker) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + ref_tracker_free(&dev->refcnt_tracker, tracker); +#endif +} + +static inline void dev_hold_track(struct net_device *dev, + netdevice_tracker *tracker, gfp_t gfp) +{ + if (dev) { + dev_hold(dev); + netdev_tracker_alloc(dev, tracker, gfp); + } +} + +static inline void dev_put_track(struct net_device *dev, + netdevice_tracker *tracker) +{ + if (dev) { + netdev_tracker_free(dev, tracker); + dev_put(dev); + } +} + +static inline void dev_replace_track(struct net_device *odev, + struct net_device *ndev, + netdevice_tracker *tracker, + gfp_t gfp) +{ +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + if (odev) + ref_tracker_free(&odev->refcnt_tracker, tracker); +#endif + dev_hold(ndev); + dev_put(odev); +#ifdef CONFIG_NET_DEV_REFCNT_TRACKER + if (ndev) + ref_tracker_alloc(&ndev->refcnt_tracker, tracker, gfp); +#endif +} + /* Carrier loss detection, dial on demand. The functions netif_carrier_on * and _off may be called from IRQ context, but it is caller * who is responsible for serialization of these calls. @@ -4451,10 +4170,21 @@ static inline void __netif_tx_unlock_bh(struct netdev_queue *txq) spin_unlock_bh(&txq->_xmit_lock); } +/* + * txq->trans_start can be read locklessly from dev_watchdog() + */ static inline void txq_trans_update(struct netdev_queue *txq) { if (txq->xmit_lock_owner != -1) - txq->trans_start = jiffies; + WRITE_ONCE(txq->trans_start, jiffies); +} + +static inline void txq_trans_cond_update(struct netdev_queue *txq) +{ + unsigned long now = jiffies; + + if (READ_ONCE(txq->trans_start) != now) + WRITE_ONCE(txq->trans_start, now); } /* legacy drivers only, netdev_start_xmit() sets txq->trans_start */ @@ -4462,8 +4192,7 @@ static inline void netif_trans_update(struct net_device *dev) { struct netdev_queue *txq = netdev_get_tx_queue(dev, 0); - if (txq->trans_start != jiffies) - txq->trans_start = jiffies; + txq_trans_cond_update(txq); } /** @@ -4472,27 +4201,7 @@ static inline void netif_trans_update(struct net_device *dev) * * Get network device transmit lock */ -static inline void netif_tx_lock(struct net_device *dev) -{ - unsigned int i; - int cpu; - - spin_lock(&dev->tx_global_lock); - cpu = smp_processor_id(); - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - - /* We are the only thread of execution doing a - * freeze, but we have to grab the _xmit_lock in - * order to synchronize with threads which are in - * the ->hard_start_xmit() handler and already - * checked the frozen bit. - */ - __netif_tx_lock(txq, cpu); - set_bit(__QUEUE_STATE_FROZEN, &txq->state); - __netif_tx_unlock(txq); - } -} +void netif_tx_lock(struct net_device *dev); static inline void netif_tx_lock_bh(struct net_device *dev) { @@ -4500,22 +4209,7 @@ static inline void netif_tx_lock_bh(struct net_device *dev) netif_tx_lock(dev); } -static inline void netif_tx_unlock(struct net_device *dev) -{ - unsigned int i; - - for (i = 0; i < dev->num_tx_queues; i++) { - struct netdev_queue *txq = netdev_get_tx_queue(dev, i); - - /* No need to grab the _xmit_lock here. If the - * queue is not stopped for another reason, we - * force a schedule. - */ - clear_bit(__QUEUE_STATE_FROZEN, &txq->state); - netif_schedule_queue(txq); - } - spin_unlock(&dev->tx_global_lock); -} +void netif_tx_unlock(struct net_device *dev); static inline void netif_tx_unlock_bh(struct net_device *dev) { @@ -4649,10 +4343,13 @@ void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list, void __hw_addr_init(struct netdev_hw_addr_list *list); /* Functions used for device addresses handling */ +void dev_addr_mod(struct net_device *dev, unsigned int offset, + const void *addr, size_t len); + static inline void __dev_addr_set(struct net_device *dev, const void *addr, size_t len) { - memcpy(dev->dev_addr, addr, len); + dev_addr_mod(dev, 0, addr, len); } static inline void dev_addr_set(struct net_device *dev, const u8 *addr) @@ -4660,19 +4357,13 @@ static inline void dev_addr_set(struct net_device *dev, const u8 *addr) __dev_addr_set(dev, addr, dev->addr_len); } -static inline void -dev_addr_mod(struct net_device *dev, unsigned int offset, - const void *addr, size_t len) -{ - memcpy(&dev->dev_addr[offset], addr, len); -} - int dev_addr_add(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); int dev_addr_del(struct net_device *dev, const unsigned char *addr, unsigned char addr_type); void dev_addr_flush(struct net_device *dev); int dev_addr_init(struct net_device *dev); +void dev_addr_check(struct net_device *dev); /* Functions used for unicast addresses handling */ int dev_uc_add(struct net_device *dev, const unsigned char *addr); @@ -5111,7 +4802,15 @@ static inline bool netif_needs_gso(struct sk_buff *skb, static inline void netif_set_gso_max_size(struct net_device *dev, unsigned int size) { - dev->gso_max_size = size; + /* dev->gso_max_size is read locklessly from sk_setup_caps() */ + WRITE_ONCE(dev->gso_max_size, size); +} + +static inline void netif_set_gso_max_segs(struct net_device *dev, + unsigned int segs) +{ + /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ + WRITE_ONCE(dev->gso_max_segs, segs); } static inline void skb_gso_error_unwind(struct sk_buff *skb, __be16 protocol, @@ -5298,7 +4997,7 @@ void netdev_info(const struct net_device *dev, const char *format, ...); #define netdev_level_once(level, dev, fmt, ...) \ do { \ - static bool __print_once __read_mostly; \ + static bool __section(".data.once") __print_once; \ \ if (!__print_once) { \ __print_once = true; \ diff --git a/include/linux/netpoll.h b/include/linux/netpoll.h index e6a2d72e0dc7..bd19c4b91e31 100644 --- a/include/linux/netpoll.h +++ b/include/linux/netpoll.h @@ -24,6 +24,7 @@ union inet_addr { struct netpoll { struct net_device *dev; + netdevice_tracker dev_tracker; char dev_name[IFNAMSIZ]; const char *name; diff --git a/include/linux/once.h b/include/linux/once.h index d361fb14ac3a..f54523052bbc 100644 --- a/include/linux/once.h +++ b/include/linux/once.h @@ -38,7 +38,7 @@ void __do_once_done(bool *done, struct static_key_true *once_key, #define DO_ONCE(func, ...) \ ({ \ bool ___ret = false; \ - static bool ___done = false; \ + static bool __section(".data.once") ___done = false; \ static DEFINE_STATIC_KEY_TRUE(___once_key); \ if (static_branch_unlikely(&___once_key)) { \ unsigned long ___flags; \ diff --git a/include/linux/phy.h b/include/linux/phy.h index cbf03a5f9cf5..6de8d7a90d78 100644 --- a/include/linux/phy.h +++ b/include/linux/phy.h @@ -99,7 +99,7 @@ extern const int phy_10gbit_features_array[1]; * @PHY_INTERFACE_MODE_RGMII_RXID: RGMII with Internal RX delay * @PHY_INTERFACE_MODE_RGMII_TXID: RGMII with Internal RX delay * @PHY_INTERFACE_MODE_RTBI: Reduced TBI - * @PHY_INTERFACE_MODE_SMII: ??? MII + * @PHY_INTERFACE_MODE_SMII: Serial MII * @PHY_INTERFACE_MODE_XGMII: 10 gigabit media-independent interface * @PHY_INTERFACE_MODE_XLGMII:40 gigabit media-independent interface * @PHY_INTERFACE_MODE_MOCA: Multimedia over Coax diff --git a/include/linux/phylink.h b/include/linux/phylink.h index f037470b6fb3..a2f266cc3442 100644 --- a/include/linux/phylink.h +++ b/include/linux/phylink.h @@ -20,6 +20,29 @@ enum { MLO_AN_PHY = 0, /* Conventional PHY */ MLO_AN_FIXED, /* Fixed-link mode */ MLO_AN_INBAND, /* In-band protocol */ + + MAC_SYM_PAUSE = BIT(0), + MAC_ASYM_PAUSE = BIT(1), + MAC_10HD = BIT(2), + MAC_10FD = BIT(3), + MAC_10 = MAC_10HD | MAC_10FD, + MAC_100HD = BIT(4), + MAC_100FD = BIT(5), + MAC_100 = MAC_100HD | MAC_100FD, + MAC_1000HD = BIT(6), + MAC_1000FD = BIT(7), + MAC_1000 = MAC_1000HD | MAC_1000FD, + MAC_2500FD = BIT(8), + MAC_5000FD = BIT(9), + MAC_10000FD = BIT(10), + MAC_20000FD = BIT(11), + MAC_25000FD = BIT(12), + MAC_40000FD = BIT(13), + MAC_50000FD = BIT(14), + MAC_56000FD = BIT(15), + MAC_100000FD = BIT(16), + MAC_200000FD = BIT(17), + MAC_400000FD = BIT(18), }; static inline bool phylink_autoneg_inband(unsigned int mode) @@ -61,6 +84,8 @@ enum phylink_op_type { * struct phylink_config - PHYLINK configuration structure * @dev: a pointer to a struct device associated with the MAC * @type: operation type of PHYLINK instance + * @legacy_pre_march2020: driver has not been updated for March 2020 updates + * (See commit 7cceb599d15d ("net: phylink: avoid mac_config calls") * @pcs_poll: MAC PCS cannot provide link change interrupt * @poll_fixed_state: if true, starts link_poll, * if MAC link is at %MLO_AN_FIXED mode. @@ -69,16 +94,19 @@ enum phylink_op_type { * if MAC link is at %MLO_AN_FIXED mode. * @supported_interfaces: bitmap describing which PHY_INTERFACE_MODE_xxx * are supported by the MAC/PCS. + * @mac_capabilities: MAC pause/speed/duplex capabilities. */ struct phylink_config { struct device *dev; enum phylink_op_type type; + bool legacy_pre_march2020; bool pcs_poll; bool poll_fixed_state; bool ovr_an_inband; void (*get_fixed_state)(struct phylink_config *config, struct phylink_link_state *state); DECLARE_PHY_INTERFACE_MASK(supported_interfaces); + unsigned long mac_capabilities; }; /** @@ -162,6 +190,10 @@ void validate(struct phylink_config *config, unsigned long *supported, * negotiation completion state in @state->an_complete, and link up state * in @state->link. If possible, @state->lp_advertising should also be * populated. + * + * Note: This is a legacy method. This function will not be called unless + * legacy_pre_march2020 is set in &struct phylink_config and there is no + * PCS attached. */ void mac_pcs_get_state(struct phylink_config *config, struct phylink_link_state *state); @@ -202,6 +234,15 @@ int mac_prepare(struct phylink_config *config, unsigned int mode, * guaranteed to be correct, and so any mac_config() implementation must * never reference these fields. * + * Note: For legacy March 2020 drivers (drivers with legacy_pre_march2020 set + * in their &phylnk_config and which don't have a PCS), this function will be + * called on each link up event, and to also change the in-band advert. For + * non-legacy drivers, it will only be called to reconfigure the MAC for a + * "major" change in e.g. interface mode. It will not be called for changes + * in speed, duplex or pause modes or to change the in-band advertisement. + * In any case, it is strongly preferred that speed, duplex and pause settings + * are handled in the mac_link_up() method and not in this method. + * * (this requires a rewrite - please refer to mac_link_up() for situations * where the PCS and MAC are not tightly integrated.) * @@ -286,6 +327,10 @@ int mac_finish(struct phylink_config *config, unsigned int mode, /** * mac_an_restart() - restart 802.3z BaseX autonegotiation * @config: a pointer to a &struct phylink_config. + * + * Note: This is a legacy method. This function will not be called unless + * legacy_pre_march2020 is set in &struct phylink_config and there is no + * PCS attached. */ void mac_an_restart(struct phylink_config *config); @@ -442,6 +487,12 @@ void pcs_link_up(struct phylink_pcs *pcs, unsigned int mode, phy_interface_t interface, int speed, int duplex); #endif +void phylink_get_linkmodes(unsigned long *linkmodes, phy_interface_t interface, + unsigned long mac_capabilities); +void phylink_generic_validate(struct phylink_config *config, + unsigned long *supported, + struct phylink_link_state *state); + struct phylink *phylink_create(struct phylink_config *, struct fwnode_handle *, phy_interface_t iface, const struct phylink_mac_ops *mac_ops); @@ -496,11 +547,12 @@ void phylink_set_port_modes(unsigned long *bits); void phylink_set_10g_modes(unsigned long *mask); void phylink_helper_basex_speed(struct phylink_link_state *state); +void phylink_mii_c22_pcs_decode_state(struct phylink_link_state *state, + u16 bmsr, u16 lpa); void phylink_mii_c22_pcs_get_state(struct mdio_device *pcs, struct phylink_link_state *state); -int phylink_mii_c22_pcs_set_advertisement(struct mdio_device *pcs, - phy_interface_t interface, - const unsigned long *advertising); +int phylink_mii_c22_pcs_encode_advertisement(phy_interface_t interface, + const unsigned long *advertising); int phylink_mii_c22_pcs_config(struct mdio_device *pcs, unsigned int mode, phy_interface_t interface, const unsigned long *advertising); diff --git a/include/linux/qed/qed_if.h b/include/linux/qed/qed_if.h index 0dae7fcc5ef2..6dc4943d8aec 100644 --- a/include/linux/qed/qed_if.h +++ b/include/linux/qed/qed_if.h @@ -652,6 +652,7 @@ struct qed_dev_info { bool wol_support; bool smart_an; + bool esl; /* MBI version */ u32 mbi_version; @@ -807,6 +808,12 @@ struct qed_devlink { struct devlink_health_reporter *fw_reporter; }; +struct qed_sb_info_dbg { + u32 igu_prod; + u32 igu_cons; + u16 pi[PIS_PER_SB]; +}; + struct qed_common_cb_ops { void (*arfs_filter_op)(void *dev, void *fltr, u8 fw_rc); void (*link_update)(void *dev, struct qed_link_output *link); @@ -1194,6 +1201,13 @@ struct qed_common_ops { struct devlink* (*devlink_register)(struct qed_dev *cdev); void (*devlink_unregister)(struct devlink *devlink); + + __printf(2, 3) void (*mfw_report)(struct qed_dev *cdev, char *fmt, ...); + + int (*get_sb_info)(struct qed_dev *cdev, struct qed_sb_info *sb, + u16 qid, struct qed_sb_info_dbg *sb_dbg); + + int (*get_esl_status)(struct qed_dev *cdev, bool *esl_active); }; #define MASK_FIELD(_name, _value) \ diff --git a/include/linux/ref_tracker.h b/include/linux/ref_tracker.h new file mode 100644 index 000000000000..c11c9db5825c --- /dev/null +++ b/include/linux/ref_tracker.h @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#ifndef _LINUX_REF_TRACKER_H +#define _LINUX_REF_TRACKER_H +#include <linux/refcount.h> +#include <linux/types.h> +#include <linux/spinlock.h> + +struct ref_tracker; + +struct ref_tracker_dir { +#ifdef CONFIG_REF_TRACKER + spinlock_t lock; + unsigned int quarantine_avail; + refcount_t untracked; + struct list_head list; /* List of active trackers */ + struct list_head quarantine; /* List of dead trackers */ +#endif +}; + +#ifdef CONFIG_REF_TRACKER +static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, + unsigned int quarantine_count) +{ + INIT_LIST_HEAD(&dir->list); + INIT_LIST_HEAD(&dir->quarantine); + spin_lock_init(&dir->lock); + dir->quarantine_avail = quarantine_count; + refcount_set(&dir->untracked, 1); +} + +void ref_tracker_dir_exit(struct ref_tracker_dir *dir); + +void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit); + +int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, gfp_t gfp); + +int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp); + +#else /* CONFIG_REF_TRACKER */ + +static inline void ref_tracker_dir_init(struct ref_tracker_dir *dir, + unsigned int quarantine_count) +{ +} + +static inline void ref_tracker_dir_exit(struct ref_tracker_dir *dir) +{ +} + +static inline void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit) +{ +} + +static inline int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, + gfp_t gfp) +{ + return 0; +} + +static inline int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp) +{ + return 0; +} + +#endif + +#endif /* _LINUX_REF_TRACKER_H */ diff --git a/include/linux/regmap.h b/include/linux/regmap.h index e3c9a25a853a..22652e5fbc38 100644 --- a/include/linux/regmap.h +++ b/include/linux/regmap.h @@ -290,6 +290,11 @@ typedef void (*regmap_unlock)(void *); * read operation on a bus such as SPI, I2C, etc. Most of the * devices do not need this. * @reg_write: Same as above for writing. + * @reg_update_bits: Optional callback that if filled will be used to perform + * all the update_bits(rmw) operation. Should only be provided + * if the function require special handling with lock and reg + * handling and the operation cannot be represented as a simple + * update_bits operation on a bus such as SPI, I2C, etc. * @fast_io: Register IO is fast. Use a spinlock instead of a mutex * to perform locking. This field is ignored if custom lock/unlock * functions are used (see fields lock/unlock of struct regmap_config). @@ -372,6 +377,8 @@ struct regmap_config { int (*reg_read)(void *context, unsigned int reg, unsigned int *val); int (*reg_write)(void *context, unsigned int reg, unsigned int val); + int (*reg_update_bits)(void *context, unsigned int reg, + unsigned int mask, unsigned int val); bool fast_io; diff --git a/include/linux/siphash.h b/include/linux/siphash.h index 0cda61855d90..cce8a9acc76c 100644 --- a/include/linux/siphash.h +++ b/include/linux/siphash.h @@ -21,6 +21,8 @@ typedef struct { u64 key[2]; } siphash_key_t; +#define siphash_aligned_key_t siphash_key_t __aligned(16) + static inline bool siphash_key_is_zero(const siphash_key_t *key) { return !(key->key[0] | key->key[1]); diff --git a/include/linux/skbuff.h b/include/linux/skbuff.h index c8cb7e697d47..6535294f6a48 100644 --- a/include/linux/skbuff.h +++ b/include/linux/skbuff.h @@ -36,6 +36,7 @@ #include <linux/splice.h> #include <linux/in6.h> #include <linux/if_packet.h> +#include <linux/llist.h> #include <net/flow.h> #include <net/page_pool.h> #if IS_ENABLED(CONFIG_NF_CONNTRACK) @@ -291,9 +292,11 @@ struct tc_skb_ext { #endif struct sk_buff_head { - /* These two members must be first. */ - struct sk_buff *next; - struct sk_buff *prev; + /* These two members must be first to match sk_buff. */ + struct_group_tagged(sk_buff_list, list, + struct sk_buff *next; + struct sk_buff *prev; + ); __u32 qlen; spinlock_t lock; @@ -626,6 +629,7 @@ typedef unsigned char *sk_buff_data_t; * for retransmit timer * @rbnode: RB tree node, alternative to next/prev for netem/tcp * @list: queue head + * @ll_node: anchor in an llist (eg socket defer_list) * @sk: Socket we are owned by * @ip_defrag_offset: (aka @sk) alternate use of @sk, used in * fragmentation management @@ -728,7 +732,7 @@ typedef unsigned char *sk_buff_data_t; struct sk_buff { union { struct { - /* These two members must be first. */ + /* These two members must be first to match sk_buff_head. */ struct sk_buff *next; struct sk_buff *prev; @@ -743,6 +747,7 @@ struct sk_buff { }; struct rb_node rbnode; /* used in netem, ip4 defrag, and tcp stack */ struct list_head list; + struct llist_node ll_node; }; union { @@ -792,7 +797,7 @@ struct sk_buff { #else #define CLONED_MASK 1 #endif -#define CLONED_OFFSET() offsetof(struct sk_buff, __cloned_offset) +#define CLONED_OFFSET offsetof(struct sk_buff, __cloned_offset) /* private: */ __u8 __cloned_offset[0]; @@ -808,25 +813,15 @@ struct sk_buff { __u8 active_extensions; #endif - /* fields enclosed in headers_start/headers_end are copied + /* Fields enclosed in headers group are copied * using a single memcpy() in __copy_skb_header() */ - /* private: */ - __u32 headers_start[0]; - /* public: */ - -/* if you move pkt_type around you also must adapt those constants */ -#ifdef __BIG_ENDIAN_BITFIELD -#define PKT_TYPE_MAX (7 << 5) -#else -#define PKT_TYPE_MAX 7 -#endif -#define PKT_TYPE_OFFSET() offsetof(struct sk_buff, __pkt_type_offset) + struct_group(headers, /* private: */ __u8 __pkt_type_offset[0]; /* public: */ - __u8 pkt_type:3; + __u8 pkt_type:3; /* see PKT_TYPE_MAX */ __u8 ignore_df:1; __u8 nf_trace:1; __u8 ip_summed:2; @@ -842,16 +837,10 @@ struct sk_buff { __u8 encap_hdr_csum:1; __u8 csum_valid:1; -#ifdef __BIG_ENDIAN_BITFIELD -#define PKT_VLAN_PRESENT_BIT 7 -#else -#define PKT_VLAN_PRESENT_BIT 0 -#endif -#define PKT_VLAN_PRESENT_OFFSET() offsetof(struct sk_buff, __pkt_vlan_present_offset) /* private: */ __u8 __pkt_vlan_present_offset[0]; /* public: */ - __u8 vlan_present:1; + __u8 vlan_present:1; /* See PKT_VLAN_PRESENT_BIT */ __u8 csum_complete_sw:1; __u8 csum_level:2; __u8 csum_not_inet:1; @@ -932,9 +921,7 @@ struct sk_buff { u64 kcov_handle; #endif - /* private: */ - __u32 headers_end[0]; - /* public: */ + ); /* end headers group */ /* These elements must be at the end, see alloc_skb() for details. */ sk_buff_data_t tail; @@ -950,6 +937,22 @@ struct sk_buff { #endif }; +/* if you move pkt_type around you also must adapt those constants */ +#ifdef __BIG_ENDIAN_BITFIELD +#define PKT_TYPE_MAX (7 << 5) +#else +#define PKT_TYPE_MAX 7 +#endif +#define PKT_TYPE_OFFSET offsetof(struct sk_buff, __pkt_type_offset) + +/* if you move pkt_vlan_present around you also must adapt these constants */ +#ifdef __BIG_ENDIAN_BITFIELD +#define PKT_VLAN_PRESENT_BIT 7 +#else +#define PKT_VLAN_PRESENT_BIT 0 +#endif +#define PKT_VLAN_PRESENT_OFFSET offsetof(struct sk_buff, __pkt_vlan_present_offset) + #ifdef __KERNEL__ /* * Handling routines are only of interest to the kernel @@ -1975,8 +1978,8 @@ static inline void __skb_insert(struct sk_buff *newsk, */ WRITE_ONCE(newsk->next, next); WRITE_ONCE(newsk->prev, prev); - WRITE_ONCE(next->prev, newsk); - WRITE_ONCE(prev->next, newsk); + WRITE_ONCE(((struct sk_buff_list *)next)->prev, newsk); + WRITE_ONCE(((struct sk_buff_list *)prev)->next, newsk); WRITE_ONCE(list->qlen, list->qlen + 1); } @@ -2072,7 +2075,7 @@ static inline void __skb_queue_after(struct sk_buff_head *list, struct sk_buff *prev, struct sk_buff *newsk) { - __skb_insert(newsk, prev, prev->next, list); + __skb_insert(newsk, prev, ((struct sk_buff_list *)prev)->next, list); } void skb_append(struct sk_buff *old, struct sk_buff *newsk, @@ -2082,7 +2085,7 @@ static inline void __skb_queue_before(struct sk_buff_head *list, struct sk_buff *next, struct sk_buff *newsk) { - __skb_insert(newsk, next->prev, next, list); + __skb_insert(newsk, ((struct sk_buff_list *)next)->prev, next, list); } /** @@ -3484,7 +3487,12 @@ __skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len, static inline void skb_postpull_rcsum(struct sk_buff *skb, const void *start, unsigned int len) { - __skb_postpull_rcsum(skb, start, len, 0); + if (skb->ip_summed == CHECKSUM_COMPLETE) + skb->csum = wsum_negate(csum_partial(start, len, + wsum_negate(skb->csum))); + else if (skb->ip_summed == CHECKSUM_PARTIAL && + skb_checksum_start_offset(skb) < 0) + skb->ip_summed = CHECKSUM_NONE; } static __always_inline void diff --git a/include/linux/skmsg.h b/include/linux/skmsg.h index 584d94be9c8b..18a717fe62eb 100644 --- a/include/linux/skmsg.h +++ b/include/linux/skmsg.h @@ -507,12 +507,6 @@ static inline bool sk_psock_strp_enabled(struct sk_psock *psock) return !!psock->saved_data_ready; } -static inline bool sk_is_tcp(const struct sock *sk) -{ - return sk->sk_type == SOCK_STREAM && - sk->sk_protocol == IPPROTO_TCP; -} - static inline bool sk_is_udp(const struct sock *sk) { return sk->sk_type == SOCK_DGRAM && diff --git a/include/linux/stmmac.h b/include/linux/stmmac.h index a6f03b36fc4f..24eea1b05ca2 100644 --- a/include/linux/stmmac.h +++ b/include/linux/stmmac.h @@ -233,6 +233,7 @@ struct plat_stmmacenet_data { int (*clks_config)(void *priv, bool enabled); int (*crosststamp)(ktime_t *device, struct system_counterval_t *system, void *ctx); + void (*dump_debug_regs)(void *priv); void *bsp_priv; struct clk *stmmac_clk; struct clk *pclk; @@ -241,6 +242,7 @@ struct plat_stmmacenet_data { unsigned int clk_ref_rate; unsigned int mult_fact_100ns; s32 ptp_max_adj; + u32 cdc_error_adj; struct reset_control *stmmac_rst; struct reset_control *stmmac_ahb_rst; struct stmmac_axi *axi; diff --git a/include/linux/tcp.h b/include/linux/tcp.h index 48d8a363319e..78b91bb92f0d 100644 --- a/include/linux/tcp.h +++ b/include/linux/tcp.h @@ -512,11 +512,13 @@ static inline u16 tcp_mss_clamp(const struct tcp_sock *tp, u16 mss) int tcp_skb_shift(struct sk_buff *to, struct sk_buff *from, int pcount, int shiftlen); +void __tcp_sock_set_cork(struct sock *sk, bool on); void tcp_sock_set_cork(struct sock *sk, bool on); int tcp_sock_set_keepcnt(struct sock *sk, int val); int tcp_sock_set_keepidle_locked(struct sock *sk, int val); int tcp_sock_set_keepidle(struct sock *sk, int val); int tcp_sock_set_keepintvl(struct sock *sk, int val); +void __tcp_sock_set_nodelay(struct sock *sk, bool on); void tcp_sock_set_nodelay(struct sock *sk); void tcp_sock_set_quickack(struct sock *sk, int val); int tcp_sock_set_syncnt(struct sock *sk, int val); diff --git a/include/linux/wwan.h b/include/linux/wwan.h index 9fac819f92e3..e143c88bf4b0 100644 --- a/include/linux/wwan.h +++ b/include/linux/wwan.h @@ -171,4 +171,13 @@ int wwan_register_ops(struct device *parent, const struct wwan_ops *ops, void wwan_unregister_ops(struct device *parent); +#ifdef CONFIG_WWAN_DEBUGFS +struct dentry *wwan_get_debugfs_dir(struct device *parent); +#else +static inline struct dentry *wwan_get_debugfs_dir(struct device *parent) +{ + return ERR_PTR(-ENODEV); +} +#endif + #endif /* __WWAN_H */ diff --git a/include/net/af_unix.h b/include/net/af_unix.h index 7d142e8a0550..a7ef624ed726 100644 --- a/include/net/af_unix.h +++ b/include/net/af_unix.h @@ -20,13 +20,12 @@ struct sock *unix_peer_get(struct sock *sk); #define UNIX_HASH_BITS 8 extern unsigned int unix_tot_inflight; -extern spinlock_t unix_table_lock; +extern spinlock_t unix_table_locks[2 * UNIX_HASH_SIZE]; extern struct hlist_head unix_socket_table[2 * UNIX_HASH_SIZE]; struct unix_address { refcount_t refcnt; int len; - unsigned int hash; struct sockaddr_un name[]; }; diff --git a/include/net/arp.h b/include/net/arp.h index 4950191f6b2b..031374ac2f22 100644 --- a/include/net/arp.h +++ b/include/net/arp.h @@ -53,13 +53,7 @@ static inline void __ipv4_confirm_neigh(struct net_device *dev, u32 key) rcu_read_lock_bh(); n = __ipv4_neigh_lookup_noref(dev, key); - if (n) { - unsigned long now = jiffies; - - /* avoid dirtying neighbour */ - if (READ_ONCE(n->confirmed) != now) - WRITE_ONCE(n->confirmed, now); - } + neigh_confirm(n); rcu_read_unlock_bh(); } diff --git a/include/net/ax25.h b/include/net/ax25.h index 03d409de61ad..526e49589197 100644 --- a/include/net/ax25.h +++ b/include/net/ax25.h @@ -229,7 +229,10 @@ struct ctl_table; typedef struct ax25_dev { struct ax25_dev *next; + struct net_device *dev; + netdevice_tracker dev_tracker; + struct net_device *forward; struct ctl_table_header *sysheader; int values[AX25_MAX_VALUES]; diff --git a/include/net/bluetooth/bluetooth.h b/include/net/bluetooth/bluetooth.h index 3271870fd85e..2f31e571f34c 100644 --- a/include/net/bluetooth/bluetooth.h +++ b/include/net/bluetooth/bluetooth.h @@ -380,6 +380,7 @@ typedef void (*hci_req_complete_skb_t)(struct hci_dev *hdev, u8 status, #define HCI_REQ_SKB BIT(1) struct hci_ctrl { + struct sock *sk; u16 opcode; u8 req_flags; u8 req_event; @@ -405,6 +406,7 @@ struct bt_skb_cb { #define hci_skb_pkt_type(skb) bt_cb((skb))->pkt_type #define hci_skb_expect(skb) bt_cb((skb))->expect #define hci_skb_opcode(skb) bt_cb((skb))->hci.opcode +#define hci_skb_sk(skb) bt_cb((skb))->hci.sk static inline struct sk_buff *bt_skb_alloc(unsigned int len, gfp_t how) { diff --git a/include/net/bluetooth/hci.h b/include/net/bluetooth/hci.h index 63065bc01b76..84db6b275231 100644 --- a/include/net/bluetooth/hci.h +++ b/include/net/bluetooth/hci.h @@ -566,6 +566,7 @@ enum { #define HCI_ERROR_INVALID_LL_PARAMS 0x1e #define HCI_ERROR_UNSPECIFIED 0x1f #define HCI_ERROR_ADVERTISING_TIMEOUT 0x3c +#define HCI_ERROR_CANCELLED_BY_HOST 0x44 /* Flow control modes */ #define HCI_FLOW_CTL_MODE_PACKET_BASED 0x00 diff --git a/include/net/bluetooth/hci_core.h b/include/net/bluetooth/hci_core.h index dd8840e70e25..2560cfe80db8 100644 --- a/include/net/bluetooth/hci_core.h +++ b/include/net/bluetooth/hci_core.h @@ -30,6 +30,7 @@ #include <linux/rculist.h> #include <net/bluetooth/hci.h> +#include <net/bluetooth/hci_sync.h> #include <net/bluetooth/hci_sock.h> /* HCI priority */ @@ -475,6 +476,9 @@ struct hci_dev { struct work_struct power_on; struct delayed_work power_off; struct work_struct error_reset; + struct work_struct cmd_sync_work; + struct list_head cmd_sync_work_list; + struct mutex cmd_sync_work_lock; __u16 discov_timeout; struct delayed_work discov_off; @@ -489,10 +493,7 @@ struct hci_dev { struct work_struct tx_work; struct work_struct discov_update; - struct work_struct bg_scan_update; struct work_struct scan_update; - struct work_struct connectable_update; - struct work_struct discoverable_update; struct delayed_work le_scan_disable; struct delayed_work le_scan_restart; @@ -519,7 +520,6 @@ struct hci_dev { bool advertising_paused; struct notifier_block suspend_notifier; - struct work_struct suspend_prepare; enum suspended_state suspend_state_next; enum suspended_state suspend_state; bool scanning_paused; @@ -528,9 +528,6 @@ struct hci_dev { bdaddr_t wake_addr; u8 wake_addr_type; - wait_queue_head_t suspend_wait_q; - DECLARE_BITMAP(suspend_tasks, __SUSPEND_NUM_TASKS); - struct hci_conn_hash conn_hash; struct list_head mgmt_pending; @@ -603,6 +600,7 @@ struct hci_dev { #if IS_ENABLED(CONFIG_BT_AOSPEXT) bool aosp_capable; + bool aosp_quality_report; #endif int (*open)(struct hci_dev *hdev); @@ -1461,8 +1459,11 @@ void hci_conn_del_sysfs(struct hci_conn *conn); #define scan_coded(dev) (((dev)->le_tx_def_phys & HCI_LE_SET_PHY_CODED) || \ ((dev)->le_rx_def_phys & HCI_LE_SET_PHY_CODED)) +#define ll_privacy_capable(dev) ((dev)->le_features[0] & HCI_LE_LL_PRIVACY) + /* Use LL Privacy based address resolution if supported */ -#define use_ll_privacy(dev) ((dev)->le_features[0] & HCI_LE_LL_PRIVACY) +#define use_ll_privacy(dev) (ll_privacy_capable(dev) && \ + hci_dev_test_flag(dev, HCI_ENABLE_LL_PRIVACY)) /* Use enhanced synchronous connection if command is supported */ #define enhanced_sco_capable(dev) ((dev)->commands[29] & 0x08) @@ -1690,10 +1691,6 @@ static inline int hci_check_conn_params(u16 min, u16 max, u16 latency, int hci_register_cb(struct hci_cb *hcb); int hci_unregister_cb(struct hci_cb *hcb); -struct sk_buff *__hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u32 timeout); -struct sk_buff *__hci_cmd_sync_ev(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u8 event, u32 timeout); int __hci_cmd_send(struct hci_dev *hdev, u16 opcode, u32 plen, const void *param); @@ -1704,9 +1701,6 @@ void hci_send_sco(struct hci_conn *conn, struct sk_buff *skb); void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode); -struct sk_buff *hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u32 timeout); - u32 hci_conn_get_phy(struct hci_conn *conn); /* ----- HCI Sockets ----- */ @@ -1806,7 +1800,6 @@ int mgmt_user_passkey_notify(struct hci_dev *hdev, bdaddr_t *bdaddr, u8 entered); void mgmt_auth_failed(struct hci_conn *conn, u8 status); void mgmt_auth_enable_complete(struct hci_dev *hdev, u8 status); -void mgmt_ssp_enable_complete(struct hci_dev *hdev, u8 enable, u8 status); void mgmt_set_class_of_dev_complete(struct hci_dev *hdev, u8 *dev_class, u8 status); void mgmt_set_local_name_complete(struct hci_dev *hdev, u8 *name, u8 status); @@ -1831,8 +1824,6 @@ void mgmt_new_conn_param(struct hci_dev *hdev, bdaddr_t *bdaddr, u16 max_interval, u16 latency, u16 timeout); void mgmt_smp_complete(struct hci_conn *conn, bool complete); bool mgmt_get_connectable(struct hci_dev *hdev); -void mgmt_set_connectable_complete(struct hci_dev *hdev, u8 status); -void mgmt_set_discoverable_complete(struct hci_dev *hdev, u8 status); u8 mgmt_get_adv_discov_flags(struct hci_dev *hdev); void mgmt_advertising_added(struct sock *sk, struct hci_dev *hdev, u8 instance); diff --git a/include/net/bluetooth/hci_sync.h b/include/net/bluetooth/hci_sync.h new file mode 100644 index 000000000000..0336c1bc5d25 --- /dev/null +++ b/include/net/bluetooth/hci_sync.h @@ -0,0 +1,102 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * BlueZ - Bluetooth protocol stack for Linux + * + * Copyright (C) 2021 Intel Corporation + */ + +typedef int (*hci_cmd_sync_work_func_t)(struct hci_dev *hdev, void *data); +typedef void (*hci_cmd_sync_work_destroy_t)(struct hci_dev *hdev, void *data, + int err); + +struct hci_cmd_sync_work_entry { + struct list_head list; + hci_cmd_sync_work_func_t func; + void *data; + hci_cmd_sync_work_destroy_t destroy; +}; + +/* Function with sync suffix shall not be called with hdev->lock held as they + * wait the command to complete and in the meantime an event could be received + * which could attempt to acquire hdev->lock causing a deadlock. + */ +struct sk_buff *__hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout); +struct sk_buff *hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout); +struct sk_buff *__hci_cmd_sync_ev(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout); +struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout, + struct sock *sk); +int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout); +int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout, + struct sock *sk); + +void hci_cmd_sync_init(struct hci_dev *hdev); +void hci_cmd_sync_clear(struct hci_dev *hdev); + +int hci_cmd_sync_queue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, + void *data, hci_cmd_sync_work_destroy_t destroy); + +int hci_update_eir_sync(struct hci_dev *hdev); +int hci_update_class_sync(struct hci_dev *hdev); + +int hci_update_eir_sync(struct hci_dev *hdev); +int hci_update_class_sync(struct hci_dev *hdev); +int hci_update_name_sync(struct hci_dev *hdev); +int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode); + +int hci_update_random_address_sync(struct hci_dev *hdev, bool require_privacy, + bool rpa, u8 *own_addr_type); + +int hci_update_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance); +int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance); +int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance, + bool force); + +int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance); +int hci_start_ext_adv_sync(struct hci_dev *hdev, u8 instance); +int hci_enable_ext_advertising_sync(struct hci_dev *hdev, u8 instance); +int hci_enable_advertising_sync(struct hci_dev *hdev); +int hci_enable_advertising(struct hci_dev *hdev); + +int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk, + u8 instance, bool force); +int hci_disable_advertising_sync(struct hci_dev *hdev); + +int hci_update_passive_scan_sync(struct hci_dev *hdev); +int hci_update_passive_scan(struct hci_dev *hdev); +int hci_read_rssi_sync(struct hci_dev *hdev, __le16 handle); +int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type); +int hci_write_sc_support_sync(struct hci_dev *hdev, u8 val); +int hci_read_clock_sync(struct hci_dev *hdev, struct hci_cp_read_clock *cp); + +int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable); +int hci_update_scan_sync(struct hci_dev *hdev); + +int hci_write_le_host_supported_sync(struct hci_dev *hdev, u8 le, u8 simul); +int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, + struct sock *sk); +struct sk_buff *hci_read_local_oob_data_sync(struct hci_dev *hdev, bool ext, + struct sock *sk); + +int hci_reset_sync(struct hci_dev *hdev); +int hci_dev_open_sync(struct hci_dev *hdev); +int hci_dev_close_sync(struct hci_dev *hdev); + +int hci_powered_update_sync(struct hci_dev *hdev); +int hci_set_powered_sync(struct hci_dev *hdev, u8 val); + +int hci_update_discoverable_sync(struct hci_dev *hdev); +int hci_update_discoverable(struct hci_dev *hdev); + +int hci_update_connectable_sync(struct hci_dev *hdev); + +int hci_start_discovery_sync(struct hci_dev *hdev); +int hci_stop_discovery_sync(struct hci_dev *hdev); + +int hci_suspend_sync(struct hci_dev *hdev); +int hci_resume_sync(struct hci_dev *hdev); diff --git a/include/net/bond_options.h b/include/net/bond_options.h index e64833a674eb..dd75c071f67e 100644 --- a/include/net/bond_options.h +++ b/include/net/bond_options.h @@ -65,6 +65,7 @@ enum { BOND_OPT_NUM_PEER_NOTIF_ALIAS, BOND_OPT_PEER_NOTIF_DELAY, BOND_OPT_LACP_ACTIVE, + BOND_OPT_MISSED_MAX, BOND_OPT_LAST }; diff --git a/include/net/bonding.h b/include/net/bonding.h index 15e083e18f75..f6ae3a4baea4 100644 --- a/include/net/bonding.h +++ b/include/net/bonding.h @@ -121,6 +121,7 @@ struct bond_params { int xmit_policy; int miimon; u8 num_peer_notif; + u8 missed_max; int arp_interval; int arp_validate; int arp_all_targets; diff --git a/include/net/checksum.h b/include/net/checksum.h index 5b96d5bd6e54..5218041e5c8f 100644 --- a/include/net/checksum.h +++ b/include/net/checksum.h @@ -180,4 +180,8 @@ static inline void remcsum_unadjust(__sum16 *psum, __wsum delta) *psum = csum_fold(csum_sub(delta, (__force __wsum)*psum)); } +static inline __wsum wsum_negate(__wsum val) +{ + return (__force __wsum)-((__force u32)val); +} #endif diff --git a/include/net/devlink.h b/include/net/devlink.h index aab3d007c577..3276a29f2b81 100644 --- a/include/net/devlink.h +++ b/include/net/devlink.h @@ -361,33 +361,6 @@ devlink_resource_size_params_init(struct devlink_resource_size_params *size_para typedef u64 devlink_resource_occ_get_t(void *priv); -/** - * struct devlink_resource - devlink resource - * @name: name of the resource - * @id: id, per devlink instance - * @size: size of the resource - * @size_new: updated size of the resource, reload is needed - * @size_valid: valid in case the total size of the resource is valid - * including its children - * @parent: parent resource - * @size_params: size parameters - * @list: parent list - * @resource_list: list of child resources - */ -struct devlink_resource { - const char *name; - u64 id; - u64 size; - u64 size_new; - bool size_valid; - struct devlink_resource *parent; - struct devlink_resource_size_params size_params; - struct list_head list; - struct list_head resource_list; - devlink_resource_occ_get_t *occ_get; - void *occ_get_priv; -}; - #define DEVLINK_RESOURCE_ID_PARENT_TOP 0 #define DEVLINK_RESOURCE_GENERIC_NAME_PORTS "physical_ports" @@ -485,6 +458,7 @@ enum devlink_param_generic_id { DEVLINK_PARAM_GENERIC_ID_ENABLE_ETH, DEVLINK_PARAM_GENERIC_ID_ENABLE_RDMA, DEVLINK_PARAM_GENERIC_ID_ENABLE_VNET, + DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP, /* add new param generic ids above here*/ __DEVLINK_PARAM_GENERIC_ID_MAX, @@ -534,6 +508,9 @@ enum devlink_param_generic_id { #define DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME "enable_vnet" #define DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE DEVLINK_PARAM_TYPE_BOOL +#define DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME "enable_iwarp" +#define DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE DEVLINK_PARAM_TYPE_BOOL + #define DEVLINK_PARAM_GENERIC(_id, _cmodes, _get, _set, _validate) \ { \ .id = DEVLINK_PARAM_GENERIC_ID_##_id, \ @@ -687,13 +664,17 @@ struct devlink_health_reporter_ops { * @trap_name: Trap name. * @trap_group_name: Trap group name. * @input_dev: Input netdevice. + * @dev_tracker: refcount tracker for @input_dev. * @fa_cookie: Flow action user cookie. * @trap_type: Trap type. */ struct devlink_trap_metadata { const char *trap_name; const char *trap_group_name; + struct net_device *input_dev; + netdevice_tracker dev_tracker; + const struct flow_action_cookie *fa_cookie; enum devlink_trap_type trap_type; }; @@ -1567,8 +1548,7 @@ int devlink_resource_register(struct devlink *devlink, u64 resource_id, u64 parent_resource_id, const struct devlink_resource_size_params *size_params); -void devlink_resources_unregister(struct devlink *devlink, - struct devlink_resource *resource); +void devlink_resources_unregister(struct devlink *devlink); int devlink_resource_size_get(struct devlink *devlink, u64 resource_id, u64 *p_resource_size); diff --git a/include/net/dsa.h b/include/net/dsa.h index eff5c44ba377..bdf308a5c55e 100644 --- a/include/net/dsa.h +++ b/include/net/dsa.h @@ -219,6 +219,12 @@ struct dsa_mall_tc_entry { }; }; +struct dsa_bridge { + struct net_device *dev; + unsigned int num; + bool tx_fwd_offload; + refcount_t refcount; +}; struct dsa_port { /* A CPU port is physically connected to a master device. @@ -256,8 +262,7 @@ struct dsa_port { /* Managed by DSA on user ports and by drivers on CPU and DSA ports */ bool learning; u8 stp_state; - struct net_device *bridge_dev; - int bridge_num; + struct dsa_bridge *bridge; struct devlink_port devlink_port; bool devlink_port_setup; struct phylink *pl; @@ -413,12 +418,12 @@ struct dsa_switch { */ unsigned int num_lag_ids; - /* Drivers that support bridge forwarding offload should set this to - * the maximum number of bridges spanning the same switch tree (or all - * trees, in the case of cross-tree bridging support) that can be - * offloaded. + /* Drivers that support bridge forwarding offload or FDB isolation + * should set this to the maximum number of bridges spanning the same + * switch tree (or all trees, in the case of cross-tree bridging + * support) that can be offloaded. */ - unsigned int num_fwd_offloading_bridges; + unsigned int max_num_bridges; size_t num_ports; }; @@ -588,7 +593,7 @@ static inline bool dsa_port_is_vlan_filtering(const struct dsa_port *dp) static inline struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp) { - if (!dp->bridge_dev) + if (!dp->bridge) return NULL; if (dp->lag_dev) @@ -599,6 +604,76 @@ struct net_device *dsa_port_to_bridge_port(const struct dsa_port *dp) return dp->slave; } +static inline struct net_device * +dsa_port_bridge_dev_get(const struct dsa_port *dp) +{ + return dp->bridge ? dp->bridge->dev : NULL; +} + +static inline unsigned int dsa_port_bridge_num_get(struct dsa_port *dp) +{ + return dp->bridge ? dp->bridge->num : 0; +} + +static inline bool dsa_port_bridge_same(const struct dsa_port *a, + const struct dsa_port *b) +{ + struct net_device *br_a = dsa_port_bridge_dev_get(a); + struct net_device *br_b = dsa_port_bridge_dev_get(b); + + /* Standalone ports are not in the same bridge with one another */ + return (!br_a || !br_b) ? false : (br_a == br_b); +} + +static inline bool dsa_port_offloads_bridge_port(struct dsa_port *dp, + const struct net_device *dev) +{ + return dsa_port_to_bridge_port(dp) == dev; +} + +static inline bool +dsa_port_offloads_bridge_dev(struct dsa_port *dp, + const struct net_device *bridge_dev) +{ + /* DSA ports connected to a bridge, and event was emitted + * for the bridge. + */ + return dsa_port_bridge_dev_get(dp) == bridge_dev; +} + +static inline bool dsa_port_offloads_bridge(struct dsa_port *dp, + const struct dsa_bridge *bridge) +{ + return dsa_port_bridge_dev_get(dp) == bridge->dev; +} + +/* Returns true if any port of this tree offloads the given net_device */ +static inline bool dsa_tree_offloads_bridge_port(struct dsa_switch_tree *dst, + const struct net_device *dev) +{ + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) + if (dsa_port_offloads_bridge_port(dp, dev)) + return true; + + return false; +} + +/* Returns true if any port of this tree offloads the given bridge */ +static inline bool +dsa_tree_offloads_bridge_dev(struct dsa_switch_tree *dst, + const struct net_device *bridge_dev) +{ + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) + if (dsa_port_offloads_bridge_dev(dp, bridge_dev)) + return true; + + return false; +} + typedef int dsa_fdb_dump_cb_t(const unsigned char *addr, u16 vid, bool is_static, void *data); struct dsa_switch_ops { @@ -645,8 +720,8 @@ struct dsa_switch_ops { /* * PHYLINK integration */ - void (*phylink_get_interfaces)(struct dsa_switch *ds, int port, - unsigned long *supported_interfaces); + void (*phylink_get_caps)(struct dsa_switch *ds, int port, + struct phylink_config *config); void (*phylink_validate)(struct dsa_switch *ds, int port, unsigned long *supported, struct phylink_link_state *state); @@ -748,17 +823,10 @@ struct dsa_switch_ops { */ int (*set_ageing_time)(struct dsa_switch *ds, unsigned int msecs); int (*port_bridge_join)(struct dsa_switch *ds, int port, - struct net_device *bridge); + struct dsa_bridge bridge, + bool *tx_fwd_offload); void (*port_bridge_leave)(struct dsa_switch *ds, int port, - struct net_device *bridge); - /* Called right after .port_bridge_join() */ - int (*port_bridge_tx_fwd_offload)(struct dsa_switch *ds, int port, - struct net_device *bridge, - int bridge_num); - /* Called right before .port_bridge_leave() */ - void (*port_bridge_tx_fwd_unoffload)(struct dsa_switch *ds, int port, - struct net_device *bridge, - int bridge_num); + struct dsa_bridge bridge); void (*port_stp_state_set)(struct dsa_switch *ds, int port, u8 state); void (*port_fast_age)(struct dsa_switch *ds, int port); @@ -830,10 +898,10 @@ struct dsa_switch_ops { */ int (*crosschip_bridge_join)(struct dsa_switch *ds, int tree_index, int sw_index, int port, - struct net_device *br); + struct dsa_bridge bridge); void (*crosschip_bridge_leave)(struct dsa_switch *ds, int tree_index, int sw_index, int port, - struct net_device *br); + struct dsa_bridge bridge); int (*crosschip_lag_change)(struct dsa_switch *ds, int sw_index, int port); int (*crosschip_lag_join)(struct dsa_switch *ds, int sw_index, diff --git a/include/net/dst.h b/include/net/dst.h index a057319aabef..6aa252c3fc55 100644 --- a/include/net/dst.h +++ b/include/net/dst.h @@ -77,6 +77,7 @@ struct dst_entry { #ifndef CONFIG_64BIT atomic_t __refcnt; /* 32-bit offset 64 */ #endif + netdevice_tracker dev_tracker; }; struct dst_metrics { diff --git a/include/net/failover.h b/include/net/failover.h index bb15438f39c7..f2b42b4b9cd6 100644 --- a/include/net/failover.h +++ b/include/net/failover.h @@ -25,6 +25,7 @@ struct failover_ops { struct failover { struct list_head list; struct net_device __rcu *failover_dev; + netdevice_tracker dev_tracker; struct failover_ops __rcu *ops; }; diff --git a/include/net/gro.h b/include/net/gro.h index 01edaf3fdda0..8f75802d50fd 100644 --- a/include/net/gro.h +++ b/include/net/gro.h @@ -4,9 +4,368 @@ #define _NET_IPV6_GRO_H #include <linux/indirect_call_wrapper.h> +#include <linux/ip.h> +#include <linux/ipv6.h> +#include <net/ip6_checksum.h> +#include <linux/skbuff.h> +#include <net/udp.h> -struct list_head; -struct sk_buff; +struct napi_gro_cb { + /* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */ + void *frag0; + + /* Length of frag0. */ + unsigned int frag0_len; + + /* This indicates where we are processing relative to skb->data. */ + int data_offset; + + /* This is non-zero if the packet cannot be merged with the new skb. */ + u16 flush; + + /* Save the IP ID here and check when we get to the transport layer */ + u16 flush_id; + + /* Number of segments aggregated. */ + u16 count; + + /* Start offset for remote checksum offload */ + u16 gro_remcsum_start; + + /* jiffies when first packet was created/queued */ + unsigned long age; + + /* Used in ipv6_gro_receive() and foo-over-udp */ + u16 proto; + + /* This is non-zero if the packet may be of the same flow. */ + u8 same_flow:1; + + /* Used in tunnel GRO receive */ + u8 encap_mark:1; + + /* GRO checksum is valid */ + u8 csum_valid:1; + + /* Number of checksums via CHECKSUM_UNNECESSARY */ + u8 csum_cnt:3; + + /* Free the skb? */ + u8 free:2; +#define NAPI_GRO_FREE 1 +#define NAPI_GRO_FREE_STOLEN_HEAD 2 + + /* Used in foo-over-udp, set in udp[46]_gro_receive */ + u8 is_ipv6:1; + + /* Used in GRE, set in fou/gue_gro_receive */ + u8 is_fou:1; + + /* Used to determine if flush_id can be ignored */ + u8 is_atomic:1; + + /* Number of gro_receive callbacks this packet already went through */ + u8 recursion_counter:4; + + /* GRO is done by frag_list pointer chaining. */ + u8 is_flist:1; + + /* used to support CHECKSUM_COMPLETE for tunneling protocols */ + __wsum csum; + + /* used in skb_gro_receive() slow path */ + struct sk_buff *last; +}; + +#define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb) + +#define GRO_RECURSION_LIMIT 15 +static inline int gro_recursion_inc_test(struct sk_buff *skb) +{ + return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT; +} + +typedef struct sk_buff *(*gro_receive_t)(struct list_head *, struct sk_buff *); +static inline struct sk_buff *call_gro_receive(gro_receive_t cb, + struct list_head *head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(head, skb); +} + +typedef struct sk_buff *(*gro_receive_sk_t)(struct sock *, struct list_head *, + struct sk_buff *); +static inline struct sk_buff *call_gro_receive_sk(gro_receive_sk_t cb, + struct sock *sk, + struct list_head *head, + struct sk_buff *skb) +{ + if (unlikely(gro_recursion_inc_test(skb))) { + NAPI_GRO_CB(skb)->flush |= 1; + return NULL; + } + + return cb(sk, head, skb); +} + +static inline unsigned int skb_gro_offset(const struct sk_buff *skb) +{ + return NAPI_GRO_CB(skb)->data_offset; +} + +static inline unsigned int skb_gro_len(const struct sk_buff *skb) +{ + return skb->len - NAPI_GRO_CB(skb)->data_offset; +} + +static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len) +{ + NAPI_GRO_CB(skb)->data_offset += len; +} + +static inline void *skb_gro_header_fast(struct sk_buff *skb, + unsigned int offset) +{ + return NAPI_GRO_CB(skb)->frag0 + offset; +} + +static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) +{ + return NAPI_GRO_CB(skb)->frag0_len < hlen; +} + +static inline void skb_gro_frag0_invalidate(struct sk_buff *skb) +{ + NAPI_GRO_CB(skb)->frag0 = NULL; + NAPI_GRO_CB(skb)->frag0_len = 0; +} + +static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, + unsigned int offset) +{ + if (!pskb_may_pull(skb, hlen)) + return NULL; + + skb_gro_frag0_invalidate(skb); + return skb->data + offset; +} + +static inline void *skb_gro_network_header(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + + skb_network_offset(skb); +} + +static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) +{ + const struct iphdr *iph = skb_gro_network_header(skb); + + return csum_tcpudp_nofold(iph->saddr, iph->daddr, + skb_gro_len(skb), proto, 0); +} + +static inline void skb_gro_postpull_rcsum(struct sk_buff *skb, + const void *start, unsigned int len) +{ + if (NAPI_GRO_CB(skb)->csum_valid) + NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len, + wsum_negate(NAPI_GRO_CB(skb)->csum))); +} + +/* GRO checksum functions. These are logical equivalents of the normal + * checksum functions (in skbuff.h) except that they operate on the GRO + * offsets and fields in sk_buff. + */ + +__sum16 __skb_gro_checksum_complete(struct sk_buff *skb); + +static inline bool skb_at_gro_remcsum_start(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb)); +} + +static inline bool __skb_gro_checksum_validate_needed(struct sk_buff *skb, + bool zero_okay, + __sum16 check) +{ + return ((skb->ip_summed != CHECKSUM_PARTIAL || + skb_checksum_start_offset(skb) < + skb_gro_offset(skb)) && + !skb_at_gro_remcsum_start(skb) && + NAPI_GRO_CB(skb)->csum_cnt == 0 && + (!zero_okay || check)); +} + +static inline __sum16 __skb_gro_checksum_validate_complete(struct sk_buff *skb, + __wsum psum) +{ + if (NAPI_GRO_CB(skb)->csum_valid && + !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum))) + return 0; + + NAPI_GRO_CB(skb)->csum = psum; + + return __skb_gro_checksum_complete(skb); +} + +static inline void skb_gro_incr_csum_unnecessary(struct sk_buff *skb) +{ + if (NAPI_GRO_CB(skb)->csum_cnt > 0) { + /* Consume a checksum from CHECKSUM_UNNECESSARY */ + NAPI_GRO_CB(skb)->csum_cnt--; + } else { + /* Update skb for CHECKSUM_UNNECESSARY and csum_level when we + * verified a new top level checksum or an encapsulated one + * during GRO. This saves work if we fallback to normal path. + */ + __skb_incr_checksum_unnecessary(skb); + } +} + +#define __skb_gro_checksum_validate(skb, proto, zero_okay, check, \ + compute_pseudo) \ +({ \ + __sum16 __ret = 0; \ + if (__skb_gro_checksum_validate_needed(skb, zero_okay, check)) \ + __ret = __skb_gro_checksum_validate_complete(skb, \ + compute_pseudo(skb, proto)); \ + if (!__ret) \ + skb_gro_incr_csum_unnecessary(skb); \ + __ret; \ +}) + +#define skb_gro_checksum_validate(skb, proto, compute_pseudo) \ + __skb_gro_checksum_validate(skb, proto, false, 0, compute_pseudo) + +#define skb_gro_checksum_validate_zero_check(skb, proto, check, \ + compute_pseudo) \ + __skb_gro_checksum_validate(skb, proto, true, check, compute_pseudo) + +#define skb_gro_checksum_simple_validate(skb) \ + __skb_gro_checksum_validate(skb, 0, false, 0, null_compute_pseudo) + +static inline bool __skb_gro_checksum_convert_check(struct sk_buff *skb) +{ + return (NAPI_GRO_CB(skb)->csum_cnt == 0 && + !NAPI_GRO_CB(skb)->csum_valid); +} + +static inline void __skb_gro_checksum_convert(struct sk_buff *skb, + __wsum pseudo) +{ + NAPI_GRO_CB(skb)->csum = ~pseudo; + NAPI_GRO_CB(skb)->csum_valid = 1; +} + +#define skb_gro_checksum_try_convert(skb, proto, compute_pseudo) \ +do { \ + if (__skb_gro_checksum_convert_check(skb)) \ + __skb_gro_checksum_convert(skb, \ + compute_pseudo(skb, proto)); \ +} while (0) + +struct gro_remcsum { + int offset; + __wsum delta; +}; + +static inline void skb_gro_remcsum_init(struct gro_remcsum *grc) +{ + grc->offset = 0; + grc->delta = 0; +} + +static inline void *skb_gro_remcsum_process(struct sk_buff *skb, void *ptr, + unsigned int off, size_t hdrlen, + int start, int offset, + struct gro_remcsum *grc, + bool nopartial) +{ + __wsum delta; + size_t plen = hdrlen + max_t(size_t, offset + sizeof(u16), start); + + BUG_ON(!NAPI_GRO_CB(skb)->csum_valid); + + if (!nopartial) { + NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start; + return ptr; + } + + ptr = skb_gro_header_fast(skb, off); + if (skb_gro_header_hard(skb, off + plen)) { + ptr = skb_gro_header_slow(skb, off + plen, off); + if (!ptr) + return NULL; + } + + delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum, + start, offset); + + /* Adjust skb->csum since we changed the packet */ + NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta); + + grc->offset = off + hdrlen + offset; + grc->delta = delta; + + return ptr; +} + +static inline void skb_gro_remcsum_cleanup(struct sk_buff *skb, + struct gro_remcsum *grc) +{ + void *ptr; + size_t plen = grc->offset + sizeof(u16); + + if (!grc->delta) + return; + + ptr = skb_gro_header_fast(skb, grc->offset); + if (skb_gro_header_hard(skb, grc->offset + sizeof(u16))) { + ptr = skb_gro_header_slow(skb, plen, grc->offset); + if (!ptr) + return; + } + + remcsum_unadjust((__sum16 *)ptr, grc->delta); +} + +#ifdef CONFIG_XFRM_OFFLOAD +static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) +{ + if (PTR_ERR(pp) != -EINPROGRESS) + NAPI_GRO_CB(skb)->flush |= flush; +} +static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, + struct sk_buff *pp, + int flush, + struct gro_remcsum *grc) +{ + if (PTR_ERR(pp) != -EINPROGRESS) { + NAPI_GRO_CB(skb)->flush |= flush; + skb_gro_remcsum_cleanup(skb, grc); + skb->remcsum_offload = 0; + } +} +#else +static inline void skb_gro_flush_final(struct sk_buff *skb, struct sk_buff *pp, int flush) +{ + NAPI_GRO_CB(skb)->flush |= flush; +} +static inline void skb_gro_flush_final_remcsum(struct sk_buff *skb, + struct sk_buff *pp, + int flush, + struct gro_remcsum *grc) +{ + NAPI_GRO_CB(skb)->flush |= flush; + skb_gro_remcsum_cleanup(skb, grc); + skb->remcsum_offload = 0; +} +#endif INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *, struct sk_buff *)); @@ -15,6 +374,14 @@ INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *, struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); +INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, + struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); + +INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, + struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); + #define indirect_call_gro_receive_inet(cb, f2, f1, head, skb) \ ({ \ unlikely(gro_recursion_inc_test(skb)) ? \ @@ -22,4 +389,54 @@ INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int)); INDIRECT_CALL_INET(cb, f2, f1, head, skb); \ }) +struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, + struct udphdr *uh, struct sock *sk); +int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); + +static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) +{ + struct udphdr *uh; + unsigned int hlen, off; + + off = skb_gro_offset(skb); + hlen = off + sizeof(*uh); + uh = skb_gro_header_fast(skb, off); + if (skb_gro_header_hard(skb, hlen)) + uh = skb_gro_header_slow(skb, hlen, off); + + return uh; +} + +static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) +{ + const struct ipv6hdr *iph = skb_gro_network_header(skb); + + return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, + skb_gro_len(skb), proto, 0)); +} + +int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb); + +/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ +static inline void gro_normal_list(struct napi_struct *napi) +{ + if (!napi->rx_count) + return; + netif_receive_skb_list_internal(&napi->rx_list); + INIT_LIST_HEAD(&napi->rx_list); + napi->rx_count = 0; +} + +/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, + * pass the whole batch up to the stack. + */ +static inline void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) +{ + list_add_tail(&skb->list, &napi->rx_list); + napi->rx_count += segs; + if (napi->rx_count >= gro_normal_batch) + gro_normal_list(napi); +} + + #endif /* _NET_IPV6_GRO_H */ diff --git a/include/net/if_inet6.h b/include/net/if_inet6.h index 653e7d0f65cb..f026cf08a8e8 100644 --- a/include/net/if_inet6.h +++ b/include/net/if_inet6.h @@ -160,6 +160,7 @@ struct ipv6_devstat { struct inet6_dev { struct net_device *dev; + netdevice_tracker dev_tracker; struct list_head addr_list; diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h index fa6a87246a7b..4ad47d9f9d27 100644 --- a/include/net/inet_connection_sock.h +++ b/include/net/inet_connection_sock.h @@ -304,7 +304,7 @@ static inline __poll_t inet_csk_listen_poll(const struct sock *sk) (EPOLLIN | EPOLLRDNORM) : 0; } -int inet_csk_listen_start(struct sock *sk, int backlog); +int inet_csk_listen_start(struct sock *sk); void inet_csk_listen_stop(struct sock *sk); void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr); diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 9e1111f5915b..234d70ae5f4c 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -372,4 +372,16 @@ static inline bool inet_can_nonlocal_bind(struct net *net, inet->freebind || inet->transparent; } +static inline bool inet_addr_valid_or_nonlocal(struct net *net, + struct inet_sock *inet, + __be32 addr, + int addr_type) +{ + return inet_can_nonlocal_bind(net, inet) || + addr == htonl(INADDR_ANY) || + addr_type == RTN_LOCAL || + addr_type == RTN_MULTICAST || + addr_type == RTN_BROADCAST; +} + #endif /* _INET_SOCK_H */ diff --git a/include/net/ip.h b/include/net/ip.h index b71e88507c4a..81e23a102a0d 100644 --- a/include/net/ip.h +++ b/include/net/ip.h @@ -568,14 +568,6 @@ static inline void iph_to_flow_copy_v4addrs(struct flow_keys *flow, flow->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS; } -static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) -{ - const struct iphdr *iph = skb_gro_network_header(skb); - - return csum_tcpudp_nofold(iph->saddr, iph->daddr, - skb_gro_len(skb), proto, 0); -} - /* * Map a multicast IP onto multicast MAC for type ethernet. */ @@ -791,5 +783,6 @@ int ip_sock_set_mtu_discover(struct sock *sk, int val); void ip_sock_set_pktinfo(struct sock *sk); void ip_sock_set_recverr(struct sock *sk); void ip_sock_set_tos(struct sock *sk, int val); +void __ip_sock_set_tos(struct sock *sk, int val); #endif /* _IP_H */ diff --git a/include/net/ip6_checksum.h b/include/net/ip6_checksum.h index b3f4eaa88672..c8a96b888277 100644 --- a/include/net/ip6_checksum.h +++ b/include/net/ip6_checksum.h @@ -43,14 +43,6 @@ static inline __wsum ip6_compute_pseudo(struct sk_buff *skb, int proto) skb->len, proto, 0)); } -static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) -{ - const struct ipv6hdr *iph = skb_gro_network_header(skb); - - return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr, - skb_gro_len(skb), proto, 0)); -} - static __inline__ __sum16 tcp_v6_check(int len, const struct in6_addr *saddr, const struct in6_addr *daddr, @@ -65,15 +57,9 @@ static inline void __tcp_v6_send_check(struct sk_buff *skb, { struct tcphdr *th = tcp_hdr(skb); - if (skb->ip_summed == CHECKSUM_PARTIAL) { - th->check = ~tcp_v6_check(skb->len, saddr, daddr, 0); - skb->csum_start = skb_transport_header(skb) - skb->head; - skb->csum_offset = offsetof(struct tcphdr, check); - } else { - th->check = tcp_v6_check(skb->len, saddr, daddr, - csum_partial(th, th->doff << 2, - skb->csum)); - } + th->check = ~tcp_v6_check(skb->len, saddr, daddr, 0); + skb->csum_start = skb_transport_header(skb) - skb->head; + skb->csum_offset = offsetof(struct tcphdr, check); } static inline void tcp_v6_gso_csum_prep(struct sk_buff *skb) diff --git a/include/net/ip6_route.h b/include/net/ip6_route.h index 5efd0b71dc67..ca2d6b60e1ec 100644 --- a/include/net/ip6_route.h +++ b/include/net/ip6_route.h @@ -263,19 +263,19 @@ static inline bool ipv6_anycast_destination(const struct dst_entry *dst, int ip6_fragment(struct net *net, struct sock *sk, struct sk_buff *skb, int (*output)(struct net *, struct sock *, struct sk_buff *)); -static inline unsigned int ip6_skb_dst_mtu(struct sk_buff *skb) +static inline unsigned int ip6_skb_dst_mtu(const struct sk_buff *skb) { - unsigned int mtu; - - struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? + const struct ipv6_pinfo *np = skb->sk && !dev_recursion_level() ? inet6_sk(skb->sk) : NULL; + const struct dst_entry *dst = skb_dst(skb); + unsigned int mtu; if (np && np->pmtudisc >= IPV6_PMTUDISC_PROBE) { - mtu = READ_ONCE(skb_dst(skb)->dev->mtu); - mtu -= lwtunnel_headroom(skb_dst(skb)->lwtstate, mtu); - } else - mtu = dst_mtu(skb_dst(skb)); - + mtu = READ_ONCE(dst->dev->mtu); + mtu -= lwtunnel_headroom(dst->lwtstate, mtu); + } else { + mtu = dst_mtu(dst); + } return mtu; } diff --git a/include/net/ip6_tunnel.h b/include/net/ip6_tunnel.h index 028eaea1c854..a38c4f1e4e5c 100644 --- a/include/net/ip6_tunnel.h +++ b/include/net/ip6_tunnel.h @@ -46,6 +46,7 @@ struct __ip6_tnl_parm { struct ip6_tnl { struct ip6_tnl __rcu *next; /* next tunnel in list */ struct net_device *dev; /* virtual device associated with tunnel */ + netdevice_tracker dev_tracker; struct net *net; /* netns for packet i/o */ struct __ip6_tnl_parm parms; /* tunnel configuration parameters */ struct flowi fl; /* flowi template for xmit */ diff --git a/include/net/ip_fib.h b/include/net/ip_fib.h index 3417ba2d27ad..c4297704bbcb 100644 --- a/include/net/ip_fib.h +++ b/include/net/ip_fib.h @@ -79,6 +79,7 @@ struct fnhe_hash_bucket { struct fib_nh_common { struct net_device *nhc_dev; + netdevice_tracker nhc_dev_tracker; int nhc_oif; unsigned char nhc_scope; u8 nhc_family; @@ -111,6 +112,7 @@ struct fib_nh { int nh_saddr_genid; #define fib_nh_family nh_common.nhc_family #define fib_nh_dev nh_common.nhc_dev +#define fib_nh_dev_tracker nh_common.nhc_dev_tracker #define fib_nh_oif nh_common.nhc_oif #define fib_nh_flags nh_common.nhc_flags #define fib_nh_lws nh_common.nhc_lwtstate diff --git a/include/net/ip_tunnels.h b/include/net/ip_tunnels.h index bc3b13ec93c9..0219fe907b26 100644 --- a/include/net/ip_tunnels.h +++ b/include/net/ip_tunnels.h @@ -104,7 +104,10 @@ struct metadata_dst; struct ip_tunnel { struct ip_tunnel __rcu *next; struct hlist_node hash_node; + struct net_device *dev; + netdevice_tracker dev_tracker; + struct net *net; /* netns for packet i/o */ unsigned long err_time; /* Time when the last ICMP error diff --git a/include/net/ipv6.h b/include/net/ipv6.h index c19bf51ded1d..53ac7707ca70 100644 --- a/include/net/ipv6.h +++ b/include/net/ipv6.h @@ -345,9 +345,9 @@ struct ipcm6_cookie { struct sockcm_cookie sockc; __s16 hlimit; __s16 tclass; + __u16 gso_size; __s8 dontfrag; struct ipv6_txoptions *opt; - __u16 gso_size; }; static inline void ipcm6_init(struct ipcm6_cookie *ipc6) diff --git a/include/net/iucv/af_iucv.h b/include/net/iucv/af_iucv.h index ff06246dbbb9..df85d19fbf84 100644 --- a/include/net/iucv/af_iucv.h +++ b/include/net/iucv/af_iucv.h @@ -112,10 +112,12 @@ enum iucv_tx_notify { struct iucv_sock { struct sock sk; - char src_user_id[8]; - char src_name[8]; - char dst_user_id[8]; - char dst_name[8]; + struct_group(init, + char src_user_id[8]; + char src_name[8]; + char dst_user_id[8]; + char dst_name[8]; + ); struct list_head accept_q; spinlock_t accept_q_lock; struct sock *parent; diff --git a/include/net/llc_conn.h b/include/net/llc_conn.h index ea985aa7a6c5..2c1ea3414640 100644 --- a/include/net/llc_conn.h +++ b/include/net/llc_conn.h @@ -38,6 +38,7 @@ struct llc_sock { struct llc_addr laddr; /* lsap/mac pair */ struct llc_addr daddr; /* dsap/mac pair */ struct net_device *dev; /* device to send to remote */ + netdevice_tracker dev_tracker; u32 copied_seq; /* head of yet unread data */ u8 retry_count; /* number of retries */ u8 ack_must_be_send; diff --git a/include/net/ndisc.h b/include/net/ndisc.h index 04341d86585d..53cb8de0e589 100644 --- a/include/net/ndisc.h +++ b/include/net/ndisc.h @@ -411,13 +411,7 @@ static inline void __ipv6_confirm_neigh(struct net_device *dev, rcu_read_lock_bh(); n = __ipv6_neigh_lookup_noref(dev, pkey); - if (n) { - unsigned long now = jiffies; - - /* avoid dirtying neighbour */ - if (READ_ONCE(n->confirmed) != now) - WRITE_ONCE(n->confirmed, now); - } + neigh_confirm(n); rcu_read_unlock_bh(); } @@ -428,13 +422,7 @@ static inline void __ipv6_confirm_neigh_stub(struct net_device *dev, rcu_read_lock_bh(); n = __ipv6_neigh_lookup_noref_stub(dev, pkey); - if (n) { - unsigned long now = jiffies; - - /* avoid dirtying neighbour */ - if (READ_ONCE(n->confirmed) != now) - WRITE_ONCE(n->confirmed, now); - } + neigh_confirm(n); rcu_read_unlock_bh(); } diff --git a/include/net/neighbour.h b/include/net/neighbour.h index 38a0c1d24570..937389e04c8e 100644 --- a/include/net/neighbour.h +++ b/include/net/neighbour.h @@ -70,6 +70,7 @@ enum { struct neigh_parms { possible_net_t net; struct net_device *dev; + netdevice_tracker dev_tracker; struct list_head list; int (*neigh_setup)(struct neighbour *); struct neigh_table *tbl; @@ -158,6 +159,7 @@ struct neighbour { struct list_head managed_list; struct rcu_head rcu; struct net_device *dev; + netdevice_tracker dev_tracker; u8 primary_key[0]; } __randomize_layout; @@ -173,6 +175,7 @@ struct pneigh_entry { struct pneigh_entry *next; possible_net_t net; struct net_device *dev; + netdevice_tracker dev_tracker; u32 flags; u8 protocol; u8 key[]; @@ -321,6 +324,17 @@ static inline struct neighbour *__neigh_lookup_noref(struct neigh_table *tbl, return ___neigh_lookup_noref(tbl, tbl->key_eq, tbl->hash, pkey, dev); } +static inline void neigh_confirm(struct neighbour *n) +{ + if (n) { + unsigned long now = jiffies; + + /* avoid dirtying neighbour */ + if (READ_ONCE(n->confirmed) != now) + WRITE_ONCE(n->confirmed, now); + } +} + void neigh_table_init(int index, struct neigh_table *tbl); int neigh_table_clear(int index, struct neigh_table *tbl); struct neighbour *neigh_lookup(struct neigh_table *tbl, const void *pkey, diff --git a/include/net/netns/core.h b/include/net/netns/core.h index 36c2d998a43c..552bc25b1933 100644 --- a/include/net/netns/core.h +++ b/include/net/netns/core.h @@ -12,7 +12,6 @@ struct netns_core { int sysctl_somaxconn; #ifdef CONFIG_PROC_FS - int __percpu *sock_inuse; struct prot_inuse __percpu *prot_inuse; #endif }; diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h index 22179b2fda72..dbf202bb1a0e 100644 --- a/include/net/sch_generic.h +++ b/include/net/sch_generic.h @@ -125,7 +125,7 @@ struct Qdisc { spinlock_t seqlock; struct rcu_head rcu; - + netdevice_tracker dev_tracker; /* private data */ long privdata[] ____cacheline_aligned; }; diff --git a/include/net/sctp/structs.h b/include/net/sctp/structs.h index 899c29c326ba..e7d1ed7a3dfb 100644 --- a/include/net/sctp/structs.h +++ b/include/net/sctp/structs.h @@ -984,12 +984,10 @@ struct sctp_transport { } cacc; struct { - __u32 last_rtx_chunks; __u16 pmtu; __u16 probe_size; __u16 probe_high; - __u8 probe_count:3; - __u8 raise_count:5; + __u8 probe_count; __u8 state; } pl; /* plpmtud related */ @@ -1011,6 +1009,7 @@ void sctp_transport_reset_t3_rtx(struct sctp_transport *); void sctp_transport_reset_hb_timer(struct sctp_transport *); void sctp_transport_reset_reconf_timer(struct sctp_transport *transport); void sctp_transport_reset_probe_timer(struct sctp_transport *transport); +void sctp_transport_reset_raise_timer(struct sctp_transport *transport); int sctp_transport_hold(struct sctp_transport *); void sctp_transport_put(struct sctp_transport *); void sctp_transport_update_rto(struct sctp_transport *, __u32); @@ -1025,7 +1024,7 @@ bool sctp_transport_update_pmtu(struct sctp_transport *t, u32 pmtu); void sctp_transport_immediate_rtx(struct sctp_transport *); void sctp_transport_dst_release(struct sctp_transport *t); void sctp_transport_dst_confirm(struct sctp_transport *t); -bool sctp_transport_pl_send(struct sctp_transport *t); +void sctp_transport_pl_send(struct sctp_transport *t); bool sctp_transport_pl_recv(struct sctp_transport *t); diff --git a/include/net/sock.h b/include/net/sock.h index bea21ff70e74..ae61cd0b650d 100644 --- a/include/net/sock.h +++ b/include/net/sock.h @@ -63,6 +63,7 @@ #include <linux/indirect_call_wrapper.h> #include <linux/atomic.h> #include <linux/refcount.h> +#include <linux/llist.h> #include <net/dst.h> #include <net/checksum.h> #include <net/tcp_states.h> @@ -284,15 +285,14 @@ struct bpf_local_storage; * @sk_no_check_tx: %SO_NO_CHECK setting, set checksum in TX packets * @sk_no_check_rx: allow zero checksum in RX packets * @sk_route_caps: route capabilities (e.g. %NETIF_F_TSO) - * @sk_route_nocaps: forbidden route capabilities (e.g NETIF_F_GSO_MASK) - * @sk_route_forced_caps: static, forced route capabilities - * (set in tcp_init_sock()) + * @sk_gso_disabled: if set, NETIF_F_GSO_MASK is forbidden. * @sk_gso_type: GSO type (e.g. %SKB_GSO_TCPV4) * @sk_gso_max_size: Maximum GSO segment size to build * @sk_gso_max_segs: Maximum number of GSO segments * @sk_pacing_shift: scaling factor for TCP Small Queues * @sk_lingertime: %SO_LINGER l_linger setting * @sk_backlog: always used with the per-socket spinlock held + * @defer_list: head of llist storing skbs to be freed * @sk_callback_lock: used with the callbacks in the end of this struct * @sk_error_queue: rarely used * @sk_prot_creator: sk_prot of original sock creator (see ipv6_setsockopt, @@ -391,6 +391,11 @@ struct sock { #define sk_flags __sk_common.skc_flags #define sk_rxhash __sk_common.skc_rxhash + /* early demux fields */ + struct dst_entry *sk_rx_dst; + int sk_rx_dst_ifindex; + u32 sk_rx_dst_cookie; + socket_lock_t sk_lock; atomic_t sk_drops; int sk_rcvlowat; @@ -410,6 +415,8 @@ struct sock { struct sk_buff *head; struct sk_buff *tail; } sk_backlog; + struct llist_head defer_list; + #define sk_rmem_alloc sk_backlog.rmem_alloc int sk_forward_alloc; @@ -431,9 +438,6 @@ struct sock { #ifdef CONFIG_XFRM struct xfrm_policy __rcu *sk_policy[2]; #endif - struct dst_entry *sk_rx_dst; - int sk_rx_dst_ifindex; - u32 sk_rx_dst_cookie; struct dst_entry __rcu *sk_dst_cache; atomic_t sk_omem_alloc; @@ -460,8 +464,6 @@ struct sock { unsigned long sk_max_pacing_rate; struct page_frag sk_frag; netdev_features_t sk_route_caps; - netdev_features_t sk_route_nocaps; - netdev_features_t sk_route_forced_caps; int sk_gso_type; unsigned int sk_gso_max_size; gfp_t sk_allocation; @@ -471,7 +473,7 @@ struct sock { * Because of non atomicity rules, all * changes are protected by socket lock. */ - u8 sk_padding : 1, + u8 sk_gso_disabled : 1, sk_kern_sock : 1, sk_no_check_tx : 1, sk_no_check_rx : 1, @@ -493,6 +495,7 @@ struct sock { u16 sk_busy_poll_budget; #endif spinlock_t sk_peer_lock; + int sk_bind_phc; struct pid *sk_peer_pid; const struct cred *sk_peer_cred; @@ -502,7 +505,6 @@ struct sock { seqlock_t sk_stamp_seq; #endif u16 sk_tsflags; - int sk_bind_phc; u8 sk_shutdown; u32 sk_tskey; atomic_t sk_zckey; @@ -1022,12 +1024,18 @@ static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *s int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb); +INDIRECT_CALLABLE_DECLARE(int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)); +INDIRECT_CALLABLE_DECLARE(int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)); + static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) { if (sk_memalloc_socks() && skb_pfmemalloc(skb)) return __sk_backlog_rcv(sk, skb); - return sk->sk_backlog_rcv(sk, skb); + return INDIRECT_CALL_INET(sk->sk_backlog_rcv, + tcp_v6_do_rcv, + tcp_v4_do_rcv, + sk, skb); } static inline void sk_incoming_cpu_update(struct sock *sk) @@ -1210,7 +1218,9 @@ struct proto { unsigned int inuse_idx; #endif +#if IS_ENABLED(CONFIG_MPTCP) int (*forward_alloc_get)(const struct sock *sk); +#endif bool (*stream_memory_free)(const struct sock *sk, int wake); bool (*sock_is_readable)(struct sock *sk); @@ -1299,10 +1309,11 @@ INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int static inline int sk_forward_alloc_get(const struct sock *sk) { - if (!sk->sk_prot->forward_alloc_get) - return sk->sk_forward_alloc; - - return sk->sk_prot->forward_alloc_get(sk); +#if IS_ENABLED(CONFIG_MPTCP) + if (sk->sk_prot->forward_alloc_get) + return sk->sk_prot->forward_alloc_get(sk); +#endif + return sk->sk_forward_alloc; } static inline bool __sk_stream_memory_free(const struct sock *sk, int wake) @@ -1419,13 +1430,32 @@ proto_memory_pressure(struct proto *prot) #ifdef CONFIG_PROC_FS -/* Called with local bh disabled */ -void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc); +#define PROTO_INUSE_NR 64 /* should be enough for the first time */ +struct prot_inuse { + int all; + int val[PROTO_INUSE_NR]; +}; + +static inline void sock_prot_inuse_add(const struct net *net, + const struct proto *prot, int val) +{ + this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); +} + +static inline void sock_inuse_add(const struct net *net, int val) +{ + this_cpu_add(net->core.prot_inuse->all, val); +} + int sock_prot_inuse_get(struct net *net, struct proto *proto); int sock_inuse_get(struct net *net); #else -static inline void sock_prot_inuse_add(struct net *net, struct proto *prot, - int inc) +static inline void sock_prot_inuse_add(const struct net *net, + const struct proto *prot, int val) +{ +} + +static inline void sock_inuse_add(const struct net *net, int val) { } #endif @@ -2118,13 +2148,10 @@ static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n) { if (skb_get_dst_pending_confirm(skb)) { struct sock *sk = skb->sk; - unsigned long now = jiffies; - /* avoid dirtying neighbour */ - if (READ_ONCE(n->confirmed) != now) - WRITE_ONCE(n->confirmed, now); if (sk && READ_ONCE(sk->sk_dst_pending_confirm)) WRITE_ONCE(sk->sk_dst_pending_confirm, 0); + neigh_confirm(n); } } @@ -2137,10 +2164,10 @@ static inline bool sk_can_gso(const struct sock *sk) void sk_setup_caps(struct sock *sk, struct dst_entry *dst); -static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags) +static inline void sk_gso_disable(struct sock *sk) { - sk->sk_route_nocaps |= flags; - sk->sk_route_caps &= ~flags; + sk->sk_gso_disabled = 1; + sk->sk_route_caps &= ~NETIF_F_GSO_MASK; } static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb, @@ -2654,6 +2681,11 @@ static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags) &skb_shinfo(skb)->tskey); } +static inline bool sk_is_tcp(const struct sock *sk) +{ + return sk->sk_type == SOCK_STREAM && sk->sk_protocol == IPPROTO_TCP; +} + /** * sk_eat_skb - Release a skb if it is no longer needed * @sk: socket to eat this skb from diff --git a/include/net/tc_act/tc_mirred.h b/include/net/tc_act/tc_mirred.h index 1cace4c69e44..32ce8ea36950 100644 --- a/include/net/tc_act/tc_mirred.h +++ b/include/net/tc_act/tc_mirred.h @@ -10,6 +10,7 @@ struct tcf_mirred { int tcfm_eaction; bool tcfm_mac_header_xmit; struct net_device __rcu *tcfm_dev; + netdevice_tracker tcfm_dev_tracker; struct list_head tcfm_list; }; #define to_mirred(a) ((struct tcf_mirred *)a) diff --git a/include/net/tcp.h b/include/net/tcp.h index 4da22b41bde6..44e442bf23f9 100644 --- a/include/net/tcp.h +++ b/include/net/tcp.h @@ -1368,6 +1368,16 @@ static inline bool tcp_checksum_complete(struct sk_buff *skb) } bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb); + +void __sk_defer_free_flush(struct sock *sk); + +static inline void sk_defer_free_flush(struct sock *sk) +{ + if (llist_empty(&sk->defer_list)) + return; + __sk_defer_free_flush(sk); +} + int tcp_filter(struct sock *sk, struct sk_buff *skb); void tcp_set_state(struct sock *sk, int state); void tcp_done(struct sock *sk); @@ -2172,9 +2182,13 @@ static inline void tcp_segs_in(struct tcp_sock *tp, const struct sk_buff *skb) u16 segs_in; segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs); - tp->segs_in += segs_in; + + /* We update these fields while other threads might + * read them from tcp_get_info() + */ + WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in); if (skb->len > tcp_hdrlen(skb)) - tp->data_segs_in += segs_in; + WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in); } /* diff --git a/include/net/udp.h b/include/net/udp.h index 909ecf447e0f..f1c2a88c9005 100644 --- a/include/net/udp.h +++ b/include/net/udp.h @@ -167,36 +167,12 @@ static inline void udp_csum_pull_header(struct sk_buff *skb) typedef struct sock *(*udp_lookup_t)(const struct sk_buff *skb, __be16 sport, __be16 dport); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp4_gro_receive(struct list_head *, - struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int udp4_gro_complete(struct sk_buff *, int)); -INDIRECT_CALLABLE_DECLARE(struct sk_buff *udp6_gro_receive(struct list_head *, - struct sk_buff *)); -INDIRECT_CALLABLE_DECLARE(int udp6_gro_complete(struct sk_buff *, int)); INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *)); INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); -struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb, - struct udphdr *uh, struct sock *sk); -int udp_gro_complete(struct sk_buff *skb, int nhoff, udp_lookup_t lookup); - struct sk_buff *__udp_gso_segment(struct sk_buff *gso_skb, netdev_features_t features, bool is_ipv6); -static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb) -{ - struct udphdr *uh; - unsigned int hlen, off; - - off = skb_gro_offset(skb); - hlen = off + sizeof(*uh); - uh = skb_gro_header_fast(skb, off); - if (skb_gro_header_hard(skb, hlen)) - uh = skb_gro_header_slow(skb, hlen, off); - - return uh; -} - /* hash routines shared between UDPv4/6 and UDP-Litev4/6 */ static inline int udp_lib_hash(struct sock *sk) { diff --git a/include/net/vxlan.h b/include/net/vxlan.h index 08537aa14f7c..5a934bebe630 100644 --- a/include/net/vxlan.h +++ b/include/net/vxlan.h @@ -10,6 +10,7 @@ #include <net/nexthop.h> #define IANA_VXLAN_UDP_PORT 4789 +#define IANA_VXLAN_GPE_UDP_PORT 4790 /* VXLAN protocol (RFC 7348) header: * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ diff --git a/include/soc/mscc/ocelot.h b/include/soc/mscc/ocelot.h index fef3a36b0210..33f2e8c9e88b 100644 --- a/include/soc/mscc/ocelot.h +++ b/include/soc/mscc/ocelot.h @@ -555,12 +555,26 @@ struct ocelot_ops { u16 (*wm_enc)(u16 value); u16 (*wm_dec)(u16 value); void (*wm_stat)(u32 val, u32 *inuse, u32 *maxuse); + void (*psfp_init)(struct ocelot *ocelot); + int (*psfp_filter_add)(struct ocelot *ocelot, int port, + struct flow_cls_offload *f); + int (*psfp_filter_del)(struct ocelot *ocelot, struct flow_cls_offload *f); + int (*psfp_stats_get)(struct ocelot *ocelot, struct flow_cls_offload *f, + struct flow_stats *stats); + void (*cut_through_fwd)(struct ocelot *ocelot); +}; + +struct ocelot_vcap_policer { + struct list_head pol_list; + u16 base; + u16 max; + u16 base2; + u16 max2; }; struct ocelot_vcap_block { struct list_head rules; int count; - int pol_lpr; }; struct ocelot_bridge_vlan { @@ -581,6 +595,12 @@ enum ocelot_port_tag_config { OCELOT_PORT_TAG_TRUNK = 3, }; +struct ocelot_psfp_list { + struct list_head stream_list; + struct list_head sfi_list; + struct list_head sgi_list; +}; + enum ocelot_sb { OCELOT_SB_BUF, OCELOT_SB_REF, @@ -593,6 +613,19 @@ enum ocelot_sb_pool { OCELOT_SB_POOL_NUM, }; +/* MAC table entry types. + * ENTRYTYPE_NORMAL is subject to aging. + * ENTRYTYPE_LOCKED is not subject to aging. + * ENTRYTYPE_MACv4 is not subject to aging. For IPv4 multicast. + * ENTRYTYPE_MACv6 is not subject to aging. For IPv6 multicast. + */ +enum macaccess_entry_type { + ENTRYTYPE_NORMAL = 0, + ENTRYTYPE_LOCKED, + ENTRYTYPE_MACv4, + ENTRYTYPE_MACv6, +}; + #define OCELOT_QUIRK_PCS_PERFORMS_RATE_ADAPTATION BIT(0) #define OCELOT_QUIRK_QSGMII_PORTS_MUST_BE_UP BIT(1) @@ -623,6 +656,8 @@ struct ocelot_port { struct net_device *bridge; u8 stp_state; + + int speed; }; struct ocelot { @@ -667,8 +702,11 @@ struct ocelot { struct list_head dummy_rules; struct ocelot_vcap_block block[3]; + struct ocelot_vcap_policer vcap_pol; struct vcap_props *vcap; + struct ocelot_psfp_list psfp; + /* Workqueue to check statistics for overflow with its lock */ struct mutex stats_lock; u64 *stats; @@ -677,6 +715,8 @@ struct ocelot { /* Lock for serializing access to the MAC table */ struct mutex mact_lock; + /* Lock for serializing forwarding domain changes */ + struct mutex fwd_domain_lock; struct workqueue_struct *owq; @@ -776,7 +816,9 @@ void ocelot_set_ageing_time(struct ocelot *ocelot, unsigned int msecs); int ocelot_port_vlan_filtering(struct ocelot *ocelot, int port, bool enabled, struct netlink_ext_ack *extack); void ocelot_bridge_stp_state_set(struct ocelot *ocelot, int port, u8 state); -void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot); +u32 ocelot_get_dsa_8021q_cpu_mask(struct ocelot *ocelot); +u32 ocelot_get_bridge_fwd_mask(struct ocelot *ocelot, int src_port); +void ocelot_apply_bridge_fwd_mask(struct ocelot *ocelot, bool joining); int ocelot_port_pre_bridge_flags(struct ocelot *ocelot, int port, struct switchdev_brport_flags val); void ocelot_port_bridge_flags(struct ocelot *ocelot, int port, @@ -870,6 +912,19 @@ void ocelot_phylink_mac_link_up(struct ocelot *ocelot, int port, bool tx_pause, bool rx_pause, unsigned long quirks); +int ocelot_mact_lookup(struct ocelot *ocelot, int *dst_idx, + const unsigned char mac[ETH_ALEN], + unsigned int vid, enum macaccess_entry_type *type); +int ocelot_mact_learn_streamdata(struct ocelot *ocelot, int dst_idx, + const unsigned char mac[ETH_ALEN], + unsigned int vid, + enum macaccess_entry_type type, + int sfid, int ssid); + +int ocelot_vcap_policer_add(struct ocelot *ocelot, u32 pol_ix, + struct ocelot_policer *pol); +int ocelot_vcap_policer_del(struct ocelot *ocelot, u32 pol_ix); + #if IS_ENABLED(CONFIG_BRIDGE_MRP) int ocelot_mrp_add(struct ocelot *ocelot, int port, const struct switchdev_obj_mrp *mrp); diff --git a/include/soc/mscc/ocelot_ana.h b/include/soc/mscc/ocelot_ana.h index 1669481d9779..67e0ae05a5ab 100644 --- a/include/soc/mscc/ocelot_ana.h +++ b/include/soc/mscc/ocelot_ana.h @@ -227,6 +227,11 @@ #define ANA_TABLES_SFIDACCESS_SFID_TBL_CMD(x) ((x) & GENMASK(1, 0)) #define ANA_TABLES_SFIDACCESS_SFID_TBL_CMD_M GENMASK(1, 0) +#define SFIDACCESS_CMD_IDLE 0 +#define SFIDACCESS_CMD_READ 1 +#define SFIDACCESS_CMD_WRITE 2 +#define SFIDACCESS_CMD_INIT 3 + #define ANA_TABLES_SFIDTIDX_SGID_VALID BIT(26) #define ANA_TABLES_SFIDTIDX_SGID(x) (((x) << 18) & GENMASK(25, 18)) #define ANA_TABLES_SFIDTIDX_SGID_M GENMASK(25, 18) @@ -255,6 +260,11 @@ #define ANA_SG_CONFIG_REG_3_INIT_IPS(x) (((x) << 21) & GENMASK(24, 21)) #define ANA_SG_CONFIG_REG_3_INIT_IPS_M GENMASK(24, 21) #define ANA_SG_CONFIG_REG_3_INIT_IPS_X(x) (((x) & GENMASK(24, 21)) >> 21) +#define ANA_SG_CONFIG_REG_3_IPV_VALID BIT(24) +#define ANA_SG_CONFIG_REG_3_IPV_INVALID(x) (((x) << 24) & GENMASK(24, 24)) +#define ANA_SG_CONFIG_REG_3_INIT_IPV(x) (((x) << 21) & GENMASK(23, 21)) +#define ANA_SG_CONFIG_REG_3_INIT_IPV_M GENMASK(23, 21) +#define ANA_SG_CONFIG_REG_3_INIT_IPV_X(x) (((x) & GENMASK(23, 21)) >> 21) #define ANA_SG_CONFIG_REG_3_INIT_GATE_STATE BIT(25) #define ANA_SG_GCL_GS_CONFIG_RSZ 0x4 diff --git a/include/soc/mscc/ocelot_vcap.h b/include/soc/mscc/ocelot_vcap.h index 4d1dfa1136b2..709cbc198fd2 100644 --- a/include/soc/mscc/ocelot_vcap.h +++ b/include/soc/mscc/ocelot_vcap.h @@ -656,6 +656,7 @@ enum ocelot_vcap_filter_type { OCELOT_VCAP_FILTER_DUMMY, OCELOT_VCAP_FILTER_PAG, OCELOT_VCAP_FILTER_OFFLOAD, + OCELOT_PSFP_FILTER_OFFLOAD, }; struct ocelot_vcap_id { diff --git a/include/soc/mscc/vsc7514_regs.h b/include/soc/mscc/vsc7514_regs.h new file mode 100644 index 000000000000..98743e252012 --- /dev/null +++ b/include/soc/mscc/vsc7514_regs.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: (GPL-2.0 OR MIT) */ +/* + * Microsemi Ocelot Switch driver + * + * Copyright (c) 2021 Innovative Advantage Inc. + */ + +#ifndef VSC7514_REGS_H +#define VSC7514_REGS_H + +extern const u32 vsc7514_ana_regmap[]; +extern const u32 vsc7514_qs_regmap[]; +extern const u32 vsc7514_qsys_regmap[]; +extern const u32 vsc7514_rew_regmap[]; +extern const u32 vsc7514_sys_regmap[]; +extern const u32 vsc7514_vcap_regmap[]; +extern const u32 vsc7514_ptp_regmap[]; +extern const u32 vsc7514_dev_gmii_regmap[]; + +extern const struct vcap_field vsc7514_vcap_es0_keys[]; +extern const struct vcap_field vsc7514_vcap_es0_actions[]; +extern const struct vcap_field vsc7514_vcap_is1_keys[]; +extern const struct vcap_field vsc7514_vcap_is1_actions[]; +extern const struct vcap_field vsc7514_vcap_is2_keys[]; +extern const struct vcap_field vsc7514_vcap_is2_actions[]; + +#endif diff --git a/include/uapi/linux/bpf.h b/include/uapi/linux/bpf.h index ba5af15e25f5..6297eafdc40f 100644 --- a/include/uapi/linux/bpf.h +++ b/include/uapi/linux/bpf.h @@ -4938,6 +4938,25 @@ union bpf_attr { * **-ENOENT** if symbol is not found. * * **-EPERM** if caller does not have permission to obtain kernel address. + * + * long bpf_find_vma(struct task_struct *task, u64 addr, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * Find vma of *task* that contains *addr*, call *callback_fn* + * function with *task*, *vma*, and *callback_ctx*. + * The *callback_fn* should be a static function and + * the *callback_ctx* should be a pointer to the stack. + * The *flags* is used to control certain aspects of the helper. + * Currently, the *flags* must be 0. + * + * The expected callback signature is + * + * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); + * + * Return + * 0 on success. + * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. + * **-EBUSY** if failed to try lock mmap_lock. + * **-EINVAL** for invalid **flags**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5120,6 +5139,7 @@ union bpf_attr { FN(trace_vprintk), \ FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ + FN(find_vma), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -6296,6 +6316,7 @@ struct bpf_sk_lookup { __u32 local_ip4; /* Network byte order */ __u32 local_ip6[4]; /* Network byte order */ __u32 local_port; /* Host byte order */ + __u32 ingress_ifindex; /* The arriving interface. Determined by inet_iif. */ }; /* diff --git a/include/uapi/linux/btf.h b/include/uapi/linux/btf.h index deb12f755f0f..b0d8fea1951d 100644 --- a/include/uapi/linux/btf.h +++ b/include/uapi/linux/btf.h @@ -43,7 +43,7 @@ struct btf_type { * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, VAR and DECL_TAG. + * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. * "type" is a type_id referring to another type. */ union { @@ -75,6 +75,7 @@ enum { BTF_KIND_DATASEC = 15, /* Section */ BTF_KIND_FLOAT = 16, /* Floating point */ BTF_KIND_DECL_TAG = 17, /* Decl Tag */ + BTF_KIND_TYPE_TAG = 18, /* Type Tag */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, diff --git a/include/uapi/linux/ethtool.h b/include/uapi/linux/ethtool.h index a2223b685451..7bc4b8def12c 100644 --- a/include/uapi/linux/ethtool.h +++ b/include/uapi/linux/ethtool.h @@ -231,6 +231,7 @@ enum tunable_id { ETHTOOL_RX_COPYBREAK, ETHTOOL_TX_COPYBREAK, ETHTOOL_PFC_PREVENTION_TOUT, /* timeout in msecs */ + ETHTOOL_TX_COPYBREAK_BUF_SIZE, /* * Add your fresh new tunable attribute above and remember to update * tunable_strings[] in net/ethtool/common.c diff --git a/include/uapi/linux/ethtool_netlink.h b/include/uapi/linux/ethtool_netlink.h index 999777d32dcf..cca6e474a085 100644 --- a/include/uapi/linux/ethtool_netlink.h +++ b/include/uapi/linux/ethtool_netlink.h @@ -329,6 +329,7 @@ enum { ETHTOOL_A_RINGS_RX_MINI, /* u32 */ ETHTOOL_A_RINGS_RX_JUMBO, /* u32 */ ETHTOOL_A_RINGS_TX, /* u32 */ + ETHTOOL_A_RINGS_RX_BUF_LEN, /* u32 */ /* add new constants above here */ __ETHTOOL_A_RINGS_CNT, diff --git a/include/uapi/linux/if_link.h b/include/uapi/linux/if_link.h index eebd3894fe89..4ac53b30b6dc 100644 --- a/include/uapi/linux/if_link.h +++ b/include/uapi/linux/if_link.h @@ -858,6 +858,7 @@ enum { IFLA_BOND_TLB_DYNAMIC_LB, IFLA_BOND_PEER_NOTIF_DELAY, IFLA_BOND_AD_LACP_ACTIVE, + IFLA_BOND_MISSED_MAX, __IFLA_BOND_MAX, }; diff --git a/include/uapi/linux/tty.h b/include/uapi/linux/tty.h index 376cccf397be..a58deb3061eb 100644 --- a/include/uapi/linux/tty.h +++ b/include/uapi/linux/tty.h @@ -38,5 +38,6 @@ #define N_NCI 25 /* NFC NCI UART */ #define N_SPEAKUP 26 /* Speakup communication with synths */ #define N_NULL 27 /* Null ldisc used for error handling */ +#define N_MCTP 28 /* MCTP-over-serial */ #endif /* _UAPI_LINUX_TTY_H */ diff --git a/kernel/bpf/bpf_task_storage.c b/kernel/bpf/bpf_task_storage.c index ebfa8bc90892..bb69aea1a777 100644 --- a/kernel/bpf/bpf_task_storage.c +++ b/kernel/bpf/bpf_task_storage.c @@ -323,7 +323,7 @@ const struct bpf_func_proto bpf_task_storage_get_proto = { .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_task_struct_ids[0], + .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL, .arg4_type = ARG_ANYTHING, }; @@ -334,5 +334,5 @@ const struct bpf_func_proto bpf_task_storage_delete_proto = { .ret_type = RET_INTEGER, .arg1_type = ARG_CONST_MAP_PTR, .arg2_type = ARG_PTR_TO_BTF_ID, - .arg2_btf_id = &btf_task_struct_ids[0], + .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], }; diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 9bdb03767db5..48cdf5b425a7 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -282,6 +282,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DATASEC] = "DATASEC", [BTF_KIND_FLOAT] = "FLOAT", [BTF_KIND_DECL_TAG] = "DECL_TAG", + [BTF_KIND_TYPE_TAG] = "TYPE_TAG", }; const char *btf_type_str(const struct btf_type *t) @@ -418,6 +419,7 @@ static bool btf_type_is_modifier(const struct btf_type *t) case BTF_KIND_VOLATILE: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: return true; } @@ -1737,6 +1739,7 @@ __btf_resolve_size(const struct btf *btf, const struct btf_type *type, case BTF_KIND_VOLATILE: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: id = type->type; type = btf_type_by_id(btf, type->type); break; @@ -2345,6 +2348,8 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env, const struct btf_type *t, u32 meta_left) { + const char *value; + if (btf_type_vlen(t)) { btf_verifier_log_type(env, t, "vlen != 0"); return -EINVAL; @@ -2360,7 +2365,7 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env, return -EINVAL; } - /* typedef type must have a valid name, and other ref types, + /* typedef/type_tag type must have a valid name, and other ref types, * volatile, const, restrict, should have a null name. */ if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPEDEF) { @@ -2369,6 +2374,12 @@ static int btf_ref_type_check_meta(struct btf_verifier_env *env, btf_verifier_log_type(env, t, "Invalid name"); return -EINVAL; } + } else if (BTF_INFO_KIND(t->info) == BTF_KIND_TYPE_TAG) { + value = btf_name_by_offset(env->btf, t->name_off); + if (!value || !value[0]) { + btf_verifier_log_type(env, t, "Invalid name"); + return -EINVAL; + } } else { if (t->name_off) { btf_verifier_log_type(env, t, "Invalid name"); @@ -4059,6 +4070,7 @@ static const struct btf_kind_operations * const kind_ops[NR_BTF_KINDS] = { [BTF_KIND_DATASEC] = &datasec_ops, [BTF_KIND_FLOAT] = &float_ops, [BTF_KIND_DECL_TAG] = &decl_tag_ops, + [BTF_KIND_TYPE_TAG] = &modifier_ops, }; static s32 btf_check_meta(struct btf_verifier_env *env, @@ -6342,7 +6354,10 @@ const struct bpf_func_proto bpf_btf_find_by_name_kind_proto = { .arg4_type = ARG_ANYTHING, }; -BTF_ID_LIST_GLOBAL_SINGLE(btf_task_struct_ids, struct, task_struct) +BTF_ID_LIST_GLOBAL(btf_tracing_ids, MAX_BTF_TRACING_TYPE) +#define BTF_TRACING_TYPE(name, type) BTF_ID(struct, type) +BTF_TRACING_TYPE_xxx +#undef BTF_TRACING_TYPE /* BTF ID set registration API for modules */ diff --git a/kernel/bpf/mmap_unlock_work.h b/kernel/bpf/mmap_unlock_work.h new file mode 100644 index 000000000000..5d18d7d85bef --- /dev/null +++ b/kernel/bpf/mmap_unlock_work.h @@ -0,0 +1,65 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (c) 2021 Facebook + */ + +#ifndef __MMAP_UNLOCK_WORK_H__ +#define __MMAP_UNLOCK_WORK_H__ +#include <linux/irq_work.h> + +/* irq_work to run mmap_read_unlock() in irq_work */ +struct mmap_unlock_irq_work { + struct irq_work irq_work; + struct mm_struct *mm; +}; + +DECLARE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); + +/* + * We cannot do mmap_read_unlock() when the irq is disabled, because of + * risk to deadlock with rq_lock. To look up vma when the irqs are + * disabled, we need to run mmap_read_unlock() in irq_work. We use a + * percpu variable to do the irq_work. If the irq_work is already used + * by another lookup, we fall over. + */ +static inline bool bpf_mmap_unlock_get_irq_work(struct mmap_unlock_irq_work **work_ptr) +{ + struct mmap_unlock_irq_work *work = NULL; + bool irq_work_busy = false; + + if (irqs_disabled()) { + if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { + work = this_cpu_ptr(&mmap_unlock_work); + if (irq_work_is_busy(&work->irq_work)) { + /* cannot queue more up_read, fallback */ + irq_work_busy = true; + } + } else { + /* + * PREEMPT_RT does not allow to trylock mmap sem in + * interrupt disabled context. Force the fallback code. + */ + irq_work_busy = true; + } + } + + *work_ptr = work; + return irq_work_busy; +} + +static inline void bpf_mmap_unlock_mm(struct mmap_unlock_irq_work *work, struct mm_struct *mm) +{ + if (!work) { + mmap_read_unlock(mm); + } else { + work->mm = mm; + + /* The lock will be released once we're out of interrupt + * context. Tell lockdep that we've released it now so + * it doesn't complain that we forgot to release it. + */ + rwsem_release(&mm->mmap_lock.dep_map, _RET_IP_); + irq_work_queue(&work->irq_work); + } +} + +#endif /* __MMAP_UNLOCK_WORK_H__ */ diff --git a/kernel/bpf/stackmap.c b/kernel/bpf/stackmap.c index 6e75bbee39f0..49e567209c6b 100644 --- a/kernel/bpf/stackmap.c +++ b/kernel/bpf/stackmap.c @@ -7,10 +7,10 @@ #include <linux/kernel.h> #include <linux/stacktrace.h> #include <linux/perf_event.h> -#include <linux/irq_work.h> #include <linux/btf_ids.h> #include <linux/buildid.h> #include "percpu_freelist.h" +#include "mmap_unlock_work.h" #define STACK_CREATE_FLAG_MASK \ (BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY | \ @@ -31,25 +31,6 @@ struct bpf_stack_map { struct stack_map_bucket *buckets[]; }; -/* irq_work to run up_read() for build_id lookup in nmi context */ -struct stack_map_irq_work { - struct irq_work irq_work; - struct mm_struct *mm; -}; - -static void do_up_read(struct irq_work *entry) -{ - struct stack_map_irq_work *work; - - if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) - return; - - work = container_of(entry, struct stack_map_irq_work, irq_work); - mmap_read_unlock_non_owner(work->mm); -} - -static DEFINE_PER_CPU(struct stack_map_irq_work, up_read_work); - static inline bool stack_map_use_build_id(struct bpf_map *map) { return (map->map_flags & BPF_F_STACK_BUILD_ID); @@ -149,35 +130,13 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, u64 *ips, u32 trace_nr, bool user) { int i; + struct mmap_unlock_irq_work *work = NULL; + bool irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); struct vm_area_struct *vma; - bool irq_work_busy = false; - struct stack_map_irq_work *work = NULL; - - if (irqs_disabled()) { - if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { - work = this_cpu_ptr(&up_read_work); - if (irq_work_is_busy(&work->irq_work)) { - /* cannot queue more up_read, fallback */ - irq_work_busy = true; - } - } else { - /* - * PREEMPT_RT does not allow to trylock mmap sem in - * interrupt disabled context. Force the fallback code. - */ - irq_work_busy = true; - } - } - /* - * We cannot do up_read() when the irq is disabled, because of - * risk to deadlock with rq_lock. To do build_id lookup when the - * irqs are disabled, we need to run up_read() in irq_work. We use - * a percpu variable to do the irq_work. If the irq_work is - * already used by another lookup, we fall back to report ips. - * - * Same fallback is used for kernel stack (!user) on a stackmap - * with build_id. + /* If the irq_work is in use, fall back to report ips. Same + * fallback is used for kernel stack (!user) on a stackmap with + * build_id. */ if (!user || !current || !current->mm || irq_work_busy || !mmap_read_trylock(current->mm)) { @@ -203,19 +162,7 @@ static void stack_map_get_build_id_offset(struct bpf_stack_build_id *id_offs, - vma->vm_start; id_offs[i].status = BPF_STACK_BUILD_ID_VALID; } - - if (!work) { - mmap_read_unlock(current->mm); - } else { - work->mm = current->mm; - - /* The lock will be released once we're out of interrupt - * context. Tell lockdep that we've released it now so - * it doesn't complain that we forgot to release it. - */ - rwsem_release(¤t->mm->mmap_lock.dep_map, _RET_IP_); - irq_work_queue(&work->irq_work); - } + bpf_mmap_unlock_mm(work, current->mm); } static struct perf_callchain_entry * @@ -542,7 +489,7 @@ const struct bpf_func_proto bpf_get_task_stack_proto = { .gpl_only = false, .ret_type = RET_INTEGER, .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_task_struct_ids[0], + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .arg2_type = ARG_PTR_TO_UNINIT_MEM, .arg3_type = ARG_CONST_SIZE_OR_ZERO, .arg4_type = ARG_ANYTHING, @@ -719,16 +666,3 @@ const struct bpf_map_ops stack_trace_map_ops = { .map_btf_name = "bpf_stack_map", .map_btf_id = &stack_trace_map_btf_id, }; - -static int __init stack_map_init(void) -{ - int cpu; - struct stack_map_irq_work *work; - - for_each_possible_cpu(cpu) { - work = per_cpu_ptr(&up_read_work, cpu); - init_irq_work(&work->irq_work, do_up_read); - } - return 0; -} -subsys_initcall(stack_map_init); diff --git a/kernel/bpf/task_iter.c b/kernel/bpf/task_iter.c index b48750bfba5a..d94696198ef8 100644 --- a/kernel/bpf/task_iter.c +++ b/kernel/bpf/task_iter.c @@ -8,6 +8,7 @@ #include <linux/fdtable.h> #include <linux/filter.h> #include <linux/btf_ids.h> +#include "mmap_unlock_work.h" struct bpf_iter_seq_task_common { struct pid_namespace *ns; @@ -524,10 +525,6 @@ static const struct seq_operations task_vma_seq_ops = { .show = task_vma_seq_show, }; -BTF_ID_LIST(btf_task_file_ids) -BTF_ID(struct, file) -BTF_ID(struct, vm_area_struct) - static const struct bpf_iter_seq_info task_seq_info = { .seq_ops = &task_seq_ops, .init_seq_private = init_seq_pidns, @@ -586,23 +583,88 @@ static struct bpf_iter_reg task_vma_reg_info = { .seq_info = &task_vma_seq_info, }; +BPF_CALL_5(bpf_find_vma, struct task_struct *, task, u64, start, + bpf_callback_t, callback_fn, void *, callback_ctx, u64, flags) +{ + struct mmap_unlock_irq_work *work = NULL; + struct vm_area_struct *vma; + bool irq_work_busy = false; + struct mm_struct *mm; + int ret = -ENOENT; + + if (flags) + return -EINVAL; + + if (!task) + return -ENOENT; + + mm = task->mm; + if (!mm) + return -ENOENT; + + irq_work_busy = bpf_mmap_unlock_get_irq_work(&work); + + if (irq_work_busy || !mmap_read_trylock(mm)) + return -EBUSY; + + vma = find_vma(mm, start); + + if (vma && vma->vm_start <= start && vma->vm_end > start) { + callback_fn((u64)(long)task, (u64)(long)vma, + (u64)(long)callback_ctx, 0, 0); + ret = 0; + } + bpf_mmap_unlock_mm(work, mm); + return ret; +} + +const struct bpf_func_proto bpf_find_vma_proto = { + .func = bpf_find_vma, + .ret_type = RET_INTEGER, + .arg1_type = ARG_PTR_TO_BTF_ID, + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], + .arg2_type = ARG_ANYTHING, + .arg3_type = ARG_PTR_TO_FUNC, + .arg4_type = ARG_PTR_TO_STACK_OR_NULL, + .arg5_type = ARG_ANYTHING, +}; + +DEFINE_PER_CPU(struct mmap_unlock_irq_work, mmap_unlock_work); + +static void do_mmap_read_unlock(struct irq_work *entry) +{ + struct mmap_unlock_irq_work *work; + + if (WARN_ON_ONCE(IS_ENABLED(CONFIG_PREEMPT_RT))) + return; + + work = container_of(entry, struct mmap_unlock_irq_work, irq_work); + mmap_read_unlock_non_owner(work->mm); +} + static int __init task_iter_init(void) { - int ret; + struct mmap_unlock_irq_work *work; + int ret, cpu; + + for_each_possible_cpu(cpu) { + work = per_cpu_ptr(&mmap_unlock_work, cpu); + init_irq_work(&work->irq_work, do_mmap_read_unlock); + } - task_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; + task_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; ret = bpf_iter_reg_target(&task_reg_info); if (ret) return ret; - task_file_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_file_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[0]; + task_file_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; + task_file_reg_info.ctx_arg_info[1].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_FILE]; ret = bpf_iter_reg_target(&task_file_reg_info); if (ret) return ret; - task_vma_reg_info.ctx_arg_info[0].btf_id = btf_task_struct_ids[0]; - task_vma_reg_info.ctx_arg_info[1].btf_id = btf_task_file_ids[1]; + task_vma_reg_info.ctx_arg_info[0].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_TASK]; + task_vma_reg_info.ctx_arg_info[1].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA]; return bpf_iter_reg_target(&task_vma_reg_info); } late_initcall(task_iter_init); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index f3001937bbb9..90675b987148 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -6130,6 +6130,33 @@ static int set_timer_callback_state(struct bpf_verifier_env *env, return 0; } +static int set_find_vma_callback_state(struct bpf_verifier_env *env, + struct bpf_func_state *caller, + struct bpf_func_state *callee, + int insn_idx) +{ + /* bpf_find_vma(struct task_struct *task, u64 addr, + * void *callback_fn, void *callback_ctx, u64 flags) + * (callback_fn)(struct task_struct *task, + * struct vm_area_struct *vma, void *callback_ctx); + */ + callee->regs[BPF_REG_1] = caller->regs[BPF_REG_1]; + + callee->regs[BPF_REG_2].type = PTR_TO_BTF_ID; + __mark_reg_known_zero(&callee->regs[BPF_REG_2]); + callee->regs[BPF_REG_2].btf = btf_vmlinux; + callee->regs[BPF_REG_2].btf_id = btf_tracing_ids[BTF_TRACING_TYPE_VMA], + + /* pointer to stack or null */ + callee->regs[BPF_REG_3] = caller->regs[BPF_REG_4]; + + /* unused */ + __mark_reg_not_init(env, &callee->regs[BPF_REG_4]); + __mark_reg_not_init(env, &callee->regs[BPF_REG_5]); + callee->in_callback_fn = true; + return 0; +} + static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) { struct bpf_verifier_state *state = env->cur_state; @@ -6487,6 +6514,13 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return -EINVAL; } + if (func_id == BPF_FUNC_find_vma) { + err = __check_func_call(env, insn, insn_idx_p, meta.subprogno, + set_find_vma_callback_state); + if (err < 0) + return -EINVAL; + } + if (func_id == BPF_FUNC_snprintf) { err = check_bpf_snprintf_call(env, regs); if (err < 0) diff --git a/kernel/trace/bpf_trace.c b/kernel/trace/bpf_trace.c index ae9755037b7e..1cd647c010da 100644 --- a/kernel/trace/bpf_trace.c +++ b/kernel/trace/bpf_trace.c @@ -764,7 +764,7 @@ const struct bpf_func_proto bpf_get_current_task_btf_proto = { .func = bpf_get_current_task_btf, .gpl_only = true, .ret_type = RET_PTR_TO_BTF_ID, - .ret_btf_id = &btf_task_struct_ids[0], + .ret_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], }; BPF_CALL_1(bpf_task_pt_regs, struct task_struct *, task) @@ -779,7 +779,7 @@ const struct bpf_func_proto bpf_task_pt_regs_proto = { .func = bpf_task_pt_regs, .gpl_only = true, .arg1_type = ARG_PTR_TO_BTF_ID, - .arg1_btf_id = &btf_task_struct_ids[0], + .arg1_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK], .ret_type = RET_PTR_TO_BTF_ID, .ret_btf_id = &bpf_task_pt_regs_ids[0], }; @@ -1206,6 +1206,8 @@ bpf_tracing_func_proto(enum bpf_func_id func_id, const struct bpf_prog *prog) return &bpf_get_func_ip_proto_tracing; case BPF_FUNC_get_branch_snapshot: return &bpf_get_branch_snapshot_proto; + case BPF_FUNC_find_vma: + return &bpf_find_vma_proto; case BPF_FUNC_trace_vprintk: return bpf_get_trace_vprintk_proto(); default: diff --git a/lib/Kconfig b/lib/Kconfig index 5e7165e6a346..655b0e43f260 100644 --- a/lib/Kconfig +++ b/lib/Kconfig @@ -680,6 +680,11 @@ config STACK_HASH_ORDER Select the hash size as a power of 2 for the stackdepot hash table. Choose a lower value to reduce the memory impact. +config REF_TRACKER + bool + depends on STACKTRACE_SUPPORT + select STACKDEPOT + config SBITMAP bool diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 5e14e32056ad..c77fe36bb3d8 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -599,6 +599,11 @@ config DEBUG_MISC Say Y here if you need to enable miscellaneous debug code that should be under a more specific debug option but isn't. +menu "Networking Debugging" + +source "net/Kconfig.debug" + +endmenu # "Networking Debugging" menu "Memory Debugging" @@ -2107,6 +2112,16 @@ config BACKTRACE_SELF_TEST Say N if you are unsure. +config TEST_REF_TRACKER + tristate "Self test for reference tracker" + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + select REF_TRACKER + help + This option provides a kernel module performing tests + using reference tracker infrastructure. + + Say N if you are unsure. + config RBTREE_TEST tristate "Red-Black tree test" depends on DEBUG_KERNEL diff --git a/lib/Makefile b/lib/Makefile index 364c23f15578..b213a7bbf3fd 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -101,7 +101,7 @@ obj-$(CONFIG_TEST_LOCKUP) += test_lockup.o obj-$(CONFIG_TEST_HMM) += test_hmm.o obj-$(CONFIG_TEST_FREE_PAGES) += test_free_pages.o obj-$(CONFIG_KPROBES_SANITY_TEST) += test_kprobes.o - +obj-$(CONFIG_TEST_REF_TRACKER) += test_ref_tracker.o # # CFLAGS for compiling floating point code inside the kernel. x86/Makefile turns # off the generation of FPU/SSE* instructions for kernel proper but FPU_FLAGS @@ -270,6 +270,8 @@ obj-$(CONFIG_STACKDEPOT) += stackdepot.o KASAN_SANITIZE_stackdepot.o := n KCOV_INSTRUMENT_stackdepot.o := n +obj-$(CONFIG_REF_TRACKER) += ref_tracker.o + libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \ fdt_empty_tree.o fdt_addresses.o $(foreach file, $(libfdt_files), \ diff --git a/lib/ref_tracker.c b/lib/ref_tracker.c new file mode 100644 index 000000000000..0ae2e66dcf0f --- /dev/null +++ b/lib/ref_tracker.c @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <linux/export.h> +#include <linux/ref_tracker.h> +#include <linux/slab.h> +#include <linux/stacktrace.h> +#include <linux/stackdepot.h> + +#define REF_TRACKER_STACK_ENTRIES 16 + +struct ref_tracker { + struct list_head head; /* anchor into dir->list or dir->quarantine */ + bool dead; + depot_stack_handle_t alloc_stack_handle; + depot_stack_handle_t free_stack_handle; +}; + +void ref_tracker_dir_exit(struct ref_tracker_dir *dir) +{ + struct ref_tracker *tracker, *n; + unsigned long flags; + bool leak = false; + + spin_lock_irqsave(&dir->lock, flags); + list_for_each_entry_safe(tracker, n, &dir->quarantine, head) { + list_del(&tracker->head); + kfree(tracker); + dir->quarantine_avail++; + } + list_for_each_entry_safe(tracker, n, &dir->list, head) { + pr_err("leaked reference.\n"); + if (tracker->alloc_stack_handle) + stack_depot_print(tracker->alloc_stack_handle); + leak = true; + list_del(&tracker->head); + kfree(tracker); + } + spin_unlock_irqrestore(&dir->lock, flags); + WARN_ON_ONCE(leak); + WARN_ON_ONCE(refcount_read(&dir->untracked) != 1); +} +EXPORT_SYMBOL(ref_tracker_dir_exit); + +void ref_tracker_dir_print(struct ref_tracker_dir *dir, + unsigned int display_limit) +{ + struct ref_tracker *tracker; + unsigned long flags; + unsigned int i = 0; + + spin_lock_irqsave(&dir->lock, flags); + list_for_each_entry(tracker, &dir->list, head) { + if (i < display_limit) { + pr_err("leaked reference.\n"); + if (tracker->alloc_stack_handle) + stack_depot_print(tracker->alloc_stack_handle); + i++; + } else { + break; + } + } + spin_unlock_irqrestore(&dir->lock, flags); +} +EXPORT_SYMBOL(ref_tracker_dir_print); + +int ref_tracker_alloc(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp, + gfp_t gfp) +{ + unsigned long entries[REF_TRACKER_STACK_ENTRIES]; + struct ref_tracker *tracker; + unsigned int nr_entries; + unsigned long flags; + + *trackerp = tracker = kzalloc(sizeof(*tracker), gfp | __GFP_NOFAIL); + if (unlikely(!tracker)) { + pr_err_once("memory allocation failure, unreliable refcount tracker.\n"); + refcount_inc(&dir->untracked); + return -ENOMEM; + } + nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + nr_entries = filter_irq_stacks(entries, nr_entries); + tracker->alloc_stack_handle = stack_depot_save(entries, nr_entries, gfp); + + spin_lock_irqsave(&dir->lock, flags); + list_add(&tracker->head, &dir->list); + spin_unlock_irqrestore(&dir->lock, flags); + return 0; +} +EXPORT_SYMBOL_GPL(ref_tracker_alloc); + +int ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp) +{ + unsigned long entries[REF_TRACKER_STACK_ENTRIES]; + struct ref_tracker *tracker = *trackerp; + depot_stack_handle_t stack_handle; + unsigned int nr_entries; + unsigned long flags; + + if (!tracker) { + refcount_dec(&dir->untracked); + return -EEXIST; + } + nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 1); + nr_entries = filter_irq_stacks(entries, nr_entries); + stack_handle = stack_depot_save(entries, nr_entries, GFP_ATOMIC); + + spin_lock_irqsave(&dir->lock, flags); + if (tracker->dead) { + pr_err("reference already released.\n"); + if (tracker->alloc_stack_handle) { + pr_err("allocated in:\n"); + stack_depot_print(tracker->alloc_stack_handle); + } + if (tracker->free_stack_handle) { + pr_err("freed in:\n"); + stack_depot_print(tracker->free_stack_handle); + } + spin_unlock_irqrestore(&dir->lock, flags); + WARN_ON_ONCE(1); + return -EINVAL; + } + tracker->dead = true; + + tracker->free_stack_handle = stack_handle; + + list_move_tail(&tracker->head, &dir->quarantine); + if (!dir->quarantine_avail) { + tracker = list_first_entry(&dir->quarantine, struct ref_tracker, head); + list_del(&tracker->head); + } else { + dir->quarantine_avail--; + tracker = NULL; + } + spin_unlock_irqrestore(&dir->lock, flags); + + kfree(tracker); + return 0; +} +EXPORT_SYMBOL_GPL(ref_tracker_free); diff --git a/lib/test_ref_tracker.c b/lib/test_ref_tracker.c new file mode 100644 index 000000000000..19d7dec70cc6 --- /dev/null +++ b/lib/test_ref_tracker.c @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Referrence tracker self test. + * + * Copyright (c) 2021 Eric Dumazet <edumazet@google.com> + */ +#include <linux/init.h> +#include <linux/module.h> +#include <linux/delay.h> +#include <linux/ref_tracker.h> +#include <linux/slab.h> +#include <linux/timer.h> + +static struct ref_tracker_dir ref_dir; +static struct ref_tracker *tracker[20]; + +#define TRT_ALLOC(X) static noinline void \ + alloctest_ref_tracker_alloc##X(struct ref_tracker_dir *dir, \ + struct ref_tracker **trackerp) \ + { \ + ref_tracker_alloc(dir, trackerp, GFP_KERNEL); \ + } + +TRT_ALLOC(1) +TRT_ALLOC(2) +TRT_ALLOC(3) +TRT_ALLOC(4) +TRT_ALLOC(5) +TRT_ALLOC(6) +TRT_ALLOC(7) +TRT_ALLOC(8) +TRT_ALLOC(9) +TRT_ALLOC(10) +TRT_ALLOC(11) +TRT_ALLOC(12) +TRT_ALLOC(13) +TRT_ALLOC(14) +TRT_ALLOC(15) +TRT_ALLOC(16) +TRT_ALLOC(17) +TRT_ALLOC(18) +TRT_ALLOC(19) + +#undef TRT_ALLOC + +static noinline void +alloctest_ref_tracker_free(struct ref_tracker_dir *dir, + struct ref_tracker **trackerp) +{ + ref_tracker_free(dir, trackerp); +} + + +static struct timer_list test_ref_tracker_timer; +static atomic_t test_ref_timer_done = ATOMIC_INIT(0); + +static void test_ref_tracker_timer_func(struct timer_list *t) +{ + ref_tracker_alloc(&ref_dir, &tracker[0], GFP_ATOMIC); + atomic_set(&test_ref_timer_done, 1); +} + +static int __init test_ref_tracker_init(void) +{ + int i; + + ref_tracker_dir_init(&ref_dir, 100); + + timer_setup(&test_ref_tracker_timer, test_ref_tracker_timer_func, 0); + mod_timer(&test_ref_tracker_timer, jiffies + 1); + + alloctest_ref_tracker_alloc1(&ref_dir, &tracker[1]); + alloctest_ref_tracker_alloc2(&ref_dir, &tracker[2]); + alloctest_ref_tracker_alloc3(&ref_dir, &tracker[3]); + alloctest_ref_tracker_alloc4(&ref_dir, &tracker[4]); + alloctest_ref_tracker_alloc5(&ref_dir, &tracker[5]); + alloctest_ref_tracker_alloc6(&ref_dir, &tracker[6]); + alloctest_ref_tracker_alloc7(&ref_dir, &tracker[7]); + alloctest_ref_tracker_alloc8(&ref_dir, &tracker[8]); + alloctest_ref_tracker_alloc9(&ref_dir, &tracker[9]); + alloctest_ref_tracker_alloc10(&ref_dir, &tracker[10]); + alloctest_ref_tracker_alloc11(&ref_dir, &tracker[11]); + alloctest_ref_tracker_alloc12(&ref_dir, &tracker[12]); + alloctest_ref_tracker_alloc13(&ref_dir, &tracker[13]); + alloctest_ref_tracker_alloc14(&ref_dir, &tracker[14]); + alloctest_ref_tracker_alloc15(&ref_dir, &tracker[15]); + alloctest_ref_tracker_alloc16(&ref_dir, &tracker[16]); + alloctest_ref_tracker_alloc17(&ref_dir, &tracker[17]); + alloctest_ref_tracker_alloc18(&ref_dir, &tracker[18]); + alloctest_ref_tracker_alloc19(&ref_dir, &tracker[19]); + + /* free all trackers but first 0 and 1. */ + for (i = 2; i < ARRAY_SIZE(tracker); i++) + alloctest_ref_tracker_free(&ref_dir, &tracker[i]); + + /* Attempt to free an already freed tracker. */ + alloctest_ref_tracker_free(&ref_dir, &tracker[2]); + + while (!atomic_read(&test_ref_timer_done)) + msleep(1); + + /* This should warn about tracker[0] & tracker[1] being not freed. */ + ref_tracker_dir_exit(&ref_dir); + + return 0; +} + +static void __exit test_ref_tracker_exit(void) +{ +} + +module_init(test_ref_tracker_init); +module_exit(test_ref_tracker_exit); + +MODULE_LICENSE("GPL v2"); diff --git a/net/802/hippi.c b/net/802/hippi.c index 887e73d520e4..1997b7dd265e 100644 --- a/net/802/hippi.c +++ b/net/802/hippi.c @@ -65,7 +65,7 @@ static int hippi_header(struct sk_buff *skb, struct net_device *dev, hip->le.src_addr_type = 2; /* 12 bit SC address */ memcpy(hip->le.src_switch_addr, dev->dev_addr + 3, 3); - memset(&hip->le.reserved, 0, 16); + memset_startat(&hip->le, 0, reserved); hip->snap.dsap = HIPPI_EXTENDED_SAP; hip->snap.ssap = HIPPI_EXTENDED_SAP; diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index abaa5d96ded2..788076b002b3 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -319,8 +319,8 @@ static void vlan_transfer_features(struct net_device *dev, { struct vlan_dev_priv *vlan = vlan_dev_priv(vlandev); - vlandev->gso_max_size = dev->gso_max_size; - vlandev->gso_max_segs = dev->gso_max_segs; + netif_set_gso_max_size(vlandev, dev->gso_max_size); + netif_set_gso_max_segs(vlandev, dev->gso_max_segs); if (vlan_hw_offload_capable(dev->features, vlan->vlan_proto)) vlandev->hard_header_len = dev->hard_header_len; diff --git a/net/8021q/vlan_core.c b/net/8021q/vlan_core.c index 59bc13b5f14f..acf8c791f320 100644 --- a/net/8021q/vlan_core.c +++ b/net/8021q/vlan_core.c @@ -476,10 +476,9 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head, type = vhdr->h_vlan_encapsulated_proto; - rcu_read_lock(); ptype = gro_find_receive_by_type(type); if (!ptype) - goto out_unlock; + goto out; flush = 0; @@ -501,8 +500,6 @@ static struct sk_buff *vlan_gro_receive(struct list_head *head, ipv6_gro_receive, inet_gro_receive, head, skb); -out_unlock: - rcu_read_unlock(); out: skb_gro_flush_final(skb, pp, flush); @@ -516,14 +513,12 @@ static int vlan_gro_complete(struct sk_buff *skb, int nhoff) struct packet_offload *ptype; int err = -ENOENT; - rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype) err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, ipv6_gro_complete, inet_gro_complete, skb, nhoff + sizeof(*vhdr)); - rcu_read_unlock(); return err; } diff --git a/net/8021q/vlan_dev.c b/net/8021q/vlan_dev.c index a54535cbcf4c..26d031a43cc1 100644 --- a/net/8021q/vlan_dev.c +++ b/net/8021q/vlan_dev.c @@ -573,8 +573,8 @@ static int vlan_dev_init(struct net_device *dev) NETIF_F_ALL_FCOE; dev->features |= dev->hw_features | NETIF_F_LLTX; - dev->gso_max_size = real_dev->gso_max_size; - dev->gso_max_segs = real_dev->gso_max_segs; + netif_set_gso_max_size(dev, real_dev->gso_max_size); + netif_set_gso_max_segs(dev, real_dev->gso_max_segs); if (dev->features & NETIF_F_VLAN_FEATURES) netdev_warn(real_dev, "VLAN features are set incorrectly. Q-in-Q configurations may not work correctly.\n"); @@ -616,7 +616,7 @@ static int vlan_dev_init(struct net_device *dev) return -ENOMEM; /* Get vlan's reference to real_dev */ - dev_hold(real_dev); + dev_hold_track(real_dev, &vlan->dev_tracker, GFP_KERNEL); return 0; } @@ -848,7 +848,7 @@ static void vlan_dev_free(struct net_device *dev) vlan->vlan_pcpu_stats = NULL; /* Get rid of the vlan's reference to real_dev */ - dev_put(vlan->real_dev); + dev_put_track(vlan->real_dev, &vlan->dev_tracker); } void vlan_setup(struct net_device *dev) diff --git a/net/8021q/vlanproc.c b/net/8021q/vlanproc.c index ec87dea23719..08bf6c839e25 100644 --- a/net/8021q/vlanproc.c +++ b/net/8021q/vlanproc.c @@ -252,7 +252,7 @@ static int vlandev_seq_show(struct seq_file *seq, void *offset) stats = dev_get_stats(vlandev, &temp); seq_printf(seq, - "%s VID: %d REORDER_HDR: %i dev->priv_flags: %hx\n", + "%s VID: %d REORDER_HDR: %i dev->priv_flags: %llx\n", vlandev->name, vlan->vlan_id, (int)(vlan->flags & 1), vlandev->priv_flags); diff --git a/net/Kconfig b/net/Kconfig index 074472dfa94a..8a1f9d0287de 100644 --- a/net/Kconfig +++ b/net/Kconfig @@ -455,4 +455,9 @@ config ETHTOOL_NETLINK netlink. It provides better extensibility and some new features, e.g. notification messages. +config NETDEV_ADDR_LIST_TEST + tristate "Unit tests for device address list" + default KUNIT_ALL_TESTS + depends on KUNIT + endif # if NET diff --git a/net/Kconfig.debug b/net/Kconfig.debug new file mode 100644 index 000000000000..fb5c70e01cb3 --- /dev/null +++ b/net/Kconfig.debug @@ -0,0 +1,10 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config NET_DEV_REFCNT_TRACKER + bool "Enable net device refcount tracking" + depends on DEBUG_KERNEL && STACKTRACE_SUPPORT + select REF_TRACKER + default n + help + Enable debugging feature to track device references. + This adds memory and cpu costs. diff --git a/net/ax25/ax25_dev.c b/net/ax25/ax25_dev.c index d0a043a51848..256fadb94df3 100644 --- a/net/ax25/ax25_dev.c +++ b/net/ax25/ax25_dev.c @@ -58,7 +58,7 @@ void ax25_dev_device_up(struct net_device *dev) dev->ax25_ptr = ax25_dev; ax25_dev->dev = dev; - dev_hold(dev); + dev_hold_track(dev, &ax25_dev->dev_tracker, GFP_ATOMIC); ax25_dev->forward = NULL; ax25_dev->values[AX25_VALUES_IPDEFMODE] = AX25_DEF_IPDEFMODE; @@ -114,7 +114,7 @@ void ax25_dev_device_down(struct net_device *dev) ax25_dev_list = s->next; spin_unlock_bh(&ax25_dev_lock); dev->ax25_ptr = NULL; - dev_put(dev); + dev_put_track(dev, &ax25_dev->dev_tracker); kfree(ax25_dev); return; } @@ -124,7 +124,7 @@ void ax25_dev_device_down(struct net_device *dev) s->next = ax25_dev->next; spin_unlock_bh(&ax25_dev_lock); dev->ax25_ptr = NULL; - dev_put(dev); + dev_put_track(dev, &ax25_dev->dev_tracker); kfree(ax25_dev); return; } @@ -188,7 +188,7 @@ void __exit ax25_dev_free(void) ax25_dev = ax25_dev_list; while (ax25_dev != NULL) { s = ax25_dev; - dev_put(ax25_dev->dev); + dev_put_track(ax25_dev->dev, &ax25_dev->dev_tracker); ax25_dev = ax25_dev->next; kfree(s); } diff --git a/net/bluetooth/Makefile b/net/bluetooth/Makefile index 291770fc9551..a52bba8500e1 100644 --- a/net/bluetooth/Makefile +++ b/net/bluetooth/Makefile @@ -15,7 +15,7 @@ bluetooth_6lowpan-y := 6lowpan.o bluetooth-y := af_bluetooth.o hci_core.o hci_conn.o hci_event.o mgmt.o \ hci_sock.o hci_sysfs.o l2cap_core.o l2cap_sock.o smp.o lib.o \ ecdh_helper.o hci_request.o mgmt_util.o mgmt_config.o hci_codec.o \ - eir.o + eir.o hci_sync.o bluetooth-$(CONFIG_BT_BREDR) += sco.o bluetooth-$(CONFIG_BT_HS) += a2mp.o amp.o diff --git a/net/bluetooth/aosp.c b/net/bluetooth/aosp.c index a1b7762335a5..432ae3aac9e3 100644 --- a/net/bluetooth/aosp.c +++ b/net/bluetooth/aosp.c @@ -8,9 +8,43 @@ #include "aosp.h" +/* Command complete parameters of LE_Get_Vendor_Capabilities_Command + * The parameters grow over time. The base version that declares the + * version_supported field is v0.95. Refer to + * https://cs.android.com/android/platform/superproject/+/master:system/ + * bt/gd/hci/controller.cc;l=452?q=le_get_vendor_capabilities_handler + */ +struct aosp_rp_le_get_vendor_capa { + /* v0.95: 15 octets */ + __u8 status; + __u8 max_advt_instances; + __u8 offloaded_resolution_of_private_address; + __le16 total_scan_results_storage; + __u8 max_irk_list_sz; + __u8 filtering_support; + __u8 max_filter; + __u8 activity_energy_info_support; + __le16 version_supported; + __le16 total_num_of_advt_tracked; + __u8 extended_scan_support; + __u8 debug_logging_supported; + /* v0.96: 16 octets */ + __u8 le_address_generation_offloading_support; + /* v0.98: 21 octets */ + __le32 a2dp_source_offload_capability_mask; + __u8 bluetooth_quality_report_support; + /* v1.00: 25 octets */ + __le32 dynamic_audio_buffer_support; +} __packed; + +#define VENDOR_CAPA_BASE_SIZE 15 +#define VENDOR_CAPA_0_98_SIZE 21 + void aosp_do_open(struct hci_dev *hdev) { struct sk_buff *skb; + struct aosp_rp_le_get_vendor_capa *rp; + u16 version_supported; if (!hdev->aosp_capable) return; @@ -20,9 +54,54 @@ void aosp_do_open(struct hci_dev *hdev) /* LE Get Vendor Capabilities Command */ skb = __hci_cmd_sync(hdev, hci_opcode_pack(0x3f, 0x153), 0, NULL, HCI_CMD_TIMEOUT); - if (IS_ERR(skb)) + if (IS_ERR(skb)) { + bt_dev_err(hdev, "AOSP get vendor capabilities (%ld)", + PTR_ERR(skb)); return; + } + + /* A basic length check */ + if (skb->len < VENDOR_CAPA_BASE_SIZE) + goto length_error; + + rp = (struct aosp_rp_le_get_vendor_capa *)skb->data; + + version_supported = le16_to_cpu(rp->version_supported); + /* AOSP displays the verion number like v0.98, v1.00, etc. */ + bt_dev_info(hdev, "AOSP extensions version v%u.%02u", + version_supported >> 8, version_supported & 0xff); + + /* Do not support very old versions. */ + if (version_supported < 95) { + bt_dev_warn(hdev, "AOSP capabilities version %u too old", + version_supported); + goto done; + } + + if (version_supported < 98) { + bt_dev_warn(hdev, "AOSP quality report is not supported"); + goto done; + } + + if (skb->len < VENDOR_CAPA_0_98_SIZE) + goto length_error; + + /* The bluetooth_quality_report_support is defined at version + * v0.98. Refer to + * https://cs.android.com/android/platform/superproject/+/ + * master:system/bt/gd/hci/controller.cc;l=477 + */ + if (rp->bluetooth_quality_report_support) { + hdev->aosp_quality_report = true; + bt_dev_info(hdev, "AOSP quality report is supported"); + } + + goto done; + +length_error: + bt_dev_err(hdev, "AOSP capabilities length %d too short", skb->len); +done: kfree_skb(skb); } @@ -33,3 +112,90 @@ void aosp_do_close(struct hci_dev *hdev) bt_dev_dbg(hdev, "Cleanup of AOSP extension"); } + +/* BQR command */ +#define BQR_OPCODE hci_opcode_pack(0x3f, 0x015e) + +/* BQR report action */ +#define REPORT_ACTION_ADD 0x00 +#define REPORT_ACTION_DELETE 0x01 +#define REPORT_ACTION_CLEAR 0x02 + +/* BQR event masks */ +#define QUALITY_MONITORING BIT(0) +#define APPRAOCHING_LSTO BIT(1) +#define A2DP_AUDIO_CHOPPY BIT(2) +#define SCO_VOICE_CHOPPY BIT(3) + +#define DEFAULT_BQR_EVENT_MASK (QUALITY_MONITORING | APPRAOCHING_LSTO | \ + A2DP_AUDIO_CHOPPY | SCO_VOICE_CHOPPY) + +/* Reporting at milliseconds so as not to stress the controller too much. + * Range: 0 ~ 65535 ms + */ +#define DEFALUT_REPORT_INTERVAL_MS 5000 + +struct aosp_bqr_cp { + __u8 report_action; + __u32 event_mask; + __u16 min_report_interval; +} __packed; + +static int enable_quality_report(struct hci_dev *hdev) +{ + struct sk_buff *skb; + struct aosp_bqr_cp cp; + + cp.report_action = REPORT_ACTION_ADD; + cp.event_mask = DEFAULT_BQR_EVENT_MASK; + cp.min_report_interval = DEFALUT_REPORT_INTERVAL_MS; + + skb = __hci_cmd_sync(hdev, BQR_OPCODE, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Enabling Android BQR failed (%ld)", + PTR_ERR(skb)); + return PTR_ERR(skb); + } + + kfree_skb(skb); + return 0; +} + +static int disable_quality_report(struct hci_dev *hdev) +{ + struct sk_buff *skb; + struct aosp_bqr_cp cp = { 0 }; + + cp.report_action = REPORT_ACTION_CLEAR; + + skb = __hci_cmd_sync(hdev, BQR_OPCODE, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Disabling Android BQR failed (%ld)", + PTR_ERR(skb)); + return PTR_ERR(skb); + } + + kfree_skb(skb); + return 0; +} + +bool aosp_has_quality_report(struct hci_dev *hdev) +{ + return hdev->aosp_quality_report; +} + +int aosp_set_quality_report(struct hci_dev *hdev, bool enable) +{ + if (!aosp_has_quality_report(hdev)) + return -EOPNOTSUPP; + + bt_dev_dbg(hdev, "quality report enable %d", enable); + + /* Enable or disable the quality report feature. */ + if (enable) + return enable_quality_report(hdev); + else + return disable_quality_report(hdev); +} diff --git a/net/bluetooth/aosp.h b/net/bluetooth/aosp.h index 328fc6d39f70..2fd8886d51b2 100644 --- a/net/bluetooth/aosp.h +++ b/net/bluetooth/aosp.h @@ -8,9 +8,22 @@ void aosp_do_open(struct hci_dev *hdev); void aosp_do_close(struct hci_dev *hdev); +bool aosp_has_quality_report(struct hci_dev *hdev); +int aosp_set_quality_report(struct hci_dev *hdev, bool enable); + #else static inline void aosp_do_open(struct hci_dev *hdev) {} static inline void aosp_do_close(struct hci_dev *hdev) {} +static inline bool aosp_has_quality_report(struct hci_dev *hdev) +{ + return false; +} + +static inline int aosp_set_quality_report(struct hci_dev *hdev, bool enable) +{ + return -EOPNOTSUPP; +} + #endif diff --git a/net/bluetooth/cmtp/core.c b/net/bluetooth/cmtp/core.c index 0a2d78e811cf..83eb84e8e688 100644 --- a/net/bluetooth/cmtp/core.c +++ b/net/bluetooth/cmtp/core.c @@ -501,9 +501,7 @@ static int __init cmtp_init(void) { BT_INFO("CMTP (CAPI Emulation) ver %s", VERSION); - cmtp_init_sockets(); - - return 0; + return cmtp_init_sockets(); } static void __exit cmtp_exit(void) diff --git a/net/bluetooth/hci_codec.c b/net/bluetooth/hci_codec.c index f0421d0edaa3..38201532f58e 100644 --- a/net/bluetooth/hci_codec.c +++ b/net/bluetooth/hci_codec.c @@ -25,9 +25,11 @@ static int hci_codec_list_add(struct list_head *list, } entry->transport = sent->transport; entry->len = len; - entry->num_caps = rp->num_caps; - if (rp->num_caps) + entry->num_caps = 0; + if (rp) { + entry->num_caps = rp->num_caps; memcpy(entry->caps, caps, len); + } list_add(&entry->list, list); return 0; @@ -58,6 +60,18 @@ static void hci_read_codec_capabilities(struct hci_dev *hdev, __u8 transport, __u32 len; cmd->transport = i; + + /* If Read_Codec_Capabilities command is not supported + * then just add codec to the list without caps + */ + if (!(hdev->commands[45] & 0x08)) { + hci_dev_lock(hdev); + hci_codec_list_add(&hdev->local_codecs, cmd, + NULL, NULL, 0); + hci_dev_unlock(hdev); + continue; + } + skb = __hci_cmd_sync(hdev, HCI_OP_READ_LOCAL_CODEC_CAPS, sizeof(*cmd), cmd, HCI_CMD_TIMEOUT); diff --git a/net/bluetooth/hci_conn.c b/net/bluetooth/hci_conn.c index bd669c95b9a7..cd6e1cf7e396 100644 --- a/net/bluetooth/hci_conn.c +++ b/net/bluetooth/hci_conn.c @@ -108,7 +108,7 @@ static void hci_connect_le_scan_cleanup(struct hci_conn *conn) break; } - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); } static void hci_conn_cleanup(struct hci_conn *conn) @@ -900,25 +900,15 @@ void hci_le_conn_failed(struct hci_conn *conn, u8 status) hci_conn_del(conn); - /* The suspend notifier is waiting for all devices to disconnect and an - * LE connect cancel will result in an hci_le_conn_failed. Once the last - * connection is deleted, we should also wake the suspend queue to - * complete suspend operations. - */ - if (list_empty(&hdev->conn_hash.list) && - test_and_clear_bit(SUSPEND_DISCONNECTING, hdev->suspend_tasks)) { - wake_up(&hdev->suspend_wait_q); - } - /* Since we may have temporarily stopped the background scanning in * favor of connection establishment, we should restart it. */ - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); - /* Re-enable advertising in case this was a failed connection + /* Enable advertising in case this was a failed connection * attempt as a peripheral. */ - hci_req_reenable_advertising(hdev); + hci_enable_advertising(hdev); } static void create_le_conn_complete(struct hci_dev *hdev, u8 status, u16 opcode) @@ -1411,7 +1401,7 @@ struct hci_conn *hci_connect_le_scan(struct hci_dev *hdev, bdaddr_t *dst, conn->conn_timeout = conn_timeout; conn->conn_reason = conn_reason; - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); done: hci_conn_hold(conn); diff --git a/net/bluetooth/hci_core.c b/net/bluetooth/hci_core.c index 8d33aa64846b..fdc0dcf8ee36 100644 --- a/net/bluetooth/hci_core.c +++ b/net/bluetooth/hci_core.c @@ -62,824 +62,6 @@ DEFINE_MUTEX(hci_cb_list_lock); /* HCI ID Numbering */ static DEFINE_IDA(hci_index_ida); -static int hci_reset_req(struct hci_request *req, unsigned long opt) -{ - BT_DBG("%s %ld", req->hdev->name, opt); - - /* Reset device */ - set_bit(HCI_RESET, &req->hdev->flags); - hci_req_add(req, HCI_OP_RESET, 0, NULL); - return 0; -} - -static void bredr_init(struct hci_request *req) -{ - req->hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_PACKET_BASED; - - /* Read Local Supported Features */ - hci_req_add(req, HCI_OP_READ_LOCAL_FEATURES, 0, NULL); - - /* Read Local Version */ - hci_req_add(req, HCI_OP_READ_LOCAL_VERSION, 0, NULL); - - /* Read BD Address */ - hci_req_add(req, HCI_OP_READ_BD_ADDR, 0, NULL); -} - -static void amp_init1(struct hci_request *req) -{ - req->hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_BLOCK_BASED; - - /* Read Local Version */ - hci_req_add(req, HCI_OP_READ_LOCAL_VERSION, 0, NULL); - - /* Read Local Supported Commands */ - hci_req_add(req, HCI_OP_READ_LOCAL_COMMANDS, 0, NULL); - - /* Read Local AMP Info */ - hci_req_add(req, HCI_OP_READ_LOCAL_AMP_INFO, 0, NULL); - - /* Read Data Blk size */ - hci_req_add(req, HCI_OP_READ_DATA_BLOCK_SIZE, 0, NULL); - - /* Read Flow Control Mode */ - hci_req_add(req, HCI_OP_READ_FLOW_CONTROL_MODE, 0, NULL); - - /* Read Location Data */ - hci_req_add(req, HCI_OP_READ_LOCATION_DATA, 0, NULL); -} - -static int amp_init2(struct hci_request *req) -{ - /* Read Local Supported Features. Not all AMP controllers - * support this so it's placed conditionally in the second - * stage init. - */ - if (req->hdev->commands[14] & 0x20) - hci_req_add(req, HCI_OP_READ_LOCAL_FEATURES, 0, NULL); - - return 0; -} - -static int hci_init1_req(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - BT_DBG("%s %ld", hdev->name, opt); - - /* Reset */ - if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) - hci_reset_req(req, 0); - - switch (hdev->dev_type) { - case HCI_PRIMARY: - bredr_init(req); - break; - case HCI_AMP: - amp_init1(req); - break; - default: - bt_dev_err(hdev, "Unknown device type %d", hdev->dev_type); - break; - } - - return 0; -} - -static void bredr_setup(struct hci_request *req) -{ - __le16 param; - __u8 flt_type; - - /* Read Buffer Size (ACL mtu, max pkt, etc.) */ - hci_req_add(req, HCI_OP_READ_BUFFER_SIZE, 0, NULL); - - /* Read Class of Device */ - hci_req_add(req, HCI_OP_READ_CLASS_OF_DEV, 0, NULL); - - /* Read Local Name */ - hci_req_add(req, HCI_OP_READ_LOCAL_NAME, 0, NULL); - - /* Read Voice Setting */ - hci_req_add(req, HCI_OP_READ_VOICE_SETTING, 0, NULL); - - /* Read Number of Supported IAC */ - hci_req_add(req, HCI_OP_READ_NUM_SUPPORTED_IAC, 0, NULL); - - /* Read Current IAC LAP */ - hci_req_add(req, HCI_OP_READ_CURRENT_IAC_LAP, 0, NULL); - - /* Clear Event Filters */ - flt_type = HCI_FLT_CLEAR_ALL; - hci_req_add(req, HCI_OP_SET_EVENT_FLT, 1, &flt_type); - - /* Connection accept timeout ~20 secs */ - param = cpu_to_le16(0x7d00); - hci_req_add(req, HCI_OP_WRITE_CA_TIMEOUT, 2, ¶m); -} - -static void le_setup(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - - /* Read LE Buffer Size */ - hci_req_add(req, HCI_OP_LE_READ_BUFFER_SIZE, 0, NULL); - - /* Read LE Local Supported Features */ - hci_req_add(req, HCI_OP_LE_READ_LOCAL_FEATURES, 0, NULL); - - /* Read LE Supported States */ - hci_req_add(req, HCI_OP_LE_READ_SUPPORTED_STATES, 0, NULL); - - /* LE-only controllers have LE implicitly enabled */ - if (!lmp_bredr_capable(hdev)) - hci_dev_set_flag(hdev, HCI_LE_ENABLED); -} - -static void hci_setup_event_mask(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - - /* The second byte is 0xff instead of 0x9f (two reserved bits - * disabled) since a Broadcom 1.2 dongle doesn't respond to the - * command otherwise. - */ - u8 events[8] = { 0xff, 0xff, 0xfb, 0xff, 0x00, 0x00, 0x00, 0x00 }; - - /* CSR 1.1 dongles does not accept any bitfield so don't try to set - * any event mask for pre 1.2 devices. - */ - if (hdev->hci_ver < BLUETOOTH_VER_1_2) - return; - - if (lmp_bredr_capable(hdev)) { - events[4] |= 0x01; /* Flow Specification Complete */ - } else { - /* Use a different default for LE-only devices */ - memset(events, 0, sizeof(events)); - events[1] |= 0x20; /* Command Complete */ - events[1] |= 0x40; /* Command Status */ - events[1] |= 0x80; /* Hardware Error */ - - /* If the controller supports the Disconnect command, enable - * the corresponding event. In addition enable packet flow - * control related events. - */ - if (hdev->commands[0] & 0x20) { - events[0] |= 0x10; /* Disconnection Complete */ - events[2] |= 0x04; /* Number of Completed Packets */ - events[3] |= 0x02; /* Data Buffer Overflow */ - } - - /* If the controller supports the Read Remote Version - * Information command, enable the corresponding event. - */ - if (hdev->commands[2] & 0x80) - events[1] |= 0x08; /* Read Remote Version Information - * Complete - */ - - if (hdev->le_features[0] & HCI_LE_ENCRYPTION) { - events[0] |= 0x80; /* Encryption Change */ - events[5] |= 0x80; /* Encryption Key Refresh Complete */ - } - } - - if (lmp_inq_rssi_capable(hdev) || - test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks)) - events[4] |= 0x02; /* Inquiry Result with RSSI */ - - if (lmp_ext_feat_capable(hdev)) - events[4] |= 0x04; /* Read Remote Extended Features Complete */ - - if (lmp_esco_capable(hdev)) { - events[5] |= 0x08; /* Synchronous Connection Complete */ - events[5] |= 0x10; /* Synchronous Connection Changed */ - } - - if (lmp_sniffsubr_capable(hdev)) - events[5] |= 0x20; /* Sniff Subrating */ - - if (lmp_pause_enc_capable(hdev)) - events[5] |= 0x80; /* Encryption Key Refresh Complete */ - - if (lmp_ext_inq_capable(hdev)) - events[5] |= 0x40; /* Extended Inquiry Result */ - - if (lmp_no_flush_capable(hdev)) - events[7] |= 0x01; /* Enhanced Flush Complete */ - - if (lmp_lsto_capable(hdev)) - events[6] |= 0x80; /* Link Supervision Timeout Changed */ - - if (lmp_ssp_capable(hdev)) { - events[6] |= 0x01; /* IO Capability Request */ - events[6] |= 0x02; /* IO Capability Response */ - events[6] |= 0x04; /* User Confirmation Request */ - events[6] |= 0x08; /* User Passkey Request */ - events[6] |= 0x10; /* Remote OOB Data Request */ - events[6] |= 0x20; /* Simple Pairing Complete */ - events[7] |= 0x04; /* User Passkey Notification */ - events[7] |= 0x08; /* Keypress Notification */ - events[7] |= 0x10; /* Remote Host Supported - * Features Notification - */ - } - - if (lmp_le_capable(hdev)) - events[7] |= 0x20; /* LE Meta-Event */ - - hci_req_add(req, HCI_OP_SET_EVENT_MASK, sizeof(events), events); -} - -static int hci_init2_req(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - if (hdev->dev_type == HCI_AMP) - return amp_init2(req); - - if (lmp_bredr_capable(hdev)) - bredr_setup(req); - else - hci_dev_clear_flag(hdev, HCI_BREDR_ENABLED); - - if (lmp_le_capable(hdev)) - le_setup(req); - - /* All Bluetooth 1.2 and later controllers should support the - * HCI command for reading the local supported commands. - * - * Unfortunately some controllers indicate Bluetooth 1.2 support, - * but do not have support for this command. If that is the case, - * the driver can quirk the behavior and skip reading the local - * supported commands. - */ - if (hdev->hci_ver > BLUETOOTH_VER_1_1 && - !test_bit(HCI_QUIRK_BROKEN_LOCAL_COMMANDS, &hdev->quirks)) - hci_req_add(req, HCI_OP_READ_LOCAL_COMMANDS, 0, NULL); - - if (lmp_ssp_capable(hdev)) { - /* When SSP is available, then the host features page - * should also be available as well. However some - * controllers list the max_page as 0 as long as SSP - * has not been enabled. To achieve proper debugging - * output, force the minimum max_page to 1 at least. - */ - hdev->max_page = 0x01; - - if (hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) { - u8 mode = 0x01; - - hci_req_add(req, HCI_OP_WRITE_SSP_MODE, - sizeof(mode), &mode); - } else { - struct hci_cp_write_eir cp; - - memset(hdev->eir, 0, sizeof(hdev->eir)); - memset(&cp, 0, sizeof(cp)); - - hci_req_add(req, HCI_OP_WRITE_EIR, sizeof(cp), &cp); - } - } - - if (lmp_inq_rssi_capable(hdev) || - test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks)) { - u8 mode; - - /* If Extended Inquiry Result events are supported, then - * they are clearly preferred over Inquiry Result with RSSI - * events. - */ - mode = lmp_ext_inq_capable(hdev) ? 0x02 : 0x01; - - hci_req_add(req, HCI_OP_WRITE_INQUIRY_MODE, 1, &mode); - } - - if (lmp_inq_tx_pwr_capable(hdev)) - hci_req_add(req, HCI_OP_READ_INQ_RSP_TX_POWER, 0, NULL); - - if (lmp_ext_feat_capable(hdev)) { - struct hci_cp_read_local_ext_features cp; - - cp.page = 0x01; - hci_req_add(req, HCI_OP_READ_LOCAL_EXT_FEATURES, - sizeof(cp), &cp); - } - - if (hci_dev_test_flag(hdev, HCI_LINK_SECURITY)) { - u8 enable = 1; - hci_req_add(req, HCI_OP_WRITE_AUTH_ENABLE, sizeof(enable), - &enable); - } - - return 0; -} - -static void hci_setup_link_policy(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_def_link_policy cp; - u16 link_policy = 0; - - if (lmp_rswitch_capable(hdev)) - link_policy |= HCI_LP_RSWITCH; - if (lmp_hold_capable(hdev)) - link_policy |= HCI_LP_HOLD; - if (lmp_sniff_capable(hdev)) - link_policy |= HCI_LP_SNIFF; - if (lmp_park_capable(hdev)) - link_policy |= HCI_LP_PARK; - - cp.policy = cpu_to_le16(link_policy); - hci_req_add(req, HCI_OP_WRITE_DEF_LINK_POLICY, sizeof(cp), &cp); -} - -static void hci_set_le_support(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_le_host_supported cp; - - /* LE-only devices do not support explicit enablement */ - if (!lmp_bredr_capable(hdev)) - return; - - memset(&cp, 0, sizeof(cp)); - - if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { - cp.le = 0x01; - cp.simul = 0x00; - } - - if (cp.le != lmp_host_le_capable(hdev)) - hci_req_add(req, HCI_OP_WRITE_LE_HOST_SUPPORTED, sizeof(cp), - &cp); -} - -static void hci_set_event_mask_page_2(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - u8 events[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; - bool changed = false; - - /* If Connectionless Peripheral Broadcast central role is supported - * enable all necessary events for it. - */ - if (lmp_cpb_central_capable(hdev)) { - events[1] |= 0x40; /* Triggered Clock Capture */ - events[1] |= 0x80; /* Synchronization Train Complete */ - events[2] |= 0x10; /* Peripheral Page Response Timeout */ - events[2] |= 0x20; /* CPB Channel Map Change */ - changed = true; - } - - /* If Connectionless Peripheral Broadcast peripheral role is supported - * enable all necessary events for it. - */ - if (lmp_cpb_peripheral_capable(hdev)) { - events[2] |= 0x01; /* Synchronization Train Received */ - events[2] |= 0x02; /* CPB Receive */ - events[2] |= 0x04; /* CPB Timeout */ - events[2] |= 0x08; /* Truncated Page Complete */ - changed = true; - } - - /* Enable Authenticated Payload Timeout Expired event if supported */ - if (lmp_ping_capable(hdev) || hdev->le_features[0] & HCI_LE_PING) { - events[2] |= 0x80; - changed = true; - } - - /* Some Broadcom based controllers indicate support for Set Event - * Mask Page 2 command, but then actually do not support it. Since - * the default value is all bits set to zero, the command is only - * required if the event mask has to be changed. In case no change - * to the event mask is needed, skip this command. - */ - if (changed) - hci_req_add(req, HCI_OP_SET_EVENT_MASK_PAGE_2, - sizeof(events), events); -} - -static int hci_init3_req(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - u8 p; - - hci_setup_event_mask(req); - - if (hdev->commands[6] & 0x20 && - !test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks)) { - struct hci_cp_read_stored_link_key cp; - - bacpy(&cp.bdaddr, BDADDR_ANY); - cp.read_all = 0x01; - hci_req_add(req, HCI_OP_READ_STORED_LINK_KEY, sizeof(cp), &cp); - } - - if (hdev->commands[5] & 0x10) - hci_setup_link_policy(req); - - if (hdev->commands[8] & 0x01) - hci_req_add(req, HCI_OP_READ_PAGE_SCAN_ACTIVITY, 0, NULL); - - if (hdev->commands[18] & 0x04 && - !test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) - hci_req_add(req, HCI_OP_READ_DEF_ERR_DATA_REPORTING, 0, NULL); - - /* Some older Broadcom based Bluetooth 1.2 controllers do not - * support the Read Page Scan Type command. Check support for - * this command in the bit mask of supported commands. - */ - if (hdev->commands[13] & 0x01) - hci_req_add(req, HCI_OP_READ_PAGE_SCAN_TYPE, 0, NULL); - - if (lmp_le_capable(hdev)) { - u8 events[8]; - - memset(events, 0, sizeof(events)); - - if (hdev->le_features[0] & HCI_LE_ENCRYPTION) - events[0] |= 0x10; /* LE Long Term Key Request */ - - /* If controller supports the Connection Parameters Request - * Link Layer Procedure, enable the corresponding event. - */ - if (hdev->le_features[0] & HCI_LE_CONN_PARAM_REQ_PROC) - events[0] |= 0x20; /* LE Remote Connection - * Parameter Request - */ - - /* If the controller supports the Data Length Extension - * feature, enable the corresponding event. - */ - if (hdev->le_features[0] & HCI_LE_DATA_LEN_EXT) - events[0] |= 0x40; /* LE Data Length Change */ - - /* If the controller supports LL Privacy feature, enable - * the corresponding event. - */ - if (hdev->le_features[0] & HCI_LE_LL_PRIVACY) - events[1] |= 0x02; /* LE Enhanced Connection - * Complete - */ - - /* If the controller supports Extended Scanner Filter - * Policies, enable the corresponding event. - */ - if (hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY) - events[1] |= 0x04; /* LE Direct Advertising - * Report - */ - - /* If the controller supports Channel Selection Algorithm #2 - * feature, enable the corresponding event. - */ - if (hdev->le_features[1] & HCI_LE_CHAN_SEL_ALG2) - events[2] |= 0x08; /* LE Channel Selection - * Algorithm - */ - - /* If the controller supports the LE Set Scan Enable command, - * enable the corresponding advertising report event. - */ - if (hdev->commands[26] & 0x08) - events[0] |= 0x02; /* LE Advertising Report */ - - /* If the controller supports the LE Create Connection - * command, enable the corresponding event. - */ - if (hdev->commands[26] & 0x10) - events[0] |= 0x01; /* LE Connection Complete */ - - /* If the controller supports the LE Connection Update - * command, enable the corresponding event. - */ - if (hdev->commands[27] & 0x04) - events[0] |= 0x04; /* LE Connection Update - * Complete - */ - - /* If the controller supports the LE Read Remote Used Features - * command, enable the corresponding event. - */ - if (hdev->commands[27] & 0x20) - events[0] |= 0x08; /* LE Read Remote Used - * Features Complete - */ - - /* If the controller supports the LE Read Local P-256 - * Public Key command, enable the corresponding event. - */ - if (hdev->commands[34] & 0x02) - events[0] |= 0x80; /* LE Read Local P-256 - * Public Key Complete - */ - - /* If the controller supports the LE Generate DHKey - * command, enable the corresponding event. - */ - if (hdev->commands[34] & 0x04) - events[1] |= 0x01; /* LE Generate DHKey Complete */ - - /* If the controller supports the LE Set Default PHY or - * LE Set PHY commands, enable the corresponding event. - */ - if (hdev->commands[35] & (0x20 | 0x40)) - events[1] |= 0x08; /* LE PHY Update Complete */ - - /* If the controller supports LE Set Extended Scan Parameters - * and LE Set Extended Scan Enable commands, enable the - * corresponding event. - */ - if (use_ext_scan(hdev)) - events[1] |= 0x10; /* LE Extended Advertising - * Report - */ - - /* If the controller supports the LE Extended Advertising - * command, enable the corresponding event. - */ - if (ext_adv_capable(hdev)) - events[2] |= 0x02; /* LE Advertising Set - * Terminated - */ - - hci_req_add(req, HCI_OP_LE_SET_EVENT_MASK, sizeof(events), - events); - - /* Read LE Advertising Channel TX Power */ - if ((hdev->commands[25] & 0x40) && !ext_adv_capable(hdev)) { - /* HCI TS spec forbids mixing of legacy and extended - * advertising commands wherein READ_ADV_TX_POWER is - * also included. So do not call it if extended adv - * is supported otherwise controller will return - * COMMAND_DISALLOWED for extended commands. - */ - hci_req_add(req, HCI_OP_LE_READ_ADV_TX_POWER, 0, NULL); - } - - if (hdev->commands[38] & 0x80) { - /* Read LE Min/Max Tx Power*/ - hci_req_add(req, HCI_OP_LE_READ_TRANSMIT_POWER, - 0, NULL); - } - - if (hdev->commands[26] & 0x40) { - /* Read LE Accept List Size */ - hci_req_add(req, HCI_OP_LE_READ_ACCEPT_LIST_SIZE, - 0, NULL); - } - - if (hdev->commands[26] & 0x80) { - /* Clear LE Accept List */ - hci_req_add(req, HCI_OP_LE_CLEAR_ACCEPT_LIST, 0, NULL); - } - - if (hdev->commands[34] & 0x40) { - /* Read LE Resolving List Size */ - hci_req_add(req, HCI_OP_LE_READ_RESOLV_LIST_SIZE, - 0, NULL); - } - - if (hdev->commands[34] & 0x20) { - /* Clear LE Resolving List */ - hci_req_add(req, HCI_OP_LE_CLEAR_RESOLV_LIST, 0, NULL); - } - - if (hdev->commands[35] & 0x04) { - __le16 rpa_timeout = cpu_to_le16(hdev->rpa_timeout); - - /* Set RPA timeout */ - hci_req_add(req, HCI_OP_LE_SET_RPA_TIMEOUT, 2, - &rpa_timeout); - } - - if (hdev->le_features[0] & HCI_LE_DATA_LEN_EXT) { - /* Read LE Maximum Data Length */ - hci_req_add(req, HCI_OP_LE_READ_MAX_DATA_LEN, 0, NULL); - - /* Read LE Suggested Default Data Length */ - hci_req_add(req, HCI_OP_LE_READ_DEF_DATA_LEN, 0, NULL); - } - - if (ext_adv_capable(hdev)) { - /* Read LE Number of Supported Advertising Sets */ - hci_req_add(req, HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS, - 0, NULL); - } - - hci_set_le_support(req); - } - - /* Read features beyond page 1 if available */ - for (p = 2; p < HCI_MAX_PAGES && p <= hdev->max_page; p++) { - struct hci_cp_read_local_ext_features cp; - - cp.page = p; - hci_req_add(req, HCI_OP_READ_LOCAL_EXT_FEATURES, - sizeof(cp), &cp); - } - - return 0; -} - -static int hci_init4_req(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - /* Some Broadcom based Bluetooth controllers do not support the - * Delete Stored Link Key command. They are clearly indicating its - * absence in the bit mask of supported commands. - * - * Check the supported commands and only if the command is marked - * as supported send it. If not supported assume that the controller - * does not have actual support for stored link keys which makes this - * command redundant anyway. - * - * Some controllers indicate that they support handling deleting - * stored link keys, but they don't. The quirk lets a driver - * just disable this command. - */ - if (hdev->commands[6] & 0x80 && - !test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks)) { - struct hci_cp_delete_stored_link_key cp; - - bacpy(&cp.bdaddr, BDADDR_ANY); - cp.delete_all = 0x01; - hci_req_add(req, HCI_OP_DELETE_STORED_LINK_KEY, - sizeof(cp), &cp); - } - - /* Set event mask page 2 if the HCI command for it is supported */ - if (hdev->commands[22] & 0x04) - hci_set_event_mask_page_2(req); - - /* Read local pairing options if the HCI command is supported */ - if (hdev->commands[41] & 0x08) - hci_req_add(req, HCI_OP_READ_LOCAL_PAIRING_OPTS, 0, NULL); - - /* Get MWS transport configuration if the HCI command is supported */ - if (hdev->commands[30] & 0x08) - hci_req_add(req, HCI_OP_GET_MWS_TRANSPORT_CONFIG, 0, NULL); - - /* Check for Synchronization Train support */ - if (lmp_sync_train_capable(hdev)) - hci_req_add(req, HCI_OP_READ_SYNC_TRAIN_PARAMS, 0, NULL); - - /* Enable Secure Connections if supported and configured */ - if (hci_dev_test_flag(hdev, HCI_SSP_ENABLED) && - bredr_sc_enabled(hdev)) { - u8 support = 0x01; - - hci_req_add(req, HCI_OP_WRITE_SC_SUPPORT, - sizeof(support), &support); - } - - /* Set erroneous data reporting if supported to the wideband speech - * setting value - */ - if (hdev->commands[18] & 0x08 && - !test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) { - bool enabled = hci_dev_test_flag(hdev, - HCI_WIDEBAND_SPEECH_ENABLED); - - if (enabled != - (hdev->err_data_reporting == ERR_DATA_REPORTING_ENABLED)) { - struct hci_cp_write_def_err_data_reporting cp; - - cp.err_data_reporting = enabled ? - ERR_DATA_REPORTING_ENABLED : - ERR_DATA_REPORTING_DISABLED; - - hci_req_add(req, HCI_OP_WRITE_DEF_ERR_DATA_REPORTING, - sizeof(cp), &cp); - } - } - - /* Set Suggested Default Data Length to maximum if supported */ - if (hdev->le_features[0] & HCI_LE_DATA_LEN_EXT) { - struct hci_cp_le_write_def_data_len cp; - - cp.tx_len = cpu_to_le16(hdev->le_max_tx_len); - cp.tx_time = cpu_to_le16(hdev->le_max_tx_time); - hci_req_add(req, HCI_OP_LE_WRITE_DEF_DATA_LEN, sizeof(cp), &cp); - } - - /* Set Default PHY parameters if command is supported */ - if (hdev->commands[35] & 0x20) { - struct hci_cp_le_set_default_phy cp; - - cp.all_phys = 0x00; - cp.tx_phys = hdev->le_tx_def_phys; - cp.rx_phys = hdev->le_rx_def_phys; - - hci_req_add(req, HCI_OP_LE_SET_DEFAULT_PHY, sizeof(cp), &cp); - } - - return 0; -} - -static int __hci_init(struct hci_dev *hdev) -{ - int err; - - err = __hci_req_sync(hdev, hci_init1_req, 0, HCI_INIT_TIMEOUT, NULL); - if (err < 0) - return err; - - if (hci_dev_test_flag(hdev, HCI_SETUP)) - hci_debugfs_create_basic(hdev); - - err = __hci_req_sync(hdev, hci_init2_req, 0, HCI_INIT_TIMEOUT, NULL); - if (err < 0) - return err; - - /* HCI_PRIMARY covers both single-mode LE, BR/EDR and dual-mode - * BR/EDR/LE type controllers. AMP controllers only need the - * first two stages of init. - */ - if (hdev->dev_type != HCI_PRIMARY) - return 0; - - err = __hci_req_sync(hdev, hci_init3_req, 0, HCI_INIT_TIMEOUT, NULL); - if (err < 0) - return err; - - err = __hci_req_sync(hdev, hci_init4_req, 0, HCI_INIT_TIMEOUT, NULL); - if (err < 0) - return err; - - /* Read local codec list if the HCI command is supported */ - if (hdev->commands[45] & 0x04) - hci_read_supported_codecs_v2(hdev); - else if (hdev->commands[29] & 0x20) - hci_read_supported_codecs(hdev); - - /* This function is only called when the controller is actually in - * configured state. When the controller is marked as unconfigured, - * this initialization procedure is not run. - * - * It means that it is possible that a controller runs through its - * setup phase and then discovers missing settings. If that is the - * case, then this function will not be called. It then will only - * be called during the config phase. - * - * So only when in setup phase or config phase, create the debugfs - * entries and register the SMP channels. - */ - if (!hci_dev_test_flag(hdev, HCI_SETUP) && - !hci_dev_test_flag(hdev, HCI_CONFIG)) - return 0; - - hci_debugfs_create_common(hdev); - - if (lmp_bredr_capable(hdev)) - hci_debugfs_create_bredr(hdev); - - if (lmp_le_capable(hdev)) - hci_debugfs_create_le(hdev); - - return 0; -} - -static int hci_init0_req(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - BT_DBG("%s %ld", hdev->name, opt); - - /* Reset */ - if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) - hci_reset_req(req, 0); - - /* Read Local Version */ - hci_req_add(req, HCI_OP_READ_LOCAL_VERSION, 0, NULL); - - /* Read BD Address */ - if (hdev->set_bdaddr) - hci_req_add(req, HCI_OP_READ_BD_ADDR, 0, NULL); - - return 0; -} - -static int __hci_unconf_init(struct hci_dev *hdev) -{ - int err; - - if (test_bit(HCI_QUIRK_RAW_DEVICE, &hdev->quirks)) - return 0; - - err = __hci_req_sync(hdev, hci_init0_req, 0, HCI_INIT_TIMEOUT, NULL); - if (err < 0) - return err; - - if (hci_dev_test_flag(hdev, HCI_SETUP)) - hci_debugfs_create_basic(hdev); - - return 0; -} - static int hci_scan_req(struct hci_request *req, unsigned long opt) { __u8 scan = opt; @@ -975,7 +157,7 @@ void hci_discovery_set_state(struct hci_dev *hdev, int state) switch (state) { case DISCOVERY_STOPPED: - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); if (old_state != DISCOVERY_STARTING) mgmt_discovering(hdev, 0); @@ -1289,32 +471,6 @@ done: return err; } -/** - * hci_dev_get_bd_addr_from_property - Get the Bluetooth Device Address - * (BD_ADDR) for a HCI device from - * a firmware node property. - * @hdev: The HCI device - * - * Search the firmware node for 'local-bd-address'. - * - * All-zero BD addresses are rejected, because those could be properties - * that exist in the firmware tables, but were not updated by the firmware. For - * example, the DTS could define 'local-bd-address', with zero BD addresses. - */ -static void hci_dev_get_bd_addr_from_property(struct hci_dev *hdev) -{ - struct fwnode_handle *fwnode = dev_fwnode(hdev->dev.parent); - bdaddr_t ba; - int ret; - - ret = fwnode_property_read_u8_array(fwnode, "local-bd-address", - (u8 *)&ba, sizeof(ba)); - if (ret < 0 || !bacmp(&ba, BDADDR_ANY)) - return; - - bacpy(&hdev->public_addr, &ba); -} - static int hci_dev_do_open(struct hci_dev *hdev) { int ret = 0; @@ -1323,205 +479,8 @@ static int hci_dev_do_open(struct hci_dev *hdev) hci_req_sync_lock(hdev); - if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) { - ret = -ENODEV; - goto done; - } - - if (!hci_dev_test_flag(hdev, HCI_SETUP) && - !hci_dev_test_flag(hdev, HCI_CONFIG)) { - /* Check for rfkill but allow the HCI setup stage to - * proceed (which in itself doesn't cause any RF activity). - */ - if (hci_dev_test_flag(hdev, HCI_RFKILLED)) { - ret = -ERFKILL; - goto done; - } - - /* Check for valid public address or a configured static - * random address, but let the HCI setup proceed to - * be able to determine if there is a public address - * or not. - * - * In case of user channel usage, it is not important - * if a public address or static random address is - * available. - * - * This check is only valid for BR/EDR controllers - * since AMP controllers do not have an address. - */ - if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - hdev->dev_type == HCI_PRIMARY && - !bacmp(&hdev->bdaddr, BDADDR_ANY) && - !bacmp(&hdev->static_addr, BDADDR_ANY)) { - ret = -EADDRNOTAVAIL; - goto done; - } - } - - if (test_bit(HCI_UP, &hdev->flags)) { - ret = -EALREADY; - goto done; - } - - if (hdev->open(hdev)) { - ret = -EIO; - goto done; - } - - set_bit(HCI_RUNNING, &hdev->flags); - hci_sock_dev_event(hdev, HCI_DEV_OPEN); - - atomic_set(&hdev->cmd_cnt, 1); - set_bit(HCI_INIT, &hdev->flags); - - if (hci_dev_test_flag(hdev, HCI_SETUP) || - test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks)) { - bool invalid_bdaddr; - - hci_sock_dev_event(hdev, HCI_DEV_SETUP); - - if (hdev->setup) - ret = hdev->setup(hdev); - - /* The transport driver can set the quirk to mark the - * BD_ADDR invalid before creating the HCI device or in - * its setup callback. - */ - invalid_bdaddr = test_bit(HCI_QUIRK_INVALID_BDADDR, - &hdev->quirks); - - if (ret) - goto setup_failed; - - if (test_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks)) { - if (!bacmp(&hdev->public_addr, BDADDR_ANY)) - hci_dev_get_bd_addr_from_property(hdev); - - if (bacmp(&hdev->public_addr, BDADDR_ANY) && - hdev->set_bdaddr) { - ret = hdev->set_bdaddr(hdev, - &hdev->public_addr); - - /* If setting of the BD_ADDR from the device - * property succeeds, then treat the address - * as valid even if the invalid BD_ADDR - * quirk indicates otherwise. - */ - if (!ret) - invalid_bdaddr = false; - } - } - -setup_failed: - /* The transport driver can set these quirks before - * creating the HCI device or in its setup callback. - * - * For the invalid BD_ADDR quirk it is possible that - * it becomes a valid address if the bootloader does - * provide it (see above). - * - * In case any of them is set, the controller has to - * start up as unconfigured. - */ - if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) || - invalid_bdaddr) - hci_dev_set_flag(hdev, HCI_UNCONFIGURED); - - /* For an unconfigured controller it is required to - * read at least the version information provided by - * the Read Local Version Information command. - * - * If the set_bdaddr driver callback is provided, then - * also the original Bluetooth public device address - * will be read using the Read BD Address command. - */ - if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) - ret = __hci_unconf_init(hdev); - } - - if (hci_dev_test_flag(hdev, HCI_CONFIG)) { - /* If public address change is configured, ensure that - * the address gets programmed. If the driver does not - * support changing the public address, fail the power - * on procedure. - */ - if (bacmp(&hdev->public_addr, BDADDR_ANY) && - hdev->set_bdaddr) - ret = hdev->set_bdaddr(hdev, &hdev->public_addr); - else - ret = -EADDRNOTAVAIL; - } - - if (!ret) { - if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED) && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) { - ret = __hci_init(hdev); - if (!ret && hdev->post_init) - ret = hdev->post_init(hdev); - } - } - - /* If the HCI Reset command is clearing all diagnostic settings, - * then they need to be reprogrammed after the init procedure - * completed. - */ - if (test_bit(HCI_QUIRK_NON_PERSISTENT_DIAG, &hdev->quirks) && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - hci_dev_test_flag(hdev, HCI_VENDOR_DIAG) && hdev->set_diag) - ret = hdev->set_diag(hdev, true); - - msft_do_open(hdev); - aosp_do_open(hdev); - - clear_bit(HCI_INIT, &hdev->flags); - - if (!ret) { - hci_dev_hold(hdev); - hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); - hci_adv_instances_set_rpa_expired(hdev, true); - set_bit(HCI_UP, &hdev->flags); - hci_sock_dev_event(hdev, HCI_DEV_UP); - hci_leds_update_powered(hdev, true); - if (!hci_dev_test_flag(hdev, HCI_SETUP) && - !hci_dev_test_flag(hdev, HCI_CONFIG) && - !hci_dev_test_flag(hdev, HCI_UNCONFIGURED) && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - hci_dev_test_flag(hdev, HCI_MGMT) && - hdev->dev_type == HCI_PRIMARY) { - ret = __hci_req_hci_power_on(hdev); - mgmt_power_on(hdev, ret); - } - } else { - /* Init failed, cleanup */ - flush_work(&hdev->tx_work); - - /* Since hci_rx_work() is possible to awake new cmd_work - * it should be flushed first to avoid unexpected call of - * hci_cmd_work() - */ - flush_work(&hdev->rx_work); - flush_work(&hdev->cmd_work); - - skb_queue_purge(&hdev->cmd_q); - skb_queue_purge(&hdev->rx_q); - - if (hdev->flush) - hdev->flush(hdev); - - if (hdev->sent_cmd) { - kfree_skb(hdev->sent_cmd); - hdev->sent_cmd = NULL; - } - - clear_bit(HCI_RUNNING, &hdev->flags); - hci_sock_dev_event(hdev, HCI_DEV_CLOSE); - - hdev->close(hdev); - hdev->flags &= BIT(HCI_RAW); - } + ret = hci_dev_open_sync(hdev); -done: hci_req_sync_unlock(hdev); return ret; } @@ -1583,155 +542,18 @@ done: return err; } -/* This function requires the caller holds hdev->lock */ -static void hci_pend_le_actions_clear(struct hci_dev *hdev) -{ - struct hci_conn_params *p; - - list_for_each_entry(p, &hdev->le_conn_params, list) { - if (p->conn) { - hci_conn_drop(p->conn); - hci_conn_put(p->conn); - p->conn = NULL; - } - list_del_init(&p->action); - } - - BT_DBG("All LE pending actions cleared"); -} - int hci_dev_do_close(struct hci_dev *hdev) { - bool auto_off; - int err = 0; + int err; BT_DBG("%s %p", hdev->name, hdev); - cancel_delayed_work(&hdev->power_off); - cancel_delayed_work(&hdev->ncmd_timer); - - hci_request_cancel_all(hdev); hci_req_sync_lock(hdev); - if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - test_bit(HCI_UP, &hdev->flags)) { - /* Execute vendor specific shutdown routine */ - if (hdev->shutdown) - err = hdev->shutdown(hdev); - } - - if (!test_and_clear_bit(HCI_UP, &hdev->flags)) { - cancel_delayed_work_sync(&hdev->cmd_timer); - hci_req_sync_unlock(hdev); - return err; - } - - hci_leds_update_powered(hdev, false); - - /* Flush RX and TX works */ - flush_work(&hdev->tx_work); - flush_work(&hdev->rx_work); - - if (hdev->discov_timeout > 0) { - hdev->discov_timeout = 0; - hci_dev_clear_flag(hdev, HCI_DISCOVERABLE); - hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); - } - - if (hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) - cancel_delayed_work(&hdev->service_cache); - - if (hci_dev_test_flag(hdev, HCI_MGMT)) { - struct adv_info *adv_instance; - - cancel_delayed_work_sync(&hdev->rpa_expired); - - list_for_each_entry(adv_instance, &hdev->adv_instances, list) - cancel_delayed_work_sync(&adv_instance->rpa_expired_cb); - } - - /* Avoid potential lockdep warnings from the *_flush() calls by - * ensuring the workqueue is empty up front. - */ - drain_workqueue(hdev->workqueue); - - hci_dev_lock(hdev); - - hci_discovery_set_state(hdev, DISCOVERY_STOPPED); - - auto_off = hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF); - - if (!auto_off && hdev->dev_type == HCI_PRIMARY && - !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && - hci_dev_test_flag(hdev, HCI_MGMT)) - __mgmt_power_off(hdev); - - hci_inquiry_cache_flush(hdev); - hci_pend_le_actions_clear(hdev); - hci_conn_hash_flush(hdev); - hci_dev_unlock(hdev); - - smp_unregister(hdev); - - hci_sock_dev_event(hdev, HCI_DEV_DOWN); - - aosp_do_close(hdev); - msft_do_close(hdev); - - if (hdev->flush) - hdev->flush(hdev); - - /* Reset device */ - skb_queue_purge(&hdev->cmd_q); - atomic_set(&hdev->cmd_cnt, 1); - if (test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks) && - !auto_off && !hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) { - set_bit(HCI_INIT, &hdev->flags); - __hci_req_sync(hdev, hci_reset_req, 0, HCI_CMD_TIMEOUT, NULL); - clear_bit(HCI_INIT, &hdev->flags); - } - - /* flush cmd work */ - flush_work(&hdev->cmd_work); - - /* Drop queues */ - skb_queue_purge(&hdev->rx_q); - skb_queue_purge(&hdev->cmd_q); - skb_queue_purge(&hdev->raw_q); - - /* Drop last sent command */ - if (hdev->sent_cmd) { - cancel_delayed_work_sync(&hdev->cmd_timer); - kfree_skb(hdev->sent_cmd); - hdev->sent_cmd = NULL; - } - - clear_bit(HCI_RUNNING, &hdev->flags); - hci_sock_dev_event(hdev, HCI_DEV_CLOSE); - - if (test_and_clear_bit(SUSPEND_POWERING_DOWN, hdev->suspend_tasks)) - wake_up(&hdev->suspend_wait_q); - - /* After this point our queues are empty - * and no tasks are scheduled. */ - hdev->close(hdev); - - /* Clear flags */ - hdev->flags &= BIT(HCI_RAW); - hci_dev_clear_volatile_flags(hdev); - - /* Controller radio is available but is currently powered down */ - hdev->amp_status = AMP_STATUS_POWERED_DOWN; - - memset(hdev->eir, 0, sizeof(hdev->eir)); - memset(hdev->dev_class, 0, sizeof(hdev->dev_class)); - bacpy(&hdev->random_addr, BDADDR_ANY); - hci_codec_list_clear(&hdev->local_codecs); + err = hci_dev_close_sync(hdev); hci_req_sync_unlock(hdev); - hci_dev_put(hdev); return err; } @@ -1787,7 +609,7 @@ static int hci_dev_do_reset(struct hci_dev *hdev) atomic_set(&hdev->cmd_cnt, 1); hdev->acl_cnt = 0; hdev->sco_cnt = 0; hdev->le_cnt = 0; - ret = __hci_req_sync(hdev, hci_reset_req, 0, HCI_INIT_TIMEOUT, NULL); + ret = hci_reset_sync(hdev); hci_req_sync_unlock(hdev); return ret; @@ -1850,7 +672,7 @@ done: return ret; } -static void hci_update_scan_state(struct hci_dev *hdev, u8 scan) +static void hci_update_passive_scan_state(struct hci_dev *hdev, u8 scan) { bool conn_changed, discov_changed; @@ -1951,7 +773,7 @@ int hci_dev_cmd(unsigned int cmd, void __user *arg) * get correctly modified as this was a non-mgmt change. */ if (!err) - hci_update_scan_state(hdev, dr.dev_opt); + hci_update_passive_scan_state(hdev, dr.dev_opt); break; case HCISETLINKPOL: @@ -2133,9 +955,7 @@ static void hci_power_on(struct work_struct *work) hci_dev_test_flag(hdev, HCI_MGMT) && hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF)) { cancel_delayed_work(&hdev->power_off); - hci_req_sync_lock(hdev); - err = __hci_req_hci_power_on(hdev); - hci_req_sync_unlock(hdev); + err = hci_powered_update_sync(hdev); mgmt_power_on(hdev, err); return; } @@ -3096,7 +1916,7 @@ bool hci_add_adv_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, switch (hci_get_adv_monitor_offload_ext(hdev)) { case HCI_ADV_MONITOR_EXT_NONE: - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); bt_dev_dbg(hdev, "%s add monitor status %d", hdev->name, *err); /* Message was not forwarded to controller - not an error */ return false; @@ -3160,7 +1980,7 @@ bool hci_remove_single_adv_monitor(struct hci_dev *hdev, u16 handle, int *err) pending = hci_remove_adv_monitor(hdev, monitor, handle, err); if (!*err && !pending) - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); bt_dev_dbg(hdev, "%s remove monitor handle %d, status %d, %spending", hdev->name, handle, *err, pending ? "" : "not "); @@ -3192,7 +2012,7 @@ bool hci_remove_all_adv_monitor(struct hci_dev *hdev, int *err) } if (update) - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); bt_dev_dbg(hdev, "%s remove all monitors status %d, %spending", hdev->name, *err, pending ? "" : "not "); @@ -3486,7 +2306,7 @@ void hci_conn_params_del(struct hci_dev *hdev, bdaddr_t *addr, u8 addr_type) hci_conn_params_free(params); - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); BT_DBG("addr %pMR (type %u)", addr, addr_type); } @@ -3554,61 +2374,6 @@ void hci_copy_identity_address(struct hci_dev *hdev, bdaddr_t *bdaddr, } } -static void hci_suspend_clear_tasks(struct hci_dev *hdev) -{ - int i; - - for (i = 0; i < __SUSPEND_NUM_TASKS; i++) - clear_bit(i, hdev->suspend_tasks); - - wake_up(&hdev->suspend_wait_q); -} - -static int hci_suspend_wait_event(struct hci_dev *hdev) -{ -#define WAKE_COND \ - (find_first_bit(hdev->suspend_tasks, __SUSPEND_NUM_TASKS) == \ - __SUSPEND_NUM_TASKS) - - int i; - int ret = wait_event_timeout(hdev->suspend_wait_q, - WAKE_COND, SUSPEND_NOTIFIER_TIMEOUT); - - if (ret == 0) { - bt_dev_err(hdev, "Timed out waiting for suspend events"); - for (i = 0; i < __SUSPEND_NUM_TASKS; ++i) { - if (test_bit(i, hdev->suspend_tasks)) - bt_dev_err(hdev, "Suspend timeout bit: %d", i); - clear_bit(i, hdev->suspend_tasks); - } - - ret = -ETIMEDOUT; - } else { - ret = 0; - } - - return ret; -} - -static void hci_prepare_suspend(struct work_struct *work) -{ - struct hci_dev *hdev = - container_of(work, struct hci_dev, suspend_prepare); - - hci_dev_lock(hdev); - hci_req_prepare_suspend(hdev, hdev->suspend_state_next); - hci_dev_unlock(hdev); -} - -static int hci_change_suspend_state(struct hci_dev *hdev, - enum suspended_state next) -{ - hdev->suspend_state_next = next; - set_bit(SUSPEND_PREPARE_NOTIFIER, hdev->suspend_tasks); - queue_work(hdev->req_workqueue, &hdev->suspend_prepare); - return hci_suspend_wait_event(hdev); -} - static void hci_clear_wake_reason(struct hci_dev *hdev) { hci_dev_lock(hdev); @@ -3745,7 +2510,8 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv) INIT_WORK(&hdev->tx_work, hci_tx_work); INIT_WORK(&hdev->power_on, hci_power_on); INIT_WORK(&hdev->error_reset, hci_error_reset); - INIT_WORK(&hdev->suspend_prepare, hci_prepare_suspend); + + hci_cmd_sync_init(hdev); INIT_DELAYED_WORK(&hdev->power_off, hci_power_off); @@ -3754,7 +2520,6 @@ struct hci_dev *hci_alloc_dev_priv(int sizeof_priv) skb_queue_head_init(&hdev->raw_q); init_waitqueue_head(&hdev->req_wait_q); - init_waitqueue_head(&hdev->suspend_wait_q); INIT_DELAYED_WORK(&hdev->cmd_timer, hci_cmd_timeout); INIT_DELAYED_WORK(&hdev->ncmd_timer, hci_ncmd_timeout); @@ -3882,6 +2647,7 @@ int hci_register_dev(struct hci_dev *hdev) return id; err_wqueue: + debugfs_remove_recursive(hdev->debugfs); destroy_workqueue(hdev->workqueue); destroy_workqueue(hdev->req_workqueue); err: @@ -3904,11 +2670,10 @@ void hci_unregister_dev(struct hci_dev *hdev) cancel_work_sync(&hdev->power_on); - if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) { - hci_suspend_clear_tasks(hdev); + hci_cmd_sync_clear(hdev); + + if (!test_bit(HCI_QUIRK_NO_SUSPEND_NOTIFIER, &hdev->quirks)) unregister_pm_notifier(&hdev->suspend_notifier); - cancel_work_sync(&hdev->suspend_prepare); - } msft_unregister(hdev); @@ -3975,7 +2740,6 @@ EXPORT_SYMBOL(hci_release_dev); int hci_suspend_dev(struct hci_dev *hdev) { int ret; - u8 state = BT_RUNNING; bt_dev_dbg(hdev, ""); @@ -3984,40 +2748,17 @@ int hci_suspend_dev(struct hci_dev *hdev) hci_dev_test_flag(hdev, HCI_UNREGISTER)) return 0; - /* If powering down, wait for completion. */ - if (mgmt_powering_down(hdev)) { - set_bit(SUSPEND_POWERING_DOWN, hdev->suspend_tasks); - ret = hci_suspend_wait_event(hdev); - if (ret) - goto done; - } - - /* Suspend consists of two actions: - * - First, disconnect everything and make the controller not - * connectable (disabling scanning) - * - Second, program event filter/accept list and enable scan - */ - ret = hci_change_suspend_state(hdev, BT_SUSPEND_DISCONNECT); - if (ret) - goto clear; - - state = BT_SUSPEND_DISCONNECT; + /* If powering down don't attempt to suspend */ + if (mgmt_powering_down(hdev)) + return 0; - /* Only configure accept list if device may wakeup. */ - if (hdev->wakeup && hdev->wakeup(hdev)) { - ret = hci_change_suspend_state(hdev, BT_SUSPEND_CONFIGURE_WAKE); - if (!ret) - state = BT_SUSPEND_CONFIGURE_WAKE; - } + hci_req_sync_lock(hdev); + ret = hci_suspend_sync(hdev); + hci_req_sync_unlock(hdev); -clear: hci_clear_wake_reason(hdev); - mgmt_suspending(hdev, state); + mgmt_suspending(hdev, hdev->suspend_state); -done: - /* We always allow suspend even if suspend preparation failed and - * attempt to recover in resume. - */ hci_sock_dev_event(hdev, HCI_DEV_SUSPEND); return ret; } @@ -4039,10 +2780,12 @@ int hci_resume_dev(struct hci_dev *hdev) if (mgmt_powering_down(hdev)) return 0; - ret = hci_change_suspend_state(hdev, BT_RUNNING); + hci_req_sync_lock(hdev); + ret = hci_resume_sync(hdev); + hci_req_sync_unlock(hdev); mgmt_resuming(hdev, hdev->wake_reason, &hdev->wake_addr, - hdev->wake_addr_type); + hdev->wake_addr_type); hci_sock_dev_event(hdev, HCI_DEV_RESUME); return ret; @@ -4270,25 +3013,6 @@ void *hci_sent_cmd_data(struct hci_dev *hdev, __u16 opcode) return hdev->sent_cmd->data + HCI_COMMAND_HDR_SIZE; } -/* Send HCI command and wait for command complete event */ -struct sk_buff *hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u32 timeout) -{ - struct sk_buff *skb; - - if (!test_bit(HCI_UP, &hdev->flags)) - return ERR_PTR(-ENETDOWN); - - bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen); - - hci_req_sync_lock(hdev); - skb = __hci_cmd_sync(hdev, opcode, plen, param, timeout); - hci_req_sync_unlock(hdev); - - return skb; -} -EXPORT_SYMBOL(hci_cmd_sync); - /* Send ACL data */ static void hci_add_acl_hdr(struct sk_buff *skb, __u16 handle, __u16 flags) { diff --git a/net/bluetooth/hci_event.c b/net/bluetooth/hci_event.c index 7d0db1ca1248..efc5458b1345 100644 --- a/net/bluetooth/hci_event.c +++ b/net/bluetooth/hci_event.c @@ -545,9 +545,7 @@ static void hci_cc_write_ssp_mode(struct hci_dev *hdev, struct sk_buff *skb) hdev->features[1][0] &= ~LMP_HOST_SSP; } - if (hci_dev_test_flag(hdev, HCI_MGMT)) - mgmt_ssp_enable_complete(hdev, sent->mode, status); - else if (!status) { + if (!status) { if (sent->mode) hci_dev_set_flag(hdev, HCI_SSP_ENABLED); else @@ -1239,6 +1237,55 @@ static void hci_cc_le_set_adv_set_random_addr(struct hci_dev *hdev, hci_dev_unlock(hdev); } +static void hci_cc_le_remove_adv_set(struct hci_dev *hdev, struct sk_buff *skb) +{ + __u8 status = *((__u8 *)skb->data); + u8 *instance; + int err; + + if (status) + return; + + instance = hci_sent_cmd_data(hdev, HCI_OP_LE_REMOVE_ADV_SET); + if (!instance) + return; + + hci_dev_lock(hdev); + + err = hci_remove_adv_instance(hdev, *instance); + if (!err) + mgmt_advertising_removed(hci_skb_sk(hdev->sent_cmd), hdev, + *instance); + + hci_dev_unlock(hdev); +} + +static void hci_cc_le_clear_adv_sets(struct hci_dev *hdev, struct sk_buff *skb) +{ + __u8 status = *((__u8 *)skb->data); + struct adv_info *adv, *n; + int err; + + if (status) + return; + + if (!hci_sent_cmd_data(hdev, HCI_OP_LE_CLEAR_ADV_SETS)) + return; + + hci_dev_lock(hdev); + + list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) { + u8 instance = adv->instance; + + err = hci_remove_adv_instance(hdev, instance); + if (!err) + mgmt_advertising_removed(hci_skb_sk(hdev->sent_cmd), + hdev, instance); + } + + hci_dev_unlock(hdev); +} + static void hci_cc_le_read_transmit_power(struct hci_dev *hdev, struct sk_buff *skb) { @@ -1326,8 +1373,10 @@ static void hci_cc_le_set_ext_adv_enable(struct hci_dev *hdev, &conn->le_conn_timeout, conn->conn_timeout); } else { - if (adv) { - adv->enabled = false; + if (cp->num_of_sets) { + if (adv) + adv->enabled = false; + /* If just one instance was disabled check if there are * any other instance enabled before clearing HCI_LE_ADV */ @@ -1463,16 +1512,10 @@ static void le_set_scan_enable_complete(struct hci_dev *hdev, u8 enable) /* The HCI_LE_SCAN_INTERRUPTED flag indicates that we * interrupted scanning due to a connect request. Mark - * therefore discovery as stopped. If this was not - * because of a connect request advertising might have - * been disabled because of active scanning, so - * re-enable it again if necessary. + * therefore discovery as stopped. */ if (hci_dev_test_and_clear_flag(hdev, HCI_LE_SCAN_INTERRUPTED)) hci_discovery_set_state(hdev, DISCOVERY_STOPPED); - else if (!hci_dev_test_flag(hdev, HCI_LE_ADV) && - hdev->discovery.state == DISCOVERY_FINDING) - hci_req_reenable_advertising(hdev); break; @@ -2371,9 +2414,14 @@ static void hci_cs_exit_sniff_mode(struct hci_dev *hdev, __u8 status) static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) { struct hci_cp_disconnect *cp; + struct hci_conn_params *params; struct hci_conn *conn; + bool mgmt_conn; - if (!status) + /* Wait for HCI_EV_DISCONN_COMPLETE if status 0x00 and not suspended + * otherwise cleanup the connection immediately. + */ + if (!status && !hdev->suspended) return; cp = hci_sent_cmd_data(hdev, HCI_OP_DISCONNECT); @@ -2383,23 +2431,60 @@ static void hci_cs_disconnect(struct hci_dev *hdev, u8 status) hci_dev_lock(hdev); conn = hci_conn_hash_lookup_handle(hdev, __le16_to_cpu(cp->handle)); - if (conn) { + if (!conn) + goto unlock; + + if (status) { mgmt_disconnect_failed(hdev, &conn->dst, conn->type, conn->dst_type, status); if (conn->type == LE_LINK && conn->role == HCI_ROLE_SLAVE) { hdev->cur_adv_instance = conn->adv_instance; - hci_req_reenable_advertising(hdev); + hci_enable_advertising(hdev); } - /* If the disconnection failed for any reason, the upper layer - * does not retry to disconnect in current implementation. - * Hence, we need to do some basic cleanup here and re-enable - * advertising if necessary. - */ - hci_conn_del(conn); + goto done; + } + + mgmt_conn = test_and_clear_bit(HCI_CONN_MGMT_CONNECTED, &conn->flags); + + if (conn->type == ACL_LINK) { + if (test_bit(HCI_CONN_FLUSH_KEY, &conn->flags)) + hci_remove_link_key(hdev, &conn->dst); + } + + params = hci_conn_params_lookup(hdev, &conn->dst, conn->dst_type); + if (params) { + switch (params->auto_connect) { + case HCI_AUTO_CONN_LINK_LOSS: + if (cp->reason != HCI_ERROR_CONNECTION_TIMEOUT) + break; + fallthrough; + + case HCI_AUTO_CONN_DIRECT: + case HCI_AUTO_CONN_ALWAYS: + list_del_init(¶ms->action); + list_add(¶ms->action, &hdev->pend_le_conns); + break; + + default: + break; + } } + mgmt_device_disconnected(hdev, &conn->dst, conn->type, conn->dst_type, + cp->reason, mgmt_conn); + + hci_disconn_cfm(conn, cp->reason); + +done: + /* If the disconnection failed for any reason, the upper layer + * does not retry to disconnect in current implementation. + * Hence, we need to do some basic cleanup here and re-enable + * advertising if necessary. + */ + hci_conn_del(conn); +unlock: hci_dev_unlock(hdev); } @@ -2977,7 +3062,7 @@ static void hci_disconn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) case HCI_AUTO_CONN_ALWAYS: list_del_init(¶ms->action); list_add(¶ms->action, &hdev->pend_le_conns); - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); break; default: @@ -2987,14 +3072,6 @@ static void hci_disconn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) hci_disconn_cfm(conn, ev->reason); - /* The suspend notifier is waiting for all devices to disconnect so - * clear the bit from pending tasks and inform the wait queue. - */ - if (list_empty(&hdev->conn_hash.list) && - test_and_clear_bit(SUSPEND_DISCONNECTING, hdev->suspend_tasks)) { - wake_up(&hdev->suspend_wait_q); - } - /* Re-enable advertising if necessary, since it might * have been disabled by the connection. From the * HCI_LE_Set_Advertise_Enable command description in @@ -3007,7 +3084,7 @@ static void hci_disconn_complete_evt(struct hci_dev *hdev, struct sk_buff *skb) */ if (conn->type == LE_LINK && conn->role == HCI_ROLE_SLAVE) { hdev->cur_adv_instance = conn->adv_instance; - hci_req_reenable_advertising(hdev); + hci_enable_advertising(hdev); } hci_conn_del(conn); @@ -3723,6 +3800,14 @@ static void hci_cmd_complete_evt(struct hci_dev *hdev, struct sk_buff *skb, hci_cc_le_set_adv_set_random_addr(hdev, skb); break; + case HCI_OP_LE_REMOVE_ADV_SET: + hci_cc_le_remove_adv_set(hdev, skb); + break; + + case HCI_OP_LE_CLEAR_ADV_SETS: + hci_cc_le_clear_adv_sets(hdev, skb); + break; + case HCI_OP_LE_READ_TRANSMIT_POWER: hci_cc_le_read_transmit_power(hdev, skb); break; @@ -4445,7 +4530,6 @@ static void hci_sync_conn_complete_evt(struct hci_dev *hdev, { struct hci_ev_sync_conn_complete *ev = (void *) skb->data; struct hci_conn *conn; - unsigned int notify_evt; BT_DBG("%s status 0x%2.2x", hdev->name, ev->status); @@ -4517,22 +4601,18 @@ static void hci_sync_conn_complete_evt(struct hci_dev *hdev, } bt_dev_dbg(hdev, "SCO connected with air mode: %02x", ev->air_mode); - - switch (ev->air_mode) { - case 0x02: - notify_evt = HCI_NOTIFY_ENABLE_SCO_CVSD; - break; - case 0x03: - notify_evt = HCI_NOTIFY_ENABLE_SCO_TRANSP; - break; - } - /* Notify only in case of SCO over HCI transport data path which * is zero and non-zero value shall be non-HCI transport data path */ - if (conn->codec.data_path == 0) { - if (hdev->notify) - hdev->notify(hdev, notify_evt); + if (conn->codec.data_path == 0 && hdev->notify) { + switch (ev->air_mode) { + case 0x02: + hdev->notify(hdev, HCI_NOTIFY_ENABLE_SCO_CVSD); + break; + case 0x03: + hdev->notify(hdev, HCI_NOTIFY_ENABLE_SCO_TRANSP); + break; + } } hci_connect_cfm(conn, ev->status); @@ -5412,7 +5492,7 @@ static void le_conn_complete_evt(struct hci_dev *hdev, u8 status, } unlock: - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); hci_dev_unlock(hdev); } @@ -5441,23 +5521,30 @@ static void hci_le_enh_conn_complete_evt(struct hci_dev *hdev, le16_to_cpu(ev->interval), le16_to_cpu(ev->latency), le16_to_cpu(ev->supervision_timeout)); - - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY) && - hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) - hci_req_disable_address_resolution(hdev); } static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, struct sk_buff *skb) { struct hci_evt_le_ext_adv_set_term *ev = (void *) skb->data; struct hci_conn *conn; - struct adv_info *adv; + struct adv_info *adv, *n; BT_DBG("%s status 0x%2.2x", hdev->name, ev->status); adv = hci_find_adv_instance(hdev, ev->handle); + /* The Bluetooth Core 5.3 specification clearly states that this event + * shall not be sent when the Host disables the advertising set. So in + * case of HCI_ERROR_CANCELLED_BY_HOST, just ignore the event. + * + * When the Host disables an advertising set, all cleanup is done via + * its command callback and not needed to be duplicated here. + */ + if (ev->status == HCI_ERROR_CANCELLED_BY_HOST) { + bt_dev_warn_ratelimited(hdev, "Unexpected advertising set terminated event"); + return; + } + if (ev->status) { if (!adv) return; @@ -5466,6 +5553,13 @@ static void hci_le_ext_adv_term_evt(struct hci_dev *hdev, struct sk_buff *skb) hci_remove_adv_instance(hdev, ev->handle); mgmt_advertising_removed(NULL, hdev, ev->handle); + list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) { + if (adv->enabled) + return; + } + + /* We are no longer advertising, clear HCI_LE_ADV */ + hci_dev_clear_flag(hdev, HCI_LE_ADV); return; } @@ -5529,8 +5623,9 @@ static struct hci_conn *check_pending_le_conn(struct hci_dev *hdev, if (adv_type != LE_ADV_IND && adv_type != LE_ADV_DIRECT_IND) return NULL; - /* Ignore if the device is blocked */ - if (hci_bdaddr_list_lookup(&hdev->reject_list, addr, addr_type)) + /* Ignore if the device is blocked or hdev is suspended */ + if (hci_bdaddr_list_lookup(&hdev->reject_list, addr, addr_type) || + hdev->suspended) return NULL; /* Most controller will fail if we try to create new connections @@ -5825,7 +5920,8 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, struct sk_buff *skb) struct hci_ev_le_advertising_info *ev = ptr; s8 rssi; - if (ev->length <= HCI_MAX_AD_LENGTH) { + if (ev->length <= HCI_MAX_AD_LENGTH && + ev->data + ev->length <= skb_tail_pointer(skb)) { rssi = ev->data[ev->length]; process_adv_report(hdev, ev->evt_type, &ev->bdaddr, ev->bdaddr_type, NULL, 0, rssi, @@ -5835,6 +5931,11 @@ static void hci_le_adv_report_evt(struct hci_dev *hdev, struct sk_buff *skb) } ptr += sizeof(*ev) + ev->length + 1; + + if (ptr > (void *) skb_tail_pointer(skb) - sizeof(*ev)) { + bt_dev_err(hdev, "Malicious advertising data. Stopping processing"); + break; + } } hci_dev_unlock(hdev); diff --git a/net/bluetooth/hci_request.c b/net/bluetooth/hci_request.c index 92611bfc0b9e..8b3205e4b23e 100644 --- a/net/bluetooth/hci_request.c +++ b/net/bluetooth/hci_request.c @@ -32,10 +32,6 @@ #include "msft.h" #include "eir.h" -#define HCI_REQ_DONE 0 -#define HCI_REQ_PEND 1 -#define HCI_REQ_CANCELED 2 - void hci_req_init(struct hci_request *req, struct hci_dev *hdev) { skb_queue_head_init(&req->cmd_q); @@ -101,8 +97,8 @@ int hci_req_run_skb(struct hci_request *req, hci_req_complete_skb_t complete) return req_run(req, NULL, complete); } -static void hci_req_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, - struct sk_buff *skb) +void hci_req_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, + struct sk_buff *skb) { bt_dev_dbg(hdev, "result 0x%2.2x", result); @@ -126,70 +122,6 @@ void hci_req_sync_cancel(struct hci_dev *hdev, int err) } } -struct sk_buff *__hci_cmd_sync_ev(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u8 event, u32 timeout) -{ - struct hci_request req; - struct sk_buff *skb; - int err = 0; - - bt_dev_dbg(hdev, ""); - - hci_req_init(&req, hdev); - - hci_req_add_ev(&req, opcode, plen, param, event); - - hdev->req_status = HCI_REQ_PEND; - - err = hci_req_run_skb(&req, hci_req_sync_complete); - if (err < 0) - return ERR_PTR(err); - - err = wait_event_interruptible_timeout(hdev->req_wait_q, - hdev->req_status != HCI_REQ_PEND, timeout); - - if (err == -ERESTARTSYS) - return ERR_PTR(-EINTR); - - switch (hdev->req_status) { - case HCI_REQ_DONE: - err = -bt_to_errno(hdev->req_result); - break; - - case HCI_REQ_CANCELED: - err = -hdev->req_result; - break; - - default: - err = -ETIMEDOUT; - break; - } - - hdev->req_status = hdev->req_result = 0; - skb = hdev->req_skb; - hdev->req_skb = NULL; - - bt_dev_dbg(hdev, "end: err %d", err); - - if (err < 0) { - kfree_skb(skb); - return ERR_PTR(err); - } - - if (!skb) - return ERR_PTR(-ENODATA); - - return skb; -} -EXPORT_SYMBOL(__hci_cmd_sync_ev); - -struct sk_buff *__hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, - const void *param, u32 timeout) -{ - return __hci_cmd_sync_ev(hdev, opcode, plen, param, 0, timeout); -} -EXPORT_SYMBOL(__hci_cmd_sync); - /* Execute request and wait for completion. */ int __hci_req_sync(struct hci_dev *hdev, int (*func)(struct hci_request *req, unsigned long opt), @@ -436,82 +368,6 @@ static bool __hci_update_interleaved_scan(struct hci_dev *hdev) return false; } -/* This function controls the background scanning based on hdev->pend_le_conns - * list. If there are pending LE connection we start the background scanning, - * otherwise we stop it. - * - * This function requires the caller holds hdev->lock. - */ -static void __hci_update_background_scan(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - - if (!test_bit(HCI_UP, &hdev->flags) || - test_bit(HCI_INIT, &hdev->flags) || - hci_dev_test_flag(hdev, HCI_SETUP) || - hci_dev_test_flag(hdev, HCI_CONFIG) || - hci_dev_test_flag(hdev, HCI_AUTO_OFF) || - hci_dev_test_flag(hdev, HCI_UNREGISTER)) - return; - - /* No point in doing scanning if LE support hasn't been enabled */ - if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) - return; - - /* If discovery is active don't interfere with it */ - if (hdev->discovery.state != DISCOVERY_STOPPED) - return; - - /* Reset RSSI and UUID filters when starting background scanning - * since these filters are meant for service discovery only. - * - * The Start Discovery and Start Service Discovery operations - * ensure to set proper values for RSSI threshold and UUID - * filter list. So it is safe to just reset them here. - */ - hci_discovery_filter_clear(hdev); - - bt_dev_dbg(hdev, "ADV monitoring is %s", - hci_is_adv_monitoring(hdev) ? "on" : "off"); - - if (list_empty(&hdev->pend_le_conns) && - list_empty(&hdev->pend_le_reports) && - !hci_is_adv_monitoring(hdev)) { - /* If there is no pending LE connections or devices - * to be scanned for or no ADV monitors, we should stop the - * background scanning. - */ - - /* If controller is not scanning we are done. */ - if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) - return; - - hci_req_add_le_scan_disable(req, false); - - bt_dev_dbg(hdev, "stopping background scanning"); - } else { - /* If there is at least one pending LE connection, we should - * keep the background scan running. - */ - - /* If controller is connecting, we should not start scanning - * since some controllers are not able to scan and connect at - * the same time. - */ - if (hci_lookup_le_connect(hdev)) - return; - - /* If controller is currently scanning, we stop it to ensure we - * don't miss any advertising (due to duplicates filter). - */ - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) - hci_req_add_le_scan_disable(req, false); - - hci_req_add_le_passive_scan(req); - bt_dev_dbg(hdev, "starting background scanning"); - } -} - void __hci_req_update_name(struct hci_request *req) { struct hci_dev *hdev = req->hdev; @@ -560,9 +416,6 @@ void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn) return; } - if (hdev->suspended) - set_bit(SUSPEND_SCAN_DISABLE, hdev->suspend_tasks); - if (use_ext_scan(hdev)) { struct hci_cp_le_set_ext_scan_enable cp; @@ -579,9 +432,7 @@ void hci_req_add_le_scan_disable(struct hci_request *req, bool rpa_le_conn) } /* Disable address resolution */ - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY) && - hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION) && !rpa_le_conn) { + if (hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION) && !rpa_le_conn) { __u8 enable = 0x00; hci_req_add(req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable); @@ -600,8 +451,7 @@ static void del_from_accept_list(struct hci_request *req, bdaddr_t *bdaddr, cp.bdaddr_type); hci_req_add(req, HCI_OP_LE_DEL_FROM_ACCEPT_LIST, sizeof(cp), &cp); - if (use_ll_privacy(req->hdev) && - hci_dev_test_flag(req->hdev, HCI_ENABLE_LL_PRIVACY)) { + if (use_ll_privacy(req->hdev)) { struct smp_irk *irk; irk = hci_find_irk_by_addr(req->hdev, bdaddr, bdaddr_type); @@ -654,8 +504,7 @@ static int add_to_accept_list(struct hci_request *req, cp.bdaddr_type); hci_req_add(req, HCI_OP_LE_ADD_TO_ACCEPT_LIST, sizeof(cp), &cp); - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) { + if (use_ll_privacy(hdev)) { struct smp_irk *irk; irk = hci_find_irk_by_addr(hdev, ¶ms->addr, @@ -694,8 +543,7 @@ static u8 update_accept_list(struct hci_request *req) */ bool allow_rpa = hdev->suspended; - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) + if (use_ll_privacy(hdev)) allow_rpa = true; /* Go through the current accept list programmed into the @@ -784,9 +632,7 @@ static void hci_req_start_scan(struct hci_request *req, u8 type, u16 interval, return; } - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY) && - addr_resolv) { + if (use_ll_privacy(hdev) && addr_resolv) { u8 enable = 0x01; hci_req_add(req, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, 1, &enable); @@ -943,8 +789,6 @@ void hci_req_add_le_passive_scan(struct hci_request *req) if (hdev->suspended) { window = hdev->le_scan_window_suspend; interval = hdev->le_scan_int_suspend; - - set_bit(SUSPEND_SCAN_ENABLE, hdev->suspend_tasks); } else if (hci_is_le_conn_scanning(hdev)) { window = hdev->le_scan_window_connect; interval = hdev->le_scan_int_connect; @@ -977,59 +821,6 @@ void hci_req_add_le_passive_scan(struct hci_request *req) addr_resolv); } -static void hci_req_clear_event_filter(struct hci_request *req) -{ - struct hci_cp_set_event_filter f; - - if (!hci_dev_test_flag(req->hdev, HCI_BREDR_ENABLED)) - return; - - if (hci_dev_test_flag(req->hdev, HCI_EVENT_FILTER_CONFIGURED)) { - memset(&f, 0, sizeof(f)); - f.flt_type = HCI_FLT_CLEAR_ALL; - hci_req_add(req, HCI_OP_SET_EVENT_FLT, 1, &f); - } -} - -static void hci_req_set_event_filter(struct hci_request *req) -{ - struct bdaddr_list_with_flags *b; - struct hci_cp_set_event_filter f; - struct hci_dev *hdev = req->hdev; - u8 scan = SCAN_DISABLED; - bool scanning = test_bit(HCI_PSCAN, &hdev->flags); - - if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) - return; - - /* Always clear event filter when starting */ - hci_req_clear_event_filter(req); - - list_for_each_entry(b, &hdev->accept_list, list) { - if (!hci_conn_test_flag(HCI_CONN_FLAG_REMOTE_WAKEUP, - b->current_flags)) - continue; - - memset(&f, 0, sizeof(f)); - bacpy(&f.addr_conn_flt.bdaddr, &b->bdaddr); - f.flt_type = HCI_FLT_CONN_SETUP; - f.cond_type = HCI_CONN_SETUP_ALLOW_BDADDR; - f.addr_conn_flt.auto_accept = HCI_CONN_SETUP_AUTO_ON; - - bt_dev_dbg(hdev, "Adding event filters for %pMR", &b->bdaddr); - hci_req_add(req, HCI_OP_SET_EVENT_FLT, sizeof(f), &f); - scan = SCAN_PAGE; - } - - if (scan && !scanning) { - set_bit(SUSPEND_SCAN_ENABLE, hdev->suspend_tasks); - hci_req_add(req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan); - } else if (!scan && scanning) { - set_bit(SUSPEND_SCAN_DISABLE, hdev->suspend_tasks); - hci_req_add(req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan); - } -} - static void cancel_adv_timeout(struct hci_dev *hdev) { if (hdev->adv_instance_timeout) { @@ -1088,185 +879,6 @@ int hci_req_resume_adv_instances(struct hci_dev *hdev) return hci_req_run(&req, NULL); } -static void suspend_req_complete(struct hci_dev *hdev, u8 status, u16 opcode) -{ - bt_dev_dbg(hdev, "Request complete opcode=0x%x, status=0x%x", opcode, - status); - if (test_bit(SUSPEND_SCAN_ENABLE, hdev->suspend_tasks) || - test_bit(SUSPEND_SCAN_DISABLE, hdev->suspend_tasks)) { - clear_bit(SUSPEND_SCAN_ENABLE, hdev->suspend_tasks); - clear_bit(SUSPEND_SCAN_DISABLE, hdev->suspend_tasks); - wake_up(&hdev->suspend_wait_q); - } - - if (test_bit(SUSPEND_SET_ADV_FILTER, hdev->suspend_tasks)) { - clear_bit(SUSPEND_SET_ADV_FILTER, hdev->suspend_tasks); - wake_up(&hdev->suspend_wait_q); - } -} - -static void hci_req_prepare_adv_monitor_suspend(struct hci_request *req, - bool suspending) -{ - struct hci_dev *hdev = req->hdev; - - switch (hci_get_adv_monitor_offload_ext(hdev)) { - case HCI_ADV_MONITOR_EXT_MSFT: - if (suspending) - msft_suspend(hdev); - else - msft_resume(hdev); - break; - default: - return; - } - - /* No need to block when enabling since it's on resume path */ - if (hdev->suspended && suspending) - set_bit(SUSPEND_SET_ADV_FILTER, hdev->suspend_tasks); -} - -/* Call with hci_dev_lock */ -void hci_req_prepare_suspend(struct hci_dev *hdev, enum suspended_state next) -{ - int old_state; - struct hci_conn *conn; - struct hci_request req; - u8 page_scan; - int disconnect_counter; - - if (next == hdev->suspend_state) { - bt_dev_dbg(hdev, "Same state before and after: %d", next); - goto done; - } - - hdev->suspend_state = next; - hci_req_init(&req, hdev); - - if (next == BT_SUSPEND_DISCONNECT) { - /* Mark device as suspended */ - hdev->suspended = true; - - /* Pause discovery if not already stopped */ - old_state = hdev->discovery.state; - if (old_state != DISCOVERY_STOPPED) { - set_bit(SUSPEND_PAUSE_DISCOVERY, hdev->suspend_tasks); - hci_discovery_set_state(hdev, DISCOVERY_STOPPING); - queue_work(hdev->req_workqueue, &hdev->discov_update); - } - - hdev->discovery_paused = true; - hdev->discovery_old_state = old_state; - - /* Stop directed advertising */ - old_state = hci_dev_test_flag(hdev, HCI_ADVERTISING); - if (old_state) { - set_bit(SUSPEND_PAUSE_ADVERTISING, hdev->suspend_tasks); - cancel_delayed_work(&hdev->discov_off); - queue_delayed_work(hdev->req_workqueue, - &hdev->discov_off, 0); - } - - /* Pause other advertisements */ - if (hdev->adv_instance_cnt) - __hci_req_pause_adv_instances(&req); - - hdev->advertising_paused = true; - hdev->advertising_old_state = old_state; - - /* Disable page scan if enabled */ - if (test_bit(HCI_PSCAN, &hdev->flags)) { - page_scan = SCAN_DISABLED; - hci_req_add(&req, HCI_OP_WRITE_SCAN_ENABLE, 1, - &page_scan); - set_bit(SUSPEND_SCAN_DISABLE, hdev->suspend_tasks); - } - - /* Disable LE passive scan if enabled */ - if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { - cancel_interleave_scan(hdev); - hci_req_add_le_scan_disable(&req, false); - } - - /* Disable advertisement filters */ - hci_req_prepare_adv_monitor_suspend(&req, true); - - /* Prevent disconnects from causing scanning to be re-enabled */ - hdev->scanning_paused = true; - - /* Run commands before disconnecting */ - hci_req_run(&req, suspend_req_complete); - - disconnect_counter = 0; - /* Soft disconnect everything (power off) */ - list_for_each_entry(conn, &hdev->conn_hash.list, list) { - hci_disconnect(conn, HCI_ERROR_REMOTE_POWER_OFF); - disconnect_counter++; - } - - if (disconnect_counter > 0) { - bt_dev_dbg(hdev, - "Had %d disconnects. Will wait on them", - disconnect_counter); - set_bit(SUSPEND_DISCONNECTING, hdev->suspend_tasks); - } - } else if (next == BT_SUSPEND_CONFIGURE_WAKE) { - /* Unpause to take care of updating scanning params */ - hdev->scanning_paused = false; - /* Enable event filter for paired devices */ - hci_req_set_event_filter(&req); - /* Enable passive scan at lower duty cycle */ - __hci_update_background_scan(&req); - /* Pause scan changes again. */ - hdev->scanning_paused = true; - hci_req_run(&req, suspend_req_complete); - } else { - hdev->suspended = false; - hdev->scanning_paused = false; - - /* Clear any event filters and restore scan state */ - hci_req_clear_event_filter(&req); - __hci_req_update_scan(&req); - - /* Reset passive/background scanning to normal */ - __hci_update_background_scan(&req); - /* Enable all of the advertisement filters */ - hci_req_prepare_adv_monitor_suspend(&req, false); - - /* Unpause directed advertising */ - hdev->advertising_paused = false; - if (hdev->advertising_old_state) { - set_bit(SUSPEND_UNPAUSE_ADVERTISING, - hdev->suspend_tasks); - hci_dev_set_flag(hdev, HCI_ADVERTISING); - queue_work(hdev->req_workqueue, - &hdev->discoverable_update); - hdev->advertising_old_state = 0; - } - - /* Resume other advertisements */ - if (hdev->adv_instance_cnt) - __hci_req_resume_adv_instances(&req); - - /* Unpause discovery */ - hdev->discovery_paused = false; - if (hdev->discovery_old_state != DISCOVERY_STOPPED && - hdev->discovery_old_state != DISCOVERY_STOPPING) { - set_bit(SUSPEND_UNPAUSE_DISCOVERY, hdev->suspend_tasks); - hci_discovery_set_state(hdev, DISCOVERY_STARTING); - queue_work(hdev->req_workqueue, &hdev->discov_update); - } - - hci_req_run(&req, suspend_req_complete); - } - - hdev->suspend_state = next; - -done: - clear_bit(SUSPEND_PREPARE_NOTIFIER, hdev->suspend_tasks); - wake_up(&hdev->suspend_wait_q); -} - static bool adv_cur_instance_is_scannable(struct hci_dev *hdev) { return hci_adv_instance_is_scannable(hdev, hdev->cur_adv_instance); @@ -1548,8 +1160,7 @@ void hci_req_disable_address_resolution(struct hci_dev *hdev) struct hci_request req; __u8 enable = 0x00; - if (!use_ll_privacy(hdev) && - !hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) + if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) return; hci_req_init(&req, hdev); @@ -1692,8 +1303,7 @@ int hci_get_random_address(struct hci_dev *hdev, bool require_privacy, /* If Controller supports LL Privacy use own address type is * 0x03 */ - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) + if (use_ll_privacy(hdev)) *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; else *own_addr_type = ADDR_LE_DEV_RANDOM; @@ -1871,7 +1481,8 @@ int __hci_req_setup_ext_adv_instance(struct hci_request *req, u8 instance) hci_req_add(req, HCI_OP_LE_SET_EXT_ADV_PARAMS, sizeof(cp), &cp); - if (own_addr_type == ADDR_LE_DEV_RANDOM && + if ((own_addr_type == ADDR_LE_DEV_RANDOM || + own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) && bacmp(&random_addr, BDADDR_ANY)) { struct hci_cp_le_set_adv_set_rand_addr cp; @@ -2160,8 +1771,7 @@ int hci_update_random_address(struct hci_request *req, bool require_privacy, /* If Controller supports LL Privacy use own address type is * 0x03 */ - if (use_ll_privacy(hdev) && - hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) + if (use_ll_privacy(hdev)) *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; else *own_addr_type = ADDR_LE_DEV_RANDOM; @@ -2301,47 +1911,6 @@ static void scan_update_work(struct work_struct *work) hci_req_sync(hdev, update_scan, 0, HCI_CMD_TIMEOUT, NULL); } -static int connectable_update(struct hci_request *req, unsigned long opt) -{ - struct hci_dev *hdev = req->hdev; - - hci_dev_lock(hdev); - - __hci_req_update_scan(req); - - /* If BR/EDR is not enabled and we disable advertising as a - * by-product of disabling connectable, we need to update the - * advertising flags. - */ - if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) - __hci_req_update_adv_data(req, hdev->cur_adv_instance); - - /* Update the advertising parameters if necessary */ - if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || - !list_empty(&hdev->adv_instances)) { - if (ext_adv_capable(hdev)) - __hci_req_start_ext_adv(req, hdev->cur_adv_instance); - else - __hci_req_enable_advertising(req); - } - - __hci_update_background_scan(req); - - hci_dev_unlock(hdev); - - return 0; -} - -static void connectable_update_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - connectable_update); - u8 status; - - hci_req_sync(hdev, connectable_update, 0, HCI_CMD_TIMEOUT, &status); - mgmt_set_connectable_complete(hdev, status); -} - static u8 get_service_classes(struct hci_dev *hdev) { struct bt_uuid *uuid; @@ -2445,16 +2014,6 @@ static int discoverable_update(struct hci_request *req, unsigned long opt) return 0; } -static void discoverable_update_work(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - discoverable_update); - u8 status; - - hci_req_sync(hdev, discoverable_update, 0, HCI_CMD_TIMEOUT, &status); - mgmt_set_discoverable_complete(hdev, status); -} - void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn, u8 reason) { @@ -2548,35 +2107,6 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason) return 0; } -static int update_bg_scan(struct hci_request *req, unsigned long opt) -{ - hci_dev_lock(req->hdev); - __hci_update_background_scan(req); - hci_dev_unlock(req->hdev); - return 0; -} - -static void bg_scan_update(struct work_struct *work) -{ - struct hci_dev *hdev = container_of(work, struct hci_dev, - bg_scan_update); - struct hci_conn *conn; - u8 status; - int err; - - err = hci_req_sync(hdev, update_bg_scan, 0, HCI_CMD_TIMEOUT, &status); - if (!err) - return; - - hci_dev_lock(hdev); - - conn = hci_conn_hash_lookup_state(hdev, LE_LINK, BT_CONNECT); - if (conn) - hci_le_conn_failed(conn, status); - - hci_dev_unlock(hdev); -} - static int le_scan_disable(struct hci_request *req, unsigned long opt) { hci_req_add_le_scan_disable(req, false); @@ -3163,10 +2693,7 @@ int __hci_req_hci_power_on(struct hci_dev *hdev) void hci_request_setup(struct hci_dev *hdev) { INIT_WORK(&hdev->discov_update, discov_update); - INIT_WORK(&hdev->bg_scan_update, bg_scan_update); INIT_WORK(&hdev->scan_update, scan_update_work); - INIT_WORK(&hdev->connectable_update, connectable_update_work); - INIT_WORK(&hdev->discoverable_update, discoverable_update_work); INIT_DELAYED_WORK(&hdev->discov_off, discov_off); INIT_DELAYED_WORK(&hdev->le_scan_disable, le_scan_disable_work); INIT_DELAYED_WORK(&hdev->le_scan_restart, le_scan_restart_work); @@ -3179,10 +2706,7 @@ void hci_request_cancel_all(struct hci_dev *hdev) hci_req_sync_cancel(hdev, ENODEV); cancel_work_sync(&hdev->discov_update); - cancel_work_sync(&hdev->bg_scan_update); cancel_work_sync(&hdev->scan_update); - cancel_work_sync(&hdev->connectable_update); - cancel_work_sync(&hdev->discoverable_update); cancel_delayed_work_sync(&hdev->discov_off); cancel_delayed_work_sync(&hdev->le_scan_disable); cancel_delayed_work_sync(&hdev->le_scan_restart); diff --git a/net/bluetooth/hci_request.h b/net/bluetooth/hci_request.h index f31420f58525..5f8e8846ec74 100644 --- a/net/bluetooth/hci_request.h +++ b/net/bluetooth/hci_request.h @@ -22,9 +22,17 @@ #include <asm/unaligned.h> +#define HCI_REQ_DONE 0 +#define HCI_REQ_PEND 1 +#define HCI_REQ_CANCELED 2 + #define hci_req_sync_lock(hdev) mutex_lock(&hdev->req_lock) #define hci_req_sync_unlock(hdev) mutex_unlock(&hdev->req_lock) +#define HCI_REQ_DONE 0 +#define HCI_REQ_PEND 1 +#define HCI_REQ_CANCELED 2 + struct hci_request { struct hci_dev *hdev; struct sk_buff_head cmd_q; @@ -40,6 +48,8 @@ void hci_req_purge(struct hci_request *req); bool hci_req_status_pend(struct hci_dev *hdev); int hci_req_run(struct hci_request *req, hci_req_complete_t complete); int hci_req_run_skb(struct hci_request *req, hci_req_complete_skb_t complete); +void hci_req_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, + struct sk_buff *skb); void hci_req_add(struct hci_request *req, u16 opcode, u32 plen, const void *param); void hci_req_add_ev(struct hci_request *req, u16 opcode, u32 plen, @@ -117,10 +127,5 @@ int hci_abort_conn(struct hci_conn *conn, u8 reason); void __hci_abort_conn(struct hci_request *req, struct hci_conn *conn, u8 reason); -static inline void hci_update_background_scan(struct hci_dev *hdev) -{ - queue_work(hdev->req_workqueue, &hdev->bg_scan_update); -} - void hci_request_setup(struct hci_dev *hdev); void hci_request_cancel_all(struct hci_dev *hdev); diff --git a/net/bluetooth/hci_sock.c b/net/bluetooth/hci_sock.c index d0dad1fafe07..446573a12571 100644 --- a/net/bluetooth/hci_sock.c +++ b/net/bluetooth/hci_sock.c @@ -889,10 +889,6 @@ static int hci_sock_release(struct socket *sock) } sock_orphan(sk); - - skb_queue_purge(&sk->sk_receive_queue); - skb_queue_purge(&sk->sk_write_queue); - release_sock(sk); sock_put(sk); return 0; @@ -2058,6 +2054,12 @@ static int hci_sock_getsockopt(struct socket *sock, int level, int optname, return err; } +static void hci_sock_destruct(struct sock *sk) +{ + skb_queue_purge(&sk->sk_receive_queue); + skb_queue_purge(&sk->sk_write_queue); +} + static const struct proto_ops hci_sock_ops = { .family = PF_BLUETOOTH, .owner = THIS_MODULE, @@ -2111,6 +2113,7 @@ static int hci_sock_create(struct net *net, struct socket *sock, int protocol, sock->state = SS_UNCONNECTED; sk->sk_state = BT_OPEN; + sk->sk_destruct = hci_sock_destruct; bt_sock_link(&hci_sk_list, sk); return 0; diff --git a/net/bluetooth/hci_sync.c b/net/bluetooth/hci_sync.c new file mode 100644 index 000000000000..ad86caf41f91 --- /dev/null +++ b/net/bluetooth/hci_sync.c @@ -0,0 +1,4922 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * BlueZ - Bluetooth protocol stack for Linux + * + * Copyright (C) 2021 Intel Corporation + */ + +#include <linux/property.h> + +#include <net/bluetooth/bluetooth.h> +#include <net/bluetooth/hci_core.h> +#include <net/bluetooth/mgmt.h> + +#include "hci_request.h" +#include "hci_debugfs.h" +#include "smp.h" +#include "eir.h" +#include "msft.h" +#include "aosp.h" +#include "leds.h" + +static void hci_cmd_sync_complete(struct hci_dev *hdev, u8 result, u16 opcode, + struct sk_buff *skb) +{ + bt_dev_dbg(hdev, "result 0x%2.2x", result); + + if (hdev->req_status != HCI_REQ_PEND) + return; + + hdev->req_result = result; + hdev->req_status = HCI_REQ_DONE; + + if (skb) { + struct sock *sk = hci_skb_sk(skb); + + /* Drop sk reference if set */ + if (sk) + sock_put(sk); + + hdev->req_skb = skb_get(skb); + } + + wake_up_interruptible(&hdev->req_wait_q); +} + +static struct sk_buff *hci_cmd_sync_alloc(struct hci_dev *hdev, u16 opcode, + u32 plen, const void *param, + struct sock *sk) +{ + int len = HCI_COMMAND_HDR_SIZE + plen; + struct hci_command_hdr *hdr; + struct sk_buff *skb; + + skb = bt_skb_alloc(len, GFP_ATOMIC); + if (!skb) + return NULL; + + hdr = skb_put(skb, HCI_COMMAND_HDR_SIZE); + hdr->opcode = cpu_to_le16(opcode); + hdr->plen = plen; + + if (plen) + skb_put_data(skb, param, plen); + + bt_dev_dbg(hdev, "skb len %d", skb->len); + + hci_skb_pkt_type(skb) = HCI_COMMAND_PKT; + hci_skb_opcode(skb) = opcode; + + /* Grab a reference if command needs to be associated with a sock (e.g. + * likely mgmt socket that initiated the command). + */ + if (sk) { + hci_skb_sk(skb) = sk; + sock_hold(sk); + } + + return skb; +} + +static void hci_cmd_sync_add(struct hci_request *req, u16 opcode, u32 plen, + const void *param, u8 event, struct sock *sk) +{ + struct hci_dev *hdev = req->hdev; + struct sk_buff *skb; + + bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen); + + /* If an error occurred during request building, there is no point in + * queueing the HCI command. We can simply return. + */ + if (req->err) + return; + + skb = hci_cmd_sync_alloc(hdev, opcode, plen, param, sk); + if (!skb) { + bt_dev_err(hdev, "no memory for command (opcode 0x%4.4x)", + opcode); + req->err = -ENOMEM; + return; + } + + if (skb_queue_empty(&req->cmd_q)) + bt_cb(skb)->hci.req_flags |= HCI_REQ_START; + + bt_cb(skb)->hci.req_event = event; + + skb_queue_tail(&req->cmd_q, skb); +} + +static int hci_cmd_sync_run(struct hci_request *req) +{ + struct hci_dev *hdev = req->hdev; + struct sk_buff *skb; + unsigned long flags; + + bt_dev_dbg(hdev, "length %u", skb_queue_len(&req->cmd_q)); + + /* If an error occurred during request building, remove all HCI + * commands queued on the HCI request queue. + */ + if (req->err) { + skb_queue_purge(&req->cmd_q); + return req->err; + } + + /* Do not allow empty requests */ + if (skb_queue_empty(&req->cmd_q)) + return -ENODATA; + + skb = skb_peek_tail(&req->cmd_q); + bt_cb(skb)->hci.req_complete_skb = hci_cmd_sync_complete; + bt_cb(skb)->hci.req_flags |= HCI_REQ_SKB; + + spin_lock_irqsave(&hdev->cmd_q.lock, flags); + skb_queue_splice_tail(&req->cmd_q, &hdev->cmd_q); + spin_unlock_irqrestore(&hdev->cmd_q.lock, flags); + + queue_work(hdev->workqueue, &hdev->cmd_work); + + return 0; +} + +/* This function requires the caller holds hdev->req_lock. */ +struct sk_buff *__hci_cmd_sync_sk(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout, + struct sock *sk) +{ + struct hci_request req; + struct sk_buff *skb; + int err = 0; + + bt_dev_dbg(hdev, "Opcode 0x%4x", opcode); + + hci_req_init(&req, hdev); + + hci_cmd_sync_add(&req, opcode, plen, param, event, sk); + + hdev->req_status = HCI_REQ_PEND; + + err = hci_cmd_sync_run(&req); + if (err < 0) + return ERR_PTR(err); + + err = wait_event_interruptible_timeout(hdev->req_wait_q, + hdev->req_status != HCI_REQ_PEND, + timeout); + + if (err == -ERESTARTSYS) + return ERR_PTR(-EINTR); + + switch (hdev->req_status) { + case HCI_REQ_DONE: + err = -bt_to_errno(hdev->req_result); + break; + + case HCI_REQ_CANCELED: + err = -hdev->req_result; + break; + + default: + err = -ETIMEDOUT; + break; + } + + hdev->req_status = 0; + hdev->req_result = 0; + skb = hdev->req_skb; + hdev->req_skb = NULL; + + bt_dev_dbg(hdev, "end: err %d", err); + + if (err < 0) { + kfree_skb(skb); + return ERR_PTR(err); + } + + return skb; +} +EXPORT_SYMBOL(__hci_cmd_sync_sk); + +/* This function requires the caller holds hdev->req_lock. */ +struct sk_buff *__hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout) +{ + return __hci_cmd_sync_sk(hdev, opcode, plen, param, 0, timeout, NULL); +} +EXPORT_SYMBOL(__hci_cmd_sync); + +/* Send HCI command and wait for command complete event */ +struct sk_buff *hci_cmd_sync(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout) +{ + struct sk_buff *skb; + + if (!test_bit(HCI_UP, &hdev->flags)) + return ERR_PTR(-ENETDOWN); + + bt_dev_dbg(hdev, "opcode 0x%4.4x plen %d", opcode, plen); + + hci_req_sync_lock(hdev); + skb = __hci_cmd_sync(hdev, opcode, plen, param, timeout); + hci_req_sync_unlock(hdev); + + return skb; +} +EXPORT_SYMBOL(hci_cmd_sync); + +/* This function requires the caller holds hdev->req_lock. */ +struct sk_buff *__hci_cmd_sync_ev(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout) +{ + return __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout, + NULL); +} +EXPORT_SYMBOL(__hci_cmd_sync_ev); + +/* This function requires the caller holds hdev->req_lock. */ +int __hci_cmd_sync_status_sk(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u8 event, u32 timeout, + struct sock *sk) +{ + struct sk_buff *skb; + u8 status; + + skb = __hci_cmd_sync_sk(hdev, opcode, plen, param, event, timeout, sk); + if (IS_ERR(skb)) { + bt_dev_err(hdev, "Opcode 0x%4x failed: %ld", opcode, + PTR_ERR(skb)); + return PTR_ERR(skb); + } + + /* If command return a status event skb will be set to NULL as there are + * no parameters, in case of failure IS_ERR(skb) would have be set to + * the actual error would be found with PTR_ERR(skb). + */ + if (!skb) + return 0; + + status = skb->data[0]; + + kfree_skb(skb); + + return status; +} +EXPORT_SYMBOL(__hci_cmd_sync_status_sk); + +int __hci_cmd_sync_status(struct hci_dev *hdev, u16 opcode, u32 plen, + const void *param, u32 timeout) +{ + return __hci_cmd_sync_status_sk(hdev, opcode, plen, param, 0, timeout, + NULL); +} +EXPORT_SYMBOL(__hci_cmd_sync_status); + +static void hci_cmd_sync_work(struct work_struct *work) +{ + struct hci_dev *hdev = container_of(work, struct hci_dev, cmd_sync_work); + struct hci_cmd_sync_work_entry *entry; + hci_cmd_sync_work_func_t func; + hci_cmd_sync_work_destroy_t destroy; + void *data; + + bt_dev_dbg(hdev, ""); + + mutex_lock(&hdev->cmd_sync_work_lock); + entry = list_first_entry(&hdev->cmd_sync_work_list, + struct hci_cmd_sync_work_entry, list); + if (entry) { + list_del(&entry->list); + func = entry->func; + data = entry->data; + destroy = entry->destroy; + kfree(entry); + } else { + func = NULL; + data = NULL; + destroy = NULL; + } + mutex_unlock(&hdev->cmd_sync_work_lock); + + if (func) { + int err; + + hci_req_sync_lock(hdev); + + err = func(hdev, data); + + if (destroy) + destroy(hdev, data, err); + + hci_req_sync_unlock(hdev); + } +} + +void hci_cmd_sync_init(struct hci_dev *hdev) +{ + INIT_WORK(&hdev->cmd_sync_work, hci_cmd_sync_work); + INIT_LIST_HEAD(&hdev->cmd_sync_work_list); + mutex_init(&hdev->cmd_sync_work_lock); +} + +void hci_cmd_sync_clear(struct hci_dev *hdev) +{ + struct hci_cmd_sync_work_entry *entry, *tmp; + + cancel_work_sync(&hdev->cmd_sync_work); + + list_for_each_entry_safe(entry, tmp, &hdev->cmd_sync_work_list, list) { + if (entry->destroy) + entry->destroy(hdev, entry->data, -ECANCELED); + + list_del(&entry->list); + kfree(entry); + } +} + +int hci_cmd_sync_queue(struct hci_dev *hdev, hci_cmd_sync_work_func_t func, + void *data, hci_cmd_sync_work_destroy_t destroy) +{ + struct hci_cmd_sync_work_entry *entry; + + entry = kmalloc(sizeof(*entry), GFP_KERNEL); + if (!entry) + return -ENOMEM; + + entry->func = func; + entry->data = data; + entry->destroy = destroy; + + mutex_lock(&hdev->cmd_sync_work_lock); + list_add_tail(&entry->list, &hdev->cmd_sync_work_list); + mutex_unlock(&hdev->cmd_sync_work_lock); + + queue_work(hdev->req_workqueue, &hdev->cmd_sync_work); + + return 0; +} +EXPORT_SYMBOL(hci_cmd_sync_queue); + +int hci_update_eir_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_eir cp; + + bt_dev_dbg(hdev, ""); + + if (!hdev_is_powered(hdev)) + return 0; + + if (!lmp_ext_inq_capable(hdev)) + return 0; + + if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) + return 0; + + if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + eir_create(hdev, cp.data); + + if (memcmp(cp.data, hdev->eir, sizeof(cp.data)) == 0) + return 0; + + memcpy(hdev->eir, cp.data, sizeof(cp.data)); + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); +} + +static u8 get_service_classes(struct hci_dev *hdev) +{ + struct bt_uuid *uuid; + u8 val = 0; + + list_for_each_entry(uuid, &hdev->uuids, list) + val |= uuid->svc_hint; + + return val; +} + +int hci_update_class_sync(struct hci_dev *hdev) +{ + u8 cod[3]; + + bt_dev_dbg(hdev, ""); + + if (!hdev_is_powered(hdev)) + return 0; + + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + return 0; + + if (hci_dev_test_flag(hdev, HCI_SERVICE_CACHE)) + return 0; + + cod[0] = hdev->minor_class; + cod[1] = hdev->major_class; + cod[2] = get_service_classes(hdev); + + if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) + cod[1] |= 0x20; + + if (memcmp(cod, hdev->dev_class, 3) == 0) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CLASS_OF_DEV, + sizeof(cod), cod, HCI_CMD_TIMEOUT); +} + +static bool is_advertising_allowed(struct hci_dev *hdev, bool connectable) +{ + /* If there is no connection we are OK to advertise. */ + if (hci_conn_num(hdev, LE_LINK) == 0) + return true; + + /* Check le_states if there is any connection in peripheral role. */ + if (hdev->conn_hash.le_num_peripheral > 0) { + /* Peripheral connection state and non connectable mode + * bit 20. + */ + if (!connectable && !(hdev->le_states[2] & 0x10)) + return false; + + /* Peripheral connection state and connectable mode bit 38 + * and scannable bit 21. + */ + if (connectable && (!(hdev->le_states[4] & 0x40) || + !(hdev->le_states[2] & 0x20))) + return false; + } + + /* Check le_states if there is any connection in central role. */ + if (hci_conn_num(hdev, LE_LINK) != hdev->conn_hash.le_num_peripheral) { + /* Central connection state and non connectable mode bit 18. */ + if (!connectable && !(hdev->le_states[2] & 0x02)) + return false; + + /* Central connection state and connectable mode bit 35 and + * scannable 19. + */ + if (connectable && (!(hdev->le_states[4] & 0x08) || + !(hdev->le_states[2] & 0x08))) + return false; + } + + return true; +} + +static bool adv_use_rpa(struct hci_dev *hdev, uint32_t flags) +{ + /* If privacy is not enabled don't use RPA */ + if (!hci_dev_test_flag(hdev, HCI_PRIVACY)) + return false; + + /* If basic privacy mode is enabled use RPA */ + if (!hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) + return true; + + /* If limited privacy mode is enabled don't use RPA if we're + * both discoverable and bondable. + */ + if ((flags & MGMT_ADV_FLAG_DISCOV) && + hci_dev_test_flag(hdev, HCI_BONDABLE)) + return false; + + /* We're neither bondable nor discoverable in the limited + * privacy mode, therefore use RPA. + */ + return true; +} + +static int hci_set_random_addr_sync(struct hci_dev *hdev, bdaddr_t *rpa) +{ + /* If we're advertising or initiating an LE connection we can't + * go ahead and change the random address at this time. This is + * because the eventual initiator address used for the + * subsequently created connection will be undefined (some + * controllers use the new address and others the one we had + * when the operation started). + * + * In this kind of scenario skip the update and let the random + * address be updated at the next cycle. + */ + if (hci_dev_test_flag(hdev, HCI_LE_ADV) || + hci_lookup_le_connect(hdev)) { + bt_dev_dbg(hdev, "Deferring random address update"); + hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); + return 0; + } + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RANDOM_ADDR, + 6, rpa, HCI_CMD_TIMEOUT); +} + +int hci_update_random_address_sync(struct hci_dev *hdev, bool require_privacy, + bool rpa, u8 *own_addr_type) +{ + int err; + + /* If privacy is enabled use a resolvable private address. If + * current RPA has expired or there is something else than + * the current RPA in use, then generate a new one. + */ + if (rpa) { + /* If Controller supports LL Privacy use own address type is + * 0x03 + */ + if (use_ll_privacy(hdev)) + *own_addr_type = ADDR_LE_DEV_RANDOM_RESOLVED; + else + *own_addr_type = ADDR_LE_DEV_RANDOM; + + /* Check if RPA is valid */ + if (rpa_valid(hdev)) + return 0; + + err = smp_generate_rpa(hdev, hdev->irk, &hdev->rpa); + if (err < 0) { + bt_dev_err(hdev, "failed to generate new RPA"); + return err; + } + + err = hci_set_random_addr_sync(hdev, &hdev->rpa); + if (err) + return err; + + return 0; + } + + /* In case of required privacy without resolvable private address, + * use an non-resolvable private address. This is useful for active + * scanning and non-connectable advertising. + */ + if (require_privacy) { + bdaddr_t nrpa; + + while (true) { + /* The non-resolvable private address is generated + * from random six bytes with the two most significant + * bits cleared. + */ + get_random_bytes(&nrpa, 6); + nrpa.b[5] &= 0x3f; + + /* The non-resolvable private address shall not be + * equal to the public address. + */ + if (bacmp(&hdev->bdaddr, &nrpa)) + break; + } + + *own_addr_type = ADDR_LE_DEV_RANDOM; + + return hci_set_random_addr_sync(hdev, &nrpa); + } + + /* If forcing static address is in use or there is no public + * address use the static address as random address (but skip + * the HCI command if the current random address is already the + * static one. + * + * In case BR/EDR has been disabled on a dual-mode controller + * and a static address has been configured, then use that + * address instead of the public BR/EDR address. + */ + if (hci_dev_test_flag(hdev, HCI_FORCE_STATIC_ADDR) || + !bacmp(&hdev->bdaddr, BDADDR_ANY) || + (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) && + bacmp(&hdev->static_addr, BDADDR_ANY))) { + *own_addr_type = ADDR_LE_DEV_RANDOM; + if (bacmp(&hdev->static_addr, &hdev->random_addr)) + return hci_set_random_addr_sync(hdev, + &hdev->static_addr); + return 0; + } + + /* Neither privacy nor static address is being used so use a + * public address. + */ + *own_addr_type = ADDR_LE_DEV_PUBLIC; + + return 0; +} + +static int hci_disable_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_ext_adv_enable *cp; + struct hci_cp_ext_adv_set *set; + u8 data[sizeof(*cp) + sizeof(*set) * 1]; + u8 size; + + /* If request specifies an instance that doesn't exist, fail */ + if (instance > 0) { + struct adv_info *adv; + + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return -EINVAL; + + /* If not enabled there is nothing to do */ + if (!adv->enabled) + return 0; + } + + memset(data, 0, sizeof(data)); + + cp = (void *)data; + set = (void *)cp->data; + + /* Instance 0x00 indicates all advertising instances will be disabled */ + cp->num_of_sets = !!instance; + cp->enable = 0x00; + + set->handle = instance; + + size = sizeof(*cp) + sizeof(*set) * cp->num_of_sets; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_ENABLE, + size, data, HCI_CMD_TIMEOUT); +} + +static int hci_set_adv_set_random_addr_sync(struct hci_dev *hdev, u8 instance, + bdaddr_t *random_addr) +{ + struct hci_cp_le_set_adv_set_rand_addr cp; + int err; + + if (!instance) { + /* Instance 0x00 doesn't have an adv_info, instead it uses + * hdev->random_addr to track its address so whenever it needs + * to be updated this also set the random address since + * hdev->random_addr is shared with scan state machine. + */ + err = hci_set_random_addr_sync(hdev, random_addr); + if (err) + return err; + } + + memset(&cp, 0, sizeof(cp)); + + cp.handle = instance; + bacpy(&cp.bdaddr, random_addr); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_SET_RAND_ADDR, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_setup_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_ext_adv_params cp; + bool connectable; + u32 flags; + bdaddr_t random_addr; + u8 own_addr_type; + int err; + struct adv_info *adv; + bool secondary_adv; + + if (instance > 0) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return -EINVAL; + } else { + adv = NULL; + } + + /* Updating parameters of an active instance will return a + * Command Disallowed error, so we must first disable the + * instance if it is active. + */ + if (adv && !adv->pending) { + err = hci_disable_ext_adv_instance_sync(hdev, instance); + if (err) + return err; + } + + flags = hci_adv_instance_flags(hdev, instance); + + /* If the "connectable" instance flag was not set, then choose between + * ADV_IND and ADV_NONCONN_IND based on the global connectable setting. + */ + connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || + mgmt_get_connectable(hdev); + + if (!is_advertising_allowed(hdev, connectable)) + return -EPERM; + + /* Set require_privacy to true only when non-connectable + * advertising is used. In that case it is fine to use a + * non-resolvable private address. + */ + err = hci_get_random_address(hdev, !connectable, + adv_use_rpa(hdev, flags), adv, + &own_addr_type, &random_addr); + if (err < 0) + return err; + + memset(&cp, 0, sizeof(cp)); + + if (adv) { + hci_cpu_to_le24(adv->min_interval, cp.min_interval); + hci_cpu_to_le24(adv->max_interval, cp.max_interval); + cp.tx_power = adv->tx_power; + } else { + hci_cpu_to_le24(hdev->le_adv_min_interval, cp.min_interval); + hci_cpu_to_le24(hdev->le_adv_max_interval, cp.max_interval); + cp.tx_power = HCI_ADV_TX_POWER_NO_PREFERENCE; + } + + secondary_adv = (flags & MGMT_ADV_FLAG_SEC_MASK); + + if (connectable) { + if (secondary_adv) + cp.evt_properties = cpu_to_le16(LE_EXT_ADV_CONN_IND); + else + cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_IND); + } else if (hci_adv_instance_is_scannable(hdev, instance) || + (flags & MGMT_ADV_PARAM_SCAN_RSP)) { + if (secondary_adv) + cp.evt_properties = cpu_to_le16(LE_EXT_ADV_SCAN_IND); + else + cp.evt_properties = cpu_to_le16(LE_LEGACY_ADV_SCAN_IND); + } else { + if (secondary_adv) + cp.evt_properties = cpu_to_le16(LE_EXT_ADV_NON_CONN_IND); + else + cp.evt_properties = cpu_to_le16(LE_LEGACY_NONCONN_IND); + } + + /* If Own_Address_Type equals 0x02 or 0x03, the Peer_Address parameter + * contains the peer’s Identity Address and the Peer_Address_Type + * parameter contains the peer’s Identity Type (i.e., 0x00 or 0x01). + * These parameters are used to locate the corresponding local IRK in + * the resolving list; this IRK is used to generate their own address + * used in the advertisement. + */ + if (own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) + hci_copy_identity_address(hdev, &cp.peer_addr, + &cp.peer_addr_type); + + cp.own_addr_type = own_addr_type; + cp.channel_map = hdev->le_adv_channel_map; + cp.handle = instance; + + if (flags & MGMT_ADV_FLAG_SEC_2M) { + cp.primary_phy = HCI_ADV_PHY_1M; + cp.secondary_phy = HCI_ADV_PHY_2M; + } else if (flags & MGMT_ADV_FLAG_SEC_CODED) { + cp.primary_phy = HCI_ADV_PHY_CODED; + cp.secondary_phy = HCI_ADV_PHY_CODED; + } else { + /* In all other cases use 1M */ + cp.primary_phy = HCI_ADV_PHY_1M; + cp.secondary_phy = HCI_ADV_PHY_1M; + } + + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_PARAMS, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (err) + return err; + + if ((own_addr_type == ADDR_LE_DEV_RANDOM || + own_addr_type == ADDR_LE_DEV_RANDOM_RESOLVED) && + bacmp(&random_addr, BDADDR_ANY)) { + /* Check if random address need to be updated */ + if (adv) { + if (!bacmp(&random_addr, &adv->random_addr)) + return 0; + } else { + if (!bacmp(&random_addr, &hdev->random_addr)) + return 0; + } + + return hci_set_adv_set_random_addr_sync(hdev, instance, + &random_addr); + } + + return 0; +} + +static int hci_set_ext_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct { + struct hci_cp_le_set_ext_scan_rsp_data cp; + u8 data[HCI_MAX_EXT_AD_LENGTH]; + } pdu; + u8 len; + + memset(&pdu, 0, sizeof(pdu)); + + len = eir_create_scan_rsp(hdev, instance, pdu.data); + + if (hdev->scan_rsp_data_len == len && + !memcmp(pdu.data, hdev->scan_rsp_data, len)) + return 0; + + memcpy(hdev->scan_rsp_data, pdu.data, len); + hdev->scan_rsp_data_len = len; + + pdu.cp.handle = instance; + pdu.cp.length = len; + pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; + pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_RSP_DATA, + sizeof(pdu.cp) + len, &pdu.cp, + HCI_CMD_TIMEOUT); +} + +static int __hci_set_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_scan_rsp_data cp; + u8 len; + + memset(&cp, 0, sizeof(cp)); + + len = eir_create_scan_rsp(hdev, instance, cp.data); + + if (hdev->scan_rsp_data_len == len && + !memcmp(cp.data, hdev->scan_rsp_data, len)) + return 0; + + memcpy(hdev->scan_rsp_data, cp.data, sizeof(cp.data)); + hdev->scan_rsp_data_len = len; + + cp.length = len; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_RSP_DATA, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_update_scan_rsp_data_sync(struct hci_dev *hdev, u8 instance) +{ + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return 0; + + if (ext_adv_capable(hdev)) + return hci_set_ext_scan_rsp_data_sync(hdev, instance); + + return __hci_set_scan_rsp_data_sync(hdev, instance); +} + +int hci_enable_ext_advertising_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_ext_adv_enable *cp; + struct hci_cp_ext_adv_set *set; + u8 data[sizeof(*cp) + sizeof(*set) * 1]; + struct adv_info *adv; + + if (instance > 0) { + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return -EINVAL; + /* If already enabled there is nothing to do */ + if (adv->enabled) + return 0; + } else { + adv = NULL; + } + + cp = (void *)data; + set = (void *)cp->data; + + memset(cp, 0, sizeof(*cp)); + + cp->enable = 0x01; + cp->num_of_sets = 0x01; + + memset(set, 0, sizeof(*set)); + + set->handle = instance; + + /* Set duration per instance since controller is responsible for + * scheduling it. + */ + if (adv && adv->timeout) { + u16 duration = adv->timeout * MSEC_PER_SEC; + + /* Time = N * 10 ms */ + set->duration = cpu_to_le16(duration / 10); + } + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_ENABLE, + sizeof(*cp) + + sizeof(*set) * cp->num_of_sets, + data, HCI_CMD_TIMEOUT); +} + +int hci_start_ext_adv_sync(struct hci_dev *hdev, u8 instance) +{ + int err; + + err = hci_setup_ext_adv_instance_sync(hdev, instance); + if (err) + return err; + + err = hci_set_ext_scan_rsp_data_sync(hdev, instance); + if (err) + return err; + + return hci_enable_ext_advertising_sync(hdev, instance); +} + +static int hci_start_adv_sync(struct hci_dev *hdev, u8 instance) +{ + int err; + + if (ext_adv_capable(hdev)) + return hci_start_ext_adv_sync(hdev, instance); + + err = hci_update_adv_data_sync(hdev, instance); + if (err) + return err; + + err = hci_update_scan_rsp_data_sync(hdev, instance); + if (err) + return err; + + return hci_enable_advertising_sync(hdev); +} + +int hci_enable_advertising_sync(struct hci_dev *hdev) +{ + struct adv_info *adv_instance; + struct hci_cp_le_set_adv_param cp; + u8 own_addr_type, enable = 0x01; + bool connectable; + u16 adv_min_interval, adv_max_interval; + u32 flags; + u8 status; + + if (ext_adv_capable(hdev)) + return hci_enable_ext_advertising_sync(hdev, + hdev->cur_adv_instance); + + flags = hci_adv_instance_flags(hdev, hdev->cur_adv_instance); + adv_instance = hci_find_adv_instance(hdev, hdev->cur_adv_instance); + + /* If the "connectable" instance flag was not set, then choose between + * ADV_IND and ADV_NONCONN_IND based on the global connectable setting. + */ + connectable = (flags & MGMT_ADV_FLAG_CONNECTABLE) || + mgmt_get_connectable(hdev); + + if (!is_advertising_allowed(hdev, connectable)) + return -EINVAL; + + status = hci_disable_advertising_sync(hdev); + if (status) + return status; + + /* Clear the HCI_LE_ADV bit temporarily so that the + * hci_update_random_address knows that it's safe to go ahead + * and write a new random address. The flag will be set back on + * as soon as the SET_ADV_ENABLE HCI command completes. + */ + hci_dev_clear_flag(hdev, HCI_LE_ADV); + + /* Set require_privacy to true only when non-connectable + * advertising is used. In that case it is fine to use a + * non-resolvable private address. + */ + status = hci_update_random_address_sync(hdev, !connectable, + adv_use_rpa(hdev, flags), + &own_addr_type); + if (status) + return status; + + memset(&cp, 0, sizeof(cp)); + + if (adv_instance) { + adv_min_interval = adv_instance->min_interval; + adv_max_interval = adv_instance->max_interval; + } else { + adv_min_interval = hdev->le_adv_min_interval; + adv_max_interval = hdev->le_adv_max_interval; + } + + if (connectable) { + cp.type = LE_ADV_IND; + } else { + if (hci_adv_instance_is_scannable(hdev, hdev->cur_adv_instance)) + cp.type = LE_ADV_SCAN_IND; + else + cp.type = LE_ADV_NONCONN_IND; + + if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE) || + hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) { + adv_min_interval = DISCOV_LE_FAST_ADV_INT_MIN; + adv_max_interval = DISCOV_LE_FAST_ADV_INT_MAX; + } + } + + cp.min_interval = cpu_to_le16(adv_min_interval); + cp.max_interval = cpu_to_le16(adv_max_interval); + cp.own_address_type = own_addr_type; + cp.channel_map = hdev->le_adv_channel_map; + + status = __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_PARAM, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (status) + return status; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE, + sizeof(enable), &enable, HCI_CMD_TIMEOUT); +} + +static int enable_advertising_sync(struct hci_dev *hdev, void *data) +{ + return hci_enable_advertising_sync(hdev); +} + +int hci_enable_advertising(struct hci_dev *hdev) +{ + if (!hci_dev_test_flag(hdev, HCI_ADVERTISING) && + list_empty(&hdev->adv_instances)) + return 0; + + return hci_cmd_sync_queue(hdev, enable_advertising_sync, NULL, NULL); +} + +int hci_remove_ext_adv_instance_sync(struct hci_dev *hdev, u8 instance, + struct sock *sk) +{ + int err; + + if (!ext_adv_capable(hdev)) + return 0; + + err = hci_disable_ext_adv_instance_sync(hdev, instance); + if (err) + return err; + + /* If request specifies an instance that doesn't exist, fail */ + if (instance > 0 && !hci_find_adv_instance(hdev, instance)) + return -EINVAL; + + return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_REMOVE_ADV_SET, + sizeof(instance), &instance, 0, + HCI_CMD_TIMEOUT, sk); +} + +static void cancel_adv_timeout(struct hci_dev *hdev) +{ + if (hdev->adv_instance_timeout) { + hdev->adv_instance_timeout = 0; + cancel_delayed_work(&hdev->adv_instance_expire); + } +} + +static int hci_set_ext_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct { + struct hci_cp_le_set_ext_adv_data cp; + u8 data[HCI_MAX_EXT_AD_LENGTH]; + } pdu; + u8 len; + + memset(&pdu, 0, sizeof(pdu)); + + len = eir_create_adv_data(hdev, instance, pdu.data); + + /* There's nothing to do if the data hasn't changed */ + if (hdev->adv_data_len == len && + memcmp(pdu.data, hdev->adv_data, len) == 0) + return 0; + + memcpy(hdev->adv_data, pdu.data, len); + hdev->adv_data_len = len; + + pdu.cp.length = len; + pdu.cp.handle = instance; + pdu.cp.operation = LE_SET_ADV_DATA_OP_COMPLETE; + pdu.cp.frag_pref = LE_SET_ADV_DATA_NO_FRAG; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_ADV_DATA, + sizeof(pdu.cp) + len, &pdu.cp, + HCI_CMD_TIMEOUT); +} + +static int hci_set_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + struct hci_cp_le_set_adv_data cp; + u8 len; + + memset(&cp, 0, sizeof(cp)); + + len = eir_create_adv_data(hdev, instance, cp.data); + + /* There's nothing to do if the data hasn't changed */ + if (hdev->adv_data_len == len && + memcmp(cp.data, hdev->adv_data, len) == 0) + return 0; + + memcpy(hdev->adv_data, cp.data, sizeof(cp.data)); + hdev->adv_data_len = len; + + cp.length = len; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_DATA, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_update_adv_data_sync(struct hci_dev *hdev, u8 instance) +{ + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return 0; + + if (ext_adv_capable(hdev)) + return hci_set_ext_adv_data_sync(hdev, instance); + + return hci_set_adv_data_sync(hdev, instance); +} + +int hci_schedule_adv_instance_sync(struct hci_dev *hdev, u8 instance, + bool force) +{ + struct adv_info *adv = NULL; + u16 timeout; + + if (hci_dev_test_flag(hdev, HCI_ADVERTISING) && !ext_adv_capable(hdev)) + return -EPERM; + + if (hdev->adv_instance_timeout) + return -EBUSY; + + adv = hci_find_adv_instance(hdev, instance); + if (!adv) + return -ENOENT; + + /* A zero timeout means unlimited advertising. As long as there is + * only one instance, duration should be ignored. We still set a timeout + * in case further instances are being added later on. + * + * If the remaining lifetime of the instance is more than the duration + * then the timeout corresponds to the duration, otherwise it will be + * reduced to the remaining instance lifetime. + */ + if (adv->timeout == 0 || adv->duration <= adv->remaining_time) + timeout = adv->duration; + else + timeout = adv->remaining_time; + + /* The remaining time is being reduced unless the instance is being + * advertised without time limit. + */ + if (adv->timeout) + adv->remaining_time = adv->remaining_time - timeout; + + /* Only use work for scheduling instances with legacy advertising */ + if (!ext_adv_capable(hdev)) { + hdev->adv_instance_timeout = timeout; + queue_delayed_work(hdev->req_workqueue, + &hdev->adv_instance_expire, + msecs_to_jiffies(timeout * 1000)); + } + + /* If we're just re-scheduling the same instance again then do not + * execute any HCI commands. This happens when a single instance is + * being advertised. + */ + if (!force && hdev->cur_adv_instance == instance && + hci_dev_test_flag(hdev, HCI_LE_ADV)) + return 0; + + hdev->cur_adv_instance = instance; + + return hci_start_adv_sync(hdev, instance); +} + +static int hci_clear_adv_sets_sync(struct hci_dev *hdev, struct sock *sk) +{ + int err; + + if (!ext_adv_capable(hdev)) + return 0; + + /* Disable instance 0x00 to disable all instances */ + err = hci_disable_ext_adv_instance_sync(hdev, 0x00); + if (err) + return err; + + return __hci_cmd_sync_status_sk(hdev, HCI_OP_LE_CLEAR_ADV_SETS, + 0, NULL, 0, HCI_CMD_TIMEOUT, sk); +} + +static int hci_clear_adv_sync(struct hci_dev *hdev, struct sock *sk, bool force) +{ + struct adv_info *adv, *n; + + if (ext_adv_capable(hdev)) + /* Remove all existing sets */ + return hci_clear_adv_sets_sync(hdev, sk); + + /* This is safe as long as there is no command send while the lock is + * held. + */ + hci_dev_lock(hdev); + + /* Cleanup non-ext instances */ + list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) { + u8 instance = adv->instance; + int err; + + if (!(force || adv->timeout)) + continue; + + err = hci_remove_adv_instance(hdev, instance); + if (!err) + mgmt_advertising_removed(sk, hdev, instance); + } + + hci_dev_unlock(hdev); + + return 0; +} + +static int hci_remove_adv_sync(struct hci_dev *hdev, u8 instance, + struct sock *sk) +{ + int err; + + /* If we use extended advertising, instance has to be removed first. */ + if (ext_adv_capable(hdev)) + return hci_remove_ext_adv_instance_sync(hdev, instance, sk); + + /* This is safe as long as there is no command send while the lock is + * held. + */ + hci_dev_lock(hdev); + + err = hci_remove_adv_instance(hdev, instance); + if (!err) + mgmt_advertising_removed(sk, hdev, instance); + + hci_dev_unlock(hdev); + + return err; +} + +/* For a single instance: + * - force == true: The instance will be removed even when its remaining + * lifetime is not zero. + * - force == false: the instance will be deactivated but kept stored unless + * the remaining lifetime is zero. + * + * For instance == 0x00: + * - force == true: All instances will be removed regardless of their timeout + * setting. + * - force == false: Only instances that have a timeout will be removed. + */ +int hci_remove_advertising_sync(struct hci_dev *hdev, struct sock *sk, + u8 instance, bool force) +{ + struct adv_info *next = NULL; + int err; + + /* Cancel any timeout concerning the removed instance(s). */ + if (!instance || hdev->cur_adv_instance == instance) + cancel_adv_timeout(hdev); + + /* Get the next instance to advertise BEFORE we remove + * the current one. This can be the same instance again + * if there is only one instance. + */ + if (hdev->cur_adv_instance == instance) + next = hci_get_next_instance(hdev, instance); + + if (!instance) { + err = hci_clear_adv_sync(hdev, sk, force); + if (err) + return err; + } else { + struct adv_info *adv = hci_find_adv_instance(hdev, instance); + + if (force || (adv && adv->timeout && !adv->remaining_time)) { + /* Don't advertise a removed instance. */ + if (next && next->instance == instance) + next = NULL; + + err = hci_remove_adv_sync(hdev, instance, sk); + if (err) + return err; + } + } + + if (!hdev_is_powered(hdev) || hci_dev_test_flag(hdev, HCI_ADVERTISING)) + return 0; + + if (next && !ext_adv_capable(hdev)) + hci_schedule_adv_instance_sync(hdev, next->instance, false); + + return 0; +} + +int hci_read_rssi_sync(struct hci_dev *hdev, __le16 handle) +{ + struct hci_cp_read_rssi cp; + + cp.handle = handle; + return __hci_cmd_sync_status(hdev, HCI_OP_READ_RSSI, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_read_clock_sync(struct hci_dev *hdev, struct hci_cp_read_clock *cp) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_CLOCK, + sizeof(*cp), cp, HCI_CMD_TIMEOUT); +} + +int hci_read_tx_power_sync(struct hci_dev *hdev, __le16 handle, u8 type) +{ + struct hci_cp_read_tx_power cp; + + cp.handle = handle; + cp.type = type; + return __hci_cmd_sync_status(hdev, HCI_OP_READ_TX_POWER, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_disable_advertising_sync(struct hci_dev *hdev) +{ + u8 enable = 0x00; + + /* If controller is not advertising we are done. */ + if (!hci_dev_test_flag(hdev, HCI_LE_ADV)) + return 0; + + if (ext_adv_capable(hdev)) + return hci_disable_ext_adv_instance_sync(hdev, 0x00); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADV_ENABLE, + sizeof(enable), &enable, HCI_CMD_TIMEOUT); +} + +static int hci_le_set_ext_scan_enable_sync(struct hci_dev *hdev, u8 val, + u8 filter_dup) +{ + struct hci_cp_le_set_ext_scan_enable cp; + + memset(&cp, 0, sizeof(cp)); + cp.enable = val; + cp.filter_dup = filter_dup; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_ENABLE, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_le_set_scan_enable_sync(struct hci_dev *hdev, u8 val, + u8 filter_dup) +{ + struct hci_cp_le_set_scan_enable cp; + + if (use_ext_scan(hdev)) + return hci_le_set_ext_scan_enable_sync(hdev, val, filter_dup); + + memset(&cp, 0, sizeof(cp)); + cp.enable = val; + cp.filter_dup = filter_dup; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_ENABLE, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_le_set_addr_resolution_enable_sync(struct hci_dev *hdev, u8 val) +{ + if (!use_ll_privacy(hdev)) + return 0; + + /* If controller is not/already resolving we are done. */ + if (val == hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_ADDR_RESOLV_ENABLE, + sizeof(val), &val, HCI_CMD_TIMEOUT); +} + +static int hci_scan_disable_sync(struct hci_dev *hdev) +{ + int err; + + /* If controller is not scanning we are done. */ + if (!hci_dev_test_flag(hdev, HCI_LE_SCAN)) + return 0; + + if (hdev->scanning_paused) { + bt_dev_dbg(hdev, "Scanning is paused for suspend"); + return 0; + } + + err = hci_le_set_scan_enable_sync(hdev, LE_SCAN_DISABLE, 0x00); + if (err) { + bt_dev_err(hdev, "Unable to disable scanning: %d", err); + return err; + } + + return err; +} + +static bool scan_use_rpa(struct hci_dev *hdev) +{ + return hci_dev_test_flag(hdev, HCI_PRIVACY); +} + +static void hci_start_interleave_scan(struct hci_dev *hdev) +{ + hdev->interleave_scan_state = INTERLEAVE_SCAN_NO_FILTER; + queue_delayed_work(hdev->req_workqueue, + &hdev->interleave_scan, 0); +} + +static bool is_interleave_scanning(struct hci_dev *hdev) +{ + return hdev->interleave_scan_state != INTERLEAVE_SCAN_NONE; +} + +static void cancel_interleave_scan(struct hci_dev *hdev) +{ + bt_dev_dbg(hdev, "cancelling interleave scan"); + + cancel_delayed_work_sync(&hdev->interleave_scan); + + hdev->interleave_scan_state = INTERLEAVE_SCAN_NONE; +} + +/* Return true if interleave_scan wasn't started until exiting this function, + * otherwise, return false + */ +static bool hci_update_interleaved_scan_sync(struct hci_dev *hdev) +{ + /* Do interleaved scan only if all of the following are true: + * - There is at least one ADV monitor + * - At least one pending LE connection or one device to be scanned for + * - Monitor offloading is not supported + * If so, we should alternate between allowlist scan and one without + * any filters to save power. + */ + bool use_interleaving = hci_is_adv_monitoring(hdev) && + !(list_empty(&hdev->pend_le_conns) && + list_empty(&hdev->pend_le_reports)) && + hci_get_adv_monitor_offload_ext(hdev) == + HCI_ADV_MONITOR_EXT_NONE; + bool is_interleaving = is_interleave_scanning(hdev); + + if (use_interleaving && !is_interleaving) { + hci_start_interleave_scan(hdev); + bt_dev_dbg(hdev, "starting interleave scan"); + return true; + } + + if (!use_interleaving && is_interleaving) + cancel_interleave_scan(hdev); + + return false; +} + +/* Removes connection to resolve list if needed.*/ +static int hci_le_del_resolve_list_sync(struct hci_dev *hdev, + bdaddr_t *bdaddr, u8 bdaddr_type) +{ + struct hci_cp_le_del_from_resolv_list cp; + struct bdaddr_list_with_irk *entry; + + if (!use_ll_privacy(hdev)) + return 0; + + /* Check if the IRK has been programmed */ + entry = hci_bdaddr_list_lookup_with_irk(&hdev->le_resolv_list, bdaddr, + bdaddr_type); + if (!entry) + return 0; + + cp.bdaddr_type = bdaddr_type; + bacpy(&cp.bdaddr, bdaddr); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_DEL_FROM_RESOLV_LIST, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_le_del_accept_list_sync(struct hci_dev *hdev, + bdaddr_t *bdaddr, u8 bdaddr_type) +{ + struct hci_cp_le_del_from_accept_list cp; + int err; + + /* Check if device is on accept list before removing it */ + if (!hci_bdaddr_list_lookup(&hdev->le_accept_list, bdaddr, bdaddr_type)) + return 0; + + cp.bdaddr_type = bdaddr_type; + bacpy(&cp.bdaddr, bdaddr); + + /* Ignore errors when removing from resolving list as that is likely + * that the device was never added. + */ + hci_le_del_resolve_list_sync(hdev, &cp.bdaddr, cp.bdaddr_type); + + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_DEL_FROM_ACCEPT_LIST, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (err) { + bt_dev_err(hdev, "Unable to remove from allow list: %d", err); + return err; + } + + bt_dev_dbg(hdev, "Remove %pMR (0x%x) from allow list", &cp.bdaddr, + cp.bdaddr_type); + + return 0; +} + +/* Adds connection to resolve list if needed. + * Setting params to NULL programs local hdev->irk + */ +static int hci_le_add_resolve_list_sync(struct hci_dev *hdev, + struct hci_conn_params *params) +{ + struct hci_cp_le_add_to_resolv_list cp; + struct smp_irk *irk; + struct bdaddr_list_with_irk *entry; + + if (!use_ll_privacy(hdev)) + return 0; + + /* Attempt to program local identity address, type and irk if params is + * NULL. + */ + if (!params) { + if (!hci_dev_test_flag(hdev, HCI_PRIVACY)) + return 0; + + hci_copy_identity_address(hdev, &cp.bdaddr, &cp.bdaddr_type); + memcpy(cp.peer_irk, hdev->irk, 16); + goto done; + } + + irk = hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type); + if (!irk) + return 0; + + /* Check if the IK has _not_ been programmed yet. */ + entry = hci_bdaddr_list_lookup_with_irk(&hdev->le_resolv_list, + ¶ms->addr, + params->addr_type); + if (entry) + return 0; + + cp.bdaddr_type = params->addr_type; + bacpy(&cp.bdaddr, ¶ms->addr); + memcpy(cp.peer_irk, irk->val, 16); + +done: + if (hci_dev_test_flag(hdev, HCI_PRIVACY)) + memcpy(cp.local_irk, hdev->irk, 16); + else + memset(cp.local_irk, 0, 16); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_ADD_TO_RESOLV_LIST, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +/* Adds connection to allow list if needed, if the device uses RPA (has IRK) + * this attempts to program the device in the resolving list as well. + */ +static int hci_le_add_accept_list_sync(struct hci_dev *hdev, + struct hci_conn_params *params, + u8 *num_entries) +{ + struct hci_cp_le_add_to_accept_list cp; + int err; + + /* Already in accept list */ + if (hci_bdaddr_list_lookup(&hdev->le_accept_list, ¶ms->addr, + params->addr_type)) + return 0; + + /* Select filter policy to accept all advertising */ + if (*num_entries >= hdev->le_accept_list_size) + return -ENOSPC; + + /* Accept list can not be used with RPAs */ + if (!use_ll_privacy(hdev) && + hci_find_irk_by_addr(hdev, ¶ms->addr, params->addr_type)) { + return -EINVAL; + } + + /* During suspend, only wakeable devices can be in acceptlist */ + if (hdev->suspended && !hci_conn_test_flag(HCI_CONN_FLAG_REMOTE_WAKEUP, + params->current_flags)) + return 0; + + /* Attempt to program the device in the resolving list first to avoid + * having to rollback in case it fails since the resolving list is + * dynamic it can probably be smaller than the accept list. + */ + err = hci_le_add_resolve_list_sync(hdev, params); + if (err) { + bt_dev_err(hdev, "Unable to add to resolve list: %d", err); + return err; + } + + *num_entries += 1; + cp.bdaddr_type = params->addr_type; + bacpy(&cp.bdaddr, ¶ms->addr); + + err = __hci_cmd_sync_status(hdev, HCI_OP_LE_ADD_TO_ACCEPT_LIST, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (err) { + bt_dev_err(hdev, "Unable to add to allow list: %d", err); + /* Rollback the device from the resolving list */ + hci_le_del_resolve_list_sync(hdev, &cp.bdaddr, cp.bdaddr_type); + return err; + } + + bt_dev_dbg(hdev, "Add %pMR (0x%x) to allow list", &cp.bdaddr, + cp.bdaddr_type); + + return 0; +} + +/* This function disables/pause all advertising instances */ +static int hci_pause_advertising_sync(struct hci_dev *hdev) +{ + int err; + int old_state; + + /* If there are no instances or advertising has already been paused + * there is nothing to do. + */ + if (!hdev->adv_instance_cnt || hdev->advertising_paused) + return 0; + + bt_dev_dbg(hdev, "Pausing directed advertising"); + + /* Stop directed advertising */ + old_state = hci_dev_test_flag(hdev, HCI_ADVERTISING); + if (old_state) { + /* When discoverable timeout triggers, then just make sure + * the limited discoverable flag is cleared. Even in the case + * of a timeout triggered from general discoverable, it is + * safe to unconditionally clear the flag. + */ + hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); + hci_dev_clear_flag(hdev, HCI_DISCOVERABLE); + hdev->discov_timeout = 0; + } + + bt_dev_dbg(hdev, "Pausing advertising instances"); + + /* Call to disable any advertisements active on the controller. + * This will succeed even if no advertisements are configured. + */ + err = hci_disable_advertising_sync(hdev); + if (err) + return err; + + /* If we are using software rotation, pause the loop */ + if (!ext_adv_capable(hdev)) + cancel_adv_timeout(hdev); + + hdev->advertising_paused = true; + hdev->advertising_old_state = old_state; + + return 0; +} + +/* This function enables all user advertising instances */ +static int hci_resume_advertising_sync(struct hci_dev *hdev) +{ + struct adv_info *adv, *tmp; + int err; + + /* If advertising has not been paused there is nothing to do. */ + if (!hdev->advertising_paused) + return 0; + + /* Resume directed advertising */ + hdev->advertising_paused = false; + if (hdev->advertising_old_state) { + hci_dev_set_flag(hdev, HCI_ADVERTISING); + hdev->advertising_old_state = 0; + } + + bt_dev_dbg(hdev, "Resuming advertising instances"); + + if (ext_adv_capable(hdev)) { + /* Call for each tracked instance to be re-enabled */ + list_for_each_entry_safe(adv, tmp, &hdev->adv_instances, list) { + err = hci_enable_ext_advertising_sync(hdev, + adv->instance); + if (!err) + continue; + + /* If the instance cannot be resumed remove it */ + hci_remove_ext_adv_instance_sync(hdev, adv->instance, + NULL); + } + } else { + /* Schedule for most recent instance to be restarted and begin + * the software rotation loop + */ + err = hci_schedule_adv_instance_sync(hdev, + hdev->cur_adv_instance, + true); + } + + hdev->advertising_paused = false; + + return err; +} + +struct sk_buff *hci_read_local_oob_data_sync(struct hci_dev *hdev, + bool extended, struct sock *sk) +{ + u16 opcode = extended ? HCI_OP_READ_LOCAL_OOB_EXT_DATA : + HCI_OP_READ_LOCAL_OOB_DATA; + + return __hci_cmd_sync_sk(hdev, opcode, 0, NULL, 0, HCI_CMD_TIMEOUT, sk); +} + +/* Device must not be scanning when updating the accept list. + * + * Update is done using the following sequence: + * + * use_ll_privacy((Disable Advertising) -> Disable Resolving List) -> + * Remove Devices From Accept List -> + * (has IRK && use_ll_privacy(Remove Devices From Resolving List))-> + * Add Devices to Accept List -> + * (has IRK && use_ll_privacy(Remove Devices From Resolving List)) -> + * use_ll_privacy(Enable Resolving List -> (Enable Advertising)) -> + * Enable Scanning + * + * In case of failure advertising shall be restored to its original state and + * return would disable accept list since either accept or resolving list could + * not be programmed. + * + */ +static u8 hci_update_accept_list_sync(struct hci_dev *hdev) +{ + struct hci_conn_params *params; + struct bdaddr_list *b, *t; + u8 num_entries = 0; + bool pend_conn, pend_report; + int err; + + /* Pause advertising if resolving list can be used as controllers are + * cannot accept resolving list modifications while advertising. + */ + if (use_ll_privacy(hdev)) { + err = hci_pause_advertising_sync(hdev); + if (err) { + bt_dev_err(hdev, "pause advertising failed: %d", err); + return 0x00; + } + } + + /* Disable address resolution while reprogramming accept list since + * devices that do have an IRK will be programmed in the resolving list + * when LL Privacy is enabled. + */ + err = hci_le_set_addr_resolution_enable_sync(hdev, 0x00); + if (err) { + bt_dev_err(hdev, "Unable to disable LL privacy: %d", err); + goto done; + } + + /* Go through the current accept list programmed into the + * controller one by one and check if that address is still + * in the list of pending connections or list of devices to + * report. If not present in either list, then remove it from + * the controller. + */ + list_for_each_entry_safe(b, t, &hdev->le_accept_list, list) { + pend_conn = hci_pend_le_action_lookup(&hdev->pend_le_conns, + &b->bdaddr, + b->bdaddr_type); + pend_report = hci_pend_le_action_lookup(&hdev->pend_le_reports, + &b->bdaddr, + b->bdaddr_type); + + /* If the device is not likely to connect or report, + * remove it from the acceptlist. + */ + if (!pend_conn && !pend_report) { + hci_le_del_accept_list_sync(hdev, &b->bdaddr, + b->bdaddr_type); + continue; + } + + num_entries++; + } + + /* Since all no longer valid accept list entries have been + * removed, walk through the list of pending connections + * and ensure that any new device gets programmed into + * the controller. + * + * If the list of the devices is larger than the list of + * available accept list entries in the controller, then + * just abort and return filer policy value to not use the + * accept list. + */ + list_for_each_entry(params, &hdev->pend_le_conns, action) { + err = hci_le_add_accept_list_sync(hdev, params, &num_entries); + if (err) + goto done; + } + + /* After adding all new pending connections, walk through + * the list of pending reports and also add these to the + * accept list if there is still space. Abort if space runs out. + */ + list_for_each_entry(params, &hdev->pend_le_reports, action) { + err = hci_le_add_accept_list_sync(hdev, params, &num_entries); + if (err) + goto done; + } + + /* Use the allowlist unless the following conditions are all true: + * - We are not currently suspending + * - There are 1 or more ADV monitors registered and it's not offloaded + * - Interleaved scanning is not currently using the allowlist + */ + if (!idr_is_empty(&hdev->adv_monitors_idr) && !hdev->suspended && + hci_get_adv_monitor_offload_ext(hdev) == HCI_ADV_MONITOR_EXT_NONE && + hdev->interleave_scan_state != INTERLEAVE_SCAN_ALLOWLIST) + err = -EINVAL; + +done: + /* Enable address resolution when LL Privacy is enabled. */ + err = hci_le_set_addr_resolution_enable_sync(hdev, 0x01); + if (err) + bt_dev_err(hdev, "Unable to enable LL privacy: %d", err); + + /* Resume advertising if it was paused */ + if (use_ll_privacy(hdev)) + hci_resume_advertising_sync(hdev); + + /* Select filter policy to use accept list */ + return err ? 0x00 : 0x01; +} + +/* Returns true if an le connection is in the scanning state */ +static inline bool hci_is_le_conn_scanning(struct hci_dev *hdev) +{ + struct hci_conn_hash *h = &hdev->conn_hash; + struct hci_conn *c; + + rcu_read_lock(); + + list_for_each_entry_rcu(c, &h->list, list) { + if (c->type == LE_LINK && c->state == BT_CONNECT && + test_bit(HCI_CONN_SCANNING, &c->flags)) { + rcu_read_unlock(); + return true; + } + } + + rcu_read_unlock(); + + return false; +} + +static int hci_le_set_ext_scan_param_sync(struct hci_dev *hdev, u8 type, + u16 interval, u16 window, + u8 own_addr_type, u8 filter_policy) +{ + struct hci_cp_le_set_ext_scan_params *cp; + struct hci_cp_le_scan_phy_params *phy; + u8 data[sizeof(*cp) + sizeof(*phy) * 2]; + u8 num_phy = 0; + + cp = (void *)data; + phy = (void *)cp->data; + + memset(data, 0, sizeof(data)); + + cp->own_addr_type = own_addr_type; + cp->filter_policy = filter_policy; + + if (scan_1m(hdev) || scan_2m(hdev)) { + cp->scanning_phys |= LE_SCAN_PHY_1M; + + phy->type = type; + phy->interval = cpu_to_le16(interval); + phy->window = cpu_to_le16(window); + + num_phy++; + phy++; + } + + if (scan_coded(hdev)) { + cp->scanning_phys |= LE_SCAN_PHY_CODED; + + phy->type = type; + phy->interval = cpu_to_le16(interval); + phy->window = cpu_to_le16(window); + + num_phy++; + phy++; + } + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EXT_SCAN_PARAMS, + sizeof(*cp) + sizeof(*phy) * num_phy, + data, HCI_CMD_TIMEOUT); +} + +static int hci_le_set_scan_param_sync(struct hci_dev *hdev, u8 type, + u16 interval, u16 window, + u8 own_addr_type, u8 filter_policy) +{ + struct hci_cp_le_set_scan_param cp; + + if (use_ext_scan(hdev)) + return hci_le_set_ext_scan_param_sync(hdev, type, interval, + window, own_addr_type, + filter_policy); + + memset(&cp, 0, sizeof(cp)); + cp.type = type; + cp.interval = cpu_to_le16(interval); + cp.window = cpu_to_le16(window); + cp.own_address_type = own_addr_type; + cp.filter_policy = filter_policy; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_SCAN_PARAM, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_start_scan_sync(struct hci_dev *hdev, u8 type, u16 interval, + u16 window, u8 own_addr_type, u8 filter_policy, + u8 filter_dup) +{ + int err; + + if (hdev->scanning_paused) { + bt_dev_dbg(hdev, "Scanning is paused for suspend"); + return 0; + } + + err = hci_le_set_scan_param_sync(hdev, type, interval, window, + own_addr_type, filter_policy); + if (err) + return err; + + return hci_le_set_scan_enable_sync(hdev, LE_SCAN_ENABLE, filter_dup); +} + +static int hci_passive_scan_sync(struct hci_dev *hdev) +{ + u8 own_addr_type; + u8 filter_policy; + u16 window, interval; + int err; + + if (hdev->scanning_paused) { + bt_dev_dbg(hdev, "Scanning is paused for suspend"); + return 0; + } + + err = hci_scan_disable_sync(hdev); + if (err) { + bt_dev_err(hdev, "disable scanning failed: %d", err); + return err; + } + + /* Set require_privacy to false since no SCAN_REQ are send + * during passive scanning. Not using an non-resolvable address + * here is important so that peer devices using direct + * advertising with our address will be correctly reported + * by the controller. + */ + if (hci_update_random_address_sync(hdev, false, scan_use_rpa(hdev), + &own_addr_type)) + return 0; + + if (hdev->enable_advmon_interleave_scan && + hci_update_interleaved_scan_sync(hdev)) + return 0; + + bt_dev_dbg(hdev, "interleave state %d", hdev->interleave_scan_state); + + /* Adding or removing entries from the accept list must + * happen before enabling scanning. The controller does + * not allow accept list modification while scanning. + */ + filter_policy = hci_update_accept_list_sync(hdev); + + /* When the controller is using random resolvable addresses and + * with that having LE privacy enabled, then controllers with + * Extended Scanner Filter Policies support can now enable support + * for handling directed advertising. + * + * So instead of using filter polices 0x00 (no acceptlist) + * and 0x01 (acceptlist enabled) use the new filter policies + * 0x02 (no acceptlist) and 0x03 (acceptlist enabled). + */ + if (hci_dev_test_flag(hdev, HCI_PRIVACY) && + (hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY)) + filter_policy |= 0x02; + + if (hdev->suspended) { + window = hdev->le_scan_window_suspend; + interval = hdev->le_scan_int_suspend; + } else if (hci_is_le_conn_scanning(hdev)) { + window = hdev->le_scan_window_connect; + interval = hdev->le_scan_int_connect; + } else if (hci_is_adv_monitoring(hdev)) { + window = hdev->le_scan_window_adv_monitor; + interval = hdev->le_scan_int_adv_monitor; + } else { + window = hdev->le_scan_window; + interval = hdev->le_scan_interval; + } + + bt_dev_dbg(hdev, "LE passive scan with acceptlist = %d", filter_policy); + + return hci_start_scan_sync(hdev, LE_SCAN_PASSIVE, interval, window, + own_addr_type, filter_policy, + LE_SCAN_FILTER_DUP_ENABLE); +} + +/* This function controls the passive scanning based on hdev->pend_le_conns + * list. If there are pending LE connection we start the background scanning, + * otherwise we stop it in the following sequence: + * + * If there are devices to scan: + * + * Disable Scanning -> Update Accept List -> + * use_ll_privacy((Disable Advertising) -> Disable Resolving List -> + * Update Resolving List -> Enable Resolving List -> (Enable Advertising)) -> + * Enable Scanning + * + * Otherwise: + * + * Disable Scanning + */ +int hci_update_passive_scan_sync(struct hci_dev *hdev) +{ + int err; + + if (!test_bit(HCI_UP, &hdev->flags) || + test_bit(HCI_INIT, &hdev->flags) || + hci_dev_test_flag(hdev, HCI_SETUP) || + hci_dev_test_flag(hdev, HCI_CONFIG) || + hci_dev_test_flag(hdev, HCI_AUTO_OFF) || + hci_dev_test_flag(hdev, HCI_UNREGISTER)) + return 0; + + /* No point in doing scanning if LE support hasn't been enabled */ + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return 0; + + /* If discovery is active don't interfere with it */ + if (hdev->discovery.state != DISCOVERY_STOPPED) + return 0; + + /* Reset RSSI and UUID filters when starting background scanning + * since these filters are meant for service discovery only. + * + * The Start Discovery and Start Service Discovery operations + * ensure to set proper values for RSSI threshold and UUID + * filter list. So it is safe to just reset them here. + */ + hci_discovery_filter_clear(hdev); + + bt_dev_dbg(hdev, "ADV monitoring is %s", + hci_is_adv_monitoring(hdev) ? "on" : "off"); + + if (list_empty(&hdev->pend_le_conns) && + list_empty(&hdev->pend_le_reports) && + !hci_is_adv_monitoring(hdev)) { + /* If there is no pending LE connections or devices + * to be scanned for or no ADV monitors, we should stop the + * background scanning. + */ + + bt_dev_dbg(hdev, "stopping background scanning"); + + err = hci_scan_disable_sync(hdev); + if (err) + bt_dev_err(hdev, "stop background scanning failed: %d", + err); + } else { + /* If there is at least one pending LE connection, we should + * keep the background scan running. + */ + + /* If controller is connecting, we should not start scanning + * since some controllers are not able to scan and connect at + * the same time. + */ + if (hci_lookup_le_connect(hdev)) + return 0; + + bt_dev_dbg(hdev, "start background scanning"); + + err = hci_passive_scan_sync(hdev); + if (err) + bt_dev_err(hdev, "start background scanning failed: %d", + err); + } + + return err; +} + +static int update_passive_scan_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_passive_scan_sync(hdev); +} + +int hci_update_passive_scan(struct hci_dev *hdev) +{ + /* Only queue if it would have any effect */ + if (!test_bit(HCI_UP, &hdev->flags) || + test_bit(HCI_INIT, &hdev->flags) || + hci_dev_test_flag(hdev, HCI_SETUP) || + hci_dev_test_flag(hdev, HCI_CONFIG) || + hci_dev_test_flag(hdev, HCI_AUTO_OFF) || + hci_dev_test_flag(hdev, HCI_UNREGISTER)) + return 0; + + return hci_cmd_sync_queue(hdev, update_passive_scan_sync, NULL, NULL); +} + +int hci_write_sc_support_sync(struct hci_dev *hdev, u8 val) +{ + int err; + + if (!bredr_sc_enabled(hdev) || lmp_host_sc_capable(hdev)) + return 0; + + err = __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SC_SUPPORT, + sizeof(val), &val, HCI_CMD_TIMEOUT); + + if (!err) { + if (val) { + hdev->features[1][0] |= LMP_HOST_SC; + hci_dev_set_flag(hdev, HCI_SC_ENABLED); + } else { + hdev->features[1][0] &= ~LMP_HOST_SC; + hci_dev_clear_flag(hdev, HCI_SC_ENABLED); + } + } + + return err; +} + +int hci_write_ssp_mode_sync(struct hci_dev *hdev, u8 mode) +{ + int err; + + if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED) || + lmp_host_ssp_capable(hdev)) + return 0; + + if (!mode && hci_dev_test_flag(hdev, HCI_USE_DEBUG_KEYS)) { + __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_DEBUG_MODE, + sizeof(mode), &mode, HCI_CMD_TIMEOUT); + } + + err = __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_MODE, + sizeof(mode), &mode, HCI_CMD_TIMEOUT); + if (err) + return err; + + return hci_write_sc_support_sync(hdev, 0x01); +} + +int hci_write_le_host_supported_sync(struct hci_dev *hdev, u8 le, u8 simul) +{ + struct hci_cp_write_le_host_supported cp; + + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED) || + !lmp_bredr_capable(hdev)) + return 0; + + /* Check first if we already have the right host state + * (host features set) + */ + if (le == lmp_host_le_capable(hdev) && + simul == lmp_host_le_br_capable(hdev)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + cp.le = le; + cp.simul = simul; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LE_HOST_SUPPORTED, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_powered_update_adv_sync(struct hci_dev *hdev) +{ + struct adv_info *adv, *tmp; + int err; + + if (!hci_dev_test_flag(hdev, HCI_LE_ENABLED)) + return 0; + + /* If RPA Resolution has not been enable yet it means the + * resolving list is empty and we should attempt to program the + * local IRK in order to support using own_addr_type + * ADDR_LE_DEV_RANDOM_RESOLVED (0x03). + */ + if (!hci_dev_test_flag(hdev, HCI_LL_RPA_RESOLUTION)) { + hci_le_add_resolve_list_sync(hdev, NULL); + hci_le_set_addr_resolution_enable_sync(hdev, 0x01); + } + + /* Make sure the controller has a good default for + * advertising data. This also applies to the case + * where BR/EDR was toggled during the AUTO_OFF phase. + */ + if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || + list_empty(&hdev->adv_instances)) { + if (ext_adv_capable(hdev)) { + err = hci_setup_ext_adv_instance_sync(hdev, 0x00); + if (!err) + hci_update_scan_rsp_data_sync(hdev, 0x00); + } else { + err = hci_update_adv_data_sync(hdev, 0x00); + if (!err) + hci_update_scan_rsp_data_sync(hdev, 0x00); + } + + if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) + hci_enable_advertising_sync(hdev); + } + + /* Call for each tracked instance to be scheduled */ + list_for_each_entry_safe(adv, tmp, &hdev->adv_instances, list) + hci_schedule_adv_instance_sync(hdev, adv->instance, true); + + return 0; +} + +static int hci_write_auth_enable_sync(struct hci_dev *hdev) +{ + u8 link_sec; + + link_sec = hci_dev_test_flag(hdev, HCI_LINK_SECURITY); + if (link_sec == test_bit(HCI_AUTH, &hdev->flags)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_AUTH_ENABLE, + sizeof(link_sec), &link_sec, + HCI_CMD_TIMEOUT); +} + +int hci_write_fast_connectable_sync(struct hci_dev *hdev, bool enable) +{ + struct hci_cp_write_page_scan_activity cp; + u8 type; + int err = 0; + + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + return 0; + + if (hdev->hci_ver < BLUETOOTH_VER_1_2) + return 0; + + memset(&cp, 0, sizeof(cp)); + + if (enable) { + type = PAGE_SCAN_TYPE_INTERLACED; + + /* 160 msec page scan interval */ + cp.interval = cpu_to_le16(0x0100); + } else { + type = hdev->def_page_scan_type; + cp.interval = cpu_to_le16(hdev->def_page_scan_int); + } + + cp.window = cpu_to_le16(hdev->def_page_scan_window); + + if (__cpu_to_le16(hdev->page_scan_interval) != cp.interval || + __cpu_to_le16(hdev->page_scan_window) != cp.window) { + err = __hci_cmd_sync_status(hdev, + HCI_OP_WRITE_PAGE_SCAN_ACTIVITY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); + if (err) + return err; + } + + if (hdev->page_scan_type != type) + err = __hci_cmd_sync_status(hdev, + HCI_OP_WRITE_PAGE_SCAN_TYPE, + sizeof(type), &type, + HCI_CMD_TIMEOUT); + + return err; +} + +static bool disconnected_accept_list_entries(struct hci_dev *hdev) +{ + struct bdaddr_list *b; + + list_for_each_entry(b, &hdev->accept_list, list) { + struct hci_conn *conn; + + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, &b->bdaddr); + if (!conn) + return true; + + if (conn->state != BT_CONNECTED && conn->state != BT_CONFIG) + return true; + } + + return false; +} + +static int hci_write_scan_enable_sync(struct hci_dev *hdev, u8 val) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SCAN_ENABLE, + sizeof(val), &val, + HCI_CMD_TIMEOUT); +} + +int hci_update_scan_sync(struct hci_dev *hdev) +{ + u8 scan; + + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + return 0; + + if (!hdev_is_powered(hdev)) + return 0; + + if (mgmt_powering_down(hdev)) + return 0; + + if (hdev->scanning_paused) + return 0; + + if (hci_dev_test_flag(hdev, HCI_CONNECTABLE) || + disconnected_accept_list_entries(hdev)) + scan = SCAN_PAGE; + else + scan = SCAN_DISABLED; + + if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) + scan |= SCAN_INQUIRY; + + if (test_bit(HCI_PSCAN, &hdev->flags) == !!(scan & SCAN_PAGE) && + test_bit(HCI_ISCAN, &hdev->flags) == !!(scan & SCAN_INQUIRY)) + return 0; + + return hci_write_scan_enable_sync(hdev, scan); +} + +int hci_update_name_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_local_name cp; + + memset(&cp, 0, sizeof(cp)); + + memcpy(cp.name, hdev->dev_name, sizeof(cp.name)); + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LOCAL_NAME, + sizeof(cp), &cp, + HCI_CMD_TIMEOUT); +} + +/* This function perform powered update HCI command sequence after the HCI init + * sequence which end up resetting all states, the sequence is as follows: + * + * HCI_SSP_ENABLED(Enable SSP) + * HCI_LE_ENABLED(Enable LE) + * HCI_LE_ENABLED(use_ll_privacy(Add local IRK to Resolving List) -> + * Update adv data) + * Enable Authentication + * lmp_bredr_capable(Set Fast Connectable -> Set Scan Type -> Set Class -> + * Set Name -> Set EIR) + */ +int hci_powered_update_sync(struct hci_dev *hdev) +{ + int err; + + /* Register the available SMP channels (BR/EDR and LE) only when + * successfully powering on the controller. This late + * registration is required so that LE SMP can clearly decide if + * the public address or static address is used. + */ + smp_register(hdev); + + err = hci_write_ssp_mode_sync(hdev, 0x01); + if (err) + return err; + + err = hci_write_le_host_supported_sync(hdev, 0x01, 0x00); + if (err) + return err; + + err = hci_powered_update_adv_sync(hdev); + if (err) + return err; + + err = hci_write_auth_enable_sync(hdev); + if (err) + return err; + + if (lmp_bredr_capable(hdev)) { + if (hci_dev_test_flag(hdev, HCI_FAST_CONNECTABLE)) + hci_write_fast_connectable_sync(hdev, true); + else + hci_write_fast_connectable_sync(hdev, false); + hci_update_scan_sync(hdev); + hci_update_class_sync(hdev); + hci_update_name_sync(hdev); + hci_update_eir_sync(hdev); + } + + return 0; +} + +/** + * hci_dev_get_bd_addr_from_property - Get the Bluetooth Device Address + * (BD_ADDR) for a HCI device from + * a firmware node property. + * @hdev: The HCI device + * + * Search the firmware node for 'local-bd-address'. + * + * All-zero BD addresses are rejected, because those could be properties + * that exist in the firmware tables, but were not updated by the firmware. For + * example, the DTS could define 'local-bd-address', with zero BD addresses. + */ +static void hci_dev_get_bd_addr_from_property(struct hci_dev *hdev) +{ + struct fwnode_handle *fwnode = dev_fwnode(hdev->dev.parent); + bdaddr_t ba; + int ret; + + ret = fwnode_property_read_u8_array(fwnode, "local-bd-address", + (u8 *)&ba, sizeof(ba)); + if (ret < 0 || !bacmp(&ba, BDADDR_ANY)) + return; + + bacpy(&hdev->public_addr, &ba); +} + +struct hci_init_stage { + int (*func)(struct hci_dev *hdev); +}; + +/* Run init stage NULL terminated function table */ +static int hci_init_stage_sync(struct hci_dev *hdev, + const struct hci_init_stage *stage) +{ + size_t i; + + for (i = 0; stage[i].func; i++) { + int err; + + err = stage[i].func(hdev); + if (err) + return err; + } + + return 0; +} + +/* Read Local Version */ +static int hci_read_local_version_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_VERSION, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read BD Address */ +static int hci_read_bd_addr_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_BD_ADDR, + 0, NULL, HCI_CMD_TIMEOUT); +} + +#define HCI_INIT(_func) \ +{ \ + .func = _func, \ +} + +static const struct hci_init_stage hci_init0[] = { + /* HCI_OP_READ_LOCAL_VERSION */ + HCI_INIT(hci_read_local_version_sync), + /* HCI_OP_READ_BD_ADDR */ + HCI_INIT(hci_read_bd_addr_sync), + {} +}; + +int hci_reset_sync(struct hci_dev *hdev) +{ + int err; + + set_bit(HCI_RESET, &hdev->flags); + + err = __hci_cmd_sync_status(hdev, HCI_OP_RESET, 0, NULL, + HCI_CMD_TIMEOUT); + if (err) + return err; + + return 0; +} + +static int hci_init0_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + /* Reset */ + if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) { + err = hci_reset_sync(hdev); + if (err) + return err; + } + + return hci_init_stage_sync(hdev, hci_init0); +} + +static int hci_unconf_init_sync(struct hci_dev *hdev) +{ + int err; + + if (test_bit(HCI_QUIRK_RAW_DEVICE, &hdev->quirks)) + return 0; + + err = hci_init0_sync(hdev); + if (err < 0) + return err; + + if (hci_dev_test_flag(hdev, HCI_SETUP)) + hci_debugfs_create_basic(hdev); + + return 0; +} + +/* Read Local Supported Features. */ +static int hci_read_local_features_sync(struct hci_dev *hdev) +{ + /* Not all AMP controllers support this command */ + if (hdev->dev_type == HCI_AMP && !(hdev->commands[14] & 0x20)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_FEATURES, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* BR Controller init stage 1 command sequence */ +static const struct hci_init_stage br_init1[] = { + /* HCI_OP_READ_LOCAL_FEATURES */ + HCI_INIT(hci_read_local_features_sync), + /* HCI_OP_READ_LOCAL_VERSION */ + HCI_INIT(hci_read_local_version_sync), + /* HCI_OP_READ_BD_ADDR */ + HCI_INIT(hci_read_bd_addr_sync), + {} +}; + +/* Read Local Commands */ +static int hci_read_local_cmds_sync(struct hci_dev *hdev) +{ + /* All Bluetooth 1.2 and later controllers should support the + * HCI command for reading the local supported commands. + * + * Unfortunately some controllers indicate Bluetooth 1.2 support, + * but do not have support for this command. If that is the case, + * the driver can quirk the behavior and skip reading the local + * supported commands. + */ + if (hdev->hci_ver > BLUETOOTH_VER_1_1 && + !test_bit(HCI_QUIRK_BROKEN_LOCAL_COMMANDS, &hdev->quirks)) + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_COMMANDS, + 0, NULL, HCI_CMD_TIMEOUT); + + return 0; +} + +/* Read Local AMP Info */ +static int hci_read_local_amp_info_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_AMP_INFO, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Data Blk size */ +static int hci_read_data_block_size_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_DATA_BLOCK_SIZE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Flow Control Mode */ +static int hci_read_flow_control_mode_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_FLOW_CONTROL_MODE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Location Data */ +static int hci_read_location_data_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCATION_DATA, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* AMP Controller init stage 1 command sequence */ +static const struct hci_init_stage amp_init1[] = { + /* HCI_OP_READ_LOCAL_VERSION */ + HCI_INIT(hci_read_local_version_sync), + /* HCI_OP_READ_LOCAL_COMMANDS */ + HCI_INIT(hci_read_local_cmds_sync), + /* HCI_OP_READ_LOCAL_AMP_INFO */ + HCI_INIT(hci_read_local_amp_info_sync), + /* HCI_OP_READ_DATA_BLOCK_SIZE */ + HCI_INIT(hci_read_data_block_size_sync), + /* HCI_OP_READ_FLOW_CONTROL_MODE */ + HCI_INIT(hci_read_flow_control_mode_sync), + /* HCI_OP_READ_LOCATION_DATA */ + HCI_INIT(hci_read_location_data_sync), +}; + +static int hci_init1_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + /* Reset */ + if (!test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks)) { + err = hci_reset_sync(hdev); + if (err) + return err; + } + + switch (hdev->dev_type) { + case HCI_PRIMARY: + hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_PACKET_BASED; + return hci_init_stage_sync(hdev, br_init1); + case HCI_AMP: + hdev->flow_ctl_mode = HCI_FLOW_CTL_MODE_BLOCK_BASED; + return hci_init_stage_sync(hdev, amp_init1); + default: + bt_dev_err(hdev, "Unknown device type %d", hdev->dev_type); + break; + } + + return 0; +} + +/* AMP Controller init stage 2 command sequence */ +static const struct hci_init_stage amp_init2[] = { + /* HCI_OP_READ_LOCAL_FEATURES */ + HCI_INIT(hci_read_local_features_sync), +}; + +/* Read Buffer Size (ACL mtu, max pkt, etc.) */ +static int hci_read_buffer_size_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_BUFFER_SIZE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Class of Device */ +static int hci_read_dev_class_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_CLASS_OF_DEV, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Local Name */ +static int hci_read_local_name_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_NAME, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Voice Setting */ +static int hci_read_voice_setting_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_VOICE_SETTING, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Number of Supported IAC */ +static int hci_read_num_supported_iac_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_NUM_SUPPORTED_IAC, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read Current IAC LAP */ +static int hci_read_current_iac_lap_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_READ_CURRENT_IAC_LAP, + 0, NULL, HCI_CMD_TIMEOUT); +} + +static int hci_set_event_filter_sync(struct hci_dev *hdev, u8 flt_type, + u8 cond_type, bdaddr_t *bdaddr, + u8 auto_accept) +{ + struct hci_cp_set_event_filter cp; + + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + return 0; + + memset(&cp, 0, sizeof(cp)); + cp.flt_type = flt_type; + + if (flt_type != HCI_FLT_CLEAR_ALL) { + cp.cond_type = cond_type; + bacpy(&cp.addr_conn_flt.bdaddr, bdaddr); + cp.addr_conn_flt.auto_accept = auto_accept; + } + + return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_FLT, + flt_type == HCI_FLT_CLEAR_ALL ? + sizeof(cp.flt_type) : sizeof(cp), &cp, + HCI_CMD_TIMEOUT); +} + +static int hci_clear_event_filter_sync(struct hci_dev *hdev) +{ + if (!hci_dev_test_flag(hdev, HCI_EVENT_FILTER_CONFIGURED)) + return 0; + + return hci_set_event_filter_sync(hdev, HCI_FLT_CLEAR_ALL, 0x00, + BDADDR_ANY, 0x00); +} + +/* Connection accept timeout ~20 secs */ +static int hci_write_ca_timeout_sync(struct hci_dev *hdev) +{ + __le16 param = cpu_to_le16(0x7d00); + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CA_TIMEOUT, + sizeof(param), ¶m, HCI_CMD_TIMEOUT); +} + +/* BR Controller init stage 2 command sequence */ +static const struct hci_init_stage br_init2[] = { + /* HCI_OP_READ_BUFFER_SIZE */ + HCI_INIT(hci_read_buffer_size_sync), + /* HCI_OP_READ_CLASS_OF_DEV */ + HCI_INIT(hci_read_dev_class_sync), + /* HCI_OP_READ_LOCAL_NAME */ + HCI_INIT(hci_read_local_name_sync), + /* HCI_OP_READ_VOICE_SETTING */ + HCI_INIT(hci_read_voice_setting_sync), + /* HCI_OP_READ_NUM_SUPPORTED_IAC */ + HCI_INIT(hci_read_num_supported_iac_sync), + /* HCI_OP_READ_CURRENT_IAC_LAP */ + HCI_INIT(hci_read_current_iac_lap_sync), + /* HCI_OP_SET_EVENT_FLT */ + HCI_INIT(hci_clear_event_filter_sync), + /* HCI_OP_WRITE_CA_TIMEOUT */ + HCI_INIT(hci_write_ca_timeout_sync), + {} +}; + +static int hci_write_ssp_mode_1_sync(struct hci_dev *hdev) +{ + u8 mode = 0x01; + + if (!lmp_ssp_capable(hdev) || !hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) + return 0; + + /* When SSP is available, then the host features page + * should also be available as well. However some + * controllers list the max_page as 0 as long as SSP + * has not been enabled. To achieve proper debugging + * output, force the minimum max_page to 1 at least. + */ + hdev->max_page = 0x01; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SSP_MODE, + sizeof(mode), &mode, HCI_CMD_TIMEOUT); +} + +static int hci_write_eir_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_eir cp; + + if (!lmp_ssp_capable(hdev) || hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) + return 0; + + memset(hdev->eir, 0, sizeof(hdev->eir)); + memset(&cp, 0, sizeof(cp)); + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_EIR, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); +} + +static int hci_write_inquiry_mode_sync(struct hci_dev *hdev) +{ + u8 mode; + + if (!lmp_inq_rssi_capable(hdev) && + !test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks)) + return 0; + + /* If Extended Inquiry Result events are supported, then + * they are clearly preferred over Inquiry Result with RSSI + * events. + */ + mode = lmp_ext_inq_capable(hdev) ? 0x02 : 0x01; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_INQUIRY_MODE, + sizeof(mode), &mode, HCI_CMD_TIMEOUT); +} + +static int hci_read_inq_rsp_tx_power_sync(struct hci_dev *hdev) +{ + if (!lmp_inq_tx_pwr_capable(hdev)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_INQ_RSP_TX_POWER, + 0, NULL, HCI_CMD_TIMEOUT); +} + +static int hci_read_local_ext_features_sync(struct hci_dev *hdev, u8 page) +{ + struct hci_cp_read_local_ext_features cp; + + if (!lmp_ext_feat_capable(hdev)) + return 0; + + memset(&cp, 0, sizeof(cp)); + cp.page = page; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_EXT_FEATURES, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_read_local_ext_features_1_sync(struct hci_dev *hdev) +{ + return hci_read_local_ext_features_sync(hdev, 0x01); +} + +/* HCI Controller init stage 2 command sequence */ +static const struct hci_init_stage hci_init2[] = { + /* HCI_OP_READ_LOCAL_COMMANDS */ + HCI_INIT(hci_read_local_cmds_sync), + /* HCI_OP_WRITE_SSP_MODE */ + HCI_INIT(hci_write_ssp_mode_1_sync), + /* HCI_OP_WRITE_EIR */ + HCI_INIT(hci_write_eir_sync), + /* HCI_OP_WRITE_INQUIRY_MODE */ + HCI_INIT(hci_write_inquiry_mode_sync), + /* HCI_OP_READ_INQ_RSP_TX_POWER */ + HCI_INIT(hci_read_inq_rsp_tx_power_sync), + /* HCI_OP_READ_LOCAL_EXT_FEATURES */ + HCI_INIT(hci_read_local_ext_features_1_sync), + /* HCI_OP_WRITE_AUTH_ENABLE */ + HCI_INIT(hci_write_auth_enable_sync), + {} +}; + +/* Read LE Buffer Size */ +static int hci_le_read_buffer_size_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_BUFFER_SIZE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read LE Local Supported Features */ +static int hci_le_read_local_features_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_LOCAL_FEATURES, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read LE Supported States */ +static int hci_le_read_supported_states_sync(struct hci_dev *hdev) +{ + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_SUPPORTED_STATES, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* LE Controller init stage 2 command sequence */ +static const struct hci_init_stage le_init2[] = { + /* HCI_OP_LE_READ_BUFFER_SIZE */ + HCI_INIT(hci_le_read_buffer_size_sync), + /* HCI_OP_LE_READ_LOCAL_FEATURES */ + HCI_INIT(hci_le_read_local_features_sync), + /* HCI_OP_LE_READ_SUPPORTED_STATES */ + HCI_INIT(hci_le_read_supported_states_sync), + {} +}; + +static int hci_init2_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + if (hdev->dev_type == HCI_AMP) + return hci_init_stage_sync(hdev, amp_init2); + + if (lmp_bredr_capable(hdev)) { + err = hci_init_stage_sync(hdev, br_init2); + if (err) + return err; + } else { + hci_dev_clear_flag(hdev, HCI_BREDR_ENABLED); + } + + if (lmp_le_capable(hdev)) { + err = hci_init_stage_sync(hdev, le_init2); + if (err) + return err; + /* LE-only controllers have LE implicitly enabled */ + if (!lmp_bredr_capable(hdev)) + hci_dev_set_flag(hdev, HCI_LE_ENABLED); + } + + return hci_init_stage_sync(hdev, hci_init2); +} + +static int hci_set_event_mask_sync(struct hci_dev *hdev) +{ + /* The second byte is 0xff instead of 0x9f (two reserved bits + * disabled) since a Broadcom 1.2 dongle doesn't respond to the + * command otherwise. + */ + u8 events[8] = { 0xff, 0xff, 0xfb, 0xff, 0x00, 0x00, 0x00, 0x00 }; + + /* CSR 1.1 dongles does not accept any bitfield so don't try to set + * any event mask for pre 1.2 devices. + */ + if (hdev->hci_ver < BLUETOOTH_VER_1_2) + return 0; + + if (lmp_bredr_capable(hdev)) { + events[4] |= 0x01; /* Flow Specification Complete */ + + /* Don't set Disconnect Complete when suspended as that + * would wakeup the host when disconnecting due to + * suspend. + */ + if (hdev->suspended) + events[0] &= 0xef; + } else { + /* Use a different default for LE-only devices */ + memset(events, 0, sizeof(events)); + events[1] |= 0x20; /* Command Complete */ + events[1] |= 0x40; /* Command Status */ + events[1] |= 0x80; /* Hardware Error */ + + /* If the controller supports the Disconnect command, enable + * the corresponding event. In addition enable packet flow + * control related events. + */ + if (hdev->commands[0] & 0x20) { + /* Don't set Disconnect Complete when suspended as that + * would wakeup the host when disconnecting due to + * suspend. + */ + if (!hdev->suspended) + events[0] |= 0x10; /* Disconnection Complete */ + events[2] |= 0x04; /* Number of Completed Packets */ + events[3] |= 0x02; /* Data Buffer Overflow */ + } + + /* If the controller supports the Read Remote Version + * Information command, enable the corresponding event. + */ + if (hdev->commands[2] & 0x80) + events[1] |= 0x08; /* Read Remote Version Information + * Complete + */ + + if (hdev->le_features[0] & HCI_LE_ENCRYPTION) { + events[0] |= 0x80; /* Encryption Change */ + events[5] |= 0x80; /* Encryption Key Refresh Complete */ + } + } + + if (lmp_inq_rssi_capable(hdev) || + test_bit(HCI_QUIRK_FIXUP_INQUIRY_MODE, &hdev->quirks)) + events[4] |= 0x02; /* Inquiry Result with RSSI */ + + if (lmp_ext_feat_capable(hdev)) + events[4] |= 0x04; /* Read Remote Extended Features Complete */ + + if (lmp_esco_capable(hdev)) { + events[5] |= 0x08; /* Synchronous Connection Complete */ + events[5] |= 0x10; /* Synchronous Connection Changed */ + } + + if (lmp_sniffsubr_capable(hdev)) + events[5] |= 0x20; /* Sniff Subrating */ + + if (lmp_pause_enc_capable(hdev)) + events[5] |= 0x80; /* Encryption Key Refresh Complete */ + + if (lmp_ext_inq_capable(hdev)) + events[5] |= 0x40; /* Extended Inquiry Result */ + + if (lmp_no_flush_capable(hdev)) + events[7] |= 0x01; /* Enhanced Flush Complete */ + + if (lmp_lsto_capable(hdev)) + events[6] |= 0x80; /* Link Supervision Timeout Changed */ + + if (lmp_ssp_capable(hdev)) { + events[6] |= 0x01; /* IO Capability Request */ + events[6] |= 0x02; /* IO Capability Response */ + events[6] |= 0x04; /* User Confirmation Request */ + events[6] |= 0x08; /* User Passkey Request */ + events[6] |= 0x10; /* Remote OOB Data Request */ + events[6] |= 0x20; /* Simple Pairing Complete */ + events[7] |= 0x04; /* User Passkey Notification */ + events[7] |= 0x08; /* Keypress Notification */ + events[7] |= 0x10; /* Remote Host Supported + * Features Notification + */ + } + + if (lmp_le_capable(hdev)) + events[7] |= 0x20; /* LE Meta-Event */ + + return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_MASK, + sizeof(events), events, HCI_CMD_TIMEOUT); +} + +static int hci_read_stored_link_key_sync(struct hci_dev *hdev) +{ + struct hci_cp_read_stored_link_key cp; + + if (!(hdev->commands[6] & 0x20) || + test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks)) + return 0; + + memset(&cp, 0, sizeof(cp)); + bacpy(&cp.bdaddr, BDADDR_ANY); + cp.read_all = 0x01; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_STORED_LINK_KEY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_setup_link_policy_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_def_link_policy cp; + u16 link_policy = 0; + + if (!(hdev->commands[5] & 0x10)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + if (lmp_rswitch_capable(hdev)) + link_policy |= HCI_LP_RSWITCH; + if (lmp_hold_capable(hdev)) + link_policy |= HCI_LP_HOLD; + if (lmp_sniff_capable(hdev)) + link_policy |= HCI_LP_SNIFF; + if (lmp_park_capable(hdev)) + link_policy |= HCI_LP_PARK; + + cp.policy = cpu_to_le16(link_policy); + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_DEF_LINK_POLICY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_read_page_scan_activity_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[8] & 0x01)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_PAGE_SCAN_ACTIVITY, + 0, NULL, HCI_CMD_TIMEOUT); +} + +static int hci_read_def_err_data_reporting_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[18] & 0x04) || + test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_DEF_ERR_DATA_REPORTING, + 0, NULL, HCI_CMD_TIMEOUT); +} + +static int hci_read_page_scan_type_sync(struct hci_dev *hdev) +{ + /* Some older Broadcom based Bluetooth 1.2 controllers do not + * support the Read Page Scan Type command. Check support for + * this command in the bit mask of supported commands. + */ + if (!(hdev->commands[13] & 0x01)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_PAGE_SCAN_TYPE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read features beyond page 1 if available */ +static int hci_read_local_ext_features_all_sync(struct hci_dev *hdev) +{ + u8 page; + int err; + + if (!lmp_ext_feat_capable(hdev)) + return 0; + + for (page = 2; page < HCI_MAX_PAGES && page <= hdev->max_page; + page++) { + err = hci_read_local_ext_features_sync(hdev, page); + if (err) + return err; + } + + return 0; +} + +/* HCI Controller init stage 3 command sequence */ +static const struct hci_init_stage hci_init3[] = { + /* HCI_OP_SET_EVENT_MASK */ + HCI_INIT(hci_set_event_mask_sync), + /* HCI_OP_READ_STORED_LINK_KEY */ + HCI_INIT(hci_read_stored_link_key_sync), + /* HCI_OP_WRITE_DEF_LINK_POLICY */ + HCI_INIT(hci_setup_link_policy_sync), + /* HCI_OP_READ_PAGE_SCAN_ACTIVITY */ + HCI_INIT(hci_read_page_scan_activity_sync), + /* HCI_OP_READ_DEF_ERR_DATA_REPORTING */ + HCI_INIT(hci_read_def_err_data_reporting_sync), + /* HCI_OP_READ_PAGE_SCAN_TYPE */ + HCI_INIT(hci_read_page_scan_type_sync), + /* HCI_OP_READ_LOCAL_EXT_FEATURES */ + HCI_INIT(hci_read_local_ext_features_all_sync), + {} +}; + +static int hci_le_set_event_mask_sync(struct hci_dev *hdev) +{ + u8 events[8]; + + if (!lmp_le_capable(hdev)) + return 0; + + memset(events, 0, sizeof(events)); + + if (hdev->le_features[0] & HCI_LE_ENCRYPTION) + events[0] |= 0x10; /* LE Long Term Key Request */ + + /* If controller supports the Connection Parameters Request + * Link Layer Procedure, enable the corresponding event. + */ + if (hdev->le_features[0] & HCI_LE_CONN_PARAM_REQ_PROC) + /* LE Remote Connection Parameter Request */ + events[0] |= 0x20; + + /* If the controller supports the Data Length Extension + * feature, enable the corresponding event. + */ + if (hdev->le_features[0] & HCI_LE_DATA_LEN_EXT) + events[0] |= 0x40; /* LE Data Length Change */ + + /* If the controller supports LL Privacy feature, enable + * the corresponding event. + */ + if (hdev->le_features[0] & HCI_LE_LL_PRIVACY) + events[1] |= 0x02; /* LE Enhanced Connection Complete */ + + /* If the controller supports Extended Scanner Filter + * Policies, enable the corresponding event. + */ + if (hdev->le_features[0] & HCI_LE_EXT_SCAN_POLICY) + events[1] |= 0x04; /* LE Direct Advertising Report */ + + /* If the controller supports Channel Selection Algorithm #2 + * feature, enable the corresponding event. + */ + if (hdev->le_features[1] & HCI_LE_CHAN_SEL_ALG2) + events[2] |= 0x08; /* LE Channel Selection Algorithm */ + + /* If the controller supports the LE Set Scan Enable command, + * enable the corresponding advertising report event. + */ + if (hdev->commands[26] & 0x08) + events[0] |= 0x02; /* LE Advertising Report */ + + /* If the controller supports the LE Create Connection + * command, enable the corresponding event. + */ + if (hdev->commands[26] & 0x10) + events[0] |= 0x01; /* LE Connection Complete */ + + /* If the controller supports the LE Connection Update + * command, enable the corresponding event. + */ + if (hdev->commands[27] & 0x04) + events[0] |= 0x04; /* LE Connection Update Complete */ + + /* If the controller supports the LE Read Remote Used Features + * command, enable the corresponding event. + */ + if (hdev->commands[27] & 0x20) + /* LE Read Remote Used Features Complete */ + events[0] |= 0x08; + + /* If the controller supports the LE Read Local P-256 + * Public Key command, enable the corresponding event. + */ + if (hdev->commands[34] & 0x02) + /* LE Read Local P-256 Public Key Complete */ + events[0] |= 0x80; + + /* If the controller supports the LE Generate DHKey + * command, enable the corresponding event. + */ + if (hdev->commands[34] & 0x04) + events[1] |= 0x01; /* LE Generate DHKey Complete */ + + /* If the controller supports the LE Set Default PHY or + * LE Set PHY commands, enable the corresponding event. + */ + if (hdev->commands[35] & (0x20 | 0x40)) + events[1] |= 0x08; /* LE PHY Update Complete */ + + /* If the controller supports LE Set Extended Scan Parameters + * and LE Set Extended Scan Enable commands, enable the + * corresponding event. + */ + if (use_ext_scan(hdev)) + events[1] |= 0x10; /* LE Extended Advertising Report */ + + /* If the controller supports the LE Extended Advertising + * command, enable the corresponding event. + */ + if (ext_adv_capable(hdev)) + events[2] |= 0x02; /* LE Advertising Set Terminated */ + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_EVENT_MASK, + sizeof(events), events, HCI_CMD_TIMEOUT); +} + +/* Read LE Advertising Channel TX Power */ +static int hci_le_read_adv_tx_power_sync(struct hci_dev *hdev) +{ + if ((hdev->commands[25] & 0x40) && !ext_adv_capable(hdev)) { + /* HCI TS spec forbids mixing of legacy and extended + * advertising commands wherein READ_ADV_TX_POWER is + * also included. So do not call it if extended adv + * is supported otherwise controller will return + * COMMAND_DISALLOWED for extended commands. + */ + return __hci_cmd_sync_status(hdev, + HCI_OP_LE_READ_ADV_TX_POWER, + 0, NULL, HCI_CMD_TIMEOUT); + } + + return 0; +} + +/* Read LE Min/Max Tx Power*/ +static int hci_le_read_tx_power_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[38] & 0x80)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_TRANSMIT_POWER, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Read LE Accept List Size */ +static int hci_le_read_accept_list_size_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[26] & 0x40)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_ACCEPT_LIST_SIZE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Clear LE Accept List */ +static int hci_le_clear_accept_list_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[26] & 0x80)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_CLEAR_ACCEPT_LIST, 0, NULL, + HCI_CMD_TIMEOUT); +} + +/* Read LE Resolving List Size */ +static int hci_le_read_resolv_list_size_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[34] & 0x40)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_RESOLV_LIST_SIZE, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Clear LE Resolving List */ +static int hci_le_clear_resolv_list_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[34] & 0x20)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_CLEAR_RESOLV_LIST, 0, NULL, + HCI_CMD_TIMEOUT); +} + +/* Set RPA timeout */ +static int hci_le_set_rpa_timeout_sync(struct hci_dev *hdev) +{ + __le16 timeout = cpu_to_le16(hdev->rpa_timeout); + + if (!(hdev->commands[35] & 0x04)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_RPA_TIMEOUT, + sizeof(timeout), &timeout, + HCI_CMD_TIMEOUT); +} + +/* Read LE Maximum Data Length */ +static int hci_le_read_max_data_len_sync(struct hci_dev *hdev) +{ + if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_MAX_DATA_LEN, 0, NULL, + HCI_CMD_TIMEOUT); +} + +/* Read LE Suggested Default Data Length */ +static int hci_le_read_def_data_len_sync(struct hci_dev *hdev) +{ + if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_READ_DEF_DATA_LEN, 0, NULL, + HCI_CMD_TIMEOUT); +} + +/* Read LE Number of Supported Advertising Sets */ +static int hci_le_read_num_support_adv_sets_sync(struct hci_dev *hdev) +{ + if (!ext_adv_capable(hdev)) + return 0; + + return __hci_cmd_sync_status(hdev, + HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Write LE Host Supported */ +static int hci_set_le_support_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_le_host_supported cp; + + /* LE-only devices do not support explicit enablement */ + if (!lmp_bredr_capable(hdev)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { + cp.le = 0x01; + cp.simul = 0x00; + } + + if (cp.le == lmp_host_le_capable(hdev)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_LE_HOST_SUPPORTED, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +/* LE Controller init stage 3 command sequence */ +static const struct hci_init_stage le_init3[] = { + /* HCI_OP_LE_SET_EVENT_MASK */ + HCI_INIT(hci_le_set_event_mask_sync), + /* HCI_OP_LE_READ_ADV_TX_POWER */ + HCI_INIT(hci_le_read_adv_tx_power_sync), + /* HCI_OP_LE_READ_TRANSMIT_POWER */ + HCI_INIT(hci_le_read_tx_power_sync), + /* HCI_OP_LE_READ_ACCEPT_LIST_SIZE */ + HCI_INIT(hci_le_read_accept_list_size_sync), + /* HCI_OP_LE_CLEAR_ACCEPT_LIST */ + HCI_INIT(hci_le_clear_accept_list_sync), + /* HCI_OP_LE_READ_RESOLV_LIST_SIZE */ + HCI_INIT(hci_le_read_resolv_list_size_sync), + /* HCI_OP_LE_CLEAR_RESOLV_LIST */ + HCI_INIT(hci_le_clear_resolv_list_sync), + /* HCI_OP_LE_SET_RPA_TIMEOUT */ + HCI_INIT(hci_le_set_rpa_timeout_sync), + /* HCI_OP_LE_READ_MAX_DATA_LEN */ + HCI_INIT(hci_le_read_max_data_len_sync), + /* HCI_OP_LE_READ_DEF_DATA_LEN */ + HCI_INIT(hci_le_read_def_data_len_sync), + /* HCI_OP_LE_READ_NUM_SUPPORTED_ADV_SETS */ + HCI_INIT(hci_le_read_num_support_adv_sets_sync), + /* HCI_OP_WRITE_LE_HOST_SUPPORTED */ + HCI_INIT(hci_set_le_support_sync), + {} +}; + +static int hci_init3_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + err = hci_init_stage_sync(hdev, hci_init3); + if (err) + return err; + + if (lmp_le_capable(hdev)) + return hci_init_stage_sync(hdev, le_init3); + + return 0; +} + +static int hci_delete_stored_link_key_sync(struct hci_dev *hdev) +{ + struct hci_cp_delete_stored_link_key cp; + + /* Some Broadcom based Bluetooth controllers do not support the + * Delete Stored Link Key command. They are clearly indicating its + * absence in the bit mask of supported commands. + * + * Check the supported commands and only if the command is marked + * as supported send it. If not supported assume that the controller + * does not have actual support for stored link keys which makes this + * command redundant anyway. + * + * Some controllers indicate that they support handling deleting + * stored link keys, but they don't. The quirk lets a driver + * just disable this command. + */ + if (!(hdev->commands[6] & 0x80) || + test_bit(HCI_QUIRK_BROKEN_STORED_LINK_KEY, &hdev->quirks)) + return 0; + + memset(&cp, 0, sizeof(cp)); + bacpy(&cp.bdaddr, BDADDR_ANY); + cp.delete_all = 0x01; + + return __hci_cmd_sync_status(hdev, HCI_OP_DELETE_STORED_LINK_KEY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_set_event_mask_page_2_sync(struct hci_dev *hdev) +{ + u8 events[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }; + bool changed = false; + + /* Set event mask page 2 if the HCI command for it is supported */ + if (!(hdev->commands[22] & 0x04)) + return 0; + + /* If Connectionless Peripheral Broadcast central role is supported + * enable all necessary events for it. + */ + if (lmp_cpb_central_capable(hdev)) { + events[1] |= 0x40; /* Triggered Clock Capture */ + events[1] |= 0x80; /* Synchronization Train Complete */ + events[2] |= 0x10; /* Peripheral Page Response Timeout */ + events[2] |= 0x20; /* CPB Channel Map Change */ + changed = true; + } + + /* If Connectionless Peripheral Broadcast peripheral role is supported + * enable all necessary events for it. + */ + if (lmp_cpb_peripheral_capable(hdev)) { + events[2] |= 0x01; /* Synchronization Train Received */ + events[2] |= 0x02; /* CPB Receive */ + events[2] |= 0x04; /* CPB Timeout */ + events[2] |= 0x08; /* Truncated Page Complete */ + changed = true; + } + + /* Enable Authenticated Payload Timeout Expired event if supported */ + if (lmp_ping_capable(hdev) || hdev->le_features[0] & HCI_LE_PING) { + events[2] |= 0x80; + changed = true; + } + + /* Some Broadcom based controllers indicate support for Set Event + * Mask Page 2 command, but then actually do not support it. Since + * the default value is all bits set to zero, the command is only + * required if the event mask has to be changed. In case no change + * to the event mask is needed, skip this command. + */ + if (!changed) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_SET_EVENT_MASK_PAGE_2, + sizeof(events), events, HCI_CMD_TIMEOUT); +} + +/* Read local codec list if the HCI command is supported */ +static int hci_read_local_codecs_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[29] & 0x20)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_CODECS, 0, NULL, + HCI_CMD_TIMEOUT); +} + +/* Read local pairing options if the HCI command is supported */ +static int hci_read_local_pairing_opts_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[41] & 0x08)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_LOCAL_PAIRING_OPTS, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Get MWS transport configuration if the HCI command is supported */ +static int hci_get_mws_transport_config_sync(struct hci_dev *hdev) +{ + if (!(hdev->commands[30] & 0x08)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_GET_MWS_TRANSPORT_CONFIG, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Check for Synchronization Train support */ +static int hci_read_sync_train_params_sync(struct hci_dev *hdev) +{ + if (!lmp_sync_train_capable(hdev)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_READ_SYNC_TRAIN_PARAMS, + 0, NULL, HCI_CMD_TIMEOUT); +} + +/* Enable Secure Connections if supported and configured */ +static int hci_write_sc_support_1_sync(struct hci_dev *hdev) +{ + u8 support = 0x01; + + if (!hci_dev_test_flag(hdev, HCI_SSP_ENABLED) || + !bredr_sc_enabled(hdev)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_SC_SUPPORT, + sizeof(support), &support, + HCI_CMD_TIMEOUT); +} + +/* Set erroneous data reporting if supported to the wideband speech + * setting value + */ +static int hci_set_err_data_report_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_def_err_data_reporting cp; + bool enabled = hci_dev_test_flag(hdev, HCI_WIDEBAND_SPEECH_ENABLED); + + if (!(hdev->commands[18] & 0x08) || + test_bit(HCI_QUIRK_BROKEN_ERR_DATA_REPORTING, &hdev->quirks)) + return 0; + + if (enabled == hdev->err_data_reporting) + return 0; + + memset(&cp, 0, sizeof(cp)); + cp.err_data_reporting = enabled ? ERR_DATA_REPORTING_ENABLED : + ERR_DATA_REPORTING_DISABLED; + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_DEF_ERR_DATA_REPORTING, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static const struct hci_init_stage hci_init4[] = { + /* HCI_OP_DELETE_STORED_LINK_KEY */ + HCI_INIT(hci_delete_stored_link_key_sync), + /* HCI_OP_SET_EVENT_MASK_PAGE_2 */ + HCI_INIT(hci_set_event_mask_page_2_sync), + /* HCI_OP_READ_LOCAL_CODECS */ + HCI_INIT(hci_read_local_codecs_sync), + /* HCI_OP_READ_LOCAL_PAIRING_OPTS */ + HCI_INIT(hci_read_local_pairing_opts_sync), + /* HCI_OP_GET_MWS_TRANSPORT_CONFIG */ + HCI_INIT(hci_get_mws_transport_config_sync), + /* HCI_OP_READ_SYNC_TRAIN_PARAMS */ + HCI_INIT(hci_read_sync_train_params_sync), + /* HCI_OP_WRITE_SC_SUPPORT */ + HCI_INIT(hci_write_sc_support_1_sync), + /* HCI_OP_WRITE_DEF_ERR_DATA_REPORTING */ + HCI_INIT(hci_set_err_data_report_sync), + {} +}; + +/* Set Suggested Default Data Length to maximum if supported */ +static int hci_le_set_write_def_data_len_sync(struct hci_dev *hdev) +{ + struct hci_cp_le_write_def_data_len cp; + + if (!(hdev->le_features[0] & HCI_LE_DATA_LEN_EXT)) + return 0; + + memset(&cp, 0, sizeof(cp)); + cp.tx_len = cpu_to_le16(hdev->le_max_tx_len); + cp.tx_time = cpu_to_le16(hdev->le_max_tx_time); + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_WRITE_DEF_DATA_LEN, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +/* Set Default PHY parameters if command is supported */ +static int hci_le_set_default_phy_sync(struct hci_dev *hdev) +{ + struct hci_cp_le_set_default_phy cp; + + if (!(hdev->commands[35] & 0x20)) + return 0; + + memset(&cp, 0, sizeof(cp)); + cp.all_phys = 0x00; + cp.tx_phys = hdev->le_tx_def_phys; + cp.rx_phys = hdev->le_rx_def_phys; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_SET_DEFAULT_PHY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static const struct hci_init_stage le_init4[] = { + /* HCI_OP_LE_WRITE_DEF_DATA_LEN */ + HCI_INIT(hci_le_set_write_def_data_len_sync), + /* HCI_OP_LE_SET_DEFAULT_PHY */ + HCI_INIT(hci_le_set_default_phy_sync), + {} +}; + +static int hci_init4_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + err = hci_init_stage_sync(hdev, hci_init4); + if (err) + return err; + + if (lmp_le_capable(hdev)) + return hci_init_stage_sync(hdev, le_init4); + + return 0; +} + +static int hci_init_sync(struct hci_dev *hdev) +{ + int err; + + err = hci_init1_sync(hdev); + if (err < 0) + return err; + + if (hci_dev_test_flag(hdev, HCI_SETUP)) + hci_debugfs_create_basic(hdev); + + err = hci_init2_sync(hdev); + if (err < 0) + return err; + + /* HCI_PRIMARY covers both single-mode LE, BR/EDR and dual-mode + * BR/EDR/LE type controllers. AMP controllers only need the + * first two stages of init. + */ + if (hdev->dev_type != HCI_PRIMARY) + return 0; + + err = hci_init3_sync(hdev); + if (err < 0) + return err; + + err = hci_init4_sync(hdev); + if (err < 0) + return err; + + /* This function is only called when the controller is actually in + * configured state. When the controller is marked as unconfigured, + * this initialization procedure is not run. + * + * It means that it is possible that a controller runs through its + * setup phase and then discovers missing settings. If that is the + * case, then this function will not be called. It then will only + * be called during the config phase. + * + * So only when in setup phase or config phase, create the debugfs + * entries and register the SMP channels. + */ + if (!hci_dev_test_flag(hdev, HCI_SETUP) && + !hci_dev_test_flag(hdev, HCI_CONFIG)) + return 0; + + hci_debugfs_create_common(hdev); + + if (lmp_bredr_capable(hdev)) + hci_debugfs_create_bredr(hdev); + + if (lmp_le_capable(hdev)) + hci_debugfs_create_le(hdev); + + return 0; +} + +int hci_dev_open_sync(struct hci_dev *hdev) +{ + int ret = 0; + + bt_dev_dbg(hdev, ""); + + if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) { + ret = -ENODEV; + goto done; + } + + if (!hci_dev_test_flag(hdev, HCI_SETUP) && + !hci_dev_test_flag(hdev, HCI_CONFIG)) { + /* Check for rfkill but allow the HCI setup stage to + * proceed (which in itself doesn't cause any RF activity). + */ + if (hci_dev_test_flag(hdev, HCI_RFKILLED)) { + ret = -ERFKILL; + goto done; + } + + /* Check for valid public address or a configured static + * random address, but let the HCI setup proceed to + * be able to determine if there is a public address + * or not. + * + * In case of user channel usage, it is not important + * if a public address or static random address is + * available. + * + * This check is only valid for BR/EDR controllers + * since AMP controllers do not have an address. + */ + if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + hdev->dev_type == HCI_PRIMARY && + !bacmp(&hdev->bdaddr, BDADDR_ANY) && + !bacmp(&hdev->static_addr, BDADDR_ANY)) { + ret = -EADDRNOTAVAIL; + goto done; + } + } + + if (test_bit(HCI_UP, &hdev->flags)) { + ret = -EALREADY; + goto done; + } + + if (hdev->open(hdev)) { + ret = -EIO; + goto done; + } + + set_bit(HCI_RUNNING, &hdev->flags); + hci_sock_dev_event(hdev, HCI_DEV_OPEN); + + atomic_set(&hdev->cmd_cnt, 1); + set_bit(HCI_INIT, &hdev->flags); + + if (hci_dev_test_flag(hdev, HCI_SETUP) || + test_bit(HCI_QUIRK_NON_PERSISTENT_SETUP, &hdev->quirks)) { + bool invalid_bdaddr; + + hci_sock_dev_event(hdev, HCI_DEV_SETUP); + + if (hdev->setup) + ret = hdev->setup(hdev); + + /* The transport driver can set the quirk to mark the + * BD_ADDR invalid before creating the HCI device or in + * its setup callback. + */ + invalid_bdaddr = test_bit(HCI_QUIRK_INVALID_BDADDR, + &hdev->quirks); + + if (ret) + goto setup_failed; + + if (test_bit(HCI_QUIRK_USE_BDADDR_PROPERTY, &hdev->quirks)) { + if (!bacmp(&hdev->public_addr, BDADDR_ANY)) + hci_dev_get_bd_addr_from_property(hdev); + + if (bacmp(&hdev->public_addr, BDADDR_ANY) && + hdev->set_bdaddr) { + ret = hdev->set_bdaddr(hdev, + &hdev->public_addr); + + /* If setting of the BD_ADDR from the device + * property succeeds, then treat the address + * as valid even if the invalid BD_ADDR + * quirk indicates otherwise. + */ + if (!ret) + invalid_bdaddr = false; + } + } + +setup_failed: + /* The transport driver can set these quirks before + * creating the HCI device or in its setup callback. + * + * For the invalid BD_ADDR quirk it is possible that + * it becomes a valid address if the bootloader does + * provide it (see above). + * + * In case any of them is set, the controller has to + * start up as unconfigured. + */ + if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) || + invalid_bdaddr) + hci_dev_set_flag(hdev, HCI_UNCONFIGURED); + + /* For an unconfigured controller it is required to + * read at least the version information provided by + * the Read Local Version Information command. + * + * If the set_bdaddr driver callback is provided, then + * also the original Bluetooth public device address + * will be read using the Read BD Address command. + */ + if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) + ret = hci_unconf_init_sync(hdev); + } + + if (hci_dev_test_flag(hdev, HCI_CONFIG)) { + /* If public address change is configured, ensure that + * the address gets programmed. If the driver does not + * support changing the public address, fail the power + * on procedure. + */ + if (bacmp(&hdev->public_addr, BDADDR_ANY) && + hdev->set_bdaddr) + ret = hdev->set_bdaddr(hdev, &hdev->public_addr); + else + ret = -EADDRNOTAVAIL; + } + + if (!ret) { + if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED) && + !hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) { + ret = hci_init_sync(hdev); + if (!ret && hdev->post_init) + ret = hdev->post_init(hdev); + } + } + + /* If the HCI Reset command is clearing all diagnostic settings, + * then they need to be reprogrammed after the init procedure + * completed. + */ + if (test_bit(HCI_QUIRK_NON_PERSISTENT_DIAG, &hdev->quirks) && + !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + hci_dev_test_flag(hdev, HCI_VENDOR_DIAG) && hdev->set_diag) + ret = hdev->set_diag(hdev, true); + + if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) { + msft_do_open(hdev); + aosp_do_open(hdev); + } + + clear_bit(HCI_INIT, &hdev->flags); + + if (!ret) { + hci_dev_hold(hdev); + hci_dev_set_flag(hdev, HCI_RPA_EXPIRED); + hci_adv_instances_set_rpa_expired(hdev, true); + set_bit(HCI_UP, &hdev->flags); + hci_sock_dev_event(hdev, HCI_DEV_UP); + hci_leds_update_powered(hdev, true); + if (!hci_dev_test_flag(hdev, HCI_SETUP) && + !hci_dev_test_flag(hdev, HCI_CONFIG) && + !hci_dev_test_flag(hdev, HCI_UNCONFIGURED) && + !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + hci_dev_test_flag(hdev, HCI_MGMT) && + hdev->dev_type == HCI_PRIMARY) { + ret = hci_powered_update_sync(hdev); + } + } else { + /* Init failed, cleanup */ + flush_work(&hdev->tx_work); + + /* Since hci_rx_work() is possible to awake new cmd_work + * it should be flushed first to avoid unexpected call of + * hci_cmd_work() + */ + flush_work(&hdev->rx_work); + flush_work(&hdev->cmd_work); + + skb_queue_purge(&hdev->cmd_q); + skb_queue_purge(&hdev->rx_q); + + if (hdev->flush) + hdev->flush(hdev); + + if (hdev->sent_cmd) { + kfree_skb(hdev->sent_cmd); + hdev->sent_cmd = NULL; + } + + clear_bit(HCI_RUNNING, &hdev->flags); + hci_sock_dev_event(hdev, HCI_DEV_CLOSE); + + hdev->close(hdev); + hdev->flags &= BIT(HCI_RAW); + } + +done: + return ret; +} + +/* This function requires the caller holds hdev->lock */ +static void hci_pend_le_actions_clear(struct hci_dev *hdev) +{ + struct hci_conn_params *p; + + list_for_each_entry(p, &hdev->le_conn_params, list) { + if (p->conn) { + hci_conn_drop(p->conn); + hci_conn_put(p->conn); + p->conn = NULL; + } + list_del_init(&p->action); + } + + BT_DBG("All LE pending actions cleared"); +} + +int hci_dev_close_sync(struct hci_dev *hdev) +{ + bool auto_off; + int err = 0; + + bt_dev_dbg(hdev, ""); + + cancel_delayed_work(&hdev->power_off); + cancel_delayed_work(&hdev->ncmd_timer); + + hci_request_cancel_all(hdev); + + if (!hci_dev_test_flag(hdev, HCI_UNREGISTER) && + !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + test_bit(HCI_UP, &hdev->flags)) { + /* Execute vendor specific shutdown routine */ + if (hdev->shutdown) + err = hdev->shutdown(hdev); + } + + if (!test_and_clear_bit(HCI_UP, &hdev->flags)) { + cancel_delayed_work_sync(&hdev->cmd_timer); + return err; + } + + hci_leds_update_powered(hdev, false); + + /* Flush RX and TX works */ + flush_work(&hdev->tx_work); + flush_work(&hdev->rx_work); + + if (hdev->discov_timeout > 0) { + hdev->discov_timeout = 0; + hci_dev_clear_flag(hdev, HCI_DISCOVERABLE); + hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); + } + + if (hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) + cancel_delayed_work(&hdev->service_cache); + + if (hci_dev_test_flag(hdev, HCI_MGMT)) { + struct adv_info *adv_instance; + + cancel_delayed_work_sync(&hdev->rpa_expired); + + list_for_each_entry(adv_instance, &hdev->adv_instances, list) + cancel_delayed_work_sync(&adv_instance->rpa_expired_cb); + } + + /* Avoid potential lockdep warnings from the *_flush() calls by + * ensuring the workqueue is empty up front. + */ + drain_workqueue(hdev->workqueue); + + hci_dev_lock(hdev); + + hci_discovery_set_state(hdev, DISCOVERY_STOPPED); + + auto_off = hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF); + + if (!auto_off && hdev->dev_type == HCI_PRIMARY && + !hci_dev_test_flag(hdev, HCI_USER_CHANNEL) && + hci_dev_test_flag(hdev, HCI_MGMT)) + __mgmt_power_off(hdev); + + hci_inquiry_cache_flush(hdev); + hci_pend_le_actions_clear(hdev); + hci_conn_hash_flush(hdev); + hci_dev_unlock(hdev); + + smp_unregister(hdev); + + hci_sock_dev_event(hdev, HCI_DEV_DOWN); + + if (!hci_dev_test_flag(hdev, HCI_USER_CHANNEL)) { + aosp_do_close(hdev); + msft_do_close(hdev); + } + + if (hdev->flush) + hdev->flush(hdev); + + /* Reset device */ + skb_queue_purge(&hdev->cmd_q); + atomic_set(&hdev->cmd_cnt, 1); + if (test_bit(HCI_QUIRK_RESET_ON_CLOSE, &hdev->quirks) && + !auto_off && !hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) { + set_bit(HCI_INIT, &hdev->flags); + hci_reset_sync(hdev); + clear_bit(HCI_INIT, &hdev->flags); + } + + /* flush cmd work */ + flush_work(&hdev->cmd_work); + + /* Drop queues */ + skb_queue_purge(&hdev->rx_q); + skb_queue_purge(&hdev->cmd_q); + skb_queue_purge(&hdev->raw_q); + + /* Drop last sent command */ + if (hdev->sent_cmd) { + cancel_delayed_work_sync(&hdev->cmd_timer); + kfree_skb(hdev->sent_cmd); + hdev->sent_cmd = NULL; + } + + clear_bit(HCI_RUNNING, &hdev->flags); + hci_sock_dev_event(hdev, HCI_DEV_CLOSE); + + /* After this point our queues are empty and no tasks are scheduled. */ + hdev->close(hdev); + + /* Clear flags */ + hdev->flags &= BIT(HCI_RAW); + hci_dev_clear_volatile_flags(hdev); + + /* Controller radio is available but is currently powered down */ + hdev->amp_status = AMP_STATUS_POWERED_DOWN; + + memset(hdev->eir, 0, sizeof(hdev->eir)); + memset(hdev->dev_class, 0, sizeof(hdev->dev_class)); + bacpy(&hdev->random_addr, BDADDR_ANY); + + hci_dev_put(hdev); + return err; +} + +/* This function perform power on HCI command sequence as follows: + * + * If controller is already up (HCI_UP) performs hci_powered_update_sync + * sequence otherwise run hci_dev_open_sync which will follow with + * hci_powered_update_sync after the init sequence is completed. + */ +static int hci_power_on_sync(struct hci_dev *hdev) +{ + int err; + + if (test_bit(HCI_UP, &hdev->flags) && + hci_dev_test_flag(hdev, HCI_MGMT) && + hci_dev_test_and_clear_flag(hdev, HCI_AUTO_OFF)) { + cancel_delayed_work(&hdev->power_off); + return hci_powered_update_sync(hdev); + } + + err = hci_dev_open_sync(hdev); + if (err < 0) + return err; + + /* During the HCI setup phase, a few error conditions are + * ignored and they need to be checked now. If they are still + * valid, it is important to return the device back off. + */ + if (hci_dev_test_flag(hdev, HCI_RFKILLED) || + hci_dev_test_flag(hdev, HCI_UNCONFIGURED) || + (hdev->dev_type == HCI_PRIMARY && + !bacmp(&hdev->bdaddr, BDADDR_ANY) && + !bacmp(&hdev->static_addr, BDADDR_ANY))) { + hci_dev_clear_flag(hdev, HCI_AUTO_OFF); + hci_dev_close_sync(hdev); + } else if (hci_dev_test_flag(hdev, HCI_AUTO_OFF)) { + queue_delayed_work(hdev->req_workqueue, &hdev->power_off, + HCI_AUTO_OFF_TIMEOUT); + } + + if (hci_dev_test_and_clear_flag(hdev, HCI_SETUP)) { + /* For unconfigured devices, set the HCI_RAW flag + * so that userspace can easily identify them. + */ + if (hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) + set_bit(HCI_RAW, &hdev->flags); + + /* For fully configured devices, this will send + * the Index Added event. For unconfigured devices, + * it will send Unconfigued Index Added event. + * + * Devices with HCI_QUIRK_RAW_DEVICE are ignored + * and no event will be send. + */ + mgmt_index_added(hdev); + } else if (hci_dev_test_and_clear_flag(hdev, HCI_CONFIG)) { + /* When the controller is now configured, then it + * is important to clear the HCI_RAW flag. + */ + if (!hci_dev_test_flag(hdev, HCI_UNCONFIGURED)) + clear_bit(HCI_RAW, &hdev->flags); + + /* Powering on the controller with HCI_CONFIG set only + * happens with the transition from unconfigured to + * configured. This will send the Index Added event. + */ + mgmt_index_added(hdev); + } + + return 0; +} + +static int hci_remote_name_cancel_sync(struct hci_dev *hdev, bdaddr_t *addr) +{ + struct hci_cp_remote_name_req_cancel cp; + + memset(&cp, 0, sizeof(cp)); + bacpy(&cp.bdaddr, addr); + + return __hci_cmd_sync_status(hdev, HCI_OP_REMOTE_NAME_REQ_CANCEL, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +int hci_stop_discovery_sync(struct hci_dev *hdev) +{ + struct discovery_state *d = &hdev->discovery; + struct inquiry_entry *e; + int err; + + bt_dev_dbg(hdev, "state %u", hdev->discovery.state); + + if (d->state == DISCOVERY_FINDING || d->state == DISCOVERY_STOPPING) { + if (test_bit(HCI_INQUIRY, &hdev->flags)) { + err = __hci_cmd_sync_status(hdev, HCI_OP_INQUIRY_CANCEL, + 0, NULL, HCI_CMD_TIMEOUT); + if (err) + return err; + } + + if (hci_dev_test_flag(hdev, HCI_LE_SCAN)) { + cancel_delayed_work(&hdev->le_scan_disable); + cancel_delayed_work(&hdev->le_scan_restart); + + err = hci_scan_disable_sync(hdev); + if (err) + return err; + } + + } else { + err = hci_scan_disable_sync(hdev); + if (err) + return err; + } + + /* Resume advertising if it was paused */ + if (use_ll_privacy(hdev)) + hci_resume_advertising_sync(hdev); + + /* No further actions needed for LE-only discovery */ + if (d->type == DISCOV_TYPE_LE) + return 0; + + if (d->state == DISCOVERY_RESOLVING || d->state == DISCOVERY_STOPPING) { + e = hci_inquiry_cache_lookup_resolve(hdev, BDADDR_ANY, + NAME_PENDING); + if (!e) + return 0; + + return hci_remote_name_cancel_sync(hdev, &e->data.bdaddr); + } + + return 0; +} + +static int hci_disconnect_phy_link_sync(struct hci_dev *hdev, u16 handle, + u8 reason) +{ + struct hci_cp_disconn_phy_link cp; + + memset(&cp, 0, sizeof(cp)); + cp.phy_handle = HCI_PHY_HANDLE(handle); + cp.reason = reason; + + return __hci_cmd_sync_status(hdev, HCI_OP_DISCONN_PHY_LINK, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_disconnect_sync(struct hci_dev *hdev, struct hci_conn *conn, + u8 reason) +{ + struct hci_cp_disconnect cp; + + if (conn->type == AMP_LINK) + return hci_disconnect_phy_link_sync(hdev, conn->handle, reason); + + memset(&cp, 0, sizeof(cp)); + cp.handle = cpu_to_le16(conn->handle); + cp.reason = reason; + + /* Wait for HCI_EV_DISCONN_COMPLETE not HCI_EV_CMD_STATUS when not + * suspending. + */ + if (!hdev->suspended) + return __hci_cmd_sync_status_sk(hdev, HCI_OP_DISCONNECT, + sizeof(cp), &cp, + HCI_EV_DISCONN_COMPLETE, + HCI_CMD_TIMEOUT, NULL); + + return __hci_cmd_sync_status(hdev, HCI_OP_DISCONNECT, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); +} + +static int hci_le_connect_cancel_sync(struct hci_dev *hdev, + struct hci_conn *conn) +{ + if (test_bit(HCI_CONN_SCANNING, &conn->flags)) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_LE_CREATE_CONN_CANCEL, + 6, &conn->dst, HCI_CMD_TIMEOUT); +} + +static int hci_connect_cancel_sync(struct hci_dev *hdev, struct hci_conn *conn) +{ + if (conn->type == LE_LINK) + return hci_le_connect_cancel_sync(hdev, conn); + + if (hdev->hci_ver < BLUETOOTH_VER_1_2) + return 0; + + return __hci_cmd_sync_status(hdev, HCI_OP_CREATE_CONN_CANCEL, + 6, &conn->dst, HCI_CMD_TIMEOUT); +} + +static int hci_reject_sco_sync(struct hci_dev *hdev, struct hci_conn *conn, + u8 reason) +{ + struct hci_cp_reject_sync_conn_req cp; + + memset(&cp, 0, sizeof(cp)); + bacpy(&cp.bdaddr, &conn->dst); + cp.reason = reason; + + /* SCO rejection has its own limited set of + * allowed error values (0x0D-0x0F). + */ + if (reason < 0x0d || reason > 0x0f) + cp.reason = HCI_ERROR_REJ_LIMITED_RESOURCES; + + return __hci_cmd_sync_status(hdev, HCI_OP_REJECT_SYNC_CONN_REQ, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_reject_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, + u8 reason) +{ + struct hci_cp_reject_conn_req cp; + + if (conn->type == SCO_LINK || conn->type == ESCO_LINK) + return hci_reject_sco_sync(hdev, conn, reason); + + memset(&cp, 0, sizeof(cp)); + bacpy(&cp.bdaddr, &conn->dst); + cp.reason = reason; + + return __hci_cmd_sync_status(hdev, HCI_OP_REJECT_CONN_REQ, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_abort_conn_sync(struct hci_dev *hdev, struct hci_conn *conn, + u8 reason) +{ + switch (conn->state) { + case BT_CONNECTED: + case BT_CONFIG: + return hci_disconnect_sync(hdev, conn, reason); + case BT_CONNECT: + return hci_connect_cancel_sync(hdev, conn); + case BT_CONNECT2: + return hci_reject_conn_sync(hdev, conn, reason); + default: + conn->state = BT_CLOSED; + break; + } + + return 0; +} + +static int hci_disconnect_all_sync(struct hci_dev *hdev, u8 reason) +{ + struct hci_conn *conn, *tmp; + int err; + + list_for_each_entry_safe(conn, tmp, &hdev->conn_hash.list, list) { + err = hci_abort_conn_sync(hdev, conn, reason); + if (err) + return err; + } + + return err; +} + +/* This function perform power off HCI command sequence as follows: + * + * Clear Advertising + * Stop Discovery + * Disconnect all connections + * hci_dev_close_sync + */ +static int hci_power_off_sync(struct hci_dev *hdev) +{ + int err; + + /* If controller is already down there is nothing to do */ + if (!test_bit(HCI_UP, &hdev->flags)) + return 0; + + if (test_bit(HCI_ISCAN, &hdev->flags) || + test_bit(HCI_PSCAN, &hdev->flags)) { + err = hci_write_scan_enable_sync(hdev, 0x00); + if (err) + return err; + } + + err = hci_clear_adv_sync(hdev, NULL, false); + if (err) + return err; + + err = hci_stop_discovery_sync(hdev); + if (err) + return err; + + /* Terminated due to Power Off */ + err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF); + if (err) + return err; + + return hci_dev_close_sync(hdev); +} + +int hci_set_powered_sync(struct hci_dev *hdev, u8 val) +{ + if (val) + return hci_power_on_sync(hdev); + + return hci_power_off_sync(hdev); +} + +static int hci_write_iac_sync(struct hci_dev *hdev) +{ + struct hci_cp_write_current_iac_lap cp; + + if (!hci_dev_test_flag(hdev, HCI_DISCOVERABLE)) + return 0; + + memset(&cp, 0, sizeof(cp)); + + if (hci_dev_test_flag(hdev, HCI_LIMITED_DISCOVERABLE)) { + /* Limited discoverable mode */ + cp.num_iac = min_t(u8, hdev->num_iac, 2); + cp.iac_lap[0] = 0x00; /* LIAC */ + cp.iac_lap[1] = 0x8b; + cp.iac_lap[2] = 0x9e; + cp.iac_lap[3] = 0x33; /* GIAC */ + cp.iac_lap[4] = 0x8b; + cp.iac_lap[5] = 0x9e; + } else { + /* General discoverable mode */ + cp.num_iac = 1; + cp.iac_lap[0] = 0x33; /* GIAC */ + cp.iac_lap[1] = 0x8b; + cp.iac_lap[2] = 0x9e; + } + + return __hci_cmd_sync_status(hdev, HCI_OP_WRITE_CURRENT_IAC_LAP, + (cp.num_iac * 3) + 1, &cp, + HCI_CMD_TIMEOUT); +} + +int hci_update_discoverable_sync(struct hci_dev *hdev) +{ + int err = 0; + + if (hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) { + err = hci_write_iac_sync(hdev); + if (err) + return err; + + err = hci_update_scan_sync(hdev); + if (err) + return err; + + err = hci_update_class_sync(hdev); + if (err) + return err; + } + + /* Advertising instances don't use the global discoverable setting, so + * only update AD if advertising was enabled using Set Advertising. + */ + if (hci_dev_test_flag(hdev, HCI_ADVERTISING)) { + err = hci_update_adv_data_sync(hdev, 0x00); + if (err) + return err; + + /* Discoverable mode affects the local advertising + * address in limited privacy mode. + */ + if (hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) { + if (ext_adv_capable(hdev)) + err = hci_start_ext_adv_sync(hdev, 0x00); + else + err = hci_enable_advertising_sync(hdev); + } + } + + return err; +} + +static int update_discoverable_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_discoverable_sync(hdev); +} + +int hci_update_discoverable(struct hci_dev *hdev) +{ + /* Only queue if it would have any effect */ + if (hdev_is_powered(hdev) && + hci_dev_test_flag(hdev, HCI_ADVERTISING) && + hci_dev_test_flag(hdev, HCI_DISCOVERABLE) && + hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) + return hci_cmd_sync_queue(hdev, update_discoverable_sync, NULL, + NULL); + + return 0; +} + +int hci_update_connectable_sync(struct hci_dev *hdev) +{ + int err; + + err = hci_update_scan_sync(hdev); + if (err) + return err; + + /* If BR/EDR is not enabled and we disable advertising as a + * by-product of disabling connectable, we need to update the + * advertising flags. + */ + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + err = hci_update_adv_data_sync(hdev, hdev->cur_adv_instance); + + /* Update the advertising parameters if necessary */ + if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || + !list_empty(&hdev->adv_instances)) { + if (ext_adv_capable(hdev)) + err = hci_start_ext_adv_sync(hdev, + hdev->cur_adv_instance); + else + err = hci_enable_advertising_sync(hdev); + + if (err) + return err; + } + + return hci_update_passive_scan_sync(hdev); +} + +static int hci_inquiry_sync(struct hci_dev *hdev, u8 length) +{ + const u8 giac[3] = { 0x33, 0x8b, 0x9e }; + const u8 liac[3] = { 0x00, 0x8b, 0x9e }; + struct hci_cp_inquiry cp; + + bt_dev_dbg(hdev, ""); + + if (hci_dev_test_flag(hdev, HCI_INQUIRY)) + return 0; + + hci_dev_lock(hdev); + hci_inquiry_cache_flush(hdev); + hci_dev_unlock(hdev); + + memset(&cp, 0, sizeof(cp)); + + if (hdev->discovery.limited) + memcpy(&cp.lap, liac, sizeof(cp.lap)); + else + memcpy(&cp.lap, giac, sizeof(cp.lap)); + + cp.length = length; + + return __hci_cmd_sync_status(hdev, HCI_OP_INQUIRY, + sizeof(cp), &cp, HCI_CMD_TIMEOUT); +} + +static int hci_active_scan_sync(struct hci_dev *hdev, uint16_t interval) +{ + u8 own_addr_type; + /* Accept list is not used for discovery */ + u8 filter_policy = 0x00; + /* Default is to enable duplicates filter */ + u8 filter_dup = LE_SCAN_FILTER_DUP_ENABLE; + int err; + + bt_dev_dbg(hdev, ""); + + /* If controller is scanning, it means the passive scanning is + * running. Thus, we should temporarily stop it in order to set the + * discovery scanning parameters. + */ + err = hci_scan_disable_sync(hdev); + if (err) { + bt_dev_err(hdev, "Unable to disable scanning: %d", err); + return err; + } + + cancel_interleave_scan(hdev); + + /* Pause advertising since active scanning disables address resolution + * which advertising depend on in order to generate its RPAs. + */ + if (use_ll_privacy(hdev)) { + err = hci_pause_advertising_sync(hdev); + if (err) { + bt_dev_err(hdev, "pause advertising failed: %d", err); + goto failed; + } + } + + /* Disable address resolution while doing active scanning since the + * accept list shall not be used and all reports shall reach the host + * anyway. + */ + err = hci_le_set_addr_resolution_enable_sync(hdev, 0x00); + if (err) { + bt_dev_err(hdev, "Unable to disable Address Resolution: %d", + err); + goto failed; + } + + /* All active scans will be done with either a resolvable private + * address (when privacy feature has been enabled) or non-resolvable + * private address. + */ + err = hci_update_random_address_sync(hdev, true, scan_use_rpa(hdev), + &own_addr_type); + if (err < 0) + own_addr_type = ADDR_LE_DEV_PUBLIC; + + if (hci_is_adv_monitoring(hdev)) { + /* Duplicate filter should be disabled when some advertisement + * monitor is activated, otherwise AdvMon can only receive one + * advertisement for one peer(*) during active scanning, and + * might report loss to these peers. + * + * Note that different controllers have different meanings of + * |duplicate|. Some of them consider packets with the same + * address as duplicate, and others consider packets with the + * same address and the same RSSI as duplicate. Although in the + * latter case we don't need to disable duplicate filter, but + * it is common to have active scanning for a short period of + * time, the power impact should be neglectable. + */ + filter_dup = LE_SCAN_FILTER_DUP_DISABLE; + } + + err = hci_start_scan_sync(hdev, LE_SCAN_ACTIVE, interval, + hdev->le_scan_window_discovery, + own_addr_type, filter_policy, filter_dup); + if (!err) + return err; + +failed: + /* Resume advertising if it was paused */ + if (use_ll_privacy(hdev)) + hci_resume_advertising_sync(hdev); + + /* Resume passive scanning */ + hci_update_passive_scan_sync(hdev); + return err; +} + +static int hci_start_interleaved_discovery_sync(struct hci_dev *hdev) +{ + int err; + + bt_dev_dbg(hdev, ""); + + err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery * 2); + if (err) + return err; + + return hci_inquiry_sync(hdev, DISCOV_BREDR_INQUIRY_LEN); +} + +int hci_start_discovery_sync(struct hci_dev *hdev) +{ + unsigned long timeout; + int err; + + bt_dev_dbg(hdev, "type %u", hdev->discovery.type); + + switch (hdev->discovery.type) { + case DISCOV_TYPE_BREDR: + return hci_inquiry_sync(hdev, DISCOV_BREDR_INQUIRY_LEN); + case DISCOV_TYPE_INTERLEAVED: + /* When running simultaneous discovery, the LE scanning time + * should occupy the whole discovery time sine BR/EDR inquiry + * and LE scanning are scheduled by the controller. + * + * For interleaving discovery in comparison, BR/EDR inquiry + * and LE scanning are done sequentially with separate + * timeouts. + */ + if (test_bit(HCI_QUIRK_SIMULTANEOUS_DISCOVERY, + &hdev->quirks)) { + timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT); + /* During simultaneous discovery, we double LE scan + * interval. We must leave some time for the controller + * to do BR/EDR inquiry. + */ + err = hci_start_interleaved_discovery_sync(hdev); + break; + } + + timeout = msecs_to_jiffies(hdev->discov_interleaved_timeout); + err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery); + break; + case DISCOV_TYPE_LE: + timeout = msecs_to_jiffies(DISCOV_LE_TIMEOUT); + err = hci_active_scan_sync(hdev, hdev->le_scan_int_discovery); + break; + default: + return -EINVAL; + } + + if (err) + return err; + + bt_dev_dbg(hdev, "timeout %u ms", jiffies_to_msecs(timeout)); + + /* When service discovery is used and the controller has a + * strict duplicate filter, it is important to remember the + * start and duration of the scan. This is required for + * restarting scanning during the discovery phase. + */ + if (test_bit(HCI_QUIRK_STRICT_DUPLICATE_FILTER, &hdev->quirks) && + hdev->discovery.result_filtering) { + hdev->discovery.scan_start = jiffies; + hdev->discovery.scan_duration = timeout; + } + + queue_delayed_work(hdev->req_workqueue, &hdev->le_scan_disable, + timeout); + return 0; +} + +static void hci_suspend_monitor_sync(struct hci_dev *hdev) +{ + switch (hci_get_adv_monitor_offload_ext(hdev)) { + case HCI_ADV_MONITOR_EXT_MSFT: + msft_suspend_sync(hdev); + break; + default: + return; + } +} + +/* This function disables discovery and mark it as paused */ +static int hci_pause_discovery_sync(struct hci_dev *hdev) +{ + int old_state = hdev->discovery.state; + int err; + + /* If discovery already stopped/stopping/paused there nothing to do */ + if (old_state == DISCOVERY_STOPPED || old_state == DISCOVERY_STOPPING || + hdev->discovery_paused) + return 0; + + hci_discovery_set_state(hdev, DISCOVERY_STOPPING); + err = hci_stop_discovery_sync(hdev); + if (err) + return err; + + hdev->discovery_paused = true; + hdev->discovery_old_state = old_state; + hci_discovery_set_state(hdev, DISCOVERY_STOPPED); + + return 0; +} + +static int hci_update_event_filter_sync(struct hci_dev *hdev) +{ + struct bdaddr_list_with_flags *b; + u8 scan = SCAN_DISABLED; + bool scanning = test_bit(HCI_PSCAN, &hdev->flags); + int err; + + if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED)) + return 0; + + /* Always clear event filter when starting */ + hci_clear_event_filter_sync(hdev); + + list_for_each_entry(b, &hdev->accept_list, list) { + if (!hci_conn_test_flag(HCI_CONN_FLAG_REMOTE_WAKEUP, + b->current_flags)) + continue; + + bt_dev_dbg(hdev, "Adding event filters for %pMR", &b->bdaddr); + + err = hci_set_event_filter_sync(hdev, HCI_FLT_CONN_SETUP, + HCI_CONN_SETUP_ALLOW_BDADDR, + &b->bdaddr, + HCI_CONN_SETUP_AUTO_ON); + if (err) + bt_dev_dbg(hdev, "Failed to set event filter for %pMR", + &b->bdaddr); + else + scan = SCAN_PAGE; + } + + if (scan && !scanning) + hci_write_scan_enable_sync(hdev, scan); + else if (!scan && scanning) + hci_write_scan_enable_sync(hdev, scan); + + return 0; +} + +/* This function performs the HCI suspend procedures in the follow order: + * + * Pause discovery (active scanning/inquiry) + * Pause Directed Advertising/Advertising + * Disconnect all connections + * Set suspend_status to BT_SUSPEND_DISCONNECT if hdev cannot wakeup + * otherwise: + * Update event mask (only set events that are allowed to wake up the host) + * Update event filter (with devices marked with HCI_CONN_FLAG_REMOTE_WAKEUP) + * Update passive scanning (lower duty cycle) + * Set suspend_status to BT_SUSPEND_CONFIGURE_WAKE + */ +int hci_suspend_sync(struct hci_dev *hdev) +{ + int err; + + /* If marked as suspended there nothing to do */ + if (hdev->suspended) + return 0; + + /* Mark device as suspended */ + hdev->suspended = true; + + /* Pause discovery if not already stopped */ + hci_pause_discovery_sync(hdev); + + /* Pause other advertisements */ + hci_pause_advertising_sync(hdev); + + /* Disable page scan if enabled */ + if (test_bit(HCI_PSCAN, &hdev->flags)) + hci_write_scan_enable_sync(hdev, SCAN_DISABLED); + + /* Suspend monitor filters */ + hci_suspend_monitor_sync(hdev); + + /* Prevent disconnects from causing scanning to be re-enabled */ + hdev->scanning_paused = true; + + /* Soft disconnect everything (power off) */ + err = hci_disconnect_all_sync(hdev, HCI_ERROR_REMOTE_POWER_OFF); + if (err) { + /* Set state to BT_RUNNING so resume doesn't notify */ + hdev->suspend_state = BT_RUNNING; + hci_resume_sync(hdev); + return err; + } + + /* Only configure accept list if disconnect succeeded and wake + * isn't being prevented. + */ + if (!hdev->wakeup || !hdev->wakeup(hdev)) { + hdev->suspend_state = BT_SUSPEND_DISCONNECT; + return 0; + } + + /* Unpause to take care of updating scanning params */ + hdev->scanning_paused = false; + + /* Update event mask so only the allowed event can wakeup the host */ + hci_set_event_mask_sync(hdev); + + /* Enable event filter for paired devices */ + hci_update_event_filter_sync(hdev); + + /* Update LE passive scan if enabled */ + hci_update_passive_scan_sync(hdev); + + /* Pause scan changes again. */ + hdev->scanning_paused = true; + + hdev->suspend_state = BT_SUSPEND_CONFIGURE_WAKE; + + return 0; +} + +/* This function resumes discovery */ +static int hci_resume_discovery_sync(struct hci_dev *hdev) +{ + int err; + + /* If discovery not paused there nothing to do */ + if (!hdev->discovery_paused) + return 0; + + hdev->discovery_paused = false; + + hci_discovery_set_state(hdev, DISCOVERY_STARTING); + + err = hci_start_discovery_sync(hdev); + + hci_discovery_set_state(hdev, err ? DISCOVERY_STOPPED : + DISCOVERY_FINDING); + + return err; +} + +static void hci_resume_monitor_sync(struct hci_dev *hdev) +{ + switch (hci_get_adv_monitor_offload_ext(hdev)) { + case HCI_ADV_MONITOR_EXT_MSFT: + msft_resume_sync(hdev); + break; + default: + return; + } +} + +/* This function performs the HCI suspend procedures in the follow order: + * + * Restore event mask + * Clear event filter + * Update passive scanning (normal duty cycle) + * Resume Directed Advertising/Advertising + * Resume discovery (active scanning/inquiry) + */ +int hci_resume_sync(struct hci_dev *hdev) +{ + /* If not marked as suspended there nothing to do */ + if (!hdev->suspended) + return 0; + + hdev->suspended = false; + hdev->scanning_paused = false; + + /* Restore event mask */ + hci_set_event_mask_sync(hdev); + + /* Clear any event filters and restore scan state */ + hci_clear_event_filter_sync(hdev); + hci_update_scan_sync(hdev); + + /* Reset passive scanning to normal */ + hci_update_passive_scan_sync(hdev); + + /* Resume monitor filters */ + hci_resume_monitor_sync(hdev); + + /* Resume other advertisements */ + hci_resume_advertising_sync(hdev); + + /* Resume discovery */ + hci_resume_discovery_sync(hdev); + + return 0; +} diff --git a/net/bluetooth/hci_sysfs.c b/net/bluetooth/hci_sysfs.c index 7827639ecf5c..4e3e0451b08c 100644 --- a/net/bluetooth/hci_sysfs.c +++ b/net/bluetooth/hci_sysfs.c @@ -86,6 +86,8 @@ static void bt_host_release(struct device *dev) if (hci_dev_test_flag(hdev, HCI_UNREGISTER)) hci_release_dev(hdev); + else + kfree(hdev); module_put(THIS_MODULE); } diff --git a/net/bluetooth/l2cap_sock.c b/net/bluetooth/l2cap_sock.c index 160c016a5dfb..4574c5cb1b59 100644 --- a/net/bluetooth/l2cap_sock.c +++ b/net/bluetooth/l2cap_sock.c @@ -172,6 +172,21 @@ done: return err; } +static void l2cap_sock_init_pid(struct sock *sk) +{ + struct l2cap_chan *chan = l2cap_pi(sk)->chan; + + /* Only L2CAP_MODE_EXT_FLOWCTL ever need to access the PID in order to + * group the channels being requested. + */ + if (chan->mode != L2CAP_MODE_EXT_FLOWCTL) + return; + + spin_lock(&sk->sk_peer_lock); + sk->sk_peer_pid = get_pid(task_tgid(current)); + spin_unlock(&sk->sk_peer_lock); +} + static int l2cap_sock_connect(struct socket *sock, struct sockaddr *addr, int alen, int flags) { @@ -243,6 +258,8 @@ static int l2cap_sock_connect(struct socket *sock, struct sockaddr *addr, if (chan->psm && bdaddr_type_is_le(chan->src_type) && !chan->mode) chan->mode = L2CAP_MODE_LE_FLOWCTL; + l2cap_sock_init_pid(sk); + err = l2cap_chan_connect(chan, la.l2_psm, __le16_to_cpu(la.l2_cid), &la.l2_bdaddr, la.l2_bdaddr_type); if (err) @@ -298,6 +315,8 @@ static int l2cap_sock_listen(struct socket *sock, int backlog) goto done; } + l2cap_sock_init_pid(sk); + sk->sk_max_ack_backlog = backlog; sk->sk_ack_backlog = 0; diff --git a/net/bluetooth/mgmt.c b/net/bluetooth/mgmt.c index 3e5283607b97..f8f74d344297 100644 --- a/net/bluetooth/mgmt.c +++ b/net/bluetooth/mgmt.c @@ -39,6 +39,7 @@ #include "mgmt_config.h" #include "msft.h" #include "eir.h" +#include "aosp.h" #define MGMT_VERSION 1 #define MGMT_REVISION 21 @@ -276,10 +277,39 @@ static const u8 mgmt_status_table[] = { MGMT_STATUS_CONNECT_FAILED, /* MAC Connection Failed */ }; -static u8 mgmt_status(u8 hci_status) +static u8 mgmt_errno_status(int err) { - if (hci_status < ARRAY_SIZE(mgmt_status_table)) - return mgmt_status_table[hci_status]; + switch (err) { + case 0: + return MGMT_STATUS_SUCCESS; + case -EPERM: + return MGMT_STATUS_REJECTED; + case -EINVAL: + return MGMT_STATUS_INVALID_PARAMS; + case -EOPNOTSUPP: + return MGMT_STATUS_NOT_SUPPORTED; + case -EBUSY: + return MGMT_STATUS_BUSY; + case -ETIMEDOUT: + return MGMT_STATUS_AUTH_FAILED; + case -ENOMEM: + return MGMT_STATUS_NO_RESOURCES; + case -EISCONN: + return MGMT_STATUS_ALREADY_CONNECTED; + case -ENOTCONN: + return MGMT_STATUS_DISCONNECTED; + } + + return MGMT_STATUS_FAILED; +} + +static u8 mgmt_status(int err) +{ + if (err < 0) + return mgmt_errno_status(err); + + if (err < ARRAY_SIZE(mgmt_status_table)) + return mgmt_status_table[err]; return MGMT_STATUS_FAILED; } @@ -810,12 +840,7 @@ static u32 get_supported_settings(struct hci_dev *hdev) settings |= MGMT_SETTING_SECURE_CONN; settings |= MGMT_SETTING_PRIVACY; settings |= MGMT_SETTING_STATIC_ADDRESS; - - /* When the experimental feature for LL Privacy support is - * enabled, then advertising is no longer supported. - */ - if (!hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) - settings |= MGMT_SETTING_ADVERTISING; + settings |= MGMT_SETTING_ADVERTISING; } if (test_bit(HCI_QUIRK_EXTERNAL_CONFIG, &hdev->quirks) || @@ -903,13 +928,6 @@ static struct mgmt_pending_cmd *pending_find(u16 opcode, struct hci_dev *hdev) return mgmt_pending_find(HCI_CHANNEL_CONTROL, opcode, hdev); } -static struct mgmt_pending_cmd *pending_find_data(u16 opcode, - struct hci_dev *hdev, - const void *data) -{ - return mgmt_pending_find_data(HCI_CHANNEL_CONTROL, opcode, hdev, data); -} - u8 mgmt_get_adv_discov_flags(struct hci_dev *hdev) { struct mgmt_pending_cmd *cmd; @@ -951,32 +969,41 @@ bool mgmt_get_connectable(struct hci_dev *hdev) return hci_dev_test_flag(hdev, HCI_CONNECTABLE); } +static int service_cache_sync(struct hci_dev *hdev, void *data) +{ + hci_update_eir_sync(hdev); + hci_update_class_sync(hdev); + + return 0; +} + static void service_cache_off(struct work_struct *work) { struct hci_dev *hdev = container_of(work, struct hci_dev, service_cache.work); - struct hci_request req; if (!hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) return; - hci_req_init(&req, hdev); - - hci_dev_lock(hdev); - - __hci_req_update_eir(&req); - __hci_req_update_class(&req); - - hci_dev_unlock(hdev); + hci_cmd_sync_queue(hdev, service_cache_sync, NULL, NULL); +} - hci_req_run(&req, NULL); +static int rpa_expired_sync(struct hci_dev *hdev, void *data) +{ + /* The generation of a new RPA and programming it into the + * controller happens in the hci_req_enable_advertising() + * function. + */ + if (ext_adv_capable(hdev)) + return hci_start_ext_adv_sync(hdev, hdev->cur_adv_instance); + else + return hci_enable_advertising_sync(hdev); } static void rpa_expired(struct work_struct *work) { struct hci_dev *hdev = container_of(work, struct hci_dev, rpa_expired.work); - struct hci_request req; bt_dev_dbg(hdev, ""); @@ -985,16 +1012,7 @@ static void rpa_expired(struct work_struct *work) if (!hci_dev_test_flag(hdev, HCI_ADVERTISING)) return; - /* The generation of a new RPA and programming it into the - * controller happens in the hci_req_enable_advertising() - * function. - */ - hci_req_init(&req, hdev); - if (ext_adv_capable(hdev)) - __hci_req_start_ext_adv(&req, hdev->cur_adv_instance); - else - __hci_req_enable_advertising(&req); - hci_req_run(&req, NULL); + hci_cmd_sync_queue(hdev, rpa_expired_sync, NULL, NULL); } static void mgmt_init_hdev(struct sock *sk, struct hci_dev *hdev) @@ -1131,16 +1149,6 @@ static int send_settings_rsp(struct sock *sk, u16 opcode, struct hci_dev *hdev) sizeof(settings)); } -static void clean_up_hci_complete(struct hci_dev *hdev, u8 status, u16 opcode) -{ - bt_dev_dbg(hdev, "status 0x%02x", status); - - if (hci_conn_count(hdev) == 0) { - cancel_delayed_work(&hdev->power_off); - queue_work(hdev->req_workqueue, &hdev->power_off.work); - } -} - void mgmt_advertising_added(struct sock *sk, struct hci_dev *hdev, u8 instance) { struct mgmt_ev_advertising_added ev; @@ -1168,38 +1176,77 @@ static void cancel_adv_timeout(struct hci_dev *hdev) } } -static int clean_up_hci_state(struct hci_dev *hdev) +/* This function requires the caller holds hdev->lock */ +static void restart_le_actions(struct hci_dev *hdev) { - struct hci_request req; - struct hci_conn *conn; - bool discov_stopped; - int err; + struct hci_conn_params *p; - hci_req_init(&req, hdev); + list_for_each_entry(p, &hdev->le_conn_params, list) { + /* Needed for AUTO_OFF case where might not "really" + * have been powered off. + */ + list_del_init(&p->action); - if (test_bit(HCI_ISCAN, &hdev->flags) || - test_bit(HCI_PSCAN, &hdev->flags)) { - u8 scan = 0x00; - hci_req_add(&req, HCI_OP_WRITE_SCAN_ENABLE, 1, &scan); + switch (p->auto_connect) { + case HCI_AUTO_CONN_DIRECT: + case HCI_AUTO_CONN_ALWAYS: + list_add(&p->action, &hdev->pend_le_conns); + break; + case HCI_AUTO_CONN_REPORT: + list_add(&p->action, &hdev->pend_le_reports); + break; + default: + break; + } } +} + +static int new_settings(struct hci_dev *hdev, struct sock *skip) +{ + __le32 ev = cpu_to_le32(get_current_settings(hdev)); - hci_req_clear_adv_instance(hdev, NULL, NULL, 0x00, false); + return mgmt_limited_event(MGMT_EV_NEW_SETTINGS, hdev, &ev, + sizeof(ev), HCI_MGMT_SETTING_EVENTS, skip); +} - if (hci_dev_test_flag(hdev, HCI_LE_ADV)) - __hci_req_disable_advertising(&req); +static void mgmt_set_powered_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; - discov_stopped = hci_req_stop_discovery(&req); + bt_dev_dbg(hdev, "err %d", err); - list_for_each_entry(conn, &hdev->conn_hash.list, list) { - /* 0x15 == Terminated due to Power Off */ - __hci_abort_conn(&req, conn, 0x15); + if (!err) { + if (cp->val) { + hci_dev_lock(hdev); + restart_le_actions(hdev); + hci_update_passive_scan(hdev); + hci_dev_unlock(hdev); + } + + send_settings_rsp(cmd->sk, cmd->opcode, hdev); + + /* Only call new_setting for power on as power off is deferred + * to hdev->power_off work which does call hci_dev_do_close. + */ + if (cp->val) + new_settings(hdev, cmd->sk); + } else { + mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_SET_POWERED, + mgmt_status(err)); } - err = hci_req_run(&req, clean_up_hci_complete); - if (!err && discov_stopped) - hci_discovery_set_state(hdev, DISCOVERY_STOPPING); + mgmt_pending_free(cmd); +} - return err; +static int set_powered_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + + BT_DBG("%s", hdev->name); + + return hci_set_powered_sync(hdev, cp->val); } static int set_powered(struct sock *sk, struct hci_dev *hdev, void *data, @@ -1228,43 +1275,20 @@ static int set_powered(struct sock *sk, struct hci_dev *hdev, void *data, goto failed; } - cmd = mgmt_pending_add(sk, MGMT_OP_SET_POWERED, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_SET_POWERED, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; } - if (cp->val) { - queue_work(hdev->req_workqueue, &hdev->power_on); - err = 0; - } else { - /* Disconnect connections, stop scans, etc */ - err = clean_up_hci_state(hdev); - if (!err) - queue_delayed_work(hdev->req_workqueue, &hdev->power_off, - HCI_POWER_OFF_TIMEOUT); - - /* ENODATA means there were no HCI commands queued */ - if (err == -ENODATA) { - cancel_delayed_work(&hdev->power_off); - queue_work(hdev->req_workqueue, &hdev->power_off.work); - err = 0; - } - } + err = hci_cmd_sync_queue(hdev, set_powered_sync, cmd, + mgmt_set_powered_complete); failed: hci_dev_unlock(hdev); return err; } -static int new_settings(struct hci_dev *hdev, struct sock *skip) -{ - __le32 ev = cpu_to_le32(get_current_settings(hdev)); - - return mgmt_limited_event(MGMT_EV_NEW_SETTINGS, hdev, &ev, - sizeof(ev), HCI_MGMT_SETTING_EVENTS, skip); -} - int mgmt_new_settings(struct hci_dev *hdev) { return new_settings(hdev, NULL); @@ -1346,23 +1370,20 @@ static u8 mgmt_le_support(struct hci_dev *hdev) return MGMT_STATUS_SUCCESS; } -void mgmt_set_discoverable_complete(struct hci_dev *hdev, u8 status) +static void mgmt_set_discoverable_complete(struct hci_dev *hdev, void *data, + int err) { - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; - bt_dev_dbg(hdev, "status 0x%02x", status); + bt_dev_dbg(hdev, "err %d", err); hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_SET_DISCOVERABLE, hdev); - if (!cmd) - goto unlock; - - if (status) { - u8 mgmt_err = mgmt_status(status); + if (err) { + u8 mgmt_err = mgmt_status(err); mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, mgmt_err); hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); - goto remove_cmd; + goto done; } if (hci_dev_test_flag(hdev, HCI_DISCOVERABLE) && @@ -1374,13 +1395,18 @@ void mgmt_set_discoverable_complete(struct hci_dev *hdev, u8 status) send_settings_rsp(cmd->sk, MGMT_OP_SET_DISCOVERABLE, hdev); new_settings(hdev, cmd->sk); -remove_cmd: - mgmt_pending_remove(cmd); - -unlock: +done: + mgmt_pending_free(cmd); hci_dev_unlock(hdev); } +static int set_discoverable_sync(struct hci_dev *hdev, void *data) +{ + BT_DBG("%s", hdev->name); + + return hci_update_discoverable_sync(hdev); +} + static int set_discoverable(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -1479,7 +1505,7 @@ static int set_discoverable(struct sock *sk, struct hci_dev *hdev, void *data, goto failed; } - cmd = mgmt_pending_add(sk, MGMT_OP_SET_DISCOVERABLE, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_SET_DISCOVERABLE, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; @@ -1503,39 +1529,34 @@ static int set_discoverable(struct sock *sk, struct hci_dev *hdev, void *data, else hci_dev_clear_flag(hdev, HCI_LIMITED_DISCOVERABLE); - queue_work(hdev->req_workqueue, &hdev->discoverable_update); - err = 0; + err = hci_cmd_sync_queue(hdev, set_discoverable_sync, cmd, + mgmt_set_discoverable_complete); failed: hci_dev_unlock(hdev); return err; } -void mgmt_set_connectable_complete(struct hci_dev *hdev, u8 status) +static void mgmt_set_connectable_complete(struct hci_dev *hdev, void *data, + int err) { - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; - bt_dev_dbg(hdev, "status 0x%02x", status); + bt_dev_dbg(hdev, "err %d", err); hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_SET_CONNECTABLE, hdev); - if (!cmd) - goto unlock; - - if (status) { - u8 mgmt_err = mgmt_status(status); + if (err) { + u8 mgmt_err = mgmt_status(err); mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, mgmt_err); - goto remove_cmd; + goto done; } send_settings_rsp(cmd->sk, MGMT_OP_SET_CONNECTABLE, hdev); new_settings(hdev, cmd->sk); -remove_cmd: - mgmt_pending_remove(cmd); - -unlock: +done: + mgmt_pending_free(cmd); hci_dev_unlock(hdev); } @@ -1561,13 +1582,20 @@ static int set_connectable_update_settings(struct hci_dev *hdev, if (changed) { hci_req_update_scan(hdev); - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); return new_settings(hdev, sk); } return 0; } +static int set_connectable_sync(struct hci_dev *hdev, void *data) +{ + BT_DBG("%s", hdev->name); + + return hci_update_connectable_sync(hdev); +} + static int set_connectable(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -1600,7 +1628,7 @@ static int set_connectable(struct sock *sk, struct hci_dev *hdev, void *data, goto failed; } - cmd = mgmt_pending_add(sk, MGMT_OP_SET_CONNECTABLE, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_SET_CONNECTABLE, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; @@ -1617,8 +1645,8 @@ static int set_connectable(struct sock *sk, struct hci_dev *hdev, void *data, hci_dev_clear_flag(hdev, HCI_CONNECTABLE); } - queue_work(hdev->req_workqueue, &hdev->connectable_update); - err = 0; + err = hci_cmd_sync_queue(hdev, set_connectable_sync, cmd, + mgmt_set_connectable_complete); failed: hci_dev_unlock(hdev); @@ -1653,12 +1681,7 @@ static int set_bondable(struct sock *sk, struct hci_dev *hdev, void *data, /* In limited privacy mode the change of bondable mode * may affect the local advertising address. */ - if (hdev_is_powered(hdev) && - hci_dev_test_flag(hdev, HCI_ADVERTISING) && - hci_dev_test_flag(hdev, HCI_DISCOVERABLE) && - hci_dev_test_flag(hdev, HCI_LIMITED_PRIVACY)) - queue_work(hdev->req_workqueue, - &hdev->discoverable_update); + hci_update_discoverable(hdev); err = new_settings(hdev, sk); } @@ -1737,6 +1760,69 @@ failed: return err; } +static void set_ssp_complete(struct hci_dev *hdev, void *data, int err) +{ + struct cmd_lookup match = { NULL, hdev }; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + u8 enable = cp->val; + bool changed; + + if (err) { + u8 mgmt_err = mgmt_status(err); + + if (enable && hci_dev_test_and_clear_flag(hdev, + HCI_SSP_ENABLED)) { + hci_dev_clear_flag(hdev, HCI_HS_ENABLED); + new_settings(hdev, NULL); + } + + mgmt_pending_foreach(MGMT_OP_SET_SSP, hdev, cmd_status_rsp, + &mgmt_err); + return; + } + + if (enable) { + changed = !hci_dev_test_and_set_flag(hdev, HCI_SSP_ENABLED); + } else { + changed = hci_dev_test_and_clear_flag(hdev, HCI_SSP_ENABLED); + + if (!changed) + changed = hci_dev_test_and_clear_flag(hdev, + HCI_HS_ENABLED); + else + hci_dev_clear_flag(hdev, HCI_HS_ENABLED); + } + + mgmt_pending_foreach(MGMT_OP_SET_SSP, hdev, settings_rsp, &match); + + if (changed) + new_settings(hdev, match.sk); + + if (match.sk) + sock_put(match.sk); + + hci_update_eir_sync(hdev); +} + +static int set_ssp_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + bool changed = false; + int err; + + if (cp->val) + changed = !hci_dev_test_and_set_flag(hdev, HCI_SSP_ENABLED); + + err = hci_write_ssp_mode_sync(hdev, cp->val); + + if (!err && changed) + hci_dev_clear_flag(hdev, HCI_SSP_ENABLED); + + return err; +} + static int set_ssp(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_mode *cp = data; @@ -1798,19 +1884,18 @@ static int set_ssp(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) } cmd = mgmt_pending_add(sk, MGMT_OP_SET_SSP, hdev, data, len); - if (!cmd) { + if (!cmd) err = -ENOMEM; - goto failed; - } - - if (!cp->val && hci_dev_test_flag(hdev, HCI_USE_DEBUG_KEYS)) - hci_send_cmd(hdev, HCI_OP_WRITE_SSP_DEBUG_MODE, - sizeof(cp->val), &cp->val); + else + err = hci_cmd_sync_queue(hdev, set_ssp_sync, cmd, + set_ssp_complete); - err = hci_send_cmd(hdev, HCI_OP_WRITE_SSP_MODE, 1, &cp->val); if (err < 0) { - mgmt_pending_remove(cmd); - goto failed; + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_SSP, + MGMT_STATUS_FAILED); + + if (cmd) + mgmt_pending_remove(cmd); } failed: @@ -1879,18 +1964,17 @@ unlock: return err; } -static void le_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static void set_le_complete(struct hci_dev *hdev, void *data, int err) { struct cmd_lookup match = { NULL, hdev }; + u8 status = mgmt_status(err); - hci_dev_lock(hdev); + bt_dev_dbg(hdev, "err %d", err); if (status) { - u8 mgmt_err = mgmt_status(status); - mgmt_pending_foreach(MGMT_OP_SET_LE, hdev, cmd_status_rsp, - &mgmt_err); - goto unlock; + &status); + return; } mgmt_pending_foreach(MGMT_OP_SET_LE, hdev, settings_rsp, &match); @@ -1899,39 +1983,54 @@ static void le_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode) if (match.sk) sock_put(match.sk); +} + +static int set_le_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + u8 val = !!cp->val; + int err; + + if (!val) { + if (hci_dev_test_flag(hdev, HCI_LE_ADV)) + hci_disable_advertising_sync(hdev); + + if (ext_adv_capable(hdev)) + hci_remove_ext_adv_instance_sync(hdev, 0, cmd->sk); + } else { + hci_dev_set_flag(hdev, HCI_LE_ENABLED); + } + + err = hci_write_le_host_supported_sync(hdev, val, 0); /* Make sure the controller has a good default for * advertising data. Restrict the update to when LE * has actually been enabled. During power on, the * update in powered_update_hci will take care of it. */ - if (hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { - struct hci_request req; - hci_req_init(&req, hdev); + if (!err && hci_dev_test_flag(hdev, HCI_LE_ENABLED)) { if (ext_adv_capable(hdev)) { - int err; + int status; - err = __hci_req_setup_ext_adv_instance(&req, 0x00); - if (!err) - __hci_req_update_scan_rsp_data(&req, 0x00); + status = hci_setup_ext_adv_instance_sync(hdev, 0x00); + if (!status) + hci_update_scan_rsp_data_sync(hdev, 0x00); } else { - __hci_req_update_adv_data(&req, 0x00); - __hci_req_update_scan_rsp_data(&req, 0x00); + hci_update_adv_data_sync(hdev, 0x00); + hci_update_scan_rsp_data_sync(hdev, 0x00); } - hci_req_run(&req, NULL); - hci_update_background_scan(hdev); + + hci_update_passive_scan(hdev); } -unlock: - hci_dev_unlock(hdev); + return err; } static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_mode *cp = data; - struct hci_cp_write_le_host_supported hci_cp; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; u8 val, enabled; @@ -2001,33 +2100,20 @@ static int set_le(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) } cmd = mgmt_pending_add(sk, MGMT_OP_SET_LE, hdev, data, len); - if (!cmd) { + if (!cmd) err = -ENOMEM; - goto unlock; - } - - hci_req_init(&req, hdev); - - memset(&hci_cp, 0, sizeof(hci_cp)); + else + err = hci_cmd_sync_queue(hdev, set_le_sync, cmd, + set_le_complete); - if (val) { - hci_cp.le = val; - hci_cp.simul = 0x00; - } else { - if (hci_dev_test_flag(hdev, HCI_LE_ADV)) - __hci_req_disable_advertising(&req); + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_LE, + MGMT_STATUS_FAILED); - if (ext_adv_capable(hdev)) - __hci_req_clear_ext_adv_sets(&req); + if (cmd) + mgmt_pending_remove(cmd); } - hci_req_add(&req, HCI_OP_WRITE_LE_HOST_SUPPORTED, sizeof(hci_cp), - &hci_cp); - - err = hci_req_run(&req, le_enable_complete); - if (err < 0) - mgmt_pending_remove(cmd); - unlock: hci_dev_unlock(hdev); return err; @@ -2075,37 +2161,33 @@ static u8 get_uuid_size(const u8 *uuid) return 16; } -static void mgmt_class_complete(struct hci_dev *hdev, u16 mgmt_op, u8 status) +static void mgmt_class_complete(struct hci_dev *hdev, void *data, int err) { - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; - hci_dev_lock(hdev); - - cmd = pending_find(mgmt_op, hdev); - if (!cmd) - goto unlock; + bt_dev_dbg(hdev, "err %d", err); mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status), hdev->dev_class, 3); - - mgmt_pending_remove(cmd); + mgmt_status(err), hdev->dev_class, 3); -unlock: - hci_dev_unlock(hdev); + mgmt_pending_free(cmd); } -static void add_uuid_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static int add_uuid_sync(struct hci_dev *hdev, void *data) { - bt_dev_dbg(hdev, "status 0x%02x", status); + int err; + + err = hci_update_class_sync(hdev); + if (err) + return err; - mgmt_class_complete(hdev, MGMT_OP_ADD_UUID, status); + return hci_update_eir_sync(hdev); } static int add_uuid(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_cp_add_uuid *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; struct bt_uuid *uuid; int err; @@ -2131,28 +2213,17 @@ static int add_uuid(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) list_add_tail(&uuid->list, &hdev->uuids); - hci_req_init(&req, hdev); - - __hci_req_update_class(&req); - __hci_req_update_eir(&req); - - err = hci_req_run(&req, add_uuid_complete); - if (err < 0) { - if (err != -ENODATA) - goto failed; - - err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_ADD_UUID, 0, - hdev->dev_class, 3); - goto failed; - } - - cmd = mgmt_pending_add(sk, MGMT_OP_ADD_UUID, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_ADD_UUID, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; } - err = 0; + err = hci_cmd_sync_queue(hdev, add_uuid_sync, cmd, mgmt_class_complete); + if (err < 0) { + mgmt_pending_free(cmd); + goto failed; + } failed: hci_dev_unlock(hdev); @@ -2173,11 +2244,15 @@ static bool enable_service_cache(struct hci_dev *hdev) return false; } -static void remove_uuid_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static int remove_uuid_sync(struct hci_dev *hdev, void *data) { - bt_dev_dbg(hdev, "status 0x%02x", status); + int err; + + err = hci_update_class_sync(hdev); + if (err) + return err; - mgmt_class_complete(hdev, MGMT_OP_REMOVE_UUID, status); + return hci_update_eir_sync(hdev); } static int remove_uuid(struct sock *sk, struct hci_dev *hdev, void *data, @@ -2187,7 +2262,6 @@ static int remove_uuid(struct sock *sk, struct hci_dev *hdev, void *data, struct mgmt_pending_cmd *cmd; struct bt_uuid *match, *tmp; u8 bt_uuid_any[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; - struct hci_request req; int err, found; bt_dev_dbg(hdev, "sock %p", sk); @@ -2231,39 +2305,35 @@ static int remove_uuid(struct sock *sk, struct hci_dev *hdev, void *data, } update_class: - hci_req_init(&req, hdev); - - __hci_req_update_class(&req); - __hci_req_update_eir(&req); - - err = hci_req_run(&req, remove_uuid_complete); - if (err < 0) { - if (err != -ENODATA) - goto unlock; - - err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_REMOVE_UUID, 0, - hdev->dev_class, 3); - goto unlock; - } - - cmd = mgmt_pending_add(sk, MGMT_OP_REMOVE_UUID, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_REMOVE_UUID, hdev, data, len); if (!cmd) { err = -ENOMEM; goto unlock; } - err = 0; + err = hci_cmd_sync_queue(hdev, remove_uuid_sync, cmd, + mgmt_class_complete); + if (err < 0) + mgmt_pending_free(cmd); unlock: hci_dev_unlock(hdev); return err; } -static void set_class_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static int set_class_sync(struct hci_dev *hdev, void *data) { - bt_dev_dbg(hdev, "status 0x%02x", status); + int err = 0; + + if (hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) { + cancel_delayed_work_sync(&hdev->service_cache); + err = hci_update_eir_sync(hdev); + } + + if (err) + return err; - mgmt_class_complete(hdev, MGMT_OP_SET_DEV_CLASS, status); + return hci_update_class_sync(hdev); } static int set_dev_class(struct sock *sk, struct hci_dev *hdev, void *data, @@ -2271,7 +2341,6 @@ static int set_dev_class(struct sock *sk, struct hci_dev *hdev, void *data, { struct mgmt_cp_set_dev_class *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); @@ -2303,34 +2372,16 @@ static int set_dev_class(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - hci_req_init(&req, hdev); - - if (hci_dev_test_and_clear_flag(hdev, HCI_SERVICE_CACHE)) { - hci_dev_unlock(hdev); - cancel_delayed_work_sync(&hdev->service_cache); - hci_dev_lock(hdev); - __hci_req_update_eir(&req); - } - - __hci_req_update_class(&req); - - err = hci_req_run(&req, set_class_complete); - if (err < 0) { - if (err != -ENODATA) - goto unlock; - - err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_SET_DEV_CLASS, 0, - hdev->dev_class, 3); - goto unlock; - } - - cmd = mgmt_pending_add(sk, MGMT_OP_SET_DEV_CLASS, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_SET_DEV_CLASS, hdev, data, len); if (!cmd) { err = -ENOMEM; goto unlock; } - err = 0; + err = hci_cmd_sync_queue(hdev, set_class_sync, cmd, + mgmt_class_complete); + if (err < 0) + mgmt_pending_free(cmd); unlock: hci_dev_unlock(hdev); @@ -3228,65 +3279,70 @@ static int user_passkey_neg_reply(struct sock *sk, struct hci_dev *hdev, HCI_OP_USER_PASSKEY_NEG_REPLY, 0); } -static void adv_expire(struct hci_dev *hdev, u32 flags) +static int adv_expire_sync(struct hci_dev *hdev, u32 flags) { struct adv_info *adv_instance; - struct hci_request req; - int err; adv_instance = hci_find_adv_instance(hdev, hdev->cur_adv_instance); if (!adv_instance) - return; + return 0; /* stop if current instance doesn't need to be changed */ if (!(adv_instance->flags & flags)) - return; + return 0; cancel_adv_timeout(hdev); adv_instance = hci_get_next_instance(hdev, adv_instance->instance); if (!adv_instance) - return; + return 0; - hci_req_init(&req, hdev); - err = __hci_req_schedule_adv_instance(&req, adv_instance->instance, - true); - if (err) - return; + hci_schedule_adv_instance_sync(hdev, adv_instance->instance, true); - hci_req_run(&req, NULL); + return 0; } -static void set_name_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static int name_changed_sync(struct hci_dev *hdev, void *data) { - struct mgmt_cp_set_local_name *cp; - struct mgmt_pending_cmd *cmd; - - bt_dev_dbg(hdev, "status 0x%02x", status); - - hci_dev_lock(hdev); + return adv_expire_sync(hdev, MGMT_ADV_FLAG_LOCAL_NAME); +} - cmd = pending_find(MGMT_OP_SET_LOCAL_NAME, hdev); - if (!cmd) - goto unlock; +static void set_name_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_set_local_name *cp = cmd->param; + u8 status = mgmt_status(err); - cp = cmd->param; + bt_dev_dbg(hdev, "err %d", err); if (status) { mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_SET_LOCAL_NAME, - mgmt_status(status)); + status); } else { mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_SET_LOCAL_NAME, 0, cp, sizeof(*cp)); if (hci_dev_test_flag(hdev, HCI_LE_ADV)) - adv_expire(hdev, MGMT_ADV_FLAG_LOCAL_NAME); + hci_cmd_sync_queue(hdev, name_changed_sync, NULL, NULL); } mgmt_pending_remove(cmd); +} -unlock: - hci_dev_unlock(hdev); +static int set_name_sync(struct hci_dev *hdev, void *data) +{ + if (lmp_bredr_capable(hdev)) { + hci_update_name_sync(hdev); + hci_update_eir_sync(hdev); + } + + /* The name is stored in the scan response data and so + * no need to update the advertising data here. + */ + if (lmp_le_capable(hdev) && hci_dev_test_flag(hdev, HCI_ADVERTISING)) + hci_update_scan_rsp_data_sync(hdev, hdev->cur_adv_instance); + + return 0; } static int set_local_name(struct sock *sk, struct hci_dev *hdev, void *data, @@ -3294,7 +3350,6 @@ static int set_local_name(struct sock *sk, struct hci_dev *hdev, void *data, { struct mgmt_cp_set_local_name *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); @@ -3330,35 +3385,34 @@ static int set_local_name(struct sock *sk, struct hci_dev *hdev, void *data, } cmd = mgmt_pending_add(sk, MGMT_OP_SET_LOCAL_NAME, hdev, data, len); - if (!cmd) { + if (!cmd) err = -ENOMEM; - goto failed; - } + else + err = hci_cmd_sync_queue(hdev, set_name_sync, cmd, + set_name_complete); - memcpy(hdev->dev_name, cp->name, sizeof(hdev->dev_name)); + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_LOCAL_NAME, + MGMT_STATUS_FAILED); - hci_req_init(&req, hdev); + if (cmd) + mgmt_pending_remove(cmd); - if (lmp_bredr_capable(hdev)) { - __hci_req_update_name(&req); - __hci_req_update_eir(&req); + goto failed; } - /* The name is stored in the scan response data and so - * no need to update the advertising data here. - */ - if (lmp_le_capable(hdev) && hci_dev_test_flag(hdev, HCI_ADVERTISING)) - __hci_req_update_scan_rsp_data(&req, hdev->cur_adv_instance); - - err = hci_req_run(&req, set_name_complete); - if (err < 0) - mgmt_pending_remove(cmd); + memcpy(hdev->dev_name, cp->name, sizeof(hdev->dev_name)); failed: hci_dev_unlock(hdev); return err; } +static int appearance_changed_sync(struct hci_dev *hdev, void *data) +{ + return adv_expire_sync(hdev, MGMT_ADV_FLAG_APPEARANCE); +} + static int set_appearance(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -3380,7 +3434,8 @@ static int set_appearance(struct sock *sk, struct hci_dev *hdev, void *data, hdev->appearance = appearance; if (hci_dev_test_flag(hdev, HCI_LE_ADV)) - adv_expire(hdev, MGMT_ADV_FLAG_APPEARANCE); + hci_cmd_sync_queue(hdev, appearance_changed_sync, NULL, + NULL); ext_info_changed(hdev, sk); } @@ -3426,23 +3481,26 @@ int mgmt_phy_configuration_changed(struct hci_dev *hdev, struct sock *skip) sizeof(ev), skip); } -static void set_default_phy_complete(struct hci_dev *hdev, u8 status, - u16 opcode, struct sk_buff *skb) +static void set_default_phy_complete(struct hci_dev *hdev, void *data, int err) { - struct mgmt_pending_cmd *cmd; - - bt_dev_dbg(hdev, "status 0x%02x", status); + struct mgmt_pending_cmd *cmd = data; + struct sk_buff *skb = cmd->skb; + u8 status = mgmt_status(err); - hci_dev_lock(hdev); + if (!status) { + if (!skb) + status = MGMT_STATUS_FAILED; + else if (IS_ERR(skb)) + status = mgmt_status(PTR_ERR(skb)); + else + status = mgmt_status(skb->data[0]); + } - cmd = pending_find(MGMT_OP_SET_PHY_CONFIGURATION, hdev); - if (!cmd) - goto unlock; + bt_dev_dbg(hdev, "status %d", status); if (status) { mgmt_cmd_status(cmd->sk, hdev->id, - MGMT_OP_SET_PHY_CONFIGURATION, - mgmt_status(status)); + MGMT_OP_SET_PHY_CONFIGURATION, status); } else { mgmt_cmd_complete(cmd->sk, hdev->id, MGMT_OP_SET_PHY_CONFIGURATION, 0, @@ -3451,19 +3509,56 @@ static void set_default_phy_complete(struct hci_dev *hdev, u8 status, mgmt_phy_configuration_changed(hdev, cmd->sk); } + if (skb && !IS_ERR(skb)) + kfree_skb(skb); + mgmt_pending_remove(cmd); +} -unlock: - hci_dev_unlock(hdev); +static int set_default_phy_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_set_phy_configuration *cp = cmd->param; + struct hci_cp_le_set_default_phy cp_phy; + u32 selected_phys = __le32_to_cpu(cp->selected_phys); + + memset(&cp_phy, 0, sizeof(cp_phy)); + + if (!(selected_phys & MGMT_PHY_LE_TX_MASK)) + cp_phy.all_phys |= 0x01; + + if (!(selected_phys & MGMT_PHY_LE_RX_MASK)) + cp_phy.all_phys |= 0x02; + + if (selected_phys & MGMT_PHY_LE_1M_TX) + cp_phy.tx_phys |= HCI_LE_SET_PHY_1M; + + if (selected_phys & MGMT_PHY_LE_2M_TX) + cp_phy.tx_phys |= HCI_LE_SET_PHY_2M; + + if (selected_phys & MGMT_PHY_LE_CODED_TX) + cp_phy.tx_phys |= HCI_LE_SET_PHY_CODED; + + if (selected_phys & MGMT_PHY_LE_1M_RX) + cp_phy.rx_phys |= HCI_LE_SET_PHY_1M; + + if (selected_phys & MGMT_PHY_LE_2M_RX) + cp_phy.rx_phys |= HCI_LE_SET_PHY_2M; + + if (selected_phys & MGMT_PHY_LE_CODED_RX) + cp_phy.rx_phys |= HCI_LE_SET_PHY_CODED; + + cmd->skb = __hci_cmd_sync(hdev, HCI_OP_LE_SET_DEFAULT_PHY, + sizeof(cp_phy), &cp_phy, HCI_CMD_TIMEOUT); + + return 0; } static int set_phy_configuration(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_cp_set_phy_configuration *cp = data; - struct hci_cp_le_set_default_phy cp_phy; struct mgmt_pending_cmd *cmd; - struct hci_request req; u32 selected_phys, configurable_phys, supported_phys, unconfigure_phys; u16 pkt_type = (HCI_DH1 | HCI_DM1); bool changed = false; @@ -3567,44 +3662,20 @@ static int set_phy_configuration(struct sock *sk, struct hci_dev *hdev, cmd = mgmt_pending_add(sk, MGMT_OP_SET_PHY_CONFIGURATION, hdev, data, len); - if (!cmd) { + if (!cmd) err = -ENOMEM; - goto unlock; - } - - hci_req_init(&req, hdev); - - memset(&cp_phy, 0, sizeof(cp_phy)); - - if (!(selected_phys & MGMT_PHY_LE_TX_MASK)) - cp_phy.all_phys |= 0x01; - - if (!(selected_phys & MGMT_PHY_LE_RX_MASK)) - cp_phy.all_phys |= 0x02; - - if (selected_phys & MGMT_PHY_LE_1M_TX) - cp_phy.tx_phys |= HCI_LE_SET_PHY_1M; - - if (selected_phys & MGMT_PHY_LE_2M_TX) - cp_phy.tx_phys |= HCI_LE_SET_PHY_2M; - - if (selected_phys & MGMT_PHY_LE_CODED_TX) - cp_phy.tx_phys |= HCI_LE_SET_PHY_CODED; - - if (selected_phys & MGMT_PHY_LE_1M_RX) - cp_phy.rx_phys |= HCI_LE_SET_PHY_1M; - - if (selected_phys & MGMT_PHY_LE_2M_RX) - cp_phy.rx_phys |= HCI_LE_SET_PHY_2M; - - if (selected_phys & MGMT_PHY_LE_CODED_RX) - cp_phy.rx_phys |= HCI_LE_SET_PHY_CODED; + else + err = hci_cmd_sync_queue(hdev, set_default_phy_sync, cmd, + set_default_phy_complete); - hci_req_add(&req, HCI_OP_LE_SET_DEFAULT_PHY, sizeof(cp_phy), &cp_phy); + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_PHY_CONFIGURATION, + MGMT_STATUS_FAILED); - err = hci_req_run_skb(&req, set_default_phy_complete); - if (err < 0) - mgmt_pending_remove(cmd); + if (cmd) + mgmt_pending_remove(cmd); + } unlock: hci_dev_unlock(hdev); @@ -3852,7 +3923,7 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, idx++; } - if (hdev && use_ll_privacy(hdev)) { + if (hdev && ll_privacy_capable(hdev)) { if (hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) flags = BIT(0) | BIT(1); else @@ -3863,7 +3934,8 @@ static int read_exp_features_info(struct sock *sk, struct hci_dev *hdev, idx++; } - if (hdev && hdev->set_quality_report) { + if (hdev && (aosp_has_quality_report(hdev) || + hdev->set_quality_report)) { if (hci_dev_test_flag(hdev, HCI_QUALITY_REPORT)) flags = BIT(0); else @@ -3927,7 +3999,9 @@ static int exp_debug_feature_changed(bool enabled, struct sock *skip) } #endif -static int exp_quality_report_feature_changed(bool enabled, struct sock *skip) +static int exp_quality_report_feature_changed(bool enabled, + struct hci_dev *hdev, + struct sock *skip) { struct mgmt_ev_exp_feature_changed ev; @@ -3935,7 +4009,7 @@ static int exp_quality_report_feature_changed(bool enabled, struct sock *skip) memcpy(ev.uuid, quality_report_uuid, 16); ev.flags = cpu_to_le32(enabled ? BIT(0) : 0); - return mgmt_limited_event(MGMT_EV_EXP_FEATURE_CHANGED, NULL, + return mgmt_limited_event(MGMT_EV_EXP_FEATURE_CHANGED, hdev, &ev, sizeof(ev), HCI_MGMT_EXP_FEATURE_EVENTS, skip); } @@ -4125,7 +4199,7 @@ static int set_quality_report_func(struct sock *sk, struct hci_dev *hdev, val = !!cp->param[0]; changed = (val != hci_dev_test_flag(hdev, HCI_QUALITY_REPORT)); - if (!hdev->set_quality_report) { + if (!aosp_has_quality_report(hdev) && !hdev->set_quality_report) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_EXP_FEATURE, MGMT_STATUS_NOT_SUPPORTED); @@ -4133,13 +4207,18 @@ static int set_quality_report_func(struct sock *sk, struct hci_dev *hdev, } if (changed) { - err = hdev->set_quality_report(hdev, val); + if (hdev->set_quality_report) + err = hdev->set_quality_report(hdev, val); + else + err = aosp_set_quality_report(hdev, val); + if (err) { err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_EXP_FEATURE, MGMT_STATUS_FAILED); goto unlock_quality_report; } + if (val) hci_dev_set_flag(hdev, HCI_QUALITY_REPORT); else @@ -4151,19 +4230,20 @@ static int set_quality_report_func(struct sock *sk, struct hci_dev *hdev, memcpy(rp.uuid, quality_report_uuid, 16); rp.flags = cpu_to_le32(val ? BIT(0) : 0); hci_sock_set_flag(sk, HCI_MGMT_EXP_FEATURE_EVENTS); - err = mgmt_cmd_complete(sk, hdev->id, - MGMT_OP_SET_EXP_FEATURE, 0, + + err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_SET_EXP_FEATURE, 0, &rp, sizeof(rp)); if (changed) - exp_quality_report_feature_changed(val, sk); + exp_quality_report_feature_changed(val, hdev, sk); unlock_quality_report: hci_req_sync_unlock(hdev); return err; } -static int exp_offload_codec_feature_changed(bool enabled, struct sock *skip) +static int exp_offload_codec_feature_changed(bool enabled, struct hci_dev *hdev, + struct sock *skip) { struct mgmt_ev_exp_feature_changed ev; @@ -4171,7 +4251,7 @@ static int exp_offload_codec_feature_changed(bool enabled, struct sock *skip) memcpy(ev.uuid, offload_codecs_uuid, 16); ev.flags = cpu_to_le32(enabled ? BIT(0) : 0); - return mgmt_limited_event(MGMT_EV_EXP_FEATURE_CHANGED, NULL, + return mgmt_limited_event(MGMT_EV_EXP_FEATURE_CHANGED, hdev, &ev, sizeof(ev), HCI_MGMT_EXP_FEATURE_EVENTS, skip); } @@ -4229,7 +4309,7 @@ static int set_offload_codec_func(struct sock *sk, struct hci_dev *hdev, &rp, sizeof(rp)); if (changed) - exp_offload_codec_feature_changed(val, sk); + exp_offload_codec_feature_changed(val, hdev, sk); return err; } @@ -4496,7 +4576,7 @@ int mgmt_add_adv_patterns_monitor_complete(struct hci_dev *hdev, u8 status) hdev->adv_monitors_cnt++; if (monitor->state == ADV_MONITOR_STATE_NOT_REGISTERED) monitor->state = ADV_MONITOR_STATE_REGISTERED; - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); } err = mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, @@ -4722,7 +4802,7 @@ int mgmt_remove_adv_monitor_complete(struct hci_dev *hdev, u8 status) rp.monitor_handle = cp->monitor_handle; if (!status) - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); err = mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, mgmt_status(status), &rp, sizeof(rp)); @@ -4801,28 +4881,33 @@ unlock: status); } -static void read_local_oob_data_complete(struct hci_dev *hdev, u8 status, - u16 opcode, struct sk_buff *skb) +static void read_local_oob_data_complete(struct hci_dev *hdev, void *data, int err) { struct mgmt_rp_read_local_oob_data mgmt_rp; size_t rp_size = sizeof(mgmt_rp); - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; + struct sk_buff *skb = cmd->skb; + u8 status = mgmt_status(err); - bt_dev_dbg(hdev, "status %u", status); + if (!status) { + if (!skb) + status = MGMT_STATUS_FAILED; + else if (IS_ERR(skb)) + status = mgmt_status(PTR_ERR(skb)); + else + status = mgmt_status(skb->data[0]); + } - cmd = pending_find(MGMT_OP_READ_LOCAL_OOB_DATA, hdev); - if (!cmd) - return; + bt_dev_dbg(hdev, "status %d", status); - if (status || !skb) { - mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_READ_LOCAL_OOB_DATA, - status ? mgmt_status(status) : MGMT_STATUS_FAILED); + if (status) { + mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_READ_LOCAL_OOB_DATA, status); goto remove; } memset(&mgmt_rp, 0, sizeof(mgmt_rp)); - if (opcode == HCI_OP_READ_LOCAL_OOB_DATA) { + if (!bredr_sc_enabled(hdev)) { struct hci_rp_read_local_oob_data *rp = (void *) skb->data; if (skb->len < sizeof(*rp)) { @@ -4857,14 +4942,31 @@ static void read_local_oob_data_complete(struct hci_dev *hdev, u8 status, MGMT_STATUS_SUCCESS, &mgmt_rp, rp_size); remove: - mgmt_pending_remove(cmd); + if (skb && !IS_ERR(skb)) + kfree_skb(skb); + + mgmt_pending_free(cmd); +} + +static int read_local_oob_data_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + + if (bredr_sc_enabled(hdev)) + cmd->skb = hci_read_local_oob_data_sync(hdev, true, cmd->sk); + else + cmd->skb = hci_read_local_oob_data_sync(hdev, false, cmd->sk); + + if (IS_ERR(cmd->skb)) + return PTR_ERR(cmd->skb); + else + return 0; } static int read_local_oob_data(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); @@ -4889,22 +4991,20 @@ static int read_local_oob_data(struct sock *sk, struct hci_dev *hdev, goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_READ_LOCAL_OOB_DATA, hdev, NULL, 0); - if (!cmd) { + cmd = mgmt_pending_new(sk, MGMT_OP_READ_LOCAL_OOB_DATA, hdev, NULL, 0); + if (!cmd) err = -ENOMEM; - goto unlock; - } - - hci_req_init(&req, hdev); - - if (bredr_sc_enabled(hdev)) - hci_req_add(&req, HCI_OP_READ_LOCAL_OOB_EXT_DATA, 0, NULL); else - hci_req_add(&req, HCI_OP_READ_LOCAL_OOB_DATA, 0, NULL); + err = hci_cmd_sync_queue(hdev, read_local_oob_data_sync, cmd, + read_local_oob_data_complete); - err = hci_req_run_skb(&req, read_local_oob_data_complete); - if (err < 0) - mgmt_pending_remove(cmd); + if (err < 0) { + err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_READ_LOCAL_OOB_DATA, + MGMT_STATUS_FAILED); + + if (cmd) + mgmt_pending_free(cmd); + } unlock: hci_dev_unlock(hdev); @@ -5077,13 +5177,6 @@ void mgmt_start_discovery_complete(struct hci_dev *hdev, u8 status) } hci_dev_unlock(hdev); - - /* Handle suspend notifier */ - if (test_and_clear_bit(SUSPEND_UNPAUSE_DISCOVERY, - hdev->suspend_tasks)) { - bt_dev_dbg(hdev, "Unpaused discovery"); - wake_up(&hdev->suspend_wait_q); - } } static bool discovery_type_is_valid(struct hci_dev *hdev, uint8_t type, @@ -5113,6 +5206,25 @@ static bool discovery_type_is_valid(struct hci_dev *hdev, uint8_t type, return true; } +static void start_discovery_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + + bt_dev_dbg(hdev, "err %d", err); + + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, mgmt_status(err), + cmd->param, 1); + mgmt_pending_free(cmd); + + hci_discovery_set_state(hdev, err ? DISCOVERY_STOPPED: + DISCOVERY_FINDING); +} + +static int start_discovery_sync(struct hci_dev *hdev, void *data) +{ + return hci_start_discovery_sync(hdev); +} + static int start_discovery_internal(struct sock *sk, struct hci_dev *hdev, u16 op, void *data, u16 len) { @@ -5164,17 +5276,20 @@ static int start_discovery_internal(struct sock *sk, struct hci_dev *hdev, else hdev->discovery.limited = false; - cmd = mgmt_pending_add(sk, op, hdev, data, len); + cmd = mgmt_pending_new(sk, op, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; } - cmd->cmd_complete = generic_cmd_complete; + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, + start_discovery_complete); + if (err < 0) { + mgmt_pending_free(cmd); + goto failed; + } hci_discovery_set_state(hdev, DISCOVERY_STARTING); - queue_work(hdev->req_workqueue, &hdev->discov_update); - err = 0; failed: hci_dev_unlock(hdev); @@ -5196,13 +5311,6 @@ static int start_limited_discovery(struct sock *sk, struct hci_dev *hdev, data, len); } -static int service_discovery_cmd_complete(struct mgmt_pending_cmd *cmd, - u8 status) -{ - return mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, status, - cmd->param, 1); -} - static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -5271,15 +5379,13 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, goto failed; } - cmd = mgmt_pending_add(sk, MGMT_OP_START_SERVICE_DISCOVERY, + cmd = mgmt_pending_new(sk, MGMT_OP_START_SERVICE_DISCOVERY, hdev, data, len); if (!cmd) { err = -ENOMEM; goto failed; } - cmd->cmd_complete = service_discovery_cmd_complete; - /* Clear the discovery filter first to free any previously * allocated memory for the UUID list. */ @@ -5303,9 +5409,14 @@ static int start_service_discovery(struct sock *sk, struct hci_dev *hdev, } } + err = hci_cmd_sync_queue(hdev, start_discovery_sync, cmd, + start_discovery_complete); + if (err < 0) { + mgmt_pending_free(cmd); + goto failed; + } + hci_discovery_set_state(hdev, DISCOVERY_STARTING); - queue_work(hdev->req_workqueue, &hdev->discov_update); - err = 0; failed: hci_dev_unlock(hdev); @@ -5327,12 +5438,25 @@ void mgmt_stop_discovery_complete(struct hci_dev *hdev, u8 status) } hci_dev_unlock(hdev); +} - /* Handle suspend notifier */ - if (test_and_clear_bit(SUSPEND_PAUSE_DISCOVERY, hdev->suspend_tasks)) { - bt_dev_dbg(hdev, "Paused discovery"); - wake_up(&hdev->suspend_wait_q); - } +static void stop_discovery_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + + bt_dev_dbg(hdev, "err %d", err); + + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, mgmt_status(err), + cmd->param, 1); + mgmt_pending_free(cmd); + + if (!err) + hci_discovery_set_state(hdev, DISCOVERY_STOPPED); +} + +static int stop_discovery_sync(struct hci_dev *hdev, void *data) +{ + return hci_stop_discovery_sync(hdev); } static int stop_discovery(struct sock *sk, struct hci_dev *hdev, void *data, @@ -5360,17 +5484,20 @@ static int stop_discovery(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_STOP_DISCOVERY, hdev, data, len); + cmd = mgmt_pending_new(sk, MGMT_OP_STOP_DISCOVERY, hdev, data, len); if (!cmd) { err = -ENOMEM; goto unlock; } - cmd->cmd_complete = generic_cmd_complete; + err = hci_cmd_sync_queue(hdev, stop_discovery_sync, cmd, + stop_discovery_complete); + if (err < 0) { + mgmt_pending_free(cmd); + goto unlock; + } hci_discovery_set_state(hdev, DISCOVERY_STOPPING); - queue_work(hdev->req_workqueue, &hdev->discov_update); - err = 0; unlock: hci_dev_unlock(hdev); @@ -5491,11 +5618,15 @@ done: return err; } +static int set_device_id_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_eir_sync(hdev); +} + static int set_device_id(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_cp_set_device_id *cp = data; - struct hci_request req; int err; __u16 source; @@ -5517,38 +5648,32 @@ static int set_device_id(struct sock *sk, struct hci_dev *hdev, void *data, err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_SET_DEVICE_ID, 0, NULL, 0); - hci_req_init(&req, hdev); - __hci_req_update_eir(&req); - hci_req_run(&req, NULL); + hci_cmd_sync_queue(hdev, set_device_id_sync, NULL, NULL); hci_dev_unlock(hdev); return err; } -static void enable_advertising_instance(struct hci_dev *hdev, u8 status, - u16 opcode) +static void enable_advertising_instance(struct hci_dev *hdev, int err) { - bt_dev_dbg(hdev, "status %u", status); + if (err) + bt_dev_err(hdev, "failed to re-configure advertising %d", err); + else + bt_dev_dbg(hdev, "status %d", err); } -static void set_advertising_complete(struct hci_dev *hdev, u8 status, - u16 opcode) +static void set_advertising_complete(struct hci_dev *hdev, void *data, int err) { struct cmd_lookup match = { NULL, hdev }; - struct hci_request req; u8 instance; struct adv_info *adv_instance; - int err; - - hci_dev_lock(hdev); + u8 status = mgmt_status(err); if (status) { - u8 mgmt_err = mgmt_status(status); - mgmt_pending_foreach(MGMT_OP_SET_ADVERTISING, hdev, - cmd_status_rsp, &mgmt_err); - goto unlock; + cmd_status_rsp, &status); + return; } if (hci_dev_test_flag(hdev, HCI_LE_ADV)) @@ -5564,46 +5689,60 @@ static void set_advertising_complete(struct hci_dev *hdev, u8 status, if (match.sk) sock_put(match.sk); - /* Handle suspend notifier */ - if (test_and_clear_bit(SUSPEND_PAUSE_ADVERTISING, - hdev->suspend_tasks)) { - bt_dev_dbg(hdev, "Paused advertising"); - wake_up(&hdev->suspend_wait_q); - } else if (test_and_clear_bit(SUSPEND_UNPAUSE_ADVERTISING, - hdev->suspend_tasks)) { - bt_dev_dbg(hdev, "Unpaused advertising"); - wake_up(&hdev->suspend_wait_q); - } - /* If "Set Advertising" was just disabled and instance advertising was * set up earlier, then re-enable multi-instance advertising. */ if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || list_empty(&hdev->adv_instances)) - goto unlock; + return; instance = hdev->cur_adv_instance; if (!instance) { adv_instance = list_first_entry_or_null(&hdev->adv_instances, struct adv_info, list); if (!adv_instance) - goto unlock; + return; instance = adv_instance->instance; } - hci_req_init(&req, hdev); + err = hci_schedule_adv_instance_sync(hdev, instance, true); - err = __hci_req_schedule_adv_instance(&req, instance, true); + enable_advertising_instance(hdev, err); +} - if (!err) - err = hci_req_run(&req, enable_advertising_instance); +static int set_adv_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + u8 val = !!cp->val; - if (err) - bt_dev_err(hdev, "failed to re-configure advertising"); + if (cp->val == 0x02) + hci_dev_set_flag(hdev, HCI_ADVERTISING_CONNECTABLE); + else + hci_dev_clear_flag(hdev, HCI_ADVERTISING_CONNECTABLE); -unlock: - hci_dev_unlock(hdev); + cancel_adv_timeout(hdev); + + if (val) { + /* Switch to instance "0" for the Set Advertising setting. + * We cannot use update_[adv|scan_rsp]_data() here as the + * HCI_ADVERTISING flag is not yet set. + */ + hdev->cur_adv_instance = 0x00; + + if (ext_adv_capable(hdev)) { + hci_start_ext_adv_sync(hdev, 0x00); + } else { + hci_update_adv_data_sync(hdev, 0x00); + hci_update_scan_rsp_data_sync(hdev, 0x00); + hci_enable_advertising_sync(hdev); + } + } else { + hci_disable_advertising_sync(hdev); + } + + return 0; } static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data, @@ -5611,7 +5750,6 @@ static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data, { struct mgmt_mode *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; u8 val, status; int err; @@ -5622,13 +5760,6 @@ static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_ADVERTISING, status); - /* Enabling the experimental LL Privay support disables support for - * advertising. - */ - if (hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_ADVERTISING, - MGMT_STATUS_NOT_SUPPORTED); - if (cp->val != 0x00 && cp->val != 0x01 && cp->val != 0x02) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_ADVERTISING, MGMT_STATUS_INVALID_PARAMS); @@ -5684,40 +5815,13 @@ static int set_advertising(struct sock *sk, struct hci_dev *hdev, void *data, } cmd = mgmt_pending_add(sk, MGMT_OP_SET_ADVERTISING, hdev, data, len); - if (!cmd) { + if (!cmd) err = -ENOMEM; - goto unlock; - } - - hci_req_init(&req, hdev); - - if (cp->val == 0x02) - hci_dev_set_flag(hdev, HCI_ADVERTISING_CONNECTABLE); else - hci_dev_clear_flag(hdev, HCI_ADVERTISING_CONNECTABLE); - - cancel_adv_timeout(hdev); - - if (val) { - /* Switch to instance "0" for the Set Advertising setting. - * We cannot use update_[adv|scan_rsp]_data() here as the - * HCI_ADVERTISING flag is not yet set. - */ - hdev->cur_adv_instance = 0x00; + err = hci_cmd_sync_queue(hdev, set_adv_sync, cmd, + set_advertising_complete); - if (ext_adv_capable(hdev)) { - __hci_req_start_ext_adv(&req, 0x00); - } else { - __hci_req_update_adv_data(&req, 0x00); - __hci_req_update_scan_rsp_data(&req, 0x00); - __hci_req_enable_advertising(&req); - } - } else { - __hci_req_disable_advertising(&req); - } - - err = hci_req_run(&req, set_advertising_complete); - if (err < 0) + if (err < 0 && cmd) mgmt_pending_remove(cmd); unlock: @@ -5810,38 +5914,23 @@ static int set_scan_params(struct sock *sk, struct hci_dev *hdev, * loaded. */ if (hci_dev_test_flag(hdev, HCI_LE_SCAN) && - hdev->discovery.state == DISCOVERY_STOPPED) { - struct hci_request req; - - hci_req_init(&req, hdev); - - hci_req_add_le_scan_disable(&req, false); - hci_req_add_le_passive_scan(&req); - - hci_req_run(&req, NULL); - } + hdev->discovery.state == DISCOVERY_STOPPED) + hci_update_passive_scan(hdev); hci_dev_unlock(hdev); return err; } -static void fast_connectable_complete(struct hci_dev *hdev, u8 status, - u16 opcode) +static void fast_connectable_complete(struct hci_dev *hdev, void *data, int err) { - struct mgmt_pending_cmd *cmd; - - bt_dev_dbg(hdev, "status 0x%02x", status); + struct mgmt_pending_cmd *cmd = data; - hci_dev_lock(hdev); - - cmd = pending_find(MGMT_OP_SET_FAST_CONNECTABLE, hdev); - if (!cmd) - goto unlock; + bt_dev_dbg(hdev, "err %d", err); - if (status) { + if (err) { mgmt_cmd_status(cmd->sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, - mgmt_status(status)); + mgmt_status(err)); } else { struct mgmt_mode *cp = cmd->param; @@ -5854,10 +5943,15 @@ static void fast_connectable_complete(struct hci_dev *hdev, u8 status, new_settings(hdev, cmd->sk); } - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); +} -unlock: - hci_dev_unlock(hdev); +static int write_fast_connectable_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + + return hci_write_fast_connectable_sync(hdev, cp->val); } static int set_fast_connectable(struct sock *sk, struct hci_dev *hdev, @@ -5865,58 +5959,49 @@ static int set_fast_connectable(struct sock *sk, struct hci_dev *hdev, { struct mgmt_mode *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); if (!hci_dev_test_flag(hdev, HCI_BREDR_ENABLED) || hdev->hci_ver < BLUETOOTH_VER_1_2) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_FAST_CONNECTABLE, MGMT_STATUS_NOT_SUPPORTED); if (cp->val != 0x00 && cp->val != 0x01) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, + return mgmt_cmd_status(sk, hdev->id, + MGMT_OP_SET_FAST_CONNECTABLE, MGMT_STATUS_INVALID_PARAMS); hci_dev_lock(hdev); - if (pending_find(MGMT_OP_SET_FAST_CONNECTABLE, hdev)) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, - MGMT_STATUS_BUSY); - goto unlock; - } - if (!!cp->val == hci_dev_test_flag(hdev, HCI_FAST_CONNECTABLE)) { - err = send_settings_rsp(sk, MGMT_OP_SET_FAST_CONNECTABLE, - hdev); + err = send_settings_rsp(sk, MGMT_OP_SET_FAST_CONNECTABLE, hdev); goto unlock; } if (!hdev_is_powered(hdev)) { hci_dev_change_flag(hdev, HCI_FAST_CONNECTABLE); - err = send_settings_rsp(sk, MGMT_OP_SET_FAST_CONNECTABLE, - hdev); + err = send_settings_rsp(sk, MGMT_OP_SET_FAST_CONNECTABLE, hdev); new_settings(hdev, sk); goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_SET_FAST_CONNECTABLE, hdev, - data, len); - if (!cmd) { + cmd = mgmt_pending_new(sk, MGMT_OP_SET_FAST_CONNECTABLE, hdev, data, + len); + if (!cmd) err = -ENOMEM; - goto unlock; - } - - hci_req_init(&req, hdev); - - __hci_req_write_fast_connectable(&req, cp->val); + else + err = hci_cmd_sync_queue(hdev, write_fast_connectable_sync, cmd, + fast_connectable_complete); - err = hci_req_run(&req, fast_connectable_complete); if (err < 0) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, - MGMT_STATUS_FAILED); - mgmt_pending_remove(cmd); + mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_FAST_CONNECTABLE, + MGMT_STATUS_FAILED); + + if (cmd) + mgmt_pending_free(cmd); } unlock: @@ -5925,20 +6010,14 @@ unlock: return err; } -static void set_bredr_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static void set_bredr_complete(struct hci_dev *hdev, void *data, int err) { - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; - bt_dev_dbg(hdev, "status 0x%02x", status); - - hci_dev_lock(hdev); - - cmd = pending_find(MGMT_OP_SET_BREDR, hdev); - if (!cmd) - goto unlock; + bt_dev_dbg(hdev, "err %d", err); - if (status) { - u8 mgmt_err = mgmt_status(status); + if (err) { + u8 mgmt_err = mgmt_status(err); /* We need to restore the flag if related HCI commands * failed. @@ -5951,17 +6030,31 @@ static void set_bredr_complete(struct hci_dev *hdev, u8 status, u16 opcode) new_settings(hdev, cmd->sk); } - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); +} -unlock: - hci_dev_unlock(hdev); +static int set_bredr_sync(struct hci_dev *hdev, void *data) +{ + int status; + + status = hci_write_fast_connectable_sync(hdev, false); + + if (!status) + status = hci_update_scan_sync(hdev); + + /* Since only the advertising data flags will change, there + * is no need to update the scan response data. + */ + if (!status) + status = hci_update_adv_data_sync(hdev, hdev->cur_adv_instance); + + return status; } static int set_bredr(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { struct mgmt_mode *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); @@ -6033,15 +6126,19 @@ static int set_bredr(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) } } - if (pending_find(MGMT_OP_SET_BREDR, hdev)) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_BREDR, - MGMT_STATUS_BUSY); - goto unlock; - } - - cmd = mgmt_pending_add(sk, MGMT_OP_SET_BREDR, hdev, data, len); - if (!cmd) { + cmd = mgmt_pending_new(sk, MGMT_OP_SET_BREDR, hdev, data, len); + if (!cmd) err = -ENOMEM; + else + err = hci_cmd_sync_queue(hdev, set_bredr_sync, cmd, + set_bredr_complete); + + if (err < 0) { + mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_BREDR, + MGMT_STATUS_FAILED); + if (cmd) + mgmt_pending_free(cmd); + goto unlock; } @@ -6050,42 +6147,23 @@ static int set_bredr(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) */ hci_dev_set_flag(hdev, HCI_BREDR_ENABLED); - hci_req_init(&req, hdev); - - __hci_req_write_fast_connectable(&req, false); - __hci_req_update_scan(&req); - - /* Since only the advertising data flags will change, there - * is no need to update the scan response data. - */ - __hci_req_update_adv_data(&req, hdev->cur_adv_instance); - - err = hci_req_run(&req, set_bredr_complete); - if (err < 0) - mgmt_pending_remove(cmd); - unlock: hci_dev_unlock(hdev); return err; } -static void sc_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static void set_secure_conn_complete(struct hci_dev *hdev, void *data, int err) { - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; struct mgmt_mode *cp; - bt_dev_dbg(hdev, "status %u", status); - - hci_dev_lock(hdev); + bt_dev_dbg(hdev, "err %d", err); - cmd = pending_find(MGMT_OP_SET_SECURE_CONN, hdev); - if (!cmd) - goto unlock; + if (err) { + u8 mgmt_err = mgmt_status(err); - if (status) { - mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status)); - goto remove; + mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, mgmt_err); + goto done; } cp = cmd->param; @@ -6105,13 +6183,23 @@ static void sc_enable_complete(struct hci_dev *hdev, u8 status, u16 opcode) break; } - send_settings_rsp(cmd->sk, MGMT_OP_SET_SECURE_CONN, hdev); + send_settings_rsp(cmd->sk, cmd->opcode, hdev); new_settings(hdev, cmd->sk); -remove: - mgmt_pending_remove(cmd); -unlock: - hci_dev_unlock(hdev); +done: + mgmt_pending_free(cmd); +} + +static int set_secure_conn_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_mode *cp = cmd->param; + u8 val = !!cp->val; + + /* Force write of val */ + hci_dev_set_flag(hdev, HCI_SC_ENABLED); + + return hci_write_sc_support_sync(hdev, val); } static int set_secure_conn(struct sock *sk, struct hci_dev *hdev, @@ -6119,7 +6207,6 @@ static int set_secure_conn(struct sock *sk, struct hci_dev *hdev, { struct mgmt_mode *cp = data; struct mgmt_pending_cmd *cmd; - struct hci_request req; u8 val; int err; @@ -6138,7 +6225,7 @@ static int set_secure_conn(struct sock *sk, struct hci_dev *hdev, if (cp->val != 0x00 && cp->val != 0x01 && cp->val != 0x02) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_SECURE_CONN, - MGMT_STATUS_INVALID_PARAMS); + MGMT_STATUS_INVALID_PARAMS); hci_dev_lock(hdev); @@ -6169,12 +6256,6 @@ static int set_secure_conn(struct sock *sk, struct hci_dev *hdev, goto failed; } - if (pending_find(MGMT_OP_SET_SECURE_CONN, hdev)) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_SECURE_CONN, - MGMT_STATUS_BUSY); - goto failed; - } - val = !!cp->val; if (val == hci_dev_test_flag(hdev, HCI_SC_ENABLED) && @@ -6183,18 +6264,18 @@ static int set_secure_conn(struct sock *sk, struct hci_dev *hdev, goto failed; } - cmd = mgmt_pending_add(sk, MGMT_OP_SET_SECURE_CONN, hdev, data, len); - if (!cmd) { + cmd = mgmt_pending_new(sk, MGMT_OP_SET_SECURE_CONN, hdev, data, len); + if (!cmd) err = -ENOMEM; - goto failed; - } + else + err = hci_cmd_sync_queue(hdev, set_secure_conn_sync, cmd, + set_secure_conn_complete); - hci_req_init(&req, hdev); - hci_req_add(&req, HCI_OP_WRITE_SC_SUPPORT, 1, &val); - err = hci_req_run(&req, sc_enable_complete); if (err < 0) { - mgmt_pending_remove(cmd); - goto failed; + mgmt_cmd_status(sk, hdev->id, MGMT_OP_SET_SECURE_CONN, + MGMT_STATUS_FAILED); + if (cmd) + mgmt_pending_free(cmd); } failed: @@ -6508,14 +6589,19 @@ static int load_long_term_keys(struct sock *sk, struct hci_dev *hdev, return err; } -static int conn_info_cmd_complete(struct mgmt_pending_cmd *cmd, u8 status) +static void get_conn_info_complete(struct hci_dev *hdev, void *data, int err) { + struct mgmt_pending_cmd *cmd = data; struct hci_conn *conn = cmd->user_data; + struct mgmt_cp_get_conn_info *cp = cmd->param; struct mgmt_rp_get_conn_info rp; - int err; + u8 status; - memcpy(&rp.addr, cmd->param, sizeof(rp.addr)); + bt_dev_dbg(hdev, "err %d", err); + + memcpy(&rp.addr, &cp->addr.bdaddr, sizeof(rp.addr)); + status = mgmt_status(err); if (status == MGMT_STATUS_SUCCESS) { rp.rssi = conn->rssi; rp.tx_power = conn->tx_power; @@ -6526,67 +6612,58 @@ static int conn_info_cmd_complete(struct mgmt_pending_cmd *cmd, u8 status) rp.max_tx_power = HCI_TX_POWER_INVALID; } - err = mgmt_cmd_complete(cmd->sk, cmd->index, MGMT_OP_GET_CONN_INFO, - status, &rp, sizeof(rp)); + mgmt_cmd_complete(cmd->sk, cmd->index, MGMT_OP_GET_CONN_INFO, status, + &rp, sizeof(rp)); - hci_conn_drop(conn); - hci_conn_put(conn); + if (conn) { + hci_conn_drop(conn); + hci_conn_put(conn); + } - return err; + mgmt_pending_free(cmd); } -static void conn_info_refresh_complete(struct hci_dev *hdev, u8 hci_status, - u16 opcode) +static int get_conn_info_sync(struct hci_dev *hdev, void *data) { - struct hci_cp_read_rssi *cp; - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_get_conn_info *cp = cmd->param; struct hci_conn *conn; - u16 handle; - u8 status; - - bt_dev_dbg(hdev, "status 0x%02x", hci_status); + int err; + __le16 handle; - hci_dev_lock(hdev); + /* Make sure we are still connected */ + if (cp->addr.type == BDADDR_BREDR) + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, + &cp->addr.bdaddr); + else + conn = hci_conn_hash_lookup_ba(hdev, LE_LINK, &cp->addr.bdaddr); - /* Commands sent in request are either Read RSSI or Read Transmit Power - * Level so we check which one was last sent to retrieve connection - * handle. Both commands have handle as first parameter so it's safe to - * cast data on the same command struct. - * - * First command sent is always Read RSSI and we fail only if it fails. - * In other case we simply override error to indicate success as we - * already remembered if TX power value is actually valid. - */ - cp = hci_sent_cmd_data(hdev, HCI_OP_READ_RSSI); - if (!cp) { - cp = hci_sent_cmd_data(hdev, HCI_OP_READ_TX_POWER); - status = MGMT_STATUS_SUCCESS; - } else { - status = mgmt_status(hci_status); + if (!conn || conn != cmd->user_data || conn->state != BT_CONNECTED) { + if (cmd->user_data) { + hci_conn_drop(cmd->user_data); + hci_conn_put(cmd->user_data); + cmd->user_data = NULL; + } + return MGMT_STATUS_NOT_CONNECTED; } - if (!cp) { - bt_dev_err(hdev, "invalid sent_cmd in conn_info response"); - goto unlock; - } + handle = cpu_to_le16(conn->handle); - handle = __le16_to_cpu(cp->handle); - conn = hci_conn_hash_lookup_handle(hdev, handle); - if (!conn) { - bt_dev_err(hdev, "unknown handle (%u) in conn_info response", - handle); - goto unlock; - } + /* Refresh RSSI each time */ + err = hci_read_rssi_sync(hdev, handle); - cmd = pending_find_data(MGMT_OP_GET_CONN_INFO, hdev, conn); - if (!cmd) - goto unlock; + /* For LE links TX power does not change thus we don't need to + * query for it once value is known. + */ + if (!err && (!bdaddr_type_is_le(cp->addr.type) || + conn->tx_power == HCI_TX_POWER_INVALID)) + err = hci_read_tx_power_sync(hdev, handle, 0x00); - cmd->cmd_complete(cmd, status); - mgmt_pending_remove(cmd); + /* Max TX power needs to be read only once per connection */ + if (!err && conn->max_tx_power == HCI_TX_POWER_INVALID) + err = hci_read_tx_power_sync(hdev, handle, 0x01); -unlock: - hci_dev_unlock(hdev); + return err; } static int get_conn_info(struct sock *sk, struct hci_dev *hdev, void *data, @@ -6631,12 +6708,6 @@ static int get_conn_info(struct sock *sk, struct hci_dev *hdev, void *data, goto unlock; } - if (pending_find_data(MGMT_OP_GET_CONN_INFO, hdev, conn)) { - err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_GET_CONN_INFO, - MGMT_STATUS_BUSY, &rp, sizeof(rp)); - goto unlock; - } - /* To avoid client trying to guess when to poll again for information we * calculate conn info age as random value between min/max set in hdev. */ @@ -6650,49 +6721,28 @@ static int get_conn_info(struct sock *sk, struct hci_dev *hdev, void *data, if (time_after(jiffies, conn->conn_info_timestamp + msecs_to_jiffies(conn_info_age)) || !conn->conn_info_timestamp) { - struct hci_request req; - struct hci_cp_read_tx_power req_txp_cp; - struct hci_cp_read_rssi req_rssi_cp; struct mgmt_pending_cmd *cmd; - hci_req_init(&req, hdev); - req_rssi_cp.handle = cpu_to_le16(conn->handle); - hci_req_add(&req, HCI_OP_READ_RSSI, sizeof(req_rssi_cp), - &req_rssi_cp); - - /* For LE links TX power does not change thus we don't need to - * query for it once value is known. - */ - if (!bdaddr_type_is_le(cp->addr.type) || - conn->tx_power == HCI_TX_POWER_INVALID) { - req_txp_cp.handle = cpu_to_le16(conn->handle); - req_txp_cp.type = 0x00; - hci_req_add(&req, HCI_OP_READ_TX_POWER, - sizeof(req_txp_cp), &req_txp_cp); - } + cmd = mgmt_pending_new(sk, MGMT_OP_GET_CONN_INFO, hdev, data, + len); + if (!cmd) + err = -ENOMEM; + else + err = hci_cmd_sync_queue(hdev, get_conn_info_sync, + cmd, get_conn_info_complete); - /* Max TX power needs to be read only once per connection */ - if (conn->max_tx_power == HCI_TX_POWER_INVALID) { - req_txp_cp.handle = cpu_to_le16(conn->handle); - req_txp_cp.type = 0x01; - hci_req_add(&req, HCI_OP_READ_TX_POWER, - sizeof(req_txp_cp), &req_txp_cp); - } + if (err < 0) { + mgmt_cmd_complete(sk, hdev->id, MGMT_OP_GET_CONN_INFO, + MGMT_STATUS_FAILED, &rp, sizeof(rp)); - err = hci_req_run(&req, conn_info_refresh_complete); - if (err < 0) - goto unlock; + if (cmd) + mgmt_pending_free(cmd); - cmd = mgmt_pending_add(sk, MGMT_OP_GET_CONN_INFO, hdev, - data, len); - if (!cmd) { - err = -ENOMEM; goto unlock; } hci_conn_hold(conn); cmd->user_data = hci_conn_get(conn); - cmd->cmd_complete = conn_info_cmd_complete; conn->conn_info_timestamp = jiffies; } else { @@ -6710,82 +6760,76 @@ unlock: return err; } -static int clock_info_cmd_complete(struct mgmt_pending_cmd *cmd, u8 status) +static void get_clock_info_complete(struct hci_dev *hdev, void *data, int err) { - struct hci_conn *conn = cmd->user_data; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_get_clock_info *cp = cmd->param; struct mgmt_rp_get_clock_info rp; - struct hci_dev *hdev; - int err; + struct hci_conn *conn = cmd->user_data; + u8 status = mgmt_status(err); + + bt_dev_dbg(hdev, "err %d", err); memset(&rp, 0, sizeof(rp)); - memcpy(&rp.addr, cmd->param, sizeof(rp.addr)); + bacpy(&rp.addr.bdaddr, &cp->addr.bdaddr); + rp.addr.type = cp->addr.type; - if (status) + if (err) goto complete; - hdev = hci_dev_get(cmd->index); - if (hdev) { - rp.local_clock = cpu_to_le32(hdev->clock); - hci_dev_put(hdev); - } + rp.local_clock = cpu_to_le32(hdev->clock); if (conn) { rp.piconet_clock = cpu_to_le32(conn->clock); rp.accuracy = cpu_to_le16(conn->clock_accuracy); - } - -complete: - err = mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, status, &rp, - sizeof(rp)); - - if (conn) { hci_conn_drop(conn); hci_conn_put(conn); } - return err; +complete: + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, status, &rp, + sizeof(rp)); + + mgmt_pending_free(cmd); } -static void get_clock_info_complete(struct hci_dev *hdev, u8 status, u16 opcode) +static int get_clock_info_sync(struct hci_dev *hdev, void *data) { - struct hci_cp_read_clock *hci_cp; - struct mgmt_pending_cmd *cmd; - struct hci_conn *conn; - - bt_dev_dbg(hdev, "status %u", status); + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_get_clock_info *cp = cmd->param; + struct hci_cp_read_clock hci_cp; + struct hci_conn *conn = cmd->user_data; + int err; - hci_dev_lock(hdev); + memset(&hci_cp, 0, sizeof(hci_cp)); + err = hci_read_clock_sync(hdev, &hci_cp); - hci_cp = hci_sent_cmd_data(hdev, HCI_OP_READ_CLOCK); - if (!hci_cp) - goto unlock; + if (conn) { + /* Make sure connection still exists */ + conn = hci_conn_hash_lookup_ba(hdev, ACL_LINK, + &cp->addr.bdaddr); - if (hci_cp->which) { - u16 handle = __le16_to_cpu(hci_cp->handle); - conn = hci_conn_hash_lookup_handle(hdev, handle); - } else { - conn = NULL; + if (conn && conn == cmd->user_data && + conn->state == BT_CONNECTED) { + hci_cp.handle = cpu_to_le16(conn->handle); + hci_cp.which = 0x01; /* Piconet clock */ + err = hci_read_clock_sync(hdev, &hci_cp); + } else if (cmd->user_data) { + hci_conn_drop(cmd->user_data); + hci_conn_put(cmd->user_data); + cmd->user_data = NULL; + } } - cmd = pending_find_data(MGMT_OP_GET_CLOCK_INFO, hdev, conn); - if (!cmd) - goto unlock; - - cmd->cmd_complete(cmd, mgmt_status(status)); - mgmt_pending_remove(cmd); - -unlock: - hci_dev_unlock(hdev); + return err; } static int get_clock_info(struct sock *sk, struct hci_dev *hdev, void *data, - u16 len) + u16 len) { struct mgmt_cp_get_clock_info *cp = data; struct mgmt_rp_get_clock_info rp; - struct hci_cp_read_clock hci_cp; struct mgmt_pending_cmd *cmd; - struct hci_request req; struct hci_conn *conn; int err; @@ -6823,31 +6867,25 @@ static int get_clock_info(struct sock *sk, struct hci_dev *hdev, void *data, conn = NULL; } - cmd = mgmt_pending_add(sk, MGMT_OP_GET_CLOCK_INFO, hdev, data, len); - if (!cmd) { + cmd = mgmt_pending_new(sk, MGMT_OP_GET_CLOCK_INFO, hdev, data, len); + if (!cmd) err = -ENOMEM; - goto unlock; - } - - cmd->cmd_complete = clock_info_cmd_complete; + else + err = hci_cmd_sync_queue(hdev, get_clock_info_sync, cmd, + get_clock_info_complete); - hci_req_init(&req, hdev); + if (err < 0) { + err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_GET_CLOCK_INFO, + MGMT_STATUS_FAILED, &rp, sizeof(rp)); - memset(&hci_cp, 0, sizeof(hci_cp)); - hci_req_add(&req, HCI_OP_READ_CLOCK, sizeof(hci_cp), &hci_cp); + if (cmd) + mgmt_pending_free(cmd); - if (conn) { + } else if (conn) { hci_conn_hold(conn); cmd->user_data = hci_conn_get(conn); - - hci_cp.handle = cpu_to_le16(conn->handle); - hci_cp.which = 0x01; /* Piconet clock */ - hci_req_add(&req, HCI_OP_READ_CLOCK, sizeof(hci_cp), &hci_cp); } - err = hci_req_run(&req, get_clock_info_complete); - if (err < 0) - mgmt_pending_remove(cmd); unlock: hci_dev_unlock(hdev); @@ -6928,6 +6966,11 @@ static void device_added(struct sock *sk, struct hci_dev *hdev, mgmt_event(MGMT_EV_DEVICE_ADDED, hdev, &ev, sizeof(ev), sk); } +static int add_device_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_passive_scan_sync(hdev); +} + static int add_device(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -7010,7 +7053,9 @@ static int add_device(struct sock *sk, struct hci_dev *hdev, current_flags = params->current_flags; } - hci_update_background_scan(hdev); + err = hci_cmd_sync_queue(hdev, add_device_sync, NULL, NULL); + if (err < 0) + goto unlock; added: device_added(sk, hdev, &cp->addr.bdaddr, cp->addr.type, cp->action); @@ -7037,6 +7082,11 @@ static void device_removed(struct sock *sk, struct hci_dev *hdev, mgmt_event(MGMT_EV_DEVICE_REMOVED, hdev, &ev, sizeof(ev), sk); } +static int remove_device_sync(struct hci_dev *hdev, void *data) +{ + return hci_update_passive_scan_sync(hdev); +} + static int remove_device(struct sock *sk, struct hci_dev *hdev, void *data, u16 len) { @@ -7116,7 +7166,6 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev, list_del(¶ms->action); list_del(¶ms->list); kfree(params); - hci_update_background_scan(hdev); device_removed(sk, hdev, &cp->addr.bdaddr, cp->addr.type); } else { @@ -7153,10 +7202,10 @@ static int remove_device(struct sock *sk, struct hci_dev *hdev, } bt_dev_dbg(hdev, "All LE connection parameters were removed"); - - hci_update_background_scan(hdev); } + hci_cmd_sync_queue(hdev, remove_device_sync, NULL, NULL); + complete: err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_REMOVE_DEVICE, MGMT_STATUS_SUCCESS, &cp->addr, @@ -7359,21 +7408,27 @@ unlock: return err; } -static void read_local_oob_ext_data_complete(struct hci_dev *hdev, u8 status, - u16 opcode, struct sk_buff *skb) +static void read_local_oob_ext_data_complete(struct hci_dev *hdev, void *data, + int err) { const struct mgmt_cp_read_local_oob_ext_data *mgmt_cp; struct mgmt_rp_read_local_oob_ext_data *mgmt_rp; u8 *h192, *r192, *h256, *r256; - struct mgmt_pending_cmd *cmd; + struct mgmt_pending_cmd *cmd = data; + struct sk_buff *skb = cmd->skb; + u8 status = mgmt_status(err); u16 eir_len; - int err; - bt_dev_dbg(hdev, "status %u", status); + if (!status) { + if (!skb) + status = MGMT_STATUS_FAILED; + else if (IS_ERR(skb)) + status = mgmt_status(PTR_ERR(skb)); + else + status = mgmt_status(skb->data[0]); + } - cmd = pending_find(MGMT_OP_READ_LOCAL_OOB_EXT_DATA, hdev); - if (!cmd) - return; + bt_dev_dbg(hdev, "status %u", status); mgmt_cp = cmd->param; @@ -7385,7 +7440,7 @@ static void read_local_oob_ext_data_complete(struct hci_dev *hdev, u8 status, r192 = NULL; h256 = NULL; r256 = NULL; - } else if (opcode == HCI_OP_READ_LOCAL_OOB_DATA) { + } else if (!bredr_sc_enabled(hdev)) { struct hci_rp_read_local_oob_data *rp; if (skb->len != sizeof(*rp)) { @@ -7466,6 +7521,9 @@ send_rsp: mgmt_rp, sizeof(*mgmt_rp) + eir_len, HCI_MGMT_OOB_DATA_EVENTS, cmd->sk); done: + if (skb && !IS_ERR(skb)) + kfree_skb(skb); + kfree(mgmt_rp); mgmt_pending_remove(cmd); } @@ -7474,7 +7532,6 @@ static int read_local_ssp_oob_req(struct hci_dev *hdev, struct sock *sk, struct mgmt_cp_read_local_oob_ext_data *cp) { struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; cmd = mgmt_pending_add(sk, MGMT_OP_READ_LOCAL_OOB_EXT_DATA, hdev, @@ -7482,14 +7539,9 @@ static int read_local_ssp_oob_req(struct hci_dev *hdev, struct sock *sk, if (!cmd) return -ENOMEM; - hci_req_init(&req, hdev); - - if (bredr_sc_enabled(hdev)) - hci_req_add(&req, HCI_OP_READ_LOCAL_OOB_EXT_DATA, 0, NULL); - else - hci_req_add(&req, HCI_OP_READ_LOCAL_OOB_DATA, 0, NULL); + err = hci_cmd_sync_queue(hdev, read_local_oob_data_sync, cmd, + read_local_oob_ext_data_complete); - err = hci_req_run_skb(&req, read_local_oob_ext_data_complete); if (err < 0) { mgmt_pending_remove(cmd); return err; @@ -7713,13 +7765,6 @@ static int read_adv_features(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_READ_ADV_FEATURES, MGMT_STATUS_REJECTED); - /* Enabling the experimental LL Privay support disables support for - * advertising. - */ - if (hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_READ_ADV_FEATURES, - MGMT_STATUS_NOT_SUPPORTED); - hci_dev_lock(hdev); rp_len = sizeof(*rp) + hdev->adv_instance_cnt; @@ -7876,58 +7921,66 @@ static bool adv_busy(struct hci_dev *hdev) pending_find(MGMT_OP_ADD_EXT_ADV_DATA, hdev)); } -static void add_advertising_complete(struct hci_dev *hdev, u8 status, - u16 opcode) +static void add_adv_complete(struct hci_dev *hdev, struct sock *sk, u8 instance, + int err) { - struct mgmt_pending_cmd *cmd; - struct mgmt_cp_add_advertising *cp; - struct mgmt_rp_add_advertising rp; - struct adv_info *adv_instance, *n; - u8 instance; + struct adv_info *adv, *n; - bt_dev_dbg(hdev, "status %u", status); + bt_dev_dbg(hdev, "err %d", err); hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_ADD_ADVERTISING, hdev); - if (!cmd) - cmd = pending_find(MGMT_OP_ADD_EXT_ADV_DATA, hdev); + list_for_each_entry_safe(adv, n, &hdev->adv_instances, list) { + u8 instance; - list_for_each_entry_safe(adv_instance, n, &hdev->adv_instances, list) { - if (!adv_instance->pending) + if (!adv->pending) continue; - if (!status) { - adv_instance->pending = false; + if (!err) { + adv->pending = false; continue; } - instance = adv_instance->instance; + instance = adv->instance; if (hdev->cur_adv_instance == instance) cancel_adv_timeout(hdev); hci_remove_adv_instance(hdev, instance); - mgmt_advertising_removed(cmd ? cmd->sk : NULL, hdev, instance); + mgmt_advertising_removed(sk, hdev, instance); } - if (!cmd) - goto unlock; + hci_dev_unlock(hdev); +} + +static void add_advertising_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_advertising *cp = cmd->param; + struct mgmt_rp_add_advertising rp; + + memset(&rp, 0, sizeof(rp)); - cp = cmd->param; rp.instance = cp->instance; - if (status) + if (err) mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status)); + mgmt_status(err)); else mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status), &rp, sizeof(rp)); + mgmt_status(err), &rp, sizeof(rp)); - mgmt_pending_remove(cmd); + add_adv_complete(hdev, cmd->sk, cp->instance, err); -unlock: - hci_dev_unlock(hdev); + mgmt_pending_free(cmd); +} + +static int add_advertising_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_advertising *cp = cmd->param; + + return hci_schedule_adv_instance_sync(hdev, cp->instance, true); } static int add_advertising(struct sock *sk, struct hci_dev *hdev, @@ -7943,7 +7996,6 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, struct adv_info *next_instance; int err; struct mgmt_pending_cmd *cmd; - struct hci_request req; bt_dev_dbg(hdev, "sock %p", sk); @@ -7952,13 +8004,6 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, status); - /* Enabling the experimental LL Privay support disables support for - * advertising. - */ - if (hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, - MGMT_STATUS_NOT_SUPPORTED); - if (cp->instance < 1 || cp->instance > hdev->le_num_of_adv_sets) return mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, MGMT_STATUS_INVALID_PARAMS); @@ -8051,25 +8096,19 @@ static int add_advertising(struct sock *sk, struct hci_dev *hdev, /* We're good to go, update advertising data, parameters, and start * advertising. */ - cmd = mgmt_pending_add(sk, MGMT_OP_ADD_ADVERTISING, hdev, data, + cmd = mgmt_pending_new(sk, MGMT_OP_ADD_ADVERTISING, hdev, data, data_len); if (!cmd) { err = -ENOMEM; goto unlock; } - hci_req_init(&req, hdev); - - err = __hci_req_schedule_adv_instance(&req, schedule_instance, true); - - if (!err) - err = hci_req_run(&req, add_advertising_complete); + cp->instance = schedule_instance; - if (err < 0) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_ADVERTISING, - MGMT_STATUS_FAILED); - mgmt_pending_remove(cmd); - } + err = hci_cmd_sync_queue(hdev, add_advertising_sync, cmd, + add_advertising_complete); + if (err < 0) + mgmt_pending_free(cmd); unlock: hci_dev_unlock(hdev); @@ -8077,30 +8116,25 @@ unlock: return err; } -static void add_ext_adv_params_complete(struct hci_dev *hdev, u8 status, - u16 opcode) +static void add_ext_adv_params_complete(struct hci_dev *hdev, void *data, + int err) { - struct mgmt_pending_cmd *cmd; - struct mgmt_cp_add_ext_adv_params *cp; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_ext_adv_params *cp = cmd->param; struct mgmt_rp_add_ext_adv_params rp; - struct adv_info *adv_instance; + struct adv_info *adv; u32 flags; BT_DBG("%s", hdev->name); hci_dev_lock(hdev); - cmd = pending_find(MGMT_OP_ADD_EXT_ADV_PARAMS, hdev); - if (!cmd) - goto unlock; - - cp = cmd->param; - adv_instance = hci_find_adv_instance(hdev, cp->instance); - if (!adv_instance) + adv = hci_find_adv_instance(hdev, cp->instance); + if (!adv) goto unlock; rp.instance = cp->instance; - rp.tx_power = adv_instance->tx_power; + rp.tx_power = adv->tx_power; /* While we're at it, inform userspace of the available space for this * advertisement, given the flags that will be used. @@ -8109,39 +8143,44 @@ static void add_ext_adv_params_complete(struct hci_dev *hdev, u8 status, rp.max_adv_data_len = tlv_data_max_len(hdev, flags, true); rp.max_scan_rsp_len = tlv_data_max_len(hdev, flags, false); - if (status) { + if (err) { /* If this advertisement was previously advertising and we * failed to update it, we signal that it has been removed and * delete its structure */ - if (!adv_instance->pending) + if (!adv->pending) mgmt_advertising_removed(cmd->sk, hdev, cp->instance); hci_remove_adv_instance(hdev, cp->instance); mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status)); - + mgmt_status(err)); } else { mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, - mgmt_status(status), &rp, sizeof(rp)); + mgmt_status(err), &rp, sizeof(rp)); } unlock: if (cmd) - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); hci_dev_unlock(hdev); } +static int add_ext_adv_params_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_ext_adv_params *cp = cmd->param; + + return hci_setup_ext_adv_instance_sync(hdev, cp->instance); +} + static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { struct mgmt_cp_add_ext_adv_params *cp = data; struct mgmt_rp_add_ext_adv_params rp; struct mgmt_pending_cmd *cmd = NULL; - struct adv_info *adv_instance; - struct hci_request req; u32 flags, min_interval, max_interval; u16 timeout, duration; u8 status; @@ -8223,29 +8262,18 @@ static int add_ext_adv_params(struct sock *sk, struct hci_dev *hdev, /* Submit request for advertising params if ext adv available */ if (ext_adv_capable(hdev)) { - hci_req_init(&req, hdev); - adv_instance = hci_find_adv_instance(hdev, cp->instance); - - /* Updating parameters of an active instance will return a - * Command Disallowed error, so we must first disable the - * instance if it is active. - */ - if (!adv_instance->pending) - __hci_req_disable_ext_adv_instance(&req, cp->instance); - - __hci_req_setup_ext_adv_instance(&req, cp->instance); - - err = hci_req_run(&req, add_ext_adv_params_complete); - - if (!err) - cmd = mgmt_pending_add(sk, MGMT_OP_ADD_EXT_ADV_PARAMS, - hdev, data, data_len); + cmd = mgmt_pending_new(sk, MGMT_OP_ADD_EXT_ADV_PARAMS, hdev, + data, data_len); if (!cmd) { err = -ENOMEM; hci_remove_adv_instance(hdev, cp->instance); goto unlock; } + err = hci_cmd_sync_queue(hdev, add_ext_adv_params_sync, cmd, + add_ext_adv_params_complete); + if (err < 0) + mgmt_pending_free(cmd); } else { rp.instance = cp->instance; rp.tx_power = HCI_ADV_TX_POWER_NO_PREFERENCE; @@ -8262,6 +8290,49 @@ unlock: return err; } +static void add_ext_adv_data_complete(struct hci_dev *hdev, void *data, int err) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_ext_adv_data *cp = cmd->param; + struct mgmt_rp_add_advertising rp; + + add_adv_complete(hdev, cmd->sk, cp->instance, err); + + memset(&rp, 0, sizeof(rp)); + + rp.instance = cp->instance; + + if (err) + mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, + mgmt_status(err)); + else + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, + mgmt_status(err), &rp, sizeof(rp)); + + mgmt_pending_free(cmd); +} + +static int add_ext_adv_data_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_add_ext_adv_data *cp = cmd->param; + int err; + + if (ext_adv_capable(hdev)) { + err = hci_update_adv_data_sync(hdev, cp->instance); + if (err) + return err; + + err = hci_update_scan_rsp_data_sync(hdev, cp->instance); + if (err) + return err; + + return hci_enable_ext_advertising_sync(hdev, cp->instance); + } + + return hci_schedule_adv_instance_sync(hdev, cp->instance, true); +} + static int add_ext_adv_data(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { @@ -8272,7 +8343,6 @@ static int add_ext_adv_data(struct sock *sk, struct hci_dev *hdev, void *data, struct adv_info *adv_instance; int err = 0; struct mgmt_pending_cmd *cmd; - struct hci_request req; BT_DBG("%s", hdev->name); @@ -8314,78 +8384,52 @@ static int add_ext_adv_data(struct sock *sk, struct hci_dev *hdev, void *data, cp->data, cp->scan_rsp_len, cp->data + cp->adv_data_len); - /* We're good to go, update advertising data, parameters, and start - * advertising. - */ - - hci_req_init(&req, hdev); - - hci_req_add(&req, HCI_OP_READ_LOCAL_NAME, 0, NULL); - - if (ext_adv_capable(hdev)) { - __hci_req_update_adv_data(&req, cp->instance); - __hci_req_update_scan_rsp_data(&req, cp->instance); - __hci_req_enable_ext_advertising(&req, cp->instance); - - } else { - /* If using software rotation, determine next instance to use */ - - if (hdev->cur_adv_instance == cp->instance) { - /* If the currently advertised instance is being changed - * then cancel the current advertising and schedule the - * next instance. If there is only one instance then the - * overridden advertising data will be visible right - * away - */ - cancel_adv_timeout(hdev); - - next_instance = hci_get_next_instance(hdev, - cp->instance); - if (next_instance) - schedule_instance = next_instance->instance; - } else if (!hdev->adv_instance_timeout) { - /* Immediately advertise the new instance if no other - * instance is currently being advertised. - */ - schedule_instance = cp->instance; - } + /* If using software rotation, determine next instance to use */ + if (hdev->cur_adv_instance == cp->instance) { + /* If the currently advertised instance is being changed + * then cancel the current advertising and schedule the + * next instance. If there is only one instance then the + * overridden advertising data will be visible right + * away + */ + cancel_adv_timeout(hdev); - /* If the HCI_ADVERTISING flag is set or there is no instance to - * be advertised then we have no HCI communication to make. - * Simply return. + next_instance = hci_get_next_instance(hdev, cp->instance); + if (next_instance) + schedule_instance = next_instance->instance; + } else if (!hdev->adv_instance_timeout) { + /* Immediately advertise the new instance if no other + * instance is currently being advertised. */ - if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || - !schedule_instance) { - if (adv_instance->pending) { - mgmt_advertising_added(sk, hdev, cp->instance); - adv_instance->pending = false; - } - rp.instance = cp->instance; - err = mgmt_cmd_complete(sk, hdev->id, - MGMT_OP_ADD_EXT_ADV_DATA, - MGMT_STATUS_SUCCESS, &rp, - sizeof(rp)); - goto unlock; - } + schedule_instance = cp->instance; + } - err = __hci_req_schedule_adv_instance(&req, schedule_instance, - true); + /* If the HCI_ADVERTISING flag is set or there is no instance to + * be advertised then we have no HCI communication to make. + * Simply return. + */ + if (hci_dev_test_flag(hdev, HCI_ADVERTISING) || !schedule_instance) { + if (adv_instance->pending) { + mgmt_advertising_added(sk, hdev, cp->instance); + adv_instance->pending = false; + } + rp.instance = cp->instance; + err = mgmt_cmd_complete(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_DATA, + MGMT_STATUS_SUCCESS, &rp, sizeof(rp)); + goto unlock; } - cmd = mgmt_pending_add(sk, MGMT_OP_ADD_EXT_ADV_DATA, hdev, data, + cmd = mgmt_pending_new(sk, MGMT_OP_ADD_EXT_ADV_DATA, hdev, data, data_len); if (!cmd) { err = -ENOMEM; goto clear_new_instance; } - if (!err) - err = hci_req_run(&req, add_advertising_complete); - + err = hci_cmd_sync_queue(hdev, add_ext_adv_data_sync, cmd, + add_ext_adv_data_complete); if (err < 0) { - err = mgmt_cmd_status(sk, hdev->id, MGMT_OP_ADD_EXT_ADV_DATA, - MGMT_STATUS_FAILED); - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); goto clear_new_instance; } @@ -8408,54 +8452,53 @@ unlock: return err; } -static void remove_advertising_complete(struct hci_dev *hdev, u8 status, - u16 opcode) +static void remove_advertising_complete(struct hci_dev *hdev, void *data, + int err) { - struct mgmt_pending_cmd *cmd; - struct mgmt_cp_remove_advertising *cp; + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_remove_advertising *cp = cmd->param; struct mgmt_rp_remove_advertising rp; - bt_dev_dbg(hdev, "status %u", status); + bt_dev_dbg(hdev, "err %d", err); - hci_dev_lock(hdev); + memset(&rp, 0, sizeof(rp)); + rp.instance = cp->instance; - /* A failure status here only means that we failed to disable - * advertising. Otherwise, the advertising instance has been removed, - * so report success. - */ - cmd = pending_find(MGMT_OP_REMOVE_ADVERTISING, hdev); - if (!cmd) - goto unlock; + if (err) + mgmt_cmd_status(cmd->sk, cmd->index, cmd->opcode, + mgmt_status(err)); + else + mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, + MGMT_STATUS_SUCCESS, &rp, sizeof(rp)); - cp = cmd->param; - rp.instance = cp->instance; + mgmt_pending_free(cmd); +} - mgmt_cmd_complete(cmd->sk, cmd->index, cmd->opcode, MGMT_STATUS_SUCCESS, - &rp, sizeof(rp)); - mgmt_pending_remove(cmd); +static int remove_advertising_sync(struct hci_dev *hdev, void *data) +{ + struct mgmt_pending_cmd *cmd = data; + struct mgmt_cp_remove_advertising *cp = cmd->param; + int err; -unlock: - hci_dev_unlock(hdev); + err = hci_remove_advertising_sync(hdev, cmd->sk, cp->instance, true); + if (err) + return err; + + if (list_empty(&hdev->adv_instances)) + err = hci_disable_advertising_sync(hdev); + + return err; } static int remove_advertising(struct sock *sk, struct hci_dev *hdev, void *data, u16 data_len) { struct mgmt_cp_remove_advertising *cp = data; - struct mgmt_rp_remove_advertising rp; struct mgmt_pending_cmd *cmd; - struct hci_request req; int err; bt_dev_dbg(hdev, "sock %p", sk); - /* Enabling the experimental LL Privay support disables support for - * advertising. - */ - if (hci_dev_test_flag(hdev, HCI_ENABLE_LL_PRIVACY)) - return mgmt_cmd_status(sk, hdev->id, MGMT_OP_REMOVE_ADVERTISING, - MGMT_STATUS_NOT_SUPPORTED); - hci_dev_lock(hdev); if (cp->instance && !hci_find_adv_instance(hdev, cp->instance)) { @@ -8479,44 +8522,17 @@ static int remove_advertising(struct sock *sk, struct hci_dev *hdev, goto unlock; } - hci_req_init(&req, hdev); - - /* If we use extended advertising, instance is disabled and removed */ - if (ext_adv_capable(hdev)) { - __hci_req_disable_ext_adv_instance(&req, cp->instance); - __hci_req_remove_ext_adv_instance(&req, cp->instance); - } - - hci_req_clear_adv_instance(hdev, sk, &req, cp->instance, true); - - if (list_empty(&hdev->adv_instances)) - __hci_req_disable_advertising(&req); - - /* If no HCI commands have been collected so far or the HCI_ADVERTISING - * flag is set or the device isn't powered then we have no HCI - * communication to make. Simply return. - */ - if (skb_queue_empty(&req.cmd_q) || - !hdev_is_powered(hdev) || - hci_dev_test_flag(hdev, HCI_ADVERTISING)) { - hci_req_purge(&req); - rp.instance = cp->instance; - err = mgmt_cmd_complete(sk, hdev->id, - MGMT_OP_REMOVE_ADVERTISING, - MGMT_STATUS_SUCCESS, &rp, sizeof(rp)); - goto unlock; - } - - cmd = mgmt_pending_add(sk, MGMT_OP_REMOVE_ADVERTISING, hdev, data, + cmd = mgmt_pending_new(sk, MGMT_OP_REMOVE_ADVERTISING, hdev, data, data_len); if (!cmd) { err = -ENOMEM; goto unlock; } - err = hci_req_run(&req, remove_advertising_complete); + err = hci_cmd_sync_queue(hdev, remove_advertising_sync, cmd, + remove_advertising_complete); if (err < 0) - mgmt_pending_remove(cmd); + mgmt_pending_free(cmd); unlock: hci_dev_unlock(hdev); @@ -8758,31 +8774,6 @@ void mgmt_index_removed(struct hci_dev *hdev) HCI_MGMT_EXT_INDEX_EVENTS); } -/* This function requires the caller holds hdev->lock */ -static void restart_le_actions(struct hci_dev *hdev) -{ - struct hci_conn_params *p; - - list_for_each_entry(p, &hdev->le_conn_params, list) { - /* Needed for AUTO_OFF case where might not "really" - * have been powered off. - */ - list_del_init(&p->action); - - switch (p->auto_connect) { - case HCI_AUTO_CONN_DIRECT: - case HCI_AUTO_CONN_ALWAYS: - list_add(&p->action, &hdev->pend_le_conns); - break; - case HCI_AUTO_CONN_REPORT: - list_add(&p->action, &hdev->pend_le_reports); - break; - default: - break; - } - } -} - void mgmt_power_on(struct hci_dev *hdev, int err) { struct cmd_lookup match = { NULL, hdev }; @@ -8793,7 +8784,7 @@ void mgmt_power_on(struct hci_dev *hdev, int err) if (!err) { restart_le_actions(hdev); - hci_update_background_scan(hdev); + hci_update_passive_scan(hdev); } mgmt_pending_foreach(MGMT_OP_SET_POWERED, hdev, settings_rsp, &match); @@ -9349,74 +9340,6 @@ void mgmt_auth_enable_complete(struct hci_dev *hdev, u8 status) sock_put(match.sk); } -static void clear_eir(struct hci_request *req) -{ - struct hci_dev *hdev = req->hdev; - struct hci_cp_write_eir cp; - - if (!lmp_ext_inq_capable(hdev)) - return; - - memset(hdev->eir, 0, sizeof(hdev->eir)); - - memset(&cp, 0, sizeof(cp)); - - hci_req_add(req, HCI_OP_WRITE_EIR, sizeof(cp), &cp); -} - -void mgmt_ssp_enable_complete(struct hci_dev *hdev, u8 enable, u8 status) -{ - struct cmd_lookup match = { NULL, hdev }; - struct hci_request req; - bool changed = false; - - if (status) { - u8 mgmt_err = mgmt_status(status); - - if (enable && hci_dev_test_and_clear_flag(hdev, - HCI_SSP_ENABLED)) { - hci_dev_clear_flag(hdev, HCI_HS_ENABLED); - new_settings(hdev, NULL); - } - - mgmt_pending_foreach(MGMT_OP_SET_SSP, hdev, cmd_status_rsp, - &mgmt_err); - return; - } - - if (enable) { - changed = !hci_dev_test_and_set_flag(hdev, HCI_SSP_ENABLED); - } else { - changed = hci_dev_test_and_clear_flag(hdev, HCI_SSP_ENABLED); - if (!changed) - changed = hci_dev_test_and_clear_flag(hdev, - HCI_HS_ENABLED); - else - hci_dev_clear_flag(hdev, HCI_HS_ENABLED); - } - - mgmt_pending_foreach(MGMT_OP_SET_SSP, hdev, settings_rsp, &match); - - if (changed) - new_settings(hdev, match.sk); - - if (match.sk) - sock_put(match.sk); - - hci_req_init(&req, hdev); - - if (hci_dev_test_flag(hdev, HCI_SSP_ENABLED)) { - if (hci_dev_test_flag(hdev, HCI_USE_DEBUG_KEYS)) - hci_req_add(&req, HCI_OP_WRITE_SSP_DEBUG_MODE, - sizeof(enable), &enable); - __hci_req_update_eir(&req); - } else { - clear_eir(&req); - } - - hci_req_run(&req, NULL); -} - static void sk_lookup(struct mgmt_pending_cmd *cmd, void *data) { struct cmd_lookup *match = data; diff --git a/net/bluetooth/mgmt_util.c b/net/bluetooth/mgmt_util.c index 0d0a6d77b9e8..83875f2a0604 100644 --- a/net/bluetooth/mgmt_util.c +++ b/net/bluetooth/mgmt_util.c @@ -227,7 +227,7 @@ void mgmt_pending_foreach(u16 opcode, struct hci_dev *hdev, } } -struct mgmt_pending_cmd *mgmt_pending_add(struct sock *sk, u16 opcode, +struct mgmt_pending_cmd *mgmt_pending_new(struct sock *sk, u16 opcode, struct hci_dev *hdev, void *data, u16 len) { @@ -251,6 +251,19 @@ struct mgmt_pending_cmd *mgmt_pending_add(struct sock *sk, u16 opcode, cmd->sk = sk; sock_hold(sk); + return cmd; +} + +struct mgmt_pending_cmd *mgmt_pending_add(struct sock *sk, u16 opcode, + struct hci_dev *hdev, + void *data, u16 len) +{ + struct mgmt_pending_cmd *cmd; + + cmd = mgmt_pending_new(sk, opcode, hdev, data, len); + if (!cmd) + return NULL; + list_add(&cmd->list, &hdev->mgmt_pending); return cmd; diff --git a/net/bluetooth/mgmt_util.h b/net/bluetooth/mgmt_util.h index 6559f189213c..63b965eaaaac 100644 --- a/net/bluetooth/mgmt_util.h +++ b/net/bluetooth/mgmt_util.h @@ -27,6 +27,7 @@ struct mgmt_pending_cmd { void *param; size_t param_len; struct sock *sk; + struct sk_buff *skb; void *user_data; int (*cmd_complete)(struct mgmt_pending_cmd *cmd, u8 status); }; @@ -49,5 +50,8 @@ void mgmt_pending_foreach(u16 opcode, struct hci_dev *hdev, struct mgmt_pending_cmd *mgmt_pending_add(struct sock *sk, u16 opcode, struct hci_dev *hdev, void *data, u16 len); +struct mgmt_pending_cmd *mgmt_pending_new(struct sock *sk, u16 opcode, + struct hci_dev *hdev, + void *data, u16 len); void mgmt_pending_free(struct mgmt_pending_cmd *cmd); void mgmt_pending_remove(struct mgmt_pending_cmd *cmd); diff --git a/net/bluetooth/msft.c b/net/bluetooth/msft.c index 255cffa554ee..1122097e1e49 100644 --- a/net/bluetooth/msft.c +++ b/net/bluetooth/msft.c @@ -93,7 +93,7 @@ struct msft_data { struct list_head handle_map; __u16 pending_add_handle; __u16 pending_remove_handle; - __u8 reregistering; + __u8 resuming; __u8 suspending; __u8 filter_enabled; }; @@ -156,7 +156,6 @@ failed: return false; } -/* This function requires the caller holds hdev->lock */ static void reregister_monitor(struct hci_dev *hdev, int handle) { struct adv_monitor *monitor; @@ -166,9 +165,9 @@ static void reregister_monitor(struct hci_dev *hdev, int handle) while (1) { monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); if (!monitor) { - /* All monitors have been reregistered */ - msft->reregistering = false; - hci_update_background_scan(hdev); + /* All monitors have been resumed */ + msft->resuming = false; + hci_update_passive_scan(hdev); return; } @@ -185,67 +184,317 @@ static void reregister_monitor(struct hci_dev *hdev, int handle) } } -/* This function requires the caller holds hdev->lock */ -static void remove_monitor_on_suspend(struct hci_dev *hdev, int handle) +/* is_mgmt = true matches the handle exposed to userspace via mgmt. + * is_mgmt = false matches the handle used by the msft controller. + * This function requires the caller holds hdev->lock + */ +static struct msft_monitor_advertisement_handle_data *msft_find_handle_data + (struct hci_dev *hdev, u16 handle, bool is_mgmt) +{ + struct msft_monitor_advertisement_handle_data *entry; + struct msft_data *msft = hdev->msft_data; + + list_for_each_entry(entry, &msft->handle_map, list) { + if (is_mgmt && entry->mgmt_handle == handle) + return entry; + if (!is_mgmt && entry->msft_handle == handle) + return entry; + } + + return NULL; +} + +static void msft_le_monitor_advertisement_cb(struct hci_dev *hdev, + u8 status, u16 opcode, + struct sk_buff *skb) +{ + struct msft_rp_le_monitor_advertisement *rp; + struct adv_monitor *monitor; + struct msft_monitor_advertisement_handle_data *handle_data; + struct msft_data *msft = hdev->msft_data; + + hci_dev_lock(hdev); + + monitor = idr_find(&hdev->adv_monitors_idr, msft->pending_add_handle); + if (!monitor) { + bt_dev_err(hdev, "msft add advmon: monitor %u is not found!", + msft->pending_add_handle); + status = HCI_ERROR_UNSPECIFIED; + goto unlock; + } + + if (status) + goto unlock; + + rp = (struct msft_rp_le_monitor_advertisement *)skb->data; + if (skb->len < sizeof(*rp)) { + status = HCI_ERROR_UNSPECIFIED; + goto unlock; + } + + handle_data = kmalloc(sizeof(*handle_data), GFP_KERNEL); + if (!handle_data) { + status = HCI_ERROR_UNSPECIFIED; + goto unlock; + } + + handle_data->mgmt_handle = monitor->handle; + handle_data->msft_handle = rp->handle; + INIT_LIST_HEAD(&handle_data->list); + list_add(&handle_data->list, &msft->handle_map); + + monitor->state = ADV_MONITOR_STATE_OFFLOADED; + +unlock: + if (status && monitor) + hci_free_adv_monitor(hdev, monitor); + + hci_dev_unlock(hdev); + + if (!msft->resuming) + hci_add_adv_patterns_monitor_complete(hdev, status); +} + +static void msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, + u8 status, u16 opcode, + struct sk_buff *skb) { + struct msft_cp_le_cancel_monitor_advertisement *cp; + struct msft_rp_le_cancel_monitor_advertisement *rp; struct adv_monitor *monitor; + struct msft_monitor_advertisement_handle_data *handle_data; struct msft_data *msft = hdev->msft_data; int err; + bool pending; - while (1) { - monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); - if (!monitor) { - /* All monitors have been removed */ - msft->suspending = false; - hci_update_background_scan(hdev); + if (status) + goto done; + + rp = (struct msft_rp_le_cancel_monitor_advertisement *)skb->data; + if (skb->len < sizeof(*rp)) { + status = HCI_ERROR_UNSPECIFIED; + goto done; + } + + hci_dev_lock(hdev); + + cp = hci_sent_cmd_data(hdev, hdev->msft_opcode); + handle_data = msft_find_handle_data(hdev, cp->handle, false); + + if (handle_data) { + monitor = idr_find(&hdev->adv_monitors_idr, + handle_data->mgmt_handle); + + if (monitor && monitor->state == ADV_MONITOR_STATE_OFFLOADED) + monitor->state = ADV_MONITOR_STATE_REGISTERED; + + /* Do not free the monitor if it is being removed due to + * suspend. It will be re-monitored on resume. + */ + if (monitor && !msft->suspending) + hci_free_adv_monitor(hdev, monitor); + + list_del(&handle_data->list); + kfree(handle_data); + } + + /* If remove all monitors is required, we need to continue the process + * here because the earlier it was paused when waiting for the + * response from controller. + */ + if (msft->pending_remove_handle == 0) { + pending = hci_remove_all_adv_monitor(hdev, &err); + if (pending) { + hci_dev_unlock(hdev); return; } - msft->pending_remove_handle = (u16)handle; - err = __msft_remove_monitor(hdev, monitor, handle); + if (err) + status = HCI_ERROR_UNSPECIFIED; + } - /* If success, return and wait for monitor removed callback */ - if (!err) - return; + hci_dev_unlock(hdev); + +done: + if (!msft->suspending) + hci_remove_adv_monitor_complete(hdev, status); +} + +static int msft_remove_monitor_sync(struct hci_dev *hdev, + struct adv_monitor *monitor) +{ + struct msft_cp_le_cancel_monitor_advertisement cp; + struct msft_monitor_advertisement_handle_data *handle_data; + struct sk_buff *skb; + u8 status; + + handle_data = msft_find_handle_data(hdev, monitor->handle, true); + + /* If no matched handle, just remove without telling controller */ + if (!handle_data) + return -ENOENT; + + cp.sub_opcode = MSFT_OP_LE_CANCEL_MONITOR_ADVERTISEMENT; + cp.handle = handle_data->msft_handle; + + skb = __hci_cmd_sync(hdev, hdev->msft_opcode, sizeof(cp), &cp, + HCI_CMD_TIMEOUT); + if (IS_ERR(skb)) + return PTR_ERR(skb); + + status = skb->data[0]; + skb_pull(skb, 1); + + msft_le_cancel_monitor_advertisement_cb(hdev, status, hdev->msft_opcode, + skb); + + return status; +} + +/* This function requires the caller holds hci_req_sync_lock */ +int msft_suspend_sync(struct hci_dev *hdev) +{ + struct msft_data *msft = hdev->msft_data; + struct adv_monitor *monitor; + int handle = 0; + + if (!msft || !msft_monitor_supported(hdev)) + return 0; + + msft->suspending = true; + + while (1) { + monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); + if (!monitor) + break; + + msft_remove_monitor_sync(hdev, monitor); - /* Otherwise free the monitor and keep removing */ - hci_free_adv_monitor(hdev, monitor); handle++; } + + /* All monitors have been removed */ + msft->suspending = false; + + return 0; } -/* This function requires the caller holds hdev->lock */ -void msft_suspend(struct hci_dev *hdev) +static bool msft_monitor_rssi_valid(struct adv_monitor *monitor) { - struct msft_data *msft = hdev->msft_data; + struct adv_rssi_thresholds *r = &monitor->rssi; - if (!msft) - return; + if (r->high_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || + r->high_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX || + r->low_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || + r->low_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX) + return false; - if (msft_monitor_supported(hdev)) { - msft->suspending = true; - /* Quitely remove all monitors on suspend to avoid waking up - * the system. - */ - remove_monitor_on_suspend(hdev, 0); + /* High_threshold_timeout is not supported, + * once high_threshold is reached, events are immediately reported. + */ + if (r->high_threshold_timeout != 0) + return false; + + if (r->low_threshold_timeout > MSFT_RSSI_LOW_TIMEOUT_MAX) + return false; + + /* Sampling period from 0x00 to 0xFF are all allowed */ + return true; +} + +static bool msft_monitor_pattern_valid(struct adv_monitor *monitor) +{ + return msft_monitor_rssi_valid(monitor); + /* No additional check needed for pattern-based monitor */ +} + +static int msft_add_monitor_sync(struct hci_dev *hdev, + struct adv_monitor *monitor) +{ + struct msft_cp_le_monitor_advertisement *cp; + struct msft_le_monitor_advertisement_pattern_data *pattern_data; + struct msft_le_monitor_advertisement_pattern *pattern; + struct adv_pattern *entry; + size_t total_size = sizeof(*cp) + sizeof(*pattern_data); + ptrdiff_t offset = 0; + u8 pattern_count = 0; + struct sk_buff *skb; + u8 status; + + if (!msft_monitor_pattern_valid(monitor)) + return -EINVAL; + + list_for_each_entry(entry, &monitor->patterns, list) { + pattern_count++; + total_size += sizeof(*pattern) + entry->length; } + + cp = kmalloc(total_size, GFP_KERNEL); + if (!cp) + return -ENOMEM; + + cp->sub_opcode = MSFT_OP_LE_MONITOR_ADVERTISEMENT; + cp->rssi_high = monitor->rssi.high_threshold; + cp->rssi_low = monitor->rssi.low_threshold; + cp->rssi_low_interval = (u8)monitor->rssi.low_threshold_timeout; + cp->rssi_sampling_period = monitor->rssi.sampling_period; + + cp->cond_type = MSFT_MONITOR_ADVERTISEMENT_TYPE_PATTERN; + + pattern_data = (void *)cp->data; + pattern_data->count = pattern_count; + + list_for_each_entry(entry, &monitor->patterns, list) { + pattern = (void *)(pattern_data->data + offset); + /* the length also includes data_type and offset */ + pattern->length = entry->length + 2; + pattern->data_type = entry->ad_type; + pattern->start_byte = entry->offset; + memcpy(pattern->pattern, entry->value, entry->length); + offset += sizeof(*pattern) + entry->length; + } + + skb = __hci_cmd_sync(hdev, hdev->msft_opcode, total_size, cp, + HCI_CMD_TIMEOUT); + kfree(cp); + + if (IS_ERR(skb)) + return PTR_ERR(skb); + + status = skb->data[0]; + skb_pull(skb, 1); + + msft_le_monitor_advertisement_cb(hdev, status, hdev->msft_opcode, skb); + + return status; } -/* This function requires the caller holds hdev->lock */ -void msft_resume(struct hci_dev *hdev) +/* This function requires the caller holds hci_req_sync_lock */ +int msft_resume_sync(struct hci_dev *hdev) { struct msft_data *msft = hdev->msft_data; + struct adv_monitor *monitor; + int handle = 0; - if (!msft) - return; + if (!msft || !msft_monitor_supported(hdev)) + return 0; - if (msft_monitor_supported(hdev)) { - msft->reregistering = true; - /* Monitors are removed on suspend, so we need to add all - * monitors on resume. - */ - reregister_monitor(hdev, 0); + msft->resuming = true; + + while (1) { + monitor = idr_get_next(&hdev->adv_monitors_idr, &handle); + if (!monitor) + break; + + msft_add_monitor_sync(hdev, monitor); + + handle++; } + + /* All monitors have been resumed */ + msft->resuming = false; + + return 0; } void msft_do_open(struct hci_dev *hdev) @@ -275,7 +524,7 @@ void msft_do_open(struct hci_dev *hdev) } if (msft_monitor_supported(hdev)) { - msft->reregistering = true; + msft->resuming = true; msft_set_filter_enable(hdev, true); /* Monitors get removed on power off, so we need to explicitly * tell the controller to re-monitor. @@ -381,151 +630,6 @@ __u64 msft_get_features(struct hci_dev *hdev) return msft ? msft->features : 0; } -/* is_mgmt = true matches the handle exposed to userspace via mgmt. - * is_mgmt = false matches the handle used by the msft controller. - * This function requires the caller holds hdev->lock - */ -static struct msft_monitor_advertisement_handle_data *msft_find_handle_data - (struct hci_dev *hdev, u16 handle, bool is_mgmt) -{ - struct msft_monitor_advertisement_handle_data *entry; - struct msft_data *msft = hdev->msft_data; - - list_for_each_entry(entry, &msft->handle_map, list) { - if (is_mgmt && entry->mgmt_handle == handle) - return entry; - if (!is_mgmt && entry->msft_handle == handle) - return entry; - } - - return NULL; -} - -static void msft_le_monitor_advertisement_cb(struct hci_dev *hdev, - u8 status, u16 opcode, - struct sk_buff *skb) -{ - struct msft_rp_le_monitor_advertisement *rp; - struct adv_monitor *monitor; - struct msft_monitor_advertisement_handle_data *handle_data; - struct msft_data *msft = hdev->msft_data; - - hci_dev_lock(hdev); - - monitor = idr_find(&hdev->adv_monitors_idr, msft->pending_add_handle); - if (!monitor) { - bt_dev_err(hdev, "msft add advmon: monitor %u is not found!", - msft->pending_add_handle); - status = HCI_ERROR_UNSPECIFIED; - goto unlock; - } - - if (status) - goto unlock; - - rp = (struct msft_rp_le_monitor_advertisement *)skb->data; - if (skb->len < sizeof(*rp)) { - status = HCI_ERROR_UNSPECIFIED; - goto unlock; - } - - handle_data = kmalloc(sizeof(*handle_data), GFP_KERNEL); - if (!handle_data) { - status = HCI_ERROR_UNSPECIFIED; - goto unlock; - } - - handle_data->mgmt_handle = monitor->handle; - handle_data->msft_handle = rp->handle; - INIT_LIST_HEAD(&handle_data->list); - list_add(&handle_data->list, &msft->handle_map); - - monitor->state = ADV_MONITOR_STATE_OFFLOADED; - -unlock: - if (status && monitor) - hci_free_adv_monitor(hdev, monitor); - - /* If in restart/reregister sequence, keep registering. */ - if (msft->reregistering) - reregister_monitor(hdev, msft->pending_add_handle + 1); - - hci_dev_unlock(hdev); - - if (!msft->reregistering) - hci_add_adv_patterns_monitor_complete(hdev, status); -} - -static void msft_le_cancel_monitor_advertisement_cb(struct hci_dev *hdev, - u8 status, u16 opcode, - struct sk_buff *skb) -{ - struct msft_cp_le_cancel_monitor_advertisement *cp; - struct msft_rp_le_cancel_monitor_advertisement *rp; - struct adv_monitor *monitor; - struct msft_monitor_advertisement_handle_data *handle_data; - struct msft_data *msft = hdev->msft_data; - int err; - bool pending; - - if (status) - goto done; - - rp = (struct msft_rp_le_cancel_monitor_advertisement *)skb->data; - if (skb->len < sizeof(*rp)) { - status = HCI_ERROR_UNSPECIFIED; - goto done; - } - - hci_dev_lock(hdev); - - cp = hci_sent_cmd_data(hdev, hdev->msft_opcode); - handle_data = msft_find_handle_data(hdev, cp->handle, false); - - if (handle_data) { - monitor = idr_find(&hdev->adv_monitors_idr, - handle_data->mgmt_handle); - - if (monitor && monitor->state == ADV_MONITOR_STATE_OFFLOADED) - monitor->state = ADV_MONITOR_STATE_REGISTERED; - - /* Do not free the monitor if it is being removed due to - * suspend. It will be re-monitored on resume. - */ - if (monitor && !msft->suspending) - hci_free_adv_monitor(hdev, monitor); - - list_del(&handle_data->list); - kfree(handle_data); - } - - /* If in suspend/remove sequence, keep removing. */ - if (msft->suspending) - remove_monitor_on_suspend(hdev, - msft->pending_remove_handle + 1); - - /* If remove all monitors is required, we need to continue the process - * here because the earlier it was paused when waiting for the - * response from controller. - */ - if (msft->pending_remove_handle == 0) { - pending = hci_remove_all_adv_monitor(hdev, &err); - if (pending) { - hci_dev_unlock(hdev); - return; - } - - if (err) - status = HCI_ERROR_UNSPECIFIED; - } - - hci_dev_unlock(hdev); - -done: - if (!msft->suspending) - hci_remove_adv_monitor_complete(hdev, status); -} - static void msft_le_set_advertisement_filter_enable_cb(struct hci_dev *hdev, u8 status, u16 opcode, struct sk_buff *skb) @@ -560,35 +664,6 @@ static void msft_le_set_advertisement_filter_enable_cb(struct hci_dev *hdev, hci_dev_unlock(hdev); } -static bool msft_monitor_rssi_valid(struct adv_monitor *monitor) -{ - struct adv_rssi_thresholds *r = &monitor->rssi; - - if (r->high_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || - r->high_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX || - r->low_threshold < MSFT_RSSI_THRESHOLD_VALUE_MIN || - r->low_threshold > MSFT_RSSI_THRESHOLD_VALUE_MAX) - return false; - - /* High_threshold_timeout is not supported, - * once high_threshold is reached, events are immediately reported. - */ - if (r->high_threshold_timeout != 0) - return false; - - if (r->low_threshold_timeout > MSFT_RSSI_LOW_TIMEOUT_MAX) - return false; - - /* Sampling period from 0x00 to 0xFF are all allowed */ - return true; -} - -static bool msft_monitor_pattern_valid(struct adv_monitor *monitor) -{ - return msft_monitor_rssi_valid(monitor); - /* No additional check needed for pattern-based monitor */ -} - /* This function requires the caller holds hdev->lock */ static int __msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor) @@ -656,7 +731,7 @@ int msft_add_monitor_pattern(struct hci_dev *hdev, struct adv_monitor *monitor) if (!msft) return -EOPNOTSUPP; - if (msft->reregistering || msft->suspending) + if (msft->resuming || msft->suspending) return -EBUSY; return __msft_add_monitor_pattern(hdev, monitor); @@ -700,7 +775,7 @@ int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, if (!msft) return -EOPNOTSUPP; - if (msft->reregistering || msft->suspending) + if (msft->resuming || msft->suspending) return -EBUSY; return __msft_remove_monitor(hdev, monitor, handle); diff --git a/net/bluetooth/msft.h b/net/bluetooth/msft.h index 59c6e081c789..b59b63dc0ea8 100644 --- a/net/bluetooth/msft.h +++ b/net/bluetooth/msft.h @@ -24,8 +24,8 @@ int msft_remove_monitor(struct hci_dev *hdev, struct adv_monitor *monitor, u16 handle); void msft_req_add_set_filter_enable(struct hci_request *req, bool enable); int msft_set_filter_enable(struct hci_dev *hdev, bool enable); -void msft_suspend(struct hci_dev *hdev); -void msft_resume(struct hci_dev *hdev); +int msft_suspend_sync(struct hci_dev *hdev); +int msft_resume_sync(struct hci_dev *hdev); bool msft_curve_validity(struct hci_dev *hdev); #else @@ -61,8 +61,15 @@ static inline int msft_set_filter_enable(struct hci_dev *hdev, bool enable) return -EOPNOTSUPP; } -static inline void msft_suspend(struct hci_dev *hdev) {} -static inline void msft_resume(struct hci_dev *hdev) {} +static inline int msft_suspend_sync(struct hci_dev *hdev) +{ + return -EOPNOTSUPP; +} + +static inline int msft_resume_sync(struct hci_dev *hdev) +{ + return -EOPNOTSUPP; +} static inline bool msft_curve_validity(struct hci_dev *hdev) { diff --git a/net/bridge/br_if.c b/net/bridge/br_if.c index c1183fef1f21..a52ad81596b7 100644 --- a/net/bridge/br_if.c +++ b/net/bridge/br_if.c @@ -274,7 +274,7 @@ static void destroy_nbp(struct net_bridge_port *p) p->br = NULL; p->dev = NULL; - dev_put(dev); + dev_put_track(dev, &p->dev_tracker); kobject_put(&p->kobj); } @@ -397,10 +397,10 @@ static int find_portno(struct net_bridge *br) if (!inuse) return -ENOMEM; - set_bit(0, inuse); /* zero is reserved */ - list_for_each_entry(p, &br->port_list, list) { - set_bit(p->port_no, inuse); - } + __set_bit(0, inuse); /* zero is reserved */ + list_for_each_entry(p, &br->port_list, list) + __set_bit(p->port_no, inuse); + index = find_first_zero_bit(inuse, BR_MAX_PORTS); bitmap_free(inuse); @@ -423,7 +423,7 @@ static struct net_bridge_port *new_nbp(struct net_bridge *br, return ERR_PTR(-ENOMEM); p->br = br; - dev_hold(dev); + dev_hold_track(dev, &p->dev_tracker, GFP_KERNEL); p->dev = dev; p->path_cost = port_cost(dev); p->priority = 0x8000 >> BR_PORT_BITS; @@ -434,7 +434,7 @@ static struct net_bridge_port *new_nbp(struct net_bridge *br, br_stp_port_timer_init(p); err = br_multicast_add_port(p); if (err) { - dev_put(dev); + dev_put_track(dev, &p->dev_tracker); kfree(p); p = ERR_PTR(err); } @@ -525,8 +525,8 @@ static void br_set_gso_limits(struct net_bridge *br) gso_max_size = min(gso_max_size, p->dev->gso_max_size); gso_max_segs = min(gso_max_segs, p->dev->gso_max_segs); } - br->dev->gso_max_size = gso_max_size; - br->dev->gso_max_segs = gso_max_segs; + netif_set_gso_max_size(br->dev, gso_max_size); + netif_set_gso_max_segs(br->dev, gso_max_segs); } /* diff --git a/net/bridge/br_private.h b/net/bridge/br_private.h index c0efd697865a..af2b3512d86c 100644 --- a/net/bridge/br_private.h +++ b/net/bridge/br_private.h @@ -344,6 +344,7 @@ struct net_bridge_mdb_entry { struct net_bridge_port { struct net_bridge *br; struct net_device *dev; + netdevice_tracker dev_tracker; struct list_head list; unsigned long flags; diff --git a/net/bridge/br_sysfs_br.c b/net/bridge/br_sysfs_br.c index d9a89ddd0331..159590d5c2af 100644 --- a/net/bridge/br_sysfs_br.c +++ b/net/bridge/br_sysfs_br.c @@ -36,15 +36,14 @@ static ssize_t store_bridge_parm(struct device *d, struct net_bridge *br = to_bridge(d); struct netlink_ext_ack extack = {0}; unsigned long val; - char *endp; int err; if (!ns_capable(dev_net(br->dev)->user_ns, CAP_NET_ADMIN)) return -EPERM; - val = simple_strtoul(buf, &endp, 0); - if (endp == buf) - return -EINVAL; + err = kstrtoul(buf, 0, &val); + if (err != 0) + return err; if (!rtnl_trylock()) return restart_syscall(); diff --git a/net/bridge/br_vlan.c b/net/bridge/br_vlan.c index 49e105e0a447..84ba456a78cc 100644 --- a/net/bridge/br_vlan.c +++ b/net/bridge/br_vlan.c @@ -1063,7 +1063,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, if (br_vlan_delete(br, old_pvid)) br_vlan_notify(br, NULL, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(br, NULL, pvid, 0, RTM_NEWVLAN); - set_bit(0, changed); + __set_bit(0, changed); } list_for_each_entry(p, &br->port_list, list) { @@ -1085,7 +1085,7 @@ int __br_vlan_set_default_pvid(struct net_bridge *br, u16 pvid, if (nbp_vlan_delete(p, old_pvid)) br_vlan_notify(br, p, old_pvid, 0, RTM_DELVLAN); br_vlan_notify(p->br, p, pvid, 0, RTM_NEWVLAN); - set_bit(p->port_no, changed); + __set_bit(p->port_no, changed); } br->default_pvid = pvid; diff --git a/net/core/Makefile b/net/core/Makefile index 4268846f2f47..a8e4f737692b 100644 --- a/net/core/Makefile +++ b/net/core/Makefile @@ -11,7 +11,9 @@ obj-$(CONFIG_SYSCTL) += sysctl_net_core.o obj-y += dev.o dev_addr_lists.o dst.o netevent.o \ neighbour.o rtnetlink.o utils.o link_watch.o filter.o \ sock_diag.o dev_ioctl.o tso.o sock_reuseport.o \ - fib_notifier.o xdp.o flow_offload.o + fib_notifier.o xdp.o flow_offload.o gro.o + +obj-$(CONFIG_NETDEV_ADDR_LIST_TEST) += dev_addr_lists_test.o obj-y += net-sysfs.o obj-$(CONFIG_PAGE_POOL) += page_pool.o diff --git a/net/core/dev.c b/net/core/dev.c index 2a352e668d10..4420086f3aeb 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -153,16 +153,10 @@ #include "net-sysfs.h" -#define MAX_GRO_SKBS 8 - -/* This should be increased if a protocol with a bigger head is added. */ -#define GRO_MAX_HEAD (MAX_HEADER + 128) static DEFINE_SPINLOCK(ptype_lock); -static DEFINE_SPINLOCK(offload_lock); struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; struct list_head ptype_all __read_mostly; /* Taps */ -static struct list_head offload_base __read_mostly; static int netif_rx_internal(struct sk_buff *skb); static int call_netdevice_notifiers_info(unsigned long val, @@ -371,12 +365,12 @@ static void list_netdevice(struct net_device *dev) ASSERT_RTNL(); - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); netdev_name_node_add(net, dev->name_node); hlist_add_head_rcu(&dev->index_hlist, dev_index_hash(net, dev->ifindex)); - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); dev_base_seq_inc(net); } @@ -389,11 +383,11 @@ static void unlist_netdevice(struct net_device *dev) ASSERT_RTNL(); /* Unlink dev from the device chain */ - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); list_del_rcu(&dev->dev_list); netdev_name_node_del(dev->name_node); hlist_del_rcu(&dev->index_hlist); - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); dev_base_seq_inc(dev_net(dev)); } @@ -604,84 +598,6 @@ void dev_remove_pack(struct packet_type *pt) EXPORT_SYMBOL(dev_remove_pack); -/** - * dev_add_offload - register offload handlers - * @po: protocol offload declaration - * - * Add protocol offload handlers to the networking stack. The passed - * &proto_offload is linked into kernel lists and may not be freed until - * it has been removed from the kernel lists. - * - * This call does not sleep therefore it can not - * guarantee all CPU's that are in middle of receiving packets - * will see the new offload handlers (until the next received packet). - */ -void dev_add_offload(struct packet_offload *po) -{ - struct packet_offload *elem; - - spin_lock(&offload_lock); - list_for_each_entry(elem, &offload_base, list) { - if (po->priority < elem->priority) - break; - } - list_add_rcu(&po->list, elem->list.prev); - spin_unlock(&offload_lock); -} -EXPORT_SYMBOL(dev_add_offload); - -/** - * __dev_remove_offload - remove offload handler - * @po: packet offload declaration - * - * Remove a protocol offload handler that was previously added to the - * kernel offload handlers by dev_add_offload(). The passed &offload_type - * is removed from the kernel lists and can be freed or reused once this - * function returns. - * - * The packet type might still be in use by receivers - * and must not be freed until after all the CPU's have gone - * through a quiescent state. - */ -static void __dev_remove_offload(struct packet_offload *po) -{ - struct list_head *head = &offload_base; - struct packet_offload *po1; - - spin_lock(&offload_lock); - - list_for_each_entry(po1, head, list) { - if (po == po1) { - list_del_rcu(&po->list); - goto out; - } - } - - pr_warn("dev_remove_offload: %p not found\n", po); -out: - spin_unlock(&offload_lock); -} - -/** - * dev_remove_offload - remove packet offload handler - * @po: packet offload declaration - * - * Remove a packet offload handler that was previously added to the kernel - * offload handlers by dev_add_offload(). The passed &offload_type is - * removed from the kernel lists and can be freed or reused once this - * function returns. - * - * This call sleeps to guarantee that no CPU is looking at the packet - * type after return. - */ -void dev_remove_offload(struct packet_offload *po) -{ - __dev_remove_offload(po); - - synchronize_net(); -} -EXPORT_SYMBOL(dev_remove_offload); - /******************************************************************************* * * Device Interface Subroutines @@ -1272,15 +1188,15 @@ rollback: netdev_adjacent_rename_links(dev, oldname); - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); netdev_name_node_del(dev->name_node); - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); synchronize_rcu(); - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); netdev_name_node_add(net, dev->name_node); - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); ret = notifier_to_errno(ret); @@ -1461,6 +1377,7 @@ static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) int ret; ASSERT_RTNL(); + dev_addr_check(dev); if (!netif_device_present(dev)) { /* may be detached because parent is runtime-suspended */ @@ -3315,40 +3232,6 @@ __be16 skb_network_protocol(struct sk_buff *skb, int *depth) return __vlan_get_protocol(skb, type, depth); } -/** - * skb_mac_gso_segment - mac layer segmentation handler. - * @skb: buffer to segment - * @features: features for the output path (see dev->features) - */ -struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, - netdev_features_t features) -{ - struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); - struct packet_offload *ptype; - int vlan_depth = skb->mac_len; - __be16 type = skb_network_protocol(skb, &vlan_depth); - - if (unlikely(!type)) - return ERR_PTR(-EINVAL); - - __skb_pull(skb, vlan_depth); - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, &offload_base, list) { - if (ptype->type == type && ptype->callbacks.gso_segment) { - segs = ptype->callbacks.gso_segment(skb, features); - break; - } - } - rcu_read_unlock(); - - __skb_push(skb, skb->data - skb_mac_header(skb)); - - return segs; -} -EXPORT_SYMBOL(skb_mac_gso_segment); - - /* openvswitch calls this on rx path, so we need a different check. */ static inline bool skb_needs_check(struct sk_buff *skb, bool tx_path) @@ -3513,7 +3396,7 @@ static netdev_features_t gso_features_check(const struct sk_buff *skb, { u16 gso_segs = skb_shinfo(skb)->gso_segs; - if (gso_segs > dev->gso_max_segs) + if (gso_segs > READ_ONCE(dev->gso_max_segs)) return features & ~NETIF_F_GSO_MASK; if (!skb_shinfo(skb)->gso_type) { @@ -4323,8 +4206,6 @@ int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ int dev_rx_weight __read_mostly = 64; int dev_tx_weight __read_mostly = 64; -/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ -int gro_normal_batch __read_mostly = 8; /* Called with irq disabled */ static inline void ____napi_schedule(struct softnet_data *sd, @@ -5667,7 +5548,7 @@ static int netif_receive_skb_internal(struct sk_buff *skb) return ret; } -static void netif_receive_skb_list_internal(struct list_head *head) +void netif_receive_skb_list_internal(struct list_head *head) { struct sk_buff *skb, *next; struct list_head sublist; @@ -5845,550 +5726,6 @@ static void flush_all_backlogs(void) cpus_read_unlock(); } -/* Pass the currently batched GRO_NORMAL SKBs up to the stack. */ -static void gro_normal_list(struct napi_struct *napi) -{ - if (!napi->rx_count) - return; - netif_receive_skb_list_internal(&napi->rx_list); - INIT_LIST_HEAD(&napi->rx_list); - napi->rx_count = 0; -} - -/* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded, - * pass the whole batch up to the stack. - */ -static void gro_normal_one(struct napi_struct *napi, struct sk_buff *skb, int segs) -{ - list_add_tail(&skb->list, &napi->rx_list); - napi->rx_count += segs; - if (napi->rx_count >= gro_normal_batch) - gro_normal_list(napi); -} - -static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) -{ - struct packet_offload *ptype; - __be16 type = skb->protocol; - struct list_head *head = &offload_base; - int err = -ENOENT; - - BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); - - if (NAPI_GRO_CB(skb)->count == 1) { - skb_shinfo(skb)->gso_size = 0; - goto out; - } - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, head, list) { - if (ptype->type != type || !ptype->callbacks.gro_complete) - continue; - - err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, - ipv6_gro_complete, inet_gro_complete, - skb, 0); - break; - } - rcu_read_unlock(); - - if (err) { - WARN_ON(&ptype->list == head); - kfree_skb(skb); - return; - } - -out: - gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); -} - -static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, - bool flush_old) -{ - struct list_head *head = &napi->gro_hash[index].list; - struct sk_buff *skb, *p; - - list_for_each_entry_safe_reverse(skb, p, head, list) { - if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) - return; - skb_list_del_init(skb); - napi_gro_complete(napi, skb); - napi->gro_hash[index].count--; - } - - if (!napi->gro_hash[index].count) - __clear_bit(index, &napi->gro_bitmask); -} - -/* napi->gro_hash[].list contains packets ordered by age. - * youngest packets at the head of it. - * Complete skbs in reverse order to reduce latencies. - */ -void napi_gro_flush(struct napi_struct *napi, bool flush_old) -{ - unsigned long bitmask = napi->gro_bitmask; - unsigned int i, base = ~0U; - - while ((i = ffs(bitmask)) != 0) { - bitmask >>= i; - base += i; - __napi_gro_flush_chain(napi, base, flush_old); - } -} -EXPORT_SYMBOL(napi_gro_flush); - -static void gro_list_prepare(const struct list_head *head, - const struct sk_buff *skb) -{ - unsigned int maclen = skb->dev->hard_header_len; - u32 hash = skb_get_hash_raw(skb); - struct sk_buff *p; - - list_for_each_entry(p, head, list) { - unsigned long diffs; - - NAPI_GRO_CB(p)->flush = 0; - - if (hash != skb_get_hash_raw(p)) { - NAPI_GRO_CB(p)->same_flow = 0; - continue; - } - - diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; - diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); - if (skb_vlan_tag_present(p)) - diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); - diffs |= skb_metadata_differs(p, skb); - if (maclen == ETH_HLEN) - diffs |= compare_ether_header(skb_mac_header(p), - skb_mac_header(skb)); - else if (!diffs) - diffs = memcmp(skb_mac_header(p), - skb_mac_header(skb), - maclen); - - /* in most common scenarions 'slow_gro' is 0 - * otherwise we are already on some slower paths - * either skip all the infrequent tests altogether or - * avoid trying too hard to skip each of them individually - */ - if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { -#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) - struct tc_skb_ext *skb_ext; - struct tc_skb_ext *p_ext; -#endif - - diffs |= p->sk != skb->sk; - diffs |= skb_metadata_dst_cmp(p, skb); - diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); - -#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) - skb_ext = skb_ext_find(skb, TC_SKB_EXT); - p_ext = skb_ext_find(p, TC_SKB_EXT); - - diffs |= (!!p_ext) ^ (!!skb_ext); - if (!diffs && unlikely(skb_ext)) - diffs |= p_ext->chain ^ skb_ext->chain; -#endif - } - - NAPI_GRO_CB(p)->same_flow = !diffs; - } -} - -static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) -{ - const struct skb_shared_info *pinfo = skb_shinfo(skb); - const skb_frag_t *frag0 = &pinfo->frags[0]; - - NAPI_GRO_CB(skb)->data_offset = 0; - NAPI_GRO_CB(skb)->frag0 = NULL; - NAPI_GRO_CB(skb)->frag0_len = 0; - - if (!skb_headlen(skb) && pinfo->nr_frags && - !PageHighMem(skb_frag_page(frag0)) && - (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { - NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); - NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, - skb_frag_size(frag0), - skb->end - skb->tail); - } -} - -static void gro_pull_from_frag0(struct sk_buff *skb, int grow) -{ - struct skb_shared_info *pinfo = skb_shinfo(skb); - - BUG_ON(skb->end - skb->tail < grow); - - memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); - - skb->data_len -= grow; - skb->tail += grow; - - skb_frag_off_add(&pinfo->frags[0], grow); - skb_frag_size_sub(&pinfo->frags[0], grow); - - if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { - skb_frag_unref(skb, 0); - memmove(pinfo->frags, pinfo->frags + 1, - --pinfo->nr_frags * sizeof(pinfo->frags[0])); - } -} - -static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) -{ - struct sk_buff *oldest; - - oldest = list_last_entry(head, struct sk_buff, list); - - /* We are called with head length >= MAX_GRO_SKBS, so this is - * impossible. - */ - if (WARN_ON_ONCE(!oldest)) - return; - - /* Do not adjust napi->gro_hash[].count, caller is adding a new - * SKB to the chain. - */ - skb_list_del_init(oldest); - napi_gro_complete(napi, oldest); -} - -static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -{ - u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); - struct gro_list *gro_list = &napi->gro_hash[bucket]; - struct list_head *head = &offload_base; - struct packet_offload *ptype; - __be16 type = skb->protocol; - struct sk_buff *pp = NULL; - enum gro_result ret; - int same_flow; - int grow; - - if (netif_elide_gro(skb->dev)) - goto normal; - - gro_list_prepare(&gro_list->list, skb); - - rcu_read_lock(); - list_for_each_entry_rcu(ptype, head, list) { - if (ptype->type != type || !ptype->callbacks.gro_receive) - continue; - - skb_set_network_header(skb, skb_gro_offset(skb)); - skb_reset_mac_len(skb); - NAPI_GRO_CB(skb)->same_flow = 0; - NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); - NAPI_GRO_CB(skb)->free = 0; - NAPI_GRO_CB(skb)->encap_mark = 0; - NAPI_GRO_CB(skb)->recursion_counter = 0; - NAPI_GRO_CB(skb)->is_fou = 0; - NAPI_GRO_CB(skb)->is_atomic = 1; - NAPI_GRO_CB(skb)->gro_remcsum_start = 0; - - /* Setup for GRO checksum validation */ - switch (skb->ip_summed) { - case CHECKSUM_COMPLETE: - NAPI_GRO_CB(skb)->csum = skb->csum; - NAPI_GRO_CB(skb)->csum_valid = 1; - NAPI_GRO_CB(skb)->csum_cnt = 0; - break; - case CHECKSUM_UNNECESSARY: - NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; - NAPI_GRO_CB(skb)->csum_valid = 0; - break; - default: - NAPI_GRO_CB(skb)->csum_cnt = 0; - NAPI_GRO_CB(skb)->csum_valid = 0; - } - - pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, - ipv6_gro_receive, inet_gro_receive, - &gro_list->list, skb); - break; - } - rcu_read_unlock(); - - if (&ptype->list == head) - goto normal; - - if (PTR_ERR(pp) == -EINPROGRESS) { - ret = GRO_CONSUMED; - goto ok; - } - - same_flow = NAPI_GRO_CB(skb)->same_flow; - ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; - - if (pp) { - skb_list_del_init(pp); - napi_gro_complete(napi, pp); - gro_list->count--; - } - - if (same_flow) - goto ok; - - if (NAPI_GRO_CB(skb)->flush) - goto normal; - - if (unlikely(gro_list->count >= MAX_GRO_SKBS)) - gro_flush_oldest(napi, &gro_list->list); - else - gro_list->count++; - - NAPI_GRO_CB(skb)->count = 1; - NAPI_GRO_CB(skb)->age = jiffies; - NAPI_GRO_CB(skb)->last = skb; - skb_shinfo(skb)->gso_size = skb_gro_len(skb); - list_add(&skb->list, &gro_list->list); - ret = GRO_HELD; - -pull: - grow = skb_gro_offset(skb) - skb_headlen(skb); - if (grow > 0) - gro_pull_from_frag0(skb, grow); -ok: - if (gro_list->count) { - if (!test_bit(bucket, &napi->gro_bitmask)) - __set_bit(bucket, &napi->gro_bitmask); - } else if (test_bit(bucket, &napi->gro_bitmask)) { - __clear_bit(bucket, &napi->gro_bitmask); - } - - return ret; - -normal: - ret = GRO_NORMAL; - goto pull; -} - -struct packet_offload *gro_find_receive_by_type(__be16 type) -{ - struct list_head *offload_head = &offload_base; - struct packet_offload *ptype; - - list_for_each_entry_rcu(ptype, offload_head, list) { - if (ptype->type != type || !ptype->callbacks.gro_receive) - continue; - return ptype; - } - return NULL; -} -EXPORT_SYMBOL(gro_find_receive_by_type); - -struct packet_offload *gro_find_complete_by_type(__be16 type) -{ - struct list_head *offload_head = &offload_base; - struct packet_offload *ptype; - - list_for_each_entry_rcu(ptype, offload_head, list) { - if (ptype->type != type || !ptype->callbacks.gro_complete) - continue; - return ptype; - } - return NULL; -} -EXPORT_SYMBOL(gro_find_complete_by_type); - -static gro_result_t napi_skb_finish(struct napi_struct *napi, - struct sk_buff *skb, - gro_result_t ret) -{ - switch (ret) { - case GRO_NORMAL: - gro_normal_one(napi, skb, 1); - break; - - case GRO_MERGED_FREE: - if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) - napi_skb_free_stolen_head(skb); - else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) - __kfree_skb(skb); - else - __kfree_skb_defer(skb); - break; - - case GRO_HELD: - case GRO_MERGED: - case GRO_CONSUMED: - break; - } - - return ret; -} - -gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) -{ - gro_result_t ret; - - skb_mark_napi_id(skb, napi); - trace_napi_gro_receive_entry(skb); - - skb_gro_reset_offset(skb, 0); - - ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); - trace_napi_gro_receive_exit(ret); - - return ret; -} -EXPORT_SYMBOL(napi_gro_receive); - -static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) -{ - if (unlikely(skb->pfmemalloc)) { - consume_skb(skb); - return; - } - __skb_pull(skb, skb_headlen(skb)); - /* restore the reserve we had after netdev_alloc_skb_ip_align() */ - skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); - __vlan_hwaccel_clear_tag(skb); - skb->dev = napi->dev; - skb->skb_iif = 0; - - /* eth_type_trans() assumes pkt_type is PACKET_HOST */ - skb->pkt_type = PACKET_HOST; - - skb->encapsulation = 0; - skb_shinfo(skb)->gso_type = 0; - skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); - if (unlikely(skb->slow_gro)) { - skb_orphan(skb); - skb_ext_reset(skb); - nf_reset_ct(skb); - skb->slow_gro = 0; - } - - napi->skb = skb; -} - -struct sk_buff *napi_get_frags(struct napi_struct *napi) -{ - struct sk_buff *skb = napi->skb; - - if (!skb) { - skb = napi_alloc_skb(napi, GRO_MAX_HEAD); - if (skb) { - napi->skb = skb; - skb_mark_napi_id(skb, napi); - } - } - return skb; -} -EXPORT_SYMBOL(napi_get_frags); - -static gro_result_t napi_frags_finish(struct napi_struct *napi, - struct sk_buff *skb, - gro_result_t ret) -{ - switch (ret) { - case GRO_NORMAL: - case GRO_HELD: - __skb_push(skb, ETH_HLEN); - skb->protocol = eth_type_trans(skb, skb->dev); - if (ret == GRO_NORMAL) - gro_normal_one(napi, skb, 1); - break; - - case GRO_MERGED_FREE: - if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) - napi_skb_free_stolen_head(skb); - else - napi_reuse_skb(napi, skb); - break; - - case GRO_MERGED: - case GRO_CONSUMED: - break; - } - - return ret; -} - -/* Upper GRO stack assumes network header starts at gro_offset=0 - * Drivers could call both napi_gro_frags() and napi_gro_receive() - * We copy ethernet header into skb->data to have a common layout. - */ -static struct sk_buff *napi_frags_skb(struct napi_struct *napi) -{ - struct sk_buff *skb = napi->skb; - const struct ethhdr *eth; - unsigned int hlen = sizeof(*eth); - - napi->skb = NULL; - - skb_reset_mac_header(skb); - skb_gro_reset_offset(skb, hlen); - - if (unlikely(skb_gro_header_hard(skb, hlen))) { - eth = skb_gro_header_slow(skb, hlen, 0); - if (unlikely(!eth)) { - net_warn_ratelimited("%s: dropping impossible skb from %s\n", - __func__, napi->dev->name); - napi_reuse_skb(napi, skb); - return NULL; - } - } else { - eth = (const struct ethhdr *)skb->data; - gro_pull_from_frag0(skb, hlen); - NAPI_GRO_CB(skb)->frag0 += hlen; - NAPI_GRO_CB(skb)->frag0_len -= hlen; - } - __skb_pull(skb, hlen); - - /* - * This works because the only protocols we care about don't require - * special handling. - * We'll fix it up properly in napi_frags_finish() - */ - skb->protocol = eth->h_proto; - - return skb; -} - -gro_result_t napi_gro_frags(struct napi_struct *napi) -{ - gro_result_t ret; - struct sk_buff *skb = napi_frags_skb(napi); - - trace_napi_gro_frags_entry(skb); - - ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); - trace_napi_gro_frags_exit(ret); - - return ret; -} -EXPORT_SYMBOL(napi_gro_frags); - -/* Compute the checksum from gro_offset and return the folded value - * after adding in any pseudo checksum. - */ -__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) -{ - __wsum wsum; - __sum16 sum; - - wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); - - /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ - sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); - /* See comments in __skb_checksum_complete(). */ - if (likely(!sum)) { - if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && - !skb->csum_complete_sw) - netdev_rx_csum_fault(skb->dev, skb); - } - - NAPI_GRO_CB(skb)->csum = wsum; - NAPI_GRO_CB(skb)->csum_valid = 1; - - return sum; -} -EXPORT_SYMBOL(__skb_gro_checksum_complete); - static void net_rps_send_ipi(struct softnet_data *remsd) { #ifdef CONFIG_RPS @@ -7200,6 +6537,7 @@ static __latent_entropy void net_rx_action(struct softirq_action *h) struct netdev_adjacent { struct net_device *dev; + netdevice_tracker dev_tracker; /* upper master flag, there can only be one master device per list */ bool master; @@ -7964,7 +7302,7 @@ static int __netdev_adjacent_dev_insert(struct net_device *dev, adj->ref_nr = 1; adj->private = private; adj->ignore = false; - dev_hold(adj_dev); + dev_hold_track(adj_dev, &adj->dev_tracker, GFP_KERNEL); pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); @@ -7993,8 +7331,8 @@ remove_symlinks: if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); free_adj: + dev_put_track(adj_dev, &adj->dev_tracker); kfree(adj); - dev_put(adj_dev); return ret; } @@ -8035,7 +7373,7 @@ static void __netdev_adjacent_dev_remove(struct net_device *dev, list_del_rcu(&adj->list); pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", adj_dev->name, dev->name, adj_dev->name); - dev_put(adj_dev); + dev_put_track(adj_dev, &adj->dev_tracker); kfree_rcu(adj, rcu); } @@ -9224,35 +8562,17 @@ bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) EXPORT_SYMBOL(netdev_port_same_parent_id); /** - * dev_change_proto_down - update protocol port state information + * dev_change_proto_down - set carrier according to proto_down. + * * @dev: device * @proto_down: new value - * - * This info can be used by switch drivers to set the phys state of the - * port. */ int dev_change_proto_down(struct net_device *dev, bool proto_down) { - const struct net_device_ops *ops = dev->netdev_ops; - - if (!ops->ndo_change_proto_down) + if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) return -EOPNOTSUPP; if (!netif_device_present(dev)) return -ENODEV; - return ops->ndo_change_proto_down(dev, proto_down); -} -EXPORT_SYMBOL(dev_change_proto_down); - -/** - * dev_change_proto_down_generic - generic implementation for - * ndo_change_proto_down that sets carrier according to - * proto_down. - * - * @dev: device - * @proto_down: new value - */ -int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) -{ if (proto_down) netif_carrier_off(dev); else @@ -9260,7 +8580,7 @@ int dev_change_proto_down_generic(struct net_device *dev, bool proto_down) dev->proto_down = proto_down; return 0; } -EXPORT_SYMBOL(dev_change_proto_down_generic); +EXPORT_SYMBOL(dev_change_proto_down); /** * dev_change_proto_down_reason - proto down reason @@ -10545,6 +9865,7 @@ static void netdev_wait_allrefs(struct net_device *dev) netdev_unregister_timeout_secs * HZ)) { pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", dev->name, refcnt); + ref_tracker_dir_print(&dev->refcnt_tracker, 10); warning_time = jiffies; } } @@ -10835,6 +10156,7 @@ struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, dev = PTR_ALIGN(p, NETDEV_ALIGN); dev->padded = (char *)dev - (char *)p; + ref_tracker_dir_init(&dev->refcnt_tracker, 128); #ifdef CONFIG_PCPU_DEV_REFCNT dev->pcpu_refcnt = alloc_percpu(int); if (!dev->pcpu_refcnt) @@ -10951,6 +10273,7 @@ void free_netdev(struct net_device *dev) list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) netif_napi_del(p); + ref_tracker_dir_exit(&dev->refcnt_tracker); #ifdef CONFIG_PCPU_DEV_REFCNT free_percpu(dev->pcpu_refcnt); dev->pcpu_refcnt = NULL; @@ -11643,8 +10966,6 @@ static int __init net_dev_init(void) for (i = 0; i < PTYPE_HASH_SIZE; i++) INIT_LIST_HEAD(&ptype_base[i]); - INIT_LIST_HEAD(&offload_base); - if (register_pernet_subsys(&netdev_net_ops)) goto out; diff --git a/net/core/dev_addr_lists.c b/net/core/dev_addr_lists.c index f0cb38344126..bead38ca50bd 100644 --- a/net/core/dev_addr_lists.c +++ b/net/core/dev_addr_lists.c @@ -16,6 +16,35 @@ * General list handling functions */ +static int __hw_addr_insert(struct netdev_hw_addr_list *list, + struct netdev_hw_addr *new, int addr_len) +{ + struct rb_node **ins_point = &list->tree.rb_node, *parent = NULL; + struct netdev_hw_addr *ha; + + while (*ins_point) { + int diff; + + ha = rb_entry(*ins_point, struct netdev_hw_addr, node); + diff = memcmp(new->addr, ha->addr, addr_len); + if (diff == 0) + diff = memcmp(&new->type, &ha->type, sizeof(new->type)); + + parent = *ins_point; + if (diff < 0) + ins_point = &parent->rb_left; + else if (diff > 0) + ins_point = &parent->rb_right; + else + return -EEXIST; + } + + rb_link_node_rcu(&new->node, parent, ins_point); + rb_insert_color(&new->node, &list->tree); + + return 0; +} + static struct netdev_hw_addr* __hw_addr_create(const unsigned char *addr, int addr_len, unsigned char addr_type, bool global, bool sync) @@ -50,11 +79,6 @@ static int __hw_addr_add_ex(struct netdev_hw_addr_list *list, if (addr_len > MAX_ADDR_LEN) return -EINVAL; - ha = list_first_entry(&list->list, struct netdev_hw_addr, list); - if (ha && !memcmp(addr, ha->addr, addr_len) && - (!addr_type || addr_type == ha->type)) - goto found_it; - while (*ins_point) { int diff; @@ -69,7 +93,6 @@ static int __hw_addr_add_ex(struct netdev_hw_addr_list *list, } else if (diff > 0) { ins_point = &parent->rb_right; } else { -found_it: if (exclusive) return -EEXIST; if (global) { @@ -94,16 +117,8 @@ found_it: if (!ha) return -ENOMEM; - /* The first address in dev->dev_addrs is pointed to by dev->dev_addr - * and mutated freely by device drivers and netdev ops, so if we insert - * it into the tree we'll end up with an invalid rbtree. - */ - if (list->count > 0) { - rb_link_node(&ha->node, parent, ins_point); - rb_insert_color(&ha->node, &list->tree); - } else { - RB_CLEAR_NODE(&ha->node); - } + rb_link_node(&ha->node, parent, ins_point); + rb_insert_color(&ha->node, &list->tree); list_add_tail_rcu(&ha->list, &list->list); list->count++; @@ -138,8 +153,7 @@ static int __hw_addr_del_entry(struct netdev_hw_addr_list *list, if (--ha->refcount) return 0; - if (!RB_EMPTY_NODE(&ha->node)) - rb_erase(&ha->node, &list->tree); + rb_erase(&ha->node, &list->tree); list_del_rcu(&ha->list); kfree_rcu(ha, rcu_head); @@ -151,18 +165,8 @@ static struct netdev_hw_addr *__hw_addr_lookup(struct netdev_hw_addr_list *list, const unsigned char *addr, int addr_len, unsigned char addr_type) { - struct netdev_hw_addr *ha; struct rb_node *node; - /* The first address isn't inserted into the tree because in the dev->dev_addrs - * list it's the address pointed to by dev->dev_addr which is freely mutated - * in place, so we need to check it separately. - */ - ha = list_first_entry(&list->list, struct netdev_hw_addr, list); - if (ha && !memcmp(addr, ha->addr, addr_len) && - (!addr_type || addr_type == ha->type)) - return ha; - node = list->tree.rb_node; while (node) { @@ -498,6 +502,21 @@ EXPORT_SYMBOL(__hw_addr_init); * Device addresses handling functions */ +/* Check that netdev->dev_addr is not written to directly as this would + * break the rbtree layout. All changes should go thru dev_addr_set() and co. + * Remove this check in mid-2024. + */ +void dev_addr_check(struct net_device *dev) +{ + if (!memcmp(dev->dev_addr, dev->dev_addr_shadow, MAX_ADDR_LEN)) + return; + + netdev_warn(dev, "Current addr: %*ph\n", MAX_ADDR_LEN, dev->dev_addr); + netdev_warn(dev, "Expected addr: %*ph\n", + MAX_ADDR_LEN, dev->dev_addr_shadow); + netdev_WARN(dev, "Incorrect netdev->dev_addr\n"); +} + /** * dev_addr_flush - Flush device address list * @dev: device @@ -509,11 +528,11 @@ EXPORT_SYMBOL(__hw_addr_init); void dev_addr_flush(struct net_device *dev) { /* rtnl_mutex must be held here */ + dev_addr_check(dev); __hw_addr_flush(&dev->dev_addrs); dev->dev_addr = NULL; } -EXPORT_SYMBOL(dev_addr_flush); /** * dev_addr_init - Init device address list @@ -547,7 +566,21 @@ int dev_addr_init(struct net_device *dev) } return err; } -EXPORT_SYMBOL(dev_addr_init); + +void dev_addr_mod(struct net_device *dev, unsigned int offset, + const void *addr, size_t len) +{ + struct netdev_hw_addr *ha; + + dev_addr_check(dev); + + ha = container_of(dev->dev_addr, struct netdev_hw_addr, addr[0]); + rb_erase(&ha->node, &dev->dev_addrs.tree); + memcpy(&ha->addr[offset], addr, len); + memcpy(&dev->dev_addr_shadow[offset], addr, len); + WARN_ON(__hw_addr_insert(&dev->dev_addrs, ha, dev->addr_len)); +} +EXPORT_SYMBOL(dev_addr_mod); /** * dev_addr_add - Add a device address diff --git a/net/core/dev_addr_lists_test.c b/net/core/dev_addr_lists_test.c new file mode 100644 index 000000000000..049cfbc58aa9 --- /dev/null +++ b/net/core/dev_addr_lists_test.c @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: GPL-2.0-or-later + +#include <kunit/test.h> +#include <linux/etherdevice.h> +#include <linux/netdevice.h> +#include <linux/rtnetlink.h> + +static const struct net_device_ops dummy_netdev_ops = { +}; + +struct dev_addr_test_priv { + u32 addr_seen; +}; + +static int dev_addr_test_sync(struct net_device *netdev, const unsigned char *a) +{ + struct dev_addr_test_priv *datp = netdev_priv(netdev); + + if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN)) + datp->addr_seen |= 1 << a[0]; + return 0; +} + +static int dev_addr_test_unsync(struct net_device *netdev, + const unsigned char *a) +{ + struct dev_addr_test_priv *datp = netdev_priv(netdev); + + if (a[0] < 31 && !memchr_inv(a, a[0], ETH_ALEN)) + datp->addr_seen &= ~(1 << a[0]); + return 0; +} + +static int dev_addr_test_init(struct kunit *test) +{ + struct dev_addr_test_priv *datp; + struct net_device *netdev; + int err; + + netdev = alloc_etherdev(sizeof(*datp)); + KUNIT_ASSERT_TRUE(test, !!netdev); + + test->priv = netdev; + netdev->netdev_ops = &dummy_netdev_ops; + + err = register_netdev(netdev); + if (err) { + free_netdev(netdev); + KUNIT_FAIL(test, "Can't register netdev %d", err); + } + + rtnl_lock(); + return 0; +} + +static void dev_addr_test_exit(struct kunit *test) +{ + struct net_device *netdev = test->priv; + + rtnl_unlock(); + unregister_netdev(netdev); + free_netdev(netdev); +} + +static void dev_addr_test_basic(struct kunit *test) +{ + struct net_device *netdev = test->priv; + u8 addr[ETH_ALEN]; + + KUNIT_EXPECT_TRUE(test, !!netdev->dev_addr); + + memset(addr, 2, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + KUNIT_EXPECT_EQ(test, 0, memcmp(netdev->dev_addr, addr, sizeof(addr))); + + memset(addr, 3, sizeof(addr)); + dev_addr_set(netdev, addr); + KUNIT_EXPECT_EQ(test, 0, memcmp(netdev->dev_addr, addr, sizeof(addr))); +} + +static void dev_addr_test_sync_one(struct kunit *test) +{ + struct net_device *netdev = test->priv; + struct dev_addr_test_priv *datp; + u8 addr[ETH_ALEN]; + + datp = netdev_priv(netdev); + + memset(addr, 1, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + KUNIT_EXPECT_EQ(test, 2, datp->addr_seen); + + memset(addr, 2, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + + datp->addr_seen = 0; + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + /* It's not going to sync anything because the main address is + * considered synced and we overwrite in place. + */ + KUNIT_EXPECT_EQ(test, 0, datp->addr_seen); +} + +static void dev_addr_test_add_del(struct kunit *test) +{ + struct net_device *netdev = test->priv; + struct dev_addr_test_priv *datp; + u8 addr[ETH_ALEN]; + int i; + + datp = netdev_priv(netdev); + + for (i = 1; i < 4; i++) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, 0, dev_addr_add(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + } + /* Add 3 again */ + KUNIT_EXPECT_EQ(test, 0, dev_addr_add(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + KUNIT_EXPECT_EQ(test, 0xf, datp->addr_seen); + + KUNIT_EXPECT_EQ(test, 0, dev_addr_del(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + KUNIT_EXPECT_EQ(test, 0xf, datp->addr_seen); + + for (i = 1; i < 4; i++) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, 0, dev_addr_del(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + } + + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + KUNIT_EXPECT_EQ(test, 1, datp->addr_seen); +} + +static void dev_addr_test_del_main(struct kunit *test) +{ + struct net_device *netdev = test->priv; + u8 addr[ETH_ALEN]; + + memset(addr, 1, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + + KUNIT_EXPECT_EQ(test, -ENOENT, dev_addr_del(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + KUNIT_EXPECT_EQ(test, 0, dev_addr_add(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + KUNIT_EXPECT_EQ(test, 0, dev_addr_del(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + KUNIT_EXPECT_EQ(test, -ENOENT, dev_addr_del(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); +} + +static void dev_addr_test_add_set(struct kunit *test) +{ + struct net_device *netdev = test->priv; + struct dev_addr_test_priv *datp; + u8 addr[ETH_ALEN]; + int i; + + datp = netdev_priv(netdev); + + /* There is no external API like dev_addr_add_excl(), + * so shuffle the tree a little bit and exploit aliasing. + */ + for (i = 1; i < 16; i++) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, 0, dev_addr_add(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + } + + memset(addr, i, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + KUNIT_EXPECT_EQ(test, 0, dev_addr_add(netdev, addr, + NETDEV_HW_ADDR_T_LAN)); + memset(addr, 0, sizeof(addr)); + eth_hw_addr_set(netdev, addr); + + __hw_addr_sync_dev(&netdev->dev_addrs, netdev, dev_addr_test_sync, + dev_addr_test_unsync); + KUNIT_EXPECT_EQ(test, 0xffff, datp->addr_seen); +} + +static void dev_addr_test_add_excl(struct kunit *test) +{ + struct net_device *netdev = test->priv; + u8 addr[ETH_ALEN]; + int i; + + for (i = 0; i < 10; i++) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, 0, dev_uc_add_excl(netdev, addr)); + } + KUNIT_EXPECT_EQ(test, -EEXIST, dev_uc_add_excl(netdev, addr)); + + for (i = 0; i < 10; i += 2) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, 0, dev_uc_del(netdev, addr)); + } + for (i = 1; i < 10; i += 2) { + memset(addr, i, sizeof(addr)); + KUNIT_EXPECT_EQ(test, -EEXIST, dev_uc_add_excl(netdev, addr)); + } +} + +static struct kunit_case dev_addr_test_cases[] = { + KUNIT_CASE(dev_addr_test_basic), + KUNIT_CASE(dev_addr_test_sync_one), + KUNIT_CASE(dev_addr_test_add_del), + KUNIT_CASE(dev_addr_test_del_main), + KUNIT_CASE(dev_addr_test_add_set), + KUNIT_CASE(dev_addr_test_add_excl), + {} +}; + +static struct kunit_suite dev_addr_test_suite = { + .name = "dev-addr-list-test", + .test_cases = dev_addr_test_cases, + .init = dev_addr_test_init, + .exit = dev_addr_test_exit, +}; +kunit_test_suite(dev_addr_test_suite); + +MODULE_LICENSE("GPL"); diff --git a/net/core/dev_ioctl.c b/net/core/dev_ioctl.c index cbab5fec64b1..1d309a666932 100644 --- a/net/core/dev_ioctl.c +++ b/net/core/dev_ioctl.c @@ -313,6 +313,7 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, void __user *data, int err; struct net_device *dev = __dev_get_by_name(net, ifr->ifr_name); const struct net_device_ops *ops; + netdevice_tracker dev_tracker; if (!dev) return -ENODEV; @@ -381,10 +382,10 @@ static int dev_ifsioc(struct net *net, struct ifreq *ifr, void __user *data, return -ENODEV; if (!netif_is_bridge_master(dev)) return -EOPNOTSUPP; - dev_hold(dev); + dev_hold_track(dev, &dev_tracker, GFP_KERNEL); rtnl_unlock(); err = br_ioctl_call(net, netdev_priv(dev), cmd, ifr, NULL); - dev_put(dev); + dev_put_track(dev, &dev_tracker); rtnl_lock(); return err; diff --git a/net/core/devlink.c b/net/core/devlink.c index c06c9ba6e8c5..0a9349a02cad 100644 --- a/net/core/devlink.c +++ b/net/core/devlink.c @@ -69,6 +69,35 @@ struct devlink { char priv[] __aligned(NETDEV_ALIGN); }; +/** + * struct devlink_resource - devlink resource + * @name: name of the resource + * @id: id, per devlink instance + * @size: size of the resource + * @size_new: updated size of the resource, reload is needed + * @size_valid: valid in case the total size of the resource is valid + * including its children + * @parent: parent resource + * @size_params: size parameters + * @list: parent list + * @resource_list: list of child resources + * @occ_get: occupancy getter callback + * @occ_get_priv: occupancy getter callback priv + */ +struct devlink_resource { + const char *name; + u64 id; + u64 size; + u64 size_new; + bool size_valid; + struct devlink_resource *parent; + struct devlink_resource_size_params size_params; + struct list_head list; + struct list_head resource_list; + devlink_resource_occ_get_t *occ_get; + void *occ_get_priv; +}; + void *devlink_priv(struct devlink *devlink) { return &devlink->priv; @@ -4432,6 +4461,11 @@ static const struct devlink_param devlink_param_generic[] = { .name = DEVLINK_PARAM_GENERIC_ENABLE_VNET_NAME, .type = DEVLINK_PARAM_GENERIC_ENABLE_VNET_TYPE, }, + { + .id = DEVLINK_PARAM_GENERIC_ID_ENABLE_IWARP, + .name = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_NAME, + .type = DEVLINK_PARAM_GENERIC_ENABLE_IWARP_TYPE, + }, }; static int devlink_param_generic_verify(const struct devlink_param *param) @@ -8840,8 +8874,6 @@ static const struct genl_small_ops devlink_nl_ops[] = { GENL_DONT_VALIDATE_DUMP_STRICT, .dumpit = devlink_nl_cmd_health_reporter_dump_get_dumpit, .flags = GENL_ADMIN_PERM, - .internal_flags = DEVLINK_NL_FLAG_NEED_DEVLINK_OR_PORT | - DEVLINK_NL_FLAG_NO_LOCK, }, { .cmd = DEVLINK_CMD_HEALTH_REPORTER_DUMP_CLEAR, @@ -9905,34 +9937,38 @@ out: } EXPORT_SYMBOL_GPL(devlink_resource_register); +static void devlink_resource_unregister(struct devlink *devlink, + struct devlink_resource *resource) +{ + struct devlink_resource *tmp, *child_resource; + + list_for_each_entry_safe(child_resource, tmp, &resource->resource_list, + list) { + devlink_resource_unregister(devlink, child_resource); + list_del(&child_resource->list); + kfree(child_resource); + } +} + /** * devlink_resources_unregister - free all resources * * @devlink: devlink - * @resource: resource */ -void devlink_resources_unregister(struct devlink *devlink, - struct devlink_resource *resource) +void devlink_resources_unregister(struct devlink *devlink) { struct devlink_resource *tmp, *child_resource; - struct list_head *resource_list; - if (resource) - resource_list = &resource->resource_list; - else - resource_list = &devlink->resource_list; - - if (!resource) - mutex_lock(&devlink->lock); + mutex_lock(&devlink->lock); - list_for_each_entry_safe(child_resource, tmp, resource_list, list) { - devlink_resources_unregister(devlink, child_resource); + list_for_each_entry_safe(child_resource, tmp, &devlink->resource_list, + list) { + devlink_resource_unregister(devlink, child_resource); list_del(&child_resource->list); kfree(child_resource); } - if (!resource) - mutex_unlock(&devlink->lock); + mutex_unlock(&devlink->lock); } EXPORT_SYMBOL_GPL(devlink_resources_unregister); diff --git a/net/core/drop_monitor.c b/net/core/drop_monitor.c index 49442cae6f69..3d0ab2eec916 100644 --- a/net/core/drop_monitor.c +++ b/net/core/drop_monitor.c @@ -850,7 +850,7 @@ net_dm_hw_metadata_copy(const struct devlink_trap_metadata *metadata) } hw_metadata->input_dev = metadata->input_dev; - dev_hold(hw_metadata->input_dev); + dev_hold_track(hw_metadata->input_dev, &hw_metadata->dev_tracker, GFP_ATOMIC); return hw_metadata; @@ -864,9 +864,9 @@ free_hw_metadata: } static void -net_dm_hw_metadata_free(const struct devlink_trap_metadata *hw_metadata) +net_dm_hw_metadata_free(struct devlink_trap_metadata *hw_metadata) { - dev_put(hw_metadata->input_dev); + dev_put_track(hw_metadata->input_dev, &hw_metadata->dev_tracker); kfree(hw_metadata->fa_cookie); kfree(hw_metadata->trap_name); kfree(hw_metadata->trap_group_name); diff --git a/net/core/dst.c b/net/core/dst.c index 497ef9b3fc6a..d16c2c9bfebd 100644 --- a/net/core/dst.c +++ b/net/core/dst.c @@ -49,7 +49,7 @@ void dst_init(struct dst_entry *dst, struct dst_ops *ops, unsigned short flags) { dst->dev = dev; - dev_hold(dev); + dev_hold_track(dev, &dst->dev_tracker, GFP_ATOMIC); dst->ops = ops; dst_init_metrics(dst, dst_default_metrics.metrics, true); dst->expires = 0UL; @@ -117,7 +117,7 @@ struct dst_entry *dst_destroy(struct dst_entry * dst) if (dst->ops->destroy) dst->ops->destroy(dst); - dev_put(dst->dev); + dev_put_track(dst->dev, &dst->dev_tracker); lwtstate_put(dst->lwtstate); @@ -159,8 +159,8 @@ void dst_dev_put(struct dst_entry *dst) dst->input = dst_discard; dst->output = dst_discard_out; dst->dev = blackhole_netdev; - dev_hold(dst->dev); - dev_put(dev); + dev_replace_track(dev, blackhole_netdev, &dst->dev_tracker, + GFP_ATOMIC); } EXPORT_SYMBOL(dst_dev_put); diff --git a/net/core/failover.c b/net/core/failover.c index b5cd3c727285..dcaa92a85ea2 100644 --- a/net/core/failover.c +++ b/net/core/failover.c @@ -252,7 +252,7 @@ struct failover *failover_register(struct net_device *dev, return ERR_PTR(-ENOMEM); rcu_assign_pointer(failover->ops, ops); - dev_hold(dev); + dev_hold_track(dev, &failover->dev_tracker, GFP_KERNEL); dev->priv_flags |= IFF_FAILOVER; rcu_assign_pointer(failover->failover_dev, dev); @@ -285,7 +285,7 @@ void failover_unregister(struct failover *failover) failover_dev->name); failover_dev->priv_flags &= ~IFF_FAILOVER; - dev_put(failover_dev); + dev_put_track(failover_dev, &failover->dev_tracker); spin_lock(&failover_lock); list_del(&failover->list); diff --git a/net/core/filter.c b/net/core/filter.c index 6102f093d59a..8271624a19aa 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -301,7 +301,7 @@ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, break; case SKF_AD_PKTTYPE: - *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_TYPE_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, dst_reg, PKT_TYPE_MAX); #ifdef __BIG_ENDIAN_BITFIELD *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, 5); @@ -323,7 +323,7 @@ static u32 convert_skb_access(int skb_field, int dst_reg, int src_reg, offsetof(struct sk_buff, vlan_tci)); break; case SKF_AD_VLAN_TAG_PRESENT: - *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET); if (PKT_VLAN_PRESENT_BIT) *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT); if (PKT_VLAN_PRESENT_BIT < 7) @@ -8029,7 +8029,7 @@ static int bpf_unclone_prologue(struct bpf_insn *insn_buf, bool direct_write, * (Fast-path, otherwise approximation that we might be * a clone, do the rest in helper.) */ - *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET()); + *insn++ = BPF_LDX_MEM(BPF_B, BPF_REG_6, BPF_REG_1, CLONED_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, BPF_REG_6, CLONED_MASK); *insn++ = BPF_JMP_IMM(BPF_JEQ, BPF_REG_6, 0, 7); @@ -8617,7 +8617,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, case offsetof(struct __sk_buff, pkt_type): *target_size = 1; *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, - PKT_TYPE_OFFSET()); + PKT_TYPE_OFFSET); *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, PKT_TYPE_MAX); #ifdef __BIG_ENDIAN_BITFIELD *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, 5); @@ -8642,7 +8642,7 @@ static u32 bpf_convert_ctx_access(enum bpf_access_type type, case offsetof(struct __sk_buff, vlan_present): *target_size = 1; *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg, - PKT_VLAN_PRESENT_OFFSET()); + PKT_VLAN_PRESENT_OFFSET); if (PKT_VLAN_PRESENT_BIT) *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT); if (PKT_VLAN_PRESENT_BIT < 7) @@ -10543,6 +10543,7 @@ static bool sk_lookup_is_valid_access(int off, int size, case bpf_ctx_range_till(struct bpf_sk_lookup, local_ip6[0], local_ip6[3]): case bpf_ctx_range(struct bpf_sk_lookup, remote_port): case bpf_ctx_range(struct bpf_sk_lookup, local_port): + case bpf_ctx_range(struct bpf_sk_lookup, ingress_ifindex): bpf_ctx_record_field_size(info, sizeof(__u32)); return bpf_ctx_narrow_access_ok(off, size, sizeof(__u32)); @@ -10632,6 +10633,12 @@ static u32 sk_lookup_convert_ctx_access(enum bpf_access_type type, bpf_target_off(struct bpf_sk_lookup_kern, dport, 2, target_size)); break; + + case offsetof(struct bpf_sk_lookup, ingress_ifindex): + *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, + bpf_target_off(struct bpf_sk_lookup_kern, + ingress_ifindex, 4, target_size)); + break; } return insn - insn_buf; @@ -10656,14 +10663,10 @@ void bpf_prog_change_xdp(struct bpf_prog *prev_prog, struct bpf_prog *prog) bpf_dispatcher_change_prog(BPF_DISPATCHER_PTR(xdp), prev_prog, prog); } -#ifdef CONFIG_DEBUG_INFO_BTF -BTF_ID_LIST_GLOBAL(btf_sock_ids) +BTF_ID_LIST_GLOBAL(btf_sock_ids, MAX_BTF_SOCK_TYPE) #define BTF_SOCK_TYPE(name, type) BTF_ID(struct, type) BTF_SOCK_TYPE_xxx #undef BTF_SOCK_TYPE -#else -u32 btf_sock_ids[MAX_BTF_SOCK_TYPE]; -#endif BPF_CALL_1(bpf_skc_to_tcp6_sock, struct sock *, sk) { diff --git a/net/core/flow_dissector.c b/net/core/flow_dissector.c index 3255f57f5131..257976cb55ce 100644 --- a/net/core/flow_dissector.c +++ b/net/core/flow_dissector.c @@ -1460,7 +1460,7 @@ out_bad: } EXPORT_SYMBOL(__skb_flow_dissect); -static siphash_key_t hashrnd __read_mostly; +static siphash_aligned_key_t hashrnd; static __always_inline void __flow_hash_secret_init(void) { net_get_random_once(&hashrnd, sizeof(hashrnd)); diff --git a/net/core/gro.c b/net/core/gro.c new file mode 100644 index 000000000000..8ec8b44596da --- /dev/null +++ b/net/core/gro.c @@ -0,0 +1,766 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +#include <net/gro.h> +#include <net/dst_metadata.h> +#include <net/busy_poll.h> +#include <trace/events/net.h> + +#define MAX_GRO_SKBS 8 + +/* This should be increased if a protocol with a bigger head is added. */ +#define GRO_MAX_HEAD (MAX_HEADER + 128) + +static DEFINE_SPINLOCK(offload_lock); +static struct list_head offload_base __read_mostly = LIST_HEAD_INIT(offload_base); +/* Maximum number of GRO_NORMAL skbs to batch up for list-RX */ +int gro_normal_batch __read_mostly = 8; + +/** + * dev_add_offload - register offload handlers + * @po: protocol offload declaration + * + * Add protocol offload handlers to the networking stack. The passed + * &proto_offload is linked into kernel lists and may not be freed until + * it has been removed from the kernel lists. + * + * This call does not sleep therefore it can not + * guarantee all CPU's that are in middle of receiving packets + * will see the new offload handlers (until the next received packet). + */ +void dev_add_offload(struct packet_offload *po) +{ + struct packet_offload *elem; + + spin_lock(&offload_lock); + list_for_each_entry(elem, &offload_base, list) { + if (po->priority < elem->priority) + break; + } + list_add_rcu(&po->list, elem->list.prev); + spin_unlock(&offload_lock); +} +EXPORT_SYMBOL(dev_add_offload); + +/** + * __dev_remove_offload - remove offload handler + * @po: packet offload declaration + * + * Remove a protocol offload handler that was previously added to the + * kernel offload handlers by dev_add_offload(). The passed &offload_type + * is removed from the kernel lists and can be freed or reused once this + * function returns. + * + * The packet type might still be in use by receivers + * and must not be freed until after all the CPU's have gone + * through a quiescent state. + */ +static void __dev_remove_offload(struct packet_offload *po) +{ + struct list_head *head = &offload_base; + struct packet_offload *po1; + + spin_lock(&offload_lock); + + list_for_each_entry(po1, head, list) { + if (po == po1) { + list_del_rcu(&po->list); + goto out; + } + } + + pr_warn("dev_remove_offload: %p not found\n", po); +out: + spin_unlock(&offload_lock); +} + +/** + * dev_remove_offload - remove packet offload handler + * @po: packet offload declaration + * + * Remove a packet offload handler that was previously added to the kernel + * offload handlers by dev_add_offload(). The passed &offload_type is + * removed from the kernel lists and can be freed or reused once this + * function returns. + * + * This call sleeps to guarantee that no CPU is looking at the packet + * type after return. + */ +void dev_remove_offload(struct packet_offload *po) +{ + __dev_remove_offload(po); + + synchronize_net(); +} +EXPORT_SYMBOL(dev_remove_offload); + +/** + * skb_mac_gso_segment - mac layer segmentation handler. + * @skb: buffer to segment + * @features: features for the output path (see dev->features) + */ +struct sk_buff *skb_mac_gso_segment(struct sk_buff *skb, + netdev_features_t features) +{ + struct sk_buff *segs = ERR_PTR(-EPROTONOSUPPORT); + struct packet_offload *ptype; + int vlan_depth = skb->mac_len; + __be16 type = skb_network_protocol(skb, &vlan_depth); + + if (unlikely(!type)) + return ERR_PTR(-EINVAL); + + __skb_pull(skb, vlan_depth); + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, &offload_base, list) { + if (ptype->type == type && ptype->callbacks.gso_segment) { + segs = ptype->callbacks.gso_segment(skb, features); + break; + } + } + rcu_read_unlock(); + + __skb_push(skb, skb->data - skb_mac_header(skb)); + + return segs; +} +EXPORT_SYMBOL(skb_mac_gso_segment); + +int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) +{ + struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); + unsigned int offset = skb_gro_offset(skb); + unsigned int headlen = skb_headlen(skb); + unsigned int len = skb_gro_len(skb); + unsigned int delta_truesize; + unsigned int new_truesize; + struct sk_buff *lp; + + if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) + return -E2BIG; + + lp = NAPI_GRO_CB(p)->last; + pinfo = skb_shinfo(lp); + + if (headlen <= offset) { + skb_frag_t *frag; + skb_frag_t *frag2; + int i = skbinfo->nr_frags; + int nr_frags = pinfo->nr_frags + i; + + if (nr_frags > MAX_SKB_FRAGS) + goto merge; + + offset -= headlen; + pinfo->nr_frags = nr_frags; + skbinfo->nr_frags = 0; + + frag = pinfo->frags + nr_frags; + frag2 = skbinfo->frags + i; + do { + *--frag = *--frag2; + } while (--i); + + skb_frag_off_add(frag, offset); + skb_frag_size_sub(frag, offset); + + /* all fragments truesize : remove (head size + sk_buff) */ + new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); + delta_truesize = skb->truesize - new_truesize; + + skb->truesize = new_truesize; + skb->len -= skb->data_len; + skb->data_len = 0; + + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; + goto done; + } else if (skb->head_frag) { + int nr_frags = pinfo->nr_frags; + skb_frag_t *frag = pinfo->frags + nr_frags; + struct page *page = virt_to_head_page(skb->head); + unsigned int first_size = headlen - offset; + unsigned int first_offset; + + if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) + goto merge; + + first_offset = skb->data - + (unsigned char *)page_address(page) + + offset; + + pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; + + __skb_frag_set_page(frag, page); + skb_frag_off_set(frag, first_offset); + skb_frag_size_set(frag, first_size); + + memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); + /* We dont need to clear skbinfo->nr_frags here */ + + new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); + delta_truesize = skb->truesize - new_truesize; + skb->truesize = new_truesize; + NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; + goto done; + } + +merge: + /* sk owenrship - if any - completely transferred to the aggregated packet */ + skb->destructor = NULL; + delta_truesize = skb->truesize; + if (offset > headlen) { + unsigned int eat = offset - headlen; + + skb_frag_off_add(&skbinfo->frags[0], eat); + skb_frag_size_sub(&skbinfo->frags[0], eat); + skb->data_len -= eat; + skb->len -= eat; + offset = headlen; + } + + __skb_pull(skb, offset); + + if (NAPI_GRO_CB(p)->last == p) + skb_shinfo(p)->frag_list = skb; + else + NAPI_GRO_CB(p)->last->next = skb; + NAPI_GRO_CB(p)->last = skb; + __skb_header_release(skb); + lp = p; + +done: + NAPI_GRO_CB(p)->count++; + p->data_len += len; + p->truesize += delta_truesize; + p->len += len; + if (lp != p) { + lp->data_len += len; + lp->truesize += delta_truesize; + lp->len += len; + } + NAPI_GRO_CB(skb)->same_flow = 1; + return 0; +} + + +static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb) +{ + struct packet_offload *ptype; + __be16 type = skb->protocol; + struct list_head *head = &offload_base; + int err = -ENOENT; + + BUILD_BUG_ON(sizeof(struct napi_gro_cb) > sizeof(skb->cb)); + + if (NAPI_GRO_CB(skb)->count == 1) { + skb_shinfo(skb)->gso_size = 0; + goto out; + } + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, head, list) { + if (ptype->type != type || !ptype->callbacks.gro_complete) + continue; + + err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, + ipv6_gro_complete, inet_gro_complete, + skb, 0); + break; + } + rcu_read_unlock(); + + if (err) { + WARN_ON(&ptype->list == head); + kfree_skb(skb); + return; + } + +out: + gro_normal_one(napi, skb, NAPI_GRO_CB(skb)->count); +} + +static void __napi_gro_flush_chain(struct napi_struct *napi, u32 index, + bool flush_old) +{ + struct list_head *head = &napi->gro_hash[index].list; + struct sk_buff *skb, *p; + + list_for_each_entry_safe_reverse(skb, p, head, list) { + if (flush_old && NAPI_GRO_CB(skb)->age == jiffies) + return; + skb_list_del_init(skb); + napi_gro_complete(napi, skb); + napi->gro_hash[index].count--; + } + + if (!napi->gro_hash[index].count) + __clear_bit(index, &napi->gro_bitmask); +} + +/* napi->gro_hash[].list contains packets ordered by age. + * youngest packets at the head of it. + * Complete skbs in reverse order to reduce latencies. + */ +void napi_gro_flush(struct napi_struct *napi, bool flush_old) +{ + unsigned long bitmask = napi->gro_bitmask; + unsigned int i, base = ~0U; + + while ((i = ffs(bitmask)) != 0) { + bitmask >>= i; + base += i; + __napi_gro_flush_chain(napi, base, flush_old); + } +} +EXPORT_SYMBOL(napi_gro_flush); + +static void gro_list_prepare(const struct list_head *head, + const struct sk_buff *skb) +{ + unsigned int maclen = skb->dev->hard_header_len; + u32 hash = skb_get_hash_raw(skb); + struct sk_buff *p; + + list_for_each_entry(p, head, list) { + unsigned long diffs; + + NAPI_GRO_CB(p)->flush = 0; + + if (hash != skb_get_hash_raw(p)) { + NAPI_GRO_CB(p)->same_flow = 0; + continue; + } + + diffs = (unsigned long)p->dev ^ (unsigned long)skb->dev; + diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb); + if (skb_vlan_tag_present(p)) + diffs |= skb_vlan_tag_get(p) ^ skb_vlan_tag_get(skb); + diffs |= skb_metadata_differs(p, skb); + if (maclen == ETH_HLEN) + diffs |= compare_ether_header(skb_mac_header(p), + skb_mac_header(skb)); + else if (!diffs) + diffs = memcmp(skb_mac_header(p), + skb_mac_header(skb), + maclen); + + /* in most common scenarions 'slow_gro' is 0 + * otherwise we are already on some slower paths + * either skip all the infrequent tests altogether or + * avoid trying too hard to skip each of them individually + */ + if (!diffs && unlikely(skb->slow_gro | p->slow_gro)) { +#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) + struct tc_skb_ext *skb_ext; + struct tc_skb_ext *p_ext; +#endif + + diffs |= p->sk != skb->sk; + diffs |= skb_metadata_dst_cmp(p, skb); + diffs |= skb_get_nfct(p) ^ skb_get_nfct(skb); + +#if IS_ENABLED(CONFIG_SKB_EXTENSIONS) && IS_ENABLED(CONFIG_NET_TC_SKB_EXT) + skb_ext = skb_ext_find(skb, TC_SKB_EXT); + p_ext = skb_ext_find(p, TC_SKB_EXT); + + diffs |= (!!p_ext) ^ (!!skb_ext); + if (!diffs && unlikely(skb_ext)) + diffs |= p_ext->chain ^ skb_ext->chain; +#endif + } + + NAPI_GRO_CB(p)->same_flow = !diffs; + } +} + +static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) +{ + const struct skb_shared_info *pinfo = skb_shinfo(skb); + const skb_frag_t *frag0 = &pinfo->frags[0]; + + NAPI_GRO_CB(skb)->data_offset = 0; + NAPI_GRO_CB(skb)->frag0 = NULL; + NAPI_GRO_CB(skb)->frag0_len = 0; + + if (!skb_headlen(skb) && pinfo->nr_frags && + !PageHighMem(skb_frag_page(frag0)) && + (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { + NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); + NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, + skb_frag_size(frag0), + skb->end - skb->tail); + } +} + +static void gro_pull_from_frag0(struct sk_buff *skb, int grow) +{ + struct skb_shared_info *pinfo = skb_shinfo(skb); + + BUG_ON(skb->end - skb->tail < grow); + + memcpy(skb_tail_pointer(skb), NAPI_GRO_CB(skb)->frag0, grow); + + skb->data_len -= grow; + skb->tail += grow; + + skb_frag_off_add(&pinfo->frags[0], grow); + skb_frag_size_sub(&pinfo->frags[0], grow); + + if (unlikely(!skb_frag_size(&pinfo->frags[0]))) { + skb_frag_unref(skb, 0); + memmove(pinfo->frags, pinfo->frags + 1, + --pinfo->nr_frags * sizeof(pinfo->frags[0])); + } +} + +static void gro_flush_oldest(struct napi_struct *napi, struct list_head *head) +{ + struct sk_buff *oldest; + + oldest = list_last_entry(head, struct sk_buff, list); + + /* We are called with head length >= MAX_GRO_SKBS, so this is + * impossible. + */ + if (WARN_ON_ONCE(!oldest)) + return; + + /* Do not adjust napi->gro_hash[].count, caller is adding a new + * SKB to the chain. + */ + skb_list_del_init(oldest); + napi_gro_complete(napi, oldest); +} + +static enum gro_result dev_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +{ + u32 bucket = skb_get_hash_raw(skb) & (GRO_HASH_BUCKETS - 1); + struct gro_list *gro_list = &napi->gro_hash[bucket]; + struct list_head *head = &offload_base; + struct packet_offload *ptype; + __be16 type = skb->protocol; + struct sk_buff *pp = NULL; + enum gro_result ret; + int same_flow; + int grow; + + if (netif_elide_gro(skb->dev)) + goto normal; + + gro_list_prepare(&gro_list->list, skb); + + rcu_read_lock(); + list_for_each_entry_rcu(ptype, head, list) { + if (ptype->type != type || !ptype->callbacks.gro_receive) + continue; + + skb_set_network_header(skb, skb_gro_offset(skb)); + skb_reset_mac_len(skb); + NAPI_GRO_CB(skb)->same_flow = 0; + NAPI_GRO_CB(skb)->flush = skb_is_gso(skb) || skb_has_frag_list(skb); + NAPI_GRO_CB(skb)->free = 0; + NAPI_GRO_CB(skb)->encap_mark = 0; + NAPI_GRO_CB(skb)->recursion_counter = 0; + NAPI_GRO_CB(skb)->is_fou = 0; + NAPI_GRO_CB(skb)->is_atomic = 1; + NAPI_GRO_CB(skb)->gro_remcsum_start = 0; + + /* Setup for GRO checksum validation */ + switch (skb->ip_summed) { + case CHECKSUM_COMPLETE: + NAPI_GRO_CB(skb)->csum = skb->csum; + NAPI_GRO_CB(skb)->csum_valid = 1; + NAPI_GRO_CB(skb)->csum_cnt = 0; + break; + case CHECKSUM_UNNECESSARY: + NAPI_GRO_CB(skb)->csum_cnt = skb->csum_level + 1; + NAPI_GRO_CB(skb)->csum_valid = 0; + break; + default: + NAPI_GRO_CB(skb)->csum_cnt = 0; + NAPI_GRO_CB(skb)->csum_valid = 0; + } + + pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive, + ipv6_gro_receive, inet_gro_receive, + &gro_list->list, skb); + break; + } + rcu_read_unlock(); + + if (&ptype->list == head) + goto normal; + + if (PTR_ERR(pp) == -EINPROGRESS) { + ret = GRO_CONSUMED; + goto ok; + } + + same_flow = NAPI_GRO_CB(skb)->same_flow; + ret = NAPI_GRO_CB(skb)->free ? GRO_MERGED_FREE : GRO_MERGED; + + if (pp) { + skb_list_del_init(pp); + napi_gro_complete(napi, pp); + gro_list->count--; + } + + if (same_flow) + goto ok; + + if (NAPI_GRO_CB(skb)->flush) + goto normal; + + if (unlikely(gro_list->count >= MAX_GRO_SKBS)) + gro_flush_oldest(napi, &gro_list->list); + else + gro_list->count++; + + NAPI_GRO_CB(skb)->count = 1; + NAPI_GRO_CB(skb)->age = jiffies; + NAPI_GRO_CB(skb)->last = skb; + skb_shinfo(skb)->gso_size = skb_gro_len(skb); + list_add(&skb->list, &gro_list->list); + ret = GRO_HELD; + +pull: + grow = skb_gro_offset(skb) - skb_headlen(skb); + if (grow > 0) + gro_pull_from_frag0(skb, grow); +ok: + if (gro_list->count) { + if (!test_bit(bucket, &napi->gro_bitmask)) + __set_bit(bucket, &napi->gro_bitmask); + } else if (test_bit(bucket, &napi->gro_bitmask)) { + __clear_bit(bucket, &napi->gro_bitmask); + } + + return ret; + +normal: + ret = GRO_NORMAL; + goto pull; +} + +struct packet_offload *gro_find_receive_by_type(__be16 type) +{ + struct list_head *offload_head = &offload_base; + struct packet_offload *ptype; + + list_for_each_entry_rcu(ptype, offload_head, list) { + if (ptype->type != type || !ptype->callbacks.gro_receive) + continue; + return ptype; + } + return NULL; +} +EXPORT_SYMBOL(gro_find_receive_by_type); + +struct packet_offload *gro_find_complete_by_type(__be16 type) +{ + struct list_head *offload_head = &offload_base; + struct packet_offload *ptype; + + list_for_each_entry_rcu(ptype, offload_head, list) { + if (ptype->type != type || !ptype->callbacks.gro_complete) + continue; + return ptype; + } + return NULL; +} +EXPORT_SYMBOL(gro_find_complete_by_type); + +static gro_result_t napi_skb_finish(struct napi_struct *napi, + struct sk_buff *skb, + gro_result_t ret) +{ + switch (ret) { + case GRO_NORMAL: + gro_normal_one(napi, skb, 1); + break; + + case GRO_MERGED_FREE: + if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) + napi_skb_free_stolen_head(skb); + else if (skb->fclone != SKB_FCLONE_UNAVAILABLE) + __kfree_skb(skb); + else + __kfree_skb_defer(skb); + break; + + case GRO_HELD: + case GRO_MERGED: + case GRO_CONSUMED: + break; + } + + return ret; +} + +gro_result_t napi_gro_receive(struct napi_struct *napi, struct sk_buff *skb) +{ + gro_result_t ret; + + skb_mark_napi_id(skb, napi); + trace_napi_gro_receive_entry(skb); + + skb_gro_reset_offset(skb, 0); + + ret = napi_skb_finish(napi, skb, dev_gro_receive(napi, skb)); + trace_napi_gro_receive_exit(ret); + + return ret; +} +EXPORT_SYMBOL(napi_gro_receive); + +static void napi_reuse_skb(struct napi_struct *napi, struct sk_buff *skb) +{ + if (unlikely(skb->pfmemalloc)) { + consume_skb(skb); + return; + } + __skb_pull(skb, skb_headlen(skb)); + /* restore the reserve we had after netdev_alloc_skb_ip_align() */ + skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN - skb_headroom(skb)); + __vlan_hwaccel_clear_tag(skb); + skb->dev = napi->dev; + skb->skb_iif = 0; + + /* eth_type_trans() assumes pkt_type is PACKET_HOST */ + skb->pkt_type = PACKET_HOST; + + skb->encapsulation = 0; + skb_shinfo(skb)->gso_type = 0; + skb->truesize = SKB_TRUESIZE(skb_end_offset(skb)); + if (unlikely(skb->slow_gro)) { + skb_orphan(skb); + skb_ext_reset(skb); + nf_reset_ct(skb); + skb->slow_gro = 0; + } + + napi->skb = skb; +} + +struct sk_buff *napi_get_frags(struct napi_struct *napi) +{ + struct sk_buff *skb = napi->skb; + + if (!skb) { + skb = napi_alloc_skb(napi, GRO_MAX_HEAD); + if (skb) { + napi->skb = skb; + skb_mark_napi_id(skb, napi); + } + } + return skb; +} +EXPORT_SYMBOL(napi_get_frags); + +static gro_result_t napi_frags_finish(struct napi_struct *napi, + struct sk_buff *skb, + gro_result_t ret) +{ + switch (ret) { + case GRO_NORMAL: + case GRO_HELD: + __skb_push(skb, ETH_HLEN); + skb->protocol = eth_type_trans(skb, skb->dev); + if (ret == GRO_NORMAL) + gro_normal_one(napi, skb, 1); + break; + + case GRO_MERGED_FREE: + if (NAPI_GRO_CB(skb)->free == NAPI_GRO_FREE_STOLEN_HEAD) + napi_skb_free_stolen_head(skb); + else + napi_reuse_skb(napi, skb); + break; + + case GRO_MERGED: + case GRO_CONSUMED: + break; + } + + return ret; +} + +/* Upper GRO stack assumes network header starts at gro_offset=0 + * Drivers could call both napi_gro_frags() and napi_gro_receive() + * We copy ethernet header into skb->data to have a common layout. + */ +static struct sk_buff *napi_frags_skb(struct napi_struct *napi) +{ + struct sk_buff *skb = napi->skb; + const struct ethhdr *eth; + unsigned int hlen = sizeof(*eth); + + napi->skb = NULL; + + skb_reset_mac_header(skb); + skb_gro_reset_offset(skb, hlen); + + if (unlikely(skb_gro_header_hard(skb, hlen))) { + eth = skb_gro_header_slow(skb, hlen, 0); + if (unlikely(!eth)) { + net_warn_ratelimited("%s: dropping impossible skb from %s\n", + __func__, napi->dev->name); + napi_reuse_skb(napi, skb); + return NULL; + } + } else { + eth = (const struct ethhdr *)skb->data; + gro_pull_from_frag0(skb, hlen); + NAPI_GRO_CB(skb)->frag0 += hlen; + NAPI_GRO_CB(skb)->frag0_len -= hlen; + } + __skb_pull(skb, hlen); + + /* + * This works because the only protocols we care about don't require + * special handling. + * We'll fix it up properly in napi_frags_finish() + */ + skb->protocol = eth->h_proto; + + return skb; +} + +gro_result_t napi_gro_frags(struct napi_struct *napi) +{ + gro_result_t ret; + struct sk_buff *skb = napi_frags_skb(napi); + + trace_napi_gro_frags_entry(skb); + + ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb)); + trace_napi_gro_frags_exit(ret); + + return ret; +} +EXPORT_SYMBOL(napi_gro_frags); + +/* Compute the checksum from gro_offset and return the folded value + * after adding in any pseudo checksum. + */ +__sum16 __skb_gro_checksum_complete(struct sk_buff *skb) +{ + __wsum wsum; + __sum16 sum; + + wsum = skb_checksum(skb, skb_gro_offset(skb), skb_gro_len(skb), 0); + + /* NAPI_GRO_CB(skb)->csum holds pseudo checksum */ + sum = csum_fold(csum_add(NAPI_GRO_CB(skb)->csum, wsum)); + /* See comments in __skb_checksum_complete(). */ + if (likely(!sum)) { + if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) && + !skb->csum_complete_sw) + netdev_rx_csum_fault(skb->dev, skb); + } + + NAPI_GRO_CB(skb)->csum = wsum; + NAPI_GRO_CB(skb)->csum_valid = 1; + + return sum; +} +EXPORT_SYMBOL(__skb_gro_checksum_complete); diff --git a/net/core/link_watch.c b/net/core/link_watch.c index 1a455847da54..d7d089963b1d 100644 --- a/net/core/link_watch.c +++ b/net/core/link_watch.c @@ -55,7 +55,7 @@ static void rfc2863_policy(struct net_device *dev) if (operstate == dev->operstate) return; - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); switch(dev->link_mode) { case IF_LINK_MODE_TESTING: @@ -74,7 +74,7 @@ static void rfc2863_policy(struct net_device *dev) dev->operstate = operstate; - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); } @@ -109,7 +109,7 @@ static void linkwatch_add_event(struct net_device *dev) spin_lock_irqsave(&lweventlist_lock, flags); if (list_empty(&dev->link_watch_list)) { list_add_tail(&dev->link_watch_list, &lweventlist); - dev_hold(dev); + dev_hold_track(dev, &dev->linkwatch_dev_tracker, GFP_ATOMIC); } spin_unlock_irqrestore(&lweventlist_lock, flags); } @@ -166,7 +166,7 @@ static void linkwatch_do_dev(struct net_device *dev) netdev_state_change(dev); } - dev_put(dev); + dev_put_track(dev, &dev->linkwatch_dev_tracker); } static void __linkwatch_run_queue(int urgent_only) diff --git a/net/core/neighbour.c b/net/core/neighbour.c index dda12fbd177b..506aa01776df 100644 --- a/net/core/neighbour.c +++ b/net/core/neighbour.c @@ -624,7 +624,7 @@ ___neigh_create(struct neigh_table *tbl, const void *pkey, memcpy(n->primary_key, pkey, key_len); n->dev = dev; - dev_hold(dev); + dev_hold_track(dev, &n->dev_tracker, GFP_ATOMIC); /* Protocol specific setup. */ if (tbl->constructor && (error = tbl->constructor(n)) < 0) { @@ -770,10 +770,10 @@ struct pneigh_entry * pneigh_lookup(struct neigh_table *tbl, write_pnet(&n->net, net); memcpy(n->key, pkey, key_len); n->dev = dev; - dev_hold(dev); + dev_hold_track(dev, &n->dev_tracker, GFP_KERNEL); if (tbl->pconstructor && tbl->pconstructor(n)) { - dev_put(dev); + dev_put_track(dev, &n->dev_tracker); kfree(n); n = NULL; goto out; @@ -805,7 +805,7 @@ int pneigh_delete(struct neigh_table *tbl, struct net *net, const void *pkey, write_unlock_bh(&tbl->lock); if (tbl->pdestructor) tbl->pdestructor(n); - dev_put(n->dev); + dev_put_track(n->dev, &n->dev_tracker); kfree(n); return 0; } @@ -838,7 +838,7 @@ static int pneigh_ifdown_and_unlock(struct neigh_table *tbl, n->next = NULL; if (tbl->pdestructor) tbl->pdestructor(n); - dev_put(n->dev); + dev_put_track(n->dev, &n->dev_tracker); kfree(n); } return -ENOENT; @@ -879,7 +879,7 @@ void neigh_destroy(struct neighbour *neigh) if (dev->netdev_ops->ndo_neigh_destroy) dev->netdev_ops->ndo_neigh_destroy(dev, neigh); - dev_put(dev); + dev_put_track(dev, &neigh->dev_tracker); neigh_parms_put(neigh->parms); neigh_dbg(2, "neigh %p is destroyed\n", neigh); @@ -1665,13 +1665,13 @@ struct neigh_parms *neigh_parms_alloc(struct net_device *dev, refcount_set(&p->refcnt, 1); p->reachable_time = neigh_rand_reach_time(NEIGH_VAR(p, BASE_REACHABLE_TIME)); - dev_hold(dev); + dev_hold_track(dev, &p->dev_tracker, GFP_KERNEL); p->dev = dev; write_pnet(&p->net, net); p->sysctl_table = NULL; if (ops->ndo_neigh_setup && ops->ndo_neigh_setup(dev, p)) { - dev_put(dev); + dev_put_track(dev, &p->dev_tracker); kfree(p); return NULL; } @@ -1702,7 +1702,7 @@ void neigh_parms_release(struct neigh_table *tbl, struct neigh_parms *parms) list_del(&parms->list); parms->dead = 1; write_unlock_bh(&tbl->lock); - dev_put(parms->dev); + dev_put_track(parms->dev, &parms->dev_tracker); call_rcu(&parms->rcu_head, neigh_rcu_free_parms); } EXPORT_SYMBOL(neigh_parms_release); diff --git a/net/core/net-sysfs.c b/net/core/net-sysfs.c index 9c01c642cf9e..53ea262ecafd 100644 --- a/net/core/net-sysfs.c +++ b/net/core/net-sysfs.c @@ -488,14 +488,6 @@ static ssize_t proto_down_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { - struct net_device *netdev = to_net_dev(dev); - - /* The check is also done in change_proto_down; this helps returning - * early without hitting the trylock/restart in netdev_store. - */ - if (!netdev->netdev_ops->ndo_change_proto_down) - return -EOPNOTSUPP; - return netdev_store(dev, attr, buf, len, change_proto_down); } NETDEVICE_SHOW_RW(proto_down, fmt_dec); @@ -1012,7 +1004,7 @@ static void rx_queue_release(struct kobject *kobj) #endif memset(kobj, 0, sizeof(*kobj)); - dev_put(queue->dev); + dev_put_track(queue->dev, &queue->dev_tracker); } static const void *rx_queue_namespace(struct kobject *kobj) @@ -1052,7 +1044,7 @@ static int rx_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger rx_queue_release call which * decreases dev refcount: Take that reference here */ - dev_hold(queue->dev); + dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &rx_queue_ktype, NULL, @@ -1201,11 +1193,7 @@ static const struct sysfs_ops netdev_queue_sysfs_ops = { static ssize_t tx_timeout_show(struct netdev_queue *queue, char *buf) { - unsigned long trans_timeout; - - spin_lock_irq(&queue->_xmit_lock); - trans_timeout = queue->trans_timeout; - spin_unlock_irq(&queue->_xmit_lock); + unsigned long trans_timeout = atomic_long_read(&queue->trans_timeout); return sprintf(buf, fmt_ulong, trans_timeout); } @@ -1452,7 +1440,7 @@ static ssize_t xps_queue_show(struct net_device *dev, unsigned int index, for (i = map->len; i--;) { if (map->queues[i] == index) { - set_bit(j, mask); + __set_bit(j, mask); break; } } @@ -1619,7 +1607,7 @@ static void netdev_queue_release(struct kobject *kobj) struct netdev_queue *queue = to_netdev_queue(kobj); memset(kobj, 0, sizeof(*kobj)); - dev_put(queue->dev); + dev_put_track(queue->dev, &queue->dev_tracker); } static const void *netdev_queue_namespace(struct kobject *kobj) @@ -1659,7 +1647,7 @@ static int netdev_queue_add_kobject(struct net_device *dev, int index) /* Kobject_put later will trigger netdev_queue_release call * which decreases dev refcount: Take that reference here */ - dev_hold(queue->dev); + dev_hold_track(queue->dev, &queue->dev_tracker, GFP_KERNEL); kobj->kset = dev->queues_kset; error = kobject_init_and_add(kobj, &netdev_queue_ktype, NULL, @@ -1706,6 +1694,13 @@ netdev_queue_update_kobjects(struct net_device *dev, int old_num, int new_num) int i; int error = 0; + /* Tx queue kobjects are allowed to be updated when a device is being + * unregistered, but solely to remove queues from qdiscs. Any path + * adding queues should be fixed. + */ + WARN(dev->reg_state == NETREG_UNREGISTERING && new_num > old_num, + "New queues can't be registered after device unregistration."); + for (i = old_num; i < new_num; i++) { error = netdev_queue_add_kobject(dev, i); if (error) { @@ -1820,6 +1815,9 @@ static void remove_queue_kobjects(struct net_device *dev) net_rx_queue_update_kobjects(dev, real_rx, 0); netdev_queue_update_kobjects(dev, real_tx, 0); + + dev->real_num_rx_queues = 0; + dev->real_num_tx_queues = 0; #ifdef CONFIG_SYSFS kset_unregister(dev->queues_kset); #endif diff --git a/net/core/netpoll.c b/net/core/netpoll.c index edfc0f8011f8..db724463e7cd 100644 --- a/net/core/netpoll.c +++ b/net/core/netpoll.c @@ -776,7 +776,7 @@ put_noaddr: err = __netpoll_setup(np, ndev); if (err) goto put; - + netdev_tracker_alloc(ndev, &np->dev_tracker, GFP_KERNEL); rtnl_unlock(); return 0; @@ -853,7 +853,7 @@ void netpoll_cleanup(struct netpoll *np) if (!np->dev) goto out; __netpoll_cleanup(np); - dev_put(np->dev); + dev_put_track(np->dev, &np->dev_tracker); np->dev = NULL; out: rtnl_unlock(); diff --git a/net/core/pktgen.c b/net/core/pktgen.c index a3d74e2704c4..560a5e712dc3 100644 --- a/net/core/pktgen.c +++ b/net/core/pktgen.c @@ -410,6 +410,7 @@ struct pktgen_dev { * device name (not when the inject is * started as it used to do.) */ + netdevice_tracker dev_tracker; char odevname[32]; struct flow_state *flows; unsigned int cflows; /* Concurrent flows (config) */ @@ -2099,7 +2100,7 @@ static int pktgen_setup_dev(const struct pktgen_net *pn, /* Clean old setups */ if (pkt_dev->odev) { - dev_put(pkt_dev->odev); + dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); pkt_dev->odev = NULL; } @@ -2117,6 +2118,7 @@ static int pktgen_setup_dev(const struct pktgen_net *pn, err = -ENETDOWN; } else { pkt_dev->odev = odev; + netdev_tracker_alloc(odev, &pkt_dev->dev_tracker, GFP_KERNEL); return 0; } @@ -3805,7 +3807,7 @@ static int pktgen_add_device(struct pktgen_thread *t, const char *ifname) return add_dev_to_thread(t, pkt_dev); out2: - dev_put(pkt_dev->odev); + dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); out1: #ifdef CONFIG_XFRM free_SAs(pkt_dev); @@ -3899,7 +3901,7 @@ static int pktgen_remove_device(struct pktgen_thread *t, /* Dis-associate from the interface */ if (pkt_dev->odev) { - dev_put(pkt_dev->odev); + dev_put_track(pkt_dev->odev, &pkt_dev->dev_tracker); pkt_dev->odev = NULL; } diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c index 2af8aeeadadf..d6eba554b137 100644 --- a/net/core/rtnetlink.c +++ b/net/core/rtnetlink.c @@ -842,9 +842,9 @@ static void set_operstate(struct net_device *dev, unsigned char transition) } if (dev->operstate != operstate) { - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); dev->operstate = operstate; - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); netdev_state_change(dev); } } @@ -2539,13 +2539,12 @@ static int do_set_proto_down(struct net_device *dev, struct netlink_ext_ack *extack) { struct nlattr *pdreason[IFLA_PROTO_DOWN_REASON_MAX + 1]; - const struct net_device_ops *ops = dev->netdev_ops; unsigned long mask = 0; u32 value; bool proto_down; int err; - if (!ops->ndo_change_proto_down) { + if (!(dev->priv_flags & IFF_CHANGE_PROTO_DOWN)) { NL_SET_ERR_MSG(extack, "Protodown not supported by device"); return -EOPNOTSUPP; } @@ -2768,7 +2767,7 @@ static int do_setlink(const struct sk_buff *skb, } if (dev->gso_max_segs ^ max_segs) { - dev->gso_max_segs = max_segs; + netif_set_gso_max_segs(dev, max_segs); status |= DO_SETLINK_MODIFIED; } } @@ -2779,11 +2778,11 @@ static int do_setlink(const struct sk_buff *skb, if (tb[IFLA_LINKMODE]) { unsigned char value = nla_get_u8(tb[IFLA_LINKMODE]); - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); if (dev->link_mode ^ value) status |= DO_SETLINK_NOTIFY; dev->link_mode = value; - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); } if (tb[IFLA_VFINFO_LIST]) { @@ -3222,7 +3221,7 @@ struct net_device *rtnl_create_link(struct net *net, const char *ifname, if (tb[IFLA_GSO_MAX_SIZE]) netif_set_gso_max_size(dev, nla_get_u32(tb[IFLA_GSO_MAX_SIZE])); if (tb[IFLA_GSO_MAX_SEGS]) - dev->gso_max_segs = nla_get_u32(tb[IFLA_GSO_MAX_SEGS]); + netif_set_gso_max_segs(dev, nla_get_u32(tb[IFLA_GSO_MAX_SEGS])); return dev; } diff --git a/net/core/secure_seq.c b/net/core/secure_seq.c index b5bc680d4755..9b8443774449 100644 --- a/net/core/secure_seq.c +++ b/net/core/secure_seq.c @@ -19,8 +19,8 @@ #include <linux/in6.h> #include <net/tcp.h> -static siphash_key_t net_secret __read_mostly; -static siphash_key_t ts_secret __read_mostly; +static siphash_aligned_key_t net_secret; +static siphash_aligned_key_t ts_secret; static __always_inline void net_secret_init(void) { diff --git a/net/core/skbuff.c b/net/core/skbuff.c index ba2f38246f07..a33247fdb8f5 100644 --- a/net/core/skbuff.c +++ b/net/core/skbuff.c @@ -992,12 +992,10 @@ void napi_consume_skb(struct sk_buff *skb, int budget) } EXPORT_SYMBOL(napi_consume_skb); -/* Make sure a field is enclosed inside headers_start/headers_end section */ +/* Make sure a field is contained by headers group */ #define CHECK_SKB_FIELD(field) \ - BUILD_BUG_ON(offsetof(struct sk_buff, field) < \ - offsetof(struct sk_buff, headers_start)); \ - BUILD_BUG_ON(offsetof(struct sk_buff, field) > \ - offsetof(struct sk_buff, headers_end)); \ + BUILD_BUG_ON(offsetof(struct sk_buff, field) != \ + offsetof(struct sk_buff, headers.field)); \ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) { @@ -1009,14 +1007,12 @@ static void __copy_skb_header(struct sk_buff *new, const struct sk_buff *old) __skb_ext_copy(new, old); __nf_copy(new, old, false); - /* Note : this field could be in headers_start/headers_end section + /* Note : this field could be in the headers group. * It is not yet because we do not want to have a 16 bit hole */ new->queue_mapping = old->queue_mapping; - memcpy(&new->headers_start, &old->headers_start, - offsetof(struct sk_buff, headers_end) - - offsetof(struct sk_buff, headers_start)); + memcpy(&new->headers, &old->headers, sizeof(new->headers)); CHECK_SKB_FIELD(protocol); CHECK_SKB_FIELD(csum); CHECK_SKB_FIELD(hash); @@ -3919,32 +3915,6 @@ err_linearize: } EXPORT_SYMBOL_GPL(skb_segment_list); -int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) -{ - if (unlikely(p->len + skb->len >= 65536)) - return -E2BIG; - - if (NAPI_GRO_CB(p)->last == p) - skb_shinfo(p)->frag_list = skb; - else - NAPI_GRO_CB(p)->last->next = skb; - - skb_pull(skb, skb_gro_offset(skb)); - - NAPI_GRO_CB(p)->last = skb; - NAPI_GRO_CB(p)->count++; - p->data_len += skb->len; - - /* sk owenrship - if any - completely transferred to the aggregated packet */ - skb->destructor = NULL; - p->truesize += skb->truesize; - p->len += skb->len; - - NAPI_GRO_CB(skb)->same_flow = 1; - - return 0; -} - /** * skb_segment - Perform protocol segmentation on skb. * @head_skb: buffer to segment @@ -4297,122 +4267,6 @@ err: } EXPORT_SYMBOL_GPL(skb_segment); -int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb) -{ - struct skb_shared_info *pinfo, *skbinfo = skb_shinfo(skb); - unsigned int offset = skb_gro_offset(skb); - unsigned int headlen = skb_headlen(skb); - unsigned int len = skb_gro_len(skb); - unsigned int delta_truesize; - unsigned int new_truesize; - struct sk_buff *lp; - - if (unlikely(p->len + len >= 65536 || NAPI_GRO_CB(skb)->flush)) - return -E2BIG; - - lp = NAPI_GRO_CB(p)->last; - pinfo = skb_shinfo(lp); - - if (headlen <= offset) { - skb_frag_t *frag; - skb_frag_t *frag2; - int i = skbinfo->nr_frags; - int nr_frags = pinfo->nr_frags + i; - - if (nr_frags > MAX_SKB_FRAGS) - goto merge; - - offset -= headlen; - pinfo->nr_frags = nr_frags; - skbinfo->nr_frags = 0; - - frag = pinfo->frags + nr_frags; - frag2 = skbinfo->frags + i; - do { - *--frag = *--frag2; - } while (--i); - - skb_frag_off_add(frag, offset); - skb_frag_size_sub(frag, offset); - - /* all fragments truesize : remove (head size + sk_buff) */ - new_truesize = SKB_TRUESIZE(skb_end_offset(skb)); - delta_truesize = skb->truesize - new_truesize; - - skb->truesize = new_truesize; - skb->len -= skb->data_len; - skb->data_len = 0; - - NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE; - goto done; - } else if (skb->head_frag) { - int nr_frags = pinfo->nr_frags; - skb_frag_t *frag = pinfo->frags + nr_frags; - struct page *page = virt_to_head_page(skb->head); - unsigned int first_size = headlen - offset; - unsigned int first_offset; - - if (nr_frags + 1 + skbinfo->nr_frags > MAX_SKB_FRAGS) - goto merge; - - first_offset = skb->data - - (unsigned char *)page_address(page) + - offset; - - pinfo->nr_frags = nr_frags + 1 + skbinfo->nr_frags; - - __skb_frag_set_page(frag, page); - skb_frag_off_set(frag, first_offset); - skb_frag_size_set(frag, first_size); - - memcpy(frag + 1, skbinfo->frags, sizeof(*frag) * skbinfo->nr_frags); - /* We dont need to clear skbinfo->nr_frags here */ - - new_truesize = SKB_DATA_ALIGN(sizeof(struct sk_buff)); - delta_truesize = skb->truesize - new_truesize; - skb->truesize = new_truesize; - NAPI_GRO_CB(skb)->free = NAPI_GRO_FREE_STOLEN_HEAD; - goto done; - } - -merge: - /* sk owenrship - if any - completely transferred to the aggregated packet */ - skb->destructor = NULL; - delta_truesize = skb->truesize; - if (offset > headlen) { - unsigned int eat = offset - headlen; - - skb_frag_off_add(&skbinfo->frags[0], eat); - skb_frag_size_sub(&skbinfo->frags[0], eat); - skb->data_len -= eat; - skb->len -= eat; - offset = headlen; - } - - __skb_pull(skb, offset); - - if (NAPI_GRO_CB(p)->last == p) - skb_shinfo(p)->frag_list = skb; - else - NAPI_GRO_CB(p)->last->next = skb; - NAPI_GRO_CB(p)->last = skb; - __skb_header_release(skb); - lp = p; - -done: - NAPI_GRO_CB(p)->count++; - p->data_len += len; - p->truesize += delta_truesize; - p->len += len; - if (lp != p) { - lp->data_len += len; - lp->truesize += delta_truesize; - lp->len += len; - } - NAPI_GRO_CB(skb)->same_flow = 1; - return 0; -} - #ifdef CONFIG_SKB_EXTENSIONS #define SKB_EXT_ALIGN_VALUE 8 #define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE) @@ -4849,8 +4703,7 @@ static void __skb_complete_tx_timestamp(struct sk_buff *skb, serr->header.h4.iif = skb->dev ? skb->dev->ifindex : 0; if (sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) { serr->ee.ee_data = skb_shinfo(skb)->tskey; - if (sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) + if (sk_is_tcp(sk)) serr->ee.ee_data -= sk->sk_tskey; } @@ -4919,8 +4772,7 @@ void __skb_tstamp_tx(struct sk_buff *orig_skb, if (tsonly) { #ifdef CONFIG_INET if ((sk->sk_tsflags & SOF_TIMESTAMPING_OPT_STATS) && - sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) { + sk_is_tcp(sk)) { skb = tcp_get_timestamping_opt_stats(sk, orig_skb, ack_skb); opt_stats = true; diff --git a/net/core/sock.c b/net/core/sock.c index 41e91d0f7061..4a499d255f40 100644 --- a/net/core/sock.c +++ b/net/core/sock.c @@ -144,8 +144,6 @@ static DEFINE_MUTEX(proto_list_mutex); static LIST_HEAD(proto_list); -static void sock_inuse_add(struct net *net, int val); - /** * sk_ns_capable - General socket capability test * @sk: Socket to use a capability on or through @@ -327,7 +325,10 @@ int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb) BUG_ON(!sock_flag(sk, SOCK_MEMALLOC)); noreclaim_flag = memalloc_noreclaim_save(); - ret = sk->sk_backlog_rcv(sk, skb); + ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv, + tcp_v6_do_rcv, + tcp_v4_do_rcv, + sk, skb); memalloc_noreclaim_restore(noreclaim_flag); return ret; @@ -872,8 +873,7 @@ int sock_set_timestamping(struct sock *sk, int optname, if (val & SOF_TIMESTAMPING_OPT_ID && !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) { - if (sk->sk_protocol == IPPROTO_TCP && - sk->sk_type == SOCK_STREAM) { + if (sk_is_tcp(sk)) { if ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) return -EINVAL; @@ -1135,6 +1135,7 @@ set_sndbuf: case SO_PRIORITY: if ((val >= 0 && val <= 6) || + ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) || ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) sk->sk_priority = val; else @@ -1280,7 +1281,8 @@ set_sndbuf: clear_bit(SOCK_PASSSEC, &sock->flags); break; case SO_MARK: - if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { + if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) && + !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) { ret = -EPERM; break; } @@ -1370,8 +1372,7 @@ set_sndbuf: case SO_ZEROCOPY: if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) { - if (!((sk->sk_type == SOCK_STREAM && - sk->sk_protocol == IPPROTO_TCP) || + if (!(sk_is_tcp(sk) || (sk->sk_type == SOCK_DGRAM && sk->sk_protocol == IPPROTO_UDP))) ret = -ENOTSUPP; @@ -2246,17 +2247,22 @@ void sk_setup_caps(struct sock *sk, struct dst_entry *dst) u32 max_segs = 1; sk_dst_set(sk, dst); - sk->sk_route_caps = dst->dev->features | sk->sk_route_forced_caps; + sk->sk_route_caps = dst->dev->features; + if (sk_is_tcp(sk)) + sk->sk_route_caps |= NETIF_F_GSO; if (sk->sk_route_caps & NETIF_F_GSO) sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE; - sk->sk_route_caps &= ~sk->sk_route_nocaps; + if (unlikely(sk->sk_gso_disabled)) + sk->sk_route_caps &= ~NETIF_F_GSO_MASK; if (sk_can_gso(sk)) { if (dst->header_len && !xfrm_dst_offload_ok(dst)) { sk->sk_route_caps &= ~NETIF_F_GSO_MASK; } else { sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM; - sk->sk_gso_max_size = dst->dev->gso_max_size; - max_segs = max_t(u32, dst->dev->gso_max_segs, 1); + /* pairs with the WRITE_ONCE() in netif_set_gso_max_size() */ + sk->sk_gso_max_size = READ_ONCE(dst->dev->gso_max_size); + /* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */ + max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1); } } sk->sk_gso_max_segs = max_segs; @@ -3532,19 +3538,8 @@ void sk_get_meminfo(const struct sock *sk, u32 *mem) } #ifdef CONFIG_PROC_FS -#define PROTO_INUSE_NR 64 /* should be enough for the first time */ -struct prot_inuse { - int val[PROTO_INUSE_NR]; -}; - static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR); -void sock_prot_inuse_add(struct net *net, struct proto *prot, int val) -{ - __this_cpu_add(net->core.prot_inuse->val[prot->inuse_idx], val); -} -EXPORT_SYMBOL_GPL(sock_prot_inuse_add); - int sock_prot_inuse_get(struct net *net, struct proto *prot) { int cpu, idx = prot->inuse_idx; @@ -3557,17 +3552,12 @@ int sock_prot_inuse_get(struct net *net, struct proto *prot) } EXPORT_SYMBOL_GPL(sock_prot_inuse_get); -static void sock_inuse_add(struct net *net, int val) -{ - this_cpu_add(*net->core.sock_inuse, val); -} - int sock_inuse_get(struct net *net) { int cpu, res = 0; for_each_possible_cpu(cpu) - res += *per_cpu_ptr(net->core.sock_inuse, cpu); + res += per_cpu_ptr(net->core.prot_inuse, cpu)->all; return res; } @@ -3579,22 +3569,12 @@ static int __net_init sock_inuse_init_net(struct net *net) net->core.prot_inuse = alloc_percpu(struct prot_inuse); if (net->core.prot_inuse == NULL) return -ENOMEM; - - net->core.sock_inuse = alloc_percpu(int); - if (net->core.sock_inuse == NULL) - goto out; - return 0; - -out: - free_percpu(net->core.prot_inuse); - return -ENOMEM; } static void __net_exit sock_inuse_exit_net(struct net *net) { free_percpu(net->core.prot_inuse); - free_percpu(net->core.sock_inuse); } static struct pernet_operations net_inuse_ops = { @@ -3640,9 +3620,6 @@ static inline void release_proto_idx(struct proto *prot) { } -static void sock_inuse_add(struct net *net, int val) -{ -} #endif static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot) diff --git a/net/dccp/proto.c b/net/dccp/proto.c index fc44dadc778b..a976b4d29892 100644 --- a/net/dccp/proto.c +++ b/net/dccp/proto.c @@ -238,17 +238,6 @@ void dccp_destroy_sock(struct sock *sk) EXPORT_SYMBOL_GPL(dccp_destroy_sock); -static inline int dccp_listen_start(struct sock *sk, int backlog) -{ - struct dccp_sock *dp = dccp_sk(sk); - - dp->dccps_role = DCCP_ROLE_LISTEN; - /* do not start to listen if feature negotiation setup fails */ - if (dccp_feat_finalise_settings(dp)) - return -EPROTO; - return inet_csk_listen_start(sk, backlog); -} - static inline int dccp_need_reset(int state) { return state != DCCP_CLOSED && state != DCCP_LISTEN && @@ -931,11 +920,17 @@ int inet_dccp_listen(struct socket *sock, int backlog) * we can only allow the backlog to be adjusted. */ if (old_state != DCCP_LISTEN) { - /* - * FIXME: here it probably should be sk->sk_prot->listen_start - * see tcp_listen_start - */ - err = dccp_listen_start(sk, backlog); + struct dccp_sock *dp = dccp_sk(sk); + + dp->dccps_role = DCCP_ROLE_LISTEN; + + /* do not start to listen if feature negotiation setup fails */ + if (dccp_feat_finalise_settings(dp)) { + err = -EPROTO; + goto out; + } + + err = inet_csk_listen_start(sk); if (err) goto out; } diff --git a/net/dccp/trace.h b/net/dccp/trace.h index 5062421beee9..5a43b3508c7f 100644 --- a/net/dccp/trace.h +++ b/net/dccp/trace.h @@ -60,9 +60,7 @@ TRACE_EVENT(dccp_probe, __entry->tx_t_ipi = hc->tx_t_ipi; } else { __entry->tx_s = 0; - memset(&__entry->tx_rtt, 0, (void *)&__entry->tx_t_ipi - - (void *)&__entry->tx_rtt + - sizeof(__entry->tx_t_ipi)); + memset_startat(__entry, 0, tx_rtt); } ), diff --git a/net/dsa/dsa.c b/net/dsa/dsa.c index ea5169e671ae..d9d0d227092c 100644 --- a/net/dsa/dsa.c +++ b/net/dsa/dsa.c @@ -406,7 +406,7 @@ EXPORT_SYMBOL_GPL(dsa_devlink_resource_register); void dsa_devlink_resources_unregister(struct dsa_switch *ds) { - devlink_resources_unregister(ds->devlink, NULL); + devlink_resources_unregister(ds->devlink); } EXPORT_SYMBOL_GPL(dsa_devlink_resources_unregister); diff --git a/net/dsa/dsa2.c b/net/dsa/dsa2.c index 826957b6442b..8814fa0e44c8 100644 --- a/net/dsa/dsa2.c +++ b/net/dsa/dsa2.c @@ -129,35 +129,52 @@ void dsa_lag_unmap(struct dsa_switch_tree *dst, struct net_device *lag) } } +struct dsa_bridge *dsa_tree_bridge_find(struct dsa_switch_tree *dst, + const struct net_device *br) +{ + struct dsa_port *dp; + + list_for_each_entry(dp, &dst->ports, list) + if (dsa_port_bridge_dev_get(dp) == br) + return dp->bridge; + + return NULL; +} + static int dsa_bridge_num_find(const struct net_device *bridge_dev) { struct dsa_switch_tree *dst; - struct dsa_port *dp; - /* When preparing the offload for a port, it will have a valid - * dp->bridge_dev pointer but a not yet valid dp->bridge_num. - * However there might be other ports having the same dp->bridge_dev - * and a valid dp->bridge_num, so just ignore this port. - */ - list_for_each_entry(dst, &dsa_tree_list, list) - list_for_each_entry(dp, &dst->ports, list) - if (dp->bridge_dev == bridge_dev && - dp->bridge_num != -1) - return dp->bridge_num; + list_for_each_entry(dst, &dsa_tree_list, list) { + struct dsa_bridge *bridge; - return -1; + bridge = dsa_tree_bridge_find(dst, bridge_dev); + if (bridge) + return bridge->num; + } + + return 0; } -int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) +unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) { - int bridge_num = dsa_bridge_num_find(bridge_dev); + unsigned int bridge_num = dsa_bridge_num_find(bridge_dev); - if (bridge_num < 0) { - /* First port that offloads TX forwarding for this bridge */ - bridge_num = find_first_zero_bit(&dsa_fwd_offloading_bridges, - DSA_MAX_NUM_OFFLOADING_BRIDGES); + /* Switches without FDB isolation support don't get unique + * bridge numbering + */ + if (!max) + return 0; + + if (!bridge_num) { + /* First port that requests FDB isolation or TX forwarding + * offload for this bridge + */ + bridge_num = find_next_zero_bit(&dsa_fwd_offloading_bridges, + DSA_MAX_NUM_OFFLOADING_BRIDGES, + 1); if (bridge_num >= max) - return -1; + return 0; set_bit(bridge_num, &dsa_fwd_offloading_bridges); } @@ -165,13 +182,14 @@ int dsa_bridge_num_get(const struct net_device *bridge_dev, int max) return bridge_num; } -void dsa_bridge_num_put(const struct net_device *bridge_dev, int bridge_num) +void dsa_bridge_num_put(const struct net_device *bridge_dev, + unsigned int bridge_num) { - /* Check if the bridge is still in use, otherwise it is time - * to clean it up so we can reuse this bridge_num later. + /* Since we refcount bridges, we know that when we call this function + * it is no longer in use, so we can just go ahead and remove it from + * the bit mask. */ - if (dsa_bridge_num_find(bridge_dev) < 0) - clear_bit(bridge_num, &dsa_fwd_offloading_bridges); + clear_bit(bridge_num, &dsa_fwd_offloading_bridges); } struct dsa_switch *dsa_switch_find(int tree_index, int sw_index) @@ -1184,7 +1202,6 @@ static struct dsa_port *dsa_port_touch(struct dsa_switch *ds, int index) dp->ds = ds; dp->index = index; - dp->bridge_num = -1; INIT_LIST_HEAD(&dp->list); list_add_tail(&dp->list, &dst->ports); diff --git a/net/dsa/dsa_priv.h b/net/dsa/dsa_priv.h index a5c9bc7b66c6..38ce5129a33d 100644 --- a/net/dsa/dsa_priv.h +++ b/net/dsa/dsa_priv.h @@ -52,10 +52,11 @@ struct dsa_notifier_ageing_time_info { /* DSA_NOTIFIER_BRIDGE_* */ struct dsa_notifier_bridge_info { - struct net_device *br; + struct dsa_bridge bridge; int tree_index; int sw_index; int port; + bool tx_fwd_offload; }; /* DSA_NOTIFIER_FDB_* */ @@ -258,54 +259,13 @@ int dsa_port_mrp_add_ring_role(const struct dsa_port *dp, const struct switchdev_obj_ring_role_mrp *mrp); int dsa_port_mrp_del_ring_role(const struct dsa_port *dp, const struct switchdev_obj_ring_role_mrp *mrp); +int dsa_port_phylink_create(struct dsa_port *dp); int dsa_port_link_register_of(struct dsa_port *dp); void dsa_port_link_unregister_of(struct dsa_port *dp); int dsa_port_hsr_join(struct dsa_port *dp, struct net_device *hsr); void dsa_port_hsr_leave(struct dsa_port *dp, struct net_device *hsr); int dsa_port_tag_8021q_vlan_add(struct dsa_port *dp, u16 vid, bool broadcast); void dsa_port_tag_8021q_vlan_del(struct dsa_port *dp, u16 vid, bool broadcast); -extern const struct phylink_mac_ops dsa_port_phylink_mac_ops; - -static inline bool dsa_port_offloads_bridge_port(struct dsa_port *dp, - const struct net_device *dev) -{ - return dsa_port_to_bridge_port(dp) == dev; -} - -static inline bool dsa_port_offloads_bridge(struct dsa_port *dp, - const struct net_device *bridge_dev) -{ - /* DSA ports connected to a bridge, and event was emitted - * for the bridge. - */ - return dp->bridge_dev == bridge_dev; -} - -/* Returns true if any port of this tree offloads the given net_device */ -static inline bool dsa_tree_offloads_bridge_port(struct dsa_switch_tree *dst, - const struct net_device *dev) -{ - struct dsa_port *dp; - - list_for_each_entry(dp, &dst->ports, list) - if (dsa_port_offloads_bridge_port(dp, dev)) - return true; - - return false; -} - -/* Returns true if any port of this tree offloads the given bridge */ -static inline bool dsa_tree_offloads_bridge(struct dsa_switch_tree *dst, - const struct net_device *bridge_dev) -{ - struct dsa_port *dp; - - list_for_each_entry(dp, &dst->ports, list) - if (dsa_port_offloads_bridge(dp, bridge_dev)) - return true; - - return false; -} /* slave.c */ extern const struct dsa_device_ops notag_netdev_ops; @@ -345,7 +305,7 @@ dsa_slave_to_master(const struct net_device *dev) static inline struct sk_buff *dsa_untag_bridge_pvid(struct sk_buff *skb) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - struct net_device *br = dp->bridge_dev; + struct net_device *br = dsa_port_bridge_dev_get(dp); struct net_device *dev = skb->dev; struct net_device *upper_dev; u16 vid, pvid, proto; @@ -415,7 +375,7 @@ dsa_find_designated_bridge_port_by_vid(struct net_device *master, u16 vid) if (dp->type != DSA_PORT_TYPE_USER) continue; - if (!dp->bridge_dev) + if (!dp->bridge) continue; if (dp->stp_state != BR_STATE_LEARNING && @@ -444,7 +404,7 @@ dsa_find_designated_bridge_port_by_vid(struct net_device *master, u16 vid) /* If the ingress port offloads the bridge, we mark the frame as autonomously * forwarded by hardware, so the software bridge doesn't forward in twice, back * to us, because we already did. However, if we're in fallback mode and we do - * software bridging, we are not offloading it, therefore the dp->bridge_dev + * software bridging, we are not offloading it, therefore the dp->bridge * pointer is not populated, and flooding needs to be done by software (we are * effectively operating in standalone ports mode). */ @@ -452,7 +412,7 @@ static inline void dsa_default_offload_fwd_mark(struct sk_buff *skb) { struct dsa_port *dp = dsa_slave_to_port(skb->dev); - skb->offload_fwd_mark = !!(dp->bridge_dev); + skb->offload_fwd_mark = !!(dp->bridge); } /* Helper for removing DSA header tags from packets in the RX path. @@ -546,8 +506,11 @@ int dsa_tree_change_tag_proto(struct dsa_switch_tree *dst, struct net_device *master, const struct dsa_device_ops *tag_ops, const struct dsa_device_ops *old_tag_ops); -int dsa_bridge_num_get(const struct net_device *bridge_dev, int max); -void dsa_bridge_num_put(const struct net_device *bridge_dev, int bridge_num); +unsigned int dsa_bridge_num_get(const struct net_device *bridge_dev, int max); +void dsa_bridge_num_put(const struct net_device *bridge_dev, + unsigned int bridge_num); +struct dsa_bridge *dsa_tree_bridge_find(struct dsa_switch_tree *dst, + const struct net_device *br); /* tag_8021q.c */ int dsa_tag_8021q_bridge_join(struct dsa_switch *ds, diff --git a/net/dsa/port.c b/net/dsa/port.c index f6f12ad2b525..05677e016982 100644 --- a/net/dsa/port.c +++ b/net/dsa/port.c @@ -130,7 +130,7 @@ int dsa_port_enable_rt(struct dsa_port *dp, struct phy_device *phy) return err; } - if (!dp->bridge_dev) + if (!dp->bridge) dsa_port_set_state_now(dp, BR_STATE_FORWARDING, false); if (dp->pl) @@ -158,7 +158,7 @@ void dsa_port_disable_rt(struct dsa_port *dp) if (dp->pl) phylink_stop(dp->pl); - if (!dp->bridge_dev) + if (!dp->bridge) dsa_port_set_state_now(dp, BR_STATE_DISABLED, false); if (ds->ops->port_disable) @@ -221,7 +221,7 @@ static int dsa_port_switchdev_sync_attrs(struct dsa_port *dp, struct netlink_ext_ack *extack) { struct net_device *brport_dev = dsa_port_to_bridge_port(dp); - struct net_device *br = dp->bridge_dev; + struct net_device *br = dsa_port_bridge_dev_get(dp); int err; err = dsa_port_inherit_brport_flags(dp, extack); @@ -270,52 +270,55 @@ static void dsa_port_switchdev_unsync_attrs(struct dsa_port *dp) */ } -static void dsa_port_bridge_tx_fwd_unoffload(struct dsa_port *dp, - struct net_device *bridge_dev) +static int dsa_port_bridge_create(struct dsa_port *dp, + struct net_device *br, + struct netlink_ext_ack *extack) { - int bridge_num = dp->bridge_num; struct dsa_switch *ds = dp->ds; + struct dsa_bridge *bridge; - /* No bridge TX forwarding offload => do nothing */ - if (!ds->ops->port_bridge_tx_fwd_unoffload || dp->bridge_num == -1) - return; + bridge = dsa_tree_bridge_find(ds->dst, br); + if (bridge) { + refcount_inc(&bridge->refcount); + dp->bridge = bridge; + return 0; + } - dp->bridge_num = -1; + bridge = kzalloc(sizeof(*bridge), GFP_KERNEL); + if (!bridge) + return -ENOMEM; - dsa_bridge_num_put(bridge_dev, bridge_num); + refcount_set(&bridge->refcount, 1); - /* Notify the chips only once the offload has been deactivated, so - * that they can update their configuration accordingly. - */ - ds->ops->port_bridge_tx_fwd_unoffload(ds, dp->index, bridge_dev, - bridge_num); + bridge->dev = br; + + bridge->num = dsa_bridge_num_get(br, ds->max_num_bridges); + if (ds->max_num_bridges && !bridge->num) { + NL_SET_ERR_MSG_MOD(extack, + "Range of offloadable bridges exceeded"); + kfree(bridge); + return -EOPNOTSUPP; + } + + dp->bridge = bridge; + + return 0; } -static bool dsa_port_bridge_tx_fwd_offload(struct dsa_port *dp, - struct net_device *bridge_dev) +static void dsa_port_bridge_destroy(struct dsa_port *dp, + const struct net_device *br) { - struct dsa_switch *ds = dp->ds; - int bridge_num, err; + struct dsa_bridge *bridge = dp->bridge; - if (!ds->ops->port_bridge_tx_fwd_offload) - return false; + dp->bridge = NULL; - bridge_num = dsa_bridge_num_get(bridge_dev, - ds->num_fwd_offloading_bridges); - if (bridge_num < 0) - return false; + if (!refcount_dec_and_test(&bridge->refcount)) + return; - dp->bridge_num = bridge_num; + if (bridge->num) + dsa_bridge_num_put(br, bridge->num); - /* Notify the driver */ - err = ds->ops->port_bridge_tx_fwd_offload(ds, dp->index, bridge_dev, - bridge_num); - if (err) { - dsa_port_bridge_tx_fwd_unoffload(dp, bridge_dev); - return false; - } - - return true; + kfree(bridge); } int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br, @@ -325,30 +328,32 @@ int dsa_port_bridge_join(struct dsa_port *dp, struct net_device *br, .tree_index = dp->ds->dst->index, .sw_index = dp->ds->index, .port = dp->index, - .br = br, }; struct net_device *dev = dp->slave; struct net_device *brport_dev; - bool tx_fwd_offload; int err; /* Here the interface is already bridged. Reflect the current * configuration so that drivers can program their chips accordingly. */ - dp->bridge_dev = br; + err = dsa_port_bridge_create(dp, br, extack); + if (err) + return err; brport_dev = dsa_port_to_bridge_port(dp); + info.bridge = *dp->bridge; err = dsa_broadcast(DSA_NOTIFIER_BRIDGE_JOIN, &info); if (err) goto out_rollback; - tx_fwd_offload = dsa_port_bridge_tx_fwd_offload(dp, br); + /* Drivers which support bridge TX forwarding should set this */ + dp->bridge->tx_fwd_offload = info.tx_fwd_offload; err = switchdev_bridge_port_offload(brport_dev, dev, dp, &dsa_slave_switchdev_notifier, &dsa_slave_switchdev_blocking_notifier, - tx_fwd_offload, extack); + dp->bridge->tx_fwd_offload, extack); if (err) goto out_rollback_unbridge; @@ -365,7 +370,7 @@ out_rollback_unoffload: out_rollback_unbridge: dsa_broadcast(DSA_NOTIFIER_BRIDGE_LEAVE, &info); out_rollback: - dp->bridge_dev = NULL; + dsa_port_bridge_destroy(dp, br); return err; } @@ -390,16 +395,14 @@ void dsa_port_bridge_leave(struct dsa_port *dp, struct net_device *br) .tree_index = dp->ds->dst->index, .sw_index = dp->ds->index, .port = dp->index, - .br = br, + .bridge = *dp->bridge, }; int err; /* Here the port is already unbridged. Reflect the current configuration * so that drivers can program their chips accordingly. */ - dp->bridge_dev = NULL; - - dsa_port_bridge_tx_fwd_unoffload(dp, br); + dsa_port_bridge_destroy(dp, br); err = dsa_broadcast(DSA_NOTIFIER_BRIDGE_LEAVE, &info); if (err) @@ -477,12 +480,15 @@ err_lag_join: void dsa_port_pre_lag_leave(struct dsa_port *dp, struct net_device *lag) { - if (dp->bridge_dev) - dsa_port_pre_bridge_leave(dp, dp->bridge_dev); + struct net_device *br = dsa_port_bridge_dev_get(dp); + + if (br) + dsa_port_pre_bridge_leave(dp, br); } void dsa_port_lag_leave(struct dsa_port *dp, struct net_device *lag) { + struct net_device *br = dsa_port_bridge_dev_get(dp); struct dsa_notifier_lag_info info = { .sw_index = dp->ds->index, .port = dp->index, @@ -496,8 +502,8 @@ void dsa_port_lag_leave(struct dsa_port *dp, struct net_device *lag) /* Port might have been part of a LAG that in turn was * attached to a bridge. */ - if (dp->bridge_dev) - dsa_port_bridge_leave(dp, dp->bridge_dev); + if (br) + dsa_port_bridge_leave(dp, br); dp->lag_tx_enabled = false; dp->lag_dev = NULL; @@ -526,8 +532,8 @@ static bool dsa_port_can_apply_vlan_filtering(struct dsa_port *dp, * as long as we have 8021q uppers. */ if (vlan_filtering && dsa_port_is_user(dp)) { + struct net_device *br = dsa_port_bridge_dev_get(dp); struct net_device *upper_dev, *slave = dp->slave; - struct net_device *br = dp->bridge_dev; struct list_head *iter; netdev_for_each_upper_dev_rcu(slave, upper_dev, iter) { @@ -561,17 +567,15 @@ static bool dsa_port_can_apply_vlan_filtering(struct dsa_port *dp, * different setting than what is being requested. */ dsa_switch_for_each_port(other_dp, ds) { - struct net_device *other_bridge; + struct net_device *other_br = dsa_port_bridge_dev_get(other_dp); - other_bridge = other_dp->bridge_dev; - if (!other_bridge) - continue; /* If it's the same bridge, it also has same * vlan_filtering setting => no need to check */ - if (other_bridge == dp->bridge_dev) + if (!other_br || other_br == dsa_port_bridge_dev_get(dp)) continue; - if (br_vlan_enabled(other_bridge) != vlan_filtering) { + + if (br_vlan_enabled(other_br) != vlan_filtering) { NL_SET_ERR_MSG_MOD(extack, "VLAN filtering is a global setting"); return false; @@ -655,13 +659,13 @@ restore: */ bool dsa_port_skip_vlan_configuration(struct dsa_port *dp) { + struct net_device *br = dsa_port_bridge_dev_get(dp); struct dsa_switch *ds = dp->ds; - if (!dp->bridge_dev) + if (!br) return false; - return (!ds->configure_vlan_while_not_filtering && - !br_vlan_enabled(dp->bridge_dev)); + return !ds->configure_vlan_while_not_filtering && !br_vlan_enabled(br); } int dsa_port_ageing_time(struct dsa_port *dp, clock_t ageing_clock) @@ -981,8 +985,11 @@ static void dsa_port_phylink_validate(struct phylink_config *config, struct dsa_port *dp = container_of(config, struct dsa_port, pl_config); struct dsa_switch *ds = dp->ds; - if (!ds->ops->phylink_validate) + if (!ds->ops->phylink_validate) { + if (config->mac_capabilities) + phylink_generic_validate(config, supported, state); return; + } ds->ops->phylink_validate(ds, dp->index, supported, state); } @@ -1072,7 +1079,7 @@ static void dsa_port_phylink_mac_link_up(struct phylink_config *config, speed, duplex, tx_pause, rx_pause); } -const struct phylink_mac_ops dsa_port_phylink_mac_ops = { +static const struct phylink_mac_ops dsa_port_phylink_mac_ops = { .validate = dsa_port_phylink_validate, .mac_pcs_get_state = dsa_port_phylink_mac_pcs_get_state, .mac_config = dsa_port_phylink_mac_config, @@ -1081,6 +1088,36 @@ const struct phylink_mac_ops dsa_port_phylink_mac_ops = { .mac_link_up = dsa_port_phylink_mac_link_up, }; +int dsa_port_phylink_create(struct dsa_port *dp) +{ + struct dsa_switch *ds = dp->ds; + phy_interface_t mode; + int err; + + err = of_get_phy_mode(dp->dn, &mode); + if (err) + mode = PHY_INTERFACE_MODE_NA; + + /* Presence of phylink_mac_link_state or phylink_mac_an_restart is + * an indicator of a legacy phylink driver. + */ + if (ds->ops->phylink_mac_link_state || + ds->ops->phylink_mac_an_restart) + dp->pl_config.legacy_pre_march2020 = true; + + if (ds->ops->phylink_get_caps) + ds->ops->phylink_get_caps(ds, dp->index, &dp->pl_config); + + dp->pl = phylink_create(&dp->pl_config, of_fwnode_handle(dp->dn), + mode, &dsa_port_phylink_mac_ops); + if (IS_ERR(dp->pl)) { + pr_err("error creating PHYLINK: %ld\n", PTR_ERR(dp->pl)); + return PTR_ERR(dp->pl); + } + + return 0; +} + static int dsa_port_setup_phy_of(struct dsa_port *dp, bool enable) { struct dsa_switch *ds = dp->ds; @@ -1157,27 +1194,15 @@ static int dsa_port_phylink_register(struct dsa_port *dp) { struct dsa_switch *ds = dp->ds; struct device_node *port_dn = dp->dn; - phy_interface_t mode; int err; - err = of_get_phy_mode(port_dn, &mode); - if (err) - mode = PHY_INTERFACE_MODE_NA; - dp->pl_config.dev = ds->dev; dp->pl_config.type = PHYLINK_DEV; dp->pl_config.pcs_poll = ds->pcs_poll; - if (ds->ops->phylink_get_interfaces) - ds->ops->phylink_get_interfaces(ds, dp->index, - dp->pl_config.supported_interfaces); - - dp->pl = phylink_create(&dp->pl_config, of_fwnode_handle(port_dn), - mode, &dsa_port_phylink_mac_ops); - if (IS_ERR(dp->pl)) { - pr_err("error creating PHYLINK: %ld\n", PTR_ERR(dp->pl)); - return PTR_ERR(dp->pl); - } + err = dsa_port_phylink_create(dp); + if (err) + return err; err = phylink_of_phy_connect(dp->pl, port_dn, 0); if (err && err != -ENODEV) { diff --git a/net/dsa/slave.c b/net/dsa/slave.c index ad61f6bc8886..88f7b8686dac 100644 --- a/net/dsa/slave.c +++ b/net/dsa/slave.c @@ -289,14 +289,14 @@ static int dsa_slave_port_attr_set(struct net_device *dev, const void *ctx, ret = dsa_port_set_state(dp, attr->u.stp_state, true); break; case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_FILTERING: - if (!dsa_port_offloads_bridge(dp, attr->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev)) return -EOPNOTSUPP; ret = dsa_port_vlan_filtering(dp, attr->u.vlan_filtering, extack); break; case SWITCHDEV_ATTR_ID_BRIDGE_AGEING_TIME: - if (!dsa_port_offloads_bridge(dp, attr->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, attr->orig_dev)) return -EOPNOTSUPP; ret = dsa_port_ageing_time(dp, attr->u.ageing_time); @@ -363,7 +363,7 @@ static int dsa_slave_vlan_add(struct net_device *dev, /* Deny adding a bridge VLAN when there is already an 802.1Q upper with * the same VID. */ - if (br_vlan_enabled(dp->bridge_dev)) { + if (br_vlan_enabled(dsa_port_bridge_dev_get(dp))) { rcu_read_lock(); err = dsa_slave_vlan_check_for_8021q_uppers(dev, &vlan); rcu_read_unlock(); @@ -409,7 +409,7 @@ static int dsa_slave_port_obj_add(struct net_device *dev, const void *ctx, err = dsa_port_mdb_add(dp, SWITCHDEV_OBJ_PORT_MDB(obj)); break; case SWITCHDEV_OBJ_ID_HOST_MDB: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_host_mdb_add(dp, SWITCHDEV_OBJ_PORT_MDB(obj)); @@ -421,13 +421,13 @@ static int dsa_slave_port_obj_add(struct net_device *dev, const void *ctx, err = dsa_slave_vlan_add(dev, obj, extack); break; case SWITCHDEV_OBJ_ID_MRP: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_mrp_add(dp, SWITCHDEV_OBJ_MRP(obj)); break; case SWITCHDEV_OBJ_ID_RING_ROLE_MRP: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_mrp_add_ring_role(dp, @@ -483,7 +483,7 @@ static int dsa_slave_port_obj_del(struct net_device *dev, const void *ctx, err = dsa_port_mdb_del(dp, SWITCHDEV_OBJ_PORT_MDB(obj)); break; case SWITCHDEV_OBJ_ID_HOST_MDB: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_host_mdb_del(dp, SWITCHDEV_OBJ_PORT_MDB(obj)); @@ -495,13 +495,13 @@ static int dsa_slave_port_obj_del(struct net_device *dev, const void *ctx, err = dsa_slave_vlan_del(dev, obj); break; case SWITCHDEV_OBJ_ID_MRP: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_mrp_del(dp, SWITCHDEV_OBJ_MRP(obj)); break; case SWITCHDEV_OBJ_ID_RING_ROLE_MRP: - if (!dsa_port_offloads_bridge(dp, obj->orig_dev)) + if (!dsa_port_offloads_bridge_dev(dp, obj->orig_dev)) return -EOPNOTSUPP; err = dsa_port_mrp_del_ring_role(dp, @@ -1564,7 +1564,7 @@ static void dsa_bridge_mtu_normalization(struct dsa_port *dp) if (!dp->ds->mtu_enforcement_ingress) return; - if (!dp->bridge_dev) + if (!dp->bridge) return; INIT_LIST_HEAD(&hw_port_list); @@ -1580,7 +1580,7 @@ static void dsa_bridge_mtu_normalization(struct dsa_port *dp) if (other_dp->type != DSA_PORT_TYPE_USER) continue; - if (other_dp->bridge_dev != dp->bridge_dev) + if (!dsa_port_bridge_same(dp, other_dp)) continue; if (!other_dp->ds->mtu_enforcement_ingress) @@ -1851,14 +1851,9 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev) struct dsa_port *dp = dsa_slave_to_port(slave_dev); struct device_node *port_dn = dp->dn; struct dsa_switch *ds = dp->ds; - phy_interface_t mode; u32 phy_flags = 0; int ret; - ret = of_get_phy_mode(port_dn, &mode); - if (ret) - mode = PHY_INTERFACE_MODE_NA; - dp->pl_config.dev = &slave_dev->dev; dp->pl_config.type = PHYLINK_NETDEV; @@ -1871,17 +1866,9 @@ static int dsa_slave_phy_setup(struct net_device *slave_dev) dp->pl_config.poll_fixed_state = true; } - if (ds->ops->phylink_get_interfaces) - ds->ops->phylink_get_interfaces(ds, dp->index, - dp->pl_config.supported_interfaces); - - dp->pl = phylink_create(&dp->pl_config, of_fwnode_handle(port_dn), mode, - &dsa_port_phylink_mac_ops); - if (IS_ERR(dp->pl)) { - netdev_err(slave_dev, - "error creating PHYLINK: %ld\n", PTR_ERR(dp->pl)); - return PTR_ERR(dp->pl); - } + ret = dsa_port_phylink_create(dp); + if (ret) + return ret; if (ds->ops->get_phy_flags) phy_flags = ds->ops->get_phy_flags(ds, dp->index); @@ -2233,7 +2220,7 @@ dsa_prevent_bridging_8021q_upper(struct net_device *dev, struct netdev_notifier_changeupper_info *info) { struct netlink_ext_ack *ext_ack; - struct net_device *slave; + struct net_device *slave, *br; struct dsa_port *dp; ext_ack = netdev_notifier_info_to_extack(&info->info); @@ -2246,11 +2233,12 @@ dsa_prevent_bridging_8021q_upper(struct net_device *dev, return NOTIFY_DONE; dp = dsa_slave_to_port(slave); - if (!dp->bridge_dev) + br = dsa_port_bridge_dev_get(dp); + if (!br) return NOTIFY_DONE; /* Deny enslaving a VLAN device into a VLAN-aware bridge */ - if (br_vlan_enabled(dp->bridge_dev) && + if (br_vlan_enabled(br) && netif_is_bridge_master(info->upper_dev) && info->linking) { NL_SET_ERR_MSG_MOD(ext_ack, "Cannot enslave VLAN device into VLAN aware bridge"); @@ -2265,7 +2253,7 @@ dsa_slave_check_8021q_upper(struct net_device *dev, struct netdev_notifier_changeupper_info *info) { struct dsa_port *dp = dsa_slave_to_port(dev); - struct net_device *br = dp->bridge_dev; + struct net_device *br = dsa_port_bridge_dev_get(dp); struct bridge_vlan_info br_info; struct netlink_ext_ack *extack; int err = NOTIFY_DONE; @@ -2462,7 +2450,7 @@ static bool dsa_foreign_dev_check(const struct net_device *dev, struct dsa_switch_tree *dst = dp->ds->dst; if (netif_is_bridge_master(foreign_dev)) - return !dsa_tree_offloads_bridge(dst, foreign_dev); + return !dsa_tree_offloads_bridge_dev(dst, foreign_dev); if (netif_is_bridge_port(foreign_dev)) return !dsa_tree_offloads_bridge_port(dst, foreign_dev); diff --git a/net/dsa/switch.c b/net/dsa/switch.c index bb155a16d454..9c92edd96961 100644 --- a/net/dsa/switch.c +++ b/net/dsa/switch.c @@ -95,7 +95,8 @@ static int dsa_switch_bridge_join(struct dsa_switch *ds, if (!ds->ops->port_bridge_join) return -EOPNOTSUPP; - err = ds->ops->port_bridge_join(ds, info->port, info->br); + err = ds->ops->port_bridge_join(ds, info->port, info->bridge, + &info->tx_fwd_offload); if (err) return err; } @@ -104,7 +105,7 @@ static int dsa_switch_bridge_join(struct dsa_switch *ds, ds->ops->crosschip_bridge_join) { err = ds->ops->crosschip_bridge_join(ds, info->tree_index, info->sw_index, - info->port, info->br); + info->port, info->bridge); if (err) return err; } @@ -124,19 +125,20 @@ static int dsa_switch_bridge_leave(struct dsa_switch *ds, if (dst->index == info->tree_index && ds->index == info->sw_index && ds->ops->port_bridge_leave) - ds->ops->port_bridge_leave(ds, info->port, info->br); + ds->ops->port_bridge_leave(ds, info->port, info->bridge); if ((dst->index != info->tree_index || ds->index != info->sw_index) && ds->ops->crosschip_bridge_leave) ds->ops->crosschip_bridge_leave(ds, info->tree_index, info->sw_index, info->port, - info->br); + info->bridge); - if (ds->needs_standalone_vlan_filtering && !br_vlan_enabled(info->br)) { + if (ds->needs_standalone_vlan_filtering && + !br_vlan_enabled(info->bridge.dev)) { change_vlan_filtering = true; vlan_filtering = true; } else if (!ds->needs_standalone_vlan_filtering && - br_vlan_enabled(info->br)) { + br_vlan_enabled(info->bridge.dev)) { change_vlan_filtering = true; vlan_filtering = false; } @@ -151,11 +153,9 @@ static int dsa_switch_bridge_leave(struct dsa_switch *ds, */ if (change_vlan_filtering && ds->vlan_filtering_is_global) { dsa_switch_for_each_port(dp, ds) { - struct net_device *bridge_dev; + struct net_device *br = dsa_port_bridge_dev_get(dp); - bridge_dev = dp->bridge_dev; - - if (bridge_dev && br_vlan_enabled(bridge_dev)) { + if (br && br_vlan_enabled(br)) { change_vlan_filtering = false; break; } diff --git a/net/dsa/tag_8021q.c b/net/dsa/tag_8021q.c index 72cac2c0af7b..27712a81c967 100644 --- a/net/dsa/tag_8021q.c +++ b/net/dsa/tag_8021q.c @@ -67,10 +67,12 @@ #define DSA_8021Q_PORT(x) (((x) << DSA_8021Q_PORT_SHIFT) & \ DSA_8021Q_PORT_MASK) -u16 dsa_8021q_bridge_tx_fwd_offload_vid(int bridge_num) +u16 dsa_8021q_bridge_tx_fwd_offload_vid(unsigned int bridge_num) { - /* The VBID value of 0 is reserved for precise TX */ - return DSA_8021Q_DIR_TX | DSA_8021Q_VBID(bridge_num + 1); + /* The VBID value of 0 is reserved for precise TX, but it is also + * reserved/invalid for the bridge_num, so all is well. + */ + return DSA_8021Q_DIR_TX | DSA_8021Q_VBID(bridge_num); } EXPORT_SYMBOL_GPL(dsa_8021q_bridge_tx_fwd_offload_vid); @@ -335,7 +337,7 @@ dsa_port_tag_8021q_bridge_match(struct dsa_port *dp, return false; if (dsa_port_is_user(dp)) - return dp->bridge_dev == info->br; + return dsa_port_offloads_bridge(dp, &info->bridge); return false; } @@ -408,10 +410,9 @@ int dsa_tag_8021q_bridge_leave(struct dsa_switch *ds, } int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num) + struct dsa_bridge bridge) { - u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); + u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge.num); return dsa_port_tag_8021q_vlan_add(dsa_to_port(ds, port), tx_vid, true); @@ -419,10 +420,9 @@ int dsa_tag_8021q_bridge_tx_fwd_offload(struct dsa_switch *ds, int port, EXPORT_SYMBOL_GPL(dsa_tag_8021q_bridge_tx_fwd_offload); void dsa_tag_8021q_bridge_tx_fwd_unoffload(struct dsa_switch *ds, int port, - struct net_device *br, - int bridge_num) + struct dsa_bridge bridge) { - u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); + u16 tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge.num); dsa_port_tag_8021q_vlan_del(dsa_to_port(ds, port), tx_vid, true); } diff --git a/net/dsa/tag_dsa.c b/net/dsa/tag_dsa.c index b3da4b2ea11c..8abf39dcac64 100644 --- a/net/dsa/tag_dsa.c +++ b/net/dsa/tag_dsa.c @@ -132,6 +132,7 @@ static struct sk_buff *dsa_xmit_ll(struct sk_buff *skb, struct net_device *dev, u8 *dsa_header; if (skb->offload_fwd_mark) { + unsigned int bridge_num = dsa_port_bridge_num_get(dp); struct dsa_switch_tree *dst = dp->ds->dst; cmd = DSA_CMD_FORWARD; @@ -140,7 +141,7 @@ static struct sk_buff *dsa_xmit_ll(struct sk_buff *skb, struct net_device *dev, * packets on behalf of a virtual switch device with an index * past the physical switches. */ - tag_dev = dst->last_switch + 1 + dp->bridge_num; + tag_dev = dst->last_switch + bridge_num; tag_port = 0; } else { cmd = DSA_CMD_FROM_CPU; @@ -165,7 +166,7 @@ static struct sk_buff *dsa_xmit_ll(struct sk_buff *skb, struct net_device *dev, dsa_header[2] &= ~0x10; } } else { - struct net_device *br = dp->bridge_dev; + struct net_device *br = dsa_port_bridge_dev_get(dp); u16 vid; vid = br ? MV88E6XXX_VID_BRIDGED : MV88E6XXX_VID_STANDALONE; diff --git a/net/dsa/tag_ocelot.c b/net/dsa/tag_ocelot.c index de1c849a0a70..4ba460c5a880 100644 --- a/net/dsa/tag_ocelot.c +++ b/net/dsa/tag_ocelot.c @@ -12,7 +12,7 @@ static void ocelot_xmit_get_vlan_info(struct sk_buff *skb, struct dsa_port *dp, u64 *vlan_tci, u64 *tag_type) { - struct net_device *br = READ_ONCE(dp->bridge_dev); + struct net_device *br = dsa_port_bridge_dev_get(dp); struct vlan_ethhdr *hdr; u16 proto, tci; diff --git a/net/dsa/tag_sja1105.c b/net/dsa/tag_sja1105.c index 262c8833a910..6c293c2a3008 100644 --- a/net/dsa/tag_sja1105.c +++ b/net/dsa/tag_sja1105.c @@ -159,14 +159,16 @@ static u16 sja1105_xmit_tpid(struct dsa_port *dp) * need to find it. */ dsa_switch_for_each_port(other_dp, ds) { - if (!other_dp->bridge_dev) + struct net_device *br = dsa_port_bridge_dev_get(other_dp); + + if (!br) continue; /* Error is returned only if CONFIG_BRIDGE_VLAN_FILTERING, * which seems pointless to handle, as our port cannot become * VLAN-aware in that case. */ - br_vlan_get_proto(other_dp->bridge_dev, &proto); + br_vlan_get_proto(br, &proto); return proto; } @@ -180,7 +182,8 @@ static struct sk_buff *sja1105_imprecise_xmit(struct sk_buff *skb, struct net_device *netdev) { struct dsa_port *dp = dsa_slave_to_port(netdev); - struct net_device *br = dp->bridge_dev; + unsigned int bridge_num = dsa_port_bridge_num_get(dp); + struct net_device *br = dsa_port_bridge_dev_get(dp); u16 tx_vid; /* If the port is under a VLAN-aware bridge, just slide the @@ -196,7 +199,7 @@ static struct sk_buff *sja1105_imprecise_xmit(struct sk_buff *skb, * TX VLAN that targets the bridge's entire broadcast domain, * instead of just the specific port. */ - tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(dp->bridge_num); + tx_vid = dsa_8021q_bridge_tx_fwd_offload_vid(bridge_num); return dsa_8021q_xmit(skb, netdev, sja1105_xmit_tpid(dp), tx_vid); } diff --git a/net/ethernet/eth.c b/net/ethernet/eth.c index c7d9e08107cb..ebcc812735a4 100644 --- a/net/ethernet/eth.c +++ b/net/ethernet/eth.c @@ -436,11 +436,10 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb) type = eh->h_proto; - rcu_read_lock(); ptype = gro_find_receive_by_type(type); if (ptype == NULL) { flush = 1; - goto out_unlock; + goto out; } skb_gro_pull(skb, sizeof(*eh)); @@ -450,8 +449,6 @@ struct sk_buff *eth_gro_receive(struct list_head *head, struct sk_buff *skb) ipv6_gro_receive, inet_gro_receive, head, skb); -out_unlock: - rcu_read_unlock(); out: skb_gro_flush_final(skb, pp, flush); @@ -469,14 +466,12 @@ int eth_gro_complete(struct sk_buff *skb, int nhoff) if (skb->encapsulation) skb_set_inner_mac_header(skb, nhoff); - rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype != NULL) err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete, ipv6_gro_complete, inet_gro_complete, skb, nhoff + sizeof(*eh)); - rcu_read_unlock(); return err; } EXPORT_SYMBOL(eth_gro_complete); diff --git a/net/ethtool/common.c b/net/ethtool/common.c index c63e0739dc6a..0c5210015911 100644 --- a/net/ethtool/common.c +++ b/net/ethtool/common.c @@ -89,6 +89,7 @@ tunable_strings[__ETHTOOL_TUNABLE_COUNT][ETH_GSTRING_LEN] = { [ETHTOOL_RX_COPYBREAK] = "rx-copybreak", [ETHTOOL_TX_COPYBREAK] = "tx-copybreak", [ETHTOOL_PFC_PREVENTION_TOUT] = "pfc-prevention-tout", + [ETHTOOL_TX_COPYBREAK_BUF_SIZE] = "tx-copybreak-buf-size", }; const char diff --git a/net/ethtool/features.c b/net/ethtool/features.c index 1c9f4df273bd..2e7331b23996 100644 --- a/net/ethtool/features.c +++ b/net/ethtool/features.c @@ -136,7 +136,6 @@ static void ethnl_features_to_bitmap(unsigned long *dest, netdev_features_t val) const unsigned int words = BITS_TO_LONGS(NETDEV_FEATURE_COUNT); unsigned int i; - bitmap_zero(dest, NETDEV_FEATURE_COUNT); for (i = 0; i < words; i++) dest[i] = (unsigned long)(val >> (i * BITS_PER_LONG)); } diff --git a/net/ethtool/ioctl.c b/net/ethtool/ioctl.c index 20bcf86970ff..9a113d893521 100644 --- a/net/ethtool/ioctl.c +++ b/net/ethtool/ioctl.c @@ -734,6 +734,9 @@ ethtool_get_drvinfo(struct net_device *dev, struct ethtool_devlink_compat *rsp) sizeof(rsp->info.bus_info)); strlcpy(rsp->info.driver, dev->dev.parent->driver->name, sizeof(rsp->info.driver)); + } else if (dev->rtnl_link_ops) { + strlcpy(rsp->info.driver, dev->rtnl_link_ops->kind, + sizeof(rsp->info.driver)); } else { return -EOPNOTSUPP; } @@ -1743,11 +1746,13 @@ static noinline_for_stack int ethtool_set_coalesce(struct net_device *dev, static int ethtool_get_ringparam(struct net_device *dev, void __user *useraddr) { struct ethtool_ringparam ringparam = { .cmd = ETHTOOL_GRINGPARAM }; + struct kernel_ethtool_ringparam kernel_ringparam = {}; if (!dev->ethtool_ops->get_ringparam) return -EOPNOTSUPP; - dev->ethtool_ops->get_ringparam(dev, &ringparam); + dev->ethtool_ops->get_ringparam(dev, &ringparam, + &kernel_ringparam, NULL); if (copy_to_user(useraddr, &ringparam, sizeof(ringparam))) return -EFAULT; @@ -1757,6 +1762,7 @@ static int ethtool_get_ringparam(struct net_device *dev, void __user *useraddr) static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) { struct ethtool_ringparam ringparam, max = { .cmd = ETHTOOL_GRINGPARAM }; + struct kernel_ethtool_ringparam kernel_ringparam; int ret; if (!dev->ethtool_ops->set_ringparam || !dev->ethtool_ops->get_ringparam) @@ -1765,7 +1771,7 @@ static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) if (copy_from_user(&ringparam, useraddr, sizeof(ringparam))) return -EFAULT; - dev->ethtool_ops->get_ringparam(dev, &max); + dev->ethtool_ops->get_ringparam(dev, &max, &kernel_ringparam, NULL); /* ensure new ring parameters are within the maximums */ if (ringparam.rx_pending > max.rx_max_pending || @@ -1774,7 +1780,8 @@ static int ethtool_set_ringparam(struct net_device *dev, void __user *useraddr) ringparam.tx_pending > max.tx_max_pending) return -EINVAL; - ret = dev->ethtool_ops->set_ringparam(dev, &ringparam); + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, + &kernel_ringparam, NULL); if (!ret) ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF, NULL); return ret; @@ -1982,6 +1989,7 @@ static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) struct ethtool_value id; static bool busy; const struct ethtool_ops *ops = dev->ethtool_ops; + netdevice_tracker dev_tracker; int rc; if (!ops->set_phys_id) @@ -2001,7 +2009,7 @@ static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) * removal of the device. */ busy = true; - dev_hold(dev); + dev_hold_track(dev, &dev_tracker, GFP_KERNEL); rtnl_unlock(); if (rc == 0) { @@ -2025,7 +2033,7 @@ static int ethtool_phys_id(struct net_device *dev, void __user *useraddr) } rtnl_lock(); - dev_put(dev); + dev_put_track(dev, &dev_tracker); busy = false; (void) ops->set_phys_id(dev, ETHTOOL_ID_INACTIVE); @@ -2396,6 +2404,7 @@ static int ethtool_tunable_valid(const struct ethtool_tunable *tuna) switch (tuna->id) { case ETHTOOL_RX_COPYBREAK: case ETHTOOL_TX_COPYBREAK: + case ETHTOOL_TX_COPYBREAK_BUF_SIZE: if (tuna->len != sizeof(u32) || tuna->type_id != ETHTOOL_TUNABLE_U32) return -EINVAL; diff --git a/net/ethtool/netlink.c b/net/ethtool/netlink.c index 96f4180aabd2..23f32a995099 100644 --- a/net/ethtool/netlink.c +++ b/net/ethtool/netlink.c @@ -142,6 +142,7 @@ int ethnl_parse_header_dev_get(struct ethnl_req_info *req_info, } req_info->dev = dev; + netdev_tracker_alloc(dev, &req_info->dev_tracker, GFP_KERNEL); req_info->flags = flags; return 0; } @@ -400,7 +401,7 @@ static int ethnl_default_doit(struct sk_buff *skb, struct genl_info *info) ops->cleanup_data(reply_data); genlmsg_end(rskb, reply_payload); - dev_put(req_info->dev); + dev_put_track(req_info->dev, &req_info->dev_tracker); kfree(reply_data); kfree(req_info); return genlmsg_reply(rskb, info); @@ -412,7 +413,7 @@ err_cleanup: if (ops->cleanup_data) ops->cleanup_data(reply_data); err_dev: - dev_put(req_info->dev); + dev_put_track(req_info->dev, &req_info->dev_tracker); kfree(reply_data); kfree(req_info); return ret; @@ -548,7 +549,7 @@ static int ethnl_default_start(struct netlink_callback *cb) * same parser as for non-dump (doit) requests is used, it * would take reference to the device if it finds one */ - dev_put(req_info->dev); + dev_put_track(req_info->dev, &req_info->dev_tracker); req_info->dev = NULL; } if (ret < 0) @@ -625,6 +626,7 @@ static void ethnl_default_notify(struct net_device *dev, unsigned int cmd, } req_info->dev = dev; + netdev_tracker_alloc(dev, &req_info->dev_tracker, GFP_KERNEL); req_info->flags |= ETHTOOL_FLAG_COMPACT_BITSETS; ethnl_init_reply_data(reply_data, ops, dev); diff --git a/net/ethtool/netlink.h b/net/ethtool/netlink.h index 836ee7157848..a779bbb0c524 100644 --- a/net/ethtool/netlink.h +++ b/net/ethtool/netlink.h @@ -222,6 +222,7 @@ static inline unsigned int ethnl_reply_header_size(void) /** * struct ethnl_req_info - base type of request information for GET requests * @dev: network device the request is for (may be null) + * @dev_tracker: refcount tracker for @dev reference * @flags: request flags common for all request types * * This is a common base for request specific structures holding data from @@ -230,6 +231,7 @@ static inline unsigned int ethnl_reply_header_size(void) */ struct ethnl_req_info { struct net_device *dev; + netdevice_tracker dev_tracker; u32 flags; }; @@ -356,7 +358,7 @@ extern const struct nla_policy ethnl_features_set_policy[ETHTOOL_A_FEATURES_WANT extern const struct nla_policy ethnl_privflags_get_policy[ETHTOOL_A_PRIVFLAGS_HEADER + 1]; extern const struct nla_policy ethnl_privflags_set_policy[ETHTOOL_A_PRIVFLAGS_FLAGS + 1]; extern const struct nla_policy ethnl_rings_get_policy[ETHTOOL_A_RINGS_HEADER + 1]; -extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_TX + 1]; +extern const struct nla_policy ethnl_rings_set_policy[ETHTOOL_A_RINGS_RX_BUF_LEN + 1]; extern const struct nla_policy ethnl_channels_get_policy[ETHTOOL_A_CHANNELS_HEADER + 1]; extern const struct nla_policy ethnl_channels_set_policy[ETHTOOL_A_CHANNELS_COMBINED_COUNT + 1]; extern const struct nla_policy ethnl_coalesce_get_policy[ETHTOOL_A_COALESCE_HEADER + 1]; diff --git a/net/ethtool/rings.c b/net/ethtool/rings.c index 4e097812a967..450b8866373d 100644 --- a/net/ethtool/rings.c +++ b/net/ethtool/rings.c @@ -10,6 +10,7 @@ struct rings_req_info { struct rings_reply_data { struct ethnl_reply_data base; struct ethtool_ringparam ringparam; + struct kernel_ethtool_ringparam kernel_ringparam; }; #define RINGS_REPDATA(__reply_base) \ @@ -25,6 +26,7 @@ static int rings_prepare_data(const struct ethnl_req_info *req_base, struct genl_info *info) { struct rings_reply_data *data = RINGS_REPDATA(reply_base); + struct netlink_ext_ack *extack = info ? info->extack : NULL; struct net_device *dev = reply_base->dev; int ret; @@ -33,7 +35,8 @@ static int rings_prepare_data(const struct ethnl_req_info *req_base, ret = ethnl_ops_begin(dev); if (ret < 0) return ret; - dev->ethtool_ops->get_ringparam(dev, &data->ringparam); + dev->ethtool_ops->get_ringparam(dev, &data->ringparam, + &data->kernel_ringparam, extack); ethnl_ops_complete(dev); return 0; @@ -49,7 +52,8 @@ static int rings_reply_size(const struct ethnl_req_info *req_base, nla_total_size(sizeof(u32)) + /* _RINGS_RX */ nla_total_size(sizeof(u32)) + /* _RINGS_RX_MINI */ nla_total_size(sizeof(u32)) + /* _RINGS_RX_JUMBO */ - nla_total_size(sizeof(u32)); /* _RINGS_TX */ + nla_total_size(sizeof(u32)) + /* _RINGS_TX */ + nla_total_size(sizeof(u32)); /* _RINGS_RX_BUF_LEN */ } static int rings_fill_reply(struct sk_buff *skb, @@ -57,6 +61,7 @@ static int rings_fill_reply(struct sk_buff *skb, const struct ethnl_reply_data *reply_base) { const struct rings_reply_data *data = RINGS_REPDATA(reply_base); + const struct kernel_ethtool_ringparam *kernel_ringparam = &data->kernel_ringparam; const struct ethtool_ringparam *ringparam = &data->ringparam; if ((ringparam->rx_max_pending && @@ -78,7 +83,10 @@ static int rings_fill_reply(struct sk_buff *skb, (nla_put_u32(skb, ETHTOOL_A_RINGS_TX_MAX, ringparam->tx_max_pending) || nla_put_u32(skb, ETHTOOL_A_RINGS_TX, - ringparam->tx_pending)))) + ringparam->tx_pending))) || + (kernel_ringparam->rx_buf_len && + (nla_put_u32(skb, ETHTOOL_A_RINGS_RX_BUF_LEN, + kernel_ringparam->rx_buf_len)))) return -EMSGSIZE; return 0; @@ -105,10 +113,12 @@ const struct nla_policy ethnl_rings_set_policy[] = { [ETHTOOL_A_RINGS_RX_MINI] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_RX_JUMBO] = { .type = NLA_U32 }, [ETHTOOL_A_RINGS_TX] = { .type = NLA_U32 }, + [ETHTOOL_A_RINGS_RX_BUF_LEN] = NLA_POLICY_MIN(NLA_U32, 1), }; int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) { + struct kernel_ethtool_ringparam kernel_ringparam = {}; struct ethtool_ringparam ringparam = {}; struct ethnl_req_info req_info = {}; struct nlattr **tb = info->attrs; @@ -134,7 +144,7 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) ret = ethnl_ops_begin(dev); if (ret < 0) goto out_rtnl; - ops->get_ringparam(dev, &ringparam); + ops->get_ringparam(dev, &ringparam, &kernel_ringparam, info->extack); ethnl_update_u32(&ringparam.rx_pending, tb[ETHTOOL_A_RINGS_RX], &mod); ethnl_update_u32(&ringparam.rx_mini_pending, @@ -142,6 +152,8 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) ethnl_update_u32(&ringparam.rx_jumbo_pending, tb[ETHTOOL_A_RINGS_RX_JUMBO], &mod); ethnl_update_u32(&ringparam.tx_pending, tb[ETHTOOL_A_RINGS_TX], &mod); + ethnl_update_u32(&kernel_ringparam.rx_buf_len, + tb[ETHTOOL_A_RINGS_RX_BUF_LEN], &mod); ret = 0; if (!mod) goto out_ops; @@ -164,7 +176,17 @@ int ethnl_set_rings(struct sk_buff *skb, struct genl_info *info) goto out_ops; } - ret = dev->ethtool_ops->set_ringparam(dev, &ringparam); + if (kernel_ringparam.rx_buf_len != 0 && + !(ops->supported_ring_params & ETHTOOL_RING_USE_RX_BUF_LEN)) { + ret = -EOPNOTSUPP; + NL_SET_ERR_MSG_ATTR(info->extack, + tb[ETHTOOL_A_RINGS_RX_BUF_LEN], + "setting rx buf len not supported"); + goto out_ops; + } + + ret = dev->ethtool_ops->set_ringparam(dev, &ringparam, + &kernel_ringparam, info->extack); if (ret < 0) goto out_ops; ethtool_notify(dev, ETHTOOL_MSG_RINGS_NTF, NULL); diff --git a/net/ethtool/stats.c b/net/ethtool/stats.c index ec07f5765e03..a20e0a24ff61 100644 --- a/net/ethtool/stats.c +++ b/net/ethtool/stats.c @@ -14,10 +14,12 @@ struct stats_req_info { struct stats_reply_data { struct ethnl_reply_data base; - struct ethtool_eth_phy_stats phy_stats; - struct ethtool_eth_mac_stats mac_stats; - struct ethtool_eth_ctrl_stats ctrl_stats; - struct ethtool_rmon_stats rmon_stats; + struct_group(stats, + struct ethtool_eth_phy_stats phy_stats; + struct ethtool_eth_mac_stats mac_stats; + struct ethtool_eth_ctrl_stats ctrl_stats; + struct ethtool_rmon_stats rmon_stats; + ); const struct ethtool_rmon_hist_range *rmon_ranges; }; @@ -117,10 +119,7 @@ static int stats_prepare_data(const struct ethnl_req_info *req_base, /* Mark all stats as unset (see ETHTOOL_STAT_NOT_SET) to prevent them * from being reported to user space in case driver did not set them. */ - memset(&data->phy_stats, 0xff, sizeof(data->phy_stats)); - memset(&data->mac_stats, 0xff, sizeof(data->mac_stats)); - memset(&data->ctrl_stats, 0xff, sizeof(data->ctrl_stats)); - memset(&data->rmon_stats, 0xff, sizeof(data->rmon_stats)); + memset(&data->stats, 0xff, sizeof(data->stats)); if (test_bit(ETHTOOL_STATS_ETH_PHY, req_info->stat_mask) && dev->ethtool_ops->get_eth_phy_stats) diff --git a/net/hsr/hsr_device.c b/net/hsr/hsr_device.c index 737e4f17e1c6..e57fdad9ef94 100644 --- a/net/hsr/hsr_device.c +++ b/net/hsr/hsr_device.c @@ -30,13 +30,13 @@ static bool is_slave_up(struct net_device *dev) static void __hsr_set_operstate(struct net_device *dev, int transition) { - write_lock_bh(&dev_base_lock); + write_lock(&dev_base_lock); if (dev->operstate != transition) { dev->operstate = transition; - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); netdev_state_change(dev); } else { - write_unlock_bh(&dev_base_lock); + write_unlock(&dev_base_lock); } } diff --git a/net/ieee802154/socket.c b/net/ieee802154/socket.c index 7bb9ef35c570..3b2366a88c3c 100644 --- a/net/ieee802154/socket.c +++ b/net/ieee802154/socket.c @@ -174,8 +174,8 @@ static int raw_hash(struct sock *sk) { write_lock_bh(&raw_lock); sk_add_node(sk, &raw_head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&raw_lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } @@ -453,8 +453,8 @@ static int dgram_hash(struct sock *sk) { write_lock_bh(&dgram_lock); sk_add_node(sk, &dgram_head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&dgram_lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } diff --git a/net/ipv4/af_inet.c b/net/ipv4/af_inet.c index 0189e3cd4a7d..04067b249bf3 100644 --- a/net/ipv4/af_inet.c +++ b/net/ipv4/af_inet.c @@ -99,6 +99,7 @@ #include <net/route.h> #include <net/ip_fib.h> #include <net/inet_connection_sock.h> +#include <net/gro.h> #include <net/tcp.h> #include <net/udp.h> #include <net/udplite.h> @@ -224,7 +225,7 @@ int inet_listen(struct socket *sock, int backlog) tcp_fastopen_init_key_once(sock_net(sk)); } - err = inet_csk_listen_start(sk, backlog); + err = inet_csk_listen_start(sk); if (err) goto out; tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_LISTEN_CB, 0, NULL); @@ -488,11 +489,8 @@ int __inet_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, * is temporarily down) */ err = -EADDRNOTAVAIL; - if (!inet_can_nonlocal_bind(net, inet) && - addr->sin_addr.s_addr != htonl(INADDR_ANY) && - chk_addr_ret != RTN_LOCAL && - chk_addr_ret != RTN_MULTICAST && - chk_addr_ret != RTN_BROADCAST) + if (!inet_addr_valid_or_nonlocal(net, inet, addr->sin_addr.s_addr, + chk_addr_ret)) goto out; snum = ntohs(addr->sin_port); @@ -1454,19 +1452,18 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) proto = iph->protocol; - rcu_read_lock(); ops = rcu_dereference(inet_offloads[proto]); if (!ops || !ops->callbacks.gro_receive) - goto out_unlock; + goto out; if (*(u8 *)iph != 0x45) - goto out_unlock; + goto out; if (ip_is_fragment(iph)) - goto out_unlock; + goto out; if (unlikely(ip_fast_csum((u8 *)iph, 5))) - goto out_unlock; + goto out; id = ntohl(*(__be32 *)&iph->id); flush = (u16)((ntohl(*(__be32 *)iph) ^ skb_gro_len(skb)) | (id & ~IP_DF)); @@ -1543,9 +1540,6 @@ struct sk_buff *inet_gro_receive(struct list_head *head, struct sk_buff *skb) pp = indirect_call_gro_receive(tcp4_gro_receive, udp4_gro_receive, ops->callbacks.gro_receive, head, skb); -out_unlock: - rcu_read_unlock(); - out: skb_gro_flush_final(skb, pp, flush); @@ -1618,10 +1612,9 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff) csum_replace2(&iph->check, iph->tot_len, newlen); iph->tot_len = newlen; - rcu_read_lock(); ops = rcu_dereference(inet_offloads[proto]); if (WARN_ON(!ops || !ops->callbacks.gro_complete)) - goto out_unlock; + goto out; /* Only need to add sizeof(*iph) to get to the next hdr below * because any hdr with option will have been flushed in @@ -1631,9 +1624,7 @@ int inet_gro_complete(struct sk_buff *skb, int nhoff) tcp4_gro_complete, udp4_gro_complete, skb, nhoff + sizeof(*iph)); -out_unlock: - rcu_read_unlock(); - +out: return err; } diff --git a/net/ipv4/arp.c b/net/ipv4/arp.c index 857a144b1ea9..4db0325f6e1a 100644 --- a/net/ipv4/arp.c +++ b/net/ipv4/arp.c @@ -1299,21 +1299,6 @@ static struct packet_type arp_packet_type __read_mostly = { .func = arp_rcv, }; -static int arp_proc_init(void); - -void __init arp_init(void) -{ - neigh_table_init(NEIGH_ARP_TABLE, &arp_tbl); - - dev_add_pack(&arp_packet_type); - arp_proc_init(); -#ifdef CONFIG_SYSCTL - neigh_sysctl_register(NULL, &arp_tbl.parms, NULL); -#endif - register_netdevice_notifier(&arp_netdev_notifier); -} - -#ifdef CONFIG_PROC_FS #if IS_ENABLED(CONFIG_AX25) /* ------------------------------------------------------------------------ */ @@ -1451,16 +1436,14 @@ static struct pernet_operations arp_net_ops = { .exit = arp_net_exit, }; -static int __init arp_proc_init(void) +void __init arp_init(void) { - return register_pernet_subsys(&arp_net_ops); -} - -#else /* CONFIG_PROC_FS */ + neigh_table_init(NEIGH_ARP_TABLE, &arp_tbl); -static int __init arp_proc_init(void) -{ - return 0; + dev_add_pack(&arp_packet_type); + register_pernet_subsys(&arp_net_ops); +#ifdef CONFIG_SYSCTL + neigh_sysctl_register(NULL, &arp_tbl.parms, NULL); +#endif + register_netdevice_notifier(&arp_netdev_notifier); } - -#endif /* CONFIG_PROC_FS */ diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index 323e622ff9b7..fba2bffd65f7 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -243,7 +243,7 @@ void in_dev_finish_destroy(struct in_device *idev) #ifdef NET_REFCNT_DEBUG pr_debug("%s: %p=%s\n", __func__, idev, dev ? dev->name : "NIL"); #endif - dev_put(dev); + dev_put_track(dev, &idev->dev_tracker); if (!idev->dead) pr_err("Freeing alive in_device %p\n", idev); else @@ -271,7 +271,7 @@ static struct in_device *inetdev_init(struct net_device *dev) if (IPV4_DEVCONF(in_dev->cnf, FORWARDING)) dev_disable_lro(dev); /* Reference in_dev->dev */ - dev_hold(dev); + dev_hold_track(dev, &in_dev->dev_tracker, GFP_KERNEL); /* Account for reference dev->ip_ptr (below) */ refcount_set(&in_dev->refcnt, 1); diff --git a/net/ipv4/esp4_offload.c b/net/ipv4/esp4_offload.c index 8e4e9aa12130..d87f02a6e934 100644 --- a/net/ipv4/esp4_offload.c +++ b/net/ipv4/esp4_offload.c @@ -16,6 +16,7 @@ #include <crypto/authenc.h> #include <linux/err.h> #include <linux/module.h> +#include <net/gro.h> #include <net/ip.h> #include <net/xfrm.h> #include <net/esp.h> diff --git a/net/ipv4/fib_semantics.c b/net/ipv4/fib_semantics.c index fde7797b5806..3cad543dc747 100644 --- a/net/ipv4/fib_semantics.c +++ b/net/ipv4/fib_semantics.c @@ -208,7 +208,7 @@ static void rt_fibinfo_free_cpus(struct rtable __rcu * __percpu *rtp) void fib_nh_common_release(struct fib_nh_common *nhc) { - dev_put(nhc->nhc_dev); + dev_put_track(nhc->nhc_dev, &nhc->nhc_dev_tracker); lwtstate_put(nhc->nhc_lwtstate); rt_fibinfo_free_cpus(nhc->nhc_pcpu_rth_output); rt_fibinfo_free(&nhc->nhc_rth_input); @@ -1006,7 +1006,7 @@ static int fib_check_nh_v6_gw(struct net *net, struct fib_nh *nh, err = ipv6_stub->fib6_nh_init(net, &fib6_nh, &cfg, GFP_KERNEL, extack); if (!err) { nh->fib_nh_dev = fib6_nh.fib_nh_dev; - dev_hold(nh->fib_nh_dev); + dev_hold_track(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_KERNEL); nh->fib_nh_oif = nh->fib_nh_dev->ifindex; nh->fib_nh_scope = RT_SCOPE_LINK; @@ -1090,7 +1090,7 @@ static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table, if (!netif_carrier_ok(dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; nh->fib_nh_dev = dev; - dev_hold(dev); + dev_hold_track(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); nh->fib_nh_scope = RT_SCOPE_LINK; return 0; } @@ -1144,7 +1144,7 @@ static int fib_check_nh_v4_gw(struct net *net, struct fib_nh *nh, u32 table, "No egress device for nexthop gateway"); goto out; } - dev_hold(dev); + dev_hold_track(dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); if (!netif_carrier_ok(dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; err = (dev->flags & IFF_UP) ? 0 : -ENETDOWN; @@ -1178,7 +1178,7 @@ static int fib_check_nh_nongw(struct net *net, struct fib_nh *nh, } nh->fib_nh_dev = in_dev->dev; - dev_hold(nh->fib_nh_dev); + dev_hold_track(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, GFP_ATOMIC); nh->fib_nh_scope = RT_SCOPE_HOST; if (!netif_carrier_ok(nh->fib_nh_dev)) nh->fib_nh_flags |= RTNH_F_LINKDOWN; @@ -1508,6 +1508,8 @@ struct fib_info *fib_create_info(struct fib_config *cfg, err = -ENODEV; if (!nh->fib_nh_dev) goto failure; + netdev_tracker_alloc(nh->fib_nh_dev, &nh->fib_nh_dev_tracker, + GFP_KERNEL); } else { int linkdown = 0; diff --git a/net/ipv4/fou.c b/net/ipv4/fou.c index 8fcbc6258ec5..0d085cc8d96c 100644 --- a/net/ipv4/fou.c +++ b/net/ipv4/fou.c @@ -9,6 +9,7 @@ #include <linux/types.h> #include <linux/kernel.h> #include <net/genetlink.h> +#include <net/gro.h> #include <net/gue.h> #include <net/fou.h> #include <net/ip.h> @@ -246,17 +247,14 @@ static struct sk_buff *fou_gro_receive(struct sock *sk, /* Flag this frame as already having an outer encap header */ NAPI_GRO_CB(skb)->is_fou = 1; - rcu_read_lock(); offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; ops = rcu_dereference(offloads[proto]); if (!ops || !ops->callbacks.gro_receive) - goto out_unlock; + goto out; pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); -out_unlock: - rcu_read_unlock(); - +out: return pp; } @@ -268,19 +266,16 @@ static int fou_gro_complete(struct sock *sk, struct sk_buff *skb, const struct net_offload *ops; int err = -ENOSYS; - rcu_read_lock(); offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; ops = rcu_dereference(offloads[proto]); if (WARN_ON(!ops || !ops->callbacks.gro_complete)) - goto out_unlock; + goto out; err = ops->callbacks.gro_complete(skb, nhoff); skb_set_inner_mac_header(skb, nhoff); -out_unlock: - rcu_read_unlock(); - +out: return err; } @@ -438,17 +433,14 @@ next_proto: /* Flag this frame as already having an outer encap header */ NAPI_GRO_CB(skb)->is_fou = 1; - rcu_read_lock(); offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; ops = rcu_dereference(offloads[proto]); if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive)) - goto out_unlock; + goto out; pp = call_gro_receive(ops->callbacks.gro_receive, head, skb); flush = 0; -out_unlock: - rcu_read_unlock(); out: skb_gro_flush_final_remcsum(skb, pp, flush, &grc); @@ -485,18 +477,16 @@ static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff) return err; } - rcu_read_lock(); offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads; ops = rcu_dereference(offloads[proto]); if (WARN_ON(!ops || !ops->callbacks.gro_complete)) - goto out_unlock; + goto out; err = ops->callbacks.gro_complete(skb, nhoff + guehlen); skb_set_inner_mac_header(skb, nhoff + guehlen); -out_unlock: - rcu_read_unlock(); +out: return err; } diff --git a/net/ipv4/gre_offload.c b/net/ipv4/gre_offload.c index 1121a9d5fed9..07073fa35205 100644 --- a/net/ipv4/gre_offload.c +++ b/net/ipv4/gre_offload.c @@ -10,6 +10,7 @@ #include <linux/init.h> #include <net/protocol.h> #include <net/gre.h> +#include <net/gro.h> static struct sk_buff *gre_gso_segment(struct sk_buff *skb, netdev_features_t features) @@ -162,10 +163,9 @@ static struct sk_buff *gre_gro_receive(struct list_head *head, type = greh->protocol; - rcu_read_lock(); ptype = gro_find_receive_by_type(type); if (!ptype) - goto out_unlock; + goto out; grehlen = GRE_HEADER_SECTION; @@ -179,13 +179,13 @@ static struct sk_buff *gre_gro_receive(struct list_head *head, if (skb_gro_header_hard(skb, hlen)) { greh = skb_gro_header_slow(skb, hlen, off); if (unlikely(!greh)) - goto out_unlock; + goto out; } /* Don't bother verifying checksum if we're going to flush anyway. */ if ((greh->flags & GRE_CSUM) && !NAPI_GRO_CB(skb)->flush) { if (skb_gro_checksum_simple_validate(skb)) - goto out_unlock; + goto out; skb_gro_checksum_try_convert(skb, IPPROTO_GRE, null_compute_pseudo); @@ -229,8 +229,6 @@ static struct sk_buff *gre_gro_receive(struct list_head *head, pp = call_gro_receive(ptype->callbacks.gro_receive, head, skb); flush = 0; -out_unlock: - rcu_read_unlock(); out: skb_gro_flush_final(skb, pp, flush); @@ -255,13 +253,10 @@ static int gre_gro_complete(struct sk_buff *skb, int nhoff) if (greh->flags & GRE_CSUM) grehlen += GRE_HEADER_SECTION; - rcu_read_lock(); ptype = gro_find_complete_by_type(type); if (ptype) err = ptype->callbacks.gro_complete(skb, nhoff + grehlen); - rcu_read_unlock(); - skb_set_inner_mac_header(skb, nhoff + grehlen); return err; diff --git a/net/ipv4/igmp.c b/net/ipv4/igmp.c index d2e2b3d18c66..2ad3c7b42d6d 100644 --- a/net/ipv4/igmp.c +++ b/net/ipv4/igmp.c @@ -2558,7 +2558,6 @@ int ip_mc_msfget(struct sock *sk, struct ip_msfilter *msf, msf->imsf_fmode = pmc->sfmode; psl = rtnl_dereference(pmc->sflist); if (!psl) { - len = 0; count = 0; } else { count = psl->sl_count; diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c index 62a67fdc344c..fc2a985f6064 100644 --- a/net/ipv4/inet_connection_sock.c +++ b/net/ipv4/inet_connection_sock.c @@ -1035,7 +1035,7 @@ void inet_csk_prepare_forced_close(struct sock *sk) } EXPORT_SYMBOL(inet_csk_prepare_forced_close); -int inet_csk_listen_start(struct sock *sk, int backlog) +int inet_csk_listen_start(struct sock *sk) { struct inet_connection_sock *icsk = inet_csk(sk); struct inet_sock *inet = inet_sk(sk); diff --git a/net/ipv4/inet_hashtables.c b/net/ipv4/inet_hashtables.c index 75737267746f..30ab717ff1b8 100644 --- a/net/ipv4/inet_hashtables.c +++ b/net/ipv4/inet_hashtables.c @@ -307,7 +307,7 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net, struct inet_hashinfo *hashinfo, struct sk_buff *skb, int doff, __be32 saddr, __be16 sport, - __be32 daddr, u16 hnum) + __be32 daddr, u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -315,8 +315,8 @@ static inline struct sock *inet_lookup_run_bpf(struct net *net, if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ - no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_TCP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -340,7 +340,7 @@ struct sock *__inet_lookup_listener(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet_lookup_run_bpf(net, hashinfo, skb, doff, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (result) goto done; } diff --git a/net/ipv4/ip_output.c b/net/ipv4/ip_output.c index 9bca57ef8b83..57c1d8431386 100644 --- a/net/ipv4/ip_output.c +++ b/net/ipv4/ip_output.c @@ -672,7 +672,6 @@ struct sk_buff *ip_frag_next(struct sk_buff *skb, struct ip_frag_state *state) struct sk_buff *skb2; struct iphdr *iph; - len = state->left; /* IF: it doesn't fit, use 'mtu' - the data space left */ if (len > state->mtu) len = state->mtu; diff --git a/net/ipv4/ip_sockglue.c b/net/ipv4/ip_sockglue.c index 38d29b175ca6..445a9ecaefa1 100644 --- a/net/ipv4/ip_sockglue.c +++ b/net/ipv4/ip_sockglue.c @@ -576,7 +576,7 @@ out: return err; } -static void __ip_sock_set_tos(struct sock *sk, int val) +void __ip_sock_set_tos(struct sock *sk, int val) { if (sk->sk_type == SOCK_STREAM) { val &= ~INET_ECN_MASK; diff --git a/net/ipv4/ipmr.c b/net/ipv4/ipmr.c index 2dda856ca260..4c7aca884fa9 100644 --- a/net/ipv4/ipmr.c +++ b/net/ipv4/ipmr.c @@ -696,7 +696,7 @@ static int vif_delete(struct mr_table *mrt, int vifi, int notify, if (v->flags & (VIFF_TUNNEL | VIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put(dev); + dev_put_track(dev, &v->dev_tracker); return 0; } @@ -896,6 +896,7 @@ static int vif_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); v->dev = dev; + netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); if (v->flags & VIFF_REGISTER) mrt->mroute_reg_vif_num = vifi; if (vifi+1 > mrt->maxvif) diff --git a/net/ipv4/nexthop.c b/net/ipv4/nexthop.c index 5dbd4b5505eb..1319d093cdda 100644 --- a/net/ipv4/nexthop.c +++ b/net/ipv4/nexthop.c @@ -1918,9 +1918,6 @@ static void nh_rt_cache_flush(struct net *net, struct nexthop *nh, if (!replaced_nh->is_group) return; - /* new dsts must use only the new nexthop group */ - synchronize_net(); - nhg = rtnl_dereference(replaced_nh->nh_grp); for (i = 0; i < nhg->num_nh; i++) { struct nh_grp_entry *nhge = &nhg->nh_entries[i]; @@ -2002,9 +1999,10 @@ static int replace_nexthop_grp(struct net *net, struct nexthop *old, rcu_assign_pointer(old->nh_grp, newg); + /* Make sure concurrent readers are not using 'oldg' anymore. */ + synchronize_net(); + if (newg->resilient) { - /* Make sure concurrent readers are not using 'oldg' anymore. */ - synchronize_net(); rcu_assign_pointer(oldg->res_table, tmp_table); rcu_assign_pointer(oldg->spare->res_table, tmp_table); } diff --git a/net/ipv4/ping.c b/net/ipv4/ping.c index 1e44a43acfe2..e540b0dcf085 100644 --- a/net/ipv4/ping.c +++ b/net/ipv4/ping.c @@ -311,15 +311,11 @@ static int ping_check_bind_addr(struct sock *sk, struct inet_sock *isk, pr_debug("ping_check_bind_addr(sk=%p,addr=%pI4,port=%d)\n", sk, &addr->sin_addr.s_addr, ntohs(addr->sin_port)); - if (addr->sin_addr.s_addr == htonl(INADDR_ANY)) - chk_addr_ret = RTN_LOCAL; - else - chk_addr_ret = inet_addr_type(net, addr->sin_addr.s_addr); - - if ((!inet_can_nonlocal_bind(net, isk) && - chk_addr_ret != RTN_LOCAL) || - chk_addr_ret == RTN_MULTICAST || - chk_addr_ret == RTN_BROADCAST) + chk_addr_ret = inet_addr_type(net, addr->sin_addr.s_addr); + + if (!inet_addr_valid_or_nonlocal(net, inet_sk(sk), + addr->sin_addr.s_addr, + chk_addr_ret)) return -EADDRNOTAVAIL; #if IS_ENABLED(CONFIG_IPV6) diff --git a/net/ipv4/raw.c b/net/ipv4/raw.c index bb446e60cf58..a53f256bf9d3 100644 --- a/net/ipv4/raw.c +++ b/net/ipv4/raw.c @@ -99,8 +99,8 @@ int raw_hash_sk(struct sock *sk) write_lock_bh(&h->lock); sk_add_node(sk, head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&h->lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } @@ -717,6 +717,7 @@ static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len) { struct inet_sock *inet = inet_sk(sk); struct sockaddr_in *addr = (struct sockaddr_in *) uaddr; + struct net *net = sock_net(sk); u32 tb_id = RT_TABLE_LOCAL; int ret = -EINVAL; int chk_addr_ret; @@ -725,16 +726,16 @@ static int raw_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len) goto out; if (sk->sk_bound_dev_if) - tb_id = l3mdev_fib_table_by_index(sock_net(sk), - sk->sk_bound_dev_if) ? : tb_id; + tb_id = l3mdev_fib_table_by_index(net, + sk->sk_bound_dev_if) ? : tb_id; - chk_addr_ret = inet_addr_type_table(sock_net(sk), addr->sin_addr.s_addr, - tb_id); + chk_addr_ret = inet_addr_type_table(net, addr->sin_addr.s_addr, tb_id); ret = -EADDRNOTAVAIL; - if (addr->sin_addr.s_addr && chk_addr_ret != RTN_LOCAL && - chk_addr_ret != RTN_MULTICAST && chk_addr_ret != RTN_BROADCAST) + if (!inet_addr_valid_or_nonlocal(net, inet, addr->sin_addr.s_addr, + chk_addr_ret)) goto out; + inet->inet_rcv_saddr = inet->inet_saddr = addr->sin_addr.s_addr; if (chk_addr_ret == RTN_MULTICAST || chk_addr_ret == RTN_BROADCAST) inet->inet_saddr = 0; /* Use device */ diff --git a/net/ipv4/route.c b/net/ipv4/route.c index 0b4103b1e622..843a7a3699fe 100644 --- a/net/ipv4/route.c +++ b/net/ipv4/route.c @@ -602,7 +602,7 @@ static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash) static u32 fnhe_hashfun(__be32 daddr) { - static siphash_key_t fnhe_hash_key __read_mostly; + static siphash_aligned_key_t fnhe_hash_key; u64 hval; net_get_random_once(&fnhe_hash_key, sizeof(fnhe_hash_key)); @@ -1531,8 +1531,9 @@ void rt_flush_dev(struct net_device *dev) if (rt->dst.dev != dev) continue; rt->dst.dev = blackhole_netdev; - dev_hold(rt->dst.dev); - dev_put(dev); + dev_replace_track(dev, blackhole_netdev, + &rt->dst.dev_tracker, + GFP_ATOMIC); } spin_unlock_bh(&ul->lock); } @@ -2819,7 +2820,7 @@ struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_or new->output = dst_discard_out; new->dev = net->loopback_dev; - dev_hold(new->dev); + dev_hold_track(new->dev, &new->dev_tracker, GFP_ATOMIC); rt->rt_is_input = ort->rt_is_input; rt->rt_iif = ort->rt_iif; diff --git a/net/ipv4/syncookies.c b/net/ipv4/syncookies.c index 8696dc343ad2..2cb3b852d148 100644 --- a/net/ipv4/syncookies.c +++ b/net/ipv4/syncookies.c @@ -14,7 +14,7 @@ #include <net/tcp.h> #include <net/route.h> -static siphash_key_t syncookie_secret[2] __read_mostly; +static siphash_aligned_key_t syncookie_secret[2]; #define COOKIEBITS 24 /* Upper bits store count */ #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1) diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c index bbb3d39c69af..20054618c87e 100644 --- a/net/ipv4/tcp.c +++ b/net/ipv4/tcp.c @@ -292,7 +292,7 @@ EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count); long sysctl_tcp_mem[3] __read_mostly; EXPORT_SYMBOL(sysctl_tcp_mem); -atomic_long_t tcp_memory_allocated; /* Current allocated memory. */ +atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */ EXPORT_SYMBOL(tcp_memory_allocated); #if IS_ENABLED(CONFIG_SMC) @@ -303,7 +303,7 @@ EXPORT_SYMBOL(tcp_have_smc); /* * Current number of TCP sockets. */ -struct percpu_counter tcp_sockets_allocated; +struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp; EXPORT_SYMBOL(tcp_sockets_allocated); /* @@ -456,7 +456,6 @@ void tcp_init_sock(struct sock *sk) WRITE_ONCE(sk->sk_rcvbuf, sock_net(sk)->ipv4.sysctl_tcp_rmem[1]); sk_sockets_allocated_inc(sk); - sk->sk_route_forced_caps = NETIF_F_GSO; } EXPORT_SYMBOL(tcp_init_sock); @@ -546,10 +545,11 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) if (state != TCP_SYN_SENT && (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) { int target = sock_rcvlowat(sk, 0, INT_MAX); + u16 urg_data = READ_ONCE(tp->urg_data); - if (READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && - !sock_flag(sk, SOCK_URGINLINE) && - tp->urg_data) + if (unlikely(urg_data) && + READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) && + !sock_flag(sk, SOCK_URGINLINE)) target++; if (tcp_stream_is_readable(sk, target)) @@ -574,7 +574,7 @@ __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait) } else mask |= EPOLLOUT | EPOLLWRNORM; - if (tp->urg_data & TCP_URG_VALID) + if (urg_data & TCP_URG_VALID) mask |= EPOLLPRI; } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) { /* Active TCP fastopen socket with defer_connect @@ -608,7 +608,7 @@ int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg) unlock_sock_fast(sk, slow); break; case SIOCATMARK: - answ = tp->urg_data && + answ = READ_ONCE(tp->urg_data) && READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq); break; case SIOCOUTQ: @@ -1466,7 +1466,7 @@ static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags) char c = tp->urg_data; if (!(flags & MSG_PEEK)) - tp->urg_data = TCP_URG_READ; + WRITE_ONCE(tp->urg_data, TCP_URG_READ); /* Read urgent data. */ msg->msg_flags |= MSG_OOB; @@ -1580,6 +1580,36 @@ void tcp_cleanup_rbuf(struct sock *sk, int copied) tcp_send_ack(sk); } +void __sk_defer_free_flush(struct sock *sk) +{ + struct llist_node *head; + struct sk_buff *skb, *n; + + head = llist_del_all(&sk->defer_list); + llist_for_each_entry_safe(skb, n, head, ll_node) { + prefetch(n); + skb_mark_not_on_list(skb); + __kfree_skb(skb); + } +} +EXPORT_SYMBOL(__sk_defer_free_flush); + +static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb) +{ + __skb_unlink(skb, &sk->sk_receive_queue); + if (likely(skb->destructor == sock_rfree)) { + sock_rfree(skb); + skb->destructor = NULL; + skb->sk = NULL; + if (!skb_queue_empty(&sk->sk_receive_queue) || + !llist_empty(&sk->defer_list)) { + llist_add(&skb->ll_node, &sk->defer_list); + return; + } + } + __kfree_skb(skb); +} + static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) { struct sk_buff *skb; @@ -1599,7 +1629,7 @@ static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off) * splitted a fat GRO packet, while we released socket lock * in skb_splice_bits() */ - sk_eat_skb(sk, skb); + tcp_eat_recv_skb(sk, skb); } return NULL; } @@ -1633,7 +1663,7 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, len = skb->len - offset; /* Stop reading if we hit a patch of urgent data */ - if (tp->urg_data) { + if (unlikely(tp->urg_data)) { u32 urg_offset = tp->urg_seq - seq; if (urg_offset < len) len = urg_offset; @@ -1665,11 +1695,11 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc, continue; } if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) { - sk_eat_skb(sk, skb); + tcp_eat_recv_skb(sk, skb); ++seq; break; } - sk_eat_skb(sk, skb); + tcp_eat_recv_skb(sk, skb); if (!desc->count) break; WRITE_ONCE(tp->copied_seq, seq); @@ -2329,7 +2359,7 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, u32 offset; /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */ - if (tp->urg_data && tp->urg_seq == *seq) { + if (unlikely(tp->urg_data) && tp->urg_seq == *seq) { if (copied) break; if (signal_pending(current)) { @@ -2372,10 +2402,10 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, break; if (copied) { - if (sk->sk_err || + if (!timeo || + sk->sk_err || sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN) || - !timeo || signal_pending(current)) break; } else { @@ -2409,13 +2439,12 @@ static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len, } } - tcp_cleanup_rbuf(sk, copied); - if (copied >= target) { /* Do not sleep, just process backlog. */ - release_sock(sk); - lock_sock(sk); + __sk_flush_backlog(sk); } else { + tcp_cleanup_rbuf(sk, copied); + sk_defer_free_flush(sk); sk_wait_data(sk, &timeo, last); } @@ -2435,7 +2464,7 @@ found_ok_skb: used = len; /* Do we have urgent data here? */ - if (tp->urg_data) { + if (unlikely(tp->urg_data)) { u32 urg_offset = tp->urg_seq - *seq; if (urg_offset < used) { if (!urg_offset) { @@ -2469,8 +2498,8 @@ found_ok_skb: tcp_rcv_space_adjust(sk); skip_copy: - if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) { - tp->urg_data = 0; + if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) { + WRITE_ONCE(tp->urg_data, 0); tcp_fast_path_check(sk); } @@ -2485,14 +2514,14 @@ skip_copy: if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) goto found_fin_ok; if (!(flags & MSG_PEEK)) - sk_eat_skb(sk, skb); + tcp_eat_recv_skb(sk, skb); continue; found_fin_ok: /* Process the FIN. */ WRITE_ONCE(*seq, *seq + 1); if (!(flags & MSG_PEEK)) - sk_eat_skb(sk, skb); + tcp_eat_recv_skb(sk, skb); break; } while (len > 0); @@ -2534,6 +2563,7 @@ int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, ret = tcp_recvmsg_locked(sk, msg, len, nonblock, flags, &tss, &cmsg_flags); release_sock(sk); + sk_defer_free_flush(sk); if (cmsg_flags && ret >= 0) { if (cmsg_flags & TCP_CMSG_TS) @@ -2964,7 +2994,7 @@ int tcp_disconnect(struct sock *sk, int flags) tcp_clear_xmit_timers(sk); __skb_queue_purge(&sk->sk_receive_queue); WRITE_ONCE(tp->copied_seq, tp->rcv_nxt); - tp->urg_data = 0; + WRITE_ONCE(tp->urg_data, 0); tcp_write_queue_purge(sk); tcp_fastopen_active_disable_ofo_check(sk); skb_rbtree_purge(&tp->out_of_order_queue); @@ -3059,7 +3089,7 @@ int tcp_disconnect(struct sock *sk, int flags) sk->sk_frag.page = NULL; sk->sk_frag.offset = 0; } - + sk_defer_free_flush(sk); sk_error_report(sk); return 0; } @@ -3177,7 +3207,7 @@ static void tcp_enable_tx_delay(void) * TCP_CORK can be set together with TCP_NODELAY and it is stronger than * TCP_NODELAY. */ -static void __tcp_sock_set_cork(struct sock *sk, bool on) +void __tcp_sock_set_cork(struct sock *sk, bool on) { struct tcp_sock *tp = tcp_sk(sk); @@ -3205,7 +3235,7 @@ EXPORT_SYMBOL(tcp_sock_set_cork); * However, when TCP_NODELAY is set we make an explicit push, which overrides * even TCP_CORK for currently queued segments. */ -static void __tcp_sock_set_nodelay(struct sock *sk, bool on) +void __tcp_sock_set_nodelay(struct sock *sk, bool on) { if (on) { tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH; @@ -3774,10 +3804,12 @@ void tcp_get_info(struct sock *sk, struct tcp_info *info) tcp_get_info_chrono_stats(tp, info); info->tcpi_segs_out = tp->segs_out; - info->tcpi_segs_in = tp->segs_in; + + /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */ + info->tcpi_segs_in = READ_ONCE(tp->segs_in); + info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in); info->tcpi_min_rtt = tcp_min_rtt(tp); - info->tcpi_data_segs_in = tp->data_segs_in; info->tcpi_data_segs_out = tp->data_segs_out; info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0; @@ -4186,6 +4218,7 @@ static int do_tcp_getsockopt(struct sock *sk, int level, err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname, &zc, &len, err); release_sock(sk); + sk_defer_free_flush(sk); if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags)) goto zerocopy_rcv_cmsg; switch (len) { diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c index 246ab7b5e857..3658b9c3dd2b 100644 --- a/net/ipv4/tcp_input.c +++ b/net/ipv4/tcp_input.c @@ -5591,7 +5591,7 @@ static void tcp_check_urg(struct sock *sk, const struct tcphdr *th) } } - tp->urg_data = TCP_URG_NOTYET; + WRITE_ONCE(tp->urg_data, TCP_URG_NOTYET); WRITE_ONCE(tp->urg_seq, ptr); /* Disable header prediction. */ @@ -5604,11 +5604,11 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t struct tcp_sock *tp = tcp_sk(sk); /* Check if we get a new urgent pointer - normally not. */ - if (th->urg) + if (unlikely(th->urg)) tcp_check_urg(sk, th); /* Do we wait for any urgent data? - normally not... */ - if (tp->urg_data == TCP_URG_NOTYET) { + if (unlikely(tp->urg_data == TCP_URG_NOTYET)) { u32 ptr = tp->urg_seq - ntohl(th->seq) + (th->doff * 4) - th->syn; @@ -5617,7 +5617,7 @@ static void tcp_urg(struct sock *sk, struct sk_buff *skb, const struct tcphdr *t u8 tmp; if (skb_copy_bits(skb, ptr, &tmp, 1)) BUG(); - tp->urg_data = TCP_URG_VALID | tmp; + WRITE_ONCE(tp->urg_data, TCP_URG_VALID | tmp); if (!sock_flag(sk, SOCK_DEAD)) sk->sk_data_ready(sk); } diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index 13d868c43284..3dd19a2bf06c 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -1182,7 +1182,7 @@ int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr, if (!md5sig) return -ENOMEM; - sk_nocaps_add(sk, NETIF_F_GSO_MASK); + sk_gso_disable(sk); INIT_HLIST_HEAD(&md5sig->head); rcu_assign_pointer(tp->md5sig_info, md5sig); } @@ -1620,7 +1620,7 @@ struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb, */ tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index, key->flags, key->key, key->keylen, GFP_ATOMIC); - sk_nocaps_add(newsk, NETIF_F_GSO_MASK); + sk_gso_disable(newsk); } #endif @@ -1800,8 +1800,7 @@ int tcp_v4_early_demux(struct sk_buff *skb) bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb) { - u32 limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf); - u32 tail_gso_size, tail_gso_segs; + u32 limit, tail_gso_size, tail_gso_segs; struct skb_shared_info *shinfo; const struct tcphdr *th; struct tcphdr *thtail; @@ -1909,7 +1908,7 @@ no_coalesce: * to reduce memory overhead, so add a little headroom here. * Few sockets backlog are possibly concurrently non empty. */ - limit += 64*1024; + limit = READ_ONCE(sk->sk_rcvbuf) + READ_ONCE(sk->sk_sndbuf) + 64*1024; if (unlikely(sk_add_backlog(sk, skb, limit))) { bh_unlock_sock(sk); @@ -2103,6 +2102,7 @@ process: sk_incoming_cpu_update(sk); + sk_defer_free_flush(sk); bh_lock_sock_nested(sk); tcp_segs_in(tcp_sk(sk), skb); ret = 0; diff --git a/net/ipv4/tcp_offload.c b/net/ipv4/tcp_offload.c index fc61cd3fea65..30abde86db45 100644 --- a/net/ipv4/tcp_offload.c +++ b/net/ipv4/tcp_offload.c @@ -8,6 +8,7 @@ #include <linux/indirect_call_wrapper.h> #include <linux/skbuff.h> +#include <net/gro.h> #include <net/tcp.h> #include <net/protocol.h> diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c index 2e6e5a70168e..5079832af5c1 100644 --- a/net/ipv4/tcp_output.c +++ b/net/ipv4/tcp_output.c @@ -1359,7 +1359,7 @@ static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, #ifdef CONFIG_TCP_MD5SIG /* Calculate the MD5 hash, as we have all we need now */ if (md5) { - sk_nocaps_add(sk, NETIF_F_GSO_MASK); + sk_gso_disable(sk); tp->af_specific->calc_md5_hash(opts.hash_location, md5, sk, skb); } diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c index 23b05e28490b..69d30053fed9 100644 --- a/net/ipv4/udp.c +++ b/net/ipv4/udp.c @@ -122,7 +122,7 @@ EXPORT_SYMBOL(udp_table); long sysctl_udp_mem[3] __read_mostly; EXPORT_SYMBOL(sysctl_udp_mem); -atomic_long_t udp_memory_allocated; +atomic_long_t udp_memory_allocated ____cacheline_aligned_in_smp; EXPORT_SYMBOL(udp_memory_allocated); #define MAX_UDP_PORTS 65536 @@ -459,7 +459,7 @@ static struct sock *udp4_lookup_run_bpf(struct net *net, struct udp_table *udptable, struct sk_buff *skb, __be32 saddr, __be16 sport, - __be32 daddr, u16 hnum) + __be32 daddr, u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -467,8 +467,8 @@ static struct sock *udp4_lookup_run_bpf(struct net *net, if (udptable != &udp_table) return NULL; /* only UDP is supported */ - no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v4(net, IPPROTO_UDP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -504,7 +504,7 @@ struct sock *__udp4_lib_lookup(struct net *net, __be32 saddr, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { sk = udp4_lookup_run_bpf(net, udptable, skb, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (sk) { result = sk; goto done; diff --git a/net/ipv4/udp_offload.c b/net/ipv4/udp_offload.c index 86d32a1e62ac..6d1a4bec2614 100644 --- a/net/ipv4/udp_offload.c +++ b/net/ipv4/udp_offload.c @@ -7,6 +7,7 @@ */ #include <linux/skbuff.h> +#include <net/gro.h> #include <net/udp.h> #include <net/protocol.h> #include <net/inet_common.h> @@ -424,6 +425,33 @@ out: return segs; } +static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb) +{ + if (unlikely(p->len + skb->len >= 65536)) + return -E2BIG; + + if (NAPI_GRO_CB(p)->last == p) + skb_shinfo(p)->frag_list = skb; + else + NAPI_GRO_CB(p)->last->next = skb; + + skb_pull(skb, skb_gro_offset(skb)); + + NAPI_GRO_CB(p)->last = skb; + NAPI_GRO_CB(p)->count++; + p->data_len += skb->len; + + /* sk owenrship - if any - completely transferred to the aggregated packet */ + skb->destructor = NULL; + p->truesize += skb->truesize; + p->len += skb->len; + + NAPI_GRO_CB(skb)->same_flow = 1; + + return 0; +} + + #define UDP_GRO_CNT_MAX 64 static struct sk_buff *udp_gro_receive_segment(struct list_head *head, struct sk_buff *skb) @@ -600,13 +628,11 @@ struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb) inet_gro_compute_pseudo); skip: NAPI_GRO_CB(skb)->is_ipv6 = 0; - rcu_read_lock(); if (static_branch_unlikely(&udp_encap_needed_key)) sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest); pp = udp_gro_receive(head, skb, uh, sk); - rcu_read_unlock(); return pp; flush: @@ -641,7 +667,6 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, uh->len = newlen; - rcu_read_lock(); sk = INDIRECT_CALL_INET(lookup, udp6_lib_lookup_skb, udp4_lib_lookup_skb, skb, uh->source, uh->dest); if (sk && udp_sk(sk)->gro_complete) { @@ -662,7 +687,6 @@ int udp_gro_complete(struct sk_buff *skb, int nhoff, } else { err = udp_gro_complete_segment(skb); } - rcu_read_unlock(); if (skb->remcsum_offload) skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM; diff --git a/net/ipv4/xfrm4_policy.c b/net/ipv4/xfrm4_policy.c index 9ebd54752e03..9e83bcb6bc99 100644 --- a/net/ipv4/xfrm4_policy.c +++ b/net/ipv4/xfrm4_policy.c @@ -77,7 +77,7 @@ static int xfrm4_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, xdst->u.rt.rt_iif = fl4->flowi4_iif; xdst->u.dst.dev = dev; - dev_hold(dev); + dev_hold_track(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); /* Sheit... I remember I did this right. Apparently, * it was magically lost, so this code needs audit */ diff --git a/net/ipv6/addrconf.c b/net/ipv6/addrconf.c index 3445f8017430..3eee17790a82 100644 --- a/net/ipv6/addrconf.c +++ b/net/ipv6/addrconf.c @@ -405,13 +405,13 @@ static struct inet6_dev *ipv6_add_dev(struct net_device *dev) if (ndev->cnf.forwarding) dev_disable_lro(dev); /* We refer to the device */ - dev_hold(dev); + dev_hold_track(dev, &ndev->dev_tracker, GFP_KERNEL); if (snmp6_alloc_dev(ndev) < 0) { netdev_dbg(dev, "%s: cannot allocate memory for statistics\n", __func__); neigh_parms_release(&nd_tbl, ndev->nd_parms); - dev_put(dev); + dev_put_track(dev, &ndev->dev_tracker); kfree(ndev); return ERR_PTR(err); } diff --git a/net/ipv6/addrconf_core.c b/net/ipv6/addrconf_core.c index 1d4054bb345b..881d1477d24a 100644 --- a/net/ipv6/addrconf_core.c +++ b/net/ipv6/addrconf_core.c @@ -263,7 +263,7 @@ void in6_dev_finish_destroy(struct inet6_dev *idev) #ifdef NET_REFCNT_DEBUG pr_debug("%s: %s\n", __func__, dev ? dev->name : "NIL"); #endif - dev_put(dev); + dev_put_track(dev, &idev->dev_tracker); if (!idev->dead) { pr_warn("Freeing alive inet6 device %p\n", idev); return; diff --git a/net/ipv6/af_inet6.c b/net/ipv6/af_inet6.c index dab4a047590b..d1636425654e 100644 --- a/net/ipv6/af_inet6.c +++ b/net/ipv6/af_inet6.c @@ -337,11 +337,8 @@ static int __inet6_bind(struct sock *sk, struct sockaddr *uaddr, int addr_len, chk_addr_ret = inet_addr_type_dev_table(net, dev, v4addr); rcu_read_unlock(); - if (!inet_can_nonlocal_bind(net, inet) && - v4addr != htonl(INADDR_ANY) && - chk_addr_ret != RTN_LOCAL && - chk_addr_ret != RTN_MULTICAST && - chk_addr_ret != RTN_BROADCAST) { + if (!inet_addr_valid_or_nonlocal(net, inet, v4addr, + chk_addr_ret)) { err = -EADDRNOTAVAIL; goto out; } diff --git a/net/ipv6/ah6.c b/net/ipv6/ah6.c index 828e62514260..b5995c1f4d7a 100644 --- a/net/ipv6/ah6.c +++ b/net/ipv6/ah6.c @@ -175,7 +175,6 @@ static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *des * See 11.3.2 of RFC 3775 for details. */ if (opt[off] == IPV6_TLV_HAO) { - struct in6_addr final_addr; struct ipv6_destopt_hao *hao; hao = (struct ipv6_destopt_hao *)&opt[off]; @@ -184,9 +183,7 @@ static void ipv6_rearrange_destopt(struct ipv6hdr *iph, struct ipv6_opt_hdr *des hao->length); goto bad; } - final_addr = hao->addr; - hao->addr = iph->saddr; - iph->saddr = final_addr; + swap(hao->addr, iph->saddr); } break; } diff --git a/net/ipv6/esp6_offload.c b/net/ipv6/esp6_offload.c index a349d4798077..ba5e81cd569c 100644 --- a/net/ipv6/esp6_offload.c +++ b/net/ipv6/esp6_offload.c @@ -16,6 +16,7 @@ #include <crypto/authenc.h> #include <linux/err.h> #include <linux/module.h> +#include <net/gro.h> #include <net/ip.h> #include <net/xfrm.h> #include <net/esp.h> diff --git a/net/ipv6/exthdrs.c b/net/ipv6/exthdrs.c index 38ece3b7b839..77e34aec7e82 100644 --- a/net/ipv6/exthdrs.c +++ b/net/ipv6/exthdrs.c @@ -686,7 +686,6 @@ static int ipv6_rthdr_rcv(struct sk_buff *skb) struct net *net = dev_net(skb->dev); int accept_source_route = net->ipv6.devconf_all->accept_source_route; - idev = __in6_dev_get(skb->dev); if (idev && accept_source_route > idev->cnf.accept_source_route) accept_source_route = idev->cnf.accept_source_route; diff --git a/net/ipv6/inet6_hashtables.c b/net/ipv6/inet6_hashtables.c index 67c9114835c8..4514444e96c8 100644 --- a/net/ipv6/inet6_hashtables.c +++ b/net/ipv6/inet6_hashtables.c @@ -165,7 +165,7 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net, const struct in6_addr *saddr, const __be16 sport, const struct in6_addr *daddr, - const u16 hnum) + const u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -173,8 +173,8 @@ static inline struct sock *inet6_lookup_run_bpf(struct net *net, if (hashinfo != &tcp_hashinfo) return NULL; /* only TCP is supported */ - no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -198,7 +198,7 @@ struct sock *inet6_lookup_listener(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { result = inet6_lookup_run_bpf(net, hashinfo, skb, doff, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (result) goto done; } diff --git a/net/ipv6/ip6_gre.c b/net/ipv6/ip6_gre.c index d831d2439693..110839a88bc2 100644 --- a/net/ipv6/ip6_gre.c +++ b/net/ipv6/ip6_gre.c @@ -403,7 +403,7 @@ static void ip6erspan_tunnel_uninit(struct net_device *dev) ip6erspan_tunnel_unlink_md(ign, t); ip6gre_tunnel_unlink(ign, t); dst_cache_reset(&t->dst_cache); - dev_put(dev); + dev_put_track(dev, &t->dev_tracker); } static void ip6gre_tunnel_uninit(struct net_device *dev) @@ -416,7 +416,7 @@ static void ip6gre_tunnel_uninit(struct net_device *dev) if (ign->fb_tunnel_dev == dev) WRITE_ONCE(ign->fb_tunnel_dev, NULL); dst_cache_reset(&t->dst_cache); - dev_put(dev); + dev_put_track(dev, &t->dev_tracker); } @@ -1496,7 +1496,7 @@ static int ip6gre_tunnel_init_common(struct net_device *dev) } ip6gre_tnl_init_features(dev); - dev_hold(dev); + dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; cleanup_dst_cache_init: @@ -1888,7 +1888,7 @@ static int ip6erspan_tap_init(struct net_device *dev) dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; ip6erspan_tnl_link_config(tunnel, 1); - dev_hold(dev); + dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; cleanup_dst_cache_init: diff --git a/net/ipv6/ip6_offload.c b/net/ipv6/ip6_offload.c index 1cbd49d5788d..b29e9ba5e113 100644 --- a/net/ipv6/ip6_offload.c +++ b/net/ipv6/ip6_offload.c @@ -208,7 +208,6 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, flush += ntohs(iph->payload_len) != skb_gro_len(skb); - rcu_read_lock(); proto = iph->nexthdr; ops = rcu_dereference(inet6_offloads[proto]); if (!ops || !ops->callbacks.gro_receive) { @@ -221,7 +220,7 @@ INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head, ops = rcu_dereference(inet6_offloads[proto]); if (!ops || !ops->callbacks.gro_receive) - goto out_unlock; + goto out; iph = ipv6_hdr(skb); } @@ -279,9 +278,6 @@ not_same_flow: pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive, ops->callbacks.gro_receive, head, skb); -out_unlock: - rcu_read_unlock(); - out: skb_gro_flush_final(skb, pp, flush); @@ -331,18 +327,14 @@ INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff) iph->payload_len = htons(skb->len - nhoff - sizeof(*iph)); - rcu_read_lock(); - nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops); if (WARN_ON(!ops || !ops->callbacks.gro_complete)) - goto out_unlock; + goto out; err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete, udp6_gro_complete, skb, nhoff); -out_unlock: - rcu_read_unlock(); - +out: return err; } diff --git a/net/ipv6/ip6_output.c b/net/ipv6/ip6_output.c index ff4e83e2a506..2995f8d89e7e 100644 --- a/net/ipv6/ip6_output.c +++ b/net/ipv6/ip6_output.c @@ -977,7 +977,7 @@ slow_path: fail_toobig: if (skb->sk && dst_allfrag(skb_dst(skb))) - sk_nocaps_add(skb->sk, NETIF_F_GSO_MASK); + sk_gso_disable(skb->sk); icmpv6_send(skb, ICMPV6_PKT_TOOBIG, 0, mtu); err = -EMSGSIZE; diff --git a/net/ipv6/ip6_tunnel.c b/net/ipv6/ip6_tunnel.c index 484aca492cc0..fe786df4f849 100644 --- a/net/ipv6/ip6_tunnel.c +++ b/net/ipv6/ip6_tunnel.c @@ -383,7 +383,7 @@ ip6_tnl_dev_uninit(struct net_device *dev) else ip6_tnl_unlink(ip6n, t); dst_cache_reset(&t->dst_cache); - dev_put(dev); + dev_put_track(dev, &t->dev_tracker); } /** @@ -1883,7 +1883,7 @@ ip6_tnl_dev_init_gen(struct net_device *dev) dev->min_mtu = ETH_MIN_MTU; dev->max_mtu = IP6_MAX_MTU - dev->hard_header_len; - dev_hold(dev); + dev_hold_track(dev, &t->dev_tracker, GFP_KERNEL); return 0; destroy_dst: diff --git a/net/ipv6/ip6_vti.c b/net/ipv6/ip6_vti.c index 527e9ead7449..ed9b6d6ca65e 100644 --- a/net/ipv6/ip6_vti.c +++ b/net/ipv6/ip6_vti.c @@ -293,7 +293,7 @@ static void vti6_dev_uninit(struct net_device *dev) RCU_INIT_POINTER(ip6n->tnls_wc[0], NULL); else vti6_tnl_unlink(ip6n, t); - dev_put(dev); + dev_put_track(dev, &t->dev_tracker); } static int vti6_input_proto(struct sk_buff *skb, int nexthdr, __be32 spi, @@ -934,7 +934,7 @@ static inline int vti6_dev_init_gen(struct net_device *dev) dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); if (!dev->tstats) return -ENOMEM; - dev_hold(dev); + dev_hold_track(dev, &t->dev_tracker, GFP_KERNEL); return 0; } diff --git a/net/ipv6/ip6mr.c b/net/ipv6/ip6mr.c index 36ed9efb8825..a77a15a7f3dc 100644 --- a/net/ipv6/ip6mr.c +++ b/net/ipv6/ip6mr.c @@ -746,7 +746,7 @@ static int mif6_delete(struct mr_table *mrt, int vifi, int notify, if ((v->flags & MIFF_REGISTER) && !notify) unregister_netdevice_queue(dev, head); - dev_put(dev); + dev_put_track(dev, &v->dev_tracker); return 0; } @@ -919,6 +919,7 @@ static int mif6_add(struct net *net, struct mr_table *mrt, /* And finish update writing critical data */ write_lock_bh(&mrt_lock); v->dev = dev; + netdev_tracker_alloc(dev, &v->dev_tracker, GFP_ATOMIC); #ifdef CONFIG_IPV6_PIMSM_V2 if (v->flags & MIFF_REGISTER) mrt->mroute_reg_vif_num = vifi; diff --git a/net/ipv6/ipv6_sockglue.c b/net/ipv6/ipv6_sockglue.c index 41efca817db4..a733803a710c 100644 --- a/net/ipv6/ipv6_sockglue.c +++ b/net/ipv6/ipv6_sockglue.c @@ -471,10 +471,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (sk->sk_protocol == IPPROTO_TCP) { struct inet_connection_sock *icsk = inet_csk(sk); - local_bh_disable(); + sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, &tcp_prot, 1); - local_bh_enable(); + sk->sk_prot = &tcp_prot; icsk->icsk_af_ops = &ipv4_specific; sk->sk_socket->ops = &inet_stream_ops; @@ -485,10 +485,10 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, if (sk->sk_protocol == IPPROTO_UDPLITE) prot = &udplite_prot; - local_bh_disable(); + sock_prot_inuse_add(net, sk->sk_prot, -1); sock_prot_inuse_add(net, prot, 1); - local_bh_enable(); + sk->sk_prot = prot; sk->sk_socket->ops = &inet_dgram_ops; sk->sk_family = PF_INET; @@ -599,7 +599,14 @@ static int do_ipv6_setsockopt(struct sock *sk, int level, int optname, /* RFC 3542, 6.5: default traffic class of 0x0 */ if (val == -1) val = 0; - np->tclass = val; + if (sk->sk_type == SOCK_STREAM) { + val &= ~INET_ECN_MASK; + val |= np->tclass & INET_ECN_MASK; + } + if (np->tclass != val) { + np->tclass = val; + sk_dst_reset(sk); + } retv = 0; break; diff --git a/net/ipv6/route.c b/net/ipv6/route.c index 42d60c76d30a..4d02a329ab60 100644 --- a/net/ipv6/route.c +++ b/net/ipv6/route.c @@ -182,8 +182,9 @@ static void rt6_uncached_list_flush_dev(struct net *net, struct net_device *dev) if (rt_dev == dev) { rt->dst.dev = blackhole_netdev; - dev_hold(rt->dst.dev); - dev_put(rt_dev); + dev_replace_track(rt_dev, blackhole_netdev, + &rt->dst.dev_tracker, + GFP_ATOMIC); } } spin_unlock_bh(&ul->lock); @@ -328,9 +329,7 @@ static const struct rt6_info ip6_blk_hole_entry_template = { static void rt6_info_init(struct rt6_info *rt) { - struct dst_entry *dst = &rt->dst; - - memset(dst + 1, 0, sizeof(*rt) - sizeof(*dst)); + memset_after(rt, 0, dst); INIT_LIST_HEAD(&rt->rt6i_uncached); } @@ -594,6 +593,7 @@ struct __rt6_probe_work { struct work_struct work; struct in6_addr target; struct net_device *dev; + netdevice_tracker dev_tracker; }; static void rt6_probe_deferred(struct work_struct *w) @@ -604,7 +604,7 @@ static void rt6_probe_deferred(struct work_struct *w) addrconf_addr_solict_mult(&work->target, &mcaddr); ndisc_send_ns(work->dev, &work->target, &mcaddr, NULL, 0); - dev_put(work->dev); + dev_put_track(work->dev, &work->dev_tracker); kfree(work); } @@ -658,7 +658,7 @@ static void rt6_probe(struct fib6_nh *fib6_nh) } else { INIT_WORK(&work->work, rt6_probe_deferred); work->target = *nh_gw; - dev_hold(dev); + dev_hold_track(dev, &work->dev_tracker, GFP_KERNEL); work->dev = dev; schedule_work(&work->work); } @@ -1485,7 +1485,7 @@ static void rt6_exception_remove_oldest(struct rt6_exception_bucket *bucket) static u32 rt6_exception_hash(const struct in6_addr *dst, const struct in6_addr *src) { - static siphash_key_t rt6_exception_key __read_mostly; + static siphash_aligned_key_t rt6_exception_key; struct { struct in6_addr dst; struct in6_addr src; @@ -3628,6 +3628,8 @@ pcpu_alloc: } fib6_nh->fib_nh_dev = dev; + netdev_tracker_alloc(dev, &fib6_nh->fib_nh_dev_tracker, gfp_flags); + fib6_nh->fib_nh_oif = dev->ifindex; err = 0; out: @@ -3658,24 +3660,8 @@ void fib6_nh_release(struct fib6_nh *fib6_nh) rcu_read_unlock(); - if (fib6_nh->rt6i_pcpu) { - int cpu; - - for_each_possible_cpu(cpu) { - struct rt6_info **ppcpu_rt; - struct rt6_info *pcpu_rt; - - ppcpu_rt = per_cpu_ptr(fib6_nh->rt6i_pcpu, cpu); - pcpu_rt = *ppcpu_rt; - if (pcpu_rt) { - dst_dev_put(&pcpu_rt->dst); - dst_release(&pcpu_rt->dst); - *ppcpu_rt = NULL; - } - } - - free_percpu(fib6_nh->rt6i_pcpu); - } + fib6_nh_release_dsts(fib6_nh); + free_percpu(fib6_nh->rt6i_pcpu); fib_nh_common_release(&fib6_nh->nh_common); } diff --git a/net/ipv6/sit.c b/net/ipv6/sit.c index 1b57ee36d668..057c0f83c800 100644 --- a/net/ipv6/sit.c +++ b/net/ipv6/sit.c @@ -521,7 +521,7 @@ static void ipip6_tunnel_uninit(struct net_device *dev) ipip6_tunnel_del_prl(tunnel, NULL); } dst_cache_reset(&tunnel->dst_cache); - dev_put(dev); + dev_put_track(dev, &tunnel->dev_tracker); } static int ipip6_err(struct sk_buff *skb, u32 info) @@ -1463,7 +1463,7 @@ static int ipip6_tunnel_init(struct net_device *dev) dev->tstats = NULL; return err; } - dev_hold(dev); + dev_hold_track(dev, &tunnel->dev_tracker, GFP_KERNEL); return 0; } diff --git a/net/ipv6/syncookies.c b/net/ipv6/syncookies.c index e8cfb9e997bf..d1b61d00368e 100644 --- a/net/ipv6/syncookies.c +++ b/net/ipv6/syncookies.c @@ -20,7 +20,7 @@ #define COOKIEBITS 24 /* Upper bits store count */ #define COOKIEMASK (((__u32)1 << COOKIEBITS) - 1) -static siphash_key_t syncookie6_secret[2] __read_mostly; +static siphash_aligned_key_t syncookie6_secret[2]; /* RFC 2460, Section 8.3: * [ipv6 tcp] MSS must be computed as the maximum packet size minus 60 [..] diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 551fce49841d..3b7d6ede1364 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -72,7 +72,7 @@ static void tcp_v6_send_reset(const struct sock *sk, struct sk_buff *skb); static void tcp_v6_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb, struct request_sock *req); -static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); +INDIRECT_CALLABLE_SCOPE int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb); static const struct inet_connection_sock_af_ops ipv6_mapped; const struct inet_connection_sock_af_ops ipv6_specific; @@ -1466,7 +1466,8 @@ INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *, * This is because we cannot sleep with the original spinlock * held. */ -static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) +INDIRECT_CALLABLE_SCOPE +int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) { struct ipv6_pinfo *np = tcp_inet6_sk(sk); struct sk_buff *opt_skb = NULL; @@ -1757,6 +1758,7 @@ process: sk_incoming_cpu_update(sk); + sk_defer_free_flush(sk); bh_lock_sock_nested(sk); tcp_segs_in(tcp_sk(sk), skb); ret = 0; @@ -1893,9 +1895,7 @@ static struct timewait_sock_ops tcp6_timewait_sock_ops = { INDIRECT_CALLABLE_SCOPE void tcp_v6_send_check(struct sock *sk, struct sk_buff *skb) { - struct ipv6_pinfo *np = inet6_sk(sk); - - __tcp_v6_send_check(skb, &np->saddr, &sk->sk_v6_daddr); + __tcp_v6_send_check(skb, &sk->sk_v6_rcv_saddr, &sk->sk_v6_daddr); } const struct inet_connection_sock_af_ops ipv6_specific = { diff --git a/net/ipv6/tcpv6_offload.c b/net/ipv6/tcpv6_offload.c index 1796856bc24f..39db5a226855 100644 --- a/net/ipv6/tcpv6_offload.c +++ b/net/ipv6/tcpv6_offload.c @@ -7,6 +7,7 @@ */ #include <linux/indirect_call_wrapper.h> #include <linux/skbuff.h> +#include <net/gro.h> #include <net/protocol.h> #include <net/tcp.h> #include <net/ip6_checksum.h> diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index e43b31d25fb6..6a0e569f0bb8 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -195,7 +195,7 @@ static inline struct sock *udp6_lookup_run_bpf(struct net *net, const struct in6_addr *saddr, __be16 sport, const struct in6_addr *daddr, - u16 hnum) + u16 hnum, const int dif) { struct sock *sk, *reuse_sk; bool no_reuseport; @@ -203,8 +203,8 @@ static inline struct sock *udp6_lookup_run_bpf(struct net *net, if (udptable != &udp_table) return NULL; /* only UDP is supported */ - no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP, - saddr, sport, daddr, hnum, &sk); + no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_UDP, saddr, sport, + daddr, hnum, dif, &sk); if (no_reuseport || IS_ERR_OR_NULL(sk)) return sk; @@ -240,7 +240,7 @@ struct sock *__udp6_lib_lookup(struct net *net, /* Lookup redirect from BPF */ if (static_branch_unlikely(&bpf_sk_lookup_enabled)) { sk = udp6_lookup_run_bpf(net, udptable, skb, - saddr, sport, daddr, hnum); + saddr, sport, daddr, hnum, dif); if (sk) { result = sk; goto done; diff --git a/net/ipv6/udp_offload.c b/net/ipv6/udp_offload.c index b3d9ed96e5ea..7720d04ed396 100644 --- a/net/ipv6/udp_offload.c +++ b/net/ipv6/udp_offload.c @@ -13,6 +13,7 @@ #include <net/udp.h> #include <net/ip6_checksum.h> #include "ip6_offload.h" +#include <net/gro.h> static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb, netdev_features_t features) @@ -144,13 +145,11 @@ struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb) skip: NAPI_GRO_CB(skb)->is_ipv6 = 1; - rcu_read_lock(); if (static_branch_unlikely(&udpv6_encap_needed_key)) sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest); pp = udp_gro_receive(head, skb, uh, sk); - rcu_read_unlock(); return pp; flush: diff --git a/net/ipv6/xfrm6_policy.c b/net/ipv6/xfrm6_policy.c index af7a4b8b1e9c..fad687ee6dd8 100644 --- a/net/ipv6/xfrm6_policy.c +++ b/net/ipv6/xfrm6_policy.c @@ -74,11 +74,11 @@ static int xfrm6_fill_dst(struct xfrm_dst *xdst, struct net_device *dev, struct rt6_info *rt = (struct rt6_info *)xdst->route; xdst->u.dst.dev = dev; - dev_hold(dev); + dev_hold_track(dev, &xdst->u.dst.dev_tracker, GFP_ATOMIC); xdst->u.rt6.rt6i_idev = in6_dev_get(dev); if (!xdst->u.rt6.rt6i_idev) { - dev_put(dev); + dev_put_track(dev, &xdst->u.dst.dev_tracker); return -ENODEV; } diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c index 18316ee3c692..49ecbe8d176a 100644 --- a/net/iucv/af_iucv.c +++ b/net/iucv/af_iucv.c @@ -142,7 +142,7 @@ static inline size_t iucv_msg_length(struct iucv_message *msg) * iucv_sock_in_state() - check for specific states * @sk: sock structure * @state: first iucv sk state - * @state: second iucv sk state + * @state2: second iucv sk state * * Returns true if the socket in either in the first or second state. */ @@ -172,7 +172,7 @@ static inline int iucv_below_msglim(struct sock *sk) (atomic_read(&iucv->pendings) <= 0)); } -/** +/* * iucv_sock_wake_msglim() - Wake up thread waiting on msg limit */ static void iucv_sock_wake_msglim(struct sock *sk) @@ -187,7 +187,7 @@ static void iucv_sock_wake_msglim(struct sock *sk) rcu_read_unlock(); } -/** +/* * afiucv_hs_send() - send a message through HiperSockets transport */ static int afiucv_hs_send(struct iucv_message *imsg, struct sock *sock, @@ -473,7 +473,7 @@ static struct sock *iucv_sock_alloc(struct socket *sock, int proto, gfp_t prio, atomic_set(&iucv->msg_recv, 0); iucv->path = NULL; iucv->sk_txnotify = afiucv_hs_callback_txnotify; - memset(&iucv->src_user_id , 0, 32); + memset(&iucv->init, 0, sizeof(iucv->init)); if (pr_iucv) iucv->transport = AF_IUCV_TRANS_IUCV; else @@ -1831,9 +1831,9 @@ static void afiucv_swap_src_dest(struct sk_buff *skb) memset(skb->data, 0, ETH_HLEN); } -/** +/* * afiucv_hs_callback_syn - react on received SYN - **/ + */ static int afiucv_hs_callback_syn(struct sock *sk, struct sk_buff *skb) { struct af_iucv_trans_hdr *trans_hdr = iucv_trans_hdr(skb); @@ -1896,9 +1896,9 @@ out: return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_callback_synack() - react on received SYN-ACK - **/ + */ static int afiucv_hs_callback_synack(struct sock *sk, struct sk_buff *skb) { struct iucv_sock *iucv = iucv_sk(sk); @@ -1917,9 +1917,9 @@ static int afiucv_hs_callback_synack(struct sock *sk, struct sk_buff *skb) return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_callback_synfin() - react on received SYN_FIN - **/ + */ static int afiucv_hs_callback_synfin(struct sock *sk, struct sk_buff *skb) { struct iucv_sock *iucv = iucv_sk(sk); @@ -1937,9 +1937,9 @@ static int afiucv_hs_callback_synfin(struct sock *sk, struct sk_buff *skb) return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_callback_fin() - react on received FIN - **/ + */ static int afiucv_hs_callback_fin(struct sock *sk, struct sk_buff *skb) { struct iucv_sock *iucv = iucv_sk(sk); @@ -1960,9 +1960,9 @@ static int afiucv_hs_callback_fin(struct sock *sk, struct sk_buff *skb) return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_callback_win() - react on received WIN - **/ + */ static int afiucv_hs_callback_win(struct sock *sk, struct sk_buff *skb) { struct iucv_sock *iucv = iucv_sk(sk); @@ -1978,9 +1978,9 @@ static int afiucv_hs_callback_win(struct sock *sk, struct sk_buff *skb) return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_callback_rx() - react on received data - **/ + */ static int afiucv_hs_callback_rx(struct sock *sk, struct sk_buff *skb) { struct iucv_sock *iucv = iucv_sk(sk); @@ -2022,11 +2022,11 @@ static int afiucv_hs_callback_rx(struct sock *sk, struct sk_buff *skb) return NET_RX_SUCCESS; } -/** +/* * afiucv_hs_rcv() - base function for arriving data through HiperSockets * transport * called from netif RX softirq - **/ + */ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, struct packet_type *pt, struct net_device *orig_dev) { @@ -2128,10 +2128,10 @@ static int afiucv_hs_rcv(struct sk_buff *skb, struct net_device *dev, return err; } -/** +/* * afiucv_hs_callback_txnotify() - handle send notifications from HiperSockets * transport - **/ + */ static void afiucv_hs_callback_txnotify(struct sock *sk, enum iucv_tx_notify n) { struct iucv_sock *iucv = iucv_sk(sk); diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index f3343a8541a5..8f4d49a7d3e8 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -276,8 +276,8 @@ static union iucv_param *iucv_param[NR_CPUS]; static union iucv_param *iucv_param_irq[NR_CPUS]; /** - * iucv_call_b2f0 - * @code: identifier of IUCV call to CP. + * __iucv_call_b2f0 + * @command: identifier of IUCV call to CP. * @parm: pointer to a struct iucv_parm block * * Calls CP to execute IUCV commands. @@ -309,7 +309,7 @@ static inline int iucv_call_b2f0(int command, union iucv_param *parm) return ccode == 1 ? parm->ctrl.iprcode : ccode; } -/** +/* * iucv_query_maxconn * * Determines the maximum number of connections that may be established. @@ -493,8 +493,8 @@ static void iucv_retrieve_cpu(void *data) cpumask_clear_cpu(cpu, &iucv_buffer_cpumask); } -/** - * iucv_setmask_smp +/* + * iucv_setmask_mp * * Allow iucv interrupts on all cpus. */ @@ -512,7 +512,7 @@ static void iucv_setmask_mp(void) cpus_read_unlock(); } -/** +/* * iucv_setmask_up * * Allow iucv interrupts on a single cpu. @@ -529,7 +529,7 @@ static void iucv_setmask_up(void) smp_call_function_single(cpu, iucv_block_cpu, NULL, 1); } -/** +/* * iucv_enable * * This function makes iucv ready for use. It allocates the pathid @@ -564,7 +564,7 @@ out: return rc; } -/** +/* * iucv_disable * * This function shuts down iucv. It disables iucv interrupts, retrieves @@ -1347,8 +1347,9 @@ EXPORT_SYMBOL(iucv_message_send); * @srccls: source class of message * @buffer: address of send buffer or address of struct iucv_array * @size: length of send buffer - * @ansbuf: address of answer buffer or address of struct iucv_array + * @answer: address of answer buffer or address of struct iucv_array * @asize: size of reply buffer + * @residual: ignored * * This function transmits data to another application. Data to be * transmitted is in a buffer. The receiver of the send is expected to @@ -1400,13 +1401,6 @@ out: } EXPORT_SYMBOL(iucv_message_send2way); -/** - * iucv_path_pending - * @data: Pointer to external interrupt buffer - * - * Process connection pending work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_path_pending { u16 ippathid; u8 ipflags1; @@ -1420,6 +1414,13 @@ struct iucv_path_pending { u8 res4[3]; } __packed; +/** + * iucv_path_pending + * @data: Pointer to external interrupt buffer + * + * Process connection pending work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_path_pending(struct iucv_irq_data *data) { struct iucv_path_pending *ipp = (void *) data; @@ -1461,13 +1462,6 @@ out_sever: iucv_sever_pathid(ipp->ippathid, error); } -/** - * iucv_path_complete - * @data: Pointer to external interrupt buffer - * - * Process connection complete work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_path_complete { u16 ippathid; u8 ipflags1; @@ -1481,6 +1475,13 @@ struct iucv_path_complete { u8 res4[3]; } __packed; +/** + * iucv_path_complete + * @data: Pointer to external interrupt buffer + * + * Process connection complete work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_path_complete(struct iucv_irq_data *data) { struct iucv_path_complete *ipc = (void *) data; @@ -1492,13 +1493,6 @@ static void iucv_path_complete(struct iucv_irq_data *data) path->handler->path_complete(path, ipc->ipuser); } -/** - * iucv_path_severed - * @data: Pointer to external interrupt buffer - * - * Process connection severed work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_path_severed { u16 ippathid; u8 res1; @@ -1511,6 +1505,13 @@ struct iucv_path_severed { u8 res5[3]; } __packed; +/** + * iucv_path_severed + * @data: Pointer to external interrupt buffer + * + * Process connection severed work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_path_severed(struct iucv_irq_data *data) { struct iucv_path_severed *ips = (void *) data; @@ -1528,13 +1529,6 @@ static void iucv_path_severed(struct iucv_irq_data *data) } } -/** - * iucv_path_quiesced - * @data: Pointer to external interrupt buffer - * - * Process connection quiesced work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_path_quiesced { u16 ippathid; u8 res1; @@ -1547,6 +1541,13 @@ struct iucv_path_quiesced { u8 res5[3]; } __packed; +/** + * iucv_path_quiesced + * @data: Pointer to external interrupt buffer + * + * Process connection quiesced work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_path_quiesced(struct iucv_irq_data *data) { struct iucv_path_quiesced *ipq = (void *) data; @@ -1556,13 +1557,6 @@ static void iucv_path_quiesced(struct iucv_irq_data *data) path->handler->path_quiesced(path, ipq->ipuser); } -/** - * iucv_path_resumed - * @data: Pointer to external interrupt buffer - * - * Process connection resumed work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_path_resumed { u16 ippathid; u8 res1; @@ -1575,6 +1569,13 @@ struct iucv_path_resumed { u8 res5[3]; } __packed; +/** + * iucv_path_resumed + * @data: Pointer to external interrupt buffer + * + * Process connection resumed work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_path_resumed(struct iucv_irq_data *data) { struct iucv_path_resumed *ipr = (void *) data; @@ -1584,13 +1585,6 @@ static void iucv_path_resumed(struct iucv_irq_data *data) path->handler->path_resumed(path, ipr->ipuser); } -/** - * iucv_message_complete - * @data: Pointer to external interrupt buffer - * - * Process message complete work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_message_complete { u16 ippathid; u8 ipflags1; @@ -1606,6 +1600,13 @@ struct iucv_message_complete { u8 res2[3]; } __packed; +/** + * iucv_message_complete + * @data: Pointer to external interrupt buffer + * + * Process message complete work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_message_complete(struct iucv_irq_data *data) { struct iucv_message_complete *imc = (void *) data; @@ -1624,13 +1625,6 @@ static void iucv_message_complete(struct iucv_irq_data *data) } } -/** - * iucv_message_pending - * @data: Pointer to external interrupt buffer - * - * Process message pending work item. Called from tasklet while holding - * iucv_table_lock. - */ struct iucv_message_pending { u16 ippathid; u8 ipflags1; @@ -1653,6 +1647,13 @@ struct iucv_message_pending { u8 res2[3]; } __packed; +/** + * iucv_message_pending + * @data: Pointer to external interrupt buffer + * + * Process message pending work item. Called from tasklet while holding + * iucv_table_lock. + */ static void iucv_message_pending(struct iucv_irq_data *data) { struct iucv_message_pending *imp = (void *) data; @@ -1673,7 +1674,7 @@ static void iucv_message_pending(struct iucv_irq_data *data) } } -/** +/* * iucv_tasklet_fn: * * This tasklet loops over the queue of irq buffers created by @@ -1717,7 +1718,7 @@ static void iucv_tasklet_fn(unsigned long ignored) spin_unlock(&iucv_table_lock); } -/** +/* * iucv_work_fn: * * This work function loops over the queue of path pending irq blocks @@ -1748,9 +1749,8 @@ static void iucv_work_fn(struct work_struct *work) spin_unlock_bh(&iucv_table_lock); } -/** +/* * iucv_external_interrupt - * @code: irq code * * Handles external interrupts coming in from CP. * Places the interrupt buffer on a queue and schedules iucv_tasklet_fn(). diff --git a/net/l2tp/l2tp_core.c b/net/l2tp/l2tp_core.c index 93271a2632b8..7499c51b1850 100644 --- a/net/l2tp/l2tp_core.c +++ b/net/l2tp/l2tp_core.c @@ -250,15 +250,15 @@ struct l2tp_session *l2tp_tunnel_get_session(struct l2tp_tunnel *tunnel, session_list = l2tp_session_id_hash(tunnel, session_id); - read_lock_bh(&tunnel->hlist_lock); - hlist_for_each_entry(session, session_list, hlist) + rcu_read_lock_bh(); + hlist_for_each_entry_rcu(session, session_list, hlist) if (session->session_id == session_id) { l2tp_session_inc_refcount(session); - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return session; } - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return NULL; } @@ -291,18 +291,18 @@ struct l2tp_session *l2tp_session_get_nth(struct l2tp_tunnel *tunnel, int nth) struct l2tp_session *session; int count = 0; - read_lock_bh(&tunnel->hlist_lock); + rcu_read_lock_bh(); for (hash = 0; hash < L2TP_HASH_SIZE; hash++) { - hlist_for_each_entry(session, &tunnel->session_hlist[hash], hlist) { + hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) { if (++count > nth) { l2tp_session_inc_refcount(session); - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return session; } } } - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); return NULL; } @@ -347,7 +347,7 @@ int l2tp_session_register(struct l2tp_session *session, head = l2tp_session_id_hash(tunnel, session->session_id); - write_lock_bh(&tunnel->hlist_lock); + spin_lock_bh(&tunnel->hlist_lock); if (!tunnel->acpt_newsess) { err = -ENODEV; goto err_tlock; @@ -384,8 +384,8 @@ int l2tp_session_register(struct l2tp_session *session, l2tp_tunnel_inc_refcount(tunnel); } - hlist_add_head(&session->hlist, head); - write_unlock_bh(&tunnel->hlist_lock); + hlist_add_head_rcu(&session->hlist, head); + spin_unlock_bh(&tunnel->hlist_lock); trace_register_session(session); @@ -394,7 +394,7 @@ int l2tp_session_register(struct l2tp_session *session, err_tlock_pnlock: spin_unlock_bh(&pn->l2tp_session_hlist_lock); err_tlock: - write_unlock_bh(&tunnel->hlist_lock); + spin_unlock_bh(&tunnel->hlist_lock); return err; } @@ -1170,9 +1170,9 @@ static void l2tp_session_unhash(struct l2tp_session *session) /* Remove the session from core hashes */ if (tunnel) { /* Remove from the per-tunnel hash */ - write_lock_bh(&tunnel->hlist_lock); - hlist_del_init(&session->hlist); - write_unlock_bh(&tunnel->hlist_lock); + spin_lock_bh(&tunnel->hlist_lock); + hlist_del_init_rcu(&session->hlist); + spin_unlock_bh(&tunnel->hlist_lock); /* For L2TPv3 we have a per-net hash: remove from there, too */ if (tunnel->version != L2TP_HDR_VER_2) { @@ -1181,8 +1181,9 @@ static void l2tp_session_unhash(struct l2tp_session *session) spin_lock_bh(&pn->l2tp_session_hlist_lock); hlist_del_init_rcu(&session->global_hlist); spin_unlock_bh(&pn->l2tp_session_hlist_lock); - synchronize_rcu(); } + + synchronize_rcu(); } } @@ -1190,22 +1191,19 @@ static void l2tp_session_unhash(struct l2tp_session *session) */ static void l2tp_tunnel_closeall(struct l2tp_tunnel *tunnel) { - int hash; - struct hlist_node *walk; - struct hlist_node *tmp; struct l2tp_session *session; + int hash; - write_lock_bh(&tunnel->hlist_lock); + spin_lock_bh(&tunnel->hlist_lock); tunnel->acpt_newsess = false; for (hash = 0; hash < L2TP_HASH_SIZE; hash++) { again: - hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) { - session = hlist_entry(walk, struct l2tp_session, hlist); - hlist_del_init(&session->hlist); + hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) { + hlist_del_init_rcu(&session->hlist); - write_unlock_bh(&tunnel->hlist_lock); + spin_unlock_bh(&tunnel->hlist_lock); l2tp_session_delete(session); - write_lock_bh(&tunnel->hlist_lock); + spin_lock_bh(&tunnel->hlist_lock); /* Now restart from the beginning of this hash * chain. We always remove a session from the @@ -1215,7 +1213,7 @@ again: goto again; } } - write_unlock_bh(&tunnel->hlist_lock); + spin_unlock_bh(&tunnel->hlist_lock); } /* Tunnel socket destroy hook for UDP encapsulation */ @@ -1408,7 +1406,7 @@ int l2tp_tunnel_create(int fd, int version, u32 tunnel_id, u32 peer_tunnel_id, tunnel->magic = L2TP_TUNNEL_MAGIC; sprintf(&tunnel->name[0], "tunl %u", tunnel_id); - rwlock_init(&tunnel->hlist_lock); + spin_lock_init(&tunnel->hlist_lock); tunnel->acpt_newsess = true; tunnel->encap = encap; diff --git a/net/l2tp/l2tp_core.h b/net/l2tp/l2tp_core.h index 98ea98eb9567..a88e070b431d 100644 --- a/net/l2tp/l2tp_core.h +++ b/net/l2tp/l2tp_core.h @@ -160,7 +160,7 @@ struct l2tp_tunnel { unsigned long dead; struct rcu_head rcu; - rwlock_t hlist_lock; /* protect session_hlist */ + spinlock_t hlist_lock; /* write-protection for session_hlist */ bool acpt_newsess; /* indicates whether this tunnel accepts * new sessions. Protected by hlist_lock. */ diff --git a/net/l2tp/l2tp_debugfs.c b/net/l2tp/l2tp_debugfs.c index bca75bef8282..acf6e1343b88 100644 --- a/net/l2tp/l2tp_debugfs.c +++ b/net/l2tp/l2tp_debugfs.c @@ -120,24 +120,21 @@ static void l2tp_dfs_seq_stop(struct seq_file *p, void *v) static void l2tp_dfs_seq_tunnel_show(struct seq_file *m, void *v) { struct l2tp_tunnel *tunnel = v; + struct l2tp_session *session; int session_count = 0; int hash; - struct hlist_node *walk; - struct hlist_node *tmp; - read_lock_bh(&tunnel->hlist_lock); + rcu_read_lock_bh(); for (hash = 0; hash < L2TP_HASH_SIZE; hash++) { - hlist_for_each_safe(walk, tmp, &tunnel->session_hlist[hash]) { - struct l2tp_session *session; - - session = hlist_entry(walk, struct l2tp_session, hlist); + hlist_for_each_entry_rcu(session, &tunnel->session_hlist[hash], hlist) { + /* Session ID of zero is a dummy/reserved value used by pppol2tp */ if (session->session_id == 0) continue; session_count++; } } - read_unlock_bh(&tunnel->hlist_lock); + rcu_read_unlock_bh(); seq_printf(m, "\nTUNNEL %u peer %u", tunnel->tunnel_id, tunnel->peer_tunnel_id); if (tunnel->sock) { diff --git a/net/llc/af_llc.c b/net/llc/af_llc.c index 3086f4a6ae68..26c00ebf4fba 100644 --- a/net/llc/af_llc.c +++ b/net/llc/af_llc.c @@ -224,7 +224,7 @@ static int llc_ui_release(struct socket *sock) } else { release_sock(sk); } - dev_put(llc->dev); + dev_put_track(llc->dev, &llc->dev_tracker); sock_put(sk); llc_sk_free(sk); out: @@ -295,6 +295,7 @@ static int llc_ui_autobind(struct socket *sock, struct sockaddr_llc *addr) llc->dev = dev_getfirstbyhwtype(&init_net, addr->sllc_arphrd); if (!llc->dev) goto out; + netdev_tracker_alloc(llc->dev, &llc->dev_tracker, GFP_KERNEL); rc = -EUSERS; llc->laddr.lsap = llc_ui_autoport(); if (!llc->laddr.lsap) @@ -362,7 +363,7 @@ static int llc_ui_bind(struct socket *sock, struct sockaddr *uaddr, int addrlen) } else llc->dev = dev_getbyhwaddr_rcu(&init_net, addr->sllc_arphrd, addr->sllc_mac); - dev_hold(llc->dev); + dev_hold_track(llc->dev, &llc->dev_tracker, GFP_ATOMIC); rcu_read_unlock(); if (!llc->dev) goto out; diff --git a/net/mac80211/ethtool.c b/net/mac80211/ethtool.c index 99a2e30b3833..b2253df54413 100644 --- a/net/mac80211/ethtool.c +++ b/net/mac80211/ethtool.c @@ -14,7 +14,9 @@ #include "driver-ops.h" static int ieee80211_set_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct ieee80211_local *local = wiphy_priv(dev->ieee80211_ptr->wiphy); @@ -25,7 +27,9 @@ static int ieee80211_set_ringparam(struct net_device *dev, } static void ieee80211_get_ringparam(struct net_device *dev, - struct ethtool_ringparam *rp) + struct ethtool_ringparam *rp, + struct kernel_ethtool_ringparam *kernel_rp, + struct netlink_ext_ack *extack) { struct ieee80211_local *local = wiphy_priv(dev->ieee80211_ptr->wiphy); diff --git a/net/mctp/af_mctp.c b/net/mctp/af_mctp.c index 871cf6266125..c921de63b494 100644 --- a/net/mctp/af_mctp.c +++ b/net/mctp/af_mctp.c @@ -405,8 +405,7 @@ static void mctp_sk_unhash(struct sock *sk) trace_mctp_key_release(key, MCTP_TRACE_KEY_CLOSED); spin_lock(&key->lock); - if (key->reasm_head) - kfree_skb(key->reasm_head); + kfree_skb(key->reasm_head); key->reasm_head = NULL; key->reasm_dead = true; key->valid = false; diff --git a/net/mctp/route.c b/net/mctp/route.c index cdf09c2a7007..8d9f4ff3e285 100644 --- a/net/mctp/route.c +++ b/net/mctp/route.c @@ -231,9 +231,7 @@ static void __mctp_key_unlock_drop(struct mctp_sk_key *key, struct net *net, /* and one for the local reference */ mctp_key_unref(key); - if (skb) - kfree_skb(skb); - + kfree_skb(skb); } #ifdef CONFIG_MCTP_FLOWS @@ -892,8 +890,7 @@ out_release: if (!ext_rt) mctp_route_release(rt); - if (dev) - dev_put(dev); + dev_put(dev); return rc; diff --git a/net/mctp/test/route-test.c b/net/mctp/test/route-test.c index 36fac3daf86a..86ad15abf897 100644 --- a/net/mctp/test/route-test.c +++ b/net/mctp/test/route-test.c @@ -150,11 +150,6 @@ static void mctp_test_fragment(struct kunit *test) rt = mctp_test_create_route(&init_net, NULL, 10, mtu); KUNIT_ASSERT_TRUE(test, rt); - /* The refcount would usually be incremented as part of a route lookup, - * but we're setting the route directly here. - */ - refcount_inc(&rt->rt.refs); - rc = mctp_do_fragment_route(&rt->rt, skb, mtu, MCTP_TAG_OWNER); KUNIT_EXPECT_FALSE(test, rc); diff --git a/net/mpls/af_mpls.c b/net/mpls/af_mpls.c index 0c7bde1c14a6..48f75a56f4ae 100644 --- a/net/mpls/af_mpls.c +++ b/net/mpls/af_mpls.c @@ -230,8 +230,8 @@ static struct mpls_nh *mpls_get_nexthop(struct mpls_route *rt, u8 index) * Since those fields can change at any moment, use READ_ONCE to * access both. */ -static struct mpls_nh *mpls_select_multipath(struct mpls_route *rt, - struct sk_buff *skb) +static const struct mpls_nh *mpls_select_multipath(struct mpls_route *rt, + struct sk_buff *skb) { u32 hash = 0; int nh_index = 0; @@ -343,8 +343,8 @@ static int mpls_forward(struct sk_buff *skb, struct net_device *dev, { struct net *net = dev_net(dev); struct mpls_shim_hdr *hdr; + const struct mpls_nh *nh; struct mpls_route *rt; - struct mpls_nh *nh; struct mpls_entry_decoded dec; struct net_device *out_dev; struct mpls_dev *out_mdev; @@ -2360,12 +2360,12 @@ static int mpls_getroute(struct sk_buff *in_skb, struct nlmsghdr *in_nlh, u32 labels[MAX_NEW_LABELS]; struct mpls_shim_hdr *hdr; unsigned int hdr_size = 0; + const struct mpls_nh *nh; struct net_device *dev; struct mpls_route *rt; struct rtmsg *rtm, *r; struct nlmsghdr *nlh; struct sk_buff *skb; - struct mpls_nh *nh; u8 n_labels; int err; diff --git a/net/mpls/internal.h b/net/mpls/internal.h index 893df00b77b6..b9f492ddf93b 100644 --- a/net/mpls/internal.h +++ b/net/mpls/internal.h @@ -158,17 +158,16 @@ struct mpls_route { /* next hop label forwarding entry */ }; #define for_nexthops(rt) { \ - int nhsel; struct mpls_nh *nh; u8 *__nh; \ - for (nhsel = 0, nh = (rt)->rt_nh, __nh = (u8 *)((rt)->rt_nh); \ + int nhsel; const struct mpls_nh *nh; \ + for (nhsel = 0, nh = (rt)->rt_nh; \ nhsel < (rt)->rt_nhn; \ - __nh += rt->rt_nh_size, nh = (struct mpls_nh *)__nh, nhsel++) + nh = (void *)nh + (rt)->rt_nh_size, nhsel++) #define change_nexthops(rt) { \ - int nhsel; struct mpls_nh *nh; u8 *__nh; \ - for (nhsel = 0, nh = (struct mpls_nh *)((rt)->rt_nh), \ - __nh = (u8 *)((rt)->rt_nh); \ + int nhsel; struct mpls_nh *nh; \ + for (nhsel = 0, nh = (rt)->rt_nh; \ nhsel < (rt)->rt_nhn; \ - __nh += rt->rt_nh_size, nh = (struct mpls_nh *)__nh, nhsel++) + nh = (void *)nh + (rt)->rt_nh_size, nhsel++) #define endfor_nexthops(rt) } diff --git a/net/mptcp/pm_netlink.c b/net/mptcp/pm_netlink.c index 7b96be1e9f14..4ff8d55cbe82 100644 --- a/net/mptcp/pm_netlink.c +++ b/net/mptcp/pm_netlink.c @@ -1702,22 +1702,28 @@ next: static int mptcp_nl_cmd_set_flags(struct sk_buff *skb, struct genl_info *info) { + struct mptcp_pm_addr_entry addr = { .addr = { .family = AF_UNSPEC }, }, *entry; struct nlattr *attr = info->attrs[MPTCP_PM_ATTR_ADDR]; struct pm_nl_pernet *pernet = genl_info_pm_nl(info); - struct mptcp_pm_addr_entry addr, *entry; struct net *net = sock_net(skb->sk); - u8 bkup = 0; + u8 bkup = 0, lookup_by_id = 0; int ret; - ret = mptcp_pm_parse_addr(attr, info, true, &addr); + ret = mptcp_pm_parse_addr(attr, info, false, &addr); if (ret < 0) return ret; if (addr.flags & MPTCP_PM_ADDR_FLAG_BACKUP) bkup = 1; + if (addr.addr.family == AF_UNSPEC) { + lookup_by_id = 1; + if (!addr.addr.id) + return -EOPNOTSUPP; + } list_for_each_entry(entry, &pernet->local_addr_list, list) { - if (addresses_equal(&entry->addr, &addr.addr, true)) { + if ((!lookup_by_id && addresses_equal(&entry->addr, &addr.addr, true)) || + (lookup_by_id && entry->addr.id == addr.addr.id)) { mptcp_nl_addr_backup(net, &entry->addr, bkup); if (bkup) diff --git a/net/mptcp/protocol.c b/net/mptcp/protocol.c index c82a76d2d0bf..f124cca125d2 100644 --- a/net/mptcp/protocol.c +++ b/net/mptcp/protocol.c @@ -22,6 +22,7 @@ #endif #include <net/mptcp.h> #include <net/xfrm.h> +#include <asm/ioctls.h> #include "protocol.h" #include "mib.h" @@ -46,9 +47,10 @@ struct mptcp_skb_cb { enum { MPTCP_CMSG_TS = BIT(0), + MPTCP_CMSG_INQ = BIT(1), }; -static struct percpu_counter mptcp_sockets_allocated; +static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; static void __mptcp_destroy_sock(struct sock *sk); static void __mptcp_check_send_data_fin(struct sock *sk); @@ -738,6 +740,7 @@ static bool __mptcp_ofo_queue(struct mptcp_sock *msk) MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, delta); MPTCP_SKB_CB(skb)->offset += delta; + MPTCP_SKB_CB(skb)->map_seq += delta; __skb_queue_tail(&sk->sk_receive_queue, skb); } msk->ack_seq = end_seq; @@ -1499,7 +1502,7 @@ static void mptcp_update_post_push(struct mptcp_sock *msk, msk->snd_nxt = snd_nxt_new; } -static void mptcp_check_and_set_pending(struct sock *sk) +void mptcp_check_and_set_pending(struct sock *sk) { if (mptcp_send_head(sk) && !test_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->flags)) @@ -1784,8 +1787,10 @@ static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, copied += count; if (count < data_len) { - if (!(flags & MSG_PEEK)) + if (!(flags & MSG_PEEK)) { MPTCP_SKB_CB(skb)->offset += count; + MPTCP_SKB_CB(skb)->map_seq += count; + } break; } @@ -1965,6 +1970,27 @@ static bool __mptcp_move_skbs(struct mptcp_sock *msk) return !skb_queue_empty(&msk->receive_queue); } +static unsigned int mptcp_inq_hint(const struct sock *sk) +{ + const struct mptcp_sock *msk = mptcp_sk(sk); + const struct sk_buff *skb; + + skb = skb_peek(&msk->receive_queue); + if (skb) { + u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; + + if (hint_val >= INT_MAX) + return INT_MAX; + + return (unsigned int)hint_val; + } + + if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) + return 1; + + return 0; +} + static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock, int flags, int *addr_len) { @@ -1989,6 +2015,9 @@ static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, len = min_t(size_t, len, INT_MAX); target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); + if (unlikely(msk->recvmsg_inq)) + cmsg_flags = MPTCP_CMSG_INQ; + while (copied < len) { int bytes_read; @@ -2062,6 +2091,12 @@ out_err: if (cmsg_flags && copied >= 0) { if (cmsg_flags & MPTCP_CMSG_TS) tcp_recv_timestamp(msg, sk, &tss); + + if (cmsg_flags & MPTCP_CMSG_INQ) { + unsigned int inq = mptcp_inq_hint(sk); + + put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); + } } pr_debug("msk=%p rx queue empty=%d:%d copied=%d", @@ -3177,6 +3212,57 @@ static int mptcp_forward_alloc_get(const struct sock *sk) return sk->sk_forward_alloc + mptcp_sk(sk)->rmem_fwd_alloc; } +static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) +{ + const struct sock *sk = (void *)msk; + u64 delta; + + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) + return 0; + + delta = msk->write_seq - v; + if (delta > INT_MAX) + delta = INT_MAX; + + return (int)delta; +} + +static int mptcp_ioctl(struct sock *sk, int cmd, unsigned long arg) +{ + struct mptcp_sock *msk = mptcp_sk(sk); + bool slow; + int answ; + + switch (cmd) { + case SIOCINQ: + if (sk->sk_state == TCP_LISTEN) + return -EINVAL; + + lock_sock(sk); + __mptcp_move_skbs(msk); + answ = mptcp_inq_hint(sk); + release_sock(sk); + break; + case SIOCOUTQ: + slow = lock_sock_fast(sk); + answ = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); + unlock_sock_fast(sk, slow); + break; + case SIOCOUTQNSD: + slow = lock_sock_fast(sk); + answ = mptcp_ioctl_outq(msk, msk->snd_nxt); + unlock_sock_fast(sk, slow); + break; + default: + return -ENOIOCTLCMD; + } + + return put_user(answ, (int __user *)arg); +} + static struct proto mptcp_prot = { .name = "MPTCP", .owner = THIS_MODULE, @@ -3189,6 +3275,7 @@ static struct proto mptcp_prot = { .shutdown = mptcp_shutdown, .destroy = mptcp_destroy, .sendmsg = mptcp_sendmsg, + .ioctl = mptcp_ioctl, .recvmsg = mptcp_recvmsg, .release_cb = mptcp_release_cb, .hash = mptcp_hash, diff --git a/net/mptcp/protocol.h b/net/mptcp/protocol.h index d87cc040352e..e1469155fb15 100644 --- a/net/mptcp/protocol.h +++ b/net/mptcp/protocol.h @@ -249,6 +249,9 @@ struct mptcp_sock { bool rcv_fastclose; bool use_64bit_ack; /* Set when we received a 64-bit DSN */ bool csum_enabled; + u8 recvmsg_inq:1, + cork:1, + nodelay:1; spinlock_t join_list_lock; struct work_struct work; struct sk_buff *ooo_last_skb; @@ -554,6 +557,7 @@ unsigned int mptcp_stale_loss_cnt(const struct net *net); void mptcp_subflow_fully_established(struct mptcp_subflow_context *subflow, struct mptcp_options_received *mp_opt); bool __mptcp_retransmit_pending_data(struct sock *sk); +void mptcp_check_and_set_pending(struct sock *sk); void __mptcp_push_pending(struct sock *sk, unsigned int flags); bool mptcp_subflow_data_available(struct sock *sk); void __init mptcp_subflow_init(void); diff --git a/net/mptcp/sockopt.c b/net/mptcp/sockopt.c index 0f1e661c2032..3c3db22fd36a 100644 --- a/net/mptcp/sockopt.c +++ b/net/mptcp/sockopt.c @@ -390,6 +390,8 @@ static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, switch (optname) { case IPV6_V6ONLY: + case IPV6_TRANSPARENT: + case IPV6_FREEBIND: lock_sock(sk); ssock = __mptcp_nmpc_socket(msk); if (!ssock) { @@ -398,8 +400,24 @@ static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, } ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen); - if (ret == 0) + if (ret != 0) { + release_sock(sk); + return ret; + } + + sockopt_seq_inc(msk); + + switch (optname) { + case IPV6_V6ONLY: sk->sk_ipv6only = ssock->sk->sk_ipv6only; + break; + case IPV6_TRANSPARENT: + inet_sk(sk)->transparent = inet_sk(ssock->sk)->transparent; + break; + case IPV6_FREEBIND: + inet_sk(sk)->freebind = inet_sk(ssock->sk)->freebind; + break; + } release_sock(sk); break; @@ -539,6 +557,7 @@ static bool mptcp_supported_sockopt(int level, int optname) case TCP_TIMESTAMP: case TCP_NOTSENT_LOWAT: case TCP_TX_DELAY: + case TCP_INQ: return true; } @@ -550,7 +569,6 @@ static bool mptcp_supported_sockopt(int level, int optname) /* TCP_FASTOPEN_KEY, TCP_FASTOPEN TCP_FASTOPEN_CONNECT, TCP_FASTOPEN_NO_COOKIE, * are not supported fastopen is currently unsupported */ - /* TCP_INQ is currently unsupported, needs some recvmsg work */ } return false; } @@ -598,14 +616,171 @@ static int mptcp_setsockopt_sol_tcp_congestion(struct mptcp_sock *msk, sockptr_t return ret; } +static int mptcp_setsockopt_sol_tcp_cork(struct mptcp_sock *msk, sockptr_t optval, + unsigned int optlen) +{ + struct mptcp_subflow_context *subflow; + struct sock *sk = (struct sock *)msk; + int val; + + if (optlen < sizeof(int)) + return -EINVAL; + + if (copy_from_sockptr(&val, optval, sizeof(val))) + return -EFAULT; + + lock_sock(sk); + sockopt_seq_inc(msk); + msk->cork = !!val; + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + + lock_sock(ssk); + __tcp_sock_set_cork(ssk, !!val); + release_sock(ssk); + } + if (!val) + mptcp_check_and_set_pending(sk); + release_sock(sk); + + return 0; +} + +static int mptcp_setsockopt_sol_tcp_nodelay(struct mptcp_sock *msk, sockptr_t optval, + unsigned int optlen) +{ + struct mptcp_subflow_context *subflow; + struct sock *sk = (struct sock *)msk; + int val; + + if (optlen < sizeof(int)) + return -EINVAL; + + if (copy_from_sockptr(&val, optval, sizeof(val))) + return -EFAULT; + + lock_sock(sk); + sockopt_seq_inc(msk); + msk->nodelay = !!val; + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + + lock_sock(ssk); + __tcp_sock_set_nodelay(ssk, !!val); + release_sock(ssk); + } + if (val) + mptcp_check_and_set_pending(sk); + release_sock(sk); + + return 0; +} + +static int mptcp_setsockopt_sol_ip_set_transparent(struct mptcp_sock *msk, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct sock *sk = (struct sock *)msk; + struct inet_sock *issk; + struct socket *ssock; + int err; + + err = ip_setsockopt(sk, SOL_IP, optname, optval, optlen); + if (err != 0) + return err; + + lock_sock(sk); + + ssock = __mptcp_nmpc_socket(msk); + if (!ssock) { + release_sock(sk); + return -EINVAL; + } + + issk = inet_sk(ssock->sk); + + switch (optname) { + case IP_FREEBIND: + issk->freebind = inet_sk(sk)->freebind; + break; + case IP_TRANSPARENT: + issk->transparent = inet_sk(sk)->transparent; + break; + default: + release_sock(sk); + WARN_ON_ONCE(1); + return -EOPNOTSUPP; + } + + sockopt_seq_inc(msk); + release_sock(sk); + return 0; +} + +static int mptcp_setsockopt_v4_set_tos(struct mptcp_sock *msk, int optname, + sockptr_t optval, unsigned int optlen) +{ + struct mptcp_subflow_context *subflow; + struct sock *sk = (struct sock *)msk; + int err, val; + + err = ip_setsockopt(sk, SOL_IP, optname, optval, optlen); + + if (err != 0) + return err; + + lock_sock(sk); + sockopt_seq_inc(msk); + val = inet_sk(sk)->tos; + mptcp_for_each_subflow(msk, subflow) { + struct sock *ssk = mptcp_subflow_tcp_sock(subflow); + + __ip_sock_set_tos(ssk, val); + } + release_sock(sk); + + return err; +} + +static int mptcp_setsockopt_v4(struct mptcp_sock *msk, int optname, + sockptr_t optval, unsigned int optlen) +{ + switch (optname) { + case IP_FREEBIND: + case IP_TRANSPARENT: + return mptcp_setsockopt_sol_ip_set_transparent(msk, optname, optval, optlen); + case IP_TOS: + return mptcp_setsockopt_v4_set_tos(msk, optname, optval, optlen); + } + + return -EOPNOTSUPP; +} + static int mptcp_setsockopt_sol_tcp(struct mptcp_sock *msk, int optname, sockptr_t optval, unsigned int optlen) { + struct sock *sk = (void *)msk; + int ret, val; + switch (optname) { + case TCP_INQ: + ret = mptcp_get_int_option(msk, optval, optlen, &val); + if (ret) + return ret; + if (val < 0 || val > 1) + return -EINVAL; + + lock_sock(sk); + msk->recvmsg_inq = !!val; + release_sock(sk); + return 0; case TCP_ULP: return -EOPNOTSUPP; case TCP_CONGESTION: return mptcp_setsockopt_sol_tcp_congestion(msk, optval, optlen); + case TCP_CORK: + return mptcp_setsockopt_sol_tcp_cork(msk, optval, optlen); + case TCP_NODELAY: + return mptcp_setsockopt_sol_tcp_nodelay(msk, optval, optlen); } return -EOPNOTSUPP; @@ -637,6 +812,9 @@ int mptcp_setsockopt(struct sock *sk, int level, int optname, if (ssk) return tcp_setsockopt(ssk, level, optname, optval, optlen); + if (level == SOL_IP) + return mptcp_setsockopt_v4(msk, optname, optval, optlen); + if (level == SOL_IPV6) return mptcp_setsockopt_v6(msk, optname, optval, optlen); @@ -932,6 +1110,35 @@ static int mptcp_getsockopt_subflow_addrs(struct mptcp_sock *msk, char __user *o return 0; } +static int mptcp_put_int_option(struct mptcp_sock *msk, char __user *optval, + int __user *optlen, int val) +{ + int len; + + if (get_user(len, optlen)) + return -EFAULT; + if (len < 0) + return -EINVAL; + + if (len < sizeof(int) && len > 0 && val >= 0 && val <= 255) { + unsigned char ucval = (unsigned char)val; + + len = 1; + if (put_user(len, optlen)) + return -EFAULT; + if (copy_to_user(optval, &ucval, 1)) + return -EFAULT; + } else { + len = min_t(unsigned int, len, sizeof(int)); + if (put_user(len, optlen)) + return -EFAULT; + if (copy_to_user(optval, &val, len)) + return -EFAULT; + } + + return 0; +} + static int mptcp_getsockopt_sol_tcp(struct mptcp_sock *msk, int optname, char __user *optval, int __user *optlen) { @@ -942,10 +1149,29 @@ static int mptcp_getsockopt_sol_tcp(struct mptcp_sock *msk, int optname, case TCP_CC_INFO: return mptcp_getsockopt_first_sf_only(msk, SOL_TCP, optname, optval, optlen); + case TCP_INQ: + return mptcp_put_int_option(msk, optval, optlen, msk->recvmsg_inq); + case TCP_CORK: + return mptcp_put_int_option(msk, optval, optlen, msk->cork); + case TCP_NODELAY: + return mptcp_put_int_option(msk, optval, optlen, msk->nodelay); } return -EOPNOTSUPP; } +static int mptcp_getsockopt_v4(struct mptcp_sock *msk, int optname, + char __user *optval, int __user *optlen) +{ + struct sock *sk = (void *)msk; + + switch (optname) { + case IP_TOS: + return mptcp_put_int_option(msk, optval, optlen, inet_sk(sk)->tos); + } + + return -EOPNOTSUPP; +} + static int mptcp_getsockopt_sol_mptcp(struct mptcp_sock *msk, int optname, char __user *optval, int __user *optlen) { @@ -981,6 +1207,8 @@ int mptcp_getsockopt(struct sock *sk, int level, int optname, if (ssk) return tcp_getsockopt(ssk, level, optname, optval, option); + if (level == SOL_IP) + return mptcp_getsockopt_v4(msk, optname, optval, option); if (level == SOL_TCP) return mptcp_getsockopt_sol_tcp(msk, optname, optval, option); if (level == SOL_MPTCP) @@ -1003,6 +1231,7 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk) ssk->sk_priority = sk->sk_priority; ssk->sk_bound_dev_if = sk->sk_bound_dev_if; ssk->sk_incoming_cpu = sk->sk_incoming_cpu; + __ip_sock_set_tos(ssk, inet_sk(sk)->tos); if (sk->sk_userlocks & tx_rx_locks) { ssk->sk_userlocks |= sk->sk_userlocks & tx_rx_locks; @@ -1028,6 +1257,11 @@ static void sync_socket_options(struct mptcp_sock *msk, struct sock *ssk) if (inet_csk(sk)->icsk_ca_ops != inet_csk(ssk)->icsk_ca_ops) tcp_set_congestion_control(ssk, msk->ca_name, false, true); + __tcp_sock_set_cork(ssk, !!msk->cork); + __tcp_sock_set_nodelay(ssk, !!msk->nodelay); + + inet_sk(ssk)->transparent = inet_sk(sk)->transparent; + inet_sk(ssk)->freebind = inet_sk(sk)->freebind; } static void __mptcp_sockopt_sync(struct mptcp_sock *msk, struct sock *ssk) diff --git a/net/mptcp/subflow.c b/net/mptcp/subflow.c index 6172f380dfb7..b8dd3441f7d0 100644 --- a/net/mptcp/subflow.c +++ b/net/mptcp/subflow.c @@ -1425,6 +1425,8 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, if (addr.ss_family == AF_INET6) addrlen = sizeof(struct sockaddr_in6); #endif + mptcp_sockopt_sync(msk, ssk); + ssk->sk_bound_dev_if = ifindex; err = kernel_bind(sf, (struct sockaddr *)&addr, addrlen); if (err) @@ -1441,7 +1443,6 @@ int __mptcp_subflow_connect(struct sock *sk, const struct mptcp_addr_info *loc, mptcp_info2sockaddr(remote, &addr, ssk->sk_family); mptcp_add_pending_subflow(msk, subflow); - mptcp_sockopt_sync(msk, ssk); err = kernel_connect(sf, (struct sockaddr *)&addr, addrlen, O_NONBLOCK); if (err && err != -EINPROGRESS) goto failed_unlink; @@ -1534,9 +1535,7 @@ int mptcp_subflow_create_socket(struct sock *sk, struct socket **new_sock) */ sf->sk->sk_net_refcnt = 1; get_net(net); -#ifdef CONFIG_PROC_FS - this_cpu_add(*net->core.sock_inuse, 1); -#endif + sock_inuse_add(net, 1); err = tcp_set_ulp(sf->sk, "mptcp"); release_sock(sf->sk); diff --git a/net/netfilter/nf_conntrack_core.c b/net/netfilter/nf_conntrack_core.c index 4712a90a1820..9fbce31baf75 100644 --- a/net/netfilter/nf_conntrack_core.c +++ b/net/netfilter/nf_conntrack_core.c @@ -189,7 +189,7 @@ EXPORT_SYMBOL_GPL(nf_conntrack_htable_size); unsigned int nf_conntrack_max __read_mostly; EXPORT_SYMBOL_GPL(nf_conntrack_max); seqcount_spinlock_t nf_conntrack_generation __read_mostly; -static siphash_key_t nf_conntrack_hash_rnd __read_mostly; +static siphash_aligned_key_t nf_conntrack_hash_rnd; static u32 hash_conntrack_raw(const struct nf_conntrack_tuple *tuple, unsigned int zoneid, @@ -482,7 +482,7 @@ EXPORT_SYMBOL_GPL(nf_ct_invert_tuple); */ u32 nf_ct_get_id(const struct nf_conn *ct) { - static __read_mostly siphash_key_t ct_id_seed; + static siphash_aligned_key_t ct_id_seed; unsigned long a, b, c, d; net_get_random_once(&ct_id_seed, sizeof(ct_id_seed)); diff --git a/net/netfilter/nf_conntrack_expect.c b/net/netfilter/nf_conntrack_expect.c index f562eeef4234..1e89b595ecd0 100644 --- a/net/netfilter/nf_conntrack_expect.c +++ b/net/netfilter/nf_conntrack_expect.c @@ -41,7 +41,7 @@ EXPORT_SYMBOL_GPL(nf_ct_expect_hash); unsigned int nf_ct_expect_max __read_mostly; static struct kmem_cache *nf_ct_expect_cachep __read_mostly; -static siphash_key_t nf_ct_expect_hashrnd __read_mostly; +static siphash_aligned_key_t nf_ct_expect_hashrnd; /* nf_conntrack_expect helper functions */ void nf_ct_unlink_expect_report(struct nf_conntrack_expect *exp, diff --git a/net/netfilter/nf_conntrack_netlink.c b/net/netfilter/nf_conntrack_netlink.c index 81d03acf68d4..47c72f28995a 100644 --- a/net/netfilter/nf_conntrack_netlink.c +++ b/net/netfilter/nf_conntrack_netlink.c @@ -2995,7 +2995,7 @@ static const union nf_inet_addr any_addr; static __be32 nf_expect_get_id(const struct nf_conntrack_expect *exp) { - static __read_mostly siphash_key_t exp_id_seed; + static siphash_aligned_key_t exp_id_seed; unsigned long a, b, c, d; net_get_random_once(&exp_id_seed, sizeof(exp_id_seed)); diff --git a/net/netfilter/nf_nat_core.c b/net/netfilter/nf_nat_core.c index 4d50d51db796..ab9f6c75524d 100644 --- a/net/netfilter/nf_nat_core.c +++ b/net/netfilter/nf_nat_core.c @@ -34,7 +34,7 @@ static unsigned int nat_net_id __read_mostly; static struct hlist_head *nf_nat_bysource __read_mostly; static unsigned int nf_nat_htable_size __read_mostly; -static siphash_key_t nf_nat_hash_rnd __read_mostly; +static siphash_aligned_key_t nf_nat_hash_rnd; struct nf_nat_lookup_hook_priv { struct nf_hook_entries __rcu *entries; diff --git a/net/netlink/af_netlink.c b/net/netlink/af_netlink.c index 9eba2e648385..4be2d97ff93e 100644 --- a/net/netlink/af_netlink.c +++ b/net/netlink/af_netlink.c @@ -707,9 +707,7 @@ static int netlink_create(struct net *net, struct socket *sock, int protocol, if (err < 0) goto out_module; - local_bh_disable(); sock_prot_inuse_add(net, &netlink_proto, 1); - local_bh_enable(); nlk = nlk_sk(sock->sk); nlk->module = module; @@ -809,9 +807,7 @@ static int netlink_release(struct socket *sock) netlink_table_ungrab(); } - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), &netlink_proto, -1); - local_bh_enable(); call_rcu(&nlk->rcu, deferred_put_nlk_sk); return 0; } diff --git a/net/openvswitch/vport-netdev.c b/net/openvswitch/vport-netdev.c index 8e1a88f13622..b498dac4e1e0 100644 --- a/net/openvswitch/vport-netdev.c +++ b/net/openvswitch/vport-netdev.c @@ -82,7 +82,7 @@ struct vport *ovs_netdev_link(struct vport *vport, const char *name) err = -ENODEV; goto error_free_vport; } - + netdev_tracker_alloc(vport->dev, &vport->dev_tracker, GFP_KERNEL); if (vport->dev->flags & IFF_LOOPBACK || (vport->dev->type != ARPHRD_ETHER && vport->dev->type != ARPHRD_NONE) || @@ -115,7 +115,7 @@ error_master_upper_dev_unlink: error_unlock: rtnl_unlock(); error_put: - dev_put(vport->dev); + dev_put_track(vport->dev, &vport->dev_tracker); error_free_vport: ovs_vport_free(vport); return ERR_PTR(err); @@ -137,8 +137,7 @@ static void vport_netdev_free(struct rcu_head *rcu) { struct vport *vport = container_of(rcu, struct vport, rcu); - if (vport->dev) - dev_put(vport->dev); + dev_put_track(vport->dev, &vport->dev_tracker); ovs_vport_free(vport); } @@ -174,7 +173,7 @@ void ovs_netdev_tunnel_destroy(struct vport *vport) */ if (vport->dev->reg_state == NETREG_REGISTERED) rtnl_delete_link(vport->dev); - dev_put(vport->dev); + dev_put_track(vport->dev, &vport->dev_tracker); vport->dev = NULL; rtnl_unlock(); diff --git a/net/openvswitch/vport.h b/net/openvswitch/vport.h index 8a930ca6d6b1..9de5030d9801 100644 --- a/net/openvswitch/vport.h +++ b/net/openvswitch/vport.h @@ -58,6 +58,7 @@ struct vport_portids { /** * struct vport - one port within a datapath * @dev: Pointer to net_device. + * @dev_tracker: refcount tracker for @dev reference * @dp: Datapath to which this port belongs. * @upcall_portids: RCU protected 'struct vport_portids'. * @port_no: Index into @dp's @ports array. @@ -69,6 +70,7 @@ struct vport_portids { */ struct vport { struct net_device *dev; + netdevice_tracker dev_tracker; struct datapath *dp; struct vport_portids __rcu *upcall_portids; u16 port_no; diff --git a/net/packet/af_packet.c b/net/packet/af_packet.c index 46943a18a10d..a1ffdb48cc47 100644 --- a/net/packet/af_packet.c +++ b/net/packet/af_packet.c @@ -3102,9 +3102,7 @@ static int packet_release(struct socket *sock) sk_del_node_init_rcu(sk); mutex_unlock(&net->packet.sklist_lock); - preempt_disable(); sock_prot_inuse_add(net, sk->sk_prot, -1); - preempt_enable(); spin_lock(&po->bind_lock); unregister_prot_hook(sk, false); @@ -3368,9 +3366,7 @@ static int packet_create(struct net *net, struct socket *sock, int protocol, sk_add_node_tail_rcu(sk, &net->packet.sklist); mutex_unlock(&net->packet.sklist_lock); - preempt_disable(); sock_prot_inuse_add(net, &packet_proto, 1); - preempt_enable(); return 0; out2: diff --git a/net/rds/send.c b/net/rds/send.c index 53444397de66..0c5504068e3c 100644 --- a/net/rds/send.c +++ b/net/rds/send.c @@ -272,7 +272,7 @@ restart: /* Unfortunately, the way Infiniband deals with * RDMA to a bad MR key is by moving the entire - * queue pair to error state. We cold possibly + * queue pair to error state. We could possibly * recover from that, but right now we drop the * connection. * Therefore, we never retransmit messages with RDMA ops. diff --git a/net/sched/act_ct.c b/net/sched/act_ct.c index 90866ae45573..ab1810f2e660 100644 --- a/net/sched/act_ct.c +++ b/net/sched/act_ct.c @@ -393,7 +393,8 @@ static void tcf_ct_flow_table_process_conn(struct tcf_ct_flow_table *ct_ft, { bool tcp = false; - if (ctinfo != IP_CT_ESTABLISHED && ctinfo != IP_CT_ESTABLISHED_REPLY) + if ((ctinfo != IP_CT_ESTABLISHED && ctinfo != IP_CT_ESTABLISHED_REPLY) || + !test_bit(IPS_ASSURED_BIT, &ct->status)) return; switch (nf_ct_protonum(ct)) { diff --git a/net/sched/act_mirred.c b/net/sched/act_mirred.c index efc963ab995a..952416bd65e6 100644 --- a/net/sched/act_mirred.c +++ b/net/sched/act_mirred.c @@ -79,7 +79,7 @@ static void tcf_mirred_release(struct tc_action *a) /* last reference to action, no need to lock */ dev = rcu_dereference_protected(m->tcfm_dev, 1); - dev_put(dev); + dev_put_track(dev, &m->tcfm_dev_tracker); } static const struct nla_policy mirred_policy[TCA_MIRRED_MAX + 1] = { @@ -101,7 +101,6 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, bool mac_header_xmit = false; struct tc_mirred *parm; struct tcf_mirred *m; - struct net_device *dev; bool exists = false; int ret, err; u32 index; @@ -171,16 +170,19 @@ static int tcf_mirred_init(struct net *net, struct nlattr *nla, spin_lock_bh(&m->tcf_lock); if (parm->ifindex) { - dev = dev_get_by_index(net, parm->ifindex); - if (!dev) { + struct net_device *odev, *ndev; + + ndev = dev_get_by_index(net, parm->ifindex); + if (!ndev) { spin_unlock_bh(&m->tcf_lock); err = -ENODEV; goto put_chain; } - mac_header_xmit = dev_is_mac_header_xmit(dev); - dev = rcu_replace_pointer(m->tcfm_dev, dev, + mac_header_xmit = dev_is_mac_header_xmit(ndev); + odev = rcu_replace_pointer(m->tcfm_dev, ndev, lockdep_is_held(&m->tcf_lock)); - dev_put(dev); + dev_put_track(odev, &m->tcfm_dev_tracker); + netdev_tracker_alloc(ndev, &m->tcfm_dev_tracker, GFP_ATOMIC); m->tcfm_mac_header_xmit = mac_header_xmit; } goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); @@ -400,7 +402,7 @@ static int mirred_device_event(struct notifier_block *unused, list_for_each_entry(m, &mirred_list, tcfm_list) { spin_lock_bh(&m->tcf_lock); if (tcf_mirred_dev_dereference(m) == dev) { - dev_put(dev); + dev_put_track(dev, &m->tcfm_dev_tracker); /* Note : no rcu grace period necessary, as * net_device are already rcu protected. */ diff --git a/net/sched/sch_generic.c b/net/sched/sch_generic.c index 3b0f62095803..b07bd1c7330f 100644 --- a/net/sched/sch_generic.c +++ b/net/sched/sch_generic.c @@ -434,9 +434,9 @@ unsigned long dev_trans_start(struct net_device *dev) dev = vlan_dev_real_dev(dev); else if (netif_is_macvlan(dev)) dev = macvlan_dev_real_dev(dev); - res = netdev_get_tx_queue(dev, 0)->trans_start; + res = READ_ONCE(netdev_get_tx_queue(dev, 0)->trans_start); for (i = 1; i < dev->num_tx_queues; i++) { - val = netdev_get_tx_queue(dev, i)->trans_start; + val = READ_ONCE(netdev_get_tx_queue(dev, i)->trans_start); if (val && time_after(val, res)) res = val; } @@ -445,11 +445,63 @@ unsigned long dev_trans_start(struct net_device *dev) } EXPORT_SYMBOL(dev_trans_start); +static void netif_freeze_queues(struct net_device *dev) +{ + unsigned int i; + int cpu; + + cpu = smp_processor_id(); + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + + /* We are the only thread of execution doing a + * freeze, but we have to grab the _xmit_lock in + * order to synchronize with threads which are in + * the ->hard_start_xmit() handler and already + * checked the frozen bit. + */ + __netif_tx_lock(txq, cpu); + set_bit(__QUEUE_STATE_FROZEN, &txq->state); + __netif_tx_unlock(txq); + } +} + +void netif_tx_lock(struct net_device *dev) +{ + spin_lock(&dev->tx_global_lock); + netif_freeze_queues(dev); +} +EXPORT_SYMBOL(netif_tx_lock); + +static void netif_unfreeze_queues(struct net_device *dev) +{ + unsigned int i; + + for (i = 0; i < dev->num_tx_queues; i++) { + struct netdev_queue *txq = netdev_get_tx_queue(dev, i); + + /* No need to grab the _xmit_lock here. If the + * queue is not stopped for another reason, we + * force a schedule. + */ + clear_bit(__QUEUE_STATE_FROZEN, &txq->state); + netif_schedule_queue(txq); + } +} + +void netif_tx_unlock(struct net_device *dev) +{ + netif_unfreeze_queues(dev); + spin_unlock(&dev->tx_global_lock); +} +EXPORT_SYMBOL(netif_tx_unlock); + static void dev_watchdog(struct timer_list *t) { struct net_device *dev = from_timer(dev, t, watchdog_timer); + bool release = true; - netif_tx_lock(dev); + spin_lock(&dev->tx_global_lock); if (!qdisc_tx_is_noop(dev)) { if (netif_device_present(dev) && netif_running(dev) && @@ -462,31 +514,34 @@ static void dev_watchdog(struct timer_list *t) struct netdev_queue *txq; txq = netdev_get_tx_queue(dev, i); - trans_start = txq->trans_start; + trans_start = READ_ONCE(txq->trans_start); if (netif_xmit_stopped(txq) && time_after(jiffies, (trans_start + dev->watchdog_timeo))) { some_queue_timedout = 1; - txq->trans_timeout++; + atomic_long_inc(&txq->trans_timeout); break; } } - if (some_queue_timedout) { + if (unlikely(some_queue_timedout)) { trace_net_dev_xmit_timeout(dev, i); WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n", dev->name, netdev_drivername(dev), i); + netif_freeze_queues(dev); dev->netdev_ops->ndo_tx_timeout(dev, i); + netif_unfreeze_queues(dev); } if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) - dev_hold(dev); + release = false; } } - netif_tx_unlock(dev); + spin_unlock(&dev->tx_global_lock); - dev_put(dev); + if (release) + dev_put_track(dev, &dev->watchdog_dev_tracker); } void __netdev_watchdog_up(struct net_device *dev) @@ -496,7 +551,7 @@ void __netdev_watchdog_up(struct net_device *dev) dev->watchdog_timeo = 5*HZ; if (!mod_timer(&dev->watchdog_timer, round_jiffies(jiffies + dev->watchdog_timeo))) - dev_hold(dev); + dev_hold_track(dev, &dev->watchdog_dev_tracker, GFP_ATOMIC); } } EXPORT_SYMBOL_GPL(__netdev_watchdog_up); @@ -510,7 +565,7 @@ static void dev_watchdog_down(struct net_device *dev) { netif_tx_lock_bh(dev); if (del_timer(&dev->watchdog_timer)) - dev_put(dev); + dev_put_track(dev, &dev->watchdog_dev_tracker); netif_tx_unlock_bh(dev); } @@ -920,7 +975,7 @@ struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue, sch->enqueue = ops->enqueue; sch->dequeue = ops->dequeue; sch->dev_queue = dev_queue; - dev_hold(dev); + dev_hold_track(dev, &sch->dev_tracker, GFP_KERNEL); refcount_set(&sch->refcnt, 1); return sch; @@ -1020,7 +1075,7 @@ static void qdisc_destroy(struct Qdisc *qdisc) ops->destroy(qdisc); module_put(ops->owner); - dev_put(qdisc_dev(qdisc)); + dev_put_track(qdisc_dev(qdisc), &qdisc->dev_tracker); trace_qdisc_destroy(qdisc); @@ -1148,7 +1203,7 @@ static void transition_one_qdisc(struct net_device *dev, rcu_assign_pointer(dev_queue->qdisc, new_qdisc); if (need_watchdog_p) { - dev_queue->trans_start = 0; + WRITE_ONCE(dev_queue->trans_start, 0); *need_watchdog_p = 1; } } diff --git a/net/sched/sch_netem.c b/net/sched/sch_netem.c index ecbb10db1111..ed4ccef5d6a8 100644 --- a/net/sched/sch_netem.c +++ b/net/sched/sch_netem.c @@ -208,17 +208,17 @@ static bool loss_4state(struct netem_sched_data *q) * next state and if the next packet has to be transmitted or lost. * The four states correspond to: * TX_IN_GAP_PERIOD => successfully transmitted packets within a gap period - * LOST_IN_BURST_PERIOD => isolated losses within a gap period - * LOST_IN_GAP_PERIOD => lost packets within a burst period - * TX_IN_GAP_PERIOD => successfully transmitted packets within a burst period + * LOST_IN_GAP_PERIOD => isolated losses within a gap period + * LOST_IN_BURST_PERIOD => lost packets within a burst period + * TX_IN_BURST_PERIOD => successfully transmitted packets within a burst period */ switch (clg->state) { case TX_IN_GAP_PERIOD: if (rnd < clg->a4) { - clg->state = LOST_IN_BURST_PERIOD; + clg->state = LOST_IN_GAP_PERIOD; return true; } else if (clg->a4 < rnd && rnd < clg->a1 + clg->a4) { - clg->state = LOST_IN_GAP_PERIOD; + clg->state = LOST_IN_BURST_PERIOD; return true; } else if (clg->a1 + clg->a4 < rnd) { clg->state = TX_IN_GAP_PERIOD; @@ -227,24 +227,24 @@ static bool loss_4state(struct netem_sched_data *q) break; case TX_IN_BURST_PERIOD: if (rnd < clg->a5) { - clg->state = LOST_IN_GAP_PERIOD; + clg->state = LOST_IN_BURST_PERIOD; return true; } else { clg->state = TX_IN_BURST_PERIOD; } break; - case LOST_IN_GAP_PERIOD: + case LOST_IN_BURST_PERIOD: if (rnd < clg->a3) clg->state = TX_IN_BURST_PERIOD; else if (clg->a3 < rnd && rnd < clg->a2 + clg->a3) { clg->state = TX_IN_GAP_PERIOD; } else if (clg->a2 + clg->a3 < rnd) { - clg->state = LOST_IN_GAP_PERIOD; + clg->state = LOST_IN_BURST_PERIOD; return true; } break; - case LOST_IN_BURST_PERIOD: + case LOST_IN_GAP_PERIOD: clg->state = TX_IN_GAP_PERIOD; break; } diff --git a/net/sctp/output.c b/net/sctp/output.c index cdfdbd353c67..72fe6669c50d 100644 --- a/net/sctp/output.c +++ b/net/sctp/output.c @@ -134,7 +134,7 @@ void sctp_packet_config(struct sctp_packet *packet, __u32 vtag, dst_hold(tp->dst); sk_setup_caps(sk, tp->dst); } - packet->max_size = sk_can_gso(sk) ? tp->dst->dev->gso_max_size + packet->max_size = sk_can_gso(sk) ? READ_ONCE(tp->dst->dev->gso_max_size) : asoc->pathmtu; rcu_read_unlock(); } diff --git a/net/sctp/outqueue.c b/net/sctp/outqueue.c index ff47091c385e..a18609f608fb 100644 --- a/net/sctp/outqueue.c +++ b/net/sctp/outqueue.c @@ -547,6 +547,9 @@ void sctp_retransmit(struct sctp_outq *q, struct sctp_transport *transport, sctp_assoc_update_retran_path(transport->asoc); transport->asoc->rtx_data_chunks += transport->asoc->unack_data; + if (transport->pl.state == SCTP_PL_COMPLETE && + transport->asoc->unack_data) + sctp_transport_reset_probe_timer(transport); break; case SCTP_RTXR_FAST_RTX: SCTP_INC_STATS(net, SCTP_MIB_FAST_RETRANSMITS); diff --git a/net/sctp/sm_statefuns.c b/net/sctp/sm_statefuns.c index 354c1c4de19b..cc544a97c4af 100644 --- a/net/sctp/sm_statefuns.c +++ b/net/sctp/sm_statefuns.c @@ -1124,12 +1124,11 @@ enum sctp_disposition sctp_sf_send_probe(struct net *net, if (!sctp_transport_pl_enabled(transport)) return SCTP_DISPOSITION_CONSUME; - if (sctp_transport_pl_send(transport)) { - reply = sctp_make_heartbeat(asoc, transport, transport->pl.probe_size); - if (!reply) - return SCTP_DISPOSITION_NOMEM; - sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(reply)); - } + sctp_transport_pl_send(transport); + reply = sctp_make_heartbeat(asoc, transport, transport->pl.probe_size); + if (!reply) + return SCTP_DISPOSITION_NOMEM; + sctp_add_cmd_sf(commands, SCTP_CMD_REPLY, SCTP_CHUNK(reply)); sctp_add_cmd_sf(commands, SCTP_CMD_PROBE_TIMER_UPDATE, SCTP_TRANSPORT(transport)); diff --git a/net/sctp/socket.c b/net/sctp/socket.c index 33391254fa82..055a6d3ec6e2 100644 --- a/net/sctp/socket.c +++ b/net/sctp/socket.c @@ -5068,12 +5068,9 @@ static int sctp_init_sock(struct sock *sk) SCTP_DBG_OBJCNT_INC(sock); - local_bh_disable(); sk_sockets_allocated_inc(sk); sock_prot_inuse_add(net, sk->sk_prot, 1); - local_bh_enable(); - return 0; } @@ -5099,10 +5096,8 @@ static void sctp_destroy_sock(struct sock *sk) list_del(&sp->auto_asconf_list); } sctp_endpoint_free(sp->ep); - local_bh_disable(); sk_sockets_allocated_dec(sk); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); - local_bh_enable(); } /* Triggered when there are no references on the socket anymore */ diff --git a/net/sctp/transport.c b/net/sctp/transport.c index 133f1719bf1b..f8fd98784977 100644 --- a/net/sctp/transport.c +++ b/net/sctp/transport.c @@ -213,13 +213,18 @@ void sctp_transport_reset_reconf_timer(struct sctp_transport *transport) void sctp_transport_reset_probe_timer(struct sctp_transport *transport) { - if (timer_pending(&transport->probe_timer)) - return; if (!mod_timer(&transport->probe_timer, jiffies + transport->probe_interval)) sctp_transport_hold(transport); } +void sctp_transport_reset_raise_timer(struct sctp_transport *transport) +{ + if (!mod_timer(&transport->probe_timer, + jiffies + transport->probe_interval * 30)) + sctp_transport_hold(transport); +} + /* This transport has been assigned to an association. * Initialize fields from the association or from the sock itself. * Register the reference count in the association. @@ -258,12 +263,11 @@ void sctp_transport_pmtu(struct sctp_transport *transport, struct sock *sk) sctp_transport_pl_update(transport); } -bool sctp_transport_pl_send(struct sctp_transport *t) +void sctp_transport_pl_send(struct sctp_transport *t) { if (t->pl.probe_count < SCTP_MAX_PROBES) goto out; - t->pl.last_rtx_chunks = t->asoc->rtx_data_chunks; t->pl.probe_count = 0; if (t->pl.state == SCTP_PL_BASE) { if (t->pl.probe_size == SCTP_BASE_PLPMTU) { /* BASE_PLPMTU Confirmation Failed */ @@ -298,17 +302,9 @@ bool sctp_transport_pl_send(struct sctp_transport *t) } out: - if (t->pl.state == SCTP_PL_COMPLETE && t->pl.raise_count < 30 && - !t->pl.probe_count && t->pl.last_rtx_chunks == t->asoc->rtx_data_chunks) { - t->pl.raise_count++; - return false; - } - pr_debug("%s: PLPMTUD: transport: %p, state: %d, pmtu: %d, size: %d, high: %d\n", __func__, t, t->pl.state, t->pl.pmtu, t->pl.probe_size, t->pl.probe_high); - t->pl.probe_count++; - return true; } bool sctp_transport_pl_recv(struct sctp_transport *t) @@ -316,7 +312,6 @@ bool sctp_transport_pl_recv(struct sctp_transport *t) pr_debug("%s: PLPMTUD: transport: %p, state: %d, pmtu: %d, size: %d, high: %d\n", __func__, t, t->pl.state, t->pl.pmtu, t->pl.probe_size, t->pl.probe_high); - t->pl.last_rtx_chunks = t->asoc->rtx_data_chunks; t->pl.pmtu = t->pl.probe_size; t->pl.probe_count = 0; if (t->pl.state == SCTP_PL_BASE) { @@ -338,14 +333,14 @@ bool sctp_transport_pl_recv(struct sctp_transport *t) t->pl.probe_size += SCTP_PL_MIN_STEP; if (t->pl.probe_size >= t->pl.probe_high) { t->pl.probe_high = 0; - t->pl.raise_count = 0; t->pl.state = SCTP_PL_COMPLETE; /* Search -> Search Complete */ t->pl.probe_size = t->pl.pmtu; t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t); sctp_assoc_sync_pmtu(t->asoc); + sctp_transport_reset_raise_timer(t); } - } else if (t->pl.state == SCTP_PL_COMPLETE && t->pl.raise_count == 30) { + } else if (t->pl.state == SCTP_PL_COMPLETE) { /* Raise probe_size again after 30 * interval in Search Complete */ t->pl.state = SCTP_PL_SEARCH; /* Search Complete -> Search */ t->pl.probe_size += SCTP_PL_MIN_STEP; @@ -393,6 +388,7 @@ static bool sctp_transport_pl_toobig(struct sctp_transport *t, u32 pmtu) t->pl.probe_high = 0; t->pl.pmtu = SCTP_BASE_PLPMTU; t->pathmtu = t->pl.pmtu + sctp_transport_pl_hlen(t); + sctp_transport_reset_probe_timer(t); return true; } } diff --git a/net/smc/af_smc.c b/net/smc/af_smc.c index 230072f9ec48..67a78bbf305f 100644 --- a/net/smc/af_smc.c +++ b/net/smc/af_smc.c @@ -89,8 +89,8 @@ int smc_hash_sk(struct sock *sk) write_lock_bh(&h->lock); sk_add_node(sk, head); - sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); write_unlock_bh(&h->lock); + sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); return 0; } diff --git a/net/smc/smc_core.c b/net/smc/smc_core.c index 387d28b2f8dd..85be94cabb01 100644 --- a/net/smc/smc_core.c +++ b/net/smc/smc_core.c @@ -1101,18 +1101,24 @@ static void smcr_buf_unuse(struct smc_buf_desc *rmb_desc, smc_buf_free(lgr, true, rmb_desc); } else { rmb_desc->used = 0; + memset(rmb_desc->cpu_addr, 0, rmb_desc->len); } } static void smc_buf_unuse(struct smc_connection *conn, struct smc_link_group *lgr) { - if (conn->sndbuf_desc) + if (conn->sndbuf_desc) { conn->sndbuf_desc->used = 0; - if (conn->rmb_desc && lgr->is_smcd) + memset(conn->sndbuf_desc->cpu_addr, 0, conn->sndbuf_desc->len); + } + if (conn->rmb_desc && lgr->is_smcd) { conn->rmb_desc->used = 0; - else if (conn->rmb_desc) + memset(conn->rmb_desc->cpu_addr, 0, conn->rmb_desc->len + + sizeof(struct smcd_cdc_msg)); + } else if (conn->rmb_desc) { smcr_buf_unuse(conn->rmb_desc, lgr); + } } /* remove a finished connection from its link group */ @@ -2148,7 +2154,6 @@ static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) if (buf_desc) { SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, bufsize); SMC_STAT_BUF_REUSE(smc, is_smcd, is_rmb); - memset(buf_desc->cpu_addr, 0, bufsize); break; /* found reusable slot */ } diff --git a/net/smc/smc_pnet.c b/net/smc/smc_pnet.c index 67e9d9fde085..e171cc6483f8 100644 --- a/net/smc/smc_pnet.c +++ b/net/smc/smc_pnet.c @@ -64,6 +64,7 @@ struct smc_pnetentry { struct { char eth_name[IFNAMSIZ + 1]; struct net_device *ndev; + netdevice_tracker dev_tracker; }; struct { char ib_name[IB_DEVICE_NAME_MAX + 1]; @@ -119,7 +120,7 @@ static int smc_pnet_remove_by_pnetid(struct net *net, char *pnet_name) smc_pnet_match(pnetelem->pnet_name, pnet_name)) { list_del(&pnetelem->list); if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev) { - dev_put(pnetelem->ndev); + dev_put_track(pnetelem->ndev, &pnetelem->dev_tracker); pr_warn_ratelimited("smc: net device %s " "erased user defined " "pnetid %.16s\n", @@ -195,7 +196,7 @@ static int smc_pnet_add_by_ndev(struct net_device *ndev) list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) { if (pnetelem->type == SMC_PNET_ETH && !pnetelem->ndev && !strncmp(pnetelem->eth_name, ndev->name, IFNAMSIZ)) { - dev_hold(ndev); + dev_hold_track(ndev, &pnetelem->dev_tracker, GFP_ATOMIC); pnetelem->ndev = ndev; rc = 0; pr_warn_ratelimited("smc: adding net device %s with " @@ -226,7 +227,7 @@ static int smc_pnet_remove_by_ndev(struct net_device *ndev) write_lock(&pnettable->lock); list_for_each_entry_safe(pnetelem, tmp_pe, &pnettable->pnetlist, list) { if (pnetelem->type == SMC_PNET_ETH && pnetelem->ndev == ndev) { - dev_put(pnetelem->ndev); + dev_put_track(pnetelem->ndev, &pnetelem->dev_tracker); pnetelem->ndev = NULL; rc = 0; pr_warn_ratelimited("smc: removing net device %s with " @@ -368,7 +369,7 @@ static int smc_pnet_add_eth(struct smc_pnettable *pnettable, struct net *net, memcpy(new_pe->pnet_name, pnet_name, SMC_MAX_PNETID_LEN); strncpy(new_pe->eth_name, eth_name, IFNAMSIZ); new_pe->ndev = ndev; - + netdev_tracker_alloc(ndev, &new_pe->dev_tracker, GFP_KERNEL); rc = -EEXIST; new_netdev = true; write_lock(&pnettable->lock); diff --git a/net/switchdev/switchdev.c b/net/switchdev/switchdev.c index 83460470e883..b62565278fac 100644 --- a/net/switchdev/switchdev.c +++ b/net/switchdev/switchdev.c @@ -28,6 +28,7 @@ typedef void switchdev_deferred_func_t(struct net_device *dev, struct switchdev_deferred_item { struct list_head list; struct net_device *dev; + netdevice_tracker dev_tracker; switchdev_deferred_func_t *func; unsigned long data[]; }; @@ -63,7 +64,7 @@ void switchdev_deferred_process(void) while ((dfitem = switchdev_deferred_dequeue())) { dfitem->func(dfitem->dev, dfitem->data); - dev_put(dfitem->dev); + dev_put_track(dfitem->dev, &dfitem->dev_tracker); kfree(dfitem); } } @@ -90,7 +91,7 @@ static int switchdev_deferred_enqueue(struct net_device *dev, dfitem->dev = dev; dfitem->func = func; memcpy(dfitem->data, data, data_len); - dev_hold(dev); + dev_hold_track(dev, &dfitem->dev_tracker, GFP_ATOMIC); spin_lock_bh(&deferred_lock); list_add_tail(&dfitem->list, &deferred); spin_unlock_bh(&deferred_lock); diff --git a/net/tipc/crypto.c b/net/tipc/crypto.c index b4d9419a015b..81116312b753 100644 --- a/net/tipc/crypto.c +++ b/net/tipc/crypto.c @@ -761,21 +761,10 @@ static int tipc_aead_encrypt(struct tipc_aead *aead, struct sk_buff *skb, skb_tailroom(skb), tailen); } - if (unlikely(!skb_cloned(skb) && tailen <= skb_tailroom(skb))) { - nsg = 1; - trailer = skb; - } else { - /* TODO: We could avoid skb_cow_data() if skb has no frag_list - * e.g. by skb_fill_page_desc() to add another page to the skb - * with the wanted tailen... However, page skbs look not often, - * so take it easy now! - * Cloned skbs e.g. from link_xmit() seems no choice though :( - */ - nsg = skb_cow_data(skb, tailen, &trailer); - if (unlikely(nsg < 0)) { - pr_err("TX: skb_cow_data() returned %d\n", nsg); - return nsg; - } + nsg = skb_cow_data(skb, tailen, &trailer); + if (unlikely(nsg < 0)) { + pr_err("TX: skb_cow_data() returned %d\n", nsg); + return nsg; } pskb_put(skb, trailer, tailen); diff --git a/net/tls/tls_sw.c b/net/tls/tls_sw.c index dfe623a4e72f..3f271e29812f 100644 --- a/net/tls/tls_sw.c +++ b/net/tls/tls_sw.c @@ -2328,10 +2328,6 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) struct tls_context *tls_ctx = tls_get_ctx(sk); struct tls_prot_info *prot = &tls_ctx->prot_info; struct tls_crypto_info *crypto_info; - struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; - struct tls12_crypto_info_aes_gcm_256 *gcm_256_info; - struct tls12_crypto_info_aes_ccm_128 *ccm_128_info; - struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305_info; struct tls_sw_context_tx *sw_ctx_tx = NULL; struct tls_sw_context_rx *sw_ctx_rx = NULL; struct cipher_context *cctx; @@ -2394,15 +2390,15 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) switch (crypto_info->cipher_type) { case TLS_CIPHER_AES_GCM_128: { + struct tls12_crypto_info_aes_gcm_128 *gcm_128_info; + + gcm_128_info = (void *)crypto_info; nonce_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; tag_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; - iv = ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->iv; + iv = gcm_128_info->iv; rec_seq_size = TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE; - rec_seq = - ((struct tls12_crypto_info_aes_gcm_128 *)crypto_info)->rec_seq; - gcm_128_info = - (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + rec_seq = gcm_128_info->rec_seq; keysize = TLS_CIPHER_AES_GCM_128_KEY_SIZE; key = gcm_128_info->key; salt = gcm_128_info->salt; @@ -2411,15 +2407,15 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) break; } case TLS_CIPHER_AES_GCM_256: { + struct tls12_crypto_info_aes_gcm_256 *gcm_256_info; + + gcm_256_info = (void *)crypto_info; nonce_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; tag_size = TLS_CIPHER_AES_GCM_256_TAG_SIZE; iv_size = TLS_CIPHER_AES_GCM_256_IV_SIZE; - iv = ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->iv; + iv = gcm_256_info->iv; rec_seq_size = TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE; - rec_seq = - ((struct tls12_crypto_info_aes_gcm_256 *)crypto_info)->rec_seq; - gcm_256_info = - (struct tls12_crypto_info_aes_gcm_256 *)crypto_info; + rec_seq = gcm_256_info->rec_seq; keysize = TLS_CIPHER_AES_GCM_256_KEY_SIZE; key = gcm_256_info->key; salt = gcm_256_info->salt; @@ -2428,15 +2424,15 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) break; } case TLS_CIPHER_AES_CCM_128: { + struct tls12_crypto_info_aes_ccm_128 *ccm_128_info; + + ccm_128_info = (void *)crypto_info; nonce_size = TLS_CIPHER_AES_CCM_128_IV_SIZE; tag_size = TLS_CIPHER_AES_CCM_128_TAG_SIZE; iv_size = TLS_CIPHER_AES_CCM_128_IV_SIZE; - iv = ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->iv; + iv = ccm_128_info->iv; rec_seq_size = TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE; - rec_seq = - ((struct tls12_crypto_info_aes_ccm_128 *)crypto_info)->rec_seq; - ccm_128_info = - (struct tls12_crypto_info_aes_ccm_128 *)crypto_info; + rec_seq = ccm_128_info->rec_seq; keysize = TLS_CIPHER_AES_CCM_128_KEY_SIZE; key = ccm_128_info->key; salt = ccm_128_info->salt; @@ -2445,6 +2441,8 @@ int tls_set_sw_offload(struct sock *sk, struct tls_context *ctx, int tx) break; } case TLS_CIPHER_CHACHA20_POLY1305: { + struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305_info; + chacha20_poly1305_info = (void *)crypto_info; nonce_size = 0; tag_size = TLS_CIPHER_CHACHA20_POLY1305_TAG_SIZE; diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c index b0bfc78e421c..4d6e33bbd446 100644 --- a/net/unix/af_unix.c +++ b/net/unix/af_unix.c @@ -117,24 +117,64 @@ #include "scm.h" +spinlock_t unix_table_locks[2 * UNIX_HASH_SIZE]; +EXPORT_SYMBOL_GPL(unix_table_locks); struct hlist_head unix_socket_table[2 * UNIX_HASH_SIZE]; EXPORT_SYMBOL_GPL(unix_socket_table); -DEFINE_SPINLOCK(unix_table_lock); -EXPORT_SYMBOL_GPL(unix_table_lock); static atomic_long_t unix_nr_socks; +/* SMP locking strategy: + * hash table is protected with spinlock unix_table_locks + * each socket state is protected by separate spin lock. + */ -static struct hlist_head *unix_sockets_unbound(void *addr) +static unsigned int unix_unbound_hash(struct sock *sk) { - unsigned long hash = (unsigned long)addr; + unsigned long hash = (unsigned long)sk; hash ^= hash >> 16; hash ^= hash >> 8; - hash %= UNIX_HASH_SIZE; - return &unix_socket_table[UNIX_HASH_SIZE + hash]; + hash ^= sk->sk_type; + + return UNIX_HASH_SIZE + (hash & (UNIX_HASH_SIZE - 1)); +} + +static unsigned int unix_bsd_hash(struct inode *i) +{ + return i->i_ino & (UNIX_HASH_SIZE - 1); +} + +static unsigned int unix_abstract_hash(struct sockaddr_un *sunaddr, + int addr_len, int type) +{ + __wsum csum = csum_partial(sunaddr, addr_len, 0); + unsigned int hash; + + hash = (__force unsigned int)csum_fold(csum); + hash ^= hash >> 8; + hash ^= type; + + return hash & (UNIX_HASH_SIZE - 1); } -#define UNIX_ABSTRACT(sk) (unix_sk(sk)->addr->hash < UNIX_HASH_SIZE) +static void unix_table_double_lock(unsigned int hash1, unsigned int hash2) +{ + /* hash1 and hash2 is never the same because + * one is between 0 and UNIX_HASH_SIZE - 1, and + * another is between UNIX_HASH_SIZE and UNIX_HASH_SIZE * 2. + */ + if (hash1 > hash2) + swap(hash1, hash2); + + spin_lock(&unix_table_locks[hash1]); + spin_lock_nested(&unix_table_locks[hash2], SINGLE_DEPTH_NESTING); +} + +static void unix_table_double_unlock(unsigned int hash1, unsigned int hash2) +{ + spin_unlock(&unix_table_locks[hash1]); + spin_unlock(&unix_table_locks[hash2]); +} #ifdef CONFIG_SECURITY_NETWORK static void unix_get_secdata(struct scm_cookie *scm, struct sk_buff *skb) @@ -164,20 +204,6 @@ static inline bool unix_secdata_eq(struct scm_cookie *scm, struct sk_buff *skb) } #endif /* CONFIG_SECURITY_NETWORK */ -/* - * SMP locking strategy: - * hash table is protected with spinlock unix_table_lock - * each socket state is protected by separate spin lock. - */ - -static inline unsigned int unix_hash_fold(__wsum n) -{ - unsigned int hash = (__force unsigned int)csum_fold(n); - - hash ^= hash>>8; - return hash&(UNIX_HASH_SIZE-1); -} - #define unix_peer(sk) (unix_sk(sk)->peer) static inline int unix_our_peer(struct sock *sk, struct sock *osk) @@ -214,6 +240,22 @@ struct sock *unix_peer_get(struct sock *s) } EXPORT_SYMBOL_GPL(unix_peer_get); +static struct unix_address *unix_create_addr(struct sockaddr_un *sunaddr, + int addr_len) +{ + struct unix_address *addr; + + addr = kmalloc(sizeof(*addr) + addr_len, GFP_KERNEL); + if (!addr) + return NULL; + + refcount_set(&addr->refcnt, 1); + addr->len = addr_len; + memcpy(addr->name, sunaddr, addr_len); + + return addr; +} + static inline void unix_release_addr(struct unix_address *addr) { if (refcount_dec_and_test(&addr->refcnt)) @@ -227,29 +269,29 @@ static inline void unix_release_addr(struct unix_address *addr) * - if started by zero, it is abstract name. */ -static int unix_mkname(struct sockaddr_un *sunaddr, int len, unsigned int *hashp) +static int unix_validate_addr(struct sockaddr_un *sunaddr, int addr_len) { - *hashp = 0; - - if (len <= sizeof(short) || len > sizeof(*sunaddr)) + if (addr_len <= offsetof(struct sockaddr_un, sun_path) || + addr_len > sizeof(*sunaddr)) return -EINVAL; - if (!sunaddr || sunaddr->sun_family != AF_UNIX) + + if (sunaddr->sun_family != AF_UNIX) return -EINVAL; - if (sunaddr->sun_path[0]) { - /* - * This may look like an off by one error but it is a bit more - * subtle. 108 is the longest valid AF_UNIX path for a binding. - * sun_path[108] doesn't as such exist. However in kernel space - * we are guaranteed that it is a valid memory location in our - * kernel address buffer. - */ - ((char *)sunaddr)[len] = 0; - len = strlen(sunaddr->sun_path)+1+sizeof(short); - return len; - } - *hashp = unix_hash_fold(csum_partial(sunaddr, len, 0)); - return len; + return 0; +} + +static void unix_mkname_bsd(struct sockaddr_un *sunaddr, int addr_len) +{ + /* This may look like an off by one error but it is a bit more + * subtle. 108 is the longest valid AF_UNIX path for a binding. + * sun_path[108] doesn't as such exist. However in kernel space + * we are guaranteed that it is a valid memory location in our + * kernel address buffer because syscall functions always pass + * a pointer of struct sockaddr_storage which has a bigger buffer + * than 108. + */ + ((char *)sunaddr)[addr_len] = 0; } static void __unix_remove_socket(struct sock *sk) @@ -257,32 +299,34 @@ static void __unix_remove_socket(struct sock *sk) sk_del_node_init(sk); } -static void __unix_insert_socket(struct hlist_head *list, struct sock *sk) +static void __unix_insert_socket(struct sock *sk) { WARN_ON(!sk_unhashed(sk)); - sk_add_node(sk, list); + sk_add_node(sk, &unix_socket_table[sk->sk_hash]); } -static void __unix_set_addr(struct sock *sk, struct unix_address *addr, - unsigned hash) +static void __unix_set_addr_hash(struct sock *sk, struct unix_address *addr, + unsigned int hash) { __unix_remove_socket(sk); smp_store_release(&unix_sk(sk)->addr, addr); - __unix_insert_socket(&unix_socket_table[hash], sk); + + sk->sk_hash = hash; + __unix_insert_socket(sk); } -static inline void unix_remove_socket(struct sock *sk) +static void unix_remove_socket(struct sock *sk) { - spin_lock(&unix_table_lock); + spin_lock(&unix_table_locks[sk->sk_hash]); __unix_remove_socket(sk); - spin_unlock(&unix_table_lock); + spin_unlock(&unix_table_locks[sk->sk_hash]); } -static inline void unix_insert_socket(struct hlist_head *list, struct sock *sk) +static void unix_insert_unbound_socket(struct sock *sk) { - spin_lock(&unix_table_lock); - __unix_insert_socket(list, sk); - spin_unlock(&unix_table_lock); + spin_lock(&unix_table_locks[sk->sk_hash]); + __unix_insert_socket(sk); + spin_unlock(&unix_table_locks[sk->sk_hash]); } static struct sock *__unix_find_socket_byname(struct net *net, @@ -310,32 +354,31 @@ static inline struct sock *unix_find_socket_byname(struct net *net, { struct sock *s; - spin_lock(&unix_table_lock); + spin_lock(&unix_table_locks[hash]); s = __unix_find_socket_byname(net, sunname, len, hash); if (s) sock_hold(s); - spin_unlock(&unix_table_lock); + spin_unlock(&unix_table_locks[hash]); return s; } static struct sock *unix_find_socket_byinode(struct inode *i) { + unsigned int hash = unix_bsd_hash(i); struct sock *s; - spin_lock(&unix_table_lock); - sk_for_each(s, - &unix_socket_table[i->i_ino & (UNIX_HASH_SIZE - 1)]) { + spin_lock(&unix_table_locks[hash]); + sk_for_each(s, &unix_socket_table[hash]) { struct dentry *dentry = unix_sk(s)->path.dentry; if (dentry && d_backing_inode(dentry) == i) { sock_hold(s); - goto found; + spin_unlock(&unix_table_locks[hash]); + return s; } } - s = NULL; -found: - spin_unlock(&unix_table_lock); - return s; + spin_unlock(&unix_table_locks[hash]); + return NULL; } /* Support code for asymmetrically connected dgram sockets @@ -522,9 +565,7 @@ static void unix_sock_destructor(struct sock *sk) unix_release_addr(u->addr); atomic_long_dec(&unix_nr_socks); - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); - local_bh_enable(); #ifdef UNIX_REFCNT_DEBUG pr_debug("UNIX %p is destroyed, %ld are still alive.\n", sk, atomic_long_read(&unix_nr_socks)); @@ -872,6 +913,7 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, sock_init_data(sock, sk); + sk->sk_hash = unix_unbound_hash(sk); sk->sk_allocation = GFP_KERNEL_ACCOUNT; sk->sk_write_space = unix_write_space; sk->sk_max_ack_backlog = net->unx.sysctl_max_dgram_qlen; @@ -887,11 +929,9 @@ static struct sock *unix_create1(struct net *net, struct socket *sock, int kern, init_waitqueue_head(&u->peer_wait); init_waitqueue_func_entry(&u->peer_wake, unix_dgram_peer_wake_relay); memset(&u->scm_stat, 0, sizeof(struct scm_stat)); - unix_insert_socket(unix_sockets_unbound(sk), sk); + unix_insert_unbound_socket(sk); - local_bh_disable(); sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); - local_bh_enable(); return sk; @@ -952,15 +992,90 @@ static int unix_release(struct socket *sock) return 0; } -static int unix_autobind(struct socket *sock) +static struct sock *unix_find_bsd(struct net *net, struct sockaddr_un *sunaddr, + int addr_len, int type) { - struct sock *sk = sock->sk; - struct net *net = sock_net(sk); + struct inode *inode; + struct path path; + struct sock *sk; + int err; + + unix_mkname_bsd(sunaddr, addr_len); + err = kern_path(sunaddr->sun_path, LOOKUP_FOLLOW, &path); + if (err) + goto fail; + + err = path_permission(&path, MAY_WRITE); + if (err) + goto path_put; + + err = -ECONNREFUSED; + inode = d_backing_inode(path.dentry); + if (!S_ISSOCK(inode->i_mode)) + goto path_put; + + sk = unix_find_socket_byinode(inode); + if (!sk) + goto path_put; + + err = -EPROTOTYPE; + if (sk->sk_type == type) + touch_atime(&path); + else + goto sock_put; + + path_put(&path); + + return sk; + +sock_put: + sock_put(sk); +path_put: + path_put(&path); +fail: + return ERR_PTR(err); +} + +static struct sock *unix_find_abstract(struct net *net, + struct sockaddr_un *sunaddr, + int addr_len, int type) +{ + unsigned int hash = unix_abstract_hash(sunaddr, addr_len, type); + struct dentry *dentry; + struct sock *sk; + + sk = unix_find_socket_byname(net, sunaddr, addr_len, hash); + if (!sk) + return ERR_PTR(-ECONNREFUSED); + + dentry = unix_sk(sk)->path.dentry; + if (dentry) + touch_atime(&unix_sk(sk)->path); + + return sk; +} + +static struct sock *unix_find_other(struct net *net, + struct sockaddr_un *sunaddr, + int addr_len, int type) +{ + struct sock *sk; + + if (sunaddr->sun_path[0]) + sk = unix_find_bsd(net, sunaddr, addr_len, type); + else + sk = unix_find_abstract(net, sunaddr, addr_len, type); + + return sk; +} + +static int unix_autobind(struct sock *sk) +{ + unsigned int new_hash, old_hash = sk->sk_hash; struct unix_sock *u = unix_sk(sk); - static u32 ordernum = 1; struct unix_address *addr; + u32 lastnum, ordernum; int err; - unsigned int retries = 0; err = mutex_lock_interruptible(&u->bindlock); if (err) @@ -970,141 +1085,103 @@ static int unix_autobind(struct socket *sock) goto out; err = -ENOMEM; - addr = kzalloc(sizeof(*addr) + sizeof(short) + 16, GFP_KERNEL); + addr = kzalloc(sizeof(*addr) + + offsetof(struct sockaddr_un, sun_path) + 16, GFP_KERNEL); if (!addr) goto out; + addr->len = offsetof(struct sockaddr_un, sun_path) + 6; addr->name->sun_family = AF_UNIX; refcount_set(&addr->refcnt, 1); + ordernum = prandom_u32(); + lastnum = ordernum & 0xFFFFF; retry: - addr->len = sprintf(addr->name->sun_path+1, "%05x", ordernum) + 1 + sizeof(short); - addr->hash = unix_hash_fold(csum_partial(addr->name, addr->len, 0)); - addr->hash ^= sk->sk_type; + ordernum = (ordernum + 1) & 0xFFFFF; + sprintf(addr->name->sun_path + 1, "%05x", ordernum); - spin_lock(&unix_table_lock); - ordernum = (ordernum+1)&0xFFFFF; + new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type); + unix_table_double_lock(old_hash, new_hash); - if (__unix_find_socket_byname(net, addr->name, addr->len, addr->hash)) { - spin_unlock(&unix_table_lock); - /* - * __unix_find_socket_byname() may take long time if many names + if (__unix_find_socket_byname(sock_net(sk), addr->name, addr->len, + new_hash)) { + unix_table_double_unlock(old_hash, new_hash); + + /* __unix_find_socket_byname() may take long time if many names * are already in use. */ cond_resched(); - /* Give up if all names seems to be in use. */ - if (retries++ == 0xFFFFF) { + + if (ordernum == lastnum) { + /* Give up if all names seems to be in use. */ err = -ENOSPC; - kfree(addr); + unix_release_addr(addr); goto out; } + goto retry; } - __unix_set_addr(sk, addr, addr->hash); - spin_unlock(&unix_table_lock); + __unix_set_addr_hash(sk, addr, new_hash); + unix_table_double_unlock(old_hash, new_hash); err = 0; out: mutex_unlock(&u->bindlock); return err; } -static struct sock *unix_find_other(struct net *net, - struct sockaddr_un *sunname, int len, - int type, unsigned int hash, int *error) -{ - struct sock *u; - struct path path; - int err = 0; - - if (sunname->sun_path[0]) { - struct inode *inode; - err = kern_path(sunname->sun_path, LOOKUP_FOLLOW, &path); - if (err) - goto fail; - inode = d_backing_inode(path.dentry); - err = path_permission(&path, MAY_WRITE); - if (err) - goto put_fail; - - err = -ECONNREFUSED; - if (!S_ISSOCK(inode->i_mode)) - goto put_fail; - u = unix_find_socket_byinode(inode); - if (!u) - goto put_fail; - - if (u->sk_type == type) - touch_atime(&path); - - path_put(&path); - - err = -EPROTOTYPE; - if (u->sk_type != type) { - sock_put(u); - goto fail; - } - } else { - err = -ECONNREFUSED; - u = unix_find_socket_byname(net, sunname, len, type ^ hash); - if (u) { - struct dentry *dentry; - dentry = unix_sk(u)->path.dentry; - if (dentry) - touch_atime(&unix_sk(u)->path); - } else - goto fail; - } - return u; - -put_fail: - path_put(&path); -fail: - *error = err; - return NULL; -} - -static int unix_bind_bsd(struct sock *sk, struct unix_address *addr) +static int unix_bind_bsd(struct sock *sk, struct sockaddr_un *sunaddr, + int addr_len) { - struct unix_sock *u = unix_sk(sk); umode_t mode = S_IFSOCK | (SOCK_INODE(sk->sk_socket)->i_mode & ~current_umask()); + unsigned int new_hash, old_hash = sk->sk_hash; + struct unix_sock *u = unix_sk(sk); struct user_namespace *ns; // barf... - struct path parent; + struct unix_address *addr; struct dentry *dentry; - unsigned int hash; + struct path parent; int err; + unix_mkname_bsd(sunaddr, addr_len); + addr_len = strlen(sunaddr->sun_path) + + offsetof(struct sockaddr_un, sun_path) + 1; + + addr = unix_create_addr(sunaddr, addr_len); + if (!addr) + return -ENOMEM; + /* * Get the parent directory, calculate the hash for last * component. */ dentry = kern_path_create(AT_FDCWD, addr->name->sun_path, &parent, 0); - if (IS_ERR(dentry)) - return PTR_ERR(dentry); - ns = mnt_user_ns(parent.mnt); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto out; + } /* * All right, let's create it. */ + ns = mnt_user_ns(parent.mnt); err = security_path_mknod(&parent, dentry, mode, 0); if (!err) err = vfs_mknod(ns, d_inode(parent.dentry), dentry, mode, 0); if (err) - goto out; + goto out_path; err = mutex_lock_interruptible(&u->bindlock); if (err) goto out_unlink; if (u->addr) goto out_unlock; - addr->hash = UNIX_HASH_SIZE; - hash = d_backing_inode(dentry)->i_ino & (UNIX_HASH_SIZE - 1); - spin_lock(&unix_table_lock); + new_hash = unix_bsd_hash(d_backing_inode(dentry)); + unix_table_double_lock(old_hash, new_hash); u->path.mnt = mntget(parent.mnt); u->path.dentry = dget(dentry); - __unix_set_addr(sk, addr, hash); - spin_unlock(&unix_table_lock); + __unix_set_addr_hash(sk, addr, new_hash); + unix_table_double_unlock(old_hash, new_hash); mutex_unlock(&u->bindlock); done_path_create(&parent, dentry); return 0; @@ -1115,74 +1192,76 @@ out_unlock: out_unlink: /* failed after successful mknod? unlink what we'd created... */ vfs_unlink(ns, d_inode(parent.dentry), dentry, NULL); -out: +out_path: done_path_create(&parent, dentry); - return err; +out: + unix_release_addr(addr); + return err == -EEXIST ? -EADDRINUSE : err; } -static int unix_bind_abstract(struct sock *sk, struct unix_address *addr) +static int unix_bind_abstract(struct sock *sk, struct sockaddr_un *sunaddr, + int addr_len) { + unsigned int new_hash, old_hash = sk->sk_hash; struct unix_sock *u = unix_sk(sk); + struct unix_address *addr; int err; + addr = unix_create_addr(sunaddr, addr_len); + if (!addr) + return -ENOMEM; + err = mutex_lock_interruptible(&u->bindlock); if (err) - return err; + goto out; if (u->addr) { - mutex_unlock(&u->bindlock); - return -EINVAL; + err = -EINVAL; + goto out_mutex; } - spin_lock(&unix_table_lock); + new_hash = unix_abstract_hash(addr->name, addr->len, sk->sk_type); + unix_table_double_lock(old_hash, new_hash); + if (__unix_find_socket_byname(sock_net(sk), addr->name, addr->len, - addr->hash)) { - spin_unlock(&unix_table_lock); - mutex_unlock(&u->bindlock); - return -EADDRINUSE; - } - __unix_set_addr(sk, addr, addr->hash); - spin_unlock(&unix_table_lock); + new_hash)) + goto out_spin; + + __unix_set_addr_hash(sk, addr, new_hash); + unix_table_double_unlock(old_hash, new_hash); mutex_unlock(&u->bindlock); return 0; + +out_spin: + unix_table_double_unlock(old_hash, new_hash); + err = -EADDRINUSE; +out_mutex: + mutex_unlock(&u->bindlock); +out: + unix_release_addr(addr); + return err; } static int unix_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) { - struct sock *sk = sock->sk; struct sockaddr_un *sunaddr = (struct sockaddr_un *)uaddr; - char *sun_path = sunaddr->sun_path; + struct sock *sk = sock->sk; int err; - unsigned int hash; - struct unix_address *addr; - - if (addr_len < offsetofend(struct sockaddr_un, sun_family) || - sunaddr->sun_family != AF_UNIX) - return -EINVAL; - if (addr_len == sizeof(short)) - return unix_autobind(sock); + if (addr_len == offsetof(struct sockaddr_un, sun_path) && + sunaddr->sun_family == AF_UNIX) + return unix_autobind(sk); - err = unix_mkname(sunaddr, addr_len, &hash); - if (err < 0) + err = unix_validate_addr(sunaddr, addr_len); + if (err) return err; - addr_len = err; - addr = kmalloc(sizeof(*addr)+addr_len, GFP_KERNEL); - if (!addr) - return -ENOMEM; - memcpy(addr->name, sunaddr, addr_len); - addr->len = addr_len; - addr->hash = hash ^ sk->sk_type; - refcount_set(&addr->refcnt, 1); - - if (sun_path[0]) - err = unix_bind_bsd(sk, addr); + if (sunaddr->sun_path[0]) + err = unix_bind_bsd(sk, sunaddr, addr_len); else - err = unix_bind_abstract(sk, addr); - if (err) - unix_release_addr(addr); - return err == -EEXIST ? -EADDRINUSE : err; + err = unix_bind_abstract(sk, sunaddr, addr_len); + + return err; } static void unix_state_double_lock(struct sock *sk1, struct sock *sk2) @@ -1217,7 +1296,6 @@ static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr, struct net *net = sock_net(sk); struct sockaddr_un *sunaddr = (struct sockaddr_un *)addr; struct sock *other; - unsigned int hash; int err; err = -EINVAL; @@ -1225,19 +1303,23 @@ static int unix_dgram_connect(struct socket *sock, struct sockaddr *addr, goto out; if (addr->sa_family != AF_UNSPEC) { - err = unix_mkname(sunaddr, alen, &hash); - if (err < 0) + err = unix_validate_addr(sunaddr, alen); + if (err) goto out; - alen = err; if (test_bit(SOCK_PASSCRED, &sock->flags) && - !unix_sk(sk)->addr && (err = unix_autobind(sock)) != 0) - goto out; + !unix_sk(sk)->addr) { + err = unix_autobind(sk); + if (err) + goto out; + } restart: - other = unix_find_other(net, sunaddr, alen, sock->type, hash, &err); - if (!other) + other = unix_find_other(net, sunaddr, alen, sock->type); + if (IS_ERR(other)) { + err = PTR_ERR(other); goto out; + } unix_state_double_lock(sk, other); @@ -1327,19 +1409,19 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, struct sock *newsk = NULL; struct sock *other = NULL; struct sk_buff *skb = NULL; - unsigned int hash; int st; int err; long timeo; - err = unix_mkname(sunaddr, addr_len, &hash); - if (err < 0) + err = unix_validate_addr(sunaddr, addr_len); + if (err) goto out; - addr_len = err; - if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr && - (err = unix_autobind(sock)) != 0) - goto out; + if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr) { + err = unix_autobind(sk); + if (err) + goto out; + } timeo = sock_sndtimeo(sk, flags & O_NONBLOCK); @@ -1365,9 +1447,12 @@ static int unix_stream_connect(struct socket *sock, struct sockaddr *uaddr, restart: /* Find listening sock. */ - other = unix_find_other(net, sunaddr, addr_len, sk->sk_type, hash, &err); - if (!other) + other = unix_find_other(net, sunaddr, addr_len, sk->sk_type); + if (IS_ERR(other)) { + err = PTR_ERR(other); + other = NULL; goto out; + } /* Latch state of peer */ unix_state_lock(other); @@ -1455,9 +1540,9 @@ restart: * * The contents of *(otheru->addr) and otheru->path * are seen fully set up here, since we have found - * otheru in hash under unix_table_lock. Insertion + * otheru in hash under unix_table_locks. Insertion * into the hash chain we'd found it in had been done - * in an earlier critical area protected by unix_table_lock, + * in an earlier critical area protected by unix_table_locks, * the same one where we'd set *(otheru->addr) contents, * as well as otheru->path and otheru->addr itself. * @@ -1604,7 +1689,7 @@ static int unix_getname(struct socket *sock, struct sockaddr *uaddr, int peer) if (!addr) { sunaddr->sun_family = AF_UNIX; sunaddr->sun_path[0] = 0; - err = sizeof(short); + err = offsetof(struct sockaddr_un, sun_path); } else { err = addr->len; memcpy(sunaddr, addr->name, addr->len); @@ -1760,9 +1845,7 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, struct unix_sock *u = unix_sk(sk); DECLARE_SOCKADDR(struct sockaddr_un *, sunaddr, msg->msg_name); struct sock *other = NULL; - int namelen = 0; /* fake GCC */ int err; - unsigned int hash; struct sk_buff *skb; long timeo; struct scm_cookie scm; @@ -1779,10 +1862,9 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, goto out; if (msg->msg_namelen) { - err = unix_mkname(sunaddr, msg->msg_namelen, &hash); - if (err < 0) + err = unix_validate_addr(sunaddr, msg->msg_namelen); + if (err) goto out; - namelen = err; } else { sunaddr = NULL; err = -ENOTCONN; @@ -1791,9 +1873,11 @@ static int unix_dgram_sendmsg(struct socket *sock, struct msghdr *msg, goto out; } - if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr - && (err = unix_autobind(sock)) != 0) - goto out; + if (test_bit(SOCK_PASSCRED, &sock->flags) && !u->addr) { + err = unix_autobind(sk); + if (err) + goto out; + } err = -EMSGSIZE; if (len > sk->sk_sndbuf - 32) @@ -1833,10 +1917,13 @@ restart: if (sunaddr == NULL) goto out_free; - other = unix_find_other(net, sunaddr, namelen, sk->sk_type, - hash, &err); - if (other == NULL) + other = unix_find_other(net, sunaddr, msg->msg_namelen, + sk->sk_type); + if (IS_ERR(other)) { + err = PTR_ERR(other); + other = NULL; goto out_free; + } } if (sk_filter(other, skb) < 0) { @@ -3132,7 +3219,7 @@ static __poll_t unix_dgram_poll(struct file *file, struct socket *sock, #define BUCKET_SPACE (BITS_PER_LONG - (UNIX_HASH_BITS + 1) - 1) #define get_bucket(x) ((x) >> BUCKET_SPACE) -#define get_offset(x) ((x) & ((1L << BUCKET_SPACE) - 1)) +#define get_offset(x) ((x) & ((1UL << BUCKET_SPACE) - 1)) #define set_bucket_offset(b, o) ((b) << BUCKET_SPACE | (o)) static struct sock *unix_from_bucket(struct seq_file *seq, loff_t *pos) @@ -3156,7 +3243,7 @@ static struct sock *unix_next_socket(struct seq_file *seq, struct sock *sk, loff_t *pos) { - unsigned long bucket; + unsigned long bucket = get_bucket(*pos); while (sk > (struct sock *)SEQ_START_TOKEN) { sk = sk_next(sk); @@ -3167,12 +3254,13 @@ static struct sock *unix_next_socket(struct seq_file *seq, } do { + spin_lock(&unix_table_locks[bucket]); sk = unix_from_bucket(seq, pos); if (sk) return sk; next_bucket: - bucket = get_bucket(*pos) + 1; + spin_unlock(&unix_table_locks[bucket++]); *pos = set_bucket_offset(bucket, 1); } while (bucket < ARRAY_SIZE(unix_socket_table)); @@ -3180,10 +3268,7 @@ next_bucket: } static void *unix_seq_start(struct seq_file *seq, loff_t *pos) - __acquires(unix_table_lock) { - spin_lock(&unix_table_lock); - if (!*pos) return SEQ_START_TOKEN; @@ -3200,9 +3285,11 @@ static void *unix_seq_next(struct seq_file *seq, void *v, loff_t *pos) } static void unix_seq_stop(struct seq_file *seq, void *v) - __releases(unix_table_lock) { - spin_unlock(&unix_table_lock); + struct sock *sk = v; + + if (sk) + spin_unlock(&unix_table_locks[sk->sk_hash]); } static int unix_seq_show(struct seq_file *seq, void *v) @@ -3227,15 +3314,16 @@ static int unix_seq_show(struct seq_file *seq, void *v) (s->sk_state == TCP_ESTABLISHED ? SS_CONNECTING : SS_DISCONNECTING), sock_i_ino(s)); - if (u->addr) { // under unix_table_lock here + if (u->addr) { // under unix_table_locks here int i, len; seq_putc(seq, ' '); i = 0; - len = u->addr->len - sizeof(short); - if (!UNIX_ABSTRACT(s)) + len = u->addr->len - + offsetof(struct sockaddr_un, sun_path); + if (u->addr->name->sun_path[0]) { len--; - else { + } else { seq_putc(seq, '@'); i++; } @@ -3385,10 +3473,13 @@ static void __init bpf_iter_register(void) static int __init af_unix_init(void) { - int rc = -1; + int i, rc = -1; BUILD_BUG_ON(sizeof(struct unix_skb_parms) > sizeof_field(struct sk_buff, cb)); + for (i = 0; i < 2 * UNIX_HASH_SIZE; i++) + spin_lock_init(&unix_table_locks[i]); + rc = proto_register(&unix_dgram_proto, 1); if (rc != 0) { pr_crit("%s: Cannot create unix_sock SLAB cache!\n", __func__); diff --git a/net/unix/diag.c b/net/unix/diag.c index 7e7d7f45685a..bb0b5ea1655f 100644 --- a/net/unix/diag.c +++ b/net/unix/diag.c @@ -13,13 +13,14 @@ static int sk_diag_dump_name(struct sock *sk, struct sk_buff *nlskb) { - /* might or might not have unix_table_lock */ + /* might or might not have unix_table_locks */ struct unix_address *addr = smp_load_acquire(&unix_sk(sk)->addr); if (!addr) return 0; - return nla_put(nlskb, UNIX_DIAG_NAME, addr->len - sizeof(short), + return nla_put(nlskb, UNIX_DIAG_NAME, + addr->len - offsetof(struct sockaddr_un, sun_path), addr->name->sun_path); } @@ -203,13 +204,13 @@ static int unix_diag_dump(struct sk_buff *skb, struct netlink_callback *cb) s_slot = cb->args[0]; num = s_num = cb->args[1]; - spin_lock(&unix_table_lock); for (slot = s_slot; slot < ARRAY_SIZE(unix_socket_table); s_num = 0, slot++) { struct sock *sk; num = 0; + spin_lock(&unix_table_locks[slot]); sk_for_each(sk, &unix_socket_table[slot]) { if (!net_eq(sock_net(sk), net)) continue; @@ -220,14 +221,16 @@ static int unix_diag_dump(struct sk_buff *skb, struct netlink_callback *cb) if (sk_diag_dump(sk, skb, req, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, - NLM_F_MULTI) < 0) + NLM_F_MULTI) < 0) { + spin_unlock(&unix_table_locks[slot]); goto done; + } next: num++; } + spin_unlock(&unix_table_locks[slot]); } done: - spin_unlock(&unix_table_lock); cb->args[0] = slot; cb->args[1] = num; @@ -236,21 +239,19 @@ done: static struct sock *unix_lookup_by_ino(unsigned int ino) { - int i; struct sock *sk; + int i; - spin_lock(&unix_table_lock); for (i = 0; i < ARRAY_SIZE(unix_socket_table); i++) { + spin_lock(&unix_table_locks[i]); sk_for_each(sk, &unix_socket_table[i]) if (ino == sock_i_ino(sk)) { sock_hold(sk); - spin_unlock(&unix_table_lock); - + spin_unlock(&unix_table_locks[i]); return sk; } + spin_unlock(&unix_table_locks[i]); } - - spin_unlock(&unix_table_lock); return NULL; } diff --git a/net/vmw_vsock/hyperv_transport.c b/net/vmw_vsock/hyperv_transport.c index 19189cf30a72..e111e13b6660 100644 --- a/net/vmw_vsock/hyperv_transport.c +++ b/net/vmw_vsock/hyperv_transport.c @@ -225,14 +225,20 @@ static size_t hvs_channel_writable_bytes(struct vmbus_channel *chan) return round_down(ret, 8); } +static int __hvs_send_data(struct vmbus_channel *chan, + struct vmpipe_proto_header *hdr, + size_t to_write) +{ + hdr->pkt_type = 1; + hdr->data_size = to_write; + return vmbus_sendpacket(chan, hdr, sizeof(*hdr) + to_write, + 0, VM_PKT_DATA_INBAND, 0); +} + static int hvs_send_data(struct vmbus_channel *chan, struct hvs_send_buf *send_buf, size_t to_write) { - send_buf->hdr.pkt_type = 1; - send_buf->hdr.data_size = to_write; - return vmbus_sendpacket(chan, &send_buf->hdr, - sizeof(send_buf->hdr) + to_write, - 0, VM_PKT_DATA_INBAND, 0); + return __hvs_send_data(chan, &send_buf->hdr, to_write); } static void hvs_channel_cb(void *ctx) @@ -468,7 +474,7 @@ static void hvs_shutdown_lock_held(struct hvsock *hvs, int mode) return; /* It can't fail: see hvs_channel_writable_bytes(). */ - (void)hvs_send_data(hvs->chan, (struct hvs_send_buf *)&hdr, 0); + (void)__hvs_send_data(hvs->chan, &hdr, 0); hvs->fin_sent = true; } diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c index f16074eb53c7..28ef3f4465ae 100644 --- a/net/xdp/xsk.c +++ b/net/xdp/xsk.c @@ -794,9 +794,7 @@ static int xsk_release(struct socket *sock) sk_del_node_init_rcu(sk); mutex_unlock(&net->xdp.lock); - local_bh_disable(); sock_prot_inuse_add(net, sk->sk_prot, -1); - local_bh_enable(); xsk_delete_from_maps(xs); mutex_lock(&xs->mutex); @@ -1396,9 +1394,7 @@ static int xsk_create(struct net *net, struct socket *sock, int protocol, sk_add_node_rcu(sk, &net->xdp.list); mutex_unlock(&net->xdp.lock); - local_bh_disable(); sock_prot_inuse_add(net, &xsk_proto, 1); - local_bh_enable(); return 0; } diff --git a/tools/bpf/bpftool/.gitignore b/tools/bpf/bpftool/.gitignore index 05ce4446b780..a736f64dc5dc 100644 --- a/tools/bpf/bpftool/.gitignore +++ b/tools/bpf/bpftool/.gitignore @@ -1,4 +1,4 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) *.d /bootstrap/ /bpftool diff --git a/tools/bpf/bpftool/Documentation/Makefile b/tools/bpf/bpftool/Documentation/Makefile index c49487905ceb..692e1b947490 100644 --- a/tools/bpf/bpftool/Documentation/Makefile +++ b/tools/bpf/bpftool/Documentation/Makefile @@ -1,6 +1,5 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) include ../../../scripts/Makefile.include -include ../../../scripts/utilities.mak INSTALL ?= install RM ?= rm -f diff --git a/tools/bpf/bpftool/Documentation/bpftool-btf.rst b/tools/bpf/bpftool/Documentation/bpftool-btf.rst index 88b28aa7431f..4425d942dd39 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-btf.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-btf.rst @@ -13,7 +13,7 @@ SYNOPSIS **bpftool** [*OPTIONS*] **btf** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | {**-d** | **--debug** } | - { **-B** | **--base-btf** } } + { **-B** | **--base-btf** } } *COMMANDS* := { **dump** | **help** } diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst index 3e4395eede4f..8069d37dd991 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst @@ -13,7 +13,7 @@ SYNOPSIS **bpftool** [*OPTIONS*] **cgroup** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } | - { **-f** | **--bpffs** } } + { **-f** | **--bpffs** } } *COMMANDS* := { **show** | **list** | **tree** | **attach** | **detach** | **help** } @@ -30,9 +30,9 @@ CGROUP COMMANDS | *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* } | *ATTACH_TYPE* := { **ingress** | **egress** | **sock_create** | **sock_ops** | **device** | | **bind4** | **bind6** | **post_bind4** | **post_bind6** | **connect4** | **connect6** | -| **getpeername4** | **getpeername6** | **getsockname4** | **getsockname6** | **sendmsg4** | -| **sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** | **getsockopt** | **setsockopt** | -| **sock_release** } +| **getpeername4** | **getpeername6** | **getsockname4** | **getsockname6** | **sendmsg4** | +| **sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** | **getsockopt** | **setsockopt** | +| **sock_release** } | *ATTACH_FLAGS* := { **multi** | **override** } DESCRIPTION @@ -98,9 +98,9 @@ DESCRIPTION **sendmsg6** call to sendto(2), sendmsg(2), sendmmsg(2) for an unconnected udp6 socket (since 4.18); **recvmsg4** call to recvfrom(2), recvmsg(2), recvmmsg(2) for - an unconnected udp4 socket (since 5.2); + an unconnected udp4 socket (since 5.2); **recvmsg6** call to recvfrom(2), recvmsg(2), recvmmsg(2) for - an unconnected udp6 socket (since 5.2); + an unconnected udp6 socket (since 5.2); **sysctl** sysctl access (since 5.2); **getsockopt** call to getsockopt (since 5.3); **setsockopt** call to setsockopt (since 5.3); diff --git a/tools/bpf/bpftool/Documentation/bpftool-gen.rst b/tools/bpf/bpftool/Documentation/bpftool-gen.rst index 2ef2f2df0279..2a137f8a4cea 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-gen.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-gen.rst @@ -13,7 +13,7 @@ SYNOPSIS **bpftool** [*OPTIONS*] **gen** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } | - { **-L** | **--use-loader** } } + { **-L** | **--use-loader** } } *COMMAND* := { **object** | **skeleton** | **help** } diff --git a/tools/bpf/bpftool/Documentation/bpftool-link.rst b/tools/bpf/bpftool/Documentation/bpftool-link.rst index 0de90f086238..9434349636a5 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-link.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-link.rst @@ -13,7 +13,7 @@ SYNOPSIS **bpftool** [*OPTIONS*] **link** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } | - { **-f** | **--bpffs** } | { **-n** | **--nomount** } } + { **-f** | **--bpffs** } | { **-n** | **--nomount** } } *COMMANDS* := { **show** | **list** | **pin** | **help** } diff --git a/tools/bpf/bpftool/Documentation/bpftool-map.rst b/tools/bpf/bpftool/Documentation/bpftool-map.rst index d0c4abe08aba..991d18fd84f2 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-map.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-map.rst @@ -13,11 +13,11 @@ SYNOPSIS **bpftool** [*OPTIONS*] **map** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } | - { **-f** | **--bpffs** } | { **-n** | **--nomount** } } + { **-f** | **--bpffs** } | { **-n** | **--nomount** } } *COMMANDS* := - { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext** - | **delete** | **pin** | **help** } + { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext** | + **delete** | **pin** | **help** } MAP COMMANDS ============= @@ -52,7 +52,7 @@ MAP COMMANDS | | **devmap** | **devmap_hash** | **sockmap** | **cpumap** | **xskmap** | **sockhash** | | **cgroup_storage** | **reuseport_sockarray** | **percpu_cgroup_storage** | | **queue** | **stack** | **sk_storage** | **struct_ops** | **ringbuf** | **inode_storage** - | **task_storage** } +| | **task_storage** | **bloom_filter** } DESCRIPTION =========== diff --git a/tools/bpf/bpftool/Documentation/bpftool-net.rst b/tools/bpf/bpftool/Documentation/bpftool-net.rst index 1ae0375e8fea..7ec57535a7c1 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-net.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-net.rst @@ -31,44 +31,44 @@ NET COMMANDS DESCRIPTION =========== **bpftool net { show | list }** [ **dev** *NAME* ] - List bpf program attachments in the kernel networking subsystem. - - Currently, only device driver xdp attachments and tc filter - classification/action attachments are implemented, i.e., for - program types **BPF_PROG_TYPE_SCHED_CLS**, - **BPF_PROG_TYPE_SCHED_ACT** and **BPF_PROG_TYPE_XDP**. - For programs attached to a particular cgroup, e.g., - **BPF_PROG_TYPE_CGROUP_SKB**, **BPF_PROG_TYPE_CGROUP_SOCK**, - **BPF_PROG_TYPE_SOCK_OPS** and **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, - users can use **bpftool cgroup** to dump cgroup attachments. - For sk_{filter, skb, msg, reuseport} and lwt/seg6 - bpf programs, users should consult other tools, e.g., iproute2. - - The current output will start with all xdp program attachments, followed by - all tc class/qdisc bpf program attachments. Both xdp programs and - tc programs are ordered based on ifindex number. If multiple bpf - programs attached to the same networking device through **tc filter**, - the order will be first all bpf programs attached to tc classes, then - all bpf programs attached to non clsact qdiscs, and finally all - bpf programs attached to root and clsact qdisc. + List bpf program attachments in the kernel networking subsystem. + + Currently, only device driver xdp attachments and tc filter + classification/action attachments are implemented, i.e., for + program types **BPF_PROG_TYPE_SCHED_CLS**, + **BPF_PROG_TYPE_SCHED_ACT** and **BPF_PROG_TYPE_XDP**. + For programs attached to a particular cgroup, e.g., + **BPF_PROG_TYPE_CGROUP_SKB**, **BPF_PROG_TYPE_CGROUP_SOCK**, + **BPF_PROG_TYPE_SOCK_OPS** and **BPF_PROG_TYPE_CGROUP_SOCK_ADDR**, + users can use **bpftool cgroup** to dump cgroup attachments. + For sk_{filter, skb, msg, reuseport} and lwt/seg6 + bpf programs, users should consult other tools, e.g., iproute2. + + The current output will start with all xdp program attachments, followed by + all tc class/qdisc bpf program attachments. Both xdp programs and + tc programs are ordered based on ifindex number. If multiple bpf + programs attached to the same networking device through **tc filter**, + the order will be first all bpf programs attached to tc classes, then + all bpf programs attached to non clsact qdiscs, and finally all + bpf programs attached to root and clsact qdisc. **bpftool** **net attach** *ATTACH_TYPE* *PROG* **dev** *NAME* [ **overwrite** ] - Attach bpf program *PROG* to network interface *NAME* with - type specified by *ATTACH_TYPE*. Previously attached bpf program - can be replaced by the command used with **overwrite** option. - Currently, only XDP-related modes are supported for *ATTACH_TYPE*. + Attach bpf program *PROG* to network interface *NAME* with + type specified by *ATTACH_TYPE*. Previously attached bpf program + can be replaced by the command used with **overwrite** option. + Currently, only XDP-related modes are supported for *ATTACH_TYPE*. - *ATTACH_TYPE* can be of: - **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it; - **xdpgeneric** - Generic XDP. runs at generic XDP hook when packet already enters receive path as skb; - **xdpdrv** - Native XDP. runs earliest point in driver's receive path; - **xdpoffload** - Offload XDP. runs directly on NIC on each packet reception; + *ATTACH_TYPE* can be of: + **xdp** - try native XDP and fallback to generic XDP if NIC driver does not support it; + **xdpgeneric** - Generic XDP. runs at generic XDP hook when packet already enters receive path as skb; + **xdpdrv** - Native XDP. runs earliest point in driver's receive path; + **xdpoffload** - Offload XDP. runs directly on NIC on each packet reception; **bpftool** **net detach** *ATTACH_TYPE* **dev** *NAME* - Detach bpf program attached to network interface *NAME* with - type specified by *ATTACH_TYPE*. To detach bpf program, same - *ATTACH_TYPE* previously used for attach must be specified. - Currently, only XDP-related modes are supported for *ATTACH_TYPE*. + Detach bpf program attached to network interface *NAME* with + type specified by *ATTACH_TYPE*. To detach bpf program, same + *ATTACH_TYPE* previously used for attach must be specified. + Currently, only XDP-related modes are supported for *ATTACH_TYPE*. **bpftool net help** Print short help message. diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index 91608cb7e44a..f27265bd589b 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -13,12 +13,12 @@ SYNOPSIS **bpftool** [*OPTIONS*] **prog** *COMMAND* *OPTIONS* := { { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } | - { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } | - { **-L** | **--use-loader** } } + { **-f** | **--bpffs** } | { **-m** | **--mapcompat** } | { **-n** | **--nomount** } | + { **-L** | **--use-loader** } } *COMMANDS* := - { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** - | **loadall** | **help** } + { **show** | **list** | **dump xlated** | **dump jited** | **pin** | **load** | + **loadall** | **help** } PROG COMMANDS ============= diff --git a/tools/bpf/bpftool/Documentation/bpftool.rst b/tools/bpf/bpftool/Documentation/bpftool.rst index bb23f55bb05a..8ac86565c501 100644 --- a/tools/bpf/bpftool/Documentation/bpftool.rst +++ b/tools/bpf/bpftool/Documentation/bpftool.rst @@ -19,14 +19,14 @@ SYNOPSIS *OBJECT* := { **map** | **program** | **cgroup** | **perf** | **net** | **feature** } *OPTIONS* := { { **-V** | **--version** } | - { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } } + { **-j** | **--json** } [{ **-p** | **--pretty** }] | { **-d** | **--debug** } } *MAP-COMMANDS* := { **show** | **list** | **create** | **dump** | **update** | **lookup** | **getnext** | - **delete** | **pin** | **event_pipe** | **help** } + **delete** | **pin** | **event_pipe** | **help** } *PROG-COMMANDS* := { **show** | **list** | **dump jited** | **dump xlated** | **pin** | - **load** | **attach** | **detach** | **help** } + **load** | **attach** | **detach** | **help** } *CGROUP-COMMANDS* := { **show** | **list** | **attach** | **detach** | **help** } diff --git a/tools/bpf/bpftool/Documentation/common_options.rst b/tools/bpf/bpftool/Documentation/common_options.rst index 05d06c74dcaa..75adf23202d8 100644 --- a/tools/bpf/bpftool/Documentation/common_options.rst +++ b/tools/bpf/bpftool/Documentation/common_options.rst @@ -20,3 +20,12 @@ Print all logs available, even debug-level information. This includes logs from libbpf as well as from the verifier, when attempting to load programs. + +-l, --legacy + Use legacy libbpf mode which has more relaxed BPF program + requirements. By default, bpftool has more strict requirements + about section names, changes pinning logic and doesn't support + some of the older non-BTF map declarations. + + See https://github.com/libbpf/libbpf/wiki/Libbpf:-the-road-to-v1.0 + for details. diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index 7cfba11c3014..42eb8eee3d89 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -1,6 +1,5 @@ -# SPDX-License-Identifier: GPL-2.0-only +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) include ../../scripts/Makefile.include -include ../../scripts/utilities.mak ifeq ($(srctree),) srctree := $(patsubst %/,%,$(dir $(CURDIR))) @@ -187,7 +186,8 @@ $(OUTPUT)%.bpf.o: skeleton/%.bpf.c $(OUTPUT)vmlinux.h $(LIBBPF_BOOTSTRAP) -I$(if $(OUTPUT),$(OUTPUT),.) \ -I$(srctree)/tools/include/uapi/ \ -I$(LIBBPF_BOOTSTRAP_INCLUDE) \ - -g -O2 -Wall -target bpf -c $< -o $@ && $(LLVM_STRIP) -g $@ + -g -O2 -Wall -target bpf -c $< -o $@ + $(Q)$(LLVM_STRIP) -g $@ $(OUTPUT)%.skel.h: $(OUTPUT)%.bpf.o $(BPFTOOL_BOOTSTRAP) $(QUIET_GEN)$(BPFTOOL_BOOTSTRAP) gen skeleton $< > $@ @@ -202,10 +202,10 @@ endif CFLAGS += $(if $(BUILD_BPF_SKELS),,-DBPFTOOL_WITHOUT_SKELETONS) $(BOOTSTRAP_OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c - $(QUIET_CC)$(HOSTCC) $(CFLAGS) -c -MMD -o $@ $< + $(QUIET_CC)$(HOSTCC) $(CFLAGS) -c -MMD $< -o $@ $(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -MMD -o $@ $< + $(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@ $(OUTPUT)feature.o: ifneq ($(feature-zlib), 1) @@ -213,19 +213,18 @@ ifneq ($(feature-zlib), 1) endif $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP) - $(QUIET_LINK)$(HOSTCC) $(CFLAGS) $(LDFLAGS) -o $@ $(BOOTSTRAP_OBJS) \ - $(LIBS_BOOTSTRAP) + $(QUIET_LINK)$(HOSTCC) $(CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@ $(OUTPUT)bpftool: $(OBJS) $(LIBBPF) - $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) -o $@ $(OBJS) $(LIBS) + $(QUIET_LINK)$(CC) $(CFLAGS) $(LDFLAGS) $(OBJS) $(LIBS) -o $@ $(BOOTSTRAP_OUTPUT)%.o: %.c $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS) | $(BOOTSTRAP_OUTPUT) $(QUIET_CC)$(HOSTCC) \ $(subst -I$(LIBBPF_INCLUDE),-I$(LIBBPF_BOOTSTRAP_INCLUDE),$(CFLAGS)) \ - -c -MMD -o $@ $< + -c -MMD $< -o $@ $(OUTPUT)%.o: %.c - $(QUIET_CC)$(CC) $(CFLAGS) -c -MMD -o $@ $< + $(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@ feature-detect-clean: $(call QUIET_CLEAN, feature-detect) diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 88e2bcf16cca..493753a4962e 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -261,7 +261,7 @@ _bpftool() # Deal with options if [[ ${words[cword]} == -* ]]; then local c='--version --json --pretty --bpffs --mapcompat --debug \ - --use-loader --base-btf' + --use-loader --base-btf --legacy' COMPREPLY=( $( compgen -W "$c" -- "$cur" ) ) return 0 fi @@ -710,7 +710,8 @@ _bpftool() hash_of_maps devmap devmap_hash sockmap cpumap \ xskmap sockhash cgroup_storage reuseport_sockarray \ percpu_cgroup_storage queue stack sk_storage \ - struct_ops inode_storage task_storage ringbuf' + struct_ops ringbuf inode_storage task_storage \ + bloom_filter' COMPREPLY=( $( compgen -W "$BPFTOOL_MAP_CREATE_TYPES" -- "$cur" ) ) return 0 ;; diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index 015d2758f826..59833125ac0a 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -39,6 +39,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_DATASEC] = "DATASEC", [BTF_KIND_FLOAT] = "FLOAT", [BTF_KIND_DECL_TAG] = "DECL_TAG", + [BTF_KIND_TYPE_TAG] = "TYPE_TAG", }; struct btf_attach_point { @@ -142,6 +143,7 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_TYPEDEF: + case BTF_KIND_TYPE_TAG: if (json_output) jsonw_uint_field(w, "type_id", t->type); else @@ -418,9 +420,10 @@ static int dump_btf_c(const struct btf *btf, struct btf_dump *d; int err = 0, i; - d = btf_dump__new(btf, NULL, NULL, btf_dump_printf); - if (IS_ERR(d)) - return PTR_ERR(d); + d = btf_dump__new(btf, btf_dump_printf, NULL, NULL); + err = libbpf_get_error(d); + if (err) + return err; printf("#ifndef __VMLINUX_H__\n"); printf("#define __VMLINUX_H__\n"); @@ -547,8 +550,8 @@ static int do_dump(int argc, char **argv) } btf = btf__parse_split(*argv, base ?: base_btf); - if (IS_ERR(btf)) { - err = -PTR_ERR(btf); + err = libbpf_get_error(btf); + if (err) { btf = NULL; p_err("failed to load BTF from %s: %s", *argv, strerror(err)); diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index 9c25286a5c73..f5dddf8ef404 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -32,14 +32,16 @@ static int dump_prog_id_as_func_ptr(const struct btf_dumper *d, const struct btf_type *func_proto, __u32 prog_id) { - struct bpf_prog_info_linear *prog_info = NULL; const struct btf_type *func_type; + int prog_fd = -1, func_sig_len; + struct bpf_prog_info info = {}; + __u32 info_len = sizeof(info); const char *prog_name = NULL; - struct bpf_func_info *finfo; struct btf *prog_btf = NULL; - struct bpf_prog_info *info; - int prog_fd, func_sig_len; + struct bpf_func_info finfo; + __u32 finfo_rec_size; char prog_str[1024]; + int err; /* Get the ptr's func_proto */ func_sig_len = btf_dump_func(d->btf, prog_str, func_proto, NULL, 0, @@ -52,25 +54,30 @@ static int dump_prog_id_as_func_ptr(const struct btf_dumper *d, /* Get the bpf_prog's name. Obtain from func_info. */ prog_fd = bpf_prog_get_fd_by_id(prog_id); - if (prog_fd == -1) + if (prog_fd < 0) goto print; - prog_info = bpf_program__get_prog_info_linear(prog_fd, - 1UL << BPF_PROG_INFO_FUNC_INFO); - close(prog_fd); - if (IS_ERR(prog_info)) { - prog_info = NULL; + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + if (err) goto print; - } - info = &prog_info->info; - if (!info->btf_id || !info->nr_func_info) + if (!info.btf_id || !info.nr_func_info) + goto print; + + finfo_rec_size = info.func_info_rec_size; + memset(&info, 0, sizeof(info)); + info.nr_func_info = 1; + info.func_info_rec_size = finfo_rec_size; + info.func_info = ptr_to_u64(&finfo); + + err = bpf_obj_get_info_by_fd(prog_fd, &info, &info_len); + if (err) goto print; - prog_btf = btf__load_from_kernel_by_id(info->btf_id); + + prog_btf = btf__load_from_kernel_by_id(info.btf_id); if (libbpf_get_error(prog_btf)) goto print; - finfo = u64_to_ptr(info->func_info); - func_type = btf__type_by_id(prog_btf, finfo->type_id); + func_type = btf__type_by_id(prog_btf, finfo.type_id); if (!func_type || !btf_is_func(func_type)) goto print; @@ -92,7 +99,8 @@ print: prog_str[sizeof(prog_str) - 1] = '\0'; jsonw_string(d->jw, prog_str); btf__free(prog_btf); - free(prog_info); + if (prog_fd >= 0) + close(prog_fd); return 0; } diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 511eccdbdfe6..fa8eb8134344 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -74,6 +74,7 @@ const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { [BPF_XDP] = "xdp", [BPF_SK_REUSEPORT_SELECT] = "sk_skb_reuseport_select", [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_skb_reuseport_select_or_migrate", + [BPF_PERF_EVENT] = "perf_event", }; void p_err(const char *fmt, ...) diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index ade44577688e..5397077d0d9e 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -467,7 +467,7 @@ static bool probe_bpf_syscall(const char *define_prefix) { bool res; - bpf_load_program(BPF_PROG_TYPE_UNSPEC, NULL, 0, NULL, 0, NULL, 0); + bpf_prog_load(BPF_PROG_TYPE_UNSPEC, NULL, NULL, NULL, 0, NULL); res = (errno != ENOSYS); print_bool_feature("have_bpf_syscall", diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 5c18351290f0..997a2865e04a 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -218,9 +218,10 @@ static int codegen_datasecs(struct bpf_object *obj, const char *obj_name) char sec_ident[256], map_ident[256]; int i, err = 0; - d = btf_dump__new(btf, NULL, NULL, codegen_btf_dump_printf); - if (IS_ERR(d)) - return PTR_ERR(d); + d = btf_dump__new(btf, codegen_btf_dump_printf, NULL, NULL); + err = libbpf_get_error(d); + if (err) + return err; bpf_object__for_each_map(map, obj) { /* only generate definitions for memory-mapped internal maps */ @@ -719,10 +720,11 @@ static int do_skeleton(int argc, char **argv) get_obj_name(obj_name, file); opts.object_name = obj_name; obj = bpf_object__open_mem(obj_data, file_sz, &opts); - if (IS_ERR(obj)) { + err = libbpf_get_error(obj); + if (err) { char err_buf[256]; - libbpf_strerror(PTR_ERR(obj), err_buf, sizeof(err_buf)); + libbpf_strerror(err, err_buf, sizeof(err_buf)); p_err("failed to open BPF object file: %s", err_buf); obj = NULL; goto out; diff --git a/tools/bpf/bpftool/iter.c b/tools/bpf/bpftool/iter.c index 6c0de647b8ad..f88fdc820d23 100644 --- a/tools/bpf/bpftool/iter.c +++ b/tools/bpf/bpftool/iter.c @@ -46,7 +46,8 @@ static int do_pin(int argc, char **argv) } obj = bpf_object__open(objfile); - if (IS_ERR(obj)) { + err = libbpf_get_error(obj); + if (err) { p_err("can't open objfile %s", objfile); goto close_map_fd; } @@ -64,8 +65,8 @@ static int do_pin(int argc, char **argv) } link = bpf_program__attach_iter(prog, &iter_opts); - if (IS_ERR(link)) { - err = PTR_ERR(link); + err = libbpf_get_error(link); + if (err) { p_err("attach_iter failed for program %s", bpf_program__name(prog)); goto close_obj; diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index 28237d7cef67..473791e87f7d 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -31,6 +31,7 @@ bool block_mount; bool verifier_logs; bool relaxed_maps; bool use_loader; +bool legacy_libbpf; struct btf *base_btf; struct hashmap *refs_table; @@ -396,6 +397,7 @@ int main(int argc, char **argv) { "debug", no_argument, NULL, 'd' }, { "use-loader", no_argument, NULL, 'L' }, { "base-btf", required_argument, NULL, 'B' }, + { "legacy", no_argument, NULL, 'l' }, { 0 } }; int opt, ret; @@ -408,7 +410,7 @@ int main(int argc, char **argv) bin_name = argv[0]; opterr = 0; - while ((opt = getopt_long(argc, argv, "VhpjfLmndB:", + while ((opt = getopt_long(argc, argv, "VhpjfLmndB:l", options, NULL)) >= 0) { switch (opt) { case 'V': @@ -454,6 +456,9 @@ int main(int argc, char **argv) case 'L': use_loader = true; break; + case 'l': + legacy_libbpf = true; + break; default: p_err("unrecognized option '%s'", argv[optind - 1]); if (json_output) @@ -463,6 +468,12 @@ int main(int argc, char **argv) } } + if (!legacy_libbpf) { + ret = libbpf_set_strict_mode(LIBBPF_STRICT_ALL); + if (ret) + p_err("failed to enable libbpf strict mode: %d", ret); + } + argc -= optind; argv += optind; if (argc < 0) diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 383835c2604d..8d76d937a62b 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -57,7 +57,7 @@ static inline void *u64_to_ptr(__u64 ptr) #define HELP_SPEC_PROGRAM \ "PROG := { id PROG_ID | pinned FILE | tag PROG_TAG | name PROG_NAME }" #define HELP_SPEC_OPTIONS \ - "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug}" + "OPTIONS := { {-j|--json} [{-p|--pretty}] | {-d|--debug} | {-l|--legacy}" #define HELP_SPEC_MAP \ "MAP := { id MAP_ID | pinned FILE | name MAP_NAME }" #define HELP_SPEC_LINK \ @@ -90,6 +90,7 @@ extern bool block_mount; extern bool verifier_logs; extern bool relaxed_maps; extern bool use_loader; +extern bool legacy_libbpf; extern struct btf *base_btf; extern struct hashmap *refs_table; diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index cae1f1119296..25b258804f11 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -53,6 +53,7 @@ const char * const map_type_name[] = { [BPF_MAP_TYPE_RINGBUF] = "ringbuf", [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", + [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", }; const size_t map_type_name_size = ARRAY_SIZE(map_type_name); @@ -811,7 +812,7 @@ static struct btf *get_map_kv_btf(const struct bpf_map_info *info) if (info->btf_vmlinux_value_type_id) { if (!btf_vmlinux) { btf_vmlinux = libbpf_find_kernel_btf(); - if (IS_ERR(btf_vmlinux)) + if (libbpf_get_error(btf_vmlinux)) p_err("failed to get kernel btf"); } return btf_vmlinux; @@ -831,13 +832,13 @@ static struct btf *get_map_kv_btf(const struct bpf_map_info *info) static void free_map_kv_btf(struct btf *btf) { - if (!IS_ERR(btf) && btf != btf_vmlinux) + if (!libbpf_get_error(btf) && btf != btf_vmlinux) btf__free(btf); } static void free_btf_vmlinux(void) { - if (!IS_ERR(btf_vmlinux)) + if (!libbpf_get_error(btf_vmlinux)) btf__free(btf_vmlinux); } @@ -862,8 +863,8 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, if (wtr) { btf = get_map_kv_btf(info); - if (IS_ERR(btf)) { - err = PTR_ERR(btf); + err = libbpf_get_error(btf); + if (err) { goto exit_free; } @@ -1477,7 +1478,7 @@ static int do_help(int argc, char **argv) " devmap | devmap_hash | sockmap | cpumap | xskmap | sockhash |\n" " cgroup_storage | reuseport_sockarray | percpu_cgroup_storage |\n" " queue | stack | sk_storage | struct_ops | ringbuf | inode_storage |\n" - " task_storage }\n" + " task_storage | bloom_filter }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-n|--nomount} }\n" "", diff --git a/tools/bpf/bpftool/map_perf_ring.c b/tools/bpf/bpftool/map_perf_ring.c index b98ea702d284..6b0c410152de 100644 --- a/tools/bpf/bpftool/map_perf_ring.c +++ b/tools/bpf/bpftool/map_perf_ring.c @@ -124,7 +124,7 @@ int do_event_pipe(int argc, char **argv) .wakeup_events = 1, }; struct bpf_map_info map_info = {}; - struct perf_buffer_raw_opts opts = {}; + LIBBPF_OPTS(perf_buffer_raw_opts, opts); struct event_pipe_ctx ctx = { .all_cpus = true, .cpu = -1, @@ -190,14 +190,11 @@ int do_event_pipe(int argc, char **argv) ctx.idx = 0; } - opts.attr = &perf_attr; - opts.event_cb = print_bpf_output; - opts.ctx = &ctx; opts.cpu_cnt = ctx.all_cpus ? 0 : 1; opts.cpus = &ctx.cpu; opts.map_keys = &ctx.idx; - - pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &opts); + pb = perf_buffer__new_raw(map_fd, MMAP_PAGE_CNT, &perf_attr, + print_bpf_output, &ctx, &opts); err = libbpf_get_error(pb); if (err) { p_err("failed to create perf buffer: %s (%d)", diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index 515d22952602..e47e8b06cc3d 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -100,6 +100,76 @@ static enum bpf_attach_type parse_attach_type(const char *str) return __MAX_BPF_ATTACH_TYPE; } +static int prep_prog_info(struct bpf_prog_info *const info, enum dump_mode mode, + void **info_data, size_t *const info_data_sz) +{ + struct bpf_prog_info holder = {}; + size_t needed = 0; + void *ptr; + + if (mode == DUMP_JITED) { + holder.jited_prog_len = info->jited_prog_len; + needed += info->jited_prog_len; + } else { + holder.xlated_prog_len = info->xlated_prog_len; + needed += info->xlated_prog_len; + } + + holder.nr_jited_ksyms = info->nr_jited_ksyms; + needed += info->nr_jited_ksyms * sizeof(__u64); + + holder.nr_jited_func_lens = info->nr_jited_func_lens; + needed += info->nr_jited_func_lens * sizeof(__u32); + + holder.nr_func_info = info->nr_func_info; + holder.func_info_rec_size = info->func_info_rec_size; + needed += info->nr_func_info * info->func_info_rec_size; + + holder.nr_line_info = info->nr_line_info; + holder.line_info_rec_size = info->line_info_rec_size; + needed += info->nr_line_info * info->line_info_rec_size; + + holder.nr_jited_line_info = info->nr_jited_line_info; + holder.jited_line_info_rec_size = info->jited_line_info_rec_size; + needed += info->nr_jited_line_info * info->jited_line_info_rec_size; + + if (needed > *info_data_sz) { + ptr = realloc(*info_data, needed); + if (!ptr) + return -1; + + *info_data = ptr; + *info_data_sz = needed; + } + ptr = *info_data; + + if (mode == DUMP_JITED) { + holder.jited_prog_insns = ptr_to_u64(ptr); + ptr += holder.jited_prog_len; + } else { + holder.xlated_prog_insns = ptr_to_u64(ptr); + ptr += holder.xlated_prog_len; + } + + holder.jited_ksyms = ptr_to_u64(ptr); + ptr += holder.nr_jited_ksyms * sizeof(__u64); + + holder.jited_func_lens = ptr_to_u64(ptr); + ptr += holder.nr_jited_func_lens * sizeof(__u32); + + holder.func_info = ptr_to_u64(ptr); + ptr += holder.nr_func_info * holder.func_info_rec_size; + + holder.line_info = ptr_to_u64(ptr); + ptr += holder.nr_line_info * holder.line_info_rec_size; + + holder.jited_line_info = ptr_to_u64(ptr); + ptr += holder.nr_jited_line_info * holder.jited_line_info_rec_size; + + *info = holder; + return 0; +} + static void print_boot_time(__u64 nsecs, char *buf, unsigned int size) { struct timespec real_time_ts, boot_time_ts; @@ -639,8 +709,8 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode, char func_sig[1024]; unsigned char *buf; __u32 member_len; + int fd, err = -1; ssize_t n; - int fd; if (mode == DUMP_JITED) { if (info->jited_prog_len == 0 || !info->jited_prog_insns) { @@ -679,7 +749,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode, if (fd < 0) { p_err("can't open file %s: %s", filepath, strerror(errno)); - return -1; + goto exit_free; } n = write(fd, buf, member_len); @@ -687,7 +757,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode, if (n != (ssize_t)member_len) { p_err("error writing output file: %s", n < 0 ? strerror(errno) : "short write"); - return -1; + goto exit_free; } if (json_output) @@ -701,7 +771,7 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode, info->netns_ino, &disasm_opt); if (!name) - return -1; + goto exit_free; } if (info->nr_jited_func_lens && info->jited_func_lens) { @@ -796,23 +866,28 @@ prog_dump(struct bpf_prog_info *info, enum dump_mode mode, kernel_syms_destroy(&dd); } - btf__free(btf); + err = 0; - return 0; +exit_free: + btf__free(btf); + bpf_prog_linfo__free(prog_linfo); + return err; } static int do_dump(int argc, char **argv) { - struct bpf_prog_info_linear *info_linear; + struct bpf_prog_info info; + __u32 info_len = sizeof(info); + size_t info_data_sz = 0; + void *info_data = NULL; char *filepath = NULL; bool opcodes = false; bool visual = false; enum dump_mode mode; bool linum = false; - int *fds = NULL; int nb_fds, i = 0; + int *fds = NULL; int err = -1; - __u64 arrays; if (is_prefix(*argv, "jited")) { if (disasm_init()) @@ -872,43 +947,44 @@ static int do_dump(int argc, char **argv) goto exit_close; } - if (mode == DUMP_JITED) - arrays = 1UL << BPF_PROG_INFO_JITED_INSNS; - else - arrays = 1UL << BPF_PROG_INFO_XLATED_INSNS; - - arrays |= 1UL << BPF_PROG_INFO_JITED_KSYMS; - arrays |= 1UL << BPF_PROG_INFO_JITED_FUNC_LENS; - arrays |= 1UL << BPF_PROG_INFO_FUNC_INFO; - arrays |= 1UL << BPF_PROG_INFO_LINE_INFO; - arrays |= 1UL << BPF_PROG_INFO_JITED_LINE_INFO; - if (json_output && nb_fds > 1) jsonw_start_array(json_wtr); /* root array */ for (i = 0; i < nb_fds; i++) { - info_linear = bpf_program__get_prog_info_linear(fds[i], arrays); - if (IS_ERR_OR_NULL(info_linear)) { + memset(&info, 0, sizeof(info)); + + err = bpf_obj_get_info_by_fd(fds[i], &info, &info_len); + if (err) { + p_err("can't get prog info: %s", strerror(errno)); + break; + } + + err = prep_prog_info(&info, mode, &info_data, &info_data_sz); + if (err) { + p_err("can't grow prog info_data"); + break; + } + + err = bpf_obj_get_info_by_fd(fds[i], &info, &info_len); + if (err) { p_err("can't get prog info: %s", strerror(errno)); break; } if (json_output && nb_fds > 1) { jsonw_start_object(json_wtr); /* prog object */ - print_prog_header_json(&info_linear->info); + print_prog_header_json(&info); jsonw_name(json_wtr, "insns"); } else if (nb_fds > 1) { - print_prog_header_plain(&info_linear->info); + print_prog_header_plain(&info); } - err = prog_dump(&info_linear->info, mode, filepath, opcodes, - visual, linum); + err = prog_dump(&info, mode, filepath, opcodes, visual, linum); if (json_output && nb_fds > 1) jsonw_end_object(json_wtr); /* prog object */ else if (i != nb_fds - 1 && nb_fds > 1) printf("\n"); - free(info_linear); if (err) break; close(fds[i]); @@ -920,6 +996,7 @@ exit_close: for (; i < nb_fds; i++) close(fds[i]); exit_free: + free(info_data); free(fds); return err; } @@ -1409,8 +1486,6 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) while (argc) { if (is_prefix(*argv, "type")) { - char *type; - NEXT_ARG(); if (common_prog_type != BPF_PROG_TYPE_UNSPEC) { @@ -1420,21 +1495,26 @@ static int load_with_options(int argc, char **argv, bool first_prog_only) if (!REQ_ARGS(1)) goto err_free_reuse_maps; - /* Put a '/' at the end of type to appease libbpf */ - type = malloc(strlen(*argv) + 2); - if (!type) { - p_err("mem alloc failed"); - goto err_free_reuse_maps; - } - *type = 0; - strcat(type, *argv); - strcat(type, "/"); + err = libbpf_prog_type_by_name(*argv, &common_prog_type, + &expected_attach_type); + if (err < 0) { + /* Put a '/' at the end of type to appease libbpf */ + char *type = malloc(strlen(*argv) + 2); - err = get_prog_type_by_name(type, &common_prog_type, - &expected_attach_type); - free(type); - if (err < 0) - goto err_free_reuse_maps; + if (!type) { + p_err("mem alloc failed"); + goto err_free_reuse_maps; + } + *type = 0; + strcat(type, *argv); + strcat(type, "/"); + + err = get_prog_type_by_name(type, &common_prog_type, + &expected_attach_type); + free(type); + if (err < 0) + goto err_free_reuse_maps; + } NEXT_ARG(); } else if (is_prefix(*argv, "map")) { @@ -1657,6 +1737,11 @@ err_unpin: else bpf_object__unpin_programs(obj, pinfile); err_close_obj: + if (!legacy_libbpf) { + p_info("Warning: bpftool is now running in libbpf strict mode and has more stringent requirements about BPF programs.\n" + "If it used to work for this object file but now doesn't, see --legacy option for more details.\n"); + } + bpf_object__close(obj); err_free_reuse_maps: for (i = 0; i < old_map_fds; i++) @@ -2016,41 +2101,58 @@ static void profile_print_readings(void) static char *profile_target_name(int tgt_fd) { - struct bpf_prog_info_linear *info_linear; - struct bpf_func_info *func_info; + struct bpf_func_info func_info; + struct bpf_prog_info info = {}; + __u32 info_len = sizeof(info); const struct btf_type *t; + __u32 func_info_rec_size; struct btf *btf = NULL; char *name = NULL; + int err; - info_linear = bpf_program__get_prog_info_linear( - tgt_fd, 1UL << BPF_PROG_INFO_FUNC_INFO); - if (IS_ERR_OR_NULL(info_linear)) { - p_err("failed to get info_linear for prog FD %d", tgt_fd); - return NULL; + err = bpf_obj_get_info_by_fd(tgt_fd, &info, &info_len); + if (err) { + p_err("failed to bpf_obj_get_info_by_fd for prog FD %d", tgt_fd); + goto out; } - if (info_linear->info.btf_id == 0) { + if (info.btf_id == 0) { p_err("prog FD %d doesn't have valid btf", tgt_fd); goto out; } - btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); + func_info_rec_size = info.func_info_rec_size; + if (info.nr_func_info == 0) { + p_err("bpf_obj_get_info_by_fd for prog FD %d found 0 func_info", tgt_fd); + goto out; + } + + memset(&info, 0, sizeof(info)); + info.nr_func_info = 1; + info.func_info_rec_size = func_info_rec_size; + info.func_info = ptr_to_u64(&func_info); + + err = bpf_obj_get_info_by_fd(tgt_fd, &info, &info_len); + if (err) { + p_err("failed to get func_info for prog FD %d", tgt_fd); + goto out; + } + + btf = btf__load_from_kernel_by_id(info.btf_id); if (libbpf_get_error(btf)) { p_err("failed to load btf for prog FD %d", tgt_fd); goto out; } - func_info = u64_to_ptr(info_linear->info.func_info); - t = btf__type_by_id(btf, func_info[0].type_id); + t = btf__type_by_id(btf, func_info.type_id); if (!t) { p_err("btf %d doesn't have type %d", - info_linear->info.btf_id, func_info[0].type_id); + info.btf_id, func_info.type_id); goto out; } name = strdup(btf__name_by_offset(btf, t->name_off)); out: btf__free(btf); - free(info_linear); return name; } diff --git a/tools/bpf/bpftool/struct_ops.c b/tools/bpf/bpftool/struct_ops.c index ab2d2290569a..cbdca37a53f0 100644 --- a/tools/bpf/bpftool/struct_ops.c +++ b/tools/bpf/bpftool/struct_ops.c @@ -32,7 +32,7 @@ static const struct btf *get_btf_vmlinux(void) return btf_vmlinux; btf_vmlinux = libbpf_find_kernel_btf(); - if (IS_ERR(btf_vmlinux)) + if (libbpf_get_error(btf_vmlinux)) p_err("struct_ops requires kernel CONFIG_DEBUG_INFO_BTF=y"); return btf_vmlinux; @@ -45,7 +45,7 @@ static const char *get_kern_struct_ops_name(const struct bpf_map_info *info) const char *st_ops_name; kern_btf = get_btf_vmlinux(); - if (IS_ERR(kern_btf)) + if (libbpf_get_error(kern_btf)) return "<btf_vmlinux_not_found>"; t = btf__type_by_id(kern_btf, info->btf_vmlinux_value_type_id); @@ -63,7 +63,7 @@ static __s32 get_map_info_type_id(void) return map_info_type_id; kern_btf = get_btf_vmlinux(); - if (IS_ERR(kern_btf)) { + if (libbpf_get_error(kern_btf)) { map_info_type_id = PTR_ERR(kern_btf); return map_info_type_id; } @@ -252,7 +252,7 @@ static struct res do_one_id(const char *id_str, work_func func, void *data, } fd = bpf_map_get_fd_by_id(id); - if (fd == -1) { + if (fd < 0) { p_err("can't get map by id (%lu): %s", id, strerror(errno)); res.nr_errs++; return res; @@ -415,7 +415,7 @@ static int do_dump(int argc, char **argv) } kern_btf = get_btf_vmlinux(); - if (IS_ERR(kern_btf)) + if (libbpf_get_error(kern_btf)) return -1; if (!json_output) { @@ -495,7 +495,7 @@ static int do_register(int argc, char **argv) file = GET_ARG(); obj = bpf_object__open(file); - if (IS_ERR_OR_NULL(obj)) + if (libbpf_get_error(obj)) return -1; set_max_rlimit(); @@ -516,7 +516,7 @@ static int do_register(int argc, char **argv) continue; link = bpf_map__attach_struct_ops(map); - if (IS_ERR(link)) { + if (libbpf_get_error(link)) { p_err("can't register struct_ops %s: %s", bpf_map__name(map), strerror(-PTR_ERR(link))); @@ -596,7 +596,7 @@ int do_struct_ops(int argc, char **argv) err = cmd_select(cmds, argc, argv, do_help); - if (!IS_ERR(btf_vmlinux)) + if (!libbpf_get_error(btf_vmlinux)) btf__free(btf_vmlinux); return err; diff --git a/tools/bpf/runqslower/runqslower.c b/tools/bpf/runqslower/runqslower.c index d89715844952..2414cc764461 100644 --- a/tools/bpf/runqslower/runqslower.c +++ b/tools/bpf/runqslower/runqslower.c @@ -123,7 +123,6 @@ int main(int argc, char **argv) .parser = parse_arg, .doc = argp_program_doc, }; - struct perf_buffer_opts pb_opts; struct perf_buffer *pb = NULL; struct runqslower_bpf *obj; int err; @@ -165,9 +164,8 @@ int main(int argc, char **argv) printf("Tracing run queue latency higher than %llu us\n", env.min_us); printf("%-8s %-16s %-6s %14s\n", "TIME", "COMM", "PID", "LAT(us)"); - pb_opts.sample_cb = handle_event; - pb_opts.lost_cb = handle_lost_events; - pb = perf_buffer__new(bpf_map__fd(obj->maps.events), 64, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(obj->maps.events), 64, + handle_event, handle_lost_events, NULL, NULL); err = libbpf_get_error(pb); if (err) { pb = NULL; diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index ba5af15e25f5..6297eafdc40f 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -4938,6 +4938,25 @@ union bpf_attr { * **-ENOENT** if symbol is not found. * * **-EPERM** if caller does not have permission to obtain kernel address. + * + * long bpf_find_vma(struct task_struct *task, u64 addr, void *callback_fn, void *callback_ctx, u64 flags) + * Description + * Find vma of *task* that contains *addr*, call *callback_fn* + * function with *task*, *vma*, and *callback_ctx*. + * The *callback_fn* should be a static function and + * the *callback_ctx* should be a pointer to the stack. + * The *flags* is used to control certain aspects of the helper. + * Currently, the *flags* must be 0. + * + * The expected callback signature is + * + * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); + * + * Return + * 0 on success. + * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. + * **-EBUSY** if failed to try lock mmap_lock. + * **-EINVAL** for invalid **flags**. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5120,6 +5139,7 @@ union bpf_attr { FN(trace_vprintk), \ FN(skc_to_unix_sock), \ FN(kallsyms_lookup_name), \ + FN(find_vma), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -6296,6 +6316,7 @@ struct bpf_sk_lookup { __u32 local_ip4; /* Network byte order */ __u32 local_ip6[4]; /* Network byte order */ __u32 local_port; /* Host byte order */ + __u32 ingress_ifindex; /* The arriving interface. Determined by inet_iif. */ }; /* diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index deb12f755f0f..b0d8fea1951d 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h @@ -43,7 +43,7 @@ struct btf_type { * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, - * FUNC, FUNC_PROTO, VAR and DECL_TAG. + * FUNC, FUNC_PROTO, VAR, DECL_TAG and TYPE_TAG. * "type" is a type_id referring to another type. */ union { @@ -75,6 +75,7 @@ enum { BTF_KIND_DATASEC = 15, /* Section */ BTF_KIND_FLOAT = 16, /* Floating point */ BTF_KIND_DECL_TAG = 17, /* Decl Tag */ + BTF_KIND_TYPE_TAG = 18, /* Type Tag */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index eebd3894fe89..4ac53b30b6dc 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -858,6 +858,7 @@ enum { IFLA_BOND_TLB_DYNAMIC_LB, IFLA_BOND_PEER_NOTIF_DELAY, IFLA_BOND_AD_LACP_ACTIVE, + IFLA_BOND_MISSED_MAX, __IFLA_BOND_MAX, }; diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index b393b5e82380..5f7086fae31c 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -84,6 +84,7 @@ else endif # Append required CFLAGS +override CFLAGS += -std=gnu89 override CFLAGS += $(EXTRA_WARNINGS) -Wno-switch-enum override CFLAGS += -Werror -Wall override CFLAGS += $(INCLUDES) diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index 725701235fd8..94560ba31724 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -28,6 +28,7 @@ #include <asm/unistd.h> #include <errno.h> #include <linux/bpf.h> +#include <limits.h> #include "bpf.h" #include "libbpf.h" #include "libbpf_internal.h" @@ -74,14 +75,15 @@ static inline int sys_bpf_fd(enum bpf_cmd cmd, union bpf_attr *attr, return ensure_good_fd(fd); } -static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size) +#define PROG_LOAD_ATTEMPTS 5 + +static inline int sys_bpf_prog_load(union bpf_attr *attr, unsigned int size, int attempts) { - int retries = 5; int fd; do { fd = sys_bpf_fd(BPF_PROG_LOAD, attr, size); - } while (fd < 0 && errno == EAGAIN && retries-- > 0); + } while (fd < 0 && errno == EAGAIN && --attempts > 0); return fd; } @@ -253,58 +255,91 @@ alloc_zero_tailing_info(const void *orecord, __u32 cnt, return info; } -int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr) +DEFAULT_VERSION(bpf_prog_load_v0_6_0, bpf_prog_load, LIBBPF_0.6.0) +int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, + const char *prog_name, const char *license, + const struct bpf_insn *insns, size_t insn_cnt, + const struct bpf_prog_load_opts *opts) { void *finfo = NULL, *linfo = NULL; + const char *func_info, *line_info; + __u32 log_size, log_level, attach_prog_fd, attach_btf_obj_fd; + __u32 func_info_rec_size, line_info_rec_size; + int fd, attempts; union bpf_attr attr; - int fd; + char *log_buf; - if (!load_attr->log_buf != !load_attr->log_buf_sz) + if (!OPTS_VALID(opts, bpf_prog_load_opts)) return libbpf_err(-EINVAL); - if (load_attr->log_level > (4 | 2 | 1) || (load_attr->log_level && !load_attr->log_buf)) + attempts = OPTS_GET(opts, attempts, 0); + if (attempts < 0) return libbpf_err(-EINVAL); + if (attempts == 0) + attempts = PROG_LOAD_ATTEMPTS; memset(&attr, 0, sizeof(attr)); - attr.prog_type = load_attr->prog_type; - attr.expected_attach_type = load_attr->expected_attach_type; - if (load_attr->attach_prog_fd) - attr.attach_prog_fd = load_attr->attach_prog_fd; - else - attr.attach_btf_obj_fd = load_attr->attach_btf_obj_fd; - attr.attach_btf_id = load_attr->attach_btf_id; + attr.prog_type = prog_type; + attr.expected_attach_type = OPTS_GET(opts, expected_attach_type, 0); - attr.prog_ifindex = load_attr->prog_ifindex; - attr.kern_version = load_attr->kern_version; + attr.prog_btf_fd = OPTS_GET(opts, prog_btf_fd, 0); + attr.prog_flags = OPTS_GET(opts, prog_flags, 0); + attr.prog_ifindex = OPTS_GET(opts, prog_ifindex, 0); + attr.kern_version = OPTS_GET(opts, kern_version, 0); - attr.insn_cnt = (__u32)load_attr->insn_cnt; - attr.insns = ptr_to_u64(load_attr->insns); - attr.license = ptr_to_u64(load_attr->license); + if (prog_name) + strncat(attr.prog_name, prog_name, sizeof(attr.prog_name) - 1); + attr.license = ptr_to_u64(license); - attr.log_level = load_attr->log_level; - if (attr.log_level) { - attr.log_buf = ptr_to_u64(load_attr->log_buf); - attr.log_size = load_attr->log_buf_sz; - } + if (insn_cnt > UINT_MAX) + return libbpf_err(-E2BIG); + + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = (__u32)insn_cnt; + + attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); + attach_btf_obj_fd = OPTS_GET(opts, attach_btf_obj_fd, 0); + + if (attach_prog_fd && attach_btf_obj_fd) + return libbpf_err(-EINVAL); + + attr.attach_btf_id = OPTS_GET(opts, attach_btf_id, 0); + if (attach_prog_fd) + attr.attach_prog_fd = attach_prog_fd; + else + attr.attach_btf_obj_fd = attach_btf_obj_fd; - attr.prog_btf_fd = load_attr->prog_btf_fd; - attr.prog_flags = load_attr->prog_flags; + log_buf = OPTS_GET(opts, log_buf, NULL); + log_size = OPTS_GET(opts, log_size, 0); + log_level = OPTS_GET(opts, log_level, 0); - attr.func_info_rec_size = load_attr->func_info_rec_size; - attr.func_info_cnt = load_attr->func_info_cnt; - attr.func_info = ptr_to_u64(load_attr->func_info); + if (!!log_buf != !!log_size) + return libbpf_err(-EINVAL); + if (log_level > (4 | 2 | 1)) + return libbpf_err(-EINVAL); + if (log_level && !log_buf) + return libbpf_err(-EINVAL); - attr.line_info_rec_size = load_attr->line_info_rec_size; - attr.line_info_cnt = load_attr->line_info_cnt; - attr.line_info = ptr_to_u64(load_attr->line_info); - attr.fd_array = ptr_to_u64(load_attr->fd_array); + attr.log_level = log_level; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_size; - if (load_attr->name) - memcpy(attr.prog_name, load_attr->name, - min(strlen(load_attr->name), (size_t)BPF_OBJ_NAME_LEN - 1)); + func_info_rec_size = OPTS_GET(opts, func_info_rec_size, 0); + func_info = OPTS_GET(opts, func_info, NULL); + attr.func_info_rec_size = func_info_rec_size; + attr.func_info = ptr_to_u64(func_info); + attr.func_info_cnt = OPTS_GET(opts, func_info_cnt, 0); - fd = sys_bpf_prog_load(&attr, sizeof(attr)); + line_info_rec_size = OPTS_GET(opts, line_info_rec_size, 0); + line_info = OPTS_GET(opts, line_info, NULL); + attr.line_info_rec_size = line_info_rec_size; + attr.line_info = ptr_to_u64(line_info); + attr.line_info_cnt = OPTS_GET(opts, line_info_cnt, 0); + + attr.fd_array = ptr_to_u64(OPTS_GET(opts, fd_array, NULL)); + + fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); if (fd >= 0) return fd; @@ -314,11 +349,11 @@ int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr) */ while (errno == E2BIG && (!finfo || !linfo)) { if (!finfo && attr.func_info_cnt && - attr.func_info_rec_size < load_attr->func_info_rec_size) { + attr.func_info_rec_size < func_info_rec_size) { /* try with corrected func info records */ - finfo = alloc_zero_tailing_info(load_attr->func_info, - load_attr->func_info_cnt, - load_attr->func_info_rec_size, + finfo = alloc_zero_tailing_info(func_info, + attr.func_info_cnt, + func_info_rec_size, attr.func_info_rec_size); if (!finfo) { errno = E2BIG; @@ -326,13 +361,12 @@ int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr) } attr.func_info = ptr_to_u64(finfo); - attr.func_info_rec_size = load_attr->func_info_rec_size; + attr.func_info_rec_size = func_info_rec_size; } else if (!linfo && attr.line_info_cnt && - attr.line_info_rec_size < - load_attr->line_info_rec_size) { - linfo = alloc_zero_tailing_info(load_attr->line_info, - load_attr->line_info_cnt, - load_attr->line_info_rec_size, + attr.line_info_rec_size < line_info_rec_size) { + linfo = alloc_zero_tailing_info(line_info, + attr.line_info_cnt, + line_info_rec_size, attr.line_info_rec_size); if (!linfo) { errno = E2BIG; @@ -340,26 +374,26 @@ int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr) } attr.line_info = ptr_to_u64(linfo); - attr.line_info_rec_size = load_attr->line_info_rec_size; + attr.line_info_rec_size = line_info_rec_size; } else { break; } - fd = sys_bpf_prog_load(&attr, sizeof(attr)); + fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); if (fd >= 0) goto done; } - if (load_attr->log_level || !load_attr->log_buf) + if (log_level || !log_buf) goto done; /* Try again with log */ - attr.log_buf = ptr_to_u64(load_attr->log_buf); - attr.log_size = load_attr->log_buf_sz; + log_buf[0] = 0; + attr.log_buf = ptr_to_u64(log_buf); + attr.log_size = log_size; attr.log_level = 1; - load_attr->log_buf[0] = 0; - fd = sys_bpf_prog_load(&attr, sizeof(attr)); + fd = sys_bpf_prog_load(&attr, sizeof(attr), attempts); done: /* free() doesn't affect errno, so we don't need to restore it */ free(finfo); @@ -367,17 +401,20 @@ done: return libbpf_err_errno(fd); } +__attribute__((alias("bpf_load_program_xattr2"))) int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz) + char *log_buf, size_t log_buf_sz); + +static int bpf_load_program_xattr2(const struct bpf_load_program_attr *load_attr, + char *log_buf, size_t log_buf_sz) { - struct bpf_prog_load_params p = {}; + LIBBPF_OPTS(bpf_prog_load_opts, p); if (!load_attr || !log_buf != !log_buf_sz) return libbpf_err(-EINVAL); - p.prog_type = load_attr->prog_type; p.expected_attach_type = load_attr->expected_attach_type; - switch (p.prog_type) { + switch (load_attr->prog_type) { case BPF_PROG_TYPE_STRUCT_OPS: case BPF_PROG_TYPE_LSM: p.attach_btf_id = load_attr->attach_btf_id; @@ -391,12 +428,9 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, p.prog_ifindex = load_attr->prog_ifindex; p.kern_version = load_attr->kern_version; } - p.insn_cnt = load_attr->insns_cnt; - p.insns = load_attr->insns; - p.license = load_attr->license; p.log_level = load_attr->log_level; p.log_buf = log_buf; - p.log_buf_sz = log_buf_sz; + p.log_size = log_buf_sz; p.prog_btf_fd = load_attr->prog_btf_fd; p.func_info_rec_size = load_attr->func_info_rec_size; p.func_info_cnt = load_attr->func_info_cnt; @@ -404,10 +438,10 @@ int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, p.line_info_rec_size = load_attr->line_info_rec_size; p.line_info_cnt = load_attr->line_info_cnt; p.line_info = load_attr->line_info; - p.name = load_attr->name; p.prog_flags = load_attr->prog_flags; - return libbpf__bpf_prog_load(&p); + return bpf_prog_load(load_attr->prog_type, load_attr->name, load_attr->license, + load_attr->insns, load_attr->insns_cnt, &p); } int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, @@ -426,7 +460,7 @@ int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, load_attr.license = license; load_attr.kern_version = kern_version; - return bpf_load_program_xattr(&load_attr, log_buf, log_buf_sz); + return bpf_load_program_xattr2(&load_attr, log_buf, log_buf_sz); } int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, @@ -449,7 +483,7 @@ int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, attr.kern_version = kern_version; attr.prog_flags = prog_flags; - fd = sys_bpf_prog_load(&attr, sizeof(attr)); + fd = sys_bpf_prog_load(&attr, sizeof(attr), PROG_LOAD_ATTEMPTS); return libbpf_err_errno(fd); } diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index 6fffb3cdf39b..079cc81ac51e 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -29,6 +29,7 @@ #include <stdint.h> #include "libbpf_common.h" +#include "libbpf_legacy.h" #ifdef __cplusplus extern "C" { @@ -71,6 +72,71 @@ LIBBPF_API int bpf_create_map_in_map(enum bpf_map_type map_type, int inner_map_fd, int max_entries, __u32 map_flags); +struct bpf_prog_load_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + + /* libbpf can retry BPF_PROG_LOAD command if bpf() syscall returns + * -EAGAIN. This field determines how many attempts libbpf has to + * make. If not specified, libbpf will use default value of 5. + */ + int attempts; + + enum bpf_attach_type expected_attach_type; + __u32 prog_btf_fd; + __u32 prog_flags; + __u32 prog_ifindex; + __u32 kern_version; + + __u32 attach_btf_id; + __u32 attach_prog_fd; + __u32 attach_btf_obj_fd; + + const int *fd_array; + + /* .BTF.ext func info data */ + const void *func_info; + __u32 func_info_cnt; + __u32 func_info_rec_size; + + /* .BTF.ext line info data */ + const void *line_info; + __u32 line_info_cnt; + __u32 line_info_rec_size; + + /* verifier log options */ + __u32 log_level; + __u32 log_size; + char *log_buf; +}; +#define bpf_prog_load_opts__last_field log_buf + +LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, + const char *prog_name, const char *license, + const struct bpf_insn *insns, size_t insn_cnt, + const struct bpf_prog_load_opts *opts); +/* this "specialization" should go away in libbpf 1.0 */ +LIBBPF_API int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, + const char *prog_name, const char *license, + const struct bpf_insn *insns, size_t insn_cnt, + const struct bpf_prog_load_opts *opts); + +/* This is an elaborate way to not conflict with deprecated bpf_prog_load() + * API, defined in libbpf.h. Once we hit libbpf 1.0, all this will be gone. + * With this approach, if someone is calling bpf_prog_load() with + * 4 arguments, they will use the deprecated API, which keeps backwards + * compatibility (both source code and binary). If bpf_prog_load() is called + * with 6 arguments, though, it gets redirected to __bpf_prog_load. + * So looking forward to libbpf 1.0 when this hack will be gone and + * __bpf_prog_load() will be called just bpf_prog_load(). + */ +#ifndef bpf_prog_load +#define bpf_prog_load(...) ___libbpf_overload(___bpf_prog_load, __VA_ARGS__) +#define ___bpf_prog_load4(file, type, pobj, prog_fd) \ + bpf_prog_load_deprecated(file, type, pobj, prog_fd) +#define ___bpf_prog_load6(prog_type, prog_name, license, insns, insn_cnt, opts) \ + bpf_prog_load(prog_type, prog_name, license, insns, insn_cnt, opts) +#endif /* bpf_prog_load */ + struct bpf_load_program_attr { enum bpf_prog_type prog_type; enum bpf_attach_type expected_attach_type; @@ -102,13 +168,15 @@ struct bpf_load_program_attr { /* Recommend log buffer size */ #define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */ -LIBBPF_API int -bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") +LIBBPF_API int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, + char *log_buf, size_t log_buf_sz); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") LIBBPF_API int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, size_t insns_cnt, const char *license, __u32 kern_version, char *log_buf, size_t log_buf_sz); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") LIBBPF_API int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, size_t insns_cnt, __u32 prog_flags, diff --git a/tools/lib/bpf/bpf_gen_internal.h b/tools/lib/bpf/bpf_gen_internal.h index 6f3df004479b..cc486a77db65 100644 --- a/tools/lib/bpf/bpf_gen_internal.h +++ b/tools/lib/bpf/bpf_gen_internal.h @@ -3,6 +3,8 @@ #ifndef __BPF_GEN_INTERNAL_H #define __BPF_GEN_INTERNAL_H +#include "bpf.h" + struct ksym_relo_desc { const char *name; int kind; @@ -50,8 +52,10 @@ int bpf_gen__finish(struct bpf_gen *gen, int nr_progs, int nr_maps); void bpf_gen__free(struct bpf_gen *gen); void bpf_gen__load_btf(struct bpf_gen *gen, const void *raw_data, __u32 raw_size); void bpf_gen__map_create(struct bpf_gen *gen, struct bpf_create_map_params *map_attr, int map_idx); -struct bpf_prog_load_params; -void bpf_gen__prog_load(struct bpf_gen *gen, struct bpf_prog_load_params *load_attr, int prog_idx); +void bpf_gen__prog_load(struct bpf_gen *gen, + enum bpf_prog_type prog_type, const char *prog_name, + const char *license, struct bpf_insn *insns, size_t insn_cnt, + struct bpf_prog_load_opts *load_attr, int prog_idx); void bpf_gen__map_update_elem(struct bpf_gen *gen, int map_idx, void *value, __u32 value_size); void bpf_gen__map_freeze(struct bpf_gen *gen, int map_idx); void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *name, enum bpf_attach_type type); diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 7e4c5586bd87..fadf089ae8fe 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -299,6 +299,7 @@ static int btf_type_size(const struct btf_type *t) case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: + case BTF_KIND_TYPE_TAG: return base_size; case BTF_KIND_INT: return base_size + sizeof(__u32); @@ -349,6 +350,7 @@ static int btf_bswap_type_rest(struct btf_type *t) case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: + case BTF_KIND_TYPE_TAG: return 0; case BTF_KIND_INT: *(__u32 *)(t + 1) = bswap_32(*(__u32 *)(t + 1)); @@ -649,6 +651,7 @@ int btf__align_of(const struct btf *btf, __u32 id) case BTF_KIND_VOLATILE: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: return btf__align_of(btf, t->type); case BTF_KIND_ARRAY: return btf__align_of(btf, btf_array(t)->type); @@ -2236,6 +2239,22 @@ int btf__add_restrict(struct btf *btf, int ref_type_id) } /* + * Append new BTF_KIND_TYPE_TAG type with: + * - *value*, non-empty/non-NULL tag value; + * - *ref_type_id* - referenced type ID, it might not exist yet; + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id) +{ + if (!value|| !value[0]) + return libbpf_err(-EINVAL); + + return btf_add_ref_kind(btf, BTF_KIND_TYPE_TAG, value, ref_type_id); +} + +/* * Append new BTF_KIND_FUNC type with: * - *name*, non-empty/non-NULL name; * - *proto_type_id* - FUNC_PROTO's type ID, it might not exist yet; @@ -2846,8 +2865,7 @@ __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext) struct btf_dedup; -static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext, - const struct btf_dedup_opts *opts); +static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); static void btf_dedup_free(struct btf_dedup *d); static int btf_dedup_prep(struct btf_dedup *d); static int btf_dedup_strings(struct btf_dedup *d); @@ -2994,12 +3012,17 @@ static int btf_dedup_remap_types(struct btf_dedup *d); * deduplicating structs/unions is described in greater details in comments for * `btf_dedup_is_equiv` function. */ -int btf__dedup(struct btf *btf, struct btf_ext *btf_ext, - const struct btf_dedup_opts *opts) + +DEFAULT_VERSION(btf__dedup_v0_6_0, btf__dedup, LIBBPF_0.6.0) +int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts) { - struct btf_dedup *d = btf_dedup_new(btf, btf_ext, opts); + struct btf_dedup *d; int err; + if (!OPTS_VALID(opts, btf_dedup_opts)) + return libbpf_err(-EINVAL); + + d = btf_dedup_new(btf, opts); if (IS_ERR(d)) { pr_debug("btf_dedup_new failed: %ld", PTR_ERR(d)); return libbpf_err(-EINVAL); @@ -3051,6 +3074,19 @@ done: return libbpf_err(err); } +COMPAT_VERSION(bpf__dedup_deprecated, btf__dedup, LIBBPF_0.0.2) +int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *unused_opts) +{ + LIBBPF_OPTS(btf_dedup_opts, opts, .btf_ext = btf_ext); + + if (unused_opts) { + pr_warn("please use new version of btf__dedup() that supports options\n"); + return libbpf_err(-ENOTSUP); + } + + return btf__dedup(btf, &opts); +} + #define BTF_UNPROCESSED_ID ((__u32)-1) #define BTF_IN_PROGRESS_ID ((__u32)-2) @@ -3163,8 +3199,7 @@ static bool btf_dedup_equal_fn(const void *k1, const void *k2, void *ctx) return k1 == k2; } -static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext, - const struct btf_dedup_opts *opts) +static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts) { struct btf_dedup *d = calloc(1, sizeof(struct btf_dedup)); hashmap_hash_fn hash_fn = btf_dedup_identity_hash_fn; @@ -3173,13 +3208,11 @@ static struct btf_dedup *btf_dedup_new(struct btf *btf, struct btf_ext *btf_ext, if (!d) return ERR_PTR(-ENOMEM); - d->opts.dont_resolve_fwds = opts && opts->dont_resolve_fwds; - /* dedup_table_size is now used only to force collisions in tests */ - if (opts && opts->dedup_table_size == 1) + if (OPTS_GET(opts, force_collisions, false)) hash_fn = btf_dedup_collision_hash_fn; d->btf = btf; - d->btf_ext = btf_ext; + d->btf_ext = OPTS_GET(opts, btf_ext, NULL); d->dedup_table = hashmap__new(hash_fn, btf_dedup_equal_fn, NULL); if (IS_ERR(d->dedup_table)) { @@ -3625,6 +3658,7 @@ static int btf_dedup_prep(struct btf_dedup *d) case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: + case BTF_KIND_TYPE_TAG: h = btf_hash_common(t); break; case BTF_KIND_INT: @@ -3685,6 +3719,7 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_VAR: case BTF_KIND_DATASEC: case BTF_KIND_DECL_TAG: + case BTF_KIND_TYPE_TAG: return 0; case BTF_KIND_INT: @@ -3708,8 +3743,6 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) new_id = cand_id; break; } - if (d->opts.dont_resolve_fwds) - continue; if (btf_compat_enum(t, cand)) { if (btf_is_enum_fwd(t)) { /* resolve fwd to full enum */ @@ -3952,8 +3985,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, return 0; /* FWD <--> STRUCT/UNION equivalence check, if enabled */ - if (!d->opts.dont_resolve_fwds - && (cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) + if ((cand_kind == BTF_KIND_FWD || canon_kind == BTF_KIND_FWD) && cand_kind != canon_kind) { __u16 real_kind; __u16 fwd_kind; @@ -3979,10 +4011,7 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, return btf_equal_int_tag(cand_type, canon_type); case BTF_KIND_ENUM: - if (d->opts.dont_resolve_fwds) - return btf_equal_enum(cand_type, canon_type); - else - return btf_compat_enum(cand_type, canon_type); + return btf_compat_enum(cand_type, canon_type); case BTF_KIND_FWD: case BTF_KIND_FLOAT: @@ -4289,6 +4318,7 @@ static int btf_dedup_ref_type(struct btf_dedup *d, __u32 type_id) case BTF_KIND_PTR: case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: + case BTF_KIND_TYPE_TAG: ref_type_id = btf_dedup_ref_type(d, t->type); if (ref_type_id < 0) return ref_type_id; @@ -4595,6 +4625,7 @@ int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ct case BTF_KIND_FUNC: case BTF_KIND_VAR: case BTF_KIND_DECL_TAG: + case BTF_KIND_TYPE_TAG: return visit(&t->type, ctx); case BTF_KIND_ARRAY: { diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index bc005ba3ceec..5c73a5b0a044 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -227,6 +227,7 @@ LIBBPF_API int btf__add_typedef(struct btf *btf, const char *name, int ref_type_ LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id); LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id); LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id); +LIBBPF_API int btf__add_type_tag(struct btf *btf, const char *value, int ref_type_id); /* func and func_proto construction APIs */ LIBBPF_API int btf__add_func(struct btf *btf, const char *name, @@ -245,25 +246,80 @@ LIBBPF_API int btf__add_decl_tag(struct btf *btf, const char *value, int ref_typ int component_idx); struct btf_dedup_opts { - unsigned int dedup_table_size; - bool dont_resolve_fwds; + size_t sz; + /* optional .BTF.ext info to dedup along the main BTF info */ + struct btf_ext *btf_ext; + /* force hash collisions (used for testing) */ + bool force_collisions; + size_t :0; }; +#define btf_dedup_opts__last_field force_collisions + +LIBBPF_API int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts); -LIBBPF_API int btf__dedup(struct btf *btf, struct btf_ext *btf_ext, - const struct btf_dedup_opts *opts); +LIBBPF_API int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts); + +LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__dedup() instead") +LIBBPF_API int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *opts); +#define btf__dedup(...) ___libbpf_overload(___btf_dedup, __VA_ARGS__) +#define ___btf_dedup3(btf, btf_ext, opts) btf__dedup_deprecated(btf, btf_ext, opts) +#define ___btf_dedup2(btf, opts) btf__dedup(btf, opts) struct btf_dump; struct btf_dump_opts { - void *ctx; + union { + size_t sz; + void *ctx; /* DEPRECATED: will be gone in v1.0 */ + }; }; typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args); LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf, - const struct btf_ext *btf_ext, - const struct btf_dump_opts *opts, - btf_dump_printf_fn_t printf_fn); + btf_dump_printf_fn_t printf_fn, + void *ctx, + const struct btf_dump_opts *opts); + +LIBBPF_API struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf, + btf_dump_printf_fn_t printf_fn, + void *ctx, + const struct btf_dump_opts *opts); + +LIBBPF_API struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, + const struct btf_ext *btf_ext, + const struct btf_dump_opts *opts, + btf_dump_printf_fn_t printf_fn); + +/* Choose either btf_dump__new() or btf_dump__new_deprecated() based on the + * type of 4th argument. If it's btf_dump's print callback, use deprecated + * API; otherwise, choose the new btf_dump__new(). ___libbpf_override() + * doesn't work here because both variants have 4 input arguments. + * + * (void *) casts are necessary to avoid compilation warnings about type + * mismatches, because even though __builtin_choose_expr() only ever evaluates + * one side the other side still has to satisfy type constraints (this is + * compiler implementation limitation which might be lifted eventually, + * according to the documentation). So passing struct btf_ext in place of + * btf_dump_printf_fn_t would be generating compilation warning. Casting to + * void * avoids this issue. + * + * Also, two type compatibility checks for a function and function pointer are + * required because passing function reference into btf_dump__new() as + * btf_dump__new(..., my_callback, ...) and as btf_dump__new(..., + * &my_callback, ...) (not explicit ampersand in the latter case) actually + * differs as far as __builtin_types_compatible_p() is concerned. Thus two + * checks are combined to detect callback argument. + * + * The rest works just like in case of ___libbpf_override() usage with symbol + * versioning. + */ +#define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \ + __builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) || \ + __builtin_types_compatible_p(typeof(a4), void(void *, const char *, va_list)), \ + btf_dump__new_deprecated((void *)a1, (void *)a2, (void *)a3, (void *)a4), \ + btf_dump__new((void *)a1, (void *)a2, (void *)a3, (void *)a4)) + LIBBPF_API void btf_dump__free(struct btf_dump *d); LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id); @@ -403,7 +459,8 @@ static inline bool btf_is_mod(const struct btf_type *t) return kind == BTF_KIND_VOLATILE || kind == BTF_KIND_CONST || - kind == BTF_KIND_RESTRICT; + kind == BTF_KIND_RESTRICT || + kind == BTF_KIND_TYPE_TAG; } static inline bool btf_is_func(const struct btf_type *t) @@ -436,6 +493,11 @@ static inline bool btf_is_decl_tag(const struct btf_type *t) return btf_kind(t) == BTF_KIND_DECL_TAG; } +static inline bool btf_is_type_tag(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_TYPE_TAG; +} + static inline __u8 btf_int_encoding(const struct btf_type *t) { return BTF_INT_ENCODING(*(__u32 *)(t + 1)); diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 17db62b5002e..05f3e7dfec0a 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -77,9 +77,8 @@ struct btf_dump_data { struct btf_dump { const struct btf *btf; - const struct btf_ext *btf_ext; btf_dump_printf_fn_t printf_fn; - struct btf_dump_opts opts; + void *cb_ctx; int ptr_sz; bool strip_mods; bool skip_anon_defs; @@ -138,29 +137,32 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) va_list args; va_start(args, fmt); - d->printf_fn(d->opts.ctx, fmt, args); + d->printf_fn(d->cb_ctx, fmt, args); va_end(args); } static int btf_dump_mark_referenced(struct btf_dump *d); static int btf_dump_resize(struct btf_dump *d); -struct btf_dump *btf_dump__new(const struct btf *btf, - const struct btf_ext *btf_ext, - const struct btf_dump_opts *opts, - btf_dump_printf_fn_t printf_fn) +DEFAULT_VERSION(btf_dump__new_v0_6_0, btf_dump__new, LIBBPF_0.6.0) +struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf, + btf_dump_printf_fn_t printf_fn, + void *ctx, + const struct btf_dump_opts *opts) { struct btf_dump *d; int err; + if (!printf_fn) + return libbpf_err_ptr(-EINVAL); + d = calloc(1, sizeof(struct btf_dump)); if (!d) return libbpf_err_ptr(-ENOMEM); d->btf = btf; - d->btf_ext = btf_ext; d->printf_fn = printf_fn; - d->opts.ctx = opts ? opts->ctx : NULL; + d->cb_ctx = ctx; d->ptr_sz = btf__pointer_size(btf) ? : sizeof(void *); d->type_names = hashmap__new(str_hash_fn, str_equal_fn, NULL); @@ -186,6 +188,17 @@ err: return libbpf_err_ptr(err); } +COMPAT_VERSION(btf_dump__new_deprecated, btf_dump__new, LIBBPF_0.0.4) +struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, + const struct btf_ext *btf_ext, + const struct btf_dump_opts *opts, + btf_dump_printf_fn_t printf_fn) +{ + if (!printf_fn) + return libbpf_err_ptr(-EINVAL); + return btf_dump__new_v0_6_0(btf, printf_fn, opts ? opts->ctx : NULL, opts); +} + static int btf_dump_resize(struct btf_dump *d) { int err, last_id = btf__type_cnt(d->btf) - 1; @@ -317,6 +330,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) case BTF_KIND_FUNC: case BTF_KIND_VAR: case BTF_KIND_DECL_TAG: + case BTF_KIND_TYPE_TAG: d->type_states[t->type].referenced = 1; break; @@ -560,6 +574,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) case BTF_KIND_VOLATILE: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: return btf_dump_order_type(d, t->type, through_ptr); case BTF_KIND_FUNC_PROTO: { @@ -734,6 +749,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) case BTF_KIND_VOLATILE: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: + case BTF_KIND_TYPE_TAG: btf_dump_emit_type(d, t->type, cont_id); break; case BTF_KIND_ARRAY: @@ -1154,6 +1170,7 @@ skip_mod: case BTF_KIND_CONST: case BTF_KIND_RESTRICT: case BTF_KIND_FUNC_PROTO: + case BTF_KIND_TYPE_TAG: id = t->type; break; case BTF_KIND_ARRAY: @@ -1322,6 +1339,11 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, case BTF_KIND_RESTRICT: btf_dump_printf(d, " restrict"); break; + case BTF_KIND_TYPE_TAG: + btf_dump_emit_mods(d, decls); + name = btf_name_of(d, t->name_off); + btf_dump_printf(d, " __attribute__((btf_type_tag(\"%s\")))", name); + break; case BTF_KIND_ARRAY: { const struct btf_array *a = btf_array(t); const struct btf_type *next_t; diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index 9934851ccde7..c28ab24d1bff 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -597,8 +597,9 @@ void bpf_gen__record_extern(struct bpf_gen *gen, const char *name, bool is_weak, static struct ksym_desc *get_ksym_desc(struct bpf_gen *gen, struct ksym_relo_desc *relo) { struct ksym_desc *kdesc; + int i; - for (int i = 0; i < gen->nr_ksyms; i++) { + for (i = 0; i < gen->nr_ksyms; i++) { if (!strcmp(gen->ksyms[i].name, relo->name)) { gen->ksyms[i].ref++; return &gen->ksyms[i]; @@ -913,27 +914,27 @@ static void cleanup_relos(struct bpf_gen *gen, int insns) } void bpf_gen__prog_load(struct bpf_gen *gen, - struct bpf_prog_load_params *load_attr, int prog_idx) + enum bpf_prog_type prog_type, const char *prog_name, + const char *license, struct bpf_insn *insns, size_t insn_cnt, + struct bpf_prog_load_opts *load_attr, int prog_idx) { int attr_size = offsetofend(union bpf_attr, fd_array); - int prog_load_attr, license, insns, func_info, line_info; + int prog_load_attr, license_off, insns_off, func_info, line_info; union bpf_attr attr; memset(&attr, 0, attr_size); - pr_debug("gen: prog_load: type %d insns_cnt %zd\n", - load_attr->prog_type, load_attr->insn_cnt); + pr_debug("gen: prog_load: type %d insns_cnt %zd\n", prog_type, insn_cnt); /* add license string to blob of bytes */ - license = add_data(gen, load_attr->license, strlen(load_attr->license) + 1); + license_off = add_data(gen, license, strlen(license) + 1); /* add insns to blob of bytes */ - insns = add_data(gen, load_attr->insns, - load_attr->insn_cnt * sizeof(struct bpf_insn)); + insns_off = add_data(gen, insns, insn_cnt * sizeof(struct bpf_insn)); - attr.prog_type = load_attr->prog_type; + attr.prog_type = prog_type; attr.expected_attach_type = load_attr->expected_attach_type; attr.attach_btf_id = load_attr->attach_btf_id; attr.prog_ifindex = load_attr->prog_ifindex; attr.kern_version = 0; - attr.insn_cnt = (__u32)load_attr->insn_cnt; + attr.insn_cnt = (__u32)insn_cnt; attr.prog_flags = load_attr->prog_flags; attr.func_info_rec_size = load_attr->func_info_rec_size; @@ -946,15 +947,15 @@ void bpf_gen__prog_load(struct bpf_gen *gen, line_info = add_data(gen, load_attr->line_info, attr.line_info_cnt * attr.line_info_rec_size); - memcpy(attr.prog_name, load_attr->name, - min((unsigned)strlen(load_attr->name), BPF_OBJ_NAME_LEN - 1)); + memcpy(attr.prog_name, prog_name, + min((unsigned)strlen(prog_name), BPF_OBJ_NAME_LEN - 1)); prog_load_attr = add_data(gen, &attr, attr_size); /* populate union bpf_attr with a pointer to license */ - emit_rel_store(gen, attr_field(prog_load_attr, license), license); + emit_rel_store(gen, attr_field(prog_load_attr, license), license_off); /* populate union bpf_attr with a pointer to instructions */ - emit_rel_store(gen, attr_field(prog_load_attr, insns), insns); + emit_rel_store(gen, attr_field(prog_load_attr, insns), insns_off); /* populate union bpf_attr with a pointer to func_info */ emit_rel_store(gen, attr_field(prog_load_attr, func_info), func_info); @@ -986,12 +987,12 @@ void bpf_gen__prog_load(struct bpf_gen *gen, emit(gen, BPF_STX_MEM(BPF_W, BPF_REG_0, BPF_REG_7, offsetof(union bpf_attr, attach_btf_obj_fd))); } - emit_relos(gen, insns); + emit_relos(gen, insns_off); /* emit PROG_LOAD command */ emit_sys_bpf(gen, BPF_PROG_LOAD, prog_load_attr, attr_size); debug_ret(gen, "prog_load %s insn_cnt %d", attr.prog_name, attr.insn_cnt); /* successful or not, close btf module FDs used in extern ksyms and attach_btf_obj_fd */ - cleanup_relos(gen, insns); + cleanup_relos(gen, insns_off); if (gen->attach_kind) emit_sys_close_blob(gen, attr_field(prog_load_attr, attach_btf_obj_fd)); diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 7c74342bb668..f6faa33c80fa 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -197,6 +197,8 @@ enum kern_feature_id { FEAT_PERF_LINK, /* BTF_KIND_DECL_TAG support */ FEAT_BTF_DECL_TAG, + /* BTF_KIND_TYPE_TAG support */ + FEAT_BTF_TYPE_TAG, __FEAT_CNT, }; @@ -221,7 +223,7 @@ struct reloc_desc { struct bpf_sec_def; typedef int (*init_fn_t)(struct bpf_program *prog, long cookie); -typedef int (*preload_fn_t)(struct bpf_program *prog, struct bpf_prog_load_params *attr, long cookie); +typedef int (*preload_fn_t)(struct bpf_program *prog, struct bpf_prog_load_opts *opts, long cookie); typedef struct bpf_link *(*attach_fn_t)(const struct bpf_program *prog, long cookie); /* stored as sec_def->cookie for all libbpf-supported SEC()s */ @@ -2076,6 +2078,7 @@ static const char *__btf_kind_str(__u16 kind) case BTF_KIND_DATASEC: return "datasec"; case BTF_KIND_FLOAT: return "float"; case BTF_KIND_DECL_TAG: return "decl_tag"; + case BTF_KIND_TYPE_TAG: return "type_tag"; default: return "unknown"; } } @@ -2588,8 +2591,10 @@ static bool btf_needs_sanitization(struct bpf_object *obj) bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); + bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); - return !has_func || !has_datasec || !has_func_global || !has_float || !has_decl_tag; + return !has_func || !has_datasec || !has_func_global || !has_float || + !has_decl_tag || !has_type_tag; } static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) @@ -2599,6 +2604,7 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) bool has_float = kernel_supports(obj, FEAT_BTF_FLOAT); bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); + bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); struct btf_type *t; int i, j, vlen; @@ -2657,6 +2663,10 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) */ t->name_off = 0; t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0); + } else if (!has_type_tag && btf_is_type_tag(t)) { + /* replace TYPE_TAG with a CONST */ + t->name_off = 0; + t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); } } } @@ -2752,13 +2762,12 @@ static int btf_fixup_datasec(struct bpf_object *obj, struct btf *btf, for (i = 0, vsi = btf_var_secinfos(t); i < vars; i++, vsi++) { t_var = btf__type_by_id(btf, vsi->type); - var = btf_var(t_var); - - if (!btf_is_var(t_var)) { + if (!t_var || !btf_is_var(t_var)) { pr_debug("Non-VAR type seen in section %s\n", name); return -EINVAL; } + var = btf_var(t_var); if (var->linkage == BTF_VAR_STATIC) continue; @@ -3191,11 +3200,11 @@ static int bpf_object__elf_collect(struct bpf_object *obj) Elf_Scn *scn; Elf64_Shdr *sh; - /* ELF section indices are 1-based, so allocate +1 element to keep - * indexing simple. Also include 0th invalid section into sec_cnt for - * simpler and more traditional iteration logic. + /* ELF section indices are 0-based, but sec #0 is special "invalid" + * section. e_shnum does include sec #0, so e_shnum is the necessary + * size of an array to keep all the sections. */ - obj->efile.sec_cnt = 1 + obj->efile.ehdr->e_shnum; + obj->efile.sec_cnt = obj->efile.ehdr->e_shnum; obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs)); if (!obj->efile.secs) return -ENOMEM; @@ -3271,8 +3280,12 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (strcmp(name, MAPS_ELF_SEC) == 0) { obj->efile.btf_maps_shndx = idx; } else if (strcmp(name, BTF_ELF_SEC) == 0) { + if (sh->sh_type != SHT_PROGBITS) + return -LIBBPF_ERRNO__FORMAT; btf_data = data; } else if (strcmp(name, BTF_EXT_ELF_SEC) == 0) { + if (sh->sh_type != SHT_PROGBITS) + return -LIBBPF_ERRNO__FORMAT; btf_ext_data = data; } else if (sh->sh_type == SHT_SYMTAB) { /* already processed during the first pass above */ @@ -3303,6 +3316,10 @@ static int bpf_object__elf_collect(struct bpf_object *obj) } else if (sh->sh_type == SHT_REL) { int targ_sec_idx = sh->sh_info; /* points to other section */ + if (sh->sh_entsize != sizeof(Elf64_Rel) || + targ_sec_idx >= obj->efile.sec_cnt) + return -LIBBPF_ERRNO__FORMAT; + /* Only do relo for section with exec instructions */ if (!section_have_execinstr(obj, targ_sec_idx) && strcmp(name, ".rel" STRUCT_OPS_SEC) && @@ -3555,7 +3572,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx); sh = elf_sec_hdr(obj, scn); - if (!sh) + if (!sh || sh->sh_entsize != sizeof(Elf64_Sym)) return -LIBBPF_ERRNO__FORMAT; dummy_var_btf_id = add_dummy_ksym_var(obj->btf); @@ -4022,7 +4039,7 @@ static int bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Data *data) { const char *relo_sec_name, *sec_name; - size_t sec_idx = shdr->sh_info; + size_t sec_idx = shdr->sh_info, sym_idx; struct bpf_program *prog; struct reloc_desc *relos; int err, i, nrels; @@ -4033,6 +4050,9 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Dat Elf64_Sym *sym; Elf64_Rel *rel; + if (sec_idx >= obj->efile.sec_cnt) + return -EINVAL; + scn = elf_sec_by_idx(obj, sec_idx); scn_data = elf_sec_data(obj, scn); @@ -4052,16 +4072,23 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Dat return -LIBBPF_ERRNO__FORMAT; } - sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info)); + sym_idx = ELF64_R_SYM(rel->r_info); + sym = elf_sym_by_idx(obj, sym_idx); if (!sym) { - pr_warn("sec '%s': symbol 0x%zx not found for relo #%d\n", - relo_sec_name, (size_t)ELF64_R_SYM(rel->r_info), i); + pr_warn("sec '%s': symbol #%zu not found for relo #%d\n", + relo_sec_name, sym_idx, i); + return -LIBBPF_ERRNO__FORMAT; + } + + if (sym->st_shndx >= obj->efile.sec_cnt) { + pr_warn("sec '%s': corrupted symbol #%zu pointing to invalid section #%zu for relo #%d\n", + relo_sec_name, sym_idx, (size_t)sym->st_shndx, i); return -LIBBPF_ERRNO__FORMAT; } if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) { pr_warn("sec '%s': invalid offset 0x%zx for relo #%d\n", - relo_sec_name, (size_t)ELF64_R_SYM(rel->r_info), i); + relo_sec_name, (size_t)rel->r_offset, i); return -LIBBPF_ERRNO__FORMAT; } @@ -4265,30 +4292,20 @@ int bpf_map__resize(struct bpf_map *map, __u32 max_entries) static int bpf_object__probe_loading(struct bpf_object *obj) { - struct bpf_load_program_attr attr; char *cp, errmsg[STRERR_BUFSIZE]; struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - int ret; + int ret, insn_cnt = ARRAY_SIZE(insns); if (obj->gen_loader) return 0; /* make sure basic loading works */ - - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.insns = insns; - attr.insns_cnt = ARRAY_SIZE(insns); - attr.license = "GPL"; - - ret = bpf_load_program_xattr(&attr, NULL, 0); - if (ret < 0) { - attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; - ret = bpf_load_program_xattr(&attr, NULL, 0); - } + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); + if (ret < 0) + ret = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); if (ret < 0) { ret = errno; cp = libbpf_strerror_r(ret, errmsg, sizeof(errmsg)); @@ -4312,28 +4329,19 @@ static int probe_fd(int fd) static int probe_kern_prog_name(void) { - struct bpf_load_program_attr attr; struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - int ret; + int ret, insn_cnt = ARRAY_SIZE(insns); /* make sure loading with name works */ - - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - attr.insns = insns; - attr.insns_cnt = ARRAY_SIZE(insns); - attr.license = "GPL"; - attr.name = "test"; - ret = bpf_load_program_xattr(&attr, NULL, 0); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "test", "GPL", insns, insn_cnt, NULL); return probe_fd(ret); } static int probe_kern_global_data(void) { - struct bpf_load_program_attr prg_attr; struct bpf_create_map_attr map_attr; char *cp, errmsg[STRERR_BUFSIZE]; struct bpf_insn insns[] = { @@ -4342,7 +4350,7 @@ static int probe_kern_global_data(void) BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - int ret, map; + int ret, map, insn_cnt = ARRAY_SIZE(insns); memset(&map_attr, 0, sizeof(map_attr)); map_attr.map_type = BPF_MAP_TYPE_ARRAY; @@ -4361,13 +4369,7 @@ static int probe_kern_global_data(void) insns[0].imm = map; - memset(&prg_attr, 0, sizeof(prg_attr)); - prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - prg_attr.insns = insns; - prg_attr.insns_cnt = ARRAY_SIZE(insns); - prg_attr.license = "GPL"; - - ret = bpf_load_program_xattr(&prg_attr, NULL, 0); + ret = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); close(map); return probe_fd(ret); } @@ -4468,6 +4470,22 @@ static int probe_kern_btf_decl_tag(void) strs, sizeof(strs))); } +static int probe_kern_btf_type_tag(void) +{ + static const char strs[] = "\0tag"; + __u32 types[] = { + /* int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + /* attr */ + BTF_TYPE_TYPE_TAG_ENC(1, 1), /* [2] */ + /* ptr */ + BTF_TYPE_ENC(0, BTF_INFO_ENC(BTF_KIND_PTR, 0, 0), 2), /* [3] */ + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + static int probe_kern_array_mmap(void) { struct bpf_create_map_attr attr = { @@ -4483,30 +4501,24 @@ static int probe_kern_array_mmap(void) static int probe_kern_exp_attach_type(void) { - struct bpf_load_program_attr attr; + LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE); struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; + int fd, insn_cnt = ARRAY_SIZE(insns); - memset(&attr, 0, sizeof(attr)); /* use any valid combination of program type and (optional) * non-zero expected attach type (i.e., not a BPF_CGROUP_INET_INGRESS) * to see if kernel supports expected_attach_type field for * BPF_PROG_LOAD command */ - attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; - attr.expected_attach_type = BPF_CGROUP_INET_SOCK_CREATE; - attr.insns = insns; - attr.insns_cnt = ARRAY_SIZE(insns); - attr.license = "GPL"; - - return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); + fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", insns, insn_cnt, &opts); + return probe_fd(fd); } static int probe_kern_probe_read_kernel(void) { - struct bpf_load_program_attr attr; struct bpf_insn insns[] = { BPF_MOV64_REG(BPF_REG_1, BPF_REG_10), /* r1 = r10 (fp) */ BPF_ALU64_IMM(BPF_ADD, BPF_REG_1, -8), /* r1 += -8 */ @@ -4515,26 +4527,21 @@ static int probe_kern_probe_read_kernel(void) BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_probe_read_kernel), BPF_EXIT_INSN(), }; + int fd, insn_cnt = ARRAY_SIZE(insns); - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_KPROBE; - attr.insns = insns; - attr.insns_cnt = ARRAY_SIZE(insns); - attr.license = "GPL"; - - return probe_fd(bpf_load_program_xattr(&attr, NULL, 0)); + fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); + return probe_fd(fd); } static int probe_prog_bind_map(void) { - struct bpf_load_program_attr prg_attr; struct bpf_create_map_attr map_attr; char *cp, errmsg[STRERR_BUFSIZE]; struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - int ret, map, prog; + int ret, map, prog, insn_cnt = ARRAY_SIZE(insns); memset(&map_attr, 0, sizeof(map_attr)); map_attr.map_type = BPF_MAP_TYPE_ARRAY; @@ -4551,13 +4558,7 @@ static int probe_prog_bind_map(void) return ret; } - memset(&prg_attr, 0, sizeof(prg_attr)); - prg_attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; - prg_attr.insns = insns; - prg_attr.insns_cnt = ARRAY_SIZE(insns); - prg_attr.license = "GPL"; - - prog = bpf_load_program_xattr(&prg_attr, NULL, 0); + prog = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", insns, insn_cnt, NULL); if (prog < 0) { close(map); return 0; @@ -4602,19 +4603,14 @@ static int probe_module_btf(void) static int probe_perf_link(void) { - struct bpf_load_program_attr attr; struct bpf_insn insns[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; int prog_fd, link_fd, err; - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_TRACEPOINT; - attr.insns = insns; - attr.insns_cnt = ARRAY_SIZE(insns); - attr.license = "GPL"; - prog_fd = bpf_load_program_xattr(&attr, NULL, 0); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", + insns, ARRAY_SIZE(insns), NULL); if (prog_fd < 0) return -errno; @@ -4687,6 +4683,9 @@ static struct kern_feature_desc { [FEAT_BTF_DECL_TAG] = { "BTF_KIND_DECL_TAG support", probe_kern_btf_decl_tag, }, + [FEAT_BTF_TYPE_TAG] = { + "BTF_KIND_TYPE_TAG support", probe_kern_btf_type_tag, + }, }; static bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) @@ -6374,16 +6373,16 @@ static int libbpf_find_attach_btf_id(struct bpf_program *prog, const char *attac /* this is called as prog->sec_def->preload_fn for libbpf-supported sec_defs */ static int libbpf_preload_prog(struct bpf_program *prog, - struct bpf_prog_load_params *attr, long cookie) + struct bpf_prog_load_opts *opts, long cookie) { enum sec_def_flags def = cookie; /* old kernels might not support specifying expected_attach_type */ if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE)) - attr->expected_attach_type = 0; + opts->expected_attach_type = 0; if (def & SEC_SLEEPABLE) - attr->prog_flags |= BPF_F_SLEEPABLE; + opts->prog_flags |= BPF_F_SLEEPABLE; if ((prog->type == BPF_PROG_TYPE_TRACING || prog->type == BPF_PROG_TYPE_LSM || @@ -6402,21 +6401,22 @@ static int libbpf_preload_prog(struct bpf_program *prog, /* but by now libbpf common logic is not utilizing * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because - * this callback is called after attrs were populated by - * libbpf, so this callback has to update attr explicitly here + * this callback is called after opts were populated by + * libbpf, so this callback has to update opts explicitly here */ - attr->attach_btf_obj_fd = btf_obj_fd; - attr->attach_btf_id = btf_type_id; + opts->attach_btf_obj_fd = btf_obj_fd; + opts->attach_btf_id = btf_type_id; } return 0; } -static int -load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, - char *license, __u32 kern_version, int *pfd) +static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_insn *insns, int insns_cnt, + const char *license, __u32 kern_version, + int *prog_fd) { - struct bpf_prog_load_params load_attr = {}; - struct bpf_object *obj = prog->obj; + LIBBPF_OPTS(bpf_prog_load_opts, load_attr); + const char *prog_name = NULL; char *cp, errmsg[STRERR_BUFSIZE]; size_t log_buf_size = 0; char *log_buf = NULL; @@ -6435,13 +6435,9 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, if (!insns || !insns_cnt) return -EINVAL; - load_attr.prog_type = prog->type; load_attr.expected_attach_type = prog->expected_attach_type; if (kernel_supports(obj, FEAT_PROG_NAME)) - load_attr.name = prog->name; - load_attr.insns = insns; - load_attr.insn_cnt = insns_cnt; - load_attr.license = license; + prog_name = prog->name; load_attr.attach_btf_id = prog->attach_btf_id; load_attr.attach_prog_fd = prog->attach_prog_fd; load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd; @@ -6475,9 +6471,10 @@ load_program(struct bpf_program *prog, struct bpf_insn *insns, int insns_cnt, } if (obj->gen_loader) { - bpf_gen__prog_load(obj->gen_loader, &load_attr, + bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name, + license, insns, insns_cnt, &load_attr, prog - obj->programs); - *pfd = -1; + *prog_fd = -1; return 0; } retry_load: @@ -6490,8 +6487,8 @@ retry_load: } load_attr.log_buf = log_buf; - load_attr.log_buf_sz = log_buf_size; - ret = libbpf__bpf_prog_load(&load_attr); + load_attr.log_size = log_buf_size; + ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr); if (ret >= 0) { if (log_buf && load_attr.log_level) @@ -6515,7 +6512,7 @@ retry_load: } } - *pfd = ret; + *prog_fd = ret; ret = 0; goto out; } @@ -6537,19 +6534,19 @@ retry_load: pr_warn("-- BEGIN DUMP LOG ---\n"); pr_warn("\n%s\n", log_buf); pr_warn("-- END LOG --\n"); - } else if (load_attr.insn_cnt >= BPF_MAXINSNS) { - pr_warn("Program too large (%zu insns), at most %d insns\n", - load_attr.insn_cnt, BPF_MAXINSNS); + } else if (insns_cnt >= BPF_MAXINSNS) { + pr_warn("Program too large (%d insns), at most %d insns\n", + insns_cnt, BPF_MAXINSNS); ret = -LIBBPF_ERRNO__PROG2BIG; - } else if (load_attr.prog_type != BPF_PROG_TYPE_KPROBE) { + } else if (prog->type != BPF_PROG_TYPE_KPROBE) { /* Wrong program type? */ int fd; - load_attr.prog_type = BPF_PROG_TYPE_KPROBE; load_attr.expected_attach_type = 0; load_attr.log_buf = NULL; - load_attr.log_buf_sz = 0; - fd = libbpf__bpf_prog_load(&load_attr); + load_attr.log_size = 0; + fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, prog_name, license, + insns, insns_cnt, &load_attr); if (fd >= 0) { close(fd); ret = -LIBBPF_ERRNO__PROGTYPE; @@ -6591,11 +6588,12 @@ static int bpf_program__record_externs(struct bpf_program *prog) return 0; } -int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) +static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, + const char *license, __u32 kern_ver) { int err = 0, fd, i; - if (prog->obj->loaded) { + if (obj->loaded) { pr_warn("prog '%s': can't load after object was loaded\n", prog->name); return libbpf_err(-EINVAL); } @@ -6621,10 +6619,11 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) pr_warn("prog '%s': inconsistent nr(%d) != 1\n", prog->name, prog->instances.nr); } - if (prog->obj->gen_loader) + if (obj->gen_loader) bpf_program__record_externs(prog); - err = load_program(prog, prog->insns, prog->insns_cnt, - license, kern_ver, &fd); + err = bpf_object_load_prog_instance(obj, prog, + prog->insns, prog->insns_cnt, + license, kern_ver, &fd); if (!err) prog->instances.fds[0] = fd; goto out; @@ -6652,8 +6651,9 @@ int bpf_program__load(struct bpf_program *prog, char *license, __u32 kern_ver) continue; } - err = load_program(prog, result.new_insn_ptr, - result.new_insn_cnt, license, kern_ver, &fd); + err = bpf_object_load_prog_instance(obj, prog, + result.new_insn_ptr, result.new_insn_cnt, + license, kern_ver, &fd); if (err) { pr_warn("Loading the %dth instance of program '%s' failed\n", i, prog->name); @@ -6670,6 +6670,11 @@ out: return libbpf_err(err); } +int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_ver) +{ + return bpf_object_load_prog(prog->obj, prog, license, kern_ver); +} + static int bpf_object__load_progs(struct bpf_object *obj, int log_level) { @@ -6693,7 +6698,7 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) continue; } prog->log_level |= log_level; - err = bpf_program__load(prog, obj->license, obj->kern_version); + err = bpf_object_load_prog(obj, prog, obj->license, obj->kern_version); if (err) return err; } @@ -7733,7 +7738,7 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path) return 0; err_unpin_maps: - while ((map = bpf_map__prev(map, obj))) { + while ((map = bpf_object__prev_map(obj, map))) { if (!map->pin_path) continue; @@ -7813,7 +7818,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) return 0; err_unpin_programs: - while ((prog = bpf_program__prev(prog, obj))) { + while ((prog = bpf_object__prev_program(obj, prog))) { char buf[PATH_MAX]; int len; @@ -8154,9 +8159,11 @@ int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) return 0; } +static int bpf_program_nth_fd(const struct bpf_program *prog, int n); + int bpf_program__fd(const struct bpf_program *prog) { - return bpf_program__nth_fd(prog, 0); + return bpf_program_nth_fd(prog, 0); } size_t bpf_program__size(const struct bpf_program *prog) @@ -8202,7 +8209,10 @@ int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, return 0; } -int bpf_program__nth_fd(const struct bpf_program *prog, int n) +__attribute__((alias("bpf_program_nth_fd"))) +int bpf_program__nth_fd(const struct bpf_program *prog, int n); + +static int bpf_program_nth_fd(const struct bpf_program *prog, int n) { int fd; @@ -8281,6 +8291,20 @@ void bpf_program__set_expected_attach_type(struct bpf_program *prog, prog->expected_attach_type = type; } +__u32 bpf_program__flags(const struct bpf_program *prog) +{ + return prog->prog_flags; +} + +int bpf_program__set_extra_flags(struct bpf_program *prog, __u32 extra_flags) +{ + if (prog->obj->loaded) + return libbpf_err(-EBUSY); + + prog->prog_flags |= extra_flags; + return 0; +} + #define SEC_DEF(sec_pfx, ptype, atype, flags, ...) { \ .sec = sec_pfx, \ .prog_type = BPF_PROG_TYPE_##ptype, \ @@ -9028,7 +9052,10 @@ int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd) pr_warn("error: inner_map_fd already specified\n"); return libbpf_err(-EINVAL); } - zfree(&map->inner_map); + if (map->inner_map) { + bpf_map__destroy(map->inner_map); + zfree(&map->inner_map); + } map->inner_map_fd = fd; return 0; } @@ -9145,21 +9172,12 @@ long libbpf_get_error(const void *ptr) return -errno; } -int bpf_prog_load(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd) -{ - struct bpf_prog_load_attr attr; - - memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); - attr.file = file; - attr.prog_type = type; - attr.expected_attach_type = 0; - - return bpf_prog_load_xattr(&attr, pobj, prog_fd); -} - +__attribute__((alias("bpf_prog_load_xattr2"))) int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, - struct bpf_object **pobj, int *prog_fd) + struct bpf_object **pobj, int *prog_fd); + +static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr, + struct bpf_object **pobj, int *prog_fd) { struct bpf_object_open_attr open_attr = {}; struct bpf_program *prog, *first_prog = NULL; @@ -9230,6 +9248,20 @@ int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, return 0; } +COMPAT_VERSION(bpf_prog_load_deprecated, bpf_prog_load, LIBBPF_0.0.1) +int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type, + struct bpf_object **pobj, int *prog_fd) +{ + struct bpf_prog_load_attr attr; + + memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); + attr.file = file; + attr.prog_type = type; + attr.expected_attach_type = 0; + + return bpf_prog_load_xattr2(&attr, pobj, prog_fd); +} + struct bpf_link { int (*detach)(struct bpf_link *link); void (*dealloc)(struct bpf_link *link); @@ -10575,11 +10607,18 @@ error: static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, struct perf_buffer_params *p); -struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, - const struct perf_buffer_opts *opts) +DEFAULT_VERSION(perf_buffer__new_v0_6_0, perf_buffer__new, LIBBPF_0.6.0) +struct perf_buffer *perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt, + perf_buffer_sample_fn sample_cb, + perf_buffer_lost_fn lost_cb, + void *ctx, + const struct perf_buffer_opts *opts) { struct perf_buffer_params p = {}; - struct perf_event_attr attr = { 0, }; + struct perf_event_attr attr = {}; + + if (!OPTS_VALID(opts, perf_buffer_opts)) + return libbpf_err_ptr(-EINVAL); attr.config = PERF_COUNT_SW_BPF_OUTPUT; attr.type = PERF_TYPE_SOFTWARE; @@ -10588,29 +10627,62 @@ struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, attr.wakeup_events = 1; p.attr = &attr; - p.sample_cb = opts ? opts->sample_cb : NULL; - p.lost_cb = opts ? opts->lost_cb : NULL; - p.ctx = opts ? opts->ctx : NULL; + p.sample_cb = sample_cb; + p.lost_cb = lost_cb; + p.ctx = ctx; return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); } -struct perf_buffer * -perf_buffer__new_raw(int map_fd, size_t page_cnt, - const struct perf_buffer_raw_opts *opts) +COMPAT_VERSION(perf_buffer__new_deprecated, perf_buffer__new, LIBBPF_0.0.4) +struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt, + const struct perf_buffer_opts *opts) +{ + return perf_buffer__new_v0_6_0(map_fd, page_cnt, + opts ? opts->sample_cb : NULL, + opts ? opts->lost_cb : NULL, + opts ? opts->ctx : NULL, + NULL); +} + +DEFAULT_VERSION(perf_buffer__new_raw_v0_6_0, perf_buffer__new_raw, LIBBPF_0.6.0) +struct perf_buffer *perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, + struct perf_event_attr *attr, + perf_buffer_event_fn event_cb, void *ctx, + const struct perf_buffer_raw_opts *opts) { struct perf_buffer_params p = {}; - p.attr = opts->attr; - p.event_cb = opts->event_cb; - p.ctx = opts->ctx; - p.cpu_cnt = opts->cpu_cnt; - p.cpus = opts->cpus; - p.map_keys = opts->map_keys; + if (page_cnt == 0 || !attr) + return libbpf_err_ptr(-EINVAL); + + if (!OPTS_VALID(opts, perf_buffer_raw_opts)) + return libbpf_err_ptr(-EINVAL); + + p.attr = attr; + p.event_cb = event_cb; + p.ctx = ctx; + p.cpu_cnt = OPTS_GET(opts, cpu_cnt, 0); + p.cpus = OPTS_GET(opts, cpus, NULL); + p.map_keys = OPTS_GET(opts, map_keys, NULL); return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); } +COMPAT_VERSION(perf_buffer__new_raw_deprecated, perf_buffer__new_raw, LIBBPF_0.0.4) +struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt, + const struct perf_buffer_raw_opts *opts) +{ + LIBBPF_OPTS(perf_buffer_raw_opts, inner_opts, + .cpu_cnt = opts->cpu_cnt, + .cpus = opts->cpus, + .map_keys = opts->map_keys, + ); + + return perf_buffer__new_raw_v0_6_0(map_fd, page_cnt, opts->attr, + opts->event_cb, opts->ctx, &inner_opts); +} + static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, struct perf_buffer_params *p) { diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 9de0f299706b..4ec69f224342 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -262,8 +262,8 @@ LIBBPF_API const struct bpf_insn *bpf_program__insns(const struct bpf_program *p */ LIBBPF_API size_t bpf_program__insn_cnt(const struct bpf_program *prog); -LIBBPF_API int bpf_program__load(struct bpf_program *prog, char *license, - __u32 kern_version); +LIBBPF_DEPRECATED_SINCE(0, 6, "use bpf_object__load() instead") +LIBBPF_API int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_version); LIBBPF_API int bpf_program__fd(const struct bpf_program *prog); LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated") LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog, @@ -431,7 +431,6 @@ bpf_program__attach_iter(const struct bpf_program *prog, * one instance. In this case bpf_program__fd(prog) is equal to * bpf_program__nth_fd(prog, 0). */ -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__insns() for getting bpf_program instructions") struct bpf_prog_prep_result { /* * If not NULL, load new instruction array. @@ -494,6 +493,9 @@ LIBBPF_API void bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type); +LIBBPF_API __u32 bpf_program__flags(const struct bpf_program *prog); +LIBBPF_API int bpf_program__set_extra_flags(struct bpf_program *prog, __u32 extra_flags); + LIBBPF_API int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, const char *attach_func_name); @@ -676,8 +678,9 @@ struct bpf_prog_load_attr { LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, struct bpf_object **pobj, int *prog_fd); -LIBBPF_API int bpf_prog_load(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd); +LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open() and bpf_object__load() instead") +LIBBPF_API int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type, + struct bpf_object **pobj, int *prog_fd); /* XDP related API */ struct xdp_link_info { @@ -775,18 +778,52 @@ typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt); /* common use perf buffer options */ struct perf_buffer_opts { - /* if specified, sample_cb is called for each sample */ - perf_buffer_sample_fn sample_cb; - /* if specified, lost_cb is called for each batch of lost samples */ - perf_buffer_lost_fn lost_cb; - /* ctx is provided to sample_cb and lost_cb */ - void *ctx; + union { + size_t sz; + struct { /* DEPRECATED: will be removed in v1.0 */ + /* if specified, sample_cb is called for each sample */ + perf_buffer_sample_fn sample_cb; + /* if specified, lost_cb is called for each batch of lost samples */ + perf_buffer_lost_fn lost_cb; + /* ctx is provided to sample_cb and lost_cb */ + void *ctx; + }; + }; }; +#define perf_buffer_opts__last_field sz +/** + * @brief **perf_buffer__new()** creates BPF perfbuf manager for a specified + * BPF_PERF_EVENT_ARRAY map + * @param map_fd FD of BPF_PERF_EVENT_ARRAY BPF map that will be used by BPF + * code to send data over to user-space + * @param page_cnt number of memory pages allocated for each per-CPU buffer + * @param sample_cb function called on each received data record + * @param lost_cb function called when record loss has occurred + * @param ctx user-provided extra context passed into *sample_cb* and *lost_cb* + * @return a new instance of struct perf_buffer on success, NULL on error with + * *errno* containing an error code + */ LIBBPF_API struct perf_buffer * perf_buffer__new(int map_fd, size_t page_cnt, + perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx, const struct perf_buffer_opts *opts); +LIBBPF_API struct perf_buffer * +perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt, + perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx, + const struct perf_buffer_opts *opts); + +LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new() instead") +struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt, + const struct perf_buffer_opts *opts); + +#define perf_buffer__new(...) ___libbpf_overload(___perf_buffer_new, __VA_ARGS__) +#define ___perf_buffer_new6(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts) \ + perf_buffer__new(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts) +#define ___perf_buffer_new3(map_fd, page_cnt, opts) \ + perf_buffer__new_deprecated(map_fd, page_cnt, opts) + enum bpf_perf_event_ret { LIBBPF_PERF_EVENT_DONE = 0, LIBBPF_PERF_EVENT_ERROR = -1, @@ -800,12 +837,21 @@ typedef enum bpf_perf_event_ret /* raw perf buffer options, giving most power and control */ struct perf_buffer_raw_opts { - /* perf event attrs passed directly into perf_event_open() */ - struct perf_event_attr *attr; - /* raw event callback */ - perf_buffer_event_fn event_cb; - /* ctx is provided to event_cb */ - void *ctx; + union { + struct { + size_t sz; + long :0; + long :0; + }; + struct { /* DEPRECATED: will be removed in v1.0 */ + /* perf event attrs passed directly into perf_event_open() */ + struct perf_event_attr *attr; + /* raw event callback */ + perf_buffer_event_fn event_cb; + /* ctx is provided to event_cb */ + void *ctx; + }; + }; /* if cpu_cnt == 0, open all on all possible CPUs (up to the number of * max_entries of given PERF_EVENT_ARRAY map) */ @@ -815,11 +861,28 @@ struct perf_buffer_raw_opts { /* if cpu_cnt > 0, map_keys specify map keys to set per-CPU FDs for */ int *map_keys; }; +#define perf_buffer_raw_opts__last_field map_keys LIBBPF_API struct perf_buffer * -perf_buffer__new_raw(int map_fd, size_t page_cnt, +perf_buffer__new_raw(int map_fd, size_t page_cnt, struct perf_event_attr *attr, + perf_buffer_event_fn event_cb, void *ctx, const struct perf_buffer_raw_opts *opts); +LIBBPF_API struct perf_buffer * +perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, struct perf_event_attr *attr, + perf_buffer_event_fn event_cb, void *ctx, + const struct perf_buffer_raw_opts *opts); + +LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new_raw() instead") +struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt, + const struct perf_buffer_raw_opts *opts); + +#define perf_buffer__new_raw(...) ___libbpf_overload(___perf_buffer_new_raw, __VA_ARGS__) +#define ___perf_buffer_new_raw6(map_fd, page_cnt, attr, event_cb, ctx, opts) \ + perf_buffer__new_raw(map_fd, page_cnt, attr, event_cb, ctx, opts) +#define ___perf_buffer_new_raw3(map_fd, page_cnt, opts) \ + perf_buffer__new_raw_deprecated(map_fd, page_cnt, opts) + LIBBPF_API void perf_buffer__free(struct perf_buffer *pb); LIBBPF_API int perf_buffer__epoll_fd(const struct perf_buffer *pb); LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms); @@ -918,12 +981,15 @@ struct bpf_prog_info_linear { __u8 data[]; }; +LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") LIBBPF_API struct bpf_prog_info_linear * bpf_program__get_prog_info_linear(int fd, __u64 arrays); +LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") LIBBPF_API void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear); +LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") LIBBPF_API void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear); diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index 43580eb47740..6a59514a48cf 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -395,10 +395,23 @@ LIBBPF_0.6.0 { bpf_object__next_program; bpf_object__prev_map; bpf_object__prev_program; + bpf_prog_load_deprecated; + bpf_prog_load; + bpf_program__flags; bpf_program__insn_cnt; bpf_program__insns; + bpf_program__set_extra_flags; btf__add_btf; btf__add_decl_tag; + btf__add_type_tag; + btf__dedup; + btf__dedup_deprecated; btf__raw_data; btf__type_cnt; + btf_dump__new; + btf_dump__new_deprecated; + perf_buffer__new; + perf_buffer__new_deprecated; + perf_buffer__new_raw; + perf_buffer__new_raw_deprecated; } LIBBPF_0.5.0; diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h index aaa1efbf6f51..b21cefc9c3b6 100644 --- a/tools/lib/bpf/libbpf_common.h +++ b/tools/lib/bpf/libbpf_common.h @@ -41,6 +41,18 @@ #define __LIBBPF_MARK_DEPRECATED_0_7(X) #endif +/* This set of internal macros allows to do "function overloading" based on + * number of arguments provided by used in backwards-compatible way during the + * transition to libbpf 1.0 + * It's ugly but necessary evil that will be cleaned up when we get to 1.0. + * See bpf_prog_load() overload for example. + */ +#define ___libbpf_cat(A, B) A ## B +#define ___libbpf_select(NAME, NUM) ___libbpf_cat(NAME, NUM) +#define ___libbpf_nth(_1, _2, _3, _4, _5, _6, N, ...) N +#define ___libbpf_cnt(...) ___libbpf_nth(__VA_ARGS__, 6, 5, 4, 3, 2, 1) +#define ___libbpf_overload(NAME, ...) ___libbpf_select(NAME, ___libbpf_cnt(__VA_ARGS__))(__VA_ARGS__) + /* Helper macro to declare and initialize libbpf options struct * * This dance with uninitialized declaration, followed by memset to zero, @@ -54,7 +66,7 @@ * including any extra padding, it with memset() and then assigns initial * values provided by users in struct initializer-syntax as varargs. */ -#define DECLARE_LIBBPF_OPTS(TYPE, NAME, ...) \ +#define LIBBPF_OPTS(TYPE, NAME, ...) \ struct TYPE NAME = ({ \ memset(&NAME, 0, sizeof(struct TYPE)); \ (struct TYPE) { \ diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index aeb79e3a8ff9..f7ac349650a1 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -73,6 +73,8 @@ BTF_TYPE_ENC(name, BTF_INFO_ENC(BTF_KIND_FLOAT, 0, 0), sz) #define BTF_TYPE_DECL_TAG_ENC(value, type, component_idx) \ BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx) +#define BTF_TYPE_TYPE_TAG_ENC(value, type) \ + BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type) #ifndef likely #define likely(x) __builtin_expect(!!(x), 1) @@ -276,37 +278,6 @@ int parse_cpu_mask_file(const char *fcpu, bool **mask, int *mask_sz); int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len); -struct bpf_prog_load_params { - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - const char *name; - const struct bpf_insn *insns; - size_t insn_cnt; - const char *license; - __u32 kern_version; - __u32 attach_prog_fd; - __u32 attach_btf_obj_fd; - __u32 attach_btf_id; - __u32 prog_ifindex; - __u32 prog_btf_fd; - __u32 prog_flags; - - __u32 func_info_rec_size; - const void *func_info; - __u32 func_info_cnt; - - __u32 line_info_rec_size; - const void *line_info; - __u32 line_info_cnt; - - __u32 log_level; - char *log_buf; - size_t log_buf_sz; - int *fd_array; -}; - -int libbpf__bpf_prog_load(const struct bpf_prog_load_params *load_attr); - struct bpf_create_map_params { const char *name; enum bpf_map_type map_type; diff --git a/tools/lib/bpf/libbpf_legacy.h b/tools/lib/bpf/libbpf_legacy.h index 5ba5c9beccfa..bb03c568af7b 100644 --- a/tools/lib/bpf/libbpf_legacy.h +++ b/tools/lib/bpf/libbpf_legacy.h @@ -69,6 +69,7 @@ enum libbpf_strict_mode { LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode); +#define DECLARE_LIBBPF_OPTS LIBBPF_OPTS #ifdef __cplusplus } /* extern "C" */ diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 68f2dbf364aa..02c401e314c7 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -68,21 +68,21 @@ static void probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insns_cnt, char *buf, size_t buf_len, __u32 ifindex) { - struct bpf_load_program_attr xattr = {}; + LIBBPF_OPTS(bpf_prog_load_opts, opts); int fd; switch (prog_type) { case BPF_PROG_TYPE_CGROUP_SOCK_ADDR: - xattr.expected_attach_type = BPF_CGROUP_INET4_CONNECT; + opts.expected_attach_type = BPF_CGROUP_INET4_CONNECT; break; case BPF_PROG_TYPE_CGROUP_SOCKOPT: - xattr.expected_attach_type = BPF_CGROUP_GETSOCKOPT; + opts.expected_attach_type = BPF_CGROUP_GETSOCKOPT; break; case BPF_PROG_TYPE_SK_LOOKUP: - xattr.expected_attach_type = BPF_SK_LOOKUP; + opts.expected_attach_type = BPF_SK_LOOKUP; break; case BPF_PROG_TYPE_KPROBE: - xattr.kern_version = get_kernel_version(); + opts.kern_version = get_kernel_version(); break; case BPF_PROG_TYPE_UNSPEC: case BPF_PROG_TYPE_SOCKET_FILTER: @@ -115,13 +115,11 @@ probe_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, break; } - xattr.prog_type = prog_type; - xattr.insns = insns; - xattr.insns_cnt = insns_cnt; - xattr.license = "GPL"; - xattr.prog_ifindex = ifindex; + opts.prog_ifindex = ifindex; + opts.log_buf = buf; + opts.log_size = buf_len; - fd = bpf_load_program_xattr(&xattr, buf, buf_len); + fd = bpf_prog_load(prog_type, NULL, "GPL", insns, insns_cnt, NULL); if (fd >= 0) close(fd); } diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index f677dccdeae4..594b206fa674 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -2650,6 +2650,7 @@ static int emit_elf_data_sec(struct bpf_linker *linker, const char *sec_name, static int finalize_btf(struct bpf_linker *linker) { + LIBBPF_OPTS(btf_dedup_opts, opts); struct btf *btf = linker->btf; const void *raw_data; int i, j, id, err; @@ -2686,7 +2687,8 @@ static int finalize_btf(struct bpf_linker *linker) return err; } - err = btf__dedup(linker->btf, linker->btf_ext, NULL); + opts.btf_ext = linker->btf_ext; + err = btf__dedup(linker->btf, &opts); if (err) { pr_warn("BTF dedup failed: %d\n", err); return err; diff --git a/tools/lib/bpf/xsk.c b/tools/lib/bpf/xsk.c index 81f8fbc85e70..fdb22f5405c9 100644 --- a/tools/lib/bpf/xsk.c +++ b/tools/lib/bpf/xsk.c @@ -364,7 +364,6 @@ int xsk_umem__create_v0_0_2(struct xsk_umem **umem_ptr, void *umem_area, static enum xsk_prog get_xsk_prog(void) { enum xsk_prog detected = XSK_PROG_FALLBACK; - struct bpf_load_program_attr prog_attr; struct bpf_create_map_attr map_attr; __u32 size_out, retval, duration; char data_in = 0, data_out; @@ -375,7 +374,7 @@ static enum xsk_prog get_xsk_prog(void) BPF_EMIT_CALL(BPF_FUNC_redirect_map), BPF_EXIT_INSN(), }; - int prog_fd, map_fd, ret; + int prog_fd, map_fd, ret, insn_cnt = ARRAY_SIZE(insns); memset(&map_attr, 0, sizeof(map_attr)); map_attr.map_type = BPF_MAP_TYPE_XSKMAP; @@ -389,13 +388,7 @@ static enum xsk_prog get_xsk_prog(void) insns[0].imm = map_fd; - memset(&prog_attr, 0, sizeof(prog_attr)); - prog_attr.prog_type = BPF_PROG_TYPE_XDP; - prog_attr.insns = insns; - prog_attr.insns_cnt = ARRAY_SIZE(insns); - prog_attr.license = "GPL"; - - prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); if (prog_fd < 0) { close(map_fd); return detected; @@ -495,10 +488,13 @@ static int xsk_load_xdp_prog(struct xsk_socket *xsk) }; struct bpf_insn *progs[] = {prog, prog_redirect_flags}; enum xsk_prog option = get_xsk_prog(); + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .log_buf = log_buf, + .log_size = log_buf_size, + ); - prog_fd = bpf_load_program(BPF_PROG_TYPE_XDP, progs[option], insns_cnt[option], - "LGPL-2.1 or BSD-2-Clause", 0, log_buf, - log_buf_size); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "LGPL-2.1 or BSD-2-Clause", + progs[option], insns_cnt[option], &opts); if (prog_fd < 0) { pr_warn("BPF log buffer:\n%s", log_buf); return prog_fd; @@ -725,14 +721,12 @@ static int xsk_link_lookup(int ifindex, __u32 *prog_id, int *link_fd) static bool xsk_probe_bpf_link(void) { - DECLARE_LIBBPF_OPTS(bpf_link_create_opts, opts, - .flags = XDP_FLAGS_SKB_MODE); - struct bpf_load_program_attr prog_attr; + LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = XDP_FLAGS_SKB_MODE); struct bpf_insn insns[2] = { BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), BPF_EXIT_INSN() }; - int prog_fd, link_fd = -1; + int prog_fd, link_fd = -1, insn_cnt = ARRAY_SIZE(insns); int ifindex_lo = 1; bool ret = false; int err; @@ -744,13 +738,7 @@ static bool xsk_probe_bpf_link(void) if (link_fd >= 0) return true; - memset(&prog_attr, 0, sizeof(prog_attr)); - prog_attr.prog_type = BPF_PROG_TYPE_XDP; - prog_attr.insns = insns; - prog_attr.insns_cnt = ARRAY_SIZE(insns); - prog_attr.license = "GPL"; - - prog_fd = bpf_load_program_xattr(&prog_attr, NULL, 0); + prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); if (prog_fd < 0) return ret; diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile index 62fafbeb4672..5d42db2e129a 100644 --- a/tools/testing/selftests/bpf/Makefile +++ b/tools/testing/selftests/bpf/Makefile @@ -23,9 +23,8 @@ BPF_GCC ?= $(shell command -v bpf-gcc;) SAN_CFLAGS ?= CFLAGS += -g -O0 -rdynamic -Wall $(GENFLAGS) $(SAN_CFLAGS) \ -I$(CURDIR) -I$(INCLUDE_DIR) -I$(GENDIR) -I$(LIBDIR) \ - -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT) \ - -Dbpf_prog_load=bpf_prog_test_load \ - -Dbpf_load_program=bpf_test_load_program + -I$(TOOLSINCDIR) -I$(APIDIR) -I$(OUTPUT) +LDFLAGS += $(SAN_CFLAGS) LDLIBS += -lcap -lelf -lz -lrt -lpthread # Silence some warnings when compiled with clang @@ -46,10 +45,8 @@ ifneq ($(BPF_GCC),) TEST_GEN_PROGS += test_progs-bpf_gcc endif -TEST_GEN_FILES = test_lwt_ip_encap.o \ - test_tc_edt.o -TEST_FILES = xsk_prereqs.sh \ - $(wildcard progs/btf_dump_test_case_*.c) +TEST_GEN_FILES = test_lwt_ip_encap.o test_tc_edt.o +TEST_FILES = xsk_prereqs.sh $(wildcard progs/btf_dump_test_case_*.c) # Order correspond to 'make run_tests' order TEST_PROGS := test_kmod.sh \ @@ -108,7 +105,10 @@ endif OVERRIDE_TARGETS := 1 override define CLEAN $(call msg,CLEAN) - $(Q)$(RM) -r $(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED) $(TEST_GEN_FILES) $(EXTRA_CLEAN) + $(Q)$(RM) -r $(TEST_GEN_PROGS) + $(Q)$(RM) -r $(TEST_GEN_PROGS_EXTENDED) + $(Q)$(RM) -r $(TEST_GEN_FILES) + $(Q)$(RM) -r $(EXTRA_CLEAN) $(Q)$(MAKE) -C bpf_testmod clean $(Q)$(MAKE) docs-clean endef @@ -170,7 +170,7 @@ $(OUTPUT)/%:%.c $(OUTPUT)/urandom_read: urandom_read.c $(call msg,BINARY,,$@) - $(Q)$(CC) $(LDFLAGS) -o $@ $< $(LDLIBS) -Wl,--build-id=sha1 + $(Q)$(CC) $(LDFLAGS) $< $(LDLIBS) -Wl,--build-id=sha1 -o $@ $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_testmod/*.[ch]) $(call msg,MOD,,$@) @@ -178,10 +178,6 @@ $(OUTPUT)/bpf_testmod.ko: $(VMLINUX_BTF) $(wildcard bpf_testmod/Makefile bpf_tes $(Q)$(MAKE) $(submake_extras) -C bpf_testmod $(Q)cp bpf_testmod/bpf_testmod.ko $@ -$(OUTPUT)/test_stub.o: test_stub.c $(BPFOBJ) - $(call msg,CC,,$@) - $(Q)$(CC) -c $(CFLAGS) -o $@ $< - DEFAULT_BPFTOOL := $(HOST_SCRATCH_DIR)/sbin/bpftool $(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT) @@ -194,18 +190,24 @@ $(OUTPUT)/runqslower: $(BPFOBJ) | $(DEFAULT_BPFTOOL) $(RUNQSLOWER_OUTPUT) TEST_GEN_PROGS_EXTENDED += $(DEFAULT_BPFTOOL) -$(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/test_stub.o $(BPFOBJ) - -$(OUTPUT)/test_dev_cgroup: cgroup_helpers.c -$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c -$(OUTPUT)/test_sock: cgroup_helpers.c -$(OUTPUT)/test_sock_addr: cgroup_helpers.c -$(OUTPUT)/test_sockmap: cgroup_helpers.c -$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c -$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c -$(OUTPUT)/test_cgroup_storage: cgroup_helpers.c -$(OUTPUT)/test_sock_fields: cgroup_helpers.c -$(OUTPUT)/test_sysctl: cgroup_helpers.c +$(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(BPFOBJ) + +$(OUTPUT)/test_dev_cgroup: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_skb_cgroup_id_user: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_sock: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_sock_addr: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_sockmap: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_tcpnotify_user: cgroup_helpers.c trace_helpers.c testing_helpers.o +$(OUTPUT)/get_cgroup_id_user: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_cgroup_storage: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_sock_fields: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_sysctl: cgroup_helpers.c testing_helpers.o +$(OUTPUT)/test_tag: testing_helpers.o +$(OUTPUT)/test_lirc_mode2_user: testing_helpers.o +$(OUTPUT)/xdping: testing_helpers.o +$(OUTPUT)/flow_dissector_load: testing_helpers.o +$(OUTPUT)/test_maps: testing_helpers.o +$(OUTPUT)/test_verifier: testing_helpers.o BPFTOOL ?= $(DEFAULT_BPFTOOL) $(DEFAULT_BPFTOOL): $(wildcard $(BPFTOOLDIR)/*.[ch] $(BPFTOOLDIR)/Makefile) \ @@ -231,16 +233,16 @@ docs-clean: prefix= OUTPUT=$(OUTPUT)/ DESTDIR=$(OUTPUT)/ $@ $(BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ - ../../../include/uapi/linux/bpf.h \ + $(APIDIR)/linux/bpf.h \ | $(BUILD_DIR)/libbpf $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) OUTPUT=$(BUILD_DIR)/libbpf/ \ EXTRA_CFLAGS='-g -O0' \ DESTDIR=$(SCRATCH_DIR) prefix= all install_headers ifneq ($(BPFOBJ),$(HOST_BPFOBJ)) -$(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ - ../../../include/uapi/linux/bpf.h \ - | $(HOST_BUILD_DIR)/libbpf +$(HOST_BPFOBJ): $(wildcard $(BPFDIR)/*.[ch] $(BPFDIR)/Makefile) \ + $(APIDIR)/linux/bpf.h \ + | $(HOST_BUILD_DIR)/libbpf $(Q)$(MAKE) $(submake_extras) -C $(BPFDIR) \ EXTRA_CFLAGS='-g -O0' \ OUTPUT=$(HOST_BUILD_DIR)/libbpf/ CC=$(HOSTCC) LD=$(HOSTLD) \ @@ -304,12 +306,12 @@ $(OUTPUT)/flow_dissector_load.o: flow_dissector_load.h # $3 - CFLAGS define CLANG_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -o $2 -mcpu=v3 + $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v3 -o $2 endef # Similar to CLANG_BPF_BUILD_RULE, but with disabled alu32 define CLANG_NOALU32_BPF_BUILD_RULE $(call msg,CLNG-BPF,$(TRUNNER_BINARY),$2) - $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -o $2 -mcpu=v2 + $(Q)$(CLANG) $3 -O2 -target bpf -c $1 -mcpu=v2 -o $2 endef # Build BPF object using GCC define GCC_BPF_BUILD_RULE @@ -471,13 +473,12 @@ TRUNNER_TESTS_DIR := prog_tests TRUNNER_BPF_PROGS_DIR := progs TRUNNER_EXTRA_SOURCES := test_progs.c cgroup_helpers.c trace_helpers.c \ network_helpers.c testing_helpers.c \ - btf_helpers.c flow_dissector_load.h + btf_helpers.c flow_dissector_load.h TRUNNER_EXTRA_FILES := $(OUTPUT)/urandom_read $(OUTPUT)/bpf_testmod.ko \ ima_setup.sh \ $(wildcard progs/btf_dump_test_case_*.c) TRUNNER_BPF_BUILD_RULE := CLANG_BPF_BUILD_RULE -TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -TRUNNER_BPF_CFLAGS += -DENABLE_ATOMICS_TESTS +TRUNNER_BPF_CFLAGS := $(BPF_CFLAGS) $(CLANG_CFLAGS) -DENABLE_ATOMICS_TESTS $(eval $(call DEFINE_TEST_RUNNER,test_progs)) # Define test_progs-no_alu32 test runner. @@ -539,7 +540,7 @@ $(OUTPUT)/bench: $(OUTPUT)/bench.o $(OUTPUT)/testing_helpers.o \ $(OUTPUT)/bench_ringbufs.o \ $(OUTPUT)/bench_bloom_filter_map.o $(call msg,BINARY,,$@) - $(Q)$(CC) $(LDFLAGS) -o $@ $(filter %.a %.o,$^) $(LDLIBS) + $(Q)$(CC) $(LDFLAGS) $(filter %.a %.o,$^) $(LDLIBS) -o $@ EXTRA_CLEAN := $(TEST_CUSTOM_PROGS) $(SCRATCH_DIR) $(HOST_SCRATCH_DIR) \ prog_tests/tests.h map_tests/tests.h verifier/tests.h \ diff --git a/tools/testing/selftests/bpf/README.rst b/tools/testing/selftests/bpf/README.rst index 5e287e445f75..42ef250c7acc 100644 --- a/tools/testing/selftests/bpf/README.rst +++ b/tools/testing/selftests/bpf/README.rst @@ -204,16 +204,17 @@ __ https://reviews.llvm.org/D93563 btf_tag test and Clang version ============================== -The btf_tag selftest require LLVM support to recognize the btf_decl_tag attribute. -It was introduced in `Clang 14`__. +The btf_tag selftest requires LLVM support to recognize the btf_decl_tag and +btf_type_tag attributes. They are introduced in `Clang 14` [0_, 1_]. -Without it, the btf_tag selftest will be skipped and you will observe: +Without them, the btf_tag selftest will be skipped and you will observe: .. code-block:: console #<test_num> btf_tag:SKIP -__ https://reviews.llvm.org/D111588 +.. _0: https://reviews.llvm.org/D111588 +.. _1: https://reviews.llvm.org/D111199 Clang dependencies for static linking tests =========================================== diff --git a/tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c b/tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c index 6eeeed2913e6..5bcb8a8cdeb2 100644 --- a/tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c +++ b/tools/testing/selftests/bpf/benchs/bench_bloom_filter_map.c @@ -63,29 +63,34 @@ static const struct argp_option opts[] = { static error_t parse_arg(int key, char *arg, struct argp_state *state) { + long ret; + switch (key) { case ARG_NR_ENTRIES: - args.nr_entries = strtol(arg, NULL, 10); - if (args.nr_entries == 0) { + ret = strtol(arg, NULL, 10); + if (ret < 1 || ret > UINT_MAX) { fprintf(stderr, "Invalid nr_entries count."); argp_usage(state); } + args.nr_entries = ret; break; case ARG_NR_HASH_FUNCS: - args.nr_hash_funcs = strtol(arg, NULL, 10); - if (args.nr_hash_funcs == 0 || args.nr_hash_funcs > 15) { + ret = strtol(arg, NULL, 10); + if (ret < 1 || ret > 15) { fprintf(stderr, "The bloom filter must use 1 to 15 hash functions."); argp_usage(state); } + args.nr_hash_funcs = ret; break; case ARG_VALUE_SIZE: - args.value_size = strtol(arg, NULL, 10); - if (args.value_size < 2 || args.value_size > 256) { + ret = strtol(arg, NULL, 10); + if (ret < 2 || ret > 256) { fprintf(stderr, "Invalid value size. Must be between 2 and 256 bytes"); argp_usage(state); } + args.value_size = ret; break; default: return ARGP_ERR_UNKNOWN; diff --git a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c index d167bffac679..52d4a2f91dbd 100644 --- a/tools/testing/selftests/bpf/benchs/bench_ringbufs.c +++ b/tools/testing/selftests/bpf/benchs/bench_ringbufs.c @@ -394,11 +394,6 @@ static void perfbuf_libbpf_setup() { struct perfbuf_libbpf_ctx *ctx = &perfbuf_libbpf_ctx; struct perf_event_attr attr; - struct perf_buffer_raw_opts pb_opts = { - .event_cb = perfbuf_process_sample_raw, - .ctx = (void *)(long)0, - .attr = &attr, - }; struct bpf_link *link; ctx->skel = perfbuf_setup_skeleton(); @@ -423,7 +418,8 @@ static void perfbuf_libbpf_setup() } ctx->perfbuf = perf_buffer__new_raw(bpf_map__fd(ctx->skel->maps.perfbuf), - args.perfbuf_sz, &pb_opts); + args.perfbuf_sz, &attr, + perfbuf_process_sample_raw, NULL, NULL); if (!ctx->perfbuf) { fprintf(stderr, "failed to create perfbuf\n"); exit(1); diff --git a/tools/testing/selftests/bpf/btf_helpers.c b/tools/testing/selftests/bpf/btf_helpers.c index b5b6b013a245..b5941d514e17 100644 --- a/tools/testing/selftests/bpf/btf_helpers.c +++ b/tools/testing/selftests/bpf/btf_helpers.c @@ -25,11 +25,12 @@ static const char * const btf_kind_str_mapping[] = { [BTF_KIND_DATASEC] = "DATASEC", [BTF_KIND_FLOAT] = "FLOAT", [BTF_KIND_DECL_TAG] = "DECL_TAG", + [BTF_KIND_TYPE_TAG] = "TYPE_TAG", }; static const char *btf_kind_str(__u16 kind) { - if (kind > BTF_KIND_DECL_TAG) + if (kind > BTF_KIND_TYPE_TAG) return "UNKNOWN"; return btf_kind_str_mapping[kind]; } @@ -109,6 +110,7 @@ int fprintf_btf_type_raw(FILE *out, const struct btf *btf, __u32 id) case BTF_KIND_VOLATILE: case BTF_KIND_RESTRICT: case BTF_KIND_TYPEDEF: + case BTF_KIND_TYPE_TAG: fprintf(out, " type_id=%u", t->type); break; case BTF_KIND_ARRAY: { @@ -238,7 +240,6 @@ const char *btf_type_c_dump(const struct btf *btf) static char buf[16 * 1024]; FILE *buf_file; struct btf_dump *d = NULL; - struct btf_dump_opts opts = {}; int err, i; buf_file = fmemopen(buf, sizeof(buf) - 1, "w"); @@ -247,22 +248,26 @@ const char *btf_type_c_dump(const struct btf *btf) return NULL; } - opts.ctx = buf_file; - d = btf_dump__new(btf, NULL, &opts, btf_dump_printf); + d = btf_dump__new(btf, btf_dump_printf, buf_file, NULL); if (libbpf_get_error(d)) { fprintf(stderr, "Failed to create btf_dump instance: %ld\n", libbpf_get_error(d)); - return NULL; + goto err_out; } for (i = 1; i < btf__type_cnt(btf); i++) { err = btf_dump__dump_type(d, i); if (err) { fprintf(stderr, "Failed to dump type [%d]: %d\n", i, err); - return NULL; + goto err_out; } } + btf_dump__free(d); fflush(buf_file); fclose(buf_file); return buf; +err_out: + btf_dump__free(d); + fclose(buf_file); + return NULL; } diff --git a/tools/testing/selftests/bpf/flow_dissector_load.h b/tools/testing/selftests/bpf/flow_dissector_load.h index 9d0acc2fc6cc..f40b585f4e7e 100644 --- a/tools/testing/selftests/bpf/flow_dissector_load.h +++ b/tools/testing/selftests/bpf/flow_dissector_load.h @@ -4,6 +4,7 @@ #include <bpf/bpf.h> #include <bpf/libbpf.h> +#include "testing_helpers.h" static inline int bpf_flow_load(struct bpf_object **obj, const char *path, @@ -18,7 +19,7 @@ static inline int bpf_flow_load(struct bpf_object **obj, int prog_array_fd; int ret, fd, i; - ret = bpf_prog_load(path, BPF_PROG_TYPE_FLOW_DISSECTOR, obj, + ret = bpf_prog_test_load(path, BPF_PROG_TYPE_FLOW_DISSECTOR, obj, prog_fd); if (ret) return ret; diff --git a/tools/testing/selftests/bpf/get_cgroup_id_user.c b/tools/testing/selftests/bpf/get_cgroup_id_user.c index 99628e1a1e58..3a7b82bd9e94 100644 --- a/tools/testing/selftests/bpf/get_cgroup_id_user.c +++ b/tools/testing/selftests/bpf/get_cgroup_id_user.c @@ -19,6 +19,7 @@ #include <bpf/libbpf.h> #include "cgroup_helpers.h" +#include "testing_helpers.h" #include "bpf_rlimit.h" #define CHECK(condition, tag, format...) ({ \ @@ -66,8 +67,8 @@ int main(int argc, char **argv) if (CHECK(cgroup_fd < 0, "cgroup_setup_and_join", "err %d errno %d\n", cgroup_fd, errno)) return 1; - err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); - if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno)) + err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); + if (CHECK(err, "bpf_prog_test_load", "err %d errno %d\n", err, errno)) goto cleanup_cgroup_env; cgidmap_fd = bpf_find_map(__func__, obj, "cg_ids"); diff --git a/tools/testing/selftests/bpf/prog_tests/align.c b/tools/testing/selftests/bpf/prog_tests/align.c index 5861446d0777..837f67c6bfda 100644 --- a/tools/testing/selftests/bpf/prog_tests/align.c +++ b/tools/testing/selftests/bpf/prog_tests/align.c @@ -594,6 +594,12 @@ static int do_test_single(struct bpf_align_test *test) struct bpf_insn *prog = test->insns; int prog_type = test->prog_type; char bpf_vlog_copy[32768]; + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .prog_flags = BPF_F_STRICT_ALIGNMENT, + .log_buf = bpf_vlog, + .log_size = sizeof(bpf_vlog), + .log_level = 2, + ); const char *line_ptr; int cur_line = -1; int prog_len, i; @@ -601,9 +607,8 @@ static int do_test_single(struct bpf_align_test *test) int ret; prog_len = probe_filter_length(prog); - fd_prog = bpf_verify_program(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, - prog, prog_len, BPF_F_STRICT_ALIGNMENT, - "GPL", 0, bpf_vlog, sizeof(bpf_vlog), 2); + fd_prog = bpf_prog_load(prog_type ? : BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", + prog, prog_len, &opts); if (fd_prog < 0 && test->result != REJECT) { printf("Failed to load program.\n"); printf("%s", bpf_vlog); diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c index 9454331aaf85..3e10abce3e5a 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_iter.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_iter.c @@ -699,14 +699,13 @@ static void test_bpf_percpu_hash_map(void) char buf[64]; void *val; - val = malloc(8 * bpf_num_possible_cpus()); - skel = bpf_iter_bpf_percpu_hash_map__open(); if (CHECK(!skel, "bpf_iter_bpf_percpu_hash_map__open", "skeleton open failed\n")) return; skel->rodata->num_cpus = bpf_num_possible_cpus(); + val = malloc(8 * bpf_num_possible_cpus()); err = bpf_iter_bpf_percpu_hash_map__load(skel); if (CHECK(!skel, "bpf_iter_bpf_percpu_hash_map__load", @@ -770,6 +769,7 @@ free_link: bpf_link__destroy(link); out: bpf_iter_bpf_percpu_hash_map__destroy(skel); + free(val); } static void test_bpf_array_map(void) @@ -870,14 +870,13 @@ static void test_bpf_percpu_array_map(void) void *val; int len; - val = malloc(8 * bpf_num_possible_cpus()); - skel = bpf_iter_bpf_percpu_array_map__open(); if (CHECK(!skel, "bpf_iter_bpf_percpu_array_map__open", "skeleton open failed\n")) return; skel->rodata->num_cpus = bpf_num_possible_cpus(); + val = malloc(8 * bpf_num_possible_cpus()); err = bpf_iter_bpf_percpu_array_map__load(skel); if (CHECK(!skel, "bpf_iter_bpf_percpu_array_map__load", @@ -933,6 +932,7 @@ free_link: bpf_link__destroy(link); out: bpf_iter_bpf_percpu_array_map__destroy(skel); + free(val); } /* An iterator program deletes all local storage in a map. */ diff --git a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c index eb8eeebe6935..0a6c5f00abd4 100644 --- a/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c +++ b/tools/testing/selftests/bpf/prog_tests/bpf_obj_id.c @@ -48,7 +48,7 @@ void serial_test_bpf_obj_id(void) bzero(zeros, sizeof(zeros)); for (i = 0; i < nr_iters; i++) { now = time(NULL); - err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, + err = bpf_prog_test_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &objs[i], &prog_fds[i]); /* test_obj_id.o is a dumb prog. It should never fail * to load. diff --git a/tools/testing/selftests/bpf/prog_tests/btf.c b/tools/testing/selftests/bpf/prog_tests/btf.c index ac596cb06e40..4aa6343dc4c8 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf.c +++ b/tools/testing/selftests/bpf/prog_tests/btf.c @@ -3939,6 +3939,23 @@ static struct btf_raw_test raw_tests[] = { .btf_load_err = true, .err_str = "Invalid component_idx", }, +{ + .descr = "type_tag test #1", + .raw_types = { + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_TBD, 1), /* [2] */ + BTF_PTR_ENC(2), /* [3] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag"), + .map_type = BPF_MAP_TYPE_ARRAY, + .map_name = "tag_type_check_btf", + .key_size = sizeof(int), + .value_size = 4, + .key_type_id = 1, + .value_type_id = 1, + .max_entries = 1, +}, }; /* struct btf_raw_test raw_tests[] */ @@ -4046,11 +4063,9 @@ static void *btf_raw_create(const struct btf_header *hdr, next_str_idx < strs_cnt ? strs_idx[next_str_idx] : NULL; done: + free(strs_idx); if (err) { - if (raw_btf) - free(raw_btf); - if (strs_idx) - free(strs_idx); + free(raw_btf); return NULL; } return raw_btf; @@ -6629,7 +6644,7 @@ struct btf_dedup_test { struct btf_dedup_opts opts; }; -const struct btf_dedup_test dedup_tests[] = { +static struct btf_dedup_test dedup_tests[] = { { .descr = "dedup: unused strings filtering", @@ -6649,9 +6664,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0int\0long"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: strings deduplication", @@ -6674,9 +6686,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0int\0long int"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: struct example #1", @@ -6757,9 +6766,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0a\0b\0c\0d\0int\0float\0next\0s"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: struct <-> fwd resolution w/ hash collision", @@ -6802,8 +6808,7 @@ const struct btf_dedup_test dedup_tests[] = { BTF_STR_SEC("\0s\0x"), }, .opts = { - .dont_resolve_fwds = false, - .dedup_table_size = 1, /* force hash collisions */ + .force_collisions = true, /* force hash collisions */ }, }, { @@ -6849,8 +6854,7 @@ const struct btf_dedup_test dedup_tests[] = { BTF_STR_SEC("\0s\0x"), }, .opts = { - .dont_resolve_fwds = false, - .dedup_table_size = 1, /* force hash collisions */ + .force_collisions = true, /* force hash collisions */ }, }, { @@ -6874,15 +6878,16 @@ const struct btf_dedup_test dedup_tests[] = { BTF_RESTRICT_ENC(8), /* [11] restrict */ BTF_FUNC_PROTO_ENC(1, 2), /* [12] func_proto */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), - BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 18), BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */ BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */ BTF_DECL_TAG_ENC(NAME_TBD, 13, -1), /* [15] decl_tag */ BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */ BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */ + BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */ BTF_END_RAW, }, - BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q"), + BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R"), }, .expect = { .raw_types = { @@ -6903,18 +6908,16 @@ const struct btf_dedup_test dedup_tests[] = { BTF_RESTRICT_ENC(8), /* [11] restrict */ BTF_FUNC_PROTO_ENC(1, 2), /* [12] func_proto */ BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 1), - BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 8), + BTF_FUNC_PROTO_ARG_ENC(NAME_TBD, 18), BTF_FUNC_ENC(NAME_TBD, 12), /* [13] func */ BTF_TYPE_FLOAT_ENC(NAME_TBD, 2), /* [14] float */ BTF_DECL_TAG_ENC(NAME_TBD, 13, -1), /* [15] decl_tag */ BTF_DECL_TAG_ENC(NAME_TBD, 13, 1), /* [16] decl_tag */ BTF_DECL_TAG_ENC(NAME_TBD, 7, -1), /* [17] decl_tag */ + BTF_TYPE_TAG_ENC(NAME_TBD, 8), /* [18] type_tag */ BTF_END_RAW, }, - BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q"), - }, - .opts = { - .dont_resolve_fwds = false, + BTF_STR_SEC("\0A\0B\0C\0D\0E\0F\0G\0H\0I\0J\0K\0L\0M\0N\0O\0P\0Q\0R"), }, }, { @@ -6967,9 +6970,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0int\0some other int\0float"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: enum fwd resolution", @@ -7011,9 +7011,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0e1\0e1_val\0e2\0e2_val"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: datasec and vars pass-through", @@ -7056,8 +7053,7 @@ const struct btf_dedup_test dedup_tests[] = { BTF_STR_SEC("\0.bss\0t"), }, .opts = { - .dont_resolve_fwds = false, - .dedup_table_size = 1 + .force_collisions = true }, }, { @@ -7101,9 +7097,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0t\0a1\0a2\0f\0tag"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: func/func_param tags", @@ -7154,9 +7147,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0a1\0a2\0f\0tag1\0tag2\0tag3"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: struct/struct_member tags", @@ -7202,9 +7192,6 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0t\0m1\0m2\0tag1\0tag2\0tag3"), }, - .opts = { - .dont_resolve_fwds = false, - }, }, { .descr = "dedup: typedef tags", @@ -7235,8 +7222,134 @@ const struct btf_dedup_test dedup_tests[] = { }, BTF_STR_SEC("\0t\0tag1\0tag2\0tag3"), }, - .opts = { - .dont_resolve_fwds = false, +}, +{ + .descr = "dedup: btf_type_tag #1", + .input = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [5] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 5), /* [6] */ + BTF_PTR_ENC(6), /* [7] */ + /* ptr -> tag1 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [8] */ + BTF_PTR_ENC(8), /* [9] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, + .expect = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag1 -> int */ + BTF_PTR_ENC(2), /* [5] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, +}, +{ + .descr = "dedup: btf_type_tag #2", + .input = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag2 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 1), /* [5] */ + BTF_PTR_ENC(5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, + .expect = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag2 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 1), /* [5] */ + BTF_PTR_ENC(5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, +}, +{ + .descr = "dedup: btf_type_tag #3", + .input = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag1 -> tag2 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 1), /* [5] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 5), /* [6] */ + BTF_PTR_ENC(6), /* [7] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, + .expect = { + .raw_types = { + /* ptr -> tag2 -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 2), /* [3] */ + BTF_PTR_ENC(3), /* [4] */ + /* ptr -> tag1 -> tag2 -> int */ + BTF_TYPE_TAG_ENC(NAME_NTH(2), 1), /* [5] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 5), /* [6] */ + BTF_PTR_ENC(6), /* [7] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1\0tag2"), + }, +}, +{ + .descr = "dedup: btf_type_tag #4", + .input = { + .raw_types = { + /* ptr -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_PTR_ENC(2), /* [3] */ + /* ptr -> tag1 -> long */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 64, 8), /* [4] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 4), /* [5] */ + BTF_PTR_ENC(5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1"), + }, + .expect = { + .raw_types = { + /* ptr -> tag1 -> int */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 32, 4), /* [1] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 1), /* [2] */ + BTF_PTR_ENC(2), /* [3] */ + /* ptr -> tag1 -> long */ + BTF_TYPE_INT_ENC(0, BTF_INT_SIGNED, 0, 64, 8), /* [4] */ + BTF_TYPE_TAG_ENC(NAME_NTH(1), 4), /* [5] */ + BTF_PTR_ENC(5), /* [6] */ + BTF_END_RAW, + }, + BTF_STR_SEC("\0tag1"), }, }, @@ -7257,6 +7370,7 @@ static int btf_type_size(const struct btf_type *t) case BTF_KIND_TYPEDEF: case BTF_KIND_FUNC: case BTF_KIND_FLOAT: + case BTF_KIND_TYPE_TAG: return base_size; case BTF_KIND_INT: return base_size + sizeof(__u32); @@ -7295,7 +7409,7 @@ static void dump_btf_strings(const char *strs, __u32 len) static void do_test_dedup(unsigned int test_num) { - const struct btf_dedup_test *test = &dedup_tests[test_num - 1]; + struct btf_dedup_test *test = &dedup_tests[test_num - 1]; __u32 test_nr_types, expect_nr_types, test_btf_size, expect_btf_size; const struct btf_header *test_hdr, *expect_hdr; struct btf *test_btf = NULL, *expect_btf = NULL; @@ -7339,7 +7453,8 @@ static void do_test_dedup(unsigned int test_num) goto done; } - err = btf__dedup(test_btf, NULL, &test->opts); + test->opts.sz = sizeof(test->opts); + err = btf__dedup(test_btf, &test->opts); if (CHECK(err, "btf_dedup failed errno:%d", err)) { err = -1; goto done; diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c index 64554fd33547..9d3b8d7a1537 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dedup_split.c @@ -92,7 +92,7 @@ struct s2 {\n\ int *f3;\n\ };\n\n", "c_dump"); - err = btf__dedup(btf2, NULL, NULL); + err = btf__dedup(btf2, NULL); if (!ASSERT_OK(err, "btf_dedup")) goto cleanup; @@ -186,7 +186,7 @@ static void test_split_fwd_resolve() { "\t'f1' type_id=7 bits_offset=0\n" "\t'f2' type_id=9 bits_offset=64"); - err = btf__dedup(btf2, NULL, NULL); + err = btf__dedup(btf2, NULL); if (!ASSERT_OK(err, "btf_dedup")) goto cleanup; @@ -283,7 +283,7 @@ static void test_split_struct_duped() { "[13] STRUCT 's3' size=8 vlen=1\n" "\t'f1' type_id=12 bits_offset=0"); - err = btf__dedup(btf2, NULL, NULL); + err = btf__dedup(btf2, NULL); if (!ASSERT_OK(err, "btf_dedup")) goto cleanup; diff --git a/tools/testing/selftests/bpf/prog_tests/btf_dump.c b/tools/testing/selftests/bpf/prog_tests/btf_dump.c index aa76360d8f49..d6272013a5a3 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_dump.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_dump.c @@ -13,25 +13,23 @@ static struct btf_dump_test_case { const char *name; const char *file; bool known_ptr_sz; - struct btf_dump_opts opts; } btf_dump_test_cases[] = { - {"btf_dump: syntax", "btf_dump_test_case_syntax", true, {}}, - {"btf_dump: ordering", "btf_dump_test_case_ordering", false, {}}, - {"btf_dump: padding", "btf_dump_test_case_padding", true, {}}, - {"btf_dump: packing", "btf_dump_test_case_packing", true, {}}, - {"btf_dump: bitfields", "btf_dump_test_case_bitfields", true, {}}, - {"btf_dump: multidim", "btf_dump_test_case_multidim", false, {}}, - {"btf_dump: namespacing", "btf_dump_test_case_namespacing", false, {}}, + {"btf_dump: syntax", "btf_dump_test_case_syntax", true}, + {"btf_dump: ordering", "btf_dump_test_case_ordering", false}, + {"btf_dump: padding", "btf_dump_test_case_padding", true}, + {"btf_dump: packing", "btf_dump_test_case_packing", true}, + {"btf_dump: bitfields", "btf_dump_test_case_bitfields", true}, + {"btf_dump: multidim", "btf_dump_test_case_multidim", false}, + {"btf_dump: namespacing", "btf_dump_test_case_namespacing", false}, }; -static int btf_dump_all_types(const struct btf *btf, - const struct btf_dump_opts *opts) +static int btf_dump_all_types(const struct btf *btf, void *ctx) { size_t type_cnt = btf__type_cnt(btf); struct btf_dump *d; int err = 0, id; - d = btf_dump__new(btf, NULL, opts, btf_dump_printf); + d = btf_dump__new(btf, btf_dump_printf, ctx, NULL); err = libbpf_get_error(d); if (err) return err; @@ -88,8 +86,7 @@ static int test_btf_dump_case(int n, struct btf_dump_test_case *t) goto done; } - t->opts.ctx = f; - err = btf_dump_all_types(btf, &t->opts); + err = btf_dump_all_types(btf, f); fclose(f); close(fd); if (CHECK(err, "btf_dump", "failure during C dumping: %d\n", err)) { @@ -137,7 +134,6 @@ static void test_btf_dump_incremental(void) { struct btf *btf = NULL; struct btf_dump *d = NULL; - struct btf_dump_opts opts; int id, err, i; dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz); @@ -146,8 +142,7 @@ static void test_btf_dump_incremental(void) btf = btf__new_empty(); if (!ASSERT_OK_PTR(btf, "new_empty")) goto err_out; - opts.ctx = dump_buf_file; - d = btf_dump__new(btf, NULL, &opts, btf_dump_printf); + d = btf_dump__new(btf, btf_dump_printf, dump_buf_file, NULL); if (!ASSERT_OK(libbpf_get_error(d), "btf_dump__new")) goto err_out; @@ -814,26 +809,28 @@ static void test_btf_datasec(struct btf *btf, struct btf_dump *d, char *str, static void test_btf_dump_datasec_data(char *str) { - struct btf *btf = btf__parse("xdping_kern.o", NULL); - struct btf_dump_opts opts = { .ctx = str }; + struct btf *btf; char license[4] = "GPL"; struct btf_dump *d; + btf = btf__parse("xdping_kern.o", NULL); if (!ASSERT_OK_PTR(btf, "xdping_kern.o BTF not found")) return; - d = btf_dump__new(btf, NULL, &opts, btf_dump_snprintf); + d = btf_dump__new(btf, btf_dump_snprintf, str, NULL); if (!ASSERT_OK_PTR(d, "could not create BTF dump")) - return; + goto out; test_btf_datasec(btf, d, str, "license", "SEC(\"license\") char[4] _license = (char[4])['G','P','L',];", license, sizeof(license)); +out: + btf_dump__free(d); + btf__free(btf); } void test_btf_dump() { char str[STRSIZE]; - struct btf_dump_opts opts = { .ctx = str }; struct btf_dump *d; struct btf *btf; int i; @@ -853,7 +850,7 @@ void test_btf_dump() { if (!ASSERT_OK_PTR(btf, "no kernel BTF found")) return; - d = btf_dump__new(btf, NULL, &opts, btf_dump_snprintf); + d = btf_dump__new(btf, btf_dump_snprintf, str, NULL); if (!ASSERT_OK_PTR(d, "could not create BTF dump")) return; diff --git a/tools/testing/selftests/bpf/prog_tests/btf_split.c b/tools/testing/selftests/bpf/prog_tests/btf_split.c index b1ffe61f2aa9..eef1158676ed 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_split.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_split.c @@ -13,7 +13,6 @@ static void btf_dump_printf(void *ctx, const char *fmt, va_list args) } void test_btf_split() { - struct btf_dump_opts opts; struct btf_dump *d = NULL; const struct btf_type *t; struct btf *btf1, *btf2; @@ -68,8 +67,7 @@ void test_btf_split() { dump_buf_file = open_memstream(&dump_buf, &dump_buf_sz); if (!ASSERT_OK_PTR(dump_buf_file, "dump_memstream")) return; - opts.ctx = dump_buf_file; - d = btf_dump__new(btf2, NULL, &opts, btf_dump_printf); + d = btf_dump__new(btf2, btf_dump_printf, dump_buf_file, NULL); if (!ASSERT_OK_PTR(d, "btf_dump__new")) goto cleanup; for (i = 1; i < btf__type_cnt(btf2); i++) { diff --git a/tools/testing/selftests/bpf/prog_tests/btf_tag.c b/tools/testing/selftests/bpf/prog_tests/btf_tag.c index 91821f42714d..88d63e23e35f 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_tag.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_tag.c @@ -1,20 +1,50 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2021 Facebook */ #include <test_progs.h> -#include "tag.skel.h" +#include "btf_decl_tag.skel.h" -void test_btf_tag(void) +/* struct btf_type_tag_test is referenced in btf_type_tag.skel.h */ +struct btf_type_tag_test { + int **p; +}; +#include "btf_type_tag.skel.h" + +static void test_btf_decl_tag(void) +{ + struct btf_decl_tag *skel; + + skel = btf_decl_tag__open_and_load(); + if (!ASSERT_OK_PTR(skel, "btf_decl_tag")) + return; + + if (skel->rodata->skip_tests) { + printf("%s:SKIP: btf_decl_tag attribute not supported", __func__); + test__skip(); + } + + btf_decl_tag__destroy(skel); +} + +static void test_btf_type_tag(void) { - struct tag *skel; + struct btf_type_tag *skel; - skel = tag__open_and_load(); - if (!ASSERT_OK_PTR(skel, "btf_tag")) + skel = btf_type_tag__open_and_load(); + if (!ASSERT_OK_PTR(skel, "btf_type_tag")) return; if (skel->rodata->skip_tests) { - printf("%s:SKIP: btf_tag attribute not supported", __func__); + printf("%s:SKIP: btf_type_tag attribute not supported", __func__); test__skip(); } - tag__destroy(skel); + btf_type_tag__destroy(skel); +} + +void test_btf_tag(void) +{ + if (test__start_subtest("btf_decl_tag")) + test_btf_decl_tag(); + if (test__start_subtest("btf_type_tag")) + test_btf_type_tag(); } diff --git a/tools/testing/selftests/bpf/prog_tests/btf_write.c b/tools/testing/selftests/bpf/prog_tests/btf_write.c index b912eeb0b6b4..addf99c05896 100644 --- a/tools/testing/selftests/bpf/prog_tests/btf_write.c +++ b/tools/testing/selftests/bpf/prog_tests/btf_write.c @@ -297,6 +297,16 @@ static void gen_btf(struct btf *btf) ASSERT_EQ(btf_decl_tag(t)->component_idx, 1, "tag_component_idx"); ASSERT_STREQ(btf_type_raw_dump(btf, 19), "[19] DECL_TAG 'tag2' type_id=14 component_idx=1", "raw_dump"); + + /* TYPE_TAG */ + id = btf__add_type_tag(btf, "tag1", 1); + ASSERT_EQ(id, 20, "tag_id"); + t = btf__type_by_id(btf, 20); + ASSERT_STREQ(btf__str_by_offset(btf, t->name_off), "tag1", "tag_value"); + ASSERT_EQ(btf_kind(t), BTF_KIND_TYPE_TAG, "tag_kind"); + ASSERT_EQ(t->type, 1, "tag_type"); + ASSERT_STREQ(btf_type_raw_dump(btf, 20), + "[20] TYPE_TAG 'tag1' type_id=1", "raw_dump"); } static void test_btf_add() @@ -337,7 +347,8 @@ static void test_btf_add() "[17] DATASEC 'datasec1' size=12 vlen=1\n" "\ttype_id=1 offset=4 size=8", "[18] DECL_TAG 'tag1' type_id=16 component_idx=-1", - "[19] DECL_TAG 'tag2' type_id=14 component_idx=1"); + "[19] DECL_TAG 'tag2' type_id=14 component_idx=1", + "[20] TYPE_TAG 'tag1' type_id=1"); btf__free(btf); } @@ -359,7 +370,7 @@ static void test_btf_add_btf() gen_btf(btf2); id = btf__add_btf(btf1, btf2); - if (!ASSERT_EQ(id, 20, "id")) + if (!ASSERT_EQ(id, 21, "id")) goto cleanup; VALIDATE_RAW_BTF( @@ -391,35 +402,37 @@ static void test_btf_add_btf() "\ttype_id=1 offset=4 size=8", "[18] DECL_TAG 'tag1' type_id=16 component_idx=-1", "[19] DECL_TAG 'tag2' type_id=14 component_idx=1", + "[20] TYPE_TAG 'tag1' type_id=1", /* types appended from the second BTF */ - "[20] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", - "[21] PTR '(anon)' type_id=20", - "[22] CONST '(anon)' type_id=24", - "[23] VOLATILE '(anon)' type_id=22", - "[24] RESTRICT '(anon)' type_id=23", - "[25] ARRAY '(anon)' type_id=21 index_type_id=20 nr_elems=10", - "[26] STRUCT 's1' size=8 vlen=2\n" - "\t'f1' type_id=20 bits_offset=0\n" - "\t'f2' type_id=20 bits_offset=32 bitfield_size=16", - "[27] UNION 'u1' size=8 vlen=1\n" - "\t'f1' type_id=20 bits_offset=0 bitfield_size=16", - "[28] ENUM 'e1' size=4 vlen=2\n" + "[21] INT 'int' size=4 bits_offset=0 nr_bits=32 encoding=SIGNED", + "[22] PTR '(anon)' type_id=21", + "[23] CONST '(anon)' type_id=25", + "[24] VOLATILE '(anon)' type_id=23", + "[25] RESTRICT '(anon)' type_id=24", + "[26] ARRAY '(anon)' type_id=22 index_type_id=21 nr_elems=10", + "[27] STRUCT 's1' size=8 vlen=2\n" + "\t'f1' type_id=21 bits_offset=0\n" + "\t'f2' type_id=21 bits_offset=32 bitfield_size=16", + "[28] UNION 'u1' size=8 vlen=1\n" + "\t'f1' type_id=21 bits_offset=0 bitfield_size=16", + "[29] ENUM 'e1' size=4 vlen=2\n" "\t'v1' val=1\n" "\t'v2' val=2", - "[29] FWD 'struct_fwd' fwd_kind=struct", - "[30] FWD 'union_fwd' fwd_kind=union", - "[31] ENUM 'enum_fwd' size=4 vlen=0", - "[32] TYPEDEF 'typedef1' type_id=20", - "[33] FUNC 'func1' type_id=34 linkage=global", - "[34] FUNC_PROTO '(anon)' ret_type_id=20 vlen=2\n" - "\t'p1' type_id=20\n" - "\t'p2' type_id=21", - "[35] VAR 'var1' type_id=20, linkage=global-alloc", - "[36] DATASEC 'datasec1' size=12 vlen=1\n" - "\ttype_id=20 offset=4 size=8", - "[37] DECL_TAG 'tag1' type_id=35 component_idx=-1", - "[38] DECL_TAG 'tag2' type_id=33 component_idx=1"); + "[30] FWD 'struct_fwd' fwd_kind=struct", + "[31] FWD 'union_fwd' fwd_kind=union", + "[32] ENUM 'enum_fwd' size=4 vlen=0", + "[33] TYPEDEF 'typedef1' type_id=21", + "[34] FUNC 'func1' type_id=35 linkage=global", + "[35] FUNC_PROTO '(anon)' ret_type_id=21 vlen=2\n" + "\t'p1' type_id=21\n" + "\t'p2' type_id=22", + "[36] VAR 'var1' type_id=21, linkage=global-alloc", + "[37] DATASEC 'datasec1' size=12 vlen=1\n" + "\ttype_id=21 offset=4 size=8", + "[38] DECL_TAG 'tag1' type_id=36 component_idx=-1", + "[39] DECL_TAG 'tag2' type_id=34 component_idx=1", + "[40] TYPE_TAG 'tag1' type_id=21"); cleanup: btf__free(btf1); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_autodetach.c b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_autodetach.c index 5de485c7370f..858916d11e2e 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_autodetach.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_autodetach.c @@ -16,7 +16,7 @@ static int prog_load(void) }; size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn); - return bpf_load_program(BPF_PROG_TYPE_CGROUP_SKB, + return bpf_test_load_program(BPF_PROG_TYPE_CGROUP_SKB, prog, insns_cnt, "GPL", 0, bpf_log_buf, BPF_LOG_BUF_SIZE); } diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c index 731bea84d8ed..de9c3e12b0ea 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_multi.c @@ -66,7 +66,7 @@ static int prog_load_cnt(int verdict, int val) size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn); int ret; - ret = bpf_load_program(BPF_PROG_TYPE_CGROUP_SKB, + ret = bpf_test_load_program(BPF_PROG_TYPE_CGROUP_SKB, prog, insns_cnt, "GPL", 0, bpf_log_buf, BPF_LOG_BUF_SIZE); diff --git a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_override.c b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_override.c index 10d3c33821a7..356547e849e2 100644 --- a/tools/testing/selftests/bpf/prog_tests/cgroup_attach_override.c +++ b/tools/testing/selftests/bpf/prog_tests/cgroup_attach_override.c @@ -18,7 +18,7 @@ static int prog_load(int verdict) }; size_t insns_cnt = sizeof(prog) / sizeof(struct bpf_insn); - return bpf_load_program(BPF_PROG_TYPE_CGROUP_SKB, + return bpf_test_load_program(BPF_PROG_TYPE_CGROUP_SKB, prog, insns_cnt, "GPL", 0, bpf_log_buf, BPF_LOG_BUF_SIZE); } diff --git a/tools/testing/selftests/bpf/prog_tests/core_reloc.c b/tools/testing/selftests/bpf/prog_tests/core_reloc.c index 55ec85ba7375..1041d0c593f6 100644 --- a/tools/testing/selftests/bpf/prog_tests/core_reloc.c +++ b/tools/testing/selftests/bpf/prog_tests/core_reloc.c @@ -433,7 +433,7 @@ static int setup_type_id_case_local(struct core_reloc_test_case *test) static int setup_type_id_case_success(struct core_reloc_test_case *test) { struct core_reloc_type_id_output *exp = (void *)test->output; - struct btf *targ_btf = btf__parse(test->btf_src_file, NULL); + struct btf *targ_btf; int err; err = setup_type_id_case_local(test); diff --git a/tools/testing/selftests/bpf/prog_tests/exhandler.c b/tools/testing/selftests/bpf/prog_tests/exhandler.c new file mode 100644 index 000000000000..118bb182ee20 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/exhandler.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021, Oracle and/or its affiliates. */ + +#include <test_progs.h> + +/* Test that verifies exception handling is working. fork() + * triggers task_newtask tracepoint; that new task will have a + * NULL pointer task_works, and the associated task->task_works->func + * should not be NULL if task_works itself is non-NULL. + * + * So to verify exception handling we want to see a NULL task_works + * and task_works->func; if we see this we can conclude that the + * exception handler ran when we attempted to dereference task->task_works + * and zeroed the destination register. + */ +#include "exhandler_kern.skel.h" + +void test_exhandler(void) +{ + int err = 0, duration = 0, status; + struct exhandler_kern *skel; + pid_t cpid; + + skel = exhandler_kern__open_and_load(); + if (CHECK(!skel, "skel_load", "skeleton failed: %d\n", err)) + goto cleanup; + + skel->bss->test_pid = getpid(); + + err = exhandler_kern__attach(skel); + if (!ASSERT_OK(err, "attach")) + goto cleanup; + cpid = fork(); + if (!ASSERT_GT(cpid, -1, "fork failed")) + goto cleanup; + if (cpid == 0) + _exit(0); + waitpid(cpid, &status, 0); + + ASSERT_NEQ(skel->bss->exception_triggered, 0, "verify exceptions occurred"); +cleanup: + exhandler_kern__destroy(skel); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 9cff14a23bb7..fdd603ebda28 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -65,7 +65,7 @@ static void test_fexit_bpf2bpf_common(const char *obj_file, int err, tgt_fd, i; struct btf *btf; - err = bpf_prog_load(target_obj_file, BPF_PROG_TYPE_UNSPEC, + err = bpf_prog_test_load(target_obj_file, BPF_PROG_TYPE_UNSPEC, &tgt_obj, &tgt_fd); if (!ASSERT_OK(err, "tgt_prog_load")) return; @@ -224,7 +224,7 @@ static int test_second_attach(struct bpf_object *obj) if (CHECK(!prog, "find_prog", "prog %s not found\n", prog_name)) return -ENOENT; - err = bpf_prog_load(tgt_obj_file, BPF_PROG_TYPE_UNSPEC, + err = bpf_prog_test_load(tgt_obj_file, BPF_PROG_TYPE_UNSPEC, &tgt_obj, &tgt_fd); if (CHECK(err, "second_prog_load", "file %s err %d errno %d\n", tgt_obj_file, err, errno)) @@ -274,7 +274,7 @@ static void test_fmod_ret_freplace(void) __u32 duration = 0; int err, pkt_fd, attach_prog_fd; - err = bpf_prog_load(tgt_name, BPF_PROG_TYPE_UNSPEC, + err = bpf_prog_test_load(tgt_name, BPF_PROG_TYPE_UNSPEC, &pkt_obj, &pkt_fd); /* the target prog should load fine */ if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n", @@ -341,7 +341,7 @@ static void test_obj_load_failure_common(const char *obj_file, int err, pkt_fd; __u32 duration = 0; - err = bpf_prog_load(target_obj_file, BPF_PROG_TYPE_UNSPEC, + err = bpf_prog_test_load(target_obj_file, BPF_PROG_TYPE_UNSPEC, &pkt_obj, &pkt_fd); /* the target prog should load fine */ if (CHECK(err, "tgt_prog_load", "file %s err %d errno %d\n", diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_stress.c b/tools/testing/selftests/bpf/prog_tests/fexit_stress.c index 7c9b62e971f1..e4cede6b4b2d 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_stress.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_stress.c @@ -20,34 +20,33 @@ void test_fexit_stress(void) BPF_EXIT_INSN(), }; - struct bpf_load_program_attr load_attr = { - .prog_type = BPF_PROG_TYPE_TRACING, - .license = "GPL", - .insns = trace_program, - .insns_cnt = sizeof(trace_program) / sizeof(struct bpf_insn), + LIBBPF_OPTS(bpf_prog_load_opts, trace_opts, .expected_attach_type = BPF_TRACE_FEXIT, - }; + .log_buf = error, + .log_size = sizeof(error), + ); const struct bpf_insn skb_program[] = { BPF_MOV64_IMM(BPF_REG_0, 0), BPF_EXIT_INSN(), }; - struct bpf_load_program_attr skb_load_attr = { - .prog_type = BPF_PROG_TYPE_SOCKET_FILTER, - .license = "GPL", - .insns = skb_program, - .insns_cnt = sizeof(skb_program) / sizeof(struct bpf_insn), - }; + LIBBPF_OPTS(bpf_prog_load_opts, skb_opts, + .log_buf = error, + .log_size = sizeof(error), + ); err = libbpf_find_vmlinux_btf_id("bpf_fentry_test1", - load_attr.expected_attach_type); + trace_opts.expected_attach_type); if (CHECK(err <= 0, "find_vmlinux_btf_id", "failed: %d\n", err)) goto out; - load_attr.attach_btf_id = err; + trace_opts.attach_btf_id = err; for (i = 0; i < CNT; i++) { - fexit_fd[i] = bpf_load_program_xattr(&load_attr, error, sizeof(error)); + fexit_fd[i] = bpf_prog_load(BPF_PROG_TYPE_TRACING, NULL, "GPL", + trace_program, + sizeof(trace_program) / sizeof(struct bpf_insn), + &trace_opts); if (CHECK(fexit_fd[i] < 0, "fexit loaded", "failed: %d errno %d\n", fexit_fd[i], errno)) goto out; @@ -57,7 +56,9 @@ void test_fexit_stress(void) goto out; } - filter_fd = bpf_load_program_xattr(&skb_load_attr, error, sizeof(error)); + filter_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL", + skb_program, sizeof(skb_program) / sizeof(struct bpf_insn), + &skb_opts); if (CHECK(filter_fd < 0, "test_program_loaded", "failed: %d errno %d\n", filter_fd, errno)) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/find_vma.c b/tools/testing/selftests/bpf/prog_tests/find_vma.c new file mode 100644 index 000000000000..b74b3c0c555a --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/find_vma.c @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include <test_progs.h> +#include <sys/types.h> +#include <unistd.h> +#include "find_vma.skel.h" +#include "find_vma_fail1.skel.h" +#include "find_vma_fail2.skel.h" + +static void test_and_reset_skel(struct find_vma *skel, int expected_find_zero_ret) +{ + ASSERT_EQ(skel->bss->found_vm_exec, 1, "found_vm_exec"); + ASSERT_EQ(skel->data->find_addr_ret, 0, "find_addr_ret"); + ASSERT_EQ(skel->data->find_zero_ret, expected_find_zero_ret, "find_zero_ret"); + ASSERT_OK_PTR(strstr(skel->bss->d_iname, "test_progs"), "find_test_progs"); + + skel->bss->found_vm_exec = 0; + skel->data->find_addr_ret = -1; + skel->data->find_zero_ret = -1; + skel->bss->d_iname[0] = 0; +} + +static int open_pe(void) +{ + struct perf_event_attr attr = {0}; + int pfd; + + /* create perf event */ + attr.size = sizeof(attr); + attr.type = PERF_TYPE_HARDWARE; + attr.config = PERF_COUNT_HW_CPU_CYCLES; + attr.freq = 1; + attr.sample_freq = 4000; + pfd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, PERF_FLAG_FD_CLOEXEC); + + return pfd >= 0 ? pfd : -errno; +} + +static void test_find_vma_pe(struct find_vma *skel) +{ + struct bpf_link *link = NULL; + volatile int j = 0; + int pfd, i; + + pfd = open_pe(); + if (pfd < 0) { + if (pfd == -ENOENT || pfd == -EOPNOTSUPP) { + printf("%s:SKIP:no PERF_COUNT_HW_CPU_CYCLES\n", __func__); + test__skip(); + goto cleanup; + } + if (!ASSERT_GE(pfd, 0, "perf_event_open")) + goto cleanup; + } + + link = bpf_program__attach_perf_event(skel->progs.handle_pe, pfd); + if (!ASSERT_OK_PTR(link, "attach_perf_event")) + goto cleanup; + + for (i = 0; i < 1000000; ++i) + ++j; + + test_and_reset_skel(skel, -EBUSY /* in nmi, irq_work is busy */); +cleanup: + bpf_link__destroy(link); + close(pfd); +} + +static void test_find_vma_kprobe(struct find_vma *skel) +{ + int err; + + err = find_vma__attach(skel); + if (!ASSERT_OK(err, "get_branch_snapshot__attach")) + return; + + getpgid(skel->bss->target_pid); + test_and_reset_skel(skel, -ENOENT /* could not find vma for ptr 0 */); +} + +static void test_illegal_write_vma(void) +{ + struct find_vma_fail1 *skel; + + skel = find_vma_fail1__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "find_vma_fail1__open_and_load")) + find_vma_fail1__destroy(skel); +} + +static void test_illegal_write_task(void) +{ + struct find_vma_fail2 *skel; + + skel = find_vma_fail2__open_and_load(); + if (!ASSERT_ERR_PTR(skel, "find_vma_fail2__open_and_load")) + find_vma_fail2__destroy(skel); +} + +void serial_test_find_vma(void) +{ + struct find_vma *skel; + + skel = find_vma__open_and_load(); + if (!ASSERT_OK_PTR(skel, "find_vma__open_and_load")) + return; + + skel->bss->target_pid = getpid(); + skel->bss->addr = (__u64)(uintptr_t)test_find_vma_pe; + + test_find_vma_pe(skel); + usleep(100000); /* allow the irq_work to finish */ + test_find_vma_kprobe(skel); + + find_vma__destroy(skel); + test_illegal_write_vma(); + test_illegal_write_task(); +} diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c index 6093728497c7..93ac3f28226c 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_load_bytes.c @@ -30,7 +30,7 @@ void serial_test_flow_dissector_load_bytes(void) /* make sure bpf_skb_load_bytes is not allowed from skb-less context */ - fd = bpf_load_program(BPF_PROG_TYPE_FLOW_DISSECTOR, prog, + fd = bpf_test_load_program(BPF_PROG_TYPE_FLOW_DISSECTOR, prog, ARRAY_SIZE(prog), "GPL", 0, NULL, 0); CHECK(fd < 0, "flow_dissector-bpf_skb_load_bytes-load", diff --git a/tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c b/tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c index f0c6c226aba8..7c79462d2702 100644 --- a/tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c +++ b/tools/testing/selftests/bpf/prog_tests/flow_dissector_reattach.c @@ -47,9 +47,9 @@ static int load_prog(enum bpf_prog_type type) }; int fd; - fd = bpf_load_program(type, prog, ARRAY_SIZE(prog), "GPL", 0, NULL, 0); + fd = bpf_test_load_program(type, prog, ARRAY_SIZE(prog), "GPL", 0, NULL, 0); if (CHECK_FAIL(fd < 0)) - perror("bpf_load_program"); + perror("bpf_test_load_program"); return fd; } diff --git a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c index 522237aa4470..4184c399d4c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/get_stack_raw_tp.c @@ -85,7 +85,6 @@ void test_get_stack_raw_tp(void) const char *file_err = "./test_get_stack_rawtp_err.o"; const char *prog_name = "raw_tracepoint/sys_enter"; int i, err, prog_fd, exp_cnt = MAX_CNT_RAWTP; - struct perf_buffer_opts pb_opts = {}; struct perf_buffer *pb = NULL; struct bpf_link *link = NULL; struct timespec tv = {0, 10}; @@ -94,11 +93,11 @@ void test_get_stack_raw_tp(void) struct bpf_map *map; cpu_set_t cpu_set; - err = bpf_prog_load(file_err, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file_err, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); if (CHECK(err >= 0, "prog_load raw tp", "err %d errno %d\n", err, errno)) return; - err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) return; @@ -124,8 +123,8 @@ void test_get_stack_raw_tp(void) if (!ASSERT_OK_PTR(link, "attach_raw_tp")) goto close_prog; - pb_opts.sample_cb = get_stack_print_output; - pb = perf_buffer__new(bpf_map__fd(map), 8, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(map), 8, get_stack_print_output, + NULL, NULL, NULL); if (!ASSERT_OK_PTR(pb, "perf_buf__new")) goto close_prog; diff --git a/tools/testing/selftests/bpf/prog_tests/global_data.c b/tools/testing/selftests/bpf/prog_tests/global_data.c index afd8639f9a94..9da131b32e13 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_data.c +++ b/tools/testing/selftests/bpf/prog_tests/global_data.c @@ -136,7 +136,7 @@ void test_global_data(void) struct bpf_object *obj; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK(err, "load program", "error %d loading %s\n", err, file)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/global_func_args.c b/tools/testing/selftests/bpf/prog_tests/global_func_args.c index 8bcc2869102f..93a2439237b0 100644 --- a/tools/testing/selftests/bpf/prog_tests/global_func_args.c +++ b/tools/testing/selftests/bpf/prog_tests/global_func_args.c @@ -44,7 +44,7 @@ void test_global_func_args(void) struct bpf_object *obj; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); if (CHECK(err, "load program", "error %d loading %s\n", err, file)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c index 01e51d16c8b8..2a49f8fcde06 100644 --- a/tools/testing/selftests/bpf/prog_tests/kfree_skb.c +++ b/tools/testing/selftests/bpf/prog_tests/kfree_skb.c @@ -66,7 +66,6 @@ void serial_test_kfree_skb(void) struct bpf_map *perf_buf_map, *global_data; struct bpf_program *prog, *fentry, *fexit; struct bpf_object *obj, *obj2 = NULL; - struct perf_buffer_opts pb_opts = {}; struct perf_buffer *pb = NULL; int err, kfree_skb_fd; bool passed = false; @@ -74,7 +73,7 @@ void serial_test_kfree_skb(void) const int zero = 0; bool test_ok[2]; - err = bpf_prog_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("./test_pkt_access.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &tattr.prog_fd); if (CHECK(err, "prog_load sched cls", "err %d errno %d\n", err, errno)) return; @@ -112,9 +111,8 @@ void serial_test_kfree_skb(void) goto close_prog; /* set up perf buffer */ - pb_opts.sample_cb = on_sample; - pb_opts.ctx = &passed; - pb = perf_buffer__new(bpf_map__fd(perf_buf_map), 1, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(perf_buf_map), 1, + on_sample, NULL, &passed, NULL); if (!ASSERT_OK_PTR(pb, "perf_buf__new")) goto close_prog; diff --git a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c index 8073105548ff..540ef28fabff 100644 --- a/tools/testing/selftests/bpf/prog_tests/l4lb_all.c +++ b/tools/testing/selftests/bpf/prog_tests/l4lb_all.c @@ -30,7 +30,7 @@ static void test_l4lb(const char *file) char buf[128]; u32 *magic = (u32 *)buf; - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c index 5a2a689dbb68..4e0b2ec057aa 100644 --- a/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c +++ b/tools/testing/selftests/bpf/prog_tests/load_bytes_relative.c @@ -27,7 +27,7 @@ void test_load_bytes_relative(void) if (CHECK_FAIL(server_fd < 0)) goto close_cgroup_fd; - err = bpf_prog_load("./load_bytes_relative.o", BPF_PROG_TYPE_CGROUP_SKB, + err = bpf_prog_test_load("./load_bytes_relative.o", BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); if (CHECK_FAIL(err)) goto close_server_fd; diff --git a/tools/testing/selftests/bpf/prog_tests/map_lock.c b/tools/testing/selftests/bpf/prog_tests/map_lock.c index ce17b1ed8709..23d19e9cf26a 100644 --- a/tools/testing/selftests/bpf/prog_tests/map_lock.c +++ b/tools/testing/selftests/bpf/prog_tests/map_lock.c @@ -53,9 +53,9 @@ void test_map_lock(void) int err = 0, key = 0, i; void *ret; - err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); if (CHECK_FAIL(err)) { - printf("test_map_lock:bpf_prog_load errno %d\n", errno); + printf("test_map_lock:bpf_prog_test_load errno %d\n", errno); goto close_prog; } map_fd[0] = bpf_find_map(__func__, obj, "hash_map"); diff --git a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c index 7589c03fd26b..eb2feaac81fe 100644 --- a/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c +++ b/tools/testing/selftests/bpf/prog_tests/migrate_reuseport.c @@ -204,8 +204,8 @@ static int pass_ack(struct migrate_reuseport_test_case *test_case) { int err; - err = bpf_link__detach(test_case->link); - if (!ASSERT_OK(err, "bpf_link__detach")) + err = bpf_link__destroy(test_case->link); + if (!ASSERT_OK(err, "bpf_link__destroy")) return -1; test_case->link = NULL; diff --git a/tools/testing/selftests/bpf/prog_tests/perf_buffer.c b/tools/testing/selftests/bpf/prog_tests/perf_buffer.c index 4e32f3586a75..5fc2b3a0711e 100644 --- a/tools/testing/selftests/bpf/prog_tests/perf_buffer.c +++ b/tools/testing/selftests/bpf/prog_tests/perf_buffer.c @@ -47,7 +47,6 @@ void serial_test_perf_buffer(void) { int err, on_len, nr_on_cpus = 0, nr_cpus, i, j; int zero = 0, my_pid = getpid(); - struct perf_buffer_opts pb_opts = {}; struct test_perf_buffer *skel; cpu_set_t cpu_seen; struct perf_buffer *pb; @@ -82,9 +81,8 @@ void serial_test_perf_buffer(void) goto out_close; /* set up perf buffer */ - pb_opts.sample_cb = on_sample; - pb_opts.ctx = &cpu_seen; - pb = perf_buffer__new(bpf_map__fd(skel->maps.perf_buf_map), 1, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(skel->maps.perf_buf_map), 1, + on_sample, NULL, &cpu_seen, NULL); if (!ASSERT_OK_PTR(pb, "perf_buf__new")) goto out_close; diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_access.c index 44b514fabccd..6628710ec3c6 100644 --- a/tools/testing/selftests/bpf/prog_tests/pkt_access.c +++ b/tools/testing/selftests/bpf/prog_tests/pkt_access.c @@ -9,7 +9,7 @@ void test_pkt_access(void) __u32 duration, retval; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c index 939015cd6dba..c9d2d6a1bfcc 100644 --- a/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c +++ b/tools/testing/selftests/bpf/prog_tests/pkt_md_access.c @@ -9,7 +9,7 @@ void test_pkt_md_access(void) __u32 duration, retval; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c index f47e7b1cb32c..8ccba3ab70ee 100644 --- a/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c +++ b/tools/testing/selftests/bpf/prog_tests/queue_stack_map.c @@ -27,7 +27,7 @@ static void test_queue_stack_map_by_type(int type) else return; - err = bpf_prog_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c index 9807336a3016..e2f1445b0e10 100644 --- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_reject_nbd_invalid.c @@ -18,15 +18,15 @@ void test_raw_tp_writable_reject_nbd_invalid(void) BPF_EXIT_INSN(), }; - struct bpf_load_program_attr load_attr = { - .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, - .license = "GPL v2", - .insns = program, - .insns_cnt = sizeof(program) / sizeof(struct bpf_insn), + LIBBPF_OPTS(bpf_prog_load_opts, opts, .log_level = 2, - }; + .log_buf = error, + .log_size = sizeof(error), + ); - bpf_fd = bpf_load_program_xattr(&load_attr, error, sizeof(error)); + bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", + program, sizeof(program) / sizeof(struct bpf_insn), + &opts); if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable load", "failed: %d errno %d\n", bpf_fd, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_test_run.c b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_test_run.c index ddefa1192e5d..239baccabccb 100644 --- a/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_test_run.c +++ b/tools/testing/selftests/bpf/prog_tests/raw_tp_writable_test_run.c @@ -17,15 +17,15 @@ void serial_test_raw_tp_writable_test_run(void) BPF_EXIT_INSN(), }; - struct bpf_load_program_attr load_attr = { - .prog_type = BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, - .license = "GPL v2", - .insns = trace_program, - .insns_cnt = sizeof(trace_program) / sizeof(struct bpf_insn), + LIBBPF_OPTS(bpf_prog_load_opts, trace_opts, .log_level = 2, - }; + .log_buf = error, + .log_size = sizeof(error), + ); - int bpf_fd = bpf_load_program_xattr(&load_attr, error, sizeof(error)); + int bpf_fd = bpf_prog_load(BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE, NULL, "GPL v2", + trace_program, sizeof(trace_program) / sizeof(struct bpf_insn), + &trace_opts); if (CHECK(bpf_fd < 0, "bpf_raw_tracepoint_writable loaded", "failed: %d errno %d\n", bpf_fd, errno)) return; @@ -35,15 +35,14 @@ void serial_test_raw_tp_writable_test_run(void) BPF_EXIT_INSN(), }; - struct bpf_load_program_attr skb_load_attr = { - .prog_type = BPF_PROG_TYPE_SOCKET_FILTER, - .license = "GPL v2", - .insns = skb_program, - .insns_cnt = sizeof(skb_program) / sizeof(struct bpf_insn), - }; + LIBBPF_OPTS(bpf_prog_load_opts, skb_opts, + .log_buf = error, + .log_size = sizeof(error), + ); - int filter_fd = - bpf_load_program_xattr(&skb_load_attr, error, sizeof(error)); + int filter_fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, NULL, "GPL v2", + skb_program, sizeof(skb_program) / sizeof(struct bpf_insn), + &skb_opts); if (CHECK(filter_fd < 0, "test_program_loaded", "failed: %d errno %d\n", filter_fd, errno)) goto out_bpffd; diff --git a/tools/testing/selftests/bpf/prog_tests/signal_pending.c b/tools/testing/selftests/bpf/prog_tests/signal_pending.c index fdfdcff6cbef..aecfe662c070 100644 --- a/tools/testing/selftests/bpf/prog_tests/signal_pending.c +++ b/tools/testing/selftests/bpf/prog_tests/signal_pending.c @@ -22,7 +22,7 @@ static void test_signal_pending_by_type(enum bpf_prog_type prog_type) prog[i] = BPF_ALU64_IMM(BPF_MOV, BPF_REG_0, 0); prog[ARRAY_SIZE(prog) - 1] = BPF_EXIT_INSN(); - prog_fd = bpf_load_program(prog_type, prog, ARRAY_SIZE(prog), + prog_fd = bpf_test_load_program(prog_type, prog, ARRAY_SIZE(prog), "GPL", 0, NULL, 0); CHECK(prog_fd < 0, "test-run", "errno %d\n", errno); diff --git a/tools/testing/selftests/bpf/prog_tests/sk_lookup.c b/tools/testing/selftests/bpf/prog_tests/sk_lookup.c index 6db07401bc49..57846cc7ce36 100644 --- a/tools/testing/selftests/bpf/prog_tests/sk_lookup.c +++ b/tools/testing/selftests/bpf/prog_tests/sk_lookup.c @@ -937,6 +937,37 @@ static void test_drop_on_lookup(struct test_sk_lookup *skel) .connect_to = { EXT_IP6, EXT_PORT }, .listen_at = { EXT_IP6, INT_PORT }, }, + /* The program will drop on success, meaning that the ifindex + * was 1. + */ + { + .desc = "TCP IPv4 drop on valid ifindex", + .lookup_prog = skel->progs.check_ifindex, + .sotype = SOCK_STREAM, + .connect_to = { EXT_IP4, EXT_PORT }, + .listen_at = { EXT_IP4, EXT_PORT }, + }, + { + .desc = "TCP IPv6 drop on valid ifindex", + .lookup_prog = skel->progs.check_ifindex, + .sotype = SOCK_STREAM, + .connect_to = { EXT_IP6, EXT_PORT }, + .listen_at = { EXT_IP6, EXT_PORT }, + }, + { + .desc = "UDP IPv4 drop on valid ifindex", + .lookup_prog = skel->progs.check_ifindex, + .sotype = SOCK_DGRAM, + .connect_to = { EXT_IP4, EXT_PORT }, + .listen_at = { EXT_IP4, EXT_PORT }, + }, + { + .desc = "UDP IPv6 drop on valid ifindex", + .lookup_prog = skel->progs.check_ifindex, + .sotype = SOCK_DGRAM, + .connect_to = { EXT_IP6, EXT_PORT }, + .listen_at = { EXT_IP6, EXT_PORT }, + }, }; const struct test *t; diff --git a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c index c437e6ba8fe2..b5319ba2ee27 100644 --- a/tools/testing/selftests/bpf/prog_tests/skb_ctx.c +++ b/tools/testing/selftests/bpf/prog_tests/skb_ctx.c @@ -32,7 +32,7 @@ void test_skb_ctx(void) int err; int i; - err = bpf_prog_load("./test_skb_ctx.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("./test_skb_ctx.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &tattr.prog_fd); if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) return; @@ -111,4 +111,6 @@ void test_skb_ctx(void) "ctx_out_mark", "skb->mark == %u, expected %d\n", skb.mark, 10); + + bpf_object__close(obj); } diff --git a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c index f302ad84a298..6f802a1c0800 100644 --- a/tools/testing/selftests/bpf/prog_tests/skb_helpers.c +++ b/tools/testing/selftests/bpf/prog_tests/skb_helpers.c @@ -20,7 +20,7 @@ void test_skb_helpers(void) struct bpf_object *obj; int err; - err = bpf_prog_load("./test_skb_helpers.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("./test_skb_helpers.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &tattr.prog_fd); if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/sockopt.c b/tools/testing/selftests/bpf/prog_tests/sockopt.c index 3e8517a8395a..cd09f4c7dd92 100644 --- a/tools/testing/selftests/bpf/prog_tests/sockopt.c +++ b/tools/testing/selftests/bpf/prog_tests/sockopt.c @@ -852,22 +852,21 @@ static struct sockopt_test { static int load_prog(const struct bpf_insn *insns, enum bpf_attach_type expected_attach_type) { - struct bpf_load_program_attr attr = { - .prog_type = BPF_PROG_TYPE_CGROUP_SOCKOPT, + LIBBPF_OPTS(bpf_prog_load_opts, opts, .expected_attach_type = expected_attach_type, - .insns = insns, - .license = "GPL", .log_level = 2, - }; - int fd; + .log_buf = bpf_log_buf, + .log_size = sizeof(bpf_log_buf), + ); + int fd, insns_cnt = 0; for (; - insns[attr.insns_cnt].code != (BPF_JMP | BPF_EXIT); - attr.insns_cnt++) { + insns[insns_cnt].code != (BPF_JMP | BPF_EXIT); + insns_cnt++) { } - attr.insns_cnt++; + insns_cnt++; - fd = bpf_load_program_xattr(&attr, bpf_log_buf, sizeof(bpf_log_buf)); + fd = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCKOPT, NULL, "GPL", insns, insns_cnt, &opts); if (verbose && fd < 0) fprintf(stderr, "%s\n", bpf_log_buf); diff --git a/tools/testing/selftests/bpf/prog_tests/spinlock.c b/tools/testing/selftests/bpf/prog_tests/spinlock.c index 7577a77a4c4c..6307f5d2b417 100644 --- a/tools/testing/selftests/bpf/prog_tests/spinlock.c +++ b/tools/testing/selftests/bpf/prog_tests/spinlock.c @@ -24,9 +24,9 @@ void test_spinlock(void) int err = 0, i; void *ret; - err = bpf_prog_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_CGROUP_SKB, &obj, &prog_fd); if (CHECK_FAIL(err)) { - printf("test_spin_lock:bpf_prog_load errno %d\n", errno); + printf("test_spin_lock:bpf_prog_test_load errno %d\n", errno); goto close_prog; } for (i = 0; i < 4; i++) diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c index 04b476bd62b9..337493d74ec5 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map.c @@ -12,7 +12,7 @@ void test_stacktrace_map(void) struct bpf_object *obj; struct bpf_link *link; - err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c index 4fd30bb651ad..063a14a2060d 100644 --- a/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/stacktrace_map_raw_tp.c @@ -12,7 +12,7 @@ void test_stacktrace_map_raw_tp(void) struct bpf_object *obj; struct bpf_link *link = NULL; - err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/tailcalls.c b/tools/testing/selftests/bpf/prog_tests/tailcalls.c index 9825f1f7bfcc..5dc0f425bd11 100644 --- a/tools/testing/selftests/bpf/prog_tests/tailcalls.c +++ b/tools/testing/selftests/bpf/prog_tests/tailcalls.c @@ -16,7 +16,7 @@ static void test_tailcall_1(void) char prog_name[32]; char buff[128] = {}; - err = bpf_prog_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("tailcall1.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -154,7 +154,7 @@ static void test_tailcall_2(void) char prog_name[32]; char buff[128] = {}; - err = bpf_prog_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("tailcall2.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -228,7 +228,7 @@ static void test_tailcall_count(const char *which) __u32 retval, duration; char buff[128] = {}; - err = bpf_prog_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load(which, BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -324,7 +324,7 @@ static void test_tailcall_4(void) char buff[128] = {}; char prog_name[32]; - err = bpf_prog_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("tailcall4.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -412,7 +412,7 @@ static void test_tailcall_5(void) char buff[128] = {}; char prog_name[32]; - err = bpf_prog_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj, + err = bpf_prog_test_load("tailcall5.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -498,7 +498,7 @@ static void test_tailcall_bpf2bpf_1(void) __u32 retval, duration; char prog_name[32]; - err = bpf_prog_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("tailcall_bpf2bpf1.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -582,7 +582,7 @@ static void test_tailcall_bpf2bpf_2(void) __u32 retval, duration; char buff[128] = {}; - err = bpf_prog_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("tailcall_bpf2bpf2.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -660,7 +660,7 @@ static void test_tailcall_bpf2bpf_3(void) __u32 retval, duration; char prog_name[32]; - err = bpf_prog_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("tailcall_bpf2bpf3.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -757,7 +757,7 @@ static void test_tailcall_bpf2bpf_4(bool noise) __u32 retval, duration; char prog_name[32]; - err = bpf_prog_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS, + err = bpf_prog_test_load("tailcall_bpf2bpf4.o", BPF_PROG_TYPE_SCHED_CLS, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c index 1bdc1d86a50c..17947c9e1d66 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c +++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_rawtp.c @@ -11,7 +11,7 @@ void test_task_fd_query_rawtp(void) __u32 duration = 0; char buf[256]; - err = bpf_prog_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_RAW_TRACEPOINT, &obj, &prog_fd); if (CHECK(err, "prog_load raw tp", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c index 3f131b8fe328..c2a98a7a8dfc 100644 --- a/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c +++ b/tools/testing/selftests/bpf/prog_tests/task_fd_query_tp.c @@ -13,8 +13,8 @@ static void test_task_fd_query_tp_core(const char *probe_name, __u32 duration = 0; char buf[256]; - err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); - if (CHECK(err, "bpf_prog_load", "err %d errno %d\n", err, errno)) + err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); + if (CHECK(err, "bpf_prog_test_load", "err %d errno %d\n", err, errno)) goto close_prog; snprintf(buf, sizeof(buf), diff --git a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c index 594307dffd13..11bf755be4c9 100644 --- a/tools/testing/selftests/bpf/prog_tests/tcp_estats.c +++ b/tools/testing/selftests/bpf/prog_tests/tcp_estats.c @@ -8,7 +8,7 @@ void test_tcp_estats(void) struct bpf_object *obj; __u32 duration = 0; - err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj, &prog_fd); CHECK(err, "", "err %d errno %d\n", err, errno); if (err) return; diff --git a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c index 8652d0a46c87..39e79291c82b 100644 --- a/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c +++ b/tools/testing/selftests/bpf/prog_tests/tp_attach_query.c @@ -35,7 +35,7 @@ void serial_test_tp_attach_query(void) query = malloc(sizeof(*query) + sizeof(__u32) * num_progs); for (i = 0; i < num_progs; i++) { - err = bpf_prog_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i], + err = bpf_prog_test_load(file, BPF_PROG_TYPE_TRACEPOINT, &obj[i], &prog_fd[i]); if (CHECK(err, "prog_load", "err %d errno %d\n", err, errno)) goto cleanup1; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp.c b/tools/testing/selftests/bpf/prog_tests/xdp.c index 48921ff74850..7a7ef9d4e151 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp.c @@ -16,7 +16,7 @@ void test_xdp(void) __u32 duration, retval, size; int err, prog_fd, map_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c index f529e3c923ae..3f5a17c38be5 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_adjust_tail.c @@ -10,7 +10,7 @@ static void test_xdp_adjust_tail_shrink(void) int err, prog_fd; char buf[128]; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -38,7 +38,7 @@ static void test_xdp_adjust_tail_grow(void) __u32 duration, retval, size, expect_sz; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); if (CHECK_FAIL(err)) return; @@ -75,7 +75,7 @@ static void test_xdp_adjust_tail_grow2(void) .data_size_out = 0, /* Per test */ }; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &tattr.prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &tattr.prog_fd); if (CHECK_ATTR(err, "load", "err %d errno %d\n", err, errno)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c index 4c4057262cd8..c6fa390e3aa1 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_attach.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_attach.c @@ -16,7 +16,7 @@ void serial_test_xdp_attach(void) len = sizeof(info); - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj1, &fd1); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj1, &fd1); if (CHECK_FAIL(err)) return; err = bpf_obj_get_info_by_fd(fd1, &info, &len); @@ -24,7 +24,7 @@ void serial_test_xdp_attach(void) goto out_1; id1 = info.id; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj2, &fd2); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj2, &fd2); if (CHECK_FAIL(err)) goto out_1; @@ -34,7 +34,7 @@ void serial_test_xdp_attach(void) goto out_2; id2 = info.id; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj3, &fd3); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj3, &fd3); if (CHECK_FAIL(err)) goto out_2; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/xdp_bpf2bpf.c index 3bd5904b4db5..f99386d1dc4c 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_bpf2bpf.c @@ -49,7 +49,6 @@ void test_xdp_bpf2bpf(void) struct vip key4 = {.protocol = 6, .family = AF_INET}; struct bpf_program *prog; struct perf_buffer *pb = NULL; - struct perf_buffer_opts pb_opts = {}; /* Load XDP program to introspect */ pkt_skel = test_xdp__open_and_load(); @@ -86,10 +85,8 @@ void test_xdp_bpf2bpf(void) goto out; /* Set up perf buffer */ - pb_opts.sample_cb = on_sample; - pb_opts.ctx = &passed; - pb = perf_buffer__new(bpf_map__fd(ftrace_skel->maps.perf_buf_map), - 1, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(ftrace_skel->maps.perf_buf_map), 1, + on_sample, NULL, &passed, NULL); if (!ASSERT_OK_PTR(pb, "perf_buf__new")) goto out; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_info.c b/tools/testing/selftests/bpf/prog_tests/xdp_info.c index 4e2a4fd56f67..abe48e82e1dc 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_info.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_info.c @@ -29,7 +29,7 @@ void serial_test_xdp_info(void) /* Setup prog */ - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c index 7185bee16fe4..15a3900e4370 100644 --- a/tools/testing/selftests/bpf/prog_tests/xdp_perf.c +++ b/tools/testing/selftests/bpf/prog_tests/xdp_perf.c @@ -9,7 +9,7 @@ void test_xdp_perf(void) char in[128], out[128]; int err, prog_fd; - err = bpf_prog_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); + err = bpf_prog_test_load(file, BPF_PROG_TYPE_XDP, &obj, &prog_fd); if (CHECK_FAIL(err)) return; diff --git a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c index 94423902685d..c21e3f545371 100644 --- a/tools/testing/selftests/bpf/progs/bpf_iter_unix.c +++ b/tools/testing/selftests/bpf/progs/bpf_iter_unix.c @@ -49,7 +49,7 @@ int dump_unix(struct bpf_iter__unix *ctx) sock_i_ino(sk)); if (unix_sk->addr) { - if (!UNIX_ABSTRACT(unix_sk)) { + if (unix_sk->addr->name->sun_path[0]) { BPF_SEQ_PRINTF(seq, " %s", unix_sk->addr->name->sun_path); } else { /* The name of the abstract UNIX domain socket starts diff --git a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h index eef5646ddb19..e0f42601be9b 100644 --- a/tools/testing/selftests/bpf/progs/bpf_tracing_net.h +++ b/tools/testing/selftests/bpf/progs/bpf_tracing_net.h @@ -6,8 +6,6 @@ #define AF_INET6 10 #define __SO_ACCEPTCON (1 << 16) -#define UNIX_HASH_SIZE 256 -#define UNIX_ABSTRACT(unix_sk) (unix_sk->addr->hash < UNIX_HASH_SIZE) #define SOL_TCP 6 #define TCP_CONGESTION 13 diff --git a/tools/testing/selftests/bpf/progs/tag.c b/tools/testing/selftests/bpf/progs/btf_decl_tag.c index 1792f4eda095..c88ccc53529a 100644 --- a/tools/testing/selftests/bpf/progs/tag.c +++ b/tools/testing/selftests/bpf/progs/btf_decl_tag.c @@ -4,10 +4,6 @@ #include <bpf/bpf_helpers.h> #include <bpf/bpf_tracing.h> -#ifndef __has_attribute -#define __has_attribute(x) 0 -#endif - #if __has_attribute(btf_decl_tag) #define __tag1 __attribute__((btf_decl_tag("tag1"))) #define __tag2 __attribute__((btf_decl_tag("tag2"))) diff --git a/tools/testing/selftests/bpf/progs/btf_type_tag.c b/tools/testing/selftests/bpf/progs/btf_type_tag.c new file mode 100644 index 000000000000..1d488da7e920 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/btf_type_tag.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +#if __has_attribute(btf_type_tag) +#define __tag1 __attribute__((btf_type_tag("tag1"))) +#define __tag2 __attribute__((btf_type_tag("tag2"))) +volatile const bool skip_tests = false; +#else +#define __tag1 +#define __tag2 +volatile const bool skip_tests = true; +#endif + +struct btf_type_tag_test { + int __tag1 * __tag1 __tag2 *p; +} g; + +SEC("fentry/bpf_fentry_test1") +int BPF_PROG(sub, int x) +{ + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/exhandler_kern.c b/tools/testing/selftests/bpf/progs/exhandler_kern.c new file mode 100644 index 000000000000..f5ca142abf8f --- /dev/null +++ b/tools/testing/selftests/bpf/progs/exhandler_kern.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021, Oracle and/or its affiliates. */ + +#include "vmlinux.h" + +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +char _license[] SEC("license") = "GPL"; + +unsigned int exception_triggered; +int test_pid; + +/* TRACE_EVENT(task_newtask, + * TP_PROTO(struct task_struct *p, u64 clone_flags) + */ +SEC("tp_btf/task_newtask") +int BPF_PROG(trace_task_newtask, struct task_struct *task, u64 clone_flags) +{ + int pid = bpf_get_current_pid_tgid() >> 32; + struct callback_head *work; + void *func; + + if (test_pid != pid) + return 0; + + /* To verify we hit an exception we dereference task->task_works->func. + * If task work has been added, + * - task->task_works is non-NULL; and + * - task->task_works->func is non-NULL also (the callback function + * must be specified for the task work. + * + * However, for a newly-created task, task->task_works is NULLed, + * so we know the exception handler triggered if task_works is + * NULL and func is NULL. + */ + work = task->task_works; + func = work->func; + if (!work && !func) + exception_triggered++; + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c index 49a84a3a2306..48cd14b43741 100644 --- a/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/progs/fexit_bpf2bpf.c @@ -73,7 +73,7 @@ int test_subprog2(struct args_subprog2 *ctx) __builtin_preserve_access_index(&skb->len)); ret = ctx->ret; - /* bpf_prog_load() loads "test_pkt_access.o" with BPF_F_TEST_RND_HI32 + /* bpf_prog_test_load() loads "test_pkt_access.o" with BPF_F_TEST_RND_HI32 * which randomizes upper 32 bits after BPF_ALU32 insns. * Hence after 'w0 <<= 1' upper bits of $rax are random. * That is expected and correct. Trim them. diff --git a/tools/testing/selftests/bpf/progs/find_vma.c b/tools/testing/selftests/bpf/progs/find_vma.c new file mode 100644 index 000000000000..38034fb82530 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/find_vma.c @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> + +char _license[] SEC("license") = "GPL"; + +struct callback_ctx { + int dummy; +}; + +#define VM_EXEC 0x00000004 +#define DNAME_INLINE_LEN 32 + +pid_t target_pid = 0; +char d_iname[DNAME_INLINE_LEN] = {0}; +__u32 found_vm_exec = 0; +__u64 addr = 0; +int find_zero_ret = -1; +int find_addr_ret = -1; + +static long check_vma(struct task_struct *task, struct vm_area_struct *vma, + struct callback_ctx *data) +{ + if (vma->vm_file) + bpf_probe_read_kernel_str(d_iname, DNAME_INLINE_LEN - 1, + vma->vm_file->f_path.dentry->d_iname); + + /* check for VM_EXEC */ + if (vma->vm_flags & VM_EXEC) + found_vm_exec = 1; + + return 0; +} + +SEC("raw_tp/sys_enter") +int handle_getpid(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct callback_ctx data = {}; + + if (task->pid != target_pid) + return 0; + + find_addr_ret = bpf_find_vma(task, addr, check_vma, &data, 0); + + /* this should return -ENOENT */ + find_zero_ret = bpf_find_vma(task, 0, check_vma, &data, 0); + return 0; +} + +SEC("perf_event") +int handle_pe(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct callback_ctx data = {}; + + if (task->pid != target_pid) + return 0; + + find_addr_ret = bpf_find_vma(task, addr, check_vma, &data, 0); + + /* In NMI, this should return -EBUSY, as the previous call is using + * the irq_work. + */ + find_zero_ret = bpf_find_vma(task, 0, check_vma, &data, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/find_vma_fail1.c b/tools/testing/selftests/bpf/progs/find_vma_fail1.c new file mode 100644 index 000000000000..b3b326b8e2d1 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/find_vma_fail1.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +struct callback_ctx { + int dummy; +}; + +static long write_vma(struct task_struct *task, struct vm_area_struct *vma, + struct callback_ctx *data) +{ + /* writing to vma, which is illegal */ + vma->vm_flags |= 0x55; + + return 0; +} + +SEC("raw_tp/sys_enter") +int handle_getpid(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct callback_ctx data = {}; + + bpf_find_vma(task, 0, write_vma, &data, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/find_vma_fail2.c b/tools/testing/selftests/bpf/progs/find_vma_fail2.c new file mode 100644 index 000000000000..9bcf3203e26b --- /dev/null +++ b/tools/testing/selftests/bpf/progs/find_vma_fail2.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2021 Facebook */ +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> + +char _license[] SEC("license") = "GPL"; + +struct callback_ctx { + int dummy; +}; + +static long write_task(struct task_struct *task, struct vm_area_struct *vma, + struct callback_ctx *data) +{ + /* writing to task, which is illegal */ + task->mm = NULL; + + return 0; +} + +SEC("raw_tp/sys_enter") +int handle_getpid(void) +{ + struct task_struct *task = bpf_get_current_task_btf(); + struct callback_ctx data = {}; + + bpf_find_vma(task, 0, write_task, &data, 0); + return 0; +} diff --git a/tools/testing/selftests/bpf/progs/test_l4lb.c b/tools/testing/selftests/bpf/progs/test_l4lb.c index 04fee08863cb..c26057ec46dc 100644 --- a/tools/testing/selftests/bpf/progs/test_l4lb.c +++ b/tools/testing/selftests/bpf/progs/test_l4lb.c @@ -448,7 +448,7 @@ static __always_inline int process_packet(void *data, __u64 off, void *data_end, return bpf_redirect(ifindex, 0); } -SEC("l4lb-demo") +SEC("tc") int balancer_ingress(struct __sk_buff *ctx) { void *data_end = (void *)(long)ctx->data_end; diff --git a/tools/testing/selftests/bpf/progs/test_l4lb_noinline.c b/tools/testing/selftests/bpf/progs/test_l4lb_noinline.c index b9e2753f4f91..19e4d2071c60 100644 --- a/tools/testing/selftests/bpf/progs/test_l4lb_noinline.c +++ b/tools/testing/selftests/bpf/progs/test_l4lb_noinline.c @@ -447,7 +447,7 @@ static __noinline int process_packet(void *data, __u64 off, void *data_end, return bpf_redirect(ifindex, 0); } -SEC("l4lb-demo") +SEC("tc") int balancer_ingress(struct __sk_buff *ctx) { void *data_end = (void *)(long)ctx->data_end; diff --git a/tools/testing/selftests/bpf/progs/test_map_lock.c b/tools/testing/selftests/bpf/progs/test_map_lock.c index b5c07ae7b68f..acf073db9e8b 100644 --- a/tools/testing/selftests/bpf/progs/test_map_lock.c +++ b/tools/testing/selftests/bpf/progs/test_map_lock.c @@ -30,7 +30,7 @@ struct { __type(value, struct array_elem); } array_map SEC(".maps"); -SEC("map_lock_demo") +SEC("cgroup/skb") int bpf_map_lock_test(struct __sk_buff *skb) { struct hmap_elem zero = {}, *val; diff --git a/tools/testing/selftests/bpf/progs/test_queue_stack_map.h b/tools/testing/selftests/bpf/progs/test_queue_stack_map.h index 0fcd3ff0e38a..648e8cab7a23 100644 --- a/tools/testing/selftests/bpf/progs/test_queue_stack_map.h +++ b/tools/testing/selftests/bpf/progs/test_queue_stack_map.h @@ -24,7 +24,7 @@ struct { __uint(value_size, sizeof(__u32)); } map_out SEC(".maps"); -SEC("test") +SEC("tc") int _test(struct __sk_buff *skb) { void *data_end = (void *)(long)skb->data_end; diff --git a/tools/testing/selftests/bpf/progs/test_sk_lookup.c b/tools/testing/selftests/bpf/progs/test_sk_lookup.c index 19d2465d9442..83b0aaa52ef7 100644 --- a/tools/testing/selftests/bpf/progs/test_sk_lookup.c +++ b/tools/testing/selftests/bpf/progs/test_sk_lookup.c @@ -84,6 +84,14 @@ int lookup_drop(struct bpf_sk_lookup *ctx) return SK_DROP; } +SEC("sk_lookup") +int check_ifindex(struct bpf_sk_lookup *ctx) +{ + if (ctx->ingress_ifindex == 1) + return SK_DROP; + return SK_PASS; +} + SEC("sk_reuseport") int reuseport_pass(struct sk_reuseport_md *ctx) { diff --git a/tools/testing/selftests/bpf/progs/test_sk_storage_tracing.c b/tools/testing/selftests/bpf/progs/test_sk_storage_tracing.c index 8e94e5c080aa..6dc1f28fc4b6 100644 --- a/tools/testing/selftests/bpf/progs/test_sk_storage_tracing.c +++ b/tools/testing/selftests/bpf/progs/test_sk_storage_tracing.c @@ -68,7 +68,7 @@ static void set_task_info(struct sock *sk) } SEC("fentry/inet_csk_listen_start") -int BPF_PROG(trace_inet_csk_listen_start, struct sock *sk, int backlog) +int BPF_PROG(trace_inet_csk_listen_start, struct sock *sk) { set_task_info(sk); diff --git a/tools/testing/selftests/bpf/progs/test_skb_ctx.c b/tools/testing/selftests/bpf/progs/test_skb_ctx.c index 1d61b36e6067..c482110cfc95 100644 --- a/tools/testing/selftests/bpf/progs/test_skb_ctx.c +++ b/tools/testing/selftests/bpf/progs/test_skb_ctx.c @@ -5,7 +5,7 @@ char _license[] SEC("license") = "GPL"; -SEC("skb_ctx") +SEC("tc") int process(struct __sk_buff *skb) { #pragma clang loop unroll(full) diff --git a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c index a408ec95cba4..eacda9fe07eb 100644 --- a/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c +++ b/tools/testing/selftests/bpf/progs/test_skc_to_unix_sock.c @@ -23,7 +23,7 @@ int BPF_PROG(unix_listen, struct socket *sock, int backlog) if (!unix_sk) return 0; - if (!UNIX_ABSTRACT(unix_sk)) + if (unix_sk->addr->name->sun_path[0]) return 0; len = unix_sk->addr->len - sizeof(short); diff --git a/tools/testing/selftests/bpf/progs/test_spin_lock.c b/tools/testing/selftests/bpf/progs/test_spin_lock.c index 0d31a3b3505f..7e88309d3229 100644 --- a/tools/testing/selftests/bpf/progs/test_spin_lock.c +++ b/tools/testing/selftests/bpf/progs/test_spin_lock.c @@ -45,7 +45,7 @@ struct { #define CREDIT_PER_NS(delta, rate) (((delta) * rate) >> 20) -SEC("spin_lock_demo") +SEC("tc") int bpf_sping_lock_test(struct __sk_buff *skb) { volatile int credit = 0, max_credit = 100, pkt_len = 64; diff --git a/tools/testing/selftests/bpf/progs/test_tcp_estats.c b/tools/testing/selftests/bpf/progs/test_tcp_estats.c index 2c5c602c6011..e2ae049c2f85 100644 --- a/tools/testing/selftests/bpf/progs/test_tcp_estats.c +++ b/tools/testing/selftests/bpf/progs/test_tcp_estats.c @@ -244,7 +244,7 @@ static __always_inline void send_basic_event(struct sock *sk, bpf_map_update_elem(&ev_record_map, &key, &ev, BPF_ANY); } -SEC("dummy_tracepoint") +SEC("tp/dummy/tracepoint") int _dummy_tracepoint(struct dummy_tracepoint_args *arg) { if (!arg->sock) diff --git a/tools/testing/selftests/bpf/test_btf.h b/tools/testing/selftests/bpf/test_btf.h index 32c7a57867da..128989bed8b7 100644 --- a/tools/testing/selftests/bpf/test_btf.h +++ b/tools/testing/selftests/bpf/test_btf.h @@ -72,4 +72,7 @@ #define BTF_DECL_TAG_ENC(value, type, component_idx) \ BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_DECL_TAG, 0, 0), type), (component_idx) +#define BTF_TYPE_TAG_ENC(value, type) \ + BTF_TYPE_ENC(value, BTF_INFO_ENC(BTF_KIND_TYPE_TAG, 0, 0), type) + #endif /* _TEST_BTF_H */ diff --git a/tools/testing/selftests/bpf/test_cgroup_storage.c b/tools/testing/selftests/bpf/test_cgroup_storage.c index 0cda61da5d39..a63787e7bb1a 100644 --- a/tools/testing/selftests/bpf/test_cgroup_storage.c +++ b/tools/testing/selftests/bpf/test_cgroup_storage.c @@ -8,6 +8,7 @@ #include "bpf_rlimit.h" #include "cgroup_helpers.h" +#include "testing_helpers.h" char bpf_log_buf[BPF_LOG_BUF_SIZE]; @@ -66,7 +67,7 @@ int main(int argc, char **argv) prog[0].imm = percpu_map_fd; prog[7].imm = map_fd; - prog_fd = bpf_load_program(BPF_PROG_TYPE_CGROUP_SKB, + prog_fd = bpf_test_load_program(BPF_PROG_TYPE_CGROUP_SKB, prog, insns_cnt, "GPL", 0, bpf_log_buf, BPF_LOG_BUF_SIZE); if (prog_fd < 0) { diff --git a/tools/testing/selftests/bpf/test_dev_cgroup.c b/tools/testing/selftests/bpf/test_dev_cgroup.c index 804dddd97d4c..c299d3452695 100644 --- a/tools/testing/selftests/bpf/test_dev_cgroup.c +++ b/tools/testing/selftests/bpf/test_dev_cgroup.c @@ -14,6 +14,7 @@ #include <bpf/libbpf.h> #include "cgroup_helpers.h" +#include "testing_helpers.h" #include "bpf_rlimit.h" #define DEV_CGROUP_PROG "./dev_cgroup.o" @@ -27,7 +28,7 @@ int main(int argc, char **argv) int prog_fd, cgroup_fd; __u32 prog_cnt; - if (bpf_prog_load(DEV_CGROUP_PROG, BPF_PROG_TYPE_CGROUP_DEVICE, + if (bpf_prog_test_load(DEV_CGROUP_PROG, BPF_PROG_TYPE_CGROUP_DEVICE, &obj, &prog_fd)) { printf("Failed to load DEV_CGROUP program\n"); goto out; diff --git a/tools/testing/selftests/bpf/test_lirc_mode2_user.c b/tools/testing/selftests/bpf/test_lirc_mode2_user.c index fb5fd6841ef3..ebf68dce5504 100644 --- a/tools/testing/selftests/bpf/test_lirc_mode2_user.c +++ b/tools/testing/selftests/bpf/test_lirc_mode2_user.c @@ -45,6 +45,8 @@ #include <bpf/bpf.h> #include <bpf/libbpf.h> +#include "testing_helpers.h" + int main(int argc, char **argv) { struct bpf_object *obj; @@ -58,8 +60,8 @@ int main(int argc, char **argv) return 2; } - ret = bpf_prog_load("test_lirc_mode2_kern.o", - BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); + ret = bpf_prog_test_load("test_lirc_mode2_kern.o", + BPF_PROG_TYPE_LIRC_MODE2, &obj, &progfd); if (ret) { printf("Failed to load bpf program\n"); return 1; diff --git a/tools/testing/selftests/bpf/test_lru_map.c b/tools/testing/selftests/bpf/test_lru_map.c index 7e9049fa3edf..7f3d1d8460b4 100644 --- a/tools/testing/selftests/bpf/test_lru_map.c +++ b/tools/testing/selftests/bpf/test_lru_map.c @@ -42,7 +42,6 @@ static int create_map(int map_type, int map_flags, unsigned int size) static int bpf_map_lookup_elem_with_ref_bit(int fd, unsigned long long key, void *value) { - struct bpf_load_program_attr prog; struct bpf_create_map_attr map; struct bpf_insn insns[] = { BPF_LD_MAP_VALUE(BPF_REG_9, 0, 0), @@ -76,13 +75,7 @@ static int bpf_map_lookup_elem_with_ref_bit(int fd, unsigned long long key, insns[0].imm = mfd; - memset(&prog, 0, sizeof(prog)); - prog.prog_type = BPF_PROG_TYPE_SCHED_CLS; - prog.insns = insns; - prog.insns_cnt = ARRAY_SIZE(insns); - prog.license = "GPL"; - - pfd = bpf_load_program_xattr(&prog, NULL, 0); + pfd = bpf_prog_load(BPF_PROG_TYPE_SCHED_CLS, NULL, "GPL", insns, ARRAY_SIZE(insns), NULL); if (pfd < 0) { close(mfd); return -1; diff --git a/tools/testing/selftests/bpf/test_maps.c b/tools/testing/selftests/bpf/test_maps.c index c7a36a9378f8..8b31bc1a801d 100644 --- a/tools/testing/selftests/bpf/test_maps.c +++ b/tools/testing/selftests/bpf/test_maps.c @@ -25,6 +25,7 @@ #include "bpf_util.h" #include "bpf_rlimit.h" #include "test_maps.h" +#include "testing_helpers.h" #ifndef ENOTSUPP #define ENOTSUPP 524 @@ -830,21 +831,21 @@ static void test_sockmap(unsigned int tasks, void *data) } /* Load SK_SKB program and Attach */ - err = bpf_prog_load(SOCKMAP_PARSE_PROG, + err = bpf_prog_test_load(SOCKMAP_PARSE_PROG, BPF_PROG_TYPE_SK_SKB, &obj, &parse_prog); if (err) { printf("Failed to load SK_SKB parse prog\n"); goto out_sockmap; } - err = bpf_prog_load(SOCKMAP_TCP_MSG_PROG, + err = bpf_prog_test_load(SOCKMAP_TCP_MSG_PROG, BPF_PROG_TYPE_SK_MSG, &obj, &msg_prog); if (err) { printf("Failed to load SK_SKB msg prog\n"); goto out_sockmap; } - err = bpf_prog_load(SOCKMAP_VERDICT_PROG, + err = bpf_prog_test_load(SOCKMAP_VERDICT_PROG, BPF_PROG_TYPE_SK_SKB, &obj, &verdict_prog); if (err) { printf("Failed to load SK_SKB verdict prog\n"); diff --git a/tools/testing/selftests/bpf/test_sock.c b/tools/testing/selftests/bpf/test_sock.c index 9613f7538840..e8edd3dd3ec2 100644 --- a/tools/testing/selftests/bpf/test_sock.c +++ b/tools/testing/selftests/bpf/test_sock.c @@ -328,18 +328,17 @@ static size_t probe_prog_length(const struct bpf_insn *fp) static int load_sock_prog(const struct bpf_insn *prog, enum bpf_attach_type attach_type) { - struct bpf_load_program_attr attr; - int ret; - - memset(&attr, 0, sizeof(struct bpf_load_program_attr)); - attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK; - attr.expected_attach_type = attach_type; - attr.insns = prog; - attr.insns_cnt = probe_prog_length(attr.insns); - attr.license = "GPL"; - attr.log_level = 2; - - ret = bpf_load_program_xattr(&attr, bpf_log_buf, BPF_LOG_BUF_SIZE); + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int ret, insn_cnt; + + insn_cnt = probe_prog_length(prog); + + opts.expected_attach_type = attach_type; + opts.log_buf = bpf_log_buf; + opts.log_size = BPF_LOG_BUF_SIZE; + opts.log_level = 2; + + ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK, NULL, "GPL", prog, insn_cnt, &opts); if (verbose && ret < 0) fprintf(stderr, "%s\n", bpf_log_buf); diff --git a/tools/testing/selftests/bpf/test_sock_addr.c b/tools/testing/selftests/bpf/test_sock_addr.c index aa3f185fcb89..05c9e4944c01 100644 --- a/tools/testing/selftests/bpf/test_sock_addr.c +++ b/tools/testing/selftests/bpf/test_sock_addr.c @@ -645,17 +645,14 @@ static int mk_sockaddr(int domain, const char *ip, unsigned short port, static int load_insns(const struct sock_addr_test *test, const struct bpf_insn *insns, size_t insns_cnt) { - struct bpf_load_program_attr load_attr; + LIBBPF_OPTS(bpf_prog_load_opts, opts); int ret; - memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); - load_attr.prog_type = BPF_PROG_TYPE_CGROUP_SOCK_ADDR; - load_attr.expected_attach_type = test->expected_attach_type; - load_attr.insns = insns; - load_attr.insns_cnt = insns_cnt; - load_attr.license = "GPL"; + opts.expected_attach_type = test->expected_attach_type; + opts.log_buf = bpf_log_buf; + opts.log_size = BPF_LOG_BUF_SIZE; - ret = bpf_load_program_xattr(&load_attr, bpf_log_buf, BPF_LOG_BUF_SIZE); + ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SOCK_ADDR, NULL, "GPL", insns, insns_cnt, &opts); if (ret < 0 && test->expected_result != LOAD_REJECT) { log_err(">>> Loading program error.\n" ">>> Verifier output:\n%s\n-------\n", bpf_log_buf); diff --git a/tools/testing/selftests/bpf/test_stub.c b/tools/testing/selftests/bpf/test_stub.c deleted file mode 100644 index 47e132726203..000000000000 --- a/tools/testing/selftests/bpf/test_stub.c +++ /dev/null @@ -1,44 +0,0 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) -/* Copyright (C) 2019 Netronome Systems, Inc. */ - -#include <bpf/bpf.h> -#include <bpf/libbpf.h> -#include <string.h> - -int extra_prog_load_log_flags = 0; - -int bpf_prog_test_load(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd) -{ - struct bpf_prog_load_attr attr; - - memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); - attr.file = file; - attr.prog_type = type; - attr.expected_attach_type = 0; - attr.prog_flags = BPF_F_TEST_RND_HI32; - attr.log_level = extra_prog_load_log_flags; - - return bpf_prog_load_xattr(&attr, pobj, prog_fd); -} - -int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, - size_t insns_cnt, const char *license, - __u32 kern_version, char *log_buf, - size_t log_buf_sz) -{ - struct bpf_load_program_attr load_attr; - - memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); - load_attr.prog_type = type; - load_attr.expected_attach_type = 0; - load_attr.name = NULL; - load_attr.insns = insns; - load_attr.insns_cnt = insns_cnt; - load_attr.license = license; - load_attr.kern_version = kern_version; - load_attr.prog_flags = BPF_F_TEST_RND_HI32; - load_attr.log_level = extra_prog_load_log_flags; - - return bpf_load_program_xattr(&load_attr, log_buf, log_buf_sz); -} diff --git a/tools/testing/selftests/bpf/test_sysctl.c b/tools/testing/selftests/bpf/test_sysctl.c index a3bb6d399daa..4f6cf833b522 100644 --- a/tools/testing/selftests/bpf/test_sysctl.c +++ b/tools/testing/selftests/bpf/test_sysctl.c @@ -17,6 +17,7 @@ #include "bpf_rlimit.h" #include "bpf_util.h" #include "cgroup_helpers.h" +#include "testing_helpers.h" #define CG_PATH "/foo" #define MAX_INSNS 512 @@ -1435,14 +1436,10 @@ static int load_sysctl_prog_insns(struct sysctl_test *test, const char *sysctl_path) { struct bpf_insn *prog = test->insns; - struct bpf_load_program_attr attr; - int ret; + LIBBPF_OPTS(bpf_prog_load_opts, opts); + int ret, insn_cnt; - memset(&attr, 0, sizeof(struct bpf_load_program_attr)); - attr.prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL; - attr.insns = prog; - attr.insns_cnt = probe_prog_length(attr.insns); - attr.license = "GPL"; + insn_cnt = probe_prog_length(prog); if (test->fixup_value_insn) { char buf[128]; @@ -1465,7 +1462,10 @@ static int load_sysctl_prog_insns(struct sysctl_test *test, return -1; } - ret = bpf_load_program_xattr(&attr, bpf_log_buf, BPF_LOG_BUF_SIZE); + opts.log_buf = bpf_log_buf; + opts.log_size = BPF_LOG_BUF_SIZE; + + ret = bpf_prog_load(BPF_PROG_TYPE_CGROUP_SYSCTL, NULL, "GPL", prog, insn_cnt, &opts); if (ret < 0 && test->result != LOAD_REJECT) { log_err(">>> Loading program error.\n" ">>> Verifier output:\n%s\n-------\n", bpf_log_buf); @@ -1476,15 +1476,10 @@ static int load_sysctl_prog_insns(struct sysctl_test *test, static int load_sysctl_prog_file(struct sysctl_test *test) { - struct bpf_prog_load_attr attr; struct bpf_object *obj; int prog_fd; - memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); - attr.file = test->prog_file; - attr.prog_type = BPF_PROG_TYPE_CGROUP_SYSCTL; - - if (bpf_prog_load_xattr(&attr, &obj, &prog_fd)) { + if (bpf_prog_test_load(test->prog_file, BPF_PROG_TYPE_CGROUP_SYSCTL, &obj, &prog_fd)) { if (test->result != LOAD_REJECT) log_err(">>> Loading program (%s) error.\n", test->prog_file); diff --git a/tools/testing/selftests/bpf/test_tag.c b/tools/testing/selftests/bpf/test_tag.c index 6272c784ca2a..5c7bea525626 100644 --- a/tools/testing/selftests/bpf/test_tag.c +++ b/tools/testing/selftests/bpf/test_tag.c @@ -21,6 +21,7 @@ #include "../../../include/linux/filter.h" #include "bpf_rlimit.h" +#include "testing_helpers.h" static struct bpf_insn prog[BPF_MAXINSNS]; @@ -57,7 +58,7 @@ static int bpf_try_load_prog(int insns, int fd_map, int fd_prog; bpf_filler(insns, fd_map); - fd_prog = bpf_load_program(BPF_PROG_TYPE_SCHED_CLS, prog, insns, "", 0, + fd_prog = bpf_test_load_program(BPF_PROG_TYPE_SCHED_CLS, prog, insns, "", 0, NULL, 0); assert(fd_prog > 0); if (fd_map > 0) diff --git a/tools/testing/selftests/bpf/test_tcpnotify_user.c b/tools/testing/selftests/bpf/test_tcpnotify_user.c index 4a39304cc5a6..4c5114765b23 100644 --- a/tools/testing/selftests/bpf/test_tcpnotify_user.c +++ b/tools/testing/selftests/bpf/test_tcpnotify_user.c @@ -25,6 +25,7 @@ #include "test_tcpnotify.h" #include "trace_helpers.h" +#include "testing_helpers.h" #define SOCKET_BUFFER_SIZE (getpagesize() < 8192L ? getpagesize() : 8192L) @@ -71,7 +72,6 @@ int main(int argc, char **argv) { const char *file = "test_tcpnotify_kern.o"; struct bpf_map *perf_map, *global_map; - struct perf_buffer_opts pb_opts = {}; struct tcpnotify_globals g = {0}; struct perf_buffer *pb = NULL; const char *cg_path = "/foo"; @@ -92,7 +92,7 @@ int main(int argc, char **argv) if (cg_fd < 0) goto err; - if (bpf_prog_load(file, BPF_PROG_TYPE_SOCK_OPS, &obj, &prog_fd)) { + if (bpf_prog_test_load(file, BPF_PROG_TYPE_SOCK_OPS, &obj, &prog_fd)) { printf("FAILED: load_bpf_file failed for: %s\n", file); goto err; } @@ -116,8 +116,7 @@ int main(int argc, char **argv) return -1; } - pb_opts.sample_cb = dummyfn; - pb = perf_buffer__new(bpf_map__fd(perf_map), 8, &pb_opts); + pb = perf_buffer__new(bpf_map__fd(perf_map), 8, dummyfn, NULL, NULL, NULL); if (!pb) goto err; diff --git a/tools/testing/selftests/bpf/test_verifier.c b/tools/testing/selftests/bpf/test_verifier.c index 465ef3f112c0..9fbef396a716 100644 --- a/tools/testing/selftests/bpf/test_verifier.c +++ b/tools/testing/selftests/bpf/test_verifier.c @@ -499,8 +499,7 @@ static int create_prog_dummy_simple(enum bpf_prog_type prog_type, int ret) BPF_EXIT_INSN(), }; - return bpf_load_program(prog_type, prog, - ARRAY_SIZE(prog), "GPL", 0, NULL, 0); + return bpf_prog_load(prog_type, NULL, "GPL", prog, ARRAY_SIZE(prog), NULL); } static int create_prog_dummy_loop(enum bpf_prog_type prog_type, int mfd, @@ -515,8 +514,7 @@ static int create_prog_dummy_loop(enum bpf_prog_type prog_type, int mfd, BPF_EXIT_INSN(), }; - return bpf_load_program(prog_type, prog, - ARRAY_SIZE(prog), "GPL", 0, NULL, 0); + return bpf_prog_load(prog_type, NULL, "GPL", prog, ARRAY_SIZE(prog), NULL); } static int create_prog_array(enum bpf_prog_type prog_type, uint32_t max_elem, @@ -1089,7 +1087,7 @@ static void do_test_single(struct bpf_test *test, bool unpriv, int fd_prog, expected_ret, alignment_prevented_execution; int prog_len, prog_type = test->prog_type; struct bpf_insn *prog = test->insns; - struct bpf_load_program_attr attr; + LIBBPF_OPTS(bpf_prog_load_opts, opts); int run_errs, run_successes; int map_fds[MAX_NR_MAPS]; const char *expected_err; @@ -1129,32 +1127,34 @@ static void do_test_single(struct bpf_test *test, bool unpriv, test->result_unpriv : test->result; expected_err = unpriv && test->errstr_unpriv ? test->errstr_unpriv : test->errstr; - memset(&attr, 0, sizeof(attr)); - attr.prog_type = prog_type; - attr.expected_attach_type = test->expected_attach_type; - attr.insns = prog; - attr.insns_cnt = prog_len; - attr.license = "GPL"; + + opts.expected_attach_type = test->expected_attach_type; if (verbose) - attr.log_level = 1; + opts.log_level = 1; else if (expected_ret == VERBOSE_ACCEPT) - attr.log_level = 2; + opts.log_level = 2; else - attr.log_level = 4; - attr.prog_flags = pflags; + opts.log_level = 4; + opts.prog_flags = pflags; if (prog_type == BPF_PROG_TYPE_TRACING && test->kfunc) { - attr.attach_btf_id = libbpf_find_vmlinux_btf_id(test->kfunc, - attr.expected_attach_type); - if (attr.attach_btf_id < 0) { + int attach_btf_id; + + attach_btf_id = libbpf_find_vmlinux_btf_id(test->kfunc, + opts.expected_attach_type); + if (attach_btf_id < 0) { printf("FAIL\nFailed to find BTF ID for '%s'!\n", test->kfunc); (*errors)++; return; } + + opts.attach_btf_id = attach_btf_id; } - fd_prog = bpf_load_program_xattr(&attr, bpf_vlog, sizeof(bpf_vlog)); + opts.log_buf = bpf_vlog; + opts.log_size = sizeof(bpf_vlog); + fd_prog = bpf_prog_load(prog_type, NULL, "GPL", prog, prog_len, &opts); saved_errno = errno; /* BPF_PROG_TYPE_TRACING requires more setup and diff --git a/tools/testing/selftests/bpf/testing_helpers.c b/tools/testing/selftests/bpf/testing_helpers.c index 800d503e5cb4..52c2f24e0898 100644 --- a/tools/testing/selftests/bpf/testing_helpers.c +++ b/tools/testing/selftests/bpf/testing_helpers.c @@ -1,7 +1,11 @@ // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +/* Copyright (C) 2019 Netronome Systems, Inc. */ /* Copyright (C) 2020 Facebook, Inc. */ #include <stdlib.h> +#include <string.h> #include <errno.h> +#include <bpf/bpf.h> +#include <bpf/libbpf.h> #include "testing_helpers.h" int parse_num_list(const char *s, bool **num_set, int *num_set_len) @@ -78,3 +82,59 @@ __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info) } return info->prog_id; } + +int extra_prog_load_log_flags = 0; + +int bpf_prog_test_load(const char *file, enum bpf_prog_type type, + struct bpf_object **pobj, int *prog_fd) +{ + struct bpf_object_load_attr attr = {}; + struct bpf_object *obj; + struct bpf_program *prog; + int err; + + obj = bpf_object__open(file); + if (!obj) + return -errno; + + prog = bpf_object__next_program(obj, NULL); + if (!prog) { + err = -ENOENT; + goto err_out; + } + + if (type != BPF_PROG_TYPE_UNSPEC) + bpf_program__set_type(prog, type); + + bpf_program__set_extra_flags(prog, BPF_F_TEST_RND_HI32); + + attr.obj = obj; + attr.log_level = extra_prog_load_log_flags; + err = bpf_object__load_xattr(&attr); + if (err) + goto err_out; + + *pobj = obj; + *prog_fd = bpf_program__fd(prog); + + return 0; +err_out: + bpf_object__close(obj); + return err; +} + +int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, + size_t insns_cnt, const char *license, + __u32 kern_version, char *log_buf, + size_t log_buf_sz) +{ + LIBBPF_OPTS(bpf_prog_load_opts, opts, + .kern_version = kern_version, + .prog_flags = BPF_F_TEST_RND_HI32, + .log_level = extra_prog_load_log_flags, + .log_buf = log_buf, + .log_size = log_buf_sz, + ); + + return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts); +} diff --git a/tools/testing/selftests/bpf/testing_helpers.h b/tools/testing/selftests/bpf/testing_helpers.h index d4f8e749611b..f46ebc476ee8 100644 --- a/tools/testing/selftests/bpf/testing_helpers.h +++ b/tools/testing/selftests/bpf/testing_helpers.h @@ -6,3 +6,9 @@ int parse_num_list(const char *s, bool **set, int *set_len); __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info); +int bpf_prog_test_load(const char *file, enum bpf_prog_type type, + struct bpf_object **pobj, int *prog_fd); +int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, + size_t insns_cnt, const char *license, + __u32 kern_version, char *log_buf, + size_t log_buf_sz); diff --git a/tools/testing/selftests/bpf/verifier/ctx_sk_lookup.c b/tools/testing/selftests/bpf/verifier/ctx_sk_lookup.c index d78627be060f..a2b006e2fd06 100644 --- a/tools/testing/selftests/bpf/verifier/ctx_sk_lookup.c +++ b/tools/testing/selftests/bpf/verifier/ctx_sk_lookup.c @@ -229,6 +229,24 @@ BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, offsetof(struct bpf_sk_lookup, local_port)), + /* 1-byte read from ingress_ifindex field */ + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex)), + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex) + 1), + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex) + 2), + BPF_LDX_MEM(BPF_B, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex) + 3), + /* 2-byte read from ingress_ifindex field */ + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex)), + BPF_LDX_MEM(BPF_H, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex) + 2), + /* 4-byte read from ingress_ifindex field */ + BPF_LDX_MEM(BPF_W, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex)), + /* 8-byte read from sk field */ BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, offsetof(struct bpf_sk_lookup, sk)), @@ -351,6 +369,20 @@ .expected_attach_type = BPF_SK_LOOKUP, .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, }, +{ + "invalid 8-byte read from bpf_sk_lookup ingress_ifindex field", + .insns = { + BPF_LDX_MEM(BPF_DW, BPF_REG_0, BPF_REG_1, + offsetof(struct bpf_sk_lookup, ingress_ifindex)), + BPF_MOV32_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }, + .errstr = "invalid bpf_context access", + .result = REJECT, + .prog_type = BPF_PROG_TYPE_SK_LOOKUP, + .expected_attach_type = BPF_SK_LOOKUP, + .flags = F_NEEDS_EFFICIENT_UNALIGNED_ACCESS, +}, /* invalid 1,2,4-byte reads from 8-byte fields in bpf_sk_lookup */ { "invalid 4-byte read from bpf_sk_lookup sk field", diff --git a/tools/testing/selftests/bpf/xdping.c b/tools/testing/selftests/bpf/xdping.c index 30f12637f4e4..baa870a759a2 100644 --- a/tools/testing/selftests/bpf/xdping.c +++ b/tools/testing/selftests/bpf/xdping.c @@ -22,6 +22,7 @@ #include "bpf/libbpf.h" #include "xdping.h" +#include "testing_helpers.h" static int ifindex; static __u32 xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; @@ -173,7 +174,7 @@ int main(int argc, char **argv) snprintf(filename, sizeof(filename), "%s_kern.o", argv[0]); - if (bpf_prog_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) { + if (bpf_prog_test_load(filename, BPF_PROG_TYPE_XDP, &obj, &prog_fd)) { fprintf(stderr, "load of %s failed\n", filename); return 1; } diff --git a/tools/testing/selftests/bpf/xdpxceiver.c b/tools/testing/selftests/bpf/xdpxceiver.c index 6c7cf8aadc79..fe7f423b8c3f 100644 --- a/tools/testing/selftests/bpf/xdpxceiver.c +++ b/tools/testing/selftests/bpf/xdpxceiver.c @@ -744,7 +744,6 @@ static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info * struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream); struct xsk_umem_info *umem = xsk->umem; u32 idx_rx = 0, idx_fq = 0, rcvd, i; - u32 total = 0; int ret; while (pkt) { @@ -799,7 +798,6 @@ static void receive_pkts(struct pkt_stream *pkt_stream, struct xsk_socket_info * pthread_mutex_lock(&pacing_mutex); pkts_in_flight -= rcvd; - total += rcvd; if (pkts_in_flight < umem->num_frames) pthread_cond_signal(&pacing_cond); pthread_mutex_unlock(&pacing_mutex); diff --git a/tools/testing/selftests/net/fcnal-test.sh b/tools/testing/selftests/net/fcnal-test.sh index a1da013d847b..98e4d5a2f587 100755 --- a/tools/testing/selftests/net/fcnal-test.sh +++ b/tools/testing/selftests/net/fcnal-test.sh @@ -66,6 +66,10 @@ NSB_LO_IP=172.16.2.2 NSA_LO_IP6=2001:db8:2::1 NSB_LO_IP6=2001:db8:2::2 +# non-local addresses for freebind tests +NL_IP=172.17.1.1 +NL_IP6=2001:db8:4::1 + MD5_PW=abc123 MD5_WRONG_PW=abc1234 @@ -316,6 +320,9 @@ addr2str() ${NSB_LO_IP6}) echo "ns-B loopback IPv6";; ${NSB_LINKIP6}|${NSB_LINKIP6}%*) echo "ns-B IPv6 LLA";; + ${NL_IP}) echo "nonlocal IP";; + ${NL_IP6}) echo "nonlocal IPv6";; + ${VRF_IP}) echo "VRF IP";; ${VRF_IP6}) echo "VRF IPv6";; @@ -1768,6 +1775,14 @@ ipv4_addr_bind_novrf() done # + # raw socket with nonlocal bind + # + a=${NL_IP} + log_start + run_cmd nettest -s -R -P icmp -f -l ${a} -I ${NSA_DEV} -b + log_test_addr ${a} $? 0 "Raw socket bind to nonlocal address after device bind" + + # # tcp sockets # a=${NSA_IP} @@ -1816,6 +1831,14 @@ ipv4_addr_bind_vrf() log_test_addr ${a} $? 1 "Raw socket bind to out of scope address after VRF bind" # + # raw socket with nonlocal bind + # + a=${NL_IP} + log_start + run_cmd nettest -s -R -P icmp -f -l ${a} -I ${VRF} -b + log_test_addr ${a} $? 0 "Raw socket bind to nonlocal address after VRF bind" + + # # tcp sockets # for a in ${NSA_IP} ${VRF_IP} @@ -1965,6 +1988,7 @@ ipv4_rt() a=${NSA_IP} log_start + run_cmd nettest ${varg} -s & sleep 1 run_cmd nettest ${varg} -d ${NSA_DEV} -r ${a} & @@ -3403,6 +3427,14 @@ ipv6_addr_bind_novrf() done # + # raw socket with nonlocal bind + # + a=${NL_IP6} + log_start + run_cmd nettest -6 -s -R -P icmp -f -l ${a} -I ${NSA_DEV} -b + log_test_addr ${a} $? 0 "Raw socket bind to nonlocal address" + + # # tcp sockets # a=${NSA_IP6} @@ -3444,6 +3476,14 @@ ipv6_addr_bind_vrf() log_test_addr ${a} $? 1 "Raw socket bind to invalid local address after vrf bind" # + # raw socket with nonlocal bind + # + a=${NL_IP6} + log_start + run_cmd nettest -6 -s -R -P icmp -f -l ${a} -I ${VRF} -b + log_test_addr ${a} $? 0 "Raw socket bind to nonlocal address after VRF bind" + + # # tcp sockets # # address on enslaved device is valid for the VRF or device in a VRF @@ -4068,8 +4108,6 @@ do # setup namespaces and config, but do not run any tests setup) setup; exit 0;; vrf_setup) setup "yes"; exit 0;; - - help) echo "Test names: $TESTS"; exit 0;; esac done diff --git a/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh new file mode 100755 index 000000000000..8748d1b1d95b --- /dev/null +++ b/tools/testing/selftests/net/forwarding/bridge_vlan_mcast.sh @@ -0,0 +1,543 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0 + +ALL_TESTS="vlmc_control_test vlmc_querier_test vlmc_igmp_mld_version_test \ + vlmc_last_member_test vlmc_startup_query_test vlmc_membership_test \ + vlmc_querier_intvl_test vlmc_query_intvl_test vlmc_query_response_intvl_test \ + vlmc_router_port_test vlmc_filtering_test" +NUM_NETIFS=4 +CHECK_TC="yes" +TEST_GROUP="239.10.10.10" + +source lib.sh + +h1_create() +{ + simple_if_init $h1 192.0.2.1/24 2001:db8:1::1/64 + ip link add l $h1 $h1.10 up type vlan id 10 +} + +h1_destroy() +{ + ip link del $h1.10 + simple_if_fini $h1 192.0.2.1/24 2001:db8:1::1/64 +} + +h2_create() +{ + simple_if_init $h2 192.0.2.2/24 2001:db8:1::2/64 + ip link add l $h2 $h2.10 up type vlan id 10 +} + +h2_destroy() +{ + ip link del $h2.10 + simple_if_fini $h2 192.0.2.2/24 2001:db8:1::2/64 +} + +switch_create() +{ + ip link add dev br0 type bridge mcast_snooping 1 mcast_querier 1 vlan_filtering 1 + + ip link set dev $swp1 master br0 + ip link set dev $swp2 master br0 + + ip link set dev br0 up + ip link set dev $swp1 up + ip link set dev $swp2 up + + tc qdisc add dev $swp1 clsact + tc qdisc add dev $swp2 clsact + + bridge vlan add vid 10-11 dev $swp1 master + bridge vlan add vid 10-11 dev $swp2 master + + ip link set dev br0 type bridge mcast_vlan_snooping 1 + check_err $? "Could not enable global vlan multicast snooping" + log_test "Vlan multicast snooping enable" +} + +switch_destroy() +{ + ip link set dev $swp2 down + ip link set dev $swp1 down + + ip link del dev br0 +} + +setup_prepare() +{ + h1=${NETIFS[p1]} + swp1=${NETIFS[p2]} + + swp2=${NETIFS[p3]} + h2=${NETIFS[p4]} + + vrf_prepare + + h1_create + h2_create + + switch_create +} + +cleanup() +{ + pre_cleanup + + switch_destroy + + h2_destroy + h1_destroy + + vrf_cleanup +} + +vlmc_v2join_test() +{ + local expect=$1 + + RET=0 + ip address add dev $h2.10 $TEST_GROUP/32 autojoin + check_err $? "Could not join $TEST_GROUP" + + sleep 5 + bridge -j mdb show dev br0 | + jq -e ".[].mdb[] | select(.grp == \"$TEST_GROUP\" and .vid == 10)" &>/dev/null + if [ $expect -eq 0 ]; then + check_err $? "IGMPv2 report didn't create mdb entry for $TEST_GROUP" + else + check_fail $? "IGMPv2 report shouldn't have created mdb entry for $TEST_GROUP" + fi + + # check if we need to cleanup + if [ $RET -eq 0 ]; then + ip address del dev $h2.10 $TEST_GROUP/32 2>&1 1>/dev/null + sleep 5 + bridge -j mdb show dev br0 | + jq -e ".[].mdb[] | select(.grp == \"$TEST_GROUP\" and \ + .vid == 10)" &>/dev/null + check_fail $? "IGMPv2 leave didn't remove mdb entry for $TEST_GROUP" + fi +} + +vlmc_control_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + log_test "Vlan global options existence" + + RET=0 + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and .mcast_snooping == 1) " &>/dev/null + check_err $? "Wrong default mcast_snooping global option value" + log_test "Vlan mcast_snooping global option default value" + + RET=0 + vlmc_v2join_test 0 + bridge vlan global set vid 10 dev br0 mcast_snooping 0 + check_err $? "Could not disable multicast snooping in vlan 10" + vlmc_v2join_test 1 + log_test "Vlan 10 multicast snooping control" +} + +# setup for general query counting +vlmc_query_cnt_xstats() +{ + local type=$1 + local version=$2 + local dev=$3 + + ip -j link xstats type bridge_slave dev $dev | \ + jq -e ".[].multicast.${type}_queries.tx_v${version}" +} + +vlmc_query_cnt_setup() +{ + local type=$1 + local dev=$2 + + if [[ $type == "igmp" ]]; then + tc filter add dev $dev egress pref 10 prot 802.1Q \ + flower vlan_id 10 vlan_ethtype ipv4 dst_ip 224.0.0.1 ip_proto 2 \ + action pass + else + tc filter add dev $dev egress pref 10 prot 802.1Q \ + flower vlan_id 10 vlan_ethtype ipv6 dst_ip ff02::1 ip_proto icmpv6 \ + action pass + fi + + ip link set dev br0 type bridge mcast_stats_enabled 1 +} + +vlmc_query_cnt_cleanup() +{ + local dev=$1 + + ip link set dev br0 type bridge mcast_stats_enabled 0 + tc filter del dev $dev egress pref 10 +} + +vlmc_check_query() +{ + local type=$1 + local version=$2 + local dev=$3 + local expect=$4 + local time=$5 + local ret=0 + + vlmc_query_cnt_setup $type $dev + + local pre_tx_xstats=$(vlmc_query_cnt_xstats $type $version $dev) + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier 1 + ret=$? + if [[ $ret -eq 0 ]]; then + sleep $time + + local tcstats=$(tc_rule_stats_get $dev 10 egress) + local post_tx_xstats=$(vlmc_query_cnt_xstats $type $version $dev) + + if [[ $tcstats != $expect || \ + $(($post_tx_xstats-$pre_tx_xstats)) != $expect || \ + $tcstats != $(($post_tx_xstats-$pre_tx_xstats)) ]]; then + ret=1 + fi + fi + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier 0 + vlmc_query_cnt_cleanup $dev + + return $ret +} + +vlmc_querier_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and .mcast_querier == 0) " &>/dev/null + check_err $? "Wrong default mcast_querier global vlan option value" + log_test "Vlan mcast_querier global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier 1 + check_err $? "Could not enable querier in vlan 10" + log_test "Vlan 10 multicast querier enable" + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier 0 + + RET=0 + vlmc_check_query igmp 2 $swp1 1 1 + check_err $? "No vlan tagged IGMPv2 general query packets sent" + log_test "Vlan 10 tagged IGMPv2 general query sent" + + RET=0 + vlmc_check_query mld 1 $swp1 1 1 + check_err $? "No vlan tagged MLD general query packets sent" + log_test "Vlan 10 tagged MLD general query sent" +} + +vlmc_igmp_mld_version_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and .mcast_igmp_version == 2) " &>/dev/null + check_err $? "Wrong default mcast_igmp_version global vlan option value" + log_test "Vlan mcast_igmp_version global option default value" + + RET=0 + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and .mcast_mld_version == 1) " &>/dev/null + check_err $? "Wrong default mcast_mld_version global vlan option value" + log_test "Vlan mcast_mld_version global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_igmp_version 3 + check_err $? "Could not set mcast_igmp_version in vlan 10" + log_test "Vlan 10 mcast_igmp_version option changed to 3" + + RET=0 + vlmc_check_query igmp 3 $swp1 1 1 + check_err $? "No vlan tagged IGMPv3 general query packets sent" + log_test "Vlan 10 tagged IGMPv3 general query sent" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_mld_version 2 + check_err $? "Could not set mcast_mld_version in vlan 10" + log_test "Vlan 10 mcast_mld_version option changed to 2" + + RET=0 + vlmc_check_query mld 2 $swp1 1 1 + check_err $? "No vlan tagged MLDv2 general query packets sent" + log_test "Vlan 10 tagged MLDv2 general query sent" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_igmp_version 2 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_mld_version 1 +} + +vlmc_last_member_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_last_member_count == 2) " &>/dev/null + check_err $? "Wrong default mcast_last_member_count global vlan option value" + log_test "Vlan mcast_last_member_count global option default value" + + RET=0 + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_last_member_interval == 100) " &>/dev/null + check_err $? "Wrong default mcast_last_member_interval global vlan option value" + log_test "Vlan mcast_last_member_interval global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_last_member_count 3 + check_err $? "Could not set mcast_last_member_count in vlan 10" + log_test "Vlan 10 mcast_last_member_count option changed to 3" + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_last_member_count 2 + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_last_member_interval 200 + check_err $? "Could not set mcast_last_member_interval in vlan 10" + log_test "Vlan 10 mcast_last_member_interval option changed to 200" + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_last_member_interval 100 +} + +vlmc_startup_query_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_startup_query_interval == 3125) " &>/dev/null + check_err $? "Wrong default mcast_startup_query_interval global vlan option value" + log_test "Vlan mcast_startup_query_interval global option default value" + + RET=0 + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_startup_query_count == 2) " &>/dev/null + check_err $? "Wrong default mcast_startup_query_count global vlan option value" + log_test "Vlan mcast_startup_query_count global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_interval 100 + check_err $? "Could not set mcast_startup_query_interval in vlan 10" + vlmc_check_query igmp 2 $swp1 2 3 + check_err $? "Wrong number of tagged IGMPv2 general queries sent" + log_test "Vlan 10 mcast_startup_query_interval option changed to 100" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_count 3 + check_err $? "Could not set mcast_startup_query_count in vlan 10" + vlmc_check_query igmp 2 $swp1 3 4 + check_err $? "Wrong number of tagged IGMPv2 general queries sent" + log_test "Vlan 10 mcast_startup_query_count option changed to 3" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_interval 3125 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_count 2 +} + +vlmc_membership_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_membership_interval == 26000) " &>/dev/null + check_err $? "Wrong default mcast_membership_interval global vlan option value" + log_test "Vlan mcast_membership_interval global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_membership_interval 200 + check_err $? "Could not set mcast_membership_interval in vlan 10" + log_test "Vlan 10 mcast_membership_interval option changed to 200" + + RET=0 + vlmc_v2join_test 1 + log_test "Vlan 10 mcast_membership_interval mdb entry expire" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_membership_interval 26000 +} + +vlmc_querier_intvl_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_querier_interval == 25500) " &>/dev/null + check_err $? "Wrong default mcast_querier_interval global vlan option value" + log_test "Vlan mcast_querier_interval global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier_interval 100 + check_err $? "Could not set mcast_querier_interval in vlan 10" + log_test "Vlan 10 mcast_querier_interval option changed to 100" + + RET=0 + ip link add dev br1 type bridge mcast_snooping 1 mcast_querier 1 vlan_filtering 1 \ + mcast_vlan_snooping 1 + bridge vlan add vid 10 dev br1 self pvid untagged + ip link set dev $h1 master br1 + ip link set dev br1 up + bridge vlan add vid 10 dev $h1 master + bridge vlan global set vid 10 dev br1 mcast_snooping 1 mcast_querier 1 + sleep 2 + ip link del dev br1 + ip addr replace 2001:db8:1::1/64 dev $h1 + vlmc_check_query igmp 2 $swp1 1 1 + check_err $? "Wrong number of IGMPv2 general queries after querier interval" + log_test "Vlan 10 mcast_querier_interval expire after outside query" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier_interval 25500 +} + +vlmc_query_intvl_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_query_interval == 12500) " &>/dev/null + check_err $? "Wrong default mcast_query_interval global vlan option value" + log_test "Vlan mcast_query_interval global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_count 0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_query_interval 200 + check_err $? "Could not set mcast_query_interval in vlan 10" + # 1 is sent immediately, then 2 more in the next 5 seconds + vlmc_check_query igmp 2 $swp1 3 5 + check_err $? "Wrong number of tagged IGMPv2 general queries sent" + log_test "Vlan 10 mcast_query_interval option changed to 200" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_startup_query_count 2 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_query_interval 12500 +} + +vlmc_query_response_intvl_test() +{ + RET=0 + local goutput=`bridge -j vlan global show` + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10)" &>/dev/null + check_err $? "Could not find vlan 10's global options" + + echo -n $goutput | + jq -e ".[].vlans[] | select(.vlan == 10 and \ + .mcast_query_response_interval == 1000) " &>/dev/null + check_err $? "Wrong default mcast_query_response_interval global vlan option value" + log_test "Vlan mcast_query_response_interval global option default value" + + RET=0 + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_query_response_interval 200 + check_err $? "Could not set mcast_query_response_interval in vlan 10" + log_test "Vlan 10 mcast_query_response_interval option changed to 200" + + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_query_response_interval 1000 +} + +vlmc_router_port_test() +{ + RET=0 + local goutput=`bridge -j -d vlan show` + echo -n $goutput | + jq -e ".[] | select(.ifname == \"$swp1\" and \ + .vlans[].vlan == 10)" &>/dev/null + check_err $? "Could not find port vlan 10's options" + + echo -n $goutput | + jq -e ".[] | select(.ifname == \"$swp1\" and \ + .vlans[].vlan == 10 and \ + .vlans[].mcast_router == 1)" &>/dev/null + check_err $? "Wrong default port mcast_router option value" + log_test "Port vlan 10 option mcast_router default value" + + RET=0 + bridge vlan set vid 10 dev $swp1 mcast_router 2 + check_err $? "Could not set port vlan 10's mcast_router option" + log_test "Port vlan 10 mcast_router option changed to 2" + + RET=0 + tc filter add dev $swp1 egress pref 10 prot 802.1Q \ + flower vlan_id 10 vlan_ethtype ipv4 dst_ip 239.1.1.1 ip_proto udp action pass + tc filter add dev $swp2 egress pref 10 prot 802.1Q \ + flower vlan_id 10 vlan_ethtype ipv4 dst_ip 239.1.1.1 ip_proto udp action pass + bridge vlan set vid 10 dev $swp2 mcast_router 0 + # we need to enable querier and disable query response interval to + # make sure packets are flooded only to router ports + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_querier 1 \ + mcast_query_response_interval 0 + bridge vlan add vid 10 dev br0 self + sleep 1 + mausezahn br0 -Q 10 -c 10 -p 128 -b 01:00:5e:01:01:01 -B 239.1.1.1 \ + -t udp "dp=1024" &>/dev/null + local swp1_tcstats=$(tc_rule_stats_get $swp1 10 egress) + if [[ $swp1_tcstats != 10 ]]; then + check_err 1 "Wrong number of vlan 10 multicast packets flooded" + fi + local swp2_tcstats=$(tc_rule_stats_get $swp2 10 egress) + check_err $swp2_tcstats "Vlan 10 multicast packets flooded to non-router port" + log_test "Flood unknown vlan multicast packets to router port only" + + tc filter del dev $swp2 egress pref 10 + tc filter del dev $swp1 egress pref 10 + bridge vlan del vid 10 dev br0 self + bridge vlan global set vid 10 dev br0 mcast_snooping 1 mcast_query_response_interval 1000 + bridge vlan set vid 10 dev $swp2 mcast_router 1 + bridge vlan set vid 10 dev $swp1 mcast_router 1 +} + +vlmc_filtering_test() +{ + RET=0 + ip link set dev br0 type bridge vlan_filtering 0 + ip -j -d link show dev br0 | \ + jq -e "select(.[0].linkinfo.info_data.mcast_vlan_snooping == 1)" &>/dev/null + check_fail $? "Vlan filtering is disabled but multicast vlan snooping is still enabled" + log_test "Disable multicast vlan snooping when vlan filtering is disabled" +} + +trap cleanup EXIT + +setup_prepare +setup_wait + +tests_run + +exit $EXIT_STATUS diff --git a/tools/testing/selftests/net/mptcp/.gitignore b/tools/testing/selftests/net/mptcp/.gitignore index 7569d892967a..49daae73c41e 100644 --- a/tools/testing/selftests/net/mptcp/.gitignore +++ b/tools/testing/selftests/net/mptcp/.gitignore @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only mptcp_connect +mptcp_inq mptcp_sockopt pm_nl_ctl *.pcap diff --git a/tools/testing/selftests/net/mptcp/Makefile b/tools/testing/selftests/net/mptcp/Makefile index bbf4e448bad9..0356c4501c99 100644 --- a/tools/testing/selftests/net/mptcp/Makefile +++ b/tools/testing/selftests/net/mptcp/Makefile @@ -8,7 +8,7 @@ CFLAGS = -Wall -Wl,--no-as-needed -O2 -g -I$(top_srcdir)/usr/include TEST_PROGS := mptcp_connect.sh pm_netlink.sh mptcp_join.sh diag.sh \ simult_flows.sh mptcp_sockopt.sh -TEST_GEN_FILES = mptcp_connect pm_nl_ctl mptcp_sockopt +TEST_GEN_FILES = mptcp_connect pm_nl_ctl mptcp_sockopt mptcp_inq TEST_FILES := settings diff --git a/tools/testing/selftests/net/mptcp/config b/tools/testing/selftests/net/mptcp/config index 0faaccd21447..419e71560fd1 100644 --- a/tools/testing/selftests/net/mptcp/config +++ b/tools/testing/selftests/net/mptcp/config @@ -13,5 +13,9 @@ CONFIG_NFT_COUNTER=m CONFIG_NFT_COMPAT=m CONFIG_NETFILTER_XTABLES=m CONFIG_NETFILTER_XT_MATCH_BPF=m -CONFIG_NF_TABLES_IPV4=y -CONFIG_NF_TABLES_IPV6=y +CONFIG_NF_TABLES_INET=y +CONFIG_NFT_TPROXY=m +CONFIG_NFT_SOCKET=m +CONFIG_IP_ADVANCED_ROUTER=y +CONFIG_IP_MULTIPLE_TABLES=y +CONFIG_IPV6_MULTIPLE_TABLES=y diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.c b/tools/testing/selftests/net/mptcp/mptcp_connect.c index 95e81d557b08..98de28ac3ba8 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.c +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.c @@ -73,9 +73,22 @@ static uint32_t cfg_mark; struct cfg_cmsg_types { unsigned int cmsg_enabled:1; unsigned int timestampns:1; + unsigned int tcp_inq:1; }; +struct cfg_sockopt_types { + unsigned int transparent:1; +}; + +struct tcp_inq_state { + unsigned int last; + bool expect_eof; +}; + +static struct tcp_inq_state tcp_inq; + static struct cfg_cmsg_types cfg_cmsg_types; +static struct cfg_sockopt_types cfg_sockopt_types; static void die_usage(void) { @@ -93,6 +106,7 @@ static void die_usage(void) fprintf(stderr, "\t-u -- check mptcp ulp\n"); fprintf(stderr, "\t-w num -- wait num sec before closing the socket\n"); fprintf(stderr, "\t-c cmsg -- test cmsg type <cmsg>\n"); + fprintf(stderr, "\t-o option -- test sockopt <option>\n"); fprintf(stderr, "\t-P [saveWithPeek|saveAfterPeek] -- save data with/after MSG_PEEK form tcp socket\n"); exit(1); @@ -185,6 +199,22 @@ static void set_mark(int fd, uint32_t mark) } } +static void set_transparent(int fd, int pf) +{ + int one = 1; + + switch (pf) { + case AF_INET: + if (-1 == setsockopt(fd, SOL_IP, IP_TRANSPARENT, &one, sizeof(one))) + perror("IP_TRANSPARENT"); + break; + case AF_INET6: + if (-1 == setsockopt(fd, IPPROTO_IPV6, IPV6_TRANSPARENT, &one, sizeof(one))) + perror("IPV6_TRANSPARENT"); + break; + } +} + static int sock_listen_mptcp(const char * const listenaddr, const char * const port) { @@ -212,6 +242,9 @@ static int sock_listen_mptcp(const char * const listenaddr, sizeof(one))) perror("setsockopt"); + if (cfg_sockopt_types.transparent) + set_transparent(sock, pf); + if (bind(sock, a->ai_addr, a->ai_addrlen) == 0) break; /* success */ @@ -364,7 +397,9 @@ static size_t do_write(const int fd, char *buf, const size_t len) static void process_cmsg(struct msghdr *msgh) { struct __kernel_timespec ts; + bool inq_found = false; bool ts_found = false; + unsigned int inq = 0; struct cmsghdr *cmsg; for (cmsg = CMSG_FIRSTHDR(msgh); cmsg ; cmsg = CMSG_NXTHDR(msgh, cmsg)) { @@ -373,12 +408,27 @@ static void process_cmsg(struct msghdr *msgh) ts_found = true; continue; } + if (cmsg->cmsg_level == IPPROTO_TCP && cmsg->cmsg_type == TCP_CM_INQ) { + memcpy(&inq, CMSG_DATA(cmsg), sizeof(inq)); + inq_found = true; + continue; + } + } if (cfg_cmsg_types.timestampns) { if (!ts_found) xerror("TIMESTAMPNS not present\n"); } + + if (cfg_cmsg_types.tcp_inq) { + if (!inq_found) + xerror("TCP_INQ not present\n"); + + if (inq > 1024) + xerror("tcp_inq %u is larger than one kbyte\n", inq); + tcp_inq.last = inq; + } } static ssize_t do_recvmsg_cmsg(const int fd, char *buf, const size_t len) @@ -395,10 +445,23 @@ static ssize_t do_recvmsg_cmsg(const int fd, char *buf, const size_t len) .msg_controllen = sizeof(msg_buf), }; int flags = 0; + unsigned int last_hint = tcp_inq.last; int ret = recvmsg(fd, &msg, flags); - if (ret <= 0) + if (ret <= 0) { + if (ret == 0 && tcp_inq.expect_eof) + return ret; + + if (ret == 0 && cfg_cmsg_types.tcp_inq) + if (last_hint != 1 && last_hint != 0) + xerror("EOF but last tcp_inq hint was %u\n", last_hint); + return ret; + } + + if (tcp_inq.expect_eof) + xerror("expected EOF, last_hint %u, now %u\n", + last_hint, tcp_inq.last); if (msg.msg_controllen && !cfg_cmsg_types.cmsg_enabled) xerror("got %lu bytes of cmsg data, expected 0\n", @@ -410,6 +473,19 @@ static ssize_t do_recvmsg_cmsg(const int fd, char *buf, const size_t len) if (msg.msg_controllen) process_cmsg(&msg); + if (cfg_cmsg_types.tcp_inq) { + if ((size_t)ret < len && last_hint > (unsigned int)ret) { + if (ret + 1 != (int)last_hint) { + int next = read(fd, msg_buf, sizeof(msg_buf)); + + xerror("read %u of %u, last_hint was %u tcp_inq hint now %u next_read returned %d/%m\n", + ret, (unsigned int)len, last_hint, tcp_inq.last, next); + } else { + tcp_inq.expect_eof = true; + } + } + } + return ret; } @@ -919,6 +995,8 @@ static void apply_cmsg_types(int fd, const struct cfg_cmsg_types *cmsg) if (cmsg->timestampns) xsetsockopt(fd, SOL_SOCKET, SO_TIMESTAMPNS_NEW, &on, sizeof(on)); + if (cmsg->tcp_inq) + xsetsockopt(fd, IPPROTO_TCP, TCP_INQ, &on, sizeof(on)); } static void parse_cmsg_types(const char *type) @@ -940,10 +1018,36 @@ static void parse_cmsg_types(const char *type) return; } + if (strncmp(type, "TCPINQ", len) == 0) { + cfg_cmsg_types.tcp_inq = 1; + return; + } + fprintf(stderr, "Unrecognized cmsg option %s\n", type); exit(1); } +static void parse_setsock_options(const char *name) +{ + char *next = strchr(name, ','); + unsigned int len = 0; + + if (next) { + parse_setsock_options(next + 1); + len = next - name; + } else { + len = strlen(name); + } + + if (strncmp(name, "TRANSPARENT", len) == 0) { + cfg_sockopt_types.transparent = 1; + return; + } + + fprintf(stderr, "Unrecognized setsockopt option %s\n", name); + exit(1); +} + int main_loop(void) { int fd; @@ -1047,7 +1151,7 @@ static void parse_opts(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "6jr:lp:s:hut:T:m:S:R:w:M:P:c:")) != -1) { + while ((c = getopt(argc, argv, "6jr:lp:s:hut:T:m:S:R:w:M:P:c:o:")) != -1) { switch (c) { case 'j': cfg_join = true; @@ -1108,6 +1212,9 @@ static void parse_opts(int argc, char **argv) case 'c': parse_cmsg_types(optarg); break; + case 'o': + parse_setsock_options(optarg); + break; } } diff --git a/tools/testing/selftests/net/mptcp/mptcp_connect.sh b/tools/testing/selftests/net/mptcp/mptcp_connect.sh index 559173a8e387..a4226b608c68 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_connect.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_connect.sh @@ -671,6 +671,82 @@ run_tests() run_tests_lo $1 $2 $3 0 } +run_test_transparent() +{ + local connect_addr="$1" + local msg="$2" + + local connector_ns="$ns1" + local listener_ns="$ns2" + local lret=0 + local r6flag="" + + # skip if we don't want v6 + if ! $ipv6 && is_v6 "${connect_addr}"; then + return 0 + fi + +ip netns exec "$listener_ns" nft -f /dev/stdin <<"EOF" +flush ruleset +table inet mangle { + chain divert { + type filter hook prerouting priority -150; + + meta l4proto tcp socket transparent 1 meta mark set 1 accept + tcp dport 20000 tproxy to :20000 meta mark set 1 accept + } +} +EOF + if [ $? -ne 0 ]; then + echo "SKIP: $msg, could not load nft ruleset" + return + fi + + local local_addr + if is_v6 "${connect_addr}"; then + local_addr="::" + r6flag="-6" + else + local_addr="0.0.0.0" + fi + + ip -net "$listener_ns" $r6flag rule add fwmark 1 lookup 100 + if [ $? -ne 0 ]; then + ip netns exec "$listener_ns" nft flush ruleset + echo "SKIP: $msg, ip $r6flag rule failed" + return + fi + + ip -net "$listener_ns" route add local $local_addr/0 dev lo table 100 + if [ $? -ne 0 ]; then + ip netns exec "$listener_ns" nft flush ruleset + ip -net "$listener_ns" $r6flag rule del fwmark 1 lookup 100 + echo "SKIP: $msg, ip route add local $local_addr failed" + return + fi + + echo "INFO: test $msg" + + TEST_COUNT=10000 + local extra_args="-o TRANSPARENT" + do_transfer ${listener_ns} ${connector_ns} MPTCP MPTCP \ + ${connect_addr} ${local_addr} "${extra_args}" + lret=$? + + ip netns exec "$listener_ns" nft flush ruleset + ip -net "$listener_ns" $r6flag rule del fwmark 1 lookup 100 + ip -net "$listener_ns" route del local $local_addr/0 dev lo table 100 + + if [ $lret -ne 0 ]; then + echo "FAIL: $msg, mptcp connection error" 1>&2 + ret=$lret + return 1 + fi + + echo "PASS: $msg" + return 0 +} + run_tests_peekmode() { local peekmode="$1" @@ -794,5 +870,9 @@ run_tests_peekmode "saveWithPeek" run_tests_peekmode "saveAfterPeek" stop_if_error "Tests with peek mode have failed" +# connect to ns4 ip address, ns2 should intercept/proxy +run_test_transparent 10.0.3.1 "tproxy ipv4" +run_test_transparent dead:beef:3::1 "tproxy ipv6" + display_time exit $ret diff --git a/tools/testing/selftests/net/mptcp/mptcp_inq.c b/tools/testing/selftests/net/mptcp/mptcp_inq.c new file mode 100644 index 000000000000..b8debd4fb5ed --- /dev/null +++ b/tools/testing/selftests/net/mptcp/mptcp_inq.c @@ -0,0 +1,603 @@ +// SPDX-License-Identifier: GPL-2.0 + +#define _GNU_SOURCE + +#include <assert.h> +#include <errno.h> +#include <fcntl.h> +#include <limits.h> +#include <string.h> +#include <stdarg.h> +#include <stdbool.h> +#include <stdint.h> +#include <inttypes.h> +#include <stdio.h> +#include <stdlib.h> +#include <strings.h> +#include <unistd.h> +#include <time.h> + +#include <sys/ioctl.h> +#include <sys/ioctl.h> +#include <sys/socket.h> +#include <sys/types.h> +#include <sys/wait.h> + +#include <netdb.h> +#include <netinet/in.h> + +#include <linux/tcp.h> +#include <linux/sockios.h> + +#ifndef IPPROTO_MPTCP +#define IPPROTO_MPTCP 262 +#endif +#ifndef SOL_MPTCP +#define SOL_MPTCP 284 +#endif + +static int pf = AF_INET; +static int proto_tx = IPPROTO_MPTCP; +static int proto_rx = IPPROTO_MPTCP; + +static void die_perror(const char *msg) +{ + perror(msg); + exit(1); +} + +static void die_usage(int r) +{ + fprintf(stderr, "Usage: mptcp_inq [-6] [ -t tcp|mptcp ] [ -r tcp|mptcp]\n"); + exit(r); +} + +static void xerror(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fputc('\n', stderr); + exit(1); +} + +static const char *getxinfo_strerr(int err) +{ + if (err == EAI_SYSTEM) + return strerror(errno); + + return gai_strerror(err); +} + +static void xgetaddrinfo(const char *node, const char *service, + const struct addrinfo *hints, + struct addrinfo **res) +{ + int err = getaddrinfo(node, service, hints, res); + + if (err) { + const char *errstr = getxinfo_strerr(err); + + fprintf(stderr, "Fatal: getaddrinfo(%s:%s): %s\n", + node ? node : "", service ? service : "", errstr); + exit(1); + } +} + +static int sock_listen_mptcp(const char * const listenaddr, + const char * const port) +{ + int sock; + struct addrinfo hints = { + .ai_protocol = IPPROTO_TCP, + .ai_socktype = SOCK_STREAM, + .ai_flags = AI_PASSIVE | AI_NUMERICHOST + }; + + hints.ai_family = pf; + + struct addrinfo *a, *addr; + int one = 1; + + xgetaddrinfo(listenaddr, port, &hints, &addr); + hints.ai_family = pf; + + for (a = addr; a; a = a->ai_next) { + sock = socket(a->ai_family, a->ai_socktype, proto_rx); + if (sock < 0) + continue; + + if (-1 == setsockopt(sock, SOL_SOCKET, SO_REUSEADDR, &one, + sizeof(one))) + perror("setsockopt"); + + if (bind(sock, a->ai_addr, a->ai_addrlen) == 0) + break; /* success */ + + perror("bind"); + close(sock); + sock = -1; + } + + freeaddrinfo(addr); + + if (sock < 0) + xerror("could not create listen socket"); + + if (listen(sock, 20)) + die_perror("listen"); + + return sock; +} + +static int sock_connect_mptcp(const char * const remoteaddr, + const char * const port, int proto) +{ + struct addrinfo hints = { + .ai_protocol = IPPROTO_TCP, + .ai_socktype = SOCK_STREAM, + }; + struct addrinfo *a, *addr; + int sock = -1; + + hints.ai_family = pf; + + xgetaddrinfo(remoteaddr, port, &hints, &addr); + for (a = addr; a; a = a->ai_next) { + sock = socket(a->ai_family, a->ai_socktype, proto); + if (sock < 0) + continue; + + if (connect(sock, a->ai_addr, a->ai_addrlen) == 0) + break; /* success */ + + die_perror("connect"); + } + + if (sock < 0) + xerror("could not create connect socket"); + + freeaddrinfo(addr); + return sock; +} + +static int protostr_to_num(const char *s) +{ + if (strcasecmp(s, "tcp") == 0) + return IPPROTO_TCP; + if (strcasecmp(s, "mptcp") == 0) + return IPPROTO_MPTCP; + + die_usage(1); + return 0; +} + +static void parse_opts(int argc, char **argv) +{ + int c; + + while ((c = getopt(argc, argv, "h6t:r:")) != -1) { + switch (c) { + case 'h': + die_usage(0); + break; + case '6': + pf = AF_INET6; + break; + case 't': + proto_tx = protostr_to_num(optarg); + break; + case 'r': + proto_rx = protostr_to_num(optarg); + break; + default: + die_usage(1); + break; + } + } +} + +/* wait up to timeout milliseconds */ +static void wait_for_ack(int fd, int timeout, size_t total) +{ + int i; + + for (i = 0; i < timeout; i++) { + int nsd, ret, queued = -1; + struct timespec req; + + ret = ioctl(fd, TIOCOUTQ, &queued); + if (ret < 0) + die_perror("TIOCOUTQ"); + + ret = ioctl(fd, SIOCOUTQNSD, &nsd); + if (ret < 0) + die_perror("SIOCOUTQNSD"); + + if ((size_t)queued > total) + xerror("TIOCOUTQ %u, but only %zu expected\n", queued, total); + assert(nsd <= queued); + + if (queued == 0) + return; + + /* wait for peer to ack rx of all data */ + req.tv_sec = 0; + req.tv_nsec = 1 * 1000 * 1000ul; /* 1ms */ + nanosleep(&req, NULL); + } + + xerror("still tx data queued after %u ms\n", timeout); +} + +static void connect_one_server(int fd, int unixfd) +{ + size_t len, i, total, sent; + char buf[4096], buf2[4096]; + ssize_t ret; + + len = rand() % (sizeof(buf) - 1); + + if (len < 128) + len = 128; + + for (i = 0; i < len ; i++) { + buf[i] = rand() % 26; + buf[i] += 'A'; + } + + buf[i] = '\n'; + + /* un-block server */ + ret = read(unixfd, buf2, 4); + assert(ret == 4); + + assert(strncmp(buf2, "xmit", 4) == 0); + + ret = write(unixfd, &len, sizeof(len)); + assert(ret == (ssize_t)sizeof(len)); + + ret = write(fd, buf, len); + if (ret < 0) + die_perror("write"); + + if (ret != (ssize_t)len) + xerror("short write"); + + ret = read(unixfd, buf2, 4); + assert(strncmp(buf2, "huge", 4) == 0); + + total = rand() % (16 * 1024 * 1024); + total += (1 * 1024 * 1024); + sent = total; + + ret = write(unixfd, &total, sizeof(total)); + assert(ret == (ssize_t)sizeof(total)); + + wait_for_ack(fd, 5000, len); + + while (total > 0) { + if (total > sizeof(buf)) + len = sizeof(buf); + else + len = total; + + ret = write(fd, buf, len); + if (ret < 0) + die_perror("write"); + total -= ret; + + /* we don't have to care about buf content, only + * number of total bytes sent + */ + } + + ret = read(unixfd, buf2, 4); + assert(ret == 4); + assert(strncmp(buf2, "shut", 4) == 0); + + wait_for_ack(fd, 5000, sent); + + ret = write(fd, buf, 1); + assert(ret == 1); + close(fd); + ret = write(unixfd, "closed", 6); + assert(ret == 6); + + close(unixfd); +} + +static void get_tcp_inq(struct msghdr *msgh, unsigned int *inqv) +{ + struct cmsghdr *cmsg; + + for (cmsg = CMSG_FIRSTHDR(msgh); cmsg ; cmsg = CMSG_NXTHDR(msgh, cmsg)) { + if (cmsg->cmsg_level == IPPROTO_TCP && cmsg->cmsg_type == TCP_CM_INQ) { + memcpy(inqv, CMSG_DATA(cmsg), sizeof(*inqv)); + return; + } + } + + xerror("could not find TCP_CM_INQ cmsg type"); +} + +static void process_one_client(int fd, int unixfd) +{ + unsigned int tcp_inq; + size_t expect_len; + char msg_buf[4096]; + char buf[4096]; + char tmp[16]; + struct iovec iov = { + .iov_base = buf, + .iov_len = 1, + }; + struct msghdr msg = { + .msg_iov = &iov, + .msg_iovlen = 1, + .msg_control = msg_buf, + .msg_controllen = sizeof(msg_buf), + }; + ssize_t ret, tot; + + ret = write(unixfd, "xmit", 4); + assert(ret == 4); + + ret = read(unixfd, &expect_len, sizeof(expect_len)); + assert(ret == (ssize_t)sizeof(expect_len)); + + if (expect_len > sizeof(buf)) + xerror("expect len %zu exceeds buffer size", expect_len); + + for (;;) { + struct timespec req; + unsigned int queued; + + ret = ioctl(fd, FIONREAD, &queued); + if (ret < 0) + die_perror("FIONREAD"); + if (queued > expect_len) + xerror("FIONREAD returned %u, but only %zu expected\n", + queued, expect_len); + if (queued == expect_len) + break; + + req.tv_sec = 0; + req.tv_nsec = 1000 * 1000ul; + nanosleep(&req, NULL); + } + + /* read one byte, expect cmsg to return expected - 1 */ + ret = recvmsg(fd, &msg, 0); + if (ret < 0) + die_perror("recvmsg"); + + if (msg.msg_controllen == 0) + xerror("msg_controllen is 0"); + + get_tcp_inq(&msg, &tcp_inq); + + assert((size_t)tcp_inq == (expect_len - 1)); + + iov.iov_len = sizeof(buf); + ret = recvmsg(fd, &msg, 0); + if (ret < 0) + die_perror("recvmsg"); + + /* should have gotten exact remainder of all pending data */ + assert(ret == (ssize_t)tcp_inq); + + /* should be 0, all drained */ + get_tcp_inq(&msg, &tcp_inq); + assert(tcp_inq == 0); + + /* request a large swath of data. */ + ret = write(unixfd, "huge", 4); + assert(ret == 4); + + ret = read(unixfd, &expect_len, sizeof(expect_len)); + assert(ret == (ssize_t)sizeof(expect_len)); + + /* peer should send us a few mb of data */ + if (expect_len <= sizeof(buf)) + xerror("expect len %zu too small\n", expect_len); + + tot = 0; + do { + iov.iov_len = sizeof(buf); + ret = recvmsg(fd, &msg, 0); + if (ret < 0) + die_perror("recvmsg"); + + tot += ret; + + get_tcp_inq(&msg, &tcp_inq); + + if (tcp_inq > expect_len - tot) + xerror("inq %d, remaining %d total_len %d\n", + tcp_inq, expect_len - tot, (int)expect_len); + + assert(tcp_inq <= expect_len - tot); + } while ((size_t)tot < expect_len); + + ret = write(unixfd, "shut", 4); + assert(ret == 4); + + /* wait for hangup. Should have received one more byte of data. */ + ret = read(unixfd, tmp, sizeof(tmp)); + assert(ret == 6); + assert(strncmp(tmp, "closed", 6) == 0); + + sleep(1); + + iov.iov_len = 1; + ret = recvmsg(fd, &msg, 0); + if (ret < 0) + die_perror("recvmsg"); + assert(ret == 1); + + get_tcp_inq(&msg, &tcp_inq); + + /* tcp_inq should be 1 due to received fin. */ + assert(tcp_inq == 1); + + iov.iov_len = 1; + ret = recvmsg(fd, &msg, 0); + if (ret < 0) + die_perror("recvmsg"); + + /* expect EOF */ + assert(ret == 0); + get_tcp_inq(&msg, &tcp_inq); + assert(tcp_inq == 1); + + close(fd); +} + +static int xaccept(int s) +{ + int fd = accept(s, NULL, 0); + + if (fd < 0) + die_perror("accept"); + + return fd; +} + +static int server(int unixfd) +{ + int fd = -1, r, on = 1; + + switch (pf) { + case AF_INET: + fd = sock_listen_mptcp("127.0.0.1", "15432"); + break; + case AF_INET6: + fd = sock_listen_mptcp("::1", "15432"); + break; + default: + xerror("Unknown pf %d\n", pf); + break; + } + + r = write(unixfd, "conn", 4); + assert(r == 4); + + alarm(15); + r = xaccept(fd); + + if (-1 == setsockopt(r, IPPROTO_TCP, TCP_INQ, &on, sizeof(on))) + die_perror("setsockopt"); + + process_one_client(r, unixfd); + + return 0; +} + +static int client(int unixfd) +{ + int fd = -1; + + alarm(15); + + switch (pf) { + case AF_INET: + fd = sock_connect_mptcp("127.0.0.1", "15432", proto_tx); + break; + case AF_INET6: + fd = sock_connect_mptcp("::1", "15432", proto_tx); + break; + default: + xerror("Unknown pf %d\n", pf); + } + + connect_one_server(fd, unixfd); + + return 0; +} + +static void init_rng(void) +{ + int fd = open("/dev/urandom", O_RDONLY); + unsigned int foo; + + if (fd > 0) { + int ret = read(fd, &foo, sizeof(foo)); + + if (ret < 0) + srand(fd + foo); + close(fd); + } + + srand(foo); +} + +static pid_t xfork(void) +{ + pid_t p = fork(); + + if (p < 0) + die_perror("fork"); + else if (p == 0) + init_rng(); + + return p; +} + +static int rcheck(int wstatus, const char *what) +{ + if (WIFEXITED(wstatus)) { + if (WEXITSTATUS(wstatus) == 0) + return 0; + fprintf(stderr, "%s exited, status=%d\n", what, WEXITSTATUS(wstatus)); + return WEXITSTATUS(wstatus); + } else if (WIFSIGNALED(wstatus)) { + xerror("%s killed by signal %d\n", what, WTERMSIG(wstatus)); + } else if (WIFSTOPPED(wstatus)) { + xerror("%s stopped by signal %d\n", what, WSTOPSIG(wstatus)); + } + + return 111; +} + +int main(int argc, char *argv[]) +{ + int e1, e2, wstatus; + pid_t s, c, ret; + int unixfds[2]; + + parse_opts(argc, argv); + + e1 = socketpair(AF_UNIX, SOCK_DGRAM, 0, unixfds); + if (e1 < 0) + die_perror("pipe"); + + s = xfork(); + if (s == 0) + return server(unixfds[1]); + + close(unixfds[1]); + + /* wait until server bound a socket */ + e1 = read(unixfds[0], &e1, 4); + assert(e1 == 4); + + c = xfork(); + if (c == 0) + return client(unixfds[0]); + + close(unixfds[0]); + + ret = waitpid(s, &wstatus, 0); + if (ret == -1) + die_perror("waitpid"); + e1 = rcheck(wstatus, "server"); + ret = waitpid(c, &wstatus, 0); + if (ret == -1) + die_perror("waitpid"); + e2 = rcheck(wstatus, "client"); + + return e1 ? e1 : e2; +} diff --git a/tools/testing/selftests/net/mptcp/mptcp_sockopt.c b/tools/testing/selftests/net/mptcp/mptcp_sockopt.c index 417b11cafafe..ac9a4d9c1764 100644 --- a/tools/testing/selftests/net/mptcp/mptcp_sockopt.c +++ b/tools/testing/selftests/net/mptcp/mptcp_sockopt.c @@ -4,6 +4,7 @@ #include <assert.h> #include <errno.h> +#include <fcntl.h> #include <limits.h> #include <string.h> #include <stdarg.h> @@ -13,6 +14,7 @@ #include <stdio.h> #include <stdlib.h> #include <strings.h> +#include <time.h> #include <unistd.h> #include <sys/socket.h> @@ -594,6 +596,44 @@ static int server(int pipefd) return 0; } +static void test_ip_tos_sockopt(int fd) +{ + uint8_t tos_in, tos_out; + socklen_t s; + int r; + + tos_in = rand() & 0xfc; + r = setsockopt(fd, SOL_IP, IP_TOS, &tos_in, sizeof(tos_out)); + if (r != 0) + die_perror("setsockopt IP_TOS"); + + tos_out = 0; + s = sizeof(tos_out); + r = getsockopt(fd, SOL_IP, IP_TOS, &tos_out, &s); + if (r != 0) + die_perror("getsockopt IP_TOS"); + + if (tos_in != tos_out) + xerror("tos %x != %x socklen_t %d\n", tos_in, tos_out, s); + + if (s != 1) + xerror("tos should be 1 byte"); + + s = 0; + r = getsockopt(fd, SOL_IP, IP_TOS, &tos_out, &s); + if (r != 0) + die_perror("getsockopt IP_TOS 0"); + if (s != 0) + xerror("expect socklen_t == 0"); + + s = -1; + r = getsockopt(fd, SOL_IP, IP_TOS, &tos_out, &s); + if (r != -1 && errno != EINVAL) + die_perror("getsockopt IP_TOS did not indicate -EINVAL"); + if (s != -1) + xerror("expect socklen_t == -1"); +} + static int client(int pipefd) { int fd = -1; @@ -611,6 +651,8 @@ static int client(int pipefd) xerror("Unknown pf %d\n", pf); } + test_ip_tos_sockopt(fd); + connect_one_server(fd, pipefd); return 0; @@ -642,6 +684,25 @@ static int rcheck(int wstatus, const char *what) return 111; } +static void init_rng(void) +{ + int fd = open("/dev/urandom", O_RDONLY); + + if (fd >= 0) { + unsigned int foo; + ssize_t ret; + + /* can't fail */ + ret = read(fd, &foo, sizeof(foo)); + assert(ret == sizeof(foo)); + + close(fd); + srand(foo); + } else { + srand(time(NULL)); + } +} + int main(int argc, char *argv[]) { int e1, e2, wstatus; @@ -650,6 +711,8 @@ int main(int argc, char *argv[]) parse_opts(argc, argv); + init_rng(); + e1 = pipe(pipefds); if (e1 < 0) die_perror("pipe"); diff --git a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh index 41de643788b8..0879da915014 100755 --- a/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh +++ b/tools/testing/selftests/net/mptcp/mptcp_sockopt.sh @@ -178,7 +178,7 @@ do_transfer() timeout ${timeout_test} \ ip netns exec ${listener_ns} \ - $mptcp_connect -t ${timeout_poll} -l -M 1 -p $port -s ${srv_proto} -c TIMESTAMPNS \ + $mptcp_connect -t ${timeout_poll} -l -M 1 -p $port -s ${srv_proto} -c TIMESTAMPNS,TCPINQ \ ${local_addr} < "$sin" > "$sout" & spid=$! @@ -186,7 +186,7 @@ do_transfer() timeout ${timeout_test} \ ip netns exec ${connector_ns} \ - $mptcp_connect -t ${timeout_poll} -M 2 -p $port -s ${cl_proto} -c TIMESTAMPNS \ + $mptcp_connect -t ${timeout_poll} -M 2 -p $port -s ${cl_proto} -c TIMESTAMPNS,TCPINQ \ $connect_addr < "$cin" > "$cout" & cpid=$! @@ -279,6 +279,45 @@ run_tests() fi } +do_tcpinq_test() +{ + ip netns exec "$ns1" ./mptcp_inq "$@" + lret=$? + if [ $lret -ne 0 ];then + ret=$lret + echo "FAIL: mptcp_inq $@" 1>&2 + return $lret + fi + + echo "PASS: TCP_INQ cmsg/ioctl $@" + return $lret +} + +do_tcpinq_tests() +{ + local lret=0 + + ip netns exec "$ns1" iptables -F + ip netns exec "$ns1" ip6tables -F + + for args in "-t tcp" "-r tcp"; do + do_tcpinq_test $args + lret=$? + if [ $lret -ne 0 ] ; then + return $lret + fi + do_tcpinq_test -6 $args + lret=$? + if [ $lret -ne 0 ] ; then + return $lret + fi + done + + do_tcpinq_test -r tcp -t tcp + + return $? +} + sin=$(mktemp) sout=$(mktemp) cin=$(mktemp) @@ -300,4 +339,5 @@ if [ $ret -eq 0 ];then echo "PASS: SOL_MPTCP getsockopt has expected information" fi +do_tcpinq_tests exit $ret diff --git a/tools/testing/selftests/net/nettest.c b/tools/testing/selftests/net/nettest.c index b599003eb5ba..d9a6fd2cd9d3 100644 --- a/tools/testing/selftests/net/nettest.c +++ b/tools/testing/selftests/net/nettest.c @@ -85,6 +85,7 @@ struct sock_args { int version; /* AF_INET/AF_INET6 */ int use_setsockopt; + int use_freebind; int use_cmsg; const char *dev; const char *server_dev; @@ -514,6 +515,29 @@ static int set_membership(int sd, uint32_t grp, uint32_t addr, int ifindex) return 0; } +static int set_freebind(int sd, int version) +{ + unsigned int one = 1; + int rc = 0; + + switch (version) { + case AF_INET: + if (setsockopt(sd, SOL_IP, IP_FREEBIND, &one, sizeof(one))) { + log_err_errno("setsockopt(IP_FREEBIND)"); + rc = -1; + } + break; + case AF_INET6: + if (setsockopt(sd, SOL_IPV6, IPV6_FREEBIND, &one, sizeof(one))) { + log_err_errno("setsockopt(IPV6_FREEBIND"); + rc = -1; + } + break; + } + + return rc; +} + static int set_broadcast(int sd) { unsigned int one = 1; @@ -1419,6 +1443,9 @@ static int lsock_init(struct sock_args *args) set_unicast_if(sd, args->ifindex, args->version)) goto err; + if (args->use_freebind && set_freebind(sd, args->version)) + goto err; + if (bind_socket(sd, args)) goto err; @@ -1827,7 +1854,7 @@ static int ipc_parent(int cpid, int fd, struct sock_args *args) return client_status; } -#define GETOPT_STR "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SCi6xL:0:1:2:3:Fbq" +#define GETOPT_STR "sr:l:c:p:t:g:P:DRn:M:X:m:d:I:BN:O:SCi6xL:0:1:2:3:Fbqf" #define OPT_FORCE_BIND_KEY_IFINDEX 1001 #define OPT_NO_BIND_KEY_IFINDEX 1002 @@ -1864,6 +1891,7 @@ static void print_usage(char *prog) " -I dev bind socket to given device name - server mode\n" " -S use setsockopt (IP_UNICAST_IF or IP_MULTICAST_IF)\n" " to set device binding\n" + " -f bind socket with the IP[V6]_FREEBIND option\n" " -C use cmsg and IP_PKTINFO to specify device binding\n" "\n" " -L len send random message of given length\n" @@ -1999,6 +2027,9 @@ int main(int argc, char *argv[]) case 'S': args.use_setsockopt = 1; break; + case 'f': + args.use_freebind = 1; + break; case 'C': args.use_cmsg = 1; break; |