diff options
1487 files changed, 33216 insertions, 16525 deletions
diff --git a/Documentation/dev-tools/index.rst b/Documentation/dev-tools/index.rst index 09dee10d2592..f7809c7b1ba9 100644 --- a/Documentation/dev-tools/index.rst +++ b/Documentation/dev-tools/index.rst @@ -21,6 +21,7 @@ whole; patches welcome! kasan ubsan kmemleak + kcsan gdb-kernel-debugging kgdb kselftest diff --git a/Documentation/dev-tools/kcsan.rst b/Documentation/dev-tools/kcsan.rst new file mode 100644 index 000000000000..ce4bbd918648 --- /dev/null +++ b/Documentation/dev-tools/kcsan.rst @@ -0,0 +1,321 @@ +The Kernel Concurrency Sanitizer (KCSAN) +======================================== + +The Kernel Concurrency Sanitizer (KCSAN) is a dynamic race detector, which +relies on compile-time instrumentation, and uses a watchpoint-based sampling +approach to detect races. KCSAN's primary purpose is to detect `data races`_. + +Usage +----- + +KCSAN requires Clang version 11 or later. + +To enable KCSAN configure the kernel with:: + + CONFIG_KCSAN = y + +KCSAN provides several other configuration options to customize behaviour (see +the respective help text in ``lib/Kconfig.kcsan`` for more info). + +Error reports +~~~~~~~~~~~~~ + +A typical data race report looks like this:: + + ================================================================== + BUG: KCSAN: data-race in generic_permission / kernfs_refresh_inode + + write to 0xffff8fee4c40700c of 4 bytes by task 175 on cpu 4: + kernfs_refresh_inode+0x70/0x170 + kernfs_iop_permission+0x4f/0x90 + inode_permission+0x190/0x200 + link_path_walk.part.0+0x503/0x8e0 + path_lookupat.isra.0+0x69/0x4d0 + filename_lookup+0x136/0x280 + user_path_at_empty+0x47/0x60 + vfs_statx+0x9b/0x130 + __do_sys_newlstat+0x50/0xb0 + __x64_sys_newlstat+0x37/0x50 + do_syscall_64+0x85/0x260 + entry_SYSCALL_64_after_hwframe+0x44/0xa9 + + read to 0xffff8fee4c40700c of 4 bytes by task 166 on cpu 6: + generic_permission+0x5b/0x2a0 + kernfs_iop_permission+0x66/0x90 + inode_permission+0x190/0x200 + link_path_walk.part.0+0x503/0x8e0 + path_lookupat.isra.0+0x69/0x4d0 + filename_lookup+0x136/0x280 + user_path_at_empty+0x47/0x60 + do_faccessat+0x11a/0x390 + __x64_sys_access+0x3c/0x50 + do_syscall_64+0x85/0x260 + entry_SYSCALL_64_after_hwframe+0x44/0xa9 + + Reported by Kernel Concurrency Sanitizer on: + CPU: 6 PID: 166 Comm: systemd-journal Not tainted 5.3.0-rc7+ #1 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 + ================================================================== + +The header of the report provides a short summary of the functions involved in +the race. It is followed by the access types and stack traces of the 2 threads +involved in the data race. + +The other less common type of data race report looks like this:: + + ================================================================== + BUG: KCSAN: data-race in e1000_clean_rx_irq+0x551/0xb10 + + race at unknown origin, with read to 0xffff933db8a2ae6c of 1 bytes by interrupt on cpu 0: + e1000_clean_rx_irq+0x551/0xb10 + e1000_clean+0x533/0xda0 + net_rx_action+0x329/0x900 + __do_softirq+0xdb/0x2db + irq_exit+0x9b/0xa0 + do_IRQ+0x9c/0xf0 + ret_from_intr+0x0/0x18 + default_idle+0x3f/0x220 + arch_cpu_idle+0x21/0x30 + do_idle+0x1df/0x230 + cpu_startup_entry+0x14/0x20 + rest_init+0xc5/0xcb + arch_call_rest_init+0x13/0x2b + start_kernel+0x6db/0x700 + + Reported by Kernel Concurrency Sanitizer on: + CPU: 0 PID: 0 Comm: swapper/0 Not tainted 5.3.0-rc7+ #2 + Hardware name: QEMU Standard PC (i440FX + PIIX, 1996), BIOS 1.12.0-1 04/01/2014 + ================================================================== + +This report is generated where it was not possible to determine the other +racing thread, but a race was inferred due to the data value of the watched +memory location having changed. These can occur either due to missing +instrumentation or e.g. DMA accesses. These reports will only be generated if +``CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN=y`` (selected by default). + +Selective analysis +~~~~~~~~~~~~~~~~~~ + +It may be desirable to disable data race detection for specific accesses, +functions, compilation units, or entire subsystems. For static blacklisting, +the below options are available: + +* KCSAN understands the ``data_race(expr)`` annotation, which tells KCSAN that + any data races due to accesses in ``expr`` should be ignored and resulting + behaviour when encountering a data race is deemed safe. + +* Disabling data race detection for entire functions can be accomplished by + using the function attribute ``__no_kcsan``:: + + __no_kcsan + void foo(void) { + ... + + To dynamically limit for which functions to generate reports, see the + `DebugFS interface`_ blacklist/whitelist feature. + + For ``__always_inline`` functions, replace ``__always_inline`` with + ``__no_kcsan_or_inline`` (which implies ``__always_inline``):: + + static __no_kcsan_or_inline void foo(void) { + ... + +* To disable data race detection for a particular compilation unit, add to the + ``Makefile``:: + + KCSAN_SANITIZE_file.o := n + +* To disable data race detection for all compilation units listed in a + ``Makefile``, add to the respective ``Makefile``:: + + KCSAN_SANITIZE := n + +Furthermore, it is possible to tell KCSAN to show or hide entire classes of +data races, depending on preferences. These can be changed via the following +Kconfig options: + +* ``CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY``: If enabled and a conflicting write + is observed via a watchpoint, but the data value of the memory location was + observed to remain unchanged, do not report the data race. + +* ``CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC``: Assume that plain aligned writes + up to word size are atomic by default. Assumes that such writes are not + subject to unsafe compiler optimizations resulting in data races. The option + causes KCSAN to not report data races due to conflicts where the only plain + accesses are aligned writes up to word size. + +DebugFS interface +~~~~~~~~~~~~~~~~~ + +The file ``/sys/kernel/debug/kcsan`` provides the following interface: + +* Reading ``/sys/kernel/debug/kcsan`` returns various runtime statistics. + +* Writing ``on`` or ``off`` to ``/sys/kernel/debug/kcsan`` allows turning KCSAN + on or off, respectively. + +* Writing ``!some_func_name`` to ``/sys/kernel/debug/kcsan`` adds + ``some_func_name`` to the report filter list, which (by default) blacklists + reporting data races where either one of the top stackframes are a function + in the list. + +* Writing either ``blacklist`` or ``whitelist`` to ``/sys/kernel/debug/kcsan`` + changes the report filtering behaviour. For example, the blacklist feature + can be used to silence frequently occurring data races; the whitelist feature + can help with reproduction and testing of fixes. + +Tuning performance +~~~~~~~~~~~~~~~~~~ + +Core parameters that affect KCSAN's overall performance and bug detection +ability are exposed as kernel command-line arguments whose defaults can also be +changed via the corresponding Kconfig options. + +* ``kcsan.skip_watch`` (``CONFIG_KCSAN_SKIP_WATCH``): Number of per-CPU memory + operations to skip, before another watchpoint is set up. Setting up + watchpoints more frequently will result in the likelihood of races to be + observed to increase. This parameter has the most significant impact on + overall system performance and race detection ability. + +* ``kcsan.udelay_task`` (``CONFIG_KCSAN_UDELAY_TASK``): For tasks, the + microsecond delay to stall execution after a watchpoint has been set up. + Larger values result in the window in which we may observe a race to + increase. + +* ``kcsan.udelay_interrupt`` (``CONFIG_KCSAN_UDELAY_INTERRUPT``): For + interrupts, the microsecond delay to stall execution after a watchpoint has + been set up. Interrupts have tighter latency requirements, and their delay + should generally be smaller than the one chosen for tasks. + +They may be tweaked at runtime via ``/sys/module/kcsan/parameters/``. + +Data Races +---------- + +In an execution, two memory accesses form a *data race* if they *conflict*, +they happen concurrently in different threads, and at least one of them is a +*plain access*; they *conflict* if both access the same memory location, and at +least one is a write. For a more thorough discussion and definition, see `"Plain +Accesses and Data Races" in the LKMM`_. + +.. _"Plain Accesses and Data Races" in the LKMM: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/tools/memory-model/Documentation/explanation.txt#n1922 + +Relationship with the Linux-Kernel Memory Consistency Model (LKMM) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The LKMM defines the propagation and ordering rules of various memory +operations, which gives developers the ability to reason about concurrent code. +Ultimately this allows to determine the possible executions of concurrent code, +and if that code is free from data races. + +KCSAN is aware of *marked atomic operations* (``READ_ONCE``, ``WRITE_ONCE``, +``atomic_*``, etc.), but is oblivious of any ordering guarantees and simply +assumes that memory barriers are placed correctly. In other words, KCSAN +assumes that as long as a plain access is not observed to race with another +conflicting access, memory operations are correctly ordered. + +This means that KCSAN will not report *potential* data races due to missing +memory ordering. Developers should therefore carefully consider the required +memory ordering requirements that remain unchecked. If, however, missing +memory ordering (that is observable with a particular compiler and +architecture) leads to an observable data race (e.g. entering a critical +section erroneously), KCSAN would report the resulting data race. + +Race Detection Beyond Data Races +-------------------------------- + +For code with complex concurrency design, race-condition bugs may not always +manifest as data races. Race conditions occur if concurrently executing +operations result in unexpected system behaviour. On the other hand, data races +are defined at the C-language level. The following macros can be used to check +properties of concurrent code where bugs would not manifest as data races. + +.. kernel-doc:: include/linux/kcsan-checks.h + :functions: ASSERT_EXCLUSIVE_WRITER ASSERT_EXCLUSIVE_WRITER_SCOPED + ASSERT_EXCLUSIVE_ACCESS ASSERT_EXCLUSIVE_ACCESS_SCOPED + ASSERT_EXCLUSIVE_BITS + +Implementation Details +---------------------- + +KCSAN relies on observing that two accesses happen concurrently. Crucially, we +want to (a) increase the chances of observing races (especially for races that +manifest rarely), and (b) be able to actually observe them. We can accomplish +(a) by injecting various delays, and (b) by using address watchpoints (or +breakpoints). + +If we deliberately stall a memory access, while we have a watchpoint for its +address set up, and then observe the watchpoint to fire, two accesses to the +same address just raced. Using hardware watchpoints, this is the approach taken +in `DataCollider +<http://usenix.org/legacy/events/osdi10/tech/full_papers/Erickson.pdf>`_. +Unlike DataCollider, KCSAN does not use hardware watchpoints, but instead +relies on compiler instrumentation and "soft watchpoints". + +In KCSAN, watchpoints are implemented using an efficient encoding that stores +access type, size, and address in a long; the benefits of using "soft +watchpoints" are portability and greater flexibility. KCSAN then relies on the +compiler instrumenting plain accesses. For each instrumented plain access: + +1. Check if a matching watchpoint exists; if yes, and at least one access is a + write, then we encountered a racing access. + +2. Periodically, if no matching watchpoint exists, set up a watchpoint and + stall for a small randomized delay. + +3. Also check the data value before the delay, and re-check the data value + after delay; if the values mismatch, we infer a race of unknown origin. + +To detect data races between plain and marked accesses, KCSAN also annotates +marked accesses, but only to check if a watchpoint exists; i.e. KCSAN never +sets up a watchpoint on marked accesses. By never setting up watchpoints for +marked operations, if all accesses to a variable that is accessed concurrently +are properly marked, KCSAN will never trigger a watchpoint and therefore never +report the accesses. + +Key Properties +~~~~~~~~~~~~~~ + +1. **Memory Overhead:** The overall memory overhead is only a few MiB + depending on configuration. The current implementation uses a small array of + longs to encode watchpoint information, which is negligible. + +2. **Performance Overhead:** KCSAN's runtime aims to be minimal, using an + efficient watchpoint encoding that does not require acquiring any shared + locks in the fast-path. For kernel boot on a system with 8 CPUs: + + - 5.0x slow-down with the default KCSAN config; + - 2.8x slow-down from runtime fast-path overhead only (set very large + ``KCSAN_SKIP_WATCH`` and unset ``KCSAN_SKIP_WATCH_RANDOMIZE``). + +3. **Annotation Overheads:** Minimal annotations are required outside the KCSAN + runtime. As a result, maintenance overheads are minimal as the kernel + evolves. + +4. **Detects Racy Writes from Devices:** Due to checking data values upon + setting up watchpoints, racy writes from devices can also be detected. + +5. **Memory Ordering:** KCSAN is *not* explicitly aware of the LKMM's ordering + rules; this may result in missed data races (false negatives). + +6. **Analysis Accuracy:** For observed executions, due to using a sampling + strategy, the analysis is *unsound* (false negatives possible), but aims to + be complete (no false positives). + +Alternatives Considered +----------------------- + +An alternative data race detection approach for the kernel can be found in the +`Kernel Thread Sanitizer (KTSAN) <https://github.com/google/ktsan/wiki>`_. +KTSAN is a happens-before data race detector, which explicitly establishes the +happens-before order between memory operations, which can then be used to +determine data races as defined in `Data Races`_. + +To build a correct happens-before relation, KTSAN must be aware of all ordering +rules of the LKMM and synchronization primitives. Unfortunately, any omission +leads to large numbers of false positives, which is especially detrimental in +the context of the kernel which includes numerous custom synchronization +mechanisms. To track the happens-before relation, KTSAN's implementation +requires metadata for each memory location (shadow memory), which for each page +corresponds to 4 pages of shadow memory, and can translate into overhead of +tens of GiB on a large system. diff --git a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-pmc.yaml b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-pmc.yaml index 81534d04094b..b71a20af5f70 100644 --- a/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-pmc.yaml +++ b/Documentation/devicetree/bindings/arm/tegra/nvidia,tegra20-pmc.yaml @@ -85,9 +85,8 @@ properties: CPU power good signal from external PMIC to PMC is enabled. nvidia,suspend-mode: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2] description: The suspend mode that the platform should use. Mode 0 is for LP0, CPU + Core voltage off and DRAM in self-refresh diff --git a/Documentation/devicetree/bindings/ata/sata_highbank.yaml b/Documentation/devicetree/bindings/ata/sata_highbank.yaml index b195457006cc..5e2a2394e600 100644 --- a/Documentation/devicetree/bindings/ata/sata_highbank.yaml +++ b/Documentation/devicetree/bindings/ata/sata_highbank.yaml @@ -40,28 +40,25 @@ properties: calxeda,led-order: description: Maps port numbers to offsets within the SGPIO bitstream. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 8 + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 calxeda,port-phys: description: | phandle-combophy and lane assignment, which maps each SATA port to a combophy and a lane within that combophy - allOf: - - $ref: /schemas/types.yaml#/definitions/phandle-array - - minItems: 1 - maxItems: 8 + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 8 calxeda,tx-atten: description: | Contains TX attenuation override codes, one per port. The upper 24 bits of each entry are always 0 and thus ignored. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 8 + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 calxeda,sgpio-gpio: description: | diff --git a/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml b/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml index d6a3b71ea835..68b0131a31d0 100644 --- a/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml +++ b/Documentation/devicetree/bindings/bus/baikal,bt1-apb.yaml @@ -71,8 +71,8 @@ examples: bus@1f059000 { compatible = "baikal,bt1-apb", "simple-bus"; - reg = <0 0x1f059000 0 0x1000>, - <0 0x1d000000 0 0x2040000>; + reg = <0x1f059000 0x1000>, + <0x1d000000 0x2040000>; reg-names = "ehb", "nodev"; #address-cells = <1>; #size-cells = <1>; diff --git a/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml b/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml index 203bc0e5346b..29e1aaea132b 100644 --- a/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml +++ b/Documentation/devicetree/bindings/bus/baikal,bt1-axi.yaml @@ -85,8 +85,8 @@ examples: bus@1f05a000 { compatible = "baikal,bt1-axi", "simple-bus"; - reg = <0 0x1f05a000 0 0x1000>, - <0 0x1f04d110 0 0x8>; + reg = <0x1f05a000 0x1000>, + <0x1f04d110 0x8>; reg-names = "qos", "ehb"; #address-cells = <1>; #size-cells = <1>; diff --git a/Documentation/devicetree/bindings/clock/imx6q-clock.yaml b/Documentation/devicetree/bindings/clock/imx6q-clock.yaml index 429e3b62b965..92a8e545e212 100644 --- a/Documentation/devicetree/bindings/clock/imx6q-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx6q-clock.yaml @@ -23,7 +23,6 @@ properties: items: - description: CCM interrupt request 1 - description: CCM interrupt request 2 - maxItems: 2 '#clock-cells': const: 1 diff --git a/Documentation/devicetree/bindings/clock/imx6sl-clock.yaml b/Documentation/devicetree/bindings/clock/imx6sl-clock.yaml index 135568c46350..c97bf95b4150 100644 --- a/Documentation/devicetree/bindings/clock/imx6sl-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx6sl-clock.yaml @@ -23,7 +23,6 @@ properties: items: - description: CCM interrupt request 1 - description: CCM interrupt request 2 - maxItems: 2 '#clock-cells': const: 1 diff --git a/Documentation/devicetree/bindings/clock/imx6sll-clock.yaml b/Documentation/devicetree/bindings/clock/imx6sll-clock.yaml index fa55f1ce3e57..de48924be191 100644 --- a/Documentation/devicetree/bindings/clock/imx6sll-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx6sll-clock.yaml @@ -23,7 +23,6 @@ properties: items: - description: CCM interrupt request 1 - description: CCM interrupt request 2 - maxItems: 2 '#clock-cells': const: 1 diff --git a/Documentation/devicetree/bindings/clock/imx6sx-clock.yaml b/Documentation/devicetree/bindings/clock/imx6sx-clock.yaml index 982d698e8c54..e50cddee43c3 100644 --- a/Documentation/devicetree/bindings/clock/imx6sx-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx6sx-clock.yaml @@ -23,7 +23,6 @@ properties: items: - description: CCM interrupt request 1 - description: CCM interrupt request 2 - maxItems: 2 '#clock-cells': const: 1 diff --git a/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml b/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml index 3c779eea6394..36ce7667c972 100644 --- a/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml +++ b/Documentation/devicetree/bindings/clock/imx6ul-clock.yaml @@ -23,7 +23,6 @@ properties: items: - description: CCM interrupt request 1 - description: CCM interrupt request 2 - maxItems: 2 '#clock-cells': const: 1 diff --git a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml index 1695e3e4bcec..ed8148e26e24 100644 --- a/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml +++ b/Documentation/devicetree/bindings/display/rockchip/rockchip-vop.yaml @@ -106,8 +106,8 @@ examples: #include <dt-bindings/power/rk3288-power.h> vopb: vopb@ff930000 { compatible = "rockchip,rk3288-vop"; - reg = <0x0 0xff930000 0x0 0x19c>, - <0x0 0xff931000 0x0 0x1000>; + reg = <0xff930000 0x19c>, + <0xff931000 0x1000>; interrupts = <GIC_SPI 15 IRQ_TYPE_LEVEL_HIGH>; clocks = <&cru ACLK_VOP0>, <&cru DCLK_VOP0>, diff --git a/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml b/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml index 04a3c51e1dc1..1240f6289249 100644 --- a/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml +++ b/Documentation/devicetree/bindings/gpio/snps,dw-apb-gpio.yaml @@ -63,11 +63,10 @@ patternProperties: snps,nr-gpios: description: The number of GPIO pins exported by the port. + $ref: /schemas/types.yaml#/definitions/uint32 default: 32 - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 1 - maximum: 32 + minimum: 1 + maximum: 32 interrupts: description: | diff --git a/Documentation/devicetree/bindings/i2c/i2c-designware.txt b/Documentation/devicetree/bindings/i2c/i2c-designware.txt deleted file mode 100644 index 08be4d3846e5..000000000000 --- a/Documentation/devicetree/bindings/i2c/i2c-designware.txt +++ /dev/null @@ -1,73 +0,0 @@ -* Synopsys DesignWare I2C - -Required properties : - - - compatible : should be "snps,designware-i2c" - or "mscc,ocelot-i2c" with "snps,designware-i2c" for fallback - - reg : Offset and length of the register set for the device - - interrupts : <IRQ> where IRQ is the interrupt number. - - clocks : phandles for the clocks, see the description of clock-names below. - The phandle for the "ic_clk" clock is required. The phandle for the "pclk" - clock is optional. If a single clock is specified but no clock-name, it is - the "ic_clk" clock. If both clocks are listed, the "ic_clk" must be first. - -Recommended properties : - - - clock-frequency : desired I2C bus clock frequency in Hz. - -Optional properties : - - - clock-names : Contains the names of the clocks: - "ic_clk", for the core clock used to generate the external I2C clock. - "pclk", the interface clock, required for register access. - - - reg : for "mscc,ocelot-i2c", a second register set to configure the SDA hold - time, named ICPU_CFG:TWI_DELAY in the datasheet. - - - i2c-sda-hold-time-ns : should contain the SDA hold time in nanoseconds. - This option is only supported in hardware blocks version 1.11a or newer and - on Microsemi SoCs ("mscc,ocelot-i2c" compatible). - - - i2c-scl-falling-time-ns : should contain the SCL falling time in nanoseconds. - This value which is by default 300ns is used to compute the tLOW period. - - - i2c-sda-falling-time-ns : should contain the SDA falling time in nanoseconds. - This value which is by default 300ns is used to compute the tHIGH period. - -Examples : - - i2c@f0000 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "snps,designware-i2c"; - reg = <0xf0000 0x1000>; - interrupts = <11>; - clock-frequency = <400000>; - }; - - i2c@1120000 { - #address-cells = <1>; - #size-cells = <0>; - compatible = "snps,designware-i2c"; - reg = <0x1120000 0x1000>; - interrupt-parent = <&ictl>; - interrupts = <12 1>; - clock-frequency = <400000>; - i2c-sda-hold-time-ns = <300>; - i2c-sda-falling-time-ns = <300>; - i2c-scl-falling-time-ns = <300>; - }; - - i2c@1120000 { - #address-cells = <1>; - #size-cells = <0>; - reg = <0x2000 0x100>; - clock-frequency = <400000>; - clocks = <&i2cclk>; - interrupts = <0>; - - eeprom@64 { - compatible = "linux,slave-24c02"; - reg = <0x40000064>; - }; - }; diff --git a/Documentation/devicetree/bindings/i2c/i2c-qcom-cci.txt b/Documentation/devicetree/bindings/i2c/i2c-qcom-cci.txt new file mode 100644 index 000000000000..c6668b7c66e6 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/i2c-qcom-cci.txt @@ -0,0 +1,92 @@ +Qualcomm Camera Control Interface (CCI) I2C controller + +PROPERTIES: + +- compatible: + Usage: required + Value type: <string> + Definition: must be one of: + "qcom,msm8916-cci" + "qcom,msm8996-cci" + "qcom,sdm845-cci" + +- reg + Usage: required + Value type: <prop-encoded-array> + Definition: base address CCI I2C controller and length of memory + mapped region. + +- interrupts: + Usage: required + Value type: <prop-encoded-array> + Definition: specifies the CCI I2C interrupt. The format of the + specifier is defined by the binding document describing + the node's interrupt parent. + +- clocks: + Usage: required + Value type: <prop-encoded-array> + Definition: a list of phandle, should contain an entry for each + entries in clock-names. + +- clock-names + Usage: required + Value type: <string> + Definition: a list of clock names, must include "cci" clock. + +- power-domains + Usage: required for "qcom,msm8996-cci" + Value type: <prop-encoded-array> + Definition: + +SUBNODES: + +The CCI provides I2C masters for one (msm8916) or two i2c busses (msm8996 and +sdm845), described as subdevices named "i2c-bus@0" and "i2c-bus@1". + +PROPERTIES: + +- reg: + Usage: required + Value type: <u32> + Definition: Index of the CCI bus/master + +- clock-frequency: + Usage: optional + Value type: <u32> + Definition: Desired I2C bus clock frequency in Hz, defaults to 100 + kHz if omitted. + +Example: + + cci@a0c000 { + compatible = "qcom,msm8996-cci"; + #address-cells = <1>; + #size-cells = <0>; + reg = <0xa0c000 0x1000>; + interrupts = <GIC_SPI 295 IRQ_TYPE_EDGE_RISING>; + clocks = <&mmcc MMSS_MMAGIC_AHB_CLK>, + <&mmcc CAMSS_TOP_AHB_CLK>, + <&mmcc CAMSS_CCI_AHB_CLK>, + <&mmcc CAMSS_CCI_CLK>, + <&mmcc CAMSS_AHB_CLK>; + clock-names = "mmss_mmagic_ahb", + "camss_top_ahb", + "cci_ahb", + "cci", + "camss_ahb"; + + i2c-bus@0 { + reg = <0>; + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; + }; + + i2c-bus@1 { + reg = <1>; + clock-frequency = <400000>; + #address-cells = <1>; + #size-cells = <0>; + }; + }; diff --git a/Documentation/devicetree/bindings/i2c/i2c.txt b/Documentation/devicetree/bindings/i2c/i2c.txt index 9a53df4243c6..438ae123107e 100644 --- a/Documentation/devicetree/bindings/i2c/i2c.txt +++ b/Documentation/devicetree/bindings/i2c/i2c.txt @@ -2,32 +2,26 @@ Generic device tree bindings for I2C busses =========================================== This document describes generic bindings which can be used to describe I2C -busses in a device tree. +busses and their child devices in a device tree. -Required properties -------------------- +Required properties (per bus) +----------------------------- - #address-cells - should be <1>. Read more about addresses below. - #size-cells - should be <0>. -- compatible - name of I2C bus controller following generic names - recommended practice. +- compatible - name of I2C bus controller For other required properties e.g. to describe register sets, clocks, etc. check the binding documentation of the specific driver. The cells properties above define that an address of children of an I2C bus -are described by a single value. This is usually a 7 bit address. However, -flags can be attached to the address. I2C_TEN_BIT_ADDRESS is used to mark a 10 -bit address. It is needed to avoid the ambiguity between e.g. a 7 bit address -of 0x50 and a 10 bit address of 0x050 which, in theory, can be on the same bus. -Another flag is I2C_OWN_SLAVE_ADDRESS to mark addresses on which we listen to -be devices ourselves. +are described by a single value. -Optional properties -------------------- +Optional properties (per bus) +----------------------------- These properties may not be supported by all drivers. However, if a driver -wants to support one of the below features, it should adapt the bindings below. +wants to support one of the below features, it should adapt these bindings. - clock-frequency frequency of bus clock in Hz. @@ -73,31 +67,54 @@ wants to support one of the below features, it should adapt the bindings below. i2c bus clock frequency (clock-frequency). Specified in Hz. -- interrupts - interrupts used by the device. - -- interrupt-names - "irq", "wakeup" and "smbus_alert" names are recognized by I2C core, - other names are left to individual drivers. - -- host-notify - device uses SMBus host notify protocol instead of interrupt line. - - multi-master states that there is another master active on this bus. The OS can use this information to adapt power management to keep the arbitration awake - all the time, for example. + all the time, for example. Can not be combined with 'single-master'. -- wakeup-source - device can be used as a wakeup source. +- single-master + states that there is no other master active on this bus. The OS can use + this information to detect a stalled bus more reliably, for example. + Can not be combined with 'multi-master'. + +Required properties (per child device) +-------------------------------------- + +- compatible + name of I2C slave device - reg - I2C slave addresses + One or many I2C slave addresses. These are usually a 7 bit addresses. + However, flags can be attached to an address. I2C_TEN_BIT_ADDRESS is + used to mark a 10 bit address. It is needed to avoid the ambiguity + between e.g. a 7 bit address of 0x50 and a 10 bit address of 0x050 + which, in theory, can be on the same bus. + Another flag is I2C_OWN_SLAVE_ADDRESS to mark addresses on which we + listen to be devices ourselves. + +Optional properties (per child device) +-------------------------------------- + +These properties may not be supported by all drivers. However, if a driver +wants to support one of the below features, it should adapt these bindings. + +- host-notify + device uses SMBus host notify protocol instead of interrupt line. + +- interrupts + interrupts used by the device. + +- interrupt-names + "irq", "wakeup" and "smbus_alert" names are recognized by I2C core, + other names are left to individual drivers. - reg-names Names of map programmable addresses. It can contain any map needing another address than default one. +- wakeup-source + device can be used as a wakeup source. + Binding may contain optional "interrupts" property, describing interrupts used by the device. I2C core will assign "irq" interrupt (or the very first interrupt if not using interrupt names) as primary interrupt for the slave. diff --git a/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml new file mode 100644 index 000000000000..e3ef2d36f372 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/nuvoton,npcm7xx-i2c.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: GPL-2.0-only OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/nuvoton,npcm7xx-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: nuvoton NPCM7XX I2C Controller Device Tree Bindings + +description: | + The NPCM750x includes sixteen I2C bus controllers. All Controllers support + both master and slave mode. Each controller can switch between master and slave + at run time (i.e. IPMB mode). Each controller has two 16 byte HW FIFO for TX and + RX. + +maintainers: + - Tali Perry <tali.perry1@gmail.com> + +properties: + compatible: + const: nuvoton,npcm7xx-i2c + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + description: Reference clock for the I2C bus + + clock-frequency: + description: Desired I2C bus clock frequency in Hz. If not specified, + the default 100 kHz frequency will be used. + possible values are 100000, 400000 and 1000000. + default: 100000 + enum: [100000, 400000, 1000000] + +required: + - compatible + - reg + - interrupts + - clocks + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + +unevaluatedProperties: false + +examples: + - | + #include <dt-bindings/clock/nuvoton,npcm7xx-clock.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + i2c0: i2c@80000 { + reg = <0x80000 0x1000>; + clocks = <&clk NPCM7XX_CLK_APB2>; + clock-frequency = <100000>; + interrupts = <GIC_SPI 64 IRQ_TYPE_LEVEL_HIGH>; + compatible = "nuvoton,npcm750-i2c"; + }; + +... diff --git a/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml new file mode 100644 index 000000000000..4f746bef2374 --- /dev/null +++ b/Documentation/devicetree/bindings/i2c/snps,designware-i2c.yaml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0-only +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/i2c/snps,designware-i2c.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Synopsys DesignWare APB I2C Controller + +maintainers: + - Jarkko Nikula <jarkko.nikula@linux.intel.com> + +allOf: + - $ref: /schemas/i2c/i2c-controller.yaml# + - if: + properties: + compatible: + not: + contains: + const: mscc,ocelot-i2c + then: + properties: + reg: + maxItems: 1 + +properties: + compatible: + oneOf: + - description: Generic Synopsys DesignWare I2C controller + const: snps,designware-i2c + - description: Microsemi Ocelot SoCs I2C controller + items: + - const: mscc,ocelot-i2c + - const: snps,designware-i2c + - description: Baikal-T1 SoC System I2C controller + const: baikal,bt1-sys-i2c + + reg: + minItems: 1 + items: + - description: DW APB I2C controller memory mapped registers + - description: | + ICPU_CFG:TWI_DELAY registers to setup the SDA hold time. + This registers are specific to the Ocelot I2C-controller. + + interrupts: + maxItems: 1 + + clocks: + minItems: 1 + items: + - description: I2C controller reference clock source + - description: APB interface clock source + + clock-names: + minItems: 1 + items: + - const: ref + - const: pclk + + resets: + maxItems: 1 + + clock-frequency: + description: Desired I2C bus clock frequency in Hz + enum: [100000, 400000, 1000000, 3400000] + default: 400000 + + i2c-sda-hold-time-ns: + maxItems: 1 + description: | + The property should contain the SDA hold time in nanoseconds. This option + is only supported in hardware blocks version 1.11a or newer or on + Microsemi SoCs. + + i2c-scl-falling-time-ns: + maxItems: 1 + description: | + The property should contain the SCL falling time in nanoseconds. + This value is used to compute the tLOW period. + default: 300 + + i2c-sda-falling-time-ns: + maxItems: 1 + description: | + The property should contain the SDA falling time in nanoseconds. + This value is used to compute the tHIGH period. + default: 300 + + dmas: + items: + - description: TX DMA Channel + - description: RX DMA Channel + + dma-names: + items: + - const: tx + - const: rx + +unevaluatedProperties: false + +required: + - compatible + - reg + - "#address-cells" + - "#size-cells" + - interrupts + +examples: + - | + i2c@f0000 { + compatible = "snps,designware-i2c"; + reg = <0xf0000 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <11>; + clock-frequency = <400000>; + }; + - | + i2c@1120000 { + compatible = "snps,designware-i2c"; + reg = <0x1120000 0x1000>; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <12 1>; + clock-frequency = <400000>; + i2c-sda-hold-time-ns = <300>; + i2c-sda-falling-time-ns = <300>; + i2c-scl-falling-time-ns = <300>; + }; + - | + i2c@2000 { + compatible = "snps,designware-i2c"; + reg = <0x2000 0x100>; + #address-cells = <1>; + #size-cells = <0>; + clock-frequency = <400000>; + clocks = <&i2cclk>; + interrupts = <0>; + + eeprom@64 { + compatible = "atmel,24c02"; + reg = <0x64>; + }; + }; + - | + i2c@100400 { + compatible = "mscc,ocelot-i2c", "snps,designware-i2c"; + reg = <0x100400 0x100>, <0x198 0x8>; + pinctrl-0 = <&i2c_pins>; + pinctrl-names = "default"; + #address-cells = <1>; + #size-cells = <0>; + interrupts = <8>; + clocks = <&ahb_clk>; + }; +... diff --git a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml index 7b3342354bbb..f2fcbb361180 100644 --- a/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml +++ b/Documentation/devicetree/bindings/i2c/st,stm32-i2c.yaml @@ -81,11 +81,11 @@ properties: clock-frequency: description: Desired I2C bus clock frequency in Hz. If not specified, the default 100 kHz frequency will be used. - For STM32F7, STM32H7 and STM32MP1 SoCs, Standard-mode, - Fast-mode and Fast-mode Plus are supported, possible - values are 100000, 400000 and 1000000. + For STM32F7, STM32H7 and STM32MP1 SoCs, if timing parameters + match, the bus clock frequency can be from 1Hz to 1MHz. default: 100000 - enum: [100000, 400000, 1000000] + minimum: 1 + maximum: 1000000 required: - compatible diff --git a/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml b/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml index 98baecb4b98a..208faaffa58d 100644 --- a/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml +++ b/Documentation/devicetree/bindings/iio/imu/adi,adis16475.yaml @@ -67,8 +67,7 @@ properties: 1 - direct_sync 2 - scaled_sync 3 - pulse_sync - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 minimum: 0 maximum: 3 diff --git a/Documentation/devicetree/bindings/iio/light/amstaos,tsl2563.yaml b/Documentation/devicetree/bindings/iio/light/amstaos,tsl2563.yaml index efd2eba5f23c..e201a06d8fdc 100644 --- a/Documentation/devicetree/bindings/iio/light/amstaos,tsl2563.yaml +++ b/Documentation/devicetree/bindings/iio/light/amstaos,tsl2563.yaml @@ -25,9 +25,8 @@ properties: amstaos,cover-comp-gain: description: Multiplier for gain compensation - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [1, 16] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 16] required: - compatible diff --git a/Documentation/devicetree/bindings/input/iqs269a.yaml b/Documentation/devicetree/bindings/input/iqs269a.yaml index f0242bb4be81..9c154e5e1a91 100644 --- a/Documentation/devicetree/bindings/input/iqs269a.yaml +++ b/Documentation/devicetree/bindings/input/iqs269a.yaml @@ -40,10 +40,9 @@ properties: posed on channels 6 and 7 by Hall-effect sensing. azoteq,suspend-mode: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: | Specifies the power mode during suspend as follows: 0: Automatic (same as normal runtime, i.e. suspend/resume disabled) @@ -56,11 +55,10 @@ properties: description: Divides the device's core clock by a factor of 4. azoteq,ulp-update: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 7 - default: 3 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 7 + default: 3 description: Specifies the ultra-low-power mode update rate. azoteq,reseed-offset: @@ -70,34 +68,30 @@ properties: reseed events. azoteq,filt-str-lp-lta: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: Specifies the long-term average filter strength during low-power mode. azoteq,filt-str-lp-cnt: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: Specifies the raw count filter strength during low-power mode. azoteq,filt-str-np-lta: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: Specifies the long-term average filter strength during normal-power mode. azoteq,filt-str-np-cnt: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: Specifies the raw count filter strength during normal-power mode. @@ -156,11 +150,10 @@ properties: description: Disables all raw count filtering. azoteq,gpio3-select: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 7 - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 7 + default: 0 description: Selects the channel for which the GPIO3 pin represents touch state. @@ -172,10 +165,9 @@ properties: in either direction. azoteq,tx-freq: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: | Specifies the inductive sensing excitation frequency as follows (paren- thesized numbers represent the frequency if 'azoteq,clk-div' is present): @@ -189,10 +181,9 @@ properties: description: Increases the global capacitance adder from 0.5 pF to 1.5 pF. azoteq,reseed-select: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 0 description: | Specifies the event(s) that prompt the device to reseed (i.e. reset the long-term average) of an associated channel as follows: @@ -208,10 +199,9 @@ properties: channels. azoteq,filt-str-slider: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 1 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 1 description: Specifies the slider coordinate filter strength. patternProperties: @@ -246,27 +236,25 @@ patternProperties: description: Specifies that the channel participates in slider 1. azoteq,rx-enable: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 8 - items: - minimum: 0 - maximum: 7 + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + items: + minimum: 0 + maximum: 7 description: Specifies the CRX pin(s) associated with the channel. By default, only the CRX pin corresponding to the channel's index is enabled (e.g. CRX0 for channel 0). azoteq,tx-enable: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 8 - items: - minimum: 0 - maximum: 7 - default: [0, 1, 2, 3, 4, 5, 6, 7] + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + items: + minimum: 0 + maximum: 7 + default: [0, 1, 2, 3, 4, 5, 6, 7] description: Specifies the TX pin(s) associated with the channel. azoteq,meas-cap-decrease: @@ -279,10 +267,9 @@ patternProperties: description: Floats any inactive CRX pins instead of grounding them. azoteq,local-cap-size: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2] + default: 0 description: | Specifies the capacitance to be added to the channel as follows: 0: None @@ -296,10 +283,9 @@ patternProperties: deep-touch events relative to their respective thresholds. azoteq,proj-bias: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 2 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 2 description: | Specifies the bias current applied during projected-capacitance sensing as follows: @@ -309,10 +295,9 @@ patternProperties: 3: 20 uA azoteq,sense-mode: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 9, 14, 15] - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 9, 14, 15] + default: 0 description: | Specifies the channel's sensing mode as follows: 0: Self capacitance @@ -322,10 +307,9 @@ patternProperties: 15: Temperature azoteq,sense-freq: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 1 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 1 description: | Specifies the channel's sensing frequency as follows (parenthesized numbers represent the frequency if 'azoteq,clk-div' is present): @@ -339,10 +323,9 @@ patternProperties: description: Enables the static front-end for the channel. azoteq,ati-mode: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [0, 1, 2, 3] - default: 3 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [0, 1, 2, 3] + default: 3 description: | Specifies the channel's ATI mode as follows: 0: Disabled @@ -351,39 +334,35 @@ patternProperties: 3: Full azoteq,ati-base: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [75, 100, 150, 200] - default: 100 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [75, 100, 150, 200] + default: 100 description: Specifies the channel's ATI base. azoteq,ati-target: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - multipleOf: 32 - minimum: 0 - maximum: 2016 - default: 512 + $ref: /schemas/types.yaml#/definitions/uint32 + multipleOf: 32 + minimum: 0 + maximum: 2016 + default: 512 description: Specifies the channel's ATI target. azoteq,assoc-select: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 8 - items: - minimum: 0 - maximum: 7 + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 8 + items: + minimum: 0 + maximum: 7 description: Specifies the associated channels for which the channel serves as a reference channel. By default, no channels are selected. azoteq,assoc-weight: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 255 - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + default: 0 description: Specifies the channel's impact weight if it acts as an associated channel (0 = 0% impact, 255 = 200% impact). @@ -411,11 +390,10 @@ patternProperties: properties: azoteq,thresh: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 255 - default: 10 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + default: 10 description: Specifies the threshold for the event. linux,code: @@ -430,19 +408,17 @@ patternProperties: properties: azoteq,thresh: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 255 - default: 8 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + default: 8 description: Specifies the threshold for the event. azoteq,hyst: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 15 - default: 4 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 15 + default: 4 description: Specifies the hysteresis for the event. linux,code: @@ -457,19 +433,17 @@ patternProperties: properties: azoteq,thresh: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 255 - default: 26 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 255 + default: 26 description: Specifies the threshold for the event. azoteq,hyst: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 0 - maximum: 15 - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 15 + default: 0 description: Specifies the hysteresis for the event. linux,code: diff --git a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml index 1a5ebbdd219a..1b256d9dd92a 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-msi.yaml @@ -25,19 +25,17 @@ properties: description: u32 value of the base of parent HyperTransport vector allocated to PCH MSI. - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 0 - maximum: 255 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 0 + maximum: 255 loongson,msi-num-vecs: description: u32 value of the number of parent HyperTransport vectors allocated to PCH MSI. - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 1 - maximum: 256 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 1 + maximum: 256 msi-controller: true diff --git a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml index 274adea13f33..a6dcbb2971a9 100644 --- a/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml +++ b/Documentation/devicetree/bindings/interrupt-controller/loongson,pch-pic.yaml @@ -25,10 +25,9 @@ properties: description: u32 value of the base of parent HyperTransport vector allocated to PCH PIC. - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 0 - maximum: 192 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 0 + maximum: 192 interrupt-controller: true diff --git a/Documentation/devicetree/bindings/ipmi/ipmi-smic.yaml b/Documentation/devicetree/bindings/ipmi/ipmi-smic.yaml index f0bb157e9417..58fa76ee6176 100644 --- a/Documentation/devicetree/bindings/ipmi/ipmi-smic.yaml +++ b/Documentation/devicetree/bindings/ipmi/ipmi-smic.yaml @@ -31,9 +31,8 @@ properties: reg-size: description: The access width of the register in bytes. Defaults to 1. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [1, 2, 4, 8] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [1, 2, 4, 8] reg-spacing: $ref: /schemas/types.yaml#/definitions/uint32 @@ -43,9 +42,8 @@ properties: description: | The amount of bits to shift the register content to the right to get the data into bit zero. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - maximum: 56 + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 56 required: - compatible diff --git a/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml b/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml index 01c7d93dc658..32e0896c6bc1 100644 --- a/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml +++ b/Documentation/devicetree/bindings/leds/backlight/qcom-wled.yaml @@ -57,8 +57,7 @@ properties: description: | mA; per-string current limit. This property is supported only for WLED3. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 default: 20 minimum: 0 maximum: 25 @@ -74,38 +73,33 @@ properties: qcom,current-boost-limit: description: | mA; boost current limit. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 qcom,switching-freq: description: | kHz; switching frequency. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 600, 640, 685, 738, 800, 872, 960, 1066, 1200, 1371, 1600, 1920, 2400, 3200, 4800, 9600 ] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 600, 640, 685, 738, 800, 872, 960, 1066, 1200, 1371, 1600, 1920, 2400, 3200, 4800, 9600 ] qcom,ovp: description: | V; Over-voltage protection limit. This property is supported only for WLED3. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 27, 29, 32, 35 ] - - default: 29 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 27, 29, 32, 35 ] + default: 29 qcom,ovp-millivolt: description: | Over-voltage protection limit. This property is for WLED4 only. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 18100, 19600, 29600, 31100 ] - - default: 29600 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 18100, 19600, 29600, 31100 ] + default: 29600 qcom,num-strings: description: | number of led strings attached. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 qcom,enabled-strings: description: | @@ -113,8 +107,7 @@ properties: string of leds are operated individually. Specify the list of strings used by the device. Any combination of led strings can be used. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array + $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 maxItems: 4 @@ -150,10 +143,9 @@ properties: 0 - Modulator A 1 - Modulator B This property is applicable only to WLED5 peripheral. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 0, 1 ] - - default: 0 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 0, 1 ] + default: 0 qcom,cabc-sel: description: | @@ -164,9 +156,8 @@ properties: 2 - CABC 2 3 - External signal (e.g. LPG) is used for dimming This property is applicable only to WLED5 peripheral. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 0, 1, 2, 3 ] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 0, 1, 2, 3 ] allOf: - if: diff --git a/Documentation/devicetree/bindings/leds/leds-aw2013.yaml b/Documentation/devicetree/bindings/leds/leds-aw2013.yaml index f118721df1e8..e24b0d15ef01 100644 --- a/Documentation/devicetree/bindings/leds/leds-aw2013.yaml +++ b/Documentation/devicetree/bindings/leds/leds-aw2013.yaml @@ -32,8 +32,7 @@ properties: patternProperties: "^led@[0-2]$": type: object - allOf: - - $ref: common.yaml# + $ref: common.yaml# properties: reg: diff --git a/Documentation/devicetree/bindings/leds/leds-sgm3140.yaml b/Documentation/devicetree/bindings/leds/leds-sgm3140.yaml index ecf7ac9ab067..f68259619488 100644 --- a/Documentation/devicetree/bindings/leds/leds-sgm3140.yaml +++ b/Documentation/devicetree/bindings/leds/leds-sgm3140.yaml @@ -33,8 +33,7 @@ properties: led: type: object - allOf: - - $ref: common.yaml# + $ref: common.yaml# required: - compatible diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt deleted file mode 100644 index beec612dbe6a..000000000000 --- a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.txt +++ /dev/null @@ -1,88 +0,0 @@ -Binding for the Qualcomm APCS global block -========================================== - -This binding describes the APCS "global" block found in various Qualcomm -platforms. - -- compatible: - Usage: required - Value type: <string> - Definition: must be one of: - "qcom,msm8916-apcs-kpss-global", - "qcom,msm8996-apcs-hmss-global" - "qcom,msm8998-apcs-hmss-global" - "qcom,qcs404-apcs-apps-global" - "qcom,sc7180-apss-shared" - "qcom,sdm845-apss-shared" - "qcom,sm8150-apss-shared" - "qcom,ipq8074-apcs-apps-global" - -- reg: - Usage: required - Value type: <prop-encoded-array> - Definition: must specify the base address and size of the global block - -- clocks: - Usage: required if #clock-names property is present - Value type: <phandle array> - Definition: phandles to the two parent clocks of the clock driver. - -- #mbox-cells: - Usage: required - Value type: <u32> - Definition: as described in mailbox.txt, must be 1 - -- #clock-cells: - Usage: optional - Value type: <u32> - Definition: as described in clock.txt, must be 0 - -- clock-names: - Usage: required if the platform data based clock driver needs to - retrieve the parent clock names from device tree. - This will requires two mandatory clocks to be defined. - Value type: <string-array> - Definition: must be "pll" and "aux" - -= EXAMPLE -The following example describes the APCS HMSS found in MSM8996 and part of the -GLINK RPM referencing the "rpm_hlos" doorbell therein. - - apcs_glb: mailbox@9820000 { - compatible = "qcom,msm8996-apcs-hmss-global"; - reg = <0x9820000 0x1000>; - - #mbox-cells = <1>; - }; - - rpm-glink { - compatible = "qcom,glink-rpm"; - - interrupts = <GIC_SPI 168 IRQ_TYPE_EDGE_RISING>; - - qcom,rpm-msg-ram = <&rpm_msg_ram>; - - mboxes = <&apcs_glb 0>; - mbox-names = "rpm_hlos"; - }; - -Below is another example of the APCS binding on MSM8916 platforms: - - apcs: mailbox@b011000 { - compatible = "qcom,msm8916-apcs-kpss-global"; - reg = <0xb011000 0x1000>; - #mbox-cells = <1>; - clocks = <&a53pll>; - #clock-cells = <0>; - }; - -Below is another example of the APCS binding on QCS404 platforms: - - apcs_glb: mailbox@b011000 { - compatible = "qcom,qcs404-apcs-apps-global", "syscon"; - reg = <0x0b011000 0x1000>; - #mbox-cells = <1>; - clocks = <&apcs_hfpll>, <&gcc GCC_GPLL0_AO_OUT_MAIN>; - clock-names = "pll", "aux"; - #clock-cells = <0>; - }; diff --git a/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml new file mode 100644 index 000000000000..12eff942708d --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/qcom,apcs-kpss-global.yaml @@ -0,0 +1,86 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/mailbox/qcom,apcs-kpss-global.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Qualcomm APCS global block bindings + +description: + This binding describes the APCS "global" block found in various Qualcomm + platforms. + +maintainers: + - Sivaprakash Murugesan <sivaprak@codeaurora.org> + +properties: + compatible: + enum: + - qcom,ipq8074-apcs-apps-global + - qcom,msm8916-apcs-kpss-global + - qcom,msm8996-apcs-hmss-global + - qcom,msm8998-apcs-hmss-global + - qcom,qcs404-apcs-apps-global + - qcom,sc7180-apss-shared + - qcom,sdm845-apss-shared + - qcom,sm8150-apss-shared + + reg: + maxItems: 1 + + clocks: + description: phandles to the parent clocks of the clock driver + items: + - description: primary pll parent of the clock driver + - description: auxiliary parent + + '#mbox-cells': + const: 1 + + '#clock-cells': + const: 0 + + clock-names: + items: + - const: pll + - const: aux + +required: + - compatible + - reg + - '#mbox-cells' + +additionalProperties: false + +examples: + + # Example apcs with msm8996 + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + apcs_glb: mailbox@9820000 { + compatible = "qcom,msm8996-apcs-hmss-global"; + reg = <0x9820000 0x1000>; + + #mbox-cells = <1>; + }; + + rpm-glink { + compatible = "qcom,glink-rpm"; + interrupts = <GIC_SPI 168 IRQ_TYPE_EDGE_RISING>; + qcom,rpm-msg-ram = <&rpm_msg_ram>; + mboxes = <&apcs_glb 0>; + mbox-names = "rpm_hlos"; + }; + + # Example apcs with qcs404 + - | + #define GCC_APSS_AHB_CLK_SRC 1 + #define GCC_GPLL0_AO_OUT_MAIN 123 + apcs: mailbox@b011000 { + compatible = "qcom,qcs404-apcs-apps-global"; + reg = <0x0b011000 0x1000>; + #mbox-cells = <1>; + clocks = <&apcs_hfpll>, <&gcc GCC_GPLL0_AO_OUT_MAIN>; + clock-names = "pll", "aux"; + #clock-cells = <0>; + }; diff --git a/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml new file mode 100644 index 000000000000..4ac2123d9193 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/mailbox/qcom-ipcc.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Qualcomm Technologies, Inc. Inter-Processor Communication Controller + +maintainers: + - Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> + +description: + The Inter-Processor Communication Controller (IPCC) is a centralized hardware + to route interrupts across various subsystems. It involves a three-level + addressing scheme called protocol, client and signal. For example, consider an + entity on the Application Processor Subsystem (APSS) that wants to listen to + Modem's interrupts via Shared Memory Point to Point (SMP2P) interface. In such + a case, the client would be Modem (client-id is 2) and the signal would be + SMP2P (signal-id is 2). The SMP2P itself falls under the Multiprocessor (MPROC) + protocol (protocol-id is 0). Refer include/dt-bindings/mailbox/qcom-ipcc.h + for the list of such IDs. + +properties: + compatible: + items: + - enum: + - qcom,sm8250-ipcc + - const: qcom,ipcc + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + interrupt-controller: true + + "#interrupt-cells": + const: 3 + description: + The first cell is the client-id, the second cell is the signal-id and the + third cell is the interrupt type. + + "#mbox-cells": + const: 2 + description: + The first cell is the client-id, and the second cell is the signal-id. + +required: + - compatible + - reg + - interrupts + - interrupt-controller + - "#interrupt-cells" + - "#mbox-cells" + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/mailbox/qcom-ipcc.h> + + mailbox@408000 { + compatible = "qcom,sm8250-ipcc", "qcom,ipcc"; + reg = <0x408000 0x1000>; + interrupts = <GIC_SPI 229 IRQ_TYPE_LEVEL_HIGH>; + interrupt-controller; + #interrupt-cells = <3>; + #mbox-cells = <2>; + }; + + smp2p-modem { + compatible = "qcom,smp2p"; + interrupts-extended = <&ipcc_mproc IPCC_CLIENT_MPSS + IPCC_MPROC_SIGNAL_SMP2P IRQ_TYPE_EDGE_RISING>; + mboxes = <&ipcc_mproc IPCC_CLIENT_MPSS IPCC_MPROC_SIGNAL_SMP2P>; + + /* Other SMP2P fields */ + }; diff --git a/Documentation/devicetree/bindings/mailbox/sprd-mailbox.yaml b/Documentation/devicetree/bindings/mailbox/sprd-mailbox.yaml new file mode 100644 index 000000000000..26a5cca3f838 --- /dev/null +++ b/Documentation/devicetree/bindings/mailbox/sprd-mailbox.yaml @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: "http://devicetree.org/schemas/mailbox/sprd-mailbox.yaml#" +$schema: "http://devicetree.org/meta-schemas/core.yaml#" + +title: Spreadtrum mailbox controller bindings + +maintainers: + - Orson Zhai <orsonzhai@gmail.com> + - Baolin Wang <baolin.wang7@gmail.com> + - Chunyan Zhang <zhang.lyra@gmail.com> + +properties: + compatible: + enum: + - sprd,sc9860-mailbox + + reg: + items: + - description: inbox registers' base address + - description: outbox registers' base address + + interrupts: + items: + - description: inbox interrupt + - description: outbox interrupt + + clocks: + maxItems: 1 + + clock-names: + items: + - const: enable + + "#mbox-cells": + const: 1 + +required: + - compatible + - reg + - interrupts + - "#mbox-cells" + - clocks + - clock-names + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + mailbox: mailbox@400a0000 { + compatible = "sprd,sc9860-mailbox"; + reg = <0x400a0000 0x8000>, <0x400a8000 0x8000>; + #mbox-cells = <1>; + clock-names = "enable"; + clocks = <&aon_gate 53>; + interrupts = <GIC_SPI 28 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 29 IRQ_TYPE_LEVEL_HIGH>; + }; +... diff --git a/Documentation/devicetree/bindings/media/i2c/ov8856.yaml b/Documentation/devicetree/bindings/media/i2c/ov8856.yaml index d6af685ad3ca..1956b2a32bf4 100644 --- a/Documentation/devicetree/bindings/media/i2c/ov8856.yaml +++ b/Documentation/devicetree/bindings/media/i2c/ov8856.yaml @@ -79,8 +79,7 @@ properties: - const: 4 link-frequencies: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint64-array + $ref: /schemas/types.yaml#/definitions/uint64-array description: Allowed data bus frequencies. 360000000, 180000000 Hz or both are supported by the driver. diff --git a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml index 0c68cdad9a31..8d35c327018b 100644 --- a/Documentation/devicetree/bindings/media/rockchip,vdec.yaml +++ b/Documentation/devicetree/bindings/media/rockchip,vdec.yaml @@ -61,7 +61,7 @@ examples: vdec: video-codec@ff660000 { compatible = "rockchip,rk3399-vdec"; - reg = <0x0 0xff660000 0x0 0x400>; + reg = <0xff660000 0x400>; interrupts = <GIC_SPI 116 IRQ_TYPE_LEVEL_HIGH 0>; clocks = <&cru ACLK_VDU>, <&cru HCLK_VDU>, <&cru SCLK_VDU_CA>, <&cru SCLK_VDU_CORE>; diff --git a/Documentation/devicetree/bindings/media/rockchip-vpu.yaml b/Documentation/devicetree/bindings/media/rockchip-vpu.yaml index 27df18ad6a81..2b629456d75f 100644 --- a/Documentation/devicetree/bindings/media/rockchip-vpu.yaml +++ b/Documentation/devicetree/bindings/media/rockchip-vpu.yaml @@ -66,7 +66,7 @@ examples: vpu: video-codec@ff9a0000 { compatible = "rockchip,rk3288-vpu"; - reg = <0x0 0xff9a0000 0x0 0x800>; + reg = <0xff9a0000 0x800>; interrupts = <GIC_SPI 9 IRQ_TYPE_LEVEL_HIGH>, <GIC_SPI 10 IRQ_TYPE_LEVEL_HIGH>; interrupt-names = "vepu", "vdpu"; diff --git a/Documentation/devicetree/bindings/memory-controllers/ingenic,nemc.yaml b/Documentation/devicetree/bindings/memory-controllers/ingenic,nemc.yaml index 9b478da0c479..17ba45a6c260 100644 --- a/Documentation/devicetree/bindings/memory-controllers/ingenic,nemc.yaml +++ b/Documentation/devicetree/bindings/memory-controllers/ingenic,nemc.yaml @@ -45,9 +45,8 @@ patternProperties: maxItems: 255 ingenic,nemc-bus-width: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [8, 16] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [8, 16] description: Specifies the bus width in bits. ingenic,nemc-tAS: diff --git a/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml b/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml index db8f115a13ec..cb9794edff24 100644 --- a/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml +++ b/Documentation/devicetree/bindings/mtd/arasan,nand-controller.yaml @@ -53,7 +53,7 @@ examples: - | nfc: nand-controller@ff100000 { compatible = "xlnx,zynqmp-nand-controller", "arasan,nfc-v3p10"; - reg = <0x0 0xff100000 0x0 0x1000>; + reg = <0xff100000 0x1000>; clock-names = "controller", "bus"; clocks = <&clk200>, <&clk100>; interrupt-parent = <&gic>; diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml index 71d9e6c1c72e..174579370a22 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpsw-nuss.yaml @@ -145,8 +145,7 @@ patternProperties: "^cpts@[0-9a-f]+": type: object - allOf: - - $ref: "ti,k3-am654-cpts.yaml#" + $ref: "ti,k3-am654-cpts.yaml#" description: CPSW Common Platform Time Sync (CPTS) module. diff --git a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml index 50e027911dd4..9b7117920d90 100644 --- a/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml +++ b/Documentation/devicetree/bindings/net/ti,k3-am654-cpts.yaml @@ -74,15 +74,13 @@ properties: - const: cpts ti,cpts-ext-ts-inputs: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 maximum: 8 description: Number of hardware timestamp push inputs (HWx_TS_PUSH) ti,cpts-periodic-outputs: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 maximum: 8 description: Number of timestamp Generator function outputs (TS_GENFx) @@ -123,7 +121,7 @@ examples: cpts@310d0000 { compatible = "ti,am65-cpts"; - reg = <0x0 0x310d0000 0x0 0x400>; + reg = <0x310d0000 0x400>; reg-names = "cpts"; clocks = <&main_cpts_mux>; clock-names = "cpts"; diff --git a/Documentation/devicetree/bindings/pci/cdns-pcie-ep.yaml b/Documentation/devicetree/bindings/pci/cdns-pcie-ep.yaml index 6150a7a7bdbf..016a5f61592d 100644 --- a/Documentation/devicetree/bindings/pci/cdns-pcie-ep.yaml +++ b/Documentation/devicetree/bindings/pci/cdns-pcie-ep.yaml @@ -15,8 +15,7 @@ allOf: properties: cdns,max-outbound-regions: description: maximum number of outbound regions - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 minimum: 1 maximum: 32 default: 32 diff --git a/Documentation/devicetree/bindings/pci/cdns-pcie-host.yaml b/Documentation/devicetree/bindings/pci/cdns-pcie-host.yaml index c87a3a36ccd2..303078a7b7a8 100644 --- a/Documentation/devicetree/bindings/pci/cdns-pcie-host.yaml +++ b/Documentation/devicetree/bindings/pci/cdns-pcie-host.yaml @@ -16,8 +16,7 @@ allOf: properties: cdns,max-outbound-regions: description: maximum number of outbound regions - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 minimum: 1 maximum: 32 default: 32 diff --git a/Documentation/devicetree/bindings/phy/calxeda-combophy.yaml b/Documentation/devicetree/bindings/phy/calxeda-combophy.yaml index 16a8bd7644bf..41ee16e21f8d 100644 --- a/Documentation/devicetree/bindings/phy/calxeda-combophy.yaml +++ b/Documentation/devicetree/bindings/phy/calxeda-combophy.yaml @@ -29,9 +29,8 @@ properties: phydev: description: device ID for programming the ComboPHY. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - maximum: 31 + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 31 required: - compatible diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml index 973b2d196f46..f80f8896d527 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-phy.yaml @@ -44,6 +44,8 @@ properties: "#size-cells": enum: [ 1, 2 ] + ranges: true + clocks: minItems: 1 maxItems: 4 @@ -87,6 +89,7 @@ required: - "#clock-cells" - "#address-cells" - "#size-cells" + - ranges - clocks - clock-names - resets @@ -281,10 +284,11 @@ examples: #include <dt-bindings/clock/qcom,gcc-sdm845.h> usb_2_qmpphy: phy-wrapper@88eb000 { compatible = "qcom,sdm845-qmp-usb3-uni-phy"; - reg = <0 0x088eb000 0 0x18c>; + reg = <0x088eb000 0x18c>; #clock-cells = <1>; - #address-cells = <2>; - #size-cells = <2>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x088eb000 0x2000>; clocks = <&gcc GCC_USB3_SEC_PHY_AUX_CLK >, <&gcc GCC_USB_PHY_CFG_AHB2PHY_CLK>, @@ -299,11 +303,11 @@ examples: vdda-phy-supply = <&vdda_usb2_ss_1p2>; vdda-pll-supply = <&vdda_usb2_ss_core>; - usb_2_ssphy: phy@88eb200 { - reg = <0 0x088eb200 0 0x128>, - <0 0x088eb400 0 0x1fc>, - <0 0x088eb800 0 0x218>, - <0 0x088eb600 0 0x70>; + usb_2_ssphy: phy@200 { + reg = <0x200 0x128>, + <0x400 0x1fc>, + <0x800 0x218>, + <0x600 0x70>; #clock-cells = <0>; #phy-cells = <0>; clocks = <&gcc GCC_USB3_SEC_PHY_PIPE_CLK>; diff --git a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml index b770e637df1d..6e2487501457 100644 --- a/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,qmp-usb3-dp-phy.yaml @@ -34,6 +34,8 @@ properties: "#size-cells": enum: [ 1, 2 ] + ranges: true + clocks: items: - description: Phy aux clock. @@ -86,6 +88,7 @@ required: - "#clock-cells" - "#address-cells" - "#size-cells" + - ranges - clocks - clock-names - resets @@ -100,12 +103,13 @@ examples: #include <dt-bindings/clock/qcom,gcc-sdm845.h> usb_1_qmpphy: phy-wrapper@88e9000 { compatible = "qcom,sdm845-qmp-usb3-phy"; - reg = <0 0x088e9000 0 0x18c>, - <0 0x088e8000 0 0x10>; + reg = <0x088e9000 0x18c>, + <0x088e8000 0x10>; reg-names = "reg-base", "dp_com"; #clock-cells = <1>; - #address-cells = <2>; - #size-cells = <2>; + #address-cells = <1>; + #size-cells = <1>; + ranges = <0x0 0x088e9000 0x1000>; clocks = <&gcc GCC_USB3_PRIM_PHY_AUX_CLK>, <&gcc GCC_USB_PHY_CFG_AHB2PHY_CLK>, @@ -120,17 +124,17 @@ examples: vdda-phy-supply = <&vdda_usb2_ss_1p2>; vdda-pll-supply = <&vdda_usb2_ss_core>; - usb_1_ssphy: phy@88e9200 { - reg = <0 0x088e9200 0 0x128>, - <0 0x088e9400 0 0x200>, - <0 0x088e9c00 0 0x218>, - <0 0x088e9600 0 0x128>, - <0 0x088e9800 0 0x200>, - <0 0x088e9a00 0 0x100>; - #clock-cells = <0>; - #phy-cells = <0>; - clocks = <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>; - clock-names = "pipe0"; - clock-output-names = "usb3_phy_pipe_clk_src"; - }; + phy@200 { + reg = <0x200 0x128>, + <0x400 0x200>, + <0xc00 0x218>, + <0x600 0x128>, + <0x800 0x200>, + <0xa00 0x100>; + #clock-cells = <0>; + #phy-cells = <0>; + clocks = <&gcc GCC_USB3_PRIM_PHY_PIPE_CLK>; + clock-names = "pipe0"; + clock-output-names = "usb3_phy_pipe_clk_src"; }; + }; diff --git a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml index 574f890fab1d..4949a2851532 100644 --- a/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml +++ b/Documentation/devicetree/bindings/phy/qcom,usb-snps-femto-v2.yaml @@ -65,7 +65,7 @@ examples: #include <dt-bindings/clock/qcom,gcc-sm8150.h> phy@88e2000 { compatible = "qcom,sm8150-usb-hs-phy"; - reg = <0 0x088e2000 0 0x400>; + reg = <0x088e2000 0x400>; #phy-cells = <0>; clocks = <&rpmhcc RPMH_CXO_CLK>; diff --git a/Documentation/devicetree/bindings/phy/renesas,usb3-phy.yaml b/Documentation/devicetree/bindings/phy/renesas,usb3-phy.yaml index f459eaf55278..68cf9dd0390d 100644 --- a/Documentation/devicetree/bindings/phy/renesas,usb3-phy.yaml +++ b/Documentation/devicetree/bindings/phy/renesas,usb3-phy.yaml @@ -52,9 +52,8 @@ properties: description: | Enable/disable spread spectrum clock (ssc). 0 or the property doesn't exist means disabling the ssc. The actual value will be -<value> ppm. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 0, 4003, 4492, 4980 ] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 0, 4003, 4492, 4980 ] required: - compatible diff --git a/Documentation/devicetree/bindings/power/supply/cw2015_battery.yaml b/Documentation/devicetree/bindings/power/supply/cw2015_battery.yaml index 4a265d4234b9..2036977ecc2f 100644 --- a/Documentation/devicetree/bindings/power/supply/cw2015_battery.yaml +++ b/Documentation/devicetree/bindings/power/supply/cw2015_battery.yaml @@ -27,11 +27,9 @@ properties: of this binary blob is kept secret by CellWise. The only way to obtain it is to mail two batteries to a test facility of CellWise and receive back a test report with the binary blob. - allOf: - - $ref: /schemas/types.yaml#definitions/uint8-array - items: - - minItems: 64 - maxItems: 64 + $ref: /schemas/types.yaml#definitions/uint8-array + minItems: 64 + maxItems: 64 cellwise,monitor-interval-ms: description: @@ -41,10 +39,9 @@ properties: power-supplies: description: Specifies supplies used for charging the battery connected to this gauge - allOf: - - $ref: /schemas/types.yaml#/definitions/phandle-array - - minItems: 1 - maxItems: 8 # Should be enough + $ref: /schemas/types.yaml#/definitions/phandle-array + minItems: 1 + maxItems: 8 # Should be enough monitored-battery: description: diff --git a/Documentation/devicetree/bindings/power/supply/sbs,sbs-battery.yaml b/Documentation/devicetree/bindings/power/supply/sbs,sbs-battery.yaml index 205bc826bd20..a90b3601e695 100644 --- a/Documentation/devicetree/bindings/power/supply/sbs,sbs-battery.yaml +++ b/Documentation/devicetree/bindings/power/supply/sbs,sbs-battery.yaml @@ -32,16 +32,14 @@ properties: description: The number of times to retry I2C transactions on I2C IO failure. default: 0 - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 sbs,poll-retry-count: description: The number of times to try looking for new status after an external change notification. default: 0 - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 sbs,battery-detect-gpios: description: diff --git a/Documentation/devicetree/bindings/pwm/imx-pwm.yaml b/Documentation/devicetree/bindings/pwm/imx-pwm.yaml index 4b62af27d4b3..01df06777cba 100644 --- a/Documentation/devicetree/bindings/pwm/imx-pwm.yaml +++ b/Documentation/devicetree/bindings/pwm/imx-pwm.yaml @@ -30,13 +30,11 @@ properties: items: - description: SoC PWM ipg clock - description: SoC PWM per clock - maxItems: 2 clock-names: items: - const: ipg - const: per - maxItems: 2 interrupts: maxItems: 1 diff --git a/Documentation/devicetree/bindings/pwm/renesas,pwm-rcar.yaml b/Documentation/devicetree/bindings/pwm/renesas,pwm-rcar.yaml index 461afb4c1f5d..daadde9ff9c4 100644 --- a/Documentation/devicetree/bindings/pwm/renesas,pwm-rcar.yaml +++ b/Documentation/devicetree/bindings/pwm/renesas,pwm-rcar.yaml @@ -27,6 +27,7 @@ properties: - renesas,pwm-r8a7794 # R-Car E2 - renesas,pwm-r8a7795 # R-Car H3 - renesas,pwm-r8a7796 # R-Car M3-W + - renesas,pwm-r8a77961 # R-Car M3-W+ - renesas,pwm-r8a77965 # R-Car M3-N - renesas,pwm-r8a77970 # R-Car V3M - renesas,pwm-r8a77980 # R-Car V3H diff --git a/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml b/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml index 19cbd5eb2897..78c0b63243f7 100644 --- a/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml +++ b/Documentation/devicetree/bindings/regulator/maxim,max77826.yaml @@ -21,8 +21,7 @@ properties: regulators: type: object - allOf: - - $ref: regulator.yaml# + $ref: regulator.yaml# description: | list of regulators provided by this controller, must be named after their hardware counterparts LDO[1-15], BUCK and BUCKBOOST @@ -30,13 +29,11 @@ properties: patternProperties: "^LDO([1-9]|1[0-5])$": type: object - allOf: - - $ref: regulator.yaml# + $ref: regulator.yaml# "^BUCK|BUCKBOOST$": type: object - allOf: - - $ref: regulator.yaml# + $ref: regulator.yaml# additionalProperties: false diff --git a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml index d797cc23406f..07256a4b50b9 100644 --- a/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml +++ b/Documentation/devicetree/bindings/regulator/rohm,bd71847-regulator.yaml @@ -29,8 +29,7 @@ description: | patternProperties: "^LDO[1-6]$": type: object - allOf: - - $ref: regulator.yaml# + $ref: regulator.yaml# description: Properties for single LDO regulator. @@ -44,8 +43,7 @@ patternProperties: "^BUCK[1-6]$": type: object - allOf: - - $ref: regulator.yaml# + $ref: regulator.yaml# description: Properties for single BUCK regulator. @@ -56,28 +54,25 @@ patternProperties: should be "buck1", ..., "buck6" rohm,dvs-run-voltage: - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 0 - maximum: 1300000 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 0 + maximum: 1300000 description: PMIC default "RUN" state voltage in uV. See below table for bucks which support this. 0 means disabled. rohm,dvs-idle-voltage: - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 0 - maximum: 1300000 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 0 + maximum: 1300000 description: PMIC default "IDLE" state voltage in uV. See below table for bucks which support this. 0 means disabled. rohm,dvs-suspend-voltage: - allOf: - - $ref: "/schemas/types.yaml#/definitions/uint32" - - minimum: 0 - maximum: 1300000 + $ref: "/schemas/types.yaml#/definitions/uint32" + minimum: 0 + maximum: 1300000 description: PMIC default "SUSPEND" state voltage in uV. See below table for bucks which support this. 0 means disabled. diff --git a/Documentation/devicetree/bindings/rng/arm-cctrng.yaml b/Documentation/devicetree/bindings/rng/arm-cctrng.yaml index ca6aad19b6ba..c471e4c10558 100644 --- a/Documentation/devicetree/bindings/rng/arm-cctrng.yaml +++ b/Documentation/devicetree/bindings/rng/arm-cctrng.yaml @@ -28,10 +28,8 @@ properties: description: Arm TrustZone CryptoCell TRNG engine has 4 ring oscillators. Sampling ratio values for these 4 ring oscillators. (from calibration) - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - items: - maxItems: 4 + $ref: /schemas/types.yaml#/definitions/uint32-array + maxItems: 4 clocks: maxItems: 1 diff --git a/Documentation/devicetree/bindings/serial/samsung_uart.yaml b/Documentation/devicetree/bindings/serial/samsung_uart.yaml index 32a5e1e30833..96414ac65d06 100644 --- a/Documentation/devicetree/bindings/serial/samsung_uart.yaml +++ b/Documentation/devicetree/bindings/serial/samsung_uart.yaml @@ -33,9 +33,8 @@ properties: description: | The size (in bytes) of the IO accesses that should be performed on the device. - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [ 1, 4 ] + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [ 1, 4 ] clocks: minItems: 2 diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.yaml b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.yaml index dee8bb2b69fe..a2b29cc3e93b 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.yaml +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,geni-se.yaml @@ -100,8 +100,7 @@ patternProperties: supports up to 50MHz, up to four chip selects, programmable data path from 4 bits to 32 bits and numerous protocol variants. - allOf: - - $ref: /spi/spi-controller.yaml# + $ref: /spi/spi-controller.yaml# properties: compatible: @@ -126,8 +125,7 @@ patternProperties: "i2c@[0-9a-f]+$": type: object description: GENI serial engine based I2C controller. - allOf: - - $ref: /schemas/i2c/i2c-controller.yaml# + $ref: /schemas/i2c/i2c-controller.yaml# properties: compatible: @@ -156,8 +154,7 @@ patternProperties: "serial@[0-9a-f]+$": type: object description: GENI Serial Engine based UART Controller. - allOf: - - $ref: /schemas/serial.yaml# + $ref: /schemas/serial.yaml# properties: compatible: diff --git a/Documentation/devicetree/bindings/sound/fsl,easrc.yaml b/Documentation/devicetree/bindings/sound/fsl,easrc.yaml index 73cdcf053a9c..32d547af9ce7 100644 --- a/Documentation/devicetree/bindings/sound/fsl,easrc.yaml +++ b/Documentation/devicetree/bindings/sound/fsl,easrc.yaml @@ -45,23 +45,20 @@ properties: - const: ctx3_tx firmware-name: - allOf: - - $ref: /schemas/types.yaml#/definitions/string - - const: imx/easrc/easrc-imx8mn.bin + $ref: /schemas/types.yaml#/definitions/string + const: imx/easrc/easrc-imx8mn.bin description: The coefficient table for the filters fsl,asrc-rate: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - minimum: 8000 - - maximum: 192000 + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 8000 + maximum: 192000 description: Defines a mutual sample rate used by DPCM Back Ends fsl,asrc-format: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - enum: [2, 6, 10, 32, 36] - default: 2 + $ref: /schemas/types.yaml#/definitions/uint32 + enum: [2, 6, 10, 32, 36] + default: 2 description: Defines a mutual sample format used by DPCM Back Ends @@ -83,7 +80,7 @@ examples: easrc: easrc@300c0000 { compatible = "fsl,imx8mn-easrc"; - reg = <0x0 0x300c0000 0x0 0x10000>; + reg = <0x300c0000 0x10000>; interrupts = <0x0 122 0x4>; clocks = <&clk IMX8MN_CLK_ASRC_ROOT>; clock-names = "mem"; diff --git a/Documentation/devicetree/bindings/sound/simple-card.yaml b/Documentation/devicetree/bindings/sound/simple-card.yaml index cb2bb5fac0e1..8132d0c0f00a 100644 --- a/Documentation/devicetree/bindings/sound/simple-card.yaml +++ b/Documentation/devicetree/bindings/sound/simple-card.yaml @@ -13,15 +13,13 @@ definitions: frame-master: description: Indicates dai-link frame master. - allOf: - - $ref: /schemas/types.yaml#/definitions/phandle-array - - maxItems: 1 + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 bitclock-master: description: Indicates dai-link bit clock master - allOf: - - $ref: /schemas/types.yaml#/definitions/phandle-array - - maxItems: 1 + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 frame-inversion: description: dai-link uses frame clock inversion diff --git a/Documentation/devicetree/bindings/sound/tdm-slot.txt b/Documentation/devicetree/bindings/sound/tdm-slot.txt index 34cf70e2cbc4..4bb513ae62fc 100644 --- a/Documentation/devicetree/bindings/sound/tdm-slot.txt +++ b/Documentation/devicetree/bindings/sound/tdm-slot.txt @@ -14,8 +14,8 @@ For instance: dai-tdm-slot-tx-mask = <0 1>; dai-tdm-slot-rx-mask = <1 0>; -And for each spcified driver, there could be one .of_xlate_tdm_slot_mask() -to specify a explicit mapping of the channels and the slots. If it's absent +And for each specified driver, there could be one .of_xlate_tdm_slot_mask() +to specify an explicit mapping of the channels and the slots. If it's absent the default snd_soc_of_xlate_tdm_slot_mask() will be used to generating the tx and rx masks. diff --git a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml index c5b5b4260496..2e6ac5d2ee96 100644 --- a/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml +++ b/Documentation/devicetree/bindings/sound/tlv320adcx140.yaml @@ -76,13 +76,12 @@ properties: PDMIN3 - PDMCLK latching edge used for channel 5 and 6 data PDMIN4 - PDMCLK latching edge used for channel 7 and 8 data - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 4 - items: - maximum: 1 - default: [0, 0, 0, 0] + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + items: + maximum: 1 + default: [0, 0, 0, 0] ti,gpi-config: description: | @@ -102,13 +101,12 @@ properties: 7 - GPIX is configured as a PDM data input for channel 7 and channel (PDMDIN4) - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - minItems: 1 - maxItems: 4 - items: - maximum: 7 - default: [0, 0, 0, 0] + $ref: /schemas/types.yaml#/definitions/uint32-array + minItems: 1 + maxItems: 4 + items: + maximum: 7 + default: [0, 0, 0, 0] required: - compatible diff --git a/Documentation/devicetree/bindings/thermal/socionext,uniphier-thermal.yaml b/Documentation/devicetree/bindings/thermal/socionext,uniphier-thermal.yaml index bb9594bb2cf1..553c9dcdaeeb 100644 --- a/Documentation/devicetree/bindings/thermal/socionext,uniphier-thermal.yaml +++ b/Documentation/devicetree/bindings/thermal/socionext,uniphier-thermal.yaml @@ -28,9 +28,8 @@ properties: const: 0 socionext,tmod-calibration: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32-array - - maxItems: 2 + $ref: /schemas/types.yaml#/definitions/uint32-array + maxItems: 2 description: A pair of calibrated values referred from PVT, in case that the values aren't set on SoC, like a reference board. diff --git a/Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml b/Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml new file mode 100644 index 000000000000..5145883d932e --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/thermal-cooling-devices.yaml @@ -0,0 +1,116 @@ +# SPDX-License-Identifier: (GPL-2.0) +# Copyright 2020 Linaro Ltd. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/thermal-cooling-devices.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Thermal cooling device binding + +maintainers: + - Amit Kucheria <amitk@kernel.org> + +description: | + Thermal management is achieved in devicetree by describing the sensor hardware + and the software abstraction of cooling devices and thermal zones required to + take appropriate action to mitigate thermal overload. + + The following node types are used to completely describe a thermal management + system in devicetree: + - thermal-sensor: device that measures temperature, has SoC-specific bindings + - cooling-device: device used to dissipate heat either passively or actively + - thermal-zones: a container of the following node types used to describe all + thermal data for the platform + + This binding describes the cooling devices. + + There are essentially two ways to provide control on power dissipation: + - Passive cooling: by means of regulating device performance. A typical + passive cooling mechanism is a CPU that has dynamic voltage and frequency + scaling (DVFS), and uses lower frequencies as cooling states. + - Active cooling: by means of activating devices in order to remove the + dissipated heat, e.g. regulating fan speeds. + + Any cooling device has a range of cooling states (i.e. different levels of + heat dissipation). They also have a way to determine the state of cooling in + which the device is. For example, a fan's cooling states correspond to the + different fan speeds possible. Cooling states are referred to by single + unsigned integers, where larger numbers mean greater heat dissipation. The + precise set of cooling states associated with a device should be defined in + a particular device's binding. + +select: true + +properties: + "#cooling-cells": + description: + Must be 2, in order to specify minimum and maximum cooling state used in + the cooling-maps reference. The first cell is the minimum cooling state + and the second cell is the maximum cooling state requested. + const: 2 + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/thermal/thermal.h> + + // Example 1: Cpufreq cooling device on CPU0 + cpus { + #address-cells = <2>; + #size-cells = <0>; + + CPU0: cpu@0 { + device_type = "cpu"; + compatible = "qcom,kryo385"; + reg = <0x0 0x0>; + enable-method = "psci"; + cpu-idle-states = <&LITTLE_CPU_SLEEP_0 + &LITTLE_CPU_SLEEP_1 + &CLUSTER_SLEEP_0>; + capacity-dmips-mhz = <607>; + dynamic-power-coefficient = <100>; + qcom,freq-domain = <&cpufreq_hw 0>; + #cooling-cells = <2>; + next-level-cache = <&L2_0>; + L2_0: l2-cache { + compatible = "cache"; + next-level-cache = <&L3_0>; + L3_0: l3-cache { + compatible = "cache"; + }; + }; + }; + + /* ... */ + + }; + + /* ... */ + + thermal-zones { + cpu0-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + + thermal-sensors = <&tsens0 1>; + + trips { + cpu0_alert0: trip-point0 { + temperature = <90000>; + hysteresis = <2000>; + type = "passive"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu0_alert0>; + /* Corresponds to 1000MHz in OPP table */ + cooling-device = <&CPU0 5 5>; + }; + }; + }; + + /* ... */ + }; +... diff --git a/Documentation/devicetree/bindings/thermal/thermal-idle.yaml b/Documentation/devicetree/bindings/thermal/thermal-idle.yaml new file mode 100644 index 000000000000..7a922f540934 --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/thermal-idle.yaml @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +# Copyright 2020 Linaro Ltd. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/thermal-idle.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Thermal idle cooling device binding + +maintainers: + - Daniel Lezcano <daniel.lezcano@linaro.org> + +description: | + The thermal idle cooling device allows the system to passively + mitigate the temperature on the device by injecting idle cycles, + forcing it to cool down. + + This binding describes the thermal idle node. + +properties: + $nodename: + const: thermal-idle + description: | + A thermal-idle node describes the idle cooling device properties to + cool down efficiently the attached thermal zone. + + '#cooling-cells': + const: 2 + description: | + Must be 2, in order to specify minimum and maximum cooling state used in + the cooling-maps reference. The first cell is the minimum cooling state + and the second cell is the maximum cooling state requested. + + duration-us: + description: | + The idle duration in microsecond the device should cool down. + + exit-latency-us: + description: | + The exit latency constraint in microsecond for the injected + idle state for the device. It is the latency constraint to + apply when selecting an idle state from among all the present + ones. + +required: + - '#cooling-cells' + +examples: + - | + #include <dt-bindings/thermal/thermal.h> + + // Example: Combining idle cooling device on big CPUs with cpufreq cooling device + cpus { + #address-cells = <2>; + #size-cells = <0>; + + /* ... */ + + cpu_b0: cpu@100 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + reg = <0x0 0x100>; + enable-method = "psci"; + capacity-dmips-mhz = <1024>; + dynamic-power-coefficient = <436>; + #cooling-cells = <2>; /* min followed by max */ + cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + thermal-idle { + #cooling-cells = <2>; + duration-us = <10000>; + exit-latency-us = <500>; + }; + }; + + cpu_b1: cpu@101 { + device_type = "cpu"; + compatible = "arm,cortex-a72"; + reg = <0x0 0x101>; + enable-method = "psci"; + capacity-dmips-mhz = <1024>; + dynamic-power-coefficient = <436>; + #cooling-cells = <2>; /* min followed by max */ + cpu-idle-states = <&CPU_SLEEP &CLUSTER_SLEEP>; + thermal-idle { + #cooling-cells = <2>; + duration-us = <10000>; + exit-latency-us = <500>; + }; + }; + + /* ... */ + + }; + + /* ... */ + + thermal_zones { + cpu_thermal: cpu { + polling-delay-passive = <100>; + polling-delay = <1000>; + + /* ... */ + + trips { + cpu_alert0: cpu_alert0 { + temperature = <65000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_alert1: cpu_alert1 { + temperature = <70000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_alert2: cpu_alert2 { + temperature = <75000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu_crit: cpu_crit { + temperature = <95000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu_alert1>; + cooling-device = <&{/cpus/cpu@100/thermal-idle} 0 15 >, + <&{/cpus/cpu@101/thermal-idle} 0 15>; + }; + + map1 { + trip = <&cpu_alert2>; + cooling-device = + <&cpu_b0 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>, + <&cpu_b1 THERMAL_NO_LIMIT THERMAL_NO_LIMIT>; + }; + }; + }; + }; diff --git a/Documentation/devicetree/bindings/thermal/thermal-sensor.yaml b/Documentation/devicetree/bindings/thermal/thermal-sensor.yaml new file mode 100644 index 000000000000..fcd25a0af38c --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/thermal-sensor.yaml @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: (GPL-2.0) +# Copyright 2020 Linaro Ltd. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/thermal-sensor.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Thermal sensor binding + +maintainers: + - Amit Kucheria <amitk@kernel.org> + +description: | + Thermal management is achieved in devicetree by describing the sensor hardware + and the software abstraction of thermal zones required to take appropriate + action to mitigate thermal overloads. + + The following node types are used to completely describe a thermal management + system in devicetree: + - thermal-sensor: device that measures temperature, has SoC-specific bindings + - cooling-device: device used to dissipate heat either passively or actively + - thermal-zones: a container of the following node types used to describe all + thermal data for the platform + + This binding describes the thermal-sensor. + + Thermal sensor devices provide temperature sensing capabilities on thermal + zones. Typical devices are I2C ADC converters and bandgaps. Thermal sensor + devices may control one or more internal sensors. + +properties: + "#thermal-sensor-cells": + description: + Used to uniquely identify a thermal sensor instance within an IC. Will be + 0 on sensor nodes with only a single sensor and at least 1 on nodes + containing several internal sensors. + enum: [0, 1] + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + + // Example 1: SDM845 TSENS + soc: soc@0 { + #address-cells = <2>; + #size-cells = <2>; + + /* ... */ + + tsens0: thermal-sensor@c263000 { + compatible = "qcom,sdm845-tsens", "qcom,tsens-v2"; + reg = <0 0x0c263000 0 0x1ff>, /* TM */ + <0 0x0c222000 0 0x1ff>; /* SROT */ + #qcom,sensors = <13>; + interrupts = <GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "uplow", "critical"; + #thermal-sensor-cells = <1>; + }; + + tsens1: thermal-sensor@c265000 { + compatible = "qcom,sdm845-tsens", "qcom,tsens-v2"; + reg = <0 0x0c265000 0 0x1ff>, /* TM */ + <0 0x0c223000 0 0x1ff>; /* SROT */ + #qcom,sensors = <8>; + interrupts = <GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "uplow", "critical"; + #thermal-sensor-cells = <1>; + }; + }; +... diff --git a/Documentation/devicetree/bindings/thermal/thermal-zones.yaml b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml new file mode 100644 index 000000000000..b8515d3eeaa2 --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/thermal-zones.yaml @@ -0,0 +1,341 @@ +# SPDX-License-Identifier: (GPL-2.0) +# Copyright 2020 Linaro Ltd. +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/thermal-zones.yaml# +$schema: http://devicetree.org/meta-schemas/base.yaml# + +title: Thermal zone binding + +maintainers: + - Amit Kucheria <amitk@kernel.org> + +description: | + Thermal management is achieved in devicetree by describing the sensor hardware + and the software abstraction of cooling devices and thermal zones required to + take appropriate action to mitigate thermal overloads. + + The following node types are used to completely describe a thermal management + system in devicetree: + - thermal-sensor: device that measures temperature, has SoC-specific bindings + - cooling-device: device used to dissipate heat either passively or actively + - thermal-zones: a container of the following node types used to describe all + thermal data for the platform + + This binding describes the thermal-zones. + + The polling-delay properties of a thermal-zone are bound to the maximum dT/dt + (temperature derivative over time) in two situations for a thermal zone: + 1. when passive cooling is activated (polling-delay-passive) + 2. when the zone just needs to be monitored (polling-delay) or when + active cooling is activated. + + The maximum dT/dt is highly bound to hardware power consumption and + dissipation capability. The delays should be chosen to account for said + max dT/dt, such that a device does not cross several trip boundaries + unexpectedly between polls. Choosing the right polling delays shall avoid + having the device in temperature ranges that may damage the silicon structures + and reduce silicon lifetime. + +properties: + $nodename: + const: thermal-zones + description: + A /thermal-zones node is required in order to use the thermal framework to + manage input from the various thermal zones in the system in order to + mitigate thermal overload conditions. It does not represent a real device + in the system, but acts as a container to link a thermal sensor device, + platform-data regarding temperature thresholds and the mitigation actions + to take when the temperature crosses those thresholds. + +patternProperties: + "^[a-zA-Z][a-zA-Z0-9\\-]{1,12}-thermal$": + type: object + description: + Each thermal zone node contains information about how frequently it + must be checked, the sensor responsible for reporting temperature for + this zone, one sub-node containing the various trip points for this + zone and one sub-node containing all the zone cooling-maps. + + properties: + polling-delay: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + The maximum number of milliseconds to wait between polls when + checking this thermal zone. Setting this to 0 disables the polling + timers setup by the thermal framework and assumes that the thermal + sensors in this zone support interrupts. + + polling-delay-passive: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + The maximum number of milliseconds to wait between polls when + checking this thermal zone while doing passive cooling. Setting + this to 0 disables the polling timers setup by the thermal + framework and assumes that the thermal sensors in this zone + support interrupts. + + thermal-sensors: + $ref: /schemas/types.yaml#/definitions/phandle-array + maxItems: 1 + description: + The thermal sensor phandle and sensor specifier used to monitor this + thermal zone. + + coefficients: + $ref: /schemas/types.yaml#/definitions/uint32-array + description: + An array of integers containing the coefficients of a linear equation + that binds all the sensors listed in this thermal zone. + + The linear equation used is as follows, + z = c0 * x0 + c1 * x1 + ... + c(n-1) * x(n-1) + cn + where c0, c1, .., cn are the coefficients. + + Coefficients default to 1 in case this property is not specified. The + coefficients are ordered and are matched with sensors by means of the + sensor ID. Additional coefficients are interpreted as constant offset. + + sustainable-power: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + An estimate of the sustainable power (in mW) that this thermal zone + can dissipate at the desired control temperature. For reference, the + sustainable power of a 4-inch phone is typically 2000mW, while on a + 10-inch tablet is around 4500mW. + + trips: + type: object + description: + This node describes a set of points in the temperature domain at + which the thermal framework needs to take action. The actions to + be taken are defined in another node called cooling-maps. + + patternProperties: + "^[a-zA-Z][a-zA-Z0-9\\-_]{0,63}$": + type: object + + properties: + temperature: + $ref: /schemas/types.yaml#/definitions/int32 + minimum: -273000 + maximum: 200000 + description: + An integer expressing the trip temperature in millicelsius. + + hysteresis: + $ref: /schemas/types.yaml#/definitions/uint32 + description: + An unsigned integer expressing the hysteresis delta with + respect to the trip temperature property above, also in + millicelsius. Any cooling action initiated by the framework is + maintained until the temperature falls below + (trip temperature - hysteresis). This potentially prevents a + situation where the trip gets constantly triggered soon after + cooling action is removed. + + type: + $ref: /schemas/types.yaml#/definitions/string + enum: + - active # enable active cooling e.g. fans + - passive # enable passive cooling e.g. throttling cpu + - hot # send notification to driver + - critical # send notification to driver, trigger shutdown + description: | + There are four valid trip types: active, passive, hot, + critical. + + The critical trip type is used to set the maximum + temperature threshold above which the HW becomes + unstable and underlying firmware might even trigger a + reboot. Hitting the critical threshold triggers a system + shutdown. + + The hot trip type can be used to send a notification to + the thermal driver (if a .notify callback is registered). + The action to be taken is left to the driver. + + The passive trip type can be used to slow down HW e.g. run + the CPU, GPU, bus at a lower frequency. + + The active trip type can be used to control other HW to + help in cooling e.g. fans can be sped up or slowed down + + required: + - temperature + - hysteresis + - type + additionalProperties: false + + additionalProperties: false + + cooling-maps: + type: object + description: + This node describes the action to be taken when a thermal zone + crosses one of the temperature thresholds described in the trips + node. The action takes the form of a mapping relation between a + trip and the target cooling device state. + + patternProperties: + "^map[-a-zA-Z0-9]*$": + type: object + + properties: + trip: + $ref: /schemas/types.yaml#/definitions/phandle + description: + A phandle of a trip point node within this thermal zone. + + cooling-device: + $ref: /schemas/types.yaml#/definitions/phandle-array + description: + A list of cooling device phandles along with the minimum + and maximum cooling state specifiers for each cooling + device. Using the THERMAL_NO_LIMIT (-1UL) constant in the + cooling-device phandle limit specifier lets the framework + use the minimum and maximum cooling state for that cooling + device automatically. + + contribution: + $ref: /schemas/types.yaml#/definitions/uint32 + minimum: 0 + maximum: 100 + description: + The percentage contribution of the cooling devices at the + specific trip temperature referenced in this map + to this thermal zone + + required: + - trip + - cooling-device + additionalProperties: false + + required: + - polling-delay + - polling-delay-passive + - thermal-sensors + - trips + additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/arm-gic.h> + #include <dt-bindings/thermal/thermal.h> + + // Example 1: SDM845 TSENS + soc: soc@0 { + #address-cells = <2>; + #size-cells = <2>; + + /* ... */ + + tsens0: thermal-sensor@c263000 { + compatible = "qcom,sdm845-tsens", "qcom,tsens-v2"; + reg = <0 0x0c263000 0 0x1ff>, /* TM */ + <0 0x0c222000 0 0x1ff>; /* SROT */ + #qcom,sensors = <13>; + interrupts = <GIC_SPI 506 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 508 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "uplow", "critical"; + #thermal-sensor-cells = <1>; + }; + + tsens1: thermal-sensor@c265000 { + compatible = "qcom,sdm845-tsens", "qcom,tsens-v2"; + reg = <0 0x0c265000 0 0x1ff>, /* TM */ + <0 0x0c223000 0 0x1ff>; /* SROT */ + #qcom,sensors = <8>; + interrupts = <GIC_SPI 507 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 509 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "uplow", "critical"; + #thermal-sensor-cells = <1>; + }; + }; + + /* ... */ + + thermal-zones { + cpu0-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + + thermal-sensors = <&tsens0 1>; + + trips { + cpu0_alert0: trip-point0 { + temperature = <90000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu0_alert1: trip-point1 { + temperature = <95000>; + hysteresis = <2000>; + type = "passive"; + }; + + cpu0_crit: cpu_crit { + temperature = <110000>; + hysteresis = <1000>; + type = "critical"; + }; + }; + + cooling-maps { + map0 { + trip = <&cpu0_alert0>; + /* Corresponds to 1400MHz in OPP table */ + cooling-device = <&CPU0 3 3>, <&CPU1 3 3>, + <&CPU2 3 3>, <&CPU3 3 3>; + }; + + map1 { + trip = <&cpu0_alert1>; + /* Corresponds to 1000MHz in OPP table */ + cooling-device = <&CPU0 5 5>, <&CPU1 5 5>, + <&CPU2 5 5>, <&CPU3 5 5>; + }; + }; + }; + + /* ... */ + + cluster0-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + + thermal-sensors = <&tsens0 5>; + + trips { + cluster0_alert0: trip-point0 { + temperature = <90000>; + hysteresis = <2000>; + type = "hot"; + }; + cluster0_crit: cluster0_crit { + temperature = <110000>; + hysteresis = <2000>; + type = "critical"; + }; + }; + }; + + /* ... */ + + gpu-top-thermal { + polling-delay-passive = <250>; + polling-delay = <1000>; + + thermal-sensors = <&tsens0 11>; + + trips { + gpu1_alert0: trip-point0 { + temperature = <90000>; + hysteresis = <2000>; + type = "hot"; + }; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/thermal/ti,am654-thermal.yaml b/Documentation/devicetree/bindings/thermal/ti,am654-thermal.yaml new file mode 100644 index 000000000000..25b9209c2e5d --- /dev/null +++ b/Documentation/devicetree/bindings/thermal/ti,am654-thermal.yaml @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/thermal/ti,am654-thermal.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Texas Instruments AM654 VTM (DTS) binding + +maintainers: + - Keerthy <j-keerthy@ti.com> + +properties: + compatible: + const: ti,am654-vtm + + reg: + maxItems: 1 + + power-domains: + maxItems: 1 + + "#thermal-sensor-cells": + const: 1 + +required: + - compatible + - reg + - power-domains + - "#thermal-sensor-cells" + +additionalProperties: false + +examples: + - | + #include <dt-bindings/soc/ti,sci_pm_domain.h> + vtm: thermal@42050000 { + compatible = "ti,am654-vtm"; + reg = <0x0 0x42050000 0x0 0x25c>; + power-domains = <&k3_pds 80 TI_SCI_PD_EXCLUSIVE>; + #thermal-sensor-cells = <1>; + }; + + mpu0_thermal: mpu0_thermal { + polling-delay-passive = <250>; /* milliseconds */ + polling-delay = <500>; /* milliseconds */ + thermal-sensors = <&vtm0 0>; + + trips { + mpu0_crit: mpu0_crit { + temperature = <125000>; /* milliCelsius */ + hysteresis = <2000>; /* milliCelsius */ + type = "critical"; + }; + }; + }; +... diff --git a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml index ccc67d03d4bb..e4e83d3971ac 100644 --- a/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml +++ b/Documentation/devicetree/bindings/usb/aspeed,usb-vhub.yaml @@ -52,21 +52,18 @@ properties: vhub-vendor-id: description: vhub Vendor ID - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - maximum: 65535 + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 65535 vhub-product-id: description: vhub Product ID - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - maximum: 65535 + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 65535 vhub-device-revision: description: vhub Device Revision in binary-coded decimal - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 - - maximum: 65535 + $ref: /schemas/types.yaml#/definitions/uint32 + maximum: 65535 vhub-strings: type: object @@ -90,18 +87,15 @@ properties: manufacturer: description: vhub manufacturer - allOf: - - $ref: /schemas/types.yaml#/definitions/string + $ref: /schemas/types.yaml#/definitions/string product: description: vhub product name - allOf: - - $ref: /schemas/types.yaml#/definitions/string + $ref: /schemas/types.yaml#/definitions/string serial-number: description: vhub device serial number - allOf: - - $ref: /schemas/types.yaml#/definitions/string + $ref: /schemas/types.yaml#/definitions/string required: - compatible diff --git a/Documentation/devicetree/bindings/usb/nvidia,tegra-xudc.yaml b/Documentation/devicetree/bindings/usb/nvidia,tegra-xudc.yaml index c4ddc0adf101..0073763a30d8 100644 --- a/Documentation/devicetree/bindings/usb/nvidia,tegra-xudc.yaml +++ b/Documentation/devicetree/bindings/usb/nvidia,tegra-xudc.yaml @@ -64,13 +64,11 @@ properties: - const: hs_src power-domains: - maxItems: 2 items: - description: XUSBB(device) power-domain - description: XUSBA(superspeed) power-domain power-domain-names: - maxItems: 2 items: - const: dev - const: ss diff --git a/Documentation/devicetree/bindings/watchdog/arm-smc-wdt.yaml b/Documentation/devicetree/bindings/watchdog/arm-smc-wdt.yaml index bec651541e0c..8e4c7c69bc1c 100644 --- a/Documentation/devicetree/bindings/watchdog/arm-smc-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/arm-smc-wdt.yaml @@ -17,8 +17,7 @@ properties: enum: - arm,smc-wdt arm,smc-id: - allOf: - - $ref: /schemas/types.yaml#/definitions/uint32 + $ref: /schemas/types.yaml#/definitions/uint32 description: | The ATF smc function id used by the firmware. Defaults to 0x82003D06 if unset. diff --git a/Documentation/driver-api/media/v4l2-subdev.rst b/Documentation/driver-api/media/v4l2-subdev.rst index 6ced2381952a..6e71f67455bb 100644 --- a/Documentation/driver-api/media/v4l2-subdev.rst +++ b/Documentation/driver-api/media/v4l2-subdev.rst @@ -319,8 +319,7 @@ The device node handles a subset of the V4L2 API. events can also be reported by one (or several) V4L2 device nodes. Sub-device drivers that want to use events need to set the - ``V4L2_SUBDEV_USES_EVENTS`` :c:type:`v4l2_subdev`.flags and initialize - :c:type:`v4l2_subdev`.nevents to events queue depth before registering + ``V4L2_SUBDEV_FL_HAS_EVENTS`` :c:type:`v4l2_subdev`.flags before registering the sub-device. After registration events can be queued as usual on the :c:type:`v4l2_subdev`.devnode device node. diff --git a/Documentation/filesystems/locking.rst b/Documentation/filesystems/locking.rst index eb71156bcb7c..318605de83f3 100644 --- a/Documentation/filesystems/locking.rst +++ b/Documentation/filesystems/locking.rst @@ -429,6 +429,7 @@ prototypes:: int (*lm_grant)(struct file_lock *, struct file_lock *, int); void (*lm_break)(struct file_lock *); /* break_lease callback */ int (*lm_change)(struct file_lock **, int); + bool (*lm_breaker_owns_lease)(struct file_lock *); locking rules: @@ -439,6 +440,7 @@ lm_notify: yes yes no lm_grant: no no no lm_break: yes no no lm_change yes no no +lm_breaker_owns_lease: no no no ========== ============= ================= ========= buffer_head diff --git a/Documentation/lzo.txt b/Documentation/lzo.txt index ca983328976b..f65b51523014 100644 --- a/Documentation/lzo.txt +++ b/Documentation/lzo.txt @@ -159,11 +159,15 @@ Byte sequences distance = 16384 + (H << 14) + D state = S (copy S literals after this block) End of stream is reached if distance == 16384 + In version 1 only, to prevent ambiguity with the RLE case when + ((distance & 0x803f) == 0x803f) && (261 <= length <= 264), the + compressor must not emit block copies where distance and length + meet these conditions. In version 1 only, this instruction is also used to encode a run of - zeros if distance = 0xbfff, i.e. H = 1 and the D bits are all 1. + zeros if distance = 0xbfff, i.e. H = 1 and the D bits are all 1. In this case, it is followed by a fourth byte, X. - run length = ((X << 3) | (0 0 0 0 0 L L L)) + 4. + run length = ((X << 3) | (0 0 0 0 0 L L L)) + 4 0 0 1 L L L L L (32..63) Copy of small block within 16kB distance (preferably less than 34B) diff --git a/Documentation/security/keys/core.rst b/Documentation/security/keys/core.rst index 9367d0fe4a02..cdc42ccc12e4 100644 --- a/Documentation/security/keys/core.rst +++ b/Documentation/security/keys/core.rst @@ -1030,6 +1030,63 @@ The keyctl syscall functions are: written into the output buffer. Verification returns 0 on success. + * Watch a key or keyring for changes:: + + long keyctl(KEYCTL_WATCH_KEY, key_serial_t key, int queue_fd, + const struct watch_notification_filter *filter); + + This will set or remove a watch for changes on the specified key or + keyring. + + "key" is the ID of the key to be watched. + + "queue_fd" is a file descriptor referring to an open "/dev/watch_queue" + which manages the buffer into which notifications will be delivered. + + "filter" is either NULL to remove a watch or a filter specification to + indicate what events are required from the key. + + See Documentation/watch_queue.rst for more information. + + Note that only one watch may be emplaced for any particular { key, + queue_fd } combination. + + Notification records look like:: + + struct key_notification { + struct watch_notification watch; + __u32 key_id; + __u32 aux; + }; + + In this, watch::type will be "WATCH_TYPE_KEY_NOTIFY" and subtype will be + one of:: + + NOTIFY_KEY_INSTANTIATED + NOTIFY_KEY_UPDATED + NOTIFY_KEY_LINKED + NOTIFY_KEY_UNLINKED + NOTIFY_KEY_CLEARED + NOTIFY_KEY_REVOKED + NOTIFY_KEY_INVALIDATED + NOTIFY_KEY_SETATTR + + Where these indicate a key being instantiated/rejected, updated, a link + being made in a keyring, a link being removed from a keyring, a keyring + being cleared, a key being revoked, a key being invalidated or a key + having one of its attributes changed (user, group, perm, timeout, + restriction). + + If a watched key is deleted, a basic watch_notification will be issued + with "type" set to WATCH_TYPE_META and "subtype" set to + watch_meta_removal_notification. The watchpoint ID will be set in the + "info" field. + + This needs to be configured by enabling: + + "Provide key/keyring change notifications" (KEY_NOTIFICATIONS) + + Kernel Services =============== diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 1f3da8f32fc1..59472cd6a11d 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -202,6 +202,7 @@ Code Seq# Include File Comments 'W' 00-1F linux/wanrouter.h conflict! (pre 3.9) 'W' 00-3F sound/asound.h conflict! 'W' 40-5F drivers/pci/switch/switchtec.c +'W' 60-61 linux/watch_queue.h 'X' all fs/xfs/xfs_fs.h, conflict! fs/xfs/linux-2.6/xfs_ioctl32.h, include/linux/falloc.h, diff --git a/Documentation/watch_queue.rst b/Documentation/watch_queue.rst new file mode 100644 index 000000000000..849fad6893ef --- /dev/null +++ b/Documentation/watch_queue.rst @@ -0,0 +1,339 @@ +============================== +General notification mechanism +============================== + +The general notification mechanism is built on top of the standard pipe driver +whereby it effectively splices notification messages from the kernel into pipes +opened by userspace. This can be used in conjunction with:: + + * Key/keyring notifications + + +The notifications buffers can be enabled by: + + "General setup"/"General notification queue" + (CONFIG_WATCH_QUEUE) + +This document has the following sections: + +.. contents:: :local: + + +Overview +======== + +This facility appears as a pipe that is opened in a special mode. The pipe's +internal ring buffer is used to hold messages that are generated by the kernel. +These messages are then read out by read(). Splice and similar are disabled on +such pipes due to them wanting to, under some circumstances, revert their +additions to the ring - which might end up interleaved with notification +messages. + +The owner of the pipe has to tell the kernel which sources it would like to +watch through that pipe. Only sources that have been connected to a pipe will +insert messages into it. Note that a source may be bound to multiple pipes and +insert messages into all of them simultaneously. + +Filters may also be emplaced on a pipe so that certain source types and +subevents can be ignored if they're not of interest. + +A message will be discarded if there isn't a slot available in the ring or if +no preallocated message buffer is available. In both of these cases, read() +will insert a WATCH_META_LOSS_NOTIFICATION message into the output buffer after +the last message currently in the buffer has been read. + +Note that when producing a notification, the kernel does not wait for the +consumers to collect it, but rather just continues on. This means that +notifications can be generated whilst spinlocks are held and also protects the +kernel from being held up indefinitely by a userspace malfunction. + + +Message Structure +================= + +Notification messages begin with a short header:: + + struct watch_notification { + __u32 type:24; + __u32 subtype:8; + __u32 info; + }; + +"type" indicates the source of the notification record and "subtype" indicates +the type of record from that source (see the Watch Sources section below). The +type may also be "WATCH_TYPE_META". This is a special record type generated +internally by the watch queue itself. There are two subtypes: + + * WATCH_META_REMOVAL_NOTIFICATION + * WATCH_META_LOSS_NOTIFICATION + +The first indicates that an object on which a watch was installed was removed +or destroyed and the second indicates that some messages have been lost. + +"info" indicates a bunch of things, including: + + * The length of the message in bytes, including the header (mask with + WATCH_INFO_LENGTH and shift by WATCH_INFO_LENGTH__SHIFT). This indicates + the size of the record, which may be between 8 and 127 bytes. + + * The watch ID (mask with WATCH_INFO_ID and shift by WATCH_INFO_ID__SHIFT). + This indicates that caller's ID of the watch, which may be between 0 + and 255. Multiple watches may share a queue, and this provides a means to + distinguish them. + + * A type-specific field (WATCH_INFO_TYPE_INFO). This is set by the + notification producer to indicate some meaning specific to the type and + subtype. + +Everything in info apart from the length can be used for filtering. + +The header can be followed by supplementary information. The format of this is +at the discretion is defined by the type and subtype. + + +Watch List (Notification Source) API +==================================== + +A "watch list" is a list of watchers that are subscribed to a source of +notifications. A list may be attached to an object (say a key or a superblock) +or may be global (say for device events). From a userspace perspective, a +non-global watch list is typically referred to by reference to the object it +belongs to (such as using KEYCTL_NOTIFY and giving it a key serial number to +watch that specific key). + +To manage a watch list, the following functions are provided: + + * ``void init_watch_list(struct watch_list *wlist, + void (*release_watch)(struct watch *wlist));`` + + Initialise a watch list. If ``release_watch`` is not NULL, then this + indicates a function that should be called when the watch_list object is + destroyed to discard any references the watch list holds on the watched + object. + + * ``void remove_watch_list(struct watch_list *wlist);`` + + This removes all of the watches subscribed to a watch_list and frees them + and then destroys the watch_list object itself. + + +Watch Queue (Notification Output) API +===================================== + +A "watch queue" is the buffer allocated by an application that notification +records will be written into. The workings of this are hidden entirely inside +of the pipe device driver, but it is necessary to gain a reference to it to set +a watch. These can be managed with: + + * ``struct watch_queue *get_watch_queue(int fd);`` + + Since watch queues are indicated to the kernel by the fd of the pipe that + implements the buffer, userspace must hand that fd through a system call. + This can be used to look up an opaque pointer to the watch queue from the + system call. + + * ``void put_watch_queue(struct watch_queue *wqueue);`` + + This discards the reference obtained from ``get_watch_queue()``. + + +Watch Subscription API +====================== + +A "watch" is a subscription on a watch list, indicating the watch queue, and +thus the buffer, into which notification records should be written. The watch +queue object may also carry filtering rules for that object, as set by +userspace. Some parts of the watch struct can be set by the driver:: + + struct watch { + union { + u32 info_id; /* ID to be OR'd in to info field */ + ... + }; + void *private; /* Private data for the watched object */ + u64 id; /* Internal identifier */ + ... + }; + +The ``info_id`` value should be an 8-bit number obtained from userspace and +shifted by WATCH_INFO_ID__SHIFT. This is OR'd into the WATCH_INFO_ID field of +struct watch_notification::info when and if the notification is written into +the associated watch queue buffer. + +The ``private`` field is the driver's data associated with the watch_list and +is cleaned up by the ``watch_list::release_watch()`` method. + +The ``id`` field is the source's ID. Notifications that are posted with a +different ID are ignored. + +The following functions are provided to manage watches: + + * ``void init_watch(struct watch *watch, struct watch_queue *wqueue);`` + + Initialise a watch object, setting its pointer to the watch queue, using + appropriate barriering to avoid lockdep complaints. + + * ``int add_watch_to_object(struct watch *watch, struct watch_list *wlist);`` + + Subscribe a watch to a watch list (notification source). The + driver-settable fields in the watch struct must have been set before this + is called. + + * ``int remove_watch_from_object(struct watch_list *wlist, + struct watch_queue *wqueue, + u64 id, false);`` + + Remove a watch from a watch list, where the watch must match the specified + watch queue (``wqueue``) and object identifier (``id``). A notification + (``WATCH_META_REMOVAL_NOTIFICATION``) is sent to the watch queue to + indicate that the watch got removed. + + * ``int remove_watch_from_object(struct watch_list *wlist, NULL, 0, true);`` + + Remove all the watches from a watch list. It is expected that this will be + called preparatory to destruction and that the watch list will be + inaccessible to new watches by this point. A notification + (``WATCH_META_REMOVAL_NOTIFICATION``) is sent to the watch queue of each + subscribed watch to indicate that the watch got removed. + + +Notification Posting API +======================== + +To post a notification to watch list so that the subscribed watches can see it, +the following function should be used:: + + void post_watch_notification(struct watch_list *wlist, + struct watch_notification *n, + const struct cred *cred, + u64 id); + +The notification should be preformatted and a pointer to the header (``n``) +should be passed in. The notification may be larger than this and the size in +units of buffer slots is noted in ``n->info & WATCH_INFO_LENGTH``. + +The ``cred`` struct indicates the credentials of the source (subject) and is +passed to the LSMs, such as SELinux, to allow or suppress the recording of the +note in each individual queue according to the credentials of that queue +(object). + +The ``id`` is the ID of the source object (such as the serial number on a key). +Only watches that have the same ID set in them will see this notification. + + +Watch Sources +============= + +Any particular buffer can be fed from multiple sources. Sources include: + + * WATCH_TYPE_KEY_NOTIFY + + Notifications of this type indicate changes to keys and keyrings, including + the changes of keyring contents or the attributes of keys. + + See Documentation/security/keys/core.rst for more information. + + +Event Filtering +=============== + +Once a watch queue has been created, a set of filters can be applied to limit +the events that are received using:: + + struct watch_notification_filter filter = { + ... + }; + ioctl(fd, IOC_WATCH_QUEUE_SET_FILTER, &filter) + +The filter description is a variable of type:: + + struct watch_notification_filter { + __u32 nr_filters; + __u32 __reserved; + struct watch_notification_type_filter filters[]; + }; + +Where "nr_filters" is the number of filters in filters[] and "__reserved" +should be 0. The "filters" array has elements of the following type:: + + struct watch_notification_type_filter { + __u32 type; + __u32 info_filter; + __u32 info_mask; + __u32 subtype_filter[8]; + }; + +Where: + + * ``type`` is the event type to filter for and should be something like + "WATCH_TYPE_KEY_NOTIFY" + + * ``info_filter`` and ``info_mask`` act as a filter on the info field of the + notification record. The notification is only written into the buffer if:: + + (watch.info & info_mask) == info_filter + + This could be used, for example, to ignore events that are not exactly on + the watched point in a mount tree. + + * ``subtype_filter`` is a bitmask indicating the subtypes that are of + interest. Bit 0 of subtype_filter[0] corresponds to subtype 0, bit 1 to + subtype 1, and so on. + +If the argument to the ioctl() is NULL, then the filters will be removed and +all events from the watched sources will come through. + + +Userspace Code Example +====================== + +A buffer is created with something like the following:: + + pipe2(fds, O_TMPFILE); + ioctl(fds[1], IOC_WATCH_QUEUE_SET_SIZE, 256); + +It can then be set to receive keyring change notifications:: + + keyctl(KEYCTL_WATCH_KEY, KEY_SPEC_SESSION_KEYRING, fds[1], 0x01); + +The notifications can then be consumed by something like the following:: + + static void consumer(int rfd, struct watch_queue_buffer *buf) + { + unsigned char buffer[128]; + ssize_t buf_len; + + while (buf_len = read(rfd, buffer, sizeof(buffer)), + buf_len > 0 + ) { + void *p = buffer; + void *end = buffer + buf_len; + while (p < end) { + union { + struct watch_notification n; + unsigned char buf1[128]; + } n; + size_t largest, len; + + largest = end - p; + if (largest > 128) + largest = 128; + memcpy(&n, p, largest); + + len = (n->info & WATCH_INFO_LENGTH) >> + WATCH_INFO_LENGTH__SHIFT; + if (len == 0 || len > largest) + return; + + switch (n.n.type) { + case WATCH_TYPE_META: + got_meta(&n.n); + case WATCH_TYPE_KEY_NOTIFY: + saw_key_change(&n.n); + break; + } + + p += len; + } + } + } diff --git a/MAINTAINERS b/MAINTAINERS index 9b4e836543a3..68f21d46614c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -884,7 +884,7 @@ M: Joerg Roedel <joro@8bytes.org> L: iommu@lists.linux-foundation.org S: Maintained T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git -F: drivers/iommu/amd_iommu*.[ch] +F: drivers/iommu/amd/ F: include/linux/amd-iommu.h AMD KFD @@ -2174,6 +2174,7 @@ F: Documentation/devicetree/bindings/*/*npcm* F: arch/arm/boot/dts/nuvoton-npcm* F: arch/arm/mach-npcm/ F: drivers/*/*npcm* +F: drivers/*/*/*npcm* F: include/dt-bindings/clock/nuvoton,npcm7xx-clock.h ARM/OPENMOKO NEO FREERUNNER (GTA02) MACHINE SUPPORT @@ -8735,8 +8736,7 @@ M: Lu Baolu <baolu.lu@linux.intel.com> L: iommu@lists.linux-foundation.org S: Supported T: git git://git.kernel.org/pub/scm/linux/kernel/git/joro/iommu.git -F: drivers/iommu/dmar.c -F: drivers/iommu/intel*.[ch] +F: drivers/iommu/intel/ F: include/linux/intel-iommu.h F: include/linux/intel-svm.h @@ -9305,6 +9305,17 @@ F: Documentation/kbuild/kconfig* F: scripts/Kconfig.include F: scripts/kconfig/ +KCSAN +M: Marco Elver <elver@google.com> +R: Dmitry Vyukov <dvyukov@google.com> +L: kasan-dev@googlegroups.com +S: Maintained +F: Documentation/dev-tools/kcsan.rst +F: include/linux/kcsan*.h +F: kernel/kcsan/ +F: lib/Kconfig.kcsan +F: scripts/Makefile.kcsan + KDUMP M: Dave Young <dyoung@redhat.com> M: Baoquan He <bhe@redhat.com> @@ -12900,7 +12911,7 @@ F: include/uapi/linux/ppdev.h PARAVIRT_OPS INTERFACE M: Juergen Gross <jgross@suse.com> -M: Thomas Hellstrom <thellstrom@vmware.com> +M: Deep Shah <sdeep@vmware.com> M: "VMware, Inc." <pv-drivers@vmware.com> L: virtualization@lists.linux-foundation.org S: Supported @@ -13909,6 +13920,7 @@ F: drivers/media/rc/pwm-ir-tx.c PWM SUBSYSTEM M: Thierry Reding <thierry.reding@gmail.com> R: Uwe Kleine-König <u.kleine-koenig@pengutronix.de> +M: Lee Jones <lee.jones@linaro.org> L: linux-pwm@vger.kernel.org S: Maintained Q: https://patchwork.ozlabs.org/project/linux-pwm/list/ @@ -14187,6 +14199,14 @@ L: linux-arm-msm@vger.kernel.org S: Maintained F: drivers/iommu/qcom_iommu.c +QUALCOMM IPCC MAILBOX DRIVER +M: Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org> +L: linux-arm-msm@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/mailbox/qcom-ipcc.yaml +F: drivers/mailbox/qcom-ipcc.c +F: include/dt-bindings/mailbox/qcom-ipcc.h + QUALCOMM RMNET DRIVER M: Subash Abhinov Kasiviswanathan <subashab@codeaurora.org> M: Sean Tranchetti <stranche@codeaurora.org> @@ -14537,6 +14557,15 @@ F: Documentation/devicetree/bindings/i2c/renesas,iic.txt F: drivers/i2c/busses/i2c-rcar.c F: drivers/i2c/busses/i2c-sh_mobile.c +RENESAS R-CAR THERMAL DRIVERS +M: Niklas Söderlund <niklas.soderlund@ragnatech.se> +L: linux-renesas-soc@vger.kernel.org +S: Supported +F: Documentation/devicetree/bindings/thermal/rcar-gen3-thermal.txt +F: Documentation/devicetree/bindings/thermal/rcar-thermal.txt +F: drivers/thermal/rcar_gen3_thermal.c +F: drivers/thermal/rcar_thermal.c + RENESAS RIIC DRIVER M: Chris Brandt <chris.brandt@renesas.com> S: Supported @@ -18268,7 +18297,7 @@ S: Maintained F: drivers/misc/vmw_balloon.c VMWARE HYPERVISOR INTERFACE -M: Thomas Hellstrom <thellstrom@vmware.com> +M: Deep Shah <sdeep@vmware.com> M: "VMware, Inc." <pv-drivers@vmware.com> L: virtualization@lists.linux-foundation.org S: Supported @@ -517,7 +517,7 @@ export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE export KBUILD_CPPFLAGS NOSTDINC_FLAGS LINUXINCLUDE OBJCOPYFLAGS KBUILD_LDFLAGS export KBUILD_CFLAGS CFLAGS_KERNEL CFLAGS_MODULE -export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN +export CFLAGS_KASAN CFLAGS_KASAN_NOSANITIZE CFLAGS_UBSAN CFLAGS_KCSAN export KBUILD_AFLAGS AFLAGS_KERNEL AFLAGS_MODULE export KBUILD_AFLAGS_MODULE KBUILD_CFLAGS_MODULE KBUILD_LDFLAGS_MODULE export KBUILD_AFLAGS_KERNEL KBUILD_CFLAGS_KERNEL @@ -951,6 +951,7 @@ endif include scripts/Makefile.kasan include scripts/Makefile.extrawarn include scripts/Makefile.ubsan +include scripts/Makefile.kcsan # Add user supplied CPPFLAGS, AFLAGS and CFLAGS as the last assignments KBUILD_CPPFLAGS += $(KCPPFLAGS) diff --git a/arch/alpha/Kconfig b/arch/alpha/Kconfig index 92f3b0d2ddf5..10862c5a8c76 100644 --- a/arch/alpha/Kconfig +++ b/arch/alpha/Kconfig @@ -545,7 +545,7 @@ config NR_CPUS default "4" if !ALPHA_GENERIC && !ALPHA_MARVEL help MARVEL support can handle a maximum of 32 CPUs, all the others - with working support have a maximum of 4 CPUs. + with working support have a maximum of 4 CPUs. config ARCH_DISCONTIGMEM_ENABLE bool "Discontiguous Memory Support" @@ -657,7 +657,7 @@ choice endchoice config HZ - int + int default 32 if HZ_32 default 64 if HZ_64 default 128 if HZ_128 diff --git a/arch/alpha/boot/tools/objstrip.c b/arch/alpha/boot/tools/objstrip.c index 825a16f5f622..08b430d25a31 100644 --- a/arch/alpha/boot/tools/objstrip.c +++ b/arch/alpha/boot/tools/objstrip.c @@ -148,7 +148,7 @@ main (int argc, char *argv[]) #ifdef __ELF__ elf = (struct elfhdr *) buf; - if (elf->e_ident[0] == 0x7f && strncmp((char *)elf->e_ident + 1, "ELF", 3) == 0) { + if (elf->e_ident[0] == 0x7f && str_has_prefix((char *)elf->e_ident + 1, "ELF")) { if (elf->e_type != ET_EXEC) { fprintf(stderr, "%s: %s is not an ELF executable\n", prog_name, inname); diff --git a/arch/alpha/include/asm/io.h b/arch/alpha/include/asm/io.h index 13bea465f1c0..a4d0c19f1e79 100644 --- a/arch/alpha/include/asm/io.h +++ b/arch/alpha/include/asm/io.h @@ -309,14 +309,18 @@ static inline int __is_mmio(const volatile void __iomem *addr) #if IO_CONCAT(__IO_PREFIX,trivial_io_bw) extern inline unsigned int ioread8(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread8)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread8)(addr); mb(); return ret; } extern inline unsigned int ioread16(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread16)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread16)(addr); mb(); return ret; } @@ -357,7 +361,9 @@ extern inline void outw(u16 b, unsigned long port) #if IO_CONCAT(__IO_PREFIX,trivial_io_lq) extern inline unsigned int ioread32(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread32)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread32)(addr); mb(); return ret; } @@ -402,14 +408,18 @@ extern inline void __raw_writew(u16 b, volatile void __iomem *addr) extern inline u8 readb(const volatile void __iomem *addr) { - u8 ret = __raw_readb(addr); + u8 ret; + mb(); + ret = __raw_readb(addr); mb(); return ret; } extern inline u16 readw(const volatile void __iomem *addr) { - u16 ret = __raw_readw(addr); + u16 ret; + mb(); + ret = __raw_readw(addr); mb(); return ret; } @@ -450,14 +460,18 @@ extern inline void __raw_writeq(u64 b, volatile void __iomem *addr) extern inline u32 readl(const volatile void __iomem *addr) { - u32 ret = __raw_readl(addr); + u32 ret; + mb(); + ret = __raw_readl(addr); mb(); return ret; } extern inline u64 readq(const volatile void __iomem *addr) { - u64 ret = __raw_readq(addr); + u64 ret; + mb(); + ret = __raw_readq(addr); mb(); return ret; } @@ -486,14 +500,44 @@ extern inline void writeq(u64 b, volatile void __iomem *addr) #define outb_p outb #define outw_p outw #define outl_p outl -#define readb_relaxed(addr) __raw_readb(addr) -#define readw_relaxed(addr) __raw_readw(addr) -#define readl_relaxed(addr) __raw_readl(addr) -#define readq_relaxed(addr) __raw_readq(addr) -#define writeb_relaxed(b, addr) __raw_writeb(b, addr) -#define writew_relaxed(b, addr) __raw_writew(b, addr) -#define writel_relaxed(b, addr) __raw_writel(b, addr) -#define writeq_relaxed(b, addr) __raw_writeq(b, addr) + +extern u8 readb_relaxed(const volatile void __iomem *addr); +extern u16 readw_relaxed(const volatile void __iomem *addr); +extern u32 readl_relaxed(const volatile void __iomem *addr); +extern u64 readq_relaxed(const volatile void __iomem *addr); + +#if IO_CONCAT(__IO_PREFIX,trivial_io_bw) +extern inline u8 readb_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readb(addr); +} + +extern inline u16 readw_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readw(addr); +} +#endif + +#if IO_CONCAT(__IO_PREFIX,trivial_io_lq) +extern inline u32 readl_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readl(addr); +} + +extern inline u64 readq_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readq(addr); +} +#endif + +#define writeb_relaxed writeb +#define writew_relaxed writew +#define writel_relaxed writel +#define writeq_relaxed writeq /* * String version of IO memory access ops: diff --git a/arch/alpha/kernel/io.c b/arch/alpha/kernel/io.c index c025a3e5e357..938de13adfbf 100644 --- a/arch/alpha/kernel/io.c +++ b/arch/alpha/kernel/io.c @@ -16,21 +16,27 @@ unsigned int ioread8(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread8)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread8)(addr); mb(); return ret; } unsigned int ioread16(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread16)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread16)(addr); mb(); return ret; } unsigned int ioread32(void __iomem *addr) { - unsigned int ret = IO_CONCAT(__IO_PREFIX,ioread32)(addr); + unsigned int ret; + mb(); + ret = IO_CONCAT(__IO_PREFIX,ioread32)(addr); mb(); return ret; } @@ -148,28 +154,36 @@ EXPORT_SYMBOL(__raw_writeq); u8 readb(const volatile void __iomem *addr) { - u8 ret = __raw_readb(addr); + u8 ret; + mb(); + ret = __raw_readb(addr); mb(); return ret; } u16 readw(const volatile void __iomem *addr) { - u16 ret = __raw_readw(addr); + u16 ret; + mb(); + ret = __raw_readw(addr); mb(); return ret; } u32 readl(const volatile void __iomem *addr) { - u32 ret = __raw_readl(addr); + u32 ret; + mb(); + ret = __raw_readl(addr); mb(); return ret; } u64 readq(const volatile void __iomem *addr) { - u64 ret = __raw_readq(addr); + u64 ret; + mb(); + ret = __raw_readq(addr); mb(); return ret; } @@ -207,6 +221,38 @@ EXPORT_SYMBOL(writew); EXPORT_SYMBOL(writel); EXPORT_SYMBOL(writeq); +/* + * The _relaxed functions must be ordered w.r.t. each other, but they don't + * have to be ordered w.r.t. other memory accesses. + */ +u8 readb_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readb(addr); +} + +u16 readw_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readw(addr); +} + +u32 readl_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readl(addr); +} + +u64 readq_relaxed(const volatile void __iomem *addr) +{ + mb(); + return __raw_readq(addr); +} + +EXPORT_SYMBOL(readb_relaxed); +EXPORT_SYMBOL(readw_relaxed); +EXPORT_SYMBOL(readl_relaxed); +EXPORT_SYMBOL(readq_relaxed); /* * Read COUNT 8-bit bytes from port PORT into memory starting at SRC. diff --git a/arch/alpha/kernel/osf_sys.c b/arch/alpha/kernel/osf_sys.c index 94e4cde8071a..d5367a1c6300 100644 --- a/arch/alpha/kernel/osf_sys.c +++ b/arch/alpha/kernel/osf_sys.c @@ -677,7 +677,7 @@ SYSCALL_DEFINE2(osf_proplist_syscall, enum pl_code, code, default: error = -EOPNOTSUPP; break; - }; + } return error; } diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index 7f1925a32c99..81037907268d 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -638,7 +638,7 @@ sg_fill(struct device *dev, struct scatterlist *leader, struct scatterlist *end, while (sg+1 < end && (int) sg[1].dma_address == -1) { size += sg[1].length; - sg++; + sg = sg_next(sg); } npages = iommu_num_pages(paddr, size, PAGE_SIZE); diff --git a/arch/alpha/kernel/setup.c b/arch/alpha/kernel/setup.c index f5c42a8fcf9c..916e42d74a86 100644 --- a/arch/alpha/kernel/setup.c +++ b/arch/alpha/kernel/setup.c @@ -253,7 +253,7 @@ reserve_std_resources(void) /* Fix up for the Jensen's queer RTC placement. */ standard_io_resources[0].start = RTC_PORT(0); - standard_io_resources[0].end = RTC_PORT(0) + 0x10; + standard_io_resources[0].end = RTC_PORT(0) + 0x0f; for (i = 0; i < ARRAY_SIZE(standard_io_resources); ++i) request_resource(io, standard_io_resources+i); @@ -430,8 +430,13 @@ register_cpus(void) arch_initcall(register_cpus); #ifdef CONFIG_MAGIC_SYSRQ +static void sysrq_reboot_handler(int unused) +{ + machine_halt(); +} + static const struct sysrq_key_op srm_sysrq_reboot_op = { - .handler = machine_halt, + .handler = sysrq_reboot_handler, .help_msg = "reboot(b)", .action_msg = "Resetting", .enable_mask = SYSRQ_ENABLE_BOOT, @@ -474,7 +479,7 @@ setup_arch(char **cmdline_p) #ifndef alpha_using_srm /* Assume that we've booted from SRM if we haven't booted from MILO. Detect the later by looking for "MILO" in the system serial nr. */ - alpha_using_srm = strncmp((const char *)hwrpb->ssn, "MILO", 4) != 0; + alpha_using_srm = !str_has_prefix((const char *)hwrpb->ssn, "MILO"); #endif #ifndef alpha_using_qemu /* Similarly, look for QEMU. */ @@ -1420,6 +1425,7 @@ c_start(struct seq_file *f, loff_t *pos) static void * c_next(struct seq_file *f, void *v, loff_t *pos) { + (*pos)++; return NULL; } diff --git a/arch/alpha/kernel/sys_eiger.c b/arch/alpha/kernel/sys_eiger.c index bf99dcfd40c4..aea8a54da4bc 100644 --- a/arch/alpha/kernel/sys_eiger.c +++ b/arch/alpha/kernel/sys_eiger.c @@ -175,7 +175,7 @@ eiger_swizzle(struct pci_dev *dev, u8 *pinp) case 0x03: bridge_count = 2; break; /* 2 */ case 0x07: bridge_count = 3; break; /* 3 */ case 0x0f: bridge_count = 4; break; /* 4 */ - }; + } slot = PCI_SLOT(dev->devfn); while (dev->bus->self) { diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 7b68cec30d45..2ac74904a3ce 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1749,7 +1749,7 @@ config DEPRECATED_PARAM_STRUCT # TEXT and BSS so we preserve their values in the config files. config ZBOOT_ROM_TEXT hex "Compressed ROM boot loader base address" - default "0" + default 0x0 help The physical address at which the ROM-able zImage is to be placed in the target. Platforms which normally make use of @@ -1760,7 +1760,7 @@ config ZBOOT_ROM_TEXT config ZBOOT_ROM_BSS hex "Compressed ROM boot loader BSS address" - default "0" + default 0x0 help The base address of an area of read/write memory in the target for the ROM-able zImage which must be available while the diff --git a/arch/arm/boot/compressed/head.S b/arch/arm/boot/compressed/head.S index c79db44ba128..434a16982e34 100644 --- a/arch/arm/boot/compressed/head.S +++ b/arch/arm/boot/compressed/head.S @@ -1410,7 +1410,11 @@ memdump: mov r12, r0 __hyp_reentry_vectors: W(b) . @ reset W(b) . @ undef +#ifdef CONFIG_EFI_STUB + W(b) __enter_kernel_from_hyp @ hvc from HYP +#else W(b) . @ svc +#endif W(b) . @ pabort W(b) . @ dabort W(b) __enter_kernel @ hyp @@ -1429,14 +1433,72 @@ __enter_kernel: reloc_code_end: #ifdef CONFIG_EFI_STUB +__enter_kernel_from_hyp: + mrc p15, 4, r0, c1, c0, 0 @ read HSCTLR + bic r0, r0, #0x5 @ disable MMU and caches + mcr p15, 4, r0, c1, c0, 0 @ write HSCTLR + isb + b __enter_kernel + ENTRY(efi_enter_kernel) mov r4, r0 @ preserve image base mov r8, r1 @ preserve DT pointer + ARM( adrl r0, call_cache_fn ) + THUMB( adr r0, call_cache_fn ) + adr r1, 0f @ clean the region of code we + bl cache_clean_flush @ may run with the MMU off + +#ifdef CONFIG_ARM_VIRT_EXT + @ + @ The EFI spec does not support booting on ARM in HYP mode, + @ since it mandates that the MMU and caches are on, with all + @ 32-bit addressable DRAM mapped 1:1 using short descriptors. + @ + @ While the EDK2 reference implementation adheres to this, + @ U-Boot might decide to enter the EFI stub in HYP mode + @ anyway, with the MMU and caches either on or off. + @ + mrs r0, cpsr @ get the current mode + msr spsr_cxsf, r0 @ record boot mode + and r0, r0, #MODE_MASK @ are we running in HYP mode? + cmp r0, #HYP_MODE + bne .Lefi_svc + + mrc p15, 4, r1, c1, c0, 0 @ read HSCTLR + tst r1, #0x1 @ MMU enabled at HYP? + beq 1f + + @ + @ When running in HYP mode with the caches on, we're better + @ off just carrying on using the cached 1:1 mapping that the + @ firmware provided. Set up the HYP vectors so HVC instructions + @ issued from HYP mode take us to the correct handler code. We + @ will disable the MMU before jumping to the kernel proper. + @ + adr r0, __hyp_reentry_vectors + mcr p15, 4, r0, c12, c0, 0 @ set HYP vector base (HVBAR) + isb + b .Lefi_hyp + + @ + @ When running in HYP mode with the caches off, we need to drop + @ into SVC mode now, and let the decompressor set up its cached + @ 1:1 mapping as usual. + @ +1: mov r9, r4 @ preserve image base + bl __hyp_stub_install @ install HYP stub vectors + safe_svcmode_maskall r1 @ drop to SVC mode + msr spsr_cxsf, r0 @ record boot mode + orr r4, r9, #1 @ restore image base and set LSB + b .Lefi_hyp +.Lefi_svc: +#endif mrc p15, 0, r0, c1, c0, 0 @ read SCTLR tst r0, #0x1 @ MMU enabled? orreq r4, r4, #1 @ set LSB if not +.Lefi_hyp: mov r0, r8 @ DT start add r1, r8, r2 @ DT end bl cache_clean_flush diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig index fdca57124852..31380da53689 100644 --- a/arch/arm64/Kconfig +++ b/arch/arm64/Kconfig @@ -1299,6 +1299,14 @@ config COMPAT_VDSO You must have a 32-bit build of glibc 2.22 or later for programs to seamlessly take advantage of this. +config THUMB2_COMPAT_VDSO + bool "Compile the 32-bit vDSO for Thumb-2 mode" if EXPERT + depends on COMPAT_VDSO + default y + help + Compile the compat vDSO with '-mthumb -fomit-frame-pointer' if y, + otherwise with '-marm'. + menuconfig ARMV8_DEPRECATED bool "Emulate deprecated/obsolete ARMv8 instructions" depends on SYSCTL @@ -1740,8 +1748,9 @@ config ARM64_DEBUG_PRIORITY_MASKING endif config RELOCATABLE - bool + bool "Build a relocatable kernel image" if EXPERT select ARCH_HAS_RELR + default y help This builds the kernel as a Position Independent Executable (PIE), which retains all relocation metadata required to relocate the diff --git a/arch/arm64/include/asm/acpi.h b/arch/arm64/include/asm/acpi.h index b263e239cb59..a45366c3909b 100644 --- a/arch/arm64/include/asm/acpi.h +++ b/arch/arm64/include/asm/acpi.h @@ -12,6 +12,7 @@ #include <linux/efi.h> #include <linux/memblock.h> #include <linux/psci.h> +#include <linux/stddef.h> #include <asm/cputype.h> #include <asm/io.h> @@ -31,14 +32,14 @@ * is therefore used to delimit the MADT GICC structure minimum length * appropriately. */ -#define ACPI_MADT_GICC_MIN_LENGTH ACPI_OFFSET( \ +#define ACPI_MADT_GICC_MIN_LENGTH offsetof( \ struct acpi_madt_generic_interrupt, efficiency_class) #define BAD_MADT_GICC_ENTRY(entry, end) \ (!(entry) || (entry)->header.length < ACPI_MADT_GICC_MIN_LENGTH || \ (unsigned long)(entry) + (entry)->header.length > (end)) -#define ACPI_MADT_GICC_SPE (ACPI_OFFSET(struct acpi_madt_generic_interrupt, \ +#define ACPI_MADT_GICC_SPE (offsetof(struct acpi_madt_generic_interrupt, \ spe_interrupt) + sizeof(u16)) /* Basic configuration for ACPI */ diff --git a/arch/arm64/include/asm/atomic.h b/arch/arm64/include/asm/atomic.h index 9543b5e0534d..a08890da696c 100644 --- a/arch/arm64/include/asm/atomic.h +++ b/arch/arm64/include/asm/atomic.h @@ -101,8 +101,8 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) #define ATOMIC_INIT(i) { (i) } -#define arch_atomic_read(v) READ_ONCE((v)->counter) -#define arch_atomic_set(v, i) WRITE_ONCE(((v)->counter), (i)) +#define arch_atomic_read(v) __READ_ONCE((v)->counter) +#define arch_atomic_set(v, i) __WRITE_ONCE(((v)->counter), (i)) #define arch_atomic_add_return_relaxed arch_atomic_add_return_relaxed #define arch_atomic_add_return_acquire arch_atomic_add_return_acquire @@ -225,6 +225,6 @@ static inline long arch_atomic64_dec_if_positive(atomic64_t *v) #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive -#include <asm-generic/atomic-instrumented.h> +#define ARCH_ATOMIC #endif /* __ASM_ATOMIC_H */ diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h index 0c9b5fc4ba0a..352aaebf4198 100644 --- a/arch/arm64/include/asm/kvm_asm.h +++ b/arch/arm64/include/asm/kvm_asm.h @@ -81,12 +81,39 @@ extern u32 __kvm_get_mdcr_el2(void); extern char __smccc_workaround_1_smc[__SMCCC_WORKAROUND_1_SMC_SZ]; -/* Home-grown __this_cpu_{ptr,read} variants that always work at HYP */ +/* + * Obtain the PC-relative address of a kernel symbol + * s: symbol + * + * The goal of this macro is to return a symbol's address based on a + * PC-relative computation, as opposed to a loading the VA from a + * constant pool or something similar. This works well for HYP, as an + * absolute VA is guaranteed to be wrong. Only use this if trying to + * obtain the address of a symbol (i.e. not something you obtained by + * following a pointer). + */ +#define hyp_symbol_addr(s) \ + ({ \ + typeof(s) *addr; \ + asm("adrp %0, %1\n" \ + "add %0, %0, :lo12:%1\n" \ + : "=r" (addr) : "S" (&s)); \ + addr; \ + }) + +/* + * Home-grown __this_cpu_{ptr,read} variants that always work at HYP, + * provided that sym is really a *symbol* and not a pointer obtained from + * a data structure. As for SHIFT_PERCPU_PTR(), the creative casting keeps + * sparse quiet. + */ #define __hyp_this_cpu_ptr(sym) \ ({ \ - void *__ptr = hyp_symbol_addr(sym); \ + void *__ptr; \ + __verify_pcpu_ptr(&sym); \ + __ptr = hyp_symbol_addr(sym); \ __ptr += read_sysreg(tpidr_el2); \ - (typeof(&sym))__ptr; \ + (typeof(sym) __kernel __force *)__ptr; \ }) #define __hyp_this_cpu_read(sym) \ diff --git a/arch/arm64/include/asm/kvm_emulate.h b/arch/arm64/include/asm/kvm_emulate.h index 6ea53e6e8b26..4d0f8ea600ba 100644 --- a/arch/arm64/include/asm/kvm_emulate.h +++ b/arch/arm64/include/asm/kvm_emulate.h @@ -112,12 +112,6 @@ static inline void vcpu_ptrauth_disable(struct kvm_vcpu *vcpu) vcpu->arch.hcr_el2 &= ~(HCR_API | HCR_APK); } -static inline void vcpu_ptrauth_setup_lazy(struct kvm_vcpu *vcpu) -{ - if (vcpu_has_ptrauth(vcpu)) - vcpu_ptrauth_disable(vcpu); -} - static inline unsigned long vcpu_get_vsesr(struct kvm_vcpu *vcpu) { return vcpu->arch.vsesr_el2; diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h index abbdf9703e20..c3e6fcc664b1 100644 --- a/arch/arm64/include/asm/kvm_host.h +++ b/arch/arm64/include/asm/kvm_host.h @@ -284,9 +284,6 @@ struct kvm_vcpu_arch { struct kvm_guest_debug_arch vcpu_debug_state; struct kvm_guest_debug_arch external_debug_state; - /* Pointer to host CPU context */ - struct kvm_cpu_context *host_cpu_context; - struct thread_info *host_thread_info; /* hyp VA */ struct user_fpsimd_state *host_fpsimd_state; /* hyp VA */ @@ -404,8 +401,10 @@ void vcpu_write_sys_reg(struct kvm_vcpu *vcpu, u64 val, int reg); * CP14 and CP15 live in the same array, as they are backed by the * same system registers. */ -#define vcpu_cp14(v,r) ((v)->arch.ctxt.copro[(r)]) -#define vcpu_cp15(v,r) ((v)->arch.ctxt.copro[(r)]) +#define CPx_BIAS IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) + +#define vcpu_cp14(v,r) ((v)->arch.ctxt.copro[(r) ^ CPx_BIAS]) +#define vcpu_cp15(v,r) ((v)->arch.ctxt.copro[(r) ^ CPx_BIAS]) struct kvm_vm_stat { ulong remote_tlb_flush; diff --git a/arch/arm64/include/asm/kvm_mmu.h b/arch/arm64/include/asm/kvm_mmu.h index 094260aaafdd..b12bfc1f051a 100644 --- a/arch/arm64/include/asm/kvm_mmu.h +++ b/arch/arm64/include/asm/kvm_mmu.h @@ -108,26 +108,6 @@ static __always_inline unsigned long __kern_hyp_va(unsigned long v) #define kern_hyp_va(v) ((typeof(v))(__kern_hyp_va((unsigned long)(v)))) /* - * Obtain the PC-relative address of a kernel symbol - * s: symbol - * - * The goal of this macro is to return a symbol's address based on a - * PC-relative computation, as opposed to a loading the VA from a - * constant pool or something similar. This works well for HYP, as an - * absolute VA is guaranteed to be wrong. Only use this if trying to - * obtain the address of a symbol (i.e. not something you obtained by - * following a pointer). - */ -#define hyp_symbol_addr(s) \ - ({ \ - typeof(s) *addr; \ - asm("adrp %0, %1\n" \ - "add %0, %0, :lo12:%1\n" \ - : "=r" (addr) : "S" (&s)); \ - addr; \ - }) - -/* * We currently support using a VM-specified IPA size. For backward * compatibility, the default IPA size is fixed to 40bits. */ diff --git a/arch/arm64/kernel/debug-monitors.c b/arch/arm64/kernel/debug-monitors.c index 15e80c876d46..5df49366e9ab 100644 --- a/arch/arm64/kernel/debug-monitors.c +++ b/arch/arm64/kernel/debug-monitors.c @@ -130,7 +130,7 @@ static int clear_os_lock(unsigned int cpu) return 0; } -static int debug_monitors_init(void) +static int __init debug_monitors_init(void) { return cpuhp_setup_state(CPUHP_AP_ARM64_DEBUG_MONITORS_STARTING, "arm64/debug_monitors:starting", diff --git a/arch/arm64/kernel/ftrace.c b/arch/arm64/kernel/ftrace.c index 8618faa82e6d..86a5cf9bc19a 100644 --- a/arch/arm64/kernel/ftrace.c +++ b/arch/arm64/kernel/ftrace.c @@ -69,7 +69,8 @@ static struct plt_entry *get_ftrace_plt(struct module *mod, unsigned long addr) if (addr == FTRACE_ADDR) return &plt[FTRACE_PLT_IDX]; - if (addr == FTRACE_REGS_ADDR && IS_ENABLED(CONFIG_FTRACE_WITH_REGS)) + if (addr == FTRACE_REGS_ADDR && + IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS)) return &plt[FTRACE_REGS_PLT_IDX]; #endif return NULL; diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 3fd2c11c09fc..93b3844cf442 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -319,6 +319,10 @@ void __init setup_arch(char **cmdline_p) xen_early_init(); efi_init(); + + if (!efi_enabled(EFI_BOOT) && ((u64)_text % MIN_KIMG_ALIGN) != 0) + pr_warn(FW_BUG "Kernel image misaligned at boot, please fix your bootloader!"); + arm64_memblock_init(); paging_init(); diff --git a/arch/arm64/kernel/vdso32/Makefile b/arch/arm64/kernel/vdso32/Makefile index 3964738ebbde..7ea1e827e505 100644 --- a/arch/arm64/kernel/vdso32/Makefile +++ b/arch/arm64/kernel/vdso32/Makefile @@ -105,6 +105,14 @@ VDSO_CFLAGS += -D__uint128_t='void*' VDSO_CFLAGS += $(call cc32-disable-warning,shift-count-overflow) VDSO_CFLAGS += -Wno-int-to-pointer-cast +# Compile as THUMB2 or ARM. Unwinding via frame-pointers in THUMB2 is +# unreliable. +ifeq ($(CONFIG_THUMB2_COMPAT_VDSO), y) +VDSO_CFLAGS += -mthumb -fomit-frame-pointer +else +VDSO_CFLAGS += -marm +endif + VDSO_AFLAGS := $(VDSO_CAFLAGS) VDSO_AFLAGS += -D__ASSEMBLY__ diff --git a/arch/arm64/kvm/aarch32.c b/arch/arm64/kvm/aarch32.c index 0a356aa91aa1..40a62a99fbf8 100644 --- a/arch/arm64/kvm/aarch32.c +++ b/arch/arm64/kvm/aarch32.c @@ -33,6 +33,26 @@ static const u8 return_offsets[8][2] = { [7] = { 4, 4 }, /* FIQ, unused */ }; +static bool pre_fault_synchronize(struct kvm_vcpu *vcpu) +{ + preempt_disable(); + if (vcpu->arch.sysregs_loaded_on_cpu) { + kvm_arch_vcpu_put(vcpu); + return true; + } + + preempt_enable(); + return false; +} + +static void post_fault_synchronize(struct kvm_vcpu *vcpu, bool loaded) +{ + if (loaded) { + kvm_arch_vcpu_load(vcpu, smp_processor_id()); + preempt_enable(); + } +} + /* * When an exception is taken, most CPSR fields are left unchanged in the * handler. However, some are explicitly overridden (e.g. M[4:0]). @@ -155,7 +175,10 @@ static void prepare_fault32(struct kvm_vcpu *vcpu, u32 mode, u32 vect_offset) void kvm_inject_undef32(struct kvm_vcpu *vcpu) { + bool loaded = pre_fault_synchronize(vcpu); + prepare_fault32(vcpu, PSR_AA32_MODE_UND, 4); + post_fault_synchronize(vcpu, loaded); } /* @@ -168,6 +191,9 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, u32 vect_offset; u32 *far, *fsr; bool is_lpae; + bool loaded; + + loaded = pre_fault_synchronize(vcpu); if (is_pabt) { vect_offset = 12; @@ -191,6 +217,8 @@ static void inject_abt32(struct kvm_vcpu *vcpu, bool is_pabt, /* no need to shuffle FS[4] into DFSR[10] as its 0 */ *fsr = DFSR_FSC_EXTABT_nLPAE; } + + post_fault_synchronize(vcpu, loaded); } void kvm_inject_dabt32(struct kvm_vcpu *vcpu, unsigned long addr) diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index 7a57381c05e8..90cb90561446 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -144,11 +144,6 @@ out_fail_alloc: return ret; } -int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) -{ - return 0; -} - vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) { return VM_FAULT_SIGBUS; @@ -340,10 +335,8 @@ void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) { int *last_ran; - kvm_host_data_t *cpu_data; last_ran = this_cpu_ptr(vcpu->kvm->arch.last_vcpu_ran); - cpu_data = this_cpu_ptr(&kvm_host_data); /* * We might get preempted before the vCPU actually runs, but @@ -355,7 +348,6 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) } vcpu->cpu = cpu; - vcpu->arch.host_cpu_context = &cpu_data->host_ctxt; kvm_vgic_load(vcpu); kvm_timer_vcpu_load(vcpu); @@ -370,7 +362,8 @@ void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) else vcpu_set_wfx_traps(vcpu); - vcpu_ptrauth_setup_lazy(vcpu); + if (vcpu_has_ptrauth(vcpu)) + vcpu_ptrauth_disable(vcpu); } void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) @@ -990,11 +983,17 @@ static int kvm_arch_vcpu_ioctl_vcpu_init(struct kvm_vcpu *vcpu, * Ensure a rebooted VM will fault in RAM pages and detect if the * guest MMU is turned off and flush the caches as needed. * - * S2FWB enforces all memory accesses to RAM being cacheable, we - * ensure that the cache is always coherent. + * S2FWB enforces all memory accesses to RAM being cacheable, + * ensuring that the data side is always coherent. We still + * need to invalidate the I-cache though, as FWB does *not* + * imply CTR_EL0.DIC. */ - if (vcpu->arch.has_run_once && !cpus_have_const_cap(ARM64_HAS_STAGE2_FWB)) - stage2_unmap_vm(vcpu->kvm); + if (vcpu->arch.has_run_once) { + if (!cpus_have_final_cap(ARM64_HAS_STAGE2_FWB)) + stage2_unmap_vm(vcpu->kvm); + else + __flush_icache_all(); + } vcpu_reset_hcr(vcpu); diff --git a/arch/arm64/kvm/handle_exit.c b/arch/arm64/kvm/handle_exit.c index eb194696ef62..5a02d4c90559 100644 --- a/arch/arm64/kvm/handle_exit.c +++ b/arch/arm64/kvm/handle_exit.c @@ -162,40 +162,14 @@ static int handle_sve(struct kvm_vcpu *vcpu, struct kvm_run *run) return 1; } -#define __ptrauth_save_key(regs, key) \ -({ \ - regs[key ## KEYLO_EL1] = read_sysreg_s(SYS_ ## key ## KEYLO_EL1); \ - regs[key ## KEYHI_EL1] = read_sysreg_s(SYS_ ## key ## KEYHI_EL1); \ -}) - -/* - * Handle the guest trying to use a ptrauth instruction, or trying to access a - * ptrauth register. - */ -void kvm_arm_vcpu_ptrauth_trap(struct kvm_vcpu *vcpu) -{ - struct kvm_cpu_context *ctxt; - - if (vcpu_has_ptrauth(vcpu)) { - vcpu_ptrauth_enable(vcpu); - ctxt = vcpu->arch.host_cpu_context; - __ptrauth_save_key(ctxt->sys_regs, APIA); - __ptrauth_save_key(ctxt->sys_regs, APIB); - __ptrauth_save_key(ctxt->sys_regs, APDA); - __ptrauth_save_key(ctxt->sys_regs, APDB); - __ptrauth_save_key(ctxt->sys_regs, APGA); - } else { - kvm_inject_undefined(vcpu); - } -} - /* * Guest usage of a ptrauth instruction (which the guest EL1 did not turn into - * a NOP). + * a NOP). If we get here, it is that we didn't fixup ptrauth on exit, and all + * that we can do is give the guest an UNDEF. */ static int kvm_handle_ptrauth(struct kvm_vcpu *vcpu, struct kvm_run *run) { - kvm_arm_vcpu_ptrauth_trap(vcpu); + kvm_inject_undefined(vcpu); return 1; } diff --git a/arch/arm64/kvm/hyp/debug-sr.c b/arch/arm64/kvm/hyp/debug-sr.c index 0fc9872a1467..e95af204fec7 100644 --- a/arch/arm64/kvm/hyp/debug-sr.c +++ b/arch/arm64/kvm/hyp/debug-sr.c @@ -185,7 +185,7 @@ void __hyp_text __debug_switch_to_guest(struct kvm_vcpu *vcpu) if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)) return; - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; guest_ctxt = &vcpu->arch.ctxt; host_dbg = &vcpu->arch.host_debug_state.regs; guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr); @@ -207,7 +207,7 @@ void __hyp_text __debug_switch_to_host(struct kvm_vcpu *vcpu) if (!(vcpu->arch.flags & KVM_ARM64_DEBUG_DIRTY)) return; - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; guest_ctxt = &vcpu->arch.ctxt; host_dbg = &vcpu->arch.host_debug_state.regs; guest_dbg = kern_hyp_va(vcpu->arch.debug_ptr); diff --git a/arch/arm64/kvm/hyp/switch.c b/arch/arm64/kvm/hyp/switch.c index 676b6585e5ae..db1c4487d95d 100644 --- a/arch/arm64/kvm/hyp/switch.c +++ b/arch/arm64/kvm/hyp/switch.c @@ -490,6 +490,64 @@ static bool __hyp_text handle_tx2_tvm(struct kvm_vcpu *vcpu) return true; } +static bool __hyp_text esr_is_ptrauth_trap(u32 esr) +{ + u32 ec = ESR_ELx_EC(esr); + + if (ec == ESR_ELx_EC_PAC) + return true; + + if (ec != ESR_ELx_EC_SYS64) + return false; + + switch (esr_sys64_to_sysreg(esr)) { + case SYS_APIAKEYLO_EL1: + case SYS_APIAKEYHI_EL1: + case SYS_APIBKEYLO_EL1: + case SYS_APIBKEYHI_EL1: + case SYS_APDAKEYLO_EL1: + case SYS_APDAKEYHI_EL1: + case SYS_APDBKEYLO_EL1: + case SYS_APDBKEYHI_EL1: + case SYS_APGAKEYLO_EL1: + case SYS_APGAKEYHI_EL1: + return true; + } + + return false; +} + +#define __ptrauth_save_key(regs, key) \ +({ \ + regs[key ## KEYLO_EL1] = read_sysreg_s(SYS_ ## key ## KEYLO_EL1); \ + regs[key ## KEYHI_EL1] = read_sysreg_s(SYS_ ## key ## KEYHI_EL1); \ +}) + +static bool __hyp_text __hyp_handle_ptrauth(struct kvm_vcpu *vcpu) +{ + struct kvm_cpu_context *ctxt; + u64 val; + + if (!vcpu_has_ptrauth(vcpu) || + !esr_is_ptrauth_trap(kvm_vcpu_get_hsr(vcpu))) + return false; + + ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; + __ptrauth_save_key(ctxt->sys_regs, APIA); + __ptrauth_save_key(ctxt->sys_regs, APIB); + __ptrauth_save_key(ctxt->sys_regs, APDA); + __ptrauth_save_key(ctxt->sys_regs, APDB); + __ptrauth_save_key(ctxt->sys_regs, APGA); + + vcpu_ptrauth_enable(vcpu); + + val = read_sysreg(hcr_el2); + val |= (HCR_API | HCR_APK); + write_sysreg(val, hcr_el2); + + return true; +} + /* * Return true when we were able to fixup the guest exit and should return to * the guest, false when we should restore the host state and return to the @@ -524,6 +582,9 @@ static bool __hyp_text fixup_guest_exit(struct kvm_vcpu *vcpu, u64 *exit_code) if (__hyp_handle_fpsimd(vcpu)) return true; + if (__hyp_handle_ptrauth(vcpu)) + return true; + if (!__populate_fault_info(vcpu)) return true; @@ -642,7 +703,7 @@ static int __kvm_vcpu_run_vhe(struct kvm_vcpu *vcpu) struct kvm_cpu_context *guest_ctxt; u64 exit_code; - host_ctxt = vcpu->arch.host_cpu_context; + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; host_ctxt->__hyp_running_vcpu = vcpu; guest_ctxt = &vcpu->arch.ctxt; @@ -747,7 +808,7 @@ int __hyp_text __kvm_vcpu_run_nvhe(struct kvm_vcpu *vcpu) vcpu = kern_hyp_va(vcpu); - host_ctxt = kern_hyp_va(vcpu->arch.host_cpu_context); + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; host_ctxt->__hyp_running_vcpu = vcpu; guest_ctxt = &vcpu->arch.ctxt; diff --git a/arch/arm64/kvm/hyp/sysreg-sr.c b/arch/arm64/kvm/hyp/sysreg-sr.c index ea5d22fbdacf..cc7e957f5b2c 100644 --- a/arch/arm64/kvm/hyp/sysreg-sr.c +++ b/arch/arm64/kvm/hyp/sysreg-sr.c @@ -39,7 +39,6 @@ static void __hyp_text __sysreg_save_el1_state(struct kvm_cpu_context *ctxt) { ctxt->sys_regs[CSSELR_EL1] = read_sysreg(csselr_el1); ctxt->sys_regs[SCTLR_EL1] = read_sysreg_el1(SYS_SCTLR); - ctxt->sys_regs[ACTLR_EL1] = read_sysreg(actlr_el1); ctxt->sys_regs[CPACR_EL1] = read_sysreg_el1(SYS_CPACR); ctxt->sys_regs[TTBR0_EL1] = read_sysreg_el1(SYS_TTBR0); ctxt->sys_regs[TTBR1_EL1] = read_sysreg_el1(SYS_TTBR1); @@ -123,7 +122,6 @@ static void __hyp_text __sysreg_restore_el1_state(struct kvm_cpu_context *ctxt) isb(); } - write_sysreg(ctxt->sys_regs[ACTLR_EL1], actlr_el1); write_sysreg_el1(ctxt->sys_regs[CPACR_EL1], SYS_CPACR); write_sysreg_el1(ctxt->sys_regs[TTBR0_EL1], SYS_TTBR0); write_sysreg_el1(ctxt->sys_regs[TTBR1_EL1], SYS_TTBR1); @@ -267,12 +265,13 @@ void __hyp_text __sysreg32_restore_state(struct kvm_vcpu *vcpu) */ void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) { - struct kvm_cpu_context *host_ctxt = vcpu->arch.host_cpu_context; struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; + struct kvm_cpu_context *host_ctxt; if (!has_vhe()) return; + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; __sysreg_save_user_state(host_ctxt); /* @@ -303,12 +302,13 @@ void kvm_vcpu_load_sysregs(struct kvm_vcpu *vcpu) */ void kvm_vcpu_put_sysregs(struct kvm_vcpu *vcpu) { - struct kvm_cpu_context *host_ctxt = vcpu->arch.host_cpu_context; struct kvm_cpu_context *guest_ctxt = &vcpu->arch.ctxt; + struct kvm_cpu_context *host_ctxt; if (!has_vhe()) return; + host_ctxt = &__hyp_this_cpu_ptr(kvm_host_data)->host_ctxt; deactivate_traps_vhe_put(); __sysreg_save_el1_state(guest_ctxt); diff --git a/arch/arm64/kvm/pmu.c b/arch/arm64/kvm/pmu.c index e71d00bb5271..b5ae3a5d509e 100644 --- a/arch/arm64/kvm/pmu.c +++ b/arch/arm64/kvm/pmu.c @@ -163,15 +163,13 @@ static void kvm_vcpu_pmu_disable_el0(unsigned long events) */ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) { - struct kvm_cpu_context *host_ctxt; struct kvm_host_data *host; u32 events_guest, events_host; if (!has_vhe()) return; - host_ctxt = vcpu->arch.host_cpu_context; - host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); + host = this_cpu_ptr(&kvm_host_data); events_guest = host->pmu_events.events_guest; events_host = host->pmu_events.events_host; @@ -184,15 +182,13 @@ void kvm_vcpu_pmu_restore_guest(struct kvm_vcpu *vcpu) */ void kvm_vcpu_pmu_restore_host(struct kvm_vcpu *vcpu) { - struct kvm_cpu_context *host_ctxt; struct kvm_host_data *host; u32 events_guest, events_host; if (!has_vhe()) return; - host_ctxt = vcpu->arch.host_cpu_context; - host = container_of(host_ctxt, struct kvm_host_data, host_ctxt); + host = this_cpu_ptr(&kvm_host_data); events_guest = host->pmu_events.events_guest; events_host = host->pmu_events.events_host; diff --git a/arch/arm64/kvm/sys_regs.c b/arch/arm64/kvm/sys_regs.c index 80985439bfb2..baf5ce9225ce 100644 --- a/arch/arm64/kvm/sys_regs.c +++ b/arch/arm64/kvm/sys_regs.c @@ -78,7 +78,6 @@ static bool __vcpu_read_sys_reg_from_cpu(int reg, u64 *val) switch (reg) { case CSSELR_EL1: *val = read_sysreg_s(SYS_CSSELR_EL1); break; case SCTLR_EL1: *val = read_sysreg_s(SYS_SCTLR_EL12); break; - case ACTLR_EL1: *val = read_sysreg_s(SYS_ACTLR_EL1); break; case CPACR_EL1: *val = read_sysreg_s(SYS_CPACR_EL12); break; case TTBR0_EL1: *val = read_sysreg_s(SYS_TTBR0_EL12); break; case TTBR1_EL1: *val = read_sysreg_s(SYS_TTBR1_EL12); break; @@ -118,7 +117,6 @@ static bool __vcpu_write_sys_reg_to_cpu(u64 val, int reg) switch (reg) { case CSSELR_EL1: write_sysreg_s(val, SYS_CSSELR_EL1); break; case SCTLR_EL1: write_sysreg_s(val, SYS_SCTLR_EL12); break; - case ACTLR_EL1: write_sysreg_s(val, SYS_ACTLR_EL1); break; case CPACR_EL1: write_sysreg_s(val, SYS_CPACR_EL12); break; case TTBR0_EL1: write_sysreg_s(val, SYS_TTBR0_EL12); break; case TTBR1_EL1: write_sysreg_s(val, SYS_TTBR1_EL12); break; @@ -1034,16 +1032,13 @@ static bool trap_ptrauth(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *rd) { - kvm_arm_vcpu_ptrauth_trap(vcpu); - /* - * Return false for both cases as we never skip the trapped - * instruction: - * - * - Either we re-execute the same key register access instruction - * after enabling ptrauth. - * - Or an UNDEF is injected as ptrauth is not supported/enabled. + * If we land here, that is because we didn't fixup the access on exit + * by allowing the PtrAuth sysregs. The only way this happens is when + * the guest does not have PtrAuth support enabled. */ + kvm_inject_undefined(vcpu); + return false; } @@ -1319,10 +1314,16 @@ static bool access_clidr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, static bool access_csselr(struct kvm_vcpu *vcpu, struct sys_reg_params *p, const struct sys_reg_desc *r) { + int reg = r->reg; + + /* See the 32bit mapping in kvm_host.h */ + if (p->is_aarch32) + reg = r->reg / 2; + if (p->is_write) - vcpu_write_sys_reg(vcpu, p->regval, r->reg); + vcpu_write_sys_reg(vcpu, p->regval, reg); else - p->regval = vcpu_read_sys_reg(vcpu, r->reg); + p->regval = vcpu_read_sys_reg(vcpu, reg); return true; } diff --git a/arch/arm64/kvm/sys_regs_generic_v8.c b/arch/arm64/kvm/sys_regs_generic_v8.c index 9cb6b4c8355a..aa9d356451eb 100644 --- a/arch/arm64/kvm/sys_regs_generic_v8.c +++ b/arch/arm64/kvm/sys_regs_generic_v8.c @@ -27,6 +27,14 @@ static bool access_actlr(struct kvm_vcpu *vcpu, return ignore_write(vcpu, p); p->regval = vcpu_read_sys_reg(vcpu, ACTLR_EL1); + + if (p->is_aarch32) { + if (r->Op2 & 2) + p->regval = upper_32_bits(p->regval); + else + p->regval = lower_32_bits(p->regval); + } + return true; } @@ -47,6 +55,8 @@ static const struct sys_reg_desc genericv8_cp15_regs[] = { /* ACTLR */ { Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b001), access_actlr }, + { Op1(0b000), CRn(0b0001), CRm(0b0000), Op2(0b011), + access_actlr }, }; static struct kvm_sys_reg_target_table genericv8_target_table = { diff --git a/arch/m68k/coldfire/pci.c b/arch/m68k/coldfire/pci.c index 62b0eb6cf69a..84eab0f5e00a 100644 --- a/arch/m68k/coldfire/pci.c +++ b/arch/m68k/coldfire/pci.c @@ -216,8 +216,10 @@ static int __init mcf_pci_init(void) /* Keep a virtual mapping to IO/config space active */ iospace = (unsigned long) ioremap(PCI_IO_PA, PCI_IO_SIZE); - if (iospace == 0) + if (iospace == 0) { + pci_free_host_bridge(bridge); return -ENODEV; + } pr_info("Coldfire: PCI IO/config window mapped to 0x%x\n", (u32) iospace); diff --git a/arch/m68k/configs/stmark2_defconfig b/arch/m68k/configs/stmark2_defconfig index 27fa9465d19d..2b746f55f419 100644 --- a/arch/m68k/configs/stmark2_defconfig +++ b/arch/m68k/configs/stmark2_defconfig @@ -48,7 +48,6 @@ CONFIG_MTD_CFI_STAA=y CONFIG_MTD_ROM=y CONFIG_MTD_COMPLEX_MAPPINGS=y CONFIG_MTD_PLATRAM=y -CONFIG_MTD_M25P80=y CONFIG_MTD_SPI_NOR=y # CONFIG_INPUT_KEYBOARD is not set # CONFIG_INPUT_MOUSE is not set diff --git a/arch/m68k/include/asm/uaccess_no.h b/arch/m68k/include/asm/uaccess_no.h index a24cfe4a0d32..dcfb69361408 100644 --- a/arch/m68k/include/asm/uaccess_no.h +++ b/arch/m68k/include/asm/uaccess_no.h @@ -42,7 +42,7 @@ static inline int _access_ok(unsigned long addr, unsigned long size) __put_user_asm(__pu_err, __pu_val, ptr, l); \ break; \ case 8: \ - memcpy(ptr, &__pu_val, sizeof (*(ptr))); \ + memcpy((void __force *)ptr, &__pu_val, sizeof(*(ptr))); \ break; \ default: \ __pu_err = __put_user_bad(); \ @@ -60,7 +60,7 @@ extern int __put_user_bad(void); * aliasing issues. */ -#define __ptr(x) ((unsigned long *)(x)) +#define __ptr(x) ((unsigned long __user *)(x)) #define __put_user_asm(err,x,ptr,bwl) \ __asm__ ("move" #bwl " %0,%1" \ @@ -85,7 +85,7 @@ extern int __put_user_bad(void); u64 l; \ __typeof__(*(ptr)) t; \ } __gu_val; \ - memcpy(&__gu_val.l, ptr, sizeof(__gu_val.l)); \ + memcpy(&__gu_val.l, (const void __force *)ptr, sizeof(__gu_val.l)); \ (x) = __gu_val.t; \ break; \ } \ diff --git a/arch/mips/Kconfig b/arch/mips/Kconfig index 7f2dbfb74e26..6fee1a133e9d 100644 --- a/arch/mips/Kconfig +++ b/arch/mips/Kconfig @@ -1403,6 +1403,7 @@ config CPU_LOONGSON64 select MIPS_L1_CACHE_SHIFT_6 select GPIOLIB select SWIOTLB + select HAVE_KVM help The Loongson GSx64(GS264/GS464/GS464E/GS464V) series of processor cores implements the MIPS64R2 instruction set with many extensions, diff --git a/arch/mips/include/asm/cpu-features.h b/arch/mips/include/asm/cpu-features.h index caecbae4b599..724dfddcab92 100644 --- a/arch/mips/include/asm/cpu-features.h +++ b/arch/mips/include/asm/cpu-features.h @@ -682,6 +682,9 @@ #ifndef cpu_guest_has_htw #define cpu_guest_has_htw (cpu_data[0].guest.options & MIPS_CPU_HTW) #endif +#ifndef cpu_guest_has_ldpte +#define cpu_guest_has_ldpte (cpu_data[0].guest.options & MIPS_CPU_LDPTE) +#endif #ifndef cpu_guest_has_mvh #define cpu_guest_has_mvh (cpu_data[0].guest.options & MIPS_CPU_MVH) #endif diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h index e28b5a946e26..363e7a89d173 100644 --- a/arch/mips/include/asm/kvm_host.h +++ b/arch/mips/include/asm/kvm_host.h @@ -23,6 +23,8 @@ #include <asm/inst.h> #include <asm/mipsregs.h> +#include <kvm/iodev.h> + /* MIPS KVM register ids */ #define MIPS_CP0_32(_R, _S) \ (KVM_REG_MIPS_CP0 | KVM_REG_SIZE_U32 | (8 * (_R) + (_S))) @@ -66,9 +68,11 @@ #define KVM_REG_MIPS_CP0_CONFIG3 MIPS_CP0_32(16, 3) #define KVM_REG_MIPS_CP0_CONFIG4 MIPS_CP0_32(16, 4) #define KVM_REG_MIPS_CP0_CONFIG5 MIPS_CP0_32(16, 5) +#define KVM_REG_MIPS_CP0_CONFIG6 MIPS_CP0_32(16, 6) #define KVM_REG_MIPS_CP0_CONFIG7 MIPS_CP0_32(16, 7) #define KVM_REG_MIPS_CP0_MAARI MIPS_CP0_64(17, 2) #define KVM_REG_MIPS_CP0_XCONTEXT MIPS_CP0_64(20, 0) +#define KVM_REG_MIPS_CP0_DIAG MIPS_CP0_32(22, 0) #define KVM_REG_MIPS_CP0_ERROREPC MIPS_CP0_64(30, 0) #define KVM_REG_MIPS_CP0_KSCRATCH1 MIPS_CP0_64(31, 2) #define KVM_REG_MIPS_CP0_KSCRATCH2 MIPS_CP0_64(31, 3) @@ -78,8 +82,8 @@ #define KVM_REG_MIPS_CP0_KSCRATCH6 MIPS_CP0_64(31, 7) -#define KVM_MAX_VCPUS 8 -#define KVM_USER_MEM_SLOTS 8 +#define KVM_MAX_VCPUS 16 +#define KVM_USER_MEM_SLOTS 16 /* memory slots that does not exposed to userspace */ #define KVM_PRIVATE_MEM_SLOTS 0 @@ -171,6 +175,9 @@ struct kvm_vcpu_stat { u64 vz_ghfc_exits; u64 vz_gpa_exits; u64 vz_resvd_exits; +#ifdef CONFIG_CPU_LOONGSON64 + u64 vz_cpucfg_exits; +#endif #endif u64 halt_successful_poll; u64 halt_attempted_poll; @@ -183,11 +190,39 @@ struct kvm_vcpu_stat { struct kvm_arch_memory_slot { }; +#ifdef CONFIG_CPU_LOONGSON64 +struct ipi_state { + uint32_t status; + uint32_t en; + uint32_t set; + uint32_t clear; + uint64_t buf[4]; +}; + +struct loongson_kvm_ipi; + +struct ipi_io_device { + int node_id; + struct loongson_kvm_ipi *ipi; + struct kvm_io_device device; +}; + +struct loongson_kvm_ipi { + spinlock_t lock; + struct kvm *kvm; + struct ipi_state ipistate[16]; + struct ipi_io_device dev_ipi[4]; +}; +#endif + struct kvm_arch { /* Guest physical mm */ struct mm_struct gpa_mm; /* Mask of CPUs needing GPA ASID flush */ cpumask_t asid_flush_mask; +#ifdef CONFIG_CPU_LOONGSON64 + struct loongson_kvm_ipi ipi; +#endif }; #define N_MIPS_COPROC_REGS 32 @@ -225,6 +260,7 @@ struct mips_coproc { #define MIPS_CP0_WATCH_LO 18 #define MIPS_CP0_WATCH_HI 19 #define MIPS_CP0_TLB_XCONTEXT 20 +#define MIPS_CP0_DIAG 22 #define MIPS_CP0_ECC 26 #define MIPS_CP0_CACHE_ERR 27 #define MIPS_CP0_TAG_LO 28 @@ -276,8 +312,12 @@ enum emulation_result { #define MIPS3_PG_SHIFT 6 #define MIPS3_PG_FRAME 0x3fffffc0 +#if defined(CONFIG_64BIT) +#define VPN2_MASK GENMASK(cpu_vmbits - 1, 13) +#else #define VPN2_MASK 0xffffe000 -#define KVM_ENTRYHI_ASID MIPS_ENTRYHI_ASID +#endif +#define KVM_ENTRYHI_ASID cpu_asid_mask(&boot_cpu_data) #define TLB_IS_GLOBAL(x) ((x).tlb_lo[0] & (x).tlb_lo[1] & ENTRYLO_G) #define TLB_VPN2(x) ((x).tlb_hi & VPN2_MASK) #define TLB_ASID(x) ((x).tlb_hi & KVM_ENTRYHI_ASID) @@ -892,6 +932,10 @@ void kvm_vz_save_guesttlb(struct kvm_mips_tlb *buf, unsigned int index, unsigned int count); void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index, unsigned int count); +#ifdef CONFIG_CPU_LOONGSON64 +void kvm_loongson_clear_guest_vtlb(void); +void kvm_loongson_clear_guest_ftlb(void); +#endif #endif void kvm_mips_suspend_mm(int cpu); @@ -1131,6 +1175,8 @@ extern int kvm_mips_trans_mtc0(union mips_instruction inst, u32 *opc, /* Misc */ extern void kvm_mips_dump_stats(struct kvm_vcpu *vcpu); extern unsigned long kvm_mips_get_ramsize(struct kvm *kvm); +extern int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, + struct kvm_mips_interrupt *irq); static inline void kvm_arch_hardware_unsetup(void) {} static inline void kvm_arch_sync_events(struct kvm *kvm) {} diff --git a/arch/mips/include/asm/mipsregs.h b/arch/mips/include/asm/mipsregs.h index 796dbb86575b..20d6d40c59a4 100644 --- a/arch/mips/include/asm/mipsregs.h +++ b/arch/mips/include/asm/mipsregs.h @@ -1038,6 +1038,8 @@ /* Disable Branch Return Cache */ #define R10K_DIAG_D_BRC (_ULCAST_(1) << 22) +/* Flush BTB */ +#define LOONGSON_DIAG_BTB (_ULCAST_(1) << 1) /* Flush ITLB */ #define LOONGSON_DIAG_ITLB (_ULCAST_(1) << 2) /* Flush DTLB */ @@ -2874,7 +2876,9 @@ __BUILD_SET_C0(status) __BUILD_SET_C0(cause) __BUILD_SET_C0(config) __BUILD_SET_C0(config5) +__BUILD_SET_C0(config6) __BUILD_SET_C0(config7) +__BUILD_SET_C0(diag) __BUILD_SET_C0(intcontrol) __BUILD_SET_C0(intctl) __BUILD_SET_C0(srsmap) diff --git a/arch/mips/include/uapi/asm/inst.h b/arch/mips/include/uapi/asm/inst.h index 98f97c85e059..43d1faa02933 100644 --- a/arch/mips/include/uapi/asm/inst.h +++ b/arch/mips/include/uapi/asm/inst.h @@ -1012,6 +1012,16 @@ struct loongson3_lsdc2_format { /* Loongson-3 overridden ldc2/sdc2 Load/Store fo ;)))))) }; +struct loongson3_lscsr_format { /* Loongson-3 CPUCFG&CSR read/write format */ + __BITFIELD_FIELD(unsigned int opcode : 6, + __BITFIELD_FIELD(unsigned int rs : 5, + __BITFIELD_FIELD(unsigned int fr : 5, + __BITFIELD_FIELD(unsigned int rd : 5, + __BITFIELD_FIELD(unsigned int fd : 5, + __BITFIELD_FIELD(unsigned int func : 6, + ;)))))) +}; + /* * MIPS16e instruction formats (16-bit length) */ @@ -1114,6 +1124,7 @@ union mips_instruction { struct mm16_r5_format mm16_r5_format; struct loongson3_lswc2_format loongson3_lswc2_format; struct loongson3_lsdc2_format loongson3_lsdc2_format; + struct loongson3_lscsr_format loongson3_lscsr_format; }; union mips16e_instruction { diff --git a/arch/mips/kernel/cpu-probe.c b/arch/mips/kernel/cpu-probe.c index 6b93162d7c5a..def1659fe262 100644 --- a/arch/mips/kernel/cpu-probe.c +++ b/arch/mips/kernel/cpu-probe.c @@ -2017,8 +2017,10 @@ static inline void decode_cpucfg(struct cpuinfo_mips *c) if (cfg2 & LOONGSON_CFG2_LEXT2) c->ases |= MIPS_ASE_LOONGSON_EXT2; - if (cfg2 & LOONGSON_CFG2_LSPW) + if (cfg2 & LOONGSON_CFG2_LSPW) { c->options |= MIPS_CPU_LDPTE; + c->guest.options |= MIPS_CPU_LDPTE; + } if (cfg3 & LOONGSON_CFG3_LCAMP) c->ases |= MIPS_ASE_LOONGSON_CAM; @@ -2074,6 +2076,7 @@ static inline void cpu_probe_loongson(struct cpuinfo_mips *c, unsigned int cpu) c->writecombine = _CACHE_UNCACHED_ACCELERATED; c->ases |= (MIPS_ASE_LOONGSON_MMI | MIPS_ASE_LOONGSON_CAM | MIPS_ASE_LOONGSON_EXT | MIPS_ASE_LOONGSON_EXT2); + c->ases &= ~MIPS_ASE_VZ; /* VZ of Loongson-3A2000/3000 is incomplete */ break; case PRID_IMP_LOONGSON_64G: c->cputype = CPU_LOONGSON64; diff --git a/arch/mips/kvm/Kconfig b/arch/mips/kvm/Kconfig index 6f5930285fd5..2bf02d849a3a 100644 --- a/arch/mips/kvm/Kconfig +++ b/arch/mips/kvm/Kconfig @@ -22,6 +22,7 @@ config KVM select EXPORT_UASM select PREEMPT_NOTIFIERS select KVM_GENERIC_DIRTYLOG_READ_PROTECT + select HAVE_KVM_EVENTFD select HAVE_KVM_VCPU_ASYNC_IOCTL select KVM_MMIO select MMU_NOTIFIER diff --git a/arch/mips/kvm/Makefile b/arch/mips/kvm/Makefile index 01affc1d21c5..506c4ac0ba1c 100644 --- a/arch/mips/kvm/Makefile +++ b/arch/mips/kvm/Makefile @@ -2,7 +2,7 @@ # Makefile for KVM support for MIPS # -common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o) +common-objs-y = $(addprefix ../../../virt/kvm/, kvm_main.o coalesced_mmio.o eventfd.o) EXTRA_CFLAGS += -Ivirt/kvm -Iarch/mips/kvm @@ -13,6 +13,9 @@ kvm-objs := $(common-objs-y) mips.o emulate.o entry.o \ fpu.o kvm-objs += hypcall.o kvm-objs += mmu.o +ifdef CONFIG_CPU_LOONGSON64 +kvm-objs += loongson_ipi.o +endif ifdef CONFIG_KVM_MIPS_VZ kvm-objs += vz.o diff --git a/arch/mips/kvm/emulate.c b/arch/mips/kvm/emulate.c index 7ccf9b096783..5ae82d925197 100644 --- a/arch/mips/kvm/emulate.c +++ b/arch/mips/kvm/emulate.c @@ -1600,9 +1600,11 @@ enum emulation_result kvm_mips_emulate_store(union mips_instruction inst, struct kvm_run *run, struct kvm_vcpu *vcpu) { + int r; enum emulation_result er; u32 rt; void *data = run->mmio.data; + unsigned int imme; unsigned long curr_pc; /* @@ -1660,15 +1662,229 @@ enum emulation_result kvm_mips_emulate_store(union mips_instruction inst, vcpu->arch.gprs[rt], *(u8 *)data); break; + case swl_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x3); + run->mmio.len = 4; + imme = vcpu->arch.host_cp0_badvaddr & 0x3; + switch (imme) { + case 0: + *(u32 *)data = ((*(u32 *)data) & 0xffffff00) | + (vcpu->arch.gprs[rt] >> 24); + break; + case 1: + *(u32 *)data = ((*(u32 *)data) & 0xffff0000) | + (vcpu->arch.gprs[rt] >> 16); + break; + case 2: + *(u32 *)data = ((*(u32 *)data) & 0xff000000) | + (vcpu->arch.gprs[rt] >> 8); + break; + case 3: + *(u32 *)data = vcpu->arch.gprs[rt]; + break; + default: + break; + } + + kvm_debug("[%#lx] OP_SWL: eaddr: %#lx, gpr: %#lx, data: %#x\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u32 *)data); + break; + + case swr_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x3); + run->mmio.len = 4; + imme = vcpu->arch.host_cp0_badvaddr & 0x3; + switch (imme) { + case 0: + *(u32 *)data = vcpu->arch.gprs[rt]; + break; + case 1: + *(u32 *)data = ((*(u32 *)data) & 0xff) | + (vcpu->arch.gprs[rt] << 8); + break; + case 2: + *(u32 *)data = ((*(u32 *)data) & 0xffff) | + (vcpu->arch.gprs[rt] << 16); + break; + case 3: + *(u32 *)data = ((*(u32 *)data) & 0xffffff) | + (vcpu->arch.gprs[rt] << 24); + break; + default: + break; + } + + kvm_debug("[%#lx] OP_SWR: eaddr: %#lx, gpr: %#lx, data: %#x\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u32 *)data); + break; + + case sdl_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x7); + + run->mmio.len = 8; + imme = vcpu->arch.host_cp0_badvaddr & 0x7; + switch (imme) { + case 0: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffffffff00) | + ((vcpu->arch.gprs[rt] >> 56) & 0xff); + break; + case 1: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffffff0000) | + ((vcpu->arch.gprs[rt] >> 48) & 0xffff); + break; + case 2: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffff000000) | + ((vcpu->arch.gprs[rt] >> 40) & 0xffffff); + break; + case 3: + *(u64 *)data = ((*(u64 *)data) & 0xffffffff00000000) | + ((vcpu->arch.gprs[rt] >> 32) & 0xffffffff); + break; + case 4: + *(u64 *)data = ((*(u64 *)data) & 0xffffff0000000000) | + ((vcpu->arch.gprs[rt] >> 24) & 0xffffffffff); + break; + case 5: + *(u64 *)data = ((*(u64 *)data) & 0xffff000000000000) | + ((vcpu->arch.gprs[rt] >> 16) & 0xffffffffffff); + break; + case 6: + *(u64 *)data = ((*(u64 *)data) & 0xff00000000000000) | + ((vcpu->arch.gprs[rt] >> 8) & 0xffffffffffffff); + break; + case 7: + *(u64 *)data = vcpu->arch.gprs[rt]; + break; + default: + break; + } + + kvm_debug("[%#lx] OP_SDL: eaddr: %#lx, gpr: %#lx, data: %llx\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u64 *)data); + break; + + case sdr_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x7); + + run->mmio.len = 8; + imme = vcpu->arch.host_cp0_badvaddr & 0x7; + switch (imme) { + case 0: + *(u64 *)data = vcpu->arch.gprs[rt]; + break; + case 1: + *(u64 *)data = ((*(u64 *)data) & 0xff) | + (vcpu->arch.gprs[rt] << 8); + break; + case 2: + *(u64 *)data = ((*(u64 *)data) & 0xffff) | + (vcpu->arch.gprs[rt] << 16); + break; + case 3: + *(u64 *)data = ((*(u64 *)data) & 0xffffff) | + (vcpu->arch.gprs[rt] << 24); + break; + case 4: + *(u64 *)data = ((*(u64 *)data) & 0xffffffff) | + (vcpu->arch.gprs[rt] << 32); + break; + case 5: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffff) | + (vcpu->arch.gprs[rt] << 40); + break; + case 6: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffffff) | + (vcpu->arch.gprs[rt] << 48); + break; + case 7: + *(u64 *)data = ((*(u64 *)data) & 0xffffffffffffff) | + (vcpu->arch.gprs[rt] << 56); + break; + default: + break; + } + + kvm_debug("[%#lx] OP_SDR: eaddr: %#lx, gpr: %#lx, data: %llx\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u64 *)data); + break; + +#ifdef CONFIG_CPU_LOONGSON64 + case sdc2_op: + rt = inst.loongson3_lsdc2_format.rt; + switch (inst.loongson3_lsdc2_format.opcode1) { + /* + * Loongson-3 overridden sdc2 instructions. + * opcode1 instruction + * 0x0 gssbx: store 1 bytes from GPR + * 0x1 gsshx: store 2 bytes from GPR + * 0x2 gsswx: store 4 bytes from GPR + * 0x3 gssdx: store 8 bytes from GPR + */ + case 0x0: + run->mmio.len = 1; + *(u8 *)data = vcpu->arch.gprs[rt]; + + kvm_debug("[%#lx] OP_GSSBX: eaddr: %#lx, gpr: %#lx, data: %#x\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u8 *)data); + break; + case 0x1: + run->mmio.len = 2; + *(u16 *)data = vcpu->arch.gprs[rt]; + + kvm_debug("[%#lx] OP_GSSSHX: eaddr: %#lx, gpr: %#lx, data: %#x\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u16 *)data); + break; + case 0x2: + run->mmio.len = 4; + *(u32 *)data = vcpu->arch.gprs[rt]; + + kvm_debug("[%#lx] OP_GSSWX: eaddr: %#lx, gpr: %#lx, data: %#x\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u32 *)data); + break; + case 0x3: + run->mmio.len = 8; + *(u64 *)data = vcpu->arch.gprs[rt]; + + kvm_debug("[%#lx] OP_GSSDX: eaddr: %#lx, gpr: %#lx, data: %#llx\n", + vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, + vcpu->arch.gprs[rt], *(u64 *)data); + break; + default: + kvm_err("Godson Exteneded GS-Store not yet supported (inst=0x%08x)\n", + inst.word); + break; + } + break; +#endif default: kvm_err("Store not yet supported (inst=0x%08x)\n", inst.word); goto out_fail; } - run->mmio.is_write = 1; vcpu->mmio_needed = 1; + run->mmio.is_write = 1; vcpu->mmio_is_write = 1; + + r = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, + run->mmio.phys_addr, run->mmio.len, data); + + if (!r) { + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + return EMULATE_DO_MMIO; out_fail: @@ -1681,9 +1897,11 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, u32 cause, struct kvm_run *run, struct kvm_vcpu *vcpu) { + int r; enum emulation_result er; unsigned long curr_pc; u32 op, rt; + unsigned int imme; rt = inst.i_format.rt; op = inst.i_format.opcode; @@ -1736,6 +1954,162 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, run->mmio.len = 1; break; + case lwl_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x3); + + run->mmio.len = 4; + imme = vcpu->arch.host_cp0_badvaddr & 0x3; + switch (imme) { + case 0: + vcpu->mmio_needed = 3; /* 1 byte */ + break; + case 1: + vcpu->mmio_needed = 4; /* 2 bytes */ + break; + case 2: + vcpu->mmio_needed = 5; /* 3 bytes */ + break; + case 3: + vcpu->mmio_needed = 6; /* 4 bytes */ + break; + default: + break; + } + break; + + case lwr_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x3); + + run->mmio.len = 4; + imme = vcpu->arch.host_cp0_badvaddr & 0x3; + switch (imme) { + case 0: + vcpu->mmio_needed = 7; /* 4 bytes */ + break; + case 1: + vcpu->mmio_needed = 8; /* 3 bytes */ + break; + case 2: + vcpu->mmio_needed = 9; /* 2 bytes */ + break; + case 3: + vcpu->mmio_needed = 10; /* 1 byte */ + break; + default: + break; + } + break; + + case ldl_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x7); + + run->mmio.len = 8; + imme = vcpu->arch.host_cp0_badvaddr & 0x7; + switch (imme) { + case 0: + vcpu->mmio_needed = 11; /* 1 byte */ + break; + case 1: + vcpu->mmio_needed = 12; /* 2 bytes */ + break; + case 2: + vcpu->mmio_needed = 13; /* 3 bytes */ + break; + case 3: + vcpu->mmio_needed = 14; /* 4 bytes */ + break; + case 4: + vcpu->mmio_needed = 15; /* 5 bytes */ + break; + case 5: + vcpu->mmio_needed = 16; /* 6 bytes */ + break; + case 6: + vcpu->mmio_needed = 17; /* 7 bytes */ + break; + case 7: + vcpu->mmio_needed = 18; /* 8 bytes */ + break; + default: + break; + } + break; + + case ldr_op: + run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( + vcpu->arch.host_cp0_badvaddr) & (~0x7); + + run->mmio.len = 8; + imme = vcpu->arch.host_cp0_badvaddr & 0x7; + switch (imme) { + case 0: + vcpu->mmio_needed = 19; /* 8 bytes */ + break; + case 1: + vcpu->mmio_needed = 20; /* 7 bytes */ + break; + case 2: + vcpu->mmio_needed = 21; /* 6 bytes */ + break; + case 3: + vcpu->mmio_needed = 22; /* 5 bytes */ + break; + case 4: + vcpu->mmio_needed = 23; /* 4 bytes */ + break; + case 5: + vcpu->mmio_needed = 24; /* 3 bytes */ + break; + case 6: + vcpu->mmio_needed = 25; /* 2 bytes */ + break; + case 7: + vcpu->mmio_needed = 26; /* 1 byte */ + break; + default: + break; + } + break; + +#ifdef CONFIG_CPU_LOONGSON64 + case ldc2_op: + rt = inst.loongson3_lsdc2_format.rt; + switch (inst.loongson3_lsdc2_format.opcode1) { + /* + * Loongson-3 overridden ldc2 instructions. + * opcode1 instruction + * 0x0 gslbx: store 1 bytes from GPR + * 0x1 gslhx: store 2 bytes from GPR + * 0x2 gslwx: store 4 bytes from GPR + * 0x3 gsldx: store 8 bytes from GPR + */ + case 0x0: + run->mmio.len = 1; + vcpu->mmio_needed = 27; /* signed */ + break; + case 0x1: + run->mmio.len = 2; + vcpu->mmio_needed = 28; /* signed */ + break; + case 0x2: + run->mmio.len = 4; + vcpu->mmio_needed = 29; /* signed */ + break; + case 0x3: + run->mmio.len = 8; + vcpu->mmio_needed = 30; /* signed */ + break; + default: + kvm_err("Godson Exteneded GS-Load for float not yet supported (inst=0x%08x)\n", + inst.word); + break; + } + break; +#endif + default: kvm_err("Load not yet supported (inst=0x%08x)\n", inst.word); @@ -1745,6 +2119,16 @@ enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, run->mmio.is_write = 0; vcpu->mmio_is_write = 0; + + r = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, + run->mmio.phys_addr, run->mmio.len, run->mmio.data); + + if (!r) { + kvm_mips_complete_mmio_load(vcpu, run); + vcpu->mmio_needed = 0; + return EMULATE_DONE; + } + return EMULATE_DO_MMIO; } @@ -2591,28 +2975,125 @@ enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu, switch (run->mmio.len) { case 8: - *gpr = *(s64 *)run->mmio.data; + switch (vcpu->mmio_needed) { + case 11: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffffff) | + (((*(s64 *)run->mmio.data) & 0xff) << 56); + break; + case 12: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffff) | + (((*(s64 *)run->mmio.data) & 0xffff) << 48); + break; + case 13: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffff) | + (((*(s64 *)run->mmio.data) & 0xffffff) << 40); + break; + case 14: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffff) | + (((*(s64 *)run->mmio.data) & 0xffffffff) << 32); + break; + case 15: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff) | + (((*(s64 *)run->mmio.data) & 0xffffffffff) << 24); + break; + case 16: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff) | + (((*(s64 *)run->mmio.data) & 0xffffffffffff) << 16); + break; + case 17: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff) | + (((*(s64 *)run->mmio.data) & 0xffffffffffffff) << 8); + break; + case 18: + case 19: + *gpr = *(s64 *)run->mmio.data; + break; + case 20: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff00000000000000) | + ((((*(s64 *)run->mmio.data)) >> 8) & 0xffffffffffffff); + break; + case 21: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff000000000000) | + ((((*(s64 *)run->mmio.data)) >> 16) & 0xffffffffffff); + break; + case 22: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff0000000000) | + ((((*(s64 *)run->mmio.data)) >> 24) & 0xffffffffff); + break; + case 23: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffff00000000) | + ((((*(s64 *)run->mmio.data)) >> 32) & 0xffffffff); + break; + case 24: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffff000000) | + ((((*(s64 *)run->mmio.data)) >> 40) & 0xffffff); + break; + case 25: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffff0000) | + ((((*(s64 *)run->mmio.data)) >> 48) & 0xffff); + break; + case 26: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffffff00) | + ((((*(s64 *)run->mmio.data)) >> 56) & 0xff); + break; + default: + *gpr = *(s64 *)run->mmio.data; + } break; case 4: - if (vcpu->mmio_needed == 2) - *gpr = *(s32 *)run->mmio.data; - else + switch (vcpu->mmio_needed) { + case 1: *gpr = *(u32 *)run->mmio.data; + break; + case 2: + *gpr = *(s32 *)run->mmio.data; + break; + case 3: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff) | + (((*(s32 *)run->mmio.data) & 0xff) << 24); + break; + case 4: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff) | + (((*(s32 *)run->mmio.data) & 0xffff) << 16); + break; + case 5: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff) | + (((*(s32 *)run->mmio.data) & 0xffffff) << 8); + break; + case 6: + case 7: + *gpr = *(s32 *)run->mmio.data; + break; + case 8: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff000000) | + ((((*(s32 *)run->mmio.data)) >> 8) & 0xffffff); + break; + case 9: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff0000) | + ((((*(s32 *)run->mmio.data)) >> 16) & 0xffff); + break; + case 10: + *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff00) | + ((((*(s32 *)run->mmio.data)) >> 24) & 0xff); + break; + default: + *gpr = *(s32 *)run->mmio.data; + } break; case 2: - if (vcpu->mmio_needed == 2) - *gpr = *(s16 *) run->mmio.data; - else + if (vcpu->mmio_needed == 1) *gpr = *(u16 *)run->mmio.data; + else + *gpr = *(s16 *)run->mmio.data; break; case 1: - if (vcpu->mmio_needed == 2) - *gpr = *(s8 *) run->mmio.data; + if (vcpu->mmio_needed == 1) + *gpr = *(u8 *)run->mmio.data; else - *gpr = *(u8 *) run->mmio.data; + *gpr = *(s8 *)run->mmio.data; break; } diff --git a/arch/mips/kvm/entry.c b/arch/mips/kvm/entry.c index 16e1c93b484f..fd716942e302 100644 --- a/arch/mips/kvm/entry.c +++ b/arch/mips/kvm/entry.c @@ -56,6 +56,7 @@ #define C0_BADVADDR 8, 0 #define C0_BADINSTR 8, 1 #define C0_BADINSTRP 8, 2 +#define C0_PGD 9, 7 #define C0_ENTRYHI 10, 0 #define C0_GUESTCTL1 10, 4 #define C0_STATUS 12, 0 @@ -307,7 +308,10 @@ static void *kvm_mips_build_enter_guest(void *addr) #ifdef CONFIG_KVM_MIPS_VZ /* Save normal linux process pgd (VZ guarantees pgd_reg is set) */ - UASM_i_MFC0(&p, K0, c0_kscratch(), pgd_reg); + if (cpu_has_ldpte) + UASM_i_MFC0(&p, K0, C0_PWBASE); + else + UASM_i_MFC0(&p, K0, c0_kscratch(), pgd_reg); UASM_i_SW(&p, K0, offsetof(struct kvm_vcpu_arch, host_pgd), K1); /* @@ -469,8 +473,10 @@ void *kvm_mips_build_tlb_refill_exception(void *addr, void *handler) u32 *p = addr; struct uasm_label labels[2]; struct uasm_reloc relocs[2]; +#ifndef CONFIG_CPU_LOONGSON64 struct uasm_label *l = labels; struct uasm_reloc *r = relocs; +#endif memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); @@ -490,6 +496,16 @@ void *kvm_mips_build_tlb_refill_exception(void *addr, void *handler) */ preempt_disable(); +#ifdef CONFIG_CPU_LOONGSON64 + UASM_i_MFC0(&p, K1, C0_PGD); + uasm_i_lddir(&p, K0, K1, 3); /* global page dir */ +#ifndef __PAGETABLE_PMD_FOLDED + uasm_i_lddir(&p, K1, K0, 1); /* middle page dir */ +#endif + uasm_i_ldpte(&p, K1, 0); /* even */ + uasm_i_ldpte(&p, K1, 1); /* odd */ + uasm_i_tlbwr(&p); +#else /* * Now for the actual refill bit. A lot of this can be common with the * Linux TLB refill handler, however we don't need to handle so many @@ -512,6 +528,7 @@ void *kvm_mips_build_tlb_refill_exception(void *addr, void *handler) build_get_ptep(&p, K0, K1); build_update_entries(&p, K0, K1); build_tlb_write_entry(&p, &l, &r, tlb_random); +#endif preempt_enable(); diff --git a/arch/mips/kvm/interrupt.c b/arch/mips/kvm/interrupt.c index 7257e8b6f5a9..d28c2c9c343e 100644 --- a/arch/mips/kvm/interrupt.c +++ b/arch/mips/kvm/interrupt.c @@ -61,27 +61,8 @@ void kvm_mips_queue_io_int_cb(struct kvm_vcpu *vcpu, * the EXC code will be set when we are actually * delivering the interrupt: */ - switch (intr) { - case 2: - kvm_set_c0_guest_cause(vcpu->arch.cop0, (C_IRQ0)); - /* Queue up an INT exception for the core */ - kvm_mips_queue_irq(vcpu, MIPS_EXC_INT_IO); - break; - - case 3: - kvm_set_c0_guest_cause(vcpu->arch.cop0, (C_IRQ1)); - kvm_mips_queue_irq(vcpu, MIPS_EXC_INT_IPI_1); - break; - - case 4: - kvm_set_c0_guest_cause(vcpu->arch.cop0, (C_IRQ2)); - kvm_mips_queue_irq(vcpu, MIPS_EXC_INT_IPI_2); - break; - - default: - break; - } - + kvm_set_c0_guest_cause(vcpu->arch.cop0, 1 << (intr + 8)); + kvm_mips_queue_irq(vcpu, kvm_irq_to_priority(intr)); } void kvm_mips_dequeue_io_int_cb(struct kvm_vcpu *vcpu, @@ -89,26 +70,8 @@ void kvm_mips_dequeue_io_int_cb(struct kvm_vcpu *vcpu, { int intr = (int)irq->irq; - switch (intr) { - case -2: - kvm_clear_c0_guest_cause(vcpu->arch.cop0, (C_IRQ0)); - kvm_mips_dequeue_irq(vcpu, MIPS_EXC_INT_IO); - break; - - case -3: - kvm_clear_c0_guest_cause(vcpu->arch.cop0, (C_IRQ1)); - kvm_mips_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_1); - break; - - case -4: - kvm_clear_c0_guest_cause(vcpu->arch.cop0, (C_IRQ2)); - kvm_mips_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_2); - break; - - default: - break; - } - + kvm_clear_c0_guest_cause(vcpu->arch.cop0, 1 << (-intr + 8)); + kvm_mips_dequeue_irq(vcpu, kvm_irq_to_priority(-intr)); } /* Deliver the interrupt of the corresponding priority, if possible. */ @@ -116,50 +79,20 @@ int kvm_mips_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, u32 cause) { int allowed = 0; - u32 exccode; + u32 exccode, ie; struct kvm_vcpu_arch *arch = &vcpu->arch; struct mips_coproc *cop0 = vcpu->arch.cop0; - switch (priority) { - case MIPS_EXC_INT_TIMER: - if ((kvm_read_c0_guest_status(cop0) & ST0_IE) - && (!(kvm_read_c0_guest_status(cop0) & (ST0_EXL | ST0_ERL))) - && (kvm_read_c0_guest_status(cop0) & IE_IRQ5)) { - allowed = 1; - exccode = EXCCODE_INT; - } - break; - - case MIPS_EXC_INT_IO: - if ((kvm_read_c0_guest_status(cop0) & ST0_IE) - && (!(kvm_read_c0_guest_status(cop0) & (ST0_EXL | ST0_ERL))) - && (kvm_read_c0_guest_status(cop0) & IE_IRQ0)) { - allowed = 1; - exccode = EXCCODE_INT; - } - break; - - case MIPS_EXC_INT_IPI_1: - if ((kvm_read_c0_guest_status(cop0) & ST0_IE) - && (!(kvm_read_c0_guest_status(cop0) & (ST0_EXL | ST0_ERL))) - && (kvm_read_c0_guest_status(cop0) & IE_IRQ1)) { - allowed = 1; - exccode = EXCCODE_INT; - } - break; - - case MIPS_EXC_INT_IPI_2: - if ((kvm_read_c0_guest_status(cop0) & ST0_IE) - && (!(kvm_read_c0_guest_status(cop0) & (ST0_EXL | ST0_ERL))) - && (kvm_read_c0_guest_status(cop0) & IE_IRQ2)) { - allowed = 1; - exccode = EXCCODE_INT; - } - break; + if (priority == MIPS_EXC_MAX) + return 0; - default: - break; + ie = 1 << (kvm_priority_to_irq[priority] + 8); + if ((kvm_read_c0_guest_status(cop0) & ST0_IE) + && (!(kvm_read_c0_guest_status(cop0) & (ST0_EXL | ST0_ERL))) + && (kvm_read_c0_guest_status(cop0) & ie)) { + allowed = 1; + exccode = EXCCODE_INT; } /* Are we allowed to deliver the interrupt ??? */ diff --git a/arch/mips/kvm/interrupt.h b/arch/mips/kvm/interrupt.h index 3bf0a49725e8..c3e878ca3e07 100644 --- a/arch/mips/kvm/interrupt.h +++ b/arch/mips/kvm/interrupt.h @@ -21,11 +21,12 @@ #define MIPS_EXC_NMI 5 #define MIPS_EXC_MCHK 6 #define MIPS_EXC_INT_TIMER 7 -#define MIPS_EXC_INT_IO 8 -#define MIPS_EXC_EXECUTE 9 -#define MIPS_EXC_INT_IPI_1 10 -#define MIPS_EXC_INT_IPI_2 11 -#define MIPS_EXC_MAX 12 +#define MIPS_EXC_INT_IO_1 8 +#define MIPS_EXC_INT_IO_2 9 +#define MIPS_EXC_EXECUTE 10 +#define MIPS_EXC_INT_IPI_1 11 +#define MIPS_EXC_INT_IPI_2 12 +#define MIPS_EXC_MAX 13 /* XXXSL More to follow */ #define C_TI (_ULCAST_(1) << 30) @@ -38,6 +39,9 @@ #define KVM_MIPS_IRQ_CLEAR_ALL_AT_ONCE (0) #endif +extern u32 *kvm_priority_to_irq; +u32 kvm_irq_to_priority(u32 irq); + void kvm_mips_queue_irq(struct kvm_vcpu *vcpu, unsigned int priority); void kvm_mips_dequeue_irq(struct kvm_vcpu *vcpu, unsigned int priority); int kvm_mips_pending_timer(struct kvm_vcpu *vcpu); diff --git a/arch/mips/kvm/loongson_ipi.c b/arch/mips/kvm/loongson_ipi.c new file mode 100644 index 000000000000..3681fc8fba38 --- /dev/null +++ b/arch/mips/kvm/loongson_ipi.c @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Loongson-3 Virtual IPI interrupt support. + * + * Copyright (C) 2019 Loongson Technologies, Inc. All rights reserved. + * + * Authors: Chen Zhu <zhuchen@loongson.cn> + * Authors: Huacai Chen <chenhc@lemote.com> + */ + +#include <linux/kvm_host.h> + +#define IPI_BASE 0x3ff01000ULL + +#define CORE0_STATUS_OFF 0x000 +#define CORE0_EN_OFF 0x004 +#define CORE0_SET_OFF 0x008 +#define CORE0_CLEAR_OFF 0x00c +#define CORE0_BUF_20 0x020 +#define CORE0_BUF_28 0x028 +#define CORE0_BUF_30 0x030 +#define CORE0_BUF_38 0x038 + +#define CORE1_STATUS_OFF 0x100 +#define CORE1_EN_OFF 0x104 +#define CORE1_SET_OFF 0x108 +#define CORE1_CLEAR_OFF 0x10c +#define CORE1_BUF_20 0x120 +#define CORE1_BUF_28 0x128 +#define CORE1_BUF_30 0x130 +#define CORE1_BUF_38 0x138 + +#define CORE2_STATUS_OFF 0x200 +#define CORE2_EN_OFF 0x204 +#define CORE2_SET_OFF 0x208 +#define CORE2_CLEAR_OFF 0x20c +#define CORE2_BUF_20 0x220 +#define CORE2_BUF_28 0x228 +#define CORE2_BUF_30 0x230 +#define CORE2_BUF_38 0x238 + +#define CORE3_STATUS_OFF 0x300 +#define CORE3_EN_OFF 0x304 +#define CORE3_SET_OFF 0x308 +#define CORE3_CLEAR_OFF 0x30c +#define CORE3_BUF_20 0x320 +#define CORE3_BUF_28 0x328 +#define CORE3_BUF_30 0x330 +#define CORE3_BUF_38 0x338 + +static int loongson_vipi_read(struct loongson_kvm_ipi *ipi, + gpa_t addr, int len, void *val) +{ + uint32_t core = (addr >> 8) & 3; + uint32_t node = (addr >> 44) & 3; + uint32_t id = core + node * 4; + uint64_t offset = addr & 0xff; + void *pbuf; + struct ipi_state *s = &(ipi->ipistate[id]); + + BUG_ON(offset & (len - 1)); + + switch (offset) { + case CORE0_STATUS_OFF: + *(uint64_t *)val = s->status; + break; + + case CORE0_EN_OFF: + *(uint64_t *)val = s->en; + break; + + case CORE0_SET_OFF: + *(uint64_t *)val = 0; + break; + + case CORE0_CLEAR_OFF: + *(uint64_t *)val = 0; + break; + + case CORE0_BUF_20 ... CORE0_BUF_38: + pbuf = (void *)s->buf + (offset - 0x20); + if (len == 8) + *(uint64_t *)val = *(uint64_t *)pbuf; + else /* Assume len == 4 */ + *(uint32_t *)val = *(uint32_t *)pbuf; + break; + + default: + pr_notice("%s with unknown addr %llx\n", __func__, addr); + break; + } + + return 0; +} + +static int loongson_vipi_write(struct loongson_kvm_ipi *ipi, + gpa_t addr, int len, const void *val) +{ + uint32_t core = (addr >> 8) & 3; + uint32_t node = (addr >> 44) & 3; + uint32_t id = core + node * 4; + uint64_t data, offset = addr & 0xff; + void *pbuf; + struct kvm *kvm = ipi->kvm; + struct kvm_mips_interrupt irq; + struct ipi_state *s = &(ipi->ipistate[id]); + + data = *(uint64_t *)val; + BUG_ON(offset & (len - 1)); + + switch (offset) { + case CORE0_STATUS_OFF: + break; + + case CORE0_EN_OFF: + s->en = data; + break; + + case CORE0_SET_OFF: + s->status |= data; + irq.cpu = id; + irq.irq = 6; + kvm_vcpu_ioctl_interrupt(kvm->vcpus[id], &irq); + break; + + case CORE0_CLEAR_OFF: + s->status &= ~data; + if (!s->status) { + irq.cpu = id; + irq.irq = -6; + kvm_vcpu_ioctl_interrupt(kvm->vcpus[id], &irq); + } + break; + + case CORE0_BUF_20 ... CORE0_BUF_38: + pbuf = (void *)s->buf + (offset - 0x20); + if (len == 8) + *(uint64_t *)pbuf = (uint64_t)data; + else /* Assume len == 4 */ + *(uint32_t *)pbuf = (uint32_t)data; + break; + + default: + pr_notice("%s with unknown addr %llx\n", __func__, addr); + break; + } + + return 0; +} + +static int kvm_ipi_read(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, void *val) +{ + unsigned long flags; + struct loongson_kvm_ipi *ipi; + struct ipi_io_device *ipi_device; + + ipi_device = container_of(dev, struct ipi_io_device, device); + ipi = ipi_device->ipi; + + spin_lock_irqsave(&ipi->lock, flags); + loongson_vipi_read(ipi, addr, len, val); + spin_unlock_irqrestore(&ipi->lock, flags); + + return 0; +} + +static int kvm_ipi_write(struct kvm_vcpu *vcpu, struct kvm_io_device *dev, + gpa_t addr, int len, const void *val) +{ + unsigned long flags; + struct loongson_kvm_ipi *ipi; + struct ipi_io_device *ipi_device; + + ipi_device = container_of(dev, struct ipi_io_device, device); + ipi = ipi_device->ipi; + + spin_lock_irqsave(&ipi->lock, flags); + loongson_vipi_write(ipi, addr, len, val); + spin_unlock_irqrestore(&ipi->lock, flags); + + return 0; +} + +static const struct kvm_io_device_ops kvm_ipi_ops = { + .read = kvm_ipi_read, + .write = kvm_ipi_write, +}; + +void kvm_init_loongson_ipi(struct kvm *kvm) +{ + int i; + unsigned long addr; + struct loongson_kvm_ipi *s; + struct kvm_io_device *device; + + s = &kvm->arch.ipi; + s->kvm = kvm; + spin_lock_init(&s->lock); + + /* + * Initialize IPI device + */ + for (i = 0; i < 4; i++) { + device = &s->dev_ipi[i].device; + kvm_iodevice_init(device, &kvm_ipi_ops); + addr = (((unsigned long)i) << 44) + IPI_BASE; + mutex_lock(&kvm->slots_lock); + kvm_io_bus_register_dev(kvm, KVM_MMIO_BUS, addr, 0x400, device); + mutex_unlock(&kvm->slots_lock); + s->dev_ipi[i].ipi = s; + s->dev_ipi[i].node_id = i; + } +} diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c index 2416fa40b687..521bd5891e84 100644 --- a/arch/mips/kvm/mips.c +++ b/arch/mips/kvm/mips.c @@ -67,6 +67,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = { VCPU_STAT("vz_ghfc", vz_ghfc_exits), VCPU_STAT("vz_gpa", vz_gpa_exits), VCPU_STAT("vz_resvd", vz_resvd_exits), + VCPU_STAT("vz_cpucfg", vz_cpucfg_exits), #endif VCPU_STAT("halt_successful_poll", halt_successful_poll), VCPU_STAT("halt_attempted_poll", halt_attempted_poll), @@ -129,6 +130,8 @@ int kvm_arch_check_processor_compat(void *opaque) return 0; } +extern void kvm_init_loongson_ipi(struct kvm *kvm); + int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) { switch (type) { @@ -148,6 +151,10 @@ int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) if (!kvm->arch.gpa_mm.pgd) return -ENOMEM; +#ifdef CONFIG_CPU_LOONGSON64 + kvm_init_loongson_ipi(kvm); +#endif + return 0; } @@ -490,7 +497,10 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, int intr = (int)irq->irq; struct kvm_vcpu *dvcpu = NULL; - if (intr == 3 || intr == -3 || intr == 4 || intr == -4) + if (intr == kvm_priority_to_irq[MIPS_EXC_INT_IPI_1] || + intr == kvm_priority_to_irq[MIPS_EXC_INT_IPI_2] || + intr == (-kvm_priority_to_irq[MIPS_EXC_INT_IPI_1]) || + intr == (-kvm_priority_to_irq[MIPS_EXC_INT_IPI_2])) kvm_debug("%s: CPU: %d, INTR: %d\n", __func__, irq->cpu, (int)intr); @@ -499,10 +509,10 @@ int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, else dvcpu = vcpu->kvm->vcpus[irq->cpu]; - if (intr == 2 || intr == 3 || intr == 4) { + if (intr == 2 || intr == 3 || intr == 4 || intr == 6) { kvm_mips_callbacks->queue_io_int(dvcpu, irq); - } else if (intr == -2 || intr == -3 || intr == -4) { + } else if (intr == -2 || intr == -3 || intr == -4 || intr == -6) { kvm_mips_callbacks->dequeue_io_int(dvcpu, irq); } else { kvm_err("%s: invalid interrupt ioctl (%d:%d)\n", __func__, @@ -1620,6 +1630,34 @@ static struct notifier_block kvm_mips_csr_die_notifier = { .notifier_call = kvm_mips_csr_die_notify, }; +static u32 kvm_default_priority_to_irq[MIPS_EXC_MAX] = { + [MIPS_EXC_INT_TIMER] = C_IRQ5, + [MIPS_EXC_INT_IO_1] = C_IRQ0, + [MIPS_EXC_INT_IPI_1] = C_IRQ1, + [MIPS_EXC_INT_IPI_2] = C_IRQ2, +}; + +static u32 kvm_loongson3_priority_to_irq[MIPS_EXC_MAX] = { + [MIPS_EXC_INT_TIMER] = C_IRQ5, + [MIPS_EXC_INT_IO_1] = C_IRQ0, + [MIPS_EXC_INT_IO_2] = C_IRQ1, + [MIPS_EXC_INT_IPI_1] = C_IRQ4, +}; + +u32 *kvm_priority_to_irq = kvm_default_priority_to_irq; + +u32 kvm_irq_to_priority(u32 irq) +{ + int i; + + for (i = MIPS_EXC_INT_TIMER; i < MIPS_EXC_MAX; i++) { + if (kvm_priority_to_irq[i] == (1 << (irq + 8))) + return i; + } + + return MIPS_EXC_MAX; +} + static int __init kvm_mips_init(void) { int ret; @@ -1638,6 +1676,9 @@ static int __init kvm_mips_init(void) if (ret) return ret; + if (boot_cpu_type() == CPU_LOONGSON64) + kvm_priority_to_irq = kvm_loongson3_priority_to_irq; + register_die_notifier(&kvm_mips_csr_die_notifier); return 0; diff --git a/arch/mips/kvm/tlb.c b/arch/mips/kvm/tlb.c index fc8aee05d10b..1c1fbce3f566 100644 --- a/arch/mips/kvm/tlb.c +++ b/arch/mips/kvm/tlb.c @@ -20,6 +20,7 @@ #include <asm/cpu.h> #include <asm/bootinfo.h> +#include <asm/mipsregs.h> #include <asm/mmu_context.h> #include <asm/cacheflush.h> #include <asm/tlb.h> @@ -621,6 +622,46 @@ void kvm_vz_load_guesttlb(const struct kvm_mips_tlb *buf, unsigned int index, } EXPORT_SYMBOL_GPL(kvm_vz_load_guesttlb); +#ifdef CONFIG_CPU_LOONGSON64 +void kvm_loongson_clear_guest_vtlb(void) +{ + int idx = read_gc0_index(); + + /* Set root GuestID for root probe and write of guest TLB entry */ + set_root_gid_to_guest_gid(); + + write_gc0_index(0); + guest_tlbinvf(); + write_gc0_index(idx); + + clear_root_gid(); + set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB); +} +EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_vtlb); + +void kvm_loongson_clear_guest_ftlb(void) +{ + int i; + int idx = read_gc0_index(); + + /* Set root GuestID for root probe and write of guest TLB entry */ + set_root_gid_to_guest_gid(); + + for (i = current_cpu_data.tlbsizevtlb; + i < (current_cpu_data.tlbsizevtlb + + current_cpu_data.tlbsizeftlbsets); + i++) { + write_gc0_index(i); + guest_tlbinvf(); + } + write_gc0_index(idx); + + clear_root_gid(); + set_c0_diag(LOONGSON_DIAG_ITLB | LOONGSON_DIAG_DTLB); +} +EXPORT_SYMBOL_GPL(kvm_loongson_clear_guest_ftlb); +#endif + #endif /** diff --git a/arch/mips/kvm/trap_emul.c b/arch/mips/kvm/trap_emul.c index 60763efa723e..34ad0b46e610 100644 --- a/arch/mips/kvm/trap_emul.c +++ b/arch/mips/kvm/trap_emul.c @@ -529,6 +529,9 @@ static int kvm_trap_emul_check_extension(struct kvm *kvm, long ext) case KVM_CAP_MIPS_TE: r = 1; break; + case KVM_CAP_IOEVENTFD: + r = 1; + break; default: r = 0; break; diff --git a/arch/mips/kvm/vz.c b/arch/mips/kvm/vz.c index 51f51009a53f..d9c462c14163 100644 --- a/arch/mips/kvm/vz.c +++ b/arch/mips/kvm/vz.c @@ -29,6 +29,7 @@ #include <linux/kvm_host.h> #include "interrupt.h" +#include "loongson_regs.h" #include "trace.h" @@ -126,6 +127,11 @@ static inline unsigned int kvm_vz_config5_guest_wrmask(struct kvm_vcpu *vcpu) return mask; } +static inline unsigned int kvm_vz_config6_guest_wrmask(struct kvm_vcpu *vcpu) +{ + return MIPS_CONF6_LOONGSON_INTIMER | MIPS_CONF6_LOONGSON_EXTIMER; +} + /* * VZ optionally allows these additional Config bits to be written by root: * Config: M, [MT] @@ -180,6 +186,12 @@ static inline unsigned int kvm_vz_config5_user_wrmask(struct kvm_vcpu *vcpu) return kvm_vz_config5_guest_wrmask(vcpu) | MIPS_CONF5_MRP; } +static inline unsigned int kvm_vz_config6_user_wrmask(struct kvm_vcpu *vcpu) +{ + return kvm_vz_config6_guest_wrmask(vcpu) | + MIPS_CONF6_LOONGSON_SFBEN | MIPS_CONF6_LOONGSON_FTLBDIS; +} + static gpa_t kvm_vz_gva_to_gpa_cb(gva_t gva) { /* VZ guest has already converted gva to gpa */ @@ -225,23 +237,7 @@ static void kvm_vz_queue_io_int_cb(struct kvm_vcpu *vcpu, * interrupts are asynchronous to vcpu execution therefore defer guest * cp0 accesses */ - switch (intr) { - case 2: - kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IO); - break; - - case 3: - kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_1); - break; - - case 4: - kvm_vz_queue_irq(vcpu, MIPS_EXC_INT_IPI_2); - break; - - default: - break; - } - + kvm_vz_queue_irq(vcpu, kvm_irq_to_priority(intr)); } static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, @@ -253,44 +249,22 @@ static void kvm_vz_dequeue_io_int_cb(struct kvm_vcpu *vcpu, * interrupts are asynchronous to vcpu execution therefore defer guest * cp0 accesses */ - switch (intr) { - case -2: - kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IO); - break; - - case -3: - kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_1); - break; - - case -4: - kvm_vz_dequeue_irq(vcpu, MIPS_EXC_INT_IPI_2); - break; - - default: - break; - } - + kvm_vz_dequeue_irq(vcpu, kvm_irq_to_priority(-intr)); } -static u32 kvm_vz_priority_to_irq[MIPS_EXC_MAX] = { - [MIPS_EXC_INT_TIMER] = C_IRQ5, - [MIPS_EXC_INT_IO] = C_IRQ0, - [MIPS_EXC_INT_IPI_1] = C_IRQ1, - [MIPS_EXC_INT_IPI_2] = C_IRQ2, -}; - static int kvm_vz_irq_deliver_cb(struct kvm_vcpu *vcpu, unsigned int priority, u32 cause) { u32 irq = (priority < MIPS_EXC_MAX) ? - kvm_vz_priority_to_irq[priority] : 0; + kvm_priority_to_irq[priority] : 0; switch (priority) { case MIPS_EXC_INT_TIMER: set_gc0_cause(C_TI); break; - case MIPS_EXC_INT_IO: + case MIPS_EXC_INT_IO_1: + case MIPS_EXC_INT_IO_2: case MIPS_EXC_INT_IPI_1: case MIPS_EXC_INT_IPI_2: if (cpu_has_guestctl2) @@ -311,7 +285,7 @@ static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, u32 cause) { u32 irq = (priority < MIPS_EXC_MAX) ? - kvm_vz_priority_to_irq[priority] : 0; + kvm_priority_to_irq[priority] : 0; switch (priority) { case MIPS_EXC_INT_TIMER: @@ -329,7 +303,8 @@ static int kvm_vz_irq_clear_cb(struct kvm_vcpu *vcpu, unsigned int priority, } break; - case MIPS_EXC_INT_IO: + case MIPS_EXC_INT_IO_1: + case MIPS_EXC_INT_IO_2: case MIPS_EXC_INT_IPI_1: case MIPS_EXC_INT_IPI_2: /* Clear GuestCtl2.VIP irq if not using Hardware Clear */ @@ -966,7 +941,8 @@ static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, (sel == 2 || /* SRSCtl */ sel == 3)) || /* SRSMap */ (rd == MIPS_CP0_CONFIG && - (sel == 7)) || /* Config7 */ + (sel == 6 || /* Config6 */ + sel == 7)) || /* Config7 */ (rd == MIPS_CP0_LLADDR && (sel == 2) && /* MAARI */ cpu_guest_has_maar && @@ -974,6 +950,11 @@ static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, (rd == MIPS_CP0_ERRCTL && (sel == 0))) { /* ErrCtl */ val = cop0->reg[rd][sel]; +#ifdef CONFIG_CPU_LOONGSON64 + } else if (rd == MIPS_CP0_DIAG && + (sel == 0)) { /* Diag */ + val = cop0->reg[rd][sel]; +#endif } else { val = 0; er = EMULATE_FAIL; @@ -1036,9 +1017,40 @@ static enum emulation_result kvm_vz_gpsi_cop0(union mips_instruction inst, cpu_guest_has_maar && !cpu_guest_has_dyn_maar) { kvm_write_maari(vcpu, val); + } else if (rd == MIPS_CP0_CONFIG && + (sel == 6)) { + cop0->reg[rd][sel] = (int)val; } else if (rd == MIPS_CP0_ERRCTL && (sel == 0)) { /* ErrCtl */ /* ignore the written value */ +#ifdef CONFIG_CPU_LOONGSON64 + } else if (rd == MIPS_CP0_DIAG && + (sel == 0)) { /* Diag */ + unsigned long flags; + + local_irq_save(flags); + if (val & LOONGSON_DIAG_BTB) { + /* Flush BTB */ + set_c0_diag(LOONGSON_DIAG_BTB); + } + if (val & LOONGSON_DIAG_ITLB) { + /* Flush ITLB */ + set_c0_diag(LOONGSON_DIAG_ITLB); + } + if (val & LOONGSON_DIAG_DTLB) { + /* Flush DTLB */ + set_c0_diag(LOONGSON_DIAG_DTLB); + } + if (val & LOONGSON_DIAG_VTLB) { + /* Flush VTLB */ + kvm_loongson_clear_guest_vtlb(); + } + if (val & LOONGSON_DIAG_FTLB) { + /* Flush FTLB */ + kvm_loongson_clear_guest_ftlb(); + } + local_irq_restore(flags); +#endif } else { er = EMULATE_FAIL; } @@ -1129,6 +1141,77 @@ static enum emulation_result kvm_vz_gpsi_cache(union mips_instruction inst, return EMULATE_FAIL; } +#ifdef CONFIG_CPU_LOONGSON64 +static enum emulation_result kvm_vz_gpsi_lwc2(union mips_instruction inst, + u32 *opc, u32 cause, + struct kvm_run *run, + struct kvm_vcpu *vcpu) +{ + unsigned int rs, rd; + unsigned int hostcfg; + unsigned long curr_pc; + enum emulation_result er = EMULATE_DONE; + + /* + * Update PC and hold onto current PC in case there is + * an error and we want to rollback the PC + */ + curr_pc = vcpu->arch.pc; + er = update_pc(vcpu, cause); + if (er == EMULATE_FAIL) + return er; + + rs = inst.loongson3_lscsr_format.rs; + rd = inst.loongson3_lscsr_format.rd; + switch (inst.loongson3_lscsr_format.fr) { + case 0x8: /* Read CPUCFG */ + ++vcpu->stat.vz_cpucfg_exits; + hostcfg = read_cpucfg(vcpu->arch.gprs[rs]); + + switch (vcpu->arch.gprs[rs]) { + case LOONGSON_CFG0: + vcpu->arch.gprs[rd] = 0x14c000; + break; + case LOONGSON_CFG1: + hostcfg &= (LOONGSON_CFG1_FP | LOONGSON_CFG1_MMI | + LOONGSON_CFG1_MSA1 | LOONGSON_CFG1_MSA2 | + LOONGSON_CFG1_SFBP); + vcpu->arch.gprs[rd] = hostcfg; + break; + case LOONGSON_CFG2: + hostcfg &= (LOONGSON_CFG2_LEXT1 | LOONGSON_CFG2_LEXT2 | + LOONGSON_CFG2_LEXT3 | LOONGSON_CFG2_LSPW); + vcpu->arch.gprs[rd] = hostcfg; + break; + case LOONGSON_CFG3: + vcpu->arch.gprs[rd] = hostcfg; + break; + default: + /* Don't export any other advanced features to guest */ + vcpu->arch.gprs[rd] = 0; + break; + } + break; + + default: + kvm_err("lwc2 emulate not impl %d rs %lx @%lx\n", + inst.loongson3_lscsr_format.fr, vcpu->arch.gprs[rs], curr_pc); + er = EMULATE_FAIL; + break; + } + + /* Rollback PC only if emulation was unsuccessful */ + if (er == EMULATE_FAIL) { + kvm_err("[%#lx]%s: unsupported lwc2 instruction 0x%08x 0x%08x\n", + curr_pc, __func__, inst.word, inst.loongson3_lscsr_format.fr); + + vcpu->arch.pc = curr_pc; + } + + return er; +} +#endif + static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, struct kvm_vcpu *vcpu) { @@ -1158,6 +1241,11 @@ static enum emulation_result kvm_trap_vz_handle_gpsi(u32 cause, u32 *opc, er = kvm_vz_gpsi_cache(inst, opc, cause, run, vcpu); break; #endif +#ifdef CONFIG_CPU_LOONGSON64 + case lwc2_op: + er = kvm_vz_gpsi_lwc2(inst, opc, cause, run, vcpu); + break; +#endif case spec3_op: switch (inst.spec3_format.func) { #ifdef CONFIG_CPU_MIPSR6 @@ -1652,6 +1740,7 @@ static u64 kvm_vz_get_one_regs[] = { KVM_REG_MIPS_CP0_CONFIG3, KVM_REG_MIPS_CP0_CONFIG4, KVM_REG_MIPS_CP0_CONFIG5, + KVM_REG_MIPS_CP0_CONFIG6, #ifdef CONFIG_64BIT KVM_REG_MIPS_CP0_XCONTEXT, #endif @@ -1706,7 +1795,7 @@ static unsigned long kvm_vz_num_regs(struct kvm_vcpu *vcpu) ret += ARRAY_SIZE(kvm_vz_get_one_regs_contextconfig); if (cpu_guest_has_segments) ret += ARRAY_SIZE(kvm_vz_get_one_regs_segments); - if (cpu_guest_has_htw) + if (cpu_guest_has_htw || cpu_guest_has_ldpte) ret += ARRAY_SIZE(kvm_vz_get_one_regs_htw); if (cpu_guest_has_maar && !cpu_guest_has_dyn_maar) ret += 1 + ARRAY_SIZE(vcpu->arch.maar); @@ -1755,7 +1844,7 @@ static int kvm_vz_copy_reg_indices(struct kvm_vcpu *vcpu, u64 __user *indices) return -EFAULT; indices += ARRAY_SIZE(kvm_vz_get_one_regs_segments); } - if (cpu_guest_has_htw) { + if (cpu_guest_has_htw || cpu_guest_has_ldpte) { if (copy_to_user(indices, kvm_vz_get_one_regs_htw, sizeof(kvm_vz_get_one_regs_htw))) return -EFAULT; @@ -1878,17 +1967,17 @@ static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, *v = read_gc0_segctl2(); break; case KVM_REG_MIPS_CP0_PWBASE: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; *v = read_gc0_pwbase(); break; case KVM_REG_MIPS_CP0_PWFIELD: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; *v = read_gc0_pwfield(); break; case KVM_REG_MIPS_CP0_PWSIZE: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; *v = read_gc0_pwsize(); break; @@ -1896,7 +1985,7 @@ static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, *v = (long)read_gc0_wired(); break; case KVM_REG_MIPS_CP0_PWCTL: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; *v = read_gc0_pwctl(); break; @@ -1979,6 +2068,9 @@ static int kvm_vz_get_one_reg(struct kvm_vcpu *vcpu, return -EINVAL; *v = read_gc0_config5(); break; + case KVM_REG_MIPS_CP0_CONFIG6: + *v = kvm_read_sw_gc0_config6(cop0); + break; case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) return -EINVAL; @@ -2101,17 +2193,17 @@ static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, write_gc0_segctl2(v); break; case KVM_REG_MIPS_CP0_PWBASE: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; write_gc0_pwbase(v); break; case KVM_REG_MIPS_CP0_PWFIELD: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; write_gc0_pwfield(v); break; case KVM_REG_MIPS_CP0_PWSIZE: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; write_gc0_pwsize(v); break; @@ -2119,7 +2211,7 @@ static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, change_gc0_wired(MIPSR6_WIRED_WIRED, v); break; case KVM_REG_MIPS_CP0_PWCTL: - if (!cpu_guest_has_htw) + if (!cpu_guest_has_htw && !cpu_guest_has_ldpte) return -EINVAL; write_gc0_pwctl(v); break; @@ -2248,6 +2340,14 @@ static int kvm_vz_set_one_reg(struct kvm_vcpu *vcpu, write_gc0_config5(v); } break; + case KVM_REG_MIPS_CP0_CONFIG6: + cur = kvm_read_sw_gc0_config6(cop0); + change = (cur ^ v) & kvm_vz_config6_user_wrmask(vcpu); + if (change) { + v = cur ^ change; + kvm_write_sw_gc0_config6(cop0, (int)v); + } + break; case KVM_REG_MIPS_CP0_MAAR(0) ... KVM_REG_MIPS_CP0_MAAR(0x3f): if (!cpu_guest_has_maar || cpu_guest_has_dyn_maar) return -EINVAL; @@ -2580,7 +2680,7 @@ static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) } /* restore HTW registers */ - if (cpu_guest_has_htw) { + if (cpu_guest_has_htw || cpu_guest_has_ldpte) { kvm_restore_gc0_pwbase(cop0); kvm_restore_gc0_pwfield(cop0); kvm_restore_gc0_pwsize(cop0); @@ -2597,7 +2697,7 @@ static int kvm_vz_vcpu_load(struct kvm_vcpu *vcpu, int cpu) * prevents a SC on the next VCPU from succeeding by matching a LL on * the previous VCPU. */ - if (cpu_guest_has_rw_llb) + if (vcpu->kvm->created_vcpus > 1) write_gc0_lladdr(0); return 0; @@ -2685,8 +2785,8 @@ static int kvm_vz_vcpu_put(struct kvm_vcpu *vcpu, int cpu) } /* save HTW registers if enabled in guest */ - if (cpu_guest_has_htw && - kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW) { + if (cpu_guest_has_ldpte || (cpu_guest_has_htw && + kvm_read_sw_gc0_config3(cop0) & MIPS_CONF3_PW)) { kvm_save_gc0_pwbase(cop0); kvm_save_gc0_pwfield(cop0); kvm_save_gc0_pwsize(cop0); @@ -2853,8 +2953,12 @@ static int kvm_vz_hardware_enable(void) write_c0_guestctl0(MIPS_GCTL0_CP0 | (MIPS_GCTL0_AT_GUEST << MIPS_GCTL0_AT_SHIFT) | MIPS_GCTL0_CG | MIPS_GCTL0_CF); - if (cpu_has_guestctl0ext) - set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); + if (cpu_has_guestctl0ext) { + if (current_cpu_type() != CPU_LOONGSON64) + set_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); + else + clear_c0_guestctl0ext(MIPS_GCTL0EXT_CGI); + } if (cpu_has_guestid) { write_c0_guestctl1(0); @@ -2871,6 +2975,12 @@ static int kvm_vz_hardware_enable(void) if (cpu_has_guestctl2) clear_c0_guestctl2(0x3f << 10); +#ifdef CONFIG_CPU_LOONGSON64 + /* Control guest CCA attribute */ + if (cpu_has_csr()) + csr_writel(csr_readl(0xffffffec) | 0x1, 0xffffffec); +#endif + return 0; } @@ -2927,6 +3037,9 @@ static int kvm_vz_check_extension(struct kvm *kvm, long ext) r = 2; break; #endif + case KVM_CAP_IOEVENTFD: + r = 1; + break; default: r = 0; break; diff --git a/arch/nios2/kernel/signal.c b/arch/nios2/kernel/signal.c index a42dd09c6578..d8a087cf2b42 100644 --- a/arch/nios2/kernel/signal.c +++ b/arch/nios2/kernel/signal.c @@ -252,6 +252,7 @@ static int do_signal(struct pt_regs *regs) switch (retval) { case ERESTART_RESTARTBLOCK: restart = -2; + fallthrough; case ERESTARTNOHAND: case ERESTARTSYS: case ERESTARTNOINTR: diff --git a/arch/openrisc/kernel/entry.S b/arch/openrisc/kernel/entry.S index a1d25c3e28d1..bc657e55c15f 100644 --- a/arch/openrisc/kernel/entry.S +++ b/arch/openrisc/kernel/entry.S @@ -1166,13 +1166,13 @@ ENTRY(__sys_clone) l.movhi r29,hi(sys_clone) l.ori r29,r29,lo(sys_clone) l.j _fork_save_extra_regs_and_call - l.addi r7,r1,0 + l.nop ENTRY(__sys_fork) l.movhi r29,hi(sys_fork) l.ori r29,r29,lo(sys_fork) l.j _fork_save_extra_regs_and_call - l.addi r3,r1,0 + l.nop ENTRY(sys_rt_sigreturn) l.jal _sys_rt_sigreturn diff --git a/arch/powerpc/include/asm/kvm_book3s.h b/arch/powerpc/include/asm/kvm_book3s.h index 8dd24c7692a0..d32ec9ae73bd 100644 --- a/arch/powerpc/include/asm/kvm_book3s.h +++ b/arch/powerpc/include/asm/kvm_book3s.h @@ -155,12 +155,11 @@ extern void kvmppc_mmu_unmap_page(struct kvm_vcpu *vcpu, struct kvmppc_pte *pte) extern int kvmppc_mmu_map_segment(struct kvm_vcpu *vcpu, ulong eaddr); extern void kvmppc_mmu_flush_segment(struct kvm_vcpu *vcpu, ulong eaddr, ulong seg_size); extern void kvmppc_mmu_flush_segments(struct kvm_vcpu *vcpu); -extern int kvmppc_book3s_hv_page_fault(struct kvm_run *run, - struct kvm_vcpu *vcpu, unsigned long addr, - unsigned long status); +extern int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu, + unsigned long addr, unsigned long status); extern long kvmppc_hv_find_lock_hpte(struct kvm *kvm, gva_t eaddr, unsigned long slb_v, unsigned long valid); -extern int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_hv_emulate_mmio(struct kvm_vcpu *vcpu, unsigned long gpa, gva_t ea, int is_store); extern void kvmppc_mmu_hpte_cache_map(struct kvm_vcpu *vcpu, struct hpte_cache *pte); @@ -174,8 +173,7 @@ extern void kvmppc_mmu_hpte_sysexit(void); extern int kvmppc_mmu_hv_init(void); extern int kvmppc_book3s_hcall_implemented(struct kvm *kvm, unsigned long hc); -extern int kvmppc_book3s_radix_page_fault(struct kvm_run *run, - struct kvm_vcpu *vcpu, +extern int kvmppc_book3s_radix_page_fault(struct kvm_vcpu *vcpu, unsigned long ea, unsigned long dsisr); extern unsigned long __kvmhv_copy_tofrom_guest_radix(int lpid, int pid, gva_t eaddr, void *to, void *from, @@ -234,7 +232,7 @@ extern void kvmppc_trigger_fac_interrupt(struct kvm_vcpu *vcpu, ulong fac); extern void kvmppc_set_bat(struct kvm_vcpu *vcpu, struct kvmppc_bat *bat, bool upper, u32 val); extern void kvmppc_giveup_ext(struct kvm_vcpu *vcpu, ulong msr); -extern int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu); +extern int kvmppc_emulate_paired_single(struct kvm_vcpu *vcpu); extern kvm_pfn_t kvmppc_gpa_to_pfn(struct kvm_vcpu *vcpu, gpa_t gpa, bool writing, bool *writable); extern void kvmppc_add_revmap_chain(struct kvm *kvm, struct revmap_entry *rev, @@ -300,12 +298,12 @@ void kvmhv_set_ptbl_entry(unsigned int lpid, u64 dw0, u64 dw1); void kvmhv_release_all_nested(struct kvm *kvm); long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu); long kvmhv_do_nested_tlbie(struct kvm_vcpu *vcpu); -int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu, +int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr); void kvmhv_save_hv_regs(struct kvm_vcpu *vcpu, struct hv_guest_state *hr); void kvmhv_restore_hv_return_state(struct kvm_vcpu *vcpu, struct hv_guest_state *hr); -long int kvmhv_nested_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu); +long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu); void kvmppc_giveup_fac(struct kvm_vcpu *vcpu, ulong fac); diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h index 337047ba4a56..7e2d061d0445 100644 --- a/arch/powerpc/include/asm/kvm_host.h +++ b/arch/powerpc/include/asm/kvm_host.h @@ -795,7 +795,6 @@ struct kvm_vcpu_arch { struct mmio_hpte_cache_entry *pgfault_cache; struct task_struct *run_task; - struct kvm_run *kvm_run; spinlock_t vpa_update_lock; struct kvmppc_vpa vpa; diff --git a/arch/powerpc/include/asm/kvm_ppc.h b/arch/powerpc/include/asm/kvm_ppc.h index 94f5a32acaf1..ccf66b3a4c1d 100644 --- a/arch/powerpc/include/asm/kvm_ppc.h +++ b/arch/powerpc/include/asm/kvm_ppc.h @@ -58,28 +58,28 @@ enum xlate_readwrite { XLATE_WRITE /* check for write permissions */ }; -extern int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); -extern int __kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu); +extern int kvmppc_vcpu_run(struct kvm_vcpu *vcpu); +extern int __kvmppc_vcpu_run(struct kvm_run *run, struct kvm_vcpu *vcpu); extern void kvmppc_handler_highmem(void); extern void kvmppc_dump_vcpu(struct kvm_vcpu *vcpu); -extern int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian); -extern int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_loads(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian); -extern int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian, int mmio_sign_extend); -extern int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian); -extern int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu, unsigned int rs, unsigned int bytes, int is_default_endian); -extern int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_store(struct kvm_vcpu *vcpu, u64 val, unsigned int bytes, int is_default_endian); -extern int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu, int rs, unsigned int bytes, int is_default_endian); @@ -90,10 +90,9 @@ extern int kvmppc_ld(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); extern int kvmppc_st(struct kvm_vcpu *vcpu, ulong *eaddr, int size, void *ptr, bool data); -extern int kvmppc_emulate_instruction(struct kvm_run *run, - struct kvm_vcpu *vcpu); +extern int kvmppc_emulate_instruction(struct kvm_vcpu *vcpu); extern int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu); -extern int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu); +extern int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu); extern void kvmppc_emulate_dec(struct kvm_vcpu *vcpu); extern u32 kvmppc_get_dec(struct kvm_vcpu *vcpu, u64 tb); extern void kvmppc_decrementer_func(struct kvm_vcpu *vcpu); @@ -267,7 +266,7 @@ struct kvmppc_ops { void (*vcpu_put)(struct kvm_vcpu *vcpu); void (*inject_interrupt)(struct kvm_vcpu *vcpu, int vec, u64 srr1_flags); void (*set_msr)(struct kvm_vcpu *vcpu, u64 msr); - int (*vcpu_run)(struct kvm_run *run, struct kvm_vcpu *vcpu); + int (*vcpu_run)(struct kvm_vcpu *vcpu); int (*vcpu_create)(struct kvm_vcpu *vcpu); void (*vcpu_free)(struct kvm_vcpu *vcpu); int (*check_requests)(struct kvm_vcpu *vcpu); @@ -291,7 +290,7 @@ struct kvmppc_ops { int (*init_vm)(struct kvm *kvm); void (*destroy_vm)(struct kvm *kvm); int (*get_smmu_info)(struct kvm *kvm, struct kvm_ppc_smmu_info *info); - int (*emulate_op)(struct kvm_run *run, struct kvm_vcpu *vcpu, + int (*emulate_op)(struct kvm_vcpu *vcpu, unsigned int inst, int *advance); int (*emulate_mtspr)(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); int (*emulate_mfspr)(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val); diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index 37508a356f28..41fedec69ac3 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -755,9 +755,9 @@ void kvmppc_set_msr(struct kvm_vcpu *vcpu, u64 msr) } EXPORT_SYMBOL_GPL(kvmppc_set_msr); -int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +int kvmppc_vcpu_run(struct kvm_vcpu *vcpu) { - return vcpu->kvm->arch.kvm_ops->vcpu_run(kvm_run, vcpu); + return vcpu->kvm->arch.kvm_ops->vcpu_run(vcpu); } int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, diff --git a/arch/powerpc/kvm/book3s.h b/arch/powerpc/kvm/book3s.h index eae259ee49af..9b6323ec8e60 100644 --- a/arch/powerpc/kvm/book3s.h +++ b/arch/powerpc/kvm/book3s.h @@ -18,7 +18,7 @@ extern void kvm_set_spte_hva_hv(struct kvm *kvm, unsigned long hva, pte_t pte); extern int kvmppc_mmu_init_pr(struct kvm_vcpu *vcpu); extern void kvmppc_mmu_destroy_pr(struct kvm_vcpu *vcpu); -extern int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, +extern int kvmppc_core_emulate_op_pr(struct kvm_vcpu *vcpu, unsigned int inst, int *advance); extern int kvmppc_core_emulate_mtspr_pr(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index ddfc4c90ebb6..7c5a1812a1c3 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -412,7 +412,7 @@ static int instruction_is_store(unsigned int instr) return (instr & mask) != 0; } -int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_hv_emulate_mmio(struct kvm_vcpu *vcpu, unsigned long gpa, gva_t ea, int is_store) { u32 last_inst; @@ -472,10 +472,10 @@ int kvmppc_hv_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu, vcpu->arch.paddr_accessed = gpa; vcpu->arch.vaddr_accessed = ea; - return kvmppc_emulate_mmio(run, vcpu); + return kvmppc_emulate_mmio(vcpu); } -int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_book3s_hv_page_fault(struct kvm_vcpu *vcpu, unsigned long ea, unsigned long dsisr) { struct kvm *kvm = vcpu->kvm; @@ -498,7 +498,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, pte_t pte, *ptep; if (kvm_is_radix(kvm)) - return kvmppc_book3s_radix_page_fault(run, vcpu, ea, dsisr); + return kvmppc_book3s_radix_page_fault(vcpu, ea, dsisr); /* * Real-mode code has already searched the HPT and found the @@ -518,7 +518,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, gpa_base = r & HPTE_R_RPN & ~(psize - 1); gfn_base = gpa_base >> PAGE_SHIFT; gpa = gpa_base | (ea & (psize - 1)); - return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea, + return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, dsisr & DSISR_ISSTORE); } } @@ -554,7 +554,7 @@ int kvmppc_book3s_hv_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, /* No memslot means it's an emulated MMIO region */ if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) - return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea, + return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, dsisr & DSISR_ISSTORE); /* diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 2a2fad9a1c13..3cb0c9843d01 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -353,7 +353,13 @@ static struct kmem_cache *kvm_pmd_cache; static pte_t *kvmppc_pte_alloc(void) { - return kmem_cache_alloc(kvm_pte_cache, GFP_KERNEL); + pte_t *pte; + + pte = kmem_cache_alloc(kvm_pte_cache, GFP_KERNEL); + /* pmd_populate() will only reference _pa(pte). */ + kmemleak_ignore(pte); + + return pte; } static void kvmppc_pte_free(pte_t *ptep) @@ -363,7 +369,13 @@ static void kvmppc_pte_free(pte_t *ptep) static pmd_t *kvmppc_pmd_alloc(void) { - return kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL); + pmd_t *pmd; + + pmd = kmem_cache_alloc(kvm_pmd_cache, GFP_KERNEL); + /* pud_populate() will only reference _pa(pmd). */ + kmemleak_ignore(pmd); + + return pmd; } static void kvmppc_pmd_free(pmd_t *pmdp) @@ -417,9 +429,13 @@ void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, * Callers are responsible for flushing the PWC. * * When page tables are being unmapped/freed as part of page fault path - * (full == false), ptes are not expected. There is code to unmap them - * and emit a warning if encountered, but there may already be data - * corruption due to the unexpected mappings. + * (full == false), valid ptes are generally not expected; however, there + * is one situation where they arise, which is when dirty page logging is + * turned off for a memslot while the VM is running. The new memslot + * becomes visible to page faults before the memslot commit function + * gets to flush the memslot, which can lead to a 2MB page mapping being + * installed for a guest physical address where there are already 64kB + * (or 4kB) mappings (of sub-pages of the same 2MB page). */ static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full, unsigned int lpid) @@ -433,7 +449,6 @@ static void kvmppc_unmap_free_pte(struct kvm *kvm, pte_t *pte, bool full, for (it = 0; it < PTRS_PER_PTE; ++it, ++p) { if (pte_val(*p) == 0) continue; - WARN_ON_ONCE(1); kvmppc_unmap_pte(kvm, p, pte_pfn(*p) << PAGE_SHIFT, PAGE_SHIFT, NULL, lpid); @@ -891,7 +906,7 @@ int kvmppc_book3s_instantiate_page(struct kvm_vcpu *vcpu, return ret; } -int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_book3s_radix_page_fault(struct kvm_vcpu *vcpu, unsigned long ea, unsigned long dsisr) { struct kvm *kvm = vcpu->kvm; @@ -937,7 +952,7 @@ int kvmppc_book3s_radix_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu, kvmppc_core_queue_data_storage(vcpu, ea, dsisr); return RESUME_GUEST; } - return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea, writing); + return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing); } if (memslot->flags & KVM_MEM_READONLY) { @@ -1142,6 +1157,11 @@ void kvmppc_radix_flush_memslot(struct kvm *kvm, kvm->arch.lpid); gpa += PAGE_SIZE; } + /* + * Increase the mmu notifier sequence number to prevent any page + * fault that read the memslot earlier from writing a PTE. + */ + kvm->mmu_notifier_seq++; spin_unlock(&kvm->mmu_lock); } diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 50555ad1db93..1a529df0ab44 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -73,6 +73,7 @@ extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, struct kvmppc_spapr_tce_iommu_table *stit, *tmp; struct iommu_table_group *table_group = NULL; + rcu_read_lock(); list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { table_group = iommu_group_get_iommudata(grp); @@ -87,7 +88,9 @@ extern void kvm_spapr_tce_release_iommu_group(struct kvm *kvm, kref_put(&stit->kref, kvm_spapr_tce_liobn_put); } } + cond_resched_rcu(); } + rcu_read_unlock(); } extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, @@ -105,12 +108,14 @@ extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, if (!f.file) return -EBADF; + rcu_read_lock(); list_for_each_entry_rcu(stt, &kvm->arch.spapr_tce_tables, list) { if (stt == f.file->private_data) { found = true; break; } } + rcu_read_unlock(); fdput(f); @@ -143,6 +148,7 @@ extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, if (!tbl) return -EINVAL; + rcu_read_lock(); list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { if (tbl != stit->tbl) continue; @@ -150,14 +156,17 @@ extern long kvm_spapr_tce_attach_iommu_group(struct kvm *kvm, int tablefd, if (!kref_get_unless_zero(&stit->kref)) { /* stit is being destroyed */ iommu_tce_table_put(tbl); + rcu_read_unlock(); return -ENOTTY; } /* * The table is already known to this KVM, we just increased * its KVM reference counter and can return. */ + rcu_read_unlock(); return 0; } + rcu_read_unlock(); stit = kzalloc(sizeof(*stit), GFP_KERNEL); if (!stit) { @@ -365,18 +374,19 @@ static long kvmppc_tce_validate(struct kvmppc_spapr_tce_table *stt, if (kvmppc_tce_to_ua(stt->kvm, tce, &ua)) return H_TOO_HARD; + rcu_read_lock(); list_for_each_entry_rcu(stit, &stt->iommu_tables, next) { unsigned long hpa = 0; struct mm_iommu_table_group_mem_t *mem; long shift = stit->tbl->it_page_shift; mem = mm_iommu_lookup(stt->kvm->mm, ua, 1ULL << shift); - if (!mem) - return H_TOO_HARD; - - if (mm_iommu_ua_to_hpa(mem, ua, shift, &hpa)) + if (!mem || mm_iommu_ua_to_hpa(mem, ua, shift, &hpa)) { + rcu_read_unlock(); return H_TOO_HARD; + } } + rcu_read_unlock(); return H_SUCCESS; } diff --git a/arch/powerpc/kvm/book3s_emulate.c b/arch/powerpc/kvm/book3s_emulate.c index dad71d276b91..0effd48c8f4d 100644 --- a/arch/powerpc/kvm/book3s_emulate.c +++ b/arch/powerpc/kvm/book3s_emulate.c @@ -235,7 +235,7 @@ void kvmppc_emulate_tabort(struct kvm_vcpu *vcpu, int ra_val) #endif -int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_core_emulate_op_pr(struct kvm_vcpu *vcpu, unsigned int inst, int *advance) { int emulated = EMULATE_DONE; @@ -371,13 +371,13 @@ int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, if (kvmppc_h_pr(vcpu, cmd) == EMULATE_DONE) break; - run->papr_hcall.nr = cmd; + vcpu->run->papr_hcall.nr = cmd; for (i = 0; i < 9; ++i) { ulong gpr = kvmppc_get_gpr(vcpu, 4 + i); - run->papr_hcall.args[i] = gpr; + vcpu->run->papr_hcall.args[i] = gpr; } - run->exit_reason = KVM_EXIT_PAPR_HCALL; + vcpu->run->exit_reason = KVM_EXIT_PAPR_HCALL; vcpu->arch.hcall_needed = 1; emulated = EMULATE_EXIT_USER; break; @@ -629,7 +629,7 @@ int kvmppc_core_emulate_op_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, } if (emulated == EMULATE_FAIL) - emulated = kvmppc_emulate_paired_single(run, vcpu); + emulated = kvmppc_emulate_paired_single(vcpu); return emulated; } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index 7f5d58663f13..6bf66649ab92 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -1094,9 +1094,14 @@ int kvmppc_pseries_do_hcall(struct kvm_vcpu *vcpu) ret = kvmppc_h_svm_init_done(vcpu->kvm); break; case H_SVM_INIT_ABORT: - ret = H_UNSUPPORTED; - if (kvmppc_get_srr1(vcpu) & MSR_S) - ret = kvmppc_h_svm_init_abort(vcpu->kvm); + /* + * Even if that call is made by the Ultravisor, the SSR1 value + * is the guest context one, with the secure bit clear as it has + * not yet been secured. So we can't check it here. + * Instead the kvm->arch.secure_guest flag is checked inside + * kvmppc_h_svm_init_abort(). + */ + ret = kvmppc_h_svm_init_abort(vcpu->kvm); break; default: @@ -1151,8 +1156,7 @@ static int kvmppc_hcall_impl_hv(unsigned long cmd) return kvmppc_hcall_impl_hv_realmode(cmd); } -static int kvmppc_emulate_debug_inst(struct kvm_run *run, - struct kvm_vcpu *vcpu) +static int kvmppc_emulate_debug_inst(struct kvm_vcpu *vcpu) { u32 last_inst; @@ -1166,8 +1170,8 @@ static int kvmppc_emulate_debug_inst(struct kvm_run *run, } if (last_inst == KVMPPC_INST_SW_BREAKPOINT) { - run->exit_reason = KVM_EXIT_DEBUG; - run->debug.arch.address = kvmppc_get_pc(vcpu); + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu); return RESUME_HOST; } else { kvmppc_core_queue_program(vcpu, SRR1_PROGILL); @@ -1268,9 +1272,10 @@ static int kvmppc_emulate_doorbell_instr(struct kvm_vcpu *vcpu) return RESUME_GUEST; } -static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_handle_exit_hv(struct kvm_vcpu *vcpu, struct task_struct *tsk) { + struct kvm_run *run = vcpu->run; int r = RESUME_HOST; vcpu->stat.sum_exits++; @@ -1405,7 +1410,7 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, swab32(vcpu->arch.emul_inst) : vcpu->arch.emul_inst; if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) { - r = kvmppc_emulate_debug_inst(run, vcpu); + r = kvmppc_emulate_debug_inst(vcpu); } else { kvmppc_core_queue_program(vcpu, SRR1_PROGILL); r = RESUME_GUEST; @@ -1457,7 +1462,7 @@ static int kvmppc_handle_exit_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, return r; } -static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) +static int kvmppc_handle_nested_exit(struct kvm_vcpu *vcpu) { int r; int srcu_idx; @@ -1515,7 +1520,7 @@ static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) */ case BOOK3S_INTERRUPT_H_DATA_STORAGE: srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvmhv_nested_page_fault(run, vcpu); + r = kvmhv_nested_page_fault(vcpu); srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); break; case BOOK3S_INTERRUPT_H_INST_STORAGE: @@ -1525,7 +1530,7 @@ static int kvmppc_handle_nested_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) if (vcpu->arch.shregs.msr & HSRR1_HISI_WRITE) vcpu->arch.fault_dsisr |= DSISR_ISSTORE; srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvmhv_nested_page_fault(run, vcpu); + r = kvmhv_nested_page_fault(vcpu); srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); break; @@ -2929,7 +2934,7 @@ static void post_guest_process(struct kvmppc_vcore *vc, bool is_master) ret = RESUME_GUEST; if (vcpu->arch.trap) - ret = kvmppc_handle_exit_hv(vcpu->arch.kvm_run, vcpu, + ret = kvmppc_handle_exit_hv(vcpu, vcpu->arch.run_task); vcpu->arch.ret = ret; @@ -3894,15 +3899,16 @@ static int kvmhv_setup_mmu(struct kvm_vcpu *vcpu) return r; } -static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +static int kvmppc_run_vcpu(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int n_ceded, i, r; struct kvmppc_vcore *vc; struct kvm_vcpu *v; trace_kvmppc_run_vcpu_enter(vcpu); - kvm_run->exit_reason = 0; + run->exit_reason = 0; vcpu->arch.ret = RESUME_GUEST; vcpu->arch.trap = 0; kvmppc_update_vpas(vcpu); @@ -3914,7 +3920,6 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) spin_lock(&vc->lock); vcpu->arch.ceded = 0; vcpu->arch.run_task = current; - vcpu->arch.kvm_run = kvm_run; vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; vcpu->arch.busy_preempt = TB_NIL; @@ -3947,8 +3952,8 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) r = kvmhv_setup_mmu(vcpu); spin_lock(&vc->lock); if (r) { - kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; - kvm_run->fail_entry. + run->exit_reason = KVM_EXIT_FAIL_ENTRY; + run->fail_entry. hardware_entry_failure_reason = 0; vcpu->arch.ret = r; break; @@ -3967,7 +3972,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) if (signal_pending(v->arch.run_task)) { kvmppc_remove_runnable(vc, v); v->stat.signal_exits++; - v->arch.kvm_run->exit_reason = KVM_EXIT_INTR; + v->run->exit_reason = KVM_EXIT_INTR; v->arch.ret = -EINTR; wake_up(&v->arch.cpu_run); } @@ -4008,7 +4013,7 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) if (vcpu->arch.state == KVMPPC_VCPU_RUNNABLE) { kvmppc_remove_runnable(vc, vcpu); vcpu->stat.signal_exits++; - kvm_run->exit_reason = KVM_EXIT_INTR; + run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; } @@ -4019,15 +4024,15 @@ static int kvmppc_run_vcpu(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) wake_up(&v->arch.cpu_run); } - trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); + trace_kvmppc_run_vcpu_exit(vcpu); spin_unlock(&vc->lock); return vcpu->arch.ret; } -int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, - struct kvm_vcpu *vcpu, u64 time_limit, +int kvmhv_run_single_vcpu(struct kvm_vcpu *vcpu, u64 time_limit, unsigned long lpcr) { + struct kvm_run *run = vcpu->run; int trap, r, pcpu; int srcu_idx, lpid; struct kvmppc_vcore *vc; @@ -4036,14 +4041,13 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, trace_kvmppc_run_vcpu_enter(vcpu); - kvm_run->exit_reason = 0; + run->exit_reason = 0; vcpu->arch.ret = RESUME_GUEST; vcpu->arch.trap = 0; vc = vcpu->arch.vcore; vcpu->arch.ceded = 0; vcpu->arch.run_task = current; - vcpu->arch.kvm_run = kvm_run; vcpu->arch.stolen_logged = vcore_stolen_time(vc, mftb()); vcpu->arch.state = KVMPPC_VCPU_RUNNABLE; vcpu->arch.busy_preempt = TB_NIL; @@ -4161,9 +4165,9 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, r = RESUME_GUEST; if (trap) { if (!nested) - r = kvmppc_handle_exit_hv(kvm_run, vcpu, current); + r = kvmppc_handle_exit_hv(vcpu, current); else - r = kvmppc_handle_nested_exit(kvm_run, vcpu); + r = kvmppc_handle_nested_exit(vcpu); } vcpu->arch.ret = r; @@ -4173,7 +4177,7 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, while (vcpu->arch.ceded && !kvmppc_vcpu_woken(vcpu)) { if (signal_pending(current)) { vcpu->stat.signal_exits++; - kvm_run->exit_reason = KVM_EXIT_INTR; + run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; break; } @@ -4189,13 +4193,13 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, done: kvmppc_remove_runnable(vc, vcpu); - trace_kvmppc_run_vcpu_exit(vcpu, kvm_run); + trace_kvmppc_run_vcpu_exit(vcpu); return vcpu->arch.ret; sigpend: vcpu->stat.signal_exits++; - kvm_run->exit_reason = KVM_EXIT_INTR; + run->exit_reason = KVM_EXIT_INTR; vcpu->arch.ret = -EINTR; out: local_irq_enable(); @@ -4203,8 +4207,9 @@ int kvmhv_run_single_vcpu(struct kvm_run *kvm_run, goto done; } -static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) +static int kvmppc_vcpu_run_hv(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int r; int srcu_idx; unsigned long ebb_regs[3] = {}; /* shut up GCC */ @@ -4288,10 +4293,10 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) */ if (kvm->arch.threads_indep && kvm_is_radix(kvm) && !no_mixing_hpt_and_radix) - r = kvmhv_run_single_vcpu(run, vcpu, ~(u64)0, + r = kvmhv_run_single_vcpu(vcpu, ~(u64)0, vcpu->arch.vcore->lpcr); else - r = kvmppc_run_vcpu(run, vcpu); + r = kvmppc_run_vcpu(vcpu); if (run->exit_reason == KVM_EXIT_PAPR_HCALL && !(vcpu->arch.shregs.msr & MSR_PR)) { @@ -4301,7 +4306,7 @@ static int kvmppc_vcpu_run_hv(struct kvm_run *run, struct kvm_vcpu *vcpu) kvmppc_core_prepare_to_enter(vcpu); } else if (r == RESUME_PAGE_FAULT) { srcu_idx = srcu_read_lock(&kvm->srcu); - r = kvmppc_book3s_hv_page_fault(run, vcpu, + r = kvmppc_book3s_hv_page_fault(vcpu, vcpu->arch.fault_dar, vcpu->arch.fault_dsisr); srcu_read_unlock(&kvm->srcu, srcu_idx); } else if (r == RESUME_PASSTHROUGH) { @@ -4975,7 +4980,7 @@ static void kvmppc_core_destroy_vm_hv(struct kvm *kvm) } /* We don't need to emulate any privileged instructions or dcbz */ -static int kvmppc_core_emulate_op_hv(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_core_emulate_op_hv(struct kvm_vcpu *vcpu, unsigned int inst, int *advance) { return EMULATE_FAIL; diff --git a/arch/powerpc/kvm/book3s_hv_nested.c b/arch/powerpc/kvm/book3s_hv_nested.c index a221cae792dc..2c849a65db77 100644 --- a/arch/powerpc/kvm/book3s_hv_nested.c +++ b/arch/powerpc/kvm/book3s_hv_nested.c @@ -290,8 +290,7 @@ long kvmhv_enter_nested_guest(struct kvm_vcpu *vcpu) r = RESUME_HOST; break; } - r = kvmhv_run_single_vcpu(vcpu->arch.kvm_run, vcpu, hdec_exp, - lpcr); + r = kvmhv_run_single_vcpu(vcpu, hdec_exp, lpcr); } while (is_kvmppc_resume_guest(r)); /* save L2 state for return */ @@ -1234,7 +1233,7 @@ static long kvmhv_handle_nested_set_rc(struct kvm_vcpu *vcpu, /* Set the rc bit in the pte of the shadow_pgtable for the nest guest */ ret = kvmppc_hv_handle_set_rc(kvm, true, writing, - n_gpa, gp->shadow_lpid); + n_gpa, gp->l1_lpid); if (!ret) ret = -EINVAL; else @@ -1270,8 +1269,7 @@ static inline int kvmppc_radix_shift_to_level(int shift) } /* called with gp->tlb_lock held */ -static long int __kvmhv_nested_page_fault(struct kvm_run *run, - struct kvm_vcpu *vcpu, +static long int __kvmhv_nested_page_fault(struct kvm_vcpu *vcpu, struct kvm_nested_guest *gp) { struct kvm *kvm = vcpu->kvm; @@ -1354,7 +1352,7 @@ static long int __kvmhv_nested_page_fault(struct kvm_run *run, } /* passthrough of emulated MMIO case */ - return kvmppc_hv_emulate_mmio(run, vcpu, gpa, ea, writing); + return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing); } if (memslot->flags & KVM_MEM_READONLY) { if (writing) { @@ -1429,8 +1427,7 @@ static long int __kvmhv_nested_page_fault(struct kvm_run *run, rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn]; ret = kvmppc_create_pte(kvm, gp->shadow_pgtable, pte, n_gpa, level, mmu_seq, gp->shadow_lpid, rmapp, &n_rmap); - if (n_rmap) - kfree(n_rmap); + kfree(n_rmap); if (ret == -EAGAIN) ret = RESUME_GUEST; /* Let the guest try again */ @@ -1441,13 +1438,13 @@ static long int __kvmhv_nested_page_fault(struct kvm_run *run, return RESUME_GUEST; } -long int kvmhv_nested_page_fault(struct kvm_run *run, struct kvm_vcpu *vcpu) +long int kvmhv_nested_page_fault(struct kvm_vcpu *vcpu) { struct kvm_nested_guest *gp = vcpu->arch.nested; long int ret; mutex_lock(&gp->tlb_lock); - ret = __kvmhv_nested_page_fault(run, vcpu, gp); + ret = __kvmhv_nested_page_fault(vcpu, gp); mutex_unlock(&gp->tlb_lock); return ret; } diff --git a/arch/powerpc/kvm/book3s_hv_uvmem.c b/arch/powerpc/kvm/book3s_hv_uvmem.c index f91224ea034a..09d8119024db 100644 --- a/arch/powerpc/kvm/book3s_hv_uvmem.c +++ b/arch/powerpc/kvm/book3s_hv_uvmem.c @@ -749,6 +749,20 @@ static u64 kvmppc_get_secmem_size(void) const __be32 *prop; u64 size = 0; + /* + * First try the new ibm,secure-memory nodes which supersede the + * secure-memory-ranges property. + * If we found some, no need to read the deprecated ones. + */ + for_each_compatible_node(np, NULL, "ibm,secure-memory") { + prop = of_get_property(np, "reg", &len); + if (!prop) + continue; + size += of_read_number(prop + 2, 2); + } + if (size) + return size; + np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware"); if (!np) goto out; diff --git a/arch/powerpc/kvm/book3s_paired_singles.c b/arch/powerpc/kvm/book3s_paired_singles.c index bf0282775e37..a11436720a8c 100644 --- a/arch/powerpc/kvm/book3s_paired_singles.c +++ b/arch/powerpc/kvm/book3s_paired_singles.c @@ -169,7 +169,7 @@ static void kvmppc_inject_pf(struct kvm_vcpu *vcpu, ulong eaddr, bool is_store) kvmppc_book3s_queue_irqprio(vcpu, BOOK3S_INTERRUPT_DATA_STORAGE); } -static int kvmppc_emulate_fpr_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_emulate_fpr_load(struct kvm_vcpu *vcpu, int rs, ulong addr, int ls_type) { int emulated = EMULATE_FAIL; @@ -188,7 +188,7 @@ static int kvmppc_emulate_fpr_load(struct kvm_run *run, struct kvm_vcpu *vcpu, kvmppc_inject_pf(vcpu, addr, false); goto done_load; } else if (r == EMULATE_DO_MMIO) { - emulated = kvmppc_handle_load(run, vcpu, KVM_MMIO_REG_FPR | rs, + emulated = kvmppc_handle_load(vcpu, KVM_MMIO_REG_FPR | rs, len, 1); goto done_load; } @@ -213,7 +213,7 @@ done_load: return emulated; } -static int kvmppc_emulate_fpr_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_emulate_fpr_store(struct kvm_vcpu *vcpu, int rs, ulong addr, int ls_type) { int emulated = EMULATE_FAIL; @@ -248,7 +248,7 @@ static int kvmppc_emulate_fpr_store(struct kvm_run *run, struct kvm_vcpu *vcpu, if (r < 0) { kvmppc_inject_pf(vcpu, addr, true); } else if (r == EMULATE_DO_MMIO) { - emulated = kvmppc_handle_store(run, vcpu, val, len, 1); + emulated = kvmppc_handle_store(vcpu, val, len, 1); } else { emulated = EMULATE_DONE; } @@ -259,7 +259,7 @@ static int kvmppc_emulate_fpr_store(struct kvm_run *run, struct kvm_vcpu *vcpu, return emulated; } -static int kvmppc_emulate_psq_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_emulate_psq_load(struct kvm_vcpu *vcpu, int rs, ulong addr, bool w, int i) { int emulated = EMULATE_FAIL; @@ -279,12 +279,12 @@ static int kvmppc_emulate_psq_load(struct kvm_run *run, struct kvm_vcpu *vcpu, kvmppc_inject_pf(vcpu, addr, false); goto done_load; } else if ((r == EMULATE_DO_MMIO) && w) { - emulated = kvmppc_handle_load(run, vcpu, KVM_MMIO_REG_FPR | rs, + emulated = kvmppc_handle_load(vcpu, KVM_MMIO_REG_FPR | rs, 4, 1); vcpu->arch.qpr[rs] = tmp[1]; goto done_load; } else if (r == EMULATE_DO_MMIO) { - emulated = kvmppc_handle_load(run, vcpu, KVM_MMIO_REG_FQPR | rs, + emulated = kvmppc_handle_load(vcpu, KVM_MMIO_REG_FQPR | rs, 8, 1); goto done_load; } @@ -302,7 +302,7 @@ done_load: return emulated; } -static int kvmppc_emulate_psq_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_emulate_psq_store(struct kvm_vcpu *vcpu, int rs, ulong addr, bool w, int i) { int emulated = EMULATE_FAIL; @@ -318,10 +318,10 @@ static int kvmppc_emulate_psq_store(struct kvm_run *run, struct kvm_vcpu *vcpu, if (r < 0) { kvmppc_inject_pf(vcpu, addr, true); } else if ((r == EMULATE_DO_MMIO) && w) { - emulated = kvmppc_handle_store(run, vcpu, tmp[0], 4, 1); + emulated = kvmppc_handle_store(vcpu, tmp[0], 4, 1); } else if (r == EMULATE_DO_MMIO) { u64 val = ((u64)tmp[0] << 32) | tmp[1]; - emulated = kvmppc_handle_store(run, vcpu, val, 8, 1); + emulated = kvmppc_handle_store(vcpu, val, 8, 1); } else { emulated = EMULATE_DONE; } @@ -618,7 +618,7 @@ static int kvmppc_ps_one_in(struct kvm_vcpu *vcpu, bool rc, return EMULATE_DONE; } -int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) +int kvmppc_emulate_paired_single(struct kvm_vcpu *vcpu) { u32 inst; enum emulation_result emulated = EMULATE_DONE; @@ -680,7 +680,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 17, 19); addr += get_d_signext(inst); - emulated = kvmppc_emulate_psq_load(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_load(vcpu, ax_rd, addr, w, i); break; } case OP_PSQ_LU: @@ -690,7 +690,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 17, 19); addr += get_d_signext(inst); - emulated = kvmppc_emulate_psq_load(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_load(vcpu, ax_rd, addr, w, i); if (emulated == EMULATE_DONE) kvmppc_set_gpr(vcpu, ax_ra, addr); @@ -703,7 +703,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 17, 19); addr += get_d_signext(inst); - emulated = kvmppc_emulate_psq_store(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_store(vcpu, ax_rd, addr, w, i); break; } case OP_PSQ_STU: @@ -713,7 +713,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 17, 19); addr += get_d_signext(inst); - emulated = kvmppc_emulate_psq_store(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_store(vcpu, ax_rd, addr, w, i); if (emulated == EMULATE_DONE) kvmppc_set_gpr(vcpu, ax_ra, addr); @@ -733,7 +733,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 22, 24); addr += kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_psq_load(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_load(vcpu, ax_rd, addr, w, i); break; } case OP_4X_PS_CMPO0: @@ -747,7 +747,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 22, 24); addr += kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_psq_load(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_load(vcpu, ax_rd, addr, w, i); if (emulated == EMULATE_DONE) kvmppc_set_gpr(vcpu, ax_ra, addr); @@ -824,7 +824,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 22, 24); addr += kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_psq_store(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_store(vcpu, ax_rd, addr, w, i); break; } case OP_4XW_PSQ_STUX: @@ -834,7 +834,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) int i = inst_get_field(inst, 22, 24); addr += kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_psq_store(run, vcpu, ax_rd, addr, w, i); + emulated = kvmppc_emulate_psq_store(vcpu, ax_rd, addr, w, i); if (emulated == EMULATE_DONE) kvmppc_set_gpr(vcpu, ax_ra, addr); @@ -922,7 +922,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + full_d; - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_SINGLE); break; } @@ -930,7 +930,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + full_d; - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_SINGLE); if (emulated == EMULATE_DONE) @@ -941,7 +941,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + full_d; - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_DOUBLE); break; } @@ -949,7 +949,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + full_d; - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_DOUBLE); if (emulated == EMULATE_DONE) @@ -960,7 +960,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + full_d; - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_SINGLE); break; } @@ -968,7 +968,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + full_d; - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_SINGLE); if (emulated == EMULATE_DONE) @@ -979,7 +979,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + full_d; - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_DOUBLE); break; } @@ -987,7 +987,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) { ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + full_d; - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, addr, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_DOUBLE); if (emulated == EMULATE_DONE) @@ -1001,7 +1001,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0; addr += kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_SINGLE); break; } @@ -1010,7 +1010,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_SINGLE); if (emulated == EMULATE_DONE) @@ -1022,7 +1022,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_DOUBLE); break; } @@ -1031,7 +1031,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_load(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_load(vcpu, ax_rd, addr, FPU_LS_DOUBLE); if (emulated == EMULATE_DONE) @@ -1043,7 +1043,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_SINGLE); break; } @@ -1052,7 +1052,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_SINGLE); if (emulated == EMULATE_DONE) @@ -1064,7 +1064,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_DOUBLE); break; } @@ -1073,7 +1073,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = kvmppc_get_gpr(vcpu, ax_ra) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_DOUBLE); if (emulated == EMULATE_DONE) @@ -1085,7 +1085,7 @@ int kvmppc_emulate_paired_single(struct kvm_run *run, struct kvm_vcpu *vcpu) ulong addr = (ax_ra ? kvmppc_get_gpr(vcpu, ax_ra) : 0) + kvmppc_get_gpr(vcpu, ax_rb); - emulated = kvmppc_emulate_fpr_store(run, vcpu, ax_rd, + emulated = kvmppc_emulate_fpr_store(vcpu, ax_rd, addr, FPU_LS_SINGLE_LOW); break; diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index a0f6813f4560..ef54f917bdaf 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -700,7 +700,7 @@ static bool kvmppc_visible_gpa(struct kvm_vcpu *vcpu, gpa_t gpa) return kvm_is_visible_gfn(vcpu->kvm, gpa >> PAGE_SHIFT); } -int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_handle_pagefault(struct kvm_vcpu *vcpu, ulong eaddr, int vec) { bool data = (vec == BOOK3S_INTERRUPT_DATA_STORAGE); @@ -795,7 +795,7 @@ int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, /* The guest's PTE is not mapped yet. Map on the host */ if (kvmppc_mmu_map_page(vcpu, &pte, iswrite) == -EIO) { /* Exit KVM if mapping failed */ - run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; return RESUME_HOST; } if (data) @@ -808,7 +808,7 @@ int kvmppc_handle_pagefault(struct kvm_run *run, struct kvm_vcpu *vcpu, vcpu->stat.mmio_exits++; vcpu->arch.paddr_accessed = pte.raddr; vcpu->arch.vaddr_accessed = pte.eaddr; - r = kvmppc_emulate_mmio(run, vcpu); + r = kvmppc_emulate_mmio(vcpu); if ( r == RESUME_HOST_NV ) r = RESUME_HOST; } @@ -992,7 +992,7 @@ static void kvmppc_emulate_fac(struct kvm_vcpu *vcpu, ulong fac) enum emulation_result er = EMULATE_FAIL; if (!(kvmppc_get_msr(vcpu) & MSR_PR)) - er = kvmppc_emulate_instruction(vcpu->run, vcpu); + er = kvmppc_emulate_instruction(vcpu); if ((er != EMULATE_DONE) && (er != EMULATE_AGAIN)) { /* Couldn't emulate, trigger interrupt in guest */ @@ -1089,8 +1089,7 @@ static void kvmppc_clear_debug(struct kvm_vcpu *vcpu) } } -static int kvmppc_exit_pr_progint(struct kvm_run *run, struct kvm_vcpu *vcpu, - unsigned int exit_nr) +static int kvmppc_exit_pr_progint(struct kvm_vcpu *vcpu, unsigned int exit_nr) { enum emulation_result er; ulong flags; @@ -1124,7 +1123,7 @@ static int kvmppc_exit_pr_progint(struct kvm_run *run, struct kvm_vcpu *vcpu, } vcpu->stat.emulated_inst_exits++; - er = kvmppc_emulate_instruction(run, vcpu); + er = kvmppc_emulate_instruction(vcpu); switch (er) { case EMULATE_DONE: r = RESUME_GUEST_NV; @@ -1139,7 +1138,7 @@ static int kvmppc_exit_pr_progint(struct kvm_run *run, struct kvm_vcpu *vcpu, r = RESUME_GUEST; break; case EMULATE_DO_MMIO: - run->exit_reason = KVM_EXIT_MMIO; + vcpu->run->exit_reason = KVM_EXIT_MMIO; r = RESUME_HOST_NV; break; case EMULATE_EXIT_USER: @@ -1198,7 +1197,7 @@ int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, /* only care about PTEG not found errors, but leave NX alone */ if (shadow_srr1 & 0x40000000) { int idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvmppc_handle_pagefault(run, vcpu, kvmppc_get_pc(vcpu), exit_nr); + r = kvmppc_handle_pagefault(vcpu, kvmppc_get_pc(vcpu), exit_nr); srcu_read_unlock(&vcpu->kvm->srcu, idx); vcpu->stat.sp_instruc++; } else if (vcpu->arch.mmu.is_dcbz32(vcpu) && @@ -1248,7 +1247,7 @@ int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, */ if (fault_dsisr & (DSISR_NOHPTE | DSISR_PROTFAULT)) { int idx = srcu_read_lock(&vcpu->kvm->srcu); - r = kvmppc_handle_pagefault(run, vcpu, dar, exit_nr); + r = kvmppc_handle_pagefault(vcpu, dar, exit_nr); srcu_read_unlock(&vcpu->kvm->srcu, idx); } else { kvmppc_core_queue_data_storage(vcpu, dar, fault_dsisr); @@ -1292,7 +1291,7 @@ int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, break; case BOOK3S_INTERRUPT_PROGRAM: case BOOK3S_INTERRUPT_H_EMUL_ASSIST: - r = kvmppc_exit_pr_progint(run, vcpu, exit_nr); + r = kvmppc_exit_pr_progint(vcpu, exit_nr); break; case BOOK3S_INTERRUPT_SYSCALL: { @@ -1370,7 +1369,7 @@ int kvmppc_handle_exit_pr(struct kvm_run *run, struct kvm_vcpu *vcpu, emul = kvmppc_get_last_inst(vcpu, INST_GENERIC, &last_inst); if (emul == EMULATE_DONE) - r = kvmppc_exit_pr_progint(run, vcpu, exit_nr); + r = kvmppc_exit_pr_progint(vcpu, exit_nr); else r = RESUME_GUEST; @@ -1825,8 +1824,9 @@ static void kvmppc_core_vcpu_free_pr(struct kvm_vcpu *vcpu) vfree(vcpu_book3s); } -static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +static int kvmppc_vcpu_run_pr(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int ret; #ifdef CONFIG_ALTIVEC unsigned long uninitialized_var(vrsave); @@ -1834,7 +1834,7 @@ static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) /* Check if we can run the vcpu at all */ if (!vcpu->arch.sane) { - kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; ret = -EINVAL; goto out; } @@ -1861,7 +1861,7 @@ static int kvmppc_vcpu_run_pr(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) kvmppc_fix_ee_before_entry(); - ret = __kvmppc_vcpu_run(kvm_run, vcpu); + ret = __kvmppc_vcpu_run(run, vcpu); kvmppc_clear_debug(vcpu); diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c index 888afe8d35cc..c0d62a917e20 100644 --- a/arch/powerpc/kvm/booke.c +++ b/arch/powerpc/kvm/booke.c @@ -729,13 +729,14 @@ int kvmppc_core_check_requests(struct kvm_vcpu *vcpu) return r; } -int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) +int kvmppc_vcpu_run(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; int ret, s; struct debug_reg debug; if (!vcpu->arch.sane) { - kvm_run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + run->exit_reason = KVM_EXIT_INTERNAL_ERROR; return -EINVAL; } @@ -777,7 +778,7 @@ int kvmppc_vcpu_run(struct kvm_run *kvm_run, struct kvm_vcpu *vcpu) vcpu->arch.pgdir = vcpu->kvm->mm->pgd; kvmppc_fix_ee_before_entry(); - ret = __kvmppc_vcpu_run(kvm_run, vcpu); + ret = __kvmppc_vcpu_run(run, vcpu); /* No need for guest_exit. It's done in handle_exit. We also get here with interrupts enabled. */ @@ -799,11 +800,11 @@ out: return ret; } -static int emulation_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) +static int emulation_exit(struct kvm_vcpu *vcpu) { enum emulation_result er; - er = kvmppc_emulate_instruction(run, vcpu); + er = kvmppc_emulate_instruction(vcpu); switch (er) { case EMULATE_DONE: /* don't overwrite subtypes, just account kvm_stats */ @@ -820,8 +821,8 @@ static int emulation_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) __func__, vcpu->arch.regs.nip, vcpu->arch.last_inst); /* For debugging, encode the failing instruction and * report it to userspace. */ - run->hw.hardware_exit_reason = ~0ULL << 32; - run->hw.hardware_exit_reason |= vcpu->arch.last_inst; + vcpu->run->hw.hardware_exit_reason = ~0ULL << 32; + vcpu->run->hw.hardware_exit_reason |= vcpu->arch.last_inst; kvmppc_core_queue_program(vcpu, ESR_PIL); return RESUME_HOST; @@ -833,8 +834,9 @@ static int emulation_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) } } -static int kvmppc_handle_debug(struct kvm_run *run, struct kvm_vcpu *vcpu) +static int kvmppc_handle_debug(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; struct debug_reg *dbg_reg = &(vcpu->arch.dbg_reg); u32 dbsr = vcpu->arch.dbsr; @@ -953,7 +955,7 @@ static void kvmppc_restart_interrupt(struct kvm_vcpu *vcpu, } } -static int kvmppc_resume_inst_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_resume_inst_load(struct kvm_vcpu *vcpu, enum emulation_result emulated, u32 last_inst) { switch (emulated) { @@ -965,8 +967,8 @@ static int kvmppc_resume_inst_load(struct kvm_run *run, struct kvm_vcpu *vcpu, __func__, vcpu->arch.regs.nip); /* For debugging, encode the failing instruction and * report it to userspace. */ - run->hw.hardware_exit_reason = ~0ULL << 32; - run->hw.hardware_exit_reason |= last_inst; + vcpu->run->hw.hardware_exit_reason = ~0ULL << 32; + vcpu->run->hw.hardware_exit_reason |= last_inst; kvmppc_core_queue_program(vcpu, ESR_PIL); return RESUME_HOST; @@ -1023,7 +1025,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, run->ready_for_interrupt_injection = 1; if (emulated != EMULATE_DONE) { - r = kvmppc_resume_inst_load(run, vcpu, emulated, last_inst); + r = kvmppc_resume_inst_load(vcpu, emulated, last_inst); goto out; } @@ -1083,7 +1085,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, break; case BOOKE_INTERRUPT_HV_PRIV: - r = emulation_exit(run, vcpu); + r = emulation_exit(vcpu); break; case BOOKE_INTERRUPT_PROGRAM: @@ -1093,7 +1095,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, * We are here because of an SW breakpoint instr, * so lets return to host to handle. */ - r = kvmppc_handle_debug(run, vcpu); + r = kvmppc_handle_debug(vcpu); run->exit_reason = KVM_EXIT_DEBUG; kvmppc_account_exit(vcpu, DEBUG_EXITS); break; @@ -1114,7 +1116,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, break; } - r = emulation_exit(run, vcpu); + r = emulation_exit(vcpu); break; case BOOKE_INTERRUPT_FP_UNAVAIL: @@ -1281,7 +1283,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, * actually RAM. */ vcpu->arch.paddr_accessed = gpaddr; vcpu->arch.vaddr_accessed = eaddr; - r = kvmppc_emulate_mmio(run, vcpu); + r = kvmppc_emulate_mmio(vcpu); kvmppc_account_exit(vcpu, MMIO_EXITS); } @@ -1332,7 +1334,7 @@ int kvmppc_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu, } case BOOKE_INTERRUPT_DEBUG: { - r = kvmppc_handle_debug(run, vcpu); + r = kvmppc_handle_debug(vcpu); if (r == RESUME_HOST) run->exit_reason = KVM_EXIT_DEBUG; kvmppc_account_exit(vcpu, DEBUG_EXITS); diff --git a/arch/powerpc/kvm/booke.h b/arch/powerpc/kvm/booke.h index 65b4d337d337..be9da96d9f06 100644 --- a/arch/powerpc/kvm/booke.h +++ b/arch/powerpc/kvm/booke.h @@ -70,7 +70,7 @@ void kvmppc_set_tcr(struct kvm_vcpu *vcpu, u32 new_tcr); void kvmppc_set_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits); void kvmppc_clr_tsr_bits(struct kvm_vcpu *vcpu, u32 tsr_bits); -int kvmppc_booke_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_booke_emulate_op(struct kvm_vcpu *vcpu, unsigned int inst, int *advance); int kvmppc_booke_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val); int kvmppc_booke_emulate_mtspr(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); @@ -94,16 +94,12 @@ enum int_class { void kvmppc_set_pending_interrupt(struct kvm_vcpu *vcpu, enum int_class type); -extern int kvmppc_core_emulate_op_e500(struct kvm_run *run, - struct kvm_vcpu *vcpu, +extern int kvmppc_core_emulate_op_e500(struct kvm_vcpu *vcpu, unsigned int inst, int *advance); extern int kvmppc_core_emulate_mtspr_e500(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); extern int kvmppc_core_emulate_mfspr_e500(struct kvm_vcpu *vcpu, int sprn, ulong *spr_val); -extern int kvmppc_core_emulate_op_e500(struct kvm_run *run, - struct kvm_vcpu *vcpu, - unsigned int inst, int *advance); extern int kvmppc_core_emulate_mtspr_e500(struct kvm_vcpu *vcpu, int sprn, ulong spr_val); extern int kvmppc_core_emulate_mfspr_e500(struct kvm_vcpu *vcpu, int sprn, diff --git a/arch/powerpc/kvm/booke_emulate.c b/arch/powerpc/kvm/booke_emulate.c index 689ff5f90e9e..d8d38aca71bd 100644 --- a/arch/powerpc/kvm/booke_emulate.c +++ b/arch/powerpc/kvm/booke_emulate.c @@ -39,7 +39,7 @@ static void kvmppc_emul_rfci(struct kvm_vcpu *vcpu) kvmppc_set_msr(vcpu, vcpu->arch.csrr1); } -int kvmppc_booke_emulate_op(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_booke_emulate_op(struct kvm_vcpu *vcpu, unsigned int inst, int *advance) { int emulated = EMULATE_DONE; diff --git a/arch/powerpc/kvm/e500_emulate.c b/arch/powerpc/kvm/e500_emulate.c index 3d0d3ec5be96..64eb833e9f02 100644 --- a/arch/powerpc/kvm/e500_emulate.c +++ b/arch/powerpc/kvm/e500_emulate.c @@ -83,16 +83,16 @@ static int kvmppc_e500_emul_msgsnd(struct kvm_vcpu *vcpu, int rb) } #endif -static int kvmppc_e500_emul_ehpriv(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int kvmppc_e500_emul_ehpriv(struct kvm_vcpu *vcpu, unsigned int inst, int *advance) { int emulated = EMULATE_DONE; switch (get_oc(inst)) { case EHPRIV_OC_DEBUG: - run->exit_reason = KVM_EXIT_DEBUG; - run->debug.arch.address = vcpu->arch.regs.nip; - run->debug.arch.status = 0; + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + vcpu->run->debug.arch.address = vcpu->arch.regs.nip; + vcpu->run->debug.arch.status = 0; kvmppc_account_exit(vcpu, DEBUG_EXITS); emulated = EMULATE_EXIT_USER; *advance = 0; @@ -125,7 +125,7 @@ static int kvmppc_e500_emul_mftmr(struct kvm_vcpu *vcpu, unsigned int inst, return EMULATE_FAIL; } -int kvmppc_core_emulate_op_e500(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_core_emulate_op_e500(struct kvm_vcpu *vcpu, unsigned int inst, int *advance) { int emulated = EMULATE_DONE; @@ -182,8 +182,7 @@ int kvmppc_core_emulate_op_e500(struct kvm_run *run, struct kvm_vcpu *vcpu, break; case XOP_EHPRIV: - emulated = kvmppc_e500_emul_ehpriv(run, vcpu, inst, - advance); + emulated = kvmppc_e500_emul_ehpriv(vcpu, inst, advance); break; default: @@ -197,7 +196,7 @@ int kvmppc_core_emulate_op_e500(struct kvm_run *run, struct kvm_vcpu *vcpu, } if (emulated == EMULATE_FAIL) - emulated = kvmppc_booke_emulate_op(run, vcpu, inst, advance); + emulated = kvmppc_booke_emulate_op(vcpu, inst, advance); return emulated; } diff --git a/arch/powerpc/kvm/emulate.c b/arch/powerpc/kvm/emulate.c index 6fca38ca791f..ee1147c98cd8 100644 --- a/arch/powerpc/kvm/emulate.c +++ b/arch/powerpc/kvm/emulate.c @@ -191,7 +191,7 @@ static int kvmppc_emulate_mfspr(struct kvm_vcpu *vcpu, int sprn, int rt) /* XXX Should probably auto-generate instruction decoding for a particular core * from opcode tables in the future. */ -int kvmppc_emulate_instruction(struct kvm_run *run, struct kvm_vcpu *vcpu) +int kvmppc_emulate_instruction(struct kvm_vcpu *vcpu) { u32 inst; int rs, rt, sprn; @@ -270,9 +270,9 @@ int kvmppc_emulate_instruction(struct kvm_run *run, struct kvm_vcpu *vcpu) * these are illegal instructions. */ if (inst == KVMPPC_INST_SW_BREAKPOINT) { - run->exit_reason = KVM_EXIT_DEBUG; - run->debug.arch.status = 0; - run->debug.arch.address = kvmppc_get_pc(vcpu); + vcpu->run->exit_reason = KVM_EXIT_DEBUG; + vcpu->run->debug.arch.status = 0; + vcpu->run->debug.arch.address = kvmppc_get_pc(vcpu); emulated = EMULATE_EXIT_USER; advance = 0; } else @@ -285,7 +285,7 @@ int kvmppc_emulate_instruction(struct kvm_run *run, struct kvm_vcpu *vcpu) } if (emulated == EMULATE_FAIL) { - emulated = vcpu->kvm->arch.kvm_ops->emulate_op(run, vcpu, inst, + emulated = vcpu->kvm->arch.kvm_ops->emulate_op(vcpu, inst, &advance); if (emulated == EMULATE_AGAIN) { advance = 0; diff --git a/arch/powerpc/kvm/emulate_loadstore.c b/arch/powerpc/kvm/emulate_loadstore.c index 135d0e686622..48272a9b9c30 100644 --- a/arch/powerpc/kvm/emulate_loadstore.c +++ b/arch/powerpc/kvm/emulate_loadstore.c @@ -71,7 +71,6 @@ static bool kvmppc_check_altivec_disabled(struct kvm_vcpu *vcpu) */ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) { - struct kvm_run *run = vcpu->run; u32 inst; enum emulation_result emulated = EMULATE_FAIL; int advance = 1; @@ -104,10 +103,10 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) int instr_byte_swap = op.type & BYTEREV; if (op.type & SIGNEXT) - emulated = kvmppc_handle_loads(run, vcpu, + emulated = kvmppc_handle_loads(vcpu, op.reg, size, !instr_byte_swap); else - emulated = kvmppc_handle_load(run, vcpu, + emulated = kvmppc_handle_load(vcpu, op.reg, size, !instr_byte_swap); if ((op.type & UPDATE) && (emulated != EMULATE_FAIL)) @@ -124,10 +123,10 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) vcpu->arch.mmio_sp64_extend = 1; if (op.type & SIGNEXT) - emulated = kvmppc_handle_loads(run, vcpu, + emulated = kvmppc_handle_loads(vcpu, KVM_MMIO_REG_FPR|op.reg, size, 1); else - emulated = kvmppc_handle_load(run, vcpu, + emulated = kvmppc_handle_load(vcpu, KVM_MMIO_REG_FPR|op.reg, size, 1); if ((op.type & UPDATE) && (emulated != EMULATE_FAIL)) @@ -164,12 +163,12 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) if (size == 16) { vcpu->arch.mmio_vmx_copy_nums = 2; - emulated = kvmppc_handle_vmx_load(run, - vcpu, KVM_MMIO_REG_VMX|op.reg, + emulated = kvmppc_handle_vmx_load(vcpu, + KVM_MMIO_REG_VMX|op.reg, 8, 1); } else { vcpu->arch.mmio_vmx_copy_nums = 1; - emulated = kvmppc_handle_vmx_load(run, vcpu, + emulated = kvmppc_handle_vmx_load(vcpu, KVM_MMIO_REG_VMX|op.reg, size, 1); } @@ -217,7 +216,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) io_size_each = op.element_size; } - emulated = kvmppc_handle_vsx_load(run, vcpu, + emulated = kvmppc_handle_vsx_load(vcpu, KVM_MMIO_REG_VSX|op.reg, io_size_each, 1, op.type & SIGNEXT); break; @@ -227,8 +226,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) /* if need byte reverse, op.val has been reversed by * analyse_instr(). */ - emulated = kvmppc_handle_store(run, vcpu, op.val, - size, 1); + emulated = kvmppc_handle_store(vcpu, op.val, size, 1); if ((op.type & UPDATE) && (emulated != EMULATE_FAIL)) kvmppc_set_gpr(vcpu, op.update_reg, op.ea); @@ -250,7 +248,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) if (op.type & FPCONV) vcpu->arch.mmio_sp64_extend = 1; - emulated = kvmppc_handle_store(run, vcpu, + emulated = kvmppc_handle_store(vcpu, VCPU_FPR(vcpu, op.reg), size, 1); if ((op.type & UPDATE) && (emulated != EMULATE_FAIL)) @@ -290,12 +288,12 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) if (size == 16) { vcpu->arch.mmio_vmx_copy_nums = 2; - emulated = kvmppc_handle_vmx_store(run, - vcpu, op.reg, 8, 1); + emulated = kvmppc_handle_vmx_store(vcpu, + op.reg, 8, 1); } else { vcpu->arch.mmio_vmx_copy_nums = 1; - emulated = kvmppc_handle_vmx_store(run, - vcpu, op.reg, size, 1); + emulated = kvmppc_handle_vmx_store(vcpu, + op.reg, size, 1); } break; @@ -338,7 +336,7 @@ int kvmppc_emulate_loadstore(struct kvm_vcpu *vcpu) io_size_each = op.element_size; } - emulated = kvmppc_handle_vsx_store(run, vcpu, + emulated = kvmppc_handle_vsx_store(vcpu, op.reg, io_size_each, 1); break; } diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 27ccff612903..dd7d141e33e8 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -279,7 +279,7 @@ out: } EXPORT_SYMBOL_GPL(kvmppc_sanity_check); -int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu) +int kvmppc_emulate_mmio(struct kvm_vcpu *vcpu) { enum emulation_result er; int r; @@ -295,7 +295,7 @@ int kvmppc_emulate_mmio(struct kvm_run *run, struct kvm_vcpu *vcpu) r = RESUME_GUEST; break; case EMULATE_DO_MMIO: - run->exit_reason = KVM_EXIT_MMIO; + vcpu->run->exit_reason = KVM_EXIT_MMIO; /* We must reload nonvolatiles because "update" load/store * instructions modify register state. */ /* Future optimization: only reload non-volatiles if they were @@ -1107,9 +1107,9 @@ static inline u32 dp_to_sp(u64 fprd) #define dp_to_sp(x) (x) #endif /* CONFIG_PPC_FPU */ -static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu, - struct kvm_run *run) +static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; u64 uninitialized_var(gpr); if (run->mmio.len > sizeof(gpr)) { @@ -1219,10 +1219,11 @@ static void kvmppc_complete_mmio_load(struct kvm_vcpu *vcpu, } } -static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +static int __kvmppc_handle_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian, int sign_extend) { + struct kvm_run *run = vcpu->run; int idx, ret; bool host_swabbed; @@ -1256,7 +1257,7 @@ static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, srcu_read_unlock(&vcpu->kvm->srcu, idx); if (!ret) { - kvmppc_complete_mmio_load(vcpu, run); + kvmppc_complete_mmio_load(vcpu); vcpu->mmio_needed = 0; return EMULATE_DONE; } @@ -1264,24 +1265,24 @@ static int __kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, return EMULATE_DO_MMIO; } -int kvmppc_handle_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian) { - return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 0); + return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 0); } EXPORT_SYMBOL_GPL(kvmppc_handle_load); /* Same as above, but sign extends */ -int kvmppc_handle_loads(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_loads(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian) { - return __kvmppc_handle_load(run, vcpu, rt, bytes, is_default_endian, 1); + return __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 1); } #ifdef CONFIG_VSX -int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_vsx_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian, int mmio_sign_extend) { @@ -1292,13 +1293,13 @@ int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, return EMULATE_FAIL; while (vcpu->arch.mmio_vsx_copy_nums) { - emulated = __kvmppc_handle_load(run, vcpu, rt, bytes, + emulated = __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, mmio_sign_extend); if (emulated != EMULATE_DONE) break; - vcpu->arch.paddr_accessed += run->mmio.len; + vcpu->arch.paddr_accessed += vcpu->run->mmio.len; vcpu->arch.mmio_vsx_copy_nums--; vcpu->arch.mmio_vsx_offset++; @@ -1307,9 +1308,10 @@ int kvmppc_handle_vsx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, } #endif /* CONFIG_VSX */ -int kvmppc_handle_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_store(struct kvm_vcpu *vcpu, u64 val, unsigned int bytes, int is_default_endian) { + struct kvm_run *run = vcpu->run; void *data = run->mmio.data; int idx, ret; bool host_swabbed; @@ -1423,7 +1425,7 @@ static inline int kvmppc_get_vsr_data(struct kvm_vcpu *vcpu, int rs, u64 *val) return result; } -int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_vsx_store(struct kvm_vcpu *vcpu, int rs, unsigned int bytes, int is_default_endian) { u64 val; @@ -1439,13 +1441,13 @@ int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, if (kvmppc_get_vsr_data(vcpu, rs, &val) == -1) return EMULATE_FAIL; - emulated = kvmppc_handle_store(run, vcpu, + emulated = kvmppc_handle_store(vcpu, val, bytes, is_default_endian); if (emulated != EMULATE_DONE) break; - vcpu->arch.paddr_accessed += run->mmio.len; + vcpu->arch.paddr_accessed += vcpu->run->mmio.len; vcpu->arch.mmio_vsx_copy_nums--; vcpu->arch.mmio_vsx_offset++; @@ -1454,19 +1456,19 @@ int kvmppc_handle_vsx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, return emulated; } -static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu, - struct kvm_run *run) +static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; enum emulation_result emulated = EMULATE_FAIL; int r; vcpu->arch.paddr_accessed += run->mmio.len; if (!vcpu->mmio_is_write) { - emulated = kvmppc_handle_vsx_load(run, vcpu, vcpu->arch.io_gpr, + emulated = kvmppc_handle_vsx_load(vcpu, vcpu->arch.io_gpr, run->mmio.len, 1, vcpu->arch.mmio_sign_extend); } else { - emulated = kvmppc_handle_vsx_store(run, vcpu, + emulated = kvmppc_handle_vsx_store(vcpu, vcpu->arch.io_gpr, run->mmio.len, 1); } @@ -1490,7 +1492,7 @@ static int kvmppc_emulate_mmio_vsx_loadstore(struct kvm_vcpu *vcpu, #endif /* CONFIG_VSX */ #ifdef CONFIG_ALTIVEC -int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_vmx_load(struct kvm_vcpu *vcpu, unsigned int rt, unsigned int bytes, int is_default_endian) { enum emulation_result emulated = EMULATE_DONE; @@ -1499,13 +1501,13 @@ int kvmppc_handle_vmx_load(struct kvm_run *run, struct kvm_vcpu *vcpu, return EMULATE_FAIL; while (vcpu->arch.mmio_vmx_copy_nums) { - emulated = __kvmppc_handle_load(run, vcpu, rt, bytes, + emulated = __kvmppc_handle_load(vcpu, rt, bytes, is_default_endian, 0); if (emulated != EMULATE_DONE) break; - vcpu->arch.paddr_accessed += run->mmio.len; + vcpu->arch.paddr_accessed += vcpu->run->mmio.len; vcpu->arch.mmio_vmx_copy_nums--; vcpu->arch.mmio_vmx_offset++; } @@ -1585,7 +1587,7 @@ int kvmppc_get_vmx_byte(struct kvm_vcpu *vcpu, int index, u64 *val) return result; } -int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, +int kvmppc_handle_vmx_store(struct kvm_vcpu *vcpu, unsigned int rs, unsigned int bytes, int is_default_endian) { u64 val = 0; @@ -1620,12 +1622,12 @@ int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, return EMULATE_FAIL; } - emulated = kvmppc_handle_store(run, vcpu, val, bytes, + emulated = kvmppc_handle_store(vcpu, val, bytes, is_default_endian); if (emulated != EMULATE_DONE) break; - vcpu->arch.paddr_accessed += run->mmio.len; + vcpu->arch.paddr_accessed += vcpu->run->mmio.len; vcpu->arch.mmio_vmx_copy_nums--; vcpu->arch.mmio_vmx_offset++; } @@ -1633,19 +1635,19 @@ int kvmppc_handle_vmx_store(struct kvm_run *run, struct kvm_vcpu *vcpu, return emulated; } -static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu, - struct kvm_run *run) +static int kvmppc_emulate_mmio_vmx_loadstore(struct kvm_vcpu *vcpu) { + struct kvm_run *run = vcpu->run; enum emulation_result emulated = EMULATE_FAIL; int r; vcpu->arch.paddr_accessed += run->mmio.len; if (!vcpu->mmio_is_write) { - emulated = kvmppc_handle_vmx_load(run, vcpu, + emulated = kvmppc_handle_vmx_load(vcpu, vcpu->arch.io_gpr, run->mmio.len, 1); } else { - emulated = kvmppc_handle_vmx_store(run, vcpu, + emulated = kvmppc_handle_vmx_store(vcpu, vcpu->arch.io_gpr, run->mmio.len, 1); } @@ -1775,7 +1777,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) if (vcpu->mmio_needed) { vcpu->mmio_needed = 0; if (!vcpu->mmio_is_write) - kvmppc_complete_mmio_load(vcpu, run); + kvmppc_complete_mmio_load(vcpu); #ifdef CONFIG_VSX if (vcpu->arch.mmio_vsx_copy_nums > 0) { vcpu->arch.mmio_vsx_copy_nums--; @@ -1783,7 +1785,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } if (vcpu->arch.mmio_vsx_copy_nums > 0) { - r = kvmppc_emulate_mmio_vsx_loadstore(vcpu, run); + r = kvmppc_emulate_mmio_vsx_loadstore(vcpu); if (r == RESUME_HOST) { vcpu->mmio_needed = 1; goto out; @@ -1797,7 +1799,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) } if (vcpu->arch.mmio_vmx_copy_nums > 0) { - r = kvmppc_emulate_mmio_vmx_loadstore(vcpu, run); + r = kvmppc_emulate_mmio_vmx_loadstore(vcpu); if (r == RESUME_HOST) { vcpu->mmio_needed = 1; goto out; @@ -1830,7 +1832,7 @@ int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) if (run->immediate_exit) r = -EINTR; else - r = kvmppc_vcpu_run(run, vcpu); + r = kvmppc_vcpu_run(vcpu); kvm_sigset_deactivate(vcpu); diff --git a/arch/powerpc/kvm/trace_hv.h b/arch/powerpc/kvm/trace_hv.h index 8a1e3b0047f1..4a61a971c34e 100644 --- a/arch/powerpc/kvm/trace_hv.h +++ b/arch/powerpc/kvm/trace_hv.h @@ -472,9 +472,9 @@ TRACE_EVENT(kvmppc_run_vcpu_enter, ); TRACE_EVENT(kvmppc_run_vcpu_exit, - TP_PROTO(struct kvm_vcpu *vcpu, struct kvm_run *run), + TP_PROTO(struct kvm_vcpu *vcpu), - TP_ARGS(vcpu, run), + TP_ARGS(vcpu), TP_STRUCT__entry( __field(int, vcpu_id) @@ -484,7 +484,7 @@ TRACE_EVENT(kvmppc_run_vcpu_exit, TP_fast_assign( __entry->vcpu_id = vcpu->vcpu_id; - __entry->exit = run->exit_reason; + __entry->exit = vcpu->run->exit_reason; __entry->ret = vcpu->arch.ret; ), diff --git a/arch/powerpc/platforms/powernv/vas-fault.c b/arch/powerpc/platforms/powernv/vas-fault.c index 25db70be4c9c..266a6ca5e15e 100644 --- a/arch/powerpc/platforms/powernv/vas-fault.c +++ b/arch/powerpc/platforms/powernv/vas-fault.c @@ -127,7 +127,7 @@ static void update_csb(struct vas_window *window, return; } - use_mm(window->mm); + kthread_use_mm(window->mm); rc = copy_to_user(csb_addr, &csb, sizeof(csb)); /* * User space polls on csb.flags (first byte). So add barrier @@ -139,7 +139,7 @@ static void update_csb(struct vas_window *window, smp_mb(); rc = copy_to_user(csb_addr, &csb, sizeof(u8)); } - unuse_mm(window->mm); + kthread_unuse_mm(window->mm); put_task_struct(tsk); /* Success */ diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig index c733007b90ab..128192e14ff2 100644 --- a/arch/riscv/Kconfig +++ b/arch/riscv/Kconfig @@ -12,64 +12,70 @@ config 32BIT config RISCV def_bool y - select OF - select OF_EARLY_FLATTREE - select OF_IRQ + select ARCH_CLOCKSOURCE_INIT select ARCH_HAS_BINFMT_FLAT + select ARCH_HAS_DEBUG_VIRTUAL if MMU select ARCH_HAS_DEBUG_WX + select ARCH_HAS_GCOV_PROFILE_ALL + select ARCH_HAS_GIGANTIC_PAGE + select ARCH_HAS_MMIOWB + select ARCH_HAS_PTE_SPECIAL + select ARCH_HAS_SET_DIRECT_MAP + select ARCH_HAS_SET_MEMORY + select ARCH_HAS_STRICT_KERNEL_RWX if MMU + select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU select ARCH_WANT_FRAME_POINTERS + select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select CLONE_BACKWARDS select COMMON_CLK + select EDAC_SUPPORT + select GENERIC_ARCH_TOPOLOGY if SMP + select GENERIC_ATOMIC64 if !64BIT select GENERIC_CLOCKEVENTS + select GENERIC_GETTIMEOFDAY if HAVE_GENERIC_VDSO + select GENERIC_IOREMAP + select GENERIC_IRQ_MULTI_HANDLER select GENERIC_IRQ_SHOW select GENERIC_PCI_IOMAP + select GENERIC_PTDUMP if MMU select GENERIC_SCHED_CLOCK + select GENERIC_SMP_IDLE_THREAD select GENERIC_STRNCPY_FROM_USER if MMU select GENERIC_STRNLEN_USER if MMU - select GENERIC_SMP_IDLE_THREAD - select GENERIC_ATOMIC64 if !64BIT - select GENERIC_IOREMAP - select GENERIC_PTDUMP if MMU + select GENERIC_TIME_VSYSCALL if MMU && 64BIT + select HANDLE_DOMAIN_IRQ select HAVE_ARCH_AUDITSYSCALL + select HAVE_ARCH_KASAN if MMU && 64BIT + select HAVE_ARCH_KGDB + select HAVE_ARCH_KGDB_QXFER_PKT + select HAVE_ARCH_MMAP_RND_BITS if MMU select HAVE_ARCH_SECCOMP_FILTER + select HAVE_ARCH_TRACEHOOK select HAVE_ASM_MODVERSIONS + select HAVE_COPY_THREAD_TLS select HAVE_DMA_CONTIGUOUS if MMU + select HAVE_EBPF_JIT if MMU select HAVE_FUTEX_CMPXCHG if FUTEX + select HAVE_GENERIC_VDSO if MMU && 64BIT + select HAVE_PCI select HAVE_PERF_EVENTS select HAVE_PERF_REGS select HAVE_PERF_USER_STACK_DUMP select HAVE_SYSCALL_TRACEPOINTS select IRQ_DOMAIN - select SPARSE_IRQ - select SYSCTL_EXCEPTION_TRACE - select HAVE_ARCH_TRACEHOOK - select HAVE_PCI select MODULES_USE_ELF_RELA if MODULES select MODULE_SECTIONS if MODULES - select THREAD_INFO_IN_TASK + select OF + select OF_EARLY_FLATTREE + select OF_IRQ select PCI_DOMAINS_GENERIC if PCI select PCI_MSI if PCI + select RISCV_INTC select RISCV_TIMER - select GENERIC_IRQ_MULTI_HANDLER - select GENERIC_ARCH_TOPOLOGY if SMP - select ARCH_HAS_PTE_SPECIAL - select ARCH_HAS_MMIOWB - select ARCH_HAS_DEBUG_VIRTUAL if MMU - select HAVE_EBPF_JIT if MMU - select EDAC_SUPPORT - select ARCH_HAS_GIGANTIC_PAGE - select ARCH_HAS_SET_DIRECT_MAP - select ARCH_HAS_SET_MEMORY - select ARCH_HAS_STRICT_KERNEL_RWX if MMU - select ARCH_WANT_HUGE_PMD_SHARE if 64BIT select SPARSEMEM_STATIC if 32BIT - select ARCH_WANT_DEFAULT_TOPDOWN_MMAP_LAYOUT if MMU - select HAVE_ARCH_MMAP_RND_BITS if MMU - select ARCH_HAS_GCOV_PROFILE_ALL - select HAVE_COPY_THREAD_TLS - select HAVE_ARCH_KASAN if MMU && 64BIT - select HAVE_ARCH_KGDB - select HAVE_ARCH_KGDB_QXFER_PKT + select SPARSE_IRQ + select SYSCTL_EXCEPTION_TRACE + select THREAD_INFO_IN_TASK config ARCH_MMAP_RND_BITS_MIN default 18 if 64BIT @@ -196,11 +202,11 @@ config ARCH_RV64I bool "RV64I" select 64BIT select ARCH_SUPPORTS_INT128 if CC_HAS_INT128 && GCC_VERSION >= 50000 - select HAVE_FUNCTION_TRACER - select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FTRACE_MCOUNT_RECORD select HAVE_DYNAMIC_FTRACE if MMU select HAVE_DYNAMIC_FTRACE_WITH_REGS if HAVE_DYNAMIC_FTRACE + select HAVE_FTRACE_MCOUNT_RECORD + select HAVE_FUNCTION_GRAPH_TRACER + select HAVE_FUNCTION_TRACER select SWIOTLB if MMU endchoice diff --git a/arch/riscv/include/asm/clocksource.h b/arch/riscv/include/asm/clocksource.h new file mode 100644 index 000000000000..482185566b0c --- /dev/null +++ b/arch/riscv/include/asm/clocksource.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_CLOCKSOURCE_H +#define _ASM_CLOCKSOURCE_H + +#include <asm/vdso/clocksource.h> + +#endif diff --git a/arch/riscv/include/asm/irq.h b/arch/riscv/include/asm/irq.h index 6e1b0e0325eb..9807ad164015 100644 --- a/arch/riscv/include/asm/irq.h +++ b/arch/riscv/include/asm/irq.h @@ -10,11 +10,6 @@ #include <linux/interrupt.h> #include <linux/linkage.h> -#define NR_IRQS 0 - -void riscv_timer_interrupt(void); -void riscv_software_interrupt(void); - #include <asm-generic/irq.h> #endif /* _ASM_RISCV_IRQ_H */ diff --git a/arch/riscv/include/asm/processor.h b/arch/riscv/include/asm/processor.h index 3ddb798264f1..bdddcd5c1b71 100644 --- a/arch/riscv/include/asm/processor.h +++ b/arch/riscv/include/asm/processor.h @@ -8,6 +8,8 @@ #include <linux/const.h> +#include <vdso/processor.h> + #include <asm/ptrace.h> /* @@ -58,16 +60,6 @@ static inline void release_thread(struct task_struct *dead_task) extern unsigned long get_wchan(struct task_struct *p); -static inline void cpu_relax(void) -{ -#ifdef __riscv_muldiv - int dummy; - /* In lieu of a halt instruction, induce a long-latency stall. */ - __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); -#endif - barrier(); -} - static inline void wait_for_interrupt(void) { __asm__ __volatile__ ("wfi"); @@ -75,6 +67,7 @@ static inline void wait_for_interrupt(void) struct device_node; int riscv_of_processor_hartid(struct device_node *node); +int riscv_of_parent_hartid(struct device_node *node); extern void riscv_fill_hwcap(void); diff --git a/arch/riscv/include/asm/smp.h b/arch/riscv/include/asm/smp.h index f4c7cfda6b7f..40bb1c15a731 100644 --- a/arch/riscv/include/asm/smp.h +++ b/arch/riscv/include/asm/smp.h @@ -28,6 +28,9 @@ void show_ipi_stats(struct seq_file *p, int prec); /* SMP initialization hook for setup_arch */ void __init setup_smp(void); +/* Called from C code, this handles an IPI. */ +void handle_IPI(struct pt_regs *regs); + /* Hook for the generic smp_call_function_many() routine. */ void arch_send_call_function_ipi_mask(struct cpumask *mask); diff --git a/arch/riscv/include/asm/vdso.h b/arch/riscv/include/asm/vdso.h index 7a7fce63c474..8454f746bbfd 100644 --- a/arch/riscv/include/asm/vdso.h +++ b/arch/riscv/include/asm/vdso.h @@ -10,8 +10,10 @@ #include <linux/types.h> +#ifndef GENERIC_TIME_VSYSCALL struct vdso_data { }; +#endif /* * The VDSO symbols are mapped into Linux so we can just use regular symbol diff --git a/arch/riscv/include/asm/vdso/clocksource.h b/arch/riscv/include/asm/vdso/clocksource.h new file mode 100644 index 000000000000..df6ea65c1dec --- /dev/null +++ b/arch/riscv/include/asm/vdso/clocksource.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSOCLOCKSOURCE_H +#define __ASM_VDSOCLOCKSOURCE_H + +#define VDSO_ARCH_CLOCKMODES \ + VDSO_CLOCKMODE_ARCHTIMER + +#endif diff --git a/arch/riscv/include/asm/vdso/gettimeofday.h b/arch/riscv/include/asm/vdso/gettimeofday.h new file mode 100644 index 000000000000..c8e818688ec1 --- /dev/null +++ b/arch/riscv/include/asm/vdso/gettimeofday.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_GETTIMEOFDAY_H +#define __ASM_VDSO_GETTIMEOFDAY_H + +#ifndef __ASSEMBLY__ + +#include <asm/unistd.h> +#include <asm/csr.h> +#include <uapi/linux/time.h> + +#define VDSO_HAS_CLOCK_GETRES 1 + +static __always_inline +int gettimeofday_fallback(struct __kernel_old_timeval *_tv, + struct timezone *_tz) +{ + register struct __kernel_old_timeval *tv asm("a0") = _tv; + register struct timezone *tz asm("a1") = _tz; + register long ret asm("a0"); + register long nr asm("a7") = __NR_gettimeofday; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(tv), "r"(tz), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +long clock_gettime_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_gettime; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline +int clock_getres_fallback(clockid_t _clkid, struct __kernel_timespec *_ts) +{ + register clockid_t clkid asm("a0") = _clkid; + register struct __kernel_timespec *ts asm("a1") = _ts; + register long ret asm("a0"); + register long nr asm("a7") = __NR_clock_getres; + + asm volatile ("ecall\n" + : "=r" (ret) + : "r"(clkid), "r"(ts), "r"(nr) + : "memory"); + + return ret; +} + +static __always_inline u64 __arch_get_hw_counter(s32 clock_mode) +{ + /* + * The purpose of csr_read(CSR_TIME) is to trap the system into + * M-mode to obtain the value of CSR_TIME. Hence, unlike other + * architecture, no fence instructions surround the csr_read() + */ + return csr_read(CSR_TIME); +} + +static __always_inline const struct vdso_data *__arch_get_vdso_data(void) +{ + return _vdso_data; +} + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_GETTIMEOFDAY_H */ diff --git a/arch/riscv/include/asm/vdso/processor.h b/arch/riscv/include/asm/vdso/processor.h new file mode 100644 index 000000000000..82a5693b1861 --- /dev/null +++ b/arch/riscv/include/asm/vdso/processor.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __ASM_VDSO_PROCESSOR_H +#define __ASM_VDSO_PROCESSOR_H + +#ifndef __ASSEMBLY__ + +static inline void cpu_relax(void) +{ +#ifdef __riscv_muldiv + int dummy; + /* In lieu of a halt instruction, induce a long-latency stall. */ + __asm__ __volatile__ ("div %0, %0, zero" : "=r" (dummy)); +#endif + barrier(); +} + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_VDSO_PROCESSOR_H */ diff --git a/arch/riscv/include/asm/vdso/vsyscall.h b/arch/riscv/include/asm/vdso/vsyscall.h new file mode 100644 index 000000000000..82fd5d83bd60 --- /dev/null +++ b/arch/riscv/include/asm/vdso/vsyscall.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_VDSO_VSYSCALL_H +#define __ASM_VDSO_VSYSCALL_H + +#ifndef __ASSEMBLY__ + +#include <linux/timekeeper_internal.h> +#include <vdso/datapage.h> + +extern struct vdso_data *vdso_data; + +/* + * Update the vDSO data page to keep in sync with kernel timekeeping. + */ +static __always_inline struct vdso_data *__riscv_get_k_vdso_data(void) +{ + return vdso_data; +} + +#define __arch_get_k_vdso_data __riscv_get_k_vdso_data + +/* The asm-generic header needs to be included after the definitions above */ +#include <asm-generic/vdso/vsyscall.h> + +#endif /* !__ASSEMBLY__ */ + +#endif /* __ASM_VDSO_VSYSCALL_H */ diff --git a/arch/riscv/kernel/cpu.c b/arch/riscv/kernel/cpu.c index 40a3c442ac5f..6d59e6906fdd 100644 --- a/arch/riscv/kernel/cpu.c +++ b/arch/riscv/kernel/cpu.c @@ -44,6 +44,22 @@ int riscv_of_processor_hartid(struct device_node *node) return hart; } +/* + * Find hart ID of the CPU DT node under which given DT node falls. + * + * To achieve this, we walk up the DT tree until we find an active + * RISC-V core (HART) node and extract the cpuid from it. + */ +int riscv_of_parent_hartid(struct device_node *node) +{ + for (; node; node = node->parent) { + if (of_device_is_compatible(node, "riscv")) + return riscv_of_processor_hartid(node); + } + + return -1; +} + #ifdef CONFIG_PROC_FS static void print_isa(struct seq_file *f, const char *isa) diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S index 56d071b2c0a1..cae7e6d4c7ef 100644 --- a/arch/riscv/kernel/entry.S +++ b/arch/riscv/kernel/entry.S @@ -106,7 +106,9 @@ _save_context: /* Handle interrupts */ move a0, sp /* pt_regs */ - tail do_IRQ + la a1, handle_arch_irq + REG_L a1, (a1) + jr a1 1: /* * Exceptions run with interrupts enabled or disabled depending on the diff --git a/arch/riscv/kernel/irq.c b/arch/riscv/kernel/irq.c index 345c4f2eba13..7207fa08d78f 100644 --- a/arch/riscv/kernel/irq.c +++ b/arch/riscv/kernel/irq.c @@ -7,7 +7,6 @@ #include <linux/interrupt.h> #include <linux/irqchip.h> -#include <linux/irqdomain.h> #include <linux/seq_file.h> #include <asm/smp.h> @@ -17,37 +16,9 @@ int arch_show_interrupts(struct seq_file *p, int prec) return 0; } -asmlinkage __visible void __irq_entry do_IRQ(struct pt_regs *regs) -{ - struct pt_regs *old_regs = set_irq_regs(regs); - - irq_enter(); - switch (regs->cause & ~CAUSE_IRQ_FLAG) { - case RV_IRQ_TIMER: - riscv_timer_interrupt(); - break; -#ifdef CONFIG_SMP - case RV_IRQ_SOFT: - /* - * We only use software interrupts to pass IPIs, so if a non-SMP - * system gets one, then we don't know what to do. - */ - riscv_software_interrupt(); - break; -#endif - case RV_IRQ_EXT: - handle_arch_irq(regs); - break; - default: - pr_alert("unexpected interrupt cause 0x%lx", regs->cause); - BUG(); - } - irq_exit(); - - set_irq_regs(old_regs); -} - void __init init_IRQ(void) { irqchip_init(); + if (!handle_arch_irq) + panic("No interrupt controller found."); } diff --git a/arch/riscv/kernel/patch.c b/arch/riscv/kernel/patch.c index 5805791cd5b5..d4a64dfed342 100644 --- a/arch/riscv/kernel/patch.c +++ b/arch/riscv/kernel/patch.c @@ -11,6 +11,7 @@ #include <asm/kprobes.h> #include <asm/cacheflush.h> #include <asm/fixmap.h> +#include <asm/patch.h> struct patch_insn { void *addr; diff --git a/arch/riscv/kernel/smp.c b/arch/riscv/kernel/smp.c index a65a8fa0c22d..b1d4f452f843 100644 --- a/arch/riscv/kernel/smp.c +++ b/arch/riscv/kernel/smp.c @@ -123,11 +123,14 @@ static inline void clear_ipi(void) clint_clear_ipi(cpuid_to_hartid_map(smp_processor_id())); } -void riscv_software_interrupt(void) +void handle_IPI(struct pt_regs *regs) { + struct pt_regs *old_regs = set_irq_regs(regs); unsigned long *pending_ipis = &ipi_data[smp_processor_id()].bits; unsigned long *stats = ipi_data[smp_processor_id()].stats; + irq_enter(); + clear_ipi(); while (true) { @@ -138,7 +141,7 @@ void riscv_software_interrupt(void) ops = xchg(pending_ipis, 0); if (ops == 0) - return; + goto done; if (ops & (1 << IPI_RESCHEDULE)) { stats[IPI_RESCHEDULE]++; @@ -160,6 +163,10 @@ void riscv_software_interrupt(void) /* Order data access and bit testing. */ mb(); } + +done: + irq_exit(); + set_irq_regs(old_regs); } static const char * const ipi_names[] = { diff --git a/arch/riscv/kernel/time.c b/arch/riscv/kernel/time.c index 6a53c02e9c73..4d3a1048ad8b 100644 --- a/arch/riscv/kernel/time.c +++ b/arch/riscv/kernel/time.c @@ -26,3 +26,12 @@ void __init time_init(void) lpj_fine = riscv_timebase / HZ; timer_probe(); } + +void clocksource_arch_init(struct clocksource *cs) +{ +#ifdef CONFIG_GENERIC_GETTIMEOFDAY + cs->vdso_clock_mode = VDSO_CLOCKMODE_ARCHTIMER; +#else + cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; +#endif +} diff --git a/arch/riscv/kernel/traps.c b/arch/riscv/kernel/traps.c index 5080fdf8c296..ecec1778e3a4 100644 --- a/arch/riscv/kernel/traps.c +++ b/arch/riscv/kernel/traps.c @@ -183,6 +183,4 @@ void trap_init(void) csr_write(CSR_SCRATCH, 0); /* Set the exception vector address */ csr_write(CSR_TVEC, &handle_exception); - /* Enable interrupts */ - csr_write(CSR_IE, IE_SIE); } diff --git a/arch/riscv/kernel/vdso.c b/arch/riscv/kernel/vdso.c index e827fae3bf90..678204231700 100644 --- a/arch/riscv/kernel/vdso.c +++ b/arch/riscv/kernel/vdso.c @@ -11,8 +11,12 @@ #include <linux/slab.h> #include <linux/binfmts.h> #include <linux/err.h> - +#include <asm/page.h> +#ifdef GENERIC_TIME_VSYSCALL +#include <vdso/datapage.h> +#else #include <asm/vdso.h> +#endif extern char vdso_start[], vdso_end[]; @@ -26,7 +30,7 @@ static union { struct vdso_data data; u8 page[PAGE_SIZE]; } vdso_data_store __page_aligned_data; -static struct vdso_data *vdso_data = &vdso_data_store.data; +struct vdso_data *vdso_data = &vdso_data_store.data; static int __init vdso_init(void) { @@ -75,13 +79,22 @@ int arch_setup_additional_pages(struct linux_binprm *bprm, */ mm->context.vdso = (void *)vdso_base; - ret = install_special_mapping(mm, vdso_base, vdso_len, + ret = + install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT, (VM_READ | VM_EXEC | VM_MAYREAD | VM_MAYWRITE | VM_MAYEXEC), vdso_pagelist); - if (unlikely(ret)) + if (unlikely(ret)) { mm->context.vdso = NULL; + goto end; + } + vdso_base += (vdso_pages << PAGE_SHIFT); + ret = install_special_mapping(mm, vdso_base, PAGE_SIZE, + (VM_READ | VM_MAYREAD), &vdso_pagelist[vdso_pages]); + + if (unlikely(ret)) + mm->context.vdso = NULL; end: mmap_write_unlock(mm); return ret; @@ -91,5 +104,8 @@ const char *arch_vma_name(struct vm_area_struct *vma) { if (vma->vm_mm && (vma->vm_start == (long)vma->vm_mm->context.vdso)) return "[vdso]"; + if (vma->vm_mm && (vma->vm_start == + (long)vma->vm_mm->context.vdso + PAGE_SIZE)) + return "[vdso_data]"; return NULL; } diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile index 4c8b2a4a6a70..38ba55b0eb9d 100644 --- a/arch/riscv/kernel/vdso/Makefile +++ b/arch/riscv/kernel/vdso/Makefile @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-only # Copied from arch/tile/kernel/vdso/Makefile +# Absolute relocation type $(ARCH_REL_TYPE_ABS) needs to be defined before +# the inclusion of generic Makefile. +ARCH_REL_TYPE_ABS := R_RISCV_32|R_RISCV_64|R_RISCV_JUMP_SLOT +include $(srctree)/lib/vdso/Makefile # Symbols present in the vdso vdso-syms = rt_sigreturn ifdef CONFIG_64BIT -vdso-syms += gettimeofday -vdso-syms += clock_gettime -vdso-syms += clock_getres +vdso-syms += vgettimeofday endif vdso-syms += getcpu vdso-syms += flush_icache @@ -14,6 +16,10 @@ vdso-syms += flush_icache # Files to link into the vdso obj-vdso = $(patsubst %, %.o, $(vdso-syms)) note.o +ifneq ($(c-gettimeofday-y),) + CFLAGS_vgettimeofday.o += -include $(c-gettimeofday-y) +endif + # Build rules targets := $(obj-vdso) vdso.so vdso.so.dbg vdso.lds vdso-dummy.o obj-vdso := $(addprefix $(obj)/, $(obj-vdso)) diff --git a/arch/riscv/kernel/vdso/clock_getres.S b/arch/riscv/kernel/vdso/clock_getres.S deleted file mode 100644 index 91378a52eb22..000000000000 --- a/arch/riscv/kernel/vdso/clock_getres.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_clock_getres(clockid_t clock_id, struct timespec *res); */ -ENTRY(__vdso_clock_getres) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_getres - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_getres) diff --git a/arch/riscv/kernel/vdso/clock_gettime.S b/arch/riscv/kernel/vdso/clock_gettime.S deleted file mode 100644 index 5371fd9bc01f..000000000000 --- a/arch/riscv/kernel/vdso/clock_gettime.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_clock_gettime(clockid_t clock_id, struct timespec *tp); */ -ENTRY(__vdso_clock_gettime) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_clock_gettime - ecall - ret - .cfi_endproc -ENDPROC(__vdso_clock_gettime) diff --git a/arch/riscv/kernel/vdso/gettimeofday.S b/arch/riscv/kernel/vdso/gettimeofday.S deleted file mode 100644 index e6fb8af88632..000000000000 --- a/arch/riscv/kernel/vdso/gettimeofday.S +++ /dev/null @@ -1,18 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * Copyright (C) 2017 SiFive - */ - -#include <linux/linkage.h> -#include <asm/unistd.h> - - .text -/* int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz); */ -ENTRY(__vdso_gettimeofday) - .cfi_startproc - /* For now, just do the syscall. */ - li a7, __NR_gettimeofday - ecall - ret - .cfi_endproc -ENDPROC(__vdso_gettimeofday) diff --git a/arch/riscv/kernel/vdso/vdso.lds.S b/arch/riscv/kernel/vdso/vdso.lds.S index f66a091cb890..e6f558bca71b 100644 --- a/arch/riscv/kernel/vdso/vdso.lds.S +++ b/arch/riscv/kernel/vdso/vdso.lds.S @@ -2,11 +2,13 @@ /* * Copyright (C) 2012 Regents of the University of California */ +#include <asm/page.h> OUTPUT_ARCH(riscv) SECTIONS { + PROVIDE(_vdso_data = . + PAGE_SIZE); . = SIZEOF_HEADERS; .hash : { *(.hash) } :text diff --git a/arch/riscv/kernel/vdso/vgettimeofday.c b/arch/riscv/kernel/vdso/vgettimeofday.c new file mode 100644 index 000000000000..d264943e2e47 --- /dev/null +++ b/arch/riscv/kernel/vdso/vgettimeofday.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from arch/arm64/kernel/vdso/vgettimeofday.c + * + * Copyright (C) 2018 ARM Ltd. + * Copyright (C) 2020 SiFive + */ + +#include <linux/time.h> +#include <linux/types.h> + +int __vdso_clock_gettime(clockid_t clock, struct __kernel_timespec *ts) +{ + return __cvdso_clock_gettime(clock, ts); +} + +int __vdso_gettimeofday(struct __kernel_old_timeval *tv, struct timezone *tz) +{ + return __cvdso_gettimeofday(tv, tz); +} + +int __vdso_clock_getres(clockid_t clock_id, struct __kernel_timespec *res) +{ + return __cvdso_clock_getres(clock_id, res); +} diff --git a/arch/riscv/mm/init.c b/arch/riscv/mm/init.c index 9996f49959bd..f4adb3684f3d 100644 --- a/arch/riscv/mm/init.c +++ b/arch/riscv/mm/init.c @@ -480,17 +480,6 @@ static void __init setup_vm_final(void) csr_write(CSR_SATP, PFN_DOWN(__pa_symbol(swapper_pg_dir)) | SATP_MODE); local_flush_tlb_all(); } - -void free_initmem(void) -{ - unsigned long init_begin = (unsigned long)__init_begin; - unsigned long init_end = (unsigned long)__init_end; - - /* Make the region as non-execuatble. */ - set_memory_nx(init_begin, (init_end - init_begin) >> PAGE_SHIFT); - free_initmem_default(POISON_FREE_INITMEM); -} - #else asmlinkage void __init setup_vm(uintptr_t dtb_pa) { diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h index 3d554887794e..cee3cb6455a2 100644 --- a/arch/s390/include/asm/kvm_host.h +++ b/arch/s390/include/asm/kvm_host.h @@ -978,7 +978,7 @@ bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu); void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); -void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c index d0ff26d157bc..d47c19718615 100644 --- a/arch/s390/kvm/kvm-s390.c +++ b/arch/s390/kvm/kvm-s390.c @@ -3923,11 +3923,13 @@ static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, } } -void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) { trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); + + return true; } void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 8ea987dd44f1..6a0cc524882d 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -181,7 +181,6 @@ config X86 select HAVE_HW_BREAKPOINT select HAVE_IDE select HAVE_IOREMAP_PROT - select HAVE_IRQ_EXIT_ON_IRQ_STACK if X86_64 select HAVE_IRQ_TIME_ACCOUNTING select HAVE_KERNEL_BZIP2 select HAVE_KERNEL_GZIP @@ -233,6 +232,7 @@ config X86 select THREAD_INFO_IN_TASK select USER_STACKTRACE_SUPPORT select VIRT_TO_BUS + select HAVE_ARCH_KCSAN if X86_64 select X86_FEATURE_NAMES if PROC_FS select PROC_PID_ARCH_STATUS if PROC_FS imply IMA_SECURE_AND_OR_TRUSTED_BOOT if EFI @@ -822,14 +822,6 @@ config PVH This option enables the PVH entry point for guest virtual machines as specified in the x86/HVM direct boot ABI. -config KVM_DEBUG_FS - bool "Enable debug information for KVM Guests in debugfs" - depends on KVM_GUEST && DEBUG_FS - help - This option enables collection of various statistics for KVM guest. - Statistics are displayed in debugfs filesystem. Enabling this option - may incur significant overhead. - config PARAVIRT_TIME_ACCOUNTING bool "Paravirtual steal time accounting" depends on PARAVIRT diff --git a/arch/x86/boot/Makefile b/arch/x86/boot/Makefile index 4c5355684321..fe605205b4ce 100644 --- a/arch/x86/boot/Makefile +++ b/arch/x86/boot/Makefile @@ -9,7 +9,9 @@ # Changed by many, many contributors over the years. # +# Sanitizer runtimes are unavailable and cannot be linked for early boot code. KASAN_SANITIZE := n +KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Kernel does not boot with kcov instrumentation here. diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile index 5f7c262bcc99..7619742f91c9 100644 --- a/arch/x86/boot/compressed/Makefile +++ b/arch/x86/boot/compressed/Makefile @@ -17,7 +17,9 @@ # (see scripts/Makefile.lib size_append) # compressed vmlinux.bin.all + u32 size of vmlinux.bin.all +# Sanitizer runtimes are unavailable and cannot be linked for early boot code. KASAN_SANITIZE := n +KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. diff --git a/arch/x86/entry/Makefile b/arch/x86/entry/Makefile index 85eb381259c2..b7a5790d8d63 100644 --- a/arch/x86/entry/Makefile +++ b/arch/x86/entry/Makefile @@ -3,7 +3,13 @@ # Makefile for the x86 low level entry code # -OBJECT_FILES_NON_STANDARD_entry_64_compat.o := y +KASAN_SANITIZE := n +UBSAN_SANITIZE := n +KCOV_INSTRUMENT := n + +CFLAGS_REMOVE_common.o = $(CC_FLAGS_FTRACE) -fstack-protector -fstack-protector-strong +CFLAGS_REMOVE_syscall_32.o = $(CC_FLAGS_FTRACE) -fstack-protector -fstack-protector-strong +CFLAGS_REMOVE_syscall_64.o = $(CC_FLAGS_FTRACE) -fstack-protector -fstack-protector-strong CFLAGS_syscall_64.o += $(call cc-option,-Wno-override-init,) CFLAGS_syscall_32.o += $(call cc-option,-Wno-override-init,) diff --git a/arch/x86/entry/calling.h b/arch/x86/entry/calling.h index 1c7f13bb6728..4208c1e3f601 100644 --- a/arch/x86/entry/calling.h +++ b/arch/x86/entry/calling.h @@ -341,30 +341,13 @@ For 32-bit we have the following conventions - kernel is built with #endif .endm -#endif /* CONFIG_X86_64 */ +#else /* CONFIG_X86_64 */ +# undef UNWIND_HINT_IRET_REGS +# define UNWIND_HINT_IRET_REGS +#endif /* !CONFIG_X86_64 */ .macro STACKLEAK_ERASE #ifdef CONFIG_GCC_PLUGIN_STACKLEAK call stackleak_erase #endif .endm - -/* - * This does 'call enter_from_user_mode' unless we can avoid it based on - * kernel config or using the static jump infrastructure. - */ -.macro CALL_enter_from_user_mode -#ifdef CONFIG_CONTEXT_TRACKING -#ifdef CONFIG_JUMP_LABEL - STATIC_JUMP_IF_FALSE .Lafter_call_\@, context_tracking_key, def=0 -#endif - call enter_from_user_mode -.Lafter_call_\@: -#endif -.endm - -#ifdef CONFIG_PARAVIRT_XXL -#define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg -#else -#define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg -#endif diff --git a/arch/x86/entry/common.c b/arch/x86/entry/common.c index 76735ec813e6..bd3f14175193 100644 --- a/arch/x86/entry/common.c +++ b/arch/x86/entry/common.c @@ -27,6 +27,11 @@ #include <linux/syscalls.h> #include <linux/uaccess.h> +#ifdef CONFIG_XEN_PV +#include <xen/xen-ops.h> +#include <xen/events.h> +#endif + #include <asm/desc.h> #include <asm/traps.h> #include <asm/vdso.h> @@ -35,21 +40,67 @@ #include <asm/nospec-branch.h> #include <asm/io_bitmap.h> #include <asm/syscall.h> +#include <asm/irq_stack.h> #define CREATE_TRACE_POINTS #include <trace/events/syscalls.h> #ifdef CONFIG_CONTEXT_TRACKING -/* Called on entry from user mode with IRQs off. */ -__visible inline void enter_from_user_mode(void) +/** + * enter_from_user_mode - Establish state when coming from user mode + * + * Syscall entry disables interrupts, but user mode is traced as interrupts + * enabled. Also with NO_HZ_FULL RCU might be idle. + * + * 1) Tell lockdep that interrupts are disabled + * 2) Invoke context tracking if enabled to reactivate RCU + * 3) Trace interrupts off state + */ +static noinstr void enter_from_user_mode(void) { - CT_WARN_ON(ct_state() != CONTEXT_USER); + enum ctx_state state = ct_state(); + + lockdep_hardirqs_off(CALLER_ADDR0); user_exit_irqoff(); + + instrumentation_begin(); + CT_WARN_ON(state != CONTEXT_USER); + trace_hardirqs_off_finish(); + instrumentation_end(); } #else -static inline void enter_from_user_mode(void) {} +static __always_inline void enter_from_user_mode(void) +{ + lockdep_hardirqs_off(CALLER_ADDR0); + instrumentation_begin(); + trace_hardirqs_off_finish(); + instrumentation_end(); +} #endif +/** + * exit_to_user_mode - Fixup state when exiting to user mode + * + * Syscall exit enables interrupts, but the kernel state is interrupts + * disabled when this is invoked. Also tell RCU about it. + * + * 1) Trace interrupts on state + * 2) Invoke context tracking if enabled to adjust RCU state + * 3) Clear CPU buffers if CPU is affected by MDS and the migitation is on. + * 4) Tell lockdep that interrupts are enabled + */ +static __always_inline void exit_to_user_mode(void) +{ + instrumentation_begin(); + trace_hardirqs_on_prepare(); + lockdep_hardirqs_on_prepare(CALLER_ADDR0); + instrumentation_end(); + + user_enter_irqoff(); + mds_user_clear_cpu_buffers(); + lockdep_hardirqs_on(CALLER_ADDR0); +} + static void do_audit_syscall_entry(struct pt_regs *regs, u32 arch) { #ifdef CONFIG_X86_64 @@ -179,8 +230,7 @@ static void exit_to_usermode_loop(struct pt_regs *regs, u32 cached_flags) } } -/* Called with IRQs disabled. */ -__visible inline void prepare_exit_to_usermode(struct pt_regs *regs) +static void __prepare_exit_to_usermode(struct pt_regs *regs) { struct thread_info *ti = current_thread_info(); u32 cached_flags; @@ -219,10 +269,14 @@ __visible inline void prepare_exit_to_usermode(struct pt_regs *regs) */ ti->status &= ~(TS_COMPAT|TS_I386_REGS_POKED); #endif +} - user_enter_irqoff(); - - mds_user_clear_cpu_buffers(); +__visible noinstr void prepare_exit_to_usermode(struct pt_regs *regs) +{ + instrumentation_begin(); + __prepare_exit_to_usermode(regs); + instrumentation_end(); + exit_to_user_mode(); } #define SYSCALL_EXIT_WORK_FLAGS \ @@ -251,11 +305,7 @@ static void syscall_slow_exit_work(struct pt_regs *regs, u32 cached_flags) tracehook_report_syscall_exit(regs, step); } -/* - * Called with IRQs on and fully valid regs. Returns with IRQs off in a - * state such that we can immediately switch to user mode. - */ -__visible inline void syscall_return_slowpath(struct pt_regs *regs) +static void __syscall_return_slowpath(struct pt_regs *regs) { struct thread_info *ti = current_thread_info(); u32 cached_flags = READ_ONCE(ti->flags); @@ -276,15 +326,29 @@ __visible inline void syscall_return_slowpath(struct pt_regs *regs) syscall_slow_exit_work(regs, cached_flags); local_irq_disable(); - prepare_exit_to_usermode(regs); + __prepare_exit_to_usermode(regs); +} + +/* + * Called with IRQs on and fully valid regs. Returns with IRQs off in a + * state such that we can immediately switch to user mode. + */ +__visible noinstr void syscall_return_slowpath(struct pt_regs *regs) +{ + instrumentation_begin(); + __syscall_return_slowpath(regs); + instrumentation_end(); + exit_to_user_mode(); } #ifdef CONFIG_X86_64 -__visible void do_syscall_64(unsigned long nr, struct pt_regs *regs) +__visible noinstr void do_syscall_64(unsigned long nr, struct pt_regs *regs) { struct thread_info *ti; enter_from_user_mode(); + instrumentation_begin(); + local_irq_enable(); ti = current_thread_info(); if (READ_ONCE(ti->flags) & _TIF_WORK_SYSCALL_ENTRY) @@ -301,8 +365,10 @@ __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs) regs->ax = x32_sys_call_table[nr](regs); #endif } + __syscall_return_slowpath(regs); - syscall_return_slowpath(regs); + instrumentation_end(); + exit_to_user_mode(); } #endif @@ -313,7 +379,7 @@ __visible void do_syscall_64(unsigned long nr, struct pt_regs *regs) * extremely hot in workloads that use it, and it's usually called from * do_fast_syscall_32, so forcibly inline it to improve performance. */ -static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs) +static void do_syscall_32_irqs_on(struct pt_regs *regs) { struct thread_info *ti = current_thread_info(); unsigned int nr = (unsigned int)regs->orig_ax; @@ -337,27 +403,62 @@ static __always_inline void do_syscall_32_irqs_on(struct pt_regs *regs) regs->ax = ia32_sys_call_table[nr](regs); } - syscall_return_slowpath(regs); + __syscall_return_slowpath(regs); } /* Handles int $0x80 */ -__visible void do_int80_syscall_32(struct pt_regs *regs) +__visible noinstr void do_int80_syscall_32(struct pt_regs *regs) { enter_from_user_mode(); + instrumentation_begin(); + local_irq_enable(); do_syscall_32_irqs_on(regs); + + instrumentation_end(); + exit_to_user_mode(); +} + +static bool __do_fast_syscall_32(struct pt_regs *regs) +{ + int res; + + /* Fetch EBP from where the vDSO stashed it. */ + if (IS_ENABLED(CONFIG_X86_64)) { + /* + * Micro-optimization: the pointer we're following is + * explicitly 32 bits, so it can't be out of range. + */ + res = __get_user(*(u32 *)®s->bp, + (u32 __user __force *)(unsigned long)(u32)regs->sp); + } else { + res = get_user(*(u32 *)®s->bp, + (u32 __user __force *)(unsigned long)(u32)regs->sp); + } + + if (res) { + /* User code screwed up. */ + regs->ax = -EFAULT; + local_irq_disable(); + __prepare_exit_to_usermode(regs); + return false; + } + + /* Now this is just like a normal syscall. */ + do_syscall_32_irqs_on(regs); + return true; } /* Returns 0 to return using IRET or 1 to return using SYSEXIT/SYSRETL. */ -__visible long do_fast_syscall_32(struct pt_regs *regs) +__visible noinstr long do_fast_syscall_32(struct pt_regs *regs) { /* * Called using the internal vDSO SYSENTER/SYSCALL32 calling * convention. Adjust regs so it looks like we entered using int80. */ - unsigned long landing_pad = (unsigned long)current->mm->context.vdso + - vdso_image_32.sym_int80_landing_pad; + vdso_image_32.sym_int80_landing_pad; + bool success; /* * SYSENTER loses EIP, and even SYSCALL32 needs us to skip forward @@ -367,33 +468,17 @@ __visible long do_fast_syscall_32(struct pt_regs *regs) regs->ip = landing_pad; enter_from_user_mode(); + instrumentation_begin(); local_irq_enable(); + success = __do_fast_syscall_32(regs); - /* Fetch EBP from where the vDSO stashed it. */ - if ( -#ifdef CONFIG_X86_64 - /* - * Micro-optimization: the pointer we're following is explicitly - * 32 bits, so it can't be out of range. - */ - __get_user(*(u32 *)®s->bp, - (u32 __user __force *)(unsigned long)(u32)regs->sp) -#else - get_user(*(u32 *)®s->bp, - (u32 __user __force *)(unsigned long)(u32)regs->sp) -#endif - ) { - - /* User code screwed up. */ - local_irq_disable(); - regs->ax = -EFAULT; - prepare_exit_to_usermode(regs); - return 0; /* Keep it simple: use IRET. */ - } + instrumentation_end(); + exit_to_user_mode(); - /* Now this is just like a normal syscall. */ - do_syscall_32_irqs_on(regs); + /* If it failed, keep it simple: use IRET. */ + if (!success) + return 0; #ifdef CONFIG_X86_64 /* @@ -431,3 +516,266 @@ SYSCALL_DEFINE0(ni_syscall) { return -ENOSYS; } + +/** + * idtentry_enter_cond_rcu - Handle state tracking on idtentry with conditional + * RCU handling + * @regs: Pointer to pt_regs of interrupted context + * + * Invokes: + * - lockdep irqflag state tracking as low level ASM entry disabled + * interrupts. + * + * - Context tracking if the exception hit user mode. + * + * - The hardirq tracer to keep the state consistent as low level ASM + * entry disabled interrupts. + * + * For kernel mode entries RCU handling is done conditional. If RCU is + * watching then the only RCU requirement is to check whether the tick has + * to be restarted. If RCU is not watching then rcu_irq_enter() has to be + * invoked on entry and rcu_irq_exit() on exit. + * + * Avoiding the rcu_irq_enter/exit() calls is an optimization but also + * solves the problem of kernel mode pagefaults which can schedule, which + * is not possible after invoking rcu_irq_enter() without undoing it. + * + * For user mode entries enter_from_user_mode() must be invoked to + * establish the proper context for NOHZ_FULL. Otherwise scheduling on exit + * would not be possible. + * + * Returns: True if RCU has been adjusted on a kernel entry + * False otherwise + * + * The return value must be fed into the rcu_exit argument of + * idtentry_exit_cond_rcu(). + */ +bool noinstr idtentry_enter_cond_rcu(struct pt_regs *regs) +{ + if (user_mode(regs)) { + enter_from_user_mode(); + return false; + } + + /* + * If this entry hit the idle task invoke rcu_irq_enter() whether + * RCU is watching or not. + * + * Interupts can nest when the first interrupt invokes softirq + * processing on return which enables interrupts. + * + * Scheduler ticks in the idle task can mark quiescent state and + * terminate a grace period, if and only if the timer interrupt is + * not nested into another interrupt. + * + * Checking for __rcu_is_watching() here would prevent the nesting + * interrupt to invoke rcu_irq_enter(). If that nested interrupt is + * the tick then rcu_flavor_sched_clock_irq() would wrongfully + * assume that it is the first interupt and eventually claim + * quiescient state and end grace periods prematurely. + * + * Unconditionally invoke rcu_irq_enter() so RCU state stays + * consistent. + * + * TINY_RCU does not support EQS, so let the compiler eliminate + * this part when enabled. + */ + if (!IS_ENABLED(CONFIG_TINY_RCU) && is_idle_task(current)) { + /* + * If RCU is not watching then the same careful + * sequence vs. lockdep and tracing is required + * as in enter_from_user_mode(). + */ + lockdep_hardirqs_off(CALLER_ADDR0); + rcu_irq_enter(); + instrumentation_begin(); + trace_hardirqs_off_finish(); + instrumentation_end(); + + return true; + } + + /* + * If RCU is watching then RCU only wants to check whether it needs + * to restart the tick in NOHZ mode. rcu_irq_enter_check_tick() + * already contains a warning when RCU is not watching, so no point + * in having another one here. + */ + instrumentation_begin(); + rcu_irq_enter_check_tick(); + /* Use the combo lockdep/tracing function */ + trace_hardirqs_off(); + instrumentation_end(); + + return false; +} + +static void idtentry_exit_cond_resched(struct pt_regs *regs, bool may_sched) +{ + if (may_sched && !preempt_count()) { + /* Sanity check RCU and thread stack */ + rcu_irq_exit_check_preempt(); + if (IS_ENABLED(CONFIG_DEBUG_ENTRY)) + WARN_ON_ONCE(!on_thread_stack()); + if (need_resched()) + preempt_schedule_irq(); + } + /* Covers both tracing and lockdep */ + trace_hardirqs_on(); +} + +/** + * idtentry_exit_cond_rcu - Handle return from exception with conditional RCU + * handling + * @regs: Pointer to pt_regs (exception entry regs) + * @rcu_exit: Invoke rcu_irq_exit() if true + * + * Depending on the return target (kernel/user) this runs the necessary + * preemption and work checks if possible and reguired and returns to + * the caller with interrupts disabled and no further work pending. + * + * This is the last action before returning to the low level ASM code which + * just needs to return to the appropriate context. + * + * Counterpart to idtentry_enter_cond_rcu(). The return value of the entry + * function must be fed into the @rcu_exit argument. + */ +void noinstr idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit) +{ + lockdep_assert_irqs_disabled(); + + /* Check whether this returns to user mode */ + if (user_mode(regs)) { + prepare_exit_to_usermode(regs); + } else if (regs->flags & X86_EFLAGS_IF) { + /* + * If RCU was not watching on entry this needs to be done + * carefully and needs the same ordering of lockdep/tracing + * and RCU as the return to user mode path. + */ + if (rcu_exit) { + instrumentation_begin(); + /* Tell the tracer that IRET will enable interrupts */ + trace_hardirqs_on_prepare(); + lockdep_hardirqs_on_prepare(CALLER_ADDR0); + instrumentation_end(); + rcu_irq_exit(); + lockdep_hardirqs_on(CALLER_ADDR0); + return; + } + + instrumentation_begin(); + idtentry_exit_cond_resched(regs, IS_ENABLED(CONFIG_PREEMPTION)); + instrumentation_end(); + } else { + /* + * IRQ flags state is correct already. Just tell RCU if it + * was not watching on entry. + */ + if (rcu_exit) + rcu_irq_exit(); + } +} + +/** + * idtentry_enter_user - Handle state tracking on idtentry from user mode + * @regs: Pointer to pt_regs of interrupted context + * + * Invokes enter_from_user_mode() to establish the proper context for + * NOHZ_FULL. Otherwise scheduling on exit would not be possible. + */ +void noinstr idtentry_enter_user(struct pt_regs *regs) +{ + enter_from_user_mode(); +} + +/** + * idtentry_exit_user - Handle return from exception to user mode + * @regs: Pointer to pt_regs (exception entry regs) + * + * Runs the necessary preemption and work checks and returns to the caller + * with interrupts disabled and no further work pending. + * + * This is the last action before returning to the low level ASM code which + * just needs to return to the appropriate context. + * + * Counterpart to idtentry_enter_user(). + */ +void noinstr idtentry_exit_user(struct pt_regs *regs) +{ + lockdep_assert_irqs_disabled(); + + prepare_exit_to_usermode(regs); +} + +#ifdef CONFIG_XEN_PV +#ifndef CONFIG_PREEMPTION +/* + * Some hypercalls issued by the toolstack can take many 10s of + * seconds. Allow tasks running hypercalls via the privcmd driver to + * be voluntarily preempted even if full kernel preemption is + * disabled. + * + * Such preemptible hypercalls are bracketed by + * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end() + * calls. + */ +DEFINE_PER_CPU(bool, xen_in_preemptible_hcall); +EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall); + +/* + * In case of scheduling the flag must be cleared and restored after + * returning from schedule as the task might move to a different CPU. + */ +static __always_inline bool get_and_clear_inhcall(void) +{ + bool inhcall = __this_cpu_read(xen_in_preemptible_hcall); + + __this_cpu_write(xen_in_preemptible_hcall, false); + return inhcall; +} + +static __always_inline void restore_inhcall(bool inhcall) +{ + __this_cpu_write(xen_in_preemptible_hcall, inhcall); +} +#else +static __always_inline bool get_and_clear_inhcall(void) { return false; } +static __always_inline void restore_inhcall(bool inhcall) { } +#endif + +static void __xen_pv_evtchn_do_upcall(void) +{ + irq_enter_rcu(); + inc_irq_stat(irq_hv_callback_count); + + xen_hvm_evtchn_do_upcall(); + + irq_exit_rcu(); +} + +__visible noinstr void xen_pv_evtchn_do_upcall(struct pt_regs *regs) +{ + struct pt_regs *old_regs; + bool inhcall, rcu_exit; + + rcu_exit = idtentry_enter_cond_rcu(regs); + old_regs = set_irq_regs(regs); + + instrumentation_begin(); + run_on_irqstack_cond(__xen_pv_evtchn_do_upcall, NULL, regs); + instrumentation_begin(); + + set_irq_regs(old_regs); + + inhcall = get_and_clear_inhcall(); + if (inhcall && !WARN_ON_ONCE(rcu_exit)) { + instrumentation_begin(); + idtentry_exit_cond_resched(regs, true); + instrumentation_end(); + restore_inhcall(inhcall); + } else { + idtentry_exit_cond_rcu(regs, rcu_exit); + } +} +#endif /* CONFIG_XEN_PV */ diff --git a/arch/x86/entry/entry_32.S b/arch/x86/entry/entry_32.S index a5eed844e948..024d7d276cd4 100644 --- a/arch/x86/entry/entry_32.S +++ b/arch/x86/entry/entry_32.S @@ -44,40 +44,13 @@ #include <asm/asm.h> #include <asm/smap.h> #include <asm/frame.h> +#include <asm/trapnr.h> #include <asm/nospec-branch.h> #include "calling.h" .section .entry.text, "ax" -/* - * We use macros for low-level operations which need to be overridden - * for paravirtualization. The following will never clobber any registers: - * INTERRUPT_RETURN (aka. "iret") - * GET_CR0_INTO_EAX (aka. "movl %cr0, %eax") - * ENABLE_INTERRUPTS_SYSEXIT (aka "sti; sysexit"). - * - * For DISABLE_INTERRUPTS/ENABLE_INTERRUPTS (aka "cli"/"sti"), you must - * specify what registers can be overwritten (CLBR_NONE, CLBR_EAX/EDX/ECX/ANY). - * Allowing a register to be clobbered can shrink the paravirt replacement - * enough to patch inline, increasing performance. - */ - -#ifdef CONFIG_PREEMPTION -# define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF -#else -# define preempt_stop(clobbers) -#endif - -.macro TRACE_IRQS_IRET -#ifdef CONFIG_TRACE_IRQFLAGS - testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off? - jz 1f - TRACE_IRQS_ON -1: -#endif -.endm - #define PTI_SWITCH_MASK (1 << PAGE_SHIFT) /* @@ -726,10 +699,68 @@ .Lend_\@: .endm + +/** + * idtentry - Macro to generate entry stubs for simple IDT entries + * @vector: Vector number + * @asmsym: ASM symbol for the entry point + * @cfunc: C function to be called + * @has_error_code: Hardware pushed error code on stack + */ +.macro idtentry vector asmsym cfunc has_error_code:req +SYM_CODE_START(\asmsym) + ASM_CLAC + cld + + .if \has_error_code == 0 + pushl $0 /* Clear the error code */ + .endif + + /* Push the C-function address into the GS slot */ + pushl $\cfunc + /* Invoke the common exception entry */ + jmp handle_exception +SYM_CODE_END(\asmsym) +.endm + +.macro idtentry_irq vector cfunc + .p2align CONFIG_X86_L1_CACHE_SHIFT +SYM_CODE_START_LOCAL(asm_\cfunc) + ASM_CLAC + SAVE_ALL switch_stacks=1 + ENCODE_FRAME_POINTER + movl %esp, %eax + movl PT_ORIG_EAX(%esp), %edx /* get the vector from stack */ + movl $-1, PT_ORIG_EAX(%esp) /* no syscall to restart */ + call \cfunc + jmp handle_exception_return +SYM_CODE_END(asm_\cfunc) +.endm + +.macro idtentry_sysvec vector cfunc + idtentry \vector asm_\cfunc \cfunc has_error_code=0 +.endm + +/* + * Include the defines which emit the idt entries which are shared + * shared between 32 and 64 bit and emit the __irqentry_text_* markers + * so the stacktrace boundary checks work. + */ + .align 16 + .globl __irqentry_text_start +__irqentry_text_start: + +#include <asm/idtentry.h> + + .align 16 + .globl __irqentry_text_end +__irqentry_text_end: + /* * %eax: prev task * %edx: next task */ +.pushsection .text, "ax" SYM_CODE_START(__switch_to_asm) /* * Save callee-saved registers @@ -776,6 +807,7 @@ SYM_CODE_START(__switch_to_asm) jmp __switch_to SYM_CODE_END(__switch_to_asm) +.popsection /* * The unwinder expects the last frame on the stack to always be at the same @@ -784,6 +816,7 @@ SYM_CODE_END(__switch_to_asm) * asmlinkage function so its argument has to be pushed on the stack. This * wrapper creates a proper "end of stack" frame header before the call. */ +.pushsection .text, "ax" SYM_FUNC_START(schedule_tail_wrapper) FRAME_BEGIN @@ -794,6 +827,8 @@ SYM_FUNC_START(schedule_tail_wrapper) FRAME_END ret SYM_FUNC_END(schedule_tail_wrapper) +.popsection + /* * A newly forked process directly context switches into this address. * @@ -801,6 +836,7 @@ SYM_FUNC_END(schedule_tail_wrapper) * ebx: kernel thread func (NULL for user thread) * edi: kernel thread arg */ +.pushsection .text, "ax" SYM_CODE_START(ret_from_fork) call schedule_tail_wrapper @@ -811,8 +847,7 @@ SYM_CODE_START(ret_from_fork) /* When we fork, we trace the syscall return in the child, too. */ movl %esp, %eax call syscall_return_slowpath - STACKLEAK_ERASE - jmp restore_all + jmp .Lsyscall_32_done /* kernel thread */ 1: movl %edi, %eax @@ -825,38 +860,7 @@ SYM_CODE_START(ret_from_fork) movl $0, PT_EAX(%esp) jmp 2b SYM_CODE_END(ret_from_fork) - -/* - * Return to user mode is not as complex as all this looks, - * but we want the default path for a system call return to - * go as quickly as possible which is why some of this is - * less clear than it otherwise should be. - */ - - # userspace resumption stub bypassing syscall exit tracing -SYM_CODE_START_LOCAL(ret_from_exception) - preempt_stop(CLBR_ANY) -ret_from_intr: -#ifdef CONFIG_VM86 - movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS - movb PT_CS(%esp), %al - andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax -#else - /* - * We can be coming here from child spawned by kernel_thread(). - */ - movl PT_CS(%esp), %eax - andl $SEGMENT_RPL_MASK, %eax -#endif - cmpl $USER_RPL, %eax - jb restore_all_kernel # not returning to v8086 or userspace - - DISABLE_INTERRUPTS(CLBR_ANY) - TRACE_IRQS_OFF - movl %esp, %eax - call prepare_exit_to_usermode - jmp restore_all -SYM_CODE_END(ret_from_exception) +.popsection SYM_ENTRY(__begin_SYSENTER_singlestep_region, SYM_L_GLOBAL, SYM_A_NONE) /* @@ -960,12 +964,6 @@ SYM_FUNC_START(entry_SYSENTER_32) jnz .Lsysenter_fix_flags .Lsysenter_flags_fixed: - /* - * User mode is traced as though IRQs are on, and SYSENTER - * turned them off. - */ - TRACE_IRQS_OFF - movl %esp, %eax call do_fast_syscall_32 /* XEN PV guests always use IRET path */ @@ -974,8 +972,7 @@ SYM_FUNC_START(entry_SYSENTER_32) STACKLEAK_ERASE -/* Opportunistic SYSEXIT */ - TRACE_IRQS_ON /* User mode traces as IRQs on. */ + /* Opportunistic SYSEXIT */ /* * Setup entry stack - we keep the pointer in %eax and do the @@ -1075,20 +1072,12 @@ SYM_FUNC_START(entry_INT80_32) SAVE_ALL pt_regs_ax=$-ENOSYS switch_stacks=1 /* save rest */ - /* - * User mode is traced as though IRQs are on, and the interrupt gate - * turned them off. - */ - TRACE_IRQS_OFF - movl %esp, %eax call do_int80_syscall_32 .Lsyscall_32_done: - STACKLEAK_ERASE -restore_all: - TRACE_IRQS_ON +restore_all_switch_stack: SWITCH_TO_ENTRY_STACK CHECK_AND_APPLY_ESPFIX @@ -1107,26 +1096,10 @@ restore_all: */ INTERRUPT_RETURN -restore_all_kernel: -#ifdef CONFIG_PREEMPTION - DISABLE_INTERRUPTS(CLBR_ANY) - cmpl $0, PER_CPU_VAR(__preempt_count) - jnz .Lno_preempt - testl $X86_EFLAGS_IF, PT_EFLAGS(%esp) # interrupts off (exception path) ? - jz .Lno_preempt - call preempt_schedule_irq -.Lno_preempt: -#endif - TRACE_IRQS_IRET - PARANOID_EXIT_TO_KERNEL_MODE - BUG_IF_WRONG_CR3 - RESTORE_REGS 4 - jmp .Lirq_return - .section .fixup, "ax" -SYM_CODE_START(iret_exc) +SYM_CODE_START(asm_iret_error) pushl $0 # no error code - pushl $do_iret_error + pushl $iret_error #ifdef CONFIG_DEBUG_ENTRY /* @@ -1140,10 +1113,10 @@ SYM_CODE_START(iret_exc) popl %eax #endif - jmp common_exception -SYM_CODE_END(iret_exc) + jmp handle_exception +SYM_CODE_END(asm_iret_error) .previous - _ASM_EXTABLE(.Lirq_return, iret_exc) + _ASM_EXTABLE(.Lirq_return, asm_iret_error) SYM_FUNC_END(entry_INT80_32) .macro FIXUP_ESPFIX_STACK @@ -1193,192 +1166,21 @@ SYM_FUNC_END(entry_INT80_32) #endif .endm -/* - * Build the entry stubs with some assembler magic. - * We pack 1 stub into every 8-byte block. - */ - .align 8 -SYM_CODE_START(irq_entries_start) - vector=FIRST_EXTERNAL_VECTOR - .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) - pushl $(~vector+0x80) /* Note: always in signed byte range */ - vector=vector+1 - jmp common_interrupt - .align 8 - .endr -SYM_CODE_END(irq_entries_start) - -#ifdef CONFIG_X86_LOCAL_APIC - .align 8 -SYM_CODE_START(spurious_entries_start) - vector=FIRST_SYSTEM_VECTOR - .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR) - pushl $(~vector+0x80) /* Note: always in signed byte range */ - vector=vector+1 - jmp common_spurious - .align 8 - .endr -SYM_CODE_END(spurious_entries_start) - -SYM_CODE_START_LOCAL(common_spurious) - ASM_CLAC - addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ - SAVE_ALL switch_stacks=1 - ENCODE_FRAME_POINTER - TRACE_IRQS_OFF - movl %esp, %eax - call smp_spurious_interrupt - jmp ret_from_intr -SYM_CODE_END(common_spurious) -#endif - -/* - * the CPU automatically disables interrupts when executing an IRQ vector, - * so IRQ-flags tracing has to follow that: - */ - .p2align CONFIG_X86_L1_CACHE_SHIFT -SYM_CODE_START_LOCAL(common_interrupt) - ASM_CLAC - addl $-0x80, (%esp) /* Adjust vector into the [-256, -1] range */ - - SAVE_ALL switch_stacks=1 - ENCODE_FRAME_POINTER - TRACE_IRQS_OFF - movl %esp, %eax - call do_IRQ - jmp ret_from_intr -SYM_CODE_END(common_interrupt) - -#define BUILD_INTERRUPT3(name, nr, fn) \ -SYM_FUNC_START(name) \ - ASM_CLAC; \ - pushl $~(nr); \ - SAVE_ALL switch_stacks=1; \ - ENCODE_FRAME_POINTER; \ - TRACE_IRQS_OFF \ - movl %esp, %eax; \ - call fn; \ - jmp ret_from_intr; \ -SYM_FUNC_END(name) - -#define BUILD_INTERRUPT(name, nr) \ - BUILD_INTERRUPT3(name, nr, smp_##name); \ - -/* The include is where all of the SMP etc. interrupts come from */ -#include <asm/entry_arch.h> - -SYM_CODE_START(coprocessor_error) - ASM_CLAC - pushl $0 - pushl $do_coprocessor_error - jmp common_exception -SYM_CODE_END(coprocessor_error) - -SYM_CODE_START(simd_coprocessor_error) - ASM_CLAC - pushl $0 -#ifdef CONFIG_X86_INVD_BUG - /* AMD 486 bug: invd from userspace calls exception 19 instead of #GP */ - ALTERNATIVE "pushl $do_general_protection", \ - "pushl $do_simd_coprocessor_error", \ - X86_FEATURE_XMM -#else - pushl $do_simd_coprocessor_error -#endif - jmp common_exception -SYM_CODE_END(simd_coprocessor_error) - -SYM_CODE_START(device_not_available) - ASM_CLAC - pushl $0 - pushl $do_device_not_available - jmp common_exception -SYM_CODE_END(device_not_available) - #ifdef CONFIG_PARAVIRT SYM_CODE_START(native_iret) iret - _ASM_EXTABLE(native_iret, iret_exc) + _ASM_EXTABLE(native_iret, asm_iret_error) SYM_CODE_END(native_iret) #endif -SYM_CODE_START(overflow) - ASM_CLAC - pushl $0 - pushl $do_overflow - jmp common_exception -SYM_CODE_END(overflow) - -SYM_CODE_START(bounds) - ASM_CLAC - pushl $0 - pushl $do_bounds - jmp common_exception -SYM_CODE_END(bounds) - -SYM_CODE_START(invalid_op) - ASM_CLAC - pushl $0 - pushl $do_invalid_op - jmp common_exception -SYM_CODE_END(invalid_op) - -SYM_CODE_START(coprocessor_segment_overrun) - ASM_CLAC - pushl $0 - pushl $do_coprocessor_segment_overrun - jmp common_exception -SYM_CODE_END(coprocessor_segment_overrun) - -SYM_CODE_START(invalid_TSS) - ASM_CLAC - pushl $do_invalid_TSS - jmp common_exception -SYM_CODE_END(invalid_TSS) - -SYM_CODE_START(segment_not_present) - ASM_CLAC - pushl $do_segment_not_present - jmp common_exception -SYM_CODE_END(segment_not_present) - -SYM_CODE_START(stack_segment) - ASM_CLAC - pushl $do_stack_segment - jmp common_exception -SYM_CODE_END(stack_segment) - -SYM_CODE_START(alignment_check) - ASM_CLAC - pushl $do_alignment_check - jmp common_exception -SYM_CODE_END(alignment_check) - -SYM_CODE_START(divide_error) - ASM_CLAC - pushl $0 # no error code - pushl $do_divide_error - jmp common_exception -SYM_CODE_END(divide_error) - -#ifdef CONFIG_X86_MCE -SYM_CODE_START(machine_check) - ASM_CLAC - pushl $0 - pushl $do_mce - jmp common_exception -SYM_CODE_END(machine_check) -#endif - -SYM_CODE_START(spurious_interrupt_bug) - ASM_CLAC - pushl $0 - pushl $do_spurious_interrupt_bug - jmp common_exception -SYM_CODE_END(spurious_interrupt_bug) - #ifdef CONFIG_XEN_PV -SYM_FUNC_START(xen_hypervisor_callback) +/* + * See comment in entry_64.S for further explanation + * + * Note: This is not an actual IDT entry point. It's a XEN specific entry + * point and therefore named to match the 64-bit trampoline counterpart. + */ +SYM_FUNC_START(xen_asm_exc_xen_hypervisor_callback) /* * Check to see if we got the event in the critical * region in xen_iret_direct, after we've reenabled @@ -1395,14 +1197,11 @@ SYM_FUNC_START(xen_hypervisor_callback) pushl $-1 /* orig_ax = -1 => not a system call */ SAVE_ALL ENCODE_FRAME_POINTER - TRACE_IRQS_OFF + mov %esp, %eax - call xen_evtchn_do_upcall -#ifndef CONFIG_PREEMPTION - call xen_maybe_preempt_hcall -#endif - jmp ret_from_intr -SYM_FUNC_END(xen_hypervisor_callback) + call xen_pv_evtchn_do_upcall + jmp handle_exception_return +SYM_FUNC_END(xen_asm_exc_xen_hypervisor_callback) /* * Hypervisor uses this for application faults while it executes. @@ -1429,11 +1228,11 @@ SYM_FUNC_START(xen_failsafe_callback) popl %eax lea 16(%esp), %esp jz 5f - jmp iret_exc + jmp asm_iret_error 5: pushl $-1 /* orig_ax = -1 => not a system call */ SAVE_ALL ENCODE_FRAME_POINTER - jmp ret_from_exception + jmp handle_exception_return .section .fixup, "ax" 6: xorl %eax, %eax @@ -1456,56 +1255,7 @@ SYM_FUNC_START(xen_failsafe_callback) SYM_FUNC_END(xen_failsafe_callback) #endif /* CONFIG_XEN_PV */ -#ifdef CONFIG_XEN_PVHVM -BUILD_INTERRUPT3(xen_hvm_callback_vector, HYPERVISOR_CALLBACK_VECTOR, - xen_evtchn_do_upcall) -#endif - - -#if IS_ENABLED(CONFIG_HYPERV) - -BUILD_INTERRUPT3(hyperv_callback_vector, HYPERVISOR_CALLBACK_VECTOR, - hyperv_vector_handler) - -BUILD_INTERRUPT3(hyperv_reenlightenment_vector, HYPERV_REENLIGHTENMENT_VECTOR, - hyperv_reenlightenment_intr) - -BUILD_INTERRUPT3(hv_stimer0_callback_vector, HYPERV_STIMER0_VECTOR, - hv_stimer0_vector_handler) - -#endif /* CONFIG_HYPERV */ - -SYM_CODE_START(page_fault) - ASM_CLAC - pushl $do_page_fault - jmp common_exception_read_cr2 -SYM_CODE_END(page_fault) - -SYM_CODE_START_LOCAL_NOALIGN(common_exception_read_cr2) - /* the function address is in %gs's slot on the stack */ - SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1 - - ENCODE_FRAME_POINTER - - /* fixup %gs */ - GS_TO_REG %ecx - movl PT_GS(%esp), %edi - REG_TO_PTGS %ecx - SET_KERNEL_GS %ecx - - GET_CR2_INTO(%ecx) # might clobber %eax - - /* fixup orig %eax */ - movl PT_ORIG_EAX(%esp), %edx # get the error code - movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart - - TRACE_IRQS_OFF - movl %esp, %eax # pt_regs pointer - CALL_NOSPEC edi - jmp ret_from_exception -SYM_CODE_END(common_exception_read_cr2) - -SYM_CODE_START_LOCAL_NOALIGN(common_exception) +SYM_CODE_START_LOCAL_NOALIGN(handle_exception) /* the function address is in %gs's slot on the stack */ SAVE_ALL switch_stacks=1 skip_gs=1 unwind_espfix=1 ENCODE_FRAME_POINTER @@ -1520,23 +1270,35 @@ SYM_CODE_START_LOCAL_NOALIGN(common_exception) movl PT_ORIG_EAX(%esp), %edx # get the error code movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart - TRACE_IRQS_OFF movl %esp, %eax # pt_regs pointer CALL_NOSPEC edi - jmp ret_from_exception -SYM_CODE_END(common_exception) -SYM_CODE_START(debug) +handle_exception_return: +#ifdef CONFIG_VM86 + movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS + movb PT_CS(%esp), %al + andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax +#else /* - * Entry from sysenter is now handled in common_exception + * We can be coming here from child spawned by kernel_thread(). */ - ASM_CLAC - pushl $0 - pushl $do_debug - jmp common_exception -SYM_CODE_END(debug) + movl PT_CS(%esp), %eax + andl $SEGMENT_RPL_MASK, %eax +#endif + cmpl $USER_RPL, %eax # returning to v8086 or userspace ? + jnb ret_to_user -SYM_CODE_START(double_fault) + PARANOID_EXIT_TO_KERNEL_MODE + BUG_IF_WRONG_CR3 + RESTORE_REGS 4 + jmp .Lirq_return + +ret_to_user: + movl %esp, %eax + jmp restore_all_switch_stack +SYM_CODE_END(handle_exception) + +SYM_CODE_START(asm_exc_double_fault) 1: /* * This is a task gate handler, not an interrupt gate handler. @@ -1574,7 +1336,7 @@ SYM_CODE_START(double_fault) 1: hlt jmp 1b -SYM_CODE_END(double_fault) +SYM_CODE_END(asm_exc_double_fault) /* * NMI is doubly nasty. It can happen on the first instruction of @@ -1583,7 +1345,7 @@ SYM_CODE_END(double_fault) * switched stacks. We handle both conditions by simply checking whether we * interrupted kernel code running on the SYSENTER stack. */ -SYM_CODE_START(nmi) +SYM_CODE_START(asm_exc_nmi) ASM_CLAC #ifdef CONFIG_X86_ESPFIX32 @@ -1612,7 +1374,7 @@ SYM_CODE_START(nmi) jb .Lnmi_from_sysenter_stack /* Not on SYSENTER stack. */ - call do_nmi + call exc_nmi jmp .Lnmi_return .Lnmi_from_sysenter_stack: @@ -1622,7 +1384,7 @@ SYM_CODE_START(nmi) */ movl %esp, %ebx movl PER_CPU_VAR(cpu_current_top_of_stack), %esp - call do_nmi + call exc_nmi movl %ebx, %esp .Lnmi_return: @@ -1676,21 +1438,9 @@ SYM_CODE_START(nmi) lss (1+5+6)*4(%esp), %esp # back to espfix stack jmp .Lirq_return #endif -SYM_CODE_END(nmi) - -SYM_CODE_START(int3) - ASM_CLAC - pushl $0 - pushl $do_int3 - jmp common_exception -SYM_CODE_END(int3) - -SYM_CODE_START(general_protection) - ASM_CLAC - pushl $do_general_protection - jmp common_exception -SYM_CODE_END(general_protection) +SYM_CODE_END(asm_exc_nmi) +.pushsection .text, "ax" SYM_CODE_START(rewind_stack_do_exit) /* Prevent any naive code from trying to unwind to our caller. */ xorl %ebp, %ebp @@ -1701,3 +1451,4 @@ SYM_CODE_START(rewind_stack_do_exit) call do_exit 1: jmp 1b SYM_CODE_END(rewind_stack_do_exit) +.popsection diff --git a/arch/x86/entry/entry_64.S b/arch/x86/entry/entry_64.S index eead1e2bebd5..d2a00c97e53f 100644 --- a/arch/x86/entry/entry_64.S +++ b/arch/x86/entry/entry_64.S @@ -16,7 +16,6 @@ * * Some macro usage: * - SYM_FUNC_START/END:Define functions in the symbol table. - * - TRACE_IRQ_*: Trace hardirq state for lock debugging. * - idtentry: Define exception entry points. */ #include <linux/linkage.h> @@ -37,6 +36,7 @@ #include <asm/pgtable_types.h> #include <asm/export.h> #include <asm/frame.h> +#include <asm/trapnr.h> #include <asm/nospec-branch.h> #include <linux/err.h> @@ -53,57 +53,6 @@ SYM_CODE_START(native_usergs_sysret64) SYM_CODE_END(native_usergs_sysret64) #endif /* CONFIG_PARAVIRT */ -.macro TRACE_IRQS_FLAGS flags:req -#ifdef CONFIG_TRACE_IRQFLAGS - btl $9, \flags /* interrupts off? */ - jnc 1f - TRACE_IRQS_ON -1: -#endif -.endm - -.macro TRACE_IRQS_IRETQ - TRACE_IRQS_FLAGS EFLAGS(%rsp) -.endm - -/* - * When dynamic function tracer is enabled it will add a breakpoint - * to all locations that it is about to modify, sync CPUs, update - * all the code, sync CPUs, then remove the breakpoints. In this time - * if lockdep is enabled, it might jump back into the debug handler - * outside the updating of the IST protection. (TRACE_IRQS_ON/OFF). - * - * We need to change the IDT table before calling TRACE_IRQS_ON/OFF to - * make sure the stack pointer does not get reset back to the top - * of the debug stack, and instead just reuses the current stack. - */ -#if defined(CONFIG_DYNAMIC_FTRACE) && defined(CONFIG_TRACE_IRQFLAGS) - -.macro TRACE_IRQS_OFF_DEBUG - call debug_stack_set_zero - TRACE_IRQS_OFF - call debug_stack_reset -.endm - -.macro TRACE_IRQS_ON_DEBUG - call debug_stack_set_zero - TRACE_IRQS_ON - call debug_stack_reset -.endm - -.macro TRACE_IRQS_IRETQ_DEBUG - btl $9, EFLAGS(%rsp) /* interrupts off? */ - jnc 1f - TRACE_IRQS_ON_DEBUG -1: -.endm - -#else -# define TRACE_IRQS_OFF_DEBUG TRACE_IRQS_OFF -# define TRACE_IRQS_ON_DEBUG TRACE_IRQS_ON -# define TRACE_IRQS_IRETQ_DEBUG TRACE_IRQS_IRETQ -#endif - /* * 64-bit SYSCALL instruction entry. Up to 6 arguments in registers. * @@ -144,11 +93,6 @@ SYM_CODE_END(native_usergs_sysret64) SYM_CODE_START(entry_SYSCALL_64) UNWIND_HINT_EMPTY - /* - * Interrupts are off on entry. - * We do not frame this tiny irq-off block with TRACE_IRQS_OFF/ON, - * it is too small to ever cause noticeable irq latency. - */ swapgs /* tss.sp2 is scratch space. */ @@ -167,15 +111,11 @@ SYM_INNER_LABEL(entry_SYSCALL_64_after_hwframe, SYM_L_GLOBAL) PUSH_AND_CLEAR_REGS rax=$-ENOSYS - TRACE_IRQS_OFF - /* IRQs are off. */ movq %rax, %rdi movq %rsp, %rsi call do_syscall_64 /* returns with IRQs disabled */ - TRACE_IRQS_ON /* return enables interrupts */ - /* * Try to use SYSRET instead of IRET if we're returning to * a completely clean 64-bit userspace context. If we're not, @@ -279,6 +219,7 @@ SYM_CODE_END(entry_SYSCALL_64) * %rdi: prev task * %rsi: next task */ +.pushsection .text, "ax" SYM_FUNC_START(__switch_to_asm) /* * Save callee-saved registers @@ -321,6 +262,7 @@ SYM_FUNC_START(__switch_to_asm) jmp __switch_to SYM_FUNC_END(__switch_to_asm) +.popsection /* * A newly forked process directly context switches into this address. @@ -329,6 +271,7 @@ SYM_FUNC_END(__switch_to_asm) * rbx: kernel thread func (NULL for user thread) * r12: kernel thread arg */ +.pushsection .text, "ax" SYM_CODE_START(ret_from_fork) UNWIND_HINT_EMPTY movq %rax, %rdi @@ -341,7 +284,6 @@ SYM_CODE_START(ret_from_fork) UNWIND_HINT_REGS movq %rsp, %rdi call syscall_return_slowpath /* returns with IRQs disabled */ - TRACE_IRQS_ON /* user mode is traced as IRQS on */ jmp swapgs_restore_regs_and_return_to_usermode 1: @@ -357,34 +299,7 @@ SYM_CODE_START(ret_from_fork) movq $0, RAX(%rsp) jmp 2b SYM_CODE_END(ret_from_fork) - -/* - * Build the entry stubs with some assembler magic. - * We pack 1 stub into every 8-byte block. - */ - .align 8 -SYM_CODE_START(irq_entries_start) - vector=FIRST_EXTERNAL_VECTOR - .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) - UNWIND_HINT_IRET_REGS - pushq $(~vector+0x80) /* Note: always in signed byte range */ - jmp common_interrupt - .align 8 - vector=vector+1 - .endr -SYM_CODE_END(irq_entries_start) - - .align 8 -SYM_CODE_START(spurious_entries_start) - vector=FIRST_SYSTEM_VECTOR - .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR) - UNWIND_HINT_IRET_REGS - pushq $(~vector+0x80) /* Note: always in signed byte range */ - jmp common_spurious - .align 8 - vector=vector+1 - .endr -SYM_CODE_END(spurious_entries_start) +.popsection .macro DEBUG_ENTRY_ASSERT_IRQS_OFF #ifdef CONFIG_DEBUG_ENTRY @@ -398,228 +313,185 @@ SYM_CODE_END(spurious_entries_start) #endif .endm -/* - * Enters the IRQ stack if we're not already using it. NMI-safe. Clobbers - * flags and puts old RSP into old_rsp, and leaves all other GPRs alone. - * Requires kernel GSBASE. - * - * The invariant is that, if irq_count != -1, then the IRQ stack is in use. +/** + * idtentry_body - Macro to emit code calling the C function + * @cfunc: C function to be called + * @has_error_code: Hardware pushed error code on stack */ -.macro ENTER_IRQ_STACK regs=1 old_rsp save_ret=0 - DEBUG_ENTRY_ASSERT_IRQS_OFF +.macro idtentry_body cfunc has_error_code:req - .if \save_ret - /* - * If save_ret is set, the original stack contains one additional - * entry -- the return address. Therefore, move the address one - * entry below %rsp to \old_rsp. - */ - leaq 8(%rsp), \old_rsp - .else - movq %rsp, \old_rsp - .endif - - .if \regs - UNWIND_HINT_REGS base=\old_rsp - .endif + call error_entry + UNWIND_HINT_REGS - incl PER_CPU_VAR(irq_count) - jnz .Lirq_stack_push_old_rsp_\@ + movq %rsp, %rdi /* pt_regs pointer into 1st argument*/ - /* - * Right now, if we just incremented irq_count to zero, we've - * claimed the IRQ stack but we haven't switched to it yet. - * - * If anything is added that can interrupt us here without using IST, - * it must be *extremely* careful to limit its stack usage. This - * could include kprobes and a hypothetical future IST-less #DB - * handler. - * - * The OOPS unwinder relies on the word at the top of the IRQ - * stack linking back to the previous RSP for the entire time we're - * on the IRQ stack. For this to work reliably, we need to write - * it before we actually move ourselves to the IRQ stack. - */ + .if \has_error_code == 1 + movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ + movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ + .endif - movq \old_rsp, PER_CPU_VAR(irq_stack_backing_store + IRQ_STACK_SIZE - 8) - movq PER_CPU_VAR(hardirq_stack_ptr), %rsp + call \cfunc -#ifdef CONFIG_DEBUG_ENTRY - /* - * If the first movq above becomes wrong due to IRQ stack layout - * changes, the only way we'll notice is if we try to unwind right - * here. Assert that we set up the stack right to catch this type - * of bug quickly. - */ - cmpq -8(%rsp), \old_rsp - je .Lirq_stack_okay\@ - ud2 - .Lirq_stack_okay\@: -#endif + jmp error_return +.endm -.Lirq_stack_push_old_rsp_\@: - pushq \old_rsp +/** + * idtentry - Macro to generate entry stubs for simple IDT entries + * @vector: Vector number + * @asmsym: ASM symbol for the entry point + * @cfunc: C function to be called + * @has_error_code: Hardware pushed error code on stack + * + * The macro emits code to set up the kernel context for straight forward + * and simple IDT entries. No IST stack, no paranoid entry checks. + */ +.macro idtentry vector asmsym cfunc has_error_code:req +SYM_CODE_START(\asmsym) + UNWIND_HINT_IRET_REGS offset=\has_error_code*8 + ASM_CLAC - .if \regs - UNWIND_HINT_REGS indirect=1 + .if \has_error_code == 0 + pushq $-1 /* ORIG_RAX: no syscall to restart */ .endif - .if \save_ret - /* - * Push the return address to the stack. This return address can - * be found at the "real" original RSP, which was offset by 8 at - * the beginning of this macro. - */ - pushq -8(\old_rsp) + .if \vector == X86_TRAP_BP + /* + * If coming from kernel space, create a 6-word gap to allow the + * int3 handler to emulate a call instruction. + */ + testb $3, CS-ORIG_RAX(%rsp) + jnz .Lfrom_usermode_no_gap_\@ + .rept 6 + pushq 5*8(%rsp) + .endr + UNWIND_HINT_IRET_REGS offset=8 +.Lfrom_usermode_no_gap_\@: .endif + + idtentry_body \cfunc \has_error_code + +_ASM_NOKPROBE(\asmsym) +SYM_CODE_END(\asmsym) .endm /* - * Undoes ENTER_IRQ_STACK. + * Interrupt entry/exit. + * + + The interrupt stubs push (vector) onto the stack, which is the error_code + * position of idtentry exceptions, and jump to one of the two idtentry points + * (common/spurious). + * + * common_interrupt is a hotpath, align it to a cache line */ -.macro LEAVE_IRQ_STACK regs=1 - DEBUG_ENTRY_ASSERT_IRQS_OFF - /* We need to be off the IRQ stack before decrementing irq_count. */ - popq %rsp - - .if \regs - UNWIND_HINT_REGS - .endif - - /* - * As in ENTER_IRQ_STACK, irq_count == 0, we are still claiming - * the irq stack but we're not on it. - */ - - decl PER_CPU_VAR(irq_count) +.macro idtentry_irq vector cfunc + .p2align CONFIG_X86_L1_CACHE_SHIFT + idtentry \vector asm_\cfunc \cfunc has_error_code=1 .endm /* - * Interrupt entry helper function. + * System vectors which invoke their handlers directly and are not + * going through the regular common device interrupt handling code. + */ +.macro idtentry_sysvec vector cfunc + idtentry \vector asm_\cfunc \cfunc has_error_code=0 +.endm + +/** + * idtentry_mce_db - Macro to generate entry stubs for #MC and #DB + * @vector: Vector number + * @asmsym: ASM symbol for the entry point + * @cfunc: C function to be called + * + * The macro emits code to set up the kernel context for #MC and #DB + * + * If the entry comes from user space it uses the normal entry path + * including the return to user space work and preemption checks on + * exit. * - * Entry runs with interrupts off. Stack layout at entry: - * +----------------------------------------------------+ - * | regs->ss | - * | regs->rsp | - * | regs->eflags | - * | regs->cs | - * | regs->ip | - * +----------------------------------------------------+ - * | regs->orig_ax = ~(interrupt number) | - * +----------------------------------------------------+ - * | return address | - * +----------------------------------------------------+ + * If hits in kernel mode then it needs to go through the paranoid + * entry as the exception can hit any random state. No preemption + * check on exit to keep the paranoid path simple. */ -SYM_CODE_START(interrupt_entry) - UNWIND_HINT_IRET_REGS offset=16 +.macro idtentry_mce_db vector asmsym cfunc +SYM_CODE_START(\asmsym) + UNWIND_HINT_IRET_REGS ASM_CLAC - cld - testb $3, CS-ORIG_RAX+8(%rsp) - jz 1f - SWAPGS - FENCE_SWAPGS_USER_ENTRY + pushq $-1 /* ORIG_RAX: no syscall to restart */ + /* - * Switch to the thread stack. The IRET frame and orig_ax are - * on the stack, as well as the return address. RDI..R12 are - * not (yet) on the stack and space has not (yet) been - * allocated for them. + * If the entry is from userspace, switch stacks and treat it as + * a normal entry. */ - pushq %rdi + testb $3, CS-ORIG_RAX(%rsp) + jnz .Lfrom_usermode_switch_stack_\@ - /* Need to switch before accessing the thread stack. */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi - movq %rsp, %rdi - movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp + /* + * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. + * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS + */ + call paranoid_entry - /* - * We have RDI, return address, and orig_ax on the stack on - * top of the IRET frame. That means offset=24 - */ - UNWIND_HINT_IRET_REGS base=%rdi offset=24 - - pushq 7*8(%rdi) /* regs->ss */ - pushq 6*8(%rdi) /* regs->rsp */ - pushq 5*8(%rdi) /* regs->eflags */ - pushq 4*8(%rdi) /* regs->cs */ - pushq 3*8(%rdi) /* regs->ip */ - UNWIND_HINT_IRET_REGS - pushq 2*8(%rdi) /* regs->orig_ax */ - pushq 8(%rdi) /* return address */ + UNWIND_HINT_REGS - movq (%rdi), %rdi - jmp 2f -1: - FENCE_SWAPGS_KERNEL_ENTRY -2: - PUSH_AND_CLEAR_REGS save_ret=1 - ENCODE_FRAME_POINTER 8 + movq %rsp, %rdi /* pt_regs pointer */ - testb $3, CS+8(%rsp) - jz 1f + call \cfunc - /* - * IRQ from user mode. - * - * We need to tell lockdep that IRQs are off. We can't do this until - * we fix gsbase, and we should do it before enter_from_user_mode - * (which can take locks). Since TRACE_IRQS_OFF is idempotent, - * the simplest way to handle it is to just call it twice if - * we enter from user mode. There's no reason to optimize this since - * TRACE_IRQS_OFF is a no-op if lockdep is off. - */ - TRACE_IRQS_OFF + jmp paranoid_exit - CALL_enter_from_user_mode + /* Switch to the regular task stack and use the noist entry point */ +.Lfrom_usermode_switch_stack_\@: + idtentry_body noist_\cfunc, has_error_code=0 -1: - ENTER_IRQ_STACK old_rsp=%rdi save_ret=1 - /* We entered an interrupt context - irqs are off: */ - TRACE_IRQS_OFF +_ASM_NOKPROBE(\asmsym) +SYM_CODE_END(\asmsym) +.endm - ret -SYM_CODE_END(interrupt_entry) -_ASM_NOKPROBE(interrupt_entry) +/* + * Double fault entry. Straight paranoid. No checks from which context + * this comes because for the espfix induced #DF this would do the wrong + * thing. + */ +.macro idtentry_df vector asmsym cfunc +SYM_CODE_START(\asmsym) + UNWIND_HINT_IRET_REGS offset=8 + ASM_CLAC + /* + * paranoid_entry returns SWAPGS flag for paranoid_exit in EBX. + * EBX == 0 -> SWAPGS, EBX == 1 -> no SWAPGS + */ + call paranoid_entry + UNWIND_HINT_REGS -/* Interrupt entry/exit. */ + movq %rsp, %rdi /* pt_regs pointer into first argument */ + movq ORIG_RAX(%rsp), %rsi /* get error code into 2nd argument*/ + movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ + call \cfunc + + jmp paranoid_exit + +_ASM_NOKPROBE(\asmsym) +SYM_CODE_END(\asmsym) +.endm /* - * The interrupt stubs push (~vector+0x80) onto the stack and - * then jump to common_spurious/interrupt. + * Include the defines which emit the idt entries which are shared + * shared between 32 and 64 bit and emit the __irqentry_text_* markers + * so the stacktrace boundary checks work. */ -SYM_CODE_START_LOCAL(common_spurious) - addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ - call interrupt_entry - UNWIND_HINT_REGS indirect=1 - call smp_spurious_interrupt /* rdi points to pt_regs */ - jmp ret_from_intr -SYM_CODE_END(common_spurious) -_ASM_NOKPROBE(common_spurious) - -/* common_interrupt is a hotpath. Align it */ - .p2align CONFIG_X86_L1_CACHE_SHIFT -SYM_CODE_START_LOCAL(common_interrupt) - addq $-0x80, (%rsp) /* Adjust vector to [-256, -1] range */ - call interrupt_entry - UNWIND_HINT_REGS indirect=1 - call do_IRQ /* rdi points to pt_regs */ - /* 0(%rsp): old RSP */ -ret_from_intr: - DISABLE_INTERRUPTS(CLBR_ANY) - TRACE_IRQS_OFF - - LEAVE_IRQ_STACK + .align 16 + .globl __irqentry_text_start +__irqentry_text_start: - testb $3, CS(%rsp) - jz retint_kernel +#include <asm/idtentry.h> - /* Interrupt came from user space */ -.Lretint_user: - mov %rsp,%rdi - call prepare_exit_to_usermode - TRACE_IRQS_ON + .align 16 + .globl __irqentry_text_end +__irqentry_text_end: +SYM_CODE_START_LOCAL(common_interrupt_return) SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) #ifdef CONFIG_DEBUG_ENTRY /* Assert that pt_regs indicates user mode. */ @@ -662,23 +534,6 @@ SYM_INNER_LABEL(swapgs_restore_regs_and_return_to_usermode, SYM_L_GLOBAL) INTERRUPT_RETURN -/* Returning to kernel space */ -retint_kernel: -#ifdef CONFIG_PREEMPTION - /* Interrupts are off */ - /* Check if we need preemption */ - btl $9, EFLAGS(%rsp) /* were interrupts off? */ - jnc 1f - cmpl $0, PER_CPU_VAR(__preempt_count) - jnz 1f - call preempt_schedule_irq -1: -#endif - /* - * The iretq could re-enable interrupts: - */ - TRACE_IRQS_IRETQ - SYM_INNER_LABEL(restore_regs_and_return_to_kernel, SYM_L_GLOBAL) #ifdef CONFIG_DEBUG_ENTRY /* Assert that pt_regs indicates kernel mode. */ @@ -710,7 +565,7 @@ SYM_INNER_LABEL(native_irq_return_iret, SYM_L_GLOBAL) /* * This may fault. Non-paranoid faults on return to userspace are * handled by fixup_bad_iret. These include #SS, #GP, and #NP. - * Double-faults due to espfix64 are handled in do_double_fault. + * Double-faults due to espfix64 are handled in exc_double_fault. * Other faults here are fatal. */ iretq @@ -788,280 +643,32 @@ native_irq_return_ldt: */ jmp native_irq_return_iret #endif -SYM_CODE_END(common_interrupt) -_ASM_NOKPROBE(common_interrupt) - -/* - * APIC interrupts. - */ -.macro apicinterrupt3 num sym do_sym -SYM_CODE_START(\sym) - UNWIND_HINT_IRET_REGS - pushq $~(\num) -.Lcommon_\sym: - call interrupt_entry - UNWIND_HINT_REGS indirect=1 - call \do_sym /* rdi points to pt_regs */ - jmp ret_from_intr -SYM_CODE_END(\sym) -_ASM_NOKPROBE(\sym) -.endm - -/* Make sure APIC interrupt handlers end up in the irqentry section: */ -#define PUSH_SECTION_IRQENTRY .pushsection .irqentry.text, "ax" -#define POP_SECTION_IRQENTRY .popsection - -.macro apicinterrupt num sym do_sym -PUSH_SECTION_IRQENTRY -apicinterrupt3 \num \sym \do_sym -POP_SECTION_IRQENTRY -.endm - -#ifdef CONFIG_SMP -apicinterrupt3 IRQ_MOVE_CLEANUP_VECTOR irq_move_cleanup_interrupt smp_irq_move_cleanup_interrupt -apicinterrupt3 REBOOT_VECTOR reboot_interrupt smp_reboot_interrupt -#endif - -#ifdef CONFIG_X86_UV -apicinterrupt3 UV_BAU_MESSAGE uv_bau_message_intr1 uv_bau_message_interrupt -#endif - -apicinterrupt LOCAL_TIMER_VECTOR apic_timer_interrupt smp_apic_timer_interrupt -apicinterrupt X86_PLATFORM_IPI_VECTOR x86_platform_ipi smp_x86_platform_ipi - -#ifdef CONFIG_HAVE_KVM -apicinterrupt3 POSTED_INTR_VECTOR kvm_posted_intr_ipi smp_kvm_posted_intr_ipi -apicinterrupt3 POSTED_INTR_WAKEUP_VECTOR kvm_posted_intr_wakeup_ipi smp_kvm_posted_intr_wakeup_ipi -apicinterrupt3 POSTED_INTR_NESTED_VECTOR kvm_posted_intr_nested_ipi smp_kvm_posted_intr_nested_ipi -#endif - -#ifdef CONFIG_X86_MCE_THRESHOLD -apicinterrupt THRESHOLD_APIC_VECTOR threshold_interrupt smp_threshold_interrupt -#endif - -#ifdef CONFIG_X86_MCE_AMD -apicinterrupt DEFERRED_ERROR_VECTOR deferred_error_interrupt smp_deferred_error_interrupt -#endif - -#ifdef CONFIG_X86_THERMAL_VECTOR -apicinterrupt THERMAL_APIC_VECTOR thermal_interrupt smp_thermal_interrupt -#endif - -#ifdef CONFIG_SMP -apicinterrupt CALL_FUNCTION_SINGLE_VECTOR call_function_single_interrupt smp_call_function_single_interrupt -apicinterrupt CALL_FUNCTION_VECTOR call_function_interrupt smp_call_function_interrupt -apicinterrupt RESCHEDULE_VECTOR reschedule_interrupt smp_reschedule_interrupt -#endif - -apicinterrupt ERROR_APIC_VECTOR error_interrupt smp_error_interrupt -apicinterrupt SPURIOUS_APIC_VECTOR spurious_interrupt smp_spurious_interrupt - -#ifdef CONFIG_IRQ_WORK -apicinterrupt IRQ_WORK_VECTOR irq_work_interrupt smp_irq_work_interrupt -#endif +SYM_CODE_END(common_interrupt_return) +_ASM_NOKPROBE(common_interrupt_return) /* - * Exception entry points. - */ -#define CPU_TSS_IST(x) PER_CPU_VAR(cpu_tss_rw) + (TSS_ist + (x) * 8) - -.macro idtentry_part do_sym, has_error_code:req, read_cr2:req, paranoid:req, shift_ist=-1, ist_offset=0 - - .if \paranoid - call paranoid_entry - /* returned flag: ebx=0: need swapgs on exit, ebx=1: don't need it */ - .else - call error_entry - .endif - UNWIND_HINT_REGS - - .if \read_cr2 - /* - * Store CR2 early so subsequent faults cannot clobber it. Use R12 as - * intermediate storage as RDX can be clobbered in enter_from_user_mode(). - * GET_CR2_INTO can clobber RAX. - */ - GET_CR2_INTO(%r12); - .endif - - .if \shift_ist != -1 - TRACE_IRQS_OFF_DEBUG /* reload IDT in case of recursion */ - .else - TRACE_IRQS_OFF - .endif - - .if \paranoid == 0 - testb $3, CS(%rsp) - jz .Lfrom_kernel_no_context_tracking_\@ - CALL_enter_from_user_mode -.Lfrom_kernel_no_context_tracking_\@: - .endif - - movq %rsp, %rdi /* pt_regs pointer */ - - .if \has_error_code - movq ORIG_RAX(%rsp), %rsi /* get error code */ - movq $-1, ORIG_RAX(%rsp) /* no syscall to restart */ - .else - xorl %esi, %esi /* no error code */ - .endif - - .if \shift_ist != -1 - subq $\ist_offset, CPU_TSS_IST(\shift_ist) - .endif - - .if \read_cr2 - movq %r12, %rdx /* Move CR2 into 3rd argument */ - .endif - - call \do_sym - - .if \shift_ist != -1 - addq $\ist_offset, CPU_TSS_IST(\shift_ist) - .endif - - .if \paranoid - /* this procedure expect "no swapgs" flag in ebx */ - jmp paranoid_exit - .else - jmp error_exit - .endif - -.endm - -/** - * idtentry - Generate an IDT entry stub - * @sym: Name of the generated entry point - * @do_sym: C function to be called - * @has_error_code: True if this IDT vector has an error code on the stack - * @paranoid: non-zero means that this vector may be invoked from - * kernel mode with user GSBASE and/or user CR3. - * 2 is special -- see below. - * @shift_ist: Set to an IST index if entries from kernel mode should - * decrement the IST stack so that nested entries get a - * fresh stack. (This is for #DB, which has a nasty habit - * of recursing.) - * @create_gap: create a 6-word stack gap when coming from kernel mode. - * @read_cr2: load CR2 into the 3rd argument; done before calling any C code - * - * idtentry generates an IDT stub that sets up a usable kernel context, - * creates struct pt_regs, and calls @do_sym. The stub has the following - * special behaviors: - * - * On an entry from user mode, the stub switches from the trampoline or - * IST stack to the normal thread stack. On an exit to user mode, the - * normal exit-to-usermode path is invoked. - * - * On an exit to kernel mode, if @paranoid == 0, we check for preemption, - * whereas we omit the preemption check if @paranoid != 0. This is purely - * because the implementation is simpler this way. The kernel only needs - * to check for asynchronous kernel preemption when IRQ handlers return. - * - * If @paranoid == 0, then the stub will handle IRET faults by pretending - * that the fault came from user mode. It will handle gs_change faults by - * pretending that the fault happened with kernel GSBASE. Since this handling - * is omitted for @paranoid != 0, the #GP, #SS, and #NP stubs must have - * @paranoid == 0. This special handling will do the wrong thing for - * espfix-induced #DF on IRET, so #DF must not use @paranoid == 0. + * Reload gs selector with exception handling + * edi: new selector * - * @paranoid == 2 is special: the stub will never switch stacks. This is for - * #DF: if the thread stack is somehow unusable, we'll still get a useful OOPS. + * Is in entry.text as it shouldn't be instrumented. */ -.macro idtentry sym do_sym has_error_code:req paranoid=0 shift_ist=-1 ist_offset=0 create_gap=0 read_cr2=0 -SYM_CODE_START(\sym) - UNWIND_HINT_IRET_REGS offset=\has_error_code*8 - - /* Sanity check */ - .if \shift_ist != -1 && \paranoid != 1 - .error "using shift_ist requires paranoid=1" - .endif - - .if \create_gap && \paranoid - .error "using create_gap requires paranoid=0" - .endif - - ASM_CLAC - - .if \has_error_code == 0 - pushq $-1 /* ORIG_RAX: no syscall to restart */ - .endif - - .if \paranoid == 1 - testb $3, CS-ORIG_RAX(%rsp) /* If coming from userspace, switch stacks */ - jnz .Lfrom_usermode_switch_stack_\@ - .endif - - .if \create_gap == 1 - /* - * If coming from kernel space, create a 6-word gap to allow the - * int3 handler to emulate a call instruction. - */ - testb $3, CS-ORIG_RAX(%rsp) - jnz .Lfrom_usermode_no_gap_\@ - .rept 6 - pushq 5*8(%rsp) - .endr - UNWIND_HINT_IRET_REGS offset=8 -.Lfrom_usermode_no_gap_\@: - .endif - - idtentry_part \do_sym, \has_error_code, \read_cr2, \paranoid, \shift_ist, \ist_offset - - .if \paranoid == 1 - /* - * Entry from userspace. Switch stacks and treat it - * as a normal entry. This means that paranoid handlers - * run in real process context if user_mode(regs). - */ -.Lfrom_usermode_switch_stack_\@: - idtentry_part \do_sym, \has_error_code, \read_cr2, paranoid=0 - .endif - -_ASM_NOKPROBE(\sym) -SYM_CODE_END(\sym) -.endm - -idtentry divide_error do_divide_error has_error_code=0 -idtentry overflow do_overflow has_error_code=0 -idtentry bounds do_bounds has_error_code=0 -idtentry invalid_op do_invalid_op has_error_code=0 -idtentry device_not_available do_device_not_available has_error_code=0 -idtentry double_fault do_double_fault has_error_code=1 paranoid=2 read_cr2=1 -idtentry coprocessor_segment_overrun do_coprocessor_segment_overrun has_error_code=0 -idtentry invalid_TSS do_invalid_TSS has_error_code=1 -idtentry segment_not_present do_segment_not_present has_error_code=1 -idtentry spurious_interrupt_bug do_spurious_interrupt_bug has_error_code=0 -idtentry coprocessor_error do_coprocessor_error has_error_code=0 -idtentry alignment_check do_alignment_check has_error_code=1 -idtentry simd_coprocessor_error do_simd_coprocessor_error has_error_code=0 - - - /* - * Reload gs selector with exception handling - * edi: new selector - */ -SYM_FUNC_START(native_load_gs_index) +SYM_FUNC_START(asm_load_gs_index) FRAME_BEGIN - pushfq - DISABLE_INTERRUPTS(CLBR_ANY & ~CLBR_RDI) - TRACE_IRQS_OFF - SWAPGS + swapgs .Lgs_change: movl %edi, %gs 2: ALTERNATIVE "", "mfence", X86_BUG_SWAPGS_FENCE - SWAPGS - TRACE_IRQS_FLAGS (%rsp) - popfq + swapgs FRAME_END ret -SYM_FUNC_END(native_load_gs_index) -EXPORT_SYMBOL(native_load_gs_index) +SYM_FUNC_END(asm_load_gs_index) +EXPORT_SYMBOL(asm_load_gs_index) _ASM_EXTABLE(.Lgs_change, .Lbad_gs) .section .fixup, "ax" /* running with kernelgs */ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) - SWAPGS /* switch back to user gs */ + swapgs /* switch back to user gs */ .macro ZAP_GS /* This can't be a string because the preprocessor needs to see it. */ movl $__USER_DS, %eax @@ -1074,20 +681,46 @@ SYM_CODE_START_LOCAL_NOALIGN(.Lbad_gs) SYM_CODE_END(.Lbad_gs) .previous -/* Call softirq on interrupt stack. Interrupts are off. */ -SYM_FUNC_START(do_softirq_own_stack) - pushq %rbp - mov %rsp, %rbp - ENTER_IRQ_STACK regs=0 old_rsp=%r11 - call __do_softirq - LEAVE_IRQ_STACK regs=0 +/* + * rdi: New stack pointer points to the top word of the stack + * rsi: Function pointer + * rdx: Function argument (can be NULL if none) + */ +SYM_FUNC_START(asm_call_on_stack) + /* + * Save the frame pointer unconditionally. This allows the ORC + * unwinder to handle the stack switch. + */ + pushq %rbp + mov %rsp, %rbp + + /* + * The unwinder relies on the word at the top of the new stack + * page linking back to the previous RSP. + */ + mov %rsp, (%rdi) + mov %rdi, %rsp + /* Move the argument to the right place */ + mov %rdx, %rdi + +1: + .pushsection .discard.instr_begin + .long 1b - . + .popsection + + CALL_NOSPEC rsi + +2: + .pushsection .discard.instr_end + .long 2b - . + .popsection + + /* Restore the previous stack pointer from RBP. */ leaveq ret -SYM_FUNC_END(do_softirq_own_stack) +SYM_FUNC_END(asm_call_on_stack) #ifdef CONFIG_XEN_PV -idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 - /* * A note on the "critical region" in our callback handler. * We want to avoid stacking callback handlers due to events occurring @@ -1100,9 +733,10 @@ idtentry hypervisor_callback xen_do_hypervisor_callback has_error_code=0 * So, on entry to the handler we detect whether we interrupted an * existing activation in its critical region -- if so, we pop the current * activation and restart the handler using the previous one. + * + * C calling convention: exc_xen_hypervisor_callback(struct *pt_regs) */ -/* do_hypervisor_callback(struct *pt_regs) */ -SYM_CODE_START_LOCAL(xen_do_hypervisor_callback) +SYM_CODE_START_LOCAL(exc_xen_hypervisor_callback) /* * Since we don't modify %rdi, evtchn_do_upall(struct *pt_regs) will @@ -1112,15 +746,10 @@ SYM_CODE_START_LOCAL(xen_do_hypervisor_callback) movq %rdi, %rsp /* we don't return, adjust the stack frame */ UNWIND_HINT_REGS - ENTER_IRQ_STACK old_rsp=%r10 - call xen_evtchn_do_upcall - LEAVE_IRQ_STACK + call xen_pv_evtchn_do_upcall -#ifndef CONFIG_PREEMPTION - call xen_maybe_preempt_hcall -#endif - jmp error_exit -SYM_CODE_END(xen_do_hypervisor_callback) + jmp error_return +SYM_CODE_END(exc_xen_hypervisor_callback) /* * Hypervisor uses this for application faults while it executes. @@ -1155,7 +784,7 @@ SYM_CODE_START(xen_failsafe_callback) addq $0x30, %rsp pushq $0 /* RIP */ UNWIND_HINT_IRET_REGS offset=8 - jmp general_protection + jmp asm_exc_general_protection 1: /* Segment mismatch => Category 1 (Bad segment). Retry the IRET. */ movq (%rsp), %rcx movq 8(%rsp), %r11 @@ -1164,48 +793,10 @@ SYM_CODE_START(xen_failsafe_callback) pushq $-1 /* orig_ax = -1 => not a system call */ PUSH_AND_CLEAR_REGS ENCODE_FRAME_POINTER - jmp error_exit + jmp error_return SYM_CODE_END(xen_failsafe_callback) #endif /* CONFIG_XEN_PV */ -#ifdef CONFIG_XEN_PVHVM -apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ - xen_hvm_callback_vector xen_evtchn_do_upcall -#endif - - -#if IS_ENABLED(CONFIG_HYPERV) -apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ - hyperv_callback_vector hyperv_vector_handler - -apicinterrupt3 HYPERV_REENLIGHTENMENT_VECTOR \ - hyperv_reenlightenment_vector hyperv_reenlightenment_intr - -apicinterrupt3 HYPERV_STIMER0_VECTOR \ - hv_stimer0_callback_vector hv_stimer0_vector_handler -#endif /* CONFIG_HYPERV */ - -#if IS_ENABLED(CONFIG_ACRN_GUEST) -apicinterrupt3 HYPERVISOR_CALLBACK_VECTOR \ - acrn_hv_callback_vector acrn_hv_vector_handler -#endif - -idtentry debug do_debug has_error_code=0 paranoid=1 shift_ist=IST_INDEX_DB ist_offset=DB_STACK_OFFSET -idtentry int3 do_int3 has_error_code=0 create_gap=1 -idtentry stack_segment do_stack_segment has_error_code=1 - -#ifdef CONFIG_XEN_PV -idtentry xennmi do_nmi has_error_code=0 -idtentry xendebug do_debug has_error_code=0 -#endif - -idtentry general_protection do_general_protection has_error_code=1 -idtentry page_fault do_page_fault has_error_code=1 read_cr2=1 - -#ifdef CONFIG_X86_MCE -idtentry machine_check do_mce has_error_code=0 paranoid=1 -#endif - /* * Save all registers in pt_regs, and switch gs if needed. * Use slow, but surefire "are we in kernel?" check. @@ -1261,17 +852,13 @@ SYM_CODE_END(paranoid_entry) */ SYM_CODE_START_LOCAL(paranoid_exit) UNWIND_HINT_REGS - DISABLE_INTERRUPTS(CLBR_ANY) - TRACE_IRQS_OFF_DEBUG testl %ebx, %ebx /* swapgs needed? */ jnz .Lparanoid_exit_no_swapgs - TRACE_IRQS_IRETQ /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 SWAPGS_UNSAFE_STACK jmp restore_regs_and_return_to_kernel .Lparanoid_exit_no_swapgs: - TRACE_IRQS_IRETQ_DEBUG /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%rbx save_reg=%r14 jmp restore_regs_and_return_to_kernel @@ -1335,7 +922,6 @@ SYM_CODE_START_LOCAL(error_entry) */ SWAPGS FENCE_SWAPGS_USER_ENTRY - SWITCH_TO_KERNEL_CR3 scratch_reg=%rax jmp .Lerror_entry_done .Lbstep_iret: @@ -1362,14 +948,13 @@ SYM_CODE_START_LOCAL(error_entry) jmp .Lerror_entry_from_usermode_after_swapgs SYM_CODE_END(error_entry) -SYM_CODE_START_LOCAL(error_exit) +SYM_CODE_START_LOCAL(error_return) UNWIND_HINT_REGS - DISABLE_INTERRUPTS(CLBR_ANY) - TRACE_IRQS_OFF + DEBUG_ENTRY_ASSERT_IRQS_OFF testb $3, CS(%rsp) - jz retint_kernel - jmp .Lretint_user -SYM_CODE_END(error_exit) + jz restore_regs_and_return_to_kernel + jmp swapgs_restore_regs_and_return_to_usermode +SYM_CODE_END(error_return) /* * Runs on exception stack. Xen PV does not go through this path at all, @@ -1379,7 +964,7 @@ SYM_CODE_END(error_exit) * %r14: Used to save/restore the CR3 of the interrupted context * when PAGE_TABLE_ISOLATION is in use. Do not clobber. */ -SYM_CODE_START(nmi) +SYM_CODE_START(asm_exc_nmi) UNWIND_HINT_IRET_REGS /* @@ -1464,7 +1049,7 @@ SYM_CODE_START(nmi) movq %rsp, %rdi movq $-1, %rsi - call do_nmi + call exc_nmi /* * Return back to user mode. We must *not* do the normal exit @@ -1521,7 +1106,7 @@ SYM_CODE_START(nmi) * end_repeat_nmi, then we are a nested NMI. We must not * modify the "iret" frame because it's being written by * the outer NMI. That's okay; the outer NMI handler is - * about to about to call do_nmi anyway, so we can just + * about to about to call exc_nmi() anyway, so we can just * resume the outer NMI. */ @@ -1640,7 +1225,7 @@ repeat_nmi: * RSP is pointing to "outermost RIP". gsbase is unknown, but, if * we're repeating an NMI, gsbase has the same value that it had on * the first iteration. paranoid_entry will load the kernel - * gsbase if needed before we call do_nmi. "NMI executing" + * gsbase if needed before we call exc_nmi(). "NMI executing" * is zero. */ movq $1, 10*8(%rsp) /* Set "NMI executing". */ @@ -1674,10 +1259,9 @@ end_repeat_nmi: call paranoid_entry UNWIND_HINT_REGS - /* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */ movq %rsp, %rdi movq $-1, %rsi - call do_nmi + call exc_nmi /* Always restore stashed CR3 value (see paranoid_entry) */ RESTORE_CR3 scratch_reg=%r15 save_reg=%r14 @@ -1714,7 +1298,7 @@ nmi_restore: * about espfix64 on the way back to kernel mode. */ iretq -SYM_CODE_END(nmi) +SYM_CODE_END(asm_exc_nmi) #ifndef CONFIG_IA32_EMULATION /* @@ -1728,6 +1312,7 @@ SYM_CODE_START(ignore_sysret) SYM_CODE_END(ignore_sysret) #endif +.pushsection .text, "ax" SYM_CODE_START(rewind_stack_do_exit) UNWIND_HINT_FUNC /* Prevent any naive code from trying to unwind to our caller. */ @@ -1739,3 +1324,4 @@ SYM_CODE_START(rewind_stack_do_exit) call do_exit SYM_CODE_END(rewind_stack_do_exit) +.popsection diff --git a/arch/x86/entry/entry_64_compat.S b/arch/x86/entry/entry_64_compat.S index f1d3ccae5dd5..0f974ae01e62 100644 --- a/arch/x86/entry/entry_64_compat.S +++ b/arch/x86/entry/entry_64_compat.S @@ -46,12 +46,14 @@ * ebp user stack * 0(%ebp) arg6 */ -SYM_FUNC_START(entry_SYSENTER_compat) +SYM_CODE_START(entry_SYSENTER_compat) + UNWIND_HINT_EMPTY /* Interrupts are off on entry. */ SWAPGS - /* We are about to clobber %rsp anyway, clobbering here is OK */ - SWITCH_TO_KERNEL_CR3 scratch_reg=%rsp + pushq %rax + SWITCH_TO_KERNEL_CR3 scratch_reg=%rax + popq %rax movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp @@ -104,6 +106,9 @@ SYM_FUNC_START(entry_SYSENTER_compat) xorl %r14d, %r14d /* nospec r14 */ pushq $0 /* pt_regs->r15 = 0 */ xorl %r15d, %r15d /* nospec r15 */ + + UNWIND_HINT_REGS + cld /* @@ -129,17 +134,11 @@ SYM_FUNC_START(entry_SYSENTER_compat) jnz .Lsysenter_fix_flags .Lsysenter_flags_fixed: - /* - * User mode is traced as though IRQs are on, and SYSENTER - * turned them off. - */ - TRACE_IRQS_OFF - movq %rsp, %rdi call do_fast_syscall_32 /* XEN PV guests always use IRET path */ - ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ - "jmp .Lsyscall_32_done", X86_FEATURE_XENPV + ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \ + "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV jmp sysret32_from_system_call .Lsysenter_fix_flags: @@ -147,7 +146,7 @@ SYM_FUNC_START(entry_SYSENTER_compat) popfq jmp .Lsysenter_flags_fixed SYM_INNER_LABEL(__end_entry_SYSENTER_compat, SYM_L_GLOBAL) -SYM_FUNC_END(entry_SYSENTER_compat) +SYM_CODE_END(entry_SYSENTER_compat) /* * 32-bit SYSCALL entry. @@ -197,6 +196,7 @@ SYM_FUNC_END(entry_SYSENTER_compat) * 0(%esp) arg6 */ SYM_CODE_START(entry_SYSCALL_compat) + UNWIND_HINT_EMPTY /* Interrupts are off on entry. */ swapgs @@ -247,17 +247,13 @@ SYM_INNER_LABEL(entry_SYSCALL_compat_after_hwframe, SYM_L_GLOBAL) pushq $0 /* pt_regs->r15 = 0 */ xorl %r15d, %r15d /* nospec r15 */ - /* - * User mode is traced as though IRQs are on, and SYSENTER - * turned them off. - */ - TRACE_IRQS_OFF + UNWIND_HINT_REGS movq %rsp, %rdi call do_fast_syscall_32 /* XEN PV guests always use IRET path */ - ALTERNATIVE "testl %eax, %eax; jz .Lsyscall_32_done", \ - "jmp .Lsyscall_32_done", X86_FEATURE_XENPV + ALTERNATIVE "testl %eax, %eax; jz swapgs_restore_regs_and_return_to_usermode", \ + "jmp swapgs_restore_regs_and_return_to_usermode", X86_FEATURE_XENPV /* Opportunistic SYSRET */ sysret32_from_system_call: @@ -266,7 +262,7 @@ sysret32_from_system_call: * stack. So let's erase the thread stack right now. */ STACKLEAK_ERASE - TRACE_IRQS_ON /* User mode traces as IRQs on. */ + movq RBX(%rsp), %rbx /* pt_regs->rbx */ movq RBP(%rsp), %rbp /* pt_regs->rbp */ movq EFLAGS(%rsp), %r11 /* pt_regs->flags (in r11) */ @@ -340,6 +336,7 @@ SYM_CODE_END(entry_SYSCALL_compat) * ebp arg6 */ SYM_CODE_START(entry_INT80_compat) + UNWIND_HINT_EMPTY /* * Interrupts are off on entry. */ @@ -361,8 +358,11 @@ SYM_CODE_START(entry_INT80_compat) /* Need to switch before accessing the thread stack. */ SWITCH_TO_KERNEL_CR3 scratch_reg=%rdi + /* In the Xen PV case we already run on the thread stack. */ - ALTERNATIVE "movq %rsp, %rdi", "jmp .Lint80_keep_stack", X86_FEATURE_XENPV + ALTERNATIVE "", "jmp .Lint80_keep_stack", X86_FEATURE_XENPV + + movq %rsp, %rdi movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp pushq 6*8(%rdi) /* regs->ss */ @@ -401,19 +401,12 @@ SYM_CODE_START(entry_INT80_compat) xorl %r14d, %r14d /* nospec r14 */ pushq %r15 /* pt_regs->r15 */ xorl %r15d, %r15d /* nospec r15 */ - cld - /* - * User mode is traced as though IRQs are on, and the interrupt - * gate turned them off. - */ - TRACE_IRQS_OFF + UNWIND_HINT_REGS + + cld movq %rsp, %rdi call do_int80_syscall_32 -.Lsyscall_32_done: - - /* Go back to user mode. */ - TRACE_IRQS_ON jmp swapgs_restore_regs_and_return_to_usermode SYM_CODE_END(entry_INT80_compat) diff --git a/arch/x86/entry/thunk_64.S b/arch/x86/entry/thunk_64.S index dbe4493b534e..ccd32877a3c4 100644 --- a/arch/x86/entry/thunk_64.S +++ b/arch/x86/entry/thunk_64.S @@ -3,7 +3,6 @@ * Save registers before calling assembly functions. This avoids * disturbance of register allocation in some inline assembly constructs. * Copyright 2001,2002 by Andi Kleen, SuSE Labs. - * Added trace_hardirqs callers - Copyright 2007 Steven Rostedt, Red Hat, Inc. */ #include <linux/linkage.h> #include "calling.h" @@ -37,15 +36,6 @@ SYM_FUNC_END(\name) _ASM_NOKPROBE(\name) .endm -#ifdef CONFIG_TRACE_IRQFLAGS - THUNK trace_hardirqs_on_thunk,trace_hardirqs_on_caller,1 - THUNK trace_hardirqs_off_thunk,trace_hardirqs_off_caller,1 -#endif - -#ifdef CONFIG_DEBUG_LOCK_ALLOC - THUNK lockdep_sys_exit_thunk,lockdep_sys_exit -#endif - #ifdef CONFIG_PREEMPTION THUNK preempt_schedule_thunk, preempt_schedule THUNK preempt_schedule_notrace_thunk, preempt_schedule_notrace @@ -53,9 +43,7 @@ SYM_FUNC_END(\name) EXPORT_SYMBOL(preempt_schedule_notrace_thunk) #endif -#if defined(CONFIG_TRACE_IRQFLAGS) \ - || defined(CONFIG_DEBUG_LOCK_ALLOC) \ - || defined(CONFIG_PREEMPTION) +#ifdef CONFIG_PREEMPTION SYM_CODE_START_LOCAL_NOALIGN(.L_restore) popq %r11 popq %r10 diff --git a/arch/x86/entry/vdso/Makefile b/arch/x86/entry/vdso/Makefile index 54e03ab26ff3..04e65f0698f6 100644 --- a/arch/x86/entry/vdso/Makefile +++ b/arch/x86/entry/vdso/Makefile @@ -10,8 +10,11 @@ ARCH_REL_TYPE_ABS += R_386_GLOB_DAT|R_386_JMP_SLOT|R_386_RELATIVE include $(srctree)/lib/vdso/Makefile KBUILD_CFLAGS += $(DISABLE_LTO) + +# Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n UBSAN_SANITIZE := n +KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. @@ -29,6 +32,9 @@ vobjs32-y += vdso32/vclock_gettime.o # files to link into kernel obj-y += vma.o +KASAN_SANITIZE_vma.o := y +UBSAN_SANITIZE_vma.o := y +KCSAN_SANITIZE_vma.o := y OBJECT_FILES_NON_STANDARD_vma.o := n # vDSO images to build diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c index e2137070386a..a54c6a401581 100644 --- a/arch/x86/hyperv/hv_init.c +++ b/arch/x86/hyperv/hv_init.c @@ -15,6 +15,7 @@ #include <asm/hypervisor.h> #include <asm/hyperv-tlfs.h> #include <asm/mshyperv.h> +#include <asm/idtentry.h> #include <linux/version.h> #include <linux/vmalloc.h> #include <linux/mm.h> @@ -152,15 +153,11 @@ static inline bool hv_reenlightenment_available(void) ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT; } -__visible void __irq_entry hyperv_reenlightenment_intr(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_reenlightenment) { - entering_ack_irq(); - + ack_APIC_irq(); inc_irq_stat(irq_hv_reenlightenment_count); - schedule_delayed_work(&hv_reenlightenment_work, HZ/10); - - exiting_irq(); } void set_hv_tscchange_cb(void (*cb)(void)) diff --git a/arch/x86/include/asm/acrn.h b/arch/x86/include/asm/acrn.h deleted file mode 100644 index 4adb13f08af7..000000000000 --- a/arch/x86/include/asm/acrn.h +++ /dev/null @@ -1,11 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef _ASM_X86_ACRN_H -#define _ASM_X86_ACRN_H - -extern void acrn_hv_callback_vector(void); -#ifdef CONFIG_TRACING -#define trace_acrn_hv_callback_vector acrn_hv_callback_vector -#endif - -extern void acrn_hv_vector_handler(struct pt_regs *regs); -#endif /* _ASM_X86_ACRN_H */ diff --git a/arch/x86/include/asm/amd_nb.h b/arch/x86/include/asm/amd_nb.h index c7df20e78b09..455066a06f60 100644 --- a/arch/x86/include/asm/amd_nb.h +++ b/arch/x86/include/asm/amd_nb.h @@ -57,6 +57,7 @@ struct threshold_bank { /* initialized to the number of CPUs on the node sharing this bank */ refcount_t cpus; + unsigned int shared; }; struct amd_northbridge { diff --git a/arch/x86/include/asm/apic.h b/arch/x86/include/asm/apic.h index 19e94af9cc5d..2cc44e957c31 100644 --- a/arch/x86/include/asm/apic.h +++ b/arch/x86/include/asm/apic.h @@ -519,39 +519,6 @@ static inline bool apic_id_is_primary_thread(unsigned int id) { return false; } static inline void apic_smt_update(void) { } #endif -extern void irq_enter(void); -extern void irq_exit(void); - -static inline void entering_irq(void) -{ - irq_enter(); - kvm_set_cpu_l1tf_flush_l1d(); -} - -static inline void entering_ack_irq(void) -{ - entering_irq(); - ack_APIC_irq(); -} - -static inline void ipi_entering_ack_irq(void) -{ - irq_enter(); - ack_APIC_irq(); - kvm_set_cpu_l1tf_flush_l1d(); -} - -static inline void exiting_irq(void) -{ - irq_exit(); -} - -static inline void exiting_ack_irq(void) -{ - ack_APIC_irq(); - irq_exit(); -} - extern void ioapic_zap_locks(void); #endif /* _ASM_X86_APIC_H */ diff --git a/arch/x86/include/asm/atomic.h b/arch/x86/include/asm/atomic.h index 115127c7ad28..bf35e476a776 100644 --- a/arch/x86/include/asm/atomic.h +++ b/arch/x86/include/asm/atomic.h @@ -28,7 +28,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) * Note for KASAN: we deliberately don't use READ_ONCE_NOCHECK() here, * it's non-inlined function that increases binary size and stack usage. */ - return READ_ONCE((v)->counter); + return __READ_ONCE((v)->counter); } /** @@ -40,7 +40,7 @@ static __always_inline int arch_atomic_read(const atomic_t *v) */ static __always_inline void arch_atomic_set(atomic_t *v, int i) { - WRITE_ONCE(v->counter, i); + __WRITE_ONCE(v->counter, i); } /** @@ -166,6 +166,7 @@ static __always_inline int arch_atomic_add_return(int i, atomic_t *v) { return i + xadd(&v->counter, i); } +#define arch_atomic_add_return arch_atomic_add_return /** * arch_atomic_sub_return - subtract integer and return @@ -178,34 +179,39 @@ static __always_inline int arch_atomic_sub_return(int i, atomic_t *v) { return arch_atomic_add_return(-i, v); } +#define arch_atomic_sub_return arch_atomic_sub_return static __always_inline int arch_atomic_fetch_add(int i, atomic_t *v) { return xadd(&v->counter, i); } +#define arch_atomic_fetch_add arch_atomic_fetch_add static __always_inline int arch_atomic_fetch_sub(int i, atomic_t *v) { return xadd(&v->counter, -i); } +#define arch_atomic_fetch_sub arch_atomic_fetch_sub static __always_inline int arch_atomic_cmpxchg(atomic_t *v, int old, int new) { return arch_cmpxchg(&v->counter, old, new); } +#define arch_atomic_cmpxchg arch_atomic_cmpxchg -#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg static __always_inline bool arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) { return try_cmpxchg(&v->counter, old, new); } +#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg -static inline int arch_atomic_xchg(atomic_t *v, int new) +static __always_inline int arch_atomic_xchg(atomic_t *v, int new) { return arch_xchg(&v->counter, new); } +#define arch_atomic_xchg arch_atomic_xchg -static inline void arch_atomic_and(int i, atomic_t *v) +static __always_inline void arch_atomic_and(int i, atomic_t *v) { asm volatile(LOCK_PREFIX "andl %1,%0" : "+m" (v->counter) @@ -213,7 +219,7 @@ static inline void arch_atomic_and(int i, atomic_t *v) : "memory"); } -static inline int arch_atomic_fetch_and(int i, atomic_t *v) +static __always_inline int arch_atomic_fetch_and(int i, atomic_t *v) { int val = arch_atomic_read(v); @@ -221,8 +227,9 @@ static inline int arch_atomic_fetch_and(int i, atomic_t *v) return val; } +#define arch_atomic_fetch_and arch_atomic_fetch_and -static inline void arch_atomic_or(int i, atomic_t *v) +static __always_inline void arch_atomic_or(int i, atomic_t *v) { asm volatile(LOCK_PREFIX "orl %1,%0" : "+m" (v->counter) @@ -230,7 +237,7 @@ static inline void arch_atomic_or(int i, atomic_t *v) : "memory"); } -static inline int arch_atomic_fetch_or(int i, atomic_t *v) +static __always_inline int arch_atomic_fetch_or(int i, atomic_t *v) { int val = arch_atomic_read(v); @@ -238,8 +245,9 @@ static inline int arch_atomic_fetch_or(int i, atomic_t *v) return val; } +#define arch_atomic_fetch_or arch_atomic_fetch_or -static inline void arch_atomic_xor(int i, atomic_t *v) +static __always_inline void arch_atomic_xor(int i, atomic_t *v) { asm volatile(LOCK_PREFIX "xorl %1,%0" : "+m" (v->counter) @@ -247,7 +255,7 @@ static inline void arch_atomic_xor(int i, atomic_t *v) : "memory"); } -static inline int arch_atomic_fetch_xor(int i, atomic_t *v) +static __always_inline int arch_atomic_fetch_xor(int i, atomic_t *v) { int val = arch_atomic_read(v); @@ -255,6 +263,7 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) return val; } +#define arch_atomic_fetch_xor arch_atomic_fetch_xor #ifdef CONFIG_X86_32 # include <asm/atomic64_32.h> @@ -262,6 +271,6 @@ static inline int arch_atomic_fetch_xor(int i, atomic_t *v) # include <asm/atomic64_64.h> #endif -#include <asm-generic/atomic-instrumented.h> +#define ARCH_ATOMIC #endif /* _ASM_X86_ATOMIC_H */ diff --git a/arch/x86/include/asm/atomic64_32.h b/arch/x86/include/asm/atomic64_32.h index 52cfaecb13f9..5efd01b548d1 100644 --- a/arch/x86/include/asm/atomic64_32.h +++ b/arch/x86/include/asm/atomic64_32.h @@ -75,6 +75,7 @@ static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) { return arch_cmpxchg64(&v->counter, o, n); } +#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg /** * arch_atomic64_xchg - xchg atomic64 variable @@ -94,6 +95,7 @@ static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 n) : "memory"); return o; } +#define arch_atomic64_xchg arch_atomic64_xchg /** * arch_atomic64_set - set atomic64 variable @@ -138,6 +140,7 @@ static inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) ASM_NO_INPUT_CLOBBER("memory")); return i; } +#define arch_atomic64_add_return arch_atomic64_add_return /* * Other variants with different arithmetic operators: @@ -149,6 +152,7 @@ static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) ASM_NO_INPUT_CLOBBER("memory")); return i; } +#define arch_atomic64_sub_return arch_atomic64_sub_return static inline s64 arch_atomic64_inc_return(atomic64_t *v) { @@ -242,6 +246,7 @@ static inline int arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) "S" (v) : "memory"); return (int)a; } +#define arch_atomic64_add_unless arch_atomic64_add_unless static inline int arch_atomic64_inc_not_zero(atomic64_t *v) { @@ -281,6 +286,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) return old; } +#define arch_atomic64_fetch_and arch_atomic64_fetch_and static inline void arch_atomic64_or(s64 i, atomic64_t *v) { @@ -299,6 +305,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) return old; } +#define arch_atomic64_fetch_or arch_atomic64_fetch_or static inline void arch_atomic64_xor(s64 i, atomic64_t *v) { @@ -317,6 +324,7 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) return old; } +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { @@ -327,6 +335,7 @@ static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) return old; } +#define arch_atomic64_fetch_add arch_atomic64_fetch_add #define arch_atomic64_fetch_sub(i, v) arch_atomic64_fetch_add(-(i), (v)) diff --git a/arch/x86/include/asm/atomic64_64.h b/arch/x86/include/asm/atomic64_64.h index 95c6ceac66b9..809bd010a751 100644 --- a/arch/x86/include/asm/atomic64_64.h +++ b/arch/x86/include/asm/atomic64_64.h @@ -19,7 +19,7 @@ */ static inline s64 arch_atomic64_read(const atomic64_t *v) { - return READ_ONCE((v)->counter); + return __READ_ONCE((v)->counter); } /** @@ -31,7 +31,7 @@ static inline s64 arch_atomic64_read(const atomic64_t *v) */ static inline void arch_atomic64_set(atomic64_t *v, s64 i) { - WRITE_ONCE(v->counter, i); + __WRITE_ONCE(v->counter, i); } /** @@ -159,37 +159,43 @@ static __always_inline s64 arch_atomic64_add_return(s64 i, atomic64_t *v) { return i + xadd(&v->counter, i); } +#define arch_atomic64_add_return arch_atomic64_add_return static inline s64 arch_atomic64_sub_return(s64 i, atomic64_t *v) { return arch_atomic64_add_return(-i, v); } +#define arch_atomic64_sub_return arch_atomic64_sub_return static inline s64 arch_atomic64_fetch_add(s64 i, atomic64_t *v) { return xadd(&v->counter, i); } +#define arch_atomic64_fetch_add arch_atomic64_fetch_add static inline s64 arch_atomic64_fetch_sub(s64 i, atomic64_t *v) { return xadd(&v->counter, -i); } +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub static inline s64 arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { return arch_cmpxchg(&v->counter, old, new); } +#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg -#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg static __always_inline bool arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { return try_cmpxchg(&v->counter, old, new); } +#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg static inline s64 arch_atomic64_xchg(atomic64_t *v, s64 new) { return arch_xchg(&v->counter, new); } +#define arch_atomic64_xchg arch_atomic64_xchg static inline void arch_atomic64_and(s64 i, atomic64_t *v) { @@ -207,6 +213,7 @@ static inline s64 arch_atomic64_fetch_and(s64 i, atomic64_t *v) } while (!arch_atomic64_try_cmpxchg(v, &val, val & i)); return val; } +#define arch_atomic64_fetch_and arch_atomic64_fetch_and static inline void arch_atomic64_or(s64 i, atomic64_t *v) { @@ -224,6 +231,7 @@ static inline s64 arch_atomic64_fetch_or(s64 i, atomic64_t *v) } while (!arch_atomic64_try_cmpxchg(v, &val, val | i)); return val; } +#define arch_atomic64_fetch_or arch_atomic64_fetch_or static inline void arch_atomic64_xor(s64 i, atomic64_t *v) { @@ -241,5 +249,6 @@ static inline s64 arch_atomic64_fetch_xor(s64 i, atomic64_t *v) } while (!arch_atomic64_try_cmpxchg(v, &val, val ^ i)); return val; } +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor #endif /* _ASM_X86_ATOMIC64_64_H */ diff --git a/arch/x86/include/asm/bitops.h b/arch/x86/include/asm/bitops.h index 0367efdc5b7a..35460fef39b8 100644 --- a/arch/x86/include/asm/bitops.h +++ b/arch/x86/include/asm/bitops.h @@ -201,8 +201,12 @@ arch_test_and_change_bit(long nr, volatile unsigned long *addr) return GEN_BINARY_RMWcc(LOCK_PREFIX __ASM_SIZE(btc), *addr, c, "Ir", nr); } -static __always_inline bool constant_test_bit(long nr, const volatile unsigned long *addr) +static __no_kcsan_or_inline bool constant_test_bit(long nr, const volatile unsigned long *addr) { + /* + * Because this is a plain access, we need to disable KCSAN here to + * avoid double instrumentation via instrumented bitops. + */ return ((1UL << (nr & (BITS_PER_LONG-1))) & (addr[nr >> _BITOPS_LONG_SHIFT])) != 0; } diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index facba9bc30ca..fb34ff641e0a 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -70,14 +70,17 @@ do { \ #define HAVE_ARCH_BUG #define BUG() \ do { \ + instrumentation_begin(); \ _BUG_FLAGS(ASM_UD2, 0); \ unreachable(); \ } while (0) #define __WARN_FLAGS(flags) \ do { \ + instrumentation_begin(); \ _BUG_FLAGS(ASM_UD2, BUGFLAG_WARNING|(flags)); \ annotate_reachable(); \ + instrumentation_end(); \ } while (0) #include <asm-generic/bug.h> diff --git a/arch/x86/include/asm/cpu_entry_area.h b/arch/x86/include/asm/cpu_entry_area.h index 02c0078d3787..8902fdb7de13 100644 --- a/arch/x86/include/asm/cpu_entry_area.h +++ b/arch/x86/include/asm/cpu_entry_area.h @@ -11,15 +11,11 @@ #ifdef CONFIG_X86_64 /* Macro to enforce the same ordering and stack sizes */ -#define ESTACKS_MEMBERS(guardsize, db2_holesize)\ +#define ESTACKS_MEMBERS(guardsize) \ char DF_stack_guard[guardsize]; \ char DF_stack[EXCEPTION_STKSZ]; \ char NMI_stack_guard[guardsize]; \ char NMI_stack[EXCEPTION_STKSZ]; \ - char DB2_stack_guard[guardsize]; \ - char DB2_stack[db2_holesize]; \ - char DB1_stack_guard[guardsize]; \ - char DB1_stack[EXCEPTION_STKSZ]; \ char DB_stack_guard[guardsize]; \ char DB_stack[EXCEPTION_STKSZ]; \ char MCE_stack_guard[guardsize]; \ @@ -28,12 +24,12 @@ /* The exception stacks' physical storage. No guard pages required */ struct exception_stacks { - ESTACKS_MEMBERS(0, 0) + ESTACKS_MEMBERS(0) }; /* The effective cpu entry area mapping with guard pages. */ struct cea_exception_stacks { - ESTACKS_MEMBERS(PAGE_SIZE, EXCEPTION_STKSZ) + ESTACKS_MEMBERS(PAGE_SIZE) }; /* @@ -42,8 +38,6 @@ struct cea_exception_stacks { enum exception_stack_ordering { ESTACK_DF, ESTACK_NMI, - ESTACK_DB2, - ESTACK_DB1, ESTACK_DB, ESTACK_MCE, N_EXCEPTION_STACKS diff --git a/arch/x86/include/asm/debugreg.h b/arch/x86/include/asm/debugreg.h index 1a8609a15856..e89558a3fe4a 100644 --- a/arch/x86/include/asm/debugreg.h +++ b/arch/x86/include/asm/debugreg.h @@ -18,7 +18,7 @@ DECLARE_PER_CPU(unsigned long, cpu_dr7); native_set_debugreg(register, value) #endif -static inline unsigned long native_get_debugreg(int regno) +static __always_inline unsigned long native_get_debugreg(int regno) { unsigned long val = 0; /* Damn you, gcc! */ @@ -47,7 +47,7 @@ static inline unsigned long native_get_debugreg(int regno) return val; } -static inline void native_set_debugreg(int regno, unsigned long value) +static __always_inline void native_set_debugreg(int regno, unsigned long value) { switch (regno) { case 0: @@ -85,7 +85,7 @@ static inline void hw_breakpoint_disable(void) set_debugreg(0UL, 3); } -static inline int hw_breakpoint_active(void) +static __always_inline bool hw_breakpoint_active(void) { return __this_cpu_read(cpu_dr7) & DR_GLOBAL_ENABLE_MASK; } @@ -94,24 +94,38 @@ extern void aout_dump_debugregs(struct user *dump); extern void hw_breakpoint_restore(void); -#ifdef CONFIG_X86_64 -DECLARE_PER_CPU(int, debug_stack_usage); -static inline void debug_stack_usage_inc(void) +static __always_inline unsigned long local_db_save(void) { - __this_cpu_inc(debug_stack_usage); + unsigned long dr7; + + if (static_cpu_has(X86_FEATURE_HYPERVISOR) && !hw_breakpoint_active()) + return 0; + + get_debugreg(dr7, 7); + dr7 &= ~0x400; /* architecturally set bit */ + if (dr7) + set_debugreg(0, 7); + /* + * Ensure the compiler doesn't lower the above statements into + * the critical section; disabling breakpoints late would not + * be good. + */ + barrier(); + + return dr7; } -static inline void debug_stack_usage_dec(void) + +static __always_inline void local_db_restore(unsigned long dr7) { - __this_cpu_dec(debug_stack_usage); + /* + * Ensure the compiler doesn't raise this statement into + * the critical section; enabling breakpoints early would + * not be good. + */ + barrier(); + if (dr7) + set_debugreg(dr7, 7); } -void debug_stack_set_zero(void); -void debug_stack_reset(void); -#else /* !X86_64 */ -static inline void debug_stack_set_zero(void) { } -static inline void debug_stack_reset(void) { } -static inline void debug_stack_usage_inc(void) { } -static inline void debug_stack_usage_dec(void) { } -#endif /* X86_64 */ #ifdef CONFIG_CPU_SUP_AMD extern void set_dr_addr_mask(unsigned long mask, int dr); diff --git a/arch/x86/include/asm/desc.h b/arch/x86/include/asm/desc.h index 68a99d2a5f33..1ced11d31932 100644 --- a/arch/x86/include/asm/desc.h +++ b/arch/x86/include/asm/desc.h @@ -40,11 +40,6 @@ static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *in desc->l = 0; } -extern struct desc_ptr idt_descr; -extern gate_desc idt_table[]; -extern const struct desc_ptr debug_idt_descr; -extern gate_desc debug_idt_table[]; - struct gdt_page { struct desc_struct gdt[GDT_ENTRIES]; } __attribute__((aligned(PAGE_SIZE))); @@ -214,7 +209,7 @@ static inline void native_load_gdt(const struct desc_ptr *dtr) asm volatile("lgdt %0"::"m" (*dtr)); } -static inline void native_load_idt(const struct desc_ptr *dtr) +static __always_inline void native_load_idt(const struct desc_ptr *dtr) { asm volatile("lidt %0"::"m" (*dtr)); } @@ -386,64 +381,23 @@ static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit) desc->limit1 = (limit >> 16) & 0xf; } -void update_intr_gate(unsigned int n, const void *addr); void alloc_intr_gate(unsigned int n, const void *addr); extern unsigned long system_vectors[]; -#ifdef CONFIG_X86_64 -DECLARE_PER_CPU(u32, debug_idt_ctr); -static inline bool is_debug_idt_enabled(void) -{ - if (this_cpu_read(debug_idt_ctr)) - return true; - - return false; -} - -static inline void load_debug_idt(void) -{ - load_idt((const struct desc_ptr *)&debug_idt_descr); -} -#else -static inline bool is_debug_idt_enabled(void) -{ - return false; -} - -static inline void load_debug_idt(void) -{ -} -#endif - -/* - * The load_current_idt() must be called with interrupts disabled - * to avoid races. That way the IDT will always be set back to the expected - * descriptor. It's also called when a CPU is being initialized, and - * that doesn't need to disable interrupts, as nothing should be - * bothering the CPU then. - */ -static inline void load_current_idt(void) -{ - if (is_debug_idt_enabled()) - load_debug_idt(); - else - load_idt((const struct desc_ptr *)&idt_descr); -} - +extern void load_current_idt(void); extern void idt_setup_early_handler(void); extern void idt_setup_early_traps(void); extern void idt_setup_traps(void); extern void idt_setup_apic_and_irq_gates(void); +extern bool idt_is_f00f_address(unsigned long address); #ifdef CONFIG_X86_64 extern void idt_setup_early_pf(void); extern void idt_setup_ist_traps(void); -extern void idt_setup_debugidt_traps(void); #else static inline void idt_setup_early_pf(void) { } static inline void idt_setup_ist_traps(void) { } -static inline void idt_setup_debugidt_traps(void) { } #endif extern void idt_invalidate(void *addr); diff --git a/arch/x86/include/asm/entry_arch.h b/arch/x86/include/asm/entry_arch.h deleted file mode 100644 index 416422762845..000000000000 --- a/arch/x86/include/asm/entry_arch.h +++ /dev/null @@ -1,56 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * This file is designed to contain the BUILD_INTERRUPT specifications for - * all of the extra named interrupt vectors used by the architecture. - * Usually this is the Inter Process Interrupts (IPIs) - */ - -/* - * The following vectors are part of the Linux architecture, there - * is no hardware IRQ pin equivalent for them, they are triggered - * through the ICC by us (IPIs) - */ -#ifdef CONFIG_SMP -BUILD_INTERRUPT(reschedule_interrupt,RESCHEDULE_VECTOR) -BUILD_INTERRUPT(call_function_interrupt,CALL_FUNCTION_VECTOR) -BUILD_INTERRUPT(call_function_single_interrupt,CALL_FUNCTION_SINGLE_VECTOR) -BUILD_INTERRUPT(irq_move_cleanup_interrupt, IRQ_MOVE_CLEANUP_VECTOR) -BUILD_INTERRUPT(reboot_interrupt, REBOOT_VECTOR) -#endif - -#ifdef CONFIG_HAVE_KVM -BUILD_INTERRUPT(kvm_posted_intr_ipi, POSTED_INTR_VECTOR) -BUILD_INTERRUPT(kvm_posted_intr_wakeup_ipi, POSTED_INTR_WAKEUP_VECTOR) -BUILD_INTERRUPT(kvm_posted_intr_nested_ipi, POSTED_INTR_NESTED_VECTOR) -#endif - -/* - * every pentium local APIC has two 'local interrupts', with a - * soft-definable vector attached to both interrupts, one of - * which is a timer interrupt, the other one is error counter - * overflow. Linux uses the local APIC timer interrupt to get - * a much simpler SMP time architecture: - */ -#ifdef CONFIG_X86_LOCAL_APIC - -BUILD_INTERRUPT(apic_timer_interrupt,LOCAL_TIMER_VECTOR) -BUILD_INTERRUPT(error_interrupt,ERROR_APIC_VECTOR) -BUILD_INTERRUPT(spurious_interrupt,SPURIOUS_APIC_VECTOR) -BUILD_INTERRUPT(x86_platform_ipi, X86_PLATFORM_IPI_VECTOR) - -#ifdef CONFIG_IRQ_WORK -BUILD_INTERRUPT(irq_work_interrupt, IRQ_WORK_VECTOR) -#endif - -#ifdef CONFIG_X86_THERMAL_VECTOR -BUILD_INTERRUPT(thermal_interrupt,THERMAL_APIC_VECTOR) -#endif - -#ifdef CONFIG_X86_MCE_THRESHOLD -BUILD_INTERRUPT(threshold_interrupt,THRESHOLD_APIC_VECTOR) -#endif - -#ifdef CONFIG_X86_MCE_AMD -BUILD_INTERRUPT(deferred_error_interrupt, DEFERRED_ERROR_VECTOR) -#endif -#endif diff --git a/arch/x86/include/asm/hw_irq.h b/arch/x86/include/asm/hw_irq.h index 4154bc5f6a4e..74c12437401e 100644 --- a/arch/x86/include/asm/hw_irq.h +++ b/arch/x86/include/asm/hw_irq.h @@ -28,28 +28,6 @@ #include <asm/irq.h> #include <asm/sections.h> -/* Interrupt handlers registered during init_IRQ */ -extern asmlinkage void apic_timer_interrupt(void); -extern asmlinkage void x86_platform_ipi(void); -extern asmlinkage void kvm_posted_intr_ipi(void); -extern asmlinkage void kvm_posted_intr_wakeup_ipi(void); -extern asmlinkage void kvm_posted_intr_nested_ipi(void); -extern asmlinkage void error_interrupt(void); -extern asmlinkage void irq_work_interrupt(void); -extern asmlinkage void uv_bau_message_intr1(void); - -extern asmlinkage void spurious_interrupt(void); -extern asmlinkage void thermal_interrupt(void); -extern asmlinkage void reschedule_interrupt(void); - -extern asmlinkage void irq_move_cleanup_interrupt(void); -extern asmlinkage void reboot_interrupt(void); -extern asmlinkage void threshold_interrupt(void); -extern asmlinkage void deferred_error_interrupt(void); - -extern asmlinkage void call_function_interrupt(void); -extern asmlinkage void call_function_single_interrupt(void); - #ifdef CONFIG_X86_LOCAL_APIC struct irq_data; struct pci_dev; diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h new file mode 100644 index 000000000000..cf51c50eb356 --- /dev/null +++ b/arch/x86/include/asm/idtentry.h @@ -0,0 +1,652 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_IDTENTRY_H +#define _ASM_X86_IDTENTRY_H + +/* Interrupts/Exceptions */ +#include <asm/trapnr.h> + +#ifndef __ASSEMBLY__ +#include <linux/hardirq.h> + +#include <asm/irq_stack.h> + +void idtentry_enter_user(struct pt_regs *regs); +void idtentry_exit_user(struct pt_regs *regs); + +bool idtentry_enter_cond_rcu(struct pt_regs *regs); +void idtentry_exit_cond_rcu(struct pt_regs *regs, bool rcu_exit); + +/** + * DECLARE_IDTENTRY - Declare functions for simple IDT entry points + * No error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Note: This is the C variant of DECLARE_IDTENTRY(). As the name says it + * declares the entry points for usage in C code. There is an ASM variant + * as well which is used to emit the entry stubs in entry_32/64.S. + */ +#define DECLARE_IDTENTRY(vector, func) \ + asmlinkage void asm_##func(void); \ + asmlinkage void xen_asm_##func(void); \ + __visible void func(struct pt_regs *regs) + +/** + * DEFINE_IDTENTRY - Emit code for simple IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * idtentry_enter() contains common code which has to be invoked before + * arbitrary code in the body. idtentry_exit() contains common code + * which has to run before returning to the low level assembly code. + */ +#define DEFINE_IDTENTRY(func) \ +static __always_inline void __##func(struct pt_regs *regs); \ + \ +__visible noinstr void func(struct pt_regs *regs) \ +{ \ + bool rcu_exit = idtentry_enter_cond_rcu(regs); \ + \ + instrumentation_begin(); \ + __##func (regs); \ + instrumentation_end(); \ + idtentry_exit_cond_rcu(regs, rcu_exit); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs) + +/* Special case for 32bit IRET 'trap' */ +#define DECLARE_IDTENTRY_SW DECLARE_IDTENTRY +#define DEFINE_IDTENTRY_SW DEFINE_IDTENTRY + +/** + * DECLARE_IDTENTRY_ERRORCODE - Declare functions for simple IDT entry points + * Error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Same as DECLARE_IDTENTRY, but has an extra error_code argument for the + * C-handler. + */ +#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ + asmlinkage void asm_##func(void); \ + asmlinkage void xen_asm_##func(void); \ + __visible void func(struct pt_regs *regs, unsigned long error_code) + +/** + * DEFINE_IDTENTRY_ERRORCODE - Emit code for simple IDT entry points + * Error code pushed by hardware + * @func: Function name of the entry point + * + * Same as DEFINE_IDTENTRY, but has an extra error_code argument + */ +#define DEFINE_IDTENTRY_ERRORCODE(func) \ +static __always_inline void __##func(struct pt_regs *regs, \ + unsigned long error_code); \ + \ +__visible noinstr void func(struct pt_regs *regs, \ + unsigned long error_code) \ +{ \ + bool rcu_exit = idtentry_enter_cond_rcu(regs); \ + \ + instrumentation_begin(); \ + __##func (regs, error_code); \ + instrumentation_end(); \ + idtentry_exit_cond_rcu(regs, rcu_exit); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs, \ + unsigned long error_code) + +/** + * DECLARE_IDTENTRY_RAW - Declare functions for raw IDT entry points + * No error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY(). + */ +#define DECLARE_IDTENTRY_RAW(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +/** + * DEFINE_IDTENTRY_RAW - Emit code for raw IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * Contrary to DEFINE_IDTENTRY() this does not invoke the + * idtentry_enter/exit() helpers before and after the body invocation. This + * needs to be done in the body itself if applicable. Use if extra work + * is required before the enter/exit() helpers are invoked. + */ +#define DEFINE_IDTENTRY_RAW(func) \ +__visible noinstr void func(struct pt_regs *regs) + +/** + * DECLARE_IDTENTRY_RAW_ERRORCODE - Declare functions for raw IDT entry points + * Error code pushed by hardware + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_ERRORCODE() + */ +#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ + DECLARE_IDTENTRY_ERRORCODE(vector, func) + +/** + * DEFINE_IDTENTRY_RAW_ERRORCODE - Emit code for raw IDT entry points + * @func: Function name of the entry point + * + * @func is called from ASM entry code with interrupts disabled. + * + * The macro is written so it acts as function definition. Append the + * body with a pair of curly brackets. + * + * Contrary to DEFINE_IDTENTRY_ERRORCODE() this does not invoke the + * idtentry_enter/exit() helpers before and after the body invocation. This + * needs to be done in the body itself if applicable. Use if extra work + * is required before the enter/exit() helpers are invoked. + */ +#define DEFINE_IDTENTRY_RAW_ERRORCODE(func) \ +__visible noinstr void func(struct pt_regs *regs, unsigned long error_code) + +/** + * DECLARE_IDTENTRY_IRQ - Declare functions for device interrupt IDT entry + * points (common/spurious) + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_ERRORCODE() + */ +#define DECLARE_IDTENTRY_IRQ(vector, func) \ + DECLARE_IDTENTRY_ERRORCODE(vector, func) + +/** + * DEFINE_IDTENTRY_IRQ - Emit code for device interrupt IDT entry points + * @func: Function name of the entry point + * + * The vector number is pushed by the low level entry stub and handed + * to the function as error_code argument which needs to be truncated + * to an u8 because the push is sign extending. + * + * On 64-bit idtentry_enter/exit() are invoked in the ASM entry code before + * and after switching to the interrupt stack. On 32-bit this happens in C. + * + * irq_enter/exit_rcu() are invoked before the function body and the + * KVM L1D flush request is set. + */ +#define DEFINE_IDTENTRY_IRQ(func) \ +static __always_inline void __##func(struct pt_regs *regs, u8 vector); \ + \ +__visible noinstr void func(struct pt_regs *regs, \ + unsigned long error_code) \ +{ \ + bool rcu_exit = idtentry_enter_cond_rcu(regs); \ + \ + instrumentation_begin(); \ + irq_enter_rcu(); \ + kvm_set_cpu_l1tf_flush_l1d(); \ + __##func (regs, (u8)error_code); \ + irq_exit_rcu(); \ + instrumentation_end(); \ + idtentry_exit_cond_rcu(regs, rcu_exit); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs, u8 vector) + +/** + * DECLARE_IDTENTRY_SYSVEC - Declare functions for system vector entry points + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Maps to DECLARE_IDTENTRY(). + */ +#define DECLARE_IDTENTRY_SYSVEC(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +/** + * DEFINE_IDTENTRY_SYSVEC - Emit code for system vector IDT entry points + * @func: Function name of the entry point + * + * idtentry_enter/exit() and irq_enter/exit_rcu() are invoked before the + * function body. KVM L1D flush request is set. + * + * Runs the function on the interrupt stack if the entry hit kernel mode + */ +#define DEFINE_IDTENTRY_SYSVEC(func) \ +static void __##func(struct pt_regs *regs); \ + \ +__visible noinstr void func(struct pt_regs *regs) \ +{ \ + bool rcu_exit = idtentry_enter_cond_rcu(regs); \ + \ + instrumentation_begin(); \ + irq_enter_rcu(); \ + kvm_set_cpu_l1tf_flush_l1d(); \ + run_on_irqstack_cond(__##func, regs, regs); \ + irq_exit_rcu(); \ + instrumentation_end(); \ + idtentry_exit_cond_rcu(regs, rcu_exit); \ +} \ + \ +static noinline void __##func(struct pt_regs *regs) + +/** + * DEFINE_IDTENTRY_SYSVEC_SIMPLE - Emit code for simple system vector IDT + * entry points + * @func: Function name of the entry point + * + * Runs the function on the interrupted stack. No switch to IRQ stack and + * only the minimal __irq_enter/exit() handling. + * + * Only use for 'empty' vectors like reschedule IPI and KVM posted + * interrupt vectors. + */ +#define DEFINE_IDTENTRY_SYSVEC_SIMPLE(func) \ +static __always_inline void __##func(struct pt_regs *regs); \ + \ +__visible noinstr void func(struct pt_regs *regs) \ +{ \ + bool rcu_exit = idtentry_enter_cond_rcu(regs); \ + \ + instrumentation_begin(); \ + __irq_enter_raw(); \ + kvm_set_cpu_l1tf_flush_l1d(); \ + __##func (regs); \ + __irq_exit_raw(); \ + instrumentation_end(); \ + idtentry_exit_cond_rcu(regs, rcu_exit); \ +} \ + \ +static __always_inline void __##func(struct pt_regs *regs) + +/** + * DECLARE_IDTENTRY_XENCB - Declare functions for XEN HV callback entry point + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares three functions: + * - The ASM entry point: asm_##func + * - The XEN PV trap entry point: xen_##func (maybe unused) + * - The C handler called from the ASM entry point + * + * Maps to DECLARE_IDTENTRY(). Distinct entry point to handle the 32/64-bit + * difference + */ +#define DECLARE_IDTENTRY_XENCB(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +#ifdef CONFIG_X86_64 +/** + * DECLARE_IDTENTRY_IST - Declare functions for IST handling IDT entry points + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW, but declares also the NOIST C handler + * which is called from the ASM entry point on user mode entry + */ +#define DECLARE_IDTENTRY_IST(vector, func) \ + DECLARE_IDTENTRY_RAW(vector, func); \ + __visible void noist_##func(struct pt_regs *regs) + +/** + * DEFINE_IDTENTRY_IST - Emit code for IST entry points + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW + */ +#define DEFINE_IDTENTRY_IST(func) \ + DEFINE_IDTENTRY_RAW(func) + +/** + * DEFINE_IDTENTRY_NOIST - Emit code for NOIST entry points which + * belong to a IST entry point (MCE, DB) + * @func: Function name of the entry point. Must be the same as + * the function name of the corresponding IST variant + * + * Maps to DEFINE_IDTENTRY_RAW(). + */ +#define DEFINE_IDTENTRY_NOIST(func) \ + DEFINE_IDTENTRY_RAW(noist_##func) + +/** + * DECLARE_IDTENTRY_DF - Declare functions for double fault + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Maps to DECLARE_IDTENTRY_RAW_ERRORCODE + */ +#define DECLARE_IDTENTRY_DF(vector, func) \ + DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) + +/** + * DEFINE_IDTENTRY_DF - Emit code for double fault + * @func: Function name of the entry point + * + * Maps to DEFINE_IDTENTRY_RAW_ERRORCODE + */ +#define DEFINE_IDTENTRY_DF(func) \ + DEFINE_IDTENTRY_RAW_ERRORCODE(func) + +#else /* CONFIG_X86_64 */ + +/* Maps to a regular IDTENTRY on 32bit for now */ +# define DECLARE_IDTENTRY_IST DECLARE_IDTENTRY +# define DEFINE_IDTENTRY_IST DEFINE_IDTENTRY + +/** + * DECLARE_IDTENTRY_DF - Declare functions for double fault 32bit variant + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Declares two functions: + * - The ASM entry point: asm_##func + * - The C handler called from the C shim + */ +#define DECLARE_IDTENTRY_DF(vector, func) \ + asmlinkage void asm_##func(void); \ + __visible void func(struct pt_regs *regs, \ + unsigned long error_code, \ + unsigned long address) + +/** + * DEFINE_IDTENTRY_DF - Emit code for double fault on 32bit + * @func: Function name of the entry point + * + * This is called through the doublefault shim which already provides + * cr2 in the address argument. + */ +#define DEFINE_IDTENTRY_DF(func) \ +__visible noinstr void func(struct pt_regs *regs, \ + unsigned long error_code, \ + unsigned long address) + +#endif /* !CONFIG_X86_64 */ + +/* C-Code mapping */ +#define DECLARE_IDTENTRY_MCE DECLARE_IDTENTRY_IST +#define DEFINE_IDTENTRY_MCE DEFINE_IDTENTRY_IST +#define DEFINE_IDTENTRY_MCE_USER DEFINE_IDTENTRY_NOIST + +#define DECLARE_IDTENTRY_NMI DECLARE_IDTENTRY_RAW +#define DEFINE_IDTENTRY_NMI DEFINE_IDTENTRY_RAW + +#define DECLARE_IDTENTRY_DEBUG DECLARE_IDTENTRY_IST +#define DEFINE_IDTENTRY_DEBUG DEFINE_IDTENTRY_IST +#define DEFINE_IDTENTRY_DEBUG_USER DEFINE_IDTENTRY_NOIST + +/** + * DECLARE_IDTENTRY_XEN - Declare functions for XEN redirect IDT entry points + * @vector: Vector number (ignored for C) + * @func: Function name of the entry point + * + * Used for xennmi and xendebug redirections. No DEFINE as this is all ASM + * indirection magic. + */ +#define DECLARE_IDTENTRY_XEN(vector, func) \ + asmlinkage void xen_asm_exc_xen##func(void); \ + asmlinkage void asm_exc_xen##func(void) + +#else /* !__ASSEMBLY__ */ + +/* + * The ASM variants for DECLARE_IDTENTRY*() which emit the ASM entry stubs. + */ +#define DECLARE_IDTENTRY(vector, func) \ + idtentry vector asm_##func func has_error_code=0 + +#define DECLARE_IDTENTRY_ERRORCODE(vector, func) \ + idtentry vector asm_##func func has_error_code=1 + +/* Special case for 32bit IRET 'trap'. Do not emit ASM code */ +#define DECLARE_IDTENTRY_SW(vector, func) + +#define DECLARE_IDTENTRY_RAW(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +#define DECLARE_IDTENTRY_RAW_ERRORCODE(vector, func) \ + DECLARE_IDTENTRY_ERRORCODE(vector, func) + +/* Entries for common/spurious (device) interrupts */ +#define DECLARE_IDTENTRY_IRQ(vector, func) \ + idtentry_irq vector func + +/* System vector entries */ +#define DECLARE_IDTENTRY_SYSVEC(vector, func) \ + idtentry_sysvec vector func + +#ifdef CONFIG_X86_64 +# define DECLARE_IDTENTRY_MCE(vector, func) \ + idtentry_mce_db vector asm_##func func + +# define DECLARE_IDTENTRY_DEBUG(vector, func) \ + idtentry_mce_db vector asm_##func func + +# define DECLARE_IDTENTRY_DF(vector, func) \ + idtentry_df vector asm_##func func + +# define DECLARE_IDTENTRY_XENCB(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +#else +# define DECLARE_IDTENTRY_MCE(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +# define DECLARE_IDTENTRY_DEBUG(vector, func) \ + DECLARE_IDTENTRY(vector, func) + +/* No ASM emitted for DF as this goes through a C shim */ +# define DECLARE_IDTENTRY_DF(vector, func) + +/* No ASM emitted for XEN hypervisor callback */ +# define DECLARE_IDTENTRY_XENCB(vector, func) + +#endif + +/* No ASM code emitted for NMI */ +#define DECLARE_IDTENTRY_NMI(vector, func) + +/* XEN NMI and DB wrapper */ +#define DECLARE_IDTENTRY_XEN(vector, func) \ + idtentry vector asm_exc_xen##func exc_##func has_error_code=0 + +/* + * ASM code to emit the common vector entry stubs where each stub is + * packed into 8 bytes. + * + * Note, that the 'pushq imm8' is emitted via '.byte 0x6a, vector' because + * GCC treats the local vector variable as unsigned int and would expand + * all vectors above 0x7F to a 5 byte push. The original code did an + * adjustment of the vector number to be in the signed byte range to avoid + * this. While clever it's mindboggling counterintuitive and requires the + * odd conversion back to a real vector number in the C entry points. Using + * .byte achieves the same thing and the only fixup needed in the C entry + * point is to mask off the bits above bit 7 because the push is sign + * extending. + */ + .align 8 +SYM_CODE_START(irq_entries_start) + vector=FIRST_EXTERNAL_VECTOR + pos = . + .rept (FIRST_SYSTEM_VECTOR - FIRST_EXTERNAL_VECTOR) + UNWIND_HINT_IRET_REGS + .byte 0x6a, vector + jmp asm_common_interrupt + nop + /* Ensure that the above is 8 bytes max */ + . = pos + 8 + pos=pos+8 + vector=vector+1 + .endr +SYM_CODE_END(irq_entries_start) + +#ifdef CONFIG_X86_LOCAL_APIC + .align 8 +SYM_CODE_START(spurious_entries_start) + vector=FIRST_SYSTEM_VECTOR + pos = . + .rept (NR_VECTORS - FIRST_SYSTEM_VECTOR) + UNWIND_HINT_IRET_REGS + .byte 0x6a, vector + jmp asm_spurious_interrupt + nop + /* Ensure that the above is 8 bytes max */ + . = pos + 8 + pos=pos+8 + vector=vector+1 + .endr +SYM_CODE_END(spurious_entries_start) +#endif + +#endif /* __ASSEMBLY__ */ + +/* + * The actual entry points. Note that DECLARE_IDTENTRY*() serves two + * purposes: + * - provide the function declarations when included from C-Code + * - emit the ASM stubs when included from entry_32/64.S + * + * This avoids duplicate defines and ensures that everything is consistent. + */ + +/* + * Dummy trap number so the low level ASM macro vector number checks do not + * match which results in emitting plain IDTENTRY stubs without bells and + * whistels. + */ +#define X86_TRAP_OTHER 0xFFFF + +/* Simple exception entry points. No hardware error code */ +DECLARE_IDTENTRY(X86_TRAP_DE, exc_divide_error); +DECLARE_IDTENTRY(X86_TRAP_OF, exc_overflow); +DECLARE_IDTENTRY(X86_TRAP_BR, exc_bounds); +DECLARE_IDTENTRY(X86_TRAP_NM, exc_device_not_available); +DECLARE_IDTENTRY(X86_TRAP_OLD_MF, exc_coproc_segment_overrun); +DECLARE_IDTENTRY(X86_TRAP_SPURIOUS, exc_spurious_interrupt_bug); +DECLARE_IDTENTRY(X86_TRAP_MF, exc_coprocessor_error); +DECLARE_IDTENTRY(X86_TRAP_XF, exc_simd_coprocessor_error); + +/* 32bit software IRET trap. Do not emit ASM code */ +DECLARE_IDTENTRY_SW(X86_TRAP_IRET, iret_error); + +/* Simple exception entries with error code pushed by hardware */ +DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_TS, exc_invalid_tss); +DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_NP, exc_segment_not_present); +DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_SS, exc_stack_segment); +DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_GP, exc_general_protection); +DECLARE_IDTENTRY_ERRORCODE(X86_TRAP_AC, exc_alignment_check); + +/* Raw exception entries which need extra work */ +DECLARE_IDTENTRY_RAW(X86_TRAP_UD, exc_invalid_op); +DECLARE_IDTENTRY_RAW(X86_TRAP_BP, exc_int3); +DECLARE_IDTENTRY_RAW_ERRORCODE(X86_TRAP_PF, exc_page_fault); + +#ifdef CONFIG_X86_MCE +DECLARE_IDTENTRY_MCE(X86_TRAP_MC, exc_machine_check); +#endif + +/* NMI */ +DECLARE_IDTENTRY_NMI(X86_TRAP_NMI, exc_nmi); +DECLARE_IDTENTRY_XEN(X86_TRAP_NMI, nmi); + +/* #DB */ +DECLARE_IDTENTRY_DEBUG(X86_TRAP_DB, exc_debug); +DECLARE_IDTENTRY_XEN(X86_TRAP_DB, debug); + +/* #DF */ +DECLARE_IDTENTRY_DF(X86_TRAP_DF, exc_double_fault); + +#ifdef CONFIG_XEN_PV +DECLARE_IDTENTRY_XENCB(X86_TRAP_OTHER, exc_xen_hypervisor_callback); +#endif + +/* Device interrupts common/spurious */ +DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, common_interrupt); +#ifdef CONFIG_X86_LOCAL_APIC +DECLARE_IDTENTRY_IRQ(X86_TRAP_OTHER, spurious_interrupt); +#endif + +/* System vector entry points */ +#ifdef CONFIG_X86_LOCAL_APIC +DECLARE_IDTENTRY_SYSVEC(ERROR_APIC_VECTOR, sysvec_error_interrupt); +DECLARE_IDTENTRY_SYSVEC(SPURIOUS_APIC_VECTOR, sysvec_spurious_apic_interrupt); +DECLARE_IDTENTRY_SYSVEC(LOCAL_TIMER_VECTOR, sysvec_apic_timer_interrupt); +DECLARE_IDTENTRY_SYSVEC(X86_PLATFORM_IPI_VECTOR, sysvec_x86_platform_ipi); +#endif + +#ifdef CONFIG_SMP +DECLARE_IDTENTRY(RESCHEDULE_VECTOR, sysvec_reschedule_ipi); +DECLARE_IDTENTRY_SYSVEC(IRQ_MOVE_CLEANUP_VECTOR, sysvec_irq_move_cleanup); +DECLARE_IDTENTRY_SYSVEC(REBOOT_VECTOR, sysvec_reboot); +DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_SINGLE_VECTOR, sysvec_call_function_single); +DECLARE_IDTENTRY_SYSVEC(CALL_FUNCTION_VECTOR, sysvec_call_function); +#endif + +#ifdef CONFIG_X86_LOCAL_APIC +# ifdef CONFIG_X86_UV +DECLARE_IDTENTRY_SYSVEC(UV_BAU_MESSAGE, sysvec_uv_bau_message); +# endif + +# ifdef CONFIG_X86_MCE_THRESHOLD +DECLARE_IDTENTRY_SYSVEC(THRESHOLD_APIC_VECTOR, sysvec_threshold); +# endif + +# ifdef CONFIG_X86_MCE_AMD +DECLARE_IDTENTRY_SYSVEC(DEFERRED_ERROR_VECTOR, sysvec_deferred_error); +# endif + +# ifdef CONFIG_X86_THERMAL_VECTOR +DECLARE_IDTENTRY_SYSVEC(THERMAL_APIC_VECTOR, sysvec_thermal); +# endif + +# ifdef CONFIG_IRQ_WORK +DECLARE_IDTENTRY_SYSVEC(IRQ_WORK_VECTOR, sysvec_irq_work); +# endif +#endif + +#ifdef CONFIG_HAVE_KVM +DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_VECTOR, sysvec_kvm_posted_intr_ipi); +DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_WAKEUP_VECTOR, sysvec_kvm_posted_intr_wakeup_ipi); +DECLARE_IDTENTRY_SYSVEC(POSTED_INTR_NESTED_VECTOR, sysvec_kvm_posted_intr_nested_ipi); +#endif + +#if IS_ENABLED(CONFIG_HYPERV) +DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_hyperv_callback); +DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_REENLIGHTENMENT_VECTOR, sysvec_hyperv_reenlightenment); +DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_STIMER0_VECTOR, sysvec_hyperv_stimer0); +#endif + +#if IS_ENABLED(CONFIG_ACRN_GUEST) +DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_acrn_hv_callback); +#endif + +#ifdef CONFIG_XEN_PVHVM +DECLARE_IDTENTRY_SYSVEC(HYPERVISOR_CALLBACK_VECTOR, sysvec_xen_hvm_callback); +#endif + +#undef X86_TRAP_OTHER + +#endif diff --git a/arch/x86/include/asm/intel-family.h b/arch/x86/include/asm/intel-family.h index 8f1e94f29a16..a338a6deb950 100644 --- a/arch/x86/include/asm/intel-family.h +++ b/arch/x86/include/asm/intel-family.h @@ -89,6 +89,8 @@ #define INTEL_FAM6_COMETLAKE 0xA5 #define INTEL_FAM6_COMETLAKE_L 0xA6 +#define INTEL_FAM6_SAPPHIRERAPIDS_X 0x8F + /* "Small Core" Processors (Atom) */ #define INTEL_FAM6_ATOM_BONNELL 0x1C /* Diamondville, Pineview */ diff --git a/arch/x86/include/asm/irq.h b/arch/x86/include/asm/irq.h index 72fba0eeeb30..528c8a71fe7f 100644 --- a/arch/x86/include/asm/irq.h +++ b/arch/x86/include/asm/irq.h @@ -11,6 +11,13 @@ #include <asm/apicdef.h> #include <asm/irq_vectors.h> +/* + * The irq entry code is in the noinstr section and the start/end of + * __irqentry_text is emitted via labels. Make the build fail if + * something moves a C function into the __irq_entry section. + */ +#define __irq_entry __invalid_section + static inline int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); @@ -26,17 +33,14 @@ extern void fixup_irqs(void); #ifdef CONFIG_HAVE_KVM extern void kvm_set_posted_intr_wakeup_handler(void (*handler)(void)); -extern __visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs); -extern __visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs); -extern __visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs); #endif extern void (*x86_platform_ipi_callback)(void); extern void native_init_IRQ(void); -extern void handle_irq(struct irq_desc *desc, struct pt_regs *regs); +extern void __handle_irq(struct irq_desc *desc, struct pt_regs *regs); -extern __visible void do_IRQ(struct pt_regs *regs); +extern __visible void do_IRQ(struct pt_regs *regs, unsigned long vector); extern void init_ISA_irqs(void); @@ -46,7 +50,6 @@ extern void __init init_IRQ(void); void arch_trigger_cpumask_backtrace(const struct cpumask *mask, bool exclude_self); -extern __visible void smp_x86_platform_ipi(struct pt_regs *regs); #define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace #endif diff --git a/arch/x86/include/asm/irq_regs.h b/arch/x86/include/asm/irq_regs.h deleted file mode 100644 index 187ce59aea28..000000000000 --- a/arch/x86/include/asm/irq_regs.h +++ /dev/null @@ -1,32 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -/* - * Per-cpu current frame pointer - the location of the last exception frame on - * the stack, stored in the per-cpu area. - * - * Jeremy Fitzhardinge <jeremy@goop.org> - */ -#ifndef _ASM_X86_IRQ_REGS_H -#define _ASM_X86_IRQ_REGS_H - -#include <asm/percpu.h> - -#define ARCH_HAS_OWN_IRQ_REGS - -DECLARE_PER_CPU(struct pt_regs *, irq_regs); - -static inline struct pt_regs *get_irq_regs(void) -{ - return __this_cpu_read(irq_regs); -} - -static inline struct pt_regs *set_irq_regs(struct pt_regs *new_regs) -{ - struct pt_regs *old_regs; - - old_regs = get_irq_regs(); - __this_cpu_write(irq_regs, new_regs); - - return old_regs; -} - -#endif /* _ASM_X86_IRQ_REGS_32_H */ diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h new file mode 100644 index 000000000000..4ae66f097101 --- /dev/null +++ b/arch/x86/include/asm/irq_stack.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_IRQ_STACK_H +#define _ASM_X86_IRQ_STACK_H + +#include <linux/ptrace.h> + +#include <asm/processor.h> + +#ifdef CONFIG_X86_64 +static __always_inline bool irqstack_active(void) +{ + return __this_cpu_read(irq_count) != -1; +} + +void asm_call_on_stack(void *sp, void *func, void *arg); + +static __always_inline void __run_on_irqstack(void *func, void *arg) +{ + void *tos = __this_cpu_read(hardirq_stack_ptr); + + __this_cpu_add(irq_count, 1); + asm_call_on_stack(tos - 8, func, arg); + __this_cpu_sub(irq_count, 1); +} + +#else /* CONFIG_X86_64 */ +static inline bool irqstack_active(void) { return false; } +static inline void __run_on_irqstack(void *func, void *arg) { } +#endif /* !CONFIG_X86_64 */ + +static __always_inline bool irq_needs_irq_stack(struct pt_regs *regs) +{ + if (IS_ENABLED(CONFIG_X86_32)) + return false; + if (!regs) + return !irqstack_active(); + return !user_mode(regs) && !irqstack_active(); +} + +static __always_inline void run_on_irqstack_cond(void *func, void *arg, + struct pt_regs *regs) +{ + void (*__func)(void *arg) = func; + + lockdep_assert_irqs_disabled(); + + if (irq_needs_irq_stack(regs)) + __run_on_irqstack(__func, arg); + else + __func(arg); +} + +#endif diff --git a/arch/x86/include/asm/irq_work.h b/arch/x86/include/asm/irq_work.h index 80b35e3adf03..800ffce0db29 100644 --- a/arch/x86/include/asm/irq_work.h +++ b/arch/x86/include/asm/irq_work.h @@ -10,7 +10,6 @@ static inline bool arch_irq_work_has_interrupt(void) return boot_cpu_has(X86_FEATURE_APIC); } extern void arch_irq_work_raise(void); -extern __visible void smp_irq_work_interrupt(struct pt_regs *regs); #else static inline bool arch_irq_work_has_interrupt(void) { diff --git a/arch/x86/include/asm/irqflags.h b/arch/x86/include/asm/irqflags.h index 8a0e56e1dcc9..02a0cf547d7b 100644 --- a/arch/x86/include/asm/irqflags.h +++ b/arch/x86/include/asm/irqflags.h @@ -17,7 +17,7 @@ /* Declaration required for gcc < 4.9 to prevent -Werror=missing-prototypes */ extern inline unsigned long native_save_fl(void); -extern inline unsigned long native_save_fl(void) +extern __always_inline unsigned long native_save_fl(void) { unsigned long flags; @@ -44,12 +44,12 @@ extern inline void native_restore_fl(unsigned long flags) :"memory", "cc"); } -static inline void native_irq_disable(void) +static __always_inline void native_irq_disable(void) { asm volatile("cli": : :"memory"); } -static inline void native_irq_enable(void) +static __always_inline void native_irq_enable(void) { asm volatile("sti": : :"memory"); } @@ -74,22 +74,22 @@ static inline __cpuidle void native_halt(void) #ifndef __ASSEMBLY__ #include <linux/types.h> -static inline notrace unsigned long arch_local_save_flags(void) +static __always_inline unsigned long arch_local_save_flags(void) { return native_save_fl(); } -static inline notrace void arch_local_irq_restore(unsigned long flags) +static __always_inline void arch_local_irq_restore(unsigned long flags) { native_restore_fl(flags); } -static inline notrace void arch_local_irq_disable(void) +static __always_inline void arch_local_irq_disable(void) { native_irq_disable(); } -static inline notrace void arch_local_irq_enable(void) +static __always_inline void arch_local_irq_enable(void) { native_irq_enable(); } @@ -115,7 +115,7 @@ static inline __cpuidle void halt(void) /* * For spinlocks, etc: */ -static inline notrace unsigned long arch_local_irq_save(void) +static __always_inline unsigned long arch_local_irq_save(void) { unsigned long flags = arch_local_save_flags(); arch_local_irq_disable(); @@ -159,12 +159,12 @@ static inline notrace unsigned long arch_local_irq_save(void) #endif /* CONFIG_PARAVIRT_XXL */ #ifndef __ASSEMBLY__ -static inline int arch_irqs_disabled_flags(unsigned long flags) +static __always_inline int arch_irqs_disabled_flags(unsigned long flags) { return !(flags & X86_EFLAGS_IF); } -static inline int arch_irqs_disabled(void) +static __always_inline int arch_irqs_disabled(void) { unsigned long flags = arch_local_save_flags(); @@ -172,38 +172,4 @@ static inline int arch_irqs_disabled(void) } #endif /* !__ASSEMBLY__ */ -#ifdef __ASSEMBLY__ -#ifdef CONFIG_TRACE_IRQFLAGS -# define TRACE_IRQS_ON call trace_hardirqs_on_thunk; -# define TRACE_IRQS_OFF call trace_hardirqs_off_thunk; -#else -# define TRACE_IRQS_ON -# define TRACE_IRQS_OFF -#endif -#ifdef CONFIG_DEBUG_LOCK_ALLOC -# ifdef CONFIG_X86_64 -# define LOCKDEP_SYS_EXIT call lockdep_sys_exit_thunk -# define LOCKDEP_SYS_EXIT_IRQ \ - TRACE_IRQS_ON; \ - sti; \ - call lockdep_sys_exit_thunk; \ - cli; \ - TRACE_IRQS_OFF; -# else -# define LOCKDEP_SYS_EXIT \ - pushl %eax; \ - pushl %ecx; \ - pushl %edx; \ - call lockdep_sys_exit; \ - popl %edx; \ - popl %ecx; \ - popl %eax; -# define LOCKDEP_SYS_EXIT_IRQ -# endif -#else -# define LOCKDEP_SYS_EXIT -# define LOCKDEP_SYS_EXIT_IRQ -#endif -#endif /* __ASSEMBLY__ */ - #endif diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h index 1da5858501ca..f8998e97457f 100644 --- a/arch/x86/include/asm/kvm_host.h +++ b/arch/x86/include/asm/kvm_host.h @@ -1306,7 +1306,6 @@ struct kvm_arch_async_pf { extern u64 __read_mostly host_efer; extern struct kvm_x86_ops kvm_x86_ops; -extern struct kmem_cache *x86_fpu_cache; #define __KVM_HAVE_ARCH_VM_ALLOC static inline struct kvm *kvm_arch_alloc_vm(void) @@ -1671,7 +1670,7 @@ void kvm_make_scan_ioapic_request(struct kvm *kvm); void kvm_make_scan_ioapic_request_mask(struct kvm *kvm, unsigned long *vcpu_bitmap); -void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work); diff --git a/arch/x86/include/asm/kvm_para.h b/arch/x86/include/asm/kvm_para.h index 57fd1966c4ea..49d3a9edb06f 100644 --- a/arch/x86/include/asm/kvm_para.h +++ b/arch/x86/include/asm/kvm_para.h @@ -141,7 +141,7 @@ static inline void kvm_disable_steal_time(void) return; } -static inline bool kvm_handle_async_pf(struct pt_regs *regs, u32 token) +static __always_inline bool kvm_handle_async_pf(struct pt_regs *regs, u32 token) { return false; } diff --git a/arch/x86/include/asm/mce.h b/arch/x86/include/asm/mce.h index f9cea081c05b..cf503824529c 100644 --- a/arch/x86/include/asm/mce.h +++ b/arch/x86/include/asm/mce.h @@ -127,6 +127,17 @@ #define MSR_AMD64_SMCA_MCx_DEADDR(x) (MSR_AMD64_SMCA_MC0_DEADDR + 0x10*(x)) #define MSR_AMD64_SMCA_MCx_MISCy(x, y) ((MSR_AMD64_SMCA_MC0_MISC1 + y) + (0x10*(x))) +#define XEC(x, mask) (((x) >> 16) & mask) + +/* mce.kflags flag bits for logging etc. */ +#define MCE_HANDLED_CEC BIT_ULL(0) +#define MCE_HANDLED_UC BIT_ULL(1) +#define MCE_HANDLED_EXTLOG BIT_ULL(2) +#define MCE_HANDLED_NFIT BIT_ULL(3) +#define MCE_HANDLED_EDAC BIT_ULL(4) +#define MCE_HANDLED_MCELOG BIT_ULL(5) +#define MCE_IN_KERNEL_RECOV BIT_ULL(6) + /* * This structure contains all data related to the MCE log. Also * carries a signature to make it easier to find from external @@ -142,14 +153,16 @@ struct mce_log_buffer { struct mce entry[]; }; +/* Highest last */ enum mce_notifier_prios { - MCE_PRIO_FIRST = INT_MAX, - MCE_PRIO_UC = INT_MAX - 1, - MCE_PRIO_EXTLOG = INT_MAX - 2, - MCE_PRIO_NFIT = INT_MAX - 3, - MCE_PRIO_EDAC = INT_MAX - 4, - MCE_PRIO_MCELOG = 1, - MCE_PRIO_LOWEST = 0, + MCE_PRIO_LOWEST, + MCE_PRIO_MCELOG, + MCE_PRIO_EDAC, + MCE_PRIO_NFIT, + MCE_PRIO_EXTLOG, + MCE_PRIO_UC, + MCE_PRIO_EARLY, + MCE_PRIO_CEC }; struct notifier_block; @@ -238,7 +251,7 @@ extern void mce_disable_bank(int bank); /* * Exception handler */ -void do_machine_check(struct pt_regs *, long); +void do_machine_check(struct pt_regs *pt_regs); /* * Threshold handler @@ -347,5 +360,4 @@ umc_normaddr_to_sysaddr(u64 norm_addr, u16 nid, u8 umc, u64 *sys_addr) { return #endif static inline void mce_hygon_feature_init(struct cpuinfo_x86 *c) { return mce_amd_feature_init(c); } - #endif /* _ASM_X86_MCE_H */ diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h index d30805ed323e..60b944dd2df1 100644 --- a/arch/x86/include/asm/mshyperv.h +++ b/arch/x86/include/asm/mshyperv.h @@ -54,20 +54,8 @@ typedef int (*hyperv_fill_flush_list_func)( vclocks_set_used(VDSO_CLOCKMODE_HVCLOCK); #define hv_get_raw_timer() rdtsc_ordered() -void hyperv_callback_vector(void); -void hyperv_reenlightenment_vector(void); -#ifdef CONFIG_TRACING -#define trace_hyperv_callback_vector hyperv_callback_vector -#endif void hyperv_vector_handler(struct pt_regs *regs); -/* - * Routines for stimer0 Direct Mode handling. - * On x86/x64, there are no percpu actions to take. - */ -void hv_stimer0_vector_handler(struct pt_regs *regs); -void hv_stimer0_callback_vector(void); - static inline void hv_enable_stimer0_percpu_irq(int irq) {} static inline void hv_disable_stimer0_percpu_irq(int irq) {} @@ -226,7 +214,6 @@ void hyperv_setup_mmu_ops(void); void *hv_alloc_hyperv_page(void); void *hv_alloc_hyperv_zeroed_page(void); void hv_free_hyperv_page(unsigned long addr); -void hyperv_reenlightenment_intr(struct pt_regs *regs); void set_hv_tscchange_cb(void (*cb)(void)); void clear_hv_tscchange_cb(void); void hyperv_stop_tsc_emulation(void); diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index d52d1aacdd97..e7752b4038ff 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -262,7 +262,7 @@ DECLARE_STATIC_KEY_FALSE(mds_idle_clear); * combination with microcode which triggers a CPU buffer flush when the * instruction is executed. */ -static inline void mds_clear_cpu_buffers(void) +static __always_inline void mds_clear_cpu_buffers(void) { static const u16 ds = __KERNEL_DS; @@ -283,7 +283,7 @@ static inline void mds_clear_cpu_buffers(void) * * Clear CPU buffers if the corresponding static key is enabled */ -static inline void mds_user_clear_cpu_buffers(void) +static __always_inline void mds_user_clear_cpu_buffers(void) { if (static_branch_likely(&mds_user_clear)) mds_clear_cpu_buffers(); diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h index 29ee0c088009..42cd333616c4 100644 --- a/arch/x86/include/asm/processor.h +++ b/arch/x86/include/asm/processor.h @@ -823,7 +823,7 @@ static inline void prefetch(const void *x) * Useful for spinlocks to avoid one state transition in the * cache coherency protocol: */ -static inline void prefetchw(const void *x) +static __always_inline void prefetchw(const void *x) { alternative_input(BASE_PREFETCH, "prefetchw %P1", X86_FEATURE_3DNOWPREFETCH, diff --git a/arch/x86/include/asm/ptrace.h b/arch/x86/include/asm/ptrace.h index 6d6475fdd327..ebedeab48704 100644 --- a/arch/x86/include/asm/ptrace.h +++ b/arch/x86/include/asm/ptrace.h @@ -123,7 +123,7 @@ static inline void regs_set_return_value(struct pt_regs *regs, unsigned long rc) * On x86_64, vm86 mode is mercifully nonexistent, and we don't need * the extra check. */ -static inline int user_mode(struct pt_regs *regs) +static __always_inline int user_mode(struct pt_regs *regs) { #ifdef CONFIG_X86_32 return ((regs->cs & SEGMENT_RPL_MASK) | (regs->flags & X86_VM_MASK)) >= USER_RPL; diff --git a/arch/x86/include/asm/set_memory.h b/arch/x86/include/asm/set_memory.h index ec2c0a094b5d..5948218f35c5 100644 --- a/arch/x86/include/asm/set_memory.h +++ b/arch/x86/include/asm/set_memory.h @@ -86,28 +86,35 @@ int set_direct_map_default_noflush(struct page *page); extern int kernel_set_to_readonly; #ifdef CONFIG_X86_64 -static inline int set_mce_nospec(unsigned long pfn) +/* + * Prevent speculative access to the page by either unmapping + * it (if we do not require access to any part of the page) or + * marking it uncacheable (if we want to try to retrieve data + * from non-poisoned lines in the page). + */ +static inline int set_mce_nospec(unsigned long pfn, bool unmap) { unsigned long decoy_addr; int rc; /* - * Mark the linear address as UC to make sure we don't log more - * errors because of speculative access to the page. * We would like to just call: - * set_memory_uc((unsigned long)pfn_to_kaddr(pfn), 1); + * set_memory_XX((unsigned long)pfn_to_kaddr(pfn), 1); * but doing that would radically increase the odds of a * speculative access to the poison page because we'd have * the virtual address of the kernel 1:1 mapping sitting * around in registers. * Instead we get tricky. We create a non-canonical address * that looks just like the one we want, but has bit 63 flipped. - * This relies on set_memory_uc() properly sanitizing any __pa() + * This relies on set_memory_XX() properly sanitizing any __pa() * results with __PHYSICAL_MASK or PTE_PFN_MASK. */ decoy_addr = (pfn << PAGE_SHIFT) + (PAGE_OFFSET ^ BIT(63)); - rc = set_memory_uc(decoy_addr, 1); + if (unmap) + rc = set_memory_np(decoy_addr, 1); + else + rc = set_memory_uc(decoy_addr, 1); if (rc) pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn); return rc; diff --git a/arch/x86/include/asm/special_insns.h b/arch/x86/include/asm/special_insns.h index 6d37b8fcfc77..eb8e781c4353 100644 --- a/arch/x86/include/asm/special_insns.h +++ b/arch/x86/include/asm/special_insns.h @@ -7,6 +7,7 @@ #include <asm/nops.h> #include <asm/processor-flags.h> +#include <linux/irqflags.h> #include <linux/jump_label.h> /* @@ -27,14 +28,14 @@ static inline unsigned long native_read_cr0(void) return val; } -static inline unsigned long native_read_cr2(void) +static __always_inline unsigned long native_read_cr2(void) { unsigned long val; asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } -static inline void native_write_cr2(unsigned long val) +static __always_inline void native_write_cr2(unsigned long val) { asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); } @@ -129,7 +130,16 @@ static inline void native_wbinvd(void) asm volatile("wbinvd": : :"memory"); } -extern asmlinkage void native_load_gs_index(unsigned); +extern asmlinkage void asm_load_gs_index(unsigned int selector); + +static inline void native_load_gs_index(unsigned int selector) +{ + unsigned long flags; + + local_irq_save(flags); + asm_load_gs_index(selector); + local_irq_restore(flags); +} static inline unsigned long __read_cr4(void) { @@ -150,12 +160,12 @@ static inline void write_cr0(unsigned long x) native_write_cr0(x); } -static inline unsigned long read_cr2(void) +static __always_inline unsigned long read_cr2(void) { return native_read_cr2(); } -static inline void write_cr2(unsigned long x) +static __always_inline void write_cr2(unsigned long x) { native_write_cr2(x); } @@ -186,7 +196,7 @@ static inline void wbinvd(void) #ifdef CONFIG_X86_64 -static inline void load_gs_index(unsigned selector) +static inline void load_gs_index(unsigned int selector) { native_load_gs_index(selector); } diff --git a/arch/x86/include/asm/text-patching.h b/arch/x86/include/asm/text-patching.h index 67315fa3956a..6593b42cb379 100644 --- a/arch/x86/include/asm/text-patching.h +++ b/arch/x86/include/asm/text-patching.h @@ -64,7 +64,7 @@ extern void text_poke_finish(void); #define DISP32_SIZE 4 -static inline int text_opcode_size(u8 opcode) +static __always_inline int text_opcode_size(u8 opcode) { int size = 0; @@ -118,12 +118,14 @@ extern __ro_after_init struct mm_struct *poking_mm; extern __ro_after_init unsigned long poking_addr; #ifndef CONFIG_UML_X86 -static inline void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip) +static __always_inline +void int3_emulate_jmp(struct pt_regs *regs, unsigned long ip) { regs->ip = ip; } -static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val) +static __always_inline +void int3_emulate_push(struct pt_regs *regs, unsigned long val) { /* * The int3 handler in entry_64.S adds a gap between the @@ -138,7 +140,8 @@ static inline void int3_emulate_push(struct pt_regs *regs, unsigned long val) *(unsigned long *)regs->sp = val; } -static inline void int3_emulate_call(struct pt_regs *regs, unsigned long func) +static __always_inline +void int3_emulate_call(struct pt_regs *regs, unsigned long func) { int3_emulate_push(regs, regs->ip - INT3_INSN_SIZE + CALL_INSN_SIZE); int3_emulate_jmp(regs, func); diff --git a/arch/x86/include/asm/trace/common.h b/arch/x86/include/asm/trace/common.h index 57c8da027d99..f0f9bcdb74d9 100644 --- a/arch/x86/include/asm/trace/common.h +++ b/arch/x86/include/asm/trace/common.h @@ -5,12 +5,8 @@ DECLARE_STATIC_KEY_FALSE(trace_pagefault_key); #define trace_pagefault_enabled() \ static_branch_unlikely(&trace_pagefault_key) -DECLARE_STATIC_KEY_FALSE(trace_resched_ipi_key); -#define trace_resched_ipi_enabled() \ - static_branch_unlikely(&trace_resched_ipi_key) #else static inline bool trace_pagefault_enabled(void) { return false; } -static inline bool trace_resched_ipi_enabled(void) { return false; } #endif #endif diff --git a/arch/x86/include/asm/trace/irq_vectors.h b/arch/x86/include/asm/trace/irq_vectors.h index 33b9d0f0aafe..88e7f0f3bf62 100644 --- a/arch/x86/include/asm/trace/irq_vectors.h +++ b/arch/x86/include/asm/trace/irq_vectors.h @@ -10,9 +10,6 @@ #ifdef CONFIG_X86_LOCAL_APIC -extern int trace_resched_ipi_reg(void); -extern void trace_resched_ipi_unreg(void); - DECLARE_EVENT_CLASS(x86_irq_vector, TP_PROTO(int vector), @@ -37,18 +34,6 @@ DEFINE_EVENT_FN(x86_irq_vector, name##_exit, \ TP_PROTO(int vector), \ TP_ARGS(vector), NULL, NULL); -#define DEFINE_RESCHED_IPI_EVENT(name) \ -DEFINE_EVENT_FN(x86_irq_vector, name##_entry, \ - TP_PROTO(int vector), \ - TP_ARGS(vector), \ - trace_resched_ipi_reg, \ - trace_resched_ipi_unreg); \ -DEFINE_EVENT_FN(x86_irq_vector, name##_exit, \ - TP_PROTO(int vector), \ - TP_ARGS(vector), \ - trace_resched_ipi_reg, \ - trace_resched_ipi_unreg); - /* * local_timer - called when entering/exiting a local timer interrupt * vector handler @@ -99,7 +84,7 @@ TRACE_EVENT_PERF_PERM(irq_work_exit, is_sampling_event(p_event) ? -EPERM : 0); /* * reschedule - called when entering/exiting a reschedule vector handler */ -DEFINE_RESCHED_IPI_EVENT(reschedule); +DEFINE_IRQ_VECTOR_EVENT(reschedule); /* * call_function - called when entering/exiting a call function interrupt diff --git a/arch/x86/include/asm/trapnr.h b/arch/x86/include/asm/trapnr.h new file mode 100644 index 000000000000..082f45631fa9 --- /dev/null +++ b/arch/x86/include/asm/trapnr.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_X86_TRAPNR_H +#define _ASM_X86_TRAPNR_H + +/* Interrupts/Exceptions */ + +#define X86_TRAP_DE 0 /* Divide-by-zero */ +#define X86_TRAP_DB 1 /* Debug */ +#define X86_TRAP_NMI 2 /* Non-maskable Interrupt */ +#define X86_TRAP_BP 3 /* Breakpoint */ +#define X86_TRAP_OF 4 /* Overflow */ +#define X86_TRAP_BR 5 /* Bound Range Exceeded */ +#define X86_TRAP_UD 6 /* Invalid Opcode */ +#define X86_TRAP_NM 7 /* Device Not Available */ +#define X86_TRAP_DF 8 /* Double Fault */ +#define X86_TRAP_OLD_MF 9 /* Coprocessor Segment Overrun */ +#define X86_TRAP_TS 10 /* Invalid TSS */ +#define X86_TRAP_NP 11 /* Segment Not Present */ +#define X86_TRAP_SS 12 /* Stack Segment Fault */ +#define X86_TRAP_GP 13 /* General Protection Fault */ +#define X86_TRAP_PF 14 /* Page Fault */ +#define X86_TRAP_SPURIOUS 15 /* Spurious Interrupt */ +#define X86_TRAP_MF 16 /* x87 Floating-Point Exception */ +#define X86_TRAP_AC 17 /* Alignment Check */ +#define X86_TRAP_MC 18 /* Machine Check */ +#define X86_TRAP_XF 19 /* SIMD Floating-Point Exception */ +#define X86_TRAP_VE 20 /* Virtualization Exception */ +#define X86_TRAP_CP 21 /* Control Protection Exception */ +#define X86_TRAP_IRET 32 /* IRET Exception */ + +#endif diff --git a/arch/x86/include/asm/traps.h b/arch/x86/include/asm/traps.h index 2ae904bf25e4..714b1a30e7b0 100644 --- a/arch/x86/include/asm/traps.h +++ b/arch/x86/include/asm/traps.h @@ -6,85 +6,9 @@ #include <linux/kprobes.h> #include <asm/debugreg.h> +#include <asm/idtentry.h> #include <asm/siginfo.h> /* TRAP_TRACE, ... */ -#define dotraplinkage __visible - -asmlinkage void divide_error(void); -asmlinkage void debug(void); -asmlinkage void nmi(void); -asmlinkage void int3(void); -asmlinkage void overflow(void); -asmlinkage void bounds(void); -asmlinkage void invalid_op(void); -asmlinkage void device_not_available(void); -#ifdef CONFIG_X86_64 -asmlinkage void double_fault(void); -#endif -asmlinkage void coprocessor_segment_overrun(void); -asmlinkage void invalid_TSS(void); -asmlinkage void segment_not_present(void); -asmlinkage void stack_segment(void); -asmlinkage void general_protection(void); -asmlinkage void page_fault(void); -asmlinkage void async_page_fault(void); -asmlinkage void spurious_interrupt_bug(void); -asmlinkage void coprocessor_error(void); -asmlinkage void alignment_check(void); -#ifdef CONFIG_X86_MCE -asmlinkage void machine_check(void); -#endif /* CONFIG_X86_MCE */ -asmlinkage void simd_coprocessor_error(void); - -#if defined(CONFIG_X86_64) && defined(CONFIG_XEN_PV) -asmlinkage void xen_divide_error(void); -asmlinkage void xen_xennmi(void); -asmlinkage void xen_xendebug(void); -asmlinkage void xen_int3(void); -asmlinkage void xen_overflow(void); -asmlinkage void xen_bounds(void); -asmlinkage void xen_invalid_op(void); -asmlinkage void xen_device_not_available(void); -asmlinkage void xen_double_fault(void); -asmlinkage void xen_coprocessor_segment_overrun(void); -asmlinkage void xen_invalid_TSS(void); -asmlinkage void xen_segment_not_present(void); -asmlinkage void xen_stack_segment(void); -asmlinkage void xen_general_protection(void); -asmlinkage void xen_page_fault(void); -asmlinkage void xen_spurious_interrupt_bug(void); -asmlinkage void xen_coprocessor_error(void); -asmlinkage void xen_alignment_check(void); -#ifdef CONFIG_X86_MCE -asmlinkage void xen_machine_check(void); -#endif /* CONFIG_X86_MCE */ -asmlinkage void xen_simd_coprocessor_error(void); -#endif - -dotraplinkage void do_divide_error(struct pt_regs *regs, long error_code); -dotraplinkage void do_debug(struct pt_regs *regs, long error_code); -dotraplinkage void do_nmi(struct pt_regs *regs, long error_code); -dotraplinkage void do_int3(struct pt_regs *regs, long error_code); -dotraplinkage void do_overflow(struct pt_regs *regs, long error_code); -dotraplinkage void do_bounds(struct pt_regs *regs, long error_code); -dotraplinkage void do_invalid_op(struct pt_regs *regs, long error_code); -dotraplinkage void do_device_not_available(struct pt_regs *regs, long error_code); -dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2); -dotraplinkage void do_coprocessor_segment_overrun(struct pt_regs *regs, long error_code); -dotraplinkage void do_invalid_TSS(struct pt_regs *regs, long error_code); -dotraplinkage void do_segment_not_present(struct pt_regs *regs, long error_code); -dotraplinkage void do_stack_segment(struct pt_regs *regs, long error_code); -dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code); -dotraplinkage void do_page_fault(struct pt_regs *regs, unsigned long error_code, unsigned long address); -dotraplinkage void do_spurious_interrupt_bug(struct pt_regs *regs, long error_code); -dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code); -dotraplinkage void do_alignment_check(struct pt_regs *regs, long error_code); -dotraplinkage void do_simd_coprocessor_error(struct pt_regs *regs, long error_code); -#ifdef CONFIG_X86_32 -dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code); -#endif -dotraplinkage void do_mce(struct pt_regs *regs, long error_code); - #ifdef CONFIG_X86_64 asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs); asmlinkage __visible notrace @@ -92,6 +16,11 @@ struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s); void __init trap_init(void); #endif +#ifdef CONFIG_X86_F00F_BUG +/* For handling the FOOF bug */ +void handle_invalid_op(struct pt_regs *regs); +#endif + static inline int get_si_code(unsigned long condition) { if (condition & DR_STEP) @@ -105,16 +34,6 @@ static inline int get_si_code(unsigned long condition) extern int panic_on_unrecovered_nmi; void math_emulate(struct math_emu_info *); -#ifndef CONFIG_X86_32 -asmlinkage void smp_thermal_interrupt(struct pt_regs *regs); -asmlinkage void smp_threshold_interrupt(struct pt_regs *regs); -asmlinkage void smp_deferred_error_interrupt(struct pt_regs *regs); -#endif - -void smp_apic_timer_interrupt(struct pt_regs *regs); -void smp_spurious_interrupt(struct pt_regs *regs); -void smp_error_interrupt(struct pt_regs *regs); -asmlinkage void smp_irq_move_cleanup_interrupt(void); #ifdef CONFIG_VMAP_STACK void __noreturn handle_stack_overflow(const char *message, @@ -122,31 +41,6 @@ void __noreturn handle_stack_overflow(const char *message, unsigned long fault_address); #endif -/* Interrupts/Exceptions */ -enum { - X86_TRAP_DE = 0, /* 0, Divide-by-zero */ - X86_TRAP_DB, /* 1, Debug */ - X86_TRAP_NMI, /* 2, Non-maskable Interrupt */ - X86_TRAP_BP, /* 3, Breakpoint */ - X86_TRAP_OF, /* 4, Overflow */ - X86_TRAP_BR, /* 5, Bound Range Exceeded */ - X86_TRAP_UD, /* 6, Invalid Opcode */ - X86_TRAP_NM, /* 7, Device Not Available */ - X86_TRAP_DF, /* 8, Double Fault */ - X86_TRAP_OLD_MF, /* 9, Coprocessor Segment Overrun */ - X86_TRAP_TS, /* 10, Invalid TSS */ - X86_TRAP_NP, /* 11, Segment Not Present */ - X86_TRAP_SS, /* 12, Stack Segment Fault */ - X86_TRAP_GP, /* 13, General Protection Fault */ - X86_TRAP_PF, /* 14, Page Fault */ - X86_TRAP_SPURIOUS, /* 15, Spurious Interrupt */ - X86_TRAP_MF, /* 16, x87 Floating-Point Exception */ - X86_TRAP_AC, /* 17, Alignment Check */ - X86_TRAP_MC, /* 18, Machine Check */ - X86_TRAP_XF, /* 19, SIMD Floating-Point Exception */ - X86_TRAP_IRET = 32, /* 32, IRET Exception */ -}; - /* * Page fault error code bits: * diff --git a/arch/x86/include/asm/uv/uv_bau.h b/arch/x86/include/asm/uv/uv_bau.h index 13687bf0e0a9..f1188bd47658 100644 --- a/arch/x86/include/asm/uv/uv_bau.h +++ b/arch/x86/include/asm/uv/uv_bau.h @@ -12,6 +12,8 @@ #define _ASM_X86_UV_UV_BAU_H #include <linux/bitmap.h> +#include <asm/idtentry.h> + #define BITSPERBYTE 8 /* @@ -799,12 +801,6 @@ static inline void bau_cpubits_clear(struct bau_local_cpumask *dstp, int nbits) bitmap_zero(&dstp->bits, nbits); } -extern void uv_bau_message_intr1(void); -#ifdef CONFIG_TRACING -#define trace_uv_bau_message_intr1 uv_bau_message_intr1 -#endif -extern void uv_bau_timeout_intr1(void); - struct atomic_short { short counter; }; diff --git a/arch/x86/include/asm/vdso/gettimeofday.h b/arch/x86/include/asm/vdso/gettimeofday.h index 9a6dc9b4ec99..fb81fea99093 100644 --- a/arch/x86/include/asm/vdso/gettimeofday.h +++ b/arch/x86/include/asm/vdso/gettimeofday.h @@ -271,6 +271,24 @@ static __always_inline const struct vdso_data *__arch_get_vdso_data(void) return __vdso_data; } +static inline bool arch_vdso_clocksource_ok(const struct vdso_data *vd) +{ + return true; +} +#define vdso_clocksource_ok arch_vdso_clocksource_ok + +/* + * Clocksource read value validation to handle PV and HyperV clocksources + * which can be invalidated asynchronously and indicate invalidation by + * returning U64_MAX, which can be effectively tested by checking for a + * negative value after casting it to s64. + */ +static inline bool arch_vdso_cycles_ok(u64 cycles) +{ + return (s64)cycles >= 0; +} +#define vdso_cycles_ok arch_vdso_cycles_ok + /* * x86 specific delta calculation. * diff --git a/arch/x86/include/uapi/asm/mce.h b/arch/x86/include/uapi/asm/mce.h index 955c2a2e1cf9..db9adc081c5a 100644 --- a/arch/x86/include/uapi/asm/mce.h +++ b/arch/x86/include/uapi/asm/mce.h @@ -35,6 +35,7 @@ struct mce { __u64 ipid; /* MCA_IPID MSR: only valid on SMCA systems */ __u64 ppin; /* Protected Processor Inventory Number */ __u32 microcode; /* Microcode revision */ + __u64 kflags; /* Internal kernel use */ }; #define MCE_GET_RECORD_LEN _IOR('M', 1, int) diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 8ef4369a4f06..e77261db2391 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -28,6 +28,10 @@ KASAN_SANITIZE_dumpstack_$(BITS).o := n KASAN_SANITIZE_stacktrace.o := n KASAN_SANITIZE_paravirt.o := n +# With some compiler versions the generated code results in boot hangs, caused +# by several compilation units. To be safe, disable all instrumentation. +KCSAN_SANITIZE := n + OBJECT_FILES_NON_STANDARD_test_nx.o := y OBJECT_FILES_NON_STANDARD_paravirt_patch.o := y diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index a9195ce8265d..8fd39ff74a49 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -1011,28 +1011,29 @@ struct bp_patching_desc { static struct bp_patching_desc *bp_desc; -static inline struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) +static __always_inline +struct bp_patching_desc *try_get_desc(struct bp_patching_desc **descp) { - struct bp_patching_desc *desc = READ_ONCE(*descp); /* rcu_dereference */ + struct bp_patching_desc *desc = __READ_ONCE(*descp); /* rcu_dereference */ - if (!desc || !atomic_inc_not_zero(&desc->refs)) + if (!desc || !arch_atomic_inc_not_zero(&desc->refs)) return NULL; return desc; } -static inline void put_desc(struct bp_patching_desc *desc) +static __always_inline void put_desc(struct bp_patching_desc *desc) { smp_mb__before_atomic(); - atomic_dec(&desc->refs); + arch_atomic_dec(&desc->refs); } -static inline void *text_poke_addr(struct text_poke_loc *tp) +static __always_inline void *text_poke_addr(struct text_poke_loc *tp) { return _stext + tp->rel_addr; } -static int notrace patch_cmp(const void *key, const void *elt) +static __always_inline int patch_cmp(const void *key, const void *elt) { struct text_poke_loc *tp = (struct text_poke_loc *) elt; @@ -1042,9 +1043,8 @@ static int notrace patch_cmp(const void *key, const void *elt) return 1; return 0; } -NOKPROBE_SYMBOL(patch_cmp); -int notrace poke_int3_handler(struct pt_regs *regs) +int noinstr poke_int3_handler(struct pt_regs *regs) { struct bp_patching_desc *desc; struct text_poke_loc *tp; @@ -1077,9 +1077,9 @@ int notrace poke_int3_handler(struct pt_regs *regs) * Skip the binary search if there is a single member in the vector. */ if (unlikely(desc->nr_entries > 1)) { - tp = bsearch(ip, desc->vec, desc->nr_entries, - sizeof(struct text_poke_loc), - patch_cmp); + tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, + sizeof(struct text_poke_loc), + patch_cmp); if (!tp) goto out_put; } else { @@ -1118,7 +1118,6 @@ out_put: put_desc(desc); return ret; } -NOKPROBE_SYMBOL(poke_int3_handler); #define TP_VEC_MAX (PAGE_SIZE / sizeof(struct text_poke_loc)) static struct text_poke_loc tp_vec[TP_VEC_MAX]; diff --git a/arch/x86/kernel/amd_nb.c b/arch/x86/kernel/amd_nb.c index b6b3297851f3..18f6b7c4bd79 100644 --- a/arch/x86/kernel/amd_nb.c +++ b/arch/x86/kernel/amd_nb.c @@ -18,9 +18,11 @@ #define PCI_DEVICE_ID_AMD_17H_ROOT 0x1450 #define PCI_DEVICE_ID_AMD_17H_M10H_ROOT 0x15d0 #define PCI_DEVICE_ID_AMD_17H_M30H_ROOT 0x1480 +#define PCI_DEVICE_ID_AMD_17H_M60H_ROOT 0x1630 #define PCI_DEVICE_ID_AMD_17H_DF_F4 0x1464 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F4 0x15ec #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F4 0x1494 +#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F4 0x144c #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F4 0x1444 #define PCI_DEVICE_ID_AMD_19H_DF_F4 0x1654 @@ -33,6 +35,7 @@ static const struct pci_device_id amd_root_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_ROOT) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_ROOT) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_ROOT) }, {} }; @@ -50,6 +53,7 @@ static const struct pci_device_id amd_nb_misc_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F3) }, @@ -65,6 +69,7 @@ static const struct pci_device_id amd_nb_link_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F4) }, + { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_19H_DF_F4) }, { PCI_DEVICE(PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_CNB17H_F4) }, diff --git a/arch/x86/kernel/apic/apic.c b/arch/x86/kernel/apic/apic.c index 4b1d31be50b4..e0e2f020ec02 100644 --- a/arch/x86/kernel/apic/apic.c +++ b/arch/x86/kernel/apic/apic.c @@ -1088,23 +1088,14 @@ static void local_apic_timer_interrupt(void) * [ if a single-CPU system runs an SMP kernel then we call the local * interrupt as well. Thus we cannot inline the local irq ... ] */ -__visible void __irq_entry smp_apic_timer_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_apic_timer_interrupt) { struct pt_regs *old_regs = set_irq_regs(regs); - /* - * NOTE! We'd better ACK the irq immediately, - * because timer handling can be slow. - * - * update_process_times() expects us to have done irq_enter(). - * Besides, if we don't timer interrupts ignore the global - * interrupt lock, which is the WrongThing (tm) to do. - */ - entering_ack_irq(); + ack_APIC_irq(); trace_local_timer_entry(LOCAL_TIMER_VECTOR); local_apic_timer_interrupt(); trace_local_timer_exit(LOCAL_TIMER_VECTOR); - exiting_irq(); set_irq_regs(old_regs); } @@ -2060,7 +2051,7 @@ void __init init_apic_mappings(void) unsigned int new_apicid; if (apic_validate_deadline_timer()) - pr_debug("TSC deadline timer available\n"); + pr_info("TSC deadline timer available\n"); if (x2apic_mode) { boot_cpu_physical_apicid = read_apic_id(); @@ -2120,15 +2111,21 @@ void __init register_lapic_address(unsigned long address) * Local APIC interrupts */ -/* - * This interrupt should _never_ happen with our APIC/SMP architecture +/** + * spurious_interrupt - Catch all for interrupts raised on unused vectors + * @regs: Pointer to pt_regs on stack + * @vector: The vector number + * + * This is invoked from ASM entry code to catch all interrupts which + * trigger on an entry which is routed to the common_spurious idtentry + * point. + * + * Also called from sysvec_spurious_apic_interrupt(). */ -__visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_IRQ(spurious_interrupt) { - u8 vector = ~regs->orig_ax; u32 v; - entering_irq(); trace_spurious_apic_entry(vector); inc_irq_stat(irq_spurious_count); @@ -2158,13 +2155,17 @@ __visible void __irq_entry smp_spurious_interrupt(struct pt_regs *regs) } out: trace_spurious_apic_exit(vector); - exiting_irq(); +} + +DEFINE_IDTENTRY_SYSVEC(sysvec_spurious_apic_interrupt) +{ + __spurious_interrupt(regs, SPURIOUS_APIC_VECTOR); } /* * This interrupt should never happen with our APIC/SMP architecture */ -__visible void __irq_entry smp_error_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_error_interrupt) { static const char * const error_interrupt_reason[] = { "Send CS error", /* APIC Error Bit 0 */ @@ -2178,7 +2179,6 @@ __visible void __irq_entry smp_error_interrupt(struct pt_regs *regs) }; u32 v, i = 0; - entering_irq(); trace_error_apic_entry(ERROR_APIC_VECTOR); /* First tickle the hardware, only then report what went on. -- REW */ @@ -2202,7 +2202,6 @@ __visible void __irq_entry smp_error_interrupt(struct pt_regs *regs) apic_printk(APIC_DEBUG, KERN_CONT "\n"); trace_error_apic_exit(ERROR_APIC_VECTOR); - exiting_irq(); } /** diff --git a/arch/x86/kernel/apic/msi.c b/arch/x86/kernel/apic/msi.c index 159bd0cb8548..5cbaca58af95 100644 --- a/arch/x86/kernel/apic/msi.c +++ b/arch/x86/kernel/apic/msi.c @@ -115,7 +115,8 @@ msi_set_affinity(struct irq_data *irqd, const struct cpumask *mask, bool force) * denote it as spurious which is no harm as this is a rare event * and interrupt handlers have to cope with spurious interrupts * anyway. If the vector is unused, then it is marked so it won't - * trigger the 'No irq handler for vector' warning in do_IRQ(). + * trigger the 'No irq handler for vector' warning in + * common_interrupt(). * * This requires to hold vector lock to prevent concurrent updates to * the affected vector. diff --git a/arch/x86/kernel/apic/vector.c b/arch/x86/kernel/apic/vector.c index 67768e54438b..c48be6e1f676 100644 --- a/arch/x86/kernel/apic/vector.c +++ b/arch/x86/kernel/apic/vector.c @@ -861,13 +861,13 @@ static void free_moved_vector(struct apic_chip_data *apicd) apicd->move_in_progress = 0; } -asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void) +DEFINE_IDTENTRY_SYSVEC(sysvec_irq_move_cleanup) { struct hlist_head *clhead = this_cpu_ptr(&cleanup_list); struct apic_chip_data *apicd; struct hlist_node *tmp; - entering_ack_irq(); + ack_APIC_irq(); /* Prevent vectors vanishing under us */ raw_spin_lock(&vector_lock); @@ -892,7 +892,6 @@ asmlinkage __visible void __irq_entry smp_irq_move_cleanup_interrupt(void) } raw_spin_unlock(&vector_lock); - exiting_irq(); } static void __send_cleanup_vector(struct apic_chip_data *apicd) diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index c2a47016f243..828be792231e 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -57,9 +57,6 @@ int main(void) BLANK(); #undef ENTRY - OFFSET(TSS_ist, tss_struct, x86_tss.ist); - DEFINE(DB_STACK_OFFSET, offsetof(struct cea_exception_stacks, DB_stack) - - offsetof(struct cea_exception_stacks, DB1_stack)); BLANK(); #ifdef CONFIG_STACKPROTECTOR diff --git a/arch/x86/kernel/cpu/Makefile b/arch/x86/kernel/cpu/Makefile index 7dc4ad68eb41..dba6a83bc349 100644 --- a/arch/x86/kernel/cpu/Makefile +++ b/arch/x86/kernel/cpu/Makefile @@ -13,6 +13,9 @@ endif KCOV_INSTRUMENT_common.o := n KCOV_INSTRUMENT_perf_event.o := n +# As above, instrumenting secondary CPU boot code causes boot hangs. +KCSAN_SANITIZE_common.o := n + # Make sure load_percpu_segment has no stackprotector nostackp := $(call cc-option, -fno-stack-protector) CFLAGS_common.o := $(nostackp) diff --git a/arch/x86/kernel/cpu/acrn.c b/arch/x86/kernel/cpu/acrn.c index 676022e71791..1da9b1c9a2db 100644 --- a/arch/x86/kernel/cpu/acrn.c +++ b/arch/x86/kernel/cpu/acrn.c @@ -10,10 +10,10 @@ */ #include <linux/interrupt.h> -#include <asm/acrn.h> #include <asm/apic.h> #include <asm/desc.h> #include <asm/hypervisor.h> +#include <asm/idtentry.h> #include <asm/irq_regs.h> static uint32_t __init acrn_detect(void) @@ -24,7 +24,7 @@ static uint32_t __init acrn_detect(void) static void __init acrn_init_platform(void) { /* Setup the IDT for ACRN hypervisor callback */ - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, acrn_hv_callback_vector); + alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_acrn_hv_callback); } static bool acrn_x2apic_available(void) @@ -39,7 +39,7 @@ static bool acrn_x2apic_available(void) static void (*acrn_intr_handler)(void); -__visible void __irq_entry acrn_hv_vector_handler(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_acrn_hv_callback) { struct pt_regs *old_regs = set_irq_regs(regs); @@ -50,13 +50,12 @@ __visible void __irq_entry acrn_hv_vector_handler(struct pt_regs *regs) * will block the interrupt whose vector is lower than * HYPERVISOR_CALLBACK_VECTOR. */ - entering_ack_irq(); + ack_APIC_irq(); inc_irq_stat(irq_hv_callback_count); if (acrn_intr_handler) acrn_intr_handler(); - exiting_irq(); set_irq_regs(old_regs); } diff --git a/arch/x86/kernel/cpu/bugs.c b/arch/x86/kernel/cpu/bugs.c index b6f887be440c..0b71970d2d3d 100644 --- a/arch/x86/kernel/cpu/bugs.c +++ b/arch/x86/kernel/cpu/bugs.c @@ -588,7 +588,9 @@ early_param("nospectre_v1", nospectre_v1_cmdline); static enum spectre_v2_mitigation spectre_v2_enabled __ro_after_init = SPECTRE_V2_NONE; -static enum spectre_v2_user_mitigation spectre_v2_user __ro_after_init = +static enum spectre_v2_user_mitigation spectre_v2_user_stibp __ro_after_init = + SPECTRE_V2_USER_NONE; +static enum spectre_v2_user_mitigation spectre_v2_user_ibpb __ro_after_init = SPECTRE_V2_USER_NONE; #ifdef CONFIG_RETPOLINE @@ -734,15 +736,6 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) break; } - /* - * At this point, an STIBP mode other than "off" has been set. - * If STIBP support is not being forced, check if STIBP always-on - * is preferred. - */ - if (mode != SPECTRE_V2_USER_STRICT && - boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) - mode = SPECTRE_V2_USER_STRICT_PREFERRED; - /* Initialize Indirect Branch Prediction Barrier */ if (boot_cpu_has(X86_FEATURE_IBPB)) { setup_force_cpu_cap(X86_FEATURE_USE_IBPB); @@ -765,23 +758,36 @@ spectre_v2_user_select_mitigation(enum spectre_v2_mitigation_cmd v2_cmd) pr_info("mitigation: Enabling %s Indirect Branch Prediction Barrier\n", static_key_enabled(&switch_mm_always_ibpb) ? "always-on" : "conditional"); + + spectre_v2_user_ibpb = mode; } - /* If enhanced IBRS is enabled no STIBP required */ - if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) + /* + * If enhanced IBRS is enabled or SMT impossible, STIBP is not + * required. + */ + if (!smt_possible || spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) return; /* - * If SMT is not possible or STIBP is not available clear the STIBP - * mode. + * At this point, an STIBP mode other than "off" has been set. + * If STIBP support is not being forced, check if STIBP always-on + * is preferred. */ - if (!smt_possible || !boot_cpu_has(X86_FEATURE_STIBP)) + if (mode != SPECTRE_V2_USER_STRICT && + boot_cpu_has(X86_FEATURE_AMD_STIBP_ALWAYS_ON)) + mode = SPECTRE_V2_USER_STRICT_PREFERRED; + + /* + * If STIBP is not available, clear the STIBP mode. + */ + if (!boot_cpu_has(X86_FEATURE_STIBP)) mode = SPECTRE_V2_USER_NONE; + + spectre_v2_user_stibp = mode; + set_mode: - spectre_v2_user = mode; - /* Only print the STIBP mode when SMT possible */ - if (smt_possible) - pr_info("%s\n", spectre_v2_user_strings[mode]); + pr_info("%s\n", spectre_v2_user_strings[mode]); } static const char * const spectre_v2_strings[] = { @@ -1014,7 +1020,7 @@ void cpu_bugs_smt_update(void) { mutex_lock(&spec_ctrl_mutex); - switch (spectre_v2_user) { + switch (spectre_v2_user_stibp) { case SPECTRE_V2_USER_NONE: break; case SPECTRE_V2_USER_STRICT: @@ -1257,14 +1263,19 @@ static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) { switch (ctrl) { case PR_SPEC_ENABLE: - if (spectre_v2_user == SPECTRE_V2_USER_NONE) + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && + spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) return 0; /* * Indirect branch speculation is always disabled in strict - * mode. + * mode. It can neither be enabled if it was force-disabled + * by a previous prctl call. + */ - if (spectre_v2_user == SPECTRE_V2_USER_STRICT || - spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED || + task_spec_ib_force_disable(task)) return -EPERM; task_clear_spec_ib_disable(task); task_update_spec_tif(task); @@ -1275,10 +1286,12 @@ static int ib_prctl_set(struct task_struct *task, unsigned long ctrl) * Indirect branch speculation is always allowed when * mitigation is force disabled. */ - if (spectre_v2_user == SPECTRE_V2_USER_NONE) + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && + spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) return -EPERM; - if (spectre_v2_user == SPECTRE_V2_USER_STRICT || - spectre_v2_user == SPECTRE_V2_USER_STRICT_PREFERRED) + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) return 0; task_set_spec_ib_disable(task); if (ctrl == PR_SPEC_FORCE_DISABLE) @@ -1309,7 +1322,8 @@ void arch_seccomp_spec_mitigate(struct task_struct *task) { if (ssb_mode == SPEC_STORE_BYPASS_SECCOMP) ssb_prctl_set(task, PR_SPEC_FORCE_DISABLE); - if (spectre_v2_user == SPECTRE_V2_USER_SECCOMP) + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || + spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) ib_prctl_set(task, PR_SPEC_FORCE_DISABLE); } #endif @@ -1340,22 +1354,24 @@ static int ib_prctl_get(struct task_struct *task) if (!boot_cpu_has_bug(X86_BUG_SPECTRE_V2)) return PR_SPEC_NOT_AFFECTED; - switch (spectre_v2_user) { - case SPECTRE_V2_USER_NONE: + if (spectre_v2_user_ibpb == SPECTRE_V2_USER_NONE && + spectre_v2_user_stibp == SPECTRE_V2_USER_NONE) return PR_SPEC_ENABLE; - case SPECTRE_V2_USER_PRCTL: - case SPECTRE_V2_USER_SECCOMP: + else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT || + spectre_v2_user_stibp == SPECTRE_V2_USER_STRICT_PREFERRED) + return PR_SPEC_DISABLE; + else if (spectre_v2_user_ibpb == SPECTRE_V2_USER_PRCTL || + spectre_v2_user_ibpb == SPECTRE_V2_USER_SECCOMP || + spectre_v2_user_stibp == SPECTRE_V2_USER_PRCTL || + spectre_v2_user_stibp == SPECTRE_V2_USER_SECCOMP) { if (task_spec_ib_force_disable(task)) return PR_SPEC_PRCTL | PR_SPEC_FORCE_DISABLE; if (task_spec_ib_disable(task)) return PR_SPEC_PRCTL | PR_SPEC_DISABLE; return PR_SPEC_PRCTL | PR_SPEC_ENABLE; - case SPECTRE_V2_USER_STRICT: - case SPECTRE_V2_USER_STRICT_PREFERRED: - return PR_SPEC_DISABLE; - default: + } else return PR_SPEC_NOT_AFFECTED; - } } int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which) @@ -1594,7 +1610,7 @@ static char *stibp_state(void) if (spectre_v2_enabled == SPECTRE_V2_IBRS_ENHANCED) return ""; - switch (spectre_v2_user) { + switch (spectre_v2_user_stibp) { case SPECTRE_V2_USER_NONE: return ", STIBP: disabled"; case SPECTRE_V2_USER_STRICT: diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 8be042df12c3..043d93cdcaad 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -1706,25 +1706,6 @@ void syscall_init(void) X86_EFLAGS_IOPL|X86_EFLAGS_AC|X86_EFLAGS_NT); } -DEFINE_PER_CPU(int, debug_stack_usage); -DEFINE_PER_CPU(u32, debug_idt_ctr); - -void debug_stack_set_zero(void) -{ - this_cpu_inc(debug_idt_ctr); - load_current_idt(); -} -NOKPROBE_SYMBOL(debug_stack_set_zero); - -void debug_stack_reset(void) -{ - if (WARN_ON(!this_cpu_read(debug_idt_ctr))) - return; - if (this_cpu_dec_return(debug_idt_ctr) == 0) - load_current_idt(); -} -NOKPROBE_SYMBOL(debug_stack_reset); - #else /* CONFIG_X86_64 */ DEFINE_PER_CPU(struct task_struct *, current_task) = &init_task; diff --git a/arch/x86/kernel/cpu/intel.c b/arch/x86/kernel/cpu/intel.c index 63926c94eb5f..c25a67a34bd3 100644 --- a/arch/x86/kernel/cpu/intel.c +++ b/arch/x86/kernel/cpu/intel.c @@ -1142,9 +1142,12 @@ void switch_to_sld(unsigned long tifn) static const struct x86_cpu_id split_lock_cpu_ids[] __initconst = { X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_X, 0), X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_L, 0), + X86_MATCH_INTEL_FAM6_MODEL(ICELAKE_D, 0), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT, 1), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_D, 1), X86_MATCH_INTEL_FAM6_MODEL(ATOM_TREMONT_L, 1), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE_L, 1), + X86_MATCH_INTEL_FAM6_MODEL(TIGERLAKE, 1), {} }; diff --git a/arch/x86/kernel/cpu/mce/amd.c b/arch/x86/kernel/cpu/mce/amd.c index 52de616a8065..99be063fcb1b 100644 --- a/arch/x86/kernel/cpu/mce/amd.c +++ b/arch/x86/kernel/cpu/mce/amd.c @@ -192,7 +192,12 @@ EXPORT_SYMBOL_GPL(smca_banks); static char buf_mcatype[MAX_MCATYPE_NAME_LEN]; static DEFINE_PER_CPU(struct threshold_bank **, threshold_banks); -static DEFINE_PER_CPU(unsigned int, bank_map); /* see which banks are on */ + +/* + * A list of the banks enabled on each logical CPU. Controls which respective + * descriptors to initialize later in mce_threshold_create_device(). + */ +static DEFINE_PER_CPU(unsigned int, bank_map); /* Map of banks that have more than MCA_MISC0 available. */ static DEFINE_PER_CPU(u32, smca_misc_banks_map); @@ -381,6 +386,10 @@ static void threshold_restart_bank(void *_tr) struct thresh_restart *tr = _tr; u32 hi, lo; + /* sysfs write might race against an offline operation */ + if (this_cpu_read(threshold_banks)) + return; + rdmsr(tr->b->address, lo, hi); if (tr->b->threshold_limit < (hi & THRESHOLD_MAX)) @@ -568,14 +577,19 @@ bool amd_filter_mce(struct mce *m) { enum smca_bank_types bank_type = smca_get_bank_type(m->bank); struct cpuinfo_x86 *c = &boot_cpu_data; - u8 xec = (m->status >> 16) & 0x3F; /* See Family 17h Models 10h-2Fh Erratum #1114. */ if (c->x86 == 0x17 && c->x86_model >= 0x10 && c->x86_model <= 0x2F && - bank_type == SMCA_IF && xec == 10) + bank_type == SMCA_IF && XEC(m->status, 0x3f) == 10) return true; + /* NB GART TLB error reporting is disabled by default. */ + if (c->x86 < 0x17) { + if (m->bank == 4 && XEC(m->status, 0x1f) == 0x5) + return true; + } + return false; } @@ -907,14 +921,13 @@ static void __log_error(unsigned int bank, u64 status, u64 addr, u64 misc) mce_log(&m); } -asmlinkage __visible void __irq_entry smp_deferred_error_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_deferred_error) { - entering_irq(); trace_deferred_error_apic_entry(DEFERRED_ERROR_VECTOR); inc_irq_stat(irq_deferred_error_count); deferred_error_int_vector(); trace_deferred_error_apic_exit(DEFERRED_ERROR_VECTOR); - exiting_ack_irq(); + ack_APIC_irq(); } /* @@ -1016,13 +1029,22 @@ static void log_and_reset_block(struct threshold_block *block) static void amd_threshold_interrupt(void) { struct threshold_block *first_block = NULL, *block = NULL, *tmp = NULL; + struct threshold_bank **bp = this_cpu_read(threshold_banks); unsigned int bank, cpu = smp_processor_id(); + /* + * Validate that the threshold bank has been initialized already. The + * handler is installed at boot time, but on a hotplug event the + * interrupt might fire before the data has been initialized. + */ + if (!bp) + return; + for (bank = 0; bank < this_cpu_read(mce_num_banks); ++bank) { if (!(per_cpu(bank_map, cpu) & (1 << bank))) continue; - first_block = per_cpu(threshold_banks, cpu)[bank]->blocks; + first_block = bp[bank]->blocks; if (!first_block) continue; @@ -1071,7 +1093,8 @@ store_interrupt_enable(struct threshold_block *b, const char *buf, size_t size) memset(&tr, 0, sizeof(tr)); tr.b = b; - smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); + if (smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1)) + return -ENODEV; return size; } @@ -1095,7 +1118,8 @@ store_threshold_limit(struct threshold_block *b, const char *buf, size_t size) b->threshold_limit = new; tr.b = b; - smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1); + if (smp_call_function_single(b->cpu, threshold_restart_bank, &tr, 1)) + return -ENODEV; return size; } @@ -1104,7 +1128,9 @@ static ssize_t show_error_count(struct threshold_block *b, char *buf) { u32 lo, hi; - rdmsr_on_cpu(b->cpu, b->address, &lo, &hi); + /* CPU might be offline by now */ + if (rdmsr_on_cpu(b->cpu, b->address, &lo, &hi)) + return -ENODEV; return sprintf(buf, "%u\n", ((hi & THRESHOLD_MAX) - (THRESHOLD_MAX - b->threshold_limit))); @@ -1209,10 +1235,10 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb u32 low, high; int err; - if ((bank >= per_cpu(mce_num_banks, cpu)) || (block >= NR_BLOCKS)) + if ((bank >= this_cpu_read(mce_num_banks)) || (block >= NR_BLOCKS)) return 0; - if (rdmsr_safe_on_cpu(cpu, address, &low, &high)) + if (rdmsr_safe(address, &low, &high)) return 0; if (!(high & MASK_VALID_HI)) { @@ -1247,6 +1273,7 @@ static int allocate_threshold_blocks(unsigned int cpu, struct threshold_bank *tb INIT_LIST_HEAD(&b->miscj); + /* This is safe as @tb is not visible yet */ if (tb->blocks) list_add(&b->miscj, &tb->blocks->miscj); else @@ -1267,13 +1294,12 @@ recurse: if (b) kobject_uevent(&b->kobj, KOBJ_ADD); - return err; + return 0; out_free: if (b) { - kobject_put(&b->kobj); list_del(&b->miscj); - kfree(b); + kobject_put(&b->kobj); } return err; } @@ -1302,9 +1328,10 @@ static int __threshold_add_blocks(struct threshold_bank *b) return err; } -static int threshold_create_bank(unsigned int cpu, unsigned int bank) +static int threshold_create_bank(struct threshold_bank **bp, unsigned int cpu, + unsigned int bank) { - struct device *dev = per_cpu(mce_device, cpu); + struct device *dev = this_cpu_read(mce_device); struct amd_northbridge *nb = NULL; struct threshold_bank *b = NULL; const char *name = get_name(bank, NULL); @@ -1324,7 +1351,7 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank) if (err) goto out; - per_cpu(threshold_banks, cpu)[bank] = b; + bp[bank] = b; refcount_inc(&b->cpus); err = __threshold_add_blocks(b); @@ -1339,6 +1366,7 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank) goto out; } + /* Associate the bank with the per-CPU MCE device */ b->kobj = kobject_create_and_add(name, &dev->kobj); if (!b->kobj) { err = -EINVAL; @@ -1346,6 +1374,7 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank) } if (is_shared_bank(bank)) { + b->shared = 1; refcount_set(&b->cpus, 1); /* nb is already initialized, see above */ @@ -1357,16 +1386,16 @@ static int threshold_create_bank(unsigned int cpu, unsigned int bank) err = allocate_threshold_blocks(cpu, b, bank, 0, msr_ops.misc(bank)); if (err) - goto out_free; - - per_cpu(threshold_banks, cpu)[bank] = b; + goto out_kobj; + bp[bank] = b; return 0; - out_free: +out_kobj: + kobject_put(b->kobj); +out_free: kfree(b); - - out: +out: return err; } @@ -1375,21 +1404,16 @@ static void threshold_block_release(struct kobject *kobj) kfree(to_block(kobj)); } -static void deallocate_threshold_block(unsigned int cpu, unsigned int bank) +static void deallocate_threshold_blocks(struct threshold_bank *bank) { - struct threshold_block *pos = NULL; - struct threshold_block *tmp = NULL; - struct threshold_bank *head = per_cpu(threshold_banks, cpu)[bank]; + struct threshold_block *pos, *tmp; - if (!head) - return; - - list_for_each_entry_safe(pos, tmp, &head->blocks->miscj, miscj) { + list_for_each_entry_safe(pos, tmp, &bank->blocks->miscj, miscj) { list_del(&pos->miscj); kobject_put(&pos->kobj); } - kobject_put(&head->blocks->kobj); + kobject_put(&bank->blocks->kobj); } static void __threshold_remove_blocks(struct threshold_bank *b) @@ -1403,122 +1427,102 @@ static void __threshold_remove_blocks(struct threshold_bank *b) kobject_del(&pos->kobj); } -static void threshold_remove_bank(unsigned int cpu, int bank) +static void threshold_remove_bank(struct threshold_bank *bank) { struct amd_northbridge *nb; - struct threshold_bank *b; - b = per_cpu(threshold_banks, cpu)[bank]; - if (!b) - return; + if (!bank->blocks) + goto out_free; - if (!b->blocks) - goto free_out; + if (!bank->shared) + goto out_dealloc; - if (is_shared_bank(bank)) { - if (!refcount_dec_and_test(&b->cpus)) { - __threshold_remove_blocks(b); - per_cpu(threshold_banks, cpu)[bank] = NULL; - return; - } else { - /* - * the last CPU on this node using the shared bank is - * going away, remove that bank now. - */ - nb = node_to_amd_nb(amd_get_nb_id(cpu)); - nb->bank4 = NULL; - } + if (!refcount_dec_and_test(&bank->cpus)) { + __threshold_remove_blocks(bank); + return; + } else { + /* + * The last CPU on this node using the shared bank is going + * away, remove that bank now. + */ + nb = node_to_amd_nb(amd_get_nb_id(smp_processor_id())); + nb->bank4 = NULL; } - deallocate_threshold_block(cpu, bank); +out_dealloc: + deallocate_threshold_blocks(bank); -free_out: - kobject_del(b->kobj); - kobject_put(b->kobj); - kfree(b); - per_cpu(threshold_banks, cpu)[bank] = NULL; +out_free: + kobject_put(bank->kobj); + kfree(bank); } int mce_threshold_remove_device(unsigned int cpu) { - unsigned int bank; + struct threshold_bank **bp = this_cpu_read(threshold_banks); + unsigned int bank, numbanks = this_cpu_read(mce_num_banks); - for (bank = 0; bank < per_cpu(mce_num_banks, cpu); ++bank) { - if (!(per_cpu(bank_map, cpu) & (1 << bank))) - continue; - threshold_remove_bank(cpu, bank); + if (!bp) + return 0; + + /* + * Clear the pointer before cleaning up, so that the interrupt won't + * touch anything of this. + */ + this_cpu_write(threshold_banks, NULL); + + for (bank = 0; bank < numbanks; bank++) { + if (bp[bank]) { + threshold_remove_bank(bp[bank]); + bp[bank] = NULL; + } } - kfree(per_cpu(threshold_banks, cpu)); - per_cpu(threshold_banks, cpu) = NULL; + kfree(bp); return 0; } -/* create dir/files for all valid threshold banks */ +/** + * mce_threshold_create_device - Create the per-CPU MCE threshold device + * @cpu: The plugged in CPU + * + * Create directories and files for all valid threshold banks. + * + * This is invoked from the CPU hotplug callback which was installed in + * mcheck_init_device(). The invocation happens in context of the hotplug + * thread running on @cpu. The callback is invoked on all CPUs which are + * online when the callback is installed or during a real hotplug event. + */ int mce_threshold_create_device(unsigned int cpu) { - unsigned int bank; + unsigned int numbanks, bank; struct threshold_bank **bp; - int err = 0; + int err; - bp = per_cpu(threshold_banks, cpu); + if (!mce_flags.amd_threshold) + return 0; + + bp = this_cpu_read(threshold_banks); if (bp) return 0; - bp = kcalloc(per_cpu(mce_num_banks, cpu), sizeof(struct threshold_bank *), - GFP_KERNEL); + numbanks = this_cpu_read(mce_num_banks); + bp = kcalloc(numbanks, sizeof(*bp), GFP_KERNEL); if (!bp) return -ENOMEM; - per_cpu(threshold_banks, cpu) = bp; - - for (bank = 0; bank < per_cpu(mce_num_banks, cpu); ++bank) { - if (!(per_cpu(bank_map, cpu) & (1 << bank))) + for (bank = 0; bank < numbanks; ++bank) { + if (!(this_cpu_read(bank_map) & (1 << bank))) continue; - err = threshold_create_bank(cpu, bank); + err = threshold_create_bank(bp, cpu, bank); if (err) - goto err; - } - return err; -err: - mce_threshold_remove_device(cpu); - return err; -} - -static __init int threshold_init_device(void) -{ - unsigned lcpu = 0; - - /* to hit CPUs online before the notifier is up */ - for_each_online_cpu(lcpu) { - int err = mce_threshold_create_device(lcpu); - - if (err) - return err; + goto out_err; } + this_cpu_write(threshold_banks, bp); if (thresholding_irq_en) mce_threshold_vector = amd_threshold_interrupt; - return 0; +out_err: + mce_threshold_remove_device(cpu); + return err; } -/* - * there are 3 funcs which need to be _initcalled in a logic sequence: - * 1. xen_late_init_mcelog - * 2. mcheck_init_device - * 3. threshold_init_device - * - * xen_late_init_mcelog must register xen_mce_chrdev_device before - * native mce_chrdev_device registration if running under xen platform; - * - * mcheck_init_device should be inited before threshold_init_device to - * initialize mce_device, otherwise a NULL ptr dereference will cause panic. - * - * so we use following _initcalls - * 1. device_initcall(xen_late_init_mcelog); - * 2. device_initcall_sync(mcheck_init_device); - * 3. late_initcall(threshold_init_device); - * - * when running under xen, the initcall order is 1,2,3; - * on baremetal, we skip 1 and we do only 2 and 3. - */ -late_initcall(threshold_init_device); diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index e9265e2f28c9..ce9120c4f740 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -130,7 +130,7 @@ static void (*quirk_no_way_out)(int bank, struct mce *m, struct pt_regs *regs); BLOCKING_NOTIFIER_HEAD(x86_mce_decoder_chain); /* Do initial initialization of a struct mce */ -void mce_setup(struct mce *m) +noinstr void mce_setup(struct mce *m) { memset(m, 0, sizeof(struct mce)); m->cpu = m->extcpu = smp_processor_id(); @@ -140,12 +140,12 @@ void mce_setup(struct mce *m) m->cpuid = cpuid_eax(1); m->socketid = cpu_data(m->extcpu).phys_proc_id; m->apicid = cpu_data(m->extcpu).initial_apicid; - rdmsrl(MSR_IA32_MCG_CAP, m->mcgcap); + m->mcgcap = __rdmsr(MSR_IA32_MCG_CAP); if (this_cpu_has(X86_FEATURE_INTEL_PPIN)) - rdmsrl(MSR_PPIN, m->ppin); + m->ppin = __rdmsr(MSR_PPIN); else if (this_cpu_has(X86_FEATURE_AMD_PPIN)) - rdmsrl(MSR_AMD_PPIN, m->ppin); + m->ppin = __rdmsr(MSR_AMD_PPIN); m->microcode = boot_cpu_data.microcode; } @@ -160,29 +160,17 @@ void mce_log(struct mce *m) } EXPORT_SYMBOL_GPL(mce_log); -/* - * We run the default notifier if we have only the UC, the first and the - * default notifier registered. I.e., the mandatory NUM_DEFAULT_NOTIFIERS - * notifiers registered on the chain. - */ -#define NUM_DEFAULT_NOTIFIERS 3 -static atomic_t num_notifiers; - void mce_register_decode_chain(struct notifier_block *nb) { if (WARN_ON(nb->priority > MCE_PRIO_MCELOG && nb->priority < MCE_PRIO_EDAC)) return; - atomic_inc(&num_notifiers); - blocking_notifier_chain_register(&x86_mce_decoder_chain, nb); } EXPORT_SYMBOL_GPL(mce_register_decode_chain); void mce_unregister_decode_chain(struct notifier_block *nb) { - atomic_dec(&num_notifiers); - blocking_notifier_chain_unregister(&x86_mce_decoder_chain, nb); } EXPORT_SYMBOL_GPL(mce_unregister_decode_chain); @@ -265,6 +253,7 @@ static void __print_mce(struct mce *m) } pr_cont("\n"); + /* * Note this output is parsed by external tools and old fields * should not be changed. @@ -531,6 +520,14 @@ bool mce_is_memory_error(struct mce *m) } EXPORT_SYMBOL_GPL(mce_is_memory_error); +static bool whole_page(struct mce *m) +{ + if (!mca_cfg.ser || !(m->status & MCI_STATUS_MISCV)) + return true; + + return MCI_MISC_ADDR_LSB(m->misc) >= PAGE_SHIFT; +} + bool mce_is_correctable(struct mce *m) { if (m->cpuvendor == X86_VENDOR_AMD && m->status & MCI_STATUS_DEFERRED) @@ -546,22 +543,7 @@ bool mce_is_correctable(struct mce *m) } EXPORT_SYMBOL_GPL(mce_is_correctable); -static bool cec_add_mce(struct mce *m) -{ - if (!m) - return false; - - /* We eat only correctable DRAM errors with usable addresses. */ - if (mce_is_memory_error(m) && - mce_is_correctable(m) && - mce_usable_address(m)) - if (!cec_add_elem(m->addr >> PAGE_SHIFT)) - return true; - - return false; -} - -static int mce_first_notifier(struct notifier_block *nb, unsigned long val, +static int mce_early_notifier(struct notifier_block *nb, unsigned long val, void *data) { struct mce *m = (struct mce *)data; @@ -569,9 +551,6 @@ static int mce_first_notifier(struct notifier_block *nb, unsigned long val, if (!m) return NOTIFY_DONE; - if (cec_add_mce(m)) - return NOTIFY_STOP; - /* Emit the trace record: */ trace_mce_record(m); @@ -582,9 +561,9 @@ static int mce_first_notifier(struct notifier_block *nb, unsigned long val, return NOTIFY_DONE; } -static struct notifier_block first_nb = { - .notifier_call = mce_first_notifier, - .priority = MCE_PRIO_FIRST, +static struct notifier_block early_nb = { + .notifier_call = mce_early_notifier, + .priority = MCE_PRIO_EARLY, }; static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, @@ -601,8 +580,10 @@ static int uc_decode_notifier(struct notifier_block *nb, unsigned long val, return NOTIFY_DONE; pfn = mce->addr >> PAGE_SHIFT; - if (!memory_failure(pfn, 0)) - set_mce_nospec(pfn); + if (!memory_failure(pfn, 0)) { + set_mce_nospec(pfn, whole_page(mce)); + mce->kflags |= MCE_HANDLED_UC; + } return NOTIFY_OK; } @@ -620,10 +601,8 @@ static int mce_default_notifier(struct notifier_block *nb, unsigned long val, if (!m) return NOTIFY_DONE; - if (atomic_read(&num_notifiers) > NUM_DEFAULT_NOTIFIERS) - return NOTIFY_DONE; - - __print_mce(m); + if (mca_cfg.print_all || !m->kflags) + __print_mce(m); return NOTIFY_DONE; } @@ -1100,13 +1079,15 @@ static void mce_clear_state(unsigned long *toclear) * kdump kernel establishing a new #MC handler where a broadcasted MCE * might not get handled properly. */ -static bool __mc_check_crashing_cpu(int cpu) +static noinstr bool mce_check_crashing_cpu(void) { + unsigned int cpu = smp_processor_id(); + if (cpu_is_offline(cpu) || (crashing_cpu != -1 && crashing_cpu != cpu)) { u64 mcgstatus; - mcgstatus = mce_rdmsrl(MSR_IA32_MCG_STATUS); + mcgstatus = __rdmsr(MSR_IA32_MCG_STATUS); if (boot_cpu_data.x86_vendor == X86_VENDOR_ZHAOXIN) { if (mcgstatus & MCG_STATUS_LMCES) @@ -1114,7 +1095,7 @@ static bool __mc_check_crashing_cpu(int cpu) } if (mcgstatus & MCG_STATUS_RIPV) { - mce_wrmsrl(MSR_IA32_MCG_STATUS, 0); + __wrmsr(MSR_IA32_MCG_STATUS, 0, 0); return true; } } @@ -1200,11 +1181,12 @@ static void kill_me_maybe(struct callback_head *cb) int flags = MF_ACTION_REQUIRED; pr_err("Uncorrected hardware memory error in user-access at %llx", p->mce_addr); - if (!(p->mce_status & MCG_STATUS_RIPV)) + + if (!p->mce_ripv) flags |= MF_MUST_KILL; if (!memory_failure(p->mce_addr >> PAGE_SHIFT, flags)) { - set_mce_nospec(p->mce_addr >> PAGE_SHIFT); + set_mce_nospec(p->mce_addr >> PAGE_SHIFT, p->mce_whole_page); return; } @@ -1230,12 +1212,11 @@ static void kill_me_maybe(struct callback_head *cb) * backing the user stack, tracing that reads the user stack will cause * potentially infinite recursion. */ -void noinstr do_machine_check(struct pt_regs *regs, long error_code) +void noinstr do_machine_check(struct pt_regs *regs) { DECLARE_BITMAP(valid_banks, MAX_NR_BANKS); DECLARE_BITMAP(toclear, MAX_NR_BANKS); struct mca_config *cfg = &mca_cfg; - int cpu = smp_processor_id(); struct mce m, *final; char *msg = NULL; int worst = 0; @@ -1264,11 +1245,6 @@ void noinstr do_machine_check(struct pt_regs *regs, long error_code) */ int lmce = 1; - if (__mc_check_crashing_cpu(cpu)) - return; - - nmi_enter(); - this_cpu_inc(mce_exception_count); mce_gather_info(&m, regs); @@ -1356,7 +1332,7 @@ void noinstr do_machine_check(struct pt_regs *regs, long error_code) sync_core(); if (worst != MCE_AR_SEVERITY && !kill_it) - goto out_ist; + return; /* Fault was in user mode and we need to take some action */ if ((m.cs & 3) == 3) { @@ -1364,18 +1340,27 @@ void noinstr do_machine_check(struct pt_regs *regs, long error_code) BUG_ON(!on_thread_stack() || !user_mode(regs)); current->mce_addr = m.addr; - current->mce_status = m.mcgstatus; + current->mce_ripv = !!(m.mcgstatus & MCG_STATUS_RIPV); + current->mce_whole_page = whole_page(&m); current->mce_kill_me.func = kill_me_maybe; if (kill_it) current->mce_kill_me.func = kill_me_now; task_work_add(current, ¤t->mce_kill_me, true); } else { - if (!fixup_exception(regs, X86_TRAP_MC, error_code, 0)) - mce_panic("Failed kernel mode recovery", &m, msg); + /* + * Handle an MCE which has happened in kernel space but from + * which the kernel can recover: ex_has_fault_handler() has + * already verified that the rIP at which the error happened is + * a rIP from which the kernel can recover (by jumping to + * recovery code specified in _ASM_EXTABLE_FAULT()) and the + * corresponding exception handler which would do that is the + * proper one. + */ + if (m.kflags & MCE_IN_KERNEL_RECOV) { + if (!fixup_exception(regs, X86_TRAP_MC, 0, 0)) + mce_panic("Failed kernel mode recovery", &m, msg); + } } - -out_ist: - nmi_exit(); } EXPORT_SYMBOL_GPL(do_machine_check); @@ -1765,6 +1750,7 @@ static void __mcheck_cpu_init_early(struct cpuinfo_x86 *c) mce_flags.overflow_recov = !!cpu_has(c, X86_FEATURE_OVERFLOW_RECOV); mce_flags.succor = !!cpu_has(c, X86_FEATURE_SUCCOR); mce_flags.smca = !!cpu_has(c, X86_FEATURE_SMCA); + mce_flags.amd_threshold = 1; if (mce_flags.smca) { msr_ops.ctl = smca_ctl_reg; @@ -1902,21 +1888,84 @@ bool filter_mce(struct mce *m) } /* Handle unconfigured int18 (should never happen) */ -static void unexpected_machine_check(struct pt_regs *regs, long error_code) +static noinstr void unexpected_machine_check(struct pt_regs *regs) { + instrumentation_begin(); pr_err("CPU#%d: Unexpected int18 (Machine Check)\n", smp_processor_id()); + instrumentation_end(); } /* Call the installed machine check handler for this CPU setup. */ -void (*machine_check_vector)(struct pt_regs *, long error_code) = - unexpected_machine_check; +void (*machine_check_vector)(struct pt_regs *) = unexpected_machine_check; -dotraplinkage notrace void do_mce(struct pt_regs *regs, long error_code) +static __always_inline void exc_machine_check_kernel(struct pt_regs *regs) { - machine_check_vector(regs, error_code); + /* + * Only required when from kernel mode. See + * mce_check_crashing_cpu() for details. + */ + if (machine_check_vector == do_machine_check && + mce_check_crashing_cpu()) + return; + + nmi_enter(); + /* + * The call targets are marked noinstr, but objtool can't figure + * that out because it's an indirect call. Annotate it. + */ + instrumentation_begin(); + trace_hardirqs_off_finish(); + machine_check_vector(regs); + if (regs->flags & X86_EFLAGS_IF) + trace_hardirqs_on_prepare(); + instrumentation_end(); + nmi_exit(); +} + +static __always_inline void exc_machine_check_user(struct pt_regs *regs) +{ + idtentry_enter_user(regs); + instrumentation_begin(); + machine_check_vector(regs); + instrumentation_end(); + idtentry_exit_user(regs); } -NOKPROBE_SYMBOL(do_mce); + +#ifdef CONFIG_X86_64 +/* MCE hit kernel mode */ +DEFINE_IDTENTRY_MCE(exc_machine_check) +{ + unsigned long dr7; + + dr7 = local_db_save(); + exc_machine_check_kernel(regs); + local_db_restore(dr7); +} + +/* The user mode variant. */ +DEFINE_IDTENTRY_MCE_USER(exc_machine_check) +{ + unsigned long dr7; + + dr7 = local_db_save(); + exc_machine_check_user(regs); + local_db_restore(dr7); +} +#else +/* 32bit unified entry point */ +DEFINE_IDTENTRY_MCE(exc_machine_check) +{ + unsigned long dr7; + + dr7 = local_db_save(); + if (user_mode(regs)) + exc_machine_check_user(regs); + else + exc_machine_check_kernel(regs); + local_db_restore(dr7); +} +#endif /* * Called for each booted CPU to set up machine checks. @@ -1999,6 +2048,7 @@ void mce_disable_bank(int bank) * mce=no_cmci Disables CMCI * mce=no_lmce Disables LMCE * mce=dont_log_ce Clears corrected events silently, no log created for CEs. + * mce=print_all Print all machine check logs to console * mce=ignore_ce Disables polling and CMCI, corrected events are not cleared. * mce=TOLERANCELEVEL[,monarchtimeout] (number, see above) * monarchtimeout is how long to wait for other CPUs on machine @@ -2027,6 +2077,8 @@ static int __init mcheck_enable(char *str) cfg->lmce_disabled = 1; else if (!strcmp(str, "dont_log_ce")) cfg->dont_log_ce = true; + else if (!strcmp(str, "print_all")) + cfg->print_all = true; else if (!strcmp(str, "ignore_ce")) cfg->ignore_ce = true; else if (!strcmp(str, "bootlog") || !strcmp(str, "nobootlog")) @@ -2049,7 +2101,7 @@ __setup("mce", mcheck_enable); int __init mcheck_init(void) { mcheck_intel_therm_init(); - mce_register_decode_chain(&first_nb); + mce_register_decode_chain(&early_nb); mce_register_decode_chain(&mce_uc_nb); mce_register_decode_chain(&mce_default_nb); mcheck_vendor_init_severity(); @@ -2293,6 +2345,7 @@ static ssize_t store_int_with_restart(struct device *s, static DEVICE_INT_ATTR(tolerant, 0644, mca_cfg.tolerant); static DEVICE_INT_ATTR(monarch_timeout, 0644, mca_cfg.monarch_timeout); static DEVICE_BOOL_ATTR(dont_log_ce, 0644, mca_cfg.dont_log_ce); +static DEVICE_BOOL_ATTR(print_all, 0644, mca_cfg.print_all); static struct dev_ext_attribute dev_attr_check_interval = { __ATTR(check_interval, 0644, device_show_int, store_int_with_restart), @@ -2317,6 +2370,7 @@ static struct device_attribute *mce_device_attrs[] = { #endif &dev_attr_monarch_timeout.attr, &dev_attr_dont_log_ce.attr, + &dev_attr_print_all.attr, &dev_attr_ignore_ce.attr, &dev_attr_cmci_disabled.attr, NULL @@ -2489,6 +2543,13 @@ static __init void mce_init_banks(void) } } +/* + * When running on XEN, this initcall is ordered against the XEN mcelog + * initcall: + * + * device_initcall(xen_late_init_mcelog); + * device_initcall_sync(mcheck_init_device); + */ static __init int mcheck_init_device(void) { int err; @@ -2520,6 +2581,10 @@ static __init int mcheck_init_device(void) if (err) goto err_out_mem; + /* + * Invokes mce_cpu_online() on all CPUs which are online when + * the state is installed. + */ err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "x86/mce:online", mce_cpu_online, mce_cpu_pre_down); if (err < 0) @@ -2609,7 +2674,6 @@ static int __init mcheck_late_init(void) static_branch_inc(&mcsafe_key); mcheck_debugfs_init(); - cec_init(); /* * Flush out everything that has been logged during early boot, now that diff --git a/arch/x86/kernel/cpu/mce/dev-mcelog.c b/arch/x86/kernel/cpu/mce/dev-mcelog.c index d089567a9ce8..43c466020ed5 100644 --- a/arch/x86/kernel/cpu/mce/dev-mcelog.c +++ b/arch/x86/kernel/cpu/mce/dev-mcelog.c @@ -39,6 +39,9 @@ static int dev_mce_log(struct notifier_block *nb, unsigned long val, struct mce *mce = (struct mce *)data; unsigned int entry; + if (mce->kflags & MCE_HANDLED_CEC) + return NOTIFY_DONE; + mutex_lock(&mce_chrdev_read_mutex); entry = mcelog->next; @@ -56,6 +59,7 @@ static int dev_mce_log(struct notifier_block *nb, unsigned long val, memcpy(mcelog->entry + entry, mce, sizeof(struct mce)); mcelog->entry[entry].finished = 1; + mcelog->entry[entry].kflags = 0; /* wake processes polling /dev/mcelog */ wake_up_interruptible(&mce_chrdev_wait); @@ -63,6 +67,7 @@ static int dev_mce_log(struct notifier_block *nb, unsigned long val, unlock: mutex_unlock(&mce_chrdev_read_mutex); + mce->kflags |= MCE_HANDLED_MCELOG; return NOTIFY_OK; } @@ -324,6 +329,7 @@ static const struct file_operations mce_chrdev_ops = { .write = mce_chrdev_write, .poll = mce_chrdev_poll, .unlocked_ioctl = mce_chrdev_ioctl, + .compat_ioctl = compat_ptr_ioctl, .llseek = no_llseek, }; @@ -343,7 +349,7 @@ static __init int dev_mcelog_init_device(void) if (!mcelog) return -ENOMEM; - strncpy(mcelog->signature, MCE_LOG_SIGNATURE, sizeof(mcelog->signature)); + memcpy(mcelog->signature, MCE_LOG_SIGNATURE, sizeof(mcelog->signature)); mcelog->len = mce_log_len; mcelog->recordlen = sizeof(struct mce); diff --git a/arch/x86/kernel/cpu/mce/inject.c b/arch/x86/kernel/cpu/mce/inject.c index 3413b41b8d55..0593b192eb8f 100644 --- a/arch/x86/kernel/cpu/mce/inject.c +++ b/arch/x86/kernel/cpu/mce/inject.c @@ -146,9 +146,9 @@ static void raise_exception(struct mce *m, struct pt_regs *pregs) regs.cs = m->cs; pregs = ®s; } - /* in mcheck exeception handler, irq will be disabled */ + /* do_machine_check() expects interrupts disabled -- at least */ local_irq_save(flags); - do_machine_check(pregs, 0); + do_machine_check(pregs); local_irq_restore(flags); m->finished = 0; } diff --git a/arch/x86/kernel/cpu/mce/internal.h b/arch/x86/kernel/cpu/mce/internal.h index 3b008172ad73..6473070b5da4 100644 --- a/arch/x86/kernel/cpu/mce/internal.h +++ b/arch/x86/kernel/cpu/mce/internal.h @@ -9,7 +9,7 @@ #include <asm/mce.h> /* Pointer to the installed machine check handler for this CPU setup. */ -extern void (*machine_check_vector)(struct pt_regs *, long error_code); +extern void (*machine_check_vector)(struct pt_regs *); enum severity_level { MCE_NO_SEVERITY, @@ -119,6 +119,7 @@ struct mca_config { bool dont_log_ce; bool cmci_disabled; bool ignore_ce; + bool print_all; __u64 lmce_disabled : 1, disabled : 1, @@ -148,7 +149,7 @@ struct mce_vendor_flags { * Recovery. It indicates support for data poisoning in HW and deferred * error interrupts. */ - succor : 1, + succor : 1, /* * (AMD) SMCA: This bit indicates support for Scalable MCA which expands @@ -156,9 +157,12 @@ struct mce_vendor_flags { * banks. Also, to accommodate the new banks and registers, the MCA * register space is moved to a new MSR range. */ - smca : 1, + smca : 1, - __reserved_0 : 61; + /* AMD-style error thresholding banks present. */ + amd_threshold : 1, + + __reserved_0 : 60; }; extern struct mce_vendor_flags mce_flags; diff --git a/arch/x86/kernel/cpu/mce/p5.c b/arch/x86/kernel/cpu/mce/p5.c index 5ee94aa1b766..19e90cae8e97 100644 --- a/arch/x86/kernel/cpu/mce/p5.c +++ b/arch/x86/kernel/cpu/mce/p5.c @@ -21,12 +21,11 @@ int mce_p5_enabled __read_mostly; /* Machine check handler for Pentium class Intel CPUs: */ -static void pentium_machine_check(struct pt_regs *regs, long error_code) +static noinstr void pentium_machine_check(struct pt_regs *regs) { u32 loaddr, hi, lotype; - nmi_enter(); - + instrumentation_begin(); rdmsr(MSR_IA32_P5_MC_ADDR, loaddr, hi); rdmsr(MSR_IA32_P5_MC_TYPE, lotype, hi); @@ -39,8 +38,7 @@ static void pentium_machine_check(struct pt_regs *regs, long error_code) } add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); - - nmi_exit(); + instrumentation_end(); } /* Set up machine check reporting for processors with Intel style MCE: */ diff --git a/arch/x86/kernel/cpu/mce/severity.c b/arch/x86/kernel/cpu/mce/severity.c index 87bcdc6dc2f0..e1da619add19 100644 --- a/arch/x86/kernel/cpu/mce/severity.c +++ b/arch/x86/kernel/cpu/mce/severity.c @@ -213,8 +213,12 @@ static int error_context(struct mce *m) { if ((m->cs & 3) == 3) return IN_USER; - if (mc_recoverable(m->mcgstatus) && ex_has_fault_handler(m->ip)) + + if (mc_recoverable(m->mcgstatus) && ex_has_fault_handler(m->ip)) { + m->kflags |= MCE_IN_KERNEL_RECOV; return IN_KERNEL_RECOV; + } + return IN_KERNEL; } diff --git a/arch/x86/kernel/cpu/mce/therm_throt.c b/arch/x86/kernel/cpu/mce/therm_throt.c index f36dc0742085..a7cd2d203ced 100644 --- a/arch/x86/kernel/cpu/mce/therm_throt.c +++ b/arch/x86/kernel/cpu/mce/therm_throt.c @@ -614,14 +614,13 @@ static void unexpected_thermal_interrupt(void) static void (*smp_thermal_vector)(void) = unexpected_thermal_interrupt; -asmlinkage __visible void __irq_entry smp_thermal_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_thermal) { - entering_irq(); trace_thermal_apic_entry(THERMAL_APIC_VECTOR); inc_irq_stat(irq_thermal_count); smp_thermal_vector(); trace_thermal_apic_exit(THERMAL_APIC_VECTOR); - exiting_ack_irq(); + ack_APIC_irq(); } /* Thermal monitoring depends on APIC, ACPI and clock modulation */ diff --git a/arch/x86/kernel/cpu/mce/threshold.c b/arch/x86/kernel/cpu/mce/threshold.c index 28812cc15300..6a059a035021 100644 --- a/arch/x86/kernel/cpu/mce/threshold.c +++ b/arch/x86/kernel/cpu/mce/threshold.c @@ -21,12 +21,11 @@ static void default_threshold_interrupt(void) void (*mce_threshold_vector)(void) = default_threshold_interrupt; -asmlinkage __visible void __irq_entry smp_threshold_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_threshold) { - entering_irq(); trace_threshold_apic_entry(THRESHOLD_APIC_VECTOR); inc_irq_stat(irq_threshold_count); mce_threshold_vector(); trace_threshold_apic_exit(THRESHOLD_APIC_VECTOR); - exiting_ack_irq(); + ack_APIC_irq(); } diff --git a/arch/x86/kernel/cpu/mce/winchip.c b/arch/x86/kernel/cpu/mce/winchip.c index b3938c195365..9c9f0abd2d7f 100644 --- a/arch/x86/kernel/cpu/mce/winchip.c +++ b/arch/x86/kernel/cpu/mce/winchip.c @@ -17,14 +17,12 @@ #include "internal.h" /* Machine check handler for WinChip C6: */ -static void winchip_machine_check(struct pt_regs *regs, long error_code) +static noinstr void winchip_machine_check(struct pt_regs *regs) { - nmi_enter(); - + instrumentation_begin(); pr_emerg("CPU0: Machine Check Exception.\n"); add_taint(TAINT_MACHINE_CHECK, LOCKDEP_NOW_UNRELIABLE); - - nmi_exit(); + instrumentation_end(); } /* Set up machine check reporting on the Winchip C6 series */ diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c index ebf34c7bc8bc..af94f05a5c66 100644 --- a/arch/x86/kernel/cpu/mshyperv.c +++ b/arch/x86/kernel/cpu/mshyperv.c @@ -23,6 +23,7 @@ #include <asm/hyperv-tlfs.h> #include <asm/mshyperv.h> #include <asm/desc.h> +#include <asm/idtentry.h> #include <asm/irq_regs.h> #include <asm/i8259.h> #include <asm/apic.h> @@ -40,11 +41,10 @@ static void (*hv_stimer0_handler)(void); static void (*hv_kexec_handler)(void); static void (*hv_crash_handler)(struct pt_regs *regs); -__visible void __irq_entry hyperv_vector_handler(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_callback) { struct pt_regs *old_regs = set_irq_regs(regs); - entering_irq(); inc_irq_stat(irq_hv_callback_count); if (vmbus_handler) vmbus_handler(); @@ -52,7 +52,6 @@ __visible void __irq_entry hyperv_vector_handler(struct pt_regs *regs) if (ms_hyperv.hints & HV_DEPRECATING_AEOI_RECOMMENDED) ack_APIC_irq(); - exiting_irq(); set_irq_regs(old_regs); } @@ -73,19 +72,16 @@ EXPORT_SYMBOL_GPL(hv_remove_vmbus_irq); * Routines to do per-architecture handling of stimer0 * interrupts when in Direct Mode */ - -__visible void __irq_entry hv_stimer0_vector_handler(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_hyperv_stimer0) { struct pt_regs *old_regs = set_irq_regs(regs); - entering_irq(); inc_irq_stat(hyperv_stimer0_count); if (hv_stimer0_handler) hv_stimer0_handler(); add_interrupt_randomness(HYPERV_STIMER0_VECTOR, 0); ack_APIC_irq(); - exiting_irq(); set_irq_regs(old_regs); } @@ -331,17 +327,19 @@ static void __init ms_hyperv_init_platform(void) x86_platform.apic_post_init = hyperv_init; hyperv_setup_mmu_ops(); /* Setup the IDT for hypervisor callback */ - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, hyperv_callback_vector); + alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_hyperv_callback); /* Setup the IDT for reenlightenment notifications */ - if (ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT) + if (ms_hyperv.features & HV_X64_ACCESS_REENLIGHTENMENT) { alloc_intr_gate(HYPERV_REENLIGHTENMENT_VECTOR, - hyperv_reenlightenment_vector); + asm_sysvec_hyperv_reenlightenment); + } /* Setup the IDT for stimer0 */ - if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE) + if (ms_hyperv.misc_features & HV_STIMER_DIRECT_MODE_AVAILABLE) { alloc_intr_gate(HYPERV_STIMER0_VECTOR, - hv_stimer0_callback_vector); + asm_sysvec_hyperv_stimer0); + } # ifdef CONFIG_SMP smp_ops.smp_prepare_boot_cpu = hv_smp_prepare_boot_cpu; diff --git a/arch/x86/kernel/doublefault_32.c b/arch/x86/kernel/doublefault_32.c index 2ccc57f152a4..759d392cbe9f 100644 --- a/arch/x86/kernel/doublefault_32.c +++ b/arch/x86/kernel/doublefault_32.c @@ -10,7 +10,6 @@ #include <asm/desc.h> #include <asm/traps.h> -extern void double_fault(void); #define ptr_ok(x) ((x) > PAGE_OFFSET && (x) < PAGE_OFFSET + MAXMEM) #define TSS(x) this_cpu_read(cpu_tss_rw.x86_tss.x) @@ -21,7 +20,7 @@ static void set_df_gdt_entry(unsigned int cpu); * Called by double_fault with CR0.TS and EFLAGS.NT cleared. The CPU thinks * we're running the doublefault task. Cannot return. */ -asmlinkage notrace void __noreturn doublefault_shim(void) +asmlinkage noinstr void __noreturn doublefault_shim(void) { unsigned long cr2; struct pt_regs regs; @@ -40,7 +39,7 @@ asmlinkage notrace void __noreturn doublefault_shim(void) * Fill in pt_regs. A downside of doing this in C is that the unwinder * won't see it (no ENCODE_FRAME_POINTER), so a nested stack dump * won't successfully unwind to the source of the double fault. - * The main dump from do_double_fault() is fine, though, since it + * The main dump from exc_double_fault() is fine, though, since it * uses these regs directly. * * If anyone ever cares, this could be moved to asm. @@ -70,7 +69,7 @@ asmlinkage notrace void __noreturn doublefault_shim(void) regs.cx = TSS(cx); regs.bx = TSS(bx); - do_double_fault(®s, 0, cr2); + exc_double_fault(®s, 0, cr2); /* * x86_32 does not save the original CR3 anywhere on a task switch. @@ -84,7 +83,6 @@ asmlinkage notrace void __noreturn doublefault_shim(void) */ panic("cannot return from double fault\n"); } -NOKPROBE_SYMBOL(doublefault_shim); DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = { .tss = { @@ -95,7 +93,7 @@ DEFINE_PER_CPU_PAGE_ALIGNED(struct doublefault_stack, doublefault_stack) = { .ldt = 0, .io_bitmap_base = IO_BITMAP_OFFSET_INVALID, - .ip = (unsigned long) double_fault, + .ip = (unsigned long) asm_exc_double_fault, .flags = X86_EFLAGS_FIXED, .es = __USER_DS, .cs = __KERNEL_CS, diff --git a/arch/x86/kernel/dumpstack_64.c b/arch/x86/kernel/dumpstack_64.c index 460ae7f66818..4a94d38cd141 100644 --- a/arch/x86/kernel/dumpstack_64.c +++ b/arch/x86/kernel/dumpstack_64.c @@ -22,15 +22,13 @@ static const char * const exception_stack_names[] = { [ ESTACK_DF ] = "#DF", [ ESTACK_NMI ] = "NMI", - [ ESTACK_DB2 ] = "#DB2", - [ ESTACK_DB1 ] = "#DB1", [ ESTACK_DB ] = "#DB", [ ESTACK_MCE ] = "#MC", }; const char *stack_type_name(enum stack_type type) { - BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 4); if (type == STACK_TYPE_IRQ) return "IRQ"; @@ -79,7 +77,6 @@ static const struct estack_pages estack_pages[CEA_ESTACK_PAGES] ____cacheline_aligned = { EPAGERANGE(DF), EPAGERANGE(NMI), - EPAGERANGE(DB1), EPAGERANGE(DB), EPAGERANGE(MCE), }; @@ -91,7 +88,7 @@ static bool in_exception_stack(unsigned long *stack, struct stack_info *info) struct pt_regs *regs; unsigned int k; - BUILD_BUG_ON(N_EXCEPTION_STACKS != 6); + BUILD_BUG_ON(N_EXCEPTION_STACKS != 4); begin = (unsigned long)__this_cpu_read(cea_exception_stacks); /* diff --git a/arch/x86/kernel/e820.c b/arch/x86/kernel/e820.c index 4d13c57f370a..983cd53ed4c9 100644 --- a/arch/x86/kernel/e820.c +++ b/arch/x86/kernel/e820.c @@ -991,7 +991,15 @@ void __init e820__reserve_setup_data(void) while (pa_data) { data = early_memremap(pa_data, sizeof(*data)); e820__range_update(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); - e820__range_update_kexec(pa_data, sizeof(*data)+data->len, E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); + + /* + * SETUP_EFI is supplied by kexec and does not need to be + * reserved. + */ + if (data->type != SETUP_EFI) + e820__range_update_kexec(pa_data, + sizeof(*data) + data->len, + E820_TYPE_RAM, E820_TYPE_RESERVED_KERN); if (data->type == SETUP_INDIRECT && ((struct setup_indirect *)data->data)->type != SETUP_INDIRECT) { diff --git a/arch/x86/kernel/ftrace_64.S b/arch/x86/kernel/ftrace_64.S index aa5d28aeb31e..083a3da7bb73 100644 --- a/arch/x86/kernel/ftrace_64.S +++ b/arch/x86/kernel/ftrace_64.S @@ -12,7 +12,7 @@ #include <asm/frame.h> .code64 - .section .entry.text, "ax" + .section .text, "ax" #ifdef CONFIG_FRAME_POINTER /* Save parent and function stack frames (rip and rbp) */ diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S index 4fc33fdf0f16..16da4ac01597 100644 --- a/arch/x86/kernel/head_64.S +++ b/arch/x86/kernel/head_64.S @@ -29,15 +29,16 @@ #ifdef CONFIG_PARAVIRT_XXL #include <asm/asm-offsets.h> #include <asm/paravirt.h> +#define GET_CR2_INTO(reg) GET_CR2_INTO_AX ; _ASM_MOV %_ASM_AX, reg #else #define INTERRUPT_RETURN iretq +#define GET_CR2_INTO(reg) _ASM_MOV %cr2, reg #endif -/* we are not able to switch in one step to the final KERNEL ADDRESS SPACE +/* + * We are not able to switch in one step to the final KERNEL ADDRESS SPACE * because we need identity-mapped pages. - * */ - #define l4_index(x) (((x) >> 39) & 511) #define pud_index(x) (((x) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) diff --git a/arch/x86/kernel/hw_breakpoint.c b/arch/x86/kernel/hw_breakpoint.c index 4d8d53ed02c9..8cdf29ffd95f 100644 --- a/arch/x86/kernel/hw_breakpoint.c +++ b/arch/x86/kernel/hw_breakpoint.c @@ -32,6 +32,8 @@ #include <asm/processor.h> #include <asm/debugreg.h> #include <asm/user.h> +#include <asm/desc.h> +#include <asm/tlbflush.h> /* Per cpu debug control register value */ DEFINE_PER_CPU(unsigned long, cpu_dr7); @@ -97,6 +99,8 @@ int arch_install_hw_breakpoint(struct perf_event *bp) unsigned long *dr7; int i; + lockdep_assert_irqs_disabled(); + for (i = 0; i < HBP_NUM; i++) { struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); @@ -115,6 +119,12 @@ int arch_install_hw_breakpoint(struct perf_event *bp) dr7 = this_cpu_ptr(&cpu_dr7); *dr7 |= encode_dr7(i, info->len, info->type); + /* + * Ensure we first write cpu_dr7 before we set the DR7 register. + * This ensures an NMI never see cpu_dr7 0 when DR7 is not. + */ + barrier(); + set_debugreg(*dr7, 7); if (info->mask) set_dr_addr_mask(info->mask, i); @@ -134,9 +144,11 @@ int arch_install_hw_breakpoint(struct perf_event *bp) void arch_uninstall_hw_breakpoint(struct perf_event *bp) { struct arch_hw_breakpoint *info = counter_arch_bp(bp); - unsigned long *dr7; + unsigned long dr7; int i; + lockdep_assert_irqs_disabled(); + for (i = 0; i < HBP_NUM; i++) { struct perf_event **slot = this_cpu_ptr(&bp_per_reg[i]); @@ -149,12 +161,20 @@ void arch_uninstall_hw_breakpoint(struct perf_event *bp) if (WARN_ONCE(i == HBP_NUM, "Can't find any breakpoint slot")) return; - dr7 = this_cpu_ptr(&cpu_dr7); - *dr7 &= ~__encode_dr7(i, info->len, info->type); + dr7 = this_cpu_read(cpu_dr7); + dr7 &= ~__encode_dr7(i, info->len, info->type); - set_debugreg(*dr7, 7); + set_debugreg(dr7, 7); if (info->mask) set_dr_addr_mask(0, i); + + /* + * Ensure the write to cpu_dr7 is after we've set the DR7 register. + * This ensures an NMI never see cpu_dr7 0 when DR7 is not. + */ + barrier(); + + this_cpu_write(cpu_dr7, dr7); } static int arch_bp_generic_len(int x86_len) @@ -227,10 +247,76 @@ int arch_check_bp_in_kernelspace(struct arch_hw_breakpoint *hw) return (va >= TASK_SIZE_MAX) || ((va + len - 1) >= TASK_SIZE_MAX); } +/* + * Checks whether the range [addr, end], overlaps the area [base, base + size). + */ +static inline bool within_area(unsigned long addr, unsigned long end, + unsigned long base, unsigned long size) +{ + return end >= base && addr < (base + size); +} + +/* + * Checks whether the range from addr to end, inclusive, overlaps the fixed + * mapped CPU entry area range or other ranges used for CPU entry. + */ +static inline bool within_cpu_entry(unsigned long addr, unsigned long end) +{ + int cpu; + + /* CPU entry erea is always used for CPU entry */ + if (within_area(addr, end, CPU_ENTRY_AREA_BASE, + CPU_ENTRY_AREA_TOTAL_SIZE)) + return true; + + for_each_possible_cpu(cpu) { + /* The original rw GDT is being used after load_direct_gdt() */ + if (within_area(addr, end, (unsigned long)get_cpu_gdt_rw(cpu), + GDT_SIZE)) + return true; + + /* + * cpu_tss_rw is not directly referenced by hardware, but + * cpu_tss_rw is also used in CPU entry code, + */ + if (within_area(addr, end, + (unsigned long)&per_cpu(cpu_tss_rw, cpu), + sizeof(struct tss_struct))) + return true; + + /* + * cpu_tlbstate.user_pcid_flush_mask is used for CPU entry. + * If a data breakpoint on it, it will cause an unwanted #DB. + * Protect the full cpu_tlbstate structure to be sure. + */ + if (within_area(addr, end, + (unsigned long)&per_cpu(cpu_tlbstate, cpu), + sizeof(struct tlb_state))) + return true; + } + + return false; +} + static int arch_build_bp_info(struct perf_event *bp, const struct perf_event_attr *attr, struct arch_hw_breakpoint *hw) { + unsigned long bp_end; + + bp_end = attr->bp_addr + attr->bp_len - 1; + if (bp_end < attr->bp_addr) + return -EINVAL; + + /* + * Prevent any breakpoint of any type that overlaps the CPU + * entry area and data. This protects the IST stacks and also + * reduces the chance that we ever find out what happens if + * there's a data breakpoint on the GDT, IDT, or TSS. + */ + if (within_cpu_entry(attr->bp_addr, bp_end)) + return -EINVAL; + hw->address = attr->bp_addr; hw->mask = 0; @@ -439,7 +525,7 @@ static int hw_breakpoint_handler(struct die_args *args) { int i, cpu, rc = NOTIFY_STOP; struct perf_event *bp; - unsigned long dr7, dr6; + unsigned long dr6; unsigned long *dr6_p; /* The DR6 value is pointed by args->err */ @@ -454,9 +540,6 @@ static int hw_breakpoint_handler(struct die_args *args) if ((dr6 & DR_TRAP_BITS) == 0) return NOTIFY_DONE; - get_debugreg(dr7, 7); - /* Disable breakpoints during exception handling */ - set_debugreg(0UL, 7); /* * Assert that local interrupts are disabled * Reset the DRn bits in the virtualized register value. @@ -513,7 +596,6 @@ static int hw_breakpoint_handler(struct die_args *args) (dr6 & (~DR_TRAP_BITS))) rc = NOTIFY_DONE; - set_debugreg(dr7, 7); put_cpu(); return rc; diff --git a/arch/x86/kernel/idt.c b/arch/x86/kernel/idt.c index 87ef69a72c52..0db21206f2f3 100644 --- a/arch/x86/kernel/idt.c +++ b/arch/x86/kernel/idt.c @@ -4,6 +4,8 @@ */ #include <linux/interrupt.h> +#include <asm/cpu_entry_area.h> +#include <asm/set_memory.h> #include <asm/traps.h> #include <asm/proto.h> #include <asm/desc.h> @@ -51,15 +53,23 @@ struct idt_data { #define TSKG(_vector, _gdt) \ G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3) +#define IDT_TABLE_SIZE (IDT_ENTRIES * sizeof(gate_desc)) + +static bool idt_setup_done __initdata; + /* * Early traps running on the DEFAULT_STACK because the other interrupt * stacks work only after cpu_init(). */ static const __initconst struct idt_data early_idts[] = { - INTG(X86_TRAP_DB, debug), - SYSG(X86_TRAP_BP, int3), + INTG(X86_TRAP_DB, asm_exc_debug), + SYSG(X86_TRAP_BP, asm_exc_int3), + #ifdef CONFIG_X86_32 - INTG(X86_TRAP_PF, page_fault), + /* + * Not possible on 64-bit. See idt_setup_early_pf() for details. + */ + INTG(X86_TRAP_PF, asm_exc_page_fault), #endif }; @@ -70,33 +80,33 @@ static const __initconst struct idt_data early_idts[] = { * set up TSS. */ static const __initconst struct idt_data def_idts[] = { - INTG(X86_TRAP_DE, divide_error), - INTG(X86_TRAP_NMI, nmi), - INTG(X86_TRAP_BR, bounds), - INTG(X86_TRAP_UD, invalid_op), - INTG(X86_TRAP_NM, device_not_available), - INTG(X86_TRAP_OLD_MF, coprocessor_segment_overrun), - INTG(X86_TRAP_TS, invalid_TSS), - INTG(X86_TRAP_NP, segment_not_present), - INTG(X86_TRAP_SS, stack_segment), - INTG(X86_TRAP_GP, general_protection), - INTG(X86_TRAP_SPURIOUS, spurious_interrupt_bug), - INTG(X86_TRAP_MF, coprocessor_error), - INTG(X86_TRAP_AC, alignment_check), - INTG(X86_TRAP_XF, simd_coprocessor_error), + INTG(X86_TRAP_DE, asm_exc_divide_error), + INTG(X86_TRAP_NMI, asm_exc_nmi), + INTG(X86_TRAP_BR, asm_exc_bounds), + INTG(X86_TRAP_UD, asm_exc_invalid_op), + INTG(X86_TRAP_NM, asm_exc_device_not_available), + INTG(X86_TRAP_OLD_MF, asm_exc_coproc_segment_overrun), + INTG(X86_TRAP_TS, asm_exc_invalid_tss), + INTG(X86_TRAP_NP, asm_exc_segment_not_present), + INTG(X86_TRAP_SS, asm_exc_stack_segment), + INTG(X86_TRAP_GP, asm_exc_general_protection), + INTG(X86_TRAP_SPURIOUS, asm_exc_spurious_interrupt_bug), + INTG(X86_TRAP_MF, asm_exc_coprocessor_error), + INTG(X86_TRAP_AC, asm_exc_alignment_check), + INTG(X86_TRAP_XF, asm_exc_simd_coprocessor_error), #ifdef CONFIG_X86_32 TSKG(X86_TRAP_DF, GDT_ENTRY_DOUBLEFAULT_TSS), #else - INTG(X86_TRAP_DF, double_fault), + INTG(X86_TRAP_DF, asm_exc_double_fault), #endif - INTG(X86_TRAP_DB, debug), + INTG(X86_TRAP_DB, asm_exc_debug), #ifdef CONFIG_X86_MCE - INTG(X86_TRAP_MC, &machine_check), + INTG(X86_TRAP_MC, asm_exc_machine_check), #endif - SYSG(X86_TRAP_OF, overflow), + SYSG(X86_TRAP_OF, asm_exc_overflow), #if defined(CONFIG_IA32_EMULATION) SYSG(IA32_SYSCALL_VECTOR, entry_INT80_compat), #elif defined(CONFIG_X86_32) @@ -109,95 +119,63 @@ static const __initconst struct idt_data def_idts[] = { */ static const __initconst struct idt_data apic_idts[] = { #ifdef CONFIG_SMP - INTG(RESCHEDULE_VECTOR, reschedule_interrupt), - INTG(CALL_FUNCTION_VECTOR, call_function_interrupt), - INTG(CALL_FUNCTION_SINGLE_VECTOR, call_function_single_interrupt), - INTG(IRQ_MOVE_CLEANUP_VECTOR, irq_move_cleanup_interrupt), - INTG(REBOOT_VECTOR, reboot_interrupt), + INTG(RESCHEDULE_VECTOR, asm_sysvec_reschedule_ipi), + INTG(CALL_FUNCTION_VECTOR, asm_sysvec_call_function), + INTG(CALL_FUNCTION_SINGLE_VECTOR, asm_sysvec_call_function_single), + INTG(IRQ_MOVE_CLEANUP_VECTOR, asm_sysvec_irq_move_cleanup), + INTG(REBOOT_VECTOR, asm_sysvec_reboot), #endif #ifdef CONFIG_X86_THERMAL_VECTOR - INTG(THERMAL_APIC_VECTOR, thermal_interrupt), + INTG(THERMAL_APIC_VECTOR, asm_sysvec_thermal), #endif #ifdef CONFIG_X86_MCE_THRESHOLD - INTG(THRESHOLD_APIC_VECTOR, threshold_interrupt), + INTG(THRESHOLD_APIC_VECTOR, asm_sysvec_threshold), #endif #ifdef CONFIG_X86_MCE_AMD - INTG(DEFERRED_ERROR_VECTOR, deferred_error_interrupt), + INTG(DEFERRED_ERROR_VECTOR, asm_sysvec_deferred_error), #endif #ifdef CONFIG_X86_LOCAL_APIC - INTG(LOCAL_TIMER_VECTOR, apic_timer_interrupt), - INTG(X86_PLATFORM_IPI_VECTOR, x86_platform_ipi), + INTG(LOCAL_TIMER_VECTOR, asm_sysvec_apic_timer_interrupt), + INTG(X86_PLATFORM_IPI_VECTOR, asm_sysvec_x86_platform_ipi), # ifdef CONFIG_HAVE_KVM - INTG(POSTED_INTR_VECTOR, kvm_posted_intr_ipi), - INTG(POSTED_INTR_WAKEUP_VECTOR, kvm_posted_intr_wakeup_ipi), - INTG(POSTED_INTR_NESTED_VECTOR, kvm_posted_intr_nested_ipi), + INTG(POSTED_INTR_VECTOR, asm_sysvec_kvm_posted_intr_ipi), + INTG(POSTED_INTR_WAKEUP_VECTOR, asm_sysvec_kvm_posted_intr_wakeup_ipi), + INTG(POSTED_INTR_NESTED_VECTOR, asm_sysvec_kvm_posted_intr_nested_ipi), # endif # ifdef CONFIG_IRQ_WORK - INTG(IRQ_WORK_VECTOR, irq_work_interrupt), + INTG(IRQ_WORK_VECTOR, asm_sysvec_irq_work), # endif -#ifdef CONFIG_X86_UV - INTG(UV_BAU_MESSAGE, uv_bau_message_intr1), -#endif - INTG(SPURIOUS_APIC_VECTOR, spurious_interrupt), - INTG(ERROR_APIC_VECTOR, error_interrupt), +# ifdef CONFIG_X86_UV + INTG(UV_BAU_MESSAGE, asm_sysvec_uv_bau_message), +# endif + INTG(SPURIOUS_APIC_VECTOR, asm_sysvec_spurious_apic_interrupt), + INTG(ERROR_APIC_VECTOR, asm_sysvec_error_interrupt), #endif }; -#ifdef CONFIG_X86_64 -/* - * Early traps running on the DEFAULT_STACK because the other interrupt - * stacks work only after cpu_init(). - */ -static const __initconst struct idt_data early_pf_idts[] = { - INTG(X86_TRAP_PF, page_fault), -}; - -/* - * Override for the debug_idt. Same as the default, but with interrupt - * stack set to DEFAULT_STACK (0). Required for NMI trap handling. - */ -static const __initconst struct idt_data dbg_idts[] = { - INTG(X86_TRAP_DB, debug), -}; -#endif - -/* Must be page-aligned because the real IDT is used in a fixmap. */ -gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; +/* Must be page-aligned because the real IDT is used in the cpu entry area */ +static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss; struct desc_ptr idt_descr __ro_after_init = { - .size = (IDT_ENTRIES * 2 * sizeof(unsigned long)) - 1, + .size = IDT_TABLE_SIZE - 1, .address = (unsigned long) idt_table, }; -#ifdef CONFIG_X86_64 -/* No need to be aligned, but done to keep all IDTs defined the same way. */ -gate_desc debug_idt_table[IDT_ENTRIES] __page_aligned_bss; - -/* - * The exceptions which use Interrupt stacks. They are setup after - * cpu_init() when the TSS has been initialized. - */ -static const __initconst struct idt_data ist_idts[] = { - ISTG(X86_TRAP_DB, debug, IST_INDEX_DB), - ISTG(X86_TRAP_NMI, nmi, IST_INDEX_NMI), - ISTG(X86_TRAP_DF, double_fault, IST_INDEX_DF), -#ifdef CONFIG_X86_MCE - ISTG(X86_TRAP_MC, &machine_check, IST_INDEX_MCE), -#endif -}; +void load_current_idt(void) +{ + lockdep_assert_irqs_disabled(); + load_idt(&idt_descr); +} -/* - * Override for the debug_idt. Same as the default, but with interrupt - * stack set to DEFAULT_STACK (0). Required for NMI trap handling. - */ -const struct desc_ptr debug_idt_descr = { - .size = IDT_ENTRIES * 16 - 1, - .address = (unsigned long) debug_idt_table, -}; +#ifdef CONFIG_X86_F00F_BUG +bool idt_is_f00f_address(unsigned long address) +{ + return ((address - idt_descr.address) >> 3) == 6; +} #endif static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) @@ -214,7 +192,7 @@ static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d) #endif } -static void +static __init void idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys) { gate_desc desc; @@ -227,7 +205,7 @@ idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sy } } -static void set_intr_gate(unsigned int n, const void *addr) +static __init void set_intr_gate(unsigned int n, const void *addr) { struct idt_data data; @@ -266,6 +244,27 @@ void __init idt_setup_traps(void) } #ifdef CONFIG_X86_64 +/* + * Early traps running on the DEFAULT_STACK because the other interrupt + * stacks work only after cpu_init(). + */ +static const __initconst struct idt_data early_pf_idts[] = { + INTG(X86_TRAP_PF, asm_exc_page_fault), +}; + +/* + * The exceptions which use Interrupt stacks. They are setup after + * cpu_init() when the TSS has been initialized. + */ +static const __initconst struct idt_data ist_idts[] = { + ISTG(X86_TRAP_DB, asm_exc_debug, IST_INDEX_DB), + ISTG(X86_TRAP_NMI, asm_exc_nmi, IST_INDEX_NMI), + ISTG(X86_TRAP_DF, asm_exc_double_fault, IST_INDEX_DF), +#ifdef CONFIG_X86_MCE + ISTG(X86_TRAP_MC, asm_exc_machine_check, IST_INDEX_MCE), +#endif +}; + /** * idt_setup_early_pf - Initialize the idt table with early pagefault handler * @@ -273,8 +272,10 @@ void __init idt_setup_traps(void) * cpu_init() is invoked and sets up TSS. The IST variant is installed * after that. * - * FIXME: Why is 32bit and 64bit installing the PF handler at different - * places in the early setup code? + * Note, that X86_64 cannot install the real #PF handler in + * idt_setup_early_traps() because the memory intialization needs the #PF + * handler from the early_idt_handler_array to initialize the early page + * tables. */ void __init idt_setup_early_pf(void) { @@ -289,17 +290,20 @@ void __init idt_setup_ist_traps(void) { idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true); } +#endif -/** - * idt_setup_debugidt_traps - Initialize the debug idt table with debug traps - */ -void __init idt_setup_debugidt_traps(void) +static void __init idt_map_in_cea(void) { - memcpy(&debug_idt_table, &idt_table, IDT_ENTRIES * 16); - - idt_setup_from_table(debug_idt_table, dbg_idts, ARRAY_SIZE(dbg_idts), false); + /* + * Set the IDT descriptor to a fixed read-only location in the cpu + * entry area, so that the "sidt" instruction will not leak the + * location of the kernel, and to defend the IDT against arbitrary + * memory write vulnerabilities. + */ + cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table), + PAGE_KERNEL_RO); + idt_descr.address = CPU_ENTRY_AREA_RO_IDT; } -#endif /** * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates @@ -318,11 +322,23 @@ void __init idt_setup_apic_and_irq_gates(void) #ifdef CONFIG_X86_LOCAL_APIC for_each_clear_bit_from(i, system_vectors, NR_VECTORS) { - set_bit(i, system_vectors); + /* + * Don't set the non assigned system vectors in the + * system_vectors bitmap. Otherwise they show up in + * /proc/interrupts. + */ entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR); set_intr_gate(i, entry); } #endif + /* Map IDT into CPU entry area and reload it. */ + idt_map_in_cea(); + load_idt(&idt_descr); + + /* Make the IDT table read only */ + set_memory_ro((unsigned long)&idt_table, 1); + + idt_setup_done = true; } /** @@ -352,16 +368,14 @@ void idt_invalidate(void *addr) load_idt(&idt); } -void __init update_intr_gate(unsigned int n, const void *addr) +void __init alloc_intr_gate(unsigned int n, const void *addr) { - if (WARN_ON_ONCE(!test_bit(n, system_vectors))) + if (WARN_ON(n < FIRST_SYSTEM_VECTOR)) return; - set_intr_gate(n, addr); -} -void alloc_intr_gate(unsigned int n, const void *addr) -{ - BUG_ON(n < FIRST_SYSTEM_VECTOR); - if (!test_and_set_bit(n, system_vectors)) + if (WARN_ON(idt_setup_done)) + return; + + if (!WARN_ON(test_and_set_bit(n, system_vectors))) set_intr_gate(n, addr); } diff --git a/arch/x86/kernel/irq.c b/arch/x86/kernel/irq.c index c7965ff429c5..181060247e3c 100644 --- a/arch/x86/kernel/irq.c +++ b/arch/x86/kernel/irq.c @@ -13,12 +13,14 @@ #include <linux/export.h> #include <linux/irq.h> +#include <asm/irq_stack.h> #include <asm/apic.h> #include <asm/io_apic.h> #include <asm/irq.h> #include <asm/mce.h> #include <asm/hw_irq.h> #include <asm/desc.h> +#include <asm/traps.h> #define CREATE_TRACE_POINTS #include <asm/trace/irq_vectors.h> @@ -26,9 +28,6 @@ DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat); EXPORT_PER_CPU_SYMBOL(irq_stat); -DEFINE_PER_CPU(struct pt_regs *, irq_regs); -EXPORT_PER_CPU_SYMBOL(irq_regs); - atomic_t irq_err_count; /* @@ -224,35 +223,35 @@ u64 arch_irq_stat(void) return sum; } +static __always_inline void handle_irq(struct irq_desc *desc, + struct pt_regs *regs) +{ + if (IS_ENABLED(CONFIG_X86_64)) + run_on_irqstack_cond(desc->handle_irq, desc, regs); + else + __handle_irq(desc, regs); +} /* - * do_IRQ handles all normal device IRQ's (the special - * SMP cross-CPU interrupts have their own specific - * handlers). + * common_interrupt() handles all normal device IRQ's (the special SMP + * cross-CPU interrupts have their own entry points). */ -__visible void __irq_entry do_IRQ(struct pt_regs *regs) +DEFINE_IDTENTRY_IRQ(common_interrupt) { struct pt_regs *old_regs = set_irq_regs(regs); - struct irq_desc * desc; - /* high bit used in ret_from_ code */ - unsigned vector = ~regs->orig_ax; - - entering_irq(); + struct irq_desc *desc; - /* entering_irq() tells RCU that we're not quiescent. Check it. */ + /* entry code tells RCU that we're not quiescent. Check it. */ RCU_LOCKDEP_WARN(!rcu_is_watching(), "IRQ failed to wake up RCU"); desc = __this_cpu_read(vector_irq[vector]); if (likely(!IS_ERR_OR_NULL(desc))) { - if (IS_ENABLED(CONFIG_X86_32)) - handle_irq(desc, regs); - else - generic_handle_irq_desc(desc); + handle_irq(desc, regs); } else { ack_APIC_irq(); if (desc == VECTOR_UNUSED) { - pr_emerg_ratelimited("%s: %d.%d No irq handler for vector\n", + pr_emerg_ratelimited("%s: %d.%u No irq handler for vector\n", __func__, smp_processor_id(), vector); } else { @@ -260,8 +259,6 @@ __visible void __irq_entry do_IRQ(struct pt_regs *regs) } } - exiting_irq(); - set_irq_regs(old_regs); } @@ -271,17 +268,16 @@ void (*x86_platform_ipi_callback)(void) = NULL; /* * Handler for X86_PLATFORM_IPI_VECTOR. */ -__visible void __irq_entry smp_x86_platform_ipi(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_x86_platform_ipi) { struct pt_regs *old_regs = set_irq_regs(regs); - entering_ack_irq(); + ack_APIC_irq(); trace_x86_platform_ipi_entry(X86_PLATFORM_IPI_VECTOR); inc_irq_stat(x86_platform_ipis); if (x86_platform_ipi_callback) x86_platform_ipi_callback(); trace_x86_platform_ipi_exit(X86_PLATFORM_IPI_VECTOR); - exiting_irq(); set_irq_regs(old_regs); } #endif @@ -302,41 +298,29 @@ EXPORT_SYMBOL_GPL(kvm_set_posted_intr_wakeup_handler); /* * Handler for POSTED_INTERRUPT_VECTOR. */ -__visible void smp_kvm_posted_intr_ipi(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_ipi) { - struct pt_regs *old_regs = set_irq_regs(regs); - - entering_ack_irq(); + ack_APIC_irq(); inc_irq_stat(kvm_posted_intr_ipis); - exiting_irq(); - set_irq_regs(old_regs); } /* * Handler for POSTED_INTERRUPT_WAKEUP_VECTOR. */ -__visible void smp_kvm_posted_intr_wakeup_ipi(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_kvm_posted_intr_wakeup_ipi) { - struct pt_regs *old_regs = set_irq_regs(regs); - - entering_ack_irq(); + ack_APIC_irq(); inc_irq_stat(kvm_posted_intr_wakeup_ipis); kvm_posted_intr_wakeup_handler(); - exiting_irq(); - set_irq_regs(old_regs); } /* * Handler for POSTED_INTERRUPT_NESTED_VECTOR. */ -__visible void smp_kvm_posted_intr_nested_ipi(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_kvm_posted_intr_nested_ipi) { - struct pt_regs *old_regs = set_irq_regs(regs); - - entering_ack_irq(); + ack_APIC_irq(); inc_irq_stat(kvm_posted_intr_nested_ipis); - exiting_irq(); - set_irq_regs(old_regs); } #endif diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c index a759ca97cd01..0b79efc87be5 100644 --- a/arch/x86/kernel/irq_32.c +++ b/arch/x86/kernel/irq_32.c @@ -148,7 +148,7 @@ void do_softirq_own_stack(void) call_on_stack(__do_softirq, isp); } -void handle_irq(struct irq_desc *desc, struct pt_regs *regs) +void __handle_irq(struct irq_desc *desc, struct pt_regs *regs) { int overflow = check_stack_overflow(); diff --git a/arch/x86/kernel/irq_64.c b/arch/x86/kernel/irq_64.c index 6b32ab009c19..1b4fe93a86c5 100644 --- a/arch/x86/kernel/irq_64.c +++ b/arch/x86/kernel/irq_64.c @@ -20,6 +20,7 @@ #include <linux/sched/task_stack.h> #include <asm/cpu_entry_area.h> +#include <asm/irq_stack.h> #include <asm/io_apic.h> #include <asm/apic.h> @@ -70,3 +71,8 @@ int irq_init_percpu_irqstack(unsigned int cpu) return 0; return map_irq_stack(cpu); } + +void do_softirq_own_stack(void) +{ + run_on_irqstack_cond(__do_softirq, NULL, NULL); +} diff --git a/arch/x86/kernel/irq_work.c b/arch/x86/kernel/irq_work.c index 80bee7695a20..890d4778cd35 100644 --- a/arch/x86/kernel/irq_work.c +++ b/arch/x86/kernel/irq_work.c @@ -9,18 +9,18 @@ #include <linux/irq_work.h> #include <linux/hardirq.h> #include <asm/apic.h> +#include <asm/idtentry.h> #include <asm/trace/irq_vectors.h> #include <linux/interrupt.h> #ifdef CONFIG_X86_LOCAL_APIC -__visible void __irq_entry smp_irq_work_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_irq_work) { - ipi_entering_ack_irq(); + ack_APIC_irq(); trace_irq_work_entry(IRQ_WORK_VECTOR); inc_irq_stat(apic_irq_work_irqs); irq_work_run(); trace_irq_work_exit(IRQ_WORK_VECTOR); - exiting_irq(); } void arch_irq_work_raise(void) diff --git a/arch/x86/kernel/kprobes/core.c b/arch/x86/kernel/kprobes/core.c index 85de8fa69b24..3bafe1bd4dc7 100644 --- a/arch/x86/kernel/kprobes/core.c +++ b/arch/x86/kernel/kprobes/core.c @@ -1073,13 +1073,6 @@ NOKPROBE_SYMBOL(kprobe_fault_handler); int __init arch_populate_kprobe_blacklist(void) { - int ret; - - ret = kprobe_add_area_blacklist((unsigned long)__irqentry_text_start, - (unsigned long)__irqentry_text_end); - if (ret) - return ret; - return kprobe_add_area_blacklist((unsigned long)__entry_text_start, (unsigned long)__entry_text_end); } diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c index 234f58e0fe8c..321c19950285 100644 --- a/arch/x86/kernel/kprobes/opt.c +++ b/arch/x86/kernel/kprobes/opt.c @@ -286,9 +286,7 @@ static int can_optimize(unsigned long paddr) * stack handling and registers setup. */ if (((paddr >= (unsigned long)__entry_text_start) && - (paddr < (unsigned long)__entry_text_end)) || - ((paddr >= (unsigned long)__irqentry_text_start) && - (paddr < (unsigned long)__irqentry_text_end))) + (paddr < (unsigned long)__entry_text_end))) return 0; /* Check there is enough space for a relative jump. */ diff --git a/arch/x86/kernel/kvm.c b/arch/x86/kernel/kvm.c index d6f22a3a1f7d..df63786e7bfa 100644 --- a/arch/x86/kernel/kvm.c +++ b/arch/x86/kernel/kvm.c @@ -21,7 +21,6 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/kprobes.h> -#include <linux/debugfs.h> #include <linux/nmi.h> #include <linux/swait.h> #include <asm/timer.h> @@ -218,7 +217,7 @@ again: } EXPORT_SYMBOL_GPL(kvm_async_pf_task_wake); -u32 kvm_read_and_reset_apf_flags(void) +noinstr u32 kvm_read_and_reset_apf_flags(void) { u32 flags = 0; @@ -230,11 +229,11 @@ u32 kvm_read_and_reset_apf_flags(void) return flags; } EXPORT_SYMBOL_GPL(kvm_read_and_reset_apf_flags); -NOKPROBE_SYMBOL(kvm_read_and_reset_apf_flags); -bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) +noinstr bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) { u32 reason = kvm_read_and_reset_apf_flags(); + bool rcu_exit; switch (reason) { case KVM_PV_REASON_PAGE_NOT_PRESENT: @@ -244,6 +243,9 @@ bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) return false; } + rcu_exit = idtentry_enter_cond_rcu(regs); + instrumentation_begin(); + /* * If the host managed to inject an async #PF into an interrupt * disabled region, then die hard as this is not going to end well @@ -258,13 +260,13 @@ bool __kvm_handle_async_pf(struct pt_regs *regs, u32 token) /* Page is swapped out by the host. */ kvm_async_pf_task_wait_schedule(token); } else { - rcu_irq_enter(); kvm_async_pf_task_wake(token); - rcu_irq_exit(); } + + instrumentation_end(); + idtentry_exit_cond_rcu(regs, rcu_exit); return true; } -NOKPROBE_SYMBOL(__kvm_handle_async_pf); static void __init paravirt_ops_setup(void) { diff --git a/arch/x86/kernel/nmi.c b/arch/x86/kernel/nmi.c index bdcc5146de96..2de365f15684 100644 --- a/arch/x86/kernel/nmi.c +++ b/arch/x86/kernel/nmi.c @@ -303,7 +303,7 @@ NOKPROBE_SYMBOL(unknown_nmi_error); static DEFINE_PER_CPU(bool, swallow_nmi); static DEFINE_PER_CPU(unsigned long, last_nmi_rip); -static void default_do_nmi(struct pt_regs *regs) +static noinstr void default_do_nmi(struct pt_regs *regs) { unsigned char reason = 0; int handled; @@ -329,6 +329,9 @@ static void default_do_nmi(struct pt_regs *regs) __this_cpu_write(last_nmi_rip, regs->ip); + instrumentation_begin(); + trace_hardirqs_off_finish(); + handled = nmi_handle(NMI_LOCAL, regs); __this_cpu_add(nmi_stats.normal, handled); if (handled) { @@ -342,7 +345,7 @@ static void default_do_nmi(struct pt_regs *regs) */ if (handled > 1) __this_cpu_write(swallow_nmi, true); - return; + goto out; } /* @@ -374,7 +377,7 @@ static void default_do_nmi(struct pt_regs *regs) #endif __this_cpu_add(nmi_stats.external, 1); raw_spin_unlock(&nmi_reason_lock); - return; + goto out; } raw_spin_unlock(&nmi_reason_lock); @@ -412,8 +415,12 @@ static void default_do_nmi(struct pt_regs *regs) __this_cpu_add(nmi_stats.swallow, 1); else unknown_nmi_error(reason, regs); + +out: + if (regs->flags & X86_EFLAGS_IF) + trace_hardirqs_on_prepare(); + instrumentation_end(); } -NOKPROBE_SYMBOL(default_do_nmi); /* * NMIs can page fault or hit breakpoints which will cause it to lose @@ -467,44 +474,9 @@ enum nmi_states { }; static DEFINE_PER_CPU(enum nmi_states, nmi_state); static DEFINE_PER_CPU(unsigned long, nmi_cr2); +static DEFINE_PER_CPU(unsigned long, nmi_dr7); -#ifdef CONFIG_X86_64 -/* - * In x86_64, we need to handle breakpoint -> NMI -> breakpoint. Without - * some care, the inner breakpoint will clobber the outer breakpoint's - * stack. - * - * If a breakpoint is being processed, and the debug stack is being - * used, if an NMI comes in and also hits a breakpoint, the stack - * pointer will be set to the same fixed address as the breakpoint that - * was interrupted, causing that stack to be corrupted. To handle this - * case, check if the stack that was interrupted is the debug stack, and - * if so, change the IDT so that new breakpoints will use the current - * stack and not switch to the fixed address. On return of the NMI, - * switch back to the original IDT. - */ -static DEFINE_PER_CPU(int, update_debug_stack); - -static bool notrace is_debug_stack(unsigned long addr) -{ - struct cea_exception_stacks *cs = __this_cpu_read(cea_exception_stacks); - unsigned long top = CEA_ESTACK_TOP(cs, DB); - unsigned long bot = CEA_ESTACK_BOT(cs, DB1); - - if (__this_cpu_read(debug_stack_usage)) - return true; - /* - * Note, this covers the guard page between DB and DB1 as well to - * avoid two checks. But by all means @addr can never point into - * the guard page. - */ - return addr >= bot && addr < top; -} -NOKPROBE_SYMBOL(is_debug_stack); -#endif - -dotraplinkage notrace void -do_nmi(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY_RAW(exc_nmi) { if (IS_ENABLED(CONFIG_SMP) && cpu_is_offline(smp_processor_id())) return; @@ -517,18 +489,7 @@ do_nmi(struct pt_regs *regs, long error_code) this_cpu_write(nmi_cr2, read_cr2()); nmi_restart: -#ifdef CONFIG_X86_64 - /* - * If we interrupted a breakpoint, it is possible that - * the nmi handler will have breakpoints too. We need to - * change the IDT such that breakpoints that happen here - * continue to use the NMI stack. - */ - if (unlikely(is_debug_stack(regs->sp))) { - debug_stack_set_zero(); - this_cpu_write(update_debug_stack, 1); - } -#endif + this_cpu_write(nmi_dr7, local_db_save()); nmi_enter(); @@ -539,12 +500,7 @@ nmi_restart: nmi_exit(); -#ifdef CONFIG_X86_64 - if (unlikely(this_cpu_read(update_debug_stack))) { - debug_stack_reset(); - this_cpu_write(update_debug_stack, 0); - } -#endif + local_db_restore(this_cpu_read(nmi_dr7)); if (unlikely(this_cpu_read(nmi_cr2) != read_cr2())) write_cr2(this_cpu_read(nmi_cr2)); @@ -554,7 +510,6 @@ nmi_restart: if (user_mode(regs)) mds_user_clear_cpu_buffers(); } -NOKPROBE_SYMBOL(do_nmi); void stop_nmi(void) { diff --git a/arch/x86/kernel/process.c b/arch/x86/kernel/process.c index 8e3d0347b664..f362ce0d5ac0 100644 --- a/arch/x86/kernel/process.c +++ b/arch/x86/kernel/process.c @@ -545,28 +545,20 @@ static __always_inline void __speculation_ctrl_update(unsigned long tifp, lockdep_assert_irqs_disabled(); - /* - * If TIF_SSBD is different, select the proper mitigation - * method. Note that if SSBD mitigation is disabled or permanentely - * enabled this branch can't be taken because nothing can set - * TIF_SSBD. - */ - if (tif_diff & _TIF_SSBD) { - if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) { + /* Handle change of TIF_SSBD depending on the mitigation method. */ + if (static_cpu_has(X86_FEATURE_VIRT_SSBD)) { + if (tif_diff & _TIF_SSBD) amd_set_ssb_virt_state(tifn); - } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) { + } else if (static_cpu_has(X86_FEATURE_LS_CFG_SSBD)) { + if (tif_diff & _TIF_SSBD) amd_set_core_ssb_state(tifn); - } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || - static_cpu_has(X86_FEATURE_AMD_SSBD)) { - msr |= ssbd_tif_to_spec_ctrl(tifn); - updmsr = true; - } + } else if (static_cpu_has(X86_FEATURE_SPEC_CTRL_SSBD) || + static_cpu_has(X86_FEATURE_AMD_SSBD)) { + updmsr |= !!(tif_diff & _TIF_SSBD); + msr |= ssbd_tif_to_spec_ctrl(tifn); } - /* - * Only evaluate TIF_SPEC_IB if conditional STIBP is enabled, - * otherwise avoid the MSR write. - */ + /* Only evaluate TIF_SPEC_IB if conditional STIBP is enabled. */ if (IS_ENABLED(CONFIG_SMP) && static_branch_unlikely(&switch_to_cond_stibp)) { updmsr |= !!(tif_diff & _TIF_SPEC_IB); diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index e040ba6be27b..0ec7ced727fe 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -197,6 +197,14 @@ static const struct dmi_system_id reboot_dmi_table[] __initconst = { DMI_MATCH(DMI_PRODUCT_NAME, "MacBook5"), }, }, + { /* Handle problems with rebooting on Apple MacBook6,1 */ + .callback = set_pci_reboot, + .ident = "Apple MacBook6,1", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "MacBook6,1"), + }, + }, { /* Handle problems with rebooting on Apple MacBookPro5 */ .callback = set_pci_reboot, .ident = "Apple MacBookPro5", diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index b8d4e9c3c070..eff4ce3b10da 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -27,6 +27,7 @@ #include <asm/mmu_context.h> #include <asm/proto.h> #include <asm/apic.h> +#include <asm/idtentry.h> #include <asm/nmi.h> #include <asm/mce.h> #include <asm/trace/irq_vectors.h> @@ -130,13 +131,11 @@ static int smp_stop_nmi_callback(unsigned int val, struct pt_regs *regs) /* * this function calls the 'stop' function on all other CPUs in the system. */ - -asmlinkage __visible void smp_reboot_interrupt(void) +DEFINE_IDTENTRY_SYSVEC(sysvec_reboot) { - ipi_entering_ack_irq(); + ack_APIC_irq(); cpu_emergency_vmxoff(); stop_this_cpu(NULL); - irq_exit(); } static int register_stop_handler(void) @@ -221,47 +220,33 @@ static void native_stop_other_cpus(int wait) /* * Reschedule call back. KVM uses this interrupt to force a cpu out of - * guest mode + * guest mode. */ -__visible void __irq_entry smp_reschedule_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC_SIMPLE(sysvec_reschedule_ipi) { ack_APIC_irq(); + trace_reschedule_entry(RESCHEDULE_VECTOR); inc_irq_stat(irq_resched_count); - kvm_set_cpu_l1tf_flush_l1d(); - - if (trace_resched_ipi_enabled()) { - /* - * scheduler_ipi() might call irq_enter() as well, but - * nested calls are fine. - */ - irq_enter(); - trace_reschedule_entry(RESCHEDULE_VECTOR); - scheduler_ipi(); - trace_reschedule_exit(RESCHEDULE_VECTOR); - irq_exit(); - return; - } scheduler_ipi(); + trace_reschedule_exit(RESCHEDULE_VECTOR); } -__visible void __irq_entry smp_call_function_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_call_function) { - ipi_entering_ack_irq(); + ack_APIC_irq(); trace_call_function_entry(CALL_FUNCTION_VECTOR); inc_irq_stat(irq_call_count); generic_smp_call_function_interrupt(); trace_call_function_exit(CALL_FUNCTION_VECTOR); - exiting_irq(); } -__visible void __irq_entry smp_call_function_single_interrupt(struct pt_regs *r) +DEFINE_IDTENTRY_SYSVEC(sysvec_call_function_single) { - ipi_entering_ack_irq(); + ack_APIC_irq(); trace_call_function_single_entry(CALL_FUNCTION_SINGLE_VECTOR); inc_irq_stat(irq_call_count); generic_smp_call_function_single_interrupt(); trace_call_function_single_exit(CALL_FUNCTION_SINGLE_VECTOR); - exiting_irq(); } static int __init nonmi_ipi_setup(char *str) diff --git a/arch/x86/kernel/time.c b/arch/x86/kernel/time.c index 371a6b348e44..e42faa792c07 100644 --- a/arch/x86/kernel/time.c +++ b/arch/x86/kernel/time.c @@ -25,10 +25,6 @@ #include <asm/hpet.h> #include <asm/time.h> -#ifdef CONFIG_X86_64 -__visible volatile unsigned long jiffies __cacheline_aligned_in_smp = INITIAL_JIFFIES; -#endif - unsigned long profile_pc(struct pt_regs *regs) { unsigned long pc = instruction_pointer(regs); diff --git a/arch/x86/kernel/tracepoint.c b/arch/x86/kernel/tracepoint.c index 496748ed266a..fcfc077afe2d 100644 --- a/arch/x86/kernel/tracepoint.c +++ b/arch/x86/kernel/tracepoint.c @@ -25,20 +25,3 @@ void trace_pagefault_unreg(void) { static_branch_dec(&trace_pagefault_key); } - -#ifdef CONFIG_SMP - -DEFINE_STATIC_KEY_FALSE(trace_resched_ipi_key); - -int trace_resched_ipi_reg(void) -{ - static_branch_inc(&trace_resched_ipi_key); - return 0; -} - -void trace_resched_ipi_unreg(void) -{ - static_branch_dec(&trace_resched_ipi_key); -} - -#endif diff --git a/arch/x86/kernel/traps.c b/arch/x86/kernel/traps.c index 4cc541051994..af75109485c2 100644 --- a/arch/x86/kernel/traps.c +++ b/arch/x86/kernel/traps.c @@ -97,24 +97,6 @@ int is_valid_bugaddr(unsigned long addr) return ud == INSN_UD0 || ud == INSN_UD2; } -int fixup_bug(struct pt_regs *regs, int trapnr) -{ - if (trapnr != X86_TRAP_UD) - return 0; - - switch (report_bug(regs->ip, regs)) { - case BUG_TRAP_TYPE_NONE: - case BUG_TRAP_TYPE_BUG: - break; - - case BUG_TRAP_TYPE_WARN: - regs->ip += LEN_UD2; - return 1; - } - - return 0; -} - static nokprobe_inline int do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str, struct pt_regs *regs, long error_code) @@ -145,7 +127,7 @@ do_trap_no_signal(struct task_struct *tsk, int trapnr, const char *str, * process no chance to handle the signal and notice the * kernel fault information, so that won't result in polluting * the information about previously queued, but not yet - * delivered, faults. See also do_general_protection below. + * delivered, faults. See also exc_general_protection below. */ tsk->thread.error_code = error_code; tsk->thread.trap_nr = trapnr; @@ -190,41 +172,119 @@ static void do_error_trap(struct pt_regs *regs, long error_code, char *str, { RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); - /* - * WARN*()s end up here; fix them up before we call the - * notifier chain. - */ - if (!user_mode(regs) && fixup_bug(regs, trapnr)) - return; - if (notify_die(DIE_TRAP, str, regs, error_code, trapnr, signr) != NOTIFY_STOP) { cond_local_irq_enable(regs); do_trap(trapnr, signr, str, regs, error_code, sicode, addr); + cond_local_irq_disable(regs); } } -#define IP ((void __user *)uprobe_get_trap_addr(regs)) -#define DO_ERROR(trapnr, signr, sicode, addr, str, name) \ -dotraplinkage void do_##name(struct pt_regs *regs, long error_code) \ -{ \ - do_error_trap(regs, error_code, str, trapnr, signr, sicode, addr); \ +/* + * Posix requires to provide the address of the faulting instruction for + * SIGILL (#UD) and SIGFPE (#DE) in the si_addr member of siginfo_t. + * + * This address is usually regs->ip, but when an uprobe moved the code out + * of line then regs->ip points to the XOL code which would confuse + * anything which analyzes the fault address vs. the unmodified binary. If + * a trap happened in XOL code then uprobe maps regs->ip back to the + * original instruction address. + */ +static __always_inline void __user *error_get_trap_addr(struct pt_regs *regs) +{ + return (void __user *)uprobe_get_trap_addr(regs); } -DO_ERROR(X86_TRAP_DE, SIGFPE, FPE_INTDIV, IP, "divide error", divide_error) -DO_ERROR(X86_TRAP_OF, SIGSEGV, 0, NULL, "overflow", overflow) -DO_ERROR(X86_TRAP_UD, SIGILL, ILL_ILLOPN, IP, "invalid opcode", invalid_op) -DO_ERROR(X86_TRAP_OLD_MF, SIGFPE, 0, NULL, "coprocessor segment overrun", coprocessor_segment_overrun) -DO_ERROR(X86_TRAP_TS, SIGSEGV, 0, NULL, "invalid TSS", invalid_TSS) -DO_ERROR(X86_TRAP_NP, SIGBUS, 0, NULL, "segment not present", segment_not_present) -DO_ERROR(X86_TRAP_SS, SIGBUS, 0, NULL, "stack segment", stack_segment) -#undef IP +DEFINE_IDTENTRY(exc_divide_error) +{ + do_error_trap(regs, 0, "divide_error", X86_TRAP_DE, SIGFPE, + FPE_INTDIV, error_get_trap_addr(regs)); +} -dotraplinkage void do_alignment_check(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_overflow) { - char *str = "alignment check"; + do_error_trap(regs, 0, "overflow", X86_TRAP_OF, SIGSEGV, 0, NULL); +} - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); +#ifdef CONFIG_X86_F00F_BUG +void handle_invalid_op(struct pt_regs *regs) +#else +static inline void handle_invalid_op(struct pt_regs *regs) +#endif +{ + do_error_trap(regs, 0, "invalid opcode", X86_TRAP_UD, SIGILL, + ILL_ILLOPN, error_get_trap_addr(regs)); +} + +DEFINE_IDTENTRY_RAW(exc_invalid_op) +{ + bool rcu_exit; + + /* + * Handle BUG/WARN like NMIs instead of like normal idtentries: + * if we bugged/warned in a bad RCU context, for example, the last + * thing we want is to BUG/WARN again in the idtentry code, ad + * infinitum. + */ + if (!user_mode(regs) && is_valid_bugaddr(regs->ip)) { + enum bug_trap_type type; + + nmi_enter(); + instrumentation_begin(); + trace_hardirqs_off_finish(); + type = report_bug(regs->ip, regs); + if (regs->flags & X86_EFLAGS_IF) + trace_hardirqs_on_prepare(); + instrumentation_end(); + nmi_exit(); + + if (type == BUG_TRAP_TYPE_WARN) { + /* Skip the ud2. */ + regs->ip += LEN_UD2; + return; + } + + /* + * Else, if this was a BUG and report_bug returns or if this + * was just a normal #UD, we want to continue onward and + * crash. + */ + } + + rcu_exit = idtentry_enter_cond_rcu(regs); + instrumentation_begin(); + handle_invalid_op(regs); + instrumentation_end(); + idtentry_exit_cond_rcu(regs, rcu_exit); +} + +DEFINE_IDTENTRY(exc_coproc_segment_overrun) +{ + do_error_trap(regs, 0, "coprocessor segment overrun", + X86_TRAP_OLD_MF, SIGFPE, 0, NULL); +} + +DEFINE_IDTENTRY_ERRORCODE(exc_invalid_tss) +{ + do_error_trap(regs, error_code, "invalid TSS", X86_TRAP_TS, SIGSEGV, + 0, NULL); +} + +DEFINE_IDTENTRY_ERRORCODE(exc_segment_not_present) +{ + do_error_trap(regs, error_code, "segment not present", X86_TRAP_NP, + SIGBUS, 0, NULL); +} + +DEFINE_IDTENTRY_ERRORCODE(exc_stack_segment) +{ + do_error_trap(regs, error_code, "stack segment", X86_TRAP_SS, SIGBUS, + 0, NULL); +} + +DEFINE_IDTENTRY_ERRORCODE(exc_alignment_check) +{ + char *str = "alignment check"; if (notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_AC, SIGBUS) == NOTIFY_STOP) return; @@ -271,12 +331,19 @@ __visible void __noreturn handle_stack_overflow(const char *message, * from the TSS. Returning is, in principle, okay, but changes to regs will * be lost. If, for some reason, we need to return to a context with modified * regs, the shim code could be adjusted to synchronize the registers. + * + * The 32bit #DF shim provides CR2 already as an argument. On 64bit it needs + * to be read before doing anything else. */ -dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsigned long cr2) +DEFINE_IDTENTRY_DF(exc_double_fault) { static const char str[] = "double fault"; struct task_struct *tsk = current; +#ifdef CONFIG_VMAP_STACK + unsigned long address = read_cr2(); +#endif + #ifdef CONFIG_X86_ESPFIX64 extern unsigned char native_irq_return_iret[]; @@ -299,6 +366,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign regs->ip == (unsigned long)native_irq_return_iret) { struct pt_regs *gpregs = (struct pt_regs *)this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1; + unsigned long *p = (unsigned long *)regs->sp; /* * regs->sp points to the failing IRET frame on the @@ -306,7 +374,11 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign * in gpregs->ss through gpregs->ip. * */ - memmove(&gpregs->ip, (void *)regs->sp, 5*8); + gpregs->ip = p[0]; + gpregs->cs = p[1]; + gpregs->flags = p[2]; + gpregs->sp = p[3]; + gpregs->ss = p[4]; gpregs->orig_ax = 0; /* Missing (lost) #GP error code */ /* @@ -320,7 +392,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign * which is what the stub expects, given that the faulting * RIP will be the IRET instruction. */ - regs->ip = (unsigned long)general_protection; + regs->ip = (unsigned long)asm_exc_general_protection; regs->sp = (unsigned long)&gpregs->orig_ax; return; @@ -328,6 +400,7 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign #endif nmi_enter(); + instrumentation_begin(); notify_die(DIE_TRAP, str, regs, error_code, X86_TRAP_DF, SIGSEGV); tsk->thread.error_code = error_code; @@ -371,27 +444,31 @@ dotraplinkage void do_double_fault(struct pt_regs *regs, long error_code, unsign * stack even if the actual trigger for the double fault was * something else. */ - if ((unsigned long)task_stack_page(tsk) - 1 - cr2 < PAGE_SIZE) - handle_stack_overflow("kernel stack overflow (double-fault)", regs, cr2); + if ((unsigned long)task_stack_page(tsk) - 1 - address < PAGE_SIZE) { + handle_stack_overflow("kernel stack overflow (double-fault)", + regs, address); + } #endif pr_emerg("PANIC: double fault, error_code: 0x%lx\n", error_code); die("double fault", regs, error_code); panic("Machine halted."); + instrumentation_end(); } -dotraplinkage void do_bounds(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_bounds) { - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); - if (notify_die(DIE_TRAP, "bounds", regs, error_code, + if (notify_die(DIE_TRAP, "bounds", regs, 0, X86_TRAP_BR, SIGSEGV) == NOTIFY_STOP) return; cond_local_irq_enable(regs); if (!user_mode(regs)) - die("bounds", regs, error_code); + die("bounds", regs, 0); + + do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, 0, 0, NULL); - do_trap(X86_TRAP_BR, SIGSEGV, "bounds", regs, error_code, 0, NULL); + cond_local_irq_disable(regs); } enum kernel_gp_hint { @@ -438,7 +515,7 @@ static enum kernel_gp_hint get_kernel_gp_address(struct pt_regs *regs, #define GPFSTR "general protection fault" -dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY_ERRORCODE(exc_general_protection) { char desc[sizeof(GPFSTR) + 50 + 2*sizeof(unsigned long) + 1] = GPFSTR; enum kernel_gp_hint hint = GP_NO_HINT; @@ -446,17 +523,17 @@ dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code) unsigned long gp_addr; int ret; - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); cond_local_irq_enable(regs); if (static_cpu_has(X86_FEATURE_UMIP)) { if (user_mode(regs) && fixup_umip_exception(regs)) - return; + goto exit; } if (v8086_mode(regs)) { local_irq_enable(); handle_vm86_fault((struct kernel_vm86_regs *) regs, error_code); + local_irq_disable(); return; } @@ -468,12 +545,11 @@ dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code) show_signal(tsk, SIGSEGV, "", desc, regs, error_code); force_sig(SIGSEGV); - - return; + goto exit; } if (fixup_exception(regs, X86_TRAP_GP, error_code, 0)) - return; + goto exit; tsk->thread.error_code = error_code; tsk->thread.trap_nr = X86_TRAP_GP; @@ -485,11 +561,11 @@ dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code) if (!preemptible() && kprobe_running() && kprobe_fault_handler(regs, X86_TRAP_GP)) - return; + goto exit; ret = notify_die(DIE_GPF, desc, regs, error_code, X86_TRAP_GP, SIGSEGV); if (ret == NOTIFY_STOP) - return; + goto exit; if (error_code) snprintf(desc, sizeof(desc), "segment-related " GPFSTR); @@ -511,47 +587,74 @@ dotraplinkage void do_general_protection(struct pt_regs *regs, long error_code) die_addr(desc, regs, error_code, gp_addr); +exit: + cond_local_irq_disable(regs); } -NOKPROBE_SYMBOL(do_general_protection); -dotraplinkage void notrace do_int3(struct pt_regs *regs, long error_code) +static bool do_int3(struct pt_regs *regs) { - if (poke_int3_handler(regs)) - return; - - /* - * Unlike any other non-IST entry, we can be called from pretty much - * any location in the kernel through kprobes -- text_poke() will most - * likely be handled by poke_int3_handler() above. This means this - * handler is effectively NMI-like. - */ - if (!user_mode(regs)) - nmi_enter(); + int res; #ifdef CONFIG_KGDB_LOW_LEVEL_TRAP - if (kgdb_ll_trap(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, - SIGTRAP) == NOTIFY_STOP) - goto exit; + if (kgdb_ll_trap(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, + SIGTRAP) == NOTIFY_STOP) + return true; #endif /* CONFIG_KGDB_LOW_LEVEL_TRAP */ #ifdef CONFIG_KPROBES if (kprobe_int3_handler(regs)) - goto exit; + return true; #endif + res = notify_die(DIE_INT3, "int3", regs, 0, X86_TRAP_BP, SIGTRAP); - if (notify_die(DIE_INT3, "int3", regs, error_code, X86_TRAP_BP, - SIGTRAP) == NOTIFY_STOP) - goto exit; + return res == NOTIFY_STOP; +} + +static void do_int3_user(struct pt_regs *regs) +{ + if (do_int3(regs)) + return; cond_local_irq_enable(regs); - do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, error_code, 0, NULL); + do_trap(X86_TRAP_BP, SIGTRAP, "int3", regs, 0, 0, NULL); cond_local_irq_disable(regs); +} -exit: - if (!user_mode(regs)) +DEFINE_IDTENTRY_RAW(exc_int3) +{ + /* + * poke_int3_handler() is completely self contained code; it does (and + * must) *NOT* call out to anything, lest it hits upon yet another + * INT3. + */ + if (poke_int3_handler(regs)) + return; + + /* + * idtentry_enter_user() uses static_branch_{,un}likely() and therefore + * can trigger INT3, hence poke_int3_handler() must be done + * before. If the entry came from kernel mode, then use nmi_enter() + * because the INT3 could have been hit in any context including + * NMI. + */ + if (user_mode(regs)) { + idtentry_enter_user(regs); + instrumentation_begin(); + do_int3_user(regs); + instrumentation_end(); + idtentry_exit_user(regs); + } else { + nmi_enter(); + instrumentation_begin(); + trace_hardirqs_off_finish(); + if (!do_int3(regs)) + die("int3", regs, 0); + if (regs->flags & X86_EFLAGS_IF) + trace_hardirqs_on_prepare(); + instrumentation_end(); nmi_exit(); + } } -NOKPROBE_SYMBOL(do_int3); #ifdef CONFIG_X86_64 /* @@ -559,21 +662,20 @@ NOKPROBE_SYMBOL(do_int3); * to switch to the normal thread stack if the interrupted code was in * user mode. The actual stack switch is done in entry_64.S */ -asmlinkage __visible notrace struct pt_regs *sync_regs(struct pt_regs *eregs) +asmlinkage __visible noinstr struct pt_regs *sync_regs(struct pt_regs *eregs) { struct pt_regs *regs = (struct pt_regs *)this_cpu_read(cpu_current_top_of_stack) - 1; if (regs != eregs) *regs = *eregs; return regs; } -NOKPROBE_SYMBOL(sync_regs); struct bad_iret_stack { void *error_entry_ret; struct pt_regs regs; }; -asmlinkage __visible notrace +asmlinkage __visible noinstr struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s) { /* @@ -584,19 +686,21 @@ struct bad_iret_stack *fixup_bad_iret(struct bad_iret_stack *s) * just below the IRET frame) and we want to pretend that the * exception came from the IRET target. */ - struct bad_iret_stack *new_stack = - (struct bad_iret_stack *)this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1; + struct bad_iret_stack tmp, *new_stack = + (struct bad_iret_stack *)__this_cpu_read(cpu_tss_rw.x86_tss.sp0) - 1; - /* Copy the IRET target to the new stack. */ - memmove(&new_stack->regs.ip, (void *)s->regs.sp, 5*8); + /* Copy the IRET target to the temporary storage. */ + memcpy(&tmp.regs.ip, (void *)s->regs.sp, 5*8); /* Copy the remainder of the stack from the current stack. */ - memmove(new_stack, s, offsetof(struct bad_iret_stack, regs.ip)); + memcpy(&tmp, s, offsetof(struct bad_iret_stack, regs.ip)); + + /* Update the entry stack */ + memcpy(new_stack, &tmp, sizeof(tmp)); BUG_ON(!user_mode(&new_stack->regs)); return new_stack; } -NOKPROBE_SYMBOL(fixup_bad_iret); #endif static bool is_sysenter_singlestep(struct pt_regs *regs) @@ -622,6 +726,43 @@ static bool is_sysenter_singlestep(struct pt_regs *regs) #endif } +static __always_inline void debug_enter(unsigned long *dr6, unsigned long *dr7) +{ + /* + * Disable breakpoints during exception handling; recursive exceptions + * are exceedingly 'fun'. + * + * Since this function is NOKPROBE, and that also applies to + * HW_BREAKPOINT_X, we can't hit a breakpoint before this (XXX except a + * HW_BREAKPOINT_W on our stack) + * + * Entry text is excluded for HW_BP_X and cpu_entry_area, which + * includes the entry stack is excluded for everything. + */ + *dr7 = local_db_save(); + + /* + * The Intel SDM says: + * + * Certain debug exceptions may clear bits 0-3. The remaining + * contents of the DR6 register are never cleared by the + * processor. To avoid confusion in identifying debug + * exceptions, debug handlers should clear the register before + * returning to the interrupted task. + * + * Keep it simple: clear DR6 immediately. + */ + get_debugreg(*dr6, 6); + set_debugreg(0, 6); + /* Filter out all the reserved bits which are preset to 1 */ + *dr6 &= ~DR6_RESERVED; +} + +static __always_inline void debug_exit(unsigned long dr7) +{ + local_db_restore(dr7); +} + /* * Our handling of the processor debug registers is non-trivial. * We do not clear them on entry and exit from the kernel. Therefore @@ -646,86 +787,54 @@ static bool is_sysenter_singlestep(struct pt_regs *regs) * * May run on IST stack. */ -dotraplinkage void do_debug(struct pt_regs *regs, long error_code) +static void handle_debug(struct pt_regs *regs, unsigned long dr6, bool user) { struct task_struct *tsk = current; - int user_icebp = 0; - unsigned long dr6; + bool user_icebp; int si_code; - nmi_enter(); - - get_debugreg(dr6, 6); - /* - * The Intel SDM says: - * - * Certain debug exceptions may clear bits 0-3. The remaining - * contents of the DR6 register are never cleared by the - * processor. To avoid confusion in identifying debug - * exceptions, debug handlers should clear the register before - * returning to the interrupted task. - * - * Keep it simple: clear DR6 immediately. - */ - set_debugreg(0, 6); - - /* Filter out all the reserved bits which are preset to 1 */ - dr6 &= ~DR6_RESERVED; - /* * The SDM says "The processor clears the BTF flag when it * generates a debug exception." Clear TIF_BLOCKSTEP to keep * TIF_BLOCKSTEP in sync with the hardware BTF flag. */ - clear_tsk_thread_flag(tsk, TIF_BLOCKSTEP); + clear_thread_flag(TIF_BLOCKSTEP); - if (unlikely(!user_mode(regs) && (dr6 & DR_STEP) && - is_sysenter_singlestep(regs))) { - dr6 &= ~DR_STEP; - if (!dr6) - goto exit; - /* - * else we might have gotten a single-step trap and hit a - * watchpoint at the same time, in which case we should fall - * through and handle the watchpoint. - */ - } + /* + * If DR6 is zero, no point in trying to handle it. The kernel is + * not using INT1. + */ + if (!user && !dr6) + return; /* * If dr6 has no reason to give us about the origin of this trap, * then it's very likely the result of an icebp/int01 trap. * User wants a sigtrap for that. */ - if (!dr6 && user_mode(regs)) - user_icebp = 1; + user_icebp = user && !dr6; /* Store the virtualized DR6 value */ tsk->thread.debugreg6 = dr6; #ifdef CONFIG_KPROBES - if (kprobe_debug_handler(regs)) - goto exit; + if (kprobe_debug_handler(regs)) { + return; + } #endif - if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, error_code, - SIGTRAP) == NOTIFY_STOP) - goto exit; - - /* - * Let others (NMI) know that the debug stack is in use - * as we may switch to the interrupt stack. - */ - debug_stack_usage_inc(); + if (notify_die(DIE_DEBUG, "debug", regs, (long)&dr6, 0, + SIGTRAP) == NOTIFY_STOP) { + return; + } /* It's safe to allow irq's after DR6 has been saved */ cond_local_irq_enable(regs); if (v8086_mode(regs)) { - handle_vm86_trap((struct kernel_vm86_regs *) regs, error_code, - X86_TRAP_DB); - cond_local_irq_disable(regs); - debug_stack_usage_dec(); - goto exit; + handle_vm86_trap((struct kernel_vm86_regs *) regs, 0, + X86_TRAP_DB); + goto out; } if (WARN_ON_ONCE((dr6 & DR_STEP) && !user_mode(regs))) { @@ -739,23 +848,91 @@ dotraplinkage void do_debug(struct pt_regs *regs, long error_code) set_tsk_thread_flag(tsk, TIF_SINGLESTEP); regs->flags &= ~X86_EFLAGS_TF; } + si_code = get_si_code(tsk->thread.debugreg6); if (tsk->thread.debugreg6 & (DR_STEP | DR_TRAP_BITS) || user_icebp) - send_sigtrap(regs, error_code, si_code); + send_sigtrap(regs, 0, si_code); + +out: cond_local_irq_disable(regs); - debug_stack_usage_dec(); +} -exit: +static __always_inline void exc_debug_kernel(struct pt_regs *regs, + unsigned long dr6) +{ + nmi_enter(); + instrumentation_begin(); + trace_hardirqs_off_finish(); + + /* + * Catch SYSENTER with TF set and clear DR_STEP. If this hit a + * watchpoint at the same time then that will still be handled. + */ + if ((dr6 & DR_STEP) && is_sysenter_singlestep(regs)) + dr6 &= ~DR_STEP; + + handle_debug(regs, dr6, false); + + if (regs->flags & X86_EFLAGS_IF) + trace_hardirqs_on_prepare(); + instrumentation_end(); nmi_exit(); } -NOKPROBE_SYMBOL(do_debug); + +static __always_inline void exc_debug_user(struct pt_regs *regs, + unsigned long dr6) +{ + idtentry_enter_user(regs); + instrumentation_begin(); + + handle_debug(regs, dr6, true); + instrumentation_end(); + idtentry_exit_user(regs); +} + +#ifdef CONFIG_X86_64 +/* IST stack entry */ +DEFINE_IDTENTRY_DEBUG(exc_debug) +{ + unsigned long dr6, dr7; + + debug_enter(&dr6, &dr7); + exc_debug_kernel(regs, dr6); + debug_exit(dr7); +} + +/* User entry, runs on regular task stack */ +DEFINE_IDTENTRY_DEBUG_USER(exc_debug) +{ + unsigned long dr6, dr7; + + debug_enter(&dr6, &dr7); + exc_debug_user(regs, dr6); + debug_exit(dr7); +} +#else +/* 32 bit does not have separate entry points. */ +DEFINE_IDTENTRY_DEBUG(exc_debug) +{ + unsigned long dr6, dr7; + + debug_enter(&dr6, &dr7); + + if (user_mode(regs)) + exc_debug_user(regs, dr6); + else + exc_debug_kernel(regs, dr6); + + debug_exit(dr7); +} +#endif /* * Note that we play around with the 'TS' bit in an attempt to get * the correct behaviour even in the presence of the asynchronous * IRQ13 behaviour */ -static void math_error(struct pt_regs *regs, int error_code, int trapnr) +static void math_error(struct pt_regs *regs, int trapnr) { struct task_struct *task = current; struct fpu *fpu = &task->thread.fpu; @@ -766,16 +943,16 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr) cond_local_irq_enable(regs); if (!user_mode(regs)) { - if (fixup_exception(regs, trapnr, error_code, 0)) - return; + if (fixup_exception(regs, trapnr, 0, 0)) + goto exit; - task->thread.error_code = error_code; + task->thread.error_code = 0; task->thread.trap_nr = trapnr; - if (notify_die(DIE_TRAP, str, regs, error_code, - trapnr, SIGFPE) != NOTIFY_STOP) - die(str, regs, error_code); - return; + if (notify_die(DIE_TRAP, str, regs, 0, trapnr, + SIGFPE) != NOTIFY_STOP) + die(str, regs, 0); + goto exit; } /* @@ -784,32 +961,37 @@ static void math_error(struct pt_regs *regs, int error_code, int trapnr) fpu__save(fpu); task->thread.trap_nr = trapnr; - task->thread.error_code = error_code; + task->thread.error_code = 0; si_code = fpu__exception_code(fpu, trapnr); /* Retry when we get spurious exceptions: */ if (!si_code) - return; + goto exit; force_sig_fault(SIGFPE, si_code, (void __user *)uprobe_get_trap_addr(regs)); +exit: + cond_local_irq_disable(regs); } -dotraplinkage void do_coprocessor_error(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_coprocessor_error) { - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); - math_error(regs, error_code, X86_TRAP_MF); + math_error(regs, X86_TRAP_MF); } -dotraplinkage void -do_simd_coprocessor_error(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_simd_coprocessor_error) { - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); - math_error(regs, error_code, X86_TRAP_XF); + if (IS_ENABLED(CONFIG_X86_INVD_BUG)) { + /* AMD 486 bug: INVD in CPL 0 raises #XF instead of #GP */ + if (!static_cpu_has(X86_FEATURE_XMM)) { + __exc_general_protection(regs, 0); + return; + } + } + math_error(regs, X86_TRAP_XF); } -dotraplinkage void -do_spurious_interrupt_bug(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_spurious_interrupt_bug) { /* * This addresses a Pentium Pro Erratum: @@ -832,13 +1014,10 @@ do_spurious_interrupt_bug(struct pt_regs *regs, long error_code) */ } -dotraplinkage void -do_device_not_available(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY(exc_device_not_available) { unsigned long cr0 = read_cr0(); - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); - #ifdef CONFIG_MATH_EMULATION if (!boot_cpu_has(X86_FEATURE_FPU) && (cr0 & X86_CR0_EM)) { struct math_emu_info info = { }; @@ -847,6 +1026,8 @@ do_device_not_available(struct pt_regs *regs, long error_code) info.regs = regs; math_emulate(&info); + + cond_local_irq_disable(regs); return; } #endif @@ -861,22 +1042,20 @@ do_device_not_available(struct pt_regs *regs, long error_code) * to kill the task than getting stuck in a never-ending * loop of #NM faults. */ - die("unexpected #NM exception", regs, error_code); + die("unexpected #NM exception", regs, 0); } } -NOKPROBE_SYMBOL(do_device_not_available); #ifdef CONFIG_X86_32 -dotraplinkage void do_iret_error(struct pt_regs *regs, long error_code) +DEFINE_IDTENTRY_SW(iret_error) { - RCU_LOCKDEP_WARN(!rcu_is_watching(), "entry code didn't wake RCU"); local_irq_enable(); - - if (notify_die(DIE_TRAP, "iret exception", regs, error_code, + if (notify_die(DIE_TRAP, "iret exception", regs, 0, X86_TRAP_IRET, SIGILL) != NOTIFY_STOP) { - do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, error_code, + do_trap(X86_TRAP_IRET, SIGILL, "iret exception", regs, 0, ILL_BADSTK, (void __user *)NULL); } + local_irq_disable(); } #endif @@ -888,20 +1067,9 @@ void __init trap_init(void) idt_setup_traps(); /* - * Set the IDT descriptor to a fixed read-only location, so that the - * "sidt" instruction will not leak the location of the kernel, and - * to defend the IDT against arbitrary memory write vulnerabilities. - * It will be reloaded in cpu_init() */ - cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table), - PAGE_KERNEL_RO); - idt_descr.address = CPU_ENTRY_AREA_RO_IDT; - - /* * Should be a barrier for any external CPU state: */ cpu_init(); idt_setup_ist_traps(); - - idt_setup_debugidt_traps(); } diff --git a/arch/x86/kernel/unwind_frame.c b/arch/x86/kernel/unwind_frame.c index 54226110bc7f..722a85f3b2dd 100644 --- a/arch/x86/kernel/unwind_frame.c +++ b/arch/x86/kernel/unwind_frame.c @@ -74,13 +74,7 @@ static bool in_entry_code(unsigned long ip) { char *addr = (char *)ip; - if (addr >= __entry_text_start && addr < __entry_text_end) - return true; - - if (addr >= __irqentry_text_start && addr < __irqentry_text_end) - return true; - - return false; + return addr >= __entry_text_start && addr < __entry_text_end; } static inline unsigned long *last_frame(struct unwind_state *state) diff --git a/arch/x86/kernel/vmlinux.lds.S b/arch/x86/kernel/vmlinux.lds.S index 1bf7e312361f..3bfc8dd8a43d 100644 --- a/arch/x86/kernel/vmlinux.lds.S +++ b/arch/x86/kernel/vmlinux.lds.S @@ -40,13 +40,13 @@ OUTPUT_FORMAT(CONFIG_OUTPUT_FORMAT) #ifdef CONFIG_X86_32 OUTPUT_ARCH(i386) ENTRY(phys_startup_32) -jiffies = jiffies_64; #else OUTPUT_ARCH(i386:x86-64) ENTRY(phys_startup_64) -jiffies_64 = jiffies; #endif +jiffies = jiffies_64; + #if defined(CONFIG_X86_64) /* * On 64-bit, align RODATA to 2MB so we retain large page mappings for @@ -134,7 +134,6 @@ SECTIONS KPROBES_TEXT ALIGN_ENTRY_TEXT_BEGIN ENTRY_TEXT - IRQENTRY_TEXT ALIGN_ENTRY_TEXT_END SOFTIRQENTRY_TEXT *(.fixup) diff --git a/arch/x86/kvm/cpuid.c b/arch/x86/kvm/cpuid.c index 253b8e875ccd..8a294f9747aa 100644 --- a/arch/x86/kvm/cpuid.c +++ b/arch/x86/kvm/cpuid.c @@ -181,17 +181,14 @@ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, r = -E2BIG; if (cpuid->nent > KVM_MAX_CPUID_ENTRIES) goto out; - r = -ENOMEM; if (cpuid->nent) { - cpuid_entries = - vmalloc(array_size(sizeof(struct kvm_cpuid_entry), - cpuid->nent)); - if (!cpuid_entries) - goto out; - r = -EFAULT; - if (copy_from_user(cpuid_entries, entries, - cpuid->nent * sizeof(struct kvm_cpuid_entry))) + cpuid_entries = vmemdup_user(entries, + array_size(sizeof(struct kvm_cpuid_entry), + cpuid->nent)); + if (IS_ERR(cpuid_entries)) { + r = PTR_ERR(cpuid_entries); goto out; + } } for (i = 0; i < cpuid->nent; i++) { vcpu->arch.cpuid_entries[i].function = cpuid_entries[i].function; @@ -211,8 +208,8 @@ int kvm_vcpu_ioctl_set_cpuid(struct kvm_vcpu *vcpu, kvm_x86_ops.cpuid_update(vcpu); r = kvm_update_cpuid(vcpu); + kvfree(cpuid_entries); out: - vfree(cpuid_entries); return r; } @@ -325,7 +322,7 @@ void kvm_set_cpu_caps(void) ); kvm_cpu_cap_mask(CPUID_7_ECX, - F(AVX512VBMI) | F(LA57) | 0 /*PKU*/ | 0 /*OSPKE*/ | F(RDPID) | + F(AVX512VBMI) | F(LA57) | F(PKU) | 0 /*OSPKE*/ | F(RDPID) | F(AVX512_VPOPCNTDQ) | F(UMIP) | F(AVX512_VBMI2) | F(GFNI) | F(VAES) | F(VPCLMULQDQ) | F(AVX512_VNNI) | F(AVX512_BITALG) | F(CLDEMOTE) | F(MOVDIRI) | F(MOVDIR64B) | 0 /*WAITPKG*/ @@ -334,6 +331,13 @@ void kvm_set_cpu_caps(void) if (cpuid_ecx(7) & F(LA57)) kvm_cpu_cap_set(X86_FEATURE_LA57); + /* + * PKU not yet implemented for shadow paging and requires OSPKE + * to be set on the host. Clear it if that is not the case + */ + if (!tdp_enabled || !boot_cpu_has(X86_FEATURE_OSPKE)) + kvm_cpu_cap_clear(X86_FEATURE_PKU); + kvm_cpu_cap_mask(CPUID_7_EDX, F(AVX512_4VNNIW) | F(AVX512_4FMAPS) | F(SPEC_CTRL) | F(SPEC_CTRL_SSBD) | F(ARCH_CAPABILITIES) | F(INTEL_STIBP) | @@ -426,7 +430,7 @@ EXPORT_SYMBOL_GPL(kvm_set_cpu_caps); struct kvm_cpuid_array { struct kvm_cpuid_entry2 *entries; - const int maxnent; + int maxnent; int nent; }; @@ -870,7 +874,6 @@ int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid, struct kvm_cpuid_array array = { .nent = 0, - .maxnent = cpuid->nent, }; int r, i; @@ -887,6 +890,8 @@ int kvm_dev_ioctl_get_cpuid(struct kvm_cpuid2 *cpuid, if (!array.entries) return -ENOMEM; + array.maxnent = cpuid->nent; + for (i = 0; i < ARRAY_SIZE(funcs); i++) { r = get_cpuid_func(&array, funcs[i], type); if (r) diff --git a/arch/x86/kvm/debugfs.c b/arch/x86/kvm/debugfs.c index 018aebce33ff..7e818d64bb4d 100644 --- a/arch/x86/kvm/debugfs.c +++ b/arch/x86/kvm/debugfs.c @@ -43,22 +43,22 @@ static int vcpu_get_tsc_scaling_frac_bits(void *data, u64 *val) DEFINE_SIMPLE_ATTRIBUTE(vcpu_tsc_scaling_frac_fops, vcpu_get_tsc_scaling_frac_bits, NULL, "%llu\n"); -void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) +void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry) { - debugfs_create_file("tsc-offset", 0444, vcpu->debugfs_dentry, vcpu, + debugfs_create_file("tsc-offset", 0444, debugfs_dentry, vcpu, &vcpu_tsc_offset_fops); if (lapic_in_kernel(vcpu)) debugfs_create_file("lapic_timer_advance_ns", 0444, - vcpu->debugfs_dentry, vcpu, + debugfs_dentry, vcpu, &vcpu_timer_advance_ns_fops); if (kvm_has_tsc_control) { debugfs_create_file("tsc-scaling-ratio", 0444, - vcpu->debugfs_dentry, vcpu, + debugfs_dentry, vcpu, &vcpu_tsc_scaling_fops); debugfs_create_file("tsc-scaling-ratio-frac-bits", 0444, - vcpu->debugfs_dentry, vcpu, + debugfs_dentry, vcpu, &vcpu_tsc_scaling_frac_fops); } } diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c index de5476f8683e..d0e2825ae617 100644 --- a/arch/x86/kvm/emulate.c +++ b/arch/x86/kvm/emulate.c @@ -4800,8 +4800,12 @@ static const struct opcode twobyte_table[256] = { GP(ModRM | DstReg | SrcMem | Mov | Sse, &pfx_0f_10_0f_11), GP(ModRM | DstMem | SrcReg | Mov | Sse, &pfx_0f_10_0f_11), N, N, N, N, N, N, - D(ImplicitOps | ModRM | SrcMem | NoAccess), - N, N, N, N, N, N, D(ImplicitOps | ModRM | SrcMem | NoAccess), + D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 4 * prefetch + 4 * reserved NOP */ + D(ImplicitOps | ModRM | SrcMem | NoAccess), N, N, + D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */ + D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */ + D(ImplicitOps | ModRM | SrcMem | NoAccess), /* 8 * reserved NOP */ + D(ImplicitOps | ModRM | SrcMem | NoAccess), /* NOP + 7 * reserved NOP */ /* 0x20 - 0x2F */ DIP(ModRM | DstMem | Priv | Op3264 | NoMod, cr_read, check_cr_read), DIP(ModRM | DstMem | Priv | Op3264 | NoMod, dr_read, check_dr_read), diff --git a/arch/x86/kvm/i8254.c b/arch/x86/kvm/i8254.c index febca334c320..a6e218c6140d 100644 --- a/arch/x86/kvm/i8254.c +++ b/arch/x86/kvm/i8254.c @@ -462,7 +462,6 @@ static int pit_ioport_write(struct kvm_vcpu *vcpu, if (channel == 3) { /* Read-Back Command. */ for (channel = 0; channel < 3; channel++) { - s = &pit_state->channels[channel]; if (val & (2 << channel)) { if (!(val & 0x20)) pit_latch_count(pit, channel); diff --git a/arch/x86/kvm/svm/nested.c b/arch/x86/kvm/svm/nested.c index 8a6db11dcb43..6bceafb19108 100644 --- a/arch/x86/kvm/svm/nested.c +++ b/arch/x86/kvm/svm/nested.c @@ -258,7 +258,7 @@ void sync_nested_vmcb_control(struct vcpu_svm *svm) /* Only a few fields of int_ctl are written by the processor. */ mask = V_IRQ_MASK | V_TPR_MASK; if (!(svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) && - is_intercept(svm, SVM_EXIT_VINTR)) { + is_intercept(svm, INTERCEPT_VINTR)) { /* * In order to request an interrupt window, L0 is usurping * svm->vmcb->control.int_ctl and possibly setting V_IRQ diff --git a/arch/x86/kvm/svm/svm.c b/arch/x86/kvm/svm/svm.c index 9e333b91ff78..8ccfa4197d9c 100644 --- a/arch/x86/kvm/svm/svm.c +++ b/arch/x86/kvm/svm/svm.c @@ -1378,6 +1378,8 @@ static void svm_clear_vintr(struct vcpu_svm *svm) /* Drop int_ctl fields related to VINTR injection. */ svm->vmcb->control.int_ctl &= mask; if (is_guest_mode(&svm->vcpu)) { + svm->nested.hsave->control.int_ctl &= mask; + WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != (svm->nested.ctl.int_ctl & V_TPR_MASK)); svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & ~mask; @@ -1837,7 +1839,7 @@ static void kvm_machine_check(void) .flags = X86_EFLAGS_IF, }; - do_machine_check(®s, 0); + do_machine_check(®s); #endif } @@ -1999,7 +2001,7 @@ void svm_set_gif(struct vcpu_svm *svm, bool value) */ if (vgif_enabled(svm)) clr_intercept(svm, INTERCEPT_STGI); - if (is_intercept(svm, SVM_EXIT_VINTR)) + if (is_intercept(svm, INTERCEPT_VINTR)) svm_clear_vintr(svm); enable_gif(svm); diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c index 9c74a732b08d..d1af20b050a8 100644 --- a/arch/x86/kvm/vmx/nested.c +++ b/arch/x86/kvm/vmx/nested.c @@ -3087,9 +3087,9 @@ static int nested_vmx_check_vmentry_hw(struct kvm_vcpu *vcpu) /* * VMExit clears RFLAGS.IF and DR7, even on a consistency check. */ - local_irq_enable(); if (hw_breakpoint_active()) set_debugreg(__this_cpu_read(cpu_dr7), 7); + local_irq_enable(); preempt_enable(); /* @@ -4624,19 +4624,24 @@ void nested_vmx_pmu_entry_exit_ctls_update(struct kvm_vcpu *vcpu) } } -static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer) +static int nested_vmx_get_vmptr(struct kvm_vcpu *vcpu, gpa_t *vmpointer, + int *ret) { gva_t gva; struct x86_exception e; + int r; if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmcs_read32(VMX_INSTRUCTION_INFO), false, - sizeof(*vmpointer), &gva)) - return 1; + sizeof(*vmpointer), &gva)) { + *ret = 1; + return -EINVAL; + } - if (kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; + r = kvm_read_guest_virt(vcpu, gva, vmpointer, sizeof(*vmpointer), &e); + if (r != X86EMUL_CONTINUE) { + *ret = vmx_handle_memory_failure(vcpu, r, &e); + return -EINVAL; } return 0; @@ -4764,8 +4769,8 @@ static int handle_vmon(struct kvm_vcpu *vcpu) return 1; } - if (nested_vmx_get_vmptr(vcpu, &vmptr)) - return 1; + if (nested_vmx_get_vmptr(vcpu, &vmptr, &ret)) + return ret; /* * SDM 3: 24.11.5 @@ -4838,12 +4843,13 @@ static int handle_vmclear(struct kvm_vcpu *vcpu) u32 zero = 0; gpa_t vmptr; u64 evmcs_gpa; + int r; if (!nested_vmx_check_permission(vcpu)) return 1; - if (nested_vmx_get_vmptr(vcpu, &vmptr)) - return 1; + if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) + return r; if (!page_address_valid(vcpu, vmptr)) return nested_vmx_failValid(vcpu, @@ -4902,7 +4908,7 @@ static int handle_vmread(struct kvm_vcpu *vcpu) u64 value; gva_t gva = 0; short offset; - int len; + int len, r; if (!nested_vmx_check_permission(vcpu)) return 1; @@ -4943,10 +4949,9 @@ static int handle_vmread(struct kvm_vcpu *vcpu) instr_info, true, len, &gva)) return 1; /* _system ok, nested_vmx_check_permission has verified cpl=0 */ - if (kvm_write_guest_virt_system(vcpu, gva, &value, len, &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_write_guest_virt_system(vcpu, gva, &value, len, &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); } return nested_vmx_succeed(vcpu); @@ -4987,7 +4992,7 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) unsigned long field; short offset; gva_t gva; - int len; + int len, r; /* * The value to write might be 32 or 64 bits, depending on L1's long @@ -5017,10 +5022,9 @@ static int handle_vmwrite(struct kvm_vcpu *vcpu) if (get_vmx_mem_address(vcpu, exit_qualification, instr_info, false, len, &gva)) return 1; - if (kvm_read_guest_virt(vcpu, gva, &value, len, &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_read_guest_virt(vcpu, gva, &value, len, &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); } field = kvm_register_readl(vcpu, (((instr_info) >> 28) & 0xf)); @@ -5103,12 +5107,13 @@ static int handle_vmptrld(struct kvm_vcpu *vcpu) { struct vcpu_vmx *vmx = to_vmx(vcpu); gpa_t vmptr; + int r; if (!nested_vmx_check_permission(vcpu)) return 1; - if (nested_vmx_get_vmptr(vcpu, &vmptr)) - return 1; + if (nested_vmx_get_vmptr(vcpu, &vmptr, &r)) + return r; if (!page_address_valid(vcpu, vmptr)) return nested_vmx_failValid(vcpu, @@ -5170,6 +5175,7 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) gpa_t current_vmptr = to_vmx(vcpu)->nested.current_vmptr; struct x86_exception e; gva_t gva; + int r; if (!nested_vmx_check_permission(vcpu)) return 1; @@ -5181,11 +5187,11 @@ static int handle_vmptrst(struct kvm_vcpu *vcpu) true, sizeof(gpa_t), &gva)) return 1; /* *_system ok, nested_vmx_check_permission has verified cpl=0 */ - if (kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, - sizeof(gpa_t), &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_write_guest_virt_system(vcpu, gva, (void *)¤t_vmptr, + sizeof(gpa_t), &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); + return nested_vmx_succeed(vcpu); } @@ -5209,7 +5215,7 @@ static int handle_invept(struct kvm_vcpu *vcpu) struct { u64 eptp, gpa; } operand; - int i; + int i, r; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_EPT) || @@ -5236,10 +5242,9 @@ static int handle_invept(struct kvm_vcpu *vcpu) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; - if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); /* * Nested EPT roots are always held through guest_mmu, @@ -5291,6 +5296,7 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) u64 gla; } operand; u16 vpid02; + int r; if (!(vmx->nested.msrs.secondary_ctls_high & SECONDARY_EXEC_ENABLE_VPID) || @@ -5318,10 +5324,10 @@ static int handle_invvpid(struct kvm_vcpu *vcpu) if (get_vmx_mem_address(vcpu, vmx_get_exit_qual(vcpu), vmx_instruction_info, false, sizeof(operand), &gva)) return 1; - if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); + if (operand.vpid >> 16) return nested_vmx_failValid(vcpu, VMXERR_INVALID_OPERAND_TO_INVEPT_INVVPID); @@ -5666,7 +5672,7 @@ static bool nested_vmx_l0_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) { u32 intr_info; - switch (exit_reason) { + switch ((u16)exit_reason) { case EXIT_REASON_EXCEPTION_NMI: intr_info = vmx_get_intr_info(vcpu); if (is_nmi(intr_info)) @@ -5727,7 +5733,7 @@ static bool nested_vmx_l1_wants_exit(struct kvm_vcpu *vcpu, u32 exit_reason) struct vmcs12 *vmcs12 = get_vmcs12(vcpu); u32 intr_info; - switch (exit_reason) { + switch ((u16)exit_reason) { case EXIT_REASON_EXCEPTION_NMI: intr_info = vmx_get_intr_info(vcpu); if (is_nmi(intr_info)) diff --git a/arch/x86/kvm/vmx/pmu_intel.c b/arch/x86/kvm/vmx/pmu_intel.c index d33d890b605f..bdcce65c7a1d 100644 --- a/arch/x86/kvm/vmx/pmu_intel.c +++ b/arch/x86/kvm/vmx/pmu_intel.c @@ -181,7 +181,7 @@ static bool intel_is_valid_msr(struct kvm_vcpu *vcpu, u32 msr) ret = pmu->version > 1; break; case MSR_IA32_PERF_CAPABILITIES: - ret = guest_cpuid_has(vcpu, X86_FEATURE_PDCM); + ret = 1; break; default: ret = get_gp_pmc(pmu, msr, MSR_IA32_PERFCTR0) || diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c index 170cc76a581f..36c771728c8c 100644 --- a/arch/x86/kvm/vmx/vmx.c +++ b/arch/x86/kvm/vmx/vmx.c @@ -1600,6 +1600,32 @@ static int skip_emulated_instruction(struct kvm_vcpu *vcpu) return 1; } +/* + * Handles kvm_read/write_guest_virt*() result and either injects #PF or returns + * KVM_EXIT_INTERNAL_ERROR for cases not currently handled by KVM. Return value + * indicates whether exit to userspace is needed. + */ +int vmx_handle_memory_failure(struct kvm_vcpu *vcpu, int r, + struct x86_exception *e) +{ + if (r == X86EMUL_PROPAGATE_FAULT) { + kvm_inject_emulated_page_fault(vcpu, e); + return 1; + } + + /* + * In case kvm_read/write_guest_virt*() failed with X86EMUL_IO_NEEDED + * while handling a VMX instruction KVM could've handled the request + * correctly by exiting to userspace and performing I/O but there + * doesn't seem to be a real use-case behind such requests, just return + * KVM_EXIT_INTERNAL_ERROR for now. + */ + vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; + vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION; + vcpu->run->internal.ndata = 0; + + return 0; +} /* * Recognizes a pending MTF VM-exit and records the nested state for later @@ -4683,7 +4709,7 @@ static void kvm_machine_check(void) .flags = X86_EFLAGS_IF, }; - do_machine_check(®s, 0); + do_machine_check(®s); #endif } @@ -5486,6 +5512,7 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) u64 pcid; u64 gla; } operand; + int r; if (!guest_cpuid_has(vcpu, X86_FEATURE_INVPCID)) { kvm_queue_exception(vcpu, UD_VECTOR); @@ -5508,10 +5535,9 @@ static int handle_invpcid(struct kvm_vcpu *vcpu) sizeof(operand), &gva)) return 1; - if (kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e)) { - kvm_inject_emulated_page_fault(vcpu, &e); - return 1; - } + r = kvm_read_guest_virt(vcpu, gva, &operand, sizeof(operand), &e); + if (r != X86EMUL_CONTINUE) + return vmx_handle_memory_failure(vcpu, r, &e); if (operand.pcid >> 12 != 0) { kvm_inject_gp(vcpu, 0); @@ -7282,10 +7308,6 @@ static __init void vmx_set_cpu_caps(void) if (vmx_pt_mode_is_host_guest()) kvm_cpu_cap_check_and_set(X86_FEATURE_INTEL_PT); - /* PKU is not yet implemented for shadow paging. */ - if (enable_ept && boot_cpu_has(X86_FEATURE_OSPKE)) - kvm_cpu_cap_check_and_set(X86_FEATURE_PKU); - if (vmx_umip_emulated()) kvm_cpu_cap_set(X86_FEATURE_UMIP); diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h index 672c28f17e49..8a83b5edc820 100644 --- a/arch/x86/kvm/vmx/vmx.h +++ b/arch/x86/kvm/vmx/vmx.h @@ -355,6 +355,8 @@ struct shared_msr_entry *find_msr_entry(struct vcpu_vmx *vmx, u32 msr); void pt_update_intercept_for_msr(struct vcpu_vmx *vmx); void vmx_update_host_rsp(struct vcpu_vmx *vmx, unsigned long host_rsp); int vmx_find_msr_index(struct vmx_msrs *m, u32 msr); +int vmx_handle_memory_failure(struct kvm_vcpu *vcpu, int r, + struct x86_exception *e); #define POSTED_INTR_ON 0 #define POSTED_INTR_SN 1 diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 9e41b5135340..00c88c2f34e4 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -239,8 +239,7 @@ u64 __read_mostly host_xcr0; u64 __read_mostly supported_xcr0; EXPORT_SYMBOL_GPL(supported_xcr0); -struct kmem_cache *x86_fpu_cache; -EXPORT_SYMBOL_GPL(x86_fpu_cache); +static struct kmem_cache *x86_fpu_cache; static struct kmem_cache *x86_emulator_cache; @@ -5647,13 +5646,6 @@ int kvm_write_guest_virt_system(struct kvm_vcpu *vcpu, gva_t addr, void *val, /* kvm_write_guest_virt_system can pull in tons of pages. */ vcpu->arch.l1tf_flush_l1d = true; - /* - * FIXME: this should call handle_emulation_failure if X86EMUL_IO_NEEDED - * is returned, but our callers are not ready for that and they blindly - * call kvm_inject_page_fault. Ensure that they at least do not leak - * uninitialized kernel stack memory into cr2 and error code. - */ - memset(exception, 0, sizeof(*exception)); return kvm_write_guest_virt_helper(addr, val, bytes, vcpu, PFERR_WRITE_MASK, exception); } @@ -7018,7 +7010,7 @@ restart: if (!ctxt->have_exception || exception_type(ctxt->exception.vector) == EXCPT_TRAP) { kvm_rip_write(vcpu, ctxt->eip); - if (r && ctxt->tf) + if (r && (ctxt->tf || (vcpu->guest_debug & KVM_GUESTDBG_SINGLESTEP))) r = kvm_vcpu_do_singlestep(vcpu); if (kvm_x86_ops.update_emulated_instruction) kvm_x86_ops.update_emulated_instruction(vcpu); @@ -8277,9 +8269,8 @@ static void vcpu_load_eoi_exitmap(struct kvm_vcpu *vcpu) kvm_x86_ops.load_eoi_exitmap(vcpu, eoi_exit_bitmap); } -int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, - unsigned long start, unsigned long end, - bool blockable) +void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, + unsigned long start, unsigned long end) { unsigned long apic_address; @@ -8290,8 +8281,6 @@ int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, apic_address = gfn_to_hva(kvm, APIC_DEFAULT_PHYS_BASE >> PAGE_SHIFT); if (start <= apic_address && apic_address < end) kvm_make_all_cpus_request(kvm, KVM_REQ_APIC_PAGE_RELOAD); - - return 0; } void kvm_vcpu_reload_apic_access_page(struct kvm_vcpu *vcpu) @@ -9962,13 +9951,8 @@ int __x86_set_memory_region(struct kvm *kvm, int id, gpa_t gpa, u32 size) if (!slot || !slot->npages) return 0; - /* - * Stuff a non-canonical value to catch use-after-delete. This - * ends up being 0 on 32-bit KVM, but there's no better - * alternative. - */ - hva = (unsigned long)(0xdeadull << 48); old_npages = slot->npages; + hva = 0; } for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) { @@ -10140,43 +10124,65 @@ int kvm_arch_prepare_memory_region(struct kvm *kvm, } static void kvm_mmu_slot_apply_flags(struct kvm *kvm, - struct kvm_memory_slot *new) + struct kvm_memory_slot *old, + struct kvm_memory_slot *new, + enum kvm_mr_change change) { - /* Still write protect RO slot */ - if (new->flags & KVM_MEM_READONLY) { - kvm_mmu_slot_remove_write_access(kvm, new, PG_LEVEL_4K); + /* + * Nothing to do for RO slots or CREATE/MOVE/DELETE of a slot. + * See comments below. + */ + if ((change != KVM_MR_FLAGS_ONLY) || (new->flags & KVM_MEM_READONLY)) return; - } /* - * Call kvm_x86_ops dirty logging hooks when they are valid. - * - * kvm_x86_ops.slot_disable_log_dirty is called when: - * - * - KVM_MR_CREATE with dirty logging is disabled - * - KVM_MR_FLAGS_ONLY with dirty logging is disabled in new flag - * - * The reason is, in case of PML, we need to set D-bit for any slots - * with dirty logging disabled in order to eliminate unnecessary GPA - * logging in PML buffer (and potential PML buffer full VMEXIT). This - * guarantees leaving PML enabled during guest's lifetime won't have - * any additional overhead from PML when guest is running with dirty - * logging disabled for memory slots. + * Dirty logging tracks sptes in 4k granularity, meaning that large + * sptes have to be split. If live migration is successful, the guest + * in the source machine will be destroyed and large sptes will be + * created in the destination. However, if the guest continues to run + * in the source machine (for example if live migration fails), small + * sptes will remain around and cause bad performance. * - * kvm_x86_ops.slot_enable_log_dirty is called when switching new slot - * to dirty logging mode. + * Scan sptes if dirty logging has been stopped, dropping those + * which can be collapsed into a single large-page spte. Later + * page faults will create the large-page sptes. * - * If kvm_x86_ops dirty logging hooks are invalid, use write protect. + * There is no need to do this in any of the following cases: + * CREATE: No dirty mappings will already exist. + * MOVE/DELETE: The old mappings will already have been cleaned up by + * kvm_arch_flush_shadow_memslot() + */ + if ((old->flags & KVM_MEM_LOG_DIRTY_PAGES) && + !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) + kvm_mmu_zap_collapsible_sptes(kvm, new); + + /* + * Enable or disable dirty logging for the slot. * - * In case of write protect: + * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of the old + * slot have been zapped so no dirty logging updates are needed for + * the old slot. + * For KVM_MR_CREATE and KVM_MR_MOVE, once the new slot is visible + * any mappings that might be created in it will consume the + * properties of the new slot and do not need to be updated here. * - * Write protect all pages for dirty logging. + * When PML is enabled, the kvm_x86_ops dirty logging hooks are + * called to enable/disable dirty logging. * - * All the sptes including the large sptes which point to this - * slot are set to readonly. We can not create any new large - * spte on this slot until the end of the logging. + * When disabling dirty logging with PML enabled, the D-bit is set + * for sptes in the slot in order to prevent unnecessary GPA + * logging in the PML buffer (and potential PML buffer full VMEXIT). + * This guarantees leaving PML enabled for the guest's lifetime + * won't have any additional overhead from PML when the guest is + * running with dirty logging disabled. * + * When enabling dirty logging, large sptes are write-protected + * so they can be split on first write. New large sptes cannot + * be created for this slot until the end of the logging. * See the comments in fast_page_fault(). + * For small sptes, nothing is done if the dirty log is in the + * initial-all-set state. Otherwise, depending on whether pml + * is enabled the D-bit or the W-bit will be cleared. */ if (new->flags & KVM_MEM_LOG_DIRTY_PAGES) { if (kvm_x86_ops.slot_enable_log_dirty) { @@ -10213,39 +10219,9 @@ void kvm_arch_commit_memory_region(struct kvm *kvm, kvm_mmu_calculate_default_mmu_pages(kvm)); /* - * Dirty logging tracks sptes in 4k granularity, meaning that large - * sptes have to be split. If live migration is successful, the guest - * in the source machine will be destroyed and large sptes will be - * created in the destination. However, if the guest continues to run - * in the source machine (for example if live migration fails), small - * sptes will remain around and cause bad performance. - * - * Scan sptes if dirty logging has been stopped, dropping those - * which can be collapsed into a single large-page spte. Later - * page faults will create the large-page sptes. - * - * There is no need to do this in any of the following cases: - * CREATE: No dirty mappings will already exist. - * MOVE/DELETE: The old mappings will already have been cleaned up by - * kvm_arch_flush_shadow_memslot() - */ - if (change == KVM_MR_FLAGS_ONLY && - (old->flags & KVM_MEM_LOG_DIRTY_PAGES) && - !(new->flags & KVM_MEM_LOG_DIRTY_PAGES)) - kvm_mmu_zap_collapsible_sptes(kvm, new); - - /* - * Set up write protection and/or dirty logging for the new slot. - * - * For KVM_MR_DELETE and KVM_MR_MOVE, the shadow pages of old slot have - * been zapped so no dirty logging staff is needed for old slot. For - * KVM_MR_FLAGS_ONLY, the old slot is essentially the same one as the - * new and it's also covered when dealing with the new slot. - * * FIXME: const-ify all uses of struct kvm_memory_slot. */ - if (change != KVM_MR_DELETE) - kvm_mmu_slot_apply_flags(kvm, (struct kvm_memory_slot *) new); + kvm_mmu_slot_apply_flags(kvm, old, (struct kvm_memory_slot *) new, change); /* Free the arrays associated with the old memslot. */ if (change == KVM_MR_MOVE) @@ -10530,7 +10506,7 @@ bool kvm_can_do_async_pf(struct kvm_vcpu *vcpu) return kvm_arch_interrupt_allowed(vcpu); } -void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, +bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, struct kvm_async_pf *work) { struct x86_exception fault; @@ -10547,6 +10523,7 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, fault.address = work->arch.token; fault.async_page_fault = true; kvm_inject_page_fault(vcpu, &fault); + return true; } else { /* * It is not possible to deliver a paravirtualized asynchronous @@ -10557,6 +10534,7 @@ void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, * fault is retried, hopefully the page will be ready in the host. */ kvm_make_request(KVM_REQ_APF_HALT, vcpu); + return false; } } @@ -10574,7 +10552,8 @@ void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, kvm_del_async_pf_gfn(vcpu, work->arch.gfn); trace_kvm_async_pf_ready(work->arch.token, work->cr2_or_gpa); - if (kvm_pv_async_pf_enabled(vcpu) && + if ((work->wakeup_all || work->notpresent_injected) && + kvm_pv_async_pf_enabled(vcpu) && !apf_put_user_ready(vcpu, work->arch.token)) { vcpu->arch.apf.pageready_pending = true; kvm_apic_set_irq(vcpu, &irq, NULL); diff --git a/arch/x86/lib/Makefile b/arch/x86/lib/Makefile index 5246db42de45..6110bce7237b 100644 --- a/arch/x86/lib/Makefile +++ b/arch/x86/lib/Makefile @@ -6,10 +6,19 @@ # Produces uninteresting flaky coverage. KCOV_INSTRUMENT_delay.o := n +# KCSAN uses udelay for introducing watchpoint delay; avoid recursion. +KCSAN_SANITIZE_delay.o := n +ifdef CONFIG_KCSAN +# In case KCSAN+lockdep+ftrace are enabled, disable ftrace for delay.o to avoid +# lockdep -> [other libs] -> KCSAN -> udelay -> ftrace -> lockdep recursion. +CFLAGS_REMOVE_delay.o = $(CC_FLAGS_FTRACE) +endif + # Early boot use of cmdline; don't instrument it ifdef CONFIG_AMD_MEM_ENCRYPT KCOV_INSTRUMENT_cmdline.o := n KASAN_SANITIZE_cmdline.o := n +KCSAN_SANITIZE_cmdline.o := n ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_cmdline.o = -pg diff --git a/arch/x86/mm/Makefile b/arch/x86/mm/Makefile index 98f7c6fa2eaa..f7fd0e868c9c 100644 --- a/arch/x86/mm/Makefile +++ b/arch/x86/mm/Makefile @@ -7,6 +7,10 @@ KCOV_INSTRUMENT_mem_encrypt_identity.o := n KASAN_SANITIZE_mem_encrypt.o := n KASAN_SANITIZE_mem_encrypt_identity.o := n +# Disable KCSAN entirely, because otherwise we get warnings that some functions +# reference __initdata sections. +KCSAN_SANITIZE := n + ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_mem_encrypt.o = -pg CFLAGS_REMOVE_mem_encrypt_identity.o = -pg diff --git a/arch/x86/mm/cpu_entry_area.c b/arch/x86/mm/cpu_entry_area.c index 6f8b48f545f4..770b613790b3 100644 --- a/arch/x86/mm/cpu_entry_area.c +++ b/arch/x86/mm/cpu_entry_area.c @@ -107,7 +107,6 @@ static void __init percpu_setup_exception_stacks(unsigned int cpu) */ cea_map_stack(DF); cea_map_stack(NMI); - cea_map_stack(DB1); cea_map_stack(DB); cea_map_stack(MCE); } diff --git a/arch/x86/mm/extable.c b/arch/x86/mm/extable.c index b991aa4bdfae..1d6cb07f4f86 100644 --- a/arch/x86/mm/extable.c +++ b/arch/x86/mm/extable.c @@ -204,8 +204,19 @@ void __init early_fixup_exception(struct pt_regs *regs, int trapnr) if (fixup_exception(regs, trapnr, regs->orig_ax, 0)) return; - if (fixup_bug(regs, trapnr)) - return; + if (trapnr == X86_TRAP_UD) { + if (report_bug(regs->ip, regs) == BUG_TRAP_TYPE_WARN) { + /* Skip the ud2. */ + regs->ip += LEN_UD2; + return; + } + + /* + * If this was a BUG and report_bug returns or if this + * was just a normal #UD, we want to continue onward and + * crash. + */ + } fail: early_printk("PANIC: early exception 0x%02x IP %lx:%lx error %lx cr2 0x%lx\n", diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 0b03ae8c39cd..66be9bd60307 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -414,21 +414,13 @@ static int is_errata100(struct pt_regs *regs, unsigned long address) return 0; } +/* Pentium F0 0F C7 C8 bug workaround: */ static int is_f00f_bug(struct pt_regs *regs, unsigned long address) { #ifdef CONFIG_X86_F00F_BUG - unsigned long nr; - - /* - * Pentium F0 0F C7 C8 bug workaround: - */ - if (boot_cpu_has_bug(X86_BUG_F00F)) { - nr = (address - idt_descr.address) >> 3; - - if (nr == 6) { - do_invalid_op(regs, 0); - return 1; - } + if (boot_cpu_has_bug(X86_BUG_F00F) && idt_is_f00f_address(address)) { + handle_invalid_op(regs); + return 1; } #endif return 0; @@ -786,6 +778,8 @@ __bad_area_nosemaphore(struct pt_regs *regs, unsigned long error_code, force_sig_fault(SIGSEGV, si_code, (void __user *)address); + local_irq_disable(); + return; } @@ -1355,11 +1349,38 @@ trace_page_fault_entries(struct pt_regs *regs, unsigned long error_code, trace_page_fault_kernel(address, regs, error_code); } -dotraplinkage void -do_page_fault(struct pt_regs *regs, unsigned long hw_error_code, - unsigned long address) +static __always_inline void +handle_page_fault(struct pt_regs *regs, unsigned long error_code, + unsigned long address) { + trace_page_fault_entries(regs, error_code, address); + + if (unlikely(kmmio_fault(regs, address))) + return; + + /* Was the fault on kernel-controlled part of the address space? */ + if (unlikely(fault_in_kernel_space(address))) { + do_kern_addr_fault(regs, error_code, address); + } else { + do_user_addr_fault(regs, error_code, address); + /* + * User address page fault handling might have reenabled + * interrupts. Fixing up all potential exit points of + * do_user_addr_fault() and its leaf functions is just not + * doable w/o creating an unholy mess or turning the code + * upside down. + */ + local_irq_disable(); + } +} + +DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault) +{ + unsigned long address = read_cr2(); + bool rcu_exit; + prefetchw(¤t->mm->mmap_lock); + /* * KVM has two types of events that are, logically, interrupts, but * are unfortunately delivered using the #PF vector. These events are @@ -1374,19 +1395,28 @@ do_page_fault(struct pt_regs *regs, unsigned long hw_error_code, * getting values from real and async page faults mixed up. * * Fingers crossed. + * + * The async #PF handling code takes care of idtentry handling + * itself. */ if (kvm_handle_async_pf(regs, (u32)address)) return; - trace_page_fault_entries(regs, hw_error_code, address); + /* + * Entry handling for valid #PF from kernel mode is slightly + * different: RCU is already watching and rcu_irq_enter() must not + * be invoked because a kernel fault on a user space address might + * sleep. + * + * In case the fault hit a RCU idle region the conditional entry + * code reenabled RCU to avoid subsequent wreckage which helps + * debugability. + */ + rcu_exit = idtentry_enter_cond_rcu(regs); - if (unlikely(kmmio_fault(regs, address))) - return; + instrumentation_begin(); + handle_page_fault(regs, error_code, address); + instrumentation_end(); - /* Was the fault on kernel-controlled part of the address space? */ - if (unlikely(fault_in_kernel_space(address))) - do_kern_addr_fault(regs, hw_error_code, address); - else - do_user_addr_fault(regs, hw_error_code, address); + idtentry_exit_cond_rcu(regs, rcu_exit); } -NOKPROBE_SYMBOL(do_page_fault); diff --git a/arch/x86/mm/pti.c b/arch/x86/mm/pti.c index a3c6757a65c7..a8a924b3c335 100644 --- a/arch/x86/mm/pti.c +++ b/arch/x86/mm/pti.c @@ -492,12 +492,12 @@ static void __init pti_setup_espfix64(void) } /* - * Clone the populated PMDs of the entry and irqentry text and force it RO. + * Clone the populated PMDs of the entry text and force it RO. */ static void pti_clone_entry_text(void) { pti_clone_pgtable((unsigned long) __entry_text_start, - (unsigned long) __irqentry_text_end, + (unsigned long) __entry_text_end, PTI_CLONE_PMD); } diff --git a/arch/x86/platform/uv/tlb_uv.c b/arch/x86/platform/uv/tlb_uv.c index 4ea69690c3e4..0ac96ca304c7 100644 --- a/arch/x86/platform/uv/tlb_uv.c +++ b/arch/x86/platform/uv/tlb_uv.c @@ -1272,7 +1272,7 @@ static void process_uv2_message(struct msg_desc *mdp, struct bau_control *bcp) * (the resource will not be freed until noninterruptable cpus see this * interrupt; hardware may timeout the s/w ack and reply ERROR) */ -void uv_bau_message_interrupt(struct pt_regs *regs) +DEFINE_IDTENTRY_SYSVEC(sysvec_uv_bau_message) { int count = 0; cycles_t time_start; diff --git a/arch/x86/purgatory/.gitignore b/arch/x86/purgatory/.gitignore new file mode 100644 index 000000000000..d2be1500671d --- /dev/null +++ b/arch/x86/purgatory/.gitignore @@ -0,0 +1 @@ +purgatory.chk diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile index fb4ee5444379..b04e6e72a592 100644 --- a/arch/x86/purgatory/Makefile +++ b/arch/x86/purgatory/Makefile @@ -14,10 +14,18 @@ $(obj)/sha256.o: $(srctree)/lib/crypto/sha256.c FORCE CFLAGS_sha256.o := -D__DISABLE_EXPORTS -LDFLAGS_purgatory.ro := -e purgatory_start -r --no-undefined -nostdlib -z nodefaultlib -targets += purgatory.ro - +# When linking purgatory.ro with -r unresolved symbols are not checked, +# also link a purgatory.chk binary without -r to check for unresolved symbols. +PURGATORY_LDFLAGS := -e purgatory_start -nostdlib -z nodefaultlib +LDFLAGS_purgatory.ro := -r $(PURGATORY_LDFLAGS) +LDFLAGS_purgatory.chk := $(PURGATORY_LDFLAGS) +targets += purgatory.ro purgatory.chk + +# Sanitizer, etc. runtimes are unavailable and cannot be linked here. +GCOV_PROFILE := n KASAN_SANITIZE := n +UBSAN_SANITIZE := n +KCSAN_SANITIZE := n KCOV_INSTRUMENT := n # These are adjustments to the compiler flags used for objects that @@ -25,7 +33,7 @@ KCOV_INSTRUMENT := n PURGATORY_CFLAGS_REMOVE := -mcmodel=kernel PURGATORY_CFLAGS := -mcmodel=large -ffreestanding -fno-zero-initialized-in-bss -PURGATORY_CFLAGS += $(DISABLE_STACKLEAK_PLUGIN) +PURGATORY_CFLAGS += $(DISABLE_STACKLEAK_PLUGIN) -DDISABLE_BRANCH_PROFILING # Default KBUILD_CFLAGS can have -pg option set when FTRACE is enabled. That # in turn leaves some undefined symbols like __fentry__ in purgatory and not @@ -58,12 +66,15 @@ CFLAGS_string.o += $(PURGATORY_CFLAGS) $(obj)/purgatory.ro: $(PURGATORY_OBJS) FORCE $(call if_changed,ld) +$(obj)/purgatory.chk: $(obj)/purgatory.ro FORCE + $(call if_changed,ld) + targets += kexec-purgatory.c quiet_cmd_bin2c = BIN2C $@ cmd_bin2c = $(objtree)/scripts/bin2c kexec_purgatory < $< > $@ -$(obj)/kexec-purgatory.c: $(obj)/purgatory.ro FORCE +$(obj)/kexec-purgatory.c: $(obj)/purgatory.ro $(obj)/purgatory.chk FORCE $(call if_changed,bin2c) obj-$(CONFIG_KEXEC_FILE) += kexec-purgatory.o diff --git a/arch/x86/realmode/Makefile b/arch/x86/realmode/Makefile index 682c895753d9..6b1f3a4eeb44 100644 --- a/arch/x86/realmode/Makefile +++ b/arch/x86/realmode/Makefile @@ -6,7 +6,10 @@ # for more details. # # + +# Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n +KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y subdir- := rm diff --git a/arch/x86/realmode/rm/Makefile b/arch/x86/realmode/rm/Makefile index b11ec5d8f8ac..83f1b6a56449 100644 --- a/arch/x86/realmode/rm/Makefile +++ b/arch/x86/realmode/rm/Makefile @@ -6,7 +6,10 @@ # for more details. # # + +# Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n +KCSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y # Prevents link failures: __sanitizer_cov_trace_pc() is not linked in. diff --git a/arch/x86/xen/enlighten_hvm.c b/arch/x86/xen/enlighten_hvm.c index e138f7de52d2..3e89b0067ff0 100644 --- a/arch/x86/xen/enlighten_hvm.c +++ b/arch/x86/xen/enlighten_hvm.c @@ -13,6 +13,7 @@ #include <asm/smp.h> #include <asm/reboot.h> #include <asm/setup.h> +#include <asm/idtentry.h> #include <asm/hypervisor.h> #include <asm/e820/api.h> #include <asm/early_ioremap.h> @@ -118,6 +119,17 @@ static void __init init_hvm_pv_info(void) this_cpu_write(xen_vcpu_id, smp_processor_id()); } +DEFINE_IDTENTRY_SYSVEC(sysvec_xen_hvm_callback) +{ + struct pt_regs *old_regs = set_irq_regs(regs); + + inc_irq_stat(irq_hv_callback_count); + + xen_hvm_evtchn_do_upcall(); + + set_irq_regs(old_regs); +} + #ifdef CONFIG_KEXEC_CORE static void xen_hvm_shutdown(void) { diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c index c2c97faaf004..33b309d65955 100644 --- a/arch/x86/xen/enlighten_pv.c +++ b/arch/x86/xen/enlighten_pv.c @@ -604,32 +604,42 @@ struct trap_array_entry { bool ist_okay; }; +#define TRAP_ENTRY(func, ist_ok) { \ + .orig = asm_##func, \ + .xen = xen_asm_##func, \ + .ist_okay = ist_ok } + +#define TRAP_ENTRY_REDIR(func, xenfunc, ist_ok) { \ + .orig = asm_##func, \ + .xen = xen_asm_##xenfunc, \ + .ist_okay = ist_ok } + static struct trap_array_entry trap_array[] = { - { debug, xen_xendebug, true }, - { double_fault, xen_double_fault, true }, + TRAP_ENTRY_REDIR(exc_debug, exc_xendebug, true ), + TRAP_ENTRY(exc_double_fault, true ), #ifdef CONFIG_X86_MCE - { machine_check, xen_machine_check, true }, + TRAP_ENTRY(exc_machine_check, true ), #endif - { nmi, xen_xennmi, true }, - { int3, xen_int3, false }, - { overflow, xen_overflow, false }, + TRAP_ENTRY_REDIR(exc_nmi, exc_xennmi, true ), + TRAP_ENTRY(exc_int3, false ), + TRAP_ENTRY(exc_overflow, false ), #ifdef CONFIG_IA32_EMULATION { entry_INT80_compat, xen_entry_INT80_compat, false }, #endif - { page_fault, xen_page_fault, false }, - { divide_error, xen_divide_error, false }, - { bounds, xen_bounds, false }, - { invalid_op, xen_invalid_op, false }, - { device_not_available, xen_device_not_available, false }, - { coprocessor_segment_overrun, xen_coprocessor_segment_overrun, false }, - { invalid_TSS, xen_invalid_TSS, false }, - { segment_not_present, xen_segment_not_present, false }, - { stack_segment, xen_stack_segment, false }, - { general_protection, xen_general_protection, false }, - { spurious_interrupt_bug, xen_spurious_interrupt_bug, false }, - { coprocessor_error, xen_coprocessor_error, false }, - { alignment_check, xen_alignment_check, false }, - { simd_coprocessor_error, xen_simd_coprocessor_error, false }, + TRAP_ENTRY(exc_page_fault, false ), + TRAP_ENTRY(exc_divide_error, false ), + TRAP_ENTRY(exc_bounds, false ), + TRAP_ENTRY(exc_invalid_op, false ), + TRAP_ENTRY(exc_device_not_available, false ), + TRAP_ENTRY(exc_coproc_segment_overrun, false ), + TRAP_ENTRY(exc_invalid_tss, false ), + TRAP_ENTRY(exc_segment_not_present, false ), + TRAP_ENTRY(exc_stack_segment, false ), + TRAP_ENTRY(exc_general_protection, false ), + TRAP_ENTRY(exc_spurious_interrupt_bug, false ), + TRAP_ENTRY(exc_coprocessor_error, false ), + TRAP_ENTRY(exc_alignment_check, false ), + TRAP_ENTRY(exc_simd_coprocessor_error, false ), }; static bool __ref get_trap_addr(void **addr, unsigned int ist) @@ -641,7 +651,7 @@ static bool __ref get_trap_addr(void **addr, unsigned int ist) * Replace trap handler addresses by Xen specific ones. * Check for known traps using IST and whitelist them. * The debugger ones are the only ones we care about. - * Xen will handle faults like double_fault, * so we should never see + * Xen will handle faults like double_fault, so we should never see * them. Warn if there's an unexpected IST-using fault handler. */ for (nr = 0; nr < ARRAY_SIZE(trap_array); nr++) { diff --git a/arch/x86/xen/setup.c b/arch/x86/xen/setup.c index 1a2d8a50dac4..3566e37241d7 100644 --- a/arch/x86/xen/setup.c +++ b/arch/x86/xen/setup.c @@ -20,6 +20,7 @@ #include <asm/setup.h> #include <asm/acpi.h> #include <asm/numa.h> +#include <asm/idtentry.h> #include <asm/xen/hypervisor.h> #include <asm/xen/hypercall.h> @@ -993,7 +994,8 @@ static void __init xen_pvmmu_arch_setup(void) HYPERVISOR_vm_assist(VMASST_CMD_enable, VMASST_TYPE_pae_extended_cr3); - if (register_callback(CALLBACKTYPE_event, xen_hypervisor_callback) || + if (register_callback(CALLBACKTYPE_event, + xen_asm_exc_xen_hypervisor_callback) || register_callback(CALLBACKTYPE_failsafe, xen_failsafe_callback)) BUG(); diff --git a/arch/x86/xen/smp_pv.c b/arch/x86/xen/smp_pv.c index 8fa01c545460..171aff1b11f2 100644 --- a/arch/x86/xen/smp_pv.c +++ b/arch/x86/xen/smp_pv.c @@ -26,6 +26,7 @@ #include <linux/pgtable.h> #include <asm/paravirt.h> +#include <asm/idtentry.h> #include <asm/desc.h> #include <asm/cpu.h> @@ -348,7 +349,7 @@ cpu_initialize_context(unsigned int cpu, struct task_struct *idle) ctxt->gs_base_kernel = per_cpu_offset(cpu); #endif ctxt->event_callback_eip = - (unsigned long)xen_hypervisor_callback; + (unsigned long)xen_asm_exc_xen_hypervisor_callback; ctxt->failsafe_callback_eip = (unsigned long)xen_failsafe_callback; per_cpu(xen_cr3, cpu) = __pa(swapper_pg_dir); diff --git a/arch/x86/xen/suspend_hvm.c b/arch/x86/xen/suspend_hvm.c index e666b614cf6d..9d548b0c772f 100644 --- a/arch/x86/xen/suspend_hvm.c +++ b/arch/x86/xen/suspend_hvm.c @@ -2,6 +2,7 @@ #include <linux/types.h> #include <xen/xen.h> +#include <xen/hvm.h> #include <xen/features.h> #include <xen/interface/features.h> @@ -13,6 +14,6 @@ void xen_hvm_post_suspend(int suspend_cancelled) xen_hvm_init_shared_info(); xen_vcpu_restore(); } - xen_callback_vector(); + xen_setup_callback_vector(); xen_unplug_emulated_devices(); } diff --git a/arch/x86/xen/xen-asm_32.S b/arch/x86/xen/xen-asm_32.S index 2712e9155306..4757cec33abe 100644 --- a/arch/x86/xen/xen-asm_32.S +++ b/arch/x86/xen/xen-asm_32.S @@ -93,7 +93,7 @@ xen_iret_start_crit: /* * If there's something pending, mask events again so we can - * jump back into xen_hypervisor_callback. Otherwise do not + * jump back into exc_xen_hypervisor_callback. Otherwise do not * touch XEN_vcpu_info_mask. */ jne 1f @@ -113,11 +113,11 @@ iret_restore_end: * Events are masked, so jumping out of the critical region is * OK. */ - je xen_hypervisor_callback + je xen_asm_exc_xen_hypervisor_callback 1: iret xen_iret_end_crit: - _ASM_EXTABLE(1b, iret_exc) + _ASM_EXTABLE(1b, asm_iret_error) hyper_iret: /* put this out of line since its very rarely used */ @@ -127,7 +127,7 @@ SYM_CODE_END(xen_iret) .globl xen_iret_start_crit, xen_iret_end_crit /* - * This is called by xen_hypervisor_callback in entry_32.S when it sees + * This is called by xen_asm_exc_xen_hypervisor_callback in entry_32.S when it sees * that the EIP at the time of interrupt was between * xen_iret_start_crit and xen_iret_end_crit. * @@ -144,7 +144,7 @@ SYM_CODE_END(xen_iret) * eflags } * cs } nested exception info * eip } - * return address : (into xen_hypervisor_callback) + * return address : (into xen_asm_exc_xen_hypervisor_callback) * * In order to deliver the nested exception properly, we need to discard the * nested exception frame such that when we handle the exception, we do it @@ -152,7 +152,8 @@ SYM_CODE_END(xen_iret) * * The only caveat is that if the outer eax hasn't been restored yet (i.e. * it's still on stack), we need to restore its value here. - */ +*/ +.pushsection .noinstr.text, "ax" SYM_CODE_START(xen_iret_crit_fixup) /* * Paranoia: Make sure we're really coming from kernel space. @@ -181,3 +182,4 @@ SYM_CODE_START(xen_iret_crit_fixup) 2: ret SYM_CODE_END(xen_iret_crit_fixup) +.popsection diff --git a/arch/x86/xen/xen-asm_64.S b/arch/x86/xen/xen-asm_64.S index 0a0fd168683a..5d252aaeade8 100644 --- a/arch/x86/xen/xen-asm_64.S +++ b/arch/x86/xen/xen-asm_64.S @@ -28,33 +28,33 @@ SYM_CODE_END(xen_\name) _ASM_NOKPROBE(xen_\name) .endm -xen_pv_trap divide_error -xen_pv_trap debug -xen_pv_trap xendebug -xen_pv_trap int3 -xen_pv_trap xennmi -xen_pv_trap overflow -xen_pv_trap bounds -xen_pv_trap invalid_op -xen_pv_trap device_not_available -xen_pv_trap double_fault -xen_pv_trap coprocessor_segment_overrun -xen_pv_trap invalid_TSS -xen_pv_trap segment_not_present -xen_pv_trap stack_segment -xen_pv_trap general_protection -xen_pv_trap page_fault -xen_pv_trap spurious_interrupt_bug -xen_pv_trap coprocessor_error -xen_pv_trap alignment_check +xen_pv_trap asm_exc_divide_error +xen_pv_trap asm_exc_debug +xen_pv_trap asm_exc_xendebug +xen_pv_trap asm_exc_int3 +xen_pv_trap asm_exc_xennmi +xen_pv_trap asm_exc_overflow +xen_pv_trap asm_exc_bounds +xen_pv_trap asm_exc_invalid_op +xen_pv_trap asm_exc_device_not_available +xen_pv_trap asm_exc_double_fault +xen_pv_trap asm_exc_coproc_segment_overrun +xen_pv_trap asm_exc_invalid_tss +xen_pv_trap asm_exc_segment_not_present +xen_pv_trap asm_exc_stack_segment +xen_pv_trap asm_exc_general_protection +xen_pv_trap asm_exc_page_fault +xen_pv_trap asm_exc_spurious_interrupt_bug +xen_pv_trap asm_exc_coprocessor_error +xen_pv_trap asm_exc_alignment_check #ifdef CONFIG_X86_MCE -xen_pv_trap machine_check +xen_pv_trap asm_exc_machine_check #endif /* CONFIG_X86_MCE */ -xen_pv_trap simd_coprocessor_error +xen_pv_trap asm_exc_simd_coprocessor_error #ifdef CONFIG_IA32_EMULATION xen_pv_trap entry_INT80_compat #endif -xen_pv_trap hypervisor_callback +xen_pv_trap asm_exc_xen_hypervisor_callback __INIT SYM_CODE_START(xen_early_idt_handler_array) diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h index 45a441c33d6d..53b224fd6177 100644 --- a/arch/x86/xen/xen-ops.h +++ b/arch/x86/xen/xen-ops.h @@ -8,7 +8,6 @@ #include <xen/xen-ops.h> /* These are code, but not functions. Defined in entry.S */ -extern const char xen_hypervisor_callback[]; extern const char xen_failsafe_callback[]; void xen_sysenter_target(void); @@ -55,7 +54,6 @@ void xen_enable_sysenter(void); void xen_enable_syscall(void); void xen_vcpu_restore(void); -void xen_callback_vector(void); void xen_hvm_init_shared_info(void); void xen_unplug_emulated_devices(void); diff --git a/block/bio-integrity.c b/block/bio-integrity.c index 3579ac0f6ec1..23632a33ed39 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -281,7 +281,6 @@ bool bio_integrity_prep(struct bio *bio) if (ret == 0) { printk(KERN_ERR "could not attach integrity payload\n"); - kfree(buf); status = BLK_STS_RESOURCE; goto err_end_io; } diff --git a/block/bio.c b/block/bio.c index 5235da6434aa..a7366c02c9b5 100644 --- a/block/bio.c +++ b/block/bio.c @@ -1434,8 +1434,7 @@ again: } if (bio->bi_disk && bio_flagged(bio, BIO_TRACE_COMPLETION)) { - trace_block_bio_complete(bio->bi_disk->queue, bio, - blk_status_to_errno(bio->bi_status)); + trace_block_bio_complete(bio->bi_disk->queue, bio); bio_clear_flag(bio, BIO_TRACE_COMPLETION); } diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 96a39d0724a2..44f3d0967cb4 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -191,6 +191,33 @@ found_tag: return tag + tag_offset; } +bool __blk_mq_get_driver_tag(struct request *rq) +{ + struct sbitmap_queue *bt = &rq->mq_hctx->tags->bitmap_tags; + unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags; + bool shared = blk_mq_tag_busy(rq->mq_hctx); + int tag; + + if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) { + bt = &rq->mq_hctx->tags->breserved_tags; + tag_offset = 0; + } + + if (!hctx_may_queue(rq->mq_hctx, bt)) + return false; + tag = __sbitmap_queue_get(bt); + if (tag == BLK_MQ_NO_TAG) + return false; + + rq->tag = tag + tag_offset; + if (shared) { + rq->rq_flags |= RQF_MQ_INFLIGHT; + atomic_inc(&rq->mq_hctx->nr_active); + } + rq->mq_hctx->tags->rqs[rq->tag] = rq; + return true; +} + void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx, unsigned int tag) { @@ -269,6 +296,7 @@ struct bt_tags_iter_data { #define BT_TAG_ITER_RESERVED (1 << 0) #define BT_TAG_ITER_STARTED (1 << 1) +#define BT_TAG_ITER_STATIC_RQS (1 << 2) static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) { @@ -282,9 +310,12 @@ static bool bt_tags_iter(struct sbitmap *bitmap, unsigned int bitnr, void *data) /* * We can hit rq == NULL here, because the tagging functions - * test and set the bit before assining ->rqs[]. + * test and set the bit before assigning ->rqs[]. */ - rq = tags->rqs[bitnr]; + if (iter_data->flags & BT_TAG_ITER_STATIC_RQS) + rq = tags->static_rqs[bitnr]; + else + rq = tags->rqs[bitnr]; if (!rq) return true; if ((iter_data->flags & BT_TAG_ITER_STARTED) && @@ -339,11 +370,13 @@ static void __blk_mq_all_tag_iter(struct blk_mq_tags *tags, * indicates whether or not @rq is a reserved request. Return * true to continue iterating tags, false to stop. * @priv: Will be passed as second argument to @fn. + * + * Caller has to pass the tag map from which requests are allocated. */ void blk_mq_all_tag_iter(struct blk_mq_tags *tags, busy_tag_iter_fn *fn, void *priv) { - return __blk_mq_all_tag_iter(tags, fn, priv, 0); + return __blk_mq_all_tag_iter(tags, fn, priv, BT_TAG_ITER_STATIC_RQS); } /** diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index d38e48f2a0a4..2e4ef51cdb32 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -51,6 +51,14 @@ enum { BLK_MQ_TAG_MAX = BLK_MQ_NO_TAG - 1, }; +bool __blk_mq_get_driver_tag(struct request *rq); +static inline bool blk_mq_get_driver_tag(struct request *rq) +{ + if (rq->tag != BLK_MQ_NO_TAG) + return true; + return __blk_mq_get_driver_tag(rq); +} + extern bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *); extern void __blk_mq_tag_idle(struct blk_mq_hw_ctx *); diff --git a/block/blk-mq.c b/block/blk-mq.c index 9a36ac1c1fa1..4f57d27bfa73 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1052,35 +1052,6 @@ static inline unsigned int queued_to_index(unsigned int queued) return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1); } -bool blk_mq_get_driver_tag(struct request *rq) -{ - struct blk_mq_alloc_data data = { - .q = rq->q, - .hctx = rq->mq_hctx, - .flags = BLK_MQ_REQ_NOWAIT, - .cmd_flags = rq->cmd_flags, - }; - bool shared; - - if (rq->tag != BLK_MQ_NO_TAG) - return true; - - if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag)) - data.flags |= BLK_MQ_REQ_RESERVED; - - shared = blk_mq_tag_busy(data.hctx); - rq->tag = blk_mq_get_tag(&data); - if (rq->tag >= 0) { - if (shared) { - rq->rq_flags |= RQF_MQ_INFLIGHT; - atomic_inc(&data.hctx->nr_active); - } - data.hctx->tags->rqs[rq->tag] = rq; - } - - return rq->tag != BLK_MQ_NO_TAG; -} - static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode, int flags, void *key) { diff --git a/block/blk-mq.h b/block/blk-mq.h index a139b0631817..b3ce0f3a2ad2 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -44,7 +44,6 @@ bool blk_mq_dispatch_rq_list(struct request_queue *, struct list_head *, bool); void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, bool kick_requeue_list); void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list); -bool blk_mq_get_driver_tag(struct request *rq); struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *start); diff --git a/block/blk.h b/block/blk.h index aa16e524dc35..b5d1f0fc6547 100644 --- a/block/blk.h +++ b/block/blk.h @@ -420,9 +420,11 @@ static inline sector_t part_nr_sects_read(struct hd_struct *part) static inline void part_nr_sects_write(struct hd_struct *part, sector_t size) { #if BITS_PER_LONG==32 && defined(CONFIG_SMP) + preempt_disable(); write_seqcount_begin(&part->nr_sects_seq); part->nr_sects = size; write_seqcount_end(&part->nr_sects_seq); + preempt_enable(); #elif BITS_PER_LONG==32 && defined(CONFIG_PREEMPTION) preempt_disable(); part->nr_sects = size; diff --git a/drivers/acpi/acpi_extlog.c b/drivers/acpi/acpi_extlog.c index 8596a106a933..f138e12b7b82 100644 --- a/drivers/acpi/acpi_extlog.c +++ b/drivers/acpi/acpi_extlog.c @@ -42,8 +42,6 @@ struct extlog_l1_head { u8 rev1[12]; }; -static int old_edac_report_status; - static u8 extlog_dsm_uuid[] __initdata = "663E35AF-CC10-41A4-88EA-5470AF055295"; /* L1 table related physical address */ @@ -146,7 +144,7 @@ static int extlog_print(struct notifier_block *nb, unsigned long val, static u32 err_seq; estatus = extlog_elog_entry_check(cpu, bank); - if (estatus == NULL) + if (estatus == NULL || (mce->kflags & MCE_HANDLED_CEC)) return NOTIFY_DONE; memcpy(elog_buf, (void *)estatus, ELOG_ENTRY_LEN); @@ -176,7 +174,8 @@ static int extlog_print(struct notifier_block *nb, unsigned long val, } out: - return NOTIFY_STOP; + mce->kflags |= MCE_HANDLED_EXTLOG; + return NOTIFY_OK; } static bool __init extlog_get_l1addr(void) @@ -228,11 +227,6 @@ static int __init extlog_init(void) if (!(cap & MCG_ELOG_P) || !extlog_get_l1addr()) return -ENODEV; - if (edac_get_report_status() == EDAC_REPORTING_FORCE) { - pr_warn("Not loading eMCA, error reporting force-enabled through EDAC.\n"); - return -EPERM; - } - rc = -EINVAL; /* get L1 header to fetch necessary information */ l1_hdr_size = sizeof(struct extlog_l1_head); @@ -280,12 +274,6 @@ static int __init extlog_init(void) if (elog_buf == NULL) goto err_release_elog; - /* - * eMCA event report method has higher priority than EDAC method, - * unless EDAC event report method is mandatory. - */ - old_edac_report_status = edac_get_report_status(); - edac_set_report_status(EDAC_REPORTING_DISABLED); mce_register_decode_chain(&extlog_mce_dec); /* enable OS to be involved to take over management from BIOS */ ((struct extlog_l1_head *)extlog_l1_addr)->flags |= FLAG_OS_OPTIN; @@ -307,7 +295,6 @@ err: static void __exit extlog_exit(void) { - edac_set_report_status(old_edac_report_status); mce_unregister_decode_chain(&extlog_mce_dec); ((struct extlog_l1_head *)extlog_l1_addr)->flags &= ~FLAG_OS_OPTIN; if (extlog_l1_addr) diff --git a/drivers/acpi/nfit/core.c b/drivers/acpi/nfit/core.c index fa4500f9cfd1..7c138a4edc03 100644 --- a/drivers/acpi/nfit/core.c +++ b/drivers/acpi/nfit/core.c @@ -2293,7 +2293,7 @@ static int acpi_nfit_init_interleave_set(struct acpi_nfit_desc *acpi_desc, nd_set = devm_kzalloc(dev, sizeof(*nd_set), GFP_KERNEL); if (!nd_set) return -ENOMEM; - guid_copy(&nd_set->type_guid, (guid_t *) spa->range_guid); + import_guid(&nd_set->type_guid, spa->range_guid); info = devm_kzalloc(dev, sizeof_nfit_set_info(nr), GFP_KERNEL); if (!info) diff --git a/drivers/acpi/nfit/mce.c b/drivers/acpi/nfit/mce.c index f0ae48515b48..ee8d9973f60b 100644 --- a/drivers/acpi/nfit/mce.c +++ b/drivers/acpi/nfit/mce.c @@ -76,6 +76,7 @@ static int nfit_handle_mce(struct notifier_block *nb, unsigned long val, */ acpi_nfit_ars_rescan(acpi_desc, 0); } + mce->kflags |= MCE_HANDLED_NFIT; break; } diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 2e96d8b8758b..c33bbbfd1bd9 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1390,7 +1390,7 @@ loop_set_status(struct loop_device *lo, const struct loop_info64 *info) goto out_unfreeze; /* Mask out flags that can't be set using LOOP_SET_STATUS. */ - lo->lo_flags &= ~LOOP_SET_STATUS_SETTABLE_FLAGS; + lo->lo_flags &= LOOP_SET_STATUS_SETTABLE_FLAGS; /* For those flags, use the previous values instead */ lo->lo_flags |= prev_lo_flags & ~LOOP_SET_STATUS_SETTABLE_FLAGS; /* For flags that can't be cleared, use previous values too */ diff --git a/drivers/block/pktcdvd.c b/drivers/block/pktcdvd.c index 0b944ac96d6b..27a33adc41e4 100644 --- a/drivers/block/pktcdvd.c +++ b/drivers/block/pktcdvd.c @@ -1613,7 +1613,7 @@ static noinline_for_stack int pkt_get_last_written(struct pktcdvd_device *pd, disc_information di; track_information ti; __u32 last_track; - int ret = -1; + int ret; ret = pkt_get_disc_info(pd, &di); if (ret) diff --git a/drivers/block/umem.c b/drivers/block/umem.c index d84e8a878df2..1e2aa5ae2796 100644 --- a/drivers/block/umem.c +++ b/drivers/block/umem.c @@ -784,7 +784,7 @@ static const struct block_device_operations mm_fops = { static int mm_pci_probe(struct pci_dev *dev, const struct pci_device_id *id) { - int ret = -ENODEV; + int ret; struct cardinfo *card = &cards[num_cards]; unsigned char mem_present; unsigned char batt_status; diff --git a/drivers/clocksource/timer-riscv.c b/drivers/clocksource/timer-riscv.c index c4f15c4068c0..9de1dabfb126 100644 --- a/drivers/clocksource/timer-riscv.c +++ b/drivers/clocksource/timer-riscv.c @@ -12,8 +12,11 @@ #include <linux/cpu.h> #include <linux/delay.h> #include <linux/irq.h> +#include <linux/irqdomain.h> #include <linux/sched_clock.h> #include <linux/io-64-nonatomic-lo-hi.h> +#include <linux/interrupt.h> +#include <linux/of_irq.h> #include <asm/smp.h> #include <asm/sbi.h> @@ -39,6 +42,7 @@ static int riscv_clock_next_event(unsigned long delta, return 0; } +static unsigned int riscv_clock_event_irq; static DEFINE_PER_CPU(struct clock_event_device, riscv_clock_event) = { .name = "riscv_timer_clockevent", .features = CLOCK_EVT_FEAT_ONESHOT, @@ -74,30 +78,36 @@ static int riscv_timer_starting_cpu(unsigned int cpu) struct clock_event_device *ce = per_cpu_ptr(&riscv_clock_event, cpu); ce->cpumask = cpumask_of(cpu); + ce->irq = riscv_clock_event_irq; clockevents_config_and_register(ce, riscv_timebase, 100, 0x7fffffff); - csr_set(CSR_IE, IE_TIE); + enable_percpu_irq(riscv_clock_event_irq, + irq_get_trigger_type(riscv_clock_event_irq)); return 0; } static int riscv_timer_dying_cpu(unsigned int cpu) { - csr_clear(CSR_IE, IE_TIE); + disable_percpu_irq(riscv_clock_event_irq); return 0; } /* called directly from the low-level interrupt handler */ -void riscv_timer_interrupt(void) +static irqreturn_t riscv_timer_interrupt(int irq, void *dev_id) { struct clock_event_device *evdev = this_cpu_ptr(&riscv_clock_event); csr_clear(CSR_IE, IE_TIE); evdev->event_handler(evdev); + + return IRQ_HANDLED; } static int __init riscv_timer_init_dt(struct device_node *n) { int cpuid, hartid, error; + struct device_node *child; + struct irq_domain *domain; hartid = riscv_of_processor_hartid(n); if (hartid < 0) { @@ -115,6 +125,25 @@ static int __init riscv_timer_init_dt(struct device_node *n) if (cpuid != smp_processor_id()) return 0; + domain = NULL; + child = of_get_compatible_child(n, "riscv,cpu-intc"); + if (!child) { + pr_err("Failed to find INTC node [%pOF]\n", n); + return -ENODEV; + } + domain = irq_find_host(child); + of_node_put(child); + if (!domain) { + pr_err("Failed to find IRQ domain for node [%pOF]\n", n); + return -ENODEV; + } + + riscv_clock_event_irq = irq_create_mapping(domain, RV_IRQ_TIMER); + if (!riscv_clock_event_irq) { + pr_err("Failed to map timer interrupt for node [%pOF]\n", n); + return -ENODEV; + } + pr_info("%s: Registering clocksource cpuid [%d] hartid [%d]\n", __func__, cpuid, hartid); error = clocksource_register_hz(&riscv_clocksource, riscv_timebase); @@ -126,6 +155,14 @@ static int __init riscv_timer_init_dt(struct device_node *n) sched_clock_register(riscv_sched_clock, 64, riscv_timebase); + error = request_percpu_irq(riscv_clock_event_irq, + riscv_timer_interrupt, + "riscv-timer", &riscv_clock_event); + if (error) { + pr_err("registering percpu irq failed [%d]\n", error); + return error; + } + error = cpuhp_setup_state(CPUHP_AP_RISCV_TIMER_STARTING, "clockevents/riscv/timer:starting", riscv_timer_starting_cpu, riscv_timer_dying_cpu); diff --git a/drivers/cpuidle/cpuidle-arm.c b/drivers/cpuidle/cpuidle-arm.c index 9e5156d39627..8c758920d699 100644 --- a/drivers/cpuidle/cpuidle-arm.c +++ b/drivers/cpuidle/cpuidle-arm.c @@ -8,6 +8,7 @@ #define pr_fmt(fmt) "CPUidle arm: " fmt +#include <linux/cpu_cooling.h> #include <linux/cpuidle.h> #include <linux/cpumask.h> #include <linux/cpu_pm.h> @@ -124,6 +125,8 @@ static int __init arm_idle_init_cpu(int cpu) if (ret) goto out_kfree_drv; + cpuidle_cooling_register(drv); + return 0; out_kfree_drv: diff --git a/drivers/cpuidle/cpuidle-psci.c b/drivers/cpuidle/cpuidle-psci.c index d0fb585073c6..3806f911b61c 100644 --- a/drivers/cpuidle/cpuidle-psci.c +++ b/drivers/cpuidle/cpuidle-psci.c @@ -9,6 +9,7 @@ #define pr_fmt(fmt) "CPUidle PSCI: " fmt #include <linux/cpuhotplug.h> +#include <linux/cpu_cooling.h> #include <linux/cpuidle.h> #include <linux/cpumask.h> #include <linux/cpu_pm.h> @@ -319,6 +320,8 @@ static int __init psci_idle_init_cpu(int cpu) if (ret) goto out_kfree_drv; + cpuidle_cooling_register(drv); + return 0; out_kfree_drv: diff --git a/drivers/crypto/cavium/nitrox/nitrox_main.c b/drivers/crypto/cavium/nitrox/nitrox_main.c index 788c6607078b..cee2a2713038 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_main.c +++ b/drivers/crypto/cavium/nitrox/nitrox_main.c @@ -278,7 +278,7 @@ static void nitrox_remove_from_devlist(struct nitrox_device *ndev) struct nitrox_device *nitrox_get_first_device(void) { - struct nitrox_device *ndev = NULL; + struct nitrox_device *ndev; mutex_lock(&devlist_lock); list_for_each_entry(ndev, &ndevlist, list) { @@ -286,7 +286,7 @@ struct nitrox_device *nitrox_get_first_device(void) break; } mutex_unlock(&devlist_lock); - if (!ndev) + if (&ndev->list == &ndevlist) return NULL; refcount_inc(&ndev->refcnt); diff --git a/drivers/crypto/omap-aes-gcm.c b/drivers/crypto/omap-aes-gcm.c index 32dc00dc570b..9f937bdc53a7 100644 --- a/drivers/crypto/omap-aes-gcm.c +++ b/drivers/crypto/omap-aes-gcm.c @@ -77,7 +77,6 @@ static void omap_aes_gcm_done_task(struct omap_aes_dev *dd) tag = (u8 *)rctx->auth_tag; for (i = 0; i < dd->authsize; i++) { if (tag[i]) { - dev_err(dd->dev, "GCM decryption: Tag Message is wrong\n"); ret = -EBADMSG; } } diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index 824ddf2a66ff..b5aff20c5900 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1269,13 +1269,17 @@ static int omap_aes_remove(struct platform_device *pdev) spin_unlock(&list_lock); for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { crypto_unregister_skcipher( &dd->pdata->algs_info[i].algs_list[j]); + dd->pdata->algs_info[i].registered--; + } - for (i = dd->pdata->aead_algs_info->size - 1; i >= 0; i--) { + for (i = dd->pdata->aead_algs_info->registered - 1; i >= 0; i--) { aalg = &dd->pdata->aead_algs_info->algs_list[i]; crypto_unregister_aead(aalg); + dd->pdata->aead_algs_info->registered--; + } crypto_engine_exit(dd->engine); diff --git a/drivers/crypto/omap-crypto.c b/drivers/crypto/omap-crypto.c index cc88b7362bc2..94b2dba90f0d 100644 --- a/drivers/crypto/omap-crypto.c +++ b/drivers/crypto/omap-crypto.c @@ -178,11 +178,17 @@ static void omap_crypto_copy_data(struct scatterlist *src, amt = min(src->length - srco, dst->length - dsto); amt = min(len, amt); - srcb = sg_virt(src) + srco; - dstb = sg_virt(dst) + dsto; + srcb = kmap_atomic(sg_page(src)) + srco + src->offset; + dstb = kmap_atomic(sg_page(dst)) + dsto + dst->offset; memcpy(dstb, srcb, amt); + if (!PageSlab(sg_page(dst))) + flush_kernel_dcache_page(sg_page(dst)); + + kunmap_atomic(srcb); + kunmap_atomic(dstb); + srco += amt; dsto += amt; len -= amt; diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 063ad5d03f33..82691a057d2a 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -168,8 +168,6 @@ struct omap_sham_hmac_ctx { }; struct omap_sham_ctx { - struct omap_sham_dev *dd; - unsigned long flags; /* fallback stuff */ @@ -750,8 +748,17 @@ static int omap_sham_align_sgs(struct scatterlist *sg, int offset = rctx->offset; int bufcnt = rctx->bufcnt; - if (!sg || !sg->length || !nbytes) + if (!sg || !sg->length || !nbytes) { + if (bufcnt) { + bufcnt = DIV_ROUND_UP(bufcnt, bs) * bs; + sg_init_table(rctx->sgl, 1); + sg_set_buf(rctx->sgl, rctx->dd->xmit_buf, bufcnt); + rctx->sg = rctx->sgl; + rctx->sg_len = 1; + } + return 0; + } new_len = nbytes; @@ -895,7 +902,7 @@ static int omap_sham_prepare_request(struct ahash_request *req, bool update) if (hash_later < 0) hash_later = 0; - if (hash_later) { + if (hash_later && hash_later <= rctx->buflen) { scatterwalk_map_and_copy(rctx->buffer, req->src, req->nbytes - hash_later, @@ -925,27 +932,35 @@ static int omap_sham_update_dma_stop(struct omap_sham_dev *dd) return 0; } +struct omap_sham_dev *omap_sham_find_dev(struct omap_sham_reqctx *ctx) +{ + struct omap_sham_dev *dd; + + if (ctx->dd) + return ctx->dd; + + spin_lock_bh(&sham.lock); + dd = list_first_entry(&sham.dev_list, struct omap_sham_dev, list); + list_move_tail(&dd->list, &sham.dev_list); + ctx->dd = dd; + spin_unlock_bh(&sham.lock); + + return dd; +} + static int omap_sham_init(struct ahash_request *req) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct omap_sham_ctx *tctx = crypto_ahash_ctx(tfm); struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = NULL, *tmp; + struct omap_sham_dev *dd; int bs = 0; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); + ctx->dd = NULL; - ctx->dd = dd; + dd = omap_sham_find_dev(ctx); + if (!dd) + return -ENODEV; ctx->flags = 0; @@ -1215,8 +1230,7 @@ err1: static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_ctx *tctx = crypto_tfm_ctx(req->base.tfm); - struct omap_sham_dev *dd = tctx->dd; + struct omap_sham_dev *dd = ctx->dd; ctx->op = op; @@ -1226,7 +1240,7 @@ static int omap_sham_enqueue(struct ahash_request *req, unsigned int op) static int omap_sham_update(struct ahash_request *req) { struct omap_sham_reqctx *ctx = ahash_request_ctx(req); - struct omap_sham_dev *dd = ctx->dd; + struct omap_sham_dev *dd = omap_sham_find_dev(ctx); if (!req->nbytes) return 0; @@ -1319,21 +1333,8 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, struct omap_sham_hmac_ctx *bctx = tctx->base; int bs = crypto_shash_blocksize(bctx->shash); int ds = crypto_shash_digestsize(bctx->shash); - struct omap_sham_dev *dd = NULL, *tmp; int err, i; - spin_lock_bh(&sham.lock); - if (!tctx->dd) { - list_for_each_entry(tmp, &sham.dev_list, list) { - dd = tmp; - break; - } - tctx->dd = dd; - } else { - dd = tctx->dd; - } - spin_unlock_bh(&sham.lock); - err = crypto_shash_setkey(tctx->fallback, key, keylen); if (err) return err; @@ -1350,7 +1351,7 @@ static int omap_sham_setkey(struct crypto_ahash *tfm, const u8 *key, memset(bctx->ipad + keylen, 0, bs - keylen); - if (!test_bit(FLAGS_AUTO_XOR, &dd->flags)) { + if (!test_bit(FLAGS_AUTO_XOR, &sham.flags)) { memcpy(bctx->opad, bctx->ipad, bs); for (i = 0; i < bs; i++) { @@ -1571,7 +1572,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "sha224", .cra_driver_name = "omap-sha224", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1592,7 +1594,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "sha256", .cra_driver_name = "omap-sha256", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1614,7 +1617,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "hmac(sha224)", .cra_driver_name = "omap-hmac-sha224", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1637,7 +1641,8 @@ static struct ahash_alg algs_sha224_sha256[] = { .cra_name = "hmac(sha256)", .cra_driver_name = "omap-hmac-sha256", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1662,7 +1667,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "sha384", .cra_driver_name = "omap-sha384", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1683,7 +1689,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "sha512", .cra_driver_name = "omap-sha512", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx), @@ -1705,7 +1712,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "hmac(sha384)", .cra_driver_name = "omap-hmac-sha384", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -1728,7 +1736,8 @@ static struct ahash_alg algs_sha384_sha512[] = { .cra_name = "hmac(sha512)", .cra_driver_name = "omap-hmac-sha512", .cra_priority = 400, - .cra_flags = CRYPTO_ALG_ASYNC | + .cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct omap_sham_ctx) + @@ -2154,6 +2163,7 @@ static int omap_sham_probe(struct platform_device *pdev) } dd->flags |= dd->pdata->flags; + sham.flags |= dd->pdata->flags; pm_runtime_use_autosuspend(dev); pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY); @@ -2181,6 +2191,9 @@ static int omap_sham_probe(struct platform_device *pdev) spin_unlock(&sham.lock); for (i = 0; i < dd->pdata->algs_info_size; i++) { + if (dd->pdata->algs_info[i].registered) + break; + for (j = 0; j < dd->pdata->algs_info[i].size; j++) { struct ahash_alg *alg; @@ -2232,9 +2245,11 @@ static int omap_sham_remove(struct platform_device *pdev) list_del(&dd->list); spin_unlock(&sham.lock); for (i = dd->pdata->algs_info_size - 1; i >= 0; i--) - for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) + for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--) { crypto_unregister_ahash( &dd->pdata->algs_info[i].algs_list[j]); + dd->pdata->algs_info[i].registered--; + } tasklet_kill(&dd->done_task); pm_runtime_disable(&pdev->dev); diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 9cf7cc1f3f72..ef90070a9194 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -4,9 +4,6 @@ static struct edac_pci_ctl_info *pci_ctl; -static int report_gart_errors; -module_param(report_gart_errors, int, 0644); - /* * Set by command line parameter. If BIOS has enabled the ECC, this override is * cleared to prevent re-enabling the hardware by this driver. @@ -2319,6 +2316,16 @@ static struct amd64_family_type family_types[] = { .dbam_to_cs = f17_addr_mask_to_cs_size, } }, + [F17_M60H_CPUS] = { + .ctl_name = "F17h_M60h", + .f0_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F0, + .f6_id = PCI_DEVICE_ID_AMD_17H_M60H_DF_F6, + .max_mcs = 2, + .ops = { + .early_channel_count = f17_early_channel_count, + .dbam_to_cs = f17_addr_mask_to_cs_size, + } + }, [F17_M70H_CPUS] = { .ctl_name = "F17h_M70h", .f0_id = PCI_DEVICE_ID_AMD_17H_M70H_DF_F0, @@ -3357,6 +3364,10 @@ static struct amd64_family_type *per_family_init(struct amd64_pvt *pvt) fam_type = &family_types[F17_M30H_CPUS]; pvt->ops = &family_types[F17_M30H_CPUS].ops; break; + } else if (pvt->model >= 0x60 && pvt->model <= 0x6f) { + fam_type = &family_types[F17_M60H_CPUS]; + pvt->ops = &family_types[F17_M60H_CPUS].ops; + break; } else if (pvt->model >= 0x70 && pvt->model <= 0x7f) { fam_type = &family_types[F17_M70H_CPUS]; pvt->ops = &family_types[F17_M70H_CPUS].ops; @@ -3681,9 +3692,6 @@ static int __init amd64_edac_init(void) } /* register stuff with EDAC MCE */ - if (report_gart_errors) - amd_report_gart_errors(true); - if (boot_cpu_data.x86 >= 0x17) amd_register_ecc_decoder(decode_umc_error); else @@ -3718,8 +3726,6 @@ static void __exit amd64_edac_exit(void) edac_pci_release_generic_ctl(pci_ctl); /* unregister from EDAC MCE */ - amd_report_gart_errors(false); - if (boot_cpu_data.x86 >= 0x17) amd_unregister_ecc_decoder(decode_umc_error); else diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index abbf3c274d74..52b5d03eeba0 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -120,6 +120,8 @@ #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F6 0x15ee #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F0 0x1490 #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F6 0x1496 +#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F0 0x1448 +#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F6 0x144e #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F0 0x1440 #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F6 0x1446 #define PCI_DEVICE_ID_AMD_19H_DF_F0 0x1650 @@ -293,6 +295,7 @@ enum amd_families { F17_CPUS, F17_M10H_CPUS, F17_M30H_CPUS, + F17_M60H_CPUS, F17_M70H_CPUS, F19_CPUS, NUM_FAMILIES, diff --git a/drivers/edac/edac_mc.c b/drivers/edac/edac_mc.c index 75ede27bdf6a..5813e931f2f0 100644 --- a/drivers/edac/edac_mc.c +++ b/drivers/edac/edac_mc.c @@ -43,8 +43,6 @@ int edac_op_state = EDAC_OPSTATE_INVAL; EXPORT_SYMBOL_GPL(edac_op_state); -static int edac_report = EDAC_REPORTING_ENABLED; - /* lock to memory controller's control array */ static DEFINE_MUTEX(mem_ctls_mutex); static LIST_HEAD(mc_devices); @@ -60,65 +58,6 @@ static struct mem_ctl_info *error_desc_to_mci(struct edac_raw_error_desc *e) return container_of(e, struct mem_ctl_info, error_desc); } -int edac_get_report_status(void) -{ - return edac_report; -} -EXPORT_SYMBOL_GPL(edac_get_report_status); - -void edac_set_report_status(int new) -{ - if (new == EDAC_REPORTING_ENABLED || - new == EDAC_REPORTING_DISABLED || - new == EDAC_REPORTING_FORCE) - edac_report = new; -} -EXPORT_SYMBOL_GPL(edac_set_report_status); - -static int edac_report_set(const char *str, const struct kernel_param *kp) -{ - if (!str) - return -EINVAL; - - if (!strncmp(str, "on", 2)) - edac_report = EDAC_REPORTING_ENABLED; - else if (!strncmp(str, "off", 3)) - edac_report = EDAC_REPORTING_DISABLED; - else if (!strncmp(str, "force", 5)) - edac_report = EDAC_REPORTING_FORCE; - - return 0; -} - -static int edac_report_get(char *buffer, const struct kernel_param *kp) -{ - int ret = 0; - - switch (edac_report) { - case EDAC_REPORTING_ENABLED: - ret = sprintf(buffer, "on"); - break; - case EDAC_REPORTING_DISABLED: - ret = sprintf(buffer, "off"); - break; - case EDAC_REPORTING_FORCE: - ret = sprintf(buffer, "force"); - break; - default: - ret = -EINVAL; - break; - } - - return ret; -} - -static const struct kernel_param_ops edac_report_ops = { - .set = edac_report_set, - .get = edac_report_get, -}; - -module_param_cb(edac_report, &edac_report_ops, &edac_report, 0644); - unsigned int edac_dimm_info_location(struct dimm_info *dimm, char *buf, unsigned int len) { diff --git a/drivers/edac/i7core_edac.c b/drivers/edac/i7core_edac.c index b3135b208f9a..5860ca41185c 100644 --- a/drivers/edac/i7core_edac.c +++ b/drivers/edac/i7core_edac.c @@ -1815,7 +1815,7 @@ static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val, struct mem_ctl_info *mci; i7_dev = get_i7core_dev(mce->socketid); - if (!i7_dev) + if (!i7_dev || (mce->kflags & MCE_HANDLED_CEC)) return NOTIFY_DONE; mci = i7_dev->mci; @@ -1834,7 +1834,8 @@ static int i7core_mce_check_error(struct notifier_block *nb, unsigned long val, i7core_check_error(mci, mce); /* Advise mcelog that the errors were handled */ - return NOTIFY_STOP; + mce->kflags |= MCE_HANDLED_EDAC; + return NOTIFY_OK; } static struct notifier_block i7_mce_dec = { diff --git a/drivers/edac/mce_amd.c b/drivers/edac/mce_amd.c index 8874b7722b2f..2b5401db56ad 100644 --- a/drivers/edac/mce_amd.c +++ b/drivers/edac/mce_amd.c @@ -10,15 +10,8 @@ static struct amd_decoder_ops fam_ops; static u8 xec_mask = 0xf; -static bool report_gart_errors; static void (*decode_dram_ecc)(int node_id, struct mce *m); -void amd_report_gart_errors(bool v) -{ - report_gart_errors = v; -} -EXPORT_SYMBOL_GPL(amd_report_gart_errors); - void amd_register_ecc_decoder(void (*f)(int, struct mce *)) { decode_dram_ecc = f; @@ -1030,20 +1023,6 @@ static inline void amd_decode_err_code(u16 ec) pr_cont("\n"); } -/* - * Filter out unwanted MCE signatures here. - */ -static bool ignore_mce(struct mce *m) -{ - /* - * NB GART TLB error reporting is disabled by default. - */ - if (m->bank == 4 && XEC(m->status, 0x1f) == 0x5 && !report_gart_errors) - return true; - - return false; -} - static const char *decode_error_status(struct mce *m) { if (m->status & MCI_STATUS_UC) { @@ -1067,8 +1046,8 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) unsigned int fam = x86_family(m->cpuid); int ecc; - if (ignore_mce(m)) - return NOTIFY_STOP; + if (m->kflags & MCE_HANDLED_CEC) + return NOTIFY_DONE; pr_emerg(HW_ERR "%s\n", decode_error_status(m)); @@ -1170,7 +1149,8 @@ amd_decode_mce(struct notifier_block *nb, unsigned long val, void *data) err_code: amd_decode_err_code(m->status & 0xffff); - return NOTIFY_STOP; + m->kflags |= MCE_HANDLED_EDAC; + return NOTIFY_OK; } static struct notifier_block amd_mce_dec_nb = { diff --git a/drivers/edac/mce_amd.h b/drivers/edac/mce_amd.h index 4e9c5e596c6c..4811b18d9606 100644 --- a/drivers/edac/mce_amd.h +++ b/drivers/edac/mce_amd.h @@ -7,7 +7,6 @@ #include <asm/mce.h> #define EC(x) ((x) & 0xffff) -#define XEC(x, mask) (((x) >> 16) & mask) #define LOW_SYNDROME(x) (((x) >> 15) & 0xff) #define HIGH_SYNDROME(x) (((x) >> 24) & 0xff) @@ -77,7 +76,6 @@ struct amd_decoder_ops { bool (*mc2_mce)(u16, u8); }; -void amd_report_gart_errors(bool); void amd_register_ecc_decoder(void (*f)(int, struct mce *)); void amd_unregister_ecc_decoder(void (*f)(int, struct mce *)); diff --git a/drivers/edac/pnd2_edac.c b/drivers/edac/pnd2_edac.c index bc47328eb485..c1f2e6deb021 100644 --- a/drivers/edac/pnd2_edac.c +++ b/drivers/edac/pnd2_edac.c @@ -1396,11 +1396,8 @@ static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, vo struct dram_addr daddr; char *type; - if (edac_get_report_status() == EDAC_REPORTING_DISABLED) - return NOTIFY_DONE; - mci = pnd2_mci; - if (!mci) + if (!mci || (mce->kflags & MCE_HANDLED_CEC)) return NOTIFY_DONE; /* @@ -1429,7 +1426,8 @@ static int pnd2_mce_check_error(struct notifier_block *nb, unsigned long val, vo pnd2_mce_output_error(mci, mce, &daddr); /* Advice mcelog that the error were handled */ - return NOTIFY_STOP; + mce->kflags |= MCE_HANDLED_EDAC; + return NOTIFY_OK; } static struct notifier_block pnd2_mce_dec = { diff --git a/drivers/edac/sb_edac.c b/drivers/edac/sb_edac.c index 7d51c82be62b..d414698ca324 100644 --- a/drivers/edac/sb_edac.c +++ b/drivers/edac/sb_edac.c @@ -3134,7 +3134,7 @@ static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, struct mem_ctl_info *mci; char *type; - if (edac_get_report_status() == EDAC_REPORTING_DISABLED) + if (mce->kflags & MCE_HANDLED_CEC) return NOTIFY_DONE; /* @@ -3183,7 +3183,8 @@ static int sbridge_mce_check_error(struct notifier_block *nb, unsigned long val, sbridge_mce_output_error(mci, mce); /* Advice mcelog that the error were handled */ - return NOTIFY_STOP; + mce->kflags |= MCE_HANDLED_EDAC; + return NOTIFY_OK; } static struct notifier_block sbridge_mce_dec = { @@ -3523,8 +3524,6 @@ static int __init sbridge_init(void) if (rc >= 0) { mce_register_decode_chain(&sbridge_mce_dec); - if (edac_get_report_status() == EDAC_REPORTING_DISABLED) - sbridge_printk(KERN_WARNING, "Loading driver, error reporting disabled.\n"); return 0; } diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 46be1a77bd1d..6d8d6dc626bf 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -573,7 +573,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, struct mem_ctl_info *mci; char *type; - if (edac_get_report_status() == EDAC_REPORTING_DISABLED) + if (mce->kflags & MCE_HANDLED_CEC) return NOTIFY_DONE; /* ignore unless this is memory related with an address */ @@ -615,6 +615,7 @@ int skx_mce_check_error(struct notifier_block *nb, unsigned long val, skx_mce_output_error(mci, mce, &res); + mce->kflags |= MCE_HANDLED_EDAC; return NOTIFY_DONE; } diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index cce4a7436052..75daaf20374e 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -37,7 +37,9 @@ KBUILD_CFLAGS := $(cflags-y) -Os -DDISABLE_BRANCH_PROFILING \ KBUILD_CFLAGS := $(filter-out $(CC_FLAGS_SCS), $(KBUILD_CFLAGS)) GCOV_PROFILE := n +# Sanitizer runtimes are unavailable and cannot be linked here. KASAN_SANITIZE := n +KCSAN_SANITIZE := n UBSAN_SANITIZE := n OBJECT_FILES_NON_STANDARD := y diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h index 53b4126373a5..ffe149aafc39 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.h @@ -27,6 +27,7 @@ #include <linux/types.h> #include <linux/mm.h> +#include <linux/kthread.h> #include <linux/workqueue.h> #include <kgd_kfd_interface.h> #include <drm/ttm/ttm_execbuf_util.h> @@ -195,10 +196,10 @@ uint8_t amdgpu_amdkfd_get_xgmi_hops_count(struct kgd_dev *dst, struct kgd_dev *s pagefault_disable(); \ if ((mmptr) == current->mm) { \ valid = !get_user((dst), (wptr)); \ - } else if (current->mm == NULL) { \ - use_mm(mmptr); \ + } else if (current->flags & PF_KTHREAD) { \ + kthread_use_mm(mmptr); \ valid = !get_user((dst), (wptr)); \ - unuse_mm(mmptr); \ + kthread_unuse_mm(mmptr); \ } \ pagefault_enable(); \ } \ diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c index 6529caca88fe..35d4a5ab0228 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_arcturus.c @@ -22,7 +22,6 @@ #include <linux/module.h> #include <linux/fdtable.h> #include <linux/uaccess.h> -#include <linux/mmu_context.h> #include <linux/firmware.h> #include "amdgpu.h" #include "amdgpu_amdkfd.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c index 691c89705bcd..bf927f432506 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v10.c @@ -19,7 +19,6 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gc/gc_10_1_0_offset.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c index c6944739183a..744366c7ee85 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v7.c @@ -20,8 +20,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "cikd.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c index 2f4bdc80a6b2..feab4cc6e836 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v8.c @@ -20,8 +20,6 @@ * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gfx_v8_0.h" diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c index df841c2ac5e7..c7fd0c47b254 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gfx_v9.c @@ -19,8 +19,6 @@ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR * OTHER DEALINGS IN THE SOFTWARE. */ -#include <linux/mmu_context.h> - #include "amdgpu.h" #include "amdgpu_amdkfd.h" #include "gc/gc_9_0_offset.h" diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c index 7d39b858c9f1..3a3a511670c9 100644 --- a/drivers/gpu/drm/ast/ast_mode.c +++ b/drivers/gpu/drm/ast/ast_mode.c @@ -226,6 +226,7 @@ static void ast_set_vbios_color_reg(struct ast_private *ast, case 3: case 4: color_index = TrueCModeIndex; + break; default: return; } diff --git a/drivers/gpu/drm/drm_connector.c b/drivers/gpu/drm/drm_connector.c index b1099e1251a2..d877ddc6dc57 100644 --- a/drivers/gpu/drm/drm_connector.c +++ b/drivers/gpu/drm/drm_connector.c @@ -27,6 +27,7 @@ #include <drm/drm_print.h> #include <drm/drm_drv.h> #include <drm/drm_file.h> +#include <drm/drm_sysfs.h> #include <linux/uaccess.h> @@ -523,6 +524,10 @@ int drm_connector_register(struct drm_connector *connector) drm_mode_object_register(connector->dev, &connector->base); connector->registration_state = DRM_CONNECTOR_REGISTERED; + + /* Let userspace know we have a new connector */ + drm_sysfs_hotplug_event(connector->dev); + goto unlock; err_debugfs: diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c index 939f0032aab1..f0336c804639 100644 --- a/drivers/gpu/drm/drm_sysfs.c +++ b/drivers/gpu/drm/drm_sysfs.c @@ -291,9 +291,6 @@ int drm_sysfs_connector_add(struct drm_connector *connector) return PTR_ERR(connector->kdev); } - /* Let userspace know we have a new connector */ - drm_sysfs_hotplug_event(dev); - if (connector->ddc) return sysfs_create_link(&connector->kdev->kobj, &connector->ddc->dev.kobj, "ddc"); diff --git a/drivers/gpu/drm/i915/display/intel_dp.c b/drivers/gpu/drm/i915/display/intel_dp.c index 40d42dcff0b7..ed9e53c373a7 100644 --- a/drivers/gpu/drm/i915/display/intel_dp.c +++ b/drivers/gpu/drm/i915/display/intel_dp.c @@ -5206,6 +5206,9 @@ void intel_read_dp_sdp(struct intel_encoder *encoder, struct intel_crtc_state *crtc_state, unsigned int type) { + if (encoder->type != INTEL_OUTPUT_DDI) + return; + switch (type) { case DP_SDP_VSC: intel_read_dp_vsc_sdp(encoder, crtc_state, diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 3ce185670ca4..db8eb1c6afe9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -1988,6 +1988,38 @@ static const struct dma_fence_work_ops eb_parse_ops = { .release = __eb_parse_release, }; +static inline int +__parser_mark_active(struct i915_vma *vma, + struct intel_timeline *tl, + struct dma_fence *fence) +{ + struct intel_gt_buffer_pool_node *node = vma->private; + + return i915_active_ref(&node->active, tl, fence); +} + +static int +parser_mark_active(struct eb_parse_work *pw, struct intel_timeline *tl) +{ + int err; + + mutex_lock(&tl->mutex); + + err = __parser_mark_active(pw->shadow, tl, &pw->base.dma); + if (err) + goto unlock; + + if (pw->trampoline) { + err = __parser_mark_active(pw->trampoline, tl, &pw->base.dma); + if (err) + goto unlock; + } + +unlock: + mutex_unlock(&tl->mutex); + return err; +} + static int eb_parse_pipeline(struct i915_execbuffer *eb, struct i915_vma *shadow, struct i915_vma *trampoline) @@ -2022,20 +2054,25 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, pw->shadow = shadow; pw->trampoline = trampoline; + /* Mark active refs early for this worker, in case we get interrupted */ + err = parser_mark_active(pw, eb->context->timeline); + if (err) + goto err_commit; + err = dma_resv_lock_interruptible(pw->batch->resv, NULL); if (err) - goto err_trampoline; + goto err_commit; err = dma_resv_reserve_shared(pw->batch->resv, 1); if (err) - goto err_batch_unlock; + goto err_commit_unlock; /* Wait for all writes (and relocs) into the batch to complete */ err = i915_sw_fence_await_reservation(&pw->base.chain, pw->batch->resv, NULL, false, 0, I915_FENCE_GFP); if (err < 0) - goto err_batch_unlock; + goto err_commit_unlock; /* Keep the batch alive and unwritten as we parse */ dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma); @@ -2050,11 +2087,13 @@ static int eb_parse_pipeline(struct i915_execbuffer *eb, dma_fence_work_commit_imm(&pw->base); return 0; -err_batch_unlock: +err_commit_unlock: dma_resv_unlock(pw->batch->resv); -err_trampoline: - if (trampoline) - i915_active_release(&trampoline->active); +err_commit: + i915_sw_fence_set_error_once(&pw->base.chain, err); + dma_fence_work_commit_imm(&pw->base); + return err; + err_shadow: i915_active_release(&shadow->active); err_batch: @@ -2100,6 +2139,7 @@ static int eb_parse(struct i915_execbuffer *eb) goto err; } i915_gem_object_set_readonly(shadow->obj); + shadow->private = pool; trampoline = NULL; if (CMDPARSER_USES_GGTT(eb->i915)) { @@ -2113,6 +2153,7 @@ static int eb_parse(struct i915_execbuffer *eb) shadow = trampoline; goto err_shadow; } + shadow->private = pool; eb->batch_flags |= I915_DISPATCH_SECURE; } @@ -2129,7 +2170,6 @@ static int eb_parse(struct i915_execbuffer *eb) eb->trampoline = trampoline; eb->batch_start_offset = 0; - shadow->private = pool; return 0; err_trampoline: diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index eee530453aa6..ad8a9df49f29 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -31,7 +31,7 @@ #include <linux/init.h> #include <linux/device.h> #include <linux/mm.h> -#include <linux/mmu_context.h> +#include <linux/kthread.h> #include <linux/sched/mm.h> #include <linux/types.h> #include <linux/list.h> diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c index a3dde770226d..02559da61e6e 100644 --- a/drivers/gpu/drm/i915/i915_params.c +++ b/drivers/gpu/drm/i915/i915_params.c @@ -65,7 +65,7 @@ i915_param_named_unsafe(vbt_sdvo_panel_type, int, 0400, "Override/Ignore selection of SDVO panel mode in the VBT " "(-2=ignore, -1=auto [default], index in VBT BIOS table)"); -i915_param_named_unsafe(reset, int, 0600, +i915_param_named_unsafe(reset, uint, 0600, "Attempt GPU resets (0=disabled, 1=full gpu reset, 2=engine reset [default])"); i915_param_named_unsafe(vbt_firmware, charp, 0400, diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi.h b/drivers/gpu/drm/sun4i/sun4i_hdmi.h index 7ad3f06c127e..00ca35f07ba5 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi.h +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi.h @@ -148,7 +148,7 @@ #define SUN4I_HDMI_DDC_CMD_IMPLICIT_WRITE 3 #define SUN4I_HDMI_DDC_CLK_REG 0x528 -#define SUN4I_HDMI_DDC_CLK_M(m) (((m) & 0x7) << 3) +#define SUN4I_HDMI_DDC_CLK_M(m) (((m) & 0xf) << 3) #define SUN4I_HDMI_DDC_CLK_N(n) ((n) & 0x7) #define SUN4I_HDMI_DDC_LINE_CTRL_REG 0x540 diff --git a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c index 2ff780114106..12430b9d4e93 100644 --- a/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c +++ b/drivers/gpu/drm/sun4i/sun4i_hdmi_ddc_clk.c @@ -33,7 +33,7 @@ static unsigned long sun4i_ddc_calc_divider(unsigned long rate, unsigned long best_rate = 0; u8 best_m = 0, best_n = 0, _m, _n; - for (_m = 0; _m < 8; _m++) { + for (_m = 0; _m < 16; _m++) { for (_n = 0; _n < 8; _n++) { unsigned long tmp_rate; diff --git a/drivers/hwmon/k10temp.c b/drivers/hwmon/k10temp.c index 9915578533bb..8f12995ec133 100644 --- a/drivers/hwmon/k10temp.c +++ b/drivers/hwmon/k10temp.c @@ -632,6 +632,7 @@ static const struct pci_device_id k10temp_id_table[] = { { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M10H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M30H_DF_F3) }, + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M60H_DF_F3) }, { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_17H_M70H_DF_F3) }, { PCI_VDEVICE(HYGON, PCI_DEVICE_ID_AMD_17H_DF_F3) }, {} diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig index 2ddca08f8a76..735bf31a3fdf 100644 --- a/drivers/i2c/busses/Kconfig +++ b/drivers/i2c/busses/Kconfig @@ -475,8 +475,8 @@ config I2C_BCM_KONA config I2C_BRCMSTB tristate "BRCM Settop/DSL I2C controller" - depends on ARCH_BRCMSTB || BMIPS_GENERIC || ARCH_BCM_63XX || \ - COMPILE_TEST + depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC || \ + ARCH_BCM_63XX || COMPILE_TEST default y help If you say yes to this option, support will be included for the @@ -526,22 +526,12 @@ config I2C_DAVINCI config I2C_DESIGNWARE_CORE tristate - -config I2C_DESIGNWARE_PLATFORM - tristate "Synopsys DesignWare Platform" - select I2C_DESIGNWARE_CORE - depends on (ACPI && COMMON_CLK) || !ACPI - help - If you say yes to this option, support will be included for the - Synopsys DesignWare I2C adapter. - - This driver can also be built as a module. If so, the module - will be called i2c-designware-platform. + select REGMAP config I2C_DESIGNWARE_SLAVE bool "Synopsys DesignWare Slave" + depends on I2C_DESIGNWARE_CORE select I2C_SLAVE - depends on I2C_DESIGNWARE_PLATFORM help If you say yes to this option, support will be included for the Synopsys DesignWare I2C slave adapter. @@ -549,20 +539,22 @@ config I2C_DESIGNWARE_SLAVE This is not a standalone module, this module compiles together with i2c-designware-core. -config I2C_DESIGNWARE_PCI - tristate "Synopsys DesignWare PCI" - depends on PCI +config I2C_DESIGNWARE_PLATFORM + tristate "Synopsys DesignWare Platform" + depends on (ACPI && COMMON_CLK) || !ACPI select I2C_DESIGNWARE_CORE + select MFD_SYSCON if MIPS_BAIKAL_T1 help If you say yes to this option, support will be included for the - Synopsys DesignWare I2C adapter. Only master mode is supported. + Synopsys DesignWare I2C adapter. This driver can also be built as a module. If so, the module - will be called i2c-designware-pci. + will be called i2c-designware-platform. config I2C_DESIGNWARE_BAYTRAIL bool "Intel Baytrail I2C semaphore support" depends on ACPI + depends on I2C_DESIGNWARE_PLATFORM depends on (I2C_DESIGNWARE_PLATFORM=m && IOSF_MBI) || \ (I2C_DESIGNWARE_PLATFORM=y && IOSF_MBI=y) help @@ -572,6 +564,17 @@ config I2C_DESIGNWARE_BAYTRAIL the platform firmware controlling it. You should say Y if running on a BayTrail system using the AXP288. +config I2C_DESIGNWARE_PCI + tristate "Synopsys DesignWare PCI" + depends on PCI + select I2C_DESIGNWARE_CORE + help + If you say yes to this option, support will be included for the + Synopsys DesignWare I2C adapter. Only master mode is supported. + + This driver can also be built as a module. If so, the module + will be called i2c-designware-pci. + config I2C_DIGICOLOR tristate "Conexant Digicolor I2C driver" depends on ARCH_DIGICOLOR || COMPILE_TEST @@ -791,6 +794,15 @@ config I2C_NOMADIK I2C interface from ST-Ericsson's Nomadik and Ux500 architectures, as well as the STA2X11 PCIe I/O HUB. +config I2C_NPCM7XX + tristate "Nuvoton I2C Controller" + depends on ARCH_NPCM7XX || COMPILE_TEST + help + If you say yes to this option, support will be included for the + Nuvoton I2C controller, which is available on the NPCM7xx BMC + controller. + Driver can also support slave mode (select I2C_SLAVE). + config I2C_OCORES tristate "OpenCores I2C Controller" help @@ -885,6 +897,16 @@ config I2C_PXA_SLAVE is necessary for systems where the PXA may be a target on the I2C bus. +config I2C_QCOM_CCI + tristate "Qualcomm Camera Control Interface" + depends on ARCH_QCOM || COMPILE_TEST + help + If you say yes to this option, support will be included for the + built-in camera control interface on the Qualcomm SoCs. + + This driver can also be built as a module. If so, the module + will be called i2c-qcom-cci. + config I2C_QCOM_GENI tristate "Qualcomm Technologies Inc.'s GENI based I2C controller" depends on ARCH_QCOM || COMPILE_TEST diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile index 25d60889713c..306d5dc3f417 100644 --- a/drivers/i2c/busses/Makefile +++ b/drivers/i2c/busses/Makefile @@ -48,16 +48,15 @@ obj-$(CONFIG_I2C_CADENCE) += i2c-cadence.o obj-$(CONFIG_I2C_CBUS_GPIO) += i2c-cbus-gpio.o obj-$(CONFIG_I2C_CPM) += i2c-cpm.o obj-$(CONFIG_I2C_DAVINCI) += i2c-davinci.o -obj-$(CONFIG_I2C_DESIGNWARE_CORE) += i2c-designware-core.o -i2c-designware-core-objs := i2c-designware-common.o i2c-designware-master.o -ifeq ($(CONFIG_I2C_DESIGNWARE_SLAVE),y) -i2c-designware-core-objs += i2c-designware-slave.o -endif -obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM) += i2c-designware-platform.o -i2c-designware-platform-objs := i2c-designware-platdrv.o +obj-$(CONFIG_I2C_DESIGNWARE_CORE) += i2c-designware-core.o +i2c-designware-core-y := i2c-designware-common.o +i2c-designware-core-y += i2c-designware-master.o +i2c-designware-core-$(CONFIG_I2C_DESIGNWARE_SLAVE) += i2c-designware-slave.o +obj-$(CONFIG_I2C_DESIGNWARE_PLATFORM) += i2c-designware-platform.o +i2c-designware-platform-y := i2c-designware-platdrv.o i2c-designware-platform-$(CONFIG_I2C_DESIGNWARE_BAYTRAIL) += i2c-designware-baytrail.o -obj-$(CONFIG_I2C_DESIGNWARE_PCI) += i2c-designware-pci.o -i2c-designware-pci-objs := i2c-designware-pcidrv.o +obj-$(CONFIG_I2C_DESIGNWARE_PCI) += i2c-designware-pci.o +i2c-designware-pci-y := i2c-designware-pcidrv.o obj-$(CONFIG_I2C_DIGICOLOR) += i2c-digicolor.o obj-$(CONFIG_I2C_EFM32) += i2c-efm32.o obj-$(CONFIG_I2C_EG20T) += i2c-eg20t.o @@ -81,6 +80,7 @@ obj-$(CONFIG_I2C_MT7621) += i2c-mt7621.o obj-$(CONFIG_I2C_MV64XXX) += i2c-mv64xxx.o obj-$(CONFIG_I2C_MXS) += i2c-mxs.o obj-$(CONFIG_I2C_NOMADIK) += i2c-nomadik.o +obj-$(CONFIG_I2C_NPCM7XX) += i2c-npcm7xx.o obj-$(CONFIG_I2C_OCORES) += i2c-ocores.o obj-$(CONFIG_I2C_OMAP) += i2c-omap.o obj-$(CONFIG_I2C_OWL) += i2c-owl.o @@ -91,6 +91,7 @@ obj-$(CONFIG_I2C_PNX) += i2c-pnx.o obj-$(CONFIG_I2C_PUV3) += i2c-puv3.o obj-$(CONFIG_I2C_PXA) += i2c-pxa.o obj-$(CONFIG_I2C_PXA_PCI) += i2c-pxa-pci.o +obj-$(CONFIG_I2C_QCOM_CCI) += i2c-qcom-cci.o obj-$(CONFIG_I2C_QCOM_GENI) += i2c-qcom-geni.o obj-$(CONFIG_I2C_QUP) += i2c-qup.o obj-$(CONFIG_I2C_RIIC) += i2c-riic.o diff --git a/drivers/i2c/busses/i2c-altera.c b/drivers/i2c/busses/i2c-altera.c index 16ddc26c00e6..7d62cbda6e06 100644 --- a/drivers/i2c/busses/i2c-altera.c +++ b/drivers/i2c/busses/i2c-altera.c @@ -69,7 +69,6 @@ * @fifo_size: size of the FIFO passed in. * @isr_mask: cached copy of local ISR enables. * @isr_status: cached copy of local ISR status. - * @lock: spinlock for IRQ synchronization. * @isr_mutex: mutex for IRQ thread. */ struct altr_i2c_dev { @@ -86,18 +85,14 @@ struct altr_i2c_dev { u32 fifo_size; u32 isr_mask; u32 isr_status; - spinlock_t lock; /* IRQ synchronization */ struct mutex isr_mutex; }; static void altr_i2c_int_enable(struct altr_i2c_dev *idev, u32 mask, bool enable) { - unsigned long flags; u32 int_en; - spin_lock_irqsave(&idev->lock, flags); - int_en = readl(idev->base + ALTR_I2C_ISER); if (enable) idev->isr_mask = int_en | mask; @@ -105,8 +100,6 @@ altr_i2c_int_enable(struct altr_i2c_dev *idev, u32 mask, bool enable) idev->isr_mask = int_en & ~mask; writel(idev->isr_mask, idev->base + ALTR_I2C_ISER); - - spin_unlock_irqrestore(&idev->lock, flags); } static void altr_i2c_int_clear(struct altr_i2c_dev *idev, u32 mask) @@ -346,6 +339,7 @@ static int altr_i2c_xfer_msg(struct altr_i2c_dev *idev, struct i2c_msg *msg) time_left = wait_for_completion_timeout(&idev->msg_complete, ALTR_I2C_XFER_TIMEOUT); + mutex_lock(&idev->isr_mutex); altr_i2c_int_enable(idev, imask, false); value = readl(idev->base + ALTR_I2C_STATUS) & ALTR_I2C_STAT_CORE; @@ -358,6 +352,7 @@ static int altr_i2c_xfer_msg(struct altr_i2c_dev *idev, struct i2c_msg *msg) } altr_i2c_core_disable(idev); + mutex_unlock(&idev->isr_mutex); return idev->msg_err; } @@ -389,23 +384,19 @@ static const struct i2c_algorithm altr_i2c_algo = { static int altr_i2c_probe(struct platform_device *pdev) { struct altr_i2c_dev *idev = NULL; - struct resource *res; int irq, ret; idev = devm_kzalloc(&pdev->dev, sizeof(*idev), GFP_KERNEL); if (!idev) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - idev->base = devm_ioremap_resource(&pdev->dev, res); + idev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(idev->base)) return PTR_ERR(idev->base); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "missing interrupt resource\n"); + if (irq < 0) return irq; - } idev->i2c_clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(idev->i2c_clk)) { @@ -415,7 +406,6 @@ static int altr_i2c_probe(struct platform_device *pdev) idev->dev = &pdev->dev; init_completion(&idev->msg_complete); - spin_lock_init(&idev->lock); mutex_init(&idev->isr_mutex); ret = device_property_read_u32(idev->dev, "fifo-size", @@ -453,7 +443,9 @@ static int altr_i2c_probe(struct platform_device *pdev) return ret; } + mutex_lock(&idev->isr_mutex); altr_i2c_init(idev); + mutex_unlock(&idev->isr_mutex); i2c_set_adapdata(&idev->adapter, idev); strlcpy(idev->adapter.name, pdev->name, sizeof(idev->adapter.name)); diff --git a/drivers/i2c/busses/i2c-at91-core.c b/drivers/i2c/busses/i2c-at91-core.c index 3da1a8acecb5..e14edd236108 100644 --- a/drivers/i2c/busses/i2c-at91-core.c +++ b/drivers/i2c/busses/i2c-at91-core.c @@ -131,6 +131,7 @@ static struct at91_twi_pdata sama5d2_config = { .has_dig_filtr = true, .has_adv_dig_filtr = true, .has_ana_filtr = true, + .has_clear_cmd = false, /* due to errata, CLEAR cmd is not working */ }; static struct at91_twi_pdata sam9x60_config = { @@ -142,6 +143,7 @@ static struct at91_twi_pdata sam9x60_config = { .has_dig_filtr = true, .has_adv_dig_filtr = true, .has_ana_filtr = true, + .has_clear_cmd = true, }; static const struct of_device_id atmel_twi_dt_ids[] = { diff --git a/drivers/i2c/busses/i2c-at91-master.c b/drivers/i2c/busses/i2c-at91-master.c index 37b96ac9dfae..363d540a8345 100644 --- a/drivers/i2c/busses/i2c-at91-master.c +++ b/drivers/i2c/busses/i2c-at91-master.c @@ -480,7 +480,6 @@ static int at91_do_twi_transfer(struct at91_twi_dev *dev) unsigned long time_left; bool has_unre_flag = dev->pdata->has_unre_flag; bool has_alt_cmd = dev->pdata->has_alt_cmd; - struct i2c_bus_recovery_info *rinfo = &dev->rinfo; /* * WARNING: the TXCOMP bit in the Status Register is NOT a clear on @@ -641,11 +640,12 @@ error: AT91_TWI_THRCLR | AT91_TWI_LOCKCLR); } - if (rinfo->get_sda && !(rinfo->get_sda(&dev->adapter))) { - dev_dbg(dev->dev, - "SDA is down; clear bus using gpio\n"); - i2c_recover_bus(&dev->adapter); - } + /* + * some faulty I2C slave devices might hold SDA down; + * we can send a bus clear command, hoping that the pins will be + * released + */ + i2c_recover_bus(&dev->adapter); return ret; } @@ -830,7 +830,7 @@ static void at91_unprepare_twi_recovery(struct i2c_adapter *adap) pinctrl_select_state(dev->pinctrl, dev->pinctrl_pins_default); } -static int at91_init_twi_recovery_info(struct platform_device *pdev, +static int at91_init_twi_recovery_gpio(struct platform_device *pdev, struct at91_twi_dev *dev) { struct i2c_bus_recovery_info *rinfo = &dev->rinfo; @@ -894,6 +894,41 @@ static int at91_init_twi_recovery_info(struct platform_device *pdev, return 0; } +static int at91_twi_recover_bus_cmd(struct i2c_adapter *adap) +{ + struct at91_twi_dev *dev = i2c_get_adapdata(adap); + + dev->transfer_status |= at91_twi_read(dev, AT91_TWI_SR); + if (!(dev->transfer_status & AT91_TWI_SDA)) { + dev_dbg(dev->dev, "SDA is down; sending bus clear command\n"); + if (dev->use_alt_cmd) { + unsigned int acr; + + acr = at91_twi_read(dev, AT91_TWI_ACR); + acr &= ~AT91_TWI_ACR_DATAL_MASK; + at91_twi_write(dev, AT91_TWI_ACR, acr); + } + at91_twi_write(dev, AT91_TWI_CR, AT91_TWI_CLEAR); + } + + return 0; +} + +static int at91_init_twi_recovery_info(struct platform_device *pdev, + struct at91_twi_dev *dev) +{ + struct i2c_bus_recovery_info *rinfo = &dev->rinfo; + bool has_clear_cmd = dev->pdata->has_clear_cmd; + + if (!has_clear_cmd) + return at91_init_twi_recovery_gpio(pdev, dev); + + rinfo->recover_bus = at91_twi_recover_bus_cmd; + dev->adapter.bus_recovery_info = rinfo; + + return 0; +} + int at91_twi_probe_master(struct platform_device *pdev, u32 phy_addr, struct at91_twi_dev *dev) { diff --git a/drivers/i2c/busses/i2c-at91.h b/drivers/i2c/busses/i2c-at91.h index f57a6cab96b4..7e7b4955ca7f 100644 --- a/drivers/i2c/busses/i2c-at91.h +++ b/drivers/i2c/busses/i2c-at91.h @@ -36,6 +36,7 @@ #define AT91_TWI_SVDIS BIT(5) /* Slave Transfer Disable */ #define AT91_TWI_QUICK BIT(6) /* SMBus quick command */ #define AT91_TWI_SWRST BIT(7) /* Software Reset */ +#define AT91_TWI_CLEAR BIT(15) /* Bus clear command */ #define AT91_TWI_ACMEN BIT(16) /* Alternative Command Mode Enable */ #define AT91_TWI_ACMDIS BIT(17) /* Alternative Command Mode Disable */ #define AT91_TWI_THRCLR BIT(24) /* Transmit Holding Register Clear */ @@ -69,6 +70,8 @@ #define AT91_TWI_NACK BIT(8) /* Not Acknowledged */ #define AT91_TWI_EOSACC BIT(11) /* End Of Slave Access */ #define AT91_TWI_LOCK BIT(23) /* TWI Lock due to Frame Errors */ +#define AT91_TWI_SCL BIT(24) /* TWI SCL status */ +#define AT91_TWI_SDA BIT(25) /* TWI SDA status */ #define AT91_TWI_INT_MASK \ (AT91_TWI_TXCOMP | AT91_TWI_RXRDY | AT91_TWI_TXRDY | AT91_TWI_NACK \ @@ -81,7 +84,8 @@ #define AT91_TWI_THR 0x0034 /* Transmit Holding Register */ #define AT91_TWI_ACR 0x0040 /* Alternative Command Register */ -#define AT91_TWI_ACR_DATAL(len) ((len) & 0xff) +#define AT91_TWI_ACR_DATAL_MASK GENMASK(15, 0) +#define AT91_TWI_ACR_DATAL(len) ((len) & AT91_TWI_ACR_DATAL_MASK) #define AT91_TWI_ACR_DIR BIT(8) #define AT91_TWI_FILTR 0x0044 @@ -118,6 +122,7 @@ struct at91_twi_pdata { bool has_dig_filtr; bool has_adv_dig_filtr; bool has_ana_filtr; + bool has_clear_cmd; struct at_dma_slave dma_slave; }; diff --git a/drivers/i2c/busses/i2c-axxia.c b/drivers/i2c/busses/i2c-axxia.c index be3681d08a8d..5294b73beca8 100644 --- a/drivers/i2c/busses/i2c-axxia.c +++ b/drivers/i2c/busses/i2c-axxia.c @@ -734,7 +734,6 @@ static int axxia_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct axxia_i2c_dev *idev = NULL; - struct resource *res; void __iomem *base; int ret = 0; @@ -742,16 +741,13 @@ static int axxia_i2c_probe(struct platform_device *pdev) if (!idev) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - base = devm_ioremap_resource(&pdev->dev, res); + base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); idev->irq = platform_get_irq(pdev, 0); - if (idev->irq < 0) { - dev_err(&pdev->dev, "missing interrupt resource\n"); + if (idev->irq < 0) return idev->irq; - } idev->i2c_clk = devm_clk_get(&pdev->dev, "i2c"); if (IS_ERR(idev->i2c_clk)) { diff --git a/drivers/i2c/busses/i2c-bcm-iproc.c b/drivers/i2c/busses/i2c-bcm-iproc.c index d091a12596ad..8a3c98866fb7 100644 --- a/drivers/i2c/busses/i2c-bcm-iproc.c +++ b/drivers/i2c/busses/i2c-bcm-iproc.c @@ -79,6 +79,7 @@ #define M_CMD_STATUS_RX_FIFO_FULL 0x6 #define M_CMD_PROTOCOL_SHIFT 9 #define M_CMD_PROTOCOL_MASK 0xf +#define M_CMD_PROTOCOL_QUICK 0x0 #define M_CMD_PROTOCOL_BLK_WR 0x7 #define M_CMD_PROTOCOL_BLK_RD 0x8 #define M_CMD_PROTOCOL_PROCESS 0xa @@ -768,7 +769,11 @@ static int bcm_iproc_i2c_xfer_internal(struct bcm_iproc_i2c_dev *iproc_i2c, * number of bytes to read */ val = BIT(M_CMD_START_BUSY_SHIFT); - if (msg->flags & I2C_M_RD) { + + if (msg->len == 0) { + /* SMBUS QUICK Command (Read/Write) */ + val |= (M_CMD_PROTOCOL_QUICK << M_CMD_PROTOCOL_SHIFT); + } else if (msg->flags & I2C_M_RD) { u32 protocol; iproc_i2c->rx_bytes = 0; @@ -830,8 +835,7 @@ static uint32_t bcm_iproc_i2c_functionality(struct i2c_adapter *adap) { u32 val; - /* We do not support the SMBUS Quick command */ - val = I2C_FUNC_I2C | (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK); + val = I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; if (adap->algo->reg_slave) val |= I2C_FUNC_SLAVE; diff --git a/drivers/i2c/busses/i2c-bcm-kona.c b/drivers/i2c/busses/i2c-bcm-kona.c index 572aebbb254e..ed5e1275ae46 100644 --- a/drivers/i2c/busses/i2c-bcm-kona.c +++ b/drivers/i2c/busses/i2c-bcm-kona.c @@ -750,7 +750,6 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev) int rc = 0; struct bcm_kona_i2c_dev *dev; struct i2c_adapter *adap; - struct resource *iomem; /* Allocate memory for private data structure */ dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL); @@ -762,8 +761,7 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev) init_completion(&dev->done); /* Map hardware registers */ - iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->base = devm_ioremap_resource(dev->device, iomem); + dev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dev->base)) return -ENOMEM; @@ -823,8 +821,7 @@ static int bcm_kona_i2c_probe(struct platform_device *pdev) /* Get the interrupt number */ dev->irq = platform_get_irq(pdev, 0); if (dev->irq < 0) { - dev_err(dev->device, "no irq resource\n"); - rc = -ENODEV; + rc = dev->irq; goto probe_disable_clk; } diff --git a/drivers/i2c/busses/i2c-brcmstb.c b/drivers/i2c/busses/i2c-brcmstb.c index 169a2836922d..d4e0a0f6732a 100644 --- a/drivers/i2c/busses/i2c-brcmstb.c +++ b/drivers/i2c/busses/i2c-brcmstb.c @@ -647,20 +647,22 @@ static int brcmstb_i2c_probe(struct platform_device *pdev) int_name = NULL; /* Get the interrupt number */ - dev->irq = platform_get_irq(pdev, 0); + dev->irq = platform_get_irq_optional(pdev, 0); /* disable the bsc interrupt line */ brcmstb_i2c_enable_disable_irq(dev, INT_DISABLE); /* register the ISR handler */ - rc = devm_request_irq(&pdev->dev, dev->irq, brcmstb_i2c_isr, - IRQF_SHARED, - int_name ? int_name : pdev->name, - dev); - - if (rc) { - dev_dbg(dev->device, "falling back to polling mode"); - dev->irq = -1; + if (dev->irq >= 0) { + rc = devm_request_irq(&pdev->dev, dev->irq, brcmstb_i2c_isr, + IRQF_SHARED, + int_name ? int_name : pdev->name, + dev); + + if (rc) { + dev_dbg(dev->device, "falling back to polling mode"); + dev->irq = -1; + } } if (of_property_read_u32(dev->device->of_node, diff --git a/drivers/i2c/busses/i2c-cadence.c b/drivers/i2c/busses/i2c-cadence.c index 89d58f7d2a25..4b72398af505 100644 --- a/drivers/i2c/busses/i2c-cadence.c +++ b/drivers/i2c/busses/i2c-cadence.c @@ -23,6 +23,7 @@ #define CDNS_I2C_ISR_OFFSET 0x10 /* IRQ Status Register, RW */ #define CDNS_I2C_XFER_SIZE_OFFSET 0x14 /* Transfer Size Register, RW */ #define CDNS_I2C_TIME_OUT_OFFSET 0x1C /* Time Out Register, RW */ +#define CDNS_I2C_IMR_OFFSET 0x20 /* IRQ Mask Register, RO */ #define CDNS_I2C_IER_OFFSET 0x24 /* IRQ Enable Register, WO */ #define CDNS_I2C_IDR_OFFSET 0x28 /* IRQ Disable Register, WO */ @@ -40,9 +41,17 @@ #define CDNS_I2C_CR_DIVB_SHIFT 8 #define CDNS_I2C_CR_DIVB_MASK (0x3f << CDNS_I2C_CR_DIVB_SHIFT) +#define CDNS_I2C_CR_MASTER_EN_MASK (CDNS_I2C_CR_NEA | \ + CDNS_I2C_CR_ACK_EN | \ + CDNS_I2C_CR_MS) + +#define CDNS_I2C_CR_SLAVE_EN_MASK ~CDNS_I2C_CR_MASTER_EN_MASK + /* Status Register Bit mask definitions */ #define CDNS_I2C_SR_BA BIT(8) +#define CDNS_I2C_SR_TXDV BIT(6) #define CDNS_I2C_SR_RXDV BIT(5) +#define CDNS_I2C_SR_RXRW BIT(3) /* * I2C Address Register Bit mask definitions @@ -91,6 +100,14 @@ CDNS_I2C_IXR_DATA | \ CDNS_I2C_IXR_COMP) +#define CDNS_I2C_IXR_SLAVE_INTR_MASK (CDNS_I2C_IXR_RX_UNF | \ + CDNS_I2C_IXR_TX_OVF | \ + CDNS_I2C_IXR_RX_OVF | \ + CDNS_I2C_IXR_TO | \ + CDNS_I2C_IXR_NACK | \ + CDNS_I2C_IXR_DATA | \ + CDNS_I2C_IXR_COMP) + #define CDNS_I2C_TIMEOUT msecs_to_jiffies(1000) /* timeout for pm runtime autosuspend */ #define CNDS_I2C_PM_TIMEOUT 1000 /* ms */ @@ -114,6 +131,32 @@ #define cdns_i2c_readreg(offset) readl_relaxed(id->membase + offset) #define cdns_i2c_writereg(val, offset) writel_relaxed(val, id->membase + offset) +#if IS_ENABLED(CONFIG_I2C_SLAVE) +/** + * enum cdns_i2c_mode - I2C Controller current operating mode + * + * @CDNS_I2C_MODE_SLAVE: I2C controller operating in slave mode + * @CDNS_I2C_MODE_MASTER: I2C Controller operating in master mode + */ +enum cdns_i2c_mode { + CDNS_I2C_MODE_SLAVE, + CDNS_I2C_MODE_MASTER, +}; + +/** + * enum cdns_i2c_slave_mode - Slave state when I2C is operating in slave mode + * + * @CDNS_I2C_SLAVE_STATE_IDLE: I2C slave idle + * @CDNS_I2C_SLAVE_STATE_SEND: I2C slave sending data to master + * @CDNS_I2C_SLAVE_STATE_RECV: I2C slave receiving data from master + */ +enum cdns_i2c_slave_state { + CDNS_I2C_SLAVE_STATE_IDLE, + CDNS_I2C_SLAVE_STATE_SEND, + CDNS_I2C_SLAVE_STATE_RECV, +}; +#endif + /** * struct cdns_i2c - I2C device private data structure * @@ -135,6 +178,10 @@ * @clk: Pointer to struct clk * @clk_rate_change_nb: Notifier block for clock rate changes * @quirks: flag for broken hold bit usage in r1p10 + * @ctrl_reg_diva_divb: value of fields DIV_A and DIV_B from CR register + * @slave: Registered slave instance. + * @dev_mode: I2C operating role(master/slave). + * @slave_state: I2C Slave state(idle/read/write). */ struct cdns_i2c { struct device *dev; @@ -155,6 +202,12 @@ struct cdns_i2c { struct clk *clk; struct notifier_block clk_rate_change_nb; u32 quirks; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + u16 ctrl_reg_diva_divb; + struct i2c_client *slave; + enum cdns_i2c_mode dev_mode; + enum cdns_i2c_slave_state slave_state; +#endif }; struct cdns_platform_data { @@ -183,17 +236,155 @@ static inline bool cdns_is_holdquirk(struct cdns_i2c *id, bool hold_wrkaround) (id->curr_recv_count == CDNS_I2C_FIFO_DEPTH + 1)); } +#if IS_ENABLED(CONFIG_I2C_SLAVE) +static void cdns_i2c_set_mode(enum cdns_i2c_mode mode, struct cdns_i2c *id) +{ + /* Disable all interrupts */ + cdns_i2c_writereg(CDNS_I2C_IXR_ALL_INTR_MASK, CDNS_I2C_IDR_OFFSET); + + /* Clear FIFO and transfer size */ + cdns_i2c_writereg(CDNS_I2C_CR_CLR_FIFO, CDNS_I2C_CR_OFFSET); + + /* Update device mode and state */ + id->dev_mode = mode; + id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE; + + switch (mode) { + case CDNS_I2C_MODE_MASTER: + /* Enable i2c master */ + cdns_i2c_writereg(id->ctrl_reg_diva_divb | + CDNS_I2C_CR_MASTER_EN_MASK, + CDNS_I2C_CR_OFFSET); + /* + * This delay is needed to give the IP some time to switch to + * the master mode. With lower values(like 110 us) i2cdetect + * will not detect any slave and without this delay, the IP will + * trigger a timeout interrupt. + */ + usleep_range(115, 125); + break; + case CDNS_I2C_MODE_SLAVE: + /* Enable i2c slave */ + cdns_i2c_writereg(id->ctrl_reg_diva_divb & + CDNS_I2C_CR_SLAVE_EN_MASK, + CDNS_I2C_CR_OFFSET); + + /* Setting slave address */ + cdns_i2c_writereg(id->slave->addr & CDNS_I2C_ADDR_MASK, + CDNS_I2C_ADDR_OFFSET); + + /* Enable slave send/receive interrupts */ + cdns_i2c_writereg(CDNS_I2C_IXR_SLAVE_INTR_MASK, + CDNS_I2C_IER_OFFSET); + break; + } +} + +static void cdns_i2c_slave_rcv_data(struct cdns_i2c *id) +{ + u8 bytes; + unsigned char data; + + /* Prepare backend for data reception */ + if (id->slave_state == CDNS_I2C_SLAVE_STATE_IDLE) { + id->slave_state = CDNS_I2C_SLAVE_STATE_RECV; + i2c_slave_event(id->slave, I2C_SLAVE_WRITE_REQUESTED, NULL); + } + + /* Fetch number of bytes to receive */ + bytes = cdns_i2c_readreg(CDNS_I2C_XFER_SIZE_OFFSET); + + /* Read data and send to backend */ + while (bytes--) { + data = cdns_i2c_readreg(CDNS_I2C_DATA_OFFSET); + i2c_slave_event(id->slave, I2C_SLAVE_WRITE_RECEIVED, &data); + } +} + +static void cdns_i2c_slave_send_data(struct cdns_i2c *id) +{ + u8 data; + + /* Prepare backend for data transmission */ + if (id->slave_state == CDNS_I2C_SLAVE_STATE_IDLE) { + id->slave_state = CDNS_I2C_SLAVE_STATE_SEND; + i2c_slave_event(id->slave, I2C_SLAVE_READ_REQUESTED, &data); + } else { + i2c_slave_event(id->slave, I2C_SLAVE_READ_PROCESSED, &data); + } + + /* Send data over bus */ + cdns_i2c_writereg(data, CDNS_I2C_DATA_OFFSET); +} + /** - * cdns_i2c_isr - Interrupt handler for the I2C device - * @irq: irq number for the I2C device - * @ptr: void pointer to cdns_i2c structure + * cdns_i2c_slave_isr - Interrupt handler for the I2C device in slave role + * @ptr: Pointer to I2C device private data + * + * This function handles the data interrupt and transfer complete interrupt of + * the I2C device in slave role. + * + * Return: IRQ_HANDLED always + */ +static irqreturn_t cdns_i2c_slave_isr(void *ptr) +{ + struct cdns_i2c *id = ptr; + unsigned int isr_status, i2c_status; + + /* Fetch the interrupt status */ + isr_status = cdns_i2c_readreg(CDNS_I2C_ISR_OFFSET); + cdns_i2c_writereg(isr_status, CDNS_I2C_ISR_OFFSET); + + /* Ignore masked interrupts */ + isr_status &= ~cdns_i2c_readreg(CDNS_I2C_IMR_OFFSET); + + /* Fetch transfer mode (send/receive) */ + i2c_status = cdns_i2c_readreg(CDNS_I2C_SR_OFFSET); + + /* Handle data send/receive */ + if (i2c_status & CDNS_I2C_SR_RXRW) { + /* Send data to master */ + if (isr_status & CDNS_I2C_IXR_DATA) + cdns_i2c_slave_send_data(id); + + if (isr_status & CDNS_I2C_IXR_COMP) { + id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE; + i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL); + } + } else { + /* Receive data from master */ + if (isr_status & CDNS_I2C_IXR_DATA) + cdns_i2c_slave_rcv_data(id); + + if (isr_status & CDNS_I2C_IXR_COMP) { + cdns_i2c_slave_rcv_data(id); + id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE; + i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL); + } + } + + /* Master indicated xfer stop or fifo underflow/overflow */ + if (isr_status & (CDNS_I2C_IXR_NACK | CDNS_I2C_IXR_RX_OVF | + CDNS_I2C_IXR_RX_UNF | CDNS_I2C_IXR_TX_OVF)) { + id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE; + i2c_slave_event(id->slave, I2C_SLAVE_STOP, NULL); + cdns_i2c_writereg(CDNS_I2C_CR_CLR_FIFO, CDNS_I2C_CR_OFFSET); + } + + return IRQ_HANDLED; +} +#endif + +/** + * cdns_i2c_master_isr - Interrupt handler for the I2C device in master role + * @ptr: Pointer to I2C device private data * * This function handles the data interrupt, transfer complete interrupt and - * the error interrupts of the I2C device. + * the error interrupts of the I2C device in master role. * * Return: IRQ_HANDLED always */ -static irqreturn_t cdns_i2c_isr(int irq, void *ptr) +static irqreturn_t cdns_i2c_master_isr(void *ptr) { unsigned int isr_status, avail_bytes, updatetx; unsigned int bytes_to_send; @@ -358,6 +549,27 @@ static irqreturn_t cdns_i2c_isr(int irq, void *ptr) } /** + * cdns_i2c_isr - Interrupt handler for the I2C device + * @irq: irq number for the I2C device + * @ptr: void pointer to cdns_i2c structure + * + * This function passes the control to slave/master based on current role of + * i2c controller. + * + * Return: IRQ_HANDLED always + */ +static irqreturn_t cdns_i2c_isr(int irq, void *ptr) +{ +#if IS_ENABLED(CONFIG_I2C_SLAVE) + struct cdns_i2c *id = ptr; + + if (id->dev_mode == CDNS_I2C_MODE_SLAVE) + return cdns_i2c_slave_isr(ptr); +#endif + return cdns_i2c_master_isr(ptr); +} + +/** * cdns_i2c_mrecv - Prepare and start a master receive operation * @id: pointer to the i2c device structure */ @@ -577,10 +789,28 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, u32 reg; struct cdns_i2c *id = adap->algo_data; bool hold_quirk; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + bool change_role = false; +#endif ret = pm_runtime_get_sync(id->dev); if (ret < 0) return ret; + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + /* Check i2c operating mode and switch if possible */ + if (id->dev_mode == CDNS_I2C_MODE_SLAVE) { + if (id->slave_state != CDNS_I2C_SLAVE_STATE_IDLE) + return -EAGAIN; + + /* Set mode to master */ + cdns_i2c_set_mode(CDNS_I2C_MODE_MASTER, id); + + /* Mark flag to change role once xfer is completed */ + change_role = true; + } +#endif + /* Check if the bus is free */ if (cdns_i2c_readreg(CDNS_I2C_SR_OFFSET) & CDNS_I2C_SR_BA) { ret = -EAGAIN; @@ -639,7 +869,15 @@ static int cdns_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, } ret = num; + out: + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + /* Switch i2c mode to slave */ + if (change_role) + cdns_i2c_set_mode(CDNS_I2C_MODE_SLAVE, id); +#endif + pm_runtime_mark_last_busy(id->dev); pm_runtime_put_autosuspend(id->dev); return ret; @@ -653,14 +891,67 @@ out: */ static u32 cdns_i2c_func(struct i2c_adapter *adap) { - return I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | - (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) | - I2C_FUNC_SMBUS_BLOCK_DATA; + u32 func = I2C_FUNC_I2C | I2C_FUNC_10BIT_ADDR | + (I2C_FUNC_SMBUS_EMUL & ~I2C_FUNC_SMBUS_QUICK) | + I2C_FUNC_SMBUS_BLOCK_DATA; + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + func |= I2C_FUNC_SLAVE; +#endif + + return func; } +#if IS_ENABLED(CONFIG_I2C_SLAVE) +static int cdns_reg_slave(struct i2c_client *slave) +{ + int ret; + struct cdns_i2c *id = container_of(slave->adapter, struct cdns_i2c, + adap); + + if (id->slave) + return -EBUSY; + + if (slave->flags & I2C_CLIENT_TEN) + return -EAFNOSUPPORT; + + ret = pm_runtime_get_sync(id->dev); + if (ret < 0) + return ret; + + /* Store slave information */ + id->slave = slave; + + /* Enable I2C slave */ + cdns_i2c_set_mode(CDNS_I2C_MODE_SLAVE, id); + + return 0; +} + +static int cdns_unreg_slave(struct i2c_client *slave) +{ + struct cdns_i2c *id = container_of(slave->adapter, struct cdns_i2c, + adap); + + pm_runtime_put(id->dev); + + /* Remove slave information */ + id->slave = NULL; + + /* Enable I2C master */ + cdns_i2c_set_mode(CDNS_I2C_MODE_MASTER, id); + + return 0; +} +#endif + static const struct i2c_algorithm cdns_i2c_algo = { .master_xfer = cdns_i2c_master_xfer, .functionality = cdns_i2c_func, +#if IS_ENABLED(CONFIG_I2C_SLAVE) + .reg_slave = cdns_reg_slave, + .unreg_slave = cdns_unreg_slave, +#endif }; /** @@ -755,7 +1046,10 @@ static int cdns_i2c_setclk(unsigned long clk_in, struct cdns_i2c *id) ctrl_reg |= ((div_a << CDNS_I2C_CR_DIVA_SHIFT) | (div_b << CDNS_I2C_CR_DIVB_SHIFT)); cdns_i2c_writereg(ctrl_reg, CDNS_I2C_CR_OFFSET); - +#if IS_ENABLED(CONFIG_I2C_SLAVE) + id->ctrl_reg_diva_divb = ctrl_reg & (CDNS_I2C_CR_DIVA_MASK | + CDNS_I2C_CR_DIVB_MASK); +#endif return 0; } @@ -906,8 +1200,7 @@ static int cdns_i2c_probe(struct platform_device *pdev) id->quirks = data->quirks; } - r_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - id->membase = devm_ioremap_resource(&pdev->dev, r_mem); + id->membase = devm_platform_get_and_ioremap_resource(pdev, 0, &r_mem); if (IS_ERR(id->membase)) return PTR_ERR(id->membase); @@ -949,8 +1242,12 @@ static int cdns_i2c_probe(struct platform_device *pdev) if (ret || (id->i2c_clk > I2C_MAX_FAST_MODE_FREQ)) id->i2c_clk = I2C_MAX_STANDARD_MODE_FREQ; - cdns_i2c_writereg(CDNS_I2C_CR_ACK_EN | CDNS_I2C_CR_NEA | CDNS_I2C_CR_MS, - CDNS_I2C_CR_OFFSET); +#if IS_ENABLED(CONFIG_I2C_SLAVE) + /* Set initial mode to master */ + id->dev_mode = CDNS_I2C_MODE_MASTER; + id->slave_state = CDNS_I2C_SLAVE_STATE_IDLE; +#endif + cdns_i2c_writereg(CDNS_I2C_CR_MASTER_EN_MASK, CDNS_I2C_CR_OFFSET); ret = cdns_i2c_setclk(id->input_clk, id); if (ret) { diff --git a/drivers/i2c/busses/i2c-cht-wc.c b/drivers/i2c/busses/i2c-cht-wc.c index 35e55feda763..f80d79e973cd 100644 --- a/drivers/i2c/busses/i2c-cht-wc.c +++ b/drivers/i2c/busses/i2c-cht-wc.c @@ -314,10 +314,8 @@ static int cht_wc_i2c_adap_i2c_probe(struct platform_device *pdev) int ret, reg, irq; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "Error missing irq resource\n"); - return -EINVAL; - } + if (irq < 0) + return irq; adap = devm_kzalloc(&pdev->dev, sizeof(*adap), GFP_KERNEL); if (!adap) diff --git a/drivers/i2c/busses/i2c-davinci.c b/drivers/i2c/busses/i2c-davinci.c index e3ceb256a380..232a7679b69b 100644 --- a/drivers/i2c/busses/i2c-davinci.c +++ b/drivers/i2c/busses/i2c-davinci.c @@ -761,7 +761,6 @@ static int davinci_i2c_probe(struct platform_device *pdev) { struct davinci_i2c_dev *dev; struct i2c_adapter *adap; - struct resource *mem; struct i2c_bus_recovery_info *rinfo; int r, irq; @@ -814,8 +813,7 @@ static int davinci_i2c_probe(struct platform_device *pdev) if (IS_ERR(dev->clk)) return PTR_ERR(dev->clk); - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->base = devm_ioremap_resource(&pdev->dev, mem); + dev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(dev->base)) { return PTR_ERR(dev->base); } diff --git a/drivers/i2c/busses/i2c-designware-common.c b/drivers/i2c/busses/i2c-designware-common.c index c70c6fc09ee3..e3a8640db7da 100644 --- a/drivers/i2c/busses/i2c-designware-common.c +++ b/drivers/i2c/busses/i2c-designware-common.c @@ -8,17 +8,22 @@ * Copyright (C) 2007 MontaVista Software Inc. * Copyright (C) 2009 Provigent Ltd. */ +#include <linux/acpi.h> #include <linux/clk.h> #include <linux/delay.h> -#include <linux/export.h> -#include <linux/errno.h> +#include <linux/device.h> #include <linux/err.h> +#include <linux/errno.h> +#include <linux/export.h> #include <linux/i2c.h> #include <linux/interrupt.h> #include <linux/io.h> +#include <linux/kernel.h> #include <linux/module.h> #include <linux/pm_runtime.h> +#include <linux/regmap.h> #include <linux/swab.h> +#include <linux/types.h> #include "i2c-designware-core.h" @@ -53,69 +58,267 @@ static char *abort_sources[] = { "incorrect slave-transmitter mode configuration", }; -u32 dw_readl(struct dw_i2c_dev *dev, int offset) +static int dw_reg_read(void *context, unsigned int reg, unsigned int *val) { - u32 value; + struct dw_i2c_dev *dev = context; - if (dev->flags & ACCESS_16BIT) - value = readw_relaxed(dev->base + offset) | - (readw_relaxed(dev->base + offset + 2) << 16); - else - value = readl_relaxed(dev->base + offset); + *val = readl_relaxed(dev->base + reg); - if (dev->flags & ACCESS_SWAP) - return swab32(value); - else - return value; + return 0; } -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset) +static int dw_reg_write(void *context, unsigned int reg, unsigned int val) { - if (dev->flags & ACCESS_SWAP) - b = swab32(b); - - if (dev->flags & ACCESS_16BIT) { - writew_relaxed((u16)b, dev->base + offset); - writew_relaxed((u16)(b >> 16), dev->base + offset + 2); - } else { - writel_relaxed(b, dev->base + offset); - } + struct dw_i2c_dev *dev = context; + + writel_relaxed(val, dev->base + reg); + + return 0; +} + +static int dw_reg_read_swab(void *context, unsigned int reg, unsigned int *val) +{ + struct dw_i2c_dev *dev = context; + + *val = swab32(readl_relaxed(dev->base + reg)); + + return 0; +} + +static int dw_reg_write_swab(void *context, unsigned int reg, unsigned int val) +{ + struct dw_i2c_dev *dev = context; + + writel_relaxed(swab32(val), dev->base + reg); + + return 0; +} + +static int dw_reg_read_word(void *context, unsigned int reg, unsigned int *val) +{ + struct dw_i2c_dev *dev = context; + + *val = readw_relaxed(dev->base + reg) | + (readw_relaxed(dev->base + reg + 2) << 16); + + return 0; +} + +static int dw_reg_write_word(void *context, unsigned int reg, unsigned int val) +{ + struct dw_i2c_dev *dev = context; + + writew_relaxed(val, dev->base + reg); + writew_relaxed(val >> 16, dev->base + reg + 2); + + return 0; } /** - * i2c_dw_set_reg_access() - Set register access flags + * i2c_dw_init_regmap() - Initialize registers map * @dev: device private data * - * Autodetects needed register access mode and sets access flags accordingly. - * This must be called before doing any other register access. + * Autodetects needed register access mode and creates the regmap with + * corresponding read/write callbacks. This must be called before doing any + * other register access. */ -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev) +int i2c_dw_init_regmap(struct dw_i2c_dev *dev) { + struct regmap_config map_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .disable_locking = true, + .reg_read = dw_reg_read, + .reg_write = dw_reg_write, + .max_register = DW_IC_COMP_TYPE, + }; u32 reg; int ret; + /* + * Skip detecting the registers map configuration if the regmap has + * already been provided by a higher code. + */ + if (dev->map) + return 0; + ret = i2c_dw_acquire_lock(dev); if (ret) return ret; - reg = dw_readl(dev, DW_IC_COMP_TYPE); + reg = readl(dev->base + DW_IC_COMP_TYPE); i2c_dw_release_lock(dev); if (reg == swab32(DW_IC_COMP_TYPE_VALUE)) { - /* Configure register endianness access */ - dev->flags |= ACCESS_SWAP; + map_cfg.reg_read = dw_reg_read_swab; + map_cfg.reg_write = dw_reg_write_swab; } else if (reg == (DW_IC_COMP_TYPE_VALUE & 0x0000ffff)) { - /* Configure register access mode 16bit */ - dev->flags |= ACCESS_16BIT; + map_cfg.reg_read = dw_reg_read_word; + map_cfg.reg_write = dw_reg_write_word; } else if (reg != DW_IC_COMP_TYPE_VALUE) { dev_err(dev->dev, "Unknown Synopsys component type: 0x%08x\n", reg); return -ENODEV; } + /* + * Note we'll check the return value of the regmap IO accessors only + * at the probe stage. The rest of the code won't do this because + * basically we have MMIO-based regmap so non of the read/write methods + * can fail. + */ + dev->map = devm_regmap_init(dev->dev, NULL, dev, &map_cfg); + if (IS_ERR(dev->map)) { + dev_err(dev->dev, "Failed to init the registers map\n"); + return PTR_ERR(dev->map); + } + return 0; } +static const u32 supported_speeds[] = { + I2C_MAX_HIGH_SPEED_MODE_FREQ, + I2C_MAX_FAST_MODE_PLUS_FREQ, + I2C_MAX_FAST_MODE_FREQ, + I2C_MAX_STANDARD_MODE_FREQ, +}; + +int i2c_dw_validate_speed(struct dw_i2c_dev *dev) +{ + struct i2c_timings *t = &dev->timings; + unsigned int i; + + /* + * Only standard mode at 100kHz, fast mode at 400kHz, + * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported. + */ + for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { + if (t->bus_freq_hz == supported_speeds[i]) + return 0; + } + + dev_err(dev->dev, + "%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n", + t->bus_freq_hz); + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(i2c_dw_validate_speed); + +#ifdef CONFIG_ACPI + +#include <linux/dmi.h> + +/* + * The HCNT/LCNT information coming from ACPI should be the most accurate + * for given platform. However, some systems get it wrong. On such systems + * we get better results by calculating those based on the input clock. + */ +static const struct dmi_system_id i2c_dw_no_acpi_params[] = { + { + .ident = "Dell Inspiron 7348", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), + DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"), + }, + }, + {} +}; + +static void i2c_dw_acpi_params(struct device *device, char method[], + u16 *hcnt, u16 *lcnt, u32 *sda_hold) +{ + struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; + acpi_handle handle = ACPI_HANDLE(device); + union acpi_object *obj; + + if (dmi_check_system(i2c_dw_no_acpi_params)) + return; + + if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf))) + return; + + obj = (union acpi_object *)buf.pointer; + if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) { + const union acpi_object *objs = obj->package.elements; + + *hcnt = (u16)objs[0].integer.value; + *lcnt = (u16)objs[1].integer.value; + *sda_hold = (u32)objs[2].integer.value; + } + + kfree(buf.pointer); +} + +int i2c_dw_acpi_configure(struct device *device) +{ + struct dw_i2c_dev *dev = dev_get_drvdata(device); + struct i2c_timings *t = &dev->timings; + u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0; + + /* + * Try to get SDA hold time and *CNT values from an ACPI method for + * selected speed modes. + */ + i2c_dw_acpi_params(device, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht); + i2c_dw_acpi_params(device, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht); + i2c_dw_acpi_params(device, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht); + i2c_dw_acpi_params(device, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht); + + switch (t->bus_freq_hz) { + case I2C_MAX_STANDARD_MODE_FREQ: + dev->sda_hold_time = ss_ht; + break; + case I2C_MAX_FAST_MODE_PLUS_FREQ: + dev->sda_hold_time = fp_ht; + break; + case I2C_MAX_HIGH_SPEED_MODE_FREQ: + dev->sda_hold_time = hs_ht; + break; + case I2C_MAX_FAST_MODE_FREQ: + default: + dev->sda_hold_time = fs_ht; + break; + } + + return 0; +} +EXPORT_SYMBOL_GPL(i2c_dw_acpi_configure); + +void i2c_dw_acpi_adjust_bus_speed(struct device *device) +{ + struct dw_i2c_dev *dev = dev_get_drvdata(device); + struct i2c_timings *t = &dev->timings; + u32 acpi_speed; + int i; + + acpi_speed = i2c_acpi_find_bus_speed(device); + /* + * Some DSTDs use a non standard speed, round down to the lowest + * standard speed. + */ + for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { + if (acpi_speed >= supported_speeds[i]) + break; + } + acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0; + + /* + * Find bus speed from the "clock-frequency" device property, ACPI + * or by using fast mode if neither is set. + */ + if (acpi_speed && t->bus_freq_hz) + t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed); + else if (acpi_speed || t->bus_freq_hz) + t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed); + else + t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; +} +EXPORT_SYMBOL_GPL(i2c_dw_acpi_adjust_bus_speed); + +#endif /* CONFIG_ACPI */ + u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset) { /* @@ -181,11 +384,17 @@ int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev) return ret; /* Configure SDA Hold Time if required */ - reg = dw_readl(dev, DW_IC_COMP_VERSION); + ret = regmap_read(dev->map, DW_IC_COMP_VERSION, ®); + if (ret) + goto err_release_lock; + if (reg >= DW_IC_SDA_HOLD_MIN_VERS) { if (!dev->sda_hold_time) { /* Keep previous hold time setting if no one set it */ - dev->sda_hold_time = dw_readl(dev, DW_IC_SDA_HOLD); + ret = regmap_read(dev->map, DW_IC_SDA_HOLD, + &dev->sda_hold_time); + if (ret) + goto err_release_lock; } /* @@ -209,14 +418,16 @@ int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev) dev->sda_hold_time = 0; } +err_release_lock: i2c_dw_release_lock(dev); - return 0; + return ret; } void __i2c_dw_disable(struct dw_i2c_dev *dev) { int timeout = 100; + u32 status; do { __i2c_dw_disable_nowait(dev); @@ -224,7 +435,8 @@ void __i2c_dw_disable(struct dw_i2c_dev *dev) * The enable status register may be unimplemented, but * in that case this test reads zero and exits the loop. */ - if ((dw_readl(dev, DW_IC_ENABLE_STATUS) & 1) == 0) + regmap_read(dev->map, DW_IC_ENABLE_STATUS, &status); + if ((status & 1) == 0) return; /* @@ -303,22 +515,23 @@ void i2c_dw_release_lock(struct dw_i2c_dev *dev) */ int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev) { - int timeout = TIMEOUT; + u32 status; + int ret; - while (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) { - if (timeout <= 0) { - dev_warn(dev->dev, "timeout waiting for bus ready\n"); - i2c_recover_bus(&dev->adapter); + ret = regmap_read_poll_timeout(dev->map, DW_IC_STATUS, status, + !(status & DW_IC_STATUS_ACTIVITY), + 1100, 20000); + if (ret) { + dev_warn(dev->dev, "timeout waiting for bus ready\n"); - if (dw_readl(dev, DW_IC_STATUS) & DW_IC_STATUS_ACTIVITY) - return -ETIMEDOUT; - return 0; - } - timeout--; - usleep_range(1000, 1100); + i2c_recover_bus(&dev->adapter); + + regmap_read(dev->map, DW_IC_STATUS, &status); + if (!(status & DW_IC_STATUS_ACTIVITY)) + ret = 0; } - return 0; + return ret; } int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev) @@ -344,15 +557,19 @@ int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev) return -EIO; } -void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev) +int i2c_dw_set_fifo_size(struct dw_i2c_dev *dev) { u32 param, tx_fifo_depth, rx_fifo_depth; + int ret; /* * Try to detect the FIFO depth if not set by interface driver, * the depth could be from 2 to 256 from HW spec. */ - param = dw_readl(dev, DW_IC_COMP_PARAM_1); + ret = regmap_read(dev->map, DW_IC_COMP_PARAM_1, ¶m); + if (ret) + return ret; + tx_fifo_depth = ((param >> 16) & 0xff) + 1; rx_fifo_depth = ((param >> 8) & 0xff) + 1; if (!dev->tx_fifo_depth) { @@ -364,6 +581,8 @@ void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev) dev->rx_fifo_depth = min_t(u32, dev->rx_fifo_depth, rx_fifo_depth); } + + return 0; } u32 i2c_dw_func(struct i2c_adapter *adap) @@ -375,17 +594,19 @@ u32 i2c_dw_func(struct i2c_adapter *adap) void i2c_dw_disable(struct dw_i2c_dev *dev) { + u32 dummy; + /* Disable controller */ __i2c_dw_disable(dev); /* Disable all interrupts */ - dw_writel(dev, 0, DW_IC_INTR_MASK); - dw_readl(dev, DW_IC_CLR_INTR); + regmap_write(dev->map, DW_IC_INTR_MASK, 0); + regmap_read(dev->map, DW_IC_CLR_INTR, &dummy); } void i2c_dw_disable_int(struct dw_i2c_dev *dev) { - dw_writel(dev, 0, DW_IC_INTR_MASK); + regmap_write(dev->map, DW_IC_INTR_MASK, 0); } MODULE_DESCRIPTION("Synopsys DesignWare I2C bus adapter core"); diff --git a/drivers/i2c/busses/i2c-designware-core.h b/drivers/i2c/busses/i2c-designware-core.h index b220ad64c38d..556673a1f61b 100644 --- a/drivers/i2c/busses/i2c-designware-core.h +++ b/drivers/i2c/busses/i2c-designware-core.h @@ -9,7 +9,14 @@ * Copyright (C) 2009 Provigent Ltd. */ +#include <linux/bits.h> +#include <linux/compiler_types.h> +#include <linux/completion.h> +#include <linux/dev_printk.h> +#include <linux/errno.h> #include <linux/i2c.h> +#include <linux/regmap.h> +#include <linux/types.h> #define DW_IC_DEFAULT_FUNCTIONALITY (I2C_FUNC_I2C | \ I2C_FUNC_SMBUS_BYTE | \ @@ -120,8 +127,6 @@ #define STATUS_WRITE_IN_PROGRESS 0x1 #define STATUS_READ_IN_PROGRESS 0x2 -#define TIMEOUT 20 /* ms */ - /* * operation modes */ @@ -170,11 +175,17 @@ DW_IC_TX_ABRT_TXDATA_NOACK | \ DW_IC_TX_ABRT_GCALL_NOACK) +struct clk; +struct device; +struct reset_control; /** * struct dw_i2c_dev - private i2c-designware data * @dev: driver model device node + * @map: IO registers map + * @sysmap: System controller registers map * @base: IO registers pointer + * @ext: Extended IO registers pointer * @cmd_complete: tx completion indicator * @clk: input reference clock * @pclk: clock required to access the registers @@ -224,6 +235,8 @@ */ struct dw_i2c_dev { struct device *dev; + struct regmap *map; + struct regmap *sysmap; void __iomem *base; void __iomem *ext; struct completion cmd_complete; @@ -232,7 +245,6 @@ struct dw_i2c_dev { struct reset_control *rst; struct i2c_client *slave; u32 (*get_clk_rate_khz) (struct dw_i2c_dev *dev); - struct dw_pci_controller *controller; int cmd_err; struct i2c_msg *msgs; int msgs_num; @@ -276,18 +288,14 @@ struct dw_i2c_dev { bool suspended; }; -#define ACCESS_SWAP 0x00000001 -#define ACCESS_16BIT 0x00000002 -#define ACCESS_INTR_MASK 0x00000004 -#define ACCESS_NO_IRQ_SUSPEND 0x00000008 +#define ACCESS_INTR_MASK 0x00000001 +#define ACCESS_NO_IRQ_SUSPEND 0x00000002 -#define MODEL_CHERRYTRAIL 0x00000100 -#define MODEL_MSCC_OCELOT 0x00000200 +#define MODEL_MSCC_OCELOT 0x00000100 +#define MODEL_BAIKAL_BT1 0x00000200 #define MODEL_MASK 0x00000f00 -u32 dw_readl(struct dw_i2c_dev *dev, int offset); -void dw_writel(struct dw_i2c_dev *dev, u32 b, int offset); -int i2c_dw_set_reg_access(struct dw_i2c_dev *dev); +int i2c_dw_init_regmap(struct dw_i2c_dev *dev); u32 i2c_dw_scl_hcnt(u32 ic_clk, u32 tSYMBOL, u32 tf, int cond, int offset); u32 i2c_dw_scl_lcnt(u32 ic_clk, u32 tLOW, u32 tf, int offset); int i2c_dw_set_sda_hold(struct dw_i2c_dev *dev); @@ -297,32 +305,67 @@ int i2c_dw_acquire_lock(struct dw_i2c_dev *dev); void i2c_dw_release_lock(struct dw_i2c_dev *dev); int i2c_dw_wait_bus_not_busy(struct dw_i2c_dev *dev); int i2c_dw_handle_tx_abort(struct dw_i2c_dev *dev); -void i2c_dw_set_fifo_size(struct dw_i2c_dev *dev); +int i2c_dw_set_fifo_size(struct dw_i2c_dev *dev); u32 i2c_dw_func(struct i2c_adapter *adap); void i2c_dw_disable(struct dw_i2c_dev *dev); void i2c_dw_disable_int(struct dw_i2c_dev *dev); static inline void __i2c_dw_enable(struct dw_i2c_dev *dev) { - dw_writel(dev, 1, DW_IC_ENABLE); + regmap_write(dev->map, DW_IC_ENABLE, 1); } static inline void __i2c_dw_disable_nowait(struct dw_i2c_dev *dev) { - dw_writel(dev, 0, DW_IC_ENABLE); + regmap_write(dev->map, DW_IC_ENABLE, 0); } void __i2c_dw_disable(struct dw_i2c_dev *dev); -extern int i2c_dw_probe(struct dw_i2c_dev *dev); +extern void i2c_dw_configure_master(struct dw_i2c_dev *dev); +extern int i2c_dw_probe_master(struct dw_i2c_dev *dev); + #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_SLAVE) +extern void i2c_dw_configure_slave(struct dw_i2c_dev *dev); extern int i2c_dw_probe_slave(struct dw_i2c_dev *dev); #else +static inline void i2c_dw_configure_slave(struct dw_i2c_dev *dev) { } static inline int i2c_dw_probe_slave(struct dw_i2c_dev *dev) { return -EINVAL; } #endif +static inline int i2c_dw_probe(struct dw_i2c_dev *dev) +{ + switch (dev->mode) { + case DW_IC_SLAVE: + return i2c_dw_probe_slave(dev); + case DW_IC_MASTER: + return i2c_dw_probe_master(dev); + default: + dev_err(dev->dev, "Wrong operation mode: %d\n", dev->mode); + return -EINVAL; + } +} + +static inline void i2c_dw_configure(struct dw_i2c_dev *dev) +{ + if (i2c_detect_slave_mode(dev->dev)) + i2c_dw_configure_slave(dev); + else + i2c_dw_configure_master(dev); +} + #if IS_ENABLED(CONFIG_I2C_DESIGNWARE_BAYTRAIL) extern int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev); #else static inline int i2c_dw_probe_lock_support(struct dw_i2c_dev *dev) { return 0; } #endif + +int i2c_dw_validate_speed(struct dw_i2c_dev *dev); + +#if IS_ENABLED(CONFIG_ACPI) +int i2c_dw_acpi_configure(struct device *device); +void i2c_dw_acpi_adjust_bus_speed(struct device *device); +#else +static inline int i2c_dw_acpi_configure(struct device *device) { return -ENODEV; } +static inline void i2c_dw_acpi_adjust_bus_speed(struct device *device) {} +#endif diff --git a/drivers/i2c/busses/i2c-designware-master.c b/drivers/i2c/busses/i2c-designware-master.c index 3a58eef20936..d6425ad6e6a3 100644 --- a/drivers/i2c/busses/i2c-designware-master.c +++ b/drivers/i2c/busses/i2c-designware-master.c @@ -18,6 +18,7 @@ #include <linux/io.h> #include <linux/module.h> #include <linux/pm_runtime.h> +#include <linux/regmap.h> #include <linux/reset.h> #include "i2c-designware-core.h" @@ -25,11 +26,11 @@ static void i2c_dw_configure_fifo_master(struct dw_i2c_dev *dev) { /* Configure Tx/Rx FIFO threshold levels */ - dw_writel(dev, dev->tx_fifo_depth / 2, DW_IC_TX_TL); - dw_writel(dev, 0, DW_IC_RX_TL); + regmap_write(dev->map, DW_IC_TX_TL, dev->tx_fifo_depth / 2); + regmap_write(dev->map, DW_IC_RX_TL, 0); /* Configure the I2C master */ - dw_writel(dev, dev->master_cfg, DW_IC_CON); + regmap_write(dev->map, DW_IC_CON, dev->master_cfg); } static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) @@ -44,8 +45,11 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) ret = i2c_dw_acquire_lock(dev); if (ret) return ret; - comp_param1 = dw_readl(dev, DW_IC_COMP_PARAM_1); + + ret = regmap_read(dev->map, DW_IC_COMP_PARAM_1, &comp_param1); i2c_dw_release_lock(dev); + if (ret) + return ret; /* Set standard and fast speed dividers for high/low periods */ sda_falling_time = t->sda_fall_ns ?: 300; /* ns */ @@ -76,14 +80,27 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) */ if (t->bus_freq_hz == 1000000) { /* - * Check are fast mode plus parameters available and use - * fast mode if not. + * Check are Fast Mode Plus parameters available. Calculate + * SCL timing parameters for Fast Mode Plus if not set. */ if (dev->fp_hcnt && dev->fp_lcnt) { dev->fs_hcnt = dev->fp_hcnt; dev->fs_lcnt = dev->fp_lcnt; - fp_str = " Plus"; + } else { + ic_clk = i2c_dw_clk_rate(dev); + dev->fs_hcnt = + i2c_dw_scl_hcnt(ic_clk, + 260, /* tHIGH = 260 ns */ + sda_falling_time, + 0, /* DW default */ + 0); /* No offset */ + dev->fs_lcnt = + i2c_dw_scl_lcnt(ic_clk, + 500, /* tLOW = 500 ns */ + scl_falling_time, + 0); /* No offset */ } + fp_str = " Plus"; } /* * Calculate SCL timing parameters for fast mode if not set. They are @@ -116,10 +133,22 @@ static int i2c_dw_set_timings_master(struct dw_i2c_dev *dev) dev->master_cfg |= DW_IC_CON_SPEED_FAST; dev->hs_hcnt = 0; dev->hs_lcnt = 0; - } else if (dev->hs_hcnt && dev->hs_lcnt) { - dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n", - dev->hs_hcnt, dev->hs_lcnt); + } else if (!dev->hs_hcnt || !dev->hs_lcnt) { + ic_clk = i2c_dw_clk_rate(dev); + dev->hs_hcnt = + i2c_dw_scl_hcnt(ic_clk, + 160, /* tHIGH = 160 ns */ + sda_falling_time, + 0, /* DW default */ + 0); /* No offset */ + dev->hs_lcnt = + i2c_dw_scl_lcnt(ic_clk, + 320, /* tLOW = 320 ns */ + scl_falling_time, + 0); /* No offset */ } + dev_dbg(dev->dev, "High Speed Mode HCNT:LCNT = %d:%d\n", + dev->hs_hcnt, dev->hs_lcnt); } ret = i2c_dw_set_sda_hold(dev); @@ -162,22 +191,22 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev) __i2c_dw_disable(dev); /* Write standard speed timing parameters */ - dw_writel(dev, dev->ss_hcnt, DW_IC_SS_SCL_HCNT); - dw_writel(dev, dev->ss_lcnt, DW_IC_SS_SCL_LCNT); + regmap_write(dev->map, DW_IC_SS_SCL_HCNT, dev->ss_hcnt); + regmap_write(dev->map, DW_IC_SS_SCL_LCNT, dev->ss_lcnt); /* Write fast mode/fast mode plus timing parameters */ - dw_writel(dev, dev->fs_hcnt, DW_IC_FS_SCL_HCNT); - dw_writel(dev, dev->fs_lcnt, DW_IC_FS_SCL_LCNT); + regmap_write(dev->map, DW_IC_FS_SCL_HCNT, dev->fs_hcnt); + regmap_write(dev->map, DW_IC_FS_SCL_LCNT, dev->fs_lcnt); /* Write high speed timing parameters if supported */ if (dev->hs_hcnt && dev->hs_lcnt) { - dw_writel(dev, dev->hs_hcnt, DW_IC_HS_SCL_HCNT); - dw_writel(dev, dev->hs_lcnt, DW_IC_HS_SCL_LCNT); + regmap_write(dev->map, DW_IC_HS_SCL_HCNT, dev->hs_hcnt); + regmap_write(dev->map, DW_IC_HS_SCL_LCNT, dev->hs_lcnt); } /* Write SDA hold time if supported */ if (dev->sda_hold_time) - dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); + regmap_write(dev->map, DW_IC_SDA_HOLD, dev->sda_hold_time); i2c_dw_configure_fifo_master(dev); i2c_dw_release_lock(dev); @@ -188,15 +217,15 @@ static int i2c_dw_init_master(struct dw_i2c_dev *dev) static void i2c_dw_xfer_init(struct dw_i2c_dev *dev) { struct i2c_msg *msgs = dev->msgs; - u32 ic_con, ic_tar = 0; + u32 ic_con = 0, ic_tar = 0; + u32 dummy; /* Disable the adapter */ __i2c_dw_disable(dev); /* If the slave address is ten bit address, enable 10BITADDR */ - ic_con = dw_readl(dev, DW_IC_CON); if (msgs[dev->msg_write_idx].flags & I2C_M_TEN) { - ic_con |= DW_IC_CON_10BITADDR_MASTER; + ic_con = DW_IC_CON_10BITADDR_MASTER; /* * If I2C_DYNAMIC_TAR_UPDATE is set, the 10-bit addressing * mode has to be enabled via bit 12 of IC_TAR register. @@ -204,17 +233,17 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev) * detected from registers. */ ic_tar = DW_IC_TAR_10BITADDR_MASTER; - } else { - ic_con &= ~DW_IC_CON_10BITADDR_MASTER; } - dw_writel(dev, ic_con, DW_IC_CON); + regmap_update_bits(dev->map, DW_IC_CON, DW_IC_CON_10BITADDR_MASTER, + ic_con); /* * Set the slave (target) address and enable 10-bit addressing mode * if applicable. */ - dw_writel(dev, msgs[dev->msg_write_idx].addr | ic_tar, DW_IC_TAR); + regmap_write(dev->map, DW_IC_TAR, + msgs[dev->msg_write_idx].addr | ic_tar); /* Enforce disabled interrupts (due to HW issues) */ i2c_dw_disable_int(dev); @@ -223,11 +252,11 @@ static void i2c_dw_xfer_init(struct dw_i2c_dev *dev) __i2c_dw_enable(dev); /* Dummy read to avoid the register getting stuck on Bay Trail */ - dw_readl(dev, DW_IC_ENABLE_STATUS); + regmap_read(dev->map, DW_IC_ENABLE_STATUS, &dummy); /* Clear and enable interrupts */ - dw_readl(dev, DW_IC_CLR_INTR); - dw_writel(dev, DW_IC_INTR_MASTER_MASK, DW_IC_INTR_MASK); + regmap_read(dev->map, DW_IC_CLR_INTR, &dummy); + regmap_write(dev->map, DW_IC_INTR_MASK, DW_IC_INTR_MASTER_MASK); } /* @@ -246,6 +275,7 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev) u32 buf_len = dev->tx_buf_len; u8 *buf = dev->tx_buf; bool need_restart = false; + unsigned int flr; intr_mask = DW_IC_INTR_MASTER_MASK; @@ -278,8 +308,11 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev) need_restart = true; } - tx_limit = dev->tx_fifo_depth - dw_readl(dev, DW_IC_TXFLR); - rx_limit = dev->rx_fifo_depth - dw_readl(dev, DW_IC_RXFLR); + regmap_read(dev->map, DW_IC_TXFLR, &flr); + tx_limit = dev->tx_fifo_depth - flr; + + regmap_read(dev->map, DW_IC_RXFLR, &flr); + rx_limit = dev->rx_fifo_depth - flr; while (buf_len > 0 && tx_limit > 0 && rx_limit > 0) { u32 cmd = 0; @@ -312,11 +345,14 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev) if (dev->rx_outstanding >= dev->rx_fifo_depth) break; - dw_writel(dev, cmd | 0x100, DW_IC_DATA_CMD); + regmap_write(dev->map, DW_IC_DATA_CMD, + cmd | 0x100); rx_limit--; dev->rx_outstanding++; - } else - dw_writel(dev, cmd | *buf++, DW_IC_DATA_CMD); + } else { + regmap_write(dev->map, DW_IC_DATA_CMD, + cmd | *buf++); + } tx_limit--; buf_len--; } @@ -346,7 +382,7 @@ i2c_dw_xfer_msg(struct dw_i2c_dev *dev) if (dev->msg_err) intr_mask = 0; - dw_writel(dev, intr_mask, DW_IC_INTR_MASK); + regmap_write(dev->map, DW_IC_INTR_MASK, intr_mask); } static u8 @@ -371,10 +407,10 @@ static void i2c_dw_read(struct dw_i2c_dev *dev) { struct i2c_msg *msgs = dev->msgs; - int rx_valid; + unsigned int rx_valid; for (; dev->msg_read_idx < dev->msgs_num; dev->msg_read_idx++) { - u32 len; + u32 len, tmp; u8 *buf; if (!(msgs[dev->msg_read_idx].flags & I2C_M_RD)) @@ -388,18 +424,18 @@ i2c_dw_read(struct dw_i2c_dev *dev) buf = dev->rx_buf; } - rx_valid = dw_readl(dev, DW_IC_RXFLR); + regmap_read(dev->map, DW_IC_RXFLR, &rx_valid); for (; len > 0 && rx_valid > 0; len--, rx_valid--) { u32 flags = msgs[dev->msg_read_idx].flags; - *buf = dw_readl(dev, DW_IC_DATA_CMD); + regmap_read(dev->map, DW_IC_DATA_CMD, &tmp); /* Ensure length byte is a valid value */ if (flags & I2C_M_RECV_LEN && - *buf <= I2C_SMBUS_BLOCK_MAX && *buf > 0) { - len = i2c_dw_recv_len(dev, *buf); + tmp <= I2C_SMBUS_BLOCK_MAX && tmp > 0) { + len = i2c_dw_recv_len(dev, tmp); } - buf++; + *buf++ = tmp; dev->rx_outstanding--; } @@ -517,7 +553,7 @@ static const struct i2c_adapter_quirks i2c_dw_quirks = { static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev) { - u32 stat; + u32 stat, dummy; /* * The IC_INTR_STAT register just indicates "enabled" interrupts. @@ -525,47 +561,47 @@ static u32 i2c_dw_read_clear_intrbits(struct dw_i2c_dev *dev) * in the IC_RAW_INTR_STAT register. * * That is, - * stat = dw_readl(IC_INTR_STAT); + * stat = readl(IC_INTR_STAT); * equals to, - * stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK); + * stat = readl(IC_RAW_INTR_STAT) & readl(IC_INTR_MASK); * * The raw version might be useful for debugging purposes. */ - stat = dw_readl(dev, DW_IC_INTR_STAT); + regmap_read(dev->map, DW_IC_INTR_STAT, &stat); /* * Do not use the IC_CLR_INTR register to clear interrupts, or * you'll miss some interrupts, triggered during the period from - * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR). + * readl(IC_INTR_STAT) to readl(IC_CLR_INTR). * * Instead, use the separately-prepared IC_CLR_* registers. */ if (stat & DW_IC_INTR_RX_UNDER) - dw_readl(dev, DW_IC_CLR_RX_UNDER); + regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &dummy); if (stat & DW_IC_INTR_RX_OVER) - dw_readl(dev, DW_IC_CLR_RX_OVER); + regmap_read(dev->map, DW_IC_CLR_RX_OVER, &dummy); if (stat & DW_IC_INTR_TX_OVER) - dw_readl(dev, DW_IC_CLR_TX_OVER); + regmap_read(dev->map, DW_IC_CLR_TX_OVER, &dummy); if (stat & DW_IC_INTR_RD_REQ) - dw_readl(dev, DW_IC_CLR_RD_REQ); + regmap_read(dev->map, DW_IC_CLR_RD_REQ, &dummy); if (stat & DW_IC_INTR_TX_ABRT) { /* * The IC_TX_ABRT_SOURCE register is cleared whenever * the IC_CLR_TX_ABRT is read. Preserve it beforehand. */ - dev->abort_source = dw_readl(dev, DW_IC_TX_ABRT_SOURCE); - dw_readl(dev, DW_IC_CLR_TX_ABRT); + regmap_read(dev->map, DW_IC_TX_ABRT_SOURCE, &dev->abort_source); + regmap_read(dev->map, DW_IC_CLR_TX_ABRT, &dummy); } if (stat & DW_IC_INTR_RX_DONE) - dw_readl(dev, DW_IC_CLR_RX_DONE); + regmap_read(dev->map, DW_IC_CLR_RX_DONE, &dummy); if (stat & DW_IC_INTR_ACTIVITY) - dw_readl(dev, DW_IC_CLR_ACTIVITY); + regmap_read(dev->map, DW_IC_CLR_ACTIVITY, &dummy); if (stat & DW_IC_INTR_STOP_DET) - dw_readl(dev, DW_IC_CLR_STOP_DET); + regmap_read(dev->map, DW_IC_CLR_STOP_DET, &dummy); if (stat & DW_IC_INTR_START_DET) - dw_readl(dev, DW_IC_CLR_START_DET); + regmap_read(dev->map, DW_IC_CLR_START_DET, &dummy); if (stat & DW_IC_INTR_GEN_CALL) - dw_readl(dev, DW_IC_CLR_GEN_CALL); + regmap_read(dev->map, DW_IC_CLR_GEN_CALL, &dummy); return stat; } @@ -587,7 +623,7 @@ static int i2c_dw_irq_handler_master(struct dw_i2c_dev *dev) * Anytime TX_ABRT is set, the contents of the tx/rx * buffers are flushed. Make sure to skip them. */ - dw_writel(dev, 0, DW_IC_INTR_MASK); + regmap_write(dev->map, DW_IC_INTR_MASK, 0); goto tx_aborted; } @@ -608,9 +644,9 @@ tx_aborted: complete(&dev->cmd_complete); else if (unlikely(dev->flags & ACCESS_INTR_MASK)) { /* Workaround to trigger pending interrupt */ - stat = dw_readl(dev, DW_IC_INTR_MASK); + regmap_read(dev->map, DW_IC_INTR_MASK, &stat); i2c_dw_disable_int(dev); - dw_writel(dev, stat, DW_IC_INTR_MASK); + regmap_write(dev->map, DW_IC_INTR_MASK, stat); } return 0; @@ -621,8 +657,8 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id) struct dw_i2c_dev *dev = dev_id; u32 stat, enabled; - enabled = dw_readl(dev, DW_IC_ENABLE); - stat = dw_readl(dev, DW_IC_RAW_INTR_STAT); + regmap_read(dev->map, DW_IC_ENABLE, &enabled); + regmap_read(dev->map, DW_IC_RAW_INTR_STAT, &stat); dev_dbg(dev->dev, "enabled=%#x stat=%#x\n", enabled, stat); if (!enabled || !(stat & ~DW_IC_INTR_ACTIVITY)) return IRQ_NONE; @@ -632,6 +668,30 @@ static irqreturn_t i2c_dw_isr(int this_irq, void *dev_id) return IRQ_HANDLED; } +void i2c_dw_configure_master(struct dw_i2c_dev *dev) +{ + struct i2c_timings *t = &dev->timings; + + dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY; + + dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE | + DW_IC_CON_RESTART_EN; + + dev->mode = DW_IC_MASTER; + + switch (t->bus_freq_hz) { + case I2C_MAX_STANDARD_MODE_FREQ: + dev->master_cfg |= DW_IC_CON_SPEED_STD; + break; + case I2C_MAX_HIGH_SPEED_MODE_FREQ: + dev->master_cfg |= DW_IC_CON_SPEED_HIGH; + break; + default: + dev->master_cfg |= DW_IC_CON_SPEED_FAST; + } +} +EXPORT_SYMBOL_GPL(i2c_dw_configure_master); + static void i2c_dw_prepare_recovery(struct i2c_adapter *adap) { struct dw_i2c_dev *dev = i2c_get_adapdata(adap); @@ -678,7 +738,7 @@ static int i2c_dw_init_recovery_info(struct dw_i2c_dev *dev) return 0; } -int i2c_dw_probe(struct dw_i2c_dev *dev) +int i2c_dw_probe_master(struct dw_i2c_dev *dev) { struct i2c_adapter *adap = &dev->adapter; unsigned long irq_flags; @@ -690,7 +750,7 @@ int i2c_dw_probe(struct dw_i2c_dev *dev) dev->disable = i2c_dw_disable; dev->disable_int = i2c_dw_disable_int; - ret = i2c_dw_set_reg_access(dev); + ret = i2c_dw_init_regmap(dev); if (ret) return ret; @@ -698,7 +758,9 @@ int i2c_dw_probe(struct dw_i2c_dev *dev) if (ret) return ret; - i2c_dw_set_fifo_size(dev); + ret = i2c_dw_set_fifo_size(dev); + if (ret) + return ret; ret = dev->init(dev); if (ret) @@ -745,7 +807,7 @@ int i2c_dw_probe(struct dw_i2c_dev *dev) return ret; } -EXPORT_SYMBOL_GPL(i2c_dw_probe); +EXPORT_SYMBOL_GPL(i2c_dw_probe_master); MODULE_DESCRIPTION("Synopsys DesignWare I2C bus master adapter"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-designware-pcidrv.c b/drivers/i2c/busses/i2c-designware-pcidrv.c index 7a0b65b5b5b5..947c096f86e3 100644 --- a/drivers/i2c/busses/i2c-designware-pcidrv.c +++ b/drivers/i2c/busses/i2c-designware-pcidrv.c @@ -46,20 +46,12 @@ struct dw_scl_sda_cfg { struct dw_pci_controller { u32 bus_num; - u32 bus_cfg; - u32 tx_fifo_depth; - u32 rx_fifo_depth; - u32 clk_khz; - u32 functionality; u32 flags; struct dw_scl_sda_cfg *scl_sda_cfg; int (*setup)(struct pci_dev *pdev, struct dw_pci_controller *c); + u32 (*get_clk_rate_khz)(struct dw_i2c_dev *dev); }; -#define INTEL_MID_STD_CFG (DW_IC_CON_MASTER | \ - DW_IC_CON_SLAVE_DISABLE | \ - DW_IC_CON_RESTART_EN) - /* Merrifield HCNT/LCNT/SDA hold time */ static struct dw_scl_sda_cfg mrfld_config = { .ss_hcnt = 0x2f8, @@ -86,12 +78,18 @@ static struct dw_scl_sda_cfg hsw_config = { .sda_hold = 0x9, }; +static u32 mfld_get_clk_rate_khz(struct dw_i2c_dev *dev) +{ + return 25000; +} + static int mfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) { + struct dw_i2c_dev *dev = dev_get_drvdata(&pdev->dev); + switch (pdev->device) { case 0x0817: - c->bus_cfg &= ~DW_IC_CON_SPEED_MASK; - c->bus_cfg |= DW_IC_CON_SPEED_STD; + dev->timings.bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; /* fall through */ case 0x0818: case 0x0819: @@ -125,57 +123,37 @@ static int mrfld_setup(struct pci_dev *pdev, struct dw_pci_controller *c) return -ENODEV; } +static u32 ehl_get_clk_rate_khz(struct dw_i2c_dev *dev) +{ + return 100000; +} + static struct dw_pci_controller dw_pci_controllers[] = { [medfield] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 32, - .rx_fifo_depth = 32, - .functionality = I2C_FUNC_10BIT_ADDR, - .clk_khz = 25000, .setup = mfld_setup, + .get_clk_rate_khz = mfld_get_clk_rate_khz, }, [merrifield] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 64, - .rx_fifo_depth = 64, - .functionality = I2C_FUNC_10BIT_ADDR, .scl_sda_cfg = &mrfld_config, .setup = mrfld_setup, }, [baytrail] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 32, - .rx_fifo_depth = 32, - .functionality = I2C_FUNC_10BIT_ADDR, .scl_sda_cfg = &byt_config, }, [haswell] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 32, - .rx_fifo_depth = 32, - .functionality = I2C_FUNC_10BIT_ADDR, .scl_sda_cfg = &hsw_config, }, [cherrytrail] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 32, - .rx_fifo_depth = 32, - .functionality = I2C_FUNC_10BIT_ADDR, - .flags = MODEL_CHERRYTRAIL, .scl_sda_cfg = &byt_config, }, [elkhartlake] = { .bus_num = -1, - .bus_cfg = INTEL_MID_STD_CFG | DW_IC_CON_SPEED_FAST, - .tx_fifo_depth = 32, - .rx_fifo_depth = 32, - .functionality = I2C_FUNC_10BIT_ADDR, - .clk_khz = 100000, + .get_clk_rate_khz = ehl_get_clk_rate_khz, }, }; @@ -205,11 +183,6 @@ static int i2c_dw_pci_resume(struct device *dev) static UNIVERSAL_DEV_PM_OPS(i2c_dw_pm_ops, i2c_dw_pci_suspend, i2c_dw_pci_resume, NULL); -static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev) -{ - return dev->controller->clk_khz; -} - static int i2c_dw_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) { @@ -250,14 +223,15 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, if (r < 0) return r; - dev->clk = NULL; - dev->controller = controller; - dev->get_clk_rate_khz = i2c_dw_get_clk_rate_khz; + dev->get_clk_rate_khz = controller->get_clk_rate_khz; + dev->timings.bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; dev->base = pcim_iomap_table(pdev)[0]; dev->dev = &pdev->dev; dev->irq = pci_irq_vector(pdev, 0); dev->flags |= controller->flags; + pci_set_drvdata(pdev, dev); + if (controller->setup) { r = controller->setup(pdev, controller); if (r) { @@ -266,10 +240,19 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, } } - dev->functionality = controller->functionality | - DW_IC_DEFAULT_FUNCTIONALITY; + i2c_dw_acpi_adjust_bus_speed(&pdev->dev); + + if (has_acpi_companion(&pdev->dev)) + i2c_dw_acpi_configure(&pdev->dev); + + r = i2c_dw_validate_speed(dev); + if (r) { + pci_free_irq_vectors(pdev); + return r; + } + + i2c_dw_configure(dev); - dev->master_cfg = controller->bus_cfg; if (controller->scl_sda_cfg) { cfg = controller->scl_sda_cfg; dev->ss_hcnt = cfg->ss_hcnt; @@ -279,11 +262,6 @@ static int i2c_dw_pci_probe(struct pci_dev *pdev, dev->sda_hold_time = cfg->sda_hold; } - pci_set_drvdata(pdev, dev); - - dev->tx_fifo_depth = controller->tx_fifo_depth; - dev->rx_fifo_depth = controller->rx_fifo_depth; - adap = &dev->adapter; adap->owner = THIS_MODULE; adap->class = 0; diff --git a/drivers/i2c/busses/i2c-designware-platdrv.c b/drivers/i2c/busses/i2c-designware-platdrv.c index c429d664f655..0de4e302fc6a 100644 --- a/drivers/i2c/busses/i2c-designware-platdrv.c +++ b/drivers/i2c/busses/i2c-designware-platdrv.c @@ -12,13 +12,13 @@ #include <linux/clk-provider.h> #include <linux/clk.h> #include <linux/delay.h> -#include <linux/dmi.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/i2c.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/mfd/syscon.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_data/i2c-designware.h> @@ -26,6 +26,7 @@ #include <linux/pm.h> #include <linux/pm_runtime.h> #include <linux/property.h> +#include <linux/regmap.h> #include <linux/reset.h> #include <linux/sched.h> #include <linux/slab.h> @@ -39,91 +40,13 @@ static u32 i2c_dw_get_clk_rate_khz(struct dw_i2c_dev *dev) } #ifdef CONFIG_ACPI -/* - * The HCNT/LCNT information coming from ACPI should be the most accurate - * for given platform. However, some systems get it wrong. On such systems - * we get better results by calculating those based on the input clock. - */ -static const struct dmi_system_id dw_i2c_no_acpi_params[] = { - { - .ident = "Dell Inspiron 7348", - .matches = { - DMI_MATCH(DMI_SYS_VENDOR, "Dell Inc."), - DMI_MATCH(DMI_PRODUCT_NAME, "Inspiron 7348"), - }, - }, - { } -}; - -static void dw_i2c_acpi_params(struct platform_device *pdev, char method[], - u16 *hcnt, u16 *lcnt, u32 *sda_hold) -{ - struct acpi_buffer buf = { ACPI_ALLOCATE_BUFFER }; - acpi_handle handle = ACPI_HANDLE(&pdev->dev); - union acpi_object *obj; - - if (dmi_check_system(dw_i2c_no_acpi_params)) - return; - - if (ACPI_FAILURE(acpi_evaluate_object(handle, method, NULL, &buf))) - return; - - obj = (union acpi_object *)buf.pointer; - if (obj->type == ACPI_TYPE_PACKAGE && obj->package.count == 3) { - const union acpi_object *objs = obj->package.elements; - - *hcnt = (u16)objs[0].integer.value; - *lcnt = (u16)objs[1].integer.value; - *sda_hold = (u32)objs[2].integer.value; - } - - kfree(buf.pointer); -} - -static int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - struct i2c_timings *t = &dev->timings; - u32 ss_ht = 0, fp_ht = 0, hs_ht = 0, fs_ht = 0; - - dev->tx_fifo_depth = 32; - dev->rx_fifo_depth = 32; - - /* - * Try to get SDA hold time and *CNT values from an ACPI method for - * selected speed modes. - */ - dw_i2c_acpi_params(pdev, "SSCN", &dev->ss_hcnt, &dev->ss_lcnt, &ss_ht); - dw_i2c_acpi_params(pdev, "FPCN", &dev->fp_hcnt, &dev->fp_lcnt, &fp_ht); - dw_i2c_acpi_params(pdev, "HSCN", &dev->hs_hcnt, &dev->hs_lcnt, &hs_ht); - dw_i2c_acpi_params(pdev, "FMCN", &dev->fs_hcnt, &dev->fs_lcnt, &fs_ht); - - switch (t->bus_freq_hz) { - case I2C_MAX_STANDARD_MODE_FREQ: - dev->sda_hold_time = ss_ht; - break; - case I2C_MAX_FAST_MODE_PLUS_FREQ: - dev->sda_hold_time = fp_ht; - break; - case I2C_MAX_HIGH_SPEED_MODE_FREQ: - dev->sda_hold_time = hs_ht; - break; - case I2C_MAX_FAST_MODE_FREQ: - default: - dev->sda_hold_time = fs_ht; - break; - } - - return 0; -} - static const struct acpi_device_id dw_i2c_acpi_match[] = { { "INT33C2", 0 }, { "INT33C3", 0 }, { "INT3432", 0 }, { "INT3433", 0 }, { "80860F41", ACCESS_NO_IRQ_SUSPEND }, - { "808622C1", ACCESS_NO_IRQ_SUSPEND | MODEL_CHERRYTRAIL }, + { "808622C1", ACCESS_NO_IRQ_SUSPEND }, { "AMD0010", ACCESS_INTR_MASK }, { "AMDI0010", ACCESS_INTR_MASK }, { "AMDI0510", 0 }, @@ -134,14 +57,66 @@ static const struct acpi_device_id dw_i2c_acpi_match[] = { { } }; MODULE_DEVICE_TABLE(acpi, dw_i2c_acpi_match); -#else -static inline int dw_i2c_acpi_configure(struct platform_device *pdev) -{ - return -ENODEV; -} #endif #ifdef CONFIG_OF +#define BT1_I2C_CTL 0x100 +#define BT1_I2C_CTL_ADDR_MASK GENMASK(7, 0) +#define BT1_I2C_CTL_WR BIT(8) +#define BT1_I2C_CTL_GO BIT(31) +#define BT1_I2C_DI 0x104 +#define BT1_I2C_DO 0x108 + +static int bt1_i2c_read(void *context, unsigned int reg, unsigned int *val) +{ + struct dw_i2c_dev *dev = context; + int ret; + + /* + * Note these methods shouldn't ever fail because the system controller + * registers are memory mapped. We check the return value just in case. + */ + ret = regmap_write(dev->sysmap, BT1_I2C_CTL, + BT1_I2C_CTL_GO | (reg & BT1_I2C_CTL_ADDR_MASK)); + if (ret) + return ret; + + return regmap_read(dev->sysmap, BT1_I2C_DO, val); +} + +static int bt1_i2c_write(void *context, unsigned int reg, unsigned int val) +{ + struct dw_i2c_dev *dev = context; + int ret; + + ret = regmap_write(dev->sysmap, BT1_I2C_DI, val); + if (ret) + return ret; + + return regmap_write(dev->sysmap, BT1_I2C_CTL, + BT1_I2C_CTL_GO | BT1_I2C_CTL_WR | (reg & BT1_I2C_CTL_ADDR_MASK)); +} + +static struct regmap_config bt1_i2c_cfg = { + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, + .fast_io = true, + .reg_read = bt1_i2c_read, + .reg_write = bt1_i2c_write, + .max_register = DW_IC_COMP_TYPE, +}; + +static int bt1_i2c_request_regs(struct dw_i2c_dev *dev) +{ + dev->sysmap = syscon_node_to_regmap(dev->dev->of_node->parent); + if (IS_ERR(dev->sysmap)) + return PTR_ERR(dev->sysmap); + + dev->map = devm_regmap_init(dev->dev, NULL, dev, &bt1_i2c_cfg); + return PTR_ERR_OR_ZERO(dev->map); +} + #define MSCC_ICPU_CFG_TWI_DELAY 0x0 #define MSCC_ICPU_CFG_TWI_DELAY_ENABLE BIT(0) #define MSCC_ICPU_CFG_TWI_SPIKE_FILTER 0x4 @@ -157,12 +132,10 @@ static int mscc_twi_set_sda_hold_time(struct dw_i2c_dev *dev) static int dw_i2c_of_configure(struct platform_device *pdev) { struct dw_i2c_dev *dev = platform_get_drvdata(pdev); - struct resource *mem; switch (dev->flags & MODEL_MASK) { case MODEL_MSCC_OCELOT: - mem = platform_get_resource(pdev, IORESOURCE_MEM, 1); - dev->ext = devm_ioremap_resource(&pdev->dev, mem); + dev->ext = devm_platform_ioremap_resource(pdev, 1); if (!IS_ERR(dev->ext)) dev->set_sda_hold_time = mscc_twi_set_sda_hold_time; break; @@ -176,48 +149,21 @@ static int dw_i2c_of_configure(struct platform_device *pdev) static const struct of_device_id dw_i2c_of_match[] = { { .compatible = "snps,designware-i2c", }, { .compatible = "mscc,ocelot-i2c", .data = (void *)MODEL_MSCC_OCELOT }, + { .compatible = "baikal,bt1-sys-i2c", .data = (void *)MODEL_BAIKAL_BT1 }, {}, }; MODULE_DEVICE_TABLE(of, dw_i2c_of_match); #else -static inline int dw_i2c_of_configure(struct platform_device *pdev) +static int bt1_i2c_request_regs(struct dw_i2c_dev *dev) { return -ENODEV; } -#endif - -static void i2c_dw_configure_master(struct dw_i2c_dev *dev) -{ - struct i2c_timings *t = &dev->timings; - - dev->functionality = I2C_FUNC_10BIT_ADDR | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->master_cfg = DW_IC_CON_MASTER | DW_IC_CON_SLAVE_DISABLE | - DW_IC_CON_RESTART_EN; - - dev->mode = DW_IC_MASTER; - - switch (t->bus_freq_hz) { - case I2C_MAX_STANDARD_MODE_FREQ: - dev->master_cfg |= DW_IC_CON_SPEED_STD; - break; - case I2C_MAX_HIGH_SPEED_MODE_FREQ: - dev->master_cfg |= DW_IC_CON_SPEED_HIGH; - break; - default: - dev->master_cfg |= DW_IC_CON_SPEED_FAST; - } -} -static void i2c_dw_configure_slave(struct dw_i2c_dev *dev) +static inline int dw_i2c_of_configure(struct platform_device *pdev) { - dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY; - - dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL | - DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED; - - dev->mode = DW_IC_SLAVE; + return -ENODEV; } +#endif static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) { @@ -227,12 +173,23 @@ static void dw_i2c_plat_pm_cleanup(struct dw_i2c_dev *dev) pm_runtime_put_noidle(dev->dev); } -static const u32 supported_speeds[] = { - I2C_MAX_HIGH_SPEED_MODE_FREQ, - I2C_MAX_FAST_MODE_PLUS_FREQ, - I2C_MAX_FAST_MODE_FREQ, - I2C_MAX_STANDARD_MODE_FREQ, -}; +static int dw_i2c_plat_request_regs(struct dw_i2c_dev *dev) +{ + struct platform_device *pdev = to_platform_device(dev->dev); + int ret; + + switch (dev->flags & MODEL_MASK) { + case MODEL_BAIKAL_BT1: + ret = bt1_i2c_request_regs(dev); + break; + default: + dev->base = devm_platform_ioremap_resource(pdev, 0); + ret = PTR_ERR_OR_ZERO(dev->base); + break; + } + + return ret; +} static int dw_i2c_plat_probe(struct platform_device *pdev) { @@ -240,9 +197,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) struct i2c_adapter *adap; struct dw_i2c_dev *dev; struct i2c_timings *t; - u32 acpi_speed; - struct resource *mem; - int i, irq, ret; + int irq, ret; irq = platform_get_irq(pdev, 0); if (irq < 0) @@ -252,15 +207,15 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->base = devm_ioremap_resource(&pdev->dev, mem); - if (IS_ERR(dev->base)) - return PTR_ERR(dev->base); - + dev->flags = (uintptr_t)device_get_match_data(&pdev->dev); dev->dev = &pdev->dev; dev->irq = irq; platform_set_drvdata(pdev, dev); + ret = dw_i2c_plat_request_regs(dev); + if (ret) + return ret; + dev->rst = devm_reset_control_get_optional_exclusive(&pdev->dev, NULL); if (IS_ERR(dev->rst)) return PTR_ERR(dev->rst); @@ -273,60 +228,23 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) else i2c_parse_fw_timings(&pdev->dev, t, false); - acpi_speed = i2c_acpi_find_bus_speed(&pdev->dev); - /* - * Some DSTDs use a non standard speed, round down to the lowest - * standard speed. - */ - for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { - if (acpi_speed >= supported_speeds[i]) - break; - } - acpi_speed = i < ARRAY_SIZE(supported_speeds) ? supported_speeds[i] : 0; - - /* - * Find bus speed from the "clock-frequency" device property, ACPI - * or by using fast mode if neither is set. - */ - if (acpi_speed && t->bus_freq_hz) - t->bus_freq_hz = min(t->bus_freq_hz, acpi_speed); - else if (acpi_speed || t->bus_freq_hz) - t->bus_freq_hz = max(t->bus_freq_hz, acpi_speed); - else - t->bus_freq_hz = I2C_MAX_FAST_MODE_FREQ; - - dev->flags |= (uintptr_t)device_get_match_data(&pdev->dev); + i2c_dw_acpi_adjust_bus_speed(&pdev->dev); if (pdev->dev.of_node) dw_i2c_of_configure(pdev); if (has_acpi_companion(&pdev->dev)) - dw_i2c_acpi_configure(pdev); + i2c_dw_acpi_configure(&pdev->dev); - /* - * Only standard mode at 100kHz, fast mode at 400kHz, - * fast mode plus at 1MHz and high speed mode at 3.4MHz are supported. - */ - for (i = 0; i < ARRAY_SIZE(supported_speeds); i++) { - if (t->bus_freq_hz == supported_speeds[i]) - break; - } - if (i == ARRAY_SIZE(supported_speeds)) { - dev_err(&pdev->dev, - "%d Hz is unsupported, only 100kHz, 400kHz, 1MHz and 3.4MHz are supported\n", - t->bus_freq_hz); - ret = -EINVAL; + ret = i2c_dw_validate_speed(dev); + if (ret) goto exit_reset; - } ret = i2c_dw_probe_lock_support(dev); if (ret) goto exit_reset; - if (i2c_detect_slave_mode(&pdev->dev)) - i2c_dw_configure_slave(dev); - else - i2c_dw_configure_master(dev); + i2c_dw_configure(dev); /* Optional interface clock */ dev->pclk = devm_clk_get_optional(&pdev->dev, "pclk"); @@ -377,11 +295,7 @@ static int dw_i2c_plat_probe(struct platform_device *pdev) pm_runtime_enable(&pdev->dev); - if (dev->mode == DW_IC_SLAVE) - ret = i2c_dw_probe_slave(dev); - else - ret = i2c_dw_probe(dev); - + ret = i2c_dw_probe(dev); if (ret) goto exit_probe; diff --git a/drivers/i2c/busses/i2c-designware-slave.c b/drivers/i2c/busses/i2c-designware-slave.c index f5ecf76c0d02..44974b53a626 100644 --- a/drivers/i2c/busses/i2c-designware-slave.c +++ b/drivers/i2c/busses/i2c-designware-slave.c @@ -14,18 +14,19 @@ #include <linux/io.h> #include <linux/module.h> #include <linux/pm_runtime.h> +#include <linux/regmap.h> #include "i2c-designware-core.h" static void i2c_dw_configure_fifo_slave(struct dw_i2c_dev *dev) { /* Configure Tx/Rx FIFO threshold levels. */ - dw_writel(dev, 0, DW_IC_TX_TL); - dw_writel(dev, 0, DW_IC_RX_TL); + regmap_write(dev->map, DW_IC_TX_TL, 0); + regmap_write(dev->map, DW_IC_RX_TL, 0); /* Configure the I2C slave. */ - dw_writel(dev, dev->slave_cfg, DW_IC_CON); - dw_writel(dev, DW_IC_INTR_SLAVE_MASK, DW_IC_INTR_MASK); + regmap_write(dev->map, DW_IC_CON, dev->slave_cfg); + regmap_write(dev->map, DW_IC_INTR_MASK, DW_IC_INTR_SLAVE_MASK); } /** @@ -49,7 +50,7 @@ static int i2c_dw_init_slave(struct dw_i2c_dev *dev) /* Write SDA hold time if supported */ if (dev->sda_hold_time) - dw_writel(dev, dev->sda_hold_time, DW_IC_SDA_HOLD); + regmap_write(dev->map, DW_IC_SDA_HOLD, dev->sda_hold_time); i2c_dw_configure_fifo_slave(dev); i2c_dw_release_lock(dev); @@ -72,7 +73,7 @@ static int i2c_dw_reg_slave(struct i2c_client *slave) * the address to which the DW_apb_i2c responds. */ __i2c_dw_disable_nowait(dev); - dw_writel(dev, slave->addr, DW_IC_SAR); + regmap_write(dev->map, DW_IC_SAR, slave->addr); dev->slave = slave; __i2c_dw_enable(dev); @@ -103,7 +104,7 @@ static int i2c_dw_unreg_slave(struct i2c_client *slave) static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev) { - u32 stat; + u32 stat, dummy; /* * The IC_INTR_STAT register just indicates "enabled" interrupts. @@ -111,39 +112,39 @@ static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev) * in the IC_RAW_INTR_STAT register. * * That is, - * stat = dw_readl(IC_INTR_STAT); + * stat = readl(IC_INTR_STAT); * equals to, - * stat = dw_readl(IC_RAW_INTR_STAT) & dw_readl(IC_INTR_MASK); + * stat = readl(IC_RAW_INTR_STAT) & readl(IC_INTR_MASK); * * The raw version might be useful for debugging purposes. */ - stat = dw_readl(dev, DW_IC_INTR_STAT); + regmap_read(dev->map, DW_IC_INTR_STAT, &stat); /* * Do not use the IC_CLR_INTR register to clear interrupts, or * you'll miss some interrupts, triggered during the period from - * dw_readl(IC_INTR_STAT) to dw_readl(IC_CLR_INTR). + * readl(IC_INTR_STAT) to readl(IC_CLR_INTR). * * Instead, use the separately-prepared IC_CLR_* registers. */ if (stat & DW_IC_INTR_TX_ABRT) - dw_readl(dev, DW_IC_CLR_TX_ABRT); + regmap_read(dev->map, DW_IC_CLR_TX_ABRT, &dummy); if (stat & DW_IC_INTR_RX_UNDER) - dw_readl(dev, DW_IC_CLR_RX_UNDER); + regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &dummy); if (stat & DW_IC_INTR_RX_OVER) - dw_readl(dev, DW_IC_CLR_RX_OVER); + regmap_read(dev->map, DW_IC_CLR_RX_OVER, &dummy); if (stat & DW_IC_INTR_TX_OVER) - dw_readl(dev, DW_IC_CLR_TX_OVER); + regmap_read(dev->map, DW_IC_CLR_TX_OVER, &dummy); if (stat & DW_IC_INTR_RX_DONE) - dw_readl(dev, DW_IC_CLR_RX_DONE); + regmap_read(dev->map, DW_IC_CLR_RX_DONE, &dummy); if (stat & DW_IC_INTR_ACTIVITY) - dw_readl(dev, DW_IC_CLR_ACTIVITY); + regmap_read(dev->map, DW_IC_CLR_ACTIVITY, &dummy); if (stat & DW_IC_INTR_STOP_DET) - dw_readl(dev, DW_IC_CLR_STOP_DET); + regmap_read(dev->map, DW_IC_CLR_STOP_DET, &dummy); if (stat & DW_IC_INTR_START_DET) - dw_readl(dev, DW_IC_CLR_START_DET); + regmap_read(dev->map, DW_IC_CLR_START_DET, &dummy); if (stat & DW_IC_INTR_GEN_CALL) - dw_readl(dev, DW_IC_CLR_GEN_CALL); + regmap_read(dev->map, DW_IC_CLR_GEN_CALL, &dummy); return stat; } @@ -155,14 +156,14 @@ static u32 i2c_dw_read_clear_intrbits_slave(struct dw_i2c_dev *dev) static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev) { - u32 raw_stat, stat, enabled; - u8 val, slave_activity; + u32 raw_stat, stat, enabled, tmp; + u8 val = 0, slave_activity; - stat = dw_readl(dev, DW_IC_INTR_STAT); - enabled = dw_readl(dev, DW_IC_ENABLE); - raw_stat = dw_readl(dev, DW_IC_RAW_INTR_STAT); - slave_activity = ((dw_readl(dev, DW_IC_STATUS) & - DW_IC_STATUS_SLAVE_ACTIVITY) >> 6); + regmap_read(dev->map, DW_IC_INTR_STAT, &stat); + regmap_read(dev->map, DW_IC_ENABLE, &enabled); + regmap_read(dev->map, DW_IC_RAW_INTR_STAT, &raw_stat); + regmap_read(dev->map, DW_IC_STATUS, &tmp); + slave_activity = ((tmp & DW_IC_STATUS_SLAVE_ACTIVITY) >> 6); if (!enabled || !(raw_stat & ~DW_IC_INTR_ACTIVITY) || !dev->slave) return 0; @@ -177,7 +178,8 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev) if (stat & DW_IC_INTR_RD_REQ) { if (slave_activity) { if (stat & DW_IC_INTR_RX_FULL) { - val = dw_readl(dev, DW_IC_DATA_CMD); + regmap_read(dev->map, DW_IC_DATA_CMD, &tmp); + val = tmp; if (!i2c_slave_event(dev->slave, I2C_SLAVE_WRITE_RECEIVED, @@ -185,24 +187,24 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev) dev_vdbg(dev->dev, "Byte %X acked!", val); } - dw_readl(dev, DW_IC_CLR_RD_REQ); + regmap_read(dev->map, DW_IC_CLR_RD_REQ, &tmp); stat = i2c_dw_read_clear_intrbits_slave(dev); } else { - dw_readl(dev, DW_IC_CLR_RD_REQ); - dw_readl(dev, DW_IC_CLR_RX_UNDER); + regmap_read(dev->map, DW_IC_CLR_RD_REQ, &tmp); + regmap_read(dev->map, DW_IC_CLR_RX_UNDER, &tmp); stat = i2c_dw_read_clear_intrbits_slave(dev); } if (!i2c_slave_event(dev->slave, I2C_SLAVE_READ_REQUESTED, &val)) - dw_writel(dev, val, DW_IC_DATA_CMD); + regmap_write(dev->map, DW_IC_DATA_CMD, val); } } if (stat & DW_IC_INTR_RX_DONE) { if (!i2c_slave_event(dev->slave, I2C_SLAVE_READ_PROCESSED, &val)) - dw_readl(dev, DW_IC_CLR_RX_DONE); + regmap_read(dev->map, DW_IC_CLR_RX_DONE, &tmp); i2c_slave_event(dev->slave, I2C_SLAVE_STOP, &val); stat = i2c_dw_read_clear_intrbits_slave(dev); @@ -210,7 +212,8 @@ static int i2c_dw_irq_handler_slave(struct dw_i2c_dev *dev) } if (stat & DW_IC_INTR_RX_FULL) { - val = dw_readl(dev, DW_IC_DATA_CMD); + regmap_read(dev->map, DW_IC_DATA_CMD, &tmp); + val = tmp; if (!i2c_slave_event(dev->slave, I2C_SLAVE_WRITE_RECEIVED, &val)) dev_vdbg(dev->dev, "Byte %X acked!", val); @@ -241,6 +244,17 @@ static const struct i2c_algorithm i2c_dw_algo = { .unreg_slave = i2c_dw_unreg_slave, }; +void i2c_dw_configure_slave(struct dw_i2c_dev *dev) +{ + dev->functionality = I2C_FUNC_SLAVE | DW_IC_DEFAULT_FUNCTIONALITY; + + dev->slave_cfg = DW_IC_CON_RX_FIFO_FULL_HLD_CTRL | + DW_IC_CON_RESTART_EN | DW_IC_CON_STOP_DET_IFADDRESSED; + + dev->mode = DW_IC_SLAVE; +} +EXPORT_SYMBOL_GPL(i2c_dw_configure_slave); + int i2c_dw_probe_slave(struct dw_i2c_dev *dev) { struct i2c_adapter *adap = &dev->adapter; @@ -252,7 +266,7 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev) dev->disable = i2c_dw_disable; dev->disable_int = i2c_dw_disable_int; - ret = i2c_dw_set_reg_access(dev); + ret = i2c_dw_init_regmap(dev); if (ret) return ret; @@ -260,7 +274,9 @@ int i2c_dw_probe_slave(struct dw_i2c_dev *dev) if (ret) return ret; - i2c_dw_set_fifo_size(dev); + ret = i2c_dw_set_fifo_size(dev); + if (ret) + return ret; ret = dev->init(dev); if (ret) diff --git a/drivers/i2c/busses/i2c-digicolor.c b/drivers/i2c/busses/i2c-digicolor.c index 056a5c4f0833..332f00437479 100644 --- a/drivers/i2c/busses/i2c-digicolor.c +++ b/drivers/i2c/busses/i2c-digicolor.c @@ -290,7 +290,6 @@ static int dc_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct dc_i2c *i2c; - struct resource *r; int ret = 0, irq; i2c = devm_kzalloc(&pdev->dev, sizeof(struct dc_i2c), GFP_KERNEL); @@ -311,8 +310,7 @@ static int dc_i2c_probe(struct platform_device *pdev) if (IS_ERR(i2c->clk)) return PTR_ERR(i2c->clk); - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->regs = devm_ioremap_resource(&pdev->dev, r); + i2c->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->regs)) return PTR_ERR(i2c->regs); diff --git a/drivers/i2c/busses/i2c-efm32.c b/drivers/i2c/busses/i2c-efm32.c index 18cca8f56da8..838ce0947191 100644 --- a/drivers/i2c/busses/i2c-efm32.c +++ b/drivers/i2c/busses/i2c-efm32.c @@ -312,9 +312,6 @@ static int efm32_i2c_probe(struct platform_device *pdev) int ret; u32 clkdiv; - if (!np) - return -EINVAL; - ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL); if (!ddata) return -ENOMEM; @@ -352,7 +349,6 @@ static int efm32_i2c_probe(struct platform_device *pdev) ret = platform_get_irq(pdev, 0); if (ret <= 0) { - dev_err(&pdev->dev, "failed to get irq (%d)\n", ret); if (!ret) ret = -EINVAL; return ret; diff --git a/drivers/i2c/busses/i2c-emev2.c b/drivers/i2c/busses/i2c-emev2.c index 959d4912ec0d..1a319352e51b 100644 --- a/drivers/i2c/busses/i2c-emev2.c +++ b/drivers/i2c/busses/i2c-emev2.c @@ -361,15 +361,13 @@ static const struct i2c_algorithm em_i2c_algo = { static int em_i2c_probe(struct platform_device *pdev) { struct em_i2c_device *priv; - struct resource *r; int ret; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->base = devm_ioremap_resource(&pdev->dev, r); + priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); diff --git a/drivers/i2c/busses/i2c-exynos5.c b/drivers/i2c/busses/i2c-exynos5.c index 527030953ba1..6ce3ec03b595 100644 --- a/drivers/i2c/busses/i2c-exynos5.c +++ b/drivers/i2c/busses/i2c-exynos5.c @@ -736,7 +736,6 @@ static int exynos5_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct exynos5_i2c *i2c; - struct resource *mem; int ret; i2c = devm_kzalloc(&pdev->dev, sizeof(struct exynos5_i2c), GFP_KERNEL); @@ -762,8 +761,7 @@ static int exynos5_i2c_probe(struct platform_device *pdev) if (ret) return ret; - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->regs = devm_ioremap_resource(&pdev->dev, mem); + i2c->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->regs)) { ret = PTR_ERR(i2c->regs); goto err_clk; @@ -879,6 +877,6 @@ static struct platform_driver exynos5_i2c_driver = { module_platform_driver(exynos5_i2c_driver); MODULE_DESCRIPTION("Exynos5 HS-I2C Bus driver"); -MODULE_AUTHOR("Naveen Krishna Chatradhi, <ch.naveen@samsung.com>"); -MODULE_AUTHOR("Taekgyun Ko, <taeggyun.ko@samsung.com>"); +MODULE_AUTHOR("Naveen Krishna Chatradhi <ch.naveen@samsung.com>"); +MODULE_AUTHOR("Taekgyun Ko <taeggyun.ko@samsung.com>"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-hix5hd2.c b/drivers/i2c/busses/i2c-hix5hd2.c index 6610304b6dc6..ab15b1ec2ab3 100644 --- a/drivers/i2c/busses/i2c-hix5hd2.c +++ b/drivers/i2c/busses/i2c-hix5hd2.c @@ -388,7 +388,6 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct hix5hd2_i2c_priv *priv; - struct resource *mem; unsigned int freq; int irq, ret; @@ -409,8 +408,7 @@ static int hix5hd2_i2c_probe(struct platform_device *pdev) } } - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->regs = devm_ioremap_resource(&pdev->dev, mem); + priv->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->regs)) return PTR_ERR(priv->regs); diff --git a/drivers/i2c/busses/i2c-i801.c b/drivers/i2c/busses/i2c-i801.c index 4f333889489c..fea644921a76 100644 --- a/drivers/i2c/busses/i2c-i801.c +++ b/drivers/i2c/busses/i2c-i801.c @@ -1318,6 +1318,12 @@ static void i801_probe_optional_slaves(struct i801_priv *priv) if (is_dell_system_with_lis3lv02d()) register_dell_lis3lv02d_i2c_device(priv); + + /* Instantiate SPD EEPROMs unless the SMBus is multiplexed */ +#if IS_ENABLED(CONFIG_I2C_MUX_GPIO) + if (!priv->mux_drvdata) +#endif + i2c_register_spd(&priv->adapter); } #else static void __init input_apanel_init(void) {} diff --git a/drivers/i2c/busses/i2c-icy.c b/drivers/i2c/busses/i2c-icy.c index 271470f4d8a9..66c9923fc766 100644 --- a/drivers/i2c/busses/i2c-icy.c +++ b/drivers/i2c/busses/i2c-icy.c @@ -43,6 +43,7 @@ #include <linux/i2c.h> #include <linux/i2c-algo-pcf.h> +#include <asm/amigahw.h> #include <asm/amigaints.h> #include <linux/zorro.h> diff --git a/drivers/i2c/busses/i2c-img-scb.c b/drivers/i2c/busses/i2c-img-scb.c index 422097a31c95..98a89301ed2a 100644 --- a/drivers/i2c/busses/i2c-img-scb.c +++ b/drivers/i2c/busses/i2c-img-scb.c @@ -1330,7 +1330,6 @@ static int img_i2c_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; struct img_i2c *i2c; - struct resource *res; int irq, ret; u32 val; @@ -1338,16 +1337,13 @@ static int img_i2c_probe(struct platform_device *pdev) if (!i2c) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->base = devm_ioremap_resource(&pdev->dev, res); + i2c->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->base)) return PTR_ERR(i2c->base); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "can't get irq number\n"); + if (irq < 0) return irq; - } i2c->sys_clk = devm_clk_get(&pdev->dev, "sys"); if (IS_ERR(i2c->sys_clk)) { diff --git a/drivers/i2c/busses/i2c-imx-lpi2c.c b/drivers/i2c/busses/i2c-imx-lpi2c.c index 94743ba581fe..9db6ccded5e9 100644 --- a/drivers/i2c/busses/i2c-imx-lpi2c.c +++ b/drivers/i2c/busses/i2c-imx-lpi2c.c @@ -551,10 +551,8 @@ static int lpi2c_imx_probe(struct platform_device *pdev) return PTR_ERR(lpi2c_imx->base); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "can't get irq number\n"); + if (irq < 0) return irq; - } lpi2c_imx->adapter.owner = THIS_MODULE; lpi2c_imx->adapter.algo = &lpi2c_imx_algo; diff --git a/drivers/i2c/busses/i2c-jz4780.c b/drivers/i2c/busses/i2c-jz4780.c index b426fc956938..ba831df6661e 100644 --- a/drivers/i2c/busses/i2c-jz4780.c +++ b/drivers/i2c/busses/i2c-jz4780.c @@ -763,7 +763,6 @@ static int jz4780_i2c_probe(struct platform_device *pdev) int ret = 0; unsigned int clk_freq = 0; unsigned short tmp; - struct resource *r; struct jz4780_i2c *i2c; i2c = devm_kzalloc(&pdev->dev, sizeof(struct jz4780_i2c), GFP_KERNEL); @@ -787,8 +786,7 @@ static int jz4780_i2c_probe(struct platform_device *pdev) init_completion(&i2c->trans_waitq); spin_lock_init(&i2c->lock); - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->iomem = devm_ioremap_resource(&pdev->dev, r); + i2c->iomem = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->iomem)) return PTR_ERR(i2c->iomem); diff --git a/drivers/i2c/busses/i2c-lpc2k.c b/drivers/i2c/busses/i2c-lpc2k.c index 13b0c12e2dba..4e30c5267142 100644 --- a/drivers/i2c/busses/i2c-lpc2k.c +++ b/drivers/i2c/busses/i2c-lpc2k.c @@ -346,7 +346,6 @@ static const struct i2c_algorithm i2c_lpc2k_algorithm = { static int i2c_lpc2k_probe(struct platform_device *pdev) { struct lpc2k_i2c *i2c; - struct resource *res; u32 bus_clk_rate; u32 scl_high; u32 clkrate; @@ -356,16 +355,13 @@ static int i2c_lpc2k_probe(struct platform_device *pdev) if (!i2c) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->base = devm_ioremap_resource(&pdev->dev, res); + i2c->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->base)) return PTR_ERR(i2c->base); i2c->irq = platform_get_irq(pdev, 0); - if (i2c->irq < 0) { - dev_err(&pdev->dev, "can't get interrupt resource\n"); + if (i2c->irq < 0) return i2c->irq; - } init_waitqueue_head(&i2c->wait); diff --git a/drivers/i2c/busses/i2c-meson.c b/drivers/i2c/busses/i2c-meson.c index 06b3bed78421..c5dec572fc48 100644 --- a/drivers/i2c/busses/i2c-meson.c +++ b/drivers/i2c/busses/i2c-meson.c @@ -397,7 +397,6 @@ static int meson_i2c_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct meson_i2c *i2c; - struct resource *mem; struct i2c_timings timings; int irq, ret = 0; @@ -422,16 +421,13 @@ static int meson_i2c_probe(struct platform_device *pdev) return PTR_ERR(i2c->clk); } - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->regs = devm_ioremap_resource(&pdev->dev, mem); + i2c->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->regs)) return PTR_ERR(i2c->regs); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "can't find IRQ\n"); + if (irq < 0) return irq; - } ret = devm_request_irq(&pdev->dev, irq, meson_i2c_irq, 0, NULL, i2c); if (ret < 0) { diff --git a/drivers/i2c/busses/i2c-mt65xx.c b/drivers/i2c/busses/i2c-mt65xx.c index 0ca6c38a15eb..deef69e56906 100644 --- a/drivers/i2c/busses/i2c-mt65xx.c +++ b/drivers/i2c/busses/i2c-mt65xx.c @@ -40,12 +40,11 @@ #define I2C_SOFT_RST 0x0001 #define I2C_FIFO_ADDR_CLR 0x0001 #define I2C_DELAY_LEN 0x0002 -#define I2C_ST_START_CON 0x8001 -#define I2C_FS_START_CON 0x1800 #define I2C_TIME_CLR_VALUE 0x0000 #define I2C_TIME_DEFAULT_VALUE 0x0003 #define I2C_WRRD_TRANAC_VALUE 0x0002 #define I2C_RD_TRANAC_VALUE 0x0001 +#define I2C_SCL_MIS_COMP_VALUE 0x0000 #define I2C_DMA_CON_TX 0x0000 #define I2C_DMA_CON_RX 0x0001 @@ -55,10 +54,13 @@ #define I2C_DMA_HARD_RST 0x0002 #define I2C_DMA_4G_MODE 0x0001 -#define I2C_DEFAULT_CLK_DIV 5 #define MAX_SAMPLE_CNT_DIV 8 #define MAX_STEP_CNT_DIV 64 +#define MAX_CLOCK_DIV 256 #define MAX_HS_STEP_CNT_DIV 8 +#define I2C_STANDARD_MODE_BUFFER (1000 / 2) +#define I2C_FAST_MODE_BUFFER (300 / 2) +#define I2C_FAST_MODE_PLUS_BUFFER (20 / 2) #define I2C_CONTROL_RS (0x1 << 1) #define I2C_CONTROL_DMA_EN (0x1 << 2) @@ -123,6 +125,12 @@ enum I2C_REGS_OFFSET { OFFSET_TRANSFER_LEN_AUX, OFFSET_CLOCK_DIV, OFFSET_LTIMING, + OFFSET_SCL_HIGH_LOW_RATIO, + OFFSET_HS_SCL_HIGH_LOW_RATIO, + OFFSET_SCL_MIS_COMP_POINT, + OFFSET_STA_STO_AC_TIMING, + OFFSET_HS_STA_STO_AC_TIMING, + OFFSET_SDA_TIMING, }; static const u16 mt_i2c_regs_v1[] = { @@ -150,6 +158,12 @@ static const u16 mt_i2c_regs_v1[] = { [OFFSET_DEBUGCTRL] = 0x68, [OFFSET_TRANSFER_LEN_AUX] = 0x6c, [OFFSET_CLOCK_DIV] = 0x70, + [OFFSET_SCL_HIGH_LOW_RATIO] = 0x74, + [OFFSET_HS_SCL_HIGH_LOW_RATIO] = 0x78, + [OFFSET_SCL_MIS_COMP_POINT] = 0x7C, + [OFFSET_STA_STO_AC_TIMING] = 0x80, + [OFFSET_HS_STA_STO_AC_TIMING] = 0x84, + [OFFSET_SDA_TIMING] = 0x88, }; static const u16 mt_i2c_regs_v2[] = { @@ -168,9 +182,11 @@ static const u16 mt_i2c_regs_v2[] = { [OFFSET_HS] = 0x30, [OFFSET_IO_CONFIG] = 0x34, [OFFSET_FIFO_ADDR_CLR] = 0x38, + [OFFSET_SDA_TIMING] = 0x3c, [OFFSET_TRANSFER_LEN_AUX] = 0x44, [OFFSET_CLOCK_DIV] = 0x48, [OFFSET_SOFTRESET] = 0x50, + [OFFSET_SCL_MIS_COMP_POINT] = 0x90, [OFFSET_DEBUGSTAT] = 0xe0, [OFFSET_DEBUGCTRL] = 0xe8, [OFFSET_FIFO_STAT] = 0xf4, @@ -191,6 +207,19 @@ struct mtk_i2c_compatible { unsigned char ltiming_adjust: 1; }; +struct mtk_i2c_ac_timing { + u16 htiming; + u16 ltiming; + u16 hs; + u16 ext; + u16 inter_clk_div; + u16 scl_hl_ratio; + u16 hs_scl_hl_ratio; + u16 sta_stop; + u16 hs_sta_stop; + u16 sda_timing; +}; + struct mtk_i2c { struct i2c_adapter adap; /* i2c host adapter */ struct device *dev; @@ -215,9 +244,46 @@ struct mtk_i2c { u16 ltiming_reg; unsigned char auto_restart; bool ignore_restart_irq; + struct mtk_i2c_ac_timing ac_timing; const struct mtk_i2c_compatible *dev_comp; }; +/** + * struct i2c_spec_values: + * min_low_ns: min LOW period of the SCL clock + * min_su_sta_ns: min set-up time for a repeated START condition + * max_hd_dat_ns: max data hold time + * min_su_dat_ns: min data set-up time + */ +struct i2c_spec_values { + unsigned int min_low_ns; + unsigned int min_high_ns; + unsigned int min_su_sta_ns; + unsigned int max_hd_dat_ns; + unsigned int min_su_dat_ns; +}; + +static const struct i2c_spec_values standard_mode_spec = { + .min_low_ns = 4700 + I2C_STANDARD_MODE_BUFFER, + .min_su_sta_ns = 4700 + I2C_STANDARD_MODE_BUFFER, + .max_hd_dat_ns = 3450 - I2C_STANDARD_MODE_BUFFER, + .min_su_dat_ns = 250 + I2C_STANDARD_MODE_BUFFER, +}; + +static const struct i2c_spec_values fast_mode_spec = { + .min_low_ns = 1300 + I2C_FAST_MODE_BUFFER, + .min_su_sta_ns = 600 + I2C_FAST_MODE_BUFFER, + .max_hd_dat_ns = 900 - I2C_FAST_MODE_BUFFER, + .min_su_dat_ns = 100 + I2C_FAST_MODE_BUFFER, +}; + +static const struct i2c_spec_values fast_mode_plus_spec = { + .min_low_ns = 500 + I2C_FAST_MODE_PLUS_BUFFER, + .min_su_sta_ns = 260 + I2C_FAST_MODE_PLUS_BUFFER, + .max_hd_dat_ns = 400 - I2C_FAST_MODE_PLUS_BUFFER, + .min_su_dat_ns = 50 + I2C_FAST_MODE_PLUS_BUFFER, +}; + static const struct i2c_adapter_quirks mt6577_i2c_quirks = { .flags = I2C_AQ_COMB_WRITE_THEN_READ, .max_num_msgs = 1, @@ -397,14 +463,38 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c) if (i2c->dev_comp->dcm) mtk_i2c_writew(i2c, I2C_DCM_DISABLE, OFFSET_DCM_EN); - if (i2c->dev_comp->timing_adjust) - mtk_i2c_writew(i2c, I2C_DEFAULT_CLK_DIV - 1, OFFSET_CLOCK_DIV); - mtk_i2c_writew(i2c, i2c->timing_reg, OFFSET_TIMING); mtk_i2c_writew(i2c, i2c->high_speed_reg, OFFSET_HS); if (i2c->dev_comp->ltiming_adjust) mtk_i2c_writew(i2c, i2c->ltiming_reg, OFFSET_LTIMING); + if (i2c->dev_comp->timing_adjust) { + mtk_i2c_writew(i2c, i2c->ac_timing.ext, OFFSET_EXT_CONF); + mtk_i2c_writew(i2c, i2c->ac_timing.inter_clk_div, + OFFSET_CLOCK_DIV); + mtk_i2c_writew(i2c, I2C_SCL_MIS_COMP_VALUE, + OFFSET_SCL_MIS_COMP_POINT); + mtk_i2c_writew(i2c, i2c->ac_timing.sda_timing, + OFFSET_SDA_TIMING); + + if (i2c->dev_comp->ltiming_adjust) { + mtk_i2c_writew(i2c, i2c->ac_timing.htiming, + OFFSET_TIMING); + mtk_i2c_writew(i2c, i2c->ac_timing.hs, OFFSET_HS); + mtk_i2c_writew(i2c, i2c->ac_timing.ltiming, + OFFSET_LTIMING); + } else { + mtk_i2c_writew(i2c, i2c->ac_timing.scl_hl_ratio, + OFFSET_SCL_HIGH_LOW_RATIO); + mtk_i2c_writew(i2c, i2c->ac_timing.hs_scl_hl_ratio, + OFFSET_HS_SCL_HIGH_LOW_RATIO); + mtk_i2c_writew(i2c, i2c->ac_timing.sta_stop, + OFFSET_STA_STO_AC_TIMING); + mtk_i2c_writew(i2c, i2c->ac_timing.hs_sta_stop, + OFFSET_HS_STA_STO_AC_TIMING); + } + } + /* If use i2c pin from PMIC mt6397 side, need set PATH_DIR first */ if (i2c->have_pmic) mtk_i2c_writew(i2c, I2C_CONTROL_WRAPPER, OFFSET_PATH_DIR); @@ -422,6 +512,126 @@ static void mtk_i2c_init_hw(struct mtk_i2c *i2c) writel(I2C_DMA_CLR_FLAG, i2c->pdmabase + OFFSET_RST); } +static const struct i2c_spec_values *mtk_i2c_get_spec(unsigned int speed) +{ + if (speed <= I2C_MAX_STANDARD_MODE_FREQ) + return &standard_mode_spec; + else if (speed <= I2C_MAX_FAST_MODE_FREQ) + return &fast_mode_spec; + else + return &fast_mode_plus_spec; +} + +static int mtk_i2c_max_step_cnt(unsigned int target_speed) +{ + if (target_speed > I2C_MAX_FAST_MODE_FREQ) + return MAX_HS_STEP_CNT_DIV; + else + return MAX_STEP_CNT_DIV; +} + +/* + * Check and Calculate i2c ac-timing + * + * Hardware design: + * sample_ns = (1000000000 * (sample_cnt + 1)) / clk_src + * xxx_cnt_div = spec->min_xxx_ns / sample_ns + * + * Sample_ns is rounded down for xxx_cnt_div would be greater + * than the smallest spec. + * The sda_timing is chosen as the middle value between + * the largest and smallest. + */ +static int mtk_i2c_check_ac_timing(struct mtk_i2c *i2c, + unsigned int clk_src, + unsigned int check_speed, + unsigned int step_cnt, + unsigned int sample_cnt) +{ + const struct i2c_spec_values *spec; + unsigned int su_sta_cnt, low_cnt, high_cnt, max_step_cnt; + unsigned int sda_max, sda_min, clk_ns, max_sta_cnt = 0x3f; + unsigned int sample_ns = div_u64(1000000000ULL * (sample_cnt + 1), + clk_src); + + if (!i2c->dev_comp->timing_adjust) + return 0; + + if (i2c->dev_comp->ltiming_adjust) + max_sta_cnt = 0x100; + + spec = mtk_i2c_get_spec(check_speed); + + if (i2c->dev_comp->ltiming_adjust) + clk_ns = 1000000000 / clk_src; + else + clk_ns = sample_ns / 2; + + su_sta_cnt = DIV_ROUND_UP(spec->min_su_sta_ns, clk_ns); + if (su_sta_cnt > max_sta_cnt) + return -1; + + low_cnt = DIV_ROUND_UP(spec->min_low_ns, sample_ns); + max_step_cnt = mtk_i2c_max_step_cnt(check_speed); + if ((2 * step_cnt) > low_cnt && low_cnt < max_step_cnt) { + if (low_cnt > step_cnt) { + high_cnt = 2 * step_cnt - low_cnt; + } else { + high_cnt = step_cnt; + low_cnt = step_cnt; + } + } else { + return -2; + } + + sda_max = spec->max_hd_dat_ns / sample_ns; + if (sda_max > low_cnt) + sda_max = 0; + + sda_min = DIV_ROUND_UP(spec->min_su_dat_ns, sample_ns); + if (sda_min < low_cnt) + sda_min = 0; + + if (sda_min > sda_max) + return -3; + + if (check_speed > I2C_MAX_FAST_MODE_FREQ) { + if (i2c->dev_comp->ltiming_adjust) { + i2c->ac_timing.hs = I2C_TIME_DEFAULT_VALUE | + (sample_cnt << 12) | (high_cnt << 8); + i2c->ac_timing.ltiming &= ~GENMASK(15, 9); + i2c->ac_timing.ltiming |= (sample_cnt << 12) | + (low_cnt << 9); + i2c->ac_timing.ext &= ~GENMASK(7, 1); + i2c->ac_timing.ext |= (su_sta_cnt << 1) | (1 << 0); + } else { + i2c->ac_timing.hs_scl_hl_ratio = (1 << 12) | + (high_cnt << 6) | low_cnt; + i2c->ac_timing.hs_sta_stop = (su_sta_cnt << 8) | + su_sta_cnt; + } + i2c->ac_timing.sda_timing &= ~GENMASK(11, 6); + i2c->ac_timing.sda_timing |= (1 << 12) | + ((sda_max + sda_min) / 2) << 6; + } else { + if (i2c->dev_comp->ltiming_adjust) { + i2c->ac_timing.htiming = (sample_cnt << 8) | (high_cnt); + i2c->ac_timing.ltiming = (sample_cnt << 6) | (low_cnt); + i2c->ac_timing.ext = (su_sta_cnt << 8) | (1 << 0); + } else { + i2c->ac_timing.scl_hl_ratio = (1 << 12) | + (high_cnt << 6) | low_cnt; + i2c->ac_timing.sta_stop = (su_sta_cnt << 8) | + su_sta_cnt; + } + + i2c->ac_timing.sda_timing = (1 << 12) | + (sda_max + sda_min) / 2; + } + + return 0; +} + /* * Calculate i2c port speed * @@ -446,15 +656,12 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src, unsigned int opt_div; unsigned int best_mul; unsigned int cnt_mul; + int ret = -EINVAL; if (target_speed > I2C_MAX_FAST_MODE_PLUS_FREQ) target_speed = I2C_MAX_FAST_MODE_PLUS_FREQ; - if (target_speed > I2C_MAX_FAST_MODE_FREQ) - max_step_cnt = MAX_HS_STEP_CNT_DIV; - else - max_step_cnt = MAX_STEP_CNT_DIV; - + max_step_cnt = mtk_i2c_max_step_cnt(target_speed); base_step_cnt = max_step_cnt; /* Find the best combination */ opt_div = DIV_ROUND_UP(clk_src >> 1, target_speed); @@ -473,6 +680,11 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src, continue; if (cnt_mul < best_mul) { + ret = mtk_i2c_check_ac_timing(i2c, clk_src, + target_speed, step_cnt - 1, sample_cnt - 1); + if (ret) + continue; + best_mul = cnt_mul; base_sample_cnt = sample_cnt; base_step_cnt = step_cnt; @@ -481,6 +693,9 @@ static int mtk_i2c_calculate_speed(struct mtk_i2c *i2c, unsigned int clk_src, } } + if (ret) + return -EINVAL; + sample_cnt = base_sample_cnt; step_cnt = base_step_cnt; @@ -506,47 +721,68 @@ static int mtk_i2c_set_speed(struct mtk_i2c *i2c, unsigned int parent_clk) unsigned int l_step_cnt; unsigned int l_sample_cnt; unsigned int target_speed; + unsigned int clk_div; + unsigned int max_clk_div; int ret; - clk_src = parent_clk / i2c->clk_src_div; target_speed = i2c->speed_hz; + parent_clk /= i2c->clk_src_div; - if (target_speed > I2C_MAX_FAST_MODE_FREQ) { - /* Set master code speed register */ - ret = mtk_i2c_calculate_speed(i2c, clk_src, I2C_MAX_FAST_MODE_FREQ, - &l_step_cnt, &l_sample_cnt); - if (ret < 0) - return ret; - - i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt; - - /* Set the high speed mode register */ - ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed, - &step_cnt, &sample_cnt); - if (ret < 0) - return ret; - - i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE | - (sample_cnt << 12) | (step_cnt << 8); + if (i2c->dev_comp->timing_adjust) + max_clk_div = MAX_CLOCK_DIV; + else + max_clk_div = 1; + + for (clk_div = 1; clk_div <= max_clk_div; clk_div++) { + clk_src = parent_clk / clk_div; + + if (target_speed > I2C_MAX_FAST_MODE_FREQ) { + /* Set master code speed register */ + ret = mtk_i2c_calculate_speed(i2c, clk_src, + I2C_MAX_FAST_MODE_FREQ, + &l_step_cnt, + &l_sample_cnt); + if (ret < 0) + continue; + + i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt; + + /* Set the high speed mode register */ + ret = mtk_i2c_calculate_speed(i2c, clk_src, + target_speed, &step_cnt, + &sample_cnt); + if (ret < 0) + continue; + + i2c->high_speed_reg = I2C_TIME_DEFAULT_VALUE | + (sample_cnt << 12) | (step_cnt << 8); + + if (i2c->dev_comp->ltiming_adjust) + i2c->ltiming_reg = + (l_sample_cnt << 6) | l_step_cnt | + (sample_cnt << 12) | (step_cnt << 9); + } else { + ret = mtk_i2c_calculate_speed(i2c, clk_src, + target_speed, &l_step_cnt, + &l_sample_cnt); + if (ret < 0) + continue; - if (i2c->dev_comp->ltiming_adjust) - i2c->ltiming_reg = (l_sample_cnt << 6) | l_step_cnt | - (sample_cnt << 12) | (step_cnt << 9); - } else { - ret = mtk_i2c_calculate_speed(i2c, clk_src, target_speed, - &step_cnt, &sample_cnt); - if (ret < 0) - return ret; + i2c->timing_reg = (l_sample_cnt << 8) | l_step_cnt; - i2c->timing_reg = (sample_cnt << 8) | step_cnt; + /* Disable the high speed transaction */ + i2c->high_speed_reg = I2C_TIME_CLR_VALUE; - /* Disable the high speed transaction */ - i2c->high_speed_reg = I2C_TIME_CLR_VALUE; + if (i2c->dev_comp->ltiming_adjust) + i2c->ltiming_reg = + (l_sample_cnt << 6) | l_step_cnt; + } - if (i2c->dev_comp->ltiming_adjust) - i2c->ltiming_reg = (sample_cnt << 6) | step_cnt; + break; } + i2c->ac_timing.inter_clk_div = clk_div - 1; + return 0; } @@ -586,12 +822,6 @@ static int mtk_i2c_do_transfer(struct mtk_i2c *i2c, struct i2c_msg *msgs, mtk_i2c_writew(i2c, control_reg, OFFSET_CONTROL); - /* set start condition */ - if (i2c->speed_hz <= I2C_MAX_STANDARD_MODE_FREQ) - mtk_i2c_writew(i2c, I2C_ST_START_CON, OFFSET_EXT_CONF); - else - mtk_i2c_writew(i2c, I2C_FS_START_CON, OFFSET_EXT_CONF); - addr_reg = i2c_8bit_addr_from_msg(msgs); mtk_i2c_writew(i2c, addr_reg, OFFSET_SLAVE_ADDR); @@ -948,9 +1178,6 @@ static int mtk_i2c_probe(struct platform_device *pdev) if (ret) return -EINVAL; - if (i2c->dev_comp->timing_adjust) - i2c->clk_src_div *= I2C_DEFAULT_CLK_DIV; - if (i2c->have_pmic && !i2c->dev_comp->pmic_i2c) return -EINVAL; diff --git a/drivers/i2c/busses/i2c-mv64xxx.c b/drivers/i2c/busses/i2c-mv64xxx.c index 9b8f1d8552ea..829b8c98ae51 100644 --- a/drivers/i2c/busses/i2c-mv64xxx.c +++ b/drivers/i2c/busses/i2c-mv64xxx.c @@ -877,7 +877,6 @@ mv64xxx_i2c_probe(struct platform_device *pd) { struct mv64xxx_i2c_data *drv_data; struct mv64xxx_i2c_pdata *pdata = dev_get_platdata(&pd->dev); - struct resource *r; int rc; if ((!pdata && !pd->dev.of_node)) @@ -888,8 +887,7 @@ mv64xxx_i2c_probe(struct platform_device *pd) if (!drv_data) return -ENOMEM; - r = platform_get_resource(pd, IORESOURCE_MEM, 0); - drv_data->reg_base = devm_ioremap_resource(&pd->dev, r); + drv_data->reg_base = devm_platform_ioremap_resource(pd, 0); if (IS_ERR(drv_data->reg_base)) return PTR_ERR(drv_data->reg_base); diff --git a/drivers/i2c/busses/i2c-npcm7xx.c b/drivers/i2c/busses/i2c-npcm7xx.c new file mode 100644 index 000000000000..75f07138a6fa --- /dev/null +++ b/drivers/i2c/busses/i2c-npcm7xx.c @@ -0,0 +1,2342 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Nuvoton NPCM7xx I2C Controller driver + * + * Copyright (C) 2020 Nuvoton Technologies tali.perry@nuvoton.com + */ +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/debugfs.h> +#include <linux/errno.h> +#include <linux/i2c.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/irq.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +enum i2c_mode { + I2C_MASTER, + I2C_SLAVE, +}; + +/* + * External I2C Interface driver xfer indication values, which indicate status + * of the bus. + */ +enum i2c_state_ind { + I2C_NO_STATUS_IND = 0, + I2C_SLAVE_RCV_IND, + I2C_SLAVE_XMIT_IND, + I2C_SLAVE_XMIT_MISSING_DATA_IND, + I2C_SLAVE_RESTART_IND, + I2C_SLAVE_DONE_IND, + I2C_MASTER_DONE_IND, + I2C_NACK_IND, + I2C_BUS_ERR_IND, + I2C_WAKE_UP_IND, + I2C_BLOCK_BYTES_ERR_IND, + I2C_SLAVE_RCV_MISSING_DATA_IND, +}; + +/* + * Operation type values (used to define the operation currently running) + * module is interrupt driven, on each interrupt the current operation is + * checked to see if the module is currently reading or writing. + */ +enum i2c_oper { + I2C_NO_OPER = 0, + I2C_WRITE_OPER, + I2C_READ_OPER, +}; + +/* I2C Bank (module had 2 banks of registers) */ +enum i2c_bank { + I2C_BANK_0 = 0, + I2C_BANK_1, +}; + +/* Internal I2C states values (for the I2C module state machine). */ +enum i2c_state { + I2C_DISABLE = 0, + I2C_IDLE, + I2C_MASTER_START, + I2C_SLAVE_MATCH, + I2C_OPER_STARTED, + I2C_STOP_PENDING, +}; + +#if IS_ENABLED(CONFIG_I2C_SLAVE) +/* Module supports setting multiple own slave addresses */ +enum i2c_addr { + I2C_SLAVE_ADDR1 = 0, + I2C_SLAVE_ADDR2, + I2C_SLAVE_ADDR3, + I2C_SLAVE_ADDR4, + I2C_SLAVE_ADDR5, + I2C_SLAVE_ADDR6, + I2C_SLAVE_ADDR7, + I2C_SLAVE_ADDR8, + I2C_SLAVE_ADDR9, + I2C_SLAVE_ADDR10, + I2C_GC_ADDR, + I2C_ARP_ADDR, +}; +#endif + +/* init register and default value required to enable module */ +#define NPCM_I2CSEGCTL 0xE4 +#define NPCM_I2CSEGCTL_INIT_VAL 0x0333F000 + +/* Common regs */ +#define NPCM_I2CSDA 0x00 +#define NPCM_I2CST 0x02 +#define NPCM_I2CCST 0x04 +#define NPCM_I2CCTL1 0x06 +#define NPCM_I2CADDR1 0x08 +#define NPCM_I2CCTL2 0x0A +#define NPCM_I2CADDR2 0x0C +#define NPCM_I2CCTL3 0x0E +#define NPCM_I2CCST2 0x18 +#define NPCM_I2CCST3 0x19 +#define I2C_VER 0x1F + +/*BANK0 regs*/ +#define NPCM_I2CADDR3 0x10 +#define NPCM_I2CADDR7 0x11 +#define NPCM_I2CADDR4 0x12 +#define NPCM_I2CADDR8 0x13 +#define NPCM_I2CADDR5 0x14 +#define NPCM_I2CADDR9 0x15 +#define NPCM_I2CADDR6 0x16 +#define NPCM_I2CADDR10 0x17 + +#if IS_ENABLED(CONFIG_I2C_SLAVE) +/* + * npcm_i2caddr array: + * The module supports having multiple own slave addresses. + * Since the addr regs are sprinkled all over the address space, + * use this array to get the address or each register. + */ +#define I2C_NUM_OWN_ADDR 10 +static const int npcm_i2caddr[I2C_NUM_OWN_ADDR] = { + NPCM_I2CADDR1, NPCM_I2CADDR2, NPCM_I2CADDR3, NPCM_I2CADDR4, + NPCM_I2CADDR5, NPCM_I2CADDR6, NPCM_I2CADDR7, NPCM_I2CADDR8, + NPCM_I2CADDR9, NPCM_I2CADDR10, +}; +#endif + +#define NPCM_I2CCTL4 0x1A +#define NPCM_I2CCTL5 0x1B +#define NPCM_I2CSCLLT 0x1C /* SCL Low Time */ +#define NPCM_I2CFIF_CTL 0x1D /* FIFO Control */ +#define NPCM_I2CSCLHT 0x1E /* SCL High Time */ + +/* BANK 1 regs */ +#define NPCM_I2CFIF_CTS 0x10 /* Both FIFOs Control and Status */ +#define NPCM_I2CTXF_CTL 0x12 /* Tx-FIFO Control */ +#define NPCM_I2CT_OUT 0x14 /* Bus T.O. */ +#define NPCM_I2CPEC 0x16 /* PEC Data */ +#define NPCM_I2CTXF_STS 0x1A /* Tx-FIFO Status */ +#define NPCM_I2CRXF_STS 0x1C /* Rx-FIFO Status */ +#define NPCM_I2CRXF_CTL 0x1E /* Rx-FIFO Control */ + +/* NPCM_I2CST reg fields */ +#define NPCM_I2CST_XMIT BIT(0) +#define NPCM_I2CST_MASTER BIT(1) +#define NPCM_I2CST_NMATCH BIT(2) +#define NPCM_I2CST_STASTR BIT(3) +#define NPCM_I2CST_NEGACK BIT(4) +#define NPCM_I2CST_BER BIT(5) +#define NPCM_I2CST_SDAST BIT(6) +#define NPCM_I2CST_SLVSTP BIT(7) + +/* NPCM_I2CCST reg fields */ +#define NPCM_I2CCST_BUSY BIT(0) +#define NPCM_I2CCST_BB BIT(1) +#define NPCM_I2CCST_MATCH BIT(2) +#define NPCM_I2CCST_GCMATCH BIT(3) +#define NPCM_I2CCST_TSDA BIT(4) +#define NPCM_I2CCST_TGSCL BIT(5) +#define NPCM_I2CCST_MATCHAF BIT(6) +#define NPCM_I2CCST_ARPMATCH BIT(7) + +/* NPCM_I2CCTL1 reg fields */ +#define NPCM_I2CCTL1_START BIT(0) +#define NPCM_I2CCTL1_STOP BIT(1) +#define NPCM_I2CCTL1_INTEN BIT(2) +#define NPCM_I2CCTL1_EOBINTE BIT(3) +#define NPCM_I2CCTL1_ACK BIT(4) +#define NPCM_I2CCTL1_GCMEN BIT(5) +#define NPCM_I2CCTL1_NMINTE BIT(6) +#define NPCM_I2CCTL1_STASTRE BIT(7) + +/* RW1S fields (inside a RW reg): */ +#define NPCM_I2CCTL1_RWS \ + (NPCM_I2CCTL1_START | NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_ACK) + +/* npcm_i2caddr reg fields */ +#define NPCM_I2CADDR_A GENMASK(6, 0) +#define NPCM_I2CADDR_SAEN BIT(7) + +/* NPCM_I2CCTL2 reg fields */ +#define I2CCTL2_ENABLE BIT(0) +#define I2CCTL2_SCLFRQ6_0 GENMASK(7, 1) + +/* NPCM_I2CCTL3 reg fields */ +#define I2CCTL3_SCLFRQ8_7 GENMASK(1, 0) +#define I2CCTL3_ARPMEN BIT(2) +#define I2CCTL3_IDL_START BIT(3) +#define I2CCTL3_400K_MODE BIT(4) +#define I2CCTL3_BNK_SEL BIT(5) +#define I2CCTL3_SDA_LVL BIT(6) +#define I2CCTL3_SCL_LVL BIT(7) + +/* NPCM_I2CCST2 reg fields */ +#define NPCM_I2CCST2_MATCHA1F BIT(0) +#define NPCM_I2CCST2_MATCHA2F BIT(1) +#define NPCM_I2CCST2_MATCHA3F BIT(2) +#define NPCM_I2CCST2_MATCHA4F BIT(3) +#define NPCM_I2CCST2_MATCHA5F BIT(4) +#define NPCM_I2CCST2_MATCHA6F BIT(5) +#define NPCM_I2CCST2_MATCHA7F BIT(5) +#define NPCM_I2CCST2_INTSTS BIT(7) + +/* NPCM_I2CCST3 reg fields */ +#define NPCM_I2CCST3_MATCHA8F BIT(0) +#define NPCM_I2CCST3_MATCHA9F BIT(1) +#define NPCM_I2CCST3_MATCHA10F BIT(2) +#define NPCM_I2CCST3_EO_BUSY BIT(7) + +/* NPCM_I2CCTL4 reg fields */ +#define I2CCTL4_HLDT GENMASK(5, 0) +#define I2CCTL4_LVL_WE BIT(7) + +/* NPCM_I2CCTL5 reg fields */ +#define I2CCTL5_DBNCT GENMASK(3, 0) + +/* NPCM_I2CFIF_CTS reg fields */ +#define NPCM_I2CFIF_CTS_RXF_TXE BIT(1) +#define NPCM_I2CFIF_CTS_RFTE_IE BIT(3) +#define NPCM_I2CFIF_CTS_CLR_FIFO BIT(6) +#define NPCM_I2CFIF_CTS_SLVRSTR BIT(7) + +/* NPCM_I2CTXF_CTL reg fields */ +#define NPCM_I2CTXF_CTL_TX_THR GENMASK(4, 0) +#define NPCM_I2CTXF_CTL_THR_TXIE BIT(6) + +/* NPCM_I2CT_OUT reg fields */ +#define NPCM_I2CT_OUT_TO_CKDIV GENMASK(5, 0) +#define NPCM_I2CT_OUT_T_OUTIE BIT(6) +#define NPCM_I2CT_OUT_T_OUTST BIT(7) + +/* NPCM_I2CTXF_STS reg fields */ +#define NPCM_I2CTXF_STS_TX_BYTES GENMASK(4, 0) +#define NPCM_I2CTXF_STS_TX_THST BIT(6) + +/* NPCM_I2CRXF_STS reg fields */ +#define NPCM_I2CRXF_STS_RX_BYTES GENMASK(4, 0) +#define NPCM_I2CRXF_STS_RX_THST BIT(6) + +/* NPCM_I2CFIF_CTL reg fields */ +#define NPCM_I2CFIF_CTL_FIFO_EN BIT(4) + +/* NPCM_I2CRXF_CTL reg fields */ +#define NPCM_I2CRXF_CTL_RX_THR GENMASK(4, 0) +#define NPCM_I2CRXF_CTL_LAST_PEC BIT(5) +#define NPCM_I2CRXF_CTL_THR_RXIE BIT(6) + +#define I2C_HW_FIFO_SIZE 16 + +/* I2C_VER reg fields */ +#define I2C_VER_VERSION GENMASK(6, 0) +#define I2C_VER_FIFO_EN BIT(7) + +/* stall/stuck timeout in us */ +#define DEFAULT_STALL_COUNT 25 + +/* SCLFRQ field position */ +#define SCLFRQ_0_TO_6 GENMASK(6, 0) +#define SCLFRQ_7_TO_8 GENMASK(8, 7) + +/* supported clk settings. values in Hz. */ +#define I2C_FREQ_MIN_HZ 10000 +#define I2C_FREQ_MAX_HZ I2C_MAX_FAST_MODE_PLUS_FREQ + +/* Status of one I2C module */ +struct npcm_i2c { + struct i2c_adapter adap; + struct device *dev; + unsigned char __iomem *reg; + spinlock_t lock; /* IRQ synchronization */ + struct completion cmd_complete; + int cmd_err; + struct i2c_msg *msgs; + int msgs_num; + int num; + u32 apb_clk; + struct i2c_bus_recovery_info rinfo; + enum i2c_state state; + enum i2c_oper operation; + enum i2c_mode master_or_slave; + enum i2c_state_ind stop_ind; + u8 dest_addr; + u8 *rd_buf; + u16 rd_size; + u16 rd_ind; + u8 *wr_buf; + u16 wr_size; + u16 wr_ind; + bool fifo_use; + u16 PEC_mask; /* PEC bit mask per slave address */ + bool PEC_use; + bool read_block_use; + unsigned long int_time_stamp; + unsigned long bus_freq; /* in Hz */ +#if IS_ENABLED(CONFIG_I2C_SLAVE) + u8 own_slave_addr; + struct i2c_client *slave; + int slv_rd_size; + int slv_rd_ind; + int slv_wr_size; + int slv_wr_ind; + u8 slv_rd_buf[I2C_HW_FIFO_SIZE]; + u8 slv_wr_buf[I2C_HW_FIFO_SIZE]; +#endif + struct dentry *debugfs; /* debugfs device directory */ + u64 ber_cnt; + u64 rec_succ_cnt; + u64 rec_fail_cnt; + u64 nack_cnt; + u64 timeout_cnt; +}; + +static inline void npcm_i2c_select_bank(struct npcm_i2c *bus, + enum i2c_bank bank) +{ + u8 i2cctl3 = ioread8(bus->reg + NPCM_I2CCTL3); + + if (bank == I2C_BANK_0) + i2cctl3 = i2cctl3 & ~I2CCTL3_BNK_SEL; + else + i2cctl3 = i2cctl3 | I2CCTL3_BNK_SEL; + iowrite8(i2cctl3, bus->reg + NPCM_I2CCTL3); +} + +static void npcm_i2c_init_params(struct npcm_i2c *bus) +{ + bus->stop_ind = I2C_NO_STATUS_IND; + bus->rd_size = 0; + bus->wr_size = 0; + bus->rd_ind = 0; + bus->wr_ind = 0; + bus->read_block_use = false; + bus->int_time_stamp = 0; + bus->PEC_use = false; + bus->PEC_mask = 0; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + if (bus->slave) + bus->master_or_slave = I2C_SLAVE; +#endif +} + +static inline void npcm_i2c_wr_byte(struct npcm_i2c *bus, u8 data) +{ + iowrite8(data, bus->reg + NPCM_I2CSDA); +} + +static inline u8 npcm_i2c_rd_byte(struct npcm_i2c *bus) +{ + return ioread8(bus->reg + NPCM_I2CSDA); +} + +static int npcm_i2c_get_SCL(struct i2c_adapter *_adap) +{ + struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap); + + return !!(I2CCTL3_SCL_LVL & ioread32(bus->reg + NPCM_I2CCTL3)); +} + +static int npcm_i2c_get_SDA(struct i2c_adapter *_adap) +{ + struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap); + + return !!(I2CCTL3_SDA_LVL & ioread32(bus->reg + NPCM_I2CCTL3)); +} + +static inline u16 npcm_i2c_get_index(struct npcm_i2c *bus) +{ + if (bus->operation == I2C_READ_OPER) + return bus->rd_ind; + if (bus->operation == I2C_WRITE_OPER) + return bus->wr_ind; + return 0; +} + +/* quick protocol (just address) */ +static inline bool npcm_i2c_is_quick(struct npcm_i2c *bus) +{ + return bus->wr_size == 0 && bus->rd_size == 0; +} + +static void npcm_i2c_disable(struct npcm_i2c *bus) +{ + u8 i2cctl2; + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + int i; + + /* select bank 0 for I2C addresses */ + npcm_i2c_select_bank(bus, I2C_BANK_0); + + /* Slave addresses removal */ + for (i = I2C_SLAVE_ADDR1; i < I2C_NUM_OWN_ADDR; i++) + iowrite8(0, bus->reg + npcm_i2caddr[i]); + + npcm_i2c_select_bank(bus, I2C_BANK_1); +#endif + /* Disable module */ + i2cctl2 = ioread8(bus->reg + NPCM_I2CCTL2); + i2cctl2 = i2cctl2 & ~I2CCTL2_ENABLE; + iowrite8(i2cctl2, bus->reg + NPCM_I2CCTL2); + + bus->state = I2C_DISABLE; +} + +static void npcm_i2c_enable(struct npcm_i2c *bus) +{ + u8 i2cctl2 = ioread8(bus->reg + NPCM_I2CCTL2); + + i2cctl2 = i2cctl2 | I2CCTL2_ENABLE; + iowrite8(i2cctl2, bus->reg + NPCM_I2CCTL2); + bus->state = I2C_IDLE; +} + +/* enable\disable end of busy (EOB) interrupts */ +static inline void npcm_i2c_eob_int(struct npcm_i2c *bus, bool enable) +{ + u8 val; + + /* Clear EO_BUSY pending bit: */ + val = ioread8(bus->reg + NPCM_I2CCST3); + val = val | NPCM_I2CCST3_EO_BUSY; + iowrite8(val, bus->reg + NPCM_I2CCST3); + + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~NPCM_I2CCTL1_RWS; + if (enable) + val |= NPCM_I2CCTL1_EOBINTE; + else + val &= ~NPCM_I2CCTL1_EOBINTE; + iowrite8(val, bus->reg + NPCM_I2CCTL1); +} + +static inline bool npcm_i2c_tx_fifo_empty(struct npcm_i2c *bus) +{ + u8 tx_fifo_sts; + + tx_fifo_sts = ioread8(bus->reg + NPCM_I2CTXF_STS); + /* check if TX FIFO is not empty */ + if ((tx_fifo_sts & NPCM_I2CTXF_STS_TX_BYTES) == 0) + return false; + + /* check if TX FIFO status bit is set: */ + return !!FIELD_GET(NPCM_I2CTXF_STS_TX_THST, tx_fifo_sts); +} + +static inline bool npcm_i2c_rx_fifo_full(struct npcm_i2c *bus) +{ + u8 rx_fifo_sts; + + rx_fifo_sts = ioread8(bus->reg + NPCM_I2CRXF_STS); + /* check if RX FIFO is not empty: */ + if ((rx_fifo_sts & NPCM_I2CRXF_STS_RX_BYTES) == 0) + return false; + + /* check if rx fifo full status is set: */ + return !!FIELD_GET(NPCM_I2CRXF_STS_RX_THST, rx_fifo_sts); +} + +static inline void npcm_i2c_clear_fifo_int(struct npcm_i2c *bus) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CFIF_CTS); + val = (val & NPCM_I2CFIF_CTS_SLVRSTR) | NPCM_I2CFIF_CTS_RXF_TXE; + iowrite8(val, bus->reg + NPCM_I2CFIF_CTS); +} + +static inline void npcm_i2c_clear_tx_fifo(struct npcm_i2c *bus) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CTXF_STS); + val = val | NPCM_I2CTXF_STS_TX_THST; + iowrite8(val, bus->reg + NPCM_I2CTXF_STS); +} + +static inline void npcm_i2c_clear_rx_fifo(struct npcm_i2c *bus) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CRXF_STS); + val = val | NPCM_I2CRXF_STS_RX_THST; + iowrite8(val, bus->reg + NPCM_I2CRXF_STS); +} + +static void npcm_i2c_int_enable(struct npcm_i2c *bus, bool enable) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~NPCM_I2CCTL1_RWS; + if (enable) + val |= NPCM_I2CCTL1_INTEN; + else + val &= ~NPCM_I2CCTL1_INTEN; + iowrite8(val, bus->reg + NPCM_I2CCTL1); +} + +static inline void npcm_i2c_master_start(struct npcm_i2c *bus) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~(NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_ACK); + val |= NPCM_I2CCTL1_START; + iowrite8(val, bus->reg + NPCM_I2CCTL1); +} + +static inline void npcm_i2c_master_stop(struct npcm_i2c *bus) +{ + u8 val; + + /* + * override HW issue: I2C may fail to supply stop condition in Master + * Write operation. + * Need to delay at least 5 us from the last int, before issueing a stop + */ + udelay(10); /* function called from interrupt, can't sleep */ + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~(NPCM_I2CCTL1_START | NPCM_I2CCTL1_ACK); + val |= NPCM_I2CCTL1_STOP; + iowrite8(val, bus->reg + NPCM_I2CCTL1); + + if (!bus->fifo_use) + return; + + npcm_i2c_select_bank(bus, I2C_BANK_1); + + if (bus->operation == I2C_READ_OPER) + npcm_i2c_clear_rx_fifo(bus); + else + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_clear_fifo_int(bus); + iowrite8(0, bus->reg + NPCM_I2CTXF_CTL); +} + +static inline void npcm_i2c_stall_after_start(struct npcm_i2c *bus, bool stall) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~NPCM_I2CCTL1_RWS; + if (stall) + val |= NPCM_I2CCTL1_STASTRE; + else + val &= ~NPCM_I2CCTL1_STASTRE; + iowrite8(val, bus->reg + NPCM_I2CCTL1); +} + +static inline void npcm_i2c_nack(struct npcm_i2c *bus) +{ + u8 val; + + val = ioread8(bus->reg + NPCM_I2CCTL1); + val &= ~(NPCM_I2CCTL1_STOP | NPCM_I2CCTL1_START); + val |= NPCM_I2CCTL1_ACK; + iowrite8(val, bus->reg + NPCM_I2CCTL1); +} + +#if IS_ENABLED(CONFIG_I2C_SLAVE) +static void npcm_i2c_slave_int_enable(struct npcm_i2c *bus, bool enable) +{ + u8 i2cctl1; + + /* enable interrupt on slave match: */ + i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1); + i2cctl1 &= ~NPCM_I2CCTL1_RWS; + if (enable) + i2cctl1 |= NPCM_I2CCTL1_NMINTE; + else + i2cctl1 &= ~NPCM_I2CCTL1_NMINTE; + iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1); +} + +static int npcm_i2c_slave_enable(struct npcm_i2c *bus, enum i2c_addr addr_type, + u8 addr, bool enable) +{ + u8 i2cctl1; + u8 i2cctl3; + u8 sa_reg; + + sa_reg = (addr & 0x7F) | FIELD_PREP(NPCM_I2CADDR_SAEN, enable); + if (addr_type == I2C_GC_ADDR) { + i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1); + if (enable) + i2cctl1 |= NPCM_I2CCTL1_GCMEN; + else + i2cctl1 &= ~NPCM_I2CCTL1_GCMEN; + iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1); + return 0; + } + if (addr_type == I2C_ARP_ADDR) { + i2cctl3 = ioread8(bus->reg + NPCM_I2CCTL3); + if (enable) + i2cctl3 |= I2CCTL3_ARPMEN; + else + i2cctl3 &= ~I2CCTL3_ARPMEN; + iowrite8(i2cctl3, bus->reg + NPCM_I2CCTL3); + return 0; + } + if (addr_type >= I2C_ARP_ADDR) + return -EFAULT; + /* select bank 0 for address 3 to 10 */ + if (addr_type > I2C_SLAVE_ADDR2) + npcm_i2c_select_bank(bus, I2C_BANK_0); + /* Set and enable the address */ + iowrite8(sa_reg, bus->reg + npcm_i2caddr[addr_type]); + npcm_i2c_slave_int_enable(bus, enable); + if (addr_type > I2C_SLAVE_ADDR2) + npcm_i2c_select_bank(bus, I2C_BANK_1); + return 0; +} +#endif + +static void npcm_i2c_reset(struct npcm_i2c *bus) +{ + /* + * Save I2CCTL1 relevant bits. It is being cleared when the module + * is disabled. + */ + u8 i2cctl1; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + u8 addr; +#endif + + i2cctl1 = ioread8(bus->reg + NPCM_I2CCTL1); + + npcm_i2c_disable(bus); + npcm_i2c_enable(bus); + + /* Restore NPCM_I2CCTL1 Status */ + i2cctl1 &= ~NPCM_I2CCTL1_RWS; + iowrite8(i2cctl1, bus->reg + NPCM_I2CCTL1); + + /* Clear BB (BUS BUSY) bit */ + iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST); + iowrite8(0xFF, bus->reg + NPCM_I2CST); + + /* Clear EOB bit */ + iowrite8(NPCM_I2CCST3_EO_BUSY, bus->reg + NPCM_I2CCST3); + + /* Clear all fifo bits: */ + iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS); + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + if (bus->slave) { + addr = bus->slave->addr; + npcm_i2c_slave_enable(bus, I2C_SLAVE_ADDR1, addr, true); + } +#endif + + bus->state = I2C_IDLE; +} + +static inline bool npcm_i2c_is_master(struct npcm_i2c *bus) +{ + return !!FIELD_GET(NPCM_I2CST_MASTER, ioread8(bus->reg + NPCM_I2CST)); +} + +static void npcm_i2c_callback(struct npcm_i2c *bus, + enum i2c_state_ind op_status, u16 info) +{ + struct i2c_msg *msgs; + int msgs_num; + + msgs = bus->msgs; + msgs_num = bus->msgs_num; + /* + * check that transaction was not timed-out, and msgs still + * holds a valid value. + */ + if (!msgs) + return; + + if (completion_done(&bus->cmd_complete)) + return; + + switch (op_status) { + case I2C_MASTER_DONE_IND: + bus->cmd_err = bus->msgs_num; + fallthrough; + case I2C_BLOCK_BYTES_ERR_IND: + /* Master tx finished and all transmit bytes were sent */ + if (bus->msgs) { + if (msgs[0].flags & I2C_M_RD) + msgs[0].len = info; + else if (msgs_num == 2 && + msgs[1].flags & I2C_M_RD) + msgs[1].len = info; + } + if (completion_done(&bus->cmd_complete) == false) + complete(&bus->cmd_complete); + break; + + case I2C_NACK_IND: + /* MASTER transmit got a NACK before tx all bytes */ + bus->cmd_err = -ENXIO; + if (bus->master_or_slave == I2C_MASTER) + complete(&bus->cmd_complete); + + break; + case I2C_BUS_ERR_IND: + /* Bus error */ + bus->cmd_err = -EAGAIN; + if (bus->master_or_slave == I2C_MASTER) + complete(&bus->cmd_complete); + + break; + case I2C_WAKE_UP_IND: + /* I2C wake up */ + break; + default: + break; + } + + bus->operation = I2C_NO_OPER; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + if (bus->slave) + bus->master_or_slave = I2C_SLAVE; +#endif +} + +static u8 npcm_i2c_fifo_usage(struct npcm_i2c *bus) +{ + if (bus->operation == I2C_WRITE_OPER) + return FIELD_GET(NPCM_I2CTXF_STS_TX_BYTES, + ioread8(bus->reg + NPCM_I2CTXF_STS)); + if (bus->operation == I2C_READ_OPER) + return FIELD_GET(NPCM_I2CRXF_STS_RX_BYTES, + ioread8(bus->reg + NPCM_I2CRXF_STS)); + return 0; +} + +static void npcm_i2c_write_to_fifo_master(struct npcm_i2c *bus, u16 max_bytes) +{ + u8 size_free_fifo; + + /* + * Fill the FIFO, while the FIFO is not full and there are more bytes + * to write + */ + size_free_fifo = I2C_HW_FIFO_SIZE - npcm_i2c_fifo_usage(bus); + while (max_bytes-- && size_free_fifo) { + if (bus->wr_ind < bus->wr_size) + npcm_i2c_wr_byte(bus, bus->wr_buf[bus->wr_ind++]); + else + npcm_i2c_wr_byte(bus, 0xFF); + size_free_fifo = I2C_HW_FIFO_SIZE - npcm_i2c_fifo_usage(bus); + } +} + +/* + * npcm_i2c_set_fifo: + * configure the FIFO before using it. If nread is -1 RX FIFO will not be + * configured. same for nwrite + */ +static void npcm_i2c_set_fifo(struct npcm_i2c *bus, int nread, int nwrite) +{ + u8 rxf_ctl = 0; + + if (!bus->fifo_use) + return; + npcm_i2c_select_bank(bus, I2C_BANK_1); + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_clear_rx_fifo(bus); + + /* configure RX FIFO */ + if (nread > 0) { + rxf_ctl = min_t(int, nread, I2C_HW_FIFO_SIZE); + + /* set LAST bit. if LAST is set next FIFO packet is nacked */ + if (nread <= I2C_HW_FIFO_SIZE) + rxf_ctl |= NPCM_I2CRXF_CTL_LAST_PEC; + + /* + * if we are about to read the first byte in blk rd mode, + * don't NACK it. If slave returns zero size HW can't NACK + * it immidiattly, it will read extra byte and then NACK. + */ + if (bus->rd_ind == 0 && bus->read_block_use) { + /* set fifo to read one byte, no last: */ + rxf_ctl = 1; + } + + /* set fifo size: */ + iowrite8(rxf_ctl, bus->reg + NPCM_I2CRXF_CTL); + } + + /* configure TX FIFO */ + if (nwrite > 0) { + if (nwrite > I2C_HW_FIFO_SIZE) + /* data to send is more then FIFO size. */ + iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CTXF_CTL); + else + iowrite8(nwrite, bus->reg + NPCM_I2CTXF_CTL); + + npcm_i2c_clear_tx_fifo(bus); + } +} + +static void npcm_i2c_read_fifo(struct npcm_i2c *bus, u8 bytes_in_fifo) +{ + u8 data; + + while (bytes_in_fifo--) { + data = npcm_i2c_rd_byte(bus); + if (bus->rd_ind < bus->rd_size) + bus->rd_buf[bus->rd_ind++] = data; + } +} + +static inline void npcm_i2c_clear_master_status(struct npcm_i2c *bus) +{ + u8 val; + + /* Clear NEGACK, STASTR and BER bits */ + val = NPCM_I2CST_BER | NPCM_I2CST_NEGACK | NPCM_I2CST_STASTR; + iowrite8(val, bus->reg + NPCM_I2CST); +} + +static void npcm_i2c_master_abort(struct npcm_i2c *bus) +{ + /* Only current master is allowed to issue a stop condition */ + if (!npcm_i2c_is_master(bus)) + return; + + npcm_i2c_eob_int(bus, true); + npcm_i2c_master_stop(bus); + npcm_i2c_clear_master_status(bus); +} + +#if IS_ENABLED(CONFIG_I2C_SLAVE) +static u8 npcm_i2c_get_slave_addr(struct npcm_i2c *bus, enum i2c_addr addr_type) +{ + u8 slave_add; + + /* select bank 0 for address 3 to 10 */ + if (addr_type > I2C_SLAVE_ADDR2) + npcm_i2c_select_bank(bus, I2C_BANK_0); + + slave_add = ioread8(bus->reg + npcm_i2caddr[(int)addr_type]); + + if (addr_type > I2C_SLAVE_ADDR2) + npcm_i2c_select_bank(bus, I2C_BANK_1); + + return slave_add; +} + +static int npcm_i2c_remove_slave_addr(struct npcm_i2c *bus, u8 slave_add) +{ + int i; + + /* Set the enable bit */ + slave_add |= 0x80; + npcm_i2c_select_bank(bus, I2C_BANK_0); + for (i = I2C_SLAVE_ADDR1; i < I2C_NUM_OWN_ADDR; i++) { + if (ioread8(bus->reg + npcm_i2caddr[i]) == slave_add) + iowrite8(0, bus->reg + npcm_i2caddr[i]); + } + npcm_i2c_select_bank(bus, I2C_BANK_1); + return 0; +} + +static void npcm_i2c_write_fifo_slave(struct npcm_i2c *bus, u16 max_bytes) +{ + /* + * Fill the FIFO, while the FIFO is not full and there are more bytes + * to write + */ + npcm_i2c_clear_fifo_int(bus); + npcm_i2c_clear_tx_fifo(bus); + iowrite8(0, bus->reg + NPCM_I2CTXF_CTL); + while (max_bytes-- && I2C_HW_FIFO_SIZE != npcm_i2c_fifo_usage(bus)) { + if (bus->slv_wr_size <= 0) + break; + bus->slv_wr_ind = bus->slv_wr_ind % I2C_HW_FIFO_SIZE; + npcm_i2c_wr_byte(bus, bus->slv_wr_buf[bus->slv_wr_ind]); + bus->slv_wr_ind++; + bus->slv_wr_ind = bus->slv_wr_ind % I2C_HW_FIFO_SIZE; + bus->slv_wr_size--; + } +} + +static void npcm_i2c_read_fifo_slave(struct npcm_i2c *bus, u8 bytes_in_fifo) +{ + u8 data; + + if (!bus->slave) + return; + + while (bytes_in_fifo--) { + data = npcm_i2c_rd_byte(bus); + + bus->slv_rd_ind = bus->slv_rd_ind % I2C_HW_FIFO_SIZE; + bus->slv_rd_buf[bus->slv_rd_ind] = data; + bus->slv_rd_ind++; + + /* 1st byte is length in block protocol: */ + if (bus->slv_rd_ind == 1 && bus->read_block_use) + bus->slv_rd_size = data + bus->PEC_use + 1; + } +} + +static int npcm_i2c_slave_get_wr_buf(struct npcm_i2c *bus) +{ + int i; + u8 value; + int ind; + int ret = bus->slv_wr_ind; + + /* fill a cyclic buffer */ + for (i = 0; i < I2C_HW_FIFO_SIZE; i++) { + if (bus->slv_wr_size >= I2C_HW_FIFO_SIZE) + break; + i2c_slave_event(bus->slave, I2C_SLAVE_READ_REQUESTED, &value); + ind = (bus->slv_wr_ind + bus->slv_wr_size) % I2C_HW_FIFO_SIZE; + bus->slv_wr_buf[ind] = value; + bus->slv_wr_size++; + i2c_slave_event(bus->slave, I2C_SLAVE_READ_PROCESSED, &value); + } + return I2C_HW_FIFO_SIZE - ret; +} + +static void npcm_i2c_slave_send_rd_buf(struct npcm_i2c *bus) +{ + int i; + + for (i = 0; i < bus->slv_rd_ind; i++) + i2c_slave_event(bus->slave, I2C_SLAVE_WRITE_RECEIVED, + &bus->slv_rd_buf[i]); + /* + * once we send bytes up, need to reset the counter of the wr buf + * got data from master (new offset in device), ignore wr fifo: + */ + if (bus->slv_rd_ind) { + bus->slv_wr_size = 0; + bus->slv_wr_ind = 0; + } + + bus->slv_rd_ind = 0; + bus->slv_rd_size = bus->adap.quirks->max_read_len; + + npcm_i2c_clear_fifo_int(bus); + npcm_i2c_clear_rx_fifo(bus); +} + +static void npcm_i2c_slave_receive(struct npcm_i2c *bus, u16 nread, + u8 *read_data) +{ + bus->state = I2C_OPER_STARTED; + bus->operation = I2C_READ_OPER; + bus->slv_rd_size = nread; + bus->slv_rd_ind = 0; + + iowrite8(0, bus->reg + NPCM_I2CTXF_CTL); + iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CRXF_CTL); + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_clear_rx_fifo(bus); +} + +static void npcm_i2c_slave_xmit(struct npcm_i2c *bus, u16 nwrite, + u8 *write_data) +{ + if (nwrite == 0) + return; + + bus->state = I2C_OPER_STARTED; + bus->operation = I2C_WRITE_OPER; + + /* get the next buffer */ + npcm_i2c_slave_get_wr_buf(bus); + npcm_i2c_write_fifo_slave(bus, nwrite); +} + +/* + * npcm_i2c_slave_wr_buf_sync: + * currently slave IF only supports single byte operations. + * in order to utilyze the npcm HW FIFO, the driver will ask for 16 bytes + * at a time, pack them in buffer, and then transmit them all together + * to the FIFO and onward to the bus. + * NACK on read will be once reached to bus->adap->quirks->max_read_len. + * sending a NACK wherever the backend requests for it is not supported. + * the next two functions allow reading to local buffer before writing it all + * to the HW FIFO. + */ +static void npcm_i2c_slave_wr_buf_sync(struct npcm_i2c *bus) +{ + int left_in_fifo; + + left_in_fifo = FIELD_GET(NPCM_I2CTXF_STS_TX_BYTES, + ioread8(bus->reg + NPCM_I2CTXF_STS)); + + /* fifo already full: */ + if (left_in_fifo >= I2C_HW_FIFO_SIZE || + bus->slv_wr_size >= I2C_HW_FIFO_SIZE) + return; + + /* update the wr fifo index back to the untransmitted bytes: */ + bus->slv_wr_ind = bus->slv_wr_ind - left_in_fifo; + bus->slv_wr_size = bus->slv_wr_size + left_in_fifo; + + if (bus->slv_wr_ind < 0) + bus->slv_wr_ind += I2C_HW_FIFO_SIZE; +} + +static void npcm_i2c_slave_rd_wr(struct npcm_i2c *bus) +{ + if (NPCM_I2CST_XMIT & ioread8(bus->reg + NPCM_I2CST)) { + /* + * Slave got an address match with direction bit 1 so it should + * transmit data. Write till the master will NACK + */ + bus->operation = I2C_WRITE_OPER; + npcm_i2c_slave_xmit(bus, bus->adap.quirks->max_write_len, + bus->slv_wr_buf); + } else { + /* + * Slave got an address match with direction bit 0 so it should + * receive data. + * this module does not support saying no to bytes. + * it will always ACK. + */ + bus->operation = I2C_READ_OPER; + npcm_i2c_read_fifo_slave(bus, npcm_i2c_fifo_usage(bus)); + bus->stop_ind = I2C_SLAVE_RCV_IND; + npcm_i2c_slave_send_rd_buf(bus); + npcm_i2c_slave_receive(bus, bus->adap.quirks->max_read_len, + bus->slv_rd_buf); + } +} + +static irqreturn_t npcm_i2c_int_slave_handler(struct npcm_i2c *bus) +{ + u8 val; + irqreturn_t ret = IRQ_NONE; + u8 i2cst = ioread8(bus->reg + NPCM_I2CST); + + /* Slave: A NACK has occurred */ + if (NPCM_I2CST_NEGACK & i2cst) { + bus->stop_ind = I2C_NACK_IND; + npcm_i2c_slave_wr_buf_sync(bus); + if (bus->fifo_use) + /* clear the FIFO */ + iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, + bus->reg + NPCM_I2CFIF_CTS); + + /* In slave write, NACK is OK, otherwise it is a problem */ + bus->stop_ind = I2C_NO_STATUS_IND; + bus->operation = I2C_NO_OPER; + bus->own_slave_addr = 0xFF; + + /* + * Slave has to wait for STOP to decide this is the end + * of the transaction. tx is not yet considered as done + */ + iowrite8(NPCM_I2CST_NEGACK, bus->reg + NPCM_I2CST); + + ret = IRQ_HANDLED; + } + + /* Slave mode: a Bus Error (BER) has been identified */ + if (NPCM_I2CST_BER & i2cst) { + /* + * Check whether bus arbitration or Start or Stop during data + * xfer bus arbitration problem should not result in recovery + */ + bus->stop_ind = I2C_BUS_ERR_IND; + + /* wait for bus busy before clear fifo */ + iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS); + + bus->state = I2C_IDLE; + + /* + * in BER case we might get 2 interrupts: one for slave one for + * master ( for a channel which is master\slave switching) + */ + if (completion_done(&bus->cmd_complete) == false) { + bus->cmd_err = -EIO; + complete(&bus->cmd_complete); + } + bus->own_slave_addr = 0xFF; + iowrite8(NPCM_I2CST_BER, bus->reg + NPCM_I2CST); + ret = IRQ_HANDLED; + } + + /* A Slave Stop Condition has been identified */ + if (NPCM_I2CST_SLVSTP & i2cst) { + u8 bytes_in_fifo = npcm_i2c_fifo_usage(bus); + + bus->stop_ind = I2C_SLAVE_DONE_IND; + + if (bus->operation == I2C_READ_OPER) + npcm_i2c_read_fifo_slave(bus, bytes_in_fifo); + + /* if the buffer is empty nothing will be sent */ + npcm_i2c_slave_send_rd_buf(bus); + + /* Slave done transmitting or receiving */ + bus->stop_ind = I2C_NO_STATUS_IND; + + /* + * Note, just because we got here, it doesn't mean we through + * away the wr buffer. + * we keep it until the next received offset. + */ + bus->operation = I2C_NO_OPER; + bus->own_slave_addr = 0xFF; + i2c_slave_event(bus->slave, I2C_SLAVE_STOP, 0); + iowrite8(NPCM_I2CST_SLVSTP, bus->reg + NPCM_I2CST); + if (bus->fifo_use) { + npcm_i2c_clear_fifo_int(bus); + npcm_i2c_clear_rx_fifo(bus); + npcm_i2c_clear_tx_fifo(bus); + + iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, + bus->reg + NPCM_I2CFIF_CTS); + } + bus->state = I2C_IDLE; + ret = IRQ_HANDLED; + } + + /* restart condition occurred and Rx-FIFO was not empty */ + if (bus->fifo_use && FIELD_GET(NPCM_I2CFIF_CTS_SLVRSTR, + ioread8(bus->reg + NPCM_I2CFIF_CTS))) { + bus->stop_ind = I2C_SLAVE_RESTART_IND; + bus->master_or_slave = I2C_SLAVE; + if (bus->operation == I2C_READ_OPER) + npcm_i2c_read_fifo_slave(bus, npcm_i2c_fifo_usage(bus)); + bus->operation = I2C_WRITE_OPER; + iowrite8(0, bus->reg + NPCM_I2CRXF_CTL); + val = NPCM_I2CFIF_CTS_CLR_FIFO | NPCM_I2CFIF_CTS_SLVRSTR | + NPCM_I2CFIF_CTS_RXF_TXE; + iowrite8(val, bus->reg + NPCM_I2CFIF_CTS); + npcm_i2c_slave_rd_wr(bus); + ret = IRQ_HANDLED; + } + + /* A Slave Address Match has been identified */ + if (NPCM_I2CST_NMATCH & i2cst) { + u8 info = 0; + + /* Address match automatically implies slave mode */ + bus->master_or_slave = I2C_SLAVE; + npcm_i2c_clear_fifo_int(bus); + npcm_i2c_clear_rx_fifo(bus); + npcm_i2c_clear_tx_fifo(bus); + iowrite8(0, bus->reg + NPCM_I2CTXF_CTL); + iowrite8(I2C_HW_FIFO_SIZE, bus->reg + NPCM_I2CRXF_CTL); + if (NPCM_I2CST_XMIT & i2cst) { + bus->operation = I2C_WRITE_OPER; + } else { + i2c_slave_event(bus->slave, I2C_SLAVE_WRITE_REQUESTED, + &info); + bus->operation = I2C_READ_OPER; + } + if (bus->own_slave_addr == 0xFF) { + /* Check which type of address match */ + val = ioread8(bus->reg + NPCM_I2CCST); + if (NPCM_I2CCST_MATCH & val) { + u16 addr; + enum i2c_addr eaddr; + u8 i2ccst2; + u8 i2ccst3; + + i2ccst3 = ioread8(bus->reg + NPCM_I2CCST3); + i2ccst2 = ioread8(bus->reg + NPCM_I2CCST2); + + /* + * the i2c module can response to 10 own SA. + * check which one was addressed by the master. + * repond to the first one. + */ + addr = ((i2ccst3 & 0x07) << 7) | + (i2ccst2 & 0x7F); + info = ffs(addr); + eaddr = (enum i2c_addr)info; + addr = npcm_i2c_get_slave_addr(bus, eaddr); + addr &= 0x7F; + bus->own_slave_addr = addr; + if (bus->PEC_mask & BIT(info)) + bus->PEC_use = true; + else + bus->PEC_use = false; + } else { + if (NPCM_I2CCST_GCMATCH & val) + bus->own_slave_addr = 0; + if (NPCM_I2CCST_ARPMATCH & val) + bus->own_slave_addr = 0x61; + } + } else { + /* + * Slave match can happen in two options: + * 1. Start, SA, read (slave read without further ado) + * 2. Start, SA, read, data, restart, SA, read, ... + * (slave read in fragmented mode) + * 3. Start, SA, write, data, restart, SA, read, .. + * (regular write-read mode) + */ + if ((bus->state == I2C_OPER_STARTED && + bus->operation == I2C_READ_OPER && + bus->stop_ind == I2C_SLAVE_XMIT_IND) || + bus->stop_ind == I2C_SLAVE_RCV_IND) { + /* slave tx after slave rx w/o STOP */ + bus->stop_ind = I2C_SLAVE_RESTART_IND; + } + } + + if (NPCM_I2CST_XMIT & i2cst) + bus->stop_ind = I2C_SLAVE_XMIT_IND; + else + bus->stop_ind = I2C_SLAVE_RCV_IND; + bus->state = I2C_SLAVE_MATCH; + npcm_i2c_slave_rd_wr(bus); + iowrite8(NPCM_I2CST_NMATCH, bus->reg + NPCM_I2CST); + ret = IRQ_HANDLED; + } + + /* Slave SDA status is set - tx or rx */ + if ((NPCM_I2CST_SDAST & i2cst) || + (bus->fifo_use && + (npcm_i2c_tx_fifo_empty(bus) || npcm_i2c_rx_fifo_full(bus)))) { + npcm_i2c_slave_rd_wr(bus); + iowrite8(NPCM_I2CST_SDAST, bus->reg + NPCM_I2CST); + ret = IRQ_HANDLED; + } /* SDAST */ + + return ret; +} + +static int npcm_i2c_reg_slave(struct i2c_client *client) +{ + unsigned long lock_flags; + struct npcm_i2c *bus = i2c_get_adapdata(client->adapter); + + bus->slave = client; + + if (!bus->slave) + return -EINVAL; + + if (client->flags & I2C_CLIENT_TEN) + return -EAFNOSUPPORT; + + spin_lock_irqsave(&bus->lock, lock_flags); + + npcm_i2c_init_params(bus); + bus->slv_rd_size = 0; + bus->slv_wr_size = 0; + bus->slv_rd_ind = 0; + bus->slv_wr_ind = 0; + if (client->flags & I2C_CLIENT_PEC) + bus->PEC_use = true; + + dev_info(bus->dev, "i2c%d register slave SA=0x%x, PEC=%d\n", bus->num, + client->addr, bus->PEC_use); + + npcm_i2c_slave_enable(bus, I2C_SLAVE_ADDR1, client->addr, true); + npcm_i2c_clear_fifo_int(bus); + npcm_i2c_clear_rx_fifo(bus); + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_slave_int_enable(bus, true); + + spin_unlock_irqrestore(&bus->lock, lock_flags); + return 0; +} + +static int npcm_i2c_unreg_slave(struct i2c_client *client) +{ + struct npcm_i2c *bus = client->adapter->algo_data; + unsigned long lock_flags; + + spin_lock_irqsave(&bus->lock, lock_flags); + if (!bus->slave) { + spin_unlock_irqrestore(&bus->lock, lock_flags); + return -EINVAL; + } + npcm_i2c_slave_int_enable(bus, false); + npcm_i2c_remove_slave_addr(bus, client->addr); + bus->slave = NULL; + spin_unlock_irqrestore(&bus->lock, lock_flags); + return 0; +} +#endif /* CONFIG_I2C_SLAVE */ + +static void npcm_i2c_master_fifo_read(struct npcm_i2c *bus) +{ + int rcount; + int fifo_bytes; + enum i2c_state_ind ind = I2C_MASTER_DONE_IND; + + fifo_bytes = npcm_i2c_fifo_usage(bus); + rcount = bus->rd_size - bus->rd_ind; + + /* + * In order not to change the RX_TRH during transaction (we found that + * this might be problematic if it takes too much time to read the FIFO) + * we read the data in the following way. If the number of bytes to + * read == FIFO Size + C (where C < FIFO Size)then first read C bytes + * and in the next int we read rest of the data. + */ + if (rcount < (2 * I2C_HW_FIFO_SIZE) && rcount > I2C_HW_FIFO_SIZE) + fifo_bytes = rcount - I2C_HW_FIFO_SIZE; + + if (rcount <= fifo_bytes) { + /* last bytes are about to be read - end of tx */ + bus->state = I2C_STOP_PENDING; + bus->stop_ind = ind; + npcm_i2c_eob_int(bus, true); + /* Stop should be set before reading last byte. */ + npcm_i2c_master_stop(bus); + npcm_i2c_read_fifo(bus, fifo_bytes); + } else { + npcm_i2c_read_fifo(bus, fifo_bytes); + rcount = bus->rd_size - bus->rd_ind; + npcm_i2c_set_fifo(bus, rcount, -1); + } +} + +static void npcm_i2c_irq_master_handler_write(struct npcm_i2c *bus) +{ + u16 wcount; + + if (bus->fifo_use) + npcm_i2c_clear_tx_fifo(bus); /* clear the TX fifo status bit */ + + /* Master write operation - last byte handling */ + if (bus->wr_ind == bus->wr_size) { + if (bus->fifo_use && npcm_i2c_fifo_usage(bus) > 0) + /* + * No more bytes to send (to add to the FIFO), + * however the FIFO is not empty yet. It is + * still in the middle of tx. Currently there's nothing + * to do except for waiting to the end of the tx + * We will get an int when the FIFO will get empty. + */ + return; + + if (bus->rd_size == 0) { + /* all bytes have been written, in wr only operation */ + npcm_i2c_eob_int(bus, true); + bus->state = I2C_STOP_PENDING; + bus->stop_ind = I2C_MASTER_DONE_IND; + npcm_i2c_master_stop(bus); + /* Clear SDA Status bit (by writing dummy byte) */ + npcm_i2c_wr_byte(bus, 0xFF); + + } else { + /* last write-byte written on previous int - restart */ + npcm_i2c_set_fifo(bus, bus->rd_size, -1); + /* Generate repeated start upon next write to SDA */ + npcm_i2c_master_start(bus); + + /* + * Receiving one byte only - stall after successful + * completion of send address byte. If we NACK here, and + * slave doesn't ACK the address, we might + * unintentionally NACK the next multi-byte read. + */ + if (bus->rd_size == 1) + npcm_i2c_stall_after_start(bus, true); + + /* Next int will occur on read */ + bus->operation = I2C_READ_OPER; + /* send the slave address in read direction */ + npcm_i2c_wr_byte(bus, bus->dest_addr | 0x1); + } + } else { + /* write next byte not last byte and not slave address */ + if (!bus->fifo_use || bus->wr_size == 1) { + npcm_i2c_wr_byte(bus, bus->wr_buf[bus->wr_ind++]); + } else { + wcount = bus->wr_size - bus->wr_ind; + npcm_i2c_set_fifo(bus, -1, wcount); + if (wcount) + npcm_i2c_write_to_fifo_master(bus, wcount); + } + } +} + +static void npcm_i2c_irq_master_handler_read(struct npcm_i2c *bus) +{ + u16 block_extra_bytes_size; + u8 data; + + /* added bytes to the packet: */ + block_extra_bytes_size = bus->read_block_use + bus->PEC_use; + + /* + * Perform master read, distinguishing between last byte and the rest of + * the bytes. The last byte should be read when the clock is stopped + */ + if (bus->rd_ind == 0) { /* first byte handling: */ + if (bus->read_block_use) { + /* first byte in block protocol is the size: */ + data = npcm_i2c_rd_byte(bus); + data = clamp_val(data, 1, I2C_SMBUS_BLOCK_MAX); + bus->rd_size = data + block_extra_bytes_size; + bus->rd_buf[bus->rd_ind++] = data; + + /* clear RX FIFO interrupt status: */ + if (bus->fifo_use) { + data = ioread8(bus->reg + NPCM_I2CFIF_CTS); + data = data | NPCM_I2CFIF_CTS_RXF_TXE; + iowrite8(data, bus->reg + NPCM_I2CFIF_CTS); + } + + npcm_i2c_set_fifo(bus, bus->rd_size - 1, -1); + npcm_i2c_stall_after_start(bus, false); + } else { + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_master_fifo_read(bus); + } + } else { + if (bus->rd_size == block_extra_bytes_size && + bus->read_block_use) { + bus->state = I2C_STOP_PENDING; + bus->stop_ind = I2C_BLOCK_BYTES_ERR_IND; + bus->cmd_err = -EIO; + npcm_i2c_eob_int(bus, true); + npcm_i2c_master_stop(bus); + npcm_i2c_read_fifo(bus, npcm_i2c_fifo_usage(bus)); + } else { + npcm_i2c_master_fifo_read(bus); + } + } +} + +static void npcm_i2c_irq_handle_nmatch(struct npcm_i2c *bus) +{ + iowrite8(NPCM_I2CST_NMATCH, bus->reg + NPCM_I2CST); + npcm_i2c_nack(bus); + bus->stop_ind = I2C_BUS_ERR_IND; + npcm_i2c_callback(bus, bus->stop_ind, npcm_i2c_get_index(bus)); +} + +/* A NACK has occurred */ +static void npcm_i2c_irq_handle_nack(struct npcm_i2c *bus) +{ + u8 val; + + if (bus->nack_cnt < ULLONG_MAX) + bus->nack_cnt++; + + if (bus->fifo_use) { + /* + * if there are still untransmitted bytes in TX FIFO + * reduce them from wr_ind + */ + if (bus->operation == I2C_WRITE_OPER) + bus->wr_ind -= npcm_i2c_fifo_usage(bus); + + /* clear the FIFO */ + iowrite8(NPCM_I2CFIF_CTS_CLR_FIFO, bus->reg + NPCM_I2CFIF_CTS); + } + + /* In master write operation, got unexpected NACK */ + bus->stop_ind = I2C_NACK_IND; + /* Only current master is allowed to issue Stop Condition */ + if (npcm_i2c_is_master(bus)) { + /* stopping in the middle */ + npcm_i2c_eob_int(bus, false); + npcm_i2c_master_stop(bus); + + /* + * The bus is released from stall only after the SW clears + * NEGACK bit. Then a Stop condition is sent. + */ + npcm_i2c_clear_master_status(bus); + readx_poll_timeout_atomic(ioread8, bus->reg + NPCM_I2CCST, val, + !(val & NPCM_I2CCST_BUSY), 10, 200); + } + bus->state = I2C_IDLE; + + /* + * In Master mode, NACK should be cleared only after STOP. + * In such case, the bus is released from stall only after the + * software clears NACK bit. Then a Stop condition is sent. + */ + npcm_i2c_callback(bus, bus->stop_ind, bus->wr_ind); +} + + /* Master mode: a Bus Error has been identified */ +static void npcm_i2c_irq_handle_ber(struct npcm_i2c *bus) +{ + if (bus->ber_cnt < ULLONG_MAX) + bus->ber_cnt++; + bus->stop_ind = I2C_BUS_ERR_IND; + if (npcm_i2c_is_master(bus)) { + npcm_i2c_master_abort(bus); + } else { + npcm_i2c_clear_master_status(bus); + + /* Clear BB (BUS BUSY) bit */ + iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST); + + bus->cmd_err = -EAGAIN; + npcm_i2c_callback(bus, bus->stop_ind, npcm_i2c_get_index(bus)); + } + bus->state = I2C_IDLE; +} + + /* EOB: a master End Of Busy (meaning STOP completed) */ +static void npcm_i2c_irq_handle_eob(struct npcm_i2c *bus) +{ + npcm_i2c_eob_int(bus, false); + bus->state = I2C_IDLE; + npcm_i2c_callback(bus, bus->stop_ind, bus->rd_ind); +} + +/* Address sent and requested stall occurred (Master mode) */ +static void npcm_i2c_irq_handle_stall_after_start(struct npcm_i2c *bus) +{ + if (npcm_i2c_is_quick(bus)) { + bus->state = I2C_STOP_PENDING; + bus->stop_ind = I2C_MASTER_DONE_IND; + npcm_i2c_eob_int(bus, true); + npcm_i2c_master_stop(bus); + } else if ((bus->rd_size == 1) && !bus->read_block_use) { + /* + * Receiving one byte only - set NACK after ensuring + * slave ACKed the address byte. + */ + npcm_i2c_nack(bus); + } + + /* Reset stall-after-address-byte */ + npcm_i2c_stall_after_start(bus, false); + + /* Clear stall only after setting STOP */ + iowrite8(NPCM_I2CST_STASTR, bus->reg + NPCM_I2CST); +} + +/* SDA status is set - TX or RX, master */ +static void npcm_i2c_irq_handle_sda(struct npcm_i2c *bus, u8 i2cst) +{ + u8 fif_cts; + + if (!npcm_i2c_is_master(bus)) + return; + + if (bus->state == I2C_IDLE) { + bus->stop_ind = I2C_WAKE_UP_IND; + + if (npcm_i2c_is_quick(bus) || bus->read_block_use) + /* + * Need to stall after successful + * completion of sending address byte + */ + npcm_i2c_stall_after_start(bus, true); + else + npcm_i2c_stall_after_start(bus, false); + + /* + * Receiving one byte only - stall after successful completion + * of sending address byte If we NACK here, and slave doesn't + * ACK the address, we might unintentionally NACK the next + * multi-byte read + */ + if (bus->wr_size == 0 && bus->rd_size == 1) + npcm_i2c_stall_after_start(bus, true); + + /* Initiate I2C master tx */ + + /* select bank 1 for FIFO regs */ + npcm_i2c_select_bank(bus, I2C_BANK_1); + + fif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS); + fif_cts = fif_cts & ~NPCM_I2CFIF_CTS_SLVRSTR; + + /* clear FIFO and relevant status bits. */ + fif_cts = fif_cts | NPCM_I2CFIF_CTS_CLR_FIFO; + iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS); + + /* re-enable */ + fif_cts = fif_cts | NPCM_I2CFIF_CTS_RXF_TXE; + iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS); + + /* + * Configure the FIFO threshold: + * according to the needed # of bytes to read. + * Note: due to HW limitation can't config the rx fifo before it + * got and ACK on the restart. LAST bit will not be reset unless + * RX completed. It will stay set on the next tx. + */ + if (bus->wr_size) + npcm_i2c_set_fifo(bus, -1, bus->wr_size); + else + npcm_i2c_set_fifo(bus, bus->rd_size, -1); + + bus->state = I2C_OPER_STARTED; + + if (npcm_i2c_is_quick(bus) || bus->wr_size) + npcm_i2c_wr_byte(bus, bus->dest_addr); + else + npcm_i2c_wr_byte(bus, bus->dest_addr | BIT(0)); + /* SDA interrupt, after start\restart */ + } else { + if (NPCM_I2CST_XMIT & i2cst) { + bus->operation = I2C_WRITE_OPER; + npcm_i2c_irq_master_handler_write(bus); + } else { + bus->operation = I2C_READ_OPER; + npcm_i2c_irq_master_handler_read(bus); + } + } +} + +static int npcm_i2c_int_master_handler(struct npcm_i2c *bus) +{ + u8 i2cst; + int ret = -EIO; + + i2cst = ioread8(bus->reg + NPCM_I2CST); + + if (FIELD_GET(NPCM_I2CST_NMATCH, i2cst)) { + npcm_i2c_irq_handle_nmatch(bus); + return 0; + } + /* A NACK has occurred */ + if (FIELD_GET(NPCM_I2CST_NEGACK, i2cst)) { + npcm_i2c_irq_handle_nack(bus); + return 0; + } + + /* Master mode: a Bus Error has been identified */ + if (FIELD_GET(NPCM_I2CST_BER, i2cst)) { + npcm_i2c_irq_handle_ber(bus); + return 0; + } + + /* EOB: a master End Of Busy (meaning STOP completed) */ + if ((FIELD_GET(NPCM_I2CCTL1_EOBINTE, + ioread8(bus->reg + NPCM_I2CCTL1)) == 1) && + (FIELD_GET(NPCM_I2CCST3_EO_BUSY, + ioread8(bus->reg + NPCM_I2CCST3)))) { + npcm_i2c_irq_handle_eob(bus); + return 0; + } + + /* Address sent and requested stall occurred (Master mode) */ + if (FIELD_GET(NPCM_I2CST_STASTR, i2cst)) { + npcm_i2c_irq_handle_stall_after_start(bus); + ret = 0; + } + + /* SDA status is set - TX or RX, master */ + if (FIELD_GET(NPCM_I2CST_SDAST, i2cst) || + (bus->fifo_use && + (npcm_i2c_tx_fifo_empty(bus) || npcm_i2c_rx_fifo_full(bus)))) { + npcm_i2c_irq_handle_sda(bus, i2cst); + ret = 0; + } + + return ret; +} + +/* recovery using TGCLK functionality of the module */ +static int npcm_i2c_recovery_tgclk(struct i2c_adapter *_adap) +{ + u8 val; + u8 fif_cts; + bool done = false; + int status = -ENOTRECOVERABLE; + struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap); + /* Allow 3 bytes (27 toggles) to be read from the slave: */ + int iter = 27; + + if ((npcm_i2c_get_SDA(_adap) == 1) && (npcm_i2c_get_SCL(_adap) == 1)) { + dev_dbg(bus->dev, "bus%d recovery skipped, bus not stuck", + bus->num); + npcm_i2c_reset(bus); + return status; + } + + npcm_i2c_int_enable(bus, false); + npcm_i2c_disable(bus); + npcm_i2c_enable(bus); + iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST); + npcm_i2c_clear_tx_fifo(bus); + npcm_i2c_clear_rx_fifo(bus); + iowrite8(0, bus->reg + NPCM_I2CRXF_CTL); + iowrite8(0, bus->reg + NPCM_I2CTXF_CTL); + npcm_i2c_stall_after_start(bus, false); + + /* select bank 1 for FIFO regs */ + npcm_i2c_select_bank(bus, I2C_BANK_1); + + /* clear FIFO and relevant status bits. */ + fif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS); + fif_cts &= ~NPCM_I2CFIF_CTS_SLVRSTR; + fif_cts |= NPCM_I2CFIF_CTS_CLR_FIFO; + iowrite8(fif_cts, bus->reg + NPCM_I2CFIF_CTS); + npcm_i2c_set_fifo(bus, -1, 0); + + /* Repeat the following sequence until SDA is released */ + do { + /* Issue a single SCL toggle */ + iowrite8(NPCM_I2CCST_TGSCL, bus->reg + NPCM_I2CCST); + usleep_range(20, 30); + /* If SDA line is inactive (high), stop */ + if (npcm_i2c_get_SDA(_adap)) { + done = true; + status = 0; + } + } while (!done && iter--); + + /* If SDA line is released: send start-addr-stop, to re-sync. */ + if (npcm_i2c_get_SDA(_adap)) { + /* Send an address byte in write direction: */ + npcm_i2c_wr_byte(bus, bus->dest_addr); + npcm_i2c_master_start(bus); + /* Wait until START condition is sent */ + status = readx_poll_timeout(npcm_i2c_get_SCL, _adap, val, !val, + 20, 200); + /* If START condition was sent */ + if (npcm_i2c_is_master(bus) > 0) { + usleep_range(20, 30); + npcm_i2c_master_stop(bus); + usleep_range(200, 500); + } + } + npcm_i2c_reset(bus); + npcm_i2c_int_enable(bus, true); + + if ((npcm_i2c_get_SDA(_adap) == 1) && (npcm_i2c_get_SCL(_adap) == 1)) + status = 0; + else + status = -ENOTRECOVERABLE; + if (status) { + if (bus->rec_fail_cnt < ULLONG_MAX) + bus->rec_fail_cnt++; + } else { + if (bus->rec_succ_cnt < ULLONG_MAX) + bus->rec_succ_cnt++; + } + return status; +} + +/* recovery using bit banging functionality of the module */ +static void npcm_i2c_recovery_init(struct i2c_adapter *_adap) +{ + struct npcm_i2c *bus = container_of(_adap, struct npcm_i2c, adap); + struct i2c_bus_recovery_info *rinfo = &bus->rinfo; + + rinfo->recover_bus = npcm_i2c_recovery_tgclk; + + /* + * npcm i2c HW allows direct reading of SCL and SDA. + * However, it does not support setting SCL and SDA directly. + * The recovery function can togle SCL when SDA is low (but not set) + * Getter functions used internally, and can be used externaly. + */ + rinfo->get_scl = npcm_i2c_get_SCL; + rinfo->get_sda = npcm_i2c_get_SDA; + _adap->bus_recovery_info = rinfo; +} + +/* SCLFRQ min/max field values */ +#define SCLFRQ_MIN 10 +#define SCLFRQ_MAX 511 +#define clk_coef(freq, mul) DIV_ROUND_UP((freq) * (mul), 1000000) + +/* + * npcm_i2c_init_clk: init HW timing parameters. + * NPCM7XX i2c module timing parameters are depenent on module core clk (APB) + * and bus frequency. + * 100kHz bus requires tSCL = 4 * SCLFRQ * tCLK. LT and HT are simetric. + * 400kHz bus requires assymetric HT and LT. A different equation is recomended + * by the HW designer, given core clock range (equations in comments below). + * + */ +static int npcm_i2c_init_clk(struct npcm_i2c *bus, u32 bus_freq_hz) +{ + u32 k1 = 0; + u32 k2 = 0; + u8 dbnct = 0; + u32 sclfrq = 0; + u8 hldt = 7; + u8 fast_mode = 0; + u32 src_clk_khz; + u32 bus_freq_khz; + + src_clk_khz = bus->apb_clk / 1000; + bus_freq_khz = bus_freq_hz / 1000; + bus->bus_freq = bus_freq_hz; + + /* 100KHz and below: */ + if (bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ) { + sclfrq = src_clk_khz / (bus_freq_khz * 4); + + if (sclfrq < SCLFRQ_MIN || sclfrq > SCLFRQ_MAX) + return -EDOM; + + if (src_clk_khz >= 40000) + hldt = 17; + else if (src_clk_khz >= 12500) + hldt = 15; + else + hldt = 7; + } + + /* 400KHz: */ + else if (bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) { + sclfrq = 0; + fast_mode = I2CCTL3_400K_MODE; + + if (src_clk_khz < 7500) + /* 400KHZ cannot be supported for core clock < 7.5MHz */ + return -EDOM; + + else if (src_clk_khz >= 50000) { + k1 = 80; + k2 = 48; + hldt = 12; + dbnct = 7; + } + + /* Master or Slave with frequency > 25MHz */ + else if (src_clk_khz > 25000) { + hldt = clk_coef(src_clk_khz, 300) + 7; + k1 = clk_coef(src_clk_khz, 1600); + k2 = clk_coef(src_clk_khz, 900); + } + } + + /* 1MHz: */ + else if (bus_freq_hz <= I2C_MAX_FAST_MODE_PLUS_FREQ) { + sclfrq = 0; + fast_mode = I2CCTL3_400K_MODE; + + /* 1MHZ cannot be supported for core clock < 24 MHz */ + if (src_clk_khz < 24000) + return -EDOM; + + k1 = clk_coef(src_clk_khz, 620); + k2 = clk_coef(src_clk_khz, 380); + + /* Core clk > 40 MHz */ + if (src_clk_khz > 40000) { + /* + * Set HLDT: + * SDA hold time: (HLDT-7) * T(CLK) >= 120 + * HLDT = 120/T(CLK) + 7 = 120 * FREQ(CLK) + 7 + */ + hldt = clk_coef(src_clk_khz, 120) + 7; + } else { + hldt = 7; + dbnct = 2; + } + } + + /* Frequency larger than 1 MHz is not supported */ + else + return -EINVAL; + + if (bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) { + k1 = round_up(k1, 2); + k2 = round_up(k2 + 1, 2); + if (k1 < SCLFRQ_MIN || k1 > SCLFRQ_MAX || + k2 < SCLFRQ_MIN || k2 > SCLFRQ_MAX) + return -EDOM; + } + + /* write sclfrq value. bits [6:0] are in I2CCTL2 reg */ + iowrite8(FIELD_PREP(I2CCTL2_SCLFRQ6_0, sclfrq & 0x7F), + bus->reg + NPCM_I2CCTL2); + + /* bits [8:7] are in I2CCTL3 reg */ + iowrite8(fast_mode | FIELD_PREP(I2CCTL3_SCLFRQ8_7, (sclfrq >> 7) & 0x3), + bus->reg + NPCM_I2CCTL3); + + /* Select Bank 0 to access NPCM_I2CCTL4/NPCM_I2CCTL5 */ + npcm_i2c_select_bank(bus, I2C_BANK_0); + + if (bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) { + /* + * Set SCL Low/High Time: + * k1 = 2 * SCLLT7-0 -> Low Time = k1 / 2 + * k2 = 2 * SCLLT7-0 -> High Time = k2 / 2 + */ + iowrite8(k1 / 2, bus->reg + NPCM_I2CSCLLT); + iowrite8(k2 / 2, bus->reg + NPCM_I2CSCLHT); + + iowrite8(dbnct, bus->reg + NPCM_I2CCTL5); + } + + iowrite8(hldt, bus->reg + NPCM_I2CCTL4); + + /* Return to Bank 1, and stay there by default: */ + npcm_i2c_select_bank(bus, I2C_BANK_1); + + return 0; +} + +static int npcm_i2c_init_module(struct npcm_i2c *bus, enum i2c_mode mode, + u32 bus_freq_hz) +{ + u8 val; + int ret; + + /* Check whether module already enabled or frequency is out of bounds */ + if ((bus->state != I2C_DISABLE && bus->state != I2C_IDLE) || + bus_freq_hz < I2C_FREQ_MIN_HZ || bus_freq_hz > I2C_FREQ_MAX_HZ) + return -EINVAL; + + npcm_i2c_disable(bus); + + /* Configure FIFO mode : */ + if (FIELD_GET(I2C_VER_FIFO_EN, ioread8(bus->reg + I2C_VER))) { + bus->fifo_use = true; + npcm_i2c_select_bank(bus, I2C_BANK_0); + val = ioread8(bus->reg + NPCM_I2CFIF_CTL); + val |= NPCM_I2CFIF_CTL_FIFO_EN; + iowrite8(val, bus->reg + NPCM_I2CFIF_CTL); + npcm_i2c_select_bank(bus, I2C_BANK_1); + } else { + bus->fifo_use = false; + } + + /* Configure I2C module clock frequency */ + ret = npcm_i2c_init_clk(bus, bus_freq_hz); + if (ret) { + dev_err(bus->dev, "npcm_i2c_init_clk failed\n"); + return ret; + } + + /* Enable module (before configuring CTL1) */ + npcm_i2c_enable(bus); + bus->state = I2C_IDLE; + val = ioread8(bus->reg + NPCM_I2CCTL1); + val = (val | NPCM_I2CCTL1_NMINTE) & ~NPCM_I2CCTL1_RWS; + iowrite8(val, bus->reg + NPCM_I2CCTL1); + + npcm_i2c_int_enable(bus, true); + + npcm_i2c_reset(bus); + + return 0; +} + +static int __npcm_i2c_init(struct npcm_i2c *bus, struct platform_device *pdev) +{ + u32 clk_freq_hz; + int ret; + + /* Initialize the internal data structures */ + bus->state = I2C_DISABLE; + bus->master_or_slave = I2C_SLAVE; + bus->int_time_stamp = 0; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + bus->slave = NULL; +#endif + + ret = device_property_read_u32(&pdev->dev, "clock-frequency", + &clk_freq_hz); + if (ret) { + dev_info(&pdev->dev, "Could not read clock-frequency property"); + clk_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; + } + + ret = npcm_i2c_init_module(bus, I2C_MASTER, clk_freq_hz); + if (ret) { + dev_err(&pdev->dev, "npcm_i2c_init_module failed\n"); + return ret; + } + + return 0; +} + +static irqreturn_t npcm_i2c_bus_irq(int irq, void *dev_id) +{ + struct npcm_i2c *bus = dev_id; + + if (npcm_i2c_is_master(bus)) + bus->master_or_slave = I2C_MASTER; + + if (bus->master_or_slave == I2C_MASTER) { + bus->int_time_stamp = jiffies; + if (!npcm_i2c_int_master_handler(bus)) + return IRQ_HANDLED; + } +#if IS_ENABLED(CONFIG_I2C_SLAVE) + if (bus->slave) { + bus->master_or_slave = I2C_SLAVE; + return npcm_i2c_int_slave_handler(bus); + } +#endif + return IRQ_NONE; +} + +static bool npcm_i2c_master_start_xmit(struct npcm_i2c *bus, + u8 slave_addr, u16 nwrite, u16 nread, + u8 *write_data, u8 *read_data, + bool use_PEC, bool use_read_block) +{ + if (bus->state != I2C_IDLE) { + bus->cmd_err = -EBUSY; + return false; + } + bus->dest_addr = slave_addr << 1; + bus->wr_buf = write_data; + bus->wr_size = nwrite; + bus->wr_ind = 0; + bus->rd_buf = read_data; + bus->rd_size = nread; + bus->rd_ind = 0; + bus->PEC_use = 0; + + /* for tx PEC is appended to buffer from i2c IF. PEC flag is ignored */ + if (nread) + bus->PEC_use = use_PEC; + + bus->read_block_use = use_read_block; + if (nread && !nwrite) + bus->operation = I2C_READ_OPER; + else + bus->operation = I2C_WRITE_OPER; + if (bus->fifo_use) { + u8 i2cfif_cts; + + npcm_i2c_select_bank(bus, I2C_BANK_1); + /* clear FIFO and relevant status bits. */ + i2cfif_cts = ioread8(bus->reg + NPCM_I2CFIF_CTS); + i2cfif_cts &= ~NPCM_I2CFIF_CTS_SLVRSTR; + i2cfif_cts |= NPCM_I2CFIF_CTS_CLR_FIFO; + iowrite8(i2cfif_cts, bus->reg + NPCM_I2CFIF_CTS); + } + + bus->state = I2C_IDLE; + npcm_i2c_stall_after_start(bus, true); + npcm_i2c_master_start(bus); + return true; +} + +static int npcm_i2c_master_xfer(struct i2c_adapter *adap, struct i2c_msg *msgs, + int num) +{ + struct npcm_i2c *bus = container_of(adap, struct npcm_i2c, adap); + struct i2c_msg *msg0, *msg1; + unsigned long time_left, flags; + u16 nwrite, nread; + u8 *write_data, *read_data; + u8 slave_addr; + int timeout; + int ret = 0; + bool read_block = false; + bool read_PEC = false; + u8 bus_busy; + unsigned long timeout_usec; + + if (bus->state == I2C_DISABLE) { + dev_err(bus->dev, "I2C%d module is disabled", bus->num); + return -EINVAL; + } + + msg0 = &msgs[0]; + slave_addr = msg0->addr; + if (msg0->flags & I2C_M_RD) { /* read */ + nwrite = 0; + write_data = NULL; + read_data = msg0->buf; + if (msg0->flags & I2C_M_RECV_LEN) { + nread = 1; + read_block = true; + if (msg0->flags & I2C_CLIENT_PEC) + read_PEC = true; + } else { + nread = msg0->len; + } + } else { /* write */ + nwrite = msg0->len; + write_data = msg0->buf; + nread = 0; + read_data = NULL; + if (num == 2) { + msg1 = &msgs[1]; + read_data = msg1->buf; + if (msg1->flags & I2C_M_RECV_LEN) { + nread = 1; + read_block = true; + if (msg1->flags & I2C_CLIENT_PEC) + read_PEC = true; + } else { + nread = msg1->len; + read_block = false; + } + } + } + + /* Adaptive TimeOut: astimated time in usec + 100% margin */ + timeout_usec = (2 * 10000 / bus->bus_freq) * (2 + nread + nwrite); + timeout = max(msecs_to_jiffies(35), usecs_to_jiffies(timeout_usec)); + if (nwrite >= 32 * 1024 || nread >= 32 * 1024) { + dev_err(bus->dev, "i2c%d buffer too big\n", bus->num); + return -EINVAL; + } + + time_left = jiffies + msecs_to_jiffies(DEFAULT_STALL_COUNT) + 1; + do { + /* + * we must clear slave address immediately when the bus is not + * busy, so we spinlock it, but we don't keep the lock for the + * entire while since it is too long. + */ + spin_lock_irqsave(&bus->lock, flags); + bus_busy = ioread8(bus->reg + NPCM_I2CCST) & NPCM_I2CCST_BB; +#if IS_ENABLED(CONFIG_I2C_SLAVE) + if (!bus_busy && bus->slave) + iowrite8((bus->slave->addr & 0x7F), + bus->reg + NPCM_I2CADDR1); +#endif + spin_unlock_irqrestore(&bus->lock, flags); + + } while (time_is_after_jiffies(time_left) && bus_busy); + + if (bus_busy) { + iowrite8(NPCM_I2CCST_BB, bus->reg + NPCM_I2CCST); + npcm_i2c_reset(bus); + i2c_recover_bus(adap); + return -EAGAIN; + } + + npcm_i2c_init_params(bus); + bus->dest_addr = slave_addr; + bus->msgs = msgs; + bus->msgs_num = num; + bus->cmd_err = 0; + bus->read_block_use = read_block; + + reinit_completion(&bus->cmd_complete); + if (!npcm_i2c_master_start_xmit(bus, slave_addr, nwrite, nread, + write_data, read_data, read_PEC, + read_block)) + ret = -EBUSY; + + if (ret != -EBUSY) { + time_left = wait_for_completion_timeout(&bus->cmd_complete, + timeout); + + if (time_left == 0) { + if (bus->timeout_cnt < ULLONG_MAX) + bus->timeout_cnt++; + if (bus->master_or_slave == I2C_MASTER) { + i2c_recover_bus(adap); + bus->cmd_err = -EIO; + bus->state = I2C_IDLE; + } + } + } + ret = bus->cmd_err; + + /* if there was BER, check if need to recover the bus: */ + if (bus->cmd_err == -EAGAIN) + ret = i2c_recover_bus(adap); + +#if IS_ENABLED(CONFIG_I2C_SLAVE) + /* reenable slave if it was enabled */ + if (bus->slave) + iowrite8((bus->slave->addr & 0x7F) | NPCM_I2CADDR_SAEN, + bus->reg + NPCM_I2CADDR1); +#endif + return bus->cmd_err; +} + +static u32 npcm_i2c_functionality(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | + I2C_FUNC_SMBUS_EMUL | + I2C_FUNC_SMBUS_BLOCK_DATA | + I2C_FUNC_SMBUS_PEC | + I2C_FUNC_SLAVE; +} + +static const struct i2c_adapter_quirks npcm_i2c_quirks = { + .max_read_len = 32768, + .max_write_len = 32768, + .flags = I2C_AQ_COMB_WRITE_THEN_READ, +}; + +static const struct i2c_algorithm npcm_i2c_algo = { + .master_xfer = npcm_i2c_master_xfer, + .functionality = npcm_i2c_functionality, +#if IS_ENABLED(CONFIG_I2C_SLAVE) + .reg_slave = npcm_i2c_reg_slave, + .unreg_slave = npcm_i2c_unreg_slave, +#endif +}; + +/* i2c debugfs directory: used to keep health monitor of i2c devices */ +static struct dentry *npcm_i2c_debugfs_dir; + +static void npcm_i2c_init_debugfs(struct platform_device *pdev, + struct npcm_i2c *bus) +{ + struct dentry *d; + + if (!npcm_i2c_debugfs_dir) + return; + d = debugfs_create_dir(dev_name(&pdev->dev), npcm_i2c_debugfs_dir); + if (IS_ERR_OR_NULL(d)) + return; + debugfs_create_u64("ber_cnt", 0444, d, &bus->ber_cnt); + debugfs_create_u64("nack_cnt", 0444, d, &bus->nack_cnt); + debugfs_create_u64("rec_succ_cnt", 0444, d, &bus->rec_succ_cnt); + debugfs_create_u64("rec_fail_cnt", 0444, d, &bus->rec_fail_cnt); + debugfs_create_u64("timeout_cnt", 0444, d, &bus->timeout_cnt); + + bus->debugfs = d; +} + +static int npcm_i2c_probe_bus(struct platform_device *pdev) +{ + struct npcm_i2c *bus; + struct i2c_adapter *adap; + struct clk *i2c_clk; + static struct regmap *gcr_regmap; + static struct regmap *clk_regmap; + int irq; + int ret; + + bus = devm_kzalloc(&pdev->dev, sizeof(*bus), GFP_KERNEL); + if (!bus) + return -ENOMEM; + + bus->dev = &pdev->dev; + + bus->num = of_alias_get_id(pdev->dev.of_node, "i2c"); + /* core clk must be acquired to calculate module timing settings */ + i2c_clk = devm_clk_get(&pdev->dev, NULL); + if (IS_ERR(i2c_clk)) + return PTR_ERR(i2c_clk); + bus->apb_clk = clk_get_rate(i2c_clk); + + gcr_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-gcr"); + if (IS_ERR(gcr_regmap)) + return PTR_ERR(gcr_regmap); + regmap_write(gcr_regmap, NPCM_I2CSEGCTL, NPCM_I2CSEGCTL_INIT_VAL); + + clk_regmap = syscon_regmap_lookup_by_compatible("nuvoton,npcm750-clk"); + if (IS_ERR(clk_regmap)) + return PTR_ERR(clk_regmap); + + bus->reg = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(bus->reg)) + return PTR_ERR(bus->reg); + + spin_lock_init(&bus->lock); + init_completion(&bus->cmd_complete); + + adap = &bus->adap; + adap->owner = THIS_MODULE; + adap->retries = 3; + adap->timeout = HZ; + adap->algo = &npcm_i2c_algo; + adap->quirks = &npcm_i2c_quirks; + adap->algo_data = bus; + adap->dev.parent = &pdev->dev; + adap->dev.of_node = pdev->dev.of_node; + adap->nr = pdev->id; + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_irq(bus->dev, irq, npcm_i2c_bus_irq, 0, + dev_name(bus->dev), bus); + if (ret) + return ret; + + ret = __npcm_i2c_init(bus, pdev); + if (ret) + return ret; + + npcm_i2c_recovery_init(adap); + + i2c_set_adapdata(adap, bus); + + snprintf(bus->adap.name, sizeof(bus->adap.name), "npcm_i2c_%d", + bus->num); + ret = i2c_add_numbered_adapter(&bus->adap); + if (ret) + return ret; + + platform_set_drvdata(pdev, bus); + npcm_i2c_init_debugfs(pdev, bus); + return 0; +} + +static int npcm_i2c_remove_bus(struct platform_device *pdev) +{ + unsigned long lock_flags; + struct npcm_i2c *bus = platform_get_drvdata(pdev); + + debugfs_remove_recursive(bus->debugfs); + spin_lock_irqsave(&bus->lock, lock_flags); + npcm_i2c_disable(bus); + spin_unlock_irqrestore(&bus->lock, lock_flags); + i2c_del_adapter(&bus->adap); + return 0; +} + +static const struct of_device_id npcm_i2c_bus_of_table[] = { + { .compatible = "nuvoton,npcm750-i2c", }, + {} +}; +MODULE_DEVICE_TABLE(of, npcm_i2c_bus_of_table); + +static struct platform_driver npcm_i2c_bus_driver = { + .probe = npcm_i2c_probe_bus, + .remove = npcm_i2c_remove_bus, + .driver = { + .name = "nuvoton-i2c", + .of_match_table = npcm_i2c_bus_of_table, + } +}; + +static int __init npcm_i2c_init(void) +{ + npcm_i2c_debugfs_dir = debugfs_create_dir("npcm_i2c", NULL); + platform_driver_register(&npcm_i2c_bus_driver); + return 0; +} +module_init(npcm_i2c_init); + +static void __exit npcm_i2c_exit(void) +{ + platform_driver_unregister(&npcm_i2c_bus_driver); + debugfs_remove_recursive(npcm_i2c_debugfs_dir); +} +module_exit(npcm_i2c_exit); + +MODULE_AUTHOR("Avi Fishman <avi.fishman@gmail.com>"); +MODULE_AUTHOR("Tali Perry <tali.perry@nuvoton.com>"); +MODULE_AUTHOR("Tyrone Ting <kfting@nuvoton.com>"); +MODULE_DESCRIPTION("Nuvoton I2C Bus Driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-nvidia-gpu.c b/drivers/i2c/busses/i2c-nvidia-gpu.c index f5d25ce00f03..f480105000b8 100644 --- a/drivers/i2c/busses/i2c-nvidia-gpu.c +++ b/drivers/i2c/busses/i2c-nvidia-gpu.c @@ -277,10 +277,7 @@ static int gpu_populate_client(struct gpu_i2c_dev *i2cd, int irq) i2cd->gpu_ccgx_ucsi->irq = irq; i2cd->gpu_ccgx_ucsi->properties = ccgx_props; i2cd->ccgx_client = i2c_new_client_device(&i2cd->adapter, i2cd->gpu_ccgx_ucsi); - if (IS_ERR(i2cd->ccgx_client)) - return PTR_ERR(i2cd->ccgx_client); - - return 0; + return PTR_ERR_OR_ZERO(i2cd->ccgx_client); } static int gpu_i2c_probe(struct pci_dev *pdev, const struct pci_device_id *id) diff --git a/drivers/i2c/busses/i2c-octeon-platdrv.c b/drivers/i2c/busses/i2c-octeon-platdrv.c index 64bda83e65ac..0c227963c8d6 100644 --- a/drivers/i2c/busses/i2c-octeon-platdrv.c +++ b/drivers/i2c/busses/i2c-octeon-platdrv.c @@ -136,7 +136,6 @@ static int octeon_i2c_probe(struct platform_device *pdev) { struct device_node *node = pdev->dev.of_node; int irq, result = 0, hlc_irq = 0; - struct resource *res_mem; struct octeon_i2c *i2c; bool cn78xx_style; @@ -167,8 +166,7 @@ static int octeon_i2c_probe(struct platform_device *pdev) i2c->roff.twsi_int = 0x10; i2c->roff.sw_twsi_ext = 0x18; - res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->twsi_base = devm_ioremap_resource(&pdev->dev, res_mem); + i2c->twsi_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->twsi_base)) { result = PTR_ERR(i2c->twsi_base); goto out; diff --git a/drivers/i2c/busses/i2c-omap.c b/drivers/i2c/busses/i2c-omap.c index 71b4637c86b7..175c590b93b7 100644 --- a/drivers/i2c/busses/i2c-omap.c +++ b/drivers/i2c/busses/i2c-omap.c @@ -1365,10 +1365,8 @@ omap_i2c_probe(struct platform_device *pdev) u16 minor, major; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "no irq resource?\n"); + if (irq < 0) return irq; - } omap = devm_kzalloc(&pdev->dev, sizeof(struct omap_i2c_dev), GFP_KERNEL); if (!omap) diff --git a/drivers/i2c/busses/i2c-owl.c b/drivers/i2c/busses/i2c-owl.c index 3ab8be62c581..672f1f239bd6 100644 --- a/drivers/i2c/busses/i2c-owl.c +++ b/drivers/i2c/busses/i2c-owl.c @@ -396,23 +396,19 @@ static int owl_i2c_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct owl_i2c_dev *i2c_dev; - struct resource *res; int ret, irq; i2c_dev = devm_kzalloc(dev, sizeof(*i2c_dev), GFP_KERNEL); if (!i2c_dev) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c_dev->base = devm_ioremap_resource(dev, res); + i2c_dev->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c_dev->base)) return PTR_ERR(i2c_dev->base); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(dev, "failed to get IRQ number\n"); + if (irq < 0) return irq; - } if (of_property_read_u32(dev->of_node, "clock-frequency", &i2c_dev->bus_freq)) diff --git a/drivers/i2c/busses/i2c-pca-platform.c b/drivers/i2c/busses/i2c-pca-platform.c index 635dd697ac0b..546426a470cc 100644 --- a/drivers/i2c/busses/i2c-pca-platform.c +++ b/drivers/i2c/busses/i2c-pca-platform.c @@ -149,8 +149,7 @@ static int i2c_pca_pf_probe(struct platform_device *pdev) if (!i2c) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->reg_base = devm_ioremap_resource(&pdev->dev, res); + i2c->reg_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(i2c->reg_base)) return PTR_ERR(i2c->reg_base); diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c index 30ded6422e7b..69740a4ff1db 100644 --- a/drivers/i2c/busses/i2c-piix4.c +++ b/drivers/i2c/busses/i2c-piix4.c @@ -977,7 +977,8 @@ static int piix4_probe(struct pci_dev *dev, const struct pci_device_id *id) } if (dev->vendor == PCI_VENDOR_ID_AMD && - dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS) { + (dev->device == PCI_DEVICE_ID_AMD_HUDSON2_SMBUS || + dev->device == PCI_DEVICE_ID_AMD_KERNCZ_SMBUS)) { retval = piix4_setup_sb800(dev, id, 1); } diff --git a/drivers/i2c/busses/i2c-pnx.c b/drivers/i2c/busses/i2c-pnx.c index 686c06f31625..5d7207c10f1d 100644 --- a/drivers/i2c/busses/i2c-pnx.c +++ b/drivers/i2c/busses/i2c-pnx.c @@ -720,7 +720,6 @@ static int i2c_pnx_probe(struct platform_device *pdev) alg_data->irq = platform_get_irq(pdev, 0); if (alg_data->irq < 0) { - dev_err(&pdev->dev, "Failed to get IRQ from platform resource\n"); ret = alg_data->irq; goto out_clock; } diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c index d565714c1f13..3e38e114948b 100644 --- a/drivers/i2c/busses/i2c-powermac.c +++ b/drivers/i2c/busses/i2c-powermac.c @@ -207,18 +207,18 @@ static u32 i2c_powermac_get_addr(struct i2c_adapter *adap, struct pmac_i2c_bus *bus, struct device_node *node) { - const __be32 *prop; - int len; + u32 prop; + int ret; /* First check for valid "reg" */ - prop = of_get_property(node, "reg", &len); - if (prop && (len >= sizeof(int))) - return (be32_to_cpup(prop) & 0xff) >> 1; + ret = of_property_read_u32(node, "reg", &prop); + if (ret == 0) + return (prop & 0xff) >> 1; /* Then check old-style "i2c-address" */ - prop = of_get_property(node, "i2c-address", &len); - if (prop && (len >= sizeof(int))) - return (be32_to_cpup(prop) & 0xff) >> 1; + ret = of_property_read_u32(node, "i2c-address", &prop); + if (ret == 0) + return (prop & 0xff) >> 1; /* Now handle some devices with missing "reg" properties */ if (of_node_name_eq(node, "cereal")) @@ -315,7 +315,7 @@ static void i2c_powermac_register_devices(struct i2c_adapter *adap, { struct i2c_client *newdev; struct device_node *node; - bool found_onyx = 0; + bool found_onyx = false; /* * In some cases we end up with the via-pmu node itself, in this diff --git a/drivers/i2c/busses/i2c-pxa.c b/drivers/i2c/busses/i2c-pxa.c index 466e4f681d7a..35ca2c02c9b9 100644 --- a/drivers/i2c/busses/i2c-pxa.c +++ b/drivers/i2c/busses/i2c-pxa.c @@ -16,25 +16,120 @@ * Dec 2004: Added support for PXA27x and slave device probing [Liam Girdwood] * Feb 2005: Rework slave mode handling [RMK] */ -#include <linux/kernel.h> -#include <linux/module.h> -#include <linux/i2c.h> -#include <linux/init.h> -#include <linux/time.h> -#include <linux/sched.h> +#include <linux/clk.h> #include <linux/delay.h> +#include <linux/err.h> #include <linux/errno.h> +#include <linux/gpio/consumer.h> +#include <linux/i2c.h> +#include <linux/init.h> #include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> #include <linux/of.h> #include <linux/of_device.h> +#include <linux/pinctrl/consumer.h> #include <linux/platform_device.h> -#include <linux/err.h> -#include <linux/clk.h> -#include <linux/slab.h> -#include <linux/io.h> #include <linux/platform_data/i2c-pxa.h> +#include <linux/slab.h> + +/* I2C register field definitions */ +#define IBMR_SDAS (1 << 0) +#define IBMR_SCLS (1 << 1) + +#define ICR_START (1 << 0) /* start bit */ +#define ICR_STOP (1 << 1) /* stop bit */ +#define ICR_ACKNAK (1 << 2) /* send ACK(0) or NAK(1) */ +#define ICR_TB (1 << 3) /* transfer byte bit */ +#define ICR_MA (1 << 4) /* master abort */ +#define ICR_SCLE (1 << 5) /* master clock enable */ +#define ICR_IUE (1 << 6) /* unit enable */ +#define ICR_GCD (1 << 7) /* general call disable */ +#define ICR_ITEIE (1 << 8) /* enable tx interrupts */ +#define ICR_IRFIE (1 << 9) /* enable rx interrupts */ +#define ICR_BEIE (1 << 10) /* enable bus error ints */ +#define ICR_SSDIE (1 << 11) /* slave STOP detected int enable */ +#define ICR_ALDIE (1 << 12) /* enable arbitration interrupt */ +#define ICR_SADIE (1 << 13) /* slave address detected int enable */ +#define ICR_UR (1 << 14) /* unit reset */ +#define ICR_FM (1 << 15) /* fast mode */ +#define ICR_HS (1 << 16) /* High Speed mode */ +#define ICR_A3700_FM (1 << 16) /* fast mode for armada-3700 */ +#define ICR_A3700_HS (1 << 17) /* high speed mode for armada-3700 */ +#define ICR_GPIOEN (1 << 19) /* enable GPIO mode for SCL in HS */ + +#define ISR_RWM (1 << 0) /* read/write mode */ +#define ISR_ACKNAK (1 << 1) /* ack/nak status */ +#define ISR_UB (1 << 2) /* unit busy */ +#define ISR_IBB (1 << 3) /* bus busy */ +#define ISR_SSD (1 << 4) /* slave stop detected */ +#define ISR_ALD (1 << 5) /* arbitration loss detected */ +#define ISR_ITE (1 << 6) /* tx buffer empty */ +#define ISR_IRF (1 << 7) /* rx buffer full */ +#define ISR_GCAD (1 << 8) /* general call address detected */ +#define ISR_SAD (1 << 9) /* slave address detected */ +#define ISR_BED (1 << 10) /* bus error no ACK/NAK */ + +#define ILCR_SLV_SHIFT 0 +#define ILCR_SLV_MASK (0x1FF << ILCR_SLV_SHIFT) +#define ILCR_FLV_SHIFT 9 +#define ILCR_FLV_MASK (0x1FF << ILCR_FLV_SHIFT) +#define ILCR_HLVL_SHIFT 18 +#define ILCR_HLVL_MASK (0x1FF << ILCR_HLVL_SHIFT) +#define ILCR_HLVH_SHIFT 27 +#define ILCR_HLVH_MASK (0x1F << ILCR_HLVH_SHIFT) + +#define IWCR_CNT_SHIFT 0 +#define IWCR_CNT_MASK (0x1F << IWCR_CNT_SHIFT) +#define IWCR_HS_CNT1_SHIFT 5 +#define IWCR_HS_CNT1_MASK (0x1F << IWCR_HS_CNT1_SHIFT) +#define IWCR_HS_CNT2_SHIFT 10 +#define IWCR_HS_CNT2_MASK (0x1F << IWCR_HS_CNT2_SHIFT) + +/* need a longer timeout if we're dealing with the fact we may well be + * looking at a multi-master environment + */ +#define DEF_TIMEOUT 32 + +#define NO_SLAVE (-ENXIO) +#define BUS_ERROR (-EREMOTEIO) +#define XFER_NAKED (-ECONNREFUSED) +#define I2C_RETRY (-2000) /* an error has occurred retry transmit */ + +/* ICR initialize bit values + * + * 15 FM 0 (100 kHz operation) + * 14 UR 0 (No unit reset) + * 13 SADIE 0 (Disables the unit from interrupting on slave addresses + * matching its slave address) + * 12 ALDIE 0 (Disables the unit from interrupt when it loses arbitration + * in master mode) + * 11 SSDIE 0 (Disables interrupts from a slave stop detected, in slave mode) + * 10 BEIE 1 (Enable interrupts from detected bus errors, no ACK sent) + * 9 IRFIE 1 (Enable interrupts from full buffer received) + * 8 ITEIE 1 (Enables the I2C unit to interrupt when transmit buffer empty) + * 7 GCD 1 (Disables i2c unit response to general call messages as a slave) + * 6 IUE 0 (Disable unit until we change settings) + * 5 SCLE 1 (Enables the i2c clock output for master mode (drives SCL) + * 4 MA 0 (Only send stop with the ICR stop bit) + * 3 TB 0 (We are not transmitting a byte initially) + * 2 ACKNAK 0 (Send an ACK after the unit receives a byte) + * 1 STOP 0 (Do not send a STOP) + * 0 START 0 (Do not send a START) + */ +#define I2C_ICR_INIT (ICR_BEIE | ICR_IRFIE | ICR_ITEIE | ICR_GCD | ICR_SCLE) -#include <asm/irq.h> +/* I2C status register init values + * + * 10 BED 1 (Clear bus error detected) + * 9 SAD 1 (Clear slave address detected) + * 7 IRF 1 (Clear IDBR Receive Full) + * 6 ITE 1 (Clear IDBR Transmit Empty) + * 5 ALD 1 (Clear Arbitration Loss Detected) + * 4 SSD 1 (Clear Slave Stop Detected) + */ +#define I2C_ISR_INIT 0x7FF /* status register init */ struct pxa_reg_layout { u32 ibmr; @@ -56,12 +151,7 @@ enum pxa_i2c_types { REGS_A3700, }; -#define ICR_BUSMODE_FM (1 << 16) /* shifted fast mode for armada-3700 */ -#define ICR_BUSMODE_HS (1 << 17) /* shifted high speed mode for armada-3700 */ - -/* - * I2C registers definitions - */ +/* I2C register layout definitions */ static struct pxa_reg_layout pxa_reg_layout[] = { [REGS_PXA2XX] = { .ibmr = 0x00, @@ -69,6 +159,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = { .icr = 0x10, .isr = 0x18, .isar = 0x20, + .fm = ICR_FM, + .hs = ICR_HS, }, [REGS_PXA3XX] = { .ibmr = 0x00, @@ -76,6 +168,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = { .icr = 0x08, .isr = 0x0c, .isar = 0x10, + .fm = ICR_FM, + .hs = ICR_HS, }, [REGS_CE4100] = { .ibmr = 0x14, @@ -83,6 +177,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = { .icr = 0x00, .isr = 0x04, /* no isar register */ + .fm = ICR_FM, + .hs = ICR_HS, }, [REGS_PXA910] = { .ibmr = 0x00, @@ -92,6 +188,8 @@ static struct pxa_reg_layout pxa_reg_layout[] = { .isar = 0x20, .ilcr = 0x28, .iwcr = 0x30, + .fm = ICR_FM, + .hs = ICR_HS, }, [REGS_A3700] = { .ibmr = 0x00, @@ -99,11 +197,20 @@ static struct pxa_reg_layout pxa_reg_layout[] = { .icr = 0x08, .isr = 0x0c, .isar = 0x10, - .fm = ICR_BUSMODE_FM, - .hs = ICR_BUSMODE_HS, + .fm = ICR_A3700_FM, + .hs = ICR_A3700_HS, }, }; +static const struct of_device_id i2c_pxa_dt_ids[] = { + { .compatible = "mrvl,pxa-i2c", .data = (void *)REGS_PXA2XX }, + { .compatible = "mrvl,pwri2c", .data = (void *)REGS_PXA3XX }, + { .compatible = "mrvl,mmp-twsi", .data = (void *)REGS_PXA910 }, + { .compatible = "marvell,armada-3700-i2c", .data = (void *)REGS_A3700 }, + {} +}; +MODULE_DEVICE_TABLE(of, i2c_pxa_dt_ids); + static const struct platform_device_id i2c_pxa_id_table[] = { { "pxa2xx-i2c", REGS_PXA2XX }, { "pxa3xx-pwri2c", REGS_PXA3XX }, @@ -114,58 +221,6 @@ static const struct platform_device_id i2c_pxa_id_table[] = { }; MODULE_DEVICE_TABLE(platform, i2c_pxa_id_table); -/* - * I2C bit definitions - */ - -#define ICR_START (1 << 0) /* start bit */ -#define ICR_STOP (1 << 1) /* stop bit */ -#define ICR_ACKNAK (1 << 2) /* send ACK(0) or NAK(1) */ -#define ICR_TB (1 << 3) /* transfer byte bit */ -#define ICR_MA (1 << 4) /* master abort */ -#define ICR_SCLE (1 << 5) /* master clock enable */ -#define ICR_IUE (1 << 6) /* unit enable */ -#define ICR_GCD (1 << 7) /* general call disable */ -#define ICR_ITEIE (1 << 8) /* enable tx interrupts */ -#define ICR_IRFIE (1 << 9) /* enable rx interrupts */ -#define ICR_BEIE (1 << 10) /* enable bus error ints */ -#define ICR_SSDIE (1 << 11) /* slave STOP detected int enable */ -#define ICR_ALDIE (1 << 12) /* enable arbitration interrupt */ -#define ICR_SADIE (1 << 13) /* slave address detected int enable */ -#define ICR_UR (1 << 14) /* unit reset */ -#define ICR_FM (1 << 15) /* fast mode */ -#define ICR_HS (1 << 16) /* High Speed mode */ -#define ICR_GPIOEN (1 << 19) /* enable GPIO mode for SCL in HS */ - -#define ISR_RWM (1 << 0) /* read/write mode */ -#define ISR_ACKNAK (1 << 1) /* ack/nak status */ -#define ISR_UB (1 << 2) /* unit busy */ -#define ISR_IBB (1 << 3) /* bus busy */ -#define ISR_SSD (1 << 4) /* slave stop detected */ -#define ISR_ALD (1 << 5) /* arbitration loss detected */ -#define ISR_ITE (1 << 6) /* tx buffer empty */ -#define ISR_IRF (1 << 7) /* rx buffer full */ -#define ISR_GCAD (1 << 8) /* general call address detected */ -#define ISR_SAD (1 << 9) /* slave address detected */ -#define ISR_BED (1 << 10) /* bus error no ACK/NAK */ - -/* bit field shift & mask */ -#define ILCR_SLV_SHIFT 0 -#define ILCR_SLV_MASK (0x1FF << ILCR_SLV_SHIFT) -#define ILCR_FLV_SHIFT 9 -#define ILCR_FLV_MASK (0x1FF << ILCR_FLV_SHIFT) -#define ILCR_HLVL_SHIFT 18 -#define ILCR_HLVL_MASK (0x1FF << ILCR_HLVL_SHIFT) -#define ILCR_HLVH_SHIFT 27 -#define ILCR_HLVH_MASK (0x1F << ILCR_HLVH_SHIFT) - -#define IWCR_CNT_SHIFT 0 -#define IWCR_CNT_MASK (0x1F << IWCR_CNT_SHIFT) -#define IWCR_HS_CNT1_SHIFT 5 -#define IWCR_HS_CNT1_MASK (0x1F << IWCR_HS_CNT1_SHIFT) -#define IWCR_HS_CNT2_SHIFT 10 -#define IWCR_HS_CNT2_MASK (0x1F << IWCR_HS_CNT2_SHIFT) - struct pxa_i2c { spinlock_t lock; wait_queue_head_t wait; @@ -207,6 +262,11 @@ struct pxa_i2c { bool highmode_enter; u32 fm_mask; u32 hs_mask; + + struct i2c_bus_recovery_info recovery; + struct pinctrl *pinctrl; + struct pinctrl_state *pinctrl_default; + struct pinctrl_state *pinctrl_recovery; }; #define _IBMR(i2c) ((i2c)->reg_ibmr) @@ -234,13 +294,14 @@ struct bits { static inline void decode_bits(const char *prefix, const struct bits *bits, int num, u32 val) { - printk("%s %08x: ", prefix, val); + printk("%s %08x:", prefix, val); while (num--) { const char *str = val & bits->mask ? bits->set : bits->unset; if (str) - printk("%s ", str); + pr_cont(" %s", str); bits++; } + pr_cont("\n"); } static const struct bits isr_bits[] = { @@ -260,7 +321,6 @@ static const struct bits isr_bits[] = { static void decode_ISR(unsigned int val) { decode_bits(KERN_DEBUG "ISR", isr_bits, ARRAY_SIZE(isr_bits), val); - printk("\n"); } static const struct bits icr_bits[] = { @@ -285,7 +345,6 @@ static const struct bits icr_bits[] = { static void decode_ICR(unsigned int val) { decode_bits(KERN_DEBUG "ICR", icr_bits, ARRAY_SIZE(icr_bits), val); - printk("\n"); } #endif @@ -311,11 +370,10 @@ static void i2c_pxa_scream_blue_murder(struct pxa_i2c *i2c, const char *why) dev_err(dev, "IBMR: %08x IDBR: %08x ICR: %08x ISR: %08x\n", readl(_IBMR(i2c)), readl(_IDBR(i2c)), readl(_ICR(i2c)), readl(_ISR(i2c))); - dev_dbg(dev, "log: "); + dev_err(dev, "log:"); for (i = 0; i < i2c->irqlogidx; i++) - pr_debug("[%08x:%08x] ", i2c->isrlog[i], i2c->icrlog[i]); - - pr_debug("\n"); + pr_cont(" [%03x:%05x]", i2c->isrlog[i], i2c->icrlog[i]); + pr_cont("\n"); } #else /* ifdef DEBUG */ @@ -330,7 +388,6 @@ static void i2c_pxa_scream_blue_murder(struct pxa_i2c *i2c, const char *why) #endif /* ifdef DEBUG / else */ static void i2c_pxa_master_complete(struct pxa_i2c *i2c, int ret); -static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id); static inline int i2c_pxa_is_slavemode(struct pxa_i2c *i2c) { @@ -346,7 +403,7 @@ static void i2c_pxa_abort(struct pxa_i2c *i2c) return; } - while ((i > 0) && (readl(_IBMR(i2c)) & 0x1) == 0) { + while ((i > 0) && (readl(_IBMR(i2c)) & IBMR_SDAS) == 0) { unsigned long icr = readl(_ICR(i2c)); icr &= ~ICR_START; @@ -367,19 +424,26 @@ static void i2c_pxa_abort(struct pxa_i2c *i2c) static int i2c_pxa_wait_bus_not_busy(struct pxa_i2c *i2c) { int timeout = DEF_TIMEOUT; + u32 isr; + + while (1) { + isr = readl(_ISR(i2c)); + if (!(isr & (ISR_IBB | ISR_UB))) + return 0; - while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB)) { - if ((readl(_ISR(i2c)) & ISR_SAD) != 0) + if (isr & ISR_SAD) timeout += 4; + if (!timeout--) + break; + msleep(2); show_state(i2c); } - if (timeout < 0) - show_state(i2c); + show_state(i2c); - return timeout < 0 ? I2C_RETRY : 0; + return I2C_RETRY; } static int i2c_pxa_wait_master(struct pxa_i2c *i2c) @@ -401,7 +465,8 @@ static int i2c_pxa_wait_master(struct pxa_i2c *i2c) * quick check of the i2c lines themselves to ensure they've * gone high... */ - if ((readl(_ISR(i2c)) & (ISR_UB | ISR_IBB)) == 0 && readl(_IBMR(i2c)) == 3) { + if ((readl(_ISR(i2c)) & (ISR_UB | ISR_IBB)) == 0 && + readl(_IBMR(i2c)) == (IBMR_SCLS | IBMR_SDAS)) { if (i2c_debug > 0) dev_dbg(&i2c->adap.dev, "%s: done\n", __func__); return 1; @@ -501,13 +566,8 @@ static void i2c_pxa_set_slave(struct pxa_i2c *i2c, int errcode) #define i2c_pxa_set_slave(i2c, err) do { } while (0) #endif -static void i2c_pxa_reset(struct pxa_i2c *i2c) +static void i2c_pxa_do_reset(struct pxa_i2c *i2c) { - pr_debug("Resetting I2C Controller Unit\n"); - - /* abort any transfer currently under way */ - i2c_pxa_abort(i2c); - /* reset according to 9.8 */ writel(ICR_UR, _ICR(i2c)); writel(I2C_ISR_INIT, _ISR(i2c)); @@ -526,12 +586,25 @@ static void i2c_pxa_reset(struct pxa_i2c *i2c) #endif i2c_pxa_set_slave(i2c, 0); +} +static void i2c_pxa_enable(struct pxa_i2c *i2c) +{ /* enable unit */ writel(readl(_ICR(i2c)) | ICR_IUE, _ICR(i2c)); udelay(100); } +static void i2c_pxa_reset(struct pxa_i2c *i2c) +{ + pr_debug("Resetting I2C Controller Unit\n"); + + /* abort any transfer currently under way */ + i2c_pxa_abort(i2c); + i2c_pxa_do_reset(i2c); + i2c_pxa_enable(i2c); +} + #ifdef CONFIG_I2C_PXA_SLAVE /* @@ -596,7 +669,7 @@ static void i2c_pxa_slave_start(struct pxa_i2c *i2c, u32 isr) timeout = 0x10000; while (1) { - if ((readl(_IBMR(i2c)) & 2) == 2) + if ((readl(_IBMR(i2c)) & IBMR_SCLS) == IBMR_SCLS) break; timeout--; @@ -691,7 +764,7 @@ static void i2c_pxa_slave_start(struct pxa_i2c *i2c, u32 isr) timeout = 0x10000; while (1) { - if ((readl(_IBMR(i2c)) & 2) == 2) + if ((readl(_IBMR(i2c)) & IBMR_SCLS) == IBMR_SCLS) break; timeout--; @@ -716,16 +789,6 @@ static void i2c_pxa_slave_stop(struct pxa_i2c *i2c) * PXA I2C Master mode */ -static inline unsigned int i2c_pxa_addr_byte(struct i2c_msg *msg) -{ - unsigned int addr = (msg->addr & 0x7f) << 1; - - if (msg->flags & I2C_M_RD) - addr |= 1; - - return addr; -} - static inline void i2c_pxa_start_message(struct pxa_i2c *i2c) { u32 icr; @@ -733,8 +796,8 @@ static inline void i2c_pxa_start_message(struct pxa_i2c *i2c) /* * Step 1: target slave address into IDBR */ - writel(i2c_pxa_addr_byte(i2c->msg), _IDBR(i2c)); - i2c->req_slave_addr = i2c_pxa_addr_byte(i2c->msg); + i2c->req_slave_addr = i2c_8bit_addr_from_msg(i2c->msg); + writel(i2c->req_slave_addr, _IDBR(i2c)); /* * Step 2: initiate the write. @@ -747,42 +810,12 @@ static inline void i2c_pxa_stop_message(struct pxa_i2c *i2c) { u32 icr; - /* - * Clear the STOP and ACK flags - */ + /* Clear the START, STOP, ACK, TB and MA flags */ icr = readl(_ICR(i2c)); - icr &= ~(ICR_STOP | ICR_ACKNAK); + icr &= ~(ICR_START | ICR_STOP | ICR_ACKNAK | ICR_TB | ICR_MA); writel(icr, _ICR(i2c)); } -static int i2c_pxa_pio_set_master(struct pxa_i2c *i2c) -{ - /* make timeout the same as for interrupt based functions */ - long timeout = 2 * DEF_TIMEOUT; - - /* - * Wait for the bus to become free. - */ - while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB)) { - udelay(1000); - show_state(i2c); - } - - if (timeout < 0) { - show_state(i2c); - dev_err(&i2c->adap.dev, - "i2c_pxa: timeout waiting for bus free\n"); - return I2C_RETRY; - } - - /* - * Set master mode. - */ - writel(readl(_ICR(i2c)) | ICR_SCLE, _ICR(i2c)); - - return 0; -} - /* * PXA I2C send master code * 1. Load master code to IDBR and send it. @@ -811,140 +844,6 @@ static int i2c_pxa_send_mastercode(struct pxa_i2c *i2c) return (timeout == 0) ? I2C_RETRY : 0; } -static int i2c_pxa_do_pio_xfer(struct pxa_i2c *i2c, - struct i2c_msg *msg, int num) -{ - unsigned long timeout = 500000; /* 5 seconds */ - int ret = 0; - - ret = i2c_pxa_pio_set_master(i2c); - if (ret) - goto out; - - i2c->msg = msg; - i2c->msg_num = num; - i2c->msg_idx = 0; - i2c->msg_ptr = 0; - i2c->irqlogidx = 0; - - i2c_pxa_start_message(i2c); - - while (i2c->msg_num > 0 && --timeout) { - i2c_pxa_handler(0, i2c); - udelay(10); - } - - i2c_pxa_stop_message(i2c); - - /* - * We place the return code in i2c->msg_idx. - */ - ret = i2c->msg_idx; - -out: - if (timeout == 0) { - i2c_pxa_scream_blue_murder(i2c, "timeout"); - ret = I2C_RETRY; - } - - return ret; -} - -/* - * We are protected by the adapter bus mutex. - */ -static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) -{ - long timeout; - int ret; - - /* - * Wait for the bus to become free. - */ - ret = i2c_pxa_wait_bus_not_busy(i2c); - if (ret) { - dev_err(&i2c->adap.dev, "i2c_pxa: timeout waiting for bus free\n"); - goto out; - } - - /* - * Set master mode. - */ - ret = i2c_pxa_set_master(i2c); - if (ret) { - dev_err(&i2c->adap.dev, "i2c_pxa_set_master: error %d\n", ret); - goto out; - } - - if (i2c->high_mode) { - ret = i2c_pxa_send_mastercode(i2c); - if (ret) { - dev_err(&i2c->adap.dev, "i2c_pxa_send_mastercode timeout\n"); - goto out; - } - } - - spin_lock_irq(&i2c->lock); - - i2c->msg = msg; - i2c->msg_num = num; - i2c->msg_idx = 0; - i2c->msg_ptr = 0; - i2c->irqlogidx = 0; - - i2c_pxa_start_message(i2c); - - spin_unlock_irq(&i2c->lock); - - /* - * The rest of the processing occurs in the interrupt handler. - */ - timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); - i2c_pxa_stop_message(i2c); - - /* - * We place the return code in i2c->msg_idx. - */ - ret = i2c->msg_idx; - - if (!timeout && i2c->msg_num) { - i2c_pxa_scream_blue_murder(i2c, "timeout"); - ret = I2C_RETRY; - } - - out: - return ret; -} - -static int i2c_pxa_pio_xfer(struct i2c_adapter *adap, - struct i2c_msg msgs[], int num) -{ - struct pxa_i2c *i2c = adap->algo_data; - int ret, i; - - /* If the I2C controller is disabled we need to reset it - (probably due to a suspend/resume destroying state). We do - this here as we can then avoid worrying about resuming the - controller before its users. */ - if (!(readl(_ICR(i2c)) & ICR_IUE)) - i2c_pxa_reset(i2c); - - for (i = adap->retries; i >= 0; i--) { - ret = i2c_pxa_do_pio_xfer(i2c, msgs, num); - if (ret != I2C_RETRY) - goto out; - - if (i2c_debug) - dev_dbg(&adap->dev, "Retrying transmission\n"); - udelay(100); - } - i2c_pxa_scream_blue_murder(i2c, "exhausted retries"); - ret = -EREMOTEIO; - out: - i2c_pxa_set_slave(i2c, ret); - return ret; -} - /* * i2c_pxa_master_complete - complete the message and wake up. */ @@ -996,7 +895,7 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr) */ if (isr & ISR_ACKNAK) { if (i2c->msg_ptr == 0 && i2c->msg_idx == 0) - ret = I2C_RETRY; + ret = NO_SLAVE; else ret = XFER_NAKED; } @@ -1047,8 +946,8 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr) /* * Write the next address. */ - writel(i2c_pxa_addr_byte(i2c->msg), _IDBR(i2c)); - i2c->req_slave_addr = i2c_pxa_addr_byte(i2c->msg); + i2c->req_slave_addr = i2c_8bit_addr_from_msg(i2c->msg); + writel(i2c->req_slave_addr, _IDBR(i2c)); /* * And trigger a repeated start, and send the byte. @@ -1056,14 +955,8 @@ static void i2c_pxa_irq_txempty(struct pxa_i2c *i2c, u32 isr) icr &= ~ICR_ALDIE; icr |= ICR_START | ICR_TB; } else { - if (i2c->msg->len == 0) { - /* - * Device probes have a message length of zero - * and need the bus to be reset before it can - * be used again. - */ - i2c_pxa_reset(i2c); - } + if (i2c->msg->len == 0) + icr |= ICR_MA; i2c_pxa_master_complete(i2c, 0); } @@ -1151,28 +1044,108 @@ static irqreturn_t i2c_pxa_handler(int this_irq, void *dev_id) return IRQ_HANDLED; } +/* + * We are protected by the adapter bus mutex. + */ +static int i2c_pxa_do_xfer(struct pxa_i2c *i2c, struct i2c_msg *msg, int num) +{ + long timeout; + int ret; + + /* + * Wait for the bus to become free. + */ + ret = i2c_pxa_wait_bus_not_busy(i2c); + if (ret) { + dev_err(&i2c->adap.dev, "i2c_pxa: timeout waiting for bus free\n"); + i2c_recover_bus(&i2c->adap); + goto out; + } + + /* + * Set master mode. + */ + ret = i2c_pxa_set_master(i2c); + if (ret) { + dev_err(&i2c->adap.dev, "i2c_pxa_set_master: error %d\n", ret); + goto out; + } + + if (i2c->high_mode) { + ret = i2c_pxa_send_mastercode(i2c); + if (ret) { + dev_err(&i2c->adap.dev, "i2c_pxa_send_mastercode timeout\n"); + goto out; + } + } + + spin_lock_irq(&i2c->lock); + + i2c->msg = msg; + i2c->msg_num = num; + i2c->msg_idx = 0; + i2c->msg_ptr = 0; + i2c->irqlogidx = 0; + + i2c_pxa_start_message(i2c); + + spin_unlock_irq(&i2c->lock); + + /* + * The rest of the processing occurs in the interrupt handler. + */ + timeout = wait_event_timeout(i2c->wait, i2c->msg_num == 0, HZ * 5); + i2c_pxa_stop_message(i2c); + + /* + * We place the return code in i2c->msg_idx. + */ + ret = i2c->msg_idx; + + if (!timeout && i2c->msg_num) { + i2c_pxa_scream_blue_murder(i2c, "timeout with active message"); + i2c_recover_bus(&i2c->adap); + ret = I2C_RETRY; + } -static int i2c_pxa_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) + out: + return ret; +} + +static int i2c_pxa_internal_xfer(struct pxa_i2c *i2c, + struct i2c_msg *msgs, int num, + int (*xfer)(struct pxa_i2c *, + struct i2c_msg *, int num)) { - struct pxa_i2c *i2c = adap->algo_data; int ret, i; - for (i = adap->retries; i >= 0; i--) { - ret = i2c_pxa_do_xfer(i2c, msgs, num); - if (ret != I2C_RETRY) + for (i = 0; ; ) { + ret = xfer(i2c, msgs, num); + if (ret != I2C_RETRY && ret != NO_SLAVE) goto out; + if (++i >= i2c->adap.retries) + break; if (i2c_debug) - dev_dbg(&adap->dev, "Retrying transmission\n"); + dev_dbg(&i2c->adap.dev, "Retrying transmission\n"); udelay(100); } - i2c_pxa_scream_blue_murder(i2c, "exhausted retries"); + if (ret != NO_SLAVE) + i2c_pxa_scream_blue_murder(i2c, "exhausted retries"); ret = -EREMOTEIO; out: i2c_pxa_set_slave(i2c, ret); return ret; } +static int i2c_pxa_xfer(struct i2c_adapter *adap, + struct i2c_msg msgs[], int num) +{ + struct pxa_i2c *i2c = adap->algo_data; + + return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_xfer); +} + static u32 i2c_pxa_functionality(struct i2c_adapter *adap) { return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL | @@ -1188,6 +1161,87 @@ static const struct i2c_algorithm i2c_pxa_algorithm = { #endif }; +/* Non-interrupt mode support */ +static int i2c_pxa_pio_set_master(struct pxa_i2c *i2c) +{ + /* make timeout the same as for interrupt based functions */ + long timeout = 2 * DEF_TIMEOUT; + + /* + * Wait for the bus to become free. + */ + while (timeout-- && readl(_ISR(i2c)) & (ISR_IBB | ISR_UB)) + udelay(1000); + + if (timeout < 0) { + show_state(i2c); + dev_err(&i2c->adap.dev, + "i2c_pxa: timeout waiting for bus free (set_master)\n"); + return I2C_RETRY; + } + + /* + * Set master mode. + */ + writel(readl(_ICR(i2c)) | ICR_SCLE, _ICR(i2c)); + + return 0; +} + +static int i2c_pxa_do_pio_xfer(struct pxa_i2c *i2c, + struct i2c_msg *msg, int num) +{ + unsigned long timeout = 500000; /* 5 seconds */ + int ret = 0; + + ret = i2c_pxa_pio_set_master(i2c); + if (ret) + goto out; + + i2c->msg = msg; + i2c->msg_num = num; + i2c->msg_idx = 0; + i2c->msg_ptr = 0; + i2c->irqlogidx = 0; + + i2c_pxa_start_message(i2c); + + while (i2c->msg_num > 0 && --timeout) { + i2c_pxa_handler(0, i2c); + udelay(10); + } + + i2c_pxa_stop_message(i2c); + + /* + * We place the return code in i2c->msg_idx. + */ + ret = i2c->msg_idx; + +out: + if (timeout == 0) { + i2c_pxa_scream_blue_murder(i2c, "timeout (do_pio_xfer)"); + ret = I2C_RETRY; + } + + return ret; +} + +static int i2c_pxa_pio_xfer(struct i2c_adapter *adap, + struct i2c_msg msgs[], int num) +{ + struct pxa_i2c *i2c = adap->algo_data; + + /* If the I2C controller is disabled we need to reset it + (probably due to a suspend/resume destroying state). We do + this here as we can then avoid worrying about resuming the + controller before its users. */ + if (!(readl(_ICR(i2c)) & ICR_IUE)) + i2c_pxa_reset(i2c); + + return i2c_pxa_internal_xfer(i2c, msgs, num, i2c_pxa_do_pio_xfer); +} + static const struct i2c_algorithm i2c_pxa_pio_algorithm = { .master_xfer = i2c_pxa_pio_xfer, .functionality = i2c_pxa_functionality, @@ -1197,15 +1251,6 @@ static const struct i2c_algorithm i2c_pxa_pio_algorithm = { #endif }; -static const struct of_device_id i2c_pxa_dt_ids[] = { - { .compatible = "mrvl,pxa-i2c", .data = (void *)REGS_PXA2XX }, - { .compatible = "mrvl,pwri2c", .data = (void *)REGS_PXA3XX }, - { .compatible = "mrvl,mmp-twsi", .data = (void *)REGS_PXA910 }, - { .compatible = "marvell,armada-3700-i2c", .data = (void *)REGS_A3700 }, - {} -}; -MODULE_DEVICE_TABLE(of, i2c_pxa_dt_ids); - static int i2c_pxa_probe_dt(struct platform_device *pdev, struct pxa_i2c *i2c, enum pxa_i2c_types *i2c_types) { @@ -1249,6 +1294,131 @@ static int i2c_pxa_probe_pdata(struct platform_device *pdev, return 0; } +static void i2c_pxa_prepare_recovery(struct i2c_adapter *adap) +{ + struct pxa_i2c *i2c = adap->algo_data; + u32 ibmr = readl(_IBMR(i2c)); + + /* + * Program the GPIOs to reflect the current I2C bus state while + * we transition to recovery; this avoids glitching the bus. + */ + gpiod_set_value(i2c->recovery.scl_gpiod, ibmr & IBMR_SCLS); + gpiod_set_value(i2c->recovery.sda_gpiod, ibmr & IBMR_SDAS); + + WARN_ON(pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_recovery)); +} + +static void i2c_pxa_unprepare_recovery(struct i2c_adapter *adap) +{ + struct pxa_i2c *i2c = adap->algo_data; + u32 isr; + + /* + * The bus should now be free. Clear up the I2C controller before + * handing control of the bus back to avoid the bus changing state. + */ + isr = readl(_ISR(i2c)); + if (isr & (ISR_UB | ISR_IBB)) { + dev_dbg(&i2c->adap.dev, + "recovery: resetting controller, ISR=0x%08x\n", isr); + i2c_pxa_do_reset(i2c); + } + + WARN_ON(pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_default)); + + dev_dbg(&i2c->adap.dev, "recovery: IBMR 0x%08x ISR 0x%08x\n", + readl(_IBMR(i2c)), readl(_ISR(i2c))); + + i2c_pxa_enable(i2c); +} + +static int i2c_pxa_init_recovery(struct pxa_i2c *i2c) +{ + struct i2c_bus_recovery_info *bri = &i2c->recovery; + struct device *dev = i2c->adap.dev.parent; + + /* + * When slave mode is enabled, we are not the only master on the bus. + * Bus recovery can only be performed when we are the master, which + * we can't be certain of. Therefore, when slave mode is enabled, do + * not configure bus recovery. + */ + if (IS_ENABLED(CONFIG_I2C_PXA_SLAVE)) + return 0; + + i2c->pinctrl = devm_pinctrl_get(dev); + if (PTR_ERR(i2c->pinctrl) == -ENODEV) + i2c->pinctrl = NULL; + if (IS_ERR(i2c->pinctrl)) + return PTR_ERR(i2c->pinctrl); + + if (!i2c->pinctrl) + return 0; + + i2c->pinctrl_default = pinctrl_lookup_state(i2c->pinctrl, + PINCTRL_STATE_DEFAULT); + i2c->pinctrl_recovery = pinctrl_lookup_state(i2c->pinctrl, "recovery"); + + if (IS_ERR(i2c->pinctrl_default) || IS_ERR(i2c->pinctrl_recovery)) { + dev_info(dev, "missing pinmux recovery information: %ld %ld\n", + PTR_ERR(i2c->pinctrl_default), + PTR_ERR(i2c->pinctrl_recovery)); + return 0; + } + + /* + * Claiming GPIOs can influence the pinmux state, and may glitch the + * I2C bus. Do this carefully. + */ + bri->scl_gpiod = devm_gpiod_get(dev, "scl", GPIOD_OUT_HIGH_OPEN_DRAIN); + if (bri->scl_gpiod == ERR_PTR(-EPROBE_DEFER)) + return -EPROBE_DEFER; + if (IS_ERR(bri->scl_gpiod)) { + dev_info(dev, "missing scl gpio recovery information: %pe\n", + bri->scl_gpiod); + return 0; + } + + /* + * We have SCL. Pull SCL low and wait a bit so that SDA glitches + * have no effect. + */ + gpiod_direction_output(bri->scl_gpiod, 0); + udelay(10); + bri->sda_gpiod = devm_gpiod_get(dev, "sda", GPIOD_OUT_HIGH_OPEN_DRAIN); + + /* Wait a bit in case of a SDA glitch, and then release SCL. */ + udelay(10); + gpiod_direction_output(bri->scl_gpiod, 1); + + if (bri->sda_gpiod == ERR_PTR(-EPROBE_DEFER)) + return -EPROBE_DEFER; + + if (IS_ERR(bri->sda_gpiod)) { + dev_info(dev, "missing sda gpio recovery information: %pe\n", + bri->sda_gpiod); + return 0; + } + + bri->prepare_recovery = i2c_pxa_prepare_recovery; + bri->unprepare_recovery = i2c_pxa_unprepare_recovery; + bri->recover_bus = i2c_generic_scl_recovery; + + i2c->adap.bus_recovery_info = bri; + + /* + * Claiming GPIOs can change the pinmux state, which confuses the + * pinctrl since pinctrl's idea of the current setting is unaffected + * by the pinmux change caused by claiming the GPIO. Work around that + * by switching pinctrl to the GPIO state here. We do it this way to + * avoid glitching the I2C bus. + */ + pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_recovery); + + return pinctrl_select_state(i2c->pinctrl, i2c->pinctrl_default); +} + static int i2c_pxa_probe(struct platform_device *dev) { struct i2c_pxa_platform_data *plat = dev_get_platdata(&dev->dev); @@ -1261,19 +1431,28 @@ static int i2c_pxa_probe(struct platform_device *dev) if (!i2c) return -ENOMEM; + /* Default adapter num to device id; i2c_pxa_probe_dt can override. */ + i2c->adap.nr = dev->id; + i2c->adap.owner = THIS_MODULE; + i2c->adap.retries = 5; + i2c->adap.algo_data = i2c; + i2c->adap.dev.parent = &dev->dev; +#ifdef CONFIG_OF + i2c->adap.dev.of_node = dev->dev.of_node; +#endif + res = platform_get_resource(dev, IORESOURCE_MEM, 0); i2c->reg_base = devm_ioremap_resource(&dev->dev, res); if (IS_ERR(i2c->reg_base)) return PTR_ERR(i2c->reg_base); irq = platform_get_irq(dev, 0); - if (irq < 0) { - dev_err(&dev->dev, "no irq resource: %d\n", irq); + if (irq < 0) return irq; - } - /* Default adapter num to device id; i2c_pxa_probe_dt can override. */ - i2c->adap.nr = dev->id; + ret = i2c_pxa_init_recovery(i2c); + if (ret) + return ret; ret = i2c_pxa_probe_dt(dev, i2c, &i2c_type); if (ret > 0) @@ -1281,9 +1460,6 @@ static int i2c_pxa_probe(struct platform_device *dev) if (ret < 0) return ret; - i2c->adap.owner = THIS_MODULE; - i2c->adap.retries = 5; - spin_lock_init(&i2c->lock); init_waitqueue_head(&i2c->wait); @@ -1299,8 +1475,8 @@ static int i2c_pxa_probe(struct platform_device *dev) i2c->reg_idbr = i2c->reg_base + pxa_reg_layout[i2c_type].idbr; i2c->reg_icr = i2c->reg_base + pxa_reg_layout[i2c_type].icr; i2c->reg_isr = i2c->reg_base + pxa_reg_layout[i2c_type].isr; - i2c->fm_mask = pxa_reg_layout[i2c_type].fm ? : ICR_FM; - i2c->hs_mask = pxa_reg_layout[i2c_type].hs ? : ICR_HS; + i2c->fm_mask = pxa_reg_layout[i2c_type].fm; + i2c->hs_mask = pxa_reg_layout[i2c_type].hs; if (i2c_type != REGS_CE4100) i2c->reg_isar = i2c->reg_base + pxa_reg_layout[i2c_type].isar; @@ -1349,12 +1525,6 @@ static int i2c_pxa_probe(struct platform_device *dev) i2c_pxa_reset(i2c); - i2c->adap.algo_data = i2c; - i2c->adap.dev.parent = &dev->dev; -#ifdef CONFIG_OF - i2c->adap.dev.of_node = dev->dev.of_node; -#endif - ret = i2c_add_numbered_adapter(&i2c->adap); if (ret < 0) goto ereqirq; diff --git a/drivers/i2c/busses/i2c-qcom-cci.c b/drivers/i2c/busses/i2c-qcom-cci.c new file mode 100644 index 000000000000..f13735beca58 --- /dev/null +++ b/drivers/i2c/busses/i2c-qcom-cci.c @@ -0,0 +1,791 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (c) 2012-2016, The Linux Foundation. All rights reserved. +// Copyright (c) 2017-20 Linaro Limited. + +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/i2c.h> +#include <linux/io.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#define CCI_HW_VERSION 0x0 +#define CCI_RESET_CMD 0x004 +#define CCI_RESET_CMD_MASK 0x0f73f3f7 +#define CCI_RESET_CMD_M0_MASK 0x000003f1 +#define CCI_RESET_CMD_M1_MASK 0x0003f001 +#define CCI_QUEUE_START 0x008 +#define CCI_HALT_REQ 0x034 +#define CCI_HALT_REQ_I2C_M0_Q0Q1 BIT(0) +#define CCI_HALT_REQ_I2C_M1_Q0Q1 BIT(1) + +#define CCI_I2C_Mm_SCL_CTL(m) (0x100 + 0x100 * (m)) +#define CCI_I2C_Mm_SDA_CTL_0(m) (0x104 + 0x100 * (m)) +#define CCI_I2C_Mm_SDA_CTL_1(m) (0x108 + 0x100 * (m)) +#define CCI_I2C_Mm_SDA_CTL_2(m) (0x10c + 0x100 * (m)) +#define CCI_I2C_Mm_MISC_CTL(m) (0x110 + 0x100 * (m)) + +#define CCI_I2C_Mm_READ_DATA(m) (0x118 + 0x100 * (m)) +#define CCI_I2C_Mm_READ_BUF_LEVEL(m) (0x11c + 0x100 * (m)) +#define CCI_I2C_Mm_Qn_EXEC_WORD_CNT(m, n) (0x300 + 0x200 * (m) + 0x100 * (n)) +#define CCI_I2C_Mm_Qn_CUR_WORD_CNT(m, n) (0x304 + 0x200 * (m) + 0x100 * (n)) +#define CCI_I2C_Mm_Qn_CUR_CMD(m, n) (0x308 + 0x200 * (m) + 0x100 * (n)) +#define CCI_I2C_Mm_Qn_REPORT_STATUS(m, n) (0x30c + 0x200 * (m) + 0x100 * (n)) +#define CCI_I2C_Mm_Qn_LOAD_DATA(m, n) (0x310 + 0x200 * (m) + 0x100 * (n)) + +#define CCI_IRQ_GLOBAL_CLEAR_CMD 0xc00 +#define CCI_IRQ_MASK_0 0xc04 +#define CCI_IRQ_MASK_0_I2C_M0_RD_DONE BIT(0) +#define CCI_IRQ_MASK_0_I2C_M0_Q0_REPORT BIT(4) +#define CCI_IRQ_MASK_0_I2C_M0_Q1_REPORT BIT(8) +#define CCI_IRQ_MASK_0_I2C_M1_RD_DONE BIT(12) +#define CCI_IRQ_MASK_0_I2C_M1_Q0_REPORT BIT(16) +#define CCI_IRQ_MASK_0_I2C_M1_Q1_REPORT BIT(20) +#define CCI_IRQ_MASK_0_RST_DONE_ACK BIT(24) +#define CCI_IRQ_MASK_0_I2C_M0_Q0Q1_HALT_ACK BIT(25) +#define CCI_IRQ_MASK_0_I2C_M1_Q0Q1_HALT_ACK BIT(26) +#define CCI_IRQ_MASK_0_I2C_M0_ERROR 0x18000ee6 +#define CCI_IRQ_MASK_0_I2C_M1_ERROR 0x60ee6000 +#define CCI_IRQ_CLEAR_0 0xc08 +#define CCI_IRQ_STATUS_0 0xc0c +#define CCI_IRQ_STATUS_0_I2C_M0_RD_DONE BIT(0) +#define CCI_IRQ_STATUS_0_I2C_M0_Q0_REPORT BIT(4) +#define CCI_IRQ_STATUS_0_I2C_M0_Q1_REPORT BIT(8) +#define CCI_IRQ_STATUS_0_I2C_M1_RD_DONE BIT(12) +#define CCI_IRQ_STATUS_0_I2C_M1_Q0_REPORT BIT(16) +#define CCI_IRQ_STATUS_0_I2C_M1_Q1_REPORT BIT(20) +#define CCI_IRQ_STATUS_0_RST_DONE_ACK BIT(24) +#define CCI_IRQ_STATUS_0_I2C_M0_Q0Q1_HALT_ACK BIT(25) +#define CCI_IRQ_STATUS_0_I2C_M1_Q0Q1_HALT_ACK BIT(26) +#define CCI_IRQ_STATUS_0_I2C_M0_Q0_NACK_ERR BIT(27) +#define CCI_IRQ_STATUS_0_I2C_M0_Q1_NACK_ERR BIT(28) +#define CCI_IRQ_STATUS_0_I2C_M1_Q0_NACK_ERR BIT(29) +#define CCI_IRQ_STATUS_0_I2C_M1_Q1_NACK_ERR BIT(30) +#define CCI_IRQ_STATUS_0_I2C_M0_ERROR 0x18000ee6 +#define CCI_IRQ_STATUS_0_I2C_M1_ERROR 0x60ee6000 + +#define CCI_TIMEOUT (msecs_to_jiffies(100)) +#define NUM_MASTERS 2 +#define NUM_QUEUES 2 + +/* Max number of resources + 1 for a NULL terminator */ +#define CCI_RES_MAX 6 + +#define CCI_I2C_SET_PARAM 1 +#define CCI_I2C_REPORT 8 +#define CCI_I2C_WRITE 9 +#define CCI_I2C_READ 10 + +#define CCI_I2C_REPORT_IRQ_EN BIT(8) + +enum { + I2C_MODE_STANDARD, + I2C_MODE_FAST, + I2C_MODE_FAST_PLUS, +}; + +enum cci_i2c_queue_t { + QUEUE_0, + QUEUE_1 +}; + +struct hw_params { + u16 thigh; /* HIGH period of the SCL clock in clock ticks */ + u16 tlow; /* LOW period of the SCL clock */ + u16 tsu_sto; /* set-up time for STOP condition */ + u16 tsu_sta; /* set-up time for a repeated START condition */ + u16 thd_dat; /* data hold time */ + u16 thd_sta; /* hold time (repeated) START condition */ + u16 tbuf; /* bus free time between a STOP and START condition */ + u8 scl_stretch_en; + u16 trdhld; + u16 tsp; /* pulse width of spikes suppressed by the input filter */ +}; + +struct cci; + +struct cci_master { + struct i2c_adapter adap; + u16 master; + u8 mode; + int status; + struct completion irq_complete; + struct cci *cci; +}; + +struct cci_data { + unsigned int num_masters; + struct i2c_adapter_quirks quirks; + u16 queue_size[NUM_QUEUES]; + unsigned long cci_clk_rate; + struct hw_params params[3]; +}; + +struct cci { + struct device *dev; + void __iomem *base; + unsigned int irq; + const struct cci_data *data; + struct clk_bulk_data *clocks; + int nclocks; + struct cci_master master[NUM_MASTERS]; +}; + +static irqreturn_t cci_isr(int irq, void *dev) +{ + struct cci *cci = dev; + u32 val, reset = 0; + int ret = IRQ_NONE; + + val = readl(cci->base + CCI_IRQ_STATUS_0); + writel(val, cci->base + CCI_IRQ_CLEAR_0); + writel(0x1, cci->base + CCI_IRQ_GLOBAL_CLEAR_CMD); + + if (val & CCI_IRQ_STATUS_0_RST_DONE_ACK) { + complete(&cci->master[0].irq_complete); + if (cci->master[1].master) + complete(&cci->master[1].irq_complete); + ret = IRQ_HANDLED; + } + + if (val & CCI_IRQ_STATUS_0_I2C_M0_RD_DONE || + val & CCI_IRQ_STATUS_0_I2C_M0_Q0_REPORT || + val & CCI_IRQ_STATUS_0_I2C_M0_Q1_REPORT) { + cci->master[0].status = 0; + complete(&cci->master[0].irq_complete); + ret = IRQ_HANDLED; + } + + if (val & CCI_IRQ_STATUS_0_I2C_M1_RD_DONE || + val & CCI_IRQ_STATUS_0_I2C_M1_Q0_REPORT || + val & CCI_IRQ_STATUS_0_I2C_M1_Q1_REPORT) { + cci->master[1].status = 0; + complete(&cci->master[1].irq_complete); + ret = IRQ_HANDLED; + } + + if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M0_Q0Q1_HALT_ACK)) { + reset = CCI_RESET_CMD_M0_MASK; + ret = IRQ_HANDLED; + } + + if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M1_Q0Q1_HALT_ACK)) { + reset = CCI_RESET_CMD_M1_MASK; + ret = IRQ_HANDLED; + } + + if (unlikely(reset)) + writel(reset, cci->base + CCI_RESET_CMD); + + if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M0_ERROR)) { + if (val & CCI_IRQ_STATUS_0_I2C_M0_Q0_NACK_ERR || + val & CCI_IRQ_STATUS_0_I2C_M0_Q1_NACK_ERR) + cci->master[0].status = -ENXIO; + else + cci->master[0].status = -EIO; + + writel(CCI_HALT_REQ_I2C_M0_Q0Q1, cci->base + CCI_HALT_REQ); + ret = IRQ_HANDLED; + } + + if (unlikely(val & CCI_IRQ_STATUS_0_I2C_M1_ERROR)) { + if (val & CCI_IRQ_STATUS_0_I2C_M1_Q0_NACK_ERR || + val & CCI_IRQ_STATUS_0_I2C_M1_Q1_NACK_ERR) + cci->master[0].status = -ENXIO; + else + cci->master[0].status = -EIO; + + writel(CCI_HALT_REQ_I2C_M1_Q0Q1, cci->base + CCI_HALT_REQ); + ret = IRQ_HANDLED; + } + + return ret; +} + +static int cci_halt(struct cci *cci, u8 master_num) +{ + struct cci_master *master; + u32 val; + + if (master_num >= cci->data->num_masters) { + dev_err(cci->dev, "Unsupported master idx (%u)\n", master_num); + return -EINVAL; + } + + val = BIT(master_num); + master = &cci->master[master_num]; + + reinit_completion(&master->irq_complete); + writel(val, cci->base + CCI_HALT_REQ); + + if (!wait_for_completion_timeout(&master->irq_complete, CCI_TIMEOUT)) { + dev_err(cci->dev, "CCI halt timeout\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static int cci_reset(struct cci *cci) +{ + /* + * we reset the whole controller, here and for implicity use + * master[0].xxx for waiting on it. + */ + reinit_completion(&cci->master[0].irq_complete); + writel(CCI_RESET_CMD_MASK, cci->base + CCI_RESET_CMD); + + if (!wait_for_completion_timeout(&cci->master[0].irq_complete, + CCI_TIMEOUT)) { + dev_err(cci->dev, "CCI reset timeout\n"); + return -ETIMEDOUT; + } + + return 0; +} + +static int cci_init(struct cci *cci) +{ + u32 val = CCI_IRQ_MASK_0_I2C_M0_RD_DONE | + CCI_IRQ_MASK_0_I2C_M0_Q0_REPORT | + CCI_IRQ_MASK_0_I2C_M0_Q1_REPORT | + CCI_IRQ_MASK_0_I2C_M1_RD_DONE | + CCI_IRQ_MASK_0_I2C_M1_Q0_REPORT | + CCI_IRQ_MASK_0_I2C_M1_Q1_REPORT | + CCI_IRQ_MASK_0_RST_DONE_ACK | + CCI_IRQ_MASK_0_I2C_M0_Q0Q1_HALT_ACK | + CCI_IRQ_MASK_0_I2C_M1_Q0Q1_HALT_ACK | + CCI_IRQ_MASK_0_I2C_M0_ERROR | + CCI_IRQ_MASK_0_I2C_M1_ERROR; + int i; + + writel(val, cci->base + CCI_IRQ_MASK_0); + + for (i = 0; i < cci->data->num_masters; i++) { + int mode = cci->master[i].mode; + const struct hw_params *hw; + + if (!cci->master[i].cci) + continue; + + hw = &cci->data->params[mode]; + + val = hw->thigh << 16 | hw->tlow; + writel(val, cci->base + CCI_I2C_Mm_SCL_CTL(i)); + + val = hw->tsu_sto << 16 | hw->tsu_sta; + writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_0(i)); + + val = hw->thd_dat << 16 | hw->thd_sta; + writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_1(i)); + + val = hw->tbuf; + writel(val, cci->base + CCI_I2C_Mm_SDA_CTL_2(i)); + + val = hw->scl_stretch_en << 8 | hw->trdhld << 4 | hw->tsp; + writel(val, cci->base + CCI_I2C_Mm_MISC_CTL(i)); + } + + return 0; +} + +static int cci_run_queue(struct cci *cci, u8 master, u8 queue) +{ + u32 val; + + val = readl(cci->base + CCI_I2C_Mm_Qn_CUR_WORD_CNT(master, queue)); + writel(val, cci->base + CCI_I2C_Mm_Qn_EXEC_WORD_CNT(master, queue)); + + reinit_completion(&cci->master[master].irq_complete); + val = BIT(master * 2 + queue); + writel(val, cci->base + CCI_QUEUE_START); + + if (!wait_for_completion_timeout(&cci->master[master].irq_complete, + CCI_TIMEOUT)) { + dev_err(cci->dev, "master %d queue %d timeout\n", + master, queue); + cci_reset(cci); + cci_init(cci); + return -ETIMEDOUT; + } + + return cci->master[master].status; +} + +static int cci_validate_queue(struct cci *cci, u8 master, u8 queue) +{ + u32 val; + + val = readl(cci->base + CCI_I2C_Mm_Qn_CUR_WORD_CNT(master, queue)); + if (val == cci->data->queue_size[queue]) + return -EINVAL; + + if (!val) + return 0; + + val = CCI_I2C_REPORT | CCI_I2C_REPORT_IRQ_EN; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + + return cci_run_queue(cci, master, queue); +} + +static int cci_i2c_read(struct cci *cci, u16 master, + u16 addr, u8 *buf, u16 len) +{ + u32 val, words_read, words_exp; + u8 queue = QUEUE_1; + int i, index = 0, ret; + bool first = true; + + /* + * Call validate queue to make sure queue is empty before starting. + * This is to avoid overflow / underflow of queue. + */ + ret = cci_validate_queue(cci, master, queue); + if (ret < 0) + return ret; + + val = CCI_I2C_SET_PARAM | (addr & 0x7f) << 4; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + + val = CCI_I2C_READ | len << 4; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + + ret = cci_run_queue(cci, master, queue); + if (ret < 0) + return ret; + + words_read = readl(cci->base + CCI_I2C_Mm_READ_BUF_LEVEL(master)); + words_exp = len / 4 + 1; + if (words_read != words_exp) { + dev_err(cci->dev, "words read = %d, words expected = %d\n", + words_read, words_exp); + return -EIO; + } + + do { + val = readl(cci->base + CCI_I2C_Mm_READ_DATA(master)); + + for (i = 0; i < 4 && index < len; i++) { + if (first) { + /* The LS byte of this register represents the + * first byte read from the slave during a read + * access. + */ + first = false; + continue; + } + buf[index++] = (val >> (i * 8)) & 0xff; + } + } while (--words_read); + + return 0; +} + +static int cci_i2c_write(struct cci *cci, u16 master, + u16 addr, u8 *buf, u16 len) +{ + u8 queue = QUEUE_0; + u8 load[12] = { 0 }; + int i = 0, j, ret; + u32 val; + + /* + * Call validate queue to make sure queue is empty before starting. + * This is to avoid overflow / underflow of queue. + */ + ret = cci_validate_queue(cci, master, queue); + if (ret < 0) + return ret; + + val = CCI_I2C_SET_PARAM | (addr & 0x7f) << 4; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + + load[i++] = CCI_I2C_WRITE | len << 4; + + for (j = 0; j < len; j++) + load[i++] = buf[j]; + + for (j = 0; j < i; j += 4) { + val = load[j]; + val |= load[j + 1] << 8; + val |= load[j + 2] << 16; + val |= load[j + 3] << 24; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + } + + val = CCI_I2C_REPORT | CCI_I2C_REPORT_IRQ_EN; + writel(val, cci->base + CCI_I2C_Mm_Qn_LOAD_DATA(master, queue)); + + return cci_run_queue(cci, master, queue); +} + +static int cci_xfer(struct i2c_adapter *adap, struct i2c_msg msgs[], int num) +{ + struct cci_master *cci_master = i2c_get_adapdata(adap); + struct cci *cci = cci_master->cci; + int i, ret; + + ret = pm_runtime_get_sync(cci->dev); + if (ret < 0) + goto err; + + for (i = 0; i < num; i++) { + if (msgs[i].flags & I2C_M_RD) + ret = cci_i2c_read(cci, cci_master->master, + msgs[i].addr, msgs[i].buf, + msgs[i].len); + else + ret = cci_i2c_write(cci, cci_master->master, + msgs[i].addr, msgs[i].buf, + msgs[i].len); + + if (ret < 0) + break; + } + + if (!ret) + ret = num; + +err: + pm_runtime_mark_last_busy(cci->dev); + pm_runtime_put_autosuspend(cci->dev); + + return ret; +} + +static u32 cci_func(struct i2c_adapter *adap) +{ + return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL; +} + +static const struct i2c_algorithm cci_algo = { + .master_xfer = cci_xfer, + .functionality = cci_func, +}; + +static int cci_enable_clocks(struct cci *cci) +{ + return clk_bulk_prepare_enable(cci->nclocks, cci->clocks); +} + +static void cci_disable_clocks(struct cci *cci) +{ + clk_bulk_disable_unprepare(cci->nclocks, cci->clocks); +} + +static int __maybe_unused cci_suspend_runtime(struct device *dev) +{ + struct cci *cci = dev_get_drvdata(dev); + + cci_disable_clocks(cci); + return 0; +} + +static int __maybe_unused cci_resume_runtime(struct device *dev) +{ + struct cci *cci = dev_get_drvdata(dev); + int ret; + + ret = cci_enable_clocks(cci); + if (ret) + return ret; + + cci_init(cci); + return 0; +} + +static int __maybe_unused cci_suspend(struct device *dev) +{ + if (!pm_runtime_suspended(dev)) + return cci_suspend_runtime(dev); + + return 0; +} + +static int __maybe_unused cci_resume(struct device *dev) +{ + cci_resume_runtime(dev); + pm_runtime_mark_last_busy(dev); + pm_request_autosuspend(dev); + + return 0; +} + +static const struct dev_pm_ops qcom_cci_pm = { + SET_SYSTEM_SLEEP_PM_OPS(cci_suspend, cci_resume) + SET_RUNTIME_PM_OPS(cci_suspend_runtime, cci_resume_runtime, NULL) +}; + +static int cci_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + unsigned long cci_clk_rate = 0; + struct device_node *child; + struct resource *r; + struct cci *cci; + int ret, i; + u32 val; + + cci = devm_kzalloc(dev, sizeof(*cci), GFP_KERNEL); + if (!cci) + return -ENOMEM; + + cci->dev = dev; + platform_set_drvdata(pdev, cci); + cci->data = device_get_match_data(dev); + if (!cci->data) + return -ENOENT; + + for_each_available_child_of_node(dev->of_node, child) { + u32 idx; + + ret = of_property_read_u32(child, "reg", &idx); + if (ret) { + dev_err(dev, "%pOF invalid 'reg' property", child); + continue; + } + + if (idx >= cci->data->num_masters) { + dev_err(dev, "%pOF invalid 'reg' value: %u (max is %u)", + child, idx, cci->data->num_masters - 1); + continue; + } + + cci->master[idx].adap.quirks = &cci->data->quirks; + cci->master[idx].adap.algo = &cci_algo; + cci->master[idx].adap.dev.parent = dev; + cci->master[idx].adap.dev.of_node = child; + cci->master[idx].master = idx; + cci->master[idx].cci = cci; + + i2c_set_adapdata(&cci->master[idx].adap, &cci->master[idx]); + snprintf(cci->master[idx].adap.name, + sizeof(cci->master[idx].adap.name), "Qualcomm-CCI"); + + cci->master[idx].mode = I2C_MODE_STANDARD; + ret = of_property_read_u32(child, "clock-frequency", &val); + if (!ret) { + if (val == 400000) + cci->master[idx].mode = I2C_MODE_FAST; + else if (val == 1000000) + cci->master[idx].mode = I2C_MODE_FAST_PLUS; + } + + init_completion(&cci->master[idx].irq_complete); + } + + /* Memory */ + + r = platform_get_resource(pdev, IORESOURCE_MEM, 0); + cci->base = devm_ioremap_resource(dev, r); + if (IS_ERR(cci->base)) + return PTR_ERR(cci->base); + + /* Clocks */ + + ret = devm_clk_bulk_get_all(dev, &cci->clocks); + if (ret < 1) { + dev_err(dev, "failed to get clocks %d\n", ret); + return ret; + } + cci->nclocks = ret; + + /* Retrieve CCI clock rate */ + for (i = 0; i < cci->nclocks; i++) { + if (!strcmp(cci->clocks[i].id, "cci")) { + cci_clk_rate = clk_get_rate(cci->clocks[i].clk); + break; + } + } + + if (cci_clk_rate != cci->data->cci_clk_rate) { + /* cci clock set by the bootloader or via assigned clock rate + * in DT. + */ + dev_warn(dev, "Found %lu cci clk rate while %lu was expected\n", + cci_clk_rate, cci->data->cci_clk_rate); + } + + ret = cci_enable_clocks(cci); + if (ret < 0) + return ret; + + /* Interrupt */ + + ret = platform_get_irq(pdev, 0); + if (ret < 0) + goto disable_clocks; + cci->irq = ret; + + ret = devm_request_irq(dev, cci->irq, cci_isr, 0, dev_name(dev), cci); + if (ret < 0) { + dev_err(dev, "request_irq failed, ret: %d\n", ret); + goto disable_clocks; + } + + val = readl(cci->base + CCI_HW_VERSION); + dev_dbg(dev, "CCI HW version = 0x%08x", val); + + ret = cci_reset(cci); + if (ret < 0) + goto error; + + ret = cci_init(cci); + if (ret < 0) + goto error; + + for (i = 0; i < cci->data->num_masters; i++) { + if (!cci->master[i].cci) + continue; + + ret = i2c_add_adapter(&cci->master[i].adap); + if (ret < 0) + goto error_i2c; + } + + pm_runtime_set_autosuspend_delay(dev, MSEC_PER_SEC); + pm_runtime_use_autosuspend(dev); + pm_runtime_set_active(dev); + pm_runtime_enable(dev); + + return 0; + +error_i2c: + for (; i >= 0; i--) { + if (cci->master[i].cci) + i2c_del_adapter(&cci->master[i].adap); + } +error: + disable_irq(cci->irq); +disable_clocks: + cci_disable_clocks(cci); + + return ret; +} + +static int cci_remove(struct platform_device *pdev) +{ + struct cci *cci = platform_get_drvdata(pdev); + int i; + + for (i = 0; i < cci->data->num_masters; i++) { + if (cci->master[i].cci) + i2c_del_adapter(&cci->master[i].adap); + cci_halt(cci, i); + } + + disable_irq(cci->irq); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + + return 0; +} + +static const struct cci_data cci_v1_data = { + .num_masters = 1, + .queue_size = { 64, 16 }, + .quirks = { + .max_write_len = 10, + .max_read_len = 12, + }, + .cci_clk_rate = 19200000, + .params[I2C_MODE_STANDARD] = { + .thigh = 78, + .tlow = 114, + .tsu_sto = 28, + .tsu_sta = 28, + .thd_dat = 10, + .thd_sta = 77, + .tbuf = 118, + .scl_stretch_en = 0, + .trdhld = 6, + .tsp = 1 + }, + .params[I2C_MODE_FAST] = { + .thigh = 20, + .tlow = 28, + .tsu_sto = 21, + .tsu_sta = 21, + .thd_dat = 13, + .thd_sta = 18, + .tbuf = 32, + .scl_stretch_en = 0, + .trdhld = 6, + .tsp = 3 + }, +}; + +static const struct cci_data cci_v2_data = { + .num_masters = 2, + .queue_size = { 64, 16 }, + .quirks = { + .max_write_len = 11, + .max_read_len = 12, + }, + .cci_clk_rate = 37500000, + .params[I2C_MODE_STANDARD] = { + .thigh = 201, + .tlow = 174, + .tsu_sto = 204, + .tsu_sta = 231, + .thd_dat = 22, + .thd_sta = 162, + .tbuf = 227, + .scl_stretch_en = 0, + .trdhld = 6, + .tsp = 3 + }, + .params[I2C_MODE_FAST] = { + .thigh = 38, + .tlow = 56, + .tsu_sto = 40, + .tsu_sta = 40, + .thd_dat = 22, + .thd_sta = 35, + .tbuf = 62, + .scl_stretch_en = 0, + .trdhld = 6, + .tsp = 3 + }, + .params[I2C_MODE_FAST_PLUS] = { + .thigh = 16, + .tlow = 22, + .tsu_sto = 17, + .tsu_sta = 18, + .thd_dat = 16, + .thd_sta = 15, + .tbuf = 24, + .scl_stretch_en = 0, + .trdhld = 3, + .tsp = 3 + }, +}; + +static const struct of_device_id cci_dt_match[] = { + { .compatible = "qcom,msm8916-cci", .data = &cci_v1_data}, + { .compatible = "qcom,msm8996-cci", .data = &cci_v2_data}, + { .compatible = "qcom,sdm845-cci", .data = &cci_v2_data}, + {} +}; +MODULE_DEVICE_TABLE(of, cci_dt_match); + +static struct platform_driver qcom_cci_driver = { + .probe = cci_probe, + .remove = cci_remove, + .driver = { + .name = "i2c-qcom-cci", + .of_match_table = cci_dt_match, + .pm = &qcom_cci_pm, + }, +}; + +module_platform_driver(qcom_cci_driver); + +MODULE_DESCRIPTION("Qualcomm Camera Control Interface driver"); +MODULE_AUTHOR("Todor Tomov <todor.tomov@linaro.org>"); +MODULE_AUTHOR("Loic Poulain <loic.poulain@linaro.org>"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/i2c/busses/i2c-qup.c b/drivers/i2c/busses/i2c-qup.c index 748872a9b0fc..fbc04b60cfd1 100644 --- a/drivers/i2c/busses/i2c-qup.c +++ b/drivers/i2c/busses/i2c-qup.c @@ -956,10 +956,8 @@ static void qup_i2c_conf_v1(struct qup_i2c_dev *qup) u32 qup_config = I2C_MINI_CORE | I2C_N_VAL; u32 io_mode = QUP_REPACK_EN; - blk->is_tx_blk_mode = - blk->total_tx_len > qup->out_fifo_sz ? true : false; - blk->is_rx_blk_mode = - blk->total_rx_len > qup->in_fifo_sz ? true : false; + blk->is_tx_blk_mode = blk->total_tx_len > qup->out_fifo_sz; + blk->is_rx_blk_mode = blk->total_rx_len > qup->in_fifo_sz; if (blk->is_tx_blk_mode) { io_mode |= QUP_OUTPUT_BLK_MODE; @@ -1528,9 +1526,9 @@ qup_i2c_determine_mode_v2(struct qup_i2c_dev *qup, qup->use_dma = true; } else { qup->blk.is_tx_blk_mode = max_tx_len > qup->out_fifo_sz - - QUP_MAX_TAGS_LEN ? true : false; + QUP_MAX_TAGS_LEN; qup->blk.is_rx_blk_mode = max_rx_len > qup->in_fifo_sz - - READ_RX_TAGS_LEN ? true : false; + READ_RX_TAGS_LEN; } return 0; @@ -1660,7 +1658,6 @@ static int qup_i2c_probe(struct platform_device *pdev) static const int blk_sizes[] = {4, 16, 32}; struct qup_i2c_dev *qup; unsigned long one_bit_t; - struct resource *res; u32 io_mode, hw_ver, size; int ret, fs_div, hs_div; u32 src_clk_freq = DEFAULT_SRC_CLK; @@ -1757,16 +1754,13 @@ nodma: return -EINVAL; } - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - qup->base = devm_ioremap_resource(qup->dev, res); + qup->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(qup->base)) return PTR_ERR(qup->base); qup->irq = platform_get_irq(pdev, 0); - if (qup->irq < 0) { - dev_err(qup->dev, "No IRQ defined\n"); + if (qup->irq < 0) return qup->irq; - } if (has_acpi_companion(qup->dev)) { ret = device_property_read_u32(qup->dev, diff --git a/drivers/i2c/busses/i2c-rcar.c b/drivers/i2c/busses/i2c-rcar.c index 3b5397aa4ca6..a45c4bf1ec01 100644 --- a/drivers/i2c/busses/i2c-rcar.c +++ b/drivers/i2c/busses/i2c-rcar.c @@ -938,9 +938,7 @@ static int rcar_i2c_probe(struct platform_device *pdev) return PTR_ERR(priv->clk); } - priv->res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - - priv->io = devm_ioremap_resource(dev, priv->res); + priv->io = devm_platform_get_and_ioremap_resource(pdev, 0, &priv->res); if (IS_ERR(priv->io)) return PTR_ERR(priv->io); diff --git a/drivers/i2c/busses/i2c-rk3x.c b/drivers/i2c/busses/i2c-rk3x.c index 73272d4296bb..bc698240c4aa 100644 --- a/drivers/i2c/busses/i2c-rk3x.c +++ b/drivers/i2c/busses/i2c-rk3x.c @@ -1193,7 +1193,6 @@ static int rk3x_i2c_probe(struct platform_device *pdev) struct device_node *np = pdev->dev.of_node; const struct of_device_id *match; struct rk3x_i2c *i2c; - struct resource *mem; int ret = 0; int bus_nr; u32 value; @@ -1223,8 +1222,7 @@ static int rk3x_i2c_probe(struct platform_device *pdev) spin_lock_init(&i2c->lock); init_waitqueue_head(&i2c->wait); - mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->regs = devm_ioremap_resource(&pdev->dev, mem); + i2c->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->regs)) return PTR_ERR(i2c->regs); @@ -1262,10 +1260,8 @@ static int rk3x_i2c_probe(struct platform_device *pdev) /* IRQ setup */ irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "cannot find rk3x IRQ\n"); + if (irq < 0) return irq; - } ret = devm_request_irq(&pdev->dev, irq, rk3x_i2c_irq, 0, dev_name(&pdev->dev), i2c); diff --git a/drivers/i2c/busses/i2c-s3c2410.c b/drivers/i2c/busses/i2c-s3c2410.c index 57986984a90b..3eafe0eb3e4c 100644 --- a/drivers/i2c/busses/i2c-s3c2410.c +++ b/drivers/i2c/busses/i2c-s3c2410.c @@ -1266,5 +1266,5 @@ static void __exit i2c_adap_s3c_exit(void) module_exit(i2c_adap_s3c_exit); MODULE_DESCRIPTION("S3C24XX I2C Bus driver"); -MODULE_AUTHOR("Ben Dooks, <ben@simtec.co.uk>"); +MODULE_AUTHOR("Ben Dooks <ben@simtec.co.uk>"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/busses/i2c-sh_mobile.c b/drivers/i2c/busses/i2c-sh_mobile.c index d83ca4028fa0..2cca1b21e26e 100644 --- a/drivers/i2c/busses/i2c-sh_mobile.c +++ b/drivers/i2c/busses/i2c-sh_mobile.c @@ -366,7 +366,6 @@ static int sh_mobile_i2c_isr_tx(struct sh_mobile_i2c_data *pd) static int sh_mobile_i2c_isr_rx(struct sh_mobile_i2c_data *pd) { - unsigned char data; int real_pos; /* switch from TX (address) to RX (data) adds two interrupts */ @@ -387,13 +386,11 @@ static int sh_mobile_i2c_isr_rx(struct sh_mobile_i2c_data *pd) if (real_pos < 0) i2c_op(pd, OP_RX_STOP); else - data = i2c_op(pd, OP_RX_STOP_DATA); + pd->msg->buf[real_pos] = i2c_op(pd, OP_RX_STOP_DATA); } else if (real_pos >= 0) { - data = i2c_op(pd, OP_RX); + pd->msg->buf[real_pos] = i2c_op(pd, OP_RX); } - if (real_pos >= 0) - pd->msg->buf[real_pos] = data; done: pd->pos++; return pd->pos == (pd->msg->len + 2); diff --git a/drivers/i2c/busses/i2c-sirf.c b/drivers/i2c/busses/i2c-sirf.c index a459e00c6851..d7f72ec331e8 100644 --- a/drivers/i2c/busses/i2c-sirf.c +++ b/drivers/i2c/busses/i2c-sirf.c @@ -271,7 +271,6 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev) { struct sirfsoc_i2c *siic; struct i2c_adapter *adap; - struct resource *mem_res; struct clk *clk; int bitrate; int ctrl_speed; @@ -309,8 +308,7 @@ static int i2c_sirfsoc_probe(struct platform_device *pdev) adap = &siic->adapter; adap->class = I2C_CLASS_DEPRECATED; - mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - siic->base = devm_ioremap_resource(&pdev->dev, mem_res); + siic->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(siic->base)) { err = PTR_ERR(siic->base); goto out; diff --git a/drivers/i2c/busses/i2c-sprd.c b/drivers/i2c/busses/i2c-sprd.c index 123a42bfe3b1..19cda6742423 100644 --- a/drivers/i2c/busses/i2c-sprd.c +++ b/drivers/i2c/busses/i2c-sprd.c @@ -492,10 +492,8 @@ static int sprd_i2c_probe(struct platform_device *pdev) return PTR_ERR(i2c_dev->base); i2c_dev->irq = platform_get_irq(pdev, 0); - if (i2c_dev->irq < 0) { - dev_err(&pdev->dev, "failed to get irq resource\n"); + if (i2c_dev->irq < 0) return i2c_dev->irq; - } i2c_set_adapdata(&i2c_dev->adap, i2c_dev); init_completion(&i2c_dev->complete); diff --git a/drivers/i2c/busses/i2c-stm32.c b/drivers/i2c/busses/i2c-stm32.c index 1da347e6a358..3f69a3bb6119 100644 --- a/drivers/i2c/busses/i2c-stm32.c +++ b/drivers/i2c/busses/i2c-stm32.c @@ -25,8 +25,9 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev, /* Request and configure I2C TX dma channel */ dma->chan_tx = dma_request_chan(dev, "tx"); if (IS_ERR(dma->chan_tx)) { - dev_dbg(dev, "can't request DMA tx channel\n"); ret = PTR_ERR(dma->chan_tx); + if (ret != -EPROBE_DEFER) + dev_err(dev, "can't request DMA tx channel\n"); goto fail_al; } @@ -44,8 +45,10 @@ struct stm32_i2c_dma *stm32_i2c_dma_request(struct device *dev, /* Request and configure I2C RX dma channel */ dma->chan_rx = dma_request_chan(dev, "rx"); if (IS_ERR(dma->chan_rx)) { - dev_err(dev, "can't request DMA rx channel\n"); ret = PTR_ERR(dma->chan_rx); + if (ret != -EPROBE_DEFER) + dev_err(dev, "can't request DMA rx channel\n"); + goto fail_tx; } @@ -73,7 +76,8 @@ fail_tx: dma_release_channel(dma->chan_tx); fail_al: devm_kfree(dev, dma); - dev_info(dev, "can't use DMA\n"); + if (ret != -EPROBE_DEFER) + dev_info(dev, "can't use DMA\n"); return ERR_PTR(ret); } diff --git a/drivers/i2c/busses/i2c-stm32f4.c b/drivers/i2c/busses/i2c-stm32f4.c index d6a69dfcac3f..48e269284369 100644 --- a/drivers/i2c/busses/i2c-stm32f4.c +++ b/drivers/i2c/busses/i2c-stm32f4.c @@ -797,8 +797,10 @@ static int stm32f4_i2c_probe(struct platform_device *pdev) rst = devm_reset_control_get_exclusive(&pdev->dev, NULL); if (IS_ERR(rst)) { - dev_err(&pdev->dev, "Error: Missing controller reset\n"); ret = PTR_ERR(rst); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Error: Missing reset ctrl\n"); + goto clk_free; } reset_control_assert(rst); diff --git a/drivers/i2c/busses/i2c-stm32f7.c b/drivers/i2c/busses/i2c-stm32f7.c index 330ffed011e0..bff3479fe122 100644 --- a/drivers/i2c/busses/i2c-stm32f7.c +++ b/drivers/i2c/busses/i2c-stm32f7.c @@ -189,8 +189,6 @@ struct stm32f7_i2c_regs { /** * struct stm32f7_i2c_spec - private i2c specification timing * @rate: I2C bus speed (Hz) - * @rate_min: 80% of I2C bus speed (Hz) - * @rate_max: 100% of I2C bus speed (Hz) * @fall_max: Max fall time of both SDA and SCL signals (ns) * @rise_max: Max rise time of both SDA and SCL signals (ns) * @hddat_min: Min data hold time (ns) @@ -201,8 +199,6 @@ struct stm32f7_i2c_regs { */ struct stm32f7_i2c_spec { u32 rate; - u32 rate_min; - u32 rate_max; u32 fall_max; u32 rise_max; u32 hddat_min; @@ -214,7 +210,6 @@ struct stm32f7_i2c_spec { /** * struct stm32f7_i2c_setup - private I2C timing setup parameters - * @speed: I2C speed mode (standard, Fast Plus) * @speed_freq: I2C speed frequency (Hz) * @clock_src: I2C clock source frequency (Hz) * @rise_time: Rise time (ns) @@ -224,7 +219,6 @@ struct stm32f7_i2c_spec { * @fmp_clr_offset: Fast Mode Plus clear register offset from set register */ struct stm32f7_i2c_setup { - enum stm32_i2c_speed speed; u32 speed_freq; u32 clock_src; u32 rise_time; @@ -287,7 +281,7 @@ struct stm32f7_i2c_msg { * @base: virtual memory area * @complete: completion of I2C message * @clk: hw i2c clock - * @speed: I2C clock frequency of the controller. Standard, Fast or Fast+ + * @bus_rate: I2C clock frequency of the controller * @msg: Pointer to data to be written * @msg_num: number of I2C messages to be executed * @msg_id: message identifiant @@ -314,7 +308,7 @@ struct stm32f7_i2c_dev { void __iomem *base; struct completion complete; struct clk *clk; - int speed; + unsigned int bus_rate; struct i2c_msg *msg; unsigned int msg_num; unsigned int msg_id; @@ -342,11 +336,9 @@ struct stm32f7_i2c_dev { * Table10. Characteristics of the SDA and SCL bus lines for Standard, Fast, * and Fast-mode Plus I2C-bus devices */ -static struct stm32f7_i2c_spec i2c_specs[] = { - [STM32_I2C_SPEED_STANDARD] = { +static struct stm32f7_i2c_spec stm32f7_i2c_specs[] = { + { .rate = I2C_MAX_STANDARD_MODE_FREQ, - .rate_min = I2C_MAX_STANDARD_MODE_FREQ * 8 / 10, /* 80% */ - .rate_max = I2C_MAX_STANDARD_MODE_FREQ, .fall_max = 300, .rise_max = 1000, .hddat_min = 0, @@ -355,10 +347,8 @@ static struct stm32f7_i2c_spec i2c_specs[] = { .l_min = 4700, .h_min = 4000, }, - [STM32_I2C_SPEED_FAST] = { + { .rate = I2C_MAX_FAST_MODE_FREQ, - .rate_min = I2C_MAX_FAST_MODE_FREQ * 8 / 10, /* 80% */ - .rate_max = I2C_MAX_FAST_MODE_FREQ, .fall_max = 300, .rise_max = 300, .hddat_min = 0, @@ -367,10 +357,8 @@ static struct stm32f7_i2c_spec i2c_specs[] = { .l_min = 1300, .h_min = 600, }, - [STM32_I2C_SPEED_FAST_PLUS] = { + { .rate = I2C_MAX_FAST_MODE_PLUS_FREQ, - .rate_min = I2C_MAX_FAST_MODE_PLUS_FREQ * 8 / 10, /* 80% */ - .rate_max = I2C_MAX_FAST_MODE_PLUS_FREQ, .fall_max = 100, .rise_max = 120, .hddat_min = 0, @@ -411,10 +399,23 @@ static void stm32f7_i2c_disable_irq(struct stm32f7_i2c_dev *i2c_dev, u32 mask) stm32f7_i2c_clr_bits(i2c_dev->base + STM32F7_I2C_CR1, mask); } +static struct stm32f7_i2c_spec *stm32f7_get_specs(u32 rate) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(stm32f7_i2c_specs); i++) + if (rate <= stm32f7_i2c_specs[i].rate) + return &stm32f7_i2c_specs[i]; + + return ERR_PTR(-EINVAL); +} + +#define RATE_MIN(rate) ((rate) * 8 / 10) static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, struct stm32f7_i2c_setup *setup, struct stm32f7_i2c_timings *output) { + struct stm32f7_i2c_spec *specs; u32 p_prev = STM32F7_PRESC_MAX; u32 i2cclk = DIV_ROUND_CLOSEST(NSEC_PER_SEC, setup->clock_src); @@ -432,18 +433,19 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, u16 p, l, a, h; int ret = 0; - if (setup->speed >= STM32_I2C_SPEED_END) { - dev_err(i2c_dev->dev, "speed out of bound {%d/%d}\n", - setup->speed, STM32_I2C_SPEED_END - 1); + specs = stm32f7_get_specs(setup->speed_freq); + if (specs == ERR_PTR(-EINVAL)) { + dev_err(i2c_dev->dev, "speed out of bound {%d}\n", + setup->speed_freq); return -EINVAL; } - if ((setup->rise_time > i2c_specs[setup->speed].rise_max) || - (setup->fall_time > i2c_specs[setup->speed].fall_max)) { + if ((setup->rise_time > specs->rise_max) || + (setup->fall_time > specs->fall_max)) { dev_err(i2c_dev->dev, "timings out of bound Rise{%d>%d}/Fall{%d>%d}\n", - setup->rise_time, i2c_specs[setup->speed].rise_max, - setup->fall_time, i2c_specs[setup->speed].fall_max); + setup->rise_time, specs->rise_max, + setup->fall_time, specs->fall_max); return -EINVAL; } @@ -454,12 +456,6 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, return -EINVAL; } - if (setup->speed_freq > i2c_specs[setup->speed].rate) { - dev_err(i2c_dev->dev, "ERROR: Freq {%d/%d}\n", - setup->speed_freq, i2c_specs[setup->speed].rate); - return -EINVAL; - } - /* Analog and Digital Filters */ af_delay_min = (setup->analog_filter ? @@ -469,13 +465,13 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, STM32F7_I2C_ANALOG_FILTER_DELAY_MAX : 0); dnf_delay = setup->dnf * i2cclk; - sdadel_min = i2c_specs[setup->speed].hddat_min + setup->fall_time - + sdadel_min = specs->hddat_min + setup->fall_time - af_delay_min - (setup->dnf + 3) * i2cclk; - sdadel_max = i2c_specs[setup->speed].vddat_max - setup->rise_time - + sdadel_max = specs->vddat_max - setup->rise_time - af_delay_max - (setup->dnf + 4) * i2cclk; - scldel_min = setup->rise_time + i2c_specs[setup->speed].sudat_min; + scldel_min = setup->rise_time + specs->sudat_min; if (sdadel_min < 0) sdadel_min = 0; @@ -530,8 +526,8 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, tsync = af_delay_min + dnf_delay + (2 * i2cclk); s = NULL; - clk_max = NSEC_PER_SEC / i2c_specs[setup->speed].rate_min; - clk_min = NSEC_PER_SEC / i2c_specs[setup->speed].rate_max; + clk_max = NSEC_PER_SEC / RATE_MIN(setup->speed_freq); + clk_min = NSEC_PER_SEC / setup->speed_freq; /* * Among Prescaler possibilities discovered above figures out SCL Low @@ -549,7 +545,7 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, for (l = 0; l < STM32F7_SCLL_MAX; l++) { u32 tscl_l = (l + 1) * prescaler + tsync; - if ((tscl_l < i2c_specs[setup->speed].l_min) || + if ((tscl_l < specs->l_min) || (i2cclk >= ((tscl_l - af_delay_min - dnf_delay) / 4))) { continue; @@ -561,7 +557,7 @@ static int stm32f7_i2c_compute_timing(struct stm32f7_i2c_dev *i2c_dev, setup->rise_time + setup->fall_time; if ((tscl >= clk_min) && (tscl <= clk_max) && - (tscl_h >= i2c_specs[setup->speed].h_min) && + (tscl_h >= specs->h_min) && (i2cclk < tscl_h)) { int clk_error = tscl - i2cbus; @@ -607,6 +603,17 @@ exit: return ret; } +static u32 stm32f7_get_lower_rate(u32 rate) +{ + int i = ARRAY_SIZE(stm32f7_i2c_specs); + + while (--i) + if (stm32f7_i2c_specs[i].rate < rate) + break; + + return stm32f7_i2c_specs[i].rate; +} + static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev, struct stm32f7_i2c_setup *setup) { @@ -619,18 +626,15 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev, i2c_parse_fw_timings(i2c_dev->dev, t, false); - if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_PLUS_FREQ) - i2c_dev->speed = STM32_I2C_SPEED_FAST_PLUS; - else if (t->bus_freq_hz >= I2C_MAX_FAST_MODE_FREQ) - i2c_dev->speed = STM32_I2C_SPEED_FAST; - else - i2c_dev->speed = STM32_I2C_SPEED_STANDARD; + if (t->bus_freq_hz > I2C_MAX_FAST_MODE_PLUS_FREQ) { + dev_err(i2c_dev->dev, "Invalid bus speed (%i>%i)\n", + t->bus_freq_hz, I2C_MAX_FAST_MODE_PLUS_FREQ); + return -EINVAL; + } + setup->speed_freq = t->bus_freq_hz; i2c_dev->setup.rise_time = t->scl_rise_ns; i2c_dev->setup.fall_time = t->scl_fall_ns; - - setup->speed = i2c_dev->speed; - setup->speed_freq = i2c_specs[setup->speed].rate; setup->clock_src = clk_get_rate(i2c_dev->clk); if (!setup->clock_src) { @@ -644,17 +648,13 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev, if (ret) { dev_err(i2c_dev->dev, "failed to compute I2C timings.\n"); - if (i2c_dev->speed > STM32_I2C_SPEED_STANDARD) { - i2c_dev->speed--; - setup->speed = i2c_dev->speed; - setup->speed_freq = - i2c_specs[setup->speed].rate; - dev_warn(i2c_dev->dev, - "downgrade I2C Speed Freq to (%i)\n", - i2c_specs[setup->speed].rate); - } else { + if (setup->speed_freq <= I2C_MAX_STANDARD_MODE_FREQ) break; - } + setup->speed_freq = + stm32f7_get_lower_rate(setup->speed_freq); + dev_warn(i2c_dev->dev, + "downgrade I2C Speed Freq to (%i)\n", + setup->speed_freq); } } while (ret); @@ -663,13 +663,15 @@ static int stm32f7_i2c_setup_timing(struct stm32f7_i2c_dev *i2c_dev, return ret; } - dev_dbg(i2c_dev->dev, "I2C Speed(%i), Freq(%i), Clk Source(%i)\n", - setup->speed, setup->speed_freq, setup->clock_src); + dev_dbg(i2c_dev->dev, "I2C Speed(%i), Clk Source(%i)\n", + setup->speed_freq, setup->clock_src); dev_dbg(i2c_dev->dev, "I2C Rise(%i) and Fall(%i) Time\n", setup->rise_time, setup->fall_time); dev_dbg(i2c_dev->dev, "I2C Analog Filter(%s), DNF(%i)\n", (setup->analog_filter ? "On" : "Off"), setup->dnf); + i2c_dev->bus_rate = setup->speed_freq; + return 0; } @@ -1462,7 +1464,8 @@ static irqreturn_t stm32f7_i2c_isr_event(int irq, void *data) /* NACK received */ if (status & STM32F7_I2C_ISR_NACKF) { - dev_dbg(i2c_dev->dev, "<%s>: Receive NACK\n", __func__); + dev_dbg(i2c_dev->dev, "<%s>: Receive NACK (addr %x)\n", + __func__, f7_msg->addr); writel_relaxed(STM32F7_I2C_ICR_NACKCF, base + STM32F7_I2C_ICR); f7_msg->result = -ENXIO; } @@ -1866,7 +1869,7 @@ static int stm32f7_i2c_write_fm_plus_bits(struct stm32f7_i2c_dev *i2c_dev, { int ret; - if (i2c_dev->speed != STM32_I2C_SPEED_FAST_PLUS || + if (i2c_dev->bus_rate <= I2C_MAX_FAST_MODE_FREQ || IS_ERR_OR_NULL(i2c_dev->regmap)) /* Optional */ return 0; @@ -1940,8 +1943,7 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) if (!i2c_dev) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c_dev->base = devm_ioremap_resource(&pdev->dev, res); + i2c_dev->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); if (IS_ERR(i2c_dev->base)) return PTR_ERR(i2c_dev->base); phy_addr = (dma_addr_t)res->start; @@ -1967,7 +1969,8 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) i2c_dev->clk = devm_clk_get(&pdev->dev, NULL); if (IS_ERR(i2c_dev->clk)) { - dev_err(&pdev->dev, "Error: Missing controller clock\n"); + if (PTR_ERR(i2c_dev->clk) != -EPROBE_DEFER) + dev_err(&pdev->dev, "Failed to get controller clock\n"); return PTR_ERR(i2c_dev->clk); } @@ -1979,8 +1982,10 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) rst = devm_reset_control_get(&pdev->dev, NULL); if (IS_ERR(rst)) { - dev_err(&pdev->dev, "Error: Missing controller reset\n"); ret = PTR_ERR(rst); + if (ret != -EPROBE_DEFER) + dev_err(&pdev->dev, "Error: Missing reset ctrl\n"); + goto clk_free; } reset_control_assert(rst); @@ -2020,7 +2025,8 @@ static int stm32f7_i2c_probe(struct platform_device *pdev) if (ret) goto clk_free; - if (i2c_dev->speed == STM32_I2C_SPEED_FAST_PLUS) { + /* Setup Fast mode plus if necessary */ + if (i2c_dev->bus_rate > I2C_MAX_FAST_MODE_FREQ) { ret = stm32f7_i2c_setup_fm_plus_bits(pdev, i2c_dev); if (ret) goto clk_free; diff --git a/drivers/i2c/busses/i2c-stu300.c b/drivers/i2c/busses/i2c-stu300.c index ba6b60caa45e..64d739baf480 100644 --- a/drivers/i2c/busses/i2c-stu300.c +++ b/drivers/i2c/busses/i2c-stu300.c @@ -860,7 +860,6 @@ static int stu300_probe(struct platform_device *pdev) { struct stu300_dev *dev; struct i2c_adapter *adap; - struct resource *res; int bus_nr; int ret = 0; @@ -876,8 +875,7 @@ static int stu300_probe(struct platform_device *pdev) } dev->pdev = pdev; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - dev->virtbase = devm_ioremap_resource(&pdev->dev, res); + dev->virtbase = devm_platform_ioremap_resource(pdev, 0); dev_dbg(&pdev->dev, "initialize bus device I2C%d on virtual " "base %p\n", bus_nr, dev->virtbase); if (IS_ERR(dev->virtbase)) diff --git a/drivers/i2c/busses/i2c-sun6i-p2wi.c b/drivers/i2c/busses/i2c-sun6i-p2wi.c index e5293f0b3318..2f6f6468214d 100644 --- a/drivers/i2c/busses/i2c-sun6i-p2wi.c +++ b/drivers/i2c/busses/i2c-sun6i-p2wi.c @@ -187,7 +187,6 @@ static int p2wi_probe(struct platform_device *pdev) struct device_node *childnp; unsigned long parent_clk_freq; u32 clk_freq = I2C_MAX_STANDARD_MODE_FREQ; - struct resource *r; struct p2wi *p2wi; u32 slave_addr; int clk_div; @@ -231,17 +230,14 @@ static int p2wi_probe(struct platform_device *pdev) p2wi->slave_addr = slave_addr; } - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - p2wi->regs = devm_ioremap_resource(dev, r); + p2wi->regs = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(p2wi->regs)) return PTR_ERR(p2wi->regs); strlcpy(p2wi->adapter.name, pdev->name, sizeof(p2wi->adapter.name)); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(dev, "failed to retrieve irq: %d\n", irq); + if (irq < 0) return irq; - } p2wi->clk = devm_clk_get(dev, NULL); if (IS_ERR(p2wi->clk)) { diff --git a/drivers/i2c/busses/i2c-synquacer.c b/drivers/i2c/busses/i2c-synquacer.c index 9099d0a67ace..c9a3dba6a75d 100644 --- a/drivers/i2c/busses/i2c-synquacer.c +++ b/drivers/i2c/busses/i2c-synquacer.c @@ -536,7 +536,6 @@ static const struct i2c_adapter synquacer_i2c_ops = { static int synquacer_i2c_probe(struct platform_device *pdev) { struct synquacer_i2c *i2c; - struct resource *r; u32 bus_speed; int ret; @@ -574,16 +573,13 @@ static int synquacer_i2c_probe(struct platform_device *pdev) return -EINVAL; } - r = platform_get_resource(pdev, IORESOURCE_MEM, 0); - i2c->base = devm_ioremap_resource(&pdev->dev, r); + i2c->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(i2c->base)) return PTR_ERR(i2c->base); i2c->irq = platform_get_irq(pdev, 0); - if (i2c->irq < 0) { - dev_err(&pdev->dev, "no IRQ resource found\n"); + if (i2c->irq < 0) return -ENODEV; - } ret = devm_request_irq(&pdev->dev, i2c->irq, synquacer_i2c_isr, 0, dev_name(&pdev->dev), i2c); diff --git a/drivers/i2c/busses/i2c-tegra.c b/drivers/i2c/busses/i2c-tegra.c index 4c4d17ddc96b..15772964a05f 100644 --- a/drivers/i2c/busses/i2c-tegra.c +++ b/drivers/i2c/busses/i2c-tegra.c @@ -6,6 +6,7 @@ * Author: Colin Cross <ccross@android.com> */ +#include <linux/bitfield.h> #include <linux/clk.h> #include <linux/delay.h> #include <linux/dmaengine.h> @@ -29,16 +30,17 @@ #define BYTES_PER_FIFO_WORD 4 #define I2C_CNFG 0x000 -#define I2C_CNFG_DEBOUNCE_CNT_SHIFT 12 +#define I2C_CNFG_DEBOUNCE_CNT GENMASK(14, 12) #define I2C_CNFG_PACKET_MODE_EN BIT(10) #define I2C_CNFG_NEW_MASTER_FSM BIT(11) #define I2C_CNFG_MULTI_MASTER_MODE BIT(17) -#define I2C_STATUS 0x01C +#define I2C_STATUS 0x01c #define I2C_SL_CNFG 0x020 #define I2C_SL_CNFG_NACK BIT(1) #define I2C_SL_CNFG_NEWSL BIT(2) #define I2C_SL_ADDR1 0x02c #define I2C_SL_ADDR2 0x030 +#define I2C_TLOW_SEXT 0x034 #define I2C_TX_FIFO 0x050 #define I2C_RX_FIFO 0x054 #define I2C_PACKET_TRANSFER_STATUS 0x058 @@ -48,10 +50,8 @@ #define I2C_FIFO_CONTROL_TX_TRIG(x) (((x) - 1) << 5) #define I2C_FIFO_CONTROL_RX_TRIG(x) (((x) - 1) << 2) #define I2C_FIFO_STATUS 0x060 -#define I2C_FIFO_STATUS_TX_MASK 0xF0 -#define I2C_FIFO_STATUS_TX_SHIFT 4 -#define I2C_FIFO_STATUS_RX_MASK 0x0F -#define I2C_FIFO_STATUS_RX_SHIFT 0 +#define I2C_FIFO_STATUS_TX GENMASK(7, 4) +#define I2C_FIFO_STATUS_RX GENMASK(3, 0) #define I2C_INT_MASK 0x064 #define I2C_INT_STATUS 0x068 #define I2C_INT_BUS_CLR_DONE BIT(11) @@ -61,7 +61,8 @@ #define I2C_INT_TX_FIFO_DATA_REQ BIT(1) #define I2C_INT_RX_FIFO_DATA_REQ BIT(0) #define I2C_CLK_DIVISOR 0x06c -#define I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT 16 +#define I2C_CLK_DIVISOR_STD_FAST_MODE GENMASK(31, 16) +#define I2C_CLK_DIVISOR_HSMODE GENMASK(15, 0) #define DVC_CTRL_REG1 0x000 #define DVC_CTRL_REG1_INTR_EN BIT(10) @@ -77,10 +78,11 @@ #define I2C_ERR_UNKNOWN_INTERRUPT BIT(2) #define I2C_ERR_RX_BUFFER_OVERFLOW BIT(3) -#define PACKET_HEADER0_HEADER_SIZE_SHIFT 28 -#define PACKET_HEADER0_PACKET_ID_SHIFT 16 -#define PACKET_HEADER0_CONT_ID_SHIFT 12 -#define PACKET_HEADER0_PROTOCOL_I2C BIT(4) +#define PACKET_HEADER0_HEADER_SIZE GENMASK(29, 28) +#define PACKET_HEADER0_PACKET_ID GENMASK(23, 16) +#define PACKET_HEADER0_CONT_ID GENMASK(15, 12) +#define PACKET_HEADER0_PROTOCOL GENMASK(7, 4) +#define PACKET_HEADER0_PROTOCOL_I2C 1 #define I2C_HEADER_CONT_ON_NAK BIT(21) #define I2C_HEADER_READ BIT(19) @@ -91,21 +93,35 @@ #define I2C_HEADER_SLAVE_ADDR_SHIFT 1 #define I2C_BUS_CLEAR_CNFG 0x084 -#define I2C_BC_SCLK_THRESHOLD 9 -#define I2C_BC_SCLK_THRESHOLD_SHIFT 16 +#define I2C_BC_SCLK_THRESHOLD GENMASK(23, 16) #define I2C_BC_STOP_COND BIT(2) #define I2C_BC_TERMINATE BIT(1) #define I2C_BC_ENABLE BIT(0) #define I2C_BUS_CLEAR_STATUS 0x088 #define I2C_BC_STATUS BIT(0) -#define I2C_CONFIG_LOAD 0x08C +#define I2C_CONFIG_LOAD 0x08c #define I2C_MSTR_CONFIG_LOAD BIT(0) #define I2C_CLKEN_OVERRIDE 0x090 #define I2C_MST_CORE_CLKEN_OVR BIT(0) -#define I2C_CONFIG_LOAD_TIMEOUT 1000000 +#define I2C_INTERFACE_TIMING_0 0x094 +#define I2C_INTERFACE_TIMING_THIGH GENMASK(13, 8) +#define I2C_INTERFACE_TIMING_TLOW GENMASK(5, 0) +#define I2C_INTERFACE_TIMING_1 0x098 +#define I2C_INTERFACE_TIMING_TBUF GENMASK(29, 24) +#define I2C_INTERFACE_TIMING_TSU_STO GENMASK(21, 16) +#define I2C_INTERFACE_TIMING_THD_STA GENMASK(13, 8) +#define I2C_INTERFACE_TIMING_TSU_STA GENMASK(5, 0) + +#define I2C_HS_INTERFACE_TIMING_0 0x09c +#define I2C_HS_INTERFACE_TIMING_THIGH GENMASK(13, 8) +#define I2C_HS_INTERFACE_TIMING_TLOW GENMASK(5, 0) +#define I2C_HS_INTERFACE_TIMING_1 0x0a0 +#define I2C_HS_INTERFACE_TIMING_TSU_STO GENMASK(21, 16) +#define I2C_HS_INTERFACE_TIMING_THD_STA GENMASK(13, 8) +#define I2C_HS_INTERFACE_TIMING_TSU_STA GENMASK(5, 0) #define I2C_MST_FIFO_CONTROL 0x0b4 #define I2C_MST_FIFO_CONTROL_RX_FLUSH BIT(0) @@ -114,14 +130,11 @@ #define I2C_MST_FIFO_CONTROL_TX_TRIG(x) (((x) - 1) << 16) #define I2C_MST_FIFO_STATUS 0x0b8 -#define I2C_MST_FIFO_STATUS_RX_MASK 0xff -#define I2C_MST_FIFO_STATUS_RX_SHIFT 0 -#define I2C_MST_FIFO_STATUS_TX_MASK 0xff0000 -#define I2C_MST_FIFO_STATUS_TX_SHIFT 16 +#define I2C_MST_FIFO_STATUS_TX GENMASK(23, 16) +#define I2C_MST_FIFO_STATUS_RX GENMASK(7, 0) -#define I2C_INTERFACE_TIMING_0 0x94 -#define I2C_THIGH_SHIFT 8 -#define I2C_INTERFACE_TIMING_1 0x98 +/* configuration load timeout in microseconds */ +#define I2C_CONFIG_LOAD_TIMEOUT 1000000 /* Packet header size in bytes */ #define I2C_PACKET_HEADER_SIZE 12 @@ -230,6 +243,7 @@ struct tegra_i2c_hw_feature { * @cont_id: I2C controller ID, used for packet header * @irq: IRQ number of transfer complete interrupt * @is_dvc: identifies the DVC I2C controller, has a different register layout + * @is_vi: identifies the VI I2C controller, has a different register layout * @msg_complete: transfer completion notifier * @msg_err: error code for completed message * @msg_buf: pointer to current message data @@ -253,12 +267,14 @@ struct tegra_i2c_dev { struct i2c_adapter adapter; struct clk *div_clk; struct clk *fast_clk; + struct clk *slow_clk; struct reset_control *rst; void __iomem *base; phys_addr_t base_phys; int cont_id; int irq; int is_dvc; + bool is_vi; struct completion msg_complete; int msg_err; u8 *msg_buf; @@ -297,6 +313,8 @@ static unsigned long tegra_i2c_reg_addr(struct tegra_i2c_dev *i2c_dev, { if (i2c_dev->is_dvc) reg += (reg >= I2C_TX_FIFO) ? 0x10 : 0x40; + else if (i2c_dev->is_vi) + reg = 0xc00 + (reg << 2); return reg; } @@ -495,12 +513,10 @@ static int tegra_i2c_empty_rx_fifo(struct tegra_i2c_dev *i2c_dev) if (i2c_dev->hw->has_mst_fifo) { val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); - rx_fifo_avail = (val & I2C_MST_FIFO_STATUS_RX_MASK) >> - I2C_MST_FIFO_STATUS_RX_SHIFT; + rx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_RX, val); } else { val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); - rx_fifo_avail = (val & I2C_FIFO_STATUS_RX_MASK) >> - I2C_FIFO_STATUS_RX_SHIFT; + rx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_RX, val); } /* Rounds down to not include partial word at the end of buf */ @@ -551,12 +567,10 @@ static int tegra_i2c_fill_tx_fifo(struct tegra_i2c_dev *i2c_dev) if (i2c_dev->hw->has_mst_fifo) { val = i2c_readl(i2c_dev, I2C_MST_FIFO_STATUS); - tx_fifo_avail = (val & I2C_MST_FIFO_STATUS_TX_MASK) >> - I2C_MST_FIFO_STATUS_TX_SHIFT; + tx_fifo_avail = FIELD_GET(I2C_MST_FIFO_STATUS_TX, val); } else { val = i2c_readl(i2c_dev, I2C_FIFO_STATUS); - tx_fifo_avail = (val & I2C_FIFO_STATUS_TX_MASK) >> - I2C_FIFO_STATUS_TX_SHIFT; + tx_fifo_avail = FIELD_GET(I2C_FIFO_STATUS_TX, val); } /* Rounds down to not include partial word at the end of buf */ @@ -650,6 +664,14 @@ static int __maybe_unused tegra_i2c_runtime_resume(struct device *dev) } } + if (i2c_dev->slow_clk) { + ret = clk_enable(i2c_dev->slow_clk); + if (ret < 0) { + dev_err(dev, "failed to enable slow clock: %d\n", ret); + return ret; + } + } + ret = clk_enable(i2c_dev->div_clk); if (ret < 0) { dev_err(i2c_dev->dev, @@ -666,6 +688,10 @@ static int __maybe_unused tegra_i2c_runtime_suspend(struct device *dev) struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); clk_disable(i2c_dev->div_clk); + + if (i2c_dev->slow_clk) + clk_disable(i2c_dev->slow_clk); + if (!i2c_dev->hw->has_single_clk_source) clk_disable(i2c_dev->fast_clk); @@ -703,6 +729,35 @@ static int tegra_i2c_wait_for_config_load(struct tegra_i2c_dev *i2c_dev) return 0; } +static void tegra_i2c_vi_init(struct tegra_i2c_dev *i2c_dev) +{ + u32 value; + + value = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, 2) | + FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, 4); + i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_0); + + value = FIELD_PREP(I2C_INTERFACE_TIMING_TBUF, 4) | + FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STO, 7) | + FIELD_PREP(I2C_INTERFACE_TIMING_THD_STA, 4) | + FIELD_PREP(I2C_INTERFACE_TIMING_TSU_STA, 4); + i2c_writel(i2c_dev, value, I2C_INTERFACE_TIMING_1); + + value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_THIGH, 3) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_TLOW, 8); + i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_0); + + value = FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STO, 11) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_THD_STA, 11) | + FIELD_PREP(I2C_HS_INTERFACE_TIMING_TSU_STA, 11); + i2c_writel(i2c_dev, value, I2C_HS_INTERFACE_TIMING_1); + + value = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND; + i2c_writel(i2c_dev, value, I2C_BUS_CLEAR_CNFG); + + i2c_writel(i2c_dev, 0x0, I2C_TLOW_SEXT); +} + static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) { u32 val; @@ -719,7 +774,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) tegra_dvc_init(i2c_dev); val = I2C_CNFG_NEW_MASTER_FSM | I2C_CNFG_PACKET_MODE_EN | - (0x2 << I2C_CNFG_DEBOUNCE_CNT_SHIFT); + FIELD_PREP(I2C_CNFG_DEBOUNCE_CNT, 2); if (i2c_dev->hw->has_multi_master_mode) val |= I2C_CNFG_MULTI_MASTER_MODE; @@ -727,10 +782,14 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) i2c_writel(i2c_dev, val, I2C_CNFG); i2c_writel(i2c_dev, 0, I2C_INT_MASK); + if (i2c_dev->is_vi) + tegra_i2c_vi_init(i2c_dev); + /* Make sure clock divisor programmed correctly */ - clk_divisor = i2c_dev->hw->clk_divisor_hs_mode; - clk_divisor |= i2c_dev->clk_divisor_non_hs_mode << - I2C_CLK_DIVISOR_STD_FAST_MODE_SHIFT; + clk_divisor = FIELD_PREP(I2C_CLK_DIVISOR_HSMODE, + i2c_dev->hw->clk_divisor_hs_mode) | + FIELD_PREP(I2C_CLK_DIVISOR_STD_FAST_MODE, + i2c_dev->clk_divisor_non_hs_mode); i2c_writel(i2c_dev, clk_divisor, I2C_CLK_DIVISOR); if (i2c_dev->bus_clk_rate > I2C_MAX_STANDARD_MODE_FREQ && @@ -745,7 +804,8 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) } if (i2c_dev->hw->has_interface_timing_reg) { - val = (thigh << I2C_THIGH_SHIFT) | tlow; + val = FIELD_PREP(I2C_INTERFACE_TIMING_THIGH, thigh) | + FIELD_PREP(I2C_INTERFACE_TIMING_TLOW, tlow); i2c_writel(i2c_dev, val, I2C_INTERFACE_TIMING_0); } @@ -768,7 +828,7 @@ static int tegra_i2c_init(struct tegra_i2c_dev *i2c_dev, bool clk_reinit) } } - if (!i2c_dev->is_dvc) { + if (!i2c_dev->is_dvc && !i2c_dev->is_vi) { u32 sl_cfg = i2c_readl(i2c_dev, I2C_SL_CNFG); sl_cfg |= I2C_SL_CNFG_NACK | I2C_SL_CNFG_NEWSL; @@ -996,14 +1056,13 @@ tegra_i2c_poll_completion_timeout(struct tegra_i2c_dev *i2c_dev, do { u32 status = i2c_readl(i2c_dev, I2C_INT_STATUS); - if (status) { + if (status) tegra_i2c_isr(i2c_dev->irq, i2c_dev); - if (completion_done(complete)) { - s64 delta = ktime_ms_delta(ktimeout, ktime); + if (completion_done(complete)) { + s64 delta = ktime_ms_delta(ktimeout, ktime); - return msecs_to_jiffies(delta) ?: 1; - } + return msecs_to_jiffies(delta) ?: 1; } ktime = ktime_get(); @@ -1030,14 +1089,18 @@ tegra_i2c_wait_completion_timeout(struct tegra_i2c_dev *i2c_dev, disable_irq(i2c_dev->irq); /* - * There is a chance that completion may happen after IRQ - * synchronization, which is done by disable_irq(). + * Under some rare circumstances (like running KASAN + + * NFS root) CPU, which handles interrupt, may stuck in + * uninterruptible state for a significant time. In this + * case we will get timeout if I2C transfer is running on + * a sibling CPU, despite of IRQ being raised. + * + * In order to handle this rare condition, the IRQ status + * needs to be checked after timeout. */ - if (ret == 0 && completion_done(complete)) { - dev_warn(i2c_dev->dev, - "completion done after timeout\n"); - ret = 1; - } + if (ret == 0) + ret = tegra_i2c_poll_completion_timeout(i2c_dev, + complete, 0); } return ret; @@ -1051,8 +1114,8 @@ static int tegra_i2c_issue_bus_clear(struct i2c_adapter *adap) u32 reg; reinit_completion(&i2c_dev->msg_complete); - reg = (I2C_BC_SCLK_THRESHOLD << I2C_BC_SCLK_THRESHOLD_SHIFT) | - I2C_BC_STOP_COND | I2C_BC_TERMINATE; + reg = FIELD_PREP(I2C_BC_SCLK_THRESHOLD, 9) | I2C_BC_STOP_COND | + I2C_BC_TERMINATE; i2c_writel(i2c_dev, reg, I2C_BUS_CLEAR_CNFG); if (i2c_dev->hw->has_config_load_reg) { err = tegra_i2c_wait_for_config_load(i2c_dev); @@ -1145,10 +1208,11 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, } } - packet_header = (0 << PACKET_HEADER0_HEADER_SIZE_SHIFT) | - PACKET_HEADER0_PROTOCOL_I2C | - (i2c_dev->cont_id << PACKET_HEADER0_CONT_ID_SHIFT) | - (1 << PACKET_HEADER0_PACKET_ID_SHIFT); + packet_header = FIELD_PREP(PACKET_HEADER0_HEADER_SIZE, 0) | + FIELD_PREP(PACKET_HEADER0_PROTOCOL, + PACKET_HEADER0_PROTOCOL_I2C) | + FIELD_PREP(PACKET_HEADER0_CONT_ID, i2c_dev->cont_id) | + FIELD_PREP(PACKET_HEADER0_PACKET_ID, 1); if (dma && !i2c_dev->msg_read) *buffer++ = packet_header; else @@ -1216,6 +1280,15 @@ static int tegra_i2c_xfer_msg(struct tegra_i2c_dev *i2c_dev, time_left = tegra_i2c_wait_completion_timeout( i2c_dev, &i2c_dev->dma_complete, xfer_time); + /* + * Synchronize DMA first, since dmaengine_terminate_sync() + * performs synchronization after the transfer's termination + * and we want to get a completion if transfer succeeded. + */ + dmaengine_synchronize(i2c_dev->msg_read ? + i2c_dev->rx_dma_chan : + i2c_dev->tx_dma_chan); + dmaengine_terminate_sync(i2c_dev->msg_read ? i2c_dev->rx_dma_chan : i2c_dev->tx_dma_chan); @@ -1544,6 +1617,7 @@ static const struct tegra_i2c_hw_feature tegra194_i2c_hw = { static const struct of_device_id tegra_i2c_of_match[] = { { .compatible = "nvidia,tegra194-i2c", .data = &tegra194_i2c_hw, }, { .compatible = "nvidia,tegra186-i2c", .data = &tegra186_i2c_hw, }, + { .compatible = "nvidia,tegra210-i2c-vi", .data = &tegra210_i2c_hw, }, { .compatible = "nvidia,tegra210-i2c", .data = &tegra210_i2c_hw, }, { .compatible = "nvidia,tegra124-i2c", .data = &tegra124_i2c_hw, }, { .compatible = "nvidia,tegra114-i2c", .data = &tegra114_i2c_hw, }, @@ -1556,6 +1630,7 @@ MODULE_DEVICE_TABLE(of, tegra_i2c_of_match); static int tegra_i2c_probe(struct platform_device *pdev) { + struct device *dev = &pdev->dev; struct tegra_i2c_dev *i2c_dev; struct resource *res; struct clk *div_clk; @@ -1611,6 +1686,8 @@ static int tegra_i2c_probe(struct platform_device *pdev) i2c_dev->hw = of_device_get_match_data(&pdev->dev); i2c_dev->is_dvc = of_device_is_compatible(pdev->dev.of_node, "nvidia,tegra20-i2c-dvc"); + i2c_dev->is_vi = of_device_is_compatible(dev->of_node, + "nvidia,tegra210-i2c-vi"); i2c_dev->adapter.quirks = i2c_dev->hw->quirks; i2c_dev->dma_buf_size = i2c_dev->adapter.quirks->max_write_len + I2C_PACKET_HEADER_SIZE; @@ -1626,6 +1703,17 @@ static int tegra_i2c_probe(struct platform_device *pdev) i2c_dev->fast_clk = fast_clk; } + if (i2c_dev->is_vi) { + i2c_dev->slow_clk = devm_clk_get(dev, "slow"); + if (IS_ERR(i2c_dev->slow_clk)) { + if (PTR_ERR(i2c_dev->slow_clk) != -EPROBE_DEFER) + dev_err(dev, "failed to get slow clock: %ld\n", + PTR_ERR(i2c_dev->slow_clk)); + + return PTR_ERR(i2c_dev->slow_clk); + } + } + platform_set_drvdata(pdev, i2c_dev); if (!i2c_dev->hw->has_single_clk_source) { @@ -1636,6 +1724,14 @@ static int tegra_i2c_probe(struct platform_device *pdev) } } + if (i2c_dev->slow_clk) { + ret = clk_prepare(i2c_dev->slow_clk); + if (ret < 0) { + dev_err(dev, "failed to prepare slow clock: %d\n", ret); + goto unprepare_fast_clk; + } + } + if (i2c_dev->bus_clk_rate > I2C_MAX_FAST_MODE_FREQ && i2c_dev->bus_clk_rate <= I2C_MAX_FAST_MODE_PLUS_FREQ) i2c_dev->clk_divisor_non_hs_mode = @@ -1651,7 +1747,7 @@ static int tegra_i2c_probe(struct platform_device *pdev) ret = clk_prepare(i2c_dev->div_clk); if (ret < 0) { dev_err(i2c_dev->dev, "Clock prepare failed %d\n", ret); - goto unprepare_fast_clk; + goto unprepare_slow_clk; } pm_runtime_irq_safe(&pdev->dev); @@ -1694,8 +1790,8 @@ static int tegra_i2c_probe(struct platform_device *pdev) irq_set_status_flags(i2c_dev->irq, IRQ_NOAUTOEN); - ret = devm_request_irq(&pdev->dev, i2c_dev->irq, - tegra_i2c_isr, 0, dev_name(&pdev->dev), i2c_dev); + ret = devm_request_irq(&pdev->dev, i2c_dev->irq, tegra_i2c_isr, + IRQF_NO_SUSPEND, dev_name(&pdev->dev), i2c_dev); if (ret) { dev_err(&pdev->dev, "Failed to request irq %i\n", i2c_dev->irq); goto release_dma; @@ -1738,6 +1834,10 @@ disable_rpm: unprepare_div_clk: clk_unprepare(i2c_dev->div_clk); +unprepare_slow_clk: + if (i2c_dev->is_vi) + clk_unprepare(i2c_dev->slow_clk); + unprepare_fast_clk: if (!i2c_dev->hw->has_single_clk_source) clk_unprepare(i2c_dev->fast_clk); @@ -1759,6 +1859,10 @@ static int tegra_i2c_remove(struct platform_device *pdev) tegra_i2c_runtime_suspend(&pdev->dev); clk_unprepare(i2c_dev->div_clk); + + if (i2c_dev->slow_clk) + clk_unprepare(i2c_dev->slow_clk); + if (!i2c_dev->hw->has_single_clk_source) clk_unprepare(i2c_dev->fast_clk); @@ -1769,15 +1873,14 @@ static int tegra_i2c_remove(struct platform_device *pdev) static int __maybe_unused tegra_i2c_suspend(struct device *dev) { struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); - int err; + int err = 0; i2c_mark_adapter_suspended(&i2c_dev->adapter); - err = pm_runtime_force_suspend(dev); - if (err < 0) - return err; + if (!pm_runtime_status_suspended(dev)) + err = tegra_i2c_runtime_suspend(dev); - return 0; + return err; } static int __maybe_unused tegra_i2c_resume(struct device *dev) @@ -1785,6 +1888,10 @@ static int __maybe_unused tegra_i2c_resume(struct device *dev) struct tegra_i2c_dev *i2c_dev = dev_get_drvdata(dev); int err; + /* + * We need to ensure that clocks are enabled so that registers can be + * restored in tegra_i2c_init(). + */ err = tegra_i2c_runtime_resume(dev); if (err) return err; @@ -1793,13 +1900,16 @@ static int __maybe_unused tegra_i2c_resume(struct device *dev) if (err) return err; - err = tegra_i2c_runtime_suspend(dev); - if (err) - return err; - - err = pm_runtime_force_resume(dev); - if (err < 0) - return err; + /* + * In case we are runtime suspended, disable clocks again so that we + * don't unbalance the clock reference counts during the next runtime + * resume transition. + */ + if (pm_runtime_status_suspended(dev)) { + err = tegra_i2c_runtime_suspend(dev); + if (err) + return err; + } i2c_mark_adapter_resumed(&i2c_dev->adapter); diff --git a/drivers/i2c/busses/i2c-uniphier-f.c b/drivers/i2c/busses/i2c-uniphier-f.c index 2b258d54d68c..cb4666c54a23 100644 --- a/drivers/i2c/busses/i2c-uniphier-f.c +++ b/drivers/i2c/busses/i2c-uniphier-f.c @@ -529,10 +529,8 @@ static int uniphier_fi2c_probe(struct platform_device *pdev) return PTR_ERR(priv->membase); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(dev, "failed to get IRQ number\n"); + if (irq < 0) return irq; - } if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed)) bus_speed = I2C_MAX_STANDARD_MODE_FREQ; diff --git a/drivers/i2c/busses/i2c-uniphier.c b/drivers/i2c/busses/i2c-uniphier.c index 668b1fa2b0ef..ee00a44bf4c7 100644 --- a/drivers/i2c/busses/i2c-uniphier.c +++ b/drivers/i2c/busses/i2c-uniphier.c @@ -324,10 +324,8 @@ static int uniphier_i2c_probe(struct platform_device *pdev) return PTR_ERR(priv->membase); irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(dev, "failed to get IRQ number\n"); + if (irq < 0) return irq; - } if (of_property_read_u32(dev->of_node, "clock-frequency", &bus_speed)) bus_speed = I2C_MAX_STANDARD_MODE_FREQ; diff --git a/drivers/i2c/busses/i2c-xlp9xx.c b/drivers/i2c/busses/i2c-xlp9xx.c index 391c878a7cdc..f2241cedf5d3 100644 --- a/drivers/i2c/busses/i2c-xlp9xx.c +++ b/drivers/i2c/busses/i2c-xlp9xx.c @@ -506,23 +506,19 @@ static int xlp9xx_i2c_smbus_setup(struct xlp9xx_i2c_dev *priv, static int xlp9xx_i2c_probe(struct platform_device *pdev) { struct xlp9xx_i2c_dev *priv; - struct resource *res; int err = 0; priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->base = devm_ioremap_resource(&pdev->dev, res); + priv->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->base)) return PTR_ERR(priv->base); priv->irq = platform_get_irq(pdev, 0); - if (priv->irq <= 0) { - dev_err(&pdev->dev, "invalid irq!\n"); + if (priv->irq <= 0) return priv->irq; - } /* SMBAlert irq */ priv->alert_data.irq = platform_get_irq(pdev, 1); if (priv->alert_data.irq <= 0) diff --git a/drivers/i2c/busses/i2c-xlr.c b/drivers/i2c/busses/i2c-xlr.c index 282f161a8b08..126d1393e548 100644 --- a/drivers/i2c/busses/i2c-xlr.c +++ b/drivers/i2c/busses/i2c-xlr.c @@ -362,7 +362,6 @@ static int xlr_i2c_probe(struct platform_device *pdev) { const struct of_device_id *match; struct xlr_i2c_private *priv; - struct resource *res; struct clk *clk; unsigned long clk_rate; unsigned long clk_div; @@ -380,8 +379,7 @@ static int xlr_i2c_probe(struct platform_device *pdev) else priv->cfg = &xlr_i2c_config_default; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - priv->iobase = devm_ioremap_resource(&pdev->dev, res); + priv->iobase = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(priv->iobase)) return PTR_ERR(priv->iobase); diff --git a/drivers/i2c/busses/i2c-zx2967.c b/drivers/i2c/busses/i2c-zx2967.c index 5f3318559b8d..8db9519695a6 100644 --- a/drivers/i2c/busses/i2c-zx2967.c +++ b/drivers/i2c/busses/i2c-zx2967.c @@ -502,7 +502,6 @@ static int zx2967_i2c_probe(struct platform_device *pdev) { struct zx2967_i2c *i2c; void __iomem *reg_base; - struct resource *res; struct clk *clk; int ret; @@ -510,8 +509,7 @@ static int zx2967_i2c_probe(struct platform_device *pdev) if (!i2c) return -ENOMEM; - res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - reg_base = devm_ioremap_resource(&pdev->dev, res); + reg_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(reg_base)) return PTR_ERR(reg_base); diff --git a/drivers/i2c/i2c-core-acpi.c b/drivers/i2c/i2c-core-acpi.c index c8f42f2037cb..2ade99b105b9 100644 --- a/drivers/i2c/i2c-core-acpi.c +++ b/drivers/i2c/i2c-core-acpi.c @@ -468,16 +468,12 @@ struct notifier_block i2c_acpi_notifier = { struct i2c_client *i2c_acpi_new_device(struct device *dev, int index, struct i2c_board_info *info) { + struct acpi_device *adev = ACPI_COMPANION(dev); struct i2c_acpi_lookup lookup; struct i2c_adapter *adapter; - struct acpi_device *adev; LIST_HEAD(resource_list); int ret; - adev = ACPI_COMPANION(dev); - if (!adev) - return ERR_PTR(-EINVAL); - memset(&lookup, 0, sizeof(lookup)); lookup.info = info; lookup.device_handle = acpi_device_handle(adev); diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c index 1f1442dfcad7..d1f278f73011 100644 --- a/drivers/i2c/i2c-core-base.c +++ b/drivers/i2c/i2c-core-base.c @@ -1598,6 +1598,18 @@ void i2c_del_adapter(struct i2c_adapter *adap) } EXPORT_SYMBOL(i2c_del_adapter); +static void i2c_parse_timing(struct device *dev, char *prop_name, u32 *cur_val_p, + u32 def_val, bool use_def) +{ + int ret; + + ret = device_property_read_u32(dev, prop_name, cur_val_p); + if (ret && use_def) + *cur_val_p = def_val; + + dev_dbg(dev, "%s: %u\n", prop_name, *cur_val_p); +} + /** * i2c_parse_fw_timings - get I2C related timing parameters from firmware * @dev: The device to scan for I2C timing properties @@ -1616,49 +1628,28 @@ EXPORT_SYMBOL(i2c_del_adapter); */ void i2c_parse_fw_timings(struct device *dev, struct i2c_timings *t, bool use_defaults) { - int ret; - - ret = device_property_read_u32(dev, "clock-frequency", &t->bus_freq_hz); - if (ret && use_defaults) - t->bus_freq_hz = I2C_MAX_STANDARD_MODE_FREQ; - - ret = device_property_read_u32(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns); - if (ret && use_defaults) { - if (t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ) - t->scl_rise_ns = 1000; - else if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) - t->scl_rise_ns = 300; - else - t->scl_rise_ns = 120; - } - - ret = device_property_read_u32(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns); - if (ret && use_defaults) { - if (t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ) - t->scl_fall_ns = 300; - else - t->scl_fall_ns = 120; - } - - ret = device_property_read_u32(dev, "i2c-scl-internal-delay-ns", &t->scl_int_delay_ns); - if (ret && use_defaults) - t->scl_int_delay_ns = 0; + bool u = use_defaults; + u32 d; - ret = device_property_read_u32(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns); - if (ret && use_defaults) - t->sda_fall_ns = t->scl_fall_ns; + i2c_parse_timing(dev, "clock-frequency", &t->bus_freq_hz, + I2C_MAX_STANDARD_MODE_FREQ, u); - ret = device_property_read_u32(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns); - if (ret && use_defaults) - t->sda_hold_ns = 0; + d = t->bus_freq_hz <= I2C_MAX_STANDARD_MODE_FREQ ? 1000 : + t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120; + i2c_parse_timing(dev, "i2c-scl-rising-time-ns", &t->scl_rise_ns, d, u); - ret = device_property_read_u32(dev, "i2c-digital-filter-width-ns", &t->digital_filter_width_ns); - if (ret && use_defaults) - t->digital_filter_width_ns = 0; + d = t->bus_freq_hz <= I2C_MAX_FAST_MODE_FREQ ? 300 : 120; + i2c_parse_timing(dev, "i2c-scl-falling-time-ns", &t->scl_fall_ns, d, u); - ret = device_property_read_u32(dev, "i2c-analog-filter-cutoff-frequency", &t->analog_filter_cutoff_freq_hz); - if (ret && use_defaults) - t->analog_filter_cutoff_freq_hz = 0; + i2c_parse_timing(dev, "i2c-scl-internal-delay-ns", + &t->scl_int_delay_ns, 0, u); + i2c_parse_timing(dev, "i2c-sda-falling-time-ns", &t->sda_fall_ns, + t->scl_fall_ns, u); + i2c_parse_timing(dev, "i2c-sda-hold-time-ns", &t->sda_hold_ns, 0, u); + i2c_parse_timing(dev, "i2c-digital-filter-width-ns", + &t->digital_filter_width_ns, 0, u); + i2c_parse_timing(dev, "i2c-analog-filter-cutoff-frequency", + &t->analog_filter_cutoff_freq_hz, 0, u); } EXPORT_SYMBOL_GPL(i2c_parse_fw_timings); @@ -2195,7 +2186,6 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver) const unsigned short *address_list; struct i2c_client *temp_client; int i, err = 0; - int adap_id = i2c_adapter_id(adapter); address_list = driver->address_list; if (!driver->detect || !address_list) @@ -2223,7 +2213,7 @@ static int i2c_detect(struct i2c_adapter *adapter, struct i2c_driver *driver) for (i = 0; address_list[i] != I2C_CLIENT_END; i += 1) { dev_dbg(&adapter->dev, "found normal entry for adapter %d, addr 0x%02x\n", - adap_id, address_list[i]); + i2c_adapter_id(adapter), address_list[i]); temp_client->addr = address_list[i]; err = i2c_detect_address(temp_client, driver); if (unlikely(err)) diff --git a/drivers/i2c/i2c-core.h b/drivers/i2c/i2c-core.h index 517d98be68d2..94ff1693b391 100644 --- a/drivers/i2c/i2c-core.h +++ b/drivers/i2c/i2c-core.h @@ -23,9 +23,9 @@ int i2c_dev_irq_from_resources(const struct resource *resources, unsigned int num_resources); /* - * We only allow atomic transfers for very late communication, e.g. to send - * the powerdown command to a PMIC. Atomic transfers are a corner case and not - * for generic use! + * We only allow atomic transfers for very late communication, e.g. to access a + * PMIC when powering down. Atomic transfers are a corner case and not for + * generic use! */ static inline bool i2c_in_atomic_xfer_mode(void) { diff --git a/drivers/i2c/i2c-slave-eeprom.c b/drivers/i2c/i2c-slave-eeprom.c index cb415b10642f..593f2fd39d17 100644 --- a/drivers/i2c/i2c-slave-eeprom.c +++ b/drivers/i2c/i2c-slave-eeprom.c @@ -5,10 +5,9 @@ * Copyright (C) 2014 by Wolfram Sang, Sang Engineering <wsa@sang-engineering.com> * Copyright (C) 2014 by Renesas Electronics Corporation * - * Because most IP blocks can only detect one I2C slave address anyhow, this - * driver does not support simulating EEPROM types which take more than one - * address. It is prepared to simulate bigger EEPROMs with an internal 16 bit - * pointer, yet implementation is deferred until the need actually arises. + * Because most slave IP cores can only detect one I2C slave address anyhow, + * this driver does not support simulating EEPROM types which take more than + * one address. */ /* @@ -18,6 +17,7 @@ */ #include <linux/bitfield.h> +#include <linux/firmware.h> #include <linux/i2c.h> #include <linux/init.h> #include <linux/module.h> @@ -40,7 +40,7 @@ struct eeprom_data { #define I2C_SLAVE_BYTELEN GENMASK(15, 0) #define I2C_SLAVE_FLAG_ADDR16 BIT(16) #define I2C_SLAVE_FLAG_RO BIT(17) -#define I2C_SLAVE_DEVICE_MAGIC(_len, _flags) ((_flags) | (_len)) +#define I2C_SLAVE_DEVICE_MAGIC(_len, _flags) ((_flags) | ((_len) - 1)) static int i2c_slave_eeprom_slave_cb(struct i2c_client *client, enum i2c_slave_event event, u8 *val) @@ -120,24 +120,47 @@ static ssize_t i2c_slave_eeprom_bin_write(struct file *filp, struct kobject *kob return count; } +static int i2c_slave_init_eeprom_data(struct eeprom_data *eeprom, struct i2c_client *client, + unsigned int size) +{ + const struct firmware *fw; + const char *eeprom_data; + int ret = device_property_read_string(&client->dev, "firmware-name", &eeprom_data); + + if (!ret) { + ret = request_firmware_into_buf(&fw, eeprom_data, &client->dev, + eeprom->buffer, size); + if (ret) + return ret; + release_firmware(fw); + } else { + /* An empty eeprom typically has all bits set to 1 */ + memset(eeprom->buffer, 0xff, size); + } + return 0; +} + static int i2c_slave_eeprom_probe(struct i2c_client *client, const struct i2c_device_id *id) { struct eeprom_data *eeprom; int ret; - unsigned int size = FIELD_GET(I2C_SLAVE_BYTELEN, id->driver_data); + unsigned int size = FIELD_GET(I2C_SLAVE_BYTELEN, id->driver_data) + 1; unsigned int flag_addr16 = FIELD_GET(I2C_SLAVE_FLAG_ADDR16, id->driver_data); eeprom = devm_kzalloc(&client->dev, sizeof(struct eeprom_data) + size, GFP_KERNEL); if (!eeprom) return -ENOMEM; - eeprom->idx_write_cnt = 0; eeprom->num_address_bytes = flag_addr16 ? 2 : 1; eeprom->address_mask = size - 1; eeprom->read_only = FIELD_GET(I2C_SLAVE_FLAG_RO, id->driver_data); spin_lock_init(&eeprom->buffer_lock); i2c_set_clientdata(client, eeprom); + ret = i2c_slave_init_eeprom_data(eeprom, client, size); + if (ret) + return ret; + sysfs_bin_attr_init(&eeprom->bin); eeprom->bin.attr.name = "slave-eeprom"; eeprom->bin.attr.mode = S_IRUSR | S_IWUSR; @@ -175,6 +198,8 @@ static const struct i2c_device_id i2c_slave_eeprom_id[] = { { "slave-24c32ro", I2C_SLAVE_DEVICE_MAGIC(32768 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) }, { "slave-24c64", I2C_SLAVE_DEVICE_MAGIC(65536 / 8, I2C_SLAVE_FLAG_ADDR16) }, { "slave-24c64ro", I2C_SLAVE_DEVICE_MAGIC(65536 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) }, + { "slave-24c512", I2C_SLAVE_DEVICE_MAGIC(524288 / 8, I2C_SLAVE_FLAG_ADDR16) }, + { "slave-24c512ro", I2C_SLAVE_DEVICE_MAGIC(524288 / 8, I2C_SLAVE_FLAG_ADDR16 | I2C_SLAVE_FLAG_RO) }, { } }; MODULE_DEVICE_TABLE(i2c, i2c_slave_eeprom_id); diff --git a/drivers/i2c/i2c-smbus.c b/drivers/i2c/i2c-smbus.c index 809bcf8387d0..dc0108287ccf 100644 --- a/drivers/i2c/i2c-smbus.c +++ b/drivers/i2c/i2c-smbus.c @@ -3,10 +3,11 @@ * i2c-smbus.c - SMBus extensions to the I2C protocol * * Copyright (C) 2008 David Brownell - * Copyright (C) 2010 Jean Delvare <jdelvare@suse.de> + * Copyright (C) 2010-2019 Jean Delvare <jdelvare@suse.de> */ #include <linux/device.h> +#include <linux/dmi.h> #include <linux/i2c.h> #include <linux/i2c-smbus.h> #include <linux/interrupt.h> @@ -196,6 +197,107 @@ EXPORT_SYMBOL_GPL(i2c_handle_smbus_alert); module_i2c_driver(smbalert_driver); +/* + * SPD is not part of SMBus but we include it here for convenience as the + * target systems are the same. + * Restrictions to automatic SPD instantiation: + * - Only works if all filled slots have the same memory type + * - Only works for DDR2, DDR3 and DDR4 for now + * - Only works on systems with 1 to 4 memory slots + */ +#if IS_ENABLED(CONFIG_DMI) +void i2c_register_spd(struct i2c_adapter *adap) +{ + int n, slot_count = 0, dimm_count = 0; + u16 handle; + u8 common_mem_type = 0x0, mem_type; + u64 mem_size; + const char *name; + + while ((handle = dmi_memdev_handle(slot_count)) != 0xffff) { + slot_count++; + + /* Skip empty slots */ + mem_size = dmi_memdev_size(handle); + if (!mem_size) + continue; + + /* Skip undefined memory type */ + mem_type = dmi_memdev_type(handle); + if (mem_type <= 0x02) /* Invalid, Other, Unknown */ + continue; + + if (!common_mem_type) { + /* First filled slot */ + common_mem_type = mem_type; + } else { + /* Check that all filled slots have the same type */ + if (mem_type != common_mem_type) { + dev_warn(&adap->dev, + "Different memory types mixed, not instantiating SPD\n"); + return; + } + } + dimm_count++; + } + + /* No useful DMI data, bail out */ + if (!dimm_count) + return; + + dev_info(&adap->dev, "%d/%d memory slots populated (from DMI)\n", + dimm_count, slot_count); + + if (slot_count > 4) { + dev_warn(&adap->dev, + "Systems with more than 4 memory slots not supported yet, not instantiating SPD\n"); + return; + } + + switch (common_mem_type) { + case 0x13: /* DDR2 */ + case 0x18: /* DDR3 */ + case 0x1C: /* LPDDR2 */ + case 0x1D: /* LPDDR3 */ + name = "spd"; + break; + case 0x1A: /* DDR4 */ + case 0x1E: /* LPDDR4 */ + name = "ee1004"; + break; + default: + dev_info(&adap->dev, + "Memory type 0x%02x not supported yet, not instantiating SPD\n", + common_mem_type); + return; + } + + /* + * We don't know in which slots the memory modules are. We could + * try to guess from the slot names, but that would be rather complex + * and unreliable, so better probe all possible addresses until we + * have found all memory modules. + */ + for (n = 0; n < slot_count && dimm_count; n++) { + struct i2c_board_info info; + unsigned short addr_list[2]; + + memset(&info, 0, sizeof(struct i2c_board_info)); + strlcpy(info.type, name, I2C_NAME_SIZE); + addr_list[0] = 0x50 + n; + addr_list[1] = I2C_CLIENT_END; + + if (!IS_ERR(i2c_new_scanned_device(adap, &info, addr_list, NULL))) { + dev_info(&adap->dev, + "Successfully instantiated SPD at 0x%hx\n", + addr_list[0]); + dimm_count--; + } + } +} +EXPORT_SYMBOL_GPL(i2c_register_spd); +#endif + MODULE_AUTHOR("Jean Delvare <jdelvare@suse.de>"); MODULE_DESCRIPTION("SMBus protocol extensions support"); MODULE_LICENSE("GPL"); diff --git a/drivers/i2c/muxes/i2c-mux-pca954x.c b/drivers/i2c/muxes/i2c-mux-pca954x.c index a0d926ae3f86..4ad665757dd8 100644 --- a/drivers/i2c/muxes/i2c-mux-pca954x.c +++ b/drivers/i2c/muxes/i2c-mux-pca954x.c @@ -1,10 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 /* * I2C multiplexer * * Copyright (c) 2008-2009 Rodolfo Giometti <giometti@linux.it> * Copyright (c) 2008-2009 Eurotech S.p.A. <info@eurotech.it> * - * This module supports the PCA954x and PCA954x series of I2C multiplexer/switch + * This module supports the PCA954x and PCA984x series of I2C multiplexer/switch * chips made by NXP Semiconductors. * This includes the: * PCA9540, PCA9542, PCA9543, PCA9544, PCA9545, PCA9546, PCA9547, @@ -29,10 +30,6 @@ * i2c-virtual_cb.c from Brian Kuschak <bkuschak@yahoo.com> * and * pca9540.c from Jean Delvare <jdelvare@suse.de>. - * - * This file is licensed under the terms of the GNU General Public - * License version 2. This program is licensed "as is" without any - * warranty of any kind, whether express or implied. */ #include <linux/device.h> @@ -43,10 +40,8 @@ #include <linux/interrupt.h> #include <linux/irq.h> #include <linux/module.h> -#include <linux/of.h> -#include <linux/of_device.h> -#include <linux/of_irq.h> #include <linux/pm.h> +#include <linux/property.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <dt-bindings/mux/mux.h> @@ -198,7 +193,6 @@ static const struct i2c_device_id pca954x_id[] = { }; MODULE_DEVICE_TABLE(i2c, pca954x_id); -#ifdef CONFIG_OF static const struct of_device_id pca954x_of_match[] = { { .compatible = "nxp,pca9540", .data = &chips[pca_9540] }, { .compatible = "nxp,pca9542", .data = &chips[pca_9542] }, @@ -215,7 +209,6 @@ static const struct of_device_id pca954x_of_match[] = { {} }; MODULE_DEVICE_TABLE(of, pca954x_of_match); -#endif /* Write to mux register. Don't use i2c_transfer()/i2c_smbus_xfer() for this as they will try to lock adapter a second time */ @@ -327,21 +320,18 @@ static DEVICE_ATTR_RW(idle_state); static irqreturn_t pca954x_irq_handler(int irq, void *dev_id) { struct pca954x *data = dev_id; - unsigned int child_irq; - int ret, i, handled = 0; + unsigned long pending; + int ret, i; ret = i2c_smbus_read_byte(data->client); if (ret < 0) return IRQ_NONE; - for (i = 0; i < data->chip->nchans; i++) { - if (ret & BIT(PCA954X_IRQ_OFFSET + i)) { - child_irq = irq_linear_revmap(data->irq, i); - handle_nested_irq(child_irq); - handled++; - } - } - return handled ? IRQ_HANDLED : IRQ_NONE; + pending = (ret >> PCA954X_IRQ_OFFSET) & (BIT(data->chip->nchans) - 1); + for_each_set_bit(i, &pending, data->chip->nchans) + handle_nested_irq(irq_linear_revmap(data->irq, i)); + + return IRQ_RETVAL(pending); } static int pca954x_irq_set_type(struct irq_data *idata, unsigned int type) @@ -390,11 +380,8 @@ static int pca954x_irq_setup(struct i2c_mux_core *muxc) static void pca954x_cleanup(struct i2c_mux_core *muxc) { struct pca954x *data = i2c_mux_priv(muxc); - struct i2c_client *client = data->client; int c, irq; - device_remove_file(&client->dev, &dev_attr_idle_state); - if (data->irq) { for (c = 0; c < data->chip->nchans; c++) { irq = irq_find_mapping(data->irq, c); @@ -429,7 +416,6 @@ static int pca954x_probe(struct i2c_client *client, { struct i2c_adapter *adap = client->adapter; struct device *dev = &client->dev; - struct device_node *np = dev->of_node; struct gpio_desc *gpio; struct i2c_mux_core *muxc; struct pca954x *data; @@ -459,7 +445,7 @@ static int pca954x_probe(struct i2c_client *client, udelay(1); } - data->chip = of_device_get_match_data(dev); + data->chip = device_get_match_data(dev); if (!data->chip) data->chip = &chips[id->driver_data]; @@ -481,8 +467,8 @@ static int pca954x_probe(struct i2c_client *client, } data->idle_state = MUX_IDLE_AS_IS; - if (of_property_read_u32(np, "idle-state", &data->idle_state)) { - if (np && of_property_read_bool(np, "i2c-mux-idle-disconnect")) + if (device_property_read_u32(dev, "idle-state", &data->idle_state)) { + if (device_property_read_bool(dev, "i2c-mux-idle-disconnect")) data->idle_state = MUX_IDLE_DISCONNECT; } @@ -539,6 +525,8 @@ static int pca954x_remove(struct i2c_client *client) { struct i2c_mux_core *muxc = i2c_get_clientdata(client); + device_remove_file(&client->dev, &dev_attr_idle_state); + pca954x_cleanup(muxc); return 0; } @@ -565,7 +553,7 @@ static struct i2c_driver pca954x_driver = { .driver = { .name = "pca954x", .pm = &pca954x_pm, - .of_match_table = of_match_ptr(pca954x_of_match), + .of_match_table = pca954x_of_match, }, .probe = pca954x_probe, .remove = pca954x_remove, diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 57cf4ba5e27c..342190196dfb 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -11,19 +11,19 @@ obj-$(CONFIG_IOASID) += ioasid.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o obj-$(CONFIG_MSM_IOMMU) += msm_iommu.o -obj-$(CONFIG_AMD_IOMMU) += amd_iommu.o amd_iommu_init.o amd_iommu_quirks.o -obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd_iommu_debugfs.o -obj-$(CONFIG_AMD_IOMMU_V2) += amd_iommu_v2.o +obj-$(CONFIG_AMD_IOMMU) += amd/iommu.o amd/init.o amd/quirks.o +obj-$(CONFIG_AMD_IOMMU_DEBUGFS) += amd/debugfs.o +obj-$(CONFIG_AMD_IOMMU_V2) += amd/iommu_v2.o obj-$(CONFIG_ARM_SMMU) += arm_smmu.o arm_smmu-objs += arm-smmu.o arm-smmu-impl.o arm-smmu-qcom.o obj-$(CONFIG_ARM_SMMU_V3) += arm-smmu-v3.o -obj-$(CONFIG_DMAR_TABLE) += dmar.o -obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o intel-pasid.o -obj-$(CONFIG_INTEL_IOMMU) += intel-trace.o -obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel-iommu-debugfs.o -obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o +obj-$(CONFIG_DMAR_TABLE) += intel/dmar.o +obj-$(CONFIG_INTEL_IOMMU) += intel/iommu.o intel/pasid.o +obj-$(CONFIG_INTEL_IOMMU) += intel/trace.o +obj-$(CONFIG_INTEL_IOMMU_DEBUGFS) += intel/debugfs.o +obj-$(CONFIG_INTEL_IOMMU_SVM) += intel/svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o -obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o +obj-$(CONFIG_IRQ_REMAP) += intel/irq_remapping.o irq_remapping.o obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o obj-$(CONFIG_MTK_IOMMU_V1) += mtk_iommu_v1.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o diff --git a/drivers/iommu/amd_iommu.h b/drivers/iommu/amd/amd_iommu.h index f892992c8744..f892992c8744 100644 --- a/drivers/iommu/amd_iommu.h +++ b/drivers/iommu/amd/amd_iommu.h diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 30a5d412255a..30a5d412255a 100644 --- a/drivers/iommu/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h diff --git a/drivers/iommu/amd_iommu_debugfs.c b/drivers/iommu/amd/debugfs.c index 545372fcc72f..545372fcc72f 100644 --- a/drivers/iommu/amd_iommu_debugfs.c +++ b/drivers/iommu/amd/debugfs.c diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd/init.c index 3faff7f80fd2..6ebd4825e320 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd/init.c @@ -33,7 +33,7 @@ #include <linux/crash_dump.h> #include "amd_iommu.h" -#include "irq_remapping.h" +#include "../irq_remapping.h" /* * definitions for the ACPI scanning code diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd/iommu.c index 311ef7105c6d..74cca1757172 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd/iommu.c @@ -43,7 +43,7 @@ #include <asm/dma.h> #include "amd_iommu.h" -#include "irq_remapping.h" +#include "../irq_remapping.h" #define CMD_SET_TYPE(cmd, t) ((cmd)->data[1] |= ((t) << 28)) diff --git a/drivers/iommu/amd_iommu_v2.c b/drivers/iommu/amd/iommu_v2.c index e4b025c5637c..e4b025c5637c 100644 --- a/drivers/iommu/amd_iommu_v2.c +++ b/drivers/iommu/amd/iommu_v2.c diff --git a/drivers/iommu/amd_iommu_quirks.c b/drivers/iommu/amd/quirks.c index 5120ce4fdce3..5120ce4fdce3 100644 --- a/drivers/iommu/amd_iommu_quirks.c +++ b/drivers/iommu/amd/quirks.c diff --git a/drivers/iommu/intel-iommu-debugfs.c b/drivers/iommu/intel/debugfs.c index cf1ebb98e418..cf1ebb98e418 100644 --- a/drivers/iommu/intel-iommu-debugfs.c +++ b/drivers/iommu/intel/debugfs.c diff --git a/drivers/iommu/dmar.c b/drivers/iommu/intel/dmar.c index 60a2970c37ff..cc46dff98fa0 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -32,7 +32,7 @@ #include <asm/irq_remapping.h> #include <asm/iommu_table.h> -#include "irq_remapping.h" +#include "../irq_remapping.h" typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *); struct dmar_res_callback { diff --git a/drivers/iommu/intel-pasid.h b/drivers/iommu/intel/intel-pasid.h index c5318d40e0fa..c5318d40e0fa 100644 --- a/drivers/iommu/intel-pasid.h +++ b/drivers/iommu/intel/intel-pasid.h diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel/iommu.c index 648a785e078a..9129663a7406 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -47,7 +47,7 @@ #include <asm/iommu.h> #include <trace/events/intel_iommu.h> -#include "irq_remapping.h" +#include "../irq_remapping.h" #include "intel-pasid.h" #define ROOT_SIZE VTD_PAGE_SIZE diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel/irq_remapping.c index a042f123b091..7f8769800815 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel/irq_remapping.c @@ -21,7 +21,7 @@ #include <asm/pci-direct.h> #include <asm/msidef.h> -#include "irq_remapping.h" +#include "../irq_remapping.h" enum irq_mode { IRQ_REMAPPING, diff --git a/drivers/iommu/intel-pasid.c b/drivers/iommu/intel/pasid.c index c81f0f17c6ba..c81f0f17c6ba 100644 --- a/drivers/iommu/intel-pasid.c +++ b/drivers/iommu/intel/pasid.c diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel/svm.c index 6c87c807a0ab..6c87c807a0ab 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel/svm.c diff --git a/drivers/iommu/intel-trace.c b/drivers/iommu/intel/trace.c index bfb6a6e37a88..bfb6a6e37a88 100644 --- a/drivers/iommu/intel-trace.c +++ b/drivers/iommu/intel/trace.c diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index 66b9a68f5e9f..29fead208cad 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -493,6 +493,19 @@ config TI_SCI_INTA_IRQCHIP If you wish to use interrupt aggregator irq resources managed by the TI System Controller, say Y here. Otherwise, say N. +config RISCV_INTC + bool "RISC-V Local Interrupt Controller" + depends on RISCV + default y + help + This enables support for the per-HART local interrupt controller + found in standard RISC-V systems. The per-HART local interrupt + controller handles timer interrupts, software interrupts, and + hardware interrupts. Without a per-HART local interrupt controller, + a RISC-V system will be unable to handle any interrupts. + + If you don't know what to do here, say Y. + config SIFIVE_PLIC bool "SiFive Platform-Level Interrupt Controller" depends on RISCV diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 3a4ce283189a..133f9c45744a 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -98,6 +98,7 @@ obj-$(CONFIG_NDS32) += irq-ativic32.o obj-$(CONFIG_QCOM_PDC) += qcom-pdc.o obj-$(CONFIG_CSKY_MPINTC) += irq-csky-mpintc.o obj-$(CONFIG_CSKY_APB_INTC) += irq-csky-apb-intc.o +obj-$(CONFIG_RISCV_INTC) += irq-riscv-intc.o obj-$(CONFIG_SIFIVE_PLIC) += irq-sifive-plic.o obj-$(CONFIG_IMX_IRQSTEER) += irq-imx-irqsteer.o obj-$(CONFIG_IMX_INTMUX) += irq-imx-intmux.o diff --git a/drivers/irqchip/irq-riscv-intc.c b/drivers/irqchip/irq-riscv-intc.c new file mode 100644 index 000000000000..a6f97fa6ff69 --- /dev/null +++ b/drivers/irqchip/irq-riscv-intc.c @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2012 Regents of the University of California + * Copyright (C) 2017-2018 SiFive + * Copyright (C) 2020 Western Digital Corporation or its affiliates. + */ + +#define pr_fmt(fmt) "riscv-intc: " fmt +#include <linux/atomic.h> +#include <linux/bits.h> +#include <linux/cpu.h> +#include <linux/irq.h> +#include <linux/irqchip.h> +#include <linux/irqdomain.h> +#include <linux/interrupt.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/smp.h> + +static struct irq_domain *intc_domain; + +static asmlinkage void riscv_intc_irq(struct pt_regs *regs) +{ + unsigned long cause = regs->cause & ~CAUSE_IRQ_FLAG; + + if (unlikely(cause >= BITS_PER_LONG)) + panic("unexpected interrupt cause"); + + switch (cause) { +#ifdef CONFIG_SMP + case RV_IRQ_SOFT: + /* + * We only use software interrupts to pass IPIs, so if a + * non-SMP system gets one, then we don't know what to do. + */ + handle_IPI(regs); + break; +#endif + default: + handle_domain_irq(intc_domain, cause, regs); + break; + } +} + +/* + * On RISC-V systems local interrupts are masked or unmasked by writing + * the SIE (Supervisor Interrupt Enable) CSR. As CSRs can only be written + * on the local hart, these functions can only be called on the hart that + * corresponds to the IRQ chip. + */ + +static void riscv_intc_irq_mask(struct irq_data *d) +{ + csr_clear(CSR_IE, BIT(d->hwirq)); +} + +static void riscv_intc_irq_unmask(struct irq_data *d) +{ + csr_set(CSR_IE, BIT(d->hwirq)); +} + +static int riscv_intc_cpu_starting(unsigned int cpu) +{ + csr_set(CSR_IE, BIT(RV_IRQ_SOFT)); + return 0; +} + +static int riscv_intc_cpu_dying(unsigned int cpu) +{ + csr_clear(CSR_IE, BIT(RV_IRQ_SOFT)); + return 0; +} + +static struct irq_chip riscv_intc_chip = { + .name = "RISC-V INTC", + .irq_mask = riscv_intc_irq_mask, + .irq_unmask = riscv_intc_irq_unmask, +}; + +static int riscv_intc_domain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hwirq) +{ + irq_set_percpu_devid(irq); + irq_domain_set_info(d, irq, hwirq, &riscv_intc_chip, d->host_data, + handle_percpu_devid_irq, NULL, NULL); + + return 0; +} + +static const struct irq_domain_ops riscv_intc_domain_ops = { + .map = riscv_intc_domain_map, + .xlate = irq_domain_xlate_onecell, +}; + +static int __init riscv_intc_init(struct device_node *node, + struct device_node *parent) +{ + int rc, hartid; + + hartid = riscv_of_parent_hartid(node); + if (hartid < 0) { + pr_warn("unable to fine hart id for %pOF\n", node); + return 0; + } + + /* + * The DT will have one INTC DT node under each CPU (or HART) + * DT node so riscv_intc_init() function will be called once + * for each INTC DT node. We only need to do INTC initialization + * for the INTC DT node belonging to boot CPU (or boot HART). + */ + if (riscv_hartid_to_cpuid(hartid) != smp_processor_id()) + return 0; + + intc_domain = irq_domain_add_linear(node, BITS_PER_LONG, + &riscv_intc_domain_ops, NULL); + if (!intc_domain) { + pr_err("unable to add IRQ domain\n"); + return -ENXIO; + } + + rc = set_handle_irq(&riscv_intc_irq); + if (rc) { + pr_err("failed to set irq handler\n"); + return rc; + } + + cpuhp_setup_state(CPUHP_AP_IRQ_RISCV_STARTING, + "irqchip/riscv/intc:starting", + riscv_intc_cpu_starting, + riscv_intc_cpu_dying); + + pr_info("%d local interrupts mapped\n", BITS_PER_LONG); + + return 0; +} + +IRQCHIP_DECLARE(riscv, "riscv,cpu-intc", riscv_intc_init); diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index d9c53f85a68e..eaa3e9fe54e9 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -9,6 +9,7 @@ #include <linux/io.h> #include <linux/irq.h> #include <linux/irqchip.h> +#include <linux/irqchip/chained_irq.h> #include <linux/irqdomain.h> #include <linux/module.h> #include <linux/of.h> @@ -76,6 +77,7 @@ struct plic_handler { void __iomem *enable_base; struct plic_priv *priv; }; +static int plic_parent_irq; static bool plic_cpuhp_setup_done; static DEFINE_PER_CPU(struct plic_handler, plic_handlers); @@ -219,15 +221,17 @@ static const struct irq_domain_ops plic_irqdomain_ops = { * that source ID back to the same claim register. This automatically enables * and disables the interrupt, so there's nothing else to do. */ -static void plic_handle_irq(struct pt_regs *regs) +static void plic_handle_irq(struct irq_desc *desc) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + struct irq_chip *chip = irq_desc_get_chip(desc); void __iomem *claim = handler->hart_base + CONTEXT_CLAIM; irq_hw_number_t hwirq; WARN_ON_ONCE(!handler->present); - csr_clear(CSR_IE, IE_EIE); + chained_irq_enter(chip, desc); + while ((hwirq = readl(claim))) { int irq = irq_find_mapping(handler->priv->irqdomain, hwirq); @@ -237,21 +241,8 @@ static void plic_handle_irq(struct pt_regs *regs) else generic_handle_irq(irq); } - csr_set(CSR_IE, IE_EIE); -} - -/* - * Walk up the DT tree until we find an active RISC-V core (HART) node and - * extract the cpuid from it. - */ -static int plic_find_hart_id(struct device_node *node) -{ - for (; node; node = node->parent) { - if (of_device_is_compatible(node, "riscv")) - return riscv_of_processor_hartid(node); - } - return -1; + chained_irq_exit(chip, desc); } static void plic_set_threshold(struct plic_handler *handler, u32 threshold) @@ -262,10 +253,8 @@ static void plic_set_threshold(struct plic_handler *handler, u32 threshold) static int plic_dying_cpu(unsigned int cpu) { - struct plic_handler *handler = this_cpu_ptr(&plic_handlers); - - csr_clear(CSR_IE, IE_EIE); - plic_set_threshold(handler, PLIC_DISABLE_THRESHOLD); + if (plic_parent_irq) + disable_percpu_irq(plic_parent_irq); return 0; } @@ -274,7 +263,11 @@ static int plic_starting_cpu(unsigned int cpu) { struct plic_handler *handler = this_cpu_ptr(&plic_handlers); - csr_set(CSR_IE, IE_EIE); + if (plic_parent_irq) + enable_percpu_irq(plic_parent_irq, + irq_get_trigger_type(plic_parent_irq)); + else + pr_warn("cpu%d: parent irq not available\n", cpu); plic_set_threshold(handler, PLIC_ENABLE_THRESHOLD); return 0; @@ -330,7 +323,7 @@ static int __init plic_init(struct device_node *node, if (parent.args[0] != RV_IRQ_EXT) continue; - hartid = plic_find_hart_id(parent.np); + hartid = riscv_of_parent_hartid(parent.np); if (hartid < 0) { pr_warn("failed to parse hart ID for context %d.\n", i); continue; @@ -342,6 +335,14 @@ static int __init plic_init(struct device_node *node, continue; } + /* Find parent domain and register chained handler */ + if (!plic_parent_irq && irq_find_host(parent.np)) { + plic_parent_irq = irq_of_parse_and_map(node, i); + if (plic_parent_irq) + irq_set_chained_handler(plic_parent_irq, + plic_handle_irq); + } + /* * When running in M-mode we need to ignore the S-mode handler. * Here we assume it always comes later, but that might be a @@ -382,7 +383,6 @@ done: pr_info("%pOFP: mapped %d interrupts with %d handlers for" " %d contexts.\n", node, nr_irqs, nr_handlers, nr_contexts); - set_handle_irq(plic_handle_irq); return 0; out_iounmap: diff --git a/drivers/mailbox/Kconfig b/drivers/mailbox/Kconfig index 5a577a6734cf..05b1009e2820 100644 --- a/drivers/mailbox/Kconfig +++ b/drivers/mailbox/Kconfig @@ -236,4 +236,22 @@ config SUN6I_MSGBOX various Allwinner SoCs. This mailbox is used for communication between the application CPUs and the power management coprocessor. +config SPRD_MBOX + tristate "Spreadtrum Mailbox" + depends on ARCH_SPRD || COMPILE_TEST + help + Mailbox driver implementation for the Spreadtrum platform. It is used + to send message between application processors and MCU. Say Y here if + you want to build the Spreatrum mailbox controller driver. + +config QCOM_IPCC + bool "Qualcomm Technologies, Inc. IPCC driver" + depends on ARCH_QCOM || COMPILE_TEST + help + Qualcomm Technologies, Inc. Inter-Processor Communication Controller + (IPCC) driver for MSM devices. The driver provides mailbox support for + sending interrupts to the clients. On the other hand, the driver also + acts as an interrupt controller for receiving interrupts from clients. + Say Y here if you want to build this driver. + endif diff --git a/drivers/mailbox/Makefile b/drivers/mailbox/Makefile index 2e4364ef5c47..60d224b723a1 100644 --- a/drivers/mailbox/Makefile +++ b/drivers/mailbox/Makefile @@ -50,3 +50,7 @@ obj-$(CONFIG_MTK_CMDQ_MBOX) += mtk-cmdq-mailbox.o obj-$(CONFIG_ZYNQMP_IPI_MBOX) += zynqmp-ipi-mailbox.o obj-$(CONFIG_SUN6I_MSGBOX) += sun6i-msgbox.o + +obj-$(CONFIG_SPRD_MBOX) += sprd-mailbox.o + +obj-$(CONFIG_QCOM_IPCC) += qcom-ipcc.o diff --git a/drivers/mailbox/imx-mailbox.c b/drivers/mailbox/imx-mailbox.c index 7906624a731c..7205b825c8b5 100644 --- a/drivers/mailbox/imx-mailbox.c +++ b/drivers/mailbox/imx-mailbox.c @@ -12,6 +12,7 @@ #include <linux/mailbox_controller.h> #include <linux/module.h> #include <linux/of_device.h> +#include <linux/pm_runtime.h> #include <linux/slab.h> #define IMX_MU_xSR_GIPn(x) BIT(28 + (3 - (x))) @@ -66,6 +67,8 @@ struct imx_mu_priv { struct clk *clk; int irq; + u32 xcr; + bool side_b; }; @@ -154,12 +157,17 @@ static int imx_mu_scu_tx(struct imx_mu_priv *priv, switch (cp->type) { case IMX_MU_TYPE_TX: - if (msg->hdr.size > sizeof(*msg)) { + /* + * msg->hdr.size specifies the number of u32 words while + * sizeof yields bytes. + */ + + if (msg->hdr.size > sizeof(*msg) / 4) { /* * The real message size can be different to * struct imx_sc_rpc_msg_max size */ - dev_err(priv->dev, "Exceed max msg size (%zu) on TX, got: %i\n", sizeof(*msg), msg->hdr.size); + dev_err(priv->dev, "Maximal message size (%zu bytes) exceeded on TX; got: %i bytes\n", sizeof(*msg), msg->hdr.size << 2); return -EINVAL; } @@ -198,9 +206,8 @@ static int imx_mu_scu_rx(struct imx_mu_priv *priv, imx_mu_xcr_rmw(priv, 0, IMX_MU_xCR_RIEn(0)); *data++ = imx_mu_read(priv, priv->dcfg->xRR[0]); - if (msg.hdr.size > sizeof(msg)) { - dev_err(priv->dev, "Exceed max msg size (%zu) on RX, got: %i\n", - sizeof(msg), msg.hdr.size); + if (msg.hdr.size > sizeof(msg) / 4) { + dev_err(priv->dev, "Maximal message size (%zu bytes) exceeded on RX; got: %i bytes\n", sizeof(msg), msg.hdr.size << 2); return -EINVAL; } @@ -285,8 +292,10 @@ static int imx_mu_startup(struct mbox_chan *chan) { struct imx_mu_priv *priv = to_imx_mu_priv(chan->mbox); struct imx_mu_con_priv *cp = chan->con_priv; + unsigned long irq_flag = IRQF_SHARED; int ret; + pm_runtime_get_sync(priv->dev); if (cp->type == IMX_MU_TYPE_TXDB) { /* Tx doorbell don't have ACK support */ tasklet_init(&cp->txdb_tasklet, imx_mu_txdb_tasklet, @@ -294,8 +303,12 @@ static int imx_mu_startup(struct mbox_chan *chan) return 0; } - ret = request_irq(priv->irq, imx_mu_isr, IRQF_SHARED | - IRQF_NO_SUSPEND, cp->irq_desc, chan); + /* IPC MU should be with IRQF_NO_SUSPEND set */ + if (!priv->dev->pm_domain) + irq_flag |= IRQF_NO_SUSPEND; + + ret = request_irq(priv->irq, imx_mu_isr, irq_flag, + cp->irq_desc, chan); if (ret) { dev_err(priv->dev, "Unable to acquire IRQ %d\n", priv->irq); @@ -323,6 +336,7 @@ static void imx_mu_shutdown(struct mbox_chan *chan) if (cp->type == IMX_MU_TYPE_TXDB) { tasklet_kill(&cp->txdb_tasklet); + pm_runtime_put_sync(priv->dev); return; } @@ -341,6 +355,7 @@ static void imx_mu_shutdown(struct mbox_chan *chan) } free_irq(priv->irq, chan); + pm_runtime_put_sync(priv->dev); } static const struct mbox_chan_ops imx_mu_ops = { @@ -374,7 +389,7 @@ static struct mbox_chan *imx_mu_scu_xlate(struct mbox_controller *mbox, break; default: dev_err(mbox->dev, "Invalid chan type: %d\n", type); - return NULL; + return ERR_PTR(-EINVAL); } if (chan >= mbox->num_chans) { @@ -508,14 +523,39 @@ static int imx_mu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - return devm_mbox_controller_register(dev, &priv->mbox); + ret = devm_mbox_controller_register(dev, &priv->mbox); + if (ret) { + clk_disable_unprepare(priv->clk); + return ret; + } + + pm_runtime_enable(dev); + + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + pm_runtime_put_noidle(dev); + goto disable_runtime_pm; + } + + ret = pm_runtime_put_sync(dev); + if (ret < 0) + goto disable_runtime_pm; + + clk_disable_unprepare(priv->clk); + + return 0; + +disable_runtime_pm: + pm_runtime_disable(dev); + clk_disable_unprepare(priv->clk); + return ret; } static int imx_mu_remove(struct platform_device *pdev) { struct imx_mu_priv *priv = platform_get_drvdata(pdev); - clk_disable_unprepare(priv->clk); + pm_runtime_disable(priv->dev); return 0; } @@ -558,12 +598,69 @@ static const struct of_device_id imx_mu_dt_ids[] = { }; MODULE_DEVICE_TABLE(of, imx_mu_dt_ids); +static int imx_mu_suspend_noirq(struct device *dev) +{ + struct imx_mu_priv *priv = dev_get_drvdata(dev); + + if (!priv->clk) + priv->xcr = imx_mu_read(priv, priv->dcfg->xCR); + + return 0; +} + +static int imx_mu_resume_noirq(struct device *dev) +{ + struct imx_mu_priv *priv = dev_get_drvdata(dev); + + /* + * ONLY restore MU when context lost, the TIE could + * be set during noirq resume as there is MU data + * communication going on, and restore the saved + * value will overwrite the TIE and cause MU data + * send failed, may lead to system freeze. This issue + * is observed by testing freeze mode suspend. + */ + if (!imx_mu_read(priv, priv->dcfg->xCR) && !priv->clk) + imx_mu_write(priv, priv->xcr, priv->dcfg->xCR); + + return 0; +} + +static int imx_mu_runtime_suspend(struct device *dev) +{ + struct imx_mu_priv *priv = dev_get_drvdata(dev); + + clk_disable_unprepare(priv->clk); + + return 0; +} + +static int imx_mu_runtime_resume(struct device *dev) +{ + struct imx_mu_priv *priv = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(priv->clk); + if (ret) + dev_err(dev, "failed to enable clock\n"); + + return ret; +} + +static const struct dev_pm_ops imx_mu_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(imx_mu_suspend_noirq, + imx_mu_resume_noirq) + SET_RUNTIME_PM_OPS(imx_mu_runtime_suspend, + imx_mu_runtime_resume, NULL) +}; + static struct platform_driver imx_mu_driver = { .probe = imx_mu_probe, .remove = imx_mu_remove, .driver = { .name = "imx_mu", .of_match_table = imx_mu_dt_ids, + .pm = &imx_mu_pm_ops, }, }; module_platform_driver(imx_mu_driver); diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 34844b7a3675..8c7fac38bb1c 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -568,7 +568,7 @@ static int pcc_mbox_probe(struct platform_device *pdev) return ret; } -struct platform_driver pcc_mbox_driver = { +static struct platform_driver pcc_mbox_driver = { .probe = pcc_mbox_probe, .driver = { .name = "PCCT", diff --git a/drivers/mailbox/qcom-apcs-ipc-mailbox.c b/drivers/mailbox/qcom-apcs-ipc-mailbox.c index eeebafd546e5..cec34f0af6ce 100644 --- a/drivers/mailbox/qcom-apcs-ipc-mailbox.c +++ b/drivers/mailbox/qcom-apcs-ipc-mailbox.c @@ -24,6 +24,35 @@ struct qcom_apcs_ipc { struct platform_device *clk; }; +struct qcom_apcs_ipc_data { + int offset; + char *clk_name; +}; + +static const struct qcom_apcs_ipc_data ipq6018_apcs_data = { + .offset = 8, .clk_name = "qcom,apss-ipq6018-clk" +}; + +static const struct qcom_apcs_ipc_data ipq8074_apcs_data = { + .offset = 8, .clk_name = NULL +}; + +static const struct qcom_apcs_ipc_data msm8916_apcs_data = { + .offset = 8, .clk_name = "qcom-apcs-msm8916-clk" +}; + +static const struct qcom_apcs_ipc_data msm8996_apcs_data = { + .offset = 16, .clk_name = NULL +}; + +static const struct qcom_apcs_ipc_data msm8998_apcs_data = { + .offset = 8, .clk_name = NULL +}; + +static const struct qcom_apcs_ipc_data apps_shared_apcs_data = { + .offset = 12, .clk_name = NULL +}; + static const struct regmap_config apcs_regmap_config = { .reg_bits = 32, .reg_stride = 4, @@ -48,17 +77,12 @@ static const struct mbox_chan_ops qcom_apcs_ipc_ops = { static int qcom_apcs_ipc_probe(struct platform_device *pdev) { struct qcom_apcs_ipc *apcs; + const struct qcom_apcs_ipc_data *apcs_data; struct regmap *regmap; struct resource *res; - unsigned long offset; void __iomem *base; unsigned long i; int ret; - const struct of_device_id apcs_clk_match_table[] = { - { .compatible = "qcom,msm8916-apcs-kpss-global", }, - { .compatible = "qcom,qcs404-apcs-apps-global", }, - {} - }; apcs = devm_kzalloc(&pdev->dev, sizeof(*apcs), GFP_KERNEL); if (!apcs) @@ -73,10 +97,10 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev) if (IS_ERR(regmap)) return PTR_ERR(regmap); - offset = (unsigned long)of_device_get_match_data(&pdev->dev); + apcs_data = of_device_get_match_data(&pdev->dev); apcs->regmap = regmap; - apcs->offset = offset; + apcs->offset = apcs_data->offset; /* Initialize channel identifiers */ for (i = 0; i < ARRAY_SIZE(apcs->mbox_chans); i++) @@ -93,9 +117,9 @@ static int qcom_apcs_ipc_probe(struct platform_device *pdev) return ret; } - if (of_match_device(apcs_clk_match_table, &pdev->dev)) { + if (apcs_data->clk_name) { apcs->clk = platform_device_register_data(&pdev->dev, - "qcom-apcs-msm8916-clk", + apcs_data->clk_name, PLATFORM_DEVID_NONE, NULL, 0); if (IS_ERR(apcs->clk)) @@ -119,14 +143,15 @@ static int qcom_apcs_ipc_remove(struct platform_device *pdev) /* .data is the offset of the ipc register within the global block */ static const struct of_device_id qcom_apcs_ipc_of_match[] = { - { .compatible = "qcom,msm8916-apcs-kpss-global", .data = (void *)8 }, - { .compatible = "qcom,msm8996-apcs-hmss-global", .data = (void *)16 }, - { .compatible = "qcom,msm8998-apcs-hmss-global", .data = (void *)8 }, - { .compatible = "qcom,qcs404-apcs-apps-global", .data = (void *)8 }, - { .compatible = "qcom,sc7180-apss-shared", .data = (void *)12 }, - { .compatible = "qcom,sdm845-apss-shared", .data = (void *)12 }, - { .compatible = "qcom,sm8150-apss-shared", .data = (void *)12 }, - { .compatible = "qcom,ipq8074-apcs-apps-global", .data = (void *)8 }, + { .compatible = "qcom,ipq6018-apcs-apps-global", .data = &ipq6018_apcs_data }, + { .compatible = "qcom,ipq8074-apcs-apps-global", .data = &ipq8074_apcs_data }, + { .compatible = "qcom,msm8916-apcs-kpss-global", .data = &msm8916_apcs_data }, + { .compatible = "qcom,msm8996-apcs-hmss-global", .data = &msm8996_apcs_data }, + { .compatible = "qcom,msm8998-apcs-hmss-global", .data = &msm8998_apcs_data }, + { .compatible = "qcom,qcs404-apcs-apps-global", .data = &msm8916_apcs_data }, + { .compatible = "qcom,sc7180-apss-shared", .data = &apps_shared_apcs_data }, + { .compatible = "qcom,sdm845-apss-shared", .data = &apps_shared_apcs_data }, + { .compatible = "qcom,sm8150-apss-shared", .data = &apps_shared_apcs_data }, {} }; MODULE_DEVICE_TABLE(of, qcom_apcs_ipc_of_match); diff --git a/drivers/mailbox/qcom-ipcc.c b/drivers/mailbox/qcom-ipcc.c new file mode 100644 index 000000000000..2d13c72944c6 --- /dev/null +++ b/drivers/mailbox/qcom-ipcc.c @@ -0,0 +1,286 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + */ + +#include <linux/bitfield.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqdomain.h> +#include <linux/mailbox_controller.h> +#include <linux/module.h> +#include <linux/platform_device.h> + +#include <dt-bindings/mailbox/qcom-ipcc.h> + +#define IPCC_MBOX_MAX_CHAN 48 + +/* IPCC Register offsets */ +#define IPCC_REG_SEND_ID 0x0c +#define IPCC_REG_RECV_ID 0x10 +#define IPCC_REG_RECV_SIGNAL_ENABLE 0x14 +#define IPCC_REG_RECV_SIGNAL_DISABLE 0x18 +#define IPCC_REG_RECV_SIGNAL_CLEAR 0x1c +#define IPCC_REG_CLIENT_CLEAR 0x38 + +#define IPCC_SIGNAL_ID_MASK GENMASK(15, 0) +#define IPCC_CLIENT_ID_MASK GENMASK(31, 16) + +#define IPCC_NO_PENDING_IRQ GENMASK(31, 0) + +/** + * struct qcom_ipcc_chan_info - Per-mailbox-channel info + * @client_id: The client-id to which the interrupt has to be triggered + * @signal_id: The signal-id to which the interrupt has to be triggered + */ +struct qcom_ipcc_chan_info { + u16 client_id; + u16 signal_id; +}; + +/** + * struct qcom_ipcc - Holder for the mailbox driver + * @dev: Device associated with this instance + * @base: Base address of the IPCC frame associated to APSS + * @irq_domain: The irq_domain associated with this instance + * @chan: The mailbox channels array + * @mchan: The per-mailbox channel info array + * @mbox: The mailbox controller + * @irq: Summary irq + */ +struct qcom_ipcc { + struct device *dev; + void __iomem *base; + struct irq_domain *irq_domain; + struct mbox_chan chan[IPCC_MBOX_MAX_CHAN]; + struct qcom_ipcc_chan_info mchan[IPCC_MBOX_MAX_CHAN]; + struct mbox_controller mbox; + int irq; +}; + +static inline struct qcom_ipcc *to_qcom_ipcc(struct mbox_controller *mbox) +{ + return container_of(mbox, struct qcom_ipcc, mbox); +} + +static inline u32 qcom_ipcc_get_hwirq(u16 client_id, u16 signal_id) +{ + return FIELD_PREP(IPCC_CLIENT_ID_MASK, client_id) | + FIELD_PREP(IPCC_SIGNAL_ID_MASK, signal_id); +} + +static irqreturn_t qcom_ipcc_irq_fn(int irq, void *data) +{ + struct qcom_ipcc *ipcc = data; + u32 hwirq; + int virq; + + for (;;) { + hwirq = readl(ipcc->base + IPCC_REG_RECV_ID); + if (hwirq == IPCC_NO_PENDING_IRQ) + break; + + virq = irq_find_mapping(ipcc->irq_domain, hwirq); + writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_CLEAR); + generic_handle_irq(virq); + } + + return IRQ_HANDLED; +} + +static void qcom_ipcc_mask_irq(struct irq_data *irqd) +{ + struct qcom_ipcc *ipcc = irq_data_get_irq_chip_data(irqd); + irq_hw_number_t hwirq = irqd_to_hwirq(irqd); + + writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_DISABLE); +} + +static void qcom_ipcc_unmask_irq(struct irq_data *irqd) +{ + struct qcom_ipcc *ipcc = irq_data_get_irq_chip_data(irqd); + irq_hw_number_t hwirq = irqd_to_hwirq(irqd); + + writel(hwirq, ipcc->base + IPCC_REG_RECV_SIGNAL_ENABLE); +} + +static struct irq_chip qcom_ipcc_irq_chip = { + .name = "ipcc", + .irq_mask = qcom_ipcc_mask_irq, + .irq_unmask = qcom_ipcc_unmask_irq, + .flags = IRQCHIP_SKIP_SET_WAKE, +}; + +static int qcom_ipcc_domain_map(struct irq_domain *d, unsigned int irq, + irq_hw_number_t hw) +{ + struct qcom_ipcc *ipcc = d->host_data; + + irq_set_chip_and_handler(irq, &qcom_ipcc_irq_chip, handle_level_irq); + irq_set_chip_data(irq, ipcc); + irq_set_noprobe(irq); + + return 0; +} + +static int qcom_ipcc_domain_xlate(struct irq_domain *d, + struct device_node *node, const u32 *intspec, + unsigned int intsize, + unsigned long *out_hwirq, + unsigned int *out_type) +{ + if (intsize != 3) + return -EINVAL; + + *out_hwirq = qcom_ipcc_get_hwirq(intspec[0], intspec[1]); + *out_type = intspec[2] & IRQ_TYPE_SENSE_MASK; + + return 0; +} + +static const struct irq_domain_ops qcom_ipcc_irq_ops = { + .map = qcom_ipcc_domain_map, + .xlate = qcom_ipcc_domain_xlate, +}; + +static int qcom_ipcc_mbox_send_data(struct mbox_chan *chan, void *data) +{ + struct qcom_ipcc *ipcc = to_qcom_ipcc(chan->mbox); + struct qcom_ipcc_chan_info *mchan = chan->con_priv; + u32 hwirq; + + hwirq = qcom_ipcc_get_hwirq(mchan->client_id, mchan->signal_id); + writel(hwirq, ipcc->base + IPCC_REG_SEND_ID); + + return 0; +} + +static struct mbox_chan *qcom_ipcc_mbox_xlate(struct mbox_controller *mbox, + const struct of_phandle_args *ph) +{ + struct qcom_ipcc *ipcc = to_qcom_ipcc(mbox); + struct qcom_ipcc_chan_info *mchan; + struct mbox_chan *chan; + unsigned int i; + + if (ph->args_count != 2) + return ERR_PTR(-EINVAL); + + for (i = 0; i < IPCC_MBOX_MAX_CHAN; i++) { + chan = &ipcc->chan[i]; + if (!chan->con_priv) { + mchan = &ipcc->mchan[i]; + mchan->client_id = ph->args[0]; + mchan->signal_id = ph->args[1]; + chan->con_priv = mchan; + break; + } + + chan = NULL; + } + + return chan ?: ERR_PTR(-EBUSY); +} + +static const struct mbox_chan_ops ipcc_mbox_chan_ops = { + .send_data = qcom_ipcc_mbox_send_data, +}; + +static int qcom_ipcc_setup_mbox(struct qcom_ipcc *ipcc) +{ + struct mbox_controller *mbox; + struct device *dev = ipcc->dev; + + mbox = &ipcc->mbox; + mbox->dev = dev; + mbox->num_chans = IPCC_MBOX_MAX_CHAN; + mbox->chans = ipcc->chan; + mbox->ops = &ipcc_mbox_chan_ops; + mbox->of_xlate = qcom_ipcc_mbox_xlate; + mbox->txdone_irq = false; + mbox->txdone_poll = false; + + return devm_mbox_controller_register(dev, mbox); +} + +static int qcom_ipcc_probe(struct platform_device *pdev) +{ + struct qcom_ipcc *ipcc; + int ret; + + ipcc = devm_kzalloc(&pdev->dev, sizeof(*ipcc), GFP_KERNEL); + if (!ipcc) + return -ENOMEM; + + ipcc->dev = &pdev->dev; + + ipcc->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(ipcc->base)) + return PTR_ERR(ipcc->base); + + ipcc->irq = platform_get_irq(pdev, 0); + if (ipcc->irq < 0) + return ipcc->irq; + + ipcc->irq_domain = irq_domain_add_tree(pdev->dev.of_node, + &qcom_ipcc_irq_ops, ipcc); + if (!ipcc->irq_domain) + return -ENOMEM; + + ret = qcom_ipcc_setup_mbox(ipcc); + if (ret) + goto err_mbox; + + ret = devm_request_irq(&pdev->dev, ipcc->irq, qcom_ipcc_irq_fn, + IRQF_TRIGGER_HIGH, "ipcc", ipcc); + if (ret < 0) { + dev_err(&pdev->dev, "Failed to register the irq: %d\n", ret); + goto err_mbox; + } + + enable_irq_wake(ipcc->irq); + platform_set_drvdata(pdev, ipcc); + + return 0; + +err_mbox: + irq_domain_remove(ipcc->irq_domain); + + return ret; +} + +static int qcom_ipcc_remove(struct platform_device *pdev) +{ + struct qcom_ipcc *ipcc = platform_get_drvdata(pdev); + + disable_irq_wake(ipcc->irq); + irq_domain_remove(ipcc->irq_domain); + + return 0; +} + +static const struct of_device_id qcom_ipcc_of_match[] = { + { .compatible = "qcom,ipcc"}, + {} +}; +MODULE_DEVICE_TABLE(of, qcom_ipcc_of_match); + +static struct platform_driver qcom_ipcc_driver = { + .probe = qcom_ipcc_probe, + .remove = qcom_ipcc_remove, + .driver = { + .name = "qcom-ipcc", + .of_match_table = qcom_ipcc_of_match, + }, +}; + +static int __init qcom_ipcc_init(void) +{ + return platform_driver_register(&qcom_ipcc_driver); +} +arch_initcall(qcom_ipcc_init); + +MODULE_AUTHOR("Venkata Narendra Kumar Gutta <vnkgutta@codeaurora.org>"); +MODULE_AUTHOR("Manivannan Sadhasivam <manivannan.sadhasivam@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm Technologies, Inc. IPCC driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mailbox/sprd-mailbox.c b/drivers/mailbox/sprd-mailbox.c new file mode 100644 index 000000000000..f6fab24ae8a9 --- /dev/null +++ b/drivers/mailbox/sprd-mailbox.c @@ -0,0 +1,361 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Spreadtrum mailbox driver + * + * Copyright (c) 2020 Spreadtrum Communications Inc. + */ + +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/mailbox_controller.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/clk.h> + +#define SPRD_MBOX_ID 0x0 +#define SPRD_MBOX_MSG_LOW 0x4 +#define SPRD_MBOX_MSG_HIGH 0x8 +#define SPRD_MBOX_TRIGGER 0xc +#define SPRD_MBOX_FIFO_RST 0x10 +#define SPRD_MBOX_FIFO_STS 0x14 +#define SPRD_MBOX_IRQ_STS 0x18 +#define SPRD_MBOX_IRQ_MSK 0x1c +#define SPRD_MBOX_LOCK 0x20 +#define SPRD_MBOX_FIFO_DEPTH 0x24 + +/* Bit and mask definiation for inbox's SPRD_MBOX_FIFO_STS register */ +#define SPRD_INBOX_FIFO_DELIVER_MASK GENMASK(23, 16) +#define SPRD_INBOX_FIFO_OVERLOW_MASK GENMASK(15, 8) +#define SPRD_INBOX_FIFO_DELIVER_SHIFT 16 +#define SPRD_INBOX_FIFO_BUSY_MASK GENMASK(7, 0) + +/* Bit and mask definiation for SPRD_MBOX_IRQ_STS register */ +#define SPRD_MBOX_IRQ_CLR BIT(0) + +/* Bit and mask definiation for outbox's SPRD_MBOX_FIFO_STS register */ +#define SPRD_OUTBOX_FIFO_FULL BIT(0) +#define SPRD_OUTBOX_FIFO_WR_SHIFT 16 +#define SPRD_OUTBOX_FIFO_RD_SHIFT 24 +#define SPRD_OUTBOX_FIFO_POS_MASK GENMASK(7, 0) + +/* Bit and mask definiation for inbox's SPRD_MBOX_IRQ_MSK register */ +#define SPRD_INBOX_FIFO_BLOCK_IRQ BIT(0) +#define SPRD_INBOX_FIFO_OVERFLOW_IRQ BIT(1) +#define SPRD_INBOX_FIFO_DELIVER_IRQ BIT(2) +#define SPRD_INBOX_FIFO_IRQ_MASK GENMASK(2, 0) + +/* Bit and mask definiation for outbox's SPRD_MBOX_IRQ_MSK register */ +#define SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ BIT(0) +#define SPRD_OUTBOX_FIFO_IRQ_MASK GENMASK(4, 0) + +#define SPRD_MBOX_CHAN_MAX 8 + +struct sprd_mbox_priv { + struct mbox_controller mbox; + struct device *dev; + void __iomem *inbox_base; + void __iomem *outbox_base; + struct clk *clk; + u32 outbox_fifo_depth; + + struct mbox_chan chan[SPRD_MBOX_CHAN_MAX]; +}; + +static struct sprd_mbox_priv *to_sprd_mbox_priv(struct mbox_controller *mbox) +{ + return container_of(mbox, struct sprd_mbox_priv, mbox); +} + +static u32 sprd_mbox_get_fifo_len(struct sprd_mbox_priv *priv, u32 fifo_sts) +{ + u32 wr_pos = (fifo_sts >> SPRD_OUTBOX_FIFO_WR_SHIFT) & + SPRD_OUTBOX_FIFO_POS_MASK; + u32 rd_pos = (fifo_sts >> SPRD_OUTBOX_FIFO_RD_SHIFT) & + SPRD_OUTBOX_FIFO_POS_MASK; + u32 fifo_len; + + /* + * If the read pointer is equal with write pointer, which means the fifo + * is full or empty. + */ + if (wr_pos == rd_pos) { + if (fifo_sts & SPRD_OUTBOX_FIFO_FULL) + fifo_len = priv->outbox_fifo_depth; + else + fifo_len = 0; + } else if (wr_pos > rd_pos) { + fifo_len = wr_pos - rd_pos; + } else { + fifo_len = priv->outbox_fifo_depth - rd_pos + wr_pos; + } + + return fifo_len; +} + +static irqreturn_t sprd_mbox_outbox_isr(int irq, void *data) +{ + struct sprd_mbox_priv *priv = data; + struct mbox_chan *chan; + u32 fifo_sts, fifo_len, msg[2]; + int i, id; + + fifo_sts = readl(priv->outbox_base + SPRD_MBOX_FIFO_STS); + + fifo_len = sprd_mbox_get_fifo_len(priv, fifo_sts); + if (!fifo_len) { + dev_warn_ratelimited(priv->dev, "spurious outbox interrupt\n"); + return IRQ_NONE; + } + + for (i = 0; i < fifo_len; i++) { + msg[0] = readl(priv->outbox_base + SPRD_MBOX_MSG_LOW); + msg[1] = readl(priv->outbox_base + SPRD_MBOX_MSG_HIGH); + id = readl(priv->outbox_base + SPRD_MBOX_ID); + + chan = &priv->chan[id]; + mbox_chan_received_data(chan, (void *)msg); + + /* Trigger to update outbox FIFO pointer */ + writel(0x1, priv->outbox_base + SPRD_MBOX_TRIGGER); + } + + /* Clear irq status after reading all message. */ + writel(SPRD_MBOX_IRQ_CLR, priv->outbox_base + SPRD_MBOX_IRQ_STS); + + return IRQ_HANDLED; +} + +static irqreturn_t sprd_mbox_inbox_isr(int irq, void *data) +{ + struct sprd_mbox_priv *priv = data; + struct mbox_chan *chan; + u32 fifo_sts, send_sts, busy, id; + + fifo_sts = readl(priv->inbox_base + SPRD_MBOX_FIFO_STS); + + /* Get the inbox data delivery status */ + send_sts = (fifo_sts & SPRD_INBOX_FIFO_DELIVER_MASK) >> + SPRD_INBOX_FIFO_DELIVER_SHIFT; + if (!send_sts) { + dev_warn_ratelimited(priv->dev, "spurious inbox interrupt\n"); + return IRQ_NONE; + } + + while (send_sts) { + id = __ffs(send_sts); + send_sts &= (send_sts - 1); + + chan = &priv->chan[id]; + + /* + * Check if the message was fetched by remote traget, if yes, + * that means the transmission has been completed. + */ + busy = fifo_sts & SPRD_INBOX_FIFO_BUSY_MASK; + if (!(busy & BIT(id))) + mbox_chan_txdone(chan, 0); + } + + /* Clear FIFO delivery and overflow status */ + writel(fifo_sts & + (SPRD_INBOX_FIFO_DELIVER_MASK | SPRD_INBOX_FIFO_OVERLOW_MASK), + priv->inbox_base + SPRD_MBOX_FIFO_RST); + + /* Clear irq status */ + writel(SPRD_MBOX_IRQ_CLR, priv->inbox_base + SPRD_MBOX_IRQ_STS); + + return IRQ_HANDLED; +} + +static int sprd_mbox_send_data(struct mbox_chan *chan, void *msg) +{ + struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); + unsigned long id = (unsigned long)chan->con_priv; + u32 *data = msg; + + /* Write data into inbox FIFO, and only support 8 bytes every time */ + writel(data[0], priv->inbox_base + SPRD_MBOX_MSG_LOW); + writel(data[1], priv->inbox_base + SPRD_MBOX_MSG_HIGH); + + /* Set target core id */ + writel(id, priv->inbox_base + SPRD_MBOX_ID); + + /* Trigger remote request */ + writel(0x1, priv->inbox_base + SPRD_MBOX_TRIGGER); + + return 0; +} + +static int sprd_mbox_flush(struct mbox_chan *chan, unsigned long timeout) +{ + struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); + unsigned long id = (unsigned long)chan->con_priv; + u32 busy; + + timeout = jiffies + msecs_to_jiffies(timeout); + + while (time_before(jiffies, timeout)) { + busy = readl(priv->inbox_base + SPRD_MBOX_FIFO_STS) & + SPRD_INBOX_FIFO_BUSY_MASK; + if (!(busy & BIT(id))) { + mbox_chan_txdone(chan, 0); + return 0; + } + + udelay(1); + } + + return -ETIME; +} + +static int sprd_mbox_startup(struct mbox_chan *chan) +{ + struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); + u32 val; + + /* Select outbox FIFO mode and reset the outbox FIFO status */ + writel(0x0, priv->outbox_base + SPRD_MBOX_FIFO_RST); + + /* Enable inbox FIFO overflow and delivery interrupt */ + val = readl(priv->inbox_base + SPRD_MBOX_IRQ_MSK); + val &= ~(SPRD_INBOX_FIFO_OVERFLOW_IRQ | SPRD_INBOX_FIFO_DELIVER_IRQ); + writel(val, priv->inbox_base + SPRD_MBOX_IRQ_MSK); + + /* Enable outbox FIFO not empty interrupt */ + val = readl(priv->outbox_base + SPRD_MBOX_IRQ_MSK); + val &= ~SPRD_OUTBOX_FIFO_NOT_EMPTY_IRQ; + writel(val, priv->outbox_base + SPRD_MBOX_IRQ_MSK); + + return 0; +} + +static void sprd_mbox_shutdown(struct mbox_chan *chan) +{ + struct sprd_mbox_priv *priv = to_sprd_mbox_priv(chan->mbox); + + /* Disable inbox & outbox interrupt */ + writel(SPRD_INBOX_FIFO_IRQ_MASK, priv->inbox_base + SPRD_MBOX_IRQ_MSK); + writel(SPRD_OUTBOX_FIFO_IRQ_MASK, priv->outbox_base + SPRD_MBOX_IRQ_MSK); +} + +static const struct mbox_chan_ops sprd_mbox_ops = { + .send_data = sprd_mbox_send_data, + .flush = sprd_mbox_flush, + .startup = sprd_mbox_startup, + .shutdown = sprd_mbox_shutdown, +}; + +static void sprd_mbox_disable(void *data) +{ + struct sprd_mbox_priv *priv = data; + + clk_disable_unprepare(priv->clk); +} + +static int sprd_mbox_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct sprd_mbox_priv *priv; + int ret, inbox_irq, outbox_irq; + unsigned long id; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->dev = dev; + + /* + * The Spreadtrum mailbox uses an inbox to send messages to the target + * core, and uses an outbox to receive messages from other cores. + * + * Thus the mailbox controller supplies 2 different register addresses + * and IRQ numbers for inbox and outbox. + */ + priv->inbox_base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->inbox_base)) + return PTR_ERR(priv->inbox_base); + + priv->outbox_base = devm_platform_ioremap_resource(pdev, 1); + if (IS_ERR(priv->outbox_base)) + return PTR_ERR(priv->outbox_base); + + priv->clk = devm_clk_get(dev, "enable"); + if (IS_ERR(priv->clk)) { + dev_err(dev, "failed to get mailbox clock\n"); + return PTR_ERR(priv->clk); + } + + ret = clk_prepare_enable(priv->clk); + if (ret) + return ret; + + ret = devm_add_action_or_reset(dev, sprd_mbox_disable, priv); + if (ret) { + dev_err(dev, "failed to add mailbox disable action\n"); + return ret; + } + + inbox_irq = platform_get_irq(pdev, 0); + if (inbox_irq < 0) + return inbox_irq; + + ret = devm_request_irq(dev, inbox_irq, sprd_mbox_inbox_isr, + IRQF_NO_SUSPEND, dev_name(dev), priv); + if (ret) { + dev_err(dev, "failed to request inbox IRQ: %d\n", ret); + return ret; + } + + outbox_irq = platform_get_irq(pdev, 1); + if (outbox_irq < 0) + return outbox_irq; + + ret = devm_request_irq(dev, outbox_irq, sprd_mbox_outbox_isr, + IRQF_NO_SUSPEND, dev_name(dev), priv); + if (ret) { + dev_err(dev, "failed to request outbox IRQ: %d\n", ret); + return ret; + } + + /* Get the default outbox FIFO depth */ + priv->outbox_fifo_depth = + readl(priv->outbox_base + SPRD_MBOX_FIFO_DEPTH) + 1; + priv->mbox.dev = dev; + priv->mbox.chans = &priv->chan[0]; + priv->mbox.num_chans = SPRD_MBOX_CHAN_MAX; + priv->mbox.ops = &sprd_mbox_ops; + priv->mbox.txdone_irq = true; + + for (id = 0; id < SPRD_MBOX_CHAN_MAX; id++) + priv->chan[id].con_priv = (void *)id; + + ret = devm_mbox_controller_register(dev, &priv->mbox); + if (ret) { + dev_err(dev, "failed to register mailbox: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct of_device_id sprd_mbox_of_match[] = { + { .compatible = "sprd,sc9860-mailbox", }, + { }, +}; +MODULE_DEVICE_TABLE(of, sprd_mbox_of_match); + +static struct platform_driver sprd_mbox_driver = { + .driver = { + .name = "sprd-mailbox", + .of_match_table = sprd_mbox_of_match, + }, + .probe = sprd_mbox_probe, +}; +module_platform_driver(sprd_mbox_driver); + +MODULE_AUTHOR("Baolin Wang <baolin.wang@unisoc.com>"); +MODULE_DESCRIPTION("Spreadtrum mailbox driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/mailbox/zynqmp-ipi-mailbox.c b/drivers/mailbox/zynqmp-ipi-mailbox.c index 86887c9a349a..f44079d62b1a 100644 --- a/drivers/mailbox/zynqmp-ipi-mailbox.c +++ b/drivers/mailbox/zynqmp-ipi-mailbox.c @@ -504,10 +504,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, mchan->req_buf_size = resource_size(&res); mchan->req_buf = devm_ioremap(mdev, res.start, mchan->req_buf_size); - if (IS_ERR(mchan->req_buf)) { + if (!mchan->req_buf) { dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); - ret = PTR_ERR(mchan->req_buf); - return ret; + return -ENOMEM; } } else if (ret != -ENODEV) { dev_err(mdev, "Unmatched resource %s, %d.\n", name, ret); @@ -520,10 +519,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, mchan->resp_buf_size = resource_size(&res); mchan->resp_buf = devm_ioremap(mdev, res.start, mchan->resp_buf_size); - if (IS_ERR(mchan->resp_buf)) { + if (!mchan->resp_buf) { dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); - ret = PTR_ERR(mchan->resp_buf); - return ret; + return -ENOMEM; } } else if (ret != -ENODEV) { dev_err(mdev, "Unmatched resource %s.\n", name); @@ -543,10 +541,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, mchan->req_buf_size = resource_size(&res); mchan->req_buf = devm_ioremap(mdev, res.start, mchan->req_buf_size); - if (IS_ERR(mchan->req_buf)) { + if (!mchan->req_buf) { dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); - ret = PTR_ERR(mchan->req_buf); - return ret; + return -ENOMEM; } } else if (ret != -ENODEV) { dev_err(mdev, "Unmatched resource %s.\n", name); @@ -559,10 +556,9 @@ static int zynqmp_ipi_mbox_probe(struct zynqmp_ipi_mbox *ipi_mbox, mchan->resp_buf_size = resource_size(&res); mchan->resp_buf = devm_ioremap(mdev, res.start, mchan->resp_buf_size); - if (IS_ERR(mchan->resp_buf)) { + if (!mchan->resp_buf) { dev_err(mdev, "Unable to map IPI buffer I/O memory\n"); - ret = PTR_ERR(mchan->resp_buf); - return ret; + return -ENOMEM; } } else if (ret != -ENODEV) { dev_err(mdev, "Unmatched resource %s.\n", name); @@ -668,10 +664,9 @@ static int zynqmp_ipi_probe(struct platform_device *pdev) /* IPI IRQ */ ret = platform_get_irq(pdev, 0); - if (ret < 0) { - dev_err(dev, "unable to find IPI IRQ.\n"); + if (ret < 0) goto free_mbox_dev; - } + pdata->irq = ret; ret = devm_request_irq(dev, pdata->irq, zynqmp_ipi_interrupt, IRQF_SHARED, dev_name(dev), pdata); diff --git a/drivers/media/cec/Kconfig b/drivers/media/cec/Kconfig index eea74b7cfa8c..7e830444bdbb 100644 --- a/drivers/media/cec/Kconfig +++ b/drivers/media/cec/Kconfig @@ -24,7 +24,7 @@ config CEC_PIN_ERROR_INJ menuconfig MEDIA_CEC_SUPPORT bool prompt "HDMI CEC drivers" - default y if !MEDIA_SUPPORT_FILTER + default y if MEDIA_SUPPORT && !MEDIA_SUPPORT_FILTER help Enable support for HDMI CEC (Consumer Electronics Control), which is an optional HDMI feature. diff --git a/drivers/media/common/videobuf2/videobuf2-dma-contig.c b/drivers/media/common/videobuf2/videobuf2-dma-contig.c index d3a3ee5b597b..f4b4a7c135eb 100644 --- a/drivers/media/common/videobuf2/videobuf2-dma-contig.c +++ b/drivers/media/common/videobuf2/videobuf2-dma-contig.c @@ -726,9 +726,8 @@ EXPORT_SYMBOL_GPL(vb2_dma_contig_memops); int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size) { if (!dev->dma_parms) { - dev->dma_parms = kzalloc(sizeof(*dev->dma_parms), GFP_KERNEL); - if (!dev->dma_parms) - return -ENOMEM; + dev_err(dev, "Failed to set max_seg_size: dma_parms is NULL\n"); + return -ENODEV; } if (dma_get_max_seg_size(dev) < size) return dma_set_max_seg_size(dev, size); @@ -737,21 +736,6 @@ int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size) } EXPORT_SYMBOL_GPL(vb2_dma_contig_set_max_seg_size); -/* - * vb2_dma_contig_clear_max_seg_size() - release resources for DMA parameters - * @dev: device for configuring DMA parameters - * - * This function releases resources allocated to configure DMA parameters - * (see vb2_dma_contig_set_max_seg_size() function). It should be called from - * device drivers on driver remove. - */ -void vb2_dma_contig_clear_max_seg_size(struct device *dev) -{ - kfree(dev->dma_parms); - dev->dma_parms = NULL; -} -EXPORT_SYMBOL_GPL(vb2_dma_contig_clear_max_seg_size); - MODULE_DESCRIPTION("DMA-contig memory handling routines for videobuf2"); MODULE_AUTHOR("Pawel Osciak <pawel@osciak.com>"); MODULE_LICENSE("GPL"); diff --git a/drivers/media/platform/s5p-mfc/s5p_mfc.c b/drivers/media/platform/s5p-mfc/s5p_mfc.c index 5c2a23b953a4..eba2b9f040df 100644 --- a/drivers/media/platform/s5p-mfc/s5p_mfc.c +++ b/drivers/media/platform/s5p-mfc/s5p_mfc.c @@ -1089,6 +1089,10 @@ static struct device *s5p_mfc_alloc_memdev(struct device *dev, child->coherent_dma_mask = dev->coherent_dma_mask; child->dma_mask = dev->dma_mask; child->release = s5p_mfc_memdev_release; + child->dma_parms = devm_kzalloc(dev, sizeof(*child->dma_parms), + GFP_KERNEL); + if (!child->dma_parms) + goto err; /* * The memdevs are not proper OF platform devices, so in order for them @@ -1104,7 +1108,7 @@ static struct device *s5p_mfc_alloc_memdev(struct device *dev, return child; device_del(child); } - +err: put_device(child); return NULL; } diff --git a/drivers/media/v4l2-core/v4l2-ctrls.c b/drivers/media/v4l2-core/v4l2-ctrls.c index b188577db40f..3f3fbcd60cc6 100644 --- a/drivers/media/v4l2-core/v4l2-ctrls.c +++ b/drivers/media/v4l2-core/v4l2-ctrls.c @@ -1844,7 +1844,7 @@ static int std_validate_compound(const struct v4l2_ctrl *ctrl, u32 idx, sizeof(p_hevc_pps->row_height_minus1)); p_hevc_pps->flags &= - ~V4L2_HEVC_PPS_FLAG_PPS_LOOP_FILTER_ACROSS_SLICES_ENABLED; + ~V4L2_HEVC_PPS_FLAG_LOOP_FILTER_ACROSS_TILES_ENABLED; } if (p_hevc_pps->flags & diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index 036e23aef9b0..39030a324d7f 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -165,7 +165,7 @@ static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) { struct bio_integrity_payload *bip; - struct nd_namespace_blk *nsblk; + struct nd_namespace_blk *nsblk = bio->bi_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -176,7 +176,6 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; bip = bio_integrity(bio); - nsblk = q->queuedata; rw = bio_data_dir(bio); do_acct = blk_queue_io_stat(bio->bi_disk->queue); if (do_acct) @@ -260,7 +259,6 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) blk_queue_max_hw_sectors(q, UINT_MAX); blk_queue_logical_block_size(q, nsblk_sector_size(nsblk)); blk_queue_flag_set(QUEUE_FLAG_NONROT, q); - q->queuedata = nsblk; disk = alloc_disk(0); if (!disk) @@ -270,6 +268,7 @@ static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) disk->fops = &nd_blk_fops; disk->queue = q; disk->flags = GENHD_FL_EXT_DEVT; + disk->private_data = nsblk; nvdimm_namespace_disk_name(&nsblk->common, disk->disk_name); if (devm_add_action_or_reset(dev, nd_blk_release_disk, disk)) diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index 90c0c4bbe77b..48e9d169b6f9 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1442,7 +1442,7 @@ static int btt_do_bvec(struct btt *btt, struct bio_integrity_payload *bip, static blk_qc_t btt_make_request(struct request_queue *q, struct bio *bio) { struct bio_integrity_payload *bip = bio_integrity(bio); - struct btt *btt = q->queuedata; + struct btt *btt = bio->bi_disk->private_data; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -1545,7 +1545,6 @@ static int btt_blk_init(struct btt *btt) blk_queue_logical_block_size(btt->btt_queue, btt->sector_size); blk_queue_max_hw_sectors(btt->btt_queue, UINT_MAX); blk_queue_flag_set(QUEUE_FLAG_NONROT, btt->btt_queue); - btt->btt_queue->queuedata = btt; if (btt_meta_size(btt)) { int rc = nd_integrity_init(btt->btt_disk, btt_meta_size(btt)); diff --git a/drivers/nvdimm/pmem.c b/drivers/nvdimm/pmem.c index d1ecd6da11a2..d25e66fd942d 100644 --- a/drivers/nvdimm/pmem.c +++ b/drivers/nvdimm/pmem.c @@ -197,7 +197,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) unsigned long start; struct bio_vec bvec; struct bvec_iter iter; - struct pmem_device *pmem = q->queuedata; + struct pmem_device *pmem = bio->bi_disk->private_data; struct nd_region *nd_region = to_region(pmem); if (bio->bi_opf & REQ_PREFLUSH) @@ -234,7 +234,7 @@ static blk_qc_t pmem_make_request(struct request_queue *q, struct bio *bio) static int pmem_rw_page(struct block_device *bdev, sector_t sector, struct page *page, unsigned int op) { - struct pmem_device *pmem = bdev->bd_queue->queuedata; + struct pmem_device *pmem = bdev->bd_disk->private_data; blk_status_t rc; if (op_is_write(op)) @@ -467,7 +467,6 @@ static int pmem_attach_disk(struct device *dev, blk_queue_flag_set(QUEUE_FLAG_NONROT, q); if (pmem->pfn_flags & PFN_MAP) blk_queue_flag_set(QUEUE_FLAG_DAX, q); - q->queuedata = pmem; disk = alloc_disk_node(0, nid); if (!disk) @@ -477,6 +476,7 @@ static int pmem_attach_disk(struct device *dev, disk->fops = &pmem_fops; disk->queue = q; disk->flags = GENHD_FL_EXT_DEVT; + disk->private_data = pmem; disk->queue->backing_dev_info->capabilities |= BDI_CAP_SYNCHRONOUS_IO; nvdimm_namespace_disk_name(ndns, disk->disk_name); set_capacity(disk, (pmem->size - pmem->pfn_pad - pmem->data_offset) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 0585efa47d8f..c2c5bc4fb702 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -3669,7 +3669,7 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) ns->disk = disk; if (__nvme_revalidate_disk(disk, id)) - goto out_free_disk; + goto out_put_disk; if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) { ret = nvme_nvm_register(ns, disk_name, node); @@ -3696,8 +3696,6 @@ static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid) /* prevent double queue cleanup */ ns->disk->queue = NULL; put_disk(ns->disk); - out_free_disk: - del_gendisk(ns->disk); out_unlink_ns: mutex_lock(&ctrl->subsys->lock); list_del_rcu(&ns->siblings); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index cb0007592c12..e999a8c4b7e8 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -2634,10 +2634,11 @@ nvme_fc_start_fcp_op(struct nvme_fc_ctrl *ctrl, struct nvme_fc_queue *queue, opstate = atomic_xchg(&op->state, FCPOP_STATE_COMPLETE); __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); - if (!(op->flags & FCOP_FLAGS_AEN)) + if (!(op->flags & FCOP_FLAGS_AEN)) { nvme_fc_unmap_data(ctrl, op->rq, op); + nvme_cleanup_cmd(op->rq); + } - nvme_cleanup_cmd(op->rq); nvme_fc_ctrl_put(ctrl); if (ctrl->rport->remoteport.port_state == FC_OBJSTATE_ONLINE && diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index fa5c75501049..c0f4226d3299 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -599,8 +599,7 @@ static inline void nvme_trace_bio_complete(struct request *req, struct nvme_ns *ns = req->q->queuedata; if (req->cmd_flags & REQ_NVME_MPATH) - trace_block_bio_complete(ns->head->disk->queue, - req->bio, status); + trace_block_bio_complete(ns->head->disk->queue, req->bio); } extern struct device_attribute dev_attr_ana_grpid; diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index d690d5593a80..e2bacd369a88 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -2950,9 +2950,15 @@ static int nvme_suspend(struct device *dev) * the PCI bus layer to put it into D3 in order to take the PCIe link * down, so as to allow the platform to achieve its minimum low-power * state (which may not be possible if the link is up). + * + * If a host memory buffer is enabled, shut down the device as the NVMe + * specification allows the device to access the host memory buffer in + * host DRAM from all power states, but hosts will fail access to DRAM + * during S3. */ if (pm_suspend_via_firmware() || !ctrl->npss || !pcie_aspm_enabled(pdev) || + ndev->nr_host_mem_descs || (ndev->ctrl.quirks & NVME_QUIRK_SIMPLE_SUSPEND)) return nvme_disable_prepare_reset(ndev, true); diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 1843110ec34f..3345ec7efaff 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -131,8 +131,8 @@ struct nvme_tcp_ctrl { static LIST_HEAD(nvme_tcp_ctrl_list); static DEFINE_MUTEX(nvme_tcp_ctrl_mutex); static struct workqueue_struct *nvme_tcp_wq; -static struct blk_mq_ops nvme_tcp_mq_ops; -static struct blk_mq_ops nvme_tcp_admin_mq_ops; +static const struct blk_mq_ops nvme_tcp_mq_ops; +static const struct blk_mq_ops nvme_tcp_admin_mq_ops; static int nvme_tcp_try_send(struct nvme_tcp_queue *queue); static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl) @@ -2301,7 +2301,7 @@ static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx) return queue->nr_cqe; } -static struct blk_mq_ops nvme_tcp_mq_ops = { +static const struct blk_mq_ops nvme_tcp_mq_ops = { .queue_rq = nvme_tcp_queue_rq, .complete = nvme_complete_rq, .init_request = nvme_tcp_init_request, @@ -2312,7 +2312,7 @@ static struct blk_mq_ops nvme_tcp_mq_ops = { .poll = nvme_tcp_poll, }; -static struct blk_mq_ops nvme_tcp_admin_mq_ops = { +static const struct blk_mq_ops nvme_tcp_admin_mq_ops = { .queue_rq = nvme_tcp_queue_rq, .complete = nvme_complete_rq, .init_request = nvme_tcp_init_request, diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 6392bcd30bd7..6e2f623e472e 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -129,7 +129,22 @@ static u32 nvmet_async_event_result(struct nvmet_async_event *aen) return aen->event_type | (aen->event_info << 8) | (aen->log_page << 16); } -static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) +static void nvmet_async_events_failall(struct nvmet_ctrl *ctrl) +{ + u16 status = NVME_SC_INTERNAL | NVME_SC_DNR; + struct nvmet_req *req; + + mutex_lock(&ctrl->lock); + while (ctrl->nr_async_event_cmds) { + req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; + mutex_unlock(&ctrl->lock); + nvmet_req_complete(req, status); + mutex_lock(&ctrl->lock); + } + mutex_unlock(&ctrl->lock); +} + +static void nvmet_async_events_process(struct nvmet_ctrl *ctrl) { struct nvmet_async_event *aen; struct nvmet_req *req; @@ -139,15 +154,14 @@ static void nvmet_async_events_process(struct nvmet_ctrl *ctrl, u16 status) aen = list_first_entry(&ctrl->async_events, struct nvmet_async_event, entry); req = ctrl->async_event_cmds[--ctrl->nr_async_event_cmds]; - if (status == 0) - nvmet_set_result(req, nvmet_async_event_result(aen)); + nvmet_set_result(req, nvmet_async_event_result(aen)); list_del(&aen->entry); kfree(aen); mutex_unlock(&ctrl->lock); trace_nvmet_async_event(ctrl, req->cqe->result.u32); - nvmet_req_complete(req, status); + nvmet_req_complete(req, 0); mutex_lock(&ctrl->lock); } mutex_unlock(&ctrl->lock); @@ -170,7 +184,7 @@ static void nvmet_async_event_work(struct work_struct *work) struct nvmet_ctrl *ctrl = container_of(work, struct nvmet_ctrl, async_event_work); - nvmet_async_events_process(ctrl, 0); + nvmet_async_events_process(ctrl); } void nvmet_add_async_event(struct nvmet_ctrl *ctrl, u8 event_type, @@ -779,7 +793,6 @@ static void nvmet_confirm_sq(struct percpu_ref *ref) void nvmet_sq_destroy(struct nvmet_sq *sq) { - u16 status = NVME_SC_INTERNAL | NVME_SC_DNR; struct nvmet_ctrl *ctrl = sq->ctrl; /* @@ -787,7 +800,7 @@ void nvmet_sq_destroy(struct nvmet_sq *sq) * queue doesn't have outstanding requests on it. */ if (ctrl && ctrl->sqs && ctrl->sqs[0] == sq) - nvmet_async_events_process(ctrl, status); + nvmet_async_events_failall(ctrl); percpu_ref_kill_and_confirm(&sq->ref, nvmet_confirm_sq); wait_for_completion(&sq->confirm_done); wait_for_completion(&sq->free_done); diff --git a/drivers/nvme/target/tcp.c b/drivers/nvme/target/tcp.c index 1669177cd26c..de9217cfd22d 100644 --- a/drivers/nvme/target/tcp.c +++ b/drivers/nvme/target/tcp.c @@ -153,7 +153,7 @@ static LIST_HEAD(nvmet_tcp_queue_list); static DEFINE_MUTEX(nvmet_tcp_queue_mutex); static struct workqueue_struct *nvmet_tcp_wq; -static struct nvmet_fabrics_ops nvmet_tcp_ops; +static const struct nvmet_fabrics_ops nvmet_tcp_ops; static void nvmet_tcp_free_cmd(struct nvmet_tcp_cmd *c); static void nvmet_tcp_finish_cmd(struct nvmet_tcp_cmd *cmd); @@ -1713,7 +1713,7 @@ static void nvmet_tcp_disc_port_addr(struct nvmet_req *req, } } -static struct nvmet_fabrics_ops nvmet_tcp_ops = { +static const struct nvmet_fabrics_ops nvmet_tcp_ops = { .owner = THIS_MODULE, .type = NVMF_TRTYPE_TCP, .msdbd = 1, diff --git a/drivers/pci/xen-pcifront.c b/drivers/pci/xen-pcifront.c index d1b16cf3403f..fab267e359e7 100644 --- a/drivers/pci/xen-pcifront.c +++ b/drivers/pci/xen-pcifront.c @@ -77,9 +77,6 @@ static inline void pcifront_init_sd(struct pcifront_sd *sd, static DEFINE_SPINLOCK(pcifront_dev_lock); static struct pcifront_device *pcifront_dev; -static int verbose_request; -module_param(verbose_request, int, 0644); - static int errno_to_pcibios_err(int errno) { switch (errno) { @@ -190,18 +187,16 @@ static int pcifront_bus_read(struct pci_bus *bus, unsigned int devfn, struct pcifront_sd *sd = bus->sysdata; struct pcifront_device *pdev = pcifront_get_pdev(sd); - if (verbose_request) - dev_info(&pdev->xdev->dev, - "read dev=%04x:%02x:%02x.%d - offset %x size %d\n", - pci_domain_nr(bus), bus->number, PCI_SLOT(devfn), - PCI_FUNC(devfn), where, size); + dev_dbg(&pdev->xdev->dev, + "read dev=%04x:%02x:%02x.%d - offset %x size %d\n", + pci_domain_nr(bus), bus->number, PCI_SLOT(devfn), + PCI_FUNC(devfn), where, size); err = do_pci_op(pdev, &op); if (likely(!err)) { - if (verbose_request) - dev_info(&pdev->xdev->dev, "read got back value %x\n", - op.value); + dev_dbg(&pdev->xdev->dev, "read got back value %x\n", + op.value); *val = op.value; } else if (err == -ENODEV) { @@ -229,12 +224,10 @@ static int pcifront_bus_write(struct pci_bus *bus, unsigned int devfn, struct pcifront_sd *sd = bus->sysdata; struct pcifront_device *pdev = pcifront_get_pdev(sd); - if (verbose_request) - dev_info(&pdev->xdev->dev, - "write dev=%04x:%02x:%02x.%d - " - "offset %x size %d val %x\n", - pci_domain_nr(bus), bus->number, - PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val); + dev_dbg(&pdev->xdev->dev, + "write dev=%04x:%02x:%02x.%d - offset %x size %d val %x\n", + pci_domain_nr(bus), bus->number, + PCI_SLOT(devfn), PCI_FUNC(devfn), where, size, val); return errno_to_pcibios_err(do_pci_op(pdev, &op)); } diff --git a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c index 8dd1278bec04..7719ae4e2c56 100644 --- a/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c +++ b/drivers/perf/hisilicon/hisi_uncore_l3c_pmu.c @@ -35,7 +35,7 @@ /* L3C has 8-counters */ #define L3C_NR_COUNTERS 0x8 -#define L3C_PERF_CTRL_EN 0x20000 +#define L3C_PERF_CTRL_EN 0x10000 #define L3C_EVTYPE_NONE 0xff /* diff --git a/drivers/platform/mellanox/mlxreg-hotplug.c b/drivers/platform/mellanox/mlxreg-hotplug.c index 77be37a1fbcf..ed48917af162 100644 --- a/drivers/platform/mellanox/mlxreg-hotplug.c +++ b/drivers/platform/mellanox/mlxreg-hotplug.c @@ -101,6 +101,7 @@ static int mlxreg_hotplug_device_create(struct mlxreg_hotplug_priv_data *priv, struct mlxreg_core_data *data) { struct mlxreg_core_hotplug_platform_data *pdata; + struct i2c_client *client; /* Notify user by sending hwmon uevent. */ kobject_uevent(&priv->hwmon->kobj, KOBJ_CHANGE); @@ -121,18 +122,20 @@ static int mlxreg_hotplug_device_create(struct mlxreg_hotplug_priv_data *priv, return -EFAULT; } - data->hpdev.client = i2c_new_device(data->hpdev.adapter, - data->hpdev.brdinfo); - if (!data->hpdev.client) { + client = i2c_new_client_device(data->hpdev.adapter, + data->hpdev.brdinfo); + if (IS_ERR(client)) { dev_err(priv->dev, "Failed to create client %s at bus %d at addr 0x%02x\n", data->hpdev.brdinfo->type, data->hpdev.nr + pdata->shift_nr, data->hpdev.brdinfo->addr); i2c_put_adapter(data->hpdev.adapter); data->hpdev.adapter = NULL; - return -EFAULT; + return PTR_ERR(client); } + data->hpdev.client = client; + return 0; } diff --git a/drivers/powercap/idle_inject.c b/drivers/powercap/idle_inject.c index e9bbd3c42eef..c90f0990968b 100644 --- a/drivers/powercap/idle_inject.c +++ b/drivers/powercap/idle_inject.c @@ -61,12 +61,14 @@ struct idle_inject_thread { * @timer: idle injection period timer * @idle_duration_us: duration of CPU idle time to inject * @run_duration_us: duration of CPU run time to allow + * @latency_us: max allowed latency * @cpumask: mask of CPUs affected by idle injection */ struct idle_inject_device { struct hrtimer timer; unsigned int idle_duration_us; unsigned int run_duration_us; + unsigned int latency_us; unsigned long cpumask[]; }; @@ -138,7 +140,8 @@ static void idle_inject_fn(unsigned int cpu) */ iit->should_run = 0; - play_idle(READ_ONCE(ii_dev->idle_duration_us)); + play_idle_precise(READ_ONCE(ii_dev->idle_duration_us) * NSEC_PER_USEC, + READ_ONCE(ii_dev->latency_us) * NSEC_PER_USEC); } /** @@ -170,6 +173,16 @@ void idle_inject_get_duration(struct idle_inject_device *ii_dev, } /** + * idle_inject_set_latency - set the maximum latency allowed + * @latency_us: set the latency requirement for the idle state + */ +void idle_inject_set_latency(struct idle_inject_device *ii_dev, + unsigned int latency_us) +{ + WRITE_ONCE(ii_dev->latency_us, latency_us); +} + +/** * idle_inject_start - start idle injections * @ii_dev: idle injection control device structure * @@ -297,6 +310,7 @@ struct idle_inject_device *idle_inject_register(struct cpumask *cpumask) cpumask_copy(to_cpumask(ii_dev->cpumask), cpumask); hrtimer_init(&ii_dev->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); ii_dev->timer.function = idle_inject_timer_fn; + ii_dev->latency_us = UINT_MAX; for_each_cpu(cpu, to_cpumask(ii_dev->cpumask)) { diff --git a/drivers/pwm/Kconfig b/drivers/pwm/Kconfig index eebbc917ac97..cb8d739067d2 100644 --- a/drivers/pwm/Kconfig +++ b/drivers/pwm/Kconfig @@ -232,9 +232,19 @@ config PWM_IMX_TPM To compile this driver as a module, choose M here: the module will be called pwm-imx-tpm. +config PWM_IQS620A + tristate "Azoteq IQS620A PWM support" + depends on MFD_IQS62X || COMPILE_TEST + help + Generic PWM framework driver for the Azoteq IQS620A multi-function + sensor. + + To compile this driver as a module, choose M here: the module will + be called pwm-iqs620a. + config PWM_JZ4740 tristate "Ingenic JZ47xx PWM support" - depends on MACH_INGENIC + depends on MIPS depends on COMMON_CLK select MFD_SYSCON help diff --git a/drivers/pwm/Makefile b/drivers/pwm/Makefile index 9a475073dafc..a59c710e98c7 100644 --- a/drivers/pwm/Makefile +++ b/drivers/pwm/Makefile @@ -20,6 +20,7 @@ obj-$(CONFIG_PWM_IMG) += pwm-img.o obj-$(CONFIG_PWM_IMX1) += pwm-imx1.o obj-$(CONFIG_PWM_IMX27) += pwm-imx27.o obj-$(CONFIG_PWM_IMX_TPM) += pwm-imx-tpm.o +obj-$(CONFIG_PWM_IQS620A) += pwm-iqs620a.o obj-$(CONFIG_PWM_JZ4740) += pwm-jz4740.o obj-$(CONFIG_PWM_LP3943) += pwm-lp3943.o obj-$(CONFIG_PWM_LPC18XX_SCT) += pwm-lpc18xx-sct.o diff --git a/drivers/pwm/core.c b/drivers/pwm/core.c index 9973c442b455..004b2ea9b5fd 100644 --- a/drivers/pwm/core.c +++ b/drivers/pwm/core.c @@ -121,7 +121,7 @@ static int pwm_device_request(struct pwm_device *pwm, const char *label) pwm->chip->ops->get_state(pwm->chip, pwm, &pwm->state); trace_pwm_get(pwm, &pwm->state); - if (IS_ENABLED(PWM_DEBUG)) + if (IS_ENABLED(CONFIG_PWM_DEBUG)) pwm->last = pwm->state; } @@ -537,7 +537,7 @@ static void pwm_apply_state_debug(struct pwm_device *pwm, if (!state->enabled && s2.enabled && s2.duty_cycle > 0) dev_warn(chip->dev, - "requested disabled, but yielded enabled with duty > 0"); + "requested disabled, but yielded enabled with duty > 0\n"); /* reapply the state that the driver reported being configured. */ err = chip->ops->apply(chip, pwm, &s1); diff --git a/drivers/pwm/pwm-img.c b/drivers/pwm/pwm-img.c index c9e57bd109fb..599a0f66a384 100644 --- a/drivers/pwm/pwm-img.c +++ b/drivers/pwm/pwm-img.c @@ -129,8 +129,10 @@ static int img_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, duty = DIV_ROUND_UP(timebase * duty_ns, period_ns); ret = pm_runtime_get_sync(chip->dev); - if (ret < 0) + if (ret < 0) { + pm_runtime_put_autosuspend(chip->dev); return ret; + } val = img_pwm_readl(pwm_chip, PWM_CTRL_CFG); val &= ~(PWM_CTRL_CFG_DIV_MASK << PWM_CTRL_CFG_DIV_SHIFT(pwm->hwpwm)); @@ -331,8 +333,10 @@ static int img_pwm_remove(struct platform_device *pdev) int ret; ret = pm_runtime_get_sync(&pdev->dev); - if (ret < 0) + if (ret < 0) { + pm_runtime_put(&pdev->dev); return ret; + } for (i = 0; i < pwm_chip->chip.npwm; i++) { val = img_pwm_readl(pwm_chip, PWM_CTRL_CFG); diff --git a/drivers/pwm/pwm-imx27.c b/drivers/pwm/pwm-imx27.c index a6e40d4c485f..732a6f3701e8 100644 --- a/drivers/pwm/pwm-imx27.c +++ b/drivers/pwm/pwm-imx27.c @@ -150,13 +150,12 @@ static void pwm_imx27_get_state(struct pwm_chip *chip, prescaler = MX3_PWMCR_PRESCALER_GET(val); pwm_clk = clk_get_rate(imx->clk_per); - pwm_clk = DIV_ROUND_CLOSEST_ULL(pwm_clk, prescaler); val = readl(imx->mmio_base + MX3_PWMPR); period = val >= MX3_PWMPR_MAX ? MX3_PWMPR_MAX : val; /* PWMOUT (Hz) = PWMCLK / (PWMPR + 2) */ - tmp = NSEC_PER_SEC * (u64)(period + 2); - state->period = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk); + tmp = NSEC_PER_SEC * (u64)(period + 2) * prescaler; + state->period = DIV_ROUND_UP_ULL(tmp, pwm_clk); /* * PWMSAR can be read only if PWM is enabled. If the PWM is disabled, @@ -167,8 +166,8 @@ static void pwm_imx27_get_state(struct pwm_chip *chip, else val = imx->duty_cycle; - tmp = NSEC_PER_SEC * (u64)(val); - state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, pwm_clk); + tmp = NSEC_PER_SEC * (u64)(val) * prescaler; + state->duty_cycle = DIV_ROUND_UP_ULL(tmp, pwm_clk); pwm_imx27_clk_disable_unprepare(imx); } @@ -220,22 +219,23 @@ static int pwm_imx27_apply(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_imx27_chip *imx = to_pwm_imx27_chip(chip); struct pwm_state cstate; unsigned long long c; + unsigned long long clkrate; int ret; u32 cr; pwm_get_state(pwm, &cstate); - c = clk_get_rate(imx->clk_per); - c *= state->period; + clkrate = clk_get_rate(imx->clk_per); + c = clkrate * state->period; - do_div(c, 1000000000); + do_div(c, NSEC_PER_SEC); period_cycles = c; prescale = period_cycles / 0x10000 + 1; period_cycles /= prescale; - c = (unsigned long long)period_cycles * state->duty_cycle; - do_div(c, state->period); + c = clkrate * state->duty_cycle; + do_div(c, NSEC_PER_SEC * prescale); duty_cycles = c; /* diff --git a/drivers/pwm/pwm-iqs620a.c b/drivers/pwm/pwm-iqs620a.c new file mode 100644 index 000000000000..674f0e238ba0 --- /dev/null +++ b/drivers/pwm/pwm-iqs620a.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Azoteq IQS620A PWM Generator + * + * Copyright (C) 2019 Jeff LaBundy <jeff@labundy.com> + * + * Limitations: + * - The period is fixed to 1 ms and is generated continuously despite changes + * to the duty cycle or enable/disable state. + * - Changes to the duty cycle or enable/disable state take effect immediately + * and may result in a glitch during the period in which the change is made. + * - The device cannot generate a 0% duty cycle. For duty cycles below 1 / 256 + * ms, the output is disabled and relies upon an external pull-down resistor + * to hold the GPIO3/LTX pin low. + */ + +#include <linux/device.h> +#include <linux/kernel.h> +#include <linux/mfd/iqs62x.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/notifier.h> +#include <linux/platform_device.h> +#include <linux/pwm.h> +#include <linux/regmap.h> +#include <linux/slab.h> + +#define IQS620_PWR_SETTINGS 0xD2 +#define IQS620_PWR_SETTINGS_PWM_OUT BIT(7) + +#define IQS620_PWM_DUTY_CYCLE 0xD8 + +#define IQS620_PWM_PERIOD_NS 1000000 + +struct iqs620_pwm_private { + struct iqs62x_core *iqs62x; + struct pwm_chip chip; + struct notifier_block notifier; + struct mutex lock; + bool out_en; + u8 duty_val; +}; + +static int iqs620_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, + const struct pwm_state *state) +{ + struct iqs620_pwm_private *iqs620_pwm; + struct iqs62x_core *iqs62x; + int duty_scale, ret; + + if (state->polarity != PWM_POLARITY_NORMAL) + return -ENOTSUPP; + + if (state->period < IQS620_PWM_PERIOD_NS) + return -EINVAL; + + iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + iqs62x = iqs620_pwm->iqs62x; + + /* + * The duty cycle generated by the device is calculated as follows: + * + * duty_cycle = (IQS620_PWM_DUTY_CYCLE + 1) / 256 * 1 ms + * + * ...where IQS620_PWM_DUTY_CYCLE is a register value between 0 and 255 + * (inclusive). Therefore the lowest duty cycle the device can generate + * while the output is enabled is 1 / 256 ms. + * + * For lower duty cycles (e.g. 0), the PWM output is simply disabled to + * allow an external pull-down resistor to hold the GPIO3/LTX pin low. + */ + duty_scale = state->duty_cycle * 256 / IQS620_PWM_PERIOD_NS; + + mutex_lock(&iqs620_pwm->lock); + + if (!state->enabled || !duty_scale) { + ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, + IQS620_PWR_SETTINGS_PWM_OUT, 0); + if (ret) + goto err_mutex; + } + + if (duty_scale) { + u8 duty_val = min(duty_scale - 1, 0xFF); + + ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, + duty_val); + if (ret) + goto err_mutex; + + iqs620_pwm->duty_val = duty_val; + } + + if (state->enabled && duty_scale) { + ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, + IQS620_PWR_SETTINGS_PWM_OUT, 0xFF); + if (ret) + goto err_mutex; + } + + iqs620_pwm->out_en = state->enabled; + +err_mutex: + mutex_unlock(&iqs620_pwm->lock); + + return ret; +} + +static void iqs620_pwm_get_state(struct pwm_chip *chip, struct pwm_device *pwm, + struct pwm_state *state) +{ + struct iqs620_pwm_private *iqs620_pwm; + + iqs620_pwm = container_of(chip, struct iqs620_pwm_private, chip); + + mutex_lock(&iqs620_pwm->lock); + + /* + * Since the device cannot generate a 0% duty cycle, requests to do so + * cause subsequent calls to iqs620_pwm_get_state to report the output + * as disabled with duty cycle equal to that which was in use prior to + * the request. This is not ideal, but is the best compromise based on + * the capabilities of the device. + */ + state->enabled = iqs620_pwm->out_en; + state->duty_cycle = DIV_ROUND_UP((iqs620_pwm->duty_val + 1) * + IQS620_PWM_PERIOD_NS, 256); + + mutex_unlock(&iqs620_pwm->lock); + + state->period = IQS620_PWM_PERIOD_NS; +} + +static int iqs620_pwm_notifier(struct notifier_block *notifier, + unsigned long event_flags, void *context) +{ + struct iqs620_pwm_private *iqs620_pwm; + struct iqs62x_core *iqs62x; + int ret; + + if (!(event_flags & BIT(IQS62X_EVENT_SYS_RESET))) + return NOTIFY_DONE; + + iqs620_pwm = container_of(notifier, struct iqs620_pwm_private, + notifier); + iqs62x = iqs620_pwm->iqs62x; + + mutex_lock(&iqs620_pwm->lock); + + /* + * The parent MFD driver already prints an error message in the event + * of a device reset, so nothing else is printed here unless there is + * an additional failure. + */ + ret = regmap_write(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, + iqs620_pwm->duty_val); + if (ret) + goto err_mutex; + + ret = regmap_update_bits(iqs62x->regmap, IQS620_PWR_SETTINGS, + IQS620_PWR_SETTINGS_PWM_OUT, + iqs620_pwm->out_en ? 0xFF : 0); + +err_mutex: + mutex_unlock(&iqs620_pwm->lock); + + if (ret) { + dev_err(iqs620_pwm->chip.dev, + "Failed to re-initialize device: %d\n", ret); + return NOTIFY_BAD; + } + + return NOTIFY_OK; +} + +static const struct pwm_ops iqs620_pwm_ops = { + .apply = iqs620_pwm_apply, + .get_state = iqs620_pwm_get_state, + .owner = THIS_MODULE, +}; + +static void iqs620_pwm_notifier_unregister(void *context) +{ + struct iqs620_pwm_private *iqs620_pwm = context; + int ret; + + ret = blocking_notifier_chain_unregister(&iqs620_pwm->iqs62x->nh, + &iqs620_pwm->notifier); + if (ret) + dev_err(iqs620_pwm->chip.dev, + "Failed to unregister notifier: %d\n", ret); +} + +static int iqs620_pwm_probe(struct platform_device *pdev) +{ + struct iqs62x_core *iqs62x = dev_get_drvdata(pdev->dev.parent); + struct iqs620_pwm_private *iqs620_pwm; + unsigned int val; + int ret; + + iqs620_pwm = devm_kzalloc(&pdev->dev, sizeof(*iqs620_pwm), GFP_KERNEL); + if (!iqs620_pwm) + return -ENOMEM; + + platform_set_drvdata(pdev, iqs620_pwm); + iqs620_pwm->iqs62x = iqs62x; + + ret = regmap_read(iqs62x->regmap, IQS620_PWR_SETTINGS, &val); + if (ret) + return ret; + iqs620_pwm->out_en = val & IQS620_PWR_SETTINGS_PWM_OUT; + + ret = regmap_read(iqs62x->regmap, IQS620_PWM_DUTY_CYCLE, &val); + if (ret) + return ret; + iqs620_pwm->duty_val = val; + + iqs620_pwm->chip.dev = &pdev->dev; + iqs620_pwm->chip.ops = &iqs620_pwm_ops; + iqs620_pwm->chip.base = -1; + iqs620_pwm->chip.npwm = 1; + + mutex_init(&iqs620_pwm->lock); + + iqs620_pwm->notifier.notifier_call = iqs620_pwm_notifier; + ret = blocking_notifier_chain_register(&iqs620_pwm->iqs62x->nh, + &iqs620_pwm->notifier); + if (ret) { + dev_err(&pdev->dev, "Failed to register notifier: %d\n", ret); + return ret; + } + + ret = devm_add_action_or_reset(&pdev->dev, + iqs620_pwm_notifier_unregister, + iqs620_pwm); + if (ret) + return ret; + + ret = pwmchip_add(&iqs620_pwm->chip); + if (ret) + dev_err(&pdev->dev, "Failed to add device: %d\n", ret); + + return ret; +} + +static int iqs620_pwm_remove(struct platform_device *pdev) +{ + struct iqs620_pwm_private *iqs620_pwm = platform_get_drvdata(pdev); + int ret; + + ret = pwmchip_remove(&iqs620_pwm->chip); + if (ret) + dev_err(&pdev->dev, "Failed to remove device: %d\n", ret); + + return ret; +} + +static struct platform_driver iqs620_pwm_platform_driver = { + .driver = { + .name = "iqs620a-pwm", + }, + .probe = iqs620_pwm_probe, + .remove = iqs620_pwm_remove, +}; +module_platform_driver(iqs620_pwm_platform_driver); + +MODULE_AUTHOR("Jeff LaBundy <jeff@labundy.com>"); +MODULE_DESCRIPTION("Azoteq IQS620A PWM Generator"); +MODULE_LICENSE("GPL"); +MODULE_ALIAS("platform:iqs620a-pwm"); diff --git a/drivers/pwm/pwm-jz4740.c b/drivers/pwm/pwm-jz4740.c index 3cd5c054ad9a..5830ac2bdf6a 100644 --- a/drivers/pwm/pwm-jz4740.c +++ b/drivers/pwm/pwm-jz4740.c @@ -6,7 +6,6 @@ * Limitations: * - The .apply callback doesn't complete the currently running period before * reconfiguring the hardware. - * - Each period starts with the inactive part. */ #include <linux/clk.h> @@ -21,7 +20,9 @@ #include <linux/pwm.h> #include <linux/regmap.h> -#define NUM_PWM 8 +struct soc_info { + unsigned int num_pwms; +}; struct jz4740_pwm_chip { struct pwm_chip chip; @@ -37,7 +38,7 @@ static bool jz4740_pwm_can_use_chn(struct jz4740_pwm_chip *jz, unsigned int channel) { /* Enable all TCU channels for PWM use by default except channels 0/1 */ - u32 pwm_channels_mask = GENMASK(NUM_PWM - 1, 2); + u32 pwm_channels_mask = GENMASK(jz->chip.npwm - 1, 2); device_property_read_u32(jz->chip.dev->parent, "ingenic,pwm-channels-mask", @@ -158,12 +159,12 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, /* Calculate period value */ tmp = (unsigned long long)rate * state->period; do_div(tmp, NSEC_PER_SEC); - period = (unsigned long)tmp; + period = tmp; /* Calculate duty value */ - tmp = (unsigned long long)period * state->duty_cycle; - do_div(tmp, state->period); - duty = period - tmp; + tmp = (unsigned long long)rate * state->duty_cycle; + do_div(tmp, NSEC_PER_SEC); + duty = tmp; if (duty >= period) duty = period - 1; @@ -189,18 +190,26 @@ static int jz4740_pwm_apply(struct pwm_chip *chip, struct pwm_device *pwm, regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm), TCU_TCSR_PWM_SD, TCU_TCSR_PWM_SD); - /* Set polarity */ - switch (state->polarity) { - case PWM_POLARITY_NORMAL: + /* + * Set polarity. + * + * The PWM starts in inactive state until the internal timer reaches the + * duty value, then becomes active until the timer reaches the period + * value. In theory, we should then use (period - duty) as the real duty + * value, as a high duty value would otherwise result in the PWM pin + * being inactive most of the time. + * + * Here, we don't do that, and instead invert the polarity of the PWM + * when it is active. This trick makes the PWM start with its active + * state instead of its inactive state. + */ + if ((state->polarity == PWM_POLARITY_NORMAL) ^ state->enabled) regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm), TCU_TCSR_PWM_INITL_HIGH, 0); - break; - case PWM_POLARITY_INVERSED: + else regmap_update_bits(jz4740->map, TCU_REG_TCSRc(pwm->hwpwm), TCU_TCSR_PWM_INITL_HIGH, TCU_TCSR_PWM_INITL_HIGH); - break; - } if (state->enabled) jz4740_pwm_enable(chip, pwm); @@ -219,6 +228,11 @@ static int jz4740_pwm_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; struct jz4740_pwm_chip *jz4740; + const struct soc_info *info; + + info = device_get_match_data(dev); + if (!info) + return -EINVAL; jz4740 = devm_kzalloc(dev, sizeof(*jz4740), GFP_KERNEL); if (!jz4740) @@ -232,7 +246,7 @@ static int jz4740_pwm_probe(struct platform_device *pdev) jz4740->chip.dev = dev; jz4740->chip.ops = &jz4740_pwm_ops; - jz4740->chip.npwm = NUM_PWM; + jz4740->chip.npwm = info->num_pwms; jz4740->chip.base = -1; jz4740->chip.of_xlate = of_pwm_xlate_with_flags; jz4740->chip.of_pwm_n_cells = 3; @@ -249,9 +263,18 @@ static int jz4740_pwm_remove(struct platform_device *pdev) return pwmchip_remove(&jz4740->chip); } +static const struct soc_info __maybe_unused jz4740_soc_info = { + .num_pwms = 8, +}; + +static const struct soc_info __maybe_unused jz4725b_soc_info = { + .num_pwms = 6, +}; + #ifdef CONFIG_OF static const struct of_device_id jz4740_pwm_dt_ids[] = { - { .compatible = "ingenic,jz4740-pwm", }, + { .compatible = "ingenic,jz4740-pwm", .data = &jz4740_soc_info }, + { .compatible = "ingenic,jz4725b-pwm", .data = &jz4725b_soc_info }, {}, }; MODULE_DEVICE_TABLE(of, jz4740_pwm_dt_ids); diff --git a/drivers/pwm/pwm-lpss.c b/drivers/pwm/pwm-lpss.c index 75bbfe5f3bc2..9d965ffe66d1 100644 --- a/drivers/pwm/pwm-lpss.c +++ b/drivers/pwm/pwm-lpss.c @@ -158,7 +158,6 @@ static int pwm_lpss_apply(struct pwm_chip *chip, struct pwm_device *pwm, return 0; } -/* This function gets called once from pwmchip_add to get the initial state */ static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, struct pwm_state *state) { @@ -167,6 +166,8 @@ static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, unsigned long long base_unit, freq, on_time_div; u32 ctrl; + pm_runtime_get_sync(chip->dev); + base_unit_range = BIT(lpwm->info->base_unit_bits); ctrl = pwm_lpss_read(pwm); @@ -187,8 +188,7 @@ static void pwm_lpss_get_state(struct pwm_chip *chip, struct pwm_device *pwm, state->polarity = PWM_POLARITY_NORMAL; state->enabled = !!(ctrl & PWM_ENABLE); - if (state->enabled) - pm_runtime_get(chip->dev); + pm_runtime_put(chip->dev); } static const struct pwm_ops pwm_lpss_ops = { @@ -202,7 +202,8 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r, { struct pwm_lpss_chip *lpwm; unsigned long c; - int ret; + int i, ret; + u32 ctrl; if (WARN_ON(info->npwm > MAX_PWMS)) return ERR_PTR(-ENODEV); @@ -232,6 +233,12 @@ struct pwm_lpss_chip *pwm_lpss_probe(struct device *dev, struct resource *r, return ERR_PTR(ret); } + for (i = 0; i < lpwm->info->npwm; i++) { + ctrl = pwm_lpss_read(&lpwm->chip.pwms[i]); + if (ctrl & PWM_ENABLE) + pm_runtime_get(dev); + } + return lpwm; } EXPORT_SYMBOL_GPL(pwm_lpss_probe); diff --git a/drivers/pwm/pwm-rockchip.c b/drivers/pwm/pwm-rockchip.c index 73352e6fbccb..eb8c9cb645a6 100644 --- a/drivers/pwm/pwm-rockchip.c +++ b/drivers/pwm/pwm-rockchip.c @@ -83,12 +83,7 @@ static void rockchip_pwm_get_state(struct pwm_chip *chip, state->duty_cycle = DIV_ROUND_CLOSEST_ULL(tmp, clk_rate); val = readl_relaxed(pc->base + pc->data->regs.ctrl); - if (pc->data->supports_polarity) - state->enabled = ((val & enable_conf) != enable_conf) ? - false : true; - else - state->enabled = ((val & enable_conf) == enable_conf) ? - true : false; + state->enabled = (val & enable_conf) == enable_conf; if (pc->data->supports_polarity && !(val & PWM_DUTY_POSITIVE)) state->polarity = PWM_POLARITY_INVERSED; diff --git a/drivers/pwm/pwm-sun4i.c b/drivers/pwm/pwm-sun4i.c index 5c677c563349..18fbbe3277d0 100644 --- a/drivers/pwm/pwm-sun4i.c +++ b/drivers/pwm/pwm-sun4i.c @@ -352,6 +352,12 @@ static const struct sun4i_pwm_data sun4i_pwm_single_bypass = { .npwm = 1, }; +static const struct sun4i_pwm_data sun50i_a64_pwm_data = { + .has_prescaler_bypass = true, + .has_direct_mod_clk_output = true, + .npwm = 1, +}; + static const struct sun4i_pwm_data sun50i_h6_pwm_data = { .has_prescaler_bypass = true, .has_direct_mod_clk_output = true, @@ -375,6 +381,9 @@ static const struct of_device_id sun4i_pwm_dt_ids[] = { .compatible = "allwinner,sun8i-h3-pwm", .data = &sun4i_pwm_single_bypass, }, { + .compatible = "allwinner,sun50i-a64-pwm", + .data = &sun50i_a64_pwm_data, + }, { .compatible = "allwinner,sun50i-h6-pwm", .data = &sun50i_h6_pwm_data, }, { diff --git a/drivers/pwm/pwm-tegra.c b/drivers/pwm/pwm-tegra.c index d26ed8f579ff..1daf591025c0 100644 --- a/drivers/pwm/pwm-tegra.c +++ b/drivers/pwm/pwm-tegra.c @@ -4,8 +4,36 @@ * * Tegra pulse-width-modulation controller driver * - * Copyright (c) 2010, NVIDIA Corporation. + * Copyright (c) 2010-2020, NVIDIA Corporation. * Based on arch/arm/plat-mxc/pwm.c by Sascha Hauer <s.hauer@pengutronix.de> + * + * Overview of Tegra Pulse Width Modulator Register: + * 1. 13-bit: Frequency division (SCALE) + * 2. 8-bit : Pulse division (DUTY) + * 3. 1-bit : Enable bit + * + * The PWM clock frequency is divided by 256 before subdividing it based + * on the programmable frequency division value to generate the required + * frequency for PWM output. The maximum output frequency that can be + * achieved is (max rate of source clock) / 256. + * e.g. if source clock rate is 408 MHz, maximum output frequency can be: + * 408 MHz/256 = 1.6 MHz. + * This 1.6 MHz frequency can further be divided using SCALE value in PWM. + * + * PWM pulse width: 8 bits are usable [23:16] for varying pulse width. + * To achieve 100% duty cycle, program Bit [24] of this register to + * 1’b1. In which case the other bits [23:16] are set to don't care. + * + * Limitations: + * - When PWM is disabled, the output is driven to inactive. + * - It does not allow the current PWM period to complete and + * stops abruptly. + * + * - If the register is reconfigured while PWM is running, + * it does not complete the currently running period. + * + * - If the user input duty is beyond acceptible limits, + * -EINVAL is returned. */ #include <linux/clk.h> @@ -41,6 +69,7 @@ struct tegra_pwm_chip { struct reset_control*rst; unsigned long clk_rate; + unsigned long min_period_ns; void __iomem *regs; @@ -68,7 +97,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, { struct tegra_pwm_chip *pc = to_tegra_pwm_chip(chip); unsigned long long c = duty_ns, hz; - unsigned long rate; + unsigned long rate, required_clk_rate; u32 val = 0; int err; @@ -83,9 +112,47 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, val = (u32)c << PWM_DUTY_SHIFT; /* + * min period = max clock limit >> PWM_DUTY_WIDTH + */ + if (period_ns < pc->min_period_ns) + return -EINVAL; + + /* * Compute the prescaler value for which (1 << PWM_DUTY_WIDTH) * cycles at the PWM clock rate will take period_ns nanoseconds. + * + * num_channels: If single instance of PWM controller has multiple + * channels (e.g. Tegra210 or older) then it is not possible to + * configure separate clock rates to each of the channels, in such + * case the value stored during probe will be referred. + * + * If every PWM controller instance has one channel respectively, i.e. + * nums_channels == 1 then only the clock rate can be modified + * dynamically (e.g. Tegra186 or Tegra194). */ + if (pc->soc->num_channels == 1) { + /* + * Rate is multiplied with 2^PWM_DUTY_WIDTH so that it matches + * with the maximum possible rate that the controller can + * provide. Any further lower value can be derived by setting + * PFM bits[0:12]. + * + * required_clk_rate is a reference rate for source clock and + * it is derived based on user requested period. By setting the + * source clock rate as required_clk_rate, PWM controller will + * be able to configure the requested period. + */ + required_clk_rate = + (NSEC_PER_SEC / period_ns) << PWM_DUTY_WIDTH; + + err = clk_set_rate(pc->clk, required_clk_rate); + if (err < 0) + return -EINVAL; + + /* Store the new rate for further references */ + pc->clk_rate = clk_get_rate(pc->clk); + } + rate = pc->clk_rate >> PWM_DUTY_WIDTH; /* Consider precision in PWM_SCALE_WIDTH rate calculation */ @@ -94,7 +161,7 @@ static int tegra_pwm_config(struct pwm_chip *chip, struct pwm_device *pwm, /* * Since the actual PWM divider is the register's frequency divider - * field minus 1, we need to decrement to get the correct value to + * field plus 1, we need to decrement to get the correct value to * write to the register. */ if (rate > 0) @@ -205,6 +272,10 @@ static int tegra_pwm_probe(struct platform_device *pdev) */ pwm->clk_rate = clk_get_rate(pwm->clk); + /* Set minimum limit of PWM period for the IP */ + pwm->min_period_ns = + (NSEC_PER_SEC / (pwm->soc->max_frequency >> PWM_DUTY_WIDTH)) + 1; + pwm->rst = devm_reset_control_get_exclusive(&pdev->dev, "pwm"); if (IS_ERR(pwm->rst)) { ret = PTR_ERR(pwm->rst); @@ -312,5 +383,6 @@ static struct platform_driver tegra_pwm_driver = { module_platform_driver(tegra_pwm_driver); MODULE_LICENSE("GPL"); -MODULE_AUTHOR("NVIDIA Corporation"); +MODULE_AUTHOR("Sandipan Patra <spatra@nvidia.com>"); +MODULE_DESCRIPTION("Tegra PWM controller driver"); MODULE_ALIAS("platform:tegra-pwm"); diff --git a/drivers/ras/cec.c b/drivers/ras/cec.c index c09cf55e2d20..569d9ad2c594 100644 --- a/drivers/ras/cec.c +++ b/drivers/ras/cec.c @@ -309,7 +309,7 @@ static bool sanity_check(struct ce_array *ca) return ret; } -int cec_add_elem(u64 pfn) +static int cec_add_elem(u64 pfn) { struct ce_array *ca = &ce_arr; unsigned int to = 0; @@ -527,7 +527,33 @@ err: return 1; } -void __init cec_init(void) +static int cec_notifier(struct notifier_block *nb, unsigned long val, + void *data) +{ + struct mce *m = (struct mce *)data; + + if (!m) + return NOTIFY_DONE; + + /* We eat only correctable DRAM errors with usable addresses. */ + if (mce_is_memory_error(m) && + mce_is_correctable(m) && + mce_usable_address(m)) { + if (!cec_add_elem(m->addr >> PAGE_SHIFT)) { + m->kflags |= MCE_HANDLED_CEC; + return NOTIFY_OK; + } + } + + return NOTIFY_DONE; +} + +static struct notifier_block cec_nb = { + .notifier_call = cec_notifier, + .priority = MCE_PRIO_CEC, +}; + +static void __init cec_init(void) { if (ce_arr.disabled) return; @@ -546,8 +572,11 @@ void __init cec_init(void) INIT_DELAYED_WORK(&cec_work, cec_work_fn); schedule_delayed_work(&cec_work, CEC_DECAY_DEFAULT_INTERVAL); + mce_register_decode_chain(&cec_nb); + pr_info("Correctable Errors collector initialized.\n"); } +late_initcall(cec_init); int __init parse_cec_param(char *str) { diff --git a/drivers/scsi/arm/acornscsi.c b/drivers/scsi/arm/acornscsi.c index ddb52e7ba622..9a912fd0f70b 100644 --- a/drivers/scsi/arm/acornscsi.c +++ b/drivers/scsi/arm/acornscsi.c @@ -2911,8 +2911,10 @@ static int acornscsi_probe(struct expansion_card *ec, const struct ecard_id *id) ashost->base = ecardm_iomap(ec, ECARD_RES_MEMC, 0, 0); ashost->fast = ecardm_iomap(ec, ECARD_RES_IOCFAST, 0, 0); - if (!ashost->base || !ashost->fast) + if (!ashost->base || !ashost->fast) { + ret = -ENOMEM; goto out_put; + } host->irq = ec->irq; ashost->host = host; diff --git a/drivers/scsi/cxlflash/main.c b/drivers/scsi/cxlflash/main.c index fcc5aa9f6014..94250ebe9e80 100644 --- a/drivers/scsi/cxlflash/main.c +++ b/drivers/scsi/cxlflash/main.c @@ -47,9 +47,6 @@ static void process_cmd_err(struct afu_cmd *cmd, struct scsi_cmnd *scp) struct sisl_ioasa *ioasa; u32 resid; - if (unlikely(!cmd)) - return; - ioasa = &(cmd->sa); if (ioasa->rc.flags & SISL_RC_FLAGS_UNDERRUN) { diff --git a/drivers/scsi/hpsa.c b/drivers/scsi/hpsa.c index 1e9302e99d05..81d0414e2117 100644 --- a/drivers/scsi/hpsa.c +++ b/drivers/scsi/hpsa.c @@ -254,6 +254,10 @@ static irqreturn_t do_hpsa_intr_intx(int irq, void *dev_id); static irqreturn_t do_hpsa_intr_msi(int irq, void *dev_id); static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd, void __user *arg); +static int hpsa_passthru_ioctl(struct ctlr_info *h, + IOCTL_Command_struct *iocommand); +static int hpsa_big_passthru_ioctl(struct ctlr_info *h, + BIG_IOCTL_Command_struct *ioc); #ifdef CONFIG_COMPAT static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd, @@ -6217,75 +6221,63 @@ static void cmd_free(struct ctlr_info *h, struct CommandList *c) static int hpsa_ioctl32_passthru(struct scsi_device *dev, unsigned int cmd, void __user *arg) { - IOCTL32_Command_struct __user *arg32 = - (IOCTL32_Command_struct __user *) arg; + struct ctlr_info *h = sdev_to_hba(dev); + IOCTL32_Command_struct __user *arg32 = arg; IOCTL_Command_struct arg64; - IOCTL_Command_struct __user *p = compat_alloc_user_space(sizeof(arg64)); int err; u32 cp; - memset(&arg64, 0, sizeof(arg64)); - err = 0; - err |= copy_from_user(&arg64.LUN_info, &arg32->LUN_info, - sizeof(arg64.LUN_info)); - err |= copy_from_user(&arg64.Request, &arg32->Request, - sizeof(arg64.Request)); - err |= copy_from_user(&arg64.error_info, &arg32->error_info, - sizeof(arg64.error_info)); - err |= get_user(arg64.buf_size, &arg32->buf_size); - err |= get_user(cp, &arg32->buf); - arg64.buf = compat_ptr(cp); - err |= copy_to_user(p, &arg64, sizeof(arg64)); + if (!arg) + return -EINVAL; - if (err) + memset(&arg64, 0, sizeof(arg64)); + if (copy_from_user(&arg64, arg32, offsetof(IOCTL_Command_struct, buf))) + return -EFAULT; + if (get_user(cp, &arg32->buf)) return -EFAULT; + arg64.buf = compat_ptr(cp); - err = hpsa_ioctl(dev, CCISS_PASSTHRU, p); + if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0) + return -EAGAIN; + err = hpsa_passthru_ioctl(h, &arg64); + atomic_inc(&h->passthru_cmds_avail); if (err) return err; - err |= copy_in_user(&arg32->error_info, &p->error_info, - sizeof(arg32->error_info)); - if (err) + if (copy_to_user(&arg32->error_info, &arg64.error_info, + sizeof(arg32->error_info))) return -EFAULT; - return err; + return 0; } static int hpsa_ioctl32_big_passthru(struct scsi_device *dev, unsigned int cmd, void __user *arg) { - BIG_IOCTL32_Command_struct __user *arg32 = - (BIG_IOCTL32_Command_struct __user *) arg; + struct ctlr_info *h = sdev_to_hba(dev); + BIG_IOCTL32_Command_struct __user *arg32 = arg; BIG_IOCTL_Command_struct arg64; - BIG_IOCTL_Command_struct __user *p = - compat_alloc_user_space(sizeof(arg64)); int err; u32 cp; + if (!arg) + return -EINVAL; memset(&arg64, 0, sizeof(arg64)); - err = 0; - err |= copy_from_user(&arg64.LUN_info, &arg32->LUN_info, - sizeof(arg64.LUN_info)); - err |= copy_from_user(&arg64.Request, &arg32->Request, - sizeof(arg64.Request)); - err |= copy_from_user(&arg64.error_info, &arg32->error_info, - sizeof(arg64.error_info)); - err |= get_user(arg64.buf_size, &arg32->buf_size); - err |= get_user(arg64.malloc_size, &arg32->malloc_size); - err |= get_user(cp, &arg32->buf); - arg64.buf = compat_ptr(cp); - err |= copy_to_user(p, &arg64, sizeof(arg64)); - - if (err) + if (copy_from_user(&arg64, arg32, + offsetof(BIG_IOCTL32_Command_struct, buf))) + return -EFAULT; + if (get_user(cp, &arg32->buf)) return -EFAULT; + arg64.buf = compat_ptr(cp); - err = hpsa_ioctl(dev, CCISS_BIG_PASSTHRU, p); + if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0) + return -EAGAIN; + err = hpsa_big_passthru_ioctl(h, &arg64); + atomic_inc(&h->passthru_cmds_avail); if (err) return err; - err |= copy_in_user(&arg32->error_info, &p->error_info, - sizeof(arg32->error_info)); - if (err) + if (copy_to_user(&arg32->error_info, &arg64.error_info, + sizeof(arg32->error_info))) return -EFAULT; - return err; + return 0; } static int hpsa_compat_ioctl(struct scsi_device *dev, unsigned int cmd, @@ -6358,37 +6350,33 @@ static int hpsa_getdrivver_ioctl(struct ctlr_info *h, void __user *argp) return 0; } -static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp) +static int hpsa_passthru_ioctl(struct ctlr_info *h, + IOCTL_Command_struct *iocommand) { - IOCTL_Command_struct iocommand; struct CommandList *c; char *buff = NULL; u64 temp64; int rc = 0; - if (!argp) - return -EINVAL; if (!capable(CAP_SYS_RAWIO)) return -EPERM; - if (copy_from_user(&iocommand, argp, sizeof(iocommand))) - return -EFAULT; - if ((iocommand.buf_size < 1) && - (iocommand.Request.Type.Direction != XFER_NONE)) { + if ((iocommand->buf_size < 1) && + (iocommand->Request.Type.Direction != XFER_NONE)) { return -EINVAL; } - if (iocommand.buf_size > 0) { - buff = kmalloc(iocommand.buf_size, GFP_KERNEL); + if (iocommand->buf_size > 0) { + buff = kmalloc(iocommand->buf_size, GFP_KERNEL); if (buff == NULL) return -ENOMEM; - if (iocommand.Request.Type.Direction & XFER_WRITE) { + if (iocommand->Request.Type.Direction & XFER_WRITE) { /* Copy the data into the buffer we created */ - if (copy_from_user(buff, iocommand.buf, - iocommand.buf_size)) { + if (copy_from_user(buff, iocommand->buf, + iocommand->buf_size)) { rc = -EFAULT; goto out_kfree; } } else { - memset(buff, 0, iocommand.buf_size); + memset(buff, 0, iocommand->buf_size); } } c = cmd_alloc(h); @@ -6398,23 +6386,23 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp) c->scsi_cmd = SCSI_CMD_BUSY; /* Fill in Command Header */ c->Header.ReplyQueue = 0; /* unused in simple mode */ - if (iocommand.buf_size > 0) { /* buffer to fill */ + if (iocommand->buf_size > 0) { /* buffer to fill */ c->Header.SGList = 1; c->Header.SGTotal = cpu_to_le16(1); } else { /* no buffers to fill */ c->Header.SGList = 0; c->Header.SGTotal = cpu_to_le16(0); } - memcpy(&c->Header.LUN, &iocommand.LUN_info, sizeof(c->Header.LUN)); + memcpy(&c->Header.LUN, &iocommand->LUN_info, sizeof(c->Header.LUN)); /* Fill in Request block */ - memcpy(&c->Request, &iocommand.Request, + memcpy(&c->Request, &iocommand->Request, sizeof(c->Request)); /* Fill in the scatter gather information */ - if (iocommand.buf_size > 0) { + if (iocommand->buf_size > 0) { temp64 = dma_map_single(&h->pdev->dev, buff, - iocommand.buf_size, DMA_BIDIRECTIONAL); + iocommand->buf_size, DMA_BIDIRECTIONAL); if (dma_mapping_error(&h->pdev->dev, (dma_addr_t) temp64)) { c->SG[0].Addr = cpu_to_le64(0); c->SG[0].Len = cpu_to_le32(0); @@ -6422,12 +6410,12 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp) goto out; } c->SG[0].Addr = cpu_to_le64(temp64); - c->SG[0].Len = cpu_to_le32(iocommand.buf_size); + c->SG[0].Len = cpu_to_le32(iocommand->buf_size); c->SG[0].Ext = cpu_to_le32(HPSA_SG_LAST); /* not chaining */ } rc = hpsa_scsi_do_simple_cmd(h, c, DEFAULT_REPLY_QUEUE, NO_TIMEOUT); - if (iocommand.buf_size > 0) + if (iocommand->buf_size > 0) hpsa_pci_unmap(h->pdev, c, 1, DMA_BIDIRECTIONAL); check_ioctl_unit_attention(h, c); if (rc) { @@ -6436,16 +6424,12 @@ static int hpsa_passthru_ioctl(struct ctlr_info *h, void __user *argp) } /* Copy the error information out */ - memcpy(&iocommand.error_info, c->err_info, - sizeof(iocommand.error_info)); - if (copy_to_user(argp, &iocommand, sizeof(iocommand))) { - rc = -EFAULT; - goto out; - } - if ((iocommand.Request.Type.Direction & XFER_READ) && - iocommand.buf_size > 0) { + memcpy(&iocommand->error_info, c->err_info, + sizeof(iocommand->error_info)); + if ((iocommand->Request.Type.Direction & XFER_READ) && + iocommand->buf_size > 0) { /* Copy the data out of the buffer we created */ - if (copy_to_user(iocommand.buf, buff, iocommand.buf_size)) { + if (copy_to_user(iocommand->buf, buff, iocommand->buf_size)) { rc = -EFAULT; goto out; } @@ -6457,9 +6441,9 @@ out_kfree: return rc; } -static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp) +static int hpsa_big_passthru_ioctl(struct ctlr_info *h, + BIG_IOCTL_Command_struct *ioc) { - BIG_IOCTL_Command_struct *ioc; struct CommandList *c; unsigned char **buff = NULL; int *buff_size = NULL; @@ -6470,29 +6454,17 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp) u32 sz; BYTE __user *data_ptr; - if (!argp) - return -EINVAL; if (!capable(CAP_SYS_RAWIO)) return -EPERM; - ioc = vmemdup_user(argp, sizeof(*ioc)); - if (IS_ERR(ioc)) { - status = PTR_ERR(ioc); - goto cleanup1; - } + if ((ioc->buf_size < 1) && - (ioc->Request.Type.Direction != XFER_NONE)) { - status = -EINVAL; - goto cleanup1; - } + (ioc->Request.Type.Direction != XFER_NONE)) + return -EINVAL; /* Check kmalloc limits using all SGs */ - if (ioc->malloc_size > MAX_KMALLOC_SIZE) { - status = -EINVAL; - goto cleanup1; - } - if (ioc->buf_size > ioc->malloc_size * SG_ENTRIES_IN_CMD) { - status = -EINVAL; - goto cleanup1; - } + if (ioc->malloc_size > MAX_KMALLOC_SIZE) + return -EINVAL; + if (ioc->buf_size > ioc->malloc_size * SG_ENTRIES_IN_CMD) + return -EINVAL; buff = kcalloc(SG_ENTRIES_IN_CMD, sizeof(char *), GFP_KERNEL); if (!buff) { status = -ENOMEM; @@ -6565,10 +6537,6 @@ static int hpsa_big_passthru_ioctl(struct ctlr_info *h, void __user *argp) /* Copy the error information out */ memcpy(&ioc->error_info, c->err_info, sizeof(ioc->error_info)); - if (copy_to_user(argp, ioc, sizeof(*ioc))) { - status = -EFAULT; - goto cleanup0; - } if ((ioc->Request.Type.Direction & XFER_READ) && ioc->buf_size > 0) { int i; @@ -6594,7 +6562,6 @@ cleanup1: kfree(buff); } kfree(buff_size); - kvfree(ioc); return status; } @@ -6610,14 +6577,11 @@ static void check_ioctl_unit_attention(struct ctlr_info *h, * ioctl */ static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd, - void __user *arg) + void __user *argp) { - struct ctlr_info *h; - void __user *argp = (void __user *)arg; + struct ctlr_info *h = sdev_to_hba(dev); int rc; - h = sdev_to_hba(dev); - switch (cmd) { case CCISS_DEREGDISK: case CCISS_REGNEWDISK: @@ -6628,18 +6592,35 @@ static int hpsa_ioctl(struct scsi_device *dev, unsigned int cmd, return hpsa_getpciinfo_ioctl(h, argp); case CCISS_GETDRIVVER: return hpsa_getdrivver_ioctl(h, argp); - case CCISS_PASSTHRU: + case CCISS_PASSTHRU: { + IOCTL_Command_struct iocommand; + + if (!argp) + return -EINVAL; + if (copy_from_user(&iocommand, argp, sizeof(iocommand))) + return -EFAULT; if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0) return -EAGAIN; - rc = hpsa_passthru_ioctl(h, argp); + rc = hpsa_passthru_ioctl(h, &iocommand); atomic_inc(&h->passthru_cmds_avail); + if (!rc && copy_to_user(argp, &iocommand, sizeof(iocommand))) + rc = -EFAULT; return rc; - case CCISS_BIG_PASSTHRU: + } + case CCISS_BIG_PASSTHRU: { + BIG_IOCTL_Command_struct ioc; + if (!argp) + return -EINVAL; + if (copy_from_user(&ioc, argp, sizeof(ioc))) + return -EFAULT; if (atomic_dec_if_positive(&h->passthru_cmds_avail) < 0) return -EAGAIN; - rc = hpsa_big_passthru_ioctl(h, argp); + rc = hpsa_big_passthru_ioctl(h, &ioc); atomic_inc(&h->passthru_cmds_avail); + if (!rc && copy_to_user(argp, &ioc, sizeof(ioc))) + rc = -EFAULT; return rc; + } default: return -ENOTTY; } diff --git a/drivers/scsi/ibmvscsi/ibmvscsi.c b/drivers/scsi/ibmvscsi/ibmvscsi.c index 44e64aa21194..14f687e9b1f4 100644 --- a/drivers/scsi/ibmvscsi/ibmvscsi.c +++ b/drivers/scsi/ibmvscsi/ibmvscsi.c @@ -415,6 +415,8 @@ static int ibmvscsi_reenable_crq_queue(struct crq_queue *queue, int rc = 0; struct vio_dev *vdev = to_vio_dev(hostdata->dev); + set_adapter_info(hostdata); + /* Re-enable the CRQ */ do { if (rc) diff --git a/drivers/scsi/iscsi_boot_sysfs.c b/drivers/scsi/iscsi_boot_sysfs.c index e4857b728033..a64abe38db2d 100644 --- a/drivers/scsi/iscsi_boot_sysfs.c +++ b/drivers/scsi/iscsi_boot_sysfs.c @@ -352,7 +352,7 @@ iscsi_boot_create_kobj(struct iscsi_boot_kset *boot_kset, boot_kobj->kobj.kset = boot_kset->kset; if (kobject_init_and_add(&boot_kobj->kobj, &iscsi_boot_ktype, NULL, name, index)) { - kfree(boot_kobj); + kobject_put(&boot_kobj->kobj); return NULL; } boot_kobj->data = data; diff --git a/drivers/scsi/mpt3sas/mpt3sas_base.c b/drivers/scsi/mpt3sas/mpt3sas_base.c index beaea1933f5c..96b78fdc6b8a 100644 --- a/drivers/scsi/mpt3sas/mpt3sas_base.c +++ b/drivers/scsi/mpt3sas/mpt3sas_base.c @@ -4984,7 +4984,7 @@ base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz) for (i = 0; i < count; i++) { if ((i % RDPQ_MAX_INDEX_IN_ONE_CHUNK == 0) && dma_alloc_count) { ioc->reply_post[i].reply_post_free = - dma_pool_alloc(ioc->reply_post_free_dma_pool, + dma_pool_zalloc(ioc->reply_post_free_dma_pool, GFP_KERNEL, &ioc->reply_post[i].reply_post_free_dma); if (!ioc->reply_post[i].reply_post_free) @@ -5008,9 +5008,6 @@ base_alloc_rdpq_dma_pool(struct MPT3SAS_ADAPTER *ioc, int sz) ioc->reply_post[i].reply_post_free_dma)); return -EAGAIN; } - memset(ioc->reply_post[i].reply_post_free, 0, - RDPQ_MAX_INDEX_IN_ONE_CHUNK * - reply_post_free_sz); dma_alloc_count--; } else { diff --git a/drivers/scsi/qedf/qedf_fip.c b/drivers/scsi/qedf/qedf_fip.c index bb82f0875eca..ad6a56ce72a8 100644 --- a/drivers/scsi/qedf/qedf_fip.c +++ b/drivers/scsi/qedf/qedf_fip.c @@ -20,7 +20,7 @@ void qedf_fcoe_send_vlan_req(struct qedf_ctx *qedf) #define MY_FIP_ALL_FCF_MACS ((__u8[6]) { 1, 0x10, 0x18, 1, 0, 2 }) static u8 my_fcoe_all_fcfs[ETH_ALEN] = MY_FIP_ALL_FCF_MACS; unsigned long flags = 0; - int rc = -1; + int rc; skb = dev_alloc_skb(sizeof(struct fip_vlan)); if (!skb) { diff --git a/drivers/scsi/sr.c b/drivers/scsi/sr.c index 4dcd735ea49e..0c4aa4665a2f 100644 --- a/drivers/scsi/sr.c +++ b/drivers/scsi/sr.c @@ -792,7 +792,7 @@ static int sr_probe(struct device *dev) disk->queue = sdev->request_queue; if (register_cdrom(disk, &cd->cdi)) - goto fail_put; + goto fail_minor; /* * Initialize block layer runtime PM stuffs before the @@ -810,8 +810,13 @@ static int sr_probe(struct device *dev) return 0; +fail_minor: + spin_lock(&sr_index_lock); + clear_bit(minor, sr_index_bits); + spin_unlock(&sr_index_lock); fail_put: put_disk(disk); + mutex_destroy(&cd->lock); fail_free: kfree(cd); fail: diff --git a/drivers/scsi/st.c b/drivers/scsi/st.c index 4bf4ab3b70f4..87fbc0ea350b 100644 --- a/drivers/scsi/st.c +++ b/drivers/scsi/st.c @@ -4921,7 +4921,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp, unsigned long end = (uaddr + count + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long start = uaddr >> PAGE_SHIFT; const int nr_pages = end - start; - int res, i, j; + int res, i; struct page **pages; struct rq_map_data *mdata = &STbp->map_data; @@ -4943,7 +4943,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp, /* Try to fault in all of the necessary pages */ /* rw==READ means read from drive, write into memory area */ - res = get_user_pages_fast(uaddr, nr_pages, rw == READ ? FOLL_WRITE : 0, + res = pin_user_pages_fast(uaddr, nr_pages, rw == READ ? FOLL_WRITE : 0, pages); /* Errors and no page mapped should return here */ @@ -4963,8 +4963,7 @@ static int sgl_map_user_pages(struct st_buffer *STbp, return nr_pages; out_unmap: if (res > 0) { - for (j=0; j < res; j++) - put_page(pages[j]); + unpin_user_pages(pages, res); res = 0; } kfree(pages); @@ -4976,18 +4975,9 @@ static int sgl_map_user_pages(struct st_buffer *STbp, static int sgl_unmap_user_pages(struct st_buffer *STbp, const unsigned int nr_pages, int dirtied) { - int i; - - for (i=0; i < nr_pages; i++) { - struct page *page = STbp->mapped_pages[i]; + /* FIXME: cache flush missing for rw==READ */ + unpin_user_pages_dirty_lock(STbp->mapped_pages, nr_pages, dirtied); - if (dirtied) - SetPageDirty(page); - /* FIXME: cache flush missing for rw==READ - * FIXME: call the correct reference counting function - */ - put_page(page); - } kfree(STbp->mapped_pages); STbp->mapped_pages = NULL; diff --git a/drivers/scsi/storvsc_drv.c b/drivers/scsi/storvsc_drv.c index 072ed8728657..2d90cddd8ac2 100644 --- a/drivers/scsi/storvsc_drv.c +++ b/drivers/scsi/storvsc_drv.c @@ -2035,9 +2035,6 @@ static int storvsc_suspend(struct hv_device *hv_dev) vmbus_close(hv_dev->channel); - memset(stor_device->stor_chns, 0, - num_possible_cpus() * sizeof(void *)); - kfree(stor_device->stor_chns); stor_device->stor_chns = NULL; diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c index 5db18f444ea9..ad4fc829cbb2 100644 --- a/drivers/scsi/ufs/ufshcd.c +++ b/drivers/scsi/ufs/ufshcd.c @@ -5076,6 +5076,7 @@ static void ufshcd_force_reset_auto_bkops(struct ufs_hba *hba) hba->ee_ctrl_mask &= ~MASK_EE_URGENT_BKOPS; ufshcd_disable_auto_bkops(hba); } + hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; hba->is_urgent_bkops_lvl_checked = false; } @@ -5123,7 +5124,6 @@ static int ufshcd_bkops_ctrl(struct ufs_hba *hba, err = ufshcd_enable_auto_bkops(hba); else err = ufshcd_disable_auto_bkops(hba); - hba->urgent_bkops_lvl = curr_status; out: return err; } @@ -7373,10 +7373,6 @@ static int ufshcd_probe_hba(struct ufs_hba *hba, bool async) if (ret) goto out; - /* set the default level for urgent bkops */ - hba->urgent_bkops_lvl = BKOPS_STATUS_PERF_IMPACT; - hba->is_urgent_bkops_lvl_checked = false; - /* Debug counters initialization */ ufshcd_clear_dbg_ufs_stats(hba); diff --git a/drivers/staging/media/atomisp/Kconfig b/drivers/staging/media/atomisp/Kconfig index ba2855f656b6..fea06cb0eb48 100644 --- a/drivers/staging/media/atomisp/Kconfig +++ b/drivers/staging/media/atomisp/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 menuconfig INTEL_ATOMISP bool "Enable support to Intel Atom ISP camera drivers" depends on X86 && EFI && PCI && ACPI @@ -11,6 +12,7 @@ menuconfig INTEL_ATOMISP config VIDEO_ATOMISP tristate "Intel Atom Image Signal Processor Driver" depends on VIDEO_V4L2 && INTEL_ATOMISP + depends on PMIC_OPREGION select IOSF_MBI select VIDEOBUF_VMALLOC help diff --git a/drivers/staging/media/atomisp/Makefile b/drivers/staging/media/atomisp/Makefile index eedecd49bbf4..9dc8072799e3 100644 --- a/drivers/staging/media/atomisp/Makefile +++ b/drivers/staging/media/atomisp/Makefile @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # Makefile for camera drivers. # @@ -36,7 +37,6 @@ atomisp-objs += \ pci/sh_css_param_dvs.o \ pci/sh_css_param_shading.o \ pci/sh_css_params.o \ - pci/sh_css_pipe.o \ pci/sh_css_properties.o \ pci/sh_css_shading.o \ pci/sh_css_sp.o \ @@ -53,10 +53,7 @@ atomisp-objs += \ pci/hmm/hmm_dynamic_pool.o \ pci/hmm/hmm.o \ pci/hmm/hmm_reserved_pool.o \ - pci/hmm/hmm_vm.o \ - pci/hrt/hive_isp_css_mm_hrt.o \ pci/ia_css_device_access.o \ - pci/ia_css_memory_access.o \ pci/isp/kernels/aa/aa_2/ia_css_aa2.host.o \ pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.o \ pci/isp/kernels/anr/anr_2/ia_css_anr2.host.o \ @@ -117,7 +114,6 @@ atomisp-objs += \ pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.o \ pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.o \ pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.o \ - pci/memory_realloc.o \ pci/mmu/isp_mmu.o \ pci/mmu/sh_mmu_mrfld.o \ pci/runtime/binary/src/binary.o \ @@ -328,11 +324,6 @@ INCLUDES_cht += \ # -I$(atomisp)/pci/css_2401_system/hrt/ \ # -I$(atomisp)/pci/css_2401_system/hive_isp_css_2401_system_generated/ \ - -ifeq ($(CONFIG_ION),y) -INCLUDES += -I$(srctree)/drivers/staging/android/ion -endif - DEFINES := -DHRT_HW -DHRT_ISP_CSS_CUSTOM_HOST -DHRT_USE_VIR_ADDRS -D__HOST__ #DEFINES += -DUSE_DYNAMIC_BIN #DEFINES += -DISP_POWER_GATING @@ -352,12 +343,3 @@ DEFINES += -DISP2400 -DSYSTEM_hive_isp_css_2400_system endif ccflags-y += $(INCLUDES) $(DEFINES) -fno-common - -# HACK! While this driver is in bad shape, don't enable several warnings -# that would be otherwise enabled with W=1 -ccflags-y += $(call cc-disable-warning, implicit-fallthrough) -ccflags-y += $(call cc-disable-warning, missing-prototypes) -ccflags-y += $(call cc-disable-warning, missing-declarations) -ccflags-y += $(call cc-disable-warning, suggest-attribute=format) -ccflags-y += $(call cc-disable-warning, unused-const-variable) -ccflags-y += $(call cc-disable-warning, unused-but-set-variable) diff --git a/drivers/staging/media/atomisp/TODO b/drivers/staging/media/atomisp/TODO index 52683a704223..6987bb2d32cf 100644 --- a/drivers/staging/media/atomisp/TODO +++ b/drivers/staging/media/atomisp/TODO @@ -1,71 +1,127 @@ -1. A single AtomISP driver needs to be implemented to support both - Baytrail (BYT and Cherrytail (CHT) platforms at the same time. +NOTE: +===== + +While the driver probes the hardware and reports itself as a +V4L2 driver, there are still some issues preventing it to +stream (at least it doesn't with the standard V4L2 applications. +Didn't test yet with some custom-made app for this driver). +Solving the related bugs and issues preventing it to work is +needed (items 6 and 7 from the list below). + +TODO +==== + +1. The atomisp doesn't rely at the usual i2c stuff to discover the + sensors. Instead, it calls a function from atomisp_gmin_platform.c. + There are some hacks added there for it to wait for sensors to be + probed (with a timeout of 2 seconds or so). + This should be converted to the usual way, using V4L2 async subdev + framework to wait for cameras to be probed; + +2. Use ACPI _DSM table - DONE! + +3. Switch the driver to use pm_runtime stuff. Right now, it probes the + existing PMIC code and sensors call it directly. + +4. There's a problem at the sensor drivers: when trying to set a video + format, the atomisp main driver calls the sensor drivers with the + sensor turned off. This causes them to fail. + + The only exception is the atomisp-ov2880, which has a hack inside it + to turn it on when VIDIOC_S_FMT is called. + + The right fix seems to power on the sensor when a video device is + opened (or at the first VIDIOC_ ioctl - except for VIDIOC_QUERYCAP), + powering it down at close() syscall. + + Such kind of control would need to be done inside the atomisp driver, + not at the sensors code. + +5. There are several issues related to memory management, causing + crashes. The atomisp splits the memory management on three separate + regions: + + - dynamic pool; + - reserved pool; + - generic pool + + The code implementing it is at: + + drivers/staging/media/atomisp/pci/hmm/ + + It also has a separate code for managing DMA buffers at: + + drivers/staging/media/atomisp/pci/mmu/ + + The code there is really dirty, ugly and probably wrong. I fixed + one bug there already, but the best would be to just trash it and use + something else. Maybe the code from the newer intel driver could + serve as a model: + + drivers/staging/media/ipu3/ipu3-mmu.c + + But converting it to use something like that is painful and may + cause some breakages. + +6. There is some issues at the frame receive logic, causing the + DQBUF ioctls to fail. + +7. A single AtomISP driver needs to be implemented to support both + Baytrail (BYT) and Cherrytail (CHT) platforms at the same time. The current driver is a mechanical and hand combined merge of the two using several runtime macros, plus some ifdef ISP2401 to select the CHT version. Yet, there are some ISP-specific headers that change the driver's behavior during compile time. -2. The file structure needs to get tidied up to resemble a normal Linux +8. The file structure needs to get tidied up to resemble a normal Linux driver. -3. Lots of the midlayer glue. unused code and abstraction needs removing. +9. Lots of the midlayer glue. unused code and abstraction needs removing. -3. The sensor drivers read MIPI settings from EFI variables or default to the - settings hard-coded in the platform data file for different platforms. - It should be possible to improve it, by adding support for _DSM tables. +10. The AtomISP driver includes some special IOCTLS (ATOMISP_IOC_XXXX_XXXX) + and controls that require some cleanup. Some of those code may have + been removed during the cleanups. They could be needed in order to + properly support 3A algorithms -4. The sensor drivers use PMIC and the regulator framework API. In the ideal - world it would be using ACPI but that's not how the existing devices work. + Such IOCTL interface needs more documentation. The better would + be to use something close to the interface used by the IPU3 IMGU driver. -5. The AtomISP driver includes some special IOCTLS (ATOMISP_IOC_XXXX_XXXX) - and controls that require some cleanup. +11. The ISP code has some dependencies of the exact FW version. + The version defined in pci/sh_css_firmware.c: -6. Correct Coding Style. Please don't send coding style patches for this - driver until the other work is done. + BYT (isp2400): "irci_stable_candrpv_0415_20150521_0458" -7. The ISP code has some dependencies of the exact FW version. - The version defined in pci/sh_css_firmware.c: - BYT: - static const char *isp2400_release_version = STR(irci_stable_candrpv_0415_20150521_0458); + CHT (isp2401): "irci_ecr - master_20150911_0724" - CHT: - static const char *isp2401_release_version = STR(irci_ecr - master_20150911_0724); + Those versions don't seem to be available anymore. On the tests we've + done so far, this version also seems to work for CHT: - Those versions don't seem to be available anymore. On the tests we've - done so far, this version also seems to work for isp2401: + "irci_stable_candrpv_0415_20150521_0458" - irci_stable_candrpv_0415_20150521_0458 + Which can be obtainable from Yocto Atom ISP respository. - At some point we may need to round up a few driver versions and see if - there are any specific things that can be done to fold in support for - multiple firmware versions. + but this was not thoroughly tested. -8. Switch to V4L2 async API to set up sensor, lens and flash devices. - Control those devices using V4L2 sub-device API without custom - extensions. + At some point we may need to round up a few driver versions and see if + there are any specific things that can be done to fold in support for + multiple firmware versions. -9. Switch to standard V4L2 sub-device API for sensor and lens. In - particular, the user space API needs to support V4L2 controls as - defined in the V4L2 spec and references to atomisp must be removed from - these drivers. +12. Switch to standard V4L2 sub-device API for sensor and lens. In + particular, the user space API needs to support V4L2 controls as + defined in the V4L2 spec and references to atomisp must be removed from + these drivers. -10. Use LED flash API for flash LED drivers such as LM3554 (which already +13. Use LED flash API for flash LED drivers such as LM3554 (which already has a LED class driver). -11. Switch from videobuf1 to videobuf2. Videobuf1 is being removed! - -12. There are some memory management code that seems to be - forked from Kernel 3.10 inside hmm/ directory. Get rid of it, - making the driver to use a more standard memory management module. +14. Switch from videobuf1 to videobuf2. Videobuf1 is being removed! -13. While the driver probes the hardware and reports itself as a - V4L2 driver, there are still some issues preventing it to - stream (at least it doesn't with the standard V4L2 applications. - Didn't test yet with some custom-made app for this driver). - Solving the related bugs and issues preventing it to work is - needed. +15. Correct Coding Style. Please refrain sending coding style patches + for this driver until the other work is done, as there will be a lot + of code churn until this driver becomes functional again. -Limitations: +Limitations +=========== 1. To test the patches, you also need the ISP firmware @@ -76,14 +132,16 @@ Limitations: device but can also be extracted from the upgrade kit if you've managed to lose them somehow. -2. Without a 3A libary the capture behaviour is not very good. To take a good - picture, you need tune ISP parameters by IOCTL functions or use a 3A libary +2. Without a 3A library the capture behaviour is not very good. To take a good + picture, you need tune ISP parameters by IOCTL functions or use a 3A library such as libxcam. 3. The driver is intended to drive the PCI exposed versions of the device. It will not detect those devices enumerated via ACPI as a field of the i915 GPU driver. + There are some patches adding i915 GPU support floating at the Yocto's + Aero repository (so far, untested upstream). + 4. The driver supports only v2 of the IPU/Camera. It will not work with the versions of the hardware in other SoCs. - diff --git a/drivers/staging/media/atomisp/i2c/Kconfig b/drivers/staging/media/atomisp/i2c/Kconfig index 96f33a4e5498..7c7f0fc090b3 100644 --- a/drivers/staging/media/atomisp/i2c/Kconfig +++ b/drivers/staging/media/atomisp/i2c/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 # # Kconfig for sensor drivers # @@ -57,7 +58,7 @@ config VIDEO_ATOMISP_GC0310 help This is a Video4Linux2 sensor-level driver for the Galaxycore GC0310 0.3MP sensor. - + config VIDEO_ATOMISP_OV2680 tristate "Omnivision OV2680 sensor support" depends on ACPI diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c index ad1bd7d6a02b..2b71de722ec3 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc0310.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for GalaxyCore GC0310 VGA camera sensor. * @@ -1309,18 +1310,6 @@ static int gc0310_probe(struct i2c_client *client) int ret; void *pdata; unsigned int i; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() - dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c index a12dd0e858bc..78147ffb6099 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for GalaxyCore GC2235 2M camera sensor. * @@ -1051,17 +1052,6 @@ static int gc2235_probe(struct i2c_client *client) void *gcpdev; int ret; unsigned int i; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c index 33ab884f7352..b93c80471f22 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-libmsrlisthelper.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2013 Intel Corporation. All Rights Reserved. * diff --git a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c index a899145265ff..809010af7855 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-lm3554.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * LED flash driver for LM3554 * @@ -850,17 +851,6 @@ static int lm3554_probe(struct i2c_client *client) struct lm3554 *flash; unsigned int i; int ret; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() flash = kzalloc(sizeof(*flash), GFP_KERNEL); if (!flash) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c index ac61b391e3f9..0d60918a9b19 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-mt9m114.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for mt9m114 Camera Sensor. * @@ -1816,17 +1817,6 @@ static int mt9m114_probe(struct i2c_client *client) int ret = 0; unsigned int i; void *pdata; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() /* Setup sensor configuration structure */ dev = kzalloc(sizeof(*dev), GFP_KERNEL); diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c index 1b60f6a9c0e0..90d125ba080f 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2680.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for OmniVision OV2680 1080p HD camera sensor. * @@ -703,7 +704,7 @@ static int power_ctrl(struct v4l2_subdev *sd, bool flag) if (!dev || !dev->platform_data) return -ENODEV; - dev_dbg(&client->dev, "%s: %s", __func__, flag? "on" : "off"); + dev_dbg(&client->dev, "%s: %s", __func__, flag ? "on" : "off"); if (flag) { ret |= dev->platform_data->v1p8_ctrl(sd, 1); @@ -1243,17 +1244,6 @@ static int ov2680_probe(struct i2c_client *client) int ret; void *pdata; unsigned int i; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - dev_info(&client->dev, "%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c index 718d10f89d5a..eecefcd734d0 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for OmniVision OV2722 1080p HD camera sensor. * @@ -1214,17 +1215,6 @@ static int ov2722_probe(struct i2c_client *client) struct ov2722_device *dev; void *ovpdev; int ret; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() dev = kzalloc(sizeof(*dev), GFP_KERNEL); if (!dev) diff --git a/drivers/staging/media/atomisp/i2c/gc0310.h b/drivers/staging/media/atomisp/i2c/gc0310.h index 12f746e7828b..2fe3de115083 100644 --- a/drivers/staging/media/atomisp/i2c/gc0310.h +++ b/drivers/staging/media/atomisp/i2c/gc0310.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for GalaxyCore GC0310 VGA camera sensor. * diff --git a/drivers/staging/media/atomisp/i2c/gc2235.h b/drivers/staging/media/atomisp/i2c/gc2235.h index bb104de61af9..68252b8f516d 100644 --- a/drivers/staging/media/atomisp/i2c/gc2235.h +++ b/drivers/staging/media/atomisp/i2c/gc2235.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for GalaxyCore GC2235 2M camera sensor. * diff --git a/drivers/staging/media/atomisp/i2c/mt9m114.h b/drivers/staging/media/atomisp/i2c/mt9m114.h index 172cec0bb398..787bbf59e895 100644 --- a/drivers/staging/media/atomisp/i2c/mt9m114.h +++ b/drivers/staging/media/atomisp/i2c/mt9m114.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for mt9m114 Camera Sensor. * diff --git a/drivers/staging/media/atomisp/i2c/ov2680.h b/drivers/staging/media/atomisp/i2c/ov2680.h index 034e1032f6c0..49920245e064 100644 --- a/drivers/staging/media/atomisp/i2c/ov2680.h +++ b/drivers/staging/media/atomisp/i2c/ov2680.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for OmniVision OV2680 5M camera sensor. * @@ -456,6 +457,7 @@ static struct ov2680_reg const ov2680_656x496_30fps[] = { // {0x5090, 0x0c}, {} }; + /* * 800x600 30fps VBlanking 1lane 10Bit (binning) */ @@ -500,6 +502,7 @@ static struct ov2680_reg const ov2680_720x592_30fps[] = { {0x5081, 0x41}, {} }; + /* * 800x600 30fps VBlanking 1lane 10Bit (binning) */ diff --git a/drivers/staging/media/atomisp/i2c/ov2722.h b/drivers/staging/media/atomisp/i2c/ov2722.h index 1110d723968e..7b0debb6c53d 100644 --- a/drivers/staging/media/atomisp/i2c/ov2722.h +++ b/drivers/staging/media/atomisp/i2c/ov2722.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for OmniVision OV2722 1080p HD camera sensor. * diff --git a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig b/drivers/staging/media/atomisp/i2c/ov5693/Kconfig index 038b0b818c3d..c8d09f416c35 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/Kconfig +++ b/drivers/staging/media/atomisp/i2c/ov5693/Kconfig @@ -1,3 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 config VIDEO_ATOMISP_OV5693 tristate "Omnivision ov5693 sensor support" depends on ACPI diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h b/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h index c97ab24c5d2b..f1362cd69f6e 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h +++ b/drivers/staging/media/atomisp/i2c/ov5693/ad5823.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for AD5823 VCM. * diff --git a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c index 2be0ef14d53e..97ab10bc45ca 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c +++ b/drivers/staging/media/atomisp/i2c/ov5693/atomisp-ov5693.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for OmniVision OV5693 1080p HD camera sensor. * @@ -1087,7 +1088,7 @@ static int ov5693_s_ctrl(struct v4l2_ctrl *ctrl) case V4L2_CID_VCM_SLEW: ret = ov5693_t_vcm_slew(&dev->sd, ctrl->val); break; - case V4L2_CID_VCM_TIMEING: + case V4L2_CID_VCM_TIMING: ret = ov5693_t_vcm_timing(&dev->sd, ctrl->val); break; default: @@ -1230,7 +1231,7 @@ static const struct v4l2_ctrl_config ov5693_controls[] = { }, { .ops = &ctrl_ops, - .id = V4L2_CID_VCM_TIMEING, + .id = V4L2_CID_VCM_TIMING, .type = V4L2_CTRL_TYPE_INTEGER, .name = "vcm step time", .min = 0, @@ -1901,17 +1902,6 @@ static int ov5693_probe(struct i2c_client *client) int ret = 0; void *pdata; unsigned int i; - acpi_handle handle; - struct acpi_device *adev; - - handle = ACPI_HANDLE(&client->dev); - if (!handle || acpi_bus_get_device(handle, &adev)) { - dev_err(&client->dev, "Error could not get ACPI device\n"); - return -ENODEV; - } - pr_info("%s: ACPI detected it on bus ID=%s, HID=%s\n", - __func__, acpi_device_bid(adev), acpi_device_hid(adev)); - // FIXME: may need to release resources allocated by acpi_bus_get_device() /* * Firmware workaround: Some modules use a "secondary default" diff --git a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h index 0189907969c6..79df07bd69b6 100644 --- a/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h +++ b/drivers/staging/media/atomisp/i2c/ov5693/ov5693.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for OmniVision OV5693 5M camera sensor. * diff --git a/drivers/staging/media/atomisp/include/hmm/hmm.h b/drivers/staging/media/atomisp/include/hmm/hmm.h index 254a71442451..b48bdf5c274c 100644 --- a/drivers/staging/media/atomisp/include/hmm/hmm.h +++ b/drivers/staging/media/atomisp/include/hmm/hmm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -28,8 +29,8 @@ #include "hmm/hmm_pool.h" #include "ia_css_types.h" -#define HMM_CACHED true -#define HMM_UNCACHED false +#define mmgr_NULL ((ia_css_ptr)0) +#define mmgr_EXCEPTION ((ia_css_ptr) - 1) int hmm_pool_register(unsigned int pool_size, enum hmm_pool_type pool_type); void hmm_pool_unregister(enum hmm_pool_type pool_type); @@ -38,7 +39,8 @@ int hmm_init(void); void hmm_cleanup(void); ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type, - int from_highmem, const void __user *userptr, bool cached); + int from_highmem, const void __user *userptr, + const uint16_t attrs); void hmm_free(ia_css_ptr ptr); int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes); int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes); diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_bo.h b/drivers/staging/media/atomisp/include/hmm/hmm_bo.h index f847d1de860e..8c78a5d87b65 100644 --- a/drivers/staging/media/atomisp/include/hmm/hmm_bo.h +++ b/drivers/staging/media/atomisp/include/hmm/hmm_bo.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -77,9 +78,6 @@ enum hmm_bo_type { HMM_BO_PRIVATE, HMM_BO_SHARE, HMM_BO_USER, -#ifdef CONFIG_ION - HMM_BO_ION, -#endif HMM_BO_LAST, }; @@ -111,9 +109,6 @@ struct hmm_bo_device { /* list lock is used to protect the entire_bo_list */ spinlock_t list_lock; -#ifdef CONFIG_ION - struct ion_client *iclient; -#endif int flag; /* linked list for entire buffer object */ @@ -136,15 +131,14 @@ struct hmm_buffer_object { struct list_head list; struct kref kref; + struct page **pages; + /* mutex protecting this BO */ struct mutex mutex; enum hmm_bo_type type; struct hmm_page_object *page_obj; /* physical pages */ int from_highmem; int mmap_count; -#ifdef CONFIG_ION - struct ion_handle *ihandle; -#endif int status; int mem_type; void *vmap_addr; /* kernel virtual address by vmap */ diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_common.h b/drivers/staging/media/atomisp/include/hmm/hmm_common.h index 00885203fb14..7152e9b52ba4 100644 --- a/drivers/staging/media/atomisp/include/hmm/hmm_common.h +++ b/drivers/staging/media/atomisp/include/hmm/hmm_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_pool.h b/drivers/staging/media/atomisp/include/hmm/hmm_pool.h index 8caf00502d74..3fef57de973c 100644 --- a/drivers/staging/media/atomisp/include/hmm/hmm_pool.h +++ b/drivers/staging/media/atomisp/include/hmm/hmm_pool.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/include/hmm/hmm_vm.h b/drivers/staging/media/atomisp/include/hmm/hmm_vm.h deleted file mode 100644 index 93ac5e445137..000000000000 --- a/drivers/staging/media/atomisp/include/hmm/hmm_vm.h +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Support for Medifield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ - -#ifndef __HMM_VM_H__ -#define __HMM_VM_H__ - -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/mutex.h> -#include <linux/list.h> - -struct hmm_vm { - unsigned int start; - unsigned int pgnr; - unsigned int size; - struct list_head vm_node_list; - spinlock_t lock; - struct kmem_cache *cache; -}; - -struct hmm_vm_node { - struct list_head list; - unsigned int start; - unsigned int pgnr; - unsigned int size; - struct hmm_vm *vm; -}; - -#define ISP_VM_START 0x0 -#define ISP_VM_SIZE (0x7FFFFFFF) /* 2G address space */ -#define ISP_PTR_NULL NULL - -int hmm_vm_init(struct hmm_vm *vm, unsigned int start, - unsigned int size); - -void hmm_vm_clean(struct hmm_vm *vm); - -struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm, - unsigned int pgnr); - -void hmm_vm_free_node(struct hmm_vm_node *node); - -struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm, - unsigned int addr); - -struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm, - unsigned int addr); - -#endif diff --git a/drivers/staging/media/atomisp/include/linux/atomisp.h b/drivers/staging/media/atomisp/include/linux/atomisp.h index e9670749bae0..22c4103b0385 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -69,9 +70,6 @@ #define V4L2_MBUS_FMT_CUSTOM_M10MO_RAW 0x800b #endif -/* FIXME: for now, let's use a boolean to identify the type of atomisp chipset */ -extern bool atomisp_hw_is_isp2401; - /* Configuration used by Bayer noise reduction and YCC noise reduction */ struct atomisp_nr_config { /* [gain] Strength of noise reduction for Bayer NR (Used by Bayer NR) */ @@ -509,7 +507,7 @@ struct atomisp_parameters { struct atomisp_shading_table *shading_table; struct atomisp_morph_table *morph_table; struct atomisp_dvs_coefficients *dvs_coefs; /* DVS 1.0 coefficients */ - struct atomisp_dvs2_coefficients *dvs2_coefs; /* DVS 2.0 coefficients */ + struct atomisp_dis_coefficients *dvs2_coefs; /* DVS 2.0 coefficients */ struct atomisp_capture_config *capture_config; struct atomisp_anr_thres *anr_thres; @@ -917,6 +915,8 @@ struct atomisp_acc_map { #define ATOMISP_MAP_FLAG_NOFLUSH 0x0001 /* Do not flush cache */ #define ATOMISP_MAP_FLAG_CACHED 0x0002 /* Enable cache */ +#define ATOMISP_MAP_FLAG_CONTIGUOUS 0x0004 +#define ATOMISP_MAP_FLAG_CLEARED 0x0008 struct atomisp_acc_state { __u32 flags; /* Flags, see list below */ @@ -1272,7 +1272,7 @@ struct atomisp_sensor_ae_bracketing_lut { /* VCM slew control */ #define V4L2_CID_VCM_SLEW (V4L2_CID_CAMERA_LASTP1 + 11) /* VCM step time */ -#define V4L2_CID_VCM_TIMEING (V4L2_CID_CAMERA_LASTP1 + 12) +#define V4L2_CID_VCM_TIMING (V4L2_CID_CAMERA_LASTP1 + 12) /* Query Focus Status */ #define V4L2_CID_FOCUS_STATUS (V4L2_CID_CAMERA_LASTP1 + 14) diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h index 8700ffd32bdb..58e0ea5355a3 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp_gmin_platform.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel MID SoC Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h index 9cf46325a696..873344a02ccf 100644 --- a/drivers/staging/media/atomisp/include/linux/atomisp_platform.h +++ b/drivers/staging/media/atomisp/include/linux/atomisp_platform.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -17,6 +18,9 @@ #ifndef ATOMISP_PLATFORM_H_ #define ATOMISP_PLATFORM_H_ +#include <asm/intel-family.h> +#include <asm/processor.h> + #include <linux/i2c.h> #include <linux/sfi.h> #include <media/v4l2-subdev.h> @@ -237,11 +241,19 @@ const struct atomisp_camera_caps *atomisp_get_default_camera_caps(void); /* API from old platform_camera.h, new CPUID implementation */ #define __IS_SOC(x) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \ boot_cpu_data.x86 == 6 && \ - boot_cpu_data.x86_model == x) + boot_cpu_data.x86_model == (x)) +#define __IS_SOCS(x,y) (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL && \ + boot_cpu_data.x86 == 6 && \ + (boot_cpu_data.x86_model == (x) || \ + boot_cpu_data.x86_model == (y))) + +#define IS_MFLD __IS_SOC(INTEL_FAM6_ATOM_SALTWELL_MID) +#define IS_BYT __IS_SOC(INTEL_FAM6_ATOM_SILVERMONT) +#define IS_CHT __IS_SOC(INTEL_FAM6_ATOM_AIRMONT) +#define IS_MOFD __IS_SOC(INTEL_FAM6_ATOM_AIRMONT_MID) -#define IS_MFLD __IS_SOC(0x27) -#define IS_BYT __IS_SOC(0x37) -#define IS_CHT __IS_SOC(0x4C) -#define IS_MOFD __IS_SOC(0x5A) +/* Both CHT and MOFD come with ISP2401 */ +#define IS_ISP2401 __IS_SOCS(INTEL_FAM6_ATOM_AIRMONT, \ + INTEL_FAM6_ATOM_AIRMONT_MID) #endif /* ATOMISP_PLATFORM_H_ */ diff --git a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h b/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h index 1b5111ad9415..abc8fa809bce 100644 --- a/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h +++ b/drivers/staging/media/atomisp/include/linux/libmsrlisthelper.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2013 Intel Corporation. All Rights Reserved. * diff --git a/drivers/staging/media/atomisp/include/media/lm3554.h b/drivers/staging/media/atomisp/include/media/lm3554.h index 03a916ade531..812ce74f0635 100644 --- a/drivers/staging/media/atomisp/include/media/lm3554.h +++ b/drivers/staging/media/atomisp/include/media/lm3554.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * include/media/lm3554.h * diff --git a/drivers/staging/media/atomisp/include/mmu/isp_mmu.h b/drivers/staging/media/atomisp/include/mmu/isp_mmu.h index d9662c515236..268560954792 100644 --- a/drivers/staging/media/atomisp/include/mmu/isp_mmu.h +++ b/drivers/staging/media/atomisp/include/mmu/isp_mmu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h b/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h index 662e98f41da2..84fe7a368c14 100644 --- a/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h +++ b/drivers/staging/media/atomisp/include/mmu/sh_mmu_mrfld.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Merrifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp-regs.h b/drivers/staging/media/atomisp/pci/atomisp-regs.h index cc489a331a7c..de34ee28e390 100644 --- a/drivers/staging/media/atomisp/pci/atomisp-regs.h +++ b/drivers/staging/media/atomisp/pci/atomisp-regs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.c b/drivers/staging/media/atomisp/pci/atomisp_acc.c index 8d575eb0a73f..76861396ba86 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_acc.c +++ b/drivers/staging/media/atomisp/pci/atomisp_acc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Clovertrail PNW Camera Imaging ISP subsystem. * @@ -23,24 +24,24 @@ #include <linux/init.h> #include <media/v4l2-event.h> +#include "hmm.h" + #include "atomisp_acc.h" #include "atomisp_internal.h" #include "atomisp_compat.h" #include "atomisp_cmd.h" -#include "hrt/hive_isp_css_mm_hrt.h" -#include "memory_access/memory_access.h" #include "ia_css.h" static const struct { unsigned int flag; - enum atomisp_css_pipe_id pipe_id; + enum ia_css_pipe_id pipe_id; } acc_flag_to_pipe[] = { - { ATOMISP_ACC_FW_LOAD_FL_PREVIEW, CSS_PIPE_ID_PREVIEW }, - { ATOMISP_ACC_FW_LOAD_FL_COPY, CSS_PIPE_ID_COPY }, - { ATOMISP_ACC_FW_LOAD_FL_VIDEO, CSS_PIPE_ID_VIDEO }, - { ATOMISP_ACC_FW_LOAD_FL_CAPTURE, CSS_PIPE_ID_CAPTURE }, - { ATOMISP_ACC_FW_LOAD_FL_ACC, CSS_PIPE_ID_ACC } + { ATOMISP_ACC_FW_LOAD_FL_PREVIEW, IA_CSS_PIPE_ID_PREVIEW }, + { ATOMISP_ACC_FW_LOAD_FL_COPY, IA_CSS_PIPE_ID_COPY }, + { ATOMISP_ACC_FW_LOAD_FL_VIDEO, IA_CSS_PIPE_ID_VIDEO }, + { ATOMISP_ACC_FW_LOAD_FL_CAPTURE, IA_CSS_PIPE_ID_CAPTURE }, + { ATOMISP_ACC_FW_LOAD_FL_ACC, IA_CSS_PIPE_ID_ACC } }; /* @@ -353,16 +354,23 @@ int atomisp_acc_map(struct atomisp_sub_device *asd, struct atomisp_acc_map *map) } pgnr = DIV_ROUND_UP(map->length, PAGE_SIZE); - cssptr = hrt_isp_css_mm_alloc_user_ptr(map->length, - map->user_ptr, - pgnr, HRT_USR_PTR, - (map->flags & ATOMISP_MAP_FLAG_CACHED)); + if (pgnr < ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) { + dev_err(atomisp_dev, + "user space memory size is less than the expected size..\n"); + return -ENOMEM; + } else if (pgnr > ((PAGE_ALIGN(map->length)) >> PAGE_SHIFT)) { + dev_err(atomisp_dev, + "user space memory size is large than the expected size..\n"); + return -ENOMEM; + } + + cssptr = hmm_alloc(map->length, HMM_BO_USER, 0, map->user_ptr, + map->flags & ATOMISP_MAP_FLAG_CACHED); + } else { /* Allocate private buffer. */ - if (map->flags & ATOMISP_MAP_FLAG_CACHED) - cssptr = hrt_isp_css_mm_calloc_cached(map->length); - else - cssptr = hrt_isp_css_mm_calloc(map->length); + cssptr = hmm_alloc(map->length, HMM_BO_PRIVATE, 0, NULL, + map->flags & ATOMISP_MAP_FLAG_CACHED); } if (!cssptr) @@ -552,7 +560,7 @@ int atomisp_acc_set_state(struct atomisp_sub_device *asd, struct atomisp_acc_fw *acc_fw; bool enable = (arg->flags & ATOMISP_STATE_FLAG_ENABLE) != 0; struct ia_css_pipe *pipe; - enum ia_css_err r; + int r; int i; if (!asd->acc.extension_mode) @@ -574,7 +582,7 @@ int atomisp_acc_set_state(struct atomisp_sub_device *asd, pipes[acc_flag_to_pipe[i].pipe_id]; r = ia_css_pipe_set_qos_ext_state(pipe, acc_fw->handle, enable); - if (r != IA_CSS_SUCCESS) + if (r) return -EBADRQC; } } diff --git a/drivers/staging/media/atomisp/pci/atomisp_acc.h b/drivers/staging/media/atomisp/pci/atomisp_acc.h index ba14181962f8..48d94232229b 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_acc.h +++ b/drivers/staging/media/atomisp/pci/atomisp_acc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Clovertrail PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.c b/drivers/staging/media/atomisp/pci/atomisp_cmd.c index 5be690f876c1..7b936e5a5f03 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_cmd.c +++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -16,6 +17,7 @@ * * */ +#include <linux/errno.h> #include <linux/firmware.h> #include <linux/pci.h> #include <linux/interrupt.h> @@ -44,7 +46,7 @@ #include "atomisp_subdev.h" #include "atomisp_dfs_tables.h" -#include "hrt/hive_isp_css_mm_hrt.h" +#include <hmm/hmm.h> #include "sh_css_hrt.h" #include "sh_css_defs.h" @@ -57,7 +59,7 @@ #include "ia_css_types.h" #include "ia_css_stream.h" -#include "error_support.h" +#include "ia_css_debug.h" #include "bits.h" /* We should never need to run the flash for more than 2 frames. @@ -241,8 +243,12 @@ int atomisp_freq_scaling(struct atomisp_device *isp, } fps = atomisp_get_sensor_fps(asd); - if (fps == 0) - return -EINVAL; + if (fps == 0) { + dev_info(isp->dev, + "Sensor didn't report FPS. Using DFS max mode.\n"); + new_freq = dfs->highest_freq; + goto done; + } curr_rules.width = asd->fmt[asd->capture_pad].fmt.width; curr_rules.height = asd->fmt[asd->capture_pad].fmt.height; @@ -445,29 +451,29 @@ static void print_csi_rx_errors(enum mipi_port_id port, atomisp_css_rx_get_irq_info(port, &infos); dev_err(isp->dev, "CSI Receiver port %d errors:\n", port); - if (infos & CSS_RX_IRQ_INFO_BUFFER_OVERRUN) + if (infos & IA_CSS_RX_IRQ_INFO_BUFFER_OVERRUN) dev_err(isp->dev, " buffer overrun"); - if (infos & CSS_RX_IRQ_INFO_ERR_SOT) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT) dev_err(isp->dev, " start-of-transmission error"); - if (infos & CSS_RX_IRQ_INFO_ERR_SOT_SYNC) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_SOT_SYNC) dev_err(isp->dev, " start-of-transmission sync error"); - if (infos & CSS_RX_IRQ_INFO_ERR_CONTROL) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_CONTROL) dev_err(isp->dev, " control error"); - if (infos & CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE) dev_err(isp->dev, " 2 or more ECC errors"); - if (infos & CSS_RX_IRQ_INFO_ERR_CRC) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_CRC) dev_err(isp->dev, " CRC mismatch"); - if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID) dev_err(isp->dev, " unknown error"); - if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_SYNC) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_SYNC) dev_err(isp->dev, " frame sync error"); - if (infos & CSS_RX_IRQ_INFO_ERR_FRAME_DATA) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_FRAME_DATA) dev_err(isp->dev, " frame data error"); - if (infos & CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT) dev_err(isp->dev, " data timeout"); - if (infos & CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC) dev_err(isp->dev, " unknown escape command entry"); - if (infos & CSS_RX_IRQ_INFO_ERR_LINE_SYNC) + if (infos & IA_CSS_RX_IRQ_INFO_ERR_LINE_SYNC) dev_err(isp->dev, " line sync error"); } @@ -526,8 +532,6 @@ irqreturn_t atomisp_isr(int irq, void *dev) return IRQ_NONE; } - dev_dbg(isp->dev, "irq:0x%x\n", irq_infos); - clear_irq_reg(isp); if (!atomisp_streaming_count(isp) && !atomisp_is_acc_enabled(isp)) @@ -542,7 +546,7 @@ irqreturn_t atomisp_isr(int irq, void *dev) * Current SOF only support one stream, so the SOF only valid * either solely one stream is running */ - if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF) { + if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) { atomic_inc(&asd->sof_count); atomisp_sof_event(asd); @@ -559,16 +563,20 @@ irqreturn_t atomisp_isr(int irq, void *dev) atomic_set(&asd->sequence_temp, atomic_read(&asd->sof_count)); } - if (irq_infos & CSS_IRQ_INFO_EVENTS_READY) + if (irq_infos & IA_CSS_IRQ_INFO_EVENTS_READY) atomic_set(&asd->sequence, atomic_read(&asd->sequence_temp)); } - if (irq_infos & CSS_IRQ_INFO_CSS_RECEIVER_SOF) - irq_infos &= ~CSS_IRQ_INFO_CSS_RECEIVER_SOF; + if (irq_infos & IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF) { + dev_dbg_ratelimited(isp->dev, + "irq:0x%x (SOF)\n", + irq_infos); + irq_infos &= ~IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF; + } - if ((irq_infos & CSS_IRQ_INFO_INPUT_SYSTEM_ERROR) || - (irq_infos & CSS_IRQ_INFO_IF_ERROR)) { + if ((irq_infos & IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR) || + (irq_infos & IA_CSS_IRQ_INFO_IF_ERROR)) { /* handle mipi receiver error */ u32 rx_infos; enum mipi_port_id port; @@ -583,18 +591,20 @@ irqreturn_t atomisp_isr(int irq, void *dev) if (irq_infos & IA_CSS_IRQ_INFO_ISYS_EVENTS_READY) { while (ia_css_dequeue_isys_event(&eof_event.event) == - IA_CSS_SUCCESS) { + 0) { /* EOF Event does not have the css_pipe returned */ asd = __get_asd_from_port(isp, eof_event.event.port); if (!asd) { - dev_err(isp->dev, "%s:no subdev.event:%d", __func__, - eof_event.event.type); + dev_err(isp->dev, "%s: ISYS event, but no subdev.event:%d", + __func__, eof_event.event.type); continue; } atomisp_eof_event(asd, eof_event.event.exp_id); - dev_dbg(isp->dev, "%s EOF exp_id %d, asd %d\n", - __func__, eof_event.event.exp_id, asd->index); + dev_dbg_ratelimited(isp->dev, + "%s ISYS event: EOF exp_id %d, asd %d\n", + __func__, eof_event.event.exp_id, + asd->index); } irq_infos &= ~IA_CSS_IRQ_INFO_ISYS_EVENTS_READY; @@ -604,11 +614,17 @@ irqreturn_t atomisp_isr(int irq, void *dev) spin_unlock_irqrestore(&isp->lock, flags); + dev_dbg_ratelimited(isp->dev, "irq:0x%x (unhandled)\n", irq_infos); + return IRQ_WAKE_THREAD; out_nowake: spin_unlock_irqrestore(&isp->lock, flags); + if (irq_infos) + dev_dbg_ratelimited(isp->dev, "irq:0x%x (ignored, as not streaming anymore)\n", + irq_infos); + return IRQ_HANDLED; } @@ -649,6 +665,7 @@ bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe) void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr, unsigned int size) { + u32 __iomem *io_virt_addr; unsigned int data = 0; unsigned int size32 = DIV_ROUND_UP(size, sizeof(u32)); @@ -661,20 +678,20 @@ void dump_sp_dmem(struct atomisp_device *isp, unsigned int addr, return; } addr += SP_DMEM_BASE; + io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); do { - data = _hrt_master_port_uload_32(addr); - + data = *io_virt_addr; dev_dbg(isp->dev, "%s, \t [0x%x]:0x%x\n", __func__, addr, data); - addr += sizeof(unsigned int); + io_virt_addr += sizeof(u32); size32 -= 1; } while (size32 > 0); } static struct videobuf_buffer *atomisp_css_frame_to_vbuf( - struct atomisp_video_pipe *pipe, struct atomisp_css_frame *frame) + struct atomisp_video_pipe *pipe, struct ia_css_frame *frame) { struct videobuf_vmalloc_memory *vm_mem; - struct atomisp_css_frame *handle; + struct ia_css_frame *handle; int i; for (i = 0; pipe->capq.bufs[i]; i++) { @@ -756,12 +773,12 @@ static void atomisp_recover_params_queue(struct atomisp_video_pipe *pipe) static struct atomisp_video_pipe *__atomisp_get_pipe( struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id, - enum atomisp_css_buffer_type buf_type) + enum ia_css_pipe_id css_pipe_id, + enum ia_css_buffer_type buf_type) { struct atomisp_device *isp = asd->isp; - if (css_pipe_id == CSS_PIPE_ID_COPY && + if (css_pipe_id == IA_CSS_PIPE_ID_COPY && isp->inputs[asd->input_curr].camera_caps-> sensor[asd->sensor_curr].stream_num > 1) { switch (stream_id) { @@ -792,7 +809,7 @@ static struct atomisp_video_pipe *__atomisp_get_pipe( * buffering. */ return &asd->video_out_video_capture; - } else if (css_pipe_id == CSS_PIPE_ID_YUVPP) { + } else if (css_pipe_id == IA_CSS_PIPE_ID_YUVPP) { /* * to SOC camera, yuvpp pipe is run for capture/video/SDV/ZSL. */ @@ -800,11 +817,11 @@ static struct atomisp_video_pipe *__atomisp_get_pipe( if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) { /* SDV case */ switch (buf_type) { - case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: return &asd->video_out_video_capture; - case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME: return &asd->video_out_preview; - case CSS_BUFFER_TYPE_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME: return &asd->video_out_capture; default: return &asd->video_out_vf; @@ -812,15 +829,15 @@ static struct atomisp_video_pipe *__atomisp_get_pipe( } else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) { /* ZSL case */ switch (buf_type) { - case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: return &asd->video_out_preview; - case CSS_BUFFER_TYPE_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME: return &asd->video_out_capture; default: return &asd->video_out_vf; } } - } else if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) { + } else if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) { switch (asd->run_mode->val) { case ATOMISP_RUN_MODE_VIDEO: return &asd->video_out_video_capture; @@ -829,7 +846,7 @@ static struct atomisp_video_pipe *__atomisp_get_pipe( default: return &asd->video_out_capture; } - } else if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { + } else if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) return &asd->video_out_preview; else @@ -837,20 +854,20 @@ static struct atomisp_video_pipe *__atomisp_get_pipe( } } else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) { /* For online video or SDV video pipe. */ - if (css_pipe_id == CSS_PIPE_ID_VIDEO || - css_pipe_id == CSS_PIPE_ID_COPY) { - if (buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) + if (css_pipe_id == IA_CSS_PIPE_ID_VIDEO || + css_pipe_id == IA_CSS_PIPE_ID_COPY) { + if (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) return &asd->video_out_video_capture; return &asd->video_out_preview; } } else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) { /* For online preview or ZSL preview pipe. */ - if (css_pipe_id == CSS_PIPE_ID_PREVIEW || - css_pipe_id == CSS_PIPE_ID_COPY) + if (css_pipe_id == IA_CSS_PIPE_ID_PREVIEW || + css_pipe_id == IA_CSS_PIPE_ID_COPY) return &asd->video_out_preview; } /* For capture pipe. */ - if (buf_type == CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) + if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) return &asd->video_out_vf; return &asd->video_out_capture; } @@ -868,8 +885,8 @@ atomisp_get_metadata_type(struct atomisp_sub_device *asd, } void atomisp_buf_done(struct atomisp_sub_device *asd, int error, - enum atomisp_css_buffer_type buf_type, - enum atomisp_css_pipe_id css_pipe_id, + enum ia_css_buffer_type buf_type, + enum ia_css_pipe_id css_pipe_id, bool q_buffers, enum atomisp_input_stream_id stream_id) { struct videobuf_buffer *vb = NULL; @@ -878,7 +895,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, bool requeue = false; int err; unsigned long irqflags; - struct atomisp_css_frame *frame = NULL; + struct ia_css_frame *frame = NULL; struct atomisp_s3a_buf *s3a_buf = NULL, *_s3a_buf_tmp; struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf_tmp; struct atomisp_metadata_buf *md_buf = NULL, *_md_buf_tmp; @@ -888,14 +905,14 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, bool reset_wdt_timer = false; if ( - buf_type != CSS_BUFFER_TYPE_METADATA && - buf_type != CSS_BUFFER_TYPE_3A_STATISTICS && - buf_type != CSS_BUFFER_TYPE_DIS_STATISTICS && - buf_type != CSS_BUFFER_TYPE_OUTPUT_FRAME && - buf_type != CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME && - buf_type != CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME && - buf_type != CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME && - buf_type != CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { + buf_type != IA_CSS_BUFFER_TYPE_METADATA && + buf_type != IA_CSS_BUFFER_TYPE_3A_STATISTICS && + buf_type != IA_CSS_BUFFER_TYPE_DIS_STATISTICS && + buf_type != IA_CSS_BUFFER_TYPE_OUTPUT_FRAME && + buf_type != IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME && + buf_type != IA_CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME && + buf_type != IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME && + buf_type != IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { dev_err(isp->dev, "%s, unsupported buffer type: %d\n", __func__, buf_type); return; @@ -919,7 +936,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, } switch (buf_type) { - case CSS_BUFFER_TYPE_3A_STATISTICS: + case IA_CSS_BUFFER_TYPE_3A_STATISTICS: list_for_each_entry_safe(s3a_buf, _s3a_buf_tmp, &asd->s3a_stats_in_css, list) { if (s3a_buf->s3a_data == @@ -936,7 +953,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, dev_dbg(isp->dev, "%s: s3a stat with exp_id %d is ready\n", __func__, s3a_buf->s3a_data->exp_id); break; - case CSS_BUFFER_TYPE_METADATA: + case IA_CSS_BUFFER_TYPE_METADATA: if (error) break; @@ -956,7 +973,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, dev_dbg(isp->dev, "%s: metadata with exp_id %d is ready\n", __func__, md_buf->metadata->exp_id); break; - case CSS_BUFFER_TYPE_DIS_STATISTICS: + case IA_CSS_BUFFER_TYPE_DIS_STATISTICS: list_for_each_entry_safe(dis_buf, _dis_buf_tmp, &asd->dis_stats_in_css, list) { if (dis_buf->dis_data == @@ -975,9 +992,9 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, dev_dbg(isp->dev, "%s: dis stat with exp_id %d is ready\n", __func__, dis_buf->dis_data->exp_id); break; - case CSS_BUFFER_TYPE_VF_OUTPUT_FRAME: - case CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME: - if (atomisp_hw_is_isp2401) + case IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME: + if (IS_ISP2401) reset_wdt_timer = true; pipe->buffers_in_css--; @@ -993,7 +1010,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, * YUVPP doesn't set postview exp_id correctlly in SDV mode. * This is a WORKAROUND to set exp_id. see HSDES-1503911606. */ - if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME && + if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME && asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd)) frame->exp_id = (asd->postview_exp_id++) % (ATOMISP_MAX_EXP_ID + 1); @@ -1002,11 +1019,11 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, __func__, frame->exp_id); if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) { if (frame->flash_state - == CSS_FRAME_FLASH_STATE_PARTIAL) + == IA_CSS_FRAME_FLASH_STATE_PARTIAL) dev_dbg(isp->dev, "%s thumb partially flashed\n", __func__); else if (frame->flash_state - == CSS_FRAME_FLASH_STATE_FULL) + == IA_CSS_FRAME_FLASH_STATE_FULL) dev_dbg(isp->dev, "%s thumb completely flashed\n", __func__); else @@ -1026,18 +1043,18 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, asd->pending_capture_request--; - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) asd->re_trigger_capture = false; dev_dbg(isp->dev, "Trigger capture again for new buffer. err=%d\n", err); - } else if (atomisp_hw_is_isp2401) { + } else if (IS_ISP2401) { asd->re_trigger_capture = true; } break; - case CSS_BUFFER_TYPE_OUTPUT_FRAME: - case CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: - if (atomisp_hw_is_isp2401) + case IA_CSS_BUFFER_TYPE_OUTPUT_FRAME: + case IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME: + if (IS_ISP2401) reset_wdt_timer = true; pipe->buffers_in_css--; @@ -1054,7 +1071,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, * YUVPP doesn't set preview exp_id correctlly in ZSL mode. * This is a WORKAROUND to set exp_id. see HSDES-1503911606. */ - if (IS_BYT && buf_type == CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME && + if (IS_BYT && buf_type == IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME && asd->continuous_mode->val && ATOMISP_USE_YUVPP(asd)) frame->exp_id = (asd->preview_exp_id++) % (ATOMISP_MAX_EXP_ID + 1); @@ -1082,13 +1099,13 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, ctrl.id = V4L2_CID_FLASH_MODE; if (asd->params.flash_state == ATOMISP_FLASH_ONGOING) { if (frame->flash_state - == CSS_FRAME_FLASH_STATE_PARTIAL) { + == IA_CSS_FRAME_FLASH_STATE_PARTIAL) { asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_FLASH_PARTIAL; dev_dbg(isp->dev, "%s partially flashed\n", __func__); } else if (frame->flash_state - == CSS_FRAME_FLASH_STATE_FULL) { + == IA_CSS_FRAME_FLASH_STATE_FULL) { asd->frame_status[vb->i] = ATOMISP_FRAME_STATUS_FLASH_EXPOSED; asd->params.num_flash_frames--; @@ -1128,11 +1145,11 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, asd->params.last_frame_status = asd->frame_status[vb->i]; if (asd->continuous_mode->val) { - if (css_pipe_id == CSS_PIPE_ID_PREVIEW || - css_pipe_id == CSS_PIPE_ID_VIDEO) { + if (css_pipe_id == IA_CSS_PIPE_ID_PREVIEW || + css_pipe_id == IA_CSS_PIPE_ID_VIDEO) { asd->latest_preview_exp_id = frame->exp_id; } else if (css_pipe_id == - CSS_PIPE_ID_CAPTURE) { + IA_CSS_PIPE_ID_CAPTURE) { if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) dev_dbg(isp->dev, "SDV capture raw buffer id: %u\n", @@ -1148,8 +1165,8 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, * in preview/video pipe, each buffer will * be locked automatically, so record it here. */ - if (((css_pipe_id == CSS_PIPE_ID_PREVIEW) || - (css_pipe_id == CSS_PIPE_ID_VIDEO)) && + if (((css_pipe_id == IA_CSS_PIPE_ID_PREVIEW) || + (css_pipe_id == IA_CSS_PIPE_ID_VIDEO)) && asd->enable_raw_buffer_lock->val && asd->continuous_mode->val) { atomisp_set_raw_buffer_bitmap(asd, frame->exp_id); @@ -1160,8 +1177,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, atomisp_apply_css_parameters(asd, &asd->params.css_param); if (asd->params.css_param.update_flag.dz_config) - atomisp_css_set_dz_config(asd, - &asd->params.css_param.dz_config); + asd->params.config.dz_config = &asd->params.css_param.dz_config; /* New global dvs 6axis config should be blocked * here if there's a buffer with per-frame parameters * pending in CSS frame buffer queue. @@ -1208,7 +1224,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, */ wake_up(&vb->done); } - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) atomic_set(&pipe->wdt_count, 0); /* @@ -1228,7 +1244,7 @@ void atomisp_buf_done(struct atomisp_sub_device *asd, int error, if (!error && q_buffers) atomisp_qbuffers_to_css(asd); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* If there are no buffers queued then * delete wdt timer. */ if (asd->streaming != ATOMISP_DEVICE_STREAMING_ENABLED) @@ -1252,9 +1268,15 @@ void atomisp_delayed_init_work(struct work_struct *work) */ if (!ATOMISP_USE_YUVPP(asd)) { struct v4l2_event event = {0}; + struct ia_css_stream *stream; + + stream = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream; + + + if (ia_css_alloc_continuous_frame_remain(stream)) + return; - atomisp_css_allocate_continuous_frames(false, asd); - atomisp_css_update_continuous_frames(asd); + ia_css_update_continuous_frames(stream); event.type = V4L2_EVENT_ATOMISP_RAW_BUFFERS_ALLOC_DONE; v4l2_event_queue(asd->subdev.devnode, &event); @@ -1267,14 +1289,14 @@ void atomisp_delayed_init_work(struct work_struct *work) static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout) { - enum atomisp_css_pipe_id css_pipe_id; + enum ia_css_pipe_id css_pipe_id; bool stream_restart[MAX_STREAM_NUM] = {0}; bool depth_mode = false; int i, ret, depth_cnt = 0; if (!isp->sw_contex.file_input) atomisp_css_irq_enable(isp, - CSS_IRQ_INFO_CSS_RECEIVER_SOF, false); + IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, false); BUG_ON(isp->num_of_streams > MAX_STREAM_NUM); @@ -1292,7 +1314,7 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout) * By calling acc_done() for all loaded fw_handles, * HAL will be unblocked. */ - acc_pipe = asd->stream_env[i].pipes[CSS_PIPE_ID_ACC]; + acc_pipe = asd->stream_env[i].pipes[IA_CSS_PIPE_ID_ACC]; if (acc_pipe) { acc_pipeline = ia_css_pipe_get_pipeline(acc_pipe); if (acc_pipeline) { @@ -1373,7 +1395,7 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout) if (isp->inputs[asd->input_curr].type != FILE_INPUT) atomisp_css_input_set_mode(asd, - CSS_INPUT_MODE_SENSOR); + IA_CSS_INPUT_MODE_BUFFERED_SENSOR); css_pipe_id = atomisp_get_css_pipe_id(asd); if (atomisp_css_start(asd, css_pipe_id, true)) @@ -1386,14 +1408,14 @@ static void __atomisp_css_recover(struct atomisp_device *isp, bool isp_timeout) } if (!isp->sw_contex.file_input) { - atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF, + atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, atomisp_css_valid_sof(isp)); if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, true) < 0) - dev_dbg(isp->dev, "dfs failed!\n"); + dev_dbg(isp->dev, "DFS auto failed while recovering!\n"); } else { if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, true) < 0) - dev_dbg(isp->dev, "dfs failed!\n"); + dev_dbg(isp->dev, "DFS max failed while recovering!\n"); } for (i = 0; i < isp->num_of_streams; i++) { @@ -1458,7 +1480,7 @@ void atomisp_wdt_work(struct work_struct *work) return; } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { dev_err(isp->dev, "timeout %d of %d\n", atomic_read(&isp->wdt_count) + 1, ATOMISP_ISP_MAX_TIMEOUT_COUNT); @@ -1494,11 +1516,8 @@ void atomisp_wdt_work(struct work_struct *work) } if (css_recover) { - unsigned int old_dbglevel = dbg_level; - - atomisp_css_debug_dump_sp_sw_debug_info(); - atomisp_css_debug_dump_debug_info(__func__); - dbg_level = old_dbglevel; + ia_css_debug_dump_sp_sw_debug_info(); + ia_css_debug_dump_debug_info(__func__); for (i = 0; i < isp->num_of_streams; i++) { struct atomisp_sub_device *asd = &isp->asd[i]; @@ -1530,15 +1549,15 @@ void atomisp_wdt_work(struct work_struct *work) dev_err(isp->dev, "%s, s3a buffers in css preview pipe:%d\n", __func__, - asd->s3a_bufs_in_css[CSS_PIPE_ID_PREVIEW]); + asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_PREVIEW]); dev_err(isp->dev, "%s, s3a buffers in css capture pipe:%d\n", __func__, - asd->s3a_bufs_in_css[CSS_PIPE_ID_CAPTURE]); + asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_CAPTURE]); dev_err(isp->dev, "%s, s3a buffers in css video pipe:%d\n", __func__, - asd->s3a_bufs_in_css[CSS_PIPE_ID_VIDEO]); + asd->s3a_bufs_in_css[IA_CSS_PIPE_ID_VIDEO]); dev_err(isp->dev, "%s, dis buffers in css: %d\n", __func__, asd->dis_bufs_in_css); @@ -1547,19 +1566,19 @@ void atomisp_wdt_work(struct work_struct *work) __func__, asd->metadata_bufs_in_css [ATOMISP_INPUT_STREAM_GENERAL] - [CSS_PIPE_ID_PREVIEW]); + [IA_CSS_PIPE_ID_PREVIEW]); dev_err(isp->dev, "%s, metadata buffers in css capture pipe:%d\n", __func__, asd->metadata_bufs_in_css [ATOMISP_INPUT_STREAM_GENERAL] - [CSS_PIPE_ID_CAPTURE]); + [IA_CSS_PIPE_ID_CAPTURE]); dev_err(isp->dev, "%s, metadata buffers in css video pipe:%d\n", __func__, asd->metadata_bufs_in_css [ATOMISP_INPUT_STREAM_GENERAL] - [CSS_PIPE_ID_VIDEO]); + [IA_CSS_PIPE_ID_VIDEO]); if (asd->enable_raw_buffer_lock->val) { unsigned int j; @@ -1584,11 +1603,11 @@ void atomisp_wdt_work(struct work_struct *work) atomisp_flush_bufs_and_wakeup(asd); complete(&asd->init_done); } - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) atomisp_wdt_stop(asd, false); } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { atomic_set(&isp->wdt_count, 0); } else { isp->isp_fatal_error = true; @@ -1600,7 +1619,7 @@ void atomisp_wdt_work(struct work_struct *work) } __atomisp_css_recover(isp, true); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { for (i = 0; i < isp->num_of_streams; i++) { struct atomisp_sub_device *asd = &isp->asd[i]; @@ -1657,7 +1676,7 @@ void atomisp_wdt(struct timer_list *t) struct atomisp_sub_device *asd; struct atomisp_device *isp; - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { asd = from_timer(asd, t, wdt); isp = asd->isp; } else { @@ -1716,7 +1735,7 @@ void atomisp_wdt_refresh_pipe(struct atomisp_video_pipe *pipe, void atomisp_wdt_refresh(struct atomisp_sub_device *asd, unsigned int delay) { - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { unsigned long next; if (delay != ATOMISP_WDT_KEEP_CURRENT_DELAY) @@ -1779,7 +1798,7 @@ void atomisp_wdt_stop(struct atomisp_sub_device *asd, bool sync) { dev_dbg(asd->isp->dev, "WDT stop:\n"); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (sync) { del_timer_sync(&asd->wdt); cancel_work_sync(&asd->isp->wdt_work); @@ -1816,7 +1835,8 @@ void atomisp_setup_flash(struct atomisp_sub_device *asd) return; } - atomisp_css_request_flash(asd); + ia_css_stream_request_flash(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream); + asd->params.flash_state = ATOMISP_FLASH_ONGOING; } else { asd->params.flash_state = ATOMISP_FLASH_IDLE; @@ -1901,7 +1921,7 @@ out: */ int atomisp_get_frame_pgnr(struct atomisp_device *isp, - const struct atomisp_css_frame *frame, u32 *p_pgnr) + const struct ia_css_frame *frame, u32 *p_pgnr) { if (!frame) { dev_err(isp->dev, "%s: NULL frame pointer ERROR.\n", __func__); @@ -1915,39 +1935,39 @@ int atomisp_get_frame_pgnr(struct atomisp_device *isp, /* * Get internal fmt according to V4L2 fmt */ -static enum atomisp_css_frame_format +static enum ia_css_frame_format v4l2_fmt_to_sh_fmt(u32 fmt) { switch (fmt) { case V4L2_PIX_FMT_YUV420: - return CSS_FRAME_FORMAT_YUV420; + return IA_CSS_FRAME_FORMAT_YUV420; case V4L2_PIX_FMT_YVU420: - return CSS_FRAME_FORMAT_YV12; + return IA_CSS_FRAME_FORMAT_YV12; case V4L2_PIX_FMT_YUV422P: - return CSS_FRAME_FORMAT_YUV422; + return IA_CSS_FRAME_FORMAT_YUV422; case V4L2_PIX_FMT_YUV444: - return CSS_FRAME_FORMAT_YUV444; + return IA_CSS_FRAME_FORMAT_YUV444; case V4L2_PIX_FMT_NV12: - return CSS_FRAME_FORMAT_NV12; + return IA_CSS_FRAME_FORMAT_NV12; case V4L2_PIX_FMT_NV21: - return CSS_FRAME_FORMAT_NV21; + return IA_CSS_FRAME_FORMAT_NV21; case V4L2_PIX_FMT_NV16: - return CSS_FRAME_FORMAT_NV16; + return IA_CSS_FRAME_FORMAT_NV16; case V4L2_PIX_FMT_NV61: - return CSS_FRAME_FORMAT_NV61; + return IA_CSS_FRAME_FORMAT_NV61; case V4L2_PIX_FMT_UYVY: - return CSS_FRAME_FORMAT_UYVY; + return IA_CSS_FRAME_FORMAT_UYVY; case V4L2_PIX_FMT_YUYV: - return CSS_FRAME_FORMAT_YUYV; + return IA_CSS_FRAME_FORMAT_YUYV; case V4L2_PIX_FMT_RGB24: - return CSS_FRAME_FORMAT_PLANAR_RGB888; + return IA_CSS_FRAME_FORMAT_PLANAR_RGB888; case V4L2_PIX_FMT_RGB32: - return CSS_FRAME_FORMAT_RGBA888; + return IA_CSS_FRAME_FORMAT_RGBA888; case V4L2_PIX_FMT_RGB565: - return CSS_FRAME_FORMAT_RGB565; + return IA_CSS_FRAME_FORMAT_RGB565; case V4L2_PIX_FMT_JPEG: case V4L2_PIX_FMT_CUSTOM_M10MO_RAW: - return CSS_FRAME_FORMAT_BINARY_8; + return IA_CSS_FRAME_FORMAT_BINARY_8; case V4L2_PIX_FMT_SBGGR16: case V4L2_PIX_FMT_SBGGR10: case V4L2_PIX_FMT_SGBRG10: @@ -1961,7 +1981,7 @@ v4l2_fmt_to_sh_fmt(u32 fmt) { case V4L2_PIX_FMT_SGBRG8: case V4L2_PIX_FMT_SGRBG8: case V4L2_PIX_FMT_SRGGB8: - return CSS_FRAME_FORMAT_RAW; + return IA_CSS_FRAME_FORMAT_RAW; default: return -EINVAL; } @@ -1972,28 +1992,28 @@ v4l2_fmt_to_sh_fmt(u32 fmt) { */ static int raw_output_format_match_input(u32 input, u32 output) { - if ((input == CSS_FORMAT_RAW_12) && + if ((input == ATOMISP_INPUT_FORMAT_RAW_12) && ((output == V4L2_PIX_FMT_SRGGB12) || (output == V4L2_PIX_FMT_SGRBG12) || (output == V4L2_PIX_FMT_SBGGR12) || (output == V4L2_PIX_FMT_SGBRG12))) return 0; - if ((input == CSS_FORMAT_RAW_10) && + if ((input == ATOMISP_INPUT_FORMAT_RAW_10) && ((output == V4L2_PIX_FMT_SRGGB10) || (output == V4L2_PIX_FMT_SGRBG10) || (output == V4L2_PIX_FMT_SBGGR10) || (output == V4L2_PIX_FMT_SGBRG10))) return 0; - if ((input == CSS_FORMAT_RAW_8) && + if ((input == ATOMISP_INPUT_FORMAT_RAW_8) && ((output == V4L2_PIX_FMT_SRGGB8) || (output == V4L2_PIX_FMT_SGRBG8) || (output == V4L2_PIX_FMT_SBGGR8) || (output == V4L2_PIX_FMT_SGBRG8))) return 0; - if ((input == CSS_FORMAT_RAW_16) && (output == V4L2_PIX_FMT_SBGGR16)) + if ((input == ATOMISP_INPUT_FORMAT_RAW_16) && (output == V4L2_PIX_FMT_SBGGR16)) return 0; return -EINVAL; @@ -2055,13 +2075,13 @@ bool atomisp_is_mbuscode_raw(uint32_t code) static void atomisp_update_capture_mode(struct atomisp_sub_device *asd) { if (asd->params.gdc_cac_en) - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_ADVANCED); + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_ADVANCED); else if (asd->params.low_light) - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_LOW_LIGHT); - else if (asd->video_out_capture.sh_fmt == CSS_FRAME_FORMAT_RAW) - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW); + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_LOW_LIGHT); + else if (asd->video_out_capture.sh_fmt == IA_CSS_FRAME_FORMAT_RAW) + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_RAW); else - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY); + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_PRIMARY); } /* ISP2401 */ @@ -2100,10 +2120,9 @@ int atomisp_gdc_cac(struct atomisp_sub_device *asd, int flag, asd->params.gdc_cac_en = !!*value; if (asd->params.gdc_cac_en) { - atomisp_css_set_morph_table(asd, - asd->params.css_param.morph_table); + asd->params.config.morph_table = asd->params.css_param.morph_table; } else { - atomisp_css_set_morph_table(asd, NULL); + asd->params.config.morph_table = NULL; } asd->params.css_update_params_needed = true; atomisp_update_capture_mode(asd); @@ -2156,8 +2175,8 @@ int atomisp_nr(struct atomisp_sub_device *asd, int flag, } else { /* Set nr config to isp parameters */ memcpy(&asd->params.css_param.nr_config, arg, - sizeof(struct atomisp_css_nr_config)); - atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config); + sizeof(struct ia_css_nr_config)); + asd->params.config.nr_config = &asd->params.css_param.nr_config; asd->params.css_update_params_needed = true; } return 0; @@ -2177,8 +2196,8 @@ int atomisp_tnr(struct atomisp_sub_device *asd, int flag, } else { /* Set tnr config to isp parameters */ memcpy(&asd->params.css_param.tnr_config, config, - sizeof(struct atomisp_css_tnr_config)); - atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config); + sizeof(struct ia_css_tnr_config)); + asd->params.config.tnr_config = &asd->params.css_param.tnr_config; asd->params.css_update_params_needed = true; } @@ -2198,8 +2217,8 @@ int atomisp_black_level(struct atomisp_sub_device *asd, int flag, } else { /* Set ob config to isp parameters */ memcpy(&asd->params.css_param.ob_config, config, - sizeof(struct atomisp_css_ob_config)); - atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config); + sizeof(struct ia_css_ob_config)); + asd->params.config.ob_config = &asd->params.css_param.ob_config; asd->params.css_update_params_needed = true; } @@ -2220,7 +2239,7 @@ int atomisp_ee(struct atomisp_sub_device *asd, int flag, /* Set ee config to isp parameters */ memcpy(&asd->params.css_param.ee_config, config, sizeof(asd->params.css_param.ee_config)); - atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config); + asd->params.config.ee_config = &asd->params.css_param.ee_config; asd->params.css_update_params_needed = true; } @@ -2241,7 +2260,7 @@ int atomisp_gamma(struct atomisp_sub_device *asd, int flag, /* Set gamma table to isp parameters */ memcpy(&asd->params.css_param.gamma_table, config, sizeof(asd->params.css_param.gamma_table)); - atomisp_css_set_gamma_table(asd, &asd->params.css_param.gamma_table); + asd->params.config.gamma_table = &asd->params.css_param.gamma_table; } return 0; @@ -2281,7 +2300,7 @@ int atomisp_gamma_correction(struct atomisp_sub_device *asd, int flag, /* Set gamma correction params to isp parameters */ memcpy(&asd->params.css_param.gc_config, config, sizeof(asd->params.css_param.gc_config)); - atomisp_css_set_gc_config(asd, &asd->params.css_param.gc_config); + asd->params.config.gc_config = &asd->params.css_param.gc_config; asd->params.css_update_params_needed = true; } @@ -2302,7 +2321,7 @@ int atomisp_formats(struct atomisp_sub_device *asd, int flag, /* Set narrow gamma flag to isp parameters */ memcpy(&asd->params.css_param.formats_config, config, sizeof(asd->params.css_param.formats_config)); - atomisp_css_set_formats_config(asd, &asd->params.css_param.formats_config); + asd->params.config.formats_config = &asd->params.css_param.formats_config; } return 0; @@ -2313,13 +2332,13 @@ void atomisp_free_internal_buffers(struct atomisp_sub_device *asd) atomisp_free_css_parameters(&asd->params.css_param); if (asd->raw_output_frame) { - atomisp_css_frame_free(asd->raw_output_frame); + ia_css_frame_free(asd->raw_output_frame); asd->raw_output_frame = NULL; } } static void atomisp_update_grid_info(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, + enum ia_css_pipe_id pipe_id, int source_pad) { struct atomisp_device *isp = asd->isp; @@ -2371,7 +2390,7 @@ static void atomisp_curr_user_grid_info(struct atomisp_sub_device *asd, struct atomisp_grid_info *info) { memcpy(info, &asd->params.curr_grid_info.s3a_grid, - sizeof(struct atomisp_css_3a_grid_info)); + sizeof(struct ia_css_3a_grid_info)); } int atomisp_compare_grid(struct atomisp_sub_device *asd, @@ -2395,14 +2414,14 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, if (flag == 0) { /* Get gdc table from current setup */ - struct atomisp_css_morph_table tab = {0}; + struct ia_css_morph_table tab = {0}; atomisp_css_get_morph_table(asd, &tab); config->width = tab.width; config->height = tab.height; - for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) { + for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { ret = copy_to_user(config->coordinates_x[i], tab.coordinates_x[i], tab.height * tab.width * sizeof(*tab.coordinates_x[i])); @@ -2421,7 +2440,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, } } } else { - struct atomisp_css_morph_table *tab = + struct ia_css_morph_table *tab = asd->params.css_param.morph_table; /* free first if we have one */ @@ -2439,7 +2458,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, return -EINVAL; } - for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) { + for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { ret = copy_from_user(tab->coordinates_x[i], config->coordinates_x[i], config->height * config->width * @@ -2465,7 +2484,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, } asd->params.css_param.morph_table = tab; if (asd->params.gdc_cac_en) - atomisp_css_set_morph_table(asd, tab); + asd->params.config.morph_table = tab; } return 0; @@ -2474,7 +2493,7 @@ int atomisp_gdc_cac_table(struct atomisp_sub_device *asd, int flag, int atomisp_macc_table(struct atomisp_sub_device *asd, int flag, struct atomisp_macc_config *config) { - struct atomisp_css_macc_table *macc_table; + struct ia_css_macc_table *macc_table; switch (config->color_effect) { case V4L2_COLORFX_NONE: @@ -2502,12 +2521,12 @@ int atomisp_macc_table(struct atomisp_sub_device *asd, int flag, if (flag == 0) { /* Get macc table from current setup */ memcpy(&config->table, macc_table, - sizeof(struct atomisp_css_macc_table)); + sizeof(struct ia_css_macc_table)); } else { memcpy(macc_table, &config->table, - sizeof(struct atomisp_css_macc_table)); + sizeof(struct ia_css_macc_table)); if (config->color_effect == asd->params.color_effect) - atomisp_css_set_macc_table(asd, macc_table); + asd->params.config.macc_table = macc_table; } return 0; @@ -2567,7 +2586,7 @@ int atomisp_get_dvs2_bq_resolutions(struct atomisp_sub_device *asd, } pipe_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] - .pipe_configs[CSS_PIPE_ID_VIDEO]; + .pipe_configs[IA_CSS_PIPE_ID_VIDEO]; stream_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] .stream_config; input_config = &stream_cfg->input_config; @@ -2951,7 +2970,7 @@ int atomisp_get_metadata_by_type(struct atomisp_sub_device *asd, int flag, */ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd, struct ia_css_dz_config *dz_config, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_stream_env *stream_env = @@ -3000,7 +3019,7 @@ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd, * map real crop region base on above calculating base max crop region. */ - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { dz_config->zoom_region.origin.x = dz_config->zoom_region.origin.x * eff_res.width / asd->sensor_array_res.width; @@ -3144,87 +3163,85 @@ void atomisp_apply_css_parameters( struct atomisp_css_params *css_param) { if (css_param->update_flag.wb_config) - atomisp_css_set_wb_config(asd, &css_param->wb_config); + asd->params.config.wb_config = &css_param->wb_config; if (css_param->update_flag.ob_config) - atomisp_css_set_ob_config(asd, &css_param->ob_config); + asd->params.config.ob_config = &css_param->ob_config; if (css_param->update_flag.dp_config) - atomisp_css_set_dp_config(asd, &css_param->dp_config); + asd->params.config.dp_config = &css_param->dp_config; if (css_param->update_flag.nr_config) - atomisp_css_set_nr_config(asd, &css_param->nr_config); + asd->params.config.nr_config = &css_param->nr_config; if (css_param->update_flag.ee_config) - atomisp_css_set_ee_config(asd, &css_param->ee_config); + asd->params.config.ee_config = &css_param->ee_config; if (css_param->update_flag.tnr_config) - atomisp_css_set_tnr_config(asd, &css_param->tnr_config); + asd->params.config.tnr_config = &css_param->tnr_config; if (css_param->update_flag.a3a_config) - atomisp_css_set_3a_config(asd, &css_param->s3a_config); + asd->params.config.s3a_config = &css_param->s3a_config; if (css_param->update_flag.ctc_config) - atomisp_css_set_ctc_config(asd, &css_param->ctc_config); + asd->params.config.ctc_config = &css_param->ctc_config; if (css_param->update_flag.cnr_config) - atomisp_css_set_cnr_config(asd, &css_param->cnr_config); + asd->params.config.cnr_config = &css_param->cnr_config; if (css_param->update_flag.ecd_config) - atomisp_css_set_ecd_config(asd, &css_param->ecd_config); + asd->params.config.ecd_config = &css_param->ecd_config; if (css_param->update_flag.ynr_config) - atomisp_css_set_ynr_config(asd, &css_param->ynr_config); + asd->params.config.ynr_config = &css_param->ynr_config; if (css_param->update_flag.fc_config) - atomisp_css_set_fc_config(asd, &css_param->fc_config); + asd->params.config.fc_config = &css_param->fc_config; if (css_param->update_flag.macc_config) - atomisp_css_set_macc_config(asd, &css_param->macc_config); + asd->params.config.macc_config = &css_param->macc_config; if (css_param->update_flag.aa_config) - atomisp_css_set_aa_config(asd, &css_param->aa_config); + asd->params.config.aa_config = &css_param->aa_config; if (css_param->update_flag.anr_config) - atomisp_css_set_anr_config(asd, &css_param->anr_config); + asd->params.config.anr_config = &css_param->anr_config; if (css_param->update_flag.xnr_config) - atomisp_css_set_xnr_config(asd, &css_param->xnr_config); + asd->params.config.xnr_config = &css_param->xnr_config; if (css_param->update_flag.yuv2rgb_cc_config) - atomisp_css_set_yuv2rgb_cc_config(asd, - &css_param->yuv2rgb_cc_config); + asd->params.config.yuv2rgb_cc_config = &css_param->yuv2rgb_cc_config; if (css_param->update_flag.rgb2yuv_cc_config) - atomisp_css_set_rgb2yuv_cc_config(asd, - &css_param->rgb2yuv_cc_config); + asd->params.config.rgb2yuv_cc_config = &css_param->rgb2yuv_cc_config; if (css_param->update_flag.macc_table) - atomisp_css_set_macc_table(asd, &css_param->macc_table); + asd->params.config.macc_table = &css_param->macc_table; if (css_param->update_flag.xnr_table) - atomisp_css_set_xnr_table(asd, &css_param->xnr_table); + asd->params.config.xnr_table = &css_param->xnr_table; if (css_param->update_flag.r_gamma_table) - atomisp_css_set_r_gamma_table(asd, &css_param->r_gamma_table); + asd->params.config.r_gamma_table = &css_param->r_gamma_table; if (css_param->update_flag.g_gamma_table) - atomisp_css_set_g_gamma_table(asd, &css_param->g_gamma_table); + asd->params.config.g_gamma_table = &css_param->g_gamma_table; if (css_param->update_flag.b_gamma_table) - atomisp_css_set_b_gamma_table(asd, &css_param->b_gamma_table); + asd->params.config.b_gamma_table = &css_param->b_gamma_table; if (css_param->update_flag.anr_thres) atomisp_css_set_anr_thres(asd, &css_param->anr_thres); if (css_param->update_flag.shading_table) - atomisp_css_set_shading_table(asd, css_param->shading_table); + asd->params.config.shading_table = css_param->shading_table; if (css_param->update_flag.morph_table && asd->params.gdc_cac_en) - atomisp_css_set_morph_table(asd, css_param->morph_table); + asd->params.config.morph_table = css_param->morph_table; if (css_param->update_flag.dvs2_coefs) { - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info( &asd->params.curr_grid_info); @@ -3271,7 +3288,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->wb_config && (from_user || !cur_config->wb_config)) { if (copy_from_compatible(&css_param->wb_config, arg->wb_config, - sizeof(struct atomisp_css_wb_config), + sizeof(struct ia_css_wb_config), from_user)) return -EFAULT; css_param->update_flag.wb_config = @@ -3280,7 +3297,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->ob_config && (from_user || !cur_config->ob_config)) { if (copy_from_compatible(&css_param->ob_config, arg->ob_config, - sizeof(struct atomisp_css_ob_config), + sizeof(struct ia_css_ob_config), from_user)) return -EFAULT; css_param->update_flag.ob_config = @@ -3289,7 +3306,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->dp_config && (from_user || !cur_config->dp_config)) { if (copy_from_compatible(&css_param->dp_config, arg->dp_config, - sizeof(struct atomisp_css_dp_config), + sizeof(struct ia_css_dp_config), from_user)) return -EFAULT; css_param->update_flag.dp_config = @@ -3300,7 +3317,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->dz_config && (from_user || !cur_config->dz_config)) { if (copy_from_compatible(&css_param->dz_config, arg->dz_config, - sizeof(struct atomisp_css_dz_config), + sizeof(struct ia_css_dz_config), from_user)) return -EFAULT; if (!atomisp_check_zoom_region(asd, @@ -3316,7 +3333,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->nr_config && (from_user || !cur_config->nr_config)) { if (copy_from_compatible(&css_param->nr_config, arg->nr_config, - sizeof(struct atomisp_css_nr_config), + sizeof(struct ia_css_nr_config), from_user)) return -EFAULT; css_param->update_flag.nr_config = @@ -3325,7 +3342,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->ee_config && (from_user || !cur_config->ee_config)) { if (copy_from_compatible(&css_param->ee_config, arg->ee_config, - sizeof(struct atomisp_css_ee_config), + sizeof(struct ia_css_ee_config), from_user)) return -EFAULT; css_param->update_flag.ee_config = @@ -3335,7 +3352,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->tnr_config && (from_user || !cur_config->tnr_config)) { if (copy_from_compatible(&css_param->tnr_config, arg->tnr_config, - sizeof(struct atomisp_css_tnr_config), + sizeof(struct ia_css_tnr_config), from_user)) return -EFAULT; css_param->update_flag.tnr_config = @@ -3346,7 +3363,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->a3a_config && (from_user || !cur_config->a3a_config)) { if (copy_from_compatible(&css_param->s3a_config, arg->a3a_config, - sizeof(struct atomisp_css_3a_config), + sizeof(struct ia_css_3a_config), from_user)) return -EFAULT; css_param->update_flag.a3a_config = @@ -3356,7 +3373,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->ctc_config && (from_user || !cur_config->ctc_config)) { if (copy_from_compatible(&css_param->ctc_config, arg->ctc_config, - sizeof(struct atomisp_css_ctc_config), + sizeof(struct ia_css_ctc_config), from_user)) return -EFAULT; css_param->update_flag.ctc_config = @@ -3367,7 +3384,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->cnr_config && (from_user || !cur_config->cnr_config)) { if (copy_from_compatible(&css_param->cnr_config, arg->cnr_config, - sizeof(struct atomisp_css_cnr_config), + sizeof(struct ia_css_cnr_config), from_user)) return -EFAULT; css_param->update_flag.cnr_config = @@ -3378,7 +3395,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->ecd_config && (from_user || !cur_config->ecd_config)) { if (copy_from_compatible(&css_param->ecd_config, arg->ecd_config, - sizeof(struct atomisp_css_ecd_config), + sizeof(struct ia_css_ecd_config), from_user)) return -EFAULT; css_param->update_flag.ecd_config = @@ -3389,7 +3406,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->ynr_config && (from_user || !cur_config->ynr_config)) { if (copy_from_compatible(&css_param->ynr_config, arg->ynr_config, - sizeof(struct atomisp_css_ynr_config), + sizeof(struct ia_css_ynr_config), from_user)) return -EFAULT; css_param->update_flag.ynr_config = @@ -3400,7 +3417,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->fc_config && (from_user || !cur_config->fc_config)) { if (copy_from_compatible(&css_param->fc_config, arg->fc_config, - sizeof(struct atomisp_css_fc_config), + sizeof(struct ia_css_fc_config), from_user)) return -EFAULT; css_param->update_flag.fc_config = @@ -3410,7 +3427,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->macc_config && (from_user || !cur_config->macc_config)) { if (copy_from_compatible(&css_param->macc_config, arg->macc_config, - sizeof(struct atomisp_css_macc_config), + sizeof(struct ia_css_macc_config), from_user)) return -EFAULT; css_param->update_flag.macc_config = @@ -3420,7 +3437,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->aa_config && (from_user || !cur_config->aa_config)) { if (copy_from_compatible(&css_param->aa_config, arg->aa_config, - sizeof(struct atomisp_css_aa_config), + sizeof(struct ia_css_aa_config), from_user)) return -EFAULT; css_param->update_flag.aa_config = @@ -3430,7 +3447,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->anr_config && (from_user || !cur_config->anr_config)) { if (copy_from_compatible(&css_param->anr_config, arg->anr_config, - sizeof(struct atomisp_css_anr_config), + sizeof(struct ia_css_anr_config), from_user)) return -EFAULT; css_param->update_flag.anr_config = @@ -3441,7 +3458,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->xnr_config && (from_user || !cur_config->xnr_config)) { if (copy_from_compatible(&css_param->xnr_config, arg->xnr_config, - sizeof(struct atomisp_css_xnr_config), + sizeof(struct ia_css_xnr_config), from_user)) return -EFAULT; css_param->update_flag.xnr_config = @@ -3453,7 +3470,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, (from_user || !cur_config->yuv2rgb_cc_config)) { if (copy_from_compatible(&css_param->yuv2rgb_cc_config, arg->yuv2rgb_cc_config, - sizeof(struct atomisp_css_cc_config), + sizeof(struct ia_css_cc_config), from_user)) return -EFAULT; css_param->update_flag.yuv2rgb_cc_config = @@ -3465,7 +3482,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, (from_user || !cur_config->rgb2yuv_cc_config)) { if (copy_from_compatible(&css_param->rgb2yuv_cc_config, arg->rgb2yuv_cc_config, - sizeof(struct atomisp_css_cc_config), + sizeof(struct ia_css_cc_config), from_user)) return -EFAULT; css_param->update_flag.rgb2yuv_cc_config = @@ -3476,7 +3493,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->macc_table && (from_user || !cur_config->macc_table)) { if (copy_from_compatible(&css_param->macc_table, arg->macc_table, - sizeof(struct atomisp_css_macc_table), + sizeof(struct ia_css_macc_table), from_user)) return -EFAULT; css_param->update_flag.macc_table = @@ -3487,7 +3504,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->xnr_table && (from_user || !cur_config->xnr_table)) { if (copy_from_compatible(&css_param->xnr_table, arg->xnr_table, - sizeof(struct atomisp_css_xnr_table), + sizeof(struct ia_css_xnr_table), from_user)) return -EFAULT; css_param->update_flag.xnr_table = @@ -3497,7 +3514,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->r_gamma_table && (from_user || !cur_config->r_gamma_table)) { if (copy_from_compatible(&css_param->r_gamma_table, arg->r_gamma_table, - sizeof(struct atomisp_css_rgb_gamma_table), + sizeof(struct ia_css_rgb_gamma_table), from_user)) return -EFAULT; css_param->update_flag.r_gamma_table = @@ -3508,7 +3525,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->g_gamma_table && (from_user || !cur_config->g_gamma_table)) { if (copy_from_compatible(&css_param->g_gamma_table, arg->g_gamma_table, - sizeof(struct atomisp_css_rgb_gamma_table), + sizeof(struct ia_css_rgb_gamma_table), from_user)) return -EFAULT; css_param->update_flag.g_gamma_table = @@ -3519,7 +3536,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->b_gamma_table && (from_user || !cur_config->b_gamma_table)) { if (copy_from_compatible(&css_param->b_gamma_table, arg->b_gamma_table, - sizeof(struct atomisp_css_rgb_gamma_table), + sizeof(struct ia_css_rgb_gamma_table), from_user)) return -EFAULT; css_param->update_flag.b_gamma_table = @@ -3529,7 +3546,7 @@ int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, if (arg->anr_thres && (from_user || !cur_config->anr_thres)) { if (copy_from_compatible(&css_param->anr_thres, arg->anr_thres, - sizeof(struct atomisp_css_anr_thres), + sizeof(struct ia_css_anr_thres), from_user)) return -EFAULT; css_param->update_flag.anr_thres = @@ -3559,8 +3576,8 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd, { unsigned int i; unsigned int len_table; - struct atomisp_css_shading_table *shading_table; - struct atomisp_css_shading_table *old_table; + struct ia_css_shading_table *shading_table; + struct ia_css_shading_table *old_table; struct atomisp_shading_table *st, dest_st; if (!source_st) @@ -3572,7 +3589,7 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd, if (!from_user && css_param->update_flag.shading_table) return 0; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { if (copy_from_compatible(&dest_st, source_st, sizeof(struct atomisp_shading_table), from_user)) { @@ -3605,7 +3622,7 @@ int atomisp_cp_lsc_table(struct atomisp_sub_device *asd, } /* Shading table size per color */ - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (st->width > ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR || st->height > ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR) { dev_err(asd->isp->dev, "shading table w/h validate failed!"); @@ -3677,7 +3694,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd, struct atomisp_css_params *css_param, bool from_user) { - struct atomisp_css_dvs_grid_info *cur = + struct ia_css_dvs_grid_info *cur = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); int dvs_hor_coef_bytes, dvs_ver_coef_bytes; struct ia_css_dvs2_coefficients dvs2_coefs; @@ -3688,7 +3705,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd, if (!from_user && css_param->update_flag.dvs2_coefs) return 0; - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (sizeof(*cur) != sizeof(coefs->grid) || memcmp(&coefs->grid, cur, sizeof(coefs->grid))) { dev_err(asd->isp->dev, "dvs grid mis-match!\n"); @@ -3796,7 +3813,7 @@ int atomisp_css_cp_dvs2_coefs(struct atomisp_sub_device *asd, } css_param->update_flag.dvs2_coefs = - (struct atomisp_dvs2_coefficients *)css_param->dvs2_coeff; + (struct atomisp_dis_coefficients *)css_param->dvs2_coeff; return 0; } @@ -3805,11 +3822,11 @@ int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd, struct atomisp_css_params *css_param, bool from_user) { - struct atomisp_css_dvs_6axis_config *dvs_6axis_config; - struct atomisp_css_dvs_6axis_config *old_6axis_config; + struct ia_css_dvs_6axis_config *dvs_6axis_config; + struct ia_css_dvs_6axis_config *old_6axis_config; struct ia_css_stream *stream = asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream; - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); int ret = -EFAULT; @@ -3831,8 +3848,8 @@ int atomisp_cp_dvs_6axis_config(struct atomisp_sub_device *asd, old_6axis_config = css_param->dvs_6axis; dvs_6axis_config = old_6axis_config; - if (atomisp_hw_is_isp2401) { - struct atomisp_css_dvs_6axis_config t_6axis_config; + if (IS_ISP2401) { + struct ia_css_dvs_6axis_config t_6axis_config; if (copy_from_compatible(&t_6axis_config, source_6axis_config, sizeof(struct atomisp_dvs_6axis_config), @@ -3955,8 +3972,8 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd, { int ret = -EFAULT; unsigned int i; - struct atomisp_css_morph_table *morph_table; - struct atomisp_css_morph_table *old_morph_table; + struct ia_css_morph_table *morph_table; + struct ia_css_morph_table *old_morph_table; if (!source_morph_table) return 0; @@ -3966,8 +3983,8 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd, old_morph_table = css_param->morph_table; - if (atomisp_hw_is_isp2401) { - struct atomisp_css_morph_table mtbl; + if (IS_ISP2401) { + struct ia_css_morph_table mtbl; if (copy_from_compatible(&mtbl, source_morph_table, sizeof(struct atomisp_morph_table), @@ -3982,7 +3999,7 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd, if (!morph_table) return -ENOMEM; - for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) { + for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { if (copy_from_compatible(morph_table->coordinates_x[i], (__force void *)source_morph_table->coordinates_x[i], mtbl.height * mtbl.width * @@ -4004,7 +4021,7 @@ int atomisp_cp_morph_table(struct atomisp_sub_device *asd, if (!morph_table) return -ENOMEM; - for (i = 0; i < CSS_MORPH_TABLE_NUM_PLANES; i++) { + for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { if (copy_from_compatible(morph_table->coordinates_x[i], (__force void *)source_morph_table->coordinates_x[i], source_morph_table->height * source_morph_table->width * @@ -4156,7 +4173,7 @@ void atomisp_handle_parameter_and_buffer(struct atomisp_video_pipe *pipe) atomisp_qbuffers_to_css(asd); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd)) atomisp_wdt_start(asd); } else { @@ -4192,7 +4209,7 @@ int atomisp_set_parameters(struct video_device *vdev, __func__, arg->per_frame_setting, asd->index, arg->isp_config_id, vdev->name); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { if (atomisp_is_vf_pipe(pipe) && arg->per_frame_setting) { dev_err(asd->isp->dev, "%s: vf pipe not support per_frame_setting", __func__); @@ -4264,21 +4281,16 @@ apply_parameter_failed: int atomisp_param(struct atomisp_sub_device *asd, int flag, struct atomisp_parm *config) { - struct atomisp_device *isp = asd->isp; struct ia_css_pipe_config *vp_cfg = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]. pipe_configs[IA_CSS_PIPE_ID_VIDEO]; /* Read parameter for 3A binary info */ if (flag == 0) { - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info( &asd->params.curr_grid_info); - if (!&config->info) { - dev_err(isp->dev, "ERROR: NULL pointer in grid_info\n"); - return -EINVAL; - } atomisp_curr_user_grid_info(asd, &config->info); /* We always return the resolution and stride even if there is @@ -4295,7 +4307,7 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag, if (dvs_grid_info) memcpy(&config->dvs_grid, dvs_grid_info, - sizeof(struct atomisp_css_dvs_grid_info)); + sizeof(struct ia_css_dvs_grid_info)); if (asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO) { config->dvs_envelop.width = 0; @@ -4330,23 +4342,23 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag, } memcpy(&asd->params.css_param.wb_config, &config->wb_config, - sizeof(struct atomisp_css_wb_config)); + sizeof(struct ia_css_wb_config)); memcpy(&asd->params.css_param.ob_config, &config->ob_config, - sizeof(struct atomisp_css_ob_config)); + sizeof(struct ia_css_ob_config)); memcpy(&asd->params.css_param.dp_config, &config->dp_config, - sizeof(struct atomisp_css_dp_config)); + sizeof(struct ia_css_dp_config)); memcpy(&asd->params.css_param.de_config, &config->de_config, - sizeof(struct atomisp_css_de_config)); + sizeof(struct ia_css_de_config)); memcpy(&asd->params.css_param.dz_config, &config->dz_config, - sizeof(struct atomisp_css_dz_config)); + sizeof(struct ia_css_dz_config)); memcpy(&asd->params.css_param.ce_config, &config->ce_config, - sizeof(struct atomisp_css_ce_config)); + sizeof(struct ia_css_ce_config)); memcpy(&asd->params.css_param.nr_config, &config->nr_config, - sizeof(struct atomisp_css_nr_config)); + sizeof(struct ia_css_nr_config)); memcpy(&asd->params.css_param.ee_config, &config->ee_config, - sizeof(struct atomisp_css_ee_config)); + sizeof(struct ia_css_ee_config)); memcpy(&asd->params.css_param.tnr_config, &config->tnr_config, - sizeof(struct atomisp_css_tnr_config)); + sizeof(struct ia_css_tnr_config)); if (asd->params.color_effect == V4L2_COLORFX_NEGATIVE) { asd->params.css_param.cc_config.matrix[3] = -config->cc_config.matrix[3]; @@ -4360,19 +4372,19 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag, if (asd->params.color_effect != V4L2_COLORFX_SEPIA && asd->params.color_effect != V4L2_COLORFX_BW) { memcpy(&asd->params.css_param.cc_config, &config->cc_config, - sizeof(struct atomisp_css_cc_config)); - atomisp_css_set_cc_config(asd, &asd->params.css_param.cc_config); - } - - atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config); - atomisp_css_set_ob_config(asd, &asd->params.css_param.ob_config); - atomisp_css_set_de_config(asd, &asd->params.css_param.de_config); - atomisp_css_set_dz_config(asd, &asd->params.css_param.dz_config); - atomisp_css_set_ce_config(asd, &asd->params.css_param.ce_config); - atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config); - atomisp_css_set_nr_config(asd, &asd->params.css_param.nr_config); - atomisp_css_set_ee_config(asd, &asd->params.css_param.ee_config); - atomisp_css_set_tnr_config(asd, &asd->params.css_param.tnr_config); + sizeof(struct ia_css_cc_config)); + asd->params.config.cc_config = &asd->params.css_param.cc_config; + } + + asd->params.config.wb_config = &asd->params.css_param.wb_config; + asd->params.config.ob_config = &asd->params.css_param.ob_config; + asd->params.config.de_config = &asd->params.css_param.de_config; + asd->params.config.dz_config = &asd->params.css_param.dz_config; + asd->params.config.ce_config = &asd->params.css_param.ce_config; + asd->params.config.dp_config = &asd->params.css_param.dp_config; + asd->params.config.nr_config = &asd->params.css_param.nr_config; + asd->params.config.ee_config = &asd->params.css_param.ee_config; + asd->params.config.tnr_config = &asd->params.css_param.tnr_config; asd->params.css_update_params_needed = true; return 0; @@ -4384,9 +4396,9 @@ int atomisp_param(struct atomisp_sub_device *asd, int flag, int atomisp_color_effect(struct atomisp_sub_device *asd, int flag, __s32 *effect) { - struct atomisp_css_cc_config *cc_config = NULL; - struct atomisp_css_macc_table *macc_table = NULL; - struct atomisp_css_ctc_table *ctc_table = NULL; + struct ia_css_cc_config *cc_config = NULL; + struct ia_css_macc_table *macc_table = NULL; + struct ia_css_ctc_table *ctc_table = NULL; int ret = 0; struct v4l2_control control; struct atomisp_device *isp = asd->isp; @@ -4461,9 +4473,9 @@ int atomisp_color_effect(struct atomisp_sub_device *asd, int flag, atomisp_update_capture_mode(asd); if (cc_config) - atomisp_css_set_cc_config(asd, cc_config); + asd->params.config.cc_config = cc_config; if (macc_table) - atomisp_css_set_macc_table(asd, macc_table); + asd->params.config.macc_table = macc_table; if (ctc_table) atomisp_css_set_ctc_table(asd, ctc_table); asd->params.color_effect = (u32)*effect; @@ -4500,7 +4512,7 @@ int atomisp_bad_pixel_param(struct atomisp_sub_device *asd, int flag, /* Set bad pixel to isp parameters */ memcpy(&asd->params.css_param.dp_config, config, sizeof(asd->params.css_param.dp_config)); - atomisp_css_set_dp_config(asd, &asd->params.css_param.dp_config); + asd->params.config.dp_config = &asd->params.css_param.dp_config; asd->params.css_update_params_needed = true; } @@ -4543,34 +4555,34 @@ int atomisp_fixed_pattern(struct atomisp_sub_device *asd, int flag, static unsigned int atomisp_bytesperline_to_padded_width(unsigned int bytesperline, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { switch (format) { - case CSS_FRAME_FORMAT_UYVY: - case CSS_FRAME_FORMAT_YUYV: - case CSS_FRAME_FORMAT_RAW: - case CSS_FRAME_FORMAT_RGB565: + case IA_CSS_FRAME_FORMAT_UYVY: + case IA_CSS_FRAME_FORMAT_YUYV: + case IA_CSS_FRAME_FORMAT_RAW: + case IA_CSS_FRAME_FORMAT_RGB565: return bytesperline / 2; - case CSS_FRAME_FORMAT_RGBA888: + case IA_CSS_FRAME_FORMAT_RGBA888: return bytesperline / 4; /* The following cases could be removed, but we leave them in to document the formats that are included. */ - case CSS_FRAME_FORMAT_NV11: - case CSS_FRAME_FORMAT_NV12: - case CSS_FRAME_FORMAT_NV16: - case CSS_FRAME_FORMAT_NV21: - case CSS_FRAME_FORMAT_NV61: - case CSS_FRAME_FORMAT_YV12: - case CSS_FRAME_FORMAT_YV16: - case CSS_FRAME_FORMAT_YUV420: - case CSS_FRAME_FORMAT_YUV420_16: - case CSS_FRAME_FORMAT_YUV422: - case CSS_FRAME_FORMAT_YUV422_16: - case CSS_FRAME_FORMAT_YUV444: - case CSS_FRAME_FORMAT_YUV_LINE: - case CSS_FRAME_FORMAT_PLANAR_RGB888: - case CSS_FRAME_FORMAT_QPLANE6: - case CSS_FRAME_FORMAT_BINARY_8: + case IA_CSS_FRAME_FORMAT_NV11: + case IA_CSS_FRAME_FORMAT_NV12: + case IA_CSS_FRAME_FORMAT_NV16: + case IA_CSS_FRAME_FORMAT_NV21: + case IA_CSS_FRAME_FORMAT_NV61: + case IA_CSS_FRAME_FORMAT_YV12: + case IA_CSS_FRAME_FORMAT_YV16: + case IA_CSS_FRAME_FORMAT_YUV420: + case IA_CSS_FRAME_FORMAT_YUV420_16: + case IA_CSS_FRAME_FORMAT_YUV422: + case IA_CSS_FRAME_FORMAT_YUV422_16: + case IA_CSS_FRAME_FORMAT_YUV444: + case IA_CSS_FRAME_FORMAT_YUV_LINE: + case IA_CSS_FRAME_FORMAT_PLANAR_RGB888: + case IA_CSS_FRAME_FORMAT_QPLANE6: + case IA_CSS_FRAME_FORMAT_BINARY_8: default: return bytesperline; } @@ -4578,11 +4590,11 @@ atomisp_bytesperline_to_padded_width(unsigned int bytesperline, static int atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg, - struct atomisp_css_frame **result) + struct ia_css_frame **result) { - struct atomisp_css_frame *res = NULL; + struct ia_css_frame *res = NULL; unsigned int padded_width; - enum atomisp_css_frame_format sh_format; + enum ia_css_frame_format sh_format; char *tmp_buf = NULL; int ret = 0; @@ -4590,10 +4602,10 @@ atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg, padded_width = atomisp_bytesperline_to_padded_width( arg->fmt.bytesperline, sh_format); - /* Note: the padded width on an atomisp_css_frame is in elements, not in + /* Note: the padded width on an ia_css_frame is in elements, not in bytes. The RAW frame we use here should always be a 16bit RAW frame. This is why we bytesperline/2 is equal to the padded with */ - if (atomisp_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height, + if (ia_css_frame_allocate(&res, arg->fmt.width, arg->fmt.height, sh_format, padded_width, 0)) { ret = -ENOMEM; goto err; @@ -4617,7 +4629,7 @@ atomisp_v4l2_framebuffer_to_css_frame(const struct v4l2_framebuffer *arg, err: if (ret && res) - atomisp_css_frame_free(res); + ia_css_frame_free(res); if (tmp_buf) vfree(tmp_buf); if (ret == 0) @@ -4631,7 +4643,7 @@ err: int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd, struct v4l2_framebuffer *arg) { - struct atomisp_css_frame *raw_black_frame = NULL; + struct ia_css_frame *raw_black_frame = NULL; int ret; if (!arg) @@ -4640,10 +4652,12 @@ int atomisp_fixed_pattern_table(struct atomisp_sub_device *asd, ret = atomisp_v4l2_framebuffer_to_css_frame(arg, &raw_black_frame); if (ret) return ret; - if (atomisp_css_set_black_frame(asd, raw_black_frame)) - ret = -ENOMEM; - atomisp_css_frame_free(raw_black_frame); + if (sh_css_set_black_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, + raw_black_frame) != 0) + return -ENOMEM; + + ia_css_frame_free(raw_black_frame); return ret; } @@ -4661,10 +4675,10 @@ int atomisp_false_color(struct atomisp_sub_device *asd, int flag, /* Set nr config to isp parameters */ if (*value) { - atomisp_css_set_default_de_config(asd); + asd->params.config.de_config = NULL; } else { asd->params.css_param.de_config.pixelnoise = 0; - atomisp_css_set_de_config(asd, &asd->params.css_param.de_config); + asd->params.config.de_config = &asd->params.css_param.de_config; } asd->params.css_update_params_needed = true; asd->params.false_color = *value; @@ -4685,7 +4699,7 @@ int atomisp_false_color_param(struct atomisp_sub_device *asd, int flag, /* Set false color to isp parameters */ memcpy(&asd->params.css_param.de_config, config, sizeof(asd->params.css_param.de_config)); - atomisp_css_set_de_config(asd, &asd->params.css_param.de_config); + asd->params.config.de_config = &asd->params.css_param.de_config; asd->params.css_update_params_needed = true; } @@ -4706,7 +4720,7 @@ int atomisp_white_balance_param(struct atomisp_sub_device *asd, int flag, /* Set white balance to isp parameters */ memcpy(&asd->params.css_param.wb_config, config, sizeof(asd->params.css_param.wb_config)); - atomisp_css_set_wb_config(asd, &asd->params.css_param.wb_config); + asd->params.config.wb_config = &asd->params.css_param.wb_config; asd->params.css_update_params_needed = true; } @@ -4728,7 +4742,7 @@ int atomisp_3a_config_param(struct atomisp_sub_device *asd, int flag, /* Set white balance to isp parameters */ memcpy(&asd->params.css_param.s3a_config, config, sizeof(asd->params.css_param.s3a_config)); - atomisp_css_set_3a_config(asd, &asd->params.css_param.s3a_config); + asd->params.config.s3a_config = &asd->params.css_param.s3a_config; asd->params.css_update_params_needed = true; } @@ -4959,13 +4973,14 @@ enum mipi_port_id __get_mipi_port(struct atomisp_device *isp, { switch (port) { case ATOMISP_CAMERA_PORT_PRIMARY: - return MIPI_PORT0_ID; + return MIPI_PORT0_ID; case ATOMISP_CAMERA_PORT_SECONDARY: return MIPI_PORT1_ID; case ATOMISP_CAMERA_PORT_TERTIARY: - if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID) + if (MIPI_PORT1_ID + 1 != N_MIPI_PORT_ID) { return MIPI_PORT1_ID + 1; - /* go through down for else case */ + } + /* fall through */ default: dev_err(isp->dev, "unsupported port: %d\n", port); return MIPI_PORT0_ID; @@ -5018,7 +5033,7 @@ static inline int atomisp_set_sensor_mipi_to_isp( mipi_info->input_format); if (!fc) return -EINVAL; - input_format = fc->css_stream_fmt; + input_format = fc->atomisp_in_fmt; } else { struct v4l2_mbus_framefmt *sink; @@ -5028,7 +5043,7 @@ static inline int atomisp_set_sensor_mipi_to_isp( fc = atomisp_find_in_fmt_conv(sink->code); if (!fc) return -EINVAL; - input_format = fc->css_stream_fmt; + input_format = fc->atomisp_in_fmt; bayer_order = fc->bayer_order; } @@ -5039,7 +5054,7 @@ static inline int atomisp_set_sensor_mipi_to_isp( mipi_info->metadata_format); if (!fc) return -EINVAL; - input_format = fc->css_stream_fmt; + input_format = fc->atomisp_in_fmt; atomisp_css_input_configure_port(asd, __get_mipi_port(asd->isp, mipi_info->port), mipi_info->num_lanes, @@ -5060,8 +5075,8 @@ static int __enable_continuous_mode(struct atomisp_sub_device *asd, enable, asd->continuous_raw_buffer_size->val, !asd->continuous_viewfinder->val); - if (!atomisp_hw_is_isp2401) - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_PRIMARY); + if (!IS_ISP2401) + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_PRIMARY); else atomisp_update_capture_mode(asd); @@ -5085,7 +5100,7 @@ static int __enable_continuous_mode(struct atomisp_sub_device *asd, } if (isp->inputs[asd->input_curr].type != FILE_INPUT) - atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR); + atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR); return atomisp_update_run_mode(asd); } @@ -5099,13 +5114,13 @@ static int configure_pp_input_nop(struct atomisp_sub_device *asd, static int configure_output_nop(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format sh_fmt) + enum ia_css_frame_format sh_fmt) { return 0; } static int get_frame_info_nop(struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *finfo) + struct ia_css_frame_info *finfo) { return 0; } @@ -5164,8 +5179,8 @@ static int css_input_resolution_changed(struct atomisp_sub_device *asd, } static int atomisp_set_fmt_to_isp(struct video_device *vdev, - struct atomisp_css_frame_info *output_info, - struct atomisp_css_frame_info *raw_output_info, + struct ia_css_frame_info *output_info, + struct ia_css_frame_info *raw_output_info, struct v4l2_pix_format *pix, unsigned int source_pad) { @@ -5174,22 +5189,22 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, struct atomisp_sub_device *asd = atomisp_to_video_pipe(vdev)->asd; const struct atomisp_format_bridge *format; struct v4l2_rect *isp_sink_crop; - enum atomisp_css_pipe_id pipe_id; + enum ia_css_pipe_id pipe_id; struct v4l2_subdev_fh fh; int (*configure_output)(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format sh_fmt) = + enum ia_css_frame_format sh_fmt) = configure_output_nop; int (*get_frame_info)(struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *finfo) = + struct ia_css_frame_info *finfo) = get_frame_info_nop; int (*configure_pp_input)(struct atomisp_sub_device *asd, unsigned int width, unsigned int height) = configure_pp_input_nop; u16 stream_index = atomisp_source_pad_to_stream_id(asd, source_pad); const struct atomisp_in_fmt_conv *fc; - int ret; + int ret, i; v4l2_fh_init(&fh.vfh, vdev); @@ -5221,8 +5236,8 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, ATOMISP_SUBDEV_PAD_SINK)->code); if (!fc) return -EINVAL; - if (format->sh_fmt == CSS_FRAME_FORMAT_RAW && - raw_output_format_match_input(fc->css_stream_fmt, + if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW && + raw_output_format_match_input(fc->atomisp_in_fmt, pix->pixelformat)) return -EINVAL; } @@ -5255,7 +5270,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, atomisp_subdev_set_ffmt(&asd->subdev, fh.pad, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SOURCE_VF, &vf_ffmt); - asd->video_out_vf.sh_fmt = CSS_FRAME_FORMAT_NV12; + asd->video_out_vf.sh_fmt = IA_CSS_FRAME_FORMAT_NV12; if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) { atomisp_css_video_configure_viewfinder(asd, @@ -5285,25 +5300,26 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, return -EINVAL; } - atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_SENSOR); - atomisp_css_disable_vf_pp(asd, - asd->vfpp->val != ATOMISP_VFPP_ENABLE); + atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_BUFFERED_SENSOR); + + for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) + asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].pipe_extra_configs[i].disable_vf_pp = asd->vfpp->val != ATOMISP_VFPP_ENABLE; /* ISP2401 new input system need to use copy pipe */ if (asd->copy_mode) { - pipe_id = CSS_PIPE_ID_COPY; + pipe_id = IA_CSS_PIPE_ID_COPY; atomisp_css_capture_enable_online(asd, stream_index, false); } else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) { /* video same in continuouscapture and online modes */ configure_output = atomisp_css_video_configure_output; get_frame_info = atomisp_css_video_get_output_frame_info; - pipe_id = CSS_PIPE_ID_VIDEO; + pipe_id = IA_CSS_PIPE_ID_VIDEO; } else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) { if (!asd->continuous_mode->val) { configure_output = atomisp_css_video_configure_output; get_frame_info = atomisp_css_video_get_output_frame_info; - pipe_id = CSS_PIPE_ID_VIDEO; + pipe_id = IA_CSS_PIPE_ID_VIDEO; } else { if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW || source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) { @@ -5313,7 +5329,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, atomisp_css_video_get_output_frame_info; configure_pp_input = atomisp_css_video_configure_pp_input; - pipe_id = CSS_PIPE_ID_VIDEO; + pipe_id = IA_CSS_PIPE_ID_VIDEO; } else { configure_output = atomisp_css_capture_configure_output; @@ -5321,7 +5337,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, atomisp_css_capture_get_output_frame_info; configure_pp_input = atomisp_css_capture_configure_pp_input; - pipe_id = CSS_PIPE_ID_CAPTURE; + pipe_id = IA_CSS_PIPE_ID_CAPTURE; atomisp_update_capture_mode(asd); atomisp_css_capture_enable_online(asd, stream_index, false); @@ -5331,7 +5347,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, configure_output = atomisp_css_preview_configure_output; get_frame_info = atomisp_css_preview_get_output_frame_info; configure_pp_input = atomisp_css_preview_configure_pp_input; - pipe_id = CSS_PIPE_ID_PREVIEW; + pipe_id = IA_CSS_PIPE_ID_PREVIEW; } else { /* CSS doesn't support low light mode on SOC cameras, so disable * it. FIXME: if this is done elsewhere, it gives corrupted @@ -5340,8 +5356,8 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, if (isp->inputs[asd->input_curr].type == SOC_CAMERA) asd->params.low_light = false; - if (format->sh_fmt == CSS_FRAME_FORMAT_RAW) { - atomisp_css_capture_set_mode(asd, CSS_CAPTURE_MODE_RAW); + if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW) { + atomisp_css_capture_set_mode(asd, IA_CSS_CAPTURE_MODE_RAW); atomisp_css_enable_dz(asd, false); } else { atomisp_update_capture_mode(asd); @@ -5356,7 +5372,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, configure_output = atomisp_css_capture_configure_output; get_frame_info = atomisp_css_capture_get_output_frame_info; configure_pp_input = atomisp_css_capture_configure_pp_input; - pipe_id = CSS_PIPE_ID_CAPTURE; + pipe_id = IA_CSS_PIPE_ID_CAPTURE; if (!asd->params.online_process && !asd->continuous_mode->val) { @@ -5377,7 +5393,7 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, * to SOC camera, use yuvpp pipe. */ if (ATOMISP_USE_YUVPP(asd)) - pipe_id = CSS_PIPE_ID_YUVPP; + pipe_id = IA_CSS_PIPE_ID_YUVPP; if (asd->copy_mode) ret = atomisp_css_copy_configure_output(asd, stream_index, @@ -5432,12 +5448,12 @@ static int atomisp_set_fmt_to_isp(struct video_device *vdev, atomisp_update_grid_info(asd, pipe_id, source_pad); /* Free the raw_dump buffer first */ - atomisp_css_frame_free(asd->raw_output_frame); + ia_css_frame_free(asd->raw_output_frame); asd->raw_output_frame = NULL; if (!asd->continuous_mode->val && !asd->params.online_process && !isp->sw_contex.file_input && - atomisp_css_frame_allocate_from_info(&asd->raw_output_frame, + ia_css_frame_allocate_from_info(&asd->raw_output_frame, raw_output_info)) return -ENOMEM; @@ -5593,24 +5609,19 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) struct atomisp_sub_device *asd = pipe->asd; const struct atomisp_format_bridge *format_bridge; const struct atomisp_format_bridge *snr_format_bridge; - struct atomisp_css_frame_info output_info, raw_output_info; + struct ia_css_frame_info output_info, raw_output_info; struct v4l2_format snr_fmt = *f; struct v4l2_format backup_fmt = *f, s_fmt = *f; unsigned int dvs_env_w = 0, dvs_env_h = 0; unsigned int padding_w = pad_w, padding_h = pad_h; bool res_overflow = false, crop_needs_override = false; - struct v4l2_mbus_framefmt isp_sink_fmt; + struct v4l2_mbus_framefmt *isp_sink_fmt; struct v4l2_mbus_framefmt isp_source_fmt = {0}; struct v4l2_rect isp_sink_crop; u16 source_pad = atomisp_subdev_source_pad(vdev); struct v4l2_subdev_fh fh; int ret; - dev_dbg(isp->dev, - "setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n", - f->fmt.pix.width, f->fmt.pix.height, source_pad, - asd->index, f->fmt.pix.bytesperline); - if (source_pad >= ATOMISP_SUBDEV_PADS_NUM) return -EINVAL; @@ -5619,6 +5630,11 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) return -EBUSY; } + dev_dbg(isp->dev, + "setting resolution %ux%u on pad %u for asd%d, bytesperline %u\n", + f->fmt.pix.width, f->fmt.pix.height, source_pad, + asd->index, f->fmt.pix.bytesperline); + v4l2_fh_init(&fh.vfh, vdev); format_bridge = atomisp_get_format_bridge(f->fmt.pix.pixelformat); @@ -5765,22 +5781,26 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) /* get sensor resolution and format */ ret = atomisp_try_fmt(vdev, &snr_fmt, &res_overflow); - if (ret) + if (ret) { + dev_warn(isp->dev, "Try format failed with error %d\n", ret); return ret; + } f->fmt.pix.width = snr_fmt.fmt.pix.width; f->fmt.pix.height = snr_fmt.fmt.pix.height; snr_format_bridge = atomisp_get_format_bridge(snr_fmt.fmt.pix.pixelformat); - if (!snr_format_bridge) + if (!snr_format_bridge) { + dev_warn(isp->dev, "Can't find bridge format\n"); return -EINVAL; + } atomisp_subdev_get_ffmt(&asd->subdev, NULL, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK)->code = snr_format_bridge->mbus_code; - isp_sink_fmt = *atomisp_subdev_get_ffmt(&asd->subdev, NULL, + isp_sink_fmt = atomisp_subdev_get_ffmt(&asd->subdev, NULL, V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK); @@ -5838,9 +5858,9 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) * capture pipe and usually has lower resolution than capture pipe. */ if (!asd->continuous_mode->val || - isp_sink_fmt.width < (f->fmt.pix.width + padding_w + dvs_env_w) || - isp_sink_fmt.height < (f->fmt.pix.height + padding_h + - dvs_env_h)) { + isp_sink_fmt->width < (f->fmt.pix.width + padding_w + dvs_env_w) || + isp_sink_fmt->height < (f->fmt.pix.height + padding_h + + dvs_env_h)) { /* * For jpeg or custom raw format the sensor will return constant * width and height. Because we already had quried try_mbus_fmt, @@ -5859,8 +5879,11 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) ret = atomisp_set_fmt_to_snr(vdev, &s_fmt, f->fmt.pix.pixelformat, padding_w, padding_h, dvs_env_w, dvs_env_h); - if (ret) + if (ret) { + dev_warn(isp->dev, + "Set format to sensor failed with %d\n", ret); return -EINVAL; + } atomisp_csi_lane_config(isp); crop_needs_override = true; @@ -5943,7 +5966,7 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) * which appears to be related by a hardware * performance limitation. It's unclear why this * particular code triggers the issue. */ - if (!atomisp_hw_is_isp2401 || crop_needs_override) { + if (!IS_ISP2401 || crop_needs_override) { if (isp_sink_crop.width * main_compose.height > isp_sink_crop.height * main_compose.width) { sink_crop.height = isp_sink_crop.height; @@ -5977,8 +6000,10 @@ int atomisp_set_fmt(struct video_device *vdev, struct v4l2_format *f) set_fmt_to_isp: ret = atomisp_set_fmt_to_isp(vdev, &output_info, &raw_output_info, &f->fmt.pix, source_pad); - if (ret) + if (ret) { + dev_warn(isp->dev, "Can't set format on ISP. Error %d\n", ret); return -EINVAL; + } done: pipe->pix.width = f->fmt.pix.width; pipe->pix.height = f->fmt.pix.height; @@ -5995,6 +6020,9 @@ done: pipe->pix.sizeimage = PAGE_ALIGN(f->fmt.pix.height * pipe->pix.bytesperline); } + dev_dbg(isp->dev, "%s: image size: %d, %d bytes per line\n", + __func__, pipe->pix.sizeimage, pipe->pix.bytesperline); + if (f->fmt.pix.field == V4L2_FIELD_ANY) f->fmt.pix.field = V4L2_FIELD_NONE; pipe->pix.field = f->fmt.pix.field; @@ -6045,7 +6073,7 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f) } pipe->pix = f->fmt.pix; - atomisp_css_input_set_mode(asd, CSS_INPUT_MODE_FIFO); + atomisp_css_input_set_mode(asd, IA_CSS_INPUT_MODE_FIFO); atomisp_css_input_configure_port(asd, __get_mipi_port(isp, ATOMISP_CAMERA_PORT_PRIMARY), 2, 0xffff4, 0, 0, 0, 0); @@ -6062,8 +6090,8 @@ int atomisp_set_fmt_file(struct video_device *vdev, struct v4l2_format *f) int atomisp_set_shading_table(struct atomisp_sub_device *asd, struct atomisp_shading_table *user_shading_table) { - struct atomisp_css_shading_table *shading_table; - struct atomisp_css_shading_table *free_table; + struct ia_css_shading_table *shading_table; + struct ia_css_shading_table *free_table; unsigned int len_table; int i; int ret = 0; @@ -6072,7 +6100,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd, return -EINVAL; if (!user_shading_table->enable) { - atomisp_css_set_shading_table(asd, NULL); + asd->params.config.shading_table = NULL; asd->params.sc_en = false; return 0; } @@ -6084,7 +6112,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd, } /* Shading table size per color */ - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (user_shading_table->width > ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR || user_shading_table->height > ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR) return -EINVAL; @@ -6117,7 +6145,7 @@ int atomisp_set_shading_table(struct atomisp_sub_device *asd, free_table = asd->params.css_param.shading_table; asd->params.css_param.shading_table = shading_table; - atomisp_css_set_shading_table(asd, shading_table); + asd->params.config.shading_table = shading_table; asd->params.sc_en = true; out: @@ -6564,31 +6592,31 @@ static int atomisp_get_pipe_id(struct atomisp_video_pipe *pipe) struct atomisp_sub_device *asd = pipe->asd; if (ATOMISP_USE_YUVPP(asd)) - return CSS_PIPE_ID_YUVPP; + return IA_CSS_PIPE_ID_YUVPP; else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; else if (pipe == &asd->video_out_video_capture) - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; else if (pipe == &asd->video_out_vf) - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; else if (pipe == &asd->video_out_preview) { if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; else - return CSS_PIPE_ID_PREVIEW; + return IA_CSS_PIPE_ID_PREVIEW; } else if (pipe == &asd->video_out_capture) { if (asd->copy_mode) return IA_CSS_PIPE_ID_COPY; else - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; } /* fail through */ dev_warn(asd->isp->dev, "%s failed to find proper pipe\n", __func__); - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; } int atomisp_get_invalid_frame_num(struct video_device *vdev, @@ -6596,7 +6624,7 @@ int atomisp_get_invalid_frame_num(struct video_device *vdev, { struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev); struct atomisp_sub_device *asd = pipe->asd; - enum atomisp_css_pipe_id pipe_id; + enum ia_css_pipe_id pipe_id; struct ia_css_pipe_info p_info; int ret; @@ -6618,7 +6646,7 @@ int atomisp_get_invalid_frame_num(struct video_device *vdev, ret = ia_css_pipe_get_info( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] .pipes[pipe_id], &p_info); - if (ret == IA_CSS_SUCCESS) { + if (!ret) { *invalid_frame_num = p_info.num_invalid_frames; return 0; } else { diff --git a/drivers/staging/media/atomisp/pci/atomisp_cmd.h b/drivers/staging/media/atomisp/pci/atomisp_cmd.h index b5af9da3b0fb..0bde995f1a8d 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_cmd.h +++ b/drivers/staging/media/atomisp/pci/atomisp_cmd.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -32,7 +33,7 @@ #include "ia_css.h" struct atomisp_device; -struct atomisp_css_frame; +struct ia_css_frame; #define MSI_ENABLE_BIT 16 #define INTR_DISABLE_BIT 10 @@ -64,16 +65,6 @@ bool atomisp_buffers_queued(struct atomisp_sub_device *asd); /* ISP2401 */ bool atomisp_buffers_queued_pipe(struct atomisp_video_pipe *pipe); -/* TODO:should be here instead of atomisp_helper.h -extern void __iomem *atomisp_io_base; - -static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address) -{ - void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF); - return ret; -} -*/ - /* * Interrupt functions */ @@ -88,7 +79,7 @@ const struct atomisp_format_bridge *get_atomisp_format_bridge_from_mbus( u32 mbus_code); bool atomisp_is_mbuscode_raw(uint32_t code); int atomisp_get_frame_pgnr(struct atomisp_device *isp, - const struct atomisp_css_frame *frame, u32 *p_pgnr); + const struct ia_css_frame *frame, u32 *p_pgnr); void atomisp_delayed_init_work(struct work_struct *work); /* @@ -301,8 +292,8 @@ int atomisp_set_array_res(struct atomisp_sub_device *asd, * Function to calculate real zoom region for every pipe */ int atomisp_calculate_real_zoom_region(struct atomisp_sub_device *asd, - struct atomisp_css_dz_config *dz_config, - enum atomisp_css_pipe_id css_pipe_id); + struct ia_css_dz_config *dz_config, + enum ia_css_pipe_id css_pipe_id); int atomisp_cp_general_isp_parameters(struct atomisp_sub_device *asd, struct atomisp_parameters *arg, @@ -372,8 +363,8 @@ int atomisp_freq_scaling(struct atomisp_device *vdev, bool force); void atomisp_buf_done(struct atomisp_sub_device *asd, int error, - enum atomisp_css_buffer_type buf_type, - enum atomisp_css_pipe_id css_pipe_id, + enum ia_css_buffer_type buf_type, + enum ia_css_pipe_id css_pipe_id, bool q_buffers, enum atomisp_input_stream_id stream_id); void atomisp_css_flush(struct atomisp_device *isp); diff --git a/drivers/staging/media/atomisp/pci/atomisp_common.h b/drivers/staging/media/atomisp/pci/atomisp_common.h index 65c9caf72b7b..b29874f2bc0f 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_common.h +++ b/drivers/staging/media/atomisp/pci/atomisp_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -43,15 +44,11 @@ extern int pad_h; /* ISP2401 */ #define ATOMISP_CSS_ISP_PIPE_VERSION_2_7 1 -#define IS_ISP2401(isp) \ - (((isp)->media_dev.hw_revision & ATOMISP_HW_REVISION_MASK) \ - >= (ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT)) - struct atomisp_format_bridge { unsigned int pixelformat; unsigned int depth; u32 mbus_code; - enum atomisp_css_frame_format sh_fmt; + enum ia_css_frame_format sh_fmt; unsigned char description[32]; /* the same as struct v4l2_fmtdesc */ bool planar; }; diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat.h b/drivers/staging/media/atomisp/pci/atomisp_compat.h index 205c530e8090..b2ed83c2f337 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat.h +++ b/drivers/staging/media/atomisp/pci/atomisp_compat.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Clovertrail PNW Camera Imaging ISP subsystem. * @@ -23,116 +24,20 @@ #include "../../include/linux/atomisp.h" #include <media/videobuf-vmalloc.h> -#define CSS_RX_IRQ_INFO_BUFFER_OVERRUN \ - CSS_ID(CSS_RX_IRQ_INFO_BUFFER_OVERRUN) -#define CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE \ - CSS_ID(CSS_RX_IRQ_INFO_ENTER_SLEEP_MODE) -#define CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE \ - CSS_ID(CSS_RX_IRQ_INFO_EXIT_SLEEP_MODE) -#define CSS_RX_IRQ_INFO_ECC_CORRECTED \ - CSS_ID(CSS_RX_IRQ_INFO_ECC_CORRECTED) -#define CSS_RX_IRQ_INFO_ERR_SOT \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT) -#define CSS_RX_IRQ_INFO_ERR_SOT_SYNC \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_SOT_SYNC) -#define CSS_RX_IRQ_INFO_ERR_CONTROL \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_CONTROL) -#define CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_ECC_DOUBLE) -#define CSS_RX_IRQ_INFO_ERR_CRC \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_CRC) -#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ID) -#define CSS_RX_IRQ_INFO_ERR_FRAME_SYNC \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_SYNC) -#define CSS_RX_IRQ_INFO_ERR_FRAME_DATA \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_FRAME_DATA) -#define CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_DATA_TIMEOUT) -#define CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_UNKNOWN_ESC) -#define CSS_RX_IRQ_INFO_ERR_LINE_SYNC \ - CSS_ID(CSS_RX_IRQ_INFO_ERR_LINE_SYNC) -#define CSS_RX_IRQ_INFO_INIT_TIMEOUT \ - CSS_ID(CSS_RX_IRQ_INFO_INIT_TIMEOUT) - -#define CSS_IRQ_INFO_CSS_RECEIVER_SOF CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_SOF) -#define CSS_IRQ_INFO_CSS_RECEIVER_EOF CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_EOF) -#define CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW \ - CSS_ID(CSS_IRQ_INFO_CSS_RECEIVER_FIFO_OVERFLOW) -#define CSS_EVENT_OUTPUT_FRAME_DONE CSS_EVENT(OUTPUT_FRAME_DONE) -#define CSS_EVENT_SEC_OUTPUT_FRAME_DONE CSS_EVENT(SECOND_OUTPUT_FRAME_DONE) -#define CSS_EVENT_VF_OUTPUT_FRAME_DONE CSS_EVENT(VF_OUTPUT_FRAME_DONE) -#define CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE CSS_EVENT(SECOND_VF_OUTPUT_FRAME_DONE) -#define CSS_EVENT_3A_STATISTICS_DONE CSS_EVENT(3A_STATISTICS_DONE) -#define CSS_EVENT_DIS_STATISTICS_DONE CSS_EVENT(DIS_STATISTICS_DONE) -#define CSS_EVENT_PIPELINE_DONE CSS_EVENT(PIPELINE_DONE) -#define CSS_EVENT_METADATA_DONE CSS_EVENT(METADATA_DONE) -#define CSS_EVENT_ACC_STAGE_COMPLETE CSS_EVENT(ACC_STAGE_COMPLETE) -#define CSS_EVENT_TIMER CSS_EVENT(TIMER) - -#define CSS_BUFFER_TYPE_METADATA CSS_ID(CSS_BUFFER_TYPE_METADATA) -#define CSS_BUFFER_TYPE_3A_STATISTICS CSS_ID(CSS_BUFFER_TYPE_3A_STATISTICS) -#define CSS_BUFFER_TYPE_DIS_STATISTICS CSS_ID(CSS_BUFFER_TYPE_DIS_STATISTICS) -#define CSS_BUFFER_TYPE_INPUT_FRAME CSS_ID(CSS_BUFFER_TYPE_INPUT_FRAME) -#define CSS_BUFFER_TYPE_OUTPUT_FRAME CSS_ID(CSS_BUFFER_TYPE_OUTPUT_FRAME) -#define CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME CSS_ID(CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME) -#define CSS_BUFFER_TYPE_VF_OUTPUT_FRAME CSS_ID(CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) -#define CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME CSS_ID(CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME) -#define CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME \ - CSS_ID(CSS_BUFFER_TYPE_RAW_OUTPUT_FRAME) - -#define CSS_FORMAT_RAW_8 CSS_FORMAT(RAW_8) -#define CSS_FORMAT_RAW_10 CSS_FORMAT(RAW_10) -#define CSS_FORMAT_RAW_12 CSS_FORMAT(RAW_12) -#define CSS_FORMAT_RAW_16 CSS_FORMAT(RAW_16) - -#define CSS_CAPTURE_MODE_RAW CSS_ID(CSS_CAPTURE_MODE_RAW) -#define CSS_CAPTURE_MODE_BAYER CSS_ID(CSS_CAPTURE_MODE_BAYER) -#define CSS_CAPTURE_MODE_PRIMARY CSS_ID(CSS_CAPTURE_MODE_PRIMARY) -#define CSS_CAPTURE_MODE_ADVANCED CSS_ID(CSS_CAPTURE_MODE_ADVANCED) -#define CSS_CAPTURE_MODE_LOW_LIGHT CSS_ID(CSS_CAPTURE_MODE_LOW_LIGHT) - -#define CSS_MORPH_TABLE_NUM_PLANES CSS_ID(CSS_MORPH_TABLE_NUM_PLANES) - -#define CSS_FRAME_FORMAT_NV11 CSS_ID(CSS_FRAME_FORMAT_NV11) -#define CSS_FRAME_FORMAT_NV12 CSS_ID(CSS_FRAME_FORMAT_NV12) -#define CSS_FRAME_FORMAT_NV16 CSS_ID(CSS_FRAME_FORMAT_NV16) -#define CSS_FRAME_FORMAT_NV21 CSS_ID(CSS_FRAME_FORMAT_NV21) -#define CSS_FRAME_FORMAT_NV61 CSS_ID(CSS_FRAME_FORMAT_NV61) -#define CSS_FRAME_FORMAT_YV12 CSS_ID(CSS_FRAME_FORMAT_YV12) -#define CSS_FRAME_FORMAT_YV16 CSS_ID(CSS_FRAME_FORMAT_YV16) -#define CSS_FRAME_FORMAT_YUV420 CSS_ID(CSS_FRAME_FORMAT_YUV420) -#define CSS_FRAME_FORMAT_YUV420_16 CSS_ID(CSS_FRAME_FORMAT_YUV420_16) -#define CSS_FRAME_FORMAT_YUV422 CSS_ID(CSS_FRAME_FORMAT_YUV422) -#define CSS_FRAME_FORMAT_YUV422_16 CSS_ID(CSS_FRAME_FORMAT_YUV422_16) -#define CSS_FRAME_FORMAT_UYVY CSS_ID(CSS_FRAME_FORMAT_UYVY) -#define CSS_FRAME_FORMAT_YUYV CSS_ID(CSS_FRAME_FORMAT_YUYV) -#define CSS_FRAME_FORMAT_YUV444 CSS_ID(CSS_FRAME_FORMAT_YUV444) -#define CSS_FRAME_FORMAT_YUV_LINE CSS_ID(CSS_FRAME_FORMAT_YUV_LINE) -#define CSS_FRAME_FORMAT_RAW CSS_ID(CSS_FRAME_FORMAT_RAW) -#define CSS_FRAME_FORMAT_RGB565 CSS_ID(CSS_FRAME_FORMAT_RGB565) -#define CSS_FRAME_FORMAT_PLANAR_RGB888 CSS_ID(CSS_FRAME_FORMAT_PLANAR_RGB888) -#define CSS_FRAME_FORMAT_RGBA888 CSS_ID(CSS_FRAME_FORMAT_RGBA888) -#define CSS_FRAME_FORMAT_QPLANE6 CSS_ID(CSS_FRAME_FORMAT_QPLANE6) -#define CSS_FRAME_FORMAT_BINARY_8 CSS_ID(CSS_FRAME_FORMAT_BINARY_8) - struct atomisp_device; struct atomisp_sub_device; struct video_device; enum atomisp_input_stream_id; +extern void __iomem *atomisp_io_base; + struct atomisp_metadata_buf { struct ia_css_metadata *metadata; void *md_vptr; struct list_head list; }; -void atomisp_css_debug_dump_sp_sw_debug_info(void); -void atomisp_css_debug_dump_debug_info(const char *context); -void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level); - -void atomisp_store_uint32(hrt_address addr, uint32_t data); +void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data); void atomisp_load_uint32(hrt_address addr, uint32_t *data); int atomisp_css_init(struct atomisp_device *isp); @@ -155,35 +60,35 @@ void atomisp_css_rx_clear_irq_info(enum mipi_port_id port, unsigned int infos); int atomisp_css_irq_enable(struct atomisp_device *isp, - enum atomisp_css_irq_info info, bool enable); + enum ia_css_irq_info info, bool enable); int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd, struct videobuf_vmalloc_memory *vm_mem, enum atomisp_input_stream_id stream_id, - enum atomisp_css_buffer_type css_buf_type, - enum atomisp_css_pipe_id css_pipe_id); + enum ia_css_buffer_type css_buf_type, + enum ia_css_pipe_id css_pipe_id); int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_s3a_buf *s3a_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id); + enum ia_css_pipe_id css_pipe_id); int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_metadata_buf *metadata_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id); + enum ia_css_pipe_id css_pipe_id); int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_dis_buf *dis_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id); + enum ia_css_pipe_id css_pipe_id); -void atomisp_css_mmu_invalidate_cache(void); +void ia_css_mmu_invalidate_cache(void); -void atomisp_css_mmu_invalidate_tlb(void); +void ia_css_mmu_invalidate_cache(void); int atomisp_css_start(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, bool in_reset); + enum ia_css_pipe_id pipe_id, bool in_reset); void atomisp_css_update_isp_params(struct atomisp_sub_device *asd); void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd, @@ -191,14 +96,14 @@ void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd, int atomisp_css_queue_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id pipe_id, - enum atomisp_css_buffer_type buf_type, + enum ia_css_pipe_id pipe_id, + enum ia_css_buffer_type buf_type, struct atomisp_css_buffer *isp_css_buffer); int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id pipe_id, - enum atomisp_css_buffer_type buf_type, + enum ia_css_pipe_id pipe_id, + enum ia_css_buffer_type buf_type, struct atomisp_css_buffer *isp_css_buffer); int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device *asd, @@ -217,7 +122,7 @@ void atomisp_css_free_metadata_buffer(struct atomisp_metadata_buf *metadata_buf); int atomisp_css_get_grid_info(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, + enum ia_css_pipe_id pipe_id, int source_pad); int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd); @@ -287,7 +192,7 @@ void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd, void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_bayer_order bayer_order); + enum ia_css_bayer_order bayer_order); void atomisp_css_input_set_format(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, @@ -312,10 +217,10 @@ void atomisp_css_enable_raw_binning(struct atomisp_sub_device *asd, void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable); void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd, - enum atomisp_css_capture_mode mode); + enum ia_css_capture_mode mode); void atomisp_css_input_set_mode(struct atomisp_sub_device *asd, - enum atomisp_css_input_mode mode); + enum ia_css_input_mode mode); void atomisp_css_capture_enable_online(struct atomisp_sub_device *asd, unsigned short stream_index, bool enable); @@ -341,130 +246,103 @@ int atomisp_css_input_configure_port(struct atomisp_sub_device *asd, unsigned int metadata_width, unsigned int metadata_height); -int atomisp_css_frame_allocate(struct atomisp_css_frame **frame, - unsigned int width, unsigned int height, - enum atomisp_css_frame_format format, - unsigned int padded_width, - unsigned int raw_bit_depth); - -int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame, - const struct atomisp_css_frame_info *info); - -void atomisp_css_frame_free(struct atomisp_css_frame *frame); - -int atomisp_css_frame_map(struct atomisp_css_frame **frame, - const struct atomisp_css_frame_info *info, - const void __user *data, uint16_t attribute, - void *context); - -int atomisp_css_set_black_frame(struct atomisp_sub_device *asd, - const struct atomisp_css_frame *raw_black_frame); - -int atomisp_css_allocate_continuous_frames(bool init_time, - struct atomisp_sub_device *asd); - -void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd); - void atomisp_create_pipes_stream(struct atomisp_sub_device *asd); void atomisp_destroy_pipes_stream_force(struct atomisp_sub_device *asd); int atomisp_css_stop(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, bool in_reset); + enum ia_css_pipe_id pipe_id, bool in_reset); int atomisp_css_continuous_set_num_raw_frames( struct atomisp_sub_device *asd, int num_frames); -void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd, - bool disable); - int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd, unsigned int stream_index, unsigned int width, unsigned int height, unsigned int padded_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd, unsigned int stream_index, unsigned int width, unsigned int height, unsigned int padded_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_yuvpp_configure_viewfinder( struct atomisp_sub_device *asd, unsigned int stream_index, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_yuvpp_get_output_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_yuvpp_get_viewfinder_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_video_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_get_css_frame_info(struct atomisp_sub_device *asd, u16 source_pad, - struct atomisp_css_frame_info *frame_info); + struct ia_css_frame_info *frame_info); int atomisp_css_video_configure_viewfinder(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_capture_configure_viewfinder( struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format); + enum ia_css_frame_format format); int atomisp_css_video_get_viewfinder_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_capture_get_viewfinder_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_copy_get_output_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_capture_get_output_raw_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_preview_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_capture_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_video_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info); + struct ia_css_frame_info *info); int atomisp_css_preview_configure_pp_input( struct atomisp_sub_device *asd, @@ -486,60 +364,8 @@ int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id); int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd, bool enable); -void atomisp_css_send_input_frame(struct atomisp_sub_device *asd, - unsigned short *data, unsigned int width, - unsigned int height); - -bool atomisp_css_isp_has_started(void); - -void atomisp_css_request_flash(struct atomisp_sub_device *asd); - -void atomisp_css_set_wb_config(struct atomisp_sub_device *asd, - struct atomisp_css_wb_config *wb_config); - -void atomisp_css_set_ob_config(struct atomisp_sub_device *asd, - struct atomisp_css_ob_config *ob_config); - -void atomisp_css_set_dp_config(struct atomisp_sub_device *asd, - struct atomisp_css_dp_config *dp_config); - -void atomisp_css_set_de_config(struct atomisp_sub_device *asd, - struct atomisp_css_de_config *de_config); - -void atomisp_css_set_dz_config(struct atomisp_sub_device *asd, - struct atomisp_css_dz_config *dz_config); - -void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd); - -void atomisp_css_set_ce_config(struct atomisp_sub_device *asd, - struct atomisp_css_ce_config *ce_config); - -void atomisp_css_set_nr_config(struct atomisp_sub_device *asd, - struct atomisp_css_nr_config *nr_config); - -void atomisp_css_set_ee_config(struct atomisp_sub_device *asd, - struct atomisp_css_ee_config *ee_config); - -void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_tnr_config *tnr_config); - -void atomisp_css_set_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *cc_config); - -void atomisp_css_set_macc_table(struct atomisp_sub_device *asd, - struct atomisp_css_macc_table *macc_table); - -void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_gamma_table *gamma_table); - void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd, - struct atomisp_css_ctc_table *ctc_table); - -void atomisp_css_set_gc_config(struct atomisp_sub_device *asd, - struct atomisp_css_gc_config *gc_config); - -void atomisp_css_set_3a_config(struct atomisp_sub_device *asd, - struct atomisp_css_3a_config *s3a_config); + struct ia_css_ctc_table *ctc_table); void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd, struct atomisp_dis_vector *vector); @@ -590,29 +416,29 @@ int atomisp_css_get_formats_config(struct atomisp_sub_device *asd, struct atomisp_formats_config *formats_config); void atomisp_css_set_formats_config(struct atomisp_sub_device *asd, - struct atomisp_css_formats_config *formats_config); + struct ia_css_formats_config *formats_config); int atomisp_css_get_zoom_factor(struct atomisp_sub_device *asd, unsigned int *zoom); -struct atomisp_css_shading_table *atomisp_css_shading_table_alloc( +struct ia_css_shading_table *atomisp_css_shading_table_alloc( unsigned int width, unsigned int height); void atomisp_css_set_shading_table(struct atomisp_sub_device *asd, - struct atomisp_css_shading_table *table); + struct ia_css_shading_table *table); -void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table); +void atomisp_css_shading_table_free(struct ia_css_shading_table *table); -struct atomisp_css_morph_table *atomisp_css_morph_table_allocate( +struct ia_css_morph_table *atomisp_css_morph_table_allocate( unsigned int width, unsigned int height); void atomisp_css_set_morph_table(struct atomisp_sub_device *asd, - struct atomisp_css_morph_table *table); + struct ia_css_morph_table *table); void atomisp_css_get_morph_table(struct atomisp_sub_device *asd, - struct atomisp_css_morph_table *table); + struct ia_css_morph_table *table); -void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table); +void atomisp_css_morph_table_free(struct ia_css_morph_table *table); void atomisp_css_set_cont_prev_start_time(struct atomisp_device *isp, unsigned int overlap); @@ -631,20 +457,20 @@ int atomisp_css_stop_acc_pipe(struct atomisp_sub_device *asd); void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd); int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, - enum atomisp_css_pipe_id pipe_id, + struct ia_css_fw_info *fw, + enum ia_css_pipe_id pipe_id, unsigned int type); void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, - enum atomisp_css_pipe_id pipe_id); + struct ia_css_fw_info *fw, + enum ia_css_pipe_id pipe_id); int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd); void atomisp_css_acc_done(struct atomisp_sub_device *asd); int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, + struct ia_css_fw_info *fw, unsigned int index); void atomisp_css_unload_acc_binary(struct atomisp_sub_device *asd); diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c index 209bc9954a53..c1e282a974d0 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c +++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Clovertrail PNW Camera Imaging ISP subsystem. * @@ -32,8 +33,6 @@ #include "atomisp_ioctl.h" #include "atomisp_acc.h" -#include "hrt/hive_isp_css_mm_hrt.h" - #include <asm/intel-mid.h> #include "ia_css_debug.h" @@ -68,82 +67,68 @@ struct bayer_ds_factor { unsigned int denominator; }; -void atomisp_css_debug_dump_sp_sw_debug_info(void) -{ - ia_css_debug_dump_sp_sw_debug_info(); -} - -void atomisp_css_debug_dump_debug_info(const char *context) -{ - ia_css_debug_dump_debug_info(context); -} - -void atomisp_css_debug_set_dtrace_level(const unsigned int trace_level) -{ - ia_css_debug_set_dtrace_level(trace_level); -} - -unsigned int atomisp_css_debug_get_dtrace_level(void) -{ - return ia_css_debug_trace_level; -} - static void atomisp_css2_hw_store_8(hrt_address addr, uint8_t data) { + s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; spin_lock_irqsave(&mmio_lock, flags); - _hrt_master_port_store_8(addr, data); + *io_virt_addr = data; spin_unlock_irqrestore(&mmio_lock, flags); } static void atomisp_css2_hw_store_16(hrt_address addr, uint16_t data) { + s16 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; spin_lock_irqsave(&mmio_lock, flags); - _hrt_master_port_store_16(addr, data); + *io_virt_addr = data; spin_unlock_irqrestore(&mmio_lock, flags); } -static void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data) +void atomisp_css2_hw_store_32(hrt_address addr, uint32_t data) { + s32 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; spin_lock_irqsave(&mmio_lock, flags); - _hrt_master_port_store_32(addr, data); + *io_virt_addr = data; spin_unlock_irqrestore(&mmio_lock, flags); } static uint8_t atomisp_css2_hw_load_8(hrt_address addr) { + s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; u8 ret; spin_lock_irqsave(&mmio_lock, flags); - ret = _hrt_master_port_load_8(addr); + ret = *io_virt_addr; spin_unlock_irqrestore(&mmio_lock, flags); return ret; } static uint16_t atomisp_css2_hw_load_16(hrt_address addr) { + s16 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; u16 ret; spin_lock_irqsave(&mmio_lock, flags); - ret = _hrt_master_port_load_16(addr); + ret = *io_virt_addr; spin_unlock_irqrestore(&mmio_lock, flags); return ret; } static uint32_t atomisp_css2_hw_load_32(hrt_address addr) { + s32 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; u32 ret; spin_lock_irqsave(&mmio_lock, flags); - ret = _hrt_master_port_load_32(addr); + ret = *io_virt_addr; spin_unlock_irqrestore(&mmio_lock, flags); return ret; } @@ -151,27 +136,25 @@ static uint32_t atomisp_css2_hw_load_32(hrt_address addr) static void atomisp_css2_hw_store(hrt_address addr, const void *from, uint32_t n) { + s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; unsigned int i; - unsigned int _to = (unsigned int)addr; - const char *_from = (const char *)from; spin_lock_irqsave(&mmio_lock, flags); - for (i = 0; i < n; i++, _to++, _from++) - _hrt_master_port_store_8(_to, *_from); + for (i = 0; i < n; i++, io_virt_addr++, from++) + *io_virt_addr = *(s8 *)from; spin_unlock_irqrestore(&mmio_lock, flags); } static void atomisp_css2_hw_load(hrt_address addr, void *to, uint32_t n) { + s8 __iomem *io_virt_addr = atomisp_io_base + (addr & 0x003FFFFF); unsigned long flags; unsigned int i; - char *_to = (char *)to; - unsigned int _from = (unsigned int)addr; spin_lock_irqsave(&mmio_lock, flags); - for (i = 0; i < n; i++, _to++, _from++) - *_to = _hrt_master_port_load_8(_from); + for (i = 0; i < n; i++, to++, io_virt_addr++) + *(s8 *)to = *io_virt_addr; spin_unlock_irqrestore(&mmio_lock, flags); } @@ -193,11 +176,6 @@ static int atomisp_css2_err_print(const char *fmt, va_list args) return 0; } -void atomisp_store_uint32(hrt_address addr, uint32_t data) -{ - atomisp_css2_hw_store_32(addr, data); -} - void atomisp_load_uint32(hrt_address addr, uint32_t *data) { *data = atomisp_css2_hw_load_32(addr); @@ -215,16 +193,6 @@ static int hmm_get_mmu_base_addr(unsigned int *mmu_base_addr) return 0; } -static void atomisp_isp_parameters_clean_up( - struct atomisp_css_isp_config *config) -{ - /* - * Set NULL to configs pointer to avoid they are set into isp again when - * some configs are changed and need to be updated later. - */ - memset(config, 0, sizeof(*config)); -} - static void __dump_pipe_config(struct atomisp_sub_device *asd, struct atomisp_stream_env *stream_env, unsigned int pipe_id) @@ -474,7 +442,7 @@ static int __destroy_stream(struct atomisp_sub_device *asd, } if (stream_env->stream_state == CSS_STREAM_STARTED - && ia_css_stream_stop(stream_env->stream) != IA_CSS_SUCCESS) { + && ia_css_stream_stop(stream_env->stream) != 0) { dev_err(isp->dev, "stop stream failed.\n"); return -EINVAL; } @@ -496,7 +464,7 @@ static int __destroy_stream(struct atomisp_sub_device *asd, stream_env->stream_state = CSS_STREAM_STOPPED; - if (ia_css_stream_destroy(stream_env->stream) != IA_CSS_SUCCESS) { + if (ia_css_stream_destroy(stream_env->stream)) { dev_err(isp->dev, "destroy stream failed.\n"); return -EINVAL; } @@ -540,10 +508,10 @@ static int __create_stream(struct atomisp_sub_device *asd, __dump_stream_config(asd, stream_env); if (ia_css_stream_create(&stream_env->stream_config, - pipe_index, multi_pipes, &stream_env->stream) != IA_CSS_SUCCESS) + pipe_index, multi_pipes, &stream_env->stream) != 0) return -EINVAL; if (ia_css_stream_get_info(stream_env->stream, - &stream_env->stream_info) != IA_CSS_SUCCESS) { + &stream_env->stream_info) != 0) { ia_css_stream_destroy(stream_env->stream); stream_env->stream = NULL; return -EINVAL; @@ -583,7 +551,7 @@ static int __destroy_stream_pipes(struct atomisp_sub_device *asd, !(force || stream_env->update_pipe[i])) continue; if (ia_css_pipe_destroy(stream_env->pipes[i]) - != IA_CSS_SUCCESS) { + != 0) { dev_err(isp->dev, "destroy pipe[%d]failed.cannot recover.\n", i); ret = -EINVAL; @@ -645,10 +613,10 @@ static void __apply_additional_pipe_config( /* enable capture pp/dz manually or digital zoom would * fail*/ if (stream_env->pipe_configs[pipe_id]. - default_capture_config.mode == CSS_CAPTURE_MODE_RAW) + default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW) stream_env->pipe_configs[pipe_id].enable_dz = false; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* the isp default to use ISP2.2 and the camera hal will * control whether use isp2.7 */ if (asd->select_isp_version->val == ATOMISP_CSS_ISP_PIPE_VERSION_2_7) @@ -699,7 +667,7 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd, if (!asd) return false; - if (pipe_id == CSS_PIPE_ID_ACC || pipe_id == CSS_PIPE_ID_YUVPP) + if (pipe_id == IA_CSS_PIPE_ID_ACC || pipe_id == IA_CSS_PIPE_ID_YUVPP) return true; if (asd->vfpp) { @@ -726,22 +694,23 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd, case ATOMISP_RUN_MODE_STILL_CAPTURE: if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) return true; - else - return false; + + return false; case ATOMISP_RUN_MODE_PREVIEW: if (!asd->continuous_mode->val) { if (pipe_id == IA_CSS_PIPE_ID_PREVIEW) return true; - else - return false; + + return false; } - /* fall through to ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE */ + /* fall-through */ case ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE: if (pipe_id == IA_CSS_PIPE_ID_CAPTURE || pipe_id == IA_CSS_PIPE_ID_PREVIEW) return true; - else - return false; + + return false; + /* fall-through */ case ATOMISP_RUN_MODE_VIDEO: if (!asd->continuous_mode->val) { if (pipe_id == IA_CSS_PIPE_ID_VIDEO || @@ -750,13 +719,13 @@ static bool is_pipe_valid_to_current_run_mode(struct atomisp_sub_device *asd, else return false; } - /* fall through to ATOMISP_RUN_MODE_SDV */ + /* fall through */ case ATOMISP_RUN_MODE_SDV: if (pipe_id == IA_CSS_PIPE_ID_CAPTURE || pipe_id == IA_CSS_PIPE_ID_VIDEO) return true; - else - return false; + + return false; } return false; @@ -768,16 +737,16 @@ static int __create_pipe(struct atomisp_sub_device *asd, { struct atomisp_device *isp = asd->isp; struct ia_css_pipe_extra_config extra_config; - enum ia_css_err ret; + int ret; if (pipe_id >= IA_CSS_PIPE_ID_NUM) return -EINVAL; - if (pipe_id != CSS_PIPE_ID_ACC && + if (pipe_id != IA_CSS_PIPE_ID_ACC && !stream_env->pipe_configs[pipe_id].output_info[0].res.width) return 0; - if (pipe_id == CSS_PIPE_ID_ACC && + if (pipe_id == IA_CSS_PIPE_ID_ACC && !stream_env->pipe_configs[pipe_id].acc_extension) return 0; @@ -798,20 +767,20 @@ static int __create_pipe(struct atomisp_sub_device *asd, &stream_env->pipe_configs[pipe_id], &stream_env->pipe_extra_configs[pipe_id], &stream_env->pipes[pipe_id]); - if (ret != IA_CSS_SUCCESS) + if (ret) dev_err(isp->dev, "create pipe[%d] error.\n", pipe_id); return ret; } static int __create_pipes(struct atomisp_sub_device *asd) { - enum ia_css_err ret; + int ret; int i, j; for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) { for (j = 0; j < IA_CSS_PIPE_ID_NUM; j++) { ret = __create_pipe(asd, &asd->stream_env[i], j); - if (ret != IA_CSS_SUCCESS) + if (ret) break; } if (j < IA_CSS_PIPE_ID_NUM) @@ -842,20 +811,20 @@ int atomisp_css_update_stream(struct atomisp_sub_device *asd) int ret; struct atomisp_device *isp = asd->isp; - if (__destroy_streams(asd, true) != IA_CSS_SUCCESS) + if (__destroy_streams(asd, true)) dev_warn(isp->dev, "destroy stream failed.\n"); - if (__destroy_pipes(asd, true) != IA_CSS_SUCCESS) + if (__destroy_pipes(asd, true)) dev_warn(isp->dev, "destroy pipe failed.\n"); ret = __create_pipes(asd); - if (ret != IA_CSS_SUCCESS) { + if (ret) { dev_err(isp->dev, "create pipe failed %d.\n", ret); return -EIO; } ret = __create_streams(asd); - if (ret != IA_CSS_SUCCESS) { + if (ret) { dev_warn(isp->dev, "create stream failed %d.\n", ret); __destroy_pipes(asd, true); return -EIO; @@ -868,7 +837,7 @@ int atomisp_css_init(struct atomisp_device *isp) { unsigned int mmu_base_addr; int ret; - enum ia_css_err err; + int err; ret = hmm_get_mmu_base_addr(&mmu_base_addr); if (ret) @@ -877,7 +846,7 @@ int atomisp_css_init(struct atomisp_device *isp) /* Init ISP */ err = ia_css_init(isp->dev, &isp->css_env.isp_css_env, NULL, (uint32_t)mmu_base_addr, IA_CSS_IRQ_TYPE_PULSE); - if (err != IA_CSS_SUCCESS) { + if (err) { dev_err(isp->dev, "css init failed --- bad firmware?\n"); return -EINVAL; } @@ -907,17 +876,9 @@ static inline int __set_css_print_env(struct atomisp_device *isp, int opt) return ret; } -int atomisp_css_check_firmware_version(struct atomisp_device *isp) -{ - if (!sh_css_check_firmware_version(isp->dev, (void *)isp->firmware->data)) { - return -EINVAL; - } - return 0; -} - int atomisp_css_load_firmware(struct atomisp_device *isp) { - enum ia_css_err err; + int err; /* set css env */ isp->css_env.isp_css_fw.data = (void *)isp->firmware->data; @@ -946,7 +907,7 @@ int atomisp_css_load_firmware(struct atomisp_device *isp) /* load isp fw into ISP memory */ err = ia_css_load_firmware(isp->dev, &isp->css_env.isp_css_env, &isp->css_env.isp_css_fw); - if (err != IA_CSS_SUCCESS) { + if (err) { dev_err(isp->dev, "css load fw failed.\n"); return -EINVAL; } @@ -954,11 +915,6 @@ int atomisp_css_load_firmware(struct atomisp_device *isp) return 0; } -void atomisp_css_unload_firmware(struct atomisp_device *isp) -{ - ia_css_unload_firmware(); -} - void atomisp_css_uninit(struct atomisp_device *isp) { struct atomisp_sub_device *asd; @@ -966,7 +922,7 @@ void atomisp_css_uninit(struct atomisp_device *isp) for (i = 0; i < isp->num_of_streams; i++) { asd = &isp->asd[i]; - atomisp_isp_parameters_clean_up(&asd->params.config); + memset(&asd->params.config, 0, sizeof(asd->params.config)); asd->params.css_update_params_needed = false; } @@ -1009,7 +965,7 @@ int atomisp_css_irq_translate(struct atomisp_device *isp, int err; err = ia_css_irq_translate(infos); - if (err != IA_CSS_SUCCESS) { + if (err) { dev_warn(isp->dev, "%s:failed to translate irq (err = %d,infos = %d)\n", __func__, err, *infos); @@ -1038,13 +994,15 @@ void atomisp_css_rx_clear_irq_info(enum mipi_port_id port, } int atomisp_css_irq_enable(struct atomisp_device *isp, - enum atomisp_css_irq_info info, bool enable) + enum ia_css_irq_info info, bool enable) { - dev_dbg(isp->dev, "%s: css irq info 0x%08x: %s.\n", + dev_dbg(isp->dev, "%s: css irq info 0x%08x: %s (%d).\n", __func__, info, - enable ? "enable" : "disable"); - if (ia_css_irq_enable(info, enable) != IA_CSS_SUCCESS) { - dev_warn(isp->dev, "%s:Invalid irq info.\n", __func__); + enable ? "enable" : "disable", enable); + if (ia_css_irq_enable(info, enable)) { + dev_warn(isp->dev, "%s:Invalid irq info: 0x%08x when %s.\n", + __func__, info, + enable ? "enabling" : "disabling"); return -EINVAL; } @@ -1072,19 +1030,19 @@ void atomisp_css_init_struct(struct atomisp_sub_device *asd) int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd, struct videobuf_vmalloc_memory *vm_mem, enum atomisp_input_stream_id stream_id, - enum atomisp_css_buffer_type css_buf_type, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_buffer_type css_buf_type, + enum ia_css_pipe_id css_pipe_id) { struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id]; struct ia_css_buffer css_buf = {0}; - enum ia_css_err err; + int err; css_buf.type = css_buf_type; css_buf.data.frame = vm_mem->vaddr; err = ia_css_pipe_enqueue_buffer( stream_env->pipes[css_pipe_id], &css_buf); - if (err != IA_CSS_SUCCESS) + if (err) return -EINVAL; return 0; @@ -1093,7 +1051,7 @@ int atomisp_q_video_buffer_to_css(struct atomisp_sub_device *asd, int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_metadata_buf *metadata_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id]; struct ia_css_buffer buffer = {0}; @@ -1113,7 +1071,7 @@ int atomisp_q_metadata_buffer_to_css(struct atomisp_sub_device *asd, int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_s3a_buf *s3a_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id]; struct ia_css_buffer buffer = {0}; @@ -1134,7 +1092,7 @@ int atomisp_q_s3a_buffer_to_css(struct atomisp_sub_device *asd, int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd, struct atomisp_dis_buf *dis_buf, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_stream_env *stream_env = &asd->stream_env[stream_id]; struct ia_css_buffer buffer = {0}; @@ -1152,18 +1110,8 @@ int atomisp_q_dis_buffer_to_css(struct atomisp_sub_device *asd, return 0; } -void atomisp_css_mmu_invalidate_cache(void) -{ - ia_css_mmu_invalidate_cache(); -} - -void atomisp_css_mmu_invalidate_tlb(void) -{ - ia_css_mmu_invalidate_cache(); -} - int atomisp_css_start(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, bool in_reset) + enum ia_css_pipe_id pipe_id, bool in_reset) { struct atomisp_device *isp = asd->isp; bool sp_is_started = false; @@ -1222,7 +1170,7 @@ int atomisp_css_start(struct atomisp_sub_device *asd, } else { if (!sh_css_hrt_system_is_idle()) dev_err(isp->dev, "CSS HW not idle before starting SP\n"); - if (ia_css_start_sp() != IA_CSS_SUCCESS) { + if (ia_css_start_sp()) { dev_err(isp->dev, "start sp error.\n"); ret = -EINVAL; goto start_err; @@ -1234,7 +1182,7 @@ int atomisp_css_start(struct atomisp_sub_device *asd, for (i = 0; i < ATOMISP_INPUT_STREAM_NUM; i++) { if (asd->stream_env[i].stream) { if (ia_css_stream_start(asd->stream_env[i] - .stream) != IA_CSS_SUCCESS) { + .stream) != 0) { dev_err(isp->dev, "stream[%d] start error.\n", i); ret = -EINVAL; goto start_err; @@ -1285,13 +1233,13 @@ void atomisp_css_update_isp_params(struct atomisp_sub_device *asd) ia_css_stream_set_isp_config( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, &asd->params.config); - atomisp_isp_parameters_clean_up(&asd->params.config); + memset(&asd->params.config, 0, sizeof(asd->params.config)); } void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd, struct ia_css_pipe *pipe) { - enum ia_css_err ret; + int ret; if (!pipe) { atomisp_css_update_isp_params(asd); @@ -1306,22 +1254,22 @@ void atomisp_css_update_isp_params_on_pipe(struct atomisp_sub_device *asd, ret = ia_css_stream_set_isp_config_on_pipe( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, &asd->params.config, pipe); - if (ret != IA_CSS_SUCCESS) + if (ret) dev_warn(asd->isp->dev, "%s: ia_css_stream_set_isp_config_on_pipe failed %d\n", __func__, ret); - atomisp_isp_parameters_clean_up(&asd->params.config); + memset(&asd->params.config, 0, sizeof(asd->params.config)); } int atomisp_css_queue_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id pipe_id, - enum atomisp_css_buffer_type buf_type, + enum ia_css_pipe_id pipe_id, + enum ia_css_buffer_type buf_type, struct atomisp_css_buffer *isp_css_buffer) { if (ia_css_pipe_enqueue_buffer( asd->stream_env[stream_id].pipes[pipe_id], &isp_css_buffer->css_buffer) - != IA_CSS_SUCCESS) + != 0) return -EINVAL; return 0; @@ -1329,17 +1277,17 @@ int atomisp_css_queue_buffer(struct atomisp_sub_device *asd, int atomisp_css_dequeue_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id pipe_id, - enum atomisp_css_buffer_type buf_type, + enum ia_css_pipe_id pipe_id, + enum ia_css_buffer_type buf_type, struct atomisp_css_buffer *isp_css_buffer) { struct atomisp_device *isp = asd->isp; - enum ia_css_err err; + int err; err = ia_css_pipe_dequeue_buffer( asd->stream_env[stream_id].pipes[pipe_id], &isp_css_buffer->css_buffer); - if (err != IA_CSS_SUCCESS) { + if (err) { dev_err(isp->dev, "ia_css_pipe_dequeue_buffer failed: 0x%x\n", err); return -EINVAL; @@ -1355,7 +1303,7 @@ int atomisp_css_allocate_stat_buffers(struct atomisp_sub_device *asd, struct atomisp_metadata_buf *md_buf) { struct atomisp_device *isp = asd->isp; - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); if (s3a_buf && asd->params.curr_grid_info.s3a_grid.enable) { @@ -1442,7 +1390,7 @@ void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd) struct atomisp_s3a_buf *s3a_buf, *_s3a_buf; struct atomisp_dis_buf *dis_buf, *_dis_buf; struct atomisp_metadata_buf *md_buf, *_md_buf; - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); unsigned int i; @@ -1524,7 +1472,7 @@ void atomisp_css_free_stat_buffers(struct atomisp_sub_device *asd) } int atomisp_css_get_grid_info(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, + enum ia_css_pipe_id pipe_id, int source_pad) { struct ia_css_pipe_info p_info; @@ -1539,7 +1487,7 @@ int atomisp_css_get_grid_info(struct atomisp_sub_device *asd, if (ia_css_pipe_get_info( asd->stream_env[stream_index].pipes[pipe_id], - &p_info) != IA_CSS_SUCCESS) { + &p_info) != 0) { dev_err(isp->dev, "ia_css_pipe_get_info failed\n"); return -EINVAL; } @@ -1553,7 +1501,7 @@ int atomisp_css_get_grid_info(struct atomisp_sub_device *asd, * Currently would have one css pipe that need it */ if (asd->params.curr_grid_info.s3a_grid.enable) { - if (asd->params.s3a_enabled_pipe != CSS_PIPE_ID_NUM) + if (asd->params.s3a_enabled_pipe != IA_CSS_PIPE_ID_NUM) dev_dbg(isp->dev, "css pipe %d enabled s3a grid replaced by: %d.\n", asd->params.s3a_enabled_pipe, pipe_id); asd->params.s3a_enabled_pipe = pipe_id; @@ -1603,7 +1551,7 @@ int atomisp_alloc_3a_output_buf(struct atomisp_sub_device *asd) int atomisp_alloc_dis_coef_buf(struct atomisp_sub_device *asd) { - struct atomisp_css_dvs_grid_info *dvs_grid = + struct ia_css_dvs_grid_info *dvs_grid = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); if (!dvs_grid) @@ -1693,7 +1641,7 @@ void atomisp_css_get_dis_statistics(struct atomisp_sub_device *asd, int atomisp_css_dequeue_event(struct atomisp_css_event *current_event) { - if (ia_css_dequeue_event(¤t_event->event) != IA_CSS_SUCCESS) + if (ia_css_dequeue_event(¤t_event->event)) return -EINVAL; return 0; @@ -1753,7 +1701,7 @@ void atomisp_css_input_set_binning_factor(struct atomisp_sub_device *asd, void atomisp_css_input_set_bayer_order(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_bayer_order bayer_order) + enum ia_css_bayer_order bayer_order) { struct ia_css_stream_config *s_config = &asd->stream_env[stream_id].stream_config; @@ -1959,7 +1907,7 @@ void atomisp_css_enable_dz(struct atomisp_sub_device *asd, bool enable) } void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd, - enum atomisp_css_capture_mode mode) + enum ia_css_capture_mode mode) { struct atomisp_stream_env *stream_env = &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]; @@ -1974,7 +1922,7 @@ void atomisp_css_capture_set_mode(struct atomisp_sub_device *asd, } void atomisp_css_input_set_mode(struct atomisp_sub_device *asd, - enum atomisp_css_input_mode mode) + enum ia_css_input_mode mode) { int i; struct atomisp_device *isp = asd->isp; @@ -2017,7 +1965,7 @@ void atomisp_css_input_set_mode(struct atomisp_sub_device *asd, s_config->input_config.format, true, 0x13000, - &size_mem_words) != IA_CSS_SUCCESS) { + &size_mem_words) != 0) { if (intel_mid_identify_cpu() == INTEL_MID_CPU_CHIP_TANGIER) size_mem_words = CSS_MIPI_FRAME_BUFFER_SIZE_2; @@ -2157,74 +2105,8 @@ int atomisp_css_input_configure_port( return 0; } -int atomisp_css_frame_allocate(struct atomisp_css_frame **frame, - unsigned int width, unsigned int height, - enum atomisp_css_frame_format format, - unsigned int padded_width, - unsigned int raw_bit_depth) -{ - if (ia_css_frame_allocate(frame, width, height, format, - padded_width, raw_bit_depth) != IA_CSS_SUCCESS) - return -ENOMEM; - - return 0; -} - -int atomisp_css_frame_allocate_from_info(struct atomisp_css_frame **frame, - const struct atomisp_css_frame_info *info) -{ - if (ia_css_frame_allocate_from_info(frame, info) != IA_CSS_SUCCESS) - return -ENOMEM; - - return 0; -} - -void atomisp_css_frame_free(struct atomisp_css_frame *frame) -{ - ia_css_frame_free(frame); -} - -int atomisp_css_frame_map(struct atomisp_css_frame **frame, - const struct atomisp_css_frame_info *info, - const void __user *data, uint16_t attribute, - void *context) -{ - if (ia_css_frame_map(frame, info, data, attribute, context) - != IA_CSS_SUCCESS) - return -ENOMEM; - - return 0; -} - -int atomisp_css_set_black_frame(struct atomisp_sub_device *asd, - const struct atomisp_css_frame *raw_black_frame) -{ - if (sh_css_set_black_frame( - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, - raw_black_frame) != IA_CSS_SUCCESS) - return -ENOMEM; - - return 0; -} - -int atomisp_css_allocate_continuous_frames(bool init_time, - struct atomisp_sub_device *asd) -{ - if (ia_css_alloc_continuous_frame_remain( - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream) - != IA_CSS_SUCCESS) - return -EINVAL; - return 0; -} - -void atomisp_css_update_continuous_frames(struct atomisp_sub_device *asd) -{ - ia_css_update_continuous_frames( - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream); -} - int atomisp_css_stop(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, bool in_reset) + enum ia_css_pipe_id pipe_id, bool in_reset) { struct atomisp_device *isp = asd->isp; struct atomisp_s3a_buf *s3a_buf; @@ -2264,7 +2146,7 @@ int atomisp_css_stop(struct atomisp_sub_device *asd, ia_css_stream_config_defaults( &stream_env->stream_config); } - atomisp_isp_parameters_clean_up(&asd->params.config); + memset(&asd->params.config, 0, sizeof(asd->params.config)); asd->params.css_update_params_needed = false; } @@ -2345,16 +2227,6 @@ int atomisp_css_continuous_set_num_raw_frames( return 0; } -void atomisp_css_disable_vf_pp(struct atomisp_sub_device *asd, - bool disable) -{ - int i; - - for (i = 0; i < IA_CSS_PIPE_ID_NUM; i++) - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] - .pipe_extra_configs[i].disable_vf_pp = !!disable; -} - static enum ia_css_pipe_mode __pipe_id_to_pipe_mode( struct atomisp_sub_device *asd, enum ia_css_pipe_id pipe_id) @@ -2749,7 +2621,7 @@ done: static void __configure_vf_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format, + enum ia_css_frame_format format, enum ia_css_pipe_id pipe_id) { struct atomisp_device *isp = asd->isp; @@ -2772,7 +2644,7 @@ static void __configure_vf_output(struct atomisp_sub_device *asd, static void __configure_video_vf_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format, + enum ia_css_frame_format format, enum ia_css_pipe_id pipe_id) { struct atomisp_device *isp = asd->isp; @@ -2807,12 +2679,12 @@ static void __configure_video_vf_output(struct atomisp_sub_device *asd, static int __get_frame_info(struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info, + struct ia_css_frame_info *info, enum frame_info_type type, enum ia_css_pipe_id pipe_id) { struct atomisp_device *isp = asd->isp; - enum ia_css_err ret; + int ret; struct ia_css_pipe_info p_info; /* FIXME! No need to destroy/recreate all streams */ @@ -2831,7 +2703,7 @@ static int __get_frame_info(struct atomisp_sub_device *asd, ret = ia_css_pipe_get_info( asd->stream_env[stream_index] .pipes[pipe_id], &p_info); - if (ret == IA_CSS_SUCCESS) { + if (!ret) { switch (type) { case ATOMISP_CSS_VF_FRAME: *info = p_info.vf_output_info[0]; @@ -2882,16 +2754,18 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd, if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO || asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) return IA_CSS_PIPE_ID_VIDEO; - else - return IA_CSS_PIPE_ID_CAPTURE; + + return IA_CSS_PIPE_ID_CAPTURE; case ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE: if (asd->copy_mode) return IA_CSS_PIPE_ID_COPY; + return IA_CSS_PIPE_ID_CAPTURE; case ATOMISP_SUBDEV_PAD_SOURCE_VF: - if (!atomisp_is_mbuscode_raw( - asd->fmt[asd->capture_pad].fmt.code)) + if (!atomisp_is_mbuscode_raw(asd->fmt[asd->capture_pad].fmt.code)) { return IA_CSS_PIPE_ID_CAPTURE; + } + /* fall through */ case ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW: if (asd->yuvpp_mode) return IA_CSS_PIPE_ID_YUVPP; @@ -2899,8 +2773,8 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd, return IA_CSS_PIPE_ID_COPY; if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) return IA_CSS_PIPE_ID_VIDEO; - else - return IA_CSS_PIPE_ID_PREVIEW; + + return IA_CSS_PIPE_ID_PREVIEW; } dev_warn(isp->dev, "invalid source pad:%d, return default preview pipe index.\n", @@ -2910,7 +2784,7 @@ static unsigned int atomisp_get_pipe_index(struct atomisp_sub_device *asd, int atomisp_get_css_frame_info(struct atomisp_sub_device *asd, u16 source_pad, - struct atomisp_css_frame_info *frame_info) + struct ia_css_frame_info *frame_info) { struct ia_css_pipe_info info; int pipe_index = atomisp_get_pipe_index(asd, source_pad); @@ -2925,7 +2799,7 @@ int atomisp_get_css_frame_info(struct atomisp_sub_device *asd, atomisp_source_pad_to_stream_id(asd, source_pad); } - if (IA_CSS_SUCCESS != ia_css_pipe_get_info(asd->stream_env[stream_index] + if (0 != ia_css_pipe_get_info(asd->stream_env[stream_index] .pipes[pipe_index], &info)) { dev_err(isp->dev, "ia_css_pipe_get_info FAILED"); return -EINVAL; @@ -2978,11 +2852,11 @@ int atomisp_css_copy_configure_output(struct atomisp_sub_device *asd, unsigned int stream_index, unsigned int width, unsigned int height, unsigned int padded_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_COPY]. default_capture_config.mode = - CSS_CAPTURE_MODE_RAW; + IA_CSS_CAPTURE_MODE_RAW; __configure_output(asd, stream_index, width, height, padded_width, format, IA_CSS_PIPE_ID_COPY); @@ -2993,11 +2867,11 @@ int atomisp_css_yuvpp_configure_output(struct atomisp_sub_device *asd, unsigned int stream_index, unsigned int width, unsigned int height, unsigned int padded_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { asd->stream_env[stream_index].pipe_configs[IA_CSS_PIPE_ID_YUVPP]. default_capture_config.mode = - CSS_CAPTURE_MODE_RAW; + IA_CSS_CAPTURE_MODE_RAW; __configure_output(asd, stream_index, width, height, padded_width, format, IA_CSS_PIPE_ID_YUVPP); @@ -3009,7 +2883,7 @@ int atomisp_css_yuvpp_configure_viewfinder( unsigned int stream_index, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { struct atomisp_stream_env *stream_env = &asd->stream_env[stream_index]; @@ -3030,7 +2904,7 @@ int atomisp_css_yuvpp_configure_viewfinder( int atomisp_css_yuvpp_get_output_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { return __get_frame_info(asd, stream_index, info, ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_YUVPP); @@ -3039,7 +2913,7 @@ int atomisp_css_yuvpp_get_output_frame_info( int atomisp_css_yuvpp_get_viewfinder_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { return __get_frame_info(asd, stream_index, info, ATOMISP_CSS_VF_FRAME, IA_CSS_PIPE_ID_YUVPP); @@ -3048,7 +2922,7 @@ int atomisp_css_yuvpp_get_viewfinder_frame_info( int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { /* * to SOC camera, use yuvpp pipe. @@ -3066,7 +2940,7 @@ int atomisp_css_preview_configure_output(struct atomisp_sub_device *asd, int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { enum ia_css_pipe_id pipe_id; @@ -3086,7 +2960,7 @@ int atomisp_css_capture_configure_output(struct atomisp_sub_device *asd, int atomisp_css_video_configure_output(struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { /* * to SOC camera, use yuvpp pipe. @@ -3105,7 +2979,7 @@ int atomisp_css_video_configure_viewfinder( struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { /* * to SOC camera, video will use yuvpp pipe. @@ -3123,7 +2997,7 @@ int atomisp_css_capture_configure_viewfinder( struct atomisp_sub_device *asd, unsigned int width, unsigned int height, unsigned int min_width, - enum atomisp_css_frame_format format) + enum ia_css_frame_format format) { enum ia_css_pipe_id pipe_id; @@ -3142,7 +3016,7 @@ int atomisp_css_capture_configure_viewfinder( int atomisp_css_video_get_viewfinder_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { enum ia_css_pipe_id pipe_id; enum frame_info_type frame_type = ATOMISP_CSS_VF_FRAME; @@ -3161,7 +3035,7 @@ int atomisp_css_video_get_viewfinder_frame_info( int atomisp_css_capture_get_viewfinder_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { enum ia_css_pipe_id pipe_id; @@ -3176,7 +3050,7 @@ int atomisp_css_capture_get_viewfinder_frame_info( int atomisp_css_capture_get_output_raw_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { if (ATOMISP_USE_YUVPP(asd)) return 0; @@ -3188,7 +3062,7 @@ int atomisp_css_capture_get_output_raw_frame_info( int atomisp_css_copy_get_output_frame_info( struct atomisp_sub_device *asd, unsigned int stream_index, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { return __get_frame_info(asd, stream_index, info, ATOMISP_CSS_OUTPUT_FRAME, IA_CSS_PIPE_ID_COPY); @@ -3196,7 +3070,7 @@ int atomisp_css_copy_get_output_frame_info( int atomisp_css_preview_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { enum ia_css_pipe_id pipe_id; enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME; @@ -3215,7 +3089,7 @@ int atomisp_css_preview_get_output_frame_info( int atomisp_css_capture_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { enum ia_css_pipe_id pipe_id; @@ -3230,7 +3104,7 @@ int atomisp_css_capture_get_output_frame_info( int atomisp_css_video_get_output_frame_info( struct atomisp_sub_device *asd, - struct atomisp_css_frame_info *info) + struct ia_css_frame_info *info) { enum ia_css_pipe_id pipe_id; enum frame_info_type frame_type = ATOMISP_CSS_OUTPUT_FRAME; @@ -3297,7 +3171,7 @@ int atomisp_css_video_configure_pp_input( int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd, int num_captures, unsigned int skip, int offset) { - enum ia_css_err ret; + int ret; dev_dbg(asd->isp->dev, "%s num_capture:%d skip:%d offset:%d\n", __func__, num_captures, skip, offset); @@ -3305,7 +3179,7 @@ int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd, ret = ia_css_stream_capture( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, num_captures, skip, offset); - if (ret != IA_CSS_SUCCESS) + if (ret) return -EINVAL; return 0; @@ -3313,15 +3187,15 @@ int atomisp_css_offline_capture_configure(struct atomisp_sub_device *asd, int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id) { - enum ia_css_err ret; + int ret; ret = ia_css_stream_capture_frame( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, exp_id); - if (ret == IA_CSS_ERR_QUEUE_IS_FULL) { + if (ret == -ENOBUFS) { /* capture cmd queue is full */ return -EBUSY; - } else if (ret != IA_CSS_SUCCESS) { + } else if (ret) { return -EIO; } @@ -3330,14 +3204,14 @@ int atomisp_css_exp_id_capture(struct atomisp_sub_device *asd, int exp_id) int atomisp_css_exp_id_unlock(struct atomisp_sub_device *asd, int exp_id) { - enum ia_css_err ret; + int ret; ret = ia_css_unlock_raw_frame( asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, exp_id); - if (ret == IA_CSS_ERR_QUEUE_IS_FULL) + if (ret == -ENOBUFS) return -EAGAIN; - else if (ret != IA_CSS_SUCCESS) + else if (ret) return -EIO; return 0; @@ -3356,201 +3230,8 @@ int atomisp_css_capture_enable_xnr(struct atomisp_sub_device *asd, return 0; } -void atomisp_css_send_input_frame(struct atomisp_sub_device *asd, - unsigned short *data, unsigned int width, - unsigned int height) -{ - ia_css_stream_send_input_frame( - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, - data, width, height); -} - -bool atomisp_css_isp_has_started(void) -{ - return ia_css_isp_has_started(); -} - -void atomisp_css_request_flash(struct atomisp_sub_device *asd) -{ - ia_css_stream_request_flash( - asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream); -} - -void atomisp_css_set_wb_config(struct atomisp_sub_device *asd, - struct atomisp_css_wb_config *wb_config) -{ - asd->params.config.wb_config = wb_config; -} - -void atomisp_css_set_ob_config(struct atomisp_sub_device *asd, - struct atomisp_css_ob_config *ob_config) -{ - asd->params.config.ob_config = ob_config; -} - -void atomisp_css_set_dp_config(struct atomisp_sub_device *asd, - struct atomisp_css_dp_config *dp_config) -{ - asd->params.config.dp_config = dp_config; -} - -void atomisp_css_set_de_config(struct atomisp_sub_device *asd, - struct atomisp_css_de_config *de_config) -{ - asd->params.config.de_config = de_config; -} - -void atomisp_css_set_dz_config(struct atomisp_sub_device *asd, - struct atomisp_css_dz_config *dz_config) -{ - asd->params.config.dz_config = dz_config; -} - -void atomisp_css_set_default_de_config(struct atomisp_sub_device *asd) -{ - asd->params.config.de_config = NULL; -} - -void atomisp_css_set_ce_config(struct atomisp_sub_device *asd, - struct atomisp_css_ce_config *ce_config) -{ - asd->params.config.ce_config = ce_config; -} - -void atomisp_css_set_nr_config(struct atomisp_sub_device *asd, - struct atomisp_css_nr_config *nr_config) -{ - asd->params.config.nr_config = nr_config; -} - -void atomisp_css_set_ee_config(struct atomisp_sub_device *asd, - struct atomisp_css_ee_config *ee_config) -{ - asd->params.config.ee_config = ee_config; -} - -void atomisp_css_set_tnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_tnr_config *tnr_config) -{ - asd->params.config.tnr_config = tnr_config; -} - -void atomisp_css_set_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *cc_config) -{ - asd->params.config.cc_config = cc_config; -} - -void atomisp_css_set_macc_table(struct atomisp_sub_device *asd, - struct atomisp_css_macc_table *macc_table) -{ - asd->params.config.macc_table = macc_table; -} - -void atomisp_css_set_macc_config(struct atomisp_sub_device *asd, - struct atomisp_css_macc_config *macc_config) -{ - asd->params.config.macc_config = macc_config; -} - -void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd, - struct atomisp_css_ecd_config *ecd_config) -{ - asd->params.config.ecd_config = ecd_config; -} - -void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd, - struct atomisp_css_ynr_config *ynr_config) -{ - asd->params.config.ynr_config = ynr_config; -} - -void atomisp_css_set_fc_config(struct atomisp_sub_device *asd, - struct atomisp_css_fc_config *fc_config) -{ - asd->params.config.fc_config = fc_config; -} - -void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd, - struct atomisp_css_ctc_config *ctc_config) -{ - asd->params.config.ctc_config = ctc_config; -} - -void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_cnr_config *cnr_config) -{ - asd->params.config.cnr_config = cnr_config; -} - -void atomisp_css_set_aa_config(struct atomisp_sub_device *asd, - struct atomisp_css_aa_config *aa_config) -{ - asd->params.config.aa_config = aa_config; -} - -void atomisp_css_set_baa_config(struct atomisp_sub_device *asd, - struct atomisp_css_baa_config *baa_config) -{ - asd->params.config.baa_config = baa_config; -} - -void atomisp_css_set_anr_config(struct atomisp_sub_device *asd, - struct atomisp_css_anr_config *anr_config) -{ - asd->params.config.anr_config = anr_config; -} - -void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_xnr_config *xnr_config) -{ - asd->params.config.xnr_config = xnr_config; -} - -void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *yuv2rgb_cc_config) -{ - asd->params.config.yuv2rgb_cc_config = yuv2rgb_cc_config; -} - -void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *rgb2yuv_cc_config) -{ - asd->params.config.rgb2yuv_cc_config = rgb2yuv_cc_config; -} - -void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd, - struct atomisp_css_xnr_table *xnr_table) -{ - asd->params.config.xnr_table = xnr_table; -} - -void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *r_gamma_table) -{ - asd->params.config.r_gamma_table = r_gamma_table; -} - -void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *g_gamma_table) -{ - asd->params.config.g_gamma_table = g_gamma_table; -} - -void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *b_gamma_table) -{ - asd->params.config.b_gamma_table = b_gamma_table; -} - -void atomisp_css_set_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_gamma_table *gamma_table) -{ - asd->params.config.gamma_table = gamma_table; -} - void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd, - struct atomisp_css_ctc_table *ctc_table) + struct ia_css_ctc_table *ctc_table) { int i; u16 *vamem_ptr = ctc_table->data.vamem_1; @@ -3577,29 +3258,17 @@ void atomisp_css_set_ctc_table(struct atomisp_sub_device *asd, } void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd, - struct atomisp_css_anr_thres *anr_thres) + struct ia_css_anr_thres *anr_thres) { asd->params.config.anr_thres = anr_thres; } void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd, - struct atomisp_css_dvs_6axis *dvs_6axis) + struct ia_css_dvs_6axis_config *dvs_6axis) { asd->params.config.dvs_6axis_config = dvs_6axis; } -void atomisp_css_set_gc_config(struct atomisp_sub_device *asd, - struct atomisp_css_gc_config *gc_config) -{ - asd->params.config.gc_config = gc_config; -} - -void atomisp_css_set_3a_config(struct atomisp_sub_device *asd, - struct atomisp_css_3a_config *s3a_config) -{ - asd->params.config.s3a_config = s3a_config; -} - void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd, struct atomisp_dis_vector *vector) { @@ -3615,7 +3284,7 @@ void atomisp_css_video_set_dis_vector(struct atomisp_sub_device *asd, static int atomisp_compare_dvs_grid(struct atomisp_sub_device *asd, struct atomisp_dvs_grid_info *atomgrid) { - struct atomisp_css_dvs_grid_info *cur = + struct ia_css_dvs_grid_info *cur = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); if (!cur) { @@ -3696,8 +3365,8 @@ int atomisp_css_set_dis_coefs(struct atomisp_sub_device *asd, return -EFAULT; asd->params.css_param.update_flag.dvs2_coefs = - (struct atomisp_dvs2_coefficients *) - asd->params.css_param.dvs2_coeff; + (struct atomisp_dis_coefficients *) + asd->params.css_param.dvs2_coeff; /* FIXME! */ /* asd->params.dis_proj_data_valid = false; */ asd->params.css_update_params_needed = true; @@ -3727,7 +3396,7 @@ void atomisp_css_set_zoom_factor(struct atomisp_sub_device *asd, } void atomisp_css_set_formats_config(struct atomisp_sub_device *asd, - struct atomisp_css_formats_config *formats_config) + struct ia_css_formats_config *formats_config) { asd->params.config.formats_config = formats_config; } @@ -3735,7 +3404,7 @@ void atomisp_css_set_formats_config(struct atomisp_sub_device *asd, int atomisp_css_get_wb_config(struct atomisp_sub_device *asd, struct atomisp_wb_config *config) { - struct atomisp_css_wb_config wb_config; + struct ia_css_wb_config wb_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3744,7 +3413,7 @@ int atomisp_css_get_wb_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&wb_config, 0, sizeof(struct atomisp_css_wb_config)); + memset(&wb_config, 0, sizeof(struct ia_css_wb_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.wb_config = &wb_config; ia_css_stream_get_isp_config( @@ -3758,7 +3427,7 @@ int atomisp_css_get_wb_config(struct atomisp_sub_device *asd, int atomisp_css_get_ob_config(struct atomisp_sub_device *asd, struct atomisp_ob_config *config) { - struct atomisp_css_ob_config ob_config; + struct ia_css_ob_config ob_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3767,7 +3436,7 @@ int atomisp_css_get_ob_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&ob_config, 0, sizeof(struct atomisp_css_ob_config)); + memset(&ob_config, 0, sizeof(struct ia_css_ob_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.ob_config = &ob_config; ia_css_stream_get_isp_config( @@ -3781,7 +3450,7 @@ int atomisp_css_get_ob_config(struct atomisp_sub_device *asd, int atomisp_css_get_dp_config(struct atomisp_sub_device *asd, struct atomisp_dp_config *config) { - struct atomisp_css_dp_config dp_config; + struct ia_css_dp_config dp_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3790,7 +3459,7 @@ int atomisp_css_get_dp_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&dp_config, 0, sizeof(struct atomisp_css_dp_config)); + memset(&dp_config, 0, sizeof(struct ia_css_dp_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.dp_config = &dp_config; ia_css_stream_get_isp_config( @@ -3804,7 +3473,7 @@ int atomisp_css_get_dp_config(struct atomisp_sub_device *asd, int atomisp_css_get_de_config(struct atomisp_sub_device *asd, struct atomisp_de_config *config) { - struct atomisp_css_de_config de_config; + struct ia_css_de_config de_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3813,7 +3482,7 @@ int atomisp_css_get_de_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&de_config, 0, sizeof(struct atomisp_css_de_config)); + memset(&de_config, 0, sizeof(struct ia_css_de_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.de_config = &de_config; ia_css_stream_get_isp_config( @@ -3827,7 +3496,7 @@ int atomisp_css_get_de_config(struct atomisp_sub_device *asd, int atomisp_css_get_nr_config(struct atomisp_sub_device *asd, struct atomisp_nr_config *config) { - struct atomisp_css_nr_config nr_config; + struct ia_css_nr_config nr_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3836,7 +3505,7 @@ int atomisp_css_get_nr_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&nr_config, 0, sizeof(struct atomisp_css_nr_config)); + memset(&nr_config, 0, sizeof(struct ia_css_nr_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.nr_config = &nr_config; @@ -3851,7 +3520,7 @@ int atomisp_css_get_nr_config(struct atomisp_sub_device *asd, int atomisp_css_get_ee_config(struct atomisp_sub_device *asd, struct atomisp_ee_config *config) { - struct atomisp_css_ee_config ee_config; + struct ia_css_ee_config ee_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3860,7 +3529,7 @@ int atomisp_css_get_ee_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&ee_config, 0, sizeof(struct atomisp_css_ee_config)); + memset(&ee_config, 0, sizeof(struct ia_css_ee_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.ee_config = &ee_config; ia_css_stream_get_isp_config( @@ -3874,7 +3543,7 @@ int atomisp_css_get_ee_config(struct atomisp_sub_device *asd, int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd, struct atomisp_tnr_config *config) { - struct atomisp_css_tnr_config tnr_config; + struct ia_css_tnr_config tnr_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3883,7 +3552,7 @@ int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&tnr_config, 0, sizeof(struct atomisp_css_tnr_config)); + memset(&tnr_config, 0, sizeof(struct ia_css_tnr_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.tnr_config = &tnr_config; ia_css_stream_get_isp_config( @@ -3897,7 +3566,7 @@ int atomisp_css_get_tnr_config(struct atomisp_sub_device *asd, int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd, struct atomisp_ctc_table *config) { - struct atomisp_css_ctc_table *tab; + struct ia_css_ctc_table *tab; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3907,7 +3576,7 @@ int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd, return -EINVAL; } - tab = vzalloc(sizeof(struct atomisp_css_ctc_table)); + tab = vzalloc(sizeof(struct ia_css_ctc_table)); if (!tab) return -ENOMEM; @@ -3925,7 +3594,7 @@ int atomisp_css_get_ctc_table(struct atomisp_sub_device *asd, int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd, struct atomisp_gamma_table *config) { - struct atomisp_css_gamma_table *tab; + struct ia_css_gamma_table *tab; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3935,7 +3604,7 @@ int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd, return -EINVAL; } - tab = vzalloc(sizeof(struct atomisp_css_gamma_table)); + tab = vzalloc(sizeof(struct ia_css_gamma_table)); if (!tab) return -ENOMEM; @@ -3953,7 +3622,7 @@ int atomisp_css_get_gamma_table(struct atomisp_sub_device *asd, int atomisp_css_get_gc_config(struct atomisp_sub_device *asd, struct atomisp_gc_config *config) { - struct atomisp_css_gc_config gc_config; + struct ia_css_gc_config gc_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3962,7 +3631,7 @@ int atomisp_css_get_gc_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&gc_config, 0, sizeof(struct atomisp_css_gc_config)); + memset(&gc_config, 0, sizeof(struct ia_css_gc_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.gc_config = &gc_config; ia_css_stream_get_isp_config( @@ -3977,7 +3646,7 @@ int atomisp_css_get_gc_config(struct atomisp_sub_device *asd, int atomisp_css_get_3a_config(struct atomisp_sub_device *asd, struct atomisp_3a_config *config) { - struct atomisp_css_3a_config s3a_config; + struct ia_css_3a_config s3a_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -3986,7 +3655,7 @@ int atomisp_css_get_3a_config(struct atomisp_sub_device *asd, __func__); return -EINVAL; } - memset(&s3a_config, 0, sizeof(struct atomisp_css_3a_config)); + memset(&s3a_config, 0, sizeof(struct ia_css_3a_config)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.s3a_config = &s3a_config; ia_css_stream_get_isp_config( @@ -4001,7 +3670,7 @@ int atomisp_css_get_3a_config(struct atomisp_sub_device *asd, int atomisp_css_get_formats_config(struct atomisp_sub_device *asd, struct atomisp_formats_config *config) { - struct atomisp_css_formats_config formats_config; + struct ia_css_formats_config formats_config; struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -4139,37 +3808,37 @@ int atomisp_css_get_dis_stat(struct atomisp_sub_device *asd, return 0; } -struct atomisp_css_shading_table *atomisp_css_shading_table_alloc( +struct ia_css_shading_table *atomisp_css_shading_table_alloc( unsigned int width, unsigned int height) { return ia_css_shading_table_alloc(width, height); } void atomisp_css_set_shading_table(struct atomisp_sub_device *asd, - struct atomisp_css_shading_table *table) + struct ia_css_shading_table *table) { asd->params.config.shading_table = table; } -void atomisp_css_shading_table_free(struct atomisp_css_shading_table *table) +void atomisp_css_shading_table_free(struct ia_css_shading_table *table) { ia_css_shading_table_free(table); } -struct atomisp_css_morph_table *atomisp_css_morph_table_allocate( +struct ia_css_morph_table *atomisp_css_morph_table_allocate( unsigned int width, unsigned int height) { return ia_css_morph_table_allocate(width, height); } void atomisp_css_set_morph_table(struct atomisp_sub_device *asd, - struct atomisp_css_morph_table *table) + struct ia_css_morph_table *table) { asd->params.config.morph_table = table; } void atomisp_css_get_morph_table(struct atomisp_sub_device *asd, - struct atomisp_css_morph_table *table) + struct ia_css_morph_table *table) { struct ia_css_isp_config isp_config; struct atomisp_device *isp = asd->isp; @@ -4179,7 +3848,7 @@ void atomisp_css_get_morph_table(struct atomisp_sub_device *asd, "%s called after streamoff, skipping.\n", __func__); return; } - memset(table, 0, sizeof(struct atomisp_css_morph_table)); + memset(table, 0, sizeof(struct ia_css_morph_table)); memset(&isp_config, 0, sizeof(struct ia_css_isp_config)); isp_config.morph_table = table; ia_css_stream_get_isp_config( @@ -4187,7 +3856,7 @@ void atomisp_css_get_morph_table(struct atomisp_sub_device *asd, &isp_config); } -void atomisp_css_morph_table_free(struct atomisp_css_morph_table *table) +void atomisp_css_morph_table_free(struct ia_css_morph_table *table) { ia_css_morph_table_free(table); } @@ -4215,8 +3884,8 @@ int atomisp_css_wait_acc_finish(struct atomisp_sub_device *asd) if (wait_for_completion_interruptible_timeout(&asd->acc.acc_done, ATOMISP_ISP_TIMEOUT_DURATION) == 0) { dev_err(isp->dev, "<%s: completion timeout\n", __func__); - atomisp_css_debug_dump_sp_sw_debug_info(); - atomisp_css_debug_dump_debug_info(__func__); + ia_css_debug_dump_sp_sw_debug_info(); + ia_css_debug_dump_debug_info(__func__); ret = -EIO; } rt_mutex_lock(&isp->mutex); @@ -4244,11 +3913,11 @@ int atomisp_css_set_acc_parameters(struct atomisp_acc_fw *acc_fw) /* Load acc binary extension */ int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, - enum atomisp_css_pipe_id pipe_id, + struct ia_css_fw_info *fw, + enum ia_css_pipe_id pipe_id, unsigned int type) { - struct atomisp_css_fw_info **hd; + struct ia_css_fw_info **hd; fw->next = NULL; hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] @@ -4264,10 +3933,10 @@ int atomisp_css_load_acc_extension(struct atomisp_sub_device *asd, /* Unload acc binary extension */ void atomisp_css_unload_acc_extension(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, - enum atomisp_css_pipe_id pipe_id) + struct ia_css_fw_info *fw, + enum ia_css_pipe_id pipe_id) { - struct atomisp_css_fw_info **hd; + struct ia_css_fw_info **hd; hd = &(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL] .pipe_configs[pipe_id].acc_extension); @@ -4294,21 +3963,21 @@ int atomisp_css_create_acc_pipe(struct atomisp_sub_device *asd) if (stream_env->acc_stream) { if (stream_env->acc_stream_state == CSS_STREAM_STARTED) { if (ia_css_stream_stop(stream_env->acc_stream) - != IA_CSS_SUCCESS) { + != 0) { dev_err(isp->dev, "stop acc_stream failed.\n"); return -EBUSY; } } if (ia_css_stream_destroy(stream_env->acc_stream) - != IA_CSS_SUCCESS) { + != 0) { dev_err(isp->dev, "destroy acc_stream failed.\n"); return -EBUSY; } stream_env->acc_stream = NULL; } - pipe_config = &stream_env->pipe_configs[CSS_PIPE_ID_ACC]; + pipe_config = &stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC]; ia_css_pipe_config_defaults(pipe_config); asd->acc.acc_stages = kzalloc(MAX_ACC_STAGES * sizeof(void *), GFP_KERNEL); @@ -4335,7 +4004,7 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd) &stream_env->pipe_configs[IA_CSS_PIPE_ID_ACC]; if (ia_css_pipe_create(pipe_config, - &stream_env->pipes[IA_CSS_PIPE_ID_ACC]) != IA_CSS_SUCCESS) { + &stream_env->pipes[IA_CSS_PIPE_ID_ACC]) != 0) { dev_err(isp->dev, "%s: ia_css_pipe_create failed\n", __func__); return -EBADE; @@ -4345,7 +4014,7 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd) sizeof(struct ia_css_stream_config)); if (ia_css_stream_create(&stream_env->acc_stream_config, 1, &stream_env->pipes[IA_CSS_PIPE_ID_ACC], - &stream_env->acc_stream) != IA_CSS_SUCCESS) { + &stream_env->acc_stream) != 0) { dev_err(isp->dev, "%s: create acc_stream error.\n", __func__); return -EINVAL; } @@ -4356,13 +4025,13 @@ int atomisp_css_start_acc_pipe(struct atomisp_sub_device *asd) atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false); - if (ia_css_start_sp() != IA_CSS_SUCCESS) { + if (ia_css_start_sp()) { dev_err(isp->dev, "start sp error.\n"); return -EIO; } if (ia_css_stream_start(stream_env->acc_stream) - != IA_CSS_SUCCESS) { + != 0) { dev_err(isp->dev, "acc_stream start error.\n"); return -EIO; } @@ -4388,7 +4057,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd) &asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL]; if (stream_env->acc_stream) { if (ia_css_stream_destroy(stream_env->acc_stream) - != IA_CSS_SUCCESS) + != 0) dev_warn(asd->isp->dev, "destroy acc_stream failed.\n"); stream_env->acc_stream = NULL; @@ -4396,7 +4065,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd) if (stream_env->pipes[IA_CSS_PIPE_ID_ACC]) { if (ia_css_pipe_destroy(stream_env->pipes[IA_CSS_PIPE_ID_ACC]) - != IA_CSS_SUCCESS) + != 0) dev_warn(asd->isp->dev, "destroy ACC pipe failed.\n"); stream_env->pipes[IA_CSS_PIPE_ID_ACC] = NULL; @@ -4420,7 +4089,7 @@ void atomisp_css_destroy_acc_pipe(struct atomisp_sub_device *asd) } int atomisp_css_load_acc_binary(struct atomisp_sub_device *asd, - struct atomisp_css_fw_info *fw, + struct ia_css_fw_info *fw, unsigned int index) { struct ia_css_pipe_config *pipe_config = @@ -4494,7 +4163,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp, for (i = 0; i < isp->num_of_streams; i++) atomisp_wdt_stop(&isp->asd[i], 0); - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) atomisp_wdt(&isp->asd[0].wdt); else queue_work(isp->wdt_work_queue, &isp->wdt_work); @@ -4510,7 +4179,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp, asd = __get_atomisp_subdev(current_event.event.pipe, isp, &stream_id); if (!asd) { - if (current_event.event.type == CSS_EVENT_TIMER) + if (current_event.event.type == IA_CSS_EVENT_TYPE_TIMER) dev_dbg(isp->dev, "event: Timer event."); else @@ -4522,63 +4191,72 @@ int atomisp_css_isr_thread(struct atomisp_device *isp, atomisp_css_temp_pipe_to_pipe_id(asd, ¤t_event); switch (current_event.event.type) { - case CSS_EVENT_OUTPUT_FRAME_DONE: + case IA_CSS_EVENT_TYPE_OUTPUT_FRAME_DONE: + dev_dbg(isp->dev, "event: Output frame done"); frame_done_found[asd->index] = true; - atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_OUTPUT_FRAME, + atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_OUTPUT_FRAME, current_event.pipe, true, stream_id); - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) reset_wdt_timer[asd->index] = true; /* ISP running */ break; - case CSS_EVENT_SEC_OUTPUT_FRAME_DONE: + case IA_CSS_EVENT_TYPE_SECOND_OUTPUT_FRAME_DONE: + dev_dbg(isp->dev, "event: Second output frame done"); frame_done_found[asd->index] = true; - atomisp_buf_done(asd, 0, CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME, + atomisp_buf_done(asd, 0, IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME, current_event.pipe, true, stream_id); - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) reset_wdt_timer[asd->index] = true; /* ISP running */ break; - case CSS_EVENT_3A_STATISTICS_DONE: + case IA_CSS_EVENT_TYPE_3A_STATISTICS_DONE: + dev_dbg(isp->dev, "event: 3A stats frame done"); atomisp_buf_done(asd, 0, - CSS_BUFFER_TYPE_3A_STATISTICS, + IA_CSS_BUFFER_TYPE_3A_STATISTICS, current_event.pipe, false, stream_id); break; - case CSS_EVENT_METADATA_DONE: + case IA_CSS_EVENT_TYPE_METADATA_DONE: + dev_dbg(isp->dev, "event: metadata frame done"); atomisp_buf_done(asd, 0, - CSS_BUFFER_TYPE_METADATA, + IA_CSS_BUFFER_TYPE_METADATA, current_event.pipe, false, stream_id); break; - case CSS_EVENT_VF_OUTPUT_FRAME_DONE: + case IA_CSS_EVENT_TYPE_VF_OUTPUT_FRAME_DONE: + dev_dbg(isp->dev, "event: VF output frame done"); atomisp_buf_done(asd, 0, - CSS_BUFFER_TYPE_VF_OUTPUT_FRAME, + IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME, current_event.pipe, true, stream_id); - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) reset_wdt_timer[asd->index] = true; /* ISP running */ break; - case CSS_EVENT_SEC_VF_OUTPUT_FRAME_DONE: + case IA_CSS_EVENT_TYPE_SECOND_VF_OUTPUT_FRAME_DONE: + dev_dbg(isp->dev, "event: second VF output frame done"); atomisp_buf_done(asd, 0, - CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME, + IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME, current_event.pipe, true, stream_id); - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) reset_wdt_timer[asd->index] = true; /* ISP running */ break; - case CSS_EVENT_DIS_STATISTICS_DONE: + case IA_CSS_EVENT_TYPE_DIS_STATISTICS_DONE: + dev_dbg(isp->dev, "event: dis stats frame done"); atomisp_buf_done(asd, 0, - CSS_BUFFER_TYPE_DIS_STATISTICS, + IA_CSS_BUFFER_TYPE_DIS_STATISTICS, current_event.pipe, false, stream_id); break; - case CSS_EVENT_PIPELINE_DONE: + case IA_CSS_EVENT_TYPE_PIPELINE_DONE: + dev_dbg(isp->dev, "event: pipeline done"); css_pipe_done[asd->index] = true; break; - case CSS_EVENT_ACC_STAGE_COMPLETE: + case IA_CSS_EVENT_TYPE_ACC_STAGE_COMPLETE: + dev_dbg(isp->dev, "event: acc stage done"); atomisp_acc_done(asd, current_event.event.fw_handle); break; default: @@ -4588,7 +4266,7 @@ int atomisp_css_isr_thread(struct atomisp_device *isp, } } - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) return 0; /* ISP2400: If there are no buffers queued then delete wdt timer. */ @@ -4618,8 +4296,13 @@ bool atomisp_css_valid_sof(struct atomisp_device *isp) struct atomisp_sub_device *asd = &isp->asd[i]; /* Loop for each css vc stream */ for (j = 0; j < ATOMISP_INPUT_STREAM_NUM; j++) { - if (asd->stream_env[j].stream && - asd->stream_env[j].stream_config.mode == + if (!asd->stream_env[j].stream) + continue; + + dev_dbg(isp->dev, + "stream #%d: mode: %d\n", j, + asd->stream_env[j].stream_config.mode); + if (asd->stream_env[j].stream_config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) return false; } @@ -4640,6 +4323,20 @@ int atomisp_css_dump_sp_raw_copy_linecount(bool reduced) return 0; } +static const char * const fw_type_name[] = { + [ia_css_sp_firmware] = "SP", + [ia_css_isp_firmware] = "ISP", + [ia_css_bootloader_firmware] = "BootLoader", + [ia_css_acc_firmware] = "accel", +}; + +static const char * const fw_acc_type_name[] = { + [IA_CSS_ACC_NONE] = "Normal", + [IA_CSS_ACC_OUTPUT] = "Accel stage on output", + [IA_CSS_ACC_VIEWFINDER] = "Accel stage on viewfinder", + [IA_CSS_ACC_STANDALONE] = "Stand-alone acceleration", +}; + int atomisp_css_dump_blob_infor(void) { struct ia_css_blob_descr *bd = sh_css_blob_info; @@ -4650,9 +4347,28 @@ int atomisp_css_dump_blob_infor(void) if (!bd) return -EPERM; - for (i = 1; i < sh_css_num_binaries; i++) - dev_dbg(atomisp_dev, "Num%d binary id is %d, name is %s\n", i, - bd[i - 1].header.info.isp.sp.id, bd[i - 1].name); + /* + * The sh_css_load_firmware function discard the initial + * "SPS" binaries + */ + for (i = 0; i < sh_css_num_binaries - NUM_OF_SPS; i++) { + switch (bd[i].header.type) { + case ia_css_isp_firmware: + dev_dbg(atomisp_dev, + "Num%2d type %s (%s), binary id is %2d, name is %s\n", + i + NUM_OF_SPS, + fw_type_name[bd[i].header.type], + fw_acc_type_name[bd[i].header.info.isp.type], + bd[i].header.info.isp.sp.id, + bd[i].name); + break; + default: + dev_dbg(atomisp_dev, + "Num%2d type %s, name is %s\n", + i + NUM_OF_SPS, fw_type_name[bd[i].header.type], + bd[i].name); + } + } return 0; } @@ -4664,7 +4380,7 @@ void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd, } void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd, - struct atomisp_css_frame *output_frame) + struct ia_css_frame *output_frame) { asd->params.config.output_frame = output_frame; } @@ -4692,8 +4408,8 @@ void atomisp_en_dz_capt_pipe(struct atomisp_sub_device *asd, bool enable) enable); } -struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info( - struct atomisp_css_grid_info *grid_info) +struct ia_css_dvs_grid_info *atomisp_css_get_dvs_grid_info( + struct ia_css_grid_info *grid_info) { if (!grid_info) return NULL; diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h index 7abd1ff35652..8376aec18e3e 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h +++ b/drivers/staging/media/atomisp/pci/atomisp_compat_css20.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Clovertrail PNW Camera Imaging ISP subsystem. * @@ -30,97 +31,6 @@ #define ATOMISP_CSS2_NUM_OFFLINE_INIT_CONTINUOUS_FRAMES_LOCK_EN 4 #define ATOMISP_CSS2_NUM_DVS_FRAME_DELAY 2 -#define atomisp_css_pipe_id ia_css_pipe_id -#define atomisp_css_pipeline ia_css_pipe -#define atomisp_css_buffer_type ia_css_buffer_type -#define atomisp_css_dis_data ia_css_isp_dvs_statistics -#define atomisp_css_irq_info ia_css_irq_info -#define atomisp_css_isp_config ia_css_isp_config -#define atomisp_css_bayer_order ia_css_bayer_order -#define atomisp_css_capture_mode ia_css_capture_mode -#define atomisp_css_input_mode ia_css_input_mode -#define atomisp_css_frame ia_css_frame -#define atomisp_css_frame_format ia_css_frame_format -#define atomisp_css_frame_info ia_css_frame_info -#define atomisp_css_dp_config ia_css_dp_config -#define atomisp_css_wb_config ia_css_wb_config -#define atomisp_css_cc_config ia_css_cc_config -#define atomisp_css_nr_config ia_css_nr_config -#define atomisp_css_ee_config ia_css_ee_config -#define atomisp_css_ob_config ia_css_ob_config -#define atomisp_css_de_config ia_css_de_config -#define atomisp_css_dz_config ia_css_dz_config -#define atomisp_css_ce_config ia_css_ce_config -#define atomisp_css_gc_config ia_css_gc_config -#define atomisp_css_tnr_config ia_css_tnr_config -#define atomisp_css_cnr_config ia_css_cnr_config -#define atomisp_css_ctc_config ia_css_ctc_config -#define atomisp_css_3a_config ia_css_3a_config -#define atomisp_css_ecd_config ia_css_ecd_config -#define atomisp_css_ynr_config ia_css_ynr_config -#define atomisp_css_fc_config ia_css_fc_config -#define atomisp_css_aa_config ia_css_aa_config -#define atomisp_css_baa_config ia_css_aa_config -#define atomisp_css_anr_config ia_css_anr_config -#define atomisp_css_xnr_config ia_css_xnr_config -#define atomisp_css_macc_config ia_css_macc_config -#define atomisp_css_gamma_table ia_css_gamma_table -#define atomisp_css_ctc_table ia_css_ctc_table -#define atomisp_css_macc_table ia_css_macc_table -#define atomisp_css_xnr_table ia_css_xnr_table -#define atomisp_css_rgb_gamma_table ia_css_rgb_gamma_table -#define atomisp_css_anr_thres ia_css_anr_thres -#define atomisp_css_dvs_6axis ia_css_dvs_6axis_config -#define atomisp_css_grid_info ia_css_grid_info -#define atomisp_css_3a_grid_info ia_css_3a_grid_info -#define atomisp_css_dvs_grid_info ia_css_dvs_grid_info -#define atomisp_css_shading_table ia_css_shading_table -#define atomisp_css_morph_table ia_css_morph_table -#define atomisp_css_dvs_6axis_config ia_css_dvs_6axis_config -#define atomisp_css_fw_info ia_css_fw_info -#define atomisp_css_formats_config ia_css_formats_config - -#define CSS_PIPE_ID_PREVIEW IA_CSS_PIPE_ID_PREVIEW -#define CSS_PIPE_ID_COPY IA_CSS_PIPE_ID_COPY -#define CSS_PIPE_ID_VIDEO IA_CSS_PIPE_ID_VIDEO -#define CSS_PIPE_ID_CAPTURE IA_CSS_PIPE_ID_CAPTURE -#define CSS_PIPE_ID_ACC IA_CSS_PIPE_ID_ACC -#define CSS_PIPE_ID_YUVPP IA_CSS_PIPE_ID_YUVPP -#define CSS_PIPE_ID_NUM IA_CSS_PIPE_ID_NUM - -#define CSS_INPUT_MODE_SENSOR IA_CSS_INPUT_MODE_BUFFERED_SENSOR -#define CSS_INPUT_MODE_FIFO IA_CSS_INPUT_MODE_FIFO -#define CSS_INPUT_MODE_TPG IA_CSS_INPUT_MODE_TPG -#define CSS_INPUT_MODE_PRBS IA_CSS_INPUT_MODE_PRBS -#define CSS_INPUT_MODE_MEMORY IA_CSS_INPUT_MODE_MEMORY - -#define CSS_IRQ_INFO_CSS_RECEIVER_ERROR IA_CSS_IRQ_INFO_CSS_RECEIVER_ERROR -#define CSS_IRQ_INFO_EVENTS_READY IA_CSS_IRQ_INFO_EVENTS_READY -#define CSS_IRQ_INFO_INPUT_SYSTEM_ERROR \ - IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR -#define CSS_IRQ_INFO_IF_ERROR IA_CSS_IRQ_INFO_IF_ERROR - -#define CSS_BUFFER_TYPE_NUM IA_CSS_BUFFER_TYPE_NUM - -#define CSS_FRAME_FLASH_STATE_NONE IA_CSS_FRAME_FLASH_STATE_NONE -#define CSS_FRAME_FLASH_STATE_PARTIAL IA_CSS_FRAME_FLASH_STATE_PARTIAL -#define CSS_FRAME_FLASH_STATE_FULL IA_CSS_FRAME_FLASH_STATE_FULL - -#define CSS_BAYER_ORDER_GRBG IA_CSS_BAYER_ORDER_GRBG -#define CSS_BAYER_ORDER_RGGB IA_CSS_BAYER_ORDER_RGGB -#define CSS_BAYER_ORDER_BGGR IA_CSS_BAYER_ORDER_BGGR -#define CSS_BAYER_ORDER_GBRG IA_CSS_BAYER_ORDER_GBRG - -/* - * Hide IA_ naming difference in otherwise common CSS macros. - */ -#define CSS_ID(val) (IA_ ## val) -#define CSS_EVENT(val) (IA_CSS_EVENT_TYPE_ ## val) -#define CSS_FORMAT(val) (ATOMISP_INPUT_FORMAT_ ## val) - -#define CSS_EVENT_PORT_EOF CSS_EVENT(PORT_EOF) -#define CSS_EVENT_FRAME_TAGGED CSS_EVENT(FRAME_TAGGED) - #define CSS_MIPI_FRAME_BUFFER_SIZE_1 0x60000 #define CSS_MIPI_FRAME_BUFFER_SIZE_2 0x80000 @@ -181,7 +91,7 @@ struct atomisp_s3a_buf { }; struct atomisp_dis_buf { - struct atomisp_css_dis_data *dis_data; + struct ia_css_isp_dvs_statistics *dis_data; struct ia_css_isp_dvs_statistics_map *dvs_map; struct list_head list; }; @@ -191,71 +101,53 @@ struct atomisp_css_buffer { }; struct atomisp_css_event { - enum atomisp_css_pipe_id pipe; + enum ia_css_pipe_id pipe; struct ia_css_event event; }; void atomisp_css_set_macc_config(struct atomisp_sub_device *asd, - struct atomisp_css_macc_config *macc_config); + struct ia_css_macc_config *macc_config); void atomisp_css_set_ecd_config(struct atomisp_sub_device *asd, - struct atomisp_css_ecd_config *ecd_config); + struct ia_css_ecd_config *ecd_config); void atomisp_css_set_ynr_config(struct atomisp_sub_device *asd, - struct atomisp_css_ynr_config *ynr_config); + struct ia_css_ynr_config *ynr_config); void atomisp_css_set_fc_config(struct atomisp_sub_device *asd, - struct atomisp_css_fc_config *fc_config); + struct ia_css_fc_config *fc_config); void atomisp_css_set_aa_config(struct atomisp_sub_device *asd, - struct atomisp_css_aa_config *aa_config); + struct ia_css_aa_config *aa_config); void atomisp_css_set_baa_config(struct atomisp_sub_device *asd, - struct atomisp_css_baa_config *baa_config); + struct ia_css_aa_config *baa_config); void atomisp_css_set_anr_config(struct atomisp_sub_device *asd, - struct atomisp_css_anr_config *anr_config); + struct ia_css_anr_config *anr_config); void atomisp_css_set_xnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_xnr_config *xnr_config); + struct ia_css_xnr_config *xnr_config); void atomisp_css_set_cnr_config(struct atomisp_sub_device *asd, - struct atomisp_css_cnr_config *cnr_config); + struct ia_css_cnr_config *cnr_config); void atomisp_css_set_ctc_config(struct atomisp_sub_device *asd, - struct atomisp_css_ctc_config *ctc_config); + struct ia_css_ctc_config *ctc_config); void atomisp_css_set_yuv2rgb_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *yuv2rgb_cc_config); + struct ia_css_cc_config *yuv2rgb_cc_config); void atomisp_css_set_rgb2yuv_cc_config(struct atomisp_sub_device *asd, - struct atomisp_css_cc_config *rgb2yuv_cc_config); - -void atomisp_css_set_xnr_table(struct atomisp_sub_device *asd, - struct atomisp_css_xnr_table *xnr_table); - -void atomisp_css_set_r_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *r_gamma_table); - -void atomisp_css_set_g_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *g_gamma_table); - -void atomisp_css_set_b_gamma_table(struct atomisp_sub_device *asd, - struct atomisp_css_rgb_gamma_table *b_gamma_table); + struct ia_css_cc_config *rgb2yuv_cc_config); void atomisp_css_set_anr_thres(struct atomisp_sub_device *asd, - struct atomisp_css_anr_thres *anr_thres); - -int atomisp_css_check_firmware_version(struct atomisp_device *isp); + struct ia_css_anr_thres *anr_thres); int atomisp_css_load_firmware(struct atomisp_device *isp); -void atomisp_css_unload_firmware(struct atomisp_device *isp); - void atomisp_css_set_dvs_6axis(struct atomisp_sub_device *asd, - struct atomisp_css_dvs_6axis *dvs_6axis); - -unsigned int atomisp_css_debug_get_dtrace_level(void); + struct ia_css_dvs_6axis_config *dvs_6axis); int atomisp_css_debug_dump_isp_binary(void); @@ -267,11 +159,11 @@ void atomisp_css_set_isp_config_id(struct atomisp_sub_device *asd, uint32_t isp_config_id); void atomisp_css_set_isp_config_applied_frame(struct atomisp_sub_device *asd, - struct atomisp_css_frame *output_frame); + struct ia_css_frame *output_frame); int atomisp_get_css_dbgfunc(void); int atomisp_set_css_dbgfunc(struct atomisp_device *isp, int opt); -struct atomisp_css_dvs_grid_info *atomisp_css_get_dvs_grid_info( - struct atomisp_css_grid_info *grid_info); +struct ia_css_dvs_grid_info *atomisp_css_get_dvs_grid_info( + struct ia_css_grid_info *grid_info); #endif diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c index 3079043f1fac..fa5918270614 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c +++ b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * @@ -24,66 +25,61 @@ #include "atomisp_ioctl.h" #include "atomisp_compat_ioctl32.h" -static int get_atomisp_histogram32(struct atomisp_histogram *kp, +/* Macro borrowed from v4l2-compat-ioctl32.c */ +/* Use the same argument order as copy_in_user */ +#define assign_in_user(to, from) \ +({ \ + typeof(*from) __assign_tmp; \ + \ + get_user(__assign_tmp, from) || put_user(__assign_tmp, to); \ +}) + + +static int get_atomisp_histogram32(struct atomisp_histogram __user *kp, struct atomisp_histogram32 __user *up) { compat_uptr_t tmp; if (!access_ok(up, sizeof(struct atomisp_histogram32)) || - get_user(kp->num_elements, &up->num_elements) || - get_user(tmp, &up->data)) + assign_in_user(&kp->num_elements, &up->num_elements) || + get_user(tmp, &up->data) || + put_user(compat_ptr(tmp), &kp->data)) return -EFAULT; - kp->data = compat_ptr(tmp); return 0; } -static int put_atomisp_histogram32(struct atomisp_histogram *kp, +static int put_atomisp_histogram32(struct atomisp_histogram __user *kp, struct atomisp_histogram32 __user *up) { - compat_uptr_t tmp = (compat_uptr_t)((uintptr_t)kp->data); + void __user *tmp; if (!access_ok(up, sizeof(struct atomisp_histogram32)) || - put_user(kp->num_elements, &up->num_elements) || - put_user(tmp, &up->data)) + assign_in_user(&up->num_elements, &kp->num_elements) || + get_user(tmp, &kp->data) || + put_user(ptr_to_compat(tmp), &up->data)) return -EFAULT; return 0; } -static inline int get_v4l2_pix_format(struct v4l2_pix_format *kp, - struct v4l2_pix_format __user *up) -{ - if (copy_from_user(kp, up, sizeof(struct v4l2_pix_format))) - return -EFAULT; - return 0; -} - -static inline int put_v4l2_pix_format(struct v4l2_pix_format *kp, - struct v4l2_pix_format __user *up) -{ - if (copy_to_user(up, kp, sizeof(struct v4l2_pix_format))) - return -EFAULT; - return 0; -} - -static int get_v4l2_framebuffer32(struct v4l2_framebuffer *kp, - struct v4l2_framebuffer32 __user *up) +static int get_v4l2_framebuffer32(struct v4l2_framebuffer __user *kp, + struct v4l2_framebuffer32 __user *up) { compat_uptr_t tmp; if (!access_ok(up, sizeof(struct v4l2_framebuffer32)) || get_user(tmp, &up->base) || - get_user(kp->capability, &up->capability) || - get_user(kp->flags, &up->flags)) + put_user(compat_ptr(tmp), &kp->base) || + assign_in_user(&kp->capability, &up->capability) || + assign_in_user(&kp->flags, &up->flags) || + copy_in_user(&kp->fmt, &up->fmt, sizeof(kp->fmt))) return -EFAULT; - kp->base = (void __force *)compat_ptr(tmp); - get_v4l2_pix_format((struct v4l2_pix_format *)&kp->fmt, &up->fmt); return 0; } -static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp, +static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp, struct atomisp_dis_statistics32 __user *up) { compat_uptr_t hor_prod_odd_real; @@ -96,67 +92,99 @@ static int get_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp, compat_uptr_t ver_prod_even_imag; if (!access_ok(up, sizeof(struct atomisp_dis_statistics32)) || - copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || - get_user(hor_prod_odd_real, &up->dvs2_stat.hor_prod.odd_real) || - get_user(hor_prod_odd_imag, &up->dvs2_stat.hor_prod.odd_imag) || - get_user(hor_prod_even_real, &up->dvs2_stat.hor_prod.even_real) || - get_user(hor_prod_even_imag, &up->dvs2_stat.hor_prod.even_imag) || - get_user(ver_prod_odd_real, &up->dvs2_stat.ver_prod.odd_real) || - get_user(ver_prod_odd_imag, &up->dvs2_stat.ver_prod.odd_imag) || - get_user(ver_prod_even_real, &up->dvs2_stat.ver_prod.even_real) || - get_user(ver_prod_even_imag, &up->dvs2_stat.ver_prod.even_imag) || - get_user(kp->exp_id, &up->exp_id)) + copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || + get_user(hor_prod_odd_real, + &up->dvs2_stat.hor_prod.odd_real) || + get_user(hor_prod_odd_imag, + &up->dvs2_stat.hor_prod.odd_imag) || + get_user(hor_prod_even_real, + &up->dvs2_stat.hor_prod.even_real) || + get_user(hor_prod_even_imag, + &up->dvs2_stat.hor_prod.even_imag) || + get_user(ver_prod_odd_real, + &up->dvs2_stat.ver_prod.odd_real) || + get_user(ver_prod_odd_imag, + &up->dvs2_stat.ver_prod.odd_imag) || + get_user(ver_prod_even_real, + &up->dvs2_stat.ver_prod.even_real) || + get_user(ver_prod_even_imag, + &up->dvs2_stat.ver_prod.even_imag) || + assign_in_user(&kp->exp_id, &up->exp_id) || + put_user(compat_ptr(hor_prod_odd_real), + &kp->dvs2_stat.hor_prod.odd_real) || + put_user(compat_ptr(hor_prod_odd_imag), + &kp->dvs2_stat.hor_prod.odd_imag) || + put_user(compat_ptr(hor_prod_even_real), + &kp->dvs2_stat.hor_prod.even_real) || + put_user(compat_ptr(hor_prod_even_imag), + &kp->dvs2_stat.hor_prod.even_imag) || + put_user(compat_ptr(ver_prod_odd_real), + &kp->dvs2_stat.ver_prod.odd_real) || + put_user(compat_ptr(ver_prod_odd_imag), + &kp->dvs2_stat.ver_prod.odd_imag) || + put_user(compat_ptr(ver_prod_even_real), + &kp->dvs2_stat.ver_prod.even_real) || + put_user(compat_ptr(ver_prod_even_imag), + &kp->dvs2_stat.ver_prod.even_imag)) return -EFAULT; - kp->dvs2_stat.hor_prod.odd_real = compat_ptr(hor_prod_odd_real); - kp->dvs2_stat.hor_prod.odd_imag = compat_ptr(hor_prod_odd_imag); - kp->dvs2_stat.hor_prod.even_real = compat_ptr(hor_prod_even_real); - kp->dvs2_stat.hor_prod.even_imag = compat_ptr(hor_prod_even_imag); - kp->dvs2_stat.ver_prod.odd_real = compat_ptr(ver_prod_odd_real); - kp->dvs2_stat.ver_prod.odd_imag = compat_ptr(ver_prod_odd_imag); - kp->dvs2_stat.ver_prod.even_real = compat_ptr(ver_prod_even_real); - kp->dvs2_stat.ver_prod.even_imag = compat_ptr(ver_prod_even_imag); return 0; } -static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics *kp, +static int put_atomisp_dis_statistics32(struct atomisp_dis_statistics __user *kp, struct atomisp_dis_statistics32 __user *up) { - compat_uptr_t hor_prod_odd_real = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_real); - compat_uptr_t hor_prod_odd_imag = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.odd_imag); - compat_uptr_t hor_prod_even_real = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_real); - compat_uptr_t hor_prod_even_imag = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.hor_prod.even_imag); - compat_uptr_t ver_prod_odd_real = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_real); - compat_uptr_t ver_prod_odd_imag = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.odd_imag); - compat_uptr_t ver_prod_even_real = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_real); - compat_uptr_t ver_prod_even_imag = - (compat_uptr_t)((uintptr_t)kp->dvs2_stat.ver_prod.even_imag); - - if (!access_ok(up, sizeof(struct atomisp_dis_statistics32)) || - copy_to_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) || - put_user(hor_prod_odd_real, &up->dvs2_stat.hor_prod.odd_real) || - put_user(hor_prod_odd_imag, &up->dvs2_stat.hor_prod.odd_imag) || - put_user(hor_prod_even_real, &up->dvs2_stat.hor_prod.even_real) || - put_user(hor_prod_even_imag, &up->dvs2_stat.hor_prod.even_imag) || - put_user(ver_prod_odd_real, &up->dvs2_stat.ver_prod.odd_real) || - put_user(ver_prod_odd_imag, &up->dvs2_stat.ver_prod.odd_imag) || - put_user(ver_prod_even_real, &up->dvs2_stat.ver_prod.even_real) || - put_user(ver_prod_even_imag, &up->dvs2_stat.ver_prod.even_imag) || - put_user(kp->exp_id, &up->exp_id)) + void __user *hor_prod_odd_real; + void __user *hor_prod_odd_imag; + void __user *hor_prod_even_real; + void __user *hor_prod_even_imag; + void __user *ver_prod_odd_real; + void __user *ver_prod_odd_imag; + void __user *ver_prod_even_real; + void __user *ver_prod_even_imag; + + if (!!access_ok(up, sizeof(struct atomisp_dis_statistics32)) || + copy_in_user(up, kp, sizeof(struct atomisp_dvs_grid_info)) || + get_user(hor_prod_odd_real, + &kp->dvs2_stat.hor_prod.odd_real) || + get_user(hor_prod_odd_imag, + &kp->dvs2_stat.hor_prod.odd_imag) || + get_user(hor_prod_even_real, + &kp->dvs2_stat.hor_prod.even_real) || + get_user(hor_prod_even_imag, + &kp->dvs2_stat.hor_prod.even_imag) || + get_user(ver_prod_odd_real, + &kp->dvs2_stat.ver_prod.odd_real) || + get_user(ver_prod_odd_imag, + &kp->dvs2_stat.ver_prod.odd_imag) || + get_user(ver_prod_even_real, + &kp->dvs2_stat.ver_prod.even_real) || + get_user(ver_prod_even_imag, + &kp->dvs2_stat.ver_prod.even_imag) || + put_user(ptr_to_compat(hor_prod_odd_real), + &up->dvs2_stat.hor_prod.odd_real) || + put_user(ptr_to_compat(hor_prod_odd_imag), + &up->dvs2_stat.hor_prod.odd_imag) || + put_user(ptr_to_compat(hor_prod_even_real), + &up->dvs2_stat.hor_prod.even_real) || + put_user(ptr_to_compat(hor_prod_even_imag), + &up->dvs2_stat.hor_prod.even_imag) || + put_user(ptr_to_compat(ver_prod_odd_real), + &up->dvs2_stat.ver_prod.odd_real) || + put_user(ptr_to_compat(ver_prod_odd_imag), + &up->dvs2_stat.ver_prod.odd_imag) || + put_user(ptr_to_compat(ver_prod_even_real), + &up->dvs2_stat.ver_prod.even_real) || + put_user(ptr_to_compat(ver_prod_even_imag), + &up->dvs2_stat.ver_prod.even_imag) || + assign_in_user(&up->exp_id, &kp->exp_id)) return -EFAULT; return 0; } -static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp, - struct atomisp_dis_coefficients32 __user *up) +static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients __user *kp, + struct atomisp_dis_coefficients32 __user *up) { compat_uptr_t hor_coefs_odd_real; compat_uptr_t hor_coefs_odd_imag; @@ -168,7 +196,7 @@ static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp, compat_uptr_t ver_coefs_even_imag; if (!access_ok(up, sizeof(struct atomisp_dis_coefficients32)) || - copy_from_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || + copy_in_user(kp, up, sizeof(struct atomisp_dvs_grid_info)) || get_user(hor_coefs_odd_real, &up->hor_coefs.odd_real) || get_user(hor_coefs_odd_imag, &up->hor_coefs.odd_imag) || get_user(hor_coefs_even_real, &up->hor_coefs.even_real) || @@ -176,22 +204,30 @@ static int get_atomisp_dis_coefficients32(struct atomisp_dis_coefficients *kp, get_user(ver_coefs_odd_real, &up->ver_coefs.odd_real) || get_user(ver_coefs_odd_imag, &up->ver_coefs.odd_imag) || get_user(ver_coefs_even_real, &up->ver_coefs.even_real) || - get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag)) + get_user(ver_coefs_even_imag, &up->ver_coefs.even_imag) || + put_user(compat_ptr(hor_coefs_odd_real), + &kp->hor_coefs.odd_real) || + put_user(compat_ptr(hor_coefs_odd_imag), + &kp->hor_coefs.odd_imag) || + put_user(compat_ptr(hor_coefs_even_real), + &kp->hor_coefs.even_real) || + put_user(compat_ptr(hor_coefs_even_imag), + &kp->hor_coefs.even_imag) || + put_user(compat_ptr(ver_coefs_odd_real), + &kp->ver_coefs.odd_real) || + put_user(compat_ptr(ver_coefs_odd_imag), + &kp->ver_coefs.odd_imag) || + put_user(compat_ptr(ver_coefs_even_real), + &kp->ver_coefs.even_real) || + put_user(compat_ptr(ver_coefs_even_imag), + &kp->ver_coefs.even_imag)) return -EFAULT; - kp->hor_coefs.odd_real = compat_ptr(hor_coefs_odd_real); - kp->hor_coefs.odd_imag = compat_ptr(hor_coefs_odd_imag); - kp->hor_coefs.even_real = compat_ptr(hor_coefs_even_real); - kp->hor_coefs.even_imag = compat_ptr(hor_coefs_even_imag); - kp->ver_coefs.odd_real = compat_ptr(ver_coefs_odd_real); - kp->ver_coefs.odd_imag = compat_ptr(ver_coefs_odd_imag); - kp->ver_coefs.even_real = compat_ptr(ver_coefs_even_real); - kp->ver_coefs.even_imag = compat_ptr(ver_coefs_even_imag); return 0; } -static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config *kp, - struct atomisp_dvs_6axis_config32 __user *up) +static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config __user *kp, + struct atomisp_dvs_6axis_config32 __user *up) { compat_uptr_t xcoords_y; compat_uptr_t ycoords_y; @@ -199,62 +235,63 @@ static int get_atomisp_dvs_6axis_config32(struct atomisp_dvs_6axis_config *kp, compat_uptr_t ycoords_uv; if (!access_ok(up, sizeof(struct atomisp_dvs_6axis_config32)) || - get_user(kp->exp_id, &up->exp_id) || - get_user(kp->width_y, &up->width_y) || - get_user(kp->height_y, &up->height_y) || - get_user(kp->width_uv, &up->width_uv) || - get_user(kp->height_uv, &up->height_uv) || + assign_in_user(&kp->exp_id, &up->exp_id) || + assign_in_user(&kp->width_y, &up->width_y) || + assign_in_user(&kp->height_y, &up->height_y) || + assign_in_user(&kp->width_uv, &up->width_uv) || + assign_in_user(&kp->height_uv, &up->height_uv) || get_user(xcoords_y, &up->xcoords_y) || get_user(ycoords_y, &up->ycoords_y) || get_user(xcoords_uv, &up->xcoords_uv) || - get_user(ycoords_uv, &up->ycoords_uv)) + get_user(ycoords_uv, &up->ycoords_uv) || + put_user(compat_ptr(xcoords_y), &kp->xcoords_y) || + put_user(compat_ptr(ycoords_y), &kp->ycoords_y) || + put_user(compat_ptr(xcoords_uv), &kp->xcoords_uv) || + put_user(compat_ptr(ycoords_uv), &kp->ycoords_uv)) return -EFAULT; - kp->xcoords_y = (void __force *)compat_ptr(xcoords_y); - kp->ycoords_y = (void __force *)compat_ptr(ycoords_y); - kp->xcoords_uv = (void __force *)compat_ptr(xcoords_uv); - kp->ycoords_uv = (void __force *)compat_ptr(ycoords_uv); return 0; } -static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp, +static int get_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp, struct atomisp_3a_statistics32 __user *up) { compat_uptr_t data; compat_uptr_t rgby_data; if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) || - copy_from_user(kp, up, sizeof(struct atomisp_grid_info)) || + copy_in_user(kp, up, sizeof(struct atomisp_grid_info)) || get_user(rgby_data, &up->rgby_data) || + put_user(compat_ptr(rgby_data), &kp->rgby_data) || get_user(data, &up->data) || - get_user(kp->exp_id, &up->exp_id) || - get_user(kp->isp_config_id, &up->isp_config_id)) + put_user(compat_ptr(data), &kp->data) || + assign_in_user(&kp->exp_id, &up->exp_id) || + assign_in_user(&kp->isp_config_id, &up->isp_config_id)) return -EFAULT; - kp->data = compat_ptr(data); - kp->rgby_data = compat_ptr(rgby_data); - return 0; } -static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics *kp, +static int put_atomisp_3a_statistics32(struct atomisp_3a_statistics __user *kp, struct atomisp_3a_statistics32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); - compat_uptr_t rgby_data = (compat_uptr_t)((uintptr_t)kp->rgby_data); + void __user *data; + void __user *rgby_data; if (!access_ok(up, sizeof(struct atomisp_3a_statistics32)) || copy_to_user(up, kp, sizeof(struct atomisp_grid_info)) || - put_user(rgby_data, &up->rgby_data) || - put_user(data, &up->data) || - put_user(kp->exp_id, &up->exp_id) || - put_user(kp->isp_config_id, &up->isp_config_id)) + get_user(rgby_data, &kp->rgby_data) || + put_user(ptr_to_compat(rgby_data), &up->rgby_data) || + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data) || + assign_in_user(&up->exp_id, &kp->exp_id) || + assign_in_user(&up->isp_config_id, &kp->isp_config_id)) return -EFAULT; return 0; } -static int get_atomisp_metadata_stat32(struct atomisp_metadata *kp, +static int get_atomisp_metadata_stat32(struct atomisp_metadata __user *kp, struct atomisp_metadata32 __user *up) { compat_uptr_t data; @@ -262,553 +299,524 @@ static int get_atomisp_metadata_stat32(struct atomisp_metadata *kp, if (!access_ok(up, sizeof(struct atomisp_metadata32)) || get_user(data, &up->data) || - get_user(kp->width, &up->width) || - get_user(kp->height, &up->height) || - get_user(kp->stride, &up->stride) || - get_user(kp->exp_id, &up->exp_id) || - get_user(effective_width, &up->effective_width)) + put_user(compat_ptr(data), &kp->data) || + assign_in_user(&kp->width, &up->width) || + assign_in_user(&kp->height, &up->height) || + assign_in_user(&kp->stride, &up->stride) || + assign_in_user(&kp->exp_id, &up->exp_id) || + get_user(effective_width, &up->effective_width) || + put_user(compat_ptr(effective_width), &kp->effective_width)) return -EFAULT; - kp->data = compat_ptr(data); - kp->effective_width = (void __force *)compat_ptr(effective_width); return 0; } -static int put_atomisp_metadata_stat32(struct atomisp_metadata *kp, - struct atomisp_metadata32 __user *up) +static int put_atomisp_metadata_stat32(struct atomisp_metadata __user *kp, + struct atomisp_metadata32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); - compat_uptr_t effective_width = - (compat_uptr_t)((uintptr_t)kp->effective_width); + void __user *data; + void __user *effective_width; + if (!access_ok(up, sizeof(struct atomisp_metadata32)) || - put_user(data, &up->data) || - put_user(kp->width, &up->width) || - put_user(kp->height, &up->height) || - put_user(kp->stride, &up->stride) || - put_user(kp->exp_id, &up->exp_id) || - put_user(effective_width, &up->effective_width)) + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data) || + assign_in_user(&up->width, &kp->width) || + assign_in_user(&up->height, &kp->height) || + assign_in_user(&up->stride, &kp->stride) || + assign_in_user(&up->exp_id, &kp->exp_id) || + get_user(effective_width, &kp->effective_width) || + put_user(ptr_to_compat(effective_width), &up->effective_width)) return -EFAULT; return 0; } -static int put_atomisp_metadata_by_type_stat32( - struct atomisp_metadata_with_type *kp, - struct atomisp_metadata_with_type32 __user *up) +static int +put_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp, + struct atomisp_metadata_with_type32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); - compat_uptr_t effective_width = - (compat_uptr_t)((uintptr_t)kp->effective_width); + void __user *data; + void __user *effective_width; + if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) || - put_user(data, &up->data) || - put_user(kp->width, &up->width) || - put_user(kp->height, &up->height) || - put_user(kp->stride, &up->stride) || - put_user(kp->exp_id, &up->exp_id) || - put_user(effective_width, &up->effective_width) || - put_user(kp->type, &up->type)) + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data) || + assign_in_user(&up->width, &kp->width) || + assign_in_user(&up->height, &kp->height) || + assign_in_user(&up->stride, &kp->stride) || + assign_in_user(&up->exp_id, &kp->exp_id) || + get_user(effective_width, &kp->effective_width) || + put_user(ptr_to_compat(effective_width), + &up->effective_width) || + assign_in_user(&up->type, &kp->type)) return -EFAULT; return 0; } -static int get_atomisp_metadata_by_type_stat32( - struct atomisp_metadata_with_type *kp, - struct atomisp_metadata_with_type32 __user *up) +static int +get_atomisp_metadata_by_type_stat32(struct atomisp_metadata_with_type __user *kp, + struct atomisp_metadata_with_type32 __user *up) { compat_uptr_t data; compat_uptr_t effective_width; if (!access_ok(up, sizeof(struct atomisp_metadata_with_type32)) || get_user(data, &up->data) || - get_user(kp->width, &up->width) || - get_user(kp->height, &up->height) || - get_user(kp->stride, &up->stride) || - get_user(kp->exp_id, &up->exp_id) || + put_user(compat_ptr(data), &kp->data) || + assign_in_user(&kp->width, &up->width) || + assign_in_user(&kp->height, &up->height) || + assign_in_user(&kp->stride, &up->stride) || + assign_in_user(&kp->exp_id, &up->exp_id) || get_user(effective_width, &up->effective_width) || - get_user(kp->type, &up->type)) + put_user(compat_ptr(effective_width), &kp->effective_width) || + assign_in_user(&kp->type, &up->type)) return -EFAULT; - kp->data = compat_ptr(data); - kp->effective_width = (void __force *)compat_ptr(effective_width); return 0; } -static int get_atomisp_morph_table32(struct atomisp_morph_table *kp, - struct atomisp_morph_table32 __user *up) +static int +get_atomisp_morph_table32(struct atomisp_morph_table __user *kp, + struct atomisp_morph_table32 __user *up) { unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES; if (!access_ok(up, sizeof(struct atomisp_morph_table32)) || - get_user(kp->enabled, &up->enabled) || - get_user(kp->width, &up->width) || - get_user(kp->height, &up->height)) - return -EFAULT; + assign_in_user(&kp->enabled, &up->enabled) || + assign_in_user(&kp->width, &up->width) || + assign_in_user(&kp->height, &up->height)) + return -EFAULT; while (n-- > 0) { - uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n]; - - if (get_user((*coord_kp), &up->coordinates_x[n])) - return -EFAULT; + compat_uptr_t coord_kp; - coord_kp = (uintptr_t *)&kp->coordinates_y[n]; - if (get_user((*coord_kp), &up->coordinates_y[n])) + if (get_user(coord_kp, &up->coordinates_x[n]) || + put_user(compat_ptr(coord_kp), &kp->coordinates_x[n]) || + get_user(coord_kp, &up->coordinates_y[n]) || + put_user(compat_ptr(coord_kp), &kp->coordinates_y[n])) return -EFAULT; } return 0; } -static int put_atomisp_morph_table32(struct atomisp_morph_table *kp, +static int put_atomisp_morph_table32(struct atomisp_morph_table __user *kp, struct atomisp_morph_table32 __user *up) { unsigned int n = ATOMISP_MORPH_TABLE_NUM_PLANES; if (!access_ok(up, sizeof(struct atomisp_morph_table32)) || - put_user(kp->enabled, &up->enabled) || - put_user(kp->width, &up->width) || - put_user(kp->height, &up->height)) - return -EFAULT; + assign_in_user(&up->enabled, &kp->enabled) || + assign_in_user(&up->width, &kp->width) || + assign_in_user(&up->height, &kp->height)) + return -EFAULT; while (n-- > 0) { - uintptr_t *coord_kp = (uintptr_t *)&kp->coordinates_x[n]; + void __user *coord_kp; - if (put_user((*coord_kp), &up->coordinates_x[n])) - return -EFAULT; - - coord_kp = (uintptr_t *)&kp->coordinates_y[n]; - if (put_user((*coord_kp), &up->coordinates_y[n])) + if (get_user(coord_kp, &kp->coordinates_x[n]) || + put_user(ptr_to_compat(coord_kp), &up->coordinates_x[n]) || + get_user(coord_kp, &kp->coordinates_y[n]) || + put_user(ptr_to_compat(coord_kp), &up->coordinates_y[n])) return -EFAULT; } return 0; } -static int get_atomisp_overlay32(struct atomisp_overlay *kp, +static int get_atomisp_overlay32(struct atomisp_overlay __user *kp, struct atomisp_overlay32 __user *up) { compat_uptr_t frame; if (!access_ok(up, sizeof(struct atomisp_overlay32)) || get_user(frame, &up->frame) || - get_user(kp->bg_y, &up->bg_y) || - get_user(kp->bg_u, &up->bg_u) || - get_user(kp->bg_v, &up->bg_v) || - get_user(kp->blend_input_perc_y, &up->blend_input_perc_y) || - get_user(kp->blend_input_perc_u, &up->blend_input_perc_u) || - get_user(kp->blend_input_perc_v, &up->blend_input_perc_v) || - get_user(kp->blend_overlay_perc_y, &up->blend_overlay_perc_y) || - get_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) || - get_user(kp->blend_overlay_perc_v, &up->blend_overlay_perc_v) || - get_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) || - get_user(kp->overlay_start_x, &up->overlay_start_y)) + put_user(compat_ptr(frame), &kp->frame) || + assign_in_user(&kp->bg_y, &up->bg_y) || + assign_in_user(&kp->bg_u, &up->bg_u) || + assign_in_user(&kp->bg_v, &up->bg_v) || + assign_in_user(&kp->blend_input_perc_y, + &up->blend_input_perc_y) || + assign_in_user(&kp->blend_input_perc_u, + &up->blend_input_perc_u) || + assign_in_user(&kp->blend_input_perc_v, + &up->blend_input_perc_v) || + assign_in_user(&kp->blend_overlay_perc_y, + &up->blend_overlay_perc_y) || + assign_in_user(&kp->blend_overlay_perc_u, + &up->blend_overlay_perc_u) || + assign_in_user(&kp->blend_overlay_perc_v, + &up->blend_overlay_perc_v) || + assign_in_user(&kp->overlay_start_x, &up->overlay_start_x) || + assign_in_user(&kp->overlay_start_y, &up->overlay_start_y)) return -EFAULT; - kp->frame = (void __force *)compat_ptr(frame); return 0; } -static int put_atomisp_overlay32(struct atomisp_overlay *kp, +static int put_atomisp_overlay32(struct atomisp_overlay __user *kp, struct atomisp_overlay32 __user *up) { - compat_uptr_t frame = (compat_uptr_t)((uintptr_t)kp->frame); + void __user *frame; if (!access_ok(up, sizeof(struct atomisp_overlay32)) || - put_user(frame, &up->frame) || - put_user(kp->bg_y, &up->bg_y) || - put_user(kp->bg_u, &up->bg_u) || - put_user(kp->bg_v, &up->bg_v) || - put_user(kp->blend_input_perc_y, &up->blend_input_perc_y) || - put_user(kp->blend_input_perc_u, &up->blend_input_perc_u) || - put_user(kp->blend_input_perc_v, &up->blend_input_perc_v) || - put_user(kp->blend_overlay_perc_y, &up->blend_overlay_perc_y) || - put_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) || - put_user(kp->blend_overlay_perc_v, &up->blend_overlay_perc_v) || - put_user(kp->blend_overlay_perc_u, &up->blend_overlay_perc_u) || - put_user(kp->overlay_start_x, &up->overlay_start_y)) + get_user(frame, &kp->frame) || + put_user(ptr_to_compat(frame), &up->frame) || + assign_in_user(&up->bg_y, &kp->bg_y) || + assign_in_user(&up->bg_u, &kp->bg_u) || + assign_in_user(&up->bg_v, &kp->bg_v) || + assign_in_user(&up->blend_input_perc_y, + &kp->blend_input_perc_y) || + assign_in_user(&up->blend_input_perc_u, + &kp->blend_input_perc_u) || + assign_in_user(&up->blend_input_perc_v, + &kp->blend_input_perc_v) || + assign_in_user(&up->blend_overlay_perc_y, + &kp->blend_overlay_perc_y) || + assign_in_user(&up->blend_overlay_perc_u, + &kp->blend_overlay_perc_u) || + assign_in_user(&up->blend_overlay_perc_v, + &kp->blend_overlay_perc_v) || + assign_in_user(&up->overlay_start_x, &kp->overlay_start_x) || + assign_in_user(&up->overlay_start_y, &kp->overlay_start_y)) return -EFAULT; return 0; } -static int get_atomisp_calibration_group32( - struct atomisp_calibration_group *kp, - struct atomisp_calibration_group32 __user *up) +static int +get_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp, + struct atomisp_calibration_group32 __user *up) { compat_uptr_t calb_grp_values; if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) || - get_user(kp->size, &up->size) || - get_user(kp->type, &up->type) || - get_user(calb_grp_values, &up->calb_grp_values)) + assign_in_user(&kp->size, &up->size) || + assign_in_user(&kp->type, &up->type) || + get_user(calb_grp_values, &up->calb_grp_values) || + put_user(compat_ptr(calb_grp_values), &kp->calb_grp_values)) return -EFAULT; - kp->calb_grp_values = (void __force *)compat_ptr(calb_grp_values); return 0; } -static int put_atomisp_calibration_group32( - struct atomisp_calibration_group *kp, - struct atomisp_calibration_group32 __user *up) +static int +put_atomisp_calibration_group32(struct atomisp_calibration_group __user *kp, + struct atomisp_calibration_group32 __user *up) { - compat_uptr_t calb_grp_values = - (compat_uptr_t)((uintptr_t)kp->calb_grp_values); + void __user *calb_grp_values; if (!access_ok(up, sizeof(struct atomisp_calibration_group32)) || - put_user(kp->size, &up->size) || - put_user(kp->type, &up->type) || - put_user(calb_grp_values, &up->calb_grp_values)) + assign_in_user(&up->size, &kp->size) || + assign_in_user(&up->type, &kp->type) || + get_user(calb_grp_values, &kp->calb_grp_values) || + put_user(ptr_to_compat(calb_grp_values), &up->calb_grp_values)) return -EFAULT; return 0; } -static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp, +static int get_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp, struct atomisp_acc_fw_load32 __user *up) { compat_uptr_t data; if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) || - get_user(kp->size, &up->size) || - get_user(kp->fw_handle, &up->fw_handle) || - get_user(data, &up->data)) + assign_in_user(&kp->size, &up->size) || + assign_in_user(&kp->fw_handle, &up->fw_handle) || + get_user(data, &up->data) || + put_user(compat_ptr(data), &kp->data)) return -EFAULT; - kp->data = compat_ptr(data); return 0; } -static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load *kp, +static int put_atomisp_acc_fw_load32(struct atomisp_acc_fw_load __user *kp, struct atomisp_acc_fw_load32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); + void __user *data; if (!access_ok(up, sizeof(struct atomisp_acc_fw_load32)) || - put_user(kp->size, &up->size) || - put_user(kp->fw_handle, &up->fw_handle) || - put_user(data, &up->data)) + assign_in_user(&up->size, &kp->size) || + assign_in_user(&up->fw_handle, &kp->fw_handle) || + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data)) return -EFAULT; return 0; } -static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp, +static int get_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp, struct atomisp_acc_fw_arg32 __user *up) { compat_uptr_t value; if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) || - get_user(kp->fw_handle, &up->fw_handle) || - get_user(kp->index, &up->index) || + assign_in_user(&kp->fw_handle, &up->fw_handle) || + assign_in_user(&kp->index, &up->index) || get_user(value, &up->value) || - get_user(kp->size, &up->size)) + put_user(compat_ptr(value), &kp->value) || + assign_in_user(&kp->size, &up->size)) return -EFAULT; - kp->value = compat_ptr(value); return 0; } -static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg *kp, +static int put_atomisp_acc_fw_arg32(struct atomisp_acc_fw_arg __user *kp, struct atomisp_acc_fw_arg32 __user *up) { - compat_uptr_t value = (compat_uptr_t)((uintptr_t)kp->value); + void __user *value; if (!access_ok(up, sizeof(struct atomisp_acc_fw_arg32)) || - put_user(kp->fw_handle, &up->fw_handle) || - put_user(kp->index, &up->index) || - put_user(value, &up->value) || - put_user(kp->size, &up->size)) + assign_in_user(&up->fw_handle, &kp->fw_handle) || + assign_in_user(&up->index, &kp->index) || + get_user(value, &kp->value) || + put_user(ptr_to_compat(value), &up->value) || + assign_in_user(&up->size, &kp->size)) return -EFAULT; return 0; } -static int get_v4l2_private_int_data32(struct v4l2_private_int_data *kp, +static int get_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp, struct v4l2_private_int_data32 __user *up) { compat_uptr_t data; if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) || - get_user(kp->size, &up->size) || + assign_in_user(&kp->size, &up->size) || get_user(data, &up->data) || - get_user(kp->reserved[0], &up->reserved[0]) || - get_user(kp->reserved[1], &up->reserved[1])) + put_user(compat_ptr(data), &kp->data) || + assign_in_user(&kp->reserved[0], &up->reserved[0]) || + assign_in_user(&kp->reserved[1], &up->reserved[1])) return -EFAULT; - kp->data = compat_ptr(data); return 0; } -static int put_v4l2_private_int_data32(struct v4l2_private_int_data *kp, +static int put_v4l2_private_int_data32(struct v4l2_private_int_data __user *kp, struct v4l2_private_int_data32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); + void __user *data; if (!access_ok(up, sizeof(struct v4l2_private_int_data32)) || - put_user(kp->size, &up->size) || - put_user(data, &up->data) || - put_user(kp->reserved[0], &up->reserved[0]) || - put_user(kp->reserved[1], &up->reserved[1])) + assign_in_user(&up->size, &kp->size) || + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data) || + assign_in_user(&up->reserved[0], &kp->reserved[0]) || + assign_in_user(&up->reserved[1], &kp->reserved[1])) return -EFAULT; return 0; } -static int get_atomisp_shading_table32(struct atomisp_shading_table *kp, +static int get_atomisp_shading_table32(struct atomisp_shading_table __user *kp, struct atomisp_shading_table32 __user *up) { unsigned int n = ATOMISP_NUM_SC_COLORS; if (!access_ok(up, sizeof(struct atomisp_shading_table32)) || - get_user(kp->enable, &up->enable) || - get_user(kp->sensor_width, &up->sensor_width) || - get_user(kp->sensor_height, &up->sensor_height) || - get_user(kp->width, &up->width) || - get_user(kp->height, &up->height) || - get_user(kp->fraction_bits, &up->fraction_bits)) + assign_in_user(&kp->enable, &up->enable) || + assign_in_user(&kp->sensor_width, &up->sensor_width) || + assign_in_user(&kp->sensor_height, &up->sensor_height) || + assign_in_user(&kp->width, &up->width) || + assign_in_user(&kp->height, &up->height) || + assign_in_user(&kp->fraction_bits, &up->fraction_bits)) return -EFAULT; while (n-- > 0) { - uintptr_t *data_p = (uintptr_t *)&kp->data[n]; + compat_uptr_t tmp; - if (get_user((*data_p), &up->data[n])) + if (get_user(tmp, &up->data[n]) || + put_user(compat_ptr(tmp), &kp->data[n])) return -EFAULT; } return 0; } -static int get_atomisp_acc_map32(struct atomisp_acc_map *kp, +static int get_atomisp_acc_map32(struct atomisp_acc_map __user *kp, struct atomisp_acc_map32 __user *up) { compat_uptr_t user_ptr; if (!access_ok(up, sizeof(struct atomisp_acc_map32)) || - get_user(kp->flags, &up->flags) || - get_user(kp->length, &up->length) || + assign_in_user(&kp->flags, &up->flags) || + assign_in_user(&kp->length, &up->length) || get_user(user_ptr, &up->user_ptr) || - get_user(kp->css_ptr, &up->css_ptr) || - get_user(kp->reserved[0], &up->reserved[0]) || - get_user(kp->reserved[1], &up->reserved[1]) || - get_user(kp->reserved[2], &up->reserved[2]) || - get_user(kp->reserved[3], &up->reserved[3])) + put_user(compat_ptr(user_ptr), &kp->user_ptr) || + assign_in_user(&kp->css_ptr, &up->css_ptr) || + assign_in_user(&kp->reserved[0], &up->reserved[0]) || + assign_in_user(&kp->reserved[1], &up->reserved[1]) || + assign_in_user(&kp->reserved[2], &up->reserved[2]) || + assign_in_user(&kp->reserved[3], &up->reserved[3])) return -EFAULT; - kp->user_ptr = compat_ptr(user_ptr); return 0; } -static int put_atomisp_acc_map32(struct atomisp_acc_map *kp, +static int put_atomisp_acc_map32(struct atomisp_acc_map __user *kp, struct atomisp_acc_map32 __user *up) { - compat_uptr_t user_ptr = (compat_uptr_t)((uintptr_t)kp->user_ptr); + void __user *user_ptr; if (!access_ok(up, sizeof(struct atomisp_acc_map32)) || - put_user(kp->flags, &up->flags) || - put_user(kp->length, &up->length) || - put_user(user_ptr, &up->user_ptr) || - put_user(kp->css_ptr, &up->css_ptr) || - put_user(kp->reserved[0], &up->reserved[0]) || - put_user(kp->reserved[1], &up->reserved[1]) || - put_user(kp->reserved[2], &up->reserved[2]) || - put_user(kp->reserved[3], &up->reserved[3])) + assign_in_user(&up->flags, &kp->flags) || + assign_in_user(&up->length, &kp->length) || + get_user(user_ptr, &kp->user_ptr) || + put_user(ptr_to_compat(user_ptr), &up->user_ptr) || + assign_in_user(&up->css_ptr, &kp->css_ptr) || + assign_in_user(&up->reserved[0], &kp->reserved[0]) || + assign_in_user(&up->reserved[1], &kp->reserved[1]) || + assign_in_user(&up->reserved[2], &kp->reserved[2]) || + assign_in_user(&up->reserved[3], &kp->reserved[3])) return -EFAULT; return 0; } -static int get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp, - struct atomisp_acc_s_mapped_arg32 __user *up) +static int +get_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp, + struct atomisp_acc_s_mapped_arg32 __user *up) { if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) || - get_user(kp->fw_handle, &up->fw_handle) || - get_user(kp->memory, &up->memory) || - get_user(kp->length, &up->length) || - get_user(kp->css_ptr, &up->css_ptr)) + assign_in_user(&kp->fw_handle, &up->fw_handle) || + assign_in_user(&kp->memory, &up->memory) || + assign_in_user(&kp->length, &up->length) || + assign_in_user(&kp->css_ptr, &up->css_ptr)) return -EFAULT; return 0; } -static int put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg *kp, - struct atomisp_acc_s_mapped_arg32 __user *up) +static int +put_atomisp_acc_s_mapped_arg32(struct atomisp_acc_s_mapped_arg __user *kp, + struct atomisp_acc_s_mapped_arg32 __user *up) { if (!access_ok(up, sizeof(struct atomisp_acc_s_mapped_arg32)) || - put_user(kp->fw_handle, &up->fw_handle) || - put_user(kp->memory, &up->memory) || - put_user(kp->length, &up->length) || - put_user(kp->css_ptr, &up->css_ptr)) + assign_in_user(&up->fw_handle, &kp->fw_handle) || + assign_in_user(&up->memory, &kp->memory) || + assign_in_user(&up->length, &kp->length) || + assign_in_user(&up->css_ptr, &kp->css_ptr)) return -EFAULT; return 0; } -static int get_atomisp_parameters32(struct atomisp_parameters *kp, +static int get_atomisp_parameters32(struct atomisp_parameters __user *kp, struct atomisp_parameters32 __user *up) { int n = offsetof(struct atomisp_parameters32, output_frame) / sizeof(compat_uptr_t); - unsigned int size, offset = 0; - void __user *user_ptr; - unsigned int stp, mtp, dcp, dscp = 0; + compat_uptr_t stp, mtp, dcp, dscp; + struct { + struct atomisp_shading_table shading_table; + struct atomisp_morph_table morph_table; + struct atomisp_dis_coefficients dvs2_coefs; + struct atomisp_dvs_6axis_config dvs_6axis_config; + } __user *karg = (void *)(kp + 1); if (!access_ok(up, sizeof(struct atomisp_parameters32))) return -EFAULT; while (n >= 0) { - compat_uptr_t __user *src = ((compat_uptr_t __user *)up) + n; - uintptr_t *dst = ((uintptr_t *)kp) + n; + compat_uptr_t *src = (compat_uptr_t *)up + n; + void * __user *dst = (void * __user *)kp + n; + compat_uptr_t tmp; - if (get_user((*dst), src)) + if (get_user(tmp, src) || put_user(compat_ptr(tmp), dst)) return -EFAULT; n--; } - if (get_user(kp->isp_config_id, &up->isp_config_id) || - get_user(kp->per_frame_setting, &up->per_frame_setting) || + + if (assign_in_user(&kp->isp_config_id, &up->isp_config_id) || + assign_in_user(&kp->per_frame_setting, &up->per_frame_setting) || get_user(stp, &up->shading_table) || get_user(mtp, &up->morph_table) || get_user(dcp, &up->dvs2_coefs) || get_user(dscp, &up->dvs_6axis_config)) return -EFAULT; - { - union { - struct atomisp_shading_table shading_table; - struct atomisp_morph_table morph_table; - struct atomisp_dis_coefficients dvs2_coefs; - struct atomisp_dvs_6axis_config dvs_6axis_config; - } karg; - - size = sizeof(struct atomisp_shading_table) + - sizeof(struct atomisp_morph_table) + - sizeof(struct atomisp_dis_coefficients) + - sizeof(struct atomisp_dvs_6axis_config); - user_ptr = compat_alloc_user_space(size); - - /* handle shading table */ - if (stp != 0) { - if (get_atomisp_shading_table32(&karg.shading_table, - (struct atomisp_shading_table32 __user *) - (uintptr_t)stp)) - return -EFAULT; - - kp->shading_table = (void __force *)user_ptr + offset; - offset = sizeof(struct atomisp_shading_table); - if (!kp->shading_table) - return -EFAULT; - - if (copy_to_user((void __user *)kp->shading_table, - &karg.shading_table, - sizeof(struct atomisp_shading_table))) - return -EFAULT; - } - - /* handle morph table */ - if (mtp != 0) { - if (get_atomisp_morph_table32(&karg.morph_table, - (struct atomisp_morph_table32 __user *) - (uintptr_t)mtp)) - return -EFAULT; - - kp->morph_table = (void __force *)user_ptr + offset; - offset += sizeof(struct atomisp_morph_table); - if (!kp->morph_table) - return -EFAULT; - - if (copy_to_user((void __user *)kp->morph_table, - &karg.morph_table, - sizeof(struct atomisp_morph_table))) - return -EFAULT; - } - - /* handle dvs2 coefficients */ - if (dcp != 0) { - if (get_atomisp_dis_coefficients32(&karg.dvs2_coefs, - (struct atomisp_dis_coefficients32 __user *) - (uintptr_t)dcp)) - return -EFAULT; - - kp->dvs2_coefs = (void __force *)user_ptr + offset; - offset += sizeof(struct atomisp_dis_coefficients); - if (!kp->dvs2_coefs) - return -EFAULT; - - if (copy_to_user((void __user *)kp->dvs2_coefs, - &karg.dvs2_coefs, - sizeof(struct atomisp_dis_coefficients))) - return -EFAULT; - } - /* handle dvs 6axis configuration */ - if (dscp != 0) { - if (get_atomisp_dvs_6axis_config32(&karg.dvs_6axis_config, - (struct atomisp_dvs_6axis_config32 __user *) - (uintptr_t)dscp)) - return -EFAULT; - - kp->dvs_6axis_config = (void __force *)user_ptr + offset; - offset += sizeof(struct atomisp_dvs_6axis_config); - if (!kp->dvs_6axis_config) - return -EFAULT; - - if (copy_to_user((void __user *)kp->dvs_6axis_config, - &karg.dvs_6axis_config, - sizeof(struct atomisp_dvs_6axis_config))) - return -EFAULT; - } - } + /* handle shading table */ + if (stp && (get_atomisp_shading_table32(&karg->shading_table, + compat_ptr(stp)) || + put_user(&karg->shading_table, &kp->shading_table))) + return -EFAULT; + + /* handle morph table */ + if (mtp && (get_atomisp_morph_table32(&karg->morph_table, + compat_ptr(mtp)) || + put_user(&karg->morph_table, &kp->morph_table))) + return -EFAULT; + + /* handle dvs2 coefficients */ + if (dcp && (get_atomisp_dis_coefficients32(&karg->dvs2_coefs, + compat_ptr(dcp)) || + put_user(&karg->dvs2_coefs, &kp->dvs2_coefs))) + return -EFAULT; + + /* handle dvs 6axis configuration */ + if (dscp && + (get_atomisp_dvs_6axis_config32(&karg->dvs_6axis_config, + compat_ptr(dscp)) || + put_user(&karg->dvs_6axis_config, &kp->dvs_6axis_config))) + return -EFAULT; + return 0; } -static int get_atomisp_acc_fw_load_to_pipe32( - struct atomisp_acc_fw_load_to_pipe *kp, - struct atomisp_acc_fw_load_to_pipe32 __user *up) +static int +get_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp, + struct atomisp_acc_fw_load_to_pipe32 __user *up) { compat_uptr_t data; if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) || - get_user(kp->flags, &up->flags) || - get_user(kp->fw_handle, &up->fw_handle) || - get_user(kp->size, &up->size) || - get_user(kp->type, &up->type) || - get_user(kp->reserved[0], &up->reserved[0]) || - get_user(kp->reserved[1], &up->reserved[1]) || - get_user(kp->reserved[2], &up->reserved[2]) || - get_user(data, &up->data)) + assign_in_user(&kp->flags, &up->flags) || + assign_in_user(&kp->fw_handle, &up->fw_handle) || + assign_in_user(&kp->size, &up->size) || + assign_in_user(&kp->type, &up->type) || + assign_in_user(&kp->reserved[0], &up->reserved[0]) || + assign_in_user(&kp->reserved[1], &up->reserved[1]) || + assign_in_user(&kp->reserved[2], &up->reserved[2]) || + get_user(data, &up->data) || + put_user(compat_ptr(data), &kp->data)) return -EFAULT; - kp->data = compat_ptr(data); return 0; } -static int put_atomisp_acc_fw_load_to_pipe32( - struct atomisp_acc_fw_load_to_pipe *kp, - struct atomisp_acc_fw_load_to_pipe32 __user *up) +static int +put_atomisp_acc_fw_load_to_pipe32(struct atomisp_acc_fw_load_to_pipe __user *kp, + struct atomisp_acc_fw_load_to_pipe32 __user *up) { - compat_uptr_t data = (compat_uptr_t)((uintptr_t)kp->data); + void __user *data; if (!access_ok(up, sizeof(struct atomisp_acc_fw_load_to_pipe32)) || - put_user(kp->flags, &up->flags) || - put_user(kp->fw_handle, &up->fw_handle) || - put_user(kp->size, &up->size) || - put_user(kp->type, &up->type) || - put_user(kp->reserved[0], &up->reserved[0]) || - put_user(kp->reserved[1], &up->reserved[1]) || - put_user(kp->reserved[2], &up->reserved[2]) || - put_user(data, &up->data)) + assign_in_user(&up->flags, &kp->flags) || + assign_in_user(&up->fw_handle, &kp->fw_handle) || + assign_in_user(&up->size, &kp->size) || + assign_in_user(&up->type, &kp->type) || + assign_in_user(&up->reserved[0], &kp->reserved[0]) || + assign_in_user(&up->reserved[1], &kp->reserved[1]) || + assign_in_user(&up->reserved[2], &kp->reserved[2]) || + get_user(data, &kp->data) || + put_user(ptr_to_compat(data), &up->data)) return -EFAULT; return 0; } -static int get_atomisp_sensor_ae_bracketing_lut( - struct atomisp_sensor_ae_bracketing_lut *kp, - struct atomisp_sensor_ae_bracketing_lut32 __user *up) +static int +get_atomisp_sensor_ae_bracketing_lut(struct atomisp_sensor_ae_bracketing_lut __user *kp, + struct atomisp_sensor_ae_bracketing_lut32 __user *up) { compat_uptr_t lut; if (!access_ok(up, sizeof(struct atomisp_sensor_ae_bracketing_lut32)) || - get_user(kp->lut_size, &up->lut_size) || - get_user(lut, &up->lut)) + assign_in_user(&kp->lut_size, &up->lut_size) || + get_user(lut, &up->lut) || + put_user(compat_ptr(lut), &kp->lut)) return -EFAULT; - kp->lut = (void __force *)compat_ptr(lut); return 0; } @@ -846,11 +854,19 @@ static long atomisp_do_compat_ioctl(struct file *file, struct atomisp_metadata md; struct atomisp_metadata_with_type md_with_type; struct atomisp_sensor_ae_bracketing_lut lut; - } karg; - mm_segment_t old_fs; + } __user *karg; void __user *up = compat_ptr(arg); long err = -ENOIOCTLCMD; + karg = compat_alloc_user_space( + sizeof(*karg) + (cmd == ATOMISP_IOC_S_PARAMETERS32 ? + sizeof(struct atomisp_shading_table) + + sizeof(struct atomisp_morph_table) + + sizeof(struct atomisp_dis_coefficients) + + sizeof(struct atomisp_dvs_6axis_config) : 0)); + if (!karg) + return -ENOMEM; + /* First, convert the command. */ switch (cmd) { case ATOMISP_IOC_G_HISTOGRAM32: @@ -936,130 +952,127 @@ static long atomisp_do_compat_ioctl(struct file *file, switch (cmd) { case ATOMISP_IOC_G_HISTOGRAM: case ATOMISP_IOC_S_HISTOGRAM: - err = get_atomisp_histogram32(&karg.his, up); + err = get_atomisp_histogram32(&karg->his, up); break; case ATOMISP_IOC_G_DIS_STAT: - err = get_atomisp_dis_statistics32(&karg.dis_s, up); + err = get_atomisp_dis_statistics32(&karg->dis_s, up); break; case ATOMISP_IOC_S_DIS_COEFS: - err = get_atomisp_dis_coefficients32(&karg.dis_c, up); + err = get_atomisp_dis_coefficients32(&karg->dis_c, up); break; case ATOMISP_IOC_S_DIS_VECTOR: - err = get_atomisp_dvs_6axis_config32(&karg.dvs_c, up); + err = get_atomisp_dvs_6axis_config32(&karg->dvs_c, up); break; case ATOMISP_IOC_G_3A_STAT: - err = get_atomisp_3a_statistics32(&karg.s3a_s, up); + err = get_atomisp_3a_statistics32(&karg->s3a_s, up); break; case ATOMISP_IOC_G_ISP_GDC_TAB: case ATOMISP_IOC_S_ISP_GDC_TAB: - err = get_atomisp_morph_table32(&karg.mor_t, up); + err = get_atomisp_morph_table32(&karg->mor_t, up); break; case ATOMISP_IOC_S_ISP_FPN_TABLE: - err = get_v4l2_framebuffer32(&karg.v4l2_buf, up); + err = get_v4l2_framebuffer32(&karg->v4l2_buf, up); break; case ATOMISP_IOC_G_ISP_OVERLAY: case ATOMISP_IOC_S_ISP_OVERLAY: - err = get_atomisp_overlay32(&karg.overlay, up); + err = get_atomisp_overlay32(&karg->overlay, up); break; case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP: - err = get_atomisp_calibration_group32(&karg.cal_grp, up); + err = get_atomisp_calibration_group32(&karg->cal_grp, up); break; case ATOMISP_IOC_ACC_LOAD: - err = get_atomisp_acc_fw_load32(&karg.acc_fw_load, up); + err = get_atomisp_acc_fw_load32(&karg->acc_fw_load, up); break; case ATOMISP_IOC_ACC_S_ARG: case ATOMISP_IOC_ACC_DESTAB: - err = get_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up); + err = get_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up); break; case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA: case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA: - err = get_v4l2_private_int_data32(&karg.v4l2_pri_data, up); + err = get_v4l2_private_int_data32(&karg->v4l2_pri_data, up); break; case ATOMISP_IOC_S_ISP_SHD_TAB: - err = get_atomisp_shading_table32(&karg.shd_tbl, up); + err = get_atomisp_shading_table32(&karg->shd_tbl, up); break; case ATOMISP_IOC_ACC_MAP: case ATOMISP_IOC_ACC_UNMAP: - err = get_atomisp_acc_map32(&karg.acc_map, up); + err = get_atomisp_acc_map32(&karg->acc_map, up); break; case ATOMISP_IOC_ACC_S_MAPPED_ARG: - err = get_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up); + err = get_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up); break; case ATOMISP_IOC_S_PARAMETERS: - err = get_atomisp_parameters32(&karg.param, up); + err = get_atomisp_parameters32(&karg->param, up); break; case ATOMISP_IOC_ACC_LOAD_TO_PIPE: - err = get_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe, + err = get_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe, up); break; case ATOMISP_IOC_G_METADATA: - err = get_atomisp_metadata_stat32(&karg.md, up); + err = get_atomisp_metadata_stat32(&karg->md, up); break; case ATOMISP_IOC_G_METADATA_BY_TYPE: - err = get_atomisp_metadata_by_type_stat32(&karg.md_with_type, - up); + err = get_atomisp_metadata_by_type_stat32(&karg->md_with_type, + up); break; case ATOMISP_IOC_S_SENSOR_AE_BRACKETING_LUT: - err = get_atomisp_sensor_ae_bracketing_lut(&karg.lut, up); + err = get_atomisp_sensor_ae_bracketing_lut(&karg->lut, up); break; } if (err) return err; - old_fs = get_fs(); - set_fs(KERNEL_DS); - err = native_ioctl(file, cmd, (unsigned long)&karg); - set_fs(old_fs); + err = native_ioctl(file, cmd, (unsigned long)karg); if (err) return err; switch (cmd) { case ATOMISP_IOC_G_HISTOGRAM: - err = put_atomisp_histogram32(&karg.his, up); + err = put_atomisp_histogram32(&karg->his, up); break; case ATOMISP_IOC_G_DIS_STAT: - err = put_atomisp_dis_statistics32(&karg.dis_s, up); + err = put_atomisp_dis_statistics32(&karg->dis_s, up); break; case ATOMISP_IOC_G_3A_STAT: - err = put_atomisp_3a_statistics32(&karg.s3a_s, up); + err = put_atomisp_3a_statistics32(&karg->s3a_s, up); break; case ATOMISP_IOC_G_ISP_GDC_TAB: - err = put_atomisp_morph_table32(&karg.mor_t, up); + err = put_atomisp_morph_table32(&karg->mor_t, up); break; case ATOMISP_IOC_G_ISP_OVERLAY: - err = put_atomisp_overlay32(&karg.overlay, up); + err = put_atomisp_overlay32(&karg->overlay, up); break; case ATOMISP_IOC_G_SENSOR_CALIBRATION_GROUP: - err = put_atomisp_calibration_group32(&karg.cal_grp, up); + err = put_atomisp_calibration_group32(&karg->cal_grp, up); break; case ATOMISP_IOC_ACC_LOAD: - err = put_atomisp_acc_fw_load32(&karg.acc_fw_load, up); + err = put_atomisp_acc_fw_load32(&karg->acc_fw_load, up); break; case ATOMISP_IOC_ACC_S_ARG: case ATOMISP_IOC_ACC_DESTAB: - err = put_atomisp_acc_fw_arg32(&karg.acc_fw_arg, up); + err = put_atomisp_acc_fw_arg32(&karg->acc_fw_arg, up); break; case ATOMISP_IOC_G_SENSOR_PRIV_INT_DATA: case ATOMISP_IOC_G_MOTOR_PRIV_INT_DATA: - err = put_v4l2_private_int_data32(&karg.v4l2_pri_data, up); + err = put_v4l2_private_int_data32(&karg->v4l2_pri_data, up); break; case ATOMISP_IOC_ACC_MAP: case ATOMISP_IOC_ACC_UNMAP: - err = put_atomisp_acc_map32(&karg.acc_map, up); + err = put_atomisp_acc_map32(&karg->acc_map, up); break; case ATOMISP_IOC_ACC_S_MAPPED_ARG: - err = put_atomisp_acc_s_mapped_arg32(&karg.acc_map_arg, up); + err = put_atomisp_acc_s_mapped_arg32(&karg->acc_map_arg, up); break; case ATOMISP_IOC_ACC_LOAD_TO_PIPE: - err = put_atomisp_acc_fw_load_to_pipe32(&karg.acc_fw_to_pipe, + err = put_atomisp_acc_fw_load_to_pipe32(&karg->acc_fw_to_pipe, up); break; case ATOMISP_IOC_G_METADATA: - err = put_atomisp_metadata_stat32(&karg.md, up); + err = put_atomisp_metadata_stat32(&karg->md, up); break; case ATOMISP_IOC_G_METADATA_BY_TYPE: - err = put_atomisp_metadata_by_type_stat32(&karg.md_with_type, - up); + err = put_atomisp_metadata_by_type_stat32(&karg->md_with_type, + up); break; } diff --git a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h index 7e59ccb88a2e..86d3fbe01378 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h +++ b/drivers/staging/media/atomisp/pci/atomisp_compat_ioctl32.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.c b/drivers/staging/media/atomisp/pci/atomisp_csi2.c index a2638863206e..060b8765ae96 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_csi2.c +++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -384,9 +385,9 @@ static void atomisp_csi2_configure_isp2401(struct atomisp_sub_device *asd) for (n = 0; n < csi2_port_lanes[port] + 1; n++) { hrt_address base = csi2_port_base[port] + csi2_lane_base[n]; - atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_TERMEN, + atomisp_css2_hw_store_32(base + CSI2_REG_RX_CSI_DLY_CNT_TERMEN, n == 0 ? clk_termen : dat_termen); - atomisp_store_uint32(base + CSI2_REG_RX_CSI_DLY_CNT_SETTLE, + atomisp_css2_hw_store_32(base + CSI2_REG_RX_CSI_DLY_CNT_SETTLE, n == 0 ? clk_settle : dat_settle); } } diff --git a/drivers/staging/media/atomisp/pci/atomisp_csi2.h b/drivers/staging/media/atomisp/pci/atomisp_csi2.h index 739c26f0807a..59261e8f1a1a 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_csi2.h +++ b/drivers/staging/media/atomisp/pci/atomisp_csi2.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h b/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h index 9680f211d424..8f1cc3fca13a 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h +++ b/drivers/staging/media/atomisp/pci/atomisp_dfs_tables.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_drvfs.c b/drivers/staging/media/atomisp/pci/atomisp_drvfs.c index 4a6ea021ddcc..fe0e2bfde27f 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_drvfs.c +++ b/drivers/staging/media/atomisp/pci/atomisp_drvfs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for atomisp driver sysfs interface * @@ -24,6 +25,7 @@ #include "atomisp_ioctl.h" #include "atomisp_drvfs.h" #include "hmm/hmm.h" +#include "ia_css_debug.h" /* * _iunit_debug: @@ -94,7 +96,7 @@ opt_err: static ssize_t iunit_dbglvl_show(struct device_driver *drv, char *buf) { - iunit_debug.dbglvl = atomisp_css_debug_get_dtrace_level(); + iunit_debug.dbglvl = dbg_level; return sprintf(buf, "dtrace level:%u\n", iunit_debug.dbglvl); } @@ -106,7 +108,7 @@ static ssize_t iunit_dbglvl_store(struct device_driver *drv, const char *buf, || iunit_debug.dbglvl > 9) { return -ERANGE; } - atomisp_css_debug_set_dtrace_level(iunit_debug.dbglvl); + ia_css_debug_set_dtrace_level(iunit_debug.dbglvl); return size; } diff --git a/drivers/staging/media/atomisp/pci/atomisp_drvfs.h b/drivers/staging/media/atomisp/pci/atomisp_drvfs.h index 7c99240d107a..4911037231fb 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_drvfs.h +++ b/drivers/staging/media/atomisp/pci/atomisp_drvfs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for atomisp driver sysfs interface. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_file.c b/drivers/staging/media/atomisp/pci/atomisp_file.c index 4ab0390316cf..e568ca99c45a 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_file.c +++ b/drivers/staging/media/atomisp/pci/atomisp_file.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -50,11 +51,12 @@ static void file_work(struct work_struct *work) V4L2_SUBDEV_FORMAT_ACTIVE, ATOMISP_SUBDEV_PAD_SINK); - while (!atomisp_css_isp_has_started()) + while (!ia_css_isp_has_started()) usleep_range(1000, 1500); - atomisp_css_send_input_frame(asd, buf, isp_sink_fmt.width, - isp_sink_fmt.height); + ia_css_stream_send_input_frame(asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream, + buf, isp_sink_fmt.width, + isp_sink_fmt.height); dev_dbg(isp->dev, "<%s: streaming done\n", __func__); } @@ -217,7 +219,7 @@ int atomisp_file_input_init(struct atomisp_device *isp) v4l2_subdev_init(sd, &file_input_ops); sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; - strcpy(sd->name, "file_input_subdev"); + strscpy(sd->name, "file_input_subdev", sizeof(sd->name)); v4l2_set_subdevdata(sd, file_dev); pads[0].flags = MEDIA_PAD_FL_SINK; diff --git a/drivers/staging/media/atomisp/pci/atomisp_file.h b/drivers/staging/media/atomisp/pci/atomisp_file.h index e38f8bc389f1..f166a2aefff1 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_file.h +++ b/drivers/staging/media/atomisp/pci/atomisp_file.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.c b/drivers/staging/media/atomisp/pci/atomisp_fops.c index 667d6f7d1d5e..453bb6913550 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_fops.c +++ b/drivers/staging/media/atomisp/pci/atomisp_fops.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -34,11 +35,8 @@ #include "atomisp-regs.h" #include "hmm/hmm.h" -#include "hrt/hive_isp_css_mm_hrt.h" - #include "type_support.h" #include "device_access/device_access.h" -#include "memory_access/memory_access.h" #include "atomisp_acc.h" @@ -88,7 +86,7 @@ static int atomisp_buf_prepare(struct videobuf_queue *vq, static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_metadata_buf *metadata_buf; enum atomisp_metadata_type md_type = @@ -128,7 +126,7 @@ static int atomisp_q_one_metadata_buffer(struct atomisp_sub_device *asd, static int atomisp_q_one_s3a_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_s3a_buf *s3a_buf; struct list_head *s3a_list; @@ -170,7 +168,7 @@ static int atomisp_q_one_s3a_buffer(struct atomisp_sub_device *asd, static int atomisp_q_one_dis_buffer(struct atomisp_sub_device *asd, enum atomisp_input_stream_id stream_id, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_pipe_id css_pipe_id) { struct atomisp_dis_buf *dis_buf; unsigned long irqflags; @@ -213,12 +211,12 @@ static int atomisp_q_one_dis_buffer(struct atomisp_sub_device *asd, int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, struct atomisp_video_pipe *pipe, enum atomisp_input_stream_id stream_id, - enum atomisp_css_buffer_type css_buf_type, - enum atomisp_css_pipe_id css_pipe_id) + enum ia_css_buffer_type css_buf_type, + enum ia_css_pipe_id css_pipe_id) { struct videobuf_vmalloc_memory *vm_mem; struct atomisp_css_params_with_list *param; - struct atomisp_css_dvs_grid_info *dvs_grid = + struct ia_css_dvs_grid_info *dvs_grid = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); unsigned long irqflags; int err = 0; @@ -255,14 +253,13 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, err = atomisp_calculate_real_zoom_region(asd, ¶m->params.dz_config, css_pipe_id); if (!err) - atomisp_css_set_dz_config(asd, - ¶m->params.dz_config); + asd->params.config.dz_config = ¶m->params.dz_config; } atomisp_css_set_isp_config_applied_frame(asd, vm_mem->vaddr); atomisp_css_update_isp_params_on_pipe(asd, asd->stream_env[stream_id].pipes[css_pipe_id]); - asd->params.dvs_6axis = (struct atomisp_css_dvs_6axis *) + asd->params.dvs_6axis = (struct ia_css_dvs_6axis_config *) param->params.dvs_6axis; /* @@ -302,19 +299,19 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, /* enqueue 3A/DIS/metadata buffers */ if (asd->params.curr_grid_info.s3a_grid.enable && css_pipe_id == asd->params.s3a_enabled_pipe && - css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) + css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) atomisp_q_one_s3a_buffer(asd, stream_id, css_pipe_id); if (asd->stream_env[ATOMISP_INPUT_STREAM_GENERAL].stream_info. metadata_info.size && - css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) + css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) atomisp_q_one_metadata_buffer(asd, stream_id, css_pipe_id); if (dvs_grid && dvs_grid->enable && - css_pipe_id == CSS_PIPE_ID_VIDEO && - css_buf_type == CSS_BUFFER_TYPE_OUTPUT_FRAME) + css_pipe_id == IA_CSS_PIPE_ID_VIDEO && + css_buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME) atomisp_q_one_dis_buffer(asd, stream_id, css_pipe_id); } @@ -323,7 +320,7 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, } static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd, - enum atomisp_css_pipe_id pipe_id, + enum ia_css_pipe_id pipe_id, uint16_t source_pad) { if (ATOMISP_USE_YUVPP(asd)) { @@ -331,24 +328,24 @@ static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd, if (asd->continuous_mode->val && asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) { if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) - return CSS_BUFFER_TYPE_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME; else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) - return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME; else - return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; } /*when run SDV case*/ if (asd->continuous_mode->val && asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) { if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE) - return CSS_BUFFER_TYPE_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME; else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW) - return CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME; else if (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO) - return CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_SEC_OUTPUT_FRAME; else - return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; } /*other case: default setting*/ @@ -356,27 +353,27 @@ static int atomisp_get_css_buf_type(struct atomisp_sub_device *asd, source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO || (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW && asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO)) - return CSS_BUFFER_TYPE_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME; else - return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; } - if (pipe_id == CSS_PIPE_ID_COPY || + if (pipe_id == IA_CSS_PIPE_ID_COPY || source_pad == ATOMISP_SUBDEV_PAD_SOURCE_CAPTURE || source_pad == ATOMISP_SUBDEV_PAD_SOURCE_VIDEO || (source_pad == ATOMISP_SUBDEV_PAD_SOURCE_PREVIEW && asd->run_mode->val != ATOMISP_RUN_MODE_VIDEO)) - return CSS_BUFFER_TYPE_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_OUTPUT_FRAME; else - return CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; + return IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME; } static int atomisp_qbuffers_to_css_for_all_pipes(struct atomisp_sub_device *asd) { - enum atomisp_css_buffer_type buf_type; - enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_COPY; - enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_COPY; - enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_COPY; + enum ia_css_buffer_type buf_type; + enum ia_css_pipe_id css_capture_pipe_id = IA_CSS_PIPE_ID_COPY; + enum ia_css_pipe_id css_preview_pipe_id = IA_CSS_PIPE_ID_COPY; + enum ia_css_pipe_id css_video_pipe_id = IA_CSS_PIPE_ID_COPY; enum atomisp_input_stream_id input_stream_id; struct atomisp_video_pipe *capture_pipe; struct atomisp_video_pipe *preview_pipe; @@ -413,10 +410,10 @@ static int atomisp_qbuffers_to_css_for_all_pipes(struct atomisp_sub_device *asd) /* queue all available buffers to css */ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) { - enum atomisp_css_buffer_type buf_type; - enum atomisp_css_pipe_id css_capture_pipe_id = CSS_PIPE_ID_NUM; - enum atomisp_css_pipe_id css_preview_pipe_id = CSS_PIPE_ID_NUM; - enum atomisp_css_pipe_id css_video_pipe_id = CSS_PIPE_ID_NUM; + enum ia_css_buffer_type buf_type; + enum ia_css_pipe_id css_capture_pipe_id = IA_CSS_PIPE_ID_NUM; + enum ia_css_pipe_id css_preview_pipe_id = IA_CSS_PIPE_ID_NUM; + enum ia_css_pipe_id css_video_pipe_id = IA_CSS_PIPE_ID_NUM; enum atomisp_input_stream_id input_stream_id; struct atomisp_video_pipe *capture_pipe = NULL; struct atomisp_video_pipe *vf_pipe = NULL; @@ -432,43 +429,43 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) { video_pipe = &asd->video_out_video_capture; - css_video_pipe_id = CSS_PIPE_ID_VIDEO; + css_video_pipe_id = IA_CSS_PIPE_ID_VIDEO; } else if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) { preview_pipe = &asd->video_out_capture; - css_preview_pipe_id = CSS_PIPE_ID_CAPTURE; + css_preview_pipe_id = IA_CSS_PIPE_ID_CAPTURE; } else if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) { if (asd->continuous_mode->val) { capture_pipe = &asd->video_out_capture; vf_pipe = &asd->video_out_vf; - css_capture_pipe_id = CSS_PIPE_ID_CAPTURE; + css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE; } video_pipe = &asd->video_out_video_capture; preview_pipe = &asd->video_out_preview; - css_video_pipe_id = CSS_PIPE_ID_VIDEO; - css_preview_pipe_id = CSS_PIPE_ID_VIDEO; + css_video_pipe_id = IA_CSS_PIPE_ID_VIDEO; + css_preview_pipe_id = IA_CSS_PIPE_ID_VIDEO; } else if (asd->continuous_mode->val) { capture_pipe = &asd->video_out_capture; vf_pipe = &asd->video_out_vf; preview_pipe = &asd->video_out_preview; - css_preview_pipe_id = CSS_PIPE_ID_PREVIEW; - css_capture_pipe_id = CSS_PIPE_ID_CAPTURE; + css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW; + css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE; } else if (asd->run_mode->val == ATOMISP_RUN_MODE_PREVIEW) { preview_pipe = &asd->video_out_preview; - css_preview_pipe_id = CSS_PIPE_ID_PREVIEW; + css_preview_pipe_id = IA_CSS_PIPE_ID_PREVIEW; } else { /* ATOMISP_RUN_MODE_STILL_CAPTURE */ capture_pipe = &asd->video_out_capture; if (!raw_mode) vf_pipe = &asd->video_out_vf; - css_capture_pipe_id = CSS_PIPE_ID_CAPTURE; + css_capture_pipe_id = IA_CSS_PIPE_ID_CAPTURE; } #ifdef ISP2401_NEW_INPUT_SYSTEM if (asd->copy_mode) { - css_capture_pipe_id = CSS_PIPE_ID_COPY; - css_preview_pipe_id = CSS_PIPE_ID_COPY; - css_video_pipe_id = CSS_PIPE_ID_COPY; + css_capture_pipe_id = IA_CSS_PIPE_ID_COPY; + css_preview_pipe_id = IA_CSS_PIPE_ID_COPY; + css_video_pipe_id = IA_CSS_PIPE_ID_COPY; } #endif @@ -476,9 +473,9 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) capture_pipe = &asd->video_out_capture; video_pipe = &asd->video_out_video_capture; preview_pipe = &asd->video_out_preview; - css_capture_pipe_id = CSS_PIPE_ID_COPY; - css_video_pipe_id = CSS_PIPE_ID_YUVPP; - css_preview_pipe_id = CSS_PIPE_ID_YUVPP; + css_capture_pipe_id = IA_CSS_PIPE_ID_COPY; + css_video_pipe_id = IA_CSS_PIPE_ID_YUVPP; + css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP; } if (capture_pipe) { @@ -491,7 +488,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) * use yuvpp pipe for SOC camera. */ if (ATOMISP_USE_YUVPP(asd)) - css_capture_pipe_id = CSS_PIPE_ID_YUVPP; + css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP; atomisp_q_video_buffers_to_css(asd, capture_pipe, input_stream_id, @@ -511,7 +508,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) * use yuvpp pipe for SOC camera. */ if (ATOMISP_USE_YUVPP(asd)) - css_capture_pipe_id = CSS_PIPE_ID_YUVPP; + css_capture_pipe_id = IA_CSS_PIPE_ID_YUVPP; atomisp_q_video_buffers_to_css(asd, vf_pipe, input_stream_id, buf_type, css_capture_pipe_id); @@ -521,10 +518,10 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) buf_type = atomisp_get_css_buf_type( asd, css_preview_pipe_id, atomisp_subdev_source_pad(&preview_pipe->vdev)); - if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == CSS_PIPE_ID_YUVPP) + if (ATOMISP_SOC_CAMERA(asd) && css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP) input_stream_id = ATOMISP_INPUT_STREAM_GENERAL; /* else for ext isp use case */ - else if (css_preview_pipe_id == CSS_PIPE_ID_YUVPP) + else if (css_preview_pipe_id == IA_CSS_PIPE_ID_YUVPP) input_stream_id = ATOMISP_INPUT_STREAM_VIDEO; else if (asd->stream_env[ATOMISP_INPUT_STREAM_PREVIEW].stream) input_stream_id = ATOMISP_INPUT_STREAM_PREVIEW; @@ -535,7 +532,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) * use yuvpp pipe for SOC camera. */ if (ATOMISP_USE_YUVPP(asd)) - css_preview_pipe_id = CSS_PIPE_ID_YUVPP; + css_preview_pipe_id = IA_CSS_PIPE_ID_YUVPP; atomisp_q_video_buffers_to_css(asd, preview_pipe, input_stream_id, @@ -555,7 +552,7 @@ int atomisp_qbuffers_to_css(struct atomisp_sub_device *asd) * use yuvpp pipe for SOC camera. */ if (ATOMISP_USE_YUVPP(asd)) - css_video_pipe_id = CSS_PIPE_ID_YUVPP; + css_video_pipe_id = IA_CSS_PIPE_ID_YUVPP; atomisp_q_video_buffers_to_css(asd, video_pipe, input_stream_id, @@ -717,7 +714,7 @@ static void atomisp_subdev_init_struct(struct atomisp_sub_device *asd) asd->params.online_process = 1; asd->params.yuv_ds_en = 0; /* s3a grid not enabled for any pipe */ - asd->params.s3a_enabled_pipe = CSS_PIPE_ID_NUM; + asd->params.s3a_enabled_pipe = IA_CSS_PIPE_ID_NUM; asd->params.offline_parm.num_captures = 1; asd->params.offline_parm.skip_frames = 0; @@ -1000,7 +997,7 @@ subdev_uninit: atomisp_css_uninit(isp); if (defer_fw_load) { - atomisp_css_unload_firmware(isp); + ia_css_unload_firmware(); isp->css_env.isp_css_fw.data = NULL; isp->css_env.isp_css_fw.bytes = 0; } @@ -1054,7 +1051,7 @@ static int do_isp_mm_remap(struct atomisp_device *isp, } static int frame_mmap(struct atomisp_device *isp, - const struct atomisp_css_frame *frame, struct vm_area_struct *vma) + const struct ia_css_frame *frame, struct vm_area_struct *vma) { ia_css_ptr isp_virt; u32 host_virt; @@ -1127,7 +1124,7 @@ int atomisp_videobuf_mmap_mapper(struct videobuf_queue *q, * There is also padding on the right (padded_width - width). */ static int remove_pad_from_frame(struct atomisp_device *isp, - struct atomisp_css_frame *in_frame, __u32 width, __u32 height) + struct ia_css_frame *in_frame, __u32 width, __u32 height) { unsigned int i; unsigned short *buffer; @@ -1164,7 +1161,7 @@ static int atomisp_mmap(struct file *file, struct vm_area_struct *vma) struct atomisp_device *isp = video_get_drvdata(vdev); struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev); struct atomisp_sub_device *asd = pipe->asd; - struct atomisp_css_frame *raw_virt_addr; + struct ia_css_frame *raw_virt_addr; u32 start = vma->vm_start; u32 end = vma->vm_end; u32 size = end - start; diff --git a/drivers/staging/media/atomisp/pci/atomisp_fops.h b/drivers/staging/media/atomisp/pci/atomisp_fops.h index e05e8f3a4442..3f1e442ba782 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_fops.h +++ b/drivers/staging/media/atomisp/pci/atomisp_fops.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -24,8 +25,8 @@ int atomisp_q_video_buffers_to_css(struct atomisp_sub_device *asd, struct atomisp_video_pipe *pipe, enum atomisp_input_stream_id stream_id, - enum atomisp_css_buffer_type css_buf_type, - enum atomisp_css_pipe_id css_pipe_id); + enum ia_css_buffer_type css_buf_type, + enum ia_css_pipe_id css_pipe_id); unsigned int atomisp_dev_users(struct atomisp_device *isp); unsigned int atomisp_sub_dev_users(struct atomisp_sub_device *asd); diff --git a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c index b096b7d30463..1af9da8acf4c 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c +++ b/drivers/staging/media/atomisp/pci/atomisp_gmin_platform.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 #include <linux/module.h> #include <linux/i2c.h> #include <linux/dmi.h> @@ -327,15 +328,6 @@ static struct gmin_cfg_var i8880_vars[] = { {}, }; -static struct gmin_cfg_var asus_vars[] = { - {"OVTI2680:00_CsiPort", "1"}, - {"OVTI2680:00_CsiLanes", "1"}, - {"OVTI2680:00_CsiFmt", "15"}, - {"OVTI2680:00_CsiBayer", "0"}, - {"OVTI2680:00_CamClk", "1"}, - {}, -}; - static const struct dmi_system_id gmin_vars[] = { { .ident = "BYT-T FFD8", @@ -373,13 +365,6 @@ static const struct dmi_system_id gmin_vars[] = { }, .driver_data = i8880_vars, }, - { - .ident = "T101HA", - .matches = { - DMI_MATCH(DMI_BOARD_NAME, "T101HA"), - }, - .driver_data = asus_vars, - }, {} }; @@ -387,6 +372,10 @@ static const struct dmi_system_id gmin_vars[] = { 0xa9, 0x71, 0xe8, 0x77, \ 0x75, 0x60, 0x68, 0xf7) +static const guid_t atomisp_dsm_guid = GUID_INIT(0xdc2f6c4f, 0x045b, 0x4f1d, + 0x97, 0xb9, 0x88, 0x2a, + 0x68, 0x60, 0xa4, 0xbe); + #define CFG_VAR_NAME_MAX 64 #define GMIN_PMC_CLK_NAME 14 /* "pmc_plt_clk_[0..5]" */ @@ -454,15 +443,28 @@ static int gmin_i2c_write(struct device *dev, u16 i2c_addr, u8 reg, static struct gmin_subdev *gmin_subdev_add(struct v4l2_subdev *subdev) { - int i, ret; - struct device *dev; struct i2c_client *power = NULL, *client = v4l2_get_subdevdata(subdev); + struct acpi_device *adev; + acpi_handle handle; + struct device *dev; + int i, ret; if (!client) return NULL; dev = &client->dev; + handle = ACPI_HANDLE(dev); + + // FIXME: may need to release resources allocated by acpi_bus_get_device() + if (!handle || acpi_bus_get_device(handle, &adev)) { + dev_err(dev, "Error could not get ACPI device\n"); + return NULL; + } + + dev_info(&client->dev, "%s: ACPI detected it on bus ID=%s, HID=%s\n", + __func__, acpi_device_bid(adev), acpi_device_hid(adev)); + if (!pmic_id) { if (gmin_i2c_dev_exists(dev, PMIC_ACPI_TI, &power)) pmic_id = PMIC_TI; @@ -479,7 +481,6 @@ static struct gmin_subdev *gmin_subdev_add(struct v4l2_subdev *subdev) if (i >= MAX_SUBDEVS) return NULL; - if (power) { gmin_subdevs[i].pwm_i2c_addr = power->addr; dev_info(dev, @@ -616,6 +617,7 @@ static int axp_regulator_set(struct device *dev, struct gmin_subdev *gs, static int axp_v1p8_on(struct device *dev, struct gmin_subdev *gs) { int ret; + ret = axp_regulator_set(dev, gs, gs->eldo2_sel_reg, gs->eldo2_1p8v, ELDO_CTRL_REG, gs->eldo2_ctrl_shift, true); if (ret) @@ -640,6 +642,7 @@ static int axp_v1p8_on(struct device *dev, struct gmin_subdev *gs) static int axp_v1p8_off(struct device *dev, struct gmin_subdev *gs) { int ret; + ret = axp_regulator_set(dev, gs, gs->eldo1_sel_reg, gs->eldo1_1p8v, ELDO_CTRL_REG, gs->eldo1_ctrl_shift, false); if (ret) @@ -650,7 +653,6 @@ static int axp_v1p8_off(struct device *dev, struct gmin_subdev *gs) return ret; } - static int gmin_gpio0_ctrl(struct v4l2_subdev *subdev, int on) { struct gmin_subdev *gs = find_gmin_subdev(subdev); @@ -753,7 +755,6 @@ static int gmin_v1p8_ctrl(struct v4l2_subdev *subdev, int on) dev_err(subdev->dev, "Couldn't set power mode for v1p2\n"); } - return -EINVAL; } @@ -921,7 +922,8 @@ int atomisp_gmin_register_vcm_control(struct camera_vcm_control *vcmCtrl) } EXPORT_SYMBOL_GPL(atomisp_gmin_register_vcm_control); -static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist, +static int gmin_get_hardcoded_var(struct device *dev, + struct gmin_cfg_var *varlist, const char *var8, char *out, size_t *out_len) { struct gmin_cfg_var *gv; @@ -932,11 +934,13 @@ static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist, if (strcmp(var8, gv->name)) continue; + dev_info(dev, "Found DMI entry for '%s'\n", var8); + vl = strlen(gv->val); if (vl > *out_len - 1) return -ENOSPC; - strcpy(out, gv->val); + strscpy(out, gv->val, *out_len); *out_len = vl; return 0; } @@ -944,6 +948,75 @@ static int gmin_get_hardcoded_var(struct gmin_cfg_var *varlist, return -EINVAL; } + +static int gmin_get_config_dsm_var(struct device *dev, + const char *var, + char *out, size_t *out_len) +{ + acpi_handle handle = ACPI_HANDLE(dev); + union acpi_object *obj, *cur = NULL; + int i; + + obj = acpi_evaluate_dsm(handle, &atomisp_dsm_guid, 0, 0, NULL); + if (!obj) { + dev_info_once(dev, "Didn't find ACPI _DSM table.\n"); + return -EINVAL; + } + +#if 0 /* Just for debugging purposes */ + for (i = 0; i < obj->package.count; i++) { + union acpi_object *cur = &obj->package.elements[i]; + + if (cur->type == ACPI_TYPE_INTEGER) + dev_info(dev, "object #%d, type %d, value: %lld\n", + i, cur->type, cur->integer.value); + else if (cur->type == ACPI_TYPE_STRING) + dev_info(dev, "object #%d, type %d, string: %s\n", + i, cur->type, cur->string.pointer); + else + dev_info(dev, "object #%d, type %d\n", + i, cur->type); + } +#endif + + /* Seek for the desired var */ + for (i = 0; i < obj->package.count - 1; i += 2) { + if (obj->package.elements[i].type == ACPI_TYPE_STRING && + !strcmp(obj->package.elements[i].string.pointer, var)) { + /* Next element should be the required value */ + cur = &obj->package.elements[i + 1]; + break; + } + } + + if (!cur) { + dev_info(dev, "didn't found _DSM entry for '%s'\n", var); + ACPI_FREE(obj); + return -EINVAL; + } + + /* + * While it could be possible to have an ACPI_TYPE_INTEGER, + * and read the value from cur->integer.value, the table + * seen so far uses the string type. So, produce a warning + * if it founds something different than string, letting it + * to fall back to the old code. + */ + if (cur && cur->type != ACPI_TYPE_STRING) { + dev_info(dev, "found non-string _DSM entry for '%s'\n", var); + ACPI_FREE(obj); + return -EINVAL; + } + + dev_info(dev, "found _DSM entry for '%s': %s\n", var, + cur->string.pointer); + strscpy(out, cur->string.pointer, *out_len); + *out_len = strlen(cur->string.pointer); + + ACPI_FREE(obj); + return 0; +} + /* Retrieves a device-specific configuration variable. The dev * argument should be a device with an ACPI companion, as all * configuration is based on firmware ID. @@ -953,12 +1026,21 @@ static int gmin_get_config_var(struct device *maindev, const char *var, char *out, size_t *out_len) { - char var8[CFG_VAR_NAME_MAX]; efi_char16_t var16[CFG_VAR_NAME_MAX]; - struct efivar_entry *ev; const struct dmi_system_id *id; - int i, ret; struct device *dev = maindev; + char var8[CFG_VAR_NAME_MAX]; + struct efivar_entry *ev; + int i, ret; + + /* For sensors, try first to use the _DSM table */ + if (!is_gmin) { + ret = gmin_get_config_dsm_var(maindev, var, out, out_len); + if (!ret) + return 0; + } + + /* Fall-back to other approaches */ if (!is_gmin && ACPI_COMPANION(dev)) dev = &ACPI_COMPANION(dev)->dev; @@ -977,9 +1059,10 @@ static int gmin_get_config_var(struct device *maindev, */ id = dmi_first_match(gmin_vars); if (id) { - dev_info(maindev, "Found DMI entry for '%s'\n", var8); - return gmin_get_hardcoded_var(id->driver_data, var8, out, - out_len); + ret = gmin_get_hardcoded_var(maindev, id->driver_data, var8, + out, out_len); + if (!ret) + return 0; } /* Our variable names are ASCII by construction, but EFI names @@ -1009,9 +1092,9 @@ static int gmin_get_config_var(struct device *maindev, *out_len = ev->var.DataSize; dev_info(maindev, "found EFI entry for '%s'\n", var8); } else if (is_gmin) { - dev_warn(maindev, "Failed to find gmin variable %s\n", var8); + dev_info(maindev, "Failed to find EFI gmin variable %s\n", var8); } else { - dev_warn(maindev, "Failed to find variable %s\n", var8); + dev_info(maindev, "Failed to find EFI variable %s\n", var8); } kfree(ev); @@ -1030,6 +1113,8 @@ int gmin_get_var_int(struct device *dev, bool is_gmin, const char *var, int def) if (!ret) { val[len] = 0; ret = kstrtol(val, 0, &result); + } else { + dev_info(dev, "%s: using default (%d)\n", var, def); } return ret ? def : result; diff --git a/drivers/staging/media/atomisp/pci/atomisp_helper.h b/drivers/staging/media/atomisp/pci/atomisp_helper.h deleted file mode 100644 index 56035063f81d..000000000000 --- a/drivers/staging/media/atomisp/pci/atomisp_helper.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Support for Medifield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ -#ifndef _atomisp_helper_h_ -#define _atomisp_helper_h_ -extern void __iomem *atomisp_io_base; - -static inline void __iomem *atomisp_get_io_virt_addr(unsigned int address) -{ - void __iomem *ret = atomisp_io_base + (address & 0x003FFFFF); - return ret; -} -#endif diff --git a/drivers/staging/media/atomisp/pci/atomisp_internal.h b/drivers/staging/media/atomisp/pci/atomisp_internal.h index 26539f3ffb9b..ff3becd41110 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_internal.h +++ b/drivers/staging/media/atomisp/pci/atomisp_internal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c index a5e71e5b714e..9404a678fa6f 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.c +++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -39,8 +40,6 @@ #include "device_access.h" #include "irq.h" -#include "hrt/hive_isp_css_mm_hrt.h" - static const char *DRIVER = "atomisp"; /* max size 15 */ static const char *CARD = "ATOM ISP"; /* max size 31 */ @@ -341,163 +340,163 @@ const struct atomisp_format_bridge atomisp_output_fmts[] = { .pixelformat = V4L2_PIX_FMT_YUV420, .depth = 12, .mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV420, - .sh_fmt = CSS_FRAME_FORMAT_YUV420, + .sh_fmt = IA_CSS_FRAME_FORMAT_YUV420, .description = "YUV420, planar", .planar = true }, { .pixelformat = V4L2_PIX_FMT_YVU420, .depth = 12, .mbus_code = V4L2_MBUS_FMT_CUSTOM_YVU420, - .sh_fmt = CSS_FRAME_FORMAT_YV12, + .sh_fmt = IA_CSS_FRAME_FORMAT_YV12, .description = "YVU420, planar", .planar = true }, { .pixelformat = V4L2_PIX_FMT_YUV422P, .depth = 16, .mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV422P, - .sh_fmt = CSS_FRAME_FORMAT_YUV422, + .sh_fmt = IA_CSS_FRAME_FORMAT_YUV422, .description = "YUV422, planar", .planar = true }, { .pixelformat = V4L2_PIX_FMT_YUV444, .depth = 24, .mbus_code = V4L2_MBUS_FMT_CUSTOM_YUV444, - .sh_fmt = CSS_FRAME_FORMAT_YUV444, + .sh_fmt = IA_CSS_FRAME_FORMAT_YUV444, .description = "YUV444" }, { .pixelformat = V4L2_PIX_FMT_NV12, .depth = 12, .mbus_code = V4L2_MBUS_FMT_CUSTOM_NV12, - .sh_fmt = CSS_FRAME_FORMAT_NV12, + .sh_fmt = IA_CSS_FRAME_FORMAT_NV12, .description = "NV12, Y-plane, CbCr interleaved", .planar = true }, { .pixelformat = V4L2_PIX_FMT_NV21, .depth = 12, .mbus_code = V4L2_MBUS_FMT_CUSTOM_NV21, - .sh_fmt = CSS_FRAME_FORMAT_NV21, + .sh_fmt = IA_CSS_FRAME_FORMAT_NV21, .description = "NV21, Y-plane, CbCr interleaved", .planar = true }, { .pixelformat = V4L2_PIX_FMT_NV16, .depth = 16, .mbus_code = V4L2_MBUS_FMT_CUSTOM_NV16, - .sh_fmt = CSS_FRAME_FORMAT_NV16, + .sh_fmt = IA_CSS_FRAME_FORMAT_NV16, .description = "NV16, Y-plane, CbCr interleaved", .planar = true }, { .pixelformat = V4L2_PIX_FMT_YUYV, .depth = 16, .mbus_code = V4L2_MBUS_FMT_CUSTOM_YUYV, - .sh_fmt = CSS_FRAME_FORMAT_YUYV, + .sh_fmt = IA_CSS_FRAME_FORMAT_YUYV, .description = "YUYV, interleaved" }, { .pixelformat = V4L2_PIX_FMT_UYVY, .depth = 16, .mbus_code = MEDIA_BUS_FMT_UYVY8_1X16, - .sh_fmt = CSS_FRAME_FORMAT_UYVY, + .sh_fmt = IA_CSS_FRAME_FORMAT_UYVY, .description = "UYVY, interleaved" }, { /* This one is for parallel sensors! DO NOT USE! */ .pixelformat = V4L2_PIX_FMT_UYVY, .depth = 16, .mbus_code = MEDIA_BUS_FMT_UYVY8_2X8, - .sh_fmt = CSS_FRAME_FORMAT_UYVY, + .sh_fmt = IA_CSS_FRAME_FORMAT_UYVY, .description = "UYVY, interleaved" }, { .pixelformat = V4L2_PIX_FMT_SBGGR16, .depth = 16, .mbus_code = V4L2_MBUS_FMT_CUSTOM_SBGGR16, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 16" }, { .pixelformat = V4L2_PIX_FMT_SBGGR8, .depth = 8, .mbus_code = MEDIA_BUS_FMT_SBGGR8_1X8, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 8" }, { .pixelformat = V4L2_PIX_FMT_SGBRG8, .depth = 8, .mbus_code = MEDIA_BUS_FMT_SGBRG8_1X8, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 8" }, { .pixelformat = V4L2_PIX_FMT_SGRBG8, .depth = 8, .mbus_code = MEDIA_BUS_FMT_SGRBG8_1X8, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 8" }, { .pixelformat = V4L2_PIX_FMT_SRGGB8, .depth = 8, .mbus_code = MEDIA_BUS_FMT_SRGGB8_1X8, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 8" }, { .pixelformat = V4L2_PIX_FMT_SBGGR10, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SBGGR10_1X10, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 10" }, { .pixelformat = V4L2_PIX_FMT_SGBRG10, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SGBRG10_1X10, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 10" }, { .pixelformat = V4L2_PIX_FMT_SGRBG10, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SGRBG10_1X10, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 10" }, { .pixelformat = V4L2_PIX_FMT_SRGGB10, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SRGGB10_1X10, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 10" }, { .pixelformat = V4L2_PIX_FMT_SBGGR12, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SBGGR12_1X12, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 12" }, { .pixelformat = V4L2_PIX_FMT_SGBRG12, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SGBRG12_1X12, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 12" }, { .pixelformat = V4L2_PIX_FMT_SGRBG12, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SGRBG12_1X12, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 12" }, { .pixelformat = V4L2_PIX_FMT_SRGGB12, .depth = 16, .mbus_code = MEDIA_BUS_FMT_SRGGB12_1X12, - .sh_fmt = CSS_FRAME_FORMAT_RAW, + .sh_fmt = IA_CSS_FRAME_FORMAT_RAW, .description = "Bayer 12" }, { .pixelformat = V4L2_PIX_FMT_RGB32, .depth = 32, .mbus_code = V4L2_MBUS_FMT_CUSTOM_RGB32, - .sh_fmt = CSS_FRAME_FORMAT_RGBA888, + .sh_fmt = IA_CSS_FRAME_FORMAT_RGBA888, .description = "32 RGB 8-8-8-8" }, { .pixelformat = V4L2_PIX_FMT_RGB565, .depth = 16, .mbus_code = MEDIA_BUS_FMT_BGR565_2X8_LE, - .sh_fmt = CSS_FRAME_FORMAT_RGB565, + .sh_fmt = IA_CSS_FRAME_FORMAT_RGB565, .description = "16 RGB 5-6-5" }, { .pixelformat = V4L2_PIX_FMT_JPEG, .depth = 8, .mbus_code = MEDIA_BUS_FMT_JPEG_1X8, - .sh_fmt = CSS_FRAME_FORMAT_BINARY_8, + .sh_fmt = IA_CSS_FRAME_FORMAT_BINARY_8, .description = "JPEG" }, #if 0 @@ -506,7 +505,7 @@ const struct atomisp_format_bridge atomisp_output_fmts[] = { .pixelformat = V4L2_PIX_FMT_CUSTOM_M10MO_RAW, .depth = 8, .mbus_code = V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, - .sh_fmt = CSS_FRAME_FORMAT_BINARY_8, + .sh_fmt = IA_CSS_FRAME_FORMAT_BINARY_8, .description = "Custom RAW for M10MO" }, #endif @@ -548,8 +547,8 @@ static int atomisp_querycap(struct file *file, void *fh, struct video_device *vdev = video_devdata(file); struct atomisp_device *isp = video_get_drvdata(vdev); - strscpy(cap->driver, DRIVER, sizeof(cap->driver) - 1); - strscpy(cap->card, CARD, sizeof(cap->card) - 1); + strscpy(cap->driver, DRIVER, sizeof(cap->driver)); + strscpy(cap->card, CARD, sizeof(cap->card)); snprintf(cap->bus_info, sizeof(cap->bus_info), "PCI:%s", pci_name(isp->pdev)); @@ -574,8 +573,8 @@ static int atomisp_enum_input(struct file *file, void *fh, return -EINVAL; memset(input, 0, sizeof(struct v4l2_input)); - strncpy(input->name, isp->inputs[index].camera->name, - sizeof(input->name) - 1); + strscpy(input->name, isp->inputs[index].camera->name, + sizeof(input->name)); /* * HACK: append actuator's name to sensor's @@ -583,7 +582,7 @@ static int atomisp_enum_input(struct file *file, void *fh, * ioctl is the only way to enum inputs + possible external actuators * for 3A tuning purpose. */ - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) motor = isp->inputs[index].motor; else motor = isp->motor; @@ -594,8 +593,8 @@ static int atomisp_enum_input(struct file *file, void *fh, if (max_size > 1) { input->name[cur_len] = '+'; - strncpy(&input->name[cur_len + 1], - motor->name, max_size - 1); + strscpy(&input->name[cur_len + 1], + motor->name, max_size); } } @@ -733,7 +732,7 @@ static int atomisp_s_input(struct file *file, void *fh, unsigned int input) goto error; } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { motor = isp->inputs[input].motor; } else { motor = isp->motor; @@ -789,7 +788,7 @@ static int atomisp_enum_fmt_cap(struct file *file, void *fh, * Is the atomisp-supported format is valid for the * sensor (configuration)? If not, skip it. */ - if (format->sh_fmt == CSS_FRAME_FORMAT_RAW + if (format->sh_fmt == IA_CSS_FRAME_FORMAT_RAW && format->mbus_code != code.code) continue; @@ -893,7 +892,7 @@ void atomisp_videobuf_free_buf(struct videobuf_buffer *vb) vm_mem = vb->priv; if (vm_mem && vm_mem->vaddr) { - atomisp_css_frame_free(vm_mem->vaddr); + ia_css_frame_free(vm_mem->vaddr); vm_mem->vaddr = NULL; } } @@ -920,7 +919,7 @@ int atomisp_alloc_css_stat_bufs(struct atomisp_sub_device *asd, struct atomisp_dis_buf *dis_buf = NULL, *_dis_buf; struct atomisp_metadata_buf *md_buf = NULL, *_md_buf; int count; - struct atomisp_css_dvs_grid_info *dvs_grid_info = + struct ia_css_dvs_grid_info *dvs_grid_info = atomisp_css_get_dvs_grid_info(&asd->params.curr_grid_info); unsigned int i; @@ -1024,8 +1023,8 @@ int __atomisp_reqbufs(struct file *file, void *fh, struct video_device *vdev = video_devdata(file); struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev); struct atomisp_sub_device *asd = pipe->asd; - struct atomisp_css_frame_info frame_info; - struct atomisp_css_frame *frame; + struct ia_css_frame_info frame_info; + struct ia_css_frame *frame; struct videobuf_vmalloc_memory *vm_mem; u16 source_pad = atomisp_subdev_source_pad(vdev); u16 stream_id = atomisp_source_pad_to_stream_id(asd, source_pad); @@ -1054,7 +1053,7 @@ int __atomisp_reqbufs(struct file *file, void *fh, atomisp_alloc_css_stat_bufs(asd, stream_id); /* - * for user pointer type, buffers are not really allcated here, + * for user pointer type, buffers are not really allocated here, * buffers are setup in QBUF operation through v4l2_buffer structure */ if (req->memory == V4L2_MEMORY_USERPTR) @@ -1069,7 +1068,7 @@ int __atomisp_reqbufs(struct file *file, void *fh, * memory management function */ for (i = 0; i < req->count; i++) { - if (atomisp_css_frame_allocate_from_info(&frame, &frame_info)) + if (ia_css_frame_allocate_from_info(&frame, &frame_info)) goto error; vm_mem = pipe->capq.bufs[i]->priv; vm_mem->vaddr = frame; @@ -1080,11 +1079,11 @@ int __atomisp_reqbufs(struct file *file, void *fh, error: while (i--) { vm_mem = pipe->capq.bufs[i]->priv; - atomisp_css_frame_free(vm_mem->vaddr); + ia_css_frame_free(vm_mem->vaddr); } if (asd->vf_frame) - atomisp_css_frame_free(asd->vf_frame); + ia_css_frame_free(asd->vf_frame); return -ENOMEM; } @@ -1152,8 +1151,8 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) struct atomisp_sub_device *asd = pipe->asd; struct videobuf_buffer *vb; struct videobuf_vmalloc_memory *vm_mem; - struct atomisp_css_frame_info frame_info; - struct atomisp_css_frame *handle = NULL; + struct ia_css_frame_info frame_info; + struct ia_css_frame *handle = NULL; u32 length; u32 pgnr; int ret = 0; @@ -1183,8 +1182,6 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) * address and reprograme out page table properly */ if (buf->memory == V4L2_MEMORY_USERPTR) { - struct hrt_userbuffer_attr attributes; - vb = pipe->capq.bufs[buf->index]; vm_mem = vb->priv; if (!vm_mem) { @@ -1204,45 +1201,9 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) goto error; } - attributes.pgnr = pgnr; - attributes.type = HRT_USR_PTR; -#ifdef CONFIG_ION - if (!atomisp_hw_is_isp2401) { - if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_ION) - attributes.type = HRT_USR_ION; - } else { - if (buf->type == V4L2_BUF_TYPE_VIDEO_CAPTURE_ION) { - attributes.type = HRT_USR_ION; - if (asd->ion_dev_fd->val != ION_FD_UNSET) { - dev_dbg(isp->dev, "ION buffer queued, share_fd=%lddev_fd=%d.\n", - buf->m.userptr, asd->ion_dev_fd->val); - /* - * Make sure the shared fd we just got - * from user space isn't larger than - * the space we have for it. - */ - if ((buf->m.userptr & - (ATOMISP_ION_DEVICE_FD_MASK)) != 0) { - dev_err(isp->dev, - "Error: v4l2 buffer fd:0X%0lX > 0XFFFF.\n", - buf->m.userptr); - ret = -EINVAL; - goto error; - } - buf->m.userptr |= asd->ion_dev_fd->val << - ATOMISP_ION_DEVICE_FD_OFFSET; - } else { - dev_err(isp->dev, "v4l2 buffer type is ION, \ - but no dev fd set from userspace.\n"); - ret = -EINVAL; - goto error; - } - } - } -#endif - ret = atomisp_css_frame_map(&handle, &frame_info, + ret = ia_css_frame_map(&handle, &frame_info, (void __user *)buf->m.userptr, - 0, &attributes); + 0, pgnr); if (ret) { dev_err(isp->dev, "Failed to map user buffer\n"); goto error; @@ -1250,7 +1211,7 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) if (vm_mem->vaddr) { mutex_lock(&pipe->capq.vb_lock); - atomisp_css_frame_free(vm_mem->vaddr); + ia_css_frame_free(vm_mem->vaddr); vm_mem->vaddr = NULL; vb->state = VIDEOBUF_NEEDS_INIT; mutex_unlock(&pipe->capq.vb_lock); @@ -1265,6 +1226,10 @@ static int atomisp_qbuf(struct file *file, void *fh, struct v4l2_buffer *buf) buf->flags |= V4L2_BUF_FLAG_MAPPED; buf->flags |= V4L2_BUF_FLAG_QUEUED; buf->flags &= ~V4L2_BUF_FLAG_DONE; + + /* + * For mmap, frames were allocated at request buffers + */ } done: @@ -1299,7 +1264,7 @@ done: } else { atomisp_qbuffers_to_css(asd); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (!atomisp_is_wdt_running(asd) && atomisp_buffers_queued(asd)) atomisp_wdt_start(asd); } else { @@ -1323,7 +1288,7 @@ done: pipe->capq.streaming && !asd->enable_raw_buffer_lock->val && asd->params.offline_parm.num_captures == 1) { - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { asd->pending_capture_request++; dev_dbg(isp->dev, "Add one pending capture request.\n"); } else { @@ -1401,7 +1366,7 @@ static int __get_frame_exp_id(struct atomisp_video_pipe *pipe, struct v4l2_buffer *buf) { struct videobuf_vmalloc_memory *vm_mem; - struct atomisp_css_frame *handle; + struct ia_css_frame *handle; int i; for (i = 0; pipe->capq.bufs[i]; i++) { @@ -1443,7 +1408,8 @@ static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) ret = videobuf_dqbuf(&pipe->capq, buf, file->f_flags & O_NONBLOCK); if (ret) { - dev_dbg(isp->dev, "<%s: %d\n", __func__, ret); + if (ret != -EAGAIN) + dev_dbg(isp->dev, "<%s: %d\n", __func__, ret); return ret; } rt_mutex_lock(&isp->mutex); @@ -1469,16 +1435,16 @@ static int atomisp_dqbuf(struct file *file, void *fh, struct v4l2_buffer *buf) return 0; } -enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd) +enum ia_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd) { if (ATOMISP_USE_YUVPP(asd)) - return CSS_PIPE_ID_YUVPP; + return IA_CSS_PIPE_ID_YUVPP; if (asd->continuous_mode->val) { if (asd->run_mode->val == ATOMISP_RUN_MODE_VIDEO) - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; else - return CSS_PIPE_ID_PREVIEW; + return IA_CSS_PIPE_ID_PREVIEW; } /* @@ -1486,7 +1452,7 @@ enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd) * scaling but it has one frame delay due to CSS internal buffering. */ if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_SCALER) - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; /* * Disable vf_pp and run CSS in still capture mode. In this mode @@ -1494,17 +1460,17 @@ enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd) * is not available. */ if (asd->vfpp->val == ATOMISP_VFPP_DISABLE_LOWLAT) - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; switch (asd->run_mode->val) { case ATOMISP_RUN_MODE_PREVIEW: - return CSS_PIPE_ID_PREVIEW; + return IA_CSS_PIPE_ID_PREVIEW; case ATOMISP_RUN_MODE_VIDEO: - return CSS_PIPE_ID_VIDEO; + return IA_CSS_PIPE_ID_VIDEO; case ATOMISP_RUN_MODE_STILL_CAPTURE: /* fall through */ default: - return CSS_PIPE_ID_CAPTURE; + return IA_CSS_PIPE_ID_CAPTURE; } } @@ -1598,7 +1564,7 @@ int atomisp_stream_on_master_slave_sensor(struct atomisp_device *isp, /* FIXME! ISP2400 */ static void __wdt_on_master_slave_sensor(struct atomisp_device *isp, - unsigned int wdt_duration) + unsigned int wdt_duration) { if (atomisp_buffers_queued(&isp->asd[0])) atomisp_wdt_refresh(&isp->asd[0], wdt_duration); @@ -1654,9 +1620,9 @@ static void atomisp_dma_burst_len_cfg(struct atomisp_sub_device *asd) ATOMISP_SUBDEV_PAD_SINK); if (sink->width * sink->height >= 4096 * 3072) - atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x7F); + atomisp_css2_hw_store_32(DMA_BURST_SIZE_REG, 0x7F); else - atomisp_store_uint32(DMA_BURST_SIZE_REG, 0x00); + atomisp_css2_hw_store_32(DMA_BURST_SIZE_REG, 0x00); } /* @@ -1669,7 +1635,7 @@ static int atomisp_streamon(struct file *file, void *fh, struct atomisp_video_pipe *pipe = atomisp_to_video_pipe(vdev); struct atomisp_sub_device *asd = pipe->asd; struct atomisp_device *isp = video_get_drvdata(vdev); - enum atomisp_css_pipe_id css_pipe_id; + enum ia_css_pipe_id css_pipe_id; unsigned int sensor_start_stream; unsigned int wdt_duration = ATOMISP_ISP_TIMEOUT_DURATION; int ret = 0; @@ -1721,7 +1687,7 @@ static int atomisp_streamon(struct file *file, void *fh, /* Reset pending capture request count. */ asd->pending_capture_request = 0; - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) asd->re_trigger_capture = false; if ((atomisp_subdev_streaming_count(asd) > sensor_start_stream) && @@ -1793,8 +1759,7 @@ static int atomisp_streamon(struct file *file, void *fh, if (asd->params.css_update_params_needed) { atomisp_apply_css_parameters(asd, &asd->params.css_param); if (asd->params.css_param.update_flag.dz_config) - atomisp_css_set_dz_config(asd, - &asd->params.css_param.dz_config); + asd->params.config.dz_config = &asd->params.css_param.dz_config; atomisp_css_update_isp_params(asd); asd->params.css_update_params_needed = false; memset(&asd->params.css_param.update_flag, 0, @@ -1835,7 +1800,7 @@ start_sensor: } if (!isp->sw_contex.file_input) { - atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF, + atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, atomisp_css_valid_sof(isp)); atomisp_csi2_configure(asd); /* @@ -1845,15 +1810,15 @@ start_sensor: if (atomisp_streaming_count(isp) > 1) { if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false) < 0) - dev_dbg(isp->dev, "dfs failed!\n"); + dev_dbg(isp->dev, "DFS max mode failed!\n"); } else { if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_AUTO, false) < 0) - dev_dbg(isp->dev, "dfs failed!\n"); + dev_dbg(isp->dev, "DFS auto mode failed!\n"); } } else { if (atomisp_freq_scaling(isp, ATOMISP_DFS_MODE_MAX, false) < 0) - dev_dbg(isp->dev, "dfs failed!\n"); + dev_dbg(isp->dev, "DFS max mode failed!\n"); } if (asd->depth_mode->val && atomisp_streaming_count(isp) == @@ -1863,7 +1828,7 @@ start_sensor: dev_err(isp->dev, "master slave sensor stream on failed!\n"); goto out; } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { __wdt_on_master_slave_sensor(isp, wdt_duration); } else { __wdt_on_master_slave_sensor_pipe(pipe, wdt_duration, true); @@ -1871,7 +1836,7 @@ start_sensor: goto start_delay_wq; } else if (asd->depth_mode->val && (atomisp_streaming_count(isp) < ATOMISP_DEPTH_SENSOR_STREAMON_COUNT)) { - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) __wdt_on_master_slave_sensor_pipe(pipe, wdt_duration, false); goto start_delay_wq; } @@ -1893,7 +1858,7 @@ start_sensor: goto out; } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (atomisp_buffers_queued(asd)) atomisp_wdt_refresh(asd, wdt_duration); } else { @@ -1933,7 +1898,7 @@ int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type) struct atomisp_video_pipe *preview_pipe = NULL; struct atomisp_video_pipe *video_pipe = NULL; struct videobuf_buffer *vb, *_vb; - enum atomisp_css_pipe_id css_pipe_id; + enum ia_css_pipe_id css_pipe_id; int ret; unsigned long flags; bool first_streamoff = false; @@ -2039,7 +2004,7 @@ int __atomisp_streamoff(struct file *file, void *fh, enum v4l2_buf_type type) atomisp_clear_css_buffer_counters(asd); if (!isp->sw_contex.file_input) - atomisp_css_irq_enable(isp, CSS_IRQ_INFO_CSS_RECEIVER_SOF, + atomisp_css_irq_enable(isp, IA_CSS_IRQ_INFO_CSS_RECEIVER_SOF, false); if (asd->delayed_init == ATOMISP_DELAYED_INIT_QUEUED) { @@ -2352,7 +2317,7 @@ static int atomisp_queryctl(struct file *file, void *fh, case V4L2_CID_FOCUS_ABSOLUTE: case V4L2_CID_FOCUS_RELATIVE: case V4L2_CID_FOCUS_STATUS: - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { return v4l2_queryctrl(isp->inputs[asd->input_curr].camera-> ctrl_handler, qc); } @@ -2393,7 +2358,7 @@ static int atomisp_camera_g_ext_ctrls(struct file *file, void *fh, int i; int ret = 0; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) motor = isp->inputs[asd->input_curr].motor; else motor = isp->motor; @@ -2502,8 +2467,7 @@ static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh, int i; int ret = 0; - - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) motor = isp->inputs[asd->input_curr].motor; else motor = isp->motor; @@ -2519,7 +2483,7 @@ static int atomisp_camera_s_ext_ctrls(struct file *file, void *fh, case V4L2_CID_EXPOSURE_METERING: case V4L2_CID_IRIS_ABSOLUTE: case V4L2_CID_FNUMBER_ABSOLUTE: - case V4L2_CID_VCM_TIMEING: + case V4L2_CID_VCM_TIMING: case V4L2_CID_VCM_SLEW: case V4L2_CID_3A_LOCK: case V4L2_CID_TEST_PATTERN: @@ -2732,7 +2696,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh, else asd = atomisp_to_video_pipe(vdev)->asd; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) motor = isp->inputs[asd->input_curr].motor; else motor = isp->motor; @@ -2757,7 +2721,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh, } switch (cmd) { case ATOMISP_IOC_S_SENSOR_RUNMODE: - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) err = atomisp_set_sensor_runmode(asd, arg); else err = -EINVAL; @@ -2932,7 +2896,7 @@ static long atomisp_vidioc_default(struct file *file, void *fh, core, ioctl, cmd, arg); break; case ATOMISP_IOC_G_UPDATE_EXPOSURE: - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) err = v4l2_subdev_call(isp->inputs[asd->input_curr].camera, core, ioctl, cmd, arg); else diff --git a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h index 5f3f2ec2539b..412bfcf33c0f 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_ioctl.h +++ b/drivers/staging/media/atomisp/pci/atomisp_ioctl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -43,7 +44,7 @@ int __atomisp_reqbufs(struct file *file, void *fh, int atomisp_reqbufs(struct file *file, void *fh, struct v4l2_requestbuffers *req); -enum atomisp_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device +enum ia_css_pipe_id atomisp_get_css_pipe_id(struct atomisp_sub_device *asd); void atomisp_videobuf_free_buf(struct videobuf_buffer *vb); diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.c b/drivers/staging/media/atomisp/pci/atomisp_subdev.c index 46590129cbe3..6ba817f15655 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_subdev.c +++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -30,29 +31,31 @@ #include "atomisp_internal.h" const struct atomisp_in_fmt_conv atomisp_in_fmt_conv[] = { - { MEDIA_BUS_FMT_SBGGR8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_8 }, - { MEDIA_BUS_FMT_SGBRG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_8 }, - { MEDIA_BUS_FMT_SGRBG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_8 }, - { MEDIA_BUS_FMT_SRGGB8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_8 }, - { MEDIA_BUS_FMT_SBGGR10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_10 }, - { MEDIA_BUS_FMT_SGBRG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_10 }, - { MEDIA_BUS_FMT_SGRBG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_10 }, - { MEDIA_BUS_FMT_SRGGB10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_10 }, - { MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_BGGR, CSS_FORMAT_RAW_12 }, - { MEDIA_BUS_FMT_SGBRG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GBRG, CSS_FORMAT_RAW_12 }, - { MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_GRBG, CSS_FORMAT_RAW_12 }, - { MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, CSS_BAYER_ORDER_RGGB, CSS_FORMAT_RAW_12 }, - { MEDIA_BUS_FMT_UYVY8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 }, - { MEDIA_BUS_FMT_YUYV8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0, ATOMISP_INPUT_FORMAT_YUV422_8 }, - { MEDIA_BUS_FMT_JPEG_1X8, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 }, - { V4L2_MBUS_FMT_CUSTOM_NV12, 12, 12, CSS_FRAME_FORMAT_NV12, 0, CSS_FRAME_FORMAT_NV12 }, - { V4L2_MBUS_FMT_CUSTOM_NV21, 12, 12, CSS_FRAME_FORMAT_NV21, 0, CSS_FRAME_FORMAT_NV21 }, - { V4L2_MBUS_FMT_CUSTOM_YUV420, 12, 12, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, 0, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY }, + { MEDIA_BUS_FMT_SBGGR8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_BGGR }, + { MEDIA_BUS_FMT_SGBRG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_GBRG }, + { MEDIA_BUS_FMT_SGRBG8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_GRBG }, + { MEDIA_BUS_FMT_SRGGB8_1X8, 8, 8, ATOMISP_INPUT_FORMAT_RAW_8, IA_CSS_BAYER_ORDER_RGGB }, + { MEDIA_BUS_FMT_SBGGR10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_BGGR }, + { MEDIA_BUS_FMT_SGBRG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_GBRG }, + { MEDIA_BUS_FMT_SGRBG10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_GRBG }, + { MEDIA_BUS_FMT_SRGGB10_1X10, 10, 10, ATOMISP_INPUT_FORMAT_RAW_10, IA_CSS_BAYER_ORDER_RGGB }, + { MEDIA_BUS_FMT_SBGGR12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_BGGR }, + { MEDIA_BUS_FMT_SGBRG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_GBRG }, + { MEDIA_BUS_FMT_SGRBG12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_GRBG }, + { MEDIA_BUS_FMT_SRGGB12_1X12, 12, 12, ATOMISP_INPUT_FORMAT_RAW_12, IA_CSS_BAYER_ORDER_RGGB }, + { MEDIA_BUS_FMT_UYVY8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0 }, + { MEDIA_BUS_FMT_YUYV8_1X16, 8, 8, ATOMISP_INPUT_FORMAT_YUV422_8, 0 }, +#if 0 // disabled due to clang warnings + { MEDIA_BUS_FMT_JPEG_1X8, 8, 8, IA_CSS_FRAME_FORMAT_BINARY_8, 0 }, + { V4L2_MBUS_FMT_CUSTOM_NV12, 12, 12, IA_CSS_FRAME_FORMAT_NV12, 0 }, + { V4L2_MBUS_FMT_CUSTOM_NV21, 12, 12, IA_CSS_FRAME_FORMAT_NV21, 0 }, +#endif + { V4L2_MBUS_FMT_CUSTOM_YUV420, 12, 12, ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, 0 }, #if 0 - { V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, 8, 8, CSS_FRAME_FORMAT_BINARY_8, 0, ATOMISP_INPUT_FORMAT_BINARY_8 }, + { V4L2_MBUS_FMT_CUSTOM_M10MO_RAW, 8, 8, IA_CSS_FRAME_FORMAT_BINARY_8, 0 }, #endif /* no valid V4L2 MBUS code for metadata format, so leave it 0. */ - { 0, 0, 0, ATOMISP_INPUT_FORMAT_EMBEDDED, 0, ATOMISP_INPUT_FORMAT_EMBEDDED }, + { 0, 0, 0, ATOMISP_INPUT_FORMAT_EMBEDDED, 0 }, {} }; @@ -639,7 +642,7 @@ void atomisp_subdev_set_ffmt(struct v4l2_subdev *sd, atomisp_css_input_set_bayer_order(isp_sd, stream_id, fc->bayer_order); atomisp_css_input_set_format(isp_sd, stream_id, - fc->css_stream_fmt); + fc->atomisp_in_fmt); atomisp_css_set_default_isys_config(isp_sd, stream_id, ffmt); } @@ -1219,15 +1222,10 @@ static int isp_subdev_init_entities(struct atomisp_sub_device *asd) v4l2_ctrl_new_custom(&asd->ctrl_handler, &ctrl_disable_dz, NULL); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { asd->select_isp_version = v4l2_ctrl_new_custom(&asd->ctrl_handler, &ctrl_select_isp_version, NULL); -#if 0 /* #ifdef CONFIG_ION */ - asd->ion_dev_fd = v4l2_ctrl_new_custom(&asd->ctrl_handler, - &ctrl_ion_dev_fd, - NULL); -#endif } /* Make controls visible on subdev as well. */ @@ -1348,8 +1346,7 @@ int atomisp_subdev_register_entities(struct atomisp_sub_device *asd, * Should any of those use V4L2_CAP_META_OUTPUT? Probably yes. */ - device_caps = V4L2_CAP_IO_MC | - V4L2_CAP_VIDEO_CAPTURE | + device_caps = V4L2_CAP_VIDEO_CAPTURE | V4L2_CAP_STREAMING; /* Register the subdev and video node. */ diff --git a/drivers/staging/media/atomisp/pci/atomisp_subdev.h b/drivers/staging/media/atomisp/pci/atomisp_subdev.h index b0d561224beb..330a77eed8aa 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_subdev.h +++ b/drivers/staging/media/atomisp/pci/atomisp_subdev.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -59,8 +60,7 @@ struct atomisp_in_fmt_conv { u8 bpp; /* bits per pixel */ u8 depth; /* uncompressed */ enum atomisp_input_format atomisp_in_fmt; - enum atomisp_css_bayer_order bayer_order; - enum atomisp_input_format css_stream_fmt; + enum ia_css_bayer_order bayer_order; }; struct atomisp_sub_device; @@ -215,8 +215,8 @@ struct atomisp_subdev_params { unsigned int histogram_elenum; /* Current grid info */ - struct atomisp_css_grid_info curr_grid_info; - enum atomisp_css_pipe_id s3a_enabled_pipe; + struct ia_css_grid_info curr_grid_info; + enum ia_css_pipe_id s3a_enabled_pipe; int s3a_output_bytes; @@ -225,7 +225,7 @@ struct atomisp_subdev_params { struct ia_css_dz_config dz_config; /** Digital Zoom */ struct ia_css_capture_config capture_config; - struct atomisp_css_isp_config config; + struct ia_css_isp_config config; /* current configurations */ struct atomisp_css_params css_param; @@ -240,7 +240,7 @@ struct atomisp_subdev_params { u32 metadata_width_size; struct ia_css_dvs2_statistics *dvs_stat; - struct atomisp_css_dvs_6axis *dvs_6axis; + struct ia_css_dvs_6axis_config *dvs_6axis; u32 exp_id; int dvs_hor_coef_bytes; int dvs_ver_coef_bytes; @@ -265,7 +265,7 @@ struct atomisp_css_params_with_list { }; struct atomisp_acc_fw { - struct atomisp_css_fw_info *fw; + struct ia_css_fw_info *fw; unsigned int handle; unsigned int flags; unsigned int type; @@ -323,7 +323,7 @@ struct atomisp_sub_device { struct { struct list_head fw; struct list_head memory_maps; - struct atomisp_css_pipeline *pipeline; + struct ia_css_pipe *pipeline; bool extension_mode; struct ida ida; struct completion acc_done; @@ -335,11 +335,11 @@ struct atomisp_sub_device { struct atomisp_stream_env stream_env[ATOMISP_INPUT_STREAM_NUM]; struct v4l2_pix_format dvs_envelop; - unsigned int s3a_bufs_in_css[CSS_PIPE_ID_NUM]; + unsigned int s3a_bufs_in_css[IA_CSS_PIPE_ID_NUM]; unsigned int dis_bufs_in_css; unsigned int metadata_bufs_in_css - [ATOMISP_INPUT_STREAM_NUM][CSS_PIPE_ID_NUM]; + [ATOMISP_INPUT_STREAM_NUM][IA_CSS_PIPE_ID_NUM]; /* The list of free and available metadata buffers for CSS */ struct list_head metadata[ATOMISP_METADATA_TYPE_NUM]; /* The list of metadata buffers which have been en-queued to CSS */ @@ -358,8 +358,8 @@ struct atomisp_sub_device { struct list_head dis_stats_in_css; spinlock_t dis_stats_lock; - struct atomisp_css_frame *vf_frame; /* TODO: needed? */ - struct atomisp_css_frame *raw_output_frame; + struct ia_css_frame *vf_frame; /* TODO: needed? */ + struct ia_css_frame *raw_output_frame; enum atomisp_frame_status frame_status[VIDEO_MAX_FRAME]; /* This field specifies which camera (v4l2 input) is selected. */ diff --git a/drivers/staging/media/atomisp/pci/atomisp_tables.h b/drivers/staging/media/atomisp/pci/atomisp_tables.h index 22eac8a25dba..e718a3f661f9 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_tables.h +++ b/drivers/staging/media/atomisp/pci/atomisp_tables.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -22,25 +23,25 @@ #include "sh_css_params.h" /*Sepia image effect table*/ -static struct atomisp_css_cc_config sepia_cc_config = { +static struct ia_css_cc_config sepia_cc_config = { .fraction_bits = 8, .matrix = {141, 18, 68, -40, -5, -19, 35, 4, 16}, }; /*Negative image effect table*/ -static struct atomisp_css_cc_config nega_cc_config = { +static struct ia_css_cc_config nega_cc_config = { .fraction_bits = 8, .matrix = {255, 29, 120, 0, 374, 342, 0, 672, -301}, }; /*Mono image effect table*/ -static struct atomisp_css_cc_config mono_cc_config = { +static struct ia_css_cc_config mono_cc_config = { .fraction_bits = 8, .matrix = {255, 29, 120, 0, 0, 0, 0, 0, 0}, }; /*Skin whiten image effect table*/ -static struct atomisp_css_macc_table skin_low_macc_table = { +static struct ia_css_macc_table skin_low_macc_table = { .data = { 8192, 0, 0, 8192, 8192, 0, 0, 8192, @@ -61,7 +62,7 @@ static struct atomisp_css_macc_table skin_low_macc_table = { } }; -static struct atomisp_css_macc_table skin_medium_macc_table = { +static struct ia_css_macc_table skin_medium_macc_table = { .data = { 8192, 0, 0, 8192, 8192, 0, 0, 8192, @@ -82,7 +83,7 @@ static struct atomisp_css_macc_table skin_medium_macc_table = { } }; -static struct atomisp_css_macc_table skin_high_macc_table = { +static struct ia_css_macc_table skin_high_macc_table = { .data = { 8192, 0, 0, 8192, 8192, 0, 0, 8192, @@ -104,7 +105,7 @@ static struct atomisp_css_macc_table skin_high_macc_table = { }; /*Blue enhencement image effect table*/ -static struct atomisp_css_macc_table blue_macc_table = { +static struct ia_css_macc_table blue_macc_table = { .data = { 9728, -3072, 0, 8192, 8192, 0, 0, 8192, @@ -126,7 +127,7 @@ static struct atomisp_css_macc_table blue_macc_table = { }; /*Green enhencement image effect table*/ -static struct atomisp_css_macc_table green_macc_table = { +static struct ia_css_macc_table green_macc_table = { .data = { 8192, 0, 0, 8192, 8192, 0, 0, 8192, @@ -147,7 +148,7 @@ static struct atomisp_css_macc_table green_macc_table = { } }; -static struct atomisp_css_ctc_table vivid_ctc_table = { +static struct ia_css_ctc_table vivid_ctc_table = { .data.vamem_2 = { 0, 384, 837, 957, 1011, 1062, 1083, 1080, 1078, 1077, 1053, 1039, 1012, 992, 969, 951, diff --git a/drivers/staging/media/atomisp/pci/atomisp_tpg.c b/drivers/staging/media/atomisp/pci/atomisp_tpg.c index 97176b54d1ec..1def80bab180 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_tpg.c +++ b/drivers/staging/media/atomisp/pci/atomisp_tpg.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -147,7 +148,7 @@ int atomisp_tpg_init(struct atomisp_device *isp) tpg->isp = isp; v4l2_subdev_init(sd, &tpg_ops); sd->flags |= V4L2_SUBDEV_FL_HAS_DEVNODE; - strcpy(sd->name, "tpg_subdev"); + strscpy(sd->name, "tpg_subdev", sizeof(sd->name)); v4l2_set_subdevdata(sd, tpg); pads[0].flags = MEDIA_PAD_FL_SINK; diff --git a/drivers/staging/media/atomisp/pci/atomisp_tpg.h b/drivers/staging/media/atomisp/pci/atomisp_tpg.h index cf492d757773..4176e076f63d 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_tpg.h +++ b/drivers/staging/media/atomisp/pci/atomisp_tpg.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/atomisp_trace_event.h b/drivers/staging/media/atomisp/pci/atomisp_trace_event.h index 4d7a6794ee66..538d45e008b5 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_trace_event.h +++ b/drivers/staging/media/atomisp/pci/atomisp_trace_event.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support Camera Imaging tracer core. * @@ -43,7 +44,7 @@ TRACE_EVENT(camera_meminfo, ), TP_fast_assign( - strlcpy(__entry->name, name, 24); + strscpy(__entry->name, name, 24); __entry->uptr_size = uptr_size; __entry->counter = counter; __entry->sys_size = sys_size; @@ -73,8 +74,8 @@ TRACE_EVENT(camera_debug, ), TP_fast_assign( - strlcpy(__entry->name, name, 24); - strlcpy(__entry->info, info, 24); + strscpy(__entry->name, name, 24); + strscpy(__entry->info, info, 24); __entry->line = line; ), diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c index 98cff13d9f93..d36809a0182c 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.c +++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -42,7 +43,7 @@ #include "hmm/hmm.h" #include "atomisp_trace_event.h" -#include "hrt/hive_isp_css_mm_hrt.h" +#include "sh_css_firmware.h" #include "device_access.h" @@ -58,21 +59,21 @@ module_param(skip_fwload, uint, 0644); MODULE_PARM_DESC(skip_fwload, "Skip atomisp firmware load"); /* set reserved memory pool size in page */ -static unsigned int repool_pgnr; +static unsigned int repool_pgnr = 32768; module_param(repool_pgnr, uint, 0644); MODULE_PARM_DESC(repool_pgnr, - "Set the reserved memory pool size in page (default:0)"); + "Set the reserved memory pool size in page (default:32768)"); /* set dynamic memory pool size in page */ unsigned int dypool_pgnr = UINT_MAX; module_param(dypool_pgnr, uint, 0644); MODULE_PARM_DESC(dypool_pgnr, - "Set the dynamic memory pool size in page (default:0)"); + "Set the dynamic memory pool size in page (default: unlimited)"); -bool dypool_enable; +bool dypool_enable = true; module_param(dypool_enable, bool, 0644); MODULE_PARM_DESC(dypool_enable, - "dynamic memory pool enable/disable (default:disable)"); + "dynamic memory pool enable/disable (default:enabled)"); /* memory optimization: deferred firmware loading */ bool defer_fw_load; @@ -83,7 +84,7 @@ MODULE_PARM_DESC(defer_fw_load, /* cross componnet debug message flag */ int dbg_level; module_param(dbg_level, int, 0644); -MODULE_PARM_DESC(dbg_level, "debug message on/off (default:off)"); +MODULE_PARM_DESC(dbg_level, "debug message level (default:0)"); /* log function switch */ int dbg_func = 2; @@ -95,6 +96,10 @@ int mipicsi_flag; module_param(mipicsi_flag, int, 0644); MODULE_PARM_DESC(mipicsi_flag, "mipi csi compression predictor algorithm"); +static char firmware_name[256]; +module_param_string(firmware_name, firmware_name, sizeof(firmware_name), 0); +MODULE_PARM_DESC(firmware_name, "Firmware file name. Allows overriding the default firmware name."); + /*set to 16x16 since this is the amount of lines and pixels the sensor exports extra. If these are kept at the 10x8 that they were on, in yuv downscaling modes incorrect resolutions where requested to the sensor @@ -119,11 +124,6 @@ MODULE_PARM_DESC(pad_h, "extra data for ISP processing"); * be to replace this to something stored inside atomisp allocated * structures. */ -bool atomisp_hw_is_isp2401; - -/* Types of atomisp hardware */ -#define HW_IS_ISP2400 0 -#define HW_IS_ISP2401 1 struct device *atomisp_dev; @@ -349,52 +349,6 @@ static const struct atomisp_dfs_config dfs_config_byt = { .dfs_table_size = ARRAY_SIZE(dfs_rules_byt), }; -static const struct atomisp_freq_scaling_rule dfs_rules_byt_cr[] = { - { - .width = ISP_FREQ_RULE_ANY, - .height = ISP_FREQ_RULE_ANY, - .fps = ISP_FREQ_RULE_ANY, - .isp_freq = ISP_FREQ_320MHZ, - .run_mode = ATOMISP_RUN_MODE_VIDEO, - }, - { - .width = ISP_FREQ_RULE_ANY, - .height = ISP_FREQ_RULE_ANY, - .fps = ISP_FREQ_RULE_ANY, - .isp_freq = ISP_FREQ_320MHZ, - .run_mode = ATOMISP_RUN_MODE_STILL_CAPTURE, - }, - { - .width = ISP_FREQ_RULE_ANY, - .height = ISP_FREQ_RULE_ANY, - .fps = ISP_FREQ_RULE_ANY, - .isp_freq = ISP_FREQ_320MHZ, - .run_mode = ATOMISP_RUN_MODE_CONTINUOUS_CAPTURE, - }, - { - .width = ISP_FREQ_RULE_ANY, - .height = ISP_FREQ_RULE_ANY, - .fps = ISP_FREQ_RULE_ANY, - .isp_freq = ISP_FREQ_320MHZ, - .run_mode = ATOMISP_RUN_MODE_PREVIEW, - }, - { - .width = ISP_FREQ_RULE_ANY, - .height = ISP_FREQ_RULE_ANY, - .fps = ISP_FREQ_RULE_ANY, - .isp_freq = ISP_FREQ_320MHZ, - .run_mode = ATOMISP_RUN_MODE_SDV, - }, -}; - -static const struct atomisp_dfs_config dfs_config_byt_cr = { - .lowest_freq = ISP_FREQ_200MHZ, - .max_freq_at_vmin = ISP_FREQ_320MHZ, - .highest_freq = ISP_FREQ_320MHZ, - .dfs_table = dfs_rules_byt_cr, - .dfs_table_size = ARRAY_SIZE(dfs_rules_byt_cr), -}; - static const struct atomisp_freq_scaling_rule dfs_rules_cht[] = { { .width = ISP_FREQ_RULE_ANY, @@ -659,7 +613,7 @@ static int __maybe_unused atomisp_restore_iunit_reg(struct atomisp_device *isp) * which has bugs(like sighting:4567697 and 4567699) and * will be removed in B0 */ - atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1); + atomisp_css2_hw_store_32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1); return 0; } @@ -689,7 +643,7 @@ static int atomisp_mrfld_pre_power_down(struct atomisp_device *isp) if (!(irq & (1 << INTR_IIR))) goto done; - atomisp_store_uint32(MRFLD_INTR_CLEAR_REG, 0xFFFFFFFF); + atomisp_css2_hw_store_32(MRFLD_INTR_CLEAR_REG, 0xFFFFFFFF); atomisp_load_uint32(MRFLD_INTR_STATUS_REG, &irq); if (irq != 0) { dev_err(isp->dev, @@ -704,7 +658,7 @@ static int atomisp_mrfld_pre_power_down(struct atomisp_device *isp) pci_read_config_dword(dev, PCI_INTERRUPT_CTRL, &irq); if (!(irq & (1 << INTR_IIR))) { - atomisp_store_uint32(MRFLD_INTR_ENABLE_REG, 0x0); + atomisp_css2_hw_store_32(MRFLD_INTR_ENABLE_REG, 0x0); goto done; } dev_err(isp->dev, @@ -1084,15 +1038,15 @@ static int atomisp_subdev_probe(struct atomisp_device *isp) /* FIXME: should return -EPROBE_DEFER if not all subdevs were probed */ for (count = 0; count < SUBDEV_WAIT_TIMEOUT_MAX_COUNT; count++) { int camera_count = 0; + for (subdevs = pdata->subdevs; subdevs->type; ++subdevs) { if (subdevs->type == RAW_CAMERA || subdevs->type == SOC_CAMERA) - camera_count ++; + camera_count++; } if (camera_count) break; msleep(SUBDEV_WAIT_TIMEOUT); - count++; } /* Wait more time to give more time for subdev init code to finish */ msleep(5 * SUBDEV_WAIT_TIMEOUT); @@ -1143,9 +1097,9 @@ static int atomisp_subdev_probe(struct atomisp_device *isp) switch (subdevs->type) { case RAW_CAMERA: - raw_index = isp->input_cnt; dev_dbg(isp->dev, "raw_index: %d\n", raw_index); - /* pass-though */ + raw_index = isp->input_cnt; + /* fall through */ case SOC_CAMERA: dev_dbg(isp->dev, "SOC_INDEX: %d\n", isp->input_cnt); if (isp->input_cnt >= ATOM_ISP_MAX_INPUTS) { @@ -1250,7 +1204,7 @@ static int atomisp_register_entities(struct atomisp_device *isp) isp->media_dev.dev = isp->dev; - strlcpy(isp->media_dev.model, "Intel Atom ISP", + strscpy(isp->media_dev.model, "Intel Atom ISP", sizeof(isp->media_dev.model)); media_device_init(&isp->media_dev); @@ -1447,19 +1401,23 @@ atomisp_load_firmware(struct atomisp_device *isp) if (skip_fwload) return NULL; - if ((isp->media_dev.hw_revision >> ATOMISP_HW_REVISION_SHIFT) - == ATOMISP_HW_REVISION_ISP2401) - fw_path = "shisp_2401a0_v21.bin"; - - if (isp->media_dev.hw_revision == - ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT) - | ATOMISP_HW_STEPPING_A0)) - fw_path = "shisp_2401a0_legacy_v21.bin"; - - if (isp->media_dev.hw_revision == - ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT) - | ATOMISP_HW_STEPPING_B0)) - fw_path = "shisp_2400b0_v21.bin"; + if (firmware_name[0] != '\0') { + fw_path = firmware_name; + } else { + if ((isp->media_dev.hw_revision >> ATOMISP_HW_REVISION_SHIFT) + == ATOMISP_HW_REVISION_ISP2401) + fw_path = "shisp_2401a0_v21.bin"; + + if (isp->media_dev.hw_revision == + ((ATOMISP_HW_REVISION_ISP2401_LEGACY << ATOMISP_HW_REVISION_SHIFT) + | ATOMISP_HW_STEPPING_A0)) + fw_path = "shisp_2401a0_legacy_v21.bin"; + + if (isp->media_dev.hw_revision == + ((ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT) + | ATOMISP_HW_STEPPING_B0)) + fw_path = "shisp_2400b0_v21.bin"; + } if (!fw_path) { dev_err(isp->dev, "Unsupported hw_revision 0x%x\n", @@ -1494,21 +1452,17 @@ static bool is_valid_device(struct pci_dev *dev, switch (id->device & ATOMISP_PCI_DEVICE_SOC_MASK) { case ATOMISP_PCI_DEVICE_SOC_MRFLD: a0_max_id = ATOMISP_PCI_REV_MRFLD_A0_MAX; - atomisp_hw_is_isp2401 = false; name = "Merrifield"; break; case ATOMISP_PCI_DEVICE_SOC_BYT: a0_max_id = ATOMISP_PCI_REV_BYT_A0_MAX; - atomisp_hw_is_isp2401 = false; name = "Baytrail"; break; case ATOMISP_PCI_DEVICE_SOC_ANN: name = "Anniedale"; - atomisp_hw_is_isp2401 = true; break; case ATOMISP_PCI_DEVICE_SOC_CHT: name = "Cherrytrail"; - atomisp_hw_is_isp2401 = true; break; default: dev_err(&dev->dev, "%s: unknown device ID %x04:%x04\n", @@ -1528,13 +1482,13 @@ static bool is_valid_device(struct pci_dev *dev, */ #if defined(ISP2400) - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { dev_err(&dev->dev, "Support for %s (ISP2401) was disabled at compile time\n", name); return false; } #else - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { dev_err(&dev->dev, "Support for %s (ISP2400) was disabled at compile time\n", name); return false; @@ -1543,7 +1497,7 @@ static bool is_valid_device(struct pci_dev *dev, dev_info(&dev->dev, "Detected %s version %d (ISP240%c) on %s\n", name, dev->revision, - atomisp_hw_is_isp2401 ? '1' : '0', + IS_ISP2401 ? '1' : '0', product); return true; @@ -1564,7 +1518,8 @@ static int init_atomisp_wdts(struct atomisp_device *isp) for (i = 0; i < isp->num_of_streams; i++) { struct atomisp_sub_device *asd = &isp->asd[i]; - if (!atomisp_hw_is_isp2401) + + if (!IS_ISP2401) timer_setup(&asd->wdt, atomisp_wdt, 0); else { timer_setup(&asd->video_out_capture.wdt, @@ -1670,20 +1625,29 @@ static int atomisp_pci_probe(struct pci_dev *dev, (ATOMISP_HW_REVISION_ISP2400 << ATOMISP_HW_REVISION_SHIFT) | ATOMISP_HW_STEPPING_B0; -#ifdef FIXME - if (INTEL_MID_BOARD(3, TABLET, BYT, BLK, PRO, CRV2) || - INTEL_MID_BOARD(3, TABLET, BYT, BLK, ENG, CRV2)) { - isp->dfs = &dfs_config_byt_cr; - isp->hpll_freq = HPLL_FREQ_2000MHZ; - } else -#endif - { - isp->dfs = &dfs_config_byt; - isp->hpll_freq = HPLL_FREQ_1600MHZ; - } - /* HPLL frequency is known to be device-specific, but we don't + + /* + * Note: some Intel-based tablets with Android use a different + * DFS table. Based on the comments at the Yocto Aero meta + * version of this driver (at the ssid.h header), they're + * identified via a "spid" var: + * + * androidboot.spid=vend:cust:manu:plat:prod:hard + * + * As we don't have this upstream, nor we know enough details + * to use a DMI or PCI match table, the old code was just + * removed, but let's keep a note here as a reminder that, + * for certain devices, we may need to limit the max DFS + * frequency to be below certain values, adjusting the + * resolution accordingly. + */ + isp->dfs = &dfs_config_byt; + + /* + * HPLL frequency is known to be device-specific, but we don't * have specs yet for exactly how it varies. Default to - * BYT-CR but let provisioning set it via EFI variable */ + * BYT-CR but let provisioning set it via EFI variable + */ isp->hpll_freq = gmin_get_var_int(&dev->dev, false, "HpllFreq", HPLL_FREQ_2000MHZ); @@ -1735,7 +1699,7 @@ static int atomisp_pci_probe(struct pci_dev *dev, default: isp->hpll_freq = HPLL_FREQ_1600MHZ; dev_warn(isp->dev, - "read HPLL from cck failed.default 1600MHz.\n"); + "read HPLL from cck failed. Default to 1600 MHz.\n"); } break; default: @@ -1758,7 +1722,8 @@ static int atomisp_pci_probe(struct pci_dev *dev, goto load_fw_fail; } - err = atomisp_css_check_firmware_version(isp); + err = sh_css_check_firmware_version(isp->dev, + isp->firmware->data); if (err) { dev_dbg(&dev->dev, "Firmware version check failed\n"); goto fw_validation_fail; @@ -1787,7 +1752,7 @@ static int atomisp_pci_probe(struct pci_dev *dev, * bugs(like sighting:4567697 and 4567699) and will be removed * in B0 */ - atomisp_store_uint32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1); + atomisp_css2_hw_store_32(MRFLD_CSI_RECEIVER_SELECTION_REG, 1); if ((id->device & ATOMISP_PCI_DEVICE_SOC_MASK) == ATOMISP_PCI_DEVICE_SOC_MRFLD) { @@ -1938,7 +1903,7 @@ static void atomisp_pci_remove(struct pci_dev *dev) atomisp_acc_cleanup(isp); - atomisp_css_unload_firmware(isp); + ia_css_unload_firmware(); hmm_cleanup(); pm_runtime_forbid(&dev->dev); diff --git a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h index 87881fa6a698..81bb356b8172 100644 --- a/drivers/staging/media/atomisp/pci/atomisp_v4l2.h +++ b/drivers/staging/media/atomisp/pci/atomisp_v4l2.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h index 789a2e68cab8..0579deac5535 100644 --- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h +++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h index 09b049b3bd15..6fa6da859158 100644 --- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h +++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h index 47c488cec8ad..1071813a284c 100644 --- a/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h +++ b/drivers/staging/media/atomisp/pci/base/circbuf/interface/ia_css_circbuf_desc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c b/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c index 78e98268e188..d9f7c143794d 100644 --- a/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c +++ b/drivers/staging/media/atomisp/pci/base/circbuf/src/circbuf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h b/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h index 8cf3b0e0cc39..78cf0cbfb3be 100644 --- a/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h +++ b/drivers/staging/media/atomisp/pci/base/refcount/interface/ia_css_refcount.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,17 +17,18 @@ #define _IA_CSS_REFCOUNT_H_ #include <type_support.h> -#include <system_types.h> +#include <system_local.h> #include <ia_css_err.h> +#include <ia_css_types.h> -typedef void (*clear_func)(hrt_vaddress ptr); +typedef void (*clear_func)(ia_css_ptr ptr); /*! \brief Function for initializing refcount list * * \param[in] size Size of the refcount list. * \return ia_css_err */ -enum ia_css_err ia_css_refcount_init(uint32_t size); +int ia_css_refcount_init(uint32_t size); /*! \brief Function for de-initializing refcount list * @@ -38,9 +40,9 @@ void ia_css_refcount_uninit(void); * * \param[in] id ID of the object. * \param[in] ptr Data of the object (ptr). - * \return hrt_vaddress (saved address) + * \return ia_css_ptr (saved address) */ -hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr); +ia_css_ptr ia_css_refcount_increment(s32 id, ia_css_ptr ptr); /*! \brief Function for decrease reference by 1. * @@ -50,7 +52,7 @@ hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr); * - true, if it is successful. * - false, otherwise. */ -bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr); +bool ia_css_refcount_decrement(s32 id, ia_css_ptr ptr); /*! \brief Function to check if reference count is 1. * @@ -59,7 +61,7 @@ bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr); * - true, if it is successful. * - false, otherwise. */ -bool ia_css_refcount_is_single(hrt_vaddress ptr); +bool ia_css_refcount_is_single(ia_css_ptr ptr); /*! \brief Function to clear reference list objects. * @@ -78,6 +80,6 @@ void ia_css_refcount_clear(s32 id, * - true, if valid * - false, if invalid */ -bool ia_css_refcount_is_valid(hrt_vaddress ptr); +bool ia_css_refcount_is_valid(ia_css_ptr ptr); #endif /* _IA_CSS_REFCOUNT_H_ */ diff --git a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c index e39cc2132953..cf02737cf8d4 100644 --- a/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c +++ b/drivers/staging/media/atomisp/pci/base/refcount/src/refcount.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include "hmm.h" + #include "ia_css_refcount.h" -#include "memory_access/memory_access.h" #include "sh_css_defs.h" #include "platform_support.h" @@ -23,10 +25,10 @@ #include "ia_css_debug.h" /* TODO: enable for other memory aswell - now only for hrt_vaddress */ + now only for ia_css_ptr */ struct ia_css_refcount_entry { u32 count; - hrt_vaddress data; + ia_css_ptr data; s32 id; }; @@ -37,7 +39,7 @@ struct ia_css_refcount_list { static struct ia_css_refcount_list myrefcount; -static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr, +static struct ia_css_refcount_entry *refcount_find_entry(ia_css_ptr ptr, bool firstfree) { u32 i; @@ -46,7 +48,7 @@ static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr, return NULL; if (!myrefcount.items) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, - "refcount_find_entry(): Ref count not initiliazed!\n"); + "refcount_find_entry(): Ref count not initialized!\n"); return NULL; } @@ -65,25 +67,25 @@ static struct ia_css_refcount_entry *refcount_find_entry(hrt_vaddress ptr, return NULL; } -enum ia_css_err ia_css_refcount_init(uint32_t size) +int ia_css_refcount_init(uint32_t size) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (size == 0) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_refcount_init(): Size of 0 for Ref count init!\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if (myrefcount.items) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_refcount_init(): Ref count is already initialized\n"); - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } myrefcount.items = - sh_css_malloc(sizeof(struct ia_css_refcount_entry) * size); + kvmalloc(sizeof(struct ia_css_refcount_entry) * size, GFP_KERNEL); if (!myrefcount.items) - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - if (err == IA_CSS_SUCCESS) { + err = -ENOMEM; + if (!err) { memset(myrefcount.items, 0, sizeof(struct ia_css_refcount_entry) * size); myrefcount.size = size; @@ -114,14 +116,14 @@ void ia_css_refcount_uninit(void) entry->id = 0; } } - sh_css_free(myrefcount.items); + kvfree(myrefcount.items); myrefcount.items = NULL; myrefcount.size = 0; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_refcount_uninit() leave\n"); } -hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr) +ia_css_ptr ia_css_refcount_increment(s32 id, ia_css_ptr ptr) { struct ia_css_refcount_entry *entry; @@ -158,7 +160,7 @@ hrt_vaddress ia_css_refcount_increment(s32 id, hrt_vaddress ptr) return ptr; } -bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr) +bool ia_css_refcount_decrement(s32 id, ia_css_ptr ptr) { struct ia_css_refcount_entry *entry; @@ -201,7 +203,7 @@ bool ia_css_refcount_decrement(s32 id, hrt_vaddress ptr) return false; } -bool ia_css_refcount_is_single(hrt_vaddress ptr) +bool ia_css_refcount_is_single(ia_css_ptr ptr) { struct ia_css_refcount_entry *entry; @@ -262,7 +264,7 @@ void ia_css_refcount_clear(s32 id, clear_func clear_func_ptr) count); } -bool ia_css_refcount_is_valid(hrt_vaddress ptr) +bool ia_css_refcount_is_valid(ia_css_ptr ptr) { struct ia_css_refcount_entry *entry; diff --git a/drivers/staging/media/atomisp/pci/bits.h b/drivers/staging/media/atomisp/pci/bits.h index c6d2a5cba213..9fab02ebddc5 100644 --- a/drivers/staging/media/atomisp/pci/bits.h +++ b/drivers/staging/media/atomisp/pci/bits.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h index 551e8d7c5003..965cfda50707 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h +++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_binarydesc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -59,10 +60,10 @@ void ia_css_pipe_get_vfpp_binarydesc( * (= The numerator member in the sh_css_bds_factor structure.) * @param[out] bds_factor_denominator: The denominator of the bayer downscaling factor. * (= The denominator member in the sh_css_bds_factor structure.) - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err sh_css_bds_factor_get_numerator_denominator( +int sh_css_bds_factor_get_numerator_denominator( unsigned int bds_factor, unsigned int *bds_factor_numerator, unsigned int *bds_factor_denominator); @@ -75,10 +76,10 @@ enum ia_css_err sh_css_bds_factor_get_numerator_denominator( * @param[in/out] bds_out_info * @param[in/out] out_info * @param[in/out] vf_info - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipe_get_preview_binarydesc( +int ia_css_pipe_get_preview_binarydesc( struct ia_css_pipe *const pipe, struct ia_css_binary_descr *preview_descr, struct ia_css_frame_info *in_info, @@ -93,10 +94,10 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc( * @param[in/out] in_info * @param[in/out] bds_out_info * @param[in/out] vf_info - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipe_get_video_binarydesc( +int ia_css_pipe_get_video_binarydesc( struct ia_css_pipe *const pipe, struct ia_css_binary_descr *video_descr, struct ia_css_frame_info *in_info, @@ -287,9 +288,9 @@ void ia_css_pipe_get_ldc_binarydesc( * @param[in] input_res * @param[in] output_res * @param[in/out] bds_factor - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. */ -enum ia_css_err binarydesc_calculate_bds_factor( +int binarydesc_calculate_bds_factor( struct ia_css_resolution input_res, struct ia_css_resolution output_res, unsigned int *bds_factor); diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h index e58c9190310d..40c8145a0797 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h +++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_stagedesc.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h index ad60210abe95..c23d1bd915a3 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h +++ b/drivers/staging/media/atomisp/pci/camera/pipe/interface/ia_css_pipe_util.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c index c6b07d65ce3e..f20c9b02fbe0 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c +++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_binarydesc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -141,7 +142,7 @@ static struct sh_css_bds_factor bds_factors_list[] = { {8, 1, SH_CSS_BDS_FACTOR_8_00} }; -enum ia_css_err sh_css_bds_factor_get_numerator_denominator( +int sh_css_bds_factor_get_numerator_denominator( unsigned int bds_factor, unsigned int *bds_factor_numerator, unsigned int *bds_factor_denominator) @@ -153,16 +154,16 @@ enum ia_css_err sh_css_bds_factor_get_numerator_denominator( if (bds_factors_list[i].bds_factor == bds_factor) { *bds_factor_numerator = bds_factors_list[i].numerator; *bds_factor_denominator = bds_factors_list[i].denominator; - return IA_CSS_SUCCESS; + return 0; } } /* Throw an error since bds_factor cannot be found in bds_factors_list */ - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } -enum ia_css_err binarydesc_calculate_bds_factor( +int binarydesc_calculate_bds_factor( struct ia_css_resolution input_res, struct ia_css_resolution output_res, unsigned int *bds_factor) @@ -195,15 +196,15 @@ enum ia_css_err binarydesc_calculate_bds_factor( if (cond) { *bds_factor = bds_factors_list[i].bds_factor; - return IA_CSS_SUCCESS; + return 0; } } /* Throw an error since a suitable bds_factor cannot be found */ - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } -enum ia_css_err ia_css_pipe_get_preview_binarydesc( +int ia_css_pipe_get_preview_binarydesc( struct ia_css_pipe *const pipe, struct ia_css_binary_descr *preview_descr, struct ia_css_frame_info *in_info, @@ -211,7 +212,7 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc( struct ia_css_frame_info *out_info, struct ia_css_frame_info *vf_info) { - enum ia_css_err err; + int err; struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; int mode = IA_CSS_BINARY_MODE_PREVIEW; unsigned int i; @@ -264,7 +265,7 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc( binarydesc_calculate_bds_factor(in_info->res, bds_out_info->res, &preview_descr->required_bds_factor); - if (err != IA_CSS_SUCCESS) + if (err) return err; } else { bds_out_info->res.width = in_info->res.width / 2; @@ -318,11 +319,11 @@ enum ia_css_err ia_css_pipe_get_preview_binarydesc( preview_descr->enable_dpc = pipe->config.enable_dpc; preview_descr->isp_pipe_version = pipe->config.isp_pipe_version; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -enum ia_css_err ia_css_pipe_get_video_binarydesc( +int ia_css_pipe_get_video_binarydesc( struct ia_css_pipe *const pipe, struct ia_css_binary_descr *video_descr, struct ia_css_frame_info *in_info, @@ -334,7 +335,7 @@ enum ia_css_err ia_css_pipe_get_video_binarydesc( int mode = IA_CSS_BINARY_MODE_VIDEO; unsigned int i; struct ia_css_frame_info *out_infos[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool stream_dz_config = false; /* vf_info can be NULL */ @@ -407,7 +408,7 @@ enum ia_css_err ia_css_pipe_get_video_binarydesc( binarydesc_calculate_bds_factor( in_info->res, bds_out_info->res, &video_descr->required_bds_factor); - if (err != IA_CSS_SUCCESS) + if (err) return err; } else { bds_out_info->res.width = @@ -607,7 +608,7 @@ void ia_css_pipe_get_primary_binarydesc( * since it has better performance. */ if (pipe_version == IA_CSS_PIPE_VERSION_2_6_1) prim_descr->striped = false; - else if (!atomisp_hw_is_isp2401) { + else if (!IS_ISP2401) { prim_descr->striped = prim_descr->continuous && (!pipe->stream->stop_copy_preview || !pipe->stream->disable_cont_vf); } else { @@ -848,7 +849,7 @@ void ia_css_pipe_get_ldc_binarydesc( assert(out_info); IA_CSS_ENTER_PRIVATE(""); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { *in_info = *out_info; } else { if (pipe->out_yuv_ds_input_info.res.width) diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c index 43f63cc20f49..82a24aabe8ce 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c +++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_stagedesc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c index cc0631550724..03d9d168fcc9 100644 --- a/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c +++ b/drivers/staging/media/atomisp/pci/camera/pipe/src/pipe_util.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h index 75333166ed9b..59df44d696a0 100644 --- a/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h +++ b/drivers/staging/media/atomisp/pci/camera/util/interface/ia_css_util.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,8 +16,9 @@ #ifndef __IA_CSS_UTIL_H__ #define __IA_CSS_UTIL_H__ +#include <linux/errno.h> + #include <ia_css_err.h> -#include <error_support.h> #include <type_support.h> #include <ia_css_frame_public.h> #include <ia_css_stream_public.h> @@ -28,26 +30,26 @@ * @return "ia_css_err" error code * */ -enum ia_css_err ia_css_convert_errno( +int ia_css_convert_errno( int in_err); /* @brief check vf frame info. * * @param[in] info - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_util_check_vf_info( +int ia_css_util_check_vf_info( const struct ia_css_frame_info *const info); /* @brief check input configuration. * * @param[in] stream_config * @param[in] must_be_raw - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_util_check_input( +int ia_css_util_check_input( const struct ia_css_stream_config *const stream_config, bool must_be_raw, bool must_be_yuv); @@ -56,10 +58,10 @@ enum ia_css_err ia_css_util_check_input( * * @param[in] out_info * @param[in] vf_info - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_util_check_vf_out_info( +int ia_css_util_check_vf_out_info( const struct ia_css_frame_info *const out_info, const struct ia_css_frame_info *const vf_info); @@ -67,10 +69,10 @@ enum ia_css_err ia_css_util_check_vf_out_info( * * @param[in] width * @param[in] height - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_util_check_res( +int ia_css_util_check_res( unsigned int width, unsigned int height); diff --git a/drivers/staging/media/atomisp/pci/camera/util/src/util.c b/drivers/staging/media/atomisp/pci/camera/util/src/util.c index 217fe9cb54ff..40a71e37cc4e 100644 --- a/drivers/staging/media/atomisp/pci/camera/util/src/util.c +++ b/drivers/staging/media/atomisp/pci/camera/util/src/util.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -20,35 +21,6 @@ /* for ia_css_binary_max_vf_width() */ #include "ia_css_binary.h" -enum ia_css_err ia_css_convert_errno( - int in_err) -{ - enum ia_css_err out_err; - - switch (in_err) { - case 0: - out_err = IA_CSS_SUCCESS; - break; - case EINVAL: - out_err = IA_CSS_ERR_INVALID_ARGUMENTS; - break; - case ENODATA: - out_err = IA_CSS_ERR_QUEUE_IS_EMPTY; - break; - case ENOSYS: - case ENOTSUP: - out_err = IA_CSS_ERR_INTERNAL_ERROR; - break; - case ENOBUFS: - out_err = IA_CSS_ERR_QUEUE_IS_FULL; - break; - default: - out_err = IA_CSS_ERR_INTERNAL_ERROR; - break; - } - return out_err; -} - /* MW: Table look-up ??? */ unsigned int ia_css_util_input_format_bpp( enum atomisp_input_format format, @@ -113,49 +85,49 @@ unsigned int ia_css_util_input_format_bpp( return rval; } -enum ia_css_err ia_css_util_check_vf_info( +int ia_css_util_check_vf_info( const struct ia_css_frame_info *const info) { - enum ia_css_err err; + int err; unsigned int max_vf_width; assert(info); err = ia_css_frame_check_info(info); - if (err != IA_CSS_SUCCESS) + if (err) return err; max_vf_width = ia_css_binary_max_vf_width(); if (max_vf_width != 0 && info->res.width > max_vf_width * 2) - return IA_CSS_ERR_INVALID_ARGUMENTS; - return IA_CSS_SUCCESS; + return -EINVAL; + return 0; } -enum ia_css_err ia_css_util_check_vf_out_info( +int ia_css_util_check_vf_out_info( const struct ia_css_frame_info *const out_info, const struct ia_css_frame_info *const vf_info) { - enum ia_css_err err; + int err; assert(out_info); assert(vf_info); err = ia_css_frame_check_info(out_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ia_css_util_check_vf_info(vf_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err ia_css_util_check_res(unsigned int width, unsigned int height) +int ia_css_util_check_res(unsigned int width, unsigned int height) { /* height can be odd number for jpeg/embedded data from ISYS2401 */ if (((width == 0) || (height == 0) || IS_ODD(width))) { - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } - return IA_CSS_SUCCESS; + return 0; } /* ISP2401 */ @@ -200,7 +172,7 @@ bool ia_css_util_is_input_format_yuv(enum atomisp_input_format format) format == ATOMISP_INPUT_FORMAT_YUV422_16; } -enum ia_css_err ia_css_util_check_input( +int ia_css_util_check_input( const struct ia_css_stream_config *const stream_config, bool must_be_raw, bool must_be_yuv) @@ -208,18 +180,18 @@ enum ia_css_err ia_css_util_check_input( assert(stream_config); if (!stream_config) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (stream_config->input_config.effective_res.width == 0 || stream_config->input_config.effective_res.height == 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (must_be_raw && !ia_css_util_is_input_format_raw(stream_config->input_config.format)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (must_be_yuv && !ia_css_util_is_input_format_yuv(stream_config->input_config.format)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/cell_params.h b/drivers/staging/media/atomisp/pci/cell_params.h index 0eabc59ff5af..3c21a18990ba 100644 --- a/drivers/staging/media/atomisp/pci/cell_params.h +++ b/drivers/staging/media/atomisp/pci/cell_params.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c index 3ef556a64825..1a021ae841fe 100644 --- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c +++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_configs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c index 2b90a7075b9b..b786247b322b 100644 --- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c +++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_params.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c index 42e0344c677d..a6bc2e9eddea 100644 --- a/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c +++ b/drivers/staging/media/atomisp/pci/css_2400_system/hive/ia_css_isp_states.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -210,7 +211,7 @@ ia_css_initialize_ynr_state( /* Code generated by genparam/genstate.c:gen_state_init_table() */ -void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( +void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( const struct ia_css_binary *binary) = { ia_css_initialize_aa_state, ia_css_initialize_cnr_state, diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h index 4de5bb81bd23..3aabd0248e4f 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/csi_rx_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c index 29d85407cac4..1a021ae841fe 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_configs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -272,7 +273,6 @@ ia_css_configure_output( "ia_css_configure_output() leave:\n"); } - /* Code generated by genparam/genconfig.c:gen_configure_function() */ void diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c index 68297296885e..d9c672d8904e 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_params.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c index c54787f3fc24..514ffe0303cb 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hive/ia_css_isp_states.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -210,7 +211,7 @@ ia_css_initialize_ynr_state( /* Code generated by genparam/genstate.c:gen_state_init_table() */ -void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( +void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( const struct ia_css_binary *binary) = { ia_css_initialize_aa_state, ia_css_initialize_cnr_state, diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c index 50080565d0d6..8e661091f7d9 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h index a86de89b2cfc..6489ee644a4a 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h index 3fa3c3a487ab..ece45d80eb2a 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/csi_rx_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -289,12 +290,12 @@ static inline void csi_rx_be_ctrl_dump_state( * lut. */ for (i = 0; i < N_SHORT_PACKET_LUT_ENTRIES[ID]; i++) { - ia_css_print("CSI RX BE STATE Controller ID %d Short packat entry %d shart packet lut id 0x%x\n", + ia_css_print("CSI RX BE STATE Controller ID %d Short packet entry %d short packet lut id 0x%x\n", ID, i, state->short_packet_lut_entry[i]); } for (i = 0; i < N_LONG_PACKET_LUT_ENTRIES[ID]; i++) { - ia_css_print("CSI RX BE STATE Controller ID %d Long packat entry %d Long packet lut id 0x%x\n", + ia_css_print("CSI RX BE STATE Controller ID %d Long packet entry %d long packet lut id 0x%x\n", ID, i, state->long_packet_lut_entry[i]); } diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c index 8b06b2410d1d..58fec54a914d 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h index ea40284623d1..4952b42d8191 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h index a0800a5df68a..a58e8477da6e 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/ibuf_ctrl_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c index 36c026cbd7cc..5809dbb6e5aa 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h index 5c694a26386e..878933261a43 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h index a1a222372ed3..eb35b7bcead4 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_dma_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c index 567c926bd47f..99576af4713c 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h index 4fd05b29dfdb..e3d6d5e1634e 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h index c519e6f06462..91ef000d76dc 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_irq_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c index 67570138ba24..b7d893aea88d 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h index 1449c19abc86..4fbbcc2338d3 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h index e5aae5c022eb..4a5646a229b8 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/isys_stream2mmio_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h index 24f4da9aef40..efaa4da8d36d 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h index 65ea23604479..4faa519219ee 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/host/pixelgen_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -97,9 +98,9 @@ STORAGE_CLASS_PIXELGEN_C void pixelgen_ctrl_dump_state( pixelgen_ctrl_state_t *state) { ia_css_print("Pixel Generator ID %d Enable 0x%x\n", ID, state->com_enable); - ia_css_print("Pixel Generator ID %d PRBS reset vlue 0 0x%x\n", ID, + ia_css_print("Pixel Generator ID %d PRBS reset value 0 0x%x\n", ID, state->prbs_rstval0); - ia_css_print("Pixel Generator ID %d PRBS reset vlue 1 0x%x\n", ID, + ia_css_print("Pixel Generator ID %d PRBS reset value 1 0x%x\n", ID, state->prbs_rstval1); ia_css_print("Pixel Generator ID %d SYNC SID 0x%x\n", ID, state->syng_sid); ia_css_print("Pixel Generator ID %d syng free run 0x%x\n", ID, diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h index ce53ba4837ea..ae471dd51737 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/PixelGen_SysBlock_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h index 5975b094a9d0..374466e6b7bf 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/ibuf_cntrl_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h index 84fe95c16404..ac8be2d49227 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_common_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h index 45f20b524368..6fae1c262446 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/mipi_backend_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h index a8d0dbd7f6d7..d0e5b54d1afc 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/rx_csi_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h index a3940d246890..e17783f96b23 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/hrt/stream2mmio_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h index dc8d091c6769..1b9f03d57659 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/ibuf_ctrl_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h index 2ca4d5210a38..f423f34134d3 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_dma_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -46,6 +47,7 @@ struct isys2401_dma_port_cfg_s { u32 cropping; u32 width; }; + /* end of DMA Port */ /************************************************ diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h index 41d051db3987..156b4c95277e 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_irq_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h index bcb46b293b6a..0611047eabbc 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/isys_stream2mmio_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h index cde599c5d0d2..75722ef572d0 100644 --- a/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h +++ b/drivers/staging/media/atomisp/pci/css_2401_system/pixelgen_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h b/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h index 99d292164efc..d2c39f9600bd 100644 --- a/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h +++ b/drivers/staging/media/atomisp/pci/css_receiver_2400_common_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h b/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h index f4b2b41b6d94..180ff7cd9ff5 100644 --- a/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h +++ b/drivers/staging/media/atomisp/pci/css_receiver_2400_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_trace.h b/drivers/staging/media/atomisp/pci/css_trace.h index 32520c21c324..d2ce50fcfb7a 100644 --- a/drivers/staging/media/atomisp/pci/css_trace.h +++ b/drivers/staging/media/atomisp/pci/css_trace.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -171,7 +172,6 @@ enum TRACE_DUMP_FORMAT { #define TRACE_ISP_DATA_SIZE (TRACE_ISP_SIZE - TRACE_ISP_HEADER_SIZE) #define TRACE_ISP_MAX_POINTS (TRACE_ISP_DATA_SIZE / TRACE_ISP_ITEM_SIZE) - /* common majors */ /* SP0 */ #define MAJOR_MAIN 1 diff --git a/drivers/staging/media/atomisp/pci/defs.h b/drivers/staging/media/atomisp/pci/defs.h index 47505f41790c..785e7a670a00 100644 --- a/drivers/staging/media/atomisp/pci/defs.h +++ b/drivers/staging/media/atomisp/pci/defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/dma_v2_defs.h b/drivers/staging/media/atomisp/pci/dma_v2_defs.h index 8741b8347dd4..27299e3a185d 100644 --- a/drivers/staging/media/atomisp/pci/dma_v2_defs.h +++ b/drivers/staging/media/atomisp/pci/dma_v2_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/gdc_v2_defs.h b/drivers/staging/media/atomisp/pci/gdc_v2_defs.h index 3cc627aa6b09..804df8179e36 100644 --- a/drivers/staging/media/atomisp/pci/gdc_v2_defs.h +++ b/drivers/staging/media/atomisp/pci/gdc_v2_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/gp_timer_defs.h b/drivers/staging/media/atomisp/pci/gp_timer_defs.h index ffd7b38fce9d..9bc04e5b4292 100644 --- a/drivers/staging/media/atomisp/pci/gp_timer_defs.h +++ b/drivers/staging/media/atomisp/pci/gp_timer_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/gpio_block_defs.h b/drivers/staging/media/atomisp/pci/gpio_block_defs.h index 96286a141b00..e1bd638d344a 100644 --- a/drivers/staging/media/atomisp/pci/gpio_block_defs.h +++ b/drivers/staging/media/atomisp/pci/gpio_block_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h deleted file mode 100644 index 0760b95818f6..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_2401_irq_types_hrt.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#ifndef _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ -#define _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ - -/* - * These are the indices of each interrupt in the interrupt - * controller's registers. these can be used as the irq_id - * argument to the hrt functions irq_controller.h. - * - * The definitions are taken from <system>_defs.h - */ -typedef enum hrt_isp_css_irq { - hrt_isp_css_irq_gpio_pin_0 = HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID, - hrt_isp_css_irq_gpio_pin_1 = HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID, - hrt_isp_css_irq_gpio_pin_2 = HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID, - hrt_isp_css_irq_gpio_pin_3 = HIVE_GP_DEV_IRQ_GPIO_PIN_3_BIT_ID, - hrt_isp_css_irq_gpio_pin_4 = HIVE_GP_DEV_IRQ_GPIO_PIN_4_BIT_ID, - hrt_isp_css_irq_gpio_pin_5 = HIVE_GP_DEV_IRQ_GPIO_PIN_5_BIT_ID, - hrt_isp_css_irq_gpio_pin_6 = HIVE_GP_DEV_IRQ_GPIO_PIN_6_BIT_ID, - hrt_isp_css_irq_gpio_pin_7 = HIVE_GP_DEV_IRQ_GPIO_PIN_7_BIT_ID, - hrt_isp_css_irq_gpio_pin_8 = HIVE_GP_DEV_IRQ_GPIO_PIN_8_BIT_ID, - hrt_isp_css_irq_gpio_pin_9 = HIVE_GP_DEV_IRQ_GPIO_PIN_9_BIT_ID, - hrt_isp_css_irq_gpio_pin_10 = HIVE_GP_DEV_IRQ_GPIO_PIN_10_BIT_ID, - hrt_isp_css_irq_gpio_pin_11 = HIVE_GP_DEV_IRQ_GPIO_PIN_11_BIT_ID, - hrt_isp_css_irq_sp = HIVE_GP_DEV_IRQ_SP_BIT_ID, - hrt_isp_css_irq_isp = HIVE_GP_DEV_IRQ_ISP_BIT_ID, - hrt_isp_css_irq_isys = HIVE_GP_DEV_IRQ_ISYS_BIT_ID, - hrt_isp_css_irq_isel = HIVE_GP_DEV_IRQ_ISEL_BIT_ID, - hrt_isp_css_irq_ifmt = HIVE_GP_DEV_IRQ_IFMT_BIT_ID, - hrt_isp_css_irq_sp_stream_mon = HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID, - hrt_isp_css_irq_isp_stream_mon = HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID, - hrt_isp_css_irq_mod_stream_mon = HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID, - hrt_isp_css_irq_is2401 = HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID, - hrt_isp_css_irq_isp_bamem_error = HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID, - hrt_isp_css_irq_isp_dmem_error = HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID, - hrt_isp_css_irq_sp_icache_mem_error = HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID, - hrt_isp_css_irq_sp_dmem_error = HIVE_GP_DEV_IRQ_SP_DMEM_ERROR_BIT_ID, - hrt_isp_css_irq_mmu_cache_mem_error = HIVE_GP_DEV_IRQ_MMU_CACHE_MEM_ERROR_BIT_ID, - hrt_isp_css_irq_gp_timer_0 = HIVE_GP_DEV_IRQ_GP_TIMER_0_BIT_ID, - hrt_isp_css_irq_gp_timer_1 = HIVE_GP_DEV_IRQ_GP_TIMER_1_BIT_ID, - hrt_isp_css_irq_sw_pin_0 = HIVE_GP_DEV_IRQ_SW_PIN_0_BIT_ID, - hrt_isp_css_irq_sw_pin_1 = HIVE_GP_DEV_IRQ_SW_PIN_1_BIT_ID, - hrt_isp_css_irq_dma = HIVE_GP_DEV_IRQ_DMA_BIT_ID, - hrt_isp_css_irq_sp_stream_mon_b = HIVE_GP_DEV_IRQ_SP_STREAM_MON_B_BIT_ID, - /* this must (obviously) be the last on in the enum */ - hrt_isp_css_irq_num_irqs -} hrt_isp_css_irq_t; - -typedef enum hrt_isp_css_irq_status { - hrt_isp_css_irq_status_error, - hrt_isp_css_irq_status_more_irqs, - hrt_isp_css_irq_status_success -} hrt_isp_css_irq_status_t; - -#endif /* _HIVE_ISP_CSS_2401_IRQ_TYPES_HRT_H_ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h index 7580cf5c9624..b6538beca18a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/debug_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h index 85d509f5b923..135034c7245a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/dma_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h index 4df7a405cdcf..a50635b717e3 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/event_fifo_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h index f43bf0ad2468..d941c82d530b 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/fifo_monitor_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h index f3ce9e9f1ad4..599d993b832e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gdc_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h index 1c1b0667a53b..c8f416515e2c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_device_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h index ee636ad6c5b3..163003f2c759 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gp_timer_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h index a82ca2a8cada..b5f017482f89 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/gpio_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h index e4b9daa2d062..746b07097681 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/hmem_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c index d911aec24185..a502ba9f8c7f 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2016, Intel Corporation. @@ -14,12 +15,12 @@ #include "debug.h" +#include "hmm.h" + #ifndef __INLINE_DEBUG__ #include "debug_private.h" #endif /* __INLINE_DEBUG__ */ -#include "memory_access.h" - #define __INLINE_SP__ #include "sp.h" @@ -27,7 +28,7 @@ /* The address of the remote copy */ hrt_address debug_buffer_address = (hrt_address) - 1; -hrt_vaddress debug_buffer_ddr_address = (hrt_vaddress)-1; +ia_css_ptr debug_buffer_ddr_address = (ia_css_ptr)-1; /* The local copy */ static debug_data_t debug_data; debug_data_t *debug_data_ptr = &debug_data; @@ -40,7 +41,7 @@ void debug_buffer_init(const hrt_address addr) debug_data.tail = 0; } -void debug_buffer_ddr_init(const hrt_vaddress addr) +void debug_buffer_ddr_init(const ia_css_ptr addr) { debug_buf_mode_t mode = DEBUG_BUFFER_MODE_LINEAR; u32 enable = 1; @@ -48,13 +49,13 @@ void debug_buffer_ddr_init(const hrt_vaddress addr) u32 tail = 0; /* set the ddr queue */ debug_buffer_ddr_address = addr; - mmgr_store(addr + DEBUG_DATA_BUF_MODE_DDR_ADDR, + hmm_store(addr + DEBUG_DATA_BUF_MODE_DDR_ADDR, &mode, sizeof(debug_buf_mode_t)); - mmgr_store(addr + DEBUG_DATA_HEAD_DDR_ADDR, + hmm_store(addr + DEBUG_DATA_HEAD_DDR_ADDR, &head, sizeof(uint32_t)); - mmgr_store(addr + DEBUG_DATA_TAIL_DDR_ADDR, + hmm_store(addr + DEBUG_DATA_TAIL_DDR_ADDR, &tail, sizeof(uint32_t)); - mmgr_store(addr + DEBUG_DATA_ENABLE_DDR_ADDR, + hmm_store(addr + DEBUG_DATA_ENABLE_DDR_ADDR, &enable, sizeof(uint32_t)); /* set the local copy */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h index 4c95eda694f7..536a4dcf0f62 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h index 8447e33d1c04..3fea43a2125e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/debug_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -22,8 +23,6 @@ #define __INLINE_ISP__ #include "isp.h" -#include "memory_access.h" - #include "assert_support.h" STORAGE_CLASS_DEBUG_C bool is_debug_buffer_empty(void) @@ -101,22 +100,22 @@ STORAGE_CLASS_DEBUG_C void debug_synch_queue_ddr(void) { u32 remote_tail; - mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_TAIL_DDR_ADDR, &remote_tail, + hmm_load(debug_buffer_ddr_address + DEBUG_DATA_TAIL_DDR_ADDR, &remote_tail, sizeof(uint32_t)); /* We could move the remote head after the upload, but we would have to limit the upload w.r.t. the local head. This is easier */ if (remote_tail > debug_data_ptr->tail) { size_t delta = remote_tail - debug_data_ptr->tail; - mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + + hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + debug_data_ptr->tail * sizeof(uint32_t), (void *)&debug_data_ptr->buf[debug_data_ptr->tail], delta * sizeof(uint32_t)); } else if (remote_tail < debug_data_ptr->tail) { size_t delta = DEBUG_BUF_SIZE - debug_data_ptr->tail; - mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + + hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR + debug_data_ptr->tail * sizeof(uint32_t), (void *)&debug_data_ptr->buf[debug_data_ptr->tail], delta * sizeof(uint32_t)); - mmgr_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR, + hmm_load(debug_buffer_ddr_address + DEBUG_DATA_BUF_DDR_ADDR, (void *)&debug_data_ptr->buf[0], remote_tail * sizeof(uint32_t)); } /* else we are up to date */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c index 87df1da1164e..f85950c471c7 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2016, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h index d7db964d5cec..7e4cc75733cd 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h index ebb75da72e18..1f62bc2f176f 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/dma_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c index 777670948d6f..62d4809e33dd 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h index 39a9dd697096..25d3823026e8 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h index 3b6cc27ecb28..f59d45cc78b7 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/event_fifo_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c index 82f7c43bcb0a..01698064bbe1 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h index a557ff8a416f..dfdca944a40b 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h index abaef8672ae2..10d9c076c140 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/fifo_monitor_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c index 65c5296163dd..8ed1cffc5384 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h index 0c6de867e012..4b2b3282c1b2 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h index f7dec75adf78..73051112f354 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gdc_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c index 5f20ac0b492e..a80e547d47b3 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h index 113d5ed32d42..320ed35269ea 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h index cdc1b12a9e8a..f11a19f21d10 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_device_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c index 4a856f1f14bf..2a58dba3c87b 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h index 4d5961c78c16..efede25587fd 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h index 705be5e5cc70..3e1b36105bb6 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gp_timer_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h index f4652b79734d..14013733f826 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h index 56b442040ad9..cc60bed71ddb 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/gpio_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c index e48f180c9507..be102d5cec87 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h index 499f55f07253..a3ee274bdf19 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h index 270d04cc9d09..80d81983bd06 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/hmem_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c index 0c90c5ed659b..bec9c7238a78 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h index ee2c8372421c..94fff77584f7 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h index bdca709219a4..e2bc952e6694 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_formatter_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c index 2114cf4f3fda..fc000af042dc 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/input_system.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-015, Intel Corporation. @@ -855,9 +856,9 @@ input_system_error_t input_system_configuration_reset(void) input_system_network_rst(INPUT_SYSTEM0_ID); - gp_device_rst(INPUT_SYSTEM0_ID); + gp_device_rst(GP_DEVICE0_ID); - input_switch_rst(INPUT_SYSTEM0_ID); + input_switch_rst(GP_DEVICE0_ID); //target_rst(); @@ -873,7 +874,7 @@ input_system_error_t input_system_configuration_reset(void) for (i = 0; i < N_CSI_PORTS; i++) { config.csi_buffer_flags[i] = INPUT_SYSTEM_CFG_FLAG_RESET; - config.multicast[i] = INPUT_SYSTEM_CFG_FLAG_RESET; + config.multicast[i] = INPUT_SYSTEM_DISCARD_ALL; } config.source_type_flags = INPUT_SYSTEM_CFG_FLAG_RESET; @@ -1323,10 +1324,10 @@ static input_system_error_t configuration_to_registers(void) } // end of switch (source_type) // Set input selector. - input_selector_cfg_for_sensor(INPUT_SYSTEM0_ID); + input_selector_cfg_for_sensor(GP_DEVICE0_ID); // Set input switch. - input_switch_cfg(INPUT_SYSTEM0_ID, &config.input_switch_cfg); + input_switch_cfg(GP_DEVICE0_ID, &config.input_switch_cfg); // Set input formatters. // AM: IF are set dynamically. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c index fdc99cc6eae4..80b5fd0dc9f6 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -20,17 +21,14 @@ #endif #include "gp_device.h" /* _REG_GP_IRQ_REQUEST_ADDR */ -#include "platform_support.h" /* hrt_sleep() */ - static inline void irq_wait_for_write_complete( const irq_ID_t ID); static inline bool any_irq_channel_enabled( const irq_ID_t ID); -static inline irq_ID_t virq_get_irq_id( - const virq_id_t irq_ID, - unsigned int *channel_ID); +static inline irq_ID_t virq_get_irq_id(const enum virq_id irq_ID, + unsigned int *channel_ID); #ifndef __INLINE_IRQ__ #include "irq_private.h" @@ -51,7 +49,7 @@ static unsigned short IRQ_N_ID_OFFSET[N_IRQ_ID + 1] = { IRQ_END_OFFSET }; -static virq_id_t IRQ_NESTING_ID[N_IRQ_ID] = { +static enum virq_id IRQ_NESTING_ID[N_IRQ_ID] = { N_virq_id, virq_ifmt, virq_isys, @@ -227,9 +225,8 @@ void irq_raise( return; } -void irq_controller_get_state( - const irq_ID_t ID, - irq_controller_state_t *state) +void irq_controller_get_state(const irq_ID_t ID, + struct irq_controller_state *state) { assert(ID < N_IRQ_ID); assert(state); @@ -256,7 +253,7 @@ bool any_virq_signal(void) } void cnd_virq_enable_channel( - const virq_id_t irq_ID, + const enum virq_id irq_ID, const bool en) { irq_ID_t i; @@ -296,8 +293,8 @@ void virq_clear_all(void) return; } -enum hrt_isp_css_irq_status virq_get_channel_signals( - virq_info_t *irq_info) +enum hrt_isp_css_irq_status +virq_get_channel_signals(struct virq_info *irq_info) { enum hrt_isp_css_irq_status irq_status = hrt_isp_css_irq_status_error; irq_ID_t ID; @@ -326,8 +323,7 @@ enum hrt_isp_css_irq_status virq_get_channel_signals( return irq_status; } -void virq_clear_info( - virq_info_t *irq_info) +void virq_clear_info(struct virq_info *irq_info) { irq_ID_t ID; @@ -340,7 +336,7 @@ void virq_clear_info( } enum hrt_isp_css_irq_status virq_get_channel_id( - virq_id_t *irq_id) + enum virq_id *irq_id) { unsigned int irq_status = irq_reg_load(IRQ0_ID, _HRT_IRQ_CONTROLLER_STATUS_REG_IDX); @@ -367,7 +363,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id( /* Check whether we have an IRQ on one of the nested devices */ for (ID = N_IRQ_ID - 1 ; ID > (irq_ID_t)0; ID--) { - if (IRQ_NESTING_ID[ID] == (virq_id_t)idx) { + if (IRQ_NESTING_ID[ID] == (enum virq_id)idx) { break; } } @@ -404,7 +400,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id( idx += IRQ_N_ID_OFFSET[ID]; if (irq_id) - *irq_id = (virq_id_t)idx; + *irq_id = (enum virq_id)idx; return status; } @@ -432,7 +428,7 @@ static inline bool any_irq_channel_enabled( } static inline irq_ID_t virq_get_irq_id( - const virq_id_t irq_ID, + const enum virq_id irq_ID, unsigned int *channel_ID) { irq_ID_t ID; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h index 86028fde2a94..6a25345ae88e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -42,10 +43,7 @@ #define IRQ2_ID_N_CHANNEL HIVE_ISYS_IRQ_NUM_BITS #define IRQ3_ID_N_CHANNEL HIVE_ISEL_IRQ_NUM_IRQS -typedef struct virq_info_s virq_info_t; -typedef struct irq_controller_state_s irq_controller_state_t; - -typedef enum { +enum virq_id { virq_gpio_pin_0 = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_0_BIT_ID, virq_gpio_pin_1 = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_1_BIT_ID, virq_gpio_pin_2 = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_GPIO_PIN_2_BIT_ID, @@ -66,13 +64,7 @@ typedef enum { virq_sp_stream_mon = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_STREAM_MON_BIT_ID, virq_isp_stream_mon = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_STREAM_MON_BIT_ID, virq_mod_stream_mon = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_MOD_STREAM_MON_BIT_ID, -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) virq_isp_pmem_error = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID, -#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM) - virq_isys_2401 = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_PMEM_ERROR_BIT_ID, -#else -#error "irq_local.h: 2400_SYSTEM must be one of {2400, 2401 }" -#endif virq_isp_bamem_error = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_BAMEM_ERROR_BIT_ID, virq_isp_dmem_error = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_ISP_DMEM_ERROR_BIT_ID, virq_sp_icache_mem_error = IRQ0_ID_OFFSET + HIVE_GP_DEV_IRQ_SP_ICACHE_MEM_ERROR_BIT_ID, @@ -117,13 +109,13 @@ typedef enum { virq_isel_eol = IRQ3_ID_OFFSET + HIVE_ISEL_IRQ_SYNC_GEN_EOL_BIT_ID, N_virq_id = IRQ_END_OFFSET -} virq_id_t; +}; -struct virq_info_s { +struct virq_info { hrt_data irq_status_reg[N_IRQ_ID]; }; -struct irq_controller_state_s { +struct irq_controller_state { unsigned int irq_edge; unsigned int irq_mask; unsigned int irq_status; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h index 8a947aefd851..e98663ef0fcd 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/irq_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c index 7de7d08f4757..4ad5e2db8a89 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -12,6 +13,8 @@ * more details. */ +#include <linux/delay.h> + #include <system_global.h> #include "isp.h" @@ -20,7 +23,6 @@ #endif /* __INLINE_ISP__ */ #include "assert_support.h" -#include "platform_support.h" /* hrt_sleep() */ void cnd_isp_irq_enable( const isp_ID_t ID, @@ -124,5 +126,5 @@ void isp_wake(isp_ID_t ID) { assert(ID < N_ISP_ID); isp_ctrl_setbit(ID, ISP_SC_REG, ISP_START_BIT); - hrt_sleep(); + udelay(1); } diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h index b04da7f1f98c..eceeb5d160ad 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h index a6ab10711255..2f9aeb3bd9d4 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/isp_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c index a17b32b6d414..eb02835aa98a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h index 7c3ad157189f..913150504f0f 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/mmu_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c index f084b316e373..aae18465b6ae 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h index 0e477b497c98..2956c7023b33 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h index e3e24fac126e..05e6b438d2c8 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/sp_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c index aaea74389443..bc9e7f10f1ab 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h index e570813af28d..f58ee6afcff9 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h index 3c137badbd43..c19eeafed3a3 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/timed_ctrl_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h index c4e99afe0d29..c68ed984ca48 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vamem_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c index 0c6830ae7344..6620f091442f 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2016, Intel Corporation. @@ -20,7 +21,6 @@ #include "ia_css_device_access.h" #endif #include "assert_support.h" -#include "platform_support.h" /* hrt_sleep() */ typedef unsigned long long hive_uedge; typedef hive_uedge *hive_wide; @@ -154,7 +154,7 @@ static void load_vector( hive_sim_wide_unpack(data, &elem, ISP_VEC_ELEMBITS, i); to[i] = elem; } - hrt_sleep(); /* Spend at least 1 cycles per vector */ + udelay(1); /* Spend at least 1 cycles per vector */ } static void store_vector( @@ -179,7 +179,7 @@ static void store_vector( //hrt_mem_store (ISP, VMEM, (unsigned)to, &v, siz); /* This will overwrite the next vector as well */ hrt_master_port_store(ISP_BAMEM_BASE[ID] + (unsigned long)to, &v, size); #endif - hrt_sleep(); /* Spend at least 1 cycles per vector */ + udelay(1); /* Spend at least 1 cycles per vector */ } void isp_vmem_load( diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h index a42cce42f29d..d0ba59cedc92 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h index f48d1281b5a7..39cf1316b404 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/host/vmem_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h index 163521c53d4b..605cf02e520c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/input_formatter_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -19,7 +20,7 @@ #define IS_INPUT_SWITCH_VERSION2 #include <type_support.h> -#include <system_types.h> +#include <system_local.h> #include "if_defs.h" #include "str2mem_defs.h" #include "input_switch_2400_defs.h" diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h index 64554d80dc0b..4a1dea6dfd40 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/irq_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,22 +16,14 @@ #ifndef __IRQ_GLOBAL_H_INCLUDED__ #define __IRQ_GLOBAL_H_INCLUDED__ -#include <system_types.h> +#include <system_local.h> #define IS_IRQ_VERSION_2 #define IS_IRQ_MAP_VERSION_2 /* We cannot include the (hrt host ID) file defining the "CSS_RECEIVER" property without side effects */ #ifndef HAS_NO_RX -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) -/*#define CSS_RECEIVER testbench_isp_inp_sys_csi_receiver*/ -#include "hive_isp_css_irq_types_hrt.h" /* enum hrt_isp_css_irq */ -#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM) -/*#define CSS_RECEIVER testbench_isp_is_2400_inp_sys_csi_receiver*/ -#include "hive_isp_css_2401_irq_types_hrt.h" /* enum hrt_isp_css_irq */ -#else -#error "irq_global.h: 2400_SYSTEM must be one of {2400, 2401 }" -#endif +#include "irq_types_hrt.h" #endif /* The IRQ is not mapped uniformly on its related interfaces */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h index 1a8547d58435..5c6891c9b451 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/isp_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,19 +16,9 @@ #ifndef __ISP_GLOBAL_H_INCLUDED__ #define __ISP_GLOBAL_H_INCLUDED__ -#include <system_types.h> +#include <system_local.h> -#if defined(HAS_ISP_2401_MAMOIADA) -#define IS_ISP_2401_MAMOIADA - -#include "isp2401_mamoiada_params.h" -#elif defined(HAS_ISP_2400_MAMOIADA) -#define IS_ISP_2400_MAMOIADA - -#include "isp2400_mamoiada_params.h" -#else -#error "isp_global_h: ISP_2400_MAMOIDA must be one of {2400, 2401 }" -#endif +#include "mamoiada_params.h" #define ISP_PMEM_WIDTH_LOG2 ISP_LOG2_PMEM_WIDTH #define ISP_PMEM_SIZE ISP_PMEM_DEPTH diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h index 83ca418c8ff2..8738fed6afdf 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/mmu_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h index 6ec4e590e3b4..b8338f9b5c0c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/sp_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,19 +16,9 @@ #ifndef __SP_GLOBAL_H_INCLUDED__ #define __SP_GLOBAL_H_INCLUDED__ -#include <system_types.h> +#include <system_local.h> -#if defined(HAS_SP_2401) -#define IS_SP_2401 -/* 2401 uses 2400 */ #include <scalar_processor_2400_params.h> -#elif defined(HAS_SP_2400) -#define IS_SP_2400 - -#include <scalar_processor_2400_params.h> -#else -#error "sp_global.h: SP_2400 must be one of {2400, 2401 }" -#endif #define SP_PMEM_WIDTH_LOG2 SP_PMEM_LOG_WIDTH_BITS #define SP_PMEM_SIZE SP_PMEM_DEPTH diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h index f185859e3084..3f2915a78031 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/timed_ctrl_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h index 92b783fed82c..0d290e815767 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vamem_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h index 7867cd137f3f..537b074211da 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_common/vmem_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h b/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h index 52676f3610ba..e9cf2743868c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h index 4cb7e4c952c5..7382c0bbf7cb 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/assert_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h index 76856db58626..29f14e900580 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/bitop_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h index badb15705710..4602885d50e8 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/csi_rx.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h index ba11b956eb1a..0f8195ba8d1a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/debug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h index be031d41de7c..492f9e26cfff 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/device_access/device_access.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. @@ -39,7 +40,7 @@ more details. * User provided file that defines the system address types: * - hrt_address a type that can hold the (sub)system address range */ -#include "system_types.h" +#include "system_local.h" /* * We cannot assume that the global system address size is the size of * a pointer because a (say) 64-bit host can be simulated in a 32-bit diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h index b6c464a530ea..2f5ebfcd7e8b 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h deleted file mode 100644 index 4f0d259bf7ed..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/error_support.h +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#ifndef __ERROR_SUPPORT_H_INCLUDED__ -#define __ERROR_SUPPORT_H_INCLUDED__ - -#include <linux/errno.h> -/* - * Put here everything __KERNEL__ specific not covered in - * "errno.h" - */ -#define ENOTSUP 252 - -#define verifexit(cond, error_tag) \ -do { \ - if (!(cond)) { \ - goto EXIT; \ - } \ -} while (0) - -#define verifjmpexit(cond) \ -do { \ - if (!(cond)) { \ - goto EXIT; \ - } \ -} while (0) - -#endif /* __ERROR_SUPPORT_H_INCLUDED__ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h index 8bfe348772f4..0a085abd3ade 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/event_fifo.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h index 1743caa006d0..19a1bdd9171a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/fifo_monitor.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h index 4f8d7fbc8e7f..4ed57fb4530e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gdc_device.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h index 665557bae7a1..d122bdeae7e7 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_device.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h index cd26c9d16a35..0b7e92b963d0 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gp_timer.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h index ad79c03e59f4..6f16ca77cf75 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/gpio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h index f87fd6b2ba23..898facd7b17a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/hmem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h index f7cd4d7b96e5..e6f695691407 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/csi_rx_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h index 79a8446658ee..ee861ddb8e92 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/debug_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,8 @@ #define __DEBUG_PUBLIC_H_INCLUDED__ #include <type_support.h> -#include "system_types.h" +#include <ia_css_types.h> +#include "system_local.h" /*! brief * @@ -42,7 +44,7 @@ typedef struct debug_data_ddr_s debug_data_ddr_t; extern debug_data_t *debug_data_ptr; extern hrt_address debug_buffer_address; -extern hrt_vaddress debug_buffer_ddr_address; +extern ia_css_ptr debug_buffer_ddr_address; /*! Check the empty state of the local debug data buffer @@ -86,7 +88,7 @@ void debug_buffer_init( \return none */ void debug_buffer_ddr_init( - const hrt_vaddress addr); + const ia_css_ptr addr); /*! Set the (remote) operating mode of the debug buffer diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h index 385b978b703b..a23cbc9a2129 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/dma_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __DMA_PUBLIC_H_INCLUDED__ #define __DMA_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" typedef struct dma_state_s dma_state_t; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h index a84b74b3bc1e..22f1875f038e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/event_fifo_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,7 @@ #define __EVENT_FIFO_PUBLIC_H #include <type_support.h> -#include "system_types.h" +#include "system_local.h" /*! Blocking read from an event source EVENT[ID] diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h index e451d6f2a70d..7c1c3d2f24c6 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/fifo_monitor_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __FIFO_MONITOR_PUBLIC_H_INCLUDED__ #define __FIFO_MONITOR_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" typedef struct fifo_channel_state_s fifo_channel_state_t; typedef struct fifo_switch_state_s fifo_switch_state_t; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h index fc6f42e76fbe..385b79254455 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gdc_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h index 7cc0799d49ed..f017742d9ac4 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_device_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __GP_DEVICE_PUBLIC_H_INCLUDED__ #define __GP_DEVICE_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" typedef struct gp_device_state_s gp_device_state_t; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h index 2ddb8c40a5b2..13baf7236375 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gp_timer_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __GP_TIMER_PUBLIC_H_INCLUDED__ #define __GP_TIMER_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" /*! initialize mentioned timer param ID timer_id diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h index d21aab3a179d..13df9b57a5fb 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/gpio_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __GPIO_PUBLIC_H_INCLUDED__ #define __GPIO_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" /*! Write to a control register of GPIO[ID] diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h index 567fbc1d35e7..8d271fb84209 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/hmem_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h index 6b17a6b651b7..053803d2cae3 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/ibuf_ctrl_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h index e5758cb8bedd..81dc58640d83 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/input_formatter_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,7 @@ #define __INPUT_FORMATTER_PUBLIC_H_INCLUDED__ #include <type_support.h> -#include "system_types.h" +#include "system_local.h" /*! Reset INPUT_FORMATTER[ID] diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h index dfe2aa9ff257..d335e7b0a76e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/irq_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,7 @@ #define __IRQ_PUBLIC_H_INCLUDED__ #include <type_support.h> -#include "system_types.h" +#include "system_local.h" /*! Read the control registers of IRQ[ID] @@ -25,9 +26,8 @@ \return none, state = IRQ[ID].state */ -void irq_controller_get_state( - const irq_ID_t ID, - irq_controller_state_t *state); +void irq_controller_get_state(const irq_ID_t ID, + struct irq_controller_state *state); /*! Write to a control register of IRQ[ID] @@ -136,7 +136,7 @@ bool any_virq_signal(void); \return none, VIRQ.channel[irq_ID].enable = en */ void cnd_virq_enable_channel( - const virq_id_t irq_ID, + const enum virq_id irq_ID, const bool en); /*! Clear the state of all IRQ channels of the virtual super IRQ @@ -151,8 +151,7 @@ void virq_clear_all(void); \return none */ -void virq_clear_info( - virq_info_t *irq_info); +void virq_clear_info(struct virq_info *irq_info); /*! Return the ID of a signalling IRQ channel of the virtual super IRQ @@ -165,7 +164,7 @@ void virq_clear_info( \return state(IRQ[...]) */ enum hrt_isp_css_irq_status virq_get_channel_id( - virq_id_t *irq_id); + enum virq_id *irq_id); /*! Return the IDs of all signaling IRQ channels of the virtual super IRQ @@ -178,7 +177,7 @@ enum hrt_isp_css_irq_status virq_get_channel_id( \return (error(state(IRQ[...])) */ -enum hrt_isp_css_irq_status virq_get_channel_signals( - virq_info_t *irq_info); +enum hrt_isp_css_irq_status +virq_get_channel_signals(struct virq_info *irq_info); #endif /* __IRQ_PUBLIC_H_INCLUDED__ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h index 0da2937b900e..a8ff75c639e5 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isp_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,7 @@ #define __ISP_PUBLIC_H_INCLUDED__ #include <type_support.h> -#include "system_types.h" +#include "system_local.h" /*! Enable or disable the program complete irq signal of ISP[ID] diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h index 734634aedadf..23a158b81b13 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_dma_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -17,7 +18,7 @@ #ifdef USE_INPUT_SYSTEM_VERSION_2401 -#include "system_types.h" +#include "system_local.h" #include "type_support.h" STORAGE_CLASS_ISYS2401_DMA_H void isys2401_dma_reg_store( diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h index d561783e47a3..b9befdd2508e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_irq_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h index 2d1859f2c656..509f75fe025c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h index 87a8d512713e..73bcc424e472 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/isys_stream2mmio_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h index 278f9cd85a00..b8c7bbb71b01 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/mmu_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __MMU_PUBLIC_H_INCLUDED__ #define __MMU_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" #include "device_access.h" #include "assert_support.h" diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h index ba67a276ce55..ded4dce06d09 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/pixelgen_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h index b8db5469b592..b0b7f2e27854 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/sp_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -16,7 +17,7 @@ #define __SP_PUBLIC_H_INCLUDED__ #include <type_support.h> -#include "system_types.h" +#include "system_local.h" typedef struct sp_state_s sp_state_t; typedef struct sp_stall_s sp_stall_t; diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h index afd5a59489cc..b18b4a4e13ac 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/tag_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h index 5f9277adb2ab..563a2833d1d9 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/timed_ctrl_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef __TIMED_CTRL_PUBLIC_H_INCLUDED__ #define __TIMED_CTRL_PUBLIC_H_INCLUDED__ -#include "system_types.h" +#include "system_local.h" /*! Write to a control register of TIMED_CTRL[ID] diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h index 577b9b8449e8..823e3857e83c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vamem_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h index e9801c0fe147..c510d6a08017 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/host/vmem_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h index f9cf7b586045..218341041811 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/ibuf_ctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h index 377996e0536d..daeb919b5384 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_formatter.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h index 33ab8a85909e..0d951fbf42e9 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/input_system.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h index 133dd9014fef..3a83a85111a2 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h index 749610b8a831..cb64e62c5569 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h index dbdd17115018..6a759142eda8 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_dma.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h index d3f64cfd0b7d..d854124f4f97 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h index e2ebeb14e7c2..b0f09ffb4f18 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/isys_stream2mmio.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h index 0b9519c8961c..a444ec14ff9d 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/math_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h deleted file mode 100644 index dc63ff0c9c6a..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_access/memory_access.h +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015-2017, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#ifndef __MEMORY_ACCESS_H_INCLUDED__ -#define __MEMORY_ACCESS_H_INCLUDED__ - -/*! - * \brief - * Define the public interface for virtual memory - * access functions. Access types are limited to - * those defined in <stdint.h> - * - * The address representation is private to the system - * and represented as "hrt_vaddress" rather than a - * pointer, as the memory allocation cannot be accessed - * by dereferencing but reaquires load and store access - * functions - * - * The page table selection or virtual memory context; - * The page table base index; Is implicit. This page - * table base index must be set by the implementation - * of the access function - * - * "store" is a transfer to the system - * "load" is a transfer from the system - * - * Allocation properties can be specified by setting - * attributes (see below) in case of multiple physical - * memories the memory ID is encoded on the attribute - * - * Allocations in the same physical memory, but in a - * different (set of) page tables can be shared through - * a page table information mapping function - */ - -#include <type_support.h> -#include "platform_support.h" /* for __func__ */ - -/* - * User provided file that defines the (sub)system address types: - * - hrt_vaddress a type that can hold the (sub)system virtual address range - */ -#include "system_types.h" - -/* - * The MMU base address is a physical address, thus the same type is used - * as for the device base address - */ -#include "device_access.h" - -#include "hmm/hmm.h" - -/*! - * \brief - * Bit masks for specialised allocation functions - * the default is "uncached", "not contiguous", - * "not page aligned" and "not cleared" - * - * Forcing alignment (usually) returns a pointer - * at an alignment boundary that is offset from - * the allocated pointer. Without storing this - * pointer/offset, we cannot free it. The memory - * manager is responsible for the bookkeeping, e.g. - * the allocation function creates a sentinel - * within the allocation referencable from the - * returned pointer/address. - */ -#define MMGR_ATTRIBUTE_MASK 0x000f -#define MMGR_ATTRIBUTE_CACHED 0x0001 -#define MMGR_ATTRIBUTE_CONTIGUOUS 0x0002 -#define MMGR_ATTRIBUTE_PAGEALIGN 0x0004 -#define MMGR_ATTRIBUTE_CLEARED 0x0008 -#define MMGR_ATTRIBUTE_UNUSED 0xfff0 - -/* #define MMGR_ATTRIBUTE_DEFAULT (MMGR_ATTRIBUTE_CACHED) */ -#define MMGR_ATTRIBUTE_DEFAULT 0 - -extern const hrt_vaddress mmgr_NULL; -extern const hrt_vaddress mmgr_EXCEPTION; - -/*! Return the address of an allocation in memory - - \param size[in] Size in bytes of the allocation - \param caller_func[in] Caller function name - \param caller_line[in] Caller function line number - - \return vaddress - */ -hrt_vaddress mmgr_malloc(const size_t size); - -/*! Return the address of a zero initialised allocation in memory - - \param N[in] Horizontal dimension of array - \param size[in] Vertical dimension of array Total size is N*size - - \return vaddress - */ -hrt_vaddress mmgr_calloc(const size_t N, const size_t size); - -/*! Return the address of an allocation in memory - - \param size[in] Size in bytes of the allocation - \param attribute[in] Bit vector specifying the properties - of the allocation including zero initialisation - - \return vaddress - */ - -hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attribute); - -/*! Return the address of a mapped existing allocation in memory - - \param ptr[in] Pointer to an allocation in a different - virtual memory page table, but the same - physical memory - \param size[in] Size of the memory of the pointer - \param attribute[in] Bit vector specifying the properties - of the allocation - \param context Pointer of a context provided by - client/driver for additional parameters - needed by the implementation - \Note - This interface is tentative, limited to the desired function - the actual interface may require furhter parameters - - \return vaddress - */ -hrt_vaddress mmgr_mmap( - const void __user *ptr, - const size_t size, - u16 attribute, - void *context); - -/*! Zero initialise an allocation in memory - - \param vaddr[in] Address of an allocation - \param size[in] Size in bytes of the area to be cleared - - \return none - */ -void mmgr_clear(hrt_vaddress vaddr, const size_t size); - -/*! Read an array of bytes from a virtual memory address - - \param vaddr[in] Address of an allocation - \param data[out] pointer to the destination array - \param size[in] number of bytes to read - - \return none - */ -void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size); - -/*! Write an array of bytes to device registers or memory in the device - - \param vaddr[in] Address of an allocation - \param data[in] pointer to the source array - \param size[in] number of bytes to write - - \return none - */ -void mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size); - -#endif /* __MEMORY_ACCESS_H_INCLUDED__ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h deleted file mode 100644 index 546b93ca1186..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/memory_realloc.h +++ /dev/null @@ -1,38 +0,0 @@ -/** -Support for Intel Camera Imaging ISP subsystem. -Copyright (c) 2010 - 2015, Intel Corporation. - -This program is free software; you can redistribute it and/or modify it -under the terms and conditions of the GNU General Public License, -version 2, as published by the Free Software Foundation. - -This program is distributed in the hope it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -more details. -*/ -#ifndef __MEMORY_REALLOC_H_INCLUDED__ -#define __MEMORY_REALLOC_H_INCLUDED__ - -/*! - * \brief - * Define the internal reallocation of private css memory - * - */ - -#include <type_support.h> -/* - * User provided file that defines the (sub)system address types: - * - hrt_vaddress a type that can hold the (sub)system virtual address range - */ -#include "system_types.h" -#include "ia_css_err.h" - -bool reallocate_buffer( - hrt_vaddress *curr_buf, - size_t *curr_size, - size_t needed_size, - bool force, - enum ia_css_err *err); - -#endif /*__MEMORY_REALLOC_H_INCLUDED__*/ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h index 38db1ecef3c8..393452d7a3d6 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/misc_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h index 1a36cb493fd8..b6f6eda4c55e 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/mmu_device.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h index 74335fdeff7d..e34cd3c58dd7 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/pixelgen.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h index 525c34882fd7..0cdef4a5e8b1 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/platform_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -24,9 +25,6 @@ #include <linux/kernel.h> #include <linux/string.h> -/* For definition of hrt_sleep() */ -#include "hive_isp_css_custom_host_hrt.h" - #define UINT16_MAX USHRT_MAX #define UINT32_MAX UINT_MAX #define UCHAR_MAX (255) diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h index f5fcf6b1d667..a1f7a5839560 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/print_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h index 1bcadd838161..e6978750a362 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/queue.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h index 129446600067..9be45b679386 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/resource.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h index 194cd64a7da8..a7d00c7bb8bc 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/sp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h deleted file mode 100644 index 84efbbe78650..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/string_support.h +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#ifndef __STRING_SUPPORT_H_INCLUDED__ -#define __STRING_SUPPORT_H_INCLUDED__ -#include <platform_support.h> -#include <type_support.h> - -/* - * For all non microsoft cases, we need the following functions - */ - -/* @brief Copy from src_buf to dest_buf. - * - * @param[out] dest_buf. Destination buffer to copy to - * @param[in] dest_size. The size of the destination buffer in bytes - * @param[in] src_buf. The source buffer - * @param[in] src_size. The size of the source buffer in bytes - * @return 0 on success, error code on failure - * @return EINVAL on Invalid arguments - * @return ERANGE on Destination size too small - */ -static inline int memcpy_s( - void *dest_buf, - size_t dest_size, - const void *src_buf, - size_t src_size) -{ - if ((!src_buf) || (!dest_buf)) { - /* Invalid arguments*/ - return EINVAL; - } - - if ((dest_size < src_size) || (src_size == 0)) { - /* Destination too small*/ - return ERANGE; - } - - memcpy(dest_buf, src_buf, src_size); - return 0; -} - -/* @brief Get the length of the string, excluding the null terminator - * - * @param[in] src_str. The source string - * @param[in] max_len. Look only for max_len bytes in the string - * @return Return the string length excluding null character - * @return Return max_len if no null character in the first max_len bytes - * @return Returns 0 if src_str is NULL - */ -static size_t strnlen_s( - const char *src_str, - size_t max_len) -{ - size_t ix; - - if (!src_str) { - /* Invalid arguments*/ - return 0; - } - - for (ix = 0; ix < max_len && src_str[ix] != '\0'; ix++) - ; - - /* On Error, it will return src_size == max_len*/ - return ix; -} - -/* @brief Copy string from src_str to dest_str - * - * @param[out] dest_str. Destination buffer to copy to - * @param[in] dest_size. The size of the destination buffer in bytes - * @param[in] src_str. The source buffer - * @param[in] src_size. The size of the source buffer in bytes - * @return Returns 0 on success - * @return Returns EINVAL on invalid arguments - * @return Returns ERANGE on destination size too small - */ -static inline int strncpy_s( - char *dest_str, - size_t dest_size, - const char *src_str, - size_t src_size) -{ - size_t len; - - if (!dest_str) { - /* Invalid arguments*/ - return EINVAL; - } - - if ((!src_str) || (dest_size == 0)) { - /* Invalid arguments*/ - dest_str[0] = '\0'; - return EINVAL; - } - - len = strnlen_s(src_str, src_size); - - if (len >= dest_size) { - /* Destination too small*/ - dest_str[0] = '\0'; - return ERANGE; - } - - /* dest_str is big enough for the len */ - strncpy(dest_str, src_str, len); - dest_str[len] = '\0'; - return 0; -} - -/* @brief Copy string from src_str to dest_str - * - * @param[out] dest_str. Destination buffer to copy to - * @param[in] dest_size. The size of the destination buffer in bytes - * @param[in] src_str. The source buffer - * @return Returns 0 on success - * @return Returns EINVAL on invalid arguments - * @return Returns ERANGE on destination size too small - */ -static inline int strcpy_s( - char *dest_str, - size_t dest_size, - const char *src_str) -{ - size_t len; - - if (!dest_str) { - /* Invalid arguments*/ - return EINVAL; - } - - if ((!src_str) || (dest_size == 0)) { - /* Invalid arguments*/ - dest_str[0] = '\0'; - return EINVAL; - } - - len = strnlen_s(src_str, dest_size); - - if (len >= dest_size) { - /* Destination too small*/ - dest_str[0] = '\0'; - return ERANGE; - } - - /* dest_str is big enough for the len */ - strncpy(dest_str, src_str, len); - dest_str[len] = '\0'; - return 0; -} - - -#endif /* __STRING_SUPPORT_H_INCLUDED__ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h deleted file mode 100644 index 764fda8dd214..000000000000 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/system_types.h +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ -#ifndef __SYSTEM_TYPES_H_INCLUDED__ -#define __SYSTEM_TYPES_H_INCLUDED__ - -/** -* @file -* Platform specific types. -*/ - -#include "system_local.h" - -#endif /* __SYSTEM_TYPES_H_INCLUDED__ */ diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h index 1f0a5d948316..98d7e922aed9 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/tag.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h index 403abcb828bf..65b2871fb4ea 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/timed_ctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h index bc77537fa73a..b996ee54d4a5 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/type_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h index 9918ca398138..3ea6758aa798 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vamem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h index 873e01e6d054..da479b370192 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_include/vmem.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h index 9f4060319b4b..31121a22d13d 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h index 2b396955cdad..be6162dfbc66 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/queue_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c index a7089ee7462a..8931539a4c01 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h index 01a8977c189e..921e50a4554a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h index 0570a95ec5bf..b14f09adef07 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/host/tag_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h index ce0d99418538..6ae453782515 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/queue_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h index 549c0d2b7970..b256ea19c0eb 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/sw_event_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h index 9db8766b3a7b..af5a47ace32a 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_shared/tag_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h b/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h index a22b771f61f2..301dd923950c 100644 --- a/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h +++ b/drivers/staging/media/atomisp/pci/hive_isp_css_streaming_to_mipi_types_hrt.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/hive_types.h b/drivers/staging/media/atomisp/pci/hive_types.h index 9715893c8a36..addda9b81d7b 100644 --- a/drivers/staging/media/atomisp/pci/hive_types.h +++ b/drivers/staging/media/atomisp/pci/hive_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -78,9 +79,6 @@ typedef hive_uint32 hrt_address; #error adddres width not supported #endif -/* The SP side representation of an HMM virtual address */ -typedef hive_uint32 hrt_vaddress; - /* use 64 bit addresses in simulation, where possible */ typedef hive_uint64 hive_sim_address; diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm.c b/drivers/staging/media/atomisp/pci/hmm/hmm.c index cd70307ffd57..42fef1779862 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -193,7 +194,7 @@ int hmm_init(void) * at the beginning, to avoid hmm_alloc return 0 in the * further allocation. */ - dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, HMM_UNCACHED); + dummy_ptr = hmm_alloc(1, HMM_BO_PRIVATE, 0, NULL, 0); if (!ret) { ret = sysfs_create_group(&atomisp_dev->kobj, @@ -208,6 +209,8 @@ int hmm_init(void) void hmm_cleanup(void) { + if (!dummy_ptr) + return; sysfs_remove_group(&atomisp_dev->kobj, atomisp_attribute_group); /* free dummy memory first */ @@ -219,12 +222,16 @@ void hmm_cleanup(void) } ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type, - int from_highmem, const void __user *userptr, bool cached) + int from_highmem, const void __user *userptr, + const uint16_t attrs) { unsigned int pgnr; struct hmm_buffer_object *bo; + bool cached = attrs & ATOMISP_MAP_FLAG_CACHED; int ret; + WARN_ON(attrs & ATOMISP_MAP_FLAG_CONTIGUOUS); + /* * Check if we are initialized. In the ideal world we wouldn't need * this but we can tackle it once the driver is a lot cleaner @@ -249,7 +256,7 @@ ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type, goto alloc_page_err; } - /* Combind the virtual address and pages togather */ + /* Combine the virtual address and pages together */ ret = hmm_bo_bind(bo); if (ret) { dev_err(atomisp_dev, "hmm_bo_bind failed.\n"); @@ -258,6 +265,13 @@ ia_css_ptr hmm_alloc(size_t bytes, enum hmm_bo_type type, hmm_mem_stat.tol_cnt += pgnr; + if (attrs & ATOMISP_MAP_FLAG_CLEARED) + hmm_set(bo->start, 0, bytes); + + dev_dbg(atomisp_dev, + "%s: pages: 0x%08x (%ld bytes), type: %d from highmem %d, user ptr %p, cached %d\n", + __func__, bo->start, bytes, type, from_highmem, userptr, cached); + return bo->start; bind_err: @@ -272,6 +286,8 @@ void hmm_free(ia_css_ptr virt) { struct hmm_buffer_object *bo; + dev_dbg(atomisp_dev, "%s: free 0x%08x\n", __func__, virt); + WARN_ON(!virt); bo = hmm_bo_device_search_start(&bo_device, (unsigned int)virt); @@ -396,9 +412,14 @@ static int load_and_flush(ia_css_ptr virt, void *data, unsigned int bytes) /* Read function in ISP memory management */ int hmm_load(ia_css_ptr virt, void *data, unsigned int bytes) { + if (!virt) { + dev_warn(atomisp_dev, + "hmm_store: address is NULL\n"); + return -EINVAL; + } if (!data) { dev_err(atomisp_dev, - "hmm_load NULL argument\n"); + "hmm_store: data is a NULL argument\n"); return -EINVAL; } return load_and_flush(virt, data, bytes); @@ -418,6 +439,17 @@ int hmm_store(ia_css_ptr virt, const void *data, unsigned int bytes) char *src, *des; int ret; + if (!virt) { + dev_warn(atomisp_dev, + "hmm_store: address is NULL\n"); + return -EINVAL; + } + if (!data) { + dev_err(atomisp_dev, + "hmm_store: data is a NULL argument\n"); + return -EINVAL; + } + bo = hmm_bo_device_search_in_range(&bo_device, virt); ret = hmm_check_bo(bo, virt); if (ret) diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c index c9e7df0ea5a6..4fb9bfdd2f4c 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm_bo.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * @@ -659,6 +660,8 @@ static void free_private_bo_pages(struct hmm_buffer_object *bo, break; } + /* fall through */ + /* * if dynamic memory pool doesn't exist, need to free * pages to system directly. @@ -854,109 +857,20 @@ static void free_private_pages(struct hmm_buffer_object *bo, kfree(bo->page_obj); } -/* - * Hacked from kernel function __get_user_pages in mm/memory.c - * - * Handle buffers allocated by other kernel space driver and mmaped into user - * space, function Ignore the VM_PFNMAP and VM_IO flag in VMA structure - * - * Get physical pages from user space virtual address and update into page list - */ -static int __get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm, - unsigned long start, int nr_pages, - unsigned int gup_flags, struct page **pages, - struct vm_area_struct **vmas) -{ - int i, ret; - unsigned long vm_flags; - - if (nr_pages <= 0) - return 0; - - VM_BUG_ON(!!pages != !!(gup_flags & FOLL_GET)); - - /* - * Require read or write permissions. - * If FOLL_FORCE is set, we only require the "MAY" flags. - */ - vm_flags = (gup_flags & FOLL_WRITE) ? - (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD); - vm_flags &= (gup_flags & FOLL_FORCE) ? - (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE); - i = 0; - - do { - struct vm_area_struct *vma; - - vma = find_vma(mm, start); - if (!vma) { - dev_err(atomisp_dev, "find_vma failed\n"); - return i ? : -EFAULT; - } - - if (is_vm_hugetlb_page(vma)) { - /* - i = follow_hugetlb_page(mm, vma, pages, vmas, - &start, &nr_pages, i, gup_flags); - */ - continue; - } - - do { - struct page *page; - unsigned long pfn; - - /* - * If we have a pending SIGKILL, don't keep faulting - * pages and potentially allocating memory. - */ - if (unlikely(fatal_signal_pending(current))) { - dev_err(atomisp_dev, - "fatal_signal_pending in %s\n", - __func__); - return i ? i : -ERESTARTSYS; - } - - ret = follow_pfn(vma, start, &pfn); - if (ret) { - dev_err(atomisp_dev, "follow_pfn() failed\n"); - return i ? : -EFAULT; - } - - page = pfn_to_page(pfn); - if (IS_ERR(page)) - return i ? i : PTR_ERR(page); - if (pages) { - pages[i] = page; - get_page(page); - flush_anon_page(vma, page, start); - flush_dcache_page(page); - } - if (vmas) - vmas[i] = vma; - i++; - start += PAGE_SIZE; - nr_pages--; - } while (nr_pages && start < vma->vm_end); - } while (nr_pages); - - return i; -} - -static int get_pfnmap_pages(struct task_struct *tsk, struct mm_struct *mm, - unsigned long start, int nr_pages, int write, int force, - struct page **pages, struct vm_area_struct **vmas) +static void free_user_pages(struct hmm_buffer_object *bo) { - int flags = FOLL_TOUCH; + int i; - if (pages) - flags |= FOLL_GET; - if (write) - flags |= FOLL_WRITE; - if (force) - flags |= FOLL_FORCE; + hmm_mem_stat.usr_size -= bo->pgnr; - return __get_pfnmap_pages(tsk, mm, start, nr_pages, flags, pages, vmas); + if (bo->mem_type == HMM_BO_MEM_TYPE_PFN) { + unpin_user_pages(bo->pages, bo->pgnr); + } else { + for (i = 0; i < bo->pgnr; i++) + put_page(bo->pages[i]); + } + kfree(bo->pages); + kfree(bo->page_obj); } /* @@ -997,11 +911,15 @@ static int alloc_user_pages(struct hmm_buffer_object *bo, * Handle frame buffer allocated in other kerenl space driver * and map to user space */ + + userptr = untagged_addr(userptr); + + bo->pages = pages; + if (vma->vm_flags & (VM_IO | VM_PFNMAP)) { - page_nr = get_pfnmap_pages(current, current->mm, - (unsigned long)userptr, - (int)(bo->pgnr), 1, 0, - pages, NULL); + page_nr = pin_user_pages((unsigned long)userptr, bo->pgnr, + FOLL_LONGTERM | FOLL_WRITE, + pages, NULL); bo->mem_type = HMM_BO_MEM_TYPE_PFN; } else { /*Handle frame buffer allocated in user space*/ @@ -1012,6 +930,13 @@ static int alloc_user_pages(struct hmm_buffer_object *bo, bo->mem_type = HMM_BO_MEM_TYPE_USER; } + dev_dbg(atomisp_dev, "%s: %d %s pages were allocated as 0x%08x\n", + __func__, + bo->pgnr, + bo->mem_type == HMM_BO_MEM_TYPE_USER ? "user" : "pfn", page_nr); + + hmm_mem_stat.usr_size += bo->pgnr; + /* can be written by caller, not forced */ if (page_nr != bo->pgnr) { dev_err(atomisp_dev, @@ -1024,29 +949,14 @@ static int alloc_user_pages(struct hmm_buffer_object *bo, bo->page_obj[i].page = pages[i]; bo->page_obj[i].type = HMM_PAGE_TYPE_GENERAL; } - hmm_mem_stat.usr_size += bo->pgnr; - kfree(pages); return 0; out_of_mem: - for (i = 0; i < page_nr; i++) - put_page(pages[i]); - kfree(pages); - kfree(bo->page_obj); - return -ENOMEM; -} + free_user_pages(bo); -static void free_user_pages(struct hmm_buffer_object *bo) -{ - int i; - - for (i = 0; i < bo->pgnr; i++) - put_page(bo->page_obj[i].page); - hmm_mem_stat.usr_size -= bo->pgnr; - - kfree(bo->page_obj); + return -ENOMEM; } /* diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c b/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c index 1a87af68a924..eaf97e5f3b68 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm_dynamic_pool.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c b/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c index d739ed914d65..57525fece921 100644 --- a/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c +++ b/drivers/staging/media/atomisp/pci/hmm/hmm_reserved_pool.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c b/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c deleted file mode 100644 index 976a2cb51354..000000000000 --- a/drivers/staging/media/atomisp/pci/hmm/hmm_vm.c +++ /dev/null @@ -1,212 +0,0 @@ -/* - * Support for Medifield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ -/* - * This file contains function for ISP virtual address management in ISP driver - */ -#include <linux/kernel.h> -#include <linux/types.h> -#include <linux/mm.h> -#include <linux/slab.h> -#include <asm/page.h> - -#include "atomisp_internal.h" -#include "mmu/isp_mmu.h" -#include "hmm/hmm_vm.h" -#include "hmm/hmm_common.h" - -static unsigned int vm_node_end(unsigned int start, unsigned int pgnr) -{ - return start + pgnr_to_size(pgnr); -} - -static int addr_in_vm_node(unsigned int addr, - struct hmm_vm_node *node) -{ - return (addr >= node->start) && (addr < (node->start + node->size)); -} - -int hmm_vm_init(struct hmm_vm *vm, unsigned int start, - unsigned int size) -{ - if (!vm) - return -1; - - vm->start = start; - vm->pgnr = size_to_pgnr_ceil(size); - vm->size = pgnr_to_size(vm->pgnr); - - INIT_LIST_HEAD(&vm->vm_node_list); - spin_lock_init(&vm->lock); - vm->cache = kmem_cache_create("atomisp_vm", sizeof(struct hmm_vm_node), - 0, 0, NULL); - - return vm->cache ? 0 : -ENOMEM; -} - -void hmm_vm_clean(struct hmm_vm *vm) -{ - struct hmm_vm_node *node, *tmp; - struct list_head new_head; - - if (!vm) - return; - - spin_lock(&vm->lock); - list_replace_init(&vm->vm_node_list, &new_head); - spin_unlock(&vm->lock); - - list_for_each_entry_safe(node, tmp, &new_head, list) { - list_del(&node->list); - kmem_cache_free(vm->cache, node); - } - - kmem_cache_destroy(vm->cache); -} - -static struct hmm_vm_node *alloc_hmm_vm_node(unsigned int pgnr, - struct hmm_vm *vm) -{ - struct hmm_vm_node *node; - - node = kmem_cache_alloc(vm->cache, GFP_KERNEL); - if (!node) - return NULL; - - INIT_LIST_HEAD(&node->list); - node->pgnr = pgnr; - node->size = pgnr_to_size(pgnr); - node->vm = vm; - - return node; -} - -struct hmm_vm_node *hmm_vm_alloc_node(struct hmm_vm *vm, unsigned int pgnr) -{ - struct list_head *head; - struct hmm_vm_node *node, *cur, *next; - unsigned int vm_start, vm_end; - unsigned int addr; - unsigned int size; - - if (!vm) - return NULL; - - vm_start = vm->start; - vm_end = vm_node_end(vm->start, vm->pgnr); - size = pgnr_to_size(pgnr); - - addr = vm_start; - head = &vm->vm_node_list; - - node = alloc_hmm_vm_node(pgnr, vm); - if (!node) { - dev_err(atomisp_dev, "no memory to allocate hmm vm node.\n"); - return NULL; - } - - spin_lock(&vm->lock); - /* - * if list is empty, the loop code will not be executed. - */ - list_for_each_entry(cur, head, list) { - /* Add gap between vm areas as helper to not hide overflow */ - addr = PAGE_ALIGN(vm_node_end(cur->start, cur->pgnr) + 1); - - if (list_is_last(&cur->list, head)) { - if (addr + size > vm_end) { - /* vm area does not have space anymore */ - spin_unlock(&vm->lock); - kmem_cache_free(vm->cache, node); - dev_err(atomisp_dev, - "no enough virtual address space.\n"); - return NULL; - } - - /* We still have vm space to add new node to tail */ - break; - } - - next = list_entry(cur->list.next, struct hmm_vm_node, list); - if ((next->start - addr) > size) - break; - } - node->start = addr; - node->vm = vm; - list_add(&node->list, &cur->list); - spin_unlock(&vm->lock); - - return node; -} - -void hmm_vm_free_node(struct hmm_vm_node *node) -{ - struct hmm_vm *vm; - - if (!node) - return; - - vm = node->vm; - - spin_lock(&vm->lock); - list_del(&node->list); - spin_unlock(&vm->lock); - - kmem_cache_free(vm->cache, node); -} - -struct hmm_vm_node *hmm_vm_find_node_start(struct hmm_vm *vm, unsigned int addr) -{ - struct hmm_vm_node *node; - - if (!vm) - return NULL; - - spin_lock(&vm->lock); - - list_for_each_entry(node, &vm->vm_node_list, list) { - if (node->start == addr) { - spin_unlock(&vm->lock); - return node; - } - } - - spin_unlock(&vm->lock); - return NULL; -} - -struct hmm_vm_node *hmm_vm_find_node_in_range(struct hmm_vm *vm, - unsigned int addr) -{ - struct hmm_vm_node *node; - - if (!vm) - return NULL; - - spin_lock(&vm->lock); - - list_for_each_entry(node, &vm->vm_node_list, list) { - if (addr_in_vm_node(addr, node)) { - spin_unlock(&vm->lock); - return node; - } - } - - spin_unlock(&vm->lock); - return NULL; -} diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h deleted file mode 100644 index c6893d712d61..000000000000 --- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_custom_host_hrt.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Support for Medifield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ -#ifndef _hive_isp_css_custom_host_hrt_h_ -#define _hive_isp_css_custom_host_hrt_h_ - -#include <linux/delay.h> -#include "atomisp_helper.h" - -/* - * _hrt_master_port_store/load/uload -macros using __force attributed - * cast to intentional dereferencing __iomem attributed (noderef) - * pointer from atomisp_get_io_virt_addr - */ -#define _hrt_master_port_store_8(a, d) \ - (*((s8 __force *)atomisp_get_io_virt_addr(a)) = (d)) - -#define _hrt_master_port_store_16(a, d) \ - (*((s16 __force *)atomisp_get_io_virt_addr(a)) = (d)) - -#define _hrt_master_port_store_32(a, d) \ - (*((s32 __force *)atomisp_get_io_virt_addr(a)) = (d)) - -#define _hrt_master_port_load_8(a) \ - (*(s8 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_load_16(a) \ - (*(s16 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_load_32(a) \ - (*(s32 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_uload_8(a) \ - (*(u8 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_uload_16(a) \ - (*(u16 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_uload_32(a) \ - (*(u32 __force *)atomisp_get_io_virt_addr(a)) - -#define _hrt_master_port_store_8_volatile(a, d) _hrt_master_port_store_8(a, d) -#define _hrt_master_port_store_16_volatile(a, d) _hrt_master_port_store_16(a, d) -#define _hrt_master_port_store_32_volatile(a, d) _hrt_master_port_store_32(a, d) - -#define _hrt_master_port_load_8_volatile(a) _hrt_master_port_load_8(a) -#define _hrt_master_port_load_16_volatile(a) _hrt_master_port_load_16(a) -#define _hrt_master_port_load_32_volatile(a) _hrt_master_port_load_32(a) - -#define _hrt_master_port_uload_8_volatile(a) _hrt_master_port_uload_8(a) -#define _hrt_master_port_uload_16_volatile(a) _hrt_master_port_uload_16(a) -#define _hrt_master_port_uload_32_volatile(a) _hrt_master_port_uload_32(a) - -static inline void hrt_sleep(void) -{ - udelay(1); -} - -static inline u32 _hrt_mem_store(u32 to, const void *from, size_t n) -{ - unsigned int i; - u32 _to = to; - const char *_from = (const char *)from; - - for (i = 0; i < n; i++, _to++, _from++) - _hrt_master_port_store_8(_to, *_from); - return _to; -} - -static inline void *_hrt_mem_load(u32 from, void *to, size_t n) -{ - unsigned int i; - char *_to = (char *)to; - u32 _from = from; - - for (i = 0; i < n; i++, _to++, _from++) - *_to = _hrt_master_port_load_8(_from); - return _to; -} - -static inline u32 _hrt_mem_set(u32 to, int c, size_t n) -{ - unsigned int i; - u32 _to = to; - - for (i = 0; i < n; i++, _to++) - _hrt_master_port_store_8(_to, c); - return _to; -} - -#endif /* _hive_isp_css_custom_host_hrt_h_ */ diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c deleted file mode 100644 index 236f27b50386..000000000000 --- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.c +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Support for Medifield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ - -#include "atomisp_internal.h" - -#include "hive_isp_css_mm_hrt.h" -#include "hmm/hmm.h" - -#define __page_align(size) (((size) + (PAGE_SIZE - 1)) & (~(PAGE_SIZE - 1))) - -static void __user *my_userptr; -static unsigned int my_num_pages; -static enum hrt_userptr_type my_usr_type; - -void hrt_isp_css_mm_set_user_ptr(void __user *userptr, - unsigned int num_pages, - enum hrt_userptr_type type) -{ - my_userptr = userptr; - my_num_pages = num_pages; - my_usr_type = type; -} - -static ia_css_ptr __hrt_isp_css_mm_alloc(size_t bytes, - const void __user *userptr, - unsigned int num_pages, - enum hrt_userptr_type type, - bool cached) -{ -#ifdef CONFIG_ION - if (type == HRT_USR_ION) - return hmm_alloc(bytes, HMM_BO_ION, 0, - userptr, cached); - -#endif - if (type == HRT_USR_PTR) { - if (!userptr) - return hmm_alloc(bytes, HMM_BO_PRIVATE, 0, - NULL, cached); - else { - if (num_pages < ((__page_align(bytes)) >> PAGE_SHIFT)) - dev_err(atomisp_dev, - "user space memory size is less than the expected size..\n"); - else if (num_pages > ((__page_align(bytes)) - >> PAGE_SHIFT)) - dev_err(atomisp_dev, - "user space memory size is large than the expected size..\n"); - - return hmm_alloc(bytes, HMM_BO_USER, 0, - userptr, cached); - } - } else { - dev_err(atomisp_dev, "user ptr type is incorrect.\n"); - return 0; - } -} - -ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes) -{ - return __hrt_isp_css_mm_alloc(bytes, my_userptr, - my_num_pages, my_usr_type, false); -} - -ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes, - const void __user *userptr, - unsigned int num_pages, - enum hrt_userptr_type type, - bool cached) -{ - return __hrt_isp_css_mm_alloc(bytes, userptr, num_pages, - type, cached); -} - -ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes) -{ - if (!my_userptr) - return hmm_alloc(bytes, HMM_BO_PRIVATE, 0, NULL, - HMM_CACHED); - else { - if (my_num_pages < ((__page_align(bytes)) >> PAGE_SHIFT)) - dev_err(atomisp_dev, - "user space memory size is less than the expected size..\n"); - else if (my_num_pages > ((__page_align(bytes)) >> PAGE_SHIFT)) - dev_err(atomisp_dev, - "user space memory size is large than the expected size..\n"); - - return hmm_alloc(bytes, HMM_BO_USER, 0, - my_userptr, HMM_CACHED); - } -} - -ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes) -{ - ia_css_ptr ptr = hrt_isp_css_mm_alloc(bytes); - - if (ptr) - hmm_set(ptr, 0, bytes); - return ptr; -} - -ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes) -{ - ia_css_ptr ptr = hrt_isp_css_mm_alloc_cached(bytes); - - if (ptr) - hmm_set(ptr, 0, bytes); - return ptr; -} diff --git a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h b/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h deleted file mode 100644 index 818ecf90b1f5..000000000000 --- a/drivers/staging/media/atomisp/pci/hrt/hive_isp_css_mm_hrt.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Support for Medfield PNW Camera Imaging ISP subsystem. - * - * Copyright (c) 2010 Intel Corporation. All Rights Reserved. - * - * Copyright (c) 2010 Silicon Hive www.siliconhive.com. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License version - * 2 as published by the Free Software Foundation. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * - */ - -#ifndef _hive_isp_css_mm_hrt_h_ -#define _hive_isp_css_mm_hrt_h_ - -#include <hmm/hmm.h> -#include <hrt/hive_isp_css_custom_host_hrt.h> - -#define HRT_BUF_FLAG_CACHED BIT(0) - -enum hrt_userptr_type { - HRT_USR_PTR = 0, -#ifdef CONFIG_ION - HRT_USR_ION, -#endif -}; - -struct hrt_userbuffer_attr { - enum hrt_userptr_type type; - unsigned int pgnr; -}; - -void hrt_isp_css_mm_set_user_ptr(void __user *userptr, - unsigned int num_pages, enum hrt_userptr_type); - -/* Allocate memory, returns a virtual address */ -ia_css_ptr hrt_isp_css_mm_alloc(size_t bytes); -ia_css_ptr hrt_isp_css_mm_alloc_user_ptr(size_t bytes, - const void __user *userptr, - unsigned int num_pages, - enum hrt_userptr_type, - bool cached); -ia_css_ptr hrt_isp_css_mm_alloc_cached(size_t bytes); - -/* allocate memory and initialize with zeros, - returns a virtual address */ -ia_css_ptr hrt_isp_css_mm_calloc(size_t bytes); -ia_css_ptr hrt_isp_css_mm_calloc_cached(size_t bytes); - -#endif /* _hive_isp_css_mm_hrt_h_ */ diff --git a/drivers/staging/media/atomisp/pci/ia_css.h b/drivers/staging/media/atomisp/pci/ia_css.h index e44df6916d90..d83e1ae5b0b3 100644 --- a/drivers/staging/media/atomisp/pci/ia_css.h +++ b/drivers/staging/media/atomisp/pci/ia_css.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* * Support for Intel Camera Imaging ISP subsystem. diff --git a/drivers/staging/media/atomisp/pci/ia_css_3a.h b/drivers/staging/media/atomisp/pci/ia_css_3a.h index a79941a2e0f2..70cfc915cc56 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_3a.h +++ b/drivers/staging/media/atomisp/pci/ia_css_3a.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -106,7 +107,7 @@ struct ia_css_isp_3a_statistics_map { * used. * Always use this function, never copy the buffer directly. */ -enum ia_css_err +int ia_css_get_3a_statistics(struct ia_css_3a_statistics *host_stats, const struct ia_css_isp_3a_statistics *isp_stats); diff --git a/drivers/staging/media/atomisp/pci/ia_css_acc_types.h b/drivers/staging/media/atomisp/pci/ia_css_acc_types.h index d281846eeba5..36583ab12e3f 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_acc_types.h +++ b/drivers/staging/media/atomisp/pci/ia_css_acc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -19,7 +20,7 @@ * This file contains types used for acceleration */ -#include <system_types.h> /* HAS_IRQ_MAP_VERSION_# */ +#include <system_local.h> /* HAS_IRQ_MAP_VERSION_# */ #include <type_support.h> #include <platform_support.h> #include <debug_global.h> @@ -62,8 +63,8 @@ enum ia_css_cell_type { */ enum ia_css_fw_type { ia_css_sp_firmware, /** Firmware for the SP */ - ia_css_isp_firmware, /** Firmware for the ISP */ - ia_css_bootloader_firmware, /** Firmware for the BootLoader */ + ia_css_isp_firmware, /** Firmware for the ISP */ + ia_css_bootloader_firmware, /** Firmware for the BootLoader */ ia_css_acc_firmware /** Firmware for accelrations */ }; diff --git a/drivers/staging/media/atomisp/pci/ia_css_buffer.h b/drivers/staging/media/atomisp/pci/ia_css_buffer.h index 38e1f4791029..b1e8357b94b5 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_buffer.h +++ b/drivers/staging/media/atomisp/pci/ia_css_buffer.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_control.h b/drivers/staging/media/atomisp/pci/ia_css_control.h index 248040b3ec07..88f031a63ba2 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_control.h +++ b/drivers/staging/media/atomisp/pci/ia_css_control.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -37,15 +38,15 @@ * of the L1 page table. This is a physical * address or index. * @param[in] irq_type The type of interrupt to be used (edge or level) - * @return Returns IA_CSS_ERR_INTERNAL_ERROR in case of any - * errors and IA_CSS_SUCCESS otherwise. + * @return Returns -EINVAL in case of any + * errors and 0 otherwise. * * This function initializes the API which includes allocating and initializing * internal data structures. This also interprets the firmware package. All * contents of this firmware package are copied into local data structures, so * the fw pointer could be freed after this function completes. */ -enum ia_css_err ia_css_init(struct device *dev, +int ia_css_init(struct device *dev, const struct ia_css_env *env, const struct ia_css_fw *fw, u32 l1_base, @@ -77,7 +78,7 @@ ia_css_uninit(void); * This function should only be called when the SP is not running, calling it * when the SP is running will result in an error value being returned. } */ -enum ia_css_err +int ia_css_enable_isys_event_queue(bool enable); /* @brief Test whether the ISP has started. @@ -109,23 +110,23 @@ ia_css_sp_has_terminated(void); /* @brief start SP hardware * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * It will boot the SP hardware and start multi-threading infrastructure. * All threads will be started and blocked by semaphore. This function should * be called before any ia_css_stream_start(). */ -enum ia_css_err +int ia_css_start_sp(void); /* @brief stop SP hardware * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * This function will terminate all threads and shut down SP. It should be * called after all ia_css_stream_stop(). */ -enum ia_css_err +int ia_css_stop_sp(void); #endif /* __IA_CSS_CONTROL_H */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_device_access.c b/drivers/staging/media/atomisp/pci/ia_css_device_access.c index 6ad8687cf08b..9cd2d3caa5c9 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_device_access.c +++ b/drivers/staging/media/atomisp/pci/ia_css_device_access.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -14,7 +15,7 @@ #include "ia_css_device_access.h" #include <type_support.h> /* for uint*, size_t */ -#include <system_types.h> /* for hrt_address */ +#include <system_local.h> /* for hrt_address */ #include <ia_css_env.h> /* for ia_css_hw_access_env */ #include <assert_support.h> /* for assert */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_device_access.h b/drivers/staging/media/atomisp/pci/ia_css_device_access.h index b2bf7d540b62..07d611fdd19f 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_device_access.h +++ b/drivers/staging/media/atomisp/pci/ia_css_device_access.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -20,7 +21,7 @@ */ #include <type_support.h> /* for uint*, size_t */ -#include <system_types.h> /* for hrt_address */ +#include <system_local.h> /* for hrt_address */ #include <ia_css_env.h> /* for ia_css_hw_access_env */ void diff --git a/drivers/staging/media/atomisp/pci/ia_css_dvs.h b/drivers/staging/media/atomisp/pci/ia_css_dvs.h index e647f73c3bd6..3367dfd64050 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_dvs.h +++ b/drivers/staging/media/atomisp/pci/ia_css_dvs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -94,7 +95,7 @@ union ia_css_dvs_statistics_host { * advised to map the ISP memory into a host-side pointer and use * the ia_css_translate_dvs_statistics() function instead. */ -enum ia_css_err +int ia_css_get_dvs_statistics(struct ia_css_dvs_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats); @@ -128,7 +129,7 @@ ia_css_translate_dvs_statistics( * advised to map the ISP memory into a host-side pointer and use * the ia_css_translate_dvs2_statistics() function instead. */ -enum ia_css_err +int ia_css_get_dvs2_statistics(struct ia_css_dvs2_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats); diff --git a/drivers/staging/media/atomisp/pci/ia_css_env.h b/drivers/staging/media/atomisp/pci/ia_css_env.h index 8b0218ee658d..8debf334c15c 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_env.h +++ b/drivers/staging/media/atomisp/pci/ia_css_env.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_err.h b/drivers/staging/media/atomisp/pci/ia_css_err.h index 375952a7782e..98401a4a171d 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_err.h +++ b/drivers/staging/media/atomisp/pci/ia_css_err.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -20,27 +21,6 @@ * functions in the CSS-API. */ -/* Errors, these values are used as the return value for most - * functions in this API. - */ -enum ia_css_err { - IA_CSS_SUCCESS, - IA_CSS_ERR_INTERNAL_ERROR, - IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY, - IA_CSS_ERR_INVALID_ARGUMENTS, - IA_CSS_ERR_SYSTEM_NOT_IDLE, - IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER, - IA_CSS_ERR_QUEUE_IS_FULL, - IA_CSS_ERR_QUEUE_IS_EMPTY, - IA_CSS_ERR_RESOURCE_NOT_AVAILABLE, - IA_CSS_ERR_RESOURCE_LIST_TO_SMALL, - IA_CSS_ERR_RESOURCE_ITEMS_STILL_ALLOCATED, - IA_CSS_ERR_RESOURCE_EXHAUSTED, - IA_CSS_ERR_RESOURCE_ALREADY_ALLOCATED, - IA_CSS_ERR_VERSION_MISMATCH, - IA_CSS_ERR_NOT_SUPPORTED -}; - /* FW warnings. This enum contains a value for each warning that * the SP FW could indicate potential performance issue */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_event_public.h b/drivers/staging/media/atomisp/pci/ia_css_event_public.h index 5c0470fa4a74..08ea801dd5ac 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_event_public.h +++ b/drivers/staging/media/atomisp/pci/ia_css_event_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -143,9 +144,9 @@ struct ia_css_event { * * @param[out] event Pointer to the event struct which will be filled by * this function if an event is available. - * @return IA_CSS_ERR_QUEUE_IS_EMPTY if no events are + * @return -ENODATA if no events are * available or - * IA_CSS_SUCCESS otherwise. + * 0 otherwise. * * This function dequeues an event from the PSYS event queue. The queue is * between the Host CPU and the CSS system. This function can be @@ -153,31 +154,31 @@ struct ia_css_event { * was available and can be used in a polling-like situation where the NO_EVENT * return value is used to determine whether an event was available or not. */ -enum ia_css_err +int ia_css_dequeue_psys_event(struct ia_css_event *event); /* @brief Dequeue an event from the CSS system. * * @param[out] event Pointer to the event struct which will be filled by * this function if an event is available. - * @return IA_CSS_ERR_QUEUE_IS_EMPTY if no events are + * @return -ENODATA if no events are * available or - * IA_CSS_SUCCESS otherwise. + * 0 otherwise. * * deprecated{Use ia_css_dequeue_psys_event instead}. * Unless the isys event queue is explicitly enabled, this function will * dequeue both isys (EOF) and psys events (all others). */ -enum ia_css_err +int ia_css_dequeue_event(struct ia_css_event *event); /* @brief Dequeue an ISYS event from the CSS system. * * @param[out] event Pointer to the event struct which will be filled by * this function if an event is available. - * @return IA_CSS_ERR_QUEUE_IS_EMPTY if no events are + * @return -ENODATA if no events are * available or - * IA_CSS_SUCCESS otherwise. + * 0 otherwise. * * This function dequeues an event from the ISYS event queue. The queue is * between host and the CSS system. @@ -190,7 +191,7 @@ ia_css_dequeue_event(struct ia_css_event *event); * incurring additional latency due to locks being held by other CSS API * functions. */ -enum ia_css_err +int ia_css_dequeue_isys_event(struct ia_css_event *event); #endif /* __IA_CSS_EVENT_PUBLIC_H */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_firmware.h b/drivers/staging/media/atomisp/pci/ia_css_firmware.h index 50817162703b..edab72256494 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_firmware.h +++ b/drivers/staging/media/atomisp/pci/ia_css_firmware.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -36,8 +37,8 @@ struct ia_css_fw { * printing. * @param[in] fw Firmware package containing the firmware for all * predefined ISP binaries. - * @return Returns IA_CSS_ERR_INTERNAL_ERROR in case of any - * errors and IA_CSS_SUCCESS otherwise. + * @return Returns -EINVAL in case of any + * errors and 0 otherwise. * * This function interprets the firmware package. All * contents of this firmware package are copied into local data structures, so @@ -47,7 +48,7 @@ struct ia_css_fw { * speeds up ia_css_init (ia_css_init is called each time a stream is created but the * firmware only needs to be loaded once). */ -enum ia_css_err +int ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, const struct ia_css_fw *fw); diff --git a/drivers/staging/media/atomisp/pci/ia_css_frac.h b/drivers/staging/media/atomisp/pci/ia_css_frac.h index 59720370cb8e..661af9225b19 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_frac.h +++ b/drivers/staging/media/atomisp/pci/ia_css_frac.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_frame_format.h b/drivers/staging/media/atomisp/pci/ia_css_frame_format.h index 2f177edc36ac..093e23a9b079 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_frame_format.h +++ b/drivers/staging/media/atomisp/pci/ia_css_frame_format.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h index 69e9143e5418..96c86f0dc81c 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_frame_public.h +++ b/drivers/staging/media/atomisp/pci/ia_css_frame_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -121,8 +122,7 @@ struct ia_css_frame_info { struct ia_css_crop_info crop_info; }; -#define IA_CSS_BINARY_DEFAULT_FRAME_INFO \ -(struct ia_css_frame_info) { \ +#define IA_CSS_BINARY_DEFAULT_FRAME_INFO { \ .format = IA_CSS_FRAME_FORMAT_NUM, \ .raw_bayer_order = IA_CSS_BAYER_ORDER_NUM, \ } @@ -184,8 +184,7 @@ struct ia_css_frame { info.format */ }; -#define DEFAULT_FRAME \ -(struct ia_css_frame) { \ +#define DEFAULT_FRAME { \ .info = IA_CSS_BINARY_DEFAULT_FRAME_INFO, \ .dynamic_queue_id = SH_CSS_INVALID_QUEUE_ID, \ .buf_type = IA_CSS_BUFFER_TYPE_INVALID, \ @@ -214,7 +213,7 @@ void ia_css_frame_zero(struct ia_css_frame *frame); * Allocate a CSS frame structure. The memory for the frame data will be * allocated in the CSS address space. */ -enum ia_css_err +int ia_css_frame_allocate(struct ia_css_frame **frame, unsigned int width, unsigned int height, @@ -232,7 +231,7 @@ ia_css_frame_allocate(struct ia_css_frame **frame, * This is a convenience function, implemented on top of * ia_css_frame_allocate(). */ -enum ia_css_err +int ia_css_frame_allocate_from_info(struct ia_css_frame **frame, const struct ia_css_frame_info *info); /* @brief Free a CSS frame structure. @@ -260,7 +259,7 @@ ia_css_frame_free(struct ia_css_frame *frame); * physically contiguous memory. * Deprecated. */ -enum ia_css_err +int ia_css_frame_allocate_contiguous(struct ia_css_frame **frame, unsigned int width, unsigned int height, @@ -280,7 +279,7 @@ ia_css_frame_allocate_contiguous(struct ia_css_frame **frame, * Only for FPGA display driver which needs physically contiguous memory. * Deprecated. */ -enum ia_css_err +int ia_css_frame_allocate_contiguous_from_info(struct ia_css_frame **frame, const struct ia_css_frame_info *info); @@ -293,7 +292,7 @@ ia_css_frame_allocate_contiguous_from_info(struct ia_css_frame **frame, * Allocate an empty CSS frame with no data buffer using the parameters * in the frame info. */ -enum ia_css_err +int ia_css_frame_create_from_info(struct ia_css_frame **frame, const struct ia_css_frame_info *info); @@ -310,7 +309,7 @@ ia_css_frame_create_from_info(struct ia_css_frame **frame, * free the mapped_data buffer. However if ia_css_frame_free() is called and * the frame had a valid data buffer, it would be freed along with the frame. */ -enum ia_css_err +int ia_css_frame_set_data(struct ia_css_frame *frame, const ia_css_ptr mapped_data, size_t data_size_bytes); @@ -331,12 +330,12 @@ ia_css_frame_set_data(struct ia_css_frame *frame, * ia_css_frame_allocate() does, but instead of allocating the memory, it will * map the pre-allocated memory into the CSS address space. */ -enum ia_css_err +int ia_css_frame_map(struct ia_css_frame **frame, const struct ia_css_frame_info *info, const void __user *data, u16 attribute, - void *context); + unsigned int pgnr); /* @brief Unmap a CSS frame structure. * diff --git a/drivers/staging/media/atomisp/pci/ia_css_host_data.h b/drivers/staging/media/atomisp/pci/ia_css_host_data.h index bc82e97d24cb..f54cc504f5d4 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_host_data.h +++ b/drivers/staging/media/atomisp/pci/ia_css_host_data.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* * Support for Intel Camera Imaging ISP subsystem. diff --git a/drivers/staging/media/atomisp/pci/ia_css_input_port.h b/drivers/staging/media/atomisp/pci/ia_css_input_port.h index ad9ca5449369..9772b6928239 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_input_port.h +++ b/drivers/staging/media/atomisp/pci/ia_css_input_port.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_irq.h b/drivers/staging/media/atomisp/pci/ia_css_irq.h index 7716373553e0..3b81a39cfe97 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_irq.h +++ b/drivers/staging/media/atomisp/pci/ia_css_irq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -55,11 +56,8 @@ enum ia_css_irq_info { /** the css receiver received the end of frame */ IA_CSS_IRQ_INFO_CSS_RECEIVER_SOL = 1 << 4, /** the css receiver received the start of line */ - IA_CSS_IRQ_INFO_PSYS_EVENTS_READY = 1 << 5, + IA_CSS_IRQ_INFO_EVENTS_READY = 1 << 5, /** One or more events are available in the PSYS event queue */ - IA_CSS_IRQ_INFO_EVENTS_READY = IA_CSS_IRQ_INFO_PSYS_EVENTS_READY, - /** deprecated{obsolete version of IA_CSS_IRQ_INFO_PSYS_EVENTS_READY, - * same functionality.} */ IA_CSS_IRQ_INFO_CSS_RECEIVER_EOL = 1 << 6, /** the css receiver received the end of line */ IA_CSS_IRQ_INFO_CSS_RECEIVER_SIDEBAND_CHANGED = 1 << 7, @@ -144,14 +142,14 @@ struct ia_css_irq { * @return If an error is encountered during the interrupt info * and no interrupt could be translated successfully, this * will return IA_CSS_INTERNAL_ERROR. Otherwise - * IA_CSS_SUCCESS. + * 0. * * This function is expected to be executed after an interrupt has been sent * to the IA from the CSS. This function returns information about the interrupt * which is needed by the IA code to properly handle the interrupt. This * information includes the image pipe, buffer type etc. */ -enum ia_css_err +int ia_css_irq_translate(unsigned int *info); /* @brief Get CSI receiver error info. @@ -227,9 +225,9 @@ ia_css_rx_port_clear_irq_info(enum mipi_port_id port, unsigned int irq_bits); * @return Returns IA_CSS_INTERNAL_ERROR if this interrupt * type cannot be enabled/disabled which is true for * CSS internal interrupts. Otherwise returns - * IA_CSS_SUCCESS. + * 0. */ -enum ia_css_err +int ia_css_irq_enable(enum ia_css_irq_info type, bool enable); #endif /* __IA_CSS_IRQ_H */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h b/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h index 6545efd24cbe..1abb2fd6a913 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h +++ b/drivers/staging/media/atomisp/pci/ia_css_isp_configs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_params.h b/drivers/staging/media/atomisp/pci/ia_css_isp_params.h index b8b3c48492ae..6e3082b39ed6 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_isp_params.h +++ b/drivers/staging/media/atomisp/pci/ia_css_isp_params.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_isp_states.h b/drivers/staging/media/atomisp/pci/ia_css_isp_states.h index cc9cdcd0e2be..9f6c342a1705 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_isp_states.h +++ b/drivers/staging/media/atomisp/pci/ia_css_isp_states.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -65,7 +66,7 @@ struct ia_css_state_memory_offsets { #include "ia_css_binary.h" /* struct ia_css_binary */ /* Code generated by genparam/genstate.c:gen_state_init_table() */ -extern void (* ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( +extern void (*ia_css_kernel_init_state[IA_CSS_NUM_STATE_IDS])( const struct ia_css_binary *binary); #endif /* IA_CSS_INCLUDE_STATE */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_memory_access.c b/drivers/staging/media/atomisp/pci/ia_css_memory_access.c deleted file mode 100644 index 8d1356047448..000000000000 --- a/drivers/staging/media/atomisp/pci/ia_css_memory_access.c +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015-2017, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -#include <type_support.h> -#include <system_types.h> -#include <assert_support.h> -#include <memory_access.h> -#include <ia_css_env.h> -#include <hrt/hive_isp_css_mm_hrt.h> - -const hrt_vaddress mmgr_NULL = (hrt_vaddress)0; -const hrt_vaddress mmgr_EXCEPTION = (hrt_vaddress)-1; - -hrt_vaddress -mmgr_malloc(const size_t size) -{ - return mmgr_alloc_attr(size, 0); -} - -hrt_vaddress mmgr_alloc_attr(const size_t size, const uint16_t attrs) -{ - u16 masked_attrs = attrs & MMGR_ATTRIBUTE_MASK; - - WARN_ON(attrs & MMGR_ATTRIBUTE_CONTIGUOUS); - - if (masked_attrs & MMGR_ATTRIBUTE_CLEARED) { - if (masked_attrs & MMGR_ATTRIBUTE_CACHED) - return (ia_css_ptr) hrt_isp_css_mm_calloc_cached(size); - else - return (ia_css_ptr) hrt_isp_css_mm_calloc(size); - } else { - if (masked_attrs & MMGR_ATTRIBUTE_CACHED) - return (ia_css_ptr) hrt_isp_css_mm_alloc_cached(size); - else - return (ia_css_ptr) hrt_isp_css_mm_alloc(size); - } -} - -hrt_vaddress -mmgr_calloc(const size_t N, const size_t size) -{ - return mmgr_alloc_attr(size * N, MMGR_ATTRIBUTE_CLEARED); -} - -void mmgr_clear(hrt_vaddress vaddr, const size_t size) -{ - if (vaddr) - hmm_set(vaddr, 0, size); -} - -void mmgr_load(const hrt_vaddress vaddr, void *data, const size_t size) -{ - if (vaddr && data) - hmm_load(vaddr, data, size); -} - -void -mmgr_store(const hrt_vaddress vaddr, const void *data, const size_t size) -{ - if (vaddr && data) - hmm_store(vaddr, data, size); -} - -hrt_vaddress -mmgr_mmap(const void __user *ptr, const size_t size, - u16 attribute, void *context) -{ - struct hrt_userbuffer_attr *userbuffer_attr = context; - - return hrt_isp_css_mm_alloc_user_ptr( - size, ptr, userbuffer_attr->pgnr, - userbuffer_attr->type, - attribute & HRT_BUF_FLAG_CACHED); -} diff --git a/drivers/staging/media/atomisp/pci/ia_css_metadata.h b/drivers/staging/media/atomisp/pci/ia_css_metadata.h index 0212d71b3355..9eb1b76a3b2a 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_metadata.h +++ b/drivers/staging/media/atomisp/pci/ia_css_metadata.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_mipi.h b/drivers/staging/media/atomisp/pci/ia_css_mipi.h index c02138ee2511..56a2fca8117f 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_mipi.h +++ b/drivers/staging/media/atomisp/pci/ia_css_mipi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -37,7 +38,7 @@ * * Specifies a CSS MIPI frame buffer: size in memory words (32B). */ -enum ia_css_err +int ia_css_mipi_frame_specify(const unsigned int size_mem_words, const bool contiguous); @@ -54,7 +55,7 @@ ia_css_mipi_frame_specify(const unsigned int size_mem_words, * * */ -enum ia_css_err +int ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port, const unsigned int size_mem_words); #endif @@ -71,7 +72,7 @@ ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port, * * Calculate the size of a mipi frame, based on the resolution and format. */ -enum ia_css_err +int ia_css_mipi_frame_calculate_size(const unsigned int width, const unsigned int height, const enum atomisp_input_format format, diff --git a/drivers/staging/media/atomisp/pci/ia_css_mmu.h b/drivers/staging/media/atomisp/pci/ia_css_mmu.h index 13c21056bfbf..8f04d196c646 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_mmu.h +++ b/drivers/staging/media/atomisp/pci/ia_css_mmu.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h b/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h index 1021e4f380a5..dc6542aa64f2 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h +++ b/drivers/staging/media/atomisp/pci/ia_css_mmu_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_morph.h b/drivers/staging/media/atomisp/pci/ia_css_morph.h index de409638d009..9c4b41b94256 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_morph.h +++ b/drivers/staging/media/atomisp/pci/ia_css_morph.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_pipe.h b/drivers/staging/media/atomisp/pci/ia_css_pipe.h index 91653952f1a7..bb0abf9bffb1 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_pipe.h +++ b/drivers/staging/media/atomisp/pci/ia_css_pipe.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -31,7 +32,7 @@ struct ia_css_preview_settings { struct ia_css_binary vf_pp_binary; /* 2401 only for these two - do we in fact use them for anything real */ - struct ia_css_frame *delay_frames[MAX_NUM_DELAY_FRAMES]; + struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES]; struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES]; struct ia_css_pipe *copy_pipe; @@ -39,8 +40,7 @@ struct ia_css_preview_settings { struct ia_css_pipe *acc_pipe; }; -#define IA_CSS_DEFAULT_PREVIEW_SETTINGS \ -(struct ia_css_preview_settings) { \ +#define IA_CSS_DEFAULT_PREVIEW_SETTINGS { \ .copy_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ .preview_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ .vf_pp_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ @@ -64,8 +64,7 @@ struct ia_css_capture_settings { unsigned int num_yuv_scaler; }; -#define IA_CSS_DEFAULT_CAPTURE_SETTINGS \ -(struct ia_css_capture_settings) { \ +#define IA_CSS_DEFAULT_CAPTURE_SETTINGS { \ .copy_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ .primary_binary = {IA_CSS_BINARY_DEFAULT_SETTINGS}, \ .pre_isp_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ @@ -90,8 +89,7 @@ struct ia_css_video_settings { unsigned int num_yuv_scaler; }; -#define IA_CSS_DEFAULT_VIDEO_SETTINGS \ -(struct ia_css_video_settings) { \ +#define IA_CSS_DEFAULT_VIDEO_SETTINGS { \ .copy_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ .video_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ .vf_pp_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ @@ -107,8 +105,7 @@ struct ia_css_yuvpp_settings { unsigned int num_output; }; -#define IA_CSS_DEFAULT_YUVPP_SETTINGS \ -(struct ia_css_yuvpp_settings) { \ +#define IA_CSS_DEFAULT_YUVPP_SETTINGS { \ .copy_binary = IA_CSS_BINARY_DEFAULT_SETTINGS, \ } @@ -146,7 +143,7 @@ struct ia_css_pipe { struct ia_css_capture_settings capture; struct ia_css_yuvpp_settings yuvpp; } pipe_settings; - hrt_vaddress scaler_pp_lut; + ia_css_ptr scaler_pp_lut; struct osys_object *osys_obj; /* This number is unique per pipe each instance of css. This number is @@ -156,8 +153,7 @@ struct ia_css_pipe { unsigned int pipe_num; }; -#define IA_CSS_DEFAULT_PIPE \ -(struct ia_css_pipe) { \ +#define IA_CSS_DEFAULT_PIPE { \ .config = DEFAULT_PIPE_CONFIG, \ .info = DEFAULT_PIPE_INFO, \ .mode = IA_CSS_PIPE_ID_ACC, /* (pipe_id) */ \ @@ -181,7 +177,7 @@ struct ia_css_pipe { void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map); -enum ia_css_err +int sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, struct ia_css_isp_parameters *params, bool commit, struct ia_css_pipe *pipe); diff --git a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h index e782978a5ce7..4affd21f9e3f 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h +++ b/drivers/staging/media/atomisp/pci/ia_css_pipe_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -148,8 +149,7 @@ struct ia_css_pipe_config { /** * Default settings for newly created pipe configurations. */ -#define DEFAULT_PIPE_CONFIG \ -(struct ia_css_pipe_config) { \ +#define DEFAULT_PIPE_CONFIG { \ .mode = IA_CSS_PIPE_MODE_PREVIEW, \ .isp_pipe_version = 1, \ .output_info = {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \ @@ -202,8 +202,7 @@ struct ia_css_pipe_info { /** * Defaults for ia_css_pipe_info structs. */ -#define DEFAULT_PIPE_INFO \ -(struct ia_css_pipe_info) { \ +#define DEFAULT_PIPE_INFO {\ .output_info = {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \ .vf_output_info = {IA_CSS_BINARY_DEFAULT_FRAME_INFO}, \ .raw_output_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO, \ @@ -249,44 +248,44 @@ void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config); /* @brief Create a pipe * @param[in] config The pipe configuration. * @param[out] pipe The pipe. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * This function will create a pipe with the given * configuration. */ -enum ia_css_err +int ia_css_pipe_create(const struct ia_css_pipe_config *config, struct ia_css_pipe **pipe); /* @brief Destroy a pipe * @param[in] pipe The pipe. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * This function will destroy a given pipe. */ -enum ia_css_err +int ia_css_pipe_destroy(struct ia_css_pipe *pipe); /* @brief Provides information about a pipe * @param[in] pipe The pipe. * @param[out] pipe_info The pipe information. - * @return IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS. + * @return 0 or -EINVAL. * * This function will provide information about a given pipe. */ -enum ia_css_err +int ia_css_pipe_get_info(const struct ia_css_pipe *pipe, struct ia_css_pipe_info *pipe_info); /* @brief Configure a pipe with filter coefficients. * @param[in] pipe The pipe. * @param[in] config The pointer to ISP configuration. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * This function configures the filter coefficients for an image * pipe. */ -enum ia_css_err +int ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe, struct ia_css_isp_config *config); @@ -304,7 +303,7 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe, at the same moment in time. There is no control over the order of these events. Once an IRQ has been raised all remembered events are reset. - * @return IA_CSS_SUCCESS. + * @return 0. * Controls when the Event generator in the CSS raises an IRQ to the Host. The main purpose of this function is to reduce the amount of interrupts @@ -362,7 +361,7 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe, All other events (3A, VF output, pipeline done) will not raise an interrupt to the Host. These events are not lost but always stored in the event queue. */ -enum ia_css_err +int ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe, unsigned int or_mask, unsigned int and_mask); @@ -374,7 +373,7 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe, of enum ia_css_event_irq_mask_type. Pointer may be NULL. * @param[out] and_mask Current and_mask.The bits in this mask are a binary or of enum ia_css_event_irq_mask_type. Pointer may be NULL. - * @return IA_CSS_SUCCESS. + * @return 0. * Reads the current event IRQ mask from the CSS. Reading returns the actual values as used by the SP and not any mirrored values stored at the Host.\n @@ -383,7 +382,7 @@ Precondition:\n SP must be running.\n */ -enum ia_css_err +int ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, unsigned int *or_mask, unsigned int *and_mask); @@ -396,7 +395,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, * structure. Only the data pointer within it will * be passed into the internal queues. * @return IA_CSS_INTERNAL_ERROR in case of unexpected errors, - * IA_CSS_SUCCESS otherwise. + * 0 otherwise. * * This function adds a buffer (which has a certain buffer type) to the queue * for this type. This queue is owned by the image pipe. After this function @@ -406,7 +405,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, * host code via an interrupt. Buffers will be consumed in the same order they * get queued, but may be returned to the host out of order. */ -enum ia_css_err +int ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, const struct ia_css_buffer *buffer); @@ -418,7 +417,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, * The resulting buffer pointer is written into the dta * field. * @return IA_CSS_ERR_NO_BUFFER if the queue is empty or - * IA_CSS_SUCCESS otherwise. + * 0 otherwise. * * This function dequeues a buffer from a buffer queue. The queue is indicated * by the buffer type argument. This function can be called after an interrupt @@ -426,7 +425,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, * be used in a polling-like situation where the NO_BUFFER return value is used * to determine whether a buffer was available or not. */ -enum ia_css_err +int ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, struct ia_css_buffer *buffer); @@ -437,9 +436,9 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, * @param[in] enable Enable Flag (1 to enable ; 0 to disable) * * @return - * IA_CSS_SUCCESS : Success - * IA_CSS_ERR_INVALID_ARGUMENTS : Invalid Parameters - * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE : Inactive QOS Pipe + * 0 : Success + * -EINVAL : Invalid Parameters + * -EBUSY : Inactive QOS Pipe * (No active stream with this pipe) * * This function will request state change (enable or disable) for the Extension @@ -452,7 +451,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, * 4. State change cannot be guaranteed immediately OR on frame boundary * */ -enum ia_css_err +int ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, u32 fw_handle, bool enable); @@ -464,9 +463,9 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, * @param[out] *enable Enable Flag * * @return - * IA_CSS_SUCCESS : Success - * IA_CSS_ERR_INVALID_ARGUMENTS : Invalid Parameters - * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE : Inactive QOS Pipe + * 0 : Success + * -EINVAL : Invalid Parameters + * -EBUSY : Inactive QOS Pipe * (No active stream with this pipe) * * This function will query the state of the Extension stage (firmware handle) @@ -478,7 +477,7 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, * 3. Initial(Default) state of QOS Extensions is Disabled. * */ -enum ia_css_err +int ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, u32 fw_handle, bool *enable); @@ -491,18 +490,18 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, * @param[in] isp_seg Parameter memory descriptors for ISP segments. * * @return - * IA_CSS_SUCCESS : Success - * IA_CSS_ERR_INVALID_ARGUMENTS : Invalid Parameters - * IA_CSS_ERR_RESOURCE_NOT_AVAILABLE : Inactive QOS Pipe + * 0 : Success + * -EINVAL : Invalid Parameters + * -EBUSY : Inactive QOS Pipe * (No active stream with this pipe) * * \deprecated{This interface is used to temporarily support a late-developed, * specific use-case on a specific IPU2 platform. It will not be supported or * maintained on IPU3 or further.} */ -enum ia_css_err +int ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, - uint32_t fw_handle, + u32 fw_handle, struct ia_css_isp_param_css_segments *css_seg, struct ia_css_isp_param_isp_segments *isp_seg); @@ -521,8 +520,8 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe, * @param[in] lut Look up tabel * * @return - * IA_CSS_SUCCESS : Success - * IA_CSS_ERR_INVALID_ARGUMENTS : Invalid Parameters + * 0 : Success + * -EINVAL : Invalid Parameters * * Note: * 1) Note that both GDC's are programmed with the same table. @@ -531,7 +530,7 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe, * 3) This function must be called before stream start * */ -enum ia_css_err +int ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, const void *lut); /* @brief Checking of DVS statistics ability @@ -550,9 +549,9 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info); * @param[in] format Format to set * * @return - * IA_CSS_SUCCESS : Success - * IA_CSS_ERR_INVALID_ARGUMENTS : Invalid Parameters - * IA_CSS_ERR_INTERNAL_ERROR : Pipe misses binary info + * 0 : Success + * -EINVAL : Invalid Parameters + * -EINVAL : Pipe misses binary info * * Note: * 1) This is an optional function to override the formats set in the pipe. @@ -561,7 +560,7 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info); * 4) If this function is used, it MUST be called after ia_css_pipe_create. * 5) If this function is used, this function MUST be called before ia_css_stream_start. */ -enum ia_css_err +int ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe, int output_pin, enum ia_css_frame_format format); diff --git a/drivers/staging/media/atomisp/pci/ia_css_prbs.h b/drivers/staging/media/atomisp/pci/ia_css_prbs.h index 037fc4f77c77..53bbf1dce3bf 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_prbs.h +++ b/drivers/staging/media/atomisp/pci/ia_css_prbs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_properties.h b/drivers/staging/media/atomisp/pci/ia_css_properties.h index 9a167306611c..2cd014f7ae29 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_properties.h +++ b/drivers/staging/media/atomisp/pci/ia_css_properties.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_shading.h b/drivers/staging/media/atomisp/pci/ia_css_shading.h index 588f53d32b72..de7ae5cabf7d 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_shading.h +++ b/drivers/staging/media/atomisp/pci/ia_css_shading.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream.h b/drivers/staging/media/atomisp/pci/ia_css_stream.h index 5690fe832f41..e3e7a8a03b04 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_stream.h +++ b/drivers/staging/media/atomisp/pci/ia_css_stream.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -102,7 +103,7 @@ ia_css_get_isp_dvs2_coefficients(struct ia_css_stream *stream, short *ver_coefs_even_real, short *ver_coefs_even_imag); -enum ia_css_err +int ia_css_stream_isp_parameters_init(struct ia_css_stream *stream); void diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream_format.h b/drivers/staging/media/atomisp/pci/ia_css_stream_format.h index 4cd29833584f..aac22d8581a0 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_stream_format.h +++ b/drivers/staging/media/atomisp/pci/ia_css_stream_format.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_stream_public.h b/drivers/staging/media/atomisp/pci/ia_css_stream_public.h index fe11c8bf3cdc..83846e417ae5 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_stream_public.h +++ b/drivers/staging/media/atomisp/pci/ia_css_stream_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -171,11 +172,11 @@ void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config); * @param[in] num_pipes The number of pipes to incorporate in the stream. * @param[in] pipes The pipes. * @param[out] stream The stream. -* @return IA_CSS_SUCCESS or the error code. +* @return 0 or the error code. * * This function will create a stream with a given configuration and given pipes. */ -enum ia_css_err +int ia_css_stream_create(const struct ia_css_stream_config *stream_config, int num_pipes, struct ia_css_pipe *pipes[], @@ -183,37 +184,37 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* @brief Destroys a stream * @param[in] stream The stream. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * This function will destroy a given stream. */ -enum ia_css_err +int ia_css_stream_destroy(struct ia_css_stream *stream); /* @brief Provides information about a stream * @param[in] stream The stream. * @param[out] stream_info The information about the stream. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * This function will destroy a given stream. */ -enum ia_css_err +int ia_css_stream_get_info(const struct ia_css_stream *stream, struct ia_css_stream_info *stream_info); /* @brief load (rebuild) a stream that was unloaded. * @param[in] stream The stream - * @return IA_CSS_SUCCESS or the error code + * @return 0 or the error code * * Rebuild a stream, including allocating structs, setting configuration and * building the required pipes. */ -enum ia_css_err +int ia_css_stream_load(struct ia_css_stream *stream); /* @brief Starts the stream. * @param[in] stream The stream. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * The dynamic data in * the buffers are not used and need to be queued with a separate call @@ -221,17 +222,17 @@ ia_css_stream_load(struct ia_css_stream *stream); * NOTE: this function will only send start event to corresponding * thread and will not start SP any more. */ -enum ia_css_err +int ia_css_stream_start(struct ia_css_stream *stream); /* @brief Stop the stream. * @param[in] stream The stream. - * @return IA_CSS_SUCCESS or the error code. + * @return 0 or the error code. * * NOTE: this function will send stop event to pipes belong to this * stream but will not terminate threads. */ -enum ia_css_err +int ia_css_stream_stop(struct ia_css_stream *stream); /* @brief Check if a stream has stopped @@ -245,11 +246,11 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream); /* @brief destroy a stream according to the stream seed previosly saved in the seed array. * @param[in] stream The stream. - * @return IA_CSS_SUCCESS (no other errors are generated now) + * @return 0 (no other errors are generated now) * * Destroy the stream and all the pipes related to it. */ -enum ia_css_err +int ia_css_stream_unload(struct ia_css_stream *stream); /* @brief Returns stream format @@ -278,19 +279,19 @@ ia_css_stream_get_two_pixels_per_clock(const struct ia_css_stream *stream); * * This function will Set the output frame stride (at the last pipe) */ -enum ia_css_err +int ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, unsigned int output_padded_width); /* @brief Return max number of continuous RAW frames. * @param[in] stream The stream. * @param[out] buffer_depth The maximum number of continuous RAW frames. - * @return IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS + * @return 0 or -EINVAL * * This function will return the maximum number of continuous RAW frames * the system can support. */ -enum ia_css_err +int ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, int *buffer_depth); @@ -298,22 +299,22 @@ ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, * * @param[in] stream The stream. * @param[in] buffer_depth Number of frames to set. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * Set the number of continuous frames to use during continuous modes. */ -enum ia_css_err +int ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth); /* @brief Get number of continuous RAW frames to use. * @param[in] stream The stream. * @param[out] buffer_depth The number of frames to use - * @return IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS + * @return 0 or -EINVAL * * Get the currently set number of continuous frames * to use during continuous modes. */ -enum ia_css_err +int ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth); /* ===== CAPTURE ===== */ @@ -338,13 +339,13 @@ ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth); * with this offset. This allows the user to * process RAW frames that were captured in the * past or future. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * For example, to capture the current frame plus the 2 previous * frames and 2 subsequent frames, you would call * ia_css_stream_capture(5, 0, -2). */ -enum ia_css_err +int ia_css_stream_capture(struct ia_css_stream *stream, int num_captures, unsigned int skip, @@ -355,12 +356,12 @@ ia_css_stream_capture(struct ia_css_stream *stream, * @param[in] stream The stream. * @param[in] exp_id The exposure id of the raw frame to tag. * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * This function allows the user to tag a raw frame based on the exposure id * found in the viewfinder frames' frame info. */ -enum ia_css_err +int ia_css_stream_capture_frame(struct ia_css_stream *stream, unsigned int exp_id); @@ -492,7 +493,7 @@ ia_css_stream_request_flash(struct ia_css_stream *stream); * @param[in] config The set of filter coefficients. * @param[in] pipe Pipe to be updated when set isp config, NULL means to * update all pipes in the stream. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * This function configures the filter coefficients for an image * stream. For image pipes that do not execute any ISP filters, this @@ -501,7 +502,7 @@ ia_css_stream_request_flash(struct ia_css_stream *stream); * in fact this is the expected behavior most of the time. Proper * resource locking and double buffering is in place to allow for this. */ -enum ia_css_err +int ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe); @@ -511,7 +512,7 @@ ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream, * ia_css_pipe_set_isp_config()} * @param[in] stream The stream. * @param[in] config The set of filter coefficients. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * This function configures the filter coefficients for an image * stream. For image pipes that do not execute any ISP filters, this @@ -521,7 +522,7 @@ ia_css_stream_set_isp_config_on_pipe(struct ia_css_stream *stream, * in fact this is the expected behaviour most of the time. Proper * resource locking and double buffering is in place to allow for this. */ -enum ia_css_err +int ia_css_stream_set_isp_config( struct ia_css_stream *stream, const struct ia_css_isp_config *config); @@ -537,37 +538,37 @@ ia_css_stream_get_isp_config(const struct ia_css_stream *stream, /* @brief allocate continuous raw frames for continuous capture * @param[in] stream The stream. - * @return IA_CSS_SUCCESS or error code. + * @return 0 or error code. * * because this allocation takes a long time (around 120ms per frame), * we separate the allocation part and update part to let driver call * this function without locking. This function is the allocation part * and next one is update part */ -enum ia_css_err +int ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream); /* @brief allocate continuous raw frames for continuous capture * @param[in] stream The stream. - * @return IA_CSS_SUCCESS or error code. + * @return 0 or error code. * * because this allocation takes a long time (around 120ms per frame), * we separate the allocation part and update part to let driver call * this function without locking. This function is the update part */ -enum ia_css_err +int ia_css_update_continuous_frames(struct ia_css_stream *stream); /* @brief ia_css_unlock_raw_frame . unlock a raw frame (HALv3 Support) * @param[in] stream The stream. * @param[in] exp_id exposure id that uniquely identifies the locked Raw Frame Buffer - * @return ia_css_err IA_CSS_SUCCESS or error code + * @return ia_css_err 0 or error code * * As part of HALv3 Feature requirement, SP locks raw buffer until the Application * releases its reference to a raw buffer (which are managed by SP), this function allows * application to explicitly unlock that buffer in SP. */ -enum ia_css_err +int ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id); /* @brief ia_css_en_dz_capt_pipe . Enable/Disable digital zoom for capture pipe diff --git a/drivers/staging/media/atomisp/pci/ia_css_timer.h b/drivers/staging/media/atomisp/pci/ia_css_timer.h index a37cfa60ad35..c78fbda907a1 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_timer.h +++ b/drivers/staging/media/atomisp/pci/ia_css_timer.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. @@ -58,10 +59,10 @@ struct ia_css_time_meas { /* @brief API to fetch timer count directly * * @param curr_ts [out] measured count value -* @return IA_CSS_SUCCESS if success +* @return 0 if success * */ -enum ia_css_err +int ia_css_timer_get_current_tick( struct ia_css_clock_tick *curr_ts); diff --git a/drivers/staging/media/atomisp/pci/ia_css_tpg.h b/drivers/staging/media/atomisp/pci/ia_css_tpg.h index 79c4e1b3b48f..8c744bedb0a6 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_tpg.h +++ b/drivers/staging/media/atomisp/pci/ia_css_tpg.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/ia_css_types.h b/drivers/staging/media/atomisp/pci/ia_css_types.h index d3584756e34e..6e34d401f9df 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_types.h +++ b/drivers/staging/media/atomisp/pci/ia_css_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* * Support for Intel Camera Imaging ISP subsystem. @@ -306,7 +307,6 @@ struct ia_css_shading_info { u32 bayer_scale_ver_ratio_in; u32 bayer_scale_ver_ratio_out; - /** Vertical ratio of bayer scaling between input height and output height, for the scaling which should be done before shading correction. @@ -392,8 +392,7 @@ struct ia_css_grid_info { }; /* defaults for ia_css_grid_info structs */ -#define DEFAULT_GRID_INFO \ -(struct ia_css_grid_info) { \ +#define DEFAULT_GRID_INFO { \ .dvs_grid = DEFAULT_DVS_GRID_INFO, \ .vamem_type = IA_CSS_VAMEM_TYPE_1 \ } @@ -485,8 +484,7 @@ struct ia_css_capture_config { }; /* default settings for ia_css_capture_config structs */ -#define DEFAULT_CAPTURE_CONFIG \ -(struct ia_css_capture_config) { \ +#define DEFAULT_CAPTURE_CONFIG { \ .mode = IA_CSS_CAPTURE_MODE_PRIMARY, \ } diff --git a/drivers/staging/media/atomisp/pci/ia_css_version.h b/drivers/staging/media/atomisp/pci/ia_css_version.h index 1e88901e0b82..cf1d010baceb 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_version.h +++ b/drivers/staging/media/atomisp/pci/ia_css_version.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -34,7 +35,7 @@ * This function generates and returns the version string. If FW is loaded, it * attaches the FW version. */ -enum ia_css_err +int ia_css_get_version(char *version, int max_size); #endif /* __IA_CSS_VERSION_H */ diff --git a/drivers/staging/media/atomisp/pci/ia_css_version_data.h b/drivers/staging/media/atomisp/pci/ia_css_version_data.h index f630fa5d55cc..428d78e1616f 100644 --- a/drivers/staging/media/atomisp/pci/ia_css_version_data.h +++ b/drivers/staging/media/atomisp/pci/ia_css_version_data.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/if_defs.h b/drivers/staging/media/atomisp/pci/if_defs.h index 7d39e45796ae..e21efa749368 100644 --- a/drivers/staging/media/atomisp/pci/if_defs.h +++ b/drivers/staging/media/atomisp/pci/if_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h b/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h index 176456da961f..594fc36a01c7 100644 --- a/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h +++ b/drivers/staging/media/atomisp/pci/input_formatter_subsystem_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_selector_defs.h b/drivers/staging/media/atomisp/pci/input_selector_defs.h index 1dd8ea3cd6d4..61882e4cb813 100644 --- a/drivers/staging/media/atomisp/pci/input_selector_defs.h +++ b/drivers/staging/media/atomisp/pci/input_selector_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h b/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h index 2d5baae30522..8ea1d7991d38 100644 --- a/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h +++ b/drivers/staging/media/atomisp/pci/input_switch_2400_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h b/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h index fcfa8c4971be..c801ddd71192 100644 --- a/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h +++ b/drivers/staging/media/atomisp/pci/input_system_ctrl_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_system_defs.h b/drivers/staging/media/atomisp/pci/input_system_defs.h index ae62163034a6..0c6a74b1891f 100644 --- a/drivers/staging/media/atomisp/pci/input_system_defs.h +++ b/drivers/staging/media/atomisp/pci/input_system_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/input_system_global.h b/drivers/staging/media/atomisp/pci/input_system_global.h index e75c2f29042d..5ac580ce64ed 100644 --- a/drivers/staging/media/atomisp/pci/input_system_global.h +++ b/drivers/staging/media/atomisp/pci/input_system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/input_system_local.h b/drivers/staging/media/atomisp/pci/input_system_local.h index 8533a1e017e4..b33aa2838290 100644 --- a/drivers/staging/media/atomisp/pci/input_system_local.h +++ b/drivers/staging/media/atomisp/pci/input_system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/input_system_private.h b/drivers/staging/media/atomisp/pci/input_system_private.h index 69c63a79a30c..889f204e77d5 100644 --- a/drivers/staging/media/atomisp/pci/input_system_private.h +++ b/drivers/staging/media/atomisp/pci/input_system_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/input_system_public.h b/drivers/staging/media/atomisp/pci/input_system_public.h index 17682c86bceb..3f5167fd6643 100644 --- a/drivers/staging/media/atomisp/pci/input_system_public.h +++ b/drivers/staging/media/atomisp/pci/input_system_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/irq_controller_defs.h b/drivers/staging/media/atomisp/pci/irq_controller_defs.h index efb3d7e135bd..e49e61e17ee7 100644 --- a/drivers/staging/media/atomisp/pci/irq_controller_defs.h +++ b/drivers/staging/media/atomisp/pci/irq_controller_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h b/drivers/staging/media/atomisp/pci/irq_types_hrt.h index 5c636effba48..4212bb01c8d8 100644 --- a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/hive_isp_css_irq_types_hrt.h +++ b/drivers/staging/media/atomisp/pci/irq_types_hrt.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c index 9cdfe50b2835..daf2f25c1ed6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h index 71587d85ff2d..3abc125debd0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h index 3c699bae2f55..4f8bb4de4edc 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h index cc6a444ac716..900ba8f5e30c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/aa/aa_2/ia_css_aa2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c index c190483dc2b3..3f079c954c1f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h index 3855f54765e3..a4720c4a948a 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h index 6bf834cb47d9..37dcb013b76d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h index d3fa0193ae07..be3534e46c15 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_1.0/ia_css_anr_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c index feee073b5099..9cdefedc6312 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h index e99108682f5d..2b3ab01c279d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h index 47a0fb08cfcc..4b83b8100160 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -21,7 +22,7 @@ /* Advanced Noise Reduction (ANR) thresholds */ struct ia_css_isp_anr2_params { - VMEM_ARRAY(data, ANR_PARAM_SIZE *ISP_VEC_NELEMS); + VMEM_ARRAY(data, ANR_PARAM_SIZE * ISP_VEC_NELEMS); }; #endif /* __IA_CSS_ANR2_PARAM_H */ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c index 070e90e3e2b5..649283bd44f2 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h index 534119e064c1..9e383e030ac4 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h index 200df3829fc7..e12aae819dce 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/anr/anr_2/ia_css_anr2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c index 6c7aa51ec079..82aa69b74677 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -14,7 +15,6 @@ #if !defined(HAS_NO_HMEM) -#include "memory_access.h" #include "ia_css_types.h" #include "sh_css_internal.h" #include "assert_support.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h index ccd83169fe22..736b6e3f9512 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h index 692a855ba012..05d5c43e6b16 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h index 8b2a53a26b75..4c0e92f13d6c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bh/bh_2/ia_css_bh_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c index 6888a7363710..45e37dc4f1e3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -42,7 +43,7 @@ static const s32 div_lut_intercepts[BNLM_DIV_LUT_SIZE] = { */ static inline void bnlm_lut_encode(struct bnlm_lut *lut, const int32_t *lut_thr, - const int32_t *lut_val, const uint32_t lut_size) + const s32 *lut_val, const uint32_t lut_size) { u32 blk, i; const u32 block_size = 16; diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h index a57933bfb974..3632bf27cc21 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h index c7d5cadf5fd4..30672db269df 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h index 8dd1b1766c64..407b5a3b0fcd 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnlm/ia_css_bnlm_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c index a5e20596539d..c42fcb1d9100 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h index a021733dcdf7..f6ab5d2bb218 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h index 698fdc0b13fa..087723795476 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h index ee9569891747..5f3dfa59f950 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr2_2/ia_css_bnr2_2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c index 5efb0ce7f323..457a004e194d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h index 4c29b47b8177..7fc2a728a6c2 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h index 52f21ce8f4d2..4f6469315386 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/bnr/bnr_1.0/ia_css_bnr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c index c50afa6bf8a6..0eb40517e08c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h index 87250ca5842c..4d046b730f06 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h index c1af207cbf9a..971ab87af2c5 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_1.0/ia_css_cnr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c index 610871d213bb..495dc1f33ca6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h index d322359feedf..38f848137eda 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h index 0d2fb2897720..3709aa4d3652 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h index 35fc2e77eb3d..d0a25616727c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/cnr/cnr_2/ia_css_cnr2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c index e64e26089a4d..ff452e2cc23a 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h index c136d5e03511..520623e27349 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h index 3a6ede394bdc..fcbec189eff8 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h index 79a626fe3a29..34152d6d09be 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/conversion/conversion_1.0/ia_css_conversion_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c index 6e29b7eeb3ed..5d34f3256a43 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h index 6f42abdec9bb..615cb6771884 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h index 587d0c62c936..56daa1d96747 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/copy_output/copy_output_1.0/ia_css_copy_output_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c index c6a3bd4fbf80..38912062edd4 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h index 2e451a872d2a..21a259d33256 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h index 35835929d252..7416e74dd782 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h index 5c166be6c5e8..aaaae5e2abb9 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/crop/crop_1.0/ia_css_crop_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c index ea81e1d3e445..284c17970e55 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h index 347ccd864577..6b0256a73e52 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h index 53e270df2db7..3809ef73e490 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h index d49203d322bd..160f19bdfca0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/csc/csc_1.0/ia_css_csc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c index e80f42ab0e6a..149adbc57730 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h index f3c40a49f7c0..8c17e7b921b5 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h index 95cf34ef4ed2..c18cfc930db6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc1_5/ia_css_ctc1_5_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c index b247dc6bec6a..e3d3f1253422 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h index 3733aee24dcd..eb10c3884020 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h index 224bdb199942..94844da665e5 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h index 9d5dadf70f1a..f9f329a58737 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc2/ia_css_ctc2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c index 26311b0a23f9..82f2adbbfac3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h index e4ad676361dd..57d1d08e1bc8 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h index 6e541a0ebaa9..7e2fa192a0fe 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c index adb146c03a73..f13b79586963 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ + #include <type_support.h> -#include <string_support.h> /* memcpy */ #include "system_global.h" #include "vamem.h" #include "ia_css_types.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h index a350dec8b4ad..33e8a05455a3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h index f6f5ec28827f..b2d42f3c1f4d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ctc/ctc_1.0/ia_css_ctc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c index 8e4218cb70cd..25e3f0822fb8 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h index baae1d9809da..cb91062029cf 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h index c85a57e194cc..2070ce040470 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h index a4b446904570..daac1275c129 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_1.0/ia_css_de_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c index ade23d53f6bb..f90da39296ec 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h index f3749e514505..294f619a3b15 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h index 868dfaaf78c7..4c9d5c630cb0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h index 24700d256bfd..372cd9d2b803 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/de/de_2/ia_css_de2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c index 9fb37447831c..eff428c67c86 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h index 009541fafda0..e5f5a2716010 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h index 8567a620696a..e0e7f2d48237 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h index e96f83e5d47c..c1666ebf1d3b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dp/dp_1.0/ia_css_dp_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c index 4dfad4ace20b..f6fe064bdda4 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h index a31ef0e5047b..f6e019a65208 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h index 6df06fb249aa..1ccceadbb7bf 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h index f78451be8d6a..f742a8dc1d67 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dpc2/ia_css_dpc2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c index d2c3e8edf626..b8b71791466f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,6 +13,8 @@ * more details. */ +#include "hmm.h" + #include "ia_css_frame_public.h" #define IA_CSS_INCLUDE_CONFIGURATIONS #include "ia_css_isp_configs.h" @@ -22,7 +25,6 @@ #include "sh_css_params.h" #include "ia_css_binary.h" #include "ia_css_debug.h" -#include "memory_access.h" #include "assert_support.h" #include "ia_css_dvs.host.h" @@ -270,12 +272,12 @@ convert_allocate_dvs_6axis_config( return me; } -enum ia_css_err +int store_dvs_6axis_config( const struct ia_css_dvs_6axis_config *dvs_6axis_config, const struct ia_css_binary *binary, const struct ia_css_frame_info *dvs_in_frame_info, - hrt_vaddress ddr_addr_y) { + ia_css_ptr ddr_addr_y) { struct ia_css_host_data *me; assert(dvs_6axis_config); @@ -288,8 +290,8 @@ store_dvs_6axis_config( if (!me) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } ia_css_params_store_ia_css_host_data( @@ -297,5 +299,5 @@ store_dvs_6axis_config( me); ia_css_host_data_free(me); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h index d711170cf7cc..f9bc17ee0f86 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -50,11 +51,11 @@ convert_allocate_dvs_6axis_config( const struct ia_css_binary *binary, const struct ia_css_frame_info *dvs_in_frame_info); -enum ia_css_err +int store_dvs_6axis_config( const struct ia_css_dvs_6axis_config *dvs_6axis_config, const struct ia_css_binary *binary, const struct ia_css_frame_info *dvs_in_frame_info, - hrt_vaddress ddr_addr_y); + ia_css_ptr ddr_addr_y); #endif /* __IA_CSS_DVS_HOST_H */ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h index f8842dae943b..2963bb10b129 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h index a1a14d93ef29..e99ff0ce8bab 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/dvs/dvs_1.0/ia_css_dvs_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c index 03e1998b0464..bfea78171f7c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h index 05f817125d3c..f1ad07e78b3a 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h index 880454d4dcf5..6fb3b38f49e7 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h index b8fdb7a51a12..836e348c184f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/eed1_8/ia_css_eed1_8_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c index 0b96b9618ab6..bae1ca2cd505 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h index 0aac424d9d54..540423d85e9d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h index 8f36af1a5ae6..5275a1dadefa 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h index 7cfebaf05dc2..16b6a3ddcd08 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fc/fc_1.0/ia_css_formats_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h index adfd4b37171c..6cd635f3ee27 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h index 6485834704da..3a55d4c698e6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fixedbds/fixedbds_1.0/ia_css_fixedbds_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c index 7b55dfea359a..47b5c7956fbd 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h index 02e85570bd1c..12187d213d90 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h index f103ddd882fd..b21415743705 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h index 95552a0e3c45..14dc5e183184 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/fpn/fpn_1.0/ia_css_fpn_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c index 1a489c93eb97..7f3f87920dc7 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h index 2fb2927b07f1..c2dc1574a3da 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h index beeba6c9be6a..a81233add437 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c index 15cf0575aac5..f48f876777dc 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ + #include <type_support.h> -#include <string_support.h> /* memcpy */ #include "system_global.h" #include "vamem.h" #include "ia_css_types.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h index 9686623d9cdd..ee6fa07b3511 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h index c896c138b569..ccd3d91a24d3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_1.0/ia_css_gc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c index 29a1e013a9aa..76209b7c14cb 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h index ca7d54576471..eabf78737bfc 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h index 458c72a45eef..af456e75e476 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c index d2fe0052fb00..7eadb31268eb 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ + #include <type_support.h> -#include <string_support.h> /* memcpy */ #include "system_global.h" #include "vamem.h" #include "ia_css_types.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h index 8686e6e3586c..13049fbfab84 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h index 30780394ed7f..ae16409d84a5 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/gc/gc_2/ia_css_gc2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c index 643b7d9095e6..698550cc2fcc 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* Release Version: irci_ecr-master_20150911_0724 */ /* diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h index ecc8bea3542b..04599ab590cd 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* Release Version: irci_ecr-master_20150911_0724 */ /* diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h index 47651cdf94b7..97a89fd3cfda 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* Release Version: irci_ecr-master_20150911_0724 */ /* diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h index 7c2f8f213bef..1b4090880201 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/hdr/ia_css_hdr_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* Release Version: irci_stable_candrpv_0415_20150521_0458 */ /* Release Version: irci_ecr-master_20150911_0724 */ /* diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c index bf71a7f661e6..ea8055148fb3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h index f9db75a089af..635ccb1b27d0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h index 77cfed002e14..283ace8385b0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h index 59b58f30af11..d06d25c9b81c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/bayer_io_ls/ia_css_bayer_io_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h index 22aedcc4470f..5e0e4cd5bfba 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h index e49bd95f77da..0801481c4b49 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/common/ia_css_common_io_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c index ba490c5fc18e..f8bd207b28e1 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h index 556e53e05607..e7cfd380e108 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h index 1cc2aff57ef3..9b7537d508ad 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h index 990299a0d2c7..137a2a05c65b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ipu2_io_ls/yuv444_io_ls/ia_css_yuv444_io_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c index 49c1b3e3370d..6d8a35a73750 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -37,7 +38,7 @@ ia_css_iterator_config( ia_css_resolution_to_sp_resolution(&to->dvs_envelope, from->dvs_envelope); } -enum ia_css_err +int ia_css_iterator_configure( const struct ia_css_binary *binary, const struct ia_css_frame_info *in_info) { @@ -76,5 +77,5 @@ ia_css_iterator_configure( ia_css_configure_iterator(binary, &config); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h index c5e8d58e0fe1..1419fa9a07f0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -26,7 +27,7 @@ ia_css_iterator_config( const struct ia_css_iterator_configuration *from, unsigned int size); -enum ia_css_err +int ia_css_iterator_configure( const struct ia_css_binary *binary, const struct ia_css_frame_info *in_info); diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h index d308126e41d3..e062f8d06128 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/iterator/iterator_1.0/ia_css_iterator_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c index 7a6abe0c5b7d..562662ab8a44 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h index ae9ede2b685a..6c1189e1d263 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h index 497ad89ab728..66b8fb259218 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -25,7 +26,7 @@ struct sh_css_isp_macc1_5_params { }; struct sh_css_isp_macc1_5_vmem_params { - VMEM_ARRAY(data, IA_CSS_MACC_NUM_COEFS *ISP_NWAY); + VMEM_ARRAY(data, IA_CSS_MACC_NUM_COEFS * ISP_NWAY); }; #endif /* __IA_CSS_MACC1_5_PARAM_H */ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c index c094f3df10aa..d205d64e0b94 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h index 10a50aa82be8..d451efbaa184 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h index 9aa352cbcffc..5492af0dfa9f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc1_5/ia_css_macc1_5_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c index 0b1d1bf5e8a0..f2d3832a0039 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h index 0e13e9cb0547..912db92540e6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h index 3b4e440c3c30..71665204e4dd 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c index f9a430da54b8..946b074e8288 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h index 96d62c9912b8..35099cb79d6a 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h index 093302f08bca..172a518cb935 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/macc/macc_1.0/ia_css_macc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c index 102dc6feb6d1..69283b631da7 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h index 42b5143ef78f..3987abcae85c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h index d432e2e39df6..06c39fdfc9e3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/norm/norm_1.0/ia_css_norm_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c index f7403ce16c99..c4ffff630b64 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h index 936f6a08a174..26c2e43202dc 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h index c728f8791ef4..c3c9fc3f9064 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h index 0ccc09f6eb0f..51e4c35cf659 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob2/ia_css_ob2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c index 6367d94275fb..12191cd36d5c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h index d767c5856880..dfcac0c64040 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h index f5c3e14a1a8a..991aa3c4051b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h index 317b24e240d8..b74296517b03 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ob/ob_1.0/ia_css_ob_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c index df4cb9c362a4..c8e074f42353 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h index 3d8f61c225cf..1f5a2242640e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h index 3a63eee58cb6..df125674bb35 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h index 3248bc3fd6c3..e5f9c05d2120 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/output/output_1.0/ia_css_output_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c index 3de108b56005..1603fd44ece3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h index ad6d7ca783e4..8d940959f40a 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h index 87898d2df2de..9ae290450865 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h index b7ecd8f40c1c..549f1a36bb7e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/qplane/qplane_2/ia_css_qplane_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c index 1a85f20770c1..1c6f6792d57b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h index 36a4079aa24a..346928435a8b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h index a1a314272a77..c4b5f719a336 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h index 7838f59a2986..1ccaa3c48407 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw/raw_1.0/ia_css_raw_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c index 2045b974ec8a..29c707ecf9f3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -14,7 +15,6 @@ #if !defined(HAS_NO_HMEM) -#include "memory_access.h" #include "ia_css_types.h" #include "sh_css_internal.h" #include "sh_css_frac.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h index d4df1dc540a0..4c2b3de7281e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/raw_aa_binning/raw_aa_binning_1.0/ia_css_raa.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c index c3f43fd327d4..061558fbe329 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -29,10 +30,15 @@ ia_css_ref_config( { unsigned int elems_a = ISP_VEC_NELEMS, i; - (void)size; - ia_css_dma_configure_from_info(&to->port_b, &from->ref_frames[0]->info); - to->width_a_over_b = elems_a / to->port_b.elems; - to->dvs_frame_delay = from->dvs_frame_delay; + if (from->ref_frames[0]) { + ia_css_dma_configure_from_info(&to->port_b, &from->ref_frames[0]->info); + to->width_a_over_b = elems_a / to->port_b.elems; + to->dvs_frame_delay = from->dvs_frame_delay; + } else { + to->width_a_over_b = 1; + to->dvs_frame_delay = 0; + to->port_b.elems = elems_a; + } for (i = 0; i < MAX_NUM_VIDEO_DELAY_FRAMES; i++) { if (from->ref_frames[i]) { to->ref_frame_addr_y[i] = from->ref_frames[i]->data + @@ -52,7 +58,7 @@ ia_css_ref_config( void ia_css_ref_configure( const struct ia_css_binary *binary, - const struct ia_css_frame **ref_frames, + const struct ia_css_frame * const *ref_frames, const uint32_t dvs_frame_delay) { struct ia_css_ref_configuration config; diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h index 4f48a8cfc604..3ce590b436a1 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -31,7 +32,7 @@ ia_css_ref_config( void ia_css_ref_configure( const struct ia_css_binary *binary, - const struct ia_css_frame **ref_frames, + const struct ia_css_frame * const *ref_frames, const uint32_t dvs_frame_delay); void diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h index 0a0498c17fba..c727e27a8e14 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -28,8 +29,8 @@ struct ia_css_ref_configuration { struct sh_css_isp_ref_isp_config { u32 width_a_over_b; struct dma_port_config port_b; - hrt_vaddress ref_frame_addr_y[MAX_NUM_VIDEO_DELAY_FRAMES]; - hrt_vaddress ref_frame_addr_c[MAX_NUM_VIDEO_DELAY_FRAMES]; + ia_css_ptr ref_frame_addr_y[MAX_NUM_VIDEO_DELAY_FRAMES]; + ia_css_ptr ref_frame_addr_c[MAX_NUM_VIDEO_DELAY_FRAMES]; u32 dvs_frame_delay; }; diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h index 1d30ccc2c638..d4f7a66763eb 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_state.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h index 156d6cd8cf3a..07d040bcf281 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ref/ref_1.0/ia_css_ref_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c index d093565d9eb8..ba52c80df4a5 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h index 13d19dab1f1d..f9926e297568 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h index 041101767ff2..9cb75b220678 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h index 5a5b277ca0eb..f369e9b95ca8 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/s3a/s3a_1.0/ia_css_s3a_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c index 43954ed6d106..f3fb4b9b3c82 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h index efbe40b399dd..f1eb568f23d4 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h index 38a625821987..fab11d3350fd 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h index 41f3ee7158ff..aae534521b7b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sc/sc_1.0/ia_css_sc_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h index c03936fb0550..7b661e49b4d0 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -76,12 +77,12 @@ struct sh_css_isp_sdis_vert_proj_tbl { struct sh_css_isp_sdis_hori_coef_tbl { VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES], - ISP_MAX_SDIS_HOR_COEF_NUM_VECS *ISP_NWAY); + ISP_MAX_SDIS_HOR_COEF_NUM_VECS * ISP_NWAY); }; struct sh_css_isp_sdis_vert_coef_tbl { VMEM_ARRAY(tbl[ISP_DVS_NUM_COEF_TYPES], - ISP_MAX_SDIS_VER_COEF_NUM_VECS *ISP_NWAY); + ISP_MAX_SDIS_VER_COEF_NUM_VECS * ISP_NWAY); }; #endif /* defined(__ISP) || defined (MK_FIRMWARE) */ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h index e257841bba67..c2ec30b4abd4 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/common/ia_css_sdis_common_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -196,12 +197,11 @@ struct ia_css_dvs_stat_grid_info { /* DVS statistics generated by accelerator default grid info */ -#define DEFAULT_DVS_GRID_INFO \ -(union ia_css_dvs_grid_u) { \ - .dvs_stat_grid_info = (struct ia_css_dvs_stat_grid_info) { \ +#define DEFAULT_DVS_GRID_INFO { \ + .dvs_stat_grid_info = { \ .fe_roi_cfg = { \ - [1] = (struct dvs_stat_public_dvs_level_fe_roi_cfg) { \ - .x_start = 4 \ + [1] = { \ + .x_start = 4 \ } \ } \ } \ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c index 418481e016f7..3e72dabe7ef6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,7 +13,8 @@ * more details. */ -#include "memory_access.h" +#include "hmm.h" + #include "assert_support.h" #include "ia_css_debug.h" #include "ia_css_sdis_types.h" @@ -234,12 +236,12 @@ void ia_css_sdis_clear_coefficients( dvs_coefs->ver_coefs = NULL; } -enum ia_css_err +int ia_css_get_dvs_statistics( struct ia_css_dvs_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats) { struct ia_css_isp_dvs_statistics_map *map; - enum ia_css_err ret = IA_CSS_SUCCESS; + int ret = 0; IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats); @@ -249,13 +251,13 @@ ia_css_get_dvs_statistics( map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL); if (map) { - mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); + hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); ia_css_translate_dvs_statistics(host_stats, map); ia_css_isp_dvs_statistics_map_free(map); } else { IA_CSS_ERROR("out of memory"); - ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + ret = -ENOMEM; } IA_CSS_LEAVE_ERR(ret); @@ -317,7 +319,7 @@ ia_css_isp_dvs_statistics_allocate( if (!grid->enable) return NULL; - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; @@ -329,7 +331,7 @@ ia_css_isp_dvs_statistics_allocate( HIVE_ISP_DDR_WORD_BYTES); me->size = hor_size + ver_size; - me->data_ptr = mmgr_malloc(me->size); + me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0); if (me->data_ptr == mmgr_NULL) goto err; me->hor_size = hor_size; @@ -358,7 +360,7 @@ ia_css_isp_dvs_statistics_map_allocate( * so we use a local char * instead. */ char *base_ptr; - me = sh_css_malloc(sizeof(*me)); + me = kvmalloc(sizeof(*me), GFP_KERNEL); if (!me) { IA_CSS_LOG("cannot allocate memory"); goto err; @@ -368,7 +370,7 @@ ia_css_isp_dvs_statistics_map_allocate( me->data_allocated = !data_ptr; if (!me->data_ptr) { - me->data_ptr = sh_css_malloc(isp_stats->size); + me->data_ptr = kvmalloc(isp_stats->size, GFP_KERNEL); if (!me->data_ptr) { IA_CSS_LOG("cannot allocate memory"); goto err; @@ -385,7 +387,7 @@ ia_css_isp_dvs_statistics_map_allocate( return me; err: if (me) - sh_css_free(me); + kvfree(me); return NULL; } @@ -394,8 +396,8 @@ ia_css_isp_dvs_statistics_map_free(struct ia_css_isp_dvs_statistics_map *me) { if (me) { if (me->data_allocated) - sh_css_free(me->data_ptr); - sh_css_free(me); + kvfree(me->data_ptr); + kvfree(me); } } @@ -404,7 +406,7 @@ ia_css_isp_dvs_statistics_free(struct ia_css_isp_dvs_statistics *me) { if (me) { hmm_free(me->data_ptr); - sh_css_free(me); + kvfree(me); } } diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h index b1b0cb8ea175..0d0ed96e08fe 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -53,7 +54,7 @@ void ia_css_get_isp_dis_coefficients( short *horizontal_coefficients, short *vertical_coefficients); -enum ia_css_err +int ia_css_get_dvs_statistics( struct ia_css_dvs_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats); diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h index 5542fa5555b4..a8f2b8afcfd6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_1.0/ia_css_sdis_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c index 20fa7d924d58..7922198f6784 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include "hmm.h" + #include <assert_support.h> -#include "memory_access.h" #include "ia_css_debug.h" #include "ia_css_sdis2.host.h" @@ -174,12 +176,12 @@ void ia_css_sdis2_clear_coefficients( dvs2_coefs->ver_coefs.even_imag = NULL; } -enum ia_css_err +int ia_css_get_dvs2_statistics( struct ia_css_dvs2_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats) { struct ia_css_isp_dvs_statistics_map *map; - enum ia_css_err ret = IA_CSS_SUCCESS; + int ret = 0; IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats); @@ -189,13 +191,13 @@ ia_css_get_dvs2_statistics( map = ia_css_isp_dvs_statistics_map_allocate(isp_stats, NULL); if (map) { - mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); + hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); ia_css_translate_dvs2_statistics(host_stats, map); ia_css_isp_dvs_statistics_map_free(map); } else { IA_CSS_ERROR("out of memory"); - ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + ret = -ENOMEM; } IA_CSS_LEAVE_ERR(ret); @@ -284,7 +286,7 @@ ia_css_isp_dvs2_statistics_allocate( if (!grid->enable) return NULL; - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; @@ -295,7 +297,7 @@ ia_css_isp_dvs2_statistics_allocate( * grid->aligned_height * IA_CSS_DVS2_NUM_COEF_TYPES; me->size = 2 * size; - me->data_ptr = mmgr_malloc(me->size); + me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0); if (me->data_ptr == mmgr_NULL) goto err; me->hor_proj = me->data_ptr; @@ -317,7 +319,7 @@ ia_css_isp_dvs2_statistics_free(struct ia_css_isp_dvs_statistics *me) { if (me) { hmm_free(me->data_ptr); - sh_css_free(me); + kvfree(me); } } diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h index a966a6bcb692..e0e6b9c338ad 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -62,7 +63,7 @@ void ia_css_get_isp_dvs2_coefficients( void ia_css_sdis2_clear_coefficients( struct ia_css_dvs2_coefficients *dvs2_coefs); -enum ia_css_err +int ia_css_get_dvs2_statistics( struct ia_css_dvs2_statistics *host_stats, const struct ia_css_isp_dvs_statistics *isp_stats); diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h index e8ae135bfd6a..d75b72e9551b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/sdis/sdis_2/ia_css_sdis2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c index 69921c27bfae..fef8c5457cda 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h index bc6e1653e354..7e44d78c5d5d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h index a93891448cde..e904f7122142 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h index e4263afee7da..0f69f9128f0e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tdf/tdf_1.0/ia_css_tdf_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h index 349f0800bbe6..4b53fddfcd2d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr3/ia_css_tnr3_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c index ecbd3042951a..ac80e6c6e67e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -95,7 +96,7 @@ ia_css_tnr_config( void ia_css_tnr_configure( const struct ia_css_binary *binary, - const struct ia_css_frame **frames) + const struct ia_css_frame * const *frames) { struct ia_css_tnr_configuration config; unsigned int i; diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h index 3dbf962089d0..90d6e6b44a8d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -47,7 +48,7 @@ ia_css_tnr_config( void ia_css_tnr_configure( const struct ia_css_binary *binary, - const struct ia_css_frame **frames); + const struct ia_css_frame * const *frames); void ia_css_init_tnr_state( diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h index 1973766d8e41..60a2542cf685 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -34,7 +35,7 @@ struct sh_css_isp_tnr_isp_config { u32 width_a_over_b; u32 frame_height; struct dma_port_config port_b; - hrt_vaddress tnr_frame_addr[NUM_TNR_FRAMES]; + ia_css_ptr tnr_frame_addr[NUM_TNR_FRAMES]; }; #endif /* __IA_CSS_TNR_PARAM_H */ diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h index 901aa1e298e0..d57238423947 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_state.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h index 98b0daeeab39..92dbe13895c6 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/tnr/tnr_1.0/ia_css_tnr_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h index 26b7b5bc9391..784b5c4facd2 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/uds/uds_1.0/ia_css_uds_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c index be274d680caf..358cb7d2cd4c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -49,7 +50,7 @@ ia_css_vf_config( * to the requested viewfinder resolution on the upper side. The output cannot * be smaller than the requested viewfinder resolution. */ -enum ia_css_err +int sh_css_vf_downscale_log2( const struct ia_css_frame_info *out_info, const struct ia_css_frame_info *vf_info, @@ -58,12 +59,12 @@ sh_css_vf_downscale_log2( unsigned int out_width; if ((!out_info) | (!vf_info)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; out_width = out_info->res.width; if (out_width == 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* downscale until width smaller than the viewfinder width. We don't * test for the height since the vmem buffers only put restrictions on @@ -79,26 +80,26 @@ sh_css_vf_downscale_log2( ds_log2--; /* TODO: use actual max input resolution of vf_pp binary */ if ((out_info->res.width >> ds_log2) >= 2 * ia_css_binary_max_vf_width()) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; *downscale_log2 = ds_log2; - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int configure_kernel( const struct ia_css_binary_info *info, const struct ia_css_frame_info *out_info, const struct ia_css_frame_info *vf_info, unsigned int *downscale_log2, struct ia_css_vf_configuration *config) { - enum ia_css_err err; + int err; unsigned int vf_log_ds = 0; /* First compute value */ if (vf_info) { err = sh_css_vf_downscale_log2(out_info, vf_info, &vf_log_ds); - if (err != IA_CSS_SUCCESS) + if (err) return err; } vf_log_ds = min(vf_log_ds, info->vf_dec.max_log_downscale); @@ -106,7 +107,7 @@ configure_kernel( /* Then store it in isp config section */ config->vf_downscale_bits = vf_log_ds; - return IA_CSS_SUCCESS; + return 0; } static void @@ -117,13 +118,13 @@ configure_dma( config->info = vf_info; } -enum ia_css_err +int ia_css_vf_configure( const struct ia_css_binary *binary, const struct ia_css_frame_info *out_info, struct ia_css_frame_info *vf_info, unsigned int *downscale_log2) { - enum ia_css_err err; + int err; struct ia_css_vf_configuration config; const struct ia_css_binary_info *info = &binary->info->sp; @@ -134,5 +135,5 @@ ia_css_vf_configure( vf_info->raw_bit_depth = info->dma.vfdec_bits_per_pixel; ia_css_configure_vf(binary, &config); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h index 9cc594f9a840..0e8de034a00e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -25,7 +26,7 @@ * to the requested viewfinder resolution on the upper side. The output cannot * be smaller than the requested viewfinder resolution. */ -enum ia_css_err +int sh_css_vf_downscale_log2( const struct ia_css_frame_info *out_info, const struct ia_css_frame_info *vf_info, @@ -37,7 +38,7 @@ ia_css_vf_config( const struct ia_css_vf_configuration *from, unsigned int size); -enum ia_css_err +int ia_css_vf_configure( const struct ia_css_binary *binary, const struct ia_css_frame_info *out_info, diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h index 171a98508a88..487ddf163324 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h index a4d39e2e9d8e..24fbb61d349c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/vf/vf_1.0/ia_css_vf_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c index d07c500eb542..01d1a2d361c3 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h index 545dea39c2e0..ffd75c8a64fb 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h index dcf548da55cc..51b2ba8efc18 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h index 59cbd71ef332..20ae73c0ef8c 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/wb/wb_1.0/ia_css_wb_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c index e04c604ba612..1cd59660857f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h index 31833b78739f..686101c0b6a9 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h index 72a5c5fd10e7..7ebf139f3618 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c index 78653b2666a4..5566f3c16aac 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,8 +13,9 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ + #include <type_support.h> -#include <string_support.h> /* memcpy */ #include "system_global.h" #include "vamem.h" #include "ia_css_types.h" diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h index 130086713a7f..2f4ab8ad402b 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_table.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h index 22189c936f64..9a4d2e470524 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_1.0/ia_css_xnr_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c index a9db6366d20b..9c9d9b9a453e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -126,7 +127,7 @@ compute_blending(int strength) * exactly as s0.11 fixed point, but -1.0 can. */ isp_strength = -(((strength * isp_scale) + offset) / host_scale); - return MAX(MIN(isp_strength, 0), -XNR_BLENDING_SCALE_FACTOR); + return MAX(MIN(isp_strength, 0), -isp_scale); } void diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h index 959533ec29c6..6b57990b72da 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h index 7d108669e19a..c728db7ce917 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h index 6963bef3c07d..4447ba31ad69 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/xnr/xnr_3.0/ia_css_xnr3_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c index a1d0e915636d..048ffbc90b8e 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h index 20165093a298..049706e1ffa9 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h index 8f104bcb4d0f..8d9069ec28a1 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h index 1a62e1dbfc6f..da1ba21a0726 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_1.0/ia_css_ynr_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c index 9a3cd59c4507..08e9d72c143f 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h index 38204f8c5735..eaf253d59c4d 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2.host.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h index 7479bce598d5..96c80b3af426 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h index 36e4bb61b38c..ab77f4e85319 100644 --- a/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h +++ b/drivers/staging/media/atomisp/pci/isp/kernels/ynr/ynr_2/ia_css_ynr2_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h index 5774c905d8e1..f86cf9bf13a5 100644 --- a/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h +++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/input_buf.isp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h index fc392c7fb18b..11e439d838ae 100644 --- a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h +++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_const.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h index 6bdf8451e7d4..ae273c826808 100644 --- a/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h +++ b/drivers/staging/media/atomisp/pci/isp/modes/interface/isp_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h index 759141c9310a..b4142bdde51b 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h +++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h index 3c0e2efb08ae..33ebf89ca053 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h +++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -20,15 +21,15 @@ #include "input_system_global.h" #include "input_system_defs.h" /* HIVE_ISYS_GPREG_MULTICAST_A_IDX,... */ -#include "css_receiver_2400_defs.h" /* _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX, _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX,... */ -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) -#include "isp_capture_defs.h" -#elif defined(IS_ISP_2401_MAMOIADA_SYSTEM) -/* Same name, but keep the distinction,it is a different device */ + +/* + * _HRT_CSS_RECEIVER_2400_TWO_PIXEL_EN_REG_IDX, + * _HRT_CSS_RECEIVER_2400_CSI2_FUNC_PROG_REG_IDX,... + */ +#include "css_receiver_2400_defs.h" + #include "isp_capture_defs.h" -#else -#error "input_system_local.h: 2400_SYSTEM must be one of {2400, 2401 }" -#endif + #include "isp_acquisition_defs.h" #include "input_system_ctrl_defs.h" diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h index 0ce9cbc0063e..9c39ca2da923 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h +++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h b/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h index d0de27abb95a..689e451f1ce2 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h +++ b/drivers/staging/media/atomisp/pci/isp2400_input_system_public.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2400_support.h b/drivers/staging/media/atomisp/pci/isp2400_support.h index e9106d1e6a63..06d04853d4ee 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_support.h +++ b/drivers/staging/media/atomisp/pci/isp2400_support.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2400_system_global.h b/drivers/staging/media/atomisp/pci/isp2400_system_global.h index 06fce25f2034..d87ddf1d2fe9 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_system_global.h +++ b/drivers/staging/media/atomisp/pci/isp2400_system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -46,27 +47,6 @@ * N.B. the 3 input formatters are of 2 different classess */ -/* - * Since this file is visible everywhere and the system definition - * macros are not, detect the separate definitions for {host, SP, ISP} - * - * The 2401 system has the nice property that it uses a vanilla 2400 SP - * so the SP will believe it is a 2400 system rather than 2401... - */ -//#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401) -#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) -#define IS_ISP_2401_MAMOIADA_SYSTEM -#define HAS_ISP_2401_MAMOIADA -#define HAS_SP_2400 -//#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400) -#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) -#define IS_ISP_2400_MAMOIADA_SYSTEM -#define HAS_ISP_2400_MAMOIADA -#define HAS_SP_2400 -#else -#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }" -#endif - #define USE_INPUT_SYSTEM_VERSION_2 #define HAS_MMU_VERSION_2 @@ -129,21 +109,11 @@ typedef enum { N_SP_ID } sp_ID_t; -#if defined(IS_ISP_2401_MAMOIADA_SYSTEM) typedef enum { MMU0_ID = 0, MMU1_ID, N_MMU_ID } mmu_ID_t; -#elif defined(IS_ISP_2400_MAMOIADA_SYSTEM) -typedef enum { - MMU0_ID = 0, - MMU1_ID, - N_MMU_ID -} mmu_ID_t; -#else -#error "system_global.h: SYSTEM must be one of {2400, 2401}" -#endif typedef enum { DMA0_ID = 0, diff --git a/drivers/staging/media/atomisp/pci/isp2400_system_local.h b/drivers/staging/media/atomisp/pci/isp2400_system_local.h index ee38059d6ceb..675b8e5bdcc1 100644 --- a/drivers/staging/media/atomisp/pci/isp2400_system_local.h +++ b/drivers/staging/media/atomisp/pci/isp2400_system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -19,8 +20,6 @@ #ifndef HRT_USE_VIR_ADDRS #define HRT_USE_VIR_ADDRS #endif -/* This interface is deprecated */ -/*#include "hive_isp_css_custom_host_hrt.h"*/ #endif #include "system_global.h" @@ -81,7 +80,6 @@ static const hrt_address SP_PMEM_BASE[N_SP_ID] = { }; /* MMU */ -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM) /* * MMU0_ID: The data MMU * MMU1_ID: The icache MMU @@ -90,9 +88,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = { (hrt_address)0x0000000000070000ULL, (hrt_address)0x00000000000A0000ULL }; -#else -#error "system_local.h: SYSTEM must be one of {2400, 2401 }" -#endif /* DMA */ static const hrt_address DMA_BASE[N_DMA_ID] = { @@ -106,6 +101,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = { (hrt_address)0x000000000008C000ULL, (hrt_address)0x0000000000090200ULL }; + /* (hrt_address)0x0000000000000500ULL}; */ @@ -161,6 +157,7 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = { static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = { (hrt_address)0x0000000000080000ULL }; + /* (hrt_address)0x0000000000081000ULL, */ /* capture A */ /* (hrt_address)0x0000000000082000ULL, */ /* capture B */ /* (hrt_address)0x0000000000083000ULL, */ /* capture C */ @@ -222,7 +219,6 @@ static const hrt_address SP_PMEM_BASE[N_SP_ID] = { }; /* MMU */ -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM) /* * MMU0_ID: The data MMU * MMU1_ID: The icache MMU @@ -231,9 +227,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = { (hrt_address)0x00070000UL, (hrt_address)0x000A0000UL }; -#else -#error "system_local.h: SYSTEM must be one of {2400, 2401 }" -#endif /* DMA */ static const hrt_address DMA_BASE[N_DMA_ID] = { @@ -247,6 +240,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = { (hrt_address)0x0008C000UL, (hrt_address)0x00090200UL }; + /* (hrt_address)0x00000500UL}; */ @@ -297,12 +291,14 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = { (hrt_address)0x00030200UL, (hrt_address)0x00030400UL }; + /* (hrt_address)0x00030600UL, */ /* memcpy() */ /* INPUT_SYSTEM */ static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = { (hrt_address)0x00080000UL }; + /* (hrt_address)0x00081000UL, */ /* capture A */ /* (hrt_address)0x00082000UL, */ /* capture B */ /* (hrt_address)0x00083000UL, */ /* capture C */ diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h index 9c882fe134f4..5070e651f7c4 100644 --- a/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h +++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h index f199423e28da..f52a8ca5f86b 100644 --- a/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h +++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h b/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h index 3f60f59ae51f..f3ca5d1bcb01 100644 --- a/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h +++ b/drivers/staging/media/atomisp/pci/isp2401_input_system_private.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h b/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h deleted file mode 100644 index 42f821473826..000000000000 --- a/drivers/staging/media/atomisp/pci/isp2401_mamoiada_params.h +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -/* Version */ -#define RTL_VERSION - -/* Cell name */ -#define ISP_CELL_TYPE isp2401_mamoiada -#define ISP_VMEM simd_vmem -#define _HRT_ISP_VMEM isp2401_mamoiada_simd_vmem - -/* instruction pipeline depth */ -#define ISP_BRANCHDELAY 5 - -/* bus */ -#define ISP_BUS_WIDTH 32 -#define ISP_BUS_ADDR_WIDTH 32 -#define ISP_BUS_BURST_SIZE 1 - -/* data-path */ -#define ISP_SCALAR_WIDTH 32 -#define ISP_SLICE_NELEMS 4 -#define ISP_VEC_NELEMS 64 -#define ISP_VEC_ELEMBITS 14 -#define ISP_VEC_ELEM8BITS 16 -#define ISP_CLONE_DATAPATH_IS_16 1 - -/* memories */ -#define ISP_DMEM_DEPTH 4096 -#define ISP_DMEM_BSEL_DOWNSAMPLE 8 -#define ISP_VMEM_DEPTH 3072 -#define ISP_VMEM_BSEL_DOWNSAMPLE 8 -#define ISP_VMEM_ELEMBITS 14 -#define ISP_VMEM_ELEM_PRECISION 14 -#define ISP_PMEM_DEPTH 2048 -#define ISP_PMEM_WIDTH 640 -#define ISP_VAMEM_ADDRESS_BITS 12 -#define ISP_VAMEM_ELEMBITS 12 -#define ISP_VAMEM_DEPTH 2048 -#define ISP_VAMEM_ALIGNMENT 2 -#define ISP_VA_ADDRESS_WIDTH 896 -#define ISP_VEC_VALSU_LATENCY ISP_VEC_NELEMS -#define ISP_HIST_ADDRESS_BITS 12 -#define ISP_HIST_ALIGNMENT 4 -#define ISP_HIST_COMP_IN_PREC 12 -#define ISP_HIST_DEPTH 1024 -#define ISP_HIST_WIDTH 24 -#define ISP_HIST_COMPONENTS 4 - -/* program counter */ -#define ISP_PC_WIDTH 13 - -/* Template switches */ -#define ISP_SHIELD_INPUT_DMEM 0 -#define ISP_SHIELD_OUTPUT_DMEM 1 -#define ISP_SHIELD_INPUT_VMEM 0 -#define ISP_SHIELD_OUTPUT_VMEM 0 -#define ISP_SHIELD_INPUT_PMEM 1 -#define ISP_SHIELD_OUTPUT_PMEM 1 -#define ISP_SHIELD_INPUT_HIST 1 -#define ISP_SHIELD_OUTPUT_HIST 1 -/* When LUT is select the shielding is always on */ -#define ISP_SHIELD_INPUT_VAMEM 1 -#define ISP_SHIELD_OUTPUT_VAMEM 1 - -#define ISP_HAS_IRQ 1 -#define ISP_HAS_SOFT_RESET 1 -#define ISP_HAS_VEC_DIV 0 -#define ISP_HAS_VFU_W_2O 1 -#define ISP_HAS_DEINT3 1 -#define ISP_HAS_LUT 1 -#define ISP_HAS_HIST 1 -#define ISP_HAS_VALSU 1 -#define ISP_HAS_3rdVALSU 1 -#define ISP_VRF1_HAS_2P 1 - -#define ISP_SRU_GUARDING 1 -#define ISP_VLSU_GUARDING 1 - -#define ISP_VRF_RAM 1 -#define ISP_SRF_RAM 1 - -#define ISP_SPLIT_VMUL_VADD_IS 0 -#define ISP_RFSPLIT_FPGA 0 - -/* RSN or Bus pipelining */ -#define ISP_RSN_PIPE 1 -#define ISP_VSF_BUS_PIPE 0 - -/* extra slave port to vmem */ -#define ISP_IF_VMEM 0 -#define ISP_GDC_VMEM 0 - -/* Streaming ports */ -#define ISP_IF 1 -#define ISP_IF_B 1 -#define ISP_GDC 1 -#define ISP_SCL 1 -#define ISP_GPFIFO 1 -#define ISP_SP 1 - -/* Removing Issue Slot(s) */ -#define ISP_HAS_NOT_SIMD_IS2 0 -#define ISP_HAS_NOT_SIMD_IS3 0 -#define ISP_HAS_NOT_SIMD_IS4 0 -#define ISP_HAS_NOT_SIMD_IS4_VADD 0 -#define ISP_HAS_NOT_SIMD_IS5 0 -#define ISP_HAS_NOT_SIMD_IS6 0 -#define ISP_HAS_NOT_SIMD_IS7 0 -#define ISP_HAS_NOT_SIMD_IS8 0 - -/* ICache */ -#define ISP_ICACHE 1 -#define ISP_ICACHE_ONLY 0 -#define ISP_ICACHE_PREFETCH 1 -#define ISP_ICACHE_INDEX_BITS 8 -#define ISP_ICACHE_SET_BITS 5 -#define ISP_ICACHE_BLOCKS_PER_SET_BITS 1 - -/* Experimental Flags */ -#define ISP_EXP_1 0 -#define ISP_EXP_2 0 -#define ISP_EXP_3 0 -#define ISP_EXP_4 0 -#define ISP_EXP_5 0 -#define ISP_EXP_6 0 - -/* Derived values */ -#define ISP_LOG2_PMEM_WIDTH 10 -#define ISP_VEC_WIDTH 896 -#define ISP_SLICE_WIDTH 56 -#define ISP_VMEM_WIDTH 896 -#define ISP_VMEM_ALIGN 128 -#define ISP_SIMDLSU 1 -#define ISP_LSU_IMM_BITS 12 - -/* convenient shortcuts for software*/ -#define ISP_NWAY ISP_VEC_NELEMS -#define NBITS ISP_VEC_ELEMBITS - -#define _isp_ceil_div(a, b) (((a) + (b) - 1) / (b)) - -#define ISP_VEC_ALIGN ISP_VMEM_ALIGN - -/* HRT specific vector support */ -#define isp2401_mamoiada_vector_alignment ISP_VEC_ALIGN -#define isp2401_mamoiada_vector_elem_bits ISP_VMEM_ELEMBITS -#define isp2401_mamoiada_vector_elem_precision ISP_VMEM_ELEM_PRECISION -#define isp2401_mamoiada_vector_num_elems ISP_VEC_NELEMS - -/* register file sizes */ -#define ISP_RF0_SIZE 64 -#define ISP_RF1_SIZE 16 -#define ISP_RF2_SIZE 64 -#define ISP_RF3_SIZE 4 -#define ISP_RF4_SIZE 64 -#define ISP_RF5_SIZE 16 -#define ISP_RF6_SIZE 16 -#define ISP_RF7_SIZE 16 -#define ISP_RF8_SIZE 16 -#define ISP_RF9_SIZE 16 -#define ISP_RF10_SIZE 16 -#define ISP_RF11_SIZE 16 -#define ISP_VRF1_SIZE 32 -#define ISP_VRF2_SIZE 32 -#define ISP_VRF3_SIZE 32 -#define ISP_VRF4_SIZE 32 -#define ISP_VRF5_SIZE 32 -#define ISP_VRF6_SIZE 32 -#define ISP_VRF7_SIZE 32 -#define ISP_VRF8_SIZE 32 -#define ISP_SRF1_SIZE 4 -#define ISP_SRF2_SIZE 64 -#define ISP_SRF3_SIZE 64 -#define ISP_SRF4_SIZE 32 -#define ISP_SRF5_SIZE 64 -#define ISP_FRF0_SIZE 16 -#define ISP_FRF1_SIZE 4 -#define ISP_FRF2_SIZE 16 -#define ISP_FRF3_SIZE 4 -#define ISP_FRF4_SIZE 4 -#define ISP_FRF5_SIZE 8 -#define ISP_FRF6_SIZE 4 -/* register file read latency */ -#define ISP_VRF1_READ_LAT 1 -#define ISP_VRF2_READ_LAT 1 -#define ISP_VRF3_READ_LAT 1 -#define ISP_VRF4_READ_LAT 1 -#define ISP_VRF5_READ_LAT 1 -#define ISP_VRF6_READ_LAT 1 -#define ISP_VRF7_READ_LAT 1 -#define ISP_VRF8_READ_LAT 1 -#define ISP_SRF1_READ_LAT 1 -#define ISP_SRF2_READ_LAT 1 -#define ISP_SRF3_READ_LAT 1 -#define ISP_SRF4_READ_LAT 1 -#define ISP_SRF5_READ_LAT 1 -#define ISP_SRF5_READ_LAT 1 -/* immediate sizes */ -#define ISP_IS1_IMM_BITS 14 -#define ISP_IS2_IMM_BITS 13 -#define ISP_IS3_IMM_BITS 14 -#define ISP_IS4_IMM_BITS 14 -#define ISP_IS5_IMM_BITS 9 -#define ISP_IS6_IMM_BITS 16 -#define ISP_IS7_IMM_BITS 9 -#define ISP_IS8_IMM_BITS 16 -#define ISP_IS9_IMM_BITS 11 -/* fifo depths */ -#define ISP_IF_FIFO_DEPTH 0 -#define ISP_IF_B_FIFO_DEPTH 0 -#define ISP_DMA_FIFO_DEPTH 0 -#define ISP_OF_FIFO_DEPTH 0 -#define ISP_GDC_FIFO_DEPTH 0 -#define ISP_SCL_FIFO_DEPTH 0 -#define ISP_GPFIFO_FIFO_DEPTH 0 -#define ISP_SP_FIFO_DEPTH 0 diff --git a/drivers/staging/media/atomisp/pci/isp2401_system_global.h b/drivers/staging/media/atomisp/pci/isp2401_system_global.h index 213b6ee52208..8bb2a956f983 100644 --- a/drivers/staging/media/atomisp/pci/isp2401_system_global.h +++ b/drivers/staging/media/atomisp/pci/isp2401_system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -52,27 +53,6 @@ #define USE_INPUT_SYSTEM_VERSION_2401 -/* - * Since this file is visible everywhere and the system definition - * macros are not, detect the separate definitions for {host, SP, ISP} - * - * The 2401 system has the nice property that it uses a vanilla 2400 SP - * so the SP will believe it is a 2400 system rather than 2401... - */ -/* #if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) || defined(__scalar_processor_2401) */ -#if defined(SYSTEM_hive_isp_css_2401_system) || defined(__isp2401_mamoiada) -#define IS_ISP_2401_MAMOIADA_SYSTEM -#define HAS_ISP_2401_MAMOIADA -#define HAS_SP_2400 -/* #elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) || defined(__scalar_processor_2400)*/ -#elif defined(SYSTEM_hive_isp_css_2400_system) || defined(__isp2400_mamoiada) -#define IS_ISP_2400_MAMOIADA_SYSTEM -#define HAS_ISP_2400_MAMOIADA -#define HAS_SP_2400 -#else -#error "system_global.h: 2400_SYSTEM must be one of {2400, 2401 }" -#endif - #define HAS_MMU_VERSION_2 #define HAS_DMA_VERSION_2 #define HAS_GDC_VERSION_2 @@ -143,21 +123,11 @@ typedef enum { N_SP_ID } sp_ID_t; -#if defined(IS_ISP_2401_MAMOIADA_SYSTEM) -typedef enum { - MMU0_ID = 0, - MMU1_ID, - N_MMU_ID -} mmu_ID_t; -#elif defined(IS_ISP_2400_MAMOIADA_SYSTEM) typedef enum { MMU0_ID = 0, MMU1_ID, N_MMU_ID } mmu_ID_t; -#else -#error "system_global.h: SYSTEM must be one of {2400, 2401}" -#endif typedef enum { DMA0_ID = 0, diff --git a/drivers/staging/media/atomisp/pci/isp2401_system_local.h b/drivers/staging/media/atomisp/pci/isp2401_system_local.h index 4bd95b818494..b09f8faadb13 100644 --- a/drivers/staging/media/atomisp/pci/isp2401_system_local.h +++ b/drivers/staging/media/atomisp/pci/isp2401_system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -19,8 +20,6 @@ #ifndef HRT_USE_VIR_ADDRS #define HRT_USE_VIR_ADDRS #endif -/* This interface is deprecated */ -/*#include "hive_isp_css_custom_host_hrt.h"*/ #endif #include "system_global.h" @@ -75,7 +74,6 @@ static const hrt_address SP_DMEM_BASE[N_SP_ID] = { }; /* MMU */ -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM) /* * MMU0_ID: The data MMU * MMU1_ID: The icache MMU @@ -84,9 +82,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = { 0x0000000000070000ULL, 0x00000000000A0000ULL }; -#else -#error "system_local.h: SYSTEM must be one of {2400, 2401 }" -#endif /* DMA */ static const hrt_address DMA_BASE[N_DMA_ID] = { @@ -104,6 +99,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = { 0x000000000008C000ULL, 0x0000000000090200ULL }; + /* 0x0000000000000500ULL}; */ @@ -160,6 +156,7 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = { static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = { 0x0000000000080000ULL }; + /* 0x0000000000081000ULL, */ /* capture A */ /* 0x0000000000082000ULL, */ /* capture B */ /* 0x0000000000083000ULL, */ /* capture C */ @@ -258,7 +255,6 @@ static const hrt_address SP_DMEM_BASE[N_SP_ID] = { }; /* MMU */ -#if defined(IS_ISP_2400_MAMOIADA_SYSTEM) || defined(IS_ISP_2401_MAMOIADA_SYSTEM) /* * MMU0_ID: The data MMU * MMU1_ID: The icache MMU @@ -267,9 +263,6 @@ static const hrt_address MMU_BASE[N_MMU_ID] = { 0x00070000UL, 0x000A0000UL }; -#else -#error "system_local.h: SYSTEM must be one of {2400, 2401 }" -#endif /* DMA */ static const hrt_address DMA_BASE[N_DMA_ID] = { @@ -287,6 +280,7 @@ static const hrt_address IRQ_BASE[N_IRQ_ID] = { 0x0008C000UL, 0x00090200UL }; + /* 0x00000500UL}; */ @@ -336,12 +330,14 @@ static const hrt_address INPUT_FORMATTER_BASE[N_INPUT_FORMATTER_ID] = { 0x00030200UL, 0x00030400UL }; + /* 0x00030600UL, */ /* memcpy() */ /* INPUT_SYSTEM */ static const hrt_address INPUT_SYSTEM_BASE[N_INPUT_SYSTEM_ID] = { 0x00080000UL }; + /* 0x00081000UL, */ /* capture A */ /* 0x00082000UL, */ /* capture B */ /* 0x00083000UL, */ /* capture C */ diff --git a/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h b/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h index 5bdc16c71e82..8e4bddc3790d 100644 --- a/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h +++ b/drivers/staging/media/atomisp/pci/isp_acquisition_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/isp_capture_defs.h b/drivers/staging/media/atomisp/pci/isp_capture_defs.h index 5ab796e5a53f..14cbb6390d3e 100644 --- a/drivers/staging/media/atomisp/pci/isp_capture_defs.h +++ b/drivers/staging/media/atomisp/pci/isp_capture_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/isp2400_mamoiada_params.h b/drivers/staging/media/atomisp/pci/mamoiada_params.h index edc4d4ff1846..e18e5f3576df 100644 --- a/drivers/staging/media/atomisp/pci/css_2400_system/hrt/isp2400_mamoiada_params.h +++ b/drivers/staging/media/atomisp/pci/mamoiada_params.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,11 +16,6 @@ /* Version */ #define RTL_VERSION -/* Cell name */ -#define ISP_CELL_TYPE isp2400_mamoiada -#define ISP_VMEM simd_vmem -#define _HRT_ISP_VMEM isp2400_mamoiada_simd_vmem - /* instruction pipeline depth */ #define ISP_BRANCHDELAY 5 @@ -153,12 +149,6 @@ #define ISP_VEC_ALIGN ISP_VMEM_ALIGN -/* HRT specific vector support */ -#define isp2400_mamoiada_vector_alignment ISP_VEC_ALIGN -#define isp2400_mamoiada_vector_elem_bits ISP_VMEM_ELEMBITS -#define isp2400_mamoiada_vector_elem_precision ISP_VMEM_ELEM_PRECISION -#define isp2400_mamoiada_vector_num_elems ISP_VEC_NELEMS - /* register file sizes */ #define ISP_RF0_SIZE 64 #define ISP_RF1_SIZE 16 @@ -172,14 +162,7 @@ #define ISP_RF9_SIZE 16 #define ISP_RF10_SIZE 16 #define ISP_RF11_SIZE 16 -#define ISP_VRF1_SIZE 24 -#define ISP_VRF2_SIZE 24 -#define ISP_VRF3_SIZE 24 -#define ISP_VRF4_SIZE 24 -#define ISP_VRF5_SIZE 24 -#define ISP_VRF6_SIZE 24 -#define ISP_VRF7_SIZE 24 -#define ISP_VRF8_SIZE 24 + #define ISP_SRF1_SIZE 4 #define ISP_SRF2_SIZE 64 #define ISP_SRF3_SIZE 64 diff --git a/drivers/staging/media/atomisp/pci/memory_realloc.c b/drivers/staging/media/atomisp/pci/memory_realloc.c deleted file mode 100644 index e640d5daf502..000000000000 --- a/drivers/staging/media/atomisp/pci/memory_realloc.c +++ /dev/null @@ -1,81 +0,0 @@ -/* -Support for Intel Camera Imaging ISP subsystem. -Copyright (c) 2010 - 2015, Intel Corporation. - -This program is free software; you can redistribute it and/or modify it -under the terms and conditions of the GNU General Public License, -version 2, as published by the Free Software Foundation. - -This program is distributed in the hope it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -more details. -*/ -#include "memory_realloc.h" -#include "ia_css_debug.h" -#include "ia_css_refcount.h" -#include "memory_access.h" - -static bool realloc_isp_css_mm_buf( - hrt_vaddress *curr_buf, - size_t *curr_size, - size_t needed_size, - bool force, - enum ia_css_err *err, - uint16_t mmgr_attribute); - -bool reallocate_buffer( - hrt_vaddress *curr_buf, - size_t *curr_size, - size_t needed_size, - bool force, - enum ia_css_err *err) -{ - bool ret; - u16 mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT; - - IA_CSS_ENTER_PRIVATE("void"); - - ret = realloc_isp_css_mm_buf(curr_buf, - curr_size, needed_size, force, err, mmgr_attribute); - - IA_CSS_LEAVE_PRIVATE("ret=%d", ret); - return ret; -} - -static bool realloc_isp_css_mm_buf( - hrt_vaddress *curr_buf, - size_t *curr_size, - size_t needed_size, - bool force, - enum ia_css_err *err, - uint16_t mmgr_attribute) -{ - s32 id; - - *err = IA_CSS_SUCCESS; - /* Possible optimization: add a function sh_css_isp_css_mm_realloc() - * and implement on top of hmm. */ - - IA_CSS_ENTER_PRIVATE("void"); - - if (ia_css_refcount_is_single(*curr_buf) && !force && - *curr_size >= needed_size) { - IA_CSS_LEAVE_PRIVATE("false"); - return false; - } - - id = IA_CSS_REFCOUNT_PARAM_BUFFER; - ia_css_refcount_decrement(id, *curr_buf); - *curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size, - mmgr_attribute)); - - if (!*curr_buf) { - *err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - *curr_size = 0; - } else { - *curr_size = needed_size; - } - IA_CSS_LEAVE_PRIVATE("true"); - return true; -} diff --git a/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c b/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c index 8930fd629dc3..72287de75a63 100644 --- a/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c +++ b/drivers/staging/media/atomisp/pci/mmu/isp_mmu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Medifield PNW Camera Imaging ISP subsystem. * diff --git a/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c b/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c index 031d7fa00510..0fbb361f5661 100644 --- a/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c +++ b/drivers/staging/media/atomisp/pci/mmu/sh_mmu_mrfld.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Merrifield PNW Camera Imaging ISP subsystem. * @@ -19,7 +20,6 @@ #include "type_support.h" #include "mmu/isp_mmu.h" #include "mmu/sh_mmu_mrfld.h" -#include "memory_access/memory_access.h" #include "atomisp_compat.h" #define MERR_VALID_PTE_MASK 0x80000000 @@ -63,7 +63,7 @@ static unsigned int sh_get_pd_base(struct isp_mmu *mmu, */ static void sh_tlb_flush(struct isp_mmu *mmu) { - atomisp_css_mmu_invalidate_cache(); + ia_css_mmu_invalidate_cache(); } struct isp_mmu_client sh_mmu_mrfld = { diff --git a/drivers/staging/media/atomisp/pci/mmu_defs.h b/drivers/staging/media/atomisp/pci/mmu_defs.h index c038f39ffd25..03cfb5833027 100644 --- a/drivers/staging/media/atomisp/pci/mmu_defs.h +++ b/drivers/staging/media/atomisp/pci/mmu_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h index 26a3fc4d48e8..b44099dbdacd 100644 --- a/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h +++ b/drivers/staging/media/atomisp/pci/runtime/binary/interface/ia_css_binary.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. @@ -143,8 +144,7 @@ struct ia_css_binary { struct ia_css_isp_param_css_segments css_params; }; -#define IA_CSS_BINARY_DEFAULT_SETTINGS \ -(struct ia_css_binary) { \ +#define IA_CSS_BINARY_DEFAULT_SETTINGS { \ .input_format = ATOMISP_INPUT_FORMAT_YUV420_8_LEGACY, \ .in_frame_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO, \ .internal_frame_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO, \ @@ -152,13 +152,13 @@ struct ia_css_binary { .vf_frame_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO, \ } -enum ia_css_err +int ia_css_binary_init_infos(void); -enum ia_css_err +int ia_css_binary_uninit(void); -enum ia_css_err +int ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, bool online, bool two_ppc, @@ -172,7 +172,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, int stream_config_left_padding, bool accelerator); -enum ia_css_err +int ia_css_binary_find(struct ia_css_binary_descr *descr, struct ia_css_binary *binary); @@ -188,10 +188,10 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, * the shading table directly required from ISP. * @param[out] pipe_config: The pipe configuration. * The shading information related to ISP (but, not necessary as API) is stored in the pipe_config. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err +int ia_css_binary_get_shading_info(const struct ia_css_binary *binary, enum ia_css_shading_correction_type type, unsigned int required_bds_factor, @@ -199,7 +199,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary, struct ia_css_shading_info *shading_info, struct ia_css_pipe_config *pipe_config); -enum ia_css_err +int ia_css_binary_3a_grid_info(const struct ia_css_binary *binary, struct ia_css_grid_info *info, struct ia_css_pipe *pipe); diff --git a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c index 2a23b7c6aeeb..9813014c3fd3 100644 --- a/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c +++ b/drivers/staging/media/atomisp/pci/runtime/binary/src/binary.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -14,6 +15,9 @@ #include <math_support.h> #include <gdc_device.h> /* HR_GDC_N */ + +#include "hmm.h" + #include "isp.h" /* ISP_VEC_NELEMS */ #include "ia_css_binary.h" @@ -26,6 +30,8 @@ #include "sh_css_defs.h" #include "sh_css_legacy.h" +#include "atomisp_internal.h" + #include "vf/vf_1.0/ia_css_vf.host.h" #include "sc/sc_1.0/ia_css_sc.host.h" #include "sdis/sdis_1.0/ia_css_sdis.host.h" @@ -33,8 +39,6 @@ #include "camera/pipe/interface/ia_css_pipe_binarydesc.h" -#include "memory_access.h" - #include "assert_support.h" #define IMPLIES(a, b) (!(a) || (b)) /* A => B */ @@ -132,7 +136,7 @@ struct sh_css_binary_sc_requirements { }; /* Get the requirements for the shading correction. */ -static enum ia_css_err +static int #ifndef ISP2401 ia_css_binary_compute_shading_table_bayer_origin( const struct ia_css_binary *binary, /* [in] */ @@ -147,7 +151,7 @@ sh_css_binary_get_sc_requirements( struct sh_css_binary_sc_requirements *scr) /* [out] */ #endif { - enum ia_css_err err; + int err; #ifndef ISP2401 /* Numerator and denominator of the fixed bayer downscaling factor. @@ -196,7 +200,7 @@ sh_css_binary_get_sc_requirements( /* Get the numerator and denominator of bayer downscaling factor. */ err = sh_css_bds_factor_get_numerator_denominator (required_bds_factor, &bds_num, &bds_den); - if (err != IA_CSS_SUCCESS) + if (err) #else /* Flags corresponding to NEED_BDS_FACTOR_2_00/NEED_BDS_FACTOR_1_50/NEED_BDS_FACTOR_1_25 macros * defined in isp kernels. */ @@ -226,7 +230,7 @@ sh_css_binary_get_sc_requirements( /* Get the numerator and denominator of the required bayer downscaling factor. */ err = sh_css_bds_factor_get_numerator_denominator(required_bds_factor, &bds_num, &bds_den); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); #endif @@ -479,7 +483,7 @@ return err; } /* Get the shading information of Shading Correction Type 1. */ -static enum ia_css_err +static int ia_css_binary_get_shading_info_type_1(const struct ia_css_binary *binary, /* [in] */ unsigned int required_bds_factor, /* [in] */ @@ -491,7 +495,7 @@ ia_css_binary_get_shading_info_type_1(const struct ia_css_binary struct ia_css_pipe_config *pipe_config) /* [out] */ #endif { - enum ia_css_err err; + int err; #ifndef ISP2401 struct sh_css_shading_table_bayer_origin_compute_results res; #else @@ -545,12 +549,12 @@ ia_css_binary_get_shading_info_type_1(const struct ia_css_binary required_bds_factor, stream_config, &res); - if (err != IA_CSS_SUCCESS) + if (err) #else *shading_info = DEFAULT_SHADING_INFO_TYPE_1; err = sh_css_binary_get_sc_requirements(binary, required_bds_factor, stream_config, &scr); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); #endif @@ -639,8 +643,8 @@ IA_CSS_LOG("adjust_width_bqs=%d, adjust_height_bqs=%d", adjust_width_bqs, adjust if (adjust_width_bqs > tbl_width_bqs || adjust_height_bqs > tbl_height_bqs) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } /* Origin of the internal frame on the shading table. */ @@ -698,7 +702,7 @@ IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } -enum ia_css_err +int ia_css_binary_get_shading_info(const struct ia_css_binary *binary, /* [in] */ enum ia_css_shading_correction_type type, /* [in] */ unsigned int required_bds_factor, /* [in] */ @@ -706,7 +710,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary, /* [in] */ struct ia_css_shading_info *shading_info, /* [out] */ struct ia_css_pipe_config *pipe_config) /* [out] */ { - enum ia_css_err err; + int err; assert(binary); assert(shading_info); @@ -726,7 +730,7 @@ ia_css_binary_get_shading_info(const struct ia_css_binary *binary, /* [in] */ /* Other function calls can be added here when other shading correction types will be added in the future. */ else - err = IA_CSS_ERR_NOT_SUPPORTED; + err = -ENOTSUPP; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -784,12 +788,12 @@ ia_css_binary_dvs_stat_grid_info( return; } -enum ia_css_err +int ia_css_binary_3a_grid_info(const struct ia_css_binary *binary, struct ia_css_grid_info *info, struct ia_css_pipe *pipe) { struct ia_css_3a_grid_info *s3a_info; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("binary=%p, info=%p, pipe=%p", binary, info, pipe); @@ -857,18 +861,6 @@ binary_supports_output_format(const struct ia_css_binary_xinfo *info, return false; } -#ifdef ISP2401 -static bool -binary_supports_input_format(const struct ia_css_binary_xinfo *info, - enum atomisp_input_format format) -{ - assert(info); - (void)format; - - return true; -} -#endif - static bool binary_supports_vf_format(const struct ia_css_binary_xinfo *info, enum ia_css_frame_format format) @@ -892,52 +884,52 @@ supports_bds_factor(u32 supported_factors, return ((supported_factors & PACK_BDS_FACTOR(bds_factor)) != 0); } -static enum ia_css_err +static int binary_init_info(struct ia_css_binary_xinfo *info, unsigned int i, bool *binary_found) { const unsigned char *blob = sh_css_blob_info[i].blob; unsigned int size = sh_css_blob_info[i].header.blob.size; if ((!info) || (!binary_found)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; *info = sh_css_blob_info[i].header.info.isp; *binary_found = blob; info->blob_index = i; /* we don't have this binary, skip it */ if (!size) - return IA_CSS_SUCCESS; + return 0; info->xmem_addr = sh_css_load_blob(blob, size); if (!info->xmem_addr) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - return IA_CSS_SUCCESS; + return -ENOMEM; + return 0; } /* When binaries are put at the beginning, they will only * be selected if no other primary matches. */ -enum ia_css_err +int ia_css_binary_init_infos(void) { unsigned int i; unsigned int num_of_isp_binaries = sh_css_num_binaries - NUM_OF_SPS - NUM_OF_BLS; if (num_of_isp_binaries == 0) - return IA_CSS_SUCCESS; + return 0; - all_binaries = sh_css_malloc(num_of_isp_binaries * - sizeof(*all_binaries)); + all_binaries = kvmalloc(num_of_isp_binaries * sizeof(*all_binaries), + GFP_KERNEL); if (!all_binaries) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; for (i = 0; i < num_of_isp_binaries; i++) { - enum ia_css_err ret; + int ret; struct ia_css_binary_xinfo *binary = &all_binaries[i]; bool binary_found; ret = binary_init_info(binary, i, &binary_found); - if (ret != IA_CSS_SUCCESS) + if (ret) return ret; if (!binary_found) continue; @@ -947,10 +939,10 @@ ia_css_binary_init_infos(void) { binary->blob = &sh_css_blob_info[i]; binary->mem_offsets = sh_css_blob_info[i].mem_offsets; } - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int ia_css_binary_uninit(void) { unsigned int i; struct ia_css_binary_xinfo *b; @@ -964,8 +956,8 @@ ia_css_binary_uninit(void) { } binary_infos[i] = NULL; } - sh_css_free(all_binaries); - return IA_CSS_SUCCESS; + kvfree(all_binaries); + return 0; } /* @brief Compute decimation factor for 3A statistics and shading correction. @@ -1083,7 +1075,7 @@ binary_in_frame_padded_width(int in_frame_width, return rval; } -enum ia_css_err +int ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, bool online, bool two_ppc, @@ -1114,7 +1106,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, bool need_scaling = false; struct ia_css_resolution binary_dvs_env, internal_res; - enum ia_css_err err; + int err; unsigned int i; const struct ia_css_frame_info *bin_out_info = NULL; @@ -1128,7 +1120,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, err = ia_css_isp_param_allocate_isp_parameters( &binary->mem_params, &binary->css_params, &info->mem_initializers); - if (err != IA_CSS_SUCCESS) { + if (err) { return err; } } @@ -1231,7 +1223,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, { err = ia_css_vf_configure(binary, bin_out_info, (struct ia_css_frame_info *)vf_info, &vf_log_ds); - if (err != IA_CSS_SUCCESS) { + if (err) { if (!accelerator) { ia_css_isp_param_destroy_isp_parameters( &binary->mem_params, @@ -1252,7 +1244,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, binary->vf_frame_info.format = vf_info->format; if (!bin_out_info) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; vf_out_vecs = __ISP_VF_OUTPUT_WIDTH_VECS(bin_out_info->padded_width, vf_log_ds); vf_out_width = _ISP_VF_OUTPUT_WIDTH(vf_out_vecs); @@ -1350,7 +1342,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, if (info->enable.sc) { - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { binary->sctbl_width_per_color = _ISP2400_SCTBL_WIDTH_PER_COLOR(sc_3a_dis_padded_width, s3a_log_deci); binary->sctbl_aligned_width_per_color = ISP2400_SH_CSS_MAX_SCTBL_ALIGNED_WIDTH_PER_COLOR; binary->sctbl_height = _ISP2400_SCTBL_HEIGHT(sc_3a_dis_height, s3a_log_deci); @@ -1366,7 +1358,7 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, binary->sctbl_width_per_color = 0; binary->sctbl_aligned_width_per_color = 0; binary->sctbl_height = 0; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { binary->sctbl_legacy_width_per_color = 0; binary->sctbl_legacy_height = 0; } @@ -1382,12 +1374,11 @@ ia_css_binary_fill_info(const struct ia_css_binary_xinfo *xinfo, else binary->left_padding = 0; - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err -ia_css_binary_find(struct ia_css_binary_descr *descr, - struct ia_css_binary *binary) { +static int __ia_css_binary_find(struct ia_css_binary_descr *descr, + struct ia_css_binary *binary) { int mode; bool online; bool two_ppc; @@ -1413,7 +1404,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, #ifdef ISP2401 bool enable_luma_only; #endif - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; bool continuous; unsigned int isp_pipe_version; struct ia_css_resolution dvs_env, internal_res; @@ -1441,7 +1432,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, req_bin_out_info = req_out_info[i]; } if (!req_bin_out_info) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; #ifndef ISP2401 req_vf_info = descr->vf_info; #else @@ -1699,15 +1690,6 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, binary_supports_output_format(xcandidate, req_bin_out_info->format)); continue; } -#ifdef ISP2401 - if (!binary_supports_input_format(xcandidate, descr->stream_format)) { - ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, - "ia_css_binary_find() [%d] continue: !%d\n", - __LINE__, - binary_supports_input_format(xcandidate, req_in_info->format)); - continue; - } -#endif if (xcandidate->num_output_pins > 1 && /* in case we have a second output pin, */ req_vf_info && /* and we need vf output. */ @@ -1799,7 +1781,7 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, descr->stream_config_left_padding, false); - if (err != IA_CSS_SUCCESS) + if (err) break; binary_init_metrics(&binary->metrics, &binary->info->sp); break; @@ -1812,9 +1794,32 @@ ia_css_binary_find(struct ia_css_binary_descr *descr, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_binary_find() leave: return_err=%d\n", err); + if (!err && xcandidate) + dev_dbg(atomisp_dev, + "Using binary %s (id %d), type %d, mode %d, continuous %s\n", + xcandidate->blob->name, + xcandidate->sp.id, + xcandidate->type, + xcandidate->sp.pipeline.mode, + xcandidate->sp.enable.continuous ? "true" : "false"); + + return err; } +int ia_css_binary_find(struct ia_css_binary_descr *descr, + struct ia_css_binary *binary) +{ + int ret = __ia_css_binary_find(descr, binary); + + if (unlikely(ret)) { + dev_dbg(atomisp_dev, "Seeking for binary failed at:"); + dump_stack(); + } + + return ret; +} + unsigned ia_css_binary_max_vf_width(void) { diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h index 78e433fa3466..a461b0ed03f1 100644 --- a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h +++ b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -77,10 +78,10 @@ void ia_css_bufq_init(void); * * @param queue_id[in] Index of the queue in the specified thread * @param item[in] Object to enqueue. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_enqueue_buffer( +int ia_css_bufq_enqueue_buffer( int thread_index, int queue_id, uint32_t item); @@ -91,10 +92,10 @@ enum ia_css_err ia_css_bufq_enqueue_buffer( * @param queue_id[in] Specifies the index of the queue in the list where * the item has to be read. * @paramitem [out] Object to be dequeued into this item. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_dequeue_buffer( +int ia_css_bufq_dequeue_buffer( int queue_id, uint32_t *item); @@ -105,10 +106,10 @@ enum ia_css_err ia_css_bufq_dequeue_buffer( * @param[in] evt_payload_0 The event payload. * @param[in] evt_payload_1 The event payload. * @param[in] evt_payload_2 The event payload. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_enqueue_psys_event( +int ia_css_bufq_enqueue_psys_event( u8 evt_id, u8 evt_payload_0, u8 evt_payload_1, @@ -119,10 +120,10 @@ enum ia_css_err ia_css_bufq_enqueue_psys_event( * @brief Dequeue an item from SP to host communication event queue. * * @param item Object to be dequeued into this item. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_dequeue_psys_event( +int ia_css_bufq_dequeue_psys_event( u8 item[BUFQ_EVENT_SIZE] ); @@ -131,10 +132,10 @@ enum ia_css_err ia_css_bufq_dequeue_psys_event( * @brief Enqueue an event item into host to SP EOF event queue. * * @param[in] evt_id The event ID. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_enqueue_isys_event( +int ia_css_bufq_enqueue_isys_event( uint8_t evt_id); /** @@ -142,29 +143,29 @@ enum ia_css_err ia_css_bufq_enqueue_isys_event( * * @param item Object to be dequeued into this item. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_dequeue_isys_event( +int ia_css_bufq_dequeue_isys_event( u8 item[BUFQ_EVENT_SIZE]); /** * @brief Enqueue a tagger command item into tagger command queue.. * * @param item Object to be enqueue. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_enqueue_tag_cmd( +int ia_css_bufq_enqueue_tag_cmd( uint32_t item); /** * @brief Uninitializes bufq module. * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_bufq_deinit(void); +int ia_css_bufq_deinit(void); /** * @brief Dump queue states diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h index 508209711edc..cddf5882b76a 100644 --- a/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h +++ b/drivers/staging/media/atomisp/pci/runtime/bufq/interface/ia_css_bufq_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c b/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c index 7e01df257150..38e85735293b 100644 --- a/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c +++ b/drivers/staging/media/atomisp/pci/runtime/bufq/src/bufq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -22,7 +23,6 @@ #include "ia_css_debug.h" /* ia_css_debug_dtrace*/ #include "sh_css_internal.h" /* sh_css_queue_type */ #include "sp_local.h" /* sp_address_of */ -#include "ia_css_util.h" /* ia_css_convert_errno()*/ #include "sh_css_firmware.h" /* sh_css_sp_fw*/ #define BUFQ_DUMP_FILE_NAME_PREFIX_SIZE 256 @@ -329,19 +329,18 @@ void ia_css_bufq_init(void) IA_CSS_LEAVE_PRIVATE(""); } -enum ia_css_err ia_css_bufq_enqueue_buffer( +int ia_css_bufq_enqueue_buffer( int thread_index, int queue_id, uint32_t item) { - enum ia_css_err return_err = IA_CSS_SUCCESS; ia_css_queue_t *q; int error; IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id); if ((thread_index >= SH_CSS_MAX_SP_THREADS) || (thread_index < 0) || (queue_id == SH_CSS_INVALID_QUEUE_ID)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* Get the queue for communication */ q = bufq_get_qhandle(sh_css_host2sp_buffer_queue, @@ -349,22 +348,20 @@ enum ia_css_err ia_css_bufq_enqueue_buffer( thread_index); if (q) { error = ia_css_queue_enqueue(q, item); - return_err = ia_css_convert_errno(error); } else { IA_CSS_ERROR("queue is not initialized"); - return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + error = -EBUSY; } - IA_CSS_LEAVE_ERR_PRIVATE(return_err); - return return_err; + IA_CSS_LEAVE_ERR_PRIVATE(error); + return error; } -enum ia_css_err ia_css_bufq_dequeue_buffer( +int ia_css_bufq_dequeue_buffer( int queue_id, uint32_t *item) { - enum ia_css_err return_err; - int error = 0; + int error; ia_css_queue_t *q; IA_CSS_ENTER_PRIVATE("queue_id=%d", queue_id); @@ -372,52 +369,49 @@ enum ia_css_err ia_css_bufq_dequeue_buffer( (queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES) ) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; q = bufq_get_qhandle(sh_css_sp2host_buffer_queue, queue_id, -1); if (q) { error = ia_css_queue_dequeue(q, item); - return_err = ia_css_convert_errno(error); } else { IA_CSS_ERROR("queue is not initialized"); - return_err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + error = -EBUSY; } - IA_CSS_LEAVE_ERR_PRIVATE(return_err); - return return_err; + IA_CSS_LEAVE_ERR_PRIVATE(error); + return error; } -enum ia_css_err ia_css_bufq_enqueue_psys_event( +int ia_css_bufq_enqueue_psys_event( u8 evt_id, u8 evt_payload_0, u8 evt_payload_1, uint8_t evt_payload_2) { - enum ia_css_err return_err; - int error = 0; + + int error = 0; ia_css_queue_t *q; IA_CSS_ENTER_PRIVATE("evt_id=%d", evt_id); q = bufq_get_qhandle(sh_css_host2sp_psys_event_queue, -1, -1); if (!q) { IA_CSS_ERROR("queue is not initialized"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } error = ia_css_eventq_send(q, evt_id, evt_payload_0, evt_payload_1, evt_payload_2); - return_err = ia_css_convert_errno(error); - IA_CSS_LEAVE_ERR_PRIVATE(return_err); - return return_err; + IA_CSS_LEAVE_ERR_PRIVATE(error); + return error; } -enum ia_css_err ia_css_bufq_dequeue_psys_event( +int ia_css_bufq_dequeue_psys_event( u8 item[BUFQ_EVENT_SIZE]) { - enum ia_css_err; int error = 0; ia_css_queue_t *q; @@ -425,23 +419,22 @@ enum ia_css_err ia_css_bufq_dequeue_psys_event( * by some test apps. Enablign logging here floods the log * files which may cause timeouts. */ if (!item) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; q = bufq_get_qhandle(sh_css_sp2host_psys_event_queue, -1, -1); if (!q) { IA_CSS_ERROR("queue is not initialized"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } error = ia_css_eventq_recv(q, item); - return ia_css_convert_errno(error); + return error; } -enum ia_css_err ia_css_bufq_dequeue_isys_event( +int ia_css_bufq_dequeue_isys_event( u8 item[BUFQ_EVENT_SIZE]) { #if !defined(HAS_NO_INPUT_SYSTEM) - enum ia_css_err; int error = 0; ia_css_queue_t *q; @@ -449,25 +442,24 @@ enum ia_css_err ia_css_bufq_dequeue_isys_event( * by some test apps. Enablign logging here floods the log * files which may cause timeouts. */ if (!item) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; q = bufq_get_qhandle(sh_css_sp2host_isys_event_queue, -1, -1); if (!q) { IA_CSS_ERROR("queue is not initialized"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } error = ia_css_eventq_recv(q, item); - return ia_css_convert_errno(error); + return error; #else (void)item; - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; #endif } -enum ia_css_err ia_css_bufq_enqueue_isys_event(uint8_t evt_id) +int ia_css_bufq_enqueue_isys_event(uint8_t evt_id) { #if !defined(HAS_NO_INPUT_SYSTEM) - enum ia_css_err return_err; int error = 0; ia_css_queue_t *q; @@ -475,47 +467,45 @@ enum ia_css_err ia_css_bufq_enqueue_isys_event(uint8_t evt_id) q = bufq_get_qhandle(sh_css_host2sp_isys_event_queue, -1, -1); if (!q) { IA_CSS_ERROR("queue is not initialized"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } error = ia_css_eventq_send(q, evt_id, 0, 0, 0); - return_err = ia_css_convert_errno(error); - IA_CSS_LEAVE_ERR_PRIVATE(return_err); - return return_err; + + IA_CSS_LEAVE_ERR_PRIVATE(error); + return error; #else (void)evt_id; - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; #endif } -enum ia_css_err ia_css_bufq_enqueue_tag_cmd( +int ia_css_bufq_enqueue_tag_cmd( uint32_t item) { #if !defined(HAS_NO_INPUT_SYSTEM) - enum ia_css_err return_err; - int error = 0; + int error; ia_css_queue_t *q; IA_CSS_ENTER_PRIVATE("item=%d", item); q = bufq_get_qhandle(sh_css_host2sp_tag_cmd_queue, -1, -1); if (!q) { IA_CSS_ERROR("queue is not initialized"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } error = ia_css_queue_enqueue(q, item); - return_err = ia_css_convert_errno(error); - IA_CSS_LEAVE_ERR_PRIVATE(return_err); - return return_err; + IA_CSS_LEAVE_ERR_PRIVATE(error); + return error; #else (void)item; - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; #endif } -enum ia_css_err ia_css_bufq_deinit(void) +int ia_css_bufq_deinit(void) { - return IA_CSS_SUCCESS; + return 0; } static void bufq_dump_queue_info(const char *prefix, ia_css_queue_t *qhandle) diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h index 61d612ec3a05..e04d2485ea75 100644 --- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h +++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -44,8 +45,9 @@ #define IA_CSS_DEBUG_PARAM 8 /*! Level for tracing info messages */ #define IA_CSS_DEBUG_INFO 9 + /* Global variable which controls the verbosity levels of the debug tracing */ -extern unsigned int ia_css_debug_trace_level; +extern int dbg_level; /*! @brief Enum defining the different isp parameters to dump. * Values can be combined to dump a combination of sets. @@ -90,7 +92,7 @@ enum ia_css_debug_enable_param_dump { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \ "%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__) -/* Shorthand for returning an enum ia_css_err return value */ +/* Shorthand for returning an int return value */ #define IA_CSS_LEAVE_ERR(__err) \ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, \ "%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err) @@ -111,7 +113,7 @@ enum ia_css_debug_enable_param_dump { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \ "%s(): leave: " fmt "\n", __func__, ##__VA_ARGS__) -/* Shorthand for returning an enum ia_css_err return value */ +/* Shorthand for returning an int return value */ #define IA_CSS_LEAVE_ERR_PRIVATE(__err) \ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, \ "%s() %d: leave: return_err=%d\n", __func__, __LINE__, __err) @@ -130,7 +132,7 @@ enum ia_css_debug_enable_param_dump { static inline void ia_css_debug_vdtrace(unsigned int level, const char *fmt, va_list args) { - if (ia_css_debug_trace_level >= level) + if (dbg_level >= level) sh_css_vprint(fmt, args); } diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h index 27136381857f..8ec487ad4298 100644 --- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h +++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_internal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h index e9964bb421d6..538918cfb2fc 100644 --- a/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h +++ b/drivers/staging/media/atomisp/pci/runtime/debug/interface/ia_css_debug_pipe.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c index 6fadc20104bf..2bca27a04b02 100644 --- a/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c +++ b/drivers/staging/media/atomisp/pci/runtime/debug/src/ia_css_debug.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -13,7 +14,6 @@ */ #include "debug.h" -#include "memory_access.h" #ifndef __INLINE_INPUT_SYSTEM__ #define __INLINE_INPUT_SYSTEM__ @@ -31,6 +31,8 @@ #define __INLINE_STREAM2MMIO__ #endif +#include <linux/string.h> /* for strscpy() */ + #include "ia_css_debug.h" #include "ia_css_debug_pipe.h" #include "ia_css_irq.h" @@ -47,7 +49,6 @@ #include "system_local.h" #include "assert_support.h" #include "print_support.h" -#include "string_support.h" #include "fifo_monitor.h" @@ -103,9 +104,6 @@ #include "gc/gc_2/ia_css_gc2.host.h" #include "ynr/ynr_2/ia_css_ynr2.host.h" -/* Global variable to store the dtrace verbosity level */ -unsigned int ia_css_debug_trace_level = IA_CSS_DEBUG_WARNING; - #define DPG_START "ia_css_debug_pipe_graph_dump_start " #define DPG_END " ia_css_debug_pipe_graph_dump_end\n" @@ -244,13 +242,13 @@ void ia_css_debug_dump_sp_stack_info(void) void ia_css_debug_set_dtrace_level(const unsigned int trace_level) { - ia_css_debug_trace_level = trace_level; + dbg_level = trace_level; return; } unsigned int ia_css_debug_get_dtrace_level(void) { - return ia_css_debug_trace_level; + return dbg_level; } static const char *debug_stream_format2str(const enum atomisp_input_format @@ -463,7 +461,6 @@ void ia_css_debug_dump_isp_state(void) #endif ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[2] dma_FIFO stalled", stall.fifo2); -#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA) || defined(IS_ISP_2500_SYSTEM) ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[3] gdc0_FIFO stalled", stall.fifo3); @@ -475,9 +472,6 @@ void ia_css_debug_dump_isp_state(void) #endif ia_css_debug_dtrace(2, "\t%-32s: %d\n", "[6] sp_FIFO stalled", stall.fifo6); -#else -#error "ia_css_debug: ISP cell must be one of {2400_MAMOIADA,, 2401_MAMOIADA, 2500_SKYCAM}" -#endif ia_css_debug_dtrace(2, "\t%-32s: %d\n", "status & control stalled", stall.stat_ctrl); @@ -489,14 +483,12 @@ void ia_css_debug_dump_isp_state(void) stall.vamem1); ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem2 stalled", stall.vamem2); -#if defined(HAS_ISP_2400_MAMOIADA) || defined(HAS_ISP_2401_MAMOIADA) ia_css_debug_dtrace(2, "\t%-32s: %d\n", "vamem3 stalled", stall.vamem3); ia_css_debug_dtrace(2, "\t%-32s: %d\n", "hmem stalled", stall.hmem); ia_css_debug_dtrace(2, "\t%-32s: %d\n", "pmem stalled", stall.pmem); -#endif } return; } @@ -509,7 +501,6 @@ void ia_css_debug_dump_sp_state(void) sp_get_state(SP0_ID, &state, &stall); debug_print_sp_state(&state, "SP"); if (state.is_stalling) { -#if defined(HAS_SP_2400) || defined(IS_ISP_2500_SYSTEM) #if !defined(HAS_NO_INPUT_SYSTEM) ia_css_debug_dtrace(2, "\t%-32s: %d\n", "isys_FIFO stalled", stall.fifo0); @@ -540,9 +531,6 @@ void ia_css_debug_dump_sp_state(void) #endif ia_css_debug_dtrace(2, "\t%-32s: %d\n", "irq FIFO stalled", stall.fifoa); -#else -#error "ia_css_debug: SP cell must be one of {SP2400, SP2500}" -#endif ia_css_debug_dtrace(2, "\t%-32s: %d\n", "dmem stalled", stall.dmem); ia_css_debug_dtrace(2, "\t%-32s: %d\n", @@ -2231,7 +2219,7 @@ void ia_css_debug_dump_debug_info(const char *context) ia_css_debug_dump_isys_state(); { - irq_controller_state_t state; + struct irq_controller_state state; irq_controller_get_state(IRQ2_ID, &state); @@ -2492,8 +2480,8 @@ void ia_css_debug_dump_perf_counters(void) void sh_css_init_ddr_debug_queue(void) { - hrt_vaddress ddr_debug_queue_addr = - mmgr_malloc(sizeof(debug_data_ddr_t)); + ia_css_ptr ddr_debug_queue_addr = + hmm_alloc(sizeof(debug_data_ddr_t), HMM_BO_PRIVATE, 0, NULL, 0); const struct ia_css_fw_info *fw; unsigned int HIVE_ADDR_debug_buffer_ddr_address; @@ -2665,7 +2653,7 @@ void sh_css_dump_pipe_stripe_info(void) static void ia_css_debug_pipe_graph_dump_frame( - struct ia_css_frame *frame, + const struct ia_css_frame *frame, enum ia_css_pipe_id id, char const *blob_name, char const *frame_name, @@ -2782,8 +2770,9 @@ ia_css_debug_pipe_graph_dump_stage( stage->binary->info->blob->name, stage->stage_num); } else if (stage->firmware) { bin_type = "firmware"; - strncpy_s(blob_name, sizeof(blob_name), - IA_CSS_EXT_ISP_PROG_NAME(stage->firmware), sizeof(blob_name)); + + strscpy(blob_name, IA_CSS_EXT_ISP_PROG_NAME(stage->firmware), + sizeof(blob_name)); } /* Guard in case of binaries that don't have any binary_info */ @@ -2849,10 +2838,8 @@ ia_css_debug_pipe_graph_dump_stage( while (ei[p] != ',') p--; /* Last comma found, copy till that comma */ - strncpy_s(enable_info1, - sizeof(enable_info1), - ei, p); - enable_info1[p] = '\0'; + strscpy(enable_info1, ei, + p > sizeof(enable_info1) ? sizeof(enable_info1) : p); ei += p + 1; l = strlen(ei); @@ -2862,10 +2849,10 @@ ia_css_debug_pipe_graph_dump_stage( /* we cannot use ei as argument because * it is not guaranteed dword aligned */ - strncpy_s(enable_info2, - sizeof(enable_info2), - ei, l); - enable_info2[l] = '\0'; + + strscpy(enable_info2, ei, + l > sizeof(enable_info2) ? sizeof(enable_info2) : l); + snprintf(enable_info, sizeof(enable_info), "%s\\n%s", enable_info1, enable_info2); @@ -2874,10 +2861,10 @@ ia_css_debug_pipe_graph_dump_stage( p = ENABLE_LINE_MAX_LENGTH; while (ei[p] != ',') p--; - strncpy_s(enable_info2, - sizeof(enable_info2), - ei, p); - enable_info2[p] = '\0'; + + strscpy(enable_info2, ei, + p > sizeof(enable_info2) ? sizeof(enable_info2) : p); + ei += p + 1; l = strlen(ei); @@ -2886,9 +2873,8 @@ ia_css_debug_pipe_graph_dump_stage( /* we cannot use ei as argument because * it is not guaranteed dword aligned */ - strcpy_s(enable_info3, - sizeof(enable_info3), ei); - enable_info3[l] = '\0'; + strscpy(enable_info3, ei, + sizeof(enable_info3)); snprintf(enable_info, sizeof(enable_info), "%s\\n%s\\n%s", enable_info1, enable_info2, @@ -2898,13 +2884,11 @@ ia_css_debug_pipe_graph_dump_stage( p = ENABLE_LINE_MAX_LENGTH; while (ei[p] != ',') p--; - strncpy_s(enable_info3, - sizeof(enable_info3), - ei, p); - enable_info3[p] = '\0'; + strscpy(enable_info3, ei, + p > sizeof(enable_info3) ? sizeof(enable_info3) : p); ei += p + 1; - strcpy_s(enable_info3, - sizeof(enable_info3), ei); + strscpy(enable_info3, ei, + sizeof(enable_info3)); snprintf(enable_info, sizeof(enable_info), "%s\\n%s\\n%s", enable_info1, enable_info2, @@ -3100,7 +3084,7 @@ ia_css_debug_dump_pipe_config( "capt_pp_in_res"); ia_css_debug_dump_resolution(&config->vf_pp_in_res, "vf_pp_in_res"); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { ia_css_debug_dump_resolution(&config->output_system_in_res, "output_system_in_res"); } @@ -3328,7 +3312,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id) return; } - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { tmp = ia_css_device_load_uint32(start_addr); point_num = (tmp >> 16) & 0xFFFF; @@ -3362,7 +3346,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id) if ((limit == (-1)) && (trace_read_buf[i] == 0)) limit = i; } - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "Status:\n"); for (i = 0; i < SH_CSS_MAX_SP_THREADS; i++) ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -3395,7 +3379,7 @@ static void debug_dump_one_trace(enum TRACE_CORE_ID proc_id) for (i = 0; i < point_num; i++) { j = (limit + i) % point_num; if (trace_read_buf[j]) { - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { TRACE_DUMP_FORMAT dump_format = FIELD_FORMAT_UNPACK(trace_read_buf[j]); } else { tid_val = FIELD_TID_UNPACK(trace_read_buf[j]); diff --git a/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h b/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h index 1fcd0fadcac8..ebbd90b14bff 100644 --- a/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h +++ b/drivers/staging/media/atomisp/pci/runtime/event/interface/ia_css_event.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/event/src/event.c b/drivers/staging/media/atomisp/pci/runtime/event/src/event.c index c4578470ad8c..e702297b0a76 100644 --- a/drivers/staging/media/atomisp/pci/runtime/event/src/event.c +++ b/drivers/staging/media/atomisp/pci/runtime/event/src/event.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -29,10 +30,7 @@ /*#include "sp.h"*/ /* host2sp_enqueue_frame_data() */ -#include "memory_access.h" - #include "assert_support.h" -#include "platform_support.h" /* hrt_sleep() */ #include "ia_css_queue.h" /* host_sp_enqueue_XXX */ #include "ia_css_event.h" /* ia_css_event_encode */ diff --git a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h index 8602398ede52..fd001ae3522d 100644 --- a/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h +++ b/drivers/staging/media/atomisp/pci/runtime/eventq/interface/ia_css_eventq.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -23,8 +24,8 @@ * @param[in] eventq_handle eventq_handle. * @param[in] payload The event payload. * @return 0 - Successfully dequeue. - * @return EINVAL - Invalid argument. - * @return ENODATA - Queue is empty. + * @return -EINVAL - Invalid argument. + * @return -ENODATA - Queue is empty. */ int ia_css_eventq_recv( ia_css_queue_t *eventq_handle, @@ -41,8 +42,8 @@ int ia_css_eventq_recv( * @param[in] evt_payload_1 The event payload. * @param[in] evt_payload_2 The event payload. * @return 0 - Successfully enqueue. - * @return EINVAL - Invalid argument. - * @return ENOBUFS - Queue is full. + * @return -EINVAL - Invalid argument. + * @return -ENOBUFS - Queue is full. */ int ia_css_eventq_send( ia_css_queue_t *eventq_handle, diff --git a/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c b/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c index 0460f102d36f..df75cef46a51 100644 --- a/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c +++ b/drivers/staging/media/atomisp/pci/runtime/eventq/src/eventq.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -19,8 +20,6 @@ #include "ia_css_event.h" /* ia_css_event_encode() ia_css_event_decode() */ -#include "platform_support.h" /* hrt_sleep() */ - int ia_css_eventq_recv( ia_css_queue_t *eventq_handle, uint8_t *payload) @@ -50,7 +49,7 @@ int ia_css_eventq_send( { u8 tmp[4]; u32 sw_event; - int error = ENOSYS; + int error = -ENOSYS; /* * Encode the queue type, the thread ID and @@ -65,13 +64,13 @@ int ia_css_eventq_send( /* queue the software event (busy-waiting) */ for ( ; ; ) { error = ia_css_queue_enqueue(eventq_handle, sw_event); - if (error != ENOBUFS) { + if (error != -ENOBUFS) { /* We were able to successfully send the event or had a real failure. return the status*/ break; } /* Wait for the queue to be not full and try again*/ - hrt_sleep(); + udelay(1); } return error; } diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h index 613fa33ab930..31f01e0f58aa 100644 --- a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h +++ b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -80,7 +81,7 @@ bool ia_css_frame_info_is_same_resolution( * @param[in] info The frame attributes to be initialized * @return The error code. */ -enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info); +int ia_css_frame_check_info(const struct ia_css_frame_info *info); /********************************************************************* **** Frame APIs @@ -92,7 +93,7 @@ enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info); * @param[in] frame The frame attributes to be initialized * @return The error code. */ -enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame); +int ia_css_frame_init_planes(struct ia_css_frame *frame); /* @brief Free an array of frames * @@ -114,7 +115,7 @@ void ia_css_frame_free_multiple(unsigned int num_frames, * Allocate a frame using the given size in bytes. * The frame structure is partially null initialized. */ -enum ia_css_err ia_css_frame_allocate_with_buffer_size( +int ia_css_frame_allocate_with_buffer_size( struct ia_css_frame **frame, const unsigned int size_bytes, const bool contiguous); @@ -153,9 +154,9 @@ void ia_css_dma_configure_from_info( * @param[in] in_res Resolution of input image * @param[in] out_res Resolution of output image * @param[out] crop_res Crop resolution of input image - * @return Returns IA_CSS_SUCCESS or IA_CSS_ERR_INVALID_ARGUMENTS on error + * @return Returns 0 or -EINVAL on error */ -enum ia_css_err +int ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res, const struct ia_css_resolution *out_res, struct ia_css_resolution *crop_res); diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h index 8861d07193bd..ce6110efbfc9 100644 --- a/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h +++ b/drivers/staging/media/atomisp/pci/runtime/frame/interface/ia_css_frame_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -18,7 +19,7 @@ #include "type_support.h" #include "platform_support.h" #include "runtime/bufq/interface/ia_css_bufq_comm.h" -#include <system_types.h> /* hrt_vaddress */ +#include <system_local.h> /* ia_css_ptr */ /* * These structs are derived from structs defined in ia_css_types.h @@ -83,7 +84,7 @@ struct ia_css_frame_sp_info { struct ia_css_buffer_sp { union { - hrt_vaddress xmem_addr; + ia_css_ptr xmem_addr; enum sh_css_queue_id queue_id; } buf_src; enum ia_css_buffer_type buf_type; diff --git a/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c b/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c index fcd8b06034f2..10c4907187d9 100644 --- a/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c +++ b/drivers/staging/media/atomisp/pci/runtime/frame/src/frame.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,13 +13,15 @@ * more details. */ +#include "hmm.h" + #include "ia_css_frame.h" #include <math_support.h> #include "assert_support.h" #include "ia_css_debug.h" #include "isp.h" #include "sh_css_internal.h" -#include "memory_access.h" +#include "atomisp_internal.h" #define NV12_TILEY_TILE_WIDTH 128 #define NV12_TILEY_TILE_HEIGHT 32 @@ -67,9 +70,9 @@ static void frame_init_rgb_planes(struct ia_css_frame *frame, static void frame_init_qplane6_planes(struct ia_css_frame *frame); -static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame); +static int frame_allocate_buffer_data(struct ia_css_frame *frame); -static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame, +static int frame_allocate_with_data(struct ia_css_frame **frame, unsigned int width, unsigned int height, enum ia_css_frame_format format, @@ -96,16 +99,16 @@ ia_css_elems_bytes_from_info( void ia_css_frame_zero(struct ia_css_frame *frame) { assert(frame); - mmgr_clear(frame->data, frame->data_bytes); + hmm_set(frame->data, 0, frame->data_bytes); } -enum ia_css_err ia_css_frame_allocate_from_info(struct ia_css_frame **frame, +int ia_css_frame_allocate_from_info(struct ia_css_frame **frame, const struct ia_css_frame_info *info) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (!frame || !info) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_allocate_from_info() enter:\n"); err = @@ -117,17 +120,17 @@ enum ia_css_err ia_css_frame_allocate_from_info(struct ia_css_frame **frame, return err; } -enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame, +int ia_css_frame_allocate(struct ia_css_frame **frame, unsigned int width, unsigned int height, enum ia_css_frame_format format, unsigned int padded_width, unsigned int raw_bit_depth) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (!frame || width == 0 || height == 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_allocate() enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n", @@ -136,7 +139,7 @@ enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame, err = frame_allocate_with_data(frame, width, height, format, padded_width, raw_bit_depth, false); - if ((*frame) && err == IA_CSS_SUCCESS) + if ((*frame) && err == 0) ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_allocate() leave: frame=%p, data(DDR address)=0x%x\n", *frame, (*frame)->data); @@ -148,13 +151,13 @@ enum ia_css_err ia_css_frame_allocate(struct ia_css_frame **frame, return err; } -enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame, +int ia_css_frame_map(struct ia_css_frame **frame, const struct ia_css_frame_info *info, const void __user *data, u16 attribute, - void *context) + unsigned int pgnr) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *me; assert(frame); @@ -162,20 +165,32 @@ enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame, /* Create the frame structure */ err = ia_css_frame_create_from_info(&me, info); - if (err != IA_CSS_SUCCESS) + if (err) return err; - if (err == IA_CSS_SUCCESS) { - /* use mmgr_mmap to map */ - me->data = (ia_css_ptr) mmgr_mmap(data, - me->data_bytes, - attribute, context); + if (!err) { + if (pgnr < ((PAGE_ALIGN(me->data_bytes)) >> PAGE_SHIFT)) { + dev_err(atomisp_dev, + "user space memory size is less than the expected size..\n"); + err = -ENOMEM; + goto error; + } else if (pgnr > ((PAGE_ALIGN(me->data_bytes)) >> PAGE_SHIFT)) { + dev_err(atomisp_dev, + "user space memory size is large than the expected size..\n"); + err = -ENOMEM; + goto error; + } + + me->data = hmm_alloc(me->data_bytes, HMM_BO_USER, 0, data, + attribute & ATOMISP_MAP_FLAG_CACHED); + if (me->data == mmgr_NULL) - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } - if (err != IA_CSS_SUCCESS) { - sh_css_free(me); +error: + if (err) { + kvfree(me); me = NULL; } @@ -184,10 +199,10 @@ enum ia_css_err ia_css_frame_map(struct ia_css_frame **frame, return err; } -enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame, +int ia_css_frame_create_from_info(struct ia_css_frame **frame, const struct ia_css_frame_info *info) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *me; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -195,7 +210,7 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame, if (!frame || !info) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_create_from_info() leave: invalid arguments\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } me = frame_create(info->res.width, @@ -208,13 +223,13 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame, if (!me) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_create_from_info() leave: frame create failed\n"); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; } err = ia_css_frame_init_planes(me); - if (err != IA_CSS_SUCCESS) { - sh_css_free(me); + if (err) { + kvfree(me); me = NULL; } @@ -226,18 +241,18 @@ enum ia_css_err ia_css_frame_create_from_info(struct ia_css_frame **frame, return err; } -enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame, +int ia_css_frame_set_data(struct ia_css_frame *frame, const ia_css_ptr mapped_data, size_t data_bytes) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_set_data() enter:\n"); if (!frame) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_set_data() leave: NULL frame\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* If we are setting a valid data. @@ -247,7 +262,7 @@ enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame, if ((mapped_data != mmgr_NULL) && (frame->data_bytes > data_bytes)) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_frame_set_data() leave: invalid arguments\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } frame->data = mapped_data; @@ -257,18 +272,17 @@ enum ia_css_err ia_css_frame_set_data(struct ia_css_frame *frame, return err; } -enum ia_css_err ia_css_frame_allocate_contiguous(struct ia_css_frame **frame, +int ia_css_frame_allocate_contiguous(struct ia_css_frame **frame, unsigned int width, unsigned int height, enum ia_css_frame_format format, unsigned int padded_width, unsigned int raw_bit_depth) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, - "ia_css_frame_allocate_contiguous() " - "enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n", + "ia_css_frame_allocate_contiguous() enter: width=%d, height=%d, format=%d, padded_width=%d, raw_bit_depth=%d\n", width, height, format, padded_width, raw_bit_depth); err = frame_allocate_with_data(frame, width, height, format, @@ -281,11 +295,11 @@ enum ia_css_err ia_css_frame_allocate_contiguous(struct ia_css_frame **frame, return err; } -enum ia_css_err ia_css_frame_allocate_contiguous_from_info( +int ia_css_frame_allocate_contiguous_from_info( struct ia_css_frame **frame, const struct ia_css_frame_info *info) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(frame); ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -307,7 +321,7 @@ void ia_css_frame_free(struct ia_css_frame *frame) if (frame) { hmm_free(frame->data); - sh_css_free(frame); + kvfree(frame); } IA_CSS_LEAVE_PRIVATE("void"); @@ -317,15 +331,15 @@ void ia_css_frame_free(struct ia_css_frame *frame) ** Module public functions **************************************************************************/ -enum ia_css_err ia_css_frame_check_info(const struct ia_css_frame_info *info) +int ia_css_frame_check_info(const struct ia_css_frame_info *info) { assert(info); if (info->res.width == 0 || info->res.height == 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; - return IA_CSS_SUCCESS; + return -EINVAL; + return 0; } -enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame) +int ia_css_frame_init_planes(struct ia_css_frame *frame) { assert(frame); @@ -430,9 +444,9 @@ enum ia_css_err ia_css_frame_init_planes(struct ia_css_frame *frame) frame->planes.binary.size = 0; break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } - return IA_CSS_SUCCESS; + return 0; } void ia_css_frame_info_set_width(struct ia_css_frame_info *info, @@ -517,27 +531,27 @@ void ia_css_frame_free_multiple(unsigned int num_frames, } } -enum ia_css_err ia_css_frame_allocate_with_buffer_size( +int ia_css_frame_allocate_with_buffer_size( struct ia_css_frame **frame, const unsigned int buffer_size_bytes, const bool contiguous) { /* AM: Body coppied from frame_allocate_with_data(). */ - enum ia_css_err err; + int err; struct ia_css_frame *me = frame_create(0, 0, IA_CSS_FRAME_FORMAT_NUM,/* Not valid format yet */ 0, 0, contiguous, false); if (!me) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; /* Get the data size */ me->data_bytes = buffer_size_bytes; err = frame_allocate_buffer_data(me); - if (err != IA_CSS_SUCCESS) { - sh_css_free(me); + if (err) { + kvfree(me); me = NULL; } @@ -782,22 +796,22 @@ static void frame_init_qplane6_planes(struct ia_css_frame *frame) return; } -static enum ia_css_err frame_allocate_buffer_data(struct ia_css_frame *frame) +static int frame_allocate_buffer_data(struct ia_css_frame *frame) { #ifdef ISP2401 IA_CSS_ENTER_LEAVE_PRIVATE("frame->data_bytes=%d\n", frame->data_bytes); #endif - frame->data = mmgr_alloc_attr(frame->data_bytes, - frame->contiguous ? - MMGR_ATTRIBUTE_CONTIGUOUS : - MMGR_ATTRIBUTE_DEFAULT); + frame->data = hmm_alloc(frame->data_bytes, + HMM_BO_PRIVATE, 0, NULL, + frame->contiguous ? + ATOMISP_MAP_FLAG_CONTIGUOUS : 0); if (frame->data == mmgr_NULL) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - return IA_CSS_SUCCESS; + return -ENOMEM; + return 0; } -static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame, +static int frame_allocate_with_data(struct ia_css_frame **frame, unsigned int width, unsigned int height, enum ia_css_frame_format format, @@ -805,7 +819,7 @@ static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame, unsigned int raw_bit_depth, bool contiguous) { - enum ia_css_err err; + int err; struct ia_css_frame *me = frame_create(width, height, format, @@ -815,15 +829,15 @@ static enum ia_css_err frame_allocate_with_data(struct ia_css_frame **frame, true); if (!me) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; err = ia_css_frame_init_planes(me); - if (err == IA_CSS_SUCCESS) + if (!err) err = frame_allocate_buffer_data(me); - if (err != IA_CSS_SUCCESS) { - sh_css_free(me); + if (err) { + kvfree(me); #ifndef ISP2401 return err; #else @@ -844,7 +858,7 @@ static struct ia_css_frame *frame_create(unsigned int width, bool contiguous, bool valid) { - struct ia_css_frame *me = sh_css_malloc(sizeof(*me)); + struct ia_css_frame *me = kvmalloc(sizeof(*me), GFP_KERNEL); if (!me) return NULL; @@ -919,7 +933,7 @@ void ia_css_resolution_to_sp_resolution( } /* ISP2401 */ -enum ia_css_err +int ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res, const struct ia_css_resolution *out_res, struct ia_css_resolution *crop_res) { @@ -927,7 +941,7 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res, u32 in_ratio, out_ratio; if ((!in_res) || (!out_res) || (!crop_res)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; IA_CSS_ENTER_PRIVATE("in(%ux%u) -> out(%ux%u)", in_res->width, in_res->height, out_res->width, out_res->height); @@ -936,11 +950,11 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res, || (in_res->height == 0) || (out_res->width == 0) || (out_res->height == 0)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if ((out_res->width > in_res->width) || (out_res->height > in_res->height)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* If aspect ratio (width/height) of out_res is higher than the aspect * ratio of the in_res, then we crop vertically, otherwise we crop @@ -985,5 +999,5 @@ ia_css_frame_find_crop_resolution(const struct ia_css_resolution *in_res, IA_CSS_LEAVE_PRIVATE("in(%ux%u) -> out(%ux%u)", crop_res->width, crop_res->height, out_res->width, out_res->height); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h b/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h index d4b0b2361176..2c440feec3ce 100644 --- a/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h +++ b/drivers/staging/media/atomisp/pci/runtime/ifmtr/interface/ia_css_ifmtr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -27,7 +28,7 @@ unsigned int ia_css_ifmtr_lines_needed_for_bayer_order( unsigned int ia_css_ifmtr_columns_needed_for_bayer_order( const struct ia_css_stream_config *config); -enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, +int ia_css_ifmtr_configure(struct ia_css_stream_config *config, struct ia_css_binary *binary); #endif /* __IA_CSS_IFMTR_H__ */ diff --git a/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c b/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c index 7a18eae8c638..89cded6b6e2b 100644 --- a/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c +++ b/drivers/staging/media/atomisp/pci/runtime/ifmtr/src/ifmtr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -28,12 +29,12 @@ /************************************************************ * Static functions declarations ************************************************************/ -static enum ia_css_err ifmtr_start_column( +static int ifmtr_start_column( const struct ia_css_stream_config *config, unsigned int bin_in, unsigned int *start_column); -static enum ia_css_err ifmtr_input_start_line( +static int ifmtr_input_start_line( const struct ia_css_stream_config *config, unsigned int bin_in, unsigned int *start_line); @@ -71,7 +72,7 @@ unsigned int ia_css_ifmtr_columns_needed_for_bayer_order( return 0; } -enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, +int ia_css_ifmtr_configure(struct ia_css_stream_config *config, struct ia_css_binary *binary) { unsigned int start_line, start_column = 0, @@ -97,7 +98,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, left_padding = 0; input_formatter_cfg_t if_a_config, if_b_config; enum atomisp_input_format input_format; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; u8 if_config_index; /* Determine which input formatter config set is targeted. */ @@ -142,10 +143,10 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, * columns. */ err = ifmtr_input_start_line(config, cropped_height, &start_line); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ifmtr_start_column(config, cropped_width, &start_column); - if (err != IA_CSS_SUCCESS) + if (err) return err; if (config->left_padding == -1) @@ -358,7 +359,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, break; } if (width_a == 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (two_ppc) left_padding /= 2; @@ -459,7 +460,7 @@ enum ia_css_err ia_css_ifmtr_configure(struct ia_css_stream_config *config, } } - return IA_CSS_SUCCESS; + return 0; } bool ifmtr_set_if_blocking_mode_reset = true; @@ -496,7 +497,7 @@ static void ifmtr_set_if_blocking_mode( return; } -static enum ia_css_err ifmtr_start_column( +static int ifmtr_start_column( const struct ia_css_stream_config *config, unsigned int bin_in, unsigned int *start_column) @@ -505,7 +506,7 @@ static enum ia_css_err ifmtr_start_column( for_bayer = ia_css_ifmtr_columns_needed_for_bayer_order(config); if (bin_in + 2 * for_bayer > in) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* On the hardware, we want to use the middle of the input, so we * divide the start column by 2. */ @@ -519,10 +520,10 @@ static enum ia_css_err ifmtr_start_column( */ start += for_bayer; *start_column = start; - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err ifmtr_input_start_line( +static int ifmtr_input_start_line( const struct ia_css_stream_config *config, unsigned int bin_in, unsigned int *start_line) @@ -531,7 +532,7 @@ static enum ia_css_err ifmtr_input_start_line( for_bayer = ia_css_ifmtr_lines_needed_for_bayer_order(config); if (bin_in + 2 * for_bayer > in) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* On the hardware, we want to use the middle of the input, so we * divide the start line by 2. On the simulator, we cannot handle extra @@ -546,7 +547,7 @@ static enum ia_css_err ifmtr_input_start_line( /* now we add the one line (if needed) to correct for the bayer order */ start += for_bayer; *start_line = start; - return IA_CSS_SUCCESS; + return 0; } #endif diff --git a/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h b/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h index d2dd231b6296..7950c5c36693 100644 --- a/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h +++ b/drivers/staging/media/atomisp/pci/runtime/inputfifo/interface/ia_css_inputfifo.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c b/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c index e5a339fb52f2..38712530f566 100644 --- a/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c +++ b/drivers/staging/media/atomisp/pci/runtime/inputfifo/src/inputfifo.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -103,7 +104,7 @@ static inline void _sh_css_fifo_snd(unsigned int token) { while (!can_event_send_token(STR2MIPI_EVENT_ID)) - hrt_sleep(); + udelay(1); event_send_token(STR2MIPI_EVENT_ID, token); return; } diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h index 2769183a8956..0ea5d6fdc88b 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h +++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -31,7 +32,7 @@ ia_css_isp_param_set_css_mem_init( struct ia_css_isp_param_css_segments *mem_init, enum ia_css_param_class pclass, enum ia_css_isp_memories mem, - hrt_vaddress address, size_t size); + ia_css_ptr address, size_t size); void ia_css_isp_param_set_isp_mem_init( @@ -67,7 +68,7 @@ ia_css_init_memory_interface( const struct ia_css_isp_param_css_segments *css_params); /* Allocate memory parameters */ -enum ia_css_err +int ia_css_isp_param_allocate_isp_parameters( struct ia_css_isp_param_host_segments *mem_params, struct ia_css_isp_param_css_segments *css_params, @@ -88,7 +89,7 @@ ia_css_isp_param_load_fw_params( bool init); /* Copy host parameter images to ddr */ -enum ia_css_err +int ia_css_isp_param_copy_isp_mem_if_to_ddr( struct ia_css_isp_param_css_segments *ddr, const struct ia_css_isp_param_host_segments *host, diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h index 5d23b2f57719..8cdeae98bda8 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h +++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/interface/ia_css_isp_param_types.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c index 443e412d05ad..e861777385a0 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c +++ b/drivers/staging/media/atomisp/pci/runtime/isp_param/src/isp_param.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -12,7 +13,8 @@ * more details. */ -#include "memory_access.h" +#include "hmm.h" + #include "ia_css_pipeline.h" #include "ia_css_isp_param.h" @@ -34,7 +36,7 @@ ia_css_isp_param_set_css_mem_init( struct ia_css_isp_param_css_segments *mem_init, enum ia_css_param_class pclass, enum ia_css_isp_memories mem, - hrt_vaddress address, size_t size) + ia_css_ptr address, size_t size) { mem_init->params[pclass][mem].address = address; mem_init->params[pclass][mem].size = (uint32_t)size; @@ -100,12 +102,12 @@ ia_css_init_memory_interface( } } -enum ia_css_err +int ia_css_isp_param_allocate_isp_parameters( struct ia_css_isp_param_host_segments *mem_params, struct ia_css_isp_param_css_segments *css_params, const struct ia_css_isp_param_isp_segments *mem_initializers) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int mem, pclass; pclass = IA_CSS_PARAM_CLASS_PARAM; @@ -121,15 +123,17 @@ ia_css_isp_param_allocate_isp_parameters( css_params->params[pclass][mem].size = size; css_params->params[pclass][mem].address = 0x0; if (size) { - mem_params->params[pclass][mem].address = sh_css_calloc(1, size); + mem_params->params[pclass][mem].address = kvcalloc(1, + size, + GFP_KERNEL); if (!mem_params->params[pclass][mem].address) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto cleanup; } if (pclass != IA_CSS_PARAM_CLASS_PARAM) { - css_params->params[pclass][mem].address = mmgr_malloc(size); + css_params->params[pclass][mem].address = hmm_alloc(size, HMM_BO_PRIVATE, 0, NULL, 0); if (!css_params->params[pclass][mem].address) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto cleanup; } } @@ -152,7 +156,7 @@ ia_css_isp_param_destroy_isp_parameters( for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) { for (pclass = 0; pclass < IA_CSS_NUM_PARAM_CLASSES; pclass++) { if (mem_params->params[pclass][mem].address) - sh_css_free(mem_params->params[pclass][mem].address); + kvfree(mem_params->params[pclass][mem].address); if (css_params->params[pclass][mem].address) hmm_free(css_params->params[pclass][mem].address); mem_params->params[pclass][mem].address = NULL; @@ -177,7 +181,7 @@ ia_css_isp_param_load_fw_params( } } -enum ia_css_err +int ia_css_isp_param_copy_isp_mem_if_to_ddr( struct ia_css_isp_param_css_segments *ddr, const struct ia_css_isp_param_host_segments *host, @@ -187,16 +191,16 @@ ia_css_isp_param_copy_isp_mem_if_to_ddr( for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) { size_t size = host->params[pclass][mem].size; - hrt_vaddress ddr_mem_ptr = ddr->params[pclass][mem].address; + ia_css_ptr ddr_mem_ptr = ddr->params[pclass][mem].address; char *host_mem_ptr = host->params[pclass][mem].address; if (size != ddr->params[pclass][mem].size) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; if (!size) continue; - mmgr_store(ddr_mem_ptr, host_mem_ptr, size); + hmm_store(ddr_mem_ptr, host_mem_ptr, size); } - return IA_CSS_SUCCESS; + return 0; } void diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h index e2aca35452c0..f975429b8705 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -48,10 +49,10 @@ enum mipi_port_id ia_css_isys_port_to_mipi_port( * @param[in] port CSI port * @param[in] isys_stream_id Stream handle generated with ia_css_isys_generate_stream_id() * Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES - * @return IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if + * @return 0 if successful, -EINVAL if * there is already a stream registered with the same handle */ -enum ia_css_err ia_css_isys_csi_rx_register_stream( +int ia_css_isys_csi_rx_register_stream( enum mipi_port_id port, uint32_t isys_stream_id); @@ -63,14 +64,14 @@ enum ia_css_err ia_css_isys_csi_rx_register_stream( * @param[in] port CSI port * @param[in] isys_stream_id Stream handle generated with ia_css_isys_generate_stream_id() * Must be lower than SH_CSS_MAX_ISYS_CHANNEL_NODES - * @return IA_CSS_SUCCESS if successful, IA_CSS_ERR_INTERNAL_ERROR if + * @return 0 if successful, -EINVAL if * there is no stream registered with that handle */ -enum ia_css_err ia_css_isys_csi_rx_unregister_stream( +int ia_css_isys_csi_rx_unregister_stream( enum mipi_port_id port, uint32_t isys_stream_id); -enum ia_css_err ia_css_isys_convert_compressed_format( +int ia_css_isys_convert_compressed_format( struct ia_css_csi2_compression *comp, struct input_system_cfg_s *cfg); unsigned int ia_css_csi2_calculate_input_system_alignment( @@ -107,7 +108,7 @@ unsigned int ia_css_isys_rx_translate_irq_infos(unsigned int bits); * This is normally done by the sensor, but when using the input fifo, this * format type must be sumitted correctly by the application. */ -enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format( +int ia_css_isys_convert_stream_format_to_mipi_format( enum atomisp_input_format input_format, mipi_predictor_t compression, unsigned int *fmt_type); diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h index 6ad7a0cd5146..6f1a86c81d7c 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/interface/ia_css_isys_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c index 8f2ce2c057eb..5a44d8f6c196 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -126,11 +127,11 @@ void ia_css_isys_csi_rx_lut_rmgr_release( } } -enum ia_css_err ia_css_isys_csi_rx_register_stream( +int ia_css_isys_csi_rx_register_stream( enum mipi_port_id port, uint32_t isys_stream_id) { - enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR; + int retval = -EINVAL; if ((port < N_INPUT_SYSTEM_CSI_PORT) && (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) { @@ -140,17 +141,17 @@ enum ia_css_err ia_css_isys_csi_rx_register_stream( if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 0) { bitop_setbit(pipe_io_status->active[port], isys_stream_id); pipe_io_status->running[port] = 0; - retval = IA_CSS_SUCCESS; + retval = 0; } } return retval; } -enum ia_css_err ia_css_isys_csi_rx_unregister_stream( +int ia_css_isys_csi_rx_unregister_stream( enum mipi_port_id port, uint32_t isys_stream_id) { - enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR; + int retval = -EINVAL; if ((port < N_INPUT_SYSTEM_CSI_PORT) && (isys_stream_id < SH_CSS_MAX_ISYS_CHANNEL_NODES)) { @@ -159,7 +160,7 @@ enum ia_css_err ia_css_isys_csi_rx_unregister_stream( pipe_io_status = ia_css_pipeline_get_pipe_io_status(); if (bitop_getbit(pipe_io_status->active[port], isys_stream_id) == 1) { bitop_clearbit(pipe_io_status->active[port], isys_stream_id); - retval = IA_CSS_SUCCESS; + retval = 0; } } return retval; diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h index 79d7c4b29bf4..11f730dc1c08 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/csi_rx_rmgr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c index 9055ed387673..9710493c47ac 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h index 7155e2c6e05c..7c754ec7224a 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/ibuf_ctrl_rmgr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c index 930fa7a0ff53..68baec78b1c4 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h index e3d07ac390fc..88c3d5581999 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_dma_rmgr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c index b923233ec5b0..de442f1fa6ba 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_init.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c index 53355a55d05d..bc4a2ff3c0fc 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h index b55cf02c8bce..78a4c867fb1b 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/isys_stream2mmio_rmgr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c index 43665ddff8ea..4f0dcdfa13be 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/rx.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -216,7 +217,7 @@ void ia_css_isys_rx_clear_irq_info(enum mipi_port_id port, } #endif /* #if !defined(USE_INPUT_SYSTEM_VERSION_2401) */ -enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format( +int ia_css_isys_convert_stream_format_to_mipi_format( enum atomisp_input_format input_format, mipi_predictor_t compression, unsigned int *fmt_type) @@ -254,9 +255,9 @@ enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format( *fmt_type = 16; break; default: - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } - return IA_CSS_SUCCESS; + return 0; } /* * This mapping comes from the Arasan CSS function spec @@ -356,9 +357,9 @@ enum ia_css_err ia_css_isys_convert_stream_format_to_mipi_format( case ATOMISP_INPUT_FORMAT_YUV420_16: case ATOMISP_INPUT_FORMAT_YUV422_16: default: - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } - return IA_CSS_SUCCESS; + return 0; } #if defined(USE_INPUT_SYSTEM_VERSION_2401) @@ -379,11 +380,11 @@ static mipi_predictor_t sh_css_csi2_compression_type_2_mipi_predictor( return predictor; } -enum ia_css_err ia_css_isys_convert_compressed_format( +int ia_css_isys_convert_compressed_format( struct ia_css_csi2_compression *comp, struct input_system_cfg_s *cfg) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(comp); assert(cfg); @@ -414,7 +415,7 @@ enum ia_css_err ia_css_isys_convert_compressed_format( cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_10_8_10; break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } } else if (comp->uncompressed_bits_per_pixel == UNCOMPRESSED_BITS_PER_PIXEL_12) { @@ -429,10 +430,10 @@ enum ia_css_err ia_css_isys_convert_compressed_format( cfg->csi_port_attr.comp_scheme = MIPI_COMPRESSOR_12_8_12; break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } } else - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; cfg->csi_port_attr.comp_predictor = sh_css_csi2_compression_type_2_mipi_predictor(comp->type); cfg->csi_port_attr.comp_enable = true; diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c index 9a795a21d3e6..b3c6831cb9e3 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,6 +13,8 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ + #include "system_global.h" #ifdef USE_INPUT_SYSTEM_VERSION_2401 @@ -19,7 +22,6 @@ #include "ia_css_isys.h" #include "ia_css_debug.h" #include "math_support.h" -#include "string_support.h" #include "virtual_isys.h" #include "isp.h" #include "sh_css_defs.h" @@ -649,14 +651,8 @@ static bool calculate_tpg_cfg( input_system_cfg_t *isys_cfg, pixelgen_tpg_cfg_t *cfg) { - (void)channel; - (void)input_port; + memcpy(cfg, &isys_cfg->tpg_port_attr, sizeof(pixelgen_tpg_cfg_t)); - memcpy_s( - (void *)cfg, - sizeof(pixelgen_tpg_cfg_t), - (void *)(&isys_cfg->tpg_port_attr), - sizeof(pixelgen_tpg_cfg_t)); return true; } @@ -666,14 +662,8 @@ static bool calculate_prbs_cfg( input_system_cfg_t *isys_cfg, pixelgen_prbs_cfg_t *cfg) { - (void)channel; - (void)input_port; + memcpy(cfg, &isys_cfg->prbs_port_attr, sizeof(pixelgen_prbs_cfg_t)); - memcpy_s( - (void *)cfg, - sizeof(pixelgen_prbs_cfg_t), - (void *)(&isys_cfg->prbs_port_attr), - sizeof(pixelgen_prbs_cfg_t)); return true; } @@ -691,12 +681,10 @@ static bool calculate_be_cfg( bool metadata, csi_rx_backend_cfg_t *cfg) { - memcpy_s( - (void *)(&cfg->lut_entry), - sizeof(csi_rx_backend_lut_entry_t), - metadata ? (void *)(&input_port->metadata.backend_lut_entry) : - (void *)(&input_port->csi_rx.backend_lut_entry), - sizeof(csi_rx_backend_lut_entry_t)); + memcpy(&cfg->lut_entry, + metadata ? &input_port->metadata.backend_lut_entry : + &input_port->csi_rx.backend_lut_entry, + sizeof(csi_rx_backend_lut_entry_t)); cfg->csi_mipi_cfg.virtual_channel = isys_cfg->csi_port_attr.ch_id; if (metadata) { diff --git a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h index b675907791ad..fbdbca0cfcc8 100644 --- a/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h +++ b/drivers/staging/media/atomisp/pci/runtime/isys/src/virtual_isys.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h index 6a41efee5635..18a7d18e197e 100644 --- a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h +++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -56,8 +57,7 @@ struct ia_css_pipeline { u32 pipe_qos_config; }; -#define DEFAULT_PIPELINE \ -(struct ia_css_pipeline) { \ +#define DEFAULT_PIPELINE { \ .pipe_id = IA_CSS_PIPE_ID_PREVIEW, \ .in_frame = DEFAULT_FRAME, \ .out_frame = {DEFAULT_FRAME}, \ @@ -94,13 +94,13 @@ void ia_css_pipeline_init(void); * @param[out] pipeline structure to be initialized with defaults * @param[in] pipe_id * @param[in] pipe_num Number that uniquely identifies a pipeline. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * Initializes the pipeline structure with a set of default values. * This API is expected to be used when a pipeline structure is allocated * externally and needs sane defaults */ -enum ia_css_err ia_css_pipeline_create( +int ia_css_pipeline_create( struct ia_css_pipeline *pipeline, enum ia_css_pipe_id pipe_id, unsigned int pipe_num, @@ -127,10 +127,10 @@ void ia_css_pipeline_start(enum ia_css_pipe_id pipe_id, /* @brief Request to stop a pipeline * * @param[in] pipeline - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline); +int ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline); /* @brief Check whether pipeline has stopped * @@ -153,13 +153,13 @@ void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline); * @param pipeline Pointer to the pipeline to be added to. * @param[in] stage_desc The description of the stage * @param[out] stage The successor of the stage. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * Add a new stage to a non-NULL pipeline. * The stage consists of an ISP binary or firmware and input and output * arguments. */ -enum ia_css_err ia_css_pipeline_create_and_add_stage( +int ia_css_pipeline_create_and_add_stage( struct ia_css_pipeline *pipeline, struct ia_css_pipeline_stage_desc *stage_desc, struct ia_css_pipeline_stage **stage); @@ -177,10 +177,10 @@ void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline, /* @brief gets a stage from the pipeline * * @param[in] pipeline - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, +int ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, int mode, struct ia_css_pipeline_stage **stage); @@ -190,10 +190,10 @@ enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, * @param[in] fw_handle * @param[out] stage Pointer to Stage * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline +int ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline *pipeline, u32 fw_handle, struct ia_css_pipeline_stage **stage); @@ -204,10 +204,10 @@ enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline * @param[in] stage_num * @param[out] fw_handle * - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline +int ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline *pipeline, u32 stage_num, uint32_t *fw_handle); @@ -215,10 +215,10 @@ enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline /* @brief gets the output stage from the pipeline * * @param[in] pipeline - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * */ -enum ia_css_err ia_css_pipeline_get_output_stage( +int ia_css_pipeline_get_output_stage( struct ia_css_pipeline *pipeline, int mode, struct ia_css_pipeline_stage **stage); diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h index b96a5b146096..cc44f03c3b34 100644 --- a/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h +++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/interface/ia_css_pipeline_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c index 8b9982de8deb..4b8e85bc2122 100644 --- a/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c +++ b/drivers/staging/media/atomisp/pci/runtime/pipeline/src/pipeline.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -12,11 +13,12 @@ * more details. */ +#include "hmm.h" + #include "ia_css_debug.h" #include "sw_event_global.h" /* encode_sw_event */ #include "sp.h" /* cnd_sp_irq_enable() */ #include "assert_support.h" -#include "memory_access.h" #include "sh_css_sp.h" #include "ia_css_pipeline.h" #include "ia_css_isp_param.h" @@ -45,7 +47,7 @@ static void pipeline_init_defaults( unsigned int dvs_frame_delay); static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage); -static enum ia_css_err pipeline_stage_create( +static int pipeline_stage_create( struct ia_css_pipeline_stage_desc *stage_desc, struct ia_css_pipeline_stage **new_stage); static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline); @@ -60,7 +62,7 @@ void ia_css_pipeline_init(void) pipeline_init_sp_thread_map(); } -enum ia_css_err ia_css_pipeline_create( +int ia_css_pipeline_create( struct ia_css_pipeline *pipeline, enum ia_css_pipe_id pipe_id, unsigned int pipe_num, @@ -70,14 +72,14 @@ enum ia_css_err ia_css_pipeline_create( IA_CSS_ENTER_PRIVATE("pipeline = %p, pipe_id = %d, pipe_num = %d, dvs_frame_delay = %d", pipeline, pipe_id, pipe_num, dvs_frame_delay); if (!pipeline) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } pipeline_init_defaults(pipeline, pipe_id, pipe_num, dvs_frame_delay); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } void ia_css_pipeline_map(unsigned int pipe_num, bool map) @@ -195,15 +197,15 @@ void ia_css_pipeline_dump_thread_map_info(void) } } -enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline) +int ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int thread_id; assert(pipeline); if (!pipeline) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipeline_request_stop() enter: pipeline=%p\n", @@ -218,7 +220,7 @@ enum ia_css_err ia_css_pipeline_request_stop(struct ia_css_pipeline *pipeline) ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipeline_request_stop() leaving\n"); /* queues are invalid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_STOP_STREAM, (uint8_t)thread_id, @@ -263,19 +265,19 @@ void ia_css_pipeline_clean(struct ia_css_pipeline *pipeline) * @param pipeline Pointer to the pipeline to be added to. * @param[in] stage_desc The description of the stage * @param[out] stage The successor of the stage. - * @return IA_CSS_SUCCESS or error code upon error. + * @return 0 or error code upon error. * * Add a new stage to a non-NULL pipeline. * The stage consists of an ISP binary or firmware and input and * output arguments. */ -enum ia_css_err ia_css_pipeline_create_and_add_stage( +int ia_css_pipeline_create_and_add_stage( struct ia_css_pipeline *pipeline, struct ia_css_pipeline_stage_desc *stage_desc, struct ia_css_pipeline_stage **stage) { struct ia_css_pipeline_stage *last, *new_stage = NULL; - enum ia_css_err err; + int err; /* other arguments can be NULL */ assert(pipeline); @@ -289,7 +291,7 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage( ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipeline_create_and_add_stage() done: Invalid args\n"); - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } /* Find the last stage */ @@ -308,12 +310,12 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage( stage_desc->in_frame = last->args.out_frame[0]; if (!stage_desc->in_frame) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } /* Create the new stage */ err = pipeline_stage_create(stage_desc, &new_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipeline_create_and_add_stage() done: stage_create_failed\n"); return err; @@ -330,7 +332,7 @@ enum ia_css_err ia_css_pipeline_create_and_add_stage( ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipeline_create_and_add_stage() done:\n"); - return IA_CSS_SUCCESS; + return 0; } void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline, @@ -350,7 +352,7 @@ void ia_css_pipeline_finalize_stages(struct ia_css_pipeline *pipeline, ia_css_pipeline_configure_inout_port(pipeline, continuous); } -enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, +int ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, int mode, struct ia_css_pipeline_stage **stage) { @@ -363,13 +365,13 @@ enum ia_css_err ia_css_pipeline_get_stage(struct ia_css_pipeline *pipeline, for (s = pipeline->stages; s; s = s->next) { if (s->mode == mode) { *stage = s; - return IA_CSS_SUCCESS; + return 0; } } - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } -enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline +int ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline *pipeline, u32 fw_handle, struct ia_css_pipeline_stage **stage) @@ -382,13 +384,13 @@ enum ia_css_err ia_css_pipeline_get_stage_from_fw(struct ia_css_pipeline for (s = pipeline->stages; s; s = s->next) { if ((s->firmware) && (s->firmware->handle == fw_handle)) { *stage = s; - return IA_CSS_SUCCESS; + return 0; } } - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } -enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline +int ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline *pipeline, u32 stage_num, uint32_t *fw_handle) @@ -397,18 +399,18 @@ enum ia_css_err ia_css_pipeline_get_fw_from_stage(struct ia_css_pipeline ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "%s()\n", __func__); if ((!pipeline) || (!fw_handle)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; for (s = pipeline->stages; s; s = s->next) { if ((s->stage_num == stage_num) && (s->firmware)) { *fw_handle = s->firmware->handle; - return IA_CSS_SUCCESS; + return 0; } } - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } -enum ia_css_err ia_css_pipeline_get_output_stage( +int ia_css_pipeline_get_output_stage( struct ia_css_pipeline *pipeline, int mode, struct ia_css_pipeline_stage **stage) @@ -428,7 +430,7 @@ enum ia_css_err ia_css_pipeline_get_output_stage( *stage = s; } if (*stage) - return IA_CSS_SUCCESS; + return 0; /* If no firmware, find binary in pipe */ return ia_css_pipeline_get_stage(pipeline, mode, stage); } @@ -511,7 +513,7 @@ static void pipeline_stage_destroy(struct ia_css_pipeline_stage *stage) ia_css_frame_free(stage->args.out_vf_frame); stage->args.out_vf_frame = NULL; } - sh_css_free(stage); + kvfree(stage); } static void pipeline_init_sp_thread_map(void) @@ -566,11 +568,11 @@ static void pipeline_unmap_num_to_sp_thread(unsigned int pipe_num) pipeline_sp_thread_list[thread_id] = PIPELINE_SP_THREAD_EMPTY_TOKEN; } -static enum ia_css_err pipeline_stage_create( +static int pipeline_stage_create( struct ia_css_pipeline_stage_desc *stage_desc, struct ia_css_pipeline_stage **new_stage) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipeline_stage *stage = NULL; struct ia_css_binary *binary; struct ia_css_frame *vf_frame; @@ -581,7 +583,7 @@ static enum ia_css_err pipeline_stage_create( /* Verify input parameters*/ if (!(stage_desc->in_frame) && !(stage_desc->firmware) && (stage_desc->binary) && !(stage_desc->binary->online)) { - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto ERR; } @@ -592,12 +594,11 @@ static enum ia_css_err pipeline_stage_create( out_frame[i] = stage_desc->out_frame[i]; } - stage = sh_css_malloc(sizeof(*stage)); + stage = kvzalloc(sizeof(*stage), GFP_KERNEL); if (!stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } - memset(stage, 0, sizeof(*stage)); if (firmware) { stage->binary = NULL; @@ -627,7 +628,7 @@ static enum ia_css_err pipeline_stage_create( && (binary->out_frame_info[i].res.width)) { err = ia_css_frame_allocate_from_info(&out_frame[i], &binary->out_frame_info[i]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; stage->out_frame_allocated[i] = true; } @@ -641,7 +642,7 @@ static enum ia_css_err pipeline_stage_create( ) { err = ia_css_frame_allocate_from_info(&vf_frame, &binary->vf_frame_info); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; stage->vf_frame_allocated = true; } @@ -664,6 +665,8 @@ ERR: return err; } +static const struct ia_css_frame ia_css_default_frame = DEFAULT_FRAME; + static void pipeline_init_defaults( struct ia_css_pipeline *pipeline, enum ia_css_pipe_id pipe_id, @@ -676,10 +679,15 @@ static void pipeline_init_defaults( pipeline->stages = NULL; pipeline->stop_requested = false; pipeline->current_stage = NULL; - pipeline->in_frame = DEFAULT_FRAME; + + memcpy(&pipeline->in_frame, &ia_css_default_frame, + sizeof(ia_css_default_frame)); + for (i = 0; i < IA_CSS_PIPE_MAX_OUTPUT_STAGE; i++) { - pipeline->out_frame[i] = DEFAULT_FRAME; - pipeline->vf_frame[i] = DEFAULT_FRAME; + memcpy(&pipeline->out_frame[i], &ia_css_default_frame, + sizeof(ia_css_default_frame)); + memcpy(&pipeline->vf_frame[i], &ia_css_default_frame, + sizeof(ia_css_default_frame)); } pipeline->num_execs = -1; pipeline->acquire_isp_each_stage = true; @@ -690,30 +698,30 @@ static void pipeline_init_defaults( static void ia_css_pipeline_set_zoom_stage(struct ia_css_pipeline *pipeline) { struct ia_css_pipeline_stage *stage = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(pipeline); if (pipeline->pipe_id == IA_CSS_PIPE_ID_PREVIEW) { /* in preview pipeline, vf_pp stage should do zoom */ err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VF_PP, &stage); - if (err == IA_CSS_SUCCESS) + if (!err) stage->enable_zoom = true; } else if (pipeline->pipe_id == IA_CSS_PIPE_ID_CAPTURE) { /* in capture pipeline, capture_pp stage should do zoom */ err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage); - if (err == IA_CSS_SUCCESS) + if (!err) stage->enable_zoom = true; } else if (pipeline->pipe_id == IA_CSS_PIPE_ID_VIDEO) { /* in video pipeline, video stage should do zoom */ err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_VIDEO, &stage); - if (err == IA_CSS_SUCCESS) + if (!err) stage->enable_zoom = true; } else if (pipeline->pipe_id == IA_CSS_PIPE_ID_YUVPP) { /* in yuvpp pipeline, first yuv_scaler stage should do zoom */ err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage); - if (err == IA_CSS_SUCCESS) + if (!err) stage->enable_zoom = true; } } diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h index 6daeb060daf9..08112be4633f 100644 --- a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h +++ b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -40,7 +41,7 @@ typedef struct ia_css_queue ia_css_queue_t; * @param[out] qhandle. Handle to queue instance for use with API * @param[in] desc. Descriptor with queue properties filled-in * @return 0 - Successful init of local queue instance. - * @return EINVAL - Invalid argument. + * @return -EINVAL - Invalid argument. * */ int ia_css_queue_local_init( @@ -52,7 +53,7 @@ int ia_css_queue_local_init( * @param[out] qhandle. Handle to queue instance for use with API * @param[in] desc. Descriptor with queue properties filled-in * @return 0 - Successful init of remote queue instance. - * @return EINVAL - Invalid argument. + * @return -EINVAL - Invalid argument. */ int ia_css_queue_remote_init( ia_css_queue_t *qhandle, @@ -72,8 +73,8 @@ int ia_css_queue_uninit( * @param[in] qhandle. Handle to queue instance * @param[in] item. Object to be enqueued. * @return 0 - Successful enqueue. - * @return EINVAL - Invalid argument. - * @return ENOBUFS - Queue is full. + * @return -EINVAL - Invalid argument. + * @return -ENOBUFS - Queue is full. * */ int ia_css_queue_enqueue( @@ -86,8 +87,8 @@ int ia_css_queue_enqueue( * @param[out] item. Object to be dequeued into this item. * @return 0 - Successful dequeue. - * @return EINVAL - Invalid argument. - * @return ENODATA - Queue is empty. + * @return -EINVAL - Invalid argument. + * @return -ENODATA - Queue is empty. * */ int ia_css_queue_dequeue( @@ -99,8 +100,8 @@ int ia_css_queue_dequeue( * @param[in] qhandle. Handle to queue instance * @param[in] is_empty True if empty, False if not. * @return 0 - Successful access state. - * @return EINVAL - Invalid argument. - * @return ENOSYS - Function not implemented. + * @return -EINVAL - Invalid argument. + * @return -ENOSYS - Function not implemented. * */ int ia_css_queue_is_empty( @@ -112,8 +113,8 @@ int ia_css_queue_is_empty( * @param[in] qhandle. Handle to queue instance * @param[in] is_full True if Full, False if not. * @return 0 - Successfully access state. - * @return EINVAL - Invalid argument. - * @return ENOSYS - Function not implemented. + * @return -EINVAL - Invalid argument. + * @return -ENOSYS - Function not implemented. * */ int ia_css_queue_is_full( @@ -125,7 +126,7 @@ int ia_css_queue_is_full( * @param[in] qhandle. Handle to queue instance * @param[in] size Number of available elements in the queue * @return 0 - Successfully access state. - * @return EINVAL - Invalid argument. + * @return -EINVAL - Invalid argument. * */ int ia_css_queue_get_used_space( @@ -137,7 +138,7 @@ int ia_css_queue_get_used_space( * @param[in] qhandle. Handle to queue instance * @param[in] size Number of free elements in the queue * @return 0 - Successfully access state. - * @return EINVAL - Invalid argument. + * @return -EINVAL - Invalid argument. * */ int ia_css_queue_get_free_space( @@ -151,7 +152,7 @@ int ia_css_queue_get_free_space( * starting from head of queue * @param[in] element Value of element returned * @return 0 - Successfully access state. - * @return EINVAL - Invalid argument. + * @return -EINVAL - Invalid argument. * */ int ia_css_queue_peek( @@ -164,8 +165,8 @@ int ia_css_queue_peek( * @param[in] qhandle. Handle to queue instance * @param[out] size Size value to be returned here. * @return 0 - Successful get size. - * @return EINVAL - Invalid argument. - * @return ENOSYS - Function not implemented. + * @return -EINVAL - Invalid argument. + * @return -ENOSYS - Function not implemented. * */ int ia_css_queue_get_size( diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h index 87fa4288d9a6..1379ae8f8c01 100644 --- a/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h +++ b/drivers/staging/media/atomisp/pci/runtime/queue/interface/ia_css_queue_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c index dd79c6f180af..aea6c66a3cee 100644 --- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c +++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -28,7 +29,7 @@ int ia_css_queue_local_init( if (NULL == qhandle || NULL == desc || NULL == desc->cb_elems || NULL == desc->cb_desc) { /* Invalid parameters, return error*/ - return EINVAL; + return -EINVAL; } /* Mark the queue as Local */ @@ -48,7 +49,7 @@ int ia_css_queue_remote_init( { if (NULL == qhandle || NULL == desc) { /* Invalid parameters, return error*/ - return EINVAL; + return -EINVAL; } /* Mark the queue as remote*/ @@ -72,7 +73,7 @@ int ia_css_queue_uninit( ia_css_queue_t *qhandle) { if (!qhandle) - return EINVAL; + return -EINVAL; /* Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -90,7 +91,7 @@ int ia_css_queue_enqueue( int error = 0; if (!qhandle) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -99,7 +100,7 @@ int ia_css_queue_enqueue( */ if (ia_css_circbuf_is_full(&qhandle->desc.cb_local)) { /* Cannot push the element. Return*/ - return ENOBUFS; + return -ENOBUFS; } /* Push the element*/ @@ -117,7 +118,7 @@ int ia_css_queue_enqueue( /* b. Operate on the queue */ if (ia_css_circbuf_desc_is_full(&cb_desc)) - return ENOBUFS; + return -ENOBUFS; cb_elem.val = item; @@ -149,7 +150,7 @@ int ia_css_queue_dequeue( int error = 0; if (!qhandle || NULL == item) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -158,7 +159,7 @@ int ia_css_queue_dequeue( */ if (ia_css_circbuf_is_empty(&qhandle->desc.cb_local)) { /* Nothing to pop. Return empty queue*/ - return ENODATA; + return -ENODATA; } *item = ia_css_circbuf_pop(&qhandle->desc.cb_local); @@ -176,7 +177,7 @@ int ia_css_queue_dequeue( /* b. Operate on the queue */ if (ia_css_circbuf_desc_is_empty(&cb_desc)) - return ENODATA; + return -ENODATA; error = ia_css_queue_item_load(qhandle, cb_desc.start, &cb_elem); if (error != 0) @@ -206,7 +207,7 @@ int ia_css_queue_is_full( int error = 0; if ((!qhandle) || (!is_full)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -230,7 +231,7 @@ int ia_css_queue_is_full( return 0; } - return EINVAL; + return -EINVAL; } int ia_css_queue_get_free_space( @@ -240,7 +241,7 @@ int ia_css_queue_get_free_space( int error = 0; if ((!qhandle) || (!size)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -264,7 +265,7 @@ int ia_css_queue_get_free_space( return 0; } - return EINVAL; + return -EINVAL; } int ia_css_queue_get_used_space( @@ -274,7 +275,7 @@ int ia_css_queue_get_used_space( int error = 0; if ((!qhandle) || (!size)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -298,7 +299,7 @@ int ia_css_queue_get_used_space( return 0; } - return EINVAL; + return -EINVAL; } int ia_css_queue_peek( @@ -310,7 +311,7 @@ int ia_css_queue_peek( int error = 0; if ((!qhandle) || (!element)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -320,7 +321,7 @@ int ia_css_queue_peek( /* Check if offset is valid */ num_elems = ia_css_circbuf_get_num_elems(&qhandle->desc.cb_local); if (offset > num_elems) - return EINVAL; + return -EINVAL; *element = ia_css_circbuf_peek_from_start(&qhandle->desc.cb_local, (int)offset); return 0; @@ -339,7 +340,7 @@ int ia_css_queue_peek( /* Check if offset is valid */ num_elems = ia_css_circbuf_desc_get_num_elems(&cb_desc); if (offset > num_elems) - return EINVAL; + return -EINVAL; offset = OP_std_modadd(cb_desc.start, offset, cb_desc.size); error = ia_css_queue_item_load(qhandle, (uint8_t)offset, &cb_elem); @@ -350,7 +351,7 @@ int ia_css_queue_peek( return 0; } - return EINVAL; + return -EINVAL; } int ia_css_queue_is_empty( @@ -360,7 +361,7 @@ int ia_css_queue_is_empty( int error = 0; if ((!qhandle) || (!is_empty)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { @@ -384,7 +385,7 @@ int ia_css_queue_is_empty( return 0; } - return EINVAL; + return -EINVAL; } int ia_css_queue_get_size( @@ -394,7 +395,7 @@ int ia_css_queue_get_size( int error = 0; if ((!qhandle) || (!size)) - return EINVAL; + return -EINVAL; /* 1. Load the required queue object */ if (qhandle->type == IA_CSS_QUEUE_TYPE_LOCAL) { diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c index 1e8d3eb82eab..fdca743c4ab7 100644 --- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c +++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -12,11 +13,12 @@ * more details. */ +#include "hmm.h" + #include "type_support.h" #include "queue_access.h" #include "ia_css_circbuf.h" #include "sp.h" -#include "memory_access.h" #include "assert_support.h" int ia_css_queue_load( @@ -25,7 +27,7 @@ int ia_css_queue_load( uint32_t ignore_desc_flags) { if (!rdesc || !cb_desc) - return EINVAL; + return -EINVAL; if (rdesc->location == IA_CSS_QUEUE_LOC_SP) { assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX); @@ -63,12 +65,12 @@ int ia_css_queue_load( } else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) { /* doing DMA transfer of entire structure */ - mmgr_load(rdesc->desc.remote.cb_desc_addr, + hmm_load(rdesc->desc.remote.cb_desc_addr, (void *)cb_desc, sizeof(ia_css_circbuf_desc_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) { /* Not supported yet */ - return ENOTSUP; + return -ENOTSUPP; } return 0; @@ -80,7 +82,7 @@ int ia_css_queue_store( uint32_t ignore_desc_flags) { if (!rdesc || !cb_desc) - return EINVAL; + return -EINVAL; if (rdesc->location == IA_CSS_QUEUE_LOC_SP) { assert(ignore_desc_flags <= QUEUE_IGNORE_DESC_FLAGS_MAX); @@ -110,12 +112,12 @@ int ia_css_queue_store( cb_desc->step); } else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) { /* doing DMA transfer of entire structure */ - mmgr_store(rdesc->desc.remote.cb_desc_addr, + hmm_store(rdesc->desc.remote.cb_desc_addr, (void *)cb_desc, sizeof(ia_css_circbuf_desc_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) { /* Not supported yet */ - return ENOTSUP; + return -ENOTSUPP; } return 0; @@ -127,7 +129,7 @@ int ia_css_queue_item_load( ia_css_circbuf_elem_t *item) { if (!rdesc || !item) - return EINVAL; + return -EINVAL; if (rdesc->location == IA_CSS_QUEUE_LOC_SP) { sp_dmem_load(rdesc->proc_id, @@ -136,13 +138,13 @@ int ia_css_queue_item_load( item, sizeof(ia_css_circbuf_elem_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) { - mmgr_load(rdesc->desc.remote.cb_elems_addr + hmm_load(rdesc->desc.remote.cb_elems_addr + position * sizeof(ia_css_circbuf_elem_t), (void *)item, sizeof(ia_css_circbuf_elem_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) { /* Not supported yet */ - return ENOTSUP; + return -ENOTSUPP; } return 0; @@ -154,7 +156,7 @@ int ia_css_queue_item_store( ia_css_circbuf_elem_t *item) { if (!rdesc || !item) - return EINVAL; + return -EINVAL; if (rdesc->location == IA_CSS_QUEUE_LOC_SP) { sp_dmem_store(rdesc->proc_id, @@ -163,13 +165,13 @@ int ia_css_queue_item_store( item, sizeof(ia_css_circbuf_elem_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_HOST) { - mmgr_store(rdesc->desc.remote.cb_elems_addr + hmm_store(rdesc->desc.remote.cb_elems_addr + position * sizeof(ia_css_circbuf_elem_t), (void *)item, sizeof(ia_css_circbuf_elem_t)); } else if (rdesc->location == IA_CSS_QUEUE_LOC_ISP) { /* Not supported yet */ - return ENOTSUP; + return -ENOTSUPP; } return 0; diff --git a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h index 884c55a754d1..d5107adccef9 100644 --- a/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h +++ b/drivers/staging/media/atomisp/pci/runtime/queue/src/queue_access.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -15,10 +16,11 @@ #ifndef __QUEUE_ACCESS_H #define __QUEUE_ACCESS_H +#include <linux/errno.h> + #include <type_support.h> #include <ia_css_queue_comm.h> #include <ia_css_circbuf.h> -#include <error_support.h> #define QUEUE_IGNORE_START_FLAG 0x0001 #define QUEUE_IGNORE_END_FLAG 0x0002 diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h index 47a80ae8dbf3..9cd3d92b34c9 100644 --- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h +++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -28,7 +29,7 @@ /** * @brief Initialize resource manager (host/common) */ -enum ia_css_err ia_css_rmgr_init(void); +int ia_css_rmgr_init(void); /** * @brief Uninitialize resource manager (host/common) diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h index 0660b65f2e34..ac969afc8bb4 100644 --- a/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h +++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/interface/ia_css_rmgr_vbuf.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -17,13 +18,14 @@ #include "ia_css_rmgr.h" #include <type_support.h> -#include <system_types.h> +#include <ia_css_types.h> +#include <system_local.h> /** * @brief Data structure for the resource handle (host, vbuf) */ struct ia_css_rmgr_vbuf_handle { - hrt_vaddress vptr; + ia_css_ptr vptr; u8 count; u32 size; }; @@ -51,7 +53,7 @@ extern struct ia_css_rmgr_vbuf_pool *hmm_buffer_pool; * * @param pool The pointer to the pool */ -STORAGE_CLASS_RMGR_H enum ia_css_err ia_css_rmgr_init_vbuf( +STORAGE_CLASS_RMGR_H int ia_css_rmgr_init_vbuf( struct ia_css_rmgr_vbuf_pool *pool); /** diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c index 23ae19ee65ca..c94a428aadde 100644 --- a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c +++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -14,16 +15,16 @@ #include "ia_css_rmgr.h" -enum ia_css_err ia_css_rmgr_init(void) +int ia_css_rmgr_init(void) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; err = ia_css_rmgr_init_vbuf(vbuf_ref); - if (err == IA_CSS_SUCCESS) + if (!err) err = ia_css_rmgr_init_vbuf(vbuf_write); - if (err == IA_CSS_SUCCESS) + if (!err) err = ia_css_rmgr_init_vbuf(hmm_buffer_pool); - if (err != IA_CSS_SUCCESS) + if (err) ia_css_rmgr_uninit(); return err; } diff --git a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c index 2c204dceb491..1ea74296fc8d 100644 --- a/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c +++ b/drivers/staging/media/atomisp/pci/runtime/rmgr/src/rmgr_vbuf.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010-2015, Intel Corporation. @@ -12,12 +13,12 @@ * more details. */ +#include "hmm.h" #include "ia_css_rmgr.h" #include <type_support.h> #include <assert_support.h> #include <platform_support.h> /* memset */ -#include <memory_access.h> /* mmmgr_malloc, mhmm_free */ #include <ia_css_debug.h> /* @@ -137,26 +138,26 @@ void ia_css_rmgr_refcount_release_vbuf(struct ia_css_rmgr_vbuf_handle **handle) * * @param pool The pointer to the pool */ -enum ia_css_err ia_css_rmgr_init_vbuf(struct ia_css_rmgr_vbuf_pool *pool) +int ia_css_rmgr_init_vbuf(struct ia_css_rmgr_vbuf_pool *pool) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; size_t bytes_needed; rmgr_refcount_init_vbuf(); assert(pool); if (!pool) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* initialize the recycle pool if used */ if (pool->recycle && pool->size) { /* allocate memory for storing the handles */ bytes_needed = sizeof(void *) * pool->size; - pool->handles = sh_css_malloc(bytes_needed); + pool->handles = kvmalloc(bytes_needed, GFP_KERNEL); if (pool->handles) memset(pool->handles, 0, bytes_needed); else - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; } else { /* just in case, set the size to 0 */ pool->size = 0; @@ -196,7 +197,7 @@ void ia_css_rmgr_uninit_vbuf(struct ia_css_rmgr_vbuf_pool *pool) } } /* now free the pool handles list */ - sh_css_free(pool->handles); + kvfree(pool->handles); pool->handles = NULL; } } @@ -297,7 +298,7 @@ void ia_css_rmgr_acq_vbuf(struct ia_css_rmgr_vbuf_pool *pool, } if ((*handle)->vptr == 0x0) { /* we need to allocate */ - (*handle)->vptr = mmgr_malloc((*handle)->size); + (*handle)->vptr = hmm_alloc((*handle)->size, HMM_BO_PRIVATE, 0, NULL, 0); } else { /* we popped a buffer */ return; diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h index 543ca8968418..efe6c4a82caf 100644 --- a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h +++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -34,11 +35,11 @@ typedef struct { } ia_css_spctrl_cfg; /* Get the code addr in DDR of SP */ -hrt_vaddress get_sp_code_addr(sp_ID_t sp_id); +ia_css_ptr get_sp_code_addr(sp_ID_t sp_id); /* ! Load firmware on to specfied SP */ -enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id, +int ia_css_spctrl_load_fw(sp_ID_t sp_id, ia_css_spctrl_cfg *spctrl_cfg); /* ISP2401 */ @@ -47,15 +48,15 @@ void sh_css_spctrl_reload_fw(sp_ID_t sp_id); /*! Unload/release any memory allocated to hold the firmware */ -enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id); +int ia_css_spctrl_unload_fw(sp_ID_t sp_id); /*! Intilaize dmem_cfg in SP dmem and start SP program */ -enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id); +int ia_css_spctrl_start(sp_ID_t sp_id); /*! stop spctrl */ -enum ia_css_err ia_css_spctrl_stop(sp_ID_t sp_id); +int ia_css_spctrl_stop(sp_ID_t sp_id); /*! Query the state of SP */ diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h index ca37c4ab7544..78e0f3096f60 100644 --- a/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h +++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/interface/ia_css_spctrl_comm.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c b/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c index db39fa273251..753a99703f1e 100644 --- a/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c +++ b/drivers/staging/media/atomisp/pci/runtime/spctrl/src/spctrl.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -12,11 +13,12 @@ * more details. */ +#include "hmm.h" + #include "ia_css_types.h" #define __INLINE_SP__ #include "sp.h" -#include "memory_access.h" #include "assert_support.h" #include "ia_css_spctrl.h" #include "ia_css_debug.h" @@ -26,7 +28,7 @@ struct spctrl_context_info { u32 spctrl_config_dmem_addr; /* location of dmem_cfg in SP dmem */ u32 spctrl_state_dmem_addr; unsigned int sp_entry; /* entry function ptr on SP */ - hrt_vaddress code_addr; /* sp firmware location in host mem-DDR*/ + ia_css_ptr code_addr; /* sp firmware location in host mem-DDR*/ u32 code_size; char *program_name; /* used in case of PLATFORM_SIM */ }; @@ -35,14 +37,14 @@ static struct spctrl_context_info spctrl_cofig_info[N_SP_ID]; static bool spctrl_loaded[N_SP_ID] = {0}; /* Load firmware */ -enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id, +int ia_css_spctrl_load_fw(sp_ID_t sp_id, ia_css_spctrl_cfg *spctrl_cfg) { - hrt_vaddress code_addr = mmgr_NULL; + ia_css_ptr code_addr = mmgr_NULL; struct ia_css_sp_init_dmem_cfg *init_dmem_cfg; if ((sp_id >= N_SP_ID) || (!spctrl_cfg)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; spctrl_cofig_info[sp_id].code_addr = mmgr_NULL; @@ -63,17 +65,17 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id, * Data used to be stored separately, because of access alignment constraints, * fix the FW generation instead */ - code_addr = mmgr_malloc(spctrl_cfg->code_size); + code_addr = hmm_alloc(spctrl_cfg->code_size, HMM_BO_PRIVATE, 0, NULL, 0); if (code_addr == mmgr_NULL) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - mmgr_store(code_addr, spctrl_cfg->code, spctrl_cfg->code_size); + return -ENOMEM; + hmm_store(code_addr, spctrl_cfg->code, spctrl_cfg->code_size); - if (sizeof(hrt_vaddress) > sizeof(hrt_data)) { + if (sizeof(ia_css_ptr) > sizeof(hrt_data)) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, - "size of hrt_vaddress can not be greater than hrt_data\n"); + "size of ia_css_ptr can not be greater than hrt_data\n"); hmm_free(code_addr); code_addr = mmgr_NULL; - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } init_dmem_cfg->ddr_data_addr = code_addr + spctrl_cfg->ddr_data_offset; @@ -82,7 +84,7 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id, "DDR address pointer is not properly aligned for DMA transfer\n"); hmm_free(code_addr); code_addr = mmgr_NULL; - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } spctrl_cofig_info[sp_id].sp_entry = spctrl_cfg->sp_entry; @@ -96,7 +98,7 @@ enum ia_css_err ia_css_spctrl_load_fw(sp_ID_t sp_id, (hrt_data)spctrl_cofig_info[sp_id].code_addr); sp_ctrl_setbit(sp_id, SP_ICACHE_INV_REG, SP_ICACHE_INV_BIT); spctrl_loaded[sp_id] = true; - return IA_CSS_SUCCESS; + return 0; } /* ISP2401 */ @@ -112,15 +114,15 @@ void sh_css_spctrl_reload_fw(sp_ID_t sp_id) spctrl_loaded[sp_id] = true; } -hrt_vaddress get_sp_code_addr(sp_ID_t sp_id) +ia_css_ptr get_sp_code_addr(sp_ID_t sp_id) { return spctrl_cofig_info[sp_id].code_addr; } -enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id) +int ia_css_spctrl_unload_fw(sp_ID_t sp_id) { if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id]))) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* freeup the resource */ if (spctrl_cofig_info[sp_id].code_addr) { @@ -128,14 +130,14 @@ enum ia_css_err ia_css_spctrl_unload_fw(sp_ID_t sp_id) spctrl_cofig_info[sp_id].code_addr = mmgr_NULL; } spctrl_loaded[sp_id] = false; - return IA_CSS_SUCCESS; + return 0; } /* Initialize dmem_cfg in SP dmem and start SP program*/ -enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id) +int ia_css_spctrl_start(sp_ID_t sp_id) { if ((sp_id >= N_SP_ID) || ((sp_id < N_SP_ID) && (!spctrl_loaded[sp_id]))) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* Set descr in the SP to initialize the SP DMEM */ /* @@ -154,7 +156,7 @@ enum ia_css_err ia_css_spctrl_start(sp_ID_t sp_id) (hrt_data)spctrl_cofig_info[sp_id].sp_entry); sp_ctrl_setbit(sp_id, SP_SC_REG, SP_RUN_BIT); sp_ctrl_setbit(sp_id, SP_SC_REG, SP_START_BIT); - return IA_CSS_SUCCESS; + return 0; } /* Query the state of SP1 */ diff --git a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h index 899294646494..49801fbc1924 100644 --- a/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h +++ b/drivers/staging/media/atomisp/pci/runtime/tagger/interface/ia_css_tagger_common.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2010 - 2015, Intel Corporation. @@ -23,11 +24,7 @@ * * Should be one less than NUM_CONTINUOUS_FRAMES in sh_css_internal.h */ -#if defined(HAS_SP_2400) #define MAX_CB_ELEMS_FOR_TAGGER 14 -#else -#define MAX_CB_ELEMS_FOR_TAGGER 9 -#endif /** * @brief Data structure for the tagger buffer element. diff --git a/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c b/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c index 57dddd74d668..679ef8242574 100644 --- a/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c +++ b/drivers/staging/media/atomisp/pci/runtime/timer/src/timer.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -18,14 +19,14 @@ #include "gp_timer.h" /*gp_timer_read()*/ #include "assert_support.h" -enum ia_css_err +int ia_css_timer_get_current_tick( struct ia_css_clock_tick *curr_ts) { assert(curr_ts); if (!curr_ts) { - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } curr_ts->ticks = (clock_value_t)gp_timer_read(GP_TIMER_SEL); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h b/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h index 9b6c2893d950..7e7188797b0a 100644 --- a/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h +++ b/drivers/staging/media/atomisp/pci/scalar_processor_2400_params.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css.c b/drivers/staging/media/atomisp/pci/sh_css.c index d77432254a2c..6676537f0e97 100644 --- a/drivers/staging/media/atomisp/pci/sh_css.c +++ b/drivers/staging/media/atomisp/pci/sh_css.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -17,6 +18,8 @@ #include <linux/slab.h> #include <linux/vmalloc.h> +#include "hmm.h" + #include "ia_css.h" #include "sh_css_hrt.h" /* only for file 2 MIPI */ #include "ia_css_buffer.h" @@ -52,7 +55,6 @@ #include "ia_css_isys.h" #endif -#include "memory_access.h" #include "tag.h" #include "assert_support.h" #include "math_support.h" @@ -74,7 +76,6 @@ #define __INLINE_GPIO__ #include "gpio.h" #include "timed_ctrl.h" -#include "platform_support.h" /* hrt_sleep(), inline */ #include "ia_css_inputfifo.h" #define WITH_PC_MONITORING 0 @@ -181,10 +182,10 @@ static bool fw_explicitly_loaded; * Local prototypes */ -static enum ia_css_err +static int allocate_delay_frames(struct ia_css_pipe *pipe); -static enum ia_css_err +static int sh_css_pipe_start(struct ia_css_stream *stream); /* ISP 2401 */ @@ -195,7 +196,7 @@ sh_css_pipe_start(struct ia_css_stream *stream); * @param[in] stream Point to the target "ia_css_stream" instance. * * @return - * - IA_CSS_SUCCESS, if the "stop" requests have been successfully sent out. + * - 0, if the "stop" requests have been successfully sent out. * - CSS error code, otherwise. * * @@ -205,7 +206,7 @@ sh_css_pipe_start(struct ia_css_stream *stream); * return without waiting for all "ia_css_pipe" instatnces * being stopped. */ -static enum ia_css_err +static int sh_css_pipes_stop(struct ia_css_stream *stream); /* @@ -224,15 +225,15 @@ static bool sh_css_pipes_have_stopped(struct ia_css_stream *stream); /* ISP 2401 */ -static enum ia_css_err +static int ia_css_pipe_check_format(struct ia_css_pipe *pipe, enum ia_css_frame_format format); /* ISP 2401 */ -static enum ia_css_err +static int check_pipe_resolutions(const struct ia_css_pipe *pipe); -static enum ia_css_err +static int ia_css_pipe_load_extension(struct ia_css_pipe *pipe, struct ia_css_fw_info *firmware); @@ -245,7 +246,7 @@ ia_css_reset_defaults(struct sh_css *css); static void sh_css_init_host_sp_control_vars(void); -static enum ia_css_err set_num_primary_stages(unsigned int *num, +static int set_num_primary_stages(unsigned int *num, enum ia_css_pipe_version version); static bool @@ -254,7 +255,7 @@ need_capture_pp(const struct ia_css_pipe *pipe); static bool need_yuv_scaler_stage(const struct ia_css_pipe *pipe); -static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output( +static int ia_css_pipe_create_cas_scaler_desc_single_output( struct ia_css_frame_info *cas_scaler_in_info, struct ia_css_frame_info *cas_scaler_out_info, struct ia_css_frame_info *cas_scaler_vf_info, @@ -269,91 +270,91 @@ need_downscaling(const struct ia_css_resolution in_res, static bool need_capt_ldc(const struct ia_css_pipe *pipe); -static enum ia_css_err +static int sh_css_pipe_load_binaries(struct ia_css_pipe *pipe); static -enum ia_css_err sh_css_pipe_get_viewfinder_frame_info( +int sh_css_pipe_get_viewfinder_frame_info( struct ia_css_pipe *pipe, struct ia_css_frame_info *info, unsigned int idx); -static enum ia_css_err +static int sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, struct ia_css_frame_info *info, unsigned int idx); -static enum ia_css_err +static int capture_start(struct ia_css_pipe *pipe); -static enum ia_css_err +static int video_start(struct ia_css_pipe *pipe); -static enum ia_css_err +static int preview_start(struct ia_css_pipe *pipe); -static enum ia_css_err +static int yuvpp_start(struct ia_css_pipe *pipe); static bool copy_on_sp(struct ia_css_pipe *pipe); -static enum ia_css_err +static int init_vf_frameinfo_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *vf_frame, unsigned int idx); -static enum ia_css_err +static int init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *frame, enum ia_css_frame_format format); -static enum ia_css_err +static int init_out_frameinfo_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *out_frame, unsigned int idx); -static enum ia_css_err +static int sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, const void *acc_fw); -static enum ia_css_err +static int alloc_continuous_frames( struct ia_css_pipe *pipe, bool init_time); static void pipe_global_init(void); -static enum ia_css_err +static int pipe_generate_pipe_num(const struct ia_css_pipe *pipe, unsigned int *pipe_number); static void pipe_release_pipe_num(unsigned int pipe_num); -static enum ia_css_err +static int create_host_pipeline_structure(struct ia_css_stream *stream); -static enum ia_css_err +static int create_host_pipeline(struct ia_css_stream *stream); -static enum ia_css_err +static int create_host_preview_pipeline(struct ia_css_pipe *pipe); -static enum ia_css_err +static int create_host_video_pipeline(struct ia_css_pipe *pipe); -static enum ia_css_err +static int create_host_copy_pipeline(struct ia_css_pipe *pipe, unsigned int max_input_width, struct ia_css_frame *out_frame); -static enum ia_css_err +static int create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe); -static enum ia_css_err +static int create_host_capture_pipeline(struct ia_css_pipe *pipe); -static enum ia_css_err +static int create_host_yuvpp_pipeline(struct ia_css_pipe *pipe); -static enum ia_css_err +static int create_host_acc_pipeline(struct ia_css_pipe *pipe); static unsigned int @@ -383,7 +384,7 @@ static struct sh_css_hmm_buffer_record hrt_address kernel_ptr); static struct sh_css_hmm_buffer_record -*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr, +*sh_css_hmm_buffer_record_validate(ia_css_ptr ddr_buffer_addr, enum ia_css_buffer_type type); void @@ -392,7 +393,7 @@ ia_css_get_acc_configs( struct ia_css_isp_config *config); #if CONFIG_ON_FRAME_ENQUEUE() -static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info +static int set_config_on_frame_enqueue(struct ia_css_frame_info *info, struct frame_data_wrapper *frame); #endif @@ -403,7 +404,7 @@ static unsigned int get_crop_columns_for_bayer_order(const struct ia_css_stream_config *config); static void get_pipe_extra_pixel(struct ia_css_pipe *pipe, unsigned int *extra_row, unsigned int *extra_column); -static enum ia_css_err +static int aspect_ratio_crop_init(struct ia_css_stream *curr_stream, struct ia_css_pipe *pipes[], bool *do_crop_status); @@ -411,7 +412,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream, static bool aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe); -static enum ia_css_err +static int aspect_ratio_crop(struct ia_css_pipe *curr_pipe, struct ia_css_resolution *effective_res); #endif @@ -458,7 +459,7 @@ static enum ia_css_frame_format yuv422_copy_formats[] = { /* Verify whether the selected output format is can be produced * by the copy binary given the stream format. * */ -static enum ia_css_err +static int verify_copy_out_frame_format(struct ia_css_pipe *pipe) { enum ia_css_frame_format out_fmt = pipe->output_info[0].format; unsigned int i, found = 0; @@ -514,8 +515,8 @@ verify_copy_out_frame_format(struct ia_css_pipe *pipe) { break; } if (!found) - return IA_CSS_ERR_INVALID_ARGUMENTS; - return IA_CSS_SUCCESS; + return -EINVAL; + return 0; } unsigned int @@ -533,12 +534,12 @@ ia_css_stream_input_format_bits_per_pixel(struct ia_css_stream *stream) #define GP_ISEL_TPG_MODE 0x90058 #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2) -static enum ia_css_err +static int sh_css_config_input_network(struct ia_css_stream *stream) { unsigned int fmt_type; struct ia_css_pipe *pipe = stream->last_pipe; struct ia_css_binary *binary = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(stream); assert(pipe); @@ -553,7 +554,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { stream->config.input_config.format, stream->csi_rx_config.comp, &fmt_type); - if (err != IA_CSS_SUCCESS) + if (err) return err; sh_css_sp_program_input_circuit(fmt_type, stream->config.channel_id, @@ -564,7 +565,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { { err = ia_css_ifmtr_configure(&stream->config, binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -582,7 +583,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { vblank_cycles = vblank_lines * (width + hblank_cycles); sh_css_sp_configure_sync_gen(width, height, hblank_cycles, vblank_cycles); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_TPG) { /* TODO: move define to proper file in tools */ ia_css_device_store_uint32(GP_ISEL_TPG_MODE, 0); @@ -591,7 +592,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { } ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_config_input_network() leave:\n"); - return IA_CSS_SUCCESS; + return 0; } #elif !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401) static unsigned int csi2_protocol_calculate_max_subpixels_per_line( @@ -879,14 +880,14 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr( break; case IA_CSS_INPUT_MODE_BUFFERED_SENSOR: { - enum ia_css_err err; + int err; unsigned int fmt_type; err = ia_css_isys_convert_stream_format_to_mipi_format( stream_cfg->isys_config[isys_stream_idx].format, MIPI_PREDICTOR_NONE, &fmt_type); - if (err != IA_CSS_SUCCESS) + if (err) rc = false; isys_stream_descr->csi_port_attr.active_lanes = @@ -899,7 +900,7 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr( err |= ia_css_isys_convert_compressed_format( &stream_cfg->source.port.compression, isys_stream_descr); - if (err != IA_CSS_SUCCESS) + if (err) rc = false; /* metadata */ @@ -909,7 +910,7 @@ static bool sh_css_translate_stream_cfg_to_input_system_input_port_attr( stream_cfg->metadata_config.data_type, MIPI_PREDICTOR_NONE, &fmt_type); - if (err != IA_CSS_SUCCESS) + if (err) rc = false; isys_stream_descr->metadata.fmt_type = fmt_type; isys_stream_descr->metadata.bits_per_pixel = @@ -1050,7 +1051,7 @@ static bool sh_css_translate_binary_info_to_input_system_output_port_attr( return true; } -static enum ia_css_err +static int sh_css_config_input_network(struct ia_css_stream *stream) { bool rc; ia_css_isys_descr_t isys_stream_descr; @@ -1095,7 +1096,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { assert(pipe); if (!pipe) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; if (pipe->pipeline.stages) if (pipe->pipeline.stages->binary) @@ -1113,7 +1114,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { /* get the SP thread id */ rc = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &sp_thread_id); if (!rc) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; /* get the target input terminal */ sp_pipeline_input_terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input; @@ -1140,7 +1141,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { } if (!rc) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; isys_stream_id = ia_css_isys_generate_stream_id(sp_thread_id, i); @@ -1150,7 +1151,7 @@ sh_css_config_input_network(struct ia_css_stream *stream) { &sp_pipeline_input_terminal->context.virtual_input_system_stream[i], isys_stream_id); if (!rc) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; /* calculate the configuration of the virtual Input System (2401) */ rc = ia_css_isys_stream_calculate_cfg( @@ -1160,14 +1161,14 @@ sh_css_config_input_network(struct ia_css_stream *stream) { if (!rc) { ia_css_isys_stream_destroy( &sp_pipeline_input_terminal->context.virtual_input_system_stream[i]); - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } } ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_config_input_network() leave:\n"); - return IA_CSS_SUCCESS; + return 0; } static inline struct ia_css_pipe *stream_get_last_pipe( @@ -1225,11 +1226,11 @@ static inline struct ia_css_pipe *stream_get_target_pipe( return target_pipe; } -static enum ia_css_err stream_csi_rx_helper( +static int stream_csi_rx_helper( struct ia_css_stream *stream, - enum ia_css_err (*func)(enum mipi_port_id, uint32_t)) + int (*func)(enum mipi_port_id, uint32_t)) { - enum ia_css_err retval = IA_CSS_ERR_INTERNAL_ERROR; + int retval = -EINVAL; u32 sp_thread_id, stream_id; bool rc; struct ia_css_pipe *target_pipe = NULL; @@ -1258,20 +1259,20 @@ static enum ia_css_err stream_csi_rx_helper( retval = func(stream->config.source.port.port, isys_stream_id); } stream_id++; - } while ((retval == IA_CSS_SUCCESS) && + } while ((retval == 0) && (stream_id < IA_CSS_STREAM_MAX_ISYS_STREAM_PER_CH)); exit: return retval; } -static inline enum ia_css_err stream_register_with_csi_rx( +static inline int stream_register_with_csi_rx( struct ia_css_stream *stream) { return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_register_stream); } -static inline enum ia_css_err stream_unregister_with_csi_rx( +static inline int stream_unregister_with_csi_rx( struct ia_css_stream *stream) { return stream_csi_rx_helper(stream, ia_css_isys_csi_rx_unregister_stream); @@ -1324,7 +1325,7 @@ static void print_pc_histogram(void) sh_css_print(" pc_histogram for binary %d\n", metrics->id); print_pc_histo(" ISP", &metrics->isp_histogram); print_pc_histo(" SP", &metrics->sp_histogram); - sh_css_print("print_pc_histogram() done for binay->id = %d, done.\n", + sh_css_print("print_pc_histogram() done for binary->id = %d, done.\n", metrics->id); } @@ -1404,7 +1405,7 @@ start_binary(struct ia_css_pipe *pipe, } /* start the copy function on the SP */ -static enum ia_css_err +static int start_copy_on_sp(struct ia_css_pipe *pipe, struct ia_css_frame *out_frame) { (void)out_frame; @@ -1412,7 +1413,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe, assert(pipe->stream); if ((!pipe) || (!pipe->stream)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; #if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401) if (pipe->stream->reconfigure_css_rx) @@ -1420,7 +1421,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe, #endif if (pipe->stream->config.input_config.format != ATOMISP_INPUT_FORMAT_BINARY_8) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; sh_css_sp_start_binary_copy(ia_css_pipe_get_pipe_num(pipe), out_frame, pipe->stream->config.pixels_per_clock == 2); #if !defined(HAS_NO_INPUT_SYSTEM) && !defined(USE_INPUT_SYSTEM_VERSION_2401) @@ -1432,7 +1433,7 @@ start_copy_on_sp(struct ia_css_pipe *pipe, } #endif - return IA_CSS_SUCCESS; + return 0; } void sh_css_binary_args_reset(struct sh_css_binary_args *args) @@ -1469,7 +1470,7 @@ static void start_pipe( assert(me); /* all callers are in this file and call with non null argument */ - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { coord = &me->config.internal_frame_origin_bqs_on_sctbl; params = me->stream->isp_params_configs; } @@ -1544,16 +1545,16 @@ enable_interrupts(enum ia_css_irq_type irq_type) /* Enable SW interrupt 0, this is used to signal ISYS events */ cnd_virq_enable_channel( - (virq_id_t)(IRQ_SW_CHANNEL0_ID + IRQ_SW_CHANNEL_OFFSET), + (enum virq_id)(IRQ_SW_CHANNEL0_ID + IRQ_SW_CHANNEL_OFFSET), true); /* Enable SW interrupt 1, this is used to signal PSYS events */ cnd_virq_enable_channel( - (virq_id_t)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET), + (enum virq_id)(IRQ_SW_CHANNEL1_ID + IRQ_SW_CHANNEL_OFFSET), true); #if !defined(HAS_IRQ_MAP_VERSION_2) /* IRQ_SW_CHANNEL2_ID does not exist on 240x systems */ cnd_virq_enable_channel( - (virq_id_t)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET), + (enum virq_id)(IRQ_SW_CHANNEL2_ID + IRQ_SW_CHANNEL_OFFSET), true); virq_clear_all(); #endif @@ -1624,15 +1625,15 @@ ia_css_reset_defaults(struct sh_css *css) *css = default_css; } -enum ia_css_err +int ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, const struct ia_css_fw *fw) { - enum ia_css_err err; + int err; if (!env) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!fw) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_load_firmware() enter\n"); @@ -1645,10 +1646,10 @@ ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, ia_css_unload_firmware(); /* in case we are called twice */ err = sh_css_load_firmware(dev, fw->data, fw->bytes); - if (err == IA_CSS_SUCCESS) + if (!err) { err = ia_css_binary_init_infos(); - if (err == IA_CSS_SUCCESS) + if (!err) fw_explicitly_loaded = true; } @@ -1656,12 +1657,12 @@ ia_css_load_firmware(struct device *dev, const struct ia_css_env *env, return err; } -enum ia_css_err +int ia_css_init(struct device *dev, const struct ia_css_env *env, const struct ia_css_fw *fw, u32 mmu_l1_base, enum ia_css_irq_type irq_type) { - enum ia_css_err err; + int err; ia_css_spctrl_cfg spctrl_cfg; void (*flush_func)(struct ia_css_acc_fw *fw); @@ -1704,9 +1705,9 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, COMPILATION_ERROR_IF(sizeof(struct ia_css_sp_init_dmem_cfg) != SIZE_OF_IA_CSS_SP_INIT_DMEM_CFG_STRUCT); if (!fw && !fw_explicitly_loaded) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!env) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; sh_css_printf = env->print_env.debug_print; @@ -1734,7 +1735,7 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, my_css.flush = flush_func; err = ia_css_rmgr_init(); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; @@ -1771,13 +1772,13 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, gpio_reg_store(GPIO0_ID, _gpio_block_reg_do_0, 0); err = ia_css_refcount_init(REFCOUNT_SIZE); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; } err = sh_css_params_init(); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; @@ -1786,12 +1787,12 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, { ia_css_unload_firmware(); /* in case we already had firmware loaded */ err = sh_css_load_firmware(dev, fw->data, fw->bytes); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } err = ia_css_binary_init_infos(); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR(err); return err; } @@ -1801,10 +1802,10 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, #endif } if (!sh_css_setup_spctrl_config(&sh_css_sp_fw, SP_PROG_NAME, &spctrl_cfg)) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; err = ia_css_spctrl_load_fw(SP0_ID, &spctrl_cfg); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; @@ -1821,8 +1822,8 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, #endif if (!sh_css_hrt_system_is_idle()) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_SYSTEM_NOT_IDLE); - return IA_CSS_ERR_SYSTEM_NOT_IDLE; + IA_CSS_LEAVE_ERR(-EBUSY); + return -EBUSY; } /* can be called here, queuing works, but: - when sp is started later, it will wipe queued items @@ -1844,7 +1845,7 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, ISP_DMA_MAX_BURST_LENGTH); if (ia_css_isys_init() != INPUT_SYSTEM_ERR_NO_ERROR) - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; #endif sh_css_params_map_and_store_default_gdc_lut(); @@ -1853,49 +1854,12 @@ ia_css_init(struct device *dev, const struct ia_css_env *env, return err; } -enum ia_css_err +int ia_css_enable_isys_event_queue(bool enable) { if (sh_css_sp_is_running()) - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; sh_css_sp_enable_isys_event_queue(enable); - return IA_CSS_SUCCESS; -} - -void *sh_css_malloc(size_t size) -{ - ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_malloc() enter: size=%zu\n", - size); - /* FIXME: This first test can probably go away */ - if (size == 0) - return NULL; - if (size > PAGE_SIZE) - return vmalloc(size); - return kmalloc(size, GFP_KERNEL); -} - -void *sh_css_calloc(size_t N, size_t size) -{ - void *p; - - ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, - "sh_css_calloc() enter: N=%zu, size=%zu\n", N, size); - - /* FIXME: this test can probably go away */ - if (size > 0) { - p = sh_css_malloc(N * size); - if (p) - memset(p, 0, size); - return p; - } - return NULL; -} - -void sh_css_free(void *ptr) -{ - if (is_vmalloc_addr(ptr)) - vfree(ptr); - else - kfree(ptr); + return 0; } /* For Acceleration API: Flush FW (shared buffer pointer) arguments */ @@ -1911,13 +1875,13 @@ sh_css_flush(struct ia_css_acc_fw *fw) * pipelines are ready to be converted to sp pipelines. Be careful if you are * doing it from stream_create since we could run out of sp threads due to * allocation on inactive pipelines. */ -static enum ia_css_err +static int map_sp_threads(struct ia_css_stream *stream, bool map) { struct ia_css_pipe *main_pipe = NULL; struct ia_css_pipe *copy_pipe = NULL; struct ia_css_pipe *capture_pipe = NULL; struct ia_css_pipe *acc_pipe = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; enum ia_css_pipe_id pipe_id; assert(stream); @@ -1926,8 +1890,8 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { if (!stream) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } main_pipe = stream->last_pipe; @@ -1984,13 +1948,13 @@ map_sp_threads(struct ia_css_stream *stream, bool map) { /* creates a host pipeline skeleton for all pipes in a stream. Called during * stream_create. */ -static enum ia_css_err +static int create_host_pipeline_structure(struct ia_css_stream *stream) { struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL; struct ia_css_pipe *acc_pipe = NULL; enum ia_css_pipe_id pipe_id; struct ia_css_pipe *main_pipe = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int copy_pipe_delay = 0, capture_pipe_delay = 0; @@ -1999,16 +1963,16 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { if (!stream) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } main_pipe = stream->last_pipe; assert(main_pipe); if (!main_pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } pipe_id = main_pipe->mode; @@ -2050,10 +2014,10 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } - if ((err == IA_CSS_SUCCESS) && copy_pipe) + if (!(err) && copy_pipe) { err = ia_css_pipeline_create(©_pipe->pipeline, copy_pipe->mode, @@ -2061,7 +2025,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { copy_pipe_delay); } - if ((err == IA_CSS_SUCCESS) && capture_pipe) + if (!(err) && capture_pipe) { err = ia_css_pipeline_create(&capture_pipe->pipeline, capture_pipe->mode, @@ -2069,7 +2033,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { capture_pipe_delay); } - if ((err == IA_CSS_SUCCESS) && acc_pipe) + if (!(err) && acc_pipe) { err = ia_css_pipeline_create(&acc_pipe->pipeline, acc_pipe->mode, acc_pipe->pipe_num, main_pipe->dvs_frame_delay); @@ -2080,7 +2044,7 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { { int i; - for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) { + for (i = 1; i < stream->num_pipes && 0 == err; i++) { main_pipe = stream->pipes[i]; err = ia_css_pipeline_create(&main_pipe->pipeline, main_pipe->mode, @@ -2095,20 +2059,20 @@ create_host_pipeline_structure(struct ia_css_stream *stream) { /* creates a host pipeline for all pipes in a stream. Called during * stream_start. */ -static enum ia_css_err +static int create_host_pipeline(struct ia_css_stream *stream) { struct ia_css_pipe *copy_pipe = NULL, *capture_pipe = NULL; struct ia_css_pipe *acc_pipe = NULL; enum ia_css_pipe_id pipe_id; struct ia_css_pipe *main_pipe = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int max_input_width = 0; IA_CSS_ENTER_PRIVATE("stream = %p", stream); if (!stream) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } main_pipe = stream->last_pipe; @@ -2129,7 +2093,7 @@ create_host_pipeline(struct ia_css_stream *stream) { (pipe_id == IA_CSS_PIPE_ID_PREVIEW && stream->config.mode != IA_CSS_INPUT_MODE_MEMORY)) { err = alloc_continuous_frames(main_pipe, true); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } } @@ -2139,7 +2103,7 @@ create_host_pipeline(struct ia_css_stream *stream) { if (pipe_id != IA_CSS_PIPE_ID_ACC) { err = allocate_mipi_frames(main_pipe, &stream->info); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } #elif defined(USE_INPUT_SYSTEM_VERSION_2401) @@ -2147,7 +2111,7 @@ create_host_pipeline(struct ia_css_stream *stream) { (main_pipe->config.mode != IA_CSS_PIPE_MODE_COPY)) { err = allocate_mipi_frames(main_pipe, &stream->info); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } #endif @@ -2162,7 +2126,7 @@ create_host_pipeline(struct ia_css_stream *stream) { main_pipe->pipe_settings.preview.preview_binary.info->sp.input.max_width; err = create_host_preview_pipeline(main_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; break; @@ -2174,7 +2138,7 @@ create_host_pipeline(struct ia_css_stream *stream) { main_pipe->pipe_settings.video.video_binary.info->sp.input.max_width; err = create_host_video_pipeline(main_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; break; @@ -2186,42 +2150,42 @@ create_host_pipeline(struct ia_css_stream *stream) { case IA_CSS_PIPE_ID_YUVPP: err = create_host_yuvpp_pipeline(main_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; break; case IA_CSS_PIPE_ID_ACC: err = create_host_acc_pipeline(main_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; if (copy_pipe) { err = create_host_copy_pipeline(copy_pipe, max_input_width, main_pipe->continuous_frames[0]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } if (capture_pipe) { err = create_host_capture_pipeline(capture_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } if (acc_pipe) { err = create_host_acc_pipeline(acc_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -2230,7 +2194,7 @@ create_host_pipeline(struct ia_css_stream *stream) { { int i; - for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err; i++) { + for (i = 1; i < stream->num_pipes && 0 == err; i++) { switch (stream->pipes[i]->mode) { case IA_CSS_PIPE_ID_PREVIEW: err = create_host_preview_pipeline(stream->pipes[i]); @@ -2248,9 +2212,9 @@ create_host_pipeline(struct ia_css_stream *stream) { err = create_host_acc_pipeline(stream->pipes[i]); break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } } @@ -2260,25 +2224,32 @@ ERR: return err; } -static enum ia_css_err +static const struct ia_css_pipe default_pipe = IA_CSS_DEFAULT_PIPE; +static const struct ia_css_preview_settings preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS; +static const struct ia_css_capture_settings capture = IA_CSS_DEFAULT_CAPTURE_SETTINGS; +static const struct ia_css_video_settings video = IA_CSS_DEFAULT_VIDEO_SETTINGS; +static const struct ia_css_yuvpp_settings yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS; + +static int init_pipe_defaults(enum ia_css_pipe_mode mode, struct ia_css_pipe *pipe, bool copy_pipe) { + if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* Initialize pipe to pre-defined defaults */ - *pipe = IA_CSS_DEFAULT_PIPE; + memcpy(pipe, &default_pipe, sizeof(default_pipe)); /* TODO: JB should not be needed, but temporary backward reference */ switch (mode) { case IA_CSS_PIPE_MODE_PREVIEW: pipe->mode = IA_CSS_PIPE_ID_PREVIEW; - pipe->pipe_settings.preview = IA_CSS_DEFAULT_PREVIEW_SETTINGS; + memcpy(&pipe->pipe_settings.preview, &preview, sizeof(preview)); break; case IA_CSS_PIPE_MODE_CAPTURE: if (copy_pipe) { @@ -2286,11 +2257,11 @@ init_pipe_defaults(enum ia_css_pipe_mode mode, } else { pipe->mode = IA_CSS_PIPE_ID_CAPTURE; } - pipe->pipe_settings.capture = IA_CSS_DEFAULT_CAPTURE_SETTINGS; + memcpy(&pipe->pipe_settings.capture, &capture, sizeof(capture)); break; case IA_CSS_PIPE_MODE_VIDEO: pipe->mode = IA_CSS_PIPE_ID_VIDEO; - pipe->pipe_settings.video = IA_CSS_DEFAULT_VIDEO_SETTINGS; + memcpy(&pipe->pipe_settings.video, &video, sizeof(video)); break; case IA_CSS_PIPE_MODE_ACC: pipe->mode = IA_CSS_PIPE_ID_ACC; @@ -2300,13 +2271,13 @@ init_pipe_defaults(enum ia_css_pipe_mode mode, break; case IA_CSS_PIPE_MODE_YUVPP: pipe->mode = IA_CSS_PIPE_ID_YUVPP; - pipe->pipe_settings.yuvpp = IA_CSS_DEFAULT_YUVPP_SETTINGS; + memcpy(&pipe->pipe_settings.yuvpp, &yuvpp, sizeof(yuvpp)); break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } - return IA_CSS_SUCCESS; + return 0; } static void @@ -2320,7 +2291,7 @@ pipe_global_init(void) } } -static enum ia_css_err +static int pipe_generate_pipe_num(const struct ia_css_pipe *pipe, unsigned int *pipe_number) { const u8 INVALID_PIPE_NUM = (uint8_t)~(0); @@ -2330,7 +2301,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe, if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* Assign a new pipe_num .... search for empty place */ @@ -2347,7 +2318,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe, { /* Max number of pipes already allocated */ IA_CSS_ERROR("Max number of pipes already created"); - return IA_CSS_ERR_RESOURCE_EXHAUSTED; + return -ENOSPC; } my_css.pipe_counter++; @@ -2355,7 +2326,7 @@ pipe_generate_pipe_num(const struct ia_css_pipe *pipe, IA_CSS_LOG("pipe_num (%d)", pipe_num); *pipe_number = pipe_num; - return IA_CSS_SUCCESS; + return 0; } static void @@ -2367,39 +2338,39 @@ pipe_release_pipe_num(unsigned int pipe_num) "pipe_release_pipe_num (%d)\n", pipe_num); } -static enum ia_css_err +static int create_pipe(enum ia_css_pipe_mode mode, struct ia_css_pipe **pipe, bool copy_pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipe *me; if (!pipe) { IA_CSS_ERROR("NULL pipe parameter"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } me = kmalloc(sizeof(*me), GFP_KERNEL); if (!me) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; err = init_pipe_defaults(mode, me, copy_pipe); - if (err != IA_CSS_SUCCESS) + if (err) { kfree(me); return err; } err = pipe_generate_pipe_num(me, &me->pipe_num); - if (err != IA_CSS_SUCCESS) + if (err) { kfree(me); return err; } *pipe = me; - return IA_CSS_SUCCESS; + return 0; } struct ia_css_pipe * @@ -2438,23 +2409,23 @@ static void sh_css_pipe_free_acc_binaries( } } -enum ia_css_err +int ia_css_pipe_destroy(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER("pipe = %p", pipe); if (!pipe) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if (pipe->stream) { IA_CSS_LOG("ia_css_stream_destroy not called!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } switch (pipe->config.mode) @@ -2574,10 +2545,10 @@ ia_css_uninit(void) } #if defined(HAS_IRQ_MAP_VERSION_2) -enum ia_css_err ia_css_irq_translate( +int ia_css_irq_translate( unsigned int *irq_infos) { - virq_id_t irq; + enum virq_id irq; enum hrt_isp_css_irq_status status = hrt_isp_css_irq_status_more_irqs; unsigned int infos = 0; @@ -2589,7 +2560,7 @@ enum ia_css_err ia_css_irq_translate( while (status == hrt_isp_css_irq_status_more_irqs) { status = virq_get_channel_id(&irq); if (status == hrt_isp_css_irq_status_error) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; #if WITH_PC_MONITORING sh_css_print("PC_MONITORING: %s() irq = %d, sh_binary_running set to 0\n", @@ -2643,14 +2614,14 @@ enum ia_css_err ia_css_irq_translate( "ia_css_irq_translate() leave: irq_infos=%u\n", infos); - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err ia_css_irq_enable( +int ia_css_irq_enable( enum ia_css_irq_info info, bool enable) { - virq_id_t irq = N_virq_id; + enum virq_id irq = N_virq_id; IA_CSS_ENTER("info=%d, enable=%d", info, enable); @@ -2674,7 +2645,7 @@ enum ia_css_err ia_css_irq_enable( case IA_CSS_IRQ_INFO_INPUT_SYSTEM_ERROR: case IA_CSS_IRQ_INFO_IF_ERROR: /* Just ignore those unused IRQs without printing errors */ - return IA_CSS_SUCCESS; + return 0; #endif case IA_CSS_IRQ_INFO_DMA_ERROR: irq = virq_dma; @@ -2686,14 +2657,14 @@ enum ia_css_err ia_css_irq_enable( irq = virq_sw_pin_1; break; default: - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } cnd_virq_enable_channel(irq, enable); - IA_CSS_LEAVE_ERR(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR(0); + return 0; } #else @@ -2715,14 +2686,14 @@ sh_css_get_sw_interrupt_value(unsigned int irq) /* configure and load the copy binary, the next binary is used to determine whether the copy binary needs to do left padding. */ -static enum ia_css_err load_copy_binary( +static int load_copy_binary( struct ia_css_pipe *pipe, struct ia_css_binary *copy_binary, struct ia_css_binary *next_binary) { struct ia_css_frame_info copy_out_info, copy_in_info, copy_vf_info; unsigned int left_padding; - enum ia_css_err err; + int err; struct ia_css_binary_descr copy_descr; /* next_binary can be NULL */ @@ -2745,16 +2716,16 @@ static enum ia_css_err load_copy_binary( ©_in_info, ©_out_info, (next_binary) ? NULL : NULL/*TODO: ©_vf_info*/); err = ia_css_binary_find(©_descr, copy_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; copy_binary->left_padding = left_padding; - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int alloc_continuous_frames( struct ia_css_pipe *pipe, bool init_time) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame_info ref_info; enum ia_css_pipe_id pipe_id; bool continuous; @@ -2766,8 +2737,8 @@ alloc_continuous_frames( if ((!pipe) || (!pipe->stream)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } pipe_id = pipe->mode; @@ -2794,8 +2765,8 @@ alloc_continuous_frames( } else { /* should not happen */ - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } #if defined(USE_INPUT_SYSTEM_VERSION_2401) @@ -2834,8 +2805,8 @@ alloc_continuous_frames( } else { /* should not happen */ - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } if (init_time) @@ -2860,7 +2831,7 @@ alloc_continuous_frames( err = ia_css_frame_allocate_from_info( &pipe->continuous_frames[i], &ref_info); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -2869,18 +2840,18 @@ alloc_continuous_frames( &pipe->stream->info.metadata_info); } } - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -enum ia_css_err +int ia_css_alloc_continuous_frame_remain(struct ia_css_stream *stream) { if (!stream) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; return alloc_continuous_frames(stream->continuous_pipe, false); } -static enum ia_css_err +static int load_preview_binaries(struct ia_css_pipe *pipe) { struct ia_css_frame_info prev_in_info, prev_bds_out_info, @@ -2888,7 +2859,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { prev_vf_info; struct ia_css_binary_descr preview_descr; bool online; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool continuous, need_vf_pp = false; bool need_isp_copy_binary = false; #ifdef USE_INPUT_SYSTEM_VERSION_2401 @@ -2910,13 +2881,13 @@ load_preview_binaries(struct ia_css_pipe *pipe) { #endif if (mycs->preview_binary.info) - return IA_CSS_SUCCESS; + return 0; err = ia_css_util_check_input(&pipe->stream->config, false, false); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ia_css_frame_check_info(pipe_out_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; /* Note: the current selection of vf_pp binary and @@ -2960,13 +2931,13 @@ load_preview_binaries(struct ia_css_pipe *pipe) { &prev_bds_out_info, &prev_out_info, &prev_vf_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ia_css_binary_find(&preview_descr, &mycs->preview_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* The delay latency determines the number of invalid frames after * a stream is started. */ pipe->num_invalid_frames = pipe->dvs_frame_delay; @@ -3004,11 +2975,11 @@ load_preview_binaries(struct ia_css_pipe *pipe) { &prev_bds_out_info, &prev_out_info, &prev_vf_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ia_css_binary_find(&preview_descr, &mycs->preview_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -3022,7 +2993,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { pipe_out_info); err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -3037,7 +3008,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { * where the driver chooses for memory based input frames. In these cases, a copy binary (which typical * copies sensor data to DDR) does not have much use. */ - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) need_isp_copy_binary = !online && !continuous; else need_isp_copy_binary = !online && !continuous && !(pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY); @@ -3049,7 +3020,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { err = load_copy_binary(pipe, &mycs->copy_binary, &mycs->preview_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -3059,7 +3030,7 @@ load_preview_binaries(struct ia_css_pipe *pipe) { pipe->shading_table = NULL; } - return IA_CSS_SUCCESS; + return 0; } static void @@ -3068,21 +3039,21 @@ ia_css_binary_unload(struct ia_css_binary *binary) ia_css_binary_destroy_isp_parameters(binary); } -static enum ia_css_err +static int unload_preview_binaries(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_binary_unload(&pipe->pipe_settings.preview.copy_binary); ia_css_binary_unload(&pipe->pipe_settings.preview.preview_binary); ia_css_binary_unload(&pipe->pipe_settings.preview.vf_pp_binary); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } static const struct ia_css_fw_info *last_output_firmware( @@ -3101,7 +3072,7 @@ static const struct ia_css_fw_info *last_output_firmware( return last_fw; } -static enum ia_css_err add_firmwares( +static int add_firmwares( struct ia_css_pipeline *me, struct ia_css_binary *binary, const struct ia_css_fw_info *fw, @@ -3113,7 +3084,7 @@ static enum ia_css_err add_firmwares( struct ia_css_pipeline_stage **my_stage, struct ia_css_pipeline_stage **vf_stage) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipeline_stage *extra_stage = NULL; struct ia_css_pipeline_stage_desc stage_desc; @@ -3140,7 +3111,7 @@ static enum ia_css_err add_firmwares( err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, &extra_stage); - if (err != IA_CSS_SUCCESS) + if (err) return err; if (fw->info.isp.sp.enable.output != 0) in_frame = extra_stage->args.out_frame[0]; @@ -3153,7 +3124,7 @@ static enum ia_css_err add_firmwares( return err; } -static enum ia_css_err add_vf_pp_stage( +static int add_vf_pp_stage( struct ia_css_pipe *pipe, struct ia_css_frame *in_frame, struct ia_css_frame *out_frame, @@ -3162,20 +3133,20 @@ static enum ia_css_err add_vf_pp_stage( { struct ia_css_pipeline *me = NULL; const struct ia_css_fw_info *last_fw = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; struct ia_css_pipeline_stage_desc stage_desc; /* out_frame can be NULL ??? */ if (!pipe) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!in_frame) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!vf_pp_binary) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!vf_pp_stage) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_pipe_util_create_output_frames(out_frames); me = &pipe->pipeline; @@ -3197,7 +3168,7 @@ static enum ia_css_err add_vf_pp_stage( out_frames, in_frame, NULL); } err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, vf_pp_stage); - if (err != IA_CSS_SUCCESS) + if (err) return err; in_frame = (*vf_pp_stage)->args.out_frame[0]; } @@ -3208,7 +3179,7 @@ static enum ia_css_err add_vf_pp_stage( return err; } -static enum ia_css_err add_yuv_scaler_stage( +static int add_yuv_scaler_stage( struct ia_css_pipe *pipe, struct ia_css_pipeline *me, struct ia_css_frame *in_frame, @@ -3218,7 +3189,7 @@ static enum ia_css_err add_yuv_scaler_stage( struct ia_css_pipeline_stage **pre_vf_pp_stage) { const struct ia_css_fw_info *last_fw; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *vf_frame = NULL; struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; struct ia_css_pipeline_stage_desc stage_desc; @@ -3250,7 +3221,7 @@ static enum ia_css_err add_yuv_scaler_stage( err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, pre_vf_pp_stage); - if (err != IA_CSS_SUCCESS) + if (err) return err; in_frame = (*pre_vf_pp_stage)->args.out_frame[0]; @@ -3267,7 +3238,7 @@ static enum ia_css_err add_yuv_scaler_stage( return err; } -static enum ia_css_err add_capture_pp_stage( +static int add_capture_pp_stage( struct ia_css_pipe *pipe, struct ia_css_pipeline *me, struct ia_css_frame *in_frame, @@ -3276,7 +3247,7 @@ static enum ia_css_err add_capture_pp_stage( struct ia_css_pipeline_stage **capture_pp_stage) { const struct ia_css_fw_info *last_fw = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *vf_frame = NULL; struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; struct ia_css_pipeline_stage_desc stage_desc; @@ -3296,7 +3267,7 @@ static enum ia_css_err add_capture_pp_stage( last_fw = last_output_firmware(pipe->output_stage); err = ia_css_frame_allocate_from_info(&vf_frame, &capture_pp_binary->vf_frame_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; if (last_fw) { ia_css_pipe_util_set_output_frames(out_frames, 0, NULL); @@ -3310,7 +3281,7 @@ static enum ia_css_err add_capture_pp_stage( err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, capture_pp_stage); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = add_firmwares(me, capture_pp_binary, pipe->output_stage, last_fw, IA_CSS_BINARY_MODE_CAPTURE_PP, @@ -3346,10 +3317,10 @@ static void sh_css_setup_queues(void) ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "sh_css_setup_queues() leave:\n"); } -static enum ia_css_err +static int init_vf_frameinfo_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *vf_frame, unsigned int idx) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -3511,11 +3482,11 @@ ia_css_get_crop_offsets( } #endif -static enum ia_css_err +static int init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *frame, enum ia_css_frame_format format) { struct ia_css_frame *in_frame; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -3552,10 +3523,10 @@ init_in_frameinfo_memory_defaults(struct ia_css_pipe *pipe, return err; } -static enum ia_css_err +static int init_out_frameinfo_defaults(struct ia_css_pipe *pipe, struct ia_css_frame *out_frame, unsigned int idx) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; @@ -3574,7 +3545,7 @@ init_out_frameinfo_defaults(struct ia_css_pipe *pipe, } /* Create stages for video pipe */ -static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) +static int create_host_video_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline_stage_desc stage_desc; struct ia_css_binary *copy_binary, *video_binary, @@ -3588,7 +3559,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) struct ia_css_frame *out_frame; struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; struct ia_css_frame *vf_frame = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool need_copy = false; bool need_vf_pp = false; bool need_yuv_pp = false; @@ -3600,8 +3571,8 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_pipe_util_create_output_frames(out_frames); out_frame = &pipe->out_frame_struct; @@ -3629,20 +3600,20 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) in_frame = &pipe->in_frame_struct; err = init_in_frameinfo_memory_defaults(pipe, in_frame, IA_CSS_FRAME_FORMAT_RAW); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } out_frame->data = 0; err = init_out_frameinfo_defaults(pipe, out_frame, 0); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { vf_frame = &pipe->vf_frame_struct; vf_frame->data = 0; err = init_vf_frameinfo_defaults(pipe, vf_frame, 0); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -3666,7 +3637,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, ©_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; in_frame = me->stages->args.out_frame[0]; } else if (pipe->stream->config.continuous) { @@ -3695,7 +3666,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, &video_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; /* If we use copy iso video, the input must be yuv iso raw */ @@ -3711,16 +3682,13 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) in_frame = video_stage->args.out_vf_frame; err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary, &vf_pp_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } if (video_stage) { int frm; -#ifndef ISP2401 - for (frm = 0; frm < NUM_TNR_FRAMES; frm++) { -#else + for (frm = 0; frm < NUM_TNR_FRAMES; frm++) { -#endif video_stage->args.tnr_frames[frm] = pipe->pipe_settings.video.tnr_frames[frm]; } @@ -3742,7 +3710,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) /* In/Out Frame mapping to support output frame extension.*/ out = video_stage->args.out_frame[0]; err = ia_css_frame_allocate_from_info(&in, &pipe->output_info[0]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; video_stage->args.out_frame[0] = in; } @@ -3751,7 +3719,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) last_output_firmware(pipe->output_stage), IA_CSS_BINARY_MODE_VIDEO, in, out, NULL, &video_stage, NULL); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -3770,7 +3738,7 @@ static enum ia_css_err create_host_video_pipeline(struct ia_css_pipe *pipe) &yuv_scaler_binary[i], &yuv_scaler_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -3789,17 +3757,17 @@ ERR: return err; } -static enum ia_css_err +static int create_host_acc_pipeline(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; const struct ia_css_fw_info *fw; unsigned int i; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (!pipe->stream)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } pipe->pipeline.num_execs = pipe->config.acc_num_execs; @@ -3811,7 +3779,7 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) { for (i = 0; fw; fw = fw->next) { err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -3820,7 +3788,7 @@ create_host_acc_pipeline(struct ia_css_pipe *pipe) { struct ia_css_fw_info *fw = pipe->config.acc_stages[i]; err = sh_css_pipeline_add_acc_stage(&pipe->pipeline, fw); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -3832,7 +3800,7 @@ ERR: } /* Create stages for preview */ -static enum ia_css_err +static int create_host_preview_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline_stage *copy_stage = NULL; struct ia_css_pipeline_stage *preview_stage = NULL; @@ -3841,7 +3809,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline *me = NULL; struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL; struct ia_css_frame *in_frame = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame *out_frame; struct ia_css_frame *out_frames[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; bool need_in_frameinfo_memory = false; @@ -3855,8 +3823,8 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_pipe_util_create_output_frames(out_frames); @@ -3886,7 +3854,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { { err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; in_frame = &me->in_frame; @@ -3896,7 +3864,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { } err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; out_frame = &me->out_frame[0]; @@ -3913,7 +3881,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, ©_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; in_frame = me->stages->args.out_frame[0]; #ifndef ISP2401 @@ -3949,7 +3917,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, &preview_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; /* If we use copy iso preview, the input must be yuv iso raw */ preview_stage->args.copy_vf = @@ -3969,7 +3937,7 @@ create_host_preview_pipeline(struct ia_css_pipe *pipe) { in_frame = preview_stage->args.out_frame[0]; err = add_vf_pp_stage(pipe, in_frame, out_frame, vf_pp_binary, &vf_pp_stage); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } @@ -4001,11 +3969,11 @@ static void send_raw_frames(struct ia_css_pipe *pipe) return; } -static enum ia_css_err +static int preview_start(struct ia_css_pipe *pipe) { struct ia_css_pipeline *me; struct ia_css_binary *copy_binary, *preview_binary, *vf_pp_binary = NULL; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipe *copy_pipe, *capture_pipe; struct ia_css_pipe *acc_pipe; enum sh_css_pipe_config_override copy_ovrd; @@ -4016,8 +3984,8 @@ preview_start(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_PREVIEW)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } me = &pipe->pipeline; @@ -4038,8 +4006,8 @@ preview_start(struct ia_css_pipe *pipe) { #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) /* multi stream video needs mipi buffers */ err = send_mipi_frames(pipe); - if (err != IA_CSS_SUCCESS) { - IA_CSS_LEAVE_ERR_PRIVATE(err); + if (err) { + IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } #endif @@ -4059,7 +4027,7 @@ preview_start(struct ia_css_pipe *pipe) { } } - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { coord = &pipe->config.internal_frame_origin_bqs_on_sctbl; params = pipe->stream->isp_params_configs; } @@ -4137,10 +4105,10 @@ preview_start(struct ia_css_pipe *pipe) { return err; } -enum ia_css_err +int ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, const struct ia_css_buffer *buffer) { - enum ia_css_err return_err = IA_CSS_SUCCESS; + int return_err = 0; unsigned int thread_id; enum sh_css_queue_id queue_id; struct ia_css_pipeline *pipeline; @@ -4156,8 +4124,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, if ((!pipe) || (!buffer)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } buf_type = buffer->type; @@ -4177,7 +4145,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, } } if (!found_pipe) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if (buf_type == IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME) { @@ -4192,7 +4160,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, } } if (!found_pipe) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } #endif pipe_id = pipe->mode; @@ -4205,36 +4173,36 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, (buf_type >= IA_CSS_NUM_DYNAMIC_BUFFER_TYPE) || (pipe_id >= IA_CSS_PIPE_ID_NUM)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); if (!ret_err) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id); if (!ret_err) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); + IA_CSS_LEAVE_ERR(-EBUSY); /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } pipeline = &pipe->pipeline; @@ -4251,24 +4219,24 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, if (buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) { if (!buffer->data.stats_3a) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_3a); ddr_buffer.payload.s3a = *buffer->data.stats_3a; } else if (buf_type == IA_CSS_BUFFER_TYPE_DIS_STATISTICS) { if (!buffer->data.stats_dvs) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.stats_dvs); ddr_buffer.payload.dis = *buffer->data.stats_dvs; } else if (buf_type == IA_CSS_BUFFER_TYPE_METADATA) { if (!buffer->data.metadata) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.metadata); ddr_buffer.payload.metadata = *buffer->data.metadata; @@ -4279,8 +4247,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, || (buf_type == IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME)) { if (!buffer->data.frame) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ddr_buffer.kernel_ptr = HOST_ADDRESS(buffer->data.frame); ddr_buffer.payload.frame.frame_data = buffer->data.frame->data; @@ -4294,7 +4262,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, return_err = set_config_on_frame_enqueue( &buffer->data.frame->info, &ddr_buffer.payload.frame); - if (return_err != IA_CSS_SUCCESS) { + if (return_err) { IA_CSS_LEAVE_ERR(return_err); return return_err; } @@ -4314,11 +4282,11 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, if ((!h_vbuf) || (h_vbuf->vptr == 0x0)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } - mmgr_store(h_vbuf->vptr, + hmm_store(h_vbuf->vptr, (void *)(&ddr_buffer), sizeof(struct sh_css_hmm_buffer)); if ((buf_type == IA_CSS_BUFFER_TYPE_3A_STATISTICS) @@ -4328,8 +4296,8 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, if (!pipeline) { ia_css_rmgr_rel_vbuf(hmm_buffer_pool, &h_vbuf); IA_CSS_LOG("pipeline is empty!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } for (stage = pipeline->stages; stage; stage = stage->next) { @@ -4353,7 +4321,7 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, queue_id, (uint32_t)h_vbuf->vptr); #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) - if ((return_err == IA_CSS_SUCCESS) && + if (!(return_err) && (buf_type == IA_CSS_BUFFER_TYPE_OUTPUT_FRAME)) { IA_CSS_LOG("pfp: enqueued OF %d to q %d thread %d", ddr_buffer.payload.frame.frame_data, @@ -4362,14 +4330,14 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, #endif } - if (return_err == IA_CSS_SUCCESS) + if (!return_err) { if (sh_css_hmm_buffer_record_acquire( h_vbuf, buf_type, HOST_ADDRESS(ddr_buffer.kernel_ptr))) { IA_CSS_LOG("send vbuf=%p", h_vbuf); } else { - return_err = IA_CSS_ERR_INTERNAL_ERROR; + return_err = -EINVAL; IA_CSS_ERROR("hmm_buffer_record[]: no available slots\n"); } } @@ -4378,13 +4346,13 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, * Tell the SP which queues are not empty, * by sending the software event. */ - if (return_err == IA_CSS_SUCCESS) + if (!return_err) { if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ IA_CSS_LOG("SP is not running!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + IA_CSS_LEAVE_ERR(-EBUSY); + return -EBUSY; } return_err = ia_css_bufq_enqueue_psys_event( IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED, @@ -4405,12 +4373,12 @@ ia_css_pipe_enqueue_buffer(struct ia_css_pipe *pipe, /* * TODO: Free up the hmm memory space. */ -enum ia_css_err +int ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, struct ia_css_buffer *buffer) { - enum ia_css_err return_err; + int return_err; enum sh_css_queue_id queue_id; - hrt_vaddress ddr_buffer_addr = (hrt_vaddress)0; + ia_css_ptr ddr_buffer_addr = (ia_css_ptr)0; struct sh_css_hmm_buffer ddr_buffer; enum ia_css_buffer_type buf_type; enum ia_css_pipe_id pipe_id; @@ -4422,8 +4390,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, if ((!pipe) || (!buffer)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } pipe_id = pipe->mode; @@ -4437,35 +4405,35 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, ret_err = ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); if (!ret_err) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } ret_err = ia_css_query_internal_queue_id(buf_type, thread_id, &queue_id); if (!ret_err) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if ((queue_id <= SH_CSS_INVALID_QUEUE_ID) || (queue_id >= SH_CSS_MAX_NUM_QUEUES)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); + IA_CSS_LEAVE_ERR(-EBUSY); /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } return_err = ia_css_bufq_dequeue_buffer(queue_id, (uint32_t *)&ddr_buffer_addr); - if (return_err == IA_CSS_SUCCESS) + if (!return_err) { struct ia_css_frame *frame; struct sh_css_hmm_buffer_record *hmm_buffer_record = NULL; @@ -4477,7 +4445,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, ddr_buffer_addr, buf_type); if (hmm_buffer_record) { /* valid hmm_buffer_record found. Save the kernel_ptr - * for validation after performing mmgr_load. The + * for validation after performing hmm_load. The * vbuf handle and buffer_record can be released. */ kernel_ptr = hmm_buffer_record->kernel_ptr; @@ -4486,11 +4454,11 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, } else { IA_CSS_ERROR("hmm_buffer_record not found (0x%x) buf_type(%d)", ddr_buffer_addr, buf_type); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } - mmgr_load(ddr_buffer_addr, + hmm_load(ddr_buffer_addr, &ddr_buffer, sizeof(struct sh_css_hmm_buffer)); @@ -4503,8 +4471,8 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, IA_CSS_ERROR("expected: (0x%llx)", (u64)kernel_ptr); IA_CSS_ERROR("actual: (0x%llx)", (u64)HOST_ADDRESS(ddr_buffer.kernel_ptr)); IA_CSS_ERROR("buf_type: %d\n", buf_type); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if (ddr_buffer.kernel_ptr != 0) { @@ -4529,7 +4497,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, * for 2401 it is done in ia_css_stream_destroy call */ return_err = free_mipi_frames(pipe); - if (return_err != IA_CSS_SUCCESS) { + if (return_err) { IA_CSS_LOG("free_mipi_frames() failed"); IA_CSS_LEAVE_ERR(return_err); return return_err; @@ -4537,6 +4505,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, #endif pipe->stop_requested = false; } + /* fall through */ case IA_CSS_BUFFER_TYPE_VF_OUTPUT_FRAME: case IA_CSS_BUFFER_TYPE_SEC_VF_OUTPUT_FRAME: frame = (struct ia_css_frame *)HOST_ADDRESS(ddr_buffer.kernel_ptr); @@ -4597,7 +4566,7 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, buffer->data.metadata->exp_id = ddr_buffer.payload.metadata.exp_id; break; default: - return_err = IA_CSS_ERR_INTERNAL_ERROR; + return_err = -EINVAL; break; } } @@ -4607,13 +4576,13 @@ ia_css_pipe_dequeue_buffer(struct ia_css_pipe *pipe, * Tell the SP which queues are not full, * by sending the software event. */ - if (return_err == IA_CSS_SUCCESS) + if (!return_err) { if (!sh_css_sp_is_running()) { IA_CSS_LOG("SP is not running!"); - IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); + IA_CSS_LEAVE_ERR(-EBUSY); /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } ia_css_bufq_enqueue_psys_event( IA_CSS_PSYS_SW_EVENT_BUFFER_DEQUEUED, @@ -4656,16 +4625,16 @@ static enum ia_css_event_type convert_event_sp_to_host_domain[] = { 0, /* error if sp passes SH_CSS_SP_EVENT_NR_OF_TYPES as a valid event. */ }; -enum ia_css_err +int ia_css_dequeue_event(struct ia_css_event *event) { return ia_css_dequeue_psys_event(event); } -enum ia_css_err +int ia_css_dequeue_psys_event(struct ia_css_event *event) { enum ia_css_pipe_id pipe_id = 0; u8 payload[4] = {0, 0, 0, 0}; - enum ia_css_err ret_err; + int ret_err; /*TODO: * a) use generic decoding function , same as the one used by sp. @@ -4675,17 +4644,17 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { * to avoid flooding the logs when the host application * uses polling. */ if (!event) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } /* dequeue the event (if any) from the psys event queue */ ret_err = ia_css_bufq_dequeue_psys_event(payload); - if (ret_err != IA_CSS_SUCCESS) + if (ret_err) return ret_err; IA_CSS_LOG("event dequeued from psys event queue"); @@ -4717,7 +4686,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { event->timer_code = payload[2]; payload[0] = payload[1] = payload[2] = payload[3] = 0; ret_err = ia_css_bufq_dequeue_psys_event(payload); - if (ret_err != IA_CSS_SUCCESS) { + if (ret_err) { /* no 2nd event ??? an error */ /* Putting IA_CSS_ERROR is resulting in failures in * Merrifield smoke testing */ @@ -4771,7 +4740,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { pipe_id = (enum ia_css_pipe_id)payload[2]; /* Check to see if pipe still exists */ if (!event->pipe) - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; if (event->type == IA_CSS_EVENT_TYPE_FRAME_TAGGED) { /* find the capture pipe that goes with this */ @@ -4796,7 +4765,7 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { &event->pipe->pipeline, stage_num, &event->fw_handle); - if (ret_err != IA_CSS_SUCCESS) { + if (ret_err) { IA_CSS_ERROR("Invalid stage num received for ACC event. stage_num:%u", stage_num); return ret_err; @@ -4809,28 +4778,28 @@ ia_css_dequeue_psys_event(struct ia_css_event *event) { else IA_CSS_LEAVE("event_id=%d", event->type); - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int ia_css_dequeue_isys_event(struct ia_css_event *event) { u8 payload[4] = {0, 0, 0, 0}; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; /* We skip the IA_CSS_ENTER logging call * to avoid flooding the logs when the host application * uses polling. */ if (!event) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } err = ia_css_bufq_dequeue_isys_event(payload); - if (err != IA_CSS_SUCCESS) + if (err) return err; IA_CSS_LOG("event dequeued from isys event queue"); @@ -4859,9 +4828,9 @@ acc_start(struct ia_css_pipe *pipe) pipe->stream->config.mode); } -static enum ia_css_err +static int sh_css_pipe_start(struct ia_css_stream *stream) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipe *pipe; enum ia_css_pipe_id pipe_id; @@ -4871,14 +4840,14 @@ sh_css_pipe_start(struct ia_css_stream *stream) { if (!stream) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } pipe = stream->last_pipe; if (!pipe) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } pipe_id = pipe->mode; @@ -4910,14 +4879,14 @@ sh_css_pipe_start(struct ia_css_stream *stream) { acc_start(pipe); break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } /* DH regular multi pipe - not continuous mode: start the next pipes too */ if (!stream->config.continuous) { int i; - for (i = 1; i < stream->num_pipes && IA_CSS_SUCCESS == err ; i++) { + for (i = 1; i < stream->num_pipes && 0 == err ; i++) { switch (stream->pipes[i]->mode) { case IA_CSS_PIPE_ID_PREVIEW: stream->pipes[i]->stop_requested = false; @@ -4940,11 +4909,11 @@ sh_css_pipe_start(struct ia_css_stream *stream) { acc_start(stream->pipes[i]); break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } } } - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -4960,7 +4929,7 @@ sh_css_pipe_start(struct ia_css_stream *stream) { sh_css_invalidate_params(stream); err = sh_css_param_update_isp_params(pipe, stream->isp_params_configs, true, NULL); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4972,9 +4941,9 @@ sh_css_pipe_start(struct ia_css_stream *stream) { if (!sh_css_sp_is_running()) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); + IA_CSS_LEAVE_ERR_PRIVATE(-EBUSY); /* SP is not running. The queues are not valid */ - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } ia_css_bufq_enqueue_psys_event(IA_CSS_PSYS_SW_EVENT_START_STREAM, (uint8_t)thread_id, 0, 0); @@ -5005,8 +4974,8 @@ sh_css_pipe_start(struct ia_css_stream *stream) { copy_pipe = pipe->pipe_settings.video.copy_pipe; if (!copy_pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(copy_pipe), &thread_id); @@ -5025,8 +4994,8 @@ sh_css_pipe_start(struct ia_css_stream *stream) { capture_pipe = pipe->pipe_settings.video.capture_pipe; if (!capture_pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(capture_pipe), &thread_id); @@ -5087,39 +5056,39 @@ sh_css_continuous_is_enabled(uint8_t pipe_num) } /* ISP2400 */ -enum ia_css_err +int ia_css_stream_get_max_buffer_depth(struct ia_css_stream *stream, int *buffer_depth) { if (!buffer_depth) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_max_buffer_depth() enter: void\n"); (void)stream; *buffer_depth = NUM_CONTINUOUS_FRAMES; - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int ia_css_stream_set_buffer_depth(struct ia_css_stream *stream, int buffer_depth) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_set_buffer_depth() enter: num_frames=%d\n", buffer_depth); (void)stream; if (buffer_depth > NUM_CONTINUOUS_FRAMES || buffer_depth < 1) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* ok, value allowed */ stream->config.target_num_cont_raw_buf = buffer_depth; /* TODO: check what to regarding initialization */ - return IA_CSS_SUCCESS; + return 0; } /* ISP2401 */ -enum ia_css_err +int ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, int *buffer_depth) { if (!buffer_depth) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_buffer_depth() enter: void\n"); (void)stream; *buffer_depth = stream->config.target_num_cont_raw_buf; - return IA_CSS_SUCCESS; + return 0; } /* @@ -5129,10 +5098,10 @@ ia_css_stream_get_buffer_depth(struct ia_css_stream *stream, * Refer to "Local prototypes" for more info. */ /* ISP2401 */ -static enum ia_css_err +static int sh_css_pipes_stop(struct ia_css_stream *stream) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipe *main_pipe; enum ia_css_pipe_id main_pipe_id; int i; @@ -5141,7 +5110,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) if (!stream) { IA_CSS_LOG("stream does NOT exist!"); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto ERR; } @@ -5150,7 +5119,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) if (!main_pipe) { IA_CSS_LOG("main_pipe does NOT exist!"); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto ERR; } @@ -5183,7 +5152,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) * the CSS driver should capture these error code and * handle it in the driver exception handling mechanism. */ - if (err != IA_CSS_SUCCESS) { + if (err) { goto ERR; } } @@ -5211,7 +5180,7 @@ sh_css_pipes_stop(struct ia_css_stream *stream) assert(copy_pipe); if (!copy_pipe) { IA_CSS_LOG("Copy Pipe does NOT exist!"); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto ERR; } @@ -5325,7 +5294,7 @@ sh_css_get_mipi_sizes_for_check(const unsigned int port, const unsigned int idx) } #endif -static enum ia_css_err sh_css_pipe_configure_output( +static int sh_css_pipe_configure_output( struct ia_css_pipe *pipe, unsigned int width, unsigned int height, @@ -5333,17 +5302,17 @@ static enum ia_css_err sh_css_pipe_configure_output( enum ia_css_frame_format format, unsigned int idx) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; - IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, paddaed width = %d, format = %d, idx = %d", + IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, padded width = %d, format = %d, idx = %d", pipe, width, height, padded_width, format, idx); if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } err = ia_css_util_check_res(width, height); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -5357,16 +5326,16 @@ static enum ia_css_err sh_css_pipe_configure_output( format, padded_width); } - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -static enum ia_css_err +static int sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe, struct ia_css_shading_info *shading_info, struct ia_css_pipe_config *pipe_config) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_binary *binary = NULL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, @@ -5390,17 +5359,17 @@ sh_css_pipe_get_shading_info(struct ia_css_pipe *pipe, /* When the pipe does not have a binary which has the shading * correction, this function does not need to fill the shading * information. It is not a error case, and then - * this function should return IA_CSS_SUCCESS. + * this function should return 0. */ memset(shading_info, 0, sizeof(*shading_info)); } return err; } -static enum ia_css_err +static int sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe, struct ia_css_grid_info *info) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_binary *binary = NULL; assert(pipe); @@ -5413,7 +5382,7 @@ sh_css_pipe_get_grid_info(struct ia_css_pipe *pipe, if (binary) { err = ia_css_binary_3a_grid_info(binary, info, pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } else memset(&info->s3a_grid, 0, sizeof(info->s3a_grid)); @@ -5457,7 +5426,7 @@ ERR : * @brief Check if a format is supported by the pipe. * */ -static enum ia_css_err +static int ia_css_pipe_check_format(struct ia_css_pipe *pipe, enum ia_css_frame_format format) { const enum ia_css_frame_format *supported_formats; @@ -5470,8 +5439,8 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe, if (NULL == pipe || NULL == pipe->pipe_settings.video.video_binary.info) { IA_CSS_ERROR("Pipe or binary info is not set"); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } supported_formats = pipe->pipe_settings.video.video_binary.info->output_formats; @@ -5487,21 +5456,21 @@ ia_css_pipe_check_format(struct ia_css_pipe *pipe, if (!found) { IA_CSS_ERROR("Requested format is not supported by binary"); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } else { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } } -static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) +static int load_video_binaries(struct ia_css_pipe *pipe) { struct ia_css_frame_info video_in_info, tnr_info, *video_vf_info, video_bds_out_info, *pipe_out_info, *pipe_vf_out_info; bool online; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool continuous = pipe->stream->config.continuous; unsigned int i; unsigned int num_output_pins; @@ -5520,7 +5489,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) * All are always reset at the same time anyway. */ if (mycs->video_binary.info) - return IA_CSS_SUCCESS; + return 0; online = pipe->stream->config.online; pipe_out_info = &pipe->output_info[0]; @@ -5534,19 +5503,19 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) * This is checked in the binary_find(), so no need to check it here */ err = ia_css_util_check_input(&pipe->stream->config, false, false); - if (err != IA_CSS_SUCCESS) + if (err) return err; /* cannot have online video and input_mode memory */ if (online && pipe->stream->config.mode == IA_CSS_INPUT_MODE_MEMORY) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { err = ia_css_util_check_vf_out_info(pipe_out_info, pipe_vf_out_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; } else { err = ia_css_frame_check_info(pipe_out_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -5581,19 +5550,19 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) pipe_out_info, NULL, &cas_scaler_descr); - if (err != IA_CSS_SUCCESS) + if (err) return err; mycs->num_yuv_scaler = cas_scaler_descr.num_stage; mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage * sizeof(struct ia_css_binary), GFP_KERNEL); if (!mycs->yuv_scaler_binary) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; return err; } mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage * sizeof(bool), GFP_KERNEL); if (!mycs->is_output_stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; return err; } for (i = 0; i < cas_scaler_descr.num_stage; i++) { @@ -5607,7 +5576,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) &cas_scaler_descr.vf_info[i]); err = ia_css_binary_find(&yuv_scaler_descr, &mycs->yuv_scaler_binary[i]); - if (err != IA_CSS_SUCCESS) { + if (err) { kfree(mycs->is_output_stage); mycs->is_output_stage = NULL; return err; @@ -5624,7 +5593,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) &video_descr, &video_in_info, &video_bds_out_info, &video_bin_out_info, video_vf_info, pipe->stream->config.left_padding); - if (err != IA_CSS_SUCCESS) + if (err) return err; /* In the case where video_vf_info is not NULL, this allows @@ -5635,7 +5604,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) err = ia_css_binary_find(&video_descr, &mycs->video_binary); - if (err != IA_CSS_SUCCESS) { + if (err) { if (video_vf_info) { /* This will do another video binary lookup later for YUV_LINE format*/ need_vf_pp = true; @@ -5677,7 +5646,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) /* restore original vf_info format */ ia_css_frame_info_set_format(video_vf_info, vf_info_format); - if (err != IA_CSS_SUCCESS) + if (err) return err; } } @@ -5712,7 +5681,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) err = load_copy_binary(pipe, &mycs->copy_binary, &mycs->video_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } #else @@ -5739,14 +5708,14 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } #endif err = allocate_delay_frames(pipe); - if (err != IA_CSS_SUCCESS) + if (err) return err; if (mycs->video_binary.info->sp.enable.block_output) { @@ -5755,7 +5724,7 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) tnr_info = mycs->video_binary.out_frame_info[0]; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* Select resolution for TNR. If * output_system_in_resolution(GDC_out_resolution) is * being used, then select that as it will also be in resolution for @@ -5794,14 +5763,14 @@ static enum ia_css_err load_video_binaries(struct ia_css_pipe *pipe) err = ia_css_frame_allocate_from_info( &mycs->tnr_frames[i], &tnr_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; } IA_CSS_LEAVE_PRIVATE(""); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int unload_video_binaries(struct ia_css_pipe *pipe) { unsigned int i; @@ -5809,8 +5778,8 @@ unload_video_binaries(struct ia_css_pipe *pipe) { if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_binary_unload(&pipe->pipe_settings.video.copy_binary); ia_css_binary_unload(&pipe->pipe_settings.video.video_binary); @@ -5824,14 +5793,14 @@ unload_video_binaries(struct ia_css_pipe *pipe) { kfree(pipe->pipe_settings.video.yuv_scaler_binary); pipe->pipe_settings.video.yuv_scaler_binary = NULL; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -static enum ia_css_err video_start(struct ia_css_pipe *pipe) +static int video_start(struct ia_css_pipe *pipe) { struct ia_css_binary *copy_binary; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipe *copy_pipe, *capture_pipe; enum sh_css_pipe_config_override copy_ovrd; enum ia_css_input_mode video_pipe_input_mode; @@ -5841,8 +5810,8 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe) IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_VIDEO)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } video_pipe_input_mode = pipe->stream->config.mode; @@ -5858,7 +5827,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe) #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) err = send_mipi_frames(pipe); - if (err != IA_CSS_SUCCESS) + if (err) return err; #endif @@ -5876,7 +5845,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe) } } - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { coord = &pipe->config.internal_frame_origin_bqs_on_sctbl; params = pipe->stream->isp_params_configs; } @@ -5932,7 +5901,7 @@ static enum ia_css_err video_start(struct ia_css_pipe *pipe) } static -enum ia_css_err sh_css_pipe_get_viewfinder_frame_info( +int sh_css_pipe_get_viewfinder_frame_info( struct ia_css_pipe *pipe, struct ia_css_frame_info *info, unsigned int idx) @@ -5947,7 +5916,7 @@ enum ia_css_err sh_css_pipe_get_viewfinder_frame_info( if (pipe->mode == IA_CSS_PIPE_ID_CAPTURE && (pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_RAW || pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER)) - return IA_CSS_ERR_MODE_HAS_NO_VIEWFINDER; + return -EINVAL; /* offline video does not generate viewfinder output */ *info = pipe->vf_output_info[idx]; @@ -5960,27 +5929,27 @@ enum ia_css_err sh_css_pipe_get_viewfinder_frame_info( info->padded_width, info->format, info->raw_bit_depth, info->raw_bayer_order); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width, unsigned int height, unsigned int min_width, enum ia_css_frame_format format, unsigned int idx) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p, width = %d, height = %d, min_width = %d, format = %d, idx = %d\n", pipe, width, height, min_width, format, idx); if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } err = ia_css_util_check_res(width, height); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -5992,13 +5961,13 @@ sh_css_pipe_configure_viewfinder(struct ia_css_pipe *pipe, unsigned int width, ia_css_frame_info_init(&pipe->vf_output_info[idx], width, height, format, min_width); } - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -static enum ia_css_err load_copy_binaries(struct ia_css_pipe *pipe) +static int load_copy_binaries(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(pipe); IA_CSS_ENTER_PRIVATE(""); @@ -6006,14 +5975,14 @@ static enum ia_css_err load_copy_binaries(struct ia_css_pipe *pipe) assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.copy_binary.info) - return IA_CSS_SUCCESS; + return 0; err = ia_css_frame_check_info(&pipe->output_info[0]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; err = verify_copy_out_frame_format(pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; err = load_copy_binary(pipe, @@ -6034,7 +6003,7 @@ static bool need_capture_pp( assert(pipe); assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE); - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* ldc and capture_pp are not supported in the same pipeline */ if (need_capt_ldc(pipe) == true) return false; @@ -6071,13 +6040,13 @@ static bool need_capt_ldc( return (pipe->extra_config.enable_dvs_6axis) ? true : false; } -static enum ia_css_err set_num_primary_stages(unsigned int *num, +static int set_num_primary_stages(unsigned int *num, enum ia_css_pipe_version version) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (!num) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; switch (version) { case IA_CSS_PIPE_VERSION_2_6_1: @@ -6088,14 +6057,14 @@ static enum ia_css_err set_num_primary_stages(unsigned int *num, *num = NUM_PRIMARY_STAGES; break; default: - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; break; } return err; } -static enum ia_css_err load_primary_binaries( +static int load_primary_binaries( struct ia_css_pipe *pipe) { bool online = false; @@ -6113,10 +6082,11 @@ static enum ia_css_err load_primary_binaries( *vf_pp_in_info, *pipe_out_info, *pipe_vf_out_info, *capt_pp_in_info, capt_ldc_out_info; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_capture_settings *mycs; unsigned int i; bool need_extra_yuv_scaler = false; + struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES]; IA_CSS_ENTER_PRIVATE(""); assert(pipe); @@ -6136,24 +6106,24 @@ static enum ia_css_err load_primary_binaries( pipe_vf_out_info = &pipe->vf_output_info[0]; if (mycs->primary_binary[0].info) - return IA_CSS_SUCCESS; + return 0; err = set_num_primary_stages(&mycs->num_primary_stage, pipe->config.isp_pipe_version); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { err = ia_css_util_check_vf_out_info(pipe_out_info, pipe_vf_out_info); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } } else { err = ia_css_frame_check_info(pipe_out_info); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6187,18 +6157,8 @@ static enum ia_css_err load_primary_binaries( capt_pp_out_info.res.height /= MAX_PREFERRED_YUV_DS_PER_STEP; ia_css_frame_info_set_width(&capt_pp_out_info, capt_pp_out_info.res.width, 0); - /* - * WARNING: The #if def flag has been added below as a - * temporary solution to solve the problem of enabling the - * view finder in a single binary in a capture flow. The - * vf-pp stage has been removed for Skycam in the solution - * provided. The vf-pp stage should be re-introduced when - * required. This should not be considered as a clean solution. - * Proper investigation should be done to come up with the clean - * solution. - * */ need_extra_yuv_scaler = need_downscaling(capt_pp_out_info.res, - pipe_out_info->res); + pipe_out_info->res); if (need_extra_yuv_scaler) { struct ia_css_cas_binary_descr cas_scaler_descr = { }; @@ -6208,7 +6168,7 @@ static enum ia_css_err load_primary_binaries( pipe_out_info, NULL, &cas_scaler_descr); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6216,14 +6176,14 @@ static enum ia_css_err load_primary_binaries( mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage * sizeof(struct ia_css_binary), GFP_KERNEL); if (!mycs->yuv_scaler_binary) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage * sizeof(bool), GFP_KERNEL); if (!mycs->is_output_stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6238,7 +6198,7 @@ static enum ia_css_err load_primary_binaries( &cas_scaler_descr.vf_info[i]); err = ia_css_binary_find(&yuv_scaler_descr, &mycs->yuv_scaler_binary[i]); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6251,29 +6211,30 @@ static enum ia_css_err load_primary_binaries( /* TODO Do we disable ldc for skycam */ need_ldc = need_capt_ldc(pipe); - - if (atomisp_hw_is_isp2401 && need_ldc) { + if (IS_ISP2401 && need_ldc) { /* ldc and capt_pp are not supported in the same pipeline */ struct ia_css_binary_descr capt_ldc_descr; ia_css_pipe_get_ldc_binarydesc(pipe, - &capt_ldc_descr, &prim_out_info, - &capt_pp_out_info); + &capt_ldc_descr, &prim_out_info, + &capt_pp_out_info); err = ia_css_binary_find(&capt_ldc_descr, - &mycs->capture_ldc_binary); - if (err != IA_CSS_SUCCESS) { + &mycs->capture_ldc_binary); + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } need_pp = 0; need_ldc = 0; } + + /* we build up the pipeline starting at the end */ + /* Capture post-processing */ if (need_pp) { struct ia_css_binary_descr capture_pp_descr; - struct ia_css_binary_descr prim_descr[MAX_NUM_PRIMARY_STAGES]; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) capt_pp_in_info = need_ldc ? &capt_ldc_out_info : &prim_out_info; else capt_pp_in_info = &prim_out_info; @@ -6283,7 +6244,7 @@ static enum ia_css_err load_primary_binaries( &capt_pp_out_info, &vf_info); err = ia_css_binary_find(&capture_pp_descr, &mycs->capture_pp_binary); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6297,92 +6258,92 @@ static enum ia_css_err load_primary_binaries( err = ia_css_binary_find(&capt_ldc_descr, &mycs->capture_ldc_binary); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - } else { - prim_out_info = *pipe_out_info; } + } else { + prim_out_info = *pipe_out_info; + } - /* Primary */ - for (i = 0; i < mycs->num_primary_stage; i++) { - struct ia_css_frame_info *local_vf_info = NULL; + /* Primary */ + for (i = 0; i < mycs->num_primary_stage; i++) { + struct ia_css_frame_info *local_vf_info = NULL; - if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && - (i == mycs->num_primary_stage - 1)) - local_vf_info = &vf_info; - ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info, - &prim_out_info, local_vf_info, i); - err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]); - if (err != IA_CSS_SUCCESS) { - IA_CSS_LEAVE_ERR_PRIVATE(err); - return err; - } + if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0] && + (i == mycs->num_primary_stage - 1)) + local_vf_info = &vf_info; + ia_css_pipe_get_primary_binarydesc(pipe, &prim_descr[i], &prim_in_info, + &prim_out_info, local_vf_info, i); + err = ia_css_binary_find(&prim_descr[i], &mycs->primary_binary[i]); + if (err) { + IA_CSS_LEAVE_ERR_PRIVATE(err); + return err; } + } - /* Viewfinder post-processing */ - if (need_pp) - vf_pp_in_info = &mycs->capture_pp_binary.vf_frame_info; - else - vf_pp_in_info = &mycs->primary_binary[mycs->num_primary_stage - 1].vf_frame_info; + /* Viewfinder post-processing */ + if (need_pp) + vf_pp_in_info = &mycs->capture_pp_binary.vf_frame_info; + else + vf_pp_in_info = &mycs->primary_binary[mycs->num_primary_stage - 1].vf_frame_info; - /* - * WARNING: The #if def flag has been added below as a - * temporary solution to solve the problem of enabling the - * view finder in a single binary in a capture flow. The - * vf-pp stage has been removed for Skycam in the solution - * provided. The vf-pp stage should be re-introduced when - * required. Thisshould not be considered as a clean solution. - * Proper * investigation should be done to come up with the clean - * solution. - * */ - if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { - struct ia_css_binary_descr vf_pp_descr; - - ia_css_pipe_get_vfpp_binarydesc(pipe, - &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); - err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary); - if (err != IA_CSS_SUCCESS) { - IA_CSS_LEAVE_ERR_PRIVATE(err); - return err; - } - } - err = allocate_delay_frames(pipe); + /* + * WARNING: The #if def flag has been added below as a + * temporary solution to solve the problem of enabling the + * view finder in a single binary in a capture flow. The + * vf-pp stage has been removed for Skycam in the solution + * provided. The vf-pp stage should be re-introduced when + * required. Thisshould not be considered as a clean solution. + * Proper * investigation should be done to come up with the clean + * solution. + * */ + if (pipe->enable_viewfinder[IA_CSS_PIPE_OUTPUT_STAGE_0]) { + struct ia_css_binary_descr vf_pp_descr; - if (err != IA_CSS_SUCCESS) + ia_css_pipe_get_vfpp_binarydesc(pipe, + &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); + err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary); + if (err) { + IA_CSS_LEAVE_ERR_PRIVATE(err); return err; + } + } + err = allocate_delay_frames(pipe); + + if (err) + return err; #ifdef USE_INPUT_SYSTEM_VERSION_2401 - /* When the input system is 2401, only the Direct Sensor Mode - * Offline Capture uses the ISP copy binary. - */ - need_isp_copy_binary = !online && sensor; + /* When the input system is 2401, only the Direct Sensor Mode + * Offline Capture uses the ISP copy binary. + */ + need_isp_copy_binary = !online && sensor; #else - need_isp_copy_binary = !online && !continuous && !memory; + need_isp_copy_binary = !online && !continuous && !memory; #endif - /* ISP Copy */ - if (need_isp_copy_binary) { - err = load_copy_binary(pipe, - &mycs->copy_binary, - &mycs->primary_binary[0]); - if (err != IA_CSS_SUCCESS) { - IA_CSS_LEAVE_ERR_PRIVATE(err); - return err; - } + /* ISP Copy */ + if (need_isp_copy_binary) { + err = load_copy_binary(pipe, + &mycs->copy_binary, + &mycs->primary_binary[0]); + if (err) { + IA_CSS_LEAVE_ERR_PRIVATE(err); + return err; } } - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int allocate_delay_frames(struct ia_css_pipe *pipe) { unsigned int num_delay_frames = 0, i = 0; unsigned int dvs_frame_delay = 0; struct ia_css_frame_info ref_info; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; enum ia_css_pipe_id mode = IA_CSS_PIPE_ID_VIDEO; struct ia_css_frame **delay_frames = NULL; @@ -6391,7 +6352,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { if (!pipe) { IA_CSS_ERROR("Invalid args - pipe %p", pipe); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } mode = pipe->mode; @@ -6447,7 +6408,7 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { } break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } ref_info.raw_bit_depth = SH_CSS_REF_BIT_DEPTH; @@ -6456,14 +6417,14 @@ allocate_delay_frames(struct ia_css_pipe *pipe) { for (i = 0; i < num_delay_frames; i++) { err = ia_css_frame_allocate_from_info(&delay_frames[i], &ref_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; } IA_CSS_LEAVE_PRIVATE(""); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err load_advanced_binaries( +static int load_advanced_binaries( struct ia_css_pipe *pipe) { struct ia_css_frame_info pre_in_info, gdc_in_info, post_in_info, post_out_info, @@ -6471,7 +6432,7 @@ static enum ia_css_err load_advanced_binaries( *pipe_vf_out_info; bool need_pp; bool need_isp_copy = true; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE(""); @@ -6479,13 +6440,13 @@ static enum ia_css_err load_advanced_binaries( assert(pipe->mode == IA_CSS_PIPE_ID_CAPTURE || pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.pre_isp_binary.info) - return IA_CSS_SUCCESS; + return 0; pipe_out_info = &pipe->output_info[0]; pipe_vf_out_info = &pipe->vf_output_info[0]; vf_info = *pipe_vf_out_info; err = ia_css_util_check_vf_out_info(pipe_out_info, &vf_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; need_pp = need_capture_pp(pipe); @@ -6501,7 +6462,7 @@ static enum ia_css_err load_advanced_binaries( &capture_pp_descr, &post_out_info, pipe_out_info, &vf_info); err = ia_css_binary_find(&capture_pp_descr, &pipe->pipe_settings.capture.capture_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } else { post_out_info = *pipe_out_info; @@ -6515,7 +6476,7 @@ static enum ia_css_err load_advanced_binaries( &post_gdc_descr, &post_in_info, &post_out_info, &vf_info); err = ia_css_binary_find(&post_gdc_descr, &pipe->pipe_settings.capture.post_isp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -6527,7 +6488,7 @@ static enum ia_css_err load_advanced_binaries( &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); err = ia_css_binary_find(&gdc_descr, &pipe->pipe_settings.capture.anr_gdc_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } pipe->pipe_settings.capture.anr_gdc_binary.left_padding = @@ -6541,7 +6502,7 @@ static enum ia_css_err load_advanced_binaries( &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); err = ia_css_binary_find(&pre_gdc_descr, &pipe->pipe_settings.capture.pre_isp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } pipe->pipe_settings.capture.pre_isp_binary.left_padding = @@ -6563,7 +6524,7 @@ static enum ia_css_err load_advanced_binaries( &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); err = ia_css_binary_find(&vf_pp_descr, &pipe->pipe_settings.capture.vf_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -6580,10 +6541,10 @@ static enum ia_css_err load_advanced_binaries( return err; } -static enum ia_css_err load_bayer_isp_binaries( +static int load_bayer_isp_binaries( struct ia_css_pipe *pipe) { struct ia_css_frame_info pre_isp_in_info, *pipe_out_info; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_binary_descr pre_de_descr; IA_CSS_ENTER_PRIVATE(""); @@ -6593,10 +6554,10 @@ static enum ia_css_err load_bayer_isp_binaries( pipe_out_info = &pipe->output_info[0]; if (pipe->pipe_settings.capture.pre_isp_binary.info) - return IA_CSS_SUCCESS; + return 0; err = ia_css_frame_check_info(pipe_out_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; ia_css_pipe_get_pre_de_binarydesc(pipe, &pre_de_descr, @@ -6609,7 +6570,7 @@ static enum ia_css_err load_bayer_isp_binaries( return err; } -static enum ia_css_err load_low_light_binaries( +static int load_low_light_binaries( struct ia_css_pipe *pipe) { struct ia_css_frame_info pre_in_info, anr_in_info, post_in_info, post_out_info, @@ -6617,7 +6578,7 @@ static enum ia_css_err load_low_light_binaries( *vf_pp_in_info; bool need_pp; bool need_isp_copy = true; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE(""); assert(pipe); @@ -6625,14 +6586,14 @@ static enum ia_css_err load_low_light_binaries( pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.pre_isp_binary.info) - return IA_CSS_SUCCESS; + return 0; pipe_vf_out_info = &pipe->vf_output_info[0]; pipe_out_info = &pipe->output_info[0]; vf_info = *pipe_vf_out_info; err = ia_css_util_check_vf_out_info(pipe_out_info, &vf_info); - if (err != IA_CSS_SUCCESS) + if (err) return err; need_pp = need_capture_pp(pipe); @@ -6648,7 +6609,7 @@ static enum ia_css_err load_low_light_binaries( &capture_pp_descr, &post_out_info, pipe_out_info, &vf_info); err = ia_css_binary_find(&capture_pp_descr, &pipe->pipe_settings.capture.capture_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } else { post_out_info = *pipe_out_info; @@ -6662,7 +6623,7 @@ static enum ia_css_err load_low_light_binaries( &post_anr_descr, &post_in_info, &post_out_info, &vf_info); err = ia_css_binary_find(&post_anr_descr, &pipe->pipe_settings.capture.post_isp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -6674,7 +6635,7 @@ static enum ia_css_err load_low_light_binaries( &pipe->pipe_settings.capture.post_isp_binary.in_frame_info); err = ia_css_binary_find(&anr_descr, &pipe->pipe_settings.capture.anr_gdc_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } pipe->pipe_settings.capture.anr_gdc_binary.left_padding = @@ -6688,7 +6649,7 @@ static enum ia_css_err load_low_light_binaries( &pipe->pipe_settings.capture.anr_gdc_binary.in_frame_info); err = ia_css_binary_find(&pre_anr_descr, &pipe->pipe_settings.capture.pre_isp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } pipe->pipe_settings.capture.pre_isp_binary.left_padding = @@ -6710,7 +6671,7 @@ static enum ia_css_err load_low_light_binaries( &vf_pp_descr, vf_pp_in_info, pipe_vf_out_info); err = ia_css_binary_find(&vf_pp_descr, &pipe->pipe_settings.capture.vf_pp_binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; } @@ -6727,7 +6688,8 @@ static enum ia_css_err load_low_light_binaries( return err; } -static bool copy_on_sp(struct ia_css_pipe *pipe) { +static bool copy_on_sp(struct ia_css_pipe *pipe) +{ bool rval; assert(pipe); @@ -6746,9 +6708,9 @@ static bool copy_on_sp(struct ia_css_pipe *pipe) { return rval; } -static enum ia_css_err load_capture_binaries( +static int load_capture_binaries( struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool must_be_raw; IA_CSS_ENTER_PRIVATE(""); @@ -6757,8 +6719,8 @@ static enum ia_css_err load_capture_binaries( pipe->mode == IA_CSS_PIPE_ID_COPY); if (pipe->pipe_settings.capture.primary_binary[0].info) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } /* in primary, advanced,low light or bayer, @@ -6768,7 +6730,7 @@ static enum ia_css_err load_capture_binaries( pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_BAYER || pipe->config.default_capture_config.mode == IA_CSS_CAPTURE_MODE_LOW_LIGHT; err = ia_css_util_check_input(&pipe->stream->config, must_be_raw, false); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6780,15 +6742,15 @@ static enum ia_css_err load_capture_binaries( 1, IA_CSS_FRAME_FORMAT_BINARY_8, 0); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } switch (pipe->config.default_capture_config.mode) { case IA_CSS_CAPTURE_MODE_RAW: err = load_copy_binaries(pipe); #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2401) - if (err == IA_CSS_SUCCESS) + if (!err) pipe->pipe_settings.capture.copy_binary.online = pipe->stream->config.online; #endif break; @@ -6805,7 +6767,7 @@ static enum ia_css_err load_capture_binaries( err = load_low_light_binaries(pipe); break; } - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -6814,7 +6776,7 @@ static enum ia_css_err load_capture_binaries( return err; } -static enum ia_css_err +static int unload_capture_binaries(struct ia_css_pipe *pipe) { unsigned int i; @@ -6822,8 +6784,8 @@ unload_capture_binaries(struct ia_css_pipe *pipe) { if ((!pipe) || ((pipe->mode != IA_CSS_PIPE_ID_CAPTURE) && (pipe->mode != IA_CSS_PIPE_ID_COPY))) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_binary_unload(&pipe->pipe_settings.capture.copy_binary); for (i = 0; i < MAX_NUM_PRIMARY_STAGES; i++) @@ -6843,8 +6805,8 @@ unload_capture_binaries(struct ia_css_pipe *pipe) { kfree(pipe->pipe_settings.capture.yuv_scaler_binary); pipe->pipe_settings.capture.yuv_scaler_binary = NULL; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } static bool @@ -6895,14 +6857,14 @@ need_yuv_scaler_stage(const struct ia_css_pipe *pipe) { /* TODO: it is temporarily created from ia_css_pipe_create_cas_scaler_desc */ /* which has some hard-coded knowledge which prevents reuse of the function. */ /* Later, merge this with ia_css_pipe_create_cas_scaler_desc */ -static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output( +static int ia_css_pipe_create_cas_scaler_desc_single_output( struct ia_css_frame_info *cas_scaler_in_info, struct ia_css_frame_info *cas_scaler_out_info, struct ia_css_frame_info *cas_scaler_vf_info, struct ia_css_cas_binary_descr *descr) { unsigned int i; unsigned int hor_ds_factor = 0, ver_ds_factor = 0; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_frame_info tmp_in_info; unsigned int max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP; @@ -6932,30 +6894,30 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc_single_output( descr->in_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->in_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->internal_out_info = kmalloc(descr->num_stage * sizeof( struct ia_css_frame_info), GFP_KERNEL); if (!descr->internal_out_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->out_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->vf_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL); if (!descr->is_output_stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } @@ -7015,7 +6977,7 @@ ERR: } /* FIXME: merge most of this and single output version */ -static enum ia_css_err ia_css_pipe_create_cas_scaler_desc( +static int ia_css_pipe_create_cas_scaler_desc( struct ia_css_pipe *pipe, struct ia_css_cas_binary_descr *descr) { struct ia_css_frame_info in_info = IA_CSS_BINARY_DEFAULT_FRAME_INFO; @@ -7027,7 +6989,7 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc( ver_scale_factor[IA_CSS_PIPE_MAX_OUTPUT_STAGE], scale_factor = 0; unsigned int num_stages = 0; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int max_scale_factor_per_stage = MAX_PREFERRED_YUV_DS_PER_STEP; @@ -7076,30 +7038,30 @@ static enum ia_css_err ia_css_pipe_create_cas_scaler_desc( descr->in_info = kmalloc_array(descr->num_stage, sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->in_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->internal_out_info = kmalloc(descr->num_stage * sizeof( struct ia_css_frame_info), GFP_KERNEL); if (!descr->internal_out_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->out_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->out_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->vf_info = kmalloc(descr->num_stage * sizeof(struct ia_css_frame_info), GFP_KERNEL); if (!descr->vf_info) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } descr->is_output_stage = kmalloc(descr->num_stage * sizeof(bool), GFP_KERNEL); if (!descr->is_output_stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } @@ -7190,9 +7152,9 @@ static void ia_css_pipe_destroy_cas_scaler_desc(struct ia_css_cas_binary_descr "ia_css_pipe_destroy_cas_scaler_desc() leave\n"); } -static enum ia_css_err +static int load_yuvpp_binaries(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool need_scaler = false; struct ia_css_frame_info *vf_pp_in_info[IA_CSS_PIPE_MAX_OUTPUT_STAGE]; struct ia_css_yuvpp_settings *mycs; @@ -7211,7 +7173,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { /* Set both must_be_raw and must_be_yuv to false then yuvpp can take rgb inputs */ err = ia_css_util_check_input(&pipe->stream->config, false, false); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; mycs = &pipe->pipe_settings.yuvpp; @@ -7221,7 +7183,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { if (pipe->vf_output_info[i].res.width != 0) { err = ia_css_util_check_vf_out_info(&pipe->output_info[i], &pipe->vf_output_info[i]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } vf_pp_in_info[i] = NULL; @@ -7237,20 +7199,20 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { err = ia_css_pipe_create_cas_scaler_desc(pipe, &cas_scaler_descr); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; mycs->num_output = cas_scaler_descr.num_output_stage; mycs->num_yuv_scaler = cas_scaler_descr.num_stage; mycs->yuv_scaler_binary = kzalloc(cas_scaler_descr.num_stage * sizeof(struct ia_css_binary), GFP_KERNEL); if (!mycs->yuv_scaler_binary) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } mycs->is_output_stage = kzalloc(cas_scaler_descr.num_stage * sizeof(bool), GFP_KERNEL); if (!mycs->is_output_stage) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } for (i = 0; i < cas_scaler_descr.num_stage; i++) { @@ -7262,7 +7224,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { &cas_scaler_descr.vf_info[i]); err = ia_css_binary_find(&yuv_scaler_descr, &mycs->yuv_scaler_binary[i]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } ia_css_pipe_destroy_cas_scaler_desc(&cas_scaler_descr); @@ -7309,7 +7271,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { &mycs->copy_binary, next_binary); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; /* @@ -7358,7 +7320,7 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { GFP_KERNEL); if (!mycs->vf_pp_binary) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto ERR; } @@ -7371,13 +7333,13 @@ load_yuvpp_binaries(struct ia_css_pipe *pipe) { ia_css_pipe_get_vfpp_binarydesc(pipe, &vf_pp_descr, vf_pp_in_info[i], &pipe->vf_output_info[i]); err = ia_css_binary_find(&vf_pp_descr, &mycs->vf_pp_binary[i]); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } } } - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; ERR: @@ -7390,7 +7352,7 @@ ERR: return err; } -static enum ia_css_err +static int unload_yuvpp_binaries(struct ia_css_pipe *pipe) { unsigned int i; @@ -7398,8 +7360,8 @@ unload_yuvpp_binaries(struct ia_css_pipe *pipe) { if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_binary_unload(&pipe->pipe_settings.yuvpp.copy_binary); for (i = 0; i < pipe->pipe_settings.yuvpp.num_yuv_scaler; i++) @@ -7417,20 +7379,21 @@ unload_yuvpp_binaries(struct ia_css_pipe *pipe) { kfree(pipe->pipe_settings.yuvpp.vf_pp_binary); pipe->pipe_settings.yuvpp.vf_pp_binary = NULL; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) { +static int yuvpp_start(struct ia_css_pipe *pipe) +{ struct ia_css_binary *copy_binary; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; enum sh_css_pipe_config_override copy_ovrd; enum ia_css_input_mode yuvpp_pipe_input_mode; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } yuvpp_pipe_input_mode = pipe->stream->config.mode; @@ -7443,7 +7406,7 @@ static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) { #if !defined(HAS_NO_INPUT_SYSTEM) && (defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401)) err = send_mipi_frames(pipe); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7462,22 +7425,22 @@ static enum ia_css_err yuvpp_start(struct ia_css_pipe *pipe) { return err; } -static enum ia_css_err +static int sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } /* PIPE_MODE_COPY has no binaries, but has output frames to outside*/ if (pipe->config.mode == IA_CSS_PIPE_MODE_COPY) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } switch (pipe->mode) @@ -7501,9 +7464,9 @@ sh_css_pipe_unload_binaries(struct ia_css_pipe *pipe) { return err; } -static enum ia_css_err +static int sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(pipe); ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_pipe_load_binaries() enter:\n"); @@ -7529,24 +7492,24 @@ sh_css_pipe_load_binaries(struct ia_css_pipe *pipe) { case IA_CSS_PIPE_ID_ACC: break; default: - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; break; } - if (err != IA_CSS_SUCCESS) + if (err) { - if (sh_css_pipe_unload_binaries(pipe) != IA_CSS_SUCCESS) { + if (sh_css_pipe_unload_binaries(pipe)) { /* currently css does not support multiple error returns in a single function, - * using IA_CSS_ERR_INTERNAL_ERROR in this case */ - err = IA_CSS_ERR_INTERNAL_ERROR; + * using -EINVAL in this case */ + err = -EINVAL; } } return err; } -static enum ia_css_err +static int create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline *me; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipeline_stage *vf_pp_stage = NULL, *copy_stage = NULL, *yuv_scaler_stage = NULL; @@ -7573,8 +7536,8 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if ((!pipe) || (!pipe->stream) || (pipe->mode != IA_CSS_PIPE_ID_YUVPP)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } me = &pipe->pipeline; ia_css_pipeline_clean(me); @@ -7652,7 +7615,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { &me->in_frame, in_frame_format); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7668,7 +7631,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { assert(i < IA_CSS_PIPE_MAX_OUTPUT_STAGE); if (pipe->output_info[i].res.width != 0) { err = init_out_frameinfo_defaults(pipe, &me->out_frame[i], i); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7678,7 +7641,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { /* Construct vf_frame info (only in case we have VF) */ if (pipe->vf_output_info[i].res.width != 0) { err = init_vf_frameinfo_defaults(pipe, &me->vf_frame[i], i); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7715,7 +7678,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { &stage_desc, ©_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7751,7 +7714,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { &yuv_scaler_binary[i], &yuv_scaler_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7763,7 +7726,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { err = add_vf_pp_stage(pipe, in_frame, tmp_vf_frame, &vf_pp_binary[j], &vf_pp_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7778,7 +7741,7 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { err = add_vf_pp_stage(pipe, in_frame, vf_frame[0], &vf_pp_binary[0], &vf_pp_stage); } - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7786,17 +7749,17 @@ create_host_yuvpp_pipeline(struct ia_css_pipe *pipe) { ia_css_pipeline_finalize_stages(&pipe->pipeline, pipe->stream->config.continuous); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); + IA_CSS_LEAVE_ERR_PRIVATE(0); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int create_host_copy_pipeline(struct ia_css_pipe *pipe, unsigned int max_input_width, struct ia_css_frame *out_frame) { struct ia_css_pipeline *me; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipeline_stage_desc stage_desc; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, @@ -7843,10 +7806,10 @@ create_host_copy_pipeline(struct ia_css_pipe *pipe, return err; } -static enum ia_css_err +static int create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline *me = &pipe->pipeline; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_pipeline_stage_desc stage_desc; struct ia_css_frame *out_frame = &me->out_frame[0]; struct ia_css_pipeline_stage *out_stage = NULL; @@ -7860,7 +7823,7 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { /* Construct out_frame info */ err = sh_css_pipe_get_output_frame_info(pipe, &out_frame->info, 0); - if (err != IA_CSS_SUCCESS) + if (err) return err; out_frame->contiguous = false; out_frame->flash_state = IA_CSS_FRAME_FLASH_STATE_NONE; @@ -7876,7 +7839,7 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { IA_CSS_PIPELINE_ISYS_COPY, max_input_width); err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, &out_stage); - if (err != IA_CSS_SUCCESS) + if (err) return err; ia_css_pipeline_finalize_stages(me, pipe->stream->config.continuous); @@ -7887,10 +7850,10 @@ create_host_isyscopy_capture_pipeline(struct ia_css_pipe *pipe) { return err; } -static enum ia_css_err +static int create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { struct ia_css_pipeline *me; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; enum ia_css_capture_mode mode; struct ia_css_pipeline_stage *current_stage = NULL; struct ia_css_pipeline_stage *yuv_scaler_stage = NULL; @@ -7956,7 +7919,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { { err = init_in_frameinfo_memory_defaults(pipe, &me->in_frame, IA_CSS_FRAME_FORMAT_RAW); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -7968,7 +7931,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { } err = init_out_frameinfo_defaults(pipe, &me->out_frame[0], 0); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -7994,8 +7957,8 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { num_primary_stage = pipe->pipe_settings.capture.num_primary_stage; if ((num_primary_stage == 0) && (mode == IA_CSS_CAPTURE_MODE_PRIMARY)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } for (i = 0; i < num_primary_stage; i++) { @@ -8043,7 +8006,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, ¤t_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8086,7 +8049,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, ¤t_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8105,7 +8068,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { out_frames, in_frame, NULL); err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, NULL); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8114,7 +8077,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { out_frames, NULL, NULL); err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, NULL); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8131,7 +8094,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, ¤t_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8143,7 +8106,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { err = ia_css_pipeline_create_and_add_stage(me, &stage_desc, NULL); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8185,7 +8148,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { #endif capture_pp_binary, ¤t_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8206,7 +8169,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { NULL, &yuv_scaler_binary[i], &yuv_scaler_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8229,7 +8192,7 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { in_frame = current_stage->args.out_vf_frame; err = add_vf_pp_stage(pipe, in_frame, vf_frame, vf_pp_binary, ¤t_stage); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8239,12 +8202,12 @@ create_host_regular_capture_pipeline(struct ia_css_pipe *pipe) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "create_host_regular_capture_pipeline() leave:\n"); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int create_host_capture_pipeline(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); @@ -8252,7 +8215,7 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) { err = create_host_isyscopy_capture_pipeline(pipe); else err = create_host_regular_capture_pipeline(pipe); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8263,17 +8226,17 @@ create_host_capture_pipeline(struct ia_css_pipe *pipe) { return err; } -static enum ia_css_err capture_start( +static int capture_start( struct ia_css_pipe *pipe) { struct ia_css_pipeline *me; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; enum sh_css_pipe_config_override copy_ovrd; IA_CSS_ENTER_PRIVATE("pipe = %p", pipe); if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } me = &pipe->pipeline; @@ -8291,14 +8254,14 @@ static enum ia_css_err capture_start( #if defined(USE_INPUT_SYSTEM_VERSION_2) /* old isys: need to send_mipi_frames() in all pipe modes */ err = send_mipi_frames(pipe); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } #elif defined(USE_INPUT_SYSTEM_VERSION_2401) if (pipe->config.mode != IA_CSS_PIPE_MODE_COPY) { err = send_mipi_frames(pipe); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -8332,7 +8295,7 @@ static enum ia_css_err capture_start( return err; } -static enum ia_css_err +static int sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, struct ia_css_frame_info *info, unsigned int idx) { @@ -8361,7 +8324,7 @@ sh_css_pipe_get_output_frame_info(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "sh_css_pipe_get_output_frame_info() leave:\n"); - return IA_CSS_SUCCESS; + return 0; } #if !defined(HAS_NO_INPUT_SYSTEM) @@ -8448,12 +8411,13 @@ remove_firmware(struct ia_css_fw_info **l, struct ia_css_fw_info *firmware) { return; /* removing single and multiple firmware is handled in acc_unload_extension() */ } -static enum ia_css_err upload_isp_code(struct ia_css_fw_info *firmware) { - hrt_vaddress binary; +static int upload_isp_code(struct ia_css_fw_info *firmware) +{ + ia_css_ptr binary; if (!firmware) { IA_CSS_ERROR("NULL input parameter"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } binary = firmware->info.isp.xmem_addr; @@ -8473,27 +8437,27 @@ static enum ia_css_err upload_isp_code(struct ia_css_fw_info *firmware) { } if (!binary) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; - return IA_CSS_SUCCESS; + return -ENOMEM; + return 0; } -static enum ia_css_err +static int acc_load_extension(struct ia_css_fw_info *firmware) { - enum ia_css_err err; + int err; struct ia_css_fw_info *hd = firmware; while (hd) { err = upload_isp_code(hd); - if (err != IA_CSS_SUCCESS) + if (err) return err; hd = hd->next; } if (!firmware) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; firmware->loaded = true; - return IA_CSS_SUCCESS; + return 0; } static void @@ -8519,36 +8483,23 @@ acc_unload_extension(struct ia_css_fw_info *firmware) { } /* Load firmware for extension */ -static enum ia_css_err +static int ia_css_pipe_load_extension(struct ia_css_pipe *pipe, struct ia_css_fw_info *firmware) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("fw = %p pipe = %p", firmware, pipe); if ((!firmware) || (!pipe)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } if (firmware->info.isp.type == IA_CSS_ACC_OUTPUT) - { - if (&pipe->output_stage) - append_firmware(&pipe->output_stage, firmware); - else { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; - } - } else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER) - { - if (&pipe->vf_stage) - append_firmware(&pipe->vf_stage, firmware); - else { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - return IA_CSS_ERR_INTERNAL_ERROR; - } - } + append_firmware(&pipe->output_stage, firmware); + else if (firmware->info.isp.type == IA_CSS_ACC_VIEWFINDER) + append_firmware(&pipe->vf_stage, firmware); err = acc_load_extension(firmware); IA_CSS_LEAVE_ERR_PRIVATE(err); @@ -8596,12 +8547,12 @@ ia_css_pipeline_uses_params(struct ia_css_pipeline *me) { return false; } -static enum ia_css_err +static int sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, const void *acc_fw) { struct ia_css_fw_info *fw = (struct ia_css_fw_info *)acc_fw; /* In QoS case, load_extension already called, so skipping */ - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (fw->loaded == false) err = acc_load_extension(fw); @@ -8610,7 +8561,7 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, "sh_css_pipeline_add_acc_stage() enter: pipeline=%p, acc_fw=%p\n", pipeline, acc_fw); - if (err == IA_CSS_SUCCESS) + if (!err) { struct ia_css_pipeline_stage_desc stage_desc; @@ -8629,25 +8580,25 @@ sh_css_pipeline_add_acc_stage(struct ia_css_pipeline *pipeline, * @brief Tag a specific frame in continuous capture. * Refer to "sh_css_internal.h" for details. */ -enum ia_css_err ia_css_stream_capture_frame(struct ia_css_stream *stream, +int ia_css_stream_capture_frame(struct ia_css_stream *stream, unsigned int exp_id) { struct sh_css_tag_descr tag_descr; u32 encoded_tag_descr; - enum ia_css_err err; + int err; assert(stream); IA_CSS_ENTER("exp_id=%d", exp_id); /* Only continuous streams have a tagger */ if (exp_id == 0 || !stream->config.continuous) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ - IA_CSS_LEAVE_ERR(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + IA_CSS_LEAVE_ERR(-EBUSY); + return -EBUSY; } /* Create the tag descriptor from the parameters */ @@ -8668,17 +8619,17 @@ enum ia_css_err ia_css_stream_capture_frame(struct ia_css_stream *stream, * @brief Configure the continuous capture. * Refer to "sh_css_internal.h" for details. */ -enum ia_css_err ia_css_stream_capture( +int ia_css_stream_capture( struct ia_css_stream *stream, int num_captures, unsigned int skip, int offset) { struct sh_css_tag_descr tag_descr; unsigned int encoded_tag_descr; - enum ia_css_err return_err; + int return_err; if (!stream) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_capture() enter: num_captures=%d, skip=%d, offset=%d\n", @@ -8688,8 +8639,8 @@ enum ia_css_err ia_css_stream_capture( if (num_captures < SH_CSS_MINIMUM_TAG_ID) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_capture() leave: return_err=%d\n", - IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + -EINVAL); + return -EINVAL; } /* Create the tag descriptor from the parameters */ @@ -8702,7 +8653,7 @@ enum ia_css_err ia_css_stream_capture( /* SP is not running. The queues are not valid */ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_capture() leaving:queues unavailable\n"); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + return -EBUSY; } /* Enqueue the encoded tag to the host2sp queue. @@ -8718,7 +8669,8 @@ enum ia_css_err ia_css_stream_capture( return return_err; } -void ia_css_stream_request_flash(struct ia_css_stream *stream) { +void ia_css_stream_request_flash(struct ia_css_stream *stream) +{ (void)stream; assert(stream); @@ -8817,11 +8769,16 @@ sh_css_init_host_sp_control_vars(void) { } /* - * create the internal structures and fill in the configuration data - */ -void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config) { + * create the internal structures and fill in the configuration data + */ + +static const struct +ia_css_pipe_config ia_css_pipe_default_config = DEFAULT_PIPE_CONFIG; + +void ia_css_pipe_config_defaults(struct ia_css_pipe_config *pipe_config) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_config_defaults()\n"); - *pipe_config = DEFAULT_PIPE_CONFIG; + memcpy(pipe_config, &ia_css_pipe_default_config, sizeof(*pipe_config)); } void @@ -8841,7 +8798,8 @@ ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config extra_config->enable_fractional_ds = false; } -void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) { +void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) +{ ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_config_defaults()\n"); assert(stream_config); memset(stream_config, 0, sizeof(*stream_config)); @@ -8854,14 +8812,14 @@ void ia_css_stream_config_defaults(struct ia_css_stream_config *stream_config) { stream_config->source.port.rxcount = 0x04040404; } -static enum ia_css_err +static int ia_css_acc_pipe_create(struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if (!pipe) { IA_CSS_ERROR("NULL input parameter"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* There is not meaning for num_execs = 0 semantically. Run atleast once. */ @@ -8876,30 +8834,31 @@ ia_css_acc_pipe_create(struct ia_css_pipe *pipe) { return err; } -enum ia_css_err +int ia_css_pipe_create(const struct ia_css_pipe_config *config, struct ia_css_pipe **pipe) { #ifndef ISP2401 if (!config) #else - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("config = %p, pipe = %p", config, pipe); if (!config) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); #endif - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; #ifndef ISP2401 if (!pipe) #else } + if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); #endif - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; #ifndef ISP2401 return ia_css_pipe_create_extra(config, NULL, pipe); #else @@ -8907,7 +8866,7 @@ if (!pipe) err = ia_css_pipe_create_extra(config, NULL, pipe); -if (err == IA_CSS_SUCCESS) +if (err == 0) { IA_CSS_LOG("pipe created successfully = %p", *pipe); } @@ -8918,11 +8877,11 @@ return err; #endif } -enum ia_css_err +int ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, const struct ia_css_pipe_extra_config *extra_config, struct ia_css_pipe **pipe) { - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; struct ia_css_pipe *internal_pipe = NULL; unsigned int i; @@ -8931,21 +8890,21 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, /* do not allow to create more than the maximum limit */ if (my_css.pipe_counter >= IA_CSS_PIPELINE_NUM_MAX) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_EXHAUSTED); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOSPC); + return -EINVAL; } if ((!pipe) || (!config)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } ia_css_debug_dump_pipe_config(config); ia_css_debug_dump_pipe_extra_config(extra_config); err = create_pipe(config->mode, &internal_pipe, false); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -8968,8 +8927,8 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, internal_pipe->config.num_acc_stages == 0) { /* if no acc binary and no standalone stage */ *pipe = NULL; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } return ia_css_acc_pipe_create(internal_pipe); } @@ -9044,9 +9003,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, internal_pipe->config.output_info[i].padded_width, internal_pipe->config.output_info[i].format, i); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); - sh_css_free(internal_pipe); + kvfree(internal_pipe); internal_pipe = NULL; return err; } @@ -9063,9 +9022,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, internal_pipe->config.vf_output_info[i].padded_width, internal_pipe->config.vf_output_info[i].format, i); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); - sh_css_free(internal_pipe); + kvfree(internal_pipe); internal_pipe = NULL; return err; } @@ -9075,9 +9034,9 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, { err = ia_css_pipe_load_extension(internal_pipe, internal_pipe->config.acc_extension); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); - sh_css_free(internal_pipe); + kvfree(internal_pipe); return err; } } @@ -9086,11 +9045,11 @@ ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, /* all went well, return the pipe */ *pipe = internal_pipe; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } -enum ia_css_err +int ia_css_pipe_get_info(const struct ia_css_pipe *pipe, struct ia_css_pipe_info *pipe_info) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, @@ -9100,21 +9059,22 @@ ia_css_pipe_get_info(const struct ia_css_pipe *pipe, { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "ia_css_pipe_get_info: pipe_info cannot be NULL\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if (!pipe || !pipe->stream) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "ia_css_pipe_get_info: ia_css_stream_create needs to be called before ia_css_[stream/pipe]_get_info\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* we succeeded return the info */ *pipe_info = pipe->info; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_pipe_get_info() leave\n"); - return IA_CSS_SUCCESS; + return 0; } -bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) { +bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) +{ unsigned int i; if (pipe_info) { @@ -9127,38 +9087,38 @@ bool ia_css_pipe_has_dvs_stats(struct ia_css_pipe_info *pipe_info) { return false; } -enum ia_css_err +int ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe, int pin_index, enum ia_css_frame_format new_format) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("pipe = %p, pin_index = %d, new_formats = %d", pipe, pin_index, new_format); if (!pipe) { IA_CSS_ERROR("pipe is not set"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } if (0 != pin_index && 1 != pin_index) { IA_CSS_ERROR("pin index is not valid"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } if (new_format != IA_CSS_FRAME_FORMAT_NV12_TILEY) { IA_CSS_ERROR("new format is not valid"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } else { err = ia_css_pipe_check_format(pipe, new_format); - if (err == IA_CSS_SUCCESS) { + if (!err) { if (pin_index == 0) { pipe->output_info[0].format = new_format; } else { @@ -9172,7 +9132,7 @@ ia_css_pipe_override_frame_format(struct ia_css_pipe *pipe, #if defined(USE_INPUT_SYSTEM_VERSION_2) /* Configuration of INPUT_SYSTEM_VERSION_2401 is done on SP */ -static enum ia_css_err +static int ia_css_stream_configure_rx(struct ia_css_stream *stream) { struct ia_css_input_port *config; @@ -9189,10 +9149,10 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) { else if (config->num_lanes == 4) stream->csi_rx_config.mode = MONO_4L_1L_0L; else if (config->num_lanes != 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (config->port > MIPI_PORT2_ID) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; stream->csi_rx_config.port = ia_css_isys_port_to_mipi_port(config->port); stream->csi_rx_config.timeout = config->timeout; @@ -9205,11 +9165,11 @@ ia_css_stream_configure_rx(struct ia_css_stream *stream) { { /* not implemented yet, requires extension of the rx_cfg_t * struct */ - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } stream->csi_rx_config.is_two_ppc = (stream->config.pixels_per_clock == 2); stream->reconfigure_css_rx = true; - return IA_CSS_SUCCESS; + return 0; } #endif @@ -9232,18 +9192,18 @@ find_pipe(struct ia_css_pipe *pipes[], return NULL; } -static enum ia_css_err +static int ia_css_acc_stream_create(struct ia_css_stream *stream) { int i; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(stream); IA_CSS_ENTER_PRIVATE("stream = %p", stream); if (!stream) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } for (i = 0; i < stream->num_pipes; i++) @@ -9252,8 +9212,8 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { assert(pipe); if (!pipe) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } pipe->stream = stream; @@ -9261,7 +9221,7 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { /* Map SP threads before doing anything. */ err = map_sp_threads(stream, true); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -9276,7 +9236,7 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { } err = create_host_pipeline_structure(stream); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -9284,42 +9244,43 @@ ia_css_acc_stream_create(struct ia_css_stream *stream) { stream->started = false; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); + IA_CSS_LEAVE_ERR_PRIVATE(0); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int metadata_info_init(const struct ia_css_metadata_config *mdc, struct ia_css_metadata_info *md) { /* Either both width and height should be set or neither */ if ((mdc->resolution.height > 0) ^ (mdc->resolution.width > 0)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; md->resolution = mdc->resolution; /* We round up the stride to a multiple of the width * of the port going to DDR, this is a HW requirements (DMA). */ md->stride = CEIL_MUL(mdc->resolution.width, HIVE_ISP_DDR_WORD_BYTES); md->size = mdc->resolution.height * md->stride; - return IA_CSS_SUCCESS; + return 0; } /* ISP2401 */ -static enum ia_css_err check_pipe_resolutions(const struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; +static int check_pipe_resolutions(const struct ia_css_pipe *pipe) +{ + int err = 0; IA_CSS_ENTER_PRIVATE(""); if (!pipe || !pipe->stream) { IA_CSS_ERROR("null arguments"); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto EXIT; } if (ia_css_util_check_res(pipe->config.input_effective_res.width, - pipe->config.input_effective_res.height) != IA_CSS_SUCCESS) { + pipe->config.input_effective_res.height) != 0) { IA_CSS_ERROR("effective resolution not supported"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto EXIT; } if (!ia_css_util_resolution_is_zero( @@ -9327,18 +9288,18 @@ static enum ia_css_err check_pipe_resolutions(const struct ia_css_pipe *pipe) { if (!ia_css_util_res_leq(pipe->config.input_effective_res, pipe->stream->config.input_config.input_res)) { IA_CSS_ERROR("effective resolution is larger than input resolution"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto EXIT; } } if (!ia_css_util_resolution_is_even(pipe->config.output_info[0].res)) { IA_CSS_ERROR("output resolution must be even"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto EXIT; } if (!ia_css_util_resolution_is_even(pipe->config.vf_output_info[0].res)) { IA_CSS_ERROR("VF resolution must be even"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto EXIT; } EXIT: @@ -9346,7 +9307,7 @@ EXIT: return err; } -enum ia_css_err +int ia_css_stream_create(const struct ia_css_stream_config *stream_config, int num_pipes, struct ia_css_pipe *pipes[], @@ -9356,7 +9317,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, bool spcopyonly; bool sensor_binning_changed; int i, j; - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; struct ia_css_metadata_info md_info; struct ia_css_resolution effective_res; #ifdef USE_INPUT_SYSTEM_VERSION_2401 @@ -9371,7 +9332,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, !stream || !pipes) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9381,7 +9342,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, if (stream_config->input_config.format == ATOMISP_INPUT_FORMAT_BINARY_8 && stream_config->metadata_config.resolution.height > 0) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9391,7 +9352,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, if (stream_config->online && stream_config->pack_raw_pixels) { IA_CSS_LOG("online and pack raw is invalid on input system 2401"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9409,7 +9370,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, unsigned int port = (unsigned int)stream_config->source.port.port; if (port >= N_MIPI_PORT_ID) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9422,7 +9383,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_create() exit: error, need to set mipi frame size.\n"); assert(stream_config->mipi_buffer_config.size_mem_words != 0); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9437,7 +9398,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_create() exit: error, need to set number of mipi frames.\n"); assert(stream_config->mipi_buffer_config.nof_mipi_buffers != 0); - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -9446,22 +9407,21 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Currently we only supported metadata up to a certain size. */ err = metadata_info_init(&stream_config->metadata_config, &md_info); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; } /* allocate the stream instance */ - curr_stream = kmalloc(sizeof(struct ia_css_stream), GFP_KERNEL); + curr_stream = kzalloc(sizeof(struct ia_css_stream), GFP_KERNEL); if (!curr_stream) { - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_LEAVE_ERR(err); return err; } /* default all to 0 */ - memset(curr_stream, 0, sizeof(struct ia_css_stream)); curr_stream->info.metadata_info = md_info; /* allocate pipes */ @@ -9472,7 +9432,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, curr_stream->num_pipes = 0; kfree(curr_stream); curr_stream = NULL; - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_LEAVE_ERR(err); return err; } @@ -9553,7 +9513,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, err = aspect_ratio_crop_init(curr_stream, pipes, &aspect_ratio_crop_enabled); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; @@ -9579,7 +9539,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, struct ia_css_resolution crop_res; err = aspect_ratio_crop(curr_pipe, &crop_res); - if (err == IA_CSS_SUCCESS) { + if (!err) { effective_res = crop_res; } else { /* in case of error fallback to default @@ -9595,12 +9555,12 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, effective_res.height); } - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { for (i = 0; i < num_pipes; i++) { if (pipes[i]->config.mode != IA_CSS_PIPE_MODE_ACC && pipes[i]->config.mode != IA_CSS_PIPE_MODE_COPY) { err = check_pipe_resolutions(pipes[i]); - if (err != IA_CSS_SUCCESS) { + if (err) { goto ERR; } } @@ -9608,7 +9568,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, } err = ia_css_stream_isp_parameters_init(curr_stream); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; IA_CSS_LOG("isp_params_configs: %p", curr_stream->isp_params_configs); @@ -9649,7 +9609,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, curr_stream->cont_capt = true; curr_stream->disable_cont_vf = curr_stream->config.disable_cont_viewfinder; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) curr_stream->stop_copy_preview = my_css.stop_copy_preview; } @@ -9667,19 +9627,19 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, capture_pipe = find_pipe(pipes, num_pipes, IA_CSS_PIPE_MODE_CAPTURE, false); if (!capture_pipe) { - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto ERR; } } /* We do not support preview and video pipe at the same time */ if (preview_pipe && video_pipe) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto ERR; } if (preview_pipe && !preview_pipe->pipe_settings.preview.copy_pipe) { err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, ©_pipe, true); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; ia_css_pipe_config_defaults(©_pipe->config); preview_pipe->pipe_settings.preview.copy_pipe = copy_pipe; @@ -9690,7 +9650,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, } if (video_pipe && !video_pipe->pipe_settings.video.copy_pipe) { err = create_pipe(IA_CSS_PIPE_MODE_CAPTURE, ©_pipe, true); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; ia_css_pipe_config_defaults(©_pipe->config); video_pipe->pipe_settings.video.copy_pipe = copy_pipe; @@ -9709,14 +9669,14 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* set current stream */ curr_pipe->stream = curr_stream; - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { /* take over effective info */ effective_res = curr_pipe->config.input_effective_res; err = ia_css_util_check_res( effective_res.width, effective_res.height); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } /* sensor binning per pipe */ @@ -9732,7 +9692,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, curr_pipe = pipes[i]; err = sh_css_pipe_load_binaries(curr_pipe); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; /* handle each pipe */ @@ -9740,31 +9700,31 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) { err = sh_css_pipe_get_output_frame_info(curr_pipe, &pipe_info->output_info[j], j); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } - if (atomisp_hw_is_isp2401) + if (IS_ISP2401) pipe_info->output_system_in_res_info = curr_pipe->config.output_system_in_res; if (!spcopyonly) { - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) err = sh_css_pipe_get_shading_info(curr_pipe, &pipe_info->shading_info, NULL); else err = sh_css_pipe_get_shading_info(curr_pipe, &pipe_info->shading_info, &curr_pipe->config); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; err = sh_css_pipe_get_grid_info(curr_pipe, &pipe_info->grid_info); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; for (j = 0; j < IA_CSS_PIPE_MAX_OUTPUT_STAGE; j++) { sh_css_pipe_get_viewfinder_frame_info(curr_pipe, &pipe_info->vf_output_info[j], j); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; } } @@ -9776,7 +9736,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Map SP threads before doing anything. */ err = map_sp_threads(curr_stream, true); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LOG("map_sp_threads: return_err=%d", err); goto ERR; @@ -9790,7 +9750,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, /* Create host side pipeline objects without stages */ err = create_host_pipeline_structure(curr_stream); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LOG("create_host_pipeline_structure: return_err=%d", err); goto ERR; @@ -9800,7 +9760,7 @@ ia_css_stream_create(const struct ia_css_stream_config *stream_config, *stream = curr_stream; ERR: - if (err == IA_CSS_SUCCESS) { + if (!err) { /* working mode: enter into the seed list */ if (my_css_save.mode == sh_css_mode_working) { for (i = 0; i < MAX_ACTIVE_STREAMS; i++) { @@ -9828,15 +9788,15 @@ ERR: return err; } -enum ia_css_err +int ia_css_stream_destroy(struct ia_css_stream *stream) { int i; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("stream = %p", stream); if (!stream) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -9859,7 +9819,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { /* get the SP thread id */ if (ia_css_pipeline_get_sp_thread_id( ia_css_pipe_get_pipe_num(entry), &sp_thread_id) != true) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; /* get the target input terminal */ sp_pipeline_input_terminal = &sh_css_sp_group.pipe_io[sp_thread_id].input; @@ -9873,7 +9833,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { } } free_mpi = stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { free_mpi |= stream->config.mode == IA_CSS_INPUT_MODE_TPG; free_mpi |= stream->config.mode == IA_CSS_INPUT_MODE_PRBS; } @@ -9900,7 +9860,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { } err = map_sp_threads(stream, false); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -9952,7 +9912,7 @@ ia_css_stream_destroy(struct ia_css_stream *stream) { return err; } -enum ia_css_err +int ia_css_stream_get_info(const struct ia_css_stream *stream, struct ia_css_stream_info *stream_info) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_get_info: enter/exit\n"); @@ -9960,7 +9920,7 @@ ia_css_stream_get_info(const struct ia_css_stream *stream, assert(stream_info); *stream_info = stream->info; - return IA_CSS_SUCCESS; + return 0; } /* @@ -9969,12 +9929,11 @@ ia_css_stream_get_info(const struct ia_css_stream *stream, * The data is taken from the css_save struct updated upon stream creation. * The stream handle is used to identify the correct entry in the css_save struct */ -enum ia_css_err +int ia_css_stream_load(struct ia_css_stream *stream) { - - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { int i; - enum ia_css_err err; + int err; assert(stream); ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() enter,\n"); @@ -9985,7 +9944,7 @@ ia_css_stream_load(struct ia_css_stream *stream) { for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) { if ((err = ia_css_pipe_create(&my_css_save.stream_seeds[i].pipe_config[j], - &my_css_save.stream_seeds[i].pipes[j])) != IA_CSS_SUCCESS) { + &my_css_save.stream_seeds[i].pipes[j])) != 0) { if (j) { int k; @@ -9999,7 +9958,7 @@ ia_css_stream_load(struct ia_css_stream *stream) { my_css_save.stream_seeds[i].num_pipes, my_css_save.stream_seeds[i].pipes, &my_css_save.stream_seeds[i].stream); - if (err != IA_CSS_SUCCESS) { + if (err) { ia_css_stream_destroy(stream); for (j = 0; j < my_css_save.stream_seeds[i].num_pipes; j++) ia_css_pipe_destroy(my_css_save.stream_seeds[i].pipes[j]); @@ -10009,23 +9968,23 @@ ia_css_stream_load(struct ia_css_stream *stream) { } } ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_load() exit,\n"); - return IA_CSS_SUCCESS; + return 0; } else { /* TODO remove function - DEPRECATED */ (void)stream; - return IA_CSS_ERR_NOT_SUPPORTED; + return -ENOTSUPP; } } -enum ia_css_err +int ia_css_stream_start(struct ia_css_stream *stream) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER("stream = %p", stream); if ((!stream) || (!stream->last_pipe)) { - IA_CSS_LEAVE_ERR(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR(-EINVAL); + return -EINVAL; } IA_CSS_LOG("starting %d", stream->last_pipe->mode); @@ -10033,7 +9992,7 @@ ia_css_stream_start(struct ia_css_stream *stream) { /* Create host side pipeline. */ err = create_host_pipeline(stream); - if (err != IA_CSS_SUCCESS) + if (err) { IA_CSS_LEAVE_ERR(err); return err; @@ -10065,7 +10024,7 @@ ia_css_stream_start(struct ia_css_stream *stream) { if (stream->config.mode != IA_CSS_INPUT_MODE_MEMORY) { err = sh_css_config_input_network(stream); - if (err != IA_CSS_SUCCESS) + if (err) return err; } #endif /* !HAS_NO_INPUT_SYSTEM */ @@ -10075,9 +10034,9 @@ ia_css_stream_start(struct ia_css_stream *stream) { return err; } -enum ia_css_err +int ia_css_stream_stop(struct ia_css_stream *stream) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_stop() enter/exit\n"); assert(stream); @@ -10098,13 +10057,13 @@ ia_css_stream_stop(struct ia_css_stream *stream) { } #endif - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { err = ia_css_pipeline_request_stop(&stream->last_pipe->pipeline); } else { err = sh_css_pipes_stop(stream); } - if (err != IA_CSS_SUCCESS) + if (err) return err; /* Ideally, unmapping should happen after pipeline_stop, but current @@ -10120,7 +10079,7 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream) { assert(stream); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { stopped = ia_css_pipeline_has_stopped(&stream->last_pipe->pipeline); } else { stopped = sh_css_pipes_have_stopped(stream); @@ -10134,7 +10093,7 @@ ia_css_stream_has_stopped(struct ia_css_stream *stream) { * Destroy the stream and all the pipes related to it. * The stream handle is used to identify the correct entry in the css_save struct */ -enum ia_css_err +int ia_css_stream_unload(struct ia_css_stream *stream) { int i; @@ -10159,10 +10118,10 @@ ia_css_stream_unload(struct ia_css_stream *stream) { break; } ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_stream_unload() exit,\n"); - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, enum ia_css_pipe_id *pipe_id) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "ia_css_temp_pipe_to_pipe_id() enter/exit\n"); @@ -10171,7 +10130,7 @@ ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, else *pipe_id = IA_CSS_PIPE_ID_COPY; - return IA_CSS_SUCCESS; + return 0; } enum atomisp_input_format @@ -10243,7 +10202,7 @@ ia_css_stream_get_3a_binary(const struct ia_css_stream *stream) { return s3a_binary; } -enum ia_css_err +int ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, unsigned int output_padded_width) { struct ia_css_pipe *pipe; @@ -10258,7 +10217,7 @@ ia_css_stream_set_output_padded_width(struct ia_css_stream *stream, pipe->config.output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width; pipe->output_info[IA_CSS_PIPE_OUTPUT_STAGE_0].padded_width = output_padded_width; - return IA_CSS_SUCCESS; + return 0; } static struct ia_css_binary * @@ -10406,10 +10365,10 @@ ia_css_pipe_get_isp_pipe_version(const struct ia_css_pipe *pipe) { #define SP_START_TIMEOUT_US 30000000 -enum ia_css_err +int ia_css_start_sp(void) { unsigned long timeout; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER(""); sh_css_sp_start_isp(); @@ -10419,12 +10378,12 @@ ia_css_start_sp(void) { while ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_INITIALIZED) && timeout) { timeout--; - hrt_sleep(); + udelay(1); } if (timeout == 0) { IA_CSS_ERROR("timeout during SP initialization"); - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } /* Workaround, in order to run two streams in parallel. See TASK 4271*/ @@ -10449,16 +10408,16 @@ ia_css_start_sp(void) { */ #define SP_SHUTDOWN_TIMEOUT_US 200000 -enum ia_css_err +int ia_css_stop_sp(void) { unsigned long timeout; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER("void"); if (!sh_css_sp_is_running()) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE("SP already stopped : return_err=%d", err); /* Return an error - stop SP should not have been called by driver */ @@ -10466,7 +10425,7 @@ ia_css_stop_sp(void) { } /* For now, stop whole SP */ - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { sh_css_write_host2sp_command(host2sp_cmd_terminate); } else { if (!sh_css_write_host2sp_command(host2sp_cmd_terminate)) @@ -10483,7 +10442,7 @@ ia_css_stop_sp(void) { while (!ia_css_spctrl_is_idle(SP0_ID) && timeout) { timeout--; - hrt_sleep(); + udelay(1); } if ((ia_css_spctrl_get_state(SP0_ID) != IA_CSS_SP_SW_TERMINATED)) IA_CSS_WARNING("SP has not terminated (SW)"); @@ -10497,7 +10456,7 @@ ia_css_stop_sp(void) { while (!isp_ctrl_getbit(ISP0_ID, ISP_SC_REG, ISP_IDLE_BIT) && timeout) { timeout--; - hrt_sleep(); + udelay(1); } if (timeout == 0) { @@ -10514,7 +10473,7 @@ ia_css_stop_sp(void) { return err; } -enum ia_css_err +int ia_css_update_continuous_frames(struct ia_css_stream *stream) { struct ia_css_pipe *pipe; unsigned int i; @@ -10528,7 +10487,7 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) { ia_css_debug_dtrace( IA_CSS_DEBUG_TRACE, "sh_css_update_continuous_frames() leave: invalid stream, return_void\n"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } pipe = stream->continuous_pipe; @@ -10545,10 +10504,11 @@ ia_css_update_continuous_frames(struct ia_css_stream *stream) { IA_CSS_DEBUG_TRACE, "sh_css_update_continuous_frames() leave: return_void\n"); - return IA_CSS_SUCCESS; + return 0; } -void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) { +void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) +{ unsigned int thread_id; enum ia_css_pipe_id pipe_id; unsigned int pipe_num; @@ -10664,7 +10624,7 @@ void ia_css_pipe_map_queue(struct ia_css_pipe *pipe, bool map) { } #if CONFIG_ON_FRAME_ENQUEUE() -static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info +static int set_config_on_frame_enqueue(struct ia_css_frame_info *info, struct frame_data_wrapper *frame) { frame->config_on_frame_enqueue.padded_width = 0; @@ -10676,7 +10636,7 @@ static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info if (info->padded_width > info->res.width) { frame->config_on_frame_enqueue.padded_width = info->padded_width; } else if ((info->padded_width < info->res.width) && (info->padded_width > 0)) { - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* nothing to do if width == padded width or padded width is zeroed (the same) */ break; @@ -10684,13 +10644,13 @@ static enum ia_css_err set_config_on_frame_enqueue(struct ia_css_frame_info break; } - return IA_CSS_SUCCESS; + return 0; } #endif -enum ia_css_err +int ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) { - enum ia_css_err ret; + int ret; IA_CSS_ENTER(""); @@ -10699,14 +10659,14 @@ ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) { if (!stream || !stream->config.continuous) { IA_CSS_ERROR("invalid stream pointer"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if (exp_id > IA_CSS_ISYS_MAX_EXPOSURE_ID || exp_id < IA_CSS_ISYS_MIN_EXPOSURE_ID) { - IA_CSS_ERROR("invalid expsure ID: %d\n", exp_id); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_ERROR("invalid exposure ID: %d\n", exp_id); + return -EINVAL; } /* Send the event. Since we verified that the exp_id is valid, @@ -10721,12 +10681,12 @@ ia_css_unlock_raw_frame(struct ia_css_stream *stream, uint32_t exp_id) { /* @brief Set the state (Enable or Disable) of the Extension stage in the * given pipe. */ -enum ia_css_err +int ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, bool enable) { unsigned int thread_id; struct ia_css_pipeline_stage *stage; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER(""); @@ -10734,28 +10694,28 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe(No Extension Firmware)"); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); - err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + err = -EBUSY; } else { /* Query the threadid and stage_num for the Extension firmware*/ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); - if (err == IA_CSS_SUCCESS) { + if (!err) { /* Set the Extension State;. TODO: Add check for stage firmware.type (QOS)*/ err = ia_css_bufq_enqueue_psys_event( (uint8_t)IA_CSS_PSYS_SW_EVENT_STAGE_ENABLE_DISABLE, (uint8_t)thread_id, (uint8_t)stage->stage_num, enable ? 1 : 0); - if (err == IA_CSS_SUCCESS) { + if (!err) { if (enable) SH_CSS_QOS_STAGE_ENABLE(&sh_css_sp_group.pipe[thread_id], stage->stage_num); else @@ -10770,12 +10730,12 @@ ia_css_pipe_set_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, /* @brief Get the state (Enable or Disable) of the Extension stage in the * given pipe. */ -enum ia_css_err +int ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, bool *enable) { struct ia_css_pipeline_stage *stage; unsigned int thread_id; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER(""); @@ -10783,22 +10743,22 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe (No Extension Firmware)."); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); - err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + err = -EBUSY; } else { /* Query the threadid and stage_num corresponding to the Extension firmware*/ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); - if (err == IA_CSS_SUCCESS) { + if (!err) { /* Get the Extension State */ *enable = (SH_CSS_QOS_STAGE_IS_ENABLED(&sh_css_sp_group.pipe[thread_id], stage->stage_num)) ? true : false; @@ -10809,9 +10769,9 @@ ia_css_pipe_get_qos_ext_state(struct ia_css_pipe *pipe, uint32_t fw_handle, } /* ISP2401 */ -enum ia_css_err +int ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, - uint32_t fw_handle, + u32 fw_handle, struct ia_css_isp_param_css_segments *css_seg, struct ia_css_isp_param_isp_segments *isp_seg) { unsigned int HIVE_ADDR_sp_group; @@ -10821,7 +10781,7 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, const struct ia_css_fw_info *fw; unsigned int thread_id; struct ia_css_pipeline_stage *stage; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; int stage_num = 0; enum ia_css_isp_memories mem; bool enabled; @@ -10834,28 +10794,28 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, if (!pipe || !pipe->stream) { IA_CSS_ERROR("Invalid Pipe."); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!(pipe->config.acc_extension)) { IA_CSS_ERROR("Invalid Pipe (No Extension Firmware)."); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } else if (!sh_css_sp_is_running()) { IA_CSS_ERROR("Leaving: queue unavailable."); - err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + err = -EBUSY; } else { /* Query the thread_id and stage_num corresponding to the Extension firmware */ ia_css_pipeline_get_sp_thread_id(ia_css_pipe_get_pipe_num(pipe), &thread_id); err = ia_css_pipeline_get_stage_from_fw(&pipe->pipeline, fw_handle, &stage); - if (err == IA_CSS_SUCCESS) { + if (!err) { /* Get the Extension State */ enabled = (SH_CSS_QOS_STAGE_IS_ENABLED(&sh_css_sp_group.pipe[thread_id], stage->stage_num)) ? true : false; /* Update mapped arg only when extension stage is not enabled */ if (enabled) { IA_CSS_ERROR("Leaving: cannot update when stage is enabled."); - err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + err = -EBUSY; } else { stage_num = stage->stage_num; @@ -10863,10 +10823,10 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, sp_dmem_load(SP0_ID, (unsigned int)sp_address_of(sp_group), &sp_group, sizeof(struct sh_css_sp_group)); - mmgr_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num], + hmm_load(sp_group.pipe[thread_id].sp_stage_addr[stage_num], &sp_stage, sizeof(struct sh_css_sp_stage)); - mmgr_load(sp_stage.isp_stage_addr, + hmm_load(sp_stage.isp_stage_addr, &isp_stage, sizeof(struct sh_css_isp_stage)); for (mem = 0; mem < N_IA_CSS_ISP_MEMORIES; mem++) { @@ -10882,7 +10842,7 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, isp_seg->params[IA_CSS_PARAM_CLASS_PARAM][mem].size; } - mmgr_store(sp_stage.isp_stage_addr, + hmm_store(sp_stage.isp_stage_addr, &isp_stage, sizeof(struct sh_css_isp_stage)); } } @@ -10892,11 +10852,11 @@ ia_css_pipe_update_qos_ext_mapped_arg(struct ia_css_pipe *pipe, } #ifdef USE_INPUT_SYSTEM_VERSION_2401 -static enum ia_css_err +static int aspect_ratio_crop_init(struct ia_css_stream *curr_stream, struct ia_css_pipe *pipes[], bool *do_crop_status) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; int i; struct ia_css_pipe *curr_pipe; u32 pipe_mask = 0; @@ -10906,7 +10866,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream, (!pipes) || (!do_crop_status)) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -10922,7 +10882,7 @@ aspect_ratio_crop_init(struct ia_css_stream *curr_stream, (pipe_mask & (1 << IA_CSS_PIPE_MODE_VIDEO))) && (pipe_mask & (1 << IA_CSS_PIPE_MODE_CAPTURE)) && curr_stream->config.continuous); - return IA_CSS_SUCCESS; + return 0; } static bool @@ -10939,10 +10899,10 @@ aspect_ratio_crop_check(bool enabled, struct ia_css_pipe *curr_pipe) { return status; } -static enum ia_css_err +static int aspect_ratio_crop(struct ia_css_pipe *curr_pipe, struct ia_css_resolution *effective_res) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_resolution crop_res; struct ia_css_resolution *in_res = NULL; struct ia_css_resolution *out_res = NULL; @@ -10953,7 +10913,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, if ((!curr_pipe) || (!effective_res)) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -10962,7 +10922,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, (curr_pipe->config.mode != IA_CSS_PIPE_MODE_VIDEO) && (curr_pipe->config.mode != IA_CSS_PIPE_MODE_CAPTURE)) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR(err); return err; } @@ -11009,7 +10969,7 @@ aspect_ratio_crop(struct ia_css_pipe *curr_pipe, } err = ia_css_frame_find_crop_resolution(in_res, out_res, &crop_res); - if (err == IA_CSS_SUCCESS) + if (!err) { *effective_res = crop_res; } else @@ -11085,7 +11045,7 @@ static struct sh_css_hmm_buffer_record } static struct sh_css_hmm_buffer_record -*sh_css_hmm_buffer_record_validate(hrt_vaddress ddr_buffer_addr, +*sh_css_hmm_buffer_record_validate(ia_css_ptr ddr_buffer_addr, enum ia_css_buffer_type type) { int i; struct sh_css_hmm_buffer_record *buffer_record = NULL; diff --git a/drivers/staging/media/atomisp/pci/sh_css_defs.h b/drivers/staging/media/atomisp/pci/sh_css_defs.h index fcd5081edf82..92d80213860f 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_defs.h +++ b/drivers/staging/media/atomisp/pci/sh_css_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -177,7 +178,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191] #define SH_CSS_MORPH_TABLE_ELEMS_PER_DDR_WORD \ (HIVE_ISP_DDR_WORD_BYTES / SH_CSS_MORPH_TABLE_ELEM_BYTES) - #define ISP2400_SH_CSS_MAX_SCTBL_WIDTH_PER_COLOR (SH_CSS_MAX_BQ_GRID_WIDTH + 1) #define ISP2400_SH_CSS_MAX_SCTBL_HEIGHT_PER_COLOR (SH_CSS_MAX_BQ_GRID_HEIGHT + 1) @@ -225,8 +225,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191] #define NUM_VIDEO_TNR_FRAMES 2 -#define MAX_NUM_DELAY_FRAMES MAX(MAX_NUM_VIDEO_DELAY_FRAMES, NUM_PREVIEW_DVS_FRAMES) - /* Note that this is the define used to configure all data structures common for all modes */ /* It should be equal or bigger to the max number of DVS frames for all possible modes */ /* Rules: these implement logic shared between the host code and ISP firmware. @@ -277,7 +275,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191] (ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + \ SH_CSS_SCTBL_CENTERING_MARGIN + SH_CSS_SCTBL_LAST_GRID_COUNT) - /* ISP2401: Legacy API: Number of horizontal grids per color in the shading table. */ #define _ISP_SCTBL_LEGACY_WIDTH_PER_COLOR(input_width, deci_factor_log2) \ (ISP_BQ_GRID_WIDTH(input_width, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT) @@ -286,7 +283,6 @@ RGB[0,8191],coef[-8192,8191] -> RGB[0,8191] #define _ISP_SCTBL_LEGACY_HEIGHT(input_height, deci_factor_log2) \ (ISP_BQ_GRID_HEIGHT(input_height, deci_factor_log2) + SH_CSS_SCTBL_LAST_GRID_COUNT) - /* ***************************************************************** * Statistics for 3A (Auto Focus, Auto White Balance, Auto Exposure) * *****************************************************************/ diff --git a/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h b/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h index 23044aad654f..6f058f132300 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h +++ b/drivers/staging/media/atomisp/pci/sh_css_dvs_info.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /** Support for Intel Camera Imaging ISP subsystem. Copyright (c) 2010 - 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_firmware.c b/drivers/staging/media/atomisp/pci/sh_css_firmware.c index eb3c01574853..d4ab15b6d1ac 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_firmware.c +++ b/drivers/staging/media/atomisp/pci/sh_css_firmware.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,9 +13,12 @@ * more details. */ +#include <linux/string.h> /* for memcpy() */ #include <linux/slab.h> #include <linux/vmalloc.h> +#include "hmm.h" + #include <math_support.h> #include "platform_support.h" #include "sh_css_firmware.h" @@ -24,9 +28,7 @@ #include "sh_css_internal.h" #include "ia_css_isp_param.h" -#include "memory_access.h" #include "assert_support.h" -#include "string_support.h" #include "isp.h" /* PMEM_WIDTH_LOG2 */ @@ -74,13 +76,13 @@ char *sh_css_get_fw_version(void) */ /* Setup sp/sp1 binary */ -static enum ia_css_err +static int setup_binary(struct ia_css_fw_info *fw, const char *fw_data, struct ia_css_fw_info *sh_css_fw, unsigned int binary_id) { const char *blob_data; if ((!fw) || (!fw_data)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; blob_data = fw_data + fw->blob.offset; @@ -88,16 +90,16 @@ setup_binary(struct ia_css_fw_info *fw, const char *fw_data, sh_css_fw->blob.code = vmalloc(fw->blob.size); if (!sh_css_fw->blob.code) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; memcpy((void *)sh_css_fw->blob.code, blob_data, fw->blob.size); sh_css_fw->blob.data = (char *)sh_css_fw->blob.code + fw->blob.data_source; fw_minibuffer[binary_id].buffer = sh_css_fw->blob.code; - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, struct ia_css_blob_descr *bd, unsigned int index) { @@ -105,7 +107,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, const unsigned char *blob; if ((!fw) || (!bd)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; /* Special case: only one binary in fw */ if (!bi) bi = (const struct ia_css_fw_info *)fw; @@ -117,11 +119,11 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, if (bi->blob.size != bi->blob.text_size + bi->blob.icache_size + bi->blob.data_size + bi->blob.padding_size) { /* sanity check, note the padding bytes added for section to DDR alignment */ - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if ((bi->blob.offset % (1UL << (ISP_PMEM_WIDTH_LOG2 - 3))) != 0) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; bd->blob = blob; bd->header = *bi; @@ -132,7 +134,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, namebuffer = kstrdup(name, GFP_KERNEL); if (!namebuffer) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; bd->name = fw_minibuffer[index].name = namebuffer; } else { @@ -149,7 +151,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, statestruct_size, GFP_KERNEL); if (!parambuf) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; bd->mem_offsets.array[IA_CSS_PARAM_CLASS_PARAM].ptr = NULL; bd->mem_offsets.array[IA_CSS_PARAM_CLASS_CONFIG].ptr = NULL; @@ -177,7 +179,7 @@ sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, bd->mem_offsets.array[IA_CSS_PARAM_CLASS_STATE].ptr = parambuf + paramstruct_size + configstruct_size; } - return IA_CSS_SUCCESS; + return 0; } bool @@ -187,7 +189,7 @@ sh_css_check_firmware_version(struct device *dev, const char *fw_data) const char *release_version; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) release_version = isp2400_release_version; else release_version = isp2401_release_version; @@ -202,19 +204,33 @@ sh_css_check_firmware_version(struct device *dev, const char *fw_data) } /* For now, let's just accept a wrong version, even if wrong */ - return true; + return 0; } -enum ia_css_err +static const char * const fw_type_name[] = { + [ia_css_sp_firmware] = "SP", + [ia_css_isp_firmware] = "ISP", + [ia_css_bootloader_firmware] = "BootLoader", + [ia_css_acc_firmware] = "accel", +}; + +static const char * const fw_acc_type_name[] = { + [IA_CSS_ACC_NONE] = "Normal", + [IA_CSS_ACC_OUTPUT] = "Accel for output", + [IA_CSS_ACC_VIEWFINDER] = "Accel for viewfinder", + [IA_CSS_ACC_STANDALONE] = "Stand-alone accel", +}; + +int sh_css_load_firmware(struct device *dev, const char *fw_data, unsigned int fw_size) { unsigned int i; struct ia_css_fw_info *binaries; struct sh_css_fw_bi_file_h *file_header; - bool valid_firmware = false; + int ret; const char *release_version; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) release_version = isp2400_release_version; else release_version = isp2401_release_version; @@ -223,21 +239,21 @@ sh_css_load_firmware(struct device *dev, const char *fw_data, file_header = &firmware_header->file_header; binaries = &firmware_header->binary_header; strscpy(FW_rel_ver_name, file_header->version, min(sizeof(FW_rel_ver_name), sizeof(file_header->version))); - valid_firmware = sh_css_check_firmware_version(dev, fw_data); - if (!valid_firmware) { + ret = sh_css_check_firmware_version(dev, fw_data); + if (ret) { IA_CSS_ERROR("CSS code version (%s) and firmware version (%s) mismatch!", file_header->version, release_version); - return IA_CSS_ERR_VERSION_MISMATCH; + return -EINVAL; } else { IA_CSS_LOG("successfully load firmware version %s", release_version); } /* some sanity checks */ if (!fw_data || fw_size < sizeof(struct sh_css_fw_bi_file_h)) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; if (file_header->h_size != sizeof(struct sh_css_fw_bi_file_h)) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; sh_css_num_binaries = file_header->binary_nr; /* Only allocate memory for ISP blob info */ @@ -247,7 +263,7 @@ sh_css_load_firmware(struct device *dev, const char *fw_data, (sh_css_num_binaries - NUM_OF_SPS) * sizeof(*sh_css_blob_info), GFP_KERNEL); if (!sh_css_blob_info) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; } else { sh_css_blob_info = NULL; } @@ -255,7 +271,7 @@ sh_css_load_firmware(struct device *dev, const char *fw_data, fw_minibuffer = kcalloc(sh_css_num_binaries, sizeof(struct fw_param), GFP_KERNEL); if (!fw_minibuffer) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; for (i = 0; i < sh_css_num_binaries; i++) { @@ -265,36 +281,71 @@ sh_css_load_firmware(struct device *dev, const char *fw_data, cause issues for drivers */ static struct ia_css_blob_descr bd; - enum ia_css_err err; + int err; err = sh_css_load_blob_info(fw_data, bi, &bd, i); - if (err != IA_CSS_SUCCESS) - return IA_CSS_ERR_INTERNAL_ERROR; + if (err) + return -EINVAL; if (bi->blob.offset + bi->blob.size > fw_size) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; + + switch (bd.header.type) { + case ia_css_isp_firmware: + if (bd.header.info.isp.type > IA_CSS_ACC_STANDALONE) { + dev_err(dev, "binary #%2d: invalid SP type\n", + i); + return -EINVAL; + } + + dev_dbg(dev, + "binary #%-2d type %s (%s), binary id is %2d: %s\n", + i, + fw_type_name[bd.header.type], + fw_acc_type_name[bd.header.info.isp.type], + bd.header.info.isp.sp.id, + bd.name); + break; + case ia_css_sp_firmware: + case ia_css_bootloader_firmware: + case ia_css_acc_firmware: + dev_dbg(dev, + "binary #%-2d type %s: %s\n", + i, fw_type_name[bd.header.type], + bd.name); + break; + default: + if (bd.header.info.isp.type > IA_CSS_ACC_STANDALONE) { + dev_err(dev, + "binary #%2d: invalid firmware type\n", + i); + return -EINVAL; + } + break; + } if (bi->type == ia_css_sp_firmware) { if (i != SP_FIRMWARE) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; err = setup_binary(bi, fw_data, &sh_css_sp_fw, i); - if (err != IA_CSS_SUCCESS) + if (err) return err; + } else { /* All subsequent binaries (including bootloaders) (i>NUM_OF_SPS) are ISP firmware */ if (i < NUM_OF_SPS) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; if (bi->type != ia_css_isp_firmware) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; if (!sh_css_blob_info) /* cannot happen but KW does not see this */ - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; sh_css_blob_info[i - NUM_OF_SPS] = bd; } } - return IA_CSS_SUCCESS; + return 0; } void sh_css_unload_firmware(void) @@ -319,15 +370,15 @@ void sh_css_unload_firmware(void) sh_css_num_binaries = 0; } -hrt_vaddress +ia_css_ptr sh_css_load_blob(const unsigned char *blob, unsigned int size) { - hrt_vaddress target_addr = mmgr_malloc(size); + ia_css_ptr target_addr = hmm_alloc(size, HMM_BO_PRIVATE, 0, NULL, 0); /* this will allocate memory aligned to a DDR word boundary which is required for the CSS DMA to read the instructions. */ assert(blob); if (target_addr) - mmgr_store(target_addr, blob, size); + hmm_store(target_addr, blob, size); return target_addr; } diff --git a/drivers/staging/media/atomisp/pci/sh_css_firmware.h b/drivers/staging/media/atomisp/pci/sh_css_firmware.h index f6253392a6c9..66cd38f08f71 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_firmware.h +++ b/drivers/staging/media/atomisp/pci/sh_css_firmware.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -15,7 +16,7 @@ #ifndef _SH_CSS_FIRMWARE_H_ #define _SH_CSS_FIRMWARE_H_ -#include <system_types.h> +#include <system_local.h> #include <ia_css_err.h> #include <ia_css_acc_types.h> @@ -37,18 +38,19 @@ extern unsigned int sh_css_num_binaries; char *sh_css_get_fw_version(void); +struct device; bool sh_css_check_firmware_version(struct device *dev, const char *fw_data); -enum ia_css_err +int sh_css_load_firmware(struct device *dev, const char *fw_data, unsigned int fw_size); void sh_css_unload_firmware(void); -hrt_vaddress sh_css_load_blob(const unsigned char *blob, unsigned int size); +ia_css_ptr sh_css_load_blob(const unsigned char *blob, unsigned int size); -enum ia_css_err +int sh_css_load_blob_info(const char *fw, const struct ia_css_fw_info *bi, struct ia_css_blob_descr *bd, unsigned int i); diff --git a/drivers/staging/media/atomisp/pci/sh_css_frac.h b/drivers/staging/media/atomisp/pci/sh_css_frac.h index cd2d755ec523..8f08df5c88cc 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_frac.h +++ b/drivers/staging/media/atomisp/pci/sh_css_frac.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_host_data.c b/drivers/staging/media/atomisp/pci/sh_css_host_data.c index 348183a221a8..39a9b9812682 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_host_data.c +++ b/drivers/staging/media/atomisp/pci/sh_css_host_data.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -24,7 +25,7 @@ struct ia_css_host_data *ia_css_host_data_allocate(size_t size) if (!me) return NULL; me->size = (uint32_t)size; - me->address = sh_css_malloc(size); + me->address = kvmalloc(size, GFP_KERNEL); if (!me->address) { kfree(me); return NULL; @@ -35,7 +36,7 @@ struct ia_css_host_data *ia_css_host_data_allocate(size_t size) void ia_css_host_data_free(struct ia_css_host_data *me) { if (me) { - sh_css_free(me->address); + kvfree(me->address); me->address = NULL; kfree(me); } diff --git a/drivers/staging/media/atomisp/pci/sh_css_hrt.c b/drivers/staging/media/atomisp/pci/sh_css_hrt.c index 94b2de5b5ef4..06b502151af9 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_hrt.c +++ b/drivers/staging/media/atomisp/pci/sh_css_hrt.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -61,7 +62,7 @@ bool sh_css_hrt_system_is_idle(void) return !not_idle; } -enum ia_css_err sh_css_hrt_sp_wait(void) +int sh_css_hrt_sp_wait(void) { #if defined(HAS_IRQ_MAP_VERSION_2) irq_sw_channel_id_t irq_id = IRQ_SW_CHANNEL0_ID; @@ -78,8 +79,8 @@ enum ia_css_err sh_css_hrt_sp_wait(void) ((irq_reg_load(IRQ0_ID, _HRT_IRQ_CONTROLLER_STATUS_REG_IDX) & (1U << (irq_id + IRQ_SW_CHANNEL_OFFSET))) == 0)) { - hrt_sleep(); + udelay(1); } - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/sh_css_hrt.h b/drivers/staging/media/atomisp/pci/sh_css_hrt.h index fd23ed1848ec..168bbd579798 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_hrt.h +++ b/drivers/staging/media/atomisp/pci/sh_css_hrt.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -27,7 +28,7 @@ void sh_css_hrt_sp_start_copy_frame(void); void sh_css_hrt_sp_start_isp(void); -enum ia_css_err sh_css_hrt_sp_wait(void); +int sh_css_hrt_sp_wait(void); bool sh_css_hrt_system_is_idle(void); diff --git a/drivers/staging/media/atomisp/pci/sh_css_internal.h b/drivers/staging/media/atomisp/pci/sh_css_internal.h index 5f271d9ae485..5c25a25dce92 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_internal.h +++ b/drivers/staging/media/atomisp/pci/sh_css_internal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -75,11 +76,7 @@ #define SH_CSS_REF_BIT_DEPTH 8 /* keep next up to date with the definition for MAX_CB_ELEMS_FOR_TAGGER in tagger.sp.c */ -#if defined(HAS_SP_2400) #define NUM_CONTINUOUS_FRAMES 15 -#else -#define NUM_CONTINUOUS_FRAMES 10 -#endif #define NUM_MIPI_FRAMES_PER_STREAM 2 #define NUM_ONLINE_INIT_CONTINUOUS_FRAMES 2 @@ -209,28 +206,28 @@ enum sh_css_sp_event_type { }; /* xmem address map allocation per pipeline, css pointers */ -/* Note that the struct below should only consist of hrt_vaddress-es +/* Note that the struct below should only consist of ia_css_ptr-es Otherwise this will cause a fail in the function ref_sh_css_ddr_address_map */ struct sh_css_ddr_address_map { - hrt_vaddress isp_param; - hrt_vaddress isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES]; - hrt_vaddress macc_tbl; - hrt_vaddress fpn_tbl; - hrt_vaddress sc_tbl; - hrt_vaddress tetra_r_x; - hrt_vaddress tetra_r_y; - hrt_vaddress tetra_gr_x; - hrt_vaddress tetra_gr_y; - hrt_vaddress tetra_gb_x; - hrt_vaddress tetra_gb_y; - hrt_vaddress tetra_b_x; - hrt_vaddress tetra_b_y; - hrt_vaddress tetra_ratb_x; - hrt_vaddress tetra_ratb_y; - hrt_vaddress tetra_batr_x; - hrt_vaddress tetra_batr_y; - hrt_vaddress dvs_6axis_params_y; + ia_css_ptr isp_param; + ia_css_ptr isp_mem_param[SH_CSS_MAX_STAGES][IA_CSS_NUM_MEMORIES]; + ia_css_ptr macc_tbl; + ia_css_ptr fpn_tbl; + ia_css_ptr sc_tbl; + ia_css_ptr tetra_r_x; + ia_css_ptr tetra_r_y; + ia_css_ptr tetra_gr_x; + ia_css_ptr tetra_gr_y; + ia_css_ptr tetra_gb_x; + ia_css_ptr tetra_gb_y; + ia_css_ptr tetra_b_x; + ia_css_ptr tetra_b_y; + ia_css_ptr tetra_ratb_x; + ia_css_ptr tetra_ratb_y; + ia_css_ptr tetra_batr_x; + ia_css_ptr tetra_batr_y; + ia_css_ptr dvs_6axis_params_y; }; #define SIZE_OF_SH_CSS_DDR_ADDRESS_MAP_STRUCT \ @@ -279,9 +276,9 @@ struct ia_css_isp_parameter_set_info { a binary. It depends on the binary which ones are used. */ struct sh_css_binary_args { struct ia_css_frame *in_frame; /* input frame */ - struct ia_css_frame + const struct ia_css_frame *delay_frames[MAX_NUM_VIDEO_DELAY_FRAMES]; /* reference input frame */ - struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES]; /* tnr frames */ + const struct ia_css_frame *tnr_frames[NUM_TNR_FRAMES]; /* tnr frames */ struct ia_css_frame *out_frame[IA_CSS_BINARY_MAX_OUTPUT_PORTS]; /* output frame */ struct ia_css_frame *out_vf_frame; /* viewfinder output frame */ @@ -531,8 +528,8 @@ struct sh_css_sp_pipeline { u32 port_id; /* port_id for input system */ u32 num_stages; /* the pipe config */ u32 running; /* needed for pipe termination */ - hrt_vaddress sp_stage_addr[SH_CSS_MAX_STAGES]; - hrt_vaddress scaler_pp_lut; /* Early bound LUT */ + ia_css_ptr sp_stage_addr[SH_CSS_MAX_STAGES]; + ia_css_ptr scaler_pp_lut; /* Early bound LUT */ u32 dummy; /* stage ptr is only used on sp but lives in this struct; needs cleanup */ s32 num_execs; /* number of times to run if this is @@ -544,7 +541,7 @@ struct sh_css_sp_pipeline { u32 height; /* Number of lines */ u32 stride; /* Stride (in bytes) per line */ u32 size; /* Total size (in bytes) */ - hrt_vaddress cont_buf; /* Address of continuous buffer */ + ia_css_ptr cont_buf; /* Address of continuous buffer */ } metadata; #endif #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) @@ -657,9 +654,9 @@ struct sh_css_sp_stage { struct ia_css_frames_sp frames; struct ia_css_resolution dvs_envelope; struct sh_css_uds_info uds; - hrt_vaddress isp_stage_addr; - hrt_vaddress xmem_bin_addr; - hrt_vaddress xmem_map_addr; + ia_css_ptr isp_stage_addr; + ia_css_ptr xmem_bin_addr; + ia_css_ptr xmem_map_addr; u16 top_cropping; u16 row_stripes_height; @@ -692,7 +689,7 @@ struct sh_css_sp_group { /* Data in SP dmem that is set from the host every stage. */ struct sh_css_sp_per_frame_data { /* ddr address of sp_group and sp_stage */ - hrt_vaddress sp_group_addr; + ia_css_ptr sp_group_addr; }; #define SH_CSS_NUM_SDW_IRQS 3 @@ -728,25 +725,19 @@ struct sh_css_sp_output { * separate SP thread for this. */ #define IA_CSS_NUM_ELEMS_HOST2SP_ISYS_EVENT_QUEUE (2 * N_CSI_PORTS) -#if defined(HAS_SP_2400) #define IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE 13 #define IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE 19 #define IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE 26 /* holds events for all type of buffers, hence deeper */ -#else -#define IA_CSS_NUM_ELEMS_HOST2SP_PSYS_EVENT_QUEUE 6 -#define IA_CSS_NUM_ELEMS_SP2HOST_BUFFER_QUEUE 6 -#define IA_CSS_NUM_ELEMS_SP2HOST_PSYS_EVENT_QUEUE 6 -#endif struct sh_css_hmm_buffer { union { struct ia_css_isp_3a_statistics s3a; struct ia_css_isp_dvs_statistics dis; - hrt_vaddress skc_dvs_statistics; - hrt_vaddress lace_stat; + ia_css_ptr skc_dvs_statistics; + ia_css_ptr lace_stat; struct ia_css_metadata metadata; struct frame_data_wrapper { - hrt_vaddress frame_data; + ia_css_ptr frame_data; u32 flashed; u32 exp_id; u32 isp_parameters_id; /** Unique ID to track which config was @@ -755,7 +746,7 @@ struct sh_css_hmm_buffer { struct sh_css_config_on_frame_enqueue config_on_frame_enqueue; #endif } frame; - hrt_vaddress ddr_ptrs; + ia_css_ptr ddr_ptrs; } payload; /* * kernel_ptr is present for host administration purposes only. @@ -834,12 +825,12 @@ struct host_sp_communication { * TODO: * Remove it when the Host and the SP is decoupled. */ - hrt_vaddress host2sp_offline_frames[NUM_CONTINUOUS_FRAMES]; - hrt_vaddress host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES]; + ia_css_ptr host2sp_offline_frames[NUM_CONTINUOUS_FRAMES]; + ia_css_ptr host2sp_offline_metadata[NUM_CONTINUOUS_FRAMES]; #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) - hrt_vaddress host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM]; - hrt_vaddress host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM]; + ia_css_ptr host2sp_mipi_frames[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM]; + ia_css_ptr host2sp_mipi_metadata[N_CSI_PORTS][NUM_MIPI_FRAMES_PER_STREAM]; u32 host2sp_num_mipi_frames[N_CSI_PORTS]; #endif u32 host2sp_cont_avail_num_raw_frames; @@ -960,20 +951,14 @@ sh_css_vprint(const char *fmt, va_list args) issue with the firmware struct/union's. More permanent solution will be to refactor this include. */ -hrt_vaddress sh_css_params_ddr_address_map(void); +ia_css_ptr sh_css_params_ddr_address_map(void); -enum ia_css_err +int sh_css_params_init(void); void sh_css_params_uninit(void); -void *sh_css_malloc(size_t size); - -void *sh_css_calloc(size_t N, size_t size); - -void sh_css_free(void *ptr); - /* For Acceleration API: Flush FW (shared buffer pointer) arguments */ void sh_css_flush(struct ia_css_acc_fw *fw); @@ -1010,13 +995,13 @@ sh_css_get_mipi_sizes_for_check(const unsigned int port, #endif -hrt_vaddress +ia_css_ptr sh_css_store_sp_group_to_ddr(void); -hrt_vaddress +ia_css_ptr sh_css_store_sp_stage_to_ddr(unsigned int pipe, unsigned int stage); -hrt_vaddress +ia_css_ptr sh_css_store_isp_stage_to_ddr(unsigned int pipe, unsigned int stage); void diff --git a/drivers/staging/media/atomisp/pci/sh_css_legacy.h b/drivers/staging/media/atomisp/pci/sh_css_legacy.h index 99ac690ba7aa..567c8d6dcc2c 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_legacy.h +++ b/drivers/staging/media/atomisp/pci/sh_css_legacy.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -45,7 +46,7 @@ struct ia_css_pipe_extra_config { bool disable_vf_pp; }; -enum ia_css_err +int ia_css_pipe_create_extra(const struct ia_css_pipe_config *config, const struct ia_css_pipe_extra_config *extra_config, struct ia_css_pipe **pipe); @@ -54,12 +55,12 @@ void ia_css_pipe_extra_config_defaults(struct ia_css_pipe_extra_config *extra_config); -enum ia_css_err +int ia_css_temp_pipe_to_pipe_id(const struct ia_css_pipe *pipe, enum ia_css_pipe_id *pipe_id); /* DEPRECATED. FPN is not supported. */ -enum ia_css_err +int sh_css_set_black_frame(struct ia_css_stream *stream, const struct ia_css_frame *raw_black_frame); diff --git a/drivers/staging/media/atomisp/pci/sh_css_metadata.c b/drivers/staging/media/atomisp/pci/sh_css_metadata.c index ebdf84d4a138..04a4b7da85e2 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_metadata.c +++ b/drivers/staging/media/atomisp/pci/sh_css_metadata.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_metrics.c b/drivers/staging/media/atomisp/pci/sh_css_metrics.c index 17f6dd9afab4..9744bbebe1bc 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_metrics.c +++ b/drivers/staging/media/atomisp/pci/sh_css_metrics.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -67,13 +68,16 @@ make_histogram(struct sh_css_pc_histogram *histogram, unsigned int length) return; if (histogram->run) return; - histogram->run = sh_css_malloc(length * sizeof(*histogram->run)); + histogram->run = kvmalloc(length * sizeof(*histogram->run), + GFP_KERNEL); if (!histogram->run) return; - histogram->stall = sh_css_malloc(length * sizeof(*histogram->stall)); + histogram->stall = kvmalloc(length * sizeof(*histogram->stall), + GFP_KERNEL); if (!histogram->stall) return; - histogram->msink = sh_css_malloc(length * sizeof(*histogram->msink)); + histogram->msink = kvmalloc(length * sizeof(*histogram->msink), + GFP_KERNEL); if (!histogram->msink) return; diff --git a/drivers/staging/media/atomisp/pci/sh_css_metrics.h b/drivers/staging/media/atomisp/pci/sh_css_metrics.h index f465d1545b8b..f4bcd08384e9 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_metrics.h +++ b/drivers/staging/media/atomisp/pci/sh_css_metrics.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_mipi.c b/drivers/staging/media/atomisp/pci/sh_css_mipi.c index 35cbef5f9f71..2ef5dbd62a6d 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_mipi.c +++ b/drivers/staging/media/atomisp/pci/sh_css_mipi.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -34,10 +35,10 @@ static u32 ref_count_mipi_allocation[N_CSI_PORTS]; /* Initialized in mipi_init */ #endif -enum ia_css_err +int ia_css_mipi_frame_specify(const unsigned int size_mem_words, const bool contiguous) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; my_css.size_mem_words = size_mem_words; (void)contiguous; @@ -97,14 +98,14 @@ static bool ia_css_mipi_is_source_port_valid(struct ia_css_pipe *pipe, * etc.). * Result is given in DDR mem words, 32B or 256 bits */ -enum ia_css_err +int ia_css_mipi_frame_calculate_size(const unsigned int width, const unsigned int height, const enum atomisp_input_format format, const bool hasSOLandEOL, const unsigned int embedded_data_size_words, unsigned int *size_mem_words) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int bits_per_pixel = 0; unsigned int even_line_bytes = 0; @@ -182,7 +183,7 @@ ia_css_mipi_frame_calculate_size(const unsigned int width, case ATOMISP_INPUT_FORMAT_YUV422_16: /* Not supported */ case ATOMISP_INPUT_FORMAT_RAW_16: /* TODO: not specified in MIPI SPEC, check */ default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */ @@ -246,12 +247,12 @@ ia_css_mipi_frame_calculate_size(const unsigned int width, } #if !defined(HAS_NO_INPUT_SYSTEM) && defined(USE_INPUT_SYSTEM_VERSION_2) -enum ia_css_err +int ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port, const unsigned int size_mem_words) { u32 idx; - enum ia_css_err err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + int err = -EBUSY; OP___assert(port < N_CSI_PORTS); OP___assert(size_mem_words != 0); @@ -264,7 +265,7 @@ ia_css_mipi_frame_enable_check_on_size(const enum mipi_port_id port, if (idx < IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT) { my_css.mipi_sizes_for_check[port][idx] = size_mem_words; - err = IA_CSS_SUCCESS; + err = 0; } return err; @@ -282,12 +283,12 @@ mipi_init(void) #endif } -enum ia_css_err +int calculate_mipi_buff_size( struct ia_css_stream_config *stream_cfg, unsigned int *size_mem_words) { #if !defined(USE_INPUT_SYSTEM_VERSION_2401) - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; (void)stream_cfg; (void)size_mem_words; #else @@ -310,7 +311,7 @@ calculate_mipi_buff_size( unsigned int mem_words_per_buff_line = 0; unsigned int mem_words_per_buff = 0; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; /** * zhengjie.lu@intel.com @@ -354,7 +355,7 @@ calculate_mipi_buff_size( bits_per_pixel = (format == ATOMISP_INPUT_FORMAT_RAW_10 && pack_raw_pixels) ? bits_per_pixel : 16; if (bits_per_pixel == 0) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; odd_line_bytes = (width_padded * bits_per_pixel + 7) >> 3; /* ceil ( bits per line / 8) */ @@ -390,7 +391,7 @@ calculate_mipi_buff_size( static bool buffers_needed(struct ia_css_pipe *pipe) { - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { if (pipe->stream->config.mode == IA_CSS_INPUT_MODE_BUFFERED_SENSOR) return false; else @@ -405,11 +406,11 @@ static bool buffers_needed(struct ia_css_pipe *pipe) return true; } -enum ia_css_err +int allocate_mipi_frames(struct ia_css_pipe *pipe, struct ia_css_stream_info *info) { #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; unsigned int port; struct ia_css_frame_info mipi_intermediate_info; @@ -423,7 +424,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) exit: pipe or stream is null.\n", pipe); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } #ifdef USE_INPUT_SYSTEM_VERSION_2401 @@ -432,7 +433,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) exit: no buffers needed for 2401 pipe mode.\n", pipe); - return IA_CSS_SUCCESS; + return 0; } #endif @@ -441,10 +442,10 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) exit: no buffers needed for pipe mode.\n", pipe); - return IA_CSS_SUCCESS; /* AM TODO: Check */ + return 0; /* AM TODO: Check */ } - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) port = (unsigned int)pipe->stream->config.source.port.port; else err = ia_css_mipi_is_source_port_valid(pipe, &port); @@ -455,7 +456,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) exit: error: port is not correct (port=%d).\n", pipe, port); - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } #ifdef USE_INPUT_SYSTEM_VERSION_2401 @@ -470,7 +471,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) exit: already allocated for this port (port=%d).\n", pipe, port); - return IA_CSS_SUCCESS; + return 0; } #else /* 2401 system allows multiple streams to use same physical port. This is not @@ -484,7 +485,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "allocate_mipi_frames(%p) leave: nothing to do, already allocated for this port (port=%d).\n", pipe, port); - return IA_CSS_SUCCESS; + return 0; } #endif @@ -522,7 +523,7 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, &my_css.mipi_frames[port][i], my_css.mipi_frame_size[port] * HIVE_ISP_DDR_WORD_BYTES, false); - if (err != IA_CSS_SUCCESS) { + if (err) { for (j = 0; j < i; j++) { if (my_css.mipi_frames[port][j]) { ia_css_frame_free(my_css.mipi_frames[port][j]); @@ -562,14 +563,14 @@ allocate_mipi_frames(struct ia_css_pipe *pipe, #else (void)pipe; (void)info; - return IA_CSS_SUCCESS; + return 0; #endif } -enum ia_css_err +int free_mipi_frames(struct ia_css_pipe *pipe) { #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; unsigned int port; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, @@ -583,7 +584,7 @@ free_mipi_frames(struct ia_css_pipe *pipe) { ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE_PRIVATE, "free_mipi_frames(%p) exit: error: pipe or stream is null.\n", pipe); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } if (!buffers_needed(pipe)) { @@ -593,7 +594,7 @@ free_mipi_frames(struct ia_css_pipe *pipe) { return err; } - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) port = (unsigned int)pipe->stream->config.source.port.port; else err = ia_css_mipi_is_source_port_valid(pipe, &port); @@ -677,13 +678,13 @@ free_mipi_frames(struct ia_css_pipe *pipe) { #else (void)pipe; #endif - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int send_mipi_frames(struct ia_css_pipe *pipe) { #if defined(USE_INPUT_SYSTEM_VERSION_2) || defined(USE_INPUT_SYSTEM_VERSION_2401) - enum ia_css_err err = IA_CSS_ERR_INTERNAL_ERROR; + int err = -EINVAL; unsigned int i; #ifndef ISP2401 unsigned int port; @@ -698,18 +699,18 @@ send_mipi_frames(struct ia_css_pipe *pipe) { if (!pipe || !pipe->stream) { IA_CSS_ERROR("pipe or stream is null"); - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } /* multi stream video needs mipi buffers */ /* nothing to be done in other cases. */ if (!buffers_needed(pipe)) { IA_CSS_LOG("nothing to be done for this mode"); - return IA_CSS_SUCCESS; + return 0; /* TODO: AM: maybe this should be returning an error. */ } - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) port = (unsigned int)pipe->stream->config.source.port.port; else err = ia_css_mipi_is_source_port_valid(pipe, &port); @@ -749,9 +750,9 @@ send_mipi_frames(struct ia_css_pipe *pipe) { (uint8_t)port, (uint8_t)my_css.num_mipi_frames[port], 0 /* not used */); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); + IA_CSS_LEAVE_ERR_PRIVATE(0); #else (void)pipe; #endif - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/sh_css_mipi.h b/drivers/staging/media/atomisp/pci/sh_css_mipi.h index ab38589d6457..52f08a103883 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_mipi.h +++ b/drivers/staging/media/atomisp/pci/sh_css_mipi.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -22,13 +23,13 @@ void mipi_init(void); -enum ia_css_err +int allocate_mipi_frames(struct ia_css_pipe *pipe, struct ia_css_stream_info *info); -enum ia_css_err +int free_mipi_frames(struct ia_css_pipe *pipe); -enum ia_css_err +int send_mipi_frames(struct ia_css_pipe *pipe); /** @@ -41,7 +42,7 @@ send_mipi_frames(struct ia_css_pipe *pipe); * * @return */ -enum ia_css_err +int calculate_mipi_buff_size( struct ia_css_stream_config *stream_cfg, unsigned int *size_mem_words); diff --git a/drivers/staging/media/atomisp/pci/sh_css_mmu.c b/drivers/staging/media/atomisp/pci/sh_css_mmu.c index 179b6f40be49..1da7459eaa25 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_mmu.c +++ b/drivers/staging/media/atomisp/pci/sh_css_mmu.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -43,7 +44,6 @@ ia_css_mmu_invalidate_cache(void) "ia_css_mmu_invalidate_cache() leave\n"); } -/* Deprecated, this is an HRT backend function (memory_access.h) */ void sh_css_mmu_set_page_table_base_index(hrt_data base_index) { diff --git a/drivers/staging/media/atomisp/pci/sh_css_morph.c b/drivers/staging/media/atomisp/pci/sh_css_morph.c index 1f4fa25b1e79..edd1da941ccb 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_morph.c +++ b/drivers/staging/media/atomisp/pci/sh_css_morph.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c index 52e29161cb35..ff0082d02af3 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c +++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -18,7 +19,6 @@ #include <ia_css_err.h> #include <ia_css_types.h> #include "ia_css_debug.h" -#include "memory_access.h" static struct ia_css_dvs_6axis_config * alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res, @@ -28,14 +28,14 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res, unsigned int height_y = 0; unsigned int width_uv = 0; unsigned int height_uv = 0; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_dvs_6axis_config *dvs_config = NULL; - dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_malloc(sizeof( - struct ia_css_dvs_6axis_config)); + dvs_config = kvmalloc(sizeof(struct ia_css_dvs_6axis_config), + GFP_KERNEL); if (!dvs_config) { IA_CSS_ERROR("out of memory"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; } else { /*Initialize new struct with latest config settings*/ if (dvs_config_src) { @@ -58,41 +58,41 @@ alloc_dvs_6axis_table(const struct ia_css_resolution *frame_res, } /* Generate Y buffers */ - dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof( - uint32_t)); + dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->xcoords_y) { IA_CSS_ERROR("out of memory"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto exit; } - dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof( - uint32_t)); + dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->ycoords_y) { IA_CSS_ERROR("out of memory"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto exit; } /* Generate UV buffers */ IA_CSS_LOG("UV W %d H %d", width_uv, height_uv); - dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * - sizeof(uint32_t)); + dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->xcoords_uv) { IA_CSS_ERROR("out of memory"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; goto exit; } - dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * - sizeof(uint32_t)); + dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->ycoords_uv) { IA_CSS_ERROR("out of memory"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; } exit: - if (err != IA_CSS_SUCCESS) { + if (err) { free_dvs_6axis_table( &dvs_config); /* we might have allocated some memory, release this */ dvs_config = NULL; @@ -208,28 +208,28 @@ free_dvs_6axis_table(struct ia_css_dvs_6axis_config **dvs_6axis_config) if ((dvs_6axis_config) && (*dvs_6axis_config)) { IA_CSS_ENTER_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config)); if ((*dvs_6axis_config)->xcoords_y) { - sh_css_free((*dvs_6axis_config)->xcoords_y); + kvfree((*dvs_6axis_config)->xcoords_y); (*dvs_6axis_config)->xcoords_y = NULL; } if ((*dvs_6axis_config)->ycoords_y) { - sh_css_free((*dvs_6axis_config)->ycoords_y); + kvfree((*dvs_6axis_config)->ycoords_y); (*dvs_6axis_config)->ycoords_y = NULL; } /* Free up UV buffers */ if ((*dvs_6axis_config)->xcoords_uv) { - sh_css_free((*dvs_6axis_config)->xcoords_uv); + kvfree((*dvs_6axis_config)->xcoords_uv); (*dvs_6axis_config)->xcoords_uv = NULL; } if ((*dvs_6axis_config)->ycoords_uv) { - sh_css_free((*dvs_6axis_config)->ycoords_uv); + kvfree((*dvs_6axis_config)->ycoords_uv); (*dvs_6axis_config)->ycoords_uv = NULL; } IA_CSS_LEAVE_PRIVATE("dvs_6axis_config %p", (*dvs_6axis_config)); - sh_css_free(*dvs_6axis_config); + kvfree(*dvs_6axis_config); *dvs_6axis_config = NULL; } } diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h index b4cffbbdafde..7782f76b9f97 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h +++ b/drivers/staging/media/atomisp/pci/sh_css_param_dvs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c index 4b648df2d073..046f34857891 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_param_shading.c +++ b/drivers/staging/media/atomisp/pci/sh_css_param_shading.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -360,12 +361,13 @@ ia_css_shading_table_alloc( me->fraction_bits = 0; for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) { me->data[i] = - sh_css_malloc(width * height * sizeof(*me->data[0])); + kvmalloc(width * height * sizeof(*me->data[0]), + GFP_KERNEL); if (!me->data[i]) { unsigned int j; for (j = 0; j < i; j++) { - sh_css_free(me->data[j]); + kvfree(me->data[j]); me->data[j] = NULL; } kfree(me); @@ -392,7 +394,7 @@ ia_css_shading_table_free(struct ia_css_shading_table *table) for (i = 0; i < IA_CSS_SC_NUM_COLORS; i++) { if (table->data[i]) { - sh_css_free(table->data[i]); + kvfree(table->data[i]); table->data[i] = NULL; } } diff --git a/drivers/staging/media/atomisp/pci/sh_css_param_shading.h b/drivers/staging/media/atomisp/pci/sh_css_param_shading.h index 6e480d31c201..7cdfaaec0b1c 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_param_shading.h +++ b/drivers/staging/media/atomisp/pci/sh_css_param_shading.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.c b/drivers/staging/media/atomisp/pci/sh_css_params.c index 2e719f7db89e..ba42be9b06eb 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_params.c +++ b/drivers/staging/media/atomisp/pci/sh_css_params.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -24,6 +25,7 @@ #define IA_CSS_INCLUDE_PARAMETERS #define IA_CSS_INCLUDE_ACC_PARAMETERS +#include "hmm.h" #include "sh_css_params.h" #include "ia_css_queue.h" #include "sw_event_global.h" /* Event IDs */ @@ -45,10 +47,7 @@ #include "sh_css_sp.h" #include "ia_css_pipeline.h" #include "ia_css_debug.h" -#include "memory_access.h" -#if 0 /* FIXME */ -#include "memory_realloc.h" -#endif + #include "ia_css_isp_param.h" #include "ia_css_isp_params.h" #include "ia_css_mipi.h" @@ -126,17 +125,17 @@ /* We keep a second copy of the ptr struct for the SP to access. Again, this would not be necessary on the chip. */ -static hrt_vaddress sp_ddr_ptrs; +static ia_css_ptr sp_ddr_ptrs; /* sp group address on DDR */ -static hrt_vaddress xmem_sp_group_ptrs; +static ia_css_ptr xmem_sp_group_ptrs; -static hrt_vaddress xmem_sp_stage_ptrs[IA_CSS_PIPE_ID_NUM] +static ia_css_ptr xmem_sp_stage_ptrs[IA_CSS_PIPE_ID_NUM] [SH_CSS_MAX_STAGES]; -static hrt_vaddress xmem_isp_stage_ptrs[IA_CSS_PIPE_ID_NUM] +static ia_css_ptr xmem_isp_stage_ptrs[IA_CSS_PIPE_ID_NUM] [SH_CSS_MAX_STAGES]; -static hrt_vaddress default_gdc_lut; +static ia_css_ptr default_gdc_lut; static int interleaved_lut_temp[4][HRT_GDC_N]; /* END DO NOT MOVE INTO VIMALS_WORLD */ @@ -1229,20 +1228,20 @@ struct ia_css_isp_skc_dvs_statistics { ia_css_ptr p_data; }; -static enum ia_css_err +static int ref_sh_css_ddr_address_map( struct sh_css_ddr_address_map *map, struct sh_css_ddr_address_map *out); -static enum ia_css_err +static int write_ia_css_isp_parameter_set_info_to_ddr( struct ia_css_isp_parameter_set_info *me, - hrt_vaddress *out); + ia_css_ptr *out); -static enum ia_css_err -free_ia_css_isp_parameter_set_info(hrt_vaddress ptr); +static int +free_ia_css_isp_parameter_set_info(ia_css_ptr ptr); -static enum ia_css_err +static int sh_css_params_write_to_ddr_internal( struct ia_css_pipe *pipe, unsigned int pipe_id, @@ -1251,7 +1250,7 @@ sh_css_params_write_to_ddr_internal( struct sh_css_ddr_address_map *ddr_map, struct sh_css_ddr_address_map_size *ddr_map_size); -static enum ia_css_err +static int sh_css_create_isp_params(struct ia_css_stream *stream, struct ia_css_isp_parameters **isp_params_out); @@ -1261,27 +1260,27 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream, bool use_default_config, struct ia_css_pipe *pipe_in); -static enum ia_css_err +static int sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe, struct ia_css_isp_parameters *params, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe_in); -static enum ia_css_err +static int sh_css_set_global_isp_config_on_pipe( struct ia_css_pipe *curr_pipe, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe); #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) -static enum ia_css_err +static int sh_css_set_per_frame_isp_config_on_pipe( struct ia_css_stream *stream, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe); #endif -static enum ia_css_err +static int sh_css_update_uds_and_crop_info_based_on_zoom_region( const struct ia_css_binary_info *info, const struct ia_css_frame_info *in_frame_info, @@ -1294,7 +1293,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region( struct ia_css_resolution pipe_in_res, bool enable_zoom); -hrt_vaddress +ia_css_ptr sh_css_params_ddr_address_map(void) { return sp_ddr_ptrs; @@ -1351,8 +1350,8 @@ convert_allocate_fpntbl(struct ia_css_isp_parameters *params) return me; } -static enum ia_css_err -store_fpntbl(struct ia_css_isp_parameters *params, hrt_vaddress ptr) { +static int +store_fpntbl(struct ia_css_isp_parameters *params, ia_css_ptr ptr) { struct ia_css_host_data *isp_data; assert(params); @@ -1361,13 +1360,13 @@ store_fpntbl(struct ia_css_isp_parameters *params, hrt_vaddress ptr) { isp_data = convert_allocate_fpntbl(params); if (!isp_data) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } ia_css_params_store_ia_css_host_data(ptr, isp_data); ia_css_host_data_free(isp_data); - return IA_CSS_SUCCESS; + return 0; } static void @@ -1431,10 +1430,10 @@ ia_css_process_kernel(struct ia_css_stream *stream, } } -static enum ia_css_err +static int sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe, bool *is_dp_10bpp) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; /* Currently we check if 10bpp DPC configuration is required based * on the use case,i.e. if BDS and DPC is both enabled. The more cleaner * design choice would be to expose the type of DPC (either 10bpp or 13bpp) @@ -1444,8 +1443,8 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe, */ if ((!pipe) || (!is_dp_10bpp)) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INTERNAL_ERROR); - err = IA_CSS_ERR_INTERNAL_ERROR; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + err = -EINVAL; } else { *is_dp_10bpp = false; @@ -1457,7 +1456,7 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe, if ((pipe->config.bayer_ds_out_res.width != 0) && (pipe->config.bayer_ds_out_res.height != 0)) { - if (IA_CSS_SUCCESS == binarydesc_calculate_bds_factor( + if (0 == binarydesc_calculate_bds_factor( pipe->config.input_effective_res, pipe->config.bayer_ds_out_res, &required_bds_factor)) { @@ -1473,7 +1472,7 @@ sh_css_select_dp_10bpp_config(const struct ia_css_pipe *pipe, return err; } -enum ia_css_err +int sh_css_set_black_frame(struct ia_css_stream *stream, const struct ia_css_frame *raw_black_frame) { struct ia_css_isp_parameters *params; @@ -1481,7 +1480,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream, * that it can use the DMA. */ unsigned int height, width, y, x, k, data; - hrt_vaddress ptr; + ia_css_ptr ptr; assert(stream); assert(raw_black_frame); @@ -1498,16 +1497,17 @@ sh_css_set_black_frame(struct ia_css_stream *stream, if (params->fpn_config.data && (params->fpn_config.width != width || params->fpn_config.height != height)) { - sh_css_free(params->fpn_config.data); + kvfree(params->fpn_config.data); params->fpn_config.data = NULL; } if (!params->fpn_config.data) { - params->fpn_config.data = sh_css_malloc(height * width * sizeof(short)); + params->fpn_config.data = kvmalloc(height * width * + sizeof(short), GFP_KERNEL); if (!params->fpn_config.data) { IA_CSS_ERROR("out of memory"); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } params->fpn_config.width = width; params->fpn_config.height = height; @@ -1521,7 +1521,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream, int ofs = y * width + x; for (k = 0; k < ISP_VEC_NELEMS; k += 2) { - mmgr_load(ptr, (void *)(&data), sizeof(int)); + hmm_load(ptr, (void *)(&data), sizeof(int)); params->fpn_config.data[ofs + 2 * k] = (short)(data & 0xFFFF); params->fpn_config.data[ofs + 2 * k + 2] = @@ -1529,7 +1529,7 @@ sh_css_set_black_frame(struct ia_css_stream *stream, ptr += sizeof(int); /* byte system address */ } for (k = 0; k < ISP_VEC_NELEMS; k += 2) { - mmgr_load(ptr, (void *)(&data), sizeof(int)); + hmm_load(ptr, (void *)(&data), sizeof(int)); params->fpn_config.data[ofs + 2 * k + 1] = (short)(data & 0xFFFF); params->fpn_config.data[ofs + 2 * k + 3] = @@ -1545,9 +1545,9 @@ sh_css_set_black_frame(struct ia_css_stream *stream, /* overwrite isp parameter */ ia_css_process_kernel(stream, params, ia_css_kernel_process_param[IA_CSS_FPN_ID]); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); + IA_CSS_LEAVE_ERR_PRIVATE(0); - return IA_CSS_SUCCESS; + return 0; } bool @@ -1611,7 +1611,7 @@ sh_css_set_shading_table(struct ia_css_stream *stream, void ia_css_params_store_ia_css_host_data( - hrt_vaddress ddr_addr, + ia_css_ptr ddr_addr, struct ia_css_host_data *data) { assert(data); @@ -1620,7 +1620,7 @@ ia_css_params_store_ia_css_host_data( IA_CSS_ENTER_PRIVATE(""); - mmgr_store(ddr_addr, + hmm_store(ddr_addr, (void *)(data->address), (size_t)data->size); @@ -1676,9 +1676,9 @@ ia_css_params_alloc_convert_sctbl( return sctbl; } -enum ia_css_err ia_css_params_store_sctbl( +int ia_css_params_store_sctbl( const struct ia_css_pipeline_stage *stage, - hrt_vaddress sc_tbl, + ia_css_ptr sc_tbl, const struct ia_css_shading_table *sc_config) { struct ia_css_host_data *isp_sc_tbl; @@ -1687,13 +1687,13 @@ enum ia_css_err ia_css_params_store_sctbl( if (!sc_config) { IA_CSS_LEAVE_PRIVATE("void"); - return IA_CSS_SUCCESS; + return 0; } isp_sc_tbl = ia_css_params_alloc_convert_sctbl(stage, sc_config); if (!isp_sc_tbl) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } /* store the shading table to ddr */ ia_css_params_store_ia_css_host_data(sc_tbl, isp_sc_tbl); @@ -1701,7 +1701,7 @@ enum ia_css_err ia_css_params_store_sctbl( IA_CSS_LEAVE_PRIVATE("void"); - return IA_CSS_SUCCESS; + return 0; } static void @@ -1717,13 +1717,13 @@ sh_css_enable_pipeline(const struct ia_css_binary *binary) IA_CSS_LEAVE_PRIVATE("void"); } -static enum ia_css_err +static int ia_css_process_zoom_and_motion( struct ia_css_isp_parameters *params, const struct ia_css_pipeline_stage *first_stage) { /* first_stage can be NULL */ const struct ia_css_pipeline_stage *stage; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; struct ia_css_resolution pipe_in_res; pipe_in_res.width = 0; @@ -1799,7 +1799,7 @@ ia_css_process_zoom_and_motion( ¶ms->uds[stage->stage_num].crop_pos, pipe_in_res, stage->enable_zoom); - if (err != IA_CSS_SUCCESS) + if (err) return err; } } @@ -1912,16 +1912,16 @@ void ia_css_morph_table_free( for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { if (me->coordinates_x[i]) { - sh_css_free(me->coordinates_x[i]); + kvfree(me->coordinates_x[i]); me->coordinates_x[i] = NULL; } if (me->coordinates_y[i]) { - sh_css_free(me->coordinates_y[i]); + kvfree(me->coordinates_y[i]); me->coordinates_y[i] = NULL; } } - sh_css_free(me); + kvfree(me); IA_CSS_LEAVE("void"); } @@ -1934,7 +1934,7 @@ struct ia_css_morph_table *ia_css_morph_table_allocate( IA_CSS_ENTER(""); - me = sh_css_malloc(sizeof(*me)); + me = kvmalloc(sizeof(*me), GFP_KERNEL); if (!me) { IA_CSS_ERROR("out of memory"); return me; @@ -1946,12 +1946,12 @@ struct ia_css_morph_table *ia_css_morph_table_allocate( } for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { - me->coordinates_x[i] = - sh_css_malloc(height * width * - sizeof(*me->coordinates_x[i])); - me->coordinates_y[i] = - sh_css_malloc(height * width * - sizeof(*me->coordinates_y[i])); + me->coordinates_x[i] = kvmalloc(height * width * + sizeof(*me->coordinates_x[i]), + GFP_KERNEL); + me->coordinates_y[i] = kvmalloc(height * width * + sizeof(*me->coordinates_y[i]), + GFP_KERNEL); if ((!me->coordinates_x[i]) || (!me->coordinates_y[i])) { @@ -1966,7 +1966,7 @@ struct ia_css_morph_table *ia_css_morph_table_allocate( return me; } -static enum ia_css_err sh_css_params_default_morph_table( +static int sh_css_params_default_morph_table( struct ia_css_morph_table **table, const struct ia_css_binary *binary) { @@ -1987,7 +1987,7 @@ static enum ia_css_err sh_css_params_default_morph_table( tab = ia_css_morph_table_allocate(width, height); if (!tab) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; for (i = 0; i < IA_CSS_MORPH_TABLE_NUM_PLANES; i++) { short val_y = start_y[i]; @@ -2016,9 +2016,9 @@ static enum ia_css_err sh_css_params_default_morph_table( } *table = tab; - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); + IA_CSS_LEAVE_ERR_PRIVATE(0); - return IA_CSS_SUCCESS; + return 0; } static void @@ -2065,11 +2065,11 @@ ia_css_isp_3a_statistics_map_free(struct ia_css_isp_3a_statistics_map *me) { if (me) { if (me->data_allocated) { - sh_css_free(me->data_ptr); + kvfree(me->data_ptr); me->data_ptr = NULL; me->data_allocated = false; } - sh_css_free(me); + kvfree(me); } } @@ -2083,7 +2083,7 @@ ia_css_isp_3a_statistics_map_allocate( * so we use a local char * instead. */ char *base_ptr; - me = sh_css_malloc(sizeof(*me)); + me = kvmalloc(sizeof(*me), GFP_KERNEL); if (!me) { IA_CSS_LEAVE("cannot allocate memory"); goto err; @@ -2092,7 +2092,7 @@ ia_css_isp_3a_statistics_map_allocate( me->data_ptr = data_ptr; me->data_allocated = !data_ptr; if (!data_ptr) { - me->data_ptr = sh_css_malloc(isp_stats->size); + me->data_ptr = kvmalloc(isp_stats->size, GFP_KERNEL); if (!me->data_ptr) { IA_CSS_LEAVE("cannot allocate memory"); goto err; @@ -2115,15 +2115,15 @@ ia_css_isp_3a_statistics_map_allocate( err: if (me) - sh_css_free(me); + kvfree(me); return NULL; } -enum ia_css_err +int ia_css_get_3a_statistics(struct ia_css_3a_statistics *host_stats, const struct ia_css_isp_3a_statistics *isp_stats) { struct ia_css_isp_3a_statistics_map *map; - enum ia_css_err ret = IA_CSS_SUCCESS; + int ret = 0; IA_CSS_ENTER("host_stats=%p, isp_stats=%p", host_stats, isp_stats); @@ -2133,13 +2133,13 @@ ia_css_get_3a_statistics(struct ia_css_3a_statistics *host_stats, map = ia_css_isp_3a_statistics_map_allocate(isp_stats, NULL); if (map) { - mmgr_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); + hmm_load(isp_stats->data_ptr, map->data_ptr, isp_stats->size); ia_css_translate_3a_statistics(host_stats, map); ia_css_isp_3a_statistics_map_free(map); } else { IA_CSS_ERROR("out of memory"); - ret = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + ret = -ENOMEM; } IA_CSS_LEAVE_ERR(ret); @@ -2161,7 +2161,7 @@ ia_css_set_param_exceptions(const struct ia_css_pipe *pipe, params->dp_config.b = params->wb_config.b; params->dp_config.gb = params->wb_config.gb; - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { assert(pipe); assert(pipe->mode < IA_CSS_PIPE_ID_NUM); @@ -2431,22 +2431,22 @@ sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe) return pipe->config.p_isp_config; } -enum ia_css_err +int ia_css_stream_set_isp_config( struct ia_css_stream *stream, const struct ia_css_isp_config *config) { return ia_css_stream_set_isp_config_on_pipe(stream, config, NULL); } -enum ia_css_err +int ia_css_stream_set_isp_config_on_pipe( struct ia_css_stream *stream, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; if ((!stream) || (!config)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; IA_CSS_ENTER("stream=%p, config=%p, pipe=%p", stream, config, pipe); @@ -2461,16 +2461,16 @@ ia_css_stream_set_isp_config_on_pipe( return err; } -enum ia_css_err +int ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe, struct ia_css_isp_config *config) { struct ia_css_pipe *pipe_in = pipe; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER("pipe=%p", pipe); if ((!pipe) || (!pipe->stream)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; ia_css_debug_dtrace(IA_CSS_DEBUG_TRACE, "config=%p\n", config); @@ -2484,14 +2484,14 @@ ia_css_pipe_set_isp_config(struct ia_css_pipe *pipe, return err; } -static enum ia_css_err +static int sh_css_set_global_isp_config_on_pipe( struct ia_css_pipe *curr_pipe, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe) { - enum ia_css_err err = IA_CSS_SUCCESS; - enum ia_css_err err1 = IA_CSS_SUCCESS; - enum ia_css_err err2 = IA_CSS_SUCCESS; + int err = 0; + int err1 = 0; + int err2 = 0; IA_CSS_ENTER_PRIVATE("stream=%p, config=%p, pipe=%p", curr_pipe, config, pipe); @@ -2506,24 +2506,24 @@ sh_css_set_global_isp_config_on_pipe( * but instead continue with updating the ISP params to enable testing of features * which are currently in TR phase. */ - err = (err1 != IA_CSS_SUCCESS) ? err1 : ((err2 != IA_CSS_SUCCESS) ? err2 : err); + err = (err1 != 0) ? err1 : ((err2 != 0) ? err2 : err); IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) -static enum ia_css_err +static int sh_css_set_per_frame_isp_config_on_pipe( struct ia_css_stream *stream, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe) { unsigned int i; bool per_frame_config_created = false; - enum ia_css_err err = IA_CSS_SUCCESS; - enum ia_css_err err1 = IA_CSS_SUCCESS; - enum ia_css_err err2 = IA_CSS_SUCCESS; - enum ia_css_err err3 = IA_CSS_SUCCESS; + int err = 0; + int err1 = 0; + int err2 = 0; + int err3 = 0; struct sh_css_ddr_address_map *ddr_ptrs; struct sh_css_ddr_address_map_size *ddr_ptrs_size; @@ -2533,7 +2533,7 @@ sh_css_set_per_frame_isp_config_on_pipe( if (!pipe) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; goto exit; } @@ -2544,7 +2544,7 @@ sh_css_set_per_frame_isp_config_on_pipe( { err = sh_css_create_isp_params(stream, &stream->per_frame_isp_params_configs); - if (err != IA_CSS_SUCCESS) + if (err) goto exit; per_frame_config_created = true; } @@ -2554,7 +2554,7 @@ sh_css_set_per_frame_isp_config_on_pipe( /* update new ISP params object with the new config */ if (!sh_css_init_isp_params_from_global(stream, params, false, pipe)) { - err1 = IA_CSS_ERR_INVALID_ARGUMENTS; + err1 = -EINVAL; } err2 = sh_css_init_isp_params_from_config(stream->pipes[0], params, config, pipe); @@ -2578,21 +2578,21 @@ sh_css_set_per_frame_isp_config_on_pipe( * The CSS API must pass this error information to the caller, ie. the host. * We do not return this error immediately, but instead continue with updating the ISP params * to enable testing of features which are currently in TR phase. */ - err = (err1 != IA_CSS_SUCCESS) ? err1 : - (err2 != IA_CSS_SUCCESS) ? err2 : - (err3 != IA_CSS_SUCCESS) ? err3 : err; + err = (err1 != 0) ? err1 : + (err2 != 0) ? err2 : + (err3 != 0) ? err3 : err; exit: IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } #endif -static enum ia_css_err +static int sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe, struct ia_css_isp_parameters *params, const struct ia_css_isp_config *config, struct ia_css_pipe *pipe_in) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool is_dp_10bpp = true; assert(pipe); @@ -2630,12 +2630,12 @@ sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe, * before (NORM+OBC) or after. The folllowing code to set the * DPC configuration should be updated when this interface is made * available */ - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { sh_css_set_dp_config(pipe, params, config->dp_config); ia_css_set_param_exceptions(pipe, params); } - if (IA_CSS_SUCCESS == + if (0 == sh_css_select_dp_10bpp_config(pipe, &is_dp_10bpp)) { /* return an error when both DPC and BDS is enabled by the @@ -2643,15 +2643,15 @@ sh_css_init_isp_params_from_config(struct ia_css_pipe *pipe, /* we do not exit from this point immediately to allow internal * firmware feature testing. */ if (is_dp_10bpp) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } } else { - err = IA_CSS_ERR_INTERNAL_ERROR; + err = -EINVAL; goto exit; } - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) ia_css_set_param_exceptions(pipe, params); exit: @@ -2709,16 +2709,16 @@ ia_css_pipe_get_isp_config(struct ia_css_pipe *pipe, * Deprecated: Implement mmgr_realloc() */ static bool realloc_isp_css_mm_buf( - hrt_vaddress *curr_buf, + ia_css_ptr *curr_buf, size_t *curr_size, size_t needed_size, bool force, - enum ia_css_err *err, + int *err, uint16_t mmgr_attribute) { s32 id; - *err = IA_CSS_SUCCESS; + *err = 0; /* Possible optimization: add a function sh_css_isp_css_mm_realloc() * and implement on top of hmm. */ @@ -2736,11 +2736,13 @@ static bool realloc_isp_css_mm_buf( id = IA_CSS_REFCOUNT_PARAM_BUFFER; ia_css_refcount_decrement(id, *curr_buf); - *curr_buf = ia_css_refcount_increment(id, mmgr_alloc_attr(needed_size, - mmgr_attribute)); + *curr_buf = ia_css_refcount_increment(id, hmm_alloc(needed_size, + HMM_BO_PRIVATE, 0, + NULL, + mmgr_attribute)); if (!*curr_buf) { - *err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + *err = -ENOMEM; *curr_size = 0; } else { *curr_size = needed_size; @@ -2750,19 +2752,18 @@ static bool realloc_isp_css_mm_buf( } static bool reallocate_buffer( - hrt_vaddress *curr_buf, + ia_css_ptr *curr_buf, size_t *curr_size, size_t needed_size, bool force, - enum ia_css_err *err) + int *err) { bool ret; - u16 mmgr_attribute = MMGR_ATTRIBUTE_DEFAULT; IA_CSS_ENTER_PRIVATE("void"); ret = realloc_isp_css_mm_buf(curr_buf, - curr_size, needed_size, force, err, mmgr_attribute); + curr_size, needed_size, force, err, 0); IA_CSS_LEAVE_PRIVATE("ret=%d", ret); return ret; @@ -2781,7 +2782,7 @@ ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid) if (!grid->enable) return NULL; - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; @@ -2803,9 +2804,9 @@ ia_css_isp_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid) me->hmem_size = CEIL_MUL(me->hmem_size, HIVE_ISP_DDR_WORD_BYTES); me->size = me->dmem_size + me->vmem_size * 2 + me->hmem_size; - me->data_ptr = mmgr_malloc(me->size); + me->data_ptr = hmm_alloc(me->size, HMM_BO_PRIVATE, 0, NULL, 0); if (me->data_ptr == mmgr_NULL) { - sh_css_free(me); + kvfree(me); me = NULL; goto err; } @@ -2828,7 +2829,7 @@ ia_css_isp_3a_statistics_free(struct ia_css_isp_3a_statistics *me) { if (me) { hmm_free(me->data_ptr); - sh_css_free(me); + kvfree(me); } } @@ -2847,13 +2848,13 @@ ia_css_metadata_allocate(const struct ia_css_metadata_info *metadata_info) if (metadata_info->size == 0) return NULL; - md = sh_css_malloc(sizeof(*md)); + md = kvmalloc(sizeof(*md), GFP_KERNEL); if (!md) goto error; md->info = *metadata_info; md->exp_id = 0; - md->address = mmgr_malloc(metadata_info->size); + md->address = hmm_alloc(metadata_info->size, HMM_BO_PRIVATE, 0, NULL, 0); if (md->address == mmgr_NULL) goto error; @@ -2877,7 +2878,7 @@ ia_css_metadata_free(struct ia_css_metadata *me) * and debugging. */ IA_CSS_ENTER("me=%p", me); hmm_free(me->address); - sh_css_free(me); + kvfree(me); IA_CSS_LEAVE("void"); } } @@ -2897,9 +2898,9 @@ ia_css_metadata_free_multiple(unsigned int num_bufs, static unsigned int g_param_buffer_dequeue_count; static unsigned int g_param_buffer_enqueue_count; -enum ia_css_err +int ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int i; struct sh_css_ddr_address_map *ddr_ptrs; struct sh_css_ddr_address_map_size *ddr_ptrs_size; @@ -2910,8 +2911,8 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) { if (!stream) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_INVALID_ARGUMENTS); - return IA_CSS_ERR_INVALID_ARGUMENTS; + IA_CSS_LEAVE_ERR_PRIVATE(-EINVAL); + return -EINVAL; } /* TMP: tracking of paramsets */ g_param_buffer_dequeue_count = 0; @@ -2920,7 +2921,7 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) { stream->per_frame_isp_params_configs = NULL; err = sh_css_create_isp_params(stream, &stream->isp_params_configs); - if (err != IA_CSS_SUCCESS) + if (err) goto ERR; params = stream->isp_params_configs; @@ -2928,7 +2929,7 @@ ia_css_stream_isp_parameters_init(struct ia_css_stream *stream) { { /* we do not return the error immediately to enable internal * firmware feature testing */ - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; } ddr_ptrs = ¶ms->ddr_ptrs; @@ -2968,22 +2969,22 @@ ia_css_set_sdis2_config( ia_css_set_sdis2_vertproj_config(params, dvs2_coefs); } -static enum ia_css_err +static int sh_css_create_isp_params(struct ia_css_stream *stream, struct ia_css_isp_parameters **isp_params_out) { bool succ = true; unsigned int i; struct sh_css_ddr_address_map *ddr_ptrs; struct sh_css_ddr_address_map_size *ddr_ptrs_size; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; size_t params_size; struct ia_css_isp_parameters *params = - sh_css_malloc(sizeof(struct ia_css_isp_parameters)); + kvmalloc(sizeof(struct ia_css_isp_parameters), GFP_KERNEL); if (!params) { *isp_params_out = NULL; - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_ERROR("%s:%d error: cannot allocate memory", __FILE__, __LINE__); IA_CSS_LEAVE_ERR_PRIVATE(err); return err; @@ -3010,13 +3011,13 @@ sh_css_create_isp_params(struct ia_css_stream *stream, ddr_ptrs_size->isp_param = params_size; ddr_ptrs->isp_param = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER, - mmgr_malloc(params_size)); + hmm_alloc(params_size, HMM_BO_PRIVATE, 0, NULL, 0)); succ &= (ddr_ptrs->isp_param != mmgr_NULL); ddr_ptrs_size->macc_tbl = sizeof(struct ia_css_macc_table); ddr_ptrs->macc_tbl = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_BUFFER, - mmgr_malloc(sizeof(struct ia_css_macc_table))); + hmm_alloc(sizeof(struct ia_css_macc_table), HMM_BO_PRIVATE, 0, NULL, 0)); succ &= (ddr_ptrs->macc_tbl != mmgr_NULL); *isp_params_out = params; @@ -3068,12 +3069,12 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream, ia_css_set_ob_config(params, &default_ob_config); ia_css_set_dp_config(params, &default_dp_config); - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { ia_css_set_param_exceptions(pipe_in, params); } else { for (i = 0; i < stream->num_pipes; i++) { if (sh_css_select_dp_10bpp_config(stream->pipes[i], - &is_dp_10bpp) == IA_CSS_SUCCESS) { + &is_dp_10bpp) == 0) { /* set the return value as false if both DPC and * BDS is enabled by the user. But we do not return * the value immediately to enable internal firmware @@ -3180,7 +3181,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream, ia_css_set_formats_config(params, &stream_params->formats_config); for (i = 0; i < stream->num_pipes; i++) { - if (IA_CSS_SUCCESS == + if (0 == sh_css_select_dp_10bpp_config(stream->pipes[i], &is_dp_10bpp)) { /* set the return value as false if both DPC and * BDS is enabled by the user. But we do not return @@ -3195,7 +3196,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream, retval = false; goto exit; } - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { if (stream->pipes[i]->mode < IA_CSS_PIPE_ID_NUM) { sh_css_set_dp_config(stream->pipes[i], params, &stream_params->pipe_dp_config[stream->pipes[i]->mode]); @@ -3207,7 +3208,7 @@ sh_css_init_isp_params_from_global(struct ia_css_stream *stream, } } - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) ia_css_set_param_exceptions(pipe_in, params); params->fpn_config.data = stream_params->fpn_config.data; @@ -3252,7 +3253,7 @@ exit: return retval; } -enum ia_css_err +int sh_css_params_init(void) { int i, p; @@ -3267,18 +3268,20 @@ sh_css_params_init(void) { for (i = 0; i < SH_CSS_MAX_STAGES; i++) { xmem_sp_stage_ptrs[p][i] = ia_css_refcount_increment(-1, - mmgr_calloc(1, - sizeof(struct sh_css_sp_stage))); + hmm_alloc(sizeof(struct sh_css_sp_stage), + HMM_BO_PRIVATE, 0, NULL, + ATOMISP_MAP_FLAG_CLEARED)); xmem_isp_stage_ptrs[p][i] = ia_css_refcount_increment(-1, - mmgr_calloc(1, - sizeof(struct sh_css_isp_stage))); + hmm_alloc(sizeof(struct sh_css_sp_stage), + HMM_BO_PRIVATE, 0, NULL, + ATOMISP_MAP_FLAG_CLEARED)); if ((xmem_sp_stage_ptrs[p][i] == mmgr_NULL) || (xmem_isp_stage_ptrs[p][i] == mmgr_NULL)) { sh_css_params_uninit(); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } } } @@ -3288,21 +3291,25 @@ sh_css_params_init(void) { ia_css_config_rgb_gamma_tables(); ia_css_config_xnr_table(); - sp_ddr_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1, - CEIL_MUL(sizeof(struct sh_css_ddr_address_map), - HIVE_ISP_DDR_WORD_BYTES))); - xmem_sp_group_ptrs = ia_css_refcount_increment(-1, mmgr_calloc(1, - sizeof(struct sh_css_sp_group))); + sp_ddr_ptrs = ia_css_refcount_increment(-1, + hmm_alloc(CEIL_MUL(sizeof(struct sh_css_ddr_address_map), + HIVE_ISP_DDR_WORD_BYTES), + HMM_BO_PRIVATE, 0, NULL, + ATOMISP_MAP_FLAG_CLEARED)); + xmem_sp_group_ptrs = ia_css_refcount_increment(-1, + hmm_alloc(sizeof(struct sh_css_sp_group), + HMM_BO_PRIVATE, 0, NULL, + ATOMISP_MAP_FLAG_CLEARED)); if ((sp_ddr_ptrs == mmgr_NULL) || (xmem_sp_group_ptrs == mmgr_NULL)) { ia_css_uninit(); - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } static void host_lut_store(const void *lut) @@ -3314,27 +3321,27 @@ static void host_lut_store(const void *lut) } /* Note that allocation is in ipu address space. */ -inline hrt_vaddress sh_css_params_alloc_gdc_lut(void) +inline ia_css_ptr sh_css_params_alloc_gdc_lut(void) { - return mmgr_malloc(sizeof(zoom_table)); + return hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0); } -inline void sh_css_params_free_gdc_lut(hrt_vaddress addr) +inline void sh_css_params_free_gdc_lut(ia_css_ptr addr) { if (addr != mmgr_NULL) hmm_free(addr); } -enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, +int ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, const void *lut) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool stream_started = false; IA_CSS_ENTER("pipe=%p lut=%p", pipe, lut); if (!lut || !pipe) { - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE("err=%d", err); return err; } @@ -3347,7 +3354,7 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "unable to set scaler lut since stream has started\n"); stream_started = true; - err = IA_CSS_ERR_NOT_SUPPORTED; + err = -ENOTSUPP; } /* Free any existing tables. */ @@ -3355,19 +3362,19 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, pipe->scaler_pp_lut = mmgr_NULL; if (!stream_started) { - if (!atomisp_hw_is_isp2401) - pipe->scaler_pp_lut = mmgr_malloc(sizeof(zoom_table)); + if (!IS_ISP2401) + pipe->scaler_pp_lut = hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0); else pipe->scaler_pp_lut = sh_css_params_alloc_gdc_lut(); if (pipe->scaler_pp_lut == mmgr_NULL) { ia_css_debug_dtrace(IA_CSS_DEBUG_ERROR, "unable to allocate scaler_pp_lut\n"); - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; } else { gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])lut, interleaved_lut_temp); - mmgr_store(pipe->scaler_pp_lut, + hmm_store(pipe->scaler_pp_lut, (int *)interleaved_lut_temp, sizeof(zoom_table)); } @@ -3378,7 +3385,7 @@ enum ia_css_err ia_css_pipe_set_bci_scaler_lut(struct ia_css_pipe *pipe, } /* if pipe is NULL, returns default lut addr. */ -hrt_vaddress sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe) +ia_css_ptr sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe) { assert(pipe); @@ -3388,9 +3395,9 @@ hrt_vaddress sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe) return sh_css_params_get_default_gdc_lut(); } -enum ia_css_err sh_css_params_map_and_store_default_gdc_lut(void) +int sh_css_params_map_and_store_default_gdc_lut(void) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; IA_CSS_ENTER_PRIVATE("void"); @@ -3400,17 +3407,17 @@ enum ia_css_err sh_css_params_map_and_store_default_gdc_lut(void) host_lut_store((void *)zoom_table); - if (!atomisp_hw_is_isp2401) - default_gdc_lut = mmgr_malloc(sizeof(zoom_table)); + if (!IS_ISP2401) + default_gdc_lut = hmm_alloc(sizeof(zoom_table), HMM_BO_PRIVATE, 0, NULL, 0); else default_gdc_lut = sh_css_params_alloc_gdc_lut(); if (default_gdc_lut == mmgr_NULL) - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + return -ENOMEM; gdc_lut_convert_to_isp_format((const int(*)[HRT_GDC_N])zoom_table, interleaved_lut_temp); - mmgr_store(default_gdc_lut, (int *)interleaved_lut_temp, + hmm_store(default_gdc_lut, (int *)interleaved_lut_temp, sizeof(zoom_table)); IA_CSS_LEAVE_PRIVATE("lut(%u) err=%d", default_gdc_lut, err); @@ -3427,13 +3434,13 @@ void sh_css_params_free_default_gdc_lut(void) IA_CSS_LEAVE_PRIVATE("void"); } -hrt_vaddress sh_css_params_get_default_gdc_lut(void) +ia_css_ptr sh_css_params_get_default_gdc_lut(void) { return default_gdc_lut; } static void free_param_set_callback( - hrt_vaddress ptr) + ia_css_ptr ptr) { IA_CSS_ENTER_PRIVATE("void"); @@ -3443,7 +3450,7 @@ static void free_param_set_callback( } static void free_buffer_callback( - hrt_vaddress ptr) + ia_css_ptr ptr) { IA_CSS_ENTER_PRIVATE("void"); @@ -3476,7 +3483,7 @@ static void free_map(struct sh_css_ddr_address_map *map) { unsigned int i; - hrt_vaddress *addrs = (hrt_vaddress *)map; + ia_css_ptr *addrs = (ia_css_ptr *)map; IA_CSS_ENTER_PRIVATE("void"); @@ -3521,7 +3528,7 @@ ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream) free_map(&per_frame_params->ddr_ptrs); if (params->fpn_config.data) { - sh_css_free(params->fpn_config.data); + kvfree(params->fpn_config.data); params->fpn_config.data = NULL; } @@ -3537,9 +3544,9 @@ ia_css_stream_isp_parameters_uninit(struct ia_css_stream *stream) } } - sh_css_free(params); + kvfree(params); if (per_frame_params) - sh_css_free(per_frame_params); + kvfree(per_frame_params); stream->isp_params_configs = NULL; stream->per_frame_isp_params_configs = NULL; @@ -3603,7 +3610,7 @@ convert_allocate_morph_plane( me = ia_css_host_data_allocate((size_t)isp_data_size); if (!me) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); return NULL; } @@ -3622,12 +3629,12 @@ convert_allocate_morph_plane( return me; } -static enum ia_css_err +static int store_morph_plane( unsigned short *data, unsigned int width, unsigned int height, - hrt_vaddress dest, + ia_css_ptr dest, unsigned int aligned_width) { struct ia_css_host_data *isp_data; @@ -3636,18 +3643,18 @@ store_morph_plane( isp_data = convert_allocate_morph_plane(data, width, height, aligned_width); if (!isp_data) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } ia_css_params_store_ia_css_host_data(dest, isp_data); ia_css_host_data_free(isp_data); - return IA_CSS_SUCCESS; + return 0; } static void sh_css_update_isp_params_to_ddr( struct ia_css_isp_parameters *params, - hrt_vaddress ddr_ptr) + ia_css_ptr ddr_ptr) { size_t size = sizeof(params->uds); @@ -3655,13 +3662,13 @@ static void sh_css_update_isp_params_to_ddr( assert(params); - mmgr_store(ddr_ptr, ¶ms->uds, size); + hmm_store(ddr_ptr, ¶ms->uds, size); IA_CSS_LEAVE_PRIVATE("void"); } static void sh_css_update_isp_mem_params_to_ddr( const struct ia_css_binary *binary, - hrt_vaddress ddr_mem_ptr, + ia_css_ptr ddr_mem_ptr, size_t size, enum ia_css_isp_memories mem) { @@ -3671,7 +3678,7 @@ static void sh_css_update_isp_mem_params_to_ddr( params = ia_css_isp_param_get_mem_init(&binary->mem_params, IA_CSS_PARAM_CLASS_PARAM, mem); - mmgr_store(ddr_mem_ptr, params->address, size); + hmm_store(ddr_mem_ptr, params->address, size); IA_CSS_LEAVE_PRIVATE("void"); } @@ -3679,7 +3686,7 @@ static void sh_css_update_isp_mem_params_to_ddr( void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void) { unsigned int i; - hrt_vaddress cpy; + ia_css_ptr cpy; enum sh_css_queue_id param_queue_ids[3] = { IA_CSS_PARAMETER_SET_QUEUE_ID, IA_CSS_PER_FRAME_PARAMETER_SET_QUEUE_ID, SH_CSS_INVALID_QUEUE_ID @@ -3694,10 +3701,10 @@ void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void) } for (i = 0; SH_CSS_INVALID_QUEUE_ID != param_queue_ids[i]; i++) { - cpy = (hrt_vaddress)0; + cpy = (ia_css_ptr)0; /* clean-up old copy */ while (ia_css_bufq_dequeue_buffer(param_queue_ids[i], - (uint32_t *)&cpy) == IA_CSS_SUCCESS) { + (uint32_t *)&cpy) == 0) { /* TMP: keep track of dequeued param set count */ g_param_buffer_dequeue_count++; @@ -3709,7 +3716,7 @@ void ia_css_dequeue_param_buffers(/*unsigned int pipe_num*/ void) IA_CSS_LOG("dequeued param set %x from %d, release ref", cpy, 0); free_ia_css_isp_parameter_set_info(cpy); - cpy = (hrt_vaddress)0; + cpy = (ia_css_ptr)0; } } @@ -3751,13 +3758,13 @@ process_kernel_parameters(unsigned int pipe_id, } } -enum ia_css_err +int sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, struct ia_css_isp_parameters *params, bool commit, struct ia_css_pipe *pipe_in) { - enum ia_css_err err = IA_CSS_SUCCESS; - hrt_vaddress cpy; + int err = 0; + ia_css_ptr cpy; int i; unsigned int raw_bit_depth = 10; unsigned int isp_pipe_version = SH_CSS_ISP_PIPE_VERSION_1; @@ -3807,7 +3814,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, #endif if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ - err = IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + err = -EBUSY; break; } cur_map = ¶ms->pipe_ddr_ptrs[pipeline->pipe_id]; @@ -3823,7 +3830,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, /* the processing is a.o. resolution dependent */ err = ia_css_process_zoom_and_motion(params, pipeline->stages); - if (err != IA_CSS_SUCCESS) + if (err) return err; } /* check if to actually update the parameters for this pipe */ @@ -3854,14 +3861,14 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, cur_map, cur_map_size); - if (err != IA_CSS_SUCCESS) + if (err) break; for (mem = 0; mem < IA_CSS_NUM_MEMORIES; mem++) { params->isp_mem_params_changed [pipeline->pipe_id][stage->stage_num][mem] = false; } } /* for */ - if (err != IA_CSS_SUCCESS) + if (err) break; /* update isp_params to pipe specific copies */ if (params->isp_params_changed) { @@ -3870,7 +3877,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, cur_map_size->isp_param, true, &err); - if (err != IA_CSS_SUCCESS) + if (err) break; sh_css_update_isp_params_to_ddr(params, cur_map->isp_param); } @@ -3879,7 +3886,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, err = ref_sh_css_ddr_address_map( cur_map, &isp_params_info.mem_map); - if (err != IA_CSS_SUCCESS) + if (err) break; /* Update Parameters ID */ @@ -3891,14 +3898,14 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, /* now write the copy to ddr */ err = write_ia_css_isp_parameter_set_info_to_ddr(&isp_params_info, &cpy); - if (err != IA_CSS_SUCCESS) + if (err) break; /* enqueue the set to sp */ IA_CSS_LOG("queue param set %x to %d", cpy, thread_id); err = ia_css_bufq_enqueue_buffer(thread_id, queue_id, (uint32_t)cpy); - if (err != IA_CSS_SUCCESS) { + if (err) { free_ia_css_isp_parameter_set_info(cpy); #if defined(SH_CSS_ENABLE_PER_FRAME_PARAMS) IA_CSS_LOG("pfp: FAILED to add config id %d for OF %d to q %d on thread %d", @@ -3919,8 +3926,8 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, */ if (!sh_css_sp_is_running()) { /* SP is not running. The queues are not valid */ - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_RESOURCE_NOT_AVAILABLE); - return IA_CSS_ERR_RESOURCE_NOT_AVAILABLE; + IA_CSS_LEAVE_ERR_PRIVATE(-EBUSY); + return -EBUSY; } ia_css_bufq_enqueue_psys_event( IA_CSS_PSYS_SW_EVENT_BUFFER_ENQUEUED, @@ -3957,7 +3964,7 @@ sh_css_param_update_isp_params(struct ia_css_pipe *curr_pipe, return err; } -static enum ia_css_err +static int sh_css_params_write_to_ddr_internal( struct ia_css_pipe *pipe, unsigned int pipe_id, @@ -3965,7 +3972,7 @@ sh_css_params_write_to_ddr_internal( const struct ia_css_pipeline_stage *stage, struct sh_css_ddr_address_map *ddr_map, struct sh_css_ddr_address_map_size *ddr_map_size) { - enum ia_css_err err; + int err; const struct ia_css_binary *binary; unsigned int stage_num; @@ -3993,14 +4000,14 @@ sh_css_params_write_to_ddr_internal( (size_t)(FPNTBL_BYTES(binary)), params->config_changed[IA_CSS_FPN_ID], &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } if (params->config_changed[IA_CSS_FPN_ID] || buff_realloced) { if (params->fpn_config.enabled) { err = store_fpntbl(params, ddr_map->fpn_tbl); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4013,7 +4020,7 @@ sh_css_params_write_to_ddr_internal( u32 enable_conv; size_t bytes; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) bytes = ISP2400_SCTBL_BYTES(binary); else bytes = ISP2401_SCTBL_BYTES(binary); @@ -4025,7 +4032,7 @@ sh_css_params_write_to_ddr_internal( bytes, params->sc_table_changed, &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4036,7 +4043,7 @@ sh_css_params_write_to_ddr_internal( if (params->sc_table) { /* store the shading table to ddr */ err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_table); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4054,13 +4061,13 @@ sh_css_params_write_to_ddr_internal( binary->sctbl_width_per_color, binary->sctbl_height); if (!params->sc_config) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } /* store the shading table to ddr */ err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4085,13 +4092,13 @@ sh_css_params_write_to_ddr_internal( ¶ms->sc_config, binary, pipe->required_bds_factor); if (!params->sc_config) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } /* store the shading table to ddr */ err = ia_css_params_store_sctbl(stage, ddr_map->sc_tbl, params->sc_config); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4110,7 +4117,7 @@ sh_css_params_write_to_ddr_internal( /* DPC configuration is made pipe specific to allow flexibility in positioning of the * DPC kernel. The code below sets the pipe specific configuration to * individual binaries. */ - if (atomisp_hw_is_isp2401 && + if (IS_ISP2401 && params->pipe_dpc_config_changed[pipe_id] && binary->info->sp.enable.dpc) { unsigned int size = @@ -4171,11 +4178,11 @@ sh_css_params_write_to_ddr_internal( ddr_map_size->macc_tbl, true, &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - mmgr_store(ddr_map->macc_tbl, + hmm_store(ddr_map->macc_tbl, converted_macc_table.data, sizeof(converted_macc_table.data)); } @@ -4192,7 +4199,7 @@ sh_css_params_write_to_ddr_internal( (size_t)((DVS_6AXIS_BYTES(binary) / 2) * 3), params->pipe_dvs_6axis_config_changed[pipe_id], &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4213,7 +4220,7 @@ sh_css_params_write_to_ddr_internal( if (!params->pipe_dvs_6axis_config[pipe_id]) { struct ia_css_resolution dvs_offset = {0}; - if (!atomisp_hw_is_isp2401) { + if (!IS_ISP2401) { dvs_offset.width = (PIX_SHIFT_FILTER_RUN_IN_X + binary->dvs_envelope.width) / 2; } else { if (binary->dvs_envelope.width || binary->dvs_envelope.height) { @@ -4225,8 +4232,8 @@ sh_css_params_write_to_ddr_internal( params->pipe_dvs_6axis_config[pipe_id] = generate_dvs_6axis_table(&binary->out_frame_info[0].res, &dvs_offset); if (!params->pipe_dvs_6axis_config[pipe_id]) { - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY); - return IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + IA_CSS_LEAVE_ERR_PRIVATE(-ENOMEM); + return -ENOMEM; } params->pipe_dvs_6axis_config_changed[pipe_id] = true; @@ -4242,13 +4249,17 @@ sh_css_params_write_to_ddr_internal( if (binary->info->sp.enable.ca_gdc) { unsigned int i; - hrt_vaddress *virt_addr_tetra_x[ + ia_css_ptr *virt_addr_tetra_x[ + IA_CSS_MORPH_TABLE_NUM_PLANES]; size_t *virt_size_tetra_x[ + IA_CSS_MORPH_TABLE_NUM_PLANES]; - hrt_vaddress *virt_addr_tetra_y[ + ia_css_ptr *virt_addr_tetra_y[ + IA_CSS_MORPH_TABLE_NUM_PLANES]; size_t *virt_size_tetra_y[ + IA_CSS_MORPH_TABLE_NUM_PLANES]; virt_addr_tetra_x[0] = &ddr_map->tetra_r_x; @@ -4288,7 +4299,7 @@ sh_css_params_write_to_ddr_internal( (MORPH_PLANE_BYTES(binary)), params->morph_table_changed, &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4299,7 +4310,7 @@ sh_css_params_write_to_ddr_internal( (MORPH_PLANE_BYTES(binary)), params->morph_table_changed, &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4316,7 +4327,7 @@ sh_css_params_write_to_ddr_internal( if (!table) { err = sh_css_params_default_morph_table(&id_table, binary); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4354,7 +4365,7 @@ sh_css_params_write_to_ddr_internal( size, params->isp_mem_params_changed[pipe_id][stage_num][mem], &err); - if (err != IA_CSS_SUCCESS) { + if (err) { IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } @@ -4365,8 +4376,8 @@ sh_css_params_write_to_ddr_internal( } } - IA_CSS_LEAVE_ERR_PRIVATE(IA_CSS_SUCCESS); - return IA_CSS_SUCCESS; + IA_CSS_LEAVE_ERR_PRIVATE(0); + return 0; } const struct ia_css_fpn_table *ia_css_get_fpn_table(struct ia_css_stream @@ -4457,42 +4468,42 @@ struct ia_css_shading_table *ia_css_get_shading_table(struct ia_css_stream return table; } -hrt_vaddress sh_css_store_sp_group_to_ddr(void) +ia_css_ptr sh_css_store_sp_group_to_ddr(void) { IA_CSS_ENTER_LEAVE_PRIVATE("void"); - mmgr_store(xmem_sp_group_ptrs, + hmm_store(xmem_sp_group_ptrs, &sh_css_sp_group, sizeof(struct sh_css_sp_group)); return xmem_sp_group_ptrs; } -hrt_vaddress sh_css_store_sp_stage_to_ddr( +ia_css_ptr sh_css_store_sp_stage_to_ddr( unsigned int pipe, unsigned int stage) { IA_CSS_ENTER_LEAVE_PRIVATE("void"); - mmgr_store(xmem_sp_stage_ptrs[pipe][stage], + hmm_store(xmem_sp_stage_ptrs[pipe][stage], &sh_css_sp_stage, sizeof(struct sh_css_sp_stage)); return xmem_sp_stage_ptrs[pipe][stage]; } -hrt_vaddress sh_css_store_isp_stage_to_ddr( +ia_css_ptr sh_css_store_isp_stage_to_ddr( unsigned int pipe, unsigned int stage) { IA_CSS_ENTER_LEAVE_PRIVATE("void"); - mmgr_store(xmem_isp_stage_ptrs[pipe][stage], + hmm_store(xmem_isp_stage_ptrs[pipe][stage], &sh_css_isp_stage, sizeof(struct sh_css_isp_stage)); return xmem_isp_stage_ptrs[pipe][stage]; } -static enum ia_css_err ref_sh_css_ddr_address_map( +static int ref_sh_css_ddr_address_map( struct sh_css_ddr_address_map *map, struct sh_css_ddr_address_map *out) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; unsigned int i; /* we will use a union to copy things; overlaying an array @@ -4501,7 +4512,7 @@ static enum ia_css_err ref_sh_css_ddr_address_map( */ union { struct sh_css_ddr_address_map *map; - hrt_vaddress *addrs; + ia_css_ptr *addrs; } in_addrs, to_addrs; IA_CSS_ENTER_PRIVATE("void"); @@ -4512,7 +4523,7 @@ static enum ia_css_err ref_sh_css_ddr_address_map( to_addrs.map = out; assert(sizeof(struct sh_css_ddr_address_map_size) / sizeof(size_t) == - sizeof(struct sh_css_ddr_address_map) / sizeof(hrt_vaddress)); + sizeof(struct sh_css_ddr_address_map) / sizeof(ia_css_ptr)); /* copy map using size info */ for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size) / @@ -4528,11 +4539,11 @@ static enum ia_css_err ref_sh_css_ddr_address_map( return err; } -static enum ia_css_err write_ia_css_isp_parameter_set_info_to_ddr( +static int write_ia_css_isp_parameter_set_info_to_ddr( struct ia_css_isp_parameter_set_info *me, - hrt_vaddress *out) + ia_css_ptr *out) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; bool succ; IA_CSS_ENTER_PRIVATE("void"); @@ -4540,26 +4551,26 @@ static enum ia_css_err write_ia_css_isp_parameter_set_info_to_ddr( assert(me); assert(out); - *out = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_SET_POOL, mmgr_malloc( - sizeof(struct ia_css_isp_parameter_set_info))); + *out = ia_css_refcount_increment(IA_CSS_REFCOUNT_PARAM_SET_POOL, + hmm_alloc(sizeof(struct ia_css_isp_parameter_set_info), HMM_BO_PRIVATE, 0, NULL, 0)); succ = (*out != mmgr_NULL); if (succ) - mmgr_store(*out, + hmm_store(*out, me, sizeof(struct ia_css_isp_parameter_set_info)); else - err = IA_CSS_ERR_CANNOT_ALLOCATE_MEMORY; + err = -ENOMEM; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } -static enum ia_css_err +static int free_ia_css_isp_parameter_set_info( - hrt_vaddress ptr) { - enum ia_css_err err = IA_CSS_SUCCESS; + ia_css_ptr ptr) { + int err = 0; struct ia_css_isp_parameter_set_info isp_params_info; unsigned int i; - hrt_vaddress *addrs = (hrt_vaddress *) &isp_params_info.mem_map; + ia_css_ptr *addrs = (ia_css_ptr *)&isp_params_info.mem_map; IA_CSS_ENTER_PRIVATE("ptr = %u", ptr); @@ -4568,12 +4579,12 @@ free_ia_css_isp_parameter_set_info( { IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_SET_POOL(0x%x) invalid arg", __func__, ptr); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; IA_CSS_LEAVE_ERR_PRIVATE(err); return err; } - mmgr_load(ptr, &isp_params_info.mem_map, sizeof(struct sh_css_ddr_address_map)); + hmm_load(ptr, &isp_params_info.mem_map, sizeof(struct sh_css_ddr_address_map)); /* copy map using size info */ for (i = 0; i < (sizeof(struct sh_css_ddr_address_map_size) / sizeof(size_t)); i++) @@ -4585,7 +4596,7 @@ free_ia_css_isp_parameter_set_info( if (!ia_css_refcount_is_valid(addrs[i])) { IA_CSS_ERROR("%s: IA_CSS_REFCOUNT_PARAM_BUFFER(0x%x) invalid arg", __func__, ptr); - err = IA_CSS_ERR_INVALID_ARGUMENTS; + err = -EINVAL; continue; } @@ -4773,7 +4784,7 @@ sh_css_update_uds_and_crop_info( IA_CSS_LEAVE_PRIVATE("void"); } -static enum ia_css_err +static int sh_css_update_uds_and_crop_info_based_on_zoom_region( const struct ia_css_binary_info *info, const struct ia_css_frame_info *in_frame_info, @@ -4786,7 +4797,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region( struct ia_css_resolution pipe_in_res, bool enable_zoom) { unsigned int x0 = 0, y0 = 0, x1 = 0, y1 = 0; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; /* Note: * Filter_Envelope = 0 for NND/LUT * Filter_Envelope = 1 for BCI @@ -4813,7 +4824,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region( y1 = zoom->zoom_region.resolution.height + y0; if ((x0 > x1) || (y0 > y1) || (x1 > pipe_in_res.width) || (y1 > pipe_in_res.height)) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; if (!enable_zoom) { @@ -4825,7 +4836,7 @@ sh_css_update_uds_and_crop_info_based_on_zoom_region( { /* Zoom region is only supported by the UDS module on ISP * 2 and higher. It is not supported in video mode on ISP 1 */ - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } else { if (enable_zoom) { @@ -4875,19 +4886,18 @@ ia_css_3a_statistics_allocate(const struct ia_css_3a_grid_info *grid) assert(grid); - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; me->grid = *grid; grid_size = grid->width * grid->height; - me->data = sh_css_malloc(grid_size * sizeof(*me->data)); + me->data = kvmalloc(grid_size * sizeof(*me->data), GFP_KERNEL); if (!me->data) goto err; #if !defined(HAS_NO_HMEM) /* No weighted histogram, no structure, treat the histogram data as a byte dump in a byte array */ - me->rgby_data = (struct ia_css_3a_rgby_output *)sh_css_malloc(sizeof_hmem( - HMEM0_ID)); + me->rgby_data = kvmalloc(sizeof_hmem(HMEM0_ID), GFP_KERNEL); #else me->rgby_data = NULL; #endif @@ -4905,10 +4915,10 @@ void ia_css_3a_statistics_free(struct ia_css_3a_statistics *me) { if (me) { - sh_css_free(me->rgby_data); - sh_css_free(me->data); + kvfree(me->rgby_data); + kvfree(me->data); memset(me, 0, sizeof(struct ia_css_3a_statistics)); - sh_css_free(me); + kvfree(me); } } @@ -4919,18 +4929,18 @@ ia_css_dvs_statistics_allocate(const struct ia_css_dvs_grid_info *grid) assert(grid); - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; me->grid = *grid; - me->hor_proj = sh_css_malloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES * - sizeof(*me->hor_proj)); + me->hor_proj = kvmalloc(grid->height * IA_CSS_DVS_NUM_COEF_TYPES * + sizeof(*me->hor_proj), GFP_KERNEL); if (!me->hor_proj) goto err; - me->ver_proj = sh_css_malloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES * - sizeof(*me->ver_proj)); + me->ver_proj = kvmalloc(grid->width * IA_CSS_DVS_NUM_COEF_TYPES * + sizeof(*me->ver_proj), GFP_KERNEL); if (!me->ver_proj) goto err; @@ -4944,10 +4954,10 @@ void ia_css_dvs_statistics_free(struct ia_css_dvs_statistics *me) { if (me) { - sh_css_free(me->hor_proj); - sh_css_free(me->ver_proj); + kvfree(me->hor_proj); + kvfree(me->ver_proj); memset(me, 0, sizeof(struct ia_css_dvs_statistics)); - sh_css_free(me); + kvfree(me); } } @@ -4958,21 +4968,21 @@ ia_css_dvs_coefficients_allocate(const struct ia_css_dvs_grid_info *grid) assert(grid); - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; me->grid = *grid; - me->hor_coefs = sh_css_malloc(grid->num_hor_coefs * - IA_CSS_DVS_NUM_COEF_TYPES * - sizeof(*me->hor_coefs)); + me->hor_coefs = kvmalloc(grid->num_hor_coefs * + IA_CSS_DVS_NUM_COEF_TYPES * + sizeof(*me->hor_coefs), GFP_KERNEL); if (!me->hor_coefs) goto err; - me->ver_coefs = sh_css_malloc(grid->num_ver_coefs * - IA_CSS_DVS_NUM_COEF_TYPES * - sizeof(*me->ver_coefs)); + me->ver_coefs = kvmalloc(grid->num_ver_coefs * + IA_CSS_DVS_NUM_COEF_TYPES * + sizeof(*me->ver_coefs), GFP_KERNEL); if (!me->ver_coefs) goto err; @@ -4986,10 +4996,10 @@ void ia_css_dvs_coefficients_free(struct ia_css_dvs_coefficients *me) { if (me) { - sh_css_free(me->hor_coefs); - sh_css_free(me->ver_coefs); + kvfree(me->hor_coefs); + kvfree(me->ver_coefs); memset(me, 0, sizeof(struct ia_css_dvs_coefficients)); - sh_css_free(me); + kvfree(me); } } @@ -5000,49 +5010,65 @@ ia_css_dvs2_statistics_allocate(const struct ia_css_dvs_grid_info *grid) assert(grid); - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; me->grid = *grid; - me->hor_prod.odd_real = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->hor_prod.odd_real)); + me->hor_prod.odd_real = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->hor_prod.odd_real), + GFP_KERNEL); if (!me->hor_prod.odd_real) goto err; - me->hor_prod.odd_imag = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->hor_prod.odd_imag)); + me->hor_prod.odd_imag = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->hor_prod.odd_imag), + GFP_KERNEL); if (!me->hor_prod.odd_imag) goto err; - me->hor_prod.even_real = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->hor_prod.even_real)); + me->hor_prod.even_real = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->hor_prod.even_real), + GFP_KERNEL); if (!me->hor_prod.even_real) goto err; - me->hor_prod.even_imag = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->hor_prod.even_imag)); + me->hor_prod.even_imag = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->hor_prod.even_imag), + GFP_KERNEL); if (!me->hor_prod.even_imag) goto err; - me->ver_prod.odd_real = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->ver_prod.odd_real)); + me->ver_prod.odd_real = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->ver_prod.odd_real), + GFP_KERNEL); if (!me->ver_prod.odd_real) goto err; - me->ver_prod.odd_imag = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->ver_prod.odd_imag)); + me->ver_prod.odd_imag = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->ver_prod.odd_imag), + GFP_KERNEL); if (!me->ver_prod.odd_imag) goto err; - me->ver_prod.even_real = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->ver_prod.even_real)); + me->ver_prod.even_real = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->ver_prod.even_real), + GFP_KERNEL); if (!me->ver_prod.even_real) goto err; - me->ver_prod.even_imag = sh_css_malloc(grid->aligned_width * - grid->aligned_height * sizeof(*me->ver_prod.even_imag)); + me->ver_prod.even_imag = kvmalloc(grid->aligned_width * + grid->aligned_height * + sizeof(*me->ver_prod.even_imag), + GFP_KERNEL); if (!me->ver_prod.even_imag) goto err; @@ -5056,16 +5082,16 @@ void ia_css_dvs2_statistics_free(struct ia_css_dvs2_statistics *me) { if (me) { - sh_css_free(me->hor_prod.odd_real); - sh_css_free(me->hor_prod.odd_imag); - sh_css_free(me->hor_prod.even_real); - sh_css_free(me->hor_prod.even_imag); - sh_css_free(me->ver_prod.odd_real); - sh_css_free(me->ver_prod.odd_imag); - sh_css_free(me->ver_prod.even_real); - sh_css_free(me->ver_prod.even_imag); + kvfree(me->hor_prod.odd_real); + kvfree(me->hor_prod.odd_imag); + kvfree(me->hor_prod.even_real); + kvfree(me->hor_prod.even_imag); + kvfree(me->ver_prod.odd_real); + kvfree(me->ver_prod.odd_imag); + kvfree(me->ver_prod.even_real); + kvfree(me->ver_prod.even_imag); memset(me, 0, sizeof(struct ia_css_dvs2_statistics)); - sh_css_free(me); + kvfree(me); } } @@ -5076,49 +5102,57 @@ ia_css_dvs2_coefficients_allocate(const struct ia_css_dvs_grid_info *grid) assert(grid); - me = sh_css_calloc(1, sizeof(*me)); + me = kvcalloc(1, sizeof(*me), GFP_KERNEL); if (!me) goto err; me->grid = *grid; - me->hor_coefs.odd_real = sh_css_malloc(grid->num_hor_coefs * - sizeof(*me->hor_coefs.odd_real)); + me->hor_coefs.odd_real = kvmalloc(grid->num_hor_coefs * + sizeof(*me->hor_coefs.odd_real), + GFP_KERNEL); if (!me->hor_coefs.odd_real) goto err; - me->hor_coefs.odd_imag = sh_css_malloc(grid->num_hor_coefs * - sizeof(*me->hor_coefs.odd_imag)); + me->hor_coefs.odd_imag = kvmalloc(grid->num_hor_coefs * + sizeof(*me->hor_coefs.odd_imag), + GFP_KERNEL); if (!me->hor_coefs.odd_imag) goto err; - me->hor_coefs.even_real = sh_css_malloc(grid->num_hor_coefs * - sizeof(*me->hor_coefs.even_real)); + me->hor_coefs.even_real = kvmalloc(grid->num_hor_coefs * + sizeof(*me->hor_coefs.even_real), + GFP_KERNEL); if (!me->hor_coefs.even_real) goto err; - me->hor_coefs.even_imag = sh_css_malloc(grid->num_hor_coefs * - sizeof(*me->hor_coefs.even_imag)); + me->hor_coefs.even_imag = kvmalloc(grid->num_hor_coefs * + sizeof(*me->hor_coefs.even_imag), + GFP_KERNEL); if (!me->hor_coefs.even_imag) goto err; - me->ver_coefs.odd_real = sh_css_malloc(grid->num_ver_coefs * - sizeof(*me->ver_coefs.odd_real)); + me->ver_coefs.odd_real = kvmalloc(grid->num_ver_coefs * + sizeof(*me->ver_coefs.odd_real), + GFP_KERNEL); if (!me->ver_coefs.odd_real) goto err; - me->ver_coefs.odd_imag = sh_css_malloc(grid->num_ver_coefs * - sizeof(*me->ver_coefs.odd_imag)); + me->ver_coefs.odd_imag = kvmalloc(grid->num_ver_coefs * + sizeof(*me->ver_coefs.odd_imag), + GFP_KERNEL); if (!me->ver_coefs.odd_imag) goto err; - me->ver_coefs.even_real = sh_css_malloc(grid->num_ver_coefs * - sizeof(*me->ver_coefs.even_real)); + me->ver_coefs.even_real = kvmalloc(grid->num_ver_coefs * + sizeof(*me->ver_coefs.even_real), + GFP_KERNEL); if (!me->ver_coefs.even_real) goto err; - me->ver_coefs.even_imag = sh_css_malloc(grid->num_ver_coefs * - sizeof(*me->ver_coefs.even_imag)); + me->ver_coefs.even_imag = kvmalloc(grid->num_ver_coefs * + sizeof(*me->ver_coefs.even_imag), + GFP_KERNEL); if (!me->ver_coefs.even_imag) goto err; @@ -5132,16 +5166,16 @@ void ia_css_dvs2_coefficients_free(struct ia_css_dvs2_coefficients *me) { if (me) { - sh_css_free(me->hor_coefs.odd_real); - sh_css_free(me->hor_coefs.odd_imag); - sh_css_free(me->hor_coefs.even_real); - sh_css_free(me->hor_coefs.even_imag); - sh_css_free(me->ver_coefs.odd_real); - sh_css_free(me->ver_coefs.odd_imag); - sh_css_free(me->ver_coefs.even_real); - sh_css_free(me->ver_coefs.even_imag); + kvfree(me->hor_coefs.odd_real); + kvfree(me->hor_coefs.odd_imag); + kvfree(me->hor_coefs.even_real); + kvfree(me->hor_coefs.even_imag); + kvfree(me->ver_coefs.odd_real); + kvfree(me->ver_coefs.odd_imag); + kvfree(me->ver_coefs.even_real); + kvfree(me->ver_coefs.even_imag); memset(me, 0, sizeof(struct ia_css_dvs2_coefficients)); - sh_css_free(me); + kvfree(me); } } @@ -5164,8 +5198,8 @@ ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream) goto err; } - dvs_config = (struct ia_css_dvs_6axis_config *)sh_css_calloc(1, - sizeof(struct ia_css_dvs_6axis_config)); + dvs_config = kvcalloc(1, sizeof(struct ia_css_dvs_6axis_config), + GFP_KERNEL); if (!dvs_config) goto err; @@ -5179,23 +5213,25 @@ ia_css_dvs2_6axis_config_allocate(const struct ia_css_stream *stream) params->pipe_dvs_6axis_config[IA_CSS_PIPE_ID_VIDEO]->height_uv; IA_CSS_LOG("table Y: W %d H %d", width_y, height_y); IA_CSS_LOG("table UV: W %d H %d", width_uv, height_uv); - dvs_config->xcoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof( - uint32_t)); + dvs_config->xcoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->xcoords_y) goto err; - dvs_config->ycoords_y = (uint32_t *)sh_css_malloc(width_y * height_y * sizeof( - uint32_t)); + dvs_config->ycoords_y = kvmalloc(width_y * height_y * sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->ycoords_y) goto err; - dvs_config->xcoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * - sizeof(uint32_t)); + dvs_config->xcoords_uv = kvmalloc(width_uv * height_uv * + sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->xcoords_uv) goto err; - dvs_config->ycoords_uv = (uint32_t *)sh_css_malloc(width_uv * height_uv * - sizeof(uint32_t)); + dvs_config->ycoords_uv = kvmalloc(width_uv * height_uv * + sizeof(uint32_t), + GFP_KERNEL); if (!dvs_config->ycoords_uv) goto err; @@ -5209,12 +5245,12 @@ void ia_css_dvs2_6axis_config_free(struct ia_css_dvs_6axis_config *dvs_6axis_config) { if (dvs_6axis_config) { - sh_css_free(dvs_6axis_config->xcoords_y); - sh_css_free(dvs_6axis_config->ycoords_y); - sh_css_free(dvs_6axis_config->xcoords_uv); - sh_css_free(dvs_6axis_config->ycoords_uv); + kvfree(dvs_6axis_config->xcoords_y); + kvfree(dvs_6axis_config->ycoords_y); + kvfree(dvs_6axis_config->xcoords_uv); + kvfree(dvs_6axis_config->ycoords_uv); memset(dvs_6axis_config, 0, sizeof(struct ia_css_dvs_6axis_config)); - sh_css_free(dvs_6axis_config); + kvfree(dvs_6axis_config); } } @@ -5225,7 +5261,7 @@ ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable) struct ia_css_pipeline *pipeline; struct ia_css_pipeline_stage *stage; enum ia_css_pipe_id pipe_id; - enum ia_css_err err; + int err; int i; if (!stream) @@ -5239,7 +5275,7 @@ ia_css_en_dz_capt_pipe(struct ia_css_stream *stream, bool enable) if (pipe_id == IA_CSS_PIPE_ID_CAPTURE) { err = ia_css_pipeline_get_stage(pipeline, IA_CSS_BINARY_MODE_CAPTURE_PP, &stage); - if (err == IA_CSS_SUCCESS) + if (!err) stage->enable_zoom = enable; break; } diff --git a/drivers/staging/media/atomisp/pci/sh_css_params.h b/drivers/staging/media/atomisp/pci/sh_css_params.h index 96d503967fd1..62a7b6ada237 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_params.h +++ b/drivers/staging/media/atomisp/pci/sh_css_params.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -150,13 +151,13 @@ struct ia_css_isp_parameters { void ia_css_params_store_ia_css_host_data( - hrt_vaddress ddr_addr, + ia_css_ptr ddr_addr, struct ia_css_host_data *data); -enum ia_css_err +int ia_css_params_store_sctbl( const struct ia_css_pipeline_stage *stage, - hrt_vaddress ddr_addr, + ia_css_ptr ddr_addr, const struct ia_css_shading_table *shading_table); struct ia_css_host_data * @@ -168,21 +169,21 @@ struct ia_css_isp_config * sh_css_pipe_isp_config_get(struct ia_css_pipe *pipe); /* ipu address allocation/free for gdc lut */ -hrt_vaddress +ia_css_ptr sh_css_params_alloc_gdc_lut(void); void -sh_css_params_free_gdc_lut(hrt_vaddress addr); +sh_css_params_free_gdc_lut(ia_css_ptr addr); -enum ia_css_err +int sh_css_params_map_and_store_default_gdc_lut(void); void sh_css_params_free_default_gdc_lut(void); -hrt_vaddress +ia_css_ptr sh_css_params_get_default_gdc_lut(void); -hrt_vaddress +ia_css_ptr sh_css_pipe_get_pp_gdc_lut(const struct ia_css_pipe *pipe); #endif /* _SH_CSS_PARAMS_H_ */ diff --git a/drivers/staging/media/atomisp/pci/sh_css_params_internal.h b/drivers/staging/media/atomisp/pci/sh_css_params_internal.h index baca24532f9f..8e5e6f273a95 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_params_internal.h +++ b/drivers/staging/media/atomisp/pci/sh_css_params_internal.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_pipe.c b/drivers/staging/media/atomisp/pci/sh_css_pipe.c deleted file mode 100644 index 1f57ffad8921..000000000000 --- a/drivers/staging/media/atomisp/pci/sh_css_pipe.c +++ /dev/null @@ -1,16 +0,0 @@ -/* - * Support for Intel Camera Imaging ISP subsystem. - * Copyright (c) 2015, Intel Corporation. - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - */ - -/* This file will contain the code to implement the functions declared in ia_css_pipe.h and ia_css_pipe_public.h - and associated helper functions */ diff --git a/drivers/staging/media/atomisp/pci/sh_css_properties.c b/drivers/staging/media/atomisp/pci/sh_css_properties.c index 50f99c53c3d4..de588f9bd540 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_properties.c +++ b/drivers/staging/media/atomisp/pci/sh_css_properties.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_shading.c b/drivers/staging/media/atomisp/pci/sh_css_shading.c index 2a2d0f4db44b..462caf9cb571 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_shading.c +++ b/drivers/staging/media/atomisp/pci/sh_css_shading.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.c b/drivers/staging/media/atomisp/pci/sh_css_sp.c index e574396ad0f4..a26680b1d0b0 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_sp.c +++ b/drivers/staging/media/atomisp/pci/sh_css_sp.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -12,6 +13,8 @@ * more details. */ +#include "hmm.h" + #include "sh_css_sp.h" #if !defined(HAS_NO_INPUT_FORMATTER) @@ -43,10 +46,8 @@ /*#include "sp.h"*/ /* host2sp_enqueue_frame_data() */ -#include "memory_access.h" #include "assert_support.h" -#include "platform_support.h" /* hrt_sleep() */ #include "sw_event_global.h" /* Event IDs.*/ #include "ia_css_event.h" @@ -75,14 +76,14 @@ static struct sh_css_sp_per_frame_data per_frame_data; /* TODO: add code that sets this bool to false */ static bool sp_running; -static enum ia_css_err +static int set_output_frame_buffer(const struct ia_css_frame *frame, unsigned int idx); static void sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf, const enum sh_css_queue_id queue_id, - const hrt_vaddress xmem_addr, + const ia_css_ptr xmem_addr, const enum ia_css_buffer_type buf_type); static void @@ -412,7 +413,7 @@ sh_css_sp_get_sw_interrupt_value(unsigned int irq) static void sh_css_copy_buffer_attr_to_spbuffer(struct ia_css_buffer_sp *dest_buf, const enum sh_css_queue_id queue_id, - const hrt_vaddress xmem_addr, + const ia_css_ptr xmem_addr, const enum ia_css_buffer_type buf_type) { assert(buf_type < IA_CSS_NUM_BUFFER_TYPE); @@ -535,10 +536,10 @@ sh_css_copy_frame_to_spframe(struct ia_css_frame_sp *sp_frame_out, } } -static enum ia_css_err +static int set_input_frame_buffer(const struct ia_css_frame *frame) { if (!frame) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; switch (frame->info.format) { @@ -558,18 +559,18 @@ set_input_frame_buffer(const struct ia_css_frame *frame) { case IA_CSS_FRAME_FORMAT_CSI_MIPI_YUV420_10: break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.in, frame); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int set_output_frame_buffer(const struct ia_css_frame *frame, unsigned int idx) { if (!frame) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; switch (frame->info.format) { @@ -601,16 +602,16 @@ set_output_frame_buffer(const struct ia_css_frame *frame, case IA_CSS_FRAME_FORMAT_BINARY_8: break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out[idx], frame); - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int set_view_finder_buffer(const struct ia_css_frame *frame) { if (!frame) - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; switch (frame->info.format) { @@ -630,11 +631,11 @@ set_view_finder_buffer(const struct ia_css_frame *frame) { case IA_CSS_FRAME_FORMAT_YUV_LINE: break; default: - return IA_CSS_ERR_INVALID_ARGUMENTS; + return -EINVAL; } sh_css_copy_frame_to_spframe(&sh_css_sp_stage.frames.out_vf, frame); - return IA_CSS_SUCCESS; + return 0; } #if !defined(HAS_NO_INPUT_FORMATTER) @@ -736,26 +737,26 @@ sh_css_sp_set_disable_continuous_viewfinder(bool flag) sh_css_sp_group.config.disable_cont_vf = flag; } -static enum ia_css_err +static int sh_css_sp_write_frame_pointers(const struct sh_css_binary_args *args) { - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; int i; assert(args); if (args->in_frame) err = set_input_frame_buffer(args->in_frame); - if (err == IA_CSS_SUCCESS && args->out_vf_frame) + if (!err && args->out_vf_frame) err = set_view_finder_buffer(args->out_vf_frame); for (i = 0; i < IA_CSS_BINARY_MAX_OUTPUT_PORTS; i++) { - if (err == IA_CSS_SUCCESS && args->out_frame[i]) + if (!err && args->out_frame[i]) err = set_output_frame_buffer(args->out_frame[i], i); } /* we don't pass this error back to the upper layer, so we add a assert here because we actually hit the error here but it still works by accident... */ - if (err != IA_CSS_SUCCESS) assert(false); + if (err) assert(false); return err; } @@ -790,23 +791,23 @@ sh_css_stage_write_binary_info(struct ia_css_binary_info *info) sh_css_isp_stage.binary_info = *info; } -static enum ia_css_err +static int copy_isp_mem_if_to_ddr(struct ia_css_binary *binary) { - enum ia_css_err err; + int err; err = ia_css_isp_param_copy_isp_mem_if_to_ddr( &binary->css_params, &binary->mem_params, IA_CSS_PARAM_CLASS_CONFIG); - if (err != IA_CSS_SUCCESS) + if (err) return err; err = ia_css_isp_param_copy_isp_mem_if_to_ddr( &binary->css_params, &binary->mem_params, IA_CSS_PARAM_CLASS_STATE); - if (err != IA_CSS_SUCCESS) + if (err) return err; - return IA_CSS_SUCCESS; + return 0; } static bool @@ -816,7 +817,7 @@ is_sp_stage(struct ia_css_pipeline_stage *stage) return stage->sp_func != IA_CSS_PIPELINE_NO_FUNC; } -static enum ia_css_err +static int configure_isp_from_args( const struct sh_css_sp_pipeline *pipeline, const struct ia_css_binary *binary, @@ -838,10 +839,21 @@ configure_isp_from_args( ia_css_dvs_configure(binary, &args->out_frame[0]->info); ia_css_output_configure(binary, &args->out_frame[0]->info); ia_css_raw_configure(pipeline, binary, &args->in_frame->info, &binary->in_frame_info, two_ppc, deinterleaved); - ia_css_ref_configure(binary, (const struct ia_css_frame **)args->delay_frames, pipeline->dvs_frame_delay); - ia_css_tnr_configure(binary, (const struct ia_css_frame **)args->tnr_frames); + + /* + * FIXME: args->delay_frames can be NULL here + * + * Somehow, the driver at the Intel Atom Yocto tree doesn't seem to + * suffer from the same issue. + * + * Anyway, the function below should now handle a NULL delay_frames + * without crashing, but the pipeline should likely be built without + * adding it at the first place (or there are a hidden bug somewhere) + */ + ia_css_ref_configure(binary, args->delay_frames, pipeline->dvs_frame_delay); + ia_css_tnr_configure(binary, args->tnr_frames); ia_css_bayer_io_config(binary, args); - return IA_CSS_SUCCESS; + return 0; } static void @@ -880,7 +892,7 @@ initialize_stage_frames(struct ia_css_frames_sp *frames) #endif } -static enum ia_css_err +static int sh_css_sp_init_stage(struct ia_css_binary *binary, const char *binary_name, const struct ia_css_blob_info *blob_info, @@ -893,7 +905,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, bool two_ppc) { const struct ia_css_binary_xinfo *xinfo; const struct ia_css_binary_info *info; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; int i; struct ia_css_pipe *pipe = NULL; unsigned int thread_id; @@ -925,7 +937,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, if (!info) { sh_css_sp_group.pipe[thread_id].sp_stage_addr[stage] = mmgr_NULL; - return IA_CSS_SUCCESS; + return 0; } #if defined(USE_INPUT_SYSTEM_VERSION_2401) @@ -978,8 +990,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, /* Make sure binary name is smaller than allowed string size */ assert(strlen(binary_name) < SH_CSS_MAX_BINARY_NAME - 1); - strncpy(sh_css_isp_stage.binary_name, binary_name, SH_CSS_MAX_BINARY_NAME - 1); - sh_css_isp_stage.binary_name[SH_CSS_MAX_BINARY_NAME - 1] = 0; + strscpy(sh_css_isp_stage.binary_name, binary_name, SH_CSS_MAX_BINARY_NAME); sh_css_isp_stage.mem_initializers = *isp_mem_if; /* @@ -1011,38 +1022,19 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, ia_css_query_internal_queue_id(IA_CSS_BUFFER_TYPE_METADATA, thread_id, &queue_id); sh_css_copy_buffer_attr_to_spbuffer(&sh_css_sp_stage.frames.metadata_buf, queue_id, mmgr_EXCEPTION, IA_CSS_BUFFER_TYPE_METADATA); #endif - if (err != IA_CSS_SUCCESS) + if (err) return err; #ifdef USE_INPUT_SYSTEM_VERSION_2401 -#ifndef ISP2401 - if (args->in_frame) - { - pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num); - if (!pipe) - return IA_CSS_ERR_INTERNAL_ERROR; - ia_css_get_crop_offsets(pipe, &args->in_frame->info); - } else if (&binary->in_frame_info) - { + if (stage == 0) { pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num); if (!pipe) - return IA_CSS_ERR_INTERNAL_ERROR; - ia_css_get_crop_offsets(pipe, &binary->in_frame_info); -#else - if (stage == 0) - { - if (args->in_frame) { - pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num); - if (!pipe) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; + + if (args->in_frame) ia_css_get_crop_offsets(pipe, &args->in_frame->info); - } else if (&binary->in_frame_info) { - pipe = find_pipe_by_num(sh_css_sp_group.pipe[thread_id].pipe_num); - if (!pipe) - return IA_CSS_ERR_INTERNAL_ERROR; + else ia_css_get_crop_offsets(pipe, &binary->in_frame_info); - } -#endif } #else (void)pipe; /*avoid build warning*/ @@ -1050,7 +1042,7 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, err = configure_isp_from_args(&sh_css_sp_group.pipe[thread_id], binary, args, two_ppc, sh_css_sp_stage.deinterleaved); - if (err != IA_CSS_SUCCESS) + if (err) return err; initialize_isp_states(binary); @@ -1072,13 +1064,13 @@ sh_css_sp_init_stage(struct ia_css_binary *binary, <<= binary->vf_downscale_log2; } err = copy_isp_mem_if_to_ddr(binary); - if (err != IA_CSS_SUCCESS) + if (err) return err; - return IA_CSS_SUCCESS; + return 0; } -static enum ia_css_err +static int sp_init_stage(struct ia_css_pipeline_stage *stage, unsigned int pipe_num, bool xnr, @@ -1110,7 +1102,7 @@ sp_init_stage(struct ia_css_pipeline_stage *stage, */ struct ia_css_isp_param_css_segments *mem_if = &isp_mem_if; - enum ia_css_err err = IA_CSS_SUCCESS; + int err = 0; assert(stage); @@ -1155,7 +1147,7 @@ sp_init_stage(struct ia_css_pipeline_stage *stage, These will be passed to sh_css_sp_init_stage and dereferenced there, so passing a NULL pointer is no good. return an error */ - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; } err = sh_css_sp_init_stage(binary, @@ -1190,12 +1182,14 @@ sp_init_sp_stage(struct ia_css_pipeline_stage *stage, break; case IA_CSS_PIPELINE_BIN_COPY: assert(false); /* TBI */ + break; case IA_CSS_PIPELINE_ISYS_COPY: sh_css_sp_start_isys_copy(args->out_frame[0], pipe_num, stage->max_input_width, if_config_index); break; case IA_CSS_PIPELINE_NO_FUNC: assert(false); + break; } } @@ -1347,7 +1341,7 @@ sh_css_sp_init_pipeline(struct ia_css_pipeline *me, } #endif - if (atomisp_hw_is_isp2401) { + if (IS_ISP2401) { /* For the shading correction type 1 (the legacy shading table conversion in css is not used), * the parameters are passed to the isp for the shading table centering. */ @@ -1608,7 +1602,7 @@ sh_css_event_init_irq_mask(void) } } -enum ia_css_err +int ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe, unsigned int or_mask, unsigned int and_mask) { @@ -1636,7 +1630,7 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe, pipe_num = ia_css_pipe_get_pipe_num(pipe); if (pipe_num >= IA_CSS_PIPE_ID_NUM) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_event_irq_mask[pipe_num]); assert(offset % HRT_BUS_BYTES == 0); @@ -1644,10 +1638,10 @@ ia_css_pipe_set_irq_mask(struct ia_css_pipe *pipe, (unsigned int)sp_address_of(host_sp_com) + offset, &event_irq_mask, sizeof(event_irq_mask)); - return IA_CSS_SUCCESS; + return 0; } -enum ia_css_err +int ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, unsigned int *or_mask, unsigned int *and_mask) { @@ -1665,7 +1659,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, pipe_num = ia_css_pipe_get_pipe_num(pipe); if (pipe_num >= IA_CSS_PIPE_ID_NUM) - return IA_CSS_ERR_INTERNAL_ERROR; + return -EINVAL; offset = (unsigned int)offsetof(struct host_sp_communication, host2sp_event_irq_mask[pipe_num]); assert(offset % HRT_BUS_BYTES == 0); @@ -1679,7 +1673,7 @@ ia_css_event_get_irq_mask(const struct ia_css_pipe *pipe, if (and_mask) *and_mask = event_irq_mask.and_mask; - return IA_CSS_SUCCESS; + return 0; } void diff --git a/drivers/staging/media/atomisp/pci/sh_css_sp.h b/drivers/staging/media/atomisp/pci/sh_css_sp.h index 7d4e13f1e038..153b005becda 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_sp.h +++ b/drivers/staging/media/atomisp/pci/sh_css_sp.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream.c b/drivers/staging/media/atomisp/pci/sh_css_stream.c index 60bddbb3d4c6..a768ce90f51c 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_stream.c +++ b/drivers/staging/media/atomisp/pci/sh_css_stream.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream_format.c b/drivers/staging/media/atomisp/pci/sh_css_stream_format.c index 548d4a3567b2..a798f0537050 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_stream_format.c +++ b/drivers/staging/media/atomisp/pci/sh_css_stream_format.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_stream_format.h b/drivers/staging/media/atomisp/pci/sh_css_stream_format.h index 32ebd6e0f344..84b7942147ad 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_stream_format.h +++ b/drivers/staging/media/atomisp/pci/sh_css_stream_format.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_struct.h b/drivers/staging/media/atomisp/pci/sh_css_struct.h index 81b9598ef8b7..bd260252317a 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_struct.h +++ b/drivers/staging/media/atomisp/pci/sh_css_struct.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -24,7 +25,7 @@ */ #include <type_support.h> -#include <system_types.h> +#include <system_local.h> #include "ia_css_pipeline.h" #include "ia_css_pipe_public.h" #include "ia_css_frame_public.h" @@ -64,8 +65,9 @@ struct sh_css { mipi_sizes_for_check[N_CSI_PORTS][IA_CSS_MIPI_SIZE_CHECK_MAX_NOF_ENTRIES_PER_PORT]; unsigned int mipi_frame_size[N_CSI_PORTS]; #endif - hrt_vaddress sp_bin_addr; + ia_css_ptr sp_bin_addr; hrt_data page_table_base_index; + unsigned int size_mem_words; /* \deprecated{Use ia_css_mipi_buffer_config instead.}*/ enum ia_css_irq_type irq_type; diff --git a/drivers/staging/media/atomisp/pci/sh_css_uds.h b/drivers/staging/media/atomisp/pci/sh_css_uds.h index d9bcae6007bf..8d9c5c92b692 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_uds.h +++ b/drivers/staging/media/atomisp/pci/sh_css_uds.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/sh_css_version.c b/drivers/staging/media/atomisp/pci/sh_css_version.c index eb986e15c7fa..fa6de61e4995 100644 --- a/drivers/staging/media/atomisp/pci/sh_css_version.c +++ b/drivers/staging/media/atomisp/pci/sh_css_version.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0 /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. @@ -13,25 +14,26 @@ */ #include "../../include/linux/atomisp.h" +#include "../../include/linux/atomisp_platform.h" #include "ia_css_version.h" #include "ia_css_version_data.h" #include "ia_css_err.h" #include "sh_css_firmware.h" -enum ia_css_err +int ia_css_get_version(char *version, int max_size) { char *css_version; - if (!atomisp_hw_is_isp2401) + if (!IS_ISP2401) css_version = ISP2400_CSS_VERSION_STRING; else css_version = ISP2401_CSS_VERSION_STRING; if (max_size <= (int)strlen(css_version) + (int)strlen(sh_css_get_fw_version()) + 5) - return IA_CSS_ERR_INVALID_ARGUMENTS; - strcpy(version, css_version); + return -EINVAL; + strscpy(version, css_version, max_size); strcat(version, "FW:"); strcat(version, sh_css_get_fw_version()); strcat(version, "; "); - return IA_CSS_SUCCESS; + return 0; } diff --git a/drivers/staging/media/atomisp/pci/str2mem_defs.h b/drivers/staging/media/atomisp/pci/str2mem_defs.h index 1cb62444cf68..e8cb456ac9c7 100644 --- a/drivers/staging/media/atomisp/pci/str2mem_defs.h +++ b/drivers/staging/media/atomisp/pci/str2mem_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h b/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h index 60143b8743a2..e0e3a6a66245 100644 --- a/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h +++ b/drivers/staging/media/atomisp/pci/streaming_to_mipi_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/system_global.h b/drivers/staging/media/atomisp/pci/system_global.h index 7f833c15f3ce..16d0a2e9a4dc 100644 --- a/drivers/staging/media/atomisp/pci/system_global.h +++ b/drivers/staging/media/atomisp/pci/system_global.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/system_local.h b/drivers/staging/media/atomisp/pci/system_local.h index fbb5daadac9f..dfcc4c2b8f16 100644 --- a/drivers/staging/media/atomisp/pci/system_local.h +++ b/drivers/staging/media/atomisp/pci/system_local.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ // SPDX-License-Identifier: GPL-2.0-or-later /* * (c) 2020 Mauro Carvalho Chehab <mchehab+huawei@kernel.org> diff --git a/drivers/staging/media/atomisp/pci/timed_controller_defs.h b/drivers/staging/media/atomisp/pci/timed_controller_defs.h index 75451e090f4f..9ddad87702de 100644 --- a/drivers/staging/media/atomisp/pci/timed_controller_defs.h +++ b/drivers/staging/media/atomisp/pci/timed_controller_defs.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/atomisp/pci/version.h b/drivers/staging/media/atomisp/pci/version.h index bbc4948baea9..a74a7bbfdb0e 100644 --- a/drivers/staging/media/atomisp/pci/version.h +++ b/drivers/staging/media/atomisp/pci/version.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 */ /* * Support for Intel Camera Imaging ISP subsystem. * Copyright (c) 2015, Intel Corporation. diff --git a/drivers/staging/media/rkvdec/rkvdec-h264.c b/drivers/staging/media/rkvdec/rkvdec-h264.c index cd4980d06be7..7b66e2743a4f 100644 --- a/drivers/staging/media/rkvdec/rkvdec-h264.c +++ b/drivers/staging/media/rkvdec/rkvdec-h264.c @@ -18,11 +18,16 @@ /* Size with u32 units. */ #define RKV_CABAC_INIT_BUFFER_SIZE (3680 + 128) #define RKV_RPS_SIZE ((128 + 128) / 4) -#define RKV_SCALING_LIST_SIZE (6 * 16 + 6 * 64 + 128) #define RKV_ERROR_INFO_SIZE (256 * 144 * 4) #define RKVDEC_NUM_REFLIST 3 +struct rkvdec_h264_scaling_list { + u8 scaling_list_4x4[6][16]; + u8 scaling_list_8x8[6][64]; + u8 padding[128]; +}; + struct rkvdec_sps_pps_packet { u32 info[8]; }; @@ -86,7 +91,7 @@ struct rkvdec_ps_field { /* Data structure describing auxiliary buffer format. */ struct rkvdec_h264_priv_tbl { s8 cabac_table[4][464][2]; - u8 scaling_list[RKV_SCALING_LIST_SIZE]; + struct rkvdec_h264_scaling_list scaling_list; u32 rps[RKV_RPS_SIZE]; struct rkvdec_sps_pps_packet param_set[256]; u8 err_info[RKV_ERROR_INFO_SIZE]; @@ -785,56 +790,25 @@ static void assemble_hw_rps(struct rkvdec_ctx *ctx, } } -/* - * NOTE: The values in a scaling list are in zig-zag order, apply inverse - * scanning process to get the values in matrix order. - */ -static const u32 zig_zag_4x4[16] = { - 0, 1, 4, 8, 5, 2, 3, 6, 9, 12, 13, 10, 7, 11, 14, 15 -}; - -static const u32 zig_zag_8x8[64] = { - 0, 1, 8, 16, 9, 2, 3, 10, 17, 24, 32, 25, 18, 11, 4, 5, - 12, 19, 26, 33, 40, 48, 41, 34, 27, 20, 13, 6, 7, 14, 21, 28, - 35, 42, 49, 56, 57, 50, 43, 36, 29, 22, 15, 23, 30, 37, 44, 51, - 58, 59, 52, 45, 38, 31, 39, 46, 53, 60, 61, 54, 47, 55, 62, 63 -}; - -static void reorder_scaling_list(struct rkvdec_ctx *ctx, - struct rkvdec_h264_run *run) +static void assemble_hw_scaling_list(struct rkvdec_ctx *ctx, + struct rkvdec_h264_run *run) { const struct v4l2_ctrl_h264_scaling_matrix *scaling = run->scaling_matrix; - const size_t num_list_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4); - const size_t list_len_4x4 = ARRAY_SIZE(scaling->scaling_list_4x4[0]); - const size_t num_list_8x8 = ARRAY_SIZE(scaling->scaling_list_8x8); - const size_t list_len_8x8 = ARRAY_SIZE(scaling->scaling_list_8x8[0]); struct rkvdec_h264_ctx *h264_ctx = ctx->priv; struct rkvdec_h264_priv_tbl *tbl = h264_ctx->priv_tbl.cpu; - u8 *dst = tbl->scaling_list; - const u8 *src; - int i, j; - - BUILD_BUG_ON(ARRAY_SIZE(zig_zag_4x4) != list_len_4x4); - BUILD_BUG_ON(ARRAY_SIZE(zig_zag_8x8) != list_len_8x8); - BUILD_BUG_ON(ARRAY_SIZE(tbl->scaling_list) < - num_list_4x4 * list_len_4x4 + - num_list_8x8 * list_len_8x8); - - src = &scaling->scaling_list_4x4[0][0]; - for (i = 0; i < num_list_4x4; ++i) { - for (j = 0; j < list_len_4x4; ++j) - dst[zig_zag_4x4[j]] = src[j]; - src += list_len_4x4; - dst += list_len_4x4; - } - src = &scaling->scaling_list_8x8[0][0]; - for (i = 0; i < num_list_8x8; ++i) { - for (j = 0; j < list_len_8x8; ++j) - dst[zig_zag_8x8[j]] = src[j]; - src += list_len_8x8; - dst += list_len_8x8; - } + BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_4x4) != + sizeof(scaling->scaling_list_4x4)); + BUILD_BUG_ON(sizeof(tbl->scaling_list.scaling_list_8x8) != + sizeof(scaling->scaling_list_8x8)); + + memcpy(tbl->scaling_list.scaling_list_4x4, + scaling->scaling_list_4x4, + sizeof(scaling->scaling_list_4x4)); + + memcpy(tbl->scaling_list.scaling_list_8x8, + scaling->scaling_list_8x8, + sizeof(scaling->scaling_list_8x8)); } /* @@ -1126,7 +1100,7 @@ static int rkvdec_h264_run(struct rkvdec_ctx *ctx) v4l2_h264_build_b_ref_lists(&reflist_builder, h264_ctx->reflists.b0, h264_ctx->reflists.b1); - reorder_scaling_list(ctx, &run); + assemble_hw_scaling_list(ctx, &run); assemble_hw_pps(ctx, &run); assemble_hw_rps(ctx, &run); config_registers(ctx, &run); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus.c b/drivers/staging/media/sunxi/cedrus/cedrus.c index 05a85517ff60..bc27f9430eeb 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus.c @@ -16,6 +16,7 @@ #include <linux/platform_device.h> #include <linux/module.h> #include <linux/of.h> +#include <linux/pm.h> #include <media/v4l2-device.h> #include <media/v4l2-ioctl.h> @@ -551,12 +552,18 @@ static const struct of_device_id cedrus_dt_match[] = { }; MODULE_DEVICE_TABLE(of, cedrus_dt_match); +static const struct dev_pm_ops cedrus_dev_pm_ops = { + SET_RUNTIME_PM_OPS(cedrus_hw_suspend, + cedrus_hw_resume, NULL) +}; + static struct platform_driver cedrus_driver = { .probe = cedrus_probe, .remove = cedrus_remove, .driver = { .name = CEDRUS_NAME, .of_match_table = of_match_ptr(cedrus_dt_match), + .pm = &cedrus_dev_pm_ops, }, }; module_platform_driver(cedrus_driver); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c index 4a2fc33a1d79..58c48e4fdfe9 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_dec.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_dec.c @@ -74,6 +74,8 @@ void cedrus_device_run(void *priv) v4l2_m2m_buf_copy_metadata(run.src, run.dst, true); + cedrus_dst_format_set(dev, &ctx->dst_fmt); + dev->dec_ops[ctx->current_codec]->setup(ctx, &run); /* Complete request(s) controls if needed. */ diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c index daf5f244f93b..1744e6fcc999 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.c @@ -19,6 +19,7 @@ #include <linux/dma-mapping.h> #include <linux/interrupt.h> #include <linux/clk.h> +#include <linux/pm_runtime.h> #include <linux/regmap.h> #include <linux/reset.h> #include <linux/soc/sunxi/sunxi_sram.h> @@ -140,6 +141,64 @@ static irqreturn_t cedrus_irq(int irq, void *data) return IRQ_HANDLED; } +int cedrus_hw_suspend(struct device *device) +{ + struct cedrus_dev *dev = dev_get_drvdata(device); + + reset_control_assert(dev->rstc); + + clk_disable_unprepare(dev->ram_clk); + clk_disable_unprepare(dev->mod_clk); + clk_disable_unprepare(dev->ahb_clk); + + return 0; +} + +int cedrus_hw_resume(struct device *device) +{ + struct cedrus_dev *dev = dev_get_drvdata(device); + int ret; + + ret = clk_prepare_enable(dev->ahb_clk); + if (ret) { + dev_err(dev->dev, "Failed to enable AHB clock\n"); + + return ret; + } + + ret = clk_prepare_enable(dev->mod_clk); + if (ret) { + dev_err(dev->dev, "Failed to enable MOD clock\n"); + + goto err_ahb_clk; + } + + ret = clk_prepare_enable(dev->ram_clk); + if (ret) { + dev_err(dev->dev, "Failed to enable RAM clock\n"); + + goto err_mod_clk; + } + + ret = reset_control_reset(dev->rstc); + if (ret) { + dev_err(dev->dev, "Failed to apply reset\n"); + + goto err_ram_clk; + } + + return 0; + +err_ram_clk: + clk_disable_unprepare(dev->ram_clk); +err_mod_clk: + clk_disable_unprepare(dev->mod_clk); +err_ahb_clk: + clk_disable_unprepare(dev->ahb_clk); + + return ret; +} + int cedrus_hw_probe(struct cedrus_dev *dev) { const struct cedrus_variant *variant; @@ -236,42 +295,17 @@ int cedrus_hw_probe(struct cedrus_dev *dev) goto err_sram; } - ret = clk_prepare_enable(dev->ahb_clk); - if (ret) { - dev_err(dev->dev, "Failed to enable AHB clock\n"); - - goto err_sram; - } - - ret = clk_prepare_enable(dev->mod_clk); - if (ret) { - dev_err(dev->dev, "Failed to enable MOD clock\n"); - - goto err_ahb_clk; - } - - ret = clk_prepare_enable(dev->ram_clk); - if (ret) { - dev_err(dev->dev, "Failed to enable RAM clock\n"); - - goto err_mod_clk; - } - - ret = reset_control_reset(dev->rstc); - if (ret) { - dev_err(dev->dev, "Failed to apply reset\n"); - - goto err_ram_clk; + pm_runtime_enable(dev->dev); + if (!pm_runtime_enabled(dev->dev)) { + ret = cedrus_hw_resume(dev->dev); + if (ret) + goto err_pm; } return 0; -err_ram_clk: - clk_disable_unprepare(dev->ram_clk); -err_mod_clk: - clk_disable_unprepare(dev->mod_clk); -err_ahb_clk: - clk_disable_unprepare(dev->ahb_clk); +err_pm: + pm_runtime_disable(dev->dev); err_sram: sunxi_sram_release(dev->dev); err_mem: @@ -282,11 +316,9 @@ err_mem: void cedrus_hw_remove(struct cedrus_dev *dev) { - reset_control_assert(dev->rstc); - - clk_disable_unprepare(dev->ram_clk); - clk_disable_unprepare(dev->mod_clk); - clk_disable_unprepare(dev->ahb_clk); + pm_runtime_disable(dev->dev); + if (!pm_runtime_status_suspended(dev->dev)) + cedrus_hw_suspend(dev->dev); sunxi_sram_release(dev->dev); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_hw.h b/drivers/staging/media/sunxi/cedrus/cedrus_hw.h index 604ff932fbf5..45f641f0bfa2 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_hw.h +++ b/drivers/staging/media/sunxi/cedrus/cedrus_hw.h @@ -22,6 +22,9 @@ void cedrus_engine_disable(struct cedrus_dev *dev); void cedrus_dst_format_set(struct cedrus_dev *dev, struct v4l2_pix_format *fmt); +int cedrus_hw_suspend(struct device *device); +int cedrus_hw_resume(struct device *device); + int cedrus_hw_probe(struct cedrus_dev *dev); void cedrus_hw_remove(struct cedrus_dev *dev); diff --git a/drivers/staging/media/sunxi/cedrus/cedrus_video.c b/drivers/staging/media/sunxi/cedrus/cedrus_video.c index 15cf1f10221b..16d82309e7b6 100644 --- a/drivers/staging/media/sunxi/cedrus/cedrus_video.c +++ b/drivers/staging/media/sunxi/cedrus/cedrus_video.c @@ -13,6 +13,8 @@ * Marek Szyprowski, <m.szyprowski@samsung.com> */ +#include <linux/pm_runtime.h> + #include <media/videobuf2-dma-contig.h> #include <media/v4l2-device.h> #include <media/v4l2-ioctl.h> @@ -273,7 +275,6 @@ static int cedrus_s_fmt_vid_cap(struct file *file, void *priv, struct v4l2_format *f) { struct cedrus_ctx *ctx = cedrus_file2ctx(file); - struct cedrus_dev *dev = ctx->dev; struct vb2_queue *vq; int ret; @@ -287,8 +288,6 @@ static int cedrus_s_fmt_vid_cap(struct file *file, void *priv, ctx->dst_fmt = f->fmt.pix; - cedrus_dst_format_set(dev, &ctx->dst_fmt); - return 0; } @@ -453,12 +452,24 @@ static int cedrus_start_streaming(struct vb2_queue *vq, unsigned int count) return -EINVAL; } - if (V4L2_TYPE_IS_OUTPUT(vq->type) && - dev->dec_ops[ctx->current_codec]->start) - ret = dev->dec_ops[ctx->current_codec]->start(ctx); + if (V4L2_TYPE_IS_OUTPUT(vq->type)) { + ret = pm_runtime_get_sync(dev->dev); + if (ret < 0) + goto err_cleanup; - if (ret) - cedrus_queue_cleanup(vq, VB2_BUF_STATE_QUEUED); + if (dev->dec_ops[ctx->current_codec]->start) { + ret = dev->dec_ops[ctx->current_codec]->start(ctx); + if (ret) + goto err_pm; + } + } + + return 0; + +err_pm: + pm_runtime_put(dev->dev); +err_cleanup: + cedrus_queue_cleanup(vq, VB2_BUF_STATE_QUEUED); return ret; } @@ -468,9 +479,12 @@ static void cedrus_stop_streaming(struct vb2_queue *vq) struct cedrus_ctx *ctx = vb2_get_drv_priv(vq); struct cedrus_dev *dev = ctx->dev; - if (V4L2_TYPE_IS_OUTPUT(vq->type) && - dev->dec_ops[ctx->current_codec]->stop) - dev->dec_ops[ctx->current_codec]->stop(ctx); + if (V4L2_TYPE_IS_OUTPUT(vq->type)) { + if (dev->dec_ops[ctx->current_codec]->stop) + dev->dec_ops[ctx->current_codec]->stop(ctx); + + pm_runtime_put(dev->dev); + } cedrus_queue_cleanup(vq, VB2_BUF_STATE_ERROR); } diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 59379d662626..c9689610e186 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -1158,7 +1158,7 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, conn->sess->se_sess, be32_to_cpu(hdr->data_length), cmd->data_direction, sam_task_attr, - cmd->sense_buffer + 2); + cmd->sense_buffer + 2, scsilun_to_int(&hdr->lun)); pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, @@ -1167,23 +1167,26 @@ int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, target_get_sess_cmd(&cmd->se_cmd, true); - cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd, - scsilun_to_int(&hdr->lun)); - if (cmd->sense_reason) - goto attach_cmd; - - /* only used for printks or comparing with ->ref_task_tag */ - cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; - cmd->sense_reason = target_setup_cmd_from_cdb(&cmd->se_cmd, hdr->cdb); + cmd->sense_reason = target_cmd_init_cdb(&cmd->se_cmd, hdr->cdb); if (cmd->sense_reason) { if (cmd->sense_reason == TCM_OUT_OF_RESOURCES) { return iscsit_add_reject_cmd(cmd, - ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); + ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); } goto attach_cmd; } + cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd); + if (cmd->sense_reason) + goto attach_cmd; + + /* only used for printks or comparing with ->ref_task_tag */ + cmd->se_cmd.tag = (__force u32)cmd->init_task_tag; + cmd->sense_reason = target_cmd_parse_cdb(&cmd->se_cmd); + if (cmd->sense_reason) + goto attach_cmd; + if (iscsit_build_pdu_and_seq_lists(cmd, payload_length) < 0) { return iscsit_add_reject_cmd(cmd, ISCSI_REASON_BOOKMARK_NO_RESOURCES, buf); @@ -2000,7 +2003,8 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, transport_init_se_cmd(&cmd->se_cmd, &iscsi_ops, conn->sess->se_sess, 0, DMA_NONE, - TCM_SIMPLE_TAG, cmd->sense_buffer + 2); + TCM_SIMPLE_TAG, cmd->sense_buffer + 2, + scsilun_to_int(&hdr->lun)); target_get_sess_cmd(&cmd->se_cmd, true); @@ -2038,8 +2042,7 @@ iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, * Locate the struct se_lun for all TMRs not related to ERL=2 TASK_REASSIGN */ if (function != ISCSI_TM_FUNC_TASK_REASSIGN) { - ret = transport_lookup_tmr_lun(&cmd->se_cmd, - scsilun_to_int(&hdr->lun)); + ret = transport_lookup_tmr_lun(&cmd->se_cmd); if (ret < 0) { se_tmr->response = ISCSI_TMF_RSP_NO_LUN; goto attach; diff --git a/drivers/target/target_core_device.c b/drivers/target/target_core_device.c index 46b0e1ceb77f..405d82d44717 100644 --- a/drivers/target/target_core_device.c +++ b/drivers/target/target_core_device.c @@ -45,7 +45,7 @@ static struct se_hba *lun0_hba; struct se_device *g_lun0_dev; sense_reason_t -transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun) +transport_lookup_cmd_lun(struct se_cmd *se_cmd) { struct se_lun *se_lun = NULL; struct se_session *se_sess = se_cmd->se_sess; @@ -54,7 +54,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun) sense_reason_t ret = TCM_NO_SENSE; rcu_read_lock(); - deve = target_nacl_find_deve(nacl, unpacked_lun); + deve = target_nacl_find_deve(nacl, se_cmd->orig_fe_lun); if (deve) { atomic_long_inc(&deve->total_cmds); @@ -74,7 +74,6 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun) se_cmd->se_lun = se_lun; se_cmd->pr_res_key = deve->pr_res_key; - se_cmd->orig_fe_lun = unpacked_lun; se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD; se_cmd->lun_ref_active = true; @@ -83,7 +82,7 @@ transport_lookup_cmd_lun(struct se_cmd *se_cmd, u64 unpacked_lun) pr_err("TARGET_CORE[%s]: Detected WRITE_PROTECTED LUN" " Access for 0x%08llx\n", se_cmd->se_tfo->fabric_name, - unpacked_lun); + se_cmd->orig_fe_lun); rcu_read_unlock(); ret = TCM_WRITE_PROTECTED; goto ref_dev; @@ -98,18 +97,17 @@ out_unlock: * REPORT_LUNS, et al to be returned when no active * MappedLUN=0 exists for this Initiator Port. */ - if (unpacked_lun != 0) { + if (se_cmd->orig_fe_lun != 0) { pr_err("TARGET_CORE[%s]: Detected NON_EXISTENT_LUN" " Access for 0x%08llx from %s\n", se_cmd->se_tfo->fabric_name, - unpacked_lun, + se_cmd->orig_fe_lun, nacl->initiatorname); return TCM_NON_EXISTENT_LUN; } se_lun = se_sess->se_tpg->tpg_virt_lun0; se_cmd->se_lun = se_sess->se_tpg->tpg_virt_lun0; - se_cmd->orig_fe_lun = 0; se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD; percpu_ref_get(&se_lun->lun_ref); @@ -145,7 +143,7 @@ ref_dev: } EXPORT_SYMBOL(transport_lookup_cmd_lun); -int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun) +int transport_lookup_tmr_lun(struct se_cmd *se_cmd) { struct se_dev_entry *deve; struct se_lun *se_lun = NULL; @@ -155,7 +153,7 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun) unsigned long flags; rcu_read_lock(); - deve = target_nacl_find_deve(nacl, unpacked_lun); + deve = target_nacl_find_deve(nacl, se_cmd->orig_fe_lun); if (deve) { se_lun = rcu_dereference(deve->se_lun); @@ -166,7 +164,6 @@ int transport_lookup_tmr_lun(struct se_cmd *se_cmd, u64 unpacked_lun) se_cmd->se_lun = se_lun; se_cmd->pr_res_key = deve->pr_res_key; - se_cmd->orig_fe_lun = unpacked_lun; se_cmd->se_cmd_flags |= SCF_SE_LUN_CMD; se_cmd->lun_ref_active = true; } @@ -177,7 +174,7 @@ out_unlock: pr_debug("TARGET_CORE[%s]: Detected NON_EXISTENT_LUN" " Access for 0x%08llx for %s\n", se_cmd->se_tfo->fabric_name, - unpacked_lun, + se_cmd->orig_fe_lun, nacl->initiatorname); return -ENODEV; } diff --git a/drivers/target/target_core_tmr.c b/drivers/target/target_core_tmr.c index afbd492c76a9..89c84d472cd7 100644 --- a/drivers/target/target_core_tmr.c +++ b/drivers/target/target_core_tmr.c @@ -148,8 +148,8 @@ void core_tmr_abort_task( * code. */ if (!tmr->tmr_dev) - WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd, - se_cmd->orig_fe_lun) < 0); + WARN_ON_ONCE(transport_lookup_tmr_lun(tmr->task_cmd) < + 0); target_put_cmd_and_wait(se_cmd); diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index da37af9c3a5e..90ecdd706a01 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -1364,7 +1364,7 @@ void transport_init_se_cmd( u32 data_length, int data_direction, int task_attr, - unsigned char *sense_buffer) + unsigned char *sense_buffer, u64 unpacked_lun) { INIT_LIST_HEAD(&cmd->se_delayed_node); INIT_LIST_HEAD(&cmd->se_qf_node); @@ -1383,6 +1383,7 @@ void transport_init_se_cmd( cmd->data_direction = data_direction; cmd->sam_task_attr = task_attr; cmd->sense_buffer = sense_buffer; + cmd->orig_fe_lun = unpacked_lun; cmd->state_active = false; } @@ -1410,11 +1411,11 @@ transport_check_alloc_task_attr(struct se_cmd *cmd) } sense_reason_t -target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) +target_cmd_init_cdb(struct se_cmd *cmd, unsigned char *cdb) { - struct se_device *dev = cmd->se_dev; sense_reason_t ret; + cmd->t_task_cdb = &cmd->__t_task_cdb[0]; /* * Ensure that the received CDB is less than the max (252 + 8) bytes * for VARIABLE_LENGTH_CMD @@ -1423,7 +1424,8 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) pr_err("Received SCSI CDB with command_size: %d that" " exceeds SCSI_MAX_VARLEN_CDB_SIZE: %d\n", scsi_command_size(cdb), SCSI_MAX_VARLEN_CDB_SIZE); - return TCM_INVALID_CDB_FIELD; + ret = TCM_INVALID_CDB_FIELD; + goto err; } /* * If the received CDB is larger than TCM_MAX_COMMAND_SIZE, @@ -1438,16 +1440,34 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) " %u > sizeof(cmd->__t_task_cdb): %lu ops\n", scsi_command_size(cdb), (unsigned long)sizeof(cmd->__t_task_cdb)); - return TCM_OUT_OF_RESOURCES; + ret = TCM_OUT_OF_RESOURCES; + goto err; } - } else - cmd->t_task_cdb = &cmd->__t_task_cdb[0]; + } /* * Copy the original CDB into cmd-> */ memcpy(cmd->t_task_cdb, cdb, scsi_command_size(cdb)); trace_target_sequencer_start(cmd); + return 0; + +err: + /* + * Copy the CDB here to allow trace_target_cmd_complete() to + * print the cdb to the trace buffers. + */ + memcpy(cmd->t_task_cdb, cdb, min(scsi_command_size(cdb), + (unsigned int)TCM_MAX_COMMAND_SIZE)); + return ret; +} +EXPORT_SYMBOL(target_cmd_init_cdb); + +sense_reason_t +target_cmd_parse_cdb(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + sense_reason_t ret; ret = dev->transport->parse_cdb(cmd); if (ret == TCM_UNSUPPORTED_SCSI_OPCODE) @@ -1466,7 +1486,7 @@ target_setup_cmd_from_cdb(struct se_cmd *cmd, unsigned char *cdb) atomic_long_inc(&cmd->se_lun->lun_stats.cmd_pdus); return 0; } -EXPORT_SYMBOL(target_setup_cmd_from_cdb); +EXPORT_SYMBOL(target_cmd_parse_cdb); /* * Used by fabric module frontends to queue tasks directly. @@ -1588,7 +1608,8 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess * target_core_fabric_ops->queue_status() callback */ transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - data_length, data_dir, task_attr, sense); + data_length, data_dir, task_attr, sense, + unpacked_lun); if (flags & TARGET_SCF_USE_CPUID) se_cmd->se_cmd_flags |= SCF_USE_CPUID; @@ -1611,17 +1632,25 @@ int target_submit_cmd_map_sgls(struct se_cmd *se_cmd, struct se_session *se_sess */ if (flags & TARGET_SCF_BIDI_OP) se_cmd->se_cmd_flags |= SCF_BIDI; + + rc = target_cmd_init_cdb(se_cmd, cdb); + if (rc) { + transport_send_check_condition_and_sense(se_cmd, rc, 0); + target_put_sess_cmd(se_cmd); + return 0; + } + /* * Locate se_lun pointer and attach it to struct se_cmd */ - rc = transport_lookup_cmd_lun(se_cmd, unpacked_lun); + rc = transport_lookup_cmd_lun(se_cmd); if (rc) { transport_send_check_condition_and_sense(se_cmd, rc, 0); target_put_sess_cmd(se_cmd); return 0; } - rc = target_setup_cmd_from_cdb(se_cmd, cdb); + rc = target_cmd_parse_cdb(se_cmd); if (rc != 0) { transport_generic_request_failure(se_cmd, rc); return 0; @@ -1782,7 +1811,7 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, BUG_ON(!se_tpg); transport_init_se_cmd(se_cmd, se_tpg->se_tpg_tfo, se_sess, - 0, DMA_NONE, TCM_SIMPLE_TAG, sense); + 0, DMA_NONE, TCM_SIMPLE_TAG, sense, unpacked_lun); /* * FIXME: Currently expect caller to handle se_cmd->se_tmr_req * allocation failure. @@ -1810,7 +1839,7 @@ int target_submit_tmr(struct se_cmd *se_cmd, struct se_session *se_sess, goto failure; } - ret = transport_lookup_tmr_lun(se_cmd, unpacked_lun); + ret = transport_lookup_tmr_lun(se_cmd); if (ret) goto failure; diff --git a/drivers/target/target_core_user.c b/drivers/target/target_core_user.c index 28fb9441de7a..560bfec933bc 100644 --- a/drivers/target/target_core_user.c +++ b/drivers/target/target_core_user.c @@ -1007,7 +1007,7 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err) entry->hdr.cmd_id = 0; /* not used for PAD */ entry->hdr.kflags = 0; entry->hdr.uflags = 0; - tcmu_flush_dcache_range(entry, sizeof(*entry)); + tcmu_flush_dcache_range(entry, sizeof(entry->hdr)); UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size); tcmu_flush_dcache_range(mb, sizeof(*mb)); @@ -1072,7 +1072,7 @@ static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err) cdb_off = CMDR_OFF + cmd_head + base_command_size; memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb)); entry->req.cdb_off = cdb_off; - tcmu_flush_dcache_range(entry, sizeof(*entry)); + tcmu_flush_dcache_range(entry, command_size); UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size); tcmu_flush_dcache_range(mb, sizeof(*mb)); diff --git a/drivers/target/target_core_xcopy.c b/drivers/target/target_core_xcopy.c index bd3ed6ce7571..0d00ccbeb050 100644 --- a/drivers/target/target_core_xcopy.c +++ b/drivers/target/target_core_xcopy.c @@ -526,8 +526,11 @@ static int target_xcopy_setup_pt_cmd( } cmd->se_cmd_flags |= SCF_SE_LUN_CMD; + if (target_cmd_init_cdb(cmd, cdb)) + return -EINVAL; + cmd->tag = 0; - if (target_setup_cmd_from_cdb(cmd, cdb)) + if (target_cmd_parse_cdb(cmd)) return -EINVAL; if (transport_generic_map_mem_to_cmd(cmd, xop->xop_data_sg, @@ -585,7 +588,7 @@ static int target_xcopy_read_source( (unsigned long long)src_lba, src_sectors, length); transport_init_se_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, length, - DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0]); + DMA_FROM_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0); rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, src_dev, &cdb[0], remote_port); @@ -630,7 +633,7 @@ static int target_xcopy_write_destination( (unsigned long long)dst_lba, dst_sectors, length); transport_init_se_cmd(se_cmd, &xcopy_pt_tfo, &xcopy_pt_sess, length, - DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0]); + DMA_TO_DEVICE, 0, &xpt_cmd.sense_buffer[0], 0); rc = target_xcopy_setup_pt_cmd(&xpt_cmd, xop, dst_dev, &cdb[0], remote_port); diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index 91af271e9bb0..3eb2348e5242 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -273,6 +273,16 @@ config IMX8MM_THERMAL cpufreq is used as the cooling device to throttle CPUs when the passive trip is crossed. +config K3_THERMAL + tristate "Texas Instruments K3 thermal support" + depends on ARCH_K3 || COMPILE_TEST + help + If you say yes here you get thermal support for the Texas Instruments + K3 SoC family. The current chip supported is: + - AM654 + + This includes temperature reading functionality. + config MAX77620_THERMAL tristate "Temperature sensor driver for Maxim MAX77620 PMIC" depends on MFD_MAX77620 @@ -285,8 +295,8 @@ config MAX77620_THERMAL config QORIQ_THERMAL tristate "QorIQ Thermal Monitoring Unit" - depends on THERMAL_OF - depends on HAS_IOMEM + depends on THERMAL_OF && HAS_IOMEM + depends on PPC_E500MC || SOC_LS1021A || ARCH_LAYERSCAPE || (ARCH_MXC && ARM64) || COMPILE_TEST select REGMAP_MMIO help Support for Thermal Monitoring Unit (TMU) found on QorIQ platforms. diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index 8c8ed7b79915..0c8b84a09b9a 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -9,14 +9,14 @@ thermal_sys-y += thermal_core.o thermal_sysfs.o \ # interface to/from other layers providing sensors thermal_sys-$(CONFIG_THERMAL_HWMON) += thermal_hwmon.o -thermal_sys-$(CONFIG_THERMAL_OF) += of-thermal.o +thermal_sys-$(CONFIG_THERMAL_OF) += thermal_of.o # governors -thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE) += fair_share.o +thermal_sys-$(CONFIG_THERMAL_GOV_FAIR_SHARE) += gov_fair_share.o thermal_sys-$(CONFIG_THERMAL_GOV_BANG_BANG) += gov_bang_bang.o -thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE) += step_wise.o -thermal_sys-$(CONFIG_THERMAL_GOV_USER_SPACE) += user_space.o -thermal_sys-$(CONFIG_THERMAL_GOV_POWER_ALLOCATOR) += power_allocator.o +thermal_sys-$(CONFIG_THERMAL_GOV_STEP_WISE) += gov_step_wise.o +thermal_sys-$(CONFIG_THERMAL_GOV_USER_SPACE) += gov_user_space.o +thermal_sys-$(CONFIG_THERMAL_GOV_POWER_ALLOCATOR) += gov_power_allocator.o # cpufreq cooling thermal_sys-$(CONFIG_CPU_FREQ_THERMAL) += cpufreq_cooling.o @@ -28,6 +28,7 @@ thermal_sys-$(CONFIG_CLOCK_THERMAL) += clock_cooling.o # devfreq cooling thermal_sys-$(CONFIG_DEVFREQ_THERMAL) += devfreq_cooling.o +obj-$(CONFIG_K3_THERMAL) += k3_bandgap.o # platform thermal drivers obj-y += broadcom/ obj-$(CONFIG_THERMAL_MMIO) += thermal_mmio.o diff --git a/drivers/thermal/clock_cooling.c b/drivers/thermal/clock_cooling.c index 7cb3ae4b44ee..56cb1f46a428 100644 --- a/drivers/thermal/clock_cooling.c +++ b/drivers/thermal/clock_cooling.c @@ -12,15 +12,16 @@ * Copyright (C) 2012 Amit Daniel <amit.kachhap@linaro.org> */ #include <linux/clk.h> +#include <linux/clock_cooling.h> #include <linux/cpufreq.h> #include <linux/device.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/idr.h> #include <linux/mutex.h> #include <linux/pm_opp.h> #include <linux/slab.h> #include <linux/thermal.h> -#include <linux/clock_cooling.h> /** * struct clock_cooling_device - data for cooling device with clock diff --git a/drivers/thermal/cpufreq_cooling.c b/drivers/thermal/cpufreq_cooling.c index e297e135c031..9e124020519f 100644 --- a/drivers/thermal/cpufreq_cooling.c +++ b/drivers/thermal/cpufreq_cooling.c @@ -10,17 +10,17 @@ * Viresh Kumar <viresh.kumar@linaro.org> * */ -#include <linux/module.h> -#include <linux/thermal.h> +#include <linux/cpu.h> #include <linux/cpufreq.h> +#include <linux/cpu_cooling.h> +#include <linux/energy_model.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/idr.h> #include <linux/pm_opp.h> #include <linux/pm_qos.h> #include <linux/slab.h> -#include <linux/cpu.h> -#include <linux/cpu_cooling.h> -#include <linux/energy_model.h> +#include <linux/thermal.h> #include <trace/events/thermal.h> diff --git a/drivers/thermal/cpuidle_cooling.c b/drivers/thermal/cpuidle_cooling.c index 0bb843246f59..78e3e8238116 100644 --- a/drivers/thermal/cpuidle_cooling.c +++ b/drivers/thermal/cpuidle_cooling.c @@ -5,11 +5,14 @@ * Author: Daniel Lezcano <daniel.lezcano@linaro.org> * */ +#define pr_fmt(fmt) "cpuidle cooling: " fmt + #include <linux/cpu_cooling.h> #include <linux/cpuidle.h> #include <linux/err.h> #include <linux/idle_inject.h> #include <linux/idr.h> +#include <linux/of_device.h> #include <linux/slab.h> #include <linux/thermal.h> @@ -154,22 +157,25 @@ static struct thermal_cooling_device_ops cpuidle_cooling_ops = { }; /** - * cpuidle_of_cooling_register - Idle cooling device initialization function + * __cpuidle_cooling_register: register the cooling device * @drv: a cpuidle driver structure pointer - * @np: a node pointer to a device tree cooling device node + * @np: a device node structure pointer used for the thermal binding * - * This function is in charge of creating a cooling device per cpuidle - * driver and register it to thermal framework. + * This function is in charge of allocating the cpuidle cooling device + * structure, the idle injection, initialize them and register the + * cooling device to the thermal framework. * - * Return: zero on success, or negative value corresponding to the - * error detected in the underlying subsystems. + * Return: zero on success, a negative value returned by one of the + * underlying subsystem in case of error */ -int cpuidle_of_cooling_register(struct device_node *np, - struct cpuidle_driver *drv) +static int __cpuidle_cooling_register(struct device_node *np, + struct cpuidle_driver *drv) { struct idle_inject_device *ii_dev; struct cpuidle_cooling_device *idle_cdev; struct thermal_cooling_device *cdev; + unsigned int idle_duration_us = TICK_USEC; + unsigned int latency_us = UINT_MAX; char dev_name[THERMAL_NAME_LENGTH]; int id, ret; @@ -191,7 +197,11 @@ int cpuidle_of_cooling_register(struct device_node *np, goto out_id; } - idle_inject_set_duration(ii_dev, TICK_USEC, TICK_USEC); + of_property_read_u32(np, "duration-us", &idle_duration_us); + of_property_read_u32(np, "exit-latency-us", &latency_us); + + idle_inject_set_duration(ii_dev, TICK_USEC, idle_duration_us); + idle_inject_set_latency(ii_dev, latency_us); idle_cdev->ii_dev = ii_dev; @@ -204,6 +214,9 @@ int cpuidle_of_cooling_register(struct device_node *np, goto out_unregister; } + pr_debug("%s: Idle injection set with idle duration=%u, latency=%u\n", + dev_name, idle_duration_us, latency_us); + return 0; out_unregister: @@ -221,12 +234,38 @@ out: * @drv: a cpuidle driver structure pointer * * This function is in charge of creating a cooling device per cpuidle - * driver and register it to thermal framework. + * driver and register it to the thermal framework. * * Return: zero on success, or negative value corresponding to the * error detected in the underlying subsystems. */ -int cpuidle_cooling_register(struct cpuidle_driver *drv) +void cpuidle_cooling_register(struct cpuidle_driver *drv) { - return cpuidle_of_cooling_register(NULL, drv); + struct device_node *cooling_node; + struct device_node *cpu_node; + int cpu, ret; + + for_each_cpu(cpu, drv->cpumask) { + + cpu_node = of_cpu_device_node_get(cpu); + + cooling_node = of_get_child_by_name(cpu_node, "thermal-idle"); + + of_node_put(cpu_node); + + if (!cooling_node) { + pr_debug("'thermal-idle' node not found for cpu%d\n", cpu); + continue; + } + + ret = __cpuidle_cooling_register(cooling_node, drv); + + of_node_put(cooling_node); + + if (ret) { + pr_err("Failed to register the cpuidle cooling device" \ + "for cpu%d: %d\n", cpu, ret); + break; + } + } } diff --git a/drivers/thermal/devfreq_cooling.c b/drivers/thermal/devfreq_cooling.c index a87d4fa031c8..f7f32e98331b 100644 --- a/drivers/thermal/devfreq_cooling.c +++ b/drivers/thermal/devfreq_cooling.c @@ -24,11 +24,13 @@ #include <linux/idr.h> #include <linux/slab.h> #include <linux/pm_opp.h> +#include <linux/pm_qos.h> #include <linux/thermal.h> #include <trace/events/thermal.h> -#define SCALE_ERROR_MITIGATION 100 +#define HZ_PER_KHZ 1000 +#define SCALE_ERROR_MITIGATION 100 static DEFINE_IDA(devfreq_ida); @@ -54,6 +56,8 @@ static DEFINE_IDA(devfreq_ida); * The 'res_util' range is from 100 to (power_table[state] * 100) * for the corresponding 'state'. * @capped_state: index to cooling state with in dynamic power budget + * @req_max_freq: PM QoS request for limiting the maximum frequency + * of the devfreq device. */ struct devfreq_cooling_device { int id; @@ -66,49 +70,9 @@ struct devfreq_cooling_device { struct devfreq_cooling_power *power_ops; u32 res_util; int capped_state; + struct dev_pm_qos_request req_max_freq; }; -/** - * partition_enable_opps() - disable all opps above a given state - * @dfc: Pointer to devfreq we are operating on - * @cdev_state: cooling device state we're setting - * - * Go through the OPPs of the device, enabling all OPPs until - * @cdev_state and disabling those frequencies above it. - */ -static int partition_enable_opps(struct devfreq_cooling_device *dfc, - unsigned long cdev_state) -{ - int i; - struct device *dev = dfc->devfreq->dev.parent; - - for (i = 0; i < dfc->freq_table_size; i++) { - struct dev_pm_opp *opp; - int ret = 0; - unsigned int freq = dfc->freq_table[i]; - bool want_enable = i >= cdev_state ? true : false; - - opp = dev_pm_opp_find_freq_exact(dev, freq, !want_enable); - - if (PTR_ERR(opp) == -ERANGE) - continue; - else if (IS_ERR(opp)) - return PTR_ERR(opp); - - dev_pm_opp_put(opp); - - if (want_enable) - ret = dev_pm_opp_enable(dev, freq); - else - ret = dev_pm_opp_disable(dev, freq); - - if (ret) - return ret; - } - - return 0; -} - static int devfreq_cooling_get_max_state(struct thermal_cooling_device *cdev, unsigned long *state) { @@ -135,7 +99,7 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, struct devfreq_cooling_device *dfc = cdev->devdata; struct devfreq *df = dfc->devfreq; struct device *dev = df->dev.parent; - int ret; + unsigned long freq; if (state == dfc->cooling_state) return 0; @@ -145,9 +109,10 @@ static int devfreq_cooling_set_cur_state(struct thermal_cooling_device *cdev, if (state >= dfc->freq_table_size) return -EINVAL; - ret = partition_enable_opps(dfc, state); - if (ret) - return ret; + freq = dfc->freq_table[state]; + + dev_pm_qos_update_request(&dfc->req_max_freq, + DIV_ROUND_UP(freq, HZ_PER_KHZ)); dfc->cooling_state = state; @@ -530,9 +495,15 @@ of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df, if (err) goto free_dfc; - err = ida_simple_get(&devfreq_ida, 0, 0, GFP_KERNEL); + err = dev_pm_qos_add_request(df->dev.parent, &dfc->req_max_freq, + DEV_PM_QOS_MAX_FREQUENCY, + PM_QOS_MAX_FREQUENCY_DEFAULT_VALUE); if (err < 0) goto free_tables; + + err = ida_simple_get(&devfreq_ida, 0, 0, GFP_KERNEL); + if (err < 0) + goto remove_qos_req; dfc->id = err; snprintf(dev_name, sizeof(dev_name), "thermal-devfreq-%d", dfc->id); @@ -553,6 +524,10 @@ of_devfreq_cooling_register_power(struct device_node *np, struct devfreq *df, release_ida: ida_simple_remove(&devfreq_ida, dfc->id); + +remove_qos_req: + dev_pm_qos_remove_request(&dfc->req_max_freq); + free_tables: kfree(dfc->power_table); kfree(dfc->freq_table); @@ -601,6 +576,7 @@ void devfreq_cooling_unregister(struct thermal_cooling_device *cdev) thermal_cooling_device_unregister(dfc->cdev); ida_simple_remove(&devfreq_ida, dfc->id); + dev_pm_qos_remove_request(&dfc->req_max_freq); kfree(dfc->power_table); kfree(dfc->freq_table); diff --git a/drivers/thermal/fair_share.c b/drivers/thermal/gov_fair_share.c index aaa07180ab48..aaa07180ab48 100644 --- a/drivers/thermal/fair_share.c +++ b/drivers/thermal/gov_fair_share.c diff --git a/drivers/thermal/power_allocator.c b/drivers/thermal/gov_power_allocator.c index 44636475b2a3..44636475b2a3 100644 --- a/drivers/thermal/power_allocator.c +++ b/drivers/thermal/gov_power_allocator.c diff --git a/drivers/thermal/step_wise.c b/drivers/thermal/gov_step_wise.c index 2ae7198d3067..2ae7198d3067 100644 --- a/drivers/thermal/step_wise.c +++ b/drivers/thermal/gov_step_wise.c diff --git a/drivers/thermal/user_space.c b/drivers/thermal/gov_user_space.c index 293cffd9c8ad..82a7198bbe71 100644 --- a/drivers/thermal/user_space.c +++ b/drivers/thermal/gov_user_space.c @@ -10,8 +10,8 @@ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ */ -#include <linux/thermal.h> #include <linux/slab.h> +#include <linux/thermal.h> #include "thermal_core.h" diff --git a/drivers/thermal/imx8mm_thermal.c b/drivers/thermal/imx8mm_thermal.c index 0d60f8d7894f..e6061e26d4ac 100644 --- a/drivers/thermal/imx8mm_thermal.c +++ b/drivers/thermal/imx8mm_thermal.c @@ -54,7 +54,7 @@ struct imx8mm_tmu { void __iomem *base; struct clk *clk; const struct thermal_soc_data *socdata; - struct tmu_sensor sensors[0]; + struct tmu_sensor sensors[]; }; static int imx8mm_tmu_get_temp(void *data, int *temp) diff --git a/drivers/thermal/imx_sc_thermal.c b/drivers/thermal/imx_sc_thermal.c index 8938ea81a525..b01d28eca7ee 100644 --- a/drivers/thermal/imx_sc_thermal.c +++ b/drivers/thermal/imx_sc_thermal.c @@ -14,6 +14,7 @@ #include <linux/thermal.h> #include "thermal_core.h" +#include "thermal_hwmon.h" #define IMX_SC_MISC_FUNC_GET_TEMP 13 @@ -115,6 +116,9 @@ static int imx_sc_thermal_probe(struct platform_device *pdev) ret = PTR_ERR(sensor->tzd); break; } + + if (devm_thermal_add_hwmon_sysfs(sensor->tzd)) + dev_warn(&pdev->dev, "failed to add hwmon sysfs attributes\n"); } of_node_put(sensor_np); diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index ceef89c956bd..0b3a62655843 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -13,6 +13,7 @@ #include "acpi_thermal_rel.h" #define INT3400_THERMAL_TABLE_CHANGED 0x83 +#define INT3400_ODVP_CHANGED 0x88 enum int3400_thermal_uuid { INT3400_THERMAL_PASSIVE_1, @@ -41,8 +42,11 @@ static char *int3400_thermal_uuids[INT3400_THERMAL_MAXIMUM_UUID] = { "BE84BABF-C4D4-403D-B495-3128FD44dAC1", }; +struct odvp_attr; + struct int3400_thermal_priv { struct acpi_device *adev; + struct platform_device *pdev; struct thermal_zone_device *thermal; int mode; int art_count; @@ -52,6 +56,36 @@ struct int3400_thermal_priv { u8 uuid_bitmap; int rel_misc_dev_res; int current_uuid_index; + char *data_vault; + int odvp_count; + int *odvp; + struct odvp_attr *odvp_attrs; +}; + +static int evaluate_odvp(struct int3400_thermal_priv *priv); + +struct odvp_attr { + int odvp; + struct int3400_thermal_priv *priv; + struct kobj_attribute attr; +}; + +static ssize_t data_vault_read(struct file *file, struct kobject *kobj, + struct bin_attribute *attr, char *buf, loff_t off, size_t count) +{ + memcpy(buf, attr->private + off, count); + return count; +} + +static BIN_ATTR_RO(data_vault, 0); + +static struct bin_attribute *data_attributes[] = { + &bin_attr_data_vault, + NULL, +}; + +static const struct attribute_group data_attribute_group = { + .bin_attrs = data_attributes, }; static ssize_t available_uuids_show(struct device *dev, @@ -62,6 +96,9 @@ static ssize_t available_uuids_show(struct device *dev, int i; int length = 0; + if (!priv->uuid_bitmap) + return sprintf(buf, "UNKNOWN\n"); + for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; i++) { if (priv->uuid_bitmap & (1 << i)) if (PAGE_SIZE - length > 0) @@ -79,11 +116,11 @@ static ssize_t current_uuid_show(struct device *dev, { struct int3400_thermal_priv *priv = dev_get_drvdata(dev); - if (priv->uuid_bitmap & (1 << priv->current_uuid_index)) - return sprintf(buf, "%s\n", - int3400_thermal_uuids[priv->current_uuid_index]); - else + if (priv->current_uuid_index == -1) return sprintf(buf, "INVALID\n"); + + return sprintf(buf, "%s\n", + int3400_thermal_uuids[priv->current_uuid_index]); } static ssize_t current_uuid_store(struct device *dev, @@ -94,9 +131,16 @@ static ssize_t current_uuid_store(struct device *dev, int i; for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; ++i) { - if ((priv->uuid_bitmap & (1 << i)) && - !(strncmp(buf, int3400_thermal_uuids[i], - sizeof(int3400_thermal_uuids[i]) - 1))) { + if (!strncmp(buf, int3400_thermal_uuids[i], + sizeof(int3400_thermal_uuids[i]) - 1)) { + /* + * If we have a list of supported UUIDs, make sure + * this one is supported. + */ + if (priv->uuid_bitmap && + !(priv->uuid_bitmap & (1 << i))) + return -EINVAL; + priv->current_uuid_index = i; return count; } @@ -191,9 +235,110 @@ static int int3400_thermal_run_osc(acpi_handle handle, result = -EPERM; kfree(context.ret.pointer); + return result; } +static ssize_t odvp_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct odvp_attr *odvp_attr; + + odvp_attr = container_of(attr, struct odvp_attr, attr); + + return sprintf(buf, "%d\n", odvp_attr->priv->odvp[odvp_attr->odvp]); +} + +static void cleanup_odvp(struct int3400_thermal_priv *priv) +{ + int i; + + if (priv->odvp_attrs) { + for (i = 0; i < priv->odvp_count; i++) { + sysfs_remove_file(&priv->pdev->dev.kobj, + &priv->odvp_attrs[i].attr.attr); + kfree(priv->odvp_attrs[i].attr.attr.name); + } + kfree(priv->odvp_attrs); + } + kfree(priv->odvp); + priv->odvp_count = 0; +} + +static int evaluate_odvp(struct int3400_thermal_priv *priv) +{ + struct acpi_buffer odvp = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *obj = NULL; + acpi_status status; + int i, ret; + + status = acpi_evaluate_object(priv->adev->handle, "ODVP", NULL, &odvp); + if (ACPI_FAILURE(status)) { + ret = -EINVAL; + goto out_err; + } + + obj = odvp.pointer; + if (obj->type != ACPI_TYPE_PACKAGE) { + ret = -EINVAL; + goto out_err; + } + + if (priv->odvp == NULL) { + priv->odvp_count = obj->package.count; + priv->odvp = kmalloc_array(priv->odvp_count, sizeof(int), + GFP_KERNEL); + if (!priv->odvp) { + ret = -ENOMEM; + goto out_err; + } + } + + if (priv->odvp_attrs == NULL) { + priv->odvp_attrs = kcalloc(priv->odvp_count, + sizeof(struct odvp_attr), + GFP_KERNEL); + if (!priv->odvp_attrs) { + ret = -ENOMEM; + goto out_err; + } + for (i = 0; i < priv->odvp_count; i++) { + struct odvp_attr *odvp = &priv->odvp_attrs[i]; + + sysfs_attr_init(&odvp->attr.attr); + odvp->priv = priv; + odvp->odvp = i; + odvp->attr.attr.name = kasprintf(GFP_KERNEL, + "odvp%d", i); + + if (!odvp->attr.attr.name) { + ret = -ENOMEM; + goto out_err; + } + odvp->attr.attr.mode = 0444; + odvp->attr.show = odvp_show; + odvp->attr.store = NULL; + ret = sysfs_create_file(&priv->pdev->dev.kobj, + &odvp->attr.attr); + if (ret) + goto out_err; + } + } + + for (i = 0; i < obj->package.count; i++) { + if (obj->package.elements[i].type == ACPI_TYPE_INTEGER) + priv->odvp[i] = obj->package.elements[i].integer.value; + } + + kfree(obj); + return 0; + +out_err: + cleanup_odvp(priv); + kfree(obj); + return ret; +} + static void int3400_notify(acpi_handle handle, u32 event, void *data) @@ -217,6 +362,9 @@ static void int3400_notify(acpi_handle handle, kobject_uevent_env(&priv->thermal->device.kobj, KOBJ_CHANGE, thermal_prop); break; + case INT3400_ODVP_CHANGED: + evaluate_odvp(priv); + break; default: /* Ignore unknown notification codes sent to INT3400 device */ break; @@ -266,11 +414,16 @@ static int int3400_thermal_set_mode(struct thermal_zone_device *thermal, priv->current_uuid_index, enable); } + + evaluate_odvp(priv); + return result; } static struct thermal_zone_device_ops int3400_thermal_ops = { .get_temp = int3400_thermal_get_temp, + .get_mode = int3400_thermal_get_mode, + .set_mode = int3400_thermal_set_mode, }; static struct thermal_zone_params int3400_thermal_params = { @@ -278,6 +431,32 @@ static struct thermal_zone_params int3400_thermal_params = { .no_hwmon = true, }; +static void int3400_setup_gddv(struct int3400_thermal_priv *priv) +{ + struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; + union acpi_object *obj; + acpi_status status; + + status = acpi_evaluate_object(priv->adev->handle, "GDDV", NULL, + &buffer); + if (ACPI_FAILURE(status) || !buffer.length) + return; + + obj = buffer.pointer; + if (obj->type != ACPI_TYPE_PACKAGE || obj->package.count != 1 + || obj->package.elements[0].type != ACPI_TYPE_BUFFER) { + kfree(buffer.pointer); + return; + } + + priv->data_vault = kmemdup(obj->package.elements[0].buffer.pointer, + obj->package.elements[0].buffer.length, + GFP_KERNEL); + bin_attr_data_vault.private = priv->data_vault; + bin_attr_data_vault.size = obj->package.elements[0].buffer.length; + kfree(buffer.pointer); +} + static int int3400_thermal_probe(struct platform_device *pdev) { struct acpi_device *adev = ACPI_COMPANION(&pdev->dev); @@ -291,12 +470,17 @@ static int int3400_thermal_probe(struct platform_device *pdev) if (!priv) return -ENOMEM; + priv->pdev = pdev; priv->adev = adev; result = int3400_thermal_get_uuids(priv); - if (result) + + /* Missing IDSP isn't fatal */ + if (result && result != -ENODEV) goto free_priv; + priv->current_uuid_index = -1; + result = acpi_parse_art(priv->adev->handle, &priv->art_count, &priv->arts, true); if (result) @@ -309,8 +493,9 @@ static int int3400_thermal_probe(struct platform_device *pdev) platform_set_drvdata(pdev, priv); - int3400_thermal_ops.get_mode = int3400_thermal_get_mode; - int3400_thermal_ops.set_mode = int3400_thermal_set_mode; + int3400_setup_gddv(priv); + + evaluate_odvp(priv); priv->thermal = thermal_zone_device_register("INT3400 Thermal", 0, 0, priv, &int3400_thermal_ops, @@ -327,6 +512,13 @@ static int int3400_thermal_probe(struct platform_device *pdev) if (result) goto free_rel_misc; + if (priv->data_vault) { + result = sysfs_create_group(&pdev->dev.kobj, + &data_attribute_group); + if (result) + goto free_uuid; + } + result = acpi_install_notify_handler( priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify, (void *)priv); @@ -336,6 +528,12 @@ static int int3400_thermal_probe(struct platform_device *pdev) return 0; free_sysfs: + cleanup_odvp(priv); + if (priv->data_vault) { + sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group); + kfree(priv->data_vault); + } +free_uuid: sysfs_remove_group(&pdev->dev.kobj, &uuid_attribute_group); free_rel_misc: if (!priv->rel_misc_dev_res) @@ -357,11 +555,16 @@ static int int3400_thermal_remove(struct platform_device *pdev) priv->adev->handle, ACPI_DEVICE_NOTIFY, int3400_notify); + cleanup_odvp(priv); + if (!priv->rel_misc_dev_res) acpi_thermal_rel_misc_device_remove(priv->adev->handle); + if (priv->data_vault) + sysfs_remove_group(&pdev->dev.kobj, &data_attribute_group); sysfs_remove_group(&pdev->dev.kobj, &uuid_attribute_group); thermal_zone_device_unregister(priv->thermal); + kfree(priv->data_vault); kfree(priv->trts); kfree(priv->arts); kfree(priv); diff --git a/drivers/thermal/k3_bandgap.c b/drivers/thermal/k3_bandgap.c new file mode 100644 index 000000000000..35f41e8a0b75 --- /dev/null +++ b/drivers/thermal/k3_bandgap.c @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * TI Bandgap temperature sensor driver for K3 SoC Family + * + * Copyright (C) 2020 Texas Instruments Incorporated - http://www.ti.com/ + */ + +#include <linux/err.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_platform.h> +#include <linux/pm_runtime.h> +#include <linux/thermal.h> +#include <linux/types.h> + +#define K3_VTM_DEVINFO_PWR0_OFFSET 0x4 +#define K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK 0xf0 +#define K3_VTM_TMPSENS0_CTRL_OFFSET 0x80 +#define K3_VTM_REGS_PER_TS 0x10 +#define K3_VTM_TS_STAT_DTEMP_MASK 0x3ff +#define K3_VTM_TMPSENS_CTRL_CBIASSEL BIT(0) +#define K3_VTM_TMPSENS_CTRL_SOC BIT(5) +#define K3_VTM_TMPSENS_CTRL_CLRZ BIT(6) +#define K3_VTM_TMPSENS_CTRL_CLKON_REQ BIT(7) + +#define K3_VTM_ADC_BEGIN_VAL 540 +#define K3_VTM_ADC_END_VAL 944 + +static const int k3_adc_to_temp[] = { + -40000, -40000, -40000, -40000, -39800, -39400, -39000, -38600, -38200, + -37800, -37400, -37000, -36600, -36200, -35800, -35300, -34700, -34200, + -33800, -33400, -33000, -32600, -32200, -31800, -31400, -31000, -30600, + -30200, -29800, -29400, -29000, -28600, -28200, -27700, -27100, -26600, + -26200, -25800, -25400, -25000, -24600, -24200, -23800, -23400, -23000, + -22600, -22200, -21800, -21400, -21000, -20500, -19900, -19400, -19000, + -18600, -18200, -17800, -17400, -17000, -16600, -16200, -15800, -15400, + -15000, -14600, -14200, -13800, -13400, -13000, -12500, -11900, -11400, + -11000, -10600, -10200, -9800, -9400, -9000, -8600, -8200, -7800, -7400, + -7000, -6600, -6200, -5800, -5400, -5000, -4500, -3900, -3400, -3000, + -2600, -2200, -1800, -1400, -1000, -600, -200, 200, 600, 1000, 1400, + 1800, 2200, 2600, 3000, 3400, 3900, 4500, 5000, 5400, 5800, 6200, 6600, + 7000, 7400, 7800, 8200, 8600, 9000, 9400, 9800, 10200, 10600, 11000, + 11400, 11800, 12200, 12700, 13300, 13800, 14200, 14600, 15000, 15400, + 15800, 16200, 16600, 17000, 17400, 17800, 18200, 18600, 19000, 19400, + 19800, 20200, 20600, 21000, 21400, 21900, 22500, 23000, 23400, 23800, + 24200, 24600, 25000, 25400, 25800, 26200, 26600, 27000, 27400, 27800, + 28200, 28600, 29000, 29400, 29800, 30200, 30600, 31000, 31400, 31900, + 32500, 33000, 33400, 33800, 34200, 34600, 35000, 35400, 35800, 36200, + 36600, 37000, 37400, 37800, 38200, 38600, 39000, 39400, 39800, 40200, + 40600, 41000, 41400, 41800, 42200, 42600, 43100, 43700, 44200, 44600, + 45000, 45400, 45800, 46200, 46600, 47000, 47400, 47800, 48200, 48600, + 49000, 49400, 49800, 50200, 50600, 51000, 51400, 51800, 52200, 52600, + 53000, 53400, 53800, 54200, 54600, 55000, 55400, 55900, 56500, 57000, + 57400, 57800, 58200, 58600, 59000, 59400, 59800, 60200, 60600, 61000, + 61400, 61800, 62200, 62600, 63000, 63400, 63800, 64200, 64600, 65000, + 65400, 65800, 66200, 66600, 67000, 67400, 67800, 68200, 68600, 69000, + 69400, 69800, 70200, 70600, 71000, 71500, 72100, 72600, 73000, 73400, + 73800, 74200, 74600, 75000, 75400, 75800, 76200, 76600, 77000, 77400, + 77800, 78200, 78600, 79000, 79400, 79800, 80200, 80600, 81000, 81400, + 81800, 82200, 82600, 83000, 83400, 83800, 84200, 84600, 85000, 85400, + 85800, 86200, 86600, 87000, 87400, 87800, 88200, 88600, 89000, 89400, + 89800, 90200, 90600, 91000, 91400, 91800, 92200, 92600, 93000, 93400, + 93800, 94200, 94600, 95000, 95400, 95800, 96200, 96600, 97000, 97500, + 98100, 98600, 99000, 99400, 99800, 100200, 100600, 101000, 101400, + 101800, 102200, 102600, 103000, 103400, 103800, 104200, 104600, 105000, + 105400, 105800, 106200, 106600, 107000, 107400, 107800, 108200, 108600, + 109000, 109400, 109800, 110200, 110600, 111000, 111400, 111800, 112200, + 112600, 113000, 113400, 113800, 114200, 114600, 115000, 115400, 115800, + 116200, 116600, 117000, 117400, 117800, 118200, 118600, 119000, 119400, + 119800, 120200, 120600, 121000, 121400, 121800, 122200, 122600, 123000, + 123400, 123800, 124200, 124600, 124900, 125000, +}; + +struct k3_bandgap { + void __iomem *base; + const struct k3_bandgap_data *conf; +}; + +/* common data structures */ +struct k3_thermal_data { + struct thermal_zone_device *tzd; + struct k3_bandgap *bgp; + int sensor_id; + u32 ctrl_offset; + u32 stat_offset; +}; + +static unsigned int vtm_get_best_value(unsigned int s0, unsigned int s1, + unsigned int s2) +{ + int d01 = abs(s0 - s1); + int d02 = abs(s0 - s2); + int d12 = abs(s1 - s2); + + if (d01 <= d02 && d01 <= d12) + return (s0 + s1) / 2; + + if (d02 <= d01 && d02 <= d12) + return (s0 + s2) / 2; + + return (s1 + s2) / 2; +} + +static int k3_bgp_read_temp(struct k3_thermal_data *devdata, + int *temp) +{ + struct k3_bandgap *bgp; + unsigned int dtemp, s0, s1, s2; + + bgp = devdata->bgp; + + /* + * Errata is applicable for am654 pg 1.0 silicon. There + * is a variation of the order for 8-10 degree centigrade. + * Work around that by getting the average of two closest + * readings out of three readings everytime we want to + * report temperatures. + * + * Errata workaround. + */ + s0 = readl(bgp->base + devdata->stat_offset) & + K3_VTM_TS_STAT_DTEMP_MASK; + s1 = readl(bgp->base + devdata->stat_offset) & + K3_VTM_TS_STAT_DTEMP_MASK; + s2 = readl(bgp->base + devdata->stat_offset) & + K3_VTM_TS_STAT_DTEMP_MASK; + dtemp = vtm_get_best_value(s0, s1, s2); + + if (dtemp < K3_VTM_ADC_BEGIN_VAL || dtemp > K3_VTM_ADC_END_VAL) + return -EINVAL; + + *temp = k3_adc_to_temp[dtemp - K3_VTM_ADC_BEGIN_VAL]; + + return 0; +} + +static int k3_thermal_get_temp(void *devdata, int *temp) +{ + struct k3_thermal_data *data = devdata; + int ret = 0; + + ret = k3_bgp_read_temp(data, temp); + if (ret) + return ret; + + return ret; +} + +static const struct thermal_zone_of_device_ops k3_of_thermal_ops = { + .get_temp = k3_thermal_get_temp, +}; + +static const struct of_device_id of_k3_bandgap_match[]; + +static int k3_bandgap_probe(struct platform_device *pdev) +{ + int ret = 0, cnt, val, id; + struct resource *res; + struct device *dev = &pdev->dev; + struct k3_bandgap *bgp; + struct k3_thermal_data *data; + + if (ARRAY_SIZE(k3_adc_to_temp) != (K3_VTM_ADC_END_VAL + 1 - + K3_VTM_ADC_BEGIN_VAL)) + return -EINVAL; + + bgp = devm_kzalloc(&pdev->dev, sizeof(*bgp), GFP_KERNEL); + if (!bgp) + return -ENOMEM; + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + bgp->base = devm_ioremap_resource(dev, res); + if (IS_ERR(bgp->base)) + return PTR_ERR(bgp->base); + + pm_runtime_enable(dev); + ret = pm_runtime_get_sync(dev); + if (ret < 0) { + pm_runtime_put_noidle(dev); + pm_runtime_disable(dev); + return ret; + } + + /* Get the sensor count in the VTM */ + val = readl(bgp->base + K3_VTM_DEVINFO_PWR0_OFFSET); + cnt = val & K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK; + cnt >>= __ffs(K3_VTM_DEVINFO_PWR0_TEMPSENS_CT_MASK); + + data = devm_kcalloc(dev, cnt, sizeof(*data), GFP_KERNEL); + if (!data) { + ret = -ENOMEM; + goto err_alloc; + } + + /* Register the thermal sensors */ + for (id = 0; id < cnt; id++) { + data[id].sensor_id = id; + data[id].bgp = bgp; + data[id].ctrl_offset = K3_VTM_TMPSENS0_CTRL_OFFSET + + id * K3_VTM_REGS_PER_TS; + data[id].stat_offset = data[id].ctrl_offset + 0x8; + + val = readl(data[id].bgp->base + data[id].ctrl_offset); + val |= (K3_VTM_TMPSENS_CTRL_SOC | + K3_VTM_TMPSENS_CTRL_CLRZ | + K3_VTM_TMPSENS_CTRL_CLKON_REQ); + val &= ~K3_VTM_TMPSENS_CTRL_CBIASSEL; + writel(val, data[id].bgp->base + data[id].ctrl_offset); + + data[id].tzd = + devm_thermal_zone_of_sensor_register(dev, id, + &data[id], + &k3_of_thermal_ops); + if (IS_ERR(data[id].tzd)) { + dev_err(dev, "thermal zone device is NULL\n"); + ret = PTR_ERR(data[id].tzd); + goto err_alloc; + } + } + + platform_set_drvdata(pdev, bgp); + + return 0; + +err_alloc: + pm_runtime_put_sync(dev); + pm_runtime_disable(dev); + + return ret; +} + +static int k3_bandgap_remove(struct platform_device *pdev) +{ + pm_runtime_put_sync(&pdev->dev); + pm_runtime_disable(&pdev->dev); + + return 0; +} + +static const struct of_device_id of_k3_bandgap_match[] = { + { + .compatible = "ti,am654-vtm", + }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, of_k3_bandgap_match); + +static struct platform_driver k3_bandgap_sensor_driver = { + .probe = k3_bandgap_probe, + .remove = k3_bandgap_remove, + .driver = { + .name = "k3-soc-thermal", + .of_match_table = of_k3_bandgap_match, + }, +}; + +module_platform_driver(k3_bandgap_sensor_driver); + +MODULE_DESCRIPTION("K3 bandgap temperature sensor driver"); +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("J Keerthy <j-keerthy@ti.com>"); diff --git a/drivers/thermal/qcom/Makefile b/drivers/thermal/qcom/Makefile index 7c8dc6e36693..ec86eef7f6a6 100644 --- a/drivers/thermal/qcom/Makefile +++ b/drivers/thermal/qcom/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_QCOM_TSENS) += qcom_tsens.o -qcom_tsens-y += tsens.o tsens-common.o tsens-v0_1.o \ - tsens-8960.o tsens-v2.o tsens-v1.o +qcom_tsens-y += tsens.o tsens-v2.o tsens-v1.o tsens-v0_1.o \ + tsens-8960.o obj-$(CONFIG_QCOM_SPMI_TEMP_ALARM) += qcom-spmi-temp-alarm.o diff --git a/drivers/thermal/qcom/tsens-common.c b/drivers/thermal/qcom/tsens-common.c deleted file mode 100644 index 172545366636..000000000000 --- a/drivers/thermal/qcom/tsens-common.c +++ /dev/null @@ -1,843 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Copyright (c) 2015, The Linux Foundation. All rights reserved. - */ - -#include <linux/debugfs.h> -#include <linux/err.h> -#include <linux/io.h> -#include <linux/nvmem-consumer.h> -#include <linux/of_address.h> -#include <linux/of_platform.h> -#include <linux/platform_device.h> -#include <linux/regmap.h> -#include "tsens.h" - -/** - * struct tsens_irq_data - IRQ status and temperature violations - * @up_viol: upper threshold violated - * @up_thresh: upper threshold temperature value - * @up_irq_mask: mask register for upper threshold irqs - * @up_irq_clear: clear register for uppper threshold irqs - * @low_viol: lower threshold violated - * @low_thresh: lower threshold temperature value - * @low_irq_mask: mask register for lower threshold irqs - * @low_irq_clear: clear register for lower threshold irqs - * @crit_viol: critical threshold violated - * @crit_thresh: critical threshold temperature value - * @crit_irq_mask: mask register for critical threshold irqs - * @crit_irq_clear: clear register for critical threshold irqs - * - * Structure containing data about temperature threshold settings and - * irq status if they were violated. - */ -struct tsens_irq_data { - u32 up_viol; - int up_thresh; - u32 up_irq_mask; - u32 up_irq_clear; - u32 low_viol; - int low_thresh; - u32 low_irq_mask; - u32 low_irq_clear; - u32 crit_viol; - u32 crit_thresh; - u32 crit_irq_mask; - u32 crit_irq_clear; -}; - -char *qfprom_read(struct device *dev, const char *cname) -{ - struct nvmem_cell *cell; - ssize_t data; - char *ret; - - cell = nvmem_cell_get(dev, cname); - if (IS_ERR(cell)) - return ERR_CAST(cell); - - ret = nvmem_cell_read(cell, &data); - nvmem_cell_put(cell); - - return ret; -} - -/* - * Use this function on devices where slope and offset calculations - * depend on calibration data read from qfprom. On others the slope - * and offset values are derived from tz->tzp->slope and tz->tzp->offset - * resp. - */ -void compute_intercept_slope(struct tsens_priv *priv, u32 *p1, - u32 *p2, u32 mode) -{ - int i; - int num, den; - - for (i = 0; i < priv->num_sensors; i++) { - dev_dbg(priv->dev, - "%s: sensor%d - data_point1:%#x data_point2:%#x\n", - __func__, i, p1[i], p2[i]); - - priv->sensor[i].slope = SLOPE_DEFAULT; - if (mode == TWO_PT_CALIB) { - /* - * slope (m) = adc_code2 - adc_code1 (y2 - y1)/ - * temp_120_degc - temp_30_degc (x2 - x1) - */ - num = p2[i] - p1[i]; - num *= SLOPE_FACTOR; - den = CAL_DEGC_PT2 - CAL_DEGC_PT1; - priv->sensor[i].slope = num / den; - } - - priv->sensor[i].offset = (p1[i] * SLOPE_FACTOR) - - (CAL_DEGC_PT1 * - priv->sensor[i].slope); - dev_dbg(priv->dev, "%s: offset:%d\n", __func__, priv->sensor[i].offset); - } -} - -static inline u32 degc_to_code(int degc, const struct tsens_sensor *s) -{ - u64 code = div_u64(((u64)degc * s->slope + s->offset), SLOPE_FACTOR); - - pr_debug("%s: raw_code: 0x%llx, degc:%d\n", __func__, code, degc); - return clamp_val(code, THRESHOLD_MIN_ADC_CODE, THRESHOLD_MAX_ADC_CODE); -} - -static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s) -{ - int degc, num, den; - - num = (adc_code * SLOPE_FACTOR) - s->offset; - den = s->slope; - - if (num > 0) - degc = num + (den / 2); - else if (num < 0) - degc = num - (den / 2); - else - degc = num; - - degc /= den; - - return degc; -} - -/** - * tsens_hw_to_mC - Return sign-extended temperature in mCelsius. - * @s: Pointer to sensor struct - * @field: Index into regmap_field array pointing to temperature data - * - * This function handles temperature returned in ADC code or deciCelsius - * depending on IP version. - * - * Return: Temperature in milliCelsius on success, a negative errno will - * be returned in error cases - */ -static int tsens_hw_to_mC(const struct tsens_sensor *s, int field) -{ - struct tsens_priv *priv = s->priv; - u32 resolution; - u32 temp = 0; - int ret; - - resolution = priv->fields[LAST_TEMP_0].msb - - priv->fields[LAST_TEMP_0].lsb; - - ret = regmap_field_read(priv->rf[field], &temp); - if (ret) - return ret; - - /* Convert temperature from ADC code to milliCelsius */ - if (priv->feat->adc) - return code_to_degc(temp, s) * 1000; - - /* deciCelsius -> milliCelsius along with sign extension */ - return sign_extend32(temp, resolution) * 100; -} - -/** - * tsens_mC_to_hw - Convert temperature to hardware register value - * @s: Pointer to sensor struct - * @temp: temperature in milliCelsius to be programmed to hardware - * - * This function outputs the value to be written to hardware in ADC code - * or deciCelsius depending on IP version. - * - * Return: ADC code or temperature in deciCelsius. - */ -static int tsens_mC_to_hw(const struct tsens_sensor *s, int temp) -{ - struct tsens_priv *priv = s->priv; - - /* milliC to adc code */ - if (priv->feat->adc) - return degc_to_code(temp / 1000, s); - - /* milliC to deciC */ - return temp / 100; -} - -static inline enum tsens_ver tsens_version(struct tsens_priv *priv) -{ - return priv->feat->ver_major; -} - -static void tsens_set_interrupt_v1(struct tsens_priv *priv, u32 hw_id, - enum tsens_irq_type irq_type, bool enable) -{ - u32 index = 0; - - switch (irq_type) { - case UPPER: - index = UP_INT_CLEAR_0 + hw_id; - break; - case LOWER: - index = LOW_INT_CLEAR_0 + hw_id; - break; - case CRITICAL: - /* No critical interrupts before v2 */ - return; - } - regmap_field_write(priv->rf[index], enable ? 0 : 1); -} - -static void tsens_set_interrupt_v2(struct tsens_priv *priv, u32 hw_id, - enum tsens_irq_type irq_type, bool enable) -{ - u32 index_mask = 0, index_clear = 0; - - /* - * To enable the interrupt flag for a sensor: - * - clear the mask bit - * To disable the interrupt flag for a sensor: - * - Mask further interrupts for this sensor - * - Write 1 followed by 0 to clear the interrupt - */ - switch (irq_type) { - case UPPER: - index_mask = UP_INT_MASK_0 + hw_id; - index_clear = UP_INT_CLEAR_0 + hw_id; - break; - case LOWER: - index_mask = LOW_INT_MASK_0 + hw_id; - index_clear = LOW_INT_CLEAR_0 + hw_id; - break; - case CRITICAL: - index_mask = CRIT_INT_MASK_0 + hw_id; - index_clear = CRIT_INT_CLEAR_0 + hw_id; - break; - } - - if (enable) { - regmap_field_write(priv->rf[index_mask], 0); - } else { - regmap_field_write(priv->rf[index_mask], 1); - regmap_field_write(priv->rf[index_clear], 1); - regmap_field_write(priv->rf[index_clear], 0); - } -} - -/** - * tsens_set_interrupt - Set state of an interrupt - * @priv: Pointer to tsens controller private data - * @hw_id: Hardware ID aka. sensor number - * @irq_type: irq_type from enum tsens_irq_type - * @enable: false = disable, true = enable - * - * Call IP-specific function to set state of an interrupt - * - * Return: void - */ -static void tsens_set_interrupt(struct tsens_priv *priv, u32 hw_id, - enum tsens_irq_type irq_type, bool enable) -{ - dev_dbg(priv->dev, "[%u] %s: %s -> %s\n", hw_id, __func__, - irq_type ? ((irq_type == 1) ? "UP" : "CRITICAL") : "LOW", - enable ? "en" : "dis"); - if (tsens_version(priv) > VER_1_X) - tsens_set_interrupt_v2(priv, hw_id, irq_type, enable); - else - tsens_set_interrupt_v1(priv, hw_id, irq_type, enable); -} - -/** - * tsens_threshold_violated - Check if a sensor temperature violated a preset threshold - * @priv: Pointer to tsens controller private data - * @hw_id: Hardware ID aka. sensor number - * @d: Pointer to irq state data - * - * Return: 0 if threshold was not violated, 1 if it was violated and negative - * errno in case of errors - */ -static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id, - struct tsens_irq_data *d) -{ - int ret; - - ret = regmap_field_read(priv->rf[UPPER_STATUS_0 + hw_id], &d->up_viol); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[LOWER_STATUS_0 + hw_id], &d->low_viol); - if (ret) - return ret; - - if (priv->feat->crit_int) { - ret = regmap_field_read(priv->rf[CRITICAL_STATUS_0 + hw_id], - &d->crit_viol); - if (ret) - return ret; - } - - if (d->up_viol || d->low_viol || d->crit_viol) - return 1; - - return 0; -} - -static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, - const struct tsens_sensor *s, - struct tsens_irq_data *d) -{ - int ret; - - ret = regmap_field_read(priv->rf[UP_INT_CLEAR_0 + hw_id], &d->up_irq_clear); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[LOW_INT_CLEAR_0 + hw_id], &d->low_irq_clear); - if (ret) - return ret; - if (tsens_version(priv) > VER_1_X) { - ret = regmap_field_read(priv->rf[UP_INT_MASK_0 + hw_id], &d->up_irq_mask); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[LOW_INT_MASK_0 + hw_id], &d->low_irq_mask); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[CRIT_INT_CLEAR_0 + hw_id], - &d->crit_irq_clear); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[CRIT_INT_MASK_0 + hw_id], - &d->crit_irq_mask); - if (ret) - return ret; - - d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id); - } else { - /* No mask register on older TSENS */ - d->up_irq_mask = 0; - d->low_irq_mask = 0; - d->crit_irq_clear = 0; - d->crit_irq_mask = 0; - d->crit_thresh = 0; - } - - d->up_thresh = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id); - d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id); - - dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n", - hw_id, __func__, - (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", - d->low_viol, d->up_viol, d->crit_viol, - d->low_irq_clear, d->up_irq_clear, d->crit_irq_clear, - d->low_irq_mask, d->up_irq_mask, d->crit_irq_mask); - dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d:%d)\n", hw_id, __func__, - (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", - d->low_thresh, d->up_thresh, d->crit_thresh); - - return 0; -} - -static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver) -{ - if (ver > VER_1_X) - return mask & (1 << hw_id); - - /* v1, v0.1 don't have a irq mask register */ - return 0; -} - -/** - * tsens_critical_irq_thread() - Threaded handler for critical interrupts - * @irq: irq number - * @data: tsens controller private data - * - * Check FSM watchdog bark status and clear if needed. - * Check all sensors to find ones that violated their critical threshold limits. - * Clear and then re-enable the interrupt. - * - * The level-triggered interrupt might deassert if the temperature returned to - * within the threshold limits by the time the handler got scheduled. We - * consider the irq to have been handled in that case. - * - * Return: IRQ_HANDLED - */ -irqreturn_t tsens_critical_irq_thread(int irq, void *data) -{ - struct tsens_priv *priv = data; - struct tsens_irq_data d; - int temp, ret, i; - u32 wdog_status, wdog_count; - - if (priv->feat->has_watchdog) { - ret = regmap_field_read(priv->rf[WDOG_BARK_STATUS], - &wdog_status); - if (ret) - return ret; - - if (wdog_status) { - /* Clear WDOG interrupt */ - regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 1); - regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 0); - ret = regmap_field_read(priv->rf[WDOG_BARK_COUNT], - &wdog_count); - if (ret) - return ret; - if (wdog_count) - dev_dbg(priv->dev, "%s: watchdog count: %d\n", - __func__, wdog_count); - - /* Fall through to handle critical interrupts if any */ - } - } - - for (i = 0; i < priv->num_sensors; i++) { - const struct tsens_sensor *s = &priv->sensor[i]; - u32 hw_id = s->hw_id; - - if (IS_ERR(s->tzd)) - continue; - if (!tsens_threshold_violated(priv, hw_id, &d)) - continue; - ret = get_temp_tsens_valid(s, &temp); - if (ret) { - dev_err(priv->dev, "[%u] %s: error reading sensor\n", - hw_id, __func__); - continue; - } - - tsens_read_irq_state(priv, hw_id, s, &d); - if (d.crit_viol && - !masked_irq(hw_id, d.crit_irq_mask, tsens_version(priv))) { - /* Mask critical interrupts, unused on Linux */ - tsens_set_interrupt(priv, hw_id, CRITICAL, false); - } - } - - return IRQ_HANDLED; -} - -/** - * tsens_irq_thread - Threaded interrupt handler for uplow interrupts - * @irq: irq number - * @data: tsens controller private data - * - * Check all sensors to find ones that violated their threshold limits. If the - * temperature is still outside the limits, call thermal_zone_device_update() to - * update the thresholds, else re-enable the interrupts. - * - * The level-triggered interrupt might deassert if the temperature returned to - * within the threshold limits by the time the handler got scheduled. We - * consider the irq to have been handled in that case. - * - * Return: IRQ_HANDLED - */ -irqreturn_t tsens_irq_thread(int irq, void *data) -{ - struct tsens_priv *priv = data; - struct tsens_irq_data d; - bool enable = true, disable = false; - unsigned long flags; - int temp, ret, i; - - for (i = 0; i < priv->num_sensors; i++) { - bool trigger = false; - const struct tsens_sensor *s = &priv->sensor[i]; - u32 hw_id = s->hw_id; - - if (IS_ERR(s->tzd)) - continue; - if (!tsens_threshold_violated(priv, hw_id, &d)) - continue; - ret = get_temp_tsens_valid(s, &temp); - if (ret) { - dev_err(priv->dev, "[%u] %s: error reading sensor\n", hw_id, __func__); - continue; - } - - spin_lock_irqsave(&priv->ul_lock, flags); - - tsens_read_irq_state(priv, hw_id, s, &d); - - if (d.up_viol && - !masked_irq(hw_id, d.up_irq_mask, tsens_version(priv))) { - tsens_set_interrupt(priv, hw_id, UPPER, disable); - if (d.up_thresh > temp) { - dev_dbg(priv->dev, "[%u] %s: re-arm upper\n", - hw_id, __func__); - tsens_set_interrupt(priv, hw_id, UPPER, enable); - } else { - trigger = true; - /* Keep irq masked */ - } - } else if (d.low_viol && - !masked_irq(hw_id, d.low_irq_mask, tsens_version(priv))) { - tsens_set_interrupt(priv, hw_id, LOWER, disable); - if (d.low_thresh < temp) { - dev_dbg(priv->dev, "[%u] %s: re-arm low\n", - hw_id, __func__); - tsens_set_interrupt(priv, hw_id, LOWER, enable); - } else { - trigger = true; - /* Keep irq masked */ - } - } - - spin_unlock_irqrestore(&priv->ul_lock, flags); - - if (trigger) { - dev_dbg(priv->dev, "[%u] %s: TZ update trigger (%d mC)\n", - hw_id, __func__, temp); - thermal_zone_device_update(s->tzd, - THERMAL_EVENT_UNSPECIFIED); - } else { - dev_dbg(priv->dev, "[%u] %s: no violation: %d\n", - hw_id, __func__, temp); - } - } - - return IRQ_HANDLED; -} - -int tsens_set_trips(void *_sensor, int low, int high) -{ - struct tsens_sensor *s = _sensor; - struct tsens_priv *priv = s->priv; - struct device *dev = priv->dev; - struct tsens_irq_data d; - unsigned long flags; - int high_val, low_val, cl_high, cl_low; - u32 hw_id = s->hw_id; - - dev_dbg(dev, "[%u] %s: proposed thresholds: (%d:%d)\n", - hw_id, __func__, low, high); - - cl_high = clamp_val(high, -40000, 120000); - cl_low = clamp_val(low, -40000, 120000); - - high_val = tsens_mC_to_hw(s, cl_high); - low_val = tsens_mC_to_hw(s, cl_low); - - spin_lock_irqsave(&priv->ul_lock, flags); - - tsens_read_irq_state(priv, hw_id, s, &d); - - /* Write the new thresholds and clear the status */ - regmap_field_write(priv->rf[LOW_THRESH_0 + hw_id], low_val); - regmap_field_write(priv->rf[UP_THRESH_0 + hw_id], high_val); - tsens_set_interrupt(priv, hw_id, LOWER, true); - tsens_set_interrupt(priv, hw_id, UPPER, true); - - spin_unlock_irqrestore(&priv->ul_lock, flags); - - dev_dbg(dev, "[%u] %s: (%d:%d)->(%d:%d)\n", - hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high); - - return 0; -} - -int tsens_enable_irq(struct tsens_priv *priv) -{ - int ret; - int val = tsens_version(priv) > VER_1_X ? 7 : 1; - - ret = regmap_field_write(priv->rf[INT_EN], val); - if (ret < 0) - dev_err(priv->dev, "%s: failed to enable interrupts\n", __func__); - - return ret; -} - -void tsens_disable_irq(struct tsens_priv *priv) -{ - regmap_field_write(priv->rf[INT_EN], 0); -} - -int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp) -{ - struct tsens_priv *priv = s->priv; - int hw_id = s->hw_id; - u32 temp_idx = LAST_TEMP_0 + hw_id; - u32 valid_idx = VALID_0 + hw_id; - u32 valid; - int ret; - - ret = regmap_field_read(priv->rf[valid_idx], &valid); - if (ret) - return ret; - while (!valid) { - /* Valid bit is 0 for 6 AHB clock cycles. - * At 19.2MHz, 1 AHB clock is ~60ns. - * We should enter this loop very, very rarely. - */ - ndelay(400); - ret = regmap_field_read(priv->rf[valid_idx], &valid); - if (ret) - return ret; - } - - /* Valid bit is set, OK to read the temperature */ - *temp = tsens_hw_to_mC(s, temp_idx); - - return 0; -} - -int get_temp_common(const struct tsens_sensor *s, int *temp) -{ - struct tsens_priv *priv = s->priv; - int hw_id = s->hw_id; - int last_temp = 0, ret; - - ret = regmap_field_read(priv->rf[LAST_TEMP_0 + hw_id], &last_temp); - if (ret) - return ret; - - *temp = code_to_degc(last_temp, s) * 1000; - - return 0; -} - -#ifdef CONFIG_DEBUG_FS -static int dbg_sensors_show(struct seq_file *s, void *data) -{ - struct platform_device *pdev = s->private; - struct tsens_priv *priv = platform_get_drvdata(pdev); - int i; - - seq_printf(s, "max: %2d\nnum: %2d\n\n", - priv->feat->max_sensors, priv->num_sensors); - - seq_puts(s, " id slope offset\n--------------------------\n"); - for (i = 0; i < priv->num_sensors; i++) { - seq_printf(s, "%8d %8d %8d\n", priv->sensor[i].hw_id, - priv->sensor[i].slope, priv->sensor[i].offset); - } - - return 0; -} - -static int dbg_version_show(struct seq_file *s, void *data) -{ - struct platform_device *pdev = s->private; - struct tsens_priv *priv = platform_get_drvdata(pdev); - u32 maj_ver, min_ver, step_ver; - int ret; - - if (tsens_version(priv) > VER_0_1) { - ret = regmap_field_read(priv->rf[VER_MAJOR], &maj_ver); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[VER_MINOR], &min_ver); - if (ret) - return ret; - ret = regmap_field_read(priv->rf[VER_STEP], &step_ver); - if (ret) - return ret; - seq_printf(s, "%d.%d.%d\n", maj_ver, min_ver, step_ver); - } else { - seq_puts(s, "0.1.0\n"); - } - - return 0; -} - -DEFINE_SHOW_ATTRIBUTE(dbg_version); -DEFINE_SHOW_ATTRIBUTE(dbg_sensors); - -static void tsens_debug_init(struct platform_device *pdev) -{ - struct tsens_priv *priv = platform_get_drvdata(pdev); - struct dentry *root, *file; - - root = debugfs_lookup("tsens", NULL); - if (!root) - priv->debug_root = debugfs_create_dir("tsens", NULL); - else - priv->debug_root = root; - - file = debugfs_lookup("version", priv->debug_root); - if (!file) - debugfs_create_file("version", 0444, priv->debug_root, - pdev, &dbg_version_fops); - - /* A directory for each instance of the TSENS IP */ - priv->debug = debugfs_create_dir(dev_name(&pdev->dev), priv->debug_root); - debugfs_create_file("sensors", 0444, priv->debug, pdev, &dbg_sensors_fops); -} -#else -static inline void tsens_debug_init(struct platform_device *pdev) {} -#endif - -static const struct regmap_config tsens_config = { - .name = "tm", - .reg_bits = 32, - .val_bits = 32, - .reg_stride = 4, -}; - -static const struct regmap_config tsens_srot_config = { - .name = "srot", - .reg_bits = 32, - .val_bits = 32, - .reg_stride = 4, -}; - -int __init init_common(struct tsens_priv *priv) -{ - void __iomem *tm_base, *srot_base; - struct device *dev = priv->dev; - u32 ver_minor; - struct resource *res; - u32 enabled; - int ret, i, j; - struct platform_device *op = of_find_device_by_node(priv->dev->of_node); - - if (!op) - return -EINVAL; - - if (op->num_resources > 1) { - /* DT with separate SROT and TM address space */ - priv->tm_offset = 0; - res = platform_get_resource(op, IORESOURCE_MEM, 1); - srot_base = devm_ioremap_resource(dev, res); - if (IS_ERR(srot_base)) { - ret = PTR_ERR(srot_base); - goto err_put_device; - } - - priv->srot_map = devm_regmap_init_mmio(dev, srot_base, - &tsens_srot_config); - if (IS_ERR(priv->srot_map)) { - ret = PTR_ERR(priv->srot_map); - goto err_put_device; - } - } else { - /* old DTs where SROT and TM were in a contiguous 2K block */ - priv->tm_offset = 0x1000; - } - - res = platform_get_resource(op, IORESOURCE_MEM, 0); - tm_base = devm_ioremap_resource(dev, res); - if (IS_ERR(tm_base)) { - ret = PTR_ERR(tm_base); - goto err_put_device; - } - - priv->tm_map = devm_regmap_init_mmio(dev, tm_base, &tsens_config); - if (IS_ERR(priv->tm_map)) { - ret = PTR_ERR(priv->tm_map); - goto err_put_device; - } - - if (tsens_version(priv) > VER_0_1) { - for (i = VER_MAJOR; i <= VER_STEP; i++) { - priv->rf[i] = devm_regmap_field_alloc(dev, priv->srot_map, - priv->fields[i]); - if (IS_ERR(priv->rf[i])) - return PTR_ERR(priv->rf[i]); - } - ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor); - if (ret) - goto err_put_device; - } - - priv->rf[TSENS_EN] = devm_regmap_field_alloc(dev, priv->srot_map, - priv->fields[TSENS_EN]); - if (IS_ERR(priv->rf[TSENS_EN])) { - ret = PTR_ERR(priv->rf[TSENS_EN]); - goto err_put_device; - } - ret = regmap_field_read(priv->rf[TSENS_EN], &enabled); - if (ret) - goto err_put_device; - if (!enabled) { - dev_err(dev, "%s: device not enabled\n", __func__); - ret = -ENODEV; - goto err_put_device; - } - - priv->rf[SENSOR_EN] = devm_regmap_field_alloc(dev, priv->srot_map, - priv->fields[SENSOR_EN]); - if (IS_ERR(priv->rf[SENSOR_EN])) { - ret = PTR_ERR(priv->rf[SENSOR_EN]); - goto err_put_device; - } - priv->rf[INT_EN] = devm_regmap_field_alloc(dev, priv->tm_map, - priv->fields[INT_EN]); - if (IS_ERR(priv->rf[INT_EN])) { - ret = PTR_ERR(priv->rf[INT_EN]); - goto err_put_device; - } - - /* This loop might need changes if enum regfield_ids is reordered */ - for (j = LAST_TEMP_0; j <= UP_THRESH_15; j += 16) { - for (i = 0; i < priv->feat->max_sensors; i++) { - int idx = j + i; - - priv->rf[idx] = devm_regmap_field_alloc(dev, priv->tm_map, - priv->fields[idx]); - if (IS_ERR(priv->rf[idx])) { - ret = PTR_ERR(priv->rf[idx]); - goto err_put_device; - } - } - } - - if (priv->feat->crit_int) { - /* Loop might need changes if enum regfield_ids is reordered */ - for (j = CRITICAL_STATUS_0; j <= CRIT_THRESH_15; j += 16) { - for (i = 0; i < priv->feat->max_sensors; i++) { - int idx = j + i; - - priv->rf[idx] = - devm_regmap_field_alloc(dev, - priv->tm_map, - priv->fields[idx]); - if (IS_ERR(priv->rf[idx])) { - ret = PTR_ERR(priv->rf[idx]); - goto err_put_device; - } - } - } - } - - if (tsens_version(priv) > VER_1_X && ver_minor > 2) { - /* Watchdog is present only on v2.3+ */ - priv->feat->has_watchdog = 1; - for (i = WDOG_BARK_STATUS; i <= CC_MON_MASK; i++) { - priv->rf[i] = devm_regmap_field_alloc(dev, priv->tm_map, - priv->fields[i]); - if (IS_ERR(priv->rf[i])) { - ret = PTR_ERR(priv->rf[i]); - goto err_put_device; - } - } - /* - * Watchdog is already enabled, unmask the bark. - * Disable cycle completion monitoring - */ - regmap_field_write(priv->rf[WDOG_BARK_MASK], 0); - regmap_field_write(priv->rf[CC_MON_MASK], 1); - } - - spin_lock_init(&priv->ul_lock); - tsens_enable_irq(priv); - tsens_debug_init(op); - -err_put_device: - put_device(&op->dev); - return ret; -} diff --git a/drivers/thermal/qcom/tsens.c b/drivers/thermal/qcom/tsens.c index 2f77d235cf73..8d3e94d2a9ed 100644 --- a/drivers/thermal/qcom/tsens.c +++ b/drivers/thermal/qcom/tsens.c @@ -1,19 +1,857 @@ // SPDX-License-Identifier: GPL-2.0 /* * Copyright (c) 2015, The Linux Foundation. All rights reserved. + * Copyright (c) 2019, 2020, Linaro Ltd. */ #include <linux/debugfs.h> #include <linux/err.h> +#include <linux/io.h> #include <linux/module.h> +#include <linux/nvmem-consumer.h> #include <linux/of.h> +#include <linux/of_address.h> #include <linux/of_platform.h> #include <linux/platform_device.h> #include <linux/pm.h> +#include <linux/regmap.h> #include <linux/slab.h> #include <linux/thermal.h> #include "tsens.h" +/** + * struct tsens_irq_data - IRQ status and temperature violations + * @up_viol: upper threshold violated + * @up_thresh: upper threshold temperature value + * @up_irq_mask: mask register for upper threshold irqs + * @up_irq_clear: clear register for uppper threshold irqs + * @low_viol: lower threshold violated + * @low_thresh: lower threshold temperature value + * @low_irq_mask: mask register for lower threshold irqs + * @low_irq_clear: clear register for lower threshold irqs + * @crit_viol: critical threshold violated + * @crit_thresh: critical threshold temperature value + * @crit_irq_mask: mask register for critical threshold irqs + * @crit_irq_clear: clear register for critical threshold irqs + * + * Structure containing data about temperature threshold settings and + * irq status if they were violated. + */ +struct tsens_irq_data { + u32 up_viol; + int up_thresh; + u32 up_irq_mask; + u32 up_irq_clear; + u32 low_viol; + int low_thresh; + u32 low_irq_mask; + u32 low_irq_clear; + u32 crit_viol; + u32 crit_thresh; + u32 crit_irq_mask; + u32 crit_irq_clear; +}; + +char *qfprom_read(struct device *dev, const char *cname) +{ + struct nvmem_cell *cell; + ssize_t data; + char *ret; + + cell = nvmem_cell_get(dev, cname); + if (IS_ERR(cell)) + return ERR_CAST(cell); + + ret = nvmem_cell_read(cell, &data); + nvmem_cell_put(cell); + + return ret; +} + +/* + * Use this function on devices where slope and offset calculations + * depend on calibration data read from qfprom. On others the slope + * and offset values are derived from tz->tzp->slope and tz->tzp->offset + * resp. + */ +void compute_intercept_slope(struct tsens_priv *priv, u32 *p1, + u32 *p2, u32 mode) +{ + int i; + int num, den; + + for (i = 0; i < priv->num_sensors; i++) { + dev_dbg(priv->dev, + "%s: sensor%d - data_point1:%#x data_point2:%#x\n", + __func__, i, p1[i], p2[i]); + + priv->sensor[i].slope = SLOPE_DEFAULT; + if (mode == TWO_PT_CALIB) { + /* + * slope (m) = adc_code2 - adc_code1 (y2 - y1)/ + * temp_120_degc - temp_30_degc (x2 - x1) + */ + num = p2[i] - p1[i]; + num *= SLOPE_FACTOR; + den = CAL_DEGC_PT2 - CAL_DEGC_PT1; + priv->sensor[i].slope = num / den; + } + + priv->sensor[i].offset = (p1[i] * SLOPE_FACTOR) - + (CAL_DEGC_PT1 * + priv->sensor[i].slope); + dev_dbg(priv->dev, "%s: offset:%d\n", __func__, + priv->sensor[i].offset); + } +} + +static inline u32 degc_to_code(int degc, const struct tsens_sensor *s) +{ + u64 code = div_u64(((u64)degc * s->slope + s->offset), SLOPE_FACTOR); + + pr_debug("%s: raw_code: 0x%llx, degc:%d\n", __func__, code, degc); + return clamp_val(code, THRESHOLD_MIN_ADC_CODE, THRESHOLD_MAX_ADC_CODE); +} + +static inline int code_to_degc(u32 adc_code, const struct tsens_sensor *s) +{ + int degc, num, den; + + num = (adc_code * SLOPE_FACTOR) - s->offset; + den = s->slope; + + if (num > 0) + degc = num + (den / 2); + else if (num < 0) + degc = num - (den / 2); + else + degc = num; + + degc /= den; + + return degc; +} + +/** + * tsens_hw_to_mC - Return sign-extended temperature in mCelsius. + * @s: Pointer to sensor struct + * @field: Index into regmap_field array pointing to temperature data + * + * This function handles temperature returned in ADC code or deciCelsius + * depending on IP version. + * + * Return: Temperature in milliCelsius on success, a negative errno will + * be returned in error cases + */ +static int tsens_hw_to_mC(const struct tsens_sensor *s, int field) +{ + struct tsens_priv *priv = s->priv; + u32 resolution; + u32 temp = 0; + int ret; + + resolution = priv->fields[LAST_TEMP_0].msb - + priv->fields[LAST_TEMP_0].lsb; + + ret = regmap_field_read(priv->rf[field], &temp); + if (ret) + return ret; + + /* Convert temperature from ADC code to milliCelsius */ + if (priv->feat->adc) + return code_to_degc(temp, s) * 1000; + + /* deciCelsius -> milliCelsius along with sign extension */ + return sign_extend32(temp, resolution) * 100; +} + +/** + * tsens_mC_to_hw - Convert temperature to hardware register value + * @s: Pointer to sensor struct + * @temp: temperature in milliCelsius to be programmed to hardware + * + * This function outputs the value to be written to hardware in ADC code + * or deciCelsius depending on IP version. + * + * Return: ADC code or temperature in deciCelsius. + */ +static int tsens_mC_to_hw(const struct tsens_sensor *s, int temp) +{ + struct tsens_priv *priv = s->priv; + + /* milliC to adc code */ + if (priv->feat->adc) + return degc_to_code(temp / 1000, s); + + /* milliC to deciC */ + return temp / 100; +} + +static inline enum tsens_ver tsens_version(struct tsens_priv *priv) +{ + return priv->feat->ver_major; +} + +static void tsens_set_interrupt_v1(struct tsens_priv *priv, u32 hw_id, + enum tsens_irq_type irq_type, bool enable) +{ + u32 index = 0; + + switch (irq_type) { + case UPPER: + index = UP_INT_CLEAR_0 + hw_id; + break; + case LOWER: + index = LOW_INT_CLEAR_0 + hw_id; + break; + case CRITICAL: + /* No critical interrupts before v2 */ + return; + } + regmap_field_write(priv->rf[index], enable ? 0 : 1); +} + +static void tsens_set_interrupt_v2(struct tsens_priv *priv, u32 hw_id, + enum tsens_irq_type irq_type, bool enable) +{ + u32 index_mask = 0, index_clear = 0; + + /* + * To enable the interrupt flag for a sensor: + * - clear the mask bit + * To disable the interrupt flag for a sensor: + * - Mask further interrupts for this sensor + * - Write 1 followed by 0 to clear the interrupt + */ + switch (irq_type) { + case UPPER: + index_mask = UP_INT_MASK_0 + hw_id; + index_clear = UP_INT_CLEAR_0 + hw_id; + break; + case LOWER: + index_mask = LOW_INT_MASK_0 + hw_id; + index_clear = LOW_INT_CLEAR_0 + hw_id; + break; + case CRITICAL: + index_mask = CRIT_INT_MASK_0 + hw_id; + index_clear = CRIT_INT_CLEAR_0 + hw_id; + break; + } + + if (enable) { + regmap_field_write(priv->rf[index_mask], 0); + } else { + regmap_field_write(priv->rf[index_mask], 1); + regmap_field_write(priv->rf[index_clear], 1); + regmap_field_write(priv->rf[index_clear], 0); + } +} + +/** + * tsens_set_interrupt - Set state of an interrupt + * @priv: Pointer to tsens controller private data + * @hw_id: Hardware ID aka. sensor number + * @irq_type: irq_type from enum tsens_irq_type + * @enable: false = disable, true = enable + * + * Call IP-specific function to set state of an interrupt + * + * Return: void + */ +static void tsens_set_interrupt(struct tsens_priv *priv, u32 hw_id, + enum tsens_irq_type irq_type, bool enable) +{ + dev_dbg(priv->dev, "[%u] %s: %s -> %s\n", hw_id, __func__, + irq_type ? ((irq_type == 1) ? "UP" : "CRITICAL") : "LOW", + enable ? "en" : "dis"); + if (tsens_version(priv) > VER_1_X) + tsens_set_interrupt_v2(priv, hw_id, irq_type, enable); + else + tsens_set_interrupt_v1(priv, hw_id, irq_type, enable); +} + +/** + * tsens_threshold_violated - Check if a sensor temperature violated a preset threshold + * @priv: Pointer to tsens controller private data + * @hw_id: Hardware ID aka. sensor number + * @d: Pointer to irq state data + * + * Return: 0 if threshold was not violated, 1 if it was violated and negative + * errno in case of errors + */ +static int tsens_threshold_violated(struct tsens_priv *priv, u32 hw_id, + struct tsens_irq_data *d) +{ + int ret; + + ret = regmap_field_read(priv->rf[UPPER_STATUS_0 + hw_id], &d->up_viol); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[LOWER_STATUS_0 + hw_id], &d->low_viol); + if (ret) + return ret; + + if (priv->feat->crit_int) { + ret = regmap_field_read(priv->rf[CRITICAL_STATUS_0 + hw_id], + &d->crit_viol); + if (ret) + return ret; + } + + if (d->up_viol || d->low_viol || d->crit_viol) + return 1; + + return 0; +} + +static int tsens_read_irq_state(struct tsens_priv *priv, u32 hw_id, + const struct tsens_sensor *s, + struct tsens_irq_data *d) +{ + int ret; + + ret = regmap_field_read(priv->rf[UP_INT_CLEAR_0 + hw_id], &d->up_irq_clear); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[LOW_INT_CLEAR_0 + hw_id], &d->low_irq_clear); + if (ret) + return ret; + if (tsens_version(priv) > VER_1_X) { + ret = regmap_field_read(priv->rf[UP_INT_MASK_0 + hw_id], &d->up_irq_mask); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[LOW_INT_MASK_0 + hw_id], &d->low_irq_mask); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[CRIT_INT_CLEAR_0 + hw_id], + &d->crit_irq_clear); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[CRIT_INT_MASK_0 + hw_id], + &d->crit_irq_mask); + if (ret) + return ret; + + d->crit_thresh = tsens_hw_to_mC(s, CRIT_THRESH_0 + hw_id); + } else { + /* No mask register on older TSENS */ + d->up_irq_mask = 0; + d->low_irq_mask = 0; + d->crit_irq_clear = 0; + d->crit_irq_mask = 0; + d->crit_thresh = 0; + } + + d->up_thresh = tsens_hw_to_mC(s, UP_THRESH_0 + hw_id); + d->low_thresh = tsens_hw_to_mC(s, LOW_THRESH_0 + hw_id); + + dev_dbg(priv->dev, "[%u] %s%s: status(%u|%u|%u) | clr(%u|%u|%u) | mask(%u|%u|%u)\n", + hw_id, __func__, + (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", + d->low_viol, d->up_viol, d->crit_viol, + d->low_irq_clear, d->up_irq_clear, d->crit_irq_clear, + d->low_irq_mask, d->up_irq_mask, d->crit_irq_mask); + dev_dbg(priv->dev, "[%u] %s%s: thresh: (%d:%d:%d)\n", hw_id, __func__, + (d->up_viol || d->low_viol || d->crit_viol) ? "(V)" : "", + d->low_thresh, d->up_thresh, d->crit_thresh); + + return 0; +} + +static inline u32 masked_irq(u32 hw_id, u32 mask, enum tsens_ver ver) +{ + if (ver > VER_1_X) + return mask & (1 << hw_id); + + /* v1, v0.1 don't have a irq mask register */ + return 0; +} + +/** + * tsens_critical_irq_thread() - Threaded handler for critical interrupts + * @irq: irq number + * @data: tsens controller private data + * + * Check FSM watchdog bark status and clear if needed. + * Check all sensors to find ones that violated their critical threshold limits. + * Clear and then re-enable the interrupt. + * + * The level-triggered interrupt might deassert if the temperature returned to + * within the threshold limits by the time the handler got scheduled. We + * consider the irq to have been handled in that case. + * + * Return: IRQ_HANDLED + */ +irqreturn_t tsens_critical_irq_thread(int irq, void *data) +{ + struct tsens_priv *priv = data; + struct tsens_irq_data d; + int temp, ret, i; + u32 wdog_status, wdog_count; + + if (priv->feat->has_watchdog) { + ret = regmap_field_read(priv->rf[WDOG_BARK_STATUS], + &wdog_status); + if (ret) + return ret; + + if (wdog_status) { + /* Clear WDOG interrupt */ + regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 1); + regmap_field_write(priv->rf[WDOG_BARK_CLEAR], 0); + ret = regmap_field_read(priv->rf[WDOG_BARK_COUNT], + &wdog_count); + if (ret) + return ret; + if (wdog_count) + dev_dbg(priv->dev, "%s: watchdog count: %d\n", + __func__, wdog_count); + + /* Fall through to handle critical interrupts if any */ + } + } + + for (i = 0; i < priv->num_sensors; i++) { + const struct tsens_sensor *s = &priv->sensor[i]; + u32 hw_id = s->hw_id; + + if (IS_ERR(s->tzd)) + continue; + if (!tsens_threshold_violated(priv, hw_id, &d)) + continue; + ret = get_temp_tsens_valid(s, &temp); + if (ret) { + dev_err(priv->dev, "[%u] %s: error reading sensor\n", + hw_id, __func__); + continue; + } + + tsens_read_irq_state(priv, hw_id, s, &d); + if (d.crit_viol && + !masked_irq(hw_id, d.crit_irq_mask, tsens_version(priv))) { + /* Mask critical interrupts, unused on Linux */ + tsens_set_interrupt(priv, hw_id, CRITICAL, false); + } + } + + return IRQ_HANDLED; +} + +/** + * tsens_irq_thread - Threaded interrupt handler for uplow interrupts + * @irq: irq number + * @data: tsens controller private data + * + * Check all sensors to find ones that violated their threshold limits. If the + * temperature is still outside the limits, call thermal_zone_device_update() to + * update the thresholds, else re-enable the interrupts. + * + * The level-triggered interrupt might deassert if the temperature returned to + * within the threshold limits by the time the handler got scheduled. We + * consider the irq to have been handled in that case. + * + * Return: IRQ_HANDLED + */ +irqreturn_t tsens_irq_thread(int irq, void *data) +{ + struct tsens_priv *priv = data; + struct tsens_irq_data d; + bool enable = true, disable = false; + unsigned long flags; + int temp, ret, i; + + for (i = 0; i < priv->num_sensors; i++) { + bool trigger = false; + const struct tsens_sensor *s = &priv->sensor[i]; + u32 hw_id = s->hw_id; + + if (IS_ERR(s->tzd)) + continue; + if (!tsens_threshold_violated(priv, hw_id, &d)) + continue; + ret = get_temp_tsens_valid(s, &temp); + if (ret) { + dev_err(priv->dev, "[%u] %s: error reading sensor\n", + hw_id, __func__); + continue; + } + + spin_lock_irqsave(&priv->ul_lock, flags); + + tsens_read_irq_state(priv, hw_id, s, &d); + + if (d.up_viol && + !masked_irq(hw_id, d.up_irq_mask, tsens_version(priv))) { + tsens_set_interrupt(priv, hw_id, UPPER, disable); + if (d.up_thresh > temp) { + dev_dbg(priv->dev, "[%u] %s: re-arm upper\n", + hw_id, __func__); + tsens_set_interrupt(priv, hw_id, UPPER, enable); + } else { + trigger = true; + /* Keep irq masked */ + } + } else if (d.low_viol && + !masked_irq(hw_id, d.low_irq_mask, tsens_version(priv))) { + tsens_set_interrupt(priv, hw_id, LOWER, disable); + if (d.low_thresh < temp) { + dev_dbg(priv->dev, "[%u] %s: re-arm low\n", + hw_id, __func__); + tsens_set_interrupt(priv, hw_id, LOWER, enable); + } else { + trigger = true; + /* Keep irq masked */ + } + } + + spin_unlock_irqrestore(&priv->ul_lock, flags); + + if (trigger) { + dev_dbg(priv->dev, "[%u] %s: TZ update trigger (%d mC)\n", + hw_id, __func__, temp); + thermal_zone_device_update(s->tzd, + THERMAL_EVENT_UNSPECIFIED); + } else { + dev_dbg(priv->dev, "[%u] %s: no violation: %d\n", + hw_id, __func__, temp); + } + } + + return IRQ_HANDLED; +} + +int tsens_set_trips(void *_sensor, int low, int high) +{ + struct tsens_sensor *s = _sensor; + struct tsens_priv *priv = s->priv; + struct device *dev = priv->dev; + struct tsens_irq_data d; + unsigned long flags; + int high_val, low_val, cl_high, cl_low; + u32 hw_id = s->hw_id; + + dev_dbg(dev, "[%u] %s: proposed thresholds: (%d:%d)\n", + hw_id, __func__, low, high); + + cl_high = clamp_val(high, -40000, 120000); + cl_low = clamp_val(low, -40000, 120000); + + high_val = tsens_mC_to_hw(s, cl_high); + low_val = tsens_mC_to_hw(s, cl_low); + + spin_lock_irqsave(&priv->ul_lock, flags); + + tsens_read_irq_state(priv, hw_id, s, &d); + + /* Write the new thresholds and clear the status */ + regmap_field_write(priv->rf[LOW_THRESH_0 + hw_id], low_val); + regmap_field_write(priv->rf[UP_THRESH_0 + hw_id], high_val); + tsens_set_interrupt(priv, hw_id, LOWER, true); + tsens_set_interrupt(priv, hw_id, UPPER, true); + + spin_unlock_irqrestore(&priv->ul_lock, flags); + + dev_dbg(dev, "[%u] %s: (%d:%d)->(%d:%d)\n", + hw_id, __func__, d.low_thresh, d.up_thresh, cl_low, cl_high); + + return 0; +} + +int tsens_enable_irq(struct tsens_priv *priv) +{ + int ret; + int val = tsens_version(priv) > VER_1_X ? 7 : 1; + + ret = regmap_field_write(priv->rf[INT_EN], val); + if (ret < 0) + dev_err(priv->dev, "%s: failed to enable interrupts\n", + __func__); + + return ret; +} + +void tsens_disable_irq(struct tsens_priv *priv) +{ + regmap_field_write(priv->rf[INT_EN], 0); +} + +int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp) +{ + struct tsens_priv *priv = s->priv; + int hw_id = s->hw_id; + u32 temp_idx = LAST_TEMP_0 + hw_id; + u32 valid_idx = VALID_0 + hw_id; + u32 valid; + int ret; + + ret = regmap_field_read(priv->rf[valid_idx], &valid); + if (ret) + return ret; + while (!valid) { + /* Valid bit is 0 for 6 AHB clock cycles. + * At 19.2MHz, 1 AHB clock is ~60ns. + * We should enter this loop very, very rarely. + */ + ndelay(400); + ret = regmap_field_read(priv->rf[valid_idx], &valid); + if (ret) + return ret; + } + + /* Valid bit is set, OK to read the temperature */ + *temp = tsens_hw_to_mC(s, temp_idx); + + return 0; +} + +int get_temp_common(const struct tsens_sensor *s, int *temp) +{ + struct tsens_priv *priv = s->priv; + int hw_id = s->hw_id; + int last_temp = 0, ret; + + ret = regmap_field_read(priv->rf[LAST_TEMP_0 + hw_id], &last_temp); + if (ret) + return ret; + + *temp = code_to_degc(last_temp, s) * 1000; + + return 0; +} + +#ifdef CONFIG_DEBUG_FS +static int dbg_sensors_show(struct seq_file *s, void *data) +{ + struct platform_device *pdev = s->private; + struct tsens_priv *priv = platform_get_drvdata(pdev); + int i; + + seq_printf(s, "max: %2d\nnum: %2d\n\n", + priv->feat->max_sensors, priv->num_sensors); + + seq_puts(s, " id slope offset\n--------------------------\n"); + for (i = 0; i < priv->num_sensors; i++) { + seq_printf(s, "%8d %8d %8d\n", priv->sensor[i].hw_id, + priv->sensor[i].slope, priv->sensor[i].offset); + } + + return 0; +} + +static int dbg_version_show(struct seq_file *s, void *data) +{ + struct platform_device *pdev = s->private; + struct tsens_priv *priv = platform_get_drvdata(pdev); + u32 maj_ver, min_ver, step_ver; + int ret; + + if (tsens_version(priv) > VER_0_1) { + ret = regmap_field_read(priv->rf[VER_MAJOR], &maj_ver); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[VER_MINOR], &min_ver); + if (ret) + return ret; + ret = regmap_field_read(priv->rf[VER_STEP], &step_ver); + if (ret) + return ret; + seq_printf(s, "%d.%d.%d\n", maj_ver, min_ver, step_ver); + } else { + seq_puts(s, "0.1.0\n"); + } + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(dbg_version); +DEFINE_SHOW_ATTRIBUTE(dbg_sensors); + +static void tsens_debug_init(struct platform_device *pdev) +{ + struct tsens_priv *priv = platform_get_drvdata(pdev); + struct dentry *root, *file; + + root = debugfs_lookup("tsens", NULL); + if (!root) + priv->debug_root = debugfs_create_dir("tsens", NULL); + else + priv->debug_root = root; + + file = debugfs_lookup("version", priv->debug_root); + if (!file) + debugfs_create_file("version", 0444, priv->debug_root, + pdev, &dbg_version_fops); + + /* A directory for each instance of the TSENS IP */ + priv->debug = debugfs_create_dir(dev_name(&pdev->dev), priv->debug_root); + debugfs_create_file("sensors", 0444, priv->debug, pdev, &dbg_sensors_fops); +} +#else +static inline void tsens_debug_init(struct platform_device *pdev) {} +#endif + +static const struct regmap_config tsens_config = { + .name = "tm", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +static const struct regmap_config tsens_srot_config = { + .name = "srot", + .reg_bits = 32, + .val_bits = 32, + .reg_stride = 4, +}; + +int __init init_common(struct tsens_priv *priv) +{ + void __iomem *tm_base, *srot_base; + struct device *dev = priv->dev; + u32 ver_minor; + struct resource *res; + u32 enabled; + int ret, i, j; + struct platform_device *op = of_find_device_by_node(priv->dev->of_node); + + if (!op) + return -EINVAL; + + if (op->num_resources > 1) { + /* DT with separate SROT and TM address space */ + priv->tm_offset = 0; + res = platform_get_resource(op, IORESOURCE_MEM, 1); + srot_base = devm_ioremap_resource(dev, res); + if (IS_ERR(srot_base)) { + ret = PTR_ERR(srot_base); + goto err_put_device; + } + + priv->srot_map = devm_regmap_init_mmio(dev, srot_base, + &tsens_srot_config); + if (IS_ERR(priv->srot_map)) { + ret = PTR_ERR(priv->srot_map); + goto err_put_device; + } + } else { + /* old DTs where SROT and TM were in a contiguous 2K block */ + priv->tm_offset = 0x1000; + } + + res = platform_get_resource(op, IORESOURCE_MEM, 0); + tm_base = devm_ioremap_resource(dev, res); + if (IS_ERR(tm_base)) { + ret = PTR_ERR(tm_base); + goto err_put_device; + } + + priv->tm_map = devm_regmap_init_mmio(dev, tm_base, &tsens_config); + if (IS_ERR(priv->tm_map)) { + ret = PTR_ERR(priv->tm_map); + goto err_put_device; + } + + if (tsens_version(priv) > VER_0_1) { + for (i = VER_MAJOR; i <= VER_STEP; i++) { + priv->rf[i] = devm_regmap_field_alloc(dev, priv->srot_map, + priv->fields[i]); + if (IS_ERR(priv->rf[i])) + return PTR_ERR(priv->rf[i]); + } + ret = regmap_field_read(priv->rf[VER_MINOR], &ver_minor); + if (ret) + goto err_put_device; + } + + priv->rf[TSENS_EN] = devm_regmap_field_alloc(dev, priv->srot_map, + priv->fields[TSENS_EN]); + if (IS_ERR(priv->rf[TSENS_EN])) { + ret = PTR_ERR(priv->rf[TSENS_EN]); + goto err_put_device; + } + ret = regmap_field_read(priv->rf[TSENS_EN], &enabled); + if (ret) + goto err_put_device; + if (!enabled) { + dev_err(dev, "%s: device not enabled\n", __func__); + ret = -ENODEV; + goto err_put_device; + } + + priv->rf[SENSOR_EN] = devm_regmap_field_alloc(dev, priv->srot_map, + priv->fields[SENSOR_EN]); + if (IS_ERR(priv->rf[SENSOR_EN])) { + ret = PTR_ERR(priv->rf[SENSOR_EN]); + goto err_put_device; + } + priv->rf[INT_EN] = devm_regmap_field_alloc(dev, priv->tm_map, + priv->fields[INT_EN]); + if (IS_ERR(priv->rf[INT_EN])) { + ret = PTR_ERR(priv->rf[INT_EN]); + goto err_put_device; + } + + /* This loop might need changes if enum regfield_ids is reordered */ + for (j = LAST_TEMP_0; j <= UP_THRESH_15; j += 16) { + for (i = 0; i < priv->feat->max_sensors; i++) { + int idx = j + i; + + priv->rf[idx] = devm_regmap_field_alloc(dev, + priv->tm_map, + priv->fields[idx]); + if (IS_ERR(priv->rf[idx])) { + ret = PTR_ERR(priv->rf[idx]); + goto err_put_device; + } + } + } + + if (priv->feat->crit_int) { + /* Loop might need changes if enum regfield_ids is reordered */ + for (j = CRITICAL_STATUS_0; j <= CRIT_THRESH_15; j += 16) { + for (i = 0; i < priv->feat->max_sensors; i++) { + int idx = j + i; + + priv->rf[idx] = + devm_regmap_field_alloc(dev, + priv->tm_map, + priv->fields[idx]); + if (IS_ERR(priv->rf[idx])) { + ret = PTR_ERR(priv->rf[idx]); + goto err_put_device; + } + } + } + } + + if (tsens_version(priv) > VER_1_X && ver_minor > 2) { + /* Watchdog is present only on v2.3+ */ + priv->feat->has_watchdog = 1; + for (i = WDOG_BARK_STATUS; i <= CC_MON_MASK; i++) { + priv->rf[i] = devm_regmap_field_alloc(dev, priv->tm_map, + priv->fields[i]); + if (IS_ERR(priv->rf[i])) { + ret = PTR_ERR(priv->rf[i]); + goto err_put_device; + } + } + /* + * Watchdog is already enabled, unmask the bark. + * Disable cycle completion monitoring + */ + regmap_field_write(priv->rf[WDOG_BARK_MASK], 0); + regmap_field_write(priv->rf[CC_MON_MASK], 1); + } + + spin_lock_init(&priv->ul_lock); + tsens_enable_irq(priv); + tsens_debug_init(op); + +err_put_device: + put_device(&op->dev); + return ret; +} + static int tsens_get_temp(void *data, int *temp) { struct tsens_sensor *s = data; diff --git a/drivers/thermal/qcom/tsens.h b/drivers/thermal/qcom/tsens.h index 502acf0e6828..59d01162c66a 100644 --- a/drivers/thermal/qcom/tsens.h +++ b/drivers/thermal/qcom/tsens.h @@ -580,11 +580,6 @@ void compute_intercept_slope(struct tsens_priv *priv, u32 *pt1, u32 *pt2, u32 mo int init_common(struct tsens_priv *priv); int get_temp_tsens_valid(const struct tsens_sensor *s, int *temp); int get_temp_common(const struct tsens_sensor *s, int *temp); -int tsens_enable_irq(struct tsens_priv *priv); -void tsens_disable_irq(struct tsens_priv *priv); -int tsens_set_trips(void *_sensor, int low, int high); -irqreturn_t tsens_irq_thread(int irq, void *data); -irqreturn_t tsens_critical_irq_thread(int irq, void *data); /* TSENS target */ extern struct tsens_plat_data data_8960; diff --git a/drivers/thermal/qoriq_thermal.c b/drivers/thermal/qoriq_thermal.c index 028a6bbf75dc..73049f9bea25 100644 --- a/drivers/thermal/qoriq_thermal.c +++ b/drivers/thermal/qoriq_thermal.c @@ -11,6 +11,7 @@ #include <linux/regmap.h> #include <linux/sizes.h> #include <linux/thermal.h> +#include <linux/units.h> #include "thermal_core.h" #include "thermal_hwmon.h" @@ -23,6 +24,7 @@ #define TMTMIR_DEFAULT 0x0000000f #define TIER_DISABLE 0x0 #define TEUMR0_V2 0x51009c00 +#define TMSARA_V2 0xe #define TMU_VER1 0x1 #define TMU_VER2 0x2 @@ -50,6 +52,9 @@ * Site Register */ #define TRITSR_V BIT(31) +#define REGS_V2_TMSAR(n) (0x304 + 16 * (n)) /* TMU monitoring + * site adjustment register + */ #define REGS_TTRnCR(n) (0xf10 + 4 * (n)) /* Temperature Range n * Control Register */ @@ -85,12 +90,21 @@ static int tmu_get_temp(void *p, int *temp) /* * REGS_TRITSR(id) has the following layout: * + * For TMU Rev1: * 31 ... 7 6 5 4 3 2 1 0 * V TEMP * * Where V bit signifies if the measurement is ready and is * within sensor range. TEMP is an 8 bit value representing - * temperature in C. + * temperature in Celsius. + + * For TMU Rev2: + * 31 ... 8 7 6 5 4 3 2 1 0 + * V TEMP + * + * Where V bit signifies if the measurement is ready and is + * within sensor range. TEMP is an 9 bit value representing + * temperature in KelVin. */ if (regmap_read_poll_timeout(qdata->regmap, REGS_TRITSR(qsensor->id), @@ -100,7 +114,10 @@ static int tmu_get_temp(void *p, int *temp) 10 * USEC_PER_MSEC)) return -ENODATA; - *temp = (val & 0xff) * 1000; + if (qdata->ver == TMU_VER1) + *temp = (val & GENMASK(7, 0)) * MILLIDEGREE_PER_DEGREE; + else + *temp = kelvin_to_millicelsius(val & GENMASK(8, 0)); return 0; } @@ -192,6 +209,8 @@ static int qoriq_tmu_calibration(struct device *dev, static void qoriq_tmu_init_device(struct qoriq_tmu_data *data) { + int i; + /* Disable interrupt, using polling instead */ regmap_write(data->regmap, REGS_TIER, TIER_DISABLE); @@ -202,6 +221,8 @@ static void qoriq_tmu_init_device(struct qoriq_tmu_data *data) } else { regmap_write(data->regmap, REGS_V2_TMTMIR, TMTMIR_DEFAULT); regmap_write(data->regmap, REGS_V2_TEUMR(0), TEUMR0_V2); + for (i = 0; i < SITES_MAX; i++) + regmap_write(data->regmap, REGS_V2_TMSAR(i), TMSARA_V2); } /* Disable monitoring */ @@ -212,6 +233,7 @@ static const struct regmap_range qoriq_yes_ranges[] = { regmap_reg_range(REGS_TMR, REGS_TSCFGR), regmap_reg_range(REGS_TTRnCR(0), REGS_TTRnCR(3)), regmap_reg_range(REGS_V2_TEUMR(0), REGS_V2_TEUMR(2)), + regmap_reg_range(REGS_V2_TMSAR(0), REGS_V2_TMSAR(15)), regmap_reg_range(REGS_IPBRR(0), REGS_IPBRR(1)), /* Read only registers below */ regmap_reg_range(REGS_TRITSR(0), REGS_TRITSR(15)), diff --git a/drivers/thermal/rcar_thermal.c b/drivers/thermal/rcar_thermal.c index e0c1f2409035..46aeb28b4e90 100644 --- a/drivers/thermal/rcar_thermal.c +++ b/drivers/thermal/rcar_thermal.c @@ -198,8 +198,8 @@ static void _rcar_thermal_bset(struct rcar_thermal_priv *priv, u32 reg, static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) { struct device *dev = rcar_priv_to_dev(priv); - int i; - u32 ctemp, old, new; + int old, new, ctemp = -EINVAL; + unsigned int i; mutex_lock(&priv->lock); @@ -209,7 +209,6 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) */ rcar_thermal_bset(priv, THSCR, CPCTL, CPCTL); - ctemp = 0; old = ~0; for (i = 0; i < 128; i++) { /* @@ -227,7 +226,7 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) old = new; } - if (!ctemp) { + if (ctemp < 0) { dev_err(dev, "thermal sensor was broken\n"); goto err_out_unlock; } @@ -248,7 +247,7 @@ static int rcar_thermal_update_temp(struct rcar_thermal_priv *priv) err_out_unlock: mutex_unlock(&priv->lock); - return ctemp ? ctemp : -EINVAL; + return ctemp; } static int rcar_thermal_get_current_temp(struct rcar_thermal_priv *priv, diff --git a/drivers/thermal/rockchip_thermal.c b/drivers/thermal/rockchip_thermal.c index 7c1a8bccdcba..15a71ecc916c 100644 --- a/drivers/thermal/rockchip_thermal.c +++ b/drivers/thermal/rockchip_thermal.c @@ -1241,10 +1241,8 @@ static int rockchip_thermal_probe(struct platform_device *pdev) return -ENXIO; irq = platform_get_irq(pdev, 0); - if (irq < 0) { - dev_err(&pdev->dev, "no irq resource?\n"); + if (irq < 0) return -EINVAL; - } thermal = devm_kzalloc(&pdev->dev, sizeof(struct rockchip_thermal_data), GFP_KERNEL); diff --git a/drivers/thermal/st/st_thermal_memmap.c b/drivers/thermal/st/st_thermal_memmap.c index a824b78dabf8..a0114452d11f 100644 --- a/drivers/thermal/st/st_thermal_memmap.c +++ b/drivers/thermal/st/st_thermal_memmap.c @@ -94,10 +94,8 @@ static int st_mmap_register_enable_irq(struct st_thermal_sensor *sensor) int ret; sensor->irq = platform_get_irq(pdev, 0); - if (sensor->irq < 0) { - dev_err(dev, "failed to register IRQ\n"); + if (sensor->irq < 0) return sensor->irq; - } ret = devm_request_threaded_irq(dev, sensor->irq, NULL, st_mmap_thermal_trip_handler, diff --git a/drivers/thermal/st/stm_thermal.c b/drivers/thermal/st/stm_thermal.c index 9314e3df6a42..331e2b768df5 100644 --- a/drivers/thermal/st/stm_thermal.c +++ b/drivers/thermal/st/stm_thermal.c @@ -385,10 +385,8 @@ static int stm_register_irq(struct stm_thermal_sensor *sensor) int ret; sensor->irq = platform_get_irq(pdev, 0); - if (sensor->irq < 0) { - dev_err(dev, "%s: Unable to find IRQ\n", __func__); + if (sensor->irq < 0) return sensor->irq; - } ret = devm_request_threaded_irq(dev, sensor->irq, NULL, diff --git a/drivers/thermal/thermal_core.c b/drivers/thermal/thermal_core.c index 9a321dc548c8..b71196eaf90e 100644 --- a/drivers/thermal/thermal_core.c +++ b/drivers/thermal/thermal_core.c @@ -9,9 +9,9 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/module.h> #include <linux/device.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/kdev_t.h> #include <linux/idr.h> @@ -27,10 +27,6 @@ #include "thermal_core.h" #include "thermal_hwmon.h" -MODULE_AUTHOR("Zhang Rui"); -MODULE_DESCRIPTION("Generic thermal management sysfs support"); -MODULE_LICENSE("GPL v2"); - static DEFINE_IDA(thermal_tz_ida); static DEFINE_IDA(thermal_cdev_ida); @@ -447,12 +443,6 @@ static void update_temperature(struct thermal_zone_device *tz) mutex_unlock(&tz->lock); trace_thermal_temperature(tz); - if (tz->last_temperature == THERMAL_TEMP_INVALID) - dev_dbg(&tz->device, "last_temperature N/A, current_temperature=%d\n", - tz->temperature); - else - dev_dbg(&tz->device, "last_temperature=%d, current_temperature=%d\n", - tz->last_temperature, tz->temperature); } static void thermal_zone_device_init(struct thermal_zone_device *tz) diff --git a/drivers/thermal/thermal_core.h b/drivers/thermal/thermal_core.h index a9bf00e91d64..c95689586e19 100644 --- a/drivers/thermal/thermal_core.h +++ b/drivers/thermal/thermal_core.h @@ -12,6 +12,17 @@ #include <linux/device.h> #include <linux/thermal.h> +/* Default Thermal Governor */ +#if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE) +#define DEFAULT_THERMAL_GOVERNOR "step_wise" +#elif defined(CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE) +#define DEFAULT_THERMAL_GOVERNOR "fair_share" +#elif defined(CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE) +#define DEFAULT_THERMAL_GOVERNOR "user_space" +#elif defined(CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR) +#define DEFAULT_THERMAL_GOVERNOR "power_allocator" +#endif + /* Initial state of a cooling device during binding */ #define THERMAL_NO_TARGET -1UL @@ -30,6 +41,44 @@ extern struct thermal_governor *__governor_thermal_table_end[]; __governor < __governor_thermal_table_end; \ __governor++) +struct thermal_attr { + struct device_attribute attr; + char name[THERMAL_NAME_LENGTH]; +}; + +static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) +{ + return cdev->ops->get_requested_power && cdev->ops->state2power && + cdev->ops->power2state; +} + +int power_actor_get_max_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *max_power); +int power_actor_get_min_power(struct thermal_cooling_device *cdev, + struct thermal_zone_device *tz, u32 *min_power); +int power_actor_set_power(struct thermal_cooling_device *cdev, + struct thermal_instance *ti, u32 power); +/** + * struct thermal_trip - representation of a point in temperature domain + * @np: pointer to struct device_node that this trip point was created from + * @temperature: temperature value in miliCelsius + * @hysteresis: relative hysteresis in miliCelsius + * @type: trip point type + */ +struct thermal_trip { + struct device_node *np; + int temperature; + int hysteresis; + enum thermal_trip_type type; +}; + +int get_tz_trend(struct thermal_zone_device *tz, int trip); + +struct thermal_instance * +get_thermal_instance(struct thermal_zone_device *tz, + struct thermal_cooling_device *cdev, + int trip); + /* * This structure is used to describe the behavior of * a certain cooling device on a certain trip point @@ -69,6 +118,9 @@ void thermal_zone_device_unbind_exception(struct thermal_zone_device *, int thermal_zone_device_set_policy(struct thermal_zone_device *, char *); int thermal_build_list_of_policies(char *buf); +/* Helpers */ +void thermal_zone_set_trips(struct thermal_zone_device *tz); + /* sysfs I/F */ int thermal_zone_create_device_groups(struct thermal_zone_device *, int); void thermal_zone_destroy_device_groups(struct thermal_zone_device *); diff --git a/drivers/thermal/thermal_helpers.c b/drivers/thermal/thermal_helpers.c index 2ba756af76b7..87b1256fa2f2 100644 --- a/drivers/thermal/thermal_helpers.c +++ b/drivers/thermal/thermal_helpers.c @@ -12,11 +12,12 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/sysfs.h> #include <linux/device.h> #include <linux/err.h> +#include <linux/export.h> #include <linux/slab.h> #include <linux/string.h> +#include <linux/sysfs.h> #include <trace/events/thermal.h> @@ -113,6 +114,18 @@ exit: } EXPORT_SYMBOL_GPL(thermal_zone_get_temp); +/** + * thermal_zone_set_trips - Computes the next trip points for the driver + * @tz: a pointer to a thermal zone device structure + * + * The function computes the next temperature boundaries by browsing + * the trip points. The result is the closer low and high trip points + * to the current temperature. These values are passed to the backend + * driver to let it set its own notification mechanism (usually an + * interrupt). + * + * It does not return a value + */ void thermal_zone_set_trips(struct thermal_zone_device *tz) { int low = -INT_MAX; @@ -161,7 +174,6 @@ void thermal_zone_set_trips(struct thermal_zone_device *tz) exit: mutex_unlock(&tz->lock); } -EXPORT_SYMBOL_GPL(thermal_zone_set_trips); void thermal_cdev_update(struct thermal_cooling_device *cdev) { diff --git a/drivers/thermal/thermal_hwmon.c b/drivers/thermal/thermal_hwmon.c index c8d2620f2e42..8b92e00ff236 100644 --- a/drivers/thermal/thermal_hwmon.c +++ b/drivers/thermal/thermal_hwmon.c @@ -10,10 +10,12 @@ * Copyright (C) 2013 Texas Instruments * Copyright (C) 2013 Eduardo Valentin <eduardo.valentin@ti.com> */ +#include <linux/err.h> +#include <linux/export.h> #include <linux/hwmon.h> -#include <linux/thermal.h> #include <linux/slab.h> -#include <linux/err.h> +#include <linux/thermal.h> + #include "thermal_hwmon.h" /* hwmon sys I/F */ diff --git a/drivers/thermal/of-thermal.c b/drivers/thermal/thermal_of.c index 874a47d6923f..ddf88dbe7ba2 100644 --- a/drivers/thermal/of-thermal.c +++ b/drivers/thermal/thermal_of.c @@ -8,13 +8,13 @@ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt -#include <linux/thermal.h> -#include <linux/slab.h> -#include <linux/types.h> -#include <linux/of_device.h> -#include <linux/of_platform.h> #include <linux/err.h> #include <linux/export.h> +#include <linux/of_device.h> +#include <linux/of_platform.h> +#include <linux/slab.h> +#include <linux/thermal.h> +#include <linux/types.h> #include <linux/string.h> #include "thermal_core.h" diff --git a/drivers/thermal/ti-soc-thermal/ti-bandgap.c b/drivers/thermal/ti-soc-thermal/ti-bandgap.c index 263b0420fbe4..ab19ceff6e2a 100644 --- a/drivers/thermal/ti-soc-thermal/ti-bandgap.c +++ b/drivers/thermal/ti-soc-thermal/ti-bandgap.c @@ -772,10 +772,9 @@ static int ti_bandgap_talert_init(struct ti_bandgap *bgp, int ret; bgp->irq = platform_get_irq(pdev, 0); - if (bgp->irq < 0) { - dev_err(&pdev->dev, "get_irq failed\n"); + if (bgp->irq < 0) return bgp->irq; - } + ret = request_threaded_irq(bgp->irq, NULL, ti_bandgap_talert_irq_handler, IRQF_TRIGGER_HIGH | IRQF_ONESHOT, diff --git a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c index d3e959d01606..85776db4bf34 100644 --- a/drivers/thermal/ti-soc-thermal/ti-thermal-common.c +++ b/drivers/thermal/ti-soc-thermal/ti-thermal-common.c @@ -169,7 +169,7 @@ int ti_thermal_expose_sensor(struct ti_bandgap *bgp, int id, data = ti_bandgap_get_sensor_data(bgp, id); - if (!data || IS_ERR(data)) + if (!IS_ERR_OR_NULL(data)) data = ti_thermal_build_data(bgp, id); if (!data) @@ -196,7 +196,7 @@ int ti_thermal_remove_sensor(struct ti_bandgap *bgp, int id) data = ti_bandgap_get_sensor_data(bgp, id); - if (data && data->ti_thermal) { + if (!IS_ERR_OR_NULL(data) && data->ti_thermal) { if (data->our_zone) thermal_zone_device_unregister(data->ti_thermal); } @@ -262,7 +262,7 @@ int ti_thermal_unregister_cpu_cooling(struct ti_bandgap *bgp, int id) data = ti_bandgap_get_sensor_data(bgp, id); - if (data) { + if (!IS_ERR_OR_NULL(data)) { cpufreq_cooling_unregister(data->cool_dev); if (data->policy) cpufreq_cpu_put(data->policy); diff --git a/drivers/usb/gadget/function/f_fs.c b/drivers/usb/gadget/function/f_fs.c index 494f853f2206..490d353d5fde 100644 --- a/drivers/usb/gadget/function/f_fs.c +++ b/drivers/usb/gadget/function/f_fs.c @@ -32,7 +32,7 @@ #include <linux/usb/functionfs.h> #include <linux/aio.h> -#include <linux/mmu_context.h> +#include <linux/kthread.h> #include <linux/poll.h> #include <linux/eventfd.h> @@ -824,13 +824,9 @@ static void ffs_user_copy_worker(struct work_struct *work) bool kiocb_has_eventfd = io_data->kiocb->ki_flags & IOCB_EVENTFD; if (io_data->read && ret > 0) { - mm_segment_t oldfs = get_fs(); - - set_fs(USER_DS); - use_mm(io_data->mm); + kthread_use_mm(io_data->mm); ret = ffs_copy_to_iter(io_data->buf, ret, &io_data->data); - unuse_mm(io_data->mm); - set_fs(oldfs); + kthread_unuse_mm(io_data->mm); } io_data->kiocb->ki_complete(io_data->kiocb, ret, ret); diff --git a/drivers/usb/gadget/function/f_tcm.c b/drivers/usb/gadget/function/f_tcm.c index 2979cbe4d95f..eaf556ceac32 100644 --- a/drivers/usb/gadget/function/f_tcm.c +++ b/drivers/usb/gadget/function/f_tcm.c @@ -1052,7 +1052,8 @@ static void usbg_cmd_work(struct work_struct *work) transport_init_se_cmd(se_cmd, tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo, tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE, - cmd->prio_attr, cmd->sense_iu.sense); + cmd->prio_attr, cmd->sense_iu.sense, + cmd->unpacked_lun); goto out; } @@ -1182,7 +1183,8 @@ static void bot_cmd_work(struct work_struct *work) transport_init_se_cmd(se_cmd, tv_nexus->tvn_se_sess->se_tpg->se_tpg_tfo, tv_nexus->tvn_se_sess, cmd->data_len, DMA_NONE, - cmd->prio_attr, cmd->sense_iu.sense); + cmd->prio_attr, cmd->sense_iu.sense, + cmd->unpacked_lun); goto out; } diff --git a/drivers/usb/gadget/legacy/inode.c b/drivers/usb/gadget/legacy/inode.c index 3afddd3bea6e..9ee0bfe7bcda 100644 --- a/drivers/usb/gadget/legacy/inode.c +++ b/drivers/usb/gadget/legacy/inode.c @@ -21,7 +21,7 @@ #include <linux/sched.h> #include <linux/slab.h> #include <linux/poll.h> -#include <linux/mmu_context.h> +#include <linux/kthread.h> #include <linux/aio.h> #include <linux/uio.h> #include <linux/refcount.h> @@ -462,9 +462,9 @@ static void ep_user_copy_worker(struct work_struct *work) struct kiocb *iocb = priv->iocb; size_t ret; - use_mm(mm); + kthread_use_mm(mm); ret = copy_to_iter(priv->buf, priv->actual, &priv->to); - unuse_mm(mm); + kthread_unuse_mm(mm); if (!ret) ret = -EFAULT; diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 186acd8675ff..5e556ac9102a 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -27,7 +27,7 @@ #include <linux/iommu.h> #include <linux/module.h> #include <linux/mm.h> -#include <linux/mmu_context.h> +#include <linux/kthread.h> #include <linux/rbtree.h> #include <linux/sched/signal.h> #include <linux/sched/mm.h> @@ -2817,7 +2817,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, return -EPERM; if (kthread) - use_mm(mm); + kthread_use_mm(mm); else if (current->mm != mm) goto out; @@ -2844,7 +2844,7 @@ static int vfio_iommu_type1_dma_rw_chunk(struct vfio_iommu *iommu, *copied = copy_from_user(data, (void __user *)vaddr, count) ? 0 : count; if (kthread) - unuse_mm(mm); + kthread_unuse_mm(mm); out: mmput(mm); return *copied ? 0 : -EFAULT; diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index 062595ee1f83..d7b8df3edffc 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -14,7 +14,6 @@ #include <linux/vhost.h> #include <linux/uio.h> #include <linux/mm.h> -#include <linux/mmu_context.h> #include <linux/miscdevice.h> #include <linux/mutex.h> #include <linux/poll.h> @@ -335,10 +334,8 @@ static int vhost_worker(void *data) struct vhost_dev *dev = data; struct vhost_work *work, *work_next; struct llist_node *node; - mm_segment_t oldfs = get_fs(); - set_fs(USER_DS); - use_mm(dev->mm); + kthread_use_mm(dev->mm); for (;;) { /* mb paired w/ kthread_stop */ @@ -366,8 +363,7 @@ static int vhost_worker(void *data) schedule(); } } - unuse_mm(dev->mm); - set_fs(oldfs); + kthread_unuse_mm(dev->mm); return 0; } diff --git a/drivers/xen/Kconfig b/drivers/xen/Kconfig index 61212fc7f0c7..727f11eb46b2 100644 --- a/drivers/xen/Kconfig +++ b/drivers/xen/Kconfig @@ -13,12 +13,16 @@ config XEN_BALLOON config XEN_BALLOON_MEMORY_HOTPLUG bool "Memory hotplug support for Xen balloon driver" depends on XEN_BALLOON && MEMORY_HOTPLUG + default y help Memory hotplug support for Xen balloon driver allows expanding memory available for the system above limit declared at system startup. It is very useful on critical systems which require long run without rebooting. + It's also very useful for non PV domains to obtain unpopulated physical + memory ranges to use in order to map foreign memory or grants. + Memory could be hotplugged in following steps: 1) target domain: ensure that memory auto online policy is in diff --git a/drivers/xen/Makefile b/drivers/xen/Makefile index 0c4efa6fe450..0d322f3d90cd 100644 --- a/drivers/xen/Makefile +++ b/drivers/xen/Makefile @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_HOTPLUG_CPU) += cpu_hotplug.o -obj-y += grant-table.o features.o balloon.o manage.o preempt.o time.o +obj-y += grant-table.o features.o balloon.o manage.o time.o obj-y += mem-reservation.o obj-y += events/ obj-y += xenbus/ diff --git a/drivers/xen/cpu_hotplug.c b/drivers/xen/cpu_hotplug.c index ec975decb5de..b96b11e2b571 100644 --- a/drivers/xen/cpu_hotplug.c +++ b/drivers/xen/cpu_hotplug.c @@ -93,10 +93,8 @@ static int setup_cpu_watcher(struct notifier_block *notifier, (void)register_xenbus_watch(&cpu_watch); for_each_possible_cpu(cpu) { - if (vcpu_online(cpu) == 0) { - device_offline(get_cpu_device(cpu)); - set_cpu_present(cpu, false); - } + if (vcpu_online(cpu) == 0) + disable_hotplug_cpu(cpu); } return NOTIFY_DONE; @@ -119,5 +117,5 @@ static int __init setup_vcpu_hotplug_event(void) return 0; } -arch_initcall(setup_vcpu_hotplug_event); +late_initcall(setup_vcpu_hotplug_event); diff --git a/drivers/xen/events/events_base.c b/drivers/xen/events/events_base.c index 3a791c8485d0..140c7bf33a98 100644 --- a/drivers/xen/events/events_base.c +++ b/drivers/xen/events/events_base.c @@ -37,6 +37,7 @@ #ifdef CONFIG_X86 #include <asm/desc.h> #include <asm/ptrace.h> +#include <asm/idtentry.h> #include <asm/irq.h> #include <asm/io_apic.h> #include <asm/i8259.h> @@ -1236,9 +1237,6 @@ void xen_evtchn_do_upcall(struct pt_regs *regs) struct pt_regs *old_regs = set_irq_regs(regs); irq_enter(); -#ifdef CONFIG_X86 - inc_irq_stat(irq_hv_callback_count); -#endif __xen_evtchn_do_upcall(); @@ -1639,26 +1637,30 @@ EXPORT_SYMBOL_GPL(xen_set_callback_via); /* Vector callbacks are better than PCI interrupts to receive event * channel notifications because we can receive vector callbacks on any * vcpu and we don't need PCI support or APIC interactions. */ -void xen_callback_vector(void) +void xen_setup_callback_vector(void) { - int rc; uint64_t callback_via; if (xen_have_vector_callback) { callback_via = HVM_CALLBACK_VECTOR(HYPERVISOR_CALLBACK_VECTOR); - rc = xen_set_callback_via(callback_via); - if (rc) { + if (xen_set_callback_via(callback_via)) { pr_err("Request for Xen HVM callback vector failed\n"); xen_have_vector_callback = 0; - return; } - pr_info_once("Xen HVM callback vector for event delivery is enabled\n"); - alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, - xen_hvm_callback_vector); } } + +static __init void xen_alloc_callback_vector(void) +{ + if (!xen_have_vector_callback) + return; + + pr_info("Xen HVM callback vector for event delivery is enabled\n"); + alloc_intr_gate(HYPERVISOR_CALLBACK_VECTOR, asm_sysvec_xen_hvm_callback); +} #else -void xen_callback_vector(void) {} +void xen_setup_callback_vector(void) {} +static inline void xen_alloc_callback_vector(void) {} #endif #undef MODULE_PARAM_PREFIX @@ -1692,8 +1694,10 @@ void __init xen_init_IRQ(void) if (xen_initial_domain()) pci_xen_initial_domain(); } - if (xen_feature(XENFEAT_hvm_callback_vector)) - xen_callback_vector(); + if (xen_feature(XENFEAT_hvm_callback_vector)) { + xen_setup_callback_vector(); + xen_alloc_callback_vector(); + } if (xen_hvm_domain()) { native_init_IRQ(); diff --git a/drivers/xen/platform-pci.c b/drivers/xen/platform-pci.c index 59e85e408c23..dd911e1ff782 100644 --- a/drivers/xen/platform-pci.c +++ b/drivers/xen/platform-pci.c @@ -168,7 +168,7 @@ static const struct pci_device_id platform_pci_tbl[] = { {0,} }; -static struct dev_pm_ops platform_pm_ops = { +static const struct dev_pm_ops platform_pm_ops = { .resume_noirq = platform_pci_resume, }; diff --git a/drivers/xen/preempt.c b/drivers/xen/preempt.c deleted file mode 100644 index 17240c5325a3..000000000000 --- a/drivers/xen/preempt.c +++ /dev/null @@ -1,42 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Preemptible hypercalls - * - * Copyright (C) 2014 Citrix Systems R&D ltd. - */ - -#include <linux/sched.h> -#include <xen/xen-ops.h> - -#ifndef CONFIG_PREEMPTION - -/* - * Some hypercalls issued by the toolstack can take many 10s of - * seconds. Allow tasks running hypercalls via the privcmd driver to - * be voluntarily preempted even if full kernel preemption is - * disabled. - * - * Such preemptible hypercalls are bracketed by - * xen_preemptible_hcall_begin() and xen_preemptible_hcall_end() - * calls. - */ - -DEFINE_PER_CPU(bool, xen_in_preemptible_hcall); -EXPORT_SYMBOL_GPL(xen_in_preemptible_hcall); - -asmlinkage __visible void xen_maybe_preempt_hcall(void) -{ - if (unlikely(__this_cpu_read(xen_in_preemptible_hcall) - && need_resched())) { - /* - * Clear flag as we may be rescheduled on a different - * cpu. - */ - __this_cpu_write(xen_in_preemptible_hcall, false); - local_irq_enable(); - cond_resched(); - local_irq_disable(); - __this_cpu_write(xen_in_preemptible_hcall, true); - } -} -#endif /* CONFIG_PREEMPTION */ diff --git a/drivers/xen/pvcalls-back.c b/drivers/xen/pvcalls-back.c index cf4ce3e9358d..9eae1fceec1e 100644 --- a/drivers/xen/pvcalls-back.c +++ b/drivers/xen/pvcalls-back.c @@ -24,7 +24,7 @@ #define PVCALLS_VERSIONS "1" #define MAX_RING_ORDER XENBUS_MAX_RING_GRANT_ORDER -struct pvcalls_back_global { +static struct pvcalls_back_global { struct list_head frontends; struct semaphore frontends_lock; } pvcalls_back_global; @@ -1088,7 +1088,8 @@ static void set_backend_state(struct xenbus_device *dev, case XenbusStateInitialised: switch (state) { case XenbusStateConnected: - backend_connect(dev); + if (backend_connect(dev)) + return; xenbus_switch_state(dev, XenbusStateConnected); break; case XenbusStateClosing: diff --git a/drivers/xen/xen-pciback/conf_space.c b/drivers/xen/xen-pciback/conf_space.c index da51a5d34e6e..059de92aea7d 100644 --- a/drivers/xen/xen-pciback/conf_space.c +++ b/drivers/xen/xen-pciback/conf_space.c @@ -10,6 +10,8 @@ * Author: Ryan Wilson <hap9@epoch.ncsc.mil> */ +#define dev_fmt(fmt) DRV_NAME ": " fmt + #include <linux/kernel.h> #include <linux/moduleparam.h> #include <linux/pci.h> @@ -154,9 +156,7 @@ int xen_pcibk_config_read(struct pci_dev *dev, int offset, int size, * (as if device didn't respond) */ u32 value = 0, tmp_val; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: read %d bytes at 0x%x\n", - pci_name(dev), size, offset); + dev_dbg(&dev->dev, "read %d bytes at 0x%x\n", size, offset); if (!valid_request(offset, size)) { err = XEN_PCI_ERR_invalid_offset; @@ -195,9 +195,7 @@ int xen_pcibk_config_read(struct pci_dev *dev, int offset, int size, } out: - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: read %d bytes at 0x%x = %x\n", - pci_name(dev), size, offset, value); + dev_dbg(&dev->dev, "read %d bytes at 0x%x = %x\n", size, offset, value); *ret_val = value; return xen_pcibios_err_to_errno(err); @@ -212,10 +210,8 @@ int xen_pcibk_config_write(struct pci_dev *dev, int offset, int size, u32 value) u32 tmp_val; int field_start, field_end; - if (unlikely(verbose_request)) - printk(KERN_DEBUG - DRV_NAME ": %s: write request %d bytes at 0x%x = %x\n", - pci_name(dev), size, offset, value); + dev_dbg(&dev->dev, "write request %d bytes at 0x%x = %x\n", + size, offset, value); if (!valid_request(offset, size)) return XEN_PCI_ERR_invalid_offset; diff --git a/drivers/xen/xen-pciback/conf_space_header.c b/drivers/xen/xen-pciback/conf_space_header.c index fb4fccb4aecc..ac45cdc38e85 100644 --- a/drivers/xen/xen-pciback/conf_space_header.c +++ b/drivers/xen/xen-pciback/conf_space_header.c @@ -6,6 +6,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define dev_fmt pr_fmt #include <linux/kernel.h> #include <linux/pci.h> @@ -67,53 +68,39 @@ static int command_write(struct pci_dev *dev, int offset, u16 value, void *data) dev_data = pci_get_drvdata(dev); if (!pci_is_enabled(dev) && is_enable_cmd(value)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: enable\n", - pci_name(dev)); + dev_dbg(&dev->dev, "enable\n"); err = pci_enable_device(dev); if (err) return err; if (dev_data) dev_data->enable_intx = 1; } else if (pci_is_enabled(dev) && !is_enable_cmd(value)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: disable\n", - pci_name(dev)); + dev_dbg(&dev->dev, "disable\n"); pci_disable_device(dev); if (dev_data) dev_data->enable_intx = 0; } if (!dev->is_busmaster && is_master_cmd(value)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: set bus master\n", - pci_name(dev)); + dev_dbg(&dev->dev, "set bus master\n"); pci_set_master(dev); } else if (dev->is_busmaster && !is_master_cmd(value)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: clear bus master\n", - pci_name(dev)); + dev_dbg(&dev->dev, "clear bus master\n"); pci_clear_master(dev); } if (!(cmd->val & PCI_COMMAND_INVALIDATE) && (value & PCI_COMMAND_INVALIDATE)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG - DRV_NAME ": %s: enable memory-write-invalidate\n", - pci_name(dev)); + dev_dbg(&dev->dev, "enable memory-write-invalidate\n"); err = pci_set_mwi(dev); if (err) { - pr_warn("%s: cannot enable memory-write-invalidate (%d)\n", - pci_name(dev), err); + dev_warn(&dev->dev, "cannot enable memory-write-invalidate (%d)\n", + err); value &= ~PCI_COMMAND_INVALIDATE; } } else if ((cmd->val & PCI_COMMAND_INVALIDATE) && !(value & PCI_COMMAND_INVALIDATE)) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG - DRV_NAME ": %s: disable memory-write-invalidate\n", - pci_name(dev)); + dev_dbg(&dev->dev, "disable memory-write-invalidate\n"); pci_clear_mwi(dev); } @@ -157,8 +144,7 @@ static int rom_write(struct pci_dev *dev, int offset, u32 value, void *data) struct pci_bar_info *bar = data; if (unlikely(!bar)) { - pr_warn(DRV_NAME ": driver data not found for %s\n", - pci_name(dev)); + dev_warn(&dev->dev, "driver data not found\n"); return XEN_PCI_ERR_op_failed; } @@ -194,8 +180,7 @@ static int bar_write(struct pci_dev *dev, int offset, u32 value, void *data) u32 mask; if (unlikely(!bar)) { - pr_warn(DRV_NAME ": driver data not found for %s\n", - pci_name(dev)); + dev_warn(&dev->dev, "driver data not found\n"); return XEN_PCI_ERR_op_failed; } @@ -228,8 +213,7 @@ static int bar_read(struct pci_dev *dev, int offset, u32 * value, void *data) struct pci_bar_info *bar = data; if (unlikely(!bar)) { - pr_warn(DRV_NAME ": driver data not found for %s\n", - pci_name(dev)); + dev_warn(&dev->dev, "driver data not found\n"); return XEN_PCI_ERR_op_failed; } @@ -433,8 +417,8 @@ int xen_pcibk_config_header_add_fields(struct pci_dev *dev) default: err = -EINVAL; - pr_err("%s: Unsupported header type %d!\n", - pci_name(dev), dev->hdr_type); + dev_err(&dev->dev, "Unsupported header type %d!\n", + dev->hdr_type); break; } diff --git a/drivers/xen/xen-pciback/conf_space_quirks.c b/drivers/xen/xen-pciback/conf_space_quirks.c index ed593d1042a6..7dc281086302 100644 --- a/drivers/xen/xen-pciback/conf_space_quirks.c +++ b/drivers/xen/xen-pciback/conf_space_quirks.c @@ -6,6 +6,8 @@ * Author: Chris Bookholt <hap10@epoch.ncsc.mil> */ +#define dev_fmt(fmt) DRV_NAME ": " fmt + #include <linux/kernel.h> #include <linux/pci.h> #include "pciback.h" @@ -35,8 +37,8 @@ static struct xen_pcibk_config_quirk *xen_pcibk_find_quirk(struct pci_dev *dev) if (match_one_device(&tmp_quirk->devid, dev) != NULL) goto out; tmp_quirk = NULL; - printk(KERN_DEBUG DRV_NAME - ": quirk didn't match any device known\n"); + dev_printk(KERN_DEBUG, &dev->dev, + "quirk didn't match any device known\n"); out: return tmp_quirk; } diff --git a/drivers/xen/xen-pciback/pci_stub.c b/drivers/xen/xen-pciback/pci_stub.c index 7af93d65ed51..e876c3d6dad1 100644 --- a/drivers/xen/xen-pciback/pci_stub.c +++ b/drivers/xen/xen-pciback/pci_stub.c @@ -6,6 +6,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define dev_fmt pr_fmt #include <linux/module.h> #include <linux/init.h> @@ -626,11 +627,11 @@ static void pcistub_remove(struct pci_dev *dev) if (found_psdev->pdev) { int domid = xen_find_device_domain_owner(dev); - pr_warn("****** removing device %s while still in-use by domain %d! ******\n", + dev_warn(&dev->dev, "****** removing device %s while still in-use by domain %d! ******\n", pci_name(found_psdev->dev), domid); - pr_warn("****** driver domain may still access this device's i/o resources!\n"); - pr_warn("****** shutdown driver domain before binding device\n"); - pr_warn("****** to other drivers or domains\n"); + dev_warn(&dev->dev, "****** driver domain may still access this device's i/o resources!\n"); + dev_warn(&dev->dev, "****** shutdown driver domain before binding device\n"); + dev_warn(&dev->dev, "****** to other drivers or domains\n"); /* N.B. This ends up calling pcistub_put_pci_dev which ends up * doing the FLR. */ @@ -711,14 +712,12 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev, ret = xen_pcibk_get_pcifront_dev(psdev->dev, psdev->pdev, &aer_op->domain, &aer_op->bus, &aer_op->devfn); if (!ret) { - dev_err(&psdev->dev->dev, - DRV_NAME ": failed to get pcifront device\n"); + dev_err(&psdev->dev->dev, "failed to get pcifront device\n"); return PCI_ERS_RESULT_NONE; } wmb(); - dev_dbg(&psdev->dev->dev, - DRV_NAME ": aer_op %x dom %x bus %x devfn %x\n", + dev_dbg(&psdev->dev->dev, "aer_op %x dom %x bus %x devfn %x\n", aer_cmd, aer_op->domain, aer_op->bus, aer_op->devfn); /*local flag to mark there's aer request, xen_pcibk callback will use * this flag to judge whether we need to check pci-front give aer @@ -754,8 +753,7 @@ static pci_ers_result_t common_process(struct pcistub_device *psdev, if (test_bit(_XEN_PCIF_active, (unsigned long *)&sh_info->flags)) { - dev_dbg(&psdev->dev->dev, - "schedule pci_conf service in " DRV_NAME "\n"); + dev_dbg(&psdev->dev->dev, "schedule pci_conf service\n"); xen_pcibk_test_and_schedule_op(psdev->pdev); } @@ -786,13 +784,12 @@ static pci_ers_result_t xen_pcibk_slot_reset(struct pci_dev *dev) PCI_FUNC(dev->devfn)); if (!psdev || !psdev->pdev) { - dev_err(&dev->dev, - DRV_NAME " device is not found/assigned\n"); + dev_err(&dev->dev, "device is not found/assigned\n"); goto end; } if (!psdev->pdev->sh_info) { - dev_err(&dev->dev, DRV_NAME " device is not connected or owned" + dev_err(&dev->dev, "device is not connected or owned" " by HVM, kill it\n"); kill_domain_by_device(psdev); goto end; @@ -844,13 +841,12 @@ static pci_ers_result_t xen_pcibk_mmio_enabled(struct pci_dev *dev) PCI_FUNC(dev->devfn)); if (!psdev || !psdev->pdev) { - dev_err(&dev->dev, - DRV_NAME " device is not found/assigned\n"); + dev_err(&dev->dev, "device is not found/assigned\n"); goto end; } if (!psdev->pdev->sh_info) { - dev_err(&dev->dev, DRV_NAME " device is not connected or owned" + dev_err(&dev->dev, "device is not connected or owned" " by HVM, kill it\n"); kill_domain_by_device(psdev); goto end; @@ -902,13 +898,12 @@ static pci_ers_result_t xen_pcibk_error_detected(struct pci_dev *dev, PCI_FUNC(dev->devfn)); if (!psdev || !psdev->pdev) { - dev_err(&dev->dev, - DRV_NAME " device is not found/assigned\n"); + dev_err(&dev->dev, "device is not found/assigned\n"); goto end; } if (!psdev->pdev->sh_info) { - dev_err(&dev->dev, DRV_NAME " device is not connected or owned" + dev_err(&dev->dev, "device is not connected or owned" " by HVM, kill it\n"); kill_domain_by_device(psdev); goto end; @@ -956,13 +951,12 @@ static void xen_pcibk_error_resume(struct pci_dev *dev) PCI_FUNC(dev->devfn)); if (!psdev || !psdev->pdev) { - dev_err(&dev->dev, - DRV_NAME " device is not found/assigned\n"); + dev_err(&dev->dev, "device is not found/assigned\n"); goto end; } if (!psdev->pdev->sh_info) { - dev_err(&dev->dev, DRV_NAME " device is not connected or owned" + dev_err(&dev->dev, "device is not connected or owned" " by HVM, kill it\n"); kill_domain_by_device(psdev); goto end; diff --git a/drivers/xen/xen-pciback/pciback.h b/drivers/xen/xen-pciback/pciback.h index 7c95516a860f..f1ed2dbf685c 100644 --- a/drivers/xen/xen-pciback/pciback.h +++ b/drivers/xen/xen-pciback/pciback.h @@ -186,8 +186,6 @@ void xen_pcibk_do_op(struct work_struct *data); int xen_pcibk_xenbus_register(void); void xen_pcibk_xenbus_unregister(void); -extern int verbose_request; - void xen_pcibk_test_and_schedule_op(struct xen_pcibk_device *pdev); #endif diff --git a/drivers/xen/xen-pciback/pciback_ops.c b/drivers/xen/xen-pciback/pciback_ops.c index 787966f44589..e11a7438e1a2 100644 --- a/drivers/xen/xen-pciback/pciback_ops.c +++ b/drivers/xen/xen-pciback/pciback_ops.c @@ -6,6 +6,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define dev_fmt pr_fmt #include <linux/moduleparam.h> #include <linux/wait.h> @@ -14,9 +15,6 @@ #include <linux/sched.h> #include "pciback.h" -int verbose_request; -module_param(verbose_request, int, 0644); - static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id); /* Ensure a device is has the fake IRQ handler "turned on/off" and is @@ -147,9 +145,6 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev, struct xen_pcibk_dev_data *dev_data; int status; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: enable MSI\n", pci_name(dev)); - if (dev->msi_enabled) status = -EALREADY; else if (dev->msix_enabled) @@ -158,9 +153,8 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev, status = pci_enable_msi(dev); if (status) { - pr_warn_ratelimited("%s: error enabling MSI for guest %u: err %d\n", - pci_name(dev), pdev->xdev->otherend_id, - status); + dev_warn_ratelimited(&dev->dev, "error enabling MSI for guest %u: err %d\n", + pdev->xdev->otherend_id, status); op->value = 0; return XEN_PCI_ERR_op_failed; } @@ -169,9 +163,8 @@ int xen_pcibk_enable_msi(struct xen_pcibk_device *pdev, * the local domain's IRQ number. */ op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev), - op->value); + + dev_dbg(&dev->dev, "MSI: %d\n", op->value); dev_data = pci_get_drvdata(dev); if (dev_data) @@ -184,10 +177,6 @@ static int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev, struct pci_dev *dev, struct xen_pci_op *op) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: disable MSI\n", - pci_name(dev)); - if (dev->msi_enabled) { struct xen_pcibk_dev_data *dev_data; @@ -198,9 +187,9 @@ int xen_pcibk_disable_msi(struct xen_pcibk_device *pdev, dev_data->ack_intr = 1; } op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: MSI: %d\n", pci_name(dev), - op->value); + + dev_dbg(&dev->dev, "MSI: %d\n", op->value); + return 0; } @@ -213,9 +202,7 @@ int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev, struct msix_entry *entries; u16 cmd; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: enable MSI-X\n", - pci_name(dev)); + dev_dbg(&dev->dev, "enable MSI-X\n"); if (op->value > SH_INFO_MAX_VEC) return -EINVAL; @@ -248,17 +235,13 @@ int xen_pcibk_enable_msix(struct xen_pcibk_device *pdev, if (entries[i].vector) { op->msix_entries[i].vector = xen_pirq_from_irq(entries[i].vector); - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: " \ - "MSI-X[%d]: %d\n", - pci_name(dev), i, - op->msix_entries[i].vector); + dev_dbg(&dev->dev, "MSI-X[%d]: %d\n", i, + op->msix_entries[i].vector); } } } else - pr_warn_ratelimited("%s: error enabling MSI-X for guest %u: err %d!\n", - pci_name(dev), pdev->xdev->otherend_id, - result); + dev_warn_ratelimited(&dev->dev, "error enabling MSI-X for guest %u: err %d!\n", + pdev->xdev->otherend_id, result); kfree(entries); op->value = result; @@ -273,10 +256,6 @@ static int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev, struct pci_dev *dev, struct xen_pci_op *op) { - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: disable MSI-X\n", - pci_name(dev)); - if (dev->msix_enabled) { struct xen_pcibk_dev_data *dev_data; @@ -291,9 +270,9 @@ int xen_pcibk_disable_msix(struct xen_pcibk_device *pdev, * an undefined IRQ value of zero. */ op->value = dev->irq ? xen_pirq_from_irq(dev->irq) : 0; - if (unlikely(verbose_request)) - printk(KERN_DEBUG DRV_NAME ": %s: MSI-X: %d\n", - pci_name(dev), op->value); + + dev_dbg(&dev->dev, "MSI-X: %d\n", op->value); + return 0; } #endif @@ -424,7 +403,7 @@ static irqreturn_t xen_pcibk_guest_interrupt(int irq, void *dev_id) dev_data->handled++; if ((dev_data->handled % 1000) == 0) { if (xen_test_irq_shared(irq)) { - pr_info("%s IRQ line is not shared " + dev_info(&dev->dev, "%s IRQ line is not shared " "with other domains. Turning ISR off\n", dev_data->irq_name); dev_data->ack_intr = 0; diff --git a/drivers/xen/xen-pciback/vpci.c b/drivers/xen/xen-pciback/vpci.c index f6ba18191c0f..5447b5ab7c76 100644 --- a/drivers/xen/xen-pciback/vpci.c +++ b/drivers/xen/xen-pciback/vpci.c @@ -7,6 +7,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define dev_fmt pr_fmt #include <linux/list.h> #include <linux/slab.h> @@ -105,9 +106,8 @@ static int __xen_pcibk_add_pci_dev(struct xen_pcibk_device *pdev, struct pci_dev_entry, list); if (match_slot(dev, t->dev)) { - pr_info("vpci: %s: assign to virtual slot %d func %d\n", - pci_name(dev), slot, - PCI_FUNC(dev->devfn)); + dev_info(&dev->dev, "vpci: assign to virtual slot %d func %d\n", + slot, PCI_FUNC(dev->devfn)); list_add_tail(&dev_entry->list, &vpci_dev->dev_list[slot]); func = PCI_FUNC(dev->devfn); @@ -119,8 +119,8 @@ static int __xen_pcibk_add_pci_dev(struct xen_pcibk_device *pdev, /* Assign to a new slot on the virtual PCI bus */ for (slot = 0; slot < PCI_SLOT_MAX; slot++) { if (list_empty(&vpci_dev->dev_list[slot])) { - pr_info("vpci: %s: assign to virtual slot %d\n", - pci_name(dev), slot); + dev_info(&dev->dev, "vpci: assign to virtual slot %d\n", + slot); list_add_tail(&dev_entry->list, &vpci_dev->dev_list[slot]); func = dev->is_virtfn ? 0 : PCI_FUNC(dev->devfn); diff --git a/drivers/xen/xenbus/xenbus_probe.c b/drivers/xen/xenbus/xenbus_probe.c index dc81e9926a76..38725d97d909 100644 --- a/drivers/xen/xenbus/xenbus_probe.c +++ b/drivers/xen/xenbus/xenbus_probe.c @@ -31,6 +31,7 @@ */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt +#define dev_fmt pr_fmt #define DPRINTK(fmt, args...) \ pr_debug("xenbus_probe (%s:%d) " fmt ".\n", \ @@ -607,7 +608,7 @@ int xenbus_dev_suspend(struct device *dev) if (drv->suspend) err = drv->suspend(xdev); if (err) - pr_warn("suspend %s failed: %i\n", dev_name(dev), err); + dev_warn(dev, "suspend failed: %i\n", err); return 0; } EXPORT_SYMBOL_GPL(xenbus_dev_suspend); @@ -626,8 +627,7 @@ int xenbus_dev_resume(struct device *dev) drv = to_xenbus_driver(dev->driver); err = talk_to_otherend(xdev); if (err) { - pr_warn("resume (talk_to_otherend) %s failed: %i\n", - dev_name(dev), err); + dev_warn(dev, "resume (talk_to_otherend) failed: %i\n", err); return err; } @@ -636,15 +636,14 @@ int xenbus_dev_resume(struct device *dev) if (drv->resume) { err = drv->resume(xdev); if (err) { - pr_warn("resume %s failed: %i\n", dev_name(dev), err); + dev_warn(dev, "resume failed: %i\n", err); return err; } } err = watch_otherend(xdev); if (err) { - pr_warn("resume (watch_otherend) %s failed: %d.\n", - dev_name(dev), err); + dev_warn(dev, "resume (watch_otherend) failed: %d\n", err); return err; } diff --git a/fs/afs/write.c b/fs/afs/write.c index 97bccde3298b..768497f82aee 100644 --- a/fs/afs/write.c +++ b/fs/afs/write.c @@ -447,6 +447,7 @@ static int afs_store_data(struct address_space *mapping, op->store.last = last; op->store.first_offset = offset; op->store.last_to = to; + op->mtime = vnode->vfs_inode.i_mtime; op->ops = &afs_store_data_operation; try_next_key: @@ -27,7 +27,6 @@ #include <linux/file.h> #include <linux/mm.h> #include <linux/mman.h> -#include <linux/mmu_context.h> #include <linux/percpu.h> #include <linux/slab.h> #include <linux/timer.h> diff --git a/fs/io-wq.c b/fs/io-wq.c index 4023c9846860..0b65a912b036 100644 --- a/fs/io-wq.c +++ b/fs/io-wq.c @@ -10,7 +10,6 @@ #include <linux/errno.h> #include <linux/sched/signal.h> #include <linux/mm.h> -#include <linux/mmu_context.h> #include <linux/sched/mm.h> #include <linux/percpu.h> #include <linux/slab.h> @@ -112,6 +111,7 @@ struct io_wq { unsigned long state; free_work_fn *free_work; + io_wq_work_fn *do_work; struct task_struct *manager; struct user_struct *user; @@ -170,8 +170,7 @@ static bool __io_worker_unuse(struct io_wqe *wqe, struct io_worker *worker) dropped_lock = true; } __set_current_state(TASK_RUNNING); - set_fs(KERNEL_DS); - unuse_mm(worker->mm); + kthread_unuse_mm(worker->mm); mmput(worker->mm); worker->mm = NULL; } @@ -418,18 +417,15 @@ static struct io_wq_work *io_get_next_work(struct io_wqe *wqe) static void io_wq_switch_mm(struct io_worker *worker, struct io_wq_work *work) { if (worker->mm) { - unuse_mm(worker->mm); + kthread_unuse_mm(worker->mm); mmput(worker->mm); worker->mm = NULL; } - if (!work->mm) { - set_fs(KERNEL_DS); + if (!work->mm) return; - } + if (mmget_not_zero(work->mm)) { - use_mm(work->mm); - if (!worker->mm) - set_fs(USER_DS); + kthread_use_mm(work->mm); worker->mm = work->mm; /* hang on to this mm */ work->mm = NULL; @@ -528,7 +524,7 @@ get_next: hash = io_get_work_hash(work); linked = old_work = work; - linked->func(&linked); + wq->do_work(&linked); linked = (old_work == linked) ? NULL : linked; work = next_hashed; @@ -785,7 +781,7 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wqe *wqe) struct io_wq_work *old_work = work; work->flags |= IO_WQ_WORK_CANCEL; - work->func(&work); + wq->do_work(&work); work = (work == old_work) ? NULL : work; wq->free_work(old_work); } while (work); @@ -1023,7 +1019,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) int ret = -ENOMEM, node; struct io_wq *wq; - if (WARN_ON_ONCE(!data->free_work)) + if (WARN_ON_ONCE(!data->free_work || !data->do_work)) return ERR_PTR(-EINVAL); wq = kzalloc(sizeof(*wq), GFP_KERNEL); @@ -1037,6 +1033,7 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data) } wq->free_work = data->free_work; + wq->do_work = data->do_work; /* caller must already hold a reference to this */ wq->user = data->user; @@ -1093,7 +1090,7 @@ err: bool io_wq_get(struct io_wq *wq, struct io_wq_data *data) { - if (data->free_work != wq->free_work) + if (data->free_work != wq->free_work || data->do_work != wq->do_work) return false; return refcount_inc_not_zero(&wq->use_refs); diff --git a/fs/io-wq.h b/fs/io-wq.h index 5ba12de7572f..8e138fa88b9f 100644 --- a/fs/io-wq.h +++ b/fs/io-wq.h @@ -85,7 +85,6 @@ static inline void wq_list_del(struct io_wq_work_list *list, struct io_wq_work { struct io_wq_work_node list; - void (*func)(struct io_wq_work **); struct files_struct *files; struct mm_struct *mm; const struct cred *creds; @@ -94,11 +93,6 @@ struct io_wq_work { pid_t task_pid; }; -#define INIT_IO_WORK(work, _func) \ - do { \ - *(work) = (struct io_wq_work){ .func = _func }; \ - } while (0) \ - static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) { if (!work->list.next) @@ -108,10 +102,12 @@ static inline struct io_wq_work *wq_next_work(struct io_wq_work *work) } typedef void (free_work_fn)(struct io_wq_work *); +typedef void (io_wq_work_fn)(struct io_wq_work **); struct io_wq_data { struct user_struct *user; + io_wq_work_fn *do_work; free_work_fn *free_work; }; diff --git a/fs/io_uring.c b/fs/io_uring.c index 9fb0dc6033ba..155f3d830ddb 100644 --- a/fs/io_uring.c +++ b/fs/io_uring.c @@ -55,7 +55,6 @@ #include <linux/fdtable.h> #include <linux/mm.h> #include <linux/mman.h> -#include <linux/mmu_context.h> #include <linux/percpu.h> #include <linux/slab.h> #include <linux/kthread.h> @@ -529,7 +528,6 @@ enum { REQ_F_INFLIGHT_BIT, REQ_F_CUR_POS_BIT, REQ_F_NOWAIT_BIT, - REQ_F_IOPOLL_COMPLETED_BIT, REQ_F_LINK_TIMEOUT_BIT, REQ_F_TIMEOUT_BIT, REQ_F_ISREG_BIT, @@ -541,6 +539,8 @@ enum { REQ_F_POLLED_BIT, REQ_F_BUFFER_SELECTED_BIT, REQ_F_NO_FILE_TABLE_BIT, + REQ_F_QUEUE_TIMEOUT_BIT, + REQ_F_WORK_INITIALIZED_BIT, /* not a real bit, just to check we're not overflowing the space */ __REQ_F_LAST_BIT, @@ -572,8 +572,6 @@ enum { REQ_F_CUR_POS = BIT(REQ_F_CUR_POS_BIT), /* must not punt to workers */ REQ_F_NOWAIT = BIT(REQ_F_NOWAIT_BIT), - /* polled IO has completed */ - REQ_F_IOPOLL_COMPLETED = BIT(REQ_F_IOPOLL_COMPLETED_BIT), /* has linked timeout */ REQ_F_LINK_TIMEOUT = BIT(REQ_F_LINK_TIMEOUT_BIT), /* timeout request */ @@ -596,6 +594,10 @@ enum { REQ_F_BUFFER_SELECTED = BIT(REQ_F_BUFFER_SELECTED_BIT), /* doesn't need file table for this request */ REQ_F_NO_FILE_TABLE = BIT(REQ_F_NO_FILE_TABLE_BIT), + /* needs to queue linked timeout */ + REQ_F_QUEUE_TIMEOUT = BIT(REQ_F_QUEUE_TIMEOUT_BIT), + /* io_wq_work is initialized */ + REQ_F_WORK_INITIALIZED = BIT(REQ_F_WORK_INITIALIZED_BIT), }; struct async_poll { @@ -634,6 +636,8 @@ struct io_kiocb { struct io_async_ctx *io; int cflags; u8 opcode; + /* polled IO has completed */ + u8 iopoll_completed; u16 buf_index; @@ -698,6 +702,8 @@ struct io_op_def { unsigned needs_mm : 1; /* needs req->file assigned */ unsigned needs_file : 1; + /* don't fail if file grab fails */ + unsigned needs_file_no_error : 1; /* hash wq insertion if file is a regular file */ unsigned hash_reg_file : 1; /* unbound wq insertion if file is a non-regular file */ @@ -804,6 +810,8 @@ static const struct io_op_def io_op_defs[] = { .needs_fs = 1, }, [IORING_OP_CLOSE] = { + .needs_file = 1, + .needs_file_no_error = 1, .file_table = 1, }, [IORING_OP_FILES_UPDATE] = { @@ -904,6 +912,19 @@ EXPORT_SYMBOL(io_uring_get_socket); static void io_file_put_work(struct work_struct *work); +/* + * Note: must call io_req_init_async() for the first time you + * touch any members of io_wq_work. + */ +static inline void io_req_init_async(struct io_kiocb *req) +{ + if (req->flags & REQ_F_WORK_INITIALIZED) + return; + + memset(&req->work, 0, sizeof(req->work)); + req->flags |= REQ_F_WORK_INITIALIZED; +} + static inline bool io_async_submit(struct io_ring_ctx *ctx) { return ctx->flags & IORING_SETUP_SQPOLL; @@ -1030,6 +1051,9 @@ static inline void io_req_work_grab_env(struct io_kiocb *req, static inline void io_req_work_drop_env(struct io_kiocb *req) { + if (!(req->flags & REQ_F_WORK_INITIALIZED)) + return; + if (req->work.mm) { mmdrop(req->work.mm); req->work.mm = NULL; @@ -1576,16 +1600,6 @@ static void io_free_req(struct io_kiocb *req) io_queue_async_work(nxt); } -static void io_link_work_cb(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); - struct io_kiocb *link; - - link = list_first_entry(&req->link_list, struct io_kiocb, link_list); - io_queue_linked_timeout(link); - io_wq_submit_work(workptr); -} - static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt) { struct io_kiocb *link; @@ -1597,7 +1611,7 @@ static void io_wq_assign_next(struct io_wq_work **workptr, struct io_kiocb *nxt) *workptr = &nxt->work; link = io_prep_linked_timeout(nxt); if (link) - nxt->work.func = io_link_work_cb; + nxt->flags |= REQ_F_QUEUE_TIMEOUT; } /* @@ -1782,7 +1796,7 @@ static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events, * If we find a request that requires polling, break out * and complete those lists first, if we have entries there. */ - if (req->flags & REQ_F_IOPOLL_COMPLETED) { + if (READ_ONCE(req->iopoll_completed)) { list_move_tail(&req->list, &done); continue; } @@ -1963,7 +1977,7 @@ static void io_complete_rw_iopoll(struct kiocb *kiocb, long res, long res2) req_set_fail_links(req); req->result = res; if (res != -EAGAIN) - req->flags |= REQ_F_IOPOLL_COMPLETED; + WRITE_ONCE(req->iopoll_completed, 1); } /* @@ -1996,7 +2010,7 @@ static void io_iopoll_req_issued(struct io_kiocb *req) * For fast devices, IO may have already completed. If it has, add * it to the front so we find it first. */ - if (req->flags & REQ_F_IOPOLL_COMPLETED) + if (READ_ONCE(req->iopoll_completed)) list_add(&req->list, &ctx->poll_list); else list_add_tail(&req->list, &ctx->poll_list); @@ -2064,6 +2078,10 @@ static bool io_file_supports_async(struct file *file, int rw) if (S_ISREG(mode) && file->f_op != &io_uring_fops) return true; + /* any ->read/write should understand O_NONBLOCK */ + if (file->f_flags & O_NONBLOCK) + return true; + if (!(file->f_mode & FMODE_NOWAIT)) return false; @@ -2106,8 +2124,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_ioprio = get_current_ioprio(); /* don't allow async punt if RWF_NOWAIT was requested */ - if ((kiocb->ki_flags & IOCB_NOWAIT) || - (req->file->f_flags & O_NONBLOCK)) + if (kiocb->ki_flags & IOCB_NOWAIT) req->flags |= REQ_F_NOWAIT; if (force_nonblock) @@ -2121,6 +2138,7 @@ static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe, kiocb->ki_flags |= IOCB_HIPRI; kiocb->ki_complete = io_complete_rw_iopoll; req->result = 0; + req->iopoll_completed = 0; } else { if (kiocb->ki_flags & IOCB_HIPRI) return -EINVAL; @@ -2359,8 +2377,14 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, bool needs_lock) { - if (req->flags & REQ_F_BUFFER_SELECTED) + if (req->flags & REQ_F_BUFFER_SELECTED) { + struct io_buffer *kbuf; + + kbuf = (struct io_buffer *) (unsigned long) req->rw.addr; + iov[0].iov_base = u64_to_user_ptr(kbuf->addr); + iov[0].iov_len = kbuf->len; return 0; + } if (!req->rw.len) return 0; else if (req->rw.len > 1) @@ -2742,7 +2766,8 @@ copy_iov: if (ret) goto out_free; /* any defer here is final, must blocking retry */ - if (!file_can_poll(req->file)) + if (!(req->flags & REQ_F_NOWAIT) && + !file_can_poll(req->file)) req->flags |= REQ_F_MUST_PUNT; return -EAGAIN; } @@ -2762,6 +2787,8 @@ static int __io_splice_prep(struct io_kiocb *req, if (req->flags & REQ_F_NEED_CLEANUP) return 0; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; sp->file_in = NULL; sp->len = READ_ONCE(sqe->len); @@ -2776,8 +2803,14 @@ static int __io_splice_prep(struct io_kiocb *req, return ret; req->flags |= REQ_F_NEED_CLEANUP; - if (!S_ISREG(file_inode(sp->file_in)->i_mode)) + if (!S_ISREG(file_inode(sp->file_in)->i_mode)) { + /* + * Splice operation will be punted aync, and here need to + * modify io_wq_work.flags, so initialize io_wq_work firstly. + */ + io_req_init_async(req); req->work.flags |= IO_WQ_WORK_UNBOUND; + } return 0; } @@ -2886,23 +2919,15 @@ static int io_prep_fsync(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static bool io_req_cancelled(struct io_kiocb *req) -{ - if (req->work.flags & IO_WQ_WORK_CANCEL) { - req_set_fail_links(req); - io_cqring_add_event(req, -ECANCELED); - io_put_req(req); - return true; - } - - return false; -} - -static void __io_fsync(struct io_kiocb *req) +static int io_fsync(struct io_kiocb *req, bool force_nonblock) { loff_t end = req->sync.off + req->sync.len; int ret; + /* fsync always requires a blocking context */ + if (force_nonblock) + return -EAGAIN; + ret = vfs_fsync_range(req->file, req->sync.off, end > 0 ? end : LLONG_MAX, req->sync.flags & IORING_FSYNC_DATASYNC); @@ -2910,58 +2935,16 @@ static void __io_fsync(struct io_kiocb *req) req_set_fail_links(req); io_cqring_add_event(req, ret); io_put_req(req); -} - -static void io_fsync_finish(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); - - if (io_req_cancelled(req)) - return; - __io_fsync(req); - io_steal_work(req, workptr); -} - -static int io_fsync(struct io_kiocb *req, bool force_nonblock) -{ - /* fsync always requires a blocking context */ - if (force_nonblock) { - req->work.func = io_fsync_finish; - return -EAGAIN; - } - __io_fsync(req); return 0; } -static void __io_fallocate(struct io_kiocb *req) -{ - int ret; - - current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize; - ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, - req->sync.len); - current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; - if (ret < 0) - req_set_fail_links(req); - io_cqring_add_event(req, ret); - io_put_req(req); -} - -static void io_fallocate_finish(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); - - if (io_req_cancelled(req)) - return; - __io_fallocate(req); - io_steal_work(req, workptr); -} - static int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (sqe->ioprio || sqe->buf_index || sqe->rw_flags) return -EINVAL; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; req->sync.off = READ_ONCE(sqe->off); req->sync.len = READ_ONCE(sqe->addr); @@ -2972,66 +2955,74 @@ static int io_fallocate_prep(struct io_kiocb *req, static int io_fallocate(struct io_kiocb *req, bool force_nonblock) { + int ret; + /* fallocate always requiring blocking context */ - if (force_nonblock) { - req->work.func = io_fallocate_finish; + if (force_nonblock) return -EAGAIN; - } - __io_fallocate(req); + current->signal->rlim[RLIMIT_FSIZE].rlim_cur = req->fsize; + ret = vfs_fallocate(req->file, req->sync.mode, req->sync.off, + req->sync.len); + current->signal->rlim[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY; + if (ret < 0) + req_set_fail_links(req); + io_cqring_add_event(req, ret); + io_put_req(req); return 0; } -static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { const char __user *fname; int ret; - if (sqe->ioprio || sqe->buf_index) + if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) return -EINVAL; - if (req->flags & REQ_F_FIXED_FILE) + if (unlikely(sqe->ioprio || sqe->buf_index)) + return -EINVAL; + if (unlikely(req->flags & REQ_F_FIXED_FILE)) return -EBADF; - if (req->flags & REQ_F_NEED_CLEANUP) - return 0; - req->open.dfd = READ_ONCE(sqe->fd); - req->open.how.mode = READ_ONCE(sqe->len); - fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); - req->open.how.flags = READ_ONCE(sqe->open_flags); - if (force_o_largefile()) + /* open.how should be already initialised */ + if (!(req->open.how.flags & O_PATH) && force_o_largefile()) req->open.how.flags |= O_LARGEFILE; + req->open.dfd = READ_ONCE(sqe->fd); + fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); req->open.filename = getname(fname); if (IS_ERR(req->open.filename)) { ret = PTR_ERR(req->open.filename); req->open.filename = NULL; return ret; } - req->open.nofile = rlimit(RLIMIT_NOFILE); req->flags |= REQ_F_NEED_CLEANUP; return 0; } +static int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) +{ + u64 flags, mode; + + if (req->flags & REQ_F_NEED_CLEANUP) + return 0; + mode = READ_ONCE(sqe->len); + flags = READ_ONCE(sqe->open_flags); + req->open.how = build_open_how(flags, mode); + return __io_openat_prep(req, sqe); +} + static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct open_how __user *how; - const char __user *fname; size_t len; int ret; - if (sqe->ioprio || sqe->buf_index) - return -EINVAL; - if (req->flags & REQ_F_FIXED_FILE) - return -EBADF; if (req->flags & REQ_F_NEED_CLEANUP) return 0; - - req->open.dfd = READ_ONCE(sqe->fd); - fname = u64_to_user_ptr(READ_ONCE(sqe->addr)); how = u64_to_user_ptr(READ_ONCE(sqe->addr2)); len = READ_ONCE(sqe->len); - if (len < OPEN_HOW_SIZE_VER0) return -EINVAL; @@ -3040,19 +3031,7 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) if (ret) return ret; - if (!(req->open.how.flags & O_PATH) && force_o_largefile()) - req->open.how.flags |= O_LARGEFILE; - - req->open.filename = getname(fname); - if (IS_ERR(req->open.filename)) { - ret = PTR_ERR(req->open.filename); - req->open.filename = NULL; - return ret; - } - - req->open.nofile = rlimit(RLIMIT_NOFILE); - req->flags |= REQ_F_NEED_CLEANUP; - return 0; + return __io_openat_prep(req, sqe); } static int io_openat2(struct io_kiocb *req, bool force_nonblock) @@ -3092,7 +3071,6 @@ err: static int io_openat(struct io_kiocb *req, bool force_nonblock) { - req->open.how = build_open_how(req->open.how.flags, req->open.how.mode); return io_openat2(req, force_nonblock); } @@ -3181,7 +3159,7 @@ static int io_provide_buffers_prep(struct io_kiocb *req, p->addr = READ_ONCE(sqe->addr); p->len = READ_ONCE(sqe->len); - if (!access_ok(u64_to_user_ptr(p->addr), p->len)) + if (!access_ok(u64_to_user_ptr(p->addr), (p->len * p->nbufs))) return -EFAULT; p->bgid = READ_ONCE(sqe->buf_group); @@ -3259,6 +3237,8 @@ static int io_epoll_ctl_prep(struct io_kiocb *req, #if defined(CONFIG_EPOLL) if (sqe->ioprio || sqe->buf_index) return -EINVAL; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; req->epoll.epfd = READ_ONCE(sqe->fd); req->epoll.op = READ_ONCE(sqe->len); @@ -3303,6 +3283,8 @@ static int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) if (sqe->ioprio || sqe->buf_index || sqe->off) return -EINVAL; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; req->madvise.addr = READ_ONCE(sqe->addr); req->madvise.len = READ_ONCE(sqe->len); @@ -3337,6 +3319,8 @@ static int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { if (sqe->ioprio || sqe->buf_index || sqe->addr) return -EINVAL; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; req->fadvise.offset = READ_ONCE(sqe->off); req->fadvise.len = READ_ONCE(sqe->len); @@ -3370,6 +3354,8 @@ static int io_fadvise(struct io_kiocb *req, bool force_nonblock) static int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; if (sqe->ioprio || sqe->buf_index) return -EINVAL; if (req->flags & REQ_F_FIXED_FILE) @@ -3410,10 +3396,14 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { /* * If we queue this for async, it must not be cancellable. That would - * leave the 'file' in an undeterminate state. + * leave the 'file' in an undeterminate state, and here need to modify + * io_wq_work.flags, so initialize io_wq_work firstly. */ + io_req_init_async(req); req->work.flags |= IO_WQ_WORK_NO_CANCEL; + if (unlikely(req->ctx->flags & (IORING_SETUP_IOPOLL|IORING_SETUP_SQPOLL))) + return -EINVAL; if (sqe->ioprio || sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) return -EINVAL; @@ -3421,53 +3411,41 @@ static int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return -EBADF; req->close.fd = READ_ONCE(sqe->fd); - return 0; -} - -/* only called when __close_fd_get_file() is done */ -static void __io_close_finish(struct io_kiocb *req) -{ - int ret; - - ret = filp_close(req->close.put_file, req->work.files); - if (ret < 0) - req_set_fail_links(req); - io_cqring_add_event(req, ret); - fput(req->close.put_file); - io_put_req(req); -} - -static void io_close_finish(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); + if ((req->file && req->file->f_op == &io_uring_fops) || + req->close.fd == req->ctx->ring_fd) + return -EBADF; - /* not cancellable, don't do io_req_cancelled() */ - __io_close_finish(req); - io_steal_work(req, workptr); + req->close.put_file = NULL; + return 0; } static int io_close(struct io_kiocb *req, bool force_nonblock) { + struct io_close *close = &req->close; int ret; - req->close.put_file = NULL; - ret = __close_fd_get_file(req->close.fd, &req->close.put_file); - if (ret < 0) - return (ret == -ENOENT) ? -EBADF : ret; + /* might be already done during nonblock submission */ + if (!close->put_file) { + ret = __close_fd_get_file(close->fd, &close->put_file); + if (ret < 0) + return (ret == -ENOENT) ? -EBADF : ret; + } /* if the file has a flush method, be safe and punt to async */ - if (req->close.put_file->f_op->flush && force_nonblock) { + if (close->put_file->f_op->flush && force_nonblock) { /* avoid grabbing files - we don't need the files */ req->flags |= REQ_F_NO_FILE_TABLE | REQ_F_MUST_PUNT; - req->work.func = io_close_finish; return -EAGAIN; } - /* - * No ->flush(), safely close from here and just punt the - * fput() to async context. - */ - __io_close_finish(req); + /* No ->flush() or already async, safely close from here */ + ret = filp_close(close->put_file, req->work.files); + if (ret < 0) + req_set_fail_links(req); + io_cqring_add_event(req, ret); + fput(close->put_file); + close->put_file = NULL; + io_put_req(req); return 0; } @@ -3489,38 +3467,20 @@ static int io_prep_sfr(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static void __io_sync_file_range(struct io_kiocb *req) +static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) { int ret; + /* sync_file_range always requires a blocking context */ + if (force_nonblock) + return -EAGAIN; + ret = sync_file_range(req->file, req->sync.off, req->sync.len, req->sync.flags); if (ret < 0) req_set_fail_links(req); io_cqring_add_event(req, ret); io_put_req(req); -} - - -static void io_sync_file_range_finish(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); - - if (io_req_cancelled(req)) - return; - __io_sync_file_range(req); - io_steal_work(req, workptr); -} - -static int io_sync_file_range(struct io_kiocb *req, bool force_nonblock) -{ - /* sync_file_range always requires a blocking context */ - if (force_nonblock) { - req->work.func = io_sync_file_range_finish; - return -EAGAIN; - } - - __io_sync_file_range(req); return 0; } @@ -3546,6 +3506,9 @@ static int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) struct io_async_ctx *io = req->io; int ret; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; + sr->msg_flags = READ_ONCE(sqe->msg_flags); sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); @@ -3575,9 +3538,6 @@ static int io_sendmsg(struct io_kiocb *req, bool force_nonblock) struct socket *sock; int ret; - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) - return -EINVAL; - sock = sock_from_file(req->file, &ret); if (sock) { struct io_async_ctx io; @@ -3631,9 +3591,6 @@ static int io_send(struct io_kiocb *req, bool force_nonblock) struct socket *sock; int ret; - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) - return -EINVAL; - sock = sock_from_file(req->file, &ret); if (sock) { struct io_sr_msg *sr = &req->sr_msg; @@ -3786,6 +3743,9 @@ static int io_recvmsg_prep(struct io_kiocb *req, struct io_async_ctx *io = req->io; int ret; + if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) + return -EINVAL; + sr->msg_flags = READ_ONCE(sqe->msg_flags); sr->msg = u64_to_user_ptr(READ_ONCE(sqe->addr)); sr->len = READ_ONCE(sqe->len); @@ -3814,9 +3774,6 @@ static int io_recvmsg(struct io_kiocb *req, bool force_nonblock) struct socket *sock; int ret, cflags = 0; - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) - return -EINVAL; - sock = sock_from_file(req->file, &ret); if (sock) { struct io_buffer *kbuf; @@ -3878,9 +3835,6 @@ static int io_recv(struct io_kiocb *req, bool force_nonblock) struct socket *sock; int ret, cflags = 0; - if (unlikely(req->ctx->flags & IORING_SETUP_IOPOLL)) - return -EINVAL; - sock = sock_from_file(req->file, &ret); if (sock) { struct io_sr_msg *sr = &req->sr_msg; @@ -3948,49 +3902,30 @@ static int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) return 0; } -static int __io_accept(struct io_kiocb *req, bool force_nonblock) +static int io_accept(struct io_kiocb *req, bool force_nonblock) { struct io_accept *accept = &req->accept; - unsigned file_flags; + unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; int ret; - file_flags = force_nonblock ? O_NONBLOCK : 0; + if (req->file->f_flags & O_NONBLOCK) + req->flags |= REQ_F_NOWAIT; + ret = __sys_accept4_file(req->file, file_flags, accept->addr, accept->addr_len, accept->flags, accept->nofile); if (ret == -EAGAIN && force_nonblock) return -EAGAIN; - if (ret == -ERESTARTSYS) - ret = -EINTR; - if (ret < 0) + if (ret < 0) { + if (ret == -ERESTARTSYS) + ret = -EINTR; req_set_fail_links(req); + } io_cqring_add_event(req, ret); io_put_req(req); return 0; } -static void io_accept_finish(struct io_wq_work **workptr) -{ - struct io_kiocb *req = container_of(*workptr, struct io_kiocb, work); - - if (io_req_cancelled(req)) - return; - __io_accept(req, false); - io_steal_work(req, workptr); -} - -static int io_accept(struct io_kiocb *req, bool force_nonblock) -{ - int ret; - - ret = __io_accept(req, force_nonblock); - if (ret == -EAGAIN && force_nonblock) { - req->work.func = io_accept_finish; - return -EAGAIN; - } - return 0; -} - static int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { struct io_connect *conn = &req->connect; @@ -4329,7 +4264,8 @@ static void io_async_task_func(struct callback_head *cb) spin_unlock_irq(&ctx->completion_lock); /* restore ->work in case we need to retry again */ - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, sizeof(req->work)); kfree(apoll); if (!canceled) { @@ -4426,7 +4362,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req) return false; req->flags |= REQ_F_POLLED; - memcpy(&apoll->work, &req->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&apoll->work, &req->work, sizeof(req->work)); had_io = req->io != NULL; get_task_struct(current); @@ -4451,7 +4388,8 @@ static bool io_arm_poll_handler(struct io_kiocb *req) if (!had_io) io_poll_remove_double(req); spin_unlock_irq(&ctx->completion_lock); - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, sizeof(req->work)); kfree(apoll); return false; } @@ -4496,7 +4434,9 @@ static bool io_poll_remove_one(struct io_kiocb *req) * io_req_work_drop_env below when dropping the * final reference. */ - memcpy(&req->work, &apoll->work, sizeof(req->work)); + if (req->flags & REQ_F_WORK_INITIALIZED) + memcpy(&req->work, &apoll->work, + sizeof(req->work)); kfree(apoll); } } @@ -4945,6 +4885,8 @@ static int io_req_defer_prep(struct io_kiocb *req, if (!sqe) return 0; + io_req_init_async(req); + if (io_op_defs[req->opcode].file_table) { ret = io_grab_files(req); if (unlikely(ret)) @@ -5382,12 +5324,26 @@ static int io_issue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe, return 0; } +static void io_arm_async_linked_timeout(struct io_kiocb *req) +{ + struct io_kiocb *link; + + /* link head's timeout is queued in io_queue_async_work() */ + if (!(req->flags & REQ_F_QUEUE_TIMEOUT)) + return; + + link = list_first_entry(&req->link_list, struct io_kiocb, link_list); + io_queue_linked_timeout(link); +} + static void io_wq_submit_work(struct io_wq_work **workptr) { struct io_wq_work *work = *workptr; struct io_kiocb *req = container_of(work, struct io_kiocb, work); int ret = 0; + io_arm_async_linked_timeout(req); + /* if NO_CANCEL is set, we must still run the work */ if ((work->flags & (IO_WQ_WORK_CANCEL|IO_WQ_WORK_NO_CANCEL)) == IO_WQ_WORK_CANCEL) { @@ -5438,19 +5394,20 @@ static int io_file_get(struct io_submit_state *state, struct io_kiocb *req, return -EBADF; fd = array_index_nospec(fd, ctx->nr_user_files); file = io_file_from_index(ctx, fd); - if (!file) - return -EBADF; - req->fixed_file_refs = ctx->file_data->cur_refs; - percpu_ref_get(req->fixed_file_refs); + if (file) { + req->fixed_file_refs = ctx->file_data->cur_refs; + percpu_ref_get(req->fixed_file_refs); + } } else { trace_io_uring_file_get(ctx, fd); file = __io_file_get(state, fd); - if (unlikely(!file)) - return -EBADF; } - *out_file = file; - return 0; + if (file || io_op_defs[req->opcode].needs_file_no_error) { + *out_file = file; + return 0; + } + return -EBADF; } static int io_req_set_file(struct io_submit_state *state, struct io_kiocb *req, @@ -5584,7 +5541,8 @@ static void __io_queue_sqe(struct io_kiocb *req, const struct io_uring_sqe *sqe) again: linked_timeout = io_prep_linked_timeout(req); - if (req->work.creds && req->work.creds != current_cred()) { + if ((req->flags & REQ_F_WORK_INITIALIZED) && req->work.creds && + req->work.creds != current_cred()) { if (old_creds) revert_creds(old_creds); if (old_creds == req->work.creds) @@ -5607,6 +5565,8 @@ again: goto exit; } punt: + io_req_init_async(req); + if (io_op_defs[req->opcode].file_table) { ret = io_grab_files(req); if (ret) @@ -5859,7 +5819,6 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, refcount_set(&req->refs, 2); req->task = NULL; req->result = 0; - INIT_IO_WORK(&req->work, io_wq_submit_work); if (unlikely(req->opcode >= IORING_OP_LAST)) return -EINVAL; @@ -5867,7 +5826,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, if (io_op_defs[req->opcode].needs_mm && !current->mm) { if (unlikely(!mmget_not_zero(ctx->sqo_mm))) return -EFAULT; - use_mm(ctx->sqo_mm); + kthread_use_mm(ctx->sqo_mm); } sqe_flags = READ_ONCE(sqe->flags); @@ -5881,6 +5840,7 @@ static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req, id = READ_ONCE(sqe->personality); if (id) { + io_req_init_async(req); req->work.creds = idr_find(&ctx->personality_idr, id); if (unlikely(!req->work.creds)) return -EINVAL; @@ -5981,7 +5941,7 @@ static inline void io_sq_thread_drop_mm(struct io_ring_ctx *ctx) struct mm_struct *mm = current->mm; if (mm) { - unuse_mm(mm); + kthread_unuse_mm(mm); mmput(mm); } } @@ -5990,15 +5950,12 @@ static int io_sq_thread(void *data) { struct io_ring_ctx *ctx = data; const struct cred *old_cred; - mm_segment_t old_fs; DEFINE_WAIT(wait); unsigned long timeout; int ret = 0; complete(&ctx->sq_thread_comp); - old_fs = get_fs(); - set_fs(USER_DS); old_cred = override_creds(ctx->creds); timeout = jiffies + ctx->sq_thread_idle; @@ -6103,7 +6060,6 @@ static int io_sq_thread(void *data) if (current->task_works) task_work_run(); - set_fs(old_fs); io_sq_thread_drop_mm(ctx); revert_creds(old_cred); @@ -6879,6 +6835,7 @@ static int io_init_wq_offload(struct io_ring_ctx *ctx, data.user = ctx->user; data.free_work = io_free_work; + data.do_work = io_wq_submit_work; if (!(p->flags & IORING_SETUP_ATTACH_WQ)) { /* Do QD, or 4 * CPUS, whatever is smallest */ @@ -7160,8 +7117,8 @@ static int io_sqe_buffer_register(struct io_ring_ctx *ctx, void __user *arg, ret = 0; if (!pages || nr_pages > got_pages) { - kfree(vmas); - kfree(pages); + kvfree(vmas); + kvfree(pages); pages = kvmalloc_array(nr_pages, sizeof(struct page *), GFP_KERNEL); vmas = kvmalloc_array(nr_pages, diff --git a/fs/iomap/buffered-io.c b/fs/iomap/buffered-io.c index a1ed7620fbac..bcfc288dba3f 100644 --- a/fs/iomap/buffered-io.c +++ b/fs/iomap/buffered-io.c @@ -870,7 +870,7 @@ iomap_unshare_actor(struct inode *inode, loff_t pos, loff_t length, void *data, struct iomap *iomap, struct iomap *srcmap) { long status = 0; - ssize_t written = 0; + loff_t written = 0; /* don't bother with blocks that are not shared to start with */ if (!(iomap->flags & IOMAP_F_SHARED)) diff --git a/fs/locks.c b/fs/locks.c index 6fd1f6e83178..7df0f9fa66f4 100644 --- a/fs/locks.c +++ b/fs/locks.c @@ -1557,6 +1557,9 @@ static bool leases_conflict(struct file_lock *lease, struct file_lock *breaker) { bool rc; + if (lease->fl_lmops->lm_breaker_owns_lease + && lease->fl_lmops->lm_breaker_owns_lease(lease)) + return false; if ((breaker->fl_flags & FL_LAYOUT) != (lease->fl_flags & FL_LAYOUT)) { rc = false; goto trace; diff --git a/fs/nfs/direct.c b/fs/nfs/direct.c index a57e7c72c7f4..1b79dd5cf661 100644 --- a/fs/nfs/direct.c +++ b/fs/nfs/direct.c @@ -446,7 +446,7 @@ ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter) struct inode *inode = mapping->host; struct nfs_direct_req *dreq; struct nfs_lock_context *l_ctx; - ssize_t result = -EINVAL, requested; + ssize_t result, requested; size_t count = iov_iter_count(iter); nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count); @@ -731,6 +731,8 @@ static void nfs_direct_write_completion(struct nfs_pgio_header *hdr) nfs_list_remove_request(req); if (request_commit) { kref_get(&req->wb_kref); + memcpy(&req->wb_verf, &hdr->verf.verifier, + sizeof(req->wb_verf)); nfs_mark_request_commit(req, hdr->lseg, &cinfo, hdr->ds_commit_idx); } diff --git a/fs/nfs/dns_resolve.c b/fs/nfs/dns_resolve.c index 963800037609..e87d500ad95a 100644 --- a/fs/nfs/dns_resolve.c +++ b/fs/nfs/dns_resolve.c @@ -39,7 +39,6 @@ ssize_t nfs_dns_resolve_name(struct net *net, char *name, size_t namelen, #include <linux/string.h> #include <linux/kmod.h> #include <linux/slab.h> -#include <linux/module.h> #include <linux/socket.h> #include <linux/seq_file.h> #include <linux/inet.h> diff --git a/fs/nfs/inode.c b/fs/nfs/inode.c index b9d0921cb4fe..0bf1f835de01 100644 --- a/fs/nfs/inode.c +++ b/fs/nfs/inode.c @@ -833,6 +833,8 @@ int nfs_getattr(const struct path *path, struct kstat *stat, do_update |= cache_validity & NFS_INO_INVALID_ATIME; if (request_mask & (STATX_CTIME|STATX_MTIME)) do_update |= cache_validity & NFS_INO_REVAL_PAGECACHE; + if (request_mask & STATX_BLOCKS) + do_update |= cache_validity & NFS_INO_INVALID_BLOCKS; if (do_update) { /* Update the attribute cache */ if (!(server->flags & NFS_MOUNT_NOAC)) @@ -1764,7 +1766,8 @@ out_noforce: status = nfs_post_op_update_inode_locked(inode, fattr, NFS_INO_INVALID_CHANGE | NFS_INO_INVALID_CTIME - | NFS_INO_INVALID_MTIME); + | NFS_INO_INVALID_MTIME + | NFS_INO_INVALID_BLOCKS); return status; } @@ -1871,7 +1874,8 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) nfsi->cache_validity &= ~(NFS_INO_INVALID_ATTR | NFS_INO_INVALID_ATIME | NFS_INO_REVAL_FORCED - | NFS_INO_REVAL_PAGECACHE); + | NFS_INO_REVAL_PAGECACHE + | NFS_INO_INVALID_BLOCKS); /* Do atomic weak cache consistency updates */ nfs_wcc_update_inode(inode, fattr); @@ -2033,8 +2037,12 @@ static int nfs_update_inode(struct inode *inode, struct nfs_fattr *fattr) inode->i_blocks = nfs_calc_block_size(fattr->du.nfs3.used); } else if (fattr->valid & NFS_ATTR_FATTR_BLOCKS_USED) inode->i_blocks = fattr->du.nfs2.blocks; - else + else { + nfsi->cache_validity |= save_cache_validity & + (NFS_INO_INVALID_BLOCKS + | NFS_INO_REVAL_FORCED); cache_revalidated = false; + } /* Update attrtimeo value if we're out of the unstable period */ if (attr_changed) { diff --git a/fs/nfs/nfs3proc.c b/fs/nfs/nfs3proc.c index a46d1d5d16d8..2397ceedba8a 100644 --- a/fs/nfs/nfs3proc.c +++ b/fs/nfs/nfs3proc.c @@ -179,11 +179,11 @@ nfs3_proc_lookup(struct inode *dir, struct dentry *dentry, if (nfs_lookup_is_soft_revalidate(dentry)) task_flags |= RPC_TASK_TIMEOUT; - dprintk("NFS call lookup %pd2\n", dentry); res.dir_attr = nfs_alloc_fattr(); if (res.dir_attr == NULL) return -ENOMEM; + dprintk("NFS call lookup %pd2\n", dentry); nfs_fattr_init(fattr); status = rpc_call_sync(NFS_CLIENT(dir), &msg, task_flags); nfs_refresh_inode(dir, res.dir_attr); diff --git a/fs/nfs/nfs4proc.c b/fs/nfs/nfs4proc.c index 9056f3dd380e..e32717fd1169 100644 --- a/fs/nfs/nfs4proc.c +++ b/fs/nfs/nfs4proc.c @@ -7909,7 +7909,7 @@ nfs4_bind_one_conn_to_session_done(struct rpc_task *task, void *calldata) } static const struct rpc_call_ops nfs4_bind_one_conn_to_session_ops = { - .rpc_call_done = &nfs4_bind_one_conn_to_session_done, + .rpc_call_done = nfs4_bind_one_conn_to_session_done, }; /* diff --git a/fs/nfs/nfstrace.h b/fs/nfs/nfstrace.h index 7e7a97ae21ed..547cec79899f 100644 --- a/fs/nfs/nfstrace.h +++ b/fs/nfs/nfstrace.h @@ -961,6 +961,97 @@ TRACE_EVENT(nfs_readpage_done, ) ); +TRACE_EVENT(nfs_readpage_short, + TP_PROTO( + const struct rpc_task *task, + const struct nfs_pgio_header *hdr + ), + + TP_ARGS(task, hdr), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(bool, eof) + __field(int, status) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->status = task->tk_status; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->eof = hdr->res.eof; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = nfsi->fileid; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk( + "fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u status=%d%s", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, + __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, + __entry->res_count, __entry->status, + __entry->eof ? " eof" : "" + ) +); + +TRACE_EVENT(nfs_pgio_error, + TP_PROTO( + const struct nfs_pgio_header *hdr, + int error, + loff_t pos + ), + + TP_ARGS(hdr, error, pos), + + TP_STRUCT__entry( + __field(dev_t, dev) + __field(u32, fhandle) + __field(u64, fileid) + __field(loff_t, offset) + __field(u32, arg_count) + __field(u32, res_count) + __field(loff_t, pos) + __field(int, status) + ), + + TP_fast_assign( + const struct inode *inode = hdr->inode; + const struct nfs_inode *nfsi = NFS_I(inode); + const struct nfs_fh *fh = hdr->args.fh ? + hdr->args.fh : &nfsi->fh; + + __entry->status = error; + __entry->offset = hdr->args.offset; + __entry->arg_count = hdr->args.count; + __entry->res_count = hdr->res.count; + __entry->dev = inode->i_sb->s_dev; + __entry->fileid = nfsi->fileid; + __entry->fhandle = nfs_fhandle_hash(fh); + ), + + TP_printk("fileid=%02x:%02x:%llu fhandle=0x%08x " + "offset=%lld count=%u res=%u pos=%llu status=%d", + MAJOR(__entry->dev), MINOR(__entry->dev), + (unsigned long long)__entry->fileid, __entry->fhandle, + (long long)__entry->offset, __entry->arg_count, __entry->res_count, + __entry->pos, __entry->status + ) +); + TRACE_DEFINE_ENUM(NFS_UNSTABLE); TRACE_DEFINE_ENUM(NFS_DATA_SYNC); TRACE_DEFINE_ENUM(NFS_FILE_SYNC); @@ -1312,7 +1403,12 @@ TRACE_EVENT(nfs_xdr_status, __field(unsigned int, task_id) __field(unsigned int, client_id) __field(u32, xid) + __field(int, version) __field(unsigned long, error) + __string(program, + xdr->rqst->rq_task->tk_client->cl_program->name) + __string(procedure, + xdr->rqst->rq_task->tk_msg.rpc_proc->p_name) ), TP_fast_assign( @@ -1322,13 +1418,19 @@ TRACE_EVENT(nfs_xdr_status, __entry->task_id = task->tk_pid; __entry->client_id = task->tk_client->cl_clid; __entry->xid = be32_to_cpu(rqstp->rq_xid); + __entry->version = task->tk_client->cl_vers; __entry->error = error; + __assign_str(program, + task->tk_client->cl_program->name) + __assign_str(procedure, task->tk_msg.rpc_proc->p_name) ), TP_printk( - "task:%u@%d xid=0x%08x error=%ld (%s)", + "task:%u@%d xid=0x%08x %sv%d %s error=%ld (%s)", __entry->task_id, __entry->client_id, __entry->xid, - -__entry->error, nfs_show_status(__entry->error) + __get_str(program), __entry->version, + __get_str(procedure), -__entry->error, + nfs_show_status(__entry->error) ) ); diff --git a/fs/nfs/pagelist.c b/fs/nfs/pagelist.c index 6ca421cbe19c..6ea4cac41e46 100644 --- a/fs/nfs/pagelist.c +++ b/fs/nfs/pagelist.c @@ -24,6 +24,7 @@ #include "internal.h" #include "pnfs.h" +#include "nfstrace.h" #define NFSDBG_FACILITY NFSDBG_PAGECACHE @@ -64,6 +65,7 @@ void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos) { unsigned int new = pos - hdr->io_start; + trace_nfs_pgio_error(hdr, error, pos); if (hdr->good_bytes > new) { hdr->good_bytes = new; clear_bit(NFS_IOHDR_EOF, &hdr->flags); diff --git a/fs/nfs/read.c b/fs/nfs/read.c index 13b22e898116..eb854f1f86e2 100644 --- a/fs/nfs/read.c +++ b/fs/nfs/read.c @@ -264,6 +264,8 @@ static void nfs_readpage_retry(struct rpc_task *task, /* This is a short read! */ nfs_inc_stats(hdr->inode, NFSIOS_SHORTREAD); + trace_nfs_readpage_short(task, hdr); + /* Has the server at least made some progress? */ if (resp->count == 0) { nfs_set_pgio_error(hdr, -EIO, argp->offset); diff --git a/fs/nfs/sysfs.h b/fs/nfs/sysfs.h index f1b27411dcc0..ebcbdc40483b 100644 --- a/fs/nfs/sysfs.h +++ b/fs/nfs/sysfs.h @@ -1,4 +1,4 @@ -// SPDX-License-Identifier: GPL-2.0 +/* SPDX-License-Identifier: GPL-2.0 */ /* * Copyright (c) 2019 Hammerspace Inc */ diff --git a/fs/nfsd/cache.h b/fs/nfsd/cache.h index 10ec5ecdf117..65c331f75e9c 100644 --- a/fs/nfsd/cache.h +++ b/fs/nfsd/cache.h @@ -78,6 +78,8 @@ enum { /* Checksum this amount of the request */ #define RC_CSUMLEN (256U) +int nfsd_drc_slab_create(void); +void nfsd_drc_slab_free(void); int nfsd_reply_cache_init(struct nfsd_net *); void nfsd_reply_cache_shutdown(struct nfsd_net *); int nfsd_cache_lookup(struct svc_rqst *); diff --git a/fs/nfsd/netns.h b/fs/nfsd/netns.h index 09aa545825bd..9217cb64bf0e 100644 --- a/fs/nfsd/netns.h +++ b/fs/nfsd/netns.h @@ -139,7 +139,6 @@ struct nfsd_net { * Duplicate reply cache */ struct nfsd_drc_bucket *drc_hashtbl; - struct kmem_cache *drc_slab; /* max number of entries allowed in the cache */ unsigned int max_drc_entries; diff --git a/fs/nfsd/nfs4callback.c b/fs/nfsd/nfs4callback.c index 5cf91322de0f..7fbe9840a03e 100644 --- a/fs/nfsd/nfs4callback.c +++ b/fs/nfsd/nfs4callback.c @@ -38,6 +38,7 @@ #include "nfsd.h" #include "state.h" #include "netns.h" +#include "trace.h" #include "xdr4cb.h" #include "xdr4.h" @@ -904,16 +905,20 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c if (clp->cl_minorversion == 0) { if (!clp->cl_cred.cr_principal && - (clp->cl_cred.cr_flavor >= RPC_AUTH_GSS_KRB5)) + (clp->cl_cred.cr_flavor >= RPC_AUTH_GSS_KRB5)) { + trace_nfsd_cb_setup_err(clp, -EINVAL); return -EINVAL; + } args.client_name = clp->cl_cred.cr_principal; args.prognumber = conn->cb_prog; args.protocol = XPRT_TRANSPORT_TCP; args.authflavor = clp->cl_cred.cr_flavor; clp->cl_cb_ident = conn->cb_ident; } else { - if (!conn->cb_xprt) + if (!conn->cb_xprt) { + trace_nfsd_cb_setup_err(clp, -EINVAL); return -EINVAL; + } clp->cl_cb_conn.cb_xprt = conn->cb_xprt; clp->cl_cb_session = ses; args.bc_xprt = conn->cb_xprt; @@ -925,32 +930,27 @@ static int setup_callback_client(struct nfs4_client *clp, struct nfs4_cb_conn *c /* Create RPC client */ client = rpc_create(&args); if (IS_ERR(client)) { - dprintk("NFSD: couldn't create callback client: %ld\n", - PTR_ERR(client)); + trace_nfsd_cb_setup_err(clp, PTR_ERR(client)); return PTR_ERR(client); } cred = get_backchannel_cred(clp, client, ses); if (!cred) { + trace_nfsd_cb_setup_err(clp, -ENOMEM); rpc_shutdown_client(client); return -ENOMEM; } clp->cl_cb_client = client; clp->cl_cb_cred = cred; + trace_nfsd_cb_setup(clp); return 0; } -static void warn_no_callback_path(struct nfs4_client *clp, int reason) -{ - dprintk("NFSD: warning: no callback path to client %.*s: error %d\n", - (int)clp->cl_name.len, clp->cl_name.data, reason); -} - static void nfsd4_mark_cb_down(struct nfs4_client *clp, int reason) { if (test_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags)) return; clp->cl_cb_state = NFSD4_CB_DOWN; - warn_no_callback_path(clp, reason); + trace_nfsd_cb_state(clp); } static void nfsd4_mark_cb_fault(struct nfs4_client *clp, int reason) @@ -958,17 +958,20 @@ static void nfsd4_mark_cb_fault(struct nfs4_client *clp, int reason) if (test_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags)) return; clp->cl_cb_state = NFSD4_CB_FAULT; - warn_no_callback_path(clp, reason); + trace_nfsd_cb_state(clp); } static void nfsd4_cb_probe_done(struct rpc_task *task, void *calldata) { struct nfs4_client *clp = container_of(calldata, struct nfs4_client, cl_cb_null); + trace_nfsd_cb_done(clp, task->tk_status); if (task->tk_status) nfsd4_mark_cb_down(clp, task->tk_status); - else + else { clp->cl_cb_state = NFSD4_CB_UP; + trace_nfsd_cb_state(clp); + } } static void nfsd4_cb_probe_release(void *calldata) @@ -993,6 +996,7 @@ static const struct rpc_call_ops nfsd4_cb_probe_ops = { void nfsd4_probe_callback(struct nfs4_client *clp) { clp->cl_cb_state = NFSD4_CB_UNKNOWN; + trace_nfsd_cb_state(clp); set_bit(NFSD4_CLIENT_CB_UPDATE, &clp->cl_flags); nfsd4_run_cb(&clp->cl_cb_null); } @@ -1009,6 +1013,7 @@ void nfsd4_change_callback(struct nfs4_client *clp, struct nfs4_cb_conn *conn) spin_lock(&clp->cl_lock); memcpy(&clp->cl_cb_conn, conn, sizeof(struct nfs4_cb_conn)); spin_unlock(&clp->cl_lock); + trace_nfsd_cb_state(clp); } /* @@ -1165,8 +1170,7 @@ static void nfsd4_cb_done(struct rpc_task *task, void *calldata) struct nfsd4_callback *cb = calldata; struct nfs4_client *clp = cb->cb_clp; - dprintk("%s: minorversion=%d\n", __func__, - clp->cl_minorversion); + trace_nfsd_cb_done(clp, task->tk_status); if (!nfsd4_cb_sequence_done(task, cb)) return; @@ -1271,6 +1275,7 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) * kill the old client: */ if (clp->cl_cb_client) { + trace_nfsd_cb_shutdown(clp); rpc_shutdown_client(clp->cl_cb_client); clp->cl_cb_client = NULL; put_cred(clp->cl_cb_cred); @@ -1301,6 +1306,8 @@ static void nfsd4_process_cb_update(struct nfsd4_callback *cb) err = setup_callback_client(clp, &conn, ses); if (err) { nfsd4_mark_cb_down(clp, err); + if (c) + svc_xprt_put(c->cn_xprt); return; } } @@ -1314,6 +1321,8 @@ nfsd4_run_cb_work(struct work_struct *work) struct rpc_clnt *clnt; int flags; + trace_nfsd_cb_work(clp, cb->cb_msg.rpc_proc->p_name); + if (cb->cb_need_restart) { cb->cb_need_restart = false; } else { diff --git a/fs/nfsd/nfs4proc.c b/fs/nfsd/nfs4proc.c index 0e75f7fb5fec..a09c35f0f6f0 100644 --- a/fs/nfsd/nfs4proc.c +++ b/fs/nfsd/nfs4proc.c @@ -1155,7 +1155,7 @@ extern void nfs_sb_deactive(struct super_block *sb); #define NFSD42_INTERSSC_MOUNTOPS "vers=4.2,addr=%s,sec=sys" -/** +/* * Support one copy source server for now. */ static __be32 @@ -1245,10 +1245,9 @@ nfsd4_interssc_disconnect(struct vfsmount *ss_mnt) mntput(ss_mnt); } -/** - * nfsd4_setup_inter_ssc - * +/* * Verify COPY destination stateid. + * * Connect to the source server with NFSv4.1. * Create the source struct file for nfsd_copy_range. * Called with COPY cstate: @@ -2302,6 +2301,8 @@ nfsd4_proc_compound(struct svc_rqst *rqstp) } check_if_stalefh_allowed(args); + rqstp->rq_lease_breaker = (void **)&cstate->clp; + trace_nfsd_compound(rqstp, args->opcnt); while (!status && resp->opcnt < args->opcnt) { op = &args->ops[resp->opcnt++]; diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index c107caa56525..bb3d2c32664a 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -51,6 +51,7 @@ #include "netns.h" #include "pnfs.h" #include "filecache.h" +#include "trace.h" #define NFSDDBG_FACILITY NFSDDBG_PROC @@ -167,9 +168,6 @@ renew_client_locked(struct nfs4_client *clp) return; } - dprintk("renewing client (clientid %08x/%08x)\n", - clp->cl_clientid.cl_boot, - clp->cl_clientid.cl_id); list_move_tail(&clp->cl_lru, &nn->client_lru); clp->cl_time = ktime_get_boottime_seconds(); } @@ -1922,8 +1920,7 @@ STALE_CLIENTID(clientid_t *clid, struct nfsd_net *nn) */ if (clid->cl_boot == (u32)nn->boot_time) return 0; - dprintk("NFSD stale clientid (%08x/%08x) boot_time %08llx\n", - clid->cl_boot, clid->cl_id, nn->boot_time); + trace_nfsd_clid_stale(clid); return 1; } @@ -2406,6 +2403,11 @@ static void states_stop(struct seq_file *s, void *v) spin_unlock(&clp->cl_lock); } +static void nfs4_show_fname(struct seq_file *s, struct nfsd_file *f) +{ + seq_printf(s, "filename: \"%pD2\"", f->nf_file); +} + static void nfs4_show_superblock(struct seq_file *s, struct nfsd_file *f) { struct inode *inode = f->nf_inode; @@ -2422,6 +2424,12 @@ static void nfs4_show_owner(struct seq_file *s, struct nfs4_stateowner *oo) seq_quote_mem(s, oo->so_owner.data, oo->so_owner.len); } +static void nfs4_show_stateid(struct seq_file *s, stateid_t *stid) +{ + seq_printf(s, "0x%.8x", stid->si_generation); + seq_printf(s, "%12phN", &stid->si_opaque); +} + static int nfs4_show_open(struct seq_file *s, struct nfs4_stid *st) { struct nfs4_ol_stateid *ols; @@ -2437,7 +2445,9 @@ static int nfs4_show_open(struct seq_file *s, struct nfs4_stid *st) nf = st->sc_file; file = find_any_file(nf); - seq_printf(s, "- 0x%16phN: { type: open, ", &st->sc_stateid); + seq_printf(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_printf(s, ": { type: open, "); access = bmap_to_share_mode(ols->st_access_bmap); deny = bmap_to_share_mode(ols->st_deny_bmap); @@ -2451,6 +2461,8 @@ static int nfs4_show_open(struct seq_file *s, struct nfs4_stid *st) nfs4_show_superblock(s, file); seq_printf(s, ", "); + nfs4_show_fname(s, file); + seq_printf(s, ", "); nfs4_show_owner(s, oo); seq_printf(s, " }\n"); nfsd_file_put(file); @@ -2470,7 +2482,9 @@ static int nfs4_show_lock(struct seq_file *s, struct nfs4_stid *st) nf = st->sc_file; file = find_any_file(nf); - seq_printf(s, "- 0x%16phN: { type: lock, ", &st->sc_stateid); + seq_printf(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_printf(s, ": { type: lock, "); /* * Note: a lock stateid isn't really the same thing as a lock, @@ -2482,6 +2496,8 @@ static int nfs4_show_lock(struct seq_file *s, struct nfs4_stid *st) nfs4_show_superblock(s, file); /* XXX: open stateid? */ seq_printf(s, ", "); + nfs4_show_fname(s, file); + seq_printf(s, ", "); nfs4_show_owner(s, oo); seq_printf(s, " }\n"); nfsd_file_put(file); @@ -2499,7 +2515,9 @@ static int nfs4_show_deleg(struct seq_file *s, struct nfs4_stid *st) nf = st->sc_file; file = nf->fi_deleg_file; - seq_printf(s, "- 0x%16phN: { type: deleg, ", &st->sc_stateid); + seq_printf(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_printf(s, ": { type: deleg, "); /* Kinda dead code as long as we only support read delegs: */ seq_printf(s, "access: %s, ", @@ -2508,6 +2526,8 @@ static int nfs4_show_deleg(struct seq_file *s, struct nfs4_stid *st) /* XXX: lease time, whether it's being recalled. */ nfs4_show_superblock(s, file); + seq_printf(s, ", "); + nfs4_show_fname(s, file); seq_printf(s, " }\n"); return 0; @@ -2521,11 +2541,15 @@ static int nfs4_show_layout(struct seq_file *s, struct nfs4_stid *st) ls = container_of(st, struct nfs4_layout_stateid, ls_stid); file = ls->ls_file; - seq_printf(s, "- 0x%16phN: { type: layout, ", &st->sc_stateid); + seq_printf(s, "- "); + nfs4_show_stateid(s, &st->sc_stateid); + seq_printf(s, ": { type: layout, "); /* XXX: What else would be useful? */ nfs4_show_superblock(s, file); + seq_printf(s, ", "); + nfs4_show_fname(s, file); seq_printf(s, " }\n"); return 0; @@ -2845,14 +2869,12 @@ gen_callback(struct nfs4_client *clp, struct nfsd4_setclientid *se, struct svc_r conn->cb_prog = se->se_callback_prog; conn->cb_ident = se->se_callback_ident; memcpy(&conn->cb_saddr, &rqstp->rq_daddr, rqstp->rq_daddrlen); + trace_nfsd_cb_args(clp, conn); return; out_err: conn->cb_addr.ss_family = AF_UNSPEC; conn->cb_addrlen = 0; - dprintk("NFSD: this client (clientid %08x/%08x) " - "will not receive delegations\n", - clp->cl_clientid.cl_boot, clp->cl_clientid.cl_id); - + trace_nfsd_cb_nodelegs(clp); return; } @@ -3458,6 +3480,45 @@ __be32 nfsd4_backchannel_ctl(struct svc_rqst *rqstp, return nfs_ok; } +static struct nfsd4_conn *__nfsd4_find_conn(struct svc_xprt *xpt, struct nfsd4_session *s) +{ + struct nfsd4_conn *c; + + list_for_each_entry(c, &s->se_conns, cn_persession) { + if (c->cn_xprt == xpt) { + return c; + } + } + return NULL; +} + +static __be32 nfsd4_match_existing_connection(struct svc_rqst *rqst, + struct nfsd4_session *session, u32 req) +{ + struct nfs4_client *clp = session->se_client; + struct svc_xprt *xpt = rqst->rq_xprt; + struct nfsd4_conn *c; + __be32 status; + + /* Following the last paragraph of RFC 5661 Section 18.34.3: */ + spin_lock(&clp->cl_lock); + c = __nfsd4_find_conn(xpt, session); + if (!c) + status = nfserr_noent; + else if (req == c->cn_flags) + status = nfs_ok; + else if (req == NFS4_CDFC4_FORE_OR_BOTH && + c->cn_flags != NFS4_CDFC4_BACK) + status = nfs_ok; + else if (req == NFS4_CDFC4_BACK_OR_BOTH && + c->cn_flags != NFS4_CDFC4_FORE) + status = nfs_ok; + else + status = nfserr_inval; + spin_unlock(&clp->cl_lock); + return status; +} + __be32 nfsd4_bind_conn_to_session(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, union nfsd4_op_u *u) @@ -3479,6 +3540,9 @@ __be32 nfsd4_bind_conn_to_session(struct svc_rqst *rqstp, status = nfserr_wrong_cred; if (!nfsd4_mach_creds_match(session->se_client, rqstp)) goto out; + status = nfsd4_match_existing_connection(rqstp, session, bcts->dir); + if (status == nfs_ok || status == nfserr_inval) + goto out; status = nfsd4_map_bcts_dir(&bcts->dir); if (status) goto out; @@ -3544,18 +3608,6 @@ out: return status; } -static struct nfsd4_conn *__nfsd4_find_conn(struct svc_xprt *xpt, struct nfsd4_session *s) -{ - struct nfsd4_conn *c; - - list_for_each_entry(c, &s->se_conns, cn_persession) { - if (c->cn_xprt == xpt) { - return c; - } - } - return NULL; -} - static __be32 nfsd4_sequence_check_conn(struct nfsd4_conn *new, struct nfsd4_session *ses) { struct nfs4_client *clp = ses->se_client; @@ -3879,23 +3931,18 @@ nfsd4_setclientid(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, if (clp_used_exchangeid(conf)) goto out; if (!same_creds(&conf->cl_cred, &rqstp->rq_cred)) { - char addr_str[INET6_ADDRSTRLEN]; - rpc_ntop((struct sockaddr *) &conf->cl_addr, addr_str, - sizeof(addr_str)); - dprintk("NFSD: setclientid: string in use by client " - "at %s\n", addr_str); + trace_nfsd_clid_inuse_err(conf); goto out; } } unconf = find_unconfirmed_client_by_name(&clname, nn); if (unconf) unhash_client_locked(unconf); + /* We need to handle only case 1: probable callback update */ if (conf && same_verf(&conf->cl_verifier, &clverifier)) { - /* case 1: probable callback update */ copy_clid(new, conf); gen_confirm(new, nn); - } else /* case 4 (new client) or cases 2, 3 (client reboot): */ - ; + } new->cl_minorversion = 0; gen_callback(new, setclid, rqstp); add_to_unconfirmed(new); @@ -4076,7 +4123,6 @@ out_free_openowner_slab: out_free_client_slab: kmem_cache_destroy(client_slab); out: - dprintk("nfsd4: out of memory while initializing nfsv4\n"); return -ENOMEM; } @@ -4508,6 +4554,8 @@ nfsd_break_deleg_cb(struct file_lock *fl) struct nfs4_delegation *dp = (struct nfs4_delegation *)fl->fl_owner; struct nfs4_file *fp = dp->dl_stid.sc_file; + trace_nfsd_deleg_break(&dp->dl_stid.sc_stateid); + /* * We don't want the locks code to timeout the lease for us; * we'll remove it ourself if a delegation isn't returned @@ -4522,6 +4570,19 @@ nfsd_break_deleg_cb(struct file_lock *fl) return ret; } +static bool nfsd_breaker_owns_lease(struct file_lock *fl) +{ + struct nfs4_delegation *dl = fl->fl_owner; + struct svc_rqst *rqst; + struct nfs4_client *clp; + + if (!i_am_nfsd()) + return NULL; + rqst = kthread_data(current); + clp = *(rqst->rq_lease_breaker); + return dl->dl_stid.sc_client == clp; +} + static int nfsd_change_deleg_cb(struct file_lock *onlist, int arg, struct list_head *dispose) @@ -4533,6 +4594,7 @@ nfsd_change_deleg_cb(struct file_lock *onlist, int arg, } static const struct lock_manager_operations nfsd_lease_mng_ops = { + .lm_breaker_owns_lease = nfsd_breaker_owns_lease, .lm_break = nfsd_break_deleg_cb, .lm_change = nfsd_change_deleg_cb, }; @@ -5018,8 +5080,7 @@ nfs4_open_delegation(struct svc_fh *fh, struct nfsd4_open *open, memcpy(&open->op_delegate_stateid, &dp->dl_stid.sc_stateid, sizeof(dp->dl_stid.sc_stateid)); - dprintk("NFSD: delegation stateid=" STATEID_FMT "\n", - STATEID_VAL(&dp->dl_stid.sc_stateid)); + trace_nfsd_deleg_open(&dp->dl_stid.sc_stateid); open->op_delegate_type = NFS4_OPEN_DELEGATE_READ; nfs4_put_stid(&dp->dl_stid); return; @@ -5136,9 +5197,7 @@ nfsd4_process_open2(struct svc_rqst *rqstp, struct svc_fh *current_fh, struct nf nfs4_open_delegation(current_fh, open, stp); nodeleg: status = nfs_ok; - - dprintk("%s: stateid=" STATEID_FMT "\n", __func__, - STATEID_VAL(&stp->st_stid.sc_stateid)); + trace_nfsd_deleg_none(&stp->st_stid.sc_stateid); out: /* 4.1 client trying to upgrade/downgrade delegation? */ if (open->op_delegate_type == NFS4_OPEN_DELEGATE_NONE && dp && @@ -5192,8 +5251,7 @@ nfsd4_renew(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, __be32 status; struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); - dprintk("process_renew(%08x/%08x): starting\n", - clid->cl_boot, clid->cl_id); + trace_nfsd_clid_renew(clid); status = lookup_clientid(clid, cstate, nn, false); if (status) goto out; @@ -5214,6 +5272,7 @@ nfsd4_end_grace(struct nfsd_net *nn) if (nn->grace_ended) return; + trace_nfsd_grace_complete(nn); nn->grace_ended = true; /* * If the server goes down again right now, an NFSv4 @@ -5279,13 +5338,10 @@ nfs4_laundromat(struct nfsd_net *nn) copy_stateid_t *cps_t; int i; - dprintk("NFSD: laundromat service - starting\n"); - if (clients_still_reclaiming(nn)) { new_timeo = 0; goto out; } - dprintk("NFSD: end of grace period\n"); nfsd4_end_grace(nn); INIT_LIST_HEAD(&reaplist); @@ -5307,8 +5363,7 @@ nfs4_laundromat(struct nfsd_net *nn) break; } if (mark_client_expired_locked(clp)) { - dprintk("NFSD: client in use (clientid %08x)\n", - clp->cl_clientid.cl_id); + trace_nfsd_clid_expired(&clp->cl_clientid); continue; } list_add(&clp->cl_lru, &reaplist); @@ -5316,8 +5371,7 @@ nfs4_laundromat(struct nfsd_net *nn) spin_unlock(&nn->client_lock); list_for_each_safe(pos, next, &reaplist) { clp = list_entry(pos, struct nfs4_client, cl_lru); - dprintk("NFSD: purging unused client (clientid %08x)\n", - clp->cl_clientid.cl_id); + trace_nfsd_clid_purged(&clp->cl_clientid); list_del_init(&clp->cl_lru); expire_client(clp); } @@ -5407,7 +5461,6 @@ laundromat_main(struct work_struct *laundry) laundromat_work); t = nfs4_laundromat(nn); - dprintk("NFSD: laundromat_main - sleeping for %lld seconds\n", t); queue_delayed_work(laundry_wq, &nn->laundromat_work, t*HZ); } @@ -5948,8 +6001,7 @@ nfs4_preprocess_seqid_op(struct nfsd4_compound_state *cstate, u32 seqid, struct nfs4_stid *s; struct nfs4_ol_stateid *stp = NULL; - dprintk("NFSD: %s: seqid=%d stateid = " STATEID_FMT "\n", __func__, - seqid, STATEID_VAL(stateid)); + trace_nfsd_preprocess(seqid, stateid); *stpp = NULL; status = nfsd4_lookup_stateid(cstate, stateid, typemask, &s, nn); @@ -6018,9 +6070,7 @@ nfsd4_open_confirm(struct svc_rqst *rqstp, struct nfsd4_compound_state *cstate, oo->oo_flags |= NFS4_OO_CONFIRMED; nfs4_inc_and_copy_stateid(&oc->oc_resp_stateid, &stp->st_stid); mutex_unlock(&stp->st_mutex); - dprintk("NFSD: %s: success, seqid=%d stateid=" STATEID_FMT "\n", - __func__, oc->oc_seqid, STATEID_VAL(&stp->st_stid.sc_stateid)); - + trace_nfsd_open_confirm(oc->oc_seqid, &stp->st_stid.sc_stateid); nfsd4_client_record_create(oo->oo_owner.so_client); status = nfs_ok; put_stateid: @@ -7072,7 +7122,7 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, unsigned int strhashval; struct nfs4_client_reclaim *crp; - dprintk("NFSD nfs4_client_to_reclaim NAME: %.*s\n", name.len, name.data); + trace_nfsd_clid_reclaim(nn, name.len, name.data); crp = alloc_reclaim(); if (crp) { strhashval = clientstr_hashval(name); @@ -7122,7 +7172,7 @@ nfsd4_find_reclaim_client(struct xdr_netobj name, struct nfsd_net *nn) unsigned int strhashval; struct nfs4_client_reclaim *crp = NULL; - dprintk("NFSD: nfs4_find_reclaim_client for name %.*s\n", name.len, name.data); + trace_nfsd_clid_find(nn, name.len, name.data); strhashval = clientstr_hashval(name); list_for_each_entry(crp, &nn->reclaim_str_hashtbl[strhashval], cr_strhash) { @@ -7686,6 +7736,9 @@ nfsd_recall_delegations(struct list_head *reaplist) list_for_each_entry_safe(dp, next, reaplist, dl_recall_lru) { list_del_init(&dp->dl_recall_lru); clp = dp->dl_stid.sc_client; + + trace_nfsd_deleg_recall(&dp->dl_stid.sc_stateid); + /* * We skipped all entries that had a zero dl_time before, * so we can now reset the dl_time back to 0. If a delegation @@ -7868,6 +7921,7 @@ nfs4_state_start_net(struct net *net) goto skip_grace; printk(KERN_INFO "NFSD: starting %lld-second grace period (net %x)\n", nn->nfsd4_grace, net->ns.inum); + trace_nfsd_grace_start(nn); queue_delayed_work(laundry_wq, &nn->laundromat_work, nn->nfsd4_grace * HZ); return 0; diff --git a/fs/nfsd/nfscache.c b/fs/nfsd/nfscache.c index 96352ab7bd81..0a0cf1fd77d3 100644 --- a/fs/nfsd/nfscache.c +++ b/fs/nfsd/nfscache.c @@ -20,8 +20,7 @@ #include "nfsd.h" #include "cache.h" - -#define NFSDDBG_FACILITY NFSDDBG_REPCACHE +#include "trace.h" /* * We use this value to determine the number of hash buckets from the max @@ -36,6 +35,8 @@ struct nfsd_drc_bucket { spinlock_t cache_lock; }; +static struct kmem_cache *drc_slab; + static int nfsd_cache_append(struct svc_rqst *rqstp, struct kvec *vec); static unsigned long nfsd_reply_cache_count(struct shrinker *shrink, struct shrink_control *sc); @@ -95,7 +96,7 @@ nfsd_reply_cache_alloc(struct svc_rqst *rqstp, __wsum csum, { struct svc_cacherep *rp; - rp = kmem_cache_alloc(nn->drc_slab, GFP_KERNEL); + rp = kmem_cache_alloc(drc_slab, GFP_KERNEL); if (rp) { rp->c_state = RC_UNUSED; rp->c_type = RC_NOCACHE; @@ -129,7 +130,7 @@ nfsd_reply_cache_free_locked(struct nfsd_drc_bucket *b, struct svc_cacherep *rp, atomic_dec(&nn->num_drc_entries); nn->drc_mem_usage -= sizeof(*rp); } - kmem_cache_free(nn->drc_slab, rp); + kmem_cache_free(drc_slab, rp); } static void @@ -141,6 +142,18 @@ nfsd_reply_cache_free(struct nfsd_drc_bucket *b, struct svc_cacherep *rp, spin_unlock(&b->cache_lock); } +int nfsd_drc_slab_create(void) +{ + drc_slab = kmem_cache_create("nfsd_drc", + sizeof(struct svc_cacherep), 0, 0, NULL); + return drc_slab ? 0: -ENOMEM; +} + +void nfsd_drc_slab_free(void) +{ + kmem_cache_destroy(drc_slab); +} + int nfsd_reply_cache_init(struct nfsd_net *nn) { unsigned int hashsize; @@ -159,18 +172,13 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) if (status) goto out_nomem; - nn->drc_slab = kmem_cache_create("nfsd_drc", - sizeof(struct svc_cacherep), 0, 0, NULL); - if (!nn->drc_slab) - goto out_shrinker; - nn->drc_hashtbl = kcalloc(hashsize, sizeof(*nn->drc_hashtbl), GFP_KERNEL); if (!nn->drc_hashtbl) { nn->drc_hashtbl = vzalloc(array_size(hashsize, sizeof(*nn->drc_hashtbl))); if (!nn->drc_hashtbl) - goto out_slab; + goto out_shrinker; } for (i = 0; i < hashsize; i++) { @@ -180,8 +188,6 @@ int nfsd_reply_cache_init(struct nfsd_net *nn) nn->drc_hashsize = hashsize; return 0; -out_slab: - kmem_cache_destroy(nn->drc_slab); out_shrinker: unregister_shrinker(&nn->nfsd_reply_cache_shrinker); out_nomem: @@ -209,8 +215,6 @@ void nfsd_reply_cache_shutdown(struct nfsd_net *nn) nn->drc_hashtbl = NULL; nn->drc_hashsize = 0; - kmem_cache_destroy(nn->drc_slab); - nn->drc_slab = NULL; } /* @@ -323,8 +327,10 @@ nfsd_cache_key_cmp(const struct svc_cacherep *key, const struct svc_cacherep *rp, struct nfsd_net *nn) { if (key->c_key.k_xid == rp->c_key.k_xid && - key->c_key.k_csum != rp->c_key.k_csum) + key->c_key.k_csum != rp->c_key.k_csum) { ++nn->payload_misses; + trace_nfsd_drc_mismatch(nn, key, rp); + } return memcmp(&key->c_key, &rp->c_key, sizeof(key->c_key)); } @@ -377,15 +383,22 @@ out: return ret; } -/* +/** + * nfsd_cache_lookup - Find an entry in the duplicate reply cache + * @rqstp: Incoming Call to find + * * Try to find an entry matching the current call in the cache. When none * is found, we try to grab the oldest expired entry off the LRU list. If * a suitable one isn't there, then drop the cache_lock and allocate a * new one, then search again in case one got inserted while this thread * didn't hold the lock. + * + * Return values: + * %RC_DOIT: Process the request normally + * %RC_REPLY: Reply from cache + * %RC_DROPIT: Do not process the request further */ -int -nfsd_cache_lookup(struct svc_rqst *rqstp) +int nfsd_cache_lookup(struct svc_rqst *rqstp) { struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct svc_cacherep *rp, *found; @@ -399,7 +412,7 @@ nfsd_cache_lookup(struct svc_rqst *rqstp) rqstp->rq_cacherep = NULL; if (type == RC_NOCACHE) { nfsdstats.rcnocache++; - return rtn; + goto out; } csum = nfsd_cache_csum(rqstp); @@ -409,10 +422,8 @@ nfsd_cache_lookup(struct svc_rqst *rqstp) * preallocate an entry. */ rp = nfsd_reply_cache_alloc(rqstp, csum, nn); - if (!rp) { - dprintk("nfsd: unable to allocate DRC entry!\n"); - return rtn; - } + if (!rp) + goto out; spin_lock(&b->cache_lock); found = nfsd_cache_insert(b, rp, nn); @@ -431,8 +442,10 @@ nfsd_cache_lookup(struct svc_rqst *rqstp) /* go ahead and prune the cache */ prune_bucket(b, nn); - out: + +out_unlock: spin_unlock(&b->cache_lock); +out: return rtn; found_entry: @@ -442,13 +455,13 @@ found_entry: /* Request being processed */ if (rp->c_state == RC_INPROG) - goto out; + goto out_trace; /* From the hall of fame of impractical attacks: * Is this a user who tries to snoop on the cache? */ rtn = RC_DOIT; if (!test_bit(RQ_SECURE, &rqstp->rq_flags) && rp->c_secure) - goto out; + goto out_trace; /* Compose RPC reply header */ switch (rp->c_type) { @@ -460,21 +473,26 @@ found_entry: break; case RC_REPLBUFF: if (!nfsd_cache_append(rqstp, &rp->c_replvec)) - goto out; /* should not happen */ + goto out_unlock; /* should not happen */ rtn = RC_REPLY; break; default: - printk(KERN_WARNING "nfsd: bad repcache type %d\n", rp->c_type); - nfsd_reply_cache_free_locked(b, rp, nn); + WARN_ONCE(1, "nfsd: bad repcache type %d\n", rp->c_type); } - goto out; +out_trace: + trace_nfsd_drc_found(nn, rqstp, rtn); + goto out_unlock; } -/* - * Update a cache entry. This is called from nfsd_dispatch when - * the procedure has been executed and the complete reply is in - * rqstp->rq_res. +/** + * nfsd_cache_update - Update an entry in the duplicate reply cache. + * @rqstp: svc_rqst with a finished Reply + * @cachetype: which cache to update + * @statp: Reply's status code + * + * This is called from nfsd_dispatch when the procedure has been + * executed and the complete reply is in rqstp->rq_res. * * We're copying around data here rather than swapping buffers because * the toplevel loop requires max-sized buffers, which would be a waste @@ -487,8 +505,7 @@ found_entry: * nfsd failed to encode a reply that otherwise would have been cached. * In this case, nfsd_cache_update is called with statp == NULL. */ -void -nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp) +void nfsd_cache_update(struct svc_rqst *rqstp, int cachetype, __be32 *statp) { struct nfsd_net *nn = net_generic(SVC_NET(rqstp), nfsd_net_id); struct svc_cacherep *rp = rqstp->rq_cacherep; diff --git a/fs/nfsd/nfsctl.c b/fs/nfsd/nfsctl.c index 3bb2db947d29..b68e96681522 100644 --- a/fs/nfsd/nfsctl.c +++ b/fs/nfsd/nfsctl.c @@ -238,7 +238,7 @@ static inline struct net *netns(struct file *file) return file_inode(file)->i_sb->s_fs_info; } -/** +/* * write_unlock_ip - Release all locks used by a client * * Experimental. @@ -277,7 +277,7 @@ static ssize_t write_unlock_ip(struct file *file, char *buf, size_t size) return nlmsvc_unlock_all_by_ip(sap); } -/** +/* * write_unlock_fs - Release all locks on a local file system * * Experimental. @@ -327,7 +327,7 @@ static ssize_t write_unlock_fs(struct file *file, char *buf, size_t size) return error; } -/** +/* * write_filehandle - Get a variable-length NFS file handle by path * * On input, the buffer contains a '\n'-terminated C string comprised of @@ -402,7 +402,7 @@ static ssize_t write_filehandle(struct file *file, char *buf, size_t size) return mesg - buf; } -/** +/* * write_threads - Start NFSD, or report the current number of running threads * * Input: @@ -452,7 +452,7 @@ static ssize_t write_threads(struct file *file, char *buf, size_t size) return scnprintf(buf, SIMPLE_TRANSACTION_LIMIT, "%d\n", rv); } -/** +/* * write_pool_threads - Set or report the current number of threads per pool * * Input: @@ -661,7 +661,7 @@ out: return tlen + len; } -/** +/* * write_versions - Set or report the available NFS protocol versions * * Input: @@ -811,7 +811,7 @@ static ssize_t __write_ports(struct file *file, char *buf, size_t size, return -EINVAL; } -/** +/* * write_ports - Pass a socket file descriptor or transport name to listen on * * Input: @@ -867,7 +867,7 @@ static ssize_t write_ports(struct file *file, char *buf, size_t size) int nfsd_max_blksize; -/** +/* * write_maxblksize - Set or report the current NFS blksize * * Input: @@ -917,7 +917,7 @@ static ssize_t write_maxblksize(struct file *file, char *buf, size_t size) nfsd_max_blksize); } -/** +/* * write_maxconn - Set or report the current max number of connections * * Input: @@ -998,7 +998,7 @@ static ssize_t nfsd4_write_time(struct file *file, char *buf, size_t size, return rv; } -/** +/* * write_leasetime - Set or report the current NFSv4 lease time * * Input: @@ -1025,7 +1025,7 @@ static ssize_t write_leasetime(struct file *file, char *buf, size_t size) return nfsd4_write_time(file, buf, size, &nn->nfsd4_lease, nn); } -/** +/* * write_gracetime - Set or report current NFSv4 grace period time * * As above, but sets the time of the NFSv4 grace period. @@ -1069,7 +1069,7 @@ static ssize_t __write_recoverydir(struct file *file, char *buf, size_t size, nfs4_recoverydir()); } -/** +/* * write_recoverydir - Set or report the pathname of the recovery directory * * Input: @@ -1101,7 +1101,7 @@ static ssize_t write_recoverydir(struct file *file, char *buf, size_t size) return rv; } -/** +/* * write_v4_end_grace - release grace period for nfsd's v4.x lock manager * * Input: @@ -1533,6 +1533,9 @@ static int __init init_nfsd(void) goto out_free_slabs; nfsd_fault_inject_init(); /* nfsd fault injection controls */ nfsd_stat_init(); /* Statistics */ + retval = nfsd_drc_slab_create(); + if (retval) + goto out_free_stat; nfsd_lockd_init(); /* lockd->nfsd callbacks */ retval = create_proc_exports_entry(); if (retval) @@ -1546,6 +1549,8 @@ out_free_all: remove_proc_entry("fs/nfs", NULL); out_free_lockd: nfsd_lockd_shutdown(); + nfsd_drc_slab_free(); +out_free_stat: nfsd_stat_shutdown(); nfsd_fault_inject_cleanup(); nfsd4_exit_pnfs(); @@ -1560,6 +1565,7 @@ out_unregister_pernet: static void __exit exit_nfsd(void) { + nfsd_drc_slab_free(); remove_proc_entry("fs/nfs/exports", NULL); remove_proc_entry("fs/nfs", NULL); nfsd_stat_shutdown(); diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index 2ab5569126b8..36cdd81b6688 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -88,6 +88,8 @@ int nfsd_pool_stats_release(struct inode *, struct file *); void nfsd_destroy(struct net *net); +bool i_am_nfsd(void); + struct nfsdfs_client { struct kref cl_ref; void (*cl_release)(struct kref *kref); diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index ca9fd348548b..b603dfcdd361 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -601,6 +601,11 @@ static const struct svc_serv_ops nfsd_thread_sv_ops = { .svo_module = THIS_MODULE, }; +bool i_am_nfsd(void) +{ + return kthread_func(current) == nfsd; +} + int nfsd_create_serv(struct net *net) { int error; @@ -1011,6 +1016,7 @@ nfsd_dispatch(struct svc_rqst *rqstp, __be32 *statp) *statp = rpc_garbage_args; return 1; } + rqstp->rq_lease_breaker = NULL; /* * Give the xdr decoder a chance to change this if it wants * (necessary in the NFSv4.0 compound case) diff --git a/fs/nfsd/state.h b/fs/nfsd/state.h index 68d3f30ee760..3b408532a5dc 100644 --- a/fs/nfsd/state.h +++ b/fs/nfsd/state.h @@ -64,13 +64,6 @@ typedef struct { refcount_t sc_count; } copy_stateid_t; -#define STATEID_FMT "(%08x/%08x/%08x/%08x)" -#define STATEID_VAL(s) \ - (s)->si_opaque.so_clid.cl_boot, \ - (s)->si_opaque.so_clid.cl_id, \ - (s)->si_opaque.so_id, \ - (s)->si_generation - struct nfsd4_callback { struct nfs4_client *cb_clp; struct rpc_message cb_msg; diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 78c574251c60..1861db1bdc67 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -277,6 +277,7 @@ DECLARE_EVENT_CLASS(nfsd_stateid_class, DEFINE_EVENT(nfsd_stateid_class, nfsd_##name, \ TP_PROTO(stateid_t *stp), \ TP_ARGS(stp)) + DEFINE_STATEID_EVENT(layoutstate_alloc); DEFINE_STATEID_EVENT(layoutstate_unhash); DEFINE_STATEID_EVENT(layoutstate_free); @@ -288,6 +289,138 @@ DEFINE_STATEID_EVENT(layout_recall_done); DEFINE_STATEID_EVENT(layout_recall_fail); DEFINE_STATEID_EVENT(layout_recall_release); +DEFINE_STATEID_EVENT(deleg_open); +DEFINE_STATEID_EVENT(deleg_none); +DEFINE_STATEID_EVENT(deleg_break); +DEFINE_STATEID_EVENT(deleg_recall); + +DECLARE_EVENT_CLASS(nfsd_stateseqid_class, + TP_PROTO(u32 seqid, const stateid_t *stp), + TP_ARGS(seqid, stp), + TP_STRUCT__entry( + __field(u32, seqid) + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, si_id) + __field(u32, si_generation) + ), + TP_fast_assign( + __entry->seqid = seqid; + __entry->cl_boot = stp->si_opaque.so_clid.cl_boot; + __entry->cl_id = stp->si_opaque.so_clid.cl_id; + __entry->si_id = stp->si_opaque.so_id; + __entry->si_generation = stp->si_generation; + ), + TP_printk("seqid=%u client %08x:%08x stateid %08x:%08x", + __entry->seqid, __entry->cl_boot, __entry->cl_id, + __entry->si_id, __entry->si_generation) +) + +#define DEFINE_STATESEQID_EVENT(name) \ +DEFINE_EVENT(nfsd_stateseqid_class, nfsd_##name, \ + TP_PROTO(u32 seqid, const stateid_t *stp), \ + TP_ARGS(seqid, stp)) + +DEFINE_STATESEQID_EVENT(preprocess); +DEFINE_STATESEQID_EVENT(open_confirm); + +DECLARE_EVENT_CLASS(nfsd_clientid_class, + TP_PROTO(const clientid_t *clid), + TP_ARGS(clid), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + ), + TP_fast_assign( + __entry->cl_boot = clid->cl_boot; + __entry->cl_id = clid->cl_id; + ), + TP_printk("client %08x:%08x", __entry->cl_boot, __entry->cl_id) +) + +#define DEFINE_CLIENTID_EVENT(name) \ +DEFINE_EVENT(nfsd_clientid_class, nfsd_clid_##name, \ + TP_PROTO(const clientid_t *clid), \ + TP_ARGS(clid)) + +DEFINE_CLIENTID_EVENT(expired); +DEFINE_CLIENTID_EVENT(purged); +DEFINE_CLIENTID_EVENT(renew); +DEFINE_CLIENTID_EVENT(stale); + +DECLARE_EVENT_CLASS(nfsd_net_class, + TP_PROTO(const struct nfsd_net *nn), + TP_ARGS(nn), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + ), + TP_printk("boot_time=%16llx", __entry->boot_time) +) + +#define DEFINE_NET_EVENT(name) \ +DEFINE_EVENT(nfsd_net_class, nfsd_##name, \ + TP_PROTO(const struct nfsd_net *nn), \ + TP_ARGS(nn)) + +DEFINE_NET_EVENT(grace_start); +DEFINE_NET_EVENT(grace_complete); + +DECLARE_EVENT_CLASS(nfsd_clid_class, + TP_PROTO(const struct nfsd_net *nn, + unsigned int namelen, + const unsigned char *namedata), + TP_ARGS(nn, namelen, namedata), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(unsigned int, namelen) + __dynamic_array(unsigned char, name, namelen) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->namelen = namelen; + memcpy(__get_dynamic_array(name), namedata, namelen); + ), + TP_printk("boot_time=%16llx nfs4_clientid=%.*s", + __entry->boot_time, __entry->namelen, __get_str(name)) +) + +#define DEFINE_CLID_EVENT(name) \ +DEFINE_EVENT(nfsd_clid_class, nfsd_clid_##name, \ + TP_PROTO(const struct nfsd_net *nn, \ + unsigned int namelen, \ + const unsigned char *namedata), \ + TP_ARGS(nn, namelen, namedata)) + +DEFINE_CLID_EVENT(find); +DEFINE_CLID_EVENT(reclaim); + +TRACE_EVENT(nfsd_clid_inuse_err, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + __field(unsigned int, namelen) + __dynamic_array(unsigned char, name, clp->cl_name.len) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + memcpy(__entry->addr, &clp->cl_addr, + sizeof(struct sockaddr_in6)); + __entry->namelen = clp->cl_name.len; + memcpy(__get_dynamic_array(name), clp->cl_name.data, + clp->cl_name.len); + ), + TP_printk("nfs4_clientid %.*s already in use by %pISpc, client %08x:%08x", + __entry->namelen, __get_str(name), __entry->addr, + __entry->cl_boot, __entry->cl_id) +) + TRACE_DEFINE_ENUM(NFSD_FILE_HASHED); TRACE_DEFINE_ENUM(NFSD_FILE_PENDING); TRACE_DEFINE_ENUM(NFSD_FILE_BREAK_READ); @@ -432,6 +565,218 @@ TRACE_EVENT(nfsd_file_fsnotify_handle_event, __entry->nlink, __entry->mode, __entry->mask) ); +#include "cache.h" + +TRACE_DEFINE_ENUM(RC_DROPIT); +TRACE_DEFINE_ENUM(RC_REPLY); +TRACE_DEFINE_ENUM(RC_DOIT); + +#define show_drc_retval(x) \ + __print_symbolic(x, \ + { RC_DROPIT, "DROPIT" }, \ + { RC_REPLY, "REPLY" }, \ + { RC_DOIT, "DOIT" }) + +TRACE_EVENT(nfsd_drc_found, + TP_PROTO( + const struct nfsd_net *nn, + const struct svc_rqst *rqstp, + int result + ), + TP_ARGS(nn, rqstp, result), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(unsigned long, result) + __field(u32, xid) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->result = result; + __entry->xid = be32_to_cpu(rqstp->rq_xid); + ), + TP_printk("boot_time=%16llx xid=0x%08x result=%s", + __entry->boot_time, __entry->xid, + show_drc_retval(__entry->result)) + +); + +TRACE_EVENT(nfsd_drc_mismatch, + TP_PROTO( + const struct nfsd_net *nn, + const struct svc_cacherep *key, + const struct svc_cacherep *rp + ), + TP_ARGS(nn, key, rp), + TP_STRUCT__entry( + __field(unsigned long long, boot_time) + __field(u32, xid) + __field(u32, cached) + __field(u32, ingress) + ), + TP_fast_assign( + __entry->boot_time = nn->boot_time; + __entry->xid = be32_to_cpu(key->c_key.k_xid); + __entry->cached = (__force u32)key->c_key.k_csum; + __entry->ingress = (__force u32)rp->c_key.k_csum; + ), + TP_printk("boot_time=%16llx xid=0x%08x cached-csum=0x%08x ingress-csum=0x%08x", + __entry->boot_time, __entry->xid, __entry->cached, + __entry->ingress) +); + +TRACE_EVENT(nfsd_cb_args, + TP_PROTO( + const struct nfs4_client *clp, + const struct nfs4_cb_conn *conn + ), + TP_ARGS(clp, conn), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(u32, prog) + __field(u32, ident) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->prog = conn->cb_prog; + __entry->ident = conn->cb_ident; + memcpy(__entry->addr, &conn->cb_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("client %08x:%08x callback addr=%pISpc prog=%u ident=%u", + __entry->cl_boot, __entry->cl_id, + __entry->addr, __entry->prog, __entry->ident) +); + +TRACE_EVENT(nfsd_cb_nodelegs, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + ), + TP_printk("client %08x:%08x", __entry->cl_boot, __entry->cl_id) +) + +TRACE_DEFINE_ENUM(NFSD4_CB_UP); +TRACE_DEFINE_ENUM(NFSD4_CB_UNKNOWN); +TRACE_DEFINE_ENUM(NFSD4_CB_DOWN); +TRACE_DEFINE_ENUM(NFSD4_CB_FAULT); + +#define show_cb_state(val) \ + __print_symbolic(val, \ + { NFSD4_CB_UP, "UP" }, \ + { NFSD4_CB_UNKNOWN, "UNKNOWN" }, \ + { NFSD4_CB_DOWN, "DOWN" }, \ + { NFSD4_CB_FAULT, "FAULT"}) + +DECLARE_EVENT_CLASS(nfsd_cb_class, + TP_PROTO(const struct nfs4_client *clp), + TP_ARGS(clp), + TP_STRUCT__entry( + __field(unsigned long, state) + __field(u32, cl_boot) + __field(u32, cl_id) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->state = clp->cl_cb_state; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + memcpy(__entry->addr, &clp->cl_cb_conn.cb_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("addr=%pISpc client %08x:%08x state=%s", + __entry->addr, __entry->cl_boot, __entry->cl_id, + show_cb_state(__entry->state)) +); + +#define DEFINE_NFSD_CB_EVENT(name) \ +DEFINE_EVENT(nfsd_cb_class, nfsd_cb_##name, \ + TP_PROTO(const struct nfs4_client *clp), \ + TP_ARGS(clp)) + +DEFINE_NFSD_CB_EVENT(setup); +DEFINE_NFSD_CB_EVENT(state); +DEFINE_NFSD_CB_EVENT(shutdown); + +TRACE_EVENT(nfsd_cb_setup_err, + TP_PROTO( + const struct nfs4_client *clp, + long error + ), + TP_ARGS(clp, error), + TP_STRUCT__entry( + __field(long, error) + __field(u32, cl_boot) + __field(u32, cl_id) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->error = error; + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + memcpy(__entry->addr, &clp->cl_cb_conn.cb_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("addr=%pISpc client %08x:%08x error=%ld", + __entry->addr, __entry->cl_boot, __entry->cl_id, __entry->error) +); + +TRACE_EVENT(nfsd_cb_work, + TP_PROTO( + const struct nfs4_client *clp, + const char *procedure + ), + TP_ARGS(clp, procedure), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __string(procedure, procedure) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __assign_str(procedure, procedure) + memcpy(__entry->addr, &clp->cl_cb_conn.cb_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("addr=%pISpc client %08x:%08x procedure=%s", + __entry->addr, __entry->cl_boot, __entry->cl_id, + __get_str(procedure)) +); + +TRACE_EVENT(nfsd_cb_done, + TP_PROTO( + const struct nfs4_client *clp, + int status + ), + TP_ARGS(clp, status), + TP_STRUCT__entry( + __field(u32, cl_boot) + __field(u32, cl_id) + __field(int, status) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + TP_fast_assign( + __entry->cl_boot = clp->cl_clientid.cl_boot; + __entry->cl_id = clp->cl_clientid.cl_id; + __entry->status = status; + memcpy(__entry->addr, &clp->cl_cb_conn.cb_addr, + sizeof(struct sockaddr_in6)); + ), + TP_printk("addr=%pISpc client %08x:%08x status=%d", + __entry->addr, __entry->cl_boot, __entry->cl_id, + __entry->status) +); + #endif /* _NFSD_TRACE_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/nilfs2/segment.c b/fs/nilfs2/segment.c index 445eef41bfaf..91b58c897f92 100644 --- a/fs/nilfs2/segment.c +++ b/fs/nilfs2/segment.c @@ -2780,6 +2780,8 @@ int nilfs_attach_log_writer(struct super_block *sb, struct nilfs_root *root) if (!nilfs->ns_writer) return -ENOMEM; + inode_attach_wb(nilfs->ns_bdev->bd_inode, NULL); + err = nilfs_segctor_start_thread(nilfs->ns_writer); if (err) { kfree(nilfs->ns_writer); diff --git a/fs/ocfs2/Kconfig b/fs/ocfs2/Kconfig index 1177c33df895..aca16624b370 100644 --- a/fs/ocfs2/Kconfig +++ b/fs/ocfs2/Kconfig @@ -1,7 +1,7 @@ # SPDX-License-Identifier: GPL-2.0-only config OCFS2_FS tristate "OCFS2 file system support" - depends on NET && SYSFS && CONFIGFS_FS + depends on INET && SYSFS && CONFIGFS_FS select JBD2 select CRC32 select QUOTA diff --git a/fs/ocfs2/mmap.c b/fs/ocfs2/mmap.c index 3a44e461828a..25cabbfe87fc 100644 --- a/fs/ocfs2/mmap.c +++ b/fs/ocfs2/mmap.c @@ -62,7 +62,7 @@ static vm_fault_t __ocfs2_page_mkwrite(struct file *file, last_index = (size - 1) >> PAGE_SHIFT; /* - * There are cases that lead to the page no longer bebongs to the + * There are cases that lead to the page no longer belonging to the * mapping. * 1) pagecache truncates locally due to memory pressure. * 2) pagecache truncates when another is taking EX lock against diff --git a/fs/pipe.c b/fs/pipe.c index c7c4fb5f345f..60dbee457143 100644 --- a/fs/pipe.c +++ b/fs/pipe.c @@ -24,6 +24,7 @@ #include <linux/syscalls.h> #include <linux/fcntl.h> #include <linux/memcontrol.h> +#include <linux/watch_queue.h> #include <linux/uaccess.h> #include <asm/ioctls.h> @@ -259,14 +260,44 @@ pipe_read(struct kiocb *iocb, struct iov_iter *to) unsigned int tail = pipe->tail; unsigned int mask = pipe->ring_size - 1; +#ifdef CONFIG_WATCH_QUEUE + if (pipe->note_loss) { + struct watch_notification n; + + if (total_len < 8) { + if (ret == 0) + ret = -ENOBUFS; + break; + } + + n.type = WATCH_TYPE_META; + n.subtype = WATCH_META_LOSS_NOTIFICATION; + n.info = watch_sizeof(n); + if (copy_to_iter(&n, sizeof(n), to) != sizeof(n)) { + if (ret == 0) + ret = -EFAULT; + break; + } + ret += sizeof(n); + total_len -= sizeof(n); + pipe->note_loss = false; + } +#endif + if (!pipe_empty(head, tail)) { struct pipe_buffer *buf = &pipe->bufs[tail & mask]; size_t chars = buf->len; size_t written; int error; - if (chars > total_len) + if (chars > total_len) { + if (buf->flags & PIPE_BUF_FLAG_WHOLE) { + if (ret == 0) + ret = -ENOBUFS; + break; + } chars = total_len; + } error = pipe_buf_confirm(pipe, buf); if (error) { @@ -294,6 +325,10 @@ pipe_read(struct kiocb *iocb, struct iov_iter *to) if (!buf->len) { pipe_buf_release(pipe, buf); spin_lock_irq(&pipe->rd_wait.lock); +#ifdef CONFIG_WATCH_QUEUE + if (buf->flags & PIPE_BUF_FLAG_LOSS) + pipe->note_loss = true; +#endif tail++; pipe->tail = tail; spin_unlock_irq(&pipe->rd_wait.lock); @@ -405,6 +440,13 @@ pipe_write(struct kiocb *iocb, struct iov_iter *from) goto out; } +#ifdef CONFIG_WATCH_QUEUE + if (pipe->watch_queue) { + ret = -EXDEV; + goto out; + } +#endif + /* * Only wake up if the pipe started out empty, since * otherwise there should be no readers waiting. @@ -574,22 +616,37 @@ static long pipe_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) int count, head, tail, mask; switch (cmd) { - case FIONREAD: - __pipe_lock(pipe); - count = 0; - head = pipe->head; - tail = pipe->tail; - mask = pipe->ring_size - 1; + case FIONREAD: + __pipe_lock(pipe); + count = 0; + head = pipe->head; + tail = pipe->tail; + mask = pipe->ring_size - 1; - while (tail != head) { - count += pipe->bufs[tail & mask].len; - tail++; - } - __pipe_unlock(pipe); + while (tail != head) { + count += pipe->bufs[tail & mask].len; + tail++; + } + __pipe_unlock(pipe); + + return put_user(count, (int __user *)arg); - return put_user(count, (int __user *)arg); - default: - return -ENOIOCTLCMD; +#ifdef CONFIG_WATCH_QUEUE + case IOC_WATCH_QUEUE_SET_SIZE: { + int ret; + __pipe_lock(pipe); + ret = watch_queue_set_size(pipe, arg); + __pipe_unlock(pipe); + return ret; + } + + case IOC_WATCH_QUEUE_SET_FILTER: + return watch_queue_set_filter( + pipe, (struct watch_notification_filter __user *)arg); +#endif + + default: + return -ENOIOCTLCMD; } } @@ -700,27 +757,27 @@ pipe_fasync(int fd, struct file *filp, int on) return retval; } -static unsigned long account_pipe_buffers(struct user_struct *user, - unsigned long old, unsigned long new) +unsigned long account_pipe_buffers(struct user_struct *user, + unsigned long old, unsigned long new) { return atomic_long_add_return(new - old, &user->pipe_bufs); } -static bool too_many_pipe_buffers_soft(unsigned long user_bufs) +bool too_many_pipe_buffers_soft(unsigned long user_bufs) { unsigned long soft_limit = READ_ONCE(pipe_user_pages_soft); return soft_limit && user_bufs > soft_limit; } -static bool too_many_pipe_buffers_hard(unsigned long user_bufs) +bool too_many_pipe_buffers_hard(unsigned long user_bufs) { unsigned long hard_limit = READ_ONCE(pipe_user_pages_hard); return hard_limit && user_bufs > hard_limit; } -static bool is_unprivileged_user(void) +bool pipe_is_unprivileged_user(void) { return !capable(CAP_SYS_RESOURCE) && !capable(CAP_SYS_ADMIN); } @@ -742,12 +799,12 @@ struct pipe_inode_info *alloc_pipe_info(void) user_bufs = account_pipe_buffers(user, 0, pipe_bufs); - if (too_many_pipe_buffers_soft(user_bufs) && is_unprivileged_user()) { + if (too_many_pipe_buffers_soft(user_bufs) && pipe_is_unprivileged_user()) { user_bufs = account_pipe_buffers(user, pipe_bufs, 1); pipe_bufs = 1; } - if (too_many_pipe_buffers_hard(user_bufs) && is_unprivileged_user()) + if (too_many_pipe_buffers_hard(user_bufs) && pipe_is_unprivileged_user()) goto out_revert_acct; pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer), @@ -759,6 +816,7 @@ struct pipe_inode_info *alloc_pipe_info(void) pipe->r_counter = pipe->w_counter = 1; pipe->max_usage = pipe_bufs; pipe->ring_size = pipe_bufs; + pipe->nr_accounted = pipe_bufs; pipe->user = user; mutex_init(&pipe->mutex); return pipe; @@ -776,7 +834,14 @@ void free_pipe_info(struct pipe_inode_info *pipe) { int i; - (void) account_pipe_buffers(pipe->user, pipe->ring_size, 0); +#ifdef CONFIG_WATCH_QUEUE + if (pipe->watch_queue) { + watch_queue_clear(pipe->watch_queue); + put_watch_queue(pipe->watch_queue); + } +#endif + + (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0); free_uid(pipe->user); for (i = 0; i < pipe->ring_size; i++) { struct pipe_buffer *buf = pipe->bufs + i; @@ -852,6 +917,17 @@ int create_pipe_files(struct file **res, int flags) if (!inode) return -ENFILE; + if (flags & O_NOTIFICATION_PIPE) { +#ifdef CONFIG_WATCH_QUEUE + if (watch_queue_init(inode->i_pipe) < 0) { + iput(inode); + return -ENOMEM; + } +#else + return -ENOPKG; +#endif + } + f = alloc_file_pseudo(inode, pipe_mnt, "", O_WRONLY | (flags & (O_NONBLOCK | O_DIRECT)), &pipefifo_fops); @@ -882,7 +958,7 @@ static int __do_pipe_flags(int *fd, struct file **files, int flags) int error; int fdw, fdr; - if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT)) + if (flags & ~(O_CLOEXEC | O_NONBLOCK | O_DIRECT | O_NOTIFICATION_PIPE)) return -EINVAL; error = create_pipe_files(files, flags); @@ -1130,42 +1206,12 @@ unsigned int round_pipe_size(unsigned long size) } /* - * Allocate a new array of pipe buffers and copy the info over. Returns the - * pipe size if successful, or return -ERROR on error. + * Resize the pipe ring to a number of slots. */ -static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) +int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots) { struct pipe_buffer *bufs; - unsigned int size, nr_slots, head, tail, mask, n; - unsigned long user_bufs; - long ret = 0; - - size = round_pipe_size(arg); - nr_slots = size >> PAGE_SHIFT; - - if (!nr_slots) - return -EINVAL; - - /* - * If trying to increase the pipe capacity, check that an - * unprivileged user is not trying to exceed various limits - * (soft limit check here, hard limit check just below). - * Decreasing the pipe capacity is always permitted, even - * if the user is currently over a limit. - */ - if (nr_slots > pipe->ring_size && - size > pipe_max_size && !capable(CAP_SYS_RESOURCE)) - return -EPERM; - - user_bufs = account_pipe_buffers(pipe->user, pipe->ring_size, nr_slots); - - if (nr_slots > pipe->ring_size && - (too_many_pipe_buffers_hard(user_bufs) || - too_many_pipe_buffers_soft(user_bufs)) && - is_unprivileged_user()) { - ret = -EPERM; - goto out_revert_acct; - } + unsigned int head, tail, mask, n; /* * We can shrink the pipe, if arg is greater than the ring occupancy. @@ -1177,17 +1223,13 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) head = pipe->head; tail = pipe->tail; n = pipe_occupancy(pipe->head, pipe->tail); - if (nr_slots < n) { - ret = -EBUSY; - goto out_revert_acct; - } + if (nr_slots < n) + return -EBUSY; bufs = kcalloc(nr_slots, sizeof(*bufs), GFP_KERNEL_ACCOUNT | __GFP_NOWARN); - if (unlikely(!bufs)) { - ret = -ENOMEM; - goto out_revert_acct; - } + if (unlikely(!bufs)) + return -ENOMEM; /* * The pipe array wraps around, so just start the new one at zero @@ -1215,16 +1257,68 @@ static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) kfree(pipe->bufs); pipe->bufs = bufs; pipe->ring_size = nr_slots; - pipe->max_usage = nr_slots; + if (pipe->max_usage > nr_slots) + pipe->max_usage = nr_slots; pipe->tail = tail; pipe->head = head; /* This might have made more room for writers */ wake_up_interruptible(&pipe->wr_wait); + return 0; +} + +/* + * Allocate a new array of pipe buffers and copy the info over. Returns the + * pipe size if successful, or return -ERROR on error. + */ +static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg) +{ + unsigned long user_bufs; + unsigned int nr_slots, size; + long ret = 0; + +#ifdef CONFIG_WATCH_QUEUE + if (pipe->watch_queue) + return -EBUSY; +#endif + + size = round_pipe_size(arg); + nr_slots = size >> PAGE_SHIFT; + + if (!nr_slots) + return -EINVAL; + + /* + * If trying to increase the pipe capacity, check that an + * unprivileged user is not trying to exceed various limits + * (soft limit check here, hard limit check just below). + * Decreasing the pipe capacity is always permitted, even + * if the user is currently over a limit. + */ + if (nr_slots > pipe->max_usage && + size > pipe_max_size && !capable(CAP_SYS_RESOURCE)) + return -EPERM; + + user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots); + + if (nr_slots > pipe->max_usage && + (too_many_pipe_buffers_hard(user_bufs) || + too_many_pipe_buffers_soft(user_bufs)) && + pipe_is_unprivileged_user()) { + ret = -EPERM; + goto out_revert_acct; + } + + ret = pipe_resize_ring(pipe, nr_slots); + if (ret < 0) + goto out_revert_acct; + + pipe->max_usage = nr_slots; + pipe->nr_accounted = nr_slots; return pipe->max_usage * PAGE_SIZE; out_revert_acct: - (void) account_pipe_buffers(pipe->user, nr_slots, pipe->ring_size); + (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted); return ret; } @@ -1233,9 +1327,17 @@ out_revert_acct: * location, so checking ->i_pipe is not enough to verify that this is a * pipe. */ -struct pipe_inode_info *get_pipe_info(struct file *file) +struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice) { - return file->f_op == &pipefifo_fops ? file->private_data : NULL; + struct pipe_inode_info *pipe = file->private_data; + + if (file->f_op != &pipefifo_fops || !pipe) + return NULL; +#ifdef CONFIG_WATCH_QUEUE + if (for_splice && pipe->watch_queue) + return NULL; +#endif + return pipe; } long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg) @@ -1243,7 +1345,7 @@ long pipe_fcntl(struct file *file, unsigned int cmd, unsigned long arg) struct pipe_inode_info *pipe; long ret; - pipe = get_pipe_info(file); + pipe = get_pipe_info(file, false); if (!pipe) return -EBADF; diff --git a/fs/proc/inode.c b/fs/proc/inode.c index f40c2532c057..28d6105e908e 100644 --- a/fs/proc/inode.c +++ b/fs/proc/inode.c @@ -617,7 +617,7 @@ const struct inode_operations proc_link_inode_operations = { struct inode *proc_get_inode(struct super_block *sb, struct proc_dir_entry *de) { - struct inode *inode = new_inode_pseudo(sb); + struct inode *inode = new_inode(sb); if (inode) { inode->i_ino = de->low_ino; diff --git a/fs/proc/self.c b/fs/proc/self.c index ca5158fa561c..72cd69bcaf4a 100644 --- a/fs/proc/self.c +++ b/fs/proc/self.c @@ -43,7 +43,7 @@ int proc_setup_self(struct super_block *s) inode_lock(root_inode); self = d_alloc_name(s->s_root, "self"); if (self) { - struct inode *inode = new_inode_pseudo(s); + struct inode *inode = new_inode(s); if (inode) { inode->i_ino = self_inum; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); diff --git a/fs/proc/thread_self.c b/fs/proc/thread_self.c index ac284f409568..a553273fbd41 100644 --- a/fs/proc/thread_self.c +++ b/fs/proc/thread_self.c @@ -43,7 +43,7 @@ int proc_setup_thread_self(struct super_block *s) inode_lock(root_inode); thread_self = d_alloc_name(s->s_root, "thread-self"); if (thread_self) { - struct inode *inode = new_inode_pseudo(s); + struct inode *inode = new_inode(s); if (inode) { inode->i_ino = thread_self_inum; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); diff --git a/fs/splice.c b/fs/splice.c index 6b3c9a018a8e..d7c8a7c4db07 100644 --- a/fs/splice.c +++ b/fs/splice.c @@ -1101,8 +1101,8 @@ long do_splice(struct file *in, loff_t __user *off_in, !(out->f_mode & FMODE_WRITE))) return -EBADF; - ipipe = get_pipe_info(in); - opipe = get_pipe_info(out); + ipipe = get_pipe_info(in, true); + opipe = get_pipe_info(out, true); if (ipipe && opipe) { if (off_in || off_out) @@ -1252,7 +1252,7 @@ static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, static long vmsplice_to_user(struct file *file, struct iov_iter *iter, unsigned int flags) { - struct pipe_inode_info *pipe = get_pipe_info(file); + struct pipe_inode_info *pipe = get_pipe_info(file, true); struct splice_desc sd = { .total_len = iov_iter_count(iter), .flags = flags, @@ -1287,7 +1287,7 @@ static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter, if (flags & SPLICE_F_GIFT) buf_flag = PIPE_BUF_FLAG_GIFT; - pipe = get_pipe_info(file); + pipe = get_pipe_info(file, true); if (!pipe) return -EBADF; @@ -1733,8 +1733,8 @@ static int link_pipe(struct pipe_inode_info *ipipe, */ long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags) { - struct pipe_inode_info *ipipe = get_pipe_info(in); - struct pipe_inode_info *opipe = get_pipe_info(out); + struct pipe_inode_info *ipipe = get_pipe_info(in, true); + struct pipe_inode_info *opipe = get_pipe_info(out, true); int ret = -EINVAL; if (unlikely(!(in->f_mode & FMODE_READ) || diff --git a/fs/xfs/xfs_inode.c b/fs/xfs/xfs_inode.c index 4c91fb25ec66..9aea7d68d8ab 100644 --- a/fs/xfs/xfs_inode.c +++ b/fs/xfs/xfs_inode.c @@ -2634,8 +2634,10 @@ xfs_ifree_cluster( error = xfs_trans_get_buf(tp, mp->m_ddev_targp, blkno, mp->m_bsize * igeo->blocks_per_cluster, XBF_UNMAPPED, &bp); - if (error) + if (error) { + xfs_perag_put(pag); return error; + } /* * This buffer may not have been correctly initialised as we diff --git a/fs/xfs/xfs_ioctl.c b/fs/xfs/xfs_ioctl.c index a40f88cf3ab7..a190212ca85d 100644 --- a/fs/xfs/xfs_ioctl.c +++ b/fs/xfs/xfs_ioctl.c @@ -1236,64 +1236,26 @@ xfs_ioctl_setattr_xflags( return 0; } -/* - * If we are changing DAX flags, we have to ensure the file is clean and any - * cached objects in the address space are invalidated and removed. This - * requires us to lock out other IO and page faults similar to a truncate - * operation. The locks need to be held until the transaction has been committed - * so that the cache invalidation is atomic with respect to the DAX flag - * manipulation. - */ -static int -xfs_ioctl_setattr_dax_invalidate( +static void +xfs_ioctl_setattr_prepare_dax( struct xfs_inode *ip, - struct fsxattr *fa, - int *join_flags) + struct fsxattr *fa) { - struct inode *inode = VFS_I(ip); - struct super_block *sb = inode->i_sb; - int error; - - *join_flags = 0; - - /* - * It is only valid to set the DAX flag on regular files and - * directories on filesystems where the block size is equal to the page - * size. On directories it serves as an inherited hint so we don't - * have to check the device for dax support or flush pagecache. - */ - if (fa->fsx_xflags & FS_XFLAG_DAX) { - struct xfs_buftarg *target = xfs_inode_buftarg(ip); - - if (!bdev_dax_supported(target->bt_bdev, sb->s_blocksize)) - return -EINVAL; - } - - /* If the DAX state is not changing, we have nothing to do here. */ - if ((fa->fsx_xflags & FS_XFLAG_DAX) && IS_DAX(inode)) - return 0; - if (!(fa->fsx_xflags & FS_XFLAG_DAX) && !IS_DAX(inode)) - return 0; + struct xfs_mount *mp = ip->i_mount; + struct inode *inode = VFS_I(ip); if (S_ISDIR(inode->i_mode)) - return 0; - - /* lock, flush and invalidate mapping in preparation for flag change */ - xfs_ilock(ip, XFS_MMAPLOCK_EXCL | XFS_IOLOCK_EXCL); - error = filemap_write_and_wait(inode->i_mapping); - if (error) - goto out_unlock; - error = invalidate_inode_pages2(inode->i_mapping); - if (error) - goto out_unlock; - - *join_flags = XFS_MMAPLOCK_EXCL | XFS_IOLOCK_EXCL; - return 0; + return; -out_unlock: - xfs_iunlock(ip, XFS_MMAPLOCK_EXCL | XFS_IOLOCK_EXCL); - return error; + if ((mp->m_flags & XFS_MOUNT_DAX_ALWAYS) || + (mp->m_flags & XFS_MOUNT_DAX_NEVER)) + return; + if (((fa->fsx_xflags & FS_XFLAG_DAX) && + !(ip->i_d.di_flags2 & XFS_DIFLAG2_DAX)) || + (!(fa->fsx_xflags & FS_XFLAG_DAX) && + (ip->i_d.di_flags2 & XFS_DIFLAG2_DAX))) + d_mark_dontcache(inode); } /* @@ -1301,17 +1263,10 @@ out_unlock: * have permission to do so. On success, return a clean transaction and the * inode locked exclusively ready for further operation specific checks. On * failure, return an error without modifying or locking the inode. - * - * The inode might already be IO locked on call. If this is the case, it is - * indicated in @join_flags and we take full responsibility for ensuring they - * are unlocked from now on. Hence if we have an error here, we still have to - * unlock them. Otherwise, once they are joined to the transaction, they will - * be unlocked on commit/cancel. */ static struct xfs_trans * xfs_ioctl_setattr_get_trans( - struct xfs_inode *ip, - int join_flags) + struct xfs_inode *ip) { struct xfs_mount *mp = ip->i_mount; struct xfs_trans *tp; @@ -1328,8 +1283,7 @@ xfs_ioctl_setattr_get_trans( goto out_unlock; xfs_ilock(ip, XFS_ILOCK_EXCL); - xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL | join_flags); - join_flags = 0; + xfs_trans_ijoin(tp, ip, XFS_ILOCK_EXCL); /* * CAP_FOWNER overrides the following restrictions: @@ -1350,8 +1304,6 @@ xfs_ioctl_setattr_get_trans( out_cancel: xfs_trans_cancel(tp); out_unlock: - if (join_flags) - xfs_iunlock(ip, join_flags); return ERR_PTR(error); } @@ -1476,7 +1428,6 @@ xfs_ioctl_setattr( struct xfs_dquot *pdqp = NULL; struct xfs_dquot *olddquot = NULL; int code; - int join_flags = 0; trace_xfs_ioctl_setattr(ip); @@ -1500,18 +1451,9 @@ xfs_ioctl_setattr( return code; } - /* - * Changing DAX config may require inode locking for mapping - * invalidation. These need to be held all the way to transaction commit - * or cancel time, so need to be passed through to - * xfs_ioctl_setattr_get_trans() so it can apply them to the join call - * appropriately. - */ - code = xfs_ioctl_setattr_dax_invalidate(ip, fa, &join_flags); - if (code) - goto error_free_dquots; + xfs_ioctl_setattr_prepare_dax(ip, fa); - tp = xfs_ioctl_setattr_get_trans(ip, join_flags); + tp = xfs_ioctl_setattr_get_trans(ip); if (IS_ERR(tp)) { code = PTR_ERR(tp); goto error_free_dquots; @@ -1639,7 +1581,6 @@ xfs_ioc_setxflags( struct fsxattr fa; struct fsxattr old_fa; unsigned int flags; - int join_flags = 0; int error; if (copy_from_user(&flags, arg, sizeof(flags))) @@ -1656,18 +1597,9 @@ xfs_ioc_setxflags( if (error) return error; - /* - * Changing DAX config may require inode locking for mapping - * invalidation. These need to be held all the way to transaction commit - * or cancel time, so need to be passed through to - * xfs_ioctl_setattr_get_trans() so it can apply them to the join call - * appropriately. - */ - error = xfs_ioctl_setattr_dax_invalidate(ip, &fa, &join_flags); - if (error) - goto out_drop_write; + xfs_ioctl_setattr_prepare_dax(ip, &fa); - tp = xfs_ioctl_setattr_get_trans(ip, join_flags); + tp = xfs_ioctl_setattr_get_trans(ip); if (IS_ERR(tp)) { error = PTR_ERR(tp); goto out_drop_write; diff --git a/include/asm-generic/atomic-instrumented.h b/include/asm-generic/atomic-instrumented.h index e8730c6b9fe2..379986e40159 100644 --- a/include/asm-generic/atomic-instrumented.h +++ b/include/asm-generic/atomic-instrumented.h @@ -18,1623 +18,1624 @@ #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H #include <linux/build_bug.h> -#include <linux/kasan-checks.h> +#include <linux/compiler.h> +#include <linux/instrumented.h> -static inline int +static __always_inline int atomic_read(const atomic_t *v) { - kasan_check_read(v, sizeof(*v)); + instrument_atomic_read(v, sizeof(*v)); return arch_atomic_read(v); } #define atomic_read atomic_read #if defined(arch_atomic_read_acquire) -static inline int +static __always_inline int atomic_read_acquire(const atomic_t *v) { - kasan_check_read(v, sizeof(*v)); + instrument_atomic_read(v, sizeof(*v)); return arch_atomic_read_acquire(v); } #define atomic_read_acquire atomic_read_acquire #endif -static inline void +static __always_inline void atomic_set(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_set(v, i); } #define atomic_set atomic_set #if defined(arch_atomic_set_release) -static inline void +static __always_inline void atomic_set_release(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_set_release(v, i); } #define atomic_set_release atomic_set_release #endif -static inline void +static __always_inline void atomic_add(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_add(i, v); } #define atomic_add atomic_add #if !defined(arch_atomic_add_return_relaxed) || defined(arch_atomic_add_return) -static inline int +static __always_inline int atomic_add_return(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_return(i, v); } #define atomic_add_return atomic_add_return #endif #if defined(arch_atomic_add_return_acquire) -static inline int +static __always_inline int atomic_add_return_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_return_acquire(i, v); } #define atomic_add_return_acquire atomic_add_return_acquire #endif #if defined(arch_atomic_add_return_release) -static inline int +static __always_inline int atomic_add_return_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_return_release(i, v); } #define atomic_add_return_release atomic_add_return_release #endif #if defined(arch_atomic_add_return_relaxed) -static inline int +static __always_inline int atomic_add_return_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_return_relaxed(i, v); } #define atomic_add_return_relaxed atomic_add_return_relaxed #endif #if !defined(arch_atomic_fetch_add_relaxed) || defined(arch_atomic_fetch_add) -static inline int +static __always_inline int atomic_fetch_add(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_add(i, v); } #define atomic_fetch_add atomic_fetch_add #endif #if defined(arch_atomic_fetch_add_acquire) -static inline int +static __always_inline int atomic_fetch_add_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_add_acquire(i, v); } #define atomic_fetch_add_acquire atomic_fetch_add_acquire #endif #if defined(arch_atomic_fetch_add_release) -static inline int +static __always_inline int atomic_fetch_add_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_add_release(i, v); } #define atomic_fetch_add_release atomic_fetch_add_release #endif #if defined(arch_atomic_fetch_add_relaxed) -static inline int +static __always_inline int atomic_fetch_add_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_add_relaxed(i, v); } #define atomic_fetch_add_relaxed atomic_fetch_add_relaxed #endif -static inline void +static __always_inline void atomic_sub(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_sub(i, v); } #define atomic_sub atomic_sub #if !defined(arch_atomic_sub_return_relaxed) || defined(arch_atomic_sub_return) -static inline int +static __always_inline int atomic_sub_return(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_sub_return(i, v); } #define atomic_sub_return atomic_sub_return #endif #if defined(arch_atomic_sub_return_acquire) -static inline int +static __always_inline int atomic_sub_return_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_sub_return_acquire(i, v); } #define atomic_sub_return_acquire atomic_sub_return_acquire #endif #if defined(arch_atomic_sub_return_release) -static inline int +static __always_inline int atomic_sub_return_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_sub_return_release(i, v); } #define atomic_sub_return_release atomic_sub_return_release #endif #if defined(arch_atomic_sub_return_relaxed) -static inline int +static __always_inline int atomic_sub_return_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_sub_return_relaxed(i, v); } #define atomic_sub_return_relaxed atomic_sub_return_relaxed #endif #if !defined(arch_atomic_fetch_sub_relaxed) || defined(arch_atomic_fetch_sub) -static inline int +static __always_inline int atomic_fetch_sub(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_sub(i, v); } #define atomic_fetch_sub atomic_fetch_sub #endif #if defined(arch_atomic_fetch_sub_acquire) -static inline int +static __always_inline int atomic_fetch_sub_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_sub_acquire(i, v); } #define atomic_fetch_sub_acquire atomic_fetch_sub_acquire #endif #if defined(arch_atomic_fetch_sub_release) -static inline int +static __always_inline int atomic_fetch_sub_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_sub_release(i, v); } #define atomic_fetch_sub_release atomic_fetch_sub_release #endif #if defined(arch_atomic_fetch_sub_relaxed) -static inline int +static __always_inline int atomic_fetch_sub_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_sub_relaxed(i, v); } #define atomic_fetch_sub_relaxed atomic_fetch_sub_relaxed #endif #if defined(arch_atomic_inc) -static inline void +static __always_inline void atomic_inc(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_inc(v); } #define atomic_inc atomic_inc #endif #if defined(arch_atomic_inc_return) -static inline int +static __always_inline int atomic_inc_return(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_return(v); } #define atomic_inc_return atomic_inc_return #endif #if defined(arch_atomic_inc_return_acquire) -static inline int +static __always_inline int atomic_inc_return_acquire(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_return_acquire(v); } #define atomic_inc_return_acquire atomic_inc_return_acquire #endif #if defined(arch_atomic_inc_return_release) -static inline int +static __always_inline int atomic_inc_return_release(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_return_release(v); } #define atomic_inc_return_release atomic_inc_return_release #endif #if defined(arch_atomic_inc_return_relaxed) -static inline int +static __always_inline int atomic_inc_return_relaxed(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_return_relaxed(v); } #define atomic_inc_return_relaxed atomic_inc_return_relaxed #endif #if defined(arch_atomic_fetch_inc) -static inline int +static __always_inline int atomic_fetch_inc(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_inc(v); } #define atomic_fetch_inc atomic_fetch_inc #endif #if defined(arch_atomic_fetch_inc_acquire) -static inline int +static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_inc_acquire(v); } #define atomic_fetch_inc_acquire atomic_fetch_inc_acquire #endif #if defined(arch_atomic_fetch_inc_release) -static inline int +static __always_inline int atomic_fetch_inc_release(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_inc_release(v); } #define atomic_fetch_inc_release atomic_fetch_inc_release #endif #if defined(arch_atomic_fetch_inc_relaxed) -static inline int +static __always_inline int atomic_fetch_inc_relaxed(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_inc_relaxed(v); } #define atomic_fetch_inc_relaxed atomic_fetch_inc_relaxed #endif #if defined(arch_atomic_dec) -static inline void +static __always_inline void atomic_dec(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_dec(v); } #define atomic_dec atomic_dec #endif #if defined(arch_atomic_dec_return) -static inline int +static __always_inline int atomic_dec_return(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_return(v); } #define atomic_dec_return atomic_dec_return #endif #if defined(arch_atomic_dec_return_acquire) -static inline int +static __always_inline int atomic_dec_return_acquire(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_return_acquire(v); } #define atomic_dec_return_acquire atomic_dec_return_acquire #endif #if defined(arch_atomic_dec_return_release) -static inline int +static __always_inline int atomic_dec_return_release(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_return_release(v); } #define atomic_dec_return_release atomic_dec_return_release #endif #if defined(arch_atomic_dec_return_relaxed) -static inline int +static __always_inline int atomic_dec_return_relaxed(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_return_relaxed(v); } #define atomic_dec_return_relaxed atomic_dec_return_relaxed #endif #if defined(arch_atomic_fetch_dec) -static inline int +static __always_inline int atomic_fetch_dec(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_dec(v); } #define atomic_fetch_dec atomic_fetch_dec #endif #if defined(arch_atomic_fetch_dec_acquire) -static inline int +static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_dec_acquire(v); } #define atomic_fetch_dec_acquire atomic_fetch_dec_acquire #endif #if defined(arch_atomic_fetch_dec_release) -static inline int +static __always_inline int atomic_fetch_dec_release(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_dec_release(v); } #define atomic_fetch_dec_release atomic_fetch_dec_release #endif #if defined(arch_atomic_fetch_dec_relaxed) -static inline int +static __always_inline int atomic_fetch_dec_relaxed(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_dec_relaxed(v); } #define atomic_fetch_dec_relaxed atomic_fetch_dec_relaxed #endif -static inline void +static __always_inline void atomic_and(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_and(i, v); } #define atomic_and atomic_and #if !defined(arch_atomic_fetch_and_relaxed) || defined(arch_atomic_fetch_and) -static inline int +static __always_inline int atomic_fetch_and(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_and(i, v); } #define atomic_fetch_and atomic_fetch_and #endif #if defined(arch_atomic_fetch_and_acquire) -static inline int +static __always_inline int atomic_fetch_and_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_and_acquire(i, v); } #define atomic_fetch_and_acquire atomic_fetch_and_acquire #endif #if defined(arch_atomic_fetch_and_release) -static inline int +static __always_inline int atomic_fetch_and_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_and_release(i, v); } #define atomic_fetch_and_release atomic_fetch_and_release #endif #if defined(arch_atomic_fetch_and_relaxed) -static inline int +static __always_inline int atomic_fetch_and_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_and_relaxed(i, v); } #define atomic_fetch_and_relaxed atomic_fetch_and_relaxed #endif #if defined(arch_atomic_andnot) -static inline void +static __always_inline void atomic_andnot(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_andnot(i, v); } #define atomic_andnot atomic_andnot #endif #if defined(arch_atomic_fetch_andnot) -static inline int +static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_andnot(i, v); } #define atomic_fetch_andnot atomic_fetch_andnot #endif #if defined(arch_atomic_fetch_andnot_acquire) -static inline int +static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_acquire(i, v); } #define atomic_fetch_andnot_acquire atomic_fetch_andnot_acquire #endif #if defined(arch_atomic_fetch_andnot_release) -static inline int +static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_release(i, v); } #define atomic_fetch_andnot_release atomic_fetch_andnot_release #endif #if defined(arch_atomic_fetch_andnot_relaxed) -static inline int +static __always_inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_andnot_relaxed(i, v); } #define atomic_fetch_andnot_relaxed atomic_fetch_andnot_relaxed #endif -static inline void +static __always_inline void atomic_or(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_or(i, v); } #define atomic_or atomic_or #if !defined(arch_atomic_fetch_or_relaxed) || defined(arch_atomic_fetch_or) -static inline int +static __always_inline int atomic_fetch_or(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_or(i, v); } #define atomic_fetch_or atomic_fetch_or #endif #if defined(arch_atomic_fetch_or_acquire) -static inline int +static __always_inline int atomic_fetch_or_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_or_acquire(i, v); } #define atomic_fetch_or_acquire atomic_fetch_or_acquire #endif #if defined(arch_atomic_fetch_or_release) -static inline int +static __always_inline int atomic_fetch_or_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_or_release(i, v); } #define atomic_fetch_or_release atomic_fetch_or_release #endif #if defined(arch_atomic_fetch_or_relaxed) -static inline int +static __always_inline int atomic_fetch_or_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_or_relaxed(i, v); } #define atomic_fetch_or_relaxed atomic_fetch_or_relaxed #endif -static inline void +static __always_inline void atomic_xor(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic_xor(i, v); } #define atomic_xor atomic_xor #if !defined(arch_atomic_fetch_xor_relaxed) || defined(arch_atomic_fetch_xor) -static inline int +static __always_inline int atomic_fetch_xor(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_xor(i, v); } #define atomic_fetch_xor atomic_fetch_xor #endif #if defined(arch_atomic_fetch_xor_acquire) -static inline int +static __always_inline int atomic_fetch_xor_acquire(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_xor_acquire(i, v); } #define atomic_fetch_xor_acquire atomic_fetch_xor_acquire #endif #if defined(arch_atomic_fetch_xor_release) -static inline int +static __always_inline int atomic_fetch_xor_release(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_xor_release(i, v); } #define atomic_fetch_xor_release atomic_fetch_xor_release #endif #if defined(arch_atomic_fetch_xor_relaxed) -static inline int +static __always_inline int atomic_fetch_xor_relaxed(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_xor_relaxed(i, v); } #define atomic_fetch_xor_relaxed atomic_fetch_xor_relaxed #endif #if !defined(arch_atomic_xchg_relaxed) || defined(arch_atomic_xchg) -static inline int +static __always_inline int atomic_xchg(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_xchg(v, i); } #define atomic_xchg atomic_xchg #endif #if defined(arch_atomic_xchg_acquire) -static inline int +static __always_inline int atomic_xchg_acquire(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_xchg_acquire(v, i); } #define atomic_xchg_acquire atomic_xchg_acquire #endif #if defined(arch_atomic_xchg_release) -static inline int +static __always_inline int atomic_xchg_release(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_xchg_release(v, i); } #define atomic_xchg_release atomic_xchg_release #endif #if defined(arch_atomic_xchg_relaxed) -static inline int +static __always_inline int atomic_xchg_relaxed(atomic_t *v, int i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_xchg_relaxed(v, i); } #define atomic_xchg_relaxed atomic_xchg_relaxed #endif #if !defined(arch_atomic_cmpxchg_relaxed) || defined(arch_atomic_cmpxchg) -static inline int +static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_cmpxchg(v, old, new); } #define atomic_cmpxchg atomic_cmpxchg #endif #if defined(arch_atomic_cmpxchg_acquire) -static inline int +static __always_inline int atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_cmpxchg_acquire(v, old, new); } #define atomic_cmpxchg_acquire atomic_cmpxchg_acquire #endif #if defined(arch_atomic_cmpxchg_release) -static inline int +static __always_inline int atomic_cmpxchg_release(atomic_t *v, int old, int new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_cmpxchg_release(v, old, new); } #define atomic_cmpxchg_release atomic_cmpxchg_release #endif #if defined(arch_atomic_cmpxchg_relaxed) -static inline int +static __always_inline int atomic_cmpxchg_relaxed(atomic_t *v, int old, int new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_cmpxchg_relaxed(v, old, new); } #define atomic_cmpxchg_relaxed atomic_cmpxchg_relaxed #endif #if defined(arch_atomic_try_cmpxchg) -static inline bool +static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg(v, old, new); } #define atomic_try_cmpxchg atomic_try_cmpxchg #endif #if defined(arch_atomic_try_cmpxchg_acquire) -static inline bool +static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_acquire(v, old, new); } #define atomic_try_cmpxchg_acquire atomic_try_cmpxchg_acquire #endif #if defined(arch_atomic_try_cmpxchg_release) -static inline bool +static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_release(v, old, new); } #define atomic_try_cmpxchg_release atomic_try_cmpxchg_release #endif #if defined(arch_atomic_try_cmpxchg_relaxed) -static inline bool +static __always_inline bool atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic_try_cmpxchg_relaxed(v, old, new); } #define atomic_try_cmpxchg_relaxed atomic_try_cmpxchg_relaxed #endif #if defined(arch_atomic_sub_and_test) -static inline bool +static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_sub_and_test(i, v); } #define atomic_sub_and_test atomic_sub_and_test #endif #if defined(arch_atomic_dec_and_test) -static inline bool +static __always_inline bool atomic_dec_and_test(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_and_test(v); } #define atomic_dec_and_test atomic_dec_and_test #endif #if defined(arch_atomic_inc_and_test) -static inline bool +static __always_inline bool atomic_inc_and_test(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_and_test(v); } #define atomic_inc_and_test atomic_inc_and_test #endif #if defined(arch_atomic_add_negative) -static inline bool +static __always_inline bool atomic_add_negative(int i, atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_negative(i, v); } #define atomic_add_negative atomic_add_negative #endif #if defined(arch_atomic_fetch_add_unless) -static inline int +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_fetch_add_unless(v, a, u); } #define atomic_fetch_add_unless atomic_fetch_add_unless #endif #if defined(arch_atomic_add_unless) -static inline bool +static __always_inline bool atomic_add_unless(atomic_t *v, int a, int u) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_add_unless(v, a, u); } #define atomic_add_unless atomic_add_unless #endif #if defined(arch_atomic_inc_not_zero) -static inline bool +static __always_inline bool atomic_inc_not_zero(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_not_zero(v); } #define atomic_inc_not_zero atomic_inc_not_zero #endif #if defined(arch_atomic_inc_unless_negative) -static inline bool +static __always_inline bool atomic_inc_unless_negative(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_inc_unless_negative(v); } #define atomic_inc_unless_negative atomic_inc_unless_negative #endif #if defined(arch_atomic_dec_unless_positive) -static inline bool +static __always_inline bool atomic_dec_unless_positive(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_unless_positive(v); } #define atomic_dec_unless_positive atomic_dec_unless_positive #endif #if defined(arch_atomic_dec_if_positive) -static inline int +static __always_inline int atomic_dec_if_positive(atomic_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic_dec_if_positive(v); } #define atomic_dec_if_positive atomic_dec_if_positive #endif -static inline s64 +static __always_inline s64 atomic64_read(const atomic64_t *v) { - kasan_check_read(v, sizeof(*v)); + instrument_atomic_read(v, sizeof(*v)); return arch_atomic64_read(v); } #define atomic64_read atomic64_read #if defined(arch_atomic64_read_acquire) -static inline s64 +static __always_inline s64 atomic64_read_acquire(const atomic64_t *v) { - kasan_check_read(v, sizeof(*v)); + instrument_atomic_read(v, sizeof(*v)); return arch_atomic64_read_acquire(v); } #define atomic64_read_acquire atomic64_read_acquire #endif -static inline void +static __always_inline void atomic64_set(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_set(v, i); } #define atomic64_set atomic64_set #if defined(arch_atomic64_set_release) -static inline void +static __always_inline void atomic64_set_release(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_set_release(v, i); } #define atomic64_set_release atomic64_set_release #endif -static inline void +static __always_inline void atomic64_add(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_add(i, v); } #define atomic64_add atomic64_add #if !defined(arch_atomic64_add_return_relaxed) || defined(arch_atomic64_add_return) -static inline s64 +static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_return(i, v); } #define atomic64_add_return atomic64_add_return #endif #if defined(arch_atomic64_add_return_acquire) -static inline s64 +static __always_inline s64 atomic64_add_return_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_return_acquire(i, v); } #define atomic64_add_return_acquire atomic64_add_return_acquire #endif #if defined(arch_atomic64_add_return_release) -static inline s64 +static __always_inline s64 atomic64_add_return_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_return_release(i, v); } #define atomic64_add_return_release atomic64_add_return_release #endif #if defined(arch_atomic64_add_return_relaxed) -static inline s64 +static __always_inline s64 atomic64_add_return_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_return_relaxed(i, v); } #define atomic64_add_return_relaxed atomic64_add_return_relaxed #endif #if !defined(arch_atomic64_fetch_add_relaxed) || defined(arch_atomic64_fetch_add) -static inline s64 +static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_add(i, v); } #define atomic64_fetch_add atomic64_fetch_add #endif #if defined(arch_atomic64_fetch_add_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_add_acquire(i, v); } #define atomic64_fetch_add_acquire atomic64_fetch_add_acquire #endif #if defined(arch_atomic64_fetch_add_release) -static inline s64 +static __always_inline s64 atomic64_fetch_add_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_add_release(i, v); } #define atomic64_fetch_add_release atomic64_fetch_add_release #endif #if defined(arch_atomic64_fetch_add_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_add_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_add_relaxed(i, v); } #define atomic64_fetch_add_relaxed atomic64_fetch_add_relaxed #endif -static inline void +static __always_inline void atomic64_sub(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_sub(i, v); } #define atomic64_sub atomic64_sub #if !defined(arch_atomic64_sub_return_relaxed) || defined(arch_atomic64_sub_return) -static inline s64 +static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_sub_return(i, v); } #define atomic64_sub_return atomic64_sub_return #endif #if defined(arch_atomic64_sub_return_acquire) -static inline s64 +static __always_inline s64 atomic64_sub_return_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_sub_return_acquire(i, v); } #define atomic64_sub_return_acquire atomic64_sub_return_acquire #endif #if defined(arch_atomic64_sub_return_release) -static inline s64 +static __always_inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_sub_return_release(i, v); } #define atomic64_sub_return_release atomic64_sub_return_release #endif #if defined(arch_atomic64_sub_return_relaxed) -static inline s64 +static __always_inline s64 atomic64_sub_return_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_sub_return_relaxed(i, v); } #define atomic64_sub_return_relaxed atomic64_sub_return_relaxed #endif #if !defined(arch_atomic64_fetch_sub_relaxed) || defined(arch_atomic64_fetch_sub) -static inline s64 +static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_sub(i, v); } #define atomic64_fetch_sub atomic64_fetch_sub #endif #if defined(arch_atomic64_fetch_sub_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_acquire(i, v); } #define atomic64_fetch_sub_acquire atomic64_fetch_sub_acquire #endif #if defined(arch_atomic64_fetch_sub_release) -static inline s64 +static __always_inline s64 atomic64_fetch_sub_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_release(i, v); } #define atomic64_fetch_sub_release atomic64_fetch_sub_release #endif #if defined(arch_atomic64_fetch_sub_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_sub_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_sub_relaxed(i, v); } #define atomic64_fetch_sub_relaxed atomic64_fetch_sub_relaxed #endif #if defined(arch_atomic64_inc) -static inline void +static __always_inline void atomic64_inc(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_inc(v); } #define atomic64_inc atomic64_inc #endif #if defined(arch_atomic64_inc_return) -static inline s64 +static __always_inline s64 atomic64_inc_return(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_return(v); } #define atomic64_inc_return atomic64_inc_return #endif #if defined(arch_atomic64_inc_return_acquire) -static inline s64 +static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_return_acquire(v); } #define atomic64_inc_return_acquire atomic64_inc_return_acquire #endif #if defined(arch_atomic64_inc_return_release) -static inline s64 +static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_return_release(v); } #define atomic64_inc_return_release atomic64_inc_return_release #endif #if defined(arch_atomic64_inc_return_relaxed) -static inline s64 +static __always_inline s64 atomic64_inc_return_relaxed(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_return_relaxed(v); } #define atomic64_inc_return_relaxed atomic64_inc_return_relaxed #endif #if defined(arch_atomic64_fetch_inc) -static inline s64 +static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_inc(v); } #define atomic64_fetch_inc atomic64_fetch_inc #endif #if defined(arch_atomic64_fetch_inc_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_acquire(v); } #define atomic64_fetch_inc_acquire atomic64_fetch_inc_acquire #endif #if defined(arch_atomic64_fetch_inc_release) -static inline s64 +static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_release(v); } #define atomic64_fetch_inc_release atomic64_fetch_inc_release #endif #if defined(arch_atomic64_fetch_inc_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_inc_relaxed(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_inc_relaxed(v); } #define atomic64_fetch_inc_relaxed atomic64_fetch_inc_relaxed #endif #if defined(arch_atomic64_dec) -static inline void +static __always_inline void atomic64_dec(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_dec(v); } #define atomic64_dec atomic64_dec #endif #if defined(arch_atomic64_dec_return) -static inline s64 +static __always_inline s64 atomic64_dec_return(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_return(v); } #define atomic64_dec_return atomic64_dec_return #endif #if defined(arch_atomic64_dec_return_acquire) -static inline s64 +static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_return_acquire(v); } #define atomic64_dec_return_acquire atomic64_dec_return_acquire #endif #if defined(arch_atomic64_dec_return_release) -static inline s64 +static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_return_release(v); } #define atomic64_dec_return_release atomic64_dec_return_release #endif #if defined(arch_atomic64_dec_return_relaxed) -static inline s64 +static __always_inline s64 atomic64_dec_return_relaxed(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_return_relaxed(v); } #define atomic64_dec_return_relaxed atomic64_dec_return_relaxed #endif #if defined(arch_atomic64_fetch_dec) -static inline s64 +static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_dec(v); } #define atomic64_fetch_dec atomic64_fetch_dec #endif #if defined(arch_atomic64_fetch_dec_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_acquire(v); } #define atomic64_fetch_dec_acquire atomic64_fetch_dec_acquire #endif #if defined(arch_atomic64_fetch_dec_release) -static inline s64 +static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_release(v); } #define atomic64_fetch_dec_release atomic64_fetch_dec_release #endif #if defined(arch_atomic64_fetch_dec_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_dec_relaxed(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_dec_relaxed(v); } #define atomic64_fetch_dec_relaxed atomic64_fetch_dec_relaxed #endif -static inline void +static __always_inline void atomic64_and(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_and(i, v); } #define atomic64_and atomic64_and #if !defined(arch_atomic64_fetch_and_relaxed) || defined(arch_atomic64_fetch_and) -static inline s64 +static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_and(i, v); } #define atomic64_fetch_and atomic64_fetch_and #endif #if defined(arch_atomic64_fetch_and_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_and_acquire(i, v); } #define atomic64_fetch_and_acquire atomic64_fetch_and_acquire #endif #if defined(arch_atomic64_fetch_and_release) -static inline s64 +static __always_inline s64 atomic64_fetch_and_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_and_release(i, v); } #define atomic64_fetch_and_release atomic64_fetch_and_release #endif #if defined(arch_atomic64_fetch_and_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_and_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_and_relaxed(i, v); } #define atomic64_fetch_and_relaxed atomic64_fetch_and_relaxed #endif #if defined(arch_atomic64_andnot) -static inline void +static __always_inline void atomic64_andnot(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_andnot(i, v); } #define atomic64_andnot atomic64_andnot #endif #if defined(arch_atomic64_fetch_andnot) -static inline s64 +static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot(i, v); } #define atomic64_fetch_andnot atomic64_fetch_andnot #endif #if defined(arch_atomic64_fetch_andnot_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_acquire(i, v); } #define atomic64_fetch_andnot_acquire atomic64_fetch_andnot_acquire #endif #if defined(arch_atomic64_fetch_andnot_release) -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_release(i, v); } #define atomic64_fetch_andnot_release atomic64_fetch_andnot_release #endif #if defined(arch_atomic64_fetch_andnot_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_andnot_relaxed(i, v); } #define atomic64_fetch_andnot_relaxed atomic64_fetch_andnot_relaxed #endif -static inline void +static __always_inline void atomic64_or(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_or(i, v); } #define atomic64_or atomic64_or #if !defined(arch_atomic64_fetch_or_relaxed) || defined(arch_atomic64_fetch_or) -static inline s64 +static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_or(i, v); } #define atomic64_fetch_or atomic64_fetch_or #endif #if defined(arch_atomic64_fetch_or_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_or_acquire(i, v); } #define atomic64_fetch_or_acquire atomic64_fetch_or_acquire #endif #if defined(arch_atomic64_fetch_or_release) -static inline s64 +static __always_inline s64 atomic64_fetch_or_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_or_release(i, v); } #define atomic64_fetch_or_release atomic64_fetch_or_release #endif #if defined(arch_atomic64_fetch_or_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_or_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_or_relaxed(i, v); } #define atomic64_fetch_or_relaxed atomic64_fetch_or_relaxed #endif -static inline void +static __always_inline void atomic64_xor(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); arch_atomic64_xor(i, v); } #define atomic64_xor atomic64_xor #if !defined(arch_atomic64_fetch_xor_relaxed) || defined(arch_atomic64_fetch_xor) -static inline s64 +static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_xor(i, v); } #define atomic64_fetch_xor atomic64_fetch_xor #endif #if defined(arch_atomic64_fetch_xor_acquire) -static inline s64 +static __always_inline s64 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_acquire(i, v); } #define atomic64_fetch_xor_acquire atomic64_fetch_xor_acquire #endif #if defined(arch_atomic64_fetch_xor_release) -static inline s64 +static __always_inline s64 atomic64_fetch_xor_release(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_release(i, v); } #define atomic64_fetch_xor_release atomic64_fetch_xor_release #endif #if defined(arch_atomic64_fetch_xor_relaxed) -static inline s64 +static __always_inline s64 atomic64_fetch_xor_relaxed(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_xor_relaxed(i, v); } #define atomic64_fetch_xor_relaxed atomic64_fetch_xor_relaxed #endif #if !defined(arch_atomic64_xchg_relaxed) || defined(arch_atomic64_xchg) -static inline s64 +static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_xchg(v, i); } #define atomic64_xchg atomic64_xchg #endif #if defined(arch_atomic64_xchg_acquire) -static inline s64 +static __always_inline s64 atomic64_xchg_acquire(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_xchg_acquire(v, i); } #define atomic64_xchg_acquire atomic64_xchg_acquire #endif #if defined(arch_atomic64_xchg_release) -static inline s64 +static __always_inline s64 atomic64_xchg_release(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_xchg_release(v, i); } #define atomic64_xchg_release atomic64_xchg_release #endif #if defined(arch_atomic64_xchg_relaxed) -static inline s64 +static __always_inline s64 atomic64_xchg_relaxed(atomic64_t *v, s64 i) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_xchg_relaxed(v, i); } #define atomic64_xchg_relaxed atomic64_xchg_relaxed #endif #if !defined(arch_atomic64_cmpxchg_relaxed) || defined(arch_atomic64_cmpxchg) -static inline s64 +static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_cmpxchg(v, old, new); } #define atomic64_cmpxchg atomic64_cmpxchg #endif #if defined(arch_atomic64_cmpxchg_acquire) -static inline s64 +static __always_inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_acquire(v, old, new); } #define atomic64_cmpxchg_acquire atomic64_cmpxchg_acquire #endif #if defined(arch_atomic64_cmpxchg_release) -static inline s64 +static __always_inline s64 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_release(v, old, new); } #define atomic64_cmpxchg_release atomic64_cmpxchg_release #endif #if defined(arch_atomic64_cmpxchg_relaxed) -static inline s64 +static __always_inline s64 atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_cmpxchg_relaxed(v, old, new); } #define atomic64_cmpxchg_relaxed atomic64_cmpxchg_relaxed #endif #if defined(arch_atomic64_try_cmpxchg) -static inline bool +static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg(v, old, new); } #define atomic64_try_cmpxchg atomic64_try_cmpxchg #endif #if defined(arch_atomic64_try_cmpxchg_acquire) -static inline bool +static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_acquire(v, old, new); } #define atomic64_try_cmpxchg_acquire atomic64_try_cmpxchg_acquire #endif #if defined(arch_atomic64_try_cmpxchg_release) -static inline bool +static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_release(v, old, new); } #define atomic64_try_cmpxchg_release atomic64_try_cmpxchg_release #endif #if defined(arch_atomic64_try_cmpxchg_relaxed) -static inline bool +static __always_inline bool atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { - kasan_check_write(v, sizeof(*v)); - kasan_check_write(old, sizeof(*old)); + instrument_atomic_write(v, sizeof(*v)); + instrument_atomic_write(old, sizeof(*old)); return arch_atomic64_try_cmpxchg_relaxed(v, old, new); } #define atomic64_try_cmpxchg_relaxed atomic64_try_cmpxchg_relaxed #endif #if defined(arch_atomic64_sub_and_test) -static inline bool +static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_sub_and_test(i, v); } #define atomic64_sub_and_test atomic64_sub_and_test #endif #if defined(arch_atomic64_dec_and_test) -static inline bool +static __always_inline bool atomic64_dec_and_test(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_and_test(v); } #define atomic64_dec_and_test atomic64_dec_and_test #endif #if defined(arch_atomic64_inc_and_test) -static inline bool +static __always_inline bool atomic64_inc_and_test(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_and_test(v); } #define atomic64_inc_and_test atomic64_inc_and_test #endif #if defined(arch_atomic64_add_negative) -static inline bool +static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_negative(i, v); } #define atomic64_add_negative atomic64_add_negative #endif #if defined(arch_atomic64_fetch_add_unless) -static inline s64 +static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_fetch_add_unless(v, a, u); } #define atomic64_fetch_add_unless atomic64_fetch_add_unless #endif #if defined(arch_atomic64_add_unless) -static inline bool +static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_add_unless(v, a, u); } #define atomic64_add_unless atomic64_add_unless #endif #if defined(arch_atomic64_inc_not_zero) -static inline bool +static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_not_zero(v); } #define atomic64_inc_not_zero atomic64_inc_not_zero #endif #if defined(arch_atomic64_inc_unless_negative) -static inline bool +static __always_inline bool atomic64_inc_unless_negative(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_inc_unless_negative(v); } #define atomic64_inc_unless_negative atomic64_inc_unless_negative #endif #if defined(arch_atomic64_dec_unless_positive) -static inline bool +static __always_inline bool atomic64_dec_unless_positive(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_unless_positive(v); } #define atomic64_dec_unless_positive atomic64_dec_unless_positive #endif #if defined(arch_atomic64_dec_if_positive) -static inline s64 +static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) { - kasan_check_write(v, sizeof(*v)); + instrument_atomic_write(v, sizeof(*v)); return arch_atomic64_dec_if_positive(v); } #define atomic64_dec_if_positive atomic64_dec_if_positive @@ -1644,7 +1645,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define xchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1653,7 +1654,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define xchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_acquire(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1662,7 +1663,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define xchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_release(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1671,7 +1672,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define xchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_xchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1680,7 +1681,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1689,7 +1690,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_acquire(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1698,7 +1699,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_release(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1707,7 +1708,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_relaxed(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1716,7 +1717,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg64(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1725,7 +1726,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg64_acquire(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_acquire(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1734,7 +1735,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg64_release(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_release(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1743,7 +1744,7 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg64_relaxed(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_relaxed(__ai_ptr, __VA_ARGS__); \ }) #endif @@ -1751,28 +1752,28 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg_local(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg64_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_cmpxchg64_local(__ai_ptr, __VA_ARGS__); \ }) #define sync_cmpxchg(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, sizeof(*__ai_ptr)); \ arch_sync_cmpxchg(__ai_ptr, __VA_ARGS__); \ }) #define cmpxchg_double(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double(__ai_ptr, __VA_ARGS__); \ }) @@ -1780,9 +1781,9 @@ atomic64_dec_if_positive(atomic64_t *v) #define cmpxchg_double_local(ptr, ...) \ ({ \ typeof(ptr) __ai_ptr = (ptr); \ - kasan_check_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ + instrument_atomic_write(__ai_ptr, 2 * sizeof(*__ai_ptr)); \ arch_cmpxchg_double_local(__ai_ptr, __VA_ARGS__); \ }) #endif /* _ASM_GENERIC_ATOMIC_INSTRUMENTED_H */ -// b29b625d5de9280f680e42c7be859b55b15e5f6a +// 89bf97f3a7509b740845e51ddf31055b48a81f40 diff --git a/include/asm-generic/atomic-long.h b/include/asm-generic/atomic-long.h index 881c7e27af28..073cf40f431b 100644 --- a/include/asm-generic/atomic-long.h +++ b/include/asm-generic/atomic-long.h @@ -6,6 +6,7 @@ #ifndef _ASM_GENERIC_ATOMIC_LONG_H #define _ASM_GENERIC_ATOMIC_LONG_H +#include <linux/compiler.h> #include <asm/types.h> #ifdef CONFIG_64BIT @@ -22,493 +23,493 @@ typedef atomic_t atomic_long_t; #ifdef CONFIG_64BIT -static inline long +static __always_inline long atomic_long_read(const atomic_long_t *v) { return atomic64_read(v); } -static inline long +static __always_inline long atomic_long_read_acquire(const atomic_long_t *v) { return atomic64_read_acquire(v); } -static inline void +static __always_inline void atomic_long_set(atomic_long_t *v, long i) { atomic64_set(v, i); } -static inline void +static __always_inline void atomic_long_set_release(atomic_long_t *v, long i) { atomic64_set_release(v, i); } -static inline void +static __always_inline void atomic_long_add(long i, atomic_long_t *v) { atomic64_add(i, v); } -static inline long +static __always_inline long atomic_long_add_return(long i, atomic_long_t *v) { return atomic64_add_return(i, v); } -static inline long +static __always_inline long atomic_long_add_return_acquire(long i, atomic_long_t *v) { return atomic64_add_return_acquire(i, v); } -static inline long +static __always_inline long atomic_long_add_return_release(long i, atomic_long_t *v) { return atomic64_add_return_release(i, v); } -static inline long +static __always_inline long atomic_long_add_return_relaxed(long i, atomic_long_t *v) { return atomic64_add_return_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add(long i, atomic_long_t *v) { return atomic64_fetch_add(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { return atomic64_fetch_add_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_release(long i, atomic_long_t *v) { return atomic64_fetch_add_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_add_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_sub(long i, atomic_long_t *v) { atomic64_sub(i, v); } -static inline long +static __always_inline long atomic_long_sub_return(long i, atomic_long_t *v) { return atomic64_sub_return(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_acquire(long i, atomic_long_t *v) { return atomic64_sub_return_acquire(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_release(long i, atomic_long_t *v) { return atomic64_sub_return_release(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { return atomic64_sub_return_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub(long i, atomic_long_t *v) { return atomic64_fetch_sub(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { return atomic64_fetch_sub_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_release(long i, atomic_long_t *v) { return atomic64_fetch_sub_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_sub_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_inc(atomic_long_t *v) { atomic64_inc(v); } -static inline long +static __always_inline long atomic_long_inc_return(atomic_long_t *v) { return atomic64_inc_return(v); } -static inline long +static __always_inline long atomic_long_inc_return_acquire(atomic_long_t *v) { return atomic64_inc_return_acquire(v); } -static inline long +static __always_inline long atomic_long_inc_return_release(atomic_long_t *v) { return atomic64_inc_return_release(v); } -static inline long +static __always_inline long atomic_long_inc_return_relaxed(atomic_long_t *v) { return atomic64_inc_return_relaxed(v); } -static inline long +static __always_inline long atomic_long_fetch_inc(atomic_long_t *v) { return atomic64_fetch_inc(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_acquire(atomic_long_t *v) { return atomic64_fetch_inc_acquire(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_release(atomic_long_t *v) { return atomic64_fetch_inc_release(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_relaxed(atomic_long_t *v) { return atomic64_fetch_inc_relaxed(v); } -static inline void +static __always_inline void atomic_long_dec(atomic_long_t *v) { atomic64_dec(v); } -static inline long +static __always_inline long atomic_long_dec_return(atomic_long_t *v) { return atomic64_dec_return(v); } -static inline long +static __always_inline long atomic_long_dec_return_acquire(atomic_long_t *v) { return atomic64_dec_return_acquire(v); } -static inline long +static __always_inline long atomic_long_dec_return_release(atomic_long_t *v) { return atomic64_dec_return_release(v); } -static inline long +static __always_inline long atomic_long_dec_return_relaxed(atomic_long_t *v) { return atomic64_dec_return_relaxed(v); } -static inline long +static __always_inline long atomic_long_fetch_dec(atomic_long_t *v) { return atomic64_fetch_dec(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_acquire(atomic_long_t *v) { return atomic64_fetch_dec_acquire(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_release(atomic_long_t *v) { return atomic64_fetch_dec_release(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_relaxed(atomic_long_t *v) { return atomic64_fetch_dec_relaxed(v); } -static inline void +static __always_inline void atomic_long_and(long i, atomic_long_t *v) { atomic64_and(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and(long i, atomic_long_t *v) { return atomic64_fetch_and(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { return atomic64_fetch_and_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_release(long i, atomic_long_t *v) { return atomic64_fetch_and_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_and_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_andnot(long i, atomic_long_t *v) { atomic64_andnot(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot(long i, atomic_long_t *v) { return atomic64_fetch_andnot(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { return atomic64_fetch_andnot_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { return atomic64_fetch_andnot_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_andnot_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_or(long i, atomic_long_t *v) { atomic64_or(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or(long i, atomic_long_t *v) { return atomic64_fetch_or(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { return atomic64_fetch_or_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_release(long i, atomic_long_t *v) { return atomic64_fetch_or_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_or_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_xor(long i, atomic_long_t *v) { atomic64_xor(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor(long i, atomic_long_t *v) { return atomic64_fetch_xor(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { return atomic64_fetch_xor_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_release(long i, atomic_long_t *v) { return atomic64_fetch_xor_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { return atomic64_fetch_xor_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_xchg(atomic_long_t *v, long i) { return atomic64_xchg(v, i); } -static inline long +static __always_inline long atomic_long_xchg_acquire(atomic_long_t *v, long i) { return atomic64_xchg_acquire(v, i); } -static inline long +static __always_inline long atomic_long_xchg_release(atomic_long_t *v, long i) { return atomic64_xchg_release(v, i); } -static inline long +static __always_inline long atomic_long_xchg_relaxed(atomic_long_t *v, long i) { return atomic64_xchg_relaxed(v, i); } -static inline long +static __always_inline long atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { return atomic64_cmpxchg(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { return atomic64_cmpxchg_acquire(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { return atomic64_cmpxchg_release(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { return atomic64_cmpxchg_relaxed(v, old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { return atomic64_try_cmpxchg(v, (s64 *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { return atomic64_try_cmpxchg_acquire(v, (s64 *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { return atomic64_try_cmpxchg_release(v, (s64 *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { return atomic64_try_cmpxchg_relaxed(v, (s64 *)old, new); } -static inline bool +static __always_inline bool atomic_long_sub_and_test(long i, atomic_long_t *v) { return atomic64_sub_and_test(i, v); } -static inline bool +static __always_inline bool atomic_long_dec_and_test(atomic_long_t *v) { return atomic64_dec_and_test(v); } -static inline bool +static __always_inline bool atomic_long_inc_and_test(atomic_long_t *v) { return atomic64_inc_and_test(v); } -static inline bool +static __always_inline bool atomic_long_add_negative(long i, atomic_long_t *v) { return atomic64_add_negative(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { return atomic64_fetch_add_unless(v, a, u); } -static inline bool +static __always_inline bool atomic_long_add_unless(atomic_long_t *v, long a, long u) { return atomic64_add_unless(v, a, u); } -static inline bool +static __always_inline bool atomic_long_inc_not_zero(atomic_long_t *v) { return atomic64_inc_not_zero(v); } -static inline bool +static __always_inline bool atomic_long_inc_unless_negative(atomic_long_t *v) { return atomic64_inc_unless_negative(v); } -static inline bool +static __always_inline bool atomic_long_dec_unless_positive(atomic_long_t *v) { return atomic64_dec_unless_positive(v); } -static inline long +static __always_inline long atomic_long_dec_if_positive(atomic_long_t *v) { return atomic64_dec_if_positive(v); @@ -516,493 +517,493 @@ atomic_long_dec_if_positive(atomic_long_t *v) #else /* CONFIG_64BIT */ -static inline long +static __always_inline long atomic_long_read(const atomic_long_t *v) { return atomic_read(v); } -static inline long +static __always_inline long atomic_long_read_acquire(const atomic_long_t *v) { return atomic_read_acquire(v); } -static inline void +static __always_inline void atomic_long_set(atomic_long_t *v, long i) { atomic_set(v, i); } -static inline void +static __always_inline void atomic_long_set_release(atomic_long_t *v, long i) { atomic_set_release(v, i); } -static inline void +static __always_inline void atomic_long_add(long i, atomic_long_t *v) { atomic_add(i, v); } -static inline long +static __always_inline long atomic_long_add_return(long i, atomic_long_t *v) { return atomic_add_return(i, v); } -static inline long +static __always_inline long atomic_long_add_return_acquire(long i, atomic_long_t *v) { return atomic_add_return_acquire(i, v); } -static inline long +static __always_inline long atomic_long_add_return_release(long i, atomic_long_t *v) { return atomic_add_return_release(i, v); } -static inline long +static __always_inline long atomic_long_add_return_relaxed(long i, atomic_long_t *v) { return atomic_add_return_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add(long i, atomic_long_t *v) { return atomic_fetch_add(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_acquire(long i, atomic_long_t *v) { return atomic_fetch_add_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_release(long i, atomic_long_t *v) { return atomic_fetch_add_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_relaxed(long i, atomic_long_t *v) { return atomic_fetch_add_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_sub(long i, atomic_long_t *v) { atomic_sub(i, v); } -static inline long +static __always_inline long atomic_long_sub_return(long i, atomic_long_t *v) { return atomic_sub_return(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_acquire(long i, atomic_long_t *v) { return atomic_sub_return_acquire(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_release(long i, atomic_long_t *v) { return atomic_sub_return_release(i, v); } -static inline long +static __always_inline long atomic_long_sub_return_relaxed(long i, atomic_long_t *v) { return atomic_sub_return_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub(long i, atomic_long_t *v) { return atomic_fetch_sub(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_acquire(long i, atomic_long_t *v) { return atomic_fetch_sub_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_release(long i, atomic_long_t *v) { return atomic_fetch_sub_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_sub_relaxed(long i, atomic_long_t *v) { return atomic_fetch_sub_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_inc(atomic_long_t *v) { atomic_inc(v); } -static inline long +static __always_inline long atomic_long_inc_return(atomic_long_t *v) { return atomic_inc_return(v); } -static inline long +static __always_inline long atomic_long_inc_return_acquire(atomic_long_t *v) { return atomic_inc_return_acquire(v); } -static inline long +static __always_inline long atomic_long_inc_return_release(atomic_long_t *v) { return atomic_inc_return_release(v); } -static inline long +static __always_inline long atomic_long_inc_return_relaxed(atomic_long_t *v) { return atomic_inc_return_relaxed(v); } -static inline long +static __always_inline long atomic_long_fetch_inc(atomic_long_t *v) { return atomic_fetch_inc(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_acquire(atomic_long_t *v) { return atomic_fetch_inc_acquire(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_release(atomic_long_t *v) { return atomic_fetch_inc_release(v); } -static inline long +static __always_inline long atomic_long_fetch_inc_relaxed(atomic_long_t *v) { return atomic_fetch_inc_relaxed(v); } -static inline void +static __always_inline void atomic_long_dec(atomic_long_t *v) { atomic_dec(v); } -static inline long +static __always_inline long atomic_long_dec_return(atomic_long_t *v) { return atomic_dec_return(v); } -static inline long +static __always_inline long atomic_long_dec_return_acquire(atomic_long_t *v) { return atomic_dec_return_acquire(v); } -static inline long +static __always_inline long atomic_long_dec_return_release(atomic_long_t *v) { return atomic_dec_return_release(v); } -static inline long +static __always_inline long atomic_long_dec_return_relaxed(atomic_long_t *v) { return atomic_dec_return_relaxed(v); } -static inline long +static __always_inline long atomic_long_fetch_dec(atomic_long_t *v) { return atomic_fetch_dec(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_acquire(atomic_long_t *v) { return atomic_fetch_dec_acquire(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_release(atomic_long_t *v) { return atomic_fetch_dec_release(v); } -static inline long +static __always_inline long atomic_long_fetch_dec_relaxed(atomic_long_t *v) { return atomic_fetch_dec_relaxed(v); } -static inline void +static __always_inline void atomic_long_and(long i, atomic_long_t *v) { atomic_and(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and(long i, atomic_long_t *v) { return atomic_fetch_and(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_acquire(long i, atomic_long_t *v) { return atomic_fetch_and_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_release(long i, atomic_long_t *v) { return atomic_fetch_and_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_and_relaxed(long i, atomic_long_t *v) { return atomic_fetch_and_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_andnot(long i, atomic_long_t *v) { atomic_andnot(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot(long i, atomic_long_t *v) { return atomic_fetch_andnot(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_acquire(long i, atomic_long_t *v) { return atomic_fetch_andnot_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_release(long i, atomic_long_t *v) { return atomic_fetch_andnot_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_andnot_relaxed(long i, atomic_long_t *v) { return atomic_fetch_andnot_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_or(long i, atomic_long_t *v) { atomic_or(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or(long i, atomic_long_t *v) { return atomic_fetch_or(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_acquire(long i, atomic_long_t *v) { return atomic_fetch_or_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_release(long i, atomic_long_t *v) { return atomic_fetch_or_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_or_relaxed(long i, atomic_long_t *v) { return atomic_fetch_or_relaxed(i, v); } -static inline void +static __always_inline void atomic_long_xor(long i, atomic_long_t *v) { atomic_xor(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor(long i, atomic_long_t *v) { return atomic_fetch_xor(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_acquire(long i, atomic_long_t *v) { return atomic_fetch_xor_acquire(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_release(long i, atomic_long_t *v) { return atomic_fetch_xor_release(i, v); } -static inline long +static __always_inline long atomic_long_fetch_xor_relaxed(long i, atomic_long_t *v) { return atomic_fetch_xor_relaxed(i, v); } -static inline long +static __always_inline long atomic_long_xchg(atomic_long_t *v, long i) { return atomic_xchg(v, i); } -static inline long +static __always_inline long atomic_long_xchg_acquire(atomic_long_t *v, long i) { return atomic_xchg_acquire(v, i); } -static inline long +static __always_inline long atomic_long_xchg_release(atomic_long_t *v, long i) { return atomic_xchg_release(v, i); } -static inline long +static __always_inline long atomic_long_xchg_relaxed(atomic_long_t *v, long i) { return atomic_xchg_relaxed(v, i); } -static inline long +static __always_inline long atomic_long_cmpxchg(atomic_long_t *v, long old, long new) { return atomic_cmpxchg(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_acquire(atomic_long_t *v, long old, long new) { return atomic_cmpxchg_acquire(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_release(atomic_long_t *v, long old, long new) { return atomic_cmpxchg_release(v, old, new); } -static inline long +static __always_inline long atomic_long_cmpxchg_relaxed(atomic_long_t *v, long old, long new) { return atomic_cmpxchg_relaxed(v, old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg(atomic_long_t *v, long *old, long new) { return atomic_try_cmpxchg(v, (int *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_acquire(atomic_long_t *v, long *old, long new) { return atomic_try_cmpxchg_acquire(v, (int *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_release(atomic_long_t *v, long *old, long new) { return atomic_try_cmpxchg_release(v, (int *)old, new); } -static inline bool +static __always_inline bool atomic_long_try_cmpxchg_relaxed(atomic_long_t *v, long *old, long new) { return atomic_try_cmpxchg_relaxed(v, (int *)old, new); } -static inline bool +static __always_inline bool atomic_long_sub_and_test(long i, atomic_long_t *v) { return atomic_sub_and_test(i, v); } -static inline bool +static __always_inline bool atomic_long_dec_and_test(atomic_long_t *v) { return atomic_dec_and_test(v); } -static inline bool +static __always_inline bool atomic_long_inc_and_test(atomic_long_t *v) { return atomic_inc_and_test(v); } -static inline bool +static __always_inline bool atomic_long_add_negative(long i, atomic_long_t *v) { return atomic_add_negative(i, v); } -static inline long +static __always_inline long atomic_long_fetch_add_unless(atomic_long_t *v, long a, long u) { return atomic_fetch_add_unless(v, a, u); } -static inline bool +static __always_inline bool atomic_long_add_unless(atomic_long_t *v, long a, long u) { return atomic_add_unless(v, a, u); } -static inline bool +static __always_inline bool atomic_long_inc_not_zero(atomic_long_t *v) { return atomic_inc_not_zero(v); } -static inline bool +static __always_inline bool atomic_long_inc_unless_negative(atomic_long_t *v) { return atomic_inc_unless_negative(v); } -static inline bool +static __always_inline bool atomic_long_dec_unless_positive(atomic_long_t *v) { return atomic_dec_unless_positive(v); } -static inline long +static __always_inline long atomic_long_dec_if_positive(atomic_long_t *v) { return atomic_dec_if_positive(v); @@ -1010,4 +1011,4 @@ atomic_long_dec_if_positive(atomic_long_t *v) #endif /* CONFIG_64BIT */ #endif /* _ASM_GENERIC_ATOMIC_LONG_H */ -// 77558968132ce4f911ad53f6f52ce423006f6268 +// a624200981f552b2c6be4f32fe44da8289f30d87 diff --git a/include/asm-generic/bitops/instrumented-atomic.h b/include/asm-generic/bitops/instrumented-atomic.h index 18ce3c9e8eec..fb2cb33a4013 100644 --- a/include/asm-generic/bitops/instrumented-atomic.h +++ b/include/asm-generic/bitops/instrumented-atomic.h @@ -11,7 +11,7 @@ #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H #define _ASM_GENERIC_BITOPS_INSTRUMENTED_ATOMIC_H -#include <linux/kasan-checks.h> +#include <linux/instrumented.h> /** * set_bit - Atomically set a bit in memory @@ -25,7 +25,7 @@ */ static inline void set_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); arch_set_bit(nr, addr); } @@ -38,7 +38,7 @@ static inline void set_bit(long nr, volatile unsigned long *addr) */ static inline void clear_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); arch_clear_bit(nr, addr); } @@ -54,7 +54,7 @@ static inline void clear_bit(long nr, volatile unsigned long *addr) */ static inline void change_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); arch_change_bit(nr, addr); } @@ -67,7 +67,7 @@ static inline void change_bit(long nr, volatile unsigned long *addr) */ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_set_bit(nr, addr); } @@ -80,7 +80,7 @@ static inline bool test_and_set_bit(long nr, volatile unsigned long *addr) */ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_clear_bit(nr, addr); } @@ -93,7 +93,7 @@ static inline bool test_and_clear_bit(long nr, volatile unsigned long *addr) */ static inline bool test_and_change_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_change_bit(nr, addr); } diff --git a/include/asm-generic/bitops/instrumented-lock.h b/include/asm-generic/bitops/instrumented-lock.h index ec53fdeea9ec..b9bec468ae03 100644 --- a/include/asm-generic/bitops/instrumented-lock.h +++ b/include/asm-generic/bitops/instrumented-lock.h @@ -11,7 +11,7 @@ #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H #define _ASM_GENERIC_BITOPS_INSTRUMENTED_LOCK_H -#include <linux/kasan-checks.h> +#include <linux/instrumented.h> /** * clear_bit_unlock - Clear a bit in memory, for unlock @@ -22,7 +22,7 @@ */ static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); arch_clear_bit_unlock(nr, addr); } @@ -37,7 +37,7 @@ static inline void clear_bit_unlock(long nr, volatile unsigned long *addr) */ static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___clear_bit_unlock(nr, addr); } @@ -52,7 +52,7 @@ static inline void __clear_bit_unlock(long nr, volatile unsigned long *addr) */ static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); return arch_test_and_set_bit_lock(nr, addr); } @@ -71,7 +71,7 @@ static inline bool test_and_set_bit_lock(long nr, volatile unsigned long *addr) static inline bool clear_bit_unlock_is_negative_byte(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_write(addr + BIT_WORD(nr), sizeof(long)); return arch_clear_bit_unlock_is_negative_byte(nr, addr); } /* Let everybody know we have it. */ diff --git a/include/asm-generic/bitops/instrumented-non-atomic.h b/include/asm-generic/bitops/instrumented-non-atomic.h index 95ff28d128a1..20f788a25ef9 100644 --- a/include/asm-generic/bitops/instrumented-non-atomic.h +++ b/include/asm-generic/bitops/instrumented-non-atomic.h @@ -11,7 +11,7 @@ #ifndef _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H #define _ASM_GENERIC_BITOPS_INSTRUMENTED_NON_ATOMIC_H -#include <linux/kasan-checks.h> +#include <linux/instrumented.h> /** * __set_bit - Set a bit in memory @@ -24,7 +24,7 @@ */ static inline void __set_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___set_bit(nr, addr); } @@ -39,7 +39,7 @@ static inline void __set_bit(long nr, volatile unsigned long *addr) */ static inline void __clear_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___clear_bit(nr, addr); } @@ -54,7 +54,7 @@ static inline void __clear_bit(long nr, volatile unsigned long *addr) */ static inline void __change_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); arch___change_bit(nr, addr); } @@ -68,7 +68,7 @@ static inline void __change_bit(long nr, volatile unsigned long *addr) */ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); return arch___test_and_set_bit(nr, addr); } @@ -82,7 +82,7 @@ static inline bool __test_and_set_bit(long nr, volatile unsigned long *addr) */ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); return arch___test_and_clear_bit(nr, addr); } @@ -96,7 +96,7 @@ static inline bool __test_and_clear_bit(long nr, volatile unsigned long *addr) */ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) { - kasan_check_write(addr + BIT_WORD(nr), sizeof(long)); + instrument_write(addr + BIT_WORD(nr), sizeof(long)); return arch___test_and_change_bit(nr, addr); } @@ -107,7 +107,7 @@ static inline bool __test_and_change_bit(long nr, volatile unsigned long *addr) */ static inline bool test_bit(long nr, const volatile unsigned long *addr) { - kasan_check_read(addr + BIT_WORD(nr), sizeof(long)); + instrument_atomic_read(addr + BIT_WORD(nr), sizeof(long)); return arch_test_bit(nr, addr); } diff --git a/include/asm-generic/bug.h b/include/asm-generic/bug.h index 384b5c835ced..c94e33ae3e7b 100644 --- a/include/asm-generic/bug.h +++ b/include/asm-generic/bug.h @@ -83,14 +83,19 @@ extern __printf(4, 5) void warn_slowpath_fmt(const char *file, const int line, unsigned taint, const char *fmt, ...); #define __WARN() __WARN_printf(TAINT_WARN, NULL) -#define __WARN_printf(taint, arg...) \ - warn_slowpath_fmt(__FILE__, __LINE__, taint, arg) +#define __WARN_printf(taint, arg...) do { \ + instrumentation_begin(); \ + warn_slowpath_fmt(__FILE__, __LINE__, taint, arg); \ + instrumentation_end(); \ + } while (0) #else extern __printf(1, 2) void __warn_printk(const char *fmt, ...); #define __WARN() __WARN_FLAGS(BUGFLAG_TAINT(TAINT_WARN)) #define __WARN_printf(taint, arg...) do { \ + instrumentation_begin(); \ __warn_printk(arg); \ __WARN_FLAGS(BUGFLAG_NO_CUT_HERE | BUGFLAG_TAINT(taint));\ + instrumentation_end(); \ } while (0) #define WARN_ON_ONCE(condition) ({ \ int __ret_warn_on = !!(condition); \ diff --git a/include/dt-bindings/mailbox/qcom-ipcc.h b/include/dt-bindings/mailbox/qcom-ipcc.h new file mode 100644 index 000000000000..4c23eefed5f3 --- /dev/null +++ b/include/dt-bindings/mailbox/qcom-ipcc.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause */ +/* + * Copyright (c) 2018-2020, The Linux Foundation. All rights reserved. + */ + +#ifndef __DT_BINDINGS_MAILBOX_IPCC_H +#define __DT_BINDINGS_MAILBOX_IPCC_H + +/* Signal IDs for MPROC protocol */ +#define IPCC_MPROC_SIGNAL_GLINK_QMP 0 +#define IPCC_MPROC_SIGNAL_SMP2P 2 +#define IPCC_MPROC_SIGNAL_PING 3 + +/* Client IDs */ +#define IPCC_CLIENT_AOP 0 +#define IPCC_CLIENT_TZ 1 +#define IPCC_CLIENT_MPSS 2 +#define IPCC_CLIENT_LPASS 3 +#define IPCC_CLIENT_SLPI 4 +#define IPCC_CLIENT_SDC 5 +#define IPCC_CLIENT_CDSP 6 +#define IPCC_CLIENT_NPU 7 +#define IPCC_CLIENT_APSS 8 +#define IPCC_CLIENT_GPU 9 +#define IPCC_CLIENT_CVP 10 +#define IPCC_CLIENT_CAM 11 +#define IPCC_CLIENT_VPU 12 +#define IPCC_CLIENT_PCIE0 13 +#define IPCC_CLIENT_PCIE1 14 +#define IPCC_CLIENT_PCIE2 15 +#define IPCC_CLIENT_SPSS 16 + +#endif diff --git a/include/linux/atomic-arch-fallback.h b/include/linux/atomic-arch-fallback.h new file mode 100644 index 000000000000..bcb6aa27cfa6 --- /dev/null +++ b/include/linux/atomic-arch-fallback.h @@ -0,0 +1,2291 @@ +// SPDX-License-Identifier: GPL-2.0 + +// Generated by scripts/atomic/gen-atomic-fallback.sh +// DO NOT MODIFY THIS FILE DIRECTLY + +#ifndef _LINUX_ATOMIC_FALLBACK_H +#define _LINUX_ATOMIC_FALLBACK_H + +#include <linux/compiler.h> + +#ifndef arch_xchg_relaxed +#define arch_xchg_relaxed arch_xchg +#define arch_xchg_acquire arch_xchg +#define arch_xchg_release arch_xchg +#else /* arch_xchg_relaxed */ + +#ifndef arch_xchg_acquire +#define arch_xchg_acquire(...) \ + __atomic_op_acquire(arch_xchg, __VA_ARGS__) +#endif + +#ifndef arch_xchg_release +#define arch_xchg_release(...) \ + __atomic_op_release(arch_xchg, __VA_ARGS__) +#endif + +#ifndef arch_xchg +#define arch_xchg(...) \ + __atomic_op_fence(arch_xchg, __VA_ARGS__) +#endif + +#endif /* arch_xchg_relaxed */ + +#ifndef arch_cmpxchg_relaxed +#define arch_cmpxchg_relaxed arch_cmpxchg +#define arch_cmpxchg_acquire arch_cmpxchg +#define arch_cmpxchg_release arch_cmpxchg +#else /* arch_cmpxchg_relaxed */ + +#ifndef arch_cmpxchg_acquire +#define arch_cmpxchg_acquire(...) \ + __atomic_op_acquire(arch_cmpxchg, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg_release +#define arch_cmpxchg_release(...) \ + __atomic_op_release(arch_cmpxchg, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg +#define arch_cmpxchg(...) \ + __atomic_op_fence(arch_cmpxchg, __VA_ARGS__) +#endif + +#endif /* arch_cmpxchg_relaxed */ + +#ifndef arch_cmpxchg64_relaxed +#define arch_cmpxchg64_relaxed arch_cmpxchg64 +#define arch_cmpxchg64_acquire arch_cmpxchg64 +#define arch_cmpxchg64_release arch_cmpxchg64 +#else /* arch_cmpxchg64_relaxed */ + +#ifndef arch_cmpxchg64_acquire +#define arch_cmpxchg64_acquire(...) \ + __atomic_op_acquire(arch_cmpxchg64, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg64_release +#define arch_cmpxchg64_release(...) \ + __atomic_op_release(arch_cmpxchg64, __VA_ARGS__) +#endif + +#ifndef arch_cmpxchg64 +#define arch_cmpxchg64(...) \ + __atomic_op_fence(arch_cmpxchg64, __VA_ARGS__) +#endif + +#endif /* arch_cmpxchg64_relaxed */ + +#ifndef arch_atomic_read_acquire +static __always_inline int +arch_atomic_read_acquire(const atomic_t *v) +{ + return smp_load_acquire(&(v)->counter); +} +#define arch_atomic_read_acquire arch_atomic_read_acquire +#endif + +#ifndef arch_atomic_set_release +static __always_inline void +arch_atomic_set_release(atomic_t *v, int i) +{ + smp_store_release(&(v)->counter, i); +} +#define arch_atomic_set_release arch_atomic_set_release +#endif + +#ifndef arch_atomic_add_return_relaxed +#define arch_atomic_add_return_acquire arch_atomic_add_return +#define arch_atomic_add_return_release arch_atomic_add_return +#define arch_atomic_add_return_relaxed arch_atomic_add_return +#else /* arch_atomic_add_return_relaxed */ + +#ifndef arch_atomic_add_return_acquire +static __always_inline int +arch_atomic_add_return_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_add_return_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_add_return_acquire arch_atomic_add_return_acquire +#endif + +#ifndef arch_atomic_add_return_release +static __always_inline int +arch_atomic_add_return_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_add_return_relaxed(i, v); +} +#define arch_atomic_add_return_release arch_atomic_add_return_release +#endif + +#ifndef arch_atomic_add_return +static __always_inline int +arch_atomic_add_return(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_add_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_add_return arch_atomic_add_return +#endif + +#endif /* arch_atomic_add_return_relaxed */ + +#ifndef arch_atomic_fetch_add_relaxed +#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add +#define arch_atomic_fetch_add_release arch_atomic_fetch_add +#define arch_atomic_fetch_add_relaxed arch_atomic_fetch_add +#else /* arch_atomic_fetch_add_relaxed */ + +#ifndef arch_atomic_fetch_add_acquire +static __always_inline int +arch_atomic_fetch_add_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_add_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_add_acquire arch_atomic_fetch_add_acquire +#endif + +#ifndef arch_atomic_fetch_add_release +static __always_inline int +arch_atomic_fetch_add_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_add_relaxed(i, v); +} +#define arch_atomic_fetch_add_release arch_atomic_fetch_add_release +#endif + +#ifndef arch_atomic_fetch_add +static __always_inline int +arch_atomic_fetch_add(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_add_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_add arch_atomic_fetch_add +#endif + +#endif /* arch_atomic_fetch_add_relaxed */ + +#ifndef arch_atomic_sub_return_relaxed +#define arch_atomic_sub_return_acquire arch_atomic_sub_return +#define arch_atomic_sub_return_release arch_atomic_sub_return +#define arch_atomic_sub_return_relaxed arch_atomic_sub_return +#else /* arch_atomic_sub_return_relaxed */ + +#ifndef arch_atomic_sub_return_acquire +static __always_inline int +arch_atomic_sub_return_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_sub_return_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_sub_return_acquire arch_atomic_sub_return_acquire +#endif + +#ifndef arch_atomic_sub_return_release +static __always_inline int +arch_atomic_sub_return_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_sub_return_relaxed(i, v); +} +#define arch_atomic_sub_return_release arch_atomic_sub_return_release +#endif + +#ifndef arch_atomic_sub_return +static __always_inline int +arch_atomic_sub_return(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_sub_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_sub_return arch_atomic_sub_return +#endif + +#endif /* arch_atomic_sub_return_relaxed */ + +#ifndef arch_atomic_fetch_sub_relaxed +#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub +#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub +#define arch_atomic_fetch_sub_relaxed arch_atomic_fetch_sub +#else /* arch_atomic_fetch_sub_relaxed */ + +#ifndef arch_atomic_fetch_sub_acquire +static __always_inline int +arch_atomic_fetch_sub_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_sub_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_sub_acquire arch_atomic_fetch_sub_acquire +#endif + +#ifndef arch_atomic_fetch_sub_release +static __always_inline int +arch_atomic_fetch_sub_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_sub_relaxed(i, v); +} +#define arch_atomic_fetch_sub_release arch_atomic_fetch_sub_release +#endif + +#ifndef arch_atomic_fetch_sub +static __always_inline int +arch_atomic_fetch_sub(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_sub_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_sub arch_atomic_fetch_sub +#endif + +#endif /* arch_atomic_fetch_sub_relaxed */ + +#ifndef arch_atomic_inc +static __always_inline void +arch_atomic_inc(atomic_t *v) +{ + arch_atomic_add(1, v); +} +#define arch_atomic_inc arch_atomic_inc +#endif + +#ifndef arch_atomic_inc_return_relaxed +#ifdef arch_atomic_inc_return +#define arch_atomic_inc_return_acquire arch_atomic_inc_return +#define arch_atomic_inc_return_release arch_atomic_inc_return +#define arch_atomic_inc_return_relaxed arch_atomic_inc_return +#endif /* arch_atomic_inc_return */ + +#ifndef arch_atomic_inc_return +static __always_inline int +arch_atomic_inc_return(atomic_t *v) +{ + return arch_atomic_add_return(1, v); +} +#define arch_atomic_inc_return arch_atomic_inc_return +#endif + +#ifndef arch_atomic_inc_return_acquire +static __always_inline int +arch_atomic_inc_return_acquire(atomic_t *v) +{ + return arch_atomic_add_return_acquire(1, v); +} +#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire +#endif + +#ifndef arch_atomic_inc_return_release +static __always_inline int +arch_atomic_inc_return_release(atomic_t *v) +{ + return arch_atomic_add_return_release(1, v); +} +#define arch_atomic_inc_return_release arch_atomic_inc_return_release +#endif + +#ifndef arch_atomic_inc_return_relaxed +static __always_inline int +arch_atomic_inc_return_relaxed(atomic_t *v) +{ + return arch_atomic_add_return_relaxed(1, v); +} +#define arch_atomic_inc_return_relaxed arch_atomic_inc_return_relaxed +#endif + +#else /* arch_atomic_inc_return_relaxed */ + +#ifndef arch_atomic_inc_return_acquire +static __always_inline int +arch_atomic_inc_return_acquire(atomic_t *v) +{ + int ret = arch_atomic_inc_return_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_inc_return_acquire arch_atomic_inc_return_acquire +#endif + +#ifndef arch_atomic_inc_return_release +static __always_inline int +arch_atomic_inc_return_release(atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_inc_return_relaxed(v); +} +#define arch_atomic_inc_return_release arch_atomic_inc_return_release +#endif + +#ifndef arch_atomic_inc_return +static __always_inline int +arch_atomic_inc_return(atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_inc_return_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_inc_return arch_atomic_inc_return +#endif + +#endif /* arch_atomic_inc_return_relaxed */ + +#ifndef arch_atomic_fetch_inc_relaxed +#ifdef arch_atomic_fetch_inc +#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc +#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc +#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc +#endif /* arch_atomic_fetch_inc */ + +#ifndef arch_atomic_fetch_inc +static __always_inline int +arch_atomic_fetch_inc(atomic_t *v) +{ + return arch_atomic_fetch_add(1, v); +} +#define arch_atomic_fetch_inc arch_atomic_fetch_inc +#endif + +#ifndef arch_atomic_fetch_inc_acquire +static __always_inline int +arch_atomic_fetch_inc_acquire(atomic_t *v) +{ + return arch_atomic_fetch_add_acquire(1, v); +} +#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire +#endif + +#ifndef arch_atomic_fetch_inc_release +static __always_inline int +arch_atomic_fetch_inc_release(atomic_t *v) +{ + return arch_atomic_fetch_add_release(1, v); +} +#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release +#endif + +#ifndef arch_atomic_fetch_inc_relaxed +static __always_inline int +arch_atomic_fetch_inc_relaxed(atomic_t *v) +{ + return arch_atomic_fetch_add_relaxed(1, v); +} +#define arch_atomic_fetch_inc_relaxed arch_atomic_fetch_inc_relaxed +#endif + +#else /* arch_atomic_fetch_inc_relaxed */ + +#ifndef arch_atomic_fetch_inc_acquire +static __always_inline int +arch_atomic_fetch_inc_acquire(atomic_t *v) +{ + int ret = arch_atomic_fetch_inc_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_inc_acquire arch_atomic_fetch_inc_acquire +#endif + +#ifndef arch_atomic_fetch_inc_release +static __always_inline int +arch_atomic_fetch_inc_release(atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_inc_relaxed(v); +} +#define arch_atomic_fetch_inc_release arch_atomic_fetch_inc_release +#endif + +#ifndef arch_atomic_fetch_inc +static __always_inline int +arch_atomic_fetch_inc(atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_inc_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_inc arch_atomic_fetch_inc +#endif + +#endif /* arch_atomic_fetch_inc_relaxed */ + +#ifndef arch_atomic_dec +static __always_inline void +arch_atomic_dec(atomic_t *v) +{ + arch_atomic_sub(1, v); +} +#define arch_atomic_dec arch_atomic_dec +#endif + +#ifndef arch_atomic_dec_return_relaxed +#ifdef arch_atomic_dec_return +#define arch_atomic_dec_return_acquire arch_atomic_dec_return +#define arch_atomic_dec_return_release arch_atomic_dec_return +#define arch_atomic_dec_return_relaxed arch_atomic_dec_return +#endif /* arch_atomic_dec_return */ + +#ifndef arch_atomic_dec_return +static __always_inline int +arch_atomic_dec_return(atomic_t *v) +{ + return arch_atomic_sub_return(1, v); +} +#define arch_atomic_dec_return arch_atomic_dec_return +#endif + +#ifndef arch_atomic_dec_return_acquire +static __always_inline int +arch_atomic_dec_return_acquire(atomic_t *v) +{ + return arch_atomic_sub_return_acquire(1, v); +} +#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire +#endif + +#ifndef arch_atomic_dec_return_release +static __always_inline int +arch_atomic_dec_return_release(atomic_t *v) +{ + return arch_atomic_sub_return_release(1, v); +} +#define arch_atomic_dec_return_release arch_atomic_dec_return_release +#endif + +#ifndef arch_atomic_dec_return_relaxed +static __always_inline int +arch_atomic_dec_return_relaxed(atomic_t *v) +{ + return arch_atomic_sub_return_relaxed(1, v); +} +#define arch_atomic_dec_return_relaxed arch_atomic_dec_return_relaxed +#endif + +#else /* arch_atomic_dec_return_relaxed */ + +#ifndef arch_atomic_dec_return_acquire +static __always_inline int +arch_atomic_dec_return_acquire(atomic_t *v) +{ + int ret = arch_atomic_dec_return_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_dec_return_acquire arch_atomic_dec_return_acquire +#endif + +#ifndef arch_atomic_dec_return_release +static __always_inline int +arch_atomic_dec_return_release(atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_dec_return_relaxed(v); +} +#define arch_atomic_dec_return_release arch_atomic_dec_return_release +#endif + +#ifndef arch_atomic_dec_return +static __always_inline int +arch_atomic_dec_return(atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_dec_return_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_dec_return arch_atomic_dec_return +#endif + +#endif /* arch_atomic_dec_return_relaxed */ + +#ifndef arch_atomic_fetch_dec_relaxed +#ifdef arch_atomic_fetch_dec +#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec +#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec +#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec +#endif /* arch_atomic_fetch_dec */ + +#ifndef arch_atomic_fetch_dec +static __always_inline int +arch_atomic_fetch_dec(atomic_t *v) +{ + return arch_atomic_fetch_sub(1, v); +} +#define arch_atomic_fetch_dec arch_atomic_fetch_dec +#endif + +#ifndef arch_atomic_fetch_dec_acquire +static __always_inline int +arch_atomic_fetch_dec_acquire(atomic_t *v) +{ + return arch_atomic_fetch_sub_acquire(1, v); +} +#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire +#endif + +#ifndef arch_atomic_fetch_dec_release +static __always_inline int +arch_atomic_fetch_dec_release(atomic_t *v) +{ + return arch_atomic_fetch_sub_release(1, v); +} +#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release +#endif + +#ifndef arch_atomic_fetch_dec_relaxed +static __always_inline int +arch_atomic_fetch_dec_relaxed(atomic_t *v) +{ + return arch_atomic_fetch_sub_relaxed(1, v); +} +#define arch_atomic_fetch_dec_relaxed arch_atomic_fetch_dec_relaxed +#endif + +#else /* arch_atomic_fetch_dec_relaxed */ + +#ifndef arch_atomic_fetch_dec_acquire +static __always_inline int +arch_atomic_fetch_dec_acquire(atomic_t *v) +{ + int ret = arch_atomic_fetch_dec_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_dec_acquire arch_atomic_fetch_dec_acquire +#endif + +#ifndef arch_atomic_fetch_dec_release +static __always_inline int +arch_atomic_fetch_dec_release(atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_dec_relaxed(v); +} +#define arch_atomic_fetch_dec_release arch_atomic_fetch_dec_release +#endif + +#ifndef arch_atomic_fetch_dec +static __always_inline int +arch_atomic_fetch_dec(atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_dec_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_dec arch_atomic_fetch_dec +#endif + +#endif /* arch_atomic_fetch_dec_relaxed */ + +#ifndef arch_atomic_fetch_and_relaxed +#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and +#define arch_atomic_fetch_and_release arch_atomic_fetch_and +#define arch_atomic_fetch_and_relaxed arch_atomic_fetch_and +#else /* arch_atomic_fetch_and_relaxed */ + +#ifndef arch_atomic_fetch_and_acquire +static __always_inline int +arch_atomic_fetch_and_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_and_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_and_acquire arch_atomic_fetch_and_acquire +#endif + +#ifndef arch_atomic_fetch_and_release +static __always_inline int +arch_atomic_fetch_and_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_and_relaxed(i, v); +} +#define arch_atomic_fetch_and_release arch_atomic_fetch_and_release +#endif + +#ifndef arch_atomic_fetch_and +static __always_inline int +arch_atomic_fetch_and(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_and_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_and arch_atomic_fetch_and +#endif + +#endif /* arch_atomic_fetch_and_relaxed */ + +#ifndef arch_atomic_andnot +static __always_inline void +arch_atomic_andnot(int i, atomic_t *v) +{ + arch_atomic_and(~i, v); +} +#define arch_atomic_andnot arch_atomic_andnot +#endif + +#ifndef arch_atomic_fetch_andnot_relaxed +#ifdef arch_atomic_fetch_andnot +#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot +#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot +#endif /* arch_atomic_fetch_andnot */ + +#ifndef arch_atomic_fetch_andnot +static __always_inline int +arch_atomic_fetch_andnot(int i, atomic_t *v) +{ + return arch_atomic_fetch_and(~i, v); +} +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#endif + +#ifndef arch_atomic_fetch_andnot_acquire +static __always_inline int +arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_acquire(~i, v); +} +#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire +#endif + +#ifndef arch_atomic_fetch_andnot_release +static __always_inline int +arch_atomic_fetch_andnot_release(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_release(~i, v); +} +#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release +#endif + +#ifndef arch_atomic_fetch_andnot_relaxed +static __always_inline int +arch_atomic_fetch_andnot_relaxed(int i, atomic_t *v) +{ + return arch_atomic_fetch_and_relaxed(~i, v); +} +#define arch_atomic_fetch_andnot_relaxed arch_atomic_fetch_andnot_relaxed +#endif + +#else /* arch_atomic_fetch_andnot_relaxed */ + +#ifndef arch_atomic_fetch_andnot_acquire +static __always_inline int +arch_atomic_fetch_andnot_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_andnot_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_andnot_acquire arch_atomic_fetch_andnot_acquire +#endif + +#ifndef arch_atomic_fetch_andnot_release +static __always_inline int +arch_atomic_fetch_andnot_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_andnot_relaxed(i, v); +} +#define arch_atomic_fetch_andnot_release arch_atomic_fetch_andnot_release +#endif + +#ifndef arch_atomic_fetch_andnot +static __always_inline int +arch_atomic_fetch_andnot(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_andnot_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_andnot arch_atomic_fetch_andnot +#endif + +#endif /* arch_atomic_fetch_andnot_relaxed */ + +#ifndef arch_atomic_fetch_or_relaxed +#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or +#define arch_atomic_fetch_or_release arch_atomic_fetch_or +#define arch_atomic_fetch_or_relaxed arch_atomic_fetch_or +#else /* arch_atomic_fetch_or_relaxed */ + +#ifndef arch_atomic_fetch_or_acquire +static __always_inline int +arch_atomic_fetch_or_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_or_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_or_acquire arch_atomic_fetch_or_acquire +#endif + +#ifndef arch_atomic_fetch_or_release +static __always_inline int +arch_atomic_fetch_or_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_or_relaxed(i, v); +} +#define arch_atomic_fetch_or_release arch_atomic_fetch_or_release +#endif + +#ifndef arch_atomic_fetch_or +static __always_inline int +arch_atomic_fetch_or(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_or_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_or arch_atomic_fetch_or +#endif + +#endif /* arch_atomic_fetch_or_relaxed */ + +#ifndef arch_atomic_fetch_xor_relaxed +#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor +#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor +#define arch_atomic_fetch_xor_relaxed arch_atomic_fetch_xor +#else /* arch_atomic_fetch_xor_relaxed */ + +#ifndef arch_atomic_fetch_xor_acquire +static __always_inline int +arch_atomic_fetch_xor_acquire(int i, atomic_t *v) +{ + int ret = arch_atomic_fetch_xor_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_fetch_xor_acquire arch_atomic_fetch_xor_acquire +#endif + +#ifndef arch_atomic_fetch_xor_release +static __always_inline int +arch_atomic_fetch_xor_release(int i, atomic_t *v) +{ + __atomic_release_fence(); + return arch_atomic_fetch_xor_relaxed(i, v); +} +#define arch_atomic_fetch_xor_release arch_atomic_fetch_xor_release +#endif + +#ifndef arch_atomic_fetch_xor +static __always_inline int +arch_atomic_fetch_xor(int i, atomic_t *v) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_fetch_xor_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_fetch_xor arch_atomic_fetch_xor +#endif + +#endif /* arch_atomic_fetch_xor_relaxed */ + +#ifndef arch_atomic_xchg_relaxed +#define arch_atomic_xchg_acquire arch_atomic_xchg +#define arch_atomic_xchg_release arch_atomic_xchg +#define arch_atomic_xchg_relaxed arch_atomic_xchg +#else /* arch_atomic_xchg_relaxed */ + +#ifndef arch_atomic_xchg_acquire +static __always_inline int +arch_atomic_xchg_acquire(atomic_t *v, int i) +{ + int ret = arch_atomic_xchg_relaxed(v, i); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_xchg_acquire arch_atomic_xchg_acquire +#endif + +#ifndef arch_atomic_xchg_release +static __always_inline int +arch_atomic_xchg_release(atomic_t *v, int i) +{ + __atomic_release_fence(); + return arch_atomic_xchg_relaxed(v, i); +} +#define arch_atomic_xchg_release arch_atomic_xchg_release +#endif + +#ifndef arch_atomic_xchg +static __always_inline int +arch_atomic_xchg(atomic_t *v, int i) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_xchg_relaxed(v, i); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_xchg arch_atomic_xchg +#endif + +#endif /* arch_atomic_xchg_relaxed */ + +#ifndef arch_atomic_cmpxchg_relaxed +#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg +#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg +#define arch_atomic_cmpxchg_relaxed arch_atomic_cmpxchg +#else /* arch_atomic_cmpxchg_relaxed */ + +#ifndef arch_atomic_cmpxchg_acquire +static __always_inline int +arch_atomic_cmpxchg_acquire(atomic_t *v, int old, int new) +{ + int ret = arch_atomic_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_cmpxchg_acquire arch_atomic_cmpxchg_acquire +#endif + +#ifndef arch_atomic_cmpxchg_release +static __always_inline int +arch_atomic_cmpxchg_release(atomic_t *v, int old, int new) +{ + __atomic_release_fence(); + return arch_atomic_cmpxchg_relaxed(v, old, new); +} +#define arch_atomic_cmpxchg_release arch_atomic_cmpxchg_release +#endif + +#ifndef arch_atomic_cmpxchg +static __always_inline int +arch_atomic_cmpxchg(atomic_t *v, int old, int new) +{ + int ret; + __atomic_pre_full_fence(); + ret = arch_atomic_cmpxchg_relaxed(v, old, new); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_cmpxchg arch_atomic_cmpxchg +#endif + +#endif /* arch_atomic_cmpxchg_relaxed */ + +#ifndef arch_atomic_try_cmpxchg_relaxed +#ifdef arch_atomic_try_cmpxchg +#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg +#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg +#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg +#endif /* arch_atomic_try_cmpxchg */ + +#ifndef arch_atomic_try_cmpxchg +static __always_inline bool +arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + int r, o = *old; + r = arch_atomic_cmpxchg(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg +#endif + +#ifndef arch_atomic_try_cmpxchg_acquire +static __always_inline bool +arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + int r, o = *old; + r = arch_atomic_cmpxchg_acquire(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire +#endif + +#ifndef arch_atomic_try_cmpxchg_release +static __always_inline bool +arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + int r, o = *old; + r = arch_atomic_cmpxchg_release(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release +#endif + +#ifndef arch_atomic_try_cmpxchg_relaxed +static __always_inline bool +arch_atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) +{ + int r, o = *old; + r = arch_atomic_cmpxchg_relaxed(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic_try_cmpxchg_relaxed arch_atomic_try_cmpxchg_relaxed +#endif + +#else /* arch_atomic_try_cmpxchg_relaxed */ + +#ifndef arch_atomic_try_cmpxchg_acquire +static __always_inline bool +arch_atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) +{ + bool ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic_try_cmpxchg_acquire arch_atomic_try_cmpxchg_acquire +#endif + +#ifndef arch_atomic_try_cmpxchg_release +static __always_inline bool +arch_atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) +{ + __atomic_release_fence(); + return arch_atomic_try_cmpxchg_relaxed(v, old, new); +} +#define arch_atomic_try_cmpxchg_release arch_atomic_try_cmpxchg_release +#endif + +#ifndef arch_atomic_try_cmpxchg +static __always_inline bool +arch_atomic_try_cmpxchg(atomic_t *v, int *old, int new) +{ + bool ret; + __atomic_pre_full_fence(); + ret = arch_atomic_try_cmpxchg_relaxed(v, old, new); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic_try_cmpxchg arch_atomic_try_cmpxchg +#endif + +#endif /* arch_atomic_try_cmpxchg_relaxed */ + +#ifndef arch_atomic_sub_and_test +/** + * arch_atomic_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_atomic_sub_and_test(int i, atomic_t *v) +{ + return arch_atomic_sub_return(i, v) == 0; +} +#define arch_atomic_sub_and_test arch_atomic_sub_and_test +#endif + +#ifndef arch_atomic_dec_and_test +/** + * arch_atomic_dec_and_test - decrement and test + * @v: pointer of type atomic_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __always_inline bool +arch_atomic_dec_and_test(atomic_t *v) +{ + return arch_atomic_dec_return(v) == 0; +} +#define arch_atomic_dec_and_test arch_atomic_dec_and_test +#endif + +#ifndef arch_atomic_inc_and_test +/** + * arch_atomic_inc_and_test - increment and test + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_atomic_inc_and_test(atomic_t *v) +{ + return arch_atomic_inc_return(v) == 0; +} +#define arch_atomic_inc_and_test arch_atomic_inc_and_test +#endif + +#ifndef arch_atomic_add_negative +/** + * arch_atomic_add_negative - add and test if negative + * @i: integer value to add + * @v: pointer of type atomic_t + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __always_inline bool +arch_atomic_add_negative(int i, atomic_t *v) +{ + return arch_atomic_add_return(i, v) < 0; +} +#define arch_atomic_add_negative arch_atomic_add_negative +#endif + +#ifndef arch_atomic_fetch_add_unless +/** + * arch_atomic_fetch_add_unless - add unless the number is already a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as @v was not already @u. + * Returns original value of @v + */ +static __always_inline int +arch_atomic_fetch_add_unless(atomic_t *v, int a, int u) +{ + int c = arch_atomic_read(v); + + do { + if (unlikely(c == u)) + break; + } while (!arch_atomic_try_cmpxchg(v, &c, c + a)); + + return c; +} +#define arch_atomic_fetch_add_unless arch_atomic_fetch_add_unless +#endif + +#ifndef arch_atomic_add_unless +/** + * arch_atomic_add_unless - add unless the number is already a given value + * @v: pointer of type atomic_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, if @v was not already @u. + * Returns true if the addition was done. + */ +static __always_inline bool +arch_atomic_add_unless(atomic_t *v, int a, int u) +{ + return arch_atomic_fetch_add_unless(v, a, u) != u; +} +#define arch_atomic_add_unless arch_atomic_add_unless +#endif + +#ifndef arch_atomic_inc_not_zero +/** + * arch_atomic_inc_not_zero - increment unless the number is zero + * @v: pointer of type atomic_t + * + * Atomically increments @v by 1, if @v is non-zero. + * Returns true if the increment was done. + */ +static __always_inline bool +arch_atomic_inc_not_zero(atomic_t *v) +{ + return arch_atomic_add_unless(v, 1, 0); +} +#define arch_atomic_inc_not_zero arch_atomic_inc_not_zero +#endif + +#ifndef arch_atomic_inc_unless_negative +static __always_inline bool +arch_atomic_inc_unless_negative(atomic_t *v) +{ + int c = arch_atomic_read(v); + + do { + if (unlikely(c < 0)) + return false; + } while (!arch_atomic_try_cmpxchg(v, &c, c + 1)); + + return true; +} +#define arch_atomic_inc_unless_negative arch_atomic_inc_unless_negative +#endif + +#ifndef arch_atomic_dec_unless_positive +static __always_inline bool +arch_atomic_dec_unless_positive(atomic_t *v) +{ + int c = arch_atomic_read(v); + + do { + if (unlikely(c > 0)) + return false; + } while (!arch_atomic_try_cmpxchg(v, &c, c - 1)); + + return true; +} +#define arch_atomic_dec_unless_positive arch_atomic_dec_unless_positive +#endif + +#ifndef arch_atomic_dec_if_positive +static __always_inline int +arch_atomic_dec_if_positive(atomic_t *v) +{ + int dec, c = arch_atomic_read(v); + + do { + dec = c - 1; + if (unlikely(dec < 0)) + break; + } while (!arch_atomic_try_cmpxchg(v, &c, dec)); + + return dec; +} +#define arch_atomic_dec_if_positive arch_atomic_dec_if_positive +#endif + +#ifdef CONFIG_GENERIC_ATOMIC64 +#include <asm-generic/atomic64.h> +#endif + +#ifndef arch_atomic64_read_acquire +static __always_inline s64 +arch_atomic64_read_acquire(const atomic64_t *v) +{ + return smp_load_acquire(&(v)->counter); +} +#define arch_atomic64_read_acquire arch_atomic64_read_acquire +#endif + +#ifndef arch_atomic64_set_release +static __always_inline void +arch_atomic64_set_release(atomic64_t *v, s64 i) +{ + smp_store_release(&(v)->counter, i); +} +#define arch_atomic64_set_release arch_atomic64_set_release +#endif + +#ifndef arch_atomic64_add_return_relaxed +#define arch_atomic64_add_return_acquire arch_atomic64_add_return +#define arch_atomic64_add_return_release arch_atomic64_add_return +#define arch_atomic64_add_return_relaxed arch_atomic64_add_return +#else /* arch_atomic64_add_return_relaxed */ + +#ifndef arch_atomic64_add_return_acquire +static __always_inline s64 +arch_atomic64_add_return_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_add_return_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_add_return_acquire arch_atomic64_add_return_acquire +#endif + +#ifndef arch_atomic64_add_return_release +static __always_inline s64 +arch_atomic64_add_return_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_add_return_relaxed(i, v); +} +#define arch_atomic64_add_return_release arch_atomic64_add_return_release +#endif + +#ifndef arch_atomic64_add_return +static __always_inline s64 +arch_atomic64_add_return(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_add_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_add_return arch_atomic64_add_return +#endif + +#endif /* arch_atomic64_add_return_relaxed */ + +#ifndef arch_atomic64_fetch_add_relaxed +#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add +#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add +#define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add +#else /* arch_atomic64_fetch_add_relaxed */ + +#ifndef arch_atomic64_fetch_add_acquire +static __always_inline s64 +arch_atomic64_fetch_add_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_add_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_add_acquire arch_atomic64_fetch_add_acquire +#endif + +#ifndef arch_atomic64_fetch_add_release +static __always_inline s64 +arch_atomic64_fetch_add_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_add_relaxed(i, v); +} +#define arch_atomic64_fetch_add_release arch_atomic64_fetch_add_release +#endif + +#ifndef arch_atomic64_fetch_add +static __always_inline s64 +arch_atomic64_fetch_add(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_add_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_add arch_atomic64_fetch_add +#endif + +#endif /* arch_atomic64_fetch_add_relaxed */ + +#ifndef arch_atomic64_sub_return_relaxed +#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return +#define arch_atomic64_sub_return_release arch_atomic64_sub_return +#define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return +#else /* arch_atomic64_sub_return_relaxed */ + +#ifndef arch_atomic64_sub_return_acquire +static __always_inline s64 +arch_atomic64_sub_return_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_sub_return_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_sub_return_acquire arch_atomic64_sub_return_acquire +#endif + +#ifndef arch_atomic64_sub_return_release +static __always_inline s64 +arch_atomic64_sub_return_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_sub_return_relaxed(i, v); +} +#define arch_atomic64_sub_return_release arch_atomic64_sub_return_release +#endif + +#ifndef arch_atomic64_sub_return +static __always_inline s64 +arch_atomic64_sub_return(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_sub_return_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_sub_return arch_atomic64_sub_return +#endif + +#endif /* arch_atomic64_sub_return_relaxed */ + +#ifndef arch_atomic64_fetch_sub_relaxed +#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub +#define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub +#else /* arch_atomic64_fetch_sub_relaxed */ + +#ifndef arch_atomic64_fetch_sub_acquire +static __always_inline s64 +arch_atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_sub_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_sub_acquire arch_atomic64_fetch_sub_acquire +#endif + +#ifndef arch_atomic64_fetch_sub_release +static __always_inline s64 +arch_atomic64_fetch_sub_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_sub_relaxed(i, v); +} +#define arch_atomic64_fetch_sub_release arch_atomic64_fetch_sub_release +#endif + +#ifndef arch_atomic64_fetch_sub +static __always_inline s64 +arch_atomic64_fetch_sub(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_sub_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_sub arch_atomic64_fetch_sub +#endif + +#endif /* arch_atomic64_fetch_sub_relaxed */ + +#ifndef arch_atomic64_inc +static __always_inline void +arch_atomic64_inc(atomic64_t *v) +{ + arch_atomic64_add(1, v); +} +#define arch_atomic64_inc arch_atomic64_inc +#endif + +#ifndef arch_atomic64_inc_return_relaxed +#ifdef arch_atomic64_inc_return +#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return +#define arch_atomic64_inc_return_release arch_atomic64_inc_return +#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return +#endif /* arch_atomic64_inc_return */ + +#ifndef arch_atomic64_inc_return +static __always_inline s64 +arch_atomic64_inc_return(atomic64_t *v) +{ + return arch_atomic64_add_return(1, v); +} +#define arch_atomic64_inc_return arch_atomic64_inc_return +#endif + +#ifndef arch_atomic64_inc_return_acquire +static __always_inline s64 +arch_atomic64_inc_return_acquire(atomic64_t *v) +{ + return arch_atomic64_add_return_acquire(1, v); +} +#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire +#endif + +#ifndef arch_atomic64_inc_return_release +static __always_inline s64 +arch_atomic64_inc_return_release(atomic64_t *v) +{ + return arch_atomic64_add_return_release(1, v); +} +#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release +#endif + +#ifndef arch_atomic64_inc_return_relaxed +static __always_inline s64 +arch_atomic64_inc_return_relaxed(atomic64_t *v) +{ + return arch_atomic64_add_return_relaxed(1, v); +} +#define arch_atomic64_inc_return_relaxed arch_atomic64_inc_return_relaxed +#endif + +#else /* arch_atomic64_inc_return_relaxed */ + +#ifndef arch_atomic64_inc_return_acquire +static __always_inline s64 +arch_atomic64_inc_return_acquire(atomic64_t *v) +{ + s64 ret = arch_atomic64_inc_return_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_inc_return_acquire arch_atomic64_inc_return_acquire +#endif + +#ifndef arch_atomic64_inc_return_release +static __always_inline s64 +arch_atomic64_inc_return_release(atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_inc_return_relaxed(v); +} +#define arch_atomic64_inc_return_release arch_atomic64_inc_return_release +#endif + +#ifndef arch_atomic64_inc_return +static __always_inline s64 +arch_atomic64_inc_return(atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_inc_return_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_inc_return arch_atomic64_inc_return +#endif + +#endif /* arch_atomic64_inc_return_relaxed */ + +#ifndef arch_atomic64_fetch_inc_relaxed +#ifdef arch_atomic64_fetch_inc +#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc +#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc +#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc +#endif /* arch_atomic64_fetch_inc */ + +#ifndef arch_atomic64_fetch_inc +static __always_inline s64 +arch_atomic64_fetch_inc(atomic64_t *v) +{ + return arch_atomic64_fetch_add(1, v); +} +#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc +#endif + +#ifndef arch_atomic64_fetch_inc_acquire +static __always_inline s64 +arch_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + return arch_atomic64_fetch_add_acquire(1, v); +} +#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire +#endif + +#ifndef arch_atomic64_fetch_inc_release +static __always_inline s64 +arch_atomic64_fetch_inc_release(atomic64_t *v) +{ + return arch_atomic64_fetch_add_release(1, v); +} +#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release +#endif + +#ifndef arch_atomic64_fetch_inc_relaxed +static __always_inline s64 +arch_atomic64_fetch_inc_relaxed(atomic64_t *v) +{ + return arch_atomic64_fetch_add_relaxed(1, v); +} +#define arch_atomic64_fetch_inc_relaxed arch_atomic64_fetch_inc_relaxed +#endif + +#else /* arch_atomic64_fetch_inc_relaxed */ + +#ifndef arch_atomic64_fetch_inc_acquire +static __always_inline s64 +arch_atomic64_fetch_inc_acquire(atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_inc_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_inc_acquire arch_atomic64_fetch_inc_acquire +#endif + +#ifndef arch_atomic64_fetch_inc_release +static __always_inline s64 +arch_atomic64_fetch_inc_release(atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_inc_relaxed(v); +} +#define arch_atomic64_fetch_inc_release arch_atomic64_fetch_inc_release +#endif + +#ifndef arch_atomic64_fetch_inc +static __always_inline s64 +arch_atomic64_fetch_inc(atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_inc_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_inc arch_atomic64_fetch_inc +#endif + +#endif /* arch_atomic64_fetch_inc_relaxed */ + +#ifndef arch_atomic64_dec +static __always_inline void +arch_atomic64_dec(atomic64_t *v) +{ + arch_atomic64_sub(1, v); +} +#define arch_atomic64_dec arch_atomic64_dec +#endif + +#ifndef arch_atomic64_dec_return_relaxed +#ifdef arch_atomic64_dec_return +#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return +#define arch_atomic64_dec_return_release arch_atomic64_dec_return +#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return +#endif /* arch_atomic64_dec_return */ + +#ifndef arch_atomic64_dec_return +static __always_inline s64 +arch_atomic64_dec_return(atomic64_t *v) +{ + return arch_atomic64_sub_return(1, v); +} +#define arch_atomic64_dec_return arch_atomic64_dec_return +#endif + +#ifndef arch_atomic64_dec_return_acquire +static __always_inline s64 +arch_atomic64_dec_return_acquire(atomic64_t *v) +{ + return arch_atomic64_sub_return_acquire(1, v); +} +#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire +#endif + +#ifndef arch_atomic64_dec_return_release +static __always_inline s64 +arch_atomic64_dec_return_release(atomic64_t *v) +{ + return arch_atomic64_sub_return_release(1, v); +} +#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release +#endif + +#ifndef arch_atomic64_dec_return_relaxed +static __always_inline s64 +arch_atomic64_dec_return_relaxed(atomic64_t *v) +{ + return arch_atomic64_sub_return_relaxed(1, v); +} +#define arch_atomic64_dec_return_relaxed arch_atomic64_dec_return_relaxed +#endif + +#else /* arch_atomic64_dec_return_relaxed */ + +#ifndef arch_atomic64_dec_return_acquire +static __always_inline s64 +arch_atomic64_dec_return_acquire(atomic64_t *v) +{ + s64 ret = arch_atomic64_dec_return_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_dec_return_acquire arch_atomic64_dec_return_acquire +#endif + +#ifndef arch_atomic64_dec_return_release +static __always_inline s64 +arch_atomic64_dec_return_release(atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_dec_return_relaxed(v); +} +#define arch_atomic64_dec_return_release arch_atomic64_dec_return_release +#endif + +#ifndef arch_atomic64_dec_return +static __always_inline s64 +arch_atomic64_dec_return(atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_dec_return_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_dec_return arch_atomic64_dec_return +#endif + +#endif /* arch_atomic64_dec_return_relaxed */ + +#ifndef arch_atomic64_fetch_dec_relaxed +#ifdef arch_atomic64_fetch_dec +#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec +#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec +#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec +#endif /* arch_atomic64_fetch_dec */ + +#ifndef arch_atomic64_fetch_dec +static __always_inline s64 +arch_atomic64_fetch_dec(atomic64_t *v) +{ + return arch_atomic64_fetch_sub(1, v); +} +#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec +#endif + +#ifndef arch_atomic64_fetch_dec_acquire +static __always_inline s64 +arch_atomic64_fetch_dec_acquire(atomic64_t *v) +{ + return arch_atomic64_fetch_sub_acquire(1, v); +} +#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire +#endif + +#ifndef arch_atomic64_fetch_dec_release +static __always_inline s64 +arch_atomic64_fetch_dec_release(atomic64_t *v) +{ + return arch_atomic64_fetch_sub_release(1, v); +} +#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release +#endif + +#ifndef arch_atomic64_fetch_dec_relaxed +static __always_inline s64 +arch_atomic64_fetch_dec_relaxed(atomic64_t *v) +{ + return arch_atomic64_fetch_sub_relaxed(1, v); +} +#define arch_atomic64_fetch_dec_relaxed arch_atomic64_fetch_dec_relaxed +#endif + +#else /* arch_atomic64_fetch_dec_relaxed */ + +#ifndef arch_atomic64_fetch_dec_acquire +static __always_inline s64 +arch_atomic64_fetch_dec_acquire(atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_dec_relaxed(v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_dec_acquire arch_atomic64_fetch_dec_acquire +#endif + +#ifndef arch_atomic64_fetch_dec_release +static __always_inline s64 +arch_atomic64_fetch_dec_release(atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_dec_relaxed(v); +} +#define arch_atomic64_fetch_dec_release arch_atomic64_fetch_dec_release +#endif + +#ifndef arch_atomic64_fetch_dec +static __always_inline s64 +arch_atomic64_fetch_dec(atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_dec_relaxed(v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_dec arch_atomic64_fetch_dec +#endif + +#endif /* arch_atomic64_fetch_dec_relaxed */ + +#ifndef arch_atomic64_fetch_and_relaxed +#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and +#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and +#define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and +#else /* arch_atomic64_fetch_and_relaxed */ + +#ifndef arch_atomic64_fetch_and_acquire +static __always_inline s64 +arch_atomic64_fetch_and_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_and_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_and_acquire arch_atomic64_fetch_and_acquire +#endif + +#ifndef arch_atomic64_fetch_and_release +static __always_inline s64 +arch_atomic64_fetch_and_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_and_relaxed(i, v); +} +#define arch_atomic64_fetch_and_release arch_atomic64_fetch_and_release +#endif + +#ifndef arch_atomic64_fetch_and +static __always_inline s64 +arch_atomic64_fetch_and(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_and_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_and arch_atomic64_fetch_and +#endif + +#endif /* arch_atomic64_fetch_and_relaxed */ + +#ifndef arch_atomic64_andnot +static __always_inline void +arch_atomic64_andnot(s64 i, atomic64_t *v) +{ + arch_atomic64_and(~i, v); +} +#define arch_atomic64_andnot arch_atomic64_andnot +#endif + +#ifndef arch_atomic64_fetch_andnot_relaxed +#ifdef arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot +#endif /* arch_atomic64_fetch_andnot */ + +#ifndef arch_atomic64_fetch_andnot +static __always_inline s64 +arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and(~i, v); +} +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#endif + +#ifndef arch_atomic64_fetch_andnot_acquire +static __always_inline s64 +arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_acquire(~i, v); +} +#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire +#endif + +#ifndef arch_atomic64_fetch_andnot_release +static __always_inline s64 +arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_release(~i, v); +} +#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release +#endif + +#ifndef arch_atomic64_fetch_andnot_relaxed +static __always_inline s64 +arch_atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) +{ + return arch_atomic64_fetch_and_relaxed(~i, v); +} +#define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed +#endif + +#else /* arch_atomic64_fetch_andnot_relaxed */ + +#ifndef arch_atomic64_fetch_andnot_acquire +static __always_inline s64 +arch_atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_andnot_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_andnot_acquire arch_atomic64_fetch_andnot_acquire +#endif + +#ifndef arch_atomic64_fetch_andnot_release +static __always_inline s64 +arch_atomic64_fetch_andnot_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_andnot_relaxed(i, v); +} +#define arch_atomic64_fetch_andnot_release arch_atomic64_fetch_andnot_release +#endif + +#ifndef arch_atomic64_fetch_andnot +static __always_inline s64 +arch_atomic64_fetch_andnot(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_andnot_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_andnot arch_atomic64_fetch_andnot +#endif + +#endif /* arch_atomic64_fetch_andnot_relaxed */ + +#ifndef arch_atomic64_fetch_or_relaxed +#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or +#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or +#define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or +#else /* arch_atomic64_fetch_or_relaxed */ + +#ifndef arch_atomic64_fetch_or_acquire +static __always_inline s64 +arch_atomic64_fetch_or_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_or_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_or_acquire arch_atomic64_fetch_or_acquire +#endif + +#ifndef arch_atomic64_fetch_or_release +static __always_inline s64 +arch_atomic64_fetch_or_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_or_relaxed(i, v); +} +#define arch_atomic64_fetch_or_release arch_atomic64_fetch_or_release +#endif + +#ifndef arch_atomic64_fetch_or +static __always_inline s64 +arch_atomic64_fetch_or(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_or_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_or arch_atomic64_fetch_or +#endif + +#endif /* arch_atomic64_fetch_or_relaxed */ + +#ifndef arch_atomic64_fetch_xor_relaxed +#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor +#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor +#define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor +#else /* arch_atomic64_fetch_xor_relaxed */ + +#ifndef arch_atomic64_fetch_xor_acquire +static __always_inline s64 +arch_atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) +{ + s64 ret = arch_atomic64_fetch_xor_relaxed(i, v); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_fetch_xor_acquire arch_atomic64_fetch_xor_acquire +#endif + +#ifndef arch_atomic64_fetch_xor_release +static __always_inline s64 +arch_atomic64_fetch_xor_release(s64 i, atomic64_t *v) +{ + __atomic_release_fence(); + return arch_atomic64_fetch_xor_relaxed(i, v); +} +#define arch_atomic64_fetch_xor_release arch_atomic64_fetch_xor_release +#endif + +#ifndef arch_atomic64_fetch_xor +static __always_inline s64 +arch_atomic64_fetch_xor(s64 i, atomic64_t *v) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_fetch_xor_relaxed(i, v); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_fetch_xor arch_atomic64_fetch_xor +#endif + +#endif /* arch_atomic64_fetch_xor_relaxed */ + +#ifndef arch_atomic64_xchg_relaxed +#define arch_atomic64_xchg_acquire arch_atomic64_xchg +#define arch_atomic64_xchg_release arch_atomic64_xchg +#define arch_atomic64_xchg_relaxed arch_atomic64_xchg +#else /* arch_atomic64_xchg_relaxed */ + +#ifndef arch_atomic64_xchg_acquire +static __always_inline s64 +arch_atomic64_xchg_acquire(atomic64_t *v, s64 i) +{ + s64 ret = arch_atomic64_xchg_relaxed(v, i); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_xchg_acquire arch_atomic64_xchg_acquire +#endif + +#ifndef arch_atomic64_xchg_release +static __always_inline s64 +arch_atomic64_xchg_release(atomic64_t *v, s64 i) +{ + __atomic_release_fence(); + return arch_atomic64_xchg_relaxed(v, i); +} +#define arch_atomic64_xchg_release arch_atomic64_xchg_release +#endif + +#ifndef arch_atomic64_xchg +static __always_inline s64 +arch_atomic64_xchg(atomic64_t *v, s64 i) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_xchg_relaxed(v, i); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_xchg arch_atomic64_xchg +#endif + +#endif /* arch_atomic64_xchg_relaxed */ + +#ifndef arch_atomic64_cmpxchg_relaxed +#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg +#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg +#define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg +#else /* arch_atomic64_cmpxchg_relaxed */ + +#ifndef arch_atomic64_cmpxchg_acquire +static __always_inline s64 +arch_atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) +{ + s64 ret = arch_atomic64_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_cmpxchg_acquire arch_atomic64_cmpxchg_acquire +#endif + +#ifndef arch_atomic64_cmpxchg_release +static __always_inline s64 +arch_atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) +{ + __atomic_release_fence(); + return arch_atomic64_cmpxchg_relaxed(v, old, new); +} +#define arch_atomic64_cmpxchg_release arch_atomic64_cmpxchg_release +#endif + +#ifndef arch_atomic64_cmpxchg +static __always_inline s64 +arch_atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) +{ + s64 ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_cmpxchg_relaxed(v, old, new); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_cmpxchg arch_atomic64_cmpxchg +#endif + +#endif /* arch_atomic64_cmpxchg_relaxed */ + +#ifndef arch_atomic64_try_cmpxchg_relaxed +#ifdef arch_atomic64_try_cmpxchg +#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg +#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg +#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg +#endif /* arch_atomic64_try_cmpxchg */ + +#ifndef arch_atomic64_try_cmpxchg +static __always_inline bool +arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + s64 r, o = *old; + r = arch_atomic64_cmpxchg(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg +#endif + +#ifndef arch_atomic64_try_cmpxchg_acquire +static __always_inline bool +arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + s64 r, o = *old; + r = arch_atomic64_cmpxchg_acquire(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire +#endif + +#ifndef arch_atomic64_try_cmpxchg_release +static __always_inline bool +arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + s64 r, o = *old; + r = arch_atomic64_cmpxchg_release(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release +#endif + +#ifndef arch_atomic64_try_cmpxchg_relaxed +static __always_inline bool +arch_atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) +{ + s64 r, o = *old; + r = arch_atomic64_cmpxchg_relaxed(v, o, new); + if (unlikely(r != o)) + *old = r; + return likely(r == o); +} +#define arch_atomic64_try_cmpxchg_relaxed arch_atomic64_try_cmpxchg_relaxed +#endif + +#else /* arch_atomic64_try_cmpxchg_relaxed */ + +#ifndef arch_atomic64_try_cmpxchg_acquire +static __always_inline bool +arch_atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) +{ + bool ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); + __atomic_acquire_fence(); + return ret; +} +#define arch_atomic64_try_cmpxchg_acquire arch_atomic64_try_cmpxchg_acquire +#endif + +#ifndef arch_atomic64_try_cmpxchg_release +static __always_inline bool +arch_atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) +{ + __atomic_release_fence(); + return arch_atomic64_try_cmpxchg_relaxed(v, old, new); +} +#define arch_atomic64_try_cmpxchg_release arch_atomic64_try_cmpxchg_release +#endif + +#ifndef arch_atomic64_try_cmpxchg +static __always_inline bool +arch_atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) +{ + bool ret; + __atomic_pre_full_fence(); + ret = arch_atomic64_try_cmpxchg_relaxed(v, old, new); + __atomic_post_full_fence(); + return ret; +} +#define arch_atomic64_try_cmpxchg arch_atomic64_try_cmpxchg +#endif + +#endif /* arch_atomic64_try_cmpxchg_relaxed */ + +#ifndef arch_atomic64_sub_and_test +/** + * arch_atomic64_sub_and_test - subtract value from variable and test result + * @i: integer value to subtract + * @v: pointer of type atomic64_t + * + * Atomically subtracts @i from @v and returns + * true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_atomic64_sub_and_test(s64 i, atomic64_t *v) +{ + return arch_atomic64_sub_return(i, v) == 0; +} +#define arch_atomic64_sub_and_test arch_atomic64_sub_and_test +#endif + +#ifndef arch_atomic64_dec_and_test +/** + * arch_atomic64_dec_and_test - decrement and test + * @v: pointer of type atomic64_t + * + * Atomically decrements @v by 1 and + * returns true if the result is 0, or false for all other + * cases. + */ +static __always_inline bool +arch_atomic64_dec_and_test(atomic64_t *v) +{ + return arch_atomic64_dec_return(v) == 0; +} +#define arch_atomic64_dec_and_test arch_atomic64_dec_and_test +#endif + +#ifndef arch_atomic64_inc_and_test +/** + * arch_atomic64_inc_and_test - increment and test + * @v: pointer of type atomic64_t + * + * Atomically increments @v by 1 + * and returns true if the result is zero, or false for all + * other cases. + */ +static __always_inline bool +arch_atomic64_inc_and_test(atomic64_t *v) +{ + return arch_atomic64_inc_return(v) == 0; +} +#define arch_atomic64_inc_and_test arch_atomic64_inc_and_test +#endif + +#ifndef arch_atomic64_add_negative +/** + * arch_atomic64_add_negative - add and test if negative + * @i: integer value to add + * @v: pointer of type atomic64_t + * + * Atomically adds @i to @v and returns true + * if the result is negative, or false when + * result is greater than or equal to zero. + */ +static __always_inline bool +arch_atomic64_add_negative(s64 i, atomic64_t *v) +{ + return arch_atomic64_add_return(i, v) < 0; +} +#define arch_atomic64_add_negative arch_atomic64_add_negative +#endif + +#ifndef arch_atomic64_fetch_add_unless +/** + * arch_atomic64_fetch_add_unless - add unless the number is already a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, so long as @v was not already @u. + * Returns original value of @v + */ +static __always_inline s64 +arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) +{ + s64 c = arch_atomic64_read(v); + + do { + if (unlikely(c == u)) + break; + } while (!arch_atomic64_try_cmpxchg(v, &c, c + a)); + + return c; +} +#define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless +#endif + +#ifndef arch_atomic64_add_unless +/** + * arch_atomic64_add_unless - add unless the number is already a given value + * @v: pointer of type atomic64_t + * @a: the amount to add to v... + * @u: ...unless v is equal to u. + * + * Atomically adds @a to @v, if @v was not already @u. + * Returns true if the addition was done. + */ +static __always_inline bool +arch_atomic64_add_unless(atomic64_t *v, s64 a, s64 u) +{ + return arch_atomic64_fetch_add_unless(v, a, u) != u; +} +#define arch_atomic64_add_unless arch_atomic64_add_unless +#endif + +#ifndef arch_atomic64_inc_not_zero +/** + * arch_atomic64_inc_not_zero - increment unless the number is zero + * @v: pointer of type atomic64_t + * + * Atomically increments @v by 1, if @v is non-zero. + * Returns true if the increment was done. + */ +static __always_inline bool +arch_atomic64_inc_not_zero(atomic64_t *v) +{ + return arch_atomic64_add_unless(v, 1, 0); +} +#define arch_atomic64_inc_not_zero arch_atomic64_inc_not_zero +#endif + +#ifndef arch_atomic64_inc_unless_negative +static __always_inline bool +arch_atomic64_inc_unless_negative(atomic64_t *v) +{ + s64 c = arch_atomic64_read(v); + + do { + if (unlikely(c < 0)) + return false; + } while (!arch_atomic64_try_cmpxchg(v, &c, c + 1)); + + return true; +} +#define arch_atomic64_inc_unless_negative arch_atomic64_inc_unless_negative +#endif + +#ifndef arch_atomic64_dec_unless_positive +static __always_inline bool +arch_atomic64_dec_unless_positive(atomic64_t *v) +{ + s64 c = arch_atomic64_read(v); + + do { + if (unlikely(c > 0)) + return false; + } while (!arch_atomic64_try_cmpxchg(v, &c, c - 1)); + + return true; +} +#define arch_atomic64_dec_unless_positive arch_atomic64_dec_unless_positive +#endif + +#ifndef arch_atomic64_dec_if_positive +static __always_inline s64 +arch_atomic64_dec_if_positive(atomic64_t *v) +{ + s64 dec, c = arch_atomic64_read(v); + + do { + dec = c - 1; + if (unlikely(dec < 0)) + break; + } while (!arch_atomic64_try_cmpxchg(v, &c, dec)); + + return dec; +} +#define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive +#endif + +#endif /* _LINUX_ATOMIC_FALLBACK_H */ +// 90cd26cfd69d2250303d654955a0cc12620fb91b diff --git a/include/linux/atomic-fallback.h b/include/linux/atomic-fallback.h index a7d240e465c0..2c4927bf7b8d 100644 --- a/include/linux/atomic-fallback.h +++ b/include/linux/atomic-fallback.h @@ -6,6 +6,8 @@ #ifndef _LINUX_ATOMIC_FALLBACK_H #define _LINUX_ATOMIC_FALLBACK_H +#include <linux/compiler.h> + #ifndef xchg_relaxed #define xchg_relaxed xchg #define xchg_acquire xchg @@ -76,7 +78,7 @@ #endif /* cmpxchg64_relaxed */ #ifndef atomic_read_acquire -static inline int +static __always_inline int atomic_read_acquire(const atomic_t *v) { return smp_load_acquire(&(v)->counter); @@ -85,7 +87,7 @@ atomic_read_acquire(const atomic_t *v) #endif #ifndef atomic_set_release -static inline void +static __always_inline void atomic_set_release(atomic_t *v, int i) { smp_store_release(&(v)->counter, i); @@ -100,7 +102,7 @@ atomic_set_release(atomic_t *v, int i) #else /* atomic_add_return_relaxed */ #ifndef atomic_add_return_acquire -static inline int +static __always_inline int atomic_add_return_acquire(int i, atomic_t *v) { int ret = atomic_add_return_relaxed(i, v); @@ -111,7 +113,7 @@ atomic_add_return_acquire(int i, atomic_t *v) #endif #ifndef atomic_add_return_release -static inline int +static __always_inline int atomic_add_return_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -121,7 +123,7 @@ atomic_add_return_release(int i, atomic_t *v) #endif #ifndef atomic_add_return -static inline int +static __always_inline int atomic_add_return(int i, atomic_t *v) { int ret; @@ -142,7 +144,7 @@ atomic_add_return(int i, atomic_t *v) #else /* atomic_fetch_add_relaxed */ #ifndef atomic_fetch_add_acquire -static inline int +static __always_inline int atomic_fetch_add_acquire(int i, atomic_t *v) { int ret = atomic_fetch_add_relaxed(i, v); @@ -153,7 +155,7 @@ atomic_fetch_add_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_add_release -static inline int +static __always_inline int atomic_fetch_add_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -163,7 +165,7 @@ atomic_fetch_add_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_add -static inline int +static __always_inline int atomic_fetch_add(int i, atomic_t *v) { int ret; @@ -184,7 +186,7 @@ atomic_fetch_add(int i, atomic_t *v) #else /* atomic_sub_return_relaxed */ #ifndef atomic_sub_return_acquire -static inline int +static __always_inline int atomic_sub_return_acquire(int i, atomic_t *v) { int ret = atomic_sub_return_relaxed(i, v); @@ -195,7 +197,7 @@ atomic_sub_return_acquire(int i, atomic_t *v) #endif #ifndef atomic_sub_return_release -static inline int +static __always_inline int atomic_sub_return_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -205,7 +207,7 @@ atomic_sub_return_release(int i, atomic_t *v) #endif #ifndef atomic_sub_return -static inline int +static __always_inline int atomic_sub_return(int i, atomic_t *v) { int ret; @@ -226,7 +228,7 @@ atomic_sub_return(int i, atomic_t *v) #else /* atomic_fetch_sub_relaxed */ #ifndef atomic_fetch_sub_acquire -static inline int +static __always_inline int atomic_fetch_sub_acquire(int i, atomic_t *v) { int ret = atomic_fetch_sub_relaxed(i, v); @@ -237,7 +239,7 @@ atomic_fetch_sub_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_sub_release -static inline int +static __always_inline int atomic_fetch_sub_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -247,7 +249,7 @@ atomic_fetch_sub_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_sub -static inline int +static __always_inline int atomic_fetch_sub(int i, atomic_t *v) { int ret; @@ -262,7 +264,7 @@ atomic_fetch_sub(int i, atomic_t *v) #endif /* atomic_fetch_sub_relaxed */ #ifndef atomic_inc -static inline void +static __always_inline void atomic_inc(atomic_t *v) { atomic_add(1, v); @@ -278,7 +280,7 @@ atomic_inc(atomic_t *v) #endif /* atomic_inc_return */ #ifndef atomic_inc_return -static inline int +static __always_inline int atomic_inc_return(atomic_t *v) { return atomic_add_return(1, v); @@ -287,7 +289,7 @@ atomic_inc_return(atomic_t *v) #endif #ifndef atomic_inc_return_acquire -static inline int +static __always_inline int atomic_inc_return_acquire(atomic_t *v) { return atomic_add_return_acquire(1, v); @@ -296,7 +298,7 @@ atomic_inc_return_acquire(atomic_t *v) #endif #ifndef atomic_inc_return_release -static inline int +static __always_inline int atomic_inc_return_release(atomic_t *v) { return atomic_add_return_release(1, v); @@ -305,7 +307,7 @@ atomic_inc_return_release(atomic_t *v) #endif #ifndef atomic_inc_return_relaxed -static inline int +static __always_inline int atomic_inc_return_relaxed(atomic_t *v) { return atomic_add_return_relaxed(1, v); @@ -316,7 +318,7 @@ atomic_inc_return_relaxed(atomic_t *v) #else /* atomic_inc_return_relaxed */ #ifndef atomic_inc_return_acquire -static inline int +static __always_inline int atomic_inc_return_acquire(atomic_t *v) { int ret = atomic_inc_return_relaxed(v); @@ -327,7 +329,7 @@ atomic_inc_return_acquire(atomic_t *v) #endif #ifndef atomic_inc_return_release -static inline int +static __always_inline int atomic_inc_return_release(atomic_t *v) { __atomic_release_fence(); @@ -337,7 +339,7 @@ atomic_inc_return_release(atomic_t *v) #endif #ifndef atomic_inc_return -static inline int +static __always_inline int atomic_inc_return(atomic_t *v) { int ret; @@ -359,7 +361,7 @@ atomic_inc_return(atomic_t *v) #endif /* atomic_fetch_inc */ #ifndef atomic_fetch_inc -static inline int +static __always_inline int atomic_fetch_inc(atomic_t *v) { return atomic_fetch_add(1, v); @@ -368,7 +370,7 @@ atomic_fetch_inc(atomic_t *v) #endif #ifndef atomic_fetch_inc_acquire -static inline int +static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { return atomic_fetch_add_acquire(1, v); @@ -377,7 +379,7 @@ atomic_fetch_inc_acquire(atomic_t *v) #endif #ifndef atomic_fetch_inc_release -static inline int +static __always_inline int atomic_fetch_inc_release(atomic_t *v) { return atomic_fetch_add_release(1, v); @@ -386,7 +388,7 @@ atomic_fetch_inc_release(atomic_t *v) #endif #ifndef atomic_fetch_inc_relaxed -static inline int +static __always_inline int atomic_fetch_inc_relaxed(atomic_t *v) { return atomic_fetch_add_relaxed(1, v); @@ -397,7 +399,7 @@ atomic_fetch_inc_relaxed(atomic_t *v) #else /* atomic_fetch_inc_relaxed */ #ifndef atomic_fetch_inc_acquire -static inline int +static __always_inline int atomic_fetch_inc_acquire(atomic_t *v) { int ret = atomic_fetch_inc_relaxed(v); @@ -408,7 +410,7 @@ atomic_fetch_inc_acquire(atomic_t *v) #endif #ifndef atomic_fetch_inc_release -static inline int +static __always_inline int atomic_fetch_inc_release(atomic_t *v) { __atomic_release_fence(); @@ -418,7 +420,7 @@ atomic_fetch_inc_release(atomic_t *v) #endif #ifndef atomic_fetch_inc -static inline int +static __always_inline int atomic_fetch_inc(atomic_t *v) { int ret; @@ -433,7 +435,7 @@ atomic_fetch_inc(atomic_t *v) #endif /* atomic_fetch_inc_relaxed */ #ifndef atomic_dec -static inline void +static __always_inline void atomic_dec(atomic_t *v) { atomic_sub(1, v); @@ -449,7 +451,7 @@ atomic_dec(atomic_t *v) #endif /* atomic_dec_return */ #ifndef atomic_dec_return -static inline int +static __always_inline int atomic_dec_return(atomic_t *v) { return atomic_sub_return(1, v); @@ -458,7 +460,7 @@ atomic_dec_return(atomic_t *v) #endif #ifndef atomic_dec_return_acquire -static inline int +static __always_inline int atomic_dec_return_acquire(atomic_t *v) { return atomic_sub_return_acquire(1, v); @@ -467,7 +469,7 @@ atomic_dec_return_acquire(atomic_t *v) #endif #ifndef atomic_dec_return_release -static inline int +static __always_inline int atomic_dec_return_release(atomic_t *v) { return atomic_sub_return_release(1, v); @@ -476,7 +478,7 @@ atomic_dec_return_release(atomic_t *v) #endif #ifndef atomic_dec_return_relaxed -static inline int +static __always_inline int atomic_dec_return_relaxed(atomic_t *v) { return atomic_sub_return_relaxed(1, v); @@ -487,7 +489,7 @@ atomic_dec_return_relaxed(atomic_t *v) #else /* atomic_dec_return_relaxed */ #ifndef atomic_dec_return_acquire -static inline int +static __always_inline int atomic_dec_return_acquire(atomic_t *v) { int ret = atomic_dec_return_relaxed(v); @@ -498,7 +500,7 @@ atomic_dec_return_acquire(atomic_t *v) #endif #ifndef atomic_dec_return_release -static inline int +static __always_inline int atomic_dec_return_release(atomic_t *v) { __atomic_release_fence(); @@ -508,7 +510,7 @@ atomic_dec_return_release(atomic_t *v) #endif #ifndef atomic_dec_return -static inline int +static __always_inline int atomic_dec_return(atomic_t *v) { int ret; @@ -530,7 +532,7 @@ atomic_dec_return(atomic_t *v) #endif /* atomic_fetch_dec */ #ifndef atomic_fetch_dec -static inline int +static __always_inline int atomic_fetch_dec(atomic_t *v) { return atomic_fetch_sub(1, v); @@ -539,7 +541,7 @@ atomic_fetch_dec(atomic_t *v) #endif #ifndef atomic_fetch_dec_acquire -static inline int +static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { return atomic_fetch_sub_acquire(1, v); @@ -548,7 +550,7 @@ atomic_fetch_dec_acquire(atomic_t *v) #endif #ifndef atomic_fetch_dec_release -static inline int +static __always_inline int atomic_fetch_dec_release(atomic_t *v) { return atomic_fetch_sub_release(1, v); @@ -557,7 +559,7 @@ atomic_fetch_dec_release(atomic_t *v) #endif #ifndef atomic_fetch_dec_relaxed -static inline int +static __always_inline int atomic_fetch_dec_relaxed(atomic_t *v) { return atomic_fetch_sub_relaxed(1, v); @@ -568,7 +570,7 @@ atomic_fetch_dec_relaxed(atomic_t *v) #else /* atomic_fetch_dec_relaxed */ #ifndef atomic_fetch_dec_acquire -static inline int +static __always_inline int atomic_fetch_dec_acquire(atomic_t *v) { int ret = atomic_fetch_dec_relaxed(v); @@ -579,7 +581,7 @@ atomic_fetch_dec_acquire(atomic_t *v) #endif #ifndef atomic_fetch_dec_release -static inline int +static __always_inline int atomic_fetch_dec_release(atomic_t *v) { __atomic_release_fence(); @@ -589,7 +591,7 @@ atomic_fetch_dec_release(atomic_t *v) #endif #ifndef atomic_fetch_dec -static inline int +static __always_inline int atomic_fetch_dec(atomic_t *v) { int ret; @@ -610,7 +612,7 @@ atomic_fetch_dec(atomic_t *v) #else /* atomic_fetch_and_relaxed */ #ifndef atomic_fetch_and_acquire -static inline int +static __always_inline int atomic_fetch_and_acquire(int i, atomic_t *v) { int ret = atomic_fetch_and_relaxed(i, v); @@ -621,7 +623,7 @@ atomic_fetch_and_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_and_release -static inline int +static __always_inline int atomic_fetch_and_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -631,7 +633,7 @@ atomic_fetch_and_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_and -static inline int +static __always_inline int atomic_fetch_and(int i, atomic_t *v) { int ret; @@ -646,7 +648,7 @@ atomic_fetch_and(int i, atomic_t *v) #endif /* atomic_fetch_and_relaxed */ #ifndef atomic_andnot -static inline void +static __always_inline void atomic_andnot(int i, atomic_t *v) { atomic_and(~i, v); @@ -662,7 +664,7 @@ atomic_andnot(int i, atomic_t *v) #endif /* atomic_fetch_andnot */ #ifndef atomic_fetch_andnot -static inline int +static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { return atomic_fetch_and(~i, v); @@ -671,7 +673,7 @@ atomic_fetch_andnot(int i, atomic_t *v) #endif #ifndef atomic_fetch_andnot_acquire -static inline int +static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { return atomic_fetch_and_acquire(~i, v); @@ -680,7 +682,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_andnot_release -static inline int +static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { return atomic_fetch_and_release(~i, v); @@ -689,7 +691,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_andnot_relaxed -static inline int +static __always_inline int atomic_fetch_andnot_relaxed(int i, atomic_t *v) { return atomic_fetch_and_relaxed(~i, v); @@ -700,7 +702,7 @@ atomic_fetch_andnot_relaxed(int i, atomic_t *v) #else /* atomic_fetch_andnot_relaxed */ #ifndef atomic_fetch_andnot_acquire -static inline int +static __always_inline int atomic_fetch_andnot_acquire(int i, atomic_t *v) { int ret = atomic_fetch_andnot_relaxed(i, v); @@ -711,7 +713,7 @@ atomic_fetch_andnot_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_andnot_release -static inline int +static __always_inline int atomic_fetch_andnot_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -721,7 +723,7 @@ atomic_fetch_andnot_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_andnot -static inline int +static __always_inline int atomic_fetch_andnot(int i, atomic_t *v) { int ret; @@ -742,7 +744,7 @@ atomic_fetch_andnot(int i, atomic_t *v) #else /* atomic_fetch_or_relaxed */ #ifndef atomic_fetch_or_acquire -static inline int +static __always_inline int atomic_fetch_or_acquire(int i, atomic_t *v) { int ret = atomic_fetch_or_relaxed(i, v); @@ -753,7 +755,7 @@ atomic_fetch_or_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_or_release -static inline int +static __always_inline int atomic_fetch_or_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -763,7 +765,7 @@ atomic_fetch_or_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_or -static inline int +static __always_inline int atomic_fetch_or(int i, atomic_t *v) { int ret; @@ -784,7 +786,7 @@ atomic_fetch_or(int i, atomic_t *v) #else /* atomic_fetch_xor_relaxed */ #ifndef atomic_fetch_xor_acquire -static inline int +static __always_inline int atomic_fetch_xor_acquire(int i, atomic_t *v) { int ret = atomic_fetch_xor_relaxed(i, v); @@ -795,7 +797,7 @@ atomic_fetch_xor_acquire(int i, atomic_t *v) #endif #ifndef atomic_fetch_xor_release -static inline int +static __always_inline int atomic_fetch_xor_release(int i, atomic_t *v) { __atomic_release_fence(); @@ -805,7 +807,7 @@ atomic_fetch_xor_release(int i, atomic_t *v) #endif #ifndef atomic_fetch_xor -static inline int +static __always_inline int atomic_fetch_xor(int i, atomic_t *v) { int ret; @@ -826,7 +828,7 @@ atomic_fetch_xor(int i, atomic_t *v) #else /* atomic_xchg_relaxed */ #ifndef atomic_xchg_acquire -static inline int +static __always_inline int atomic_xchg_acquire(atomic_t *v, int i) { int ret = atomic_xchg_relaxed(v, i); @@ -837,7 +839,7 @@ atomic_xchg_acquire(atomic_t *v, int i) #endif #ifndef atomic_xchg_release -static inline int +static __always_inline int atomic_xchg_release(atomic_t *v, int i) { __atomic_release_fence(); @@ -847,7 +849,7 @@ atomic_xchg_release(atomic_t *v, int i) #endif #ifndef atomic_xchg -static inline int +static __always_inline int atomic_xchg(atomic_t *v, int i) { int ret; @@ -868,7 +870,7 @@ atomic_xchg(atomic_t *v, int i) #else /* atomic_cmpxchg_relaxed */ #ifndef atomic_cmpxchg_acquire -static inline int +static __always_inline int atomic_cmpxchg_acquire(atomic_t *v, int old, int new) { int ret = atomic_cmpxchg_relaxed(v, old, new); @@ -879,7 +881,7 @@ atomic_cmpxchg_acquire(atomic_t *v, int old, int new) #endif #ifndef atomic_cmpxchg_release -static inline int +static __always_inline int atomic_cmpxchg_release(atomic_t *v, int old, int new) { __atomic_release_fence(); @@ -889,7 +891,7 @@ atomic_cmpxchg_release(atomic_t *v, int old, int new) #endif #ifndef atomic_cmpxchg -static inline int +static __always_inline int atomic_cmpxchg(atomic_t *v, int old, int new) { int ret; @@ -911,7 +913,7 @@ atomic_cmpxchg(atomic_t *v, int old, int new) #endif /* atomic_try_cmpxchg */ #ifndef atomic_try_cmpxchg -static inline bool +static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { int r, o = *old; @@ -924,7 +926,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) #endif #ifndef atomic_try_cmpxchg_acquire -static inline bool +static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { int r, o = *old; @@ -937,7 +939,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) #endif #ifndef atomic_try_cmpxchg_release -static inline bool +static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { int r, o = *old; @@ -950,7 +952,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) #endif #ifndef atomic_try_cmpxchg_relaxed -static inline bool +static __always_inline bool atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) { int r, o = *old; @@ -965,7 +967,7 @@ atomic_try_cmpxchg_relaxed(atomic_t *v, int *old, int new) #else /* atomic_try_cmpxchg_relaxed */ #ifndef atomic_try_cmpxchg_acquire -static inline bool +static __always_inline bool atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) { bool ret = atomic_try_cmpxchg_relaxed(v, old, new); @@ -976,7 +978,7 @@ atomic_try_cmpxchg_acquire(atomic_t *v, int *old, int new) #endif #ifndef atomic_try_cmpxchg_release -static inline bool +static __always_inline bool atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) { __atomic_release_fence(); @@ -986,7 +988,7 @@ atomic_try_cmpxchg_release(atomic_t *v, int *old, int new) #endif #ifndef atomic_try_cmpxchg -static inline bool +static __always_inline bool atomic_try_cmpxchg(atomic_t *v, int *old, int new) { bool ret; @@ -1010,7 +1012,7 @@ atomic_try_cmpxchg(atomic_t *v, int *old, int new) * true if the result is zero, or false for all * other cases. */ -static inline bool +static __always_inline bool atomic_sub_and_test(int i, atomic_t *v) { return atomic_sub_return(i, v) == 0; @@ -1027,7 +1029,7 @@ atomic_sub_and_test(int i, atomic_t *v) * returns true if the result is 0, or false for all other * cases. */ -static inline bool +static __always_inline bool atomic_dec_and_test(atomic_t *v) { return atomic_dec_return(v) == 0; @@ -1044,7 +1046,7 @@ atomic_dec_and_test(atomic_t *v) * and returns true if the result is zero, or false for all * other cases. */ -static inline bool +static __always_inline bool atomic_inc_and_test(atomic_t *v) { return atomic_inc_return(v) == 0; @@ -1062,7 +1064,7 @@ atomic_inc_and_test(atomic_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline bool +static __always_inline bool atomic_add_negative(int i, atomic_t *v) { return atomic_add_return(i, v) < 0; @@ -1080,7 +1082,7 @@ atomic_add_negative(int i, atomic_t *v) * Atomically adds @a to @v, so long as @v was not already @u. * Returns original value of @v */ -static inline int +static __always_inline int atomic_fetch_add_unless(atomic_t *v, int a, int u) { int c = atomic_read(v); @@ -1105,7 +1107,7 @@ atomic_fetch_add_unless(atomic_t *v, int a, int u) * Atomically adds @a to @v, if @v was not already @u. * Returns true if the addition was done. */ -static inline bool +static __always_inline bool atomic_add_unless(atomic_t *v, int a, int u) { return atomic_fetch_add_unless(v, a, u) != u; @@ -1121,7 +1123,7 @@ atomic_add_unless(atomic_t *v, int a, int u) * Atomically increments @v by 1, if @v is non-zero. * Returns true if the increment was done. */ -static inline bool +static __always_inline bool atomic_inc_not_zero(atomic_t *v) { return atomic_add_unless(v, 1, 0); @@ -1130,7 +1132,7 @@ atomic_inc_not_zero(atomic_t *v) #endif #ifndef atomic_inc_unless_negative -static inline bool +static __always_inline bool atomic_inc_unless_negative(atomic_t *v) { int c = atomic_read(v); @@ -1146,7 +1148,7 @@ atomic_inc_unless_negative(atomic_t *v) #endif #ifndef atomic_dec_unless_positive -static inline bool +static __always_inline bool atomic_dec_unless_positive(atomic_t *v) { int c = atomic_read(v); @@ -1162,7 +1164,7 @@ atomic_dec_unless_positive(atomic_t *v) #endif #ifndef atomic_dec_if_positive -static inline int +static __always_inline int atomic_dec_if_positive(atomic_t *v) { int dec, c = atomic_read(v); @@ -1178,15 +1180,12 @@ atomic_dec_if_positive(atomic_t *v) #define atomic_dec_if_positive atomic_dec_if_positive #endif -#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) -#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) - #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> #endif #ifndef atomic64_read_acquire -static inline s64 +static __always_inline s64 atomic64_read_acquire(const atomic64_t *v) { return smp_load_acquire(&(v)->counter); @@ -1195,7 +1194,7 @@ atomic64_read_acquire(const atomic64_t *v) #endif #ifndef atomic64_set_release -static inline void +static __always_inline void atomic64_set_release(atomic64_t *v, s64 i) { smp_store_release(&(v)->counter, i); @@ -1210,7 +1209,7 @@ atomic64_set_release(atomic64_t *v, s64 i) #else /* atomic64_add_return_relaxed */ #ifndef atomic64_add_return_acquire -static inline s64 +static __always_inline s64 atomic64_add_return_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_add_return_relaxed(i, v); @@ -1221,7 +1220,7 @@ atomic64_add_return_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_add_return_release -static inline s64 +static __always_inline s64 atomic64_add_return_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1231,7 +1230,7 @@ atomic64_add_return_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_add_return -static inline s64 +static __always_inline s64 atomic64_add_return(s64 i, atomic64_t *v) { s64 ret; @@ -1252,7 +1251,7 @@ atomic64_add_return(s64 i, atomic64_t *v) #else /* atomic64_fetch_add_relaxed */ #ifndef atomic64_fetch_add_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_add_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_add_relaxed(i, v); @@ -1263,7 +1262,7 @@ atomic64_fetch_add_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_add_release -static inline s64 +static __always_inline s64 atomic64_fetch_add_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1273,7 +1272,7 @@ atomic64_fetch_add_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_add -static inline s64 +static __always_inline s64 atomic64_fetch_add(s64 i, atomic64_t *v) { s64 ret; @@ -1294,7 +1293,7 @@ atomic64_fetch_add(s64 i, atomic64_t *v) #else /* atomic64_sub_return_relaxed */ #ifndef atomic64_sub_return_acquire -static inline s64 +static __always_inline s64 atomic64_sub_return_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_sub_return_relaxed(i, v); @@ -1305,7 +1304,7 @@ atomic64_sub_return_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_sub_return_release -static inline s64 +static __always_inline s64 atomic64_sub_return_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1315,7 +1314,7 @@ atomic64_sub_return_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_sub_return -static inline s64 +static __always_inline s64 atomic64_sub_return(s64 i, atomic64_t *v) { s64 ret; @@ -1336,7 +1335,7 @@ atomic64_sub_return(s64 i, atomic64_t *v) #else /* atomic64_fetch_sub_relaxed */ #ifndef atomic64_fetch_sub_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_sub_relaxed(i, v); @@ -1347,7 +1346,7 @@ atomic64_fetch_sub_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_sub_release -static inline s64 +static __always_inline s64 atomic64_fetch_sub_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1357,7 +1356,7 @@ atomic64_fetch_sub_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_sub -static inline s64 +static __always_inline s64 atomic64_fetch_sub(s64 i, atomic64_t *v) { s64 ret; @@ -1372,7 +1371,7 @@ atomic64_fetch_sub(s64 i, atomic64_t *v) #endif /* atomic64_fetch_sub_relaxed */ #ifndef atomic64_inc -static inline void +static __always_inline void atomic64_inc(atomic64_t *v) { atomic64_add(1, v); @@ -1388,7 +1387,7 @@ atomic64_inc(atomic64_t *v) #endif /* atomic64_inc_return */ #ifndef atomic64_inc_return -static inline s64 +static __always_inline s64 atomic64_inc_return(atomic64_t *v) { return atomic64_add_return(1, v); @@ -1397,7 +1396,7 @@ atomic64_inc_return(atomic64_t *v) #endif #ifndef atomic64_inc_return_acquire -static inline s64 +static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { return atomic64_add_return_acquire(1, v); @@ -1406,7 +1405,7 @@ atomic64_inc_return_acquire(atomic64_t *v) #endif #ifndef atomic64_inc_return_release -static inline s64 +static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { return atomic64_add_return_release(1, v); @@ -1415,7 +1414,7 @@ atomic64_inc_return_release(atomic64_t *v) #endif #ifndef atomic64_inc_return_relaxed -static inline s64 +static __always_inline s64 atomic64_inc_return_relaxed(atomic64_t *v) { return atomic64_add_return_relaxed(1, v); @@ -1426,7 +1425,7 @@ atomic64_inc_return_relaxed(atomic64_t *v) #else /* atomic64_inc_return_relaxed */ #ifndef atomic64_inc_return_acquire -static inline s64 +static __always_inline s64 atomic64_inc_return_acquire(atomic64_t *v) { s64 ret = atomic64_inc_return_relaxed(v); @@ -1437,7 +1436,7 @@ atomic64_inc_return_acquire(atomic64_t *v) #endif #ifndef atomic64_inc_return_release -static inline s64 +static __always_inline s64 atomic64_inc_return_release(atomic64_t *v) { __atomic_release_fence(); @@ -1447,7 +1446,7 @@ atomic64_inc_return_release(atomic64_t *v) #endif #ifndef atomic64_inc_return -static inline s64 +static __always_inline s64 atomic64_inc_return(atomic64_t *v) { s64 ret; @@ -1469,7 +1468,7 @@ atomic64_inc_return(atomic64_t *v) #endif /* atomic64_fetch_inc */ #ifndef atomic64_fetch_inc -static inline s64 +static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { return atomic64_fetch_add(1, v); @@ -1478,7 +1477,7 @@ atomic64_fetch_inc(atomic64_t *v) #endif #ifndef atomic64_fetch_inc_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { return atomic64_fetch_add_acquire(1, v); @@ -1487,7 +1486,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) #endif #ifndef atomic64_fetch_inc_release -static inline s64 +static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { return atomic64_fetch_add_release(1, v); @@ -1496,7 +1495,7 @@ atomic64_fetch_inc_release(atomic64_t *v) #endif #ifndef atomic64_fetch_inc_relaxed -static inline s64 +static __always_inline s64 atomic64_fetch_inc_relaxed(atomic64_t *v) { return atomic64_fetch_add_relaxed(1, v); @@ -1507,7 +1506,7 @@ atomic64_fetch_inc_relaxed(atomic64_t *v) #else /* atomic64_fetch_inc_relaxed */ #ifndef atomic64_fetch_inc_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_inc_acquire(atomic64_t *v) { s64 ret = atomic64_fetch_inc_relaxed(v); @@ -1518,7 +1517,7 @@ atomic64_fetch_inc_acquire(atomic64_t *v) #endif #ifndef atomic64_fetch_inc_release -static inline s64 +static __always_inline s64 atomic64_fetch_inc_release(atomic64_t *v) { __atomic_release_fence(); @@ -1528,7 +1527,7 @@ atomic64_fetch_inc_release(atomic64_t *v) #endif #ifndef atomic64_fetch_inc -static inline s64 +static __always_inline s64 atomic64_fetch_inc(atomic64_t *v) { s64 ret; @@ -1543,7 +1542,7 @@ atomic64_fetch_inc(atomic64_t *v) #endif /* atomic64_fetch_inc_relaxed */ #ifndef atomic64_dec -static inline void +static __always_inline void atomic64_dec(atomic64_t *v) { atomic64_sub(1, v); @@ -1559,7 +1558,7 @@ atomic64_dec(atomic64_t *v) #endif /* atomic64_dec_return */ #ifndef atomic64_dec_return -static inline s64 +static __always_inline s64 atomic64_dec_return(atomic64_t *v) { return atomic64_sub_return(1, v); @@ -1568,7 +1567,7 @@ atomic64_dec_return(atomic64_t *v) #endif #ifndef atomic64_dec_return_acquire -static inline s64 +static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { return atomic64_sub_return_acquire(1, v); @@ -1577,7 +1576,7 @@ atomic64_dec_return_acquire(atomic64_t *v) #endif #ifndef atomic64_dec_return_release -static inline s64 +static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { return atomic64_sub_return_release(1, v); @@ -1586,7 +1585,7 @@ atomic64_dec_return_release(atomic64_t *v) #endif #ifndef atomic64_dec_return_relaxed -static inline s64 +static __always_inline s64 atomic64_dec_return_relaxed(atomic64_t *v) { return atomic64_sub_return_relaxed(1, v); @@ -1597,7 +1596,7 @@ atomic64_dec_return_relaxed(atomic64_t *v) #else /* atomic64_dec_return_relaxed */ #ifndef atomic64_dec_return_acquire -static inline s64 +static __always_inline s64 atomic64_dec_return_acquire(atomic64_t *v) { s64 ret = atomic64_dec_return_relaxed(v); @@ -1608,7 +1607,7 @@ atomic64_dec_return_acquire(atomic64_t *v) #endif #ifndef atomic64_dec_return_release -static inline s64 +static __always_inline s64 atomic64_dec_return_release(atomic64_t *v) { __atomic_release_fence(); @@ -1618,7 +1617,7 @@ atomic64_dec_return_release(atomic64_t *v) #endif #ifndef atomic64_dec_return -static inline s64 +static __always_inline s64 atomic64_dec_return(atomic64_t *v) { s64 ret; @@ -1640,7 +1639,7 @@ atomic64_dec_return(atomic64_t *v) #endif /* atomic64_fetch_dec */ #ifndef atomic64_fetch_dec -static inline s64 +static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { return atomic64_fetch_sub(1, v); @@ -1649,7 +1648,7 @@ atomic64_fetch_dec(atomic64_t *v) #endif #ifndef atomic64_fetch_dec_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { return atomic64_fetch_sub_acquire(1, v); @@ -1658,7 +1657,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) #endif #ifndef atomic64_fetch_dec_release -static inline s64 +static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { return atomic64_fetch_sub_release(1, v); @@ -1667,7 +1666,7 @@ atomic64_fetch_dec_release(atomic64_t *v) #endif #ifndef atomic64_fetch_dec_relaxed -static inline s64 +static __always_inline s64 atomic64_fetch_dec_relaxed(atomic64_t *v) { return atomic64_fetch_sub_relaxed(1, v); @@ -1678,7 +1677,7 @@ atomic64_fetch_dec_relaxed(atomic64_t *v) #else /* atomic64_fetch_dec_relaxed */ #ifndef atomic64_fetch_dec_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_dec_acquire(atomic64_t *v) { s64 ret = atomic64_fetch_dec_relaxed(v); @@ -1689,7 +1688,7 @@ atomic64_fetch_dec_acquire(atomic64_t *v) #endif #ifndef atomic64_fetch_dec_release -static inline s64 +static __always_inline s64 atomic64_fetch_dec_release(atomic64_t *v) { __atomic_release_fence(); @@ -1699,7 +1698,7 @@ atomic64_fetch_dec_release(atomic64_t *v) #endif #ifndef atomic64_fetch_dec -static inline s64 +static __always_inline s64 atomic64_fetch_dec(atomic64_t *v) { s64 ret; @@ -1720,7 +1719,7 @@ atomic64_fetch_dec(atomic64_t *v) #else /* atomic64_fetch_and_relaxed */ #ifndef atomic64_fetch_and_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_and_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_and_relaxed(i, v); @@ -1731,7 +1730,7 @@ atomic64_fetch_and_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_and_release -static inline s64 +static __always_inline s64 atomic64_fetch_and_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1741,7 +1740,7 @@ atomic64_fetch_and_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_and -static inline s64 +static __always_inline s64 atomic64_fetch_and(s64 i, atomic64_t *v) { s64 ret; @@ -1756,7 +1755,7 @@ atomic64_fetch_and(s64 i, atomic64_t *v) #endif /* atomic64_fetch_and_relaxed */ #ifndef atomic64_andnot -static inline void +static __always_inline void atomic64_andnot(s64 i, atomic64_t *v) { atomic64_and(~i, v); @@ -1772,7 +1771,7 @@ atomic64_andnot(s64 i, atomic64_t *v) #endif /* atomic64_fetch_andnot */ #ifndef atomic64_fetch_andnot -static inline s64 +static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { return atomic64_fetch_and(~i, v); @@ -1781,7 +1780,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_andnot_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { return atomic64_fetch_and_acquire(~i, v); @@ -1790,7 +1789,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_andnot_release -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { return atomic64_fetch_and_release(~i, v); @@ -1799,7 +1798,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_andnot_relaxed -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) { return atomic64_fetch_and_relaxed(~i, v); @@ -1810,7 +1809,7 @@ atomic64_fetch_andnot_relaxed(s64 i, atomic64_t *v) #else /* atomic64_fetch_andnot_relaxed */ #ifndef atomic64_fetch_andnot_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_andnot_relaxed(i, v); @@ -1821,7 +1820,7 @@ atomic64_fetch_andnot_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_andnot_release -static inline s64 +static __always_inline s64 atomic64_fetch_andnot_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1831,7 +1830,7 @@ atomic64_fetch_andnot_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_andnot -static inline s64 +static __always_inline s64 atomic64_fetch_andnot(s64 i, atomic64_t *v) { s64 ret; @@ -1852,7 +1851,7 @@ atomic64_fetch_andnot(s64 i, atomic64_t *v) #else /* atomic64_fetch_or_relaxed */ #ifndef atomic64_fetch_or_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_or_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_or_relaxed(i, v); @@ -1863,7 +1862,7 @@ atomic64_fetch_or_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_or_release -static inline s64 +static __always_inline s64 atomic64_fetch_or_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1873,7 +1872,7 @@ atomic64_fetch_or_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_or -static inline s64 +static __always_inline s64 atomic64_fetch_or(s64 i, atomic64_t *v) { s64 ret; @@ -1894,7 +1893,7 @@ atomic64_fetch_or(s64 i, atomic64_t *v) #else /* atomic64_fetch_xor_relaxed */ #ifndef atomic64_fetch_xor_acquire -static inline s64 +static __always_inline s64 atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) { s64 ret = atomic64_fetch_xor_relaxed(i, v); @@ -1905,7 +1904,7 @@ atomic64_fetch_xor_acquire(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_xor_release -static inline s64 +static __always_inline s64 atomic64_fetch_xor_release(s64 i, atomic64_t *v) { __atomic_release_fence(); @@ -1915,7 +1914,7 @@ atomic64_fetch_xor_release(s64 i, atomic64_t *v) #endif #ifndef atomic64_fetch_xor -static inline s64 +static __always_inline s64 atomic64_fetch_xor(s64 i, atomic64_t *v) { s64 ret; @@ -1936,7 +1935,7 @@ atomic64_fetch_xor(s64 i, atomic64_t *v) #else /* atomic64_xchg_relaxed */ #ifndef atomic64_xchg_acquire -static inline s64 +static __always_inline s64 atomic64_xchg_acquire(atomic64_t *v, s64 i) { s64 ret = atomic64_xchg_relaxed(v, i); @@ -1947,7 +1946,7 @@ atomic64_xchg_acquire(atomic64_t *v, s64 i) #endif #ifndef atomic64_xchg_release -static inline s64 +static __always_inline s64 atomic64_xchg_release(atomic64_t *v, s64 i) { __atomic_release_fence(); @@ -1957,7 +1956,7 @@ atomic64_xchg_release(atomic64_t *v, s64 i) #endif #ifndef atomic64_xchg -static inline s64 +static __always_inline s64 atomic64_xchg(atomic64_t *v, s64 i) { s64 ret; @@ -1978,7 +1977,7 @@ atomic64_xchg(atomic64_t *v, s64 i) #else /* atomic64_cmpxchg_relaxed */ #ifndef atomic64_cmpxchg_acquire -static inline s64 +static __always_inline s64 atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) { s64 ret = atomic64_cmpxchg_relaxed(v, old, new); @@ -1989,7 +1988,7 @@ atomic64_cmpxchg_acquire(atomic64_t *v, s64 old, s64 new) #endif #ifndef atomic64_cmpxchg_release -static inline s64 +static __always_inline s64 atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) { __atomic_release_fence(); @@ -1999,7 +1998,7 @@ atomic64_cmpxchg_release(atomic64_t *v, s64 old, s64 new) #endif #ifndef atomic64_cmpxchg -static inline s64 +static __always_inline s64 atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) { s64 ret; @@ -2021,7 +2020,7 @@ atomic64_cmpxchg(atomic64_t *v, s64 old, s64 new) #endif /* atomic64_try_cmpxchg */ #ifndef atomic64_try_cmpxchg -static inline bool +static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; @@ -2034,7 +2033,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) #endif #ifndef atomic64_try_cmpxchg_acquire -static inline bool +static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; @@ -2047,7 +2046,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) #endif #ifndef atomic64_try_cmpxchg_release -static inline bool +static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; @@ -2060,7 +2059,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) #endif #ifndef atomic64_try_cmpxchg_relaxed -static inline bool +static __always_inline bool atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) { s64 r, o = *old; @@ -2075,7 +2074,7 @@ atomic64_try_cmpxchg_relaxed(atomic64_t *v, s64 *old, s64 new) #else /* atomic64_try_cmpxchg_relaxed */ #ifndef atomic64_try_cmpxchg_acquire -static inline bool +static __always_inline bool atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) { bool ret = atomic64_try_cmpxchg_relaxed(v, old, new); @@ -2086,7 +2085,7 @@ atomic64_try_cmpxchg_acquire(atomic64_t *v, s64 *old, s64 new) #endif #ifndef atomic64_try_cmpxchg_release -static inline bool +static __always_inline bool atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) { __atomic_release_fence(); @@ -2096,7 +2095,7 @@ atomic64_try_cmpxchg_release(atomic64_t *v, s64 *old, s64 new) #endif #ifndef atomic64_try_cmpxchg -static inline bool +static __always_inline bool atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) { bool ret; @@ -2120,7 +2119,7 @@ atomic64_try_cmpxchg(atomic64_t *v, s64 *old, s64 new) * true if the result is zero, or false for all * other cases. */ -static inline bool +static __always_inline bool atomic64_sub_and_test(s64 i, atomic64_t *v) { return atomic64_sub_return(i, v) == 0; @@ -2137,7 +2136,7 @@ atomic64_sub_and_test(s64 i, atomic64_t *v) * returns true if the result is 0, or false for all other * cases. */ -static inline bool +static __always_inline bool atomic64_dec_and_test(atomic64_t *v) { return atomic64_dec_return(v) == 0; @@ -2154,7 +2153,7 @@ atomic64_dec_and_test(atomic64_t *v) * and returns true if the result is zero, or false for all * other cases. */ -static inline bool +static __always_inline bool atomic64_inc_and_test(atomic64_t *v) { return atomic64_inc_return(v) == 0; @@ -2172,7 +2171,7 @@ atomic64_inc_and_test(atomic64_t *v) * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline bool +static __always_inline bool atomic64_add_negative(s64 i, atomic64_t *v) { return atomic64_add_return(i, v) < 0; @@ -2190,7 +2189,7 @@ atomic64_add_negative(s64 i, atomic64_t *v) * Atomically adds @a to @v, so long as @v was not already @u. * Returns original value of @v */ -static inline s64 +static __always_inline s64 atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) { s64 c = atomic64_read(v); @@ -2215,7 +2214,7 @@ atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) * Atomically adds @a to @v, if @v was not already @u. * Returns true if the addition was done. */ -static inline bool +static __always_inline bool atomic64_add_unless(atomic64_t *v, s64 a, s64 u) { return atomic64_fetch_add_unless(v, a, u) != u; @@ -2231,7 +2230,7 @@ atomic64_add_unless(atomic64_t *v, s64 a, s64 u) * Atomically increments @v by 1, if @v is non-zero. * Returns true if the increment was done. */ -static inline bool +static __always_inline bool atomic64_inc_not_zero(atomic64_t *v) { return atomic64_add_unless(v, 1, 0); @@ -2240,7 +2239,7 @@ atomic64_inc_not_zero(atomic64_t *v) #endif #ifndef atomic64_inc_unless_negative -static inline bool +static __always_inline bool atomic64_inc_unless_negative(atomic64_t *v) { s64 c = atomic64_read(v); @@ -2256,7 +2255,7 @@ atomic64_inc_unless_negative(atomic64_t *v) #endif #ifndef atomic64_dec_unless_positive -static inline bool +static __always_inline bool atomic64_dec_unless_positive(atomic64_t *v) { s64 c = atomic64_read(v); @@ -2272,7 +2271,7 @@ atomic64_dec_unless_positive(atomic64_t *v) #endif #ifndef atomic64_dec_if_positive -static inline s64 +static __always_inline s64 atomic64_dec_if_positive(atomic64_t *v) { s64 dec, c = atomic64_read(v); @@ -2288,8 +2287,5 @@ atomic64_dec_if_positive(atomic64_t *v) #define atomic64_dec_if_positive atomic64_dec_if_positive #endif -#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) -#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) - #endif /* _LINUX_ATOMIC_FALLBACK_H */ -// 25de4a2804d70f57e994fe3b419148658bb5378a +// 1fac0941c79bf0ae100723cc2ac9b94061f0b67a diff --git a/include/linux/atomic.h b/include/linux/atomic.h index 4c0d009a46f0..571a11008ab5 100644 --- a/include/linux/atomic.h +++ b/include/linux/atomic.h @@ -25,6 +25,12 @@ * See Documentation/memory-barriers.txt for ACQUIRE/RELEASE definitions. */ +#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) +#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) + +#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) +#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) + /* * The idea here is to build acquire/release variants by adding explicit * barriers on top of the relaxed variant. In the case where the relaxed @@ -71,7 +77,12 @@ __ret; \ }) +#ifdef ARCH_ATOMIC +#include <linux/atomic-arch-fallback.h> +#include <asm-generic/atomic-instrumented.h> +#else #include <linux/atomic-fallback.h> +#endif #include <asm-generic/atomic-long.h> diff --git a/include/linux/bsearch.h b/include/linux/bsearch.h index 8ed53d7524ea..e66b711d091e 100644 --- a/include/linux/bsearch.h +++ b/include/linux/bsearch.h @@ -4,7 +4,29 @@ #include <linux/types.h> -void *bsearch(const void *key, const void *base, size_t num, size_t size, - cmp_func_t cmp); +static __always_inline +void *__inline_bsearch(const void *key, const void *base, size_t num, size_t size, cmp_func_t cmp) +{ + const char *pivot; + int result; + + while (num > 0) { + pivot = base + (num >> 1) * size; + result = cmp(key, pivot); + + if (result == 0) + return (void *)pivot; + + if (result > 0) { + base = pivot + size; + num--; + } + num >>= 1; + } + + return NULL; +} + +extern void *bsearch(const void *key, const void *base, size_t num, size_t size, cmp_func_t cmp); #endif /* _LINUX_BSEARCH_H */ diff --git a/include/linux/compiler-clang.h b/include/linux/compiler-clang.h index 790c0c6b8552..ee37256ec8bd 100644 --- a/include/linux/compiler-clang.h +++ b/include/linux/compiler-clang.h @@ -16,7 +16,7 @@ #define KASAN_ABI_VERSION 5 #if __has_feature(address_sanitizer) || __has_feature(hwaddress_sanitizer) -/* emulate gcc's __SANITIZE_ADDRESS__ flag */ +/* Emulate GCC's __SANITIZE_ADDRESS__ flag */ #define __SANITIZE_ADDRESS__ #define __no_sanitize_address \ __attribute__((no_sanitize("address", "hwaddress"))) @@ -24,6 +24,15 @@ #define __no_sanitize_address #endif +#if __has_feature(thread_sanitizer) +/* emulate gcc's __SANITIZE_THREAD__ flag */ +#define __SANITIZE_THREAD__ +#define __no_sanitize_thread \ + __attribute__((no_sanitize("thread"))) +#else +#define __no_sanitize_thread +#endif + /* * Not all versions of clang implement the the type-generic versions * of the builtin overflow checkers. Fortunately, clang implements diff --git a/include/linux/compiler-gcc.h b/include/linux/compiler-gcc.h index e2f725273261..7dd4e0349ef3 100644 --- a/include/linux/compiler-gcc.h +++ b/include/linux/compiler-gcc.h @@ -144,6 +144,12 @@ #define __no_sanitize_address #endif +#if defined(__SANITIZE_THREAD__) && __has_attribute(__no_sanitize_thread__) +#define __no_sanitize_thread __attribute__((no_sanitize_thread)) +#else +#define __no_sanitize_thread +#endif + #if GCC_VERSION >= 50100 #define COMPILER_HAS_GENERIC_BUILTIN_OVERFLOW 1 #endif diff --git a/include/linux/compiler.h b/include/linux/compiler.h index 33d3a2e5abab..30827f82ad62 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -250,6 +250,27 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, */ #include <asm/barrier.h> #include <linux/kasan-checks.h> +#include <linux/kcsan-checks.h> + +/** + * data_race - mark an expression as containing intentional data races + * + * This data_race() macro is useful for situations in which data races + * should be forgiven. One example is diagnostic code that accesses + * shared variables but is not a part of the core synchronization design. + * + * This macro *does not* affect normal code generation, but is a hint + * to tooling that data races here are to be ignored. + */ +#define data_race(expr) \ +({ \ + __unqual_scalar_typeof(({ expr; })) __v = ({ \ + __kcsan_disable_current(); \ + expr; \ + }); \ + __kcsan_enable_current(); \ + __v; \ +}) /* * Use __READ_ONCE() instead of READ_ONCE() if you do not require any @@ -271,30 +292,18 @@ void ftrace_likely_update(struct ftrace_likely_data *f, int val, __READ_ONCE_SCALAR(x); \ }) -#define __WRITE_ONCE(x, val) \ -do { \ - *(volatile typeof(x) *)&(x) = (val); \ +#define __WRITE_ONCE(x, val) \ +do { \ + *(volatile typeof(x) *)&(x) = (val); \ } while (0) -#define WRITE_ONCE(x, val) \ -do { \ - compiletime_assert_rwonce_type(x); \ - __WRITE_ONCE(x, val); \ +#define WRITE_ONCE(x, val) \ +do { \ + compiletime_assert_rwonce_type(x); \ + __WRITE_ONCE(x, val); \ } while (0) -#ifdef CONFIG_KASAN -/* - * We can't declare function 'inline' because __no_sanitize_address conflicts - * with inlining. Attempt to inline it may cause a build failure. - * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 - * '__maybe_unused' allows us to avoid defined-but-not-used warnings. - */ -# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused -#else -# define __no_kasan_or_inline __always_inline -#endif - -static __no_kasan_or_inline +static __no_sanitize_or_inline unsigned long __read_once_word_nocheck(const void *addr) { return __READ_ONCE(*(unsigned long *)addr); @@ -302,8 +311,8 @@ unsigned long __read_once_word_nocheck(const void *addr) /* * Use READ_ONCE_NOCHECK() instead of READ_ONCE() if you need to load a - * word from memory atomically but without telling KASAN. This is usually - * used by unwinding code when walking the stack of a running process. + * word from memory atomically but without telling KASAN/KCSAN. This is + * usually used by unwinding code when walking the stack of a running process. */ #define READ_ONCE_NOCHECK(x) \ ({ \ diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index d4e1956f903f..21aed0981edf 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -171,6 +171,38 @@ struct ftrace_likely_data { */ #define noinline_for_stack noinline +/* + * Sanitizer helper attributes: Because using __always_inline and + * __no_sanitize_* conflict, provide helper attributes that will either expand + * to __no_sanitize_* in compilation units where instrumentation is enabled + * (__SANITIZE_*__), or __always_inline in compilation units without + * instrumentation (__SANITIZE_*__ undefined). + */ +#ifdef __SANITIZE_ADDRESS__ +/* + * We can't declare function 'inline' because __no_sanitize_address conflicts + * with inlining. Attempt to inline it may cause a build failure. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=67368 + * '__maybe_unused' allows us to avoid defined-but-not-used warnings. + */ +# define __no_kasan_or_inline __no_sanitize_address notrace __maybe_unused +# define __no_sanitize_or_inline __no_kasan_or_inline +#else +# define __no_kasan_or_inline __always_inline +#endif + +#define __no_kcsan __no_sanitize_thread +#ifdef __SANITIZE_THREAD__ +# define __no_kcsan_or_inline __no_kcsan notrace __maybe_unused +# define __no_sanitize_or_inline __no_kcsan_or_inline +#else +# define __no_kcsan_or_inline __always_inline +#endif + +#ifndef __no_sanitize_or_inline +#define __no_sanitize_or_inline __always_inline +#endif + #endif /* __KERNEL__ */ #endif /* __ASSEMBLY__ */ diff --git a/include/linux/context_tracking.h b/include/linux/context_tracking.h index 8cac62ee6add..981b880d5b60 100644 --- a/include/linux/context_tracking.h +++ b/include/linux/context_tracking.h @@ -33,13 +33,13 @@ static inline void user_exit(void) } /* Called with interrupts disabled. */ -static inline void user_enter_irqoff(void) +static __always_inline void user_enter_irqoff(void) { if (context_tracking_enabled()) __context_tracking_enter(CONTEXT_USER); } -static inline void user_exit_irqoff(void) +static __always_inline void user_exit_irqoff(void) { if (context_tracking_enabled()) __context_tracking_exit(CONTEXT_USER); @@ -75,7 +75,7 @@ static inline void exception_exit(enum ctx_state prev_ctx) * is enabled. If context tracking is disabled, returns * CONTEXT_DISABLED. This should be used primarily for debugging. */ -static inline enum ctx_state ct_state(void) +static __always_inline enum ctx_state ct_state(void) { return context_tracking_enabled() ? this_cpu_read(context_tracking.state) : CONTEXT_DISABLED; diff --git a/include/linux/context_tracking_state.h b/include/linux/context_tracking_state.h index e7fe6678b7ad..65a60d3313b0 100644 --- a/include/linux/context_tracking_state.h +++ b/include/linux/context_tracking_state.h @@ -26,12 +26,12 @@ struct context_tracking { extern struct static_key_false context_tracking_key; DECLARE_PER_CPU(struct context_tracking, context_tracking); -static inline bool context_tracking_enabled(void) +static __always_inline bool context_tracking_enabled(void) { return static_branch_unlikely(&context_tracking_key); } -static inline bool context_tracking_enabled_cpu(int cpu) +static __always_inline bool context_tracking_enabled_cpu(int cpu) { return context_tracking_enabled() && per_cpu(context_tracking.active, cpu); } @@ -41,7 +41,7 @@ static inline bool context_tracking_enabled_this_cpu(void) return context_tracking_enabled() && __this_cpu_read(context_tracking.active); } -static inline bool context_tracking_in_user(void) +static __always_inline bool context_tracking_in_user(void) { return __this_cpu_read(context_tracking.state) == CONTEXT_USER; } diff --git a/include/linux/cpu_cooling.h b/include/linux/cpu_cooling.h index 65501d8f9778..a3bdc8a98f2c 100644 --- a/include/linux/cpu_cooling.h +++ b/include/linux/cpu_cooling.h @@ -63,18 +63,10 @@ of_cpufreq_cooling_register(struct cpufreq_policy *policy) struct cpuidle_driver; #ifdef CONFIG_CPU_IDLE_THERMAL -int cpuidle_cooling_register(struct cpuidle_driver *drv); -int cpuidle_of_cooling_register(struct device_node *np, - struct cpuidle_driver *drv); +void cpuidle_cooling_register(struct cpuidle_driver *drv); #else /* CONFIG_CPU_IDLE_THERMAL */ -static inline int cpuidle_cooling_register(struct cpuidle_driver *drv) +static inline void cpuidle_cooling_register(struct cpuidle_driver *drv) { - return 0; -} -static inline int cpuidle_of_cooling_register(struct device_node *np, - struct cpuidle_driver *drv) -{ - return 0; } #endif /* CONFIG_CPU_IDLE_THERMAL */ diff --git a/include/linux/cpuhotplug.h b/include/linux/cpuhotplug.h index 8377afef8806..191772d4a4d7 100644 --- a/include/linux/cpuhotplug.h +++ b/include/linux/cpuhotplug.h @@ -102,6 +102,7 @@ enum cpuhp_state { CPUHP_AP_IRQ_ARMADA_XP_STARTING, CPUHP_AP_IRQ_BCM2836_STARTING, CPUHP_AP_IRQ_MIPS_GIC_STARTING, + CPUHP_AP_IRQ_RISCV_STARTING, CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, CPUHP_AP_ARM_MVEBU_COHERENCY, CPUHP_AP_MICROCODE_LOADER, diff --git a/include/linux/debug_locks.h b/include/linux/debug_locks.h index 257ab3c92cb8..e7e45f0cc7da 100644 --- a/include/linux/debug_locks.h +++ b/include/linux/debug_locks.h @@ -12,7 +12,7 @@ extern int debug_locks __read_mostly; extern int debug_locks_silent __read_mostly; -static inline int __debug_locks_off(void) +static __always_inline int __debug_locks_off(void) { return xchg(&debug_locks, 0); } diff --git a/include/linux/edac.h b/include/linux/edac.h index 0f20b986b0ab..6eb7d55d7c3d 100644 --- a/include/linux/edac.h +++ b/include/linux/edac.h @@ -31,14 +31,6 @@ struct device; extern int edac_op_state; struct bus_type *edac_get_sysfs_subsys(void); -int edac_get_report_status(void); -void edac_set_report_status(int new); - -enum { - EDAC_REPORTING_ENABLED, - EDAC_REPORTING_DISABLED, - EDAC_REPORTING_FORCE -}; static inline void opstate_init(void) { diff --git a/include/linux/fs.h b/include/linux/fs.h index 19ef6c88c152..8e1f8f93108f 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -1048,6 +1048,7 @@ struct lock_manager_operations { bool (*lm_break)(struct file_lock *); int (*lm_change)(struct file_lock *, int, struct list_head *); void (*lm_setup)(struct file_lock *, void **); + bool (*lm_breaker_owns_lease)(struct file_lock *); }; struct lock_manager { diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index e07cf853aa16..03c9fece7d43 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -38,9 +38,24 @@ static __always_inline void rcu_irq_enter_check_tick(void) } while (0) /* + * Like __irq_enter() without time accounting for fast + * interrupts, e.g. reschedule IPI where time accounting + * is more expensive than the actual interrupt. + */ +#define __irq_enter_raw() \ + do { \ + preempt_count_add(HARDIRQ_OFFSET); \ + lockdep_hardirq_enter(); \ + } while (0) + +/* * Enter irq context (on NO_HZ, update jiffies): */ -extern void irq_enter(void); +void irq_enter(void); +/* + * Like irq_enter(), but RCU is already watching. + */ +void irq_enter_rcu(void); /* * Exit irq context without processing softirqs: @@ -53,9 +68,23 @@ extern void irq_enter(void); } while (0) /* + * Like __irq_exit() without time accounting + */ +#define __irq_exit_raw() \ + do { \ + lockdep_hardirq_exit(); \ + preempt_count_sub(HARDIRQ_OFFSET); \ + } while (0) + +/* * Exit irq context and process softirqs if needed: */ -extern void irq_exit(void); +void irq_exit(void); + +/* + * Like irq_exit(), but return with RCU watching. + */ +void irq_exit_rcu(void); #ifndef arch_nmi_enter #define arch_nmi_enter() do { } while (0) @@ -87,20 +116,24 @@ extern void rcu_nmi_exit(void); arch_nmi_enter(); \ printk_nmi_enter(); \ lockdep_off(); \ - ftrace_nmi_enter(); \ BUG_ON(in_nmi() == NMI_MASK); \ __preempt_count_add(NMI_OFFSET + HARDIRQ_OFFSET); \ rcu_nmi_enter(); \ lockdep_hardirq_enter(); \ + instrumentation_begin(); \ + ftrace_nmi_enter(); \ + instrumentation_end(); \ } while (0) #define nmi_exit() \ do { \ + instrumentation_begin(); \ + ftrace_nmi_exit(); \ + instrumentation_end(); \ lockdep_hardirq_exit(); \ rcu_nmi_exit(); \ BUG_ON(!in_nmi()); \ __preempt_count_sub(NMI_OFFSET + HARDIRQ_OFFSET); \ - ftrace_nmi_exit(); \ lockdep_on(); \ printk_nmi_exit(); \ arch_nmi_exit(); \ diff --git a/include/linux/i2c-smbus.h b/include/linux/i2c-smbus.h index 8c5459034f92..1e4e0de4ef8b 100644 --- a/include/linux/i2c-smbus.h +++ b/include/linux/i2c-smbus.h @@ -2,7 +2,7 @@ /* * i2c-smbus.h - SMBus extensions to the I2C protocol * - * Copyright (C) 2010 Jean Delvare <jdelvare@suse.de> + * Copyright (C) 2010-2019 Jean Delvare <jdelvare@suse.de> */ #ifndef _LINUX_I2C_SMBUS_H @@ -39,4 +39,10 @@ static inline int of_i2c_setup_smbus_alert(struct i2c_adapter *adap) } #endif +#if IS_ENABLED(CONFIG_I2C_SMBUS) && IS_ENABLED(CONFIG_DMI) +void i2c_register_spd(struct i2c_adapter *adap); +#else +static inline void i2c_register_spd(struct i2c_adapter *adap) { } +#endif + #endif /* _LINUX_I2C_SMBUS_H */ diff --git a/include/linux/i2c.h b/include/linux/i2c.h index 49d29054e657..c10617bb980a 100644 --- a/include/linux/i2c.h +++ b/include/linux/i2c.h @@ -351,14 +351,14 @@ static inline struct i2c_client *kobj_to_i2c_client(struct kobject *kobj) return to_i2c_client(dev); } -static inline void *i2c_get_clientdata(const struct i2c_client *dev) +static inline void *i2c_get_clientdata(const struct i2c_client *client) { - return dev_get_drvdata(&dev->dev); + return dev_get_drvdata(&client->dev); } -static inline void i2c_set_clientdata(struct i2c_client *dev, void *data) +static inline void i2c_set_clientdata(struct i2c_client *client, void *data) { - dev_set_drvdata(&dev->dev, data); + dev_set_drvdata(&client->dev, data); } /* I2C slave support */ diff --git a/include/linux/idle_inject.h b/include/linux/idle_inject.h index a445cd1a36c5..91a8612b8bf9 100644 --- a/include/linux/idle_inject.h +++ b/include/linux/idle_inject.h @@ -26,4 +26,8 @@ void idle_inject_set_duration(struct idle_inject_device *ii_dev, void idle_inject_get_duration(struct idle_inject_device *ii_dev, unsigned int *run_duration_us, unsigned int *idle_duration_us); + +void idle_inject_set_latency(struct idle_inject_device *ii_dev, + unsigned int latency_ns); + #endif /* __IDLE_INJECT_H__ */ diff --git a/include/linux/instrumented.h b/include/linux/instrumented.h new file mode 100644 index 000000000000..43e6ea591975 --- /dev/null +++ b/include/linux/instrumented.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * This header provides generic wrappers for memory access instrumentation that + * the compiler cannot emit for: KASAN, KCSAN. + */ +#ifndef _LINUX_INSTRUMENTED_H +#define _LINUX_INSTRUMENTED_H + +#include <linux/compiler.h> +#include <linux/kasan-checks.h> +#include <linux/kcsan-checks.h> +#include <linux/types.h> + +/** + * instrument_read - instrument regular read access + * + * Instrument a regular read access. The instrumentation should be inserted + * before the actual read happens. + * + * @ptr address of access + * @size size of access + */ +static __always_inline void instrument_read(const volatile void *v, size_t size) +{ + kasan_check_read(v, size); + kcsan_check_read(v, size); +} + +/** + * instrument_write - instrument regular write access + * + * Instrument a regular write access. The instrumentation should be inserted + * before the actual write happens. + * + * @ptr address of access + * @size size of access + */ +static __always_inline void instrument_write(const volatile void *v, size_t size) +{ + kasan_check_write(v, size); + kcsan_check_write(v, size); +} + +/** + * instrument_atomic_read - instrument atomic read access + * + * Instrument an atomic read access. The instrumentation should be inserted + * before the actual read happens. + * + * @ptr address of access + * @size size of access + */ +static __always_inline void instrument_atomic_read(const volatile void *v, size_t size) +{ + kasan_check_read(v, size); + kcsan_check_atomic_read(v, size); +} + +/** + * instrument_atomic_write - instrument atomic write access + * + * Instrument an atomic write access. The instrumentation should be inserted + * before the actual write happens. + * + * @ptr address of access + * @size size of access + */ +static __always_inline void instrument_atomic_write(const volatile void *v, size_t size) +{ + kasan_check_write(v, size); + kcsan_check_atomic_write(v, size); +} + +/** + * instrument_copy_to_user - instrument reads of copy_to_user + * + * Instrument reads from kernel memory, that are due to copy_to_user (and + * variants). The instrumentation must be inserted before the accesses. + * + * @to destination address + * @from source address + * @n number of bytes to copy + */ +static __always_inline void +instrument_copy_to_user(void __user *to, const void *from, unsigned long n) +{ + kasan_check_read(from, n); + kcsan_check_read(from, n); +} + +/** + * instrument_copy_from_user - instrument writes of copy_from_user + * + * Instrument writes to kernel memory, that are due to copy_from_user (and + * variants). The instrumentation should be inserted before the accesses. + * + * @to destination address + * @from source address + * @n number of bytes to copy + */ +static __always_inline void +instrument_copy_from_user(const void *to, const void __user *from, unsigned long n) +{ + kasan_check_write(to, n); + kcsan_check_write(to, n); +} + +#endif /* _LINUX_INSTRUMENTED_H */ diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h index 80f637c3a6f3..5db970b6615a 100644 --- a/include/linux/interrupt.h +++ b/include/linux/interrupt.h @@ -760,8 +760,10 @@ extern int arch_early_irq_init(void); /* * We want to know which function is an entrypoint of a hardirq or a softirq. */ -#define __irq_entry __attribute__((__section__(".irqentry.text"))) -#define __softirq_entry \ - __attribute__((__section__(".softirqentry.text"))) +#ifndef __irq_entry +# define __irq_entry __attribute__((__section__(".irqentry.text"))) +#endif + +#define __softirq_entry __attribute__((__section__(".softirqentry.text"))) #endif diff --git a/include/linux/irqflags.h b/include/linux/irqflags.h index d7f7e436c3af..6384d2813ded 100644 --- a/include/linux/irqflags.h +++ b/include/linux/irqflags.h @@ -32,7 +32,7 @@ #ifdef CONFIG_TRACE_IRQFLAGS extern void trace_hardirqs_on_prepare(void); - extern void trace_hardirqs_off_prepare(void); + extern void trace_hardirqs_off_finish(void); extern void trace_hardirqs_on(void); extern void trace_hardirqs_off(void); # define lockdep_hardirq_context(p) ((p)->hardirq_context) @@ -101,7 +101,7 @@ do { \ #else # define trace_hardirqs_on_prepare() do { } while (0) -# define trace_hardirqs_off_prepare() do { } while (0) +# define trace_hardirqs_off_finish() do { } while (0) # define trace_hardirqs_on() do { } while (0) # define trace_hardirqs_off() do { } while (0) # define lockdep_hardirq_context(p) 0 diff --git a/include/linux/kcsan-checks.h b/include/linux/kcsan-checks.h new file mode 100644 index 000000000000..7b0b9c44f5f3 --- /dev/null +++ b/include/linux/kcsan-checks.h @@ -0,0 +1,430 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_KCSAN_CHECKS_H +#define _LINUX_KCSAN_CHECKS_H + +/* Note: Only include what is already included by compiler.h. */ +#include <linux/compiler_attributes.h> +#include <linux/types.h> + +/* + * ACCESS TYPE MODIFIERS + * + * <none>: normal read access; + * WRITE : write access; + * ATOMIC: access is atomic; + * ASSERT: access is not a regular access, but an assertion; + * SCOPED: access is a scoped access; + */ +#define KCSAN_ACCESS_WRITE 0x1 +#define KCSAN_ACCESS_ATOMIC 0x2 +#define KCSAN_ACCESS_ASSERT 0x4 +#define KCSAN_ACCESS_SCOPED 0x8 + +/* + * __kcsan_*: Always calls into the runtime when KCSAN is enabled. This may be used + * even in compilation units that selectively disable KCSAN, but must use KCSAN + * to validate access to an address. Never use these in header files! + */ +#ifdef CONFIG_KCSAN +/** + * __kcsan_check_access - check generic access for races + * + * @ptr: address of access + * @size: size of access + * @type: access type modifier + */ +void __kcsan_check_access(const volatile void *ptr, size_t size, int type); + +/** + * kcsan_disable_current - disable KCSAN for the current context + * + * Supports nesting. + */ +void kcsan_disable_current(void); + +/** + * kcsan_enable_current - re-enable KCSAN for the current context + * + * Supports nesting. + */ +void kcsan_enable_current(void); +void kcsan_enable_current_nowarn(void); /* Safe in uaccess regions. */ + +/** + * kcsan_nestable_atomic_begin - begin nestable atomic region + * + * Accesses within the atomic region may appear to race with other accesses but + * should be considered atomic. + */ +void kcsan_nestable_atomic_begin(void); + +/** + * kcsan_nestable_atomic_end - end nestable atomic region + */ +void kcsan_nestable_atomic_end(void); + +/** + * kcsan_flat_atomic_begin - begin flat atomic region + * + * Accesses within the atomic region may appear to race with other accesses but + * should be considered atomic. + */ +void kcsan_flat_atomic_begin(void); + +/** + * kcsan_flat_atomic_end - end flat atomic region + */ +void kcsan_flat_atomic_end(void); + +/** + * kcsan_atomic_next - consider following accesses as atomic + * + * Force treating the next n memory accesses for the current context as atomic + * operations. + * + * @n: number of following memory accesses to treat as atomic. + */ +void kcsan_atomic_next(int n); + +/** + * kcsan_set_access_mask - set access mask + * + * Set the access mask for all accesses for the current context if non-zero. + * Only value changes to bits set in the mask will be reported. + * + * @mask: bitmask + */ +void kcsan_set_access_mask(unsigned long mask); + +/* Scoped access information. */ +struct kcsan_scoped_access { + struct list_head list; + const volatile void *ptr; + size_t size; + int type; +}; +/* + * Automatically call kcsan_end_scoped_access() when kcsan_scoped_access goes + * out of scope; relies on attribute "cleanup", which is supported by all + * compilers that support KCSAN. + */ +#define __kcsan_cleanup_scoped \ + __maybe_unused __attribute__((__cleanup__(kcsan_end_scoped_access))) + +/** + * kcsan_begin_scoped_access - begin scoped access + * + * Begin scoped access and initialize @sa, which will cause KCSAN to + * continuously check the memory range in the current thread until + * kcsan_end_scoped_access() is called for @sa. + * + * Scoped accesses are implemented by appending @sa to an internal list for the + * current execution context, and then checked on every call into the KCSAN + * runtime. + * + * @ptr: address of access + * @size: size of access + * @type: access type modifier + * @sa: struct kcsan_scoped_access to use for the scope of the access + */ +struct kcsan_scoped_access * +kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type, + struct kcsan_scoped_access *sa); + +/** + * kcsan_end_scoped_access - end scoped access + * + * End a scoped access, which will stop KCSAN checking the memory range. + * Requires that kcsan_begin_scoped_access() was previously called once for @sa. + * + * @sa: a previously initialized struct kcsan_scoped_access + */ +void kcsan_end_scoped_access(struct kcsan_scoped_access *sa); + + +#else /* CONFIG_KCSAN */ + +static inline void __kcsan_check_access(const volatile void *ptr, size_t size, + int type) { } + +static inline void kcsan_disable_current(void) { } +static inline void kcsan_enable_current(void) { } +static inline void kcsan_enable_current_nowarn(void) { } +static inline void kcsan_nestable_atomic_begin(void) { } +static inline void kcsan_nestable_atomic_end(void) { } +static inline void kcsan_flat_atomic_begin(void) { } +static inline void kcsan_flat_atomic_end(void) { } +static inline void kcsan_atomic_next(int n) { } +static inline void kcsan_set_access_mask(unsigned long mask) { } + +struct kcsan_scoped_access { }; +#define __kcsan_cleanup_scoped __maybe_unused +static inline struct kcsan_scoped_access * +kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type, + struct kcsan_scoped_access *sa) { return sa; } +static inline void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) { } + +#endif /* CONFIG_KCSAN */ + +#ifdef __SANITIZE_THREAD__ +/* + * Only calls into the runtime when the particular compilation unit has KCSAN + * instrumentation enabled. May be used in header files. + */ +#define kcsan_check_access __kcsan_check_access + +/* + * Only use these to disable KCSAN for accesses in the current compilation unit; + * calls into libraries may still perform KCSAN checks. + */ +#define __kcsan_disable_current kcsan_disable_current +#define __kcsan_enable_current kcsan_enable_current_nowarn +#else +static inline void kcsan_check_access(const volatile void *ptr, size_t size, + int type) { } +static inline void __kcsan_enable_current(void) { } +static inline void __kcsan_disable_current(void) { } +#endif + +/** + * __kcsan_check_read - check regular read access for races + * + * @ptr: address of access + * @size: size of access + */ +#define __kcsan_check_read(ptr, size) __kcsan_check_access(ptr, size, 0) + +/** + * __kcsan_check_write - check regular write access for races + * + * @ptr: address of access + * @size: size of access + */ +#define __kcsan_check_write(ptr, size) \ + __kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + +/** + * kcsan_check_read - check regular read access for races + * + * @ptr: address of access + * @size: size of access + */ +#define kcsan_check_read(ptr, size) kcsan_check_access(ptr, size, 0) + +/** + * kcsan_check_write - check regular write access for races + * + * @ptr: address of access + * @size: size of access + */ +#define kcsan_check_write(ptr, size) \ + kcsan_check_access(ptr, size, KCSAN_ACCESS_WRITE) + +/* + * Check for atomic accesses: if atomic accesses are not ignored, this simply + * aliases to kcsan_check_access(), otherwise becomes a no-op. + */ +#ifdef CONFIG_KCSAN_IGNORE_ATOMICS +#define kcsan_check_atomic_read(...) do { } while (0) +#define kcsan_check_atomic_write(...) do { } while (0) +#else +#define kcsan_check_atomic_read(ptr, size) \ + kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC) +#define kcsan_check_atomic_write(ptr, size) \ + kcsan_check_access(ptr, size, KCSAN_ACCESS_ATOMIC | KCSAN_ACCESS_WRITE) +#endif + +/** + * ASSERT_EXCLUSIVE_WRITER - assert no concurrent writes to @var + * + * Assert that there are no concurrent writes to @var; other readers are + * allowed. This assertion can be used to specify properties of concurrent code, + * where violation cannot be detected as a normal data race. + * + * For example, if we only have a single writer, but multiple concurrent + * readers, to avoid data races, all these accesses must be marked; even + * concurrent marked writes racing with the single writer are bugs. + * Unfortunately, due to being marked, they are no longer data races. For cases + * like these, we can use the macro as follows: + * + * .. code-block:: c + * + * void writer(void) { + * spin_lock(&update_foo_lock); + * ASSERT_EXCLUSIVE_WRITER(shared_foo); + * WRITE_ONCE(shared_foo, ...); + * spin_unlock(&update_foo_lock); + * } + * void reader(void) { + * // update_foo_lock does not need to be held! + * ... = READ_ONCE(shared_foo); + * } + * + * Note: ASSERT_EXCLUSIVE_WRITER_SCOPED(), if applicable, performs more thorough + * checking if a clear scope where no concurrent writes are expected exists. + * + * @var: variable to assert on + */ +#define ASSERT_EXCLUSIVE_WRITER(var) \ + __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT) + +/* + * Helper macros for implementation of for ASSERT_EXCLUSIVE_*_SCOPED(). @id is + * expected to be unique for the scope in which instances of kcsan_scoped_access + * are declared. + */ +#define __kcsan_scoped_name(c, suffix) __kcsan_scoped_##c##suffix +#define __ASSERT_EXCLUSIVE_SCOPED(var, type, id) \ + struct kcsan_scoped_access __kcsan_scoped_name(id, _) \ + __kcsan_cleanup_scoped; \ + struct kcsan_scoped_access *__kcsan_scoped_name(id, _dummy_p) \ + __maybe_unused = kcsan_begin_scoped_access( \ + &(var), sizeof(var), KCSAN_ACCESS_SCOPED | (type), \ + &__kcsan_scoped_name(id, _)) + +/** + * ASSERT_EXCLUSIVE_WRITER_SCOPED - assert no concurrent writes to @var in scope + * + * Scoped variant of ASSERT_EXCLUSIVE_WRITER(). + * + * Assert that there are no concurrent writes to @var for the duration of the + * scope in which it is introduced. This provides a better way to fully cover + * the enclosing scope, compared to multiple ASSERT_EXCLUSIVE_WRITER(), and + * increases the likelihood for KCSAN to detect racing accesses. + * + * For example, it allows finding race-condition bugs that only occur due to + * state changes within the scope itself: + * + * .. code-block:: c + * + * void writer(void) { + * spin_lock(&update_foo_lock); + * { + * ASSERT_EXCLUSIVE_WRITER_SCOPED(shared_foo); + * WRITE_ONCE(shared_foo, 42); + * ... + * // shared_foo should still be 42 here! + * } + * spin_unlock(&update_foo_lock); + * } + * void buggy(void) { + * if (READ_ONCE(shared_foo) == 42) + * WRITE_ONCE(shared_foo, 1); // bug! + * } + * + * @var: variable to assert on + */ +#define ASSERT_EXCLUSIVE_WRITER_SCOPED(var) \ + __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_ASSERT, __COUNTER__) + +/** + * ASSERT_EXCLUSIVE_ACCESS - assert no concurrent accesses to @var + * + * Assert that there are no concurrent accesses to @var (no readers nor + * writers). This assertion can be used to specify properties of concurrent + * code, where violation cannot be detected as a normal data race. + * + * For example, where exclusive access is expected after determining no other + * users of an object are left, but the object is not actually freed. We can + * check that this property actually holds as follows: + * + * .. code-block:: c + * + * if (refcount_dec_and_test(&obj->refcnt)) { + * ASSERT_EXCLUSIVE_ACCESS(*obj); + * do_some_cleanup(obj); + * release_for_reuse(obj); + * } + * + * Note: ASSERT_EXCLUSIVE_ACCESS_SCOPED(), if applicable, performs more thorough + * checking if a clear scope where no concurrent accesses are expected exists. + * + * Note: For cases where the object is freed, `KASAN <kasan.html>`_ is a better + * fit to detect use-after-free bugs. + * + * @var: variable to assert on + */ +#define ASSERT_EXCLUSIVE_ACCESS(var) \ + __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT) + +/** + * ASSERT_EXCLUSIVE_ACCESS_SCOPED - assert no concurrent accesses to @var in scope + * + * Scoped variant of ASSERT_EXCLUSIVE_ACCESS(). + * + * Assert that there are no concurrent accesses to @var (no readers nor writers) + * for the entire duration of the scope in which it is introduced. This provides + * a better way to fully cover the enclosing scope, compared to multiple + * ASSERT_EXCLUSIVE_ACCESS(), and increases the likelihood for KCSAN to detect + * racing accesses. + * + * @var: variable to assert on + */ +#define ASSERT_EXCLUSIVE_ACCESS_SCOPED(var) \ + __ASSERT_EXCLUSIVE_SCOPED(var, KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT, __COUNTER__) + +/** + * ASSERT_EXCLUSIVE_BITS - assert no concurrent writes to subset of bits in @var + * + * Bit-granular variant of ASSERT_EXCLUSIVE_WRITER(). + * + * Assert that there are no concurrent writes to a subset of bits in @var; + * concurrent readers are permitted. This assertion captures more detailed + * bit-level properties, compared to the other (word granularity) assertions. + * Only the bits set in @mask are checked for concurrent modifications, while + * ignoring the remaining bits, i.e. concurrent writes (or reads) to ~mask bits + * are ignored. + * + * Use this for variables, where some bits must not be modified concurrently, + * yet other bits are expected to be modified concurrently. + * + * For example, variables where, after initialization, some bits are read-only, + * but other bits may still be modified concurrently. A reader may wish to + * assert that this is true as follows: + * + * .. code-block:: c + * + * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK); + * foo = (READ_ONCE(flags) & READ_ONLY_MASK) >> READ_ONLY_SHIFT; + * + * Note: The access that immediately follows ASSERT_EXCLUSIVE_BITS() is assumed + * to access the masked bits only, and KCSAN optimistically assumes it is + * therefore safe, even in the presence of data races, and marking it with + * READ_ONCE() is optional from KCSAN's point-of-view. We caution, however, that + * it may still be advisable to do so, since we cannot reason about all compiler + * optimizations when it comes to bit manipulations (on the reader and writer + * side). If you are sure nothing can go wrong, we can write the above simply + * as: + * + * .. code-block:: c + * + * ASSERT_EXCLUSIVE_BITS(flags, READ_ONLY_MASK); + * foo = (flags & READ_ONLY_MASK) >> READ_ONLY_SHIFT; + * + * Another example, where this may be used, is when certain bits of @var may + * only be modified when holding the appropriate lock, but other bits may still + * be modified concurrently. Writers, where other bits may change concurrently, + * could use the assertion as follows: + * + * .. code-block:: c + * + * spin_lock(&foo_lock); + * ASSERT_EXCLUSIVE_BITS(flags, FOO_MASK); + * old_flags = flags; + * new_flags = (old_flags & ~FOO_MASK) | (new_foo << FOO_SHIFT); + * if (cmpxchg(&flags, old_flags, new_flags) != old_flags) { ... } + * spin_unlock(&foo_lock); + * + * @var: variable to assert on + * @mask: only check for modifications to bits set in @mask + */ +#define ASSERT_EXCLUSIVE_BITS(var, mask) \ + do { \ + kcsan_set_access_mask(mask); \ + __kcsan_check_access(&(var), sizeof(var), KCSAN_ACCESS_ASSERT);\ + kcsan_set_access_mask(0); \ + kcsan_atomic_next(1); \ + } while (0) + +#endif /* _LINUX_KCSAN_CHECKS_H */ diff --git a/include/linux/kcsan.h b/include/linux/kcsan.h new file mode 100644 index 000000000000..53340d8789f9 --- /dev/null +++ b/include/linux/kcsan.h @@ -0,0 +1,59 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _LINUX_KCSAN_H +#define _LINUX_KCSAN_H + +#include <linux/kcsan-checks.h> +#include <linux/types.h> + +#ifdef CONFIG_KCSAN + +/* + * Context for each thread of execution: for tasks, this is stored in + * task_struct, and interrupts access internal per-CPU storage. + */ +struct kcsan_ctx { + int disable_count; /* disable counter */ + int atomic_next; /* number of following atomic ops */ + + /* + * We distinguish between: (a) nestable atomic regions that may contain + * other nestable regions; and (b) flat atomic regions that do not keep + * track of nesting. Both (a) and (b) are entirely independent of each + * other, and a flat region may be started in a nestable region or + * vice-versa. + * + * This is required because, for example, in the annotations for + * seqlocks, we declare seqlock writer critical sections as (a) nestable + * atomic regions, but reader critical sections as (b) flat atomic + * regions, but have encountered cases where seqlock reader critical + * sections are contained within writer critical sections (the opposite + * may be possible, too). + * + * To support these cases, we independently track the depth of nesting + * for (a), and whether the leaf level is flat for (b). + */ + int atomic_nest_count; + bool in_flat_atomic; + + /* + * Access mask for all accesses if non-zero. + */ + unsigned long access_mask; + + /* List of scoped accesses. */ + struct list_head scoped_accesses; +}; + +/** + * kcsan_init - initialize KCSAN runtime + */ +void kcsan_init(void); + +#else /* CONFIG_KCSAN */ + +static inline void kcsan_init(void) { } + +#endif /* CONFIG_KCSAN */ + +#endif /* _LINUX_KCSAN_H */ diff --git a/include/linux/key.h b/include/linux/key.h index 6cf8e71cf8b7..0f2e24f13c2b 100644 --- a/include/linux/key.h +++ b/include/linux/key.h @@ -71,6 +71,23 @@ struct net; #define KEY_PERM_UNDEF 0xffffffff +/* + * The permissions required on a key that we're looking up. + */ +enum key_need_perm { + KEY_NEED_UNSPECIFIED, /* Needed permission unspecified */ + KEY_NEED_VIEW, /* Require permission to view attributes */ + KEY_NEED_READ, /* Require permission to read content */ + KEY_NEED_WRITE, /* Require permission to update / modify */ + KEY_NEED_SEARCH, /* Require permission to search (keyring) or find (key) */ + KEY_NEED_LINK, /* Require permission to link */ + KEY_NEED_SETATTR, /* Require permission to change attributes */ + KEY_NEED_UNLINK, /* Require permission to unlink key */ + KEY_SYSADMIN_OVERRIDE, /* Special: override by CAP_SYS_ADMIN */ + KEY_AUTHTOKEN_OVERRIDE, /* Special: override by possession of auth token */ + KEY_DEFER_PERM_CHECK, /* Special: permission check is deferred */ +}; + struct seq_file; struct user_struct; struct signal_struct; @@ -176,6 +193,9 @@ struct key { struct list_head graveyard_link; struct rb_node serial_node; }; +#ifdef CONFIG_KEY_NOTIFICATIONS + struct watch_list *watchers; /* Entities watching this key for changes */ +#endif struct rw_semaphore sem; /* change vs change sem */ struct key_user *user; /* owner of this key */ void *security; /* security data for this key */ @@ -417,20 +437,9 @@ static inline key_serial_t key_serial(const struct key *key) extern void key_set_timeout(struct key *, unsigned); extern key_ref_t lookup_user_key(key_serial_t id, unsigned long flags, - key_perm_t perm); + enum key_need_perm need_perm); extern void key_free_user_ns(struct user_namespace *); -/* - * The permissions required on a key that we're looking up. - */ -#define KEY_NEED_VIEW 0x01 /* Require permission to view attributes */ -#define KEY_NEED_READ 0x02 /* Require permission to read content */ -#define KEY_NEED_WRITE 0x04 /* Require permission to update / modify */ -#define KEY_NEED_SEARCH 0x08 /* Require permission to search (keyring) or find (key) */ -#define KEY_NEED_LINK 0x10 /* Require permission to link */ -#define KEY_NEED_SETATTR 0x20 /* Require permission to change attributes */ -#define KEY_NEED_ALL 0x3f /* All the above permissions */ - static inline short key_read_state(const struct key *key) { /* Barrier versus mark_key_instantiated(). */ diff --git a/include/linux/kthread.h b/include/linux/kthread.h index 8bbcaad7ef0f..65b81e0c494d 100644 --- a/include/linux/kthread.h +++ b/include/linux/kthread.h @@ -5,6 +5,8 @@ #include <linux/err.h> #include <linux/sched.h> +struct mm_struct; + __printf(4, 5) struct task_struct *kthread_create_on_node(int (*threadfn)(void *data), void *data, @@ -57,6 +59,7 @@ bool kthread_should_stop(void); bool kthread_should_park(void); bool __kthread_should_park(struct task_struct *k); bool kthread_freezable_should_stop(bool *was_frozen); +void *kthread_func(struct task_struct *k); void *kthread_data(struct task_struct *k); void *kthread_probe_data(struct task_struct *k); int kthread_park(struct task_struct *k); @@ -198,6 +201,9 @@ bool kthread_cancel_delayed_work_sync(struct kthread_delayed_work *work); void kthread_destroy_worker(struct kthread_worker *worker); +void kthread_use_mm(struct mm_struct *mm); +void kthread_unuse_mm(struct mm_struct *mm); + struct cgroup_subsys_state; #ifdef CONFIG_BLK_CGROUP diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h index f43b59b1294c..62ec926c78a0 100644 --- a/include/linux/kvm_host.h +++ b/include/linux/kvm_host.h @@ -206,6 +206,7 @@ struct kvm_async_pf { unsigned long addr; struct kvm_arch_async_pf arch; bool wakeup_all; + bool notpresent_injected; }; void kvm_clear_async_pf_completion_queue(struct kvm_vcpu *vcpu); @@ -318,7 +319,6 @@ struct kvm_vcpu { bool preempted; bool ready; struct kvm_vcpu_arch arch; - struct dentry *debugfs_dentry; }; static inline int kvm_vcpu_exiting_guest_mode(struct kvm_vcpu *vcpu) @@ -888,7 +888,7 @@ void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu); void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu); #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS -void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu); +void kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu, struct dentry *debugfs_dentry); #endif int kvm_arch_hardware_enable(void); @@ -1421,8 +1421,8 @@ static inline long kvm_arch_vcpu_async_ioctl(struct file *filp, } #endif /* CONFIG_HAVE_KVM_VCPU_ASYNC_IOCTL */ -int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, - unsigned long start, unsigned long end, bool blockable); +void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, + unsigned long start, unsigned long end); #ifdef CONFIG_HAVE_KVM_VCPU_RUN_PID_CHANGE int kvm_arch_vcpu_run_pid_change(struct kvm_vcpu *vcpu); diff --git a/include/linux/lsm_audit.h b/include/linux/lsm_audit.h index 99d629fd9944..28f23b341c1c 100644 --- a/include/linux/lsm_audit.h +++ b/include/linux/lsm_audit.h @@ -75,6 +75,7 @@ struct common_audit_data { #define LSM_AUDIT_DATA_IBPKEY 13 #define LSM_AUDIT_DATA_IBENDPORT 14 #define LSM_AUDIT_DATA_LOCKDOWN 15 +#define LSM_AUDIT_DATA_NOTIFICATION 16 union { struct path path; struct dentry *dentry; diff --git a/include/linux/lsm_hook_defs.h b/include/linux/lsm_hook_defs.h index fb3ce6cec997..4a3d70baa7f1 100644 --- a/include/linux/lsm_hook_defs.h +++ b/include/linux/lsm_hook_defs.h @@ -254,6 +254,15 @@ LSM_HOOK(int, 0, inode_setsecctx, struct dentry *dentry, void *ctx, u32 ctxlen) LSM_HOOK(int, 0, inode_getsecctx, struct inode *inode, void **ctx, u32 *ctxlen) +#if defined(CONFIG_SECURITY) && defined(CONFIG_WATCH_QUEUE) +LSM_HOOK(int, 0, post_notification, const struct cred *w_cred, + const struct cred *cred, struct watch_notification *n) +#endif /* CONFIG_SECURITY && CONFIG_WATCH_QUEUE */ + +#if defined(CONFIG_SECURITY) && defined(CONFIG_KEY_NOTIFICATIONS) +LSM_HOOK(int, 0, watch_key, struct key *key) +#endif /* CONFIG_SECURITY && CONFIG_KEY_NOTIFICATIONS */ + #ifdef CONFIG_SECURITY_NETWORK LSM_HOOK(int, 0, unix_stream_connect, struct sock *sock, struct sock *other, struct sock *newsk) diff --git a/include/linux/lsm_hooks.h b/include/linux/lsm_hooks.h index 3e62dab77699..73d87955e26b 100644 --- a/include/linux/lsm_hooks.h +++ b/include/linux/lsm_hooks.h @@ -1445,6 +1445,20 @@ * @ctx is a pointer in which to place the allocated security context. * @ctxlen points to the place to put the length of @ctx. * + * Security hooks for the general notification queue: + * + * @post_notification: + * Check to see if a watch notification can be posted to a particular + * queue. + * @w_cred: The credentials of the whoever set the watch. + * @cred: The event-triggerer's credentials + * @n: The notification being posted + * + * @watch_key: + * Check to see if a process is allowed to watch for event notifications + * from a key or keyring. + * @key: The key to watch. + * * Security hooks for using the eBPF maps and programs functionalities through * eBPF syscalls. * diff --git a/include/linux/mmu_context.h b/include/linux/mmu_context.h index d9a543a9e1cc..c51a84132d7c 100644 --- a/include/linux/mmu_context.h +++ b/include/linux/mmu_context.h @@ -4,11 +4,6 @@ #include <asm/mmu_context.h> -struct mm_struct; - -void use_mm(struct mm_struct *mm); -void unuse_mm(struct mm_struct *mm); - /* Architectures that care about IRQ state in switch_mm can override this. */ #ifndef switch_mm_irqs_off # define switch_mm_irqs_off switch_mm diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index df1f08486d81..c4c37fd12104 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -660,9 +660,21 @@ struct deferred_split { * per-zone basis. */ typedef struct pglist_data { + /* + * node_zones contains just the zones for THIS node. Not all of the + * zones may be populated, but it is the full list. It is referenced by + * this node's node_zonelists as well as other node's node_zonelists. + */ struct zone node_zones[MAX_NR_ZONES]; + + /* + * node_zonelists contains references to all zones in all nodes. + * Generally the first zones will be references to this node's + * node_zones. + */ struct zonelist node_zonelists[MAX_ZONELISTS]; - int nr_zones; + + int nr_zones; /* number of populated zones in this node */ #ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */ struct page *node_mem_map; #ifdef CONFIG_PAGE_EXTENSION diff --git a/include/linux/nfs4.h b/include/linux/nfs4.h index 82d8fb422092..4dba3c948932 100644 --- a/include/linux/nfs4.h +++ b/include/linux/nfs4.h @@ -38,7 +38,7 @@ struct nfs4_ace { struct nfs4_acl { uint32_t naces; - struct nfs4_ace aces[0]; + struct nfs4_ace aces[]; }; #define NFS4_MAXLABELLEN 2048 @@ -295,7 +295,7 @@ static inline bool seqid_mutating_err(u32 err) case NFS4ERR_NOFILEHANDLE: case NFS4ERR_MOVED: return false; - }; + } return true; } diff --git a/include/linux/nfs_fs.h b/include/linux/nfs_fs.h index 73eda45f1cfd..6ee9119acc5d 100644 --- a/include/linux/nfs_fs.h +++ b/include/linux/nfs_fs.h @@ -230,6 +230,7 @@ struct nfs4_copy_state { #define NFS_INO_INVALID_OTHER BIT(12) /* other attrs are invalid */ #define NFS_INO_DATA_INVAL_DEFER \ BIT(13) /* Deferred cache invalidation */ +#define NFS_INO_INVALID_BLOCKS BIT(14) /* cached blocks are invalid */ #define NFS_INO_INVALID_ATTR (NFS_INO_INVALID_CHANGE \ | NFS_INO_INVALID_CTIME \ diff --git a/include/linux/nfs_xdr.h b/include/linux/nfs_xdr.h index e5f3e7d8d3d5..5fd0a9ef425f 100644 --- a/include/linux/nfs_xdr.h +++ b/include/linux/nfs_xdr.h @@ -1227,7 +1227,7 @@ struct nfs4_secinfo4 { struct nfs4_secinfo_flavors { unsigned int num_flavors; - struct nfs4_secinfo4 flavors[0]; + struct nfs4_secinfo4 flavors[]; }; struct nfs4_secinfo_arg { diff --git a/include/linux/pci_ids.h b/include/linux/pci_ids.h index 9a57e6717e5c..0ad57693f392 100644 --- a/include/linux/pci_ids.h +++ b/include/linux/pci_ids.h @@ -550,6 +550,7 @@ #define PCI_DEVICE_ID_AMD_17H_DF_F3 0x1463 #define PCI_DEVICE_ID_AMD_17H_M10H_DF_F3 0x15eb #define PCI_DEVICE_ID_AMD_17H_M30H_DF_F3 0x1493 +#define PCI_DEVICE_ID_AMD_17H_M60H_DF_F3 0x144b #define PCI_DEVICE_ID_AMD_17H_M70H_DF_F3 0x1443 #define PCI_DEVICE_ID_AMD_19H_DF_F3 0x1653 #define PCI_DEVICE_ID_AMD_CNB17H_F3 0x1703 diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h index 0c31b9461262..50afd0d0084c 100644 --- a/include/linux/pipe_fs_i.h +++ b/include/linux/pipe_fs_i.h @@ -9,6 +9,10 @@ #define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */ #define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */ #define PIPE_BUF_FLAG_CAN_MERGE 0x10 /* can merge buffers */ +#define PIPE_BUF_FLAG_WHOLE 0x20 /* read() must return entire buffer or error */ +#ifdef CONFIG_WATCH_QUEUE +#define PIPE_BUF_FLAG_LOSS 0x40 /* Message loss happened after this buffer */ +#endif /** * struct pipe_buffer - a linux kernel pipe buffer @@ -34,8 +38,10 @@ struct pipe_buffer { * @wr_wait: writer wait point in case of full pipe * @head: The point of buffer production * @tail: The point of buffer consumption + * @note_loss: The next read() should insert a data-lost message * @max_usage: The maximum number of slots that may be used in the ring * @ring_size: total number of buffers (should be a power of 2) + * @nr_accounted: The amount this pipe accounts for in user->pipe_bufs * @tmp_page: cached released page * @readers: number of current readers of this pipe * @writers: number of current writers of this pipe @@ -46,6 +52,7 @@ struct pipe_buffer { * @fasync_writers: writer side fasync * @bufs: the circular array of pipe buffers * @user: the user who created this pipe + * @watch_queue: If this pipe is a watch_queue, this is the stuff for that **/ struct pipe_inode_info { struct mutex mutex; @@ -54,6 +61,10 @@ struct pipe_inode_info { unsigned int tail; unsigned int max_usage; unsigned int ring_size; +#ifdef CONFIG_WATCH_QUEUE + bool note_loss; +#endif + unsigned int nr_accounted; unsigned int readers; unsigned int writers; unsigned int files; @@ -64,6 +75,9 @@ struct pipe_inode_info { struct fasync_struct *fasync_writers; struct pipe_buffer *bufs; struct user_struct *user; +#ifdef CONFIG_WATCH_QUEUE + struct watch_queue *watch_queue; +#endif }; /* @@ -239,9 +253,20 @@ void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *); extern const struct pipe_buf_operations nosteal_pipe_buf_ops; +#ifdef CONFIG_WATCH_QUEUE +unsigned long account_pipe_buffers(struct user_struct *user, + unsigned long old, unsigned long new); +bool too_many_pipe_buffers_soft(unsigned long user_bufs); +bool too_many_pipe_buffers_hard(unsigned long user_bufs); +bool pipe_is_unprivileged_user(void); +#endif + /* for F_SETPIPE_SZ and F_GETPIPE_SZ */ +#ifdef CONFIG_WATCH_QUEUE +int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots); +#endif long pipe_fcntl(struct file *, unsigned int, unsigned long arg); -struct pipe_inode_info *get_pipe_info(struct file *file); +struct pipe_inode_info *get_pipe_info(struct file *file, bool for_splice); int create_pipe_files(struct file **, int); unsigned int round_pipe_size(unsigned long size); diff --git a/include/linux/platform_data/i2c-pxa.h b/include/linux/platform_data/i2c-pxa.h index 6a9b28399b39..24953981bd9f 100644 --- a/include/linux/platform_data/i2c-pxa.h +++ b/include/linux/platform_data/i2c-pxa.h @@ -7,54 +7,6 @@ #ifndef _I2C_PXA_H_ #define _I2C_PXA_H_ -#if 0 -#define DEF_TIMEOUT 3 -#else -/* need a longer timeout if we're dealing with the fact we may well be - * looking at a multi-master environment -*/ -#define DEF_TIMEOUT 32 -#endif - -#define BUS_ERROR (-EREMOTEIO) -#define XFER_NAKED (-ECONNREFUSED) -#define I2C_RETRY (-2000) /* an error has occurred retry transmit */ - -/* ICR initialize bit values -* -* 15. FM 0 (100 Khz operation) -* 14. UR 0 (No unit reset) -* 13. SADIE 0 (Disables the unit from interrupting on slave addresses -* matching its slave address) -* 12. ALDIE 0 (Disables the unit from interrupt when it loses arbitration -* in master mode) -* 11. SSDIE 0 (Disables interrupts from a slave stop detected, in slave mode) -* 10. BEIE 1 (Enable interrupts from detected bus errors, no ACK sent) -* 9. IRFIE 1 (Enable interrupts from full buffer received) -* 8. ITEIE 1 (Enables the I2C unit to interrupt when transmit buffer empty) -* 7. GCD 1 (Disables i2c unit response to general call messages as a slave) -* 6. IUE 0 (Disable unit until we change settings) -* 5. SCLE 1 (Enables the i2c clock output for master mode (drives SCL) -* 4. MA 0 (Only send stop with the ICR stop bit) -* 3. TB 0 (We are not transmitting a byte initially) -* 2. ACKNAK 0 (Send an ACK after the unit receives a byte) -* 1. STOP 0 (Do not send a STOP) -* 0. START 0 (Do not send a START) -* -*/ -#define I2C_ICR_INIT (ICR_BEIE | ICR_IRFIE | ICR_ITEIE | ICR_GCD | ICR_SCLE) - -/* I2C status register init values - * - * 10. BED 1 (Clear bus error detected) - * 9. SAD 1 (Clear slave address detected) - * 7. IRF 1 (Clear IDBR Receive Full) - * 6. ITE 1 (Clear IDBR Transmit Empty) - * 5. ALD 1 (Clear Arbitration Loss Detected) - * 4. SSD 1 (Clear Slave Stop Detected) - */ -#define I2C_ISR_INIT 0x7FF /* status register init */ - struct i2c_pxa_platform_data { unsigned int class; unsigned int use_pio :1; diff --git a/include/linux/ras.h b/include/linux/ras.h index 7c3debb47c87..1f4048bf2674 100644 --- a/include/linux/ras.h +++ b/include/linux/ras.h @@ -17,12 +17,7 @@ static inline int ras_add_daemon_trace(void) { return 0; } #endif #ifdef CONFIG_RAS_CEC -void __init cec_init(void); int __init parse_cec_param(char *str); -int cec_add_elem(u64 pfn); -#else -static inline void __init cec_init(void) { } -static inline int cec_add_elem(u64 pfn) { return -ENODEV; } #endif #ifdef CONFIG_RAS diff --git a/include/linux/sched.h b/include/linux/sched.h index c5d96e3e7fff..b62e6aaf28f0 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -31,6 +31,7 @@ #include <linux/task_io_accounting.h> #include <linux/posix-timers.h> #include <linux/rseq.h> +#include <linux/kcsan.h> /* task_struct member predeclarations (sorted alphabetically): */ struct audit_context; @@ -1197,6 +1198,9 @@ struct task_struct { #ifdef CONFIG_KASAN unsigned int kasan_depth; #endif +#ifdef CONFIG_KCSAN + struct kcsan_ctx kcsan_ctx; +#endif #ifdef CONFIG_FUNCTION_GRAPH_TRACER /* Index of current stored address in ret_stack: */ @@ -1304,7 +1308,9 @@ struct task_struct { #ifdef CONFIG_X86_MCE u64 mce_addr; - u64 mce_status; + __u64 mce_ripv : 1, + mce_whole_page : 1, + __mce_reserved : 62; struct callback_head mce_kill_me; #endif diff --git a/include/linux/security.h b/include/linux/security.h index b3f2cb21b4f2..469fa91f8cf8 100644 --- a/include/linux/security.h +++ b/include/linux/security.h @@ -56,6 +56,8 @@ struct mm_struct; struct fs_context; struct fs_parameter; enum fs_value_type; +struct watch; +struct watch_notification; /* Default (no) options for the capable function */ #define CAP_OPT_NONE 0x0 @@ -1282,6 +1284,28 @@ static inline int security_locked_down(enum lockdown_reason what) } #endif /* CONFIG_SECURITY */ +#if defined(CONFIG_SECURITY) && defined(CONFIG_WATCH_QUEUE) +int security_post_notification(const struct cred *w_cred, + const struct cred *cred, + struct watch_notification *n); +#else +static inline int security_post_notification(const struct cred *w_cred, + const struct cred *cred, + struct watch_notification *n) +{ + return 0; +} +#endif + +#if defined(CONFIG_SECURITY) && defined(CONFIG_KEY_NOTIFICATIONS) +int security_watch_key(struct key *key); +#else +static inline int security_watch_key(struct key *key) +{ + return 0; +} +#endif + #ifdef CONFIG_SECURITY_NETWORK int security_unix_stream_connect(struct sock *sock, struct sock *other, struct sock *newsk); @@ -1750,8 +1774,8 @@ static inline int security_path_chroot(const struct path *path) int security_key_alloc(struct key *key, const struct cred *cred, unsigned long flags); void security_key_free(struct key *key); -int security_key_permission(key_ref_t key_ref, - const struct cred *cred, unsigned perm); +int security_key_permission(key_ref_t key_ref, const struct cred *cred, + enum key_need_perm need_perm); int security_key_getsecurity(struct key *key, char **_buffer); #else @@ -1769,7 +1793,7 @@ static inline void security_key_free(struct key *key) static inline int security_key_permission(key_ref_t key_ref, const struct cred *cred, - unsigned perm) + enum key_need_perm need_perm) { return 0; } diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index 0491d963d47e..8b97204f35a7 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -37,9 +37,25 @@ #include <linux/preempt.h> #include <linux/lockdep.h> #include <linux/compiler.h> +#include <linux/kcsan-checks.h> #include <asm/processor.h> /* + * The seqlock interface does not prescribe a precise sequence of read + * begin/retry/end. For readers, typically there is a call to + * read_seqcount_begin() and read_seqcount_retry(), however, there are more + * esoteric cases which do not follow this pattern. + * + * As a consequence, we take the following best-effort approach for raw usage + * via seqcount_t under KCSAN: upon beginning a seq-reader critical section, + * pessimistically mark the next KCSAN_SEQLOCK_REGION_MAX memory accesses as + * atomics; if there is a matching read_seqcount_retry() call, no following + * memory operations are considered atomic. Usage of seqlocks via seqlock_t + * interface is not affected. + */ +#define KCSAN_SEQLOCK_REGION_MAX 1000 + +/* * Version using sequence counter only. * This can be used when code has its own mutex protecting the * updating starting before the write_seqcountbeqin() and ending @@ -115,6 +131,7 @@ repeat: cpu_relax(); goto repeat; } + kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); return ret; } @@ -131,6 +148,7 @@ static inline unsigned raw_read_seqcount(const seqcount_t *s) { unsigned ret = READ_ONCE(s->sequence); smp_rmb(); + kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); return ret; } @@ -183,6 +201,7 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) { unsigned ret = READ_ONCE(s->sequence); smp_rmb(); + kcsan_atomic_next(KCSAN_SEQLOCK_REGION_MAX); return ret & ~1; } @@ -202,7 +221,8 @@ static inline unsigned raw_seqcount_begin(const seqcount_t *s) */ static inline int __read_seqcount_retry(const seqcount_t *s, unsigned start) { - return unlikely(s->sequence != start); + kcsan_atomic_next(0); + return unlikely(READ_ONCE(s->sequence) != start); } /** @@ -225,6 +245,7 @@ static inline int read_seqcount_retry(const seqcount_t *s, unsigned start) static inline void raw_write_seqcount_begin(seqcount_t *s) { + kcsan_nestable_atomic_begin(); s->sequence++; smp_wmb(); } @@ -233,6 +254,7 @@ static inline void raw_write_seqcount_end(seqcount_t *s) { smp_wmb(); s->sequence++; + kcsan_nestable_atomic_end(); } /** @@ -243,6 +265,13 @@ static inline void raw_write_seqcount_end(seqcount_t *s) * usual consistency guarantee. It is one wmb cheaper, because we can * collapse the two back-to-back wmb()s. * + * Note that writes surrounding the barrier should be declared atomic (e.g. + * via WRITE_ONCE): a) to ensure the writes become visible to other threads + * atomically, avoiding compiler optimizations; b) to document which writes are + * meant to propagate to the reader critical section. This is necessary because + * neither writes before and after the barrier are enclosed in a seq-writer + * critical section that would ensure readers are aware of ongoing writes. + * * seqcount_t seq; * bool X = true, Y = false; * @@ -262,18 +291,20 @@ static inline void raw_write_seqcount_end(seqcount_t *s) * * void write(void) * { - * Y = true; + * WRITE_ONCE(Y, true); * * raw_write_seqcount_barrier(seq); * - * X = false; + * WRITE_ONCE(X, false); * } */ static inline void raw_write_seqcount_barrier(seqcount_t *s) { + kcsan_nestable_atomic_begin(); s->sequence++; smp_wmb(); s->sequence++; + kcsan_nestable_atomic_end(); } static inline int raw_read_seqcount_latch(seqcount_t *s) @@ -398,7 +429,9 @@ static inline void write_seqcount_end(seqcount_t *s) static inline void write_seqcount_invalidate(seqcount_t *s) { smp_wmb(); + kcsan_nestable_atomic_begin(); s->sequence+=2; + kcsan_nestable_atomic_end(); } typedef struct { @@ -430,11 +463,21 @@ typedef struct { */ static inline unsigned read_seqbegin(const seqlock_t *sl) { - return read_seqcount_begin(&sl->seqcount); + unsigned ret = read_seqcount_begin(&sl->seqcount); + + kcsan_atomic_next(0); /* non-raw usage, assume closing read_seqretry() */ + kcsan_flat_atomic_begin(); + return ret; } static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start) { + /* + * Assume not nested: read_seqretry() may be called multiple times when + * completing read critical section. + */ + kcsan_flat_atomic_end(); + return read_seqcount_retry(&sl->seqcount, start); } diff --git a/include/linux/set_memory.h b/include/linux/set_memory.h index 86281ac7c305..860e0f843c12 100644 --- a/include/linux/set_memory.h +++ b/include/linux/set_memory.h @@ -26,7 +26,7 @@ static inline int set_direct_map_default_noflush(struct page *page) #endif #ifndef set_mce_nospec -static inline int set_mce_nospec(unsigned long pfn) +static inline int set_mce_nospec(unsigned long pfn, bool unmap) { return 0; } diff --git a/include/linux/stacktrace.h b/include/linux/stacktrace.h index 83bd8cb475d7..b7af8cc13eda 100644 --- a/include/linux/stacktrace.h +++ b/include/linux/stacktrace.h @@ -64,7 +64,7 @@ void arch_stack_walk_user(stack_trace_consume_fn consume_entry, void *cookie, struct stack_trace { unsigned int nr_entries, max_entries; unsigned long *entries; - int skip; /* input argument: How many entries to skip */ + unsigned int skip; /* input argument: How many entries to skip */ }; extern void save_stack_trace(struct stack_trace *trace); diff --git a/include/linux/sunrpc/auth.h b/include/linux/sunrpc/auth.h index 4f6b28487f28..98da816b5fc2 100644 --- a/include/linux/sunrpc/auth.h +++ b/include/linux/sunrpc/auth.h @@ -76,7 +76,7 @@ struct rpc_auth { unsigned int au_verfsize; /* size of reply verifier */ unsigned int au_ralign; /* words before UL header */ - unsigned int au_flags; + unsigned long au_flags; const struct rpc_authops *au_ops; rpc_authflavor_t au_flavor; /* pseudoflavor (note may * differ from the flavor in @@ -89,7 +89,8 @@ struct rpc_auth { }; /* rpc_auth au_flags */ -#define RPCAUTH_AUTH_DATATOUCH 0x00000002 +#define RPCAUTH_AUTH_DATATOUCH (1) +#define RPCAUTH_AUTH_UPDATE_SLACK (2) struct rpc_auth_create_args { rpc_authflavor_t pseudoflavor; diff --git a/include/linux/sunrpc/gss_api.h b/include/linux/sunrpc/gss_api.h index bc07e51f20d1..bf4ac8a0268c 100644 --- a/include/linux/sunrpc/gss_api.h +++ b/include/linux/sunrpc/gss_api.h @@ -84,6 +84,7 @@ struct pf_desc { u32 service; char *name; char *auth_domain_name; + struct auth_domain *domain; bool datatouch; }; diff --git a/include/linux/sunrpc/svc.h b/include/linux/sunrpc/svc.h index fd390894a584..386628b36bc7 100644 --- a/include/linux/sunrpc/svc.h +++ b/include/linux/sunrpc/svc.h @@ -254,6 +254,7 @@ struct svc_rqst { struct page * *rq_page_end; /* one past the last page */ struct kvec rq_vec[RPCSVC_MAXPAGES]; /* generally useful.. */ + struct bio_vec rq_bvec[RPCSVC_MAXPAGES]; __be32 rq_xid; /* transmission id */ u32 rq_prog; /* program number */ @@ -299,6 +300,7 @@ struct svc_rqst { struct net *rq_bc_net; /* pointer to backchannel's * net namespace */ + void ** rq_lease_breaker; /* The v4 client breaking a lease */ }; #define SVC_NET(rqst) (rqst->rq_xprt ? rqst->rq_xprt->xpt_net : rqst->rq_bc_net) diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index cbcfbd0521e3..7ed82625dc0b 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -48,7 +48,6 @@ #include <linux/sunrpc/rpc_rdma.h> #include <rdma/ib_verbs.h> #include <rdma/rdma_cm.h> -#define SVCRDMA_DEBUG /* Default and maximum inline threshold sizes */ enum { @@ -160,9 +159,8 @@ struct svc_rdma_send_ctxt { }; /* svc_rdma_backchannel.c */ -extern int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, - __be32 *rdma_resp, - struct xdr_buf *rcvbuf); +extern void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp, + struct svc_rdma_recv_ctxt *rctxt); /* svc_rdma_recvfrom.c */ extern void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma); diff --git a/include/linux/sunrpc/svc_xprt.h b/include/linux/sunrpc/svc_xprt.h index 9e1e046de176..aca35ab5cff2 100644 --- a/include/linux/sunrpc/svc_xprt.h +++ b/include/linux/sunrpc/svc_xprt.h @@ -117,6 +117,12 @@ static inline int register_xpt_user(struct svc_xprt *xpt, struct svc_xpt_user *u return 0; } +static inline bool svc_xprt_is_dead(const struct svc_xprt *xprt) +{ + return (test_bit(XPT_DEAD, &xprt->xpt_flags) != 0) || + (test_bit(XPT_CLOSE, &xprt->xpt_flags) != 0); +} + int svc_reg_xprt_class(struct svc_xprt_class *); void svc_unreg_xprt_class(struct svc_xprt_class *); void svc_xprt_init(struct net *, struct svc_xprt_class *, struct svc_xprt *, diff --git a/include/linux/sunrpc/svcauth_gss.h b/include/linux/sunrpc/svcauth_gss.h index ca39a388dc22..f09c82b0a7ae 100644 --- a/include/linux/sunrpc/svcauth_gss.h +++ b/include/linux/sunrpc/svcauth_gss.h @@ -20,7 +20,8 @@ int gss_svc_init(void); void gss_svc_shutdown(void); int gss_svc_init_net(struct net *net); void gss_svc_shutdown_net(struct net *net); -int svcauth_gss_register_pseudoflavor(u32 pseudoflavor, char * name); +struct auth_domain *svcauth_gss_register_pseudoflavor(u32 pseudoflavor, + char *name); u32 svcauth_gss_flavor(struct auth_domain *dom); #endif /* _LINUX_SUNRPC_SVCAUTH_GSS_H */ diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index 771baadaee9d..b7ac7fe68306 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -28,7 +28,7 @@ struct svc_sock { /* private TCP part */ /* On-the-wire fragment header: */ - __be32 sk_reclen; + __be32 sk_marker; /* As we receive a record, this includes the length received so * far (including the fragment header): */ u32 sk_tcplen; @@ -41,12 +41,12 @@ struct svc_sock { static inline u32 svc_sock_reclen(struct svc_sock *svsk) { - return ntohl(svsk->sk_reclen) & RPC_FRAGMENT_SIZE_MASK; + return be32_to_cpu(svsk->sk_marker) & RPC_FRAGMENT_SIZE_MASK; } static inline u32 svc_sock_final_rec(struct svc_sock *svsk) { - return ntohl(svsk->sk_reclen) & RPC_LAST_STREAM_FRAGMENT; + return be32_to_cpu(svsk->sk_marker) & RPC_LAST_STREAM_FRAGMENT; } /* diff --git a/include/linux/thermal.h b/include/linux/thermal.h index c91b1e344d56..216185bb3014 100644 --- a/include/linux/thermal.h +++ b/include/linux/thermal.h @@ -32,20 +32,10 @@ /* use value, which < 0K, to indicate an invalid/uninitialized temperature */ #define THERMAL_TEMP_INVALID -274000 -/* Default Thermal Governor */ -#if defined(CONFIG_THERMAL_DEFAULT_GOV_STEP_WISE) -#define DEFAULT_THERMAL_GOVERNOR "step_wise" -#elif defined(CONFIG_THERMAL_DEFAULT_GOV_FAIR_SHARE) -#define DEFAULT_THERMAL_GOVERNOR "fair_share" -#elif defined(CONFIG_THERMAL_DEFAULT_GOV_USER_SPACE) -#define DEFAULT_THERMAL_GOVERNOR "user_space" -#elif defined(CONFIG_THERMAL_DEFAULT_GOV_POWER_ALLOCATOR) -#define DEFAULT_THERMAL_GOVERNOR "power_allocator" -#endif - struct thermal_zone_device; struct thermal_cooling_device; struct thermal_instance; +struct thermal_attr; enum thermal_device_mode { THERMAL_DEVICE_DISABLED = 0, @@ -130,11 +120,6 @@ struct thermal_cooling_device { struct list_head node; }; -struct thermal_attr { - struct device_attribute attr; - char name[THERMAL_NAME_LENGTH]; -}; - /** * struct thermal_zone_device - structure for a thermal zone * @id: unique id number for each thermal zone @@ -347,21 +332,6 @@ struct thermal_zone_of_device_ops { int (*set_trip_temp)(void *, int, int); }; -/** - * struct thermal_trip - representation of a point in temperature domain - * @np: pointer to struct device_node that this trip point was created from - * @temperature: temperature value in miliCelsius - * @hysteresis: relative hysteresis in miliCelsius - * @type: trip point type - */ - -struct thermal_trip { - struct device_node *np; - int temperature; - int hysteresis; - enum thermal_trip_type type; -}; - /* Function declarations */ #ifdef CONFIG_THERMAL_OF int thermal_zone_of_get_sensor_id(struct device_node *tz_np, @@ -413,19 +383,7 @@ void devm_thermal_zone_of_sensor_unregister(struct device *dev, #endif -#if IS_ENABLED(CONFIG_THERMAL) -static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) -{ - return cdev->ops->get_requested_power && cdev->ops->state2power && - cdev->ops->power2state; -} - -int power_actor_get_max_power(struct thermal_cooling_device *, - struct thermal_zone_device *tz, u32 *max_power); -int power_actor_get_min_power(struct thermal_cooling_device *, - struct thermal_zone_device *tz, u32 *min_power); -int power_actor_set_power(struct thermal_cooling_device *, - struct thermal_instance *, u32); +#ifdef CONFIG_THERMAL struct thermal_zone_device *thermal_zone_device_register(const char *, int, int, void *, struct thermal_zone_device_ops *, struct thermal_zone_params *, int, int); @@ -439,7 +397,6 @@ int thermal_zone_unbind_cooling_device(struct thermal_zone_device *, int, struct thermal_cooling_device *); void thermal_zone_device_update(struct thermal_zone_device *, enum thermal_notify_event); -void thermal_zone_set_trips(struct thermal_zone_device *); struct thermal_cooling_device *thermal_cooling_device_register(const char *, void *, const struct thermal_cooling_device_ops *); @@ -457,24 +414,9 @@ int thermal_zone_get_temp(struct thermal_zone_device *tz, int *temp); int thermal_zone_get_slope(struct thermal_zone_device *tz); int thermal_zone_get_offset(struct thermal_zone_device *tz); -int get_tz_trend(struct thermal_zone_device *, int); -struct thermal_instance *get_thermal_instance(struct thermal_zone_device *, - struct thermal_cooling_device *, int); void thermal_cdev_update(struct thermal_cooling_device *); void thermal_notify_framework(struct thermal_zone_device *, int); #else -static inline bool cdev_is_power_actor(struct thermal_cooling_device *cdev) -{ return false; } -static inline int power_actor_get_max_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, u32 *max_power) -{ return 0; } -static inline int power_actor_get_min_power(struct thermal_cooling_device *cdev, - struct thermal_zone_device *tz, - u32 *min_power) -{ return -ENODEV; } -static inline int power_actor_set_power(struct thermal_cooling_device *cdev, - struct thermal_instance *tz, u32 power) -{ return 0; } static inline struct thermal_zone_device *thermal_zone_device_register( const char *type, int trips, int mask, void *devdata, struct thermal_zone_device_ops *ops, @@ -484,21 +426,6 @@ static inline struct thermal_zone_device *thermal_zone_device_register( static inline void thermal_zone_device_unregister( struct thermal_zone_device *tz) { } -static inline int thermal_zone_bind_cooling_device( - struct thermal_zone_device *tz, int trip, - struct thermal_cooling_device *cdev, - unsigned long upper, unsigned long lower, - unsigned int weight) -{ return -ENODEV; } -static inline int thermal_zone_unbind_cooling_device( - struct thermal_zone_device *tz, int trip, - struct thermal_cooling_device *cdev) -{ return -ENODEV; } -static inline void thermal_zone_device_update(struct thermal_zone_device *tz, - enum thermal_notify_event event) -{ } -static inline void thermal_zone_set_trips(struct thermal_zone_device *tz) -{ } static inline struct thermal_cooling_device * thermal_cooling_device_register(char *type, void *devdata, const struct thermal_cooling_device_ops *ops) @@ -530,12 +457,7 @@ static inline int thermal_zone_get_slope( static inline int thermal_zone_get_offset( struct thermal_zone_device *tz) { return -ENODEV; } -static inline int get_tz_trend(struct thermal_zone_device *tz, int trip) -{ return -ENODEV; } -static inline struct thermal_instance * -get_thermal_instance(struct thermal_zone_device *tz, - struct thermal_cooling_device *cdev, int trip) -{ return ERR_PTR(-ENODEV); } + static inline void thermal_cdev_update(struct thermal_cooling_device *cdev) { } static inline void thermal_notify_framework(struct thermal_zone_device *tz, diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index dac1db05bf7e..7bcadca22100 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -2,9 +2,9 @@ #ifndef __LINUX_UACCESS_H__ #define __LINUX_UACCESS_H__ +#include <linux/instrumented.h> #include <linux/sched.h> #include <linux/thread_info.h> -#include <linux/kasan-checks.h> #define uaccess_kernel() segment_eq(get_fs(), KERNEL_DS) @@ -58,7 +58,7 @@ static __always_inline __must_check unsigned long __copy_from_user_inatomic(void *to, const void __user *from, unsigned long n) { - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); check_object_size(to, n, false); return raw_copy_from_user(to, from, n); } @@ -67,7 +67,7 @@ static __always_inline __must_check unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) { might_fault(); - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); check_object_size(to, n, false); return raw_copy_from_user(to, from, n); } @@ -88,7 +88,7 @@ __copy_from_user(void *to, const void __user *from, unsigned long n) static __always_inline __must_check unsigned long __copy_to_user_inatomic(void __user *to, const void *from, unsigned long n) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); check_object_size(from, n, true); return raw_copy_to_user(to, from, n); } @@ -97,7 +97,7 @@ static __always_inline __must_check unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); check_object_size(from, n, true); return raw_copy_to_user(to, from, n); } @@ -109,7 +109,7 @@ _copy_from_user(void *to, const void __user *from, unsigned long n) unsigned long res = n; might_fault(); if (likely(access_ok(from, n))) { - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); res = raw_copy_from_user(to, from, n); } if (unlikely(res)) @@ -127,7 +127,7 @@ _copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); if (access_ok(to, n)) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n; diff --git a/include/linux/watch_queue.h b/include/linux/watch_queue.h new file mode 100644 index 000000000000..5e08db2adc31 --- /dev/null +++ b/include/linux/watch_queue.h @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +/* User-mappable watch queue + * + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * See Documentation/watch_queue.rst + */ + +#ifndef _LINUX_WATCH_QUEUE_H +#define _LINUX_WATCH_QUEUE_H + +#include <uapi/linux/watch_queue.h> +#include <linux/kref.h> +#include <linux/rcupdate.h> + +#ifdef CONFIG_WATCH_QUEUE + +struct cred; + +struct watch_type_filter { + enum watch_notification_type type; + __u32 subtype_filter[1]; /* Bitmask of subtypes to filter on */ + __u32 info_filter; /* Filter on watch_notification::info */ + __u32 info_mask; /* Mask of relevant bits in info_filter */ +}; + +struct watch_filter { + union { + struct rcu_head rcu; + unsigned long type_filter[2]; /* Bitmask of accepted types */ + }; + u32 nr_filters; /* Number of filters */ + struct watch_type_filter filters[]; +}; + +struct watch_queue { + struct rcu_head rcu; + struct watch_filter __rcu *filter; + struct pipe_inode_info *pipe; /* The pipe we're using as a buffer */ + struct hlist_head watches; /* Contributory watches */ + struct page **notes; /* Preallocated notifications */ + unsigned long *notes_bitmap; /* Allocation bitmap for notes */ + struct kref usage; /* Object usage count */ + spinlock_t lock; + unsigned int nr_notes; /* Number of notes */ + unsigned int nr_pages; /* Number of pages in notes[] */ + bool defunct; /* T when queues closed */ +}; + +/* + * Representation of a watch on an object. + */ +struct watch { + union { + struct rcu_head rcu; + u32 info_id; /* ID to be OR'd in to info field */ + }; + struct watch_queue __rcu *queue; /* Queue to post events to */ + struct hlist_node queue_node; /* Link in queue->watches */ + struct watch_list __rcu *watch_list; + struct hlist_node list_node; /* Link in watch_list->watchers */ + const struct cred *cred; /* Creds of the owner of the watch */ + void *private; /* Private data for the watched object */ + u64 id; /* Internal identifier */ + struct kref usage; /* Object usage count */ +}; + +/* + * List of watches on an object. + */ +struct watch_list { + struct rcu_head rcu; + struct hlist_head watchers; + void (*release_watch)(struct watch *); + spinlock_t lock; +}; + +extern void __post_watch_notification(struct watch_list *, + struct watch_notification *, + const struct cred *, + u64); +extern struct watch_queue *get_watch_queue(int); +extern void put_watch_queue(struct watch_queue *); +extern void init_watch(struct watch *, struct watch_queue *); +extern int add_watch_to_object(struct watch *, struct watch_list *); +extern int remove_watch_from_object(struct watch_list *, struct watch_queue *, u64, bool); +extern long watch_queue_set_size(struct pipe_inode_info *, unsigned int); +extern long watch_queue_set_filter(struct pipe_inode_info *, + struct watch_notification_filter __user *); +extern int watch_queue_init(struct pipe_inode_info *); +extern void watch_queue_clear(struct watch_queue *); + +static inline void init_watch_list(struct watch_list *wlist, + void (*release_watch)(struct watch *)) +{ + INIT_HLIST_HEAD(&wlist->watchers); + spin_lock_init(&wlist->lock); + wlist->release_watch = release_watch; +} + +static inline void post_watch_notification(struct watch_list *wlist, + struct watch_notification *n, + const struct cred *cred, + u64 id) +{ + if (unlikely(wlist)) + __post_watch_notification(wlist, n, cred, id); +} + +static inline void remove_watch_list(struct watch_list *wlist, u64 id) +{ + if (wlist) { + remove_watch_from_object(wlist, NULL, id, true); + kfree_rcu(wlist, rcu); + } +} + +/** + * watch_sizeof - Calculate the information part of the size of a watch record, + * given the structure size. + */ +#define watch_sizeof(STRUCT) (sizeof(STRUCT) << WATCH_INFO_LENGTH__SHIFT) + +#endif + +#endif /* _LINUX_WATCH_QUEUE_H */ diff --git a/include/media/videobuf2-dma-contig.h b/include/media/videobuf2-dma-contig.h index 5604818d137e..5be313cbf7d7 100644 --- a/include/media/videobuf2-dma-contig.h +++ b/include/media/videobuf2-dma-contig.h @@ -25,7 +25,7 @@ vb2_dma_contig_plane_dma_addr(struct vb2_buffer *vb, unsigned int plane_no) } int vb2_dma_contig_set_max_seg_size(struct device *dev, unsigned int size); -void vb2_dma_contig_clear_max_seg_size(struct device *dev); +static inline void vb2_dma_contig_clear_max_seg_size(struct device *dev) { } extern const struct vb2_mem_ops vb2_dma_contig_memops; diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h index 063f133e47c2..6adf4d71acf6 100644 --- a/include/target/target_core_fabric.h +++ b/include/target/target_core_fabric.h @@ -150,9 +150,10 @@ void transport_deregister_session(struct se_session *); void transport_init_se_cmd(struct se_cmd *, const struct target_core_fabric_ops *, - struct se_session *, u32, int, int, unsigned char *); -sense_reason_t transport_lookup_cmd_lun(struct se_cmd *, u64); -sense_reason_t target_setup_cmd_from_cdb(struct se_cmd *, unsigned char *); + struct se_session *, u32, int, int, unsigned char *, u64); +sense_reason_t transport_lookup_cmd_lun(struct se_cmd *); +sense_reason_t target_cmd_init_cdb(struct se_cmd *, unsigned char *); +sense_reason_t target_cmd_parse_cdb(struct se_cmd *); int target_submit_cmd_map_sgls(struct se_cmd *, struct se_session *, unsigned char *, unsigned char *, u64, u32, int, int, int, struct scatterlist *, u32, struct scatterlist *, u32, @@ -187,7 +188,7 @@ int core_tmr_alloc_req(struct se_cmd *, void *, u8, gfp_t); void core_tmr_release_req(struct se_tmr_req *); int transport_generic_handle_tmr(struct se_cmd *); void transport_generic_request_failure(struct se_cmd *, sense_reason_t); -int transport_lookup_tmr_lun(struct se_cmd *, u64); +int transport_lookup_tmr_lun(struct se_cmd *); void core_allocate_nexus_loss_ua(struct se_node_acl *acl); struct se_node_acl *core_tpg_get_initiator_node_acl(struct se_portal_group *tpg, diff --git a/include/trace/events/block.h b/include/trace/events/block.h index 81b43f5bdf23..1257f26bb887 100644 --- a/include/trace/events/block.h +++ b/include/trace/events/block.h @@ -261,9 +261,9 @@ TRACE_EVENT(block_bio_bounce, */ TRACE_EVENT(block_bio_complete, - TP_PROTO(struct request_queue *q, struct bio *bio, int error), + TP_PROTO(struct request_queue *q, struct bio *bio), - TP_ARGS(q, bio, error), + TP_ARGS(q, bio), TP_STRUCT__entry( __field( dev_t, dev ) @@ -277,7 +277,7 @@ TRACE_EVENT(block_bio_complete, __entry->dev = bio_dev(bio); __entry->sector = bio->bi_iter.bi_sector; __entry->nr_sector = bio_sectors(bio); - __entry->error = error; + __entry->error = blk_status_to_errno(bio->bi_status); blk_fill_rwbs(__entry->rwbs, bio->bi_opf, bio->bi_iter.bi_size); ), diff --git a/include/trace/events/rpcgss.h b/include/trace/events/rpcgss.h index 32d88c4fb063..b9b51a4b1db1 100644 --- a/include/trace/events/rpcgss.h +++ b/include/trace/events/rpcgss.h @@ -17,6 +17,16 @@ ** GSS-API related trace events **/ +TRACE_DEFINE_ENUM(RPC_GSS_SVC_NONE); +TRACE_DEFINE_ENUM(RPC_GSS_SVC_INTEGRITY); +TRACE_DEFINE_ENUM(RPC_GSS_SVC_PRIVACY); + +#define show_gss_service(x) \ + __print_symbolic(x, \ + { RPC_GSS_SVC_NONE, "none" }, \ + { RPC_GSS_SVC_INTEGRITY, "integrity" }, \ + { RPC_GSS_SVC_PRIVACY, "privacy" }) + TRACE_DEFINE_ENUM(GSS_S_BAD_MECH); TRACE_DEFINE_ENUM(GSS_S_BAD_NAME); TRACE_DEFINE_ENUM(GSS_S_BAD_NAMETYPE); @@ -126,6 +136,40 @@ DEFINE_GSSAPI_EVENT(verify_mic); DEFINE_GSSAPI_EVENT(wrap); DEFINE_GSSAPI_EVENT(unwrap); +DECLARE_EVENT_CLASS(rpcgss_ctx_class, + TP_PROTO( + const struct gss_cred *gc + ), + + TP_ARGS(gc), + + TP_STRUCT__entry( + __field(const void *, cred) + __field(unsigned long, service) + __string(principal, gc->gc_principal) + ), + + TP_fast_assign( + __entry->cred = gc; + __entry->service = gc->gc_service; + __assign_str(principal, gc->gc_principal) + ), + + TP_printk("cred=%p service=%s principal='%s'", + __entry->cred, show_gss_service(__entry->service), + __get_str(principal)) +); + +#define DEFINE_CTX_EVENT(name) \ + DEFINE_EVENT(rpcgss_ctx_class, rpcgss_ctx_##name, \ + TP_PROTO( \ + const struct gss_cred *gc \ + ), \ + TP_ARGS(gc)) + +DEFINE_CTX_EVENT(init); +DEFINE_CTX_EVENT(destroy); + TRACE_EVENT(rpcgss_svc_accept_upcall, TP_PROTO( __be32 xid, @@ -291,6 +335,40 @@ TRACE_EVENT(rpcgss_need_reencode, __entry->ret ? "" : "un") ); +TRACE_EVENT(rpcgss_update_slack, + TP_PROTO( + const struct rpc_task *task, + const struct rpc_auth *auth + ), + + TP_ARGS(task, auth), + + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(u32, xid) + __field(const void *, auth) + __field(unsigned int, rslack) + __field(unsigned int, ralign) + __field(unsigned int, verfsize) + ), + + TP_fast_assign( + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client->cl_clid; + __entry->xid = be32_to_cpu(task->tk_rqstp->rq_xid); + __entry->auth = auth; + __entry->rslack = auth->au_rslack; + __entry->ralign = auth->au_ralign; + __entry->verfsize = auth->au_verfsize; + ), + + TP_printk("task:%u@%u xid=0x%08x auth=%p rslack=%u ralign=%u verfsize=%u\n", + __entry->task_id, __entry->client_id, __entry->xid, + __entry->auth, __entry->rslack, __entry->ralign, + __entry->verfsize) +); + DECLARE_EVENT_CLASS(rpcgss_svc_seqno_class, TP_PROTO( __be32 xid, @@ -371,6 +449,7 @@ TRACE_EVENT(rpcgss_upcall_result, TRACE_EVENT(rpcgss_context, TP_PROTO( + u32 window_size, unsigned long expiry, unsigned long now, unsigned int timeout, @@ -378,12 +457,13 @@ TRACE_EVENT(rpcgss_context, const u8 *data ), - TP_ARGS(expiry, now, timeout, len, data), + TP_ARGS(window_size, expiry, now, timeout, len, data), TP_STRUCT__entry( __field(unsigned long, expiry) __field(unsigned long, now) __field(unsigned int, timeout) + __field(u32, window_size) __field(int, len) __string(acceptor, data) ), @@ -392,13 +472,14 @@ TRACE_EVENT(rpcgss_context, __entry->expiry = expiry; __entry->now = now; __entry->timeout = timeout; + __entry->window_size = window_size; __entry->len = len; strncpy(__get_str(acceptor), data, len); ), - TP_printk("gc_expiry=%lu now=%lu timeout=%u acceptor=%.*s", - __entry->expiry, __entry->now, __entry->timeout, - __entry->len, __get_str(acceptor)) + TP_printk("win_size=%u expiry=%lu now=%lu timeout=%u acceptor=%.*s", + __entry->window_size, __entry->expiry, __entry->now, + __entry->timeout, __entry->len, __get_str(acceptor)) ); diff --git a/include/trace/events/rpcrdma.h b/include/trace/events/rpcrdma.h index 132c3c778a43..0f05a6e2b9cb 100644 --- a/include/trace/events/rpcrdma.h +++ b/include/trace/events/rpcrdma.h @@ -380,12 +380,8 @@ TRACE_EVENT(xprtrdma_inline_thresh, DEFINE_CONN_EVENT(connect); DEFINE_CONN_EVENT(disconnect); -DEFINE_CONN_EVENT(flush_dct); -DEFINE_RXPRT_EVENT(xprtrdma_create); -DEFINE_RXPRT_EVENT(xprtrdma_op_destroy); DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc); -DEFINE_RXPRT_EVENT(xprtrdma_op_close); DEFINE_RXPRT_EVENT(xprtrdma_op_setport); TRACE_EVENT(xprtrdma_op_connect, @@ -1279,38 +1275,42 @@ TRACE_EVENT(xprtrdma_leaked_rep, ** Server-side RPC/RDMA events **/ -DECLARE_EVENT_CLASS(svcrdma_xprt_event, +DECLARE_EVENT_CLASS(svcrdma_accept_class, TP_PROTO( - const struct svc_xprt *xprt + const struct svcxprt_rdma *rdma, + long status ), - TP_ARGS(xprt), + TP_ARGS(rdma, status), TP_STRUCT__entry( - __field(const void *, xprt) - __string(addr, xprt->xpt_remotebuf) + __field(long, status) + __string(addr, rdma->sc_xprt.xpt_remotebuf) ), TP_fast_assign( - __entry->xprt = xprt; - __assign_str(addr, xprt->xpt_remotebuf); + __entry->status = status; + __assign_str(addr, rdma->sc_xprt.xpt_remotebuf); ), - TP_printk("xprt=%p addr=%s", - __entry->xprt, __get_str(addr) + TP_printk("addr=%s status=%ld", + __get_str(addr), __entry->status ) ); -#define DEFINE_XPRT_EVENT(name) \ - DEFINE_EVENT(svcrdma_xprt_event, svcrdma_xprt_##name, \ - TP_PROTO( \ - const struct svc_xprt *xprt \ - ), \ - TP_ARGS(xprt)) +#define DEFINE_ACCEPT_EVENT(name) \ + DEFINE_EVENT(svcrdma_accept_class, svcrdma_##name##_err, \ + TP_PROTO( \ + const struct svcxprt_rdma *rdma, \ + long status \ + ), \ + TP_ARGS(rdma, status)) -DEFINE_XPRT_EVENT(accept); -DEFINE_XPRT_EVENT(fail); -DEFINE_XPRT_EVENT(free); +DEFINE_ACCEPT_EVENT(pd); +DEFINE_ACCEPT_EVENT(qp); +DEFINE_ACCEPT_EVENT(fabric); +DEFINE_ACCEPT_EVENT(initdepth); +DEFINE_ACCEPT_EVENT(accept); TRACE_DEFINE_ENUM(RDMA_MSG); TRACE_DEFINE_ENUM(RDMA_NOMSG); @@ -1355,7 +1355,7 @@ TRACE_EVENT(svcrdma_decode_rqst, show_rpcrdma_proc(__entry->proc), __entry->hdrlen) ); -TRACE_EVENT(svcrdma_decode_short, +TRACE_EVENT(svcrdma_decode_short_err, TP_PROTO( unsigned int hdrlen ), @@ -1399,7 +1399,8 @@ DECLARE_EVENT_CLASS(svcrdma_badreq_event, ); #define DEFINE_BADREQ_EVENT(name) \ - DEFINE_EVENT(svcrdma_badreq_event, svcrdma_decode_##name,\ + DEFINE_EVENT(svcrdma_badreq_event, \ + svcrdma_decode_##name##_err, \ TP_PROTO( \ __be32 *p \ ), \ @@ -1583,28 +1584,117 @@ DECLARE_EVENT_CLASS(svcrdma_dma_map_class, DEFINE_SVC_DMA_EVENT(dma_map_page); DEFINE_SVC_DMA_EVENT(dma_unmap_page); -TRACE_EVENT(svcrdma_dma_map_rwctx, +TRACE_EVENT(svcrdma_dma_map_rw_err, TP_PROTO( const struct svcxprt_rdma *rdma, + unsigned int nents, int status ), - TP_ARGS(rdma, status), + TP_ARGS(rdma, nents, status), TP_STRUCT__entry( __field(int, status) + __field(unsigned int, nents) __string(device, rdma->sc_cm_id->device->name) __string(addr, rdma->sc_xprt.xpt_remotebuf) ), TP_fast_assign( __entry->status = status; + __entry->nents = nents; + __assign_str(device, rdma->sc_cm_id->device->name); + __assign_str(addr, rdma->sc_xprt.xpt_remotebuf); + ), + + TP_printk("addr=%s device=%s nents=%u status=%d", + __get_str(addr), __get_str(device), __entry->nents, + __entry->status + ) +); + +TRACE_EVENT(svcrdma_no_rwctx_err, + TP_PROTO( + const struct svcxprt_rdma *rdma, + unsigned int num_sges + ), + + TP_ARGS(rdma, num_sges), + + TP_STRUCT__entry( + __field(unsigned int, num_sges) + __string(device, rdma->sc_cm_id->device->name) + __string(addr, rdma->sc_xprt.xpt_remotebuf) + ), + + TP_fast_assign( + __entry->num_sges = num_sges; + __assign_str(device, rdma->sc_cm_id->device->name); + __assign_str(addr, rdma->sc_xprt.xpt_remotebuf); + ), + + TP_printk("addr=%s device=%s num_sges=%d", + __get_str(addr), __get_str(device), __entry->num_sges + ) +); + +TRACE_EVENT(svcrdma_page_overrun_err, + TP_PROTO( + const struct svcxprt_rdma *rdma, + const struct svc_rqst *rqst, + unsigned int pageno + ), + + TP_ARGS(rdma, rqst, pageno), + + TP_STRUCT__entry( + __field(unsigned int, pageno) + __field(u32, xid) + __string(device, rdma->sc_cm_id->device->name) + __string(addr, rdma->sc_xprt.xpt_remotebuf) + ), + + TP_fast_assign( + __entry->pageno = pageno; + __entry->xid = __be32_to_cpu(rqst->rq_xid); + __assign_str(device, rdma->sc_cm_id->device->name); + __assign_str(addr, rdma->sc_xprt.xpt_remotebuf); + ), + + TP_printk("addr=%s device=%s xid=0x%08x pageno=%u", __get_str(addr), + __get_str(device), __entry->xid, __entry->pageno + ) +); + +TRACE_EVENT(svcrdma_small_wrch_err, + TP_PROTO( + const struct svcxprt_rdma *rdma, + unsigned int remaining, + unsigned int seg_no, + unsigned int num_segs + ), + + TP_ARGS(rdma, remaining, seg_no, num_segs), + + TP_STRUCT__entry( + __field(unsigned int, remaining) + __field(unsigned int, seg_no) + __field(unsigned int, num_segs) + __string(device, rdma->sc_cm_id->device->name) + __string(addr, rdma->sc_xprt.xpt_remotebuf) + ), + + TP_fast_assign( + __entry->remaining = remaining; + __entry->seg_no = seg_no; + __entry->num_segs = num_segs; __assign_str(device, rdma->sc_cm_id->device->name); __assign_str(addr, rdma->sc_xprt.xpt_remotebuf); ), - TP_printk("addr=%s device=%s status=%d", - __get_str(addr), __get_str(device), __entry->status + TP_printk("addr=%s device=%s remaining=%u seg_no=%u num_segs=%u", + __get_str(addr), __get_str(device), __entry->remaining, + __entry->seg_no, __entry->num_segs ) ); diff --git a/include/trace/events/sunrpc.h b/include/trace/events/sunrpc.h index ffd2215950dc..6a12935b8b14 100644 --- a/include/trace/events/sunrpc.h +++ b/include/trace/events/sunrpc.h @@ -14,14 +14,50 @@ #include <linux/net.h> #include <linux/tracepoint.h> -DECLARE_EVENT_CLASS(xdr_buf_class, +TRACE_DEFINE_ENUM(SOCK_STREAM); +TRACE_DEFINE_ENUM(SOCK_DGRAM); +TRACE_DEFINE_ENUM(SOCK_RAW); +TRACE_DEFINE_ENUM(SOCK_RDM); +TRACE_DEFINE_ENUM(SOCK_SEQPACKET); +TRACE_DEFINE_ENUM(SOCK_DCCP); +TRACE_DEFINE_ENUM(SOCK_PACKET); + +#define show_socket_type(type) \ + __print_symbolic(type, \ + { SOCK_STREAM, "STREAM" }, \ + { SOCK_DGRAM, "DGRAM" }, \ + { SOCK_RAW, "RAW" }, \ + { SOCK_RDM, "RDM" }, \ + { SOCK_SEQPACKET, "SEQPACKET" }, \ + { SOCK_DCCP, "DCCP" }, \ + { SOCK_PACKET, "PACKET" }) + +/* This list is known to be incomplete, add new enums as needed. */ +TRACE_DEFINE_ENUM(AF_UNSPEC); +TRACE_DEFINE_ENUM(AF_UNIX); +TRACE_DEFINE_ENUM(AF_LOCAL); +TRACE_DEFINE_ENUM(AF_INET); +TRACE_DEFINE_ENUM(AF_INET6); + +#define rpc_show_address_family(family) \ + __print_symbolic(family, \ + { AF_UNSPEC, "AF_UNSPEC" }, \ + { AF_UNIX, "AF_UNIX" }, \ + { AF_LOCAL, "AF_LOCAL" }, \ + { AF_INET, "AF_INET" }, \ + { AF_INET6, "AF_INET6" }) + +DECLARE_EVENT_CLASS(rpc_xdr_buf_class, TP_PROTO( + const struct rpc_task *task, const struct xdr_buf *xdr ), - TP_ARGS(xdr), + TP_ARGS(task, xdr), TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) __field(const void *, head_base) __field(size_t, head_len) __field(const void *, tail_base) @@ -31,6 +67,8 @@ DECLARE_EVENT_CLASS(xdr_buf_class, ), TP_fast_assign( + __entry->task_id = task->tk_pid; + __entry->client_id = task->tk_client->cl_clid; __entry->head_base = xdr->head[0].iov_base; __entry->head_len = xdr->head[0].iov_len; __entry->tail_base = xdr->tail[0].iov_base; @@ -39,23 +77,137 @@ DECLARE_EVENT_CLASS(xdr_buf_class, __entry->msg_len = xdr->len; ), - TP_printk("head=[%p,%zu] page=%u tail=[%p,%zu] len=%u", + TP_printk("task:%u@%u head=[%p,%zu] page=%u tail=[%p,%zu] len=%u", + __entry->task_id, __entry->client_id, __entry->head_base, __entry->head_len, __entry->page_len, __entry->tail_base, __entry->tail_len, __entry->msg_len ) ); -#define DEFINE_XDRBUF_EVENT(name) \ - DEFINE_EVENT(xdr_buf_class, name, \ +#define DEFINE_RPCXDRBUF_EVENT(name) \ + DEFINE_EVENT(rpc_xdr_buf_class, \ + rpc_xdr_##name, \ TP_PROTO( \ + const struct rpc_task *task, \ const struct xdr_buf *xdr \ ), \ - TP_ARGS(xdr)) + TP_ARGS(task, xdr)) + +DEFINE_RPCXDRBUF_EVENT(sendto); +DEFINE_RPCXDRBUF_EVENT(recvfrom); +DEFINE_RPCXDRBUF_EVENT(reply_pages); + + +DECLARE_EVENT_CLASS(rpc_clnt_class, + TP_PROTO( + const struct rpc_clnt *clnt + ), + + TP_ARGS(clnt), + + TP_STRUCT__entry( + __field(unsigned int, client_id) + ), + + TP_fast_assign( + __entry->client_id = clnt->cl_clid; + ), + + TP_printk("clid=%u", __entry->client_id) +); + +#define DEFINE_RPC_CLNT_EVENT(name) \ + DEFINE_EVENT(rpc_clnt_class, \ + rpc_clnt_##name, \ + TP_PROTO( \ + const struct rpc_clnt *clnt \ + ), \ + TP_ARGS(clnt)) + +DEFINE_RPC_CLNT_EVENT(free); +DEFINE_RPC_CLNT_EVENT(killall); +DEFINE_RPC_CLNT_EVENT(shutdown); +DEFINE_RPC_CLNT_EVENT(release); +DEFINE_RPC_CLNT_EVENT(replace_xprt); +DEFINE_RPC_CLNT_EVENT(replace_xprt_err); + +TRACE_EVENT(rpc_clnt_new, + TP_PROTO( + const struct rpc_clnt *clnt, + const struct rpc_xprt *xprt, + const char *program, + const char *server + ), + + TP_ARGS(clnt, xprt, program, server), + + TP_STRUCT__entry( + __field(unsigned int, client_id) + __string(addr, xprt->address_strings[RPC_DISPLAY_ADDR]) + __string(port, xprt->address_strings[RPC_DISPLAY_PORT]) + __string(program, program) + __string(server, server) + ), + + TP_fast_assign( + __entry->client_id = clnt->cl_clid; + __assign_str(addr, xprt->address_strings[RPC_DISPLAY_ADDR]); + __assign_str(port, xprt->address_strings[RPC_DISPLAY_PORT]); + __assign_str(program, program) + __assign_str(server, server) + ), + + TP_printk("client=%u peer=[%s]:%s program=%s server=%s", + __entry->client_id, __get_str(addr), __get_str(port), + __get_str(program), __get_str(server)) +); + +TRACE_EVENT(rpc_clnt_new_err, + TP_PROTO( + const char *program, + const char *server, + int error + ), + + TP_ARGS(program, server, error), + + TP_STRUCT__entry( + __field(int, error) + __string(program, program) + __string(server, server) + ), + + TP_fast_assign( + __entry->error = error; + __assign_str(program, program) + __assign_str(server, server) + ), + + TP_printk("program=%s server=%s error=%d", + __get_str(program), __get_str(server), __entry->error) +); + +TRACE_EVENT(rpc_clnt_clone_err, + TP_PROTO( + const struct rpc_clnt *clnt, + int error + ), + + TP_ARGS(clnt, error), + + TP_STRUCT__entry( + __field(unsigned int, client_id) + __field(int, error) + ), + + TP_fast_assign( + __entry->client_id = clnt->cl_clid; + __entry->error = error; + ), + + TP_printk("client=%u error=%d", __entry->client_id, __entry->error) +); -DEFINE_XDRBUF_EVENT(xprt_sendto); -DEFINE_XDRBUF_EVENT(xprt_recvfrom); -DEFINE_XDRBUF_EVENT(svc_recvfrom); -DEFINE_XDRBUF_EVENT(svc_sendto); TRACE_DEFINE_ENUM(RPC_AUTH_OK); TRACE_DEFINE_ENUM(RPC_AUTH_BADCRED); @@ -142,29 +294,35 @@ TRACE_EVENT(rpc_request, TRACE_DEFINE_ENUM(RPC_TASK_ASYNC); TRACE_DEFINE_ENUM(RPC_TASK_SWAPPER); +TRACE_DEFINE_ENUM(RPC_TASK_NULLCREDS); TRACE_DEFINE_ENUM(RPC_CALL_MAJORSEEN); TRACE_DEFINE_ENUM(RPC_TASK_ROOTCREDS); TRACE_DEFINE_ENUM(RPC_TASK_DYNAMIC); +TRACE_DEFINE_ENUM(RPC_TASK_NO_ROUND_ROBIN); TRACE_DEFINE_ENUM(RPC_TASK_SOFT); TRACE_DEFINE_ENUM(RPC_TASK_SOFTCONN); TRACE_DEFINE_ENUM(RPC_TASK_SENT); TRACE_DEFINE_ENUM(RPC_TASK_TIMEOUT); TRACE_DEFINE_ENUM(RPC_TASK_NOCONNECT); TRACE_DEFINE_ENUM(RPC_TASK_NO_RETRANS_TIMEOUT); +TRACE_DEFINE_ENUM(RPC_TASK_CRED_NOREF); #define rpc_show_task_flags(flags) \ __print_flags(flags, "|", \ { RPC_TASK_ASYNC, "ASYNC" }, \ { RPC_TASK_SWAPPER, "SWAPPER" }, \ + { RPC_TASK_NULLCREDS, "NULLCREDS" }, \ { RPC_CALL_MAJORSEEN, "MAJORSEEN" }, \ { RPC_TASK_ROOTCREDS, "ROOTCREDS" }, \ { RPC_TASK_DYNAMIC, "DYNAMIC" }, \ + { RPC_TASK_NO_ROUND_ROBIN, "NO_ROUND_ROBIN" }, \ { RPC_TASK_SOFT, "SOFT" }, \ { RPC_TASK_SOFTCONN, "SOFTCONN" }, \ { RPC_TASK_SENT, "SENT" }, \ { RPC_TASK_TIMEOUT, "TIMEOUT" }, \ { RPC_TASK_NOCONNECT, "NOCONNECT" }, \ - { RPC_TASK_NO_RETRANS_TIMEOUT, "NORTO" }) + { RPC_TASK_NO_RETRANS_TIMEOUT, "NORTO" }, \ + { RPC_TASK_CRED_NOREF, "CRED_NOREF" }) TRACE_DEFINE_ENUM(RPC_TASK_RUNNING); TRACE_DEFINE_ENUM(RPC_TASK_QUEUED); @@ -359,6 +517,34 @@ DEFINE_RPC_REPLY_EVENT(stale_creds); DEFINE_RPC_REPLY_EVENT(bad_creds); DEFINE_RPC_REPLY_EVENT(auth_tooweak); +TRACE_EVENT(rpc_call_rpcerror, + TP_PROTO( + const struct rpc_task *task, + int tk_status, + int rpc_status + ), + + TP_ARGS(task, tk_status, rpc_status), + + TP_STRUCT__entry( + __field(unsigned int, task_id) + __field(unsigned int, client_id) + __field(int, tk_status) + __field(int, rpc_status) + ), + + TP_fast_assign( + __entry->client_id = task->tk_client->cl_clid; + __entry->task_id = task->tk_pid; + __entry->tk_status = tk_status; + __entry->rpc_status = rpc_status; + ), + + TP_printk("task:%u@%u tk_status=%d rpc_status=%d", + __entry->task_id, __entry->client_id, + __entry->tk_status, __entry->rpc_status) +); + TRACE_EVENT(rpc_stats_latency, TP_PROTO( @@ -526,43 +712,6 @@ TRACE_EVENT(rpc_xdr_alignment, ) ); -TRACE_EVENT(rpc_reply_pages, - TP_PROTO( - const struct rpc_rqst *req - ), - - TP_ARGS(req), - - TP_STRUCT__entry( - __field(unsigned int, task_id) - __field(unsigned int, client_id) - __field(const void *, head_base) - __field(size_t, head_len) - __field(const void *, tail_base) - __field(size_t, tail_len) - __field(unsigned int, page_len) - ), - - TP_fast_assign( - __entry->task_id = req->rq_task->tk_pid; - __entry->client_id = req->rq_task->tk_client->cl_clid; - - __entry->head_base = req->rq_rcv_buf.head[0].iov_base; - __entry->head_len = req->rq_rcv_buf.head[0].iov_len; - __entry->page_len = req->rq_rcv_buf.page_len; - __entry->tail_base = req->rq_rcv_buf.tail[0].iov_base; - __entry->tail_len = req->rq_rcv_buf.tail[0].iov_len; - ), - - TP_printk( - "task:%u@%u xdr=[%p,%zu]/%u/[%p,%zu]\n", - __entry->task_id, __entry->client_id, - __entry->head_base, __entry->head_len, - __entry->page_len, - __entry->tail_base, __entry->tail_len - ) -); - /* * First define the enums in the below macros to be exported to userspace * via TRACE_DEFINE_ENUM(). @@ -575,9 +724,9 @@ TRACE_EVENT(rpc_reply_pages, #define RPC_SHOW_SOCKET \ EM( SS_FREE, "FREE" ) \ EM( SS_UNCONNECTED, "UNCONNECTED" ) \ - EM( SS_CONNECTING, "CONNECTING," ) \ - EM( SS_CONNECTED, "CONNECTED," ) \ - EMe(SS_DISCONNECTING, "DISCONNECTING" ) + EM( SS_CONNECTING, "CONNECTING" ) \ + EM( SS_CONNECTED, "CONNECTED" ) \ + EMe( SS_DISCONNECTING, "DISCONNECTING" ) #define rpc_show_socket_state(state) \ __print_symbolic(state, RPC_SHOW_SOCKET) @@ -719,6 +868,69 @@ DEFINE_RPC_SOCKET_EVENT_DONE(rpc_socket_reset_connection); DEFINE_RPC_SOCKET_EVENT(rpc_socket_close); DEFINE_RPC_SOCKET_EVENT(rpc_socket_shutdown); +TRACE_DEFINE_ENUM(XPRT_LOCKED); +TRACE_DEFINE_ENUM(XPRT_CONNECTED); +TRACE_DEFINE_ENUM(XPRT_CONNECTING); +TRACE_DEFINE_ENUM(XPRT_CLOSE_WAIT); +TRACE_DEFINE_ENUM(XPRT_BOUND); +TRACE_DEFINE_ENUM(XPRT_BINDING); +TRACE_DEFINE_ENUM(XPRT_CLOSING); +TRACE_DEFINE_ENUM(XPRT_CONGESTED); +TRACE_DEFINE_ENUM(XPRT_CWND_WAIT); +TRACE_DEFINE_ENUM(XPRT_WRITE_SPACE); + +#define rpc_show_xprt_state(x) \ + __print_flags(x, "|", \ + { (1UL << XPRT_LOCKED), "LOCKED"}, \ + { (1UL << XPRT_CONNECTED), "CONNECTED"}, \ + { (1UL << XPRT_CONNECTING), "CONNECTING"}, \ + { (1UL << XPRT_CLOSE_WAIT), "CLOSE_WAIT"}, \ + { (1UL << XPRT_BOUND), "BOUND"}, \ + { (1UL << XPRT_BINDING), "BINDING"}, \ + { (1UL << XPRT_CLOSING), "CLOSING"}, \ + { (1UL << XPRT_CONGESTED), "CONGESTED"}, \ + { (1UL << XPRT_CWND_WAIT), "CWND_WAIT"}, \ + { (1UL << XPRT_WRITE_SPACE), "WRITE_SPACE"}) + +DECLARE_EVENT_CLASS(rpc_xprt_lifetime_class, + TP_PROTO( + const struct rpc_xprt *xprt + ), + + TP_ARGS(xprt), + + TP_STRUCT__entry( + __field(unsigned long, state) + __string(addr, xprt->address_strings[RPC_DISPLAY_ADDR]) + __string(port, xprt->address_strings[RPC_DISPLAY_PORT]) + ), + + TP_fast_assign( + __entry->state = xprt->state; + __assign_str(addr, xprt->address_strings[RPC_DISPLAY_ADDR]); + __assign_str(port, xprt->address_strings[RPC_DISPLAY_PORT]); + ), + + TP_printk("peer=[%s]:%s state=%s", + __get_str(addr), __get_str(port), + rpc_show_xprt_state(__entry->state)) +); + +#define DEFINE_RPC_XPRT_LIFETIME_EVENT(name) \ + DEFINE_EVENT(rpc_xprt_lifetime_class, \ + xprt_##name, \ + TP_PROTO( \ + const struct rpc_xprt *xprt \ + ), \ + TP_ARGS(xprt)) + +DEFINE_RPC_XPRT_LIFETIME_EVENT(create); +DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_auto); +DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_done); +DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_force); +DEFINE_RPC_XPRT_LIFETIME_EVENT(disconnect_cleanup); +DEFINE_RPC_XPRT_LIFETIME_EVENT(destroy); + DECLARE_EVENT_CLASS(rpc_xprt_event, TP_PROTO( const struct rpc_xprt *xprt, @@ -990,6 +1202,54 @@ TRACE_EVENT(xs_stream_read_request, __entry->copied, __entry->reclen, __entry->offset) ); + +DECLARE_EVENT_CLASS(svc_xdr_buf_class, + TP_PROTO( + const struct svc_rqst *rqst, + const struct xdr_buf *xdr + ), + + TP_ARGS(rqst, xdr), + + TP_STRUCT__entry( + __field(u32, xid) + __field(const void *, head_base) + __field(size_t, head_len) + __field(const void *, tail_base) + __field(size_t, tail_len) + __field(unsigned int, page_len) + __field(unsigned int, msg_len) + ), + + TP_fast_assign( + __entry->xid = be32_to_cpu(rqst->rq_xid); + __entry->head_base = xdr->head[0].iov_base; + __entry->head_len = xdr->head[0].iov_len; + __entry->tail_base = xdr->tail[0].iov_base; + __entry->tail_len = xdr->tail[0].iov_len; + __entry->page_len = xdr->page_len; + __entry->msg_len = xdr->len; + ), + + TP_printk("xid=0x%08x head=[%p,%zu] page=%u tail=[%p,%zu] len=%u", + __entry->xid, + __entry->head_base, __entry->head_len, __entry->page_len, + __entry->tail_base, __entry->tail_len, __entry->msg_len + ) +); + +#define DEFINE_SVCXDRBUF_EVENT(name) \ + DEFINE_EVENT(svc_xdr_buf_class, \ + svc_xdr_##name, \ + TP_PROTO( \ + const struct svc_rqst *rqst, \ + const struct xdr_buf *xdr \ + ), \ + TP_ARGS(rqst, xdr)) + +DEFINE_SVCXDRBUF_EVENT(recvfrom); +DEFINE_SVCXDRBUF_EVENT(sendto); + #define show_rqstp_flags(flags) \ __print_flags(flags, "|", \ { (1UL << RQ_SECURE), "RQ_SECURE"}, \ @@ -1024,6 +1284,17 @@ TRACE_EVENT(svc_recv, show_rqstp_flags(__entry->flags)) ); +TRACE_DEFINE_ENUM(SVC_GARBAGE); +TRACE_DEFINE_ENUM(SVC_SYSERR); +TRACE_DEFINE_ENUM(SVC_VALID); +TRACE_DEFINE_ENUM(SVC_NEGATIVE); +TRACE_DEFINE_ENUM(SVC_OK); +TRACE_DEFINE_ENUM(SVC_DROP); +TRACE_DEFINE_ENUM(SVC_CLOSE); +TRACE_DEFINE_ENUM(SVC_DENIED); +TRACE_DEFINE_ENUM(SVC_PENDING); +TRACE_DEFINE_ENUM(SVC_COMPLETE); + #define svc_show_status(status) \ __print_symbolic(status, \ { SVC_GARBAGE, "SVC_GARBAGE" }, \ @@ -1167,28 +1438,54 @@ DEFINE_EVENT(svc_rqst_status, svc_send, { (1UL << XPT_KILL_TEMP), "XPT_KILL_TEMP"}, \ { (1UL << XPT_CONG_CTRL), "XPT_CONG_CTRL"}) +TRACE_EVENT(svc_xprt_create_err, + TP_PROTO( + const char *program, + const char *protocol, + struct sockaddr *sap, + const struct svc_xprt *xprt + ), + + TP_ARGS(program, protocol, sap, xprt), + + TP_STRUCT__entry( + __field(long, error) + __string(program, program) + __string(protocol, protocol) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->error = PTR_ERR(xprt); + __assign_str(program, program); + __assign_str(protocol, protocol); + memcpy(__entry->addr, sap, sizeof(__entry->addr)); + ), + + TP_printk("addr=%pISpc program=%s protocol=%s error=%ld", + __entry->addr, __get_str(program), __get_str(protocol), + __entry->error) +); + TRACE_EVENT(svc_xprt_do_enqueue, TP_PROTO(struct svc_xprt *xprt, struct svc_rqst *rqst), TP_ARGS(xprt, rqst), TP_STRUCT__entry( - __field(struct svc_xprt *, xprt) __field(int, pid) __field(unsigned long, flags) __string(addr, xprt->xpt_remotebuf) ), TP_fast_assign( - __entry->xprt = xprt; __entry->pid = rqst? rqst->rq_task->pid : 0; __entry->flags = xprt->xpt_flags; __assign_str(addr, xprt->xpt_remotebuf); ), - TP_printk("xprt=%p addr=%s pid=%d flags=%s", - __entry->xprt, __get_str(addr), - __entry->pid, show_svc_xprt_flags(__entry->flags)) + TP_printk("addr=%s pid=%d flags=%s", __get_str(addr), + __entry->pid, show_svc_xprt_flags(__entry->flags)) ); DECLARE_EVENT_CLASS(svc_xprt_event, @@ -1197,25 +1494,55 @@ DECLARE_EVENT_CLASS(svc_xprt_event, TP_ARGS(xprt), TP_STRUCT__entry( - __field(struct svc_xprt *, xprt) __field(unsigned long, flags) __string(addr, xprt->xpt_remotebuf) ), TP_fast_assign( - __entry->xprt = xprt; __entry->flags = xprt->xpt_flags; __assign_str(addr, xprt->xpt_remotebuf); ), - TP_printk("xprt=%p addr=%s flags=%s", - __entry->xprt, __get_str(addr), - show_svc_xprt_flags(__entry->flags)) + TP_printk("addr=%s flags=%s", __get_str(addr), + show_svc_xprt_flags(__entry->flags)) ); -DEFINE_EVENT(svc_xprt_event, svc_xprt_no_write_space, - TP_PROTO(struct svc_xprt *xprt), - TP_ARGS(xprt)); +#define DEFINE_SVC_XPRT_EVENT(name) \ + DEFINE_EVENT(svc_xprt_event, svc_xprt_##name, \ + TP_PROTO( \ + struct svc_xprt *xprt \ + ), \ + TP_ARGS(xprt)) + +DEFINE_SVC_XPRT_EVENT(no_write_space); +DEFINE_SVC_XPRT_EVENT(close); +DEFINE_SVC_XPRT_EVENT(detach); +DEFINE_SVC_XPRT_EVENT(free); + +TRACE_EVENT(svc_xprt_accept, + TP_PROTO( + const struct svc_xprt *xprt, + const char *service + ), + + TP_ARGS(xprt, service), + + TP_STRUCT__entry( + __string(addr, xprt->xpt_remotebuf) + __string(protocol, xprt->xpt_class->xcl_name) + __string(service, service) + ), + + TP_fast_assign( + __assign_str(addr, xprt->xpt_remotebuf); + __assign_str(protocol, xprt->xpt_class->xcl_name) + __assign_str(service, service); + ), + + TP_printk("addr=%s protocol=%s service=%s", + __get_str(addr), __get_str(protocol), __get_str(service) + ) +); TRACE_EVENT(svc_xprt_dequeue, TP_PROTO(struct svc_rqst *rqst), @@ -1223,24 +1550,20 @@ TRACE_EVENT(svc_xprt_dequeue, TP_ARGS(rqst), TP_STRUCT__entry( - __field(struct svc_xprt *, xprt) __field(unsigned long, flags) __field(unsigned long, wakeup) __string(addr, rqst->rq_xprt->xpt_remotebuf) ), TP_fast_assign( - __entry->xprt = rqst->rq_xprt; __entry->flags = rqst->rq_xprt->xpt_flags; __entry->wakeup = ktime_to_us(ktime_sub(ktime_get(), rqst->rq_qtime)); __assign_str(addr, rqst->rq_xprt->xpt_remotebuf); ), - TP_printk("xprt=%p addr=%s flags=%s wakeup-us=%lu", - __entry->xprt, __get_str(addr), - show_svc_xprt_flags(__entry->flags), - __entry->wakeup) + TP_printk("addr=%s flags=%s wakeup-us=%lu", __get_str(addr), + show_svc_xprt_flags(__entry->flags), __entry->wakeup) ); TRACE_EVENT(svc_wake_up, @@ -1265,21 +1588,18 @@ TRACE_EVENT(svc_handle_xprt, TP_ARGS(xprt, len), TP_STRUCT__entry( - __field(struct svc_xprt *, xprt) __field(int, len) __field(unsigned long, flags) __string(addr, xprt->xpt_remotebuf) ), TP_fast_assign( - __entry->xprt = xprt; __entry->len = len; __entry->flags = xprt->xpt_flags; __assign_str(addr, xprt->xpt_remotebuf); ), - TP_printk("xprt=%p addr=%s len=%d flags=%s", - __entry->xprt, __get_str(addr), + TP_printk("addr=%s len=%d flags=%s", __get_str(addr), __entry->len, show_svc_xprt_flags(__entry->flags)) ); @@ -1313,27 +1633,221 @@ DECLARE_EVENT_CLASS(svc_deferred_event, TP_ARGS(dr), TP_STRUCT__entry( + __field(const void *, dr) __field(u32, xid) __string(addr, dr->xprt->xpt_remotebuf) ), TP_fast_assign( + __entry->dr = dr; __entry->xid = be32_to_cpu(*(__be32 *)(dr->args + (dr->xprt_hlen>>2))); __assign_str(addr, dr->xprt->xpt_remotebuf); ), - TP_printk("addr=%s xid=0x%08x", __get_str(addr), __entry->xid) + TP_printk("addr=%s dr=%p xid=0x%08x", __get_str(addr), __entry->dr, + __entry->xid) ); + #define DEFINE_SVC_DEFERRED_EVENT(name) \ - DEFINE_EVENT(svc_deferred_event, svc_##name##_deferred, \ + DEFINE_EVENT(svc_deferred_event, svc_defer_##name, \ TP_PROTO( \ const struct svc_deferred_req *dr \ ), \ TP_ARGS(dr)) DEFINE_SVC_DEFERRED_EVENT(drop); -DEFINE_SVC_DEFERRED_EVENT(revisit); +DEFINE_SVC_DEFERRED_EVENT(queue); +DEFINE_SVC_DEFERRED_EVENT(recv); + +TRACE_EVENT(svcsock_new_socket, + TP_PROTO( + const struct socket *socket + ), + + TP_ARGS(socket), + + TP_STRUCT__entry( + __field(unsigned long, type) + __field(unsigned long, family) + __field(bool, listener) + ), + + TP_fast_assign( + __entry->type = socket->type; + __entry->family = socket->sk->sk_family; + __entry->listener = (socket->sk->sk_state == TCP_LISTEN); + ), + + TP_printk("type=%s family=%s%s", + show_socket_type(__entry->type), + rpc_show_address_family(__entry->family), + __entry->listener ? " (listener)" : "" + ) +); + +TRACE_EVENT(svcsock_marker, + TP_PROTO( + const struct svc_xprt *xprt, + __be32 marker + ), + + TP_ARGS(xprt, marker), + + TP_STRUCT__entry( + __field(unsigned int, length) + __field(bool, last) + __string(addr, xprt->xpt_remotebuf) + ), + + TP_fast_assign( + __entry->length = be32_to_cpu(marker) & RPC_FRAGMENT_SIZE_MASK; + __entry->last = be32_to_cpu(marker) & RPC_LAST_STREAM_FRAGMENT; + __assign_str(addr, xprt->xpt_remotebuf); + ), + + TP_printk("addr=%s length=%u%s", __get_str(addr), + __entry->length, __entry->last ? " (last)" : "") +); + +DECLARE_EVENT_CLASS(svcsock_class, + TP_PROTO( + const struct svc_xprt *xprt, + ssize_t result + ), + + TP_ARGS(xprt, result), + + TP_STRUCT__entry( + __field(ssize_t, result) + __field(unsigned long, flags) + __string(addr, xprt->xpt_remotebuf) + ), + + TP_fast_assign( + __entry->result = result; + __entry->flags = xprt->xpt_flags; + __assign_str(addr, xprt->xpt_remotebuf); + ), + + TP_printk("addr=%s result=%zd flags=%s", __get_str(addr), + __entry->result, show_svc_xprt_flags(__entry->flags) + ) +); + +#define DEFINE_SVCSOCK_EVENT(name) \ + DEFINE_EVENT(svcsock_class, svcsock_##name, \ + TP_PROTO( \ + const struct svc_xprt *xprt, \ + ssize_t result \ + ), \ + TP_ARGS(xprt, result)) + +DEFINE_SVCSOCK_EVENT(udp_send); +DEFINE_SVCSOCK_EVENT(udp_recv); +DEFINE_SVCSOCK_EVENT(udp_recv_err); +DEFINE_SVCSOCK_EVENT(tcp_send); +DEFINE_SVCSOCK_EVENT(tcp_recv); +DEFINE_SVCSOCK_EVENT(tcp_recv_eagain); +DEFINE_SVCSOCK_EVENT(tcp_recv_err); +DEFINE_SVCSOCK_EVENT(data_ready); +DEFINE_SVCSOCK_EVENT(write_space); + +TRACE_EVENT(svcsock_tcp_recv_short, + TP_PROTO( + const struct svc_xprt *xprt, + u32 expected, + u32 received + ), + + TP_ARGS(xprt, expected, received), + + TP_STRUCT__entry( + __field(u32, expected) + __field(u32, received) + __field(unsigned long, flags) + __string(addr, xprt->xpt_remotebuf) + ), + + TP_fast_assign( + __entry->expected = expected; + __entry->received = received; + __entry->flags = xprt->xpt_flags; + __assign_str(addr, xprt->xpt_remotebuf); + ), + + TP_printk("addr=%s flags=%s expected=%u received=%u", + __get_str(addr), show_svc_xprt_flags(__entry->flags), + __entry->expected, __entry->received + ) +); + +TRACE_EVENT(svcsock_tcp_state, + TP_PROTO( + const struct svc_xprt *xprt, + const struct socket *socket + ), + + TP_ARGS(xprt, socket), + + TP_STRUCT__entry( + __field(unsigned long, socket_state) + __field(unsigned long, sock_state) + __field(unsigned long, flags) + __string(addr, xprt->xpt_remotebuf) + ), + + TP_fast_assign( + __entry->socket_state = socket->state; + __entry->sock_state = socket->sk->sk_state; + __entry->flags = xprt->xpt_flags; + __assign_str(addr, xprt->xpt_remotebuf); + ), + + TP_printk("addr=%s state=%s sk_state=%s flags=%s", __get_str(addr), + rpc_show_socket_state(__entry->socket_state), + rpc_show_sock_state(__entry->sock_state), + show_svc_xprt_flags(__entry->flags) + ) +); + +DECLARE_EVENT_CLASS(svcsock_accept_class, + TP_PROTO( + const struct svc_xprt *xprt, + const char *service, + long status + ), + + TP_ARGS(xprt, service, status), + + TP_STRUCT__entry( + __field(long, status) + __string(service, service) + __array(unsigned char, addr, sizeof(struct sockaddr_in6)) + ), + + TP_fast_assign( + __entry->status = status; + __assign_str(service, service); + memcpy(__entry->addr, &xprt->xpt_local, sizeof(__entry->addr)); + ), + + TP_printk("listener=%pISpc service=%s status=%ld", + __entry->addr, __get_str(service), __entry->status + ) +); + +#define DEFINE_ACCEPT_EVENT(name) \ + DEFINE_EVENT(svcsock_accept_class, svcsock_##name##_err, \ + TP_PROTO( \ + const struct svc_xprt *xprt, \ + const char *service, \ + long status \ + ), \ + TP_ARGS(xprt, service, status)) + +DEFINE_ACCEPT_EVENT(accept); +DEFINE_ACCEPT_EVENT(getpeername); DECLARE_EVENT_CLASS(cache_event, TP_PROTO( @@ -1368,6 +1882,86 @@ DEFINE_CACHE_EVENT(cache_entry_update); DEFINE_CACHE_EVENT(cache_entry_make_negative); DEFINE_CACHE_EVENT(cache_entry_no_listener); +DECLARE_EVENT_CLASS(register_class, + TP_PROTO( + const char *program, + const u32 version, + const int family, + const unsigned short protocol, + const unsigned short port, + int error + ), + + TP_ARGS(program, version, family, protocol, port, error), + + TP_STRUCT__entry( + __field(u32, version) + __field(unsigned long, family) + __field(unsigned short, protocol) + __field(unsigned short, port) + __field(int, error) + __string(program, program) + ), + + TP_fast_assign( + __entry->version = version; + __entry->family = family; + __entry->protocol = protocol; + __entry->port = port; + __entry->error = error; + __assign_str(program, program); + ), + + TP_printk("program=%sv%u proto=%s port=%u family=%s error=%d", + __get_str(program), __entry->version, + __entry->protocol == IPPROTO_UDP ? "udp" : "tcp", + __entry->port, rpc_show_address_family(__entry->family), + __entry->error + ) +); + +#define DEFINE_REGISTER_EVENT(name) \ + DEFINE_EVENT(register_class, svc_##name, \ + TP_PROTO( \ + const char *program, \ + const u32 version, \ + const int family, \ + const unsigned short protocol, \ + const unsigned short port, \ + int error \ + ), \ + TP_ARGS(program, version, family, protocol, \ + port, error)) + +DEFINE_REGISTER_EVENT(register); +DEFINE_REGISTER_EVENT(noregister); + +TRACE_EVENT(svc_unregister, + TP_PROTO( + const char *program, + const u32 version, + int error + ), + + TP_ARGS(program, version, error), + + TP_STRUCT__entry( + __field(u32, version) + __field(int, error) + __string(program, program) + ), + + TP_fast_assign( + __entry->version = version; + __entry->error = error; + __assign_str(program, program); + ), + + TP_printk("program=%sv%u error=%d", + __get_str(program), __entry->version, __entry->error + ) +); + #endif /* _TRACE_SUNRPC_H */ #include <trace/define_trace.h> diff --git a/include/uapi/linux/keyctl.h b/include/uapi/linux/keyctl.h index ed3d5893830d..4c8884eea808 100644 --- a/include/uapi/linux/keyctl.h +++ b/include/uapi/linux/keyctl.h @@ -69,6 +69,7 @@ #define KEYCTL_RESTRICT_KEYRING 29 /* Restrict keys allowed to link to a keyring */ #define KEYCTL_MOVE 30 /* Move keys between keyrings */ #define KEYCTL_CAPABILITIES 31 /* Find capabilities of keyrings subsystem */ +#define KEYCTL_WATCH_KEY 32 /* Watch a key or ring of keys for changes */ /* keyctl structures */ struct keyctl_dh_params { @@ -130,5 +131,6 @@ struct keyctl_pkey_params { #define KEYCTL_CAPS0_MOVE 0x80 /* KEYCTL_MOVE supported */ #define KEYCTL_CAPS1_NS_KEYRING_NAME 0x01 /* Keyring names are per-user_namespace */ #define KEYCTL_CAPS1_NS_KEY_TAG 0x02 /* Key indexing can include a namespace tag */ +#define KEYCTL_CAPS1_NOTIFICATIONS 0x04 /* Keys generate watchable notifications */ #endif /* _LINUX_KEYCTL_H */ diff --git a/include/uapi/linux/watch_queue.h b/include/uapi/linux/watch_queue.h new file mode 100644 index 000000000000..c3d8320b5d3a --- /dev/null +++ b/include/uapi/linux/watch_queue.h @@ -0,0 +1,104 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _UAPI_LINUX_WATCH_QUEUE_H +#define _UAPI_LINUX_WATCH_QUEUE_H + +#include <linux/types.h> +#include <linux/fcntl.h> +#include <linux/ioctl.h> + +#define O_NOTIFICATION_PIPE O_EXCL /* Parameter to pipe2() selecting notification pipe */ + +#define IOC_WATCH_QUEUE_SET_SIZE _IO('W', 0x60) /* Set the size in pages */ +#define IOC_WATCH_QUEUE_SET_FILTER _IO('W', 0x61) /* Set the filter */ + +enum watch_notification_type { + WATCH_TYPE_META = 0, /* Special record */ + WATCH_TYPE_KEY_NOTIFY = 1, /* Key change event notification */ + WATCH_TYPE__NR = 2 +}; + +enum watch_meta_notification_subtype { + WATCH_META_REMOVAL_NOTIFICATION = 0, /* Watched object was removed */ + WATCH_META_LOSS_NOTIFICATION = 1, /* Data loss occurred */ +}; + +/* + * Notification record header. This is aligned to 64-bits so that subclasses + * can contain __u64 fields. + */ +struct watch_notification { + __u32 type:24; /* enum watch_notification_type */ + __u32 subtype:8; /* Type-specific subtype (filterable) */ + __u32 info; +#define WATCH_INFO_LENGTH 0x0000007f /* Length of record */ +#define WATCH_INFO_LENGTH__SHIFT 0 +#define WATCH_INFO_ID 0x0000ff00 /* ID of watchpoint */ +#define WATCH_INFO_ID__SHIFT 8 +#define WATCH_INFO_TYPE_INFO 0xffff0000 /* Type-specific info */ +#define WATCH_INFO_TYPE_INFO__SHIFT 16 +#define WATCH_INFO_FLAG_0 0x00010000 /* Type-specific info, flag bit 0 */ +#define WATCH_INFO_FLAG_1 0x00020000 /* ... */ +#define WATCH_INFO_FLAG_2 0x00040000 +#define WATCH_INFO_FLAG_3 0x00080000 +#define WATCH_INFO_FLAG_4 0x00100000 +#define WATCH_INFO_FLAG_5 0x00200000 +#define WATCH_INFO_FLAG_6 0x00400000 +#define WATCH_INFO_FLAG_7 0x00800000 +}; + +/* + * Notification filtering rules (IOC_WATCH_QUEUE_SET_FILTER). + */ +struct watch_notification_type_filter { + __u32 type; /* Type to apply filter to */ + __u32 info_filter; /* Filter on watch_notification::info */ + __u32 info_mask; /* Mask of relevant bits in info_filter */ + __u32 subtype_filter[8]; /* Bitmask of subtypes to filter on */ +}; + +struct watch_notification_filter { + __u32 nr_filters; /* Number of filters */ + __u32 __reserved; /* Must be 0 */ + struct watch_notification_type_filter filters[]; +}; + + +/* + * Extended watch removal notification. This is used optionally if the type + * wants to indicate an identifier for the object being watched, if there is + * such. This can be distinguished by the length. + * + * type -> WATCH_TYPE_META + * subtype -> WATCH_META_REMOVAL_NOTIFICATION + */ +struct watch_notification_removal { + struct watch_notification watch; + __u64 id; /* Type-dependent identifier */ +}; + +/* + * Type of key/keyring change notification. + */ +enum key_notification_subtype { + NOTIFY_KEY_INSTANTIATED = 0, /* Key was instantiated (aux is error code) */ + NOTIFY_KEY_UPDATED = 1, /* Key was updated */ + NOTIFY_KEY_LINKED = 2, /* Key (aux) was added to watched keyring */ + NOTIFY_KEY_UNLINKED = 3, /* Key (aux) was removed from watched keyring */ + NOTIFY_KEY_CLEARED = 4, /* Keyring was cleared */ + NOTIFY_KEY_REVOKED = 5, /* Key was revoked */ + NOTIFY_KEY_INVALIDATED = 6, /* Key was invalidated */ + NOTIFY_KEY_SETATTR = 7, /* Key's attributes got changed */ +}; + +/* + * Key/keyring notification record. + * - watch.type = WATCH_TYPE_KEY_NOTIFY + * - watch.subtype = enum key_notification_type + */ +struct key_notification { + struct watch_notification watch; + __u32 key_id; /* The key/keyring affected */ + __u32 aux; /* Per-type auxiliary data */ +}; + +#endif /* _UAPI_LINUX_WATCH_QUEUE_H */ diff --git a/include/xen/events.h b/include/xen/events.h index 12b0dcb6a120..df1e6391f63f 100644 --- a/include/xen/events.h +++ b/include/xen/events.h @@ -90,13 +90,6 @@ unsigned int irq_from_evtchn(evtchn_port_t evtchn); int irq_from_virq(unsigned int cpu, unsigned int virq); evtchn_port_t evtchn_from_irq(unsigned irq); -#ifdef CONFIG_XEN_PVHVM -/* Xen HVM evtchn vector callback */ -void xen_hvm_callback_vector(void); -#ifdef CONFIG_TRACING -#define trace_xen_hvm_callback_vector xen_hvm_callback_vector -#endif -#endif int xen_set_callback_via(uint64_t via); void xen_evtchn_do_upcall(struct pt_regs *regs); void xen_hvm_evtchn_do_upcall(void); diff --git a/include/xen/hvm.h b/include/xen/hvm.h index 0b15f8cb17fc..b7fd7fc9ad41 100644 --- a/include/xen/hvm.h +++ b/include/xen/hvm.h @@ -58,4 +58,6 @@ static inline int hvm_get_parameter(int idx, uint64_t *value) #define HVM_CALLBACK_VECTOR(x) (((uint64_t)HVM_CALLBACK_VIA_TYPE_VECTOR)<<\ HVM_CALLBACK_VIA_TYPE_SHIFT | (x)) +void xen_setup_callback_vector(void); + #endif /* XEN_HVM_H__ */ diff --git a/include/xen/interface/hvm/hvm_op.h b/include/xen/interface/hvm/hvm_op.h index 956a04682865..25d945ef17de 100644 --- a/include/xen/interface/hvm/hvm_op.h +++ b/include/xen/interface/hvm/hvm_op.h @@ -21,6 +21,8 @@ #ifndef __XEN_PUBLIC_HVM_HVM_OP_H__ #define __XEN_PUBLIC_HVM_HVM_OP_H__ +#include <xen/interface/xen.h> + /* Get/set subcommands: the second argument of the hypercall is a * pointer to a xen_hvm_param struct. */ #define HVMOP_set_param 0 diff --git a/include/xen/xen-ops.h b/include/xen/xen-ops.h index 095be1d66f31..39a5580f8feb 100644 --- a/include/xen/xen-ops.h +++ b/include/xen/xen-ops.h @@ -215,17 +215,7 @@ bool xen_running_on_version_or_later(unsigned int major, unsigned int minor); void xen_efi_runtime_setup(void); -#ifdef CONFIG_PREEMPTION - -static inline void xen_preemptible_hcall_begin(void) -{ -} - -static inline void xen_preemptible_hcall_end(void) -{ -} - -#else +#if defined(CONFIG_XEN_PV) && !defined(CONFIG_PREEMPTION) DECLARE_PER_CPU(bool, xen_in_preemptible_hcall); @@ -239,6 +229,11 @@ static inline void xen_preemptible_hcall_end(void) __this_cpu_write(xen_in_preemptible_hcall, false); } -#endif /* CONFIG_PREEMPTION */ +#else + +static inline void xen_preemptible_hcall_begin(void) { } +static inline void xen_preemptible_hcall_end(void) { } + +#endif /* CONFIG_XEN_PV && !CONFIG_PREEMPTION */ #endif /* INCLUDE_XEN_OPS_H */ diff --git a/init/Kconfig b/init/Kconfig index 8f90d317364e..a46aa8f3174d 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -367,6 +367,18 @@ config POSIX_MQUEUE_SYSCTL depends on SYSCTL default y +config WATCH_QUEUE + bool "General notification queue" + default n + help + + This is a general notification queue for the kernel to pass events to + userspace by splicing them into pipes. It can be used in conjunction + with watches for key/keyring change notifications and device + notifications. + + See Documentation/watch_queue.rst + config CROSS_MEMORY_ATTACH bool "Enable process_vm_readv/writev syscalls" depends on MMU diff --git a/init/init_task.c b/init/init_task.c index a90ac70cc515..15089d15010a 100644 --- a/init/init_task.c +++ b/init/init_task.c @@ -174,6 +174,16 @@ struct task_struct init_task #ifdef CONFIG_KASAN .kasan_depth = 1, #endif +#ifdef CONFIG_KCSAN + .kcsan_ctx = { + .disable_count = 0, + .atomic_next = 0, + .atomic_nest_count = 0, + .in_flat_atomic = false, + .access_mask = 0, + .scoped_accesses = {LIST_POISON1, NULL}, + }, +#endif #ifdef CONFIG_TRACE_IRQFLAGS .softirqs_enabled = 1, #endif diff --git a/init/main.c b/init/main.c index b59e09353881..0ead83e86b5a 100644 --- a/init/main.c +++ b/init/main.c @@ -95,6 +95,7 @@ #include <linux/rodata_test.h> #include <linux/jump_label.h> #include <linux/mem_encrypt.h> +#include <linux/kcsan.h> #include <asm/io.h> #include <asm/bugs.h> @@ -1036,6 +1037,7 @@ asmlinkage __visible void __init start_kernel(void) acpi_subsystem_init(); arch_post_acpi_subsys_init(); sfi_init_late(); + kcsan_init(); /* Do the rest non-__init'ed, we're now alive */ arch_call_rest_init(); diff --git a/kernel/Makefile b/kernel/Makefile index c332eb9d4841..f3218bc5ec69 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -23,6 +23,9 @@ endif # Prevents flicker of uninteresting __do_softirq()/__local_bh_disable_ip() # in coverage traces. KCOV_INSTRUMENT_softirq.o := n +# Avoid KCSAN instrumentation in softirq ("No shared variables, all the data +# are CPU local" => assume no data races), to reduce overhead in interrupts. +KCSAN_SANITIZE_softirq.o = n # These are called from save_stack_trace() on slub debug path, # and produce insane amounts of uninteresting coverage. KCOV_INSTRUMENT_module.o := n @@ -31,6 +34,7 @@ KCOV_INSTRUMENT_stacktrace.o := n # Don't self-instrument. KCOV_INSTRUMENT_kcov.o := n KASAN_SANITIZE_kcov.o := n +KCSAN_SANITIZE_kcov.o := n CFLAGS_kcov.o := $(call cc-option, -fno-conserve-stack -fno-stack-protector) # cond_syscall is currently not LTO compatible @@ -103,6 +107,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/ obj-$(CONFIG_IRQ_WORK) += irq_work.o obj-$(CONFIG_CPU_PM) += cpu_pm.o obj-$(CONFIG_BPF) += bpf/ +obj-$(CONFIG_KCSAN) += kcsan/ obj-$(CONFIG_SHADOW_CALL_STACK) += scs.o obj-$(CONFIG_PERF_EVENTS) += events/ @@ -116,11 +121,13 @@ obj-$(CONFIG_TORTURE_TEST) += torture.o obj-$(CONFIG_HAS_IOMEM) += iomem.o obj-$(CONFIG_RSEQ) += rseq.o +obj-$(CONFIG_WATCH_QUEUE) += watch_queue.o obj-$(CONFIG_SYSCTL_KUNIT_TEST) += sysctl-test.o obj-$(CONFIG_GCC_PLUGIN_STACKLEAK) += stackleak.o KASAN_SANITIZE_stackleak.o := n +KCSAN_SANITIZE_stackleak.o := n KCOV_INSTRUMENT_stackleak.o := n $(obj)/configs.o: $(obj)/config_data.gz diff --git a/kernel/context_tracking.c b/kernel/context_tracking.c index ce430885c26c..36a98c48aedc 100644 --- a/kernel/context_tracking.c +++ b/kernel/context_tracking.c @@ -31,7 +31,7 @@ EXPORT_SYMBOL_GPL(context_tracking_key); DEFINE_PER_CPU(struct context_tracking, context_tracking); EXPORT_SYMBOL_GPL(context_tracking); -static bool context_tracking_recursion_enter(void) +static noinstr bool context_tracking_recursion_enter(void) { int recursion; @@ -45,7 +45,7 @@ static bool context_tracking_recursion_enter(void) return false; } -static void context_tracking_recursion_exit(void) +static __always_inline void context_tracking_recursion_exit(void) { __this_cpu_dec(context_tracking.recursion); } @@ -59,7 +59,7 @@ static void context_tracking_recursion_exit(void) * instructions to execute won't use any RCU read side critical section * because this function sets RCU in extended quiescent state. */ -void __context_tracking_enter(enum ctx_state state) +void noinstr __context_tracking_enter(enum ctx_state state) { /* Kernel threads aren't supposed to go to userspace */ WARN_ON_ONCE(!current->mm); @@ -77,8 +77,10 @@ void __context_tracking_enter(enum ctx_state state) * on the tick. */ if (state == CONTEXT_USER) { + instrumentation_begin(); trace_user_enter(0); vtime_user_enter(current); + instrumentation_end(); } rcu_user_enter(); } @@ -99,7 +101,6 @@ void __context_tracking_enter(enum ctx_state state) } context_tracking_recursion_exit(); } -NOKPROBE_SYMBOL(__context_tracking_enter); EXPORT_SYMBOL_GPL(__context_tracking_enter); void context_tracking_enter(enum ctx_state state) @@ -142,7 +143,7 @@ NOKPROBE_SYMBOL(context_tracking_user_enter); * This call supports re-entrancy. This way it can be called from any exception * handler without needing to know if we came from userspace or not. */ -void __context_tracking_exit(enum ctx_state state) +void noinstr __context_tracking_exit(enum ctx_state state) { if (!context_tracking_recursion_enter()) return; @@ -155,15 +156,16 @@ void __context_tracking_exit(enum ctx_state state) */ rcu_user_exit(); if (state == CONTEXT_USER) { + instrumentation_begin(); vtime_user_exit(current); trace_user_exit(0); + instrumentation_end(); } } __this_cpu_write(context_tracking.state, CONTEXT_KERNEL); } context_tracking_recursion_exit(); } -NOKPROBE_SYMBOL(__context_tracking_exit); EXPORT_SYMBOL_GPL(__context_tracking_exit); void context_tracking_exit(enum ctx_state state) diff --git a/kernel/kcov.c b/kernel/kcov.c index 55c5d883a93e..6afae0bcbac4 100644 --- a/kernel/kcov.c +++ b/kernel/kcov.c @@ -427,7 +427,8 @@ void kcov_task_exit(struct task_struct *t) * WARN_ON(!kcov->remote && kcov->t != t); * * For KCOV_REMOTE_ENABLE devices, the exiting task is either: - * 2. A remote task between kcov_remote_start() and kcov_remote_stop(). + * + * 1. A remote task between kcov_remote_start() and kcov_remote_stop(). * In this case we should print a warning right away, since a task * shouldn't be exiting when it's in a kcov coverage collection * section. Here t points to the task that is collecting remote @@ -437,7 +438,7 @@ void kcov_task_exit(struct task_struct *t) * WARN_ON(kcov->remote && kcov->t != t); * * 2. The task that created kcov exiting without calling KCOV_DISABLE, - * and then again we can make sure that t->kcov->t == t: + * and then again we make sure that t->kcov->t == t: * WARN_ON(kcov->remote && kcov->t != t); * * By combining all three checks into one we get: @@ -764,7 +765,7 @@ static const struct file_operations kcov_fops = { * Internally, kcov_remote_start() looks up the kcov device associated with the * provided handle, allocates an area for coverage collection, and saves the * pointers to kcov and area into the current task_struct to allow coverage to - * be collected via __sanitizer_cov_trace_pc() + * be collected via __sanitizer_cov_trace_pc(). * In turns kcov_remote_stop() clears those pointers from task_struct to stop * collecting coverage and copies all collected coverage into the kcov area. */ @@ -972,16 +973,25 @@ void kcov_remote_stop(void) local_irq_restore(flags); return; } - kcov = t->kcov; - area = t->kcov_area; - size = t->kcov_size; - sequence = t->kcov_sequence; - + /* + * When in softirq, check if the corresponding kcov_remote_start() + * actually found the remote handle and started collecting coverage. + */ + if (in_serving_softirq() && !t->kcov_softirq) { + local_irq_restore(flags); + return; + } + /* Make sure that kcov_softirq is only set when in softirq. */ if (WARN_ON(!in_serving_softirq() && t->kcov_softirq)) { local_irq_restore(flags); return; } + kcov = t->kcov; + area = t->kcov_area; + size = t->kcov_size; + sequence = t->kcov_sequence; + kcov_stop(t); if (in_serving_softirq()) { t->kcov_softirq = 0; diff --git a/kernel/kcsan/Makefile b/kernel/kcsan/Makefile new file mode 100644 index 000000000000..d4999b38d1be --- /dev/null +++ b/kernel/kcsan/Makefile @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +KCSAN_SANITIZE := n +KCOV_INSTRUMENT := n +UBSAN_SANITIZE := n + +CFLAGS_REMOVE_core.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_debugfs.o = $(CC_FLAGS_FTRACE) +CFLAGS_REMOVE_report.o = $(CC_FLAGS_FTRACE) + +CFLAGS_core.o := $(call cc-option,-fno-conserve-stack,) \ + $(call cc-option,-fno-stack-protector,) + +obj-y := core.o debugfs.o report.o +obj-$(CONFIG_KCSAN_SELFTEST) += test.o diff --git a/kernel/kcsan/atomic.h b/kernel/kcsan/atomic.h new file mode 100644 index 000000000000..be9e625227f3 --- /dev/null +++ b/kernel/kcsan/atomic.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _KERNEL_KCSAN_ATOMIC_H +#define _KERNEL_KCSAN_ATOMIC_H + +#include <linux/jiffies.h> +#include <linux/sched.h> + +/* + * Special rules for certain memory where concurrent conflicting accesses are + * common, however, the current convention is to not mark them; returns true if + * access to @ptr should be considered atomic. Called from slow-path. + */ +static bool kcsan_is_atomic_special(const volatile void *ptr) +{ + /* volatile globals that have been observed in data races. */ + return ptr == &jiffies || ptr == ¤t->state; +} + +#endif /* _KERNEL_KCSAN_ATOMIC_H */ diff --git a/kernel/kcsan/core.c b/kernel/kcsan/core.c new file mode 100644 index 000000000000..15f67949d11e --- /dev/null +++ b/kernel/kcsan/core.c @@ -0,0 +1,850 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/atomic.h> +#include <linux/bug.h> +#include <linux/delay.h> +#include <linux/export.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/moduleparam.h> +#include <linux/percpu.h> +#include <linux/preempt.h> +#include <linux/random.h> +#include <linux/sched.h> +#include <linux/uaccess.h> + +#include "atomic.h" +#include "encoding.h" +#include "kcsan.h" + +static bool kcsan_early_enable = IS_ENABLED(CONFIG_KCSAN_EARLY_ENABLE); +unsigned int kcsan_udelay_task = CONFIG_KCSAN_UDELAY_TASK; +unsigned int kcsan_udelay_interrupt = CONFIG_KCSAN_UDELAY_INTERRUPT; +static long kcsan_skip_watch = CONFIG_KCSAN_SKIP_WATCH; +static bool kcsan_interrupt_watcher = IS_ENABLED(CONFIG_KCSAN_INTERRUPT_WATCHER); + +#ifdef MODULE_PARAM_PREFIX +#undef MODULE_PARAM_PREFIX +#endif +#define MODULE_PARAM_PREFIX "kcsan." +module_param_named(early_enable, kcsan_early_enable, bool, 0); +module_param_named(udelay_task, kcsan_udelay_task, uint, 0644); +module_param_named(udelay_interrupt, kcsan_udelay_interrupt, uint, 0644); +module_param_named(skip_watch, kcsan_skip_watch, long, 0644); +module_param_named(interrupt_watcher, kcsan_interrupt_watcher, bool, 0444); + +bool kcsan_enabled; + +/* Per-CPU kcsan_ctx for interrupts */ +static DEFINE_PER_CPU(struct kcsan_ctx, kcsan_cpu_ctx) = { + .disable_count = 0, + .atomic_next = 0, + .atomic_nest_count = 0, + .in_flat_atomic = false, + .access_mask = 0, + .scoped_accesses = {LIST_POISON1, NULL}, +}; + +/* + * Helper macros to index into adjacent slots, starting from address slot + * itself, followed by the right and left slots. + * + * The purpose is 2-fold: + * + * 1. if during insertion the address slot is already occupied, check if + * any adjacent slots are free; + * 2. accesses that straddle a slot boundary due to size that exceeds a + * slot's range may check adjacent slots if any watchpoint matches. + * + * Note that accesses with very large size may still miss a watchpoint; however, + * given this should be rare, this is a reasonable trade-off to make, since this + * will avoid: + * + * 1. excessive contention between watchpoint checks and setup; + * 2. larger number of simultaneous watchpoints without sacrificing + * performance. + * + * Example: SLOT_IDX values for KCSAN_CHECK_ADJACENT=1, where i is [0, 1, 2]: + * + * slot=0: [ 1, 2, 0] + * slot=9: [10, 11, 9] + * slot=63: [64, 65, 63] + */ +#define SLOT_IDX(slot, i) (slot + ((i + KCSAN_CHECK_ADJACENT) % NUM_SLOTS)) + +/* + * SLOT_IDX_FAST is used in the fast-path. Not first checking the address's primary + * slot (middle) is fine if we assume that races occur rarely. The set of + * indices {SLOT_IDX(slot, i) | i in [0, NUM_SLOTS)} is equivalent to + * {SLOT_IDX_FAST(slot, i) | i in [0, NUM_SLOTS)}. + */ +#define SLOT_IDX_FAST(slot, i) (slot + i) + +/* + * Watchpoints, with each entry encoded as defined in encoding.h: in order to be + * able to safely update and access a watchpoint without introducing locking + * overhead, we encode each watchpoint as a single atomic long. The initial + * zero-initialized state matches INVALID_WATCHPOINT. + * + * Add NUM_SLOTS-1 entries to account for overflow; this helps avoid having to + * use more complicated SLOT_IDX_FAST calculation with modulo in the fast-path. + */ +static atomic_long_t watchpoints[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1]; + +/* + * Instructions to skip watching counter, used in should_watch(). We use a + * per-CPU counter to avoid excessive contention. + */ +static DEFINE_PER_CPU(long, kcsan_skip); + +static __always_inline atomic_long_t *find_watchpoint(unsigned long addr, + size_t size, + bool expect_write, + long *encoded_watchpoint) +{ + const int slot = watchpoint_slot(addr); + const unsigned long addr_masked = addr & WATCHPOINT_ADDR_MASK; + atomic_long_t *watchpoint; + unsigned long wp_addr_masked; + size_t wp_size; + bool is_write; + int i; + + BUILD_BUG_ON(CONFIG_KCSAN_NUM_WATCHPOINTS < NUM_SLOTS); + + for (i = 0; i < NUM_SLOTS; ++i) { + watchpoint = &watchpoints[SLOT_IDX_FAST(slot, i)]; + *encoded_watchpoint = atomic_long_read(watchpoint); + if (!decode_watchpoint(*encoded_watchpoint, &wp_addr_masked, + &wp_size, &is_write)) + continue; + + if (expect_write && !is_write) + continue; + + /* Check if the watchpoint matches the access. */ + if (matching_access(wp_addr_masked, wp_size, addr_masked, size)) + return watchpoint; + } + + return NULL; +} + +static inline atomic_long_t * +insert_watchpoint(unsigned long addr, size_t size, bool is_write) +{ + const int slot = watchpoint_slot(addr); + const long encoded_watchpoint = encode_watchpoint(addr, size, is_write); + atomic_long_t *watchpoint; + int i; + + /* Check slot index logic, ensuring we stay within array bounds. */ + BUILD_BUG_ON(SLOT_IDX(0, 0) != KCSAN_CHECK_ADJACENT); + BUILD_BUG_ON(SLOT_IDX(0, KCSAN_CHECK_ADJACENT+1) != 0); + BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT) != ARRAY_SIZE(watchpoints)-1); + BUILD_BUG_ON(SLOT_IDX(CONFIG_KCSAN_NUM_WATCHPOINTS-1, KCSAN_CHECK_ADJACENT+1) != ARRAY_SIZE(watchpoints) - NUM_SLOTS); + + for (i = 0; i < NUM_SLOTS; ++i) { + long expect_val = INVALID_WATCHPOINT; + + /* Try to acquire this slot. */ + watchpoint = &watchpoints[SLOT_IDX(slot, i)]; + if (atomic_long_try_cmpxchg_relaxed(watchpoint, &expect_val, encoded_watchpoint)) + return watchpoint; + } + + return NULL; +} + +/* + * Return true if watchpoint was successfully consumed, false otherwise. + * + * This may return false if: + * + * 1. another thread already consumed the watchpoint; + * 2. the thread that set up the watchpoint already removed it; + * 3. the watchpoint was removed and then re-used. + */ +static __always_inline bool +try_consume_watchpoint(atomic_long_t *watchpoint, long encoded_watchpoint) +{ + return atomic_long_try_cmpxchg_relaxed(watchpoint, &encoded_watchpoint, CONSUMED_WATCHPOINT); +} + +/* Return true if watchpoint was not touched, false if already consumed. */ +static inline bool consume_watchpoint(atomic_long_t *watchpoint) +{ + return atomic_long_xchg_relaxed(watchpoint, CONSUMED_WATCHPOINT) != CONSUMED_WATCHPOINT; +} + +/* Remove the watchpoint -- its slot may be reused after. */ +static inline void remove_watchpoint(atomic_long_t *watchpoint) +{ + atomic_long_set(watchpoint, INVALID_WATCHPOINT); +} + +static __always_inline struct kcsan_ctx *get_ctx(void) +{ + /* + * In interrupts, use raw_cpu_ptr to avoid unnecessary checks, that would + * also result in calls that generate warnings in uaccess regions. + */ + return in_task() ? ¤t->kcsan_ctx : raw_cpu_ptr(&kcsan_cpu_ctx); +} + +/* Check scoped accesses; never inline because this is a slow-path! */ +static noinline void kcsan_check_scoped_accesses(void) +{ + struct kcsan_ctx *ctx = get_ctx(); + struct list_head *prev_save = ctx->scoped_accesses.prev; + struct kcsan_scoped_access *scoped_access; + + ctx->scoped_accesses.prev = NULL; /* Avoid recursion. */ + list_for_each_entry(scoped_access, &ctx->scoped_accesses, list) + __kcsan_check_access(scoped_access->ptr, scoped_access->size, scoped_access->type); + ctx->scoped_accesses.prev = prev_save; +} + +/* Rules for generic atomic accesses. Called from fast-path. */ +static __always_inline bool +is_atomic(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx) +{ + if (type & KCSAN_ACCESS_ATOMIC) + return true; + + /* + * Unless explicitly declared atomic, never consider an assertion access + * as atomic. This allows using them also in atomic regions, such as + * seqlocks, without implicitly changing their semantics. + */ + if (type & KCSAN_ACCESS_ASSERT) + return false; + + if (IS_ENABLED(CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC) && + (type & KCSAN_ACCESS_WRITE) && size <= sizeof(long) && + IS_ALIGNED((unsigned long)ptr, size)) + return true; /* Assume aligned writes up to word size are atomic. */ + + if (ctx->atomic_next > 0) { + /* + * Because we do not have separate contexts for nested + * interrupts, in case atomic_next is set, we simply assume that + * the outer interrupt set atomic_next. In the worst case, we + * will conservatively consider operations as atomic. This is a + * reasonable trade-off to make, since this case should be + * extremely rare; however, even if extremely rare, it could + * lead to false positives otherwise. + */ + if ((hardirq_count() >> HARDIRQ_SHIFT) < 2) + --ctx->atomic_next; /* in task, or outer interrupt */ + return true; + } + + return ctx->atomic_nest_count > 0 || ctx->in_flat_atomic; +} + +static __always_inline bool +should_watch(const volatile void *ptr, size_t size, int type, struct kcsan_ctx *ctx) +{ + /* + * Never set up watchpoints when memory operations are atomic. + * + * Need to check this first, before kcsan_skip check below: (1) atomics + * should not count towards skipped instructions, and (2) to actually + * decrement kcsan_atomic_next for consecutive instruction stream. + */ + if (is_atomic(ptr, size, type, ctx)) + return false; + + if (this_cpu_dec_return(kcsan_skip) >= 0) + return false; + + /* + * NOTE: If we get here, kcsan_skip must always be reset in slow path + * via reset_kcsan_skip() to avoid underflow. + */ + + /* this operation should be watched */ + return true; +} + +static inline void reset_kcsan_skip(void) +{ + long skip_count = kcsan_skip_watch - + (IS_ENABLED(CONFIG_KCSAN_SKIP_WATCH_RANDOMIZE) ? + prandom_u32_max(kcsan_skip_watch) : + 0); + this_cpu_write(kcsan_skip, skip_count); +} + +static __always_inline bool kcsan_is_enabled(void) +{ + return READ_ONCE(kcsan_enabled) && get_ctx()->disable_count == 0; +} + +static inline unsigned int get_delay(void) +{ + unsigned int delay = in_task() ? kcsan_udelay_task : kcsan_udelay_interrupt; + return delay - (IS_ENABLED(CONFIG_KCSAN_DELAY_RANDOMIZE) ? + prandom_u32_max(delay) : + 0); +} + +/* + * Pull everything together: check_access() below contains the performance + * critical operations; the fast-path (including check_access) functions should + * all be inlinable by the instrumentation functions. + * + * The slow-path (kcsan_found_watchpoint, kcsan_setup_watchpoint) are + * non-inlinable -- note that, we prefix these with "kcsan_" to ensure they can + * be filtered from the stacktrace, as well as give them unique names for the + * UACCESS whitelist of objtool. Each function uses user_access_save/restore(), + * since they do not access any user memory, but instrumentation is still + * emitted in UACCESS regions. + */ + +static noinline void kcsan_found_watchpoint(const volatile void *ptr, + size_t size, + int type, + atomic_long_t *watchpoint, + long encoded_watchpoint) +{ + unsigned long flags; + bool consumed; + + if (!kcsan_is_enabled()) + return; + + /* + * The access_mask check relies on value-change comparison. To avoid + * reporting a race where e.g. the writer set up the watchpoint, but the + * reader has access_mask!=0, we have to ignore the found watchpoint. + */ + if (get_ctx()->access_mask != 0) + return; + + /* + * Consume the watchpoint as soon as possible, to minimize the chances + * of !consumed. Consuming the watchpoint must always be guarded by + * kcsan_is_enabled() check, as otherwise we might erroneously + * triggering reports when disabled. + */ + consumed = try_consume_watchpoint(watchpoint, encoded_watchpoint); + + /* keep this after try_consume_watchpoint */ + flags = user_access_save(); + + if (consumed) { + kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_MAYBE, + KCSAN_REPORT_CONSUMED_WATCHPOINT, + watchpoint - watchpoints); + } else { + /* + * The other thread may not print any diagnostics, as it has + * already removed the watchpoint, or another thread consumed + * the watchpoint before this thread. + */ + kcsan_counter_inc(KCSAN_COUNTER_REPORT_RACES); + } + + if ((type & KCSAN_ACCESS_ASSERT) != 0) + kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES); + else + kcsan_counter_inc(KCSAN_COUNTER_DATA_RACES); + + user_access_restore(flags); +} + +static noinline void +kcsan_setup_watchpoint(const volatile void *ptr, size_t size, int type) +{ + const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0; + const bool is_assert = (type & KCSAN_ACCESS_ASSERT) != 0; + atomic_long_t *watchpoint; + union { + u8 _1; + u16 _2; + u32 _4; + u64 _8; + } expect_value; + unsigned long access_mask; + enum kcsan_value_change value_change = KCSAN_VALUE_CHANGE_MAYBE; + unsigned long ua_flags = user_access_save(); + unsigned long irq_flags = 0; + + /* + * Always reset kcsan_skip counter in slow-path to avoid underflow; see + * should_watch(). + */ + reset_kcsan_skip(); + + if (!kcsan_is_enabled()) + goto out; + + /* + * Special atomic rules: unlikely to be true, so we check them here in + * the slow-path, and not in the fast-path in is_atomic(). Call after + * kcsan_is_enabled(), as we may access memory that is not yet + * initialized during early boot. + */ + if (!is_assert && kcsan_is_atomic_special(ptr)) + goto out; + + if (!check_encodable((unsigned long)ptr, size)) { + kcsan_counter_inc(KCSAN_COUNTER_UNENCODABLE_ACCESSES); + goto out; + } + + if (!kcsan_interrupt_watcher) + /* Use raw to avoid lockdep recursion via IRQ flags tracing. */ + raw_local_irq_save(irq_flags); + + watchpoint = insert_watchpoint((unsigned long)ptr, size, is_write); + if (watchpoint == NULL) { + /* + * Out of capacity: the size of 'watchpoints', and the frequency + * with which should_watch() returns true should be tweaked so + * that this case happens very rarely. + */ + kcsan_counter_inc(KCSAN_COUNTER_NO_CAPACITY); + goto out_unlock; + } + + kcsan_counter_inc(KCSAN_COUNTER_SETUP_WATCHPOINTS); + kcsan_counter_inc(KCSAN_COUNTER_USED_WATCHPOINTS); + + /* + * Read the current value, to later check and infer a race if the data + * was modified via a non-instrumented access, e.g. from a device. + */ + expect_value._8 = 0; + switch (size) { + case 1: + expect_value._1 = READ_ONCE(*(const u8 *)ptr); + break; + case 2: + expect_value._2 = READ_ONCE(*(const u16 *)ptr); + break; + case 4: + expect_value._4 = READ_ONCE(*(const u32 *)ptr); + break; + case 8: + expect_value._8 = READ_ONCE(*(const u64 *)ptr); + break; + default: + break; /* ignore; we do not diff the values */ + } + + if (IS_ENABLED(CONFIG_KCSAN_DEBUG)) { + kcsan_disable_current(); + pr_err("KCSAN: watching %s, size: %zu, addr: %px [slot: %d, encoded: %lx]\n", + is_write ? "write" : "read", size, ptr, + watchpoint_slot((unsigned long)ptr), + encode_watchpoint((unsigned long)ptr, size, is_write)); + kcsan_enable_current(); + } + + /* + * Delay this thread, to increase probability of observing a racy + * conflicting access. + */ + udelay(get_delay()); + + /* + * Re-read value, and check if it is as expected; if not, we infer a + * racy access. + */ + access_mask = get_ctx()->access_mask; + switch (size) { + case 1: + expect_value._1 ^= READ_ONCE(*(const u8 *)ptr); + if (access_mask) + expect_value._1 &= (u8)access_mask; + break; + case 2: + expect_value._2 ^= READ_ONCE(*(const u16 *)ptr); + if (access_mask) + expect_value._2 &= (u16)access_mask; + break; + case 4: + expect_value._4 ^= READ_ONCE(*(const u32 *)ptr); + if (access_mask) + expect_value._4 &= (u32)access_mask; + break; + case 8: + expect_value._8 ^= READ_ONCE(*(const u64 *)ptr); + if (access_mask) + expect_value._8 &= (u64)access_mask; + break; + default: + break; /* ignore; we do not diff the values */ + } + + /* Were we able to observe a value-change? */ + if (expect_value._8 != 0) + value_change = KCSAN_VALUE_CHANGE_TRUE; + + /* Check if this access raced with another. */ + if (!consume_watchpoint(watchpoint)) { + /* + * Depending on the access type, map a value_change of MAYBE to + * TRUE (always report) or FALSE (never report). + */ + if (value_change == KCSAN_VALUE_CHANGE_MAYBE) { + if (access_mask != 0) { + /* + * For access with access_mask, we require a + * value-change, as it is likely that races on + * ~access_mask bits are expected. + */ + value_change = KCSAN_VALUE_CHANGE_FALSE; + } else if (size > 8 || is_assert) { + /* Always assume a value-change. */ + value_change = KCSAN_VALUE_CHANGE_TRUE; + } + } + + /* + * No need to increment 'data_races' counter, as the racing + * thread already did. + * + * Count 'assert_failures' for each failed ASSERT access, + * therefore both this thread and the racing thread may + * increment this counter. + */ + if (is_assert && value_change == KCSAN_VALUE_CHANGE_TRUE) + kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES); + + kcsan_report(ptr, size, type, value_change, KCSAN_REPORT_RACE_SIGNAL, + watchpoint - watchpoints); + } else if (value_change == KCSAN_VALUE_CHANGE_TRUE) { + /* Inferring a race, since the value should not have changed. */ + + kcsan_counter_inc(KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN); + if (is_assert) + kcsan_counter_inc(KCSAN_COUNTER_ASSERT_FAILURES); + + if (IS_ENABLED(CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN) || is_assert) + kcsan_report(ptr, size, type, KCSAN_VALUE_CHANGE_TRUE, + KCSAN_REPORT_RACE_UNKNOWN_ORIGIN, + watchpoint - watchpoints); + } + + /* + * Remove watchpoint; must be after reporting, since the slot may be + * reused after this point. + */ + remove_watchpoint(watchpoint); + kcsan_counter_dec(KCSAN_COUNTER_USED_WATCHPOINTS); +out_unlock: + if (!kcsan_interrupt_watcher) + raw_local_irq_restore(irq_flags); +out: + user_access_restore(ua_flags); +} + +static __always_inline void check_access(const volatile void *ptr, size_t size, + int type) +{ + const bool is_write = (type & KCSAN_ACCESS_WRITE) != 0; + atomic_long_t *watchpoint; + long encoded_watchpoint; + + /* + * Do nothing for 0 sized check; this comparison will be optimized out + * for constant sized instrumentation (__tsan_{read,write}N). + */ + if (unlikely(size == 0)) + return; + + /* + * Avoid user_access_save in fast-path: find_watchpoint is safe without + * user_access_save, as the address that ptr points to is only used to + * check if a watchpoint exists; ptr is never dereferenced. + */ + watchpoint = find_watchpoint((unsigned long)ptr, size, !is_write, + &encoded_watchpoint); + /* + * It is safe to check kcsan_is_enabled() after find_watchpoint in the + * slow-path, as long as no state changes that cause a race to be + * detected and reported have occurred until kcsan_is_enabled() is + * checked. + */ + + if (unlikely(watchpoint != NULL)) + kcsan_found_watchpoint(ptr, size, type, watchpoint, + encoded_watchpoint); + else { + struct kcsan_ctx *ctx = get_ctx(); /* Call only once in fast-path. */ + + if (unlikely(should_watch(ptr, size, type, ctx))) + kcsan_setup_watchpoint(ptr, size, type); + else if (unlikely(ctx->scoped_accesses.prev)) + kcsan_check_scoped_accesses(); + } +} + +/* === Public interface ===================================================== */ + +void __init kcsan_init(void) +{ + BUG_ON(!in_task()); + + kcsan_debugfs_init(); + + /* + * We are in the init task, and no other tasks should be running; + * WRITE_ONCE without memory barrier is sufficient. + */ + if (kcsan_early_enable) + WRITE_ONCE(kcsan_enabled, true); +} + +/* === Exported interface =================================================== */ + +void kcsan_disable_current(void) +{ + ++get_ctx()->disable_count; +} +EXPORT_SYMBOL(kcsan_disable_current); + +void kcsan_enable_current(void) +{ + if (get_ctx()->disable_count-- == 0) { + /* + * Warn if kcsan_enable_current() calls are unbalanced with + * kcsan_disable_current() calls, which causes disable_count to + * become negative and should not happen. + */ + kcsan_disable_current(); /* restore to 0, KCSAN still enabled */ + kcsan_disable_current(); /* disable to generate warning */ + WARN(1, "Unbalanced %s()", __func__); + kcsan_enable_current(); + } +} +EXPORT_SYMBOL(kcsan_enable_current); + +void kcsan_enable_current_nowarn(void) +{ + if (get_ctx()->disable_count-- == 0) + kcsan_disable_current(); +} +EXPORT_SYMBOL(kcsan_enable_current_nowarn); + +void kcsan_nestable_atomic_begin(void) +{ + /* + * Do *not* check and warn if we are in a flat atomic region: nestable + * and flat atomic regions are independent from each other. + * See include/linux/kcsan.h: struct kcsan_ctx comments for more + * comments. + */ + + ++get_ctx()->atomic_nest_count; +} +EXPORT_SYMBOL(kcsan_nestable_atomic_begin); + +void kcsan_nestable_atomic_end(void) +{ + if (get_ctx()->atomic_nest_count-- == 0) { + /* + * Warn if kcsan_nestable_atomic_end() calls are unbalanced with + * kcsan_nestable_atomic_begin() calls, which causes + * atomic_nest_count to become negative and should not happen. + */ + kcsan_nestable_atomic_begin(); /* restore to 0 */ + kcsan_disable_current(); /* disable to generate warning */ + WARN(1, "Unbalanced %s()", __func__); + kcsan_enable_current(); + } +} +EXPORT_SYMBOL(kcsan_nestable_atomic_end); + +void kcsan_flat_atomic_begin(void) +{ + get_ctx()->in_flat_atomic = true; +} +EXPORT_SYMBOL(kcsan_flat_atomic_begin); + +void kcsan_flat_atomic_end(void) +{ + get_ctx()->in_flat_atomic = false; +} +EXPORT_SYMBOL(kcsan_flat_atomic_end); + +void kcsan_atomic_next(int n) +{ + get_ctx()->atomic_next = n; +} +EXPORT_SYMBOL(kcsan_atomic_next); + +void kcsan_set_access_mask(unsigned long mask) +{ + get_ctx()->access_mask = mask; +} +EXPORT_SYMBOL(kcsan_set_access_mask); + +struct kcsan_scoped_access * +kcsan_begin_scoped_access(const volatile void *ptr, size_t size, int type, + struct kcsan_scoped_access *sa) +{ + struct kcsan_ctx *ctx = get_ctx(); + + __kcsan_check_access(ptr, size, type); + + ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */ + + INIT_LIST_HEAD(&sa->list); + sa->ptr = ptr; + sa->size = size; + sa->type = type; + + if (!ctx->scoped_accesses.prev) /* Lazy initialize list head. */ + INIT_LIST_HEAD(&ctx->scoped_accesses); + list_add(&sa->list, &ctx->scoped_accesses); + + ctx->disable_count--; + return sa; +} +EXPORT_SYMBOL(kcsan_begin_scoped_access); + +void kcsan_end_scoped_access(struct kcsan_scoped_access *sa) +{ + struct kcsan_ctx *ctx = get_ctx(); + + if (WARN(!ctx->scoped_accesses.prev, "Unbalanced %s()?", __func__)) + return; + + ctx->disable_count++; /* Disable KCSAN, in case list debugging is on. */ + + list_del(&sa->list); + if (list_empty(&ctx->scoped_accesses)) + /* + * Ensure we do not enter kcsan_check_scoped_accesses() + * slow-path if unnecessary, and avoids requiring list_empty() + * in the fast-path (to avoid a READ_ONCE() and potential + * uaccess warning). + */ + ctx->scoped_accesses.prev = NULL; + + ctx->disable_count--; + + __kcsan_check_access(sa->ptr, sa->size, sa->type); +} +EXPORT_SYMBOL(kcsan_end_scoped_access); + +void __kcsan_check_access(const volatile void *ptr, size_t size, int type) +{ + check_access(ptr, size, type); +} +EXPORT_SYMBOL(__kcsan_check_access); + +/* + * KCSAN uses the same instrumentation that is emitted by supported compilers + * for ThreadSanitizer (TSAN). + * + * When enabled, the compiler emits instrumentation calls (the functions + * prefixed with "__tsan" below) for all loads and stores that it generated; + * inline asm is not instrumented. + * + * Note that, not all supported compiler versions distinguish aligned/unaligned + * accesses, but e.g. recent versions of Clang do. We simply alias the unaligned + * version to the generic version, which can handle both. + */ + +#define DEFINE_TSAN_READ_WRITE(size) \ + void __tsan_read##size(void *ptr) \ + { \ + check_access(ptr, size, 0); \ + } \ + EXPORT_SYMBOL(__tsan_read##size); \ + void __tsan_unaligned_read##size(void *ptr) \ + __alias(__tsan_read##size); \ + EXPORT_SYMBOL(__tsan_unaligned_read##size); \ + void __tsan_write##size(void *ptr) \ + { \ + check_access(ptr, size, KCSAN_ACCESS_WRITE); \ + } \ + EXPORT_SYMBOL(__tsan_write##size); \ + void __tsan_unaligned_write##size(void *ptr) \ + __alias(__tsan_write##size); \ + EXPORT_SYMBOL(__tsan_unaligned_write##size) + +DEFINE_TSAN_READ_WRITE(1); +DEFINE_TSAN_READ_WRITE(2); +DEFINE_TSAN_READ_WRITE(4); +DEFINE_TSAN_READ_WRITE(8); +DEFINE_TSAN_READ_WRITE(16); + +void __tsan_read_range(void *ptr, size_t size) +{ + check_access(ptr, size, 0); +} +EXPORT_SYMBOL(__tsan_read_range); + +void __tsan_write_range(void *ptr, size_t size) +{ + check_access(ptr, size, KCSAN_ACCESS_WRITE); +} +EXPORT_SYMBOL(__tsan_write_range); + +/* + * Use of explicit volatile is generally disallowed [1], however, volatile is + * still used in various concurrent context, whether in low-level + * synchronization primitives or for legacy reasons. + * [1] https://lwn.net/Articles/233479/ + * + * We only consider volatile accesses atomic if they are aligned and would pass + * the size-check of compiletime_assert_rwonce_type(). + */ +#define DEFINE_TSAN_VOLATILE_READ_WRITE(size) \ + void __tsan_volatile_read##size(void *ptr) \ + { \ + const bool is_atomic = size <= sizeof(long long) && \ + IS_ALIGNED((unsigned long)ptr, size); \ + if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic) \ + return; \ + check_access(ptr, size, is_atomic ? KCSAN_ACCESS_ATOMIC : 0); \ + } \ + EXPORT_SYMBOL(__tsan_volatile_read##size); \ + void __tsan_unaligned_volatile_read##size(void *ptr) \ + __alias(__tsan_volatile_read##size); \ + EXPORT_SYMBOL(__tsan_unaligned_volatile_read##size); \ + void __tsan_volatile_write##size(void *ptr) \ + { \ + const bool is_atomic = size <= sizeof(long long) && \ + IS_ALIGNED((unsigned long)ptr, size); \ + if (IS_ENABLED(CONFIG_KCSAN_IGNORE_ATOMICS) && is_atomic) \ + return; \ + check_access(ptr, size, \ + KCSAN_ACCESS_WRITE | \ + (is_atomic ? KCSAN_ACCESS_ATOMIC : 0)); \ + } \ + EXPORT_SYMBOL(__tsan_volatile_write##size); \ + void __tsan_unaligned_volatile_write##size(void *ptr) \ + __alias(__tsan_volatile_write##size); \ + EXPORT_SYMBOL(__tsan_unaligned_volatile_write##size) + +DEFINE_TSAN_VOLATILE_READ_WRITE(1); +DEFINE_TSAN_VOLATILE_READ_WRITE(2); +DEFINE_TSAN_VOLATILE_READ_WRITE(4); +DEFINE_TSAN_VOLATILE_READ_WRITE(8); +DEFINE_TSAN_VOLATILE_READ_WRITE(16); + +/* + * The below are not required by KCSAN, but can still be emitted by the + * compiler. + */ +void __tsan_func_entry(void *call_pc) +{ +} +EXPORT_SYMBOL(__tsan_func_entry); +void __tsan_func_exit(void) +{ +} +EXPORT_SYMBOL(__tsan_func_exit); +void __tsan_init(void) +{ +} +EXPORT_SYMBOL(__tsan_init); diff --git a/kernel/kcsan/debugfs.c b/kernel/kcsan/debugfs.c new file mode 100644 index 000000000000..023e49c58d55 --- /dev/null +++ b/kernel/kcsan/debugfs.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/atomic.h> +#include <linux/bsearch.h> +#include <linux/bug.h> +#include <linux/debugfs.h> +#include <linux/init.h> +#include <linux/kallsyms.h> +#include <linux/sched.h> +#include <linux/seq_file.h> +#include <linux/slab.h> +#include <linux/sort.h> +#include <linux/string.h> +#include <linux/uaccess.h> + +#include "kcsan.h" + +/* + * Statistics counters. + */ +static atomic_long_t counters[KCSAN_COUNTER_COUNT]; + +/* + * Addresses for filtering functions from reporting. This list can be used as a + * whitelist or blacklist. + */ +static struct { + unsigned long *addrs; /* array of addresses */ + size_t size; /* current size */ + int used; /* number of elements used */ + bool sorted; /* if elements are sorted */ + bool whitelist; /* if list is a blacklist or whitelist */ +} report_filterlist = { + .addrs = NULL, + .size = 8, /* small initial size */ + .used = 0, + .sorted = false, + .whitelist = false, /* default is blacklist */ +}; +static DEFINE_SPINLOCK(report_filterlist_lock); + +static const char *counter_to_name(enum kcsan_counter_id id) +{ + switch (id) { + case KCSAN_COUNTER_USED_WATCHPOINTS: return "used_watchpoints"; + case KCSAN_COUNTER_SETUP_WATCHPOINTS: return "setup_watchpoints"; + case KCSAN_COUNTER_DATA_RACES: return "data_races"; + case KCSAN_COUNTER_ASSERT_FAILURES: return "assert_failures"; + case KCSAN_COUNTER_NO_CAPACITY: return "no_capacity"; + case KCSAN_COUNTER_REPORT_RACES: return "report_races"; + case KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN: return "races_unknown_origin"; + case KCSAN_COUNTER_UNENCODABLE_ACCESSES: return "unencodable_accesses"; + case KCSAN_COUNTER_ENCODING_FALSE_POSITIVES: return "encoding_false_positives"; + case KCSAN_COUNTER_COUNT: + BUG(); + } + return NULL; +} + +void kcsan_counter_inc(enum kcsan_counter_id id) +{ + atomic_long_inc(&counters[id]); +} + +void kcsan_counter_dec(enum kcsan_counter_id id) +{ + atomic_long_dec(&counters[id]); +} + +/* + * The microbenchmark allows benchmarking KCSAN core runtime only. To run + * multiple threads, pipe 'microbench=<iters>' from multiple tasks into the + * debugfs file. This will not generate any conflicts, and tests fast-path only. + */ +static noinline void microbenchmark(unsigned long iters) +{ + const struct kcsan_ctx ctx_save = current->kcsan_ctx; + const bool was_enabled = READ_ONCE(kcsan_enabled); + cycles_t cycles; + + /* We may have been called from an atomic region; reset context. */ + memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx)); + /* + * Disable to benchmark fast-path for all accesses, and (expected + * negligible) call into slow-path, but never set up watchpoints. + */ + WRITE_ONCE(kcsan_enabled, false); + + pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters); + + cycles = get_cycles(); + while (iters--) { + unsigned long addr = iters & ((PAGE_SIZE << 8) - 1); + int type = !(iters & 0x7f) ? KCSAN_ACCESS_ATOMIC : + (!(iters & 0xf) ? KCSAN_ACCESS_WRITE : 0); + __kcsan_check_access((void *)addr, sizeof(long), type); + } + cycles = get_cycles() - cycles; + + pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles); + + WRITE_ONCE(kcsan_enabled, was_enabled); + /* restore context */ + current->kcsan_ctx = ctx_save; +} + +/* + * Simple test to create conflicting accesses. Write 'test=<iters>' to KCSAN's + * debugfs file from multiple tasks to generate real conflicts and show reports. + */ +static long test_dummy; +static long test_flags; +static long test_scoped; +static noinline void test_thread(unsigned long iters) +{ + const long CHANGE_BITS = 0xff00ff00ff00ff00L; + const struct kcsan_ctx ctx_save = current->kcsan_ctx; + cycles_t cycles; + + /* We may have been called from an atomic region; reset context. */ + memset(¤t->kcsan_ctx, 0, sizeof(current->kcsan_ctx)); + + pr_info("KCSAN: %s begin | iters: %lu\n", __func__, iters); + pr_info("test_dummy@%px, test_flags@%px, test_scoped@%px,\n", + &test_dummy, &test_flags, &test_scoped); + + cycles = get_cycles(); + while (iters--) { + /* These all should generate reports. */ + __kcsan_check_read(&test_dummy, sizeof(test_dummy)); + ASSERT_EXCLUSIVE_WRITER(test_dummy); + ASSERT_EXCLUSIVE_ACCESS(test_dummy); + + ASSERT_EXCLUSIVE_BITS(test_flags, ~CHANGE_BITS); /* no report */ + __kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */ + + ASSERT_EXCLUSIVE_BITS(test_flags, CHANGE_BITS); /* report */ + __kcsan_check_read(&test_flags, sizeof(test_flags)); /* no report */ + + /* not actually instrumented */ + WRITE_ONCE(test_dummy, iters); /* to observe value-change */ + __kcsan_check_write(&test_dummy, sizeof(test_dummy)); + + test_flags ^= CHANGE_BITS; /* generate value-change */ + __kcsan_check_write(&test_flags, sizeof(test_flags)); + + BUG_ON(current->kcsan_ctx.scoped_accesses.prev); + { + /* Should generate reports anywhere in this block. */ + ASSERT_EXCLUSIVE_WRITER_SCOPED(test_scoped); + ASSERT_EXCLUSIVE_ACCESS_SCOPED(test_scoped); + BUG_ON(!current->kcsan_ctx.scoped_accesses.prev); + /* Unrelated accesses. */ + __kcsan_check_access(&cycles, sizeof(cycles), 0); + __kcsan_check_access(&cycles, sizeof(cycles), KCSAN_ACCESS_ATOMIC); + } + BUG_ON(current->kcsan_ctx.scoped_accesses.prev); + } + cycles = get_cycles() - cycles; + + pr_info("KCSAN: %s end | cycles: %llu\n", __func__, cycles); + + /* restore context */ + current->kcsan_ctx = ctx_save; +} + +static int cmp_filterlist_addrs(const void *rhs, const void *lhs) +{ + const unsigned long a = *(const unsigned long *)rhs; + const unsigned long b = *(const unsigned long *)lhs; + + return a < b ? -1 : a == b ? 0 : 1; +} + +bool kcsan_skip_report_debugfs(unsigned long func_addr) +{ + unsigned long symbolsize, offset; + unsigned long flags; + bool ret = false; + + if (!kallsyms_lookup_size_offset(func_addr, &symbolsize, &offset)) + return false; + func_addr -= offset; /* Get function start */ + + spin_lock_irqsave(&report_filterlist_lock, flags); + if (report_filterlist.used == 0) + goto out; + + /* Sort array if it is unsorted, and then do a binary search. */ + if (!report_filterlist.sorted) { + sort(report_filterlist.addrs, report_filterlist.used, + sizeof(unsigned long), cmp_filterlist_addrs, NULL); + report_filterlist.sorted = true; + } + ret = !!bsearch(&func_addr, report_filterlist.addrs, + report_filterlist.used, sizeof(unsigned long), + cmp_filterlist_addrs); + if (report_filterlist.whitelist) + ret = !ret; + +out: + spin_unlock_irqrestore(&report_filterlist_lock, flags); + return ret; +} + +static void set_report_filterlist_whitelist(bool whitelist) +{ + unsigned long flags; + + spin_lock_irqsave(&report_filterlist_lock, flags); + report_filterlist.whitelist = whitelist; + spin_unlock_irqrestore(&report_filterlist_lock, flags); +} + +/* Returns 0 on success, error-code otherwise. */ +static ssize_t insert_report_filterlist(const char *func) +{ + unsigned long flags; + unsigned long addr = kallsyms_lookup_name(func); + ssize_t ret = 0; + + if (!addr) { + pr_err("KCSAN: could not find function: '%s'\n", func); + return -ENOENT; + } + + spin_lock_irqsave(&report_filterlist_lock, flags); + + if (report_filterlist.addrs == NULL) { + /* initial allocation */ + report_filterlist.addrs = + kmalloc_array(report_filterlist.size, + sizeof(unsigned long), GFP_ATOMIC); + if (report_filterlist.addrs == NULL) { + ret = -ENOMEM; + goto out; + } + } else if (report_filterlist.used == report_filterlist.size) { + /* resize filterlist */ + size_t new_size = report_filterlist.size * 2; + unsigned long *new_addrs = + krealloc(report_filterlist.addrs, + new_size * sizeof(unsigned long), GFP_ATOMIC); + + if (new_addrs == NULL) { + /* leave filterlist itself untouched */ + ret = -ENOMEM; + goto out; + } + + report_filterlist.size = new_size; + report_filterlist.addrs = new_addrs; + } + + /* Note: deduplicating should be done in userspace. */ + report_filterlist.addrs[report_filterlist.used++] = + kallsyms_lookup_name(func); + report_filterlist.sorted = false; + +out: + spin_unlock_irqrestore(&report_filterlist_lock, flags); + + return ret; +} + +static int show_info(struct seq_file *file, void *v) +{ + int i; + unsigned long flags; + + /* show stats */ + seq_printf(file, "enabled: %i\n", READ_ONCE(kcsan_enabled)); + for (i = 0; i < KCSAN_COUNTER_COUNT; ++i) + seq_printf(file, "%s: %ld\n", counter_to_name(i), + atomic_long_read(&counters[i])); + + /* show filter functions, and filter type */ + spin_lock_irqsave(&report_filterlist_lock, flags); + seq_printf(file, "\n%s functions: %s\n", + report_filterlist.whitelist ? "whitelisted" : "blacklisted", + report_filterlist.used == 0 ? "none" : ""); + for (i = 0; i < report_filterlist.used; ++i) + seq_printf(file, " %ps\n", (void *)report_filterlist.addrs[i]); + spin_unlock_irqrestore(&report_filterlist_lock, flags); + + return 0; +} + +static int debugfs_open(struct inode *inode, struct file *file) +{ + return single_open(file, show_info, NULL); +} + +static ssize_t +debugfs_write(struct file *file, const char __user *buf, size_t count, loff_t *off) +{ + char kbuf[KSYM_NAME_LEN]; + char *arg; + int read_len = count < (sizeof(kbuf) - 1) ? count : (sizeof(kbuf) - 1); + + if (copy_from_user(kbuf, buf, read_len)) + return -EFAULT; + kbuf[read_len] = '\0'; + arg = strstrip(kbuf); + + if (!strcmp(arg, "on")) { + WRITE_ONCE(kcsan_enabled, true); + } else if (!strcmp(arg, "off")) { + WRITE_ONCE(kcsan_enabled, false); + } else if (!strncmp(arg, "microbench=", sizeof("microbench=") - 1)) { + unsigned long iters; + + if (kstrtoul(&arg[sizeof("microbench=") - 1], 0, &iters)) + return -EINVAL; + microbenchmark(iters); + } else if (!strncmp(arg, "test=", sizeof("test=") - 1)) { + unsigned long iters; + + if (kstrtoul(&arg[sizeof("test=") - 1], 0, &iters)) + return -EINVAL; + test_thread(iters); + } else if (!strcmp(arg, "whitelist")) { + set_report_filterlist_whitelist(true); + } else if (!strcmp(arg, "blacklist")) { + set_report_filterlist_whitelist(false); + } else if (arg[0] == '!') { + ssize_t ret = insert_report_filterlist(&arg[1]); + + if (ret < 0) + return ret; + } else { + return -EINVAL; + } + + return count; +} + +static const struct file_operations debugfs_ops = +{ + .read = seq_read, + .open = debugfs_open, + .write = debugfs_write, + .release = single_release +}; + +void __init kcsan_debugfs_init(void) +{ + debugfs_create_file("kcsan", 0644, NULL, NULL, &debugfs_ops); +} diff --git a/kernel/kcsan/encoding.h b/kernel/kcsan/encoding.h new file mode 100644 index 000000000000..f03562aaf2eb --- /dev/null +++ b/kernel/kcsan/encoding.h @@ -0,0 +1,95 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef _KERNEL_KCSAN_ENCODING_H +#define _KERNEL_KCSAN_ENCODING_H + +#include <linux/bits.h> +#include <linux/log2.h> +#include <linux/mm.h> + +#include "kcsan.h" + +#define SLOT_RANGE PAGE_SIZE + +#define INVALID_WATCHPOINT 0 +#define CONSUMED_WATCHPOINT 1 + +/* + * The maximum useful size of accesses for which we set up watchpoints is the + * max range of slots we check on an access. + */ +#define MAX_ENCODABLE_SIZE (SLOT_RANGE * (1 + KCSAN_CHECK_ADJACENT)) + +/* + * Number of bits we use to store size info. + */ +#define WATCHPOINT_SIZE_BITS bits_per(MAX_ENCODABLE_SIZE) +/* + * This encoding for addresses discards the upper (1 for is-write + SIZE_BITS); + * however, most 64-bit architectures do not use the full 64-bit address space. + * Also, in order for a false positive to be observable 2 things need to happen: + * + * 1. different addresses but with the same encoded address race; + * 2. and both map onto the same watchpoint slots; + * + * Both these are assumed to be very unlikely. However, in case it still happens + * happens, the report logic will filter out the false positive (see report.c). + */ +#define WATCHPOINT_ADDR_BITS (BITS_PER_LONG-1 - WATCHPOINT_SIZE_BITS) + +/* + * Masks to set/retrieve the encoded data. + */ +#define WATCHPOINT_WRITE_MASK BIT(BITS_PER_LONG-1) +#define WATCHPOINT_SIZE_MASK \ + GENMASK(BITS_PER_LONG-2, BITS_PER_LONG-2 - WATCHPOINT_SIZE_BITS) +#define WATCHPOINT_ADDR_MASK \ + GENMASK(BITS_PER_LONG-3 - WATCHPOINT_SIZE_BITS, 0) + +static inline bool check_encodable(unsigned long addr, size_t size) +{ + return size <= MAX_ENCODABLE_SIZE; +} + +static inline long +encode_watchpoint(unsigned long addr, size_t size, bool is_write) +{ + return (long)((is_write ? WATCHPOINT_WRITE_MASK : 0) | + (size << WATCHPOINT_ADDR_BITS) | + (addr & WATCHPOINT_ADDR_MASK)); +} + +static __always_inline bool decode_watchpoint(long watchpoint, + unsigned long *addr_masked, + size_t *size, + bool *is_write) +{ + if (watchpoint == INVALID_WATCHPOINT || + watchpoint == CONSUMED_WATCHPOINT) + return false; + + *addr_masked = (unsigned long)watchpoint & WATCHPOINT_ADDR_MASK; + *size = ((unsigned long)watchpoint & WATCHPOINT_SIZE_MASK) >> WATCHPOINT_ADDR_BITS; + *is_write = !!((unsigned long)watchpoint & WATCHPOINT_WRITE_MASK); + + return true; +} + +/* + * Return watchpoint slot for an address. + */ +static __always_inline int watchpoint_slot(unsigned long addr) +{ + return (addr / PAGE_SIZE) % CONFIG_KCSAN_NUM_WATCHPOINTS; +} + +static __always_inline bool matching_access(unsigned long addr1, size_t size1, + unsigned long addr2, size_t size2) +{ + unsigned long end_range1 = addr1 + size1 - 1; + unsigned long end_range2 = addr2 + size2 - 1; + + return addr1 <= end_range2 && addr2 <= end_range1; +} + +#endif /* _KERNEL_KCSAN_ENCODING_H */ diff --git a/kernel/kcsan/kcsan.h b/kernel/kcsan/kcsan.h new file mode 100644 index 000000000000..763d6d08d94b --- /dev/null +++ b/kernel/kcsan/kcsan.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * The Kernel Concurrency Sanitizer (KCSAN) infrastructure. For more info please + * see Documentation/dev-tools/kcsan.rst. + */ + +#ifndef _KERNEL_KCSAN_KCSAN_H +#define _KERNEL_KCSAN_KCSAN_H + +#include <linux/kcsan.h> + +/* The number of adjacent watchpoints to check. */ +#define KCSAN_CHECK_ADJACENT 1 +#define NUM_SLOTS (1 + 2*KCSAN_CHECK_ADJACENT) + +extern unsigned int kcsan_udelay_task; +extern unsigned int kcsan_udelay_interrupt; + +/* + * Globally enable and disable KCSAN. + */ +extern bool kcsan_enabled; + +/* + * Initialize debugfs file. + */ +void kcsan_debugfs_init(void); + +enum kcsan_counter_id { + /* + * Number of watchpoints currently in use. + */ + KCSAN_COUNTER_USED_WATCHPOINTS, + + /* + * Total number of watchpoints set up. + */ + KCSAN_COUNTER_SETUP_WATCHPOINTS, + + /* + * Total number of data races. + */ + KCSAN_COUNTER_DATA_RACES, + + /* + * Total number of ASSERT failures due to races. If the observed race is + * due to two conflicting ASSERT type accesses, then both will be + * counted. + */ + KCSAN_COUNTER_ASSERT_FAILURES, + + /* + * Number of times no watchpoints were available. + */ + KCSAN_COUNTER_NO_CAPACITY, + + /* + * A thread checking a watchpoint raced with another checking thread; + * only one will be reported. + */ + KCSAN_COUNTER_REPORT_RACES, + + /* + * Observed data value change, but writer thread unknown. + */ + KCSAN_COUNTER_RACES_UNKNOWN_ORIGIN, + + /* + * The access cannot be encoded to a valid watchpoint. + */ + KCSAN_COUNTER_UNENCODABLE_ACCESSES, + + /* + * Watchpoint encoding caused a watchpoint to fire on mismatching + * accesses. + */ + KCSAN_COUNTER_ENCODING_FALSE_POSITIVES, + + KCSAN_COUNTER_COUNT, /* number of counters */ +}; + +/* + * Increment/decrement counter with given id; avoid calling these in fast-path. + */ +extern void kcsan_counter_inc(enum kcsan_counter_id id); +extern void kcsan_counter_dec(enum kcsan_counter_id id); + +/* + * Returns true if data races in the function symbol that maps to func_addr + * (offsets are ignored) should *not* be reported. + */ +extern bool kcsan_skip_report_debugfs(unsigned long func_addr); + +/* + * Value-change states. + */ +enum kcsan_value_change { + /* + * Did not observe a value-change, however, it is valid to report the + * race, depending on preferences. + */ + KCSAN_VALUE_CHANGE_MAYBE, + + /* + * Did not observe a value-change, and it is invalid to report the race. + */ + KCSAN_VALUE_CHANGE_FALSE, + + /* + * The value was observed to change, and the race should be reported. + */ + KCSAN_VALUE_CHANGE_TRUE, +}; + +enum kcsan_report_type { + /* + * The thread that set up the watchpoint and briefly stalled was + * signalled that another thread triggered the watchpoint. + */ + KCSAN_REPORT_RACE_SIGNAL, + + /* + * A thread found and consumed a matching watchpoint. + */ + KCSAN_REPORT_CONSUMED_WATCHPOINT, + + /* + * No other thread was observed to race with the access, but the data + * value before and after the stall differs. + */ + KCSAN_REPORT_RACE_UNKNOWN_ORIGIN, +}; + +/* + * Print a race report from thread that encountered the race. + */ +extern void kcsan_report(const volatile void *ptr, size_t size, int access_type, + enum kcsan_value_change value_change, + enum kcsan_report_type type, int watchpoint_idx); + +#endif /* _KERNEL_KCSAN_KCSAN_H */ diff --git a/kernel/kcsan/report.c b/kernel/kcsan/report.c new file mode 100644 index 000000000000..ac5f8345bae9 --- /dev/null +++ b/kernel/kcsan/report.c @@ -0,0 +1,634 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/debug_locks.h> +#include <linux/delay.h> +#include <linux/jiffies.h> +#include <linux/kernel.h> +#include <linux/lockdep.h> +#include <linux/preempt.h> +#include <linux/printk.h> +#include <linux/sched.h> +#include <linux/spinlock.h> +#include <linux/stacktrace.h> + +#include "kcsan.h" +#include "encoding.h" + +/* + * Max. number of stack entries to show in the report. + */ +#define NUM_STACK_ENTRIES 64 + +/* Common access info. */ +struct access_info { + const volatile void *ptr; + size_t size; + int access_type; + int task_pid; + int cpu_id; +}; + +/* + * Other thread info: communicated from other racing thread to thread that set + * up the watchpoint, which then prints the complete report atomically. + */ +struct other_info { + struct access_info ai; + unsigned long stack_entries[NUM_STACK_ENTRIES]; + int num_stack_entries; + + /* + * Optionally pass @current. Typically we do not need to pass @current + * via @other_info since just @task_pid is sufficient. Passing @current + * has additional overhead. + * + * To safely pass @current, we must either use get_task_struct/ + * put_task_struct, or stall the thread that populated @other_info. + * + * We cannot rely on get_task_struct/put_task_struct in case + * release_report() races with a task being released, and would have to + * free it in release_report(). This may result in deadlock if we want + * to use KCSAN on the allocators. + * + * Since we also want to reliably print held locks for + * CONFIG_KCSAN_VERBOSE, the current implementation stalls the thread + * that populated @other_info until it has been consumed. + */ + struct task_struct *task; +}; + +/* + * To never block any producers of struct other_info, we need as many elements + * as we have watchpoints (upper bound on concurrent races to report). + */ +static struct other_info other_infos[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1]; + +/* + * Information about reported races; used to rate limit reporting. + */ +struct report_time { + /* + * The last time the race was reported. + */ + unsigned long time; + + /* + * The frames of the 2 threads; if only 1 thread is known, one frame + * will be 0. + */ + unsigned long frame1; + unsigned long frame2; +}; + +/* + * Since we also want to be able to debug allocators with KCSAN, to avoid + * deadlock, report_times cannot be dynamically resized with krealloc in + * rate_limit_report. + * + * Therefore, we use a fixed-size array, which at most will occupy a page. This + * still adequately rate limits reports, assuming that a) number of unique data + * races is not excessive, and b) occurrence of unique races within the + * same time window is limited. + */ +#define REPORT_TIMES_MAX (PAGE_SIZE / sizeof(struct report_time)) +#define REPORT_TIMES_SIZE \ + (CONFIG_KCSAN_REPORT_ONCE_IN_MS > REPORT_TIMES_MAX ? \ + REPORT_TIMES_MAX : \ + CONFIG_KCSAN_REPORT_ONCE_IN_MS) +static struct report_time report_times[REPORT_TIMES_SIZE]; + +/* + * Spinlock serializing report generation, and access to @other_infos. Although + * it could make sense to have a finer-grained locking story for @other_infos, + * report generation needs to be serialized either way, so not much is gained. + */ +static DEFINE_RAW_SPINLOCK(report_lock); + +/* + * Checks if the race identified by thread frames frame1 and frame2 has + * been reported since (now - KCSAN_REPORT_ONCE_IN_MS). + */ +static bool rate_limit_report(unsigned long frame1, unsigned long frame2) +{ + struct report_time *use_entry = &report_times[0]; + unsigned long invalid_before; + int i; + + BUILD_BUG_ON(CONFIG_KCSAN_REPORT_ONCE_IN_MS != 0 && REPORT_TIMES_SIZE == 0); + + if (CONFIG_KCSAN_REPORT_ONCE_IN_MS == 0) + return false; + + invalid_before = jiffies - msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS); + + /* Check if a matching race report exists. */ + for (i = 0; i < REPORT_TIMES_SIZE; ++i) { + struct report_time *rt = &report_times[i]; + + /* + * Must always select an entry for use to store info as we + * cannot resize report_times; at the end of the scan, use_entry + * will be the oldest entry, which ideally also happened before + * KCSAN_REPORT_ONCE_IN_MS ago. + */ + if (time_before(rt->time, use_entry->time)) + use_entry = rt; + + /* + * Initially, no need to check any further as this entry as well + * as following entries have never been used. + */ + if (rt->time == 0) + break; + + /* Check if entry expired. */ + if (time_before(rt->time, invalid_before)) + continue; /* before KCSAN_REPORT_ONCE_IN_MS ago */ + + /* Reported recently, check if race matches. */ + if ((rt->frame1 == frame1 && rt->frame2 == frame2) || + (rt->frame1 == frame2 && rt->frame2 == frame1)) + return true; + } + + use_entry->time = jiffies; + use_entry->frame1 = frame1; + use_entry->frame2 = frame2; + return false; +} + +/* + * Special rules to skip reporting. + */ +static bool +skip_report(enum kcsan_value_change value_change, unsigned long top_frame) +{ + /* Should never get here if value_change==FALSE. */ + WARN_ON_ONCE(value_change == KCSAN_VALUE_CHANGE_FALSE); + + /* + * The first call to skip_report always has value_change==TRUE, since we + * cannot know the value written of an instrumented access. For the 2nd + * call there are 6 cases with CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY: + * + * 1. read watchpoint, conflicting write (value_change==TRUE): report; + * 2. read watchpoint, conflicting write (value_change==MAYBE): skip; + * 3. write watchpoint, conflicting write (value_change==TRUE): report; + * 4. write watchpoint, conflicting write (value_change==MAYBE): skip; + * 5. write watchpoint, conflicting read (value_change==MAYBE): skip; + * 6. write watchpoint, conflicting read (value_change==TRUE): report; + * + * Cases 1-4 are intuitive and expected; case 5 ensures we do not report + * data races where the write may have rewritten the same value; case 6 + * is possible either if the size is larger than what we check value + * changes for or the access type is KCSAN_ACCESS_ASSERT. + */ + if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY) && + value_change == KCSAN_VALUE_CHANGE_MAYBE) { + /* + * The access is a write, but the data value did not change. + * + * We opt-out of this filter for certain functions at request of + * maintainers. + */ + char buf[64]; + int len = scnprintf(buf, sizeof(buf), "%ps", (void *)top_frame); + + if (!strnstr(buf, "rcu_", len) && + !strnstr(buf, "_rcu", len) && + !strnstr(buf, "_srcu", len)) + return true; + } + + return kcsan_skip_report_debugfs(top_frame); +} + +static const char *get_access_type(int type) +{ + if (type & KCSAN_ACCESS_ASSERT) { + if (type & KCSAN_ACCESS_SCOPED) { + if (type & KCSAN_ACCESS_WRITE) + return "assert no accesses (scoped)"; + else + return "assert no writes (scoped)"; + } else { + if (type & KCSAN_ACCESS_WRITE) + return "assert no accesses"; + else + return "assert no writes"; + } + } + + switch (type) { + case 0: + return "read"; + case KCSAN_ACCESS_ATOMIC: + return "read (marked)"; + case KCSAN_ACCESS_WRITE: + return "write"; + case KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC: + return "write (marked)"; + case KCSAN_ACCESS_SCOPED: + return "read (scoped)"; + case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_ATOMIC: + return "read (marked, scoped)"; + case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE: + return "write (scoped)"; + case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC: + return "write (marked, scoped)"; + default: + BUG(); + } +} + +static const char *get_bug_type(int type) +{ + return (type & KCSAN_ACCESS_ASSERT) != 0 ? "assert: race" : "data-race"; +} + +/* Return thread description: in task or interrupt. */ +static const char *get_thread_desc(int task_id) +{ + if (task_id != -1) { + static char buf[32]; /* safe: protected by report_lock */ + + snprintf(buf, sizeof(buf), "task %i", task_id); + return buf; + } + return "interrupt"; +} + +/* Helper to skip KCSAN-related functions in stack-trace. */ +static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries) +{ + char buf[64]; + char *cur; + int len, skip; + + for (skip = 0; skip < num_entries; ++skip) { + len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]); + + /* Never show tsan_* or {read,write}_once_size. */ + if (strnstr(buf, "tsan_", len) || + strnstr(buf, "_once_size", len)) + continue; + + cur = strnstr(buf, "kcsan_", len); + if (cur) { + cur += sizeof("kcsan_") - 1; + if (strncmp(cur, "test", sizeof("test") - 1)) + continue; /* KCSAN runtime function. */ + /* KCSAN related test. */ + } + + /* + * No match for runtime functions -- @skip entries to skip to + * get to first frame of interest. + */ + break; + } + + return skip; +} + +/* Compares symbolized strings of addr1 and addr2. */ +static int sym_strcmp(void *addr1, void *addr2) +{ + char buf1[64]; + char buf2[64]; + + snprintf(buf1, sizeof(buf1), "%pS", addr1); + snprintf(buf2, sizeof(buf2), "%pS", addr2); + + return strncmp(buf1, buf2, sizeof(buf1)); +} + +static void print_verbose_info(struct task_struct *task) +{ + if (!task) + return; + + pr_err("\n"); + debug_show_held_locks(task); + print_irqtrace_events(task); +} + +/* + * Returns true if a report was generated, false otherwise. + */ +static bool print_report(enum kcsan_value_change value_change, + enum kcsan_report_type type, + const struct access_info *ai, + const struct other_info *other_info) +{ + unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 }; + int num_stack_entries = stack_trace_save(stack_entries, NUM_STACK_ENTRIES, 1); + int skipnr = get_stack_skipnr(stack_entries, num_stack_entries); + unsigned long this_frame = stack_entries[skipnr]; + unsigned long other_frame = 0; + int other_skipnr = 0; /* silence uninit warnings */ + + /* + * Must check report filter rules before starting to print. + */ + if (skip_report(KCSAN_VALUE_CHANGE_TRUE, stack_entries[skipnr])) + return false; + + if (type == KCSAN_REPORT_RACE_SIGNAL) { + other_skipnr = get_stack_skipnr(other_info->stack_entries, + other_info->num_stack_entries); + other_frame = other_info->stack_entries[other_skipnr]; + + /* @value_change is only known for the other thread */ + if (skip_report(value_change, other_frame)) + return false; + } + + if (rate_limit_report(this_frame, other_frame)) + return false; + + /* Print report header. */ + pr_err("==================================================================\n"); + switch (type) { + case KCSAN_REPORT_RACE_SIGNAL: { + int cmp; + + /* + * Order functions lexographically for consistent bug titles. + * Do not print offset of functions to keep title short. + */ + cmp = sym_strcmp((void *)other_frame, (void *)this_frame); + pr_err("BUG: KCSAN: %s in %ps / %ps\n", + get_bug_type(ai->access_type | other_info->ai.access_type), + (void *)(cmp < 0 ? other_frame : this_frame), + (void *)(cmp < 0 ? this_frame : other_frame)); + } break; + + case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN: + pr_err("BUG: KCSAN: %s in %pS\n", get_bug_type(ai->access_type), + (void *)this_frame); + break; + + default: + BUG(); + } + + pr_err("\n"); + + /* Print information about the racing accesses. */ + switch (type) { + case KCSAN_REPORT_RACE_SIGNAL: + pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n", + get_access_type(other_info->ai.access_type), other_info->ai.ptr, + other_info->ai.size, get_thread_desc(other_info->ai.task_pid), + other_info->ai.cpu_id); + + /* Print the other thread's stack trace. */ + stack_trace_print(other_info->stack_entries + other_skipnr, + other_info->num_stack_entries - other_skipnr, + 0); + + if (IS_ENABLED(CONFIG_KCSAN_VERBOSE)) + print_verbose_info(other_info->task); + + pr_err("\n"); + pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n", + get_access_type(ai->access_type), ai->ptr, ai->size, + get_thread_desc(ai->task_pid), ai->cpu_id); + break; + + case KCSAN_REPORT_RACE_UNKNOWN_ORIGIN: + pr_err("race at unknown origin, with %s to 0x%px of %zu bytes by %s on cpu %i:\n", + get_access_type(ai->access_type), ai->ptr, ai->size, + get_thread_desc(ai->task_pid), ai->cpu_id); + break; + + default: + BUG(); + } + /* Print stack trace of this thread. */ + stack_trace_print(stack_entries + skipnr, num_stack_entries - skipnr, + 0); + + if (IS_ENABLED(CONFIG_KCSAN_VERBOSE)) + print_verbose_info(current); + + /* Print report footer. */ + pr_err("\n"); + pr_err("Reported by Kernel Concurrency Sanitizer on:\n"); + dump_stack_print_info(KERN_DEFAULT); + pr_err("==================================================================\n"); + + return true; +} + +static void release_report(unsigned long *flags, struct other_info *other_info) +{ + if (other_info) + /* + * Use size to denote valid/invalid, since KCSAN entirely + * ignores 0-sized accesses. + */ + other_info->ai.size = 0; + + raw_spin_unlock_irqrestore(&report_lock, *flags); +} + +/* + * Sets @other_info->task and awaits consumption of @other_info. + * + * Precondition: report_lock is held. + * Postcondition: report_lock is held. + */ +static void set_other_info_task_blocking(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) +{ + /* + * We may be instrumenting a code-path where current->state is already + * something other than TASK_RUNNING. + */ + const bool is_running = current->state == TASK_RUNNING; + /* + * To avoid deadlock in case we are in an interrupt here and this is a + * race with a task on the same CPU (KCSAN_INTERRUPT_WATCHER), provide a + * timeout to ensure this works in all contexts. + * + * Await approximately the worst case delay of the reporting thread (if + * we are not interrupted). + */ + int timeout = max(kcsan_udelay_task, kcsan_udelay_interrupt); + + other_info->task = current; + do { + if (is_running) { + /* + * Let lockdep know the real task is sleeping, to print + * the held locks (recall we turned lockdep off, so + * locking/unlocking @report_lock won't be recorded). + */ + set_current_state(TASK_UNINTERRUPTIBLE); + } + raw_spin_unlock_irqrestore(&report_lock, *flags); + /* + * We cannot call schedule() since we also cannot reliably + * determine if sleeping here is permitted -- see in_atomic(). + */ + + udelay(1); + raw_spin_lock_irqsave(&report_lock, *flags); + if (timeout-- < 0) { + /* + * Abort. Reset @other_info->task to NULL, since it + * appears the other thread is still going to consume + * it. It will result in no verbose info printed for + * this task. + */ + other_info->task = NULL; + break; + } + /* + * If invalid, or @ptr nor @current matches, then @other_info + * has been consumed and we may continue. If not, retry. + */ + } while (other_info->ai.size && other_info->ai.ptr == ai->ptr && + other_info->task == current); + if (is_running) + set_current_state(TASK_RUNNING); +} + +/* Populate @other_info; requires that the provided @other_info not in use. */ +static void prepare_report_producer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) +{ + raw_spin_lock_irqsave(&report_lock, *flags); + + /* + * The same @other_infos entry cannot be used concurrently, because + * there is a one-to-one mapping to watchpoint slots (@watchpoints in + * core.c), and a watchpoint is only released for reuse after reporting + * is done by the consumer of @other_info. Therefore, it is impossible + * for another concurrent prepare_report_producer() to set the same + * @other_info, and are guaranteed exclusivity for the @other_infos + * entry pointed to by @other_info. + * + * To check this property holds, size should never be non-zero here, + * because every consumer of struct other_info resets size to 0 in + * release_report(). + */ + WARN_ON(other_info->ai.size); + + other_info->ai = *ai; + other_info->num_stack_entries = stack_trace_save(other_info->stack_entries, NUM_STACK_ENTRIES, 2); + + if (IS_ENABLED(CONFIG_KCSAN_VERBOSE)) + set_other_info_task_blocking(flags, ai, other_info); + + raw_spin_unlock_irqrestore(&report_lock, *flags); +} + +/* Awaits producer to fill @other_info and then returns. */ +static bool prepare_report_consumer(unsigned long *flags, + const struct access_info *ai, + struct other_info *other_info) +{ + + raw_spin_lock_irqsave(&report_lock, *flags); + while (!other_info->ai.size) { /* Await valid @other_info. */ + raw_spin_unlock_irqrestore(&report_lock, *flags); + cpu_relax(); + raw_spin_lock_irqsave(&report_lock, *flags); + } + + /* Should always have a matching access based on watchpoint encoding. */ + if (WARN_ON(!matching_access((unsigned long)other_info->ai.ptr & WATCHPOINT_ADDR_MASK, other_info->ai.size, + (unsigned long)ai->ptr & WATCHPOINT_ADDR_MASK, ai->size))) + goto discard; + + if (!matching_access((unsigned long)other_info->ai.ptr, other_info->ai.size, + (unsigned long)ai->ptr, ai->size)) { + /* + * If the actual accesses to not match, this was a false + * positive due to watchpoint encoding. + */ + kcsan_counter_inc(KCSAN_COUNTER_ENCODING_FALSE_POSITIVES); + goto discard; + } + + return true; + +discard: + release_report(flags, other_info); + return false; +} + +/* + * Depending on the report type either sets @other_info and returns false, or + * awaits @other_info and returns true. If @other_info is not required for the + * report type, simply acquires @report_lock and returns true. + */ +static noinline bool prepare_report(unsigned long *flags, + enum kcsan_report_type type, + const struct access_info *ai, + struct other_info *other_info) +{ + switch (type) { + case KCSAN_REPORT_CONSUMED_WATCHPOINT: + prepare_report_producer(flags, ai, other_info); + return false; + case KCSAN_REPORT_RACE_SIGNAL: + return prepare_report_consumer(flags, ai, other_info); + default: + /* @other_info not required; just acquire @report_lock. */ + raw_spin_lock_irqsave(&report_lock, *flags); + return true; + } +} + +void kcsan_report(const volatile void *ptr, size_t size, int access_type, + enum kcsan_value_change value_change, + enum kcsan_report_type type, int watchpoint_idx) +{ + unsigned long flags = 0; + const struct access_info ai = { + .ptr = ptr, + .size = size, + .access_type = access_type, + .task_pid = in_task() ? task_pid_nr(current) : -1, + .cpu_id = raw_smp_processor_id() + }; + struct other_info *other_info = type == KCSAN_REPORT_RACE_UNKNOWN_ORIGIN + ? NULL : &other_infos[watchpoint_idx]; + + kcsan_disable_current(); + if (WARN_ON(watchpoint_idx < 0 || watchpoint_idx >= ARRAY_SIZE(other_infos))) + goto out; + + /* + * With TRACE_IRQFLAGS, lockdep's IRQ trace state becomes corrupted if + * we do not turn off lockdep here; this could happen due to recursion + * into lockdep via KCSAN if we detect a race in utilities used by + * lockdep. + */ + lockdep_off(); + + if (prepare_report(&flags, type, &ai, other_info)) { + /* + * Never report if value_change is FALSE, only if we it is + * either TRUE or MAYBE. In case of MAYBE, further filtering may + * be done once we know the full stack trace in print_report(). + */ + bool reported = value_change != KCSAN_VALUE_CHANGE_FALSE && + print_report(value_change, type, &ai, other_info); + + if (reported && panic_on_warn) + panic("panic_on_warn set ...\n"); + + release_report(&flags, other_info); + } + + lockdep_on(); +out: + kcsan_enable_current(); +} diff --git a/kernel/kcsan/test.c b/kernel/kcsan/test.c new file mode 100644 index 000000000000..d26a052d3383 --- /dev/null +++ b/kernel/kcsan/test.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/printk.h> +#include <linux/random.h> +#include <linux/types.h> + +#include "encoding.h" + +#define ITERS_PER_TEST 2000 + +/* Test requirements. */ +static bool test_requires(void) +{ + /* random should be initialized for the below tests */ + return prandom_u32() + prandom_u32() != 0; +} + +/* + * Test watchpoint encode and decode: check that encoding some access's info, + * and then subsequent decode preserves the access's info. + */ +static bool test_encode_decode(void) +{ + int i; + + for (i = 0; i < ITERS_PER_TEST; ++i) { + size_t size = prandom_u32_max(MAX_ENCODABLE_SIZE) + 1; + bool is_write = !!prandom_u32_max(2); + unsigned long addr; + + prandom_bytes(&addr, sizeof(addr)); + if (WARN_ON(!check_encodable(addr, size))) + return false; + + /* Encode and decode */ + { + const long encoded_watchpoint = + encode_watchpoint(addr, size, is_write); + unsigned long verif_masked_addr; + size_t verif_size; + bool verif_is_write; + + /* Check special watchpoints */ + if (WARN_ON(decode_watchpoint( + INVALID_WATCHPOINT, &verif_masked_addr, + &verif_size, &verif_is_write))) + return false; + if (WARN_ON(decode_watchpoint( + CONSUMED_WATCHPOINT, &verif_masked_addr, + &verif_size, &verif_is_write))) + return false; + + /* Check decoding watchpoint returns same data */ + if (WARN_ON(!decode_watchpoint( + encoded_watchpoint, &verif_masked_addr, + &verif_size, &verif_is_write))) + return false; + if (WARN_ON(verif_masked_addr != + (addr & WATCHPOINT_ADDR_MASK))) + goto fail; + if (WARN_ON(verif_size != size)) + goto fail; + if (WARN_ON(is_write != verif_is_write)) + goto fail; + + continue; +fail: + pr_err("%s fail: %s %zu bytes @ %lx -> encoded: %lx -> %s %zu bytes @ %lx\n", + __func__, is_write ? "write" : "read", size, + addr, encoded_watchpoint, + verif_is_write ? "write" : "read", verif_size, + verif_masked_addr); + return false; + } + } + + return true; +} + +/* Test access matching function. */ +static bool test_matching_access(void) +{ + if (WARN_ON(!matching_access(10, 1, 10, 1))) + return false; + if (WARN_ON(!matching_access(10, 2, 11, 1))) + return false; + if (WARN_ON(!matching_access(10, 1, 9, 2))) + return false; + if (WARN_ON(matching_access(10, 1, 11, 1))) + return false; + if (WARN_ON(matching_access(9, 1, 10, 1))) + return false; + + /* + * An access of size 0 could match another access, as demonstrated here. + * Rather than add more comparisons to 'matching_access()', which would + * end up in the fast-path for *all* checks, check_access() simply + * returns for all accesses of size 0. + */ + if (WARN_ON(!matching_access(8, 8, 12, 0))) + return false; + + return true; +} + +static int __init kcsan_selftest(void) +{ + int passed = 0; + int total = 0; + +#define RUN_TEST(do_test) \ + do { \ + ++total; \ + if (do_test()) \ + ++passed; \ + else \ + pr_err("KCSAN selftest: " #do_test " failed"); \ + } while (0) + + RUN_TEST(test_requires); + RUN_TEST(test_encode_decode); + RUN_TEST(test_matching_access); + + pr_info("KCSAN selftest: %d/%d tests passed\n", passed, total); + if (passed != total) + panic("KCSAN selftests failed"); + return 0; +} +postcore_initcall(kcsan_selftest); diff --git a/kernel/kthread.c b/kernel/kthread.c index bfbfa481be3a..8e3d2d7fdf5e 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -1,13 +1,17 @@ // SPDX-License-Identifier: GPL-2.0-only /* Kernel thread helper functions. * Copyright (C) 2004 IBM Corporation, Rusty Russell. + * Copyright (C) 2009 Red Hat, Inc. * * Creation is done via kthreadd, so that we get a clean environment * even if we're invoked from userspace (think modprobe, hotplug cpu, * etc.). */ #include <uapi/linux/sched/types.h> +#include <linux/mm.h> +#include <linux/mmu_context.h> #include <linux/sched.h> +#include <linux/sched/mm.h> #include <linux/sched/task.h> #include <linux/kthread.h> #include <linux/completion.h> @@ -25,6 +29,7 @@ #include <linux/numa.h> #include <trace/events/sched.h> + static DEFINE_SPINLOCK(kthread_create_lock); static LIST_HEAD(kthread_create_list); struct task_struct *kthreadd_task; @@ -46,7 +51,9 @@ struct kthread_create_info struct kthread { unsigned long flags; unsigned int cpu; + int (*threadfn)(void *); void *data; + mm_segment_t oldfs; struct completion parked; struct completion exited; #ifdef CONFIG_BLK_CGROUP @@ -153,6 +160,20 @@ bool kthread_freezable_should_stop(bool *was_frozen) EXPORT_SYMBOL_GPL(kthread_freezable_should_stop); /** + * kthread_func - return the function specified on kthread creation + * @task: kthread task in question + * + * Returns NULL if the task is not a kthread. + */ +void *kthread_func(struct task_struct *task) +{ + if (task->flags & PF_KTHREAD) + return to_kthread(task)->threadfn; + return NULL; +} +EXPORT_SYMBOL_GPL(kthread_func); + +/** * kthread_data - return data value specified on kthread creation * @task: kthread task in question * @@ -164,6 +185,7 @@ void *kthread_data(struct task_struct *task) { return to_kthread(task)->data; } +EXPORT_SYMBOL_GPL(kthread_data); /** * kthread_probe_data - speculative version of kthread_data() @@ -244,6 +266,7 @@ static int kthread(void *_create) do_exit(-ENOMEM); } + self->threadfn = threadfn; self->data = data; init_completion(&self->exited); init_completion(&self->parked); @@ -1203,6 +1226,61 @@ void kthread_destroy_worker(struct kthread_worker *worker) } EXPORT_SYMBOL(kthread_destroy_worker); +/** + * kthread_use_mm - make the calling kthread operate on an address space + * @mm: address space to operate on + */ +void kthread_use_mm(struct mm_struct *mm) +{ + struct mm_struct *active_mm; + struct task_struct *tsk = current; + + WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); + WARN_ON_ONCE(tsk->mm); + + task_lock(tsk); + active_mm = tsk->active_mm; + if (active_mm != mm) { + mmgrab(mm); + tsk->active_mm = mm; + } + tsk->mm = mm; + switch_mm(active_mm, mm, tsk); + task_unlock(tsk); +#ifdef finish_arch_post_lock_switch + finish_arch_post_lock_switch(); +#endif + + if (active_mm != mm) + mmdrop(active_mm); + + to_kthread(tsk)->oldfs = get_fs(); + set_fs(USER_DS); +} +EXPORT_SYMBOL_GPL(kthread_use_mm); + +/** + * kthread_unuse_mm - reverse the effect of kthread_use_mm() + * @mm: address space to operate on + */ +void kthread_unuse_mm(struct mm_struct *mm) +{ + struct task_struct *tsk = current; + + WARN_ON_ONCE(!(tsk->flags & PF_KTHREAD)); + WARN_ON_ONCE(!tsk->mm); + + set_fs(to_kthread(tsk)->oldfs); + + task_lock(tsk); + sync_mm_rss(mm); + tsk->mm = NULL; + /* active_mm is still 'mm' */ + enter_lazy_tlb(mm, tsk); + task_unlock(tsk); +} +EXPORT_SYMBOL_GPL(kthread_unuse_mm); + #ifdef CONFIG_BLK_CGROUP /** * kthread_associate_blkcg - associate blkcg to current kthread diff --git a/kernel/locking/Makefile b/kernel/locking/Makefile index 45452facff3b..6d11cfb9b41f 100644 --- a/kernel/locking/Makefile +++ b/kernel/locking/Makefile @@ -5,6 +5,9 @@ KCOV_INSTRUMENT := n obj-y += mutex.o semaphore.o rwsem.o percpu-rwsem.o +# Avoid recursion lockdep -> KCSAN -> ... -> lockdep. +KCSAN_SANITIZE_lockdep.o := n + ifdef CONFIG_FUNCTION_TRACER CFLAGS_REMOVE_lockdep.o = $(CC_FLAGS_FTRACE) CFLAGS_REMOVE_lockdep_proc.o = $(CC_FLAGS_FTRACE) diff --git a/kernel/locking/lockdep.c b/kernel/locking/lockdep.c index 38cce34d03dc..29a8de4c50b9 100644 --- a/kernel/locking/lockdep.c +++ b/kernel/locking/lockdep.c @@ -393,7 +393,7 @@ void lockdep_init_task(struct task_struct *task) task->lockdep_recursion = 0; } -static inline void lockdep_recursion_finish(void) +static __always_inline void lockdep_recursion_finish(void) { if (WARN_ON_ONCE(--current->lockdep_recursion)) current->lockdep_recursion = 0; @@ -801,7 +801,7 @@ static int count_matching_names(struct lock_class *new_class) } /* used from NMI context -- must be lockless */ -static inline struct lock_class * +static __always_inline struct lock_class * look_up_lock_class(const struct lockdep_map *lock, unsigned int subclass) { struct lockdep_subclass_key *key; diff --git a/kernel/panic.c b/kernel/panic.c index 85568bbfb12b..e2157ca387c8 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -680,10 +680,12 @@ device_initcall(register_warn_debugfs); * Called when gcc's -fstack-protector feature is used, and * gcc detects corruption of the on-stack canary value */ -__visible void __stack_chk_fail(void) +__visible noinstr void __stack_chk_fail(void) { + instrumentation_begin(); panic("stack-protector: Kernel stack is corrupted in: %pB", __builtin_return_address(0)); + instrumentation_end(); } EXPORT_SYMBOL(__stack_chk_fail); diff --git a/kernel/printk/printk.c b/kernel/printk/printk.c index 3132d6f860a8..8c14835be46c 100644 --- a/kernel/printk/printk.c +++ b/kernel/printk/printk.c @@ -35,7 +35,6 @@ #include <linux/memblock.h> #include <linux/syscalls.h> #include <linux/crash_core.h> -#include <linux/kdb.h> #include <linux/ratelimit.h> #include <linux/kmsg_dump.h> #include <linux/syslog.h> @@ -2047,18 +2046,7 @@ EXPORT_SYMBOL(vprintk); int vprintk_default(const char *fmt, va_list args) { - int r; - -#ifdef CONFIG_KGDB_KDB - /* Allow to pass printk() to kdb but avoid a recursion. */ - if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0)) { - r = vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args); - return r; - } -#endif - r = vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args); - - return r; + return vprintk_emit(0, LOGLEVEL_DEFAULT, NULL, 0, fmt, args); } EXPORT_SYMBOL_GPL(vprintk_default); diff --git a/kernel/printk/printk_safe.c b/kernel/printk/printk_safe.c index 4242403316bb..50aeae770434 100644 --- a/kernel/printk/printk_safe.c +++ b/kernel/printk/printk_safe.c @@ -6,6 +6,7 @@ #include <linux/preempt.h> #include <linux/spinlock.h> #include <linux/debug_locks.h> +#include <linux/kdb.h> #include <linux/smp.h> #include <linux/cpumask.h> #include <linux/irq_work.h> @@ -360,6 +361,12 @@ void __printk_safe_exit(void) __printf(1, 0) int vprintk_func(const char *fmt, va_list args) { +#ifdef CONFIG_KGDB_KDB + /* Allow to pass printk() to kdb but avoid a recursion. */ + if (unlikely(kdb_trap_printk && kdb_printf_cpu < 0)) + return vkdb_printf(KDB_MSGSRC_PRINTK, fmt, args); +#endif + /* * Try to use the main logbuf even in NMI. But avoid calling console * drivers that might have their own locks. diff --git a/kernel/sched/Makefile b/kernel/sched/Makefile index 21fb5a5662b5..5fc9c9b70862 100644 --- a/kernel/sched/Makefile +++ b/kernel/sched/Makefile @@ -7,6 +7,12 @@ endif # that is not a function of syscall inputs. E.g. involuntary context switches. KCOV_INSTRUMENT := n +# There are numerous data races here, however, most of them are due to plain accesses. +# This would make it even harder for syzbot to find reproducers, because these +# bugs trigger without specific input. Disable by default, but should re-enable +# eventually. +KCSAN_SANITIZE := n + ifneq ($(CONFIG_SCHED_OMIT_FRAME_POINTER),y) # According to Alan Modra <alan@linuxcare.com.au>, the -fno-omit-frame-pointer is # needed for x86 only. Why this used to be enabled for all architectures is beyond diff --git a/kernel/scs.c b/kernel/scs.c index 222a7a9ad543..5d4d9bbdec36 100644 --- a/kernel/scs.c +++ b/kernel/scs.c @@ -74,7 +74,7 @@ static void scs_check_usage(struct task_struct *tsk) for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { if (!READ_ONCE_NOCHECK(*p)) break; - used++; + used += sizeof(*p); } while (used > curr) { diff --git a/kernel/softirq.c b/kernel/softirq.c index a47c6dd57452..c4201b7f42b1 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -339,12 +339,11 @@ asmlinkage __visible void do_softirq(void) local_irq_restore(flags); } -/* - * Enter an interrupt context. +/** + * irq_enter_rcu - Enter an interrupt context with RCU watching */ -void irq_enter(void) +void irq_enter_rcu(void) { - rcu_irq_enter(); if (is_idle_task(current) && !in_interrupt()) { /* * Prevent raise_softirq from needlessly waking up ksoftirqd @@ -354,10 +353,18 @@ void irq_enter(void) tick_irq_enter(); _local_bh_enable(); } - __irq_enter(); } +/** + * irq_enter - Enter an interrupt context including RCU update + */ +void irq_enter(void) +{ + rcu_irq_enter(); + irq_enter_rcu(); +} + static inline void invoke_softirq(void) { if (ksoftirqd_running(local_softirq_pending())) @@ -397,10 +404,7 @@ static inline void tick_irq_exit(void) #endif } -/* - * Exit an interrupt context. Process softirqs if needed and possible: - */ -void irq_exit(void) +static inline void __irq_exit_rcu(void) { #ifndef __ARCH_IRQ_EXIT_IRQS_DISABLED local_irq_disable(); @@ -413,6 +417,28 @@ void irq_exit(void) invoke_softirq(); tick_irq_exit(); +} + +/** + * irq_exit_rcu() - Exit an interrupt context without updating RCU + * + * Also processes softirqs if needed and possible. + */ +void irq_exit_rcu(void) +{ + __irq_exit_rcu(); + /* must be last! */ + lockdep_hardirq_exit(); +} + +/** + * irq_exit - Exit an interrupt context, update RCU and lockdep + * + * Also processes softirqs if needed and possible. + */ +void irq_exit(void) +{ + __irq_exit_rcu(); rcu_irq_exit(); /* must be last! */ lockdep_hardirq_exit(); diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index 7cb09c4cf21c..02441ead3c3b 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -928,14 +928,12 @@ int __clocksource_register_scale(struct clocksource *cs, u32 scale, u32 freq) clocksource_arch_init(cs); -#ifdef CONFIG_GENERIC_VDSO_CLOCK_MODE if (cs->vdso_clock_mode < 0 || cs->vdso_clock_mode >= VDSO_CLOCKMODE_MAX) { pr_warn("clocksource %s registered with invalid VDSO mode %d. Disabling VDSO support.\n", cs->name, cs->vdso_clock_mode); cs->vdso_clock_mode = VDSO_CLOCKMODE_NONE; } -#endif /* Initialize mult/shift and max_idle_ns */ __clocksource_update_freq_scale(cs, scale, freq); diff --git a/kernel/time/timekeeping.c b/kernel/time/timekeeping.c index 9ebaab13339d..d20d489841c8 100644 --- a/kernel/time/timekeeping.c +++ b/kernel/time/timekeeping.c @@ -953,7 +953,7 @@ EXPORT_SYMBOL_GPL(ktime_get_real_seconds); * but without the sequence counter protect. This internal function * is called just when timekeeping lock is already held. */ -time64_t __ktime_get_real_seconds(void) +noinstr time64_t __ktime_get_real_seconds(void) { struct timekeeper *tk = &tk_core.timekeeper; diff --git a/kernel/trace/Makefile b/kernel/trace/Makefile index 1d8aaa546412..6575bb0a0434 100644 --- a/kernel/trace/Makefile +++ b/kernel/trace/Makefile @@ -6,6 +6,9 @@ ifdef CONFIG_FUNCTION_TRACER ORIG_CFLAGS := $(KBUILD_CFLAGS) KBUILD_CFLAGS = $(subst $(CC_FLAGS_FTRACE),,$(ORIG_CFLAGS)) +# Avoid recursion due to instrumentation. +KCSAN_SANITIZE := n + ifdef CONFIG_FTRACE_SELFTEST # selftest needs instrumentation CFLAGS_trace_selftest_dynamic.o = $(CC_FLAGS_FTRACE) diff --git a/kernel/trace/blktrace.c b/kernel/trace/blktrace.c index ea47f2084087..5773f0ba7e76 100644 --- a/kernel/trace/blktrace.c +++ b/kernel/trace/blktrace.c @@ -885,10 +885,10 @@ static void blk_add_trace_bio_bounce(void *ignore, } static void blk_add_trace_bio_complete(void *ignore, - struct request_queue *q, struct bio *bio, - int error) + struct request_queue *q, struct bio *bio) { - blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, error); + blk_add_trace_bio(q, bio, BLK_TA_COMPLETE, + blk_status_to_errno(bio->bi_status)); } static void blk_add_trace_bio_backmerge(void *ignore, @@ -995,8 +995,10 @@ static void blk_add_trace_split(void *ignore, __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, bio_op(bio), bio->bi_opf, - BLK_TA_SPLIT, bio->bi_status, sizeof(rpdu), - &rpdu, blk_trace_bio_get_cgid(q, bio)); + BLK_TA_SPLIT, + blk_status_to_errno(bio->bi_status), + sizeof(rpdu), &rpdu, + blk_trace_bio_get_cgid(q, bio)); } rcu_read_unlock(); } @@ -1033,7 +1035,8 @@ static void blk_add_trace_bio_remap(void *ignore, r.sector_from = cpu_to_be64(from); __blk_add_trace(bt, bio->bi_iter.bi_sector, bio->bi_iter.bi_size, - bio_op(bio), bio->bi_opf, BLK_TA_REMAP, bio->bi_status, + bio_op(bio), bio->bi_opf, BLK_TA_REMAP, + blk_status_to_errno(bio->bi_status), sizeof(r), &r, blk_trace_bio_get_cgid(q, bio)); rcu_read_unlock(); } @@ -1253,21 +1256,10 @@ static inline __u16 t_error(const struct trace_entry *ent) static __u64 get_pdu_int(const struct trace_entry *ent, bool has_cg) { - const __u64 *val = pdu_start(ent, has_cg); + const __be64 *val = pdu_start(ent, has_cg); return be64_to_cpu(*val); } -static void get_pdu_remap(const struct trace_entry *ent, - struct blk_io_trace_remap *r, bool has_cg) -{ - const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); - __u64 sector_from = __r->sector_from; - - r->device_from = be32_to_cpu(__r->device_from); - r->device_to = be32_to_cpu(__r->device_to); - r->sector_from = be64_to_cpu(sector_from); -} - typedef void (blk_log_action_t) (struct trace_iterator *iter, const char *act, bool has_cg); @@ -1407,13 +1399,13 @@ static void blk_log_with_error(struct trace_seq *s, static void blk_log_remap(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) { - struct blk_io_trace_remap r = { .device_from = 0, }; + const struct blk_io_trace_remap *__r = pdu_start(ent, has_cg); - get_pdu_remap(ent, &r, has_cg); trace_seq_printf(s, "%llu + %u <- (%d,%d) %llu\n", t_sector(ent), t_sec(ent), - MAJOR(r.device_from), MINOR(r.device_from), - (unsigned long long)r.sector_from); + MAJOR(be32_to_cpu(__r->device_from)), + MINOR(be32_to_cpu(__r->device_from)), + be64_to_cpu(__r->sector_from)); } static void blk_log_plug(struct trace_seq *s, const struct trace_entry *ent, bool has_cg) diff --git a/kernel/trace/trace_preemptirq.c b/kernel/trace/trace_preemptirq.c index fb0691b8a88d..f10073e62603 100644 --- a/kernel/trace/trace_preemptirq.c +++ b/kernel/trace/trace_preemptirq.c @@ -58,7 +58,7 @@ NOKPROBE_SYMBOL(trace_hardirqs_on); * and lockdep uses a staged approach which splits the lockdep hardirq * tracking into a RCU on and a RCU off section. */ -void trace_hardirqs_off_prepare(void) +void trace_hardirqs_off_finish(void) { if (!this_cpu_read(tracing_irq_cpu)) { this_cpu_write(tracing_irq_cpu, 1); @@ -68,19 +68,19 @@ void trace_hardirqs_off_prepare(void) } } -EXPORT_SYMBOL(trace_hardirqs_off_prepare); -NOKPROBE_SYMBOL(trace_hardirqs_off_prepare); +EXPORT_SYMBOL(trace_hardirqs_off_finish); +NOKPROBE_SYMBOL(trace_hardirqs_off_finish); void trace_hardirqs_off(void) { + lockdep_hardirqs_off(CALLER_ADDR0); + if (!this_cpu_read(tracing_irq_cpu)) { this_cpu_write(tracing_irq_cpu, 1); tracer_hardirqs_off(CALLER_ADDR0, CALLER_ADDR1); if (!in_nmi()) trace_irq_disable_rcuidle(CALLER_ADDR0, CALLER_ADDR1); } - - lockdep_hardirqs_off(CALLER_ADDR0); } EXPORT_SYMBOL(trace_hardirqs_off); NOKPROBE_SYMBOL(trace_hardirqs_off); diff --git a/kernel/watch_queue.c b/kernel/watch_queue.c new file mode 100644 index 000000000000..f74020f6bd9d --- /dev/null +++ b/kernel/watch_queue.c @@ -0,0 +1,655 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Watch queue and general notification mechanism, built on pipes + * + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + * + * See Documentation/watch_queue.rst + */ + +#define pr_fmt(fmt) "watchq: " fmt +#include <linux/module.h> +#include <linux/init.h> +#include <linux/sched.h> +#include <linux/slab.h> +#include <linux/printk.h> +#include <linux/miscdevice.h> +#include <linux/fs.h> +#include <linux/mm.h> +#include <linux/pagemap.h> +#include <linux/poll.h> +#include <linux/uaccess.h> +#include <linux/vmalloc.h> +#include <linux/file.h> +#include <linux/security.h> +#include <linux/cred.h> +#include <linux/sched/signal.h> +#include <linux/watch_queue.h> +#include <linux/pipe_fs_i.h> + +MODULE_DESCRIPTION("Watch queue"); +MODULE_AUTHOR("Red Hat, Inc."); +MODULE_LICENSE("GPL"); + +#define WATCH_QUEUE_NOTE_SIZE 128 +#define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE) + +static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe, + struct pipe_buffer *buf) +{ + struct watch_queue *wqueue = (struct watch_queue *)buf->private; + struct page *page; + unsigned int bit; + + /* We need to work out which note within the page this refers to, but + * the note might have been maximum size, so merely ANDing the offset + * off doesn't work. OTOH, the note must've been more than zero size. + */ + bit = buf->offset + buf->len; + if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0) + bit -= WATCH_QUEUE_NOTE_SIZE; + bit /= WATCH_QUEUE_NOTE_SIZE; + + page = buf->page; + bit += page->index; + + set_bit(bit, wqueue->notes_bitmap); +} + +// No try_steal function => no stealing +#define watch_queue_pipe_buf_try_steal NULL + +/* New data written to a pipe may be appended to a buffer with this type. */ +static const struct pipe_buf_operations watch_queue_pipe_buf_ops = { + .release = watch_queue_pipe_buf_release, + .try_steal = watch_queue_pipe_buf_try_steal, + .get = generic_pipe_buf_get, +}; + +/* + * Post a notification to a watch queue. + */ +static bool post_one_notification(struct watch_queue *wqueue, + struct watch_notification *n) +{ + void *p; + struct pipe_inode_info *pipe = wqueue->pipe; + struct pipe_buffer *buf; + struct page *page; + unsigned int head, tail, mask, note, offset, len; + bool done = false; + + if (!pipe) + return false; + + spin_lock_irq(&pipe->rd_wait.lock); + + if (wqueue->defunct) + goto out; + + mask = pipe->ring_size - 1; + head = pipe->head; + tail = pipe->tail; + if (pipe_full(head, tail, pipe->ring_size)) + goto lost; + + note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes); + if (note >= wqueue->nr_notes) + goto lost; + + page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE]; + offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE; + get_page(page); + len = n->info & WATCH_INFO_LENGTH; + p = kmap_atomic(page); + memcpy(p + offset, n, len); + kunmap_atomic(p); + + buf = &pipe->bufs[head & mask]; + buf->page = page; + buf->private = (unsigned long)wqueue; + buf->ops = &watch_queue_pipe_buf_ops; + buf->offset = offset; + buf->len = len; + buf->flags = PIPE_BUF_FLAG_WHOLE; + pipe->head = head + 1; + + if (!test_and_clear_bit(note, wqueue->notes_bitmap)) { + spin_unlock_irq(&pipe->rd_wait.lock); + BUG(); + } + wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); + done = true; + +out: + spin_unlock_irq(&pipe->rd_wait.lock); + if (done) + kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); + return done; + +lost: + buf = &pipe->bufs[(head - 1) & mask]; + buf->flags |= PIPE_BUF_FLAG_LOSS; + goto out; +} + +/* + * Apply filter rules to a notification. + */ +static bool filter_watch_notification(const struct watch_filter *wf, + const struct watch_notification *n) +{ + const struct watch_type_filter *wt; + unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8; + unsigned int st_index = n->subtype / st_bits; + unsigned int st_bit = 1U << (n->subtype % st_bits); + int i; + + if (!test_bit(n->type, wf->type_filter)) + return false; + + for (i = 0; i < wf->nr_filters; i++) { + wt = &wf->filters[i]; + if (n->type == wt->type && + (wt->subtype_filter[st_index] & st_bit) && + (n->info & wt->info_mask) == wt->info_filter) + return true; + } + + return false; /* If there is a filter, the default is to reject. */ +} + +/** + * __post_watch_notification - Post an event notification + * @wlist: The watch list to post the event to. + * @n: The notification record to post. + * @cred: The creds of the process that triggered the notification. + * @id: The ID to match on the watch. + * + * Post a notification of an event into a set of watch queues and let the users + * know. + * + * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and + * should be in units of sizeof(*n). + */ +void __post_watch_notification(struct watch_list *wlist, + struct watch_notification *n, + const struct cred *cred, + u64 id) +{ + const struct watch_filter *wf; + struct watch_queue *wqueue; + struct watch *watch; + + if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) { + WARN_ON(1); + return; + } + + rcu_read_lock(); + + hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) { + if (watch->id != id) + continue; + n->info &= ~WATCH_INFO_ID; + n->info |= watch->info_id; + + wqueue = rcu_dereference(watch->queue); + wf = rcu_dereference(wqueue->filter); + if (wf && !filter_watch_notification(wf, n)) + continue; + + if (security_post_notification(watch->cred, cred, n) < 0) + continue; + + post_one_notification(wqueue, n); + } + + rcu_read_unlock(); +} +EXPORT_SYMBOL(__post_watch_notification); + +/* + * Allocate sufficient pages to preallocation for the requested number of + * notifications. + */ +long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes) +{ + struct watch_queue *wqueue = pipe->watch_queue; + struct page **pages; + unsigned long *bitmap; + unsigned long user_bufs; + unsigned int bmsize; + int ret, i, nr_pages; + + if (!wqueue) + return -ENODEV; + if (wqueue->notes) + return -EBUSY; + + if (nr_notes < 1 || + nr_notes > 512) /* TODO: choose a better hard limit */ + return -EINVAL; + + nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1); + nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE; + user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages); + + if (nr_pages > pipe->max_usage && + (too_many_pipe_buffers_hard(user_bufs) || + too_many_pipe_buffers_soft(user_bufs)) && + pipe_is_unprivileged_user()) { + ret = -EPERM; + goto error; + } + + ret = pipe_resize_ring(pipe, nr_notes); + if (ret < 0) + goto error; + + pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL); + if (!pages) + goto error; + + for (i = 0; i < nr_pages; i++) { + pages[i] = alloc_page(GFP_KERNEL); + if (!pages[i]) + goto error_p; + pages[i]->index = i * WATCH_QUEUE_NOTES_PER_PAGE; + } + + bmsize = (nr_notes + BITS_PER_LONG - 1) / BITS_PER_LONG; + bmsize *= sizeof(unsigned long); + bitmap = kmalloc(bmsize, GFP_KERNEL); + if (!bitmap) + goto error_p; + + memset(bitmap, 0xff, bmsize); + wqueue->notes = pages; + wqueue->notes_bitmap = bitmap; + wqueue->nr_pages = nr_pages; + wqueue->nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE; + return 0; + +error_p: + for (i = 0; i < nr_pages; i++) + __free_page(pages[i]); + kfree(pages); +error: + (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted); + return ret; +} + +/* + * Set the filter on a watch queue. + */ +long watch_queue_set_filter(struct pipe_inode_info *pipe, + struct watch_notification_filter __user *_filter) +{ + struct watch_notification_type_filter *tf; + struct watch_notification_filter filter; + struct watch_type_filter *q; + struct watch_filter *wfilter; + struct watch_queue *wqueue = pipe->watch_queue; + int ret, nr_filter = 0, i; + + if (!wqueue) + return -ENODEV; + + if (!_filter) { + /* Remove the old filter */ + wfilter = NULL; + goto set; + } + + /* Grab the user's filter specification */ + if (copy_from_user(&filter, _filter, sizeof(filter)) != 0) + return -EFAULT; + if (filter.nr_filters == 0 || + filter.nr_filters > 16 || + filter.__reserved != 0) + return -EINVAL; + + tf = memdup_user(_filter->filters, filter.nr_filters * sizeof(*tf)); + if (IS_ERR(tf)) + return PTR_ERR(tf); + + ret = -EINVAL; + for (i = 0; i < filter.nr_filters; i++) { + if ((tf[i].info_filter & ~tf[i].info_mask) || + tf[i].info_mask & WATCH_INFO_LENGTH) + goto err_filter; + /* Ignore any unknown types */ + if (tf[i].type >= sizeof(wfilter->type_filter) * 8) + continue; + nr_filter++; + } + + /* Now we need to build the internal filter from only the relevant + * user-specified filters. + */ + ret = -ENOMEM; + wfilter = kzalloc(struct_size(wfilter, filters, nr_filter), GFP_KERNEL); + if (!wfilter) + goto err_filter; + wfilter->nr_filters = nr_filter; + + q = wfilter->filters; + for (i = 0; i < filter.nr_filters; i++) { + if (tf[i].type >= sizeof(wfilter->type_filter) * BITS_PER_LONG) + continue; + + q->type = tf[i].type; + q->info_filter = tf[i].info_filter; + q->info_mask = tf[i].info_mask; + q->subtype_filter[0] = tf[i].subtype_filter[0]; + __set_bit(q->type, wfilter->type_filter); + q++; + } + + kfree(tf); +set: + pipe_lock(pipe); + wfilter = rcu_replace_pointer(wqueue->filter, wfilter, + lockdep_is_held(&pipe->mutex)); + pipe_unlock(pipe); + if (wfilter) + kfree_rcu(wfilter, rcu); + return 0; + +err_filter: + kfree(tf); + return ret; +} + +static void __put_watch_queue(struct kref *kref) +{ + struct watch_queue *wqueue = + container_of(kref, struct watch_queue, usage); + struct watch_filter *wfilter; + int i; + + for (i = 0; i < wqueue->nr_pages; i++) + __free_page(wqueue->notes[i]); + + wfilter = rcu_access_pointer(wqueue->filter); + if (wfilter) + kfree_rcu(wfilter, rcu); + kfree_rcu(wqueue, rcu); +} + +/** + * put_watch_queue - Dispose of a ref on a watchqueue. + * @wqueue: The watch queue to unref. + */ +void put_watch_queue(struct watch_queue *wqueue) +{ + kref_put(&wqueue->usage, __put_watch_queue); +} +EXPORT_SYMBOL(put_watch_queue); + +static void free_watch(struct rcu_head *rcu) +{ + struct watch *watch = container_of(rcu, struct watch, rcu); + + put_watch_queue(rcu_access_pointer(watch->queue)); + put_cred(watch->cred); +} + +static void __put_watch(struct kref *kref) +{ + struct watch *watch = container_of(kref, struct watch, usage); + + call_rcu(&watch->rcu, free_watch); +} + +/* + * Discard a watch. + */ +static void put_watch(struct watch *watch) +{ + kref_put(&watch->usage, __put_watch); +} + +/** + * init_watch_queue - Initialise a watch + * @watch: The watch to initialise. + * @wqueue: The queue to assign. + * + * Initialise a watch and set the watch queue. + */ +void init_watch(struct watch *watch, struct watch_queue *wqueue) +{ + kref_init(&watch->usage); + INIT_HLIST_NODE(&watch->list_node); + INIT_HLIST_NODE(&watch->queue_node); + rcu_assign_pointer(watch->queue, wqueue); +} + +/** + * add_watch_to_object - Add a watch on an object to a watch list + * @watch: The watch to add + * @wlist: The watch list to add to + * + * @watch->queue must have been set to point to the queue to post notifications + * to and the watch list of the object to be watched. @watch->cred must also + * have been set to the appropriate credentials and a ref taken on them. + * + * The caller must pin the queue and the list both and must hold the list + * locked against racing watch additions/removals. + */ +int add_watch_to_object(struct watch *watch, struct watch_list *wlist) +{ + struct watch_queue *wqueue = rcu_access_pointer(watch->queue); + struct watch *w; + + hlist_for_each_entry(w, &wlist->watchers, list_node) { + struct watch_queue *wq = rcu_access_pointer(w->queue); + if (wqueue == wq && watch->id == w->id) + return -EBUSY; + } + + watch->cred = get_current_cred(); + rcu_assign_pointer(watch->watch_list, wlist); + + spin_lock_bh(&wqueue->lock); + kref_get(&wqueue->usage); + kref_get(&watch->usage); + hlist_add_head(&watch->queue_node, &wqueue->watches); + spin_unlock_bh(&wqueue->lock); + + hlist_add_head(&watch->list_node, &wlist->watchers); + return 0; +} +EXPORT_SYMBOL(add_watch_to_object); + +/** + * remove_watch_from_object - Remove a watch or all watches from an object. + * @wlist: The watch list to remove from + * @wq: The watch queue of interest (ignored if @all is true) + * @id: The ID of the watch to remove (ignored if @all is true) + * @all: True to remove all objects + * + * Remove a specific watch or all watches from an object. A notification is + * sent to the watcher to tell them that this happened. + */ +int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq, + u64 id, bool all) +{ + struct watch_notification_removal n; + struct watch_queue *wqueue; + struct watch *watch; + int ret = -EBADSLT; + + rcu_read_lock(); + +again: + spin_lock(&wlist->lock); + hlist_for_each_entry(watch, &wlist->watchers, list_node) { + if (all || + (watch->id == id && rcu_access_pointer(watch->queue) == wq)) + goto found; + } + spin_unlock(&wlist->lock); + goto out; + +found: + ret = 0; + hlist_del_init_rcu(&watch->list_node); + rcu_assign_pointer(watch->watch_list, NULL); + spin_unlock(&wlist->lock); + + /* We now own the reference on watch that used to belong to wlist. */ + + n.watch.type = WATCH_TYPE_META; + n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION; + n.watch.info = watch->info_id | watch_sizeof(n.watch); + n.id = id; + if (id != 0) + n.watch.info = watch->info_id | watch_sizeof(n); + + wqueue = rcu_dereference(watch->queue); + + /* We don't need the watch list lock for the next bit as RCU is + * protecting *wqueue from deallocation. + */ + if (wqueue) { + post_one_notification(wqueue, &n.watch); + + spin_lock_bh(&wqueue->lock); + + if (!hlist_unhashed(&watch->queue_node)) { + hlist_del_init_rcu(&watch->queue_node); + put_watch(watch); + } + + spin_unlock_bh(&wqueue->lock); + } + + if (wlist->release_watch) { + void (*release_watch)(struct watch *); + + release_watch = wlist->release_watch; + rcu_read_unlock(); + (*release_watch)(watch); + rcu_read_lock(); + } + put_watch(watch); + + if (all && !hlist_empty(&wlist->watchers)) + goto again; +out: + rcu_read_unlock(); + return ret; +} +EXPORT_SYMBOL(remove_watch_from_object); + +/* + * Remove all the watches that are contributory to a queue. This has the + * potential to race with removal of the watches by the destruction of the + * objects being watched or with the distribution of notifications. + */ +void watch_queue_clear(struct watch_queue *wqueue) +{ + struct watch_list *wlist; + struct watch *watch; + bool release; + + rcu_read_lock(); + spin_lock_bh(&wqueue->lock); + + /* Prevent new additions and prevent notifications from happening */ + wqueue->defunct = true; + + while (!hlist_empty(&wqueue->watches)) { + watch = hlist_entry(wqueue->watches.first, struct watch, queue_node); + hlist_del_init_rcu(&watch->queue_node); + /* We now own a ref on the watch. */ + spin_unlock_bh(&wqueue->lock); + + /* We can't do the next bit under the queue lock as we need to + * get the list lock - which would cause a deadlock if someone + * was removing from the opposite direction at the same time or + * posting a notification. + */ + wlist = rcu_dereference(watch->watch_list); + if (wlist) { + void (*release_watch)(struct watch *); + + spin_lock(&wlist->lock); + + release = !hlist_unhashed(&watch->list_node); + if (release) { + hlist_del_init_rcu(&watch->list_node); + rcu_assign_pointer(watch->watch_list, NULL); + + /* We now own a second ref on the watch. */ + } + + release_watch = wlist->release_watch; + spin_unlock(&wlist->lock); + + if (release) { + if (release_watch) { + rcu_read_unlock(); + /* This might need to call dput(), so + * we have to drop all the locks. + */ + (*release_watch)(watch); + rcu_read_lock(); + } + put_watch(watch); + } + } + + put_watch(watch); + spin_lock_bh(&wqueue->lock); + } + + spin_unlock_bh(&wqueue->lock); + rcu_read_unlock(); +} + +/** + * get_watch_queue - Get a watch queue from its file descriptor. + * @fd: The fd to query. + */ +struct watch_queue *get_watch_queue(int fd) +{ + struct pipe_inode_info *pipe; + struct watch_queue *wqueue = ERR_PTR(-EINVAL); + struct fd f; + + f = fdget(fd); + if (f.file) { + pipe = get_pipe_info(f.file, false); + if (pipe && pipe->watch_queue) { + wqueue = pipe->watch_queue; + kref_get(&wqueue->usage); + } + fdput(f); + } + + return wqueue; +} +EXPORT_SYMBOL(get_watch_queue); + +/* + * Initialise a watch queue + */ +int watch_queue_init(struct pipe_inode_info *pipe) +{ + struct watch_queue *wqueue; + + wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL); + if (!wqueue) + return -ENOMEM; + + wqueue->pipe = pipe; + kref_init(&wqueue->usage); + spin_lock_init(&wqueue->lock); + INIT_HLIST_HEAD(&wqueue->watches); + + pipe->watch_queue = wqueue; + return 0; +} diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 29bb066dceb0..d74ac0fd6b2d 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -1570,6 +1570,8 @@ config PROVIDE_OHCI1394_DMA_INIT source "samples/Kconfig" +source "lib/Kconfig.kcsan" + config ARCH_HAS_DEVMEM_IS_ALLOWED bool @@ -2052,15 +2054,15 @@ config TEST_LKM If unsure, say N. config TEST_BITOPS - tristate "Test module for compilation of clear_bit/set_bit operations" + tristate "Test module for compilation of bitops operations" depends on m help This builds the "test_bitops" module that is much like the TEST_LKM module except that it does a basic exercise of the - clear_bit and set_bit macros to make sure there are no compiler - warnings from C=1 sparse checker or -Wextra compilations. It has - no dependencies and doesn't run or load unless explicitly requested - by name. for example: modprobe test_bitops. + set/clear_bit macros and get_count_order/long to make sure there are + no compiler warnings from C=1 sparse checker or -Wextra + compilations. It has no dependencies and doesn't run or load unless + explicitly requested by name. for example: modprobe test_bitops. If unsure, say N. diff --git a/lib/Kconfig.kcsan b/lib/Kconfig.kcsan new file mode 100644 index 000000000000..5ee88e5119c2 --- /dev/null +++ b/lib/Kconfig.kcsan @@ -0,0 +1,199 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config HAVE_ARCH_KCSAN + bool + +config HAVE_KCSAN_COMPILER + def_bool CC_IS_CLANG && $(cc-option,-fsanitize=thread -mllvm -tsan-distinguish-volatile=1) + help + For the list of compilers that support KCSAN, please see + <file:Documentation/dev-tools/kcsan.rst>. + +config KCSAN_KCOV_BROKEN + def_bool KCOV && CC_HAS_SANCOV_TRACE_PC + depends on CC_IS_CLANG + depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=thread -fsanitize-coverage=trace-pc) + help + Some versions of clang support either KCSAN and KCOV but not the + combination of the two. + See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status + in newer releases. + +menuconfig KCSAN + bool "KCSAN: dynamic data race detector" + depends on HAVE_ARCH_KCSAN && HAVE_KCSAN_COMPILER + depends on DEBUG_KERNEL && !KASAN + depends on !KCSAN_KCOV_BROKEN + select STACKTRACE + help + The Kernel Concurrency Sanitizer (KCSAN) is a dynamic + data-race detector that relies on compile-time instrumentation. + KCSAN uses a watchpoint-based sampling approach to detect races. + + While KCSAN's primary purpose is to detect data races, it + also provides assertions to check data access constraints. + These assertions can expose bugs that do not manifest as + data races. + + See <file:Documentation/dev-tools/kcsan.rst> for more details. + +if KCSAN + +config KCSAN_VERBOSE + bool "Show verbose reports with more information about system state" + depends on PROVE_LOCKING + help + If enabled, reports show more information about the system state that + may help better analyze and debug races. This includes held locks and + IRQ trace events. + + While this option should generally be benign, we call into more + external functions on report generation; if a race report is + generated from any one of them, system stability may suffer due to + deadlocks or recursion. If in doubt, say N. + +config KCSAN_DEBUG + bool "Debugging of KCSAN internals" + +config KCSAN_SELFTEST + bool "Perform short selftests on boot" + default y + help + Run KCSAN selftests on boot. On test failure, causes the kernel to panic. + +config KCSAN_EARLY_ENABLE + bool "Early enable during boot" + default y + help + If KCSAN should be enabled globally as soon as possible. KCSAN can + later be enabled/disabled via debugfs. + +config KCSAN_NUM_WATCHPOINTS + int "Number of available watchpoints" + default 64 + help + Total number of available watchpoints. An address range maps into a + specific watchpoint slot as specified in kernel/kcsan/encoding.h. + Although larger number of watchpoints may not be usable due to + limited number of CPUs, a larger value helps to improve performance + due to reducing cache-line contention. The chosen default is a + conservative value; we should almost never observe "no_capacity" + events (see /sys/kernel/debug/kcsan). + +config KCSAN_UDELAY_TASK + int "Delay in microseconds (for tasks)" + default 80 + help + For tasks, the microsecond delay after setting up a watchpoint. + +config KCSAN_UDELAY_INTERRUPT + int "Delay in microseconds (for interrupts)" + default 20 + help + For interrupts, the microsecond delay after setting up a watchpoint. + Interrupts have tighter latency requirements, and their delay should + be lower than for tasks. + +config KCSAN_DELAY_RANDOMIZE + bool "Randomize above delays" + default y + help + If delays should be randomized, where the maximum is KCSAN_UDELAY_*. + If false, the chosen delays are always the KCSAN_UDELAY_* values + as defined above. + +config KCSAN_SKIP_WATCH + int "Skip instructions before setting up watchpoint" + default 4000 + help + The number of per-CPU memory operations to skip, before another + watchpoint is set up, i.e. one in KCSAN_WATCH_SKIP per-CPU + memory operations are used to set up a watchpoint. A smaller value + results in more aggressive race detection, whereas a larger value + improves system performance at the cost of missing some races. + +config KCSAN_SKIP_WATCH_RANDOMIZE + bool "Randomize watchpoint instruction skip count" + default y + help + If instruction skip count should be randomized, where the maximum is + KCSAN_WATCH_SKIP. If false, the chosen value is always + KCSAN_WATCH_SKIP. + +config KCSAN_INTERRUPT_WATCHER + bool "Interruptible watchers" + help + If enabled, a task that set up a watchpoint may be interrupted while + delayed. This option will allow KCSAN to detect races between + interrupted tasks and other threads of execution on the same CPU. + + Currently disabled by default, because not all safe per-CPU access + primitives and patterns may be accounted for, and therefore could + result in false positives. + +config KCSAN_REPORT_ONCE_IN_MS + int "Duration in milliseconds, in which any given race is only reported once" + default 3000 + help + Any given race is only reported once in the defined time window. + Different races may still generate reports within a duration that is + smaller than the duration defined here. This allows rate limiting + reporting to avoid flooding the console with reports. Setting this + to 0 disables rate limiting. + +# The main purpose of the below options is to control reported data races (e.g. +# in fuzzer configs), and are not expected to be switched frequently by other +# users. We could turn some of them into boot parameters, but given they should +# not be switched normally, let's keep them here to simplify configuration. +# +# The defaults below are chosen to be very conservative, and may miss certain +# bugs. + +config KCSAN_REPORT_RACE_UNKNOWN_ORIGIN + bool "Report races of unknown origin" + default y + help + If KCSAN should report races where only one access is known, and the + conflicting access is of unknown origin. This type of race is + reported if it was only possible to infer a race due to a data value + change while an access is being delayed on a watchpoint. + +config KCSAN_REPORT_VALUE_CHANGE_ONLY + bool "Only report races where watcher observed a data value change" + default y + help + If enabled and a conflicting write is observed via a watchpoint, but + the data value of the memory location was observed to remain + unchanged, do not report the data race. + +config KCSAN_ASSUME_PLAIN_WRITES_ATOMIC + bool "Assume that plain aligned writes up to word size are atomic" + default y + help + Assume that plain aligned writes up to word size are atomic by + default, and also not subject to other unsafe compiler optimizations + resulting in data races. This will cause KCSAN to not report data + races due to conflicts where the only plain accesses are aligned + writes up to word size: conflicts between marked reads and plain + aligned writes up to word size will not be reported as data races; + notice that data races between two conflicting plain aligned writes + will also not be reported. + +config KCSAN_IGNORE_ATOMICS + bool "Do not instrument marked atomic accesses" + help + Never instrument marked atomic accesses. This option can be used for + additional filtering. Conflicting marked atomic reads and plain + writes will never be reported as a data race, however, will cause + plain reads and marked writes to result in "unknown origin" reports. + If combined with CONFIG_KCSAN_REPORT_RACE_UNKNOWN_ORIGIN=n, data + races where at least one access is marked atomic will never be + reported. + + Similar to KCSAN_ASSUME_PLAIN_WRITES_ATOMIC, but including unaligned + accesses, conflicting marked atomic reads and plain writes will not + be reported as data races; however, unlike that option, data races + due to two conflicting plain writes will be reported (aligned and + unaligned, if CONFIG_KCSAN_ASSUME_PLAIN_WRITES_ATOMIC=n). + +endif # KCSAN diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 27bcc2568c95..774315de555a 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -26,9 +26,20 @@ config UBSAN_TRAP the system. For some system builders this is an acceptable trade-off. +config UBSAN_KCOV_BROKEN + def_bool KCOV && CC_HAS_SANCOV_TRACE_PC + depends on CC_IS_CLANG + depends on !$(cc-option,-Werror=unused-command-line-argument -fsanitize=bounds -fsanitize-coverage=trace-pc) + help + Some versions of clang support either UBSAN or KCOV but not the + combination of the two. + See https://bugs.llvm.org/show_bug.cgi?id=45831 for the status + in newer releases. + config UBSAN_BOUNDS bool "Perform array index bounds checking" default UBSAN + depends on !UBSAN_KCOV_BROKEN help This option enables detection of directly indexed out of bounds array accesses, where the array size is known at compile time. diff --git a/lib/Makefile b/lib/Makefile index 315516fa4ef4..b1c42c10073b 100644 --- a/lib/Makefile +++ b/lib/Makefile @@ -25,6 +25,9 @@ KASAN_SANITIZE_string.o := n CFLAGS_string.o := $(call cc-option, -fno-stack-protector) endif +# Used by KCSAN while enabled, avoid recursion. +KCSAN_SANITIZE_random32.o := n + lib-y := ctype.o string.o vsprintf.o cmdline.o \ rbtree.o radix-tree.o timerqueue.o xarray.o \ idr.o extable.o sha1.o irq_regs.o argv_split.o \ @@ -296,6 +299,7 @@ endif UBSAN_SANITIZE_ubsan.o := n KASAN_SANITIZE_ubsan.o := n +KCSAN_SANITIZE_ubsan.o := n CFLAGS_ubsan.o := $(call cc-option, -fno-stack-protector) $(DISABLE_STACKLEAK_PLUGIN) obj-$(CONFIG_SBITMAP) += sbitmap.o diff --git a/lib/bitmap.c b/lib/bitmap.c index 21a7640c5eed..0364452b1617 100644 --- a/lib/bitmap.c +++ b/lib/bitmap.c @@ -741,8 +741,9 @@ int bitmap_parse(const char *start, unsigned int buflen, int chunks = BITS_TO_U32(nmaskbits); u32 *bitmap = (u32 *)maskp; int unset_bit; + int chunk; - while (1) { + for (chunk = 0; ; chunk++) { end = bitmap_find_region_reverse(start, end); if (start > end) break; @@ -750,7 +751,11 @@ int bitmap_parse(const char *start, unsigned int buflen, if (!chunks--) return -EOVERFLOW; - end = bitmap_get_x32_reverse(start, end, bitmap++); +#if defined(CONFIG_64BIT) && defined(__BIG_ENDIAN) + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk ^ 1]); +#else + end = bitmap_get_x32_reverse(start, end, &bitmap[chunk]); +#endif if (IS_ERR(end)) return PTR_ERR(end); } diff --git a/lib/bsearch.c b/lib/bsearch.c index 8b3aae5ae77a..bf86aa66f2b2 100644 --- a/lib/bsearch.c +++ b/lib/bsearch.c @@ -28,27 +28,9 @@ * the key and elements in the array are of the same type, you can use * the same comparison function for both sort() and bsearch(). */ -void *bsearch(const void *key, const void *base, size_t num, size_t size, - cmp_func_t cmp) +void *bsearch(const void *key, const void *base, size_t num, size_t size, cmp_func_t cmp) { - const char *pivot; - int result; - - while (num > 0) { - pivot = base + (num >> 1) * size; - result = cmp(key, pivot); - - if (result == 0) - return (void *)pivot; - - if (result > 0) { - base = pivot + size; - num--; - } - num >>= 1; - } - - return NULL; + return __inline_bsearch(key, base, num, size, cmp); } EXPORT_SYMBOL(bsearch); NOKPROBE_SYMBOL(bsearch); diff --git a/lib/iov_iter.c b/lib/iov_iter.c index 51595bf3af85..bf538c2bec77 100644 --- a/lib/iov_iter.c +++ b/lib/iov_iter.c @@ -8,6 +8,7 @@ #include <linux/splice.h> #include <net/checksum.h> #include <linux/scatterlist.h> +#include <linux/instrumented.h> #define PIPE_PARANOIA /* for now */ @@ -138,7 +139,7 @@ static int copyout(void __user *to, const void *from, size_t n) { if (access_ok(to, n)) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n; @@ -147,7 +148,7 @@ static int copyout(void __user *to, const void *from, size_t n) static int copyin(void *to, const void __user *from, size_t n) { if (access_ok(from, n)) { - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); n = raw_copy_from_user(to, from, n); } return n; @@ -639,7 +640,7 @@ EXPORT_SYMBOL(_copy_to_iter); static int copyout_mcsafe(void __user *to, const void *from, size_t n) { if (access_ok(to, n)) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = copy_to_user_mcsafe((__force void *) to, from, n); } return n; diff --git a/lib/lz4/lz4_decompress.c b/lib/lz4/lz4_decompress.c index 0c9d3ad17e0f..5371dab6b481 100644 --- a/lib/lz4/lz4_decompress.c +++ b/lib/lz4/lz4_decompress.c @@ -141,6 +141,9 @@ static FORCE_INLINE int LZ4_decompress_generic( * space in the output for those 18 bytes earlier, upon * entering the shortcut (in other words, there is a * combined check for both stages). + * + * The & in the likely() below is intentionally not && so that + * some compilers can produce better parallelized runtime code */ if ((endOnInput ? length != RUN_MASK : length <= 8) /* diff --git a/lib/lzo/lzo1x_compress.c b/lib/lzo/lzo1x_compress.c index 717c940112f9..8ad5ba2b86e2 100644 --- a/lib/lzo/lzo1x_compress.c +++ b/lib/lzo/lzo1x_compress.c @@ -268,6 +268,19 @@ m_len_done: *op++ = (M4_MARKER | ((m_off >> 11) & 8) | (m_len - 2)); else { + if (unlikely(((m_off & 0x403f) == 0x403f) + && (m_len >= 261) + && (m_len <= 264)) + && likely(bitstream_version)) { + // Under lzo-rle, block copies + // for 261 <= length <= 264 and + // (distance & 0x80f3) == 0x80f3 + // can result in ambiguous + // output. Adjust length + // to 260 to prevent ambiguity. + ip -= m_len - 260; + m_len = 260; + } m_len -= M4_MAX_LEN; *op++ = (M4_MARKER | ((m_off >> 11) & 8)); while (unlikely(m_len > 255)) { diff --git a/lib/smp_processor_id.c b/lib/smp_processor_id.c index bd9571653288..525222e4f409 100644 --- a/lib/smp_processor_id.c +++ b/lib/smp_processor_id.c @@ -8,7 +8,7 @@ #include <linux/kprobes.h> #include <linux/sched.h> -notrace static nokprobe_inline +noinstr static unsigned int check_preemption_disabled(const char *what1, const char *what2) { int this_cpu = raw_smp_processor_id(); @@ -37,6 +37,7 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) */ preempt_disable_notrace(); + instrumentation_begin(); if (!printk_ratelimit()) goto out_enable; @@ -45,6 +46,7 @@ unsigned int check_preemption_disabled(const char *what1, const char *what2) printk("caller is %pS\n", __builtin_return_address(0)); dump_stack(); + instrumentation_end(); out_enable: preempt_enable_no_resched_notrace(); @@ -52,16 +54,14 @@ out: return this_cpu; } -notrace unsigned int debug_smp_processor_id(void) +noinstr unsigned int debug_smp_processor_id(void) { return check_preemption_disabled("smp_processor_id", ""); } EXPORT_SYMBOL(debug_smp_processor_id); -NOKPROBE_SYMBOL(debug_smp_processor_id); -notrace void __this_cpu_preempt_check(const char *op) +noinstr void __this_cpu_preempt_check(const char *op) { check_preemption_disabled("__this_cpu_", op); } EXPORT_SYMBOL(__this_cpu_preempt_check); -NOKPROBE_SYMBOL(__this_cpu_preempt_check); diff --git a/lib/test_bitops.c b/lib/test_bitops.c index fd50b3ae4a14..ced25e3a779b 100644 --- a/lib/test_bitops.c +++ b/lib/test_bitops.c @@ -9,7 +9,11 @@ #include <linux/module.h> #include <linux/printk.h> -/* a tiny module only meant to test set/clear_bit */ +/* a tiny module only meant to test + * + * set/clear_bit + * get_count_order/long + */ /* use an enum because thats the most common BITMAP usage */ enum bitops_fun { @@ -24,14 +28,59 @@ enum bitops_fun { static DECLARE_BITMAP(g_bitmap, BITOPS_LENGTH); +static unsigned int order_comb[][2] = { + {0x00000003, 2}, + {0x00000004, 2}, + {0x00001fff, 13}, + {0x00002000, 13}, + {0x50000000, 31}, + {0x80000000, 31}, + {0x80003000, 32}, +}; + +#ifdef CONFIG_64BIT +static unsigned long order_comb_long[][2] = { + {0x0000000300000000, 34}, + {0x0000000400000000, 34}, + {0x00001fff00000000, 45}, + {0x0000200000000000, 45}, + {0x5000000000000000, 63}, + {0x8000000000000000, 63}, + {0x8000300000000000, 64}, +}; +#endif + static int __init test_bitops_startup(void) { + int i; + pr_warn("Loaded test module\n"); set_bit(BITOPS_4, g_bitmap); set_bit(BITOPS_7, g_bitmap); set_bit(BITOPS_11, g_bitmap); set_bit(BITOPS_31, g_bitmap); set_bit(BITOPS_88, g_bitmap); + + for (i = 0; i < ARRAY_SIZE(order_comb); i++) { + if (order_comb[i][1] != get_count_order(order_comb[i][0])) + pr_warn("get_count_order wrong for %x\n", + order_comb[i][0]); + } + + for (i = 0; i < ARRAY_SIZE(order_comb); i++) { + if (order_comb[i][1] != get_count_order_long(order_comb[i][0])) + pr_warn("get_count_order_long wrong for %x\n", + order_comb[i][0]); + } + +#ifdef CONFIG_64BIT + for (i = 0; i < ARRAY_SIZE(order_comb_long); i++) { + if (order_comb_long[i][1] != + get_count_order_long(order_comb_long[i][0])) + pr_warn("get_count_order_long wrong for %lx\n", + order_comb_long[i][0]); + } +#endif return 0; } @@ -55,6 +104,6 @@ static void __exit test_bitops_unstartup(void) module_init(test_bitops_startup); module_exit(test_bitops_unstartup); -MODULE_AUTHOR("Jesse Brandeburg <jesse.brandeburg@intel.com>"); +MODULE_AUTHOR("Jesse Brandeburg <jesse.brandeburg@intel.com>, Wei Yang <richard.weiyang@gmail.com>"); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Bit testing module"); diff --git a/lib/usercopy.c b/lib/usercopy.c index ca2a697a2061..b26509f112f9 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 -#include <linux/uaccess.h> #include <linux/bitops.h> +#include <linux/instrumented.h> +#include <linux/uaccess.h> /* out-of-line parts */ @@ -10,7 +11,7 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n unsigned long res = n; might_fault(); if (likely(access_ok(from, n))) { - kasan_check_write(to, n); + instrument_copy_from_user(to, from, n); res = raw_copy_from_user(to, from, n); } if (unlikely(res)) @@ -25,7 +26,7 @@ unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { might_fault(); if (likely(access_ok(to, n))) { - kasan_check_read(from, n); + instrument_copy_to_user(to, from, n); n = raw_copy_to_user(to, from, n); } return n; diff --git a/lib/vdso/gettimeofday.c b/lib/vdso/gettimeofday.c index a2909af4b924..bcc9a98a0524 100644 --- a/lib/vdso/gettimeofday.c +++ b/lib/vdso/gettimeofday.c @@ -38,6 +38,13 @@ static inline bool vdso_clocksource_ok(const struct vdso_data *vd) } #endif +#ifndef vdso_cycles_ok +static inline bool vdso_cycles_ok(u64 cycles) +{ + return true; +} +#endif + #ifdef CONFIG_TIME_NS static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, struct __kernel_timespec *ts) @@ -62,6 +69,8 @@ static int do_hres_timens(const struct vdso_data *vdns, clockid_t clk, return -1; cycles = __arch_get_hw_counter(vd->clock_mode); + if (unlikely(!vdso_cycles_ok(cycles))) + return -1; ns = vdso_ts->nsec; last = vd->cycle_last; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); @@ -130,6 +139,8 @@ static __always_inline int do_hres(const struct vdso_data *vd, clockid_t clk, return -1; cycles = __arch_get_hw_counter(vd->clock_mode); + if (unlikely(!vdso_cycles_ok(cycles))) + return -1; ns = vdso_ts->nsec; last = vd->cycle_last; ns += vdso_calc_delta(cycles, last, vd->mask, vd->mult); @@ -210,7 +221,7 @@ static __always_inline int do_coarse(const struct vdso_data *vd, clockid_t clk, return 0; } -static __maybe_unused int +static __always_inline int __cvdso_clock_gettime_common(const struct vdso_data *vd, clockid_t clock, struct __kernel_timespec *ts) { diff --git a/mm/Makefile b/mm/Makefile index 662fd1504646..6e9d46b2efc9 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -8,6 +8,14 @@ KASAN_SANITIZE_slab.o := n KASAN_SANITIZE_slub.o := n KCSAN_SANITIZE_kmemleak.o := n +# These produce frequent data race reports: most of them are due to races on +# the same word but accesses to different bits of that word. Re-enable KCSAN +# for these when we have more consensus on what to do about them. +KCSAN_SANITIZE_slab_common.o := n +KCSAN_SANITIZE_slab.o := n +KCSAN_SANITIZE_slub.o := n +KCSAN_SANITIZE_page_alloc.o := n + # These files are disabled because they produce non-interesting and/or # flaky coverage that is not a function of syscall inputs. E.g. slab is out of # free pages, or a task is migrated between nodes. @@ -41,7 +49,7 @@ obj-y := filemap.o mempool.o oom_kill.o fadvise.o \ maccess.o page-writeback.o \ readahead.o swap.o truncate.o vmscan.o shmem.o \ util.o mmzone.o vmstat.o backing-dev.o \ - mm_init.o mmu_context.o percpu.o slab_common.o \ + mm_init.o percpu.o slab_common.o \ compaction.o vmacache.o \ interval_tree.o list_lru.o workingset.o \ debug.o gup.o $(mmu-y) diff --git a/mm/debug_vm_pgtable.c b/mm/debug_vm_pgtable.c index 9ec59c38d6a2..e45623016aea 100644 --- a/mm/debug_vm_pgtable.c +++ b/mm/debug_vm_pgtable.c @@ -60,6 +60,9 @@ static void __init pmd_basic_tests(unsigned long pfn, pgprot_t prot) { pmd_t pmd = pfn_pmd(pfn, prot); + if (!has_transparent_hugepage()) + return; + WARN_ON(!pmd_same(pmd, pmd)); WARN_ON(!pmd_young(pmd_mkyoung(pmd_mkold(pmd)))); WARN_ON(!pmd_dirty(pmd_mkdirty(pmd_mkclean(pmd)))); @@ -79,6 +82,9 @@ static void __init pud_basic_tests(unsigned long pfn, pgprot_t prot) { pud_t pud = pfn_pud(pfn, prot); + if (!has_transparent_hugepage()) + return; + WARN_ON(!pud_same(pud, pud)); WARN_ON(!pud_young(pud_mkyoung(pud_mkold(pud)))); WARN_ON(!pud_write(pud_mkwrite(pud_wrprotect(pud)))); diff --git a/mm/memory-failure.c b/mm/memory-failure.c index ababa368cb68..47b8ccb1fb9b 100644 --- a/mm/memory-failure.c +++ b/mm/memory-failure.c @@ -212,15 +212,13 @@ static int kill_proc(struct to_kill *tk, unsigned long pfn, int flags) short addr_lsb = tk->size_shift; int ret = 0; - if ((t->mm == current->mm) || !(flags & MF_ACTION_REQUIRED)) - pr_err("Memory failure: %#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", + pr_err("Memory failure: %#lx: Sending SIGBUS to %s:%d due to hardware memory corruption\n", pfn, t->comm, t->pid); if (flags & MF_ACTION_REQUIRED) { - if (t->mm == current->mm) - ret = force_sig_mceerr(BUS_MCEERR_AR, + WARN_ON_ONCE(t != current); + ret = force_sig_mceerr(BUS_MCEERR_AR, (void __user *)tk->addr, addr_lsb); - /* send no signal to non-current processes */ } else { /* * Don't use force here, it's convenient if the signal @@ -402,9 +400,15 @@ static struct task_struct *find_early_kill_thread(struct task_struct *tsk) { struct task_struct *t; - for_each_thread(tsk, t) - if ((t->flags & PF_MCE_PROCESS) && (t->flags & PF_MCE_EARLY)) - return t; + for_each_thread(tsk, t) { + if (t->flags & PF_MCE_PROCESS) { + if (t->flags & PF_MCE_EARLY) + return t; + } else { + if (sysctl_memory_failure_early_kill) + return t; + } + } return NULL; } @@ -413,21 +417,26 @@ static struct task_struct *find_early_kill_thread(struct task_struct *tsk) * to be signaled when some page under the process is hwpoisoned. * Return task_struct of the dedicated thread (main thread unless explicitly * specified) if the process is "early kill," and otherwise returns NULL. + * + * Note that the above is true for Action Optional case, but not for Action + * Required case where SIGBUS should sent only to the current thread. */ static struct task_struct *task_early_kill(struct task_struct *tsk, int force_early) { - struct task_struct *t; if (!tsk->mm) return NULL; - if (force_early) - return tsk; - t = find_early_kill_thread(tsk); - if (t) - return t; - if (sysctl_memory_failure_early_kill) - return tsk; - return NULL; + if (force_early) { + /* + * Comparing ->mm here because current task might represent + * a subthread, while tsk always points to the main thread. + */ + if (tsk->mm == current->mm) + return current; + else + return NULL; + } + return find_early_kill_thread(tsk); } /* diff --git a/mm/mmu_context.c b/mm/mmu_context.c deleted file mode 100644 index 3e612ae748e9..000000000000 --- a/mm/mmu_context.c +++ /dev/null @@ -1,64 +0,0 @@ -/* Copyright (C) 2009 Red Hat, Inc. - * - * See ../COPYING for licensing terms. - */ - -#include <linux/mm.h> -#include <linux/sched.h> -#include <linux/sched/mm.h> -#include <linux/sched/task.h> -#include <linux/mmu_context.h> -#include <linux/export.h> - -#include <asm/mmu_context.h> - -/* - * use_mm - * Makes the calling kernel thread take on the specified - * mm context. - * (Note: this routine is intended to be called only - * from a kernel thread context) - */ -void use_mm(struct mm_struct *mm) -{ - struct mm_struct *active_mm; - struct task_struct *tsk = current; - - task_lock(tsk); - active_mm = tsk->active_mm; - if (active_mm != mm) { - mmgrab(mm); - tsk->active_mm = mm; - } - tsk->mm = mm; - switch_mm(active_mm, mm, tsk); - task_unlock(tsk); -#ifdef finish_arch_post_lock_switch - finish_arch_post_lock_switch(); -#endif - - if (active_mm != mm) - mmdrop(active_mm); -} -EXPORT_SYMBOL_GPL(use_mm); - -/* - * unuse_mm - * Reverses the effect of use_mm, i.e. releases the - * specified mm context which was earlier taken on - * by the calling kernel thread - * (Note: this routine is intended to be called only - * from a kernel thread context) - */ -void unuse_mm(struct mm_struct *mm) -{ - struct task_struct *tsk = current; - - task_lock(tsk); - sync_mm_rss(mm); - tsk->mm = NULL; - /* active_mm is still 'mm' */ - enter_lazy_tlb(mm, tsk); - task_unlock(tsk); -} -EXPORT_SYMBOL_GPL(unuse_mm); diff --git a/mm/oom_kill.c b/mm/oom_kill.c index b4e9491cb320..6e94962893ee 100644 --- a/mm/oom_kill.c +++ b/mm/oom_kill.c @@ -126,7 +126,7 @@ static bool oom_cpuset_eligible(struct task_struct *tsk, struct oom_control *oc) /* * The process p may have detached its own ->mm while exiting or through - * use_mm(), but one or more of its subthreads may still have a valid + * kthread_use_mm(), but one or more of its subthreads may still have a valid * pointer. Return p, or any of its subthreads with a valid ->mm, with * task_lock() held. */ @@ -919,8 +919,8 @@ static void __oom_kill_process(struct task_struct *victim, const char *message) continue; } /* - * No use_mm() user needs to read from the userspace so we are - * ok to reap it. + * No kthead_use_mm() user needs to read from the userspace so + * we are ok to reap it. */ if (unlikely(p->flags & PF_KTHREAD)) continue; diff --git a/mm/vmacache.c b/mm/vmacache.c index d9092814c772..01a6e6688ec1 100644 --- a/mm/vmacache.c +++ b/mm/vmacache.c @@ -24,8 +24,8 @@ * task's vmacache pertains to a different mm (ie, its own). There is * nothing we can do here. * - * Also handle the case where a kernel thread has adopted this mm via use_mm(). - * That kernel thread's vmacache is not applicable to this mm. + * Also handle the case where a kernel thread has adopted this mm via + * kthread_use_mm(). That kernel thread's vmacache is not applicable to this mm. */ static inline bool vmacache_valid_mm(struct mm_struct *mm) { diff --git a/net/9p/trans_xen.c b/net/9p/trans_xen.c index 3963eb11c3fb..3debad93be1a 100644 --- a/net/9p/trans_xen.c +++ b/net/9p/trans_xen.c @@ -43,8 +43,8 @@ #include <net/9p/transport.h> #define XEN_9PFS_NUM_RINGS 2 -#define XEN_9PFS_RING_ORDER 6 -#define XEN_9PFS_RING_SIZE XEN_FLEX_RING_SIZE(XEN_9PFS_RING_ORDER) +#define XEN_9PFS_RING_ORDER 9 +#define XEN_9PFS_RING_SIZE(ring) XEN_FLEX_RING_SIZE(ring->intf->ring_order) struct xen_9pfs_header { uint32_t size; @@ -132,8 +132,8 @@ static bool p9_xen_write_todo(struct xen_9pfs_dataring *ring, RING_IDX size) prod = ring->intf->out_prod; virt_mb(); - return XEN_9PFS_RING_SIZE - - xen_9pfs_queued(prod, cons, XEN_9PFS_RING_SIZE) >= size; + return XEN_9PFS_RING_SIZE(ring) - + xen_9pfs_queued(prod, cons, XEN_9PFS_RING_SIZE(ring)) >= size; } static int p9_xen_request(struct p9_client *client, struct p9_req_t *p9_req) @@ -167,17 +167,18 @@ again: prod = ring->intf->out_prod; virt_mb(); - if (XEN_9PFS_RING_SIZE - xen_9pfs_queued(prod, cons, - XEN_9PFS_RING_SIZE) < size) { + if (XEN_9PFS_RING_SIZE(ring) - + xen_9pfs_queued(prod, cons, XEN_9PFS_RING_SIZE(ring)) < size) { spin_unlock_irqrestore(&ring->lock, flags); goto again; } - masked_prod = xen_9pfs_mask(prod, XEN_9PFS_RING_SIZE); - masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE); + masked_prod = xen_9pfs_mask(prod, XEN_9PFS_RING_SIZE(ring)); + masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE(ring)); xen_9pfs_write_packet(ring->data.out, p9_req->tc.sdata, size, - &masked_prod, masked_cons, XEN_9PFS_RING_SIZE); + &masked_prod, masked_cons, + XEN_9PFS_RING_SIZE(ring)); p9_req->status = REQ_STATUS_SENT; virt_wmb(); /* write ring before updating pointer */ @@ -207,19 +208,19 @@ static void p9_xen_response(struct work_struct *work) prod = ring->intf->in_prod; virt_rmb(); - if (xen_9pfs_queued(prod, cons, XEN_9PFS_RING_SIZE) < + if (xen_9pfs_queued(prod, cons, XEN_9PFS_RING_SIZE(ring)) < sizeof(h)) { notify_remote_via_irq(ring->irq); return; } - masked_prod = xen_9pfs_mask(prod, XEN_9PFS_RING_SIZE); - masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE); + masked_prod = xen_9pfs_mask(prod, XEN_9PFS_RING_SIZE(ring)); + masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE(ring)); /* First, read just the header */ xen_9pfs_read_packet(&h, ring->data.in, sizeof(h), masked_prod, &masked_cons, - XEN_9PFS_RING_SIZE); + XEN_9PFS_RING_SIZE(ring)); req = p9_tag_lookup(priv->client, h.tag); if (!req || req->status != REQ_STATUS_SENT) { @@ -233,11 +234,11 @@ static void p9_xen_response(struct work_struct *work) memcpy(&req->rc, &h, sizeof(h)); req->rc.offset = 0; - masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE); + masked_cons = xen_9pfs_mask(cons, XEN_9PFS_RING_SIZE(ring)); /* Then, read the whole packet (including the header) */ xen_9pfs_read_packet(req->rc.sdata, ring->data.in, h.size, masked_prod, &masked_cons, - XEN_9PFS_RING_SIZE); + XEN_9PFS_RING_SIZE(ring)); virt_mb(); cons += h.size; @@ -267,7 +268,7 @@ static irqreturn_t xen_9pfs_front_event_handler(int irq, void *r) static struct p9_trans_module p9_xen_trans = { .name = "xen", - .maxsize = 1 << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT), + .maxsize = 1 << (XEN_9PFS_RING_ORDER + XEN_PAGE_SHIFT - 2), .def = 1, .create = p9_xen_create, .close = p9_xen_close, @@ -295,14 +296,16 @@ static void xen_9pfs_front_free(struct xen_9pfs_front_priv *priv) if (priv->rings[i].irq > 0) unbind_from_irqhandler(priv->rings[i].irq, priv->dev); if (priv->rings[i].data.in) { - for (j = 0; j < (1 << XEN_9PFS_RING_ORDER); j++) { + for (j = 0; + j < (1 << priv->rings[i].intf->ring_order); + j++) { grant_ref_t ref; ref = priv->rings[i].intf->ref[j]; gnttab_end_foreign_access(ref, 0, 0); } free_pages((unsigned long)priv->rings[i].data.in, - XEN_9PFS_RING_ORDER - + priv->rings[i].intf->ring_order - (PAGE_SHIFT - XEN_PAGE_SHIFT)); } gnttab_end_foreign_access(priv->rings[i].ref, 0, 0); @@ -323,7 +326,8 @@ static int xen_9pfs_front_remove(struct xenbus_device *dev) } static int xen_9pfs_front_alloc_dataring(struct xenbus_device *dev, - struct xen_9pfs_dataring *ring) + struct xen_9pfs_dataring *ring, + unsigned int order) { int i = 0; int ret = -ENOMEM; @@ -342,21 +346,21 @@ static int xen_9pfs_front_alloc_dataring(struct xenbus_device *dev, goto out; ring->ref = ret; bytes = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - XEN_9PFS_RING_ORDER - (PAGE_SHIFT - XEN_PAGE_SHIFT)); + order - (PAGE_SHIFT - XEN_PAGE_SHIFT)); if (!bytes) { ret = -ENOMEM; goto out; } - for (; i < (1 << XEN_9PFS_RING_ORDER); i++) { + for (; i < (1 << order); i++) { ret = gnttab_grant_foreign_access( dev->otherend_id, virt_to_gfn(bytes) + i, 0); if (ret < 0) goto out; ring->intf->ref[i] = ret; } - ring->intf->ring_order = XEN_9PFS_RING_ORDER; + ring->intf->ring_order = order; ring->data.in = bytes; - ring->data.out = bytes + XEN_9PFS_RING_SIZE; + ring->data.out = bytes + XEN_FLEX_RING_SIZE(order); ret = xenbus_alloc_evtchn(dev, &ring->evtchn); if (ret) @@ -374,7 +378,7 @@ out: for (i--; i >= 0; i--) gnttab_end_foreign_access(ring->intf->ref[i], 0, 0); free_pages((unsigned long)bytes, - XEN_9PFS_RING_ORDER - + ring->intf->ring_order - (PAGE_SHIFT - XEN_PAGE_SHIFT)); } gnttab_end_foreign_access(ring->ref, 0, 0); @@ -404,8 +408,10 @@ static int xen_9pfs_front_probe(struct xenbus_device *dev, return -EINVAL; max_ring_order = xenbus_read_unsigned(dev->otherend, "max-ring-page-order", 0); - if (max_ring_order < XEN_9PFS_RING_ORDER) - return -EINVAL; + if (max_ring_order > XEN_9PFS_RING_ORDER) + max_ring_order = XEN_9PFS_RING_ORDER; + if (p9_xen_trans.maxsize > XEN_FLEX_RING_SIZE(max_ring_order)) + p9_xen_trans.maxsize = XEN_FLEX_RING_SIZE(max_ring_order) / 2; priv = kzalloc(sizeof(*priv), GFP_KERNEL); if (!priv) @@ -422,7 +428,8 @@ static int xen_9pfs_front_probe(struct xenbus_device *dev, for (i = 0; i < priv->num_rings; i++) { priv->rings[i].priv = priv; - ret = xen_9pfs_front_alloc_dataring(dev, &priv->rings[i]); + ret = xen_9pfs_front_alloc_dataring(dev, &priv->rings[i], + max_ring_order); if (ret < 0) goto error; } diff --git a/net/sunrpc/addr.c b/net/sunrpc/addr.c index 8b4d72b1a066..010dcb876f9d 100644 --- a/net/sunrpc/addr.c +++ b/net/sunrpc/addr.c @@ -82,11 +82,11 @@ static size_t rpc_ntop6(const struct sockaddr *sap, rc = snprintf(scopebuf, sizeof(scopebuf), "%c%u", IPV6_SCOPE_DELIMITER, sin6->sin6_scope_id); - if (unlikely((size_t)rc > sizeof(scopebuf))) + if (unlikely((size_t)rc >= sizeof(scopebuf))) return 0; len += rc; - if (unlikely(len > buflen)) + if (unlikely(len >= buflen)) return 0; strcat(buf, scopebuf); diff --git a/net/sunrpc/auth.c b/net/sunrpc/auth.c index 5748ad0ba1bd..a9f0d17fdb0d 100644 --- a/net/sunrpc/auth.c +++ b/net/sunrpc/auth.c @@ -81,7 +81,7 @@ static int param_get_hashtbl_sz(char *buffer, const struct kernel_param *kp) unsigned int nbits; nbits = *(unsigned int *)kp->arg; - return sprintf(buffer, "%u", 1U << nbits); + return sprintf(buffer, "%u\n", 1U << nbits); } #define param_check_hashtbl_sz(name, p) __param_check(name, p, unsigned int); diff --git a/net/sunrpc/auth_gss/auth_gss.c b/net/sunrpc/auth_gss/auth_gss.c index ac5cac0dd24b..4ecc2a959567 100644 --- a/net/sunrpc/auth_gss/auth_gss.c +++ b/net/sunrpc/auth_gss/auth_gss.c @@ -254,7 +254,7 @@ gss_fill_context(const void *p, const void *end, struct gss_cl_ctx *ctx, struct if (IS_ERR(p)) goto err; done: - trace_rpcgss_context(ctx->gc_expiry, now, timeout, + trace_rpcgss_context(window_size, ctx->gc_expiry, now, timeout, ctx->gc_acceptor.len, ctx->gc_acceptor.data); err: return p; @@ -697,10 +697,12 @@ retry: } schedule(); } - if (gss_msg->ctx) + if (gss_msg->ctx) { + trace_rpcgss_ctx_init(gss_cred); gss_cred_set_ctx(cred, gss_msg->ctx); - else + } else { err = gss_msg->msg.errno; + } spin_unlock(&pipe->lock); out_intr: finish_wait(&gss_msg->waitqueue, &wait); @@ -1054,11 +1056,11 @@ gss_create_new(const struct rpc_auth_create_args *args, struct rpc_clnt *clnt) auth->au_rslack = GSS_KRB5_MAX_SLACK_NEEDED >> 2; auth->au_verfsize = GSS_VERF_SLACK >> 2; auth->au_ralign = GSS_VERF_SLACK >> 2; - auth->au_flags = 0; + __set_bit(RPCAUTH_AUTH_UPDATE_SLACK, &auth->au_flags); auth->au_ops = &authgss_ops; auth->au_flavor = flavor; if (gss_pseudoflavor_to_datatouch(gss_auth->mech, flavor)) - auth->au_flags |= RPCAUTH_AUTH_DATATOUCH; + __set_bit(RPCAUTH_AUTH_DATATOUCH, &auth->au_flags); refcount_set(&auth->au_count, 1); kref_init(&gss_auth->kref); @@ -1284,8 +1286,9 @@ gss_send_destroy_context(struct rpc_cred *cred) if (new) { ctx->gc_proc = RPC_GSS_PROC_DESTROY; + trace_rpcgss_ctx_destroy(gss_cred); task = rpc_call_null(gss_auth->client, &new->gc_base, - RPC_TASK_ASYNC|RPC_TASK_SOFT); + RPC_TASK_ASYNC); if (!IS_ERR(task)) rpc_put_task(task); @@ -1349,7 +1352,6 @@ gss_destroy_nullcred(struct rpc_cred *cred) static void gss_destroy_cred(struct rpc_cred *cred) { - if (test_and_clear_bit(RPCAUTH_CRED_UPTODATE, &cred->cr_flags) != 0) gss_send_destroy_context(cred); gss_destroy_nullcred(cred); @@ -1613,6 +1615,7 @@ static int gss_renew_cred(struct rpc_task *task) new = gss_lookup_cred(auth, &acred, RPCAUTH_LOOKUP_NEW); if (IS_ERR(new)) return PTR_ERR(new); + task->tk_rqstp->rq_cred = new; put_rpccred(oldcred); return 0; @@ -1709,7 +1712,8 @@ gss_validate(struct rpc_task *task, struct xdr_stream *xdr) /* We leave it to unwrap to calculate au_rslack. For now we just * calculate the length of the verifier: */ - cred->cr_auth->au_verfsize = XDR_QUADLEN(len) + 2; + if (test_bit(RPCAUTH_AUTH_UPDATE_SLACK, &cred->cr_auth->au_flags)) + cred->cr_auth->au_verfsize = XDR_QUADLEN(len) + 2; status = 0; out: gss_put_ctx(ctx); @@ -1927,13 +1931,30 @@ out: return status; } -static int -gss_unwrap_resp_auth(struct rpc_cred *cred) +/** + * gss_update_rslack - Possibly update RPC receive buffer size estimates + * @task: rpc_task for incoming RPC Reply being unwrapped + * @cred: controlling rpc_cred for @task + * @before: XDR words needed before each RPC Reply message + * @after: XDR words needed following each RPC Reply message + * + */ +static void gss_update_rslack(struct rpc_task *task, struct rpc_cred *cred, + unsigned int before, unsigned int after) { struct rpc_auth *auth = cred->cr_auth; - auth->au_rslack = auth->au_verfsize; - auth->au_ralign = auth->au_verfsize; + if (test_and_clear_bit(RPCAUTH_AUTH_UPDATE_SLACK, &auth->au_flags)) { + auth->au_ralign = auth->au_verfsize + before; + auth->au_rslack = auth->au_verfsize + after; + trace_rpcgss_update_slack(task, auth); + } +} + +static int +gss_unwrap_resp_auth(struct rpc_task *task, struct rpc_cred *cred) +{ + gss_update_rslack(task, cred, 0, 0); return 0; } @@ -1956,7 +1977,6 @@ gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, struct xdr_stream *xdr) { struct xdr_buf gss_data, *rcv_buf = &rqstp->rq_rcv_buf; - struct rpc_auth *auth = cred->cr_auth; u32 len, offset, seqno, maj_stat; struct xdr_netobj mic; int ret; @@ -2005,8 +2025,7 @@ gss_unwrap_resp_integ(struct rpc_task *task, struct rpc_cred *cred, if (maj_stat != GSS_S_COMPLETE) goto bad_mic; - auth->au_rslack = auth->au_verfsize + 2 + 1 + XDR_QUADLEN(mic.len); - auth->au_ralign = auth->au_verfsize + 2; + gss_update_rslack(task, cred, 2, 2 + 1 + XDR_QUADLEN(mic.len)); ret = 0; out: @@ -2031,7 +2050,6 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, { struct xdr_buf *rcv_buf = &rqstp->rq_rcv_buf; struct kvec *head = rqstp->rq_rcv_buf.head; - struct rpc_auth *auth = cred->cr_auth; u32 offset, opaque_len, maj_stat; __be32 *p; @@ -2058,8 +2076,8 @@ gss_unwrap_resp_priv(struct rpc_task *task, struct rpc_cred *cred, */ xdr_init_decode(xdr, rcv_buf, p, rqstp); - auth->au_rslack = auth->au_verfsize + 2 + ctx->gc_gss_ctx->slack; - auth->au_ralign = auth->au_verfsize + 2 + ctx->gc_gss_ctx->align; + gss_update_rslack(task, cred, 2 + ctx->gc_gss_ctx->align, + 2 + ctx->gc_gss_ctx->slack); return 0; unwrap_failed: @@ -2130,7 +2148,7 @@ gss_unwrap_resp(struct rpc_task *task, struct xdr_stream *xdr) goto out_decode; switch (gss_cred->gc_service) { case RPC_GSS_SVC_NONE: - status = gss_unwrap_resp_auth(cred); + status = gss_unwrap_resp_auth(task, cred); break; case RPC_GSS_SVC_INTEGRITY: status = gss_unwrap_resp_integ(task, cred, ctx, rqstp, xdr); diff --git a/net/sunrpc/auth_gss/gss_mech_switch.c b/net/sunrpc/auth_gss/gss_mech_switch.c index 69316ab1b9fa..fae632da1058 100644 --- a/net/sunrpc/auth_gss/gss_mech_switch.c +++ b/net/sunrpc/auth_gss/gss_mech_switch.c @@ -37,6 +37,8 @@ gss_mech_free(struct gss_api_mech *gm) for (i = 0; i < gm->gm_pf_num; i++) { pf = &gm->gm_pfs[i]; + if (pf->domain) + auth_domain_put(pf->domain); kfree(pf->auth_domain_name); pf->auth_domain_name = NULL; } @@ -59,6 +61,7 @@ make_auth_domain_name(char *name) static int gss_mech_svc_setup(struct gss_api_mech *gm) { + struct auth_domain *dom; struct pf_desc *pf; int i, status; @@ -68,10 +71,13 @@ gss_mech_svc_setup(struct gss_api_mech *gm) status = -ENOMEM; if (pf->auth_domain_name == NULL) goto out; - status = svcauth_gss_register_pseudoflavor(pf->pseudoflavor, - pf->auth_domain_name); - if (status) + dom = svcauth_gss_register_pseudoflavor( + pf->pseudoflavor, pf->auth_domain_name); + if (IS_ERR(dom)) { + status = PTR_ERR(dom); goto out; + } + pf->domain = dom; } return 0; out: diff --git a/net/sunrpc/auth_gss/gss_rpc_upcall.c b/net/sunrpc/auth_gss/gss_rpc_upcall.c index 0349f455a862..af9c7f43859c 100644 --- a/net/sunrpc/auth_gss/gss_rpc_upcall.c +++ b/net/sunrpc/auth_gss/gss_rpc_upcall.c @@ -223,7 +223,7 @@ static int gssp_alloc_receive_pages(struct gssx_arg_accept_sec_context *arg) static char *gssp_stringify(struct xdr_netobj *netobj) { - return kstrndup(netobj->data, netobj->len, GFP_KERNEL); + return kmemdup_nul(netobj->data, netobj->len, GFP_KERNEL); } static void gssp_hostbased_service(char **principal) diff --git a/net/sunrpc/auth_gss/svcauth_gss.c b/net/sunrpc/auth_gss/svcauth_gss.c index 50d93c49ef1a..46027d0c903f 100644 --- a/net/sunrpc/auth_gss/svcauth_gss.c +++ b/net/sunrpc/auth_gss/svcauth_gss.c @@ -809,7 +809,7 @@ u32 svcauth_gss_flavor(struct auth_domain *dom) EXPORT_SYMBOL_GPL(svcauth_gss_flavor); -int +struct auth_domain * svcauth_gss_register_pseudoflavor(u32 pseudoflavor, char * name) { struct gss_domain *new; @@ -826,21 +826,23 @@ svcauth_gss_register_pseudoflavor(u32 pseudoflavor, char * name) new->h.flavour = &svcauthops_gss; new->pseudoflavor = pseudoflavor; - stat = 0; test = auth_domain_lookup(name, &new->h); - if (test != &new->h) { /* Duplicate registration */ + if (test != &new->h) { + pr_warn("svc: duplicate registration of gss pseudo flavour %s.\n", + name); + stat = -EADDRINUSE; auth_domain_put(test); - kfree(new->h.name); - goto out_free_dom; + goto out_free_name; } - return 0; + return test; +out_free_name: + kfree(new->h.name); out_free_dom: kfree(new); out: - return stat; + return ERR_PTR(stat); } - EXPORT_SYMBOL_GPL(svcauth_gss_register_pseudoflavor); static inline int diff --git a/net/sunrpc/auth_gss/trace.c b/net/sunrpc/auth_gss/trace.c index 5576f1e66de9..49fa583d7f91 100644 --- a/net/sunrpc/auth_gss/trace.c +++ b/net/sunrpc/auth_gss/trace.c @@ -6,6 +6,7 @@ #include <linux/sunrpc/clnt.h> #include <linux/sunrpc/sched.h> #include <linux/sunrpc/gss_err.h> +#include <linux/sunrpc/auth_gss.h> #define CREATE_TRACE_POINTS #include <trace/events/rpcgss.h> diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c index 61b21dafd7c0..a91d1cdad9d7 100644 --- a/net/sunrpc/clnt.c +++ b/net/sunrpc/clnt.c @@ -370,10 +370,6 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, const char *nodename = args->nodename; int err; - /* sanity check the name before trying to print it */ - dprintk("RPC: creating %s client for %s (xprt %p)\n", - program->name, args->servername, xprt); - err = rpciod_up(); if (err) goto out_no_rpciod; @@ -436,6 +432,8 @@ static struct rpc_clnt * rpc_new_client(const struct rpc_create_args *args, goto out_no_path; if (parent) atomic_inc(&parent->cl_count); + + trace_rpc_clnt_new(clnt, xprt, program->name, args->servername); return clnt; out_no_path: @@ -450,6 +448,7 @@ out_err: out_no_rpciod: xprt_switch_put(xps); xprt_put(xprt); + trace_rpc_clnt_new_err(program->name, args->servername, err); return ERR_PTR(err); } @@ -634,10 +633,8 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args, args->nodename = clnt->cl_nodename; new = rpc_new_client(args, xps, xprt, clnt); - if (IS_ERR(new)) { - err = PTR_ERR(new); - goto out_err; - } + if (IS_ERR(new)) + return new; /* Turn off autobind on clones */ new->cl_autobind = 0; @@ -650,7 +647,7 @@ static struct rpc_clnt *__rpc_clone_client(struct rpc_create_args *args, return new; out_err: - dprintk("RPC: %s: returned error %d\n", __func__, err); + trace_rpc_clnt_clone_err(clnt, err); return ERR_PTR(err); } @@ -723,11 +720,8 @@ int rpc_switch_client_transport(struct rpc_clnt *clnt, int err; xprt = xprt_create_transport(args); - if (IS_ERR(xprt)) { - dprintk("RPC: failed to create new xprt for clnt %p\n", - clnt); + if (IS_ERR(xprt)) return PTR_ERR(xprt); - } xps = xprt_switch_alloc(xprt, GFP_KERNEL); if (xps == NULL) { @@ -767,7 +761,7 @@ int rpc_switch_client_transport(struct rpc_clnt *clnt, rpc_release_client(parent); xprt_switch_put(oldxps); xprt_put(old); - dprintk("RPC: replaced xprt for clnt %p\n", clnt); + trace_rpc_clnt_replace_xprt(clnt); return 0; out_revert: @@ -777,7 +771,7 @@ out_revert: rpc_client_register(clnt, pseudoflavor, NULL); xprt_switch_put(xps); xprt_put(xprt); - dprintk("RPC: failed to switch xprt for clnt %p\n", clnt); + trace_rpc_clnt_replace_xprt_err(clnt); return err; } EXPORT_SYMBOL_GPL(rpc_switch_client_transport); @@ -844,10 +838,11 @@ void rpc_killall_tasks(struct rpc_clnt *clnt) if (list_empty(&clnt->cl_tasks)) return; - dprintk("RPC: killing all tasks for client %p\n", clnt); + /* * Spin lock all_tasks to prevent changes... */ + trace_rpc_clnt_killall(clnt); spin_lock(&clnt->cl_lock); list_for_each_entry(rovr, &clnt->cl_tasks, tk_task) rpc_signal_task(rovr); @@ -863,9 +858,7 @@ void rpc_shutdown_client(struct rpc_clnt *clnt) { might_sleep(); - dprintk_rcu("RPC: shutting down %s client for %s\n", - clnt->cl_program->name, - rcu_dereference(clnt->cl_xprt)->servername); + trace_rpc_clnt_shutdown(clnt); while (!list_empty(&clnt->cl_tasks)) { rpc_killall_tasks(clnt); @@ -884,6 +877,8 @@ static void rpc_free_client_work(struct work_struct *work) { struct rpc_clnt *clnt = container_of(work, struct rpc_clnt, cl_work); + trace_rpc_clnt_free(clnt); + /* These might block on processes that might allocate memory, * so they cannot be called in rpciod, so they are handled separately * here. @@ -901,9 +896,7 @@ rpc_free_client(struct rpc_clnt *clnt) { struct rpc_clnt *parent = NULL; - dprintk_rcu("RPC: destroying %s client for %s\n", - clnt->cl_program->name, - rcu_dereference(clnt->cl_xprt)->servername); + trace_rpc_clnt_release(clnt); if (clnt->cl_parent != clnt) parent = clnt->cl_parent; rpc_unregister_client(clnt); @@ -945,8 +938,6 @@ rpc_free_auth(struct rpc_clnt *clnt) void rpc_release_client(struct rpc_clnt *clnt) { - dprintk("RPC: rpc_release_client(%p)\n", clnt); - do { if (list_empty(&clnt->cl_tasks)) wake_up(&destroy_wait); @@ -1270,7 +1261,7 @@ void rpc_prepare_reply_pages(struct rpc_rqst *req, struct page **pages, hdrsize += RPC_REPHDRSIZE + req->rq_cred->cr_auth->au_ralign - 1; xdr_inline_pages(&req->rq_rcv_buf, hdrsize << 2, pages, base, len); - trace_rpc_reply_pages(req); + trace_rpc_xdr_reply_pages(req->rq_task, &req->rq_rcv_buf); } EXPORT_SYMBOL_GPL(rpc_prepare_reply_pages); @@ -1624,6 +1615,7 @@ const char static void __rpc_call_rpcerror(struct rpc_task *task, int tk_status, int rpc_status) { + trace_rpc_call_rpcerror(task, tk_status, rpc_status); task->tk_rpc_status = rpc_status; rpc_exit(task, tk_status); } @@ -2531,7 +2523,7 @@ call_decode(struct rpc_task *task) goto out; req->rq_rcv_buf.len = req->rq_private_buf.len; - trace_xprt_recvfrom(&req->rq_rcv_buf); + trace_rpc_xdr_recvfrom(task, &req->rq_rcv_buf); /* Check that the softirq receive buffer is valid */ WARN_ON(memcmp(&req->rq_rcv_buf, &req->rq_private_buf, @@ -2760,7 +2752,8 @@ struct rpc_task *rpc_call_null_helper(struct rpc_clnt *clnt, .rpc_op_cred = cred, .callback_ops = (ops != NULL) ? ops : &rpc_default_ops, .callback_data = data, - .flags = flags | RPC_TASK_NULLCREDS, + .flags = flags | RPC_TASK_SOFT | RPC_TASK_SOFTCONN | + RPC_TASK_NULLCREDS, }; return rpc_run_task(&task_setup_data); @@ -2823,8 +2816,7 @@ int rpc_clnt_test_and_add_xprt(struct rpc_clnt *clnt, goto success; } - task = rpc_call_null_helper(clnt, xprt, NULL, - RPC_TASK_SOFT|RPC_TASK_SOFTCONN|RPC_TASK_ASYNC|RPC_TASK_NULLCREDS, + task = rpc_call_null_helper(clnt, xprt, NULL, RPC_TASK_ASYNC, &rpc_cb_add_xprt_call_ops, data); rpc_put_task(task); @@ -2867,9 +2859,7 @@ int rpc_clnt_setup_test_and_add_xprt(struct rpc_clnt *clnt, goto out_err; /* Test the connection */ - task = rpc_call_null_helper(clnt, xprt, NULL, - RPC_TASK_SOFT | RPC_TASK_SOFTCONN | RPC_TASK_NULLCREDS, - NULL, NULL); + task = rpc_call_null_helper(clnt, xprt, NULL, 0, NULL, NULL); if (IS_ERR(task)) { status = PTR_ERR(task); goto out_err; diff --git a/net/sunrpc/rpcb_clnt.c b/net/sunrpc/rpcb_clnt.c index 4a020b688860..c27123e6ba80 100644 --- a/net/sunrpc/rpcb_clnt.c +++ b/net/sunrpc/rpcb_clnt.c @@ -795,12 +795,6 @@ void rpcb_getport_async(struct rpc_task *task) child = rpcb_call_async(rpcb_clnt, map, proc); rpc_release_client(rpcb_clnt); - if (IS_ERR(child)) { - /* rpcb_map_release() has freed the arguments */ - dprintk("RPC: %5u %s: rpc_run_task failed\n", - task->tk_pid, __func__); - return; - } xprt->stat.bind_count++; rpc_put_task(child); diff --git a/net/sunrpc/sunrpc.h b/net/sunrpc/sunrpc.h index 47a756503d11..f6fe2e6cd65a 100644 --- a/net/sunrpc/sunrpc.h +++ b/net/sunrpc/sunrpc.h @@ -52,4 +52,5 @@ static inline int sock_is_loopback(struct sock *sk) int rpc_clients_notifier_register(void); void rpc_clients_notifier_unregister(void); +void auth_domain_cleanup(void); #endif /* _NET_SUNRPC_SUNRPC_H */ diff --git a/net/sunrpc/sunrpc_syms.c b/net/sunrpc/sunrpc_syms.c index f9edaa9174a4..236fadc4a439 100644 --- a/net/sunrpc/sunrpc_syms.c +++ b/net/sunrpc/sunrpc_syms.c @@ -23,6 +23,7 @@ #include <linux/sunrpc/rpc_pipe_fs.h> #include <linux/sunrpc/xprtsock.h> +#include "sunrpc.h" #include "netns.h" unsigned int sunrpc_net_id; @@ -131,6 +132,7 @@ cleanup_sunrpc(void) unregister_rpc_pipefs(); rpc_destroy_mempool(); unregister_pernet_subsys(&sunrpc_net_ops); + auth_domain_cleanup(); #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) rpc_unregister_sysctl(); #endif diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index 9ed3126600ce..c211b607239e 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -88,15 +88,15 @@ param_get_pool_mode(char *buf, const struct kernel_param *kp) switch (*ip) { case SVC_POOL_AUTO: - return strlcpy(buf, "auto", 20); + return strlcpy(buf, "auto\n", 20); case SVC_POOL_GLOBAL: - return strlcpy(buf, "global", 20); + return strlcpy(buf, "global\n", 20); case SVC_POOL_PERCPU: - return strlcpy(buf, "percpu", 20); + return strlcpy(buf, "percpu\n", 20); case SVC_POOL_PERNODE: - return strlcpy(buf, "pernode", 20); + return strlcpy(buf, "pernode\n", 20); default: - return sprintf(buf, "%d", *ip); + return sprintf(buf, "%d\n", *ip); } } @@ -991,6 +991,7 @@ static int __svc_register(struct net *net, const char *progname, #endif } + trace_svc_register(progname, version, protocol, port, family, error); return error; } @@ -1000,11 +1001,6 @@ int svc_rpcbind_set_version(struct net *net, unsigned short proto, unsigned short port) { - dprintk("svc: svc_register(%sv%d, %s, %u, %u)\n", - progp->pg_name, version, - proto == IPPROTO_UDP? "udp" : "tcp", - port, family); - return __svc_register(net, progp->pg_name, progp->pg_prog, version, family, proto, port); @@ -1024,11 +1020,8 @@ int svc_generic_rpcbind_set(struct net *net, return 0; if (vers->vs_hidden) { - dprintk("svc: svc_register(%sv%d, %s, %u, %u)" - " (but not telling portmap)\n", - progp->pg_name, version, - proto == IPPROTO_UDP? "udp" : "tcp", - port, family); + trace_svc_noregister(progp->pg_name, version, proto, + port, family, 0); return 0; } @@ -1106,8 +1099,7 @@ static void __svc_unregister(struct net *net, const u32 program, const u32 versi if (error == -EPROTONOSUPPORT) error = rpcb_register(net, program, version, 0, 0); - dprintk("svc: %s(%sv%u), error %d\n", - __func__, progname, version, error); + trace_svc_unregister(progname, version, error); } /* @@ -1132,9 +1124,6 @@ static void svc_unregister(const struct svc_serv *serv, struct net *net) continue; if (progp->pg_vers[i]->vs_hidden) continue; - - dprintk("svc: attempting to unregister %sv%u\n", - progp->pg_name, i); __svc_unregister(net, progp->pg_prog, i, progp->pg_name); } } diff --git a/net/sunrpc/svc_xprt.c b/net/sunrpc/svc_xprt.c index 2284ff038dad..43cf8dbde898 100644 --- a/net/sunrpc/svc_xprt.c +++ b/net/sunrpc/svc_xprt.c @@ -153,6 +153,7 @@ static void svc_xprt_free(struct kref *kref) xprt_put(xprt->xpt_bc_xprt); if (xprt->xpt_bc_xps) xprt_switch_put(xprt->xpt_bc_xps); + trace_svc_xprt_free(xprt); xprt->xpt_ops->xpo_free(xprt); module_put(owner); } @@ -206,6 +207,7 @@ static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl, .sin6_port = htons(port), }; #endif + struct svc_xprt *xprt; struct sockaddr *sap; size_t len; @@ -224,7 +226,11 @@ static struct svc_xprt *__svc_xpo_create(struct svc_xprt_class *xcl, return ERR_PTR(-EAFNOSUPPORT); } - return xcl->xcl_ops->xpo_create(serv, net, sap, len, flags); + xprt = xcl->xcl_ops->xpo_create(serv, net, sap, len, flags); + if (IS_ERR(xprt)) + trace_svc_xprt_create_err(serv->sv_program->pg_name, + xcl->xcl_name, sap, xprt); + return xprt; } /* @@ -304,15 +310,11 @@ int svc_create_xprt(struct svc_serv *serv, const char *xprt_name, { int err; - dprintk("svc: creating transport %s[%d]\n", xprt_name, port); err = _svc_create_xprt(serv, xprt_name, net, family, port, flags, cred); if (err == -EPROTONOSUPPORT) { request_module("svc%s", xprt_name); err = _svc_create_xprt(serv, xprt_name, net, family, port, flags, cred); } - if (err < 0) - dprintk("svc: transport %s not found, err %d\n", - xprt_name, -err); return err; } EXPORT_SYMBOL_GPL(svc_create_xprt); @@ -780,7 +782,6 @@ static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt) int len = 0; if (test_bit(XPT_CLOSE, &xprt->xpt_flags)) { - dprintk("svc_recv: found XPT_CLOSE\n"); if (test_and_clear_bit(XPT_KILL_TEMP, &xprt->xpt_flags)) xprt->xpt_ops->xpo_kill_temp_xprt(xprt); svc_delete_xprt(xprt); @@ -799,6 +800,7 @@ static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt) if (newxpt) { newxpt->xpt_cred = get_cred(xprt->xpt_cred); svc_add_new_temp_xprt(serv, newxpt); + trace_svc_xprt_accept(newxpt, serv->sv_name); } else module_put(xprt->xpt_class->xcl_owner); } else if (svc_xprt_reserve_slot(rqstp, xprt)) { @@ -812,7 +814,7 @@ static int svc_handle_xprt(struct svc_rqst *rqstp, struct svc_xprt *xprt) else len = xprt->xpt_ops->xpo_recvfrom(rqstp); if (len > 0) - trace_svc_recvfrom(&rqstp->rq_arg); + trace_svc_xdr_recvfrom(rqstp, &rqstp->rq_arg); rqstp->rq_stime = ktime_get(); rqstp->rq_reserved = serv->sv_max_mesg; atomic_add(rqstp->rq_reserved, &xprt->xpt_reserved); @@ -835,14 +837,6 @@ int svc_recv(struct svc_rqst *rqstp, long timeout) struct svc_serv *serv = rqstp->rq_server; int len, err; - dprintk("svc: server %p waiting for data (to = %ld)\n", - rqstp, timeout); - - if (rqstp->rq_xprt) - printk(KERN_ERR - "svc_recv: service %p, transport not NULL!\n", - rqstp); - err = svc_alloc_arg(rqstp); if (err) goto out; @@ -890,7 +884,6 @@ EXPORT_SYMBOL_GPL(svc_recv); void svc_drop(struct svc_rqst *rqstp) { trace_svc_drop(rqstp); - dprintk("svc: xprt %p dropped request\n", rqstp->rq_xprt); svc_xprt_release(rqstp); } EXPORT_SYMBOL_GPL(svc_drop); @@ -913,17 +906,11 @@ int svc_send(struct svc_rqst *rqstp) xb->len = xb->head[0].iov_len + xb->page_len + xb->tail[0].iov_len; - trace_svc_sendto(xb); - - /* Grab mutex to serialize outgoing data. */ - mutex_lock(&xprt->xpt_mutex); + trace_svc_xdr_sendto(rqstp, xb); trace_svc_stats_latency(rqstp); - if (test_bit(XPT_DEAD, &xprt->xpt_flags) - || test_bit(XPT_CLOSE, &xprt->xpt_flags)) - len = -ENOTCONN; - else - len = xprt->xpt_ops->xpo_sendto(rqstp); - mutex_unlock(&xprt->xpt_mutex); + + len = xprt->xpt_ops->xpo_sendto(rqstp); + trace_svc_send(rqstp, len); svc_xprt_release(rqstp); @@ -1031,11 +1018,10 @@ static void svc_delete_xprt(struct svc_xprt *xprt) struct svc_serv *serv = xprt->xpt_server; struct svc_deferred_req *dr; - /* Only do this once */ if (test_and_set_bit(XPT_DEAD, &xprt->xpt_flags)) - BUG(); + return; - dprintk("svc: svc_delete_xprt(%p)\n", xprt); + trace_svc_xprt_detach(xprt); xprt->xpt_ops->xpo_detach(xprt); if (xprt->xpt_bc_xprt) xprt->xpt_bc_xprt->ops->close(xprt->xpt_bc_xprt); @@ -1056,6 +1042,7 @@ static void svc_delete_xprt(struct svc_xprt *xprt) void svc_close_xprt(struct svc_xprt *xprt) { + trace_svc_xprt_close(xprt); set_bit(XPT_CLOSE, &xprt->xpt_flags); if (test_and_set_bit(XPT_BUSY, &xprt->xpt_flags)) /* someone else will have to effect the close */ @@ -1158,16 +1145,15 @@ static void svc_revisit(struct cache_deferred_req *dreq, int too_many) set_bit(XPT_DEFERRED, &xprt->xpt_flags); if (too_many || test_bit(XPT_DEAD, &xprt->xpt_flags)) { spin_unlock(&xprt->xpt_lock); - dprintk("revisit canceled\n"); + trace_svc_defer_drop(dr); svc_xprt_put(xprt); - trace_svc_drop_deferred(dr); kfree(dr); return; } - dprintk("revisit queued\n"); dr->xprt = NULL; list_add(&dr->handle.recent, &xprt->xpt_deferred); spin_unlock(&xprt->xpt_lock); + trace_svc_defer_queue(dr); svc_xprt_enqueue(xprt); svc_xprt_put(xprt); } @@ -1213,22 +1199,24 @@ static struct cache_deferred_req *svc_defer(struct cache_req *req) memcpy(dr->args, rqstp->rq_arg.head[0].iov_base - skip, dr->argslen << 2); } + trace_svc_defer(rqstp); svc_xprt_get(rqstp->rq_xprt); dr->xprt = rqstp->rq_xprt; set_bit(RQ_DROPME, &rqstp->rq_flags); dr->handle.revisit = svc_revisit; - trace_svc_defer(rqstp); return &dr->handle; } /* * recv data from a deferred request into an active one */ -static int svc_deferred_recv(struct svc_rqst *rqstp) +static noinline int svc_deferred_recv(struct svc_rqst *rqstp) { struct svc_deferred_req *dr = rqstp->rq_deferred; + trace_svc_defer_recv(dr); + /* setup iov_base past transport header */ rqstp->rq_arg.head[0].iov_base = dr->args + (dr->xprt_hlen>>2); /* The iov_len does not include the transport header bytes */ @@ -1259,7 +1247,6 @@ static struct svc_deferred_req *svc_deferred_dequeue(struct svc_xprt *xprt) struct svc_deferred_req, handle.recent); list_del_init(&dr->handle.recent); - trace_svc_revisit_deferred(dr); } else clear_bit(XPT_DEFERRED, &xprt->xpt_flags); spin_unlock(&xprt->xpt_lock); diff --git a/net/sunrpc/svcauth.c b/net/sunrpc/svcauth.c index 552617e3467b..998b196b6176 100644 --- a/net/sunrpc/svcauth.c +++ b/net/sunrpc/svcauth.c @@ -21,6 +21,8 @@ #include <trace/events/sunrpc.h> +#include "sunrpc.h" + #define RPCDBG_FACILITY RPCDBG_AUTH @@ -205,3 +207,26 @@ struct auth_domain *auth_domain_find(char *name) return NULL; } EXPORT_SYMBOL_GPL(auth_domain_find); + +/** + * auth_domain_cleanup - check that the auth_domain table is empty + * + * On module unload the auth_domain_table must be empty. To make it + * easier to catch bugs which don't clean up domains properly, we + * warn if anything remains in the table at cleanup time. + * + * Note that we cannot proactively remove the domains at this stage. + * The ->release() function might be in a module that has already been + * unloaded. + */ + +void auth_domain_cleanup(void) +{ + int h; + struct auth_domain *hp; + + for (h = 0; h < DN_HASHMAX; h++) + hlist_for_each_entry(hp, &auth_domain_table[h], hash) + pr_warn("svc: domain %s still present at module unload.\n", + hp->name); +} diff --git a/net/sunrpc/svcauth_unix.c b/net/sunrpc/svcauth_unix.c index 6c8f802c4261..97c0bddba7a3 100644 --- a/net/sunrpc/svcauth_unix.c +++ b/net/sunrpc/svcauth_unix.c @@ -332,15 +332,6 @@ static int __ip_map_update(struct cache_detail *cd, struct ip_map *ipm, return 0; } -static inline int ip_map_update(struct net *net, struct ip_map *ipm, - struct unix_domain *udom, time64_t expiry) -{ - struct sunrpc_net *sn; - - sn = net_generic(net, sunrpc_net_id); - return __ip_map_update(sn->ip_map_cache, ipm, udom, expiry); -} - void svcauth_unix_purge(struct net *net) { struct sunrpc_net *sn; diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index e7a0037d9b56..5c4ec9386f81 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -45,7 +45,6 @@ #include <net/tcp_states.h> #include <linux/uaccess.h> #include <asm/ioctls.h> -#include <trace/events/skb.h> #include <linux/sunrpc/types.h> #include <linux/sunrpc/clnt.h> @@ -55,6 +54,8 @@ #include <linux/sunrpc/stats.h> #include <linux/sunrpc/xprt.h> +#include <trace/events/sunrpc.h> + #include "socklib.h" #include "sunrpc.h" @@ -108,31 +109,35 @@ static void svc_reclassify_socket(struct socket *sock) } #endif -/* - * Release an skbuff after use +/** + * svc_tcp_release_rqst - Release transport-related resources + * @rqstp: request structure with resources to be released + * */ -static void svc_release_skb(struct svc_rqst *rqstp) +static void svc_tcp_release_rqst(struct svc_rqst *rqstp) { struct sk_buff *skb = rqstp->rq_xprt_ctxt; if (skb) { struct svc_sock *svsk = container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); - rqstp->rq_xprt_ctxt = NULL; - dprintk("svc: service %p, releasing skb %p\n", rqstp, skb); + rqstp->rq_xprt_ctxt = NULL; skb_free_datagram_locked(svsk->sk_sk, skb); } } -static void svc_release_udp_skb(struct svc_rqst *rqstp) +/** + * svc_udp_release_rqst - Release transport-related resources + * @rqstp: request structure with resources to be released + * + */ +static void svc_udp_release_rqst(struct svc_rqst *rqstp) { struct sk_buff *skb = rqstp->rq_xprt_ctxt; if (skb) { rqstp->rq_xprt_ctxt = NULL; - - dprintk("svc: service %p, releasing skb %p\n", rqstp, skb); consume_skb(skb); } } @@ -218,34 +223,68 @@ static int svc_one_sock_name(struct svc_sock *svsk, char *buf, int remaining) return len; } +#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE +static void svc_flush_bvec(const struct bio_vec *bvec, size_t size, size_t seek) +{ + struct bvec_iter bi = { + .bi_size = size, + }; + struct bio_vec bv; + + bvec_iter_advance(bvec, &bi, seek & PAGE_MASK); + for_each_bvec(bv, bvec, bi, bi) + flush_dcache_page(bv.bv_page); +} +#else +static inline void svc_flush_bvec(const struct bio_vec *bvec, size_t size, + size_t seek) +{ +} +#endif + /* - * Generic recvfrom routine. + * Read from @rqstp's transport socket. The incoming message fills whole + * pages in @rqstp's rq_pages array until the last page of the message + * has been received into a partial page. */ -static ssize_t svc_recvfrom(struct svc_rqst *rqstp, struct kvec *iov, - unsigned int nr, size_t buflen, unsigned int base) +static ssize_t svc_tcp_read_msg(struct svc_rqst *rqstp, size_t buflen, + size_t seek) { struct svc_sock *svsk = container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); + struct bio_vec *bvec = rqstp->rq_bvec; struct msghdr msg = { NULL }; + unsigned int i; ssize_t len; + size_t t; rqstp->rq_xprt_hlen = 0; clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); - iov_iter_kvec(&msg.msg_iter, READ, iov, nr, buflen); - if (base != 0) { - iov_iter_advance(&msg.msg_iter, base); - buflen -= base; + + for (i = 0, t = 0; t < buflen; i++, t += PAGE_SIZE) { + bvec[i].bv_page = rqstp->rq_pages[i]; + bvec[i].bv_len = PAGE_SIZE; + bvec[i].bv_offset = 0; + } + rqstp->rq_respages = &rqstp->rq_pages[i]; + rqstp->rq_next_page = rqstp->rq_respages + 1; + + iov_iter_bvec(&msg.msg_iter, READ, bvec, i, buflen); + if (seek) { + iov_iter_advance(&msg.msg_iter, seek); + buflen -= seek; } len = sock_recvmsg(svsk->sk_sock, &msg, MSG_DONTWAIT); + if (len > 0) + svc_flush_bvec(bvec, len, seek); + /* If we read a full record, then assume there may be more * data to read (stream based sockets only!) */ if (len == buflen) set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); - dprintk("svc: socket %p recvfrom(%p, %zu) = %zd\n", - svsk, iov[0].iov_base, iov[0].iov_len, len); return len; } @@ -282,13 +321,10 @@ static void svc_data_ready(struct sock *sk) struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; if (svsk) { - dprintk("svc: socket %p(inet %p), busy=%d\n", - svsk, sk, - test_bit(XPT_BUSY, &svsk->sk_xprt.xpt_flags)); - /* Refer to svc_setup_socket() for details. */ rmb(); svsk->sk_odata(sk); + trace_svcsock_data_ready(&svsk->sk_xprt, 0); if (!test_and_set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags)) svc_xprt_enqueue(&svsk->sk_xprt); } @@ -302,11 +338,9 @@ static void svc_write_space(struct sock *sk) struct svc_sock *svsk = (struct svc_sock *)(sk->sk_user_data); if (svsk) { - dprintk("svc: socket %p(inet %p), write_space busy=%d\n", - svsk, sk, test_bit(XPT_BUSY, &svsk->sk_xprt.xpt_flags)); - /* Refer to svc_setup_socket() for details. */ rmb(); + trace_svcsock_write_space(&svsk->sk_xprt, 0); svsk->sk_owspace(sk); svc_xprt_enqueue(&svsk->sk_xprt); } @@ -383,8 +417,15 @@ static int svc_udp_get_dest_address(struct svc_rqst *rqstp, return 0; } -/* - * Receive a datagram from a UDP socket. +/** + * svc_udp_recvfrom - Receive a datagram from a UDP socket. + * @rqstp: request structure into which to receive an RPC Call + * + * Called in a loop when XPT_DATA has been set. + * + * Returns: + * On success, the number of bytes in a received RPC Call, or + * %0 if a complete RPC Call message was not ready to return */ static int svc_udp_recvfrom(struct svc_rqst *rqstp) { @@ -418,20 +459,14 @@ static int svc_udp_recvfrom(struct svc_rqst *rqstp) svc_sock_setbufsize(svsk, serv->sv_nrthreads + 3); clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); - skb = NULL; err = kernel_recvmsg(svsk->sk_sock, &msg, NULL, 0, 0, MSG_PEEK | MSG_DONTWAIT); - if (err >= 0) - skb = skb_recv_udp(svsk->sk_sk, 0, 1, &err); - - if (skb == NULL) { - if (err != -EAGAIN) { - /* possibly an icmp error */ - dprintk("svc: recvfrom returned error %d\n", -err); - set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); - } - return 0; - } + if (err < 0) + goto out_recv_err; + skb = skb_recv_udp(svsk->sk_sk, 0, 1, &err); + if (!skb) + goto out_recv_err; + len = svc_addr_len(svc_addr(rqstp)); rqstp->rq_addrlen = len; if (skb->tstamp == 0) { @@ -442,26 +477,21 @@ static int svc_udp_recvfrom(struct svc_rqst *rqstp) sock_write_timestamp(svsk->sk_sk, skb->tstamp); set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); /* there may be more data... */ - len = skb->len; + len = skb->len; rqstp->rq_arg.len = len; + trace_svcsock_udp_recv(&svsk->sk_xprt, len); rqstp->rq_prot = IPPROTO_UDP; - if (!svc_udp_get_dest_address(rqstp, cmh)) { - net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n", - cmh->cmsg_level, cmh->cmsg_type); - goto out_free; - } + if (!svc_udp_get_dest_address(rqstp, cmh)) + goto out_cmsg_err; rqstp->rq_daddrlen = svc_addr_len(svc_daddr(rqstp)); if (skb_is_nonlinear(skb)) { /* we have to copy */ local_bh_disable(); - if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb)) { - local_bh_enable(); - /* checksum error */ - goto out_free; - } + if (csum_partial_copy_to_xdr(&rqstp->rq_arg, skb)) + goto out_bh_enable; local_bh_enable(); consume_skb(skb); } else { @@ -489,6 +519,20 @@ static int svc_udp_recvfrom(struct svc_rqst *rqstp) serv->sv_stats->netudpcnt++; return len; + +out_recv_err: + if (err != -EAGAIN) { + /* possibly an icmp error */ + set_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); + } + trace_svcsock_udp_recv_err(&svsk->sk_xprt, err); + return 0; +out_cmsg_err: + net_warn_ratelimited("svc: received unknown control message %d/%d; dropping RPC reply datagram\n", + cmh->cmsg_level, cmh->cmsg_type); + goto out_free; +out_bh_enable: + local_bh_enable(); out_free: kfree_skb(skb); return 0; @@ -498,6 +542,9 @@ out_free: * svc_udp_sendto - Send out a reply on a UDP socket * @rqstp: completed svc_rqst * + * xpt_mutex ensures @rqstp's whole message is written to the socket + * without interruption. + * * Returns the number of bytes sent, or a negative errno. */ static int svc_udp_sendto(struct svc_rqst *rqstp) @@ -519,10 +566,15 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) unsigned int uninitialized_var(sent); int err; - svc_release_udp_skb(rqstp); + svc_udp_release_rqst(rqstp); svc_set_cmsg_data(rqstp, cmh); + mutex_lock(&xprt->xpt_mutex); + + if (svc_xprt_is_dead(xprt)) + goto out_notconn; + err = xprt_sock_sendmsg(svsk->sk_sock, &msg, xdr, 0, 0, &sent); xdr_free_bvec(xdr); if (err == -ECONNREFUSED) { @@ -530,9 +582,16 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) err = xprt_sock_sendmsg(svsk->sk_sock, &msg, xdr, 0, 0, &sent); xdr_free_bvec(xdr); } + trace_svcsock_udp_send(xprt, err); + + mutex_unlock(&xprt->xpt_mutex); if (err < 0) return err; return sent; + +out_notconn: + mutex_unlock(&xprt->xpt_mutex); + return -ENOTCONN; } static int svc_udp_has_wspace(struct svc_xprt *xprt) @@ -576,7 +635,7 @@ static const struct svc_xprt_ops svc_udp_ops = { .xpo_recvfrom = svc_udp_recvfrom, .xpo_sendto = svc_udp_sendto, .xpo_read_payload = svc_sock_read_payload, - .xpo_release_rqst = svc_release_udp_skb, + .xpo_release_rqst = svc_udp_release_rqst, .xpo_detach = svc_sock_detach, .xpo_free = svc_sock_free, .xpo_has_wspace = svc_udp_has_wspace, @@ -632,9 +691,6 @@ static void svc_tcp_listen_data_ready(struct sock *sk) { struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; - dprintk("svc: socket %p TCP (listen) state change %d\n", - sk, sk->sk_state); - if (svsk) { /* Refer to svc_setup_socket() for details. */ rmb(); @@ -655,8 +711,7 @@ static void svc_tcp_listen_data_ready(struct sock *sk) if (svsk) { set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); svc_xprt_enqueue(&svsk->sk_xprt); - } else - printk("svc: socket %p: no user data\n", sk); + } } } @@ -667,15 +722,11 @@ static void svc_tcp_state_change(struct sock *sk) { struct svc_sock *svsk = (struct svc_sock *)sk->sk_user_data; - dprintk("svc: socket %p TCP (connected) state change %d (svsk %p)\n", - sk, sk->sk_state, sk->sk_user_data); - - if (!svsk) - printk("svc: socket %p: no user data\n", sk); - else { + if (svsk) { /* Refer to svc_setup_socket() for details. */ rmb(); svsk->sk_ostate(sk); + trace_svcsock_tcp_state(&svsk->sk_xprt, svsk->sk_sock); if (sk->sk_state != TCP_ESTABLISHED) { set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags); svc_xprt_enqueue(&svsk->sk_xprt); @@ -696,9 +747,7 @@ static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) struct socket *newsock; struct svc_sock *newsvsk; int err, slen; - RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]); - dprintk("svc: tcp_accept %p sock %p\n", svsk, sock); if (!sock) return NULL; @@ -711,30 +760,18 @@ static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) else if (err != -EAGAIN) net_warn_ratelimited("%s: accept failed (err %d)!\n", serv->sv_name, -err); + trace_svcsock_accept_err(xprt, serv->sv_name, err); return NULL; } set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); err = kernel_getpeername(newsock, sin); if (err < 0) { - net_warn_ratelimited("%s: peername failed (err %d)!\n", - serv->sv_name, -err); + trace_svcsock_getpeername_err(xprt, serv->sv_name, err); goto failed; /* aborted connection or whatever */ } slen = err; - /* Ideally, we would want to reject connections from unauthorized - * hosts here, but when we get encryption, the IP of the host won't - * tell us anything. For now just warn about unpriv connections. - */ - if (!svc_port_is_privileged(sin)) { - dprintk("%s: connect from unprivileged port: %s\n", - serv->sv_name, - __svc_print_addr(sin, buf, sizeof(buf))); - } - dprintk("%s: connect from %s\n", serv->sv_name, - __svc_print_addr(sin, buf, sizeof(buf))); - /* Reset the inherited callbacks before calling svc_setup_socket */ newsock->sk->sk_state_change = svsk->sk_ostate; newsock->sk->sk_data_ready = svsk->sk_odata; @@ -752,10 +789,8 @@ static struct svc_xprt *svc_tcp_accept(struct svc_xprt *xprt) svc_xprt_set_remote(&newsvsk->sk_xprt, sin, slen); err = kernel_getsockname(newsock, sin); slen = err; - if (unlikely(err < 0)) { - dprintk("svc_tcp_accept: kernel_getsockname error %d\n", -err); + if (unlikely(err < 0)) slen = offsetof(struct sockaddr, sa_data); - } svc_xprt_set_local(&newsvsk->sk_xprt, sin, slen); if (sock_is_loopback(newsock->sk)) @@ -772,13 +807,14 @@ failed: return NULL; } -static unsigned int svc_tcp_restore_pages(struct svc_sock *svsk, struct svc_rqst *rqstp) +static size_t svc_tcp_restore_pages(struct svc_sock *svsk, + struct svc_rqst *rqstp) { - unsigned int i, len, npages; + size_t len = svsk->sk_datalen; + unsigned int i, npages; - if (svsk->sk_datalen == 0) + if (!len) return 0; - len = svsk->sk_datalen; npages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; for (i = 0; i < npages; i++) { if (rqstp->rq_pages[i] != NULL) @@ -827,47 +863,45 @@ out: } /* - * Receive fragment record header. - * If we haven't gotten the record length yet, get the next four bytes. + * Receive fragment record header into sk_marker. */ -static int svc_tcp_recv_record(struct svc_sock *svsk, struct svc_rqst *rqstp) +static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, + struct svc_rqst *rqstp) { - struct svc_serv *serv = svsk->sk_xprt.xpt_server; - unsigned int want; - int len; + ssize_t want, len; + /* If we haven't gotten the record length yet, + * get the next four bytes. + */ if (svsk->sk_tcplen < sizeof(rpc_fraghdr)) { + struct msghdr msg = { NULL }; struct kvec iov; want = sizeof(rpc_fraghdr) - svsk->sk_tcplen; - iov.iov_base = ((char *) &svsk->sk_reclen) + svsk->sk_tcplen; + iov.iov_base = ((char *)&svsk->sk_marker) + svsk->sk_tcplen; iov.iov_len = want; - len = svc_recvfrom(rqstp, &iov, 1, want, 0); + iov_iter_kvec(&msg.msg_iter, READ, &iov, 1, want); + len = sock_recvmsg(svsk->sk_sock, &msg, MSG_DONTWAIT); if (len < 0) - goto error; + return len; svsk->sk_tcplen += len; - if (len < want) { - dprintk("svc: short recvfrom while reading record " - "length (%d of %d)\n", len, want); - return -EAGAIN; + /* call again to read the remaining bytes */ + goto err_short; } - - dprintk("svc: TCP record, %d bytes\n", svc_sock_reclen(svsk)); + trace_svcsock_marker(&svsk->sk_xprt, svsk->sk_marker); if (svc_sock_reclen(svsk) + svsk->sk_datalen > - serv->sv_max_mesg) { - net_notice_ratelimited("RPC: fragment too large: %d\n", - svc_sock_reclen(svsk)); - goto err_delete; - } + svsk->sk_xprt.xpt_server->sv_max_mesg) + goto err_too_large; } - return svc_sock_reclen(svsk); -error: - dprintk("RPC: TCP recv_record got %d\n", len); - return len; -err_delete: + +err_too_large: + net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n", + __func__, svsk->sk_xprt.xpt_server->sv_name, + svc_sock_reclen(svsk)); set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags); +err_short: return -EAGAIN; } @@ -916,87 +950,58 @@ unlock_eagain: return -EAGAIN; } -static int copy_pages_to_kvecs(struct kvec *vec, struct page **pages, int len) -{ - int i = 0; - int t = 0; - - while (t < len) { - vec[i].iov_base = page_address(pages[i]); - vec[i].iov_len = PAGE_SIZE; - i++; - t += PAGE_SIZE; - } - return i; -} - static void svc_tcp_fragment_received(struct svc_sock *svsk) { /* If we have more data, signal svc_xprt_enqueue() to try again */ - dprintk("svc: TCP %s record (%d bytes)\n", - svc_sock_final_rec(svsk) ? "final" : "nonfinal", - svc_sock_reclen(svsk)); svsk->sk_tcplen = 0; - svsk->sk_reclen = 0; + svsk->sk_marker = xdr_zero; } -/* - * Receive data from a TCP socket. +/** + * svc_tcp_recvfrom - Receive data from a TCP socket + * @rqstp: request structure into which to receive an RPC Call + * + * Called in a loop when XPT_DATA has been set. + * + * Read the 4-byte stream record marker, then use the record length + * in that marker to set up exactly the resources needed to receive + * the next RPC message into @rqstp. + * + * Returns: + * On success, the number of bytes in a received RPC Call, or + * %0 if a complete RPC Call message was not ready to return + * + * The zero return case handles partial receives and callback Replies. + * The state of a partial receive is preserved in the svc_sock for + * the next call to svc_tcp_recvfrom. */ static int svc_tcp_recvfrom(struct svc_rqst *rqstp) { struct svc_sock *svsk = container_of(rqstp->rq_xprt, struct svc_sock, sk_xprt); struct svc_serv *serv = svsk->sk_xprt.xpt_server; - int len; - struct kvec *vec; - unsigned int want, base; + size_t want, base; + ssize_t len; __be32 *p; __be32 calldir; - int pnum; - - dprintk("svc: tcp_recv %p data %d conn %d close %d\n", - svsk, test_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags), - test_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags), - test_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags)); - len = svc_tcp_recv_record(svsk, rqstp); + clear_bit(XPT_DATA, &svsk->sk_xprt.xpt_flags); + len = svc_tcp_read_marker(svsk, rqstp); if (len < 0) goto error; base = svc_tcp_restore_pages(svsk, rqstp); - want = svc_sock_reclen(svsk) - (svsk->sk_tcplen - sizeof(rpc_fraghdr)); - - vec = rqstp->rq_vec; - - pnum = copy_pages_to_kvecs(&vec[0], &rqstp->rq_pages[0], base + want); - - rqstp->rq_respages = &rqstp->rq_pages[pnum]; - rqstp->rq_next_page = rqstp->rq_respages + 1; - - /* Now receive data */ - len = svc_recvfrom(rqstp, vec, pnum, base + want, base); + want = len - (svsk->sk_tcplen - sizeof(rpc_fraghdr)); + len = svc_tcp_read_msg(rqstp, base + want, base); if (len >= 0) { + trace_svcsock_tcp_recv(&svsk->sk_xprt, len); svsk->sk_tcplen += len; svsk->sk_datalen += len; } - if (len != want || !svc_sock_final_rec(svsk)) { - svc_tcp_save_pages(svsk, rqstp); - if (len < 0 && len != -EAGAIN) - goto err_delete; - if (len == want) - svc_tcp_fragment_received(svsk); - else - dprintk("svc: incomplete TCP record (%d of %d)\n", - (int)(svsk->sk_tcplen - sizeof(rpc_fraghdr)), - svc_sock_reclen(svsk)); - goto err_noclose; - } - - if (svsk->sk_datalen < 8) { - svsk->sk_datalen = 0; - goto err_delete; /* client is nuts. */ - } + if (len != want || !svc_sock_final_rec(svsk)) + goto err_incomplete; + if (svsk->sk_datalen < 8) + goto err_nuts; rqstp->rq_arg.len = svsk->sk_datalen; rqstp->rq_arg.page_base = 0; @@ -1031,14 +1036,26 @@ static int svc_tcp_recvfrom(struct svc_rqst *rqstp) return rqstp->rq_arg.len; +err_incomplete: + svc_tcp_save_pages(svsk, rqstp); + if (len < 0 && len != -EAGAIN) + goto err_delete; + if (len == want) + svc_tcp_fragment_received(svsk); + else + trace_svcsock_tcp_recv_short(&svsk->sk_xprt, + svc_sock_reclen(svsk), + svsk->sk_tcplen - sizeof(rpc_fraghdr)); + goto err_noclose; error: if (len != -EAGAIN) goto err_delete; - dprintk("RPC: TCP recvfrom got EAGAIN\n"); + trace_svcsock_tcp_recv_eagain(&svsk->sk_xprt, 0); return 0; +err_nuts: + svsk->sk_datalen = 0; err_delete: - printk(KERN_NOTICE "%s: recvfrom returned errno %d\n", - svsk->sk_xprt.xpt_server->sv_name, -len); + trace_svcsock_tcp_recv_err(&svsk->sk_xprt, len); set_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags); err_noclose: return 0; /* record not complete */ @@ -1048,6 +1065,9 @@ err_noclose: * svc_tcp_sendto - Send out a reply on a TCP socket * @rqstp: completed svc_rqst * + * xpt_mutex ensures @rqstp's whole message is written to the socket + * without interruption. + * * Returns the number of bytes sent, or a negative errno. */ static int svc_tcp_sendto(struct svc_rqst *rqstp) @@ -1063,14 +1083,22 @@ static int svc_tcp_sendto(struct svc_rqst *rqstp) unsigned int uninitialized_var(sent); int err; - svc_release_skb(rqstp); + svc_tcp_release_rqst(rqstp); + mutex_lock(&xprt->xpt_mutex); + if (svc_xprt_is_dead(xprt)) + goto out_notconn; err = xprt_sock_sendmsg(svsk->sk_sock, &msg, xdr, 0, marker, &sent); xdr_free_bvec(xdr); + trace_svcsock_tcp_send(xprt, err < 0 ? err : sent); if (err < 0 || sent != (xdr->len + sizeof(marker))) goto out_close; + mutex_unlock(&xprt->xpt_mutex); return sent; +out_notconn: + mutex_unlock(&xprt->xpt_mutex); + return -ENOTCONN; out_close: pr_notice("rpc-srv/tcp: %s: %s %d when sending %d bytes - shutting down socket\n", xprt->xpt_server->sv_name, @@ -1078,6 +1106,7 @@ out_close: (err < 0) ? err : sent, xdr->len); set_bit(XPT_CLOSE, &xprt->xpt_flags); svc_xprt_enqueue(xprt); + mutex_unlock(&xprt->xpt_mutex); return -EAGAIN; } @@ -1094,7 +1123,7 @@ static const struct svc_xprt_ops svc_tcp_ops = { .xpo_recvfrom = svc_tcp_recvfrom, .xpo_sendto = svc_tcp_sendto, .xpo_read_payload = svc_sock_read_payload, - .xpo_release_rqst = svc_release_skb, + .xpo_release_rqst = svc_tcp_release_rqst, .xpo_detach = svc_tcp_sock_detach, .xpo_free = svc_sock_free, .xpo_has_wspace = svc_tcp_has_wspace, @@ -1132,18 +1161,16 @@ static void svc_tcp_init(struct svc_sock *svsk, struct svc_serv *serv) set_bit(XPT_CACHE_AUTH, &svsk->sk_xprt.xpt_flags); set_bit(XPT_CONG_CTRL, &svsk->sk_xprt.xpt_flags); if (sk->sk_state == TCP_LISTEN) { - dprintk("setting up TCP socket for listening\n"); strcpy(svsk->sk_xprt.xpt_remotebuf, "listener"); set_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags); sk->sk_data_ready = svc_tcp_listen_data_ready; set_bit(XPT_CONN, &svsk->sk_xprt.xpt_flags); } else { - dprintk("setting up TCP socket for reading\n"); sk->sk_state_change = svc_tcp_state_change; sk->sk_data_ready = svc_data_ready; sk->sk_write_space = svc_write_space; - svsk->sk_reclen = 0; + svsk->sk_marker = xdr_zero; svsk->sk_tcplen = 0; svsk->sk_datalen = 0; memset(&svsk->sk_pages[0], 0, sizeof(svsk->sk_pages)); @@ -1188,7 +1215,6 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); int err = 0; - dprintk("svc: svc_setup_socket %p\n", sock); svsk = kzalloc(sizeof(*svsk), GFP_KERNEL); if (!svsk) return ERR_PTR(-ENOMEM); @@ -1225,12 +1251,7 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, else svc_tcp_init(svsk, serv); - dprintk("svc: svc_setup_socket created %p (inet %p), " - "listen %d close %d\n", - svsk, svsk->sk_sk, - test_bit(XPT_LISTENER, &svsk->sk_xprt.xpt_flags), - test_bit(XPT_CLOSE, &svsk->sk_xprt.xpt_flags)); - + trace_svcsock_new_socket(sock); return svsk; } @@ -1322,11 +1343,6 @@ static struct svc_xprt *svc_create_socket(struct svc_serv *serv, struct sockaddr *newsin = (struct sockaddr *)&addr; int newlen; int family; - RPC_IFDEBUG(char buf[RPC_MAX_ADDRBUFLEN]); - - dprintk("svc: svc_create_socket(%s, %d, %s)\n", - serv->sv_program->pg_name, protocol, - __svc_print_addr(sin, buf, sizeof(buf))); if (protocol != IPPROTO_UDP && protocol != IPPROTO_TCP) { printk(KERN_WARNING "svc: only UDP and TCP " @@ -1383,7 +1399,6 @@ static struct svc_xprt *svc_create_socket(struct svc_serv *serv, svc_xprt_set_local(&svsk->sk_xprt, newsin, newlen); return (struct svc_xprt *)svsk; bummer: - dprintk("svc: svc_create_socket error = %d\n", -error); sock_release(sock); return ERR_PTR(error); } @@ -1397,8 +1412,6 @@ static void svc_sock_detach(struct svc_xprt *xprt) struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); struct sock *sk = svsk->sk_sk; - dprintk("svc: svc_sock_detach(%p)\n", svsk); - /* put back the old socket callbacks */ lock_sock(sk); sk->sk_state_change = svsk->sk_ostate; @@ -1415,8 +1428,6 @@ static void svc_tcp_sock_detach(struct svc_xprt *xprt) { struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); - dprintk("svc: svc_tcp_sock_detach(%p)\n", svsk); - svc_sock_detach(xprt); if (!test_bit(XPT_LISTENER, &xprt->xpt_flags)) { @@ -1431,7 +1442,6 @@ static void svc_tcp_sock_detach(struct svc_xprt *xprt) static void svc_sock_free(struct svc_xprt *xprt) { struct svc_sock *svsk = container_of(xprt, struct svc_sock, sk_xprt); - dprintk("svc: svc_sock_free(%p)\n", svsk); if (svsk->sk_sock->file) sockfd_put(svsk->sk_sock); diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c index 493a30a296fc..d5cc5db9dbf3 100644 --- a/net/sunrpc/xprt.c +++ b/net/sunrpc/xprt.c @@ -663,6 +663,7 @@ static void xprt_autoclose(struct work_struct *work) container_of(work, struct rpc_xprt, task_cleanup); unsigned int pflags = memalloc_nofs_save(); + trace_xprt_disconnect_auto(xprt); clear_bit(XPRT_CLOSE_WAIT, &xprt->state); xprt->ops->close(xprt); xprt_release_write(xprt, NULL); @@ -677,7 +678,7 @@ static void xprt_autoclose(struct work_struct *work) */ void xprt_disconnect_done(struct rpc_xprt *xprt) { - dprintk("RPC: disconnected transport %p\n", xprt); + trace_xprt_disconnect_done(xprt); spin_lock(&xprt->transport_lock); xprt_clear_connected(xprt); xprt_clear_write_space_locked(xprt); @@ -694,6 +695,8 @@ EXPORT_SYMBOL_GPL(xprt_disconnect_done); */ void xprt_force_disconnect(struct rpc_xprt *xprt) { + trace_xprt_disconnect_force(xprt); + /* Don't race with the test_bit() in xprt_clear_locked() */ spin_lock(&xprt->transport_lock); set_bit(XPRT_CLOSE_WAIT, &xprt->state); @@ -832,8 +835,10 @@ void xprt_connect(struct rpc_task *task) if (!xprt_lock_write(xprt, task)) return; - if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) + if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state)) { + trace_xprt_disconnect_cleanup(xprt); xprt->ops->close(xprt); + } if (!xprt_connected(xprt)) { task->tk_rqstp->rq_connect_cookie = xprt->connect_cookie; @@ -1460,7 +1465,7 @@ xprt_request_transmit(struct rpc_rqst *req, struct rpc_task *snd_task) */ req->rq_ntrans++; - trace_xprt_sendto(&req->rq_snd_buf); + trace_rpc_xdr_sendto(task, &req->rq_snd_buf); connect_cookie = xprt->connect_cookie; status = xprt->ops->send_request(req); if (status != 0) { @@ -1903,11 +1908,8 @@ struct rpc_xprt *xprt_create_transport(struct xprt_create *args) found: xprt = t->setup(args); - if (IS_ERR(xprt)) { - dprintk("RPC: xprt_create_transport: failed, %ld\n", - -PTR_ERR(xprt)); + if (IS_ERR(xprt)) goto out; - } if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT) xprt->idle_timeout = 0; INIT_WORK(&xprt->task_cleanup, xprt_autoclose); @@ -1928,8 +1930,7 @@ found: rpc_xprt_debugfs_register(xprt); - dprintk("RPC: created transport %p with %u slots\n", xprt, - xprt->max_reqs); + trace_xprt_create(xprt); out: return xprt; } @@ -1939,6 +1940,8 @@ static void xprt_destroy_cb(struct work_struct *work) struct rpc_xprt *xprt = container_of(work, struct rpc_xprt, task_cleanup); + trace_xprt_destroy(xprt); + rpc_xprt_debugfs_unregister(xprt); rpc_destroy_wait_queue(&xprt->binding); rpc_destroy_wait_queue(&xprt->pending); @@ -1963,8 +1966,6 @@ static void xprt_destroy_cb(struct work_struct *work) */ static void xprt_destroy(struct rpc_xprt *xprt) { - dprintk("RPC: destroying transport %p\n", xprt); - /* * Exclude transport connect/disconnect handlers and autoclose */ diff --git a/net/sunrpc/xprtrdma/rpc_rdma.c b/net/sunrpc/xprtrdma/rpc_rdma.c index 3c627dc685cc..2081c8fbfa48 100644 --- a/net/sunrpc/xprtrdma/rpc_rdma.c +++ b/net/sunrpc/xprtrdma/rpc_rdma.c @@ -892,8 +892,8 @@ rpcrdma_marshal_req(struct rpcrdma_xprt *r_xprt, struct rpc_rqst *rqst) * or privacy, direct data placement of individual data items * is not allowed. */ - ddp_allowed = !(rqst->rq_cred->cr_auth->au_flags & - RPCAUTH_AUTH_DATATOUCH); + ddp_allowed = !test_bit(RPCAUTH_AUTH_DATATOUCH, + &rqst->rq_cred->cr_auth->au_flags); /* * Chunks needed for results? diff --git a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c index af7eb8d202ae..1ee73f7cf931 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_backchannel.c +++ b/net/sunrpc/xprtrdma/svc_rdma_backchannel.c @@ -10,59 +10,34 @@ #include "xprt_rdma.h" #include <trace/events/rpcrdma.h> -#define RPCDBG_FACILITY RPCDBG_SVCXPRT - -#undef SVCRDMA_BACKCHANNEL_DEBUG - /** - * svc_rdma_handle_bc_reply - Process incoming backchannel reply - * @xprt: controlling backchannel transport - * @rdma_resp: pointer to incoming transport header - * @rcvbuf: XDR buffer into which to decode the reply + * svc_rdma_handle_bc_reply - Process incoming backchannel Reply + * @rqstp: resources for handling the Reply + * @rctxt: Received message * - * Returns: - * %0 if @rcvbuf is filled in, xprt_complete_rqst called, - * %-EAGAIN if server should call ->recvfrom again. */ -int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp, - struct xdr_buf *rcvbuf) +void svc_rdma_handle_bc_reply(struct svc_rqst *rqstp, + struct svc_rdma_recv_ctxt *rctxt) { + struct svc_xprt *sxprt = rqstp->rq_xprt; + struct rpc_xprt *xprt = sxprt->xpt_bc_xprt; struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); + struct xdr_buf *rcvbuf = &rqstp->rq_arg; struct kvec *dst, *src = &rcvbuf->head[0]; + __be32 *rdma_resp = rctxt->rc_recv_buf; struct rpc_rqst *req; u32 credits; - size_t len; - __be32 xid; - __be32 *p; - int ret; - - p = (__be32 *)src->iov_base; - len = src->iov_len; - xid = *rdma_resp; - -#ifdef SVCRDMA_BACKCHANNEL_DEBUG - pr_info("%s: xid=%08x, length=%zu\n", - __func__, be32_to_cpu(xid), len); - pr_info("%s: RPC/RDMA: %*ph\n", - __func__, (int)RPCRDMA_HDRLEN_MIN, rdma_resp); - pr_info("%s: RPC: %*ph\n", - __func__, (int)len, p); -#endif - - ret = -EAGAIN; - if (src->iov_len < 24) - goto out_shortreply; spin_lock(&xprt->queue_lock); - req = xprt_lookup_rqst(xprt, xid); + req = xprt_lookup_rqst(xprt, *rdma_resp); if (!req) - goto out_notfound; + goto out_unlock; dst = &req->rq_private_buf.head[0]; memcpy(&req->rq_private_buf, &req->rq_rcv_buf, sizeof(struct xdr_buf)); - if (dst->iov_len < len) + if (dst->iov_len < src->iov_len) goto out_unlock; - memcpy(dst->iov_base, p, len); + memcpy(dst->iov_base, src->iov_base, src->iov_len); xprt_pin_rqst(req); spin_unlock(&xprt->queue_lock); @@ -71,31 +46,17 @@ int svc_rdma_handle_bc_reply(struct rpc_xprt *xprt, __be32 *rdma_resp, credits = 1; /* don't deadlock */ else if (credits > r_xprt->rx_buf.rb_bc_max_requests) credits = r_xprt->rx_buf.rb_bc_max_requests; - spin_lock(&xprt->transport_lock); xprt->cwnd = credits << RPC_CWNDSHIFT; spin_unlock(&xprt->transport_lock); spin_lock(&xprt->queue_lock); - ret = 0; xprt_complete_rqst(req->rq_task, rcvbuf->len); xprt_unpin_rqst(req); rcvbuf->len = 0; out_unlock: spin_unlock(&xprt->queue_lock); -out: - return ret; - -out_shortreply: - dprintk("svcrdma: short bc reply: xprt=%p, len=%zu\n", - xprt, src->iov_len); - goto out; - -out_notfound: - dprintk("svcrdma: unrecognized bc reply: xprt=%p, xid=%08x\n", - xprt, be32_to_cpu(xid)); - goto out_unlock; } /* Send a backwards direction RPC call. @@ -192,10 +153,6 @@ rpcrdma_bc_send_request(struct svcxprt_rdma *rdma, struct rpc_rqst *rqst) *p++ = xdr_zero; *p = xdr_zero; -#ifdef SVCRDMA_BACKCHANNEL_DEBUG - pr_info("%s: %*ph\n", __func__, 64, rqst->rq_buffer); -#endif - rqst->rq_xtime = ktime_get(); rc = svc_rdma_bc_sendto(rdma, rqst, ctxt); if (rc) @@ -206,45 +163,36 @@ put_ctxt: svc_rdma_send_ctxt_put(rdma, ctxt); drop_connection: - dprintk("svcrdma: failed to send bc call\n"); return -ENOTCONN; } -/* Send an RPC call on the passive end of a transport - * connection. +/** + * xprt_rdma_bc_send_request - Send a reverse-direction Call + * @rqst: rpc_rqst containing Call message to be sent + * + * Return values: + * %0 if the message was sent successfully + * %ENOTCONN if the message was not sent */ -static int -xprt_rdma_bc_send_request(struct rpc_rqst *rqst) +static int xprt_rdma_bc_send_request(struct rpc_rqst *rqst) { struct svc_xprt *sxprt = rqst->rq_xprt->bc_xprt; - struct svcxprt_rdma *rdma; + struct svcxprt_rdma *rdma = + container_of(sxprt, struct svcxprt_rdma, sc_xprt); int ret; - dprintk("svcrdma: sending bc call with xid: %08x\n", - be32_to_cpu(rqst->rq_xid)); + if (test_bit(XPT_DEAD, &sxprt->xpt_flags)) + return -ENOTCONN; - mutex_lock(&sxprt->xpt_mutex); - - ret = -ENOTCONN; - rdma = container_of(sxprt, struct svcxprt_rdma, sc_xprt); - if (!test_bit(XPT_DEAD, &sxprt->xpt_flags)) { - ret = rpcrdma_bc_send_request(rdma, rqst); - if (ret == -ENOTCONN) - svc_close_xprt(sxprt); - } - - mutex_unlock(&sxprt->xpt_mutex); - - if (ret < 0) - return ret; - return 0; + ret = rpcrdma_bc_send_request(rdma, rqst); + if (ret == -ENOTCONN) + svc_close_xprt(sxprt); + return ret; } static void xprt_rdma_bc_close(struct rpc_xprt *xprt) { - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); - xprt_disconnect_done(xprt); xprt->cwnd = RPC_CWNDSHIFT; } @@ -252,8 +200,6 @@ xprt_rdma_bc_close(struct rpc_xprt *xprt) static void xprt_rdma_bc_put(struct rpc_xprt *xprt) { - dprintk("svcrdma: %s: xprt %p\n", __func__, xprt); - xprt_rdma_free_addresses(xprt); xprt_free(xprt); } @@ -288,19 +234,14 @@ xprt_setup_rdma_bc(struct xprt_create *args) struct rpc_xprt *xprt; struct rpcrdma_xprt *new_xprt; - if (args->addrlen > sizeof(xprt->addr)) { - dprintk("RPC: %s: address too large\n", __func__); + if (args->addrlen > sizeof(xprt->addr)) return ERR_PTR(-EBADF); - } xprt = xprt_alloc(args->net, sizeof(*new_xprt), RPCRDMA_MAX_BC_REQUESTS, RPCRDMA_MAX_BC_REQUESTS); - if (!xprt) { - dprintk("RPC: %s: couldn't allocate rpc_xprt\n", - __func__); + if (!xprt) return ERR_PTR(-ENOMEM); - } xprt->timeout = &xprt_rdma_bc_timeout; xprt_set_bound(xprt); diff --git a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c index efa5fcb5793f..e426fedb9524 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c +++ b/net/sunrpc/xprtrdma/svc_rdma_recvfrom.c @@ -665,23 +665,23 @@ static int svc_rdma_xdr_decode_req(struct xdr_buf *rq_arg, return hdr_len; out_short: - trace_svcrdma_decode_short(rq_arg->len); + trace_svcrdma_decode_short_err(rq_arg->len); return -EINVAL; out_version: - trace_svcrdma_decode_badvers(rdma_argp); + trace_svcrdma_decode_badvers_err(rdma_argp); return -EPROTONOSUPPORT; out_drop: - trace_svcrdma_decode_drop(rdma_argp); + trace_svcrdma_decode_drop_err(rdma_argp); return 0; out_proc: - trace_svcrdma_decode_badproc(rdma_argp); + trace_svcrdma_decode_badproc_err(rdma_argp); return -EINVAL; out_inval: - trace_svcrdma_decode_parse(rdma_argp); + trace_svcrdma_decode_parse_err(rdma_argp); return -EINVAL; } @@ -878,12 +878,9 @@ int svc_rdma_recvfrom(struct svc_rqst *rqstp) goto out_drop; rqstp->rq_xprt_hlen = ret; - if (svc_rdma_is_backchannel_reply(xprt, p)) { - ret = svc_rdma_handle_bc_reply(xprt->xpt_bc_xprt, p, - &rqstp->rq_arg); - svc_rdma_recv_ctxt_put(rdma_xprt, ctxt); - return ret; - } + if (svc_rdma_is_backchannel_reply(xprt, p)) + goto out_backchannel; + svc_rdma_get_inv_rkey(rdma_xprt, ctxt); p += rpcrdma_fixed_maxsz; @@ -913,6 +910,8 @@ out_postfail: svc_rdma_recv_ctxt_put(rdma_xprt, ctxt); return ret; +out_backchannel: + svc_rdma_handle_bc_reply(rqstp, ctxt); out_drop: svc_rdma_recv_ctxt_put(rdma_xprt, ctxt); return 0; diff --git a/net/sunrpc/xprtrdma/svc_rdma_rw.c b/net/sunrpc/xprtrdma/svc_rdma_rw.c index 23c2d3ce0dc9..5eb35309ecef 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_rw.c +++ b/net/sunrpc/xprtrdma/svc_rdma_rw.c @@ -9,13 +9,10 @@ #include <linux/sunrpc/rpc_rdma.h> #include <linux/sunrpc/svc_rdma.h> -#include <linux/sunrpc/debug.h> #include "xprt_rdma.h" #include <trace/events/rpcrdma.h> -#define RPCDBG_FACILITY RPCDBG_SVCXPRT - static void svc_rdma_write_done(struct ib_cq *cq, struct ib_wc *wc); static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); @@ -39,7 +36,7 @@ static void svc_rdma_wc_read_done(struct ib_cq *cq, struct ib_wc *wc); struct svc_rdma_rw_ctxt { struct list_head rw_list; struct rdma_rw_ctx rw_ctx; - int rw_nents; + unsigned int rw_nents; struct sg_table rw_sg_table; struct scatterlist rw_first_sgl[]; }; @@ -67,19 +64,22 @@ svc_rdma_get_rw_ctxt(struct svcxprt_rdma *rdma, unsigned int sges) ctxt = kmalloc(struct_size(ctxt, rw_first_sgl, SG_CHUNK_SIZE), GFP_KERNEL); if (!ctxt) - goto out; + goto out_noctx; INIT_LIST_HEAD(&ctxt->rw_list); } ctxt->rw_sg_table.sgl = ctxt->rw_first_sgl; if (sg_alloc_table_chained(&ctxt->rw_sg_table, sges, ctxt->rw_sg_table.sgl, - SG_CHUNK_SIZE)) { - kfree(ctxt); - ctxt = NULL; - } -out: + SG_CHUNK_SIZE)) + goto out_free; return ctxt; + +out_free: + kfree(ctxt); +out_noctx: + trace_svcrdma_no_rwctx_err(rdma, sges); + return NULL; } static void svc_rdma_put_rw_ctxt(struct svcxprt_rdma *rdma, @@ -107,6 +107,34 @@ void svc_rdma_destroy_rw_ctxts(struct svcxprt_rdma *rdma) } } +/** + * svc_rdma_rw_ctx_init - Prepare a R/W context for I/O + * @rdma: controlling transport instance + * @ctxt: R/W context to prepare + * @offset: RDMA offset + * @handle: RDMA tag/handle + * @direction: I/O direction + * + * Returns on success, the number of WQEs that will be needed + * on the workqueue, or a negative errno. + */ +static int svc_rdma_rw_ctx_init(struct svcxprt_rdma *rdma, + struct svc_rdma_rw_ctxt *ctxt, + u64 offset, u32 handle, + enum dma_data_direction direction) +{ + int ret; + + ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, rdma->sc_port_num, + ctxt->rw_sg_table.sgl, ctxt->rw_nents, + 0, offset, handle, direction); + if (unlikely(ret < 0)) { + svc_rdma_put_rw_ctxt(rdma, ctxt); + trace_svcrdma_dma_map_rw_err(rdma, ctxt->rw_nents, ret); + } + return ret; +} + /* A chunk context tracks all I/O for moving one Read or Write * chunk. This is a a set of rdma_rw's that handle data movement * for all segments of one chunk. @@ -428,15 +456,13 @@ svc_rdma_build_writes(struct svc_rdma_write_info *info, ctxt = svc_rdma_get_rw_ctxt(rdma, (write_len >> PAGE_SHIFT) + 2); if (!ctxt) - goto out_noctx; + return -ENOMEM; constructor(info, write_len, ctxt); - ret = rdma_rw_ctx_init(&ctxt->rw_ctx, rdma->sc_qp, - rdma->sc_port_num, ctxt->rw_sg_table.sgl, - ctxt->rw_nents, 0, seg_offset, - seg_handle, DMA_TO_DEVICE); + ret = svc_rdma_rw_ctx_init(rdma, ctxt, seg_offset, seg_handle, + DMA_TO_DEVICE); if (ret < 0) - goto out_initerr; + return -EIO; trace_svcrdma_send_wseg(seg_handle, write_len, seg_offset); @@ -455,18 +481,9 @@ svc_rdma_build_writes(struct svc_rdma_write_info *info, return 0; out_overflow: - dprintk("svcrdma: inadequate space in Write chunk (%u)\n", - info->wi_nsegs); + trace_svcrdma_small_wrch_err(rdma, remaining, info->wi_seg_no, + info->wi_nsegs); return -E2BIG; - -out_noctx: - dprintk("svcrdma: no R/W ctxs available\n"); - return -ENOMEM; - -out_initerr: - svc_rdma_put_rw_ctxt(rdma, ctxt); - trace_svcrdma_dma_map_rwctx(rdma, ret); - return -EIO; } /* Send one of an xdr_buf's kvecs by itself. To send a Reply @@ -616,7 +633,7 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, sge_no = PAGE_ALIGN(info->ri_pageoff + len) >> PAGE_SHIFT; ctxt = svc_rdma_get_rw_ctxt(cc->cc_rdma, sge_no); if (!ctxt) - goto out_noctx; + return -ENOMEM; ctxt->rw_nents = sge_no; sg = ctxt->rw_sg_table.sgl; @@ -646,29 +663,18 @@ static int svc_rdma_build_read_segment(struct svc_rdma_read_info *info, goto out_overrun; } - ret = rdma_rw_ctx_init(&ctxt->rw_ctx, cc->cc_rdma->sc_qp, - cc->cc_rdma->sc_port_num, - ctxt->rw_sg_table.sgl, ctxt->rw_nents, - 0, offset, rkey, DMA_FROM_DEVICE); + ret = svc_rdma_rw_ctx_init(cc->cc_rdma, ctxt, offset, rkey, + DMA_FROM_DEVICE); if (ret < 0) - goto out_initerr; + return -EIO; list_add(&ctxt->rw_list, &cc->cc_rwctxts); cc->cc_sqecount += ret; return 0; -out_noctx: - dprintk("svcrdma: no R/W ctxs available\n"); - return -ENOMEM; - out_overrun: - dprintk("svcrdma: request overruns rq_pages\n"); + trace_svcrdma_page_overrun_err(cc->cc_rdma, rqstp, info->ri_pageno); return -EINVAL; - -out_initerr: - trace_svcrdma_dma_map_rwctx(cc->cc_rdma, ret); - svc_rdma_put_rw_ctxt(cc->cc_rdma, ctxt); - return -EIO; } /* Walk the segments in the Read chunk starting at @p and construct diff --git a/net/sunrpc/xprtrdma/svc_rdma_sendto.c b/net/sunrpc/xprtrdma/svc_rdma_sendto.c index b6c8643867f2..38e7c3c8c4a9 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_sendto.c +++ b/net/sunrpc/xprtrdma/svc_rdma_sendto.c @@ -868,12 +868,10 @@ int svc_rdma_sendto(struct svc_rqst *rqstp) __be32 *p; int ret; - /* Create the RDMA response header. xprt->xpt_mutex, - * acquired in svc_send(), serializes RPC replies. The - * code path below that inserts the credit grant value - * into each transport header runs only inside this - * critical section. - */ + ret = -ENOTCONN; + if (svc_xprt_is_dead(xprt)) + goto err0; + ret = -ENOMEM; sctxt = svc_rdma_send_ctxt_get(rdma); if (!sctxt) diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index ea54785db4f8..d38be57b00ed 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -211,7 +211,12 @@ static void handle_connect_req(struct rdma_cm_id *new_cma_id, newxprt->sc_ord = param->initiator_depth; sa = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr; - svc_xprt_set_remote(&newxprt->sc_xprt, sa, svc_addr_len(sa)); + newxprt->sc_xprt.xpt_remotelen = svc_addr_len(sa); + memcpy(&newxprt->sc_xprt.xpt_remote, sa, + newxprt->sc_xprt.xpt_remotelen); + snprintf(newxprt->sc_xprt.xpt_remotebuf, + sizeof(newxprt->sc_xprt.xpt_remotebuf) - 1, "%pISc", sa); + /* The remote port is arbitrary and not under the control of the * client ULP. Set it to a fixed value so that the DRC continues * to be effective after a reconnect. @@ -309,11 +314,8 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, struct svcxprt_rdma *cma_xprt; int ret; - dprintk("svcrdma: Creating RDMA listener\n"); - if ((sa->sa_family != AF_INET) && (sa->sa_family != AF_INET6)) { - dprintk("svcrdma: Address family %d is not supported.\n", sa->sa_family); + if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6) return ERR_PTR(-EAFNOSUPPORT); - } cma_xprt = svc_rdma_create_xprt(serv, net); if (!cma_xprt) return ERR_PTR(-ENOMEM); @@ -324,7 +326,6 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, RDMA_PS_TCP, IB_QPT_RC); if (IS_ERR(listen_id)) { ret = PTR_ERR(listen_id); - dprintk("svcrdma: rdma_create_id failed = %d\n", ret); goto err0; } @@ -333,23 +334,17 @@ static struct svc_xprt *svc_rdma_create(struct svc_serv *serv, */ #if IS_ENABLED(CONFIG_IPV6) ret = rdma_set_afonly(listen_id, 1); - if (ret) { - dprintk("svcrdma: rdma_set_afonly failed = %d\n", ret); + if (ret) goto err1; - } #endif ret = rdma_bind_addr(listen_id, sa); - if (ret) { - dprintk("svcrdma: rdma_bind_addr failed = %d\n", ret); + if (ret) goto err1; - } cma_xprt->sc_cm_id = listen_id; ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG); - if (ret) { - dprintk("svcrdma: rdma_listen failed = %d\n", ret); + if (ret) goto err1; - } /* * We need to use the address from the cm_id in case the @@ -405,9 +400,6 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) if (!newxprt) return NULL; - dprintk("svcrdma: newxprt from accept queue = %p, cm_id=%p\n", - newxprt, newxprt->sc_cm_id); - dev = newxprt->sc_cm_id->device; newxprt->sc_port_num = newxprt->sc_cm_id->port_num; @@ -443,21 +435,17 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) newxprt->sc_pd = ib_alloc_pd(dev, 0); if (IS_ERR(newxprt->sc_pd)) { - dprintk("svcrdma: error creating PD for connect request\n"); + trace_svcrdma_pd_err(newxprt, PTR_ERR(newxprt->sc_pd)); goto errout; } newxprt->sc_sq_cq = ib_alloc_cq_any(dev, newxprt, newxprt->sc_sq_depth, IB_POLL_WORKQUEUE); - if (IS_ERR(newxprt->sc_sq_cq)) { - dprintk("svcrdma: error creating SQ CQ for connect request\n"); + if (IS_ERR(newxprt->sc_sq_cq)) goto errout; - } newxprt->sc_rq_cq = ib_alloc_cq_any(dev, newxprt, rq_depth, IB_POLL_WORKQUEUE); - if (IS_ERR(newxprt->sc_rq_cq)) { - dprintk("svcrdma: error creating RQ CQ for connect request\n"); + if (IS_ERR(newxprt->sc_rq_cq)) goto errout; - } memset(&qp_attr, 0, sizeof qp_attr); qp_attr.event_handler = qp_event_handler; @@ -481,7 +469,7 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) ret = rdma_create_qp(newxprt->sc_cm_id, newxprt->sc_pd, &qp_attr); if (ret) { - dprintk("svcrdma: failed to create QP, ret=%d\n", ret); + trace_svcrdma_qp_err(newxprt, ret); goto errout; } newxprt->sc_qp = newxprt->sc_cm_id->qp; @@ -489,8 +477,10 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) if (!(dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS)) newxprt->sc_snd_w_inv = false; if (!rdma_protocol_iwarp(dev, newxprt->sc_port_num) && - !rdma_ib_or_roce(dev, newxprt->sc_port_num)) + !rdma_ib_or_roce(dev, newxprt->sc_port_num)) { + trace_svcrdma_fabric_err(newxprt, -EINVAL); goto errout; + } if (!svc_rdma_post_recvs(newxprt)) goto errout; @@ -512,15 +502,17 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) conn_param.initiator_depth = min_t(int, newxprt->sc_ord, dev->attrs.max_qp_init_rd_atom); if (!conn_param.initiator_depth) { - dprintk("svcrdma: invalid ORD setting\n"); ret = -EINVAL; + trace_svcrdma_initdepth_err(newxprt, ret); goto errout; } conn_param.private_data = &pmsg; conn_param.private_data_len = sizeof(pmsg); ret = rdma_accept(newxprt->sc_cm_id, &conn_param); - if (ret) + if (ret) { + trace_svcrdma_accept_err(newxprt, ret); goto errout; + } #if IS_ENABLED(CONFIG_SUNRPC_DEBUG) dprintk("svcrdma: new connection %p accepted:\n", newxprt); @@ -535,12 +527,9 @@ static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt) dprintk(" ord : %d\n", conn_param.initiator_depth); #endif - trace_svcrdma_xprt_accept(&newxprt->sc_xprt); return &newxprt->sc_xprt; errout: - dprintk("svcrdma: failure accepting new connection rc=%d.\n", ret); - trace_svcrdma_xprt_fail(&newxprt->sc_xprt); /* Take a reference in case the DTO handler runs */ svc_xprt_get(&newxprt->sc_xprt); if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp)) @@ -578,8 +567,6 @@ static void __svc_rdma_free(struct work_struct *work) container_of(work, struct svcxprt_rdma, sc_work); struct svc_xprt *xprt = &rdma->sc_xprt; - trace_svcrdma_xprt_free(xprt); - if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) ib_drain_qp(rdma->sc_qp); diff --git a/net/sunrpc/xprtrdma/transport.c b/net/sunrpc/xprtrdma/transport.c index 659da37020a4..0c4af7f5e241 100644 --- a/net/sunrpc/xprtrdma/transport.c +++ b/net/sunrpc/xprtrdma/transport.c @@ -68,7 +68,7 @@ * tunables */ -unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE; +static unsigned int xprt_rdma_slot_table_entries = RPCRDMA_DEF_SLOT_TABLE; unsigned int xprt_rdma_max_inline_read = RPCRDMA_DEF_INLINE; unsigned int xprt_rdma_max_inline_write = RPCRDMA_DEF_INLINE; unsigned int xprt_rdma_memreg_strategy = RPCRDMA_FRWR; @@ -281,8 +281,6 @@ xprt_rdma_destroy(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); - trace_xprtrdma_op_destroy(r_xprt); - cancel_delayed_work_sync(&r_xprt->rx_connect_worker); rpcrdma_xprt_disconnect(r_xprt); @@ -365,10 +363,6 @@ xprt_setup_rdma(struct xprt_create *args) xprt->max_payload = RPCRDMA_MAX_DATA_SEGS << PAGE_SHIFT; - dprintk("RPC: %s: %s:%s\n", __func__, - xprt->address_strings[RPC_DISPLAY_ADDR], - xprt->address_strings[RPC_DISPLAY_PORT]); - trace_xprtrdma_create(new_xprt); return xprt; } @@ -385,8 +379,6 @@ void xprt_rdma_close(struct rpc_xprt *xprt) { struct rpcrdma_xprt *r_xprt = rpcx_to_rdmax(xprt); - trace_xprtrdma_op_close(r_xprt); - rpcrdma_xprt_disconnect(r_xprt); xprt->reestablish_timeout = 0; diff --git a/net/sunrpc/xprtrdma/verbs.c b/net/sunrpc/xprtrdma/verbs.c index 05c4d3a9cda2..2ae348377806 100644 --- a/net/sunrpc/xprtrdma/verbs.c +++ b/net/sunrpc/xprtrdma/verbs.c @@ -141,7 +141,6 @@ void rpcrdma_flush_disconnect(struct ib_cq *cq, struct ib_wc *wc) if (wc->status != IB_WC_SUCCESS && r_xprt->rx_ep->re_connect_status == 1) { r_xprt->rx_ep->re_connect_status = -ECONNABORTED; - trace_xprtrdma_flush_dct(r_xprt, wc->status); xprt_force_disconnect(xprt); } } diff --git a/net/sunrpc/xprtsock.c b/net/sunrpc/xprtsock.c index 3a143e250b9a..914508ea9b84 100644 --- a/net/sunrpc/xprtsock.c +++ b/net/sunrpc/xprtsock.c @@ -2528,8 +2528,16 @@ static int bc_sendto(struct rpc_rqst *req) return sent; } -/* - * The send routine. Borrows from svc_send +/** + * bc_send_request - Send a backchannel Call on a TCP socket + * @req: rpc_rqst containing Call message to be sent + * + * xpt_mutex ensures @rqstp's whole message is written to the socket + * without interruption. + * + * Return values: + * %0 if the message was sent successfully + * %ENOTCONN if the message was not sent */ static int bc_send_request(struct rpc_rqst *req) { diff --git a/samples/Kconfig b/samples/Kconfig index 953abbdebf7b..f3ac549a53b0 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -209,4 +209,11 @@ config SAMPLE_WATCHDOG bool "watchdog sample" depends on CC_CAN_LINK +config SAMPLE_WATCH_QUEUE + bool "Build example /dev/watch_queue notification consumer" + depends on HEADERS_INSTALL + help + Build example userspace program to use the new mount_notify(), + sb_notify() syscalls and the KEYCTL_WATCH_KEY keyctl() function. + endif # SAMPLES diff --git a/samples/Makefile b/samples/Makefile index 4029d207cebb..754553597581 100644 --- a/samples/Makefile +++ b/samples/Makefile @@ -27,3 +27,4 @@ obj-y += vfio-mdev/ subdir-$(CONFIG_SAMPLE_VFS) += vfs obj-$(CONFIG_SAMPLE_INTEL_MEI) += mei/ subdir-$(CONFIG_SAMPLE_WATCHDOG) += watchdog +subdir-$(CONFIG_SAMPLE_WATCH_QUEUE) += watch_queue diff --git a/samples/watch_queue/.gitignore b/samples/watch_queue/.gitignore new file mode 100644 index 000000000000..2aa3c7e56a1a --- /dev/null +++ b/samples/watch_queue/.gitignore @@ -0,0 +1 @@ +watch_test diff --git a/samples/watch_queue/Makefile b/samples/watch_queue/Makefile new file mode 100644 index 000000000000..8511fb6c53d2 --- /dev/null +++ b/samples/watch_queue/Makefile @@ -0,0 +1,7 @@ +# List of programs to build +hostprogs := watch_test + +# Tell kbuild to always build the programs +always-y := $(hostprogs) + +HOSTCFLAGS_watch_test.o += -I$(objtree)/usr/include diff --git a/samples/watch_queue/watch_test.c b/samples/watch_queue/watch_test.c new file mode 100644 index 000000000000..46e618a897fe --- /dev/null +++ b/samples/watch_queue/watch_test.c @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Use /dev/watch_queue to watch for notifications. + * + * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. + * Written by David Howells (dhowells@redhat.com) + */ + +#define _GNU_SOURCE +#include <stdbool.h> +#include <stdarg.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <signal.h> +#include <unistd.h> +#include <errno.h> +#include <sys/ioctl.h> +#include <limits.h> +#include <linux/watch_queue.h> +#include <linux/unistd.h> +#include <linux/keyctl.h> + +#ifndef KEYCTL_WATCH_KEY +#define KEYCTL_WATCH_KEY -1 +#endif +#ifndef __NR_keyctl +#define __NR_keyctl -1 +#endif + +#define BUF_SIZE 256 + +static long keyctl_watch_key(int key, int watch_fd, int watch_id) +{ + return syscall(__NR_keyctl, KEYCTL_WATCH_KEY, key, watch_fd, watch_id); +} + +static const char *key_subtypes[256] = { + [NOTIFY_KEY_INSTANTIATED] = "instantiated", + [NOTIFY_KEY_UPDATED] = "updated", + [NOTIFY_KEY_LINKED] = "linked", + [NOTIFY_KEY_UNLINKED] = "unlinked", + [NOTIFY_KEY_CLEARED] = "cleared", + [NOTIFY_KEY_REVOKED] = "revoked", + [NOTIFY_KEY_INVALIDATED] = "invalidated", + [NOTIFY_KEY_SETATTR] = "setattr", +}; + +static void saw_key_change(struct watch_notification *n, size_t len) +{ + struct key_notification *k = (struct key_notification *)n; + + if (len != sizeof(struct key_notification)) { + fprintf(stderr, "Incorrect key message length\n"); + return; + } + + printf("KEY %08x change=%u[%s] aux=%u\n", + k->key_id, n->subtype, key_subtypes[n->subtype], k->aux); +} + +/* + * Consume and display events. + */ +static void consumer(int fd) +{ + unsigned char buffer[433], *p, *end; + union { + struct watch_notification n; + unsigned char buf1[128]; + } n; + ssize_t buf_len; + + for (;;) { + buf_len = read(fd, buffer, sizeof(buffer)); + if (buf_len == -1) { + perror("read"); + exit(1); + } + + if (buf_len == 0) { + printf("-- END --\n"); + return; + } + + if (buf_len > sizeof(buffer)) { + fprintf(stderr, "Read buffer overrun: %zd\n", buf_len); + return; + } + + printf("read() = %zd\n", buf_len); + + p = buffer; + end = buffer + buf_len; + while (p < end) { + size_t largest, len; + + largest = end - p; + if (largest > 128) + largest = 128; + if (largest < sizeof(struct watch_notification)) { + fprintf(stderr, "Short message header: %zu\n", largest); + return; + } + memcpy(&n, p, largest); + + printf("NOTIFY[%03zx]: ty=%06x sy=%02x i=%08x\n", + p - buffer, n.n.type, n.n.subtype, n.n.info); + + len = n.n.info & WATCH_INFO_LENGTH; + if (len < sizeof(n.n) || len > largest) { + fprintf(stderr, "Bad message length: %zu/%zu\n", len, largest); + exit(1); + } + + switch (n.n.type) { + case WATCH_TYPE_META: + switch (n.n.subtype) { + case WATCH_META_REMOVAL_NOTIFICATION: + printf("REMOVAL of watchpoint %08x\n", + (n.n.info & WATCH_INFO_ID) >> + WATCH_INFO_ID__SHIFT); + break; + case WATCH_META_LOSS_NOTIFICATION: + printf("-- LOSS --\n"); + break; + default: + printf("other meta record\n"); + break; + } + break; + case WATCH_TYPE_KEY_NOTIFY: + saw_key_change(&n.n, len); + break; + default: + printf("other type\n"); + break; + } + + p += len; + } + } +} + +static struct watch_notification_filter filter = { + .nr_filters = 1, + .filters = { + [0] = { + .type = WATCH_TYPE_KEY_NOTIFY, + .subtype_filter[0] = UINT_MAX, + }, + }, +}; + +int main(int argc, char **argv) +{ + int pipefd[2], fd; + + if (pipe2(pipefd, O_NOTIFICATION_PIPE) == -1) { + perror("pipe2"); + exit(1); + } + fd = pipefd[0]; + + if (ioctl(fd, IOC_WATCH_QUEUE_SET_SIZE, BUF_SIZE) == -1) { + perror("watch_queue(size)"); + exit(1); + } + + if (ioctl(fd, IOC_WATCH_QUEUE_SET_FILTER, &filter) == -1) { + perror("watch_queue(filter)"); + exit(1); + } + + if (keyctl_watch_key(KEY_SPEC_SESSION_KEYRING, fd, 0x01) == -1) { + perror("keyctl"); + exit(1); + } + + if (keyctl_watch_key(KEY_SPEC_USER_KEYRING, fd, 0x02) == -1) { + perror("keyctl"); + exit(1); + } + + consumer(fd); + exit(0); +} diff --git a/scripts/Makefile.kcsan b/scripts/Makefile.kcsan new file mode 100644 index 000000000000..bd4da1af5953 --- /dev/null +++ b/scripts/Makefile.kcsan @@ -0,0 +1,19 @@ +# SPDX-License-Identifier: GPL-2.0 +ifdef CONFIG_KCSAN + +# GCC and Clang accept backend options differently. Do not wrap in cc-option, +# because Clang accepts "--param" even if it is unused. +ifdef CONFIG_CC_IS_CLANG +cc-param = -mllvm -$(1) +else +cc-param = --param -$(1) +endif + +# Keep most options here optional, to allow enabling more compilers if absence +# of some options does not break KCSAN nor causes false positive reports. +CFLAGS_KCSAN := -fsanitize=thread \ + $(call cc-option,$(call cc-param,tsan-instrument-func-entry-exit=0) -fno-optimize-sibling-calls) \ + $(call cc-option,$(call cc-param,tsan-instrument-read-before-write=1)) \ + $(call cc-param,tsan-distinguish-volatile=1) + +endif # CONFIG_KCSAN diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index 94eeddb2e599..99ac59c59826 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -152,6 +152,16 @@ _c_flags += $(if $(patsubst n%,, \ $(CFLAGS_KCOV)) endif +# +# Enable KCSAN flags except some files or directories we don't want to check +# (depends on variables KCSAN_SANITIZE_obj.o, KCSAN_SANITIZE) +# +ifeq ($(CONFIG_KCSAN),y) +_c_flags += $(if $(patsubst n%,, \ + $(KCSAN_SANITIZE_$(basetarget).o)$(KCSAN_SANITIZE)y), \ + $(CFLAGS_KCSAN)) +endif + # $(srctree)/$(src) for including checkin headers from generated source files # $(objtree)/$(obj) for including generated headers from checkin source files ifeq ($(KBUILD_EXTMOD),) diff --git a/scripts/atomic/fallbacks/acquire b/scripts/atomic/fallbacks/acquire index e38871e64db6..59c00529dc7c 100755 --- a/scripts/atomic/fallbacks/acquire +++ b/scripts/atomic/fallbacks/acquire @@ -1,8 +1,8 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}_acquire(${params}) +static __always_inline ${ret} +${arch}${atomic}_${pfx}${name}${sfx}_acquire(${params}) { - ${ret} ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ${ret} ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args}); __atomic_acquire_fence(); return ret; } diff --git a/scripts/atomic/fallbacks/add_negative b/scripts/atomic/fallbacks/add_negative index e6f4815637de..a66635bceefb 100755 --- a/scripts/atomic/fallbacks/add_negative +++ b/scripts/atomic/fallbacks/add_negative @@ -1,6 +1,6 @@ cat <<EOF /** - * ${atomic}_add_negative - add and test if negative + * ${arch}${atomic}_add_negative - add and test if negative * @i: integer value to add * @v: pointer of type ${atomic}_t * @@ -8,9 +8,9 @@ cat <<EOF * if the result is negative, or false when * result is greater than or equal to zero. */ -static inline bool -${atomic}_add_negative(${int} i, ${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_add_negative(${int} i, ${atomic}_t *v) { - return ${atomic}_add_return(i, v) < 0; + return ${arch}${atomic}_add_return(i, v) < 0; } EOF diff --git a/scripts/atomic/fallbacks/add_unless b/scripts/atomic/fallbacks/add_unless index 792533885fbf..2ff598a3f9ec 100755 --- a/scripts/atomic/fallbacks/add_unless +++ b/scripts/atomic/fallbacks/add_unless @@ -1,6 +1,6 @@ cat << EOF /** - * ${atomic}_add_unless - add unless the number is already a given value + * ${arch}${atomic}_add_unless - add unless the number is already a given value * @v: pointer of type ${atomic}_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -8,9 +8,9 @@ cat << EOF * Atomically adds @a to @v, if @v was not already @u. * Returns true if the addition was done. */ -static inline bool -${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) +static __always_inline bool +${arch}${atomic}_add_unless(${atomic}_t *v, ${int} a, ${int} u) { - return ${atomic}_fetch_add_unless(v, a, u) != u; + return ${arch}${atomic}_fetch_add_unless(v, a, u) != u; } EOF diff --git a/scripts/atomic/fallbacks/andnot b/scripts/atomic/fallbacks/andnot index 9f3a3216b5e3..3f18663dcefb 100755 --- a/scripts/atomic/fallbacks/andnot +++ b/scripts/atomic/fallbacks/andnot @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) +static __always_inline ${ret} +${arch}${atomic}_${pfx}andnot${sfx}${order}(${int} i, ${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}and${sfx}${order}(~i, v); + ${retstmt}${arch}${atomic}_${pfx}and${sfx}${order}(~i, v); } EOF diff --git a/scripts/atomic/fallbacks/dec b/scripts/atomic/fallbacks/dec index 10bbc82be31d..e2e01f0574bb 100755 --- a/scripts/atomic/fallbacks/dec +++ b/scripts/atomic/fallbacks/dec @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) +static __always_inline ${ret} +${arch}${atomic}_${pfx}dec${sfx}${order}(${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}sub${sfx}${order}(1, v); + ${retstmt}${arch}${atomic}_${pfx}sub${sfx}${order}(1, v); } EOF diff --git a/scripts/atomic/fallbacks/dec_and_test b/scripts/atomic/fallbacks/dec_and_test index 0ce7103b3df2..e8a5e492eb5f 100755 --- a/scripts/atomic/fallbacks/dec_and_test +++ b/scripts/atomic/fallbacks/dec_and_test @@ -1,15 +1,15 @@ cat <<EOF /** - * ${atomic}_dec_and_test - decrement and test + * ${arch}${atomic}_dec_and_test - decrement and test * @v: pointer of type ${atomic}_t * * Atomically decrements @v by 1 and * returns true if the result is 0, or false for all other * cases. */ -static inline bool -${atomic}_dec_and_test(${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_dec_and_test(${atomic}_t *v) { - return ${atomic}_dec_return(v) == 0; + return ${arch}${atomic}_dec_return(v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/dec_if_positive b/scripts/atomic/fallbacks/dec_if_positive index c52eacec43c8..527adec89c37 100755 --- a/scripts/atomic/fallbacks/dec_if_positive +++ b/scripts/atomic/fallbacks/dec_if_positive @@ -1,14 +1,14 @@ cat <<EOF -static inline ${ret} -${atomic}_dec_if_positive(${atomic}_t *v) +static __always_inline ${ret} +${arch}${atomic}_dec_if_positive(${atomic}_t *v) { - ${int} dec, c = ${atomic}_read(v); + ${int} dec, c = ${arch}${atomic}_read(v); do { dec = c - 1; if (unlikely(dec < 0)) break; - } while (!${atomic}_try_cmpxchg(v, &c, dec)); + } while (!${arch}${atomic}_try_cmpxchg(v, &c, dec)); return dec; } diff --git a/scripts/atomic/fallbacks/dec_unless_positive b/scripts/atomic/fallbacks/dec_unless_positive index 8a2578f14268..dcab6848ca1e 100755 --- a/scripts/atomic/fallbacks/dec_unless_positive +++ b/scripts/atomic/fallbacks/dec_unless_positive @@ -1,13 +1,13 @@ cat <<EOF -static inline bool -${atomic}_dec_unless_positive(${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_dec_unless_positive(${atomic}_t *v) { - ${int} c = ${atomic}_read(v); + ${int} c = ${arch}${atomic}_read(v); do { if (unlikely(c > 0)) return false; - } while (!${atomic}_try_cmpxchg(v, &c, c - 1)); + } while (!${arch}${atomic}_try_cmpxchg(v, &c, c - 1)); return true; } diff --git a/scripts/atomic/fallbacks/fence b/scripts/atomic/fallbacks/fence index 82f68fa6931a..3764fc8ce945 100755 --- a/scripts/atomic/fallbacks/fence +++ b/scripts/atomic/fallbacks/fence @@ -1,10 +1,10 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}(${params}) +static __always_inline ${ret} +${arch}${atomic}_${pfx}${name}${sfx}(${params}) { ${ret} ret; __atomic_pre_full_fence(); - ret = ${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ret = ${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args}); __atomic_post_full_fence(); return ret; } diff --git a/scripts/atomic/fallbacks/fetch_add_unless b/scripts/atomic/fallbacks/fetch_add_unless index d2c091db7eae..0e0b9aef1515 100755 --- a/scripts/atomic/fallbacks/fetch_add_unless +++ b/scripts/atomic/fallbacks/fetch_add_unless @@ -1,6 +1,6 @@ cat << EOF /** - * ${atomic}_fetch_add_unless - add unless the number is already a given value + * ${arch}${atomic}_fetch_add_unless - add unless the number is already a given value * @v: pointer of type ${atomic}_t * @a: the amount to add to v... * @u: ...unless v is equal to u. @@ -8,15 +8,15 @@ cat << EOF * Atomically adds @a to @v, so long as @v was not already @u. * Returns original value of @v */ -static inline ${int} -${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) +static __always_inline ${int} +${arch}${atomic}_fetch_add_unless(${atomic}_t *v, ${int} a, ${int} u) { - ${int} c = ${atomic}_read(v); + ${int} c = ${arch}${atomic}_read(v); do { if (unlikely(c == u)) break; - } while (!${atomic}_try_cmpxchg(v, &c, c + a)); + } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + a)); return c; } diff --git a/scripts/atomic/fallbacks/inc b/scripts/atomic/fallbacks/inc index f866b3ad2353..15ec62946e8c 100755 --- a/scripts/atomic/fallbacks/inc +++ b/scripts/atomic/fallbacks/inc @@ -1,7 +1,7 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) +static __always_inline ${ret} +${arch}${atomic}_${pfx}inc${sfx}${order}(${atomic}_t *v) { - ${retstmt}${atomic}_${pfx}add${sfx}${order}(1, v); + ${retstmt}${arch}${atomic}_${pfx}add${sfx}${order}(1, v); } EOF diff --git a/scripts/atomic/fallbacks/inc_and_test b/scripts/atomic/fallbacks/inc_and_test index 4e2068869f7e..cecc8322a21f 100755 --- a/scripts/atomic/fallbacks/inc_and_test +++ b/scripts/atomic/fallbacks/inc_and_test @@ -1,15 +1,15 @@ cat <<EOF /** - * ${atomic}_inc_and_test - increment and test + * ${arch}${atomic}_inc_and_test - increment and test * @v: pointer of type ${atomic}_t * * Atomically increments @v by 1 * and returns true if the result is zero, or false for all * other cases. */ -static inline bool -${atomic}_inc_and_test(${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_inc_and_test(${atomic}_t *v) { - return ${atomic}_inc_return(v) == 0; + return ${arch}${atomic}_inc_return(v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/inc_not_zero b/scripts/atomic/fallbacks/inc_not_zero index a7c45c8d107c..50f2d4d48279 100755 --- a/scripts/atomic/fallbacks/inc_not_zero +++ b/scripts/atomic/fallbacks/inc_not_zero @@ -1,14 +1,14 @@ cat <<EOF /** - * ${atomic}_inc_not_zero - increment unless the number is zero + * ${arch}${atomic}_inc_not_zero - increment unless the number is zero * @v: pointer of type ${atomic}_t * * Atomically increments @v by 1, if @v is non-zero. * Returns true if the increment was done. */ -static inline bool -${atomic}_inc_not_zero(${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_inc_not_zero(${atomic}_t *v) { - return ${atomic}_add_unless(v, 1, 0); + return ${arch}${atomic}_add_unless(v, 1, 0); } EOF diff --git a/scripts/atomic/fallbacks/inc_unless_negative b/scripts/atomic/fallbacks/inc_unless_negative index 0c266e71dbd4..87629e0d4a80 100755 --- a/scripts/atomic/fallbacks/inc_unless_negative +++ b/scripts/atomic/fallbacks/inc_unless_negative @@ -1,13 +1,13 @@ cat <<EOF -static inline bool -${atomic}_inc_unless_negative(${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_inc_unless_negative(${atomic}_t *v) { - ${int} c = ${atomic}_read(v); + ${int} c = ${arch}${atomic}_read(v); do { if (unlikely(c < 0)) return false; - } while (!${atomic}_try_cmpxchg(v, &c, c + 1)); + } while (!${arch}${atomic}_try_cmpxchg(v, &c, c + 1)); return true; } diff --git a/scripts/atomic/fallbacks/read_acquire b/scripts/atomic/fallbacks/read_acquire index 75863b5203f7..341a88dccaa7 100755 --- a/scripts/atomic/fallbacks/read_acquire +++ b/scripts/atomic/fallbacks/read_acquire @@ -1,6 +1,6 @@ cat <<EOF -static inline ${ret} -${atomic}_read_acquire(const ${atomic}_t *v) +static __always_inline ${ret} +${arch}${atomic}_read_acquire(const ${atomic}_t *v) { return smp_load_acquire(&(v)->counter); } diff --git a/scripts/atomic/fallbacks/release b/scripts/atomic/fallbacks/release index 3f628a3802d9..f8906d537c0f 100755 --- a/scripts/atomic/fallbacks/release +++ b/scripts/atomic/fallbacks/release @@ -1,8 +1,8 @@ cat <<EOF -static inline ${ret} -${atomic}_${pfx}${name}${sfx}_release(${params}) +static __always_inline ${ret} +${arch}${atomic}_${pfx}${name}${sfx}_release(${params}) { __atomic_release_fence(); - ${retstmt}${atomic}_${pfx}${name}${sfx}_relaxed(${args}); + ${retstmt}${arch}${atomic}_${pfx}${name}${sfx}_relaxed(${args}); } EOF diff --git a/scripts/atomic/fallbacks/set_release b/scripts/atomic/fallbacks/set_release index 45bb5e0cfc08..76068272d5f5 100755 --- a/scripts/atomic/fallbacks/set_release +++ b/scripts/atomic/fallbacks/set_release @@ -1,6 +1,6 @@ cat <<EOF -static inline void -${atomic}_set_release(${atomic}_t *v, ${int} i) +static __always_inline void +${arch}${atomic}_set_release(${atomic}_t *v, ${int} i) { smp_store_release(&(v)->counter, i); } diff --git a/scripts/atomic/fallbacks/sub_and_test b/scripts/atomic/fallbacks/sub_and_test index 289ef17a2d7a..c580f4c2136e 100755 --- a/scripts/atomic/fallbacks/sub_and_test +++ b/scripts/atomic/fallbacks/sub_and_test @@ -1,6 +1,6 @@ cat <<EOF /** - * ${atomic}_sub_and_test - subtract value from variable and test result + * ${arch}${atomic}_sub_and_test - subtract value from variable and test result * @i: integer value to subtract * @v: pointer of type ${atomic}_t * @@ -8,9 +8,9 @@ cat <<EOF * true if the result is zero, or false for all * other cases. */ -static inline bool -${atomic}_sub_and_test(${int} i, ${atomic}_t *v) +static __always_inline bool +${arch}${atomic}_sub_and_test(${int} i, ${atomic}_t *v) { - return ${atomic}_sub_return(i, v) == 0; + return ${arch}${atomic}_sub_return(i, v) == 0; } EOF diff --git a/scripts/atomic/fallbacks/try_cmpxchg b/scripts/atomic/fallbacks/try_cmpxchg index 4ed85e2f5378..06db0f738e45 100755 --- a/scripts/atomic/fallbacks/try_cmpxchg +++ b/scripts/atomic/fallbacks/try_cmpxchg @@ -1,9 +1,9 @@ cat <<EOF -static inline bool -${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) +static __always_inline bool +${arch}${atomic}_try_cmpxchg${order}(${atomic}_t *v, ${int} *old, ${int} new) { ${int} r, o = *old; - r = ${atomic}_cmpxchg${order}(v, o, new); + r = ${arch}${atomic}_cmpxchg${order}(v, o, new); if (unlikely(r != o)) *old = r; return likely(r == o); diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh index 1bd7c1707633..0fd1cf0c2b94 100755 --- a/scripts/atomic/gen-atomic-fallback.sh +++ b/scripts/atomic/gen-atomic-fallback.sh @@ -2,10 +2,11 @@ # SPDX-License-Identifier: GPL-2.0 ATOMICDIR=$(dirname $0) +ARCH=$2 . ${ATOMICDIR}/atomic-tbl.sh -#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...) +#gen_template_fallback(template, meta, pfx, name, sfx, order, arch, atomic, int, args...) gen_template_fallback() { local template="$1"; shift @@ -14,10 +15,11 @@ gen_template_fallback() local name="$1"; shift local sfx="$1"; shift local order="$1"; shift + local arch="$1"; shift local atomic="$1"; shift local int="$1"; shift - local atomicname="${atomic}_${pfx}${name}${sfx}${order}" + local atomicname="${arch}${atomic}_${pfx}${name}${sfx}${order}" local ret="$(gen_ret_type "${meta}" "${int}")" local retstmt="$(gen_ret_stmt "${meta}")" @@ -32,7 +34,7 @@ gen_template_fallback() fi } -#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...) +#gen_proto_fallback(meta, pfx, name, sfx, order, arch, atomic, int, args...) gen_proto_fallback() { local meta="$1"; shift @@ -56,16 +58,17 @@ cat << EOF EOF } -#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...) +#gen_proto_order_variants(meta, pfx, name, sfx, arch, atomic, int, args...) gen_proto_order_variants() { local meta="$1"; shift local pfx="$1"; shift local name="$1"; shift local sfx="$1"; shift - local atomic="$1" + local arch="$1" + local atomic="$2" - local basename="${atomic}_${pfx}${name}${sfx}" + local basename="${arch}${atomic}_${pfx}${name}${sfx}" local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")" @@ -94,7 +97,7 @@ gen_proto_order_variants() gen_basic_fallbacks "${basename}" if [ ! -z "${template}" ]; then - printf "#endif /* ${atomic}_${pfx}${name}${sfx} */\n\n" + printf "#endif /* ${arch}${atomic}_${pfx}${name}${sfx} */\n\n" gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@" gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@" gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@" @@ -149,20 +152,19 @@ cat << EOF #ifndef _LINUX_ATOMIC_FALLBACK_H #define _LINUX_ATOMIC_FALLBACK_H +#include <linux/compiler.h> + EOF -for xchg in "xchg" "cmpxchg" "cmpxchg64"; do +for xchg in "${ARCH}xchg" "${ARCH}cmpxchg" "${ARCH}cmpxchg64"; do gen_xchg_fallbacks "${xchg}" done grep '^[a-z]' "$1" | while read name meta args; do - gen_proto "${meta}" "${name}" "atomic" "int" ${args} + gen_proto "${meta}" "${name}" "${ARCH}" "atomic" "int" ${args} done cat <<EOF -#define atomic_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) -#define atomic_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) - #ifdef CONFIG_GENERIC_ATOMIC64 #include <asm-generic/atomic64.h> #endif @@ -170,12 +172,9 @@ cat <<EOF EOF grep '^[a-z]' "$1" | while read name meta args; do - gen_proto "${meta}" "${name}" "atomic64" "s64" ${args} + gen_proto "${meta}" "${name}" "${ARCH}" "atomic64" "s64" ${args} done cat <<EOF -#define atomic64_cond_read_acquire(v, c) smp_cond_load_acquire(&(v)->counter, (c)) -#define atomic64_cond_read_relaxed(v, c) smp_cond_load_relaxed(&(v)->counter, (c)) - #endif /* _LINUX_ATOMIC_FALLBACK_H */ EOF diff --git a/scripts/atomic/gen-atomic-instrumented.sh b/scripts/atomic/gen-atomic-instrumented.sh index e09812372b17..6afadf73da17 100755 --- a/scripts/atomic/gen-atomic-instrumented.sh +++ b/scripts/atomic/gen-atomic-instrumented.sh @@ -20,7 +20,7 @@ gen_param_check() # We don't write to constant parameters [ ${type#c} != ${type} ] && rw="read" - printf "\tkasan_check_${rw}(${name}, sizeof(*${name}));\n" + printf "\tinstrument_atomic_${rw}(${name}, sizeof(*${name}));\n" } #gen_param_check(arg...) @@ -84,7 +84,7 @@ gen_proto_order_variant() [ ! -z "${guard}" ] && printf "#if ${guard}\n" cat <<EOF -static inline ${ret} +static __always_inline ${ret} ${atomicname}(${params}) { ${checks} @@ -107,7 +107,7 @@ cat <<EOF #define ${xchg}(ptr, ...) \\ ({ \\ typeof(ptr) __ai_ptr = (ptr); \\ - kasan_check_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ + instrument_atomic_write(__ai_ptr, ${mult}sizeof(*__ai_ptr)); \\ arch_${xchg}(__ai_ptr, __VA_ARGS__); \\ }) EOF @@ -147,7 +147,8 @@ cat << EOF #define _ASM_GENERIC_ATOMIC_INSTRUMENTED_H #include <linux/build_bug.h> -#include <linux/kasan-checks.h> +#include <linux/compiler.h> +#include <linux/instrumented.h> EOF diff --git a/scripts/atomic/gen-atomic-long.sh b/scripts/atomic/gen-atomic-long.sh index c240a7231b2e..e318d3f92e53 100755 --- a/scripts/atomic/gen-atomic-long.sh +++ b/scripts/atomic/gen-atomic-long.sh @@ -46,7 +46,7 @@ gen_proto_order_variant() local retstmt="$(gen_ret_stmt "${meta}")" cat <<EOF -static inline ${ret} +static __always_inline ${ret} atomic_long_${name}(${params}) { ${retstmt}${atomic}_${name}(${argscast}); @@ -64,6 +64,7 @@ cat << EOF #ifndef _ASM_GENERIC_ATOMIC_LONG_H #define _ASM_GENERIC_ATOMIC_LONG_H +#include <linux/compiler.h> #include <asm/types.h> #ifdef CONFIG_64BIT diff --git a/scripts/atomic/gen-atomics.sh b/scripts/atomic/gen-atomics.sh index 000dc6437893..d29e159ef489 100644 --- a/scripts/atomic/gen-atomics.sh +++ b/scripts/atomic/gen-atomics.sh @@ -10,10 +10,11 @@ LINUXDIR=${ATOMICDIR}/../.. cat <<EOF | gen-atomic-instrumented.sh asm-generic/atomic-instrumented.h gen-atomic-long.sh asm-generic/atomic-long.h +gen-atomic-fallback.sh linux/atomic-arch-fallback.h arch_ gen-atomic-fallback.sh linux/atomic-fallback.h EOF -while read script header; do - /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} > ${LINUXDIR}/include/${header} +while read script header args; do + /bin/sh ${ATOMICDIR}/${script} ${ATOMICTBL} ${args} > ${LINUXDIR}/include/${header} HASH="$(sha1sum ${LINUXDIR}/include/${header})" HASH="${HASH%% *}" printf "// %s\n" "${HASH}" >> ${LINUXDIR}/include/${header} diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 197436b20288..4c820607540b 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -2407,7 +2407,7 @@ sub process { if ($rawline=~/^\+\+\+\s+(\S+)/) { $setup_docs = 0; - if ($1 =~ m@Documentation/admin-guide/kernel-parameters.rst$@) { + if ($1 =~ m@Documentation/admin-guide/kernel-parameters.txt$@) { $setup_docs = 1; } #next; @@ -5945,6 +5945,14 @@ sub process { } } +# check for data_race without a comment. + if ($line =~ /\bdata_race\s*\(/) { + if (!ctx_has_comment($first_line, $linenr)) { + WARN("DATA_RACE", + "data_race without comment\n" . $herecurr); + } + } + # check for smp_read_barrier_depends and read_barrier_depends if (!$file && $line =~ /\b(smp_|)read_barrier_depends\s*\(/) { WARN("READ_BARRIER_DEPENDS", @@ -6388,7 +6396,7 @@ sub process { if (!grep(/$name/, @setup_docs)) { CHK("UNDOCUMENTED_SETUP", - "__setup appears un-documented -- check Documentation/admin-guide/kernel-parameters.rst\n" . $herecurr); + "__setup appears un-documented -- check Documentation/admin-guide/kernel-parameters.txt\n" . $herecurr); } } diff --git a/scripts/dtc/Makefile b/scripts/dtc/Makefile index ef85f8b7d4a7..0b44917f981c 100644 --- a/scripts/dtc/Makefile +++ b/scripts/dtc/Makefile @@ -20,6 +20,9 @@ endif HOST_EXTRACFLAGS += -DNO_YAML else dtc-objs += yamltree.o +# To include <yaml.h> installed in a non-default path +HOSTCFLAGS_yamltree.o := $(shell pkg-config --cflags yaml-0.1) +# To link libyaml installed in a non-default path HOSTLDLIBS_dtc := $(shell pkg-config yaml-0.1 --libs) endif diff --git a/scripts/spelling.txt b/scripts/spelling.txt index d9cd24cf0d40..c45e9afaab2d 100644 --- a/scripts/spelling.txt +++ b/scripts/spelling.txt @@ -59,6 +59,7 @@ actualy||actually acumulating||accumulating acumulative||accumulative acumulator||accumulator +acutally||actually adapater||adapter addional||additional additionaly||additionally @@ -249,6 +250,7 @@ calescing||coalescing calle||called callibration||calibration callled||called +callser||caller calucate||calculate calulate||calculate cancelation||cancellation @@ -671,6 +673,7 @@ hanlde||handle hanled||handled happend||happened harware||hardware +havind||having heirarchically||hierarchically helpfull||helpful hexdecimal||hexadecimal @@ -845,6 +848,7 @@ logile||logfile loobpack||loopback loosing||losing losted||lost +maangement||management machinary||machinery maibox||mailbox maintainance||maintenance @@ -905,6 +909,7 @@ modfiy||modify modulues||modules momery||memory memomry||memory +monitring||monitoring monochorome||monochrome monochromo||monochrome monocrome||monochrome @@ -1010,6 +1015,7 @@ partiton||partition pased||passed passin||passing pathes||paths +pattrns||patterns pecularities||peculiarities peformance||performance peforming||performing @@ -1256,6 +1262,7 @@ shoule||should shrinked||shrunk siginificantly||significantly signabl||signal +significanly||significantly similary||similarly similiar||similar simlar||similar @@ -1371,6 +1378,7 @@ thead||thread therfore||therefore thier||their threds||threads +threee||three threshhold||threshold thresold||threshold throught||through @@ -1410,6 +1418,7 @@ tyep||type udpate||update uesd||used uknown||unknown +usccess||success usupported||unsupported uncommited||uncommitted unconditionaly||unconditionally diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index 800fb3bba418..c1583d98c5e5 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -419,7 +419,8 @@ int ima_file_mprotect(struct vm_area_struct *vma, unsigned long prot) int pcr; /* Is mprotect making an mmap'ed file executable? */ - if (!vma->vm_file || !(prot & PROT_EXEC) || (vma->vm_flags & VM_EXEC)) + if (!(ima_policy_flag & IMA_APPRAISE) || !vma->vm_file || + !(prot & PROT_EXEC) || (vma->vm_flags & VM_EXEC)) return 0; security_task_getsecid(current, &secid); diff --git a/security/keys/Kconfig b/security/keys/Kconfig index 8153ea01d7bb..83bc23409164 100644 --- a/security/keys/Kconfig +++ b/security/keys/Kconfig @@ -114,3 +114,12 @@ config KEY_DH_OPERATIONS in the kernel. If you are unsure as to whether this is required, answer N. + +config KEY_NOTIFICATIONS + bool "Provide key/keyring change notifications" + depends on KEYS && WATCH_QUEUE + help + This option provides support for getting change notifications on keys + and keyrings on which the caller has View permission. This makes use + of the /dev/watch_queue misc device to handle the notification + buffer and provides KEYCTL_WATCH_KEY to enable/disable watches. diff --git a/security/keys/compat.c b/security/keys/compat.c index b975f8f11124..6ee9d8f6a4a5 100644 --- a/security/keys/compat.c +++ b/security/keys/compat.c @@ -156,6 +156,9 @@ COMPAT_SYSCALL_DEFINE5(keyctl, u32, option, case KEYCTL_CAPABILITIES: return keyctl_capabilities(compat_ptr(arg2), arg3); + case KEYCTL_WATCH_KEY: + return keyctl_watch_key(arg2, arg3, arg4); + default: return -EOPNOTSUPP; } diff --git a/security/keys/gc.c b/security/keys/gc.c index 671dd730ecfc..3c90807476eb 100644 --- a/security/keys/gc.c +++ b/security/keys/gc.c @@ -131,6 +131,11 @@ static noinline void key_gc_unused_keys(struct list_head *keys) kdebug("- %u", key->serial); key_check(key); +#ifdef CONFIG_KEY_NOTIFICATIONS + remove_watch_list(key->watchers, key->serial); + key->watchers = NULL; +#endif + /* Throw away the key data if the key is instantiated */ if (state == KEY_IS_POSITIVE && key->type->destroy) key->type->destroy(key); diff --git a/security/keys/internal.h b/security/keys/internal.h index 153d35c20d3d..338a526cbfa5 100644 --- a/security/keys/internal.h +++ b/security/keys/internal.h @@ -15,6 +15,7 @@ #include <linux/task_work.h> #include <linux/keyctl.h> #include <linux/refcount.h> +#include <linux/watch_queue.h> #include <linux/compat.h> #include <linux/mm.h> #include <linux/vmalloc.h> @@ -99,7 +100,8 @@ extern int __key_link_begin(struct key *keyring, const struct keyring_index_key *index_key, struct assoc_array_edit **_edit); extern int __key_link_check_live_key(struct key *keyring, struct key *key); -extern void __key_link(struct key *key, struct assoc_array_edit **_edit); +extern void __key_link(struct key *keyring, struct key *key, + struct assoc_array_edit **_edit); extern void __key_link_end(struct key *keyring, const struct keyring_index_key *index_key, struct assoc_array_edit *edit); @@ -165,7 +167,6 @@ extern bool lookup_user_key_possessed(const struct key *key, const struct key_match_data *match_data); #define KEY_LOOKUP_CREATE 0x01 #define KEY_LOOKUP_PARTIAL 0x02 -#define KEY_LOOKUP_FOR_UNLINK 0x04 extern long join_session_keyring(const char *name); extern void key_change_session_keyring(struct callback_head *twork); @@ -181,14 +182,32 @@ extern void key_gc_keytype(struct key_type *ktype); extern int key_task_permission(const key_ref_t key_ref, const struct cred *cred, - key_perm_t perm); + enum key_need_perm need_perm); + +static inline void notify_key(struct key *key, + enum key_notification_subtype subtype, u32 aux) +{ +#ifdef CONFIG_KEY_NOTIFICATIONS + struct key_notification n = { + .watch.type = WATCH_TYPE_KEY_NOTIFY, + .watch.subtype = subtype, + .watch.info = watch_sizeof(n), + .key_id = key_serial(key), + .aux = aux, + }; + + post_watch_notification(key->watchers, &n.watch, current_cred(), + n.key_id); +#endif +} /* * Check to see whether permission is granted to use a key in the desired way. */ -static inline int key_permission(const key_ref_t key_ref, unsigned perm) +static inline int key_permission(const key_ref_t key_ref, + enum key_need_perm need_perm) { - return key_task_permission(key_ref, current_cred(), perm); + return key_task_permission(key_ref, current_cred(), need_perm); } extern struct key_type key_type_request_key_auth; @@ -333,6 +352,15 @@ static inline long keyctl_pkey_e_d_s(int op, extern long keyctl_capabilities(unsigned char __user *_buffer, size_t buflen); +#ifdef CONFIG_KEY_NOTIFICATIONS +extern long keyctl_watch_key(key_serial_t, int, int); +#else +static inline long keyctl_watch_key(key_serial_t key_id, int watch_fd, int watch_id) +{ + return -EOPNOTSUPP; +} +#endif + /* * Debugging key validation */ diff --git a/security/keys/key.c b/security/keys/key.c index e959b3c96b48..e282c6179b21 100644 --- a/security/keys/key.c +++ b/security/keys/key.c @@ -444,6 +444,7 @@ static int __key_instantiate_and_link(struct key *key, /* mark the key as being instantiated */ atomic_inc(&key->user->nikeys); mark_key_instantiated(key, 0); + notify_key(key, NOTIFY_KEY_INSTANTIATED, 0); if (test_and_clear_bit(KEY_FLAG_USER_CONSTRUCT, &key->flags)) awaken = 1; @@ -453,7 +454,7 @@ static int __key_instantiate_and_link(struct key *key, if (test_bit(KEY_FLAG_KEEP, &keyring->flags)) set_bit(KEY_FLAG_KEEP, &key->flags); - __key_link(key, _edit); + __key_link(keyring, key, _edit); } /* disable the authorisation key */ @@ -601,6 +602,7 @@ int key_reject_and_link(struct key *key, /* mark the key as being negatively instantiated */ atomic_inc(&key->user->nikeys); mark_key_instantiated(key, -error); + notify_key(key, NOTIFY_KEY_INSTANTIATED, -error); key->expiry = ktime_get_real_seconds() + timeout; key_schedule_gc(key->expiry + key_gc_delay); @@ -611,7 +613,7 @@ int key_reject_and_link(struct key *key, /* and link it into the destination keyring */ if (keyring && link_ret == 0) - __key_link(key, &edit); + __key_link(keyring, key, &edit); /* disable the authorisation key */ if (authkey) @@ -764,9 +766,11 @@ static inline key_ref_t __key_update(key_ref_t key_ref, down_write(&key->sem); ret = key->type->update(key, prep); - if (ret == 0) + if (ret == 0) { /* Updating a negative key positively instantiates it */ mark_key_instantiated(key, 0); + notify_key(key, NOTIFY_KEY_UPDATED, 0); + } up_write(&key->sem); @@ -1023,9 +1027,11 @@ int key_update(key_ref_t key_ref, const void *payload, size_t plen) down_write(&key->sem); ret = key->type->update(key, &prep); - if (ret == 0) + if (ret == 0) { /* Updating a negative key positively instantiates it */ mark_key_instantiated(key, 0); + notify_key(key, NOTIFY_KEY_UPDATED, 0); + } up_write(&key->sem); @@ -1057,15 +1063,17 @@ void key_revoke(struct key *key) * instantiated */ down_write_nested(&key->sem, 1); - if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags) && - key->type->revoke) - key->type->revoke(key); - - /* set the death time to no more than the expiry time */ - time = ktime_get_real_seconds(); - if (key->revoked_at == 0 || key->revoked_at > time) { - key->revoked_at = time; - key_schedule_gc(key->revoked_at + key_gc_delay); + if (!test_and_set_bit(KEY_FLAG_REVOKED, &key->flags)) { + notify_key(key, NOTIFY_KEY_REVOKED, 0); + if (key->type->revoke) + key->type->revoke(key); + + /* set the death time to no more than the expiry time */ + time = ktime_get_real_seconds(); + if (key->revoked_at == 0 || key->revoked_at > time) { + key->revoked_at = time; + key_schedule_gc(key->revoked_at + key_gc_delay); + } } up_write(&key->sem); @@ -1087,8 +1095,10 @@ void key_invalidate(struct key *key) if (!test_bit(KEY_FLAG_INVALIDATED, &key->flags)) { down_write_nested(&key->sem, 1); - if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) + if (!test_and_set_bit(KEY_FLAG_INVALIDATED, &key->flags)) { + notify_key(key, NOTIFY_KEY_INVALIDATED, 0); key_schedule_gc_links(); + } up_write(&key->sem); } } diff --git a/security/keys/keyctl.c b/security/keys/keyctl.c index e5ef20a0d05e..9febd37a168f 100644 --- a/security/keys/keyctl.c +++ b/security/keys/keyctl.c @@ -37,7 +37,9 @@ static const unsigned char keyrings_capabilities[2] = { KEYCTL_CAPS0_MOVE ), [1] = (KEYCTL_CAPS1_NS_KEYRING_NAME | - KEYCTL_CAPS1_NS_KEY_TAG), + KEYCTL_CAPS1_NS_KEY_TAG | + (IS_ENABLED(CONFIG_KEY_NOTIFICATIONS) ? KEYCTL_CAPS1_NOTIFICATIONS : 0) + ), }; static int key_get_type_from_user(char *type, @@ -429,7 +431,7 @@ long keyctl_invalidate_key(key_serial_t id) /* Root is permitted to invalidate certain special keys */ if (capable(CAP_SYS_ADMIN)) { - key_ref = lookup_user_key(id, 0, 0); + key_ref = lookup_user_key(id, 0, KEY_SYSADMIN_OVERRIDE); if (IS_ERR(key_ref)) goto error; if (test_bit(KEY_FLAG_ROOT_CAN_INVAL, @@ -474,7 +476,8 @@ long keyctl_keyring_clear(key_serial_t ringid) /* Root is permitted to invalidate certain special keyrings */ if (capable(CAP_SYS_ADMIN)) { - keyring_ref = lookup_user_key(ringid, 0, 0); + keyring_ref = lookup_user_key(ringid, 0, + KEY_SYSADMIN_OVERRIDE); if (IS_ERR(keyring_ref)) goto error; if (test_bit(KEY_FLAG_ROOT_CAN_CLEAR, @@ -558,7 +561,7 @@ long keyctl_keyring_unlink(key_serial_t id, key_serial_t ringid) goto error; } - key_ref = lookup_user_key(id, KEY_LOOKUP_FOR_UNLINK, 0); + key_ref = lookup_user_key(id, KEY_LOOKUP_PARTIAL, KEY_NEED_UNLINK); if (IS_ERR(key_ref)) { ret = PTR_ERR(key_ref); goto error2; @@ -658,7 +661,7 @@ long keyctl_describe_key(key_serial_t keyid, key_put(instkey); key_ref = lookup_user_key(keyid, KEY_LOOKUP_PARTIAL, - 0); + KEY_AUTHTOKEN_OVERRIDE); if (!IS_ERR(key_ref)) goto okay; } @@ -828,7 +831,7 @@ long keyctl_read_key(key_serial_t keyid, char __user *buffer, size_t buflen) size_t key_data_len; /* find the key first */ - key_ref = lookup_user_key(keyid, 0, 0); + key_ref = lookup_user_key(keyid, 0, KEY_DEFER_PERM_CHECK); if (IS_ERR(key_ref)) { ret = -ENOKEY; goto out; @@ -1036,6 +1039,7 @@ long keyctl_chown_key(key_serial_t id, uid_t user, gid_t group) if (group != (gid_t) -1) key->gid = gid; + notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; error_put: @@ -1086,6 +1090,7 @@ long keyctl_setperm_key(key_serial_t id, key_perm_t perm) /* if we're not the sysadmin, we can only change a key that we own */ if (capable(CAP_SYS_ADMIN) || uid_eq(key->uid, current_fsuid())) { key->perm = perm; + notify_key(key, NOTIFY_KEY_SETATTR, 0); ret = 0; } @@ -1461,7 +1466,7 @@ long keyctl_set_timeout(key_serial_t id, unsigned timeout) key_put(instkey); key_ref = lookup_user_key(id, KEY_LOOKUP_PARTIAL, - 0); + KEY_AUTHTOKEN_OVERRIDE); if (!IS_ERR(key_ref)) goto okay; } @@ -1474,10 +1479,12 @@ long keyctl_set_timeout(key_serial_t id, unsigned timeout) okay: key = key_ref_to_ptr(key_ref); ret = 0; - if (test_bit(KEY_FLAG_KEEP, &key->flags)) + if (test_bit(KEY_FLAG_KEEP, &key->flags)) { ret = -EPERM; - else + } else { key_set_timeout(key, timeout); + notify_key(key, NOTIFY_KEY_SETATTR, 0); + } key_put(key); error: @@ -1567,7 +1574,8 @@ long keyctl_get_security(key_serial_t keyid, return PTR_ERR(instkey); key_put(instkey); - key_ref = lookup_user_key(keyid, KEY_LOOKUP_PARTIAL, 0); + key_ref = lookup_user_key(keyid, KEY_LOOKUP_PARTIAL, + KEY_AUTHTOKEN_OVERRIDE); if (IS_ERR(key_ref)) return PTR_ERR(key_ref); } @@ -1751,6 +1759,90 @@ error: return ret; } +#ifdef CONFIG_KEY_NOTIFICATIONS +/* + * Watch for changes to a key. + * + * The caller must have View permission to watch a key or keyring. + */ +long keyctl_watch_key(key_serial_t id, int watch_queue_fd, int watch_id) +{ + struct watch_queue *wqueue; + struct watch_list *wlist = NULL; + struct watch *watch = NULL; + struct key *key; + key_ref_t key_ref; + long ret; + + if (watch_id < -1 || watch_id > 0xff) + return -EINVAL; + + key_ref = lookup_user_key(id, KEY_LOOKUP_CREATE, KEY_NEED_VIEW); + if (IS_ERR(key_ref)) + return PTR_ERR(key_ref); + key = key_ref_to_ptr(key_ref); + + wqueue = get_watch_queue(watch_queue_fd); + if (IS_ERR(wqueue)) { + ret = PTR_ERR(wqueue); + goto err_key; + } + + if (watch_id >= 0) { + ret = -ENOMEM; + if (!key->watchers) { + wlist = kzalloc(sizeof(*wlist), GFP_KERNEL); + if (!wlist) + goto err_wqueue; + init_watch_list(wlist, NULL); + } + + watch = kzalloc(sizeof(*watch), GFP_KERNEL); + if (!watch) + goto err_wlist; + + init_watch(watch, wqueue); + watch->id = key->serial; + watch->info_id = (u32)watch_id << WATCH_INFO_ID__SHIFT; + + ret = security_watch_key(key); + if (ret < 0) + goto err_watch; + + down_write(&key->sem); + if (!key->watchers) { + key->watchers = wlist; + wlist = NULL; + } + + ret = add_watch_to_object(watch, key->watchers); + up_write(&key->sem); + + if (ret == 0) + watch = NULL; + } else { + ret = -EBADSLT; + if (key->watchers) { + down_write(&key->sem); + ret = remove_watch_from_object(key->watchers, + wqueue, key_serial(key), + false); + up_write(&key->sem); + } + } + +err_watch: + kfree(watch); +err_wlist: + kfree(wlist); +err_wqueue: + put_watch_queue(wqueue); +err_key: + key_put(key); + return ret; +} +#endif /* CONFIG_KEY_NOTIFICATIONS */ + /* * Get keyrings subsystem capabilities. */ @@ -1920,6 +2012,9 @@ SYSCALL_DEFINE5(keyctl, int, option, unsigned long, arg2, unsigned long, arg3, case KEYCTL_CAPABILITIES: return keyctl_capabilities((unsigned char __user *)arg2, (size_t)arg3); + case KEYCTL_WATCH_KEY: + return keyctl_watch_key((key_serial_t)arg2, (int)arg3, (int)arg4); + default: return -EOPNOTSUPP; } diff --git a/security/keys/keyring.c b/security/keys/keyring.c index 5ca620d31cd3..14abfe765b7e 100644 --- a/security/keys/keyring.c +++ b/security/keys/keyring.c @@ -1056,12 +1056,14 @@ int keyring_restrict(key_ref_t keyring_ref, const char *type, down_write(&keyring->sem); down_write(&keyring_serialise_restrict_sem); - if (keyring->restrict_link) + if (keyring->restrict_link) { ret = -EEXIST; - else if (keyring_detect_restriction_cycle(keyring, restrict_link)) + } else if (keyring_detect_restriction_cycle(keyring, restrict_link)) { ret = -EDEADLK; - else + } else { keyring->restrict_link = restrict_link; + notify_key(keyring, NOTIFY_KEY_SETATTR, 0); + } up_write(&keyring_serialise_restrict_sem); up_write(&keyring->sem); @@ -1362,12 +1364,14 @@ int __key_link_check_live_key(struct key *keyring, struct key *key) * holds at most one link to any given key of a particular type+description * combination. */ -void __key_link(struct key *key, struct assoc_array_edit **_edit) +void __key_link(struct key *keyring, struct key *key, + struct assoc_array_edit **_edit) { __key_get(key); assoc_array_insert_set_object(*_edit, keyring_key_to_ptr(key)); assoc_array_apply_edit(*_edit); *_edit = NULL; + notify_key(keyring, NOTIFY_KEY_LINKED, key_serial(key)); } /* @@ -1451,7 +1455,7 @@ int key_link(struct key *keyring, struct key *key) if (ret == 0) ret = __key_link_check_live_key(keyring, key); if (ret == 0) - __key_link(key, &edit); + __key_link(keyring, key, &edit); error_end: __key_link_end(keyring, &key->index_key, edit); @@ -1483,7 +1487,7 @@ static int __key_unlink_begin(struct key *keyring, struct key *key, struct assoc_array_edit *edit; BUG_ON(*_edit != NULL); - + edit = assoc_array_delete(&keyring->keys, &keyring_assoc_array_ops, &key->index_key); if (IS_ERR(edit)) @@ -1503,6 +1507,7 @@ static void __key_unlink(struct key *keyring, struct key *key, struct assoc_array_edit **_edit) { assoc_array_apply_edit(*_edit); + notify_key(keyring, NOTIFY_KEY_UNLINKED, key_serial(key)); *_edit = NULL; key_payload_reserve(keyring, keyring->datalen - KEYQUOTA_LINK_BYTES); } @@ -1621,7 +1626,7 @@ int key_move(struct key *key, goto error; __key_unlink(from_keyring, key, &from_edit); - __key_link(key, &to_edit); + __key_link(to_keyring, key, &to_edit); error: __key_link_end(to_keyring, &key->index_key, to_edit); __key_unlink_end(from_keyring, key, from_edit); @@ -1655,6 +1660,7 @@ int keyring_clear(struct key *keyring) } else { if (edit) assoc_array_apply_edit(edit); + notify_key(keyring, NOTIFY_KEY_CLEARED, 0); key_payload_reserve(keyring, 0); ret = 0; } diff --git a/security/keys/permission.c b/security/keys/permission.c index 085f907b64ac..4a61f804e80f 100644 --- a/security/keys/permission.c +++ b/security/keys/permission.c @@ -13,7 +13,7 @@ * key_task_permission - Check a key can be used * @key_ref: The key to check. * @cred: The credentials to use. - * @perm: The permissions to check for. + * @need_perm: The permission required. * * Check to see whether permission is granted to use a key in the desired way, * but permit the security modules to override. @@ -24,12 +24,30 @@ * permissions bits or the LSM check. */ int key_task_permission(const key_ref_t key_ref, const struct cred *cred, - unsigned perm) + enum key_need_perm need_perm) { struct key *key; - key_perm_t kperm; + key_perm_t kperm, mask; int ret; + switch (need_perm) { + default: + WARN_ON(1); + return -EACCES; + case KEY_NEED_UNLINK: + case KEY_SYSADMIN_OVERRIDE: + case KEY_AUTHTOKEN_OVERRIDE: + case KEY_DEFER_PERM_CHECK: + goto lsm; + + case KEY_NEED_VIEW: mask = KEY_OTH_VIEW; break; + case KEY_NEED_READ: mask = KEY_OTH_READ; break; + case KEY_NEED_WRITE: mask = KEY_OTH_WRITE; break; + case KEY_NEED_SEARCH: mask = KEY_OTH_SEARCH; break; + case KEY_NEED_LINK: mask = KEY_OTH_LINK; break; + case KEY_NEED_SETATTR: mask = KEY_OTH_SETATTR; break; + } + key = key_ref_to_ptr(key_ref); /* use the second 8-bits of permissions for keys the caller owns */ @@ -64,13 +82,12 @@ use_these_perms: if (is_key_possessed(key_ref)) kperm |= key->perm >> 24; - kperm = kperm & perm & KEY_NEED_ALL; - - if (kperm != perm) + if ((kperm & mask) != mask) return -EACCES; /* let LSM be the final arbiter */ - return security_key_permission(key_ref, cred, perm); +lsm: + return security_key_permission(key_ref, cred, need_perm); } EXPORT_SYMBOL(key_task_permission); diff --git a/security/keys/process_keys.c b/security/keys/process_keys.c index 09541de31f2f..7e0232db1707 100644 --- a/security/keys/process_keys.c +++ b/security/keys/process_keys.c @@ -609,7 +609,7 @@ bool lookup_user_key_possessed(const struct key *key, * returned key reference. */ key_ref_t lookup_user_key(key_serial_t id, unsigned long lflags, - key_perm_t perm) + enum key_need_perm need_perm) { struct keyring_search_context ctx = { .match_data.cmp = lookup_user_key_possessed, @@ -773,35 +773,33 @@ try_again: /* unlink does not use the nominated key in any way, so can skip all * the permission checks as it is only concerned with the keyring */ - if (lflags & KEY_LOOKUP_FOR_UNLINK) { - ret = 0; - goto error; - } - - if (!(lflags & KEY_LOOKUP_PARTIAL)) { - ret = wait_for_key_construction(key, true); - switch (ret) { - case -ERESTARTSYS: - goto invalid_key; - default: - if (perm) + if (need_perm != KEY_NEED_UNLINK) { + if (!(lflags & KEY_LOOKUP_PARTIAL)) { + ret = wait_for_key_construction(key, true); + switch (ret) { + case -ERESTARTSYS: + goto invalid_key; + default: + if (need_perm != KEY_AUTHTOKEN_OVERRIDE && + need_perm != KEY_DEFER_PERM_CHECK) + goto invalid_key; + case 0: + break; + } + } else if (need_perm != KEY_DEFER_PERM_CHECK) { + ret = key_validate(key); + if (ret < 0) goto invalid_key; - case 0: - break; } - } else if (perm) { - ret = key_validate(key); - if (ret < 0) + + ret = -EIO; + if (!(lflags & KEY_LOOKUP_PARTIAL) && + key_read_state(key) == KEY_IS_UNINSTANTIATED) goto invalid_key; } - ret = -EIO; - if (!(lflags & KEY_LOOKUP_PARTIAL) && - key_read_state(key) == KEY_IS_UNINSTANTIATED) - goto invalid_key; - /* check the permissions */ - ret = key_task_permission(key_ref, ctx.cred, perm); + ret = key_task_permission(key_ref, ctx.cred, need_perm); if (ret < 0) goto invalid_key; diff --git a/security/keys/request_key.c b/security/keys/request_key.c index 957b9e3e1492..e1b9f1a80676 100644 --- a/security/keys/request_key.c +++ b/security/keys/request_key.c @@ -418,7 +418,7 @@ static int construct_alloc_key(struct keyring_search_context *ctx, goto key_already_present; if (dest_keyring) - __key_link(key, &edit); + __key_link(dest_keyring, key, &edit); mutex_unlock(&key_construction_mutex); if (dest_keyring) @@ -437,7 +437,7 @@ key_already_present: if (dest_keyring) { ret = __key_link_check_live_key(dest_keyring, key); if (ret == 0) - __key_link(key, &edit); + __key_link(dest_keyring, key, &edit); __key_link_end(dest_keyring, &ctx->index_key, edit); if (ret < 0) goto link_check_failed; diff --git a/security/security.c b/security/security.c index e0290b7e6a08..2a652e59c413 100644 --- a/security/security.c +++ b/security/security.c @@ -2030,6 +2030,22 @@ int security_inode_getsecctx(struct inode *inode, void **ctx, u32 *ctxlen) } EXPORT_SYMBOL(security_inode_getsecctx); +#ifdef CONFIG_WATCH_QUEUE +int security_post_notification(const struct cred *w_cred, + const struct cred *cred, + struct watch_notification *n) +{ + return call_int_hook(post_notification, 0, w_cred, cred, n); +} +#endif /* CONFIG_WATCH_QUEUE */ + +#ifdef CONFIG_KEY_NOTIFICATIONS +int security_watch_key(struct key *key) +{ + return call_int_hook(watch_key, 0, key); +} +#endif + #ifdef CONFIG_SECURITY_NETWORK int security_unix_stream_connect(struct sock *sock, struct sock *other, struct sock *newsk) @@ -2405,10 +2421,10 @@ void security_key_free(struct key *key) call_void_hook(key_free, key); } -int security_key_permission(key_ref_t key_ref, - const struct cred *cred, unsigned perm) +int security_key_permission(key_ref_t key_ref, const struct cred *cred, + enum key_need_perm need_perm) { - return call_int_hook(key_permission, 0, key_ref, cred, perm); + return call_int_hook(key_permission, 0, key_ref, cred, need_perm); } int security_key_getsecurity(struct key *key, char **_buffer) diff --git a/security/selinux/hooks.c b/security/selinux/hooks.c index 7e954b555be6..efa6108b1ce9 100644 --- a/security/selinux/hooks.c +++ b/security/selinux/hooks.c @@ -6559,20 +6559,43 @@ static void selinux_key_free(struct key *k) static int selinux_key_permission(key_ref_t key_ref, const struct cred *cred, - unsigned perm) + enum key_need_perm need_perm) { struct key *key; struct key_security_struct *ksec; - u32 sid; + u32 perm, sid; - /* if no specific permissions are requested, we skip the - permission check. No serious, additional covert channels - appear to be created. */ - if (perm == 0) + switch (need_perm) { + case KEY_NEED_VIEW: + perm = KEY__VIEW; + break; + case KEY_NEED_READ: + perm = KEY__READ; + break; + case KEY_NEED_WRITE: + perm = KEY__WRITE; + break; + case KEY_NEED_SEARCH: + perm = KEY__SEARCH; + break; + case KEY_NEED_LINK: + perm = KEY__LINK; + break; + case KEY_NEED_SETATTR: + perm = KEY__SETATTR; + break; + case KEY_NEED_UNLINK: + case KEY_SYSADMIN_OVERRIDE: + case KEY_AUTHTOKEN_OVERRIDE: + case KEY_DEFER_PERM_CHECK: return 0; + default: + WARN_ON(1); + return -EPERM; - sid = cred_sid(cred); + } + sid = cred_sid(cred); key = key_ref_to_ptr(key_ref); ksec = key->security; @@ -6594,6 +6617,17 @@ static int selinux_key_getsecurity(struct key *key, char **_buffer) *_buffer = context; return rc; } + +#ifdef CONFIG_KEY_NOTIFICATIONS +static int selinux_watch_key(struct key *key) +{ + struct key_security_struct *ksec = key->security; + u32 sid = current_sid(); + + return avc_has_perm(&selinux_state, + sid, ksec->sid, SECCLASS_KEY, KEY__VIEW, NULL); +} +#endif #endif #ifdef CONFIG_SECURITY_INFINIBAND @@ -7109,6 +7143,9 @@ static struct security_hook_list selinux_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(key_free, selinux_key_free), LSM_HOOK_INIT(key_permission, selinux_key_permission), LSM_HOOK_INIT(key_getsecurity, selinux_key_getsecurity), +#ifdef CONFIG_KEY_NOTIFICATIONS + LSM_HOOK_INIT(watch_key, selinux_watch_key), +#endif #endif #ifdef CONFIG_AUDIT diff --git a/security/smack/smack_lsm.c b/security/smack/smack_lsm.c index cd44b79bf1f5..8ffbf951b7ed 100644 --- a/security/smack/smack_lsm.c +++ b/security/smack/smack_lsm.c @@ -41,6 +41,7 @@ #include <linux/parser.h> #include <linux/fs_context.h> #include <linux/fs_parser.h> +#include <linux/watch_queue.h> #include "smack.h" #define TRANS_TRUE "TRUE" @@ -4213,13 +4214,14 @@ static void smack_key_free(struct key *key) * smack_key_permission - Smack access on a key * @key_ref: gets to the object * @cred: the credentials to use - * @perm: requested key permissions + * @need_perm: requested key permission * * Return 0 if the task has read and write to the object, * an error code otherwise */ static int smack_key_permission(key_ref_t key_ref, - const struct cred *cred, unsigned perm) + const struct cred *cred, + enum key_need_perm need_perm) { struct key *keyp; struct smk_audit_info ad; @@ -4230,8 +4232,26 @@ static int smack_key_permission(key_ref_t key_ref, /* * Validate requested permissions */ - if (perm & ~KEY_NEED_ALL) + switch (need_perm) { + case KEY_NEED_READ: + case KEY_NEED_SEARCH: + case KEY_NEED_VIEW: + request |= MAY_READ; + break; + case KEY_NEED_WRITE: + case KEY_NEED_LINK: + case KEY_NEED_SETATTR: + request |= MAY_WRITE; + break; + case KEY_NEED_UNSPECIFIED: + case KEY_NEED_UNLINK: + case KEY_SYSADMIN_OVERRIDE: + case KEY_AUTHTOKEN_OVERRIDE: + case KEY_DEFER_PERM_CHECK: + return 0; + default: return -EINVAL; + } keyp = key_ref_to_ptr(key_ref); if (keyp == NULL) @@ -4248,7 +4268,7 @@ static int smack_key_permission(key_ref_t key_ref, if (tkp == NULL) return -EACCES; - if (smack_privileged_cred(CAP_MAC_OVERRIDE, cred)) + if (smack_privileged(CAP_MAC_OVERRIDE)) return 0; #ifdef CONFIG_AUDIT @@ -4256,10 +4276,6 @@ static int smack_key_permission(key_ref_t key_ref, ad.a.u.key_struct.key = keyp->serial; ad.a.u.key_struct.key_desc = keyp->description; #endif - if (perm & (KEY_NEED_READ | KEY_NEED_SEARCH | KEY_NEED_VIEW)) - request |= MAY_READ; - if (perm & (KEY_NEED_WRITE | KEY_NEED_LINK | KEY_NEED_SETATTR)) - request |= MAY_WRITE; rc = smk_access(tkp, keyp->security, request, &ad); rc = smk_bu_note("key access", tkp, keyp->security, request, rc); return rc; @@ -4294,8 +4310,81 @@ static int smack_key_getsecurity(struct key *key, char **_buffer) return length; } + +#ifdef CONFIG_KEY_NOTIFICATIONS +/** + * smack_watch_key - Smack access to watch a key for notifications. + * @key: The key to be watched + * + * Return 0 if the @watch->cred has permission to read from the key object and + * an error otherwise. + */ +static int smack_watch_key(struct key *key) +{ + struct smk_audit_info ad; + struct smack_known *tkp = smk_of_current(); + int rc; + + if (key == NULL) + return -EINVAL; + /* + * If the key hasn't been initialized give it access so that + * it may do so. + */ + if (key->security == NULL) + return 0; + /* + * This should not occur + */ + if (tkp == NULL) + return -EACCES; + + if (smack_privileged_cred(CAP_MAC_OVERRIDE, current_cred())) + return 0; + +#ifdef CONFIG_AUDIT + smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_KEY); + ad.a.u.key_struct.key = key->serial; + ad.a.u.key_struct.key_desc = key->description; +#endif + rc = smk_access(tkp, key->security, MAY_READ, &ad); + rc = smk_bu_note("key watch", tkp, key->security, MAY_READ, rc); + return rc; +} +#endif /* CONFIG_KEY_NOTIFICATIONS */ #endif /* CONFIG_KEYS */ +#ifdef CONFIG_WATCH_QUEUE +/** + * smack_post_notification - Smack access to post a notification to a queue + * @w_cred: The credentials of the watcher. + * @cred: The credentials of the event source (may be NULL). + * @n: The notification message to be posted. + */ +static int smack_post_notification(const struct cred *w_cred, + const struct cred *cred, + struct watch_notification *n) +{ + struct smk_audit_info ad; + struct smack_known *subj, *obj; + int rc; + + /* Always let maintenance notifications through. */ + if (n->type == WATCH_TYPE_META) + return 0; + + if (!cred) + return 0; + subj = smk_of_task(smack_cred(cred)); + obj = smk_of_task(smack_cred(w_cred)); + + smk_ad_init(&ad, __func__, LSM_AUDIT_DATA_NOTIFICATION); + rc = smk_access(subj, obj, MAY_WRITE, &ad); + rc = smk_bu_note("notification", subj, obj, MAY_WRITE, rc); + return rc; +} +#endif /* CONFIG_WATCH_QUEUE */ + /* * Smack Audit hooks * @@ -4684,8 +4773,15 @@ static struct security_hook_list smack_hooks[] __lsm_ro_after_init = { LSM_HOOK_INIT(key_free, smack_key_free), LSM_HOOK_INIT(key_permission, smack_key_permission), LSM_HOOK_INIT(key_getsecurity, smack_key_getsecurity), +#ifdef CONFIG_KEY_NOTIFICATIONS + LSM_HOOK_INIT(watch_key, smack_watch_key), +#endif #endif /* CONFIG_KEYS */ +#ifdef CONFIG_WATCH_QUEUE + LSM_HOOK_INIT(post_notification, smack_post_notification), +#endif + /* Audit hooks */ #ifdef CONFIG_AUDIT LSM_HOOK_INIT(audit_rule_init, smack_audit_rule_init), diff --git a/sound/core/pcm_native.c b/sound/core/pcm_native.c index 47838f57a647..9630d2523948 100644 --- a/sound/core/pcm_native.c +++ b/sound/core/pcm_native.c @@ -138,6 +138,16 @@ void snd_pcm_stream_lock_irq(struct snd_pcm_substream *substream) } EXPORT_SYMBOL_GPL(snd_pcm_stream_lock_irq); +static void snd_pcm_stream_lock_nested(struct snd_pcm_substream *substream) +{ + struct snd_pcm_group *group = &substream->self_group; + + if (substream->pcm->nonatomic) + mutex_lock_nested(&group->mutex, SINGLE_DEPTH_NESTING); + else + spin_lock_nested(&group->lock, SINGLE_DEPTH_NESTING); +} + /** * snd_pcm_stream_unlock_irq - Unlock the PCM stream * @substream: PCM substream @@ -2166,6 +2176,12 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) } pcm_file = f.file->private_data; substream1 = pcm_file->substream; + + if (substream == substream1) { + res = -EINVAL; + goto _badf; + } + group = kzalloc(sizeof(*group), GFP_KERNEL); if (!group) { res = -ENOMEM; @@ -2194,7 +2210,7 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) snd_pcm_stream_unlock_irq(substream); snd_pcm_group_lock_irq(target_group, nonatomic); - snd_pcm_stream_lock(substream1); + snd_pcm_stream_lock_nested(substream1); snd_pcm_group_assign(substream1, target_group); refcount_inc(&target_group->refs); snd_pcm_stream_unlock(substream1); @@ -2210,7 +2226,7 @@ static int snd_pcm_link(struct snd_pcm_substream *substream, int fd) static void relink_to_local(struct snd_pcm_substream *substream) { - snd_pcm_stream_lock(substream); + snd_pcm_stream_lock_nested(substream); snd_pcm_group_assign(substream, &substream->self_group); snd_pcm_stream_unlock(substream); } diff --git a/sound/pci/emu10k1/emu10k1x.c b/sound/pci/emu10k1/emu10k1x.c index ddb7c2ce3f7c..def8161cde4c 100644 --- a/sound/pci/emu10k1/emu10k1x.c +++ b/sound/pci/emu10k1/emu10k1x.c @@ -1040,7 +1040,7 @@ static void snd_emu10k1x_proc_reg_write(struct snd_info_entry *entry, if (sscanf(line, "%x %x %x", ®, &channel_id, &val) != 3) continue; - if (reg < 0x49 && val <= 0xffffffff && channel_id <= 2) + if (reg < 0x49 && channel_id <= 2) snd_emu10k1x_ptr_write(emu, reg, channel_id, val); } } diff --git a/sound/pci/hda/patch_realtek.c b/sound/pci/hda/patch_realtek.c index 0aa778ff7f2b..6d73f8beadb6 100644 --- a/sound/pci/hda/patch_realtek.c +++ b/sound/pci/hda/patch_realtek.c @@ -8161,6 +8161,12 @@ static const struct snd_hda_pin_quirk alc269_pin_fixup_tbl[] = { ALC225_STANDARD_PINS, {0x12, 0xb7a60130}, {0x17, 0x90170110}), + SND_HDA_PIN_QUIRK(0x10ec0623, 0x17aa, "Lenovo", ALC283_FIXUP_HEADSET_MIC, + {0x14, 0x01014010}, + {0x17, 0x90170120}, + {0x18, 0x02a11030}, + {0x19, 0x02a1103f}, + {0x21, 0x0221101f}), {} }; diff --git a/sound/soc/codecs/max98390.c b/sound/soc/codecs/max98390.c index b9ce44dda886..0d63ebfbff2f 100644 --- a/sound/soc/codecs/max98390.c +++ b/sound/soc/codecs/max98390.c @@ -754,6 +754,7 @@ static struct snd_soc_dai_driver max98390_dai[] = { static int max98390_dsm_init(struct snd_soc_component *component) { int ret; + int param_size, param_start_addr; char filename[128]; const char *vendor, *product; struct max98390_priv *max98390 = @@ -778,16 +779,31 @@ static int max98390_dsm_init(struct snd_soc_component *component) } dev_dbg(component->dev, - "max98390: param fw size %ld\n", + "max98390: param fw size %zd\n", fw->size); + if (fw->size < MAX98390_DSM_PARAM_MIN_SIZE) { + dev_err(component->dev, + "param fw is invalid.\n"); + goto err_alloc; + } dsm_param = (char *)fw->data; + param_start_addr = (dsm_param[0] & 0xff) | (dsm_param[1] & 0xff) << 8; + param_size = (dsm_param[2] & 0xff) | (dsm_param[3] & 0xff) << 8; + if (param_size > MAX98390_DSM_PARAM_MAX_SIZE || + param_start_addr < DSM_STBASS_HPF_B0_BYTE0 || + fw->size < param_size + MAX98390_DSM_PAYLOAD_OFFSET) { + dev_err(component->dev, + "param fw is invalid.\n"); + goto err_alloc; + } + regmap_write(max98390->regmap, MAX98390_R203A_AMP_EN, 0x80); dsm_param += MAX98390_DSM_PAYLOAD_OFFSET; - regmap_bulk_write(max98390->regmap, DSM_EQ_BQ1_B0_BYTE0, - dsm_param, - fw->size - MAX98390_DSM_PAYLOAD_OFFSET); - release_firmware(fw); + regmap_bulk_write(max98390->regmap, param_start_addr, + dsm_param, param_size); regmap_write(max98390->regmap, MAX98390_R23E1_DSP_GLOBAL_EN, 0x01); +err_alloc: + release_firmware(fw); err: return ret; } diff --git a/sound/soc/codecs/max98390.h b/sound/soc/codecs/max98390.h index f59cb114d957..5f444e7779b0 100644 --- a/sound/soc/codecs/max98390.h +++ b/sound/soc/codecs/max98390.h @@ -650,7 +650,8 @@ /* DSM register offset */ #define MAX98390_DSM_PAYLOAD_OFFSET 16 -#define MAX98390_DSM_PAYLOAD_OFFSET_2 495 +#define MAX98390_DSM_PARAM_MAX_SIZE 770 +#define MAX98390_DSM_PARAM_MIN_SIZE 670 struct max98390_priv { struct regmap *regmap; diff --git a/sound/soc/codecs/rl6231.c b/sound/soc/codecs/rl6231.c index 2586d1cafc0c..8c9daf32bab8 100644 --- a/sound/soc/codecs/rl6231.c +++ b/sound/soc/codecs/rl6231.c @@ -80,8 +80,8 @@ int rl6231_calc_dmic_clk(int rate) for (i = 0; i < ARRAY_SIZE(div); i++) { if ((div[i] % 3) == 0) continue; - /* find divider that gives DMIC frequency below 3.072MHz */ - if (3072000 * div[i] >= rate) + /* find divider that gives DMIC frequency below 1.536MHz */ + if (1536000 * div[i] >= rate) return i; } diff --git a/sound/soc/codecs/rt5645.c b/sound/soc/codecs/rt5645.c index 6ba1849a77b0..e2e1d5b03b38 100644 --- a/sound/soc/codecs/rt5645.c +++ b/sound/soc/codecs/rt5645.c @@ -3625,6 +3625,12 @@ static const struct rt5645_platform_data asus_t100ha_platform_data = { .inv_jd1_1 = true, }; +static const struct rt5645_platform_data asus_t101ha_platform_data = { + .dmic1_data_pin = RT5645_DMIC_DATA_IN2N, + .dmic2_data_pin = RT5645_DMIC2_DISABLE, + .jd_mode = 3, +}; + static const struct rt5645_platform_data lenovo_ideapad_miix_310_pdata = { .jd_mode = 3, .in2_diff = true, @@ -3709,6 +3715,14 @@ static const struct dmi_system_id dmi_platform_data[] = { .driver_data = (void *)&asus_t100ha_platform_data, }, { + .ident = "ASUS T101HA", + .matches = { + DMI_MATCH(DMI_SYS_VENDOR, "ASUSTeK COMPUTER INC."), + DMI_MATCH(DMI_PRODUCT_NAME, "T101HA"), + }, + .driver_data = (void *)&asus_t101ha_platform_data, + }, + { .ident = "MINIX Z83-4", .matches = { DMI_EXACT_MATCH(DMI_SYS_VENDOR, "MINIX"), diff --git a/sound/soc/fsl/fsl-asoc-card.c b/sound/soc/fsl/fsl-asoc-card.c index cf4feb835743..00be73900888 100644 --- a/sound/soc/fsl/fsl-asoc-card.c +++ b/sound/soc/fsl/fsl-asoc-card.c @@ -581,7 +581,7 @@ static int fsl_asoc_card_probe(struct platform_device *pdev) if (!fsl_asoc_card_is_ac97(priv) && !codec_dev) { dev_err(&pdev->dev, "failed to find codec device\n"); - ret = -EINVAL; + ret = -EPROBE_DEFER; goto asrc_fail; } diff --git a/sound/soc/intel/boards/bytcr_rt5640.c b/sound/soc/intel/boards/bytcr_rt5640.c index 30f70bbdf89c..1fdb70b9e478 100644 --- a/sound/soc/intel/boards/bytcr_rt5640.c +++ b/sound/soc/intel/boards/bytcr_rt5640.c @@ -754,6 +754,18 @@ static const struct dmi_system_id byt_rt5640_quirk_table[] = { BYT_RT5640_JD_NOT_INV | BYT_RT5640_MCLK_EN), }, + { /* Toshiba Encore WT10-A */ + .matches = { + DMI_EXACT_MATCH(DMI_SYS_VENDOR, "TOSHIBA"), + DMI_EXACT_MATCH(DMI_PRODUCT_NAME, "TOSHIBA WT10-A-103"), + }, + .driver_data = (void *)(BYT_RT5640_DMIC1_MAP | + BYT_RT5640_JD_SRC_JD1_IN4P | + BYT_RT5640_OVCD_TH_2000UA | + BYT_RT5640_OVCD_SF_0P75 | + BYT_RT5640_SSP0_AIF2 | + BYT_RT5640_MCLK_EN), + }, { /* Catch-all for generic Insyde tablets, must be last */ .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Insyde"), diff --git a/sound/soc/intel/boards/glk_rt5682_max98357a.c b/sound/soc/intel/boards/glk_rt5682_max98357a.c index 48eda1a8aa6c..954ab01f695b 100644 --- a/sound/soc/intel/boards/glk_rt5682_max98357a.c +++ b/sound/soc/intel/boards/glk_rt5682_max98357a.c @@ -407,7 +407,7 @@ static struct snd_soc_dai_link geminilake_dais[] = { .name = "Glk Audio Echo Reference cap", .stream_name = "Echoreference Capture", .init = NULL, - .capture_only = 1, + .dpcm_capture = 1, .nonatomic = 1, .dynamic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), diff --git a/sound/soc/intel/boards/kbl_da7219_max98927.c b/sound/soc/intel/boards/kbl_da7219_max98927.c index cc9b5eab8b4a..e29c31ffd241 100644 --- a/sound/soc/intel/boards/kbl_da7219_max98927.c +++ b/sound/soc/intel/boards/kbl_da7219_max98927.c @@ -692,7 +692,7 @@ static struct snd_soc_dai_link kabylake_dais[] = { .name = "Kbl Audio Echo Reference cap", .stream_name = "Echoreference Capture", .init = NULL, - .capture_only = 1, + .dpcm_capture = 1, .nonatomic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), }, @@ -858,7 +858,7 @@ static struct snd_soc_dai_link kabylake_max98_927_373_dais[] = { .name = "Kbl Audio Echo Reference cap", .stream_name = "Echoreference Capture", .init = NULL, - .capture_only = 1, + .dpcm_capture = 1, .nonatomic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), }, diff --git a/sound/soc/intel/boards/kbl_rt5663_max98927.c b/sound/soc/intel/boards/kbl_rt5663_max98927.c index 658a9da3a40f..09ba55fc36d5 100644 --- a/sound/soc/intel/boards/kbl_rt5663_max98927.c +++ b/sound/soc/intel/boards/kbl_rt5663_max98927.c @@ -672,7 +672,7 @@ static struct snd_soc_dai_link kabylake_dais[] = { .name = "Kbl Audio Echo Reference cap", .stream_name = "Echoreference Capture", .init = NULL, - .capture_only = 1, + .dpcm_capture = 1, .nonatomic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), }, diff --git a/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c b/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c index 1b1f8d7a4ea3..b34cf6cf1139 100644 --- a/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c +++ b/sound/soc/intel/boards/kbl_rt5663_rt5514_max98927.c @@ -566,7 +566,7 @@ static struct snd_soc_dai_link kabylake_dais[] = { .name = "Kbl Audio Echo Reference cap", .stream_name = "Echoreference Capture", .init = NULL, - .capture_only = 1, + .dpcm_capture = 1, .nonatomic = 1, SND_SOC_DAILINK_REG(echoref, dummy, platform), }, diff --git a/sound/soc/meson/axg-fifo.c b/sound/soc/meson/axg-fifo.c index 2e9b56b29d31..b2e867113226 100644 --- a/sound/soc/meson/axg-fifo.c +++ b/sound/soc/meson/axg-fifo.c @@ -249,7 +249,7 @@ int axg_fifo_pcm_open(struct snd_soc_component *component, /* Enable pclk to access registers and clock the fifo ip */ ret = clk_prepare_enable(fifo->pclk); if (ret) - return ret; + goto free_irq; /* Setup status2 so it reports the memory pointer */ regmap_update_bits(fifo->map, FIFO_CTRL1, @@ -269,8 +269,14 @@ int axg_fifo_pcm_open(struct snd_soc_component *component, /* Take memory arbitror out of reset */ ret = reset_control_deassert(fifo->arb); if (ret) - clk_disable_unprepare(fifo->pclk); + goto free_clk; + + return 0; +free_clk: + clk_disable_unprepare(fifo->pclk); +free_irq: + free_irq(fifo->irq, ss); return ret; } EXPORT_SYMBOL_GPL(axg_fifo_pcm_open); diff --git a/sound/soc/meson/meson-card-utils.c b/sound/soc/meson/meson-card-utils.c index 2ca8c98e204f..5a4a91c88734 100644 --- a/sound/soc/meson/meson-card-utils.c +++ b/sound/soc/meson/meson-card-utils.c @@ -49,19 +49,26 @@ int meson_card_reallocate_links(struct snd_soc_card *card, links = krealloc(priv->card.dai_link, num_links * sizeof(*priv->card.dai_link), GFP_KERNEL | __GFP_ZERO); + if (!links) + goto err_links; + ldata = krealloc(priv->link_data, num_links * sizeof(*priv->link_data), GFP_KERNEL | __GFP_ZERO); - - if (!links || !ldata) { - dev_err(priv->card.dev, "failed to allocate links\n"); - return -ENOMEM; - } + if (!ldata) + goto err_ldata; priv->card.dai_link = links; priv->link_data = ldata; priv->card.num_links = num_links; return 0; + +err_ldata: + kfree(links); +err_links: + dev_err(priv->card.dev, "failed to allocate links\n"); + return -ENOMEM; + } EXPORT_SYMBOL_GPL(meson_card_reallocate_links); diff --git a/sound/soc/soc-core.c b/sound/soc/soc-core.c index b07eca2c6ccc..7b387202c5db 100644 --- a/sound/soc/soc-core.c +++ b/sound/soc/soc-core.c @@ -1648,9 +1648,25 @@ match: dai_link->platforms->name = component->name; /* convert non BE into BE */ - dai_link->no_pcm = 1; - dai_link->dpcm_playback = 1; - dai_link->dpcm_capture = 1; + if (!dai_link->no_pcm) { + dai_link->no_pcm = 1; + + if (dai_link->dpcm_playback) + dev_warn(card->dev, + "invalid configuration, dailink %s has flags no_pcm=0 and dpcm_playback=1\n", + dai_link->name); + if (dai_link->dpcm_capture) + dev_warn(card->dev, + "invalid configuration, dailink %s has flags no_pcm=0 and dpcm_capture=1\n", + dai_link->name); + + /* convert normal link into DPCM one */ + if (!(dai_link->dpcm_playback || + dai_link->dpcm_capture)) { + dai_link->dpcm_playback = !dai_link->capture_only; + dai_link->dpcm_capture = !dai_link->playback_only; + } + } /* * override any BE fixups diff --git a/sound/soc/soc-pcm.c b/sound/soc/soc-pcm.c index 276505fb9d50..2c114b4542ce 100644 --- a/sound/soc/soc-pcm.c +++ b/sound/soc/soc-pcm.c @@ -2789,20 +2789,44 @@ int soc_new_pcm(struct snd_soc_pcm_runtime *rtd, int num) struct snd_pcm *pcm; char new_name[64]; int ret = 0, playback = 0, capture = 0; + int stream; int i; + if (rtd->dai_link->dynamic && rtd->num_cpus > 1) { + dev_err(rtd->dev, + "DPCM doesn't support Multi CPU for Front-Ends yet\n"); + return -EINVAL; + } + if (rtd->dai_link->dynamic || rtd->dai_link->no_pcm) { - cpu_dai = asoc_rtd_to_cpu(rtd, 0); - if (rtd->num_cpus > 1) { - dev_err(rtd->dev, - "DPCM doesn't support Multi CPU yet\n"); - return -EINVAL; + if (rtd->dai_link->dpcm_playback) { + stream = SNDRV_PCM_STREAM_PLAYBACK; + + for_each_rtd_cpu_dais(rtd, i, cpu_dai) + if (!snd_soc_dai_stream_valid(cpu_dai, + stream)) { + dev_err(rtd->card->dev, + "CPU DAI %s for rtd %s does not support playback\n", + cpu_dai->name, + rtd->dai_link->stream_name); + return -EINVAL; + } + playback = 1; + } + if (rtd->dai_link->dpcm_capture) { + stream = SNDRV_PCM_STREAM_CAPTURE; + + for_each_rtd_cpu_dais(rtd, i, cpu_dai) + if (!snd_soc_dai_stream_valid(cpu_dai, + stream)) { + dev_err(rtd->card->dev, + "CPU DAI %s for rtd %s does not support capture\n", + cpu_dai->name, + rtd->dai_link->stream_name); + return -EINVAL; + } + capture = 1; } - - playback = rtd->dai_link->dpcm_playback && - snd_soc_dai_stream_valid(cpu_dai, SNDRV_PCM_STREAM_PLAYBACK); - capture = rtd->dai_link->dpcm_capture && - snd_soc_dai_stream_valid(cpu_dai, SNDRV_PCM_STREAM_CAPTURE); } else { /* Adapt stream for codec2codec links */ int cpu_capture = rtd->dai_link->params ? diff --git a/sound/soc/sof/nocodec.c b/sound/soc/sof/nocodec.c index ce053ba8f2e8..d03b5be31255 100644 --- a/sound/soc/sof/nocodec.c +++ b/sound/soc/sof/nocodec.c @@ -52,8 +52,10 @@ static int sof_nocodec_bes_setup(struct device *dev, links[i].platforms->name = dev_name(dev); links[i].codecs->dai_name = "snd-soc-dummy-dai"; links[i].codecs->name = "snd-soc-dummy"; - links[i].dpcm_playback = 1; - links[i].dpcm_capture = 1; + if (ops->drv[i].playback.channels_min) + links[i].dpcm_playback = 1; + if (ops->drv[i].capture.channels_min) + links[i].dpcm_capture = 1; } card->dai_link = links; diff --git a/sound/usb/card.c b/sound/usb/card.c index fd6fd1726ea0..162bdd6eb4d4 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -634,7 +634,6 @@ static int usb_audio_probe(struct usb_interface *intf, id, &chip); if (err < 0) goto __error; - chip->pm_intf = intf; break; } else if (vid[i] != -1 || pid[i] != -1) { dev_info(&dev->dev, @@ -651,6 +650,13 @@ static int usb_audio_probe(struct usb_interface *intf, goto __error; } } + + if (chip->num_interfaces >= MAX_CARD_INTERFACES) { + dev_info(&dev->dev, "Too many interfaces assigned to the single USB-audio card\n"); + err = -EINVAL; + goto __error; + } + dev_set_drvdata(&dev->dev, chip); /* @@ -703,6 +709,7 @@ static int usb_audio_probe(struct usb_interface *intf, } usb_chip[chip->index] = chip; + chip->intf[chip->num_interfaces] = intf; chip->num_interfaces++; usb_set_intfdata(intf, chip); atomic_dec(&chip->active); @@ -818,19 +825,37 @@ void snd_usb_unlock_shutdown(struct snd_usb_audio *chip) int snd_usb_autoresume(struct snd_usb_audio *chip) { + int i, err; + if (atomic_read(&chip->shutdown)) return -EIO; - if (atomic_inc_return(&chip->active) == 1) - return usb_autopm_get_interface(chip->pm_intf); + if (atomic_inc_return(&chip->active) != 1) + return 0; + + for (i = 0; i < chip->num_interfaces; i++) { + err = usb_autopm_get_interface(chip->intf[i]); + if (err < 0) { + /* rollback */ + while (--i >= 0) + usb_autopm_put_interface(chip->intf[i]); + atomic_dec(&chip->active); + return err; + } + } return 0; } void snd_usb_autosuspend(struct snd_usb_audio *chip) { + int i; + if (atomic_read(&chip->shutdown)) return; - if (atomic_dec_and_test(&chip->active)) - usb_autopm_put_interface(chip->pm_intf); + if (!atomic_dec_and_test(&chip->active)) + return; + + for (i = 0; i < chip->num_interfaces; i++) + usb_autopm_put_interface(chip->intf[i]); } static int usb_audio_suspend(struct usb_interface *intf, pm_message_t message) @@ -843,9 +868,6 @@ static int usb_audio_suspend(struct usb_interface *intf, pm_message_t message) if (chip == (void *)-1L) return 0; - chip->autosuspended = !!PMSG_IS_AUTO(message); - if (!chip->autosuspended) - snd_power_change_state(chip->card, SNDRV_CTL_POWER_D3hot); if (!chip->num_suspended_intf++) { list_for_each_entry(as, &chip->pcm_list, list) { snd_usb_pcm_suspend(as); @@ -858,6 +880,11 @@ static int usb_audio_suspend(struct usb_interface *intf, pm_message_t message) snd_usb_mixer_suspend(mixer); } + if (!PMSG_IS_AUTO(message) && !chip->system_suspend) { + snd_power_change_state(chip->card, SNDRV_CTL_POWER_D3hot); + chip->system_suspend = chip->num_suspended_intf; + } + return 0; } @@ -871,10 +898,10 @@ static int __usb_audio_resume(struct usb_interface *intf, bool reset_resume) if (chip == (void *)-1L) return 0; - if (--chip->num_suspended_intf) - return 0; atomic_inc(&chip->active); /* avoid autopm */ + if (chip->num_suspended_intf > 1) + goto out; list_for_each_entry(as, &chip->pcm_list, list) { err = snd_usb_pcm_resume(as); @@ -896,9 +923,12 @@ static int __usb_audio_resume(struct usb_interface *intf, bool reset_resume) snd_usbmidi_resume(p); } - if (!chip->autosuspended) + out: + if (chip->num_suspended_intf == chip->system_suspend) { snd_power_change_state(chip->card, SNDRV_CTL_POWER_D0); - chip->autosuspended = 0; + chip->system_suspend = 0; + } + chip->num_suspended_intf--; err_out: atomic_dec(&chip->active); /* allow autopm after this point */ diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 6d6492195bdc..4ec491011b19 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -40,6 +40,18 @@ .ifnum = QUIRK_NO_INTERFACE \ } +/* HP Thunderbolt Dock Audio Headset */ +{ + USB_DEVICE(0x03f0, 0x0269), + QUIRK_DEVICE_PROFILE("HP", "Thunderbolt Dock Audio Headset", + "HP-Thunderbolt-Dock-Audio-Headset"), +}, +/* HP Thunderbolt Dock Audio Module */ +{ + USB_DEVICE(0x03f0, 0x0567), + QUIRK_DEVICE_PROFILE("HP", "Thunderbolt Dock Audio Module", + "HP-Thunderbolt-Dock-Audio-Module"), +}, /* FTDI devices */ { USB_DEVICE(0x0403, 0xb8d8), diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index 1c892c7f14d7..b91c4c0807ec 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -19,14 +19,16 @@ struct media_device; struct media_intf_devnode; +#define MAX_CARD_INTERFACES 16 + struct snd_usb_audio { int index; struct usb_device *dev; struct snd_card *card; - struct usb_interface *pm_intf; + struct usb_interface *intf[MAX_CARD_INTERFACES]; u32 usb_id; struct mutex mutex; - unsigned int autosuspended:1; + unsigned int system_suspend; atomic_t active; atomic_t shutdown; atomic_t usage_count; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 63d65a702900..5fbb90a80d23 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -505,6 +505,28 @@ static const char *uaccess_safe_builtin[] = { "__asan_report_store4_noabort", "__asan_report_store8_noabort", "__asan_report_store16_noabort", + /* KCSAN */ + "__kcsan_check_access", + "kcsan_found_watchpoint", + "kcsan_setup_watchpoint", + "kcsan_check_scoped_accesses", + "kcsan_disable_current", + "kcsan_enable_current_nowarn", + /* KCSAN/TSAN */ + "__tsan_func_entry", + "__tsan_func_exit", + "__tsan_read_range", + "__tsan_write_range", + "__tsan_read1", + "__tsan_read2", + "__tsan_read4", + "__tsan_read8", + "__tsan_read16", + "__tsan_write1", + "__tsan_write2", + "__tsan_write4", + "__tsan_write8", + "__tsan_write16", /* KCOV */ "write_comp_data", "check_kcov_mode", diff --git a/tools/testing/selftests/kvm/.gitignore b/tools/testing/selftests/kvm/.gitignore index f159718f90c0..452787152748 100644 --- a/tools/testing/selftests/kvm/.gitignore +++ b/tools/testing/selftests/kvm/.gitignore @@ -3,6 +3,7 @@ /s390x/resets /s390x/sync_regs_test /x86_64/cr4_cpuid_sync_test +/x86_64/debug_regs /x86_64/evmcs_test /x86_64/hyperv_cpuid /x86_64/mmio_warning_test diff --git a/tools/testing/selftests/kvm/Makefile b/tools/testing/selftests/kvm/Makefile index b4ff112e5c7e..4a166588d99f 100644 --- a/tools/testing/selftests/kvm/Makefile +++ b/tools/testing/selftests/kvm/Makefile @@ -83,7 +83,11 @@ LIBKVM += $(LIBKVM_$(UNAME_M)) INSTALL_HDR_PATH = $(top_srcdir)/usr LINUX_HDR_PATH = $(INSTALL_HDR_PATH)/include/ LINUX_TOOL_INCLUDE = $(top_srcdir)/tools/include +ifeq ($(ARCH),x86_64) +LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/x86/include +else LINUX_TOOL_ARCH_INCLUDE = $(top_srcdir)/tools/arch/$(ARCH)/include +endif CFLAGS += -Wall -Wstrict-prototypes -Wuninitialized -O2 -g -std=gnu99 \ -fno-stack-protector -fno-PIE -I$(LINUX_TOOL_INCLUDE) \ -I$(LINUX_TOOL_ARCH_INCLUDE) -I$(LINUX_HDR_PATH) -Iinclude \ diff --git a/tools/testing/selftests/kvm/include/x86_64/svm_util.h b/tools/testing/selftests/kvm/include/x86_64/svm_util.h index 674151d24fcf..b7531c83b8ae 100644 --- a/tools/testing/selftests/kvm/include/x86_64/svm_util.h +++ b/tools/testing/selftests/kvm/include/x86_64/svm_util.h @@ -33,6 +33,7 @@ struct svm_test_data { struct svm_test_data *vcpu_alloc_svm(struct kvm_vm *vm, vm_vaddr_t *p_svm_gva); void generic_svm_setup(struct svm_test_data *svm, void *guest_rip, void *guest_rsp); void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa); +bool nested_svm_supported(void); void nested_svm_check_supported(void); static inline bool cpu_has_svm(void) diff --git a/tools/testing/selftests/kvm/include/x86_64/vmx.h b/tools/testing/selftests/kvm/include/x86_64/vmx.h index ccff3e6e2704..16fa21ebb99c 100644 --- a/tools/testing/selftests/kvm/include/x86_64/vmx.h +++ b/tools/testing/selftests/kvm/include/x86_64/vmx.h @@ -598,15 +598,12 @@ union vmx_ctrl_msr { }; }; -union vmx_basic basic; -union vmx_ctrl_msr ctrl_pin_rev; -union vmx_ctrl_msr ctrl_exit_rev; - struct vmx_pages *vcpu_alloc_vmx(struct kvm_vm *vm, vm_vaddr_t *p_vmx_gva); bool prepare_for_vmx_operation(struct vmx_pages *vmx); void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp); bool load_vmcs(struct vmx_pages *vmx); +bool nested_vmx_supported(void); void nested_vmx_check_supported(void); void nested_pg_map(struct vmx_pages *vmx, struct kvm_vm *vm, diff --git a/tools/testing/selftests/kvm/lib/kvm_util.c b/tools/testing/selftests/kvm/lib/kvm_util.c index c9cede5c7d0d..74776ee228f2 100644 --- a/tools/testing/selftests/kvm/lib/kvm_util.c +++ b/tools/testing/selftests/kvm/lib/kvm_util.c @@ -195,11 +195,18 @@ struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm) case VM_MODE_PXXV48_4K: #ifdef __x86_64__ kvm_get_cpu_address_width(&vm->pa_bits, &vm->va_bits); - TEST_ASSERT(vm->va_bits == 48, "Linear address width " - "(%d bits) not supported", vm->va_bits); + /* + * Ignore KVM support for 5-level paging (vm->va_bits == 57), + * it doesn't take effect unless a CR4.LA57 is set, which it + * isn't for this VM_MODE. + */ + TEST_ASSERT(vm->va_bits == 48 || vm->va_bits == 57, + "Linear address width (%d bits) not supported", + vm->va_bits); pr_debug("Guest physical address width detected: %d\n", vm->pa_bits); vm->pgtable_levels = 4; + vm->va_bits = 48; #else TEST_FAIL("VM_MODE_PXXV48_4K not supported on non-x86 platforms"); #endif diff --git a/tools/testing/selftests/kvm/lib/x86_64/svm.c b/tools/testing/selftests/kvm/lib/x86_64/svm.c index c42401068373..3a5c72ed2b79 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/svm.c +++ b/tools/testing/selftests/kvm/lib/x86_64/svm.c @@ -148,14 +148,18 @@ void run_guest(struct vmcb *vmcb, uint64_t vmcb_gpa) : "r15", "memory"); } -void nested_svm_check_supported(void) +bool nested_svm_supported(void) { struct kvm_cpuid_entry2 *entry = kvm_get_supported_cpuid_entry(0x80000001); - if (!(entry->ecx & CPUID_SVM)) { + return entry->ecx & CPUID_SVM; +} + +void nested_svm_check_supported(void) +{ + if (!nested_svm_supported()) { print_skip("nested SVM not enabled"); exit(KSFT_SKIP); } } - diff --git a/tools/testing/selftests/kvm/lib/x86_64/vmx.c b/tools/testing/selftests/kvm/lib/x86_64/vmx.c index 4ae104f6ce69..f1e00d43eea2 100644 --- a/tools/testing/selftests/kvm/lib/x86_64/vmx.c +++ b/tools/testing/selftests/kvm/lib/x86_64/vmx.c @@ -379,11 +379,16 @@ void prepare_vmcs(struct vmx_pages *vmx, void *guest_rip, void *guest_rsp) init_vmcs_guest_state(guest_rip, guest_rsp); } -void nested_vmx_check_supported(void) +bool nested_vmx_supported(void) { struct kvm_cpuid_entry2 *entry = kvm_get_supported_cpuid_entry(1); - if (!(entry->ecx & CPUID_VMX)) { + return entry->ecx & CPUID_VMX; +} + +void nested_vmx_check_supported(void) +{ + if (!nested_vmx_supported()) { print_skip("nested VMX not enabled"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/evmcs_test.c b/tools/testing/selftests/kvm/x86_64/evmcs_test.c index e6e62e5e75b2..757928199f19 100644 --- a/tools/testing/selftests/kvm/x86_64/evmcs_test.c +++ b/tools/testing/selftests/kvm/x86_64/evmcs_test.c @@ -94,9 +94,10 @@ int main(int argc, char *argv[]) vcpu_set_cpuid(vm, VCPU_ID, kvm_get_supported_cpuid()); - if (!kvm_check_cap(KVM_CAP_NESTED_STATE) || + if (!nested_vmx_supported() || + !kvm_check_cap(KVM_CAP_NESTED_STATE) || !kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { - print_skip("capabilities not available"); + print_skip("Enlightened VMCS is unsupported"); exit(KSFT_SKIP); } diff --git a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c index 4a7967cca281..745b708c2d3b 100644 --- a/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c +++ b/tools/testing/selftests/kvm/x86_64/hyperv_cpuid.c @@ -170,7 +170,8 @@ int main(int argc, char *argv[]) case 1: break; case 2: - if (!kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { + if (!nested_vmx_supported() || + !kvm_check_cap(KVM_CAP_HYPERV_ENLIGHTENED_VMCS)) { print_skip("Enlightened VMCS is unsupported"); continue; } diff --git a/tools/testing/selftests/kvm/x86_64/smm_test.c b/tools/testing/selftests/kvm/x86_64/smm_test.c index 6f8f478b3ceb..ae39a220609f 100644 --- a/tools/testing/selftests/kvm/x86_64/smm_test.c +++ b/tools/testing/selftests/kvm/x86_64/smm_test.c @@ -47,10 +47,10 @@ uint8_t smi_handler[] = { 0x0f, 0xaa, /* rsm */ }; -void sync_with_host(uint64_t phase) +static inline void sync_with_host(uint64_t phase) { asm volatile("in $" XSTR(SYNC_PORT)", %%al \n" - : : "a" (phase)); + : "+a" (phase)); } void self_smi(void) @@ -118,16 +118,17 @@ int main(int argc, char *argv[]) vcpu_set_msr(vm, VCPU_ID, MSR_IA32_SMBASE, SMRAM_GPA); if (kvm_check_cap(KVM_CAP_NESTED_STATE)) { - if (kvm_get_supported_cpuid_entry(0x80000001)->ecx & CPUID_SVM) + if (nested_svm_supported()) vcpu_alloc_svm(vm, &nested_gva); - else + else if (nested_vmx_supported()) vcpu_alloc_vmx(vm, &nested_gva); - vcpu_args_set(vm, VCPU_ID, 1, nested_gva); - } else { - pr_info("will skip SMM test with VMX enabled\n"); - vcpu_args_set(vm, VCPU_ID, 1, 0); } + if (!nested_gva) + pr_info("will skip SMM test with VMX enabled\n"); + + vcpu_args_set(vm, VCPU_ID, 1, nested_gva); + for (stage = 1;; stage++) { _vcpu_run(vm, VCPU_ID); TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, diff --git a/tools/testing/selftests/kvm/x86_64/state_test.c b/tools/testing/selftests/kvm/x86_64/state_test.c index d43b6f99b66c..f6c8b9042f8a 100644 --- a/tools/testing/selftests/kvm/x86_64/state_test.c +++ b/tools/testing/selftests/kvm/x86_64/state_test.c @@ -171,16 +171,17 @@ int main(int argc, char *argv[]) vcpu_regs_get(vm, VCPU_ID, ®s1); if (kvm_check_cap(KVM_CAP_NESTED_STATE)) { - if (kvm_get_supported_cpuid_entry(0x80000001)->ecx & CPUID_SVM) + if (nested_svm_supported()) vcpu_alloc_svm(vm, &nested_gva); - else + else if (nested_vmx_supported()) vcpu_alloc_vmx(vm, &nested_gva); - vcpu_args_set(vm, VCPU_ID, 1, nested_gva); - } else { - pr_info("will skip nested state checks\n"); - vcpu_args_set(vm, VCPU_ID, 1, 0); } + if (!nested_gva) + pr_info("will skip nested state checks\n"); + + vcpu_args_set(vm, VCPU_ID, 1, nested_gva); + for (stage = 1;; stage++) { _vcpu_run(vm, VCPU_ID); TEST_ASSERT(run->exit_reason == KVM_EXIT_IO, diff --git a/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c b/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c index cc72b6188ca7..a7737af1224f 100644 --- a/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c +++ b/tools/testing/selftests/kvm/x86_64/vmx_preemption_timer_test.c @@ -31,6 +31,10 @@ bool l2_save_restore_done; static u64 l2_vmx_pt_start; volatile u64 l2_vmx_pt_finish; +union vmx_basic basic; +union vmx_ctrl_msr ctrl_pin_rev; +union vmx_ctrl_msr ctrl_exit_rev; + void l2_guest_code(void) { u64 vmx_pt_delta; diff --git a/tools/testing/selftests/vm/khugepaged.c b/tools/testing/selftests/vm/khugepaged.c index 51b89cedd09d..8b75821302a7 100644 --- a/tools/testing/selftests/vm/khugepaged.c +++ b/tools/testing/selftests/vm/khugepaged.c @@ -502,7 +502,7 @@ static bool wait_for_scan(const char *msg, char *p) madvise(p, hpage_pmd_size, MADV_NOHUGEPAGE); - return !timeout; + return timeout == -1; } static void alloc_at_fault(void) diff --git a/virt/kvm/async_pf.c b/virt/kvm/async_pf.c index 5f8f3e8b5add..45799606bb3e 100644 --- a/virt/kvm/async_pf.c +++ b/virt/kvm/async_pf.c @@ -164,7 +164,9 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, if (vcpu->async_pf.queued >= ASYNC_PF_PER_VCPU) return 0; - /* setup delayed work */ + /* Arch specific code should not do async PF in this case */ + if (unlikely(kvm_is_error_hva(hva))) + return 0; /* * do alloc nowait since if we are going to sleep anyway we @@ -183,24 +185,15 @@ int kvm_setup_async_pf(struct kvm_vcpu *vcpu, gpa_t cr2_or_gpa, mmget(work->mm); kvm_get_kvm(work->vcpu->kvm); - /* this can't really happen otherwise gfn_to_pfn_async - would succeed */ - if (unlikely(kvm_is_error_hva(work->addr))) - goto retry_sync; - INIT_WORK(&work->work, async_pf_execute); - if (!schedule_work(&work->work)) - goto retry_sync; list_add_tail(&work->queue, &vcpu->async_pf.queue); vcpu->async_pf.queued++; - kvm_arch_async_page_not_present(vcpu, work); + work->notpresent_injected = kvm_arch_async_page_not_present(vcpu, work); + + schedule_work(&work->work); + return 1; -retry_sync: - kvm_put_kvm(work->vcpu->kvm); - mmput(work->mm); - kmem_cache_free(async_pf_cache, work); - return 0; } int kvm_async_pf_wakeup_all(struct kvm_vcpu *vcpu) diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c index 0dfee7576e88..a852af5c3214 100644 --- a/virt/kvm/kvm_main.c +++ b/virt/kvm/kvm_main.c @@ -154,10 +154,9 @@ static void kvm_uevent_notify_change(unsigned int type, struct kvm *kvm); static unsigned long long kvm_createvm_count; static unsigned long long kvm_active_vms; -__weak int kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, - unsigned long start, unsigned long end, bool blockable) +__weak void kvm_arch_mmu_notifier_invalidate_range(struct kvm *kvm, + unsigned long start, unsigned long end) { - return 0; } bool kvm_is_zone_device_pfn(kvm_pfn_t pfn) @@ -383,6 +382,18 @@ static inline struct kvm *mmu_notifier_to_kvm(struct mmu_notifier *mn) return container_of(mn, struct kvm, mmu_notifier); } +static void kvm_mmu_notifier_invalidate_range(struct mmu_notifier *mn, + struct mm_struct *mm, + unsigned long start, unsigned long end) +{ + struct kvm *kvm = mmu_notifier_to_kvm(mn); + int idx; + + idx = srcu_read_lock(&kvm->srcu); + kvm_arch_mmu_notifier_invalidate_range(kvm, start, end); + srcu_read_unlock(&kvm->srcu, idx); +} + static void kvm_mmu_notifier_change_pte(struct mmu_notifier *mn, struct mm_struct *mm, unsigned long address, @@ -407,7 +418,6 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, { struct kvm *kvm = mmu_notifier_to_kvm(mn); int need_tlb_flush = 0, idx; - int ret; idx = srcu_read_lock(&kvm->srcu); spin_lock(&kvm->mmu_lock); @@ -424,14 +434,9 @@ static int kvm_mmu_notifier_invalidate_range_start(struct mmu_notifier *mn, kvm_flush_remote_tlbs(kvm); spin_unlock(&kvm->mmu_lock); - - ret = kvm_arch_mmu_notifier_invalidate_range(kvm, range->start, - range->end, - mmu_notifier_range_blockable(range)); - srcu_read_unlock(&kvm->srcu, idx); - return ret; + return 0; } static void kvm_mmu_notifier_invalidate_range_end(struct mmu_notifier *mn, @@ -537,6 +542,7 @@ static void kvm_mmu_notifier_release(struct mmu_notifier *mn, } static const struct mmu_notifier_ops kvm_mmu_notifier_ops = { + .invalidate_range = kvm_mmu_notifier_invalidate_range, .invalidate_range_start = kvm_mmu_notifier_invalidate_range_start, .invalidate_range_end = kvm_mmu_notifier_invalidate_range_end, .clear_flush_young = kvm_mmu_notifier_clear_flush_young, @@ -2970,7 +2976,6 @@ static int kvm_vcpu_release(struct inode *inode, struct file *filp) { struct kvm_vcpu *vcpu = filp->private_data; - debugfs_remove_recursive(vcpu->debugfs_dentry); kvm_put_kvm(vcpu->kvm); return 0; } @@ -2997,16 +3002,17 @@ static int create_vcpu_fd(struct kvm_vcpu *vcpu) static void kvm_create_vcpu_debugfs(struct kvm_vcpu *vcpu) { #ifdef __KVM_HAVE_ARCH_VCPU_DEBUGFS + struct dentry *debugfs_dentry; char dir_name[ITOA_MAX_LEN * 2]; if (!debugfs_initialized()) return; snprintf(dir_name, sizeof(dir_name), "vcpu%d", vcpu->vcpu_id); - vcpu->debugfs_dentry = debugfs_create_dir(dir_name, - vcpu->kvm->debugfs_dentry); + debugfs_dentry = debugfs_create_dir(dir_name, + vcpu->kvm->debugfs_dentry); - kvm_arch_create_vcpu_debugfs(vcpu); + kvm_arch_create_vcpu_debugfs(vcpu, debugfs_dentry); #endif } @@ -3743,21 +3749,18 @@ static long kvm_vm_ioctl(struct file *filp, if (routing.flags) goto out; if (routing.nr) { - r = -ENOMEM; - entries = vmalloc(array_size(sizeof(*entries), - routing.nr)); - if (!entries) - goto out; - r = -EFAULT; urouting = argp; - if (copy_from_user(entries, urouting->entries, - routing.nr * sizeof(*entries))) - goto out_free_irq_routing; + entries = vmemdup_user(urouting->entries, + array_size(sizeof(*entries), + routing.nr)); + if (IS_ERR(entries)) { + r = PTR_ERR(entries); + goto out; + } } r = kvm_set_irq_routing(kvm, entries, routing.nr, routing.flags); -out_free_irq_routing: - vfree(entries); + kvfree(entries); break; } #endif /* CONFIG_HAVE_KVM_IRQ_ROUTING */ |